fluent-plugin-openstack 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_swift.rb +47 -32
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4b27ffca51d8e3d40e7793106d6c94a6a2cc13db511aec390cdf6755536b469f
|
4
|
+
data.tar.gz: d79d1b950a0846e60c05a9b56ddff6b3d4f2aea2a195f05ab821cafc601daacd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 25daa97506ab9150f10b26a54bceb495bc88552a12c505076f365465a4c3ec6e4108a8651bec2bfe1ba213b503af5820cdafe16b070a2edb1b371df618fb8d8c
|
7
|
+
data.tar.gz: cd25abdefd40c317246e653ef3d41909c8aea18eefd8c5828d815af97aba61e4302d49bedb4ff75eec0511523a27b63b066788fc1c68e975f7f8db1536563721
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.
|
1
|
+
1.0.1
|
@@ -16,7 +16,7 @@ module Fluent::Plugin
|
|
16
16
|
MAX_HEX_RANDOM_LENGTH = 16
|
17
17
|
|
18
18
|
desc 'Path prefix of the files on Swift'
|
19
|
-
config_param :path, :string, default: ''
|
19
|
+
config_param :path, :string, default: '%Y%m%d'
|
20
20
|
# OpenStack AUTH
|
21
21
|
desc "Authentication URL. Set a value or `#{ENV['OS_AUTH_URL']}`"
|
22
22
|
config_param :auth_url, :string
|
@@ -36,13 +36,13 @@ module Fluent::Plugin
|
|
36
36
|
desc 'If false, the certificate of endpoint will not be verified'
|
37
37
|
config_param :ssl_verify, :bool, default: true
|
38
38
|
desc 'The format of Swift object keys'
|
39
|
-
config_param :swift_object_key_format, :string, default: '%{path}
|
39
|
+
config_param :swift_object_key_format, :string, default: '%{path}/%{time_slice}_%{index}.%{file_extension}'
|
40
40
|
desc 'Create Swift container if it does not exists'
|
41
41
|
config_param :auto_create_container, :bool, default: true
|
42
42
|
config_param :check_apikey_on_start, :bool, default: true
|
43
43
|
desc 'URI of proxy environment'
|
44
44
|
config_param :proxy_uri, :string, default: nil
|
45
|
-
desc 'The length of `%{hex_random}` placeholder(4-16)'
|
45
|
+
desc 'The length of `%{hex_random}` placeholder (4 - 16)'
|
46
46
|
config_param :hex_random_length, :integer, default: 4
|
47
47
|
desc '`sprintf` format for `%{index}`'
|
48
48
|
config_param :index_format, :string, default: '%d'
|
@@ -70,6 +70,8 @@ module Fluent::Plugin
|
|
70
70
|
|
71
71
|
super
|
72
72
|
|
73
|
+
$log.info("config: #{config}")
|
74
|
+
|
73
75
|
if auth_url.empty?
|
74
76
|
raise Fluent::ConfigError, 'auth_url parameter or OS_AUTH_URL variable not defined'
|
75
77
|
end
|
@@ -80,6 +82,14 @@ module Fluent::Plugin
|
|
80
82
|
raise Fluent::ConfigError, 'auth_api_key parameter or OS_PASSWORD variable not defined'
|
81
83
|
end
|
82
84
|
|
85
|
+
if hex_random_length > MAX_HEX_RANDOM_LENGTH
|
86
|
+
raise Fluent::ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
|
87
|
+
end
|
88
|
+
|
89
|
+
unless index_format =~ /^%(0\d*)?[dxX]$/
|
90
|
+
raise Fluent::ConfigError, 'index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types'
|
91
|
+
end
|
92
|
+
|
83
93
|
self.ext, self.mime_type = case store_as
|
84
94
|
when 'gzip' then ['gz', 'application/x-gzip']
|
85
95
|
when 'lzo' then
|
@@ -91,23 +101,14 @@ module Fluent::Plugin
|
|
91
101
|
['lzo', 'application/x-lzop']
|
92
102
|
when 'json' then ['json', 'application/json']
|
93
103
|
else ['txt', 'text/plain']
|
94
|
-
|
104
|
+
end
|
95
105
|
|
96
106
|
self.formatter = formatter_create
|
97
|
-
|
98
|
-
if hex_random_length > MAX_HEX_RANDOM_LENGTH
|
99
|
-
raise Fluent::ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
|
100
|
-
end
|
101
|
-
|
102
|
-
unless index_format =~ /^%(0\d*)?[dxX]$/
|
103
|
-
raise Fluent::ConfigError, 'index_format parameter should follow `%[flags][width]type`. `0` is the only supported flag, and is mandatory if width is specified. `d`, `x` and `X` are supported types'
|
104
|
-
end
|
105
|
-
|
106
107
|
self.swift_object_key_format = configure_swift_object_key_format
|
107
108
|
# For backward compatibility
|
108
109
|
# TODO: Remove time_slice_format when end of support compat_parameters
|
109
110
|
self.values_for_swift_object_chunk = {}
|
110
|
-
self.time_slice_with_tz = Fluent::Timezone.formatter(timekey_zone, config['time_slice_format']
|
111
|
+
self.time_slice_with_tz = Fluent::Timezone.formatter(timekey_zone, config['time_slice_format'])
|
111
112
|
end
|
112
113
|
|
113
114
|
def multi_workers_ready?
|
@@ -145,10 +146,15 @@ module Fluent::Plugin
|
|
145
146
|
time_slice = if metadata.timekey.nil?
|
146
147
|
''
|
147
148
|
else
|
149
|
+
$log.info("timekey: #{metadata.timekey}")
|
150
|
+
$log.info("metadata: #{metadata}")
|
148
151
|
time_slice_with_tz.call(metadata.timekey)
|
149
152
|
end
|
150
153
|
|
151
154
|
begin
|
155
|
+
$log.info("time_slice: #{time_slice}")
|
156
|
+
$log.info("index_format: #{index_format}")
|
157
|
+
|
152
158
|
values_for_swift_object_chunk[chunk.unique_id] ||= {
|
153
159
|
'%{hex_random}' => hex_random(chunk: chunk)
|
154
160
|
}
|
@@ -158,9 +164,11 @@ module Fluent::Plugin
|
|
158
164
|
}
|
159
165
|
values_for_swift_object_key_post = {
|
160
166
|
'%{time_slice}' => time_slice,
|
161
|
-
'%{index}' =>
|
167
|
+
'%{index}' => format(index_format, i)
|
162
168
|
}.merge!(values_for_swift_object_chunk[chunk.unique_id])
|
163
169
|
|
170
|
+
$log.info("values_for_swift_object_key_post: #{values_for_swift_object_key_post}")
|
171
|
+
|
164
172
|
if uuid_flush_enabled
|
165
173
|
values_for_swift_object_key_post['%{uuid_flush}'] = uuid_random
|
166
174
|
end
|
@@ -169,11 +177,18 @@ module Fluent::Plugin
|
|
169
177
|
values_for_swift_object_key_pre.fetch(matched_key, matched_key)
|
170
178
|
end
|
171
179
|
|
180
|
+
$log.info("swift_path 1: #{swift_path}")
|
181
|
+
|
172
182
|
swift_path = extract_placeholders(swift_path, metadata)
|
183
|
+
|
184
|
+
$log.info("swift_path 2: #{swift_path}")
|
185
|
+
|
173
186
|
swift_path = swift_path.gsub(/%{[^}]+}/, values_for_swift_object_key_post)
|
187
|
+
|
188
|
+
$log.info("swift_path 3: #{swift_path}")
|
174
189
|
if i.positive? && (swift_path == previous_path)
|
175
190
|
if overwrite
|
176
|
-
log.warn
|
191
|
+
log.warn("File: #{swift_path} already exists, but will overwrite!")
|
177
192
|
break
|
178
193
|
else
|
179
194
|
raise "Duplicated path is generated. Use %{index} in swift_object_key_format: Path: #{swift_path}"
|
@@ -202,7 +217,12 @@ module Fluent::Plugin
|
|
202
217
|
tmp.close
|
203
218
|
end
|
204
219
|
File.open(tmp.path) do |file|
|
205
|
-
storage.put_object(
|
220
|
+
storage.put_object(
|
221
|
+
swift_container,
|
222
|
+
swift_path,
|
223
|
+
file,
|
224
|
+
content_type: mime_type
|
225
|
+
)
|
206
226
|
values_for_swift_object_chunk.delete(chunk.unique_id)
|
207
227
|
end
|
208
228
|
ensure
|
@@ -226,7 +246,13 @@ module Fluent::Plugin
|
|
226
246
|
|
227
247
|
private
|
228
248
|
|
229
|
-
attr_accessor :uuid_flush_enabled,
|
249
|
+
attr_accessor :uuid_flush_enabled,
|
250
|
+
:storage,
|
251
|
+
:ext,
|
252
|
+
:mime_type,
|
253
|
+
:formatter,
|
254
|
+
:values_for_swift_object_chunk,
|
255
|
+
:time_slice_with_tz
|
230
256
|
|
231
257
|
def hex_random(chunk:)
|
232
258
|
unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
|
@@ -238,25 +264,14 @@ module Fluent::Plugin
|
|
238
264
|
::UUIDTools::UUID.random_create.to_s
|
239
265
|
end
|
240
266
|
|
241
|
-
# This is stolen from Fluentd
|
242
|
-
def timekey_to_timeformat(timekey:)
|
243
|
-
case timekey
|
244
|
-
when nil then ''
|
245
|
-
when 0...60 then '%Y%m%d%H%M%S'
|
246
|
-
when 60...3600 then '%Y%m%d%H%M'
|
247
|
-
when 3600...86_400 then '%Y%m%d%H'
|
248
|
-
else '%Y%m%d'
|
249
|
-
end
|
250
|
-
end
|
251
|
-
|
252
267
|
def check_container
|
253
268
|
storage.get_container(swift_container)
|
254
269
|
rescue Fog::OpenStack::Storage::NotFound
|
255
270
|
if auto_create_container
|
256
|
-
$log.info
|
271
|
+
$log.info("Creating container `#{swift_container}` on `#{auth_url}`, `#{swift_account}`.")
|
257
272
|
storage.put_container(swift_container)
|
258
273
|
else
|
259
|
-
raise "The specified container does not exist:
|
274
|
+
raise "The specified container does not exist: #{swift_container}."
|
260
275
|
end
|
261
276
|
end
|
262
277
|
|
@@ -282,7 +297,7 @@ module Fluent::Plugin
|
|
282
297
|
end
|
283
298
|
|
284
299
|
swift_object_key_format.gsub('%{hostname}') do |_expr|
|
285
|
-
log.warn
|
300
|
+
log.warn("%{hostname} will be removed in the future. Use `#{Socket.gethostname}` instead.")
|
286
301
|
Socket.gethostname
|
287
302
|
end
|
288
303
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-openstack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- brissenden
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-09-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|