fluent-plugin-openstack 1.0.7 → 1.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_swift.rb +13 -19
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 06b4ee6c0a7f57cb946378c1d2813c845fddd4f06420b3499002bef45e7fdadf
|
4
|
+
data.tar.gz: 247e316a55a1b29b2f7094a7b49dc7db56eafd42635b05ceaae60fe13edfdf9f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cf7e4f145523b545190a99123091ddf5e4b2e0c25eb2b7f3fb5871369a131a34770e7a62f651b3bc349001cf9b81eed14929ae7397a9dd9901ebc6994b08ccc2
|
7
|
+
data.tar.gz: 703015c28047f081a869a3f62fd5bffaf89da0b567eb35b7f76f187bef638296962f4c5566f941e0a4d2c1ff4abe2bfb0f46997c597ddbfc15fda7590187950d
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.
|
1
|
+
1.0.8
|
@@ -15,9 +15,9 @@ module Fluent::Plugin
|
|
15
15
|
DEFAULT_FORMAT_TYPE = 'out_file'
|
16
16
|
MAX_HEX_RANDOM_LENGTH = 16
|
17
17
|
|
18
|
+
# OpenStack
|
18
19
|
desc 'Path prefix of the files on Swift'
|
19
20
|
config_param :path, :string, default: '%Y%m%d'
|
20
|
-
# OpenStack AUTH
|
21
21
|
desc "Authentication URL. Set a value or `#{ENV['OS_AUTH_URL']}`"
|
22
22
|
config_param :auth_url, :string
|
23
23
|
desc "Authentication User Name. If you use TempAuth, auth_user is ACCOUNT:USER. Set a value or use `#{ENV['OS_USERNAME']}`"
|
@@ -28,7 +28,6 @@ module Fluent::Plugin
|
|
28
28
|
desc "Authentication Region. Optional, not required if there is only one region available. Set a value or use `#{ENV['OS_REGION_NAME']}`"
|
29
29
|
config_param :auth_region, :string, default: nil
|
30
30
|
config_param :swift_account, :string, default: nil
|
31
|
-
|
32
31
|
desc 'Swift container name'
|
33
32
|
config_param :swift_container, :string
|
34
33
|
desc 'Archive format on Swift'
|
@@ -39,9 +38,6 @@ module Fluent::Plugin
|
|
39
38
|
config_param :swift_object_key_format, :string, default: '%{path}/%H%M_%{index}.%{file_extension}'
|
40
39
|
desc 'Create Swift container if it does not exists'
|
41
40
|
config_param :auto_create_container, :bool, default: true
|
42
|
-
config_param :check_apikey_on_start, :bool, default: true
|
43
|
-
desc 'URI of proxy environment'
|
44
|
-
config_param :proxy_uri, :string, default: nil
|
45
41
|
desc 'The length of `%{hex_random}` placeholder (4 - 16)'
|
46
42
|
config_param :hex_random_length, :integer, default: 4
|
47
43
|
desc '`sprintf` format for `%{index}`'
|
@@ -53,9 +49,10 @@ module Fluent::Plugin
|
|
53
49
|
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
54
50
|
end
|
55
51
|
|
52
|
+
# https://docs.fluentd.org/configuration/buffer-section#time
|
56
53
|
config_section :buffer do
|
57
54
|
config_set_default :chunk_keys, ['time']
|
58
|
-
config_set_default :timekey,
|
55
|
+
config_set_default :timekey, 3600
|
59
56
|
config_set_default :timekey_wait, 60
|
60
57
|
end
|
61
58
|
|
@@ -104,7 +101,7 @@ module Fluent::Plugin
|
|
104
101
|
|
105
102
|
self.formatter = formatter_create
|
106
103
|
self.swift_object_key_format = configure_swift_object_key_format
|
107
|
-
self.
|
104
|
+
self.swift_object_chunk_buffer = {}
|
108
105
|
end
|
109
106
|
|
110
107
|
def multi_workers_ready?
|
@@ -125,11 +122,14 @@ module Fluent::Plugin
|
|
125
122
|
rescue StandardError => e
|
126
123
|
raise "Can't call Swift API. Please check your ENV OS_*, your credentials or `auth_url` configuration. Error: #{e.inspect}"
|
127
124
|
end
|
128
|
-
|
125
|
+
if swift_account
|
126
|
+
storage.change_account(swift_account)
|
127
|
+
end
|
129
128
|
check_container
|
130
129
|
super
|
131
130
|
end
|
132
131
|
|
132
|
+
# https://docs.fluentd.org/plugin-development/api-plugin-formatter
|
133
133
|
def format(tag, time, record)
|
134
134
|
new_record = inject_values_to_record(tag, time, record)
|
135
135
|
formatter.format(tag, time, new_record)
|
@@ -140,7 +140,7 @@ module Fluent::Plugin
|
|
140
140
|
metadata = chunk.metadata
|
141
141
|
previous_path = nil
|
142
142
|
begin
|
143
|
-
|
143
|
+
swift_object_chunk_buffer[chunk.unique_id] ||= {
|
144
144
|
'%{hex_random}' => hex_random(chunk: chunk)
|
145
145
|
}
|
146
146
|
values_for_swift_object_key_pre = {
|
@@ -150,7 +150,7 @@ module Fluent::Plugin
|
|
150
150
|
# rubocop:disable Style/FormatString
|
151
151
|
values_for_swift_object_key_post = {
|
152
152
|
'%{index}' => sprintf(index_format, i)
|
153
|
-
}.merge!(
|
153
|
+
}.merge!(swift_object_chunk_buffer[chunk.unique_id])
|
154
154
|
# rubocop:enable Style/FormatString
|
155
155
|
|
156
156
|
if uuid_flush_enabled
|
@@ -160,16 +160,10 @@ module Fluent::Plugin
|
|
160
160
|
swift_path = swift_object_key_format.gsub(/%{[^}]+}/) do |matched_key|
|
161
161
|
values_for_swift_object_key_pre.fetch(matched_key, matched_key)
|
162
162
|
end
|
163
|
-
|
164
|
-
$log.warn("swift_path: #{swift_path}")
|
165
|
-
|
166
163
|
swift_path = extract_placeholders(swift_path, metadata)
|
167
|
-
|
168
|
-
$log.warn("swift_path: #{swift_path}")
|
169
|
-
|
170
164
|
swift_path = swift_path.gsub(/%{[^}]+}/, values_for_swift_object_key_post)
|
171
165
|
|
172
|
-
$log.warn("
|
166
|
+
$log.warn("File flushing: #{swift_path}")
|
173
167
|
|
174
168
|
if i.positive? && (swift_path == previous_path)
|
175
169
|
if overwrite
|
@@ -208,7 +202,7 @@ module Fluent::Plugin
|
|
208
202
|
file,
|
209
203
|
content_type: mime_type
|
210
204
|
)
|
211
|
-
|
205
|
+
swift_object_chunk_buffer.delete(chunk.unique_id)
|
212
206
|
end
|
213
207
|
ensure
|
214
208
|
begin
|
@@ -236,7 +230,7 @@ module Fluent::Plugin
|
|
236
230
|
:ext,
|
237
231
|
:mime_type,
|
238
232
|
:formatter,
|
239
|
-
:
|
233
|
+
:swift_object_chunk_buffer
|
240
234
|
|
241
235
|
def hex_random(chunk:)
|
242
236
|
unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
|