fluent-plugin-azurestorage-gen2 0.2.4 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +8 -0
- data/VERSION +1 -1
- data/fluent-plugin-azurestorage-gen2.gemspec +2 -1
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +68 -32
- metadata +23 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4bff097a151b5af0d596f82ab181d3e141abf0de
|
4
|
+
data.tar.gz: f453dabd8d7719502820e51ae870803d3845a80f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 361a92e5ab216018f95a1f0f0e331d0401bf894da0b65ea3a2bcd1ecfb62e3ce3fd599b8633e77ae385537e9590d2cffa51ec4df38917e5f42660932edc87350
|
7
|
+
data.tar.gz: da8cec0e4867005f2e794c1733c58bfcb33c5ef7e2a6f071f93436506e2d70c3b9a8ecd755ad305973af16bbc9ee163d62d50e632a7d7d26a80964de4d844cc0
|
data/README.md
CHANGED
@@ -124,10 +124,18 @@ This plugin create container if not exist when you set 'auto_create_container' t
|
|
124
124
|
|
125
125
|
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
|
126
126
|
|
127
|
+
### failsafe_container_check
|
128
|
+
|
129
|
+
On startup, a list operation is called against a container, if that does not exists, the app tries to create it if `auto_create_container` is enabled. If that option is enabled, the application won't fail if any of these operations are failed. (Can be useful the right roles are not set on your container, but set for blobs)
|
130
|
+
|
127
131
|
### enable_retry
|
128
132
|
|
129
133
|
If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
|
130
134
|
|
135
|
+
### startup_fail_on_error
|
136
|
+
|
137
|
+
If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
|
138
|
+
|
131
139
|
### url_domain_suffix
|
132
140
|
|
133
141
|
The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.9
|
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-azurestorage-gen2"
|
6
6
|
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
-
gem.license = "
|
7
|
+
gem.license = "MIT License"
|
8
8
|
gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
|
9
9
|
gem.summary = gem.description
|
10
10
|
gem.version = File.read("VERSION").strip
|
@@ -21,6 +21,7 @@ Gem::Specification.new do |gem|
|
|
21
21
|
gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
|
22
22
|
gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
|
23
23
|
gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
|
24
|
+
gem.add_runtime_dependency 'concurrent-ruby', '~> 1.1', '>= 1.1.5'
|
24
25
|
gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
|
25
26
|
gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
|
26
27
|
gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'
|
@@ -6,6 +6,7 @@ require 'tempfile'
|
|
6
6
|
require 'time'
|
7
7
|
require 'typhoeus'
|
8
8
|
require 'fluent/plugin/output'
|
9
|
+
require 'concurrent'
|
9
10
|
require 'zlib'
|
10
11
|
|
11
12
|
module Fluent::Plugin
|
@@ -34,10 +35,13 @@ module Fluent::Plugin
|
|
34
35
|
config_param :store_as, :string, :default => "none"
|
35
36
|
config_param :auto_create_container, :bool, :default => false
|
36
37
|
config_param :skip_container_check, :bool, :default => false
|
38
|
+
config_param :failsafe_container_check, :bool, :default => false
|
37
39
|
config_param :enable_retry, :bool, :default => false
|
40
|
+
config_param :startup_fail_on_error, :bool, :default => true
|
38
41
|
config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
|
39
42
|
config_param :format, :string, :default => "out_file"
|
40
43
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
44
|
+
config_param :hex_random_length, :integer, default: 4
|
41
45
|
config_param :command_parameter, :string, :default => nil
|
42
46
|
|
43
47
|
DEFAULT_FORMAT_TYPE = "out_file"
|
@@ -72,16 +76,6 @@ module Fluent::Plugin
|
|
72
76
|
|
73
77
|
@formatter = formatter_create
|
74
78
|
|
75
|
-
if @localtime
|
76
|
-
@path_slicer = Proc.new {|path|
|
77
|
-
Time.now.strftime(path)
|
78
|
-
}
|
79
|
-
else
|
80
|
-
@path_slicer = Proc.new {|path|
|
81
|
-
Time.now.utc.strftime(path)
|
82
|
-
}
|
83
|
-
end
|
84
|
-
|
85
79
|
if @azure_container.nil?
|
86
80
|
raise Fluent::ConfigError, "azure_container is needed"
|
87
81
|
end
|
@@ -95,7 +89,7 @@ module Fluent::Plugin
|
|
95
89
|
else
|
96
90
|
@final_file_extension = @compressor.ext
|
97
91
|
end
|
98
|
-
|
92
|
+
@values_for_object_chunk = {}
|
99
93
|
end
|
100
94
|
|
101
95
|
def multi_workers_ready?
|
@@ -105,7 +99,15 @@ module Fluent::Plugin
|
|
105
99
|
def start
|
106
100
|
setup_access_token
|
107
101
|
if !@skip_container_check
|
108
|
-
|
102
|
+
if @failsafe_container_check
|
103
|
+
begin
|
104
|
+
ensure_container
|
105
|
+
rescue Exception => e
|
106
|
+
log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
|
107
|
+
end
|
108
|
+
else
|
109
|
+
ensure_container
|
110
|
+
end
|
109
111
|
end
|
110
112
|
super
|
111
113
|
end
|
@@ -116,17 +118,16 @@ module Fluent::Plugin
|
|
116
118
|
end
|
117
119
|
|
118
120
|
def write(chunk)
|
119
|
-
metadata = chunk.metadata
|
120
121
|
if @store_as.nil? || @store_as == "none"
|
121
|
-
generate_log_name(
|
122
|
+
generate_log_name(chunk, @current_index)
|
122
123
|
if @last_azure_storage_path != @azure_storage_path
|
123
124
|
@current_index = 0
|
124
|
-
generate_log_name(
|
125
|
+
generate_log_name(chunk, @current_index)
|
125
126
|
end
|
126
127
|
raw_data = chunk.read
|
127
128
|
unless raw_data.empty?
|
128
129
|
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
129
|
-
upload_blob(raw_data,
|
130
|
+
upload_blob(raw_data, chunk)
|
130
131
|
end
|
131
132
|
chunk.close rescue nil
|
132
133
|
@last_azure_storage_path = @azure_storage_path
|
@@ -136,57 +137,86 @@ module Fluent::Plugin
|
|
136
137
|
begin
|
137
138
|
@compressor.compress(chunk, tmp)
|
138
139
|
tmp.rewind
|
139
|
-
generate_log_name(
|
140
|
+
generate_log_name(chunk, @current_index)
|
140
141
|
if @last_azure_storage_path != @azure_storage_path
|
141
142
|
@current_index = 0
|
142
|
-
generate_log_name(
|
143
|
+
generate_log_name(chunk, @current_index)
|
143
144
|
end
|
144
145
|
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
|
145
146
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
146
|
-
upload_blob(content,
|
147
|
+
upload_blob(content, chunk)
|
147
148
|
@last_azure_storage_path = @azure_storage_path
|
148
149
|
ensure
|
149
150
|
tmp.close(true) rescue nil
|
150
151
|
end
|
152
|
+
@values_for_object_chunk.delete(chunk.unique_id)
|
151
153
|
end
|
152
154
|
|
153
155
|
end
|
154
156
|
|
155
157
|
private
|
156
|
-
def upload_blob(content,
|
158
|
+
def upload_blob(content, chunk)
|
157
159
|
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
158
160
|
existing_content_length = get_blob_properties(@azure_storage_path)
|
159
161
|
if existing_content_length == 0
|
160
162
|
create_blob(@azure_storage_path)
|
161
163
|
end
|
162
|
-
append_blob(content,
|
164
|
+
append_blob(content, chunk, existing_content_length)
|
163
165
|
end
|
164
166
|
|
165
167
|
private
|
166
|
-
def generate_log_name(
|
168
|
+
def generate_log_name(chunk, index)
|
169
|
+
metadata = chunk.metadata
|
167
170
|
time_slice = if metadata.timekey.nil?
|
168
171
|
''.freeze
|
169
172
|
else
|
170
173
|
Time.at(metadata.timekey).utc.strftime(@time_slice_format)
|
171
174
|
end
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
"%
|
175
|
+
if @localtime
|
176
|
+
hms_slicer = Time.now.strftime("%H%M%S")
|
177
|
+
else
|
178
|
+
hms_slicer = Time.now.utc.strftime("%H%M%S")
|
179
|
+
end
|
180
|
+
|
181
|
+
@values_for_object_chunk[chunk.unique_id] ||= {
|
182
|
+
"%{hex_random}" => hex_random(chunk),
|
183
|
+
}
|
184
|
+
values_for_object_key_pre = {
|
185
|
+
"%{path}" => @path,
|
176
186
|
"%{index}" => index,
|
177
187
|
"%{uuid_flush}" => uuid_random,
|
178
188
|
"%{file_extension}" => @final_file_extension
|
179
189
|
}
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
190
|
+
values_for_object_key_post = {
|
191
|
+
"%{date_slice}" => time_slice,
|
192
|
+
"%{time_slice}" => time_slice,
|
193
|
+
"%{hms_slice}" => hms_slicer,
|
194
|
+
}.merge!(@values_for_object_chunk[chunk.unique_id])
|
195
|
+
storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
|
196
|
+
values_for_object_key_pre.fetch(matched_key, matched_key)
|
197
|
+
end
|
198
|
+
storage_path = extract_placeholders(storage_path, chunk)
|
199
|
+
storage_path = storage_path.gsub(%r(%{[^}]+}), values_for_object_key_post)
|
200
|
+
storage_path = "/" + storage_path unless storage_path.start_with?("/")
|
201
|
+
@azure_storage_path = storage_path
|
184
202
|
end
|
185
203
|
|
186
204
|
def setup_access_token
|
187
205
|
if @azure_storage_access_key.nil?
|
188
206
|
@get_token_lock = Concurrent::ReadWriteLock.new
|
189
|
-
|
207
|
+
if @startup_fail_on_error
|
208
|
+
acquire_access_token
|
209
|
+
else
|
210
|
+
while true
|
211
|
+
begin
|
212
|
+
acquire_access_token
|
213
|
+
break
|
214
|
+
rescue Exception => e
|
215
|
+
log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
|
216
|
+
sleep 20
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
190
220
|
if @azure_oauth_refresh_interval > 0
|
191
221
|
log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
|
192
222
|
@get_token_task = Concurrent::TimerTask.new(
|
@@ -404,7 +434,7 @@ module Fluent::Plugin
|
|
404
434
|
end
|
405
435
|
|
406
436
|
private
|
407
|
-
def append_blob(content,
|
437
|
+
def append_blob(content, chunk, existing_content_length)
|
408
438
|
position = 0
|
409
439
|
log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
|
410
440
|
loop do
|
@@ -522,6 +552,12 @@ module Fluent::Plugin
|
|
522
552
|
require 'uuidtools'
|
523
553
|
::UUIDTools::UUID.random_create.to_s
|
524
554
|
end
|
555
|
+
|
556
|
+
def hex_random(chunk)
|
557
|
+
unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
|
558
|
+
unique_hex.reverse!
|
559
|
+
unique_hex[0...@hex_random_length]
|
560
|
+
end
|
525
561
|
|
526
562
|
def timekey_to_timeformat(timekey)
|
527
563
|
case timekey
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -104,6 +104,26 @@ dependencies:
|
|
104
104
|
- - "~>"
|
105
105
|
- !ruby/object:Gem::Version
|
106
106
|
version: '1.4'
|
107
|
+
- !ruby/object:Gem::Dependency
|
108
|
+
name: concurrent-ruby
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - "~>"
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '1.1'
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 1.1.5
|
117
|
+
type: :runtime
|
118
|
+
prerelease: false
|
119
|
+
version_requirements: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - "~>"
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '1.1'
|
124
|
+
- - ">="
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: 1.1.5
|
107
127
|
- !ruby/object:Gem::Dependency
|
108
128
|
name: rake
|
109
129
|
requirement: !ruby/object:Gem::Requirement
|
@@ -188,7 +208,7 @@ files:
|
|
188
208
|
- test/plugin/test_out_azurestorage_gen2.rb
|
189
209
|
homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
|
190
210
|
licenses:
|
191
|
-
-
|
211
|
+
- MIT License
|
192
212
|
metadata: {}
|
193
213
|
post_install_message:
|
194
214
|
rdoc_options: []
|