fluent-plugin-azurestorage-gen2 0.2.0 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Makefile +2 -4
- data/README.md +19 -6
- data/VERSION +1 -1
- data/fluent-plugin-azurestorage-gen2.gemspec +1 -1
- data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb +1 -0
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +55 -24
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 02170d63c314b7bb91fb1722b0ca1a4dccad9bd0
|
4
|
+
data.tar.gz: 7fc221dc0be643681800be67c954634caa7df518
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad9f64e7f2af1d65177e683aa808df6187d9ace11a0f976a925a11d31e74db15c324b10035d4610ff387b5f2b3a1939c4f8c2cf5f7f25e4a237c16350b4ec620
|
7
|
+
data.tar.gz: eeff3b1a4c7ea0b48e207c270e1679ba770c2bca21146f879bfb893a1fca314419c5a7a6c35a1437ca4967dbd87a0d703c81b66a16d1326e187da90e2fcf1ac1
|
data/Makefile
CHANGED
@@ -8,8 +8,6 @@ install: gem
|
|
8
8
|
push: gem
|
9
9
|
gem push fluent-plugin-azurestorage-gen2*.gem
|
10
10
|
|
11
|
-
tag
|
11
|
+
tag:
|
12
12
|
git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
|
13
|
-
git
|
14
|
-
git push origin "v$$(cat VERSION)"
|
15
|
-
git push -u origin "release/$$(cat VERSION)"
|
13
|
+
git push origin "v$$(cat VERSION)"
|
data/README.md
CHANGED
@@ -3,13 +3,13 @@
|
|
3
3
|
[](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
5
5
|
[](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
|
6
|
-

|
6
|
+

|
7
7
|
|
8
8
|
## Requirements
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.
|
12
|
+
| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -34,9 +34,10 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
34
34
|
azure_object_key_format %{path}-%{index}.%{file_extension}
|
35
35
|
azure_oauth_refresh_interval 3600
|
36
36
|
time_slice_format %Y%m%d-%H
|
37
|
-
file_extension log
|
37
|
+
file_extension log # only used with store_as none
|
38
38
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
39
39
|
auto_create_container true
|
40
|
+
store_as gzip
|
40
41
|
format single_value
|
41
42
|
<buffer tag,time>
|
42
43
|
@type file
|
@@ -61,8 +62,8 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
61
62
|
azure_oauth_secret <my client secret>
|
62
63
|
azure_oauth_refresh_interval 3600
|
63
64
|
time_slice_format %Y%m%d-%H
|
64
|
-
file_extension log
|
65
65
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
66
|
+
store_as gzip
|
66
67
|
auto_create_container true
|
67
68
|
format single_value
|
68
69
|
<buffer tag,time>
|
@@ -105,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
|
|
105
106
|
|
106
107
|
### azure_oauth_refresh_interval
|
107
108
|
|
108
|
-
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
109
|
+
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
109
110
|
|
110
111
|
### azure_oauth_use_azure_cli (Preview)
|
111
112
|
|
@@ -121,7 +122,19 @@ This plugin create container if not exist when you set 'auto_create_container' t
|
|
121
122
|
|
122
123
|
### skip_container_check
|
123
124
|
|
124
|
-
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform
|
125
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
|
126
|
+
|
127
|
+
### enable_retry
|
128
|
+
|
129
|
+
If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
|
130
|
+
|
131
|
+
### startup_fail_on_error
|
132
|
+
|
133
|
+
If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
|
134
|
+
|
135
|
+
### url_domain_suffix
|
136
|
+
|
137
|
+
The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
|
125
138
|
|
126
139
|
### azure_object_key_format
|
127
140
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.5
|
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-azurestorage-gen2"
|
6
6
|
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
-
gem.license = "
|
7
|
+
gem.license = "MIT License"
|
8
8
|
gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
|
9
9
|
gem.summary = gem.description
|
10
10
|
gem.version = File.read("VERSION").strip
|
@@ -27,19 +27,21 @@ module Fluent::Plugin
|
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
29
|
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
30
|
-
config_param :azure_oauth_refresh_interval, :integer, :default => 60 *
|
30
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
|
31
31
|
config_param :azure_container, :string, :default => nil
|
32
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
36
|
config_param :skip_container_check, :bool, :default => false
|
37
|
+
config_param :enable_retry, :bool, :default => false
|
38
|
+
config_param :startup_fail_on_error, :bool, :default => true
|
39
|
+
config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
|
37
40
|
config_param :format, :string, :default => "out_file"
|
38
41
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
39
42
|
config_param :command_parameter, :string, :default => nil
|
40
43
|
|
41
44
|
DEFAULT_FORMAT_TYPE = "out_file"
|
42
|
-
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
43
45
|
ACCESS_TOKEN_API_VERSION = "2018-02-01"
|
44
46
|
ABFS_API_VERSION = "2018-11-09"
|
45
47
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
@@ -127,23 +129,25 @@ module Fluent::Plugin
|
|
127
129
|
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
128
130
|
upload_blob(raw_data, metadata)
|
129
131
|
end
|
132
|
+
chunk.close rescue nil
|
130
133
|
@last_azure_storage_path = @azure_storage_path
|
131
134
|
else
|
132
135
|
tmp = Tempfile.new("azure-")
|
136
|
+
tmp.binmode
|
133
137
|
begin
|
134
138
|
@compressor.compress(chunk, tmp)
|
135
|
-
tmp.
|
139
|
+
tmp.rewind
|
136
140
|
generate_log_name(metadata, @current_index)
|
137
141
|
if @last_azure_storage_path != @azure_storage_path
|
138
142
|
@current_index = 0
|
139
143
|
generate_log_name(metadata, @current_index)
|
140
144
|
end
|
141
|
-
log.debug "Start uploading temp file: #{tmp.path}"
|
145
|
+
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
|
142
146
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
143
147
|
upload_blob(content, metadata)
|
144
148
|
@last_azure_storage_path = @azure_storage_path
|
145
149
|
ensure
|
146
|
-
tmp.
|
150
|
+
tmp.close(true) rescue nil
|
147
151
|
end
|
148
152
|
end
|
149
153
|
|
@@ -151,7 +155,7 @@ module Fluent::Plugin
|
|
151
155
|
|
152
156
|
private
|
153
157
|
def upload_blob(content, metadata)
|
154
|
-
log.debug "azurestorage_gen2:
|
158
|
+
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
155
159
|
existing_content_length = get_blob_properties(@azure_storage_path)
|
156
160
|
if existing_content_length == 0
|
157
161
|
create_blob(@azure_storage_path)
|
@@ -183,7 +187,19 @@ module Fluent::Plugin
|
|
183
187
|
def setup_access_token
|
184
188
|
if @azure_storage_access_key.nil?
|
185
189
|
@get_token_lock = Concurrent::ReadWriteLock.new
|
186
|
-
|
190
|
+
if @startup_fail_on_error
|
191
|
+
acquire_access_token
|
192
|
+
else
|
193
|
+
while true
|
194
|
+
begin
|
195
|
+
acquire_access_token
|
196
|
+
break
|
197
|
+
rescue Exception => e
|
198
|
+
log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
|
199
|
+
sleep 20
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
187
203
|
if @azure_oauth_refresh_interval > 0
|
188
204
|
log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
|
189
205
|
@get_token_task = Concurrent::TimerTask.new(
|
@@ -217,7 +233,7 @@ module Fluent::Plugin
|
|
217
233
|
# https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
|
218
234
|
private
|
219
235
|
def acquire_access_token_msi
|
220
|
-
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.
|
236
|
+
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azures.com/" }
|
221
237
|
unless @azure_instance_msi.nil?
|
222
238
|
params[:msi_res_id] = @azure_instance_msi
|
223
239
|
end
|
@@ -266,7 +282,7 @@ module Fluent::Plugin
|
|
266
282
|
params = {:resource => "filesystem" }
|
267
283
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
|
268
284
|
headers[:Authorization] = auth_header
|
269
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
285
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
|
270
286
|
request.on_complete do |response|
|
271
287
|
if response.success?
|
272
288
|
log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
|
@@ -293,12 +309,12 @@ module Fluent::Plugin
|
|
293
309
|
params = {:resource => "filesystem" }
|
294
310
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
|
295
311
|
headers[:Authorization] = auth_header
|
296
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
312
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
|
297
313
|
request.on_complete do |response|
|
298
314
|
if response.success?
|
299
315
|
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
300
316
|
elsif response.timed_out?
|
301
|
-
raise Fluent::UnrecoverableError,
|
317
|
+
raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
|
302
318
|
else
|
303
319
|
raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
304
320
|
end
|
@@ -313,16 +329,16 @@ module Fluent::Plugin
|
|
313
329
|
params = {:resource => "file", :recursive => "false"}
|
314
330
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
315
331
|
headers[:Authorization] = auth_header
|
316
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
332
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
|
317
333
|
request.on_complete do |response|
|
318
334
|
if response.success?
|
319
335
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
|
320
336
|
elsif response.timed_out?
|
321
|
-
|
337
|
+
raise_error "Creating blob '#{blob_path}' request timed out."
|
322
338
|
elsif response.code == 409
|
323
339
|
log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
|
324
340
|
else
|
325
|
-
|
341
|
+
raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
326
342
|
end
|
327
343
|
end
|
328
344
|
request.run
|
@@ -330,24 +346,24 @@ module Fluent::Plugin
|
|
330
346
|
|
331
347
|
private
|
332
348
|
def append_blob_block(blob_path, content, position)
|
333
|
-
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
349
|
+
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
|
334
350
|
datestamp = create_request_date
|
335
351
|
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
|
336
352
|
params = {:action => "append", :position => "#{position}"}
|
337
353
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
338
354
|
headers[:Authorization] = auth_header
|
339
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
355
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
|
340
356
|
request.on_complete do |response|
|
341
357
|
if response.success?
|
342
358
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
343
359
|
elsif response.timed_out?
|
344
|
-
|
360
|
+
raise_error "Appending blob #{blob_path}' request timed out."
|
345
361
|
elsif response.code == 404
|
346
362
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
|
347
363
|
elsif response.code == 409
|
348
364
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
|
349
365
|
else
|
350
|
-
|
366
|
+
raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
351
367
|
end
|
352
368
|
end
|
353
369
|
request.run
|
@@ -361,14 +377,14 @@ module Fluent::Plugin
|
|
361
377
|
params = {:action => "flush", :position => "#{position}"}
|
362
378
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
|
363
379
|
headers[:Authorization] = auth_header
|
364
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
380
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
|
365
381
|
request.on_complete do |response|
|
366
382
|
if response.success?
|
367
383
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
|
368
384
|
elsif response.timed_out?
|
369
|
-
|
385
|
+
raise_error "Bloub '#{blob_path}' flush request timed out."
|
370
386
|
else
|
371
|
-
|
387
|
+
raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
372
388
|
end
|
373
389
|
end
|
374
390
|
request.run
|
@@ -382,18 +398,18 @@ module Fluent::Plugin
|
|
382
398
|
content_length = -1
|
383
399
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
384
400
|
headers[:Authorization] = auth_header
|
385
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
401
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
|
386
402
|
request.on_complete do |response|
|
387
403
|
if response.success?
|
388
404
|
log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
|
389
405
|
content_length = response.headers['Content-Length'].to_i
|
390
406
|
elsif response.timed_out?
|
391
|
-
|
407
|
+
raise_error "Get blob properties '#{blob_path}' request timed out."
|
392
408
|
elsif response.code == 404
|
393
409
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
|
394
410
|
content_length = 0
|
395
411
|
else
|
396
|
-
|
412
|
+
raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
397
413
|
end
|
398
414
|
end
|
399
415
|
request.run
|
@@ -506,6 +522,15 @@ module Fluent::Plugin
|
|
506
522
|
Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
|
507
523
|
end
|
508
524
|
|
525
|
+
private
|
526
|
+
def raise_error(error_message)
|
527
|
+
if @enable_retry
|
528
|
+
raise BlobOperationError, error_message
|
529
|
+
else
|
530
|
+
raise Fluent::UnrecoverableError, error_message
|
531
|
+
end
|
532
|
+
end
|
533
|
+
|
509
534
|
def uuid_random
|
510
535
|
require 'uuidtools'
|
511
536
|
::UUIDTools::UUID.random_create.to_s
|
@@ -622,4 +647,10 @@ module Fluent::Plugin
|
|
622
647
|
super(message)
|
623
648
|
end
|
624
649
|
end
|
650
|
+
|
651
|
+
class BlobOperationError < StandardError
|
652
|
+
def initialize(message="Default message")
|
653
|
+
super(message)
|
654
|
+
end
|
655
|
+
end
|
625
656
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -188,7 +188,7 @@ files:
|
|
188
188
|
- test/plugin/test_out_azurestorage_gen2.rb
|
189
189
|
homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
|
190
190
|
licenses:
|
191
|
-
-
|
191
|
+
- MIT License
|
192
192
|
metadata: {}
|
193
193
|
post_install_message:
|
194
194
|
rdoc_options: []
|