fluent-plugin-azurestorage-gen2 0.2.2 → 0.2.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -3
- data/VERSION +1 -1
- data/fluent-plugin-azurestorage-gen2.gemspec +1 -1
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +39 -17
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 968d3c5f271eaabe28496c3c1a7a145fc4223068
|
4
|
+
data.tar.gz: d451b9a7fee0eeef3d2dbbced493ac49495a43cd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f91296dd56d58faad94aede59b47afa740c746dc5207154ec31e627790428597d08518bd679f87d6ab9f540600017f725428218297731280b476dadb1f993dac
|
7
|
+
data.tar.gz: 452f1de528f620b44fa317918558218d5a757c3018d67ec6dbbf43dcd86f817c86757f1bc6eea11047c68efacebfd80b187895023a480111499ede945ceba196
|
data/README.md
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
[![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
|
4
4
|
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
|
5
5
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
|
6
|
-
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
|
6
|
+
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
|
7
7
|
|
8
8
|
## Requirements
|
9
9
|
|
@@ -106,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
|
|
106
106
|
|
107
107
|
### azure_oauth_refresh_interval
|
108
108
|
|
109
|
-
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
109
|
+
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
110
110
|
|
111
111
|
### azure_oauth_use_azure_cli (Preview)
|
112
112
|
|
@@ -124,9 +124,21 @@ This plugin create container if not exist when you set 'auto_create_container' t
|
|
124
124
|
|
125
125
|
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
|
126
126
|
|
127
|
+
### failsafe_container_check
|
128
|
+
|
129
|
+
On startup, a list operation is called against a container, if that does not exists, the app tries to create it if `auto_create_container` is enabled. If that option is enabled, the application won't fail if any of these operations are failed. (Can be useful the right roles are not set on your container, but set for blobs)
|
130
|
+
|
127
131
|
### enable_retry
|
128
132
|
|
129
|
-
If you set this option, operations can be retried in the buffer. Default value is false. (
|
133
|
+
If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
|
134
|
+
|
135
|
+
### startup_fail_on_error
|
136
|
+
|
137
|
+
If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
|
138
|
+
|
139
|
+
### url_domain_suffix
|
140
|
+
|
141
|
+
The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
|
130
142
|
|
131
143
|
### azure_object_key_format
|
132
144
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.7
|
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-azurestorage-gen2"
|
6
6
|
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
-
gem.license = "
|
7
|
+
gem.license = "MIT License"
|
8
8
|
gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
|
9
9
|
gem.summary = gem.description
|
10
10
|
gem.version = File.read("VERSION").strip
|
@@ -27,20 +27,22 @@ module Fluent::Plugin
|
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
29
|
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
30
|
-
config_param :azure_oauth_refresh_interval, :integer, :default => 60 *
|
30
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
|
31
31
|
config_param :azure_container, :string, :default => nil
|
32
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
36
|
config_param :skip_container_check, :bool, :default => false
|
37
|
+
config_param :failsafe_container_check, :bool, :default => false
|
37
38
|
config_param :enable_retry, :bool, :default => false
|
39
|
+
config_param :startup_fail_on_error, :bool, :default => true
|
40
|
+
config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
|
38
41
|
config_param :format, :string, :default => "out_file"
|
39
42
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
40
43
|
config_param :command_parameter, :string, :default => nil
|
41
44
|
|
42
45
|
DEFAULT_FORMAT_TYPE = "out_file"
|
43
|
-
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
44
46
|
ACCESS_TOKEN_API_VERSION = "2018-02-01"
|
45
47
|
ABFS_API_VERSION = "2018-11-09"
|
46
48
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
@@ -105,7 +107,15 @@ module Fluent::Plugin
|
|
105
107
|
def start
|
106
108
|
setup_access_token
|
107
109
|
if !@skip_container_check
|
108
|
-
|
110
|
+
if @failsafe_container_check
|
111
|
+
begin
|
112
|
+
ensure_container
|
113
|
+
rescue Exception => e
|
114
|
+
log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
|
115
|
+
end
|
116
|
+
else
|
117
|
+
ensure_container
|
118
|
+
end
|
109
119
|
end
|
110
120
|
super
|
111
121
|
end
|
@@ -186,7 +196,19 @@ module Fluent::Plugin
|
|
186
196
|
def setup_access_token
|
187
197
|
if @azure_storage_access_key.nil?
|
188
198
|
@get_token_lock = Concurrent::ReadWriteLock.new
|
189
|
-
|
199
|
+
if @startup_fail_on_error
|
200
|
+
acquire_access_token
|
201
|
+
else
|
202
|
+
while true
|
203
|
+
begin
|
204
|
+
acquire_access_token
|
205
|
+
break
|
206
|
+
rescue Exception => e
|
207
|
+
log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
|
208
|
+
sleep 20
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
190
212
|
if @azure_oauth_refresh_interval > 0
|
191
213
|
log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
|
192
214
|
@get_token_task = Concurrent::TimerTask.new(
|
@@ -269,7 +291,7 @@ module Fluent::Plugin
|
|
269
291
|
params = {:resource => "filesystem" }
|
270
292
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
|
271
293
|
headers[:Authorization] = auth_header
|
272
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
294
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
|
273
295
|
request.on_complete do |response|
|
274
296
|
if response.success?
|
275
297
|
log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
|
@@ -296,7 +318,7 @@ module Fluent::Plugin
|
|
296
318
|
params = {:resource => "filesystem" }
|
297
319
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
|
298
320
|
headers[:Authorization] = auth_header
|
299
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
321
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
|
300
322
|
request.on_complete do |response|
|
301
323
|
if response.success?
|
302
324
|
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
@@ -316,16 +338,16 @@ module Fluent::Plugin
|
|
316
338
|
params = {:resource => "file", :recursive => "false"}
|
317
339
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
318
340
|
headers[:Authorization] = auth_header
|
319
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
341
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
|
320
342
|
request.on_complete do |response|
|
321
343
|
if response.success?
|
322
344
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
|
323
345
|
elsif response.timed_out?
|
324
|
-
|
346
|
+
raise_error "Creating blob '#{blob_path}' request timed out."
|
325
347
|
elsif response.code == 409
|
326
348
|
log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
|
327
349
|
else
|
328
|
-
|
350
|
+
raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
329
351
|
end
|
330
352
|
end
|
331
353
|
request.run
|
@@ -339,18 +361,18 @@ module Fluent::Plugin
|
|
339
361
|
params = {:action => "append", :position => "#{position}"}
|
340
362
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
341
363
|
headers[:Authorization] = auth_header
|
342
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
364
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
|
343
365
|
request.on_complete do |response|
|
344
366
|
if response.success?
|
345
367
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
346
368
|
elsif response.timed_out?
|
347
|
-
|
369
|
+
raise_error "Appending blob #{blob_path}' request timed out."
|
348
370
|
elsif response.code == 404
|
349
371
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
|
350
372
|
elsif response.code == 409
|
351
373
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
|
352
374
|
else
|
353
|
-
|
375
|
+
raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
354
376
|
end
|
355
377
|
end
|
356
378
|
request.run
|
@@ -364,12 +386,12 @@ module Fluent::Plugin
|
|
364
386
|
params = {:action => "flush", :position => "#{position}"}
|
365
387
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
|
366
388
|
headers[:Authorization] = auth_header
|
367
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
389
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
|
368
390
|
request.on_complete do |response|
|
369
391
|
if response.success?
|
370
392
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
|
371
393
|
elsif response.timed_out?
|
372
|
-
|
394
|
+
raise_error "Bloub '#{blob_path}' flush request timed out."
|
373
395
|
else
|
374
396
|
raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
375
397
|
end
|
@@ -385,18 +407,18 @@ module Fluent::Plugin
|
|
385
407
|
content_length = -1
|
386
408
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
387
409
|
headers[:Authorization] = auth_header
|
388
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
410
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
|
389
411
|
request.on_complete do |response|
|
390
412
|
if response.success?
|
391
413
|
log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
|
392
414
|
content_length = response.headers['Content-Length'].to_i
|
393
415
|
elsif response.timed_out?
|
394
|
-
|
416
|
+
raise_error "Get blob properties '#{blob_path}' request timed out."
|
395
417
|
elsif response.code == 404
|
396
418
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
|
397
419
|
content_length = 0
|
398
420
|
else
|
399
|
-
|
421
|
+
raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
400
422
|
end
|
401
423
|
end
|
402
424
|
request.run
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -188,7 +188,7 @@ files:
|
|
188
188
|
- test/plugin/test_out_azurestorage_gen2.rb
|
189
189
|
homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
|
190
190
|
licenses:
|
191
|
-
-
|
191
|
+
- MIT License
|
192
192
|
metadata: {}
|
193
193
|
post_install_message:
|
194
194
|
rdoc_options: []
|