fluent-plugin-azurestorage-gen2 0.2.2 → 0.2.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c631dda137f661c5692855eb46d97696df14a5e0
4
- data.tar.gz: 174ace2e26b75f25b12a38d3c76726d3526f0866
3
+ metadata.gz: 968d3c5f271eaabe28496c3c1a7a145fc4223068
4
+ data.tar.gz: d451b9a7fee0eeef3d2dbbced493ac49495a43cd
5
5
  SHA512:
6
- metadata.gz: 5f135ffbe42f292666e66bca2b84a8c1f49d3ddf727460e376c95658b0382419a9008b110371d48314ccef6af0dc2f3cff5b0ae64730ae29222571dd20333163
7
- data.tar.gz: e81ba7ac31051ac5813ab4f3c845ee866f7dce3061c16a72fc5f3164e4b53c9c345c1135820762671162f092b195191a93ff6650a2562fc61cb12ea9a4b97ff2
6
+ metadata.gz: f91296dd56d58faad94aede59b47afa740c746dc5207154ec31e627790428597d08518bd679f87d6ab9f540600017f725428218297731280b476dadb1f993dac
7
+ data.tar.gz: 452f1de528f620b44fa317918558218d5a757c3018d67ec6dbbf43dcd86f817c86757f1bc6eea11047c68efacebfd80b187895023a480111499ede945ceba196
data/README.md CHANGED
@@ -3,7 +3,7 @@
3
3
  [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
4
4
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
5
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
6
- ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
6
+ ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
7
7
 
8
8
  ## Requirements
9
9
 
@@ -106,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
106
106
 
107
107
  ### azure_oauth_refresh_interval
108
108
 
109
- OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
109
+ OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
110
110
 
111
111
  ### azure_oauth_use_azure_cli (Preview)
112
112
 
@@ -124,9 +124,21 @@ This plugin create container if not exist when you set 'auto_create_container' t
124
124
 
125
125
  You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
126
126
 
127
+ ### failsafe_container_check
128
+
129
+ On startup, a list operation is called against a container, if that does not exists, the app tries to create it if `auto_create_container` is enabled. If that option is enabled, the application won't fail if any of these operations are failed. (Can be useful the right roles are not set on your container, but set for blobs)
130
+
127
131
  ### enable_retry
128
132
 
129
- If you set this option, operations can be retried in the buffer. Default value is false. (Currently only flush command is supported)
133
+ If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
134
+
135
+ ### startup_fail_on_error
136
+
137
+ If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
138
+
139
+ ### url_domain_suffix
140
+
141
+ The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
130
142
 
131
143
  ### azure_object_key_format
132
144
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.2
1
+ 0.2.7
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-azurestorage-gen2"
6
6
  gem.description = "Azure Storage output plugin for Fluentd event collector"
7
- gem.license = "Apache-2.0"
7
+ gem.license = "MIT License"
8
8
  gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
9
9
  gem.summary = gem.description
10
10
  gem.version = File.read("VERSION").strip
@@ -27,20 +27,22 @@ module Fluent::Plugin
27
27
  config_param :azure_oauth_secret, :string, :default => nil, :secret => true
28
28
  config_param :azure_oauth_tenant_id, :string, :default => nil
29
29
  config_param :azure_oauth_use_azure_cli, :bool, :default => false
30
- config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
30
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
31
31
  config_param :azure_container, :string, :default => nil
32
32
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
36
  config_param :skip_container_check, :bool, :default => false
37
+ config_param :failsafe_container_check, :bool, :default => false
37
38
  config_param :enable_retry, :bool, :default => false
39
+ config_param :startup_fail_on_error, :bool, :default => true
40
+ config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
38
41
  config_param :format, :string, :default => "out_file"
39
42
  config_param :time_slice_format, :string, :default => '%Y%m%d'
40
43
  config_param :command_parameter, :string, :default => nil
41
44
 
42
45
  DEFAULT_FORMAT_TYPE = "out_file"
43
- URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
44
46
  ACCESS_TOKEN_API_VERSION = "2018-02-01"
45
47
  ABFS_API_VERSION = "2018-11-09"
46
48
  AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
@@ -105,7 +107,15 @@ module Fluent::Plugin
105
107
  def start
106
108
  setup_access_token
107
109
  if !@skip_container_check
108
- ensure_container
110
+ if @failsafe_container_check
111
+ begin
112
+ ensure_container
113
+ rescue Exception => e
114
+ log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
115
+ end
116
+ else
117
+ ensure_container
118
+ end
109
119
  end
110
120
  super
111
121
  end
@@ -186,7 +196,19 @@ module Fluent::Plugin
186
196
  def setup_access_token
187
197
  if @azure_storage_access_key.nil?
188
198
  @get_token_lock = Concurrent::ReadWriteLock.new
189
- acquire_access_token
199
+ if @startup_fail_on_error
200
+ acquire_access_token
201
+ else
202
+ while true
203
+ begin
204
+ acquire_access_token
205
+ break
206
+ rescue Exception => e
207
+ log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
208
+ sleep 20
209
+ end
210
+ end
211
+ end
190
212
  if @azure_oauth_refresh_interval > 0
191
213
  log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
192
214
  @get_token_task = Concurrent::TimerTask.new(
@@ -269,7 +291,7 @@ module Fluent::Plugin
269
291
  params = {:resource => "filesystem" }
270
292
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
271
293
  headers[:Authorization] = auth_header
272
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
294
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
273
295
  request.on_complete do |response|
274
296
  if response.success?
275
297
  log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -296,7 +318,7 @@ module Fluent::Plugin
296
318
  params = {:resource => "filesystem" }
297
319
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
298
320
  headers[:Authorization] = auth_header
299
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
321
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
300
322
  request.on_complete do |response|
301
323
  if response.success?
302
324
  log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
@@ -316,16 +338,16 @@ module Fluent::Plugin
316
338
  params = {:resource => "file", :recursive => "false"}
317
339
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
318
340
  headers[:Authorization] = auth_header
319
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
341
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
320
342
  request.on_complete do |response|
321
343
  if response.success?
322
344
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
323
345
  elsif response.timed_out?
324
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request timed out."
346
+ raise_error "Creating blob '#{blob_path}' request timed out."
325
347
  elsif response.code == 409
326
348
  log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
327
349
  else
328
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
350
+ raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
329
351
  end
330
352
  end
331
353
  request.run
@@ -339,18 +361,18 @@ module Fluent::Plugin
339
361
  params = {:action => "append", :position => "#{position}"}
340
362
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
341
363
  headers[:Authorization] = auth_header
342
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
364
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
343
365
  request.on_complete do |response|
344
366
  if response.success?
345
367
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
346
368
  elsif response.timed_out?
347
- raise Fluent::UnrecoverableError, "Appending blob #{blob_path}' request timed out."
369
+ raise_error "Appending blob #{blob_path}' request timed out."
348
370
  elsif response.code == 404
349
371
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
350
372
  elsif response.code == 409
351
373
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
352
374
  else
353
- raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
375
+ raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
354
376
  end
355
377
  end
356
378
  request.run
@@ -364,12 +386,12 @@ module Fluent::Plugin
364
386
  params = {:action => "flush", :position => "#{position}"}
365
387
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
366
388
  headers[:Authorization] = auth_header
367
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
389
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
368
390
  request.on_complete do |response|
369
391
  if response.success?
370
392
  log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
371
393
  elsif response.timed_out?
372
- raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
394
+ raise_error "Bloub '#{blob_path}' flush request timed out."
373
395
  else
374
396
  raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
375
397
  end
@@ -385,18 +407,18 @@ module Fluent::Plugin
385
407
  content_length = -1
386
408
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
387
409
  headers[:Authorization] = auth_header
388
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
410
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
389
411
  request.on_complete do |response|
390
412
  if response.success?
391
413
  log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
392
414
  content_length = response.headers['Content-Length'].to_i
393
415
  elsif response.timed_out?
394
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request timed out."
416
+ raise_error "Get blob properties '#{blob_path}' request timed out."
395
417
  elsif response.code == 404
396
418
  log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
397
419
  content_length = 0
398
420
  else
399
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
421
+ raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
400
422
  end
401
423
  end
402
424
  request.run
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-01-07 00:00:00.000000000 Z
11
+ date: 2020-02-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -188,7 +188,7 @@ files:
188
188
  - test/plugin/test_out_azurestorage_gen2.rb
189
189
  homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
190
190
  licenses:
191
- - Apache-2.0
191
+ - MIT License
192
192
  metadata: {}
193
193
  post_install_message:
194
194
  rdoc_options: []