fluent-plugin-azurestorage-gen2 0.2.0 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c4b4636ae463e1d010ac8b6c069c1c9e47d226a9
4
- data.tar.gz: b215a1aa8a4c8c9e768b38f1ee3a318b8d5f42a2
3
+ metadata.gz: 02170d63c314b7bb91fb1722b0ca1a4dccad9bd0
4
+ data.tar.gz: 7fc221dc0be643681800be67c954634caa7df518
5
5
  SHA512:
6
- metadata.gz: 8c7eb7bd110a30963f4cc5b99303829d234b9285a7a29e4b9eb3115511e0c7870f06f94cf237b60ce8cd76e74a0c634a71341747b0991b5080f7dbab2f94ec3f
7
- data.tar.gz: a02576ae303d449b2d2301c88d80e076792acf58d90b50fafa4822a3ab96581f1566ccb052cd153d42a323fb4563575b648271f89a9220d815ba1a3d5f953be2
6
+ metadata.gz: ad9f64e7f2af1d65177e683aa808df6187d9ace11a0f976a925a11d31e74db15c324b10035d4610ff387b5f2b3a1939c4f8c2cf5f7f25e4a237c16350b4ec620
7
+ data.tar.gz: eeff3b1a4c7ea0b48e207c270e1679ba770c2bca21146f879bfb893a1fca314419c5a7a6c35a1437ca4967dbd87a0d703c81b66a16d1326e187da90e2fcf1ac1
data/Makefile CHANGED
@@ -8,8 +8,6 @@ install: gem
8
8
  push: gem
9
9
  gem push fluent-plugin-azurestorage-gen2*.gem
10
10
 
11
- tag-and-branch:
11
+ tag:
12
12
  git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
13
- git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
14
- git push origin "v$$(cat VERSION)"
15
- git push -u origin "release/$$(cat VERSION)"
13
+ git push origin "v$$(cat VERSION)"
data/README.md CHANGED
@@ -3,13 +3,13 @@
3
3
  [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
4
4
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
5
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
6
- ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
6
+ ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
7
7
 
8
8
  ## Requirements
9
9
 
10
10
  | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
11
  |------------------------|---------|------|
12
- | >= 0.2.0 | >= v0.14.0 | >= 2.4 |
12
+ | >= 0.1.0 | >= v0.14.0 | >= 2.4 |
13
13
 
14
14
  ## Overview
15
15
 
@@ -34,9 +34,10 @@ $ gem install fluent-plugin-azurestorage-gen2
34
34
  azure_object_key_format %{path}-%{index}.%{file_extension}
35
35
  azure_oauth_refresh_interval 3600
36
36
  time_slice_format %Y%m%d-%H
37
- file_extension log
37
+ file_extension log # only used with store_as none
38
38
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
39
39
  auto_create_container true
40
+ store_as gzip
40
41
  format single_value
41
42
  <buffer tag,time>
42
43
  @type file
@@ -61,8 +62,8 @@ $ gem install fluent-plugin-azurestorage-gen2
61
62
  azure_oauth_secret <my client secret>
62
63
  azure_oauth_refresh_interval 3600
63
64
  time_slice_format %Y%m%d-%H
64
- file_extension log
65
65
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
66
+ store_as gzip
66
67
  auto_create_container true
67
68
  format single_value
68
69
  <buffer tag,time>
@@ -105,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
105
106
 
106
107
  ### azure_oauth_refresh_interval
107
108
 
108
- OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
109
+ OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
109
110
 
110
111
  ### azure_oauth_use_azure_cli (Preview)
111
112
 
@@ -121,7 +122,19 @@ This plugin create container if not exist when you set 'auto_create_container' t
121
122
 
122
123
  ### skip_container_check
123
124
 
124
- You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
125
+ You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
126
+
127
+ ### enable_retry
128
+
129
+ If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
130
+
131
+ ### startup_fail_on_error
132
+
133
+ If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
134
+
135
+ ### url_domain_suffix
136
+
137
+ The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
125
138
 
126
139
  ### azure_object_key_format
127
140
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.0
1
+ 0.2.5
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-azurestorage-gen2"
6
6
  gem.description = "Azure Storage output plugin for Fluentd event collector"
7
- gem.license = "Apache-2.0"
7
+ gem.license = "MIT License"
8
8
  gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
9
9
  gem.summary = gem.description
10
10
  gem.version = File.read("VERSION").strip
@@ -24,6 +24,7 @@ module Fluent
24
24
  chunk.path
25
25
  else
26
26
  w = Tempfile.new("chunk-gzip-tmp")
27
+ w.binmode
27
28
  chunk.write_to(w)
28
29
  w.close
29
30
  w.path
@@ -20,6 +20,7 @@ module Fluent
20
20
 
21
21
  def compress(chunk, tmp)
22
22
  w = Tempfile.new("chunk-xz-tmp")
23
+ w.binmode
23
24
  chunk.write_to(w)
24
25
  w.close
25
26
 
@@ -20,6 +20,7 @@ module Fluent
20
20
 
21
21
  def compress(chunk, tmp)
22
22
  w = Tempfile.new("chunk-tmp")
23
+ w.binmode
23
24
  chunk.write_to(w)
24
25
  w.close
25
26
 
@@ -27,19 +27,21 @@ module Fluent::Plugin
27
27
  config_param :azure_oauth_secret, :string, :default => nil, :secret => true
28
28
  config_param :azure_oauth_tenant_id, :string, :default => nil
29
29
  config_param :azure_oauth_use_azure_cli, :bool, :default => false
30
- config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
30
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
31
31
  config_param :azure_container, :string, :default => nil
32
32
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
36
  config_param :skip_container_check, :bool, :default => false
37
+ config_param :enable_retry, :bool, :default => false
38
+ config_param :startup_fail_on_error, :bool, :default => true
39
+ config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
37
40
  config_param :format, :string, :default => "out_file"
38
41
  config_param :time_slice_format, :string, :default => '%Y%m%d'
39
42
  config_param :command_parameter, :string, :default => nil
40
43
 
41
44
  DEFAULT_FORMAT_TYPE = "out_file"
42
- URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
43
45
  ACCESS_TOKEN_API_VERSION = "2018-02-01"
44
46
  ABFS_API_VERSION = "2018-11-09"
45
47
  AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
@@ -127,23 +129,25 @@ module Fluent::Plugin
127
129
  log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
128
130
  upload_blob(raw_data, metadata)
129
131
  end
132
+ chunk.close rescue nil
130
133
  @last_azure_storage_path = @azure_storage_path
131
134
  else
132
135
  tmp = Tempfile.new("azure-")
136
+ tmp.binmode
133
137
  begin
134
138
  @compressor.compress(chunk, tmp)
135
- tmp.close
139
+ tmp.rewind
136
140
  generate_log_name(metadata, @current_index)
137
141
  if @last_azure_storage_path != @azure_storage_path
138
142
  @current_index = 0
139
143
  generate_log_name(metadata, @current_index)
140
144
  end
141
- log.debug "Start uploading temp file: #{tmp.path}"
145
+ log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
142
146
  content = File.open(tmp.path, 'rb') { |file| file.read }
143
147
  upload_blob(content, metadata)
144
148
  @last_azure_storage_path = @azure_storage_path
145
149
  ensure
146
- tmp.unlink
150
+ tmp.close(true) rescue nil
147
151
  end
148
152
  end
149
153
 
@@ -151,7 +155,7 @@ module Fluent::Plugin
151
155
 
152
156
  private
153
157
  def upload_blob(content, metadata)
154
- log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
158
+ log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
155
159
  existing_content_length = get_blob_properties(@azure_storage_path)
156
160
  if existing_content_length == 0
157
161
  create_blob(@azure_storage_path)
@@ -183,7 +187,19 @@ module Fluent::Plugin
183
187
  def setup_access_token
184
188
  if @azure_storage_access_key.nil?
185
189
  @get_token_lock = Concurrent::ReadWriteLock.new
186
- acquire_access_token
190
+ if @startup_fail_on_error
191
+ acquire_access_token
192
+ else
193
+ while true
194
+ begin
195
+ acquire_access_token
196
+ break
197
+ rescue Exception => e
198
+ log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
199
+ sleep 20
200
+ end
201
+ end
202
+ end
187
203
  if @azure_oauth_refresh_interval > 0
188
204
  log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
189
205
  @get_token_task = Concurrent::TimerTask.new(
@@ -217,7 +233,7 @@ module Fluent::Plugin
217
233
  # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
218
234
  private
219
235
  def acquire_access_token_msi
220
- params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
236
+ params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azures.com/" }
221
237
  unless @azure_instance_msi.nil?
222
238
  params[:msi_res_id] = @azure_instance_msi
223
239
  end
@@ -266,7 +282,7 @@ module Fluent::Plugin
266
282
  params = {:resource => "filesystem" }
267
283
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
268
284
  headers[:Authorization] = auth_header
269
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
285
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
270
286
  request.on_complete do |response|
271
287
  if response.success?
272
288
  log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -293,12 +309,12 @@ module Fluent::Plugin
293
309
  params = {:resource => "filesystem" }
294
310
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
295
311
  headers[:Authorization] = auth_header
296
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
312
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
297
313
  request.on_complete do |response|
298
314
  if response.success?
299
315
  log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
300
316
  elsif response.timed_out?
301
- raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
317
+ raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
302
318
  else
303
319
  raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
304
320
  end
@@ -313,16 +329,16 @@ module Fluent::Plugin
313
329
  params = {:resource => "file", :recursive => "false"}
314
330
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
315
331
  headers[:Authorization] = auth_header
316
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
332
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
317
333
  request.on_complete do |response|
318
334
  if response.success?
319
335
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
320
336
  elsif response.timed_out?
321
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request timed out."
337
+ raise_error "Creating blob '#{blob_path}' request timed out."
322
338
  elsif response.code == 409
323
339
  log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
324
340
  else
325
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
341
+ raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
326
342
  end
327
343
  end
328
344
  request.run
@@ -330,24 +346,24 @@ module Fluent::Plugin
330
346
 
331
347
  private
332
348
  def append_blob_block(blob_path, content, position)
333
- log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
349
+ log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
334
350
  datestamp = create_request_date
335
351
  headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
336
352
  params = {:action => "append", :position => "#{position}"}
337
353
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
338
354
  headers[:Authorization] = auth_header
339
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
355
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
340
356
  request.on_complete do |response|
341
357
  if response.success?
342
358
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
343
359
  elsif response.timed_out?
344
- raise Fluent::UnrecoverableError, "Appending blob #{blob_path}' request timed out."
360
+ raise_error "Appending blob #{blob_path}' request timed out."
345
361
  elsif response.code == 404
346
362
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
347
363
  elsif response.code == 409
348
364
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
349
365
  else
350
- raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
366
+ raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
351
367
  end
352
368
  end
353
369
  request.run
@@ -361,14 +377,14 @@ module Fluent::Plugin
361
377
  params = {:action => "flush", :position => "#{position}"}
362
378
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
363
379
  headers[:Authorization] = auth_header
364
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
380
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
365
381
  request.on_complete do |response|
366
382
  if response.success?
367
383
  log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
368
384
  elsif response.timed_out?
369
- raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
385
+ raise_error "Bloub '#{blob_path}' flush request timed out."
370
386
  else
371
- raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
387
+ raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
372
388
  end
373
389
  end
374
390
  request.run
@@ -382,18 +398,18 @@ module Fluent::Plugin
382
398
  content_length = -1
383
399
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
384
400
  headers[:Authorization] = auth_header
385
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
401
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
386
402
  request.on_complete do |response|
387
403
  if response.success?
388
404
  log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
389
405
  content_length = response.headers['Content-Length'].to_i
390
406
  elsif response.timed_out?
391
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request timed out."
407
+ raise_error "Get blob properties '#{blob_path}' request timed out."
392
408
  elsif response.code == 404
393
409
  log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
394
410
  content_length = 0
395
411
  else
396
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
412
+ raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
397
413
  end
398
414
  end
399
415
  request.run
@@ -506,6 +522,15 @@ module Fluent::Plugin
506
522
  Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
507
523
  end
508
524
 
525
+ private
526
+ def raise_error(error_message)
527
+ if @enable_retry
528
+ raise BlobOperationError, error_message
529
+ else
530
+ raise Fluent::UnrecoverableError, error_message
531
+ end
532
+ end
533
+
509
534
  def uuid_random
510
535
  require 'uuidtools'
511
536
  ::UUIDTools::UUID.random_create.to_s
@@ -622,4 +647,10 @@ module Fluent::Plugin
622
647
  super(message)
623
648
  end
624
649
  end
650
+
651
+ class BlobOperationError < StandardError
652
+ def initialize(message="Default message")
653
+ super(message)
654
+ end
655
+ end
625
656
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-01-04 00:00:00.000000000 Z
11
+ date: 2020-02-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -188,7 +188,7 @@ files:
188
188
  - test/plugin/test_out_azurestorage_gen2.rb
189
189
  homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
190
190
  licenses:
191
- - Apache-2.0
191
+ - MIT License
192
192
  metadata: {}
193
193
  post_install_message:
194
194
  rdoc_options: []