fluent-plugin-azurestorage-gen2 0.2.0 → 0.2.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Makefile +2 -4
- data/README.md +19 -6
- data/VERSION +1 -1
- data/fluent-plugin-azurestorage-gen2.gemspec +1 -1
- data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb +1 -0
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +55 -24
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 02170d63c314b7bb91fb1722b0ca1a4dccad9bd0
|
4
|
+
data.tar.gz: 7fc221dc0be643681800be67c954634caa7df518
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ad9f64e7f2af1d65177e683aa808df6187d9ace11a0f976a925a11d31e74db15c324b10035d4610ff387b5f2b3a1939c4f8c2cf5f7f25e4a237c16350b4ec620
|
7
|
+
data.tar.gz: eeff3b1a4c7ea0b48e207c270e1679ba770c2bca21146f879bfb893a1fca314419c5a7a6c35a1437ca4967dbd87a0d703c81b66a16d1326e187da90e2fcf1ac1
|
data/Makefile
CHANGED
@@ -8,8 +8,6 @@ install: gem
|
|
8
8
|
push: gem
|
9
9
|
gem push fluent-plugin-azurestorage-gen2*.gem
|
10
10
|
|
11
|
-
tag
|
11
|
+
tag:
|
12
12
|
git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
|
13
|
-
git
|
14
|
-
git push origin "v$$(cat VERSION)"
|
15
|
-
git push -u origin "release/$$(cat VERSION)"
|
13
|
+
git push origin "v$$(cat VERSION)"
|
data/README.md
CHANGED
@@ -3,13 +3,13 @@
|
|
3
3
|
[![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
|
4
4
|
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
|
5
5
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
|
6
|
-
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
|
6
|
+
![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
|
7
7
|
|
8
8
|
## Requirements
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.
|
12
|
+
| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -34,9 +34,10 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
34
34
|
azure_object_key_format %{path}-%{index}.%{file_extension}
|
35
35
|
azure_oauth_refresh_interval 3600
|
36
36
|
time_slice_format %Y%m%d-%H
|
37
|
-
file_extension log
|
37
|
+
file_extension log # only used with store_as none
|
38
38
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
39
39
|
auto_create_container true
|
40
|
+
store_as gzip
|
40
41
|
format single_value
|
41
42
|
<buffer tag,time>
|
42
43
|
@type file
|
@@ -61,8 +62,8 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
61
62
|
azure_oauth_secret <my client secret>
|
62
63
|
azure_oauth_refresh_interval 3600
|
63
64
|
time_slice_format %Y%m%d-%H
|
64
|
-
file_extension log
|
65
65
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
66
|
+
store_as gzip
|
66
67
|
auto_create_container true
|
67
68
|
format single_value
|
68
69
|
<buffer tag,time>
|
@@ -105,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
|
|
105
106
|
|
106
107
|
### azure_oauth_refresh_interval
|
107
108
|
|
108
|
-
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
109
|
+
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
109
110
|
|
110
111
|
### azure_oauth_use_azure_cli (Preview)
|
111
112
|
|
@@ -121,7 +122,19 @@ This plugin create container if not exist when you set 'auto_create_container' t
|
|
121
122
|
|
122
123
|
### skip_container_check
|
123
124
|
|
124
|
-
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform
|
125
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
|
126
|
+
|
127
|
+
### enable_retry
|
128
|
+
|
129
|
+
If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
|
130
|
+
|
131
|
+
### startup_fail_on_error
|
132
|
+
|
133
|
+
If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
|
134
|
+
|
135
|
+
### url_domain_suffix
|
136
|
+
|
137
|
+
The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
|
125
138
|
|
126
139
|
### azure_object_key_format
|
127
140
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.5
|
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-azurestorage-gen2"
|
6
6
|
gem.description = "Azure Storage output plugin for Fluentd event collector"
|
7
|
-
gem.license = "
|
7
|
+
gem.license = "MIT License"
|
8
8
|
gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
|
9
9
|
gem.summary = gem.description
|
10
10
|
gem.version = File.read("VERSION").strip
|
@@ -27,19 +27,21 @@ module Fluent::Plugin
|
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
29
|
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
30
|
-
config_param :azure_oauth_refresh_interval, :integer, :default => 60 *
|
30
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
|
31
31
|
config_param :azure_container, :string, :default => nil
|
32
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
36
|
config_param :skip_container_check, :bool, :default => false
|
37
|
+
config_param :enable_retry, :bool, :default => false
|
38
|
+
config_param :startup_fail_on_error, :bool, :default => true
|
39
|
+
config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
|
37
40
|
config_param :format, :string, :default => "out_file"
|
38
41
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
39
42
|
config_param :command_parameter, :string, :default => nil
|
40
43
|
|
41
44
|
DEFAULT_FORMAT_TYPE = "out_file"
|
42
|
-
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
43
45
|
ACCESS_TOKEN_API_VERSION = "2018-02-01"
|
44
46
|
ABFS_API_VERSION = "2018-11-09"
|
45
47
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
@@ -127,23 +129,25 @@ module Fluent::Plugin
|
|
127
129
|
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
128
130
|
upload_blob(raw_data, metadata)
|
129
131
|
end
|
132
|
+
chunk.close rescue nil
|
130
133
|
@last_azure_storage_path = @azure_storage_path
|
131
134
|
else
|
132
135
|
tmp = Tempfile.new("azure-")
|
136
|
+
tmp.binmode
|
133
137
|
begin
|
134
138
|
@compressor.compress(chunk, tmp)
|
135
|
-
tmp.
|
139
|
+
tmp.rewind
|
136
140
|
generate_log_name(metadata, @current_index)
|
137
141
|
if @last_azure_storage_path != @azure_storage_path
|
138
142
|
@current_index = 0
|
139
143
|
generate_log_name(metadata, @current_index)
|
140
144
|
end
|
141
|
-
log.debug "Start uploading temp file: #{tmp.path}"
|
145
|
+
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
|
142
146
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
143
147
|
upload_blob(content, metadata)
|
144
148
|
@last_azure_storage_path = @azure_storage_path
|
145
149
|
ensure
|
146
|
-
tmp.
|
150
|
+
tmp.close(true) rescue nil
|
147
151
|
end
|
148
152
|
end
|
149
153
|
|
@@ -151,7 +155,7 @@ module Fluent::Plugin
|
|
151
155
|
|
152
156
|
private
|
153
157
|
def upload_blob(content, metadata)
|
154
|
-
log.debug "azurestorage_gen2:
|
158
|
+
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
155
159
|
existing_content_length = get_blob_properties(@azure_storage_path)
|
156
160
|
if existing_content_length == 0
|
157
161
|
create_blob(@azure_storage_path)
|
@@ -183,7 +187,19 @@ module Fluent::Plugin
|
|
183
187
|
def setup_access_token
|
184
188
|
if @azure_storage_access_key.nil?
|
185
189
|
@get_token_lock = Concurrent::ReadWriteLock.new
|
186
|
-
|
190
|
+
if @startup_fail_on_error
|
191
|
+
acquire_access_token
|
192
|
+
else
|
193
|
+
while true
|
194
|
+
begin
|
195
|
+
acquire_access_token
|
196
|
+
break
|
197
|
+
rescue Exception => e
|
198
|
+
log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
|
199
|
+
sleep 20
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
187
203
|
if @azure_oauth_refresh_interval > 0
|
188
204
|
log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
|
189
205
|
@get_token_task = Concurrent::TimerTask.new(
|
@@ -217,7 +233,7 @@ module Fluent::Plugin
|
|
217
233
|
# https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
|
218
234
|
private
|
219
235
|
def acquire_access_token_msi
|
220
|
-
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.
|
236
|
+
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azures.com/" }
|
221
237
|
unless @azure_instance_msi.nil?
|
222
238
|
params[:msi_res_id] = @azure_instance_msi
|
223
239
|
end
|
@@ -266,7 +282,7 @@ module Fluent::Plugin
|
|
266
282
|
params = {:resource => "filesystem" }
|
267
283
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
|
268
284
|
headers[:Authorization] = auth_header
|
269
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
285
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
|
270
286
|
request.on_complete do |response|
|
271
287
|
if response.success?
|
272
288
|
log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
|
@@ -293,12 +309,12 @@ module Fluent::Plugin
|
|
293
309
|
params = {:resource => "filesystem" }
|
294
310
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
|
295
311
|
headers[:Authorization] = auth_header
|
296
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
312
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
|
297
313
|
request.on_complete do |response|
|
298
314
|
if response.success?
|
299
315
|
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
300
316
|
elsif response.timed_out?
|
301
|
-
raise Fluent::UnrecoverableError,
|
317
|
+
raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
|
302
318
|
else
|
303
319
|
raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
304
320
|
end
|
@@ -313,16 +329,16 @@ module Fluent::Plugin
|
|
313
329
|
params = {:resource => "file", :recursive => "false"}
|
314
330
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
315
331
|
headers[:Authorization] = auth_header
|
316
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
332
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
|
317
333
|
request.on_complete do |response|
|
318
334
|
if response.success?
|
319
335
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
|
320
336
|
elsif response.timed_out?
|
321
|
-
|
337
|
+
raise_error "Creating blob '#{blob_path}' request timed out."
|
322
338
|
elsif response.code == 409
|
323
339
|
log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
|
324
340
|
else
|
325
|
-
|
341
|
+
raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
326
342
|
end
|
327
343
|
end
|
328
344
|
request.run
|
@@ -330,24 +346,24 @@ module Fluent::Plugin
|
|
330
346
|
|
331
347
|
private
|
332
348
|
def append_blob_block(blob_path, content, position)
|
333
|
-
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
349
|
+
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
|
334
350
|
datestamp = create_request_date
|
335
351
|
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
|
336
352
|
params = {:action => "append", :position => "#{position}"}
|
337
353
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
338
354
|
headers[:Authorization] = auth_header
|
339
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
355
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
|
340
356
|
request.on_complete do |response|
|
341
357
|
if response.success?
|
342
358
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
343
359
|
elsif response.timed_out?
|
344
|
-
|
360
|
+
raise_error "Appending blob #{blob_path}' request timed out."
|
345
361
|
elsif response.code == 404
|
346
362
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
|
347
363
|
elsif response.code == 409
|
348
364
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
|
349
365
|
else
|
350
|
-
|
366
|
+
raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
351
367
|
end
|
352
368
|
end
|
353
369
|
request.run
|
@@ -361,14 +377,14 @@ module Fluent::Plugin
|
|
361
377
|
params = {:action => "flush", :position => "#{position}"}
|
362
378
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
|
363
379
|
headers[:Authorization] = auth_header
|
364
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
380
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
|
365
381
|
request.on_complete do |response|
|
366
382
|
if response.success?
|
367
383
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
|
368
384
|
elsif response.timed_out?
|
369
|
-
|
385
|
+
raise_error "Bloub '#{blob_path}' flush request timed out."
|
370
386
|
else
|
371
|
-
|
387
|
+
raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
372
388
|
end
|
373
389
|
end
|
374
390
|
request.run
|
@@ -382,18 +398,18 @@ module Fluent::Plugin
|
|
382
398
|
content_length = -1
|
383
399
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
384
400
|
headers[:Authorization] = auth_header
|
385
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
401
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
|
386
402
|
request.on_complete do |response|
|
387
403
|
if response.success?
|
388
404
|
log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
|
389
405
|
content_length = response.headers['Content-Length'].to_i
|
390
406
|
elsif response.timed_out?
|
391
|
-
|
407
|
+
raise_error "Get blob properties '#{blob_path}' request timed out."
|
392
408
|
elsif response.code == 404
|
393
409
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
|
394
410
|
content_length = 0
|
395
411
|
else
|
396
|
-
|
412
|
+
raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
397
413
|
end
|
398
414
|
end
|
399
415
|
request.run
|
@@ -506,6 +522,15 @@ module Fluent::Plugin
|
|
506
522
|
Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
|
507
523
|
end
|
508
524
|
|
525
|
+
private
|
526
|
+
def raise_error(error_message)
|
527
|
+
if @enable_retry
|
528
|
+
raise BlobOperationError, error_message
|
529
|
+
else
|
530
|
+
raise Fluent::UnrecoverableError, error_message
|
531
|
+
end
|
532
|
+
end
|
533
|
+
|
509
534
|
def uuid_random
|
510
535
|
require 'uuidtools'
|
511
536
|
::UUIDTools::UUID.random_create.to_s
|
@@ -622,4 +647,10 @@ module Fluent::Plugin
|
|
622
647
|
super(message)
|
623
648
|
end
|
624
649
|
end
|
650
|
+
|
651
|
+
class BlobOperationError < StandardError
|
652
|
+
def initialize(message="Default message")
|
653
|
+
super(message)
|
654
|
+
end
|
655
|
+
end
|
625
656
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-02-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -188,7 +188,7 @@ files:
|
|
188
188
|
- test/plugin/test_out_azurestorage_gen2.rb
|
189
189
|
homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
|
190
190
|
licenses:
|
191
|
-
-
|
191
|
+
- MIT License
|
192
192
|
metadata: {}
|
193
193
|
post_install_message:
|
194
194
|
rdoc_options: []
|