fluent-plugin-azurestorage-gen2 0.1.6 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Makefile +2 -4
- data/README.md +18 -7
- data/VERSION +1 -1
- data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb +1 -0
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +42 -32
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 04eea9a39f7cc1d5533d895d6efd3aa725cd9c44
|
4
|
+
data.tar.gz: 0c3779b86a040e07924c0e7d4f78a01cee9ac274
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0ec050faad375da7b4b13b5686de44b9c9c7f24cef7bedc1f647392620a3dff379f2f409b475d072947215e50ff6ae6d3a976320467a0d41eb579883dc0ae79f
|
7
|
+
data.tar.gz: c71a5e917920770e4339bab5e208c51b53c9cfcd9a4afe7255996149da0b70db539d6ca12ba983c1147b9611ed8cb342831ce217c7a4adfde93ac3232a2225fb
|
data/Makefile
CHANGED
@@ -8,8 +8,6 @@ install: gem
|
|
8
8
|
push: gem
|
9
9
|
gem push fluent-plugin-azurestorage-gen2*.gem
|
10
10
|
|
11
|
-
tag
|
11
|
+
tag:
|
12
12
|
git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
|
13
|
-
git
|
14
|
-
git push origin "v$$(cat VERSION)"
|
15
|
-
git push -u origin "release/$$(cat VERSION)"
|
13
|
+
git push origin "v$$(cat VERSION)"
|
data/README.md
CHANGED
@@ -3,13 +3,13 @@
|
|
3
3
|
[](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
5
5
|
[](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
|
6
|
-

|
6
|
+

|
7
7
|
|
8
8
|
## Requirements
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.1.
|
12
|
+
| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -34,9 +34,11 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
34
34
|
azure_object_key_format %{path}-%{index}.%{file_extension}
|
35
35
|
azure_oauth_refresh_interval 3600
|
36
36
|
time_slice_format %Y%m%d-%H
|
37
|
-
file_extension log
|
37
|
+
file_extension log # only used with store_as none
|
38
38
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
39
39
|
auto_create_container true
|
40
|
+
store_as gzip
|
41
|
+
format single_value
|
40
42
|
<buffer tag,time>
|
41
43
|
@type file
|
42
44
|
path /var/log/fluent/azurestorage-buffer
|
@@ -60,9 +62,10 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
60
62
|
azure_oauth_secret <my client secret>
|
61
63
|
azure_oauth_refresh_interval 3600
|
62
64
|
time_slice_format %Y%m%d-%H
|
63
|
-
file_extension log
|
64
65
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
66
|
+
store_as gzip
|
65
67
|
auto_create_container true
|
68
|
+
format single_value
|
66
69
|
<buffer tag,time>
|
67
70
|
@type file
|
68
71
|
path /var/log/fluent/azurestorage-buffer
|
@@ -119,7 +122,15 @@ This plugin create container if not exist when you set 'auto_create_container' t
|
|
119
122
|
|
120
123
|
### skip_container_check
|
121
124
|
|
122
|
-
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform
|
125
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
|
126
|
+
|
127
|
+
### enable_retry
|
128
|
+
|
129
|
+
If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
|
130
|
+
|
131
|
+
### url_domain_suffix
|
132
|
+
|
133
|
+
The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
|
123
134
|
|
124
135
|
### azure_object_key_format
|
125
136
|
|
@@ -173,13 +184,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
|
|
173
184
|
|
174
185
|
### file_extension
|
175
186
|
|
176
|
-
File extension for the uploaded files. Only
|
187
|
+
File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
|
177
188
|
|
178
189
|
### store_as
|
179
190
|
|
180
191
|
Archive format on Azure Storage. You can use following types:
|
181
192
|
|
182
|
-
- none (default - no tmp file creation for log processing)
|
193
|
+
- none (default - no tmp file creation for log processing, use with json or single value format)
|
183
194
|
- gzip
|
184
195
|
- json
|
185
196
|
- text
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.4
|
@@ -27,20 +27,20 @@ module Fluent::Plugin
|
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
29
|
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
30
|
-
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
|
30
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
|
31
31
|
config_param :azure_container, :string, :default => nil
|
32
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
36
|
config_param :skip_container_check, :bool, :default => false
|
37
|
+
config_param :enable_retry, :bool, :default => false
|
38
|
+
config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
|
37
39
|
config_param :format, :string, :default => "out_file"
|
38
40
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
39
41
|
config_param :command_parameter, :string, :default => nil
|
40
|
-
config_param :message_field, :string, :default => nil
|
41
42
|
|
42
43
|
DEFAULT_FORMAT_TYPE = "out_file"
|
43
|
-
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
44
44
|
ACCESS_TOKEN_API_VERSION = "2018-02-01"
|
45
45
|
ABFS_API_VERSION = "2018-11-09"
|
46
46
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
@@ -118,40 +118,35 @@ module Fluent::Plugin
|
|
118
118
|
def write(chunk)
|
119
119
|
metadata = chunk.metadata
|
120
120
|
if @store_as.nil? || @store_as == "none"
|
121
|
-
raw_data=''
|
122
121
|
generate_log_name(metadata, @current_index)
|
123
122
|
if @last_azure_storage_path != @azure_storage_path
|
124
123
|
@current_index = 0
|
125
124
|
generate_log_name(metadata, @current_index)
|
126
125
|
end
|
127
|
-
chunk.
|
128
|
-
if @message_field.nil? || @message_field.empty?
|
129
|
-
raw_data << "#{Yajl.dump(record)}\n"
|
130
|
-
elsif record.key?(@message_field)
|
131
|
-
line = record[@message_field].chomp
|
132
|
-
raw_data << "#{line}\n"
|
133
|
-
end
|
134
|
-
end
|
126
|
+
raw_data = chunk.read
|
135
127
|
unless raw_data.empty?
|
128
|
+
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
136
129
|
upload_blob(raw_data, metadata)
|
137
130
|
end
|
131
|
+
chunk.close rescue nil
|
138
132
|
@last_azure_storage_path = @azure_storage_path
|
139
133
|
else
|
140
134
|
tmp = Tempfile.new("azure-")
|
135
|
+
tmp.binmode
|
141
136
|
begin
|
142
137
|
@compressor.compress(chunk, tmp)
|
143
|
-
tmp.
|
138
|
+
tmp.rewind
|
144
139
|
generate_log_name(metadata, @current_index)
|
145
140
|
if @last_azure_storage_path != @azure_storage_path
|
146
141
|
@current_index = 0
|
147
142
|
generate_log_name(metadata, @current_index)
|
148
143
|
end
|
149
|
-
log.debug "Start uploading temp file: #{tmp.path}"
|
144
|
+
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
|
150
145
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
151
146
|
upload_blob(content, metadata)
|
152
147
|
@last_azure_storage_path = @azure_storage_path
|
153
148
|
ensure
|
154
|
-
tmp.
|
149
|
+
tmp.close(true) rescue nil
|
155
150
|
end
|
156
151
|
end
|
157
152
|
|
@@ -159,7 +154,7 @@ module Fluent::Plugin
|
|
159
154
|
|
160
155
|
private
|
161
156
|
def upload_blob(content, metadata)
|
162
|
-
log.debug "azurestorage_gen2:
|
157
|
+
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
163
158
|
existing_content_length = get_blob_properties(@azure_storage_path)
|
164
159
|
if existing_content_length == 0
|
165
160
|
create_blob(@azure_storage_path)
|
@@ -274,7 +269,7 @@ module Fluent::Plugin
|
|
274
269
|
params = {:resource => "filesystem" }
|
275
270
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
|
276
271
|
headers[:Authorization] = auth_header
|
277
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
272
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
|
278
273
|
request.on_complete do |response|
|
279
274
|
if response.success?
|
280
275
|
log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
|
@@ -301,12 +296,12 @@ module Fluent::Plugin
|
|
301
296
|
params = {:resource => "filesystem" }
|
302
297
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
|
303
298
|
headers[:Authorization] = auth_header
|
304
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
299
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
|
305
300
|
request.on_complete do |response|
|
306
301
|
if response.success?
|
307
302
|
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
308
303
|
elsif response.timed_out?
|
309
|
-
raise Fluent::UnrecoverableError,
|
304
|
+
raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
|
310
305
|
else
|
311
306
|
raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
312
307
|
end
|
@@ -321,16 +316,16 @@ module Fluent::Plugin
|
|
321
316
|
params = {:resource => "file", :recursive => "false"}
|
322
317
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
323
318
|
headers[:Authorization] = auth_header
|
324
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
319
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
|
325
320
|
request.on_complete do |response|
|
326
321
|
if response.success?
|
327
322
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
|
328
323
|
elsif response.timed_out?
|
329
|
-
|
324
|
+
raise_error "Creating blob '#{blob_path}' request timed out."
|
330
325
|
elsif response.code == 409
|
331
326
|
log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
|
332
327
|
else
|
333
|
-
|
328
|
+
raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
334
329
|
end
|
335
330
|
end
|
336
331
|
request.run
|
@@ -338,24 +333,24 @@ module Fluent::Plugin
|
|
338
333
|
|
339
334
|
private
|
340
335
|
def append_blob_block(blob_path, content, position)
|
341
|
-
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
336
|
+
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
|
342
337
|
datestamp = create_request_date
|
343
338
|
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
|
344
339
|
params = {:action => "append", :position => "#{position}"}
|
345
340
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
346
341
|
headers[:Authorization] = auth_header
|
347
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
342
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
|
348
343
|
request.on_complete do |response|
|
349
344
|
if response.success?
|
350
345
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
351
346
|
elsif response.timed_out?
|
352
|
-
|
347
|
+
raise_error "Appending blob #{blob_path}' request timed out."
|
353
348
|
elsif response.code == 404
|
354
349
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
|
355
350
|
elsif response.code == 409
|
356
351
|
raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
|
357
352
|
else
|
358
|
-
|
353
|
+
raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
359
354
|
end
|
360
355
|
end
|
361
356
|
request.run
|
@@ -369,14 +364,14 @@ module Fluent::Plugin
|
|
369
364
|
params = {:action => "flush", :position => "#{position}"}
|
370
365
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
|
371
366
|
headers[:Authorization] = auth_header
|
372
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
367
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
|
373
368
|
request.on_complete do |response|
|
374
369
|
if response.success?
|
375
370
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
|
376
371
|
elsif response.timed_out?
|
377
|
-
|
372
|
+
raise_error "Bloub '#{blob_path}' flush request timed out."
|
378
373
|
else
|
379
|
-
|
374
|
+
raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
380
375
|
end
|
381
376
|
end
|
382
377
|
request.run
|
@@ -390,18 +385,18 @@ module Fluent::Plugin
|
|
390
385
|
content_length = -1
|
391
386
|
auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
392
387
|
headers[:Authorization] = auth_header
|
393
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{
|
388
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
|
394
389
|
request.on_complete do |response|
|
395
390
|
if response.success?
|
396
391
|
log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
|
397
392
|
content_length = response.headers['Content-Length'].to_i
|
398
393
|
elsif response.timed_out?
|
399
|
-
|
394
|
+
raise_error "Get blob properties '#{blob_path}' request timed out."
|
400
395
|
elsif response.code == 404
|
401
396
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
|
402
397
|
content_length = 0
|
403
398
|
else
|
404
|
-
|
399
|
+
raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
405
400
|
end
|
406
401
|
end
|
407
402
|
request.run
|
@@ -514,6 +509,15 @@ module Fluent::Plugin
|
|
514
509
|
Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
|
515
510
|
end
|
516
511
|
|
512
|
+
private
|
513
|
+
def raise_error(error_message)
|
514
|
+
if @enable_retry
|
515
|
+
raise BlobOperationError, error_message
|
516
|
+
else
|
517
|
+
raise Fluent::UnrecoverableError, error_message
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
517
521
|
def uuid_random
|
518
522
|
require 'uuidtools'
|
519
523
|
::UUIDTools::UUID.random_create.to_s
|
@@ -630,4 +634,10 @@ module Fluent::Plugin
|
|
630
634
|
super(message)
|
631
635
|
end
|
632
636
|
end
|
637
|
+
|
638
|
+
class BlobOperationError < StandardError
|
639
|
+
def initialize(message="Default message")
|
640
|
+
super(message)
|
641
|
+
end
|
642
|
+
end
|
633
643
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-01-
|
11
|
+
date: 2020-01-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|