fluent-plugin-azurestorage-gen2 0.1.4 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Makefile +2 -4
- data/README.md +17 -6
- data/VERSION +1 -1
- data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb +1 -0
- data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb +1 -0
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +55 -40
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c631dda137f661c5692855eb46d97696df14a5e0
|
4
|
+
data.tar.gz: 174ace2e26b75f25b12a38d3c76726d3526f0866
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5f135ffbe42f292666e66bca2b84a8c1f49d3ddf727460e376c95658b0382419a9008b110371d48314ccef6af0dc2f3cff5b0ae64730ae29222571dd20333163
|
7
|
+
data.tar.gz: e81ba7ac31051ac5813ab4f3c845ee866f7dce3061c16a72fc5f3164e4b53c9c345c1135820762671162f092b195191a93ff6650a2562fc61cb12ea9a4b97ff2
|
data/Makefile
CHANGED
@@ -8,8 +8,6 @@ install: gem
|
|
8
8
|
push: gem
|
9
9
|
gem push fluent-plugin-azurestorage-gen2*.gem
|
10
10
|
|
11
|
-
tag
|
11
|
+
tag:
|
12
12
|
git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
|
13
|
-
git
|
14
|
-
git push origin "v$$(cat VERSION)"
|
15
|
-
git push -u origin "release/$$(cat VERSION)"
|
13
|
+
git push origin "v$$(cat VERSION)"
|
data/README.md
CHANGED
@@ -9,7 +9,7 @@
|
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.1.
|
12
|
+
| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -34,9 +34,11 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
34
34
|
azure_object_key_format %{path}-%{index}.%{file_extension}
|
35
35
|
azure_oauth_refresh_interval 3600
|
36
36
|
time_slice_format %Y%m%d-%H
|
37
|
-
file_extension log
|
37
|
+
file_extension log # only used with store_as none
|
38
38
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
39
39
|
auto_create_container true
|
40
|
+
store_as gzip
|
41
|
+
format single_value
|
40
42
|
<buffer tag,time>
|
41
43
|
@type file
|
42
44
|
path /var/log/fluent/azurestorage-buffer
|
@@ -60,9 +62,10 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
60
62
|
azure_oauth_secret <my client secret>
|
61
63
|
azure_oauth_refresh_interval 3600
|
62
64
|
time_slice_format %Y%m%d-%H
|
63
|
-
file_extension log
|
64
65
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
66
|
+
store_as gzip
|
65
67
|
auto_create_container true
|
68
|
+
format single_value
|
66
69
|
<buffer tag,time>
|
67
70
|
@type file
|
68
71
|
path /var/log/fluent/azurestorage-buffer
|
@@ -103,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
|
|
103
106
|
|
104
107
|
### azure_oauth_refresh_interval
|
105
108
|
|
106
|
-
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
109
|
+
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
|
107
110
|
|
108
111
|
### azure_oauth_use_azure_cli (Preview)
|
109
112
|
|
@@ -117,6 +120,14 @@ Azure Storage Container name
|
|
117
120
|
|
118
121
|
This plugin create container if not exist when you set 'auto_create_container' to true.
|
119
122
|
|
123
|
+
### skip_container_check
|
124
|
+
|
125
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
|
126
|
+
|
127
|
+
### enable_retry
|
128
|
+
|
129
|
+
If you set this option, operations can be retried in the buffer. Default value is false. (Currently only flush command is supported)
|
130
|
+
|
120
131
|
### azure_object_key_format
|
121
132
|
|
122
133
|
The format of Azure Storage object keys. You can use several built-in variables:
|
@@ -169,13 +180,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
|
|
169
180
|
|
170
181
|
### file_extension
|
171
182
|
|
172
|
-
File extension for the uploaded files. Only
|
183
|
+
File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
|
173
184
|
|
174
185
|
### store_as
|
175
186
|
|
176
187
|
Archive format on Azure Storage. You can use following types:
|
177
188
|
|
178
|
-
- none (default - no tmp file creation for log processing)
|
189
|
+
- none (default - no tmp file creation for log processing, use with json or single value format)
|
179
190
|
- gzip
|
180
191
|
- json
|
181
192
|
- text
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.2
|
@@ -27,16 +27,17 @@ module Fluent::Plugin
|
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
29
|
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
30
|
-
config_param :azure_oauth_refresh_interval, :integer, :default => 60 *
|
30
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
|
31
31
|
config_param :azure_container, :string, :default => nil
|
32
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
|
+
config_param :skip_container_check, :bool, :default => false
|
37
|
+
config_param :enable_retry, :bool, :default => false
|
36
38
|
config_param :format, :string, :default => "out_file"
|
37
39
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
38
40
|
config_param :command_parameter, :string, :default => nil
|
39
|
-
config_param :message_field, :string, :default => nil
|
40
41
|
|
41
42
|
DEFAULT_FORMAT_TYPE = "out_file"
|
42
43
|
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
@@ -57,13 +58,17 @@ module Fluent::Plugin
|
|
57
58
|
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
58
59
|
super
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
61
|
+
if @store_as.nil? || @store_as == "none"
|
62
|
+
log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
|
63
|
+
else
|
64
|
+
begin
|
65
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
66
|
+
rescue => e
|
67
|
+
log.warn "#{@store_as} not found. Use 'text' instead"
|
68
|
+
@compressor = TextCompressor.new
|
69
|
+
end
|
70
|
+
@compressor.configure(conf)
|
65
71
|
end
|
66
|
-
@compressor.configure(conf)
|
67
72
|
|
68
73
|
@formatter = formatter_create
|
69
74
|
|
@@ -84,11 +89,10 @@ module Fluent::Plugin
|
|
84
89
|
@azure_storage_path = ''
|
85
90
|
@last_azure_storage_path = ''
|
86
91
|
@current_index = 0
|
92
|
+
|
87
93
|
if @store_as.nil? || @store_as == "none"
|
88
|
-
@blob_content_type = "text/plain"
|
89
94
|
@final_file_extension = @file_extension
|
90
95
|
else
|
91
|
-
@blob_content_type = @compressor.content_type
|
92
96
|
@final_file_extension = @compressor.ext
|
93
97
|
end
|
94
98
|
|
@@ -100,48 +104,49 @@ module Fluent::Plugin
|
|
100
104
|
|
101
105
|
def start
|
102
106
|
setup_access_token
|
103
|
-
|
107
|
+
if !@skip_container_check
|
108
|
+
ensure_container
|
109
|
+
end
|
104
110
|
super
|
105
111
|
end
|
106
112
|
|
113
|
+
def format(tag, time, record)
|
114
|
+
r = inject_values_to_record(tag, time, record)
|
115
|
+
@formatter.format(tag, time, r)
|
116
|
+
end
|
117
|
+
|
107
118
|
def write(chunk)
|
108
119
|
metadata = chunk.metadata
|
109
120
|
if @store_as.nil? || @store_as == "none"
|
110
|
-
raw_data=''
|
111
121
|
generate_log_name(metadata, @current_index)
|
112
122
|
if @last_azure_storage_path != @azure_storage_path
|
113
123
|
@current_index = 0
|
114
124
|
generate_log_name(metadata, @current_index)
|
115
125
|
end
|
116
|
-
chunk.
|
117
|
-
if @message_field.nil? || @message_field.empty?
|
118
|
-
raw_data << "#{Yajl.dump(record)}\n"
|
119
|
-
elsif record.key?(@message_field)
|
120
|
-
line = record[@message_field].chomp
|
121
|
-
raw_data << "#{line}\n"
|
122
|
-
end
|
123
|
-
end
|
124
|
-
raw_data = raw_data.chomp
|
126
|
+
raw_data = chunk.read
|
125
127
|
unless raw_data.empty?
|
128
|
+
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
126
129
|
upload_blob(raw_data, metadata)
|
127
130
|
end
|
131
|
+
chunk.close rescue nil
|
128
132
|
@last_azure_storage_path = @azure_storage_path
|
129
133
|
else
|
130
134
|
tmp = Tempfile.new("azure-")
|
135
|
+
tmp.binmode
|
131
136
|
begin
|
132
137
|
@compressor.compress(chunk, tmp)
|
133
|
-
tmp.
|
138
|
+
tmp.rewind
|
134
139
|
generate_log_name(metadata, @current_index)
|
135
140
|
if @last_azure_storage_path != @azure_storage_path
|
136
141
|
@current_index = 0
|
137
142
|
generate_log_name(metadata, @current_index)
|
138
143
|
end
|
139
|
-
log.debug "Start uploading temp file: #{tmp.path}"
|
144
|
+
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
|
140
145
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
141
146
|
upload_blob(content, metadata)
|
142
147
|
@last_azure_storage_path = @azure_storage_path
|
143
148
|
ensure
|
144
|
-
tmp.
|
149
|
+
tmp.close(true) rescue nil
|
145
150
|
end
|
146
151
|
end
|
147
152
|
|
@@ -149,7 +154,7 @@ module Fluent::Plugin
|
|
149
154
|
|
150
155
|
private
|
151
156
|
def upload_blob(content, metadata)
|
152
|
-
log.debug "azurestorage_gen2:
|
157
|
+
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
153
158
|
existing_content_length = get_blob_properties(@azure_storage_path)
|
154
159
|
if existing_content_length == 0
|
155
160
|
create_blob(@azure_storage_path)
|
@@ -157,11 +162,6 @@ module Fluent::Plugin
|
|
157
162
|
append_blob(content, metadata, existing_content_length)
|
158
163
|
end
|
159
164
|
|
160
|
-
def format(tag, time, record)
|
161
|
-
r = inject_values_to_record(tag, time, record)
|
162
|
-
@formatter.format(tag, time, r)
|
163
|
-
end
|
164
|
-
|
165
165
|
private
|
166
166
|
def generate_log_name(metadata, index)
|
167
167
|
time_slice = if metadata.timekey.nil?
|
@@ -229,7 +229,7 @@ module Fluent::Plugin
|
|
229
229
|
if response.success?
|
230
230
|
data = JSON.parse(response.body)
|
231
231
|
log.debug "azurestorage_gen2: Token response: #{data}"
|
232
|
-
@azure_access_token = data["access_token"]
|
232
|
+
@azure_access_token = data["access_token"].chomp
|
233
233
|
else
|
234
234
|
raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
|
235
235
|
end
|
@@ -242,12 +242,12 @@ module Fluent::Plugin
|
|
242
242
|
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
|
243
243
|
headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
|
244
244
|
content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
|
245
|
-
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers
|
245
|
+
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
|
246
246
|
request.on_complete do |response|
|
247
247
|
if response.success?
|
248
248
|
data = JSON.parse(response.body)
|
249
249
|
log.debug "azurestorage_gen2: Token response: #{data}"
|
250
|
-
@azure_access_token = data["access_token"]
|
250
|
+
@azure_access_token = data["access_token"].chomp
|
251
251
|
else
|
252
252
|
raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
|
253
253
|
end
|
@@ -259,7 +259,7 @@ module Fluent::Plugin
|
|
259
259
|
def acquire_access_token_by_az
|
260
260
|
access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
|
261
261
|
log.debug "azurestorage_gen2: Token response: #{access_token}"
|
262
|
-
@azure_access_token = access_token
|
262
|
+
@azure_access_token = access_token.chomp
|
263
263
|
end
|
264
264
|
|
265
265
|
private
|
@@ -301,7 +301,7 @@ module Fluent::Plugin
|
|
301
301
|
if response.success?
|
302
302
|
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
303
303
|
elsif response.timed_out?
|
304
|
-
raise Fluent::UnrecoverableError,
|
304
|
+
raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
|
305
305
|
else
|
306
306
|
raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
307
307
|
end
|
@@ -312,7 +312,7 @@ module Fluent::Plugin
|
|
312
312
|
private
|
313
313
|
def create_blob(blob_path)
|
314
314
|
datestamp = create_request_date
|
315
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "
|
315
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "text/plain"}
|
316
316
|
params = {:resource => "file", :recursive => "false"}
|
317
317
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
318
318
|
headers[:Authorization] = auth_header
|
@@ -333,13 +333,13 @@ module Fluent::Plugin
|
|
333
333
|
|
334
334
|
private
|
335
335
|
def append_blob_block(blob_path, content, position)
|
336
|
-
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
336
|
+
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
|
337
337
|
datestamp = create_request_date
|
338
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"
|
338
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
|
339
339
|
params = {:action => "append", :position => "#{position}"}
|
340
340
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
341
341
|
headers[:Authorization] = auth_header
|
342
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :
|
342
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
|
343
343
|
request.on_complete do |response|
|
344
344
|
if response.success?
|
345
345
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
@@ -371,7 +371,7 @@ module Fluent::Plugin
|
|
371
371
|
elsif response.timed_out?
|
372
372
|
raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
|
373
373
|
else
|
374
|
-
|
374
|
+
raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
375
375
|
end
|
376
376
|
end
|
377
377
|
request.run
|
@@ -509,6 +509,15 @@ module Fluent::Plugin
|
|
509
509
|
Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
|
510
510
|
end
|
511
511
|
|
512
|
+
private
|
513
|
+
def raise_error(error_message)
|
514
|
+
if @enable_retry
|
515
|
+
raise BlobOperationError, error_message
|
516
|
+
else
|
517
|
+
raise Fluent::UnrecoverableError, error_message
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
512
521
|
def uuid_random
|
513
522
|
require 'uuidtools'
|
514
523
|
::UUIDTools::UUID.random_create.to_s
|
@@ -625,4 +634,10 @@ module Fluent::Plugin
|
|
625
634
|
super(message)
|
626
635
|
end
|
627
636
|
end
|
637
|
+
|
638
|
+
class BlobOperationError < StandardError
|
639
|
+
def initialize(message="Default message")
|
640
|
+
super(message)
|
641
|
+
end
|
642
|
+
end
|
628
643
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|