fluent-plugin-azurestorage-gen2 0.1.3 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +12 -6
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_azurestorage_gen2.rb +52 -38
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 830b58ff51c9166a04c91f7e4994cc5c867c86e1
|
4
|
+
data.tar.gz: b257f9c0684a7797266920775cea076d9c6f67dc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2512d0e061e46d7820dc6405ea2530eeceb1f65483eb9d31789df8956c4fb1469a43d20a2ebee3b0e496a73143f406331c17aabe812d9a65dae4828729d8c6aa
|
7
|
+
data.tar.gz: 185079c3757ccdbaf51d205984f57791b5d3e20566c65ec8adea0b94f7db6667a56074a34234543afe29e429fb205f7bc2cf681b70e8e803d0992b1db772e65d
|
data/README.md
CHANGED
@@ -9,7 +9,7 @@
|
|
9
9
|
|
10
10
|
| fluent-plugin-azurestorage-gen2 | fluentd | ruby |
|
11
11
|
|------------------------|---------|------|
|
12
|
-
| >= 0.1.
|
12
|
+
| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
|
13
13
|
|
14
14
|
## Overview
|
15
15
|
|
@@ -37,6 +37,7 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
37
37
|
file_extension log
|
38
38
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
39
39
|
auto_create_container true
|
40
|
+
format single_value
|
40
41
|
<buffer tag,time>
|
41
42
|
@type file
|
42
43
|
path /var/log/fluent/azurestorage-buffer
|
@@ -63,6 +64,7 @@ $ gem install fluent-plugin-azurestorage-gen2
|
|
63
64
|
file_extension log
|
64
65
|
path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
|
65
66
|
auto_create_container true
|
67
|
+
format single_value
|
66
68
|
<buffer tag,time>
|
67
69
|
@type file
|
68
70
|
path /var/log/fluent/azurestorage-buffer
|
@@ -89,7 +91,7 @@ Your Azure Storage Access Key(Primary or Secondary). This also can be got from A
|
|
89
91
|
|
90
92
|
Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
|
91
93
|
|
92
|
-
### azure_oauth_tenant_id
|
94
|
+
### azure_oauth_tenant_id (Preview)
|
93
95
|
|
94
96
|
Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
|
95
97
|
|
@@ -101,9 +103,9 @@ OAuth client id that is used for OAuth based authentication. Required if OAuth b
|
|
101
103
|
|
102
104
|
OAuth client secret that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
|
103
105
|
|
104
|
-
### azure_oauth_refresh_interval
|
106
|
+
### azure_oauth_refresh_interval
|
105
107
|
|
106
|
-
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
|
108
|
+
OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
|
107
109
|
|
108
110
|
### azure_oauth_use_azure_cli (Preview)
|
109
111
|
|
@@ -117,6 +119,10 @@ Azure Storage Container name
|
|
117
119
|
|
118
120
|
This plugin create container if not exist when you set 'auto_create_container' to true.
|
119
121
|
|
122
|
+
### skip_container_check
|
123
|
+
|
124
|
+
You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
|
125
|
+
|
120
126
|
### azure_object_key_format
|
121
127
|
|
122
128
|
The format of Azure Storage object keys. You can use several built-in variables:
|
@@ -169,13 +175,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
|
|
169
175
|
|
170
176
|
### file_extension
|
171
177
|
|
172
|
-
File extension for the uploaded files. Only
|
178
|
+
File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
|
173
179
|
|
174
180
|
### store_as
|
175
181
|
|
176
182
|
Archive format on Azure Storage. You can use following types:
|
177
183
|
|
178
|
-
- none (default - no tmp file creation for log processing)
|
184
|
+
- none (default - no tmp file creation for log processing, use with json or single value format)
|
179
185
|
- gzip
|
180
186
|
- json
|
181
187
|
- text
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1
|
1
|
+
0.2.1
|
@@ -27,16 +27,17 @@ module Fluent::Plugin
|
|
27
27
|
config_param :azure_oauth_secret, :string, :default => nil, :secret => true
|
28
28
|
config_param :azure_oauth_tenant_id, :string, :default => nil
|
29
29
|
config_param :azure_oauth_use_azure_cli, :bool, :default => false
|
30
|
-
config_param :azure_oauth_refresh_interval, :integer, :default => 60 *
|
30
|
+
config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
|
31
31
|
config_param :azure_container, :string, :default => nil
|
32
32
|
config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
33
33
|
config_param :file_extension, :string, :default => "log"
|
34
34
|
config_param :store_as, :string, :default => "none"
|
35
35
|
config_param :auto_create_container, :bool, :default => false
|
36
|
+
config_param :skip_container_check, :bool, :default => false
|
37
|
+
config_param :enable_retry, :bool, :default => false
|
36
38
|
config_param :format, :string, :default => "out_file"
|
37
39
|
config_param :time_slice_format, :string, :default => '%Y%m%d'
|
38
40
|
config_param :command_parameter, :string, :default => nil
|
39
|
-
config_param :message_field, :string, :default => nil
|
40
41
|
|
41
42
|
DEFAULT_FORMAT_TYPE = "out_file"
|
42
43
|
URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
|
@@ -57,13 +58,17 @@ module Fluent::Plugin
|
|
57
58
|
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
58
59
|
super
|
59
60
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
61
|
+
if @store_as.nil? || @store_as == "none"
|
62
|
+
log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
|
63
|
+
else
|
64
|
+
begin
|
65
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
66
|
+
rescue => e
|
67
|
+
log.warn "#{@store_as} not found. Use 'text' instead"
|
68
|
+
@compressor = TextCompressor.new
|
69
|
+
end
|
70
|
+
@compressor.configure(conf)
|
65
71
|
end
|
66
|
-
@compressor.configure(conf)
|
67
72
|
|
68
73
|
@formatter = formatter_create
|
69
74
|
|
@@ -84,11 +89,10 @@ module Fluent::Plugin
|
|
84
89
|
@azure_storage_path = ''
|
85
90
|
@last_azure_storage_path = ''
|
86
91
|
@current_index = 0
|
92
|
+
|
87
93
|
if @store_as.nil? || @store_as == "none"
|
88
|
-
@blob_content_type = "text/plain"
|
89
94
|
@final_file_extension = @file_extension
|
90
95
|
else
|
91
|
-
@blob_content_type = @compressor.content_type
|
92
96
|
@final_file_extension = @compressor.ext
|
93
97
|
end
|
94
98
|
|
@@ -100,29 +104,28 @@ module Fluent::Plugin
|
|
100
104
|
|
101
105
|
def start
|
102
106
|
setup_access_token
|
103
|
-
|
107
|
+
if !@skip_container_check
|
108
|
+
ensure_container
|
109
|
+
end
|
104
110
|
super
|
105
111
|
end
|
106
112
|
|
113
|
+
def format(tag, time, record)
|
114
|
+
r = inject_values_to_record(tag, time, record)
|
115
|
+
@formatter.format(tag, time, r)
|
116
|
+
end
|
117
|
+
|
107
118
|
def write(chunk)
|
108
119
|
metadata = chunk.metadata
|
109
120
|
if @store_as.nil? || @store_as == "none"
|
110
|
-
raw_data=''
|
111
121
|
generate_log_name(metadata, @current_index)
|
112
122
|
if @last_azure_storage_path != @azure_storage_path
|
113
123
|
@current_index = 0
|
114
124
|
generate_log_name(metadata, @current_index)
|
115
125
|
end
|
116
|
-
chunk.
|
117
|
-
if @message_field.nil? || @message_field.empty?
|
118
|
-
raw_data << "#{Yajl.dump(record)}\n"
|
119
|
-
elsif record.key?(@message_field)
|
120
|
-
line = record[@message_field].chomp
|
121
|
-
raw_data << "#{line}\n"
|
122
|
-
end
|
123
|
-
end
|
124
|
-
raw_data = raw_data.chomp
|
126
|
+
raw_data = chunk.read
|
125
127
|
unless raw_data.empty?
|
128
|
+
log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
126
129
|
upload_blob(raw_data, metadata)
|
127
130
|
end
|
128
131
|
@last_azure_storage_path = @azure_storage_path
|
@@ -136,7 +139,7 @@ module Fluent::Plugin
|
|
136
139
|
@current_index = 0
|
137
140
|
generate_log_name(metadata, @current_index)
|
138
141
|
end
|
139
|
-
log.debug "Start uploading temp file: #{tmp.path}"
|
142
|
+
log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
|
140
143
|
content = File.open(tmp.path, 'rb') { |file| file.read }
|
141
144
|
upload_blob(content, metadata)
|
142
145
|
@last_azure_storage_path = @azure_storage_path
|
@@ -149,7 +152,7 @@ module Fluent::Plugin
|
|
149
152
|
|
150
153
|
private
|
151
154
|
def upload_blob(content, metadata)
|
152
|
-
log.debug "azurestorage_gen2:
|
155
|
+
log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
|
153
156
|
existing_content_length = get_blob_properties(@azure_storage_path)
|
154
157
|
if existing_content_length == 0
|
155
158
|
create_blob(@azure_storage_path)
|
@@ -157,11 +160,6 @@ module Fluent::Plugin
|
|
157
160
|
append_blob(content, metadata, existing_content_length)
|
158
161
|
end
|
159
162
|
|
160
|
-
def format(tag, time, record)
|
161
|
-
r = inject_values_to_record(tag, time, record)
|
162
|
-
@formatter.format(tag, time, r)
|
163
|
-
end
|
164
|
-
|
165
163
|
private
|
166
164
|
def generate_log_name(metadata, index)
|
167
165
|
time_slice = if metadata.timekey.nil?
|
@@ -229,7 +227,7 @@ module Fluent::Plugin
|
|
229
227
|
if response.success?
|
230
228
|
data = JSON.parse(response.body)
|
231
229
|
log.debug "azurestorage_gen2: Token response: #{data}"
|
232
|
-
@azure_access_token = data["access_token"]
|
230
|
+
@azure_access_token = data["access_token"].chomp
|
233
231
|
else
|
234
232
|
raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
|
235
233
|
end
|
@@ -242,12 +240,12 @@ module Fluent::Plugin
|
|
242
240
|
params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
|
243
241
|
headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
|
244
242
|
content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
|
245
|
-
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers
|
243
|
+
request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
|
246
244
|
request.on_complete do |response|
|
247
245
|
if response.success?
|
248
246
|
data = JSON.parse(response.body)
|
249
247
|
log.debug "azurestorage_gen2: Token response: #{data}"
|
250
|
-
@azure_access_token = data["access_token"]
|
248
|
+
@azure_access_token = data["access_token"].chomp
|
251
249
|
else
|
252
250
|
raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
|
253
251
|
end
|
@@ -259,7 +257,8 @@ module Fluent::Plugin
|
|
259
257
|
def acquire_access_token_by_az
|
260
258
|
access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
|
261
259
|
log.debug "azurestorage_gen2: Token response: #{access_token}"
|
262
|
-
@azure_access_token = access_token
|
260
|
+
@azure_access_token = access_token.chomp
|
261
|
+
end
|
263
262
|
|
264
263
|
private
|
265
264
|
def ensure_container
|
@@ -300,7 +299,7 @@ module Fluent::Plugin
|
|
300
299
|
if response.success?
|
301
300
|
log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
|
302
301
|
elsif response.timed_out?
|
303
|
-
raise Fluent::UnrecoverableError,
|
302
|
+
raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
|
304
303
|
else
|
305
304
|
raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
306
305
|
end
|
@@ -311,7 +310,7 @@ module Fluent::Plugin
|
|
311
310
|
private
|
312
311
|
def create_blob(blob_path)
|
313
312
|
datestamp = create_request_date
|
314
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "
|
313
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "text/plain"}
|
315
314
|
params = {:resource => "file", :recursive => "false"}
|
316
315
|
auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
317
316
|
headers[:Authorization] = auth_header
|
@@ -332,13 +331,13 @@ module Fluent::Plugin
|
|
332
331
|
|
333
332
|
private
|
334
333
|
def append_blob_block(blob_path, content, position)
|
335
|
-
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
|
334
|
+
log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
|
336
335
|
datestamp = create_request_date
|
337
|
-
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"
|
336
|
+
headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
|
338
337
|
params = {:action => "append", :position => "#{position}"}
|
339
338
|
auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
|
340
339
|
headers[:Authorization] = auth_header
|
341
|
-
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :
|
340
|
+
request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
|
342
341
|
request.on_complete do |response|
|
343
342
|
if response.success?
|
344
343
|
log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
|
@@ -370,7 +369,7 @@ module Fluent::Plugin
|
|
370
369
|
elsif response.timed_out?
|
371
370
|
raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
|
372
371
|
else
|
373
|
-
|
372
|
+
raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
|
374
373
|
end
|
375
374
|
end
|
376
375
|
request.run
|
@@ -508,6 +507,15 @@ module Fluent::Plugin
|
|
508
507
|
Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
|
509
508
|
end
|
510
509
|
|
510
|
+
private
|
511
|
+
def raise_error(error_message)
|
512
|
+
if @enable_retry
|
513
|
+
raise BlobOperationError, error_message
|
514
|
+
else
|
515
|
+
raise Fluent::UnrecoverableError, error_message
|
516
|
+
end
|
517
|
+
end
|
518
|
+
|
511
519
|
def uuid_random
|
512
520
|
require 'uuidtools'
|
513
521
|
::UUIDTools::UUID.random_create.to_s
|
@@ -624,4 +632,10 @@ module Fluent::Plugin
|
|
624
632
|
super(message)
|
625
633
|
end
|
626
634
|
end
|
635
|
+
|
636
|
+
class BlobOperationError < StandardError
|
637
|
+
def initialize(message="Default message")
|
638
|
+
super(message)
|
639
|
+
end
|
640
|
+
end
|
627
641
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azurestorage-gen2
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Szabo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|