fluent-plugin-azurestorage-gen2 0.1.3 → 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b3bcdaa488c738cafda86bcae69dbc0ac6d8521e
4
- data.tar.gz: d0f9bd8a6a2ae52dea8e57570e112d72cdd8e05a
3
+ metadata.gz: 830b58ff51c9166a04c91f7e4994cc5c867c86e1
4
+ data.tar.gz: b257f9c0684a7797266920775cea076d9c6f67dc
5
5
  SHA512:
6
- metadata.gz: 1529a5bda90bedf0c4f6a03ce89d3372fd736c844f426d2d6919aef2a776765fb8ffa1305fda0304179e99eb13cc8746008cf2461a9d997d05a4519c03579a18
7
- data.tar.gz: 48843accc54dca373c08efa02f72ec4ee1142d07003af82ed00548f5b1767568770b91fa1549633836feddad28a1733a915065cb81d157c1a680975189573075
6
+ metadata.gz: 2512d0e061e46d7820dc6405ea2530eeceb1f65483eb9d31789df8956c4fb1469a43d20a2ebee3b0e496a73143f406331c17aabe812d9a65dae4828729d8c6aa
7
+ data.tar.gz: 185079c3757ccdbaf51d205984f57791b5d3e20566c65ec8adea0b94f7db6667a56074a34234543afe29e429fb205f7bc2cf681b70e8e803d0992b1db772e65d
data/README.md CHANGED
@@ -9,7 +9,7 @@
9
9
 
10
10
  | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
11
  |------------------------|---------|------|
12
- | >= 0.1.3 | >= v0.14.0 | >= 2.4 |
12
+ | >= 0.1.0 | >= v0.14.0 | >= 2.4 |
13
13
 
14
14
  ## Overview
15
15
 
@@ -37,6 +37,7 @@ $ gem install fluent-plugin-azurestorage-gen2
37
37
  file_extension log
38
38
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
39
39
  auto_create_container true
40
+ format single_value
40
41
  <buffer tag,time>
41
42
  @type file
42
43
  path /var/log/fluent/azurestorage-buffer
@@ -63,6 +64,7 @@ $ gem install fluent-plugin-azurestorage-gen2
63
64
  file_extension log
64
65
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
65
66
  auto_create_container true
67
+ format single_value
66
68
  <buffer tag,time>
67
69
  @type file
68
70
  path /var/log/fluent/azurestorage-buffer
@@ -89,7 +91,7 @@ Your Azure Storage Access Key(Primary or Secondary). This also can be got from A
89
91
 
90
92
  Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
91
93
 
92
- ### azure_oauth_tenant_id
94
+ ### azure_oauth_tenant_id (Preview)
93
95
 
94
96
  Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
95
97
 
@@ -101,9 +103,9 @@ OAuth client id that is used for OAuth based authentication. Required if OAuth b
101
103
 
102
104
  OAuth client secret that is used for OAuth based authentication. Required if OAuth based credential mechanism is used.
103
105
 
104
- ### azure_oauth_refresh_interval (Preview)
106
+ ### azure_oauth_refresh_interval
105
107
 
106
- OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
108
+ OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
107
109
 
108
110
  ### azure_oauth_use_azure_cli (Preview)
109
111
 
@@ -117,6 +119,10 @@ Azure Storage Container name
117
119
 
118
120
  This plugin create container if not exist when you set 'auto_create_container' to true.
119
121
 
122
+ ### skip_container_check
123
+
124
+ You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
125
+
120
126
  ### azure_object_key_format
121
127
 
122
128
  The format of Azure Storage object keys. You can use several built-in variables:
@@ -169,13 +175,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
169
175
 
170
176
  ### file_extension
171
177
 
172
- File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
178
+ File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
173
179
 
174
180
  ### store_as
175
181
 
176
182
  Archive format on Azure Storage. You can use following types:
177
183
 
178
- - none (default - no tmp file creation for log processing)
184
+ - none (default - no tmp file creation for log processing, use with json or single value format)
179
185
  - gzip
180
186
  - json
181
187
  - text
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.3
1
+ 0.2.1
@@ -27,16 +27,17 @@ module Fluent::Plugin
27
27
  config_param :azure_oauth_secret, :string, :default => nil, :secret => true
28
28
  config_param :azure_oauth_tenant_id, :string, :default => nil
29
29
  config_param :azure_oauth_use_azure_cli, :bool, :default => false
30
- config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
30
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
31
31
  config_param :azure_container, :string, :default => nil
32
32
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
+ config_param :skip_container_check, :bool, :default => false
37
+ config_param :enable_retry, :bool, :default => false
36
38
  config_param :format, :string, :default => "out_file"
37
39
  config_param :time_slice_format, :string, :default => '%Y%m%d'
38
40
  config_param :command_parameter, :string, :default => nil
39
- config_param :message_field, :string, :default => nil
40
41
 
41
42
  DEFAULT_FORMAT_TYPE = "out_file"
42
43
  URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
@@ -57,13 +58,17 @@ module Fluent::Plugin
57
58
  compat_parameters_convert(conf, :buffer, :formatter, :inject)
58
59
  super
59
60
 
60
- begin
61
- @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
62
- rescue => e
63
- log.warn "#{@store_as} not found. Use 'text' instead"
64
- @compressor = TextCompressor.new
61
+ if @store_as.nil? || @store_as == "none"
62
+ log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
63
+ else
64
+ begin
65
+ @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
66
+ rescue => e
67
+ log.warn "#{@store_as} not found. Use 'text' instead"
68
+ @compressor = TextCompressor.new
69
+ end
70
+ @compressor.configure(conf)
65
71
  end
66
- @compressor.configure(conf)
67
72
 
68
73
  @formatter = formatter_create
69
74
 
@@ -84,11 +89,10 @@ module Fluent::Plugin
84
89
  @azure_storage_path = ''
85
90
  @last_azure_storage_path = ''
86
91
  @current_index = 0
92
+
87
93
  if @store_as.nil? || @store_as == "none"
88
- @blob_content_type = "text/plain"
89
94
  @final_file_extension = @file_extension
90
95
  else
91
- @blob_content_type = @compressor.content_type
92
96
  @final_file_extension = @compressor.ext
93
97
  end
94
98
 
@@ -100,29 +104,28 @@ module Fluent::Plugin
100
104
 
101
105
  def start
102
106
  setup_access_token
103
- ensure_container
107
+ if !@skip_container_check
108
+ ensure_container
109
+ end
104
110
  super
105
111
  end
106
112
 
113
+ def format(tag, time, record)
114
+ r = inject_values_to_record(tag, time, record)
115
+ @formatter.format(tag, time, r)
116
+ end
117
+
107
118
  def write(chunk)
108
119
  metadata = chunk.metadata
109
120
  if @store_as.nil? || @store_as == "none"
110
- raw_data=''
111
121
  generate_log_name(metadata, @current_index)
112
122
  if @last_azure_storage_path != @azure_storage_path
113
123
  @current_index = 0
114
124
  generate_log_name(metadata, @current_index)
115
125
  end
116
- chunk.each do |emit_time, record|
117
- if @message_field.nil? || @message_field.empty?
118
- raw_data << "#{Yajl.dump(record)}\n"
119
- elsif record.key?(@message_field)
120
- line = record[@message_field].chomp
121
- raw_data << "#{line}\n"
122
- end
123
- end
124
- raw_data = raw_data.chomp
126
+ raw_data = chunk.read
125
127
  unless raw_data.empty?
128
+ log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
126
129
  upload_blob(raw_data, metadata)
127
130
  end
128
131
  @last_azure_storage_path = @azure_storage_path
@@ -136,7 +139,7 @@ module Fluent::Plugin
136
139
  @current_index = 0
137
140
  generate_log_name(metadata, @current_index)
138
141
  end
139
- log.debug "Start uploading temp file: #{tmp.path}"
142
+ log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
140
143
  content = File.open(tmp.path, 'rb') { |file| file.read }
141
144
  upload_blob(content, metadata)
142
145
  @last_azure_storage_path = @azure_storage_path
@@ -149,7 +152,7 @@ module Fluent::Plugin
149
152
 
150
153
  private
151
154
  def upload_blob(content, metadata)
152
- log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
155
+ log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
153
156
  existing_content_length = get_blob_properties(@azure_storage_path)
154
157
  if existing_content_length == 0
155
158
  create_blob(@azure_storage_path)
@@ -157,11 +160,6 @@ module Fluent::Plugin
157
160
  append_blob(content, metadata, existing_content_length)
158
161
  end
159
162
 
160
- def format(tag, time, record)
161
- r = inject_values_to_record(tag, time, record)
162
- @formatter.format(tag, time, r)
163
- end
164
-
165
163
  private
166
164
  def generate_log_name(metadata, index)
167
165
  time_slice = if metadata.timekey.nil?
@@ -229,7 +227,7 @@ module Fluent::Plugin
229
227
  if response.success?
230
228
  data = JSON.parse(response.body)
231
229
  log.debug "azurestorage_gen2: Token response: #{data}"
232
- @azure_access_token = data["access_token"]
230
+ @azure_access_token = data["access_token"].chomp
233
231
  else
234
232
  raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
235
233
  end
@@ -242,12 +240,12 @@ module Fluent::Plugin
242
240
  params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
243
241
  headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
244
242
  content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
245
- request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
243
+ request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
246
244
  request.on_complete do |response|
247
245
  if response.success?
248
246
  data = JSON.parse(response.body)
249
247
  log.debug "azurestorage_gen2: Token response: #{data}"
250
- @azure_access_token = data["access_token"]
248
+ @azure_access_token = data["access_token"].chomp
251
249
  else
252
250
  raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
253
251
  end
@@ -259,7 +257,8 @@ module Fluent::Plugin
259
257
  def acquire_access_token_by_az
260
258
  access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
261
259
  log.debug "azurestorage_gen2: Token response: #{access_token}"
262
- @azure_access_token = access_token
260
+ @azure_access_token = access_token.chomp
261
+ end
263
262
 
264
263
  private
265
264
  def ensure_container
@@ -300,7 +299,7 @@ module Fluent::Plugin
300
299
  if response.success?
301
300
  log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
302
301
  elsif response.timed_out?
303
- raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
302
+ raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
304
303
  else
305
304
  raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
306
305
  end
@@ -311,7 +310,7 @@ module Fluent::Plugin
311
310
  private
312
311
  def create_blob(blob_path)
313
312
  datestamp = create_request_date
314
- headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "application/json"}
313
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "text/plain"}
315
314
  params = {:resource => "file", :recursive => "false"}
316
315
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
317
316
  headers[:Authorization] = auth_header
@@ -332,13 +331,13 @@ module Fluent::Plugin
332
331
 
333
332
  private
334
333
  def append_blob_block(blob_path, content, position)
335
- log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
334
+ log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
336
335
  datestamp = create_request_date
337
- headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
336
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
338
337
  params = {:action => "append", :position => "#{position}"}
339
338
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
340
339
  headers[:Authorization] = auth_header
341
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
340
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
342
341
  request.on_complete do |response|
343
342
  if response.success?
344
343
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
@@ -370,7 +369,7 @@ module Fluent::Plugin
370
369
  elsif response.timed_out?
371
370
  raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
372
371
  else
373
- raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
372
+ raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
374
373
  end
375
374
  end
376
375
  request.run
@@ -508,6 +507,15 @@ module Fluent::Plugin
508
507
  Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
509
508
  end
510
509
 
510
+ private
511
+ def raise_error(error_message)
512
+ if @enable_retry
513
+ raise BlobOperationError, error_message
514
+ else
515
+ raise Fluent::UnrecoverableError, error_message
516
+ end
517
+ end
518
+
511
519
  def uuid_random
512
520
  require 'uuidtools'
513
521
  ::UUIDTools::UUID.random_create.to_s
@@ -624,4 +632,10 @@ module Fluent::Plugin
624
632
  super(message)
625
633
  end
626
634
  end
635
+
636
+ class BlobOperationError < StandardError
637
+ def initialize(message="Default message")
638
+ super(message)
639
+ end
640
+ end
627
641
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-23 00:00:00.000000000 Z
11
+ date: 2020-01-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd