fluent-plugin-azurestorage-gen2 0.1.6 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2f30aa7c60fa869b86eb0d921aab12831c6355b6
4
- data.tar.gz: 522010167e94cb9500e89b13aa68c4ec0bba1053
3
+ metadata.gz: 04eea9a39f7cc1d5533d895d6efd3aa725cd9c44
4
+ data.tar.gz: 0c3779b86a040e07924c0e7d4f78a01cee9ac274
5
5
  SHA512:
6
- metadata.gz: 5d40bc5c3964bd9187e6710dbda66f7e3556f310425120cf2c46da19a34c8905676278f4428d744a3255ce83e55bf8bdd2791dae755b25aefcf32d39e771c8b6
7
- data.tar.gz: 149ee0c0ffc588e048fd0fa87cc343af153d6dc6de4132e9ace1d06f13b80e9d4f6ca60dba1892972693e6dfb37fc0dc649e233e538ccdcf982bbbc6540d1d1f
6
+ metadata.gz: 0ec050faad375da7b4b13b5686de44b9c9c7f24cef7bedc1f647392620a3dff379f2f409b475d072947215e50ff6ae6d3a976320467a0d41eb579883dc0ae79f
7
+ data.tar.gz: c71a5e917920770e4339bab5e208c51b53c9cfcd9a4afe7255996149da0b70db539d6ca12ba983c1147b9611ed8cb342831ce217c7a4adfde93ac3232a2225fb
data/Makefile CHANGED
@@ -8,8 +8,6 @@ install: gem
8
8
  push: gem
9
9
  gem push fluent-plugin-azurestorage-gen2*.gem
10
10
 
11
- tag-and-branch:
11
+ tag:
12
12
  git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
13
- git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
14
- git push origin "v$$(cat VERSION)"
15
- git push -u origin "release/$$(cat VERSION)"
13
+ git push origin "v$$(cat VERSION)"
data/README.md CHANGED
@@ -3,13 +3,13 @@
3
3
  [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
4
4
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
5
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
6
- ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
6
+ ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
7
7
 
8
8
  ## Requirements
9
9
 
10
10
  | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
11
  |------------------------|---------|------|
12
- | >= 0.1.6 | >= v0.14.0 | >= 2.4 |
12
+ | >= 0.1.0 | >= v0.14.0 | >= 2.4 |
13
13
 
14
14
  ## Overview
15
15
 
@@ -34,9 +34,11 @@ $ gem install fluent-plugin-azurestorage-gen2
34
34
  azure_object_key_format %{path}-%{index}.%{file_extension}
35
35
  azure_oauth_refresh_interval 3600
36
36
  time_slice_format %Y%m%d-%H
37
- file_extension log
37
+ file_extension log # only used with store_as none
38
38
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
39
39
  auto_create_container true
40
+ store_as gzip
41
+ format single_value
40
42
  <buffer tag,time>
41
43
  @type file
42
44
  path /var/log/fluent/azurestorage-buffer
@@ -60,9 +62,10 @@ $ gem install fluent-plugin-azurestorage-gen2
60
62
  azure_oauth_secret <my client secret>
61
63
  azure_oauth_refresh_interval 3600
62
64
  time_slice_format %Y%m%d-%H
63
- file_extension log
64
65
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
66
+ store_as gzip
65
67
  auto_create_container true
68
+ format single_value
66
69
  <buffer tag,time>
67
70
  @type file
68
71
  path /var/log/fluent/azurestorage-buffer
@@ -119,7 +122,15 @@ This plugin create container if not exist when you set 'auto_create_container' t
119
122
 
120
123
  ### skip_container_check
121
124
 
122
- You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
125
+ You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
126
+
127
+ ### enable_retry
128
+
129
+ If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
130
+
131
+ ### url_domain_suffix
132
+
133
+ The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
123
134
 
124
135
  ### azure_object_key_format
125
136
 
@@ -173,13 +184,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
173
184
 
174
185
  ### file_extension
175
186
 
176
- File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
187
+ File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
177
188
 
178
189
  ### store_as
179
190
 
180
191
  Archive format on Azure Storage. You can use following types:
181
192
 
182
- - none (default - no tmp file creation for log processing)
193
+ - none (default - no tmp file creation for log processing, use with json or single value format)
183
194
  - gzip
184
195
  - json
185
196
  - text
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.6
1
+ 0.2.4
@@ -24,6 +24,7 @@ module Fluent
24
24
  chunk.path
25
25
  else
26
26
  w = Tempfile.new("chunk-gzip-tmp")
27
+ w.binmode
27
28
  chunk.write_to(w)
28
29
  w.close
29
30
  w.path
@@ -20,6 +20,7 @@ module Fluent
20
20
 
21
21
  def compress(chunk, tmp)
22
22
  w = Tempfile.new("chunk-xz-tmp")
23
+ w.binmode
23
24
  chunk.write_to(w)
24
25
  w.close
25
26
 
@@ -20,6 +20,7 @@ module Fluent
20
20
 
21
21
  def compress(chunk, tmp)
22
22
  w = Tempfile.new("chunk-tmp")
23
+ w.binmode
23
24
  chunk.write_to(w)
24
25
  w.close
25
26
 
@@ -27,20 +27,20 @@ module Fluent::Plugin
27
27
  config_param :azure_oauth_secret, :string, :default => nil, :secret => true
28
28
  config_param :azure_oauth_tenant_id, :string, :default => nil
29
29
  config_param :azure_oauth_use_azure_cli, :bool, :default => false
30
- config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
30
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
31
31
  config_param :azure_container, :string, :default => nil
32
32
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
36
  config_param :skip_container_check, :bool, :default => false
37
+ config_param :enable_retry, :bool, :default => false
38
+ config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
37
39
  config_param :format, :string, :default => "out_file"
38
40
  config_param :time_slice_format, :string, :default => '%Y%m%d'
39
41
  config_param :command_parameter, :string, :default => nil
40
- config_param :message_field, :string, :default => nil
41
42
 
42
43
  DEFAULT_FORMAT_TYPE = "out_file"
43
- URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
44
44
  ACCESS_TOKEN_API_VERSION = "2018-02-01"
45
45
  ABFS_API_VERSION = "2018-11-09"
46
46
  AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
@@ -118,40 +118,35 @@ module Fluent::Plugin
118
118
  def write(chunk)
119
119
  metadata = chunk.metadata
120
120
  if @store_as.nil? || @store_as == "none"
121
- raw_data=''
122
121
  generate_log_name(metadata, @current_index)
123
122
  if @last_azure_storage_path != @azure_storage_path
124
123
  @current_index = 0
125
124
  generate_log_name(metadata, @current_index)
126
125
  end
127
- chunk.each do |emit_time, record|
128
- if @message_field.nil? || @message_field.empty?
129
- raw_data << "#{Yajl.dump(record)}\n"
130
- elsif record.key?(@message_field)
131
- line = record[@message_field].chomp
132
- raw_data << "#{line}\n"
133
- end
134
- end
126
+ raw_data = chunk.read
135
127
  unless raw_data.empty?
128
+ log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
136
129
  upload_blob(raw_data, metadata)
137
130
  end
131
+ chunk.close rescue nil
138
132
  @last_azure_storage_path = @azure_storage_path
139
133
  else
140
134
  tmp = Tempfile.new("azure-")
135
+ tmp.binmode
141
136
  begin
142
137
  @compressor.compress(chunk, tmp)
143
- tmp.close
138
+ tmp.rewind
144
139
  generate_log_name(metadata, @current_index)
145
140
  if @last_azure_storage_path != @azure_storage_path
146
141
  @current_index = 0
147
142
  generate_log_name(metadata, @current_index)
148
143
  end
149
- log.debug "Start uploading temp file: #{tmp.path}"
144
+ log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
150
145
  content = File.open(tmp.path, 'rb') { |file| file.read }
151
146
  upload_blob(content, metadata)
152
147
  @last_azure_storage_path = @azure_storage_path
153
148
  ensure
154
- tmp.unlink
149
+ tmp.close(true) rescue nil
155
150
  end
156
151
  end
157
152
 
@@ -159,7 +154,7 @@ module Fluent::Plugin
159
154
 
160
155
  private
161
156
  def upload_blob(content, metadata)
162
- log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
157
+ log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
163
158
  existing_content_length = get_blob_properties(@azure_storage_path)
164
159
  if existing_content_length == 0
165
160
  create_blob(@azure_storage_path)
@@ -274,7 +269,7 @@ module Fluent::Plugin
274
269
  params = {:resource => "filesystem" }
275
270
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
276
271
  headers[:Authorization] = auth_header
277
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
272
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
278
273
  request.on_complete do |response|
279
274
  if response.success?
280
275
  log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -301,12 +296,12 @@ module Fluent::Plugin
301
296
  params = {:resource => "filesystem" }
302
297
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
303
298
  headers[:Authorization] = auth_header
304
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
299
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
305
300
  request.on_complete do |response|
306
301
  if response.success?
307
302
  log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
308
303
  elsif response.timed_out?
309
- raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
304
+ raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
310
305
  else
311
306
  raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
312
307
  end
@@ -321,16 +316,16 @@ module Fluent::Plugin
321
316
  params = {:resource => "file", :recursive => "false"}
322
317
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
323
318
  headers[:Authorization] = auth_header
324
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
319
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
325
320
  request.on_complete do |response|
326
321
  if response.success?
327
322
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
328
323
  elsif response.timed_out?
329
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request timed out."
324
+ raise_error "Creating blob '#{blob_path}' request timed out."
330
325
  elsif response.code == 409
331
326
  log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
332
327
  else
333
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
328
+ raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
334
329
  end
335
330
  end
336
331
  request.run
@@ -338,24 +333,24 @@ module Fluent::Plugin
338
333
 
339
334
  private
340
335
  def append_blob_block(blob_path, content, position)
341
- log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
336
+ log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
342
337
  datestamp = create_request_date
343
338
  headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
344
339
  params = {:action => "append", :position => "#{position}"}
345
340
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
346
341
  headers[:Authorization] = auth_header
347
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
342
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
348
343
  request.on_complete do |response|
349
344
  if response.success?
350
345
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
351
346
  elsif response.timed_out?
352
- raise Fluent::UnrecoverableError, "Appending blob #{blob_path}' request timed out."
347
+ raise_error "Appending blob #{blob_path}' request timed out."
353
348
  elsif response.code == 404
354
349
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
355
350
  elsif response.code == 409
356
351
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
357
352
  else
358
- raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
353
+ raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
359
354
  end
360
355
  end
361
356
  request.run
@@ -369,14 +364,14 @@ module Fluent::Plugin
369
364
  params = {:action => "flush", :position => "#{position}"}
370
365
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
371
366
  headers[:Authorization] = auth_header
372
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
367
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
373
368
  request.on_complete do |response|
374
369
  if response.success?
375
370
  log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
376
371
  elsif response.timed_out?
377
- raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
372
+ raise_error "Bloub '#{blob_path}' flush request timed out."
378
373
  else
379
- raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
374
+ raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
380
375
  end
381
376
  end
382
377
  request.run
@@ -390,18 +385,18 @@ module Fluent::Plugin
390
385
  content_length = -1
391
386
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
392
387
  headers[:Authorization] = auth_header
393
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
388
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
394
389
  request.on_complete do |response|
395
390
  if response.success?
396
391
  log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
397
392
  content_length = response.headers['Content-Length'].to_i
398
393
  elsif response.timed_out?
399
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request timed out."
394
+ raise_error "Get blob properties '#{blob_path}' request timed out."
400
395
  elsif response.code == 404
401
396
  log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
402
397
  content_length = 0
403
398
  else
404
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
399
+ raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
405
400
  end
406
401
  end
407
402
  request.run
@@ -514,6 +509,15 @@ module Fluent::Plugin
514
509
  Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
515
510
  end
516
511
 
512
+ private
513
+ def raise_error(error_message)
514
+ if @enable_retry
515
+ raise BlobOperationError, error_message
516
+ else
517
+ raise Fluent::UnrecoverableError, error_message
518
+ end
519
+ end
520
+
517
521
  def uuid_random
518
522
  require 'uuidtools'
519
523
  ::UUIDTools::UUID.random_create.to_s
@@ -630,4 +634,10 @@ module Fluent::Plugin
630
634
  super(message)
631
635
  end
632
636
  end
637
+
638
+ class BlobOperationError < StandardError
639
+ def initialize(message="Default message")
640
+ super(message)
641
+ end
642
+ end
633
643
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.2.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-01-03 00:00:00.000000000 Z
11
+ date: 2020-01-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd