fluent-plugin-azurestorage-gen2 0.1.6 → 0.2.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2f30aa7c60fa869b86eb0d921aab12831c6355b6
4
- data.tar.gz: 522010167e94cb9500e89b13aa68c4ec0bba1053
3
+ metadata.gz: 04eea9a39f7cc1d5533d895d6efd3aa725cd9c44
4
+ data.tar.gz: 0c3779b86a040e07924c0e7d4f78a01cee9ac274
5
5
  SHA512:
6
- metadata.gz: 5d40bc5c3964bd9187e6710dbda66f7e3556f310425120cf2c46da19a34c8905676278f4428d744a3255ce83e55bf8bdd2791dae755b25aefcf32d39e771c8b6
7
- data.tar.gz: 149ee0c0ffc588e048fd0fa87cc343af153d6dc6de4132e9ace1d06f13b80e9d4f6ca60dba1892972693e6dfb37fc0dc649e233e538ccdcf982bbbc6540d1d1f
6
+ metadata.gz: 0ec050faad375da7b4b13b5686de44b9c9c7f24cef7bedc1f647392620a3dff379f2f409b475d072947215e50ff6ae6d3a976320467a0d41eb579883dc0ae79f
7
+ data.tar.gz: c71a5e917920770e4339bab5e208c51b53c9cfcd9a4afe7255996149da0b70db539d6ca12ba983c1147b9611ed8cb342831ce217c7a4adfde93ac3232a2225fb
data/Makefile CHANGED
@@ -8,8 +8,6 @@ install: gem
8
8
  push: gem
9
9
  gem push fluent-plugin-azurestorage-gen2*.gem
10
10
 
11
- tag-and-branch:
11
+ tag:
12
12
  git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
13
- git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
14
- git push origin "v$$(cat VERSION)"
15
- git push -u origin "release/$$(cat VERSION)"
13
+ git push origin "v$$(cat VERSION)"
data/README.md CHANGED
@@ -3,13 +3,13 @@
3
3
  [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
4
4
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
5
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
6
- ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
6
+ ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
7
7
 
8
8
  ## Requirements
9
9
 
10
10
  | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
11
  |------------------------|---------|------|
12
- | >= 0.1.6 | >= v0.14.0 | >= 2.4 |
12
+ | >= 0.1.0 | >= v0.14.0 | >= 2.4 |
13
13
 
14
14
  ## Overview
15
15
 
@@ -34,9 +34,11 @@ $ gem install fluent-plugin-azurestorage-gen2
34
34
  azure_object_key_format %{path}-%{index}.%{file_extension}
35
35
  azure_oauth_refresh_interval 3600
36
36
  time_slice_format %Y%m%d-%H
37
- file_extension log
37
+ file_extension log # only used with store_as none
38
38
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
39
39
  auto_create_container true
40
+ store_as gzip
41
+ format single_value
40
42
  <buffer tag,time>
41
43
  @type file
42
44
  path /var/log/fluent/azurestorage-buffer
@@ -60,9 +62,10 @@ $ gem install fluent-plugin-azurestorage-gen2
60
62
  azure_oauth_secret <my client secret>
61
63
  azure_oauth_refresh_interval 3600
62
64
  time_slice_format %Y%m%d-%H
63
- file_extension log
64
65
  path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
66
+ store_as gzip
65
67
  auto_create_container true
68
+ format single_value
66
69
  <buffer tag,time>
67
70
  @type file
68
71
  path /var/log/fluent/azurestorage-buffer
@@ -119,7 +122,15 @@ This plugin create container if not exist when you set 'auto_create_container' t
119
122
 
120
123
  ### skip_container_check
121
124
 
122
- You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
125
+ You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
126
+
127
+ ### enable_retry
128
+
129
+ If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
130
+
131
+ ### url_domain_suffix
132
+
133
+ The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
123
134
 
124
135
  ### azure_object_key_format
125
136
 
@@ -173,13 +184,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
173
184
 
174
185
  ### file_extension
175
186
 
176
- File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
187
+ File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
177
188
 
178
189
  ### store_as
179
190
 
180
191
  Archive format on Azure Storage. You can use following types:
181
192
 
182
- - none (default - no tmp file creation for log processing)
193
+ - none (default - no tmp file creation for log processing, use with json or single value format)
183
194
  - gzip
184
195
  - json
185
196
  - text
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.6
1
+ 0.2.4
@@ -24,6 +24,7 @@ module Fluent
24
24
  chunk.path
25
25
  else
26
26
  w = Tempfile.new("chunk-gzip-tmp")
27
+ w.binmode
27
28
  chunk.write_to(w)
28
29
  w.close
29
30
  w.path
@@ -20,6 +20,7 @@ module Fluent
20
20
 
21
21
  def compress(chunk, tmp)
22
22
  w = Tempfile.new("chunk-xz-tmp")
23
+ w.binmode
23
24
  chunk.write_to(w)
24
25
  w.close
25
26
 
@@ -20,6 +20,7 @@ module Fluent
20
20
 
21
21
  def compress(chunk, tmp)
22
22
  w = Tempfile.new("chunk-tmp")
23
+ w.binmode
23
24
  chunk.write_to(w)
24
25
  w.close
25
26
 
@@ -27,20 +27,20 @@ module Fluent::Plugin
27
27
  config_param :azure_oauth_secret, :string, :default => nil, :secret => true
28
28
  config_param :azure_oauth_tenant_id, :string, :default => nil
29
29
  config_param :azure_oauth_use_azure_cli, :bool, :default => false
30
- config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
30
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
31
31
  config_param :azure_container, :string, :default => nil
32
32
  config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
33
33
  config_param :file_extension, :string, :default => "log"
34
34
  config_param :store_as, :string, :default => "none"
35
35
  config_param :auto_create_container, :bool, :default => false
36
36
  config_param :skip_container_check, :bool, :default => false
37
+ config_param :enable_retry, :bool, :default => false
38
+ config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
37
39
  config_param :format, :string, :default => "out_file"
38
40
  config_param :time_slice_format, :string, :default => '%Y%m%d'
39
41
  config_param :command_parameter, :string, :default => nil
40
- config_param :message_field, :string, :default => nil
41
42
 
42
43
  DEFAULT_FORMAT_TYPE = "out_file"
43
- URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
44
44
  ACCESS_TOKEN_API_VERSION = "2018-02-01"
45
45
  ABFS_API_VERSION = "2018-11-09"
46
46
  AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
@@ -118,40 +118,35 @@ module Fluent::Plugin
118
118
  def write(chunk)
119
119
  metadata = chunk.metadata
120
120
  if @store_as.nil? || @store_as == "none"
121
- raw_data=''
122
121
  generate_log_name(metadata, @current_index)
123
122
  if @last_azure_storage_path != @azure_storage_path
124
123
  @current_index = 0
125
124
  generate_log_name(metadata, @current_index)
126
125
  end
127
- chunk.each do |emit_time, record|
128
- if @message_field.nil? || @message_field.empty?
129
- raw_data << "#{Yajl.dump(record)}\n"
130
- elsif record.key?(@message_field)
131
- line = record[@message_field].chomp
132
- raw_data << "#{line}\n"
133
- end
134
- end
126
+ raw_data = chunk.read
135
127
  unless raw_data.empty?
128
+ log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
136
129
  upload_blob(raw_data, metadata)
137
130
  end
131
+ chunk.close rescue nil
138
132
  @last_azure_storage_path = @azure_storage_path
139
133
  else
140
134
  tmp = Tempfile.new("azure-")
135
+ tmp.binmode
141
136
  begin
142
137
  @compressor.compress(chunk, tmp)
143
- tmp.close
138
+ tmp.rewind
144
139
  generate_log_name(metadata, @current_index)
145
140
  if @last_azure_storage_path != @azure_storage_path
146
141
  @current_index = 0
147
142
  generate_log_name(metadata, @current_index)
148
143
  end
149
- log.debug "Start uploading temp file: #{tmp.path}"
144
+ log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
150
145
  content = File.open(tmp.path, 'rb') { |file| file.read }
151
146
  upload_blob(content, metadata)
152
147
  @last_azure_storage_path = @azure_storage_path
153
148
  ensure
154
- tmp.unlink
149
+ tmp.close(true) rescue nil
155
150
  end
156
151
  end
157
152
 
@@ -159,7 +154,7 @@ module Fluent::Plugin
159
154
 
160
155
  private
161
156
  def upload_blob(content, metadata)
162
- log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
157
+ log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
163
158
  existing_content_length = get_blob_properties(@azure_storage_path)
164
159
  if existing_content_length == 0
165
160
  create_blob(@azure_storage_path)
@@ -274,7 +269,7 @@ module Fluent::Plugin
274
269
  params = {:resource => "filesystem" }
275
270
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
276
271
  headers[:Authorization] = auth_header
277
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
272
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
278
273
  request.on_complete do |response|
279
274
  if response.success?
280
275
  log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -301,12 +296,12 @@ module Fluent::Plugin
301
296
  params = {:resource => "filesystem" }
302
297
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
303
298
  headers[:Authorization] = auth_header
304
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
299
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
305
300
  request.on_complete do |response|
306
301
  if response.success?
307
302
  log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
308
303
  elsif response.timed_out?
309
- raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
304
+ raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
310
305
  else
311
306
  raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
312
307
  end
@@ -321,16 +316,16 @@ module Fluent::Plugin
321
316
  params = {:resource => "file", :recursive => "false"}
322
317
  auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
323
318
  headers[:Authorization] = auth_header
324
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
319
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
325
320
  request.on_complete do |response|
326
321
  if response.success?
327
322
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
328
323
  elsif response.timed_out?
329
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request timed out."
324
+ raise_error "Creating blob '#{blob_path}' request timed out."
330
325
  elsif response.code == 409
331
326
  log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
332
327
  else
333
- raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
328
+ raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
334
329
  end
335
330
  end
336
331
  request.run
@@ -338,24 +333,24 @@ module Fluent::Plugin
338
333
 
339
334
  private
340
335
  def append_blob_block(blob_path, content, position)
341
- log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
336
+ log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
342
337
  datestamp = create_request_date
343
338
  headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-date" => datestamp, :"Content-Length" => content.length}
344
339
  params = {:action => "append", :position => "#{position}"}
345
340
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
346
341
  headers[:Authorization] = auth_header
347
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
342
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
348
343
  request.on_complete do |response|
349
344
  if response.success?
350
345
  log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
351
346
  elsif response.timed_out?
352
- raise Fluent::UnrecoverableError, "Appending blob #{blob_path}' request timed out."
347
+ raise_error "Appending blob #{blob_path}' request timed out."
353
348
  elsif response.code == 404
354
349
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
355
350
  elsif response.code == 409
356
351
  raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
357
352
  else
358
- raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
353
+ raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
359
354
  end
360
355
  end
361
356
  request.run
@@ -369,14 +364,14 @@ module Fluent::Plugin
369
364
  params = {:action => "flush", :position => "#{position}"}
370
365
  auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
371
366
  headers[:Authorization] = auth_header
372
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
367
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
373
368
  request.on_complete do |response|
374
369
  if response.success?
375
370
  log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
376
371
  elsif response.timed_out?
377
- raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
372
+ raise_error "Bloub '#{blob_path}' flush request timed out."
378
373
  else
379
- raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
374
+ raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
380
375
  end
381
376
  end
382
377
  request.run
@@ -390,18 +385,18 @@ module Fluent::Plugin
390
385
  content_length = -1
391
386
  auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
392
387
  headers[:Authorization] = auth_header
393
- request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
388
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
394
389
  request.on_complete do |response|
395
390
  if response.success?
396
391
  log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
397
392
  content_length = response.headers['Content-Length'].to_i
398
393
  elsif response.timed_out?
399
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request timed out."
394
+ raise_error "Get blob properties '#{blob_path}' request timed out."
400
395
  elsif response.code == 404
401
396
  log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
402
397
  content_length = 0
403
398
  else
404
- raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
399
+ raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
405
400
  end
406
401
  end
407
402
  request.run
@@ -514,6 +509,15 @@ module Fluent::Plugin
514
509
  Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
515
510
  end
516
511
 
512
+ private
513
+ def raise_error(error_message)
514
+ if @enable_retry
515
+ raise BlobOperationError, error_message
516
+ else
517
+ raise Fluent::UnrecoverableError, error_message
518
+ end
519
+ end
520
+
517
521
  def uuid_random
518
522
  require 'uuidtools'
519
523
  ::UUIDTools::UUID.random_create.to_s
@@ -630,4 +634,10 @@ module Fluent::Plugin
630
634
  super(message)
631
635
  end
632
636
  end
637
+
638
+ class BlobOperationError < StandardError
639
+ def initialize(message="Default message")
640
+ super(message)
641
+ end
642
+ end
633
643
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.2.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-01-03 00:00:00.000000000 Z
11
+ date: 2020-01-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd