fluent-plugin-azurestorage-gen2 0.2.4 → 0.2.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 04eea9a39f7cc1d5533d895d6efd3aa725cd9c44
4
- data.tar.gz: 0c3779b86a040e07924c0e7d4f78a01cee9ac274
3
+ metadata.gz: 4bff097a151b5af0d596f82ab181d3e141abf0de
4
+ data.tar.gz: f453dabd8d7719502820e51ae870803d3845a80f
5
5
  SHA512:
6
- metadata.gz: 0ec050faad375da7b4b13b5686de44b9c9c7f24cef7bedc1f647392620a3dff379f2f409b475d072947215e50ff6ae6d3a976320467a0d41eb579883dc0ae79f
7
- data.tar.gz: c71a5e917920770e4339bab5e208c51b53c9cfcd9a4afe7255996149da0b70db539d6ca12ba983c1147b9611ed8cb342831ce217c7a4adfde93ac3232a2225fb
6
+ metadata.gz: 361a92e5ab216018f95a1f0f0e331d0401bf894da0b65ea3a2bcd1ecfb62e3ce3fd599b8633e77ae385537e9590d2cffa51ec4df38917e5f42660932edc87350
7
+ data.tar.gz: da8cec0e4867005f2e794c1733c58bfcb33c5ef7e2a6f071f93436506e2d70c3b9a8ecd755ad305973af16bbc9ee163d62d50e632a7d7d26a80964de4d844cc0
data/README.md CHANGED
@@ -124,10 +124,18 @@ This plugin create container if not exist when you set 'auto_create_container' t
124
124
 
125
125
  You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
126
126
 
127
+ ### failsafe_container_check
128
+
129
+ On startup, a list operation is called against a container, if that does not exists, the app tries to create it if `auto_create_container` is enabled. If that option is enabled, the application won't fail if any of these operations are failed. (Can be useful the right roles are not set on your container, but set for blobs)
130
+
127
131
  ### enable_retry
128
132
 
129
133
  If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
130
134
 
135
+ ### startup_fail_on_error
136
+
137
+ If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
138
+
131
139
  ### url_domain_suffix
132
140
 
133
141
  The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.4
1
+ 0.2.9
@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-azurestorage-gen2"
6
6
  gem.description = "Azure Storage output plugin for Fluentd event collector"
7
- gem.license = "Apache-2.0"
7
+ gem.license = "MIT License"
8
8
  gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
9
9
  gem.summary = gem.description
10
10
  gem.version = File.read("VERSION").strip
@@ -21,6 +21,7 @@ Gem::Specification.new do |gem|
21
21
  gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
22
22
  gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
23
23
  gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
24
+ gem.add_runtime_dependency 'concurrent-ruby', '~> 1.1', '>= 1.1.5'
24
25
  gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
25
26
  gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
26
27
  gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'
@@ -6,6 +6,7 @@ require 'tempfile'
6
6
  require 'time'
7
7
  require 'typhoeus'
8
8
  require 'fluent/plugin/output'
9
+ require 'concurrent'
9
10
  require 'zlib'
10
11
 
11
12
  module Fluent::Plugin
@@ -34,10 +35,13 @@ module Fluent::Plugin
34
35
  config_param :store_as, :string, :default => "none"
35
36
  config_param :auto_create_container, :bool, :default => false
36
37
  config_param :skip_container_check, :bool, :default => false
38
+ config_param :failsafe_container_check, :bool, :default => false
37
39
  config_param :enable_retry, :bool, :default => false
40
+ config_param :startup_fail_on_error, :bool, :default => true
38
41
  config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
39
42
  config_param :format, :string, :default => "out_file"
40
43
  config_param :time_slice_format, :string, :default => '%Y%m%d'
44
+ config_param :hex_random_length, :integer, default: 4
41
45
  config_param :command_parameter, :string, :default => nil
42
46
 
43
47
  DEFAULT_FORMAT_TYPE = "out_file"
@@ -72,16 +76,6 @@ module Fluent::Plugin
72
76
 
73
77
  @formatter = formatter_create
74
78
 
75
- if @localtime
76
- @path_slicer = Proc.new {|path|
77
- Time.now.strftime(path)
78
- }
79
- else
80
- @path_slicer = Proc.new {|path|
81
- Time.now.utc.strftime(path)
82
- }
83
- end
84
-
85
79
  if @azure_container.nil?
86
80
  raise Fluent::ConfigError, "azure_container is needed"
87
81
  end
@@ -95,7 +89,7 @@ module Fluent::Plugin
95
89
  else
96
90
  @final_file_extension = @compressor.ext
97
91
  end
98
-
92
+ @values_for_object_chunk = {}
99
93
  end
100
94
 
101
95
  def multi_workers_ready?
@@ -105,7 +99,15 @@ module Fluent::Plugin
105
99
  def start
106
100
  setup_access_token
107
101
  if !@skip_container_check
108
- ensure_container
102
+ if @failsafe_container_check
103
+ begin
104
+ ensure_container
105
+ rescue Exception => e
106
+ log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
107
+ end
108
+ else
109
+ ensure_container
110
+ end
109
111
  end
110
112
  super
111
113
  end
@@ -116,17 +118,16 @@ module Fluent::Plugin
116
118
  end
117
119
 
118
120
  def write(chunk)
119
- metadata = chunk.metadata
120
121
  if @store_as.nil? || @store_as == "none"
121
- generate_log_name(metadata, @current_index)
122
+ generate_log_name(chunk, @current_index)
122
123
  if @last_azure_storage_path != @azure_storage_path
123
124
  @current_index = 0
124
- generate_log_name(metadata, @current_index)
125
+ generate_log_name(chunk, @current_index)
125
126
  end
126
127
  raw_data = chunk.read
127
128
  unless raw_data.empty?
128
129
  log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
129
- upload_blob(raw_data, metadata)
130
+ upload_blob(raw_data, chunk)
130
131
  end
131
132
  chunk.close rescue nil
132
133
  @last_azure_storage_path = @azure_storage_path
@@ -136,57 +137,86 @@ module Fluent::Plugin
136
137
  begin
137
138
  @compressor.compress(chunk, tmp)
138
139
  tmp.rewind
139
- generate_log_name(metadata, @current_index)
140
+ generate_log_name(chunk, @current_index)
140
141
  if @last_azure_storage_path != @azure_storage_path
141
142
  @current_index = 0
142
- generate_log_name(metadata, @current_index)
143
+ generate_log_name(chunk, @current_index)
143
144
  end
144
145
  log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
145
146
  content = File.open(tmp.path, 'rb') { |file| file.read }
146
- upload_blob(content, metadata)
147
+ upload_blob(content, chunk)
147
148
  @last_azure_storage_path = @azure_storage_path
148
149
  ensure
149
150
  tmp.close(true) rescue nil
150
151
  end
152
+ @values_for_object_chunk.delete(chunk.unique_id)
151
153
  end
152
154
 
153
155
  end
154
156
 
155
157
  private
156
- def upload_blob(content, metadata)
158
+ def upload_blob(content, chunk)
157
159
  log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
158
160
  existing_content_length = get_blob_properties(@azure_storage_path)
159
161
  if existing_content_length == 0
160
162
  create_blob(@azure_storage_path)
161
163
  end
162
- append_blob(content, metadata, existing_content_length)
164
+ append_blob(content, chunk, existing_content_length)
163
165
  end
164
166
 
165
167
  private
166
- def generate_log_name(metadata, index)
168
+ def generate_log_name(chunk, index)
169
+ metadata = chunk.metadata
167
170
  time_slice = if metadata.timekey.nil?
168
171
  ''.freeze
169
172
  else
170
173
  Time.at(metadata.timekey).utc.strftime(@time_slice_format)
171
174
  end
172
- path = @path_slicer.call(@path)
173
- values_for_object_key = {
174
- "%{path}" => path,
175
- "%{time_slice}" => time_slice,
175
+ if @localtime
176
+ hms_slicer = Time.now.strftime("%H%M%S")
177
+ else
178
+ hms_slicer = Time.now.utc.strftime("%H%M%S")
179
+ end
180
+
181
+ @values_for_object_chunk[chunk.unique_id] ||= {
182
+ "%{hex_random}" => hex_random(chunk),
183
+ }
184
+ values_for_object_key_pre = {
185
+ "%{path}" => @path,
176
186
  "%{index}" => index,
177
187
  "%{uuid_flush}" => uuid_random,
178
188
  "%{file_extension}" => @final_file_extension
179
189
  }
180
- storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
181
- extracted_path = extract_placeholders(storage_path, metadata)
182
- extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
183
- @azure_storage_path = extracted_path
190
+ values_for_object_key_post = {
191
+ "%{date_slice}" => time_slice,
192
+ "%{time_slice}" => time_slice,
193
+ "%{hms_slice}" => hms_slicer,
194
+ }.merge!(@values_for_object_chunk[chunk.unique_id])
195
+ storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
196
+ values_for_object_key_pre.fetch(matched_key, matched_key)
197
+ end
198
+ storage_path = extract_placeholders(storage_path, chunk)
199
+ storage_path = storage_path.gsub(%r(%{[^}]+}), values_for_object_key_post)
200
+ storage_path = "/" + storage_path unless storage_path.start_with?("/")
201
+ @azure_storage_path = storage_path
184
202
  end
185
203
 
186
204
  def setup_access_token
187
205
  if @azure_storage_access_key.nil?
188
206
  @get_token_lock = Concurrent::ReadWriteLock.new
189
- acquire_access_token
207
+ if @startup_fail_on_error
208
+ acquire_access_token
209
+ else
210
+ while true
211
+ begin
212
+ acquire_access_token
213
+ break
214
+ rescue Exception => e
215
+ log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
216
+ sleep 20
217
+ end
218
+ end
219
+ end
190
220
  if @azure_oauth_refresh_interval > 0
191
221
  log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
192
222
  @get_token_task = Concurrent::TimerTask.new(
@@ -404,7 +434,7 @@ module Fluent::Plugin
404
434
  end
405
435
 
406
436
  private
407
- def append_blob(content, metadata, existing_content_length)
437
+ def append_blob(content, chunk, existing_content_length)
408
438
  position = 0
409
439
  log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
410
440
  loop do
@@ -522,6 +552,12 @@ module Fluent::Plugin
522
552
  require 'uuidtools'
523
553
  ::UUIDTools::UUID.random_create.to_s
524
554
  end
555
+
556
+ def hex_random(chunk)
557
+ unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
558
+ unique_hex.reverse!
559
+ unique_hex[0...@hex_random_length]
560
+ end
525
561
 
526
562
  def timekey_to_timeformat(timekey)
527
563
  case timekey
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-azurestorage-gen2
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.4
4
+ version: 0.2.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Oliver Szabo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-01-13 00:00:00.000000000 Z
11
+ date: 2020-02-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -104,6 +104,26 @@ dependencies:
104
104
  - - "~>"
105
105
  - !ruby/object:Gem::Version
106
106
  version: '1.4'
107
+ - !ruby/object:Gem::Dependency
108
+ name: concurrent-ruby
109
+ requirement: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - "~>"
112
+ - !ruby/object:Gem::Version
113
+ version: '1.1'
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: 1.1.5
117
+ type: :runtime
118
+ prerelease: false
119
+ version_requirements: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - "~>"
122
+ - !ruby/object:Gem::Version
123
+ version: '1.1'
124
+ - - ">="
125
+ - !ruby/object:Gem::Version
126
+ version: 1.1.5
107
127
  - !ruby/object:Gem::Dependency
108
128
  name: rake
109
129
  requirement: !ruby/object:Gem::Requirement
@@ -188,7 +208,7 @@ files:
188
208
  - test/plugin/test_out_azurestorage_gen2.rb
189
209
  homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
190
210
  licenses:
191
- - Apache-2.0
211
+ - MIT License
192
212
  metadata: {}
193
213
  post_install_message:
194
214
  rdoc_options: []