fluent-plugin-azurestorage-gen2 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 857dcdb31e03f6564e547e0183a3bf24f06c8a0e
4
+ data.tar.gz: f1f1d048e69afba0f469a1999b6786473e282092
5
+ SHA512:
6
+ metadata.gz: 6c87884fa508bc01f7470653b0a3b67d4a4cb02bddab03eb9071c1360f25abcf13558d1023a11be7d1277dcebf0794d4fb01ff19982a2000c2879ff335421773
7
+ data.tar.gz: 1491be525cbf66dbd7240ddea5a2ba33ded91e4c4c5816d238fc4ac49a76057e05b7bc7dea881220dc32fff836239d2d485175699285041bb1330c955b103e0b
@@ -0,0 +1,58 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+ GEMSPEC_ARGS
13
+ Gemfile.lock
14
+
15
+ # Used by dotenv library to load environment variables.
16
+ # .env
17
+
18
+ # Ignore Byebug command history file.
19
+ .byebug_history
20
+
21
+ ## Specific to RubyMotion:
22
+ .dat*
23
+ .repl_history
24
+ build/
25
+ *.bridgesupport
26
+ build-iPhoneOS/
27
+ build-iPhoneSimulator/
28
+
29
+ ## Specific to RubyMotion (use of CocoaPods):
30
+ #
31
+ # We recommend against adding the Pods directory to your .gitignore. However
32
+ # you should judge for yourself, the pros and cons are mentioned at:
33
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
34
+ #
35
+ # vendor/Pods/
36
+
37
+ ## Documentation cache and generated files:
38
+ /.yardoc/
39
+ /_yardoc/
40
+ /doc/
41
+ /rdoc/
42
+
43
+ ## Environment normalization:
44
+ /.bundle/
45
+ /vendor/bundle
46
+ /lib/bundler/man/
47
+
48
+ # for a library or gem, you might want to ignore these files since the code is
49
+ # intended to run in multiple environments; otherwise, check them in:
50
+ # Gemfile.lock
51
+ # .ruby-version
52
+ # .ruby-gemset
53
+
54
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
55
+ .rvmrc
56
+
57
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
58
+ # .rubocop-https?--*
@@ -0,0 +1,14 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 2.4
5
+
6
+ gemfile:
7
+ - Gemfile
8
+
9
+ branches:
10
+ only:
11
+ - master
12
+
13
+ # bundle exec rake test
14
+ script: make gem
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 Olivér Sz,
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,15 @@
1
+ gem:
2
+ rm -f fluent-plugin-azurestorage-gen2*.gem
3
+ gem build fluent-plugin-azurestorage-gen2.gemspec
4
+
5
+ install: gem
6
+ gem install fluent-plugin-azurestorage-gen2*.gem
7
+
8
+ push: gem
9
+ gem push fluent-plugin-azurestorage-gen2*.gem
10
+
11
+ tag-and-branch:
12
+ git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
13
+ git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
14
+ git push origin "v$$(cat VERSION)"
15
+ git push -u origin "release/$$(cat VERSION)"
@@ -0,0 +1,37 @@
1
+ # Azure Datalake Storage Gen2 Fluentd Output Plugin (IN PROGRESS)
2
+
3
+ [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
5
+
6
+ ## Requirements
7
+
8
+ | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
9
+ |------------------------|---------|------|
10
+ | >= 0.1.0 | >= v0.14.0 | >= 2.4 |
11
+
12
+ ## Overview
13
+
14
+ Fluent output plugin that can use ABFS api and append blobs with MSI support
15
+
16
+ ## Installation
17
+
18
+ Install from RubyGems:
19
+ ```
20
+ $ gem install fluent-plugin-azurestorage-gen2
21
+ ```
22
+
23
+ ## Configuration
24
+
25
+ TODO
26
+
27
+ ### Configuration options
28
+
29
+ TODO
30
+
31
+ ## Contributing
32
+
33
+ 1. Fork it
34
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
35
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
36
+ 4. Push to the branch (`git push origin my-new-feature`)
37
+ 5. Create new Pull Request
@@ -0,0 +1,14 @@
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rake/testtask'
6
+
7
+ Rake::TestTask.new(:test) do |test|
8
+ test.libs << 'lib' << 'test'
9
+ test.test_files = FileList['test/plugin/test_*.rb']
10
+ test.verbose = true
11
+ end
12
+
13
+ task :default => [:build]
14
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.0
@@ -0,0 +1,29 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-azurestorage-gen2"
6
+ gem.description = "Azure Storage output plugin for Fluentd event collector"
7
+ gem.license = "Apache-2.0"
8
+ gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
9
+ gem.summary = gem.description
10
+ gem.version = File.read("VERSION").strip
11
+ gem.authors = ["Oliver Szabo"]
12
+ gem.email = ["oleewere@gmail.com"]
13
+ #gem.platform = Gem::Platform::RUBY
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map {|f| File.basename(f)}
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
20
+ gem.add_runtime_dependency 'azure-storage-common', '~> 1.1', '>= 1.1.0'
21
+ gem.add_runtime_dependency 'azure-storage-blob', '~> 1.1', '>= 1.1.0'
22
+ gem.add_runtime_dependency 'uuidtools', '~> 2.1', '>= 2.1.5'
23
+ gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
24
+ gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
25
+ gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
26
+ gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
27
+ gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
28
+ gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'
29
+ end
@@ -0,0 +1,366 @@
1
+ require 'net/http'
2
+ require 'json'
3
+ require 'tempfile'
4
+ require 'time'
5
+ require 'typhoeus'
6
+ require 'fluent/plugin/output'
7
+ require 'zlib'
8
+
9
+ module Fluent::Plugin
10
+ class AzureStorageGen2Output < Fluent::Plugin::Output
11
+ Fluent::Plugin.register_output('azurestorage_gen2', self)
12
+
13
+ helpers :compat_parameters, :formatter, :inject
14
+
15
+ def initialize
16
+ super
17
+ @compressor = nil
18
+ end
19
+
20
+ config_param :path, :string, :default => ""
21
+ config_param :azure_storage_account, :string, :default => nil
22
+ config_param :azure_storage_access_key, :string, :default => nil, :secret => true
23
+ config_param :azure_instance_msi, :string, :default => nil
24
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
25
+ config_param :azure_container, :string, :default => nil
26
+ config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
27
+ config_param :file_extension, :string, :default => "log"
28
+ config_param :store_as, :string, :default => "gzip"
29
+ config_param :auto_create_container, :bool, :default => false
30
+ config_param :format, :string, :default => "out_file"
31
+ config_param :time_slice_format, :string, :default => '%Y%m%d'
32
+
33
+ DEFAULT_FORMAT_TYPE = "out_file"
34
+ URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
35
+ ACCESS_TOKEN_API_VERSION = "2018-02-01"
36
+ ABFS_API_VERSION = "2018-11-09"
37
+ AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
38
+
39
+ config_section :format do
40
+ config_set_default :@type, DEFAULT_FORMAT_TYPE
41
+ end
42
+
43
+ config_section :buffer do
44
+ config_set_default :chunk_keys, ['time']
45
+ config_set_default :timekey, (60 * 60 * 24)
46
+ end
47
+
48
+ def configure(conf)
49
+ compat_parameters_convert(conf, :buffer, :formatter, :inject)
50
+ super
51
+
52
+ @formatter = formatter_create
53
+
54
+ if @localtime
55
+ @path_slicer = Proc.new {|path|
56
+ Time.now.strftime(path)
57
+ }
58
+ else
59
+ @path_slicer = Proc.new {|path|
60
+ Time.now.utc.strftime(path)
61
+ }
62
+ end
63
+
64
+ if @azure_container.nil?
65
+ raise Fluent::ConfigError, "azure_container is needed"
66
+ end
67
+
68
+ @azure_storage_path = ''
69
+ @last_azure_storage_path = ''
70
+ @current_index = 0
71
+ end
72
+
73
+ def multi_workers_ready?
74
+ true
75
+ end
76
+
77
+ def start
78
+ setup_access_token
79
+ ensure_container
80
+ super
81
+ end
82
+
83
+ def write(chunk)
84
+ metadata = chunk.metadata
85
+
86
+ #tmp = Tempfile.new("azure-")
87
+ #begin
88
+ # tmp.close
89
+ # generate_log_name(metadata, @current_index)
90
+ # if @last_azure_storage_path != @azure_storage_path
91
+ # @current_index = 0
92
+ # generate_log_name(metadata, @current_index)
93
+ # end
94
+ # content = File.open(tmp.path, 'rb') { |file| file.read }
95
+ # raw_data = raw_data.chomp
96
+ # log.debug "Content: #{content}"
97
+ # upload_blob(content, metadata)
98
+ # @last_azure_storage_path = @azure_storage_path
99
+ #ensure
100
+ # tmp.close(true) rescue nil
101
+ # tmp.unlink
102
+ #end
103
+ raw_data=''
104
+ generate_log_name(metadata, @current_index)
105
+ if @last_azure_storage_path != @azure_storage_path
106
+ @current_index = 0
107
+ generate_log_name(metadata, @current_index)
108
+ end
109
+ chunk.each do |emit_time, record|
110
+ line = record["message"].chomp
111
+ raw_data << "#{line}\n"
112
+ end
113
+ raw_data = raw_data.chomp
114
+ unless raw_data.empty?
115
+ upload_blob(raw_data, metadata)
116
+ end
117
+ @last_azure_storage_path = @azure_storage_path
118
+
119
+ end
120
+
121
+ private
122
+ def upload_blob(content, metadata)
123
+ log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
124
+ existing_content_length = get_blob_properties(@azure_storage_path)
125
+ if existing_content_length == 0
126
+ create_blob(@azure_storage_path)
127
+ end
128
+ append_blob(content, metadata, existing_content_length)
129
+ end
130
+
131
+ def format(tag, time, record)
132
+ r = inject_values_to_record(tag, time, record)
133
+ @formatter.format(tag, time, r)
134
+ end
135
+
136
+ private
137
+ def generate_log_name(metadata, index)
138
+ time_slice = if metadata.timekey.nil?
139
+ ''.freeze
140
+ else
141
+ Time.at(metadata.timekey).utc.strftime(@time_slice_format)
142
+ end
143
+ path = @path_slicer.call(@path)
144
+ values_for_object_key = {
145
+ "%{path}" => path,
146
+ "%{time_slice}" => time_slice,
147
+ "%{index}" => index,
148
+ "%{file_extension}" => @file_extension
149
+ }
150
+ storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
151
+ extracted_path = extract_placeholders(storage_path, metadata)
152
+ extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
153
+ @azure_storage_path = extracted_path
154
+ end
155
+
156
+ def setup_access_token
157
+ @get_token_lock = Concurrent::ReadWriteLock.new
158
+ acquire_access_token
159
+ if @azure_oauth_refresh_interval > 0
160
+ log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
161
+ @get_token_task = Concurrent::TimerTask.new(
162
+ execution_interval: @azure_oauth_refresh_interval) {
163
+ begin
164
+ acquire_access_token
165
+ rescue Exception => e
166
+ log.warn("#{e.message}, continue with previous credentials.")
167
+ end
168
+ }
169
+ @get_token_task.execute
170
+ end
171
+ end
172
+
173
+ # Referenced from azure doc.
174
+ # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
175
+ def acquire_access_token
176
+ params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
177
+ unless @azure_instance_msi.nil?
178
+ params[:msi_res_id] = @azure_instance_msi
179
+ end
180
+ request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
181
+ request.on_complete do |response|
182
+ if response.success?
183
+ data = JSON.parse(response.body)
184
+ log.debug "azurestorage_gen2: Token response: #{data}"
185
+ @azure_access_token = data["access_token"]
186
+ else
187
+ raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
188
+ end
189
+ end
190
+ request.run
191
+ end
192
+
193
+ private
194
+ def ensure_container
195
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
196
+ params = {:resource => "filesystem" }
197
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
198
+ request.on_complete do |response|
199
+ if response.success?
200
+ log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
201
+ elsif response.timed_out?
202
+ raise Fluent::UnrecoverableError, "Get container '#{@azure_container}' request timed out."
203
+ elsif response.code == 404
204
+ log.info "azurestorage_gen2: Container '#{@azure_container}' does not exist. Creating it if needed..."
205
+ if @auto_create_container
206
+ create_container
207
+ else
208
+ raise Fluent::ConfigError, "The specified container does not exist: container = #{@azure_container}"
209
+ end
210
+ else
211
+ raise Fluent::UnrecoverableError, "Get container request failed - code: #{response.code}, body: #{response.body}"
212
+ end
213
+ end
214
+ request.run
215
+ end
216
+
217
+ private
218
+ def create_container
219
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
220
+ params = {:resource => "filesystem" }
221
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
222
+ request.on_complete do |response|
223
+ if response.success?
224
+ log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
225
+ elsif response.timed_out?
226
+ raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
227
+ else
228
+ raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}"
229
+ end
230
+ end
231
+ request.run
232
+ end
233
+
234
+ private
235
+ def create_blob(blob_path)
236
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0", :"Content-Type" => "application/json"}
237
+ params = {:resource => "file", :recursive => "false"}
238
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}/testprefix#{blob_path}", :method => :put, :params => params, :headers=> headers)
239
+ request.on_complete do |response|
240
+ if response.success?
241
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
242
+ elsif response.timed_out?
243
+ raise Fluent::UnrecoverableError, "Creating blob '#{@blob_path}' request timed out."
244
+ elsif response.code == 409
245
+ log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
246
+ else
247
+ raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
248
+ end
249
+ end
250
+ request.run
251
+ end
252
+
253
+ private
254
+ def append_blob_block(blob_path, content, position)
255
+ log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
256
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-content-type" => "text/plain", :"Authorization" => "Bearer #{@azure_access_token}"}
257
+ params = {:action => "append", :position => "#{position}"}
258
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
259
+ request.on_complete do |response|
260
+ if response.success?
261
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
262
+ elsif response.timed_out?
263
+ raise Fluent::UnrecoverableError, "Appending blob #{@blob_path}' request timed out."
264
+ elsif response.code == 404
265
+ raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
266
+ elsif response.code == 409
267
+ raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
268
+ else
269
+ raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
270
+ end
271
+ end
272
+ request.run
273
+ end
274
+
275
+ private
276
+ def flush(blob_path, position)
277
+ log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
278
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
279
+ params = {:action => "flush", :position => "#{position}"}
280
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
281
+ request.on_complete do |response|
282
+ if response.success?
283
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
284
+ elsif response.timed_out?
285
+ raise Fluent::UnrecoverableError, "Bloub '#{@blob_path}' flush request timed out."
286
+ else
287
+ raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}"
288
+ end
289
+ end
290
+ request.run
291
+ end
292
+
293
+ private
294
+ def get_blob_properties(blob_path)
295
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
296
+ params = {}
297
+ content_length = -1
298
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
299
+ request.on_complete do |response|
300
+ if response.success?
301
+ log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
302
+ content_length = response.headers['Content-Length'].to_i
303
+ elsif response.timed_out?
304
+ raise Fluent::UnrecoverableError, "Get blob properties '#{@blob_path}' request timed out."
305
+ elsif response.code == 404
306
+ log.debug "azurestorage_gen2: Blob '#{@blob_path}' does not exist. Creating it if needed..."
307
+ content_length = 0
308
+ else
309
+ raise Fluent::UnrecoverableError, "Get blob properties '#{@blob_path}' request failed - code: #{response.code}, body: #{response.body}"
310
+ end
311
+ end
312
+ request.run
313
+ content_length
314
+ end
315
+
316
+ private
317
+ def append_blob(content, metadata, existing_content_length)
318
+ position = 0
319
+ log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
320
+ loop do
321
+ begin
322
+ size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
323
+ log.debug "azurestorage_gen2: append_blob.chunk: content[#{position}..#{position + size}]"
324
+ append_blob_block(@azure_storage_path, content[position..position + size], existing_content_length)
325
+
326
+ position += size
327
+ existing_content_length += size
328
+ break if position >= content.length
329
+ rescue AppendBlobResponseError => ex
330
+ status_code = ex.status_code
331
+
332
+ if status_code == 409 # exceeds azure block limit
333
+ @current_index += 1
334
+ old_azure_storage_path = @azure_storage_path
335
+ generate_log_name(metadata, @current_index)
336
+
337
+ # If index is not a part of format, rethrow exception.
338
+ if old_azure_storage_path == @azure_storage_path
339
+ log.warn "azurestorage_gen2: append_blob: blocks limit reached, you need to use %{index} for the format."
340
+ raise
341
+ end
342
+ flush(old_azure_storage_path, existing_content_length)
343
+
344
+ log.info "azurestorage_gen2: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
345
+ create_blob(@azure_storage_path)
346
+ elsif status_code == 404 # blob not found
347
+ log.debug "azurestorage_gen2: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
348
+ create_blob(@azure_storage_path)
349
+ else
350
+ raise
351
+ end
352
+ end
353
+ end
354
+ flush(@azure_storage_path, existing_content_length)
355
+ log.debug "azurestorage_gen2: append_blob.complete"
356
+ end
357
+ end
358
+
359
+ class AppendBlobResponseError < StandardError
360
+ attr_reader :status_code
361
+ def initialize(message="Default message", status_code=0)
362
+ @status_code = status_code
363
+ super(message)
364
+ end
365
+ end
366
+ end
@@ -0,0 +1,8 @@
1
+ $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
2
+ require "test-unit"
3
+ require "fluent/test"
4
+ require "fluent/test/driver/output"
5
+ require "fluent/test/helpers"
6
+
7
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
8
+ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -0,0 +1,54 @@
1
+ require 'fluent/test'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/test/helpers'
4
+ require 'fluent/plugin/out_azurestorage_gen2'
5
+
6
+ require 'test/unit/rr'
7
+ require 'zlib'
8
+ require 'fileutils'
9
+
10
+ include Fluent::Test::Helpers
11
+
12
+ class AzureStorageGen2OutputTest < Test::Unit::TestCase
13
+ def setup
14
+ Fluent::Test.setup
15
+ end
16
+
17
+ CONFIG = %[
18
+ azure_storage_account test_storage_account
19
+ azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
20
+ azure_container test_container
21
+ path log
22
+ utc
23
+ buffer_type memory
24
+ ]
25
+
26
+ def create_driver(conf = CONFIG)
27
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageGen2Output) do
28
+ # for testing.
29
+ def contents
30
+ @emit_streams
31
+ end
32
+
33
+ def write(chunk)
34
+ @emit_streams = []
35
+ event = chunk.read
36
+ @emit_streams << event
37
+ end
38
+
39
+ private
40
+ def ensure_container
41
+ end
42
+
43
+ end.configure(conf)
44
+ end
45
+
46
+ def test_configure
47
+ # TODO write tests
48
+ d = create_driver
49
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
50
+ assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
51
+ assert_equal 'test_container', d.instance.azure_container
52
+ assert_equal 'log', d.instance.path
53
+ end
54
+ end
metadata ADDED
@@ -0,0 +1,252 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-azurestorage-gen2
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Oliver Szabo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-12-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '1.0'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '2'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '1.0'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: azure-storage-common
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '1.1'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 1.1.0
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '1.1'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 1.1.0
53
+ - !ruby/object:Gem::Dependency
54
+ name: azure-storage-blob
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '1.1'
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: 1.1.0
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '1.1'
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: 1.1.0
73
+ - !ruby/object:Gem::Dependency
74
+ name: uuidtools
75
+ requirement: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '2.1'
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: 2.1.5
83
+ type: :runtime
84
+ prerelease: false
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '2.1'
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: 2.1.5
93
+ - !ruby/object:Gem::Dependency
94
+ name: typhoeus
95
+ requirement: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - "~>"
98
+ - !ruby/object:Gem::Version
99
+ version: '1.0'
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: 1.0.1
103
+ type: :runtime
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '1.0'
110
+ - - ">="
111
+ - !ruby/object:Gem::Version
112
+ version: 1.0.1
113
+ - !ruby/object:Gem::Dependency
114
+ name: json
115
+ requirement: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - "~>"
118
+ - !ruby/object:Gem::Version
119
+ version: '2.1'
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: 2.1.0
123
+ type: :runtime
124
+ prerelease: false
125
+ version_requirements: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - "~>"
128
+ - !ruby/object:Gem::Version
129
+ version: '2.1'
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: 2.1.0
133
+ - !ruby/object:Gem::Dependency
134
+ name: yajl-ruby
135
+ requirement: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - "~>"
138
+ - !ruby/object:Gem::Version
139
+ version: '1.4'
140
+ type: :runtime
141
+ prerelease: false
142
+ version_requirements: !ruby/object:Gem::Requirement
143
+ requirements:
144
+ - - "~>"
145
+ - !ruby/object:Gem::Version
146
+ version: '1.4'
147
+ - !ruby/object:Gem::Dependency
148
+ name: rake
149
+ requirement: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - "~>"
152
+ - !ruby/object:Gem::Version
153
+ version: '12.3'
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: 12.3.1
157
+ type: :development
158
+ prerelease: false
159
+ version_requirements: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - "~>"
162
+ - !ruby/object:Gem::Version
163
+ version: '12.3'
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: 12.3.1
167
+ - !ruby/object:Gem::Dependency
168
+ name: test-unit
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '3.3'
174
+ - - ">="
175
+ - !ruby/object:Gem::Version
176
+ version: 3.3.3
177
+ type: :development
178
+ prerelease: false
179
+ version_requirements: !ruby/object:Gem::Requirement
180
+ requirements:
181
+ - - "~>"
182
+ - !ruby/object:Gem::Version
183
+ version: '3.3'
184
+ - - ">="
185
+ - !ruby/object:Gem::Version
186
+ version: 3.3.3
187
+ - !ruby/object:Gem::Dependency
188
+ name: test-unit-rr
189
+ requirement: !ruby/object:Gem::Requirement
190
+ requirements:
191
+ - - "~>"
192
+ - !ruby/object:Gem::Version
193
+ version: '1.0'
194
+ - - ">="
195
+ - !ruby/object:Gem::Version
196
+ version: 1.0.5
197
+ type: :development
198
+ prerelease: false
199
+ version_requirements: !ruby/object:Gem::Requirement
200
+ requirements:
201
+ - - "~>"
202
+ - !ruby/object:Gem::Version
203
+ version: '1.0'
204
+ - - ">="
205
+ - !ruby/object:Gem::Version
206
+ version: 1.0.5
207
+ description: Azure Storage output plugin for Fluentd event collector
208
+ email:
209
+ - oleewere@gmail.com
210
+ executables: []
211
+ extensions: []
212
+ extra_rdoc_files: []
213
+ files:
214
+ - ".gitignore"
215
+ - ".travis.yml"
216
+ - Gemfile
217
+ - LICENSE
218
+ - Makefile
219
+ - README.md
220
+ - Rakefile
221
+ - VERSION
222
+ - fluent-plugin-azurestorage-gen2.gemspec
223
+ - lib/fluent/plugin/out_azurestorage_gen2.rb
224
+ - test/helper.rb
225
+ - test/plugin/test_out_azurestorage_gen2.rb
226
+ homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
227
+ licenses:
228
+ - Apache-2.0
229
+ metadata: {}
230
+ post_install_message:
231
+ rdoc_options: []
232
+ require_paths:
233
+ - lib
234
+ required_ruby_version: !ruby/object:Gem::Requirement
235
+ requirements:
236
+ - - ">="
237
+ - !ruby/object:Gem::Version
238
+ version: '0'
239
+ required_rubygems_version: !ruby/object:Gem::Requirement
240
+ requirements:
241
+ - - ">="
242
+ - !ruby/object:Gem::Version
243
+ version: '0'
244
+ requirements: []
245
+ rubyforge_project:
246
+ rubygems_version: 2.5.1
247
+ signing_key:
248
+ specification_version: 4
249
+ summary: Azure Storage output plugin for Fluentd event collector
250
+ test_files:
251
+ - test/helper.rb
252
+ - test/plugin/test_out_azurestorage_gen2.rb