fluent-plugin-azurestorage-gen2 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6f15931cc13f12c4aeb6723ba2a44c604998e819
4
+ data.tar.gz: 557964c5e84e0274a840af3e18500d9197ea8cd4
5
+ SHA512:
6
+ metadata.gz: 7d0e97ee7d6b1429e98f91aa72135c4f84bf4cfd8b573cc8ec747cb36fe0e65fb98bce5d02d5416db83a4946c406c93f27ede48a1ba989d25bf675771311e6cd
7
+ data.tar.gz: 6ed3f231452a94eb0e66244648be66500af2fd82577b3796df11d0cda363238b95b50cc478ae01b54d1e8deb4cdb12101151ccfecb8177f53923db97b395248b
@@ -0,0 +1,58 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+ GEMSPEC_ARGS
13
+ Gemfile.lock
14
+
15
+ # Used by dotenv library to load environment variables.
16
+ # .env
17
+
18
+ # Ignore Byebug command history file.
19
+ .byebug_history
20
+
21
+ ## Specific to RubyMotion:
22
+ .dat*
23
+ .repl_history
24
+ build/
25
+ *.bridgesupport
26
+ build-iPhoneOS/
27
+ build-iPhoneSimulator/
28
+
29
+ ## Specific to RubyMotion (use of CocoaPods):
30
+ #
31
+ # We recommend against adding the Pods directory to your .gitignore. However
32
+ # you should judge for yourself, the pros and cons are mentioned at:
33
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
34
+ #
35
+ # vendor/Pods/
36
+
37
+ ## Documentation cache and generated files:
38
+ /.yardoc/
39
+ /_yardoc/
40
+ /doc/
41
+ /rdoc/
42
+
43
+ ## Environment normalization:
44
+ /.bundle/
45
+ /vendor/bundle
46
+ /lib/bundler/man/
47
+
48
+ # for a library or gem, you might want to ignore these files since the code is
49
+ # intended to run in multiple environments; otherwise, check them in:
50
+ # Gemfile.lock
51
+ # .ruby-version
52
+ # .ruby-gemset
53
+
54
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
55
+ .rvmrc
56
+
57
+ # Used by RuboCop. Remote config files pulled in from inherit_from directive.
58
+ # .rubocop-https?--*
@@ -0,0 +1,14 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 2.4
5
+
6
+ gemfile:
7
+ - Gemfile
8
+
9
+ branches:
10
+ only:
11
+ - master
12
+
13
+ # bundle exec rake test
14
+ script: make gem
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 Olivér Sz,
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,15 @@
1
+ gem:
2
+ rm -f fluent-plugin-azurestorage-gen2*.gem
3
+ gem build fluent-plugin-azurestorage-gen2.gemspec
4
+
5
+ install: gem
6
+ gem install fluent-plugin-azurestorage-gen2*.gem
7
+
8
+ push: gem
9
+ gem push fluent-plugin-azurestorage-gen2*.gem
10
+
11
+ tag-and-branch:
12
+ git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
13
+ git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
14
+ git push origin "v$$(cat VERSION)"
15
+ git push -u origin "release/$$(cat VERSION)"
@@ -0,0 +1,216 @@
1
+ # Azure Datalake Storage Gen2 Fluentd Output Plugin (IN PROGRESS)
2
+
3
+ [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
+ [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
6
+ ![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?extension=png)
7
+
8
+ ## Requirements
9
+
10
+ | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
11
+ |------------------------|---------|------|
12
+ | >= 0.1.1 | >= v0.14.0 | >= 2.4 |
13
+
14
+ ## Overview
15
+
16
+ Fluent output plugin that can use ABFS api and append blobs with MSI support
17
+
18
+ ## Installation
19
+
20
+ Install from RubyGems:
21
+ ```
22
+ $ gem install fluent-plugin-azurestorage-gen2
23
+ ```
24
+
25
+ ## Configuration
26
+
27
+ ```
28
+ <match **>
29
+ @type azurestorage_gen2
30
+ azure_storage_account mystorageabfs
31
+ azure_container mycontainer
32
+ azure_instance_msi /subscriptions/mysubscriptionid
33
+ azure_object_key_format %{path}-%{index}.%{file_extension}
34
+ azure_oauth_refresh_interval 3600
35
+ time_slice_format %Y%m%d-%H
36
+ file_extension log
37
+ path "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
38
+ auto_create_container true
39
+ <buffer tag,time>
40
+ @type file
41
+ path /var/log/fluent/azurestorage-buffer
42
+ timekey 5m
43
+ timekey_wait 0s
44
+ timekey_use_utc true
45
+ chunk_limit_size 64m
46
+ </buffer>
47
+ </match>
48
+ ```
49
+
50
+ ### Configuration options
51
+
52
+ ### azure_storage_account
53
+
54
+ Your Azure Storage Account Name. This can be got from Azure Management potal.
55
+ This parameter is required when environment variable 'AZURE_STORAGE_ACCOUNT' is not set.
56
+
57
+ ### azure_storage_access_key (not implemented yet - use msi)
58
+
59
+ Your Azure Storage Access Key(Primary or Secondary). This also can be got from Azure Management potal. Storage access key authentication is used when this parameter is provided or environment variable 'AZURE_STORAGE_ACCESS_KEY' is set.
60
+
61
+ ### azure_instance_msi
62
+
63
+ Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
64
+
65
+ ### azure_oauth_refresh_interval
66
+
67
+ OAuth2 access token refreshment interval in second. Only applies when MSI authentication is used.
68
+
69
+ ### azure_container (Required)
70
+
71
+ Azure Storage Container name
72
+
73
+ ### auto_create_container
74
+
75
+ This plugin create container if not exist when you set 'auto_create_container' to true.
76
+
77
+ ### azure_object_key_format
78
+
79
+ The format of Azure Storage object keys. You can use several built-in variables:
80
+
81
+ - %{path}
82
+ - %{time_slice}
83
+ - %{index}
84
+ - %{file_extension}
85
+
86
+ to decide keys dynamically.
87
+
88
+ %{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
89
+ %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
90
+ %{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
91
+ %{file_extention} is always "gz" for now.
92
+
93
+ The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
94
+
95
+ For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
96
+
97
+ ```
98
+ "logs/20130111-22_0.gz"
99
+ "logs/20130111-23_0.gz"
100
+ "logs/20130111-23_1.gz"
101
+ "logs/20130112-00_0.gz"
102
+ ```
103
+
104
+ With the configuration:
105
+
106
+ ```
107
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}.%{file_extension}
108
+ path log
109
+ time_slice_format %Y%m%d-%H
110
+ ```
111
+
112
+ You get:
113
+
114
+ ```
115
+ "log/events/ts=20130111-22/events_0.gz"
116
+ "log/events/ts=20130111-23/events_0.gz"
117
+ "log/events/ts=20130111-23/events_1.gz"
118
+ "log/events/ts=20130112-00/events_0.gz"
119
+ ```
120
+
121
+ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, %{uuid}, etc. can be used in the azure_object_key_format. This could prove useful in preventing filename conflicts when writing from multiple servers.
122
+
123
+ ```
124
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
125
+ ```
126
+
127
+ ### format
128
+
129
+ Change one line format in the Azure Storage object. Supported formats are 'out_file', 'json', 'ltsv' and 'single_value'.
130
+
131
+ - out_file (default)
132
+
133
+ ```
134
+ time\ttag\t{..json1..}
135
+ time\ttag\t{..json2..}
136
+ ...
137
+ ```
138
+
139
+ - json
140
+
141
+ ```
142
+ {..json1..}
143
+ {..json2..}
144
+ ...
145
+ ```
146
+
147
+ At this format, "time" and "tag" are omitted.
148
+ But you can set these information to the record by setting "include_tag_key" / "tag_key" and "include_time_key" / "time_key" option.
149
+ If you set following configuration in AzureStorage output:
150
+
151
+ ```
152
+ format json
153
+ include_time_key true
154
+ time_key log_time # default is time
155
+ ```
156
+
157
+ then the record has log_time field.
158
+
159
+ ```
160
+ {"log_time":"time string",...}
161
+ ```
162
+
163
+ - ltsv
164
+
165
+ ```
166
+ key1:value1\tkey2:value2
167
+ key1:value1\tkey2:value2
168
+ ...
169
+ ```
170
+
171
+ "ltsv" format also accepts "include_xxx" related options. See "json" section.
172
+
173
+ - single_value
174
+
175
+ Use specified value instead of entire recode. If you get '{"message":"my log"}', then contents are
176
+
177
+ ```
178
+ my log1
179
+ my log2
180
+ ...
181
+ ```
182
+
183
+ You can change key name by "message_key" option.
184
+
185
+ ### path
186
+
187
+ path prefix of the files on Azure Storage. Default is "" (no prefix).
188
+
189
+ ### buffer_path (required)
190
+
191
+ path prefix of the files to buffer logs.
192
+
193
+ ### time_slice_format
194
+
195
+ Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
196
+
197
+ ### time_slice_wait
198
+
199
+ The time to wait old logs. Default is 10 minutes.
200
+
201
+ ### utc
202
+
203
+ Use UTC instead of local time.
204
+
205
+ ## TODOs
206
+
207
+ - add storage key support
208
+ - add compression (if append is not used)
209
+
210
+ ## Contributing
211
+
212
+ 1. Fork it
213
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
214
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
215
+ 4. Push to the branch (`git push origin my-new-feature`)
216
+ 5. Create new Pull Request
@@ -0,0 +1,14 @@
1
+
2
+ require 'bundler'
3
+ Bundler::GemHelper.install_tasks
4
+
5
+ require 'rake/testtask'
6
+
7
+ Rake::TestTask.new(:test) do |test|
8
+ test.libs << 'lib' << 'test'
9
+ test.test_files = FileList['test/plugin/test_*.rb']
10
+ test.verbose = true
11
+ end
12
+
13
+ task :default => [:build]
14
+
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.1
@@ -0,0 +1,27 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-azurestorage-gen2"
6
+ gem.description = "Azure Storage output plugin for Fluentd event collector"
7
+ gem.license = "Apache-2.0"
8
+ gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
9
+ gem.summary = gem.description
10
+ gem.version = File.read("VERSION").strip
11
+ gem.authors = ["Oliver Szabo"]
12
+ gem.email = ["oleewere@gmail.com"]
13
+ #gem.platform = Gem::Platform::RUBY
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map {|f| File.basename(f)}
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
20
+ gem.add_runtime_dependency 'uuidtools', '~> 2.1', '>= 2.1.5'
21
+ gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
22
+ gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
23
+ gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
24
+ gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
25
+ gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
26
+ gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'
27
+ end
@@ -0,0 +1,366 @@
1
+ require 'net/http'
2
+ require 'json'
3
+ require 'tempfile'
4
+ require 'time'
5
+ require 'typhoeus'
6
+ require 'fluent/plugin/output'
7
+ require 'zlib'
8
+
9
+ module Fluent::Plugin
10
+ class AzureStorageGen2Output < Fluent::Plugin::Output
11
+ Fluent::Plugin.register_output('azurestorage_gen2', self)
12
+
13
+ helpers :compat_parameters, :formatter, :inject
14
+
15
+ def initialize
16
+ super
17
+ @compressor = nil
18
+ end
19
+
20
+ config_param :path, :string, :default => ""
21
+ config_param :azure_storage_account, :string, :default => nil
22
+ config_param :azure_storage_access_key, :string, :default => nil, :secret => true
23
+ config_param :azure_instance_msi, :string, :default => nil
24
+ config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
25
+ config_param :azure_container, :string, :default => nil
26
+ config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
27
+ config_param :file_extension, :string, :default => "log"
28
+ config_param :store_as, :string, :default => "gzip"
29
+ config_param :auto_create_container, :bool, :default => false
30
+ config_param :format, :string, :default => "out_file"
31
+ config_param :time_slice_format, :string, :default => '%Y%m%d'
32
+
33
+ DEFAULT_FORMAT_TYPE = "out_file"
34
+ URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
35
+ ACCESS_TOKEN_API_VERSION = "2018-02-01"
36
+ ABFS_API_VERSION = "2018-11-09"
37
+ AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
38
+
39
+ config_section :format do
40
+ config_set_default :@type, DEFAULT_FORMAT_TYPE
41
+ end
42
+
43
+ config_section :buffer do
44
+ config_set_default :chunk_keys, ['time']
45
+ config_set_default :timekey, (60 * 60 * 24)
46
+ end
47
+
48
+ def configure(conf)
49
+ compat_parameters_convert(conf, :buffer, :formatter, :inject)
50
+ super
51
+
52
+ @formatter = formatter_create
53
+
54
+ if @localtime
55
+ @path_slicer = Proc.new {|path|
56
+ Time.now.strftime(path)
57
+ }
58
+ else
59
+ @path_slicer = Proc.new {|path|
60
+ Time.now.utc.strftime(path)
61
+ }
62
+ end
63
+
64
+ if @azure_container.nil?
65
+ raise Fluent::ConfigError, "azure_container is needed"
66
+ end
67
+
68
+ @azure_storage_path = ''
69
+ @last_azure_storage_path = ''
70
+ @current_index = 0
71
+ end
72
+
73
+ def multi_workers_ready?
74
+ true
75
+ end
76
+
77
+ def start
78
+ setup_access_token
79
+ ensure_container
80
+ super
81
+ end
82
+
83
+ def write(chunk)
84
+ metadata = chunk.metadata
85
+
86
+ #tmp = Tempfile.new("azure-")
87
+ #begin
88
+ # tmp.close
89
+ # generate_log_name(metadata, @current_index)
90
+ # if @last_azure_storage_path != @azure_storage_path
91
+ # @current_index = 0
92
+ # generate_log_name(metadata, @current_index)
93
+ # end
94
+ # content = File.open(tmp.path, 'rb') { |file| file.read }
95
+ # raw_data = raw_data.chomp
96
+ # log.debug "Content: #{content}"
97
+ # upload_blob(content, metadata)
98
+ # @last_azure_storage_path = @azure_storage_path
99
+ #ensure
100
+ # tmp.close(true) rescue nil
101
+ # tmp.unlink
102
+ #end
103
+ raw_data=''
104
+ generate_log_name(metadata, @current_index)
105
+ if @last_azure_storage_path != @azure_storage_path
106
+ @current_index = 0
107
+ generate_log_name(metadata, @current_index)
108
+ end
109
+ chunk.each do |emit_time, record|
110
+ line = record["message"].chomp
111
+ raw_data << "#{line}\n"
112
+ end
113
+ raw_data = raw_data.chomp
114
+ unless raw_data.empty?
115
+ upload_blob(raw_data, metadata)
116
+ end
117
+ @last_azure_storage_path = @azure_storage_path
118
+
119
+ end
120
+
121
+ private
122
+ def upload_blob(content, metadata)
123
+ log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
124
+ existing_content_length = get_blob_properties(@azure_storage_path)
125
+ if existing_content_length == 0
126
+ create_blob(@azure_storage_path)
127
+ end
128
+ append_blob(content, metadata, existing_content_length)
129
+ end
130
+
131
+ def format(tag, time, record)
132
+ r = inject_values_to_record(tag, time, record)
133
+ @formatter.format(tag, time, r)
134
+ end
135
+
136
+ private
137
+ def generate_log_name(metadata, index)
138
+ time_slice = if metadata.timekey.nil?
139
+ ''.freeze
140
+ else
141
+ Time.at(metadata.timekey).utc.strftime(@time_slice_format)
142
+ end
143
+ path = @path_slicer.call(@path)
144
+ values_for_object_key = {
145
+ "%{path}" => path,
146
+ "%{time_slice}" => time_slice,
147
+ "%{index}" => index,
148
+ "%{file_extension}" => @file_extension
149
+ }
150
+ storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
151
+ extracted_path = extract_placeholders(storage_path, metadata)
152
+ extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
153
+ @azure_storage_path = extracted_path
154
+ end
155
+
156
+ def setup_access_token
157
+ @get_token_lock = Concurrent::ReadWriteLock.new
158
+ acquire_access_token
159
+ if @azure_oauth_refresh_interval > 0
160
+ log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
161
+ @get_token_task = Concurrent::TimerTask.new(
162
+ execution_interval: @azure_oauth_refresh_interval) {
163
+ begin
164
+ acquire_access_token
165
+ rescue Exception => e
166
+ log.warn("#{e.message}, continue with previous credentials.")
167
+ end
168
+ }
169
+ @get_token_task.execute
170
+ end
171
+ end
172
+
173
+ # Referenced from azure doc.
174
+ # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
175
+ def acquire_access_token
176
+ params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
177
+ unless @azure_instance_msi.nil?
178
+ params[:msi_res_id] = @azure_instance_msi
179
+ end
180
+ request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
181
+ request.on_complete do |response|
182
+ if response.success?
183
+ data = JSON.parse(response.body)
184
+ log.debug "azurestorage_gen2: Token response: #{data}"
185
+ @azure_access_token = data["access_token"]
186
+ else
187
+ raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
188
+ end
189
+ end
190
+ request.run
191
+ end
192
+
193
+ private
194
+ def ensure_container
195
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
196
+ params = {:resource => "filesystem" }
197
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
198
+ request.on_complete do |response|
199
+ if response.success?
200
+ log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
201
+ elsif response.timed_out?
202
+ raise Fluent::UnrecoverableError, "Get container '#{@azure_container}' request timed out."
203
+ elsif response.code == 404
204
+ log.info "azurestorage_gen2: Container '#{@azure_container}' does not exist. Creating it if needed..."
205
+ if @auto_create_container
206
+ create_container
207
+ else
208
+ raise Fluent::ConfigError, "The specified container does not exist: container = #{@azure_container}"
209
+ end
210
+ else
211
+ raise Fluent::UnrecoverableError, "Get container request failed - code: #{response.code}, body: #{response.body}"
212
+ end
213
+ end
214
+ request.run
215
+ end
216
+
217
+ private
218
+ def create_container
219
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
220
+ params = {:resource => "filesystem" }
221
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
222
+ request.on_complete do |response|
223
+ if response.success?
224
+ log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
225
+ elsif response.timed_out?
226
+ raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
227
+ else
228
+ raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}"
229
+ end
230
+ end
231
+ request.run
232
+ end
233
+
234
+ private
235
+ def create_blob(blob_path)
236
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0", :"Content-Type" => "application/json"}
237
+ params = {:resource => "file", :recursive => "false"}
238
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
239
+ request.on_complete do |response|
240
+ if response.success?
241
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
242
+ elsif response.timed_out?
243
+ raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request timed out."
244
+ elsif response.code == 409
245
+ log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
246
+ else
247
+ raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
248
+ end
249
+ end
250
+ request.run
251
+ end
252
+
253
+ private
254
+ def append_blob_block(blob_path, content, position)
255
+ log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
256
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"x-ms-content-type" => "text/plain", :"Authorization" => "Bearer #{@azure_access_token}"}
257
+ params = {:action => "append", :position => "#{position}"}
258
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
259
+ request.on_complete do |response|
260
+ if response.success?
261
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
262
+ elsif response.timed_out?
263
+ raise Fluent::UnrecoverableError, "Appending blob #{blob_path}' request timed out."
264
+ elsif response.code == 404
265
+ raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
266
+ elsif response.code == 409
267
+ raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
268
+ else
269
+ raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
270
+ end
271
+ end
272
+ request.run
273
+ end
274
+
275
+ private
276
+ def flush(blob_path, position)
277
+ log.debug "azurestorage_gen2: flush_blob.start: Flush blob ('#{blob_path}') called with position #{position}"
278
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
279
+ params = {:action => "flush", :position => "#{position}"}
280
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
281
+ request.on_complete do |response|
282
+ if response.success?
283
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
284
+ elsif response.timed_out?
285
+ raise Fluent::UnrecoverableError, "Bloub '#{blob_path}' flush request timed out."
286
+ else
287
+ raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}"
288
+ end
289
+ end
290
+ request.run
291
+ end
292
+
293
+ private
294
+ def get_blob_properties(blob_path)
295
+ headers = {:"x-ms-version" => ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0"}
296
+ params = {}
297
+ content_length = -1
298
+ request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
299
+ request.on_complete do |response|
300
+ if response.success?
301
+ log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
302
+ content_length = response.headers['Content-Length'].to_i
303
+ elsif response.timed_out?
304
+ raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request timed out."
305
+ elsif response.code == 404
306
+ log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
307
+ content_length = 0
308
+ else
309
+ raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
310
+ end
311
+ end
312
+ request.run
313
+ content_length
314
+ end
315
+
316
+ private
317
+ def append_blob(content, metadata, existing_content_length)
318
+ position = 0
319
+ log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
320
+ loop do
321
+ begin
322
+ size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
323
+ log.debug "azurestorage_gen2: append_blob.chunk: content[#{position}..#{position + size}]"
324
+ append_blob_block(@azure_storage_path, content[position..position + size], existing_content_length)
325
+
326
+ position += size
327
+ existing_content_length += size
328
+ break if position >= content.length
329
+ rescue AppendBlobResponseError => ex
330
+ status_code = ex.status_code
331
+
332
+ if status_code == 409 # exceeds azure block limit
333
+ @current_index += 1
334
+ old_azure_storage_path = @azure_storage_path
335
+ generate_log_name(metadata, @current_index)
336
+
337
+ # If index is not a part of format, rethrow exception.
338
+ if old_azure_storage_path == @azure_storage_path
339
+ log.warn "azurestorage_gen2: append_blob: blocks limit reached, you need to use %{index} for the format."
340
+ raise
341
+ end
342
+ flush(old_azure_storage_path, existing_content_length)
343
+
344
+ log.info "azurestorage_gen2: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
345
+ create_blob(@azure_storage_path)
346
+ elsif status_code == 404 # blob not found
347
+ log.debug "azurestorage_gen2: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
348
+ create_blob(@azure_storage_path)
349
+ else
350
+ raise
351
+ end
352
+ end
353
+ end
354
+ flush(@azure_storage_path, existing_content_length)
355
+ log.debug "azurestorage_gen2: append_blob.complete"
356
+ end
357
+ end
358
+
359
+ class AppendBlobResponseError < StandardError
360
+ attr_reader :status_code
361
+ def initialize(message="Default message", status_code=0)
362
+ @status_code = status_code
363
+ super(message)
364
+ end
365
+ end
366
+ end
@@ -0,0 +1,8 @@
1
+ $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
2
+ require "test-unit"
3
+ require "fluent/test"
4
+ require "fluent/test/driver/output"
5
+ require "fluent/test/helpers"
6
+
7
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
8
+ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -0,0 +1,54 @@
1
+ require 'fluent/test'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/test/helpers'
4
+ require 'fluent/plugin/out_azurestorage_gen2'
5
+
6
+ require 'test/unit/rr'
7
+ require 'zlib'
8
+ require 'fileutils'
9
+
10
+ include Fluent::Test::Helpers
11
+
12
+ class AzureStorageGen2OutputTest < Test::Unit::TestCase
13
+ def setup
14
+ Fluent::Test.setup
15
+ end
16
+
17
+ CONFIG = %[
18
+ azure_storage_account test_storage_account
19
+ azure_storage_access_key dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=
20
+ azure_container test_container
21
+ path log
22
+ utc
23
+ buffer_type memory
24
+ ]
25
+
26
+ def create_driver(conf = CONFIG)
27
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageGen2Output) do
28
+ # for testing.
29
+ def contents
30
+ @emit_streams
31
+ end
32
+
33
+ def write(chunk)
34
+ @emit_streams = []
35
+ event = chunk.read
36
+ @emit_streams << event
37
+ end
38
+
39
+ private
40
+ def ensure_container
41
+ end
42
+
43
+ end.configure(conf)
44
+ end
45
+
46
+ def test_configure
47
+ # TODO write tests
48
+ d = create_driver
49
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
50
+ assert_equal 'dGVzdF9zdG9yYWdlX2FjY2Vzc19rZXk=', d.instance.azure_storage_access_key
51
+ assert_equal 'test_container', d.instance.azure_container
52
+ assert_equal 'log', d.instance.path
53
+ end
54
+ end
metadata ADDED
@@ -0,0 +1,212 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-azurestorage-gen2
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Oliver Szabo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-12-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '1.0'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '2'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '1.0'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '2'
33
+ - !ruby/object:Gem::Dependency
34
+ name: uuidtools
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '2.1'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 2.1.5
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '2.1'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 2.1.5
53
+ - !ruby/object:Gem::Dependency
54
+ name: typhoeus
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: '1.0'
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: 1.0.1
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: '1.0'
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: 1.0.1
73
+ - !ruby/object:Gem::Dependency
74
+ name: json
75
+ requirement: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '2.1'
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: 2.1.0
83
+ type: :runtime
84
+ prerelease: false
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '2.1'
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: 2.1.0
93
+ - !ruby/object:Gem::Dependency
94
+ name: yajl-ruby
95
+ requirement: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - "~>"
98
+ - !ruby/object:Gem::Version
99
+ version: '1.4'
100
+ type: :runtime
101
+ prerelease: false
102
+ version_requirements: !ruby/object:Gem::Requirement
103
+ requirements:
104
+ - - "~>"
105
+ - !ruby/object:Gem::Version
106
+ version: '1.4'
107
+ - !ruby/object:Gem::Dependency
108
+ name: rake
109
+ requirement: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - "~>"
112
+ - !ruby/object:Gem::Version
113
+ version: '12.3'
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: 12.3.1
117
+ type: :development
118
+ prerelease: false
119
+ version_requirements: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - "~>"
122
+ - !ruby/object:Gem::Version
123
+ version: '12.3'
124
+ - - ">="
125
+ - !ruby/object:Gem::Version
126
+ version: 12.3.1
127
+ - !ruby/object:Gem::Dependency
128
+ name: test-unit
129
+ requirement: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - "~>"
132
+ - !ruby/object:Gem::Version
133
+ version: '3.3'
134
+ - - ">="
135
+ - !ruby/object:Gem::Version
136
+ version: 3.3.3
137
+ type: :development
138
+ prerelease: false
139
+ version_requirements: !ruby/object:Gem::Requirement
140
+ requirements:
141
+ - - "~>"
142
+ - !ruby/object:Gem::Version
143
+ version: '3.3'
144
+ - - ">="
145
+ - !ruby/object:Gem::Version
146
+ version: 3.3.3
147
+ - !ruby/object:Gem::Dependency
148
+ name: test-unit-rr
149
+ requirement: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - "~>"
152
+ - !ruby/object:Gem::Version
153
+ version: '1.0'
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: 1.0.5
157
+ type: :development
158
+ prerelease: false
159
+ version_requirements: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - "~>"
162
+ - !ruby/object:Gem::Version
163
+ version: '1.0'
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: 1.0.5
167
+ description: Azure Storage output plugin for Fluentd event collector
168
+ email:
169
+ - oleewere@gmail.com
170
+ executables: []
171
+ extensions: []
172
+ extra_rdoc_files: []
173
+ files:
174
+ - ".gitignore"
175
+ - ".travis.yml"
176
+ - Gemfile
177
+ - LICENSE
178
+ - Makefile
179
+ - README.md
180
+ - Rakefile
181
+ - VERSION
182
+ - fluent-plugin-azurestorage-gen2.gemspec
183
+ - lib/fluent/plugin/out_azurestorage_gen2.rb
184
+ - test/helper.rb
185
+ - test/plugin/test_out_azurestorage_gen2.rb
186
+ homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
187
+ licenses:
188
+ - Apache-2.0
189
+ metadata: {}
190
+ post_install_message:
191
+ rdoc_options: []
192
+ require_paths:
193
+ - lib
194
+ required_ruby_version: !ruby/object:Gem::Requirement
195
+ requirements:
196
+ - - ">="
197
+ - !ruby/object:Gem::Version
198
+ version: '0'
199
+ required_rubygems_version: !ruby/object:Gem::Requirement
200
+ requirements:
201
+ - - ">="
202
+ - !ruby/object:Gem::Version
203
+ version: '0'
204
+ requirements: []
205
+ rubyforge_project:
206
+ rubygems_version: 2.5.1
207
+ signing_key:
208
+ specification_version: 4
209
+ summary: Azure Storage output plugin for Fluentd event collector
210
+ test_files:
211
+ - test/helper.rb
212
+ - test/plugin/test_out_azurestorage_gen2.rb