fluent-plugin-azure-storage-append-blob-lts 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 335e03eeafd0648547d44b98ce7bbbc3dd8a780dc03d3e39a17005a8305c0ba1
4
+ data.tar.gz: e1361ff0a6cc0d95eec72e2b330861de638afd17601a3ba2d6f81d0a18d7ee9a
5
+ SHA512:
6
+ metadata.gz: b2c40956a6eb51ea8a0d19058527fe8c470d48dbf59130671322c99c073353ffb4b9a0f931afcf7707dbc3deda7d44974174214e3045f7d566da876b7f434e93
7
+ data.tar.gz: 31258dfaf599e2bcfdda8c268aef6295b9720407c93a8b8997376ef192b7024fe15639da751be765c43e562738ad7f50daf479055f9668ef3cd7748e41173dd2
@@ -0,0 +1,31 @@
1
+ name: Publish Ruby Gem
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ otp:
7
+ description: 'One Time Password'
8
+ required: true
9
+
10
+ jobs:
11
+ build:
12
+ name: Build + Publish
13
+ runs-on: ubuntu-latest
14
+
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - name: Set up Ruby 2.6
18
+ uses: actions/setup-ruby@v1
19
+ with:
20
+ ruby-version: 2.6.x
21
+
22
+ - name: Publish to RubyGems
23
+ run: |
24
+ mkdir -p $HOME/.gem
25
+ touch $HOME/.gem/credentials
26
+ chmod 0600 $HOME/.gem/credentials
27
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
28
+ gem build *.gemspec
29
+ gem push *.gem --otp ${{ github.event.inputs.otp }}
30
+ env:
31
+ RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
@@ -0,0 +1,23 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches: [ master ]
6
+ pull_request:
7
+ branches: [ master ]
8
+
9
+ jobs:
10
+ test:
11
+
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v2
16
+ - name: Set up Ruby
17
+ uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: 2.6
20
+ - name: Install dependencies
21
+ run: bundle install
22
+ - name: Run tests
23
+ run: bundle exec rake test
@@ -0,0 +1,50 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /spec/examples.txt
9
+ /test/tmp/
10
+ /test/version_tmp/
11
+ /tmp/
12
+
13
+ # Used by dotenv library to load environment variables.
14
+ # .env
15
+
16
+ ## Specific to RubyMotion:
17
+ .dat*
18
+ .repl_history
19
+ build/
20
+ *.bridgesupport
21
+ build-iPhoneOS/
22
+ build-iPhoneSimulator/
23
+
24
+ ## Specific to RubyMotion (use of CocoaPods):
25
+ #
26
+ # We recommend against adding the Pods directory to your .gitignore. However
27
+ # you should judge for yourself, the pros and cons are mentioned at:
28
+ # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
29
+ #
30
+ # vendor/Pods/
31
+
32
+ ## Documentation cache and generated files:
33
+ /.yardoc/
34
+ /_yardoc/
35
+ /doc/
36
+ /rdoc/
37
+
38
+ ## Environment normalization:
39
+ /.bundle/
40
+ /vendor/bundle
41
+ /lib/bundler/man/
42
+
43
+ # for a library or gem, you might want to ignore these files since the code is
44
+ # intended to run in multiple environments; otherwise, check them in:
45
+ Gemfile.lock
46
+ # .ruby-version
47
+ # .ruby-gemset
48
+
49
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
50
+ .rvmrc
@@ -0,0 +1,2 @@
1
+ Style/FrozenStringLiteralComment:
2
+ Enabled: false
@@ -0,0 +1,37 @@
1
+ FROM ruby:latest
2
+
3
+ WORKDIR /plugin
4
+
5
+ ADD . /plugin
6
+
7
+ RUN gem install bundler && \
8
+ gem install fluentd --no-doc && \
9
+ fluent-gem build fluent-plugin-azure-storage-append-blob-lts.gemspec && \
10
+ fluent-gem install fluent-plugin-azure-storage-append-blob-lts-*.gem
11
+
12
+ RUN echo "<source>\n\
13
+ @type sample\n\
14
+ sample {\"hello\":\"world\"}\n\
15
+ tag pattern\n\
16
+ </source>\n\
17
+ <match pattern>\n\
18
+ @type azure-storage-append-blob\n\
19
+ azure_storage_account \"#{ENV['STORAGE_ACCOUNT']}\"\n\
20
+ azure_storage_access_key \"#{ENV['STORAGE_ACCESS_KEY']}\"\n\
21
+ azure_storage_connection_string \"#{ENV['STORAGE_CONNECTION_STRING']}\"\n\
22
+ azure_storage_sas_token \"#{ENV['STORAGE_SAS_TOKEN']}\"\n\
23
+ azure_container fluentd\n\
24
+ auto_create_container true\n\
25
+ path logs/\n\
26
+ azure_object_key_format %{path}%{time_slice}_%{index}.log\n\
27
+ time_slice_format %Y%m%d-%H\n\
28
+ <buffer tag,time>\n\
29
+ @type file\n\
30
+ path /var/log/fluent/azurestorageappendblob\n\
31
+ timekey 120 # 2 minutes\n\
32
+ timekey_wait 60\n\
33
+ timekey_use_utc true # use utc\n\
34
+ </buffer>\n\
35
+ </match>" > /plugin/fluent.conf
36
+
37
+ ENTRYPOINT ["fluentd", "-c", "fluent.conf"]
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) Microsoft Corporation. All rights reserved.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE
@@ -0,0 +1,170 @@
1
+ # fluent-plugin-azure-storage-append-blob-lts
2
+
3
+ Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
4
+ This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
5
+
6
+ ## Installation
7
+
8
+ ### RubyGems
9
+
10
+ gem install fluent-plugin-azure-storage-append-blob-lts
11
+
12
+ ### Bundler
13
+
14
+ Add following line to your Gemfile:
15
+
16
+ gem "fluent-plugin-azure-storage-append-blob-lts"
17
+
18
+ And then execute:
19
+
20
+ bundle
21
+
22
+ ## Configuration
23
+
24
+ <match pattern>
25
+ type azure-storage-append-blob
26
+
27
+ azure_storage_account <your azure storage account>
28
+ azure_storage_access_key <your azure storage access key> # leave empty to use MSI
29
+ azure_storage_connection_string <your azure storage connection string> # leave empty to use MSI
30
+ azure_storage_sas_token <your azure storage sas token> # leave empty to use MSI
31
+ azure_imds_api_version <Azure Instance Metadata Service API Version> # only used for MSI
32
+ azure_token_refresh_interval <refresh interval in min> # only used for MSI
33
+ azure_container <your azure storage container>
34
+ auto_create_container true
35
+ path logs/
36
+ azure_object_key_format %{path}%{time_slice}_%{index}.log
37
+ time_slice_format %Y%m%d-%H
38
+ # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
39
+ # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
40
+ <buffer tag,time>
41
+ @type file
42
+ path /var/log/fluent/azurestorageappendblob
43
+ timekey 120 # 2 minutes
44
+ timekey_wait 60
45
+ timekey_use_utc true # use utc
46
+ </buffer>
47
+ </match>
48
+
49
+ ### `azure_storage_account` (Required)
50
+
51
+ Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
52
+
53
+ ### `azure_storage_access_key` or `azure_storage_sas_token` or `azure_storage_connection_string` (Any required or all empty to use MSI)
54
+
55
+ Your Azure Storage Access Key (Primary or Secondary) or shared access signature (SAS) token or full connection string.
56
+ This also can be retrieved from Azure Management portal.
57
+
58
+ If all are empty, the plugin will use the local Managed Identity endpoint to obtain a token for the target storage account.
59
+
60
+ ### `azure_imds_api_version` (Optional, only for MSI)
61
+
62
+ Default: 2019-08-15
63
+
64
+ The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
65
+
66
+ See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
67
+
68
+ ### `azure_token_refresh_interval` (Optional, only for MSI)
69
+
70
+ Default: 60 (1 hour)
71
+
72
+ When using MSI, the initial access token needs to be refreshed periodically.
73
+
74
+ ### `azure_container` (Required)
75
+
76
+ Azure Storage Container name
77
+
78
+ ### `auto_create_container`
79
+
80
+ This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
81
+ The default value is `true`
82
+
83
+ ### `azure_object_key_format`
84
+
85
+ The format of Azure Storage object keys. You can use several built-in variables:
86
+
87
+ - %{path}
88
+ - %{time_slice}
89
+ - %{index}
90
+
91
+ to decide keys dynamically.
92
+
93
+ %{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
94
+ %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
95
+ %{index} is used only if your blob exceed Azure 50000 blocks limit per blob to prevent data loss. Its not required to use this parameter.
96
+
97
+ The default format is "%{path}%{time_slice}-%{index}.log".
98
+
99
+ For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
100
+
101
+ "logs/20130111-22-0.log"
102
+ "logs/20130111-23-0.log"
103
+ "logs/20130112-00-0.log"
104
+
105
+ With the configuration:
106
+
107
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
108
+ path log
109
+ time_slice_format %Y%m%d-%H
110
+
111
+ You get:
112
+
113
+ "log/events/ts=20130111-22/events.log"
114
+ "log/events/ts=20130111-23/events.log"
115
+ "log/events/ts=20130112-00/events.log"
116
+
117
+ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
118
+
119
+ azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
120
+
121
+ ### `time_slice_format`
122
+
123
+ Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
124
+
125
+ ### Run tests
126
+
127
+ gem install bundler
128
+ bundle install
129
+ bundle exec rake test
130
+
131
+
132
+ ### Test Fluentd
133
+
134
+ 1. Create Storage Account and VM with enabled MSI
135
+ 2. Setup Docker ang Git
136
+ 3. SSH into VM
137
+ 4. Download this repo
138
+ ```
139
+ git clone https://github.com/microsoft/fluent-plugin-azure-storage-append-blob-lts.git
140
+ cd fluent-plugin-azure-storage-append-blob-lts
141
+ ```
142
+ 5. Build Docker image
143
+ `docker build -t fluent .`
144
+ 6. Run Docker image with different set of parameters:
145
+
146
+ 1. `STORAGE_ACCOUNT`: required, name of your storage account
147
+ 2. `STORAGE_ACCESS_KEY`: storage account access key
148
+ 3. `STORAGE_SAS_TOKEN`: storage sas token with enough permissions for the plugin
149
+
150
+ You need to specify `STORAGE_ACCOUNT` and one of auth ways. If you run it from VM with MSI,
151
+ just `STORAGE_ACCOUNT` is required. Keep in mind, there is no way to refresh MSI Token, so
152
+ ensure you setup proper permissions first.
153
+
154
+ ```bash
155
+ docker run -it -e STORAGE_ACCOUNT=<storage> -e STORAGE_ACCESS_KEY=<key> fluent
156
+ ```
157
+
158
+ ## Contributing
159
+
160
+ This project welcomes contributions and suggestions. Most contributions require you to agree to a
161
+ Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
162
+ the rights to use your contribution. For details, visit [https://cla.microsoft.com](https://cla.microsoft.com).
163
+
164
+ When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
165
+ a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
166
+ provided by the bot. You will only need to do this once across all repos using our CLA.
167
+
168
+ This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
169
+ For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
170
+ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
@@ -0,0 +1,13 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rake/testtask'
5
+
6
+ Rake::TestTask.new(:test) do |t|
7
+ t.libs.push('lib', 'test')
8
+ t.test_files = FileList['test/**/test_*.rb']
9
+ t.verbose = true
10
+ t.warning = true
11
+ end
12
+
13
+ task default: [:test]
@@ -0,0 +1,29 @@
1
+ lib = File.expand_path('lib', __dir__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
6
+ spec.version = '0.3.0'
7
+ spec.authors = ['Jonas-Taha El Sesiy']
8
+ spec.email = ['']
9
+
10
+ spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
11
+ spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob'
12
+ spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
13
+ spec.license = 'MIT'
14
+
15
+ test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
+ f.match(%r{^(test|spec|features)/})
17
+ end
18
+ spec.files = files
19
+ spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
+ spec.test_files = test_files
21
+ spec.require_paths = ['lib']
22
+ spec.required_ruby_version = '>= 2.6'
23
+
24
+ spec.add_development_dependency 'bundler', '~> 2.0'
25
+ spec.add_development_dependency 'rake', '~> 13.0'
26
+ spec.add_development_dependency 'test-unit', '~> 3.0'
27
+ spec.add_runtime_dependency 'azure-storage-blob', '~> 2.0'
28
+ spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
29
+ end
@@ -0,0 +1,235 @@
1
+ #---------------------------------------------------------------------------------------------
2
+ # Copyright (c) Microsoft Corporation. All rights reserved.
3
+ # Licensed under the MIT License. See License.txt in the project root for license information.
4
+ #--------------------------------------------------------------------------------------------*/
5
+
6
+ require 'azure/storage/common'
7
+ require 'azure/storage/blob'
8
+ require 'faraday'
9
+ require 'fluent/plugin/output'
10
+ require 'json'
11
+
12
+ module Fluent
13
+ module Plugin
14
+ class AzureStorageAppendBlobOut < Fluent::Plugin::Output
15
+ Fluent::Plugin.register_output('azure-storage-append-blob', self)
16
+
17
+ helpers :formatter, :inject
18
+
19
+ DEFAULT_FORMAT_TYPE = 'out_file'.freeze
20
+ AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
21
+
22
+ config_param :path, :string, default: ''
23
+ config_param :azure_storage_account, :string, default: nil
24
+ config_param :azure_storage_access_key, :string, default: nil, secret: true
25
+ config_param :azure_storage_sas_token, :string, default: nil, secret: true
26
+ config_param :azure_storage_connection_string, :string, default: nil, secret: true
27
+ config_param :azure_container, :string, default: nil
28
+ config_param :azure_imds_api_version, :string, default: '2019-08-15'
29
+ config_param :azure_token_refresh_interval, :integer, default: 60
30
+ config_param :use_msi, :bool, default: false
31
+ config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
32
+ config_param :auto_create_container, :bool, default: true
33
+ config_param :format, :string, default: DEFAULT_FORMAT_TYPE
34
+ config_param :time_slice_format, :string, default: '%Y%m%d'
35
+ config_param :localtime, :bool, default: false
36
+
37
+ config_section :format do
38
+ config_set_default :@type, DEFAULT_FORMAT_TYPE
39
+ end
40
+
41
+ config_section :buffer do
42
+ config_set_default :chunk_keys, ['time']
43
+ config_set_default :timekey, (60 * 60 * 24)
44
+ end
45
+
46
+ attr_reader :bs
47
+
48
+ def configure(conf)
49
+ super
50
+
51
+ @formatter = formatter_create
52
+
53
+ @path_slicer = if @localtime
54
+ proc do |path|
55
+ Time.now.strftime(path)
56
+ end
57
+ else
58
+ proc do |path|
59
+ Time.now.utc.strftime(path)
60
+ end
61
+ end
62
+
63
+ raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
64
+
65
+ raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
66
+
67
+ if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
68
+ (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
69
+ (@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
70
+ log.info 'Using MSI since neither azure_storage_access_key nor azure_storage_sas_token was provided.'
71
+ @use_msi = true
72
+ end
73
+ end
74
+
75
+ def multi_workers_ready?
76
+ true
77
+ end
78
+
79
+ def get_access_token
80
+ access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
81
+ "api-version=#{@azure_imds_api_version}" \
82
+ '&resource=https://storage.azure.com/',
83
+ headers: { 'Metadata' => 'true' })
84
+ .get
85
+ .body
86
+ JSON.parse(access_key_request)['access_token']
87
+ end
88
+
89
+ def start
90
+ super
91
+ if @use_msi
92
+ token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
93
+ token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
94
+ @bs = Azure::Storage::Blob::BlobService.new(storage_account_name: @azure_storage_account, signer: token_signer)
95
+
96
+ refresh_interval = @azure_token_refresh_interval * 60
97
+ cancelled = false
98
+ renew_token = Thread.new do
99
+ Thread.stop
100
+ until cancelled
101
+ sleep(refresh_interval)
102
+
103
+ token_credential.renew_token get_access_token
104
+ end
105
+ end
106
+ sleep 0.1 while renew_token.status != 'sleep'
107
+ renew_token.run
108
+ elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
109
+ @bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
110
+ else
111
+ @bs_params = { storage_account_name: @azure_storage_account }
112
+
113
+ if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
114
+ @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
115
+ elsif !@azure_storage_sas_token.nil? && !@azure_storage_sas_token.empty?
116
+ @bs_params.merge!({ storage_sas_token: @azure_storage_sas_token })
117
+ end
118
+
119
+ @bs = Azure::Storage::Blob::BlobService.create(@bs_params)
120
+
121
+ end
122
+
123
+ ensure_container
124
+ @azure_storage_path = ''
125
+ @last_azure_storage_path = ''
126
+ @current_index = 0
127
+ end
128
+
129
+ def format(tag, time, record)
130
+ r = inject_values_to_record(tag, time, record)
131
+ @formatter.format(tag, time, r)
132
+ end
133
+
134
+ def write(chunk)
135
+ metadata = chunk.metadata
136
+ tmp = Tempfile.new('azure-')
137
+ begin
138
+ chunk.write_to(tmp)
139
+
140
+ generate_log_name(metadata, @current_index)
141
+ if @last_azure_storage_path != @azure_storage_path
142
+ @current_index = 0
143
+ generate_log_name(metadata, @current_index)
144
+ end
145
+
146
+ content = File.open(tmp.path, 'rb', &:read)
147
+
148
+ append_blob(content, metadata)
149
+ @last_azure_storage_path = @azure_storage_path
150
+ ensure
151
+ tmp.close(true) rescue nil
152
+ end
153
+ end
154
+
155
+ def container_exists?(container)
156
+ begin
157
+ @bs.get_container_properties(container)
158
+ rescue Azure::Core::Http::HTTPError => e
159
+ if e.status_code == 404 # container does not exist
160
+ return false
161
+ else
162
+ raise
163
+ end
164
+ end
165
+ true
166
+ end
167
+
168
+ private
169
+
170
+ def ensure_container
171
+ unless container_exists? @azure_container
172
+ if @auto_create_container
173
+ @bs.create_container(@azure_container)
174
+ else
175
+ raise "The specified container does not exist: container = #{@azure_container}"
176
+ end
177
+ end
178
+ end
179
+
180
+ def generate_log_name(metadata, index)
181
+ time_slice = if metadata.timekey.nil?
182
+ ''.freeze
183
+ else
184
+ Time.at(metadata.timekey).utc.strftime(@time_slice_format)
185
+ end
186
+
187
+ path = @path_slicer.call(@path)
188
+ values_for_object_key = {
189
+ '%{path}' => path,
190
+ '%{time_slice}' => time_slice,
191
+ '%{index}' => index
192
+ }
193
+ storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
194
+ @azure_storage_path = extract_placeholders(storage_path, metadata)
195
+ end
196
+
197
+ def append_blob(content, metadata)
198
+ position = 0
199
+ log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
200
+ loop do
201
+ begin
202
+ size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
203
+ log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
204
+ @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1])
205
+ position += size
206
+ break if position >= content.length
207
+ rescue Azure::Core::Http::HTTPError => e
208
+ status_code = e.status_code
209
+
210
+ if status_code == 409 # exceeds azure block limit
211
+ @current_index += 1
212
+ old_azure_storage_path = @azure_storage_path
213
+ generate_log_name(metadata, @current_index)
214
+
215
+ # If index is not a part of format, rethrow exception.
216
+ if old_azure_storage_path == @azure_storage_path
217
+ log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
218
+ raise
219
+ end
220
+
221
+ log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
222
+ @bs.create_append_blob(@azure_container, @azure_storage_path)
223
+ elsif status_code == 404 # blob not found
224
+ log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
225
+ @bs.create_append_blob(@azure_container, @azure_storage_path)
226
+ else
227
+ raise
228
+ end
229
+ end
230
+ end
231
+ log.debug 'azure_storage_append_blob: append_blob.complete'
232
+ end
233
+ end
234
+ end
235
+ end
@@ -0,0 +1,8 @@
1
+ $LOAD_PATH.unshift(File.expand_path('..', __dir__))
2
+ require 'test-unit'
3
+ require 'fluent/test'
4
+ require 'fluent/test/driver/output'
5
+ require 'fluent/test/helpers'
6
+
7
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
8
+ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -0,0 +1,184 @@
1
+ require 'helper'
2
+ require 'fluent/plugin/out_azure-storage-append-blob.rb'
3
+ require 'azure/core/http/http_response'
4
+ require 'azure/core/http/http_error'
5
+
6
+ include Fluent::Test::Helpers
7
+
8
+ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
9
+ setup do
10
+ Fluent::Test.setup
11
+ end
12
+
13
+ CONFIG = %(
14
+ azure_storage_account test_storage_account
15
+ azure_storage_access_key MY_FAKE_SECRET
16
+ azure_container test_container
17
+ time_slice_format %Y%m%d-%H
18
+ path log
19
+ ).freeze
20
+
21
+ MSI_CONFIG = %(
22
+ azure_storage_account test_storage_account
23
+ azure_container test_container
24
+ azure_imds_api_version 1970-01-01
25
+ azure_token_refresh_interval 120
26
+ time_slice_format %Y%m%d-%H
27
+ path log
28
+ ).freeze
29
+
30
+ def create_driver(conf: CONFIG, service: nil)
31
+ d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
32
+ d.instance.instance_variable_set(:@bs, service)
33
+ d.instance.instance_variable_set(:@azure_storage_path, 'storage_path')
34
+ d
35
+ end
36
+
37
+ sub_test_case 'test config' do
38
+ test 'config should reject with no azure container' do
39
+ assert_raise Fluent::ConfigError do
40
+ create_driver conf: %(
41
+ azure_storage_account test_storage_account
42
+ azure_storage_access_key MY_FAKE_SECRET
43
+ time_slice_format %Y%m%d-%H
44
+ time_slice_wait 10m
45
+ path log
46
+ )
47
+ end
48
+ end
49
+
50
+ test 'config with access key should set instance variables' do
51
+ d = create_driver
52
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
53
+ assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
54
+ assert_equal 'test_container', d.instance.azure_container
55
+ assert_equal true, d.instance.auto_create_container
56
+ assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
57
+ end
58
+
59
+ test 'config with managed identity enabled should set instance variables' do
60
+ d = create_driver conf: MSI_CONFIG
61
+ assert_equal 'test_storage_account', d.instance.azure_storage_account
62
+ assert_equal 'test_container', d.instance.azure_container
63
+ assert_equal true, d.instance.use_msi
64
+ assert_equal true, d.instance.auto_create_container
65
+ assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
66
+ assert_equal 120, d.instance.azure_token_refresh_interval
67
+ assert_equal '1970-01-01', d.instance.azure_imds_api_version
68
+ end
69
+ end
70
+
71
+ sub_test_case 'test path slicing' do
72
+ test 'test path_slicing' do
73
+ config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
74
+ d = create_driver conf: config
75
+ path_slicer = d.instance.instance_variable_get(:@path_slicer)
76
+ path = d.instance.instance_variable_get(:@path)
77
+ slice = path_slicer.call(path)
78
+ assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
79
+ end
80
+
81
+ test 'path slicing utc' do
82
+ config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
83
+ config << "\nutc\n"
84
+ d = create_driver conf: config
85
+ path_slicer = d.instance.instance_variable_get(:@path_slicer)
86
+ path = d.instance.instance_variable_get(:@path)
87
+ slice = path_slicer.call(path)
88
+ assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
89
+ end
90
+ end
91
+
92
+ # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
93
+ # a response object when it is created.
94
+ class FakeResponse
95
+ def initialize(status=404)
96
+ @status = status
97
+ @body = "body"
98
+ @headers = {}
99
+ end
100
+
101
+ attr_reader :status
102
+ attr_reader :body
103
+ attr_reader :headers
104
+ end
105
+
106
+ # This class is used to test plugin functions which interact with the blob service
107
+ class FakeBlobService
108
+ def initialize(status)
109
+ @response = Azure::Core::Http::HttpResponse.new(FakeResponse.new(status))
110
+ @blocks = []
111
+ end
112
+ attr_reader :blocks
113
+
114
+ def append_blob_block(container, path, data)
115
+ @blocks.append(data)
116
+ end
117
+
118
+ def get_container_properties(container)
119
+ unless @response.status_code == 200
120
+ raise Azure::Core::Http::HTTPError.new(@response)
121
+ end
122
+ end
123
+ end
124
+
125
+ sub_test_case 'test container_exists' do
126
+ test 'container 404 returns false' do
127
+ d = create_driver service: FakeBlobService.new(404)
128
+ assert_false d.instance.container_exists? "anything"
129
+ end
130
+
131
+ test 'existing container returns true' do
132
+ d = create_driver service: FakeBlobService.new(200)
133
+ assert_true d.instance.container_exists? "anything"
134
+ end
135
+
136
+ test 'unexpected exception raises' do
137
+ d = create_driver service: FakeBlobService.new(500)
138
+ assert_raise_kind_of Azure::Core::Http::HTTPError do
139
+ d.instance.container_exists? "anything"
140
+ end
141
+ end
142
+ end
143
+
144
+ # Override the block size limit so that mocked requests do not require huge buffers
145
+ class Fluent::Plugin::AzureStorageAppendBlobOut
146
+ AZURE_BLOCK_SIZE_LIMIT = 10
147
+ end
148
+
149
+ sub_test_case 'test append blob buffering' do
150
+ def fake_appended_blocks(content)
151
+ # run the append on the fake blob service, return a list of append request buffers
152
+ svc = FakeBlobService.new(200)
153
+ d = create_driver service: svc
154
+ d.instance.send(:append_blob, content, nil)
155
+ svc.blocks
156
+ end
157
+
158
+ test 'short buffer appends once' do
159
+ content = '123456789'
160
+ blocks = fake_appended_blocks content
161
+ assert_equal [content], blocks
162
+ end
163
+
164
+ test 'single character appends once' do
165
+ content = '1'
166
+ blocks = fake_appended_blocks content
167
+ assert_equal [content], blocks
168
+ end
169
+
170
+ test 'empty appends once' do
171
+ content = ''
172
+ blocks = fake_appended_blocks content
173
+ assert_equal [''], blocks
174
+ end
175
+
176
+ test 'long buffer appends multiple times' do
177
+ limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
178
+ buf_1 = 'a' * limit
179
+ buf_2 = 'a' * 3
180
+ blocks = fake_appended_blocks buf_1 + buf_2
181
+ assert_equal [buf_1, buf_2], blocks
182
+ end
183
+ end
184
+ end
metadata ADDED
@@ -0,0 +1,135 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-azure-storage-append-blob-lts
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.3.0
5
+ platform: ruby
6
+ authors:
7
+ - Jonas-Taha El Sesiy
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-12-10 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '13.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '13.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: test-unit
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: azure-storage-blob
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '2.0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: fluentd
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: 0.14.10
76
+ - - "<"
77
+ - !ruby/object:Gem::Version
78
+ version: '2'
79
+ type: :runtime
80
+ prerelease: false
81
+ version_requirements: !ruby/object:Gem::Requirement
82
+ requirements:
83
+ - - ">="
84
+ - !ruby/object:Gem::Version
85
+ version: 0.14.10
86
+ - - "<"
87
+ - !ruby/object:Gem::Version
88
+ version: '2'
89
+ description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
90
+ https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob
91
+ email:
92
+ - ''
93
+ executables: []
94
+ extensions: []
95
+ extra_rdoc_files: []
96
+ files:
97
+ - ".github/workflows/publish.yaml"
98
+ - ".github/workflows/test.yaml"
99
+ - ".gitignore"
100
+ - ".rubocop.yml"
101
+ - Dockerfile
102
+ - Gemfile
103
+ - LICENSE
104
+ - README.md
105
+ - Rakefile
106
+ - fluent-plugin-azure-storage-append-blob-lts.gemspec
107
+ - lib/fluent/plugin/out_azure-storage-append-blob.rb
108
+ - test/helper.rb
109
+ - test/plugin/test_out_azure_storage_append_blob.rb
110
+ homepage: https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts
111
+ licenses:
112
+ - MIT
113
+ metadata: {}
114
+ post_install_message:
115
+ rdoc_options: []
116
+ require_paths:
117
+ - lib
118
+ required_ruby_version: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: '2.6'
123
+ required_rubygems_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ requirements: []
129
+ rubygems_version: 3.0.3
130
+ signing_key:
131
+ specification_version: 4
132
+ summary: Azure Storage Append Blob output plugin for Fluentd event collector
133
+ test_files:
134
+ - test/helper.rb
135
+ - test/plugin/test_out_azure_storage_append_blob.rb