fluent-plugin-azure-storage-append-blob-lts 0.4.0 → 0.6.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/workflows/publish.yaml +0 -2
- data/.github/workflows/test.yaml +0 -2
- data/Dockerfile +2 -2
- data/README.md +32 -4
- data/fluent-plugin-azure-storage-append-blob-lts.gemspec +3 -3
- data/lib/fluent/plugin/out_azure-storage-append-blob.rb +83 -46
- data/test/plugin/test_out_azure_storage_append_blob.rb +65 -16
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f9e6de27ebbf624d51968e415d502111523d3fd670d2345558381b03895934d6
|
4
|
+
data.tar.gz: 80fbe42cee9d82adb02afbaf2cfed1fd92b0d4fbbbfb1e67b50e328795d90b23
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3c2e0296f88ae875f19043e322678aa9a7ea3a26e9bb41f9f4292991ff09a6a50022d03c1da48b36e0c724c2e9e83f70ef431e1b9a3fc628507774a8de32943
|
7
|
+
data.tar.gz: a86038ca1acc515dd05df9f86d014cef337c5cc8a70847a15dd94b8f86fe2233c5b0a49be988f26a3da86f298d1aa27465aab4fb904ddd8eeef9e16535291d99
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @elsesiy
|
@@ -11,14 +11,12 @@ jobs:
|
|
11
11
|
build:
|
12
12
|
name: Build + Publish
|
13
13
|
runs-on: ubuntu-latest
|
14
|
-
|
15
14
|
steps:
|
16
15
|
- uses: actions/checkout@v2
|
17
16
|
- name: Set up Ruby 2.6
|
18
17
|
uses: actions/setup-ruby@v1
|
19
18
|
with:
|
20
19
|
ruby-version: 2.6.x
|
21
|
-
|
22
20
|
- name: Publish to RubyGems
|
23
21
|
run: |
|
24
22
|
mkdir -p $HOME/.gem
|
data/.github/workflows/test.yaml
CHANGED
data/Dockerfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# fluent-plugin-azure-storage-append-blob-lts
|
2
2
|
|
3
|
+
![Tests](https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts/workflows/Test/badge.svg?branch=master)
|
4
|
+
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts.svg)](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts)
|
5
|
+
[![Twitter](https://img.shields.io/badge/twitter-@elsesiy-blue.svg)](http://twitter.com/elsesiy)
|
6
|
+
|
3
7
|
Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
|
4
8
|
This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
|
5
9
|
|
@@ -24,6 +28,8 @@ And then execute:
|
|
24
28
|
<match pattern>
|
25
29
|
type azure-storage-append-blob
|
26
30
|
|
31
|
+
azure_cloud <azure cloud environment>
|
32
|
+
azure_storage_dns_suffix <your azure storage dns suffix> # only used for Azure Stack Cloud
|
27
33
|
azure_storage_account <your azure storage account>
|
28
34
|
azure_storage_access_key <your azure storage access key> # leave empty to use MSI
|
29
35
|
azure_storage_connection_string <your azure storage connection string> # leave empty to use MSI
|
@@ -36,6 +42,7 @@ And then execute:
|
|
36
42
|
path logs/
|
37
43
|
azure_object_key_format %{path}%{time_slice}_%{index}.log
|
38
44
|
time_slice_format %Y%m%d-%H
|
45
|
+
compute_checksums true
|
39
46
|
# if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
|
40
47
|
# need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
|
41
48
|
<buffer tag,time>
|
@@ -47,6 +54,18 @@ And then execute:
|
|
47
54
|
</buffer>
|
48
55
|
</match>
|
49
56
|
|
57
|
+
### `azure_cloud` (Optional)
|
58
|
+
|
59
|
+
Default: `AZUREPUBLICCLOUD`
|
60
|
+
|
61
|
+
Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
|
62
|
+
|
63
|
+
Use `AZURESTACKCLOUD` for Azure Stack Cloud.
|
64
|
+
|
65
|
+
### `azure_storage_dns_suffix` (Required only for Azure Stack Cloud)
|
66
|
+
|
67
|
+
Your Azure Storage endpoint suffix. This can be retrieved from Azure Storage connection string, `EndpointSuffix` section.
|
68
|
+
|
50
69
|
### `azure_storage_account` (Required)
|
51
70
|
|
52
71
|
Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
|
@@ -60,15 +79,15 @@ If all are empty, the plugin will use the local Managed Identity endpoint to obt
|
|
60
79
|
|
61
80
|
### `azure_imds_api_version` (Optional, only for MSI)
|
62
81
|
|
63
|
-
Default:
|
82
|
+
Default: `2020-12-01`
|
64
83
|
|
65
84
|
The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
|
66
85
|
|
67
|
-
See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
|
86
|
+
See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service?tabs=windows#versioning) for more details.
|
68
87
|
|
69
88
|
### `azure_token_refresh_interval` (Optional, only for MSI)
|
70
89
|
|
71
|
-
Default: 60 (1 hour)
|
90
|
+
Default: `60` (1 hour)
|
72
91
|
|
73
92
|
When using MSI, the initial access token needs to be refreshed periodically.
|
74
93
|
|
@@ -82,8 +101,9 @@ Azure Identity Client ID to use for accessing Azure Blob service.
|
|
82
101
|
|
83
102
|
### `auto_create_container`
|
84
103
|
|
104
|
+
Default: `true`
|
105
|
+
|
85
106
|
This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
|
86
|
-
The default value is `true`
|
87
107
|
|
88
108
|
### `azure_object_key_format`
|
89
109
|
|
@@ -127,6 +147,14 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
|
|
127
147
|
|
128
148
|
Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
|
129
149
|
|
150
|
+
### `compute_checksums`
|
151
|
+
|
152
|
+
Default: `true`
|
153
|
+
|
154
|
+
Whether to compute MD5 checksum of the blob contents during append operation and provide it in a header for the blob service.
|
155
|
+
|
156
|
+
You want to set it to `false` in FIPS-enabled environments.
|
157
|
+
|
130
158
|
### Run tests
|
131
159
|
|
132
160
|
gem install bundler
|
@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
|
6
|
-
spec.version = '0.
|
6
|
+
spec.version = '0.6.3'
|
7
7
|
spec.authors = ['Jonas-Taha El Sesiy']
|
8
|
-
spec.email = ['']
|
8
|
+
spec.email = ['github@elsesiy.com']
|
9
9
|
|
10
10
|
spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
|
11
|
-
spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/
|
11
|
+
spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/microsoft/fluent-plugin-azure-storage-append-blob'
|
12
12
|
spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
|
13
13
|
spec.license = 'MIT'
|
14
14
|
|
@@ -11,7 +11,7 @@ require 'json'
|
|
11
11
|
|
12
12
|
module Fluent
|
13
13
|
module Plugin
|
14
|
-
class AzureStorageAppendBlobOut <
|
14
|
+
class AzureStorageAppendBlobOut < Output
|
15
15
|
Fluent::Plugin.register_output('azure-storage-append-blob', self)
|
16
16
|
|
17
17
|
helpers :formatter, :inject
|
@@ -19,18 +19,33 @@ module Fluent
|
|
19
19
|
DEFAULT_FORMAT_TYPE = 'out_file'.freeze
|
20
20
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
21
21
|
|
22
|
+
def initialize
|
23
|
+
super
|
24
|
+
@use_msi = false
|
25
|
+
|
26
|
+
# Storage endpoint suffixes for various environments, see https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go
|
27
|
+
@storage_endpoint_mapping = {
|
28
|
+
'AZURECHINACLOUD' => 'core.chinacloudapi.cn',
|
29
|
+
'AZUREGERMANCLOUD' => 'core.cloudapi.de',
|
30
|
+
'AZUREPUBLICCLOUD' => 'core.windows.net',
|
31
|
+
'AZUREUSGOVERNMENTCLOUD' => 'core.usgovcloudapi.net'
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
22
35
|
config_param :path, :string, default: ''
|
23
36
|
config_param :azure_storage_account, :string, default: nil
|
24
37
|
config_param :azure_storage_access_key, :string, default: nil, secret: true
|
25
|
-
config_param :azure_storage_sas_token, :string, default: nil, secret: true
|
26
38
|
config_param :azure_storage_connection_string, :string, default: nil, secret: true
|
39
|
+
config_param :azure_storage_dns_suffix, :string, default: nil
|
40
|
+
config_param :azure_storage_sas_token, :string, default: nil, secret: true
|
41
|
+
config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
|
27
42
|
config_param :azure_msi_client_id, :string, default: nil
|
28
43
|
config_param :azure_container, :string, default: nil
|
29
|
-
config_param :azure_imds_api_version, :string, default: '
|
44
|
+
config_param :azure_imds_api_version, :string, default: '2020-12-01'
|
30
45
|
config_param :azure_token_refresh_interval, :integer, default: 60
|
31
|
-
config_param :use_msi, :bool, default: false
|
32
46
|
config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
|
33
47
|
config_param :auto_create_container, :bool, default: true
|
48
|
+
config_param :compute_checksums, :bool, default: true
|
34
49
|
config_param :format, :string, default: DEFAULT_FORMAT_TYPE
|
35
50
|
config_param :time_slice_format, :string, default: '%Y%m%d'
|
36
51
|
config_param :localtime, :bool, default: false
|
@@ -61,15 +76,31 @@ module Fluent
|
|
61
76
|
end
|
62
77
|
end
|
63
78
|
|
64
|
-
|
65
|
-
|
66
|
-
|
79
|
+
if @azure_cloud == 'AZURESTACKCLOUD'
|
80
|
+
if @azure_storage_dns_suffix.nil?
|
81
|
+
raise ConfigError, 'azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD'
|
82
|
+
end
|
83
|
+
else
|
84
|
+
@azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
|
85
|
+
if @azure_storage_dns_suffix.nil?
|
86
|
+
raise ConfigError, 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD'
|
87
|
+
end
|
88
|
+
end
|
67
89
|
|
68
90
|
if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
|
69
91
|
(@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
|
70
92
|
(@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
|
71
|
-
log.info 'Using MSI since neither azure_storage_access_key
|
93
|
+
log.info 'Using MSI since neither of azure_storage_access_key, azure_storage_sas_token, azure_storage_connection_string was provided.'
|
72
94
|
@use_msi = true
|
95
|
+
elsif @azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?
|
96
|
+
raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
|
97
|
+
raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
|
98
|
+
end
|
99
|
+
|
100
|
+
@blob_options = {}
|
101
|
+
|
102
|
+
if !@compute_checksums
|
103
|
+
@blob_options[:content_md5] = ''
|
73
104
|
end
|
74
105
|
end
|
75
106
|
|
@@ -81,13 +112,13 @@ module Fluent
|
|
81
112
|
access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
|
82
113
|
"api-version=#{@azure_imds_api_version}" \
|
83
114
|
'&resource=https://storage.azure.com/' \
|
84
|
-
"#{!
|
115
|
+
"#{!azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
|
85
116
|
headers: { 'Metadata' => 'true' }).get
|
86
117
|
|
87
118
|
if access_key_request.status == 200
|
88
119
|
JSON.parse(access_key_request.body)['access_token']
|
89
120
|
else
|
90
|
-
raise
|
121
|
+
raise 'Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter.'
|
91
122
|
end
|
92
123
|
end
|
93
124
|
|
@@ -96,7 +127,11 @@ module Fluent
|
|
96
127
|
if @use_msi
|
97
128
|
token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
|
98
129
|
token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
|
99
|
-
@bs = Azure::Storage::Blob::BlobService.new(
|
130
|
+
@bs = Azure::Storage::Blob::BlobService.new(
|
131
|
+
storage_account_name: @azure_storage_account,
|
132
|
+
storage_dns_suffix: @azure_storage_dns_suffix,
|
133
|
+
signer: token_signer
|
134
|
+
)
|
100
135
|
|
101
136
|
refresh_interval = @azure_token_refresh_interval * 60
|
102
137
|
cancelled = false
|
@@ -113,7 +148,7 @@ module Fluent
|
|
113
148
|
elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
|
114
149
|
@bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
|
115
150
|
else
|
116
|
-
@bs_params = { storage_account_name: @azure_storage_account }
|
151
|
+
@bs_params = { storage_account_name: @azure_storage_account, storage_dns_suffix: @azure_storage_dns_suffix }
|
117
152
|
|
118
153
|
if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
|
119
154
|
@bs_params.merge!({ storage_access_key: @azure_storage_access_key })
|
@@ -137,23 +172,26 @@ module Fluent
|
|
137
172
|
end
|
138
173
|
|
139
174
|
def write(chunk)
|
140
|
-
metadata = chunk.metadata
|
141
175
|
tmp = Tempfile.new('azure-')
|
142
176
|
begin
|
143
177
|
chunk.write_to(tmp)
|
144
178
|
|
145
|
-
generate_log_name(
|
179
|
+
generate_log_name(chunk, @current_index)
|
146
180
|
if @last_azure_storage_path != @azure_storage_path
|
147
181
|
@current_index = 0
|
148
|
-
generate_log_name(
|
182
|
+
generate_log_name(chunk, @current_index)
|
149
183
|
end
|
150
184
|
|
151
185
|
content = File.open(tmp.path, 'rb', &:read)
|
152
186
|
|
153
|
-
append_blob(content,
|
187
|
+
append_blob(content, chunk)
|
154
188
|
@last_azure_storage_path = @azure_storage_path
|
155
189
|
ensure
|
156
|
-
|
190
|
+
begin
|
191
|
+
tmp.close(true)
|
192
|
+
rescue StandardError
|
193
|
+
nil
|
194
|
+
end
|
157
195
|
end
|
158
196
|
end
|
159
197
|
|
@@ -182,7 +220,8 @@ module Fluent
|
|
182
220
|
end
|
183
221
|
end
|
184
222
|
|
185
|
-
def generate_log_name(
|
223
|
+
def generate_log_name(chunk, index)
|
224
|
+
metadata = chunk.metadata
|
186
225
|
time_slice = if metadata.timekey.nil?
|
187
226
|
''.freeze
|
188
227
|
else
|
@@ -196,41 +235,39 @@ module Fluent
|
|
196
235
|
'%{index}' => index
|
197
236
|
}
|
198
237
|
storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
|
199
|
-
@azure_storage_path = extract_placeholders(storage_path,
|
238
|
+
@azure_storage_path = extract_placeholders(storage_path, chunk)
|
200
239
|
end
|
201
240
|
|
202
|
-
def append_blob(content,
|
241
|
+
def append_blob(content, chunk)
|
203
242
|
position = 0
|
204
243
|
log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
|
205
244
|
loop do
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
|
223
|
-
raise
|
224
|
-
end
|
225
|
-
|
226
|
-
log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
|
227
|
-
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
228
|
-
elsif status_code == 404 # blob not found
|
229
|
-
log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
|
230
|
-
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
231
|
-
else
|
245
|
+
size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
|
246
|
+
log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
|
247
|
+
@bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1], options=@blob_options)
|
248
|
+
position += size
|
249
|
+
break if position >= content.length
|
250
|
+
rescue Azure::Core::Http::HTTPError => e
|
251
|
+
status_code = e.status_code
|
252
|
+
|
253
|
+
if status_code == 409 # exceeds azure block limit
|
254
|
+
@current_index += 1
|
255
|
+
old_azure_storage_path = @azure_storage_path
|
256
|
+
generate_log_name(chunk, @current_index)
|
257
|
+
|
258
|
+
# If index is not a part of format, rethrow exception.
|
259
|
+
if old_azure_storage_path == @azure_storage_path
|
260
|
+
log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
|
232
261
|
raise
|
233
262
|
end
|
263
|
+
|
264
|
+
log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
|
265
|
+
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
266
|
+
elsif status_code == 404 # blob not found
|
267
|
+
log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
|
268
|
+
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
269
|
+
else
|
270
|
+
raise
|
234
271
|
end
|
235
272
|
end
|
236
273
|
log.debug 'azure_storage_append_blob: append_blob.complete'
|
@@ -11,6 +11,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
11
11
|
end
|
12
12
|
|
13
13
|
CONFIG = %(
|
14
|
+
azure_cloud AZUREGERMANCLOUD
|
14
15
|
azure_storage_account test_storage_account
|
15
16
|
azure_storage_access_key MY_FAKE_SECRET
|
16
17
|
azure_container test_container
|
@@ -18,6 +19,12 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
18
19
|
path log
|
19
20
|
).freeze
|
20
21
|
|
22
|
+
CONNSTR_CONFIG = %(
|
23
|
+
azure_storage_connection_string https://test
|
24
|
+
time_slice_format %Y%m%d-%H
|
25
|
+
path log
|
26
|
+
).freeze
|
27
|
+
|
21
28
|
MSI_CONFIG = %(
|
22
29
|
azure_storage_account test_storage_account
|
23
30
|
azure_container test_container
|
@@ -27,6 +34,16 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
27
34
|
path log
|
28
35
|
).freeze
|
29
36
|
|
37
|
+
AZURESTACKCLOUD_CONFIG = %(
|
38
|
+
azure_cloud AZURESTACKCLOUD
|
39
|
+
azure_storage_dns_suffix test.storage.dns.suffix
|
40
|
+
azure_storage_account test_storage_account
|
41
|
+
azure_storage_access_key MY_FAKE_SECRET
|
42
|
+
azure_container test_container
|
43
|
+
time_slice_format %Y%m%d-%H
|
44
|
+
path log
|
45
|
+
).freeze
|
46
|
+
|
30
47
|
def create_driver(conf: CONFIG, service: nil)
|
31
48
|
d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
|
32
49
|
d.instance.instance_variable_set(:@bs, service)
|
@@ -36,7 +53,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
36
53
|
|
37
54
|
sub_test_case 'test config' do
|
38
55
|
test 'config should reject with no azure container' do
|
39
|
-
assert_raise Fluent::ConfigError do
|
56
|
+
assert_raise Fluent::ConfigError.new('azure_container needs to be specified') do
|
40
57
|
create_driver conf: %(
|
41
58
|
azure_storage_account test_storage_account
|
42
59
|
azure_storage_access_key MY_FAKE_SECRET
|
@@ -47,8 +64,25 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
47
64
|
end
|
48
65
|
end
|
49
66
|
|
67
|
+
test 'config should reject for invalid cloud ' do
|
68
|
+
assert_raise Fluent::ConfigError.new('azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD') do
|
69
|
+
create_driver conf: %(
|
70
|
+
azure_cloud INVALIDCLOUD
|
71
|
+
)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
test 'config should reject for Azure Stack Cloud with no azure storage dns suffix' do
|
76
|
+
assert_raise Fluent::ConfigError.new('azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD') do
|
77
|
+
create_driver conf: %(
|
78
|
+
azure_cloud AZURESTACKCLOUD
|
79
|
+
)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
50
83
|
test 'config with access key should set instance variables' do
|
51
84
|
d = create_driver
|
85
|
+
assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
|
52
86
|
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
53
87
|
assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
|
54
88
|
assert_equal 'test_container', d.instance.azure_container
|
@@ -60,12 +94,29 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
60
94
|
d = create_driver conf: MSI_CONFIG
|
61
95
|
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
62
96
|
assert_equal 'test_container', d.instance.azure_container
|
63
|
-
assert_equal true, d.instance.use_msi
|
97
|
+
assert_equal true, d.instance.instance_variable_get(:@use_msi)
|
64
98
|
assert_equal true, d.instance.auto_create_container
|
65
99
|
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
66
100
|
assert_equal 120, d.instance.azure_token_refresh_interval
|
67
101
|
assert_equal '1970-01-01', d.instance.azure_imds_api_version
|
68
102
|
end
|
103
|
+
|
104
|
+
test 'config with connection string should set instance variables' do
|
105
|
+
d = create_driver conf: CONNSTR_CONFIG
|
106
|
+
assert_equal 'https://test', d.instance.azure_storage_connection_string
|
107
|
+
assert_equal false, d.instance.instance_variable_get(:@use_msi)
|
108
|
+
assert_equal true, d.instance.auto_create_container
|
109
|
+
end
|
110
|
+
|
111
|
+
test 'config for Azure Stack Cloud should set instance variables' do
|
112
|
+
d = create_driver conf: AZURESTACKCLOUD_CONFIG
|
113
|
+
assert_equal 'test.storage.dns.suffix', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
|
114
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
115
|
+
assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
|
116
|
+
assert_equal 'test_container', d.instance.azure_container
|
117
|
+
assert_equal true, d.instance.auto_create_container
|
118
|
+
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
119
|
+
end
|
69
120
|
end
|
70
121
|
|
71
122
|
sub_test_case 'test path slicing' do
|
@@ -92,15 +143,13 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
92
143
|
# This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
|
93
144
|
# a response object when it is created.
|
94
145
|
class FakeResponse
|
95
|
-
def initialize(status=404)
|
146
|
+
def initialize(status = 404)
|
96
147
|
@status = status
|
97
|
-
@body =
|
148
|
+
@body = 'body'
|
98
149
|
@headers = {}
|
99
150
|
end
|
100
151
|
|
101
|
-
attr_reader :status
|
102
|
-
attr_reader :body
|
103
|
-
attr_reader :headers
|
152
|
+
attr_reader :status, :body, :headers
|
104
153
|
end
|
105
154
|
|
106
155
|
# This class is used to test plugin functions which interact with the blob service
|
@@ -111,11 +160,11 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
111
160
|
end
|
112
161
|
attr_reader :blocks
|
113
162
|
|
114
|
-
def append_blob_block(
|
163
|
+
def append_blob_block(_container, _path, data, options={})
|
115
164
|
@blocks.append(data)
|
116
165
|
end
|
117
166
|
|
118
|
-
def get_container_properties(
|
167
|
+
def get_container_properties(_container)
|
119
168
|
unless @response.status_code == 200
|
120
169
|
raise Azure::Core::Http::HTTPError.new(@response)
|
121
170
|
end
|
@@ -125,18 +174,18 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
125
174
|
sub_test_case 'test container_exists' do
|
126
175
|
test 'container 404 returns false' do
|
127
176
|
d = create_driver service: FakeBlobService.new(404)
|
128
|
-
assert_false d.instance.container_exists?
|
177
|
+
assert_false d.instance.container_exists? 'anything'
|
129
178
|
end
|
130
179
|
|
131
180
|
test 'existing container returns true' do
|
132
181
|
d = create_driver service: FakeBlobService.new(200)
|
133
|
-
assert_true d.instance.container_exists?
|
182
|
+
assert_true d.instance.container_exists? 'anything'
|
134
183
|
end
|
135
184
|
|
136
185
|
test 'unexpected exception raises' do
|
137
186
|
d = create_driver service: FakeBlobService.new(500)
|
138
187
|
assert_raise_kind_of Azure::Core::Http::HTTPError do
|
139
|
-
d.instance.container_exists?
|
188
|
+
d.instance.container_exists? 'anything'
|
140
189
|
end
|
141
190
|
end
|
142
191
|
end
|
@@ -175,10 +224,10 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
175
224
|
|
176
225
|
test 'long buffer appends multiple times' do
|
177
226
|
limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
|
178
|
-
|
179
|
-
|
180
|
-
blocks = fake_appended_blocks
|
181
|
-
assert_equal [
|
227
|
+
buf1 = 'a' * limit
|
228
|
+
buf2 = 'a' * 3
|
229
|
+
blocks = fake_appended_blocks buf1 + buf2
|
230
|
+
assert_equal [buf1, buf2], blocks
|
182
231
|
end
|
183
232
|
end
|
184
233
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azure-storage-append-blob-lts
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonas-Taha El Sesiy
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-04-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -87,13 +87,14 @@ dependencies:
|
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '2'
|
89
89
|
description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
|
90
|
-
https://github.com/
|
90
|
+
https://github.com/microsoft/fluent-plugin-azure-storage-append-blob
|
91
91
|
email:
|
92
|
-
-
|
92
|
+
- github@elsesiy.com
|
93
93
|
executables: []
|
94
94
|
extensions: []
|
95
95
|
extra_rdoc_files: []
|
96
96
|
files:
|
97
|
+
- ".github/CODEOWNERS"
|
97
98
|
- ".github/workflows/publish.yaml"
|
98
99
|
- ".github/workflows/test.yaml"
|
99
100
|
- ".gitignore"
|