fluent-plugin-azure-storage-append-blob-lts 0.4.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/workflows/publish.yaml +0 -2
- data/.github/workflows/test.yaml +0 -2
- data/Dockerfile +2 -2
- data/README.md +32 -4
- data/fluent-plugin-azure-storage-append-blob-lts.gemspec +3 -3
- data/lib/fluent/plugin/out_azure-storage-append-blob.rb +83 -46
- data/test/plugin/test_out_azure_storage_append_blob.rb +65 -16
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f9e6de27ebbf624d51968e415d502111523d3fd670d2345558381b03895934d6
|
4
|
+
data.tar.gz: 80fbe42cee9d82adb02afbaf2cfed1fd92b0d4fbbbfb1e67b50e328795d90b23
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3c2e0296f88ae875f19043e322678aa9a7ea3a26e9bb41f9f4292991ff09a6a50022d03c1da48b36e0c724c2e9e83f70ef431e1b9a3fc628507774a8de32943
|
7
|
+
data.tar.gz: a86038ca1acc515dd05df9f86d014cef337c5cc8a70847a15dd94b8f86fe2233c5b0a49be988f26a3da86f298d1aa27465aab4fb904ddd8eeef9e16535291d99
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @elsesiy
|
@@ -11,14 +11,12 @@ jobs:
|
|
11
11
|
build:
|
12
12
|
name: Build + Publish
|
13
13
|
runs-on: ubuntu-latest
|
14
|
-
|
15
14
|
steps:
|
16
15
|
- uses: actions/checkout@v2
|
17
16
|
- name: Set up Ruby 2.6
|
18
17
|
uses: actions/setup-ruby@v1
|
19
18
|
with:
|
20
19
|
ruby-version: 2.6.x
|
21
|
-
|
22
20
|
- name: Publish to RubyGems
|
23
21
|
run: |
|
24
22
|
mkdir -p $HOME/.gem
|
data/.github/workflows/test.yaml
CHANGED
data/Dockerfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# fluent-plugin-azure-storage-append-blob-lts
|
2
2
|
|
3
|
+

|
4
|
+
[](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts)
|
5
|
+
[](http://twitter.com/elsesiy)
|
6
|
+
|
3
7
|
Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
|
4
8
|
This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
|
5
9
|
|
@@ -24,6 +28,8 @@ And then execute:
|
|
24
28
|
<match pattern>
|
25
29
|
type azure-storage-append-blob
|
26
30
|
|
31
|
+
azure_cloud <azure cloud environment>
|
32
|
+
azure_storage_dns_suffix <your azure storage dns suffix> # only used for Azure Stack Cloud
|
27
33
|
azure_storage_account <your azure storage account>
|
28
34
|
azure_storage_access_key <your azure storage access key> # leave empty to use MSI
|
29
35
|
azure_storage_connection_string <your azure storage connection string> # leave empty to use MSI
|
@@ -36,6 +42,7 @@ And then execute:
|
|
36
42
|
path logs/
|
37
43
|
azure_object_key_format %{path}%{time_slice}_%{index}.log
|
38
44
|
time_slice_format %Y%m%d-%H
|
45
|
+
compute_checksums true
|
39
46
|
# if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
|
40
47
|
# need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
|
41
48
|
<buffer tag,time>
|
@@ -47,6 +54,18 @@ And then execute:
|
|
47
54
|
</buffer>
|
48
55
|
</match>
|
49
56
|
|
57
|
+
### `azure_cloud` (Optional)
|
58
|
+
|
59
|
+
Default: `AZUREPUBLICCLOUD`
|
60
|
+
|
61
|
+
Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
|
62
|
+
|
63
|
+
Use `AZURESTACKCLOUD` for Azure Stack Cloud.
|
64
|
+
|
65
|
+
### `azure_storage_dns_suffix` (Required only for Azure Stack Cloud)
|
66
|
+
|
67
|
+
Your Azure Storage endpoint suffix. This can be retrieved from Azure Storage connection string, `EndpointSuffix` section.
|
68
|
+
|
50
69
|
### `azure_storage_account` (Required)
|
51
70
|
|
52
71
|
Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
|
@@ -60,15 +79,15 @@ If all are empty, the plugin will use the local Managed Identity endpoint to obt
|
|
60
79
|
|
61
80
|
### `azure_imds_api_version` (Optional, only for MSI)
|
62
81
|
|
63
|
-
Default:
|
82
|
+
Default: `2020-12-01`
|
64
83
|
|
65
84
|
The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
|
66
85
|
|
67
|
-
See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
|
86
|
+
See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service?tabs=windows#versioning) for more details.
|
68
87
|
|
69
88
|
### `azure_token_refresh_interval` (Optional, only for MSI)
|
70
89
|
|
71
|
-
Default: 60 (1 hour)
|
90
|
+
Default: `60` (1 hour)
|
72
91
|
|
73
92
|
When using MSI, the initial access token needs to be refreshed periodically.
|
74
93
|
|
@@ -82,8 +101,9 @@ Azure Identity Client ID to use for accessing Azure Blob service.
|
|
82
101
|
|
83
102
|
### `auto_create_container`
|
84
103
|
|
104
|
+
Default: `true`
|
105
|
+
|
85
106
|
This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
|
86
|
-
The default value is `true`
|
87
107
|
|
88
108
|
### `azure_object_key_format`
|
89
109
|
|
@@ -127,6 +147,14 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
|
|
127
147
|
|
128
148
|
Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
|
129
149
|
|
150
|
+
### `compute_checksums`
|
151
|
+
|
152
|
+
Default: `true`
|
153
|
+
|
154
|
+
Whether to compute MD5 checksum of the blob contents during append operation and provide it in a header for the blob service.
|
155
|
+
|
156
|
+
You want to set it to `false` in FIPS-enabled environments.
|
157
|
+
|
130
158
|
### Run tests
|
131
159
|
|
132
160
|
gem install bundler
|
@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
|
6
|
-
spec.version = '0.
|
6
|
+
spec.version = '0.6.3'
|
7
7
|
spec.authors = ['Jonas-Taha El Sesiy']
|
8
|
-
spec.email = ['']
|
8
|
+
spec.email = ['github@elsesiy.com']
|
9
9
|
|
10
10
|
spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
|
11
|
-
spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/
|
11
|
+
spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/microsoft/fluent-plugin-azure-storage-append-blob'
|
12
12
|
spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
|
13
13
|
spec.license = 'MIT'
|
14
14
|
|
@@ -11,7 +11,7 @@ require 'json'
|
|
11
11
|
|
12
12
|
module Fluent
|
13
13
|
module Plugin
|
14
|
-
class AzureStorageAppendBlobOut <
|
14
|
+
class AzureStorageAppendBlobOut < Output
|
15
15
|
Fluent::Plugin.register_output('azure-storage-append-blob', self)
|
16
16
|
|
17
17
|
helpers :formatter, :inject
|
@@ -19,18 +19,33 @@ module Fluent
|
|
19
19
|
DEFAULT_FORMAT_TYPE = 'out_file'.freeze
|
20
20
|
AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
|
21
21
|
|
22
|
+
def initialize
|
23
|
+
super
|
24
|
+
@use_msi = false
|
25
|
+
|
26
|
+
# Storage endpoint suffixes for various environments, see https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go
|
27
|
+
@storage_endpoint_mapping = {
|
28
|
+
'AZURECHINACLOUD' => 'core.chinacloudapi.cn',
|
29
|
+
'AZUREGERMANCLOUD' => 'core.cloudapi.de',
|
30
|
+
'AZUREPUBLICCLOUD' => 'core.windows.net',
|
31
|
+
'AZUREUSGOVERNMENTCLOUD' => 'core.usgovcloudapi.net'
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
22
35
|
config_param :path, :string, default: ''
|
23
36
|
config_param :azure_storage_account, :string, default: nil
|
24
37
|
config_param :azure_storage_access_key, :string, default: nil, secret: true
|
25
|
-
config_param :azure_storage_sas_token, :string, default: nil, secret: true
|
26
38
|
config_param :azure_storage_connection_string, :string, default: nil, secret: true
|
39
|
+
config_param :azure_storage_dns_suffix, :string, default: nil
|
40
|
+
config_param :azure_storage_sas_token, :string, default: nil, secret: true
|
41
|
+
config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
|
27
42
|
config_param :azure_msi_client_id, :string, default: nil
|
28
43
|
config_param :azure_container, :string, default: nil
|
29
|
-
config_param :azure_imds_api_version, :string, default: '
|
44
|
+
config_param :azure_imds_api_version, :string, default: '2020-12-01'
|
30
45
|
config_param :azure_token_refresh_interval, :integer, default: 60
|
31
|
-
config_param :use_msi, :bool, default: false
|
32
46
|
config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
|
33
47
|
config_param :auto_create_container, :bool, default: true
|
48
|
+
config_param :compute_checksums, :bool, default: true
|
34
49
|
config_param :format, :string, default: DEFAULT_FORMAT_TYPE
|
35
50
|
config_param :time_slice_format, :string, default: '%Y%m%d'
|
36
51
|
config_param :localtime, :bool, default: false
|
@@ -61,15 +76,31 @@ module Fluent
|
|
61
76
|
end
|
62
77
|
end
|
63
78
|
|
64
|
-
|
65
|
-
|
66
|
-
|
79
|
+
if @azure_cloud == 'AZURESTACKCLOUD'
|
80
|
+
if @azure_storage_dns_suffix.nil?
|
81
|
+
raise ConfigError, 'azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD'
|
82
|
+
end
|
83
|
+
else
|
84
|
+
@azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
|
85
|
+
if @azure_storage_dns_suffix.nil?
|
86
|
+
raise ConfigError, 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD'
|
87
|
+
end
|
88
|
+
end
|
67
89
|
|
68
90
|
if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
|
69
91
|
(@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
|
70
92
|
(@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
|
71
|
-
log.info 'Using MSI since neither azure_storage_access_key
|
93
|
+
log.info 'Using MSI since neither of azure_storage_access_key, azure_storage_sas_token, azure_storage_connection_string was provided.'
|
72
94
|
@use_msi = true
|
95
|
+
elsif @azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?
|
96
|
+
raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
|
97
|
+
raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
|
98
|
+
end
|
99
|
+
|
100
|
+
@blob_options = {}
|
101
|
+
|
102
|
+
if !@compute_checksums
|
103
|
+
@blob_options[:content_md5] = ''
|
73
104
|
end
|
74
105
|
end
|
75
106
|
|
@@ -81,13 +112,13 @@ module Fluent
|
|
81
112
|
access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
|
82
113
|
"api-version=#{@azure_imds_api_version}" \
|
83
114
|
'&resource=https://storage.azure.com/' \
|
84
|
-
"#{!
|
115
|
+
"#{!azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
|
85
116
|
headers: { 'Metadata' => 'true' }).get
|
86
117
|
|
87
118
|
if access_key_request.status == 200
|
88
119
|
JSON.parse(access_key_request.body)['access_token']
|
89
120
|
else
|
90
|
-
raise
|
121
|
+
raise 'Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter.'
|
91
122
|
end
|
92
123
|
end
|
93
124
|
|
@@ -96,7 +127,11 @@ module Fluent
|
|
96
127
|
if @use_msi
|
97
128
|
token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
|
98
129
|
token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
|
99
|
-
@bs = Azure::Storage::Blob::BlobService.new(
|
130
|
+
@bs = Azure::Storage::Blob::BlobService.new(
|
131
|
+
storage_account_name: @azure_storage_account,
|
132
|
+
storage_dns_suffix: @azure_storage_dns_suffix,
|
133
|
+
signer: token_signer
|
134
|
+
)
|
100
135
|
|
101
136
|
refresh_interval = @azure_token_refresh_interval * 60
|
102
137
|
cancelled = false
|
@@ -113,7 +148,7 @@ module Fluent
|
|
113
148
|
elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
|
114
149
|
@bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
|
115
150
|
else
|
116
|
-
@bs_params = { storage_account_name: @azure_storage_account }
|
151
|
+
@bs_params = { storage_account_name: @azure_storage_account, storage_dns_suffix: @azure_storage_dns_suffix }
|
117
152
|
|
118
153
|
if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
|
119
154
|
@bs_params.merge!({ storage_access_key: @azure_storage_access_key })
|
@@ -137,23 +172,26 @@ module Fluent
|
|
137
172
|
end
|
138
173
|
|
139
174
|
def write(chunk)
|
140
|
-
metadata = chunk.metadata
|
141
175
|
tmp = Tempfile.new('azure-')
|
142
176
|
begin
|
143
177
|
chunk.write_to(tmp)
|
144
178
|
|
145
|
-
generate_log_name(
|
179
|
+
generate_log_name(chunk, @current_index)
|
146
180
|
if @last_azure_storage_path != @azure_storage_path
|
147
181
|
@current_index = 0
|
148
|
-
generate_log_name(
|
182
|
+
generate_log_name(chunk, @current_index)
|
149
183
|
end
|
150
184
|
|
151
185
|
content = File.open(tmp.path, 'rb', &:read)
|
152
186
|
|
153
|
-
append_blob(content,
|
187
|
+
append_blob(content, chunk)
|
154
188
|
@last_azure_storage_path = @azure_storage_path
|
155
189
|
ensure
|
156
|
-
|
190
|
+
begin
|
191
|
+
tmp.close(true)
|
192
|
+
rescue StandardError
|
193
|
+
nil
|
194
|
+
end
|
157
195
|
end
|
158
196
|
end
|
159
197
|
|
@@ -182,7 +220,8 @@ module Fluent
|
|
182
220
|
end
|
183
221
|
end
|
184
222
|
|
185
|
-
def generate_log_name(
|
223
|
+
def generate_log_name(chunk, index)
|
224
|
+
metadata = chunk.metadata
|
186
225
|
time_slice = if metadata.timekey.nil?
|
187
226
|
''.freeze
|
188
227
|
else
|
@@ -196,41 +235,39 @@ module Fluent
|
|
196
235
|
'%{index}' => index
|
197
236
|
}
|
198
237
|
storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
|
199
|
-
@azure_storage_path = extract_placeholders(storage_path,
|
238
|
+
@azure_storage_path = extract_placeholders(storage_path, chunk)
|
200
239
|
end
|
201
240
|
|
202
|
-
def append_blob(content,
|
241
|
+
def append_blob(content, chunk)
|
203
242
|
position = 0
|
204
243
|
log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
|
205
244
|
loop do
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
|
223
|
-
raise
|
224
|
-
end
|
225
|
-
|
226
|
-
log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
|
227
|
-
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
228
|
-
elsif status_code == 404 # blob not found
|
229
|
-
log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
|
230
|
-
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
231
|
-
else
|
245
|
+
size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
|
246
|
+
log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
|
247
|
+
@bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1], options=@blob_options)
|
248
|
+
position += size
|
249
|
+
break if position >= content.length
|
250
|
+
rescue Azure::Core::Http::HTTPError => e
|
251
|
+
status_code = e.status_code
|
252
|
+
|
253
|
+
if status_code == 409 # exceeds azure block limit
|
254
|
+
@current_index += 1
|
255
|
+
old_azure_storage_path = @azure_storage_path
|
256
|
+
generate_log_name(chunk, @current_index)
|
257
|
+
|
258
|
+
# If index is not a part of format, rethrow exception.
|
259
|
+
if old_azure_storage_path == @azure_storage_path
|
260
|
+
log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
|
232
261
|
raise
|
233
262
|
end
|
263
|
+
|
264
|
+
log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
|
265
|
+
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
266
|
+
elsif status_code == 404 # blob not found
|
267
|
+
log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
|
268
|
+
@bs.create_append_blob(@azure_container, @azure_storage_path)
|
269
|
+
else
|
270
|
+
raise
|
234
271
|
end
|
235
272
|
end
|
236
273
|
log.debug 'azure_storage_append_blob: append_blob.complete'
|
@@ -11,6 +11,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
11
11
|
end
|
12
12
|
|
13
13
|
CONFIG = %(
|
14
|
+
azure_cloud AZUREGERMANCLOUD
|
14
15
|
azure_storage_account test_storage_account
|
15
16
|
azure_storage_access_key MY_FAKE_SECRET
|
16
17
|
azure_container test_container
|
@@ -18,6 +19,12 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
18
19
|
path log
|
19
20
|
).freeze
|
20
21
|
|
22
|
+
CONNSTR_CONFIG = %(
|
23
|
+
azure_storage_connection_string https://test
|
24
|
+
time_slice_format %Y%m%d-%H
|
25
|
+
path log
|
26
|
+
).freeze
|
27
|
+
|
21
28
|
MSI_CONFIG = %(
|
22
29
|
azure_storage_account test_storage_account
|
23
30
|
azure_container test_container
|
@@ -27,6 +34,16 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
27
34
|
path log
|
28
35
|
).freeze
|
29
36
|
|
37
|
+
AZURESTACKCLOUD_CONFIG = %(
|
38
|
+
azure_cloud AZURESTACKCLOUD
|
39
|
+
azure_storage_dns_suffix test.storage.dns.suffix
|
40
|
+
azure_storage_account test_storage_account
|
41
|
+
azure_storage_access_key MY_FAKE_SECRET
|
42
|
+
azure_container test_container
|
43
|
+
time_slice_format %Y%m%d-%H
|
44
|
+
path log
|
45
|
+
).freeze
|
46
|
+
|
30
47
|
def create_driver(conf: CONFIG, service: nil)
|
31
48
|
d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
|
32
49
|
d.instance.instance_variable_set(:@bs, service)
|
@@ -36,7 +53,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
36
53
|
|
37
54
|
sub_test_case 'test config' do
|
38
55
|
test 'config should reject with no azure container' do
|
39
|
-
assert_raise Fluent::ConfigError do
|
56
|
+
assert_raise Fluent::ConfigError.new('azure_container needs to be specified') do
|
40
57
|
create_driver conf: %(
|
41
58
|
azure_storage_account test_storage_account
|
42
59
|
azure_storage_access_key MY_FAKE_SECRET
|
@@ -47,8 +64,25 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
47
64
|
end
|
48
65
|
end
|
49
66
|
|
67
|
+
test 'config should reject for invalid cloud ' do
|
68
|
+
assert_raise Fluent::ConfigError.new('azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD') do
|
69
|
+
create_driver conf: %(
|
70
|
+
azure_cloud INVALIDCLOUD
|
71
|
+
)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
test 'config should reject for Azure Stack Cloud with no azure storage dns suffix' do
|
76
|
+
assert_raise Fluent::ConfigError.new('azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD') do
|
77
|
+
create_driver conf: %(
|
78
|
+
azure_cloud AZURESTACKCLOUD
|
79
|
+
)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
50
83
|
test 'config with access key should set instance variables' do
|
51
84
|
d = create_driver
|
85
|
+
assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
|
52
86
|
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
53
87
|
assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
|
54
88
|
assert_equal 'test_container', d.instance.azure_container
|
@@ -60,12 +94,29 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
60
94
|
d = create_driver conf: MSI_CONFIG
|
61
95
|
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
62
96
|
assert_equal 'test_container', d.instance.azure_container
|
63
|
-
assert_equal true, d.instance.use_msi
|
97
|
+
assert_equal true, d.instance.instance_variable_get(:@use_msi)
|
64
98
|
assert_equal true, d.instance.auto_create_container
|
65
99
|
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
66
100
|
assert_equal 120, d.instance.azure_token_refresh_interval
|
67
101
|
assert_equal '1970-01-01', d.instance.azure_imds_api_version
|
68
102
|
end
|
103
|
+
|
104
|
+
test 'config with connection string should set instance variables' do
|
105
|
+
d = create_driver conf: CONNSTR_CONFIG
|
106
|
+
assert_equal 'https://test', d.instance.azure_storage_connection_string
|
107
|
+
assert_equal false, d.instance.instance_variable_get(:@use_msi)
|
108
|
+
assert_equal true, d.instance.auto_create_container
|
109
|
+
end
|
110
|
+
|
111
|
+
test 'config for Azure Stack Cloud should set instance variables' do
|
112
|
+
d = create_driver conf: AZURESTACKCLOUD_CONFIG
|
113
|
+
assert_equal 'test.storage.dns.suffix', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
|
114
|
+
assert_equal 'test_storage_account', d.instance.azure_storage_account
|
115
|
+
assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
|
116
|
+
assert_equal 'test_container', d.instance.azure_container
|
117
|
+
assert_equal true, d.instance.auto_create_container
|
118
|
+
assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
|
119
|
+
end
|
69
120
|
end
|
70
121
|
|
71
122
|
sub_test_case 'test path slicing' do
|
@@ -92,15 +143,13 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
92
143
|
# This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
|
93
144
|
# a response object when it is created.
|
94
145
|
class FakeResponse
|
95
|
-
def initialize(status=404)
|
146
|
+
def initialize(status = 404)
|
96
147
|
@status = status
|
97
|
-
@body =
|
148
|
+
@body = 'body'
|
98
149
|
@headers = {}
|
99
150
|
end
|
100
151
|
|
101
|
-
attr_reader :status
|
102
|
-
attr_reader :body
|
103
|
-
attr_reader :headers
|
152
|
+
attr_reader :status, :body, :headers
|
104
153
|
end
|
105
154
|
|
106
155
|
# This class is used to test plugin functions which interact with the blob service
|
@@ -111,11 +160,11 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
111
160
|
end
|
112
161
|
attr_reader :blocks
|
113
162
|
|
114
|
-
def append_blob_block(
|
163
|
+
def append_blob_block(_container, _path, data, options={})
|
115
164
|
@blocks.append(data)
|
116
165
|
end
|
117
166
|
|
118
|
-
def get_container_properties(
|
167
|
+
def get_container_properties(_container)
|
119
168
|
unless @response.status_code == 200
|
120
169
|
raise Azure::Core::Http::HTTPError.new(@response)
|
121
170
|
end
|
@@ -125,18 +174,18 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
125
174
|
sub_test_case 'test container_exists' do
|
126
175
|
test 'container 404 returns false' do
|
127
176
|
d = create_driver service: FakeBlobService.new(404)
|
128
|
-
assert_false d.instance.container_exists?
|
177
|
+
assert_false d.instance.container_exists? 'anything'
|
129
178
|
end
|
130
179
|
|
131
180
|
test 'existing container returns true' do
|
132
181
|
d = create_driver service: FakeBlobService.new(200)
|
133
|
-
assert_true d.instance.container_exists?
|
182
|
+
assert_true d.instance.container_exists? 'anything'
|
134
183
|
end
|
135
184
|
|
136
185
|
test 'unexpected exception raises' do
|
137
186
|
d = create_driver service: FakeBlobService.new(500)
|
138
187
|
assert_raise_kind_of Azure::Core::Http::HTTPError do
|
139
|
-
d.instance.container_exists?
|
188
|
+
d.instance.container_exists? 'anything'
|
140
189
|
end
|
141
190
|
end
|
142
191
|
end
|
@@ -175,10 +224,10 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
|
|
175
224
|
|
176
225
|
test 'long buffer appends multiple times' do
|
177
226
|
limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
|
178
|
-
|
179
|
-
|
180
|
-
blocks = fake_appended_blocks
|
181
|
-
assert_equal [
|
227
|
+
buf1 = 'a' * limit
|
228
|
+
buf2 = 'a' * 3
|
229
|
+
blocks = fake_appended_blocks buf1 + buf2
|
230
|
+
assert_equal [buf1, buf2], blocks
|
182
231
|
end
|
183
232
|
end
|
184
233
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-azure-storage-append-blob-lts
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonas-Taha El Sesiy
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-04-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -87,13 +87,14 @@ dependencies:
|
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '2'
|
89
89
|
description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
|
90
|
-
https://github.com/
|
90
|
+
https://github.com/microsoft/fluent-plugin-azure-storage-append-blob
|
91
91
|
email:
|
92
|
-
-
|
92
|
+
- github@elsesiy.com
|
93
93
|
executables: []
|
94
94
|
extensions: []
|
95
95
|
extra_rdoc_files: []
|
96
96
|
files:
|
97
|
+
- ".github/CODEOWNERS"
|
97
98
|
- ".github/workflows/publish.yaml"
|
98
99
|
- ".github/workflows/test.yaml"
|
99
100
|
- ".gitignore"
|