fluent-plugin-azure-logs-ingestion 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +3 -0
- data/README.md +53 -0
- data/README_ja.md +53 -1
- data/lib/fluent/plugin/azure_logs_ingestion/auth.rb +2 -2
- data/lib/fluent/plugin/azure_logs_ingestion/payload_builder.rb +2 -10
- data/lib/fluent/plugin/azure_logs_ingestion/version.rb +1 -1
- data/test/support/fake_azure_server.rb +2 -2
- data/test/support/helpers.rb +15 -4
- data/test/test_auth_msi.rb +52 -48
- data/test/test_auth_service_principal.rb +36 -32
- data/test/test_out_azure_logs_ingestion_write.rb +56 -48
- data/test/test_payload_builder.rb +50 -40
- metadata +4 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a1c9a4f3c9cbe762fc5a559e35a38fc3b3e056f8ecf6f830c882fcd11b73bc19
|
|
4
|
+
data.tar.gz: f47c00750ec0a3b7995b6b9ac757d325f36865cbb1e5f18c6809541ac3011e2f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 709e94c88e7ac5b734cce439943a9f00cf42ff68f4a2d4dd4b91b7ff07aec9e472c6e27ae10fddc9964e213b6269de1a98f18d55ca6ea220b35a6568eac8041d
|
|
7
|
+
data.tar.gz: 113f9f8abb4ef51952697ff64ecd6fdc9693216a32620daab2df11bd2b6d837e9c87d5ba4edc37d38151a32f363303a4e060e25d3d0868664c4846474ebe9534
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -5,6 +5,11 @@ Fluentd output plugin that sends records to Log Analytics Workspace tables by us
|
|
|
5
5
|
> [!WARNING]
|
|
6
6
|
> This plugin is experimental and has not yet been sufficiently proven in serious production workloads.
|
|
7
7
|
|
|
8
|
+
## Supported Environment
|
|
9
|
+
|
|
10
|
+
- Ruby 2.4 or later
|
|
11
|
+
- Fluentd 1.x
|
|
12
|
+
|
|
8
13
|
## Installation
|
|
9
14
|
|
|
10
15
|
### RubyGems
|
|
@@ -207,3 +212,51 @@ The `time` in `<buffer time>` is the Fluentd event time, not a `time` field insi
|
|
|
207
212
|
bundle install
|
|
208
213
|
bundle exec rake test
|
|
209
214
|
```
|
|
215
|
+
|
|
216
|
+
This project uses a single `Gemfile`. Set `FLUENTD_VERSION` when you want to test a specific Fluentd version.
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
bundle install
|
|
220
|
+
bundle exec rake test
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Use rbenv when you want to check compatibility with older Ruby / Fluentd versions locally. Environment variables are set only inside subshells, so they do not remain in your working shell. Run `rbenv exec` inside the block where `RBENV_VERSION` is set.
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
rbenv init
|
|
227
|
+
source ~/.bashrc
|
|
228
|
+
|
|
229
|
+
(
|
|
230
|
+
export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.15.3
|
|
231
|
+
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
232
|
+
rbenv install "$RBENV_VERSION" -s
|
|
233
|
+
rbenv exec gem install bundler -v 2.3.27 --no-document
|
|
234
|
+
rbenv exec bundle _2.3.27_ install
|
|
235
|
+
rbenv exec bundle _2.3.27_ exec rake test
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
(
|
|
240
|
+
export RBENV_VERSION=2.7.8 FLUENTD_VERSION=1.18.0
|
|
241
|
+
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
242
|
+
rbenv install "$RBENV_VERSION" -s
|
|
243
|
+
rbenv exec gem install bundler -v 2.3.27 --no-document
|
|
244
|
+
rbenv exec bundle _2.3.27_ install
|
|
245
|
+
rbenv exec bundle _2.3.27_ exec rake test
|
|
246
|
+
)
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
CI also checks Ruby 2.4 / Fluentd 1.15 compatibility.
|
|
250
|
+
|
|
251
|
+
RubyGems releases do not need separate gems for each Ruby / Fluentd combination. The gemspec `required_ruby_version` and `fluentd` dependency declare the supported range, so build and push one gem as usual.
|
|
252
|
+
Compatibility checks run tests against the selected Ruby / Fluentd combinations. Build the distributable gem once with the current Ruby.
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
(
|
|
256
|
+
export BUNDLE_PATH=vendor/bundle
|
|
257
|
+
bundle install
|
|
258
|
+
bundle exec rake test
|
|
259
|
+
bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
|
|
260
|
+
gem push fluent-plugin-azure-logs-ingestion-*.gem
|
|
261
|
+
)
|
|
262
|
+
```
|
data/README_ja.md
CHANGED
|
@@ -5,6 +5,11 @@ Azure Monitor Logs Ingestion API を使い、Log Analytics Workspace のテー
|
|
|
5
5
|
> [!WARNING]
|
|
6
6
|
> この plugin は試験的な実装であり、本格的な production workload での動作実績はまだ十分ではありません。
|
|
7
7
|
|
|
8
|
+
## サポート環境
|
|
9
|
+
|
|
10
|
+
- Ruby 2.4 以上
|
|
11
|
+
- Fluentd 1.x
|
|
12
|
+
|
|
8
13
|
## インストール
|
|
9
14
|
|
|
10
15
|
### RubyGems
|
|
@@ -206,4 +211,51 @@ Log Analytics Workspace の Auxiliary tier へ送信し、DCR transformation で
|
|
|
206
211
|
```bash
|
|
207
212
|
bundle install
|
|
208
213
|
bundle exec rake test
|
|
209
|
-
```
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
この project は 1 つの `Gemfile` を使います。特定の Fluentd version を確認する場合は `FLUENTD_VERSION` を指定してください。
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
bundle install
|
|
220
|
+
bundle exec rake test
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
古い Ruby / Fluentd version の互換性を手元で確認する場合は、rbenv を使います。環境変数は subshell の中だけで設定するため、作業中の shell には残りません。`rbenv exec` は `RBENV_VERSION` を設定した block の中で実行してください。
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
rbenv init
|
|
227
|
+
source ~/.bashrc
|
|
228
|
+
|
|
229
|
+
(
|
|
230
|
+
export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.15.3
|
|
231
|
+
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
232
|
+
rbenv install "$RBENV_VERSION" -s
|
|
233
|
+
rbenv exec gem install bundler -v 2.3.27 --no-document
|
|
234
|
+
rbenv exec bundle _2.3.27_ install
|
|
235
|
+
rbenv exec bundle _2.3.27_ exec rake test
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
(
|
|
239
|
+
export RBENV_VERSION=2.7.8 FLUENTD_VERSION=1.18.0
|
|
240
|
+
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
241
|
+
rbenv install "$RBENV_VERSION" -s
|
|
242
|
+
rbenv exec gem install bundler -v 2.3.27 --no-document
|
|
243
|
+
rbenv exec bundle _2.3.27_ install
|
|
244
|
+
rbenv exec bundle _2.3.27_ exec rake test
|
|
245
|
+
)
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Ruby 2.4 / Fluentd 1.15 の互換性は CI でも確認します。
|
|
249
|
+
|
|
250
|
+
RubyGems へ公開する gem は Ruby / Fluentd の組み合わせごとには分けません。gemspec の `required_ruby_version` と `fluentd` dependency が対応範囲を表すため、通常どおり 1 つの gem を build して push します。
|
|
251
|
+
互換性確認では Ruby / Fluentd の組み合わせごとに test を実行します。配布する gem の build は現在の Ruby で 1 回だけ行います。
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
(
|
|
255
|
+
export BUNDLE_PATH=vendor/bundle
|
|
256
|
+
bundle install
|
|
257
|
+
bundle exec rake test
|
|
258
|
+
bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
|
|
259
|
+
gem push fluent-plugin-azure-logs-ingestion-*.gem
|
|
260
|
+
)
|
|
261
|
+
```
|
|
@@ -10,7 +10,7 @@ module Fluent
|
|
|
10
10
|
module Plugin
|
|
11
11
|
module AzureLogsIngestion
|
|
12
12
|
class Auth
|
|
13
|
-
Token = Struct.new(:value, :expires_at
|
|
13
|
+
Token = Struct.new(:value, :expires_at)
|
|
14
14
|
|
|
15
15
|
IMDS_API_VERSION = '2018-02-01'
|
|
16
16
|
APP_SERVICE_API_VERSION = '2019-08-01'
|
|
@@ -159,7 +159,7 @@ module Fluent
|
|
|
159
159
|
def build_token_from_token_response(body)
|
|
160
160
|
token = body.fetch('access_token')
|
|
161
161
|
expires_at = parse_token_expiry(body)
|
|
162
|
-
Token.new(
|
|
162
|
+
Token.new(token, expires_at)
|
|
163
163
|
end
|
|
164
164
|
|
|
165
165
|
def parse_token_expiry(body)
|
|
@@ -15,8 +15,7 @@ module Fluent
|
|
|
15
15
|
:content_length,
|
|
16
16
|
:raw_size,
|
|
17
17
|
:gzip_size,
|
|
18
|
-
:record_count
|
|
19
|
-
keyword_init: true
|
|
18
|
+
:record_count
|
|
20
19
|
) do
|
|
21
20
|
def close!
|
|
22
21
|
io.close!
|
|
@@ -65,14 +64,7 @@ module Fluent
|
|
|
65
64
|
|
|
66
65
|
validate!(raw_size: raw_size, gzip_size: gzip_size)
|
|
67
66
|
|
|
68
|
-
Result.new(
|
|
69
|
-
io: io,
|
|
70
|
-
content_encoding: content_encoding,
|
|
71
|
-
content_length: content_length,
|
|
72
|
-
raw_size: raw_size,
|
|
73
|
-
gzip_size: gzip_size,
|
|
74
|
-
record_count: record_count
|
|
75
|
-
)
|
|
67
|
+
Result.new(io, content_encoding, content_length, raw_size, gzip_size, record_count)
|
|
76
68
|
rescue StandardError
|
|
77
69
|
gzip_file.close! if gzip_file
|
|
78
70
|
raw_file.close! if raw_file
|
|
@@ -4,7 +4,7 @@ require 'socket'
|
|
|
4
4
|
require 'thread'
|
|
5
5
|
|
|
6
6
|
class FakeAzureServer
|
|
7
|
-
Request = Struct.new(:method, :path, :headers, :body
|
|
7
|
+
Request = Struct.new(:method, :path, :headers, :body)
|
|
8
8
|
|
|
9
9
|
attr_reader :requests
|
|
10
10
|
|
|
@@ -61,7 +61,7 @@ class FakeAzureServer
|
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
body = read_body(socket, headers)
|
|
64
|
-
request = Request.new(method
|
|
64
|
+
request = Request.new(method, path, headers, body)
|
|
65
65
|
@mutex.synchronize { @requests << request }
|
|
66
66
|
status, response_headers, response_body = @handler.call(request)
|
|
67
67
|
write_response(socket, status, response_headers || {}, response_body || '')
|
data/test/support/helpers.rb
CHANGED
|
@@ -5,10 +5,21 @@ require 'time'
|
|
|
5
5
|
require 'uri'
|
|
6
6
|
|
|
7
7
|
class TestLogger
|
|
8
|
-
def debug(*)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
def debug(*)
|
|
9
|
+
nil
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def info(*)
|
|
13
|
+
nil
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def warn(*)
|
|
17
|
+
nil
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def error(*)
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
12
23
|
end
|
|
13
24
|
|
|
14
25
|
FakeChunk = Struct.new(:events, :chunk_id) do
|
data/test/test_auth_msi.rb
CHANGED
|
@@ -13,32 +13,34 @@ class AuthManagedIdentityTest < Test::Unit::TestCase
|
|
|
13
13
|
[200, { 'Content-Type' => 'application/json' }, { access_token: 'msi-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
|
|
14
14
|
end.start
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
16
|
+
begin
|
|
17
|
+
with_env(
|
|
18
|
+
'IDENTITY_ENDPOINT' => "#{server.url}/msi/token",
|
|
19
|
+
'IDENTITY_HEADER' => 'identity-header-value',
|
|
20
|
+
'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
|
|
21
|
+
) do
|
|
22
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
23
|
+
use_msi: true,
|
|
24
|
+
tenant_id: nil,
|
|
25
|
+
client_id: 'user-assigned-client-id',
|
|
26
|
+
client_secret: nil,
|
|
27
|
+
authority_host: 'https://login.microsoftonline.com',
|
|
28
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
29
|
+
token_refresh_skew: 300,
|
|
30
|
+
logger: TestLogger.new
|
|
31
|
+
)
|
|
31
32
|
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
assert_equal 'msi-token', auth.token
|
|
34
|
+
end
|
|
34
35
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
36
|
+
request = server.requests.first
|
|
37
|
+
query = URI.decode_www_form(URI(request.path).query).to_h
|
|
38
|
+
assert_equal 'identity-header-value', request.headers['x-identity-header']
|
|
39
|
+
assert_equal 'https://monitor.azure.com/', query['resource']
|
|
40
|
+
assert_equal 'user-assigned-client-id', query['client_id']
|
|
41
|
+
ensure
|
|
42
|
+
server&.stop
|
|
43
|
+
end
|
|
42
44
|
end
|
|
43
45
|
|
|
44
46
|
test 'uses IMDS endpoint when app service environment is absent' do
|
|
@@ -46,31 +48,33 @@ class AuthManagedIdentityTest < Test::Unit::TestCase
|
|
|
46
48
|
[200, { 'Content-Type' => 'application/json' }, { access_token: 'imds-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
|
|
47
49
|
end.start
|
|
48
50
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
51
|
+
begin
|
|
52
|
+
with_env(
|
|
53
|
+
'IDENTITY_ENDPOINT' => nil,
|
|
54
|
+
'IDENTITY_HEADER' => nil,
|
|
55
|
+
'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
|
|
56
|
+
) do
|
|
57
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
58
|
+
use_msi: true,
|
|
59
|
+
tenant_id: nil,
|
|
60
|
+
client_id: 'user-assigned-client-id',
|
|
61
|
+
client_secret: nil,
|
|
62
|
+
authority_host: 'https://login.microsoftonline.com',
|
|
63
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
64
|
+
token_refresh_skew: 300,
|
|
65
|
+
logger: TestLogger.new
|
|
66
|
+
)
|
|
64
67
|
|
|
65
|
-
|
|
66
|
-
|
|
68
|
+
assert_equal 'imds-token', auth.token
|
|
69
|
+
end
|
|
67
70
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
71
|
+
request = server.requests.first
|
|
72
|
+
query = URI.decode_www_form(URI(request.path).query).to_h
|
|
73
|
+
assert_equal 'true', request.headers['metadata']
|
|
74
|
+
assert_equal 'https://monitor.azure.com/', query['resource']
|
|
75
|
+
assert_equal 'user-assigned-client-id', query['client_id']
|
|
76
|
+
ensure
|
|
77
|
+
server&.stop
|
|
78
|
+
end
|
|
75
79
|
end
|
|
76
80
|
end
|
|
@@ -11,24 +11,26 @@ class AuthServicePrincipalTest < Test::Unit::TestCase
|
|
|
11
11
|
[200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_in: '3600' }.to_json]
|
|
12
12
|
end.start
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
14
|
+
begin
|
|
15
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
16
|
+
use_msi: false,
|
|
17
|
+
tenant_id: 'tenant-id',
|
|
18
|
+
client_id: 'client-id',
|
|
19
|
+
client_secret: 'secret',
|
|
20
|
+
authority_host: server.url,
|
|
21
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
22
|
+
token_refresh_skew: 300,
|
|
23
|
+
logger: TestLogger.new
|
|
24
|
+
)
|
|
24
25
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
26
|
+
assert_equal 'token-1', auth.token
|
|
27
|
+
assert_equal 'token-1', auth.token
|
|
28
|
+
assert_equal 1, server.requests.size
|
|
29
|
+
assert_match %r{/tenant-id/oauth2/v2.0/token}, server.requests.first.path
|
|
30
|
+
assert_match(/scope=https%3A%2F%2Fmonitor\.azure\.com%2F\.default/, server.requests.first.body)
|
|
31
|
+
ensure
|
|
32
|
+
server&.stop
|
|
33
|
+
end
|
|
32
34
|
end
|
|
33
35
|
|
|
34
36
|
test 'refreshes service principal token when it is already expired' do
|
|
@@ -37,21 +39,23 @@ class AuthServicePrincipalTest < Test::Unit::TestCase
|
|
|
37
39
|
[200, { 'Content-Type' => 'application/json' }, { access_token: tokens.shift, expires_in: '0' }.to_json]
|
|
38
40
|
end.start
|
|
39
41
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
42
|
+
begin
|
|
43
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
44
|
+
use_msi: false,
|
|
45
|
+
tenant_id: 'tenant-id',
|
|
46
|
+
client_id: 'client-id',
|
|
47
|
+
client_secret: 'secret',
|
|
48
|
+
authority_host: server.url,
|
|
49
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
50
|
+
token_refresh_skew: 0,
|
|
51
|
+
logger: TestLogger.new
|
|
52
|
+
)
|
|
50
53
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
assert_equal 'token-1', auth.token
|
|
55
|
+
assert_equal 'token-2', auth.token
|
|
56
|
+
assert_equal 2, server.requests.size
|
|
57
|
+
ensure
|
|
58
|
+
server&.stop
|
|
59
|
+
end
|
|
56
60
|
end
|
|
57
61
|
end
|
|
@@ -38,19 +38,21 @@ class AzureLogsIngestionWriteTest < Test::Unit::TestCase
|
|
|
38
38
|
</buffer>
|
|
39
39
|
CONFIG
|
|
40
40
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
41
|
+
begin
|
|
42
|
+
driver.instance.start
|
|
43
|
+
driver.instance.write(FakeChunk.new([
|
|
44
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
45
|
+
]))
|
|
46
|
+
|
|
47
|
+
ingestion_request = server.requests.last
|
|
48
|
+
assert_equal 'Bearer token-1', ingestion_request.headers['authorization']
|
|
49
|
+
assert_match(/"message":"hello"/, ingestion_request.body)
|
|
50
|
+
assert_not_match(/"TimeGenerated":/, ingestion_request.body)
|
|
51
|
+
ensure
|
|
52
|
+
driver&.instance&.shutdown
|
|
53
|
+
driver&.instance&.close
|
|
54
|
+
server&.stop
|
|
55
|
+
end
|
|
54
56
|
end
|
|
55
57
|
|
|
56
58
|
test 'raises unrecoverable error for 400 response' do
|
|
@@ -76,18 +78,20 @@ class AzureLogsIngestionWriteTest < Test::Unit::TestCase
|
|
|
76
78
|
</buffer>
|
|
77
79
|
CONFIG
|
|
78
80
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
begin
|
|
82
|
+
driver.instance.start
|
|
83
|
+
error = assert_raise(Fluent::UnrecoverableError) do
|
|
84
|
+
driver.instance.write(FakeChunk.new([
|
|
85
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
86
|
+
]))
|
|
87
|
+
end
|
|
85
88
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
assert_match(/400/, error.message)
|
|
90
|
+
ensure
|
|
91
|
+
driver&.instance&.shutdown
|
|
92
|
+
driver&.instance&.close
|
|
93
|
+
server&.stop
|
|
94
|
+
end
|
|
91
95
|
end
|
|
92
96
|
|
|
93
97
|
test 'sends gzip payload when enabled' do
|
|
@@ -116,19 +120,21 @@ class AzureLogsIngestionWriteTest < Test::Unit::TestCase
|
|
|
116
120
|
</buffer>
|
|
117
121
|
CONFIG
|
|
118
122
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
123
|
+
begin
|
|
124
|
+
driver.instance.start
|
|
125
|
+
driver.instance.write(FakeChunk.new([
|
|
126
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
127
|
+
]))
|
|
128
|
+
|
|
129
|
+
ingestion_request = server.requests.last
|
|
130
|
+
json = Zlib::GzipReader.new(StringIO.new(ingestion_request.body)).read
|
|
131
|
+
assert_equal 'gzip', ingestion_request.headers['content-encoding']
|
|
132
|
+
assert_match(/"message":"hello"/, json)
|
|
133
|
+
ensure
|
|
134
|
+
driver&.instance&.shutdown
|
|
135
|
+
driver&.instance&.close
|
|
136
|
+
server&.stop
|
|
137
|
+
end
|
|
132
138
|
end
|
|
133
139
|
|
|
134
140
|
test 'keeps 429 retryable' do
|
|
@@ -154,17 +160,19 @@ class AzureLogsIngestionWriteTest < Test::Unit::TestCase
|
|
|
154
160
|
</buffer>
|
|
155
161
|
CONFIG
|
|
156
162
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
+
begin
|
|
164
|
+
driver.instance.start
|
|
165
|
+
error = assert_raise(RuntimeError) do
|
|
166
|
+
driver.instance.write(FakeChunk.new([
|
|
167
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
168
|
+
]))
|
|
169
|
+
end
|
|
163
170
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
171
|
+
assert_match(/429/, error.message)
|
|
172
|
+
ensure
|
|
173
|
+
driver&.instance&.shutdown
|
|
174
|
+
driver&.instance&.close
|
|
175
|
+
server&.stop
|
|
176
|
+
end
|
|
169
177
|
end
|
|
170
178
|
end
|
|
@@ -9,30 +9,34 @@ class PayloadBuilderTest < Test::Unit::TestCase
|
|
|
9
9
|
test 'does not inject TimeGenerated by default' do
|
|
10
10
|
builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
begin
|
|
13
|
+
result = builder.build(FakeChunk.new([
|
|
14
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
15
|
+
]))
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
body = result.io.read
|
|
18
|
+
assert_match(/"message":"hello"/, body)
|
|
19
|
+
assert_not_match(/"TimeGenerated":/, body)
|
|
20
|
+
ensure
|
|
21
|
+
result&.close!
|
|
22
|
+
end
|
|
21
23
|
end
|
|
22
24
|
|
|
23
25
|
test 'builds payload from event time' do
|
|
24
26
|
builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
begin
|
|
29
|
+
result = builder.build(FakeChunk.new([
|
|
30
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
31
|
+
]))
|
|
29
32
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
body = result.io.read
|
|
34
|
+
assert_match(/"message":"hello"/, body)
|
|
35
|
+
assert_equal 1, result.record_count
|
|
36
|
+
assert_equal result.raw_size, result.content_length
|
|
37
|
+
ensure
|
|
38
|
+
result&.close!
|
|
39
|
+
end
|
|
36
40
|
end
|
|
37
41
|
|
|
38
42
|
test 'accepts wide time span' do
|
|
@@ -50,17 +54,19 @@ class PayloadBuilderTest < Test::Unit::TestCase
|
|
|
50
54
|
test 'builds gzip payload when enabled' do
|
|
51
55
|
builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: true)
|
|
52
56
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
57
|
+
begin
|
|
58
|
+
result = builder.build(FakeChunk.new([
|
|
59
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
60
|
+
]))
|
|
56
61
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
json = Zlib::GzipReader.new(result.io).read
|
|
63
|
+
assert_equal 'gzip', result.content_encoding
|
|
64
|
+
assert_match(/"message":"hello"/, json)
|
|
65
|
+
assert_not_nil result.gzip_size
|
|
66
|
+
ensure
|
|
67
|
+
result&.io&.rewind
|
|
68
|
+
result&.close!
|
|
69
|
+
end
|
|
64
70
|
end
|
|
65
71
|
|
|
66
72
|
test 'rejects oversized payload' do
|
|
@@ -78,26 +84,30 @@ class PayloadBuilderTest < Test::Unit::TestCase
|
|
|
78
84
|
test 'accepts non-parseable event time when payload is otherwise valid' do
|
|
79
85
|
builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
|
|
80
86
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
87
|
+
begin
|
|
88
|
+
result = builder.build(FakeChunk.new([
|
|
89
|
+
['not-a-time', { 'message' => 'hello' }]
|
|
90
|
+
]))
|
|
84
91
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
92
|
+
assert_match(/"message":"hello"/, result.io.read)
|
|
93
|
+
ensure
|
|
94
|
+
result&.close!
|
|
95
|
+
end
|
|
88
96
|
end
|
|
89
97
|
|
|
90
98
|
test 'accepts payload records with existing TimeGenerated' do
|
|
91
99
|
builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
|
|
92
100
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
101
|
+
begin
|
|
102
|
+
result = builder.build(FakeChunk.new([
|
|
103
|
+
['not-a-time', { 'message' => 'hello', 'TimeGenerated' => '2026-01-01T00:00:00Z' }]
|
|
104
|
+
]))
|
|
96
105
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
106
|
+
body = result.io.read
|
|
107
|
+
assert_match(/"TimeGenerated":"2026-01-01T00:00:00Z"/, body)
|
|
108
|
+
ensure
|
|
109
|
+
result&.close!
|
|
110
|
+
end
|
|
101
111
|
end
|
|
102
112
|
|
|
103
113
|
test 'rejects oversized gzip payload' do
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-azure-logs-ingestion
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- fukasawah
|
|
@@ -15,7 +15,7 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '1.
|
|
18
|
+
version: '1.15'
|
|
19
19
|
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
21
|
version: '2'
|
|
@@ -25,7 +25,7 @@ dependencies:
|
|
|
25
25
|
requirements:
|
|
26
26
|
- - ">="
|
|
27
27
|
- !ruby/object:Gem::Version
|
|
28
|
-
version: '1.
|
|
28
|
+
version: '1.15'
|
|
29
29
|
- - "<"
|
|
30
30
|
- !ruby/object:Gem::Version
|
|
31
31
|
version: '2'
|
|
@@ -95,7 +95,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
95
95
|
requirements:
|
|
96
96
|
- - ">="
|
|
97
97
|
- !ruby/object:Gem::Version
|
|
98
|
-
version: '
|
|
98
|
+
version: '2.4'
|
|
99
99
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
100
|
requirements:
|
|
101
101
|
- - ">="
|