fluent-plugin-azure-logs-ingestion 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -23
- data/README_ja.md +9 -22
- data/lib/fluent/plugin/azure_logs_ingestion/version.rb +1 -1
- data/test/helper.rb +9 -0
- data/test/test_auth_msi.rb +127 -0
- data/test/test_auth_service_principal.rb +140 -0
- data/test/test_client.rb +200 -0
- data/test/test_out_azure_logs_ingestion.rb +37 -0
- data/test/test_out_azure_logs_ingestion_write.rb +39 -0
- data/test/test_payload_builder.rb +63 -6
- data/test/test_version.rb +10 -0
- metadata +5 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2d502da5879c0445acbdf411a24c2af70b7ad9016eb922474eda10f342d919c4
|
|
4
|
+
data.tar.gz: 2fee3972f2418f56c1cf5823bec48dc0abebe4e8738e108a7f727dca5e4d5b01
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1e64e5b98b5dde247d1a73b9bb437114d87ca39caf79f5bc1dff20264e7980b12d09cfd0e9b8ee748516f9e4e1cccb8c35a5103f281e7fad93454754e348861a
|
|
7
|
+
data.tar.gz: a48c818655f21c3292fbd5e36327df8870e001935f878fcdd76cae804fb5a2341bac47cbe6647f3f2e6830c41e0cb80630b77d0ba78f0e8bfeb51a3a340a21dd
|
data/README.md
CHANGED
|
@@ -8,7 +8,7 @@ Fluentd output plugin that sends records to Log Analytics Workspace tables by us
|
|
|
8
8
|
## Supported Environment
|
|
9
9
|
|
|
10
10
|
- Ruby 2.4 or later
|
|
11
|
-
- Fluentd 1.
|
|
11
|
+
- Fluentd 1.13.0 or later, before 2.0
|
|
12
12
|
|
|
13
13
|
## Installation
|
|
14
14
|
|
|
@@ -35,7 +35,7 @@ Then run `bundle install`.
|
|
|
35
35
|
Bundler can point directly at the GitHub repository. Specify `ref` when you want to pin a specific revision.
|
|
36
36
|
|
|
37
37
|
```ruby
|
|
38
|
-
gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: '
|
|
38
|
+
gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: 'dbb0160a8048957dafb3002799dc3e8567da06e6'
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
Then run `bundle install`.
|
|
@@ -213,21 +213,14 @@ bundle install
|
|
|
213
213
|
bundle exec rake test
|
|
214
214
|
```
|
|
215
215
|
|
|
216
|
-
This project uses a single `Gemfile`.
|
|
217
|
-
|
|
218
|
-
```bash
|
|
219
|
-
bundle install
|
|
220
|
-
bundle exec rake test
|
|
221
|
-
```
|
|
222
|
-
|
|
223
|
-
Use rbenv when you want to check compatibility with older Ruby / Fluentd versions locally. Environment variables are set only inside subshells, so they do not remain in your working shell. Run `rbenv exec` inside the block where `RBENV_VERSION` is set.
|
|
216
|
+
This project uses a single `Gemfile`. Use rbenv and `FLUENTD_VERSION` when checking older Ruby / Fluentd combinations locally.
|
|
224
217
|
|
|
225
218
|
```bash
|
|
226
219
|
rbenv init
|
|
227
220
|
source ~/.bashrc
|
|
228
221
|
|
|
229
222
|
(
|
|
230
|
-
export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.
|
|
223
|
+
export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.13.0
|
|
231
224
|
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
232
225
|
rbenv install "$RBENV_VERSION" -s
|
|
233
226
|
rbenv exec gem install bundler -v 2.3.27 --no-document
|
|
@@ -235,7 +228,6 @@ source ~/.bashrc
|
|
|
235
228
|
rbenv exec bundle _2.3.27_ exec rake test
|
|
236
229
|
)
|
|
237
230
|
|
|
238
|
-
|
|
239
231
|
(
|
|
240
232
|
export RBENV_VERSION=2.7.8 FLUENTD_VERSION=1.18.0
|
|
241
233
|
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
@@ -246,17 +238,11 @@ source ~/.bashrc
|
|
|
246
238
|
)
|
|
247
239
|
```
|
|
248
240
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
RubyGems releases do not need separate gems for each Ruby / Fluentd combination. The gemspec `required_ruby_version` and `fluentd` dependency declare the supported range, so build and push one gem as usual.
|
|
252
|
-
Compatibility checks run tests against the selected Ruby / Fluentd combinations. Build the distributable gem once with the current Ruby.
|
|
241
|
+
Publish one RubyGems artifact. Compatibility is checked with tests; build the distributable gem once with the current Ruby.
|
|
253
242
|
|
|
254
243
|
```bash
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
|
|
260
|
-
gem push fluent-plugin-azure-logs-ingestion-*.gem
|
|
261
|
-
)
|
|
244
|
+
bundle install
|
|
245
|
+
bundle exec rake test
|
|
246
|
+
bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
|
|
247
|
+
gem push fluent-plugin-azure-logs-ingestion-*.gem
|
|
262
248
|
```
|
data/README_ja.md
CHANGED
|
@@ -8,7 +8,7 @@ Azure Monitor Logs Ingestion API を使い、Log Analytics Workspace のテー
|
|
|
8
8
|
## サポート環境
|
|
9
9
|
|
|
10
10
|
- Ruby 2.4 以上
|
|
11
|
-
- Fluentd 1.
|
|
11
|
+
- Fluentd 1.13.0 以上、2.0 未満
|
|
12
12
|
|
|
13
13
|
## インストール
|
|
14
14
|
|
|
@@ -35,7 +35,7 @@ gem 'fluent-plugin-azure-logs-ingestion'
|
|
|
35
35
|
Bundler で GitHub repository を直接指定でき、特定の revision に固定したい場合は、`ref` を指定します。
|
|
36
36
|
|
|
37
37
|
```ruby
|
|
38
|
-
gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: '
|
|
38
|
+
gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: 'dbb0160a8048957dafb3002799dc3e8567da06e6'
|
|
39
39
|
```
|
|
40
40
|
|
|
41
41
|
その後、`bundle install` を実行してください。
|
|
@@ -213,21 +213,14 @@ bundle install
|
|
|
213
213
|
bundle exec rake test
|
|
214
214
|
```
|
|
215
215
|
|
|
216
|
-
この project は 1 つの `Gemfile`
|
|
217
|
-
|
|
218
|
-
```bash
|
|
219
|
-
bundle install
|
|
220
|
-
bundle exec rake test
|
|
221
|
-
```
|
|
222
|
-
|
|
223
|
-
古い Ruby / Fluentd version の互換性を手元で確認する場合は、rbenv を使います。環境変数は subshell の中だけで設定するため、作業中の shell には残りません。`rbenv exec` は `RBENV_VERSION` を設定した block の中で実行してください。
|
|
216
|
+
この project は 1 つの `Gemfile` を使います。古い Ruby / Fluentd version の互換性を手元で確認する場合は、rbenv と `FLUENTD_VERSION` を使います。
|
|
224
217
|
|
|
225
218
|
```bash
|
|
226
219
|
rbenv init
|
|
227
220
|
source ~/.bashrc
|
|
228
221
|
|
|
229
222
|
(
|
|
230
|
-
export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.
|
|
223
|
+
export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.13.0
|
|
231
224
|
unset GEM_HOME GEM_PATH MY_RUBY_HOME
|
|
232
225
|
rbenv install "$RBENV_VERSION" -s
|
|
233
226
|
rbenv exec gem install bundler -v 2.3.27 --no-document
|
|
@@ -245,17 +238,11 @@ source ~/.bashrc
|
|
|
245
238
|
)
|
|
246
239
|
```
|
|
247
240
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
RubyGems へ公開する gem は Ruby / Fluentd の組み合わせごとには分けません。gemspec の `required_ruby_version` と `fluentd` dependency が対応範囲を表すため、通常どおり 1 つの gem を build して push します。
|
|
251
|
-
互換性確認では Ruby / Fluentd の組み合わせごとに test を実行します。配布する gem の build は現在の Ruby で 1 回だけ行います。
|
|
241
|
+
RubyGems へ公開する gem は 1 つだけです。互換性は test で確認し、配布する gem は現在の Ruby で build します。
|
|
252
242
|
|
|
253
243
|
```bash
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
|
|
259
|
-
gem push fluent-plugin-azure-logs-ingestion-*.gem
|
|
260
|
-
)
|
|
244
|
+
bundle install
|
|
245
|
+
bundle exec rake test
|
|
246
|
+
bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
|
|
247
|
+
gem push fluent-plugin-azure-logs-ingestion-*.gem
|
|
261
248
|
```
|
data/test/helper.rb
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
if ENV['COVERAGE']
|
|
4
|
+
require 'simplecov'
|
|
5
|
+
SimpleCov.enable_coverage :branch if SimpleCov.respond_to?(:enable_coverage)
|
|
6
|
+
SimpleCov.start do
|
|
7
|
+
add_filter '/test/'
|
|
8
|
+
add_filter '/vendor/'
|
|
9
|
+
track_files 'lib/**/*.rb'
|
|
10
|
+
end
|
|
11
|
+
end
|
|
3
12
|
require 'test/unit'
|
|
4
13
|
require 'fluent/test'
|
|
5
14
|
|
data/test/test_auth_msi.rb
CHANGED
|
@@ -43,6 +43,73 @@ class AuthManagedIdentityTest < Test::Unit::TestCase
|
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
+
test 'uses app service managed identity secret when header is absent' do
|
|
47
|
+
server = FakeAzureServer.new do |_request|
|
|
48
|
+
[200, { 'Content-Type' => 'application/json' }, { access_token: 'msi-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
|
|
49
|
+
end.start
|
|
50
|
+
|
|
51
|
+
begin
|
|
52
|
+
with_env(
|
|
53
|
+
'IDENTITY_ENDPOINT' => "#{server.url}/msi/token?existing=1",
|
|
54
|
+
'IDENTITY_HEADER' => nil,
|
|
55
|
+
'MSI_SECRET' => 'msi-secret-value',
|
|
56
|
+
'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
|
|
57
|
+
) do
|
|
58
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
59
|
+
use_msi: true,
|
|
60
|
+
tenant_id: nil,
|
|
61
|
+
client_id: '',
|
|
62
|
+
client_secret: nil,
|
|
63
|
+
authority_host: 'https://login.microsoftonline.com',
|
|
64
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
65
|
+
token_refresh_skew: 300,
|
|
66
|
+
logger: TestLogger.new
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
assert_equal 'msi-token', auth.token
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
request = server.requests.first
|
|
73
|
+
query = URI.decode_www_form(URI(request.path).query).to_h
|
|
74
|
+
assert_equal '1', query['existing']
|
|
75
|
+
assert_equal 'msi-secret-value', request.headers['secret']
|
|
76
|
+
assert_nil query['client_id']
|
|
77
|
+
ensure
|
|
78
|
+
server&.stop
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
test 'rejects invalid app service managed identity json response' do
|
|
83
|
+
server = FakeAzureServer.new do |_request|
|
|
84
|
+
[200, { 'Content-Type' => 'application/json' }, '{']
|
|
85
|
+
end.start
|
|
86
|
+
|
|
87
|
+
begin
|
|
88
|
+
with_env(
|
|
89
|
+
'IDENTITY_ENDPOINT' => "#{server.url}/msi/token",
|
|
90
|
+
'IDENTITY_HEADER' => 'identity-header-value',
|
|
91
|
+
'MSI_SECRET' => nil,
|
|
92
|
+
'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
|
|
93
|
+
) do
|
|
94
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
95
|
+
use_msi: true,
|
|
96
|
+
tenant_id: nil,
|
|
97
|
+
client_id: nil,
|
|
98
|
+
client_secret: nil,
|
|
99
|
+
authority_host: 'https://login.microsoftonline.com',
|
|
100
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
101
|
+
token_refresh_skew: 300,
|
|
102
|
+
logger: TestLogger.new
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
error = assert_raise(RuntimeError) { auth.token }
|
|
106
|
+
assert_match(/failed to parse App Service managed identity response/, error.message)
|
|
107
|
+
end
|
|
108
|
+
ensure
|
|
109
|
+
server&.stop
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
46
113
|
test 'uses IMDS endpoint when app service environment is absent' do
|
|
47
114
|
server = FakeAzureServer.new do |_request|
|
|
48
115
|
[200, { 'Content-Type' => 'application/json' }, { access_token: 'imds-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
|
|
@@ -77,4 +144,64 @@ class AuthManagedIdentityTest < Test::Unit::TestCase
|
|
|
77
144
|
server&.stop
|
|
78
145
|
end
|
|
79
146
|
end
|
|
147
|
+
|
|
148
|
+
test 'keeps retryable managed identity status retryable' do
|
|
149
|
+
server = FakeAzureServer.new do |_request|
|
|
150
|
+
[404, { 'Content-Type' => 'application/json' }, '{"error":"not ready"}']
|
|
151
|
+
end.start
|
|
152
|
+
|
|
153
|
+
begin
|
|
154
|
+
with_env(
|
|
155
|
+
'IDENTITY_ENDPOINT' => nil,
|
|
156
|
+
'IDENTITY_HEADER' => nil,
|
|
157
|
+
'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
|
|
158
|
+
) do
|
|
159
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
160
|
+
use_msi: true,
|
|
161
|
+
tenant_id: nil,
|
|
162
|
+
client_id: nil,
|
|
163
|
+
client_secret: nil,
|
|
164
|
+
authority_host: 'https://login.microsoftonline.com',
|
|
165
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
166
|
+
token_refresh_skew: 300,
|
|
167
|
+
logger: TestLogger.new
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
error = assert_raise(RuntimeError) { auth.token }
|
|
171
|
+
assert_match(/404/, error.message)
|
|
172
|
+
end
|
|
173
|
+
ensure
|
|
174
|
+
server&.stop
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
test 'raises unrecoverable error for managed identity client error' do
|
|
179
|
+
server = FakeAzureServer.new do |_request|
|
|
180
|
+
[401, { 'Content-Type' => 'application/json' }, '{"error":"unauthorized"}']
|
|
181
|
+
end.start
|
|
182
|
+
|
|
183
|
+
begin
|
|
184
|
+
with_env(
|
|
185
|
+
'IDENTITY_ENDPOINT' => nil,
|
|
186
|
+
'IDENTITY_HEADER' => nil,
|
|
187
|
+
'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
|
|
188
|
+
) do
|
|
189
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
190
|
+
use_msi: true,
|
|
191
|
+
tenant_id: nil,
|
|
192
|
+
client_id: nil,
|
|
193
|
+
client_secret: nil,
|
|
194
|
+
authority_host: 'https://login.microsoftonline.com',
|
|
195
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
196
|
+
token_refresh_skew: 300,
|
|
197
|
+
logger: TestLogger.new
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
error = assert_raise(Fluent::UnrecoverableError) { auth.token }
|
|
201
|
+
assert_match(/401/, error.message)
|
|
202
|
+
end
|
|
203
|
+
ensure
|
|
204
|
+
server&.stop
|
|
205
|
+
end
|
|
206
|
+
end
|
|
80
207
|
end
|
|
@@ -58,4 +58,144 @@ class AuthServicePrincipalTest < Test::Unit::TestCase
|
|
|
58
58
|
server&.stop
|
|
59
59
|
end
|
|
60
60
|
end
|
|
61
|
+
|
|
62
|
+
test 'raises unrecoverable error for service principal client error' do
|
|
63
|
+
server = FakeAzureServer.new do |_request|
|
|
64
|
+
[401, { 'Content-Type' => 'application/json' }, '{"error":"unauthorized"}']
|
|
65
|
+
end.start
|
|
66
|
+
|
|
67
|
+
begin
|
|
68
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
69
|
+
use_msi: false,
|
|
70
|
+
tenant_id: 'tenant-id',
|
|
71
|
+
client_id: 'client-id',
|
|
72
|
+
client_secret: 'secret',
|
|
73
|
+
authority_host: server.url,
|
|
74
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
75
|
+
token_refresh_skew: 300,
|
|
76
|
+
logger: TestLogger.new
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
error = assert_raise(Fluent::UnrecoverableError) { auth.token }
|
|
80
|
+
assert_match(/401/, error.message)
|
|
81
|
+
ensure
|
|
82
|
+
server&.stop
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
test 'raises retryable error for service principal server error' do
|
|
87
|
+
server = FakeAzureServer.new do |_request|
|
|
88
|
+
[500, { 'Content-Type' => 'application/json' }, '{"error":"server"}']
|
|
89
|
+
end.start
|
|
90
|
+
|
|
91
|
+
begin
|
|
92
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
93
|
+
use_msi: false,
|
|
94
|
+
tenant_id: 'tenant-id',
|
|
95
|
+
client_id: 'client-id',
|
|
96
|
+
client_secret: 'secret',
|
|
97
|
+
authority_host: server.url,
|
|
98
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
99
|
+
token_refresh_skew: 300,
|
|
100
|
+
logger: TestLogger.new
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
error = assert_raise(RuntimeError) { auth.token }
|
|
104
|
+
assert_match(/500/, error.message)
|
|
105
|
+
ensure
|
|
106
|
+
server&.stop
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
test 'rejects token response without expiration' do
|
|
111
|
+
server = FakeAzureServer.new do |_request|
|
|
112
|
+
[200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1' }.to_json]
|
|
113
|
+
end.start
|
|
114
|
+
|
|
115
|
+
begin
|
|
116
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
117
|
+
use_msi: false,
|
|
118
|
+
tenant_id: 'tenant-id',
|
|
119
|
+
client_id: 'client-id',
|
|
120
|
+
client_secret: 'secret',
|
|
121
|
+
authority_host: server.url,
|
|
122
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
123
|
+
token_refresh_skew: 300,
|
|
124
|
+
logger: TestLogger.new
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
error = assert_raise(Fluent::UnrecoverableError) { auth.token }
|
|
128
|
+
assert_match(/expires_on or expires_in/, error.message)
|
|
129
|
+
ensure
|
|
130
|
+
server&.stop
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
test 'rejects token response with invalid expiration' do
|
|
135
|
+
server = FakeAzureServer.new do |_request|
|
|
136
|
+
[200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_on: 'not-a-time' }.to_json]
|
|
137
|
+
end.start
|
|
138
|
+
|
|
139
|
+
begin
|
|
140
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
141
|
+
use_msi: false,
|
|
142
|
+
tenant_id: 'tenant-id',
|
|
143
|
+
client_id: 'client-id',
|
|
144
|
+
client_secret: 'secret',
|
|
145
|
+
authority_host: "#{server.url}/",
|
|
146
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
147
|
+
token_refresh_skew: 300,
|
|
148
|
+
logger: TestLogger.new
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
error = assert_raise(Fluent::UnrecoverableError) { auth.token }
|
|
152
|
+
assert_match(/invalid expiration value/, error.message)
|
|
153
|
+
ensure
|
|
154
|
+
server&.stop
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
test 'wraps service principal connection failures' do
|
|
159
|
+
socket = TCPServer.new('127.0.0.1', 0)
|
|
160
|
+
authority_host = "http://127.0.0.1:#{socket.addr[1]}"
|
|
161
|
+
socket.close
|
|
162
|
+
|
|
163
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
164
|
+
use_msi: false,
|
|
165
|
+
tenant_id: 'tenant-id',
|
|
166
|
+
client_id: 'client-id',
|
|
167
|
+
client_secret: 'secret',
|
|
168
|
+
authority_host: authority_host,
|
|
169
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
170
|
+
token_refresh_skew: 300,
|
|
171
|
+
logger: TestLogger.new
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
error = assert_raise(RuntimeError) { auth.token }
|
|
175
|
+
assert_match(/token request failed/, error.message)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
test 'rejects invalid service principal json response' do
|
|
179
|
+
server = FakeAzureServer.new do |_request|
|
|
180
|
+
[200, { 'Content-Type' => 'application/json' }, '{']
|
|
181
|
+
end.start
|
|
182
|
+
|
|
183
|
+
begin
|
|
184
|
+
auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
|
|
185
|
+
use_msi: false,
|
|
186
|
+
tenant_id: 'tenant-id',
|
|
187
|
+
client_id: 'client-id',
|
|
188
|
+
client_secret: 'secret',
|
|
189
|
+
authority_host: server.url,
|
|
190
|
+
logs_ingestion_scope: 'https://monitor.azure.com/.default',
|
|
191
|
+
token_refresh_skew: 300,
|
|
192
|
+
logger: TestLogger.new
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
error = assert_raise(RuntimeError) { auth.token }
|
|
196
|
+
assert_match(/failed to parse service principal token response/, error.message)
|
|
197
|
+
ensure
|
|
198
|
+
server&.stop
|
|
199
|
+
end
|
|
200
|
+
end
|
|
61
201
|
end
|
data/test/test_client.rb
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'helper'
|
|
4
|
+
require_relative 'support/fake_azure_server'
|
|
5
|
+
require_relative 'support/helpers'
|
|
6
|
+
require 'fluent/plugin/azure_logs_ingestion/client'
|
|
7
|
+
require 'tempfile'
|
|
8
|
+
|
|
9
|
+
class ClientTest < Test::Unit::TestCase
|
|
10
|
+
Payload = Struct.new(:io, :content_encoding, :content_length)
|
|
11
|
+
|
|
12
|
+
def build_payload(body = '[{"message":"hello"}]', content_encoding = nil)
|
|
13
|
+
file = Tempfile.new('azure-logs-ingestion-client-test')
|
|
14
|
+
file.binmode
|
|
15
|
+
file.write(body)
|
|
16
|
+
file.flush
|
|
17
|
+
file.rewind
|
|
18
|
+
Payload.new(file, content_encoding, body.bytesize)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def build_client(server)
|
|
22
|
+
Fluent::Plugin::AzureLogsIngestion::Client.new(
|
|
23
|
+
endpoint: server.url,
|
|
24
|
+
dcr_immutable_id: 'dcr-immutable-id',
|
|
25
|
+
stream_name: 'Custom-MyTable',
|
|
26
|
+
logger: TestLogger.new
|
|
27
|
+
)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def build_client_for_endpoint(endpoint)
|
|
31
|
+
Fluent::Plugin::AzureLogsIngestion::Client.new(
|
|
32
|
+
endpoint: endpoint,
|
|
33
|
+
dcr_immutable_id: 'dcr-immutable-id',
|
|
34
|
+
stream_name: 'Custom-MyTable',
|
|
35
|
+
logger: TestLogger.new
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def with_payload(body = '[{"message":"hello"}]', content_encoding = nil)
|
|
40
|
+
payload = build_payload(body, content_encoding)
|
|
41
|
+
yield payload
|
|
42
|
+
ensure
|
|
43
|
+
payload.io.close! if payload && payload.io
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
test 'sends payload to logs ingestion endpoint' do
|
|
47
|
+
server = FakeAzureServer.new do |_request|
|
|
48
|
+
[200, { 'Content-Type' => 'application/json' }, '{}']
|
|
49
|
+
end.start
|
|
50
|
+
|
|
51
|
+
begin
|
|
52
|
+
with_payload do |payload|
|
|
53
|
+
assert_equal true, build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
request = server.requests.first
|
|
57
|
+
assert_equal 'POST', request.method
|
|
58
|
+
assert_match %r{/dataCollectionRules/dcr-immutable-id/streams/Custom-MyTable\?api-version=2023-01-01}, request.path
|
|
59
|
+
assert_equal 'Bearer token-1', request.headers['authorization']
|
|
60
|
+
assert_equal 'application/json', request.headers['content-type']
|
|
61
|
+
assert_not_nil request.headers['x-ms-client-request-id']
|
|
62
|
+
assert_match(/"message":"hello"/, request.body)
|
|
63
|
+
ensure
|
|
64
|
+
server&.stop
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
test 'sends content encoding when payload is compressed' do
|
|
69
|
+
server = FakeAzureServer.new { |_request| [200, {}, ''] }.start
|
|
70
|
+
|
|
71
|
+
begin
|
|
72
|
+
with_payload('compressed', 'gzip') do |payload|
|
|
73
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
assert_equal 'gzip', server.requests.first.headers['content-encoding']
|
|
77
|
+
ensure
|
|
78
|
+
server&.stop
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
test 'preserves endpoint path when endpoint has trailing slash' do
|
|
83
|
+
server = FakeAzureServer.new { |_request| [200, {}, ''] }.start
|
|
84
|
+
|
|
85
|
+
begin
|
|
86
|
+
with_payload do |payload|
|
|
87
|
+
build_client_for_endpoint("#{server.url}/").send_payload(payload: payload, bearer_token: 'token-1')
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
assert_match %r{\A/dataCollectionRules/dcr-immutable-id/streams/Custom-MyTable\?api-version=2023-01-01\z}, server.requests.first.path
|
|
91
|
+
ensure
|
|
92
|
+
server&.stop
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
test 'rejects invalid json body on successful response' do
|
|
97
|
+
server = FakeAzureServer.new { |_request| [200, { 'Content-Type' => 'application/json' }, '{'] }.start
|
|
98
|
+
|
|
99
|
+
begin
|
|
100
|
+
with_payload do |payload|
|
|
101
|
+
error = assert_raise(RuntimeError) do
|
|
102
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
103
|
+
end
|
|
104
|
+
assert_match(/successful response body was not valid JSON/, error.message)
|
|
105
|
+
end
|
|
106
|
+
ensure
|
|
107
|
+
server&.stop
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
test 'treats documented client errors as unrecoverable' do
|
|
112
|
+
[400, 401, 403, 413].each do |status|
|
|
113
|
+
server = FakeAzureServer.new { |_request| [status, {}, 'client error'] }.start
|
|
114
|
+
|
|
115
|
+
begin
|
|
116
|
+
with_payload do |payload|
|
|
117
|
+
error = assert_raise(Fluent::UnrecoverableError) do
|
|
118
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
119
|
+
end
|
|
120
|
+
assert_match(/#{status}/, error.message)
|
|
121
|
+
end
|
|
122
|
+
ensure
|
|
123
|
+
server&.stop
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
test 'keeps 5xx retryable' do
|
|
129
|
+
server = FakeAzureServer.new { |_request| [500, {}, 'server error'] }.start
|
|
130
|
+
|
|
131
|
+
begin
|
|
132
|
+
with_payload do |payload|
|
|
133
|
+
error = assert_raise(RuntimeError) do
|
|
134
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
135
|
+
end
|
|
136
|
+
assert_match(/500/, error.message)
|
|
137
|
+
end
|
|
138
|
+
ensure
|
|
139
|
+
server&.stop
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
test 'keeps 429 retryable without retry after header' do
|
|
144
|
+
server = FakeAzureServer.new { |_request| [429, {}, 'throttled'] }.start
|
|
145
|
+
|
|
146
|
+
begin
|
|
147
|
+
with_payload do |payload|
|
|
148
|
+
error = assert_raise(RuntimeError) do
|
|
149
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
150
|
+
end
|
|
151
|
+
assert_match(/429/, error.message)
|
|
152
|
+
end
|
|
153
|
+
ensure
|
|
154
|
+
server&.stop
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
test 'treats other client errors as unrecoverable' do
|
|
159
|
+
server = FakeAzureServer.new { |_request| [418, {}, 'teapot'] }.start
|
|
160
|
+
|
|
161
|
+
begin
|
|
162
|
+
with_payload do |payload|
|
|
163
|
+
error = assert_raise(Fluent::UnrecoverableError) do
|
|
164
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
165
|
+
end
|
|
166
|
+
assert_match(/418/, error.message)
|
|
167
|
+
end
|
|
168
|
+
ensure
|
|
169
|
+
server&.stop
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
test 'treats redirects as retryable unexpected responses' do
|
|
174
|
+
server = FakeAzureServer.new { |_request| [302, { 'Location' => 'https://example.invalid/' }, 'redirect'] }.start
|
|
175
|
+
|
|
176
|
+
begin
|
|
177
|
+
with_payload do |payload|
|
|
178
|
+
error = assert_raise(RuntimeError) do
|
|
179
|
+
build_client(server).send_payload(payload: payload, bearer_token: 'token-1')
|
|
180
|
+
end
|
|
181
|
+
assert_match(/302/, error.message)
|
|
182
|
+
end
|
|
183
|
+
ensure
|
|
184
|
+
server&.stop
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
test 'wraps connection failures as retryable runtime errors' do
|
|
189
|
+
socket = TCPServer.new('127.0.0.1', 0)
|
|
190
|
+
endpoint = "http://127.0.0.1:#{socket.addr[1]}"
|
|
191
|
+
socket.close
|
|
192
|
+
|
|
193
|
+
with_payload do |payload|
|
|
194
|
+
error = assert_raise(RuntimeError) do
|
|
195
|
+
build_client_for_endpoint(endpoint).send_payload(payload: payload, bearer_token: 'token-1')
|
|
196
|
+
end
|
|
197
|
+
assert_match(/logs ingestion request failed/, error.message)
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
end
|
|
@@ -57,4 +57,41 @@ class AzureLogsIngestionOutputTest < Test::Unit::TestCase
|
|
|
57
57
|
|
|
58
58
|
assert_match(/tenant_id, client_id, and client_secret/, error.message)
|
|
59
59
|
end
|
|
60
|
+
|
|
61
|
+
test 'rejects invalid endpoint url' do
|
|
62
|
+
error = assert_raise(Fluent::ConfigError) do
|
|
63
|
+
create_driver(<<~CONFIG)
|
|
64
|
+
endpoint not-a-url
|
|
65
|
+
dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
|
|
66
|
+
stream_name Custom-MyTable
|
|
67
|
+
tenant_id test-tenant
|
|
68
|
+
client_id test-client
|
|
69
|
+
client_secret test-secret
|
|
70
|
+
<buffer>
|
|
71
|
+
@type memory
|
|
72
|
+
</buffer>
|
|
73
|
+
CONFIG
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
assert_match(/endpoint must be a valid HTTP or HTTPS URL/, error.message)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
test 'rejects invalid authority host url' do
|
|
80
|
+
error = assert_raise(Fluent::ConfigError) do
|
|
81
|
+
create_driver(<<~CONFIG)
|
|
82
|
+
endpoint https://example.eastus-1.ingest.monitor.azure.com
|
|
83
|
+
authority_host not-a-url
|
|
84
|
+
dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
|
|
85
|
+
stream_name Custom-MyTable
|
|
86
|
+
tenant_id test-tenant
|
|
87
|
+
client_id test-client
|
|
88
|
+
client_secret test-secret
|
|
89
|
+
<buffer>
|
|
90
|
+
@type memory
|
|
91
|
+
</buffer>
|
|
92
|
+
CONFIG
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
assert_match(/authority_host must be a valid HTTP or HTTPS URL/, error.message)
|
|
96
|
+
end
|
|
60
97
|
end
|
|
@@ -175,4 +175,43 @@ class AzureLogsIngestionWriteTest < Test::Unit::TestCase
|
|
|
175
175
|
server&.stop
|
|
176
176
|
end
|
|
177
177
|
end
|
|
178
|
+
|
|
179
|
+
test 'closes payload when sending fails' do
|
|
180
|
+
server = FakeAzureServer.new do |request|
|
|
181
|
+
case request.path
|
|
182
|
+
when %r{/tenant/oauth2/v2.0/token}
|
|
183
|
+
[200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_in: '3600' }.to_json]
|
|
184
|
+
else
|
|
185
|
+
[500, {}, 'server error']
|
|
186
|
+
end
|
|
187
|
+
end.start
|
|
188
|
+
|
|
189
|
+
driver = create_driver(<<~CONFIG)
|
|
190
|
+
endpoint #{server.url}
|
|
191
|
+
authority_host #{server.url}
|
|
192
|
+
dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
|
|
193
|
+
stream_name Custom-MyTable
|
|
194
|
+
tenant_id tenant
|
|
195
|
+
client_id client
|
|
196
|
+
client_secret secret
|
|
197
|
+
<buffer>
|
|
198
|
+
@type memory
|
|
199
|
+
</buffer>
|
|
200
|
+
CONFIG
|
|
201
|
+
|
|
202
|
+
begin
|
|
203
|
+
driver.instance.start
|
|
204
|
+
error = assert_raise(RuntimeError) do
|
|
205
|
+
driver.instance.write(FakeChunk.new([
|
|
206
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
207
|
+
]))
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
assert_match(/500/, error.message)
|
|
211
|
+
ensure
|
|
212
|
+
driver&.instance&.shutdown
|
|
213
|
+
driver&.instance&.close
|
|
214
|
+
server&.stop
|
|
215
|
+
end
|
|
216
|
+
end
|
|
178
217
|
end
|
|
@@ -42,13 +42,18 @@ class PayloadBuilderTest < Test::Unit::TestCase
|
|
|
42
42
|
test 'accepts wide time span' do
|
|
43
43
|
builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
begin
|
|
46
|
+
result = builder.build(FakeChunk.new([
|
|
47
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'first' }],
|
|
48
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 31, 0)), { 'message' => 'second' }]
|
|
49
|
+
]))
|
|
49
50
|
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
records = JSON.parse(result.io.read)
|
|
52
|
+
assert_equal 2, result.record_count
|
|
53
|
+
assert_equal %w[first second], records.map { |record| record['message'] }
|
|
54
|
+
ensure
|
|
55
|
+
result&.close!
|
|
56
|
+
end
|
|
52
57
|
end
|
|
53
58
|
|
|
54
59
|
test 'builds gzip payload when enabled' do
|
|
@@ -121,4 +126,56 @@ class PayloadBuilderTest < Test::Unit::TestCase
|
|
|
121
126
|
|
|
122
127
|
assert_match(/payload size|gzip payload size/, error.message)
|
|
123
128
|
end
|
|
129
|
+
|
|
130
|
+
test 'rejects oversized gzip payload even when raw payload fits' do
|
|
131
|
+
builder_class = Class.new(Fluent::Plugin::AzureLogsIngestion::PayloadBuilder) do
|
|
132
|
+
private
|
|
133
|
+
|
|
134
|
+
def gzip_file_from(source_file)
|
|
135
|
+
gzip_file = Tempfile.new('azure-logs-ingestion-test-gzip')
|
|
136
|
+
gzip_file.binmode
|
|
137
|
+
source_file.rewind
|
|
138
|
+
IO.copy_stream(source_file, gzip_file)
|
|
139
|
+
gzip_file.flush
|
|
140
|
+
gzip_file.rewind
|
|
141
|
+
[gzip_file, Fluent::Plugin::AzureLogsIngestion::PayloadBuilder::MAX_BYTES + 1]
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
builder = builder_class.new(gzip: true)
|
|
145
|
+
|
|
146
|
+
error = assert_raise(Fluent::UnrecoverableError) do
|
|
147
|
+
builder.build(FakeChunk.new([
|
|
148
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
149
|
+
]))
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
assert_match(/gzip payload size/, error.message)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
test 'closes gzip tempfile when validation fails after gzip is built' do
|
|
156
|
+
captured_file = nil
|
|
157
|
+
builder_class = Class.new(Fluent::Plugin::AzureLogsIngestion::PayloadBuilder) do
|
|
158
|
+
define_method(:captured_file) { captured_file }
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
define_method(:gzip_file_from) do |source_file|
|
|
163
|
+
gzip_file, gzip_size = super(source_file)
|
|
164
|
+
captured_file = gzip_file
|
|
165
|
+
[gzip_file, gzip_size]
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def validate!(raw_size:, gzip_size:)
|
|
169
|
+
raise 'forced validation failure'
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
builder = builder_class.new(gzip: true)
|
|
173
|
+
|
|
174
|
+
assert_raise(RuntimeError) do
|
|
175
|
+
builder.build(FakeChunk.new([
|
|
176
|
+
[Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
|
|
177
|
+
]))
|
|
178
|
+
end
|
|
179
|
+
assert_equal true, captured_file.closed?
|
|
180
|
+
end
|
|
124
181
|
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'helper'
|
|
4
|
+
require 'fluent/plugin/azure_logs_ingestion/version'
|
|
5
|
+
|
|
6
|
+
class VersionTest < Test::Unit::TestCase
|
|
7
|
+
test 'defines plugin version' do
|
|
8
|
+
assert_match(/\A\d+\.\d+\.\d+\z/, Fluent::Plugin::AzureLogsIngestion::VERSION)
|
|
9
|
+
end
|
|
10
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-azure-logs-ingestion
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- fukasawah
|
|
@@ -15,7 +15,7 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version:
|
|
18
|
+
version: 1.13.0
|
|
19
19
|
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
21
|
version: '2'
|
|
@@ -25,7 +25,7 @@ dependencies:
|
|
|
25
25
|
requirements:
|
|
26
26
|
- - ">="
|
|
27
27
|
- !ruby/object:Gem::Version
|
|
28
|
-
version:
|
|
28
|
+
version: 1.13.0
|
|
29
29
|
- - "<"
|
|
30
30
|
- !ruby/object:Gem::Version
|
|
31
31
|
version: '2'
|
|
@@ -79,9 +79,11 @@ files:
|
|
|
79
79
|
- test/support/helpers.rb
|
|
80
80
|
- test/test_auth_msi.rb
|
|
81
81
|
- test/test_auth_service_principal.rb
|
|
82
|
+
- test/test_client.rb
|
|
82
83
|
- test/test_out_azure_logs_ingestion.rb
|
|
83
84
|
- test/test_out_azure_logs_ingestion_write.rb
|
|
84
85
|
- test/test_payload_builder.rb
|
|
86
|
+
- test/test_version.rb
|
|
85
87
|
homepage: https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion
|
|
86
88
|
licenses:
|
|
87
89
|
- Apache-2.0
|