fluent-plugin-google-cloud 0.10.2 → 0.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +36 -36
- data/fluent-plugin-google-cloud.gemspec +3 -3
- data/lib/fluent/plugin/filter_analyze_config.rb +72 -31
- data/lib/fluent/plugin/monitoring.rb +66 -23
- data/lib/fluent/plugin/out_google_cloud.rb +12 -6
- data/test/plugin/asserts.rb +22 -12
- data/test/plugin/base_test.rb +152 -20
- data/test/plugin/test_filter_analyze_config.rb +68 -26
- data/test/plugin/test_out_google_cloud.rb +51 -75
- data/test/plugin/test_out_google_cloud_grpc.rb +50 -81
- metadata +6 -6
@@ -86,13 +86,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
86
86
|
end
|
87
87
|
d.run
|
88
88
|
assert_prometheus_metric_value(
|
89
|
-
:stackdriver_successful_requests_count, 1,
|
89
|
+
:stackdriver_successful_requests_count, 1,
|
90
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
91
|
+
grpc: false, code: 200)
|
90
92
|
assert_prometheus_metric_value(
|
91
|
-
:stackdriver_ingested_entries_count, 1,
|
93
|
+
:stackdriver_ingested_entries_count, 1,
|
94
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
95
|
+
grpc: false, code: 200)
|
92
96
|
assert_prometheus_metric_value(
|
93
|
-
:stackdriver_dropped_entries_count, 2,
|
97
|
+
:stackdriver_dropped_entries_count, 2,
|
98
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
99
|
+
grpc: false, code: 3)
|
94
100
|
assert_prometheus_metric_value(
|
95
|
-
:stackdriver_dropped_entries_count, 1,
|
101
|
+
:stackdriver_dropped_entries_count, 1,
|
102
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
103
|
+
grpc: false, code: 7)
|
96
104
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
97
105
|
end
|
98
106
|
|
@@ -109,13 +117,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
109
117
|
d.emit('message' => log_entry(0))
|
110
118
|
d.run
|
111
119
|
assert_prometheus_metric_value(
|
112
|
-
:stackdriver_successful_requests_count, 0,
|
120
|
+
:stackdriver_successful_requests_count, 0,
|
121
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
122
|
+
grpc: false, code: 200)
|
113
123
|
assert_prometheus_metric_value(
|
114
|
-
:stackdriver_failed_requests_count, 1,
|
124
|
+
:stackdriver_failed_requests_count, 1,
|
125
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
126
|
+
grpc: false, code: 400)
|
115
127
|
assert_prometheus_metric_value(
|
116
|
-
:stackdriver_ingested_entries_count, 0,
|
128
|
+
:stackdriver_ingested_entries_count, 0,
|
129
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
130
|
+
grpc: false, code: 200)
|
117
131
|
assert_prometheus_metric_value(
|
118
|
-
:stackdriver_dropped_entries_count, 1,
|
132
|
+
:stackdriver_dropped_entries_count, 1,
|
133
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
134
|
+
grpc: false, code: 400)
|
119
135
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
120
136
|
end
|
121
137
|
|
@@ -138,71 +154,6 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
138
154
|
assert_equal 1, exception_count
|
139
155
|
end
|
140
156
|
|
141
|
-
# TODO: The code in the non-gRPC and gRPC tests is nearly identical.
|
142
|
-
# Refactor and remove duplication.
|
143
|
-
# TODO: Use status codes instead of int literals.
|
144
|
-
def test_metrics
|
145
|
-
setup_gce_metadata_stubs
|
146
|
-
[
|
147
|
-
[ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
|
148
|
-
[ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
|
149
|
-
].each do |config, assert_metric_value|
|
150
|
-
[
|
151
|
-
# Single successful request.
|
152
|
-
[200, 1, 1, [1, 0, 1, 0, 0]],
|
153
|
-
# Several successful requests.
|
154
|
-
[200, 2, 1, [2, 0, 2, 0, 0]],
|
155
|
-
# Single successful request with several entries.
|
156
|
-
[200, 1, 2, [1, 0, 2, 0, 0]],
|
157
|
-
# Single failed request that causes logs to be dropped.
|
158
|
-
[401, 1, 1, [0, 1, 0, 1, 0]],
|
159
|
-
# Single failed request that escalates without logs being dropped with
|
160
|
-
# several entries.
|
161
|
-
[500, 1, 2, [0, 0, 0, 0, 2]]
|
162
|
-
].each do |code, request_count, entry_count, metric_values|
|
163
|
-
clear_metrics
|
164
|
-
# TODO: Do this as part of setup_logging_stubs.
|
165
|
-
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
166
|
-
.to_return(status: code, body: 'Some Message')
|
167
|
-
(1..request_count).each do
|
168
|
-
d = create_driver(config)
|
169
|
-
(1..entry_count).each do |i|
|
170
|
-
d.emit('message' => log_entry(i.to_s))
|
171
|
-
end
|
172
|
-
# rubocop:disable Lint/HandleExceptions
|
173
|
-
begin
|
174
|
-
d.run
|
175
|
-
rescue Google::Apis::AuthorizationError
|
176
|
-
rescue Google::Apis::ServerError
|
177
|
-
end
|
178
|
-
# rubocop:enable Lint/HandleExceptions
|
179
|
-
end
|
180
|
-
successful_requests_count, failed_requests_count,
|
181
|
-
ingested_entries_count, dropped_entries_count,
|
182
|
-
retried_entries_count = metric_values
|
183
|
-
assert_metric_value.call(:stackdriver_successful_requests_count,
|
184
|
-
successful_requests_count,
|
185
|
-
grpc: false, code: 200)
|
186
|
-
assert_metric_value.call(:stackdriver_ingested_entries_count,
|
187
|
-
ingested_entries_count,
|
188
|
-
grpc: false, code: 200)
|
189
|
-
assert_metric_value.call(:stackdriver_retried_entries_count,
|
190
|
-
retried_entries_count,
|
191
|
-
grpc: false, code: code)
|
192
|
-
# Skip failure assertions when code indicates success, because the
|
193
|
-
# assertion will fail in the case when a single metric contains time
|
194
|
-
# series with success and failure events.
|
195
|
-
next if code == 200
|
196
|
-
assert_metric_value.call(:stackdriver_failed_requests_count,
|
197
|
-
failed_requests_count,
|
198
|
-
grpc: false, code: code)
|
199
|
-
assert_metric_value.call(:stackdriver_dropped_entries_count,
|
200
|
-
dropped_entries_count,
|
201
|
-
grpc: false, code: code)
|
202
|
-
end
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
157
|
# This test looks similar between the grpc and non-grpc paths except that when
|
207
158
|
# parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
|
208
159
|
# responds with "100".
|
@@ -400,14 +351,39 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
400
351
|
end
|
401
352
|
|
402
353
|
# Set up http stubs to mock the external calls.
|
403
|
-
def setup_logging_stubs
|
354
|
+
def setup_logging_stubs(_error = nil, code = nil, message = 'some message')
|
404
355
|
stub_request(:post, WRITE_LOG_ENTRIES_URI).to_return do |request|
|
405
356
|
@logs_sent << JSON.parse(request.body)
|
406
|
-
{ body:
|
357
|
+
{ status: code, body: message }
|
407
358
|
end
|
408
359
|
yield
|
409
360
|
end
|
410
361
|
|
362
|
+
# Whether this is the grpc path
|
363
|
+
def use_grpc
|
364
|
+
false
|
365
|
+
end
|
366
|
+
|
367
|
+
# The OK status code for the grpc path.
|
368
|
+
def ok_status_code
|
369
|
+
200
|
370
|
+
end
|
371
|
+
|
372
|
+
# A client side error status code for the grpc path.
|
373
|
+
def client_error_status_code
|
374
|
+
401
|
375
|
+
end
|
376
|
+
|
377
|
+
# A server side error status code for the grpc path.
|
378
|
+
def server_error_status_code
|
379
|
+
500
|
380
|
+
end
|
381
|
+
|
382
|
+
# The parent error type to expect in the mock
|
383
|
+
def mock_error_type
|
384
|
+
Google::Apis::Error
|
385
|
+
end
|
386
|
+
|
411
387
|
# The conversions from user input to output.
|
412
388
|
def latency_conversion
|
413
389
|
{
|
@@ -60,8 +60,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
60
60
|
GRPC::Core::StatusCodes::ABORTED => 'Aborted',
|
61
61
|
GRPC::Core::StatusCodes::UNAUTHENTICATED => 'Unauthenticated'
|
62
62
|
}.each_with_index do |(code, message), index|
|
63
|
-
setup_logging_stubs(
|
64
|
-
GRPC::BadStatus.new_status_exception(code, message)) do
|
63
|
+
setup_logging_stubs(nil, code, message) do
|
65
64
|
d = create_driver(USE_GRPC_CONFIG, 'test')
|
66
65
|
# The API Client should not retry this and the plugin should consume the
|
67
66
|
# exception.
|
@@ -99,19 +98,24 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
99
98
|
d.run
|
100
99
|
assert_prometheus_metric_value(
|
101
100
|
:stackdriver_successful_requests_count, 1,
|
102
|
-
|
101
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
102
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
|
103
103
|
assert_prometheus_metric_value(
|
104
104
|
:stackdriver_failed_requests_count, 0,
|
105
|
-
|
105
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
106
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
|
106
107
|
assert_prometheus_metric_value(
|
107
108
|
:stackdriver_ingested_entries_count, 1,
|
108
|
-
|
109
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
110
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
|
109
111
|
assert_prometheus_metric_value(
|
110
112
|
:stackdriver_dropped_entries_count, 2,
|
111
|
-
|
113
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
114
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
|
112
115
|
assert_prometheus_metric_value(
|
113
116
|
:stackdriver_dropped_entries_count, 1,
|
114
|
-
|
117
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
118
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED)
|
115
119
|
end
|
116
120
|
end
|
117
121
|
|
@@ -128,16 +132,20 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
128
132
|
d.run
|
129
133
|
assert_prometheus_metric_value(
|
130
134
|
:stackdriver_successful_requests_count, 0,
|
131
|
-
|
135
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
136
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
|
132
137
|
assert_prometheus_metric_value(
|
133
138
|
:stackdriver_failed_requests_count, 1,
|
134
|
-
|
139
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
140
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
|
135
141
|
assert_prometheus_metric_value(
|
136
142
|
:stackdriver_ingested_entries_count, 0,
|
137
|
-
|
143
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
144
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK)
|
138
145
|
assert_prometheus_metric_value(
|
139
146
|
:stackdriver_dropped_entries_count, 1,
|
140
|
-
|
147
|
+
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
148
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT)
|
141
149
|
end
|
142
150
|
end
|
143
151
|
|
@@ -150,8 +158,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
150
158
|
GRPC::Core::StatusCodes::UNAVAILABLE => 'Unavailable'
|
151
159
|
}.each_with_index do |(code, message), index|
|
152
160
|
exception_count = 0
|
153
|
-
setup_logging_stubs(
|
154
|
-
GRPC::BadStatus.new_status_exception(code, message)) do
|
161
|
+
setup_logging_stubs(nil, code, message) do
|
155
162
|
d = create_driver(USE_GRPC_CONFIG, 'test')
|
156
163
|
# The API client should retry this once, then throw an exception which
|
157
164
|
# gets propagated through the plugin
|
@@ -168,72 +175,6 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
168
175
|
end
|
169
176
|
end
|
170
177
|
|
171
|
-
# TODO: The code in the non-gRPC and gRPC tests is nearly identical.
|
172
|
-
# Refactor and remove duplication.
|
173
|
-
# TODO: Use status codes instead of int literals.
|
174
|
-
def test_metrics
|
175
|
-
setup_gce_metadata_stubs
|
176
|
-
[
|
177
|
-
[ENABLE_PROMETHEUS_CONFIG, method(:assert_prometheus_metric_value)],
|
178
|
-
[ENABLE_OPENCENSUS_CONFIG, method(:assert_opencensus_metric_value)]
|
179
|
-
].each do |config, assert_metric_value|
|
180
|
-
[
|
181
|
-
# Single successful request.
|
182
|
-
[false, 0, 1, 1, [1, 0, 1, 0, 0]],
|
183
|
-
# Several successful requests.
|
184
|
-
[false, 0, 2, 1, [2, 0, 2, 0, 0]],
|
185
|
-
# Single successful request with several entries.
|
186
|
-
[false, 0, 1, 2, [1, 0, 2, 0, 0]],
|
187
|
-
# Single failed request that causes logs to be dropped.
|
188
|
-
[true, 16, 1, 1, [0, 1, 0, 1, 0]],
|
189
|
-
# Single failed request that escalates without logs being dropped with
|
190
|
-
# several entries.
|
191
|
-
[true, 13, 1, 2, [0, 0, 0, 0, 2]]
|
192
|
-
].each do |should_fail, code, request_count, entry_count, metric_values|
|
193
|
-
clear_metrics
|
194
|
-
(1..request_count).each do
|
195
|
-
setup_logging_stubs(
|
196
|
-
if should_fail
|
197
|
-
GRPC::BadStatus.new_status_exception(code, 'SomeMessage')
|
198
|
-
end) do
|
199
|
-
d = create_driver(USE_GRPC_CONFIG + config, 'test')
|
200
|
-
(1..entry_count).each do |i|
|
201
|
-
d.emit('message' => log_entry(i.to_s))
|
202
|
-
end
|
203
|
-
# rubocop:disable Lint/HandleExceptions
|
204
|
-
begin
|
205
|
-
d.run
|
206
|
-
rescue GRPC::BadStatus
|
207
|
-
end
|
208
|
-
# rubocop:enable Lint/HandleExceptions
|
209
|
-
end
|
210
|
-
end
|
211
|
-
successful_requests_count, failed_requests_count,
|
212
|
-
ingested_entries_count, dropped_entries_count,
|
213
|
-
retried_entries_count = metric_values
|
214
|
-
assert_metric_value.call(:stackdriver_successful_requests_count,
|
215
|
-
successful_requests_count,
|
216
|
-
grpc: true, code: 0)
|
217
|
-
assert_metric_value.call(:stackdriver_ingested_entries_count,
|
218
|
-
ingested_entries_count,
|
219
|
-
grpc: true, code: 0)
|
220
|
-
assert_metric_value.call(:stackdriver_retried_entries_count,
|
221
|
-
retried_entries_count,
|
222
|
-
grpc: true, code: code)
|
223
|
-
# Skip failure assertions when code indicates success, because the
|
224
|
-
# assertion will fail in the case when a single metric contains time
|
225
|
-
# series with success and failure events.
|
226
|
-
next if code == 0
|
227
|
-
assert_metric_value.call(:stackdriver_failed_requests_count,
|
228
|
-
failed_requests_count,
|
229
|
-
grpc: true, code: code)
|
230
|
-
assert_metric_value.call(:stackdriver_dropped_entries_count,
|
231
|
-
dropped_entries_count,
|
232
|
-
grpc: true, code: code)
|
233
|
-
end
|
234
|
-
end
|
235
|
-
end
|
236
|
-
|
237
178
|
# This test looks similar between the grpc and non-grpc paths except that when
|
238
179
|
# parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
|
239
180
|
# responds with "100".
|
@@ -428,17 +369,45 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
428
369
|
end
|
429
370
|
|
430
371
|
# Set up grpc stubs to mock the external calls.
|
431
|
-
def setup_logging_stubs(error = nil)
|
432
|
-
if error.nil?
|
372
|
+
def setup_logging_stubs(error = nil, code = nil, message = 'some message')
|
373
|
+
if error.nil? && (code.nil? || code == 0)
|
433
374
|
@requests_sent = []
|
434
375
|
@grpc_stub = GRPCLoggingMockService.new(@requests_sent)
|
435
376
|
else
|
436
377
|
@failed_attempts = []
|
378
|
+
# Only fall back to constructing an error with code and message if no
|
379
|
+
# error is passed in.
|
380
|
+
error ||= GRPC::BadStatus.new_status_exception(code, message)
|
437
381
|
@grpc_stub = GRPCLoggingMockFailingService.new(error, @failed_attempts)
|
438
382
|
end
|
439
383
|
yield
|
440
384
|
end
|
441
385
|
|
386
|
+
# Whether this is the grpc path
|
387
|
+
def use_grpc
|
388
|
+
true
|
389
|
+
end
|
390
|
+
|
391
|
+
# The OK status code for the grpc path.
|
392
|
+
def ok_status_code
|
393
|
+
0
|
394
|
+
end
|
395
|
+
|
396
|
+
# A client side error status code for the grpc path.
|
397
|
+
def client_error_status_code
|
398
|
+
16
|
399
|
+
end
|
400
|
+
|
401
|
+
# A server side error status code for the grpc path.
|
402
|
+
def server_error_status_code
|
403
|
+
13
|
404
|
+
end
|
405
|
+
|
406
|
+
# The parent error type to expect in the mock
|
407
|
+
def mock_error_type
|
408
|
+
GRPC::BadStatus
|
409
|
+
end
|
410
|
+
|
442
411
|
# Verify the number and the content of the log entries match the expectation.
|
443
412
|
# The caller can optionally provide a block which is called for each entry.
|
444
413
|
def verify_log_entries(n, params, payload_type = 'textPayload',
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stackdriver Agents Team
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - '='
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 1.3.
|
33
|
+
version: 1.3.10
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - '='
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 1.3.
|
40
|
+
version: 1.3.10
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: googleauth
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,14 +86,14 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - '='
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 3.
|
89
|
+
version: 3.14.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - '='
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: 3.
|
96
|
+
version: 3.14.0
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: grpc
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|