fluent-plugin-google-cloud 0.12.10 → 0.12.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +26 -19
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +9 -9
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -11
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +222 -166
- data/lib/fluent/plugin/statusz.rb +7 -9
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +194 -134
- data/test/plugin/constants.rb +18 -14
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +33 -21
- data/test/plugin/test_out_google_cloud_grpc.rb +35 -22
- data/test/plugin/utils.rb +8 -6
- metadata +29 -29
data/test/plugin/constants.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
# Copyright 2017 Google Inc. All rights reserved.
|
3
2
|
#
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
@@ -228,13 +227,11 @@ module Constants
|
|
228
227
|
detect_json true
|
229
228
|
).freeze
|
230
229
|
|
231
|
-
# rubocop:disable Metrics/LineLength
|
232
230
|
PRIVATE_KEY_CONFIG = %(
|
233
231
|
auth_method private_key
|
234
232
|
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
235
233
|
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
236
234
|
).freeze
|
237
|
-
# rubocop:enable Metrics/LineLength
|
238
235
|
|
239
236
|
REQUIRE_VALID_TAGS_CONFIG = %(
|
240
237
|
require_valid_tags true
|
@@ -433,7 +430,6 @@ module Constants
|
|
433
430
|
monitoring_type not_prometheus
|
434
431
|
).freeze
|
435
432
|
|
436
|
-
# rubocop:disable Metrics/LineLength
|
437
433
|
CONFIG_METRICS_RESOURCE_JSON = %(
|
438
434
|
enable_monitoring true
|
439
435
|
monitoring_type opencensus
|
@@ -481,7 +477,6 @@ module Constants
|
|
481
477
|
monitoring_type opencensus
|
482
478
|
metrics_resource {"type":"custom_resource","labels.label1":"123","random":"x"}
|
483
479
|
).freeze
|
484
|
-
# rubocop:enable Metrics/LineLength
|
485
480
|
|
486
481
|
# For statusz.
|
487
482
|
CONFIG_STATUSZ = %(
|
@@ -1028,29 +1023,36 @@ module Constants
|
|
1028
1023
|
0 => Google::Rpc::Status.new(
|
1029
1024
|
code: GRPC::Core::StatusCodes::PERMISSION_DENIED,
|
1030
1025
|
message: 'User not authorized.',
|
1031
|
-
details: []
|
1026
|
+
details: []
|
1027
|
+
),
|
1032
1028
|
1 => Google::Rpc::Status.new(
|
1033
1029
|
code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
|
1034
1030
|
message: 'Log name contains illegal character :',
|
1035
|
-
details: []
|
1031
|
+
details: []
|
1032
|
+
),
|
1036
1033
|
3 => Google::Rpc::Status.new(
|
1037
1034
|
code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
|
1038
1035
|
message: 'Log name contains illegal character :',
|
1039
|
-
details: []
|
1040
|
-
|
1036
|
+
details: []
|
1037
|
+
)
|
1038
|
+
}
|
1039
|
+
)
|
1041
1040
|
status = Google::Rpc::Status.new(
|
1042
1041
|
message: 'User not authorized.',
|
1043
|
-
details: [Google::Protobuf::Any.pack(partial_errors)]
|
1042
|
+
details: [Google::Protobuf::Any.pack(partial_errors)]
|
1043
|
+
)
|
1044
1044
|
debug_info = Google::Rpc::DebugInfo.new(
|
1045
1045
|
detail: '[ORIGINAL ERROR] generic::permission_denied: User not' \
|
1046
1046
|
' authorized. [google.rpc.error_details_ext] { message:' \
|
1047
1047
|
" #{status.message.inspect} details { type_url:" \
|
1048
1048
|
" #{status.details[0].type_url.inspect} value:" \
|
1049
|
-
" #{status.details[0].value.inspect_octal} } }"
|
1049
|
+
" #{status.details[0].value.inspect_octal} } }"
|
1050
|
+
)
|
1050
1051
|
status_details = Google::Rpc::Status.new(
|
1051
1052
|
code: 7, message: 'User not authorized.',
|
1052
1053
|
details: [Google::Protobuf::Any.pack(partial_errors),
|
1053
|
-
Google::Protobuf::Any.pack(debug_info)]
|
1054
|
+
Google::Protobuf::Any.pack(debug_info)]
|
1055
|
+
)
|
1054
1056
|
{
|
1055
1057
|
'google.logging.v2.writelogentriespartialerrors-bin' =>
|
1056
1058
|
partial_errors.to_proto,
|
@@ -1078,10 +1080,12 @@ module Constants
|
|
1078
1080
|
PARSE_ERROR_GRPC_METADATA = begin
|
1079
1081
|
debug_info = Google::Rpc::DebugInfo.new(
|
1080
1082
|
detail: '[ORIGINAL ERROR] RPC::CLIENT_ERROR: server could not parse' \
|
1081
|
-
" request sent by client; initialization error is: ''"
|
1083
|
+
" request sent by client; initialization error is: ''"
|
1084
|
+
)
|
1082
1085
|
status_details = Google::Rpc::Status.new(
|
1083
1086
|
code: 3, message: 'internal client error',
|
1084
|
-
details: [Google::Protobuf::Any.pack(debug_info)]
|
1087
|
+
details: [Google::Protobuf::Any.pack(debug_info)]
|
1088
|
+
)
|
1085
1089
|
{
|
1086
1090
|
'google.rpc.debuginfo-bin' => debug_info.to_proto,
|
1087
1091
|
'grpc-status-details-bin' => status_details.to_proto
|
data/test/plugin/test_driver.rb
CHANGED
@@ -70,7 +70,7 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
70
70
|
assert_equal TEST_MESSAGE, record['message'], "Index #{index} failed."
|
71
71
|
|
72
72
|
# Get the first insertID.
|
73
|
-
expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index
|
73
|
+
expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index.zero?
|
74
74
|
insert_id = record[DEFAULT_INSERT_ID_KEY]
|
75
75
|
assert_equal expected_insert_id, insert_id, "Index #{index} failed."
|
76
76
|
expected_insert_id = expected_insert_id.next
|
@@ -86,7 +86,8 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
86
86
|
|
87
87
|
def test_insert_ids_not_added_if_present
|
88
88
|
log_entry_with_empty_insert_id = log_entry(0).merge(
|
89
|
-
DEFAULT_INSERT_ID_KEY => ''
|
89
|
+
DEFAULT_INSERT_ID_KEY => ''
|
90
|
+
)
|
90
91
|
{
|
91
92
|
log_entry(0).merge(DEFAULT_INSERT_ID_KEY => INSERT_ID) => true,
|
92
93
|
# Still generate insertId if it's an empty string
|
@@ -123,7 +124,8 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
123
124
|
|
124
125
|
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
125
126
|
Fluent::Test::FilterTestDriver.new(
|
126
|
-
Fluent::AddInsertIdsFilter
|
127
|
+
Fluent::AddInsertIdsFilter
|
128
|
+
).configure(conf, true)
|
127
129
|
end
|
128
130
|
|
129
131
|
def log_entry(index)
|
@@ -63,7 +63,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
63
63
|
plugin_name: 'source/syslog/tcp',
|
64
64
|
is_default_plugin: true,
|
65
65
|
has_default_config: true,
|
66
|
-
has_ruby_snippet: false
|
66
|
+
has_ruby_snippet: false
|
67
|
+
)
|
67
68
|
assert_metric_value.call(
|
68
69
|
:enabled_plugins,
|
69
70
|
1,
|
@@ -73,7 +74,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
73
74
|
plugin_name: 'source/tail/apache-access',
|
74
75
|
is_default_plugin: true,
|
75
76
|
has_default_config: true,
|
76
|
-
has_ruby_snippet: false
|
77
|
+
has_ruby_snippet: false
|
78
|
+
)
|
77
79
|
assert_metric_value.call(
|
78
80
|
:enabled_plugins,
|
79
81
|
1,
|
@@ -83,7 +85,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
83
85
|
plugin_name: 'filter/add_insert_ids',
|
84
86
|
is_default_plugin: true,
|
85
87
|
has_default_config: true,
|
86
|
-
has_ruby_snippet: false
|
88
|
+
has_ruby_snippet: false
|
89
|
+
)
|
87
90
|
|
88
91
|
# Default plugins, with custom config.
|
89
92
|
assert_metric_value.call(
|
@@ -95,7 +98,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
95
98
|
plugin_name: 'match/google_cloud',
|
96
99
|
is_default_plugin: true,
|
97
100
|
has_default_config: false,
|
98
|
-
has_ruby_snippet: false
|
101
|
+
has_ruby_snippet: false
|
102
|
+
)
|
99
103
|
|
100
104
|
# Custom plugins, some with embedded Ruby.
|
101
105
|
assert_metric_value.call(
|
@@ -107,7 +111,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
107
111
|
plugin_name: 'filter',
|
108
112
|
is_default_plugin: false,
|
109
113
|
has_default_config: false,
|
110
|
-
has_ruby_snippet: false
|
114
|
+
has_ruby_snippet: false
|
115
|
+
)
|
111
116
|
assert_metric_value.call(
|
112
117
|
:enabled_plugins,
|
113
118
|
1,
|
@@ -117,7 +122,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
117
122
|
plugin_name: 'filter/record_transformer',
|
118
123
|
is_default_plugin: false,
|
119
124
|
has_default_config: false,
|
120
|
-
has_ruby_snippet: true
|
125
|
+
has_ruby_snippet: true
|
126
|
+
)
|
121
127
|
assert_metric_value.call(
|
122
128
|
:enabled_plugins,
|
123
129
|
1,
|
@@ -127,7 +133,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
127
133
|
plugin_name: 'match/stdout',
|
128
134
|
is_default_plugin: false,
|
129
135
|
has_default_config: false,
|
130
|
-
has_ruby_snippet: true
|
136
|
+
has_ruby_snippet: true
|
137
|
+
)
|
131
138
|
|
132
139
|
# For out_google_cloud, 3 params are present.
|
133
140
|
assert_metric_value.call(
|
@@ -139,7 +146,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
139
146
|
plugin_name: 'google_cloud',
|
140
147
|
param: 'adjust_invalid_timestamps',
|
141
148
|
is_present: true,
|
142
|
-
has_default_config: true
|
149
|
+
has_default_config: true
|
150
|
+
)
|
143
151
|
assert_metric_value.call(
|
144
152
|
:plugin_config,
|
145
153
|
1,
|
@@ -149,7 +157,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
149
157
|
plugin_name: 'google_cloud',
|
150
158
|
param: 'autoformat_stackdriver_trace',
|
151
159
|
is_present: true,
|
152
|
-
has_default_config: false
|
160
|
+
has_default_config: false
|
161
|
+
)
|
153
162
|
assert_metric_value.call(
|
154
163
|
:plugin_config,
|
155
164
|
1,
|
@@ -159,10 +168,11 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
159
168
|
plugin_name: 'google_cloud',
|
160
169
|
param: 'coerce_to_utf8',
|
161
170
|
is_present: true,
|
162
|
-
has_default_config: false
|
171
|
+
has_default_config: false
|
172
|
+
)
|
163
173
|
# The remaining "google_cloud" params are not present.
|
164
174
|
# The are no params for "detect_exceptions".
|
165
|
-
%w
|
175
|
+
%w[
|
166
176
|
auth_method
|
167
177
|
detect_json
|
168
178
|
enable_monitoring
|
@@ -191,7 +201,7 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
191
201
|
vm_id
|
192
202
|
vm_name
|
193
203
|
zone
|
194
|
-
|
204
|
+
].each do |p|
|
195
205
|
assert_metric_value.call(
|
196
206
|
:plugin_config,
|
197
207
|
1,
|
@@ -201,7 +211,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
201
211
|
plugin_name: 'google_cloud',
|
202
212
|
param: p,
|
203
213
|
is_present: false,
|
204
|
-
has_default_config: false
|
214
|
+
has_default_config: false
|
215
|
+
)
|
205
216
|
end
|
206
217
|
|
207
218
|
# We also export values for the bools.
|
@@ -213,7 +224,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
213
224
|
d,
|
214
225
|
plugin_name: 'google_cloud',
|
215
226
|
param: 'adjust_invalid_timestamps',
|
216
|
-
value: true
|
227
|
+
value: true
|
228
|
+
)
|
217
229
|
assert_metric_value.call(
|
218
230
|
:config_bool_values,
|
219
231
|
1,
|
@@ -222,7 +234,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
222
234
|
d,
|
223
235
|
plugin_name: 'google_cloud',
|
224
236
|
param: 'autoformat_stackdriver_trace',
|
225
|
-
value: false
|
237
|
+
value: false
|
238
|
+
)
|
226
239
|
assert_metric_value.call(
|
227
240
|
:config_bool_values,
|
228
241
|
1,
|
@@ -231,12 +244,14 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
231
244
|
d,
|
232
245
|
plugin_name: 'google_cloud',
|
233
246
|
param: 'coerce_to_utf8',
|
234
|
-
value: true
|
247
|
+
value: true
|
248
|
+
)
|
235
249
|
end
|
236
250
|
end
|
237
251
|
|
238
252
|
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
239
253
|
Fluent::Test::FilterTestDriver.new(
|
240
|
-
Fluent::AnalyzeConfigFilter
|
254
|
+
Fluent::AnalyzeConfigFilter
|
255
|
+
).configure(conf, true)
|
241
256
|
end
|
242
257
|
end
|
@@ -44,7 +44,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
44
44
|
user_agent
|
45
45
|
end
|
46
46
|
|
47
|
-
def
|
47
|
+
def test_client_status400
|
48
48
|
setup_gce_metadata_stubs
|
49
49
|
# The API Client should not retry this and the plugin should consume
|
50
50
|
# the exception.
|
@@ -57,7 +57,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
57
57
|
end
|
58
58
|
|
59
59
|
# All credentials errors resolve to a 401.
|
60
|
-
def
|
60
|
+
def test_client_status401
|
61
61
|
setup_gce_metadata_stubs
|
62
62
|
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
63
63
|
.to_return(status: 401, body: 'Unauthorized')
|
@@ -65,8 +65,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
65
65
|
d.emit('message' => log_entry(0))
|
66
66
|
begin
|
67
67
|
d.run
|
68
|
-
rescue Google::Apis::AuthorizationError =>
|
69
|
-
assert_equal 'Unauthorized',
|
68
|
+
rescue Google::Apis::AuthorizationError => e
|
69
|
+
assert_equal 'Unauthorized', e.message
|
70
70
|
end
|
71
71
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
|
72
72
|
end
|
@@ -88,19 +88,23 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
88
88
|
assert_prometheus_metric_value(
|
89
89
|
:stackdriver_successful_requests_count, 1,
|
90
90
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
91
|
-
grpc: false, code: 200
|
91
|
+
grpc: false, code: 200
|
92
|
+
)
|
92
93
|
assert_prometheus_metric_value(
|
93
94
|
:stackdriver_ingested_entries_count, 1,
|
94
95
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
95
|
-
grpc: false, code: 200
|
96
|
+
grpc: false, code: 200
|
97
|
+
)
|
96
98
|
assert_prometheus_metric_value(
|
97
99
|
:stackdriver_dropped_entries_count, 2,
|
98
100
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
99
|
-
grpc: false, code: 3
|
101
|
+
grpc: false, code: 3
|
102
|
+
)
|
100
103
|
assert_prometheus_metric_value(
|
101
104
|
:stackdriver_dropped_entries_count, 1,
|
102
105
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
103
|
-
grpc: false, code: 7
|
106
|
+
grpc: false, code: 7
|
107
|
+
)
|
104
108
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
105
109
|
end
|
106
110
|
|
@@ -119,19 +123,23 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
119
123
|
assert_prometheus_metric_value(
|
120
124
|
:stackdriver_successful_requests_count, 0,
|
121
125
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
122
|
-
grpc: false, code: 200
|
126
|
+
grpc: false, code: 200
|
127
|
+
)
|
123
128
|
assert_prometheus_metric_value(
|
124
129
|
:stackdriver_failed_requests_count, 1,
|
125
130
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
126
|
-
grpc: false, code: 400
|
131
|
+
grpc: false, code: 400
|
132
|
+
)
|
127
133
|
assert_prometheus_metric_value(
|
128
134
|
:stackdriver_ingested_entries_count, 0,
|
129
135
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
130
|
-
grpc: false, code: 200
|
136
|
+
grpc: false, code: 200
|
137
|
+
)
|
131
138
|
assert_prometheus_metric_value(
|
132
139
|
:stackdriver_dropped_entries_count, 1,
|
133
140
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
134
|
-
grpc: false, code: 400
|
141
|
+
grpc: false, code: 400
|
142
|
+
)
|
135
143
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
136
144
|
end
|
137
145
|
|
@@ -146,8 +154,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
146
154
|
exception_count = 0
|
147
155
|
begin
|
148
156
|
d.run
|
149
|
-
rescue Google::Apis::ServerError =>
|
150
|
-
assert_equal 'Server error',
|
157
|
+
rescue Google::Apis::ServerError => e
|
158
|
+
assert_equal 'Server error', e.message
|
151
159
|
exception_count += 1
|
152
160
|
end
|
153
161
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
@@ -168,7 +176,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
168
176
|
setup_logging_stubs do
|
169
177
|
d = create_driver
|
170
178
|
# Array of pairs of [parsed_severity, expected_severity]
|
171
|
-
[%w
|
179
|
+
[%w[INFO INFO], %w[warn WARNING], %w[E ERROR], %w[BLAH DEFAULT],
|
172
180
|
['105', 100], ['', 'DEFAULT']].each do |sev|
|
173
181
|
d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
|
174
182
|
expected_severity.push(sev[1])
|
@@ -188,7 +196,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
188
196
|
test_obj = Fluent::GoogleCloudOutput.new
|
189
197
|
|
190
198
|
# known severities should translate to themselves, regardless of case
|
191
|
-
%w
|
199
|
+
%w[DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY].each \
|
192
200
|
do |severity|
|
193
201
|
assert_equal(severity, test_obj.parse_severity(severity))
|
194
202
|
assert_equal(severity, test_obj.parse_severity(severity.downcase))
|
@@ -298,6 +306,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
298
306
|
resp = Net::HTTP.get('127.0.0.1', '/statusz', 5678)
|
299
307
|
must_match = [
|
300
308
|
'<h1>Status for .*</h1>.*',
|
309
|
+
'\bStarted: .*<br>',
|
310
|
+
'\bUp \d+ hr \d{2} min \d{2} sec<br>',
|
301
311
|
|
302
312
|
'\badjust_invalid_timestamps\b.*\bfalse\b',
|
303
313
|
'\bautoformat_stackdriver_trace\b.*\bfalse\b',
|
@@ -411,20 +421,22 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
411
421
|
multi_tags = false)
|
412
422
|
driver = if multi_tags
|
413
423
|
Fluent::Test::MultiTagBufferedOutputTestDriver.new(
|
414
|
-
Fluent::GoogleCloudOutput
|
424
|
+
Fluent::GoogleCloudOutput
|
425
|
+
)
|
415
426
|
else
|
416
427
|
Fluent::Test::BufferedOutputTestDriver.new(
|
417
|
-
Fluent::GoogleCloudOutput, tag
|
428
|
+
Fluent::GoogleCloudOutput, tag
|
429
|
+
)
|
418
430
|
end
|
419
431
|
driver.configure(conf, true)
|
420
432
|
end
|
421
433
|
|
422
434
|
# Verify the number and the content of the log entries match the expectation.
|
423
435
|
# The caller can optionally provide a block which is called for each entry.
|
424
|
-
def verify_log_entries(
|
436
|
+
def verify_log_entries(expected_count, params, payload_type = 'textPayload',
|
425
437
|
check_exact_entry_labels = true, &block)
|
426
|
-
verify_json_log_entries(
|
427
|
-
&block)
|
438
|
+
verify_json_log_entries(expected_count, params, payload_type,
|
439
|
+
check_exact_entry_labels, &block)
|
428
440
|
end
|
429
441
|
|
430
442
|
# For an optional field with default values, Protobuf omits the field when it
|
@@ -88,7 +88,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
88
88
|
clear_metrics
|
89
89
|
setup_logging_stubs(
|
90
90
|
GRPC::PermissionDenied.new('User not authorized.',
|
91
|
-
PARTIAL_SUCCESS_GRPC_METADATA)
|
91
|
+
PARTIAL_SUCCESS_GRPC_METADATA)
|
92
|
+
) do
|
92
93
|
# The API Client should not retry this and the plugin should consume
|
93
94
|
# the exception.
|
94
95
|
d = create_driver(ENABLE_PROMETHEUS_CONFIG)
|
@@ -99,23 +100,28 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
99
100
|
assert_prometheus_metric_value(
|
100
101
|
:stackdriver_successful_requests_count, 1,
|
101
102
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
102
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
103
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
104
|
+
)
|
103
105
|
assert_prometheus_metric_value(
|
104
106
|
:stackdriver_failed_requests_count, 0,
|
105
107
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
106
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
108
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
109
|
+
)
|
107
110
|
assert_prometheus_metric_value(
|
108
111
|
:stackdriver_ingested_entries_count, 1,
|
109
112
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
110
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
113
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
114
|
+
)
|
111
115
|
assert_prometheus_metric_value(
|
112
116
|
:stackdriver_dropped_entries_count, 2,
|
113
117
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
114
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
118
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
119
|
+
)
|
115
120
|
assert_prometheus_metric_value(
|
116
121
|
:stackdriver_dropped_entries_count, 1,
|
117
122
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
118
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
123
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
124
|
+
)
|
119
125
|
end
|
120
126
|
end
|
121
127
|
|
@@ -124,7 +130,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
124
130
|
clear_metrics
|
125
131
|
setup_logging_stubs(
|
126
132
|
GRPC::InvalidArgument.new('internal client error',
|
127
|
-
PARSE_ERROR_GRPC_METADATA)
|
133
|
+
PARSE_ERROR_GRPC_METADATA)
|
134
|
+
) do
|
128
135
|
# The API Client should not retry this and the plugin should consume
|
129
136
|
# the exception.
|
130
137
|
d = create_driver(ENABLE_PROMETHEUS_CONFIG)
|
@@ -133,19 +140,23 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
133
140
|
assert_prometheus_metric_value(
|
134
141
|
:stackdriver_successful_requests_count, 0,
|
135
142
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
136
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
143
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
144
|
+
)
|
137
145
|
assert_prometheus_metric_value(
|
138
146
|
:stackdriver_failed_requests_count, 1,
|
139
147
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
140
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
148
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
149
|
+
)
|
141
150
|
assert_prometheus_metric_value(
|
142
151
|
:stackdriver_ingested_entries_count, 0,
|
143
152
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
144
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
153
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
154
|
+
)
|
145
155
|
assert_prometheus_metric_value(
|
146
156
|
:stackdriver_dropped_entries_count, 1,
|
147
157
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
148
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
158
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
159
|
+
)
|
149
160
|
end
|
150
161
|
end
|
151
162
|
|
@@ -165,8 +176,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
165
176
|
d.emit('message' => log_entry(0))
|
166
177
|
begin
|
167
178
|
d.run
|
168
|
-
rescue GRPC::BadStatus =>
|
169
|
-
assert_equal "#{code}:#{message}",
|
179
|
+
rescue GRPC::BadStatus => e
|
180
|
+
assert_equal "#{code}:#{message}", e.message
|
170
181
|
exception_count += 1
|
171
182
|
end
|
172
183
|
end
|
@@ -189,8 +200,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
189
200
|
setup_logging_stubs do
|
190
201
|
d = create_driver
|
191
202
|
# Array of pairs of [parsed_severity, expected_severity]
|
192
|
-
[%w
|
193
|
-
%w
|
203
|
+
[%w[INFO INFO], %w[warn WARNING], %w[E ERROR], %w[BLAH DEFAULT],
|
204
|
+
%w[105 DEBUG], ['', 'DEFAULT']].each do |sev|
|
194
205
|
d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
|
195
206
|
expected_severity.push(sev[1])
|
196
207
|
emit_index += 1
|
@@ -297,10 +308,12 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
297
308
|
conf += USE_GRPC_CONFIG
|
298
309
|
driver = if multi_tags
|
299
310
|
Fluent::Test::MultiTagBufferedOutputTestDriver.new(
|
300
|
-
GoogleCloudOutputWithGRPCMock.new(@grpc_stub)
|
311
|
+
GoogleCloudOutputWithGRPCMock.new(@grpc_stub)
|
312
|
+
)
|
301
313
|
else
|
302
314
|
Fluent::Test::BufferedOutputTestDriver.new(
|
303
|
-
GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag
|
315
|
+
GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag
|
316
|
+
)
|
304
317
|
end
|
305
318
|
driver.configure(conf, true)
|
306
319
|
end
|
@@ -360,7 +373,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
360
373
|
@failed_attempts << 1
|
361
374
|
begin
|
362
375
|
raise @error
|
363
|
-
rescue
|
376
|
+
rescue StandardError
|
364
377
|
# Google::Gax::GaxError will wrap the latest thrown exception as @cause.
|
365
378
|
raise Google::Gax::GaxError, 'This test message does not matter.'
|
366
379
|
end
|
@@ -370,7 +383,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
370
383
|
|
371
384
|
# Set up grpc stubs to mock the external calls.
|
372
385
|
def setup_logging_stubs(error = nil, code = nil, message = 'some message')
|
373
|
-
if error.nil? && (code.nil? || code
|
386
|
+
if error.nil? && (code.nil? || code.zero?)
|
374
387
|
@requests_sent = []
|
375
388
|
@grpc_stub = GRPCLoggingMockService.new(@requests_sent)
|
376
389
|
else
|
@@ -410,7 +423,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
410
423
|
|
411
424
|
# Verify the number and the content of the log entries match the expectation.
|
412
425
|
# The caller can optionally provide a block which is called for each entry.
|
413
|
-
def verify_log_entries(
|
426
|
+
def verify_log_entries(expected_count, params, payload_type = 'textPayload',
|
414
427
|
check_exact_entry_labels = true, &block)
|
415
428
|
@requests_sent.each do |request|
|
416
429
|
@logs_sent << {
|
@@ -420,8 +433,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
420
433
|
'logName' => request.log_name
|
421
434
|
}
|
422
435
|
end
|
423
|
-
verify_json_log_entries(
|
424
|
-
&block)
|
436
|
+
verify_json_log_entries(expected_count, params, payload_type,
|
437
|
+
check_exact_entry_labels, &block)
|
425
438
|
end
|
426
439
|
|
427
440
|
# Use the right single quotation mark as the sample non-utf8 character.
|
data/test/plugin/utils.rb
CHANGED
@@ -36,8 +36,7 @@ module Utils
|
|
36
36
|
end
|
37
37
|
|
38
38
|
def stub_metadata_request(metadata_path, response_body)
|
39
|
-
stub_request(:get,
|
40
|
-
metadata_path)
|
39
|
+
stub_request(:get, "http://169.254.169.254/computeMetadata/v1/#{metadata_path}")
|
41
40
|
.to_return(body: response_body, status: 200,
|
42
41
|
headers: { 'Content-Length' => response_body.length })
|
43
42
|
end
|
@@ -101,7 +100,8 @@ module Utils
|
|
101
100
|
def setup_managed_vm_metadata_stubs
|
102
101
|
stub_metadata_request(
|
103
102
|
'instance/attributes/',
|
104
|
-
"attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute"
|
103
|
+
"attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute"
|
104
|
+
)
|
105
105
|
stub_metadata_request('instance/attributes/gae_backend_name',
|
106
106
|
MANAGED_VM_BACKEND_NAME)
|
107
107
|
stub_metadata_request('instance/attributes/gae_backend_version',
|
@@ -112,13 +112,14 @@ module Utils
|
|
112
112
|
if should_respond
|
113
113
|
stub_metadata_request(
|
114
114
|
'instance/attributes/',
|
115
|
-
"attribute1\ncluster-location\ncluster-name\nlast_attribute"
|
115
|
+
"attribute1\ncluster-location\ncluster-name\nlast_attribute"
|
116
|
+
)
|
116
117
|
stub_metadata_request('instance/attributes/cluster-location',
|
117
118
|
K8S_LOCATION2)
|
118
119
|
stub_metadata_request('instance/attributes/cluster-name',
|
119
120
|
K8S_CLUSTER_NAME)
|
120
121
|
else
|
121
|
-
[
|
122
|
+
%w[cluster-location cluster-name].each do |metadata_name|
|
122
123
|
stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
|
123
124
|
.to_return(status: 404,
|
124
125
|
body: 'The requested URL /computeMetadata/v1/instance/' \
|
@@ -131,7 +132,8 @@ module Utils
|
|
131
132
|
def setup_dataproc_metadata_stubs
|
132
133
|
stub_metadata_request(
|
133
134
|
'instance/attributes/',
|
134
|
-
"attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name"
|
135
|
+
"attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name"
|
136
|
+
)
|
135
137
|
stub_metadata_request('instance/attributes/dataproc-cluster-name',
|
136
138
|
DATAPROC_CLUSTER_NAME)
|
137
139
|
stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
|