fluent-plugin-google-cloud 0.12.10 → 0.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +99 -71
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +17 -21
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -14
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +269 -213
- data/lib/fluent/plugin/statusz.rb +10 -10
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +199 -136
- data/test/plugin/constants.rb +21 -28
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +37 -25
- data/test/plugin/test_out_google_cloud_grpc.rb +45 -30
- data/test/plugin/utils.rb +9 -8
- metadata +53 -39
data/test/plugin/constants.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
# coding: utf-8
|
2
1
|
# Copyright 2017 Google Inc. All rights reserved.
|
3
2
|
#
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
@@ -13,20 +12,9 @@
|
|
13
12
|
# See the License for the specific language governing permissions and
|
14
13
|
# limitations under the License.
|
15
14
|
|
15
|
+
require 'google/rpc/error_details_pb'
|
16
|
+
|
16
17
|
# Add some helper methods to standard classes.
|
17
|
-
module Google
|
18
|
-
module Protobuf
|
19
|
-
Any.class_eval do
|
20
|
-
# TODO(igorpeshansky): Remove this once
|
21
|
-
# https://github.com/google/protobuf/pull/4719 gets released.
|
22
|
-
def self.pack(msg, type_url_prefix = 'type.googleapis.com/')
|
23
|
-
any = Google::Protobuf::Any.new
|
24
|
-
any.pack(msg, type_url_prefix)
|
25
|
-
any
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
18
|
String.class_eval do
|
31
19
|
def inspect_octal
|
32
20
|
specials = {
|
@@ -228,13 +216,11 @@ module Constants
|
|
228
216
|
detect_json true
|
229
217
|
).freeze
|
230
218
|
|
231
|
-
# rubocop:disable Metrics/LineLength
|
232
219
|
PRIVATE_KEY_CONFIG = %(
|
233
220
|
auth_method private_key
|
234
221
|
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
235
222
|
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
236
223
|
).freeze
|
237
|
-
# rubocop:enable Metrics/LineLength
|
238
224
|
|
239
225
|
REQUIRE_VALID_TAGS_CONFIG = %(
|
240
226
|
require_valid_tags true
|
@@ -433,7 +419,6 @@ module Constants
|
|
433
419
|
monitoring_type not_prometheus
|
434
420
|
).freeze
|
435
421
|
|
436
|
-
# rubocop:disable Metrics/LineLength
|
437
422
|
CONFIG_METRICS_RESOURCE_JSON = %(
|
438
423
|
enable_monitoring true
|
439
424
|
monitoring_type opencensus
|
@@ -481,7 +466,6 @@ module Constants
|
|
481
466
|
monitoring_type opencensus
|
482
467
|
metrics_resource {"type":"custom_resource","labels.label1":"123","random":"x"}
|
483
468
|
).freeze
|
484
|
-
# rubocop:enable Metrics/LineLength
|
485
469
|
|
486
470
|
# For statusz.
|
487
471
|
CONFIG_STATUSZ = %(
|
@@ -1023,34 +1007,41 @@ module Constants
|
|
1023
1007
|
}.freeze
|
1024
1008
|
|
1025
1009
|
PARTIAL_SUCCESS_GRPC_METADATA = begin
|
1026
|
-
partial_errors = Google::Logging::V2::WriteLogEntriesPartialErrors.new(
|
1010
|
+
partial_errors = Google::Cloud::Logging::V2::WriteLogEntriesPartialErrors.new(
|
1027
1011
|
log_entry_errors: {
|
1028
1012
|
0 => Google::Rpc::Status.new(
|
1029
1013
|
code: GRPC::Core::StatusCodes::PERMISSION_DENIED,
|
1030
1014
|
message: 'User not authorized.',
|
1031
|
-
details: []
|
1015
|
+
details: []
|
1016
|
+
),
|
1032
1017
|
1 => Google::Rpc::Status.new(
|
1033
1018
|
code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
|
1034
1019
|
message: 'Log name contains illegal character :',
|
1035
|
-
details: []
|
1020
|
+
details: []
|
1021
|
+
),
|
1036
1022
|
3 => Google::Rpc::Status.new(
|
1037
1023
|
code: GRPC::Core::StatusCodes::INVALID_ARGUMENT,
|
1038
1024
|
message: 'Log name contains illegal character :',
|
1039
|
-
details: []
|
1040
|
-
|
1025
|
+
details: []
|
1026
|
+
)
|
1027
|
+
}
|
1028
|
+
)
|
1041
1029
|
status = Google::Rpc::Status.new(
|
1042
1030
|
message: 'User not authorized.',
|
1043
|
-
details: [Google::Protobuf::Any.pack(partial_errors)]
|
1031
|
+
details: [Google::Protobuf::Any.pack(partial_errors)]
|
1032
|
+
)
|
1044
1033
|
debug_info = Google::Rpc::DebugInfo.new(
|
1045
1034
|
detail: '[ORIGINAL ERROR] generic::permission_denied: User not' \
|
1046
1035
|
' authorized. [google.rpc.error_details_ext] { message:' \
|
1047
1036
|
" #{status.message.inspect} details { type_url:" \
|
1048
1037
|
" #{status.details[0].type_url.inspect} value:" \
|
1049
|
-
" #{status.details[0].value.inspect_octal} } }"
|
1038
|
+
" #{status.details[0].value.inspect_octal} } }"
|
1039
|
+
)
|
1050
1040
|
status_details = Google::Rpc::Status.new(
|
1051
1041
|
code: 7, message: 'User not authorized.',
|
1052
1042
|
details: [Google::Protobuf::Any.pack(partial_errors),
|
1053
|
-
Google::Protobuf::Any.pack(debug_info)]
|
1043
|
+
Google::Protobuf::Any.pack(debug_info)]
|
1044
|
+
)
|
1054
1045
|
{
|
1055
1046
|
'google.logging.v2.writelogentriespartialerrors-bin' =>
|
1056
1047
|
partial_errors.to_proto,
|
@@ -1078,10 +1069,12 @@ module Constants
|
|
1078
1069
|
PARSE_ERROR_GRPC_METADATA = begin
|
1079
1070
|
debug_info = Google::Rpc::DebugInfo.new(
|
1080
1071
|
detail: '[ORIGINAL ERROR] RPC::CLIENT_ERROR: server could not parse' \
|
1081
|
-
" request sent by client; initialization error is: ''"
|
1072
|
+
" request sent by client; initialization error is: ''"
|
1073
|
+
)
|
1082
1074
|
status_details = Google::Rpc::Status.new(
|
1083
1075
|
code: 3, message: 'internal client error',
|
1084
|
-
details: [Google::Protobuf::Any.pack(debug_info)]
|
1076
|
+
details: [Google::Protobuf::Any.pack(debug_info)]
|
1077
|
+
)
|
1085
1078
|
{
|
1086
1079
|
'google.rpc.debuginfo-bin' => debug_info.to_proto,
|
1087
1080
|
'grpc-status-details-bin' => status_details.to_proto
|
data/test/plugin/test_driver.rb
CHANGED
@@ -70,7 +70,7 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
70
70
|
assert_equal TEST_MESSAGE, record['message'], "Index #{index} failed."
|
71
71
|
|
72
72
|
# Get the first insertID.
|
73
|
-
expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index
|
73
|
+
expected_insert_id = record[DEFAULT_INSERT_ID_KEY] if index.zero?
|
74
74
|
insert_id = record[DEFAULT_INSERT_ID_KEY]
|
75
75
|
assert_equal expected_insert_id, insert_id, "Index #{index} failed."
|
76
76
|
expected_insert_id = expected_insert_id.next
|
@@ -86,7 +86,8 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
86
86
|
|
87
87
|
def test_insert_ids_not_added_if_present
|
88
88
|
log_entry_with_empty_insert_id = log_entry(0).merge(
|
89
|
-
DEFAULT_INSERT_ID_KEY => ''
|
89
|
+
DEFAULT_INSERT_ID_KEY => ''
|
90
|
+
)
|
90
91
|
{
|
91
92
|
log_entry(0).merge(DEFAULT_INSERT_ID_KEY => INSERT_ID) => true,
|
92
93
|
# Still generate insertId if it's an empty string
|
@@ -123,7 +124,8 @@ class FilterAddInsertIdsTest < Test::Unit::TestCase
|
|
123
124
|
|
124
125
|
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
125
126
|
Fluent::Test::FilterTestDriver.new(
|
126
|
-
Fluent::AddInsertIdsFilter
|
127
|
+
Fluent::AddInsertIdsFilter
|
128
|
+
).configure(conf, true)
|
127
129
|
end
|
128
130
|
|
129
131
|
def log_entry(index)
|
@@ -63,7 +63,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
63
63
|
plugin_name: 'source/syslog/tcp',
|
64
64
|
is_default_plugin: true,
|
65
65
|
has_default_config: true,
|
66
|
-
has_ruby_snippet: false
|
66
|
+
has_ruby_snippet: false
|
67
|
+
)
|
67
68
|
assert_metric_value.call(
|
68
69
|
:enabled_plugins,
|
69
70
|
1,
|
@@ -73,7 +74,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
73
74
|
plugin_name: 'source/tail/apache-access',
|
74
75
|
is_default_plugin: true,
|
75
76
|
has_default_config: true,
|
76
|
-
has_ruby_snippet: false
|
77
|
+
has_ruby_snippet: false
|
78
|
+
)
|
77
79
|
assert_metric_value.call(
|
78
80
|
:enabled_plugins,
|
79
81
|
1,
|
@@ -83,7 +85,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
83
85
|
plugin_name: 'filter/add_insert_ids',
|
84
86
|
is_default_plugin: true,
|
85
87
|
has_default_config: true,
|
86
|
-
has_ruby_snippet: false
|
88
|
+
has_ruby_snippet: false
|
89
|
+
)
|
87
90
|
|
88
91
|
# Default plugins, with custom config.
|
89
92
|
assert_metric_value.call(
|
@@ -95,7 +98,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
95
98
|
plugin_name: 'match/google_cloud',
|
96
99
|
is_default_plugin: true,
|
97
100
|
has_default_config: false,
|
98
|
-
has_ruby_snippet: false
|
101
|
+
has_ruby_snippet: false
|
102
|
+
)
|
99
103
|
|
100
104
|
# Custom plugins, some with embedded Ruby.
|
101
105
|
assert_metric_value.call(
|
@@ -107,7 +111,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
107
111
|
plugin_name: 'filter',
|
108
112
|
is_default_plugin: false,
|
109
113
|
has_default_config: false,
|
110
|
-
has_ruby_snippet: false
|
114
|
+
has_ruby_snippet: false
|
115
|
+
)
|
111
116
|
assert_metric_value.call(
|
112
117
|
:enabled_plugins,
|
113
118
|
1,
|
@@ -117,7 +122,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
117
122
|
plugin_name: 'filter/record_transformer',
|
118
123
|
is_default_plugin: false,
|
119
124
|
has_default_config: false,
|
120
|
-
has_ruby_snippet: true
|
125
|
+
has_ruby_snippet: true
|
126
|
+
)
|
121
127
|
assert_metric_value.call(
|
122
128
|
:enabled_plugins,
|
123
129
|
1,
|
@@ -127,7 +133,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
127
133
|
plugin_name: 'match/stdout',
|
128
134
|
is_default_plugin: false,
|
129
135
|
has_default_config: false,
|
130
|
-
has_ruby_snippet: true
|
136
|
+
has_ruby_snippet: true
|
137
|
+
)
|
131
138
|
|
132
139
|
# For out_google_cloud, 3 params are present.
|
133
140
|
assert_metric_value.call(
|
@@ -139,7 +146,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
139
146
|
plugin_name: 'google_cloud',
|
140
147
|
param: 'adjust_invalid_timestamps',
|
141
148
|
is_present: true,
|
142
|
-
has_default_config: true
|
149
|
+
has_default_config: true
|
150
|
+
)
|
143
151
|
assert_metric_value.call(
|
144
152
|
:plugin_config,
|
145
153
|
1,
|
@@ -149,7 +157,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
149
157
|
plugin_name: 'google_cloud',
|
150
158
|
param: 'autoformat_stackdriver_trace',
|
151
159
|
is_present: true,
|
152
|
-
has_default_config: false
|
160
|
+
has_default_config: false
|
161
|
+
)
|
153
162
|
assert_metric_value.call(
|
154
163
|
:plugin_config,
|
155
164
|
1,
|
@@ -159,10 +168,11 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
159
168
|
plugin_name: 'google_cloud',
|
160
169
|
param: 'coerce_to_utf8',
|
161
170
|
is_present: true,
|
162
|
-
has_default_config: false
|
171
|
+
has_default_config: false
|
172
|
+
)
|
163
173
|
# The remaining "google_cloud" params are not present.
|
164
174
|
# The are no params for "detect_exceptions".
|
165
|
-
%w
|
175
|
+
%w[
|
166
176
|
auth_method
|
167
177
|
detect_json
|
168
178
|
enable_monitoring
|
@@ -191,7 +201,7 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
191
201
|
vm_id
|
192
202
|
vm_name
|
193
203
|
zone
|
194
|
-
|
204
|
+
].each do |p|
|
195
205
|
assert_metric_value.call(
|
196
206
|
:plugin_config,
|
197
207
|
1,
|
@@ -201,7 +211,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
201
211
|
plugin_name: 'google_cloud',
|
202
212
|
param: p,
|
203
213
|
is_present: false,
|
204
|
-
has_default_config: false
|
214
|
+
has_default_config: false
|
215
|
+
)
|
205
216
|
end
|
206
217
|
|
207
218
|
# We also export values for the bools.
|
@@ -213,7 +224,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
213
224
|
d,
|
214
225
|
plugin_name: 'google_cloud',
|
215
226
|
param: 'adjust_invalid_timestamps',
|
216
|
-
value: true
|
227
|
+
value: true
|
228
|
+
)
|
217
229
|
assert_metric_value.call(
|
218
230
|
:config_bool_values,
|
219
231
|
1,
|
@@ -222,7 +234,8 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
222
234
|
d,
|
223
235
|
plugin_name: 'google_cloud',
|
224
236
|
param: 'autoformat_stackdriver_trace',
|
225
|
-
value: false
|
237
|
+
value: false
|
238
|
+
)
|
226
239
|
assert_metric_value.call(
|
227
240
|
:config_bool_values,
|
228
241
|
1,
|
@@ -231,12 +244,14 @@ class FilterAnalyzeConfigTest < Test::Unit::TestCase
|
|
231
244
|
d,
|
232
245
|
plugin_name: 'google_cloud',
|
233
246
|
param: 'coerce_to_utf8',
|
234
|
-
value: true
|
247
|
+
value: true
|
248
|
+
)
|
235
249
|
end
|
236
250
|
end
|
237
251
|
|
238
252
|
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
239
253
|
Fluent::Test::FilterTestDriver.new(
|
240
|
-
Fluent::AnalyzeConfigFilter
|
254
|
+
Fluent::AnalyzeConfigFilter
|
255
|
+
).configure(conf, true)
|
241
256
|
end
|
242
257
|
end
|
@@ -44,7 +44,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
44
44
|
user_agent
|
45
45
|
end
|
46
46
|
|
47
|
-
def
|
47
|
+
def test_client_status400
|
48
48
|
setup_gce_metadata_stubs
|
49
49
|
# The API Client should not retry this and the plugin should consume
|
50
50
|
# the exception.
|
@@ -57,7 +57,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
57
57
|
end
|
58
58
|
|
59
59
|
# All credentials errors resolve to a 401.
|
60
|
-
def
|
60
|
+
def test_client_status401
|
61
61
|
setup_gce_metadata_stubs
|
62
62
|
stub_request(:post, WRITE_LOG_ENTRIES_URI)
|
63
63
|
.to_return(status: 401, body: 'Unauthorized')
|
@@ -65,8 +65,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
65
65
|
d.emit('message' => log_entry(0))
|
66
66
|
begin
|
67
67
|
d.run
|
68
|
-
rescue Google::Apis::AuthorizationError =>
|
69
|
-
assert_equal 'Unauthorized',
|
68
|
+
rescue Google::Apis::AuthorizationError => e
|
69
|
+
assert_equal 'Unauthorized', e.message
|
70
70
|
end
|
71
71
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 2)
|
72
72
|
end
|
@@ -88,19 +88,23 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
88
88
|
assert_prometheus_metric_value(
|
89
89
|
:stackdriver_successful_requests_count, 1,
|
90
90
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
91
|
-
grpc: false, code: 200
|
91
|
+
grpc: false, code: 200
|
92
|
+
)
|
92
93
|
assert_prometheus_metric_value(
|
93
94
|
:stackdriver_ingested_entries_count, 1,
|
94
95
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
95
|
-
grpc: false, code: 200
|
96
|
+
grpc: false, code: 200
|
97
|
+
)
|
96
98
|
assert_prometheus_metric_value(
|
97
99
|
:stackdriver_dropped_entries_count, 2,
|
98
100
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
99
|
-
grpc: false, code: 3
|
101
|
+
grpc: false, code: 3
|
102
|
+
)
|
100
103
|
assert_prometheus_metric_value(
|
101
104
|
:stackdriver_dropped_entries_count, 1,
|
102
105
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
103
|
-
grpc: false, code: 7
|
106
|
+
grpc: false, code: 7
|
107
|
+
)
|
104
108
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
105
109
|
end
|
106
110
|
|
@@ -119,19 +123,23 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
119
123
|
assert_prometheus_metric_value(
|
120
124
|
:stackdriver_successful_requests_count, 0,
|
121
125
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
122
|
-
grpc: false, code: 200
|
126
|
+
grpc: false, code: 200
|
127
|
+
)
|
123
128
|
assert_prometheus_metric_value(
|
124
129
|
:stackdriver_failed_requests_count, 1,
|
125
130
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
126
|
-
grpc: false, code: 400
|
131
|
+
grpc: false, code: 400
|
132
|
+
)
|
127
133
|
assert_prometheus_metric_value(
|
128
134
|
:stackdriver_ingested_entries_count, 0,
|
129
135
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
130
|
-
grpc: false, code: 200
|
136
|
+
grpc: false, code: 200
|
137
|
+
)
|
131
138
|
assert_prometheus_metric_value(
|
132
139
|
:stackdriver_dropped_entries_count, 1,
|
133
140
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
134
|
-
grpc: false, code: 400
|
141
|
+
grpc: false, code: 400
|
142
|
+
)
|
135
143
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
136
144
|
end
|
137
145
|
|
@@ -146,8 +154,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
146
154
|
exception_count = 0
|
147
155
|
begin
|
148
156
|
d.run
|
149
|
-
rescue Google::Apis::ServerError =>
|
150
|
-
assert_equal 'Server error',
|
157
|
+
rescue Google::Apis::ServerError => e
|
158
|
+
assert_equal 'Server error', e.message
|
151
159
|
exception_count += 1
|
152
160
|
end
|
153
161
|
assert_requested(:post, WRITE_LOG_ENTRIES_URI, times: 1)
|
@@ -168,7 +176,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
168
176
|
setup_logging_stubs do
|
169
177
|
d = create_driver
|
170
178
|
# Array of pairs of [parsed_severity, expected_severity]
|
171
|
-
[%w
|
179
|
+
[%w[INFO INFO], %w[warn WARNING], %w[E ERROR], %w[BLAH DEFAULT],
|
172
180
|
['105', 100], ['', 'DEFAULT']].each do |sev|
|
173
181
|
d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
|
174
182
|
expected_severity.push(sev[1])
|
@@ -188,12 +196,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
188
196
|
test_obj = Fluent::GoogleCloudOutput.new
|
189
197
|
|
190
198
|
# known severities should translate to themselves, regardless of case
|
191
|
-
%w
|
199
|
+
%w[DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY].each \
|
192
200
|
do |severity|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
201
|
+
assert_equal(severity, test_obj.parse_severity(severity))
|
202
|
+
assert_equal(severity, test_obj.parse_severity(severity.downcase))
|
203
|
+
assert_equal(severity, test_obj.parse_severity(severity.capitalize))
|
204
|
+
end
|
197
205
|
|
198
206
|
# numeric levels
|
199
207
|
assert_equal(0, test_obj.parse_severity('0'))
|
@@ -298,6 +306,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
298
306
|
resp = Net::HTTP.get('127.0.0.1', '/statusz', 5678)
|
299
307
|
must_match = [
|
300
308
|
'<h1>Status for .*</h1>.*',
|
309
|
+
'\bStarted: .*<br>',
|
310
|
+
'\bUp \d+ hr \d{2} min \d{2} sec<br>',
|
301
311
|
|
302
312
|
'\badjust_invalid_timestamps\b.*\bfalse\b',
|
303
313
|
'\bautoformat_stackdriver_trace\b.*\bfalse\b',
|
@@ -411,20 +421,22 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
411
421
|
multi_tags = false)
|
412
422
|
driver = if multi_tags
|
413
423
|
Fluent::Test::MultiTagBufferedOutputTestDriver.new(
|
414
|
-
Fluent::GoogleCloudOutput
|
424
|
+
Fluent::GoogleCloudOutput
|
425
|
+
)
|
415
426
|
else
|
416
427
|
Fluent::Test::BufferedOutputTestDriver.new(
|
417
|
-
Fluent::GoogleCloudOutput, tag
|
428
|
+
Fluent::GoogleCloudOutput, tag
|
429
|
+
)
|
418
430
|
end
|
419
431
|
driver.configure(conf, true)
|
420
432
|
end
|
421
433
|
|
422
434
|
# Verify the number and the content of the log entries match the expectation.
|
423
435
|
# The caller can optionally provide a block which is called for each entry.
|
424
|
-
def verify_log_entries(
|
436
|
+
def verify_log_entries(expected_count, params, payload_type = 'textPayload',
|
425
437
|
check_exact_entry_labels = true, &block)
|
426
|
-
verify_json_log_entries(
|
427
|
-
&block)
|
438
|
+
verify_json_log_entries(expected_count, params, payload_type,
|
439
|
+
check_exact_entry_labels, &block)
|
428
440
|
end
|
429
441
|
|
430
442
|
# For an optional field with default values, Protobuf omits the field when it
|
@@ -34,6 +34,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
34
34
|
# Record user agent when creating a GRPC::Core::Channel.
|
35
35
|
GRPC::Core::Channel.class_eval do
|
36
36
|
old_initialize = instance_method(:initialize)
|
37
|
+
# Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055).
|
38
|
+
alias_method :initialize, :initialize
|
37
39
|
define_method(:initialize) do |url, args, creds|
|
38
40
|
user_agent = args['grpc.primary_user_agent']
|
39
41
|
old_initialize.bind(self).call(url, args, creds)
|
@@ -88,7 +90,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
88
90
|
clear_metrics
|
89
91
|
setup_logging_stubs(
|
90
92
|
GRPC::PermissionDenied.new('User not authorized.',
|
91
|
-
PARTIAL_SUCCESS_GRPC_METADATA)
|
93
|
+
PARTIAL_SUCCESS_GRPC_METADATA)
|
94
|
+
) do
|
92
95
|
# The API Client should not retry this and the plugin should consume
|
93
96
|
# the exception.
|
94
97
|
d = create_driver(ENABLE_PROMETHEUS_CONFIG)
|
@@ -99,23 +102,28 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
99
102
|
assert_prometheus_metric_value(
|
100
103
|
:stackdriver_successful_requests_count, 1,
|
101
104
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
102
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
105
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
106
|
+
)
|
103
107
|
assert_prometheus_metric_value(
|
104
108
|
:stackdriver_failed_requests_count, 0,
|
105
109
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
106
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
110
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
111
|
+
)
|
107
112
|
assert_prometheus_metric_value(
|
108
113
|
:stackdriver_ingested_entries_count, 1,
|
109
114
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
110
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
115
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
116
|
+
)
|
111
117
|
assert_prometheus_metric_value(
|
112
118
|
:stackdriver_dropped_entries_count, 2,
|
113
119
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
114
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
120
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
121
|
+
)
|
115
122
|
assert_prometheus_metric_value(
|
116
123
|
:stackdriver_dropped_entries_count, 1,
|
117
124
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
118
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
125
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::PERMISSION_DENIED
|
126
|
+
)
|
119
127
|
end
|
120
128
|
end
|
121
129
|
|
@@ -124,7 +132,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
124
132
|
clear_metrics
|
125
133
|
setup_logging_stubs(
|
126
134
|
GRPC::InvalidArgument.new('internal client error',
|
127
|
-
PARSE_ERROR_GRPC_METADATA)
|
135
|
+
PARSE_ERROR_GRPC_METADATA)
|
136
|
+
) do
|
128
137
|
# The API Client should not retry this and the plugin should consume
|
129
138
|
# the exception.
|
130
139
|
d = create_driver(ENABLE_PROMETHEUS_CONFIG)
|
@@ -133,19 +142,23 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
133
142
|
assert_prometheus_metric_value(
|
134
143
|
:stackdriver_successful_requests_count, 0,
|
135
144
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
136
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
145
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
146
|
+
)
|
137
147
|
assert_prometheus_metric_value(
|
138
148
|
:stackdriver_failed_requests_count, 1,
|
139
149
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
140
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
150
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
151
|
+
)
|
141
152
|
assert_prometheus_metric_value(
|
142
153
|
:stackdriver_ingested_entries_count, 0,
|
143
154
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
144
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
155
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::OK
|
156
|
+
)
|
145
157
|
assert_prometheus_metric_value(
|
146
158
|
:stackdriver_dropped_entries_count, 1,
|
147
159
|
'agent.googleapis.com/agent', OpenCensus::Stats::Aggregation::Sum, d,
|
148
|
-
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
160
|
+
grpc: use_grpc, code: GRPC::Core::StatusCodes::INVALID_ARGUMENT
|
161
|
+
)
|
149
162
|
end
|
150
163
|
end
|
151
164
|
|
@@ -165,8 +178,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
165
178
|
d.emit('message' => log_entry(0))
|
166
179
|
begin
|
167
180
|
d.run
|
168
|
-
rescue GRPC::BadStatus =>
|
169
|
-
assert_equal "#{code}:#{message}",
|
181
|
+
rescue GRPC::BadStatus => e
|
182
|
+
assert_equal "#{code}:#{message}", e.message
|
170
183
|
exception_count += 1
|
171
184
|
end
|
172
185
|
end
|
@@ -189,8 +202,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
189
202
|
setup_logging_stubs do
|
190
203
|
d = create_driver
|
191
204
|
# Array of pairs of [parsed_severity, expected_severity]
|
192
|
-
[%w
|
193
|
-
%w
|
205
|
+
[%w[INFO INFO], %w[warn WARNING], %w[E ERROR], %w[BLAH DEFAULT],
|
206
|
+
%w[105 DEBUG], ['', 'DEFAULT']].each do |sev|
|
194
207
|
d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
|
195
208
|
expected_severity.push(sev[1])
|
196
209
|
emit_index += 1
|
@@ -259,8 +272,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
259
272
|
|
260
273
|
private
|
261
274
|
|
262
|
-
WriteLogEntriesRequest = Google::Logging::V2::WriteLogEntriesRequest
|
263
|
-
WriteLogEntriesResponse = Google::Logging::V2::WriteLogEntriesResponse
|
275
|
+
WriteLogEntriesRequest = Google::Cloud::Logging::V2::WriteLogEntriesRequest
|
276
|
+
WriteLogEntriesResponse = Google::Cloud::Logging::V2::WriteLogEntriesResponse
|
264
277
|
|
265
278
|
USE_GRPC_CONFIG = %(
|
266
279
|
use_grpc true
|
@@ -297,10 +310,12 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
297
310
|
conf += USE_GRPC_CONFIG
|
298
311
|
driver = if multi_tags
|
299
312
|
Fluent::Test::MultiTagBufferedOutputTestDriver.new(
|
300
|
-
GoogleCloudOutputWithGRPCMock.new(@grpc_stub)
|
313
|
+
GoogleCloudOutputWithGRPCMock.new(@grpc_stub)
|
314
|
+
)
|
301
315
|
else
|
302
316
|
Fluent::Test::BufferedOutputTestDriver.new(
|
303
|
-
GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag
|
317
|
+
GoogleCloudOutputWithGRPCMock.new(@grpc_stub), tag
|
318
|
+
)
|
304
319
|
end
|
305
320
|
driver.configure(conf, true)
|
306
321
|
end
|
@@ -319,13 +334,13 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
319
334
|
|
320
335
|
# GRPC logging mock that successfully logs the records.
|
321
336
|
class GRPCLoggingMockService <
|
322
|
-
Google::Cloud::Logging::V2::
|
337
|
+
Google::Cloud::Logging::V2::LoggingService::Client
|
323
338
|
def initialize(requests_received)
|
324
339
|
super()
|
325
340
|
@requests_received = requests_received
|
326
341
|
end
|
327
342
|
|
328
|
-
def write_log_entries(entries
|
343
|
+
def write_log_entries(entries:,
|
329
344
|
log_name: nil,
|
330
345
|
resource: nil,
|
331
346
|
labels: nil,
|
@@ -344,7 +359,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
344
359
|
|
345
360
|
# GRPC logging mock that fails and returns server side or client side errors.
|
346
361
|
class GRPCLoggingMockFailingService <
|
347
|
-
Google::Cloud::Logging::V2::
|
362
|
+
Google::Cloud::Logging::V2::LoggingService::Client
|
348
363
|
def initialize(error, failed_attempts)
|
349
364
|
super()
|
350
365
|
@error = error
|
@@ -352,7 +367,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
352
367
|
end
|
353
368
|
|
354
369
|
# rubocop:disable Lint/UnusedMethodArgument
|
355
|
-
def write_log_entries(entries
|
370
|
+
def write_log_entries(entries:,
|
356
371
|
log_name: nil,
|
357
372
|
resource: nil,
|
358
373
|
labels: nil,
|
@@ -360,9 +375,9 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
360
375
|
@failed_attempts << 1
|
361
376
|
begin
|
362
377
|
raise @error
|
363
|
-
rescue
|
364
|
-
# Google::
|
365
|
-
raise Google::
|
378
|
+
rescue StandardError
|
379
|
+
# Google::Cloud::Error will wrap the latest thrown exception as @cause.
|
380
|
+
raise Google::Cloud::Error, 'This test message does not matter.'
|
366
381
|
end
|
367
382
|
end
|
368
383
|
# rubocop:enable Lint/UnusedMethodArgument
|
@@ -370,7 +385,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
370
385
|
|
371
386
|
# Set up grpc stubs to mock the external calls.
|
372
387
|
def setup_logging_stubs(error = nil, code = nil, message = 'some message')
|
373
|
-
if error.nil? && (code.nil? || code
|
388
|
+
if error.nil? && (code.nil? || code.zero?)
|
374
389
|
@requests_sent = []
|
375
390
|
@grpc_stub = GRPCLoggingMockService.new(@requests_sent)
|
376
391
|
else
|
@@ -410,7 +425,7 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
410
425
|
|
411
426
|
# Verify the number and the content of the log entries match the expectation.
|
412
427
|
# The caller can optionally provide a block which is called for each entry.
|
413
|
-
def verify_log_entries(
|
428
|
+
def verify_log_entries(expected_count, params, payload_type = 'textPayload',
|
414
429
|
check_exact_entry_labels = true, &block)
|
415
430
|
@requests_sent.each do |request|
|
416
431
|
@logs_sent << {
|
@@ -420,8 +435,8 @@ class GoogleCloudOutputGRPCTest < Test::Unit::TestCase
|
|
420
435
|
'logName' => request.log_name
|
421
436
|
}
|
422
437
|
end
|
423
|
-
verify_json_log_entries(
|
424
|
-
&block)
|
438
|
+
verify_json_log_entries(expected_count, params, payload_type,
|
439
|
+
check_exact_entry_labels, &block)
|
425
440
|
end
|
426
441
|
|
427
442
|
# Use the right single quotation mark as the sample non-utf8 character.
|