fluent-plugin-google-cloud 0.12.10 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +99 -71
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +17 -21
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -14
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +269 -213
- data/lib/fluent/plugin/statusz.rb +10 -10
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +199 -136
- data/test/plugin/constants.rb +21 -28
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +37 -25
- data/test/plugin/test_out_google_cloud_grpc.rb +45 -30
- data/test/plugin/utils.rb +9 -8
- metadata +53 -39
data/test/plugin/base_test.rb
CHANGED
@@ -29,8 +29,10 @@ require_relative 'utils'
|
|
29
29
|
|
30
30
|
module Monitoring
|
31
31
|
# Prevent OpenCensus from writing to the network.
|
32
|
-
|
33
|
-
|
32
|
+
OpenCensusMonitoringRegistry.class_eval do
|
33
|
+
# Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055).
|
34
|
+
alias_method :export, :export
|
35
|
+
define_method(:export) do
|
34
36
|
nil
|
35
37
|
end
|
36
38
|
end
|
@@ -55,6 +57,7 @@ module BaseTest
|
|
55
57
|
registry.unregister(:stackdriver_retried_entries_count)
|
56
58
|
|
57
59
|
setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token')
|
60
|
+
setup_auth_stubs('https://oauth2.googleapis.com/token')
|
58
61
|
@logs_sent = []
|
59
62
|
end
|
60
63
|
|
@@ -71,8 +74,8 @@ module BaseTest
|
|
71
74
|
exception_count = 0
|
72
75
|
begin
|
73
76
|
create_driver(PRIVATE_KEY_CONFIG)
|
74
|
-
rescue Fluent::ConfigError =>
|
75
|
-
assert
|
77
|
+
rescue Fluent::ConfigError => e
|
78
|
+
assert e.message.include? 'Please remove configuration parameters'
|
76
79
|
exception_count += 1
|
77
80
|
end
|
78
81
|
assert_equal 1, exception_count
|
@@ -100,23 +103,22 @@ module BaseTest
|
|
100
103
|
def test_configure_metadata_missing_parts_on_other_platforms
|
101
104
|
setup_no_metadata_service_stubs
|
102
105
|
Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
|
103
|
-
[
|
104
|
-
|
105
|
-
|
106
|
-
|
106
|
+
[
|
107
|
+
[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
|
108
|
+
[CONFIG_MISSING_METADATA_ZONE, [], true],
|
109
|
+
[CONFIG_MISSING_METADATA_VM_ID, [], true],
|
110
|
+
[CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
|
107
111
|
].each_with_index do |(config, missing_parts, is_valid_config), index|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
assert error.message.include?(part), "Index #{index} failed."
|
119
|
-
end
|
112
|
+
create_driver(config)
|
113
|
+
assert_true is_valid_config, "Invalid config at index #{index} should "\
|
114
|
+
'have raised an error.'
|
115
|
+
rescue Fluent::ConfigError => e
|
116
|
+
assert_false is_valid_config, "Valid config at index #{index} should "\
|
117
|
+
"not have raised an error #{e}."
|
118
|
+
assert e.message.include?('Unable to obtain metadata parameters:'),
|
119
|
+
"Index #{index} failed."
|
120
|
+
missing_parts.each do |part|
|
121
|
+
assert e.message.include?(part), "Index #{index} failed."
|
120
122
|
end
|
121
123
|
end
|
122
124
|
end
|
@@ -128,38 +130,52 @@ module BaseTest
|
|
128
130
|
setup_gce_metadata_stubs
|
129
131
|
create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
|
130
132
|
assert_nil(Prometheus::Client.registry.get(
|
131
|
-
:stackdriver_successful_requests_count
|
133
|
+
:stackdriver_successful_requests_count
|
134
|
+
))
|
132
135
|
assert_nil(Prometheus::Client.registry.get(
|
133
|
-
:stackdriver_failed_requests_count
|
136
|
+
:stackdriver_failed_requests_count
|
137
|
+
))
|
134
138
|
assert_nil(Prometheus::Client.registry.get(
|
135
|
-
:stackdriver_ingested_entries_count
|
139
|
+
:stackdriver_ingested_entries_count
|
140
|
+
))
|
136
141
|
assert_nil(Prometheus::Client.registry.get(
|
137
|
-
:stackdriver_dropped_entries_count
|
142
|
+
:stackdriver_dropped_entries_count
|
143
|
+
))
|
138
144
|
assert_nil(Prometheus::Client.registry.get(
|
139
|
-
:stackdriver_retried_entries_count
|
145
|
+
:stackdriver_retried_entries_count
|
146
|
+
))
|
140
147
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
141
148
|
Monitoring::MetricTranslator.new(
|
142
|
-
:stackdriver_successful_requests_count, {}
|
149
|
+
:stackdriver_successful_requests_count, {}
|
150
|
+
)
|
151
|
+
))
|
143
152
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
144
153
|
Monitoring::MetricTranslator.new(
|
145
|
-
:stackdriver_failed_requests_count, {}
|
154
|
+
:stackdriver_failed_requests_count, {}
|
155
|
+
)
|
156
|
+
))
|
146
157
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
147
158
|
Monitoring::MetricTranslator.new(
|
148
|
-
:stackdriver_ingested_entries_count, {}
|
159
|
+
:stackdriver_ingested_entries_count, {}
|
160
|
+
)
|
161
|
+
))
|
149
162
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
150
163
|
Monitoring::MetricTranslator.new(
|
151
|
-
:stackdriver_dropped_entries_count, {}
|
164
|
+
:stackdriver_dropped_entries_count, {}
|
165
|
+
)
|
166
|
+
))
|
152
167
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
153
168
|
Monitoring::MetricTranslator.new(
|
154
|
-
:stackdriver_retried_entries_count, {}
|
169
|
+
:stackdriver_retried_entries_count, {}
|
170
|
+
)
|
171
|
+
))
|
155
172
|
end
|
156
173
|
|
157
174
|
def test_configure_uses_metrics_resource
|
158
175
|
setup_gce_metadata_stubs
|
159
176
|
[CONFIG_METRICS_RESOURCE_JSON,
|
160
177
|
CONFIG_METRICS_RESOURCE_HASH,
|
161
|
-
CONFIG_METRICS_RESOURCE_JSON_HASH
|
162
|
-
].each_with_index do |config, index|
|
178
|
+
CONFIG_METRICS_RESOURCE_JSON_HASH].each_with_index do |config, index|
|
163
179
|
d = create_driver(config)
|
164
180
|
assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
|
165
181
|
"Index #{index}"
|
@@ -171,7 +187,8 @@ module BaseTest
|
|
171
187
|
registry = d.instance.instance_variable_get(:@registry)
|
172
188
|
assert_not_nil registry
|
173
189
|
monitored_resource = registry.instance_variable_get(
|
174
|
-
:@metrics_monitored_resource
|
190
|
+
:@metrics_monitored_resource
|
191
|
+
)
|
175
192
|
assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
|
176
193
|
assert_equal({ 'label1' => '123', 'label2' => 'abc' },
|
177
194
|
monitored_resource.labels, "Index #{index}")
|
@@ -190,14 +207,12 @@ module BaseTest
|
|
190
207
|
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
|
191
208
|
/unrecognized keys: \[:random\]/
|
192
209
|
}.each_with_index do |(config, pattern), index|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
"Index #{index} failed: got #{error.message}."
|
200
|
-
end
|
210
|
+
create_driver(config)
|
211
|
+
assert false,
|
212
|
+
"Invalid config at index #{index} should have raised an error."
|
213
|
+
rescue Fluent::ConfigError => e
|
214
|
+
assert e.message.match?(pattern), \
|
215
|
+
"Index #{index} failed: got #{e.message}."
|
201
216
|
end
|
202
217
|
end
|
203
218
|
|
@@ -277,16 +292,16 @@ module BaseTest
|
|
277
292
|
exception_count = 0
|
278
293
|
begin
|
279
294
|
create_driver
|
280
|
-
rescue Fluent::ConfigError =>
|
281
|
-
assert
|
282
|
-
assert
|
295
|
+
rescue Fluent::ConfigError => e
|
296
|
+
assert e.message.include? 'Unable to obtain metadata parameters:'
|
297
|
+
assert e.message.include? 'project_id'
|
283
298
|
exception_count += 1
|
284
299
|
end
|
285
300
|
assert_equal 1, exception_count
|
286
301
|
end
|
287
302
|
|
288
303
|
def test_project_id_from_credentials
|
289
|
-
%w
|
304
|
+
%w[gce ec2].each do |platform|
|
290
305
|
send("setup_#{platform}_metadata_stubs")
|
291
306
|
[IAM_CREDENTIALS, NEW_STYLE_CREDENTIALS, LEGACY_CREDENTIALS].each \
|
292
307
|
do |creds|
|
@@ -317,18 +332,19 @@ module BaseTest
|
|
317
332
|
d.run
|
318
333
|
end
|
319
334
|
verify_log_entries(1, COMPUTE_PARAMS.merge(
|
320
|
-
project_id: IAM_CREDENTIALS[:project_id]
|
335
|
+
project_id: IAM_CREDENTIALS[:project_id]
|
336
|
+
))
|
321
337
|
end
|
322
338
|
|
323
339
|
def test_invalid_json_credentials
|
324
|
-
%w
|
340
|
+
%w[gce_metadata ec2_metadata no_metadata_service].each do |platform|
|
325
341
|
send("setup_#{platform}_stubs")
|
326
342
|
exception_count = 0
|
327
343
|
ENV[CREDENTIALS_PATH_ENV_VAR] = INVALID_CREDENTIALS[:path]
|
328
344
|
begin
|
329
345
|
create_driver
|
330
|
-
rescue RuntimeError =>
|
331
|
-
assert
|
346
|
+
rescue RuntimeError => e
|
347
|
+
assert e.message.include? 'Unable to read the credential file'
|
332
348
|
exception_count += 1
|
333
349
|
end
|
334
350
|
assert_equal 1, exception_count
|
@@ -498,9 +514,9 @@ module BaseTest
|
|
498
514
|
d.emit(
|
499
515
|
'int_key' => { 1 => message },
|
500
516
|
'int_array_key' => { [1, 2, 3, 4] => message },
|
501
|
-
'string_array_key' => { %w
|
517
|
+
'string_array_key' => { %w[a b c] => message },
|
502
518
|
'hash_key' => { { 'some_key' => 'some_value' } => message },
|
503
|
-
'mixed_key' => { { 'some_key' => %w
|
519
|
+
'mixed_key' => { { 'some_key' => %w[a b c] } => message },
|
504
520
|
'symbol_key' => { some_symbol: message },
|
505
521
|
'nil_key' => { nil => message }
|
506
522
|
)
|
@@ -527,9 +543,9 @@ module BaseTest
|
|
527
543
|
'"data": 5000, "some_null_field": null}'
|
528
544
|
setup_logging_stubs do
|
529
545
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
530
|
-
d.emit('message' =>
|
546
|
+
d.emit('message' => "notJSON #{json_string}")
|
531
547
|
d.emit('message' => json_string)
|
532
|
-
d.emit('message' => " \r\n \t"
|
548
|
+
d.emit('message' => " \r\n \t#{json_string}")
|
533
549
|
d.run
|
534
550
|
end
|
535
551
|
verify_log_entries(3, COMPUTE_PARAMS, 'textPayload') do
|
@@ -543,10 +559,10 @@ module BaseTest
|
|
543
559
|
'"data": 5000, "some_null_field": null}'
|
544
560
|
setup_logging_stubs do
|
545
561
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
546
|
-
%w
|
547
|
-
d.emit(field =>
|
562
|
+
%w[log msg].each do |field|
|
563
|
+
d.emit(field => "notJSON #{json_string}")
|
548
564
|
d.emit(field => json_string)
|
549
|
-
d.emit(field => " \r\n \t"
|
565
|
+
d.emit(field => " \r\n \t#{json_string}")
|
550
566
|
end
|
551
567
|
d.run
|
552
568
|
end
|
@@ -564,7 +580,7 @@ module BaseTest
|
|
564
580
|
'"data": 5000, "some_null_field": null}'
|
565
581
|
setup_logging_stubs do
|
566
582
|
d = create_driver(DETECT_JSON_CONFIG)
|
567
|
-
d.emit('message' =>
|
583
|
+
d.emit('message' => "notJSON #{json_string}")
|
568
584
|
d.run
|
569
585
|
end
|
570
586
|
verify_log_entries(1, COMPUTE_PARAMS, 'textPayload') do
|
@@ -578,8 +594,8 @@ module BaseTest
|
|
578
594
|
'"data": 5000, "some_null_field": null}'
|
579
595
|
setup_logging_stubs do
|
580
596
|
d = create_driver(DETECT_JSON_CONFIG)
|
581
|
-
%w
|
582
|
-
d.emit(field =>
|
597
|
+
%w[log msg].each do |field|
|
598
|
+
d.emit(field => "notJSON #{json_string}")
|
583
599
|
end
|
584
600
|
d.run
|
585
601
|
end
|
@@ -668,9 +684,9 @@ module BaseTest
|
|
668
684
|
'"data": 5000, "some_null_field": null}'
|
669
685
|
setup_logging_stubs do
|
670
686
|
d = create_driver(DETECT_JSON_CONFIG)
|
671
|
-
%w
|
687
|
+
%w[message log msg].each do |field|
|
672
688
|
d.emit(field => json_string)
|
673
|
-
d.emit(field => " \r\n \t"
|
689
|
+
d.emit(field => " \r\n \t#{json_string}")
|
674
690
|
end
|
675
691
|
d.run
|
676
692
|
end
|
@@ -691,9 +707,9 @@ module BaseTest
|
|
691
707
|
'"data": 5000, "some_null_field": null}'
|
692
708
|
setup_logging_stubs do
|
693
709
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
694
|
-
d.emit(container_log_entry_with_metadata(
|
710
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
695
711
|
d.emit(container_log_entry_with_metadata(json_string))
|
696
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
712
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
697
713
|
d.run
|
698
714
|
end
|
699
715
|
verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -708,7 +724,7 @@ module BaseTest
|
|
708
724
|
'"data": 5000, "some_null_field": null}'
|
709
725
|
setup_logging_stubs do
|
710
726
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
711
|
-
d.emit(container_log_entry_with_metadata(
|
727
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
712
728
|
d.run
|
713
729
|
end
|
714
730
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -724,7 +740,7 @@ module BaseTest
|
|
724
740
|
setup_logging_stubs do
|
725
741
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
726
742
|
d.emit(container_log_entry_with_metadata(json_string))
|
727
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
743
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
728
744
|
d.run
|
729
745
|
end
|
730
746
|
verify_log_entries(2, CONTAINER_FROM_METADATA_PARAMS, 'jsonPayload') \
|
@@ -749,14 +765,16 @@ module BaseTest
|
|
749
765
|
setup_logging_stubs do
|
750
766
|
@logs_sent = []
|
751
767
|
d = create_driver(DETECT_JSON_CONFIG)
|
752
|
-
%w
|
768
|
+
%w[message log msg].each do |field|
|
753
769
|
d.emit(PRESERVED_KEYS_MAP.merge(
|
754
|
-
field => json_string
|
770
|
+
field => json_string
|
771
|
+
).merge(timestamp_fields))
|
755
772
|
end
|
756
773
|
d.run
|
757
774
|
end
|
758
775
|
expected_params = COMPUTE_PARAMS.merge(
|
759
|
-
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
776
|
+
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
777
|
+
)
|
760
778
|
verify_log_entries(3, expected_params, 'jsonPayload') do |entry|
|
761
779
|
fields = entry['jsonPayload']
|
762
780
|
assert_equal 4, fields.size, entry
|
@@ -772,7 +790,7 @@ module BaseTest
|
|
772
790
|
# any non-string tags or tags with non-utf8 characters are detected.
|
773
791
|
def test_reject_invalid_tags_with_require_valid_tags_true
|
774
792
|
setup_gce_metadata_stubs
|
775
|
-
INVALID_TAGS.
|
793
|
+
INVALID_TAGS.each_key do |tag|
|
776
794
|
setup_logging_stubs do
|
777
795
|
@logs_sent = []
|
778
796
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
@@ -800,8 +818,11 @@ module BaseTest
|
|
800
818
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
801
819
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
802
820
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
803
|
-
'container_name' => container_name
|
804
|
-
|
821
|
+
'container_name' => container_name
|
822
|
+
)
|
823
|
+
),
|
824
|
+
log_name: tag
|
825
|
+
)
|
805
826
|
verify_log_entries(1, params, 'textPayload')
|
806
827
|
end
|
807
828
|
|
@@ -824,8 +845,10 @@ module BaseTest
|
|
824
845
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
825
846
|
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
826
847
|
"#{GKE_CONSTANTS[:service]}/container_name" =>
|
827
|
-
CGI.unescape(encoded_name)
|
828
|
-
|
848
|
+
CGI.unescape(encoded_name)
|
849
|
+
),
|
850
|
+
log_name: encoded_name
|
851
|
+
)
|
829
852
|
verify_log_entries(0, params, 'textPayload')
|
830
853
|
end
|
831
854
|
end
|
@@ -861,8 +884,11 @@ module BaseTest
|
|
861
884
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
862
885
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
863
886
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
864
|
-
'container_name' => tag
|
865
|
-
|
887
|
+
'container_name' => tag
|
888
|
+
)
|
889
|
+
),
|
890
|
+
log_name: encoded_tag
|
891
|
+
)
|
866
892
|
verify_log_entries(1, params, 'textPayload')
|
867
893
|
end
|
868
894
|
end
|
@@ -907,8 +933,11 @@ module BaseTest
|
|
907
933
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
908
934
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
909
935
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
910
|
-
'container_name' => CGI.unescape(encoded_container_name)
|
911
|
-
|
936
|
+
'container_name' => CGI.unescape(encoded_container_name)
|
937
|
+
)
|
938
|
+
),
|
939
|
+
log_name: encoded_container_name
|
940
|
+
)
|
912
941
|
verify_log_entries(1, params, 'textPayload')
|
913
942
|
end
|
914
943
|
end
|
@@ -960,13 +989,15 @@ module BaseTest
|
|
960
989
|
request_count,
|
961
990
|
'agent.googleapis.com/agent',
|
962
991
|
OpenCensus::Stats::Aggregation::Sum, d,
|
963
|
-
:aggregate
|
992
|
+
:aggregate
|
993
|
+
)
|
964
994
|
assert_prometheus_metric_value(
|
965
995
|
:stackdriver_ingested_entries_count,
|
966
996
|
log_entry_count,
|
967
997
|
'agent.googleapis.com/agent',
|
968
998
|
OpenCensus::Stats::Aggregation::Sum, d,
|
969
|
-
:aggregate
|
999
|
+
:aggregate
|
1000
|
+
)
|
970
1001
|
end
|
971
1002
|
end
|
972
1003
|
end
|
@@ -1044,13 +1075,13 @@ module BaseTest
|
|
1044
1075
|
current_time)
|
1045
1076
|
end
|
1046
1077
|
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1078
|
+
december29 = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
|
1079
|
+
december31 = Time.new(2019, 12, 31, 10, 23, 35, '-08:00')
|
1080
|
+
january1 = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1050
1081
|
|
1051
1082
|
{
|
1052
1083
|
# December 29, 2019 (normal operation).
|
1053
|
-
|
1084
|
+
december29 => begin
|
1054
1085
|
one_day_later = Time.new(2019, 12, 30, 10, 23, 35, '-08:00')
|
1055
1086
|
one_day_a_year_earlier = Time.new(2018, 12, 30, 10, 23, 35, '-08:00')
|
1056
1087
|
just_under_one_day_later = Time.new(2019, 12, 30, 10, 23, 34, '-08:00')
|
@@ -1064,7 +1095,7 @@ module BaseTest
|
|
1064
1095
|
{
|
1065
1096
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1066
1097
|
Time.at(0) => Time.at(0),
|
1067
|
-
|
1098
|
+
december29 => december29,
|
1068
1099
|
one_day_later => one_day_a_year_earlier,
|
1069
1100
|
just_under_one_day_later => just_under_one_day_later,
|
1070
1101
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1074,7 +1105,7 @@ module BaseTest
|
|
1074
1105
|
}
|
1075
1106
|
end,
|
1076
1107
|
# January 1, 2020 (normal operation).
|
1077
|
-
|
1108
|
+
january1 => begin
|
1078
1109
|
one_day_later = Time.new(2020, 1, 2, 10, 23, 35, '-08:00')
|
1079
1110
|
one_day_a_year_earlier = Time.new(2019, 1, 2, 10, 23, 35, '-08:00')
|
1080
1111
|
just_under_one_day_later = Time.new(2020, 1, 2, 10, 23, 34, '-08:00')
|
@@ -1088,7 +1119,7 @@ module BaseTest
|
|
1088
1119
|
{
|
1089
1120
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1090
1121
|
Time.at(0) => Time.at(0),
|
1091
|
-
|
1122
|
+
january1 => january1,
|
1092
1123
|
one_day_later => one_day_a_year_earlier,
|
1093
1124
|
just_under_one_day_later => just_under_one_day_later,
|
1094
1125
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1098,7 +1129,7 @@ module BaseTest
|
|
1098
1129
|
}
|
1099
1130
|
end,
|
1100
1131
|
# December 31, 2019 (next day overlaps new year).
|
1101
|
-
|
1132
|
+
december31 => begin
|
1102
1133
|
one_day_later = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1103
1134
|
just_under_one_day_later = Time.new(2020, 1, 1, 10, 23, 34, '-08:00')
|
1104
1135
|
next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
|
@@ -1109,7 +1140,7 @@ module BaseTest
|
|
1109
1140
|
{
|
1110
1141
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1111
1142
|
Time.at(0) => Time.at(0),
|
1112
|
-
|
1143
|
+
december31 => december31,
|
1113
1144
|
one_day_later => Time.at(0), # Falls into the next year.
|
1114
1145
|
just_under_one_day_later => just_under_one_day_later,
|
1115
1146
|
one_second_before_next_year => one_second_before_next_year,
|
@@ -1607,7 +1638,8 @@ module BaseTest
|
|
1607
1638
|
custom_key: 'custom_labels_key',
|
1608
1639
|
custom_key_config: CONFIG_CUSTOM_LABELS_KEY_SPECIFIED,
|
1609
1640
|
sample_value: COMPUTE_PARAMS[:labels].merge(
|
1610
|
-
LABELS_MESSAGE
|
1641
|
+
LABELS_MESSAGE
|
1642
|
+
),
|
1611
1643
|
default_value: COMPUTE_PARAMS[:labels])
|
1612
1644
|
end
|
1613
1645
|
|
@@ -1659,7 +1691,8 @@ module BaseTest
|
|
1659
1691
|
verify_cascading_json_detection_with_log_entry_fields(
|
1660
1692
|
'insertId', DEFAULT_INSERT_ID_KEY,
|
1661
1693
|
root_level_value: INSERT_ID,
|
1662
|
-
nested_level_value: INSERT_ID2
|
1694
|
+
nested_level_value: INSERT_ID2
|
1695
|
+
)
|
1663
1696
|
end
|
1664
1697
|
|
1665
1698
|
def test_cascading_json_detection_with_log_entry_labels_field
|
@@ -1669,7 +1702,9 @@ module BaseTest
|
|
1669
1702
|
nested_level_value: LABELS_MESSAGE2,
|
1670
1703
|
expected_value_from_root: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE),
|
1671
1704
|
expected_value_from_nested: COMPUTE_PARAMS[:labels].merge(
|
1672
|
-
LABELS_MESSAGE2
|
1705
|
+
LABELS_MESSAGE2
|
1706
|
+
)
|
1707
|
+
)
|
1673
1708
|
end
|
1674
1709
|
|
1675
1710
|
def test_cascading_json_detection_with_log_entry_operation_field
|
@@ -1677,28 +1712,32 @@ module BaseTest
|
|
1677
1712
|
'operation', DEFAULT_OPERATION_KEY,
|
1678
1713
|
root_level_value: OPERATION_MESSAGE,
|
1679
1714
|
nested_level_value: OPERATION_MESSAGE2,
|
1680
|
-
expected_value_from_nested: expected_operation_message2
|
1715
|
+
expected_value_from_nested: expected_operation_message2
|
1716
|
+
)
|
1681
1717
|
end
|
1682
1718
|
|
1683
1719
|
def test_cascading_json_detection_with_log_entry_source_location_field
|
1684
1720
|
verify_cascading_json_detection_with_log_entry_fields(
|
1685
1721
|
'sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
|
1686
1722
|
root_level_value: SOURCE_LOCATION_MESSAGE,
|
1687
|
-
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1723
|
+
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1724
|
+
)
|
1688
1725
|
end
|
1689
1726
|
|
1690
1727
|
def test_cascading_json_detection_with_log_entry_span_id_field
|
1691
1728
|
verify_cascading_json_detection_with_log_entry_fields(
|
1692
1729
|
'spanId', DEFAULT_SPAN_ID_KEY,
|
1693
1730
|
root_level_value: SPAN_ID,
|
1694
|
-
nested_level_value: SPAN_ID2
|
1731
|
+
nested_level_value: SPAN_ID2
|
1732
|
+
)
|
1695
1733
|
end
|
1696
1734
|
|
1697
1735
|
def test_cascading_json_detection_with_log_entry_trace_field
|
1698
1736
|
verify_cascading_json_detection_with_log_entry_fields(
|
1699
1737
|
'trace', DEFAULT_TRACE_KEY,
|
1700
1738
|
root_level_value: TRACE,
|
1701
|
-
nested_level_value: TRACE2
|
1739
|
+
nested_level_value: TRACE2
|
1740
|
+
)
|
1702
1741
|
end
|
1703
1742
|
|
1704
1743
|
def test_cascading_json_detection_with_log_entry_trace_sampled_field
|
@@ -1707,7 +1746,8 @@ module BaseTest
|
|
1707
1746
|
root_level_value: TRACE_SAMPLED,
|
1708
1747
|
nested_level_value: TRACE_SAMPLED2,
|
1709
1748
|
default_value_from_root: false,
|
1710
|
-
default_value_from_nested: false
|
1749
|
+
default_value_from_nested: false
|
1750
|
+
)
|
1711
1751
|
end
|
1712
1752
|
|
1713
1753
|
# Verify that labels present in multiple inputs respect the expected priority
|
@@ -1739,9 +1779,11 @@ module BaseTest
|
|
1739
1779
|
{
|
1740
1780
|
config: CONFIG_LABLES_AND_LABLE_MAP,
|
1741
1781
|
emitted_log: PAYLOAD_FOR_LABEL_MAP.merge(
|
1742
|
-
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1782
|
+
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1783
|
+
),
|
1743
1784
|
expected_labels: LABELS_MESSAGE.merge(LABELS_FROM_LABELS_CONFIG).merge(
|
1744
|
-
LABELS_FROM_LABEL_MAP_CONFIG
|
1785
|
+
LABELS_FROM_LABEL_MAP_CONFIG
|
1786
|
+
)
|
1745
1787
|
},
|
1746
1788
|
# labels from the config "labels" and "label_map" conflict.
|
1747
1789
|
{
|
@@ -1761,14 +1803,16 @@ module BaseTest
|
|
1761
1803
|
{
|
1762
1804
|
config: CONFIG_LABEL_MAP_CONFLICTING,
|
1763
1805
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1764
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1806
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1807
|
+
),
|
1765
1808
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1766
1809
|
},
|
1767
1810
|
# All three types of labels conflict.
|
1768
1811
|
{
|
1769
1812
|
config: CONFIG_LABLES_AND_LABLE_MAP_CONFLICTING,
|
1770
1813
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1771
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1814
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1815
|
+
),
|
1772
1816
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1773
1817
|
}
|
1774
1818
|
].each do |test_params|
|
@@ -1780,7 +1824,8 @@ module BaseTest
|
|
1780
1824
|
d.run
|
1781
1825
|
end
|
1782
1826
|
expected_params = COMPUTE_PARAMS.merge(
|
1783
|
-
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1827
|
+
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1828
|
+
)
|
1784
1829
|
verify_log_entries(1, expected_params)
|
1785
1830
|
end
|
1786
1831
|
end
|
@@ -1852,7 +1897,8 @@ module BaseTest
|
|
1852
1897
|
config: APPLICATION_DEFAULT_CONFIG,
|
1853
1898
|
setup_k8s_stub: true,
|
1854
1899
|
log_entry: k8s_container_log_entry(
|
1855
|
-
log_entry(0)
|
1900
|
+
log_entry(0)
|
1901
|
+
).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
1856
1902
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1857
1903
|
},
|
1858
1904
|
{
|
@@ -1860,7 +1906,8 @@ module BaseTest
|
|
1860
1906
|
setup_k8s_stub: true,
|
1861
1907
|
log_entry: k8s_container_log_entry(
|
1862
1908
|
log_entry(0),
|
1863
|
-
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1909
|
+
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1910
|
+
),
|
1864
1911
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1865
1912
|
}
|
1866
1913
|
].each do |test_params|
|
@@ -1979,7 +2026,8 @@ module BaseTest
|
|
1979
2026
|
assert_metric_value.call(
|
1980
2027
|
:uptime, expected, 'agent.googleapis.com/agent',
|
1981
2028
|
OpenCensus::Stats::Aggregation::Sum, d,
|
1982
|
-
version: Fluent::GoogleCloudOutput.version_string
|
2029
|
+
version: Fluent::GoogleCloudOutput.version_string
|
2030
|
+
)
|
1983
2031
|
rescue Test::Unit::AssertionFailedError
|
1984
2032
|
retry if (retries += 1) < 3
|
1985
2033
|
end
|
@@ -2013,12 +2061,12 @@ module BaseTest
|
|
2013
2061
|
(1..entry_count).each do |entry_index|
|
2014
2062
|
d.emit('message' => log_entry(entry_index.to_s))
|
2015
2063
|
end
|
2016
|
-
# rubocop:disable Lint/
|
2064
|
+
# rubocop:disable Lint/SuppressedException
|
2017
2065
|
begin
|
2018
2066
|
d.run
|
2019
2067
|
rescue mock_error_type
|
2020
2068
|
end
|
2021
|
-
# rubocop:enable Lint/
|
2069
|
+
# rubocop:enable Lint/SuppressedException
|
2022
2070
|
failed_requests_count, dropped_entries_count,
|
2023
2071
|
retried_entries_count = metric_values
|
2024
2072
|
|
@@ -2067,6 +2115,7 @@ module BaseTest
|
|
2067
2115
|
# assertion will fail in the case when a single metric contains time
|
2068
2116
|
# series with success and failure events.
|
2069
2117
|
next if code == ok_status_code
|
2118
|
+
|
2070
2119
|
assert_metric_value.call(:stackdriver_failed_requests_count,
|
2071
2120
|
failed_requests_count,
|
2072
2121
|
'agent.googleapis.com/agent',
|
@@ -2100,7 +2149,8 @@ module BaseTest
|
|
2100
2149
|
end
|
2101
2150
|
|
2102
2151
|
def container_log_entry_with_metadata(
|
2103
|
-
|
2152
|
+
log, container_name = K8S_CONTAINER_NAME
|
2153
|
+
)
|
2104
2154
|
{
|
2105
2155
|
log: log,
|
2106
2156
|
stream: K8S_STREAM,
|
@@ -2159,10 +2209,10 @@ module BaseTest
|
|
2159
2209
|
}
|
2160
2210
|
end
|
2161
2211
|
|
2162
|
-
def dataflow_log_entry(
|
2212
|
+
def dataflow_log_entry(index)
|
2163
2213
|
{
|
2164
2214
|
step: DATAFLOW_STEP_ID,
|
2165
|
-
message: log_entry(
|
2215
|
+
message: log_entry(index)
|
2166
2216
|
}
|
2167
2217
|
end
|
2168
2218
|
|
@@ -2175,10 +2225,10 @@ module BaseTest
|
|
2175
2225
|
}
|
2176
2226
|
end
|
2177
2227
|
|
2178
|
-
def ml_log_entry(
|
2228
|
+
def ml_log_entry(index)
|
2179
2229
|
{
|
2180
2230
|
name: ML_LOG_AREA,
|
2181
|
-
message: log_entry(
|
2231
|
+
message: log_entry(index)
|
2182
2232
|
}
|
2183
2233
|
end
|
2184
2234
|
|
@@ -2189,8 +2239,8 @@ module BaseTest
|
|
2189
2239
|
}
|
2190
2240
|
end
|
2191
2241
|
|
2192
|
-
def log_entry(
|
2193
|
-
"test log entry #{
|
2242
|
+
def log_entry(index)
|
2243
|
+
"test log entry #{index}"
|
2194
2244
|
end
|
2195
2245
|
|
2196
2246
|
# If check_exact_labels is true, assert 'labels' and 'expected_labels' match
|
@@ -2198,6 +2248,7 @@ module BaseTest
|
|
2198
2248
|
# 'expected_labels'.
|
2199
2249
|
def check_labels(expected_labels, labels, check_exact_labels = true)
|
2200
2250
|
return if expected_labels.empty? && labels.empty?
|
2251
|
+
|
2201
2252
|
expected_labels.each do |expected_key, expected_value|
|
2202
2253
|
assert labels.key?(expected_key), "Expected label #{expected_key} not" \
|
2203
2254
|
" found. Got labels: #{labels}."
|
@@ -2207,20 +2258,21 @@ module BaseTest
|
|
2207
2258
|
assert_equal expected_value, actual_value, "Value for #{expected_key}" \
|
2208
2259
|
" mismatch. Expected #{expected_value}. Got #{actual_value}"
|
2209
2260
|
end
|
2210
|
-
|
2211
|
-
|
2212
|
-
|
2213
|
-
|
2214
|
-
|
2261
|
+
return unless check_exact_labels
|
2262
|
+
|
2263
|
+
assert_equal expected_labels.length, labels.length, 'Expected ' \
|
2264
|
+
"#{expected_labels.length} labels: #{expected_labels}, got " \
|
2265
|
+
"#{labels.length} labels: #{labels}"
|
2215
2266
|
end
|
2216
2267
|
|
2217
|
-
def verify_default_log_entry_text(text,
|
2218
|
-
assert_equal "test log entry #{
|
2219
|
-
"Entry ##{
|
2268
|
+
def verify_default_log_entry_text(text, index, entry)
|
2269
|
+
assert_equal "test log entry #{index}", text,
|
2270
|
+
"Entry ##{index} had unexpected text: #{entry}"
|
2220
2271
|
end
|
2221
2272
|
|
2222
2273
|
# The caller can optionally provide a block which is called for each entry.
|
2223
|
-
def verify_json_log_entries(
|
2274
|
+
def verify_json_log_entries(expected_count, params,
|
2275
|
+
payload_type = 'textPayload',
|
2224
2276
|
check_exact_entry_labels = true)
|
2225
2277
|
entry_count = 0
|
2226
2278
|
@logs_sent.each do |request|
|
@@ -2258,11 +2310,11 @@ module BaseTest
|
|
2258
2310
|
entry)
|
2259
2311
|
end
|
2260
2312
|
entry_count += 1
|
2261
|
-
assert entry_count <=
|
2262
|
-
"Number of entries #{entry_count} exceeds expected number #{
|
2313
|
+
assert entry_count <= expected_count,
|
2314
|
+
"Number of entries #{entry_count} exceeds expected number #{expected_count}."
|
2263
2315
|
end
|
2264
2316
|
end
|
2265
|
-
assert_equal
|
2317
|
+
assert_equal expected_count, entry_count
|
2266
2318
|
end
|
2267
2319
|
|
2268
2320
|
def verify_container_logs(log_entry_factory, expected_params)
|
@@ -2291,13 +2343,17 @@ module BaseTest
|
|
2291
2343
|
# LogEntry info from. The values are lists of two elements: the name of
|
2292
2344
|
# the subfield in LogEntry object and the expected value of that field.
|
2293
2345
|
DEFAULT_HTTP_REQUEST_KEY => [
|
2294
|
-
'httpRequest', HTTP_REQUEST_MESSAGE
|
2346
|
+
'httpRequest', HTTP_REQUEST_MESSAGE
|
2347
|
+
],
|
2295
2348
|
DEFAULT_LABELS_KEY => [
|
2296
|
-
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2349
|
+
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2350
|
+
],
|
2297
2351
|
DEFAULT_OPERATION_KEY => [
|
2298
|
-
'operation', OPERATION_MESSAGE
|
2352
|
+
'operation', OPERATION_MESSAGE
|
2353
|
+
],
|
2299
2354
|
DEFAULT_SOURCE_LOCATION_KEY => [
|
2300
|
-
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2355
|
+
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2356
|
+
]
|
2301
2357
|
}
|
2302
2358
|
end
|
2303
2359
|
|
@@ -2397,17 +2453,22 @@ module BaseTest
|
|
2397
2453
|
# left with name "log", "message" or "msg". This test verifies additional
|
2398
2454
|
# LogEntry fields like spanId and traceId do not disable that by accident.
|
2399
2455
|
def verify_cascading_json_detection_with_log_entry_fields(
|
2400
|
-
|
2456
|
+
log_entry_field, default_key, expectation
|
2457
|
+
)
|
2401
2458
|
root_level_value = expectation[:root_level_value]
|
2402
2459
|
nested_level_value = expectation[:nested_level_value]
|
2403
2460
|
expected_value_from_root = expectation.fetch(
|
2404
|
-
:expected_value_from_root, root_level_value
|
2461
|
+
:expected_value_from_root, root_level_value
|
2462
|
+
)
|
2405
2463
|
expected_value_from_nested = expectation.fetch(
|
2406
|
-
:expected_value_from_nested, nested_level_value
|
2464
|
+
:expected_value_from_nested, nested_level_value
|
2465
|
+
)
|
2407
2466
|
default_value_from_root = expectation.fetch(
|
2408
|
-
:default_value_from_root, nil
|
2467
|
+
:default_value_from_root, nil
|
2468
|
+
)
|
2409
2469
|
default_value_from_nested = expectation.fetch(
|
2410
|
-
:default_value_from_nested, nil
|
2470
|
+
:default_value_from_nested, nil
|
2471
|
+
)
|
2411
2472
|
|
2412
2473
|
setup_gce_metadata_stubs
|
2413
2474
|
|
@@ -2443,7 +2504,8 @@ module BaseTest
|
|
2443
2504
|
# }
|
2444
2505
|
# }
|
2445
2506
|
log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
|
2446
|
-
default_key => root_level_value
|
2507
|
+
default_key => root_level_value
|
2508
|
+
)
|
2447
2509
|
|
2448
2510
|
[
|
2449
2511
|
[
|
@@ -2594,7 +2656,8 @@ module BaseTest
|
|
2594
2656
|
|
2595
2657
|
# Verify the number and the content of the log entries match the expectation.
|
2596
2658
|
# The caller can optionally provide a block which is called for each entry.
|
2597
|
-
def verify_log_entries(
|
2659
|
+
def verify_log_entries(_expected_count, _params,
|
2660
|
+
_payload_type = 'textPayload',
|
2598
2661
|
_check_exact_entry_labels = true, &_block)
|
2599
2662
|
_undefined
|
2600
2663
|
end
|