fluent-plugin-google-cloud 0.12.10 → 0.12.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +26 -19
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +9 -9
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -11
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +222 -166
- data/lib/fluent/plugin/statusz.rb +7 -9
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +194 -134
- data/test/plugin/constants.rb +18 -14
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +33 -21
- data/test/plugin/test_out_google_cloud_grpc.rb +35 -22
- data/test/plugin/utils.rb +8 -6
- metadata +29 -29
data/test/plugin/base_test.rb
CHANGED
@@ -71,8 +71,8 @@ module BaseTest
|
|
71
71
|
exception_count = 0
|
72
72
|
begin
|
73
73
|
create_driver(PRIVATE_KEY_CONFIG)
|
74
|
-
rescue Fluent::ConfigError =>
|
75
|
-
assert
|
74
|
+
rescue Fluent::ConfigError => e
|
75
|
+
assert e.message.include? 'Please remove configuration parameters'
|
76
76
|
exception_count += 1
|
77
77
|
end
|
78
78
|
assert_equal 1, exception_count
|
@@ -100,23 +100,22 @@ module BaseTest
|
|
100
100
|
def test_configure_metadata_missing_parts_on_other_platforms
|
101
101
|
setup_no_metadata_service_stubs
|
102
102
|
Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
|
103
|
-
[
|
104
|
-
|
105
|
-
|
106
|
-
|
103
|
+
[
|
104
|
+
[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
|
105
|
+
[CONFIG_MISSING_METADATA_ZONE, [], true],
|
106
|
+
[CONFIG_MISSING_METADATA_VM_ID, [], true],
|
107
|
+
[CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
|
107
108
|
].each_with_index do |(config, missing_parts, is_valid_config), index|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
assert error.message.include?(part), "Index #{index} failed."
|
119
|
-
end
|
109
|
+
create_driver(config)
|
110
|
+
assert_true is_valid_config, "Invalid config at index #{index} should "\
|
111
|
+
'have raised an error.'
|
112
|
+
rescue Fluent::ConfigError => e
|
113
|
+
assert_false is_valid_config, "Valid config at index #{index} should "\
|
114
|
+
"not have raised an error #{e}."
|
115
|
+
assert e.message.include?('Unable to obtain metadata parameters:'),
|
116
|
+
"Index #{index} failed."
|
117
|
+
missing_parts.each do |part|
|
118
|
+
assert e.message.include?(part), "Index #{index} failed."
|
120
119
|
end
|
121
120
|
end
|
122
121
|
end
|
@@ -128,38 +127,52 @@ module BaseTest
|
|
128
127
|
setup_gce_metadata_stubs
|
129
128
|
create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
|
130
129
|
assert_nil(Prometheus::Client.registry.get(
|
131
|
-
:stackdriver_successful_requests_count
|
130
|
+
:stackdriver_successful_requests_count
|
131
|
+
))
|
132
132
|
assert_nil(Prometheus::Client.registry.get(
|
133
|
-
:stackdriver_failed_requests_count
|
133
|
+
:stackdriver_failed_requests_count
|
134
|
+
))
|
134
135
|
assert_nil(Prometheus::Client.registry.get(
|
135
|
-
:stackdriver_ingested_entries_count
|
136
|
+
:stackdriver_ingested_entries_count
|
137
|
+
))
|
136
138
|
assert_nil(Prometheus::Client.registry.get(
|
137
|
-
:stackdriver_dropped_entries_count
|
139
|
+
:stackdriver_dropped_entries_count
|
140
|
+
))
|
138
141
|
assert_nil(Prometheus::Client.registry.get(
|
139
|
-
:stackdriver_retried_entries_count
|
142
|
+
:stackdriver_retried_entries_count
|
143
|
+
))
|
140
144
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
141
145
|
Monitoring::MetricTranslator.new(
|
142
|
-
:stackdriver_successful_requests_count, {}
|
146
|
+
:stackdriver_successful_requests_count, {}
|
147
|
+
)
|
148
|
+
))
|
143
149
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
144
150
|
Monitoring::MetricTranslator.new(
|
145
|
-
:stackdriver_failed_requests_count, {}
|
151
|
+
:stackdriver_failed_requests_count, {}
|
152
|
+
)
|
153
|
+
))
|
146
154
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
147
155
|
Monitoring::MetricTranslator.new(
|
148
|
-
:stackdriver_ingested_entries_count, {}
|
156
|
+
:stackdriver_ingested_entries_count, {}
|
157
|
+
)
|
158
|
+
))
|
149
159
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
150
160
|
Monitoring::MetricTranslator.new(
|
151
|
-
:stackdriver_dropped_entries_count, {}
|
161
|
+
:stackdriver_dropped_entries_count, {}
|
162
|
+
)
|
163
|
+
))
|
152
164
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
153
165
|
Monitoring::MetricTranslator.new(
|
154
|
-
:stackdriver_retried_entries_count, {}
|
166
|
+
:stackdriver_retried_entries_count, {}
|
167
|
+
)
|
168
|
+
))
|
155
169
|
end
|
156
170
|
|
157
171
|
def test_configure_uses_metrics_resource
|
158
172
|
setup_gce_metadata_stubs
|
159
173
|
[CONFIG_METRICS_RESOURCE_JSON,
|
160
174
|
CONFIG_METRICS_RESOURCE_HASH,
|
161
|
-
CONFIG_METRICS_RESOURCE_JSON_HASH
|
162
|
-
].each_with_index do |config, index|
|
175
|
+
CONFIG_METRICS_RESOURCE_JSON_HASH].each_with_index do |config, index|
|
163
176
|
d = create_driver(config)
|
164
177
|
assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
|
165
178
|
"Index #{index}"
|
@@ -171,7 +184,8 @@ module BaseTest
|
|
171
184
|
registry = d.instance.instance_variable_get(:@registry)
|
172
185
|
assert_not_nil registry
|
173
186
|
monitored_resource = registry.instance_variable_get(
|
174
|
-
:@metrics_monitored_resource
|
187
|
+
:@metrics_monitored_resource
|
188
|
+
)
|
175
189
|
assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
|
176
190
|
assert_equal({ 'label1' => '123', 'label2' => 'abc' },
|
177
191
|
monitored_resource.labels, "Index #{index}")
|
@@ -190,14 +204,12 @@ module BaseTest
|
|
190
204
|
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
|
191
205
|
/unrecognized keys: \[:random\]/
|
192
206
|
}.each_with_index do |(config, pattern), index|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
"Index #{index} failed: got #{error.message}."
|
200
|
-
end
|
207
|
+
create_driver(config)
|
208
|
+
assert false,
|
209
|
+
"Invalid config at index #{index} should have raised an error."
|
210
|
+
rescue Fluent::ConfigError => e
|
211
|
+
assert e.message.match?(pattern), \
|
212
|
+
"Index #{index} failed: got #{e.message}."
|
201
213
|
end
|
202
214
|
end
|
203
215
|
|
@@ -277,16 +289,16 @@ module BaseTest
|
|
277
289
|
exception_count = 0
|
278
290
|
begin
|
279
291
|
create_driver
|
280
|
-
rescue Fluent::ConfigError =>
|
281
|
-
assert
|
282
|
-
assert
|
292
|
+
rescue Fluent::ConfigError => e
|
293
|
+
assert e.message.include? 'Unable to obtain metadata parameters:'
|
294
|
+
assert e.message.include? 'project_id'
|
283
295
|
exception_count += 1
|
284
296
|
end
|
285
297
|
assert_equal 1, exception_count
|
286
298
|
end
|
287
299
|
|
288
300
|
def test_project_id_from_credentials
|
289
|
-
%w
|
301
|
+
%w[gce ec2].each do |platform|
|
290
302
|
send("setup_#{platform}_metadata_stubs")
|
291
303
|
[IAM_CREDENTIALS, NEW_STYLE_CREDENTIALS, LEGACY_CREDENTIALS].each \
|
292
304
|
do |creds|
|
@@ -317,18 +329,19 @@ module BaseTest
|
|
317
329
|
d.run
|
318
330
|
end
|
319
331
|
verify_log_entries(1, COMPUTE_PARAMS.merge(
|
320
|
-
project_id: IAM_CREDENTIALS[:project_id]
|
332
|
+
project_id: IAM_CREDENTIALS[:project_id]
|
333
|
+
))
|
321
334
|
end
|
322
335
|
|
323
336
|
def test_invalid_json_credentials
|
324
|
-
%w
|
337
|
+
%w[gce_metadata ec2_metadata no_metadata_service].each do |platform|
|
325
338
|
send("setup_#{platform}_stubs")
|
326
339
|
exception_count = 0
|
327
340
|
ENV[CREDENTIALS_PATH_ENV_VAR] = INVALID_CREDENTIALS[:path]
|
328
341
|
begin
|
329
342
|
create_driver
|
330
|
-
rescue RuntimeError =>
|
331
|
-
assert
|
343
|
+
rescue RuntimeError => e
|
344
|
+
assert e.message.include? 'Unable to read the credential file'
|
332
345
|
exception_count += 1
|
333
346
|
end
|
334
347
|
assert_equal 1, exception_count
|
@@ -498,9 +511,9 @@ module BaseTest
|
|
498
511
|
d.emit(
|
499
512
|
'int_key' => { 1 => message },
|
500
513
|
'int_array_key' => { [1, 2, 3, 4] => message },
|
501
|
-
'string_array_key' => { %w
|
514
|
+
'string_array_key' => { %w[a b c] => message },
|
502
515
|
'hash_key' => { { 'some_key' => 'some_value' } => message },
|
503
|
-
'mixed_key' => { { 'some_key' => %w
|
516
|
+
'mixed_key' => { { 'some_key' => %w[a b c] } => message },
|
504
517
|
'symbol_key' => { some_symbol: message },
|
505
518
|
'nil_key' => { nil => message }
|
506
519
|
)
|
@@ -527,9 +540,9 @@ module BaseTest
|
|
527
540
|
'"data": 5000, "some_null_field": null}'
|
528
541
|
setup_logging_stubs do
|
529
542
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
530
|
-
d.emit('message' =>
|
543
|
+
d.emit('message' => "notJSON #{json_string}")
|
531
544
|
d.emit('message' => json_string)
|
532
|
-
d.emit('message' => " \r\n \t"
|
545
|
+
d.emit('message' => " \r\n \t#{json_string}")
|
533
546
|
d.run
|
534
547
|
end
|
535
548
|
verify_log_entries(3, COMPUTE_PARAMS, 'textPayload') do
|
@@ -543,10 +556,10 @@ module BaseTest
|
|
543
556
|
'"data": 5000, "some_null_field": null}'
|
544
557
|
setup_logging_stubs do
|
545
558
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
546
|
-
%w
|
547
|
-
d.emit(field =>
|
559
|
+
%w[log msg].each do |field|
|
560
|
+
d.emit(field => "notJSON #{json_string}")
|
548
561
|
d.emit(field => json_string)
|
549
|
-
d.emit(field => " \r\n \t"
|
562
|
+
d.emit(field => " \r\n \t#{json_string}")
|
550
563
|
end
|
551
564
|
d.run
|
552
565
|
end
|
@@ -564,7 +577,7 @@ module BaseTest
|
|
564
577
|
'"data": 5000, "some_null_field": null}'
|
565
578
|
setup_logging_stubs do
|
566
579
|
d = create_driver(DETECT_JSON_CONFIG)
|
567
|
-
d.emit('message' =>
|
580
|
+
d.emit('message' => "notJSON #{json_string}")
|
568
581
|
d.run
|
569
582
|
end
|
570
583
|
verify_log_entries(1, COMPUTE_PARAMS, 'textPayload') do
|
@@ -578,8 +591,8 @@ module BaseTest
|
|
578
591
|
'"data": 5000, "some_null_field": null}'
|
579
592
|
setup_logging_stubs do
|
580
593
|
d = create_driver(DETECT_JSON_CONFIG)
|
581
|
-
%w
|
582
|
-
d.emit(field =>
|
594
|
+
%w[log msg].each do |field|
|
595
|
+
d.emit(field => "notJSON #{json_string}")
|
583
596
|
end
|
584
597
|
d.run
|
585
598
|
end
|
@@ -668,9 +681,9 @@ module BaseTest
|
|
668
681
|
'"data": 5000, "some_null_field": null}'
|
669
682
|
setup_logging_stubs do
|
670
683
|
d = create_driver(DETECT_JSON_CONFIG)
|
671
|
-
%w
|
684
|
+
%w[message log msg].each do |field|
|
672
685
|
d.emit(field => json_string)
|
673
|
-
d.emit(field => " \r\n \t"
|
686
|
+
d.emit(field => " \r\n \t#{json_string}")
|
674
687
|
end
|
675
688
|
d.run
|
676
689
|
end
|
@@ -691,9 +704,9 @@ module BaseTest
|
|
691
704
|
'"data": 5000, "some_null_field": null}'
|
692
705
|
setup_logging_stubs do
|
693
706
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
694
|
-
d.emit(container_log_entry_with_metadata(
|
707
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
695
708
|
d.emit(container_log_entry_with_metadata(json_string))
|
696
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
709
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
697
710
|
d.run
|
698
711
|
end
|
699
712
|
verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -708,7 +721,7 @@ module BaseTest
|
|
708
721
|
'"data": 5000, "some_null_field": null}'
|
709
722
|
setup_logging_stubs do
|
710
723
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
711
|
-
d.emit(container_log_entry_with_metadata(
|
724
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
712
725
|
d.run
|
713
726
|
end
|
714
727
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -724,7 +737,7 @@ module BaseTest
|
|
724
737
|
setup_logging_stubs do
|
725
738
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
726
739
|
d.emit(container_log_entry_with_metadata(json_string))
|
727
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
740
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
728
741
|
d.run
|
729
742
|
end
|
730
743
|
verify_log_entries(2, CONTAINER_FROM_METADATA_PARAMS, 'jsonPayload') \
|
@@ -749,14 +762,16 @@ module BaseTest
|
|
749
762
|
setup_logging_stubs do
|
750
763
|
@logs_sent = []
|
751
764
|
d = create_driver(DETECT_JSON_CONFIG)
|
752
|
-
%w
|
765
|
+
%w[message log msg].each do |field|
|
753
766
|
d.emit(PRESERVED_KEYS_MAP.merge(
|
754
|
-
field => json_string
|
767
|
+
field => json_string
|
768
|
+
).merge(timestamp_fields))
|
755
769
|
end
|
756
770
|
d.run
|
757
771
|
end
|
758
772
|
expected_params = COMPUTE_PARAMS.merge(
|
759
|
-
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
773
|
+
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
774
|
+
)
|
760
775
|
verify_log_entries(3, expected_params, 'jsonPayload') do |entry|
|
761
776
|
fields = entry['jsonPayload']
|
762
777
|
assert_equal 4, fields.size, entry
|
@@ -772,7 +787,7 @@ module BaseTest
|
|
772
787
|
# any non-string tags or tags with non-utf8 characters are detected.
|
773
788
|
def test_reject_invalid_tags_with_require_valid_tags_true
|
774
789
|
setup_gce_metadata_stubs
|
775
|
-
INVALID_TAGS.
|
790
|
+
INVALID_TAGS.each_key do |tag|
|
776
791
|
setup_logging_stubs do
|
777
792
|
@logs_sent = []
|
778
793
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
@@ -800,8 +815,11 @@ module BaseTest
|
|
800
815
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
801
816
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
802
817
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
803
|
-
'container_name' => container_name
|
804
|
-
|
818
|
+
'container_name' => container_name
|
819
|
+
)
|
820
|
+
),
|
821
|
+
log_name: tag
|
822
|
+
)
|
805
823
|
verify_log_entries(1, params, 'textPayload')
|
806
824
|
end
|
807
825
|
|
@@ -824,8 +842,10 @@ module BaseTest
|
|
824
842
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
825
843
|
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
826
844
|
"#{GKE_CONSTANTS[:service]}/container_name" =>
|
827
|
-
CGI.unescape(encoded_name)
|
828
|
-
|
845
|
+
CGI.unescape(encoded_name)
|
846
|
+
),
|
847
|
+
log_name: encoded_name
|
848
|
+
)
|
829
849
|
verify_log_entries(0, params, 'textPayload')
|
830
850
|
end
|
831
851
|
end
|
@@ -861,8 +881,11 @@ module BaseTest
|
|
861
881
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
862
882
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
863
883
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
864
|
-
'container_name' => tag
|
865
|
-
|
884
|
+
'container_name' => tag
|
885
|
+
)
|
886
|
+
),
|
887
|
+
log_name: encoded_tag
|
888
|
+
)
|
866
889
|
verify_log_entries(1, params, 'textPayload')
|
867
890
|
end
|
868
891
|
end
|
@@ -907,8 +930,11 @@ module BaseTest
|
|
907
930
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
908
931
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
909
932
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
910
|
-
'container_name' => CGI.unescape(encoded_container_name)
|
911
|
-
|
933
|
+
'container_name' => CGI.unescape(encoded_container_name)
|
934
|
+
)
|
935
|
+
),
|
936
|
+
log_name: encoded_container_name
|
937
|
+
)
|
912
938
|
verify_log_entries(1, params, 'textPayload')
|
913
939
|
end
|
914
940
|
end
|
@@ -960,13 +986,15 @@ module BaseTest
|
|
960
986
|
request_count,
|
961
987
|
'agent.googleapis.com/agent',
|
962
988
|
OpenCensus::Stats::Aggregation::Sum, d,
|
963
|
-
:aggregate
|
989
|
+
:aggregate
|
990
|
+
)
|
964
991
|
assert_prometheus_metric_value(
|
965
992
|
:stackdriver_ingested_entries_count,
|
966
993
|
log_entry_count,
|
967
994
|
'agent.googleapis.com/agent',
|
968
995
|
OpenCensus::Stats::Aggregation::Sum, d,
|
969
|
-
:aggregate
|
996
|
+
:aggregate
|
997
|
+
)
|
970
998
|
end
|
971
999
|
end
|
972
1000
|
end
|
@@ -1044,13 +1072,13 @@ module BaseTest
|
|
1044
1072
|
current_time)
|
1045
1073
|
end
|
1046
1074
|
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1075
|
+
december29 = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
|
1076
|
+
december31 = Time.new(2019, 12, 31, 10, 23, 35, '-08:00')
|
1077
|
+
january1 = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1050
1078
|
|
1051
1079
|
{
|
1052
1080
|
# December 29, 2019 (normal operation).
|
1053
|
-
|
1081
|
+
december29 => begin
|
1054
1082
|
one_day_later = Time.new(2019, 12, 30, 10, 23, 35, '-08:00')
|
1055
1083
|
one_day_a_year_earlier = Time.new(2018, 12, 30, 10, 23, 35, '-08:00')
|
1056
1084
|
just_under_one_day_later = Time.new(2019, 12, 30, 10, 23, 34, '-08:00')
|
@@ -1064,7 +1092,7 @@ module BaseTest
|
|
1064
1092
|
{
|
1065
1093
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1066
1094
|
Time.at(0) => Time.at(0),
|
1067
|
-
|
1095
|
+
december29 => december29,
|
1068
1096
|
one_day_later => one_day_a_year_earlier,
|
1069
1097
|
just_under_one_day_later => just_under_one_day_later,
|
1070
1098
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1074,7 +1102,7 @@ module BaseTest
|
|
1074
1102
|
}
|
1075
1103
|
end,
|
1076
1104
|
# January 1, 2020 (normal operation).
|
1077
|
-
|
1105
|
+
january1 => begin
|
1078
1106
|
one_day_later = Time.new(2020, 1, 2, 10, 23, 35, '-08:00')
|
1079
1107
|
one_day_a_year_earlier = Time.new(2019, 1, 2, 10, 23, 35, '-08:00')
|
1080
1108
|
just_under_one_day_later = Time.new(2020, 1, 2, 10, 23, 34, '-08:00')
|
@@ -1088,7 +1116,7 @@ module BaseTest
|
|
1088
1116
|
{
|
1089
1117
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1090
1118
|
Time.at(0) => Time.at(0),
|
1091
|
-
|
1119
|
+
january1 => january1,
|
1092
1120
|
one_day_later => one_day_a_year_earlier,
|
1093
1121
|
just_under_one_day_later => just_under_one_day_later,
|
1094
1122
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1098,7 +1126,7 @@ module BaseTest
|
|
1098
1126
|
}
|
1099
1127
|
end,
|
1100
1128
|
# December 31, 2019 (next day overlaps new year).
|
1101
|
-
|
1129
|
+
december31 => begin
|
1102
1130
|
one_day_later = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1103
1131
|
just_under_one_day_later = Time.new(2020, 1, 1, 10, 23, 34, '-08:00')
|
1104
1132
|
next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
|
@@ -1109,7 +1137,7 @@ module BaseTest
|
|
1109
1137
|
{
|
1110
1138
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1111
1139
|
Time.at(0) => Time.at(0),
|
1112
|
-
|
1140
|
+
december31 => december31,
|
1113
1141
|
one_day_later => Time.at(0), # Falls into the next year.
|
1114
1142
|
just_under_one_day_later => just_under_one_day_later,
|
1115
1143
|
one_second_before_next_year => one_second_before_next_year,
|
@@ -1607,7 +1635,8 @@ module BaseTest
|
|
1607
1635
|
custom_key: 'custom_labels_key',
|
1608
1636
|
custom_key_config: CONFIG_CUSTOM_LABELS_KEY_SPECIFIED,
|
1609
1637
|
sample_value: COMPUTE_PARAMS[:labels].merge(
|
1610
|
-
LABELS_MESSAGE
|
1638
|
+
LABELS_MESSAGE
|
1639
|
+
),
|
1611
1640
|
default_value: COMPUTE_PARAMS[:labels])
|
1612
1641
|
end
|
1613
1642
|
|
@@ -1659,7 +1688,8 @@ module BaseTest
|
|
1659
1688
|
verify_cascading_json_detection_with_log_entry_fields(
|
1660
1689
|
'insertId', DEFAULT_INSERT_ID_KEY,
|
1661
1690
|
root_level_value: INSERT_ID,
|
1662
|
-
nested_level_value: INSERT_ID2
|
1691
|
+
nested_level_value: INSERT_ID2
|
1692
|
+
)
|
1663
1693
|
end
|
1664
1694
|
|
1665
1695
|
def test_cascading_json_detection_with_log_entry_labels_field
|
@@ -1669,7 +1699,9 @@ module BaseTest
|
|
1669
1699
|
nested_level_value: LABELS_MESSAGE2,
|
1670
1700
|
expected_value_from_root: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE),
|
1671
1701
|
expected_value_from_nested: COMPUTE_PARAMS[:labels].merge(
|
1672
|
-
LABELS_MESSAGE2
|
1702
|
+
LABELS_MESSAGE2
|
1703
|
+
)
|
1704
|
+
)
|
1673
1705
|
end
|
1674
1706
|
|
1675
1707
|
def test_cascading_json_detection_with_log_entry_operation_field
|
@@ -1677,28 +1709,32 @@ module BaseTest
|
|
1677
1709
|
'operation', DEFAULT_OPERATION_KEY,
|
1678
1710
|
root_level_value: OPERATION_MESSAGE,
|
1679
1711
|
nested_level_value: OPERATION_MESSAGE2,
|
1680
|
-
expected_value_from_nested: expected_operation_message2
|
1712
|
+
expected_value_from_nested: expected_operation_message2
|
1713
|
+
)
|
1681
1714
|
end
|
1682
1715
|
|
1683
1716
|
def test_cascading_json_detection_with_log_entry_source_location_field
|
1684
1717
|
verify_cascading_json_detection_with_log_entry_fields(
|
1685
1718
|
'sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
|
1686
1719
|
root_level_value: SOURCE_LOCATION_MESSAGE,
|
1687
|
-
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1720
|
+
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1721
|
+
)
|
1688
1722
|
end
|
1689
1723
|
|
1690
1724
|
def test_cascading_json_detection_with_log_entry_span_id_field
|
1691
1725
|
verify_cascading_json_detection_with_log_entry_fields(
|
1692
1726
|
'spanId', DEFAULT_SPAN_ID_KEY,
|
1693
1727
|
root_level_value: SPAN_ID,
|
1694
|
-
nested_level_value: SPAN_ID2
|
1728
|
+
nested_level_value: SPAN_ID2
|
1729
|
+
)
|
1695
1730
|
end
|
1696
1731
|
|
1697
1732
|
def test_cascading_json_detection_with_log_entry_trace_field
|
1698
1733
|
verify_cascading_json_detection_with_log_entry_fields(
|
1699
1734
|
'trace', DEFAULT_TRACE_KEY,
|
1700
1735
|
root_level_value: TRACE,
|
1701
|
-
nested_level_value: TRACE2
|
1736
|
+
nested_level_value: TRACE2
|
1737
|
+
)
|
1702
1738
|
end
|
1703
1739
|
|
1704
1740
|
def test_cascading_json_detection_with_log_entry_trace_sampled_field
|
@@ -1707,7 +1743,8 @@ module BaseTest
|
|
1707
1743
|
root_level_value: TRACE_SAMPLED,
|
1708
1744
|
nested_level_value: TRACE_SAMPLED2,
|
1709
1745
|
default_value_from_root: false,
|
1710
|
-
default_value_from_nested: false
|
1746
|
+
default_value_from_nested: false
|
1747
|
+
)
|
1711
1748
|
end
|
1712
1749
|
|
1713
1750
|
# Verify that labels present in multiple inputs respect the expected priority
|
@@ -1739,9 +1776,11 @@ module BaseTest
|
|
1739
1776
|
{
|
1740
1777
|
config: CONFIG_LABLES_AND_LABLE_MAP,
|
1741
1778
|
emitted_log: PAYLOAD_FOR_LABEL_MAP.merge(
|
1742
|
-
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1779
|
+
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1780
|
+
),
|
1743
1781
|
expected_labels: LABELS_MESSAGE.merge(LABELS_FROM_LABELS_CONFIG).merge(
|
1744
|
-
LABELS_FROM_LABEL_MAP_CONFIG
|
1782
|
+
LABELS_FROM_LABEL_MAP_CONFIG
|
1783
|
+
)
|
1745
1784
|
},
|
1746
1785
|
# labels from the config "labels" and "label_map" conflict.
|
1747
1786
|
{
|
@@ -1761,14 +1800,16 @@ module BaseTest
|
|
1761
1800
|
{
|
1762
1801
|
config: CONFIG_LABEL_MAP_CONFLICTING,
|
1763
1802
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1764
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1803
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1804
|
+
),
|
1765
1805
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1766
1806
|
},
|
1767
1807
|
# All three types of labels conflict.
|
1768
1808
|
{
|
1769
1809
|
config: CONFIG_LABLES_AND_LABLE_MAP_CONFLICTING,
|
1770
1810
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1771
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1811
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1812
|
+
),
|
1772
1813
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1773
1814
|
}
|
1774
1815
|
].each do |test_params|
|
@@ -1780,7 +1821,8 @@ module BaseTest
|
|
1780
1821
|
d.run
|
1781
1822
|
end
|
1782
1823
|
expected_params = COMPUTE_PARAMS.merge(
|
1783
|
-
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1824
|
+
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1825
|
+
)
|
1784
1826
|
verify_log_entries(1, expected_params)
|
1785
1827
|
end
|
1786
1828
|
end
|
@@ -1852,7 +1894,8 @@ module BaseTest
|
|
1852
1894
|
config: APPLICATION_DEFAULT_CONFIG,
|
1853
1895
|
setup_k8s_stub: true,
|
1854
1896
|
log_entry: k8s_container_log_entry(
|
1855
|
-
log_entry(0)
|
1897
|
+
log_entry(0)
|
1898
|
+
).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
1856
1899
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1857
1900
|
},
|
1858
1901
|
{
|
@@ -1860,7 +1903,8 @@ module BaseTest
|
|
1860
1903
|
setup_k8s_stub: true,
|
1861
1904
|
log_entry: k8s_container_log_entry(
|
1862
1905
|
log_entry(0),
|
1863
|
-
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1906
|
+
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1907
|
+
),
|
1864
1908
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1865
1909
|
}
|
1866
1910
|
].each do |test_params|
|
@@ -1979,7 +2023,8 @@ module BaseTest
|
|
1979
2023
|
assert_metric_value.call(
|
1980
2024
|
:uptime, expected, 'agent.googleapis.com/agent',
|
1981
2025
|
OpenCensus::Stats::Aggregation::Sum, d,
|
1982
|
-
version: Fluent::GoogleCloudOutput.version_string
|
2026
|
+
version: Fluent::GoogleCloudOutput.version_string
|
2027
|
+
)
|
1983
2028
|
rescue Test::Unit::AssertionFailedError
|
1984
2029
|
retry if (retries += 1) < 3
|
1985
2030
|
end
|
@@ -2013,12 +2058,12 @@ module BaseTest
|
|
2013
2058
|
(1..entry_count).each do |entry_index|
|
2014
2059
|
d.emit('message' => log_entry(entry_index.to_s))
|
2015
2060
|
end
|
2016
|
-
# rubocop:disable Lint/
|
2061
|
+
# rubocop:disable Lint/SuppressedException
|
2017
2062
|
begin
|
2018
2063
|
d.run
|
2019
2064
|
rescue mock_error_type
|
2020
2065
|
end
|
2021
|
-
# rubocop:enable Lint/
|
2066
|
+
# rubocop:enable Lint/SuppressedException
|
2022
2067
|
failed_requests_count, dropped_entries_count,
|
2023
2068
|
retried_entries_count = metric_values
|
2024
2069
|
|
@@ -2067,6 +2112,7 @@ module BaseTest
|
|
2067
2112
|
# assertion will fail in the case when a single metric contains time
|
2068
2113
|
# series with success and failure events.
|
2069
2114
|
next if code == ok_status_code
|
2115
|
+
|
2070
2116
|
assert_metric_value.call(:stackdriver_failed_requests_count,
|
2071
2117
|
failed_requests_count,
|
2072
2118
|
'agent.googleapis.com/agent',
|
@@ -2100,7 +2146,8 @@ module BaseTest
|
|
2100
2146
|
end
|
2101
2147
|
|
2102
2148
|
def container_log_entry_with_metadata(
|
2103
|
-
|
2149
|
+
log, container_name = K8S_CONTAINER_NAME
|
2150
|
+
)
|
2104
2151
|
{
|
2105
2152
|
log: log,
|
2106
2153
|
stream: K8S_STREAM,
|
@@ -2159,10 +2206,10 @@ module BaseTest
|
|
2159
2206
|
}
|
2160
2207
|
end
|
2161
2208
|
|
2162
|
-
def dataflow_log_entry(
|
2209
|
+
def dataflow_log_entry(index)
|
2163
2210
|
{
|
2164
2211
|
step: DATAFLOW_STEP_ID,
|
2165
|
-
message: log_entry(
|
2212
|
+
message: log_entry(index)
|
2166
2213
|
}
|
2167
2214
|
end
|
2168
2215
|
|
@@ -2175,10 +2222,10 @@ module BaseTest
|
|
2175
2222
|
}
|
2176
2223
|
end
|
2177
2224
|
|
2178
|
-
def ml_log_entry(
|
2225
|
+
def ml_log_entry(index)
|
2179
2226
|
{
|
2180
2227
|
name: ML_LOG_AREA,
|
2181
|
-
message: log_entry(
|
2228
|
+
message: log_entry(index)
|
2182
2229
|
}
|
2183
2230
|
end
|
2184
2231
|
|
@@ -2189,8 +2236,8 @@ module BaseTest
|
|
2189
2236
|
}
|
2190
2237
|
end
|
2191
2238
|
|
2192
|
-
def log_entry(
|
2193
|
-
"test log entry #{
|
2239
|
+
def log_entry(index)
|
2240
|
+
"test log entry #{index}"
|
2194
2241
|
end
|
2195
2242
|
|
2196
2243
|
# If check_exact_labels is true, assert 'labels' and 'expected_labels' match
|
@@ -2198,6 +2245,7 @@ module BaseTest
|
|
2198
2245
|
# 'expected_labels'.
|
2199
2246
|
def check_labels(expected_labels, labels, check_exact_labels = true)
|
2200
2247
|
return if expected_labels.empty? && labels.empty?
|
2248
|
+
|
2201
2249
|
expected_labels.each do |expected_key, expected_value|
|
2202
2250
|
assert labels.key?(expected_key), "Expected label #{expected_key} not" \
|
2203
2251
|
" found. Got labels: #{labels}."
|
@@ -2207,20 +2255,21 @@ module BaseTest
|
|
2207
2255
|
assert_equal expected_value, actual_value, "Value for #{expected_key}" \
|
2208
2256
|
" mismatch. Expected #{expected_value}. Got #{actual_value}"
|
2209
2257
|
end
|
2210
|
-
|
2211
|
-
|
2212
|
-
|
2213
|
-
|
2214
|
-
|
2258
|
+
return unless check_exact_labels
|
2259
|
+
|
2260
|
+
assert_equal expected_labels.length, labels.length, 'Expected ' \
|
2261
|
+
"#{expected_labels.length} labels: #{expected_labels}, got " \
|
2262
|
+
"#{labels.length} labels: #{labels}"
|
2215
2263
|
end
|
2216
2264
|
|
2217
|
-
def verify_default_log_entry_text(text,
|
2218
|
-
assert_equal "test log entry #{
|
2219
|
-
"Entry ##{
|
2265
|
+
def verify_default_log_entry_text(text, index, entry)
|
2266
|
+
assert_equal "test log entry #{index}", text,
|
2267
|
+
"Entry ##{index} had unexpected text: #{entry}"
|
2220
2268
|
end
|
2221
2269
|
|
2222
2270
|
# The caller can optionally provide a block which is called for each entry.
|
2223
|
-
def verify_json_log_entries(
|
2271
|
+
def verify_json_log_entries(expected_count, params,
|
2272
|
+
payload_type = 'textPayload',
|
2224
2273
|
check_exact_entry_labels = true)
|
2225
2274
|
entry_count = 0
|
2226
2275
|
@logs_sent.each do |request|
|
@@ -2258,11 +2307,11 @@ module BaseTest
|
|
2258
2307
|
entry)
|
2259
2308
|
end
|
2260
2309
|
entry_count += 1
|
2261
|
-
assert entry_count <=
|
2262
|
-
"Number of entries #{entry_count} exceeds expected number #{
|
2310
|
+
assert entry_count <= expected_count,
|
2311
|
+
"Number of entries #{entry_count} exceeds expected number #{expected_count}."
|
2263
2312
|
end
|
2264
2313
|
end
|
2265
|
-
assert_equal
|
2314
|
+
assert_equal expected_count, entry_count
|
2266
2315
|
end
|
2267
2316
|
|
2268
2317
|
def verify_container_logs(log_entry_factory, expected_params)
|
@@ -2291,13 +2340,17 @@ module BaseTest
|
|
2291
2340
|
# LogEntry info from. The values are lists of two elements: the name of
|
2292
2341
|
# the subfield in LogEntry object and the expected value of that field.
|
2293
2342
|
DEFAULT_HTTP_REQUEST_KEY => [
|
2294
|
-
'httpRequest', HTTP_REQUEST_MESSAGE
|
2343
|
+
'httpRequest', HTTP_REQUEST_MESSAGE
|
2344
|
+
],
|
2295
2345
|
DEFAULT_LABELS_KEY => [
|
2296
|
-
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2346
|
+
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2347
|
+
],
|
2297
2348
|
DEFAULT_OPERATION_KEY => [
|
2298
|
-
'operation', OPERATION_MESSAGE
|
2349
|
+
'operation', OPERATION_MESSAGE
|
2350
|
+
],
|
2299
2351
|
DEFAULT_SOURCE_LOCATION_KEY => [
|
2300
|
-
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2352
|
+
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2353
|
+
]
|
2301
2354
|
}
|
2302
2355
|
end
|
2303
2356
|
|
@@ -2397,17 +2450,22 @@ module BaseTest
|
|
2397
2450
|
# left with name "log", "message" or "msg". This test verifies additional
|
2398
2451
|
# LogEntry fields like spanId and traceId do not disable that by accident.
|
2399
2452
|
def verify_cascading_json_detection_with_log_entry_fields(
|
2400
|
-
|
2453
|
+
log_entry_field, default_key, expectation
|
2454
|
+
)
|
2401
2455
|
root_level_value = expectation[:root_level_value]
|
2402
2456
|
nested_level_value = expectation[:nested_level_value]
|
2403
2457
|
expected_value_from_root = expectation.fetch(
|
2404
|
-
:expected_value_from_root, root_level_value
|
2458
|
+
:expected_value_from_root, root_level_value
|
2459
|
+
)
|
2405
2460
|
expected_value_from_nested = expectation.fetch(
|
2406
|
-
:expected_value_from_nested, nested_level_value
|
2461
|
+
:expected_value_from_nested, nested_level_value
|
2462
|
+
)
|
2407
2463
|
default_value_from_root = expectation.fetch(
|
2408
|
-
:default_value_from_root, nil
|
2464
|
+
:default_value_from_root, nil
|
2465
|
+
)
|
2409
2466
|
default_value_from_nested = expectation.fetch(
|
2410
|
-
:default_value_from_nested, nil
|
2467
|
+
:default_value_from_nested, nil
|
2468
|
+
)
|
2411
2469
|
|
2412
2470
|
setup_gce_metadata_stubs
|
2413
2471
|
|
@@ -2443,7 +2501,8 @@ module BaseTest
|
|
2443
2501
|
# }
|
2444
2502
|
# }
|
2445
2503
|
log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
|
2446
|
-
default_key => root_level_value
|
2504
|
+
default_key => root_level_value
|
2505
|
+
)
|
2447
2506
|
|
2448
2507
|
[
|
2449
2508
|
[
|
@@ -2594,7 +2653,8 @@ module BaseTest
|
|
2594
2653
|
|
2595
2654
|
# Verify the number and the content of the log entries match the expectation.
|
2596
2655
|
# The caller can optionally provide a block which is called for each entry.
|
2597
|
-
def verify_log_entries(
|
2656
|
+
def verify_log_entries(_expected_count, _params,
|
2657
|
+
_payload_type = 'textPayload',
|
2598
2658
|
_check_exact_entry_labels = true, &_block)
|
2599
2659
|
_undefined
|
2600
2660
|
end
|