fluent-plugin-google-cloud 0.12.10 → 0.12.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +26 -19
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +9 -9
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -11
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +222 -166
- data/lib/fluent/plugin/statusz.rb +7 -9
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +194 -134
- data/test/plugin/constants.rb +18 -14
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +33 -21
- data/test/plugin/test_out_google_cloud_grpc.rb +35 -22
- data/test/plugin/utils.rb +8 -6
- metadata +29 -29
data/test/plugin/base_test.rb
CHANGED
@@ -71,8 +71,8 @@ module BaseTest
|
|
71
71
|
exception_count = 0
|
72
72
|
begin
|
73
73
|
create_driver(PRIVATE_KEY_CONFIG)
|
74
|
-
rescue Fluent::ConfigError =>
|
75
|
-
assert
|
74
|
+
rescue Fluent::ConfigError => e
|
75
|
+
assert e.message.include? 'Please remove configuration parameters'
|
76
76
|
exception_count += 1
|
77
77
|
end
|
78
78
|
assert_equal 1, exception_count
|
@@ -100,23 +100,22 @@ module BaseTest
|
|
100
100
|
def test_configure_metadata_missing_parts_on_other_platforms
|
101
101
|
setup_no_metadata_service_stubs
|
102
102
|
Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
|
103
|
-
[
|
104
|
-
|
105
|
-
|
106
|
-
|
103
|
+
[
|
104
|
+
[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
|
105
|
+
[CONFIG_MISSING_METADATA_ZONE, [], true],
|
106
|
+
[CONFIG_MISSING_METADATA_VM_ID, [], true],
|
107
|
+
[CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
|
107
108
|
].each_with_index do |(config, missing_parts, is_valid_config), index|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
assert error.message.include?(part), "Index #{index} failed."
|
119
|
-
end
|
109
|
+
create_driver(config)
|
110
|
+
assert_true is_valid_config, "Invalid config at index #{index} should "\
|
111
|
+
'have raised an error.'
|
112
|
+
rescue Fluent::ConfigError => e
|
113
|
+
assert_false is_valid_config, "Valid config at index #{index} should "\
|
114
|
+
"not have raised an error #{e}."
|
115
|
+
assert e.message.include?('Unable to obtain metadata parameters:'),
|
116
|
+
"Index #{index} failed."
|
117
|
+
missing_parts.each do |part|
|
118
|
+
assert e.message.include?(part), "Index #{index} failed."
|
120
119
|
end
|
121
120
|
end
|
122
121
|
end
|
@@ -128,38 +127,52 @@ module BaseTest
|
|
128
127
|
setup_gce_metadata_stubs
|
129
128
|
create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
|
130
129
|
assert_nil(Prometheus::Client.registry.get(
|
131
|
-
:stackdriver_successful_requests_count
|
130
|
+
:stackdriver_successful_requests_count
|
131
|
+
))
|
132
132
|
assert_nil(Prometheus::Client.registry.get(
|
133
|
-
:stackdriver_failed_requests_count
|
133
|
+
:stackdriver_failed_requests_count
|
134
|
+
))
|
134
135
|
assert_nil(Prometheus::Client.registry.get(
|
135
|
-
:stackdriver_ingested_entries_count
|
136
|
+
:stackdriver_ingested_entries_count
|
137
|
+
))
|
136
138
|
assert_nil(Prometheus::Client.registry.get(
|
137
|
-
:stackdriver_dropped_entries_count
|
139
|
+
:stackdriver_dropped_entries_count
|
140
|
+
))
|
138
141
|
assert_nil(Prometheus::Client.registry.get(
|
139
|
-
:stackdriver_retried_entries_count
|
142
|
+
:stackdriver_retried_entries_count
|
143
|
+
))
|
140
144
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
141
145
|
Monitoring::MetricTranslator.new(
|
142
|
-
:stackdriver_successful_requests_count, {}
|
146
|
+
:stackdriver_successful_requests_count, {}
|
147
|
+
)
|
148
|
+
))
|
143
149
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
144
150
|
Monitoring::MetricTranslator.new(
|
145
|
-
:stackdriver_failed_requests_count, {}
|
151
|
+
:stackdriver_failed_requests_count, {}
|
152
|
+
)
|
153
|
+
))
|
146
154
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
147
155
|
Monitoring::MetricTranslator.new(
|
148
|
-
:stackdriver_ingested_entries_count, {}
|
156
|
+
:stackdriver_ingested_entries_count, {}
|
157
|
+
)
|
158
|
+
))
|
149
159
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
150
160
|
Monitoring::MetricTranslator.new(
|
151
|
-
:stackdriver_dropped_entries_count, {}
|
161
|
+
:stackdriver_dropped_entries_count, {}
|
162
|
+
)
|
163
|
+
))
|
152
164
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
153
165
|
Monitoring::MetricTranslator.new(
|
154
|
-
:stackdriver_retried_entries_count, {}
|
166
|
+
:stackdriver_retried_entries_count, {}
|
167
|
+
)
|
168
|
+
))
|
155
169
|
end
|
156
170
|
|
157
171
|
def test_configure_uses_metrics_resource
|
158
172
|
setup_gce_metadata_stubs
|
159
173
|
[CONFIG_METRICS_RESOURCE_JSON,
|
160
174
|
CONFIG_METRICS_RESOURCE_HASH,
|
161
|
-
CONFIG_METRICS_RESOURCE_JSON_HASH
|
162
|
-
].each_with_index do |config, index|
|
175
|
+
CONFIG_METRICS_RESOURCE_JSON_HASH].each_with_index do |config, index|
|
163
176
|
d = create_driver(config)
|
164
177
|
assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
|
165
178
|
"Index #{index}"
|
@@ -171,7 +184,8 @@ module BaseTest
|
|
171
184
|
registry = d.instance.instance_variable_get(:@registry)
|
172
185
|
assert_not_nil registry
|
173
186
|
monitored_resource = registry.instance_variable_get(
|
174
|
-
:@metrics_monitored_resource
|
187
|
+
:@metrics_monitored_resource
|
188
|
+
)
|
175
189
|
assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
|
176
190
|
assert_equal({ 'label1' => '123', 'label2' => 'abc' },
|
177
191
|
monitored_resource.labels, "Index #{index}")
|
@@ -190,14 +204,12 @@ module BaseTest
|
|
190
204
|
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
|
191
205
|
/unrecognized keys: \[:random\]/
|
192
206
|
}.each_with_index do |(config, pattern), index|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
"Index #{index} failed: got #{error.message}."
|
200
|
-
end
|
207
|
+
create_driver(config)
|
208
|
+
assert false,
|
209
|
+
"Invalid config at index #{index} should have raised an error."
|
210
|
+
rescue Fluent::ConfigError => e
|
211
|
+
assert e.message.match?(pattern), \
|
212
|
+
"Index #{index} failed: got #{e.message}."
|
201
213
|
end
|
202
214
|
end
|
203
215
|
|
@@ -277,16 +289,16 @@ module BaseTest
|
|
277
289
|
exception_count = 0
|
278
290
|
begin
|
279
291
|
create_driver
|
280
|
-
rescue Fluent::ConfigError =>
|
281
|
-
assert
|
282
|
-
assert
|
292
|
+
rescue Fluent::ConfigError => e
|
293
|
+
assert e.message.include? 'Unable to obtain metadata parameters:'
|
294
|
+
assert e.message.include? 'project_id'
|
283
295
|
exception_count += 1
|
284
296
|
end
|
285
297
|
assert_equal 1, exception_count
|
286
298
|
end
|
287
299
|
|
288
300
|
def test_project_id_from_credentials
|
289
|
-
%w
|
301
|
+
%w[gce ec2].each do |platform|
|
290
302
|
send("setup_#{platform}_metadata_stubs")
|
291
303
|
[IAM_CREDENTIALS, NEW_STYLE_CREDENTIALS, LEGACY_CREDENTIALS].each \
|
292
304
|
do |creds|
|
@@ -317,18 +329,19 @@ module BaseTest
|
|
317
329
|
d.run
|
318
330
|
end
|
319
331
|
verify_log_entries(1, COMPUTE_PARAMS.merge(
|
320
|
-
project_id: IAM_CREDENTIALS[:project_id]
|
332
|
+
project_id: IAM_CREDENTIALS[:project_id]
|
333
|
+
))
|
321
334
|
end
|
322
335
|
|
323
336
|
def test_invalid_json_credentials
|
324
|
-
%w
|
337
|
+
%w[gce_metadata ec2_metadata no_metadata_service].each do |platform|
|
325
338
|
send("setup_#{platform}_stubs")
|
326
339
|
exception_count = 0
|
327
340
|
ENV[CREDENTIALS_PATH_ENV_VAR] = INVALID_CREDENTIALS[:path]
|
328
341
|
begin
|
329
342
|
create_driver
|
330
|
-
rescue RuntimeError =>
|
331
|
-
assert
|
343
|
+
rescue RuntimeError => e
|
344
|
+
assert e.message.include? 'Unable to read the credential file'
|
332
345
|
exception_count += 1
|
333
346
|
end
|
334
347
|
assert_equal 1, exception_count
|
@@ -498,9 +511,9 @@ module BaseTest
|
|
498
511
|
d.emit(
|
499
512
|
'int_key' => { 1 => message },
|
500
513
|
'int_array_key' => { [1, 2, 3, 4] => message },
|
501
|
-
'string_array_key' => { %w
|
514
|
+
'string_array_key' => { %w[a b c] => message },
|
502
515
|
'hash_key' => { { 'some_key' => 'some_value' } => message },
|
503
|
-
'mixed_key' => { { 'some_key' => %w
|
516
|
+
'mixed_key' => { { 'some_key' => %w[a b c] } => message },
|
504
517
|
'symbol_key' => { some_symbol: message },
|
505
518
|
'nil_key' => { nil => message }
|
506
519
|
)
|
@@ -527,9 +540,9 @@ module BaseTest
|
|
527
540
|
'"data": 5000, "some_null_field": null}'
|
528
541
|
setup_logging_stubs do
|
529
542
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
530
|
-
d.emit('message' =>
|
543
|
+
d.emit('message' => "notJSON #{json_string}")
|
531
544
|
d.emit('message' => json_string)
|
532
|
-
d.emit('message' => " \r\n \t"
|
545
|
+
d.emit('message' => " \r\n \t#{json_string}")
|
533
546
|
d.run
|
534
547
|
end
|
535
548
|
verify_log_entries(3, COMPUTE_PARAMS, 'textPayload') do
|
@@ -543,10 +556,10 @@ module BaseTest
|
|
543
556
|
'"data": 5000, "some_null_field": null}'
|
544
557
|
setup_logging_stubs do
|
545
558
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
546
|
-
%w
|
547
|
-
d.emit(field =>
|
559
|
+
%w[log msg].each do |field|
|
560
|
+
d.emit(field => "notJSON #{json_string}")
|
548
561
|
d.emit(field => json_string)
|
549
|
-
d.emit(field => " \r\n \t"
|
562
|
+
d.emit(field => " \r\n \t#{json_string}")
|
550
563
|
end
|
551
564
|
d.run
|
552
565
|
end
|
@@ -564,7 +577,7 @@ module BaseTest
|
|
564
577
|
'"data": 5000, "some_null_field": null}'
|
565
578
|
setup_logging_stubs do
|
566
579
|
d = create_driver(DETECT_JSON_CONFIG)
|
567
|
-
d.emit('message' =>
|
580
|
+
d.emit('message' => "notJSON #{json_string}")
|
568
581
|
d.run
|
569
582
|
end
|
570
583
|
verify_log_entries(1, COMPUTE_PARAMS, 'textPayload') do
|
@@ -578,8 +591,8 @@ module BaseTest
|
|
578
591
|
'"data": 5000, "some_null_field": null}'
|
579
592
|
setup_logging_stubs do
|
580
593
|
d = create_driver(DETECT_JSON_CONFIG)
|
581
|
-
%w
|
582
|
-
d.emit(field =>
|
594
|
+
%w[log msg].each do |field|
|
595
|
+
d.emit(field => "notJSON #{json_string}")
|
583
596
|
end
|
584
597
|
d.run
|
585
598
|
end
|
@@ -668,9 +681,9 @@ module BaseTest
|
|
668
681
|
'"data": 5000, "some_null_field": null}'
|
669
682
|
setup_logging_stubs do
|
670
683
|
d = create_driver(DETECT_JSON_CONFIG)
|
671
|
-
%w
|
684
|
+
%w[message log msg].each do |field|
|
672
685
|
d.emit(field => json_string)
|
673
|
-
d.emit(field => " \r\n \t"
|
686
|
+
d.emit(field => " \r\n \t#{json_string}")
|
674
687
|
end
|
675
688
|
d.run
|
676
689
|
end
|
@@ -691,9 +704,9 @@ module BaseTest
|
|
691
704
|
'"data": 5000, "some_null_field": null}'
|
692
705
|
setup_logging_stubs do
|
693
706
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
694
|
-
d.emit(container_log_entry_with_metadata(
|
707
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
695
708
|
d.emit(container_log_entry_with_metadata(json_string))
|
696
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
709
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
697
710
|
d.run
|
698
711
|
end
|
699
712
|
verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -708,7 +721,7 @@ module BaseTest
|
|
708
721
|
'"data": 5000, "some_null_field": null}'
|
709
722
|
setup_logging_stubs do
|
710
723
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
711
|
-
d.emit(container_log_entry_with_metadata(
|
724
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
712
725
|
d.run
|
713
726
|
end
|
714
727
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -724,7 +737,7 @@ module BaseTest
|
|
724
737
|
setup_logging_stubs do
|
725
738
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
726
739
|
d.emit(container_log_entry_with_metadata(json_string))
|
727
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
740
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
728
741
|
d.run
|
729
742
|
end
|
730
743
|
verify_log_entries(2, CONTAINER_FROM_METADATA_PARAMS, 'jsonPayload') \
|
@@ -749,14 +762,16 @@ module BaseTest
|
|
749
762
|
setup_logging_stubs do
|
750
763
|
@logs_sent = []
|
751
764
|
d = create_driver(DETECT_JSON_CONFIG)
|
752
|
-
%w
|
765
|
+
%w[message log msg].each do |field|
|
753
766
|
d.emit(PRESERVED_KEYS_MAP.merge(
|
754
|
-
field => json_string
|
767
|
+
field => json_string
|
768
|
+
).merge(timestamp_fields))
|
755
769
|
end
|
756
770
|
d.run
|
757
771
|
end
|
758
772
|
expected_params = COMPUTE_PARAMS.merge(
|
759
|
-
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
773
|
+
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
774
|
+
)
|
760
775
|
verify_log_entries(3, expected_params, 'jsonPayload') do |entry|
|
761
776
|
fields = entry['jsonPayload']
|
762
777
|
assert_equal 4, fields.size, entry
|
@@ -772,7 +787,7 @@ module BaseTest
|
|
772
787
|
# any non-string tags or tags with non-utf8 characters are detected.
|
773
788
|
def test_reject_invalid_tags_with_require_valid_tags_true
|
774
789
|
setup_gce_metadata_stubs
|
775
|
-
INVALID_TAGS.
|
790
|
+
INVALID_TAGS.each_key do |tag|
|
776
791
|
setup_logging_stubs do
|
777
792
|
@logs_sent = []
|
778
793
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
@@ -800,8 +815,11 @@ module BaseTest
|
|
800
815
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
801
816
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
802
817
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
803
|
-
'container_name' => container_name
|
804
|
-
|
818
|
+
'container_name' => container_name
|
819
|
+
)
|
820
|
+
),
|
821
|
+
log_name: tag
|
822
|
+
)
|
805
823
|
verify_log_entries(1, params, 'textPayload')
|
806
824
|
end
|
807
825
|
|
@@ -824,8 +842,10 @@ module BaseTest
|
|
824
842
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
825
843
|
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
826
844
|
"#{GKE_CONSTANTS[:service]}/container_name" =>
|
827
|
-
CGI.unescape(encoded_name)
|
828
|
-
|
845
|
+
CGI.unescape(encoded_name)
|
846
|
+
),
|
847
|
+
log_name: encoded_name
|
848
|
+
)
|
829
849
|
verify_log_entries(0, params, 'textPayload')
|
830
850
|
end
|
831
851
|
end
|
@@ -861,8 +881,11 @@ module BaseTest
|
|
861
881
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
862
882
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
863
883
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
864
|
-
'container_name' => tag
|
865
|
-
|
884
|
+
'container_name' => tag
|
885
|
+
)
|
886
|
+
),
|
887
|
+
log_name: encoded_tag
|
888
|
+
)
|
866
889
|
verify_log_entries(1, params, 'textPayload')
|
867
890
|
end
|
868
891
|
end
|
@@ -907,8 +930,11 @@ module BaseTest
|
|
907
930
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
908
931
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
909
932
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
910
|
-
'container_name' => CGI.unescape(encoded_container_name)
|
911
|
-
|
933
|
+
'container_name' => CGI.unescape(encoded_container_name)
|
934
|
+
)
|
935
|
+
),
|
936
|
+
log_name: encoded_container_name
|
937
|
+
)
|
912
938
|
verify_log_entries(1, params, 'textPayload')
|
913
939
|
end
|
914
940
|
end
|
@@ -960,13 +986,15 @@ module BaseTest
|
|
960
986
|
request_count,
|
961
987
|
'agent.googleapis.com/agent',
|
962
988
|
OpenCensus::Stats::Aggregation::Sum, d,
|
963
|
-
:aggregate
|
989
|
+
:aggregate
|
990
|
+
)
|
964
991
|
assert_prometheus_metric_value(
|
965
992
|
:stackdriver_ingested_entries_count,
|
966
993
|
log_entry_count,
|
967
994
|
'agent.googleapis.com/agent',
|
968
995
|
OpenCensus::Stats::Aggregation::Sum, d,
|
969
|
-
:aggregate
|
996
|
+
:aggregate
|
997
|
+
)
|
970
998
|
end
|
971
999
|
end
|
972
1000
|
end
|
@@ -1044,13 +1072,13 @@ module BaseTest
|
|
1044
1072
|
current_time)
|
1045
1073
|
end
|
1046
1074
|
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1075
|
+
december29 = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
|
1076
|
+
december31 = Time.new(2019, 12, 31, 10, 23, 35, '-08:00')
|
1077
|
+
january1 = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1050
1078
|
|
1051
1079
|
{
|
1052
1080
|
# December 29, 2019 (normal operation).
|
1053
|
-
|
1081
|
+
december29 => begin
|
1054
1082
|
one_day_later = Time.new(2019, 12, 30, 10, 23, 35, '-08:00')
|
1055
1083
|
one_day_a_year_earlier = Time.new(2018, 12, 30, 10, 23, 35, '-08:00')
|
1056
1084
|
just_under_one_day_later = Time.new(2019, 12, 30, 10, 23, 34, '-08:00')
|
@@ -1064,7 +1092,7 @@ module BaseTest
|
|
1064
1092
|
{
|
1065
1093
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1066
1094
|
Time.at(0) => Time.at(0),
|
1067
|
-
|
1095
|
+
december29 => december29,
|
1068
1096
|
one_day_later => one_day_a_year_earlier,
|
1069
1097
|
just_under_one_day_later => just_under_one_day_later,
|
1070
1098
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1074,7 +1102,7 @@ module BaseTest
|
|
1074
1102
|
}
|
1075
1103
|
end,
|
1076
1104
|
# January 1, 2020 (normal operation).
|
1077
|
-
|
1105
|
+
january1 => begin
|
1078
1106
|
one_day_later = Time.new(2020, 1, 2, 10, 23, 35, '-08:00')
|
1079
1107
|
one_day_a_year_earlier = Time.new(2019, 1, 2, 10, 23, 35, '-08:00')
|
1080
1108
|
just_under_one_day_later = Time.new(2020, 1, 2, 10, 23, 34, '-08:00')
|
@@ -1088,7 +1116,7 @@ module BaseTest
|
|
1088
1116
|
{
|
1089
1117
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1090
1118
|
Time.at(0) => Time.at(0),
|
1091
|
-
|
1119
|
+
january1 => january1,
|
1092
1120
|
one_day_later => one_day_a_year_earlier,
|
1093
1121
|
just_under_one_day_later => just_under_one_day_later,
|
1094
1122
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1098,7 +1126,7 @@ module BaseTest
|
|
1098
1126
|
}
|
1099
1127
|
end,
|
1100
1128
|
# December 31, 2019 (next day overlaps new year).
|
1101
|
-
|
1129
|
+
december31 => begin
|
1102
1130
|
one_day_later = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1103
1131
|
just_under_one_day_later = Time.new(2020, 1, 1, 10, 23, 34, '-08:00')
|
1104
1132
|
next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
|
@@ -1109,7 +1137,7 @@ module BaseTest
|
|
1109
1137
|
{
|
1110
1138
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1111
1139
|
Time.at(0) => Time.at(0),
|
1112
|
-
|
1140
|
+
december31 => december31,
|
1113
1141
|
one_day_later => Time.at(0), # Falls into the next year.
|
1114
1142
|
just_under_one_day_later => just_under_one_day_later,
|
1115
1143
|
one_second_before_next_year => one_second_before_next_year,
|
@@ -1607,7 +1635,8 @@ module BaseTest
|
|
1607
1635
|
custom_key: 'custom_labels_key',
|
1608
1636
|
custom_key_config: CONFIG_CUSTOM_LABELS_KEY_SPECIFIED,
|
1609
1637
|
sample_value: COMPUTE_PARAMS[:labels].merge(
|
1610
|
-
LABELS_MESSAGE
|
1638
|
+
LABELS_MESSAGE
|
1639
|
+
),
|
1611
1640
|
default_value: COMPUTE_PARAMS[:labels])
|
1612
1641
|
end
|
1613
1642
|
|
@@ -1659,7 +1688,8 @@ module BaseTest
|
|
1659
1688
|
verify_cascading_json_detection_with_log_entry_fields(
|
1660
1689
|
'insertId', DEFAULT_INSERT_ID_KEY,
|
1661
1690
|
root_level_value: INSERT_ID,
|
1662
|
-
nested_level_value: INSERT_ID2
|
1691
|
+
nested_level_value: INSERT_ID2
|
1692
|
+
)
|
1663
1693
|
end
|
1664
1694
|
|
1665
1695
|
def test_cascading_json_detection_with_log_entry_labels_field
|
@@ -1669,7 +1699,9 @@ module BaseTest
|
|
1669
1699
|
nested_level_value: LABELS_MESSAGE2,
|
1670
1700
|
expected_value_from_root: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE),
|
1671
1701
|
expected_value_from_nested: COMPUTE_PARAMS[:labels].merge(
|
1672
|
-
LABELS_MESSAGE2
|
1702
|
+
LABELS_MESSAGE2
|
1703
|
+
)
|
1704
|
+
)
|
1673
1705
|
end
|
1674
1706
|
|
1675
1707
|
def test_cascading_json_detection_with_log_entry_operation_field
|
@@ -1677,28 +1709,32 @@ module BaseTest
|
|
1677
1709
|
'operation', DEFAULT_OPERATION_KEY,
|
1678
1710
|
root_level_value: OPERATION_MESSAGE,
|
1679
1711
|
nested_level_value: OPERATION_MESSAGE2,
|
1680
|
-
expected_value_from_nested: expected_operation_message2
|
1712
|
+
expected_value_from_nested: expected_operation_message2
|
1713
|
+
)
|
1681
1714
|
end
|
1682
1715
|
|
1683
1716
|
def test_cascading_json_detection_with_log_entry_source_location_field
|
1684
1717
|
verify_cascading_json_detection_with_log_entry_fields(
|
1685
1718
|
'sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
|
1686
1719
|
root_level_value: SOURCE_LOCATION_MESSAGE,
|
1687
|
-
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1720
|
+
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1721
|
+
)
|
1688
1722
|
end
|
1689
1723
|
|
1690
1724
|
def test_cascading_json_detection_with_log_entry_span_id_field
|
1691
1725
|
verify_cascading_json_detection_with_log_entry_fields(
|
1692
1726
|
'spanId', DEFAULT_SPAN_ID_KEY,
|
1693
1727
|
root_level_value: SPAN_ID,
|
1694
|
-
nested_level_value: SPAN_ID2
|
1728
|
+
nested_level_value: SPAN_ID2
|
1729
|
+
)
|
1695
1730
|
end
|
1696
1731
|
|
1697
1732
|
def test_cascading_json_detection_with_log_entry_trace_field
|
1698
1733
|
verify_cascading_json_detection_with_log_entry_fields(
|
1699
1734
|
'trace', DEFAULT_TRACE_KEY,
|
1700
1735
|
root_level_value: TRACE,
|
1701
|
-
nested_level_value: TRACE2
|
1736
|
+
nested_level_value: TRACE2
|
1737
|
+
)
|
1702
1738
|
end
|
1703
1739
|
|
1704
1740
|
def test_cascading_json_detection_with_log_entry_trace_sampled_field
|
@@ -1707,7 +1743,8 @@ module BaseTest
|
|
1707
1743
|
root_level_value: TRACE_SAMPLED,
|
1708
1744
|
nested_level_value: TRACE_SAMPLED2,
|
1709
1745
|
default_value_from_root: false,
|
1710
|
-
default_value_from_nested: false
|
1746
|
+
default_value_from_nested: false
|
1747
|
+
)
|
1711
1748
|
end
|
1712
1749
|
|
1713
1750
|
# Verify that labels present in multiple inputs respect the expected priority
|
@@ -1739,9 +1776,11 @@ module BaseTest
|
|
1739
1776
|
{
|
1740
1777
|
config: CONFIG_LABLES_AND_LABLE_MAP,
|
1741
1778
|
emitted_log: PAYLOAD_FOR_LABEL_MAP.merge(
|
1742
|
-
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1779
|
+
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1780
|
+
),
|
1743
1781
|
expected_labels: LABELS_MESSAGE.merge(LABELS_FROM_LABELS_CONFIG).merge(
|
1744
|
-
LABELS_FROM_LABEL_MAP_CONFIG
|
1782
|
+
LABELS_FROM_LABEL_MAP_CONFIG
|
1783
|
+
)
|
1745
1784
|
},
|
1746
1785
|
# labels from the config "labels" and "label_map" conflict.
|
1747
1786
|
{
|
@@ -1761,14 +1800,16 @@ module BaseTest
|
|
1761
1800
|
{
|
1762
1801
|
config: CONFIG_LABEL_MAP_CONFLICTING,
|
1763
1802
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1764
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1803
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1804
|
+
),
|
1765
1805
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1766
1806
|
},
|
1767
1807
|
# All three types of labels conflict.
|
1768
1808
|
{
|
1769
1809
|
config: CONFIG_LABLES_AND_LABLE_MAP_CONFLICTING,
|
1770
1810
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1771
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1811
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1812
|
+
),
|
1772
1813
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1773
1814
|
}
|
1774
1815
|
].each do |test_params|
|
@@ -1780,7 +1821,8 @@ module BaseTest
|
|
1780
1821
|
d.run
|
1781
1822
|
end
|
1782
1823
|
expected_params = COMPUTE_PARAMS.merge(
|
1783
|
-
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1824
|
+
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1825
|
+
)
|
1784
1826
|
verify_log_entries(1, expected_params)
|
1785
1827
|
end
|
1786
1828
|
end
|
@@ -1852,7 +1894,8 @@ module BaseTest
|
|
1852
1894
|
config: APPLICATION_DEFAULT_CONFIG,
|
1853
1895
|
setup_k8s_stub: true,
|
1854
1896
|
log_entry: k8s_container_log_entry(
|
1855
|
-
log_entry(0)
|
1897
|
+
log_entry(0)
|
1898
|
+
).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
1856
1899
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1857
1900
|
},
|
1858
1901
|
{
|
@@ -1860,7 +1903,8 @@ module BaseTest
|
|
1860
1903
|
setup_k8s_stub: true,
|
1861
1904
|
log_entry: k8s_container_log_entry(
|
1862
1905
|
log_entry(0),
|
1863
|
-
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1906
|
+
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1907
|
+
),
|
1864
1908
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1865
1909
|
}
|
1866
1910
|
].each do |test_params|
|
@@ -1979,7 +2023,8 @@ module BaseTest
|
|
1979
2023
|
assert_metric_value.call(
|
1980
2024
|
:uptime, expected, 'agent.googleapis.com/agent',
|
1981
2025
|
OpenCensus::Stats::Aggregation::Sum, d,
|
1982
|
-
version: Fluent::GoogleCloudOutput.version_string
|
2026
|
+
version: Fluent::GoogleCloudOutput.version_string
|
2027
|
+
)
|
1983
2028
|
rescue Test::Unit::AssertionFailedError
|
1984
2029
|
retry if (retries += 1) < 3
|
1985
2030
|
end
|
@@ -2013,12 +2058,12 @@ module BaseTest
|
|
2013
2058
|
(1..entry_count).each do |entry_index|
|
2014
2059
|
d.emit('message' => log_entry(entry_index.to_s))
|
2015
2060
|
end
|
2016
|
-
# rubocop:disable Lint/
|
2061
|
+
# rubocop:disable Lint/SuppressedException
|
2017
2062
|
begin
|
2018
2063
|
d.run
|
2019
2064
|
rescue mock_error_type
|
2020
2065
|
end
|
2021
|
-
# rubocop:enable Lint/
|
2066
|
+
# rubocop:enable Lint/SuppressedException
|
2022
2067
|
failed_requests_count, dropped_entries_count,
|
2023
2068
|
retried_entries_count = metric_values
|
2024
2069
|
|
@@ -2067,6 +2112,7 @@ module BaseTest
|
|
2067
2112
|
# assertion will fail in the case when a single metric contains time
|
2068
2113
|
# series with success and failure events.
|
2069
2114
|
next if code == ok_status_code
|
2115
|
+
|
2070
2116
|
assert_metric_value.call(:stackdriver_failed_requests_count,
|
2071
2117
|
failed_requests_count,
|
2072
2118
|
'agent.googleapis.com/agent',
|
@@ -2100,7 +2146,8 @@ module BaseTest
|
|
2100
2146
|
end
|
2101
2147
|
|
2102
2148
|
def container_log_entry_with_metadata(
|
2103
|
-
|
2149
|
+
log, container_name = K8S_CONTAINER_NAME
|
2150
|
+
)
|
2104
2151
|
{
|
2105
2152
|
log: log,
|
2106
2153
|
stream: K8S_STREAM,
|
@@ -2159,10 +2206,10 @@ module BaseTest
|
|
2159
2206
|
}
|
2160
2207
|
end
|
2161
2208
|
|
2162
|
-
def dataflow_log_entry(
|
2209
|
+
def dataflow_log_entry(index)
|
2163
2210
|
{
|
2164
2211
|
step: DATAFLOW_STEP_ID,
|
2165
|
-
message: log_entry(
|
2212
|
+
message: log_entry(index)
|
2166
2213
|
}
|
2167
2214
|
end
|
2168
2215
|
|
@@ -2175,10 +2222,10 @@ module BaseTest
|
|
2175
2222
|
}
|
2176
2223
|
end
|
2177
2224
|
|
2178
|
-
def ml_log_entry(
|
2225
|
+
def ml_log_entry(index)
|
2179
2226
|
{
|
2180
2227
|
name: ML_LOG_AREA,
|
2181
|
-
message: log_entry(
|
2228
|
+
message: log_entry(index)
|
2182
2229
|
}
|
2183
2230
|
end
|
2184
2231
|
|
@@ -2189,8 +2236,8 @@ module BaseTest
|
|
2189
2236
|
}
|
2190
2237
|
end
|
2191
2238
|
|
2192
|
-
def log_entry(
|
2193
|
-
"test log entry #{
|
2239
|
+
def log_entry(index)
|
2240
|
+
"test log entry #{index}"
|
2194
2241
|
end
|
2195
2242
|
|
2196
2243
|
# If check_exact_labels is true, assert 'labels' and 'expected_labels' match
|
@@ -2198,6 +2245,7 @@ module BaseTest
|
|
2198
2245
|
# 'expected_labels'.
|
2199
2246
|
def check_labels(expected_labels, labels, check_exact_labels = true)
|
2200
2247
|
return if expected_labels.empty? && labels.empty?
|
2248
|
+
|
2201
2249
|
expected_labels.each do |expected_key, expected_value|
|
2202
2250
|
assert labels.key?(expected_key), "Expected label #{expected_key} not" \
|
2203
2251
|
" found. Got labels: #{labels}."
|
@@ -2207,20 +2255,21 @@ module BaseTest
|
|
2207
2255
|
assert_equal expected_value, actual_value, "Value for #{expected_key}" \
|
2208
2256
|
" mismatch. Expected #{expected_value}. Got #{actual_value}"
|
2209
2257
|
end
|
2210
|
-
|
2211
|
-
|
2212
|
-
|
2213
|
-
|
2214
|
-
|
2258
|
+
return unless check_exact_labels
|
2259
|
+
|
2260
|
+
assert_equal expected_labels.length, labels.length, 'Expected ' \
|
2261
|
+
"#{expected_labels.length} labels: #{expected_labels}, got " \
|
2262
|
+
"#{labels.length} labels: #{labels}"
|
2215
2263
|
end
|
2216
2264
|
|
2217
|
-
def verify_default_log_entry_text(text,
|
2218
|
-
assert_equal "test log entry #{
|
2219
|
-
"Entry ##{
|
2265
|
+
def verify_default_log_entry_text(text, index, entry)
|
2266
|
+
assert_equal "test log entry #{index}", text,
|
2267
|
+
"Entry ##{index} had unexpected text: #{entry}"
|
2220
2268
|
end
|
2221
2269
|
|
2222
2270
|
# The caller can optionally provide a block which is called for each entry.
|
2223
|
-
def verify_json_log_entries(
|
2271
|
+
def verify_json_log_entries(expected_count, params,
|
2272
|
+
payload_type = 'textPayload',
|
2224
2273
|
check_exact_entry_labels = true)
|
2225
2274
|
entry_count = 0
|
2226
2275
|
@logs_sent.each do |request|
|
@@ -2258,11 +2307,11 @@ module BaseTest
|
|
2258
2307
|
entry)
|
2259
2308
|
end
|
2260
2309
|
entry_count += 1
|
2261
|
-
assert entry_count <=
|
2262
|
-
"Number of entries #{entry_count} exceeds expected number #{
|
2310
|
+
assert entry_count <= expected_count,
|
2311
|
+
"Number of entries #{entry_count} exceeds expected number #{expected_count}."
|
2263
2312
|
end
|
2264
2313
|
end
|
2265
|
-
assert_equal
|
2314
|
+
assert_equal expected_count, entry_count
|
2266
2315
|
end
|
2267
2316
|
|
2268
2317
|
def verify_container_logs(log_entry_factory, expected_params)
|
@@ -2291,13 +2340,17 @@ module BaseTest
|
|
2291
2340
|
# LogEntry info from. The values are lists of two elements: the name of
|
2292
2341
|
# the subfield in LogEntry object and the expected value of that field.
|
2293
2342
|
DEFAULT_HTTP_REQUEST_KEY => [
|
2294
|
-
'httpRequest', HTTP_REQUEST_MESSAGE
|
2343
|
+
'httpRequest', HTTP_REQUEST_MESSAGE
|
2344
|
+
],
|
2295
2345
|
DEFAULT_LABELS_KEY => [
|
2296
|
-
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2346
|
+
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2347
|
+
],
|
2297
2348
|
DEFAULT_OPERATION_KEY => [
|
2298
|
-
'operation', OPERATION_MESSAGE
|
2349
|
+
'operation', OPERATION_MESSAGE
|
2350
|
+
],
|
2299
2351
|
DEFAULT_SOURCE_LOCATION_KEY => [
|
2300
|
-
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2352
|
+
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2353
|
+
]
|
2301
2354
|
}
|
2302
2355
|
end
|
2303
2356
|
|
@@ -2397,17 +2450,22 @@ module BaseTest
|
|
2397
2450
|
# left with name "log", "message" or "msg". This test verifies additional
|
2398
2451
|
# LogEntry fields like spanId and traceId do not disable that by accident.
|
2399
2452
|
def verify_cascading_json_detection_with_log_entry_fields(
|
2400
|
-
|
2453
|
+
log_entry_field, default_key, expectation
|
2454
|
+
)
|
2401
2455
|
root_level_value = expectation[:root_level_value]
|
2402
2456
|
nested_level_value = expectation[:nested_level_value]
|
2403
2457
|
expected_value_from_root = expectation.fetch(
|
2404
|
-
:expected_value_from_root, root_level_value
|
2458
|
+
:expected_value_from_root, root_level_value
|
2459
|
+
)
|
2405
2460
|
expected_value_from_nested = expectation.fetch(
|
2406
|
-
:expected_value_from_nested, nested_level_value
|
2461
|
+
:expected_value_from_nested, nested_level_value
|
2462
|
+
)
|
2407
2463
|
default_value_from_root = expectation.fetch(
|
2408
|
-
:default_value_from_root, nil
|
2464
|
+
:default_value_from_root, nil
|
2465
|
+
)
|
2409
2466
|
default_value_from_nested = expectation.fetch(
|
2410
|
-
:default_value_from_nested, nil
|
2467
|
+
:default_value_from_nested, nil
|
2468
|
+
)
|
2411
2469
|
|
2412
2470
|
setup_gce_metadata_stubs
|
2413
2471
|
|
@@ -2443,7 +2501,8 @@ module BaseTest
|
|
2443
2501
|
# }
|
2444
2502
|
# }
|
2445
2503
|
log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
|
2446
|
-
default_key => root_level_value
|
2504
|
+
default_key => root_level_value
|
2505
|
+
)
|
2447
2506
|
|
2448
2507
|
[
|
2449
2508
|
[
|
@@ -2594,7 +2653,8 @@ module BaseTest
|
|
2594
2653
|
|
2595
2654
|
# Verify the number and the content of the log entries match the expectation.
|
2596
2655
|
# The caller can optionally provide a block which is called for each entry.
|
2597
|
-
def verify_log_entries(
|
2656
|
+
def verify_log_entries(_expected_count, _params,
|
2657
|
+
_payload_type = 'textPayload',
|
2598
2658
|
_check_exact_entry_labels = true, &_block)
|
2599
2659
|
_undefined
|
2600
2660
|
end
|