fluent-plugin-google-cloud 0.12.10 → 0.13.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +99 -71
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +17 -21
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -14
- data/lib/fluent/plugin/filter_analyze_config.rb +65 -47
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +34 -23
- data/lib/fluent/plugin/out_google_cloud.rb +269 -213
- data/lib/fluent/plugin/statusz.rb +10 -10
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +199 -136
- data/test/plugin/constants.rb +21 -28
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +37 -25
- data/test/plugin/test_out_google_cloud_grpc.rb +45 -30
- data/test/plugin/utils.rb +9 -8
- metadata +53 -39
data/test/plugin/base_test.rb
CHANGED
@@ -29,8 +29,10 @@ require_relative 'utils'
|
|
29
29
|
|
30
30
|
module Monitoring
|
31
31
|
# Prevent OpenCensus from writing to the network.
|
32
|
-
|
33
|
-
|
32
|
+
OpenCensusMonitoringRegistry.class_eval do
|
33
|
+
# Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055).
|
34
|
+
alias_method :export, :export
|
35
|
+
define_method(:export) do
|
34
36
|
nil
|
35
37
|
end
|
36
38
|
end
|
@@ -55,6 +57,7 @@ module BaseTest
|
|
55
57
|
registry.unregister(:stackdriver_retried_entries_count)
|
56
58
|
|
57
59
|
setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token')
|
60
|
+
setup_auth_stubs('https://oauth2.googleapis.com/token')
|
58
61
|
@logs_sent = []
|
59
62
|
end
|
60
63
|
|
@@ -71,8 +74,8 @@ module BaseTest
|
|
71
74
|
exception_count = 0
|
72
75
|
begin
|
73
76
|
create_driver(PRIVATE_KEY_CONFIG)
|
74
|
-
rescue Fluent::ConfigError =>
|
75
|
-
assert
|
77
|
+
rescue Fluent::ConfigError => e
|
78
|
+
assert e.message.include? 'Please remove configuration parameters'
|
76
79
|
exception_count += 1
|
77
80
|
end
|
78
81
|
assert_equal 1, exception_count
|
@@ -100,23 +103,22 @@ module BaseTest
|
|
100
103
|
def test_configure_metadata_missing_parts_on_other_platforms
|
101
104
|
setup_no_metadata_service_stubs
|
102
105
|
Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
|
103
|
-
[
|
104
|
-
|
105
|
-
|
106
|
-
|
106
|
+
[
|
107
|
+
[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
|
108
|
+
[CONFIG_MISSING_METADATA_ZONE, [], true],
|
109
|
+
[CONFIG_MISSING_METADATA_VM_ID, [], true],
|
110
|
+
[CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
|
107
111
|
].each_with_index do |(config, missing_parts, is_valid_config), index|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
assert error.message.include?(part), "Index #{index} failed."
|
119
|
-
end
|
112
|
+
create_driver(config)
|
113
|
+
assert_true is_valid_config, "Invalid config at index #{index} should "\
|
114
|
+
'have raised an error.'
|
115
|
+
rescue Fluent::ConfigError => e
|
116
|
+
assert_false is_valid_config, "Valid config at index #{index} should "\
|
117
|
+
"not have raised an error #{e}."
|
118
|
+
assert e.message.include?('Unable to obtain metadata parameters:'),
|
119
|
+
"Index #{index} failed."
|
120
|
+
missing_parts.each do |part|
|
121
|
+
assert e.message.include?(part), "Index #{index} failed."
|
120
122
|
end
|
121
123
|
end
|
122
124
|
end
|
@@ -128,38 +130,52 @@ module BaseTest
|
|
128
130
|
setup_gce_metadata_stubs
|
129
131
|
create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
|
130
132
|
assert_nil(Prometheus::Client.registry.get(
|
131
|
-
:stackdriver_successful_requests_count
|
133
|
+
:stackdriver_successful_requests_count
|
134
|
+
))
|
132
135
|
assert_nil(Prometheus::Client.registry.get(
|
133
|
-
:stackdriver_failed_requests_count
|
136
|
+
:stackdriver_failed_requests_count
|
137
|
+
))
|
134
138
|
assert_nil(Prometheus::Client.registry.get(
|
135
|
-
:stackdriver_ingested_entries_count
|
139
|
+
:stackdriver_ingested_entries_count
|
140
|
+
))
|
136
141
|
assert_nil(Prometheus::Client.registry.get(
|
137
|
-
:stackdriver_dropped_entries_count
|
142
|
+
:stackdriver_dropped_entries_count
|
143
|
+
))
|
138
144
|
assert_nil(Prometheus::Client.registry.get(
|
139
|
-
:stackdriver_retried_entries_count
|
145
|
+
:stackdriver_retried_entries_count
|
146
|
+
))
|
140
147
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
141
148
|
Monitoring::MetricTranslator.new(
|
142
|
-
:stackdriver_successful_requests_count, {}
|
149
|
+
:stackdriver_successful_requests_count, {}
|
150
|
+
)
|
151
|
+
))
|
143
152
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
144
153
|
Monitoring::MetricTranslator.new(
|
145
|
-
:stackdriver_failed_requests_count, {}
|
154
|
+
:stackdriver_failed_requests_count, {}
|
155
|
+
)
|
156
|
+
))
|
146
157
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
147
158
|
Monitoring::MetricTranslator.new(
|
148
|
-
:stackdriver_ingested_entries_count, {}
|
159
|
+
:stackdriver_ingested_entries_count, {}
|
160
|
+
)
|
161
|
+
))
|
149
162
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
150
163
|
Monitoring::MetricTranslator.new(
|
151
|
-
:stackdriver_dropped_entries_count, {}
|
164
|
+
:stackdriver_dropped_entries_count, {}
|
165
|
+
)
|
166
|
+
))
|
152
167
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
153
168
|
Monitoring::MetricTranslator.new(
|
154
|
-
:stackdriver_retried_entries_count, {}
|
169
|
+
:stackdriver_retried_entries_count, {}
|
170
|
+
)
|
171
|
+
))
|
155
172
|
end
|
156
173
|
|
157
174
|
def test_configure_uses_metrics_resource
|
158
175
|
setup_gce_metadata_stubs
|
159
176
|
[CONFIG_METRICS_RESOURCE_JSON,
|
160
177
|
CONFIG_METRICS_RESOURCE_HASH,
|
161
|
-
CONFIG_METRICS_RESOURCE_JSON_HASH
|
162
|
-
].each_with_index do |config, index|
|
178
|
+
CONFIG_METRICS_RESOURCE_JSON_HASH].each_with_index do |config, index|
|
163
179
|
d = create_driver(config)
|
164
180
|
assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
|
165
181
|
"Index #{index}"
|
@@ -171,7 +187,8 @@ module BaseTest
|
|
171
187
|
registry = d.instance.instance_variable_get(:@registry)
|
172
188
|
assert_not_nil registry
|
173
189
|
monitored_resource = registry.instance_variable_get(
|
174
|
-
:@metrics_monitored_resource
|
190
|
+
:@metrics_monitored_resource
|
191
|
+
)
|
175
192
|
assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
|
176
193
|
assert_equal({ 'label1' => '123', 'label2' => 'abc' },
|
177
194
|
monitored_resource.labels, "Index #{index}")
|
@@ -190,14 +207,12 @@ module BaseTest
|
|
190
207
|
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
|
191
208
|
/unrecognized keys: \[:random\]/
|
192
209
|
}.each_with_index do |(config, pattern), index|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
"Index #{index} failed: got #{error.message}."
|
200
|
-
end
|
210
|
+
create_driver(config)
|
211
|
+
assert false,
|
212
|
+
"Invalid config at index #{index} should have raised an error."
|
213
|
+
rescue Fluent::ConfigError => e
|
214
|
+
assert e.message.match?(pattern), \
|
215
|
+
"Index #{index} failed: got #{e.message}."
|
201
216
|
end
|
202
217
|
end
|
203
218
|
|
@@ -277,16 +292,16 @@ module BaseTest
|
|
277
292
|
exception_count = 0
|
278
293
|
begin
|
279
294
|
create_driver
|
280
|
-
rescue Fluent::ConfigError =>
|
281
|
-
assert
|
282
|
-
assert
|
295
|
+
rescue Fluent::ConfigError => e
|
296
|
+
assert e.message.include? 'Unable to obtain metadata parameters:'
|
297
|
+
assert e.message.include? 'project_id'
|
283
298
|
exception_count += 1
|
284
299
|
end
|
285
300
|
assert_equal 1, exception_count
|
286
301
|
end
|
287
302
|
|
288
303
|
def test_project_id_from_credentials
|
289
|
-
%w
|
304
|
+
%w[gce ec2].each do |platform|
|
290
305
|
send("setup_#{platform}_metadata_stubs")
|
291
306
|
[IAM_CREDENTIALS, NEW_STYLE_CREDENTIALS, LEGACY_CREDENTIALS].each \
|
292
307
|
do |creds|
|
@@ -317,18 +332,19 @@ module BaseTest
|
|
317
332
|
d.run
|
318
333
|
end
|
319
334
|
verify_log_entries(1, COMPUTE_PARAMS.merge(
|
320
|
-
project_id: IAM_CREDENTIALS[:project_id]
|
335
|
+
project_id: IAM_CREDENTIALS[:project_id]
|
336
|
+
))
|
321
337
|
end
|
322
338
|
|
323
339
|
def test_invalid_json_credentials
|
324
|
-
%w
|
340
|
+
%w[gce_metadata ec2_metadata no_metadata_service].each do |platform|
|
325
341
|
send("setup_#{platform}_stubs")
|
326
342
|
exception_count = 0
|
327
343
|
ENV[CREDENTIALS_PATH_ENV_VAR] = INVALID_CREDENTIALS[:path]
|
328
344
|
begin
|
329
345
|
create_driver
|
330
|
-
rescue RuntimeError =>
|
331
|
-
assert
|
346
|
+
rescue RuntimeError => e
|
347
|
+
assert e.message.include? 'Unable to read the credential file'
|
332
348
|
exception_count += 1
|
333
349
|
end
|
334
350
|
assert_equal 1, exception_count
|
@@ -498,9 +514,9 @@ module BaseTest
|
|
498
514
|
d.emit(
|
499
515
|
'int_key' => { 1 => message },
|
500
516
|
'int_array_key' => { [1, 2, 3, 4] => message },
|
501
|
-
'string_array_key' => { %w
|
517
|
+
'string_array_key' => { %w[a b c] => message },
|
502
518
|
'hash_key' => { { 'some_key' => 'some_value' } => message },
|
503
|
-
'mixed_key' => { { 'some_key' => %w
|
519
|
+
'mixed_key' => { { 'some_key' => %w[a b c] } => message },
|
504
520
|
'symbol_key' => { some_symbol: message },
|
505
521
|
'nil_key' => { nil => message }
|
506
522
|
)
|
@@ -527,9 +543,9 @@ module BaseTest
|
|
527
543
|
'"data": 5000, "some_null_field": null}'
|
528
544
|
setup_logging_stubs do
|
529
545
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
530
|
-
d.emit('message' =>
|
546
|
+
d.emit('message' => "notJSON #{json_string}")
|
531
547
|
d.emit('message' => json_string)
|
532
|
-
d.emit('message' => " \r\n \t"
|
548
|
+
d.emit('message' => " \r\n \t#{json_string}")
|
533
549
|
d.run
|
534
550
|
end
|
535
551
|
verify_log_entries(3, COMPUTE_PARAMS, 'textPayload') do
|
@@ -543,10 +559,10 @@ module BaseTest
|
|
543
559
|
'"data": 5000, "some_null_field": null}'
|
544
560
|
setup_logging_stubs do
|
545
561
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
546
|
-
%w
|
547
|
-
d.emit(field =>
|
562
|
+
%w[log msg].each do |field|
|
563
|
+
d.emit(field => "notJSON #{json_string}")
|
548
564
|
d.emit(field => json_string)
|
549
|
-
d.emit(field => " \r\n \t"
|
565
|
+
d.emit(field => " \r\n \t#{json_string}")
|
550
566
|
end
|
551
567
|
d.run
|
552
568
|
end
|
@@ -564,7 +580,7 @@ module BaseTest
|
|
564
580
|
'"data": 5000, "some_null_field": null}'
|
565
581
|
setup_logging_stubs do
|
566
582
|
d = create_driver(DETECT_JSON_CONFIG)
|
567
|
-
d.emit('message' =>
|
583
|
+
d.emit('message' => "notJSON #{json_string}")
|
568
584
|
d.run
|
569
585
|
end
|
570
586
|
verify_log_entries(1, COMPUTE_PARAMS, 'textPayload') do
|
@@ -578,8 +594,8 @@ module BaseTest
|
|
578
594
|
'"data": 5000, "some_null_field": null}'
|
579
595
|
setup_logging_stubs do
|
580
596
|
d = create_driver(DETECT_JSON_CONFIG)
|
581
|
-
%w
|
582
|
-
d.emit(field =>
|
597
|
+
%w[log msg].each do |field|
|
598
|
+
d.emit(field => "notJSON #{json_string}")
|
583
599
|
end
|
584
600
|
d.run
|
585
601
|
end
|
@@ -668,9 +684,9 @@ module BaseTest
|
|
668
684
|
'"data": 5000, "some_null_field": null}'
|
669
685
|
setup_logging_stubs do
|
670
686
|
d = create_driver(DETECT_JSON_CONFIG)
|
671
|
-
%w
|
687
|
+
%w[message log msg].each do |field|
|
672
688
|
d.emit(field => json_string)
|
673
|
-
d.emit(field => " \r\n \t"
|
689
|
+
d.emit(field => " \r\n \t#{json_string}")
|
674
690
|
end
|
675
691
|
d.run
|
676
692
|
end
|
@@ -691,9 +707,9 @@ module BaseTest
|
|
691
707
|
'"data": 5000, "some_null_field": null}'
|
692
708
|
setup_logging_stubs do
|
693
709
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
694
|
-
d.emit(container_log_entry_with_metadata(
|
710
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
695
711
|
d.emit(container_log_entry_with_metadata(json_string))
|
696
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
712
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
697
713
|
d.run
|
698
714
|
end
|
699
715
|
verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -708,7 +724,7 @@ module BaseTest
|
|
708
724
|
'"data": 5000, "some_null_field": null}'
|
709
725
|
setup_logging_stubs do
|
710
726
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
711
|
-
d.emit(container_log_entry_with_metadata(
|
727
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
712
728
|
d.run
|
713
729
|
end
|
714
730
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -724,7 +740,7 @@ module BaseTest
|
|
724
740
|
setup_logging_stubs do
|
725
741
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
726
742
|
d.emit(container_log_entry_with_metadata(json_string))
|
727
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
743
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
728
744
|
d.run
|
729
745
|
end
|
730
746
|
verify_log_entries(2, CONTAINER_FROM_METADATA_PARAMS, 'jsonPayload') \
|
@@ -749,14 +765,16 @@ module BaseTest
|
|
749
765
|
setup_logging_stubs do
|
750
766
|
@logs_sent = []
|
751
767
|
d = create_driver(DETECT_JSON_CONFIG)
|
752
|
-
%w
|
768
|
+
%w[message log msg].each do |field|
|
753
769
|
d.emit(PRESERVED_KEYS_MAP.merge(
|
754
|
-
field => json_string
|
770
|
+
field => json_string
|
771
|
+
).merge(timestamp_fields))
|
755
772
|
end
|
756
773
|
d.run
|
757
774
|
end
|
758
775
|
expected_params = COMPUTE_PARAMS.merge(
|
759
|
-
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
776
|
+
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
777
|
+
)
|
760
778
|
verify_log_entries(3, expected_params, 'jsonPayload') do |entry|
|
761
779
|
fields = entry['jsonPayload']
|
762
780
|
assert_equal 4, fields.size, entry
|
@@ -772,7 +790,7 @@ module BaseTest
|
|
772
790
|
# any non-string tags or tags with non-utf8 characters are detected.
|
773
791
|
def test_reject_invalid_tags_with_require_valid_tags_true
|
774
792
|
setup_gce_metadata_stubs
|
775
|
-
INVALID_TAGS.
|
793
|
+
INVALID_TAGS.each_key do |tag|
|
776
794
|
setup_logging_stubs do
|
777
795
|
@logs_sent = []
|
778
796
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
@@ -800,8 +818,11 @@ module BaseTest
|
|
800
818
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
801
819
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
802
820
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
803
|
-
'container_name' => container_name
|
804
|
-
|
821
|
+
'container_name' => container_name
|
822
|
+
)
|
823
|
+
),
|
824
|
+
log_name: tag
|
825
|
+
)
|
805
826
|
verify_log_entries(1, params, 'textPayload')
|
806
827
|
end
|
807
828
|
|
@@ -824,8 +845,10 @@ module BaseTest
|
|
824
845
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
825
846
|
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
826
847
|
"#{GKE_CONSTANTS[:service]}/container_name" =>
|
827
|
-
CGI.unescape(encoded_name)
|
828
|
-
|
848
|
+
CGI.unescape(encoded_name)
|
849
|
+
),
|
850
|
+
log_name: encoded_name
|
851
|
+
)
|
829
852
|
verify_log_entries(0, params, 'textPayload')
|
830
853
|
end
|
831
854
|
end
|
@@ -861,8 +884,11 @@ module BaseTest
|
|
861
884
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
862
885
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
863
886
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
864
|
-
'container_name' => tag
|
865
|
-
|
887
|
+
'container_name' => tag
|
888
|
+
)
|
889
|
+
),
|
890
|
+
log_name: encoded_tag
|
891
|
+
)
|
866
892
|
verify_log_entries(1, params, 'textPayload')
|
867
893
|
end
|
868
894
|
end
|
@@ -907,8 +933,11 @@ module BaseTest
|
|
907
933
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
908
934
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
909
935
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
910
|
-
'container_name' => CGI.unescape(encoded_container_name)
|
911
|
-
|
936
|
+
'container_name' => CGI.unescape(encoded_container_name)
|
937
|
+
)
|
938
|
+
),
|
939
|
+
log_name: encoded_container_name
|
940
|
+
)
|
912
941
|
verify_log_entries(1, params, 'textPayload')
|
913
942
|
end
|
914
943
|
end
|
@@ -960,13 +989,15 @@ module BaseTest
|
|
960
989
|
request_count,
|
961
990
|
'agent.googleapis.com/agent',
|
962
991
|
OpenCensus::Stats::Aggregation::Sum, d,
|
963
|
-
:aggregate
|
992
|
+
:aggregate
|
993
|
+
)
|
964
994
|
assert_prometheus_metric_value(
|
965
995
|
:stackdriver_ingested_entries_count,
|
966
996
|
log_entry_count,
|
967
997
|
'agent.googleapis.com/agent',
|
968
998
|
OpenCensus::Stats::Aggregation::Sum, d,
|
969
|
-
:aggregate
|
999
|
+
:aggregate
|
1000
|
+
)
|
970
1001
|
end
|
971
1002
|
end
|
972
1003
|
end
|
@@ -1044,13 +1075,13 @@ module BaseTest
|
|
1044
1075
|
current_time)
|
1045
1076
|
end
|
1046
1077
|
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1078
|
+
december29 = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
|
1079
|
+
december31 = Time.new(2019, 12, 31, 10, 23, 35, '-08:00')
|
1080
|
+
january1 = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1050
1081
|
|
1051
1082
|
{
|
1052
1083
|
# December 29, 2019 (normal operation).
|
1053
|
-
|
1084
|
+
december29 => begin
|
1054
1085
|
one_day_later = Time.new(2019, 12, 30, 10, 23, 35, '-08:00')
|
1055
1086
|
one_day_a_year_earlier = Time.new(2018, 12, 30, 10, 23, 35, '-08:00')
|
1056
1087
|
just_under_one_day_later = Time.new(2019, 12, 30, 10, 23, 34, '-08:00')
|
@@ -1064,7 +1095,7 @@ module BaseTest
|
|
1064
1095
|
{
|
1065
1096
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1066
1097
|
Time.at(0) => Time.at(0),
|
1067
|
-
|
1098
|
+
december29 => december29,
|
1068
1099
|
one_day_later => one_day_a_year_earlier,
|
1069
1100
|
just_under_one_day_later => just_under_one_day_later,
|
1070
1101
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1074,7 +1105,7 @@ module BaseTest
|
|
1074
1105
|
}
|
1075
1106
|
end,
|
1076
1107
|
# January 1, 2020 (normal operation).
|
1077
|
-
|
1108
|
+
january1 => begin
|
1078
1109
|
one_day_later = Time.new(2020, 1, 2, 10, 23, 35, '-08:00')
|
1079
1110
|
one_day_a_year_earlier = Time.new(2019, 1, 2, 10, 23, 35, '-08:00')
|
1080
1111
|
just_under_one_day_later = Time.new(2020, 1, 2, 10, 23, 34, '-08:00')
|
@@ -1088,7 +1119,7 @@ module BaseTest
|
|
1088
1119
|
{
|
1089
1120
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1090
1121
|
Time.at(0) => Time.at(0),
|
1091
|
-
|
1122
|
+
january1 => january1,
|
1092
1123
|
one_day_later => one_day_a_year_earlier,
|
1093
1124
|
just_under_one_day_later => just_under_one_day_later,
|
1094
1125
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1098,7 +1129,7 @@ module BaseTest
|
|
1098
1129
|
}
|
1099
1130
|
end,
|
1100
1131
|
# December 31, 2019 (next day overlaps new year).
|
1101
|
-
|
1132
|
+
december31 => begin
|
1102
1133
|
one_day_later = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1103
1134
|
just_under_one_day_later = Time.new(2020, 1, 1, 10, 23, 34, '-08:00')
|
1104
1135
|
next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
|
@@ -1109,7 +1140,7 @@ module BaseTest
|
|
1109
1140
|
{
|
1110
1141
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1111
1142
|
Time.at(0) => Time.at(0),
|
1112
|
-
|
1143
|
+
december31 => december31,
|
1113
1144
|
one_day_later => Time.at(0), # Falls into the next year.
|
1114
1145
|
just_under_one_day_later => just_under_one_day_later,
|
1115
1146
|
one_second_before_next_year => one_second_before_next_year,
|
@@ -1607,7 +1638,8 @@ module BaseTest
|
|
1607
1638
|
custom_key: 'custom_labels_key',
|
1608
1639
|
custom_key_config: CONFIG_CUSTOM_LABELS_KEY_SPECIFIED,
|
1609
1640
|
sample_value: COMPUTE_PARAMS[:labels].merge(
|
1610
|
-
LABELS_MESSAGE
|
1641
|
+
LABELS_MESSAGE
|
1642
|
+
),
|
1611
1643
|
default_value: COMPUTE_PARAMS[:labels])
|
1612
1644
|
end
|
1613
1645
|
|
@@ -1659,7 +1691,8 @@ module BaseTest
|
|
1659
1691
|
verify_cascading_json_detection_with_log_entry_fields(
|
1660
1692
|
'insertId', DEFAULT_INSERT_ID_KEY,
|
1661
1693
|
root_level_value: INSERT_ID,
|
1662
|
-
nested_level_value: INSERT_ID2
|
1694
|
+
nested_level_value: INSERT_ID2
|
1695
|
+
)
|
1663
1696
|
end
|
1664
1697
|
|
1665
1698
|
def test_cascading_json_detection_with_log_entry_labels_field
|
@@ -1669,7 +1702,9 @@ module BaseTest
|
|
1669
1702
|
nested_level_value: LABELS_MESSAGE2,
|
1670
1703
|
expected_value_from_root: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE),
|
1671
1704
|
expected_value_from_nested: COMPUTE_PARAMS[:labels].merge(
|
1672
|
-
LABELS_MESSAGE2
|
1705
|
+
LABELS_MESSAGE2
|
1706
|
+
)
|
1707
|
+
)
|
1673
1708
|
end
|
1674
1709
|
|
1675
1710
|
def test_cascading_json_detection_with_log_entry_operation_field
|
@@ -1677,28 +1712,32 @@ module BaseTest
|
|
1677
1712
|
'operation', DEFAULT_OPERATION_KEY,
|
1678
1713
|
root_level_value: OPERATION_MESSAGE,
|
1679
1714
|
nested_level_value: OPERATION_MESSAGE2,
|
1680
|
-
expected_value_from_nested: expected_operation_message2
|
1715
|
+
expected_value_from_nested: expected_operation_message2
|
1716
|
+
)
|
1681
1717
|
end
|
1682
1718
|
|
1683
1719
|
def test_cascading_json_detection_with_log_entry_source_location_field
|
1684
1720
|
verify_cascading_json_detection_with_log_entry_fields(
|
1685
1721
|
'sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
|
1686
1722
|
root_level_value: SOURCE_LOCATION_MESSAGE,
|
1687
|
-
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1723
|
+
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1724
|
+
)
|
1688
1725
|
end
|
1689
1726
|
|
1690
1727
|
def test_cascading_json_detection_with_log_entry_span_id_field
|
1691
1728
|
verify_cascading_json_detection_with_log_entry_fields(
|
1692
1729
|
'spanId', DEFAULT_SPAN_ID_KEY,
|
1693
1730
|
root_level_value: SPAN_ID,
|
1694
|
-
nested_level_value: SPAN_ID2
|
1731
|
+
nested_level_value: SPAN_ID2
|
1732
|
+
)
|
1695
1733
|
end
|
1696
1734
|
|
1697
1735
|
def test_cascading_json_detection_with_log_entry_trace_field
|
1698
1736
|
verify_cascading_json_detection_with_log_entry_fields(
|
1699
1737
|
'trace', DEFAULT_TRACE_KEY,
|
1700
1738
|
root_level_value: TRACE,
|
1701
|
-
nested_level_value: TRACE2
|
1739
|
+
nested_level_value: TRACE2
|
1740
|
+
)
|
1702
1741
|
end
|
1703
1742
|
|
1704
1743
|
def test_cascading_json_detection_with_log_entry_trace_sampled_field
|
@@ -1707,7 +1746,8 @@ module BaseTest
|
|
1707
1746
|
root_level_value: TRACE_SAMPLED,
|
1708
1747
|
nested_level_value: TRACE_SAMPLED2,
|
1709
1748
|
default_value_from_root: false,
|
1710
|
-
default_value_from_nested: false
|
1749
|
+
default_value_from_nested: false
|
1750
|
+
)
|
1711
1751
|
end
|
1712
1752
|
|
1713
1753
|
# Verify that labels present in multiple inputs respect the expected priority
|
@@ -1739,9 +1779,11 @@ module BaseTest
|
|
1739
1779
|
{
|
1740
1780
|
config: CONFIG_LABLES_AND_LABLE_MAP,
|
1741
1781
|
emitted_log: PAYLOAD_FOR_LABEL_MAP.merge(
|
1742
|
-
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1782
|
+
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1783
|
+
),
|
1743
1784
|
expected_labels: LABELS_MESSAGE.merge(LABELS_FROM_LABELS_CONFIG).merge(
|
1744
|
-
LABELS_FROM_LABEL_MAP_CONFIG
|
1785
|
+
LABELS_FROM_LABEL_MAP_CONFIG
|
1786
|
+
)
|
1745
1787
|
},
|
1746
1788
|
# labels from the config "labels" and "label_map" conflict.
|
1747
1789
|
{
|
@@ -1761,14 +1803,16 @@ module BaseTest
|
|
1761
1803
|
{
|
1762
1804
|
config: CONFIG_LABEL_MAP_CONFLICTING,
|
1763
1805
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1764
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1806
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1807
|
+
),
|
1765
1808
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1766
1809
|
},
|
1767
1810
|
# All three types of labels conflict.
|
1768
1811
|
{
|
1769
1812
|
config: CONFIG_LABLES_AND_LABLE_MAP_CONFLICTING,
|
1770
1813
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1771
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1814
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1815
|
+
),
|
1772
1816
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1773
1817
|
}
|
1774
1818
|
].each do |test_params|
|
@@ -1780,7 +1824,8 @@ module BaseTest
|
|
1780
1824
|
d.run
|
1781
1825
|
end
|
1782
1826
|
expected_params = COMPUTE_PARAMS.merge(
|
1783
|
-
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1827
|
+
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1828
|
+
)
|
1784
1829
|
verify_log_entries(1, expected_params)
|
1785
1830
|
end
|
1786
1831
|
end
|
@@ -1852,7 +1897,8 @@ module BaseTest
|
|
1852
1897
|
config: APPLICATION_DEFAULT_CONFIG,
|
1853
1898
|
setup_k8s_stub: true,
|
1854
1899
|
log_entry: k8s_container_log_entry(
|
1855
|
-
log_entry(0)
|
1900
|
+
log_entry(0)
|
1901
|
+
).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
1856
1902
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1857
1903
|
},
|
1858
1904
|
{
|
@@ -1860,7 +1906,8 @@ module BaseTest
|
|
1860
1906
|
setup_k8s_stub: true,
|
1861
1907
|
log_entry: k8s_container_log_entry(
|
1862
1908
|
log_entry(0),
|
1863
|
-
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1909
|
+
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1910
|
+
),
|
1864
1911
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1865
1912
|
}
|
1866
1913
|
].each do |test_params|
|
@@ -1979,7 +2026,8 @@ module BaseTest
|
|
1979
2026
|
assert_metric_value.call(
|
1980
2027
|
:uptime, expected, 'agent.googleapis.com/agent',
|
1981
2028
|
OpenCensus::Stats::Aggregation::Sum, d,
|
1982
|
-
version: Fluent::GoogleCloudOutput.version_string
|
2029
|
+
version: Fluent::GoogleCloudOutput.version_string
|
2030
|
+
)
|
1983
2031
|
rescue Test::Unit::AssertionFailedError
|
1984
2032
|
retry if (retries += 1) < 3
|
1985
2033
|
end
|
@@ -2013,12 +2061,12 @@ module BaseTest
|
|
2013
2061
|
(1..entry_count).each do |entry_index|
|
2014
2062
|
d.emit('message' => log_entry(entry_index.to_s))
|
2015
2063
|
end
|
2016
|
-
# rubocop:disable Lint/
|
2064
|
+
# rubocop:disable Lint/SuppressedException
|
2017
2065
|
begin
|
2018
2066
|
d.run
|
2019
2067
|
rescue mock_error_type
|
2020
2068
|
end
|
2021
|
-
# rubocop:enable Lint/
|
2069
|
+
# rubocop:enable Lint/SuppressedException
|
2022
2070
|
failed_requests_count, dropped_entries_count,
|
2023
2071
|
retried_entries_count = metric_values
|
2024
2072
|
|
@@ -2067,6 +2115,7 @@ module BaseTest
|
|
2067
2115
|
# assertion will fail in the case when a single metric contains time
|
2068
2116
|
# series with success and failure events.
|
2069
2117
|
next if code == ok_status_code
|
2118
|
+
|
2070
2119
|
assert_metric_value.call(:stackdriver_failed_requests_count,
|
2071
2120
|
failed_requests_count,
|
2072
2121
|
'agent.googleapis.com/agent',
|
@@ -2100,7 +2149,8 @@ module BaseTest
|
|
2100
2149
|
end
|
2101
2150
|
|
2102
2151
|
def container_log_entry_with_metadata(
|
2103
|
-
|
2152
|
+
log, container_name = K8S_CONTAINER_NAME
|
2153
|
+
)
|
2104
2154
|
{
|
2105
2155
|
log: log,
|
2106
2156
|
stream: K8S_STREAM,
|
@@ -2159,10 +2209,10 @@ module BaseTest
|
|
2159
2209
|
}
|
2160
2210
|
end
|
2161
2211
|
|
2162
|
-
def dataflow_log_entry(
|
2212
|
+
def dataflow_log_entry(index)
|
2163
2213
|
{
|
2164
2214
|
step: DATAFLOW_STEP_ID,
|
2165
|
-
message: log_entry(
|
2215
|
+
message: log_entry(index)
|
2166
2216
|
}
|
2167
2217
|
end
|
2168
2218
|
|
@@ -2175,10 +2225,10 @@ module BaseTest
|
|
2175
2225
|
}
|
2176
2226
|
end
|
2177
2227
|
|
2178
|
-
def ml_log_entry(
|
2228
|
+
def ml_log_entry(index)
|
2179
2229
|
{
|
2180
2230
|
name: ML_LOG_AREA,
|
2181
|
-
message: log_entry(
|
2231
|
+
message: log_entry(index)
|
2182
2232
|
}
|
2183
2233
|
end
|
2184
2234
|
|
@@ -2189,8 +2239,8 @@ module BaseTest
|
|
2189
2239
|
}
|
2190
2240
|
end
|
2191
2241
|
|
2192
|
-
def log_entry(
|
2193
|
-
"test log entry #{
|
2242
|
+
def log_entry(index)
|
2243
|
+
"test log entry #{index}"
|
2194
2244
|
end
|
2195
2245
|
|
2196
2246
|
# If check_exact_labels is true, assert 'labels' and 'expected_labels' match
|
@@ -2198,6 +2248,7 @@ module BaseTest
|
|
2198
2248
|
# 'expected_labels'.
|
2199
2249
|
def check_labels(expected_labels, labels, check_exact_labels = true)
|
2200
2250
|
return if expected_labels.empty? && labels.empty?
|
2251
|
+
|
2201
2252
|
expected_labels.each do |expected_key, expected_value|
|
2202
2253
|
assert labels.key?(expected_key), "Expected label #{expected_key} not" \
|
2203
2254
|
" found. Got labels: #{labels}."
|
@@ -2207,20 +2258,21 @@ module BaseTest
|
|
2207
2258
|
assert_equal expected_value, actual_value, "Value for #{expected_key}" \
|
2208
2259
|
" mismatch. Expected #{expected_value}. Got #{actual_value}"
|
2209
2260
|
end
|
2210
|
-
|
2211
|
-
|
2212
|
-
|
2213
|
-
|
2214
|
-
|
2261
|
+
return unless check_exact_labels
|
2262
|
+
|
2263
|
+
assert_equal expected_labels.length, labels.length, 'Expected ' \
|
2264
|
+
"#{expected_labels.length} labels: #{expected_labels}, got " \
|
2265
|
+
"#{labels.length} labels: #{labels}"
|
2215
2266
|
end
|
2216
2267
|
|
2217
|
-
def verify_default_log_entry_text(text,
|
2218
|
-
assert_equal "test log entry #{
|
2219
|
-
"Entry ##{
|
2268
|
+
def verify_default_log_entry_text(text, index, entry)
|
2269
|
+
assert_equal "test log entry #{index}", text,
|
2270
|
+
"Entry ##{index} had unexpected text: #{entry}"
|
2220
2271
|
end
|
2221
2272
|
|
2222
2273
|
# The caller can optionally provide a block which is called for each entry.
|
2223
|
-
def verify_json_log_entries(
|
2274
|
+
def verify_json_log_entries(expected_count, params,
|
2275
|
+
payload_type = 'textPayload',
|
2224
2276
|
check_exact_entry_labels = true)
|
2225
2277
|
entry_count = 0
|
2226
2278
|
@logs_sent.each do |request|
|
@@ -2258,11 +2310,11 @@ module BaseTest
|
|
2258
2310
|
entry)
|
2259
2311
|
end
|
2260
2312
|
entry_count += 1
|
2261
|
-
assert entry_count <=
|
2262
|
-
"Number of entries #{entry_count} exceeds expected number #{
|
2313
|
+
assert entry_count <= expected_count,
|
2314
|
+
"Number of entries #{entry_count} exceeds expected number #{expected_count}."
|
2263
2315
|
end
|
2264
2316
|
end
|
2265
|
-
assert_equal
|
2317
|
+
assert_equal expected_count, entry_count
|
2266
2318
|
end
|
2267
2319
|
|
2268
2320
|
def verify_container_logs(log_entry_factory, expected_params)
|
@@ -2291,13 +2343,17 @@ module BaseTest
|
|
2291
2343
|
# LogEntry info from. The values are lists of two elements: the name of
|
2292
2344
|
# the subfield in LogEntry object and the expected value of that field.
|
2293
2345
|
DEFAULT_HTTP_REQUEST_KEY => [
|
2294
|
-
'httpRequest', HTTP_REQUEST_MESSAGE
|
2346
|
+
'httpRequest', HTTP_REQUEST_MESSAGE
|
2347
|
+
],
|
2295
2348
|
DEFAULT_LABELS_KEY => [
|
2296
|
-
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2349
|
+
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2350
|
+
],
|
2297
2351
|
DEFAULT_OPERATION_KEY => [
|
2298
|
-
'operation', OPERATION_MESSAGE
|
2352
|
+
'operation', OPERATION_MESSAGE
|
2353
|
+
],
|
2299
2354
|
DEFAULT_SOURCE_LOCATION_KEY => [
|
2300
|
-
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2355
|
+
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2356
|
+
]
|
2301
2357
|
}
|
2302
2358
|
end
|
2303
2359
|
|
@@ -2397,17 +2453,22 @@ module BaseTest
|
|
2397
2453
|
# left with name "log", "message" or "msg". This test verifies additional
|
2398
2454
|
# LogEntry fields like spanId and traceId do not disable that by accident.
|
2399
2455
|
def verify_cascading_json_detection_with_log_entry_fields(
|
2400
|
-
|
2456
|
+
log_entry_field, default_key, expectation
|
2457
|
+
)
|
2401
2458
|
root_level_value = expectation[:root_level_value]
|
2402
2459
|
nested_level_value = expectation[:nested_level_value]
|
2403
2460
|
expected_value_from_root = expectation.fetch(
|
2404
|
-
:expected_value_from_root, root_level_value
|
2461
|
+
:expected_value_from_root, root_level_value
|
2462
|
+
)
|
2405
2463
|
expected_value_from_nested = expectation.fetch(
|
2406
|
-
:expected_value_from_nested, nested_level_value
|
2464
|
+
:expected_value_from_nested, nested_level_value
|
2465
|
+
)
|
2407
2466
|
default_value_from_root = expectation.fetch(
|
2408
|
-
:default_value_from_root, nil
|
2467
|
+
:default_value_from_root, nil
|
2468
|
+
)
|
2409
2469
|
default_value_from_nested = expectation.fetch(
|
2410
|
-
:default_value_from_nested, nil
|
2470
|
+
:default_value_from_nested, nil
|
2471
|
+
)
|
2411
2472
|
|
2412
2473
|
setup_gce_metadata_stubs
|
2413
2474
|
|
@@ -2443,7 +2504,8 @@ module BaseTest
|
|
2443
2504
|
# }
|
2444
2505
|
# }
|
2445
2506
|
log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
|
2446
|
-
default_key => root_level_value
|
2507
|
+
default_key => root_level_value
|
2508
|
+
)
|
2447
2509
|
|
2448
2510
|
[
|
2449
2511
|
[
|
@@ -2594,7 +2656,8 @@ module BaseTest
|
|
2594
2656
|
|
2595
2657
|
# Verify the number and the content of the log entries match the expectation.
|
2596
2658
|
# The caller can optionally provide a block which is called for each entry.
|
2597
|
-
def verify_log_entries(
|
2659
|
+
def verify_log_entries(_expected_count, _params,
|
2660
|
+
_payload_type = 'textPayload',
|
2598
2661
|
_check_exact_entry_labels = true, &_block)
|
2599
2662
|
_undefined
|
2600
2663
|
end
|