fluent-plugin-google-cloud 0.10.9 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +133 -108
- data/Rakefile +6 -5
- data/fluent-plugin-google-cloud.gemspec +21 -22
- data/lib/fluent/plugin/common.rb +42 -29
- data/lib/fluent/plugin/filter_add_insert_ids.rb +1 -14
- data/lib/fluent/plugin/filter_analyze_config.rb +73 -48
- data/lib/fluent/plugin/in_object_space_dump.rb +1 -1
- data/lib/fluent/plugin/monitoring.rb +49 -20
- data/lib/fluent/plugin/out_google_cloud.rb +270 -213
- data/lib/fluent/plugin/statusz.rb +10 -10
- data/test/helper.rb +6 -0
- data/test/plugin/base_test.rb +200 -136
- data/test/plugin/constants.rb +21 -28
- data/test/plugin/test_driver.rb +2 -1
- data/test/plugin/test_filter_add_insert_ids.rb +5 -3
- data/test/plugin/test_filter_analyze_config.rb +32 -17
- data/test/plugin/test_out_google_cloud.rb +37 -25
- data/test/plugin/test_out_google_cloud_grpc.rb +45 -30
- data/test/plugin/utils.rb +9 -8
- metadata +58 -44
data/test/plugin/base_test.rb
CHANGED
@@ -21,6 +21,7 @@ require 'helper'
|
|
21
21
|
require 'mocha/test_unit'
|
22
22
|
require 'prometheus/client'
|
23
23
|
require 'webmock/test_unit'
|
24
|
+
require 'cgi'
|
24
25
|
|
25
26
|
require_relative 'asserts'
|
26
27
|
require_relative 'constants'
|
@@ -28,8 +29,10 @@ require_relative 'utils'
|
|
28
29
|
|
29
30
|
module Monitoring
|
30
31
|
# Prevent OpenCensus from writing to the network.
|
31
|
-
|
32
|
-
|
32
|
+
OpenCensusMonitoringRegistry.class_eval do
|
33
|
+
# Suppress redefine warning (https://bugs.ruby-lang.org/issues/17055).
|
34
|
+
alias_method :export, :export
|
35
|
+
define_method(:export) do
|
33
36
|
nil
|
34
37
|
end
|
35
38
|
end
|
@@ -54,6 +57,7 @@ module BaseTest
|
|
54
57
|
registry.unregister(:stackdriver_retried_entries_count)
|
55
58
|
|
56
59
|
setup_auth_stubs('https://www.googleapis.com/oauth2/v4/token')
|
60
|
+
setup_auth_stubs('https://oauth2.googleapis.com/token')
|
57
61
|
@logs_sent = []
|
58
62
|
end
|
59
63
|
|
@@ -70,8 +74,8 @@ module BaseTest
|
|
70
74
|
exception_count = 0
|
71
75
|
begin
|
72
76
|
create_driver(PRIVATE_KEY_CONFIG)
|
73
|
-
rescue Fluent::ConfigError =>
|
74
|
-
assert
|
77
|
+
rescue Fluent::ConfigError => e
|
78
|
+
assert e.message.include? 'Please remove configuration parameters'
|
75
79
|
exception_count += 1
|
76
80
|
end
|
77
81
|
assert_equal 1, exception_count
|
@@ -99,23 +103,22 @@ module BaseTest
|
|
99
103
|
def test_configure_metadata_missing_parts_on_other_platforms
|
100
104
|
setup_no_metadata_service_stubs
|
101
105
|
Common::Utils::CredentialsInfo.stubs(:project_id).returns(nil)
|
102
|
-
[
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
+
[
|
107
|
+
[CONFIG_MISSING_METADATA_PROJECT_ID, ['project_id'], false],
|
108
|
+
[CONFIG_MISSING_METADATA_ZONE, [], true],
|
109
|
+
[CONFIG_MISSING_METADATA_VM_ID, [], true],
|
110
|
+
[CONFIG_MISSING_METADATA_ALL, ['project_id'], false]
|
106
111
|
].each_with_index do |(config, missing_parts, is_valid_config), index|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
assert error.message.include?(part), "Index #{index} failed."
|
118
|
-
end
|
112
|
+
create_driver(config)
|
113
|
+
assert_true is_valid_config, "Invalid config at index #{index} should "\
|
114
|
+
'have raised an error.'
|
115
|
+
rescue Fluent::ConfigError => e
|
116
|
+
assert_false is_valid_config, "Valid config at index #{index} should "\
|
117
|
+
"not have raised an error #{e}."
|
118
|
+
assert e.message.include?('Unable to obtain metadata parameters:'),
|
119
|
+
"Index #{index} failed."
|
120
|
+
missing_parts.each do |part|
|
121
|
+
assert e.message.include?(part), "Index #{index} failed."
|
119
122
|
end
|
120
123
|
end
|
121
124
|
end
|
@@ -127,38 +130,52 @@ module BaseTest
|
|
127
130
|
setup_gce_metadata_stubs
|
128
131
|
create_driver(CONFIG_UNKNOWN_MONITORING_TYPE)
|
129
132
|
assert_nil(Prometheus::Client.registry.get(
|
130
|
-
:stackdriver_successful_requests_count
|
133
|
+
:stackdriver_successful_requests_count
|
134
|
+
))
|
131
135
|
assert_nil(Prometheus::Client.registry.get(
|
132
|
-
:stackdriver_failed_requests_count
|
136
|
+
:stackdriver_failed_requests_count
|
137
|
+
))
|
133
138
|
assert_nil(Prometheus::Client.registry.get(
|
134
|
-
:stackdriver_ingested_entries_count
|
139
|
+
:stackdriver_ingested_entries_count
|
140
|
+
))
|
135
141
|
assert_nil(Prometheus::Client.registry.get(
|
136
|
-
:stackdriver_dropped_entries_count
|
142
|
+
:stackdriver_dropped_entries_count
|
143
|
+
))
|
137
144
|
assert_nil(Prometheus::Client.registry.get(
|
138
|
-
:stackdriver_retried_entries_count
|
145
|
+
:stackdriver_retried_entries_count
|
146
|
+
))
|
139
147
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
140
148
|
Monitoring::MetricTranslator.new(
|
141
|
-
:stackdriver_successful_requests_count, {}
|
149
|
+
:stackdriver_successful_requests_count, {}
|
150
|
+
)
|
151
|
+
))
|
142
152
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
143
153
|
Monitoring::MetricTranslator.new(
|
144
|
-
:stackdriver_failed_requests_count, {}
|
154
|
+
:stackdriver_failed_requests_count, {}
|
155
|
+
)
|
156
|
+
))
|
145
157
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
146
158
|
Monitoring::MetricTranslator.new(
|
147
|
-
:stackdriver_ingested_entries_count, {}
|
159
|
+
:stackdriver_ingested_entries_count, {}
|
160
|
+
)
|
161
|
+
))
|
148
162
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
149
163
|
Monitoring::MetricTranslator.new(
|
150
|
-
:stackdriver_dropped_entries_count, {}
|
164
|
+
:stackdriver_dropped_entries_count, {}
|
165
|
+
)
|
166
|
+
))
|
151
167
|
assert_nil(OpenCensus::Stats::MeasureRegistry.get(
|
152
168
|
Monitoring::MetricTranslator.new(
|
153
|
-
:stackdriver_retried_entries_count, {}
|
169
|
+
:stackdriver_retried_entries_count, {}
|
170
|
+
)
|
171
|
+
))
|
154
172
|
end
|
155
173
|
|
156
174
|
def test_configure_uses_metrics_resource
|
157
175
|
setup_gce_metadata_stubs
|
158
176
|
[CONFIG_METRICS_RESOURCE_JSON,
|
159
177
|
CONFIG_METRICS_RESOURCE_HASH,
|
160
|
-
CONFIG_METRICS_RESOURCE_JSON_HASH
|
161
|
-
].each_with_index do |config, index|
|
178
|
+
CONFIG_METRICS_RESOURCE_JSON_HASH].each_with_index do |config, index|
|
162
179
|
d = create_driver(config)
|
163
180
|
assert_equal 'custom_resource', d.instance.monitoring_resource.type, \
|
164
181
|
"Index #{index}"
|
@@ -170,7 +187,8 @@ module BaseTest
|
|
170
187
|
registry = d.instance.instance_variable_get(:@registry)
|
171
188
|
assert_not_nil registry
|
172
189
|
monitored_resource = registry.instance_variable_get(
|
173
|
-
:@metrics_monitored_resource
|
190
|
+
:@metrics_monitored_resource
|
191
|
+
)
|
174
192
|
assert_equal('custom_resource', monitored_resource.type, "Index #{index}")
|
175
193
|
assert_equal({ 'label1' => '123', 'label2' => 'abc' },
|
176
194
|
monitored_resource.labels, "Index #{index}")
|
@@ -189,14 +207,12 @@ module BaseTest
|
|
189
207
|
CONFIG_METRICS_RESOURCE_JSON_BAD_KEYS_NO_LABELS =>
|
190
208
|
/unrecognized keys: \[:random\]/
|
191
209
|
}.each_with_index do |(config, pattern), index|
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
"Index #{index} failed: got #{error.message}."
|
199
|
-
end
|
210
|
+
create_driver(config)
|
211
|
+
assert false,
|
212
|
+
"Invalid config at index #{index} should have raised an error."
|
213
|
+
rescue Fluent::ConfigError => e
|
214
|
+
assert e.message.match?(pattern), \
|
215
|
+
"Index #{index} failed: got #{e.message}."
|
200
216
|
end
|
201
217
|
end
|
202
218
|
|
@@ -276,16 +292,16 @@ module BaseTest
|
|
276
292
|
exception_count = 0
|
277
293
|
begin
|
278
294
|
create_driver
|
279
|
-
rescue Fluent::ConfigError =>
|
280
|
-
assert
|
281
|
-
assert
|
295
|
+
rescue Fluent::ConfigError => e
|
296
|
+
assert e.message.include? 'Unable to obtain metadata parameters:'
|
297
|
+
assert e.message.include? 'project_id'
|
282
298
|
exception_count += 1
|
283
299
|
end
|
284
300
|
assert_equal 1, exception_count
|
285
301
|
end
|
286
302
|
|
287
303
|
def test_project_id_from_credentials
|
288
|
-
%w
|
304
|
+
%w[gce ec2].each do |platform|
|
289
305
|
send("setup_#{platform}_metadata_stubs")
|
290
306
|
[IAM_CREDENTIALS, NEW_STYLE_CREDENTIALS, LEGACY_CREDENTIALS].each \
|
291
307
|
do |creds|
|
@@ -316,18 +332,19 @@ module BaseTest
|
|
316
332
|
d.run
|
317
333
|
end
|
318
334
|
verify_log_entries(1, COMPUTE_PARAMS.merge(
|
319
|
-
project_id: IAM_CREDENTIALS[:project_id]
|
335
|
+
project_id: IAM_CREDENTIALS[:project_id]
|
336
|
+
))
|
320
337
|
end
|
321
338
|
|
322
339
|
def test_invalid_json_credentials
|
323
|
-
%w
|
340
|
+
%w[gce_metadata ec2_metadata no_metadata_service].each do |platform|
|
324
341
|
send("setup_#{platform}_stubs")
|
325
342
|
exception_count = 0
|
326
343
|
ENV[CREDENTIALS_PATH_ENV_VAR] = INVALID_CREDENTIALS[:path]
|
327
344
|
begin
|
328
345
|
create_driver
|
329
|
-
rescue RuntimeError =>
|
330
|
-
assert
|
346
|
+
rescue RuntimeError => e
|
347
|
+
assert e.message.include? 'Unable to read the credential file'
|
331
348
|
exception_count += 1
|
332
349
|
end
|
333
350
|
assert_equal 1, exception_count
|
@@ -497,9 +514,9 @@ module BaseTest
|
|
497
514
|
d.emit(
|
498
515
|
'int_key' => { 1 => message },
|
499
516
|
'int_array_key' => { [1, 2, 3, 4] => message },
|
500
|
-
'string_array_key' => { %w
|
517
|
+
'string_array_key' => { %w[a b c] => message },
|
501
518
|
'hash_key' => { { 'some_key' => 'some_value' } => message },
|
502
|
-
'mixed_key' => { { 'some_key' => %w
|
519
|
+
'mixed_key' => { { 'some_key' => %w[a b c] } => message },
|
503
520
|
'symbol_key' => { some_symbol: message },
|
504
521
|
'nil_key' => { nil => message }
|
505
522
|
)
|
@@ -526,9 +543,9 @@ module BaseTest
|
|
526
543
|
'"data": 5000, "some_null_field": null}'
|
527
544
|
setup_logging_stubs do
|
528
545
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
529
|
-
d.emit('message' =>
|
546
|
+
d.emit('message' => "notJSON #{json_string}")
|
530
547
|
d.emit('message' => json_string)
|
531
|
-
d.emit('message' => " \r\n \t"
|
548
|
+
d.emit('message' => " \r\n \t#{json_string}")
|
532
549
|
d.run
|
533
550
|
end
|
534
551
|
verify_log_entries(3, COMPUTE_PARAMS, 'textPayload') do
|
@@ -542,10 +559,10 @@ module BaseTest
|
|
542
559
|
'"data": 5000, "some_null_field": null}'
|
543
560
|
setup_logging_stubs do
|
544
561
|
d = create_driver(APPLICATION_DEFAULT_CONFIG)
|
545
|
-
%w
|
546
|
-
d.emit(field =>
|
562
|
+
%w[log msg].each do |field|
|
563
|
+
d.emit(field => "notJSON #{json_string}")
|
547
564
|
d.emit(field => json_string)
|
548
|
-
d.emit(field => " \r\n \t"
|
565
|
+
d.emit(field => " \r\n \t#{json_string}")
|
549
566
|
end
|
550
567
|
d.run
|
551
568
|
end
|
@@ -563,7 +580,7 @@ module BaseTest
|
|
563
580
|
'"data": 5000, "some_null_field": null}'
|
564
581
|
setup_logging_stubs do
|
565
582
|
d = create_driver(DETECT_JSON_CONFIG)
|
566
|
-
d.emit('message' =>
|
583
|
+
d.emit('message' => "notJSON #{json_string}")
|
567
584
|
d.run
|
568
585
|
end
|
569
586
|
verify_log_entries(1, COMPUTE_PARAMS, 'textPayload') do
|
@@ -577,8 +594,8 @@ module BaseTest
|
|
577
594
|
'"data": 5000, "some_null_field": null}'
|
578
595
|
setup_logging_stubs do
|
579
596
|
d = create_driver(DETECT_JSON_CONFIG)
|
580
|
-
%w
|
581
|
-
d.emit(field =>
|
597
|
+
%w[log msg].each do |field|
|
598
|
+
d.emit(field => "notJSON #{json_string}")
|
582
599
|
end
|
583
600
|
d.run
|
584
601
|
end
|
@@ -667,9 +684,9 @@ module BaseTest
|
|
667
684
|
'"data": 5000, "some_null_field": null}'
|
668
685
|
setup_logging_stubs do
|
669
686
|
d = create_driver(DETECT_JSON_CONFIG)
|
670
|
-
%w
|
687
|
+
%w[message log msg].each do |field|
|
671
688
|
d.emit(field => json_string)
|
672
|
-
d.emit(field => " \r\n \t"
|
689
|
+
d.emit(field => " \r\n \t#{json_string}")
|
673
690
|
end
|
674
691
|
d.run
|
675
692
|
end
|
@@ -690,9 +707,9 @@ module BaseTest
|
|
690
707
|
'"data": 5000, "some_null_field": null}'
|
691
708
|
setup_logging_stubs do
|
692
709
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
693
|
-
d.emit(container_log_entry_with_metadata(
|
710
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
694
711
|
d.emit(container_log_entry_with_metadata(json_string))
|
695
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
712
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
696
713
|
d.run
|
697
714
|
end
|
698
715
|
verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -707,7 +724,7 @@ module BaseTest
|
|
707
724
|
'"data": 5000, "some_null_field": null}'
|
708
725
|
setup_logging_stubs do
|
709
726
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
710
|
-
d.emit(container_log_entry_with_metadata(
|
727
|
+
d.emit(container_log_entry_with_metadata("notJSON#{json_string}"))
|
711
728
|
d.run
|
712
729
|
end
|
713
730
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS, 'textPayload') do
|
@@ -723,7 +740,7 @@ module BaseTest
|
|
723
740
|
setup_logging_stubs do
|
724
741
|
d = create_driver(DETECT_JSON_CONFIG, CONTAINER_TAG)
|
725
742
|
d.emit(container_log_entry_with_metadata(json_string))
|
726
|
-
d.emit(container_log_entry_with_metadata(" \r\n \t"
|
743
|
+
d.emit(container_log_entry_with_metadata(" \r\n \t#{json_string}"))
|
727
744
|
d.run
|
728
745
|
end
|
729
746
|
verify_log_entries(2, CONTAINER_FROM_METADATA_PARAMS, 'jsonPayload') \
|
@@ -748,14 +765,16 @@ module BaseTest
|
|
748
765
|
setup_logging_stubs do
|
749
766
|
@logs_sent = []
|
750
767
|
d = create_driver(DETECT_JSON_CONFIG)
|
751
|
-
%w
|
768
|
+
%w[message log msg].each do |field|
|
752
769
|
d.emit(PRESERVED_KEYS_MAP.merge(
|
753
|
-
field => json_string
|
770
|
+
field => json_string
|
771
|
+
).merge(timestamp_fields))
|
754
772
|
end
|
755
773
|
d.run
|
756
774
|
end
|
757
775
|
expected_params = COMPUTE_PARAMS.merge(
|
758
|
-
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
776
|
+
labels: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
777
|
+
)
|
759
778
|
verify_log_entries(3, expected_params, 'jsonPayload') do |entry|
|
760
779
|
fields = entry['jsonPayload']
|
761
780
|
assert_equal 4, fields.size, entry
|
@@ -771,7 +790,7 @@ module BaseTest
|
|
771
790
|
# any non-string tags or tags with non-utf8 characters are detected.
|
772
791
|
def test_reject_invalid_tags_with_require_valid_tags_true
|
773
792
|
setup_gce_metadata_stubs
|
774
|
-
INVALID_TAGS.
|
793
|
+
INVALID_TAGS.each_key do |tag|
|
775
794
|
setup_logging_stubs do
|
776
795
|
@logs_sent = []
|
777
796
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
@@ -799,8 +818,11 @@ module BaseTest
|
|
799
818
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
800
819
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
801
820
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
802
|
-
'container_name' => container_name
|
803
|
-
|
821
|
+
'container_name' => container_name
|
822
|
+
)
|
823
|
+
),
|
824
|
+
log_name: tag
|
825
|
+
)
|
804
826
|
verify_log_entries(1, params, 'textPayload')
|
805
827
|
end
|
806
828
|
|
@@ -823,8 +845,10 @@ module BaseTest
|
|
823
845
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
824
846
|
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
825
847
|
"#{GKE_CONSTANTS[:service]}/container_name" =>
|
826
|
-
|
827
|
-
|
848
|
+
CGI.unescape(encoded_name)
|
849
|
+
),
|
850
|
+
log_name: encoded_name
|
851
|
+
)
|
828
852
|
verify_log_entries(0, params, 'textPayload')
|
829
853
|
end
|
830
854
|
end
|
@@ -860,8 +884,11 @@ module BaseTest
|
|
860
884
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
861
885
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
862
886
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
863
|
-
'container_name' => tag
|
864
|
-
|
887
|
+
'container_name' => tag
|
888
|
+
)
|
889
|
+
),
|
890
|
+
log_name: encoded_tag
|
891
|
+
)
|
865
892
|
verify_log_entries(1, params, 'textPayload')
|
866
893
|
end
|
867
894
|
end
|
@@ -906,8 +933,11 @@ module BaseTest
|
|
906
933
|
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
907
934
|
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
908
935
|
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
909
|
-
'container_name' =>
|
910
|
-
|
936
|
+
'container_name' => CGI.unescape(encoded_container_name)
|
937
|
+
)
|
938
|
+
),
|
939
|
+
log_name: encoded_container_name
|
940
|
+
)
|
911
941
|
verify_log_entries(1, params, 'textPayload')
|
912
942
|
end
|
913
943
|
end
|
@@ -959,13 +989,15 @@ module BaseTest
|
|
959
989
|
request_count,
|
960
990
|
'agent.googleapis.com/agent',
|
961
991
|
OpenCensus::Stats::Aggregation::Sum, d,
|
962
|
-
:aggregate
|
992
|
+
:aggregate
|
993
|
+
)
|
963
994
|
assert_prometheus_metric_value(
|
964
995
|
:stackdriver_ingested_entries_count,
|
965
996
|
log_entry_count,
|
966
997
|
'agent.googleapis.com/agent',
|
967
998
|
OpenCensus::Stats::Aggregation::Sum, d,
|
968
|
-
:aggregate
|
999
|
+
:aggregate
|
1000
|
+
)
|
969
1001
|
end
|
970
1002
|
end
|
971
1003
|
end
|
@@ -1043,13 +1075,13 @@ module BaseTest
|
|
1043
1075
|
current_time)
|
1044
1076
|
end
|
1045
1077
|
|
1046
|
-
|
1047
|
-
|
1048
|
-
|
1078
|
+
december29 = Time.new(2019, 12, 29, 10, 23, 35, '-08:00')
|
1079
|
+
december31 = Time.new(2019, 12, 31, 10, 23, 35, '-08:00')
|
1080
|
+
january1 = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1049
1081
|
|
1050
1082
|
{
|
1051
1083
|
# December 29, 2019 (normal operation).
|
1052
|
-
|
1084
|
+
december29 => begin
|
1053
1085
|
one_day_later = Time.new(2019, 12, 30, 10, 23, 35, '-08:00')
|
1054
1086
|
one_day_a_year_earlier = Time.new(2018, 12, 30, 10, 23, 35, '-08:00')
|
1055
1087
|
just_under_one_day_later = Time.new(2019, 12, 30, 10, 23, 34, '-08:00')
|
@@ -1063,7 +1095,7 @@ module BaseTest
|
|
1063
1095
|
{
|
1064
1096
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1065
1097
|
Time.at(0) => Time.at(0),
|
1066
|
-
|
1098
|
+
december29 => december29,
|
1067
1099
|
one_day_later => one_day_a_year_earlier,
|
1068
1100
|
just_under_one_day_later => just_under_one_day_later,
|
1069
1101
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1073,7 +1105,7 @@ module BaseTest
|
|
1073
1105
|
}
|
1074
1106
|
end,
|
1075
1107
|
# January 1, 2020 (normal operation).
|
1076
|
-
|
1108
|
+
january1 => begin
|
1077
1109
|
one_day_later = Time.new(2020, 1, 2, 10, 23, 35, '-08:00')
|
1078
1110
|
one_day_a_year_earlier = Time.new(2019, 1, 2, 10, 23, 35, '-08:00')
|
1079
1111
|
just_under_one_day_later = Time.new(2020, 1, 2, 10, 23, 34, '-08:00')
|
@@ -1087,7 +1119,7 @@ module BaseTest
|
|
1087
1119
|
{
|
1088
1120
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1089
1121
|
Time.at(0) => Time.at(0),
|
1090
|
-
|
1122
|
+
january1 => january1,
|
1091
1123
|
one_day_later => one_day_a_year_earlier,
|
1092
1124
|
just_under_one_day_later => just_under_one_day_later,
|
1093
1125
|
one_second_before_next_year => one_second_before_this_year,
|
@@ -1097,7 +1129,7 @@ module BaseTest
|
|
1097
1129
|
}
|
1098
1130
|
end,
|
1099
1131
|
# December 31, 2019 (next day overlaps new year).
|
1100
|
-
|
1132
|
+
december31 => begin
|
1101
1133
|
one_day_later = Time.new(2020, 1, 1, 10, 23, 35, '-08:00')
|
1102
1134
|
just_under_one_day_later = Time.new(2020, 1, 1, 10, 23, 34, '-08:00')
|
1103
1135
|
next_year = Time.new(2020, 1, 1, 0, 0, 0, '-08:00')
|
@@ -1108,7 +1140,7 @@ module BaseTest
|
|
1108
1140
|
{
|
1109
1141
|
Time.at(123_456.789) => Time.at(123_456.789),
|
1110
1142
|
Time.at(0) => Time.at(0),
|
1111
|
-
|
1143
|
+
december31 => december31,
|
1112
1144
|
one_day_later => Time.at(0), # Falls into the next year.
|
1113
1145
|
just_under_one_day_later => just_under_one_day_later,
|
1114
1146
|
one_second_before_next_year => one_second_before_next_year,
|
@@ -1606,7 +1638,8 @@ module BaseTest
|
|
1606
1638
|
custom_key: 'custom_labels_key',
|
1607
1639
|
custom_key_config: CONFIG_CUSTOM_LABELS_KEY_SPECIFIED,
|
1608
1640
|
sample_value: COMPUTE_PARAMS[:labels].merge(
|
1609
|
-
LABELS_MESSAGE
|
1641
|
+
LABELS_MESSAGE
|
1642
|
+
),
|
1610
1643
|
default_value: COMPUTE_PARAMS[:labels])
|
1611
1644
|
end
|
1612
1645
|
|
@@ -1658,7 +1691,8 @@ module BaseTest
|
|
1658
1691
|
verify_cascading_json_detection_with_log_entry_fields(
|
1659
1692
|
'insertId', DEFAULT_INSERT_ID_KEY,
|
1660
1693
|
root_level_value: INSERT_ID,
|
1661
|
-
nested_level_value: INSERT_ID2
|
1694
|
+
nested_level_value: INSERT_ID2
|
1695
|
+
)
|
1662
1696
|
end
|
1663
1697
|
|
1664
1698
|
def test_cascading_json_detection_with_log_entry_labels_field
|
@@ -1668,7 +1702,9 @@ module BaseTest
|
|
1668
1702
|
nested_level_value: LABELS_MESSAGE2,
|
1669
1703
|
expected_value_from_root: COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE),
|
1670
1704
|
expected_value_from_nested: COMPUTE_PARAMS[:labels].merge(
|
1671
|
-
LABELS_MESSAGE2
|
1705
|
+
LABELS_MESSAGE2
|
1706
|
+
)
|
1707
|
+
)
|
1672
1708
|
end
|
1673
1709
|
|
1674
1710
|
def test_cascading_json_detection_with_log_entry_operation_field
|
@@ -1676,28 +1712,32 @@ module BaseTest
|
|
1676
1712
|
'operation', DEFAULT_OPERATION_KEY,
|
1677
1713
|
root_level_value: OPERATION_MESSAGE,
|
1678
1714
|
nested_level_value: OPERATION_MESSAGE2,
|
1679
|
-
expected_value_from_nested: expected_operation_message2
|
1715
|
+
expected_value_from_nested: expected_operation_message2
|
1716
|
+
)
|
1680
1717
|
end
|
1681
1718
|
|
1682
1719
|
def test_cascading_json_detection_with_log_entry_source_location_field
|
1683
1720
|
verify_cascading_json_detection_with_log_entry_fields(
|
1684
1721
|
'sourceLocation', DEFAULT_SOURCE_LOCATION_KEY,
|
1685
1722
|
root_level_value: SOURCE_LOCATION_MESSAGE,
|
1686
|
-
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1723
|
+
nested_level_value: SOURCE_LOCATION_MESSAGE2
|
1724
|
+
)
|
1687
1725
|
end
|
1688
1726
|
|
1689
1727
|
def test_cascading_json_detection_with_log_entry_span_id_field
|
1690
1728
|
verify_cascading_json_detection_with_log_entry_fields(
|
1691
1729
|
'spanId', DEFAULT_SPAN_ID_KEY,
|
1692
1730
|
root_level_value: SPAN_ID,
|
1693
|
-
nested_level_value: SPAN_ID2
|
1731
|
+
nested_level_value: SPAN_ID2
|
1732
|
+
)
|
1694
1733
|
end
|
1695
1734
|
|
1696
1735
|
def test_cascading_json_detection_with_log_entry_trace_field
|
1697
1736
|
verify_cascading_json_detection_with_log_entry_fields(
|
1698
1737
|
'trace', DEFAULT_TRACE_KEY,
|
1699
1738
|
root_level_value: TRACE,
|
1700
|
-
nested_level_value: TRACE2
|
1739
|
+
nested_level_value: TRACE2
|
1740
|
+
)
|
1701
1741
|
end
|
1702
1742
|
|
1703
1743
|
def test_cascading_json_detection_with_log_entry_trace_sampled_field
|
@@ -1706,7 +1746,8 @@ module BaseTest
|
|
1706
1746
|
root_level_value: TRACE_SAMPLED,
|
1707
1747
|
nested_level_value: TRACE_SAMPLED2,
|
1708
1748
|
default_value_from_root: false,
|
1709
|
-
default_value_from_nested: false
|
1749
|
+
default_value_from_nested: false
|
1750
|
+
)
|
1710
1751
|
end
|
1711
1752
|
|
1712
1753
|
# Verify that labels present in multiple inputs respect the expected priority
|
@@ -1738,9 +1779,11 @@ module BaseTest
|
|
1738
1779
|
{
|
1739
1780
|
config: CONFIG_LABLES_AND_LABLE_MAP,
|
1740
1781
|
emitted_log: PAYLOAD_FOR_LABEL_MAP.merge(
|
1741
|
-
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1782
|
+
DEFAULT_LABELS_KEY => LABELS_MESSAGE
|
1783
|
+
),
|
1742
1784
|
expected_labels: LABELS_MESSAGE.merge(LABELS_FROM_LABELS_CONFIG).merge(
|
1743
|
-
LABELS_FROM_LABEL_MAP_CONFIG
|
1785
|
+
LABELS_FROM_LABEL_MAP_CONFIG
|
1786
|
+
)
|
1744
1787
|
},
|
1745
1788
|
# labels from the config "labels" and "label_map" conflict.
|
1746
1789
|
{
|
@@ -1760,14 +1803,16 @@ module BaseTest
|
|
1760
1803
|
{
|
1761
1804
|
config: CONFIG_LABEL_MAP_CONFLICTING,
|
1762
1805
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1763
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1806
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1807
|
+
),
|
1764
1808
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1765
1809
|
},
|
1766
1810
|
# All three types of labels conflict.
|
1767
1811
|
{
|
1768
1812
|
config: CONFIG_LABLES_AND_LABLE_MAP_CONFLICTING,
|
1769
1813
|
emitted_log: PAYLOAD_FOR_LABEL_MAP_CONFLICTING.merge(
|
1770
|
-
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1814
|
+
DEFAULT_LABELS_KEY => LABELS_FROM_PAYLOAD_CONFLICTING
|
1815
|
+
),
|
1771
1816
|
expected_labels: LABELS_FROM_PAYLOAD_CONFLICTING
|
1772
1817
|
}
|
1773
1818
|
].each do |test_params|
|
@@ -1779,7 +1824,8 @@ module BaseTest
|
|
1779
1824
|
d.run
|
1780
1825
|
end
|
1781
1826
|
expected_params = COMPUTE_PARAMS.merge(
|
1782
|
-
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1827
|
+
labels: COMPUTE_PARAMS[:labels].merge(test_params[:expected_labels])
|
1828
|
+
)
|
1783
1829
|
verify_log_entries(1, expected_params)
|
1784
1830
|
end
|
1785
1831
|
end
|
@@ -1851,7 +1897,8 @@ module BaseTest
|
|
1851
1897
|
config: APPLICATION_DEFAULT_CONFIG,
|
1852
1898
|
setup_k8s_stub: true,
|
1853
1899
|
log_entry: k8s_container_log_entry(
|
1854
|
-
log_entry(0)
|
1900
|
+
log_entry(0)
|
1901
|
+
).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
|
1855
1902
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1856
1903
|
},
|
1857
1904
|
{
|
@@ -1859,7 +1906,8 @@ module BaseTest
|
|
1859
1906
|
setup_k8s_stub: true,
|
1860
1907
|
log_entry: k8s_container_log_entry(
|
1861
1908
|
log_entry(0),
|
1862
|
-
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1909
|
+
local_resource_id: RANDOM_LOCAL_RESOURCE_ID
|
1910
|
+
),
|
1863
1911
|
expected_params: CONTAINER_FROM_TAG_PARAMS
|
1864
1912
|
}
|
1865
1913
|
].each do |test_params|
|
@@ -1978,7 +2026,8 @@ module BaseTest
|
|
1978
2026
|
assert_metric_value.call(
|
1979
2027
|
:uptime, expected, 'agent.googleapis.com/agent',
|
1980
2028
|
OpenCensus::Stats::Aggregation::Sum, d,
|
1981
|
-
version: Fluent::GoogleCloudOutput.version_string
|
2029
|
+
version: Fluent::GoogleCloudOutput.version_string
|
2030
|
+
)
|
1982
2031
|
rescue Test::Unit::AssertionFailedError
|
1983
2032
|
retry if (retries += 1) < 3
|
1984
2033
|
end
|
@@ -2012,12 +2061,12 @@ module BaseTest
|
|
2012
2061
|
(1..entry_count).each do |entry_index|
|
2013
2062
|
d.emit('message' => log_entry(entry_index.to_s))
|
2014
2063
|
end
|
2015
|
-
# rubocop:disable Lint/
|
2064
|
+
# rubocop:disable Lint/SuppressedException
|
2016
2065
|
begin
|
2017
2066
|
d.run
|
2018
2067
|
rescue mock_error_type
|
2019
2068
|
end
|
2020
|
-
# rubocop:enable Lint/
|
2069
|
+
# rubocop:enable Lint/SuppressedException
|
2021
2070
|
failed_requests_count, dropped_entries_count,
|
2022
2071
|
retried_entries_count = metric_values
|
2023
2072
|
|
@@ -2066,6 +2115,7 @@ module BaseTest
|
|
2066
2115
|
# assertion will fail in the case when a single metric contains time
|
2067
2116
|
# series with success and failure events.
|
2068
2117
|
next if code == ok_status_code
|
2118
|
+
|
2069
2119
|
assert_metric_value.call(:stackdriver_failed_requests_count,
|
2070
2120
|
failed_requests_count,
|
2071
2121
|
'agent.googleapis.com/agent',
|
@@ -2099,7 +2149,8 @@ module BaseTest
|
|
2099
2149
|
end
|
2100
2150
|
|
2101
2151
|
def container_log_entry_with_metadata(
|
2102
|
-
|
2152
|
+
log, container_name = K8S_CONTAINER_NAME
|
2153
|
+
)
|
2103
2154
|
{
|
2104
2155
|
log: log,
|
2105
2156
|
stream: K8S_STREAM,
|
@@ -2158,10 +2209,10 @@ module BaseTest
|
|
2158
2209
|
}
|
2159
2210
|
end
|
2160
2211
|
|
2161
|
-
def dataflow_log_entry(
|
2212
|
+
def dataflow_log_entry(index)
|
2162
2213
|
{
|
2163
2214
|
step: DATAFLOW_STEP_ID,
|
2164
|
-
message: log_entry(
|
2215
|
+
message: log_entry(index)
|
2165
2216
|
}
|
2166
2217
|
end
|
2167
2218
|
|
@@ -2174,10 +2225,10 @@ module BaseTest
|
|
2174
2225
|
}
|
2175
2226
|
end
|
2176
2227
|
|
2177
|
-
def ml_log_entry(
|
2228
|
+
def ml_log_entry(index)
|
2178
2229
|
{
|
2179
2230
|
name: ML_LOG_AREA,
|
2180
|
-
message: log_entry(
|
2231
|
+
message: log_entry(index)
|
2181
2232
|
}
|
2182
2233
|
end
|
2183
2234
|
|
@@ -2188,8 +2239,8 @@ module BaseTest
|
|
2188
2239
|
}
|
2189
2240
|
end
|
2190
2241
|
|
2191
|
-
def log_entry(
|
2192
|
-
"test log entry #{
|
2242
|
+
def log_entry(index)
|
2243
|
+
"test log entry #{index}"
|
2193
2244
|
end
|
2194
2245
|
|
2195
2246
|
# If check_exact_labels is true, assert 'labels' and 'expected_labels' match
|
@@ -2197,6 +2248,7 @@ module BaseTest
|
|
2197
2248
|
# 'expected_labels'.
|
2198
2249
|
def check_labels(expected_labels, labels, check_exact_labels = true)
|
2199
2250
|
return if expected_labels.empty? && labels.empty?
|
2251
|
+
|
2200
2252
|
expected_labels.each do |expected_key, expected_value|
|
2201
2253
|
assert labels.key?(expected_key), "Expected label #{expected_key} not" \
|
2202
2254
|
" found. Got labels: #{labels}."
|
@@ -2206,20 +2258,21 @@ module BaseTest
|
|
2206
2258
|
assert_equal expected_value, actual_value, "Value for #{expected_key}" \
|
2207
2259
|
" mismatch. Expected #{expected_value}. Got #{actual_value}"
|
2208
2260
|
end
|
2209
|
-
|
2210
|
-
|
2211
|
-
|
2212
|
-
|
2213
|
-
|
2261
|
+
return unless check_exact_labels
|
2262
|
+
|
2263
|
+
assert_equal expected_labels.length, labels.length, 'Expected ' \
|
2264
|
+
"#{expected_labels.length} labels: #{expected_labels}, got " \
|
2265
|
+
"#{labels.length} labels: #{labels}"
|
2214
2266
|
end
|
2215
2267
|
|
2216
|
-
def verify_default_log_entry_text(text,
|
2217
|
-
assert_equal "test log entry #{
|
2218
|
-
"Entry ##{
|
2268
|
+
def verify_default_log_entry_text(text, index, entry)
|
2269
|
+
assert_equal "test log entry #{index}", text,
|
2270
|
+
"Entry ##{index} had unexpected text: #{entry}"
|
2219
2271
|
end
|
2220
2272
|
|
2221
2273
|
# The caller can optionally provide a block which is called for each entry.
|
2222
|
-
def verify_json_log_entries(
|
2274
|
+
def verify_json_log_entries(expected_count, params,
|
2275
|
+
payload_type = 'textPayload',
|
2223
2276
|
check_exact_entry_labels = true)
|
2224
2277
|
entry_count = 0
|
2225
2278
|
@logs_sent.each do |request|
|
@@ -2257,11 +2310,11 @@ module BaseTest
|
|
2257
2310
|
entry)
|
2258
2311
|
end
|
2259
2312
|
entry_count += 1
|
2260
|
-
assert entry_count <=
|
2261
|
-
"Number of entries #{entry_count} exceeds expected number #{
|
2313
|
+
assert entry_count <= expected_count,
|
2314
|
+
"Number of entries #{entry_count} exceeds expected number #{expected_count}."
|
2262
2315
|
end
|
2263
2316
|
end
|
2264
|
-
assert_equal
|
2317
|
+
assert_equal expected_count, entry_count
|
2265
2318
|
end
|
2266
2319
|
|
2267
2320
|
def verify_container_logs(log_entry_factory, expected_params)
|
@@ -2290,13 +2343,17 @@ module BaseTest
|
|
2290
2343
|
# LogEntry info from. The values are lists of two elements: the name of
|
2291
2344
|
# the subfield in LogEntry object and the expected value of that field.
|
2292
2345
|
DEFAULT_HTTP_REQUEST_KEY => [
|
2293
|
-
'httpRequest', HTTP_REQUEST_MESSAGE
|
2346
|
+
'httpRequest', HTTP_REQUEST_MESSAGE
|
2347
|
+
],
|
2294
2348
|
DEFAULT_LABELS_KEY => [
|
2295
|
-
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2349
|
+
'labels', COMPUTE_PARAMS[:labels].merge(LABELS_MESSAGE)
|
2350
|
+
],
|
2296
2351
|
DEFAULT_OPERATION_KEY => [
|
2297
|
-
'operation', OPERATION_MESSAGE
|
2352
|
+
'operation', OPERATION_MESSAGE
|
2353
|
+
],
|
2298
2354
|
DEFAULT_SOURCE_LOCATION_KEY => [
|
2299
|
-
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2355
|
+
'sourceLocation', SOURCE_LOCATION_MESSAGE
|
2356
|
+
]
|
2300
2357
|
}
|
2301
2358
|
end
|
2302
2359
|
|
@@ -2396,17 +2453,22 @@ module BaseTest
|
|
2396
2453
|
# left with name "log", "message" or "msg". This test verifies additional
|
2397
2454
|
# LogEntry fields like spanId and traceId do not disable that by accident.
|
2398
2455
|
def verify_cascading_json_detection_with_log_entry_fields(
|
2399
|
-
|
2456
|
+
log_entry_field, default_key, expectation
|
2457
|
+
)
|
2400
2458
|
root_level_value = expectation[:root_level_value]
|
2401
2459
|
nested_level_value = expectation[:nested_level_value]
|
2402
2460
|
expected_value_from_root = expectation.fetch(
|
2403
|
-
:expected_value_from_root, root_level_value
|
2461
|
+
:expected_value_from_root, root_level_value
|
2462
|
+
)
|
2404
2463
|
expected_value_from_nested = expectation.fetch(
|
2405
|
-
:expected_value_from_nested, nested_level_value
|
2464
|
+
:expected_value_from_nested, nested_level_value
|
2465
|
+
)
|
2406
2466
|
default_value_from_root = expectation.fetch(
|
2407
|
-
:default_value_from_root, nil
|
2467
|
+
:default_value_from_root, nil
|
2468
|
+
)
|
2408
2469
|
default_value_from_nested = expectation.fetch(
|
2409
|
-
:default_value_from_nested, nil
|
2470
|
+
:default_value_from_nested, nil
|
2471
|
+
)
|
2410
2472
|
|
2411
2473
|
setup_gce_metadata_stubs
|
2412
2474
|
|
@@ -2442,7 +2504,8 @@ module BaseTest
|
|
2442
2504
|
# }
|
2443
2505
|
# }
|
2444
2506
|
log_entry_with_both_level_fields = log_entry_with_nested_level_field.merge(
|
2445
|
-
default_key => root_level_value
|
2507
|
+
default_key => root_level_value
|
2508
|
+
)
|
2446
2509
|
|
2447
2510
|
[
|
2448
2511
|
[
|
@@ -2593,7 +2656,8 @@ module BaseTest
|
|
2593
2656
|
|
2594
2657
|
# Verify the number and the content of the log entries match the expectation.
|
2595
2658
|
# The caller can optionally provide a block which is called for each entry.
|
2596
|
-
def verify_log_entries(
|
2659
|
+
def verify_log_entries(_expected_count, _params,
|
2660
|
+
_payload_type = 'textPayload',
|
2597
2661
|
_check_exact_entry_labels = true, &_block)
|
2598
2662
|
_undefined
|
2599
2663
|
end
|