fluent-plugin-google-cloud 0.6.3 → 0.6.4.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -6
- data/fluent-plugin-google-cloud.gemspec +2 -1
- data/lib/fluent/plugin/out_google_cloud.rb +733 -290
- data/test/plugin/base_test.rb +316 -77
- data/test/plugin/constants.rb +143 -0
- metadata +19 -6
- data/fluent-plugin-google-cloud-0.6.2.gem +0 -0
data/test/plugin/base_test.rb
CHANGED
@@ -99,7 +99,7 @@ module BaseTest
|
|
99
99
|
assert_equal PROJECT_ID, d.instance.project_id
|
100
100
|
assert_equal ZONE, d.instance.zone
|
101
101
|
assert_equal VM_ID, d.instance.vm_id
|
102
|
-
assert_equal
|
102
|
+
assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
|
103
103
|
end
|
104
104
|
|
105
105
|
def test_managed_vm_metadata_loading
|
@@ -110,9 +110,11 @@ module BaseTest
|
|
110
110
|
assert_equal PROJECT_ID, d.instance.project_id
|
111
111
|
assert_equal ZONE, d.instance.zone
|
112
112
|
assert_equal VM_ID, d.instance.vm_id
|
113
|
-
assert_equal
|
114
|
-
assert_equal MANAGED_VM_BACKEND_NAME,
|
115
|
-
|
113
|
+
assert_equal APPENGINE_CONSTANTS[:resource_type], d.instance.resource.type
|
114
|
+
assert_equal MANAGED_VM_BACKEND_NAME,
|
115
|
+
d.instance.resource.labels['module_id']
|
116
|
+
assert_equal MANAGED_VM_BACKEND_VERSION,
|
117
|
+
d.instance.resource.labels['version_id']
|
116
118
|
end
|
117
119
|
|
118
120
|
def test_gce_metadata_does_not_load_when_use_metadata_service_is_false
|
@@ -122,7 +124,7 @@ module BaseTest
|
|
122
124
|
assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
|
123
125
|
assert_equal CUSTOM_ZONE, d.instance.zone
|
124
126
|
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
125
|
-
assert_equal
|
127
|
+
assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
|
126
128
|
end
|
127
129
|
|
128
130
|
def test_gce_used_when_detect_subservice_is_false
|
@@ -156,8 +158,6 @@ module BaseTest
|
|
156
158
|
assert_equal parts[1], d.instance.project_id, "Index #{index} failed."
|
157
159
|
assert_equal parts[2], d.instance.zone, "Index #{index} failed."
|
158
160
|
assert_equal parts[3], d.instance.vm_id, "Index #{index} failed."
|
159
|
-
assert_equal false, d.instance.running_on_managed_vm,
|
160
|
-
"Index #{index} failed."
|
161
161
|
end
|
162
162
|
end
|
163
163
|
|
@@ -701,78 +701,17 @@ module BaseTest
|
|
701
701
|
end
|
702
702
|
end
|
703
703
|
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
d.emit(container_log_entry_with_metadata(log_entry(0)))
|
710
|
-
d.run
|
711
|
-
end
|
712
|
-
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS) do |entry|
|
713
|
-
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
714
|
-
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
715
|
-
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
716
|
-
end
|
717
|
-
end
|
718
|
-
|
719
|
-
def test_multiple_container_logs_metadata_from_plugin
|
720
|
-
setup_gce_metadata_stubs
|
721
|
-
setup_container_metadata_stubs
|
722
|
-
[2, 3, 5, 11, 50].each do |n|
|
723
|
-
@logs_sent = []
|
724
|
-
setup_logging_stubs do
|
725
|
-
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
726
|
-
# The test driver doesn't clear its buffer of entries after running, so
|
727
|
-
# do it manually here.
|
728
|
-
d.instance_variable_get('@entries').clear
|
729
|
-
n.times { |i| d.emit(container_log_entry_with_metadata(log_entry(i))) }
|
730
|
-
d.run
|
731
|
-
end
|
732
|
-
verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS) do |entry|
|
733
|
-
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
|
734
|
-
entry
|
735
|
-
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
736
|
-
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
737
|
-
end
|
738
|
-
end
|
739
|
-
end
|
740
|
-
|
741
|
-
def test_multiple_container_logs_metadata_from_tag
|
742
|
-
setup_gce_metadata_stubs
|
743
|
-
setup_container_metadata_stubs
|
744
|
-
[2, 3, 5, 11, 50].each do |n|
|
745
|
-
@logs_sent = []
|
746
|
-
setup_logging_stubs do
|
747
|
-
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
748
|
-
# The test driver doesn't clear its buffer of entries after running, so
|
749
|
-
# do it manually here.
|
750
|
-
d.instance_variable_get('@entries').clear
|
751
|
-
n.times { |i| d.emit(container_log_entry(log_entry(i))) }
|
752
|
-
d.run
|
753
|
-
end
|
754
|
-
verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS) do |entry|
|
755
|
-
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
|
756
|
-
entry
|
757
|
-
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
758
|
-
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
759
|
-
end
|
760
|
-
end
|
704
|
+
# Test container logs when metadata is extracted from the 'kubernetes' field
|
705
|
+
# in the log record.
|
706
|
+
def test_container_logs_metadata_from_record
|
707
|
+
verify_container_logs(method(:container_log_entry_with_metadata),
|
708
|
+
CONTAINER_FROM_METADATA_PARAMS)
|
761
709
|
end
|
762
710
|
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
768
|
-
d.emit(container_log_entry(log_entry(0)))
|
769
|
-
d.run
|
770
|
-
end
|
771
|
-
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS) do |entry|
|
772
|
-
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
773
|
-
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
774
|
-
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
775
|
-
end
|
711
|
+
# Test container logs when metadata is extracted from the tag.
|
712
|
+
def test_container_logs_metadata_from_tag
|
713
|
+
verify_container_logs(method(:container_log_entry),
|
714
|
+
CONTAINER_FROM_TAG_PARAMS)
|
776
715
|
end
|
777
716
|
|
778
717
|
def test_one_container_log_from_tag_stderr
|
@@ -839,6 +778,134 @@ module BaseTest
|
|
839
778
|
end
|
840
779
|
end
|
841
780
|
|
781
|
+
# Docker Container.
|
782
|
+
|
783
|
+
# Test textPayload logs from Docker container stderr / stdout.
|
784
|
+
def test_text_payload_docker_container_logs
|
785
|
+
setup_gce_metadata_stubs
|
786
|
+
[1, 2, 3, 5, 11, 50].each do |n|
|
787
|
+
@logs_sent = []
|
788
|
+
setup_logging_stubs do
|
789
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, DOCKER_CONTAINER_TAG)
|
790
|
+
# Generate log entries with 'log' (simply a string) and 'time' fields.
|
791
|
+
n.times { |i| d.emit(docker_container_log_entry(log_entry(i))) }
|
792
|
+
d.run
|
793
|
+
end
|
794
|
+
# 'container_id' and 'container_name' should have been extracted from tag
|
795
|
+
# and properly set in resource.labels and common metadata labels as in
|
796
|
+
# DOCKER_CONTAINER_PARAMS.
|
797
|
+
verify_log_entries(n, DOCKER_CONTAINER_PARAMS) do |entry|
|
798
|
+
# Timestamp in 'time' field from log entry should be set properly.
|
799
|
+
assert_equal DOCKER_CONTAINER_SECONDS_EPOCH,
|
800
|
+
entry['timestamp']['seconds'], entry
|
801
|
+
assert_equal DOCKER_CONTAINER_NANOS, entry['timestamp']['nanos'],
|
802
|
+
entry
|
803
|
+
# Severity is 'DEFAULT' because 'stream' info is absent from log entry.
|
804
|
+
assert_equal_with_default entry['severity'], 'DEFAULT', 'DEFAULT', entry
|
805
|
+
end
|
806
|
+
end
|
807
|
+
end
|
808
|
+
|
809
|
+
# Test jsonPayload logs from Docker container stderr / stdout.
|
810
|
+
def test_json_payload_docker_container_logs
|
811
|
+
setup_gce_metadata_stubs
|
812
|
+
[1, 2, 3, 5, 11, 50].each do |n|
|
813
|
+
@logs_sent = []
|
814
|
+
setup_logging_stubs do
|
815
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, DOCKER_CONTAINER_TAG)
|
816
|
+
n.times do
|
817
|
+
# Generate log entries with 'log' (json) and 'time' fields.
|
818
|
+
d.emit(docker_container_log_entry('{"msg": "test log entry ' \
|
819
|
+
"#{n}" \
|
820
|
+
'", "tag2": "test", "data": ' \
|
821
|
+
'5000, "severity": "WARNING"}'))
|
822
|
+
end
|
823
|
+
d.run
|
824
|
+
end
|
825
|
+
# 'container_id' and 'container_name' should have been extracted from tag
|
826
|
+
# and properly set in resource.labels and common metadata labels as in
|
827
|
+
# DOCKER_CONTAINER_PARAMS.
|
828
|
+
verify_log_entries(n, DOCKER_CONTAINER_PARAMS, 'jsonPayload') do |entry|
|
829
|
+
# 'log' field should be detected as json and parsed properly.
|
830
|
+
fields = get_fields(entry['jsonPayload'])
|
831
|
+
assert_equal 3, fields.size, entry
|
832
|
+
assert_equal "test log entry #{n}", get_string(fields['msg']), entry
|
833
|
+
assert_equal 'test', get_string(fields['tag2']), entry
|
834
|
+
assert_equal 5000, get_number(fields['data']), entry
|
835
|
+
# Timestamp in 'time' field from log entry should be set properly.
|
836
|
+
assert_equal DOCKER_CONTAINER_SECONDS_EPOCH,
|
837
|
+
entry['timestamp']['seconds'], entry
|
838
|
+
assert_equal DOCKER_CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
839
|
+
# Severity in 'severity' field from log entry should be set properly.
|
840
|
+
assert_equal 'WARNING', entry['severity'], entry
|
841
|
+
end
|
842
|
+
end
|
843
|
+
end
|
844
|
+
|
845
|
+
# Test the following metadata is properly extracted from json record:
|
846
|
+
# * container_id
|
847
|
+
# * container_name
|
848
|
+
# * source (this field maps to the 'stream' metadata label.
|
849
|
+
def test_docker_container_logs_metadata_from_json_record
|
850
|
+
setup_gce_metadata_stubs
|
851
|
+
{
|
852
|
+
# 'stream' label should be extracted from json record when present.
|
853
|
+
docker_container_log_entry_with_metadata(
|
854
|
+
log_entry(0)
|
855
|
+
) => {
|
856
|
+
# When the 'source' field from the json record has value 'stdout',
|
857
|
+
# 'severity' should be 'INFO'.
|
858
|
+
'params' => DOCKER_CONTAINER_PARAMS_WITH_STREAM_STDOUT,
|
859
|
+
'severity' => 'INFO'
|
860
|
+
},
|
861
|
+
# 'container_id' and 'container_name' can be extracted from tag or
|
862
|
+
# json record. If present in both cases, values in the json record will
|
863
|
+
# overwrite values in tags. For example:
|
864
|
+
# DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN has different
|
865
|
+
# 'container_id' and 'container_name' values from DOCKER_CONTAINER_TAG,
|
866
|
+
# and we need to verify these two fields get overwritten as expected.
|
867
|
+
docker_container_log_entry_with_metadata(
|
868
|
+
log_entry(0), DOCKER_CONTAINER_ID2, DOCKER_CONTAINER_NAME2,
|
869
|
+
DOCKER_CONTAINER_STREAM_STDERR
|
870
|
+
) => {
|
871
|
+
# When the 'source' field from the json record has value 'stderr',
|
872
|
+
# 'severity' should be 'ERROR'.
|
873
|
+
'params' => DOCKER_CONTAINER_PARAMS_WITH_METADATA_OVERWRITTEN,
|
874
|
+
'severity' => 'ERROR'
|
875
|
+
}
|
876
|
+
}.each do |log_entry, expected|
|
877
|
+
@logs_sent = []
|
878
|
+
setup_logging_stubs do
|
879
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, DOCKER_CONTAINER_TAG)
|
880
|
+
d.emit(log_entry)
|
881
|
+
d.run
|
882
|
+
end
|
883
|
+
verify_log_entries(1, expected['params']) do |entry|
|
884
|
+
assert_equal DOCKER_CONTAINER_SECONDS_EPOCH,
|
885
|
+
entry['timestamp']['seconds'], entry
|
886
|
+
assert_equal DOCKER_CONTAINER_NANOS, entry['timestamp']['nanos'],
|
887
|
+
entry
|
888
|
+
assert_equal_with_default entry['severity'], expected['severity'],
|
889
|
+
'DEFAULT', entry
|
890
|
+
end
|
891
|
+
end
|
892
|
+
end
|
893
|
+
|
894
|
+
# Test logs from applications running in Docker containers.
|
895
|
+
def test_docker_container_application_logs
|
896
|
+
setup_gce_metadata_stubs
|
897
|
+
setup_docker_remote_api_stubs
|
898
|
+
setup_logging_stubs do
|
899
|
+
# Metadata Agent is not enabled. Will call Docker Remote API for
|
900
|
+
# container info.
|
901
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG,
|
902
|
+
DOCKER_CONTAINER_TAG_WITH_APPLICATION)
|
903
|
+
d.emit('message' => log_entry(0))
|
904
|
+
d.run
|
905
|
+
end
|
906
|
+
verify_log_entries(1, DOCKER_CONTAINER_WITH_APPLICATION_PARAMS)
|
907
|
+
end
|
908
|
+
|
842
909
|
def test_cloudfunctions_log
|
843
910
|
setup_gce_metadata_stubs
|
844
911
|
setup_cloudfunctions_metadata_stubs
|
@@ -903,6 +970,17 @@ module BaseTest
|
|
903
970
|
end
|
904
971
|
end
|
905
972
|
|
973
|
+
def test_dataproc_log
|
974
|
+
setup_gce_metadata_stubs
|
975
|
+
setup_dataproc_metadata_stubs
|
976
|
+
setup_logging_stubs do
|
977
|
+
d = create_driver
|
978
|
+
d.emit(dataproc_log_entry('test message'))
|
979
|
+
d.run
|
980
|
+
end
|
981
|
+
verify_log_entries(1, DATAPROC_PARAMS, 'jsonPayload')
|
982
|
+
end
|
983
|
+
|
906
984
|
def test_http_request_from_record
|
907
985
|
setup_gce_metadata_stubs
|
908
986
|
setup_logging_stubs do
|
@@ -1003,6 +1081,72 @@ module BaseTest
|
|
1003
1081
|
end
|
1004
1082
|
end
|
1005
1083
|
|
1084
|
+
# Metadata Agent related tests.
|
1085
|
+
def test_configure_enable_metadata_agent_default_and_false
|
1086
|
+
setup_gce_metadata_stubs
|
1087
|
+
[create_driver, create_driver(DISABLE_METADATA_AGENT_CONFIG)].each do |d|
|
1088
|
+
assert_false d.instance.instance_variable_get(:@enable_metadata_agent)
|
1089
|
+
end
|
1090
|
+
end
|
1091
|
+
|
1092
|
+
def test_configure_enable_metadata_agent_true
|
1093
|
+
setup_gce_metadata_stubs
|
1094
|
+
setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
|
1095
|
+
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
1096
|
+
assert_true d.instance.instance_variable_get(:@enable_metadata_agent)
|
1097
|
+
end
|
1098
|
+
|
1099
|
+
# Test an implicit monitored resource can be retrieved from Metadata Agent
|
1100
|
+
# with an empty string as the locally-unique id.
|
1101
|
+
def test_retrieve_implicit_monitored_resource
|
1102
|
+
# Minimum set up for gce metadata stubs. Stubs for 'vm_id' and 'location'
|
1103
|
+
# info are deliberately not set up.
|
1104
|
+
setup_gce_metadata_stubs_minimum
|
1105
|
+
setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
|
1106
|
+
setup_logging_stubs do
|
1107
|
+
d = create_driver(ENABLE_METADATA_AGENT_CONFIG)
|
1108
|
+
d.emit('message' => log_entry(0))
|
1109
|
+
d.run
|
1110
|
+
end
|
1111
|
+
verify_log_entries(1, COMPUTE_PARAMS)
|
1112
|
+
end
|
1113
|
+
|
1114
|
+
# Test a docker container monitored resource can be retrieved from Metadata
|
1115
|
+
# Agent with "container.<container_id>" as the locally-unique id.
|
1116
|
+
def test_retrieve_docker_container_monitored_resource
|
1117
|
+
setup_gce_metadata_stubs_minimum
|
1118
|
+
setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
|
1119
|
+
setup_metadata_agent_stubs(DOCKER_CONSTANTS[:resource_type])
|
1120
|
+
setup_logging_stubs do
|
1121
|
+
# Tag format here: "container.<container_id>.<container_name>"
|
1122
|
+
d = create_driver(ENABLE_METADATA_AGENT_CONFIG, DOCKER_CONTAINER_TAG)
|
1123
|
+
d.emit('message' => log_entry(0))
|
1124
|
+
d.run
|
1125
|
+
end
|
1126
|
+
# gce_metadata_stubs has ZONE1, while metadata agent stub has ZONE2. Here we
|
1127
|
+
# need to verify ZONE2 overwrites ZONE1 as expected.
|
1128
|
+
verify_log_entries(1, DOCKER_CONTAINER_PARAMS_WITH_ZONE2)
|
1129
|
+
end
|
1130
|
+
|
1131
|
+
# Test a docker container monitored resource can be retrieved from Metadata
|
1132
|
+
# Agent with "container.<container_name>" as the locally-unique id when the
|
1133
|
+
# log entry comes from an application running in the docker container.
|
1134
|
+
def test_retrieve_docker_container_monitored_resource_with_application
|
1135
|
+
setup_gce_metadata_stubs_minimum
|
1136
|
+
setup_metadata_agent_stubs(IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
|
1137
|
+
setup_metadata_agent_stubs(
|
1138
|
+
"#{DOCKER_CONSTANTS[:resource_type]}_application")
|
1139
|
+
setup_logging_stubs do
|
1140
|
+
# Tag format here: "application-container.<container_name>.
|
1141
|
+
# <additional_tag>"
|
1142
|
+
d = create_driver(ENABLE_METADATA_AGENT_CONFIG,
|
1143
|
+
DOCKER_CONTAINER_TAG_WITH_APPLICATION)
|
1144
|
+
d.emit('message' => log_entry(0))
|
1145
|
+
d.run
|
1146
|
+
end
|
1147
|
+
verify_log_entries(1, DOCKER_CONTAINER_WITH_APPLICATION_PARAMS)
|
1148
|
+
end
|
1149
|
+
|
1006
1150
|
private
|
1007
1151
|
|
1008
1152
|
def stub_metadata_request(metadata_path, response_body)
|
@@ -1039,6 +1183,26 @@ module BaseTest
|
|
1039
1183
|
'Content-Type' => 'application/json' })
|
1040
1184
|
end
|
1041
1185
|
|
1186
|
+
# The minimum stubs needed for infomation that Metadata Agent can not provide.
|
1187
|
+
def setup_gce_metadata_stubs_minimum
|
1188
|
+
# Stub the root, used for platform detection by the plugin and 'googleauth'.
|
1189
|
+
stub_request(:get, 'http://169.254.169.254')
|
1190
|
+
.to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
|
1191
|
+
|
1192
|
+
# Create stubs for all the GCE metadata lookups the agent needs to make.
|
1193
|
+
stub_metadata_request('project/project-id', PROJECT_ID)
|
1194
|
+
stub_metadata_request('instance/attributes/',
|
1195
|
+
"attribute1\nattribute2\nattribute3")
|
1196
|
+
|
1197
|
+
# Used by 'googleauth' to fetch the default service account credentials.
|
1198
|
+
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
|
1199
|
+
'instance/service-accounts/default/token')
|
1200
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
1201
|
+
status: 200,
|
1202
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
1203
|
+
'Content-Type' => 'application/json' })
|
1204
|
+
end
|
1205
|
+
|
1042
1206
|
def setup_ec2_metadata_stubs
|
1043
1207
|
# Stub the root, used for platform detection.
|
1044
1208
|
stub_request(:get, 'http://169.254.169.254')
|
@@ -1089,6 +1253,11 @@ module BaseTest
|
|
1089
1253
|
'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
|
1090
1254
|
end
|
1091
1255
|
|
1256
|
+
def setup_docker_remote_api_stubs
|
1257
|
+
stub_request(:get, "http://unix/containers/#{DOCKER_CONTAINER_NAME}/json")
|
1258
|
+
.to_return(status: 200, body: "{\"Id\":\"#{DOCKER_CONTAINER_ID}\"}")
|
1259
|
+
end
|
1260
|
+
|
1092
1261
|
def setup_cloudfunctions_metadata_stubs
|
1093
1262
|
stub_metadata_request(
|
1094
1263
|
'instance/attributes/',
|
@@ -1102,11 +1271,32 @@ module BaseTest
|
|
1102
1271
|
CLOUDFUNCTIONS_REGION)
|
1103
1272
|
end
|
1104
1273
|
|
1274
|
+
def setup_dataproc_metadata_stubs
|
1275
|
+
stub_metadata_request(
|
1276
|
+
'instance/attributes/',
|
1277
|
+
"attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
|
1278
|
+
stub_metadata_request('instance/attributes/dataproc-cluster-name',
|
1279
|
+
DATAPROC_CLUSTER_NAME)
|
1280
|
+
stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
|
1281
|
+
DATAPROC_CLUSTER_UUID)
|
1282
|
+
stub_metadata_request('instance/attributes/dataproc-region',
|
1283
|
+
DATAPROC_REGION)
|
1284
|
+
end
|
1285
|
+
|
1286
|
+
def setup_metadata_agent_stubs(resource_type)
|
1287
|
+
unique_id = MONITORED_RESOURCE_STUBS[resource_type]['unique_id']
|
1288
|
+
resource = MONITORED_RESOURCE_STUBS[resource_type]['monitored_resource']
|
1289
|
+
stub_request(:get, 'http://local-metadata-agent.stackdriver.com:8000' \
|
1290
|
+
"/monitoredResource/#{unique_id}")
|
1291
|
+
.to_return(status: 200, body: resource)
|
1292
|
+
end
|
1293
|
+
|
1105
1294
|
def container_tag_with_container_name(container_name)
|
1106
1295
|
"kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
|
1107
1296
|
"#{container_name}"
|
1108
1297
|
end
|
1109
1298
|
|
1299
|
+
# GKE Container
|
1110
1300
|
def container_log_entry_with_metadata(
|
1111
1301
|
log, container_name = CONTAINER_CONTAINER_NAME)
|
1112
1302
|
{
|
@@ -1134,6 +1324,27 @@ module BaseTest
|
|
1134
1324
|
}
|
1135
1325
|
end
|
1136
1326
|
|
1327
|
+
# Docker Container
|
1328
|
+
def docker_container_log_entry_with_metadata(
|
1329
|
+
log, container_id = DOCKER_CONTAINER_ID,
|
1330
|
+
container_name = DOCKER_CONTAINER_NAME,
|
1331
|
+
stream = DOCKER_CONTAINER_STREAM_STDOUT)
|
1332
|
+
{
|
1333
|
+
log: log,
|
1334
|
+
container_id: container_id,
|
1335
|
+
container_name: container_name,
|
1336
|
+
time: DOCKER_CONTAINER_TIMESTAMP,
|
1337
|
+
source: stream
|
1338
|
+
}
|
1339
|
+
end
|
1340
|
+
|
1341
|
+
def docker_container_log_entry(log)
|
1342
|
+
{
|
1343
|
+
log: log,
|
1344
|
+
time: CONTAINER_TIMESTAMP
|
1345
|
+
}
|
1346
|
+
end
|
1347
|
+
|
1137
1348
|
def cloudfunctions_log_entry(i)
|
1138
1349
|
{
|
1139
1350
|
stream: 'stdout',
|
@@ -1155,6 +1366,15 @@ module BaseTest
|
|
1155
1366
|
}
|
1156
1367
|
end
|
1157
1368
|
|
1369
|
+
def dataproc_log_entry(message, source_class = 'com.example.Example',
|
1370
|
+
filename = 'test.log')
|
1371
|
+
{
|
1372
|
+
filename: filename,
|
1373
|
+
class: source_class,
|
1374
|
+
message: log_entry(message)
|
1375
|
+
}
|
1376
|
+
end
|
1377
|
+
|
1158
1378
|
def ml_log_entry(i)
|
1159
1379
|
{
|
1160
1380
|
name: ML_LOG_AREA,
|
@@ -1215,6 +1435,25 @@ module BaseTest
|
|
1215
1435
|
assert i == n, "Number of entries #{i} does not match expected number #{n}"
|
1216
1436
|
end
|
1217
1437
|
|
1438
|
+
def verify_container_logs(log_entry_method, expected_params)
|
1439
|
+
setup_gce_metadata_stubs
|
1440
|
+
setup_container_metadata_stubs
|
1441
|
+
[1, 2, 3, 5, 11, 50].each do |n|
|
1442
|
+
@logs_sent = []
|
1443
|
+
setup_logging_stubs do
|
1444
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
1445
|
+
n.times { |i| d.emit(log_entry_method.call(log_entry(i))) }
|
1446
|
+
d.run
|
1447
|
+
end
|
1448
|
+
verify_log_entries(n, expected_params) do |entry|
|
1449
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
|
1450
|
+
entry
|
1451
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
1452
|
+
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
1453
|
+
end
|
1454
|
+
end
|
1455
|
+
end
|
1456
|
+
|
1218
1457
|
# Replace the 'referer' field with nil.
|
1219
1458
|
def http_request_message_with_nil_referer
|
1220
1459
|
HTTP_REQUEST_MESSAGE.merge('referer' => nil)
|