vmik-fluent-plugin-google-cloud 0.5.5 → 0.6.4.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile.lock +159 -0
- data/fluent-plugin-google-cloud.gemspec +9 -7
- data/lib/fluent/plugin/monitoring.rb +67 -0
- data/lib/fluent/plugin/out_google_cloud.rb +481 -224
- data/test/plugin/base_test.rb +263 -447
- data/test/plugin/constants.rb +452 -0
- data/test/plugin/test_out_google_cloud.rb +62 -46
- data/test/plugin/test_out_google_cloud_grpc.rb +66 -61
- metadata +83 -109
- data/lib/google/logging/type/http_request_pb.rb +0 -30
- data/lib/google/logging/type/log_severity_pb.rb +0 -26
- data/lib/google/logging/v1/log_entry_pb.rb +0 -52
- data/lib/google/logging/v1/logging_pb.rb +0 -84
- data/lib/google/logging/v1/logging_services_pb.rb +0 -150
data/test/plugin/base_test.rb
CHANGED
@@ -16,9 +16,14 @@ require 'google/apis'
|
|
16
16
|
require 'helper'
|
17
17
|
require 'mocha/test_unit'
|
18
18
|
require 'webmock/test_unit'
|
19
|
+
require 'prometheus/client'
|
20
|
+
|
21
|
+
require_relative 'constants'
|
19
22
|
|
20
23
|
# Unit tests for Google Cloud Logging plugin
|
21
24
|
module BaseTest
|
25
|
+
include Constants
|
26
|
+
|
22
27
|
def setup
|
23
28
|
Fluent::Test.setup
|
24
29
|
# delete environment variables that googleauth uses to find credentials.
|
@@ -37,317 +42,6 @@ module BaseTest
|
|
37
42
|
@logs_sent = []
|
38
43
|
end
|
39
44
|
|
40
|
-
# generic attributes
|
41
|
-
HOSTNAME = Socket.gethostname
|
42
|
-
|
43
|
-
# attributes used for the GCE metadata service
|
44
|
-
PROJECT_ID = 'test-project-id'
|
45
|
-
ZONE = 'us-central1-b'
|
46
|
-
FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
47
|
-
VM_ID = '9876543210'
|
48
|
-
|
49
|
-
# attributes used for custom (overridden) configs
|
50
|
-
CUSTOM_PROJECT_ID = 'test-custom-project-id'
|
51
|
-
CUSTOM_ZONE = 'us-custom-central1-b'
|
52
|
-
CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
|
53
|
-
CUSTOM_VM_ID = 'C9876543210'
|
54
|
-
CUSTOM_HOSTNAME = 'custom.hostname.org'
|
55
|
-
|
56
|
-
# attributes used for the EC2 metadata service
|
57
|
-
EC2_PROJECT_ID = 'test-ec2-project-id'
|
58
|
-
EC2_ZONE = 'us-west-2b'
|
59
|
-
EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
|
60
|
-
EC2_VM_ID = 'i-81c16767'
|
61
|
-
EC2_ACCOUNT_ID = '123456789012'
|
62
|
-
|
63
|
-
# The formatting here matches the format used on the VM.
|
64
|
-
EC2_IDENTITY_DOCUMENT = %({
|
65
|
-
"accountId" : "#{EC2_ACCOUNT_ID}",
|
66
|
-
"availabilityZone" : "#{EC2_ZONE}",
|
67
|
-
"instanceId" : "#{EC2_VM_ID}"
|
68
|
-
})
|
69
|
-
|
70
|
-
# Managed VMs specific labels
|
71
|
-
MANAGED_VM_BACKEND_NAME = 'default'
|
72
|
-
MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
|
73
|
-
|
74
|
-
# Container Engine / Kubernetes specific labels
|
75
|
-
CONTAINER_CLUSTER_NAME = 'cluster-1'
|
76
|
-
CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
|
77
|
-
CONTAINER_NAMESPACE_NAME = 'kube-system'
|
78
|
-
CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
|
79
|
-
CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
|
80
|
-
CONTAINER_CONTAINER_NAME = 'redis'
|
81
|
-
CONTAINER_LABEL_KEY = 'component'
|
82
|
-
CONTAINER_LABEL_VALUE = 'redis-component'
|
83
|
-
CONTAINER_STREAM = 'stdout'
|
84
|
-
CONTAINER_SEVERITY = 'INFO'
|
85
|
-
# Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch
|
86
|
-
CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
|
87
|
-
CONTAINER_SECONDS_EPOCH = 1_234_567_890
|
88
|
-
CONTAINER_NANOS = 987_654_321
|
89
|
-
|
90
|
-
# Cloud Functions specific labels
|
91
|
-
CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
|
92
|
-
CLOUDFUNCTIONS_REGION = 'us-central1'
|
93
|
-
CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
|
94
|
-
CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
|
95
|
-
CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
|
96
|
-
CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
|
97
|
-
CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
|
98
|
-
|
99
|
-
# Parameters used for authentication
|
100
|
-
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
101
|
-
FAKE_AUTH_TOKEN = 'abc123'
|
102
|
-
|
103
|
-
# Information about test credentials files.
|
104
|
-
# path: Path to the credentials file.
|
105
|
-
# project_id: ID of the project, which must correspond to the file contents.
|
106
|
-
IAM_CREDENTIALS = {
|
107
|
-
path: 'test/plugin/data/iam-credentials.json',
|
108
|
-
project_id: 'fluent-test-project'
|
109
|
-
}
|
110
|
-
LEGACY_CREDENTIALS = {
|
111
|
-
path: 'test/plugin/data/credentials.json',
|
112
|
-
project_id: '847859579879'
|
113
|
-
}
|
114
|
-
INVALID_CREDENTIALS = {
|
115
|
-
path: 'test/plugin/data/invalid_credentials.json',
|
116
|
-
project_id: ''
|
117
|
-
}
|
118
|
-
|
119
|
-
# Configuration files for various test scenarios
|
120
|
-
APPLICATION_DEFAULT_CONFIG = %(
|
121
|
-
)
|
122
|
-
|
123
|
-
# rubocop:disable Metrics/LineLength
|
124
|
-
PRIVATE_KEY_CONFIG = %(
|
125
|
-
auth_method private_key
|
126
|
-
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
127
|
-
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
128
|
-
)
|
129
|
-
# rubocop:enable Metrics/LineLength
|
130
|
-
|
131
|
-
REQUIRE_VALID_TAGS_CONFIG = %(
|
132
|
-
require_valid_tags true
|
133
|
-
)
|
134
|
-
|
135
|
-
NO_METADATA_SERVICE_CONFIG = %(
|
136
|
-
use_metadata_service false
|
137
|
-
)
|
138
|
-
|
139
|
-
NO_DETECT_SUBSERVICE_CONFIG = %(
|
140
|
-
detect_subservice false
|
141
|
-
)
|
142
|
-
|
143
|
-
CUSTOM_METADATA_CONFIG = %(
|
144
|
-
project_id #{CUSTOM_PROJECT_ID}
|
145
|
-
zone #{CUSTOM_ZONE}
|
146
|
-
vm_id #{CUSTOM_VM_ID}
|
147
|
-
vm_name #{CUSTOM_HOSTNAME}
|
148
|
-
)
|
149
|
-
|
150
|
-
CONFIG_MISSING_METADATA_PROJECT_ID = %(
|
151
|
-
zone #{CUSTOM_ZONE}
|
152
|
-
vm_id #{CUSTOM_VM_ID}
|
153
|
-
)
|
154
|
-
CONFIG_MISSING_METADATA_ZONE = %(
|
155
|
-
project_id #{CUSTOM_PROJECT_ID}
|
156
|
-
vm_id #{CUSTOM_VM_ID}
|
157
|
-
)
|
158
|
-
CONFIG_MISSING_METADATA_VM_ID = %(
|
159
|
-
project_id #{CUSTOM_PROJECT_ID}
|
160
|
-
zone #{CUSTOM_ZONE}
|
161
|
-
)
|
162
|
-
CONFIG_MISSING_METADATA_ALL = %(
|
163
|
-
)
|
164
|
-
|
165
|
-
CONFIG_EC2_PROJECT_ID = %(
|
166
|
-
project_id #{EC2_PROJECT_ID}
|
167
|
-
)
|
168
|
-
|
169
|
-
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
|
170
|
-
project_id #{EC2_PROJECT_ID}
|
171
|
-
vm_id #{CUSTOM_VM_ID}
|
172
|
-
)
|
173
|
-
|
174
|
-
# Service configurations for various services
|
175
|
-
COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
|
176
|
-
APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
|
177
|
-
CONTAINER_SERVICE_NAME = 'container.googleapis.com'
|
178
|
-
CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
|
179
|
-
EC2_SERVICE_NAME = 'ec2.amazonaws.com'
|
180
|
-
|
181
|
-
COMPUTE_PARAMS = {
|
182
|
-
service_name: COMPUTE_SERVICE_NAME,
|
183
|
-
log_name: 'test',
|
184
|
-
project_id: PROJECT_ID,
|
185
|
-
zone: ZONE,
|
186
|
-
labels: {
|
187
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
188
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
189
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
190
|
-
}
|
191
|
-
}
|
192
|
-
|
193
|
-
VMENGINE_PARAMS = {
|
194
|
-
service_name: APPENGINE_SERVICE_NAME,
|
195
|
-
log_name: "#{APPENGINE_SERVICE_NAME}%2Ftest",
|
196
|
-
project_id: PROJECT_ID,
|
197
|
-
zone: ZONE,
|
198
|
-
labels: {
|
199
|
-
"#{APPENGINE_SERVICE_NAME}/module_id" => MANAGED_VM_BACKEND_NAME,
|
200
|
-
"#{APPENGINE_SERVICE_NAME}/version_id" => MANAGED_VM_BACKEND_VERSION,
|
201
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
202
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
203
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
204
|
-
}
|
205
|
-
}
|
206
|
-
|
207
|
-
CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
|
208
|
-
"#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
|
209
|
-
|
210
|
-
CONTAINER_FROM_METADATA_PARAMS = {
|
211
|
-
service_name: CONTAINER_SERVICE_NAME,
|
212
|
-
log_name: CONTAINER_CONTAINER_NAME,
|
213
|
-
project_id: PROJECT_ID,
|
214
|
-
zone: ZONE,
|
215
|
-
labels: {
|
216
|
-
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
217
|
-
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
|
218
|
-
"#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
|
219
|
-
"#{CONTAINER_SERVICE_NAME}/namespace_id" => CONTAINER_NAMESPACE_ID,
|
220
|
-
"#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
|
221
|
-
"#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
|
222
|
-
"#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
|
223
|
-
"#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
|
224
|
-
"label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
|
225
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
226
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
227
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
228
|
-
}
|
229
|
-
}
|
230
|
-
|
231
|
-
# Almost the same as from metadata, but missing namespace_id and pod_id.
|
232
|
-
CONTAINER_FROM_TAG_PARAMS = {
|
233
|
-
service_name: CONTAINER_SERVICE_NAME,
|
234
|
-
log_name: CONTAINER_CONTAINER_NAME,
|
235
|
-
project_id: PROJECT_ID,
|
236
|
-
zone: ZONE,
|
237
|
-
labels: {
|
238
|
-
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
239
|
-
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
|
240
|
-
"#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
|
241
|
-
"#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
|
242
|
-
"#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
|
243
|
-
"#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
|
244
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
245
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
246
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
247
|
-
}
|
248
|
-
}
|
249
|
-
|
250
|
-
CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
|
251
|
-
"#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
|
252
|
-
"#{CLOUDFUNCTIONS_CONTAINER_NAME}"
|
253
|
-
|
254
|
-
CLOUDFUNCTIONS_PARAMS = {
|
255
|
-
service_name: CLOUDFUNCTIONS_SERVICE_NAME,
|
256
|
-
log_name: 'cloud-functions',
|
257
|
-
project_id: PROJECT_ID,
|
258
|
-
zone: ZONE,
|
259
|
-
labels: {
|
260
|
-
'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
|
261
|
-
"#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
|
262
|
-
CLOUDFUNCTIONS_FUNCTION_NAME,
|
263
|
-
"#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
|
264
|
-
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
265
|
-
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
|
266
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
267
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
268
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
269
|
-
}
|
270
|
-
}
|
271
|
-
|
272
|
-
CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
|
273
|
-
service_name: CLOUDFUNCTIONS_SERVICE_NAME,
|
274
|
-
log_name: 'cloud-functions',
|
275
|
-
project_id: PROJECT_ID,
|
276
|
-
zone: ZONE,
|
277
|
-
labels: {
|
278
|
-
"#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
|
279
|
-
CLOUDFUNCTIONS_FUNCTION_NAME,
|
280
|
-
"#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
|
281
|
-
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
282
|
-
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
|
283
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
284
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
285
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
286
|
-
}
|
287
|
-
}
|
288
|
-
|
289
|
-
CUSTOM_PARAMS = {
|
290
|
-
service_name: COMPUTE_SERVICE_NAME,
|
291
|
-
log_name: 'test',
|
292
|
-
project_id: CUSTOM_PROJECT_ID,
|
293
|
-
zone: CUSTOM_ZONE,
|
294
|
-
labels: {
|
295
|
-
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
296
|
-
"#{COMPUTE_SERVICE_NAME}/resource_id" => CUSTOM_VM_ID,
|
297
|
-
"#{COMPUTE_SERVICE_NAME}/resource_name" => CUSTOM_HOSTNAME
|
298
|
-
}
|
299
|
-
}
|
300
|
-
|
301
|
-
EC2_PARAMS = {
|
302
|
-
service_name: EC2_SERVICE_NAME,
|
303
|
-
log_name: 'test',
|
304
|
-
project_id: EC2_PROJECT_ID,
|
305
|
-
zone: EC2_PREFIXED_ZONE,
|
306
|
-
labels: {
|
307
|
-
"#{EC2_SERVICE_NAME}/resource_type" => 'instance',
|
308
|
-
"#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
|
309
|
-
"#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID,
|
310
|
-
"#{EC2_SERVICE_NAME}/resource_name" => HOSTNAME
|
311
|
-
}
|
312
|
-
}
|
313
|
-
|
314
|
-
HTTP_REQUEST_MESSAGE = {
|
315
|
-
'requestMethod' => 'POST',
|
316
|
-
'requestUrl' => 'http://example/',
|
317
|
-
'requestSize' => 210,
|
318
|
-
'status' => 200,
|
319
|
-
'responseSize' => 65,
|
320
|
-
'userAgent' => 'USER AGENT 1.0',
|
321
|
-
'remoteIp' => '55.55.55.55',
|
322
|
-
'referer' => 'http://referer/',
|
323
|
-
'cacheHit' => true,
|
324
|
-
'cacheValidatedWithOriginServer' => true
|
325
|
-
}
|
326
|
-
|
327
|
-
# Tags and their sanitized and encoded version.
|
328
|
-
VALID_TAGS = {
|
329
|
-
'test' => 'test',
|
330
|
-
'germanß' => 'german%C3%9F',
|
331
|
-
'chinese中' => 'chinese%E4%B8%AD',
|
332
|
-
'specialCharacter/_-.' => 'specialCharacter%2F_-.',
|
333
|
-
'abc@&^$*' => 'abc%40%26%5E%24%2A',
|
334
|
-
'@&^$*' => '%40%26%5E%24%2A'
|
335
|
-
}
|
336
|
-
INVALID_TAGS = {
|
337
|
-
# Non-string tags.
|
338
|
-
123 => '123',
|
339
|
-
1.23 => '1.23',
|
340
|
-
[1, 2, 3] => '%5B1%2C%202%2C%203%5D',
|
341
|
-
{ key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
|
342
|
-
# Non-utf8 string tags.
|
343
|
-
"nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
|
344
|
-
"abc#{[0x92].pack('C*')}" => 'abc%20',
|
345
|
-
"#{[0x92].pack('C*')}" => '%20',
|
346
|
-
# Empty string tag.
|
347
|
-
'' => '_'
|
348
|
-
}
|
349
|
-
ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
|
350
|
-
|
351
45
|
# Shared tests.
|
352
46
|
|
353
47
|
def test_configure_service_account_application_default
|
@@ -434,12 +128,12 @@ module BaseTest
|
|
434
128
|
|
435
129
|
def test_gce_used_when_detect_subservice_is_false
|
436
130
|
setup_gce_metadata_stubs
|
437
|
-
# This would cause the
|
438
|
-
# detect_subservice=false config.
|
131
|
+
# This would cause the resource type to be container.googleapis.com if not
|
132
|
+
# for the detect_subservice=false config.
|
439
133
|
setup_container_metadata_stubs
|
440
134
|
d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
|
441
135
|
d.run
|
442
|
-
assert_equal
|
136
|
+
assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
|
443
137
|
end
|
444
138
|
|
445
139
|
def test_metadata_overrides
|
@@ -554,7 +248,7 @@ module BaseTest
|
|
554
248
|
verify_log_entries(1, EC2_PARAMS)
|
555
249
|
end
|
556
250
|
|
557
|
-
def
|
251
|
+
def test_structured_payload_log
|
558
252
|
setup_gce_metadata_stubs
|
559
253
|
setup_logging_stubs do
|
560
254
|
d = create_driver
|
@@ -562,8 +256,8 @@ module BaseTest
|
|
562
256
|
'some_null_field' => nil)
|
563
257
|
d.run
|
564
258
|
end
|
565
|
-
verify_log_entries(1, COMPUTE_PARAMS, '
|
566
|
-
fields = get_fields(entry['
|
259
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
|
260
|
+
fields = get_fields(entry['jsonPayload'])
|
567
261
|
assert_equal 4, fields.size, entry
|
568
262
|
assert_equal 'test log entry 0', get_string(fields['msg']), entry
|
569
263
|
assert_equal 'test', get_string(fields['tag2']), entry
|
@@ -572,7 +266,7 @@ module BaseTest
|
|
572
266
|
end
|
573
267
|
end
|
574
268
|
|
575
|
-
def
|
269
|
+
def test_structured_payload_malformatted_log
|
576
270
|
setup_gce_metadata_stubs
|
577
271
|
message = 'test message'
|
578
272
|
setup_logging_stubs do
|
@@ -588,8 +282,8 @@ module BaseTest
|
|
588
282
|
)
|
589
283
|
d.run
|
590
284
|
end
|
591
|
-
verify_log_entries(1, COMPUTE_PARAMS, '
|
592
|
-
fields = get_fields(entry['
|
285
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
|
286
|
+
fields = get_fields(entry['jsonPayload'])
|
593
287
|
assert_equal 7, fields.size, entry
|
594
288
|
assert_equal message, get_string(get_fields(get_struct(fields \
|
595
289
|
['int_key']))['1']), entry
|
@@ -608,7 +302,7 @@ module BaseTest
|
|
608
302
|
end
|
609
303
|
end
|
610
304
|
|
611
|
-
def
|
305
|
+
def test_structured_payload_json_log
|
612
306
|
setup_gce_metadata_stubs
|
613
307
|
setup_logging_stubs do
|
614
308
|
d = create_driver
|
@@ -624,7 +318,7 @@ module BaseTest
|
|
624
318
|
end
|
625
319
|
end
|
626
320
|
|
627
|
-
def
|
321
|
+
def test_structured_payload_json_container_log
|
628
322
|
setup_gce_metadata_stubs
|
629
323
|
setup_container_metadata_stubs
|
630
324
|
setup_logging_stubs do
|
@@ -643,8 +337,8 @@ module BaseTest
|
|
643
337
|
if log_index == 1
|
644
338
|
assert entry.key?('textPayload'), 'Entry did not have textPayload'
|
645
339
|
else
|
646
|
-
assert entry.key?('
|
647
|
-
fields = get_fields(entry['
|
340
|
+
assert entry.key?('jsonPayload'), 'Entry did not have jsonPayload'
|
341
|
+
fields = get_fields(entry['jsonPayload'])
|
648
342
|
assert_equal 4, fields.size, entry
|
649
343
|
assert_equal 'test log entry 0', get_string(fields['msg']), entry
|
650
344
|
assert_equal 'test', get_string(fields['tag2']), entry
|
@@ -665,7 +359,7 @@ module BaseTest
|
|
665
359
|
d.emit('msg' => log_entry(0))
|
666
360
|
d.run
|
667
361
|
end
|
668
|
-
verify_log_entries(0, COMPUTE_PARAMS, '
|
362
|
+
verify_log_entries(0, COMPUTE_PARAMS, 'jsonPayload')
|
669
363
|
end
|
670
364
|
end
|
671
365
|
|
@@ -678,18 +372,17 @@ module BaseTest
|
|
678
372
|
# This tag will not match the kubernetes regex because it requires a
|
679
373
|
# non-empty container name.
|
680
374
|
tag = container_tag_with_container_name(container_name)
|
681
|
-
|
682
|
-
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
683
|
-
"#{CONTAINER_SERVICE_NAME}/container_name" => container_name),
|
684
|
-
log_name: tag)
|
685
|
-
setup_logging_stubs([params]) do
|
686
|
-
@logs_sent = []
|
375
|
+
setup_logging_stubs do
|
687
376
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
688
377
|
d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
|
689
378
|
d.run
|
690
379
|
end
|
380
|
+
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
381
|
+
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
382
|
+
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
383
|
+
'container_name' => container_name)),
|
384
|
+
log_name: tag)
|
691
385
|
verify_log_entries(1, params, 'textPayload')
|
692
|
-
assert_equal "projects/#{PROJECT_ID}/logs/#{tag}", @logs_sent[0]['logName']
|
693
386
|
end
|
694
387
|
|
695
388
|
# Verify that container names with non-utf8 characters should be rejected when
|
@@ -701,18 +394,18 @@ module BaseTest
|
|
701
394
|
tag.is_a?(String) && !tag.empty?
|
702
395
|
end
|
703
396
|
non_utf8_tags.each do |container_name, encoded_name|
|
704
|
-
|
705
|
-
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
706
|
-
"#{CONTAINER_SERVICE_NAME}/container_name" =>
|
707
|
-
URI.decode(encoded_name)),
|
708
|
-
log_name: encoded_name)
|
709
|
-
setup_logging_stubs([params]) do
|
397
|
+
setup_logging_stubs do
|
710
398
|
@logs_sent = []
|
711
399
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
|
712
400
|
container_tag_with_container_name(container_name))
|
713
401
|
d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
|
714
402
|
d.run
|
715
403
|
end
|
404
|
+
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
405
|
+
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
406
|
+
"#{CONTAINER_CONSTANTS[:service]}/container_name" =>
|
407
|
+
URI.decode(encoded_name)),
|
408
|
+
log_name: encoded_name)
|
716
409
|
verify_log_entries(0, params, 'textPayload')
|
717
410
|
end
|
718
411
|
end
|
@@ -722,15 +415,14 @@ module BaseTest
|
|
722
415
|
def test_encode_tags_with_require_valid_tags_true
|
723
416
|
setup_gce_metadata_stubs
|
724
417
|
VALID_TAGS.each do |tag, encoded_tag|
|
725
|
-
setup_logging_stubs
|
418
|
+
setup_logging_stubs do
|
726
419
|
@logs_sent = []
|
727
420
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
|
728
421
|
d.emit('msg' => log_entry(0))
|
729
422
|
d.run
|
730
423
|
end
|
731
|
-
verify_log_entries(1, COMPUTE_PARAMS
|
732
|
-
|
733
|
-
@logs_sent[0]['logName']
|
424
|
+
verify_log_entries(1, COMPUTE_PARAMS.merge(log_name: encoded_tag),
|
425
|
+
'jsonPayload')
|
734
426
|
end
|
735
427
|
end
|
736
428
|
|
@@ -739,20 +431,19 @@ module BaseTest
|
|
739
431
|
setup_gce_metadata_stubs
|
740
432
|
setup_container_metadata_stubs
|
741
433
|
VALID_TAGS.each do |tag, encoded_tag|
|
742
|
-
|
743
|
-
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
744
|
-
"#{CONTAINER_SERVICE_NAME}/container_name" => tag),
|
745
|
-
log_name: encoded_tag)
|
746
|
-
setup_logging_stubs([params]) do
|
434
|
+
setup_logging_stubs do
|
747
435
|
@logs_sent = []
|
748
436
|
d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
|
749
437
|
container_tag_with_container_name(tag))
|
750
438
|
d.emit(container_log_entry_with_metadata(log_entry(0), tag))
|
751
439
|
d.run
|
752
440
|
end
|
441
|
+
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
442
|
+
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
443
|
+
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
444
|
+
'container_name' => tag)),
|
445
|
+
log_name: encoded_tag)
|
753
446
|
verify_log_entries(1, params, 'textPayload')
|
754
|
-
assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
|
755
|
-
@logs_sent[0]['logName']
|
756
447
|
end
|
757
448
|
end
|
758
449
|
|
@@ -762,15 +453,14 @@ module BaseTest
|
|
762
453
|
def test_sanitize_tags_with_require_valid_tags_false
|
763
454
|
setup_gce_metadata_stubs
|
764
455
|
ALL_TAGS.each do |tag, sanitized_tag|
|
765
|
-
setup_logging_stubs
|
456
|
+
setup_logging_stubs do
|
766
457
|
@logs_sent = []
|
767
458
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, tag)
|
768
459
|
d.emit('msg' => log_entry(0))
|
769
460
|
d.run
|
770
461
|
end
|
771
|
-
verify_log_entries(1, COMPUTE_PARAMS
|
772
|
-
|
773
|
-
@logs_sent[0]['logName']
|
462
|
+
verify_log_entries(1, COMPUTE_PARAMS.merge(log_name: sanitized_tag),
|
463
|
+
'jsonPayload')
|
774
464
|
end
|
775
465
|
end
|
776
466
|
|
@@ -787,21 +477,19 @@ module BaseTest
|
|
787
477
|
string_tags.each do |container_name, encoded_container_name|
|
788
478
|
# Container name in the label is sanitized but not encoded, while the log
|
789
479
|
# name is encoded.
|
790
|
-
|
791
|
-
labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
|
792
|
-
"#{CONTAINER_SERVICE_NAME}/container_name" =>
|
793
|
-
URI.decode(encoded_container_name)),
|
794
|
-
log_name: encoded_container_name)
|
795
|
-
setup_logging_stubs([params]) do
|
480
|
+
setup_logging_stubs do
|
796
481
|
@logs_sent = []
|
797
482
|
d = create_driver(APPLICATION_DEFAULT_CONFIG,
|
798
483
|
container_tag_with_container_name(container_name))
|
799
484
|
d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
|
800
485
|
d.run
|
801
486
|
end
|
487
|
+
params = CONTAINER_FROM_METADATA_PARAMS.merge(
|
488
|
+
resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
|
489
|
+
labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
|
490
|
+
'container_name' => URI.decode(encoded_container_name))),
|
491
|
+
log_name: encoded_container_name)
|
802
492
|
verify_log_entries(1, params, 'textPayload')
|
803
|
-
assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_container_name}",
|
804
|
-
@logs_sent[0]['logName']
|
805
493
|
end
|
806
494
|
end
|
807
495
|
|
@@ -833,9 +521,9 @@ module BaseTest
|
|
833
521
|
end
|
834
522
|
verify_index = 0
|
835
523
|
verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
|
836
|
-
assert_equal_with_default entry['
|
524
|
+
assert_equal_with_default entry['timestamp']['seconds'],
|
837
525
|
expected_ts[verify_index].tv_sec, 0, entry
|
838
|
-
assert_equal_with_default entry['
|
526
|
+
assert_equal_with_default entry['timestamp']['nanos'],
|
839
527
|
expected_ts[verify_index].tv_nsec, 0, entry do
|
840
528
|
# Fluentd v0.14 onwards supports nanosecond timestamp values.
|
841
529
|
# Added in 600 ns delta to avoid flaky tests introduced
|
@@ -843,7 +531,7 @@ module BaseTest
|
|
843
531
|
# (to account for the missing 9 bits of precision ~ 512 ns).
|
844
532
|
# See http://wikipedia.org/wiki/Double-precision_floating-point_format
|
845
533
|
assert_in_delta expected_ts[verify_index].tv_nsec,
|
846
|
-
entry['
|
534
|
+
entry['timestamp']['nanos'], 600, entry
|
847
535
|
end
|
848
536
|
verify_index += 1
|
849
537
|
end
|
@@ -853,12 +541,12 @@ module BaseTest
|
|
853
541
|
setup_gce_metadata_stubs
|
854
542
|
setup_logging_stubs do
|
855
543
|
d = create_driver
|
856
|
-
# if timestamp is not a hash it is passed through to the
|
544
|
+
# if timestamp is not a hash it is passed through to the json payload.
|
857
545
|
d.emit('message' => log_entry(0), 'timestamp' => 'not-a-hash')
|
858
546
|
d.run
|
859
547
|
end
|
860
|
-
verify_log_entries(1, COMPUTE_PARAMS, '
|
861
|
-
fields = get_fields(entry['
|
548
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
|
549
|
+
fields = get_fields(entry['jsonPayload'])
|
862
550
|
assert_equal 2, fields.size, entry
|
863
551
|
assert_equal 'not-a-hash', get_string(fields['timestamp']), entry
|
864
552
|
end
|
@@ -937,7 +625,7 @@ module BaseTest
|
|
937
625
|
}
|
938
626
|
)
|
939
627
|
d = create_driver(config)
|
940
|
-
# not_a_label passes through to the
|
628
|
+
# not_a_label passes through to the json payload
|
941
629
|
d.emit('message' => log_entry(0),
|
942
630
|
'label1' => 'value1',
|
943
631
|
'label_number_two' => 'value2',
|
@@ -950,8 +638,8 @@ module BaseTest
|
|
950
638
|
params[:labels]['sent_label_1'] = 'value1'
|
951
639
|
params[:labels]['foo.googleapis.com/bar'] = 'value2'
|
952
640
|
params[:labels]['label3'] = 'value3'
|
953
|
-
verify_log_entries(1, params, '
|
954
|
-
fields = get_fields(entry['
|
641
|
+
verify_log_entries(1, params, 'jsonPayload') do |entry|
|
642
|
+
fields = get_fields(entry['jsonPayload'])
|
955
643
|
assert_equal 2, fields.size, entry
|
956
644
|
assert_equal 'test log entry 0', get_string(fields['message']), entry
|
957
645
|
assert_equal 'value4', get_string(fields['not_a_label']), entry
|
@@ -1023,11 +711,9 @@ module BaseTest
|
|
1023
711
|
d.run
|
1024
712
|
end
|
1025
713
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS) do |entry|
|
1026
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1027
|
-
|
1028
|
-
assert_equal
|
1029
|
-
entry['metadata']['timestamp']['nanos'], entry
|
1030
|
-
assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
|
714
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
715
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
716
|
+
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
1031
717
|
end
|
1032
718
|
end
|
1033
719
|
|
@@ -1045,11 +731,10 @@ module BaseTest
|
|
1045
731
|
d.run
|
1046
732
|
end
|
1047
733
|
verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS) do |entry|
|
1048
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1049
|
-
entry
|
1050
|
-
assert_equal CONTAINER_NANOS,
|
1051
|
-
|
1052
|
-
assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
|
734
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
|
735
|
+
entry
|
736
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
737
|
+
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
1053
738
|
end
|
1054
739
|
end
|
1055
740
|
end
|
@@ -1068,11 +753,10 @@ module BaseTest
|
|
1068
753
|
d.run
|
1069
754
|
end
|
1070
755
|
verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS) do |entry|
|
1071
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1072
|
-
entry
|
1073
|
-
assert_equal CONTAINER_NANOS,
|
1074
|
-
|
1075
|
-
assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
|
756
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
|
757
|
+
entry
|
758
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
759
|
+
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
1076
760
|
end
|
1077
761
|
end
|
1078
762
|
end
|
@@ -1086,11 +770,9 @@ module BaseTest
|
|
1086
770
|
d.run
|
1087
771
|
end
|
1088
772
|
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS) do |entry|
|
1089
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1090
|
-
|
1091
|
-
assert_equal
|
1092
|
-
entry['metadata']['timestamp']['nanos'], entry
|
1093
|
-
assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
|
773
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
774
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
775
|
+
assert_equal CONTAINER_SEVERITY, entry['severity'], entry
|
1094
776
|
end
|
1095
777
|
end
|
1096
778
|
|
@@ -1103,18 +785,16 @@ module BaseTest
|
|
1103
785
|
d.run
|
1104
786
|
end
|
1105
787
|
expected_params = CONTAINER_FROM_TAG_PARAMS.merge(
|
1106
|
-
labels: { "#{
|
788
|
+
labels: { "#{CONTAINER_CONSTANTS[:service]}/stream" => 'stderr' }
|
1107
789
|
) { |_, oldval, newval| oldval.merge(newval) }
|
1108
790
|
verify_log_entries(1, expected_params) do |entry|
|
1109
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1110
|
-
|
1111
|
-
assert_equal
|
1112
|
-
entry['metadata']['timestamp']['nanos'], entry
|
1113
|
-
assert_equal 'ERROR', entry['metadata']['severity'], entry
|
791
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
792
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
793
|
+
assert_equal 'ERROR', entry['severity'], entry
|
1114
794
|
end
|
1115
795
|
end
|
1116
796
|
|
1117
|
-
def
|
797
|
+
def test_json_container_log_metadata_from_plugin
|
1118
798
|
setup_gce_metadata_stubs
|
1119
799
|
setup_container_metadata_stubs
|
1120
800
|
setup_logging_stubs do
|
@@ -1125,21 +805,19 @@ module BaseTest
|
|
1125
805
|
d.run
|
1126
806
|
end
|
1127
807
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
|
1128
|
-
'
|
1129
|
-
fields = get_fields(entry['
|
808
|
+
'jsonPayload') do |entry|
|
809
|
+
fields = get_fields(entry['jsonPayload'])
|
1130
810
|
assert_equal 3, fields.size, entry
|
1131
811
|
assert_equal 'test log entry 0', get_string(fields['msg']), entry
|
1132
812
|
assert_equal 'test', get_string(fields['tag2']), entry
|
1133
813
|
assert_equal 5000, get_number(fields['data']), entry
|
1134
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1135
|
-
|
1136
|
-
assert_equal
|
1137
|
-
entry['metadata']['timestamp']['nanos'], entry
|
1138
|
-
assert_equal 'WARNING', entry['metadata']['severity'], entry
|
814
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
815
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
816
|
+
assert_equal 'WARNING', entry['severity'], entry
|
1139
817
|
end
|
1140
818
|
end
|
1141
819
|
|
1142
|
-
def
|
820
|
+
def test_json_container_log_metadata_from_tag
|
1143
821
|
setup_gce_metadata_stubs
|
1144
822
|
setup_container_metadata_stubs
|
1145
823
|
setup_logging_stubs do
|
@@ -1150,17 +828,15 @@ module BaseTest
|
|
1150
828
|
d.run
|
1151
829
|
end
|
1152
830
|
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
|
1153
|
-
'
|
1154
|
-
fields = get_fields(entry['
|
831
|
+
'jsonPayload') do |entry|
|
832
|
+
fields = get_fields(entry['jsonPayload'])
|
1155
833
|
assert_equal 3, fields.size, entry
|
1156
834
|
assert_equal 'test log entry 0', get_string(fields['msg']), entry
|
1157
835
|
assert_equal 'test', get_string(fields['tag2']), entry
|
1158
836
|
assert_equal 5000, get_number(fields['data']), entry
|
1159
|
-
assert_equal CONTAINER_SECONDS_EPOCH,
|
1160
|
-
|
1161
|
-
assert_equal
|
1162
|
-
entry['metadata']['timestamp']['nanos'], entry
|
1163
|
-
assert_equal 'WARNING', entry['metadata']['severity'], entry
|
837
|
+
assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
|
838
|
+
assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
|
839
|
+
assert_equal 'WARNING', entry['severity'], entry
|
1164
840
|
end
|
1165
841
|
end
|
1166
842
|
|
@@ -1178,7 +854,7 @@ module BaseTest
|
|
1178
854
|
d.run
|
1179
855
|
end
|
1180
856
|
verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
|
1181
|
-
assert_equal 'DEBUG', entry['
|
857
|
+
assert_equal 'DEBUG', entry['severity'],
|
1182
858
|
"Test with #{n} logs failed. \n#{entry}"
|
1183
859
|
end
|
1184
860
|
end
|
@@ -1199,7 +875,7 @@ module BaseTest
|
|
1199
875
|
end
|
1200
876
|
verify_log_entries(
|
1201
877
|
n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
|
1202
|
-
assert_equal 'INFO', entry['
|
878
|
+
assert_equal 'INFO', entry['severity'],
|
1203
879
|
"Test with #{n} logs failed. \n#{entry}"
|
1204
880
|
end
|
1205
881
|
end
|
@@ -1228,16 +904,27 @@ module BaseTest
|
|
1228
904
|
end
|
1229
905
|
end
|
1230
906
|
|
907
|
+
def test_dataproc_log
|
908
|
+
setup_gce_metadata_stubs
|
909
|
+
setup_dataproc_metadata_stubs
|
910
|
+
setup_logging_stubs do
|
911
|
+
d = create_driver
|
912
|
+
d.emit(dataproc_log_entry('test message'))
|
913
|
+
d.run
|
914
|
+
end
|
915
|
+
verify_log_entries(1, DATAPROC_PARAMS, 'jsonPayload')
|
916
|
+
end
|
917
|
+
|
1231
918
|
def test_http_request_from_record
|
1232
919
|
setup_gce_metadata_stubs
|
1233
920
|
setup_logging_stubs do
|
1234
921
|
d = create_driver
|
1235
|
-
d.emit('httpRequest' =>
|
922
|
+
d.emit('httpRequest' => HTTP_REQUEST_MESSAGE)
|
1236
923
|
d.run
|
1237
924
|
end
|
1238
925
|
verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
|
1239
|
-
assert_equal
|
1240
|
-
assert_nil get_fields(entry['
|
926
|
+
assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
|
927
|
+
assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
|
1241
928
|
end
|
1242
929
|
end
|
1243
930
|
|
@@ -1245,13 +932,13 @@ module BaseTest
|
|
1245
932
|
setup_gce_metadata_stubs
|
1246
933
|
setup_logging_stubs do
|
1247
934
|
d = create_driver
|
1248
|
-
d.emit('httpRequest' =>
|
935
|
+
d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge(
|
1249
936
|
'otherKey' => 'value'))
|
1250
937
|
d.run
|
1251
938
|
end
|
1252
939
|
verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
|
1253
|
-
assert_equal
|
1254
|
-
fields = get_fields(entry['
|
940
|
+
assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
|
941
|
+
fields = get_fields(entry['jsonPayload'])
|
1255
942
|
request = get_fields(get_struct(fields['httpRequest']))
|
1256
943
|
assert_equal 'value', get_string(request['otherKey']), entry
|
1257
944
|
end
|
@@ -1264,20 +951,72 @@ module BaseTest
|
|
1264
951
|
d.emit('httpRequest' => 'a_string')
|
1265
952
|
d.run
|
1266
953
|
end
|
1267
|
-
verify_log_entries(1, COMPUTE_PARAMS, '
|
1268
|
-
fields = get_fields(entry['
|
954
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
|
955
|
+
fields = get_fields(entry['jsonPayload'])
|
1269
956
|
assert_equal 'a_string', get_string(fields['httpRequest']), entry
|
1270
957
|
assert_nil entry['httpRequest'], entry
|
1271
958
|
end
|
1272
959
|
end
|
1273
960
|
|
1274
|
-
|
961
|
+
def test_http_request_from_record_with_referer_nil_or_absent
|
962
|
+
setup_gce_metadata_stubs
|
963
|
+
[
|
964
|
+
http_request_message_with_nil_referer,
|
965
|
+
http_request_message_with_absent_referer
|
966
|
+
].each do |input|
|
967
|
+
setup_logging_stubs do
|
968
|
+
@logs_sent = []
|
969
|
+
d = create_driver
|
970
|
+
d.emit('httpRequest' => input)
|
971
|
+
d.run
|
972
|
+
end
|
973
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
|
974
|
+
assert_equal http_request_message_with_absent_referer,
|
975
|
+
entry['httpRequest'], entry
|
976
|
+
assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
|
977
|
+
end
|
978
|
+
end
|
979
|
+
end
|
1275
980
|
|
1276
|
-
def
|
1277
|
-
|
1278
|
-
|
981
|
+
def test_http_request_with_latency
|
982
|
+
setup_gce_metadata_stubs
|
983
|
+
latency_conversion.each do |input, expected|
|
984
|
+
setup_logging_stubs do
|
985
|
+
d = create_driver
|
986
|
+
@logs_sent = []
|
987
|
+
d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
|
988
|
+
d.run
|
989
|
+
end
|
990
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
|
991
|
+
assert_equal HTTP_REQUEST_MESSAGE.merge('latency' => expected),
|
992
|
+
entry['httpRequest'], entry
|
993
|
+
assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
|
994
|
+
end
|
995
|
+
end
|
1279
996
|
end
|
1280
997
|
|
998
|
+
# Skip setting latency when the field is null or has invalid format.
|
999
|
+
def test_http_request_skip_setting_latency
|
1000
|
+
setup_gce_metadata_stubs
|
1001
|
+
[
|
1002
|
+
'', ' ', nil, 'null', '123', '1.23 seconds',
|
1003
|
+
' 123 s econds ', '1min', 'abc&^!$*('
|
1004
|
+
].each do |input|
|
1005
|
+
setup_logging_stubs do
|
1006
|
+
d = create_driver
|
1007
|
+
@logs_sent = []
|
1008
|
+
d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
|
1009
|
+
d.run
|
1010
|
+
end
|
1011
|
+
verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
|
1012
|
+
assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
|
1013
|
+
assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
|
1014
|
+
end
|
1015
|
+
end
|
1016
|
+
end
|
1017
|
+
|
1018
|
+
private
|
1019
|
+
|
1281
1020
|
def stub_metadata_request(metadata_path, response_body)
|
1282
1021
|
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
|
1283
1022
|
metadata_path)
|
@@ -1375,6 +1114,22 @@ module BaseTest
|
|
1375
1114
|
CLOUDFUNCTIONS_REGION)
|
1376
1115
|
end
|
1377
1116
|
|
1117
|
+
def setup_dataproc_metadata_stubs
|
1118
|
+
stub_metadata_request(
|
1119
|
+
'instance/attributes/',
|
1120
|
+
"attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
|
1121
|
+
stub_metadata_request('instance/attributes/dataproc-cluster-name',
|
1122
|
+
DATAPROC_CLUSTER_NAME)
|
1123
|
+
stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
|
1124
|
+
DATAPROC_CLUSTER_UUID)
|
1125
|
+
stub_metadata_request('instance/attributes/dataproc-region',
|
1126
|
+
DATAPROC_REGION)
|
1127
|
+
end
|
1128
|
+
|
1129
|
+
def setup_prometheus
|
1130
|
+
Prometheus::Client.registry.instance_variable_set('@metrics', {})
|
1131
|
+
end
|
1132
|
+
|
1378
1133
|
def container_tag_with_container_name(container_name)
|
1379
1134
|
"kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
|
1380
1135
|
"#{container_name}"
|
@@ -1421,42 +1176,74 @@ module BaseTest
|
|
1421
1176
|
}
|
1422
1177
|
end
|
1423
1178
|
|
1179
|
+
def dataflow_log_entry(i)
|
1180
|
+
{
|
1181
|
+
step: DATAFLOW_STEP_ID,
|
1182
|
+
message: log_entry(i)
|
1183
|
+
}
|
1184
|
+
end
|
1185
|
+
|
1186
|
+
def dataproc_log_entry(message, source_class = 'com.example.Example',
|
1187
|
+
filename = 'test.log')
|
1188
|
+
{
|
1189
|
+
filename: filename,
|
1190
|
+
class: source_class,
|
1191
|
+
message: log_entry(message)
|
1192
|
+
}
|
1193
|
+
end
|
1194
|
+
|
1195
|
+
def ml_log_entry(i)
|
1196
|
+
{
|
1197
|
+
name: ML_LOG_AREA,
|
1198
|
+
message: log_entry(i)
|
1199
|
+
}
|
1200
|
+
end
|
1201
|
+
|
1424
1202
|
def log_entry(i)
|
1425
1203
|
'test log entry ' + i.to_s
|
1426
1204
|
end
|
1427
1205
|
|
1428
|
-
def check_labels(
|
1429
|
-
|
1430
|
-
all_labels ||= common_labels
|
1431
|
-
all_labels.merge!(entry['metadata']['labels'] || {})
|
1432
|
-
all_labels.each do |key, value|
|
1206
|
+
def check_labels(labels, expected_labels)
|
1207
|
+
labels.each do |key, value|
|
1433
1208
|
assert value.is_a?(String), "Value #{value} for label #{key} " \
|
1434
1209
|
'is not a string: ' + value.class.name
|
1435
1210
|
assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
|
1436
1211
|
assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
|
1437
1212
|
"#{expected_labels[key]} in #{key} => #{value}"
|
1438
1213
|
end
|
1439
|
-
assert_equal expected_labels.length,
|
1440
|
-
"#{expected_labels.length} labels, got
|
1214
|
+
assert_equal expected_labels.length, labels.length, 'Expected ' \
|
1215
|
+
"#{expected_labels.length} labels: #{expected_labels}, got " \
|
1216
|
+
"#{labels.length} labels: #{labels}"
|
1441
1217
|
end
|
1442
1218
|
|
1443
1219
|
# The caller can optionally provide a block which is called for each entry.
|
1444
1220
|
def verify_json_log_entries(n, params, payload_type = 'textPayload')
|
1445
1221
|
i = 0
|
1446
|
-
@logs_sent.each do |
|
1447
|
-
|
1222
|
+
@logs_sent.each do |request|
|
1223
|
+
request['entries'].each do |entry|
|
1448
1224
|
unless payload_type.empty?
|
1449
1225
|
assert entry.key?(payload_type), 'Entry did not contain expected ' \
|
1450
1226
|
"#{payload_type} key: " + entry.to_s
|
1451
1227
|
# Check the payload for textPayload, otherwise it's up to the caller.
|
1452
1228
|
if payload_type == 'textPayload'
|
1453
|
-
assert_equal "test log entry #{i}", entry['textPayload'],
|
1229
|
+
assert_equal "test log entry #{i}", entry['textPayload'], request
|
1454
1230
|
end
|
1455
1231
|
end
|
1456
1232
|
|
1457
|
-
|
1458
|
-
|
1459
|
-
|
1233
|
+
# per-entry resource or log_name overrides the corresponding field
|
1234
|
+
# from the request. Labels are merged, with the per-entry label
|
1235
|
+
# taking precedence in case of overlap.
|
1236
|
+
resource = entry['resource'] || request['resource']
|
1237
|
+
log_name = entry['logName'] || request['logName']
|
1238
|
+
|
1239
|
+
labels ||= request['labels']
|
1240
|
+
labels.merge!(entry['labels'] || {})
|
1241
|
+
|
1242
|
+
assert_equal \
|
1243
|
+
"projects/#{params[:project_id]}/logs/#{params[:log_name]}", log_name
|
1244
|
+
assert_equal params[:resource][:type], resource['type']
|
1245
|
+
check_labels resource['labels'], params[:resource][:labels]
|
1246
|
+
check_labels labels, params[:labels]
|
1460
1247
|
yield(entry) if block_given?
|
1461
1248
|
i += 1
|
1462
1249
|
assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
|
@@ -1465,14 +1252,37 @@ module BaseTest
|
|
1465
1252
|
assert i == n, "Number of entries #{i} does not match expected number #{n}"
|
1466
1253
|
end
|
1467
1254
|
|
1468
|
-
# The http request message to test against.
|
1469
|
-
def http_request_message
|
1470
|
-
HTTP_REQUEST_MESSAGE
|
1471
|
-
end
|
1472
|
-
|
1473
1255
|
# Replace the 'referer' field with nil.
|
1474
1256
|
def http_request_message_with_nil_referer
|
1475
|
-
|
1257
|
+
HTTP_REQUEST_MESSAGE.merge('referer' => nil)
|
1258
|
+
end
|
1259
|
+
|
1260
|
+
# Unset the 'referer' field.
|
1261
|
+
def http_request_message_with_absent_referer
|
1262
|
+
HTTP_REQUEST_MESSAGE.reject do |k, _|
|
1263
|
+
k == 'referer'
|
1264
|
+
end
|
1265
|
+
end
|
1266
|
+
|
1267
|
+
# The conversions from user input to output.
|
1268
|
+
def latency_conversion
|
1269
|
+
{
|
1270
|
+
'32 s' => { 'seconds' => 32 },
|
1271
|
+
'32s' => { 'seconds' => 32 },
|
1272
|
+
'0.32s' => { 'nanos' => 320_000_000 },
|
1273
|
+
' 123 s ' => { 'seconds' => 123 },
|
1274
|
+
'1.3442 s' => { 'seconds' => 1, 'nanos' => 344_200_000 },
|
1275
|
+
|
1276
|
+
# Test whitespace.
|
1277
|
+
# \t: tab. \r: carriage return. \n: line break.
|
1278
|
+
# \v: vertical whitespace. \f: form feed.
|
1279
|
+
"\t123.5\ts\t" => { 'seconds' => 123, 'nanos' => 500_000_000 },
|
1280
|
+
"\r123.5\rs\r" => { 'seconds' => 123, 'nanos' => 500_000_000 },
|
1281
|
+
"\n123.5\ns\n" => { 'seconds' => 123, 'nanos' => 500_000_000 },
|
1282
|
+
"\v123.5\vs\v" => { 'seconds' => 123, 'nanos' => 500_000_000 },
|
1283
|
+
"\f123.5\fs\f" => { 'seconds' => 123, 'nanos' => 500_000_000 },
|
1284
|
+
"\r123.5\ts\f" => { 'seconds' => 123, 'nanos' => 500_000_000 }
|
1285
|
+
}
|
1476
1286
|
end
|
1477
1287
|
|
1478
1288
|
# This module expects the methods below to be overridden.
|
@@ -1503,8 +1313,14 @@ module BaseTest
|
|
1503
1313
|
_undefined
|
1504
1314
|
end
|
1505
1315
|
|
1506
|
-
|
1507
|
-
|
1316
|
+
def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
|
1317
|
+
metric = Prometheus::Client.registry.get(metric_name)
|
1318
|
+
assert_not_nil(metric)
|
1319
|
+
assert_equal(expected_value, metric.get(labels))
|
1320
|
+
end
|
1321
|
+
|
1322
|
+
# Get the fields of the payload.
|
1323
|
+
def get_fields(_payload)
|
1508
1324
|
_undefined
|
1509
1325
|
end
|
1510
1326
|
|