vmik-fluent-plugin-google-cloud 0.5.5 → 0.6.4.pre.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,9 +16,14 @@ require 'google/apis'
16
16
  require 'helper'
17
17
  require 'mocha/test_unit'
18
18
  require 'webmock/test_unit'
19
+ require 'prometheus/client'
20
+
21
+ require_relative 'constants'
19
22
 
20
23
  # Unit tests for Google Cloud Logging plugin
21
24
  module BaseTest
25
+ include Constants
26
+
22
27
  def setup
23
28
  Fluent::Test.setup
24
29
  # delete environment variables that googleauth uses to find credentials.
@@ -37,317 +42,6 @@ module BaseTest
37
42
  @logs_sent = []
38
43
  end
39
44
 
40
- # generic attributes
41
- HOSTNAME = Socket.gethostname
42
-
43
- # attributes used for the GCE metadata service
44
- PROJECT_ID = 'test-project-id'
45
- ZONE = 'us-central1-b'
46
- FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
47
- VM_ID = '9876543210'
48
-
49
- # attributes used for custom (overridden) configs
50
- CUSTOM_PROJECT_ID = 'test-custom-project-id'
51
- CUSTOM_ZONE = 'us-custom-central1-b'
52
- CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
53
- CUSTOM_VM_ID = 'C9876543210'
54
- CUSTOM_HOSTNAME = 'custom.hostname.org'
55
-
56
- # attributes used for the EC2 metadata service
57
- EC2_PROJECT_ID = 'test-ec2-project-id'
58
- EC2_ZONE = 'us-west-2b'
59
- EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
60
- EC2_VM_ID = 'i-81c16767'
61
- EC2_ACCOUNT_ID = '123456789012'
62
-
63
- # The formatting here matches the format used on the VM.
64
- EC2_IDENTITY_DOCUMENT = %({
65
- "accountId" : "#{EC2_ACCOUNT_ID}",
66
- "availabilityZone" : "#{EC2_ZONE}",
67
- "instanceId" : "#{EC2_VM_ID}"
68
- })
69
-
70
- # Managed VMs specific labels
71
- MANAGED_VM_BACKEND_NAME = 'default'
72
- MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
73
-
74
- # Container Engine / Kubernetes specific labels
75
- CONTAINER_CLUSTER_NAME = 'cluster-1'
76
- CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
77
- CONTAINER_NAMESPACE_NAME = 'kube-system'
78
- CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
79
- CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
80
- CONTAINER_CONTAINER_NAME = 'redis'
81
- CONTAINER_LABEL_KEY = 'component'
82
- CONTAINER_LABEL_VALUE = 'redis-component'
83
- CONTAINER_STREAM = 'stdout'
84
- CONTAINER_SEVERITY = 'INFO'
85
- # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch
86
- CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
87
- CONTAINER_SECONDS_EPOCH = 1_234_567_890
88
- CONTAINER_NANOS = 987_654_321
89
-
90
- # Cloud Functions specific labels
91
- CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
92
- CLOUDFUNCTIONS_REGION = 'us-central1'
93
- CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
94
- CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
95
- CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
96
- CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
97
- CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
98
-
99
- # Parameters used for authentication
100
- AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
101
- FAKE_AUTH_TOKEN = 'abc123'
102
-
103
- # Information about test credentials files.
104
- # path: Path to the credentials file.
105
- # project_id: ID of the project, which must correspond to the file contents.
106
- IAM_CREDENTIALS = {
107
- path: 'test/plugin/data/iam-credentials.json',
108
- project_id: 'fluent-test-project'
109
- }
110
- LEGACY_CREDENTIALS = {
111
- path: 'test/plugin/data/credentials.json',
112
- project_id: '847859579879'
113
- }
114
- INVALID_CREDENTIALS = {
115
- path: 'test/plugin/data/invalid_credentials.json',
116
- project_id: ''
117
- }
118
-
119
- # Configuration files for various test scenarios
120
- APPLICATION_DEFAULT_CONFIG = %(
121
- )
122
-
123
- # rubocop:disable Metrics/LineLength
124
- PRIVATE_KEY_CONFIG = %(
125
- auth_method private_key
126
- private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
127
- private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
128
- )
129
- # rubocop:enable Metrics/LineLength
130
-
131
- REQUIRE_VALID_TAGS_CONFIG = %(
132
- require_valid_tags true
133
- )
134
-
135
- NO_METADATA_SERVICE_CONFIG = %(
136
- use_metadata_service false
137
- )
138
-
139
- NO_DETECT_SUBSERVICE_CONFIG = %(
140
- detect_subservice false
141
- )
142
-
143
- CUSTOM_METADATA_CONFIG = %(
144
- project_id #{CUSTOM_PROJECT_ID}
145
- zone #{CUSTOM_ZONE}
146
- vm_id #{CUSTOM_VM_ID}
147
- vm_name #{CUSTOM_HOSTNAME}
148
- )
149
-
150
- CONFIG_MISSING_METADATA_PROJECT_ID = %(
151
- zone #{CUSTOM_ZONE}
152
- vm_id #{CUSTOM_VM_ID}
153
- )
154
- CONFIG_MISSING_METADATA_ZONE = %(
155
- project_id #{CUSTOM_PROJECT_ID}
156
- vm_id #{CUSTOM_VM_ID}
157
- )
158
- CONFIG_MISSING_METADATA_VM_ID = %(
159
- project_id #{CUSTOM_PROJECT_ID}
160
- zone #{CUSTOM_ZONE}
161
- )
162
- CONFIG_MISSING_METADATA_ALL = %(
163
- )
164
-
165
- CONFIG_EC2_PROJECT_ID = %(
166
- project_id #{EC2_PROJECT_ID}
167
- )
168
-
169
- CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
170
- project_id #{EC2_PROJECT_ID}
171
- vm_id #{CUSTOM_VM_ID}
172
- )
173
-
174
- # Service configurations for various services
175
- COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
176
- APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
177
- CONTAINER_SERVICE_NAME = 'container.googleapis.com'
178
- CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
179
- EC2_SERVICE_NAME = 'ec2.amazonaws.com'
180
-
181
- COMPUTE_PARAMS = {
182
- service_name: COMPUTE_SERVICE_NAME,
183
- log_name: 'test',
184
- project_id: PROJECT_ID,
185
- zone: ZONE,
186
- labels: {
187
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
188
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
189
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
190
- }
191
- }
192
-
193
- VMENGINE_PARAMS = {
194
- service_name: APPENGINE_SERVICE_NAME,
195
- log_name: "#{APPENGINE_SERVICE_NAME}%2Ftest",
196
- project_id: PROJECT_ID,
197
- zone: ZONE,
198
- labels: {
199
- "#{APPENGINE_SERVICE_NAME}/module_id" => MANAGED_VM_BACKEND_NAME,
200
- "#{APPENGINE_SERVICE_NAME}/version_id" => MANAGED_VM_BACKEND_VERSION,
201
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
202
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
203
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
204
- }
205
- }
206
-
207
- CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
208
- "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
209
-
210
- CONTAINER_FROM_METADATA_PARAMS = {
211
- service_name: CONTAINER_SERVICE_NAME,
212
- log_name: CONTAINER_CONTAINER_NAME,
213
- project_id: PROJECT_ID,
214
- zone: ZONE,
215
- labels: {
216
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
217
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
218
- "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
219
- "#{CONTAINER_SERVICE_NAME}/namespace_id" => CONTAINER_NAMESPACE_ID,
220
- "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
221
- "#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
222
- "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
223
- "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
224
- "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
225
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
226
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
227
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
228
- }
229
- }
230
-
231
- # Almost the same as from metadata, but missing namespace_id and pod_id.
232
- CONTAINER_FROM_TAG_PARAMS = {
233
- service_name: CONTAINER_SERVICE_NAME,
234
- log_name: CONTAINER_CONTAINER_NAME,
235
- project_id: PROJECT_ID,
236
- zone: ZONE,
237
- labels: {
238
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
239
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
240
- "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
241
- "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
242
- "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
243
- "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
244
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
245
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
246
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
247
- }
248
- }
249
-
250
- CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
251
- "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
252
- "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
253
-
254
- CLOUDFUNCTIONS_PARAMS = {
255
- service_name: CLOUDFUNCTIONS_SERVICE_NAME,
256
- log_name: 'cloud-functions',
257
- project_id: PROJECT_ID,
258
- zone: ZONE,
259
- labels: {
260
- 'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
261
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
262
- CLOUDFUNCTIONS_FUNCTION_NAME,
263
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
264
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
265
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
266
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
267
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
268
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
269
- }
270
- }
271
-
272
- CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
273
- service_name: CLOUDFUNCTIONS_SERVICE_NAME,
274
- log_name: 'cloud-functions',
275
- project_id: PROJECT_ID,
276
- zone: ZONE,
277
- labels: {
278
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
279
- CLOUDFUNCTIONS_FUNCTION_NAME,
280
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
281
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
282
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
283
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
284
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
285
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
286
- }
287
- }
288
-
289
- CUSTOM_PARAMS = {
290
- service_name: COMPUTE_SERVICE_NAME,
291
- log_name: 'test',
292
- project_id: CUSTOM_PROJECT_ID,
293
- zone: CUSTOM_ZONE,
294
- labels: {
295
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
296
- "#{COMPUTE_SERVICE_NAME}/resource_id" => CUSTOM_VM_ID,
297
- "#{COMPUTE_SERVICE_NAME}/resource_name" => CUSTOM_HOSTNAME
298
- }
299
- }
300
-
301
- EC2_PARAMS = {
302
- service_name: EC2_SERVICE_NAME,
303
- log_name: 'test',
304
- project_id: EC2_PROJECT_ID,
305
- zone: EC2_PREFIXED_ZONE,
306
- labels: {
307
- "#{EC2_SERVICE_NAME}/resource_type" => 'instance',
308
- "#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
309
- "#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID,
310
- "#{EC2_SERVICE_NAME}/resource_name" => HOSTNAME
311
- }
312
- }
313
-
314
- HTTP_REQUEST_MESSAGE = {
315
- 'requestMethod' => 'POST',
316
- 'requestUrl' => 'http://example/',
317
- 'requestSize' => 210,
318
- 'status' => 200,
319
- 'responseSize' => 65,
320
- 'userAgent' => 'USER AGENT 1.0',
321
- 'remoteIp' => '55.55.55.55',
322
- 'referer' => 'http://referer/',
323
- 'cacheHit' => true,
324
- 'cacheValidatedWithOriginServer' => true
325
- }
326
-
327
- # Tags and their sanitized and encoded version.
328
- VALID_TAGS = {
329
- 'test' => 'test',
330
- 'germanß' => 'german%C3%9F',
331
- 'chinese中' => 'chinese%E4%B8%AD',
332
- 'specialCharacter/_-.' => 'specialCharacter%2F_-.',
333
- 'abc@&^$*' => 'abc%40%26%5E%24%2A',
334
- '@&^$*' => '%40%26%5E%24%2A'
335
- }
336
- INVALID_TAGS = {
337
- # Non-string tags.
338
- 123 => '123',
339
- 1.23 => '1.23',
340
- [1, 2, 3] => '%5B1%2C%202%2C%203%5D',
341
- { key: 'value' } => '%7B%22key%22%3D%3E%22value%22%7D',
342
- # Non-utf8 string tags.
343
- "nonutf8#{[0x92].pack('C*')}" => 'nonutf8%20',
344
- "abc#{[0x92].pack('C*')}" => 'abc%20',
345
- "#{[0x92].pack('C*')}" => '%20',
346
- # Empty string tag.
347
- '' => '_'
348
- }
349
- ALL_TAGS = VALID_TAGS.merge(INVALID_TAGS)
350
-
351
45
  # Shared tests.
352
46
 
353
47
  def test_configure_service_account_application_default
@@ -434,12 +128,12 @@ module BaseTest
434
128
 
435
129
  def test_gce_used_when_detect_subservice_is_false
436
130
  setup_gce_metadata_stubs
437
- # This would cause the service to be container.googleapis.com if not for the
438
- # detect_subservice=false config.
131
+ # This would cause the resource type to be container.googleapis.com if not
132
+ # for the detect_subservice=false config.
439
133
  setup_container_metadata_stubs
440
134
  d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
441
135
  d.run
442
- assert_equal COMPUTE_SERVICE_NAME, d.instance.service_name
136
+ assert_equal COMPUTE_CONSTANTS[:resource_type], d.instance.resource.type
443
137
  end
444
138
 
445
139
  def test_metadata_overrides
@@ -554,7 +248,7 @@ module BaseTest
554
248
  verify_log_entries(1, EC2_PARAMS)
555
249
  end
556
250
 
557
- def test_struct_payload_log
251
+ def test_structured_payload_log
558
252
  setup_gce_metadata_stubs
559
253
  setup_logging_stubs do
560
254
  d = create_driver
@@ -562,8 +256,8 @@ module BaseTest
562
256
  'some_null_field' => nil)
563
257
  d.run
564
258
  end
565
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
566
- fields = get_fields(entry['structPayload'])
259
+ verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
260
+ fields = get_fields(entry['jsonPayload'])
567
261
  assert_equal 4, fields.size, entry
568
262
  assert_equal 'test log entry 0', get_string(fields['msg']), entry
569
263
  assert_equal 'test', get_string(fields['tag2']), entry
@@ -572,7 +266,7 @@ module BaseTest
572
266
  end
573
267
  end
574
268
 
575
- def test_struct_payload_malformatted_log
269
+ def test_structured_payload_malformatted_log
576
270
  setup_gce_metadata_stubs
577
271
  message = 'test message'
578
272
  setup_logging_stubs do
@@ -588,8 +282,8 @@ module BaseTest
588
282
  )
589
283
  d.run
590
284
  end
591
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
592
- fields = get_fields(entry['structPayload'])
285
+ verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
286
+ fields = get_fields(entry['jsonPayload'])
593
287
  assert_equal 7, fields.size, entry
594
288
  assert_equal message, get_string(get_fields(get_struct(fields \
595
289
  ['int_key']))['1']), entry
@@ -608,7 +302,7 @@ module BaseTest
608
302
  end
609
303
  end
610
304
 
611
- def test_struct_payload_json_log
305
+ def test_structured_payload_json_log
612
306
  setup_gce_metadata_stubs
613
307
  setup_logging_stubs do
614
308
  d = create_driver
@@ -624,7 +318,7 @@ module BaseTest
624
318
  end
625
319
  end
626
320
 
627
- def test_struct_payload_json_container_log
321
+ def test_structured_payload_json_container_log
628
322
  setup_gce_metadata_stubs
629
323
  setup_container_metadata_stubs
630
324
  setup_logging_stubs do
@@ -643,8 +337,8 @@ module BaseTest
643
337
  if log_index == 1
644
338
  assert entry.key?('textPayload'), 'Entry did not have textPayload'
645
339
  else
646
- assert entry.key?('structPayload'), 'Entry did not have structPayload'
647
- fields = get_fields(entry['structPayload'])
340
+ assert entry.key?('jsonPayload'), 'Entry did not have jsonPayload'
341
+ fields = get_fields(entry['jsonPayload'])
648
342
  assert_equal 4, fields.size, entry
649
343
  assert_equal 'test log entry 0', get_string(fields['msg']), entry
650
344
  assert_equal 'test', get_string(fields['tag2']), entry
@@ -665,7 +359,7 @@ module BaseTest
665
359
  d.emit('msg' => log_entry(0))
666
360
  d.run
667
361
  end
668
- verify_log_entries(0, COMPUTE_PARAMS, 'structPayload')
362
+ verify_log_entries(0, COMPUTE_PARAMS, 'jsonPayload')
669
363
  end
670
364
  end
671
365
 
@@ -678,18 +372,17 @@ module BaseTest
678
372
  # This tag will not match the kubernetes regex because it requires a
679
373
  # non-empty container name.
680
374
  tag = container_tag_with_container_name(container_name)
681
- params = CONTAINER_FROM_METADATA_PARAMS.merge(
682
- labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
683
- "#{CONTAINER_SERVICE_NAME}/container_name" => container_name),
684
- log_name: tag)
685
- setup_logging_stubs([params]) do
686
- @logs_sent = []
375
+ setup_logging_stubs do
687
376
  d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
688
377
  d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
689
378
  d.run
690
379
  end
380
+ params = CONTAINER_FROM_METADATA_PARAMS.merge(
381
+ resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
382
+ labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
383
+ 'container_name' => container_name)),
384
+ log_name: tag)
691
385
  verify_log_entries(1, params, 'textPayload')
692
- assert_equal "projects/#{PROJECT_ID}/logs/#{tag}", @logs_sent[0]['logName']
693
386
  end
694
387
 
695
388
  # Verify that container names with non-utf8 characters should be rejected when
@@ -701,18 +394,18 @@ module BaseTest
701
394
  tag.is_a?(String) && !tag.empty?
702
395
  end
703
396
  non_utf8_tags.each do |container_name, encoded_name|
704
- params = CONTAINER_FROM_METADATA_PARAMS.merge(
705
- labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
706
- "#{CONTAINER_SERVICE_NAME}/container_name" =>
707
- URI.decode(encoded_name)),
708
- log_name: encoded_name)
709
- setup_logging_stubs([params]) do
397
+ setup_logging_stubs do
710
398
  @logs_sent = []
711
399
  d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
712
400
  container_tag_with_container_name(container_name))
713
401
  d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
714
402
  d.run
715
403
  end
404
+ params = CONTAINER_FROM_METADATA_PARAMS.merge(
405
+ labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
406
+ "#{CONTAINER_CONSTANTS[:service]}/container_name" =>
407
+ URI.decode(encoded_name)),
408
+ log_name: encoded_name)
716
409
  verify_log_entries(0, params, 'textPayload')
717
410
  end
718
411
  end
@@ -722,15 +415,14 @@ module BaseTest
722
415
  def test_encode_tags_with_require_valid_tags_true
723
416
  setup_gce_metadata_stubs
724
417
  VALID_TAGS.each do |tag, encoded_tag|
725
- setup_logging_stubs([COMPUTE_PARAMS.merge(log_name: encoded_tag)]) do
418
+ setup_logging_stubs do
726
419
  @logs_sent = []
727
420
  d = create_driver(REQUIRE_VALID_TAGS_CONFIG, tag)
728
421
  d.emit('msg' => log_entry(0))
729
422
  d.run
730
423
  end
731
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload')
732
- assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
733
- @logs_sent[0]['logName']
424
+ verify_log_entries(1, COMPUTE_PARAMS.merge(log_name: encoded_tag),
425
+ 'jsonPayload')
734
426
  end
735
427
  end
736
428
 
@@ -739,20 +431,19 @@ module BaseTest
739
431
  setup_gce_metadata_stubs
740
432
  setup_container_metadata_stubs
741
433
  VALID_TAGS.each do |tag, encoded_tag|
742
- params = CONTAINER_FROM_METADATA_PARAMS.merge(
743
- labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
744
- "#{CONTAINER_SERVICE_NAME}/container_name" => tag),
745
- log_name: encoded_tag)
746
- setup_logging_stubs([params]) do
434
+ setup_logging_stubs do
747
435
  @logs_sent = []
748
436
  d = create_driver(REQUIRE_VALID_TAGS_CONFIG,
749
437
  container_tag_with_container_name(tag))
750
438
  d.emit(container_log_entry_with_metadata(log_entry(0), tag))
751
439
  d.run
752
440
  end
441
+ params = CONTAINER_FROM_METADATA_PARAMS.merge(
442
+ resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
443
+ labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
444
+ 'container_name' => tag)),
445
+ log_name: encoded_tag)
753
446
  verify_log_entries(1, params, 'textPayload')
754
- assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_tag}",
755
- @logs_sent[0]['logName']
756
447
  end
757
448
  end
758
449
 
@@ -762,15 +453,14 @@ module BaseTest
762
453
  def test_sanitize_tags_with_require_valid_tags_false
763
454
  setup_gce_metadata_stubs
764
455
  ALL_TAGS.each do |tag, sanitized_tag|
765
- setup_logging_stubs([COMPUTE_PARAMS.merge(log_name: sanitized_tag)]) do
456
+ setup_logging_stubs do
766
457
  @logs_sent = []
767
458
  d = create_driver(APPLICATION_DEFAULT_CONFIG, tag)
768
459
  d.emit('msg' => log_entry(0))
769
460
  d.run
770
461
  end
771
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload')
772
- assert_equal "projects/#{PROJECT_ID}/logs/#{sanitized_tag}",
773
- @logs_sent[0]['logName']
462
+ verify_log_entries(1, COMPUTE_PARAMS.merge(log_name: sanitized_tag),
463
+ 'jsonPayload')
774
464
  end
775
465
  end
776
466
 
@@ -787,21 +477,19 @@ module BaseTest
787
477
  string_tags.each do |container_name, encoded_container_name|
788
478
  # Container name in the label is sanitized but not encoded, while the log
789
479
  # name is encoded.
790
- params = CONTAINER_FROM_METADATA_PARAMS.merge(
791
- labels: CONTAINER_FROM_METADATA_PARAMS[:labels].merge(
792
- "#{CONTAINER_SERVICE_NAME}/container_name" =>
793
- URI.decode(encoded_container_name)),
794
- log_name: encoded_container_name)
795
- setup_logging_stubs([params]) do
480
+ setup_logging_stubs do
796
481
  @logs_sent = []
797
482
  d = create_driver(APPLICATION_DEFAULT_CONFIG,
798
483
  container_tag_with_container_name(container_name))
799
484
  d.emit(container_log_entry_with_metadata(log_entry(0), container_name))
800
485
  d.run
801
486
  end
487
+ params = CONTAINER_FROM_METADATA_PARAMS.merge(
488
+ resource: CONTAINER_FROM_METADATA_PARAMS[:resource].merge(
489
+ labels: CONTAINER_FROM_METADATA_PARAMS[:resource][:labels].merge(
490
+ 'container_name' => URI.decode(encoded_container_name))),
491
+ log_name: encoded_container_name)
802
492
  verify_log_entries(1, params, 'textPayload')
803
- assert_equal "projects/#{PROJECT_ID}/logs/#{encoded_container_name}",
804
- @logs_sent[0]['logName']
805
493
  end
806
494
  end
807
495
 
@@ -833,9 +521,9 @@ module BaseTest
833
521
  end
834
522
  verify_index = 0
835
523
  verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
836
- assert_equal_with_default entry['metadata']['timestamp']['seconds'],
524
+ assert_equal_with_default entry['timestamp']['seconds'],
837
525
  expected_ts[verify_index].tv_sec, 0, entry
838
- assert_equal_with_default entry['metadata']['timestamp']['nanos'],
526
+ assert_equal_with_default entry['timestamp']['nanos'],
839
527
  expected_ts[verify_index].tv_nsec, 0, entry do
840
528
  # Fluentd v0.14 onwards supports nanosecond timestamp values.
841
529
  # Added in 600 ns delta to avoid flaky tests introduced
@@ -843,7 +531,7 @@ module BaseTest
843
531
  # (to account for the missing 9 bits of precision ~ 512 ns).
844
532
  # See http://wikipedia.org/wiki/Double-precision_floating-point_format
845
533
  assert_in_delta expected_ts[verify_index].tv_nsec,
846
- entry['metadata']['timestamp']['nanos'], 600, entry
534
+ entry['timestamp']['nanos'], 600, entry
847
535
  end
848
536
  verify_index += 1
849
537
  end
@@ -853,12 +541,12 @@ module BaseTest
853
541
  setup_gce_metadata_stubs
854
542
  setup_logging_stubs do
855
543
  d = create_driver
856
- # if timestamp is not a hash it is passed through to the struct payload.
544
+ # if timestamp is not a hash it is passed through to the json payload.
857
545
  d.emit('message' => log_entry(0), 'timestamp' => 'not-a-hash')
858
546
  d.run
859
547
  end
860
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
861
- fields = get_fields(entry['structPayload'])
548
+ verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
549
+ fields = get_fields(entry['jsonPayload'])
862
550
  assert_equal 2, fields.size, entry
863
551
  assert_equal 'not-a-hash', get_string(fields['timestamp']), entry
864
552
  end
@@ -937,7 +625,7 @@ module BaseTest
937
625
  }
938
626
  )
939
627
  d = create_driver(config)
940
- # not_a_label passes through to the struct payload
628
+ # not_a_label passes through to the json payload
941
629
  d.emit('message' => log_entry(0),
942
630
  'label1' => 'value1',
943
631
  'label_number_two' => 'value2',
@@ -950,8 +638,8 @@ module BaseTest
950
638
  params[:labels]['sent_label_1'] = 'value1'
951
639
  params[:labels]['foo.googleapis.com/bar'] = 'value2'
952
640
  params[:labels]['label3'] = 'value3'
953
- verify_log_entries(1, params, 'structPayload') do |entry|
954
- fields = get_fields(entry['structPayload'])
641
+ verify_log_entries(1, params, 'jsonPayload') do |entry|
642
+ fields = get_fields(entry['jsonPayload'])
955
643
  assert_equal 2, fields.size, entry
956
644
  assert_equal 'test log entry 0', get_string(fields['message']), entry
957
645
  assert_equal 'value4', get_string(fields['not_a_label']), entry
@@ -1023,11 +711,9 @@ module BaseTest
1023
711
  d.run
1024
712
  end
1025
713
  verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS) do |entry|
1026
- assert_equal CONTAINER_SECONDS_EPOCH, \
1027
- entry['metadata']['timestamp']['seconds'], entry
1028
- assert_equal CONTAINER_NANOS, \
1029
- entry['metadata']['timestamp']['nanos'], entry
1030
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
714
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
715
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
716
+ assert_equal CONTAINER_SEVERITY, entry['severity'], entry
1031
717
  end
1032
718
  end
1033
719
 
@@ -1045,11 +731,10 @@ module BaseTest
1045
731
  d.run
1046
732
  end
1047
733
  verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS) do |entry|
1048
- assert_equal CONTAINER_SECONDS_EPOCH, \
1049
- entry['metadata']['timestamp']['seconds'], entry
1050
- assert_equal CONTAINER_NANOS, \
1051
- entry['metadata']['timestamp']['nanos'], entry
1052
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
734
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
735
+ entry
736
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
737
+ assert_equal CONTAINER_SEVERITY, entry['severity'], entry
1053
738
  end
1054
739
  end
1055
740
  end
@@ -1068,11 +753,10 @@ module BaseTest
1068
753
  d.run
1069
754
  end
1070
755
  verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS) do |entry|
1071
- assert_equal CONTAINER_SECONDS_EPOCH, \
1072
- entry['metadata']['timestamp']['seconds'], entry
1073
- assert_equal CONTAINER_NANOS, \
1074
- entry['metadata']['timestamp']['nanos'], entry
1075
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
756
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'],
757
+ entry
758
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
759
+ assert_equal CONTAINER_SEVERITY, entry['severity'], entry
1076
760
  end
1077
761
  end
1078
762
  end
@@ -1086,11 +770,9 @@ module BaseTest
1086
770
  d.run
1087
771
  end
1088
772
  verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS) do |entry|
1089
- assert_equal CONTAINER_SECONDS_EPOCH, \
1090
- entry['metadata']['timestamp']['seconds'], entry
1091
- assert_equal CONTAINER_NANOS, \
1092
- entry['metadata']['timestamp']['nanos'], entry
1093
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
773
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
774
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
775
+ assert_equal CONTAINER_SEVERITY, entry['severity'], entry
1094
776
  end
1095
777
  end
1096
778
 
@@ -1103,18 +785,16 @@ module BaseTest
1103
785
  d.run
1104
786
  end
1105
787
  expected_params = CONTAINER_FROM_TAG_PARAMS.merge(
1106
- labels: { "#{CONTAINER_SERVICE_NAME}/stream" => 'stderr' }
788
+ labels: { "#{CONTAINER_CONSTANTS[:service]}/stream" => 'stderr' }
1107
789
  ) { |_, oldval, newval| oldval.merge(newval) }
1108
790
  verify_log_entries(1, expected_params) do |entry|
1109
- assert_equal CONTAINER_SECONDS_EPOCH, \
1110
- entry['metadata']['timestamp']['seconds'], entry
1111
- assert_equal CONTAINER_NANOS, \
1112
- entry['metadata']['timestamp']['nanos'], entry
1113
- assert_equal 'ERROR', entry['metadata']['severity'], entry
791
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
792
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
793
+ assert_equal 'ERROR', entry['severity'], entry
1114
794
  end
1115
795
  end
1116
796
 
1117
- def test_struct_container_log_metadata_from_plugin
797
+ def test_json_container_log_metadata_from_plugin
1118
798
  setup_gce_metadata_stubs
1119
799
  setup_container_metadata_stubs
1120
800
  setup_logging_stubs do
@@ -1125,21 +805,19 @@ module BaseTest
1125
805
  d.run
1126
806
  end
1127
807
  verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
1128
- 'structPayload') do |entry|
1129
- fields = get_fields(entry['structPayload'])
808
+ 'jsonPayload') do |entry|
809
+ fields = get_fields(entry['jsonPayload'])
1130
810
  assert_equal 3, fields.size, entry
1131
811
  assert_equal 'test log entry 0', get_string(fields['msg']), entry
1132
812
  assert_equal 'test', get_string(fields['tag2']), entry
1133
813
  assert_equal 5000, get_number(fields['data']), entry
1134
- assert_equal CONTAINER_SECONDS_EPOCH, \
1135
- entry['metadata']['timestamp']['seconds'], entry
1136
- assert_equal CONTAINER_NANOS, \
1137
- entry['metadata']['timestamp']['nanos'], entry
1138
- assert_equal 'WARNING', entry['metadata']['severity'], entry
814
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
815
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
816
+ assert_equal 'WARNING', entry['severity'], entry
1139
817
  end
1140
818
  end
1141
819
 
1142
- def test_struct_container_log_metadata_from_tag
820
+ def test_json_container_log_metadata_from_tag
1143
821
  setup_gce_metadata_stubs
1144
822
  setup_container_metadata_stubs
1145
823
  setup_logging_stubs do
@@ -1150,17 +828,15 @@ module BaseTest
1150
828
  d.run
1151
829
  end
1152
830
  verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
1153
- 'structPayload') do |entry|
1154
- fields = get_fields(entry['structPayload'])
831
+ 'jsonPayload') do |entry|
832
+ fields = get_fields(entry['jsonPayload'])
1155
833
  assert_equal 3, fields.size, entry
1156
834
  assert_equal 'test log entry 0', get_string(fields['msg']), entry
1157
835
  assert_equal 'test', get_string(fields['tag2']), entry
1158
836
  assert_equal 5000, get_number(fields['data']), entry
1159
- assert_equal CONTAINER_SECONDS_EPOCH, \
1160
- entry['metadata']['timestamp']['seconds'], entry
1161
- assert_equal CONTAINER_NANOS, \
1162
- entry['metadata']['timestamp']['nanos'], entry
1163
- assert_equal 'WARNING', entry['metadata']['severity'], entry
837
+ assert_equal CONTAINER_SECONDS_EPOCH, entry['timestamp']['seconds'], entry
838
+ assert_equal CONTAINER_NANOS, entry['timestamp']['nanos'], entry
839
+ assert_equal 'WARNING', entry['severity'], entry
1164
840
  end
1165
841
  end
1166
842
 
@@ -1178,7 +854,7 @@ module BaseTest
1178
854
  d.run
1179
855
  end
1180
856
  verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
1181
- assert_equal 'DEBUG', entry['metadata']['severity'],
857
+ assert_equal 'DEBUG', entry['severity'],
1182
858
  "Test with #{n} logs failed. \n#{entry}"
1183
859
  end
1184
860
  end
@@ -1199,7 +875,7 @@ module BaseTest
1199
875
  end
1200
876
  verify_log_entries(
1201
877
  n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
1202
- assert_equal 'INFO', entry['metadata']['severity'],
878
+ assert_equal 'INFO', entry['severity'],
1203
879
  "Test with #{n} logs failed. \n#{entry}"
1204
880
  end
1205
881
  end
@@ -1228,16 +904,27 @@ module BaseTest
1228
904
  end
1229
905
  end
1230
906
 
907
+ def test_dataproc_log
908
+ setup_gce_metadata_stubs
909
+ setup_dataproc_metadata_stubs
910
+ setup_logging_stubs do
911
+ d = create_driver
912
+ d.emit(dataproc_log_entry('test message'))
913
+ d.run
914
+ end
915
+ verify_log_entries(1, DATAPROC_PARAMS, 'jsonPayload')
916
+ end
917
+
1231
918
  def test_http_request_from_record
1232
919
  setup_gce_metadata_stubs
1233
920
  setup_logging_stubs do
1234
921
  d = create_driver
1235
- d.emit('httpRequest' => http_request_message)
922
+ d.emit('httpRequest' => HTTP_REQUEST_MESSAGE)
1236
923
  d.run
1237
924
  end
1238
925
  verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
1239
- assert_equal http_request_message, entry['httpRequest'], entry
1240
- assert_nil get_fields(entry['structPayload'])['httpRequest'], entry
926
+ assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
927
+ assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
1241
928
  end
1242
929
  end
1243
930
 
@@ -1245,13 +932,13 @@ module BaseTest
1245
932
  setup_gce_metadata_stubs
1246
933
  setup_logging_stubs do
1247
934
  d = create_driver
1248
- d.emit('httpRequest' => http_request_message.merge(
935
+ d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge(
1249
936
  'otherKey' => 'value'))
1250
937
  d.run
1251
938
  end
1252
939
  verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
1253
- assert_equal http_request_message, entry['httpRequest'], entry
1254
- fields = get_fields(entry['structPayload'])
940
+ assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
941
+ fields = get_fields(entry['jsonPayload'])
1255
942
  request = get_fields(get_struct(fields['httpRequest']))
1256
943
  assert_equal 'value', get_string(request['otherKey']), entry
1257
944
  end
@@ -1264,20 +951,72 @@ module BaseTest
1264
951
  d.emit('httpRequest' => 'a_string')
1265
952
  d.run
1266
953
  end
1267
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
1268
- fields = get_fields(entry['structPayload'])
954
+ verify_log_entries(1, COMPUTE_PARAMS, 'jsonPayload') do |entry|
955
+ fields = get_fields(entry['jsonPayload'])
1269
956
  assert_equal 'a_string', get_string(fields['httpRequest']), entry
1270
957
  assert_nil entry['httpRequest'], entry
1271
958
  end
1272
959
  end
1273
960
 
1274
- private
961
+ def test_http_request_from_record_with_referer_nil_or_absent
962
+ setup_gce_metadata_stubs
963
+ [
964
+ http_request_message_with_nil_referer,
965
+ http_request_message_with_absent_referer
966
+ ].each do |input|
967
+ setup_logging_stubs do
968
+ @logs_sent = []
969
+ d = create_driver
970
+ d.emit('httpRequest' => input)
971
+ d.run
972
+ end
973
+ verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
974
+ assert_equal http_request_message_with_absent_referer,
975
+ entry['httpRequest'], entry
976
+ assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
977
+ end
978
+ end
979
+ end
1275
980
 
1276
- def uri_for_log(params)
1277
- 'https://logging.googleapis.com/v1beta3/projects/' + params[:project_id] +
1278
- '/logs/' + params[:log_name] + '/entries:write'
981
+ def test_http_request_with_latency
982
+ setup_gce_metadata_stubs
983
+ latency_conversion.each do |input, expected|
984
+ setup_logging_stubs do
985
+ d = create_driver
986
+ @logs_sent = []
987
+ d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
988
+ d.run
989
+ end
990
+ verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
991
+ assert_equal HTTP_REQUEST_MESSAGE.merge('latency' => expected),
992
+ entry['httpRequest'], entry
993
+ assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
994
+ end
995
+ end
1279
996
  end
1280
997
 
998
+ # Skip setting latency when the field is null or has invalid format.
999
+ def test_http_request_skip_setting_latency
1000
+ setup_gce_metadata_stubs
1001
+ [
1002
+ '', ' ', nil, 'null', '123', '1.23 seconds',
1003
+ ' 123 s econds ', '1min', 'abc&^!$*('
1004
+ ].each do |input|
1005
+ setup_logging_stubs do
1006
+ d = create_driver
1007
+ @logs_sent = []
1008
+ d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('latency' => input))
1009
+ d.run
1010
+ end
1011
+ verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
1012
+ assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
1013
+ assert_nil get_fields(entry['jsonPayload'])['httpRequest'], entry
1014
+ end
1015
+ end
1016
+ end
1017
+
1018
+ private
1019
+
1281
1020
  def stub_metadata_request(metadata_path, response_body)
1282
1021
  stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
1283
1022
  metadata_path)
@@ -1375,6 +1114,22 @@ module BaseTest
1375
1114
  CLOUDFUNCTIONS_REGION)
1376
1115
  end
1377
1116
 
1117
+ def setup_dataproc_metadata_stubs
1118
+ stub_metadata_request(
1119
+ 'instance/attributes/',
1120
+ "attribute1\ndataproc-cluster-uuid\ndataproc-cluster-name")
1121
+ stub_metadata_request('instance/attributes/dataproc-cluster-name',
1122
+ DATAPROC_CLUSTER_NAME)
1123
+ stub_metadata_request('instance/attributes/dataproc-cluster-uuid',
1124
+ DATAPROC_CLUSTER_UUID)
1125
+ stub_metadata_request('instance/attributes/dataproc-region',
1126
+ DATAPROC_REGION)
1127
+ end
1128
+
1129
+ def setup_prometheus
1130
+ Prometheus::Client.registry.instance_variable_set('@metrics', {})
1131
+ end
1132
+
1378
1133
  def container_tag_with_container_name(container_name)
1379
1134
  "kubernetes.#{CONTAINER_POD_NAME}_#{CONTAINER_NAMESPACE_NAME}_" \
1380
1135
  "#{container_name}"
@@ -1421,42 +1176,74 @@ module BaseTest
1421
1176
  }
1422
1177
  end
1423
1178
 
1179
+ def dataflow_log_entry(i)
1180
+ {
1181
+ step: DATAFLOW_STEP_ID,
1182
+ message: log_entry(i)
1183
+ }
1184
+ end
1185
+
1186
+ def dataproc_log_entry(message, source_class = 'com.example.Example',
1187
+ filename = 'test.log')
1188
+ {
1189
+ filename: filename,
1190
+ class: source_class,
1191
+ message: log_entry(message)
1192
+ }
1193
+ end
1194
+
1195
+ def ml_log_entry(i)
1196
+ {
1197
+ name: ML_LOG_AREA,
1198
+ message: log_entry(i)
1199
+ }
1200
+ end
1201
+
1424
1202
  def log_entry(i)
1425
1203
  'test log entry ' + i.to_s
1426
1204
  end
1427
1205
 
1428
- def check_labels(entry, common_labels, expected_labels)
1429
- # TODO(salty) test/handle overlap between common_labels and entry labels
1430
- all_labels ||= common_labels
1431
- all_labels.merge!(entry['metadata']['labels'] || {})
1432
- all_labels.each do |key, value|
1206
+ def check_labels(labels, expected_labels)
1207
+ labels.each do |key, value|
1433
1208
  assert value.is_a?(String), "Value #{value} for label #{key} " \
1434
1209
  'is not a string: ' + value.class.name
1435
1210
  assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
1436
1211
  assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
1437
1212
  "#{expected_labels[key]} in #{key} => #{value}"
1438
1213
  end
1439
- assert_equal expected_labels.length, all_labels.length, 'Expected ' \
1440
- "#{expected_labels.length} labels, got #{all_labels.length}"
1214
+ assert_equal expected_labels.length, labels.length, 'Expected ' \
1215
+ "#{expected_labels.length} labels: #{expected_labels}, got " \
1216
+ "#{labels.length} labels: #{labels}"
1441
1217
  end
1442
1218
 
1443
1219
  # The caller can optionally provide a block which is called for each entry.
1444
1220
  def verify_json_log_entries(n, params, payload_type = 'textPayload')
1445
1221
  i = 0
1446
- @logs_sent.each do |batch|
1447
- batch['entries'].each do |entry|
1222
+ @logs_sent.each do |request|
1223
+ request['entries'].each do |entry|
1448
1224
  unless payload_type.empty?
1449
1225
  assert entry.key?(payload_type), 'Entry did not contain expected ' \
1450
1226
  "#{payload_type} key: " + entry.to_s
1451
1227
  # Check the payload for textPayload, otherwise it's up to the caller.
1452
1228
  if payload_type == 'textPayload'
1453
- assert_equal "test log entry #{i}", entry['textPayload'], batch
1229
+ assert_equal "test log entry #{i}", entry['textPayload'], request
1454
1230
  end
1455
1231
  end
1456
1232
 
1457
- assert_equal params[:zone], entry['metadata']['zone']
1458
- assert_equal params[:service_name], entry['metadata']['serviceName']
1459
- check_labels entry, batch['commonLabels'], params[:labels]
1233
+ # per-entry resource or log_name overrides the corresponding field
1234
+ # from the request. Labels are merged, with the per-entry label
1235
+ # taking precedence in case of overlap.
1236
+ resource = entry['resource'] || request['resource']
1237
+ log_name = entry['logName'] || request['logName']
1238
+
1239
+ labels ||= request['labels']
1240
+ labels.merge!(entry['labels'] || {})
1241
+
1242
+ assert_equal \
1243
+ "projects/#{params[:project_id]}/logs/#{params[:log_name]}", log_name
1244
+ assert_equal params[:resource][:type], resource['type']
1245
+ check_labels resource['labels'], params[:resource][:labels]
1246
+ check_labels labels, params[:labels]
1460
1247
  yield(entry) if block_given?
1461
1248
  i += 1
1462
1249
  assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
@@ -1465,14 +1252,37 @@ module BaseTest
1465
1252
  assert i == n, "Number of entries #{i} does not match expected number #{n}"
1466
1253
  end
1467
1254
 
1468
- # The http request message to test against.
1469
- def http_request_message
1470
- HTTP_REQUEST_MESSAGE
1471
- end
1472
-
1473
1255
  # Replace the 'referer' field with nil.
1474
1256
  def http_request_message_with_nil_referer
1475
- http_request_message.merge('referer' => nil)
1257
+ HTTP_REQUEST_MESSAGE.merge('referer' => nil)
1258
+ end
1259
+
1260
+ # Unset the 'referer' field.
1261
+ def http_request_message_with_absent_referer
1262
+ HTTP_REQUEST_MESSAGE.reject do |k, _|
1263
+ k == 'referer'
1264
+ end
1265
+ end
1266
+
1267
+ # The conversions from user input to output.
1268
+ def latency_conversion
1269
+ {
1270
+ '32 s' => { 'seconds' => 32 },
1271
+ '32s' => { 'seconds' => 32 },
1272
+ '0.32s' => { 'nanos' => 320_000_000 },
1273
+ ' 123 s ' => { 'seconds' => 123 },
1274
+ '1.3442 s' => { 'seconds' => 1, 'nanos' => 344_200_000 },
1275
+
1276
+ # Test whitespace.
1277
+ # \t: tab. \r: carriage return. \n: line break.
1278
+ # \v: vertical whitespace. \f: form feed.
1279
+ "\t123.5\ts\t" => { 'seconds' => 123, 'nanos' => 500_000_000 },
1280
+ "\r123.5\rs\r" => { 'seconds' => 123, 'nanos' => 500_000_000 },
1281
+ "\n123.5\ns\n" => { 'seconds' => 123, 'nanos' => 500_000_000 },
1282
+ "\v123.5\vs\v" => { 'seconds' => 123, 'nanos' => 500_000_000 },
1283
+ "\f123.5\fs\f" => { 'seconds' => 123, 'nanos' => 500_000_000 },
1284
+ "\r123.5\ts\f" => { 'seconds' => 123, 'nanos' => 500_000_000 }
1285
+ }
1476
1286
  end
1477
1287
 
1478
1288
  # This module expects the methods below to be overridden.
@@ -1503,8 +1313,14 @@ module BaseTest
1503
1313
  _undefined
1504
1314
  end
1505
1315
 
1506
- # Get the fields of the struct payload.
1507
- def get_fields(_struct_payload)
1316
+ def assert_prometheus_metric_value(metric_name, expected_value, labels = {})
1317
+ metric = Prometheus::Client.registry.get(metric_name)
1318
+ assert_not_nil(metric)
1319
+ assert_equal(expected_value, metric.get(labels))
1320
+ end
1321
+
1322
+ # Get the fields of the payload.
1323
+ def get_fields(_payload)
1508
1324
  _undefined
1509
1325
  end
1510
1326