fluent-plugin-google-cloud 0.5.3.grpc.alpha.3 → 0.5.3.grpc.alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,802 +12,16 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- require 'helper'
16
- require 'json'
17
- require 'time'
18
- require 'mocha/test_unit'
19
- require 'webmock/test_unit'
20
- require 'google/apis'
15
+ require_relative 'base_test'
21
16
 
22
17
  # Unit tests for Google Cloud Logging plugin
23
18
  class GoogleCloudOutputTest < Test::Unit::TestCase
24
- def setup
25
- Fluent::Test.setup
26
- # delete environment variables that googleauth uses to find credentials.
27
- ENV.delete('GOOGLE_APPLICATION_CREDENTIALS')
28
- # service account env.
29
- ENV.delete('PRIVATE_KEY_VAR')
30
- ENV.delete('CLIENT_EMAIL_VAR')
31
- # authorized_user env.
32
- ENV.delete('CLIENT_ID_VAR')
33
- ENV.delete('CLIENT_SECRET_VAR')
34
- ENV.delete('REFRESH_TOKEN_VAR')
35
- # home var, which is used to find $HOME/.gcloud/...
36
- ENV.delete('HOME')
19
+ include BaseTest
37
20
 
38
- setup_auth_stubs
39
- @logs_sent = []
40
- end
41
-
42
- # generic attributes
43
- HOSTNAME = Socket.gethostname
44
-
45
- # attributes used for the GCE metadata service
46
- PROJECT_ID = 'test-project-id'
47
- ZONE = 'us-central1-b'
48
- FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
49
- VM_ID = '9876543210'
50
-
51
- # attributes used for custom (overridden) configs
52
- CUSTOM_PROJECT_ID = 'test-custom-project-id'
53
- CUSTOM_ZONE = 'us-custom-central1-b'
54
- CUSTOM_FULLY_QUALIFIED_ZONE = 'projects/' + PROJECT_ID + '/zones/' + ZONE
55
- CUSTOM_VM_ID = 'C9876543210'
56
- CUSTOM_HOSTNAME = 'custom.hostname.org'
57
-
58
- # attributes used for the EC2 metadata service
59
- EC2_PROJECT_ID = 'test-ec2-project-id'
60
- EC2_ZONE = 'us-west-2b'
61
- EC2_PREFIXED_ZONE = 'aws:' + EC2_ZONE
62
- EC2_VM_ID = 'i-81c16767'
63
- EC2_ACCOUNT_ID = '123456789012'
64
-
65
- # The formatting here matches the format used on the VM.
66
- EC2_IDENTITY_DOCUMENT = %({
67
- "accountId" : "#{EC2_ACCOUNT_ID}",
68
- "availabilityZone" : "#{EC2_ZONE}",
69
- "instanceId" : "#{EC2_VM_ID}"
70
- })
71
-
72
- # Managed VMs specific labels
73
- MANAGED_VM_BACKEND_NAME = 'default'
74
- MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
75
-
76
- # Container Engine / Kubernetes specific labels
77
- CONTAINER_CLUSTER_NAME = 'cluster-1'
78
- CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
79
- CONTAINER_NAMESPACE_NAME = 'kube-system'
80
- CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
81
- CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
82
- CONTAINER_CONTAINER_NAME = 'redis'
83
- CONTAINER_LABEL_KEY = 'component'
84
- CONTAINER_LABEL_VALUE = 'redis-component'
85
- CONTAINER_STREAM = 'stdout'
86
- CONTAINER_SEVERITY = 'INFO'
87
- # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch
88
- CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
89
- CONTAINER_SECONDS_EPOCH = 1_234_567_890
90
- CONTAINER_NANOS = 987_654_321
91
-
92
- # Cloud Functions specific labels
93
- CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
94
- CLOUDFUNCTIONS_REGION = 'us-central1'
95
- CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
96
- CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
97
- CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
98
- CLOUDFUNCTIONS_POD_NAME = 'd.dc.myu.uc.functionp.pc.name-a.a1.987-c0l82'
99
- CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
100
-
101
- # Parameters used for authentication
102
- AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
103
- FAKE_AUTH_TOKEN = 'abc123'
104
-
105
- # Information about test credentials files.
106
- # path: Path to the credentials file.
107
- # project_id: ID of the project, which must correspond to the file contents.
108
- IAM_CREDENTIALS = {
109
- path: 'test/plugin/data/iam-credentials.json',
110
- project_id: 'fluent-test-project'
111
- }
112
- LEGACY_CREDENTIALS = {
113
- path: 'test/plugin/data/credentials.json',
114
- project_id: '847859579879'
115
- }
116
- INVALID_CREDENTIALS = {
117
- path: 'test/plugin/data/invalid_credentials.json',
118
- project_id: ''
119
- }
120
-
121
- # Configuration files for various test scenarios
122
- APPLICATION_DEFAULT_CONFIG = %(
123
- )
124
-
125
- # rubocop:disable Metrics/LineLength
126
- PRIVATE_KEY_CONFIG = %(
127
- auth_method private_key
128
- private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
129
- private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
130
- )
131
- # rubocop:enable Metrics/LineLength
132
-
133
- NO_METADATA_SERVICE_CONFIG = %(
134
- use_metadata_service false
135
- )
136
-
137
- NO_DETECT_SUBSERVICE_CONFIG = %(
138
- detect_subservice false
139
- )
140
-
141
- CUSTOM_METADATA_CONFIG = %(
142
- project_id #{CUSTOM_PROJECT_ID}
143
- zone #{CUSTOM_ZONE}
144
- vm_id #{CUSTOM_VM_ID}
145
- vm_name #{CUSTOM_HOSTNAME}
146
- )
147
-
148
- CONFIG_MISSING_METADATA_PROJECT_ID = %(
149
- zone #{CUSTOM_ZONE}
150
- vm_id #{CUSTOM_VM_ID}
151
- )
152
- CONFIG_MISSING_METADATA_ZONE = %(
153
- project_id #{CUSTOM_PROJECT_ID}
154
- vm_id #{CUSTOM_VM_ID}
155
- )
156
- CONFIG_MISSING_METADATA_VM_ID = %(
157
- project_id #{CUSTOM_PROJECT_ID}
158
- zone #{CUSTOM_ZONE}
159
- )
160
- CONFIG_MISSING_METADATA_ALL = %(
161
- )
162
-
163
- CONFIG_EC2_PROJECT_ID = %(
164
- project_id #{EC2_PROJECT_ID}
165
- )
166
-
167
- CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
168
- project_id #{EC2_PROJECT_ID}
169
- vm_id #{CUSTOM_VM_ID}
170
- )
171
-
172
- # Service configurations for various services
173
- COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
174
- APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
175
- CONTAINER_SERVICE_NAME = 'container.googleapis.com'
176
- CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
177
- EC2_SERVICE_NAME = 'ec2.amazonaws.com'
178
-
179
- COMPUTE_PARAMS = {
180
- service_name: COMPUTE_SERVICE_NAME,
181
- log_name: 'test',
182
- project_id: PROJECT_ID,
183
- zone: ZONE,
184
- labels: {
185
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
186
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
187
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
188
- }
189
- }
190
-
191
- VMENGINE_PARAMS = {
192
- service_name: APPENGINE_SERVICE_NAME,
193
- log_name: "#{APPENGINE_SERVICE_NAME}%2Ftest",
194
- project_id: PROJECT_ID,
195
- zone: ZONE,
196
- labels: {
197
- "#{APPENGINE_SERVICE_NAME}/module_id" => MANAGED_VM_BACKEND_NAME,
198
- "#{APPENGINE_SERVICE_NAME}/version_id" => MANAGED_VM_BACKEND_VERSION,
199
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
200
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
201
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
202
- }
203
- }
204
-
205
- CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
206
- "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
207
-
208
- CONTAINER_FROM_METADATA_PARAMS = {
209
- service_name: CONTAINER_SERVICE_NAME,
210
- log_name: CONTAINER_CONTAINER_NAME,
211
- project_id: PROJECT_ID,
212
- zone: ZONE,
213
- labels: {
214
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
215
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
216
- "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
217
- "#{CONTAINER_SERVICE_NAME}/namespace_id" => CONTAINER_NAMESPACE_ID,
218
- "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
219
- "#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
220
- "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
221
- "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
222
- "label/#{CONTAINER_LABEL_KEY}" => CONTAINER_LABEL_VALUE,
223
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
224
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
225
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
226
- }
227
- }
228
-
229
- # Almost the same as from metadata, but missing namespace_id and pod_id.
230
- CONTAINER_FROM_TAG_PARAMS = {
231
- service_name: CONTAINER_SERVICE_NAME,
232
- log_name: CONTAINER_CONTAINER_NAME,
233
- project_id: PROJECT_ID,
234
- zone: ZONE,
235
- labels: {
236
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
237
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
238
- "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
239
- "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
240
- "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
241
- "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
242
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
243
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
244
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
245
- }
246
- }
247
-
248
- CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
249
- "#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
250
- "#{CLOUDFUNCTIONS_CONTAINER_NAME}"
251
-
252
- CLOUDFUNCTIONS_PARAMS = {
253
- service_name: CLOUDFUNCTIONS_SERVICE_NAME,
254
- log_name: 'cloud-functions',
255
- project_id: PROJECT_ID,
256
- zone: ZONE,
257
- labels: {
258
- 'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
259
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
260
- CLOUDFUNCTIONS_FUNCTION_NAME,
261
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
262
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
263
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
264
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
265
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
266
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
267
- }
268
- }
269
-
270
- CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
271
- service_name: CLOUDFUNCTIONS_SERVICE_NAME,
272
- log_name: 'cloud-functions',
273
- project_id: PROJECT_ID,
274
- zone: ZONE,
275
- labels: {
276
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
277
- CLOUDFUNCTIONS_FUNCTION_NAME,
278
- "#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
279
- "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
280
- "#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
281
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
282
- "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
283
- "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
284
- }
285
- }
286
-
287
- CUSTOM_PARAMS = {
288
- service_name: COMPUTE_SERVICE_NAME,
289
- log_name: 'test',
290
- project_id: CUSTOM_PROJECT_ID,
291
- zone: CUSTOM_ZONE,
292
- labels: {
293
- "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
294
- "#{COMPUTE_SERVICE_NAME}/resource_id" => CUSTOM_VM_ID,
295
- "#{COMPUTE_SERVICE_NAME}/resource_name" => CUSTOM_HOSTNAME
296
- }
297
- }
298
-
299
- EC2_PARAMS = {
300
- service_name: EC2_SERVICE_NAME,
301
- log_name: 'test',
302
- project_id: EC2_PROJECT_ID,
303
- zone: EC2_PREFIXED_ZONE,
304
- labels: {
305
- "#{EC2_SERVICE_NAME}/resource_type" => 'instance',
306
- "#{EC2_SERVICE_NAME}/resource_id" => EC2_VM_ID,
307
- "#{EC2_SERVICE_NAME}/account_id" => EC2_ACCOUNT_ID,
308
- "#{EC2_SERVICE_NAME}/resource_name" => HOSTNAME
309
- }
310
- }
311
-
312
- HTTP_REQUEST_MESSAGE = {
313
- 'requestMethod' => 'POST',
314
- 'requestUrl' => 'http://example/',
315
- 'requestSize' => 210,
316
- 'status' => 200,
317
- 'responseSize' => 65,
318
- 'userAgent' => 'USER AGENT 1.0',
319
- 'remoteIp' => '55.55.55.55',
320
- 'referer' => 'http://referer/',
321
- 'cacheHit' => false,
322
- 'validatedWithOriginServer' => true
323
- }
324
-
325
- def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
326
- Fluent::Test::BufferedOutputTestDriver.new(
327
- Fluent::GoogleCloudOutput, tag).configure(conf, use_v1_config: true)
328
- end
329
-
330
- def test_configure_service_account_application_default
331
- setup_gce_metadata_stubs
332
- d = create_driver(APPLICATION_DEFAULT_CONFIG)
333
- assert_equal HOSTNAME, d.instance.vm_name
334
- end
335
-
336
- def test_configure_service_account_private_key
337
- # Using out-of-date config method.
338
- setup_gce_metadata_stubs
339
- exception_count = 0
340
- begin
341
- _d = create_driver(PRIVATE_KEY_CONFIG)
342
- rescue Fluent::ConfigError => error
343
- assert error.message.include? 'Please remove configuration parameters'
344
- exception_count += 1
345
- end
346
- assert_equal 1, exception_count
347
- end
348
-
349
- def test_configure_custom_metadata
350
- setup_no_metadata_service_stubs
351
- d = create_driver(CUSTOM_METADATA_CONFIG)
352
- assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
353
- assert_equal CUSTOM_ZONE, d.instance.zone
354
- assert_equal CUSTOM_VM_ID, d.instance.vm_id
355
- end
356
-
357
- def test_configure_invalid_metadata_missing_project_id_no_metadata_service
358
- setup_no_metadata_service_stubs
359
- exception_count = 0
360
- begin
361
- _d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
362
- rescue Fluent::ConfigError => error
363
- assert error.message.include? 'Unable to obtain metadata parameters:'
364
- assert error.message.include? 'project_id'
365
- exception_count += 1
366
- end
367
- assert_equal 1, exception_count
368
- end
369
-
370
- def test_configure_invalid_metadata_missing_zone_no_metadata_service
371
- setup_no_metadata_service_stubs
372
- exception_count = 0
373
- begin
374
- _d = create_driver(CONFIG_MISSING_METADATA_ZONE)
375
- rescue Fluent::ConfigError => error
376
- assert error.message.include? 'Unable to obtain metadata parameters:'
377
- assert error.message.include? 'zone'
378
- exception_count += 1
379
- end
380
- assert_equal 1, exception_count
381
- end
382
-
383
- def test_configure_invalid_metadata_missing_vm_id_no_metadata_service
384
- setup_no_metadata_service_stubs
385
- exception_count = 0
386
- begin
387
- _d = create_driver(CONFIG_MISSING_METADATA_VM_ID)
388
- rescue Fluent::ConfigError => error
389
- assert error.message.include? 'Unable to obtain metadata parameters:'
390
- assert error.message.include? 'vm_id'
391
- exception_count += 1
392
- end
393
- assert_equal 1, exception_count
394
- end
395
-
396
- def test_configure_invalid_metadata_missing_all_no_metadata_service
397
- setup_no_metadata_service_stubs
398
- exception_count = 0
399
- begin
400
- _d = create_driver(CONFIG_MISSING_METADATA_ALL)
401
- rescue Fluent::ConfigError => error
402
- assert error.message.include? 'Unable to obtain metadata parameters:'
403
- assert error.message.include? 'project_id'
404
- assert error.message.include? 'zone'
405
- assert error.message.include? 'vm_id'
406
- exception_count += 1
407
- end
408
- assert_equal 1, exception_count
409
- end
410
-
411
- def test_metadata_loading
412
- setup_gce_metadata_stubs
413
- d = create_driver
414
- d.run
415
- assert_equal PROJECT_ID, d.instance.project_id
416
- assert_equal ZONE, d.instance.zone
417
- assert_equal VM_ID, d.instance.vm_id
418
- assert_equal false, d.instance.running_on_managed_vm
419
- end
420
-
421
- def test_managed_vm_metadata_loading
422
- setup_gce_metadata_stubs
423
- setup_managed_vm_metadata_stubs
424
- d = create_driver
425
- d.run
426
- assert_equal PROJECT_ID, d.instance.project_id
427
- assert_equal ZONE, d.instance.zone
428
- assert_equal VM_ID, d.instance.vm_id
429
- assert_equal true, d.instance.running_on_managed_vm
430
- assert_equal MANAGED_VM_BACKEND_NAME, d.instance.gae_backend_name
431
- assert_equal MANAGED_VM_BACKEND_VERSION, d.instance.gae_backend_version
432
- end
433
-
434
- def test_gce_metadata_does_not_load_when_use_metadata_service_is_false
435
- Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
436
- d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
437
- d.run
438
- assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
439
- assert_equal CUSTOM_ZONE, d.instance.zone
440
- assert_equal CUSTOM_VM_ID, d.instance.vm_id
441
- assert_equal false, d.instance.running_on_managed_vm
442
- end
443
-
444
- def test_gce_used_when_detect_subservice_is_false
445
- setup_gce_metadata_stubs
446
- # This would cause the service to be container.googleapis.com if not for the
447
- # detect_subservice=false config.
448
- setup_container_metadata_stubs
449
- d = create_driver(NO_DETECT_SUBSERVICE_CONFIG)
450
- d.run
451
- assert_equal COMPUTE_SERVICE_NAME, d.instance.service_name
452
- end
453
-
454
- def test_metadata_overrides_on_gce
455
- # In this case we are overriding all configured parameters so we should
456
- # see all "custom" values rather than the ones from the metadata server.
457
- setup_gce_metadata_stubs
458
- d = create_driver(CUSTOM_METADATA_CONFIG)
459
- d.run
460
- assert_equal CUSTOM_PROJECT_ID, d.instance.project_id
461
- assert_equal CUSTOM_ZONE, d.instance.zone
462
- assert_equal CUSTOM_VM_ID, d.instance.vm_id
463
- assert_equal false, d.instance.running_on_managed_vm
464
- end
465
-
466
- def test_metadata_partial_overrides_on_gce
467
- # Similar to above, but we are not overriding project_id in this config
468
- # so we should see the metadata value for project_id and "custom" otherwise.
469
- setup_gce_metadata_stubs
470
- d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
471
- d.run
472
- assert_equal PROJECT_ID, d.instance.project_id
473
- assert_equal CUSTOM_ZONE, d.instance.zone
474
- assert_equal CUSTOM_VM_ID, d.instance.vm_id
475
- assert_equal false, d.instance.running_on_managed_vm
476
- end
477
-
478
- def test_ec2_metadata_loading
479
- setup_ec2_metadata_stubs
480
- d = create_driver(CONFIG_EC2_PROJECT_ID)
481
- d.run
482
- assert_equal EC2_PROJECT_ID, d.instance.project_id
483
- assert_equal EC2_PREFIXED_ZONE, d.instance.zone
484
- assert_equal EC2_VM_ID, d.instance.vm_id
485
- assert_equal false, d.instance.running_on_managed_vm
486
- end
487
-
488
- def test_ec2_metadata_partial_override
489
- setup_ec2_metadata_stubs
490
- d = create_driver(CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID)
491
- d.run
492
- assert_equal EC2_PROJECT_ID, d.instance.project_id
493
- assert_equal EC2_PREFIXED_ZONE, d.instance.zone
494
- assert_equal CUSTOM_VM_ID, d.instance.vm_id
495
- assert_equal false, d.instance.running_on_managed_vm
496
- end
497
-
498
- def test_ec2_metadata_requires_project_id
499
- setup_ec2_metadata_stubs
500
- exception_count = 0
501
- begin
502
- _d = create_driver
503
- rescue Fluent::ConfigError => error
504
- assert error.message.include? 'Unable to obtain metadata parameters:'
505
- assert error.message.include? 'project_id'
506
- exception_count += 1
507
- end
508
- assert_equal 1, exception_count
509
- end
510
-
511
- def test_ec2_metadata_project_id_from_credentials
512
- setup_ec2_metadata_stubs
513
- [IAM_CREDENTIALS, LEGACY_CREDENTIALS].each do |creds|
514
- ENV['GOOGLE_APPLICATION_CREDENTIALS'] = creds[:path]
515
- d = create_driver
516
- d.run
517
- assert_equal creds[:project_id], d.instance.project_id
518
- end
519
- end
520
-
521
- def test_one_log
522
- setup_gce_metadata_stubs
523
- setup_logging_stubs
524
- d = create_driver
525
- d.emit('message' => log_entry(0))
526
- d.run
527
- verify_log_entries(1, COMPUTE_PARAMS)
528
- end
529
-
530
- def test_one_log_with_json_credentials
21
+ def test_configure_use_grpc
531
22
  setup_gce_metadata_stubs
532
- setup_logging_stubs
533
- ENV['GOOGLE_APPLICATION_CREDENTIALS'] = IAM_CREDENTIALS[:path]
534
23
  d = create_driver
535
- d.emit('message' => log_entry(0))
536
- d.run
537
- verify_log_entries(1, COMPUTE_PARAMS)
538
- end
539
-
540
- def test_one_log_with_invalid_json_credentials
541
- setup_gce_metadata_stubs
542
- setup_logging_stubs
543
- ENV['GOOGLE_APPLICATION_CREDENTIALS'] = INVALID_CREDENTIALS[:path]
544
- d = create_driver
545
- d.emit('message' => log_entry(0))
546
- exception_count = 0
547
- begin
548
- d.run
549
- rescue RuntimeError => error
550
- assert error.message.include? 'Unable to read the credential file'
551
- exception_count += 1
552
- end
553
- assert_equal 1, exception_count
554
- end
555
-
556
- def test_one_log_custom_metadata
557
- # don't set up any metadata stubs, so the test will fail if we try to
558
- # fetch metadata (and explicitly check this as well).
559
- Fluent::GoogleCloudOutput.any_instance.expects(:fetch_metadata).never
560
- ENV['GOOGLE_APPLICATION_CREDENTIALS'] = IAM_CREDENTIALS[:path]
561
- setup_logging_stubs
562
- d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
563
- d.emit('message' => log_entry(0))
564
- d.run
565
- verify_log_entries(1, CUSTOM_PARAMS)
566
- end
567
-
568
- def test_one_log_ec2
569
- ENV['GOOGLE_APPLICATION_CREDENTIALS'] = IAM_CREDENTIALS[:path]
570
- setup_ec2_metadata_stubs
571
- setup_logging_stubs
572
- d = create_driver(CONFIG_EC2_PROJECT_ID)
573
- d.emit('message' => log_entry(0))
574
- d.run
575
- verify_log_entries(1, EC2_PARAMS)
576
- end
577
-
578
- def test_struct_payload_log
579
- setup_gce_metadata_stubs
580
- setup_logging_stubs
581
- d = create_driver
582
- d.emit('msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000)
583
- d.run
584
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
585
- assert_equal 3, entry['structPayload'].size, entry
586
- assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
587
- assert_equal 'test', entry['structPayload']['tag2'], entry
588
- assert_equal 5000, entry['structPayload']['data'], entry
589
- end
590
- end
591
-
592
- def test_struct_payload_json_log
593
- setup_gce_metadata_stubs
594
- setup_logging_stubs
595
- d = create_driver
596
- json_string = '{"msg": "test log entry 0", "tag2": "test", "data": 5000}'
597
- d.emit('message' => 'notJSON ' + json_string)
598
- d.emit('message' => json_string)
599
- d.emit('message' => "\t" + json_string)
600
- d.emit('message' => ' ' + json_string)
601
- d.run
602
- verify_log_entries(4, COMPUTE_PARAMS, '') do |entry|
603
- assert entry.key?('textPayload'), 'Entry did not have textPayload'
604
- end
605
- end
606
-
607
- def test_struct_payload_json_container_log
608
- setup_gce_metadata_stubs
609
- setup_container_metadata_stubs
610
- setup_logging_stubs
611
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
612
- json_string = '{"msg": "test log entry 0", "tag2": "test", "data": 5000}'
613
- d.emit(container_log_entry_with_metadata('notJSON' + json_string))
614
- d.emit(container_log_entry_with_metadata(json_string))
615
- d.emit(container_log_entry_with_metadata(" \r\n \t" + json_string))
616
- d.run
617
- log_index = 0
618
- verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, '') do |entry|
619
- log_index += 1
620
- if log_index == 1
621
- assert entry.key?('textPayload'), 'Entry did not have textPayload'
622
- else
623
- assert entry.key?('structPayload'), 'Entry did not have structPayload'
624
- assert_equal 3, entry['structPayload'].size, entry
625
- assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
626
- assert_equal 'test', entry['structPayload']['tag2'], entry
627
- assert_equal 5000, entry['structPayload']['data'], entry
628
- end
629
- end
630
- end
631
-
632
- def test_timestamps
633
- setup_gce_metadata_stubs
634
- setup_logging_stubs
635
- d = create_driver
636
- expected_ts = []
637
- emit_index = 0
638
- [Time.at(123_456.789), Time.at(0), Time.now].each do |ts|
639
- # Test the "native" fluentd timestamp as well as our nanosecond tags.
640
- d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
641
- # The native timestamp currently only supports second granularity
642
- # (fluentd issue #461), so strip nanoseconds from the expected value.
643
- expected_ts.push(Time.at(ts.tv_sec))
644
- emit_index += 1
645
- d.emit('message' => log_entry(emit_index),
646
- 'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
647
- expected_ts.push(ts)
648
- emit_index += 1
649
- d.emit('message' => log_entry(emit_index),
650
- 'timestamp' => { 'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec })
651
- expected_ts.push(ts)
652
- emit_index += 1
653
- d.emit('message' => log_entry(emit_index),
654
- 'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec)
655
- expected_ts.push(ts)
656
- emit_index += 1
657
- end
658
- d.run
659
- verify_index = 0
660
- verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
661
- assert_equal expected_ts[verify_index].tv_sec,
662
- entry['metadata']['timestamp']['seconds'], entry
663
- assert_equal expected_ts[verify_index].tv_nsec,
664
- entry['metadata']['timestamp']['nanos'], entry
665
- verify_index += 1
666
- end
667
- end
668
-
669
- def test_malformed_timestamp
670
- setup_gce_metadata_stubs
671
- setup_logging_stubs
672
- d = create_driver
673
- # if timestamp is not a hash it is passed through to the struct payload.
674
- d.emit('message' => log_entry(0), 'timestamp' => 'not-a-hash')
675
- d.run
676
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
677
- assert_equal 2, entry['structPayload'].size, entry
678
- assert_equal 'not-a-hash', entry['structPayload']['timestamp'], entry
679
- end
680
- end
681
-
682
- def test_severities
683
- setup_gce_metadata_stubs
684
- setup_logging_stubs
685
- d = create_driver
686
- expected_severity = []
687
- emit_index = 0
688
- # Array of pairs of [parsed_severity, expected_severity]
689
- [%w(INFO INFO), %w(warn WARNING), %w(E ERROR), %w(BLAH DEFAULT),
690
- ['105', 100], ['', 'DEFAULT']].each do |sev|
691
- d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
692
- expected_severity.push(sev[1])
693
- emit_index += 1
694
- end
695
- d.run
696
- verify_index = 0
697
- verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
698
- assert_equal expected_severity[verify_index],
699
- entry['metadata']['severity'], entry
700
- verify_index += 1
701
- end
702
- end
703
-
704
- def test_label_map_without_field_present
705
- setup_gce_metadata_stubs
706
- setup_logging_stubs
707
- config = %(label_map { "label_field": "sent_label" })
708
- d = create_driver(config)
709
- d.emit('message' => log_entry(0))
710
- d.run
711
- # No additional labels should be present
712
- verify_log_entries(1, COMPUTE_PARAMS)
713
- end
714
-
715
- def test_label_map_with_field_present
716
- setup_gce_metadata_stubs
717
- setup_logging_stubs
718
- config = %(label_map { "label_field": "sent_label" })
719
- d = create_driver(config)
720
- d.emit('message' => log_entry(0), 'label_field' => 'label_value')
721
- d.run
722
- # make a deep copy of COMPUTE_PARAMS and add the parsed label.
723
- params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
724
- params[:labels]['sent_label'] = 'label_value'
725
- verify_log_entries(1, params)
726
- end
727
-
728
- def test_label_map_with_numeric_field
729
- setup_gce_metadata_stubs
730
- setup_logging_stubs
731
- config = %(label_map { "label_field": "sent_label" })
732
- d = create_driver(config)
733
- d.emit('message' => log_entry(0), 'label_field' => 123_456_789)
734
- d.run
735
- # make a deep copy of COMPUTE_PARAMS and add the parsed label.
736
- params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
737
- params[:labels]['sent_label'] = '123456789'
738
- verify_log_entries(1, params)
739
- end
740
-
741
- def test_label_map_with_hash_field
742
- setup_gce_metadata_stubs
743
- setup_logging_stubs
744
- config = %(label_map { "label_field": "sent_label" })
745
- d = create_driver(config)
746
- # I'm not sure this actually makes sense for a user to do, but make
747
- # sure that it works if they try it.
748
- d.emit('message' => log_entry(0),
749
- 'label_field' => { 'k1' => 10, 'k2' => 'val' })
750
- d.run
751
- # make a deep copy of COMPUTE_PARAMS and add the parsed label.
752
- params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
753
- params[:labels]['sent_label'] = '{"k1"=>10, "k2"=>"val"}'
754
- verify_log_entries(1, params)
755
- end
756
-
757
- def test_label_map_with_multiple_fields
758
- setup_gce_metadata_stubs
759
- setup_logging_stubs
760
- config = %(
761
- label_map {
762
- "label1": "sent_label_1",
763
- "label_number_two": "foo.googleapis.com/bar",
764
- "label3": "label3"
765
- }
766
- )
767
- d = create_driver(config)
768
- # not_a_label passes through to the struct payload
769
- d.emit('message' => log_entry(0),
770
- 'label1' => 'value1',
771
- 'label_number_two' => 'value2',
772
- 'not_a_label' => 'value4',
773
- 'label3' => 'value3')
774
- d.run
775
- # make a deep copy of COMPUTE_PARAMS and add the parsed labels.
776
- params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
777
- params[:labels]['sent_label_1'] = 'value1'
778
- params[:labels]['foo.googleapis.com/bar'] = 'value2'
779
- params[:labels]['label3'] = 'value3'
780
- verify_log_entries(1, params, 'structPayload') do |entry|
781
- assert_equal 2, entry['structPayload'].size, entry
782
- assert_equal 'test log entry 0', entry['structPayload']['message'], entry
783
- assert_equal 'value4', entry['structPayload']['not_a_label'], entry
784
- end
785
- end
786
-
787
- def test_multiple_logs
788
- setup_gce_metadata_stubs
789
- setup_logging_stubs
790
- d = create_driver
791
- # Only test a few values because otherwise the test can take minutes.
792
- [2, 3, 5, 11, 50].each do |n|
793
- # The test driver doesn't clear its buffer of entries after running, so
794
- # do it manually here.
795
- d.instance_variable_get('@entries').clear
796
- @logs_sent = []
797
- n.times { |i| d.emit('message' => log_entry(i)) }
798
- d.run
799
- verify_log_entries(n, COMPUTE_PARAMS)
800
- end
801
- end
802
-
803
- def test_malformed_log
804
- setup_gce_metadata_stubs
805
- setup_logging_stubs
806
- d = create_driver
807
- # if the entry is not a hash, the plugin should silently drop it.
808
- d.emit('a string is not a valid message')
809
- d.run
810
- assert @logs_sent.empty?
24
+ assert_false d.instance.instance_variable_get(:@use_grpc)
811
25
  end
812
26
 
813
27
  def test_client_400
@@ -856,310 +70,36 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
856
70
  assert_equal 1, exception_count
857
71
  end
858
72
 
859
- def test_one_managed_vm_log
860
- setup_gce_metadata_stubs
861
- setup_managed_vm_metadata_stubs
862
- setup_logging_stubs
863
- d = create_driver
864
- d.emit('message' => log_entry(0))
865
- d.run
866
- verify_log_entries(1, VMENGINE_PARAMS)
867
- end
868
-
869
- def test_multiple_managed_vm_logs
870
- setup_gce_metadata_stubs
871
- setup_managed_vm_metadata_stubs
872
- setup_logging_stubs
873
- d = create_driver
874
- [2, 3, 5, 11, 50].each do |n|
875
- # The test driver doesn't clear its buffer of entries after running, so
876
- # do it manually here.
877
- d.instance_variable_get('@entries').clear
878
- @logs_sent = []
879
- n.times { |i| d.emit('message' => log_entry(i)) }
880
- d.run
881
- verify_log_entries(n, VMENGINE_PARAMS)
882
- end
883
- end
884
-
885
- def test_one_container_log_metadata_from_plugin
886
- setup_gce_metadata_stubs
887
- setup_container_metadata_stubs
888
- setup_logging_stubs
889
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
890
- d.emit(container_log_entry_with_metadata(log_entry(0)))
891
- d.run
892
- verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS) do |entry|
893
- assert_equal CONTAINER_SECONDS_EPOCH, \
894
- entry['metadata']['timestamp']['seconds'], entry
895
- assert_equal CONTAINER_NANOS, \
896
- entry['metadata']['timestamp']['nanos'], entry
897
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
898
- end
899
- end
900
-
901
- def test_multiple_container_logs_metadata_from_plugin
902
- setup_gce_metadata_stubs
903
- setup_container_metadata_stubs
904
- setup_logging_stubs
905
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
906
- [2, 3, 5, 11, 50].each do |n|
907
- # The test driver doesn't clear its buffer of entries after running, so
908
- # do it manually here.
909
- d.instance_variable_get('@entries').clear
910
- @logs_sent = []
911
- n.times { |i| d.emit(container_log_entry_with_metadata(log_entry(i))) }
912
- d.run
913
- verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS) do |entry|
914
- assert_equal CONTAINER_SECONDS_EPOCH, \
915
- entry['metadata']['timestamp']['seconds'], entry
916
- assert_equal CONTAINER_NANOS, \
917
- entry['metadata']['timestamp']['nanos'], entry
918
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
919
- end
920
- end
921
- end
922
-
923
- def test_one_container_log_metadata_from_tag
924
- setup_gce_metadata_stubs
925
- setup_container_metadata_stubs
926
- setup_logging_stubs
927
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
928
- d.emit(container_log_entry(log_entry(0)))
929
- d.run
930
- verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS) do |entry|
931
- assert_equal CONTAINER_SECONDS_EPOCH, \
932
- entry['metadata']['timestamp']['seconds'], entry
933
- assert_equal CONTAINER_NANOS, \
934
- entry['metadata']['timestamp']['nanos'], entry
935
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
936
- end
937
- end
938
-
939
- def test_multiple_container_logs_metadata_from_tag
940
- setup_gce_metadata_stubs
941
- setup_container_metadata_stubs
942
- setup_logging_stubs
943
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
944
- [2, 3, 5, 11, 50].each do |n|
945
- # The test driver doesn't clear its buffer of entries after running, so
946
- # do it manually here.
947
- d.instance_variable_get('@entries').clear
948
- @logs_sent = []
949
- n.times { |i| d.emit(container_log_entry(log_entry(i))) }
950
- d.run
951
- verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS) do |entry|
952
- assert_equal CONTAINER_SECONDS_EPOCH, \
953
- entry['metadata']['timestamp']['seconds'], entry
954
- assert_equal CONTAINER_NANOS, \
955
- entry['metadata']['timestamp']['nanos'], entry
956
- assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
957
- end
958
- end
959
- end
960
-
961
- def test_one_container_log_from_tag_stderr
962
- setup_gce_metadata_stubs
963
- setup_container_metadata_stubs
964
- setup_logging_stubs
965
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
966
- d.emit(container_log_entry(log_entry(0), 'stderr'))
967
- d.run
968
- expected_params = CONTAINER_FROM_TAG_PARAMS.merge(
969
- labels: { "#{CONTAINER_SERVICE_NAME}/stream" => 'stderr' }
970
- ) { |_, oldval, newval| oldval.merge(newval) }
971
- verify_log_entries(1, expected_params) do |entry|
972
- assert_equal CONTAINER_SECONDS_EPOCH, \
973
- entry['metadata']['timestamp']['seconds'], entry
974
- assert_equal CONTAINER_NANOS, \
975
- entry['metadata']['timestamp']['nanos'], entry
976
- assert_equal 'ERROR', entry['metadata']['severity'], entry
977
- end
978
- end
979
-
980
- def test_struct_container_log_metadata_from_plugin
981
- setup_gce_metadata_stubs
982
- setup_container_metadata_stubs
983
- setup_logging_stubs
984
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
985
- d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
986
- '"tag2": "test", "data": 5000, ' \
987
- '"severity": "WARNING"}'))
988
- d.run
989
- verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
990
- 'structPayload') do |entry|
991
- assert_equal 3, entry['structPayload'].size, entry
992
- assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
993
- assert_equal 'test', entry['structPayload']['tag2'], entry
994
- assert_equal 5000, entry['structPayload']['data'], entry
995
- assert_equal CONTAINER_SECONDS_EPOCH, \
996
- entry['metadata']['timestamp']['seconds'], entry
997
- assert_equal CONTAINER_NANOS, \
998
- entry['metadata']['timestamp']['nanos'], entry
999
- assert_equal 'WARNING', entry['metadata']['severity'], entry
1000
- end
1001
- end
1002
-
1003
- def test_struct_container_log_metadata_from_tag
1004
- setup_gce_metadata_stubs
1005
- setup_container_metadata_stubs
1006
- setup_logging_stubs
1007
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
1008
- d.emit(container_log_entry('{"msg": "test log entry 0", ' \
1009
- '"tag2": "test", "data": 5000, ' \
1010
- '"severity": "W"}'))
1011
- d.run
1012
- verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
1013
- 'structPayload') do |entry|
1014
- assert_equal 3, entry['structPayload'].size, entry
1015
- assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
1016
- assert_equal 'test', entry['structPayload']['tag2'], entry
1017
- assert_equal 5000, entry['structPayload']['data'], entry
1018
- assert_equal CONTAINER_SECONDS_EPOCH, \
1019
- entry['metadata']['timestamp']['seconds'], entry
1020
- assert_equal CONTAINER_NANOS, \
1021
- entry['metadata']['timestamp']['nanos'], entry
1022
- assert_equal 'WARNING', entry['metadata']['severity'], entry
1023
- end
1024
- end
1025
-
1026
- def test_one_cloudfunctions_log
1027
- setup_gce_metadata_stubs
1028
- setup_cloudfunctions_metadata_stubs
1029
- setup_logging_stubs
1030
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
1031
- d.emit(cloudfunctions_log_entry(0))
1032
- d.run
1033
- verify_log_entries(1, CLOUDFUNCTIONS_PARAMS) do |entry|
1034
- assert_equal 'DEBUG', entry['metadata']['severity'], entry
1035
- end
1036
- end
1037
-
1038
- def test_multiple_cloudfunctions_logs
1039
- setup_gce_metadata_stubs
1040
- setup_cloudfunctions_metadata_stubs
1041
- setup_logging_stubs
1042
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
1043
- [2, 3, 5, 11, 50].each do |n|
1044
- # The test driver doesn't clear its buffer of entries after running, so
1045
- # do it manually here.
1046
- d.instance_variable_get('@entries').clear
1047
- @logs_sent = []
1048
- n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
1049
- d.run
1050
- verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
1051
- assert_equal 'DEBUG', entry['metadata']['severity'], entry
1052
- end
1053
- end
1054
- end
1055
-
1056
- def test_one_cloudfunctions_log_text_not_matched
1057
- setup_gce_metadata_stubs
1058
- setup_cloudfunctions_metadata_stubs
1059
- setup_logging_stubs
1060
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
1061
- d.emit(cloudfunctions_log_entry_text_not_matched(0))
1062
- d.run
1063
- verify_log_entries(1, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
1064
- assert_equal 'INFO', entry['metadata']['severity'], entry
1065
- end
1066
- end
1067
-
1068
- def test_multiple_cloudfunctions_logs_text_not_matched
73
+ # This test looks similar between the grpc and non-grpc paths except that when
74
+ # parsing "105", the grpc path responds with "DEBUG", while the non-grpc path
75
+ # responds with "100".
76
+ #
77
+ # TODO(lingshi) consolidate the tests between the grpc path and the non-grpc
78
+ # path, or at least split into two tests, one with string severities and one
79
+ # with numeric severities.
80
+ def test_severities
1069
81
  setup_gce_metadata_stubs
1070
- setup_cloudfunctions_metadata_stubs
1071
- setup_logging_stubs
1072
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
1073
- [2, 3, 5, 11, 50].each do |n|
1074
- # The test driver doesn't clear its buffer of entries after running, so
1075
- # do it manually here.
1076
- d.instance_variable_get('@entries').clear
1077
- @logs_sent = []
1078
- n.times { |i| d.emit(cloudfunctions_log_entry_text_not_matched(i)) }
1079
- d.run
1080
- verify_log_entries(n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
1081
- assert_equal 'INFO', entry['metadata']['severity'], entry
82
+ expected_severity = []
83
+ emit_index = 0
84
+ setup_logging_stubs do
85
+ d = create_driver
86
+ # Array of pairs of [parsed_severity, expected_severity]
87
+ [%w(INFO INFO), %w(warn WARNING), %w(E ERROR), %w(BLAH DEFAULT),
88
+ ['105', 100], ['', 'DEFAULT']].each do |sev|
89
+ d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
90
+ expected_severity.push(sev[1])
91
+ emit_index += 1
1082
92
  end
1083
- end
1084
- end
1085
-
1086
- def test_one_cloudfunctions_log_tag_not_matched
1087
- setup_gce_metadata_stubs
1088
- setup_cloudfunctions_metadata_stubs
1089
- setup_logging_stubs
1090
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
1091
- d.emit(cloudfunctions_log_entry(0))
1092
- d.run
1093
- verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS, '') do |entry|
1094
- assert_equal '[D][2015-09-25T12:34:56.789Z][123-0] test log entry 0',
1095
- entry['textPayload'], entry
1096
- end
1097
- end
1098
-
1099
- def test_multiple_cloudfunctions_logs_tag_not_matched
1100
- setup_gce_metadata_stubs
1101
- setup_cloudfunctions_metadata_stubs
1102
- setup_logging_stubs
1103
- d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
1104
- [2, 3, 5, 11, 50].each do |n|
1105
- # The test driver doesn't clear its buffer of entries after running, so
1106
- # do it manually here.
1107
- d.instance_variable_get('@entries').clear
1108
- @logs_sent = []
1109
- n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
1110
93
  d.run
1111
- i = 0
1112
- params = CONTAINER_FROM_TAG_PARAMS
1113
- verify_log_entries(n, params, '') do |entry|
1114
- assert_equal "[D][2015-09-25T12:34:56.789Z][123-0] test log entry #{i}",
1115
- entry['textPayload'], entry
1116
- i += 1
1117
- end
1118
- end
1119
- end
1120
-
1121
- def test_http_request_from_record
1122
- setup_gce_metadata_stubs
1123
- setup_logging_stubs
1124
- d = create_driver(APPLICATION_DEFAULT_CONFIG)
1125
- d.emit('httpRequest' => HTTP_REQUEST_MESSAGE)
1126
- d.run
1127
- verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
1128
- assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
1129
- assert_equal nil, entry['structPayload']['httpRequest'], entry
1130
- end
1131
- end
1132
-
1133
- def test_http_request_partial_from_record
1134
- setup_gce_metadata_stubs
1135
- setup_logging_stubs
1136
- d = create_driver(APPLICATION_DEFAULT_CONFIG)
1137
- d.emit('httpRequest' => HTTP_REQUEST_MESSAGE.merge('otherKey' => 'value'))
1138
- d.run
1139
- verify_log_entries(1, COMPUTE_PARAMS, 'httpRequest') do |entry|
1140
- assert_equal HTTP_REQUEST_MESSAGE, entry['httpRequest'], entry
1141
- assert_equal 'value', entry['structPayload']['httpRequest']['otherKey'],
1142
- entry
1143
94
  end
1144
- end
1145
-
1146
- def test_http_request_when_not_hash
1147
- setup_gce_metadata_stubs
1148
- setup_logging_stubs
1149
- d = create_driver(APPLICATION_DEFAULT_CONFIG)
1150
- d.emit('httpRequest' => 'a_string')
1151
- d.run
1152
- verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
1153
- assert_equal 'a_string', entry['structPayload']['httpRequest'], entry
1154
- assert_equal nil, entry['httpRequest'], entry
95
+ verify_index = 0
96
+ verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
97
+ assert_equal expected_severity[verify_index],
98
+ entry['metadata']['severity'], entry
99
+ verify_index += 1
1155
100
  end
1156
101
  end
1157
102
 
1158
- # Make parse_severity public so we can test it.
1159
- class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
1160
- public :parse_severity
1161
- end
1162
-
1163
103
  def test_parse_severity
1164
104
  test_obj = Fluent::GoogleCloudOutput.new
1165
105
 
@@ -1235,57 +175,27 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
1235
175
 
1236
176
  private
1237
177
 
1238
- def uri_for_log(params)
1239
- 'https://logging.googleapis.com/v1beta3/projects/' + params[:project_id] +
1240
- '/logs/' + params[:log_name] + '/entries:write'
1241
- end
1242
-
1243
- def stub_metadata_request(metadata_path, response_body)
1244
- stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
1245
- metadata_path)
1246
- .to_return(body: response_body, status: 200,
1247
- headers: { 'Content-Length' => response_body.length })
1248
- end
1249
-
1250
- def setup_no_metadata_service_stubs
1251
- # Simulate a machine with no metadata service present
1252
- stub_request(:any, %r{http://169.254.169.254/.*})
1253
- .to_raise(Errno::EHOSTUNREACH)
1254
- end
1255
-
1256
- def setup_gce_metadata_stubs
1257
- # Stub the root, used for platform detection by the plugin and 'googleauth'.
1258
- stub_request(:get, 'http://169.254.169.254')
1259
- .to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
1260
-
1261
- # Create stubs for all the GCE metadata lookups the agent needs to make.
1262
- stub_metadata_request('project/project-id', PROJECT_ID)
1263
- stub_metadata_request('instance/zone', FULLY_QUALIFIED_ZONE)
1264
- stub_metadata_request('instance/id', VM_ID)
1265
- stub_metadata_request('instance/attributes/',
1266
- "attribute1\nattribute2\nattribute3")
1267
-
1268
- # Used by 'googleauth' to fetch the default service account credentials.
1269
- stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
1270
- 'instance/service-accounts/default/token')
1271
- .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
1272
- status: 200,
1273
- headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
1274
- 'Content-Type' => 'application/json' })
1275
- end
1276
-
1277
- def setup_ec2_metadata_stubs
1278
- # Stub the root, used for platform detection
1279
- stub_request(:get, 'http://169.254.169.254')
1280
- .to_return(status: 200, headers: { 'Server' => 'EC2ws' })
178
+ # The non-grpc path has a unique field 'validatedWithOriginServer', while
179
+ # the grpc path has a unique field 'cacheValidatedWithOriginServer'.
180
+ HTTP_REQUEST_MESSAGE = {
181
+ 'requestMethod' => 'POST',
182
+ 'requestUrl' => 'http://example/',
183
+ 'requestSize' => 210,
184
+ 'status' => 200,
185
+ 'responseSize' => 65,
186
+ 'userAgent' => 'USER AGENT 1.0',
187
+ 'remoteIp' => '55.55.55.55',
188
+ 'referer' => 'http://referer/',
189
+ 'cacheHit' => false,
190
+ 'validatedWithOriginServer' => true
191
+ }
1281
192
 
1282
- # Stub the identity document lookup made by the agent.
1283
- stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
1284
- 'instance-identity/document')
1285
- .to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
1286
- headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
1287
- end
193
+ # In the non-grpc path 'referer' is nil, while in the grpc path 'referer' is
194
+ # absent.
195
+ HTTP_REQUEST_MESSAGE_WITHOUT_REFERER = HTTP_REQUEST_MESSAGE.merge(
196
+ 'referer' => nil)
1288
197
 
198
+ # Set up http stubs to mock the external calls.
1289
199
  def setup_logging_stubs
1290
200
  [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
1291
201
  CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS, CUSTOM_PARAMS,
@@ -1295,140 +205,64 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
1295
205
  { body: '' }
1296
206
  end
1297
207
  end
208
+ yield
1298
209
  end
1299
210
 
1300
- def setup_auth_stubs
1301
- # Used when loading credentials from a JSON file.
1302
- stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
1303
- .with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
1304
- .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
1305
- status: 200,
1306
- headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
1307
- 'Content-Type' => 'application/json' })
1308
-
1309
- stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
1310
- .with(body: hash_including(grant_type: 'refresh_token'))
1311
- .to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
1312
- status: 200,
1313
- headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
1314
- 'Content-Type' => 'application/json' })
1315
- end
1316
-
1317
- def setup_managed_vm_metadata_stubs
1318
- stub_metadata_request(
1319
- 'instance/attributes/',
1320
- "attribute1\ngae_backend_name\ngae_backend_version\nlast_attribute")
1321
- stub_metadata_request('instance/attributes/gae_backend_name',
1322
- MANAGED_VM_BACKEND_NAME)
1323
- stub_metadata_request('instance/attributes/gae_backend_version',
1324
- MANAGED_VM_BACKEND_VERSION)
1325
- end
1326
-
1327
- def setup_container_metadata_stubs
1328
- stub_metadata_request(
1329
- 'instance/attributes/',
1330
- "attribute1\nkube-env\nlast_attribute")
1331
- stub_metadata_request('instance/attributes/kube-env',
1332
- "ENABLE_NODE_LOGGING: \"true\"\n"\
1333
- 'INSTANCE_PREFIX: '\
1334
- "gke-#{CONTAINER_CLUSTER_NAME}-740fdafa\n"\
1335
- 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
211
+ # Create a Fluentd output test driver with the Google Cloud Output plugin.
212
+ def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
213
+ Fluent::Test::BufferedOutputTestDriver.new(
214
+ Fluent::GoogleCloudOutput, tag).configure(conf, true)
1336
215
  end
1337
216
 
1338
- def setup_cloudfunctions_metadata_stubs
1339
- stub_metadata_request(
1340
- 'instance/attributes/',
1341
- "attribute1\nkube-env\ngcf_region\nlast_attribute")
1342
- stub_metadata_request('instance/attributes/kube-env',
1343
- "ENABLE_NODE_LOGGING: \"true\"\n"\
1344
- 'INSTANCE_PREFIX: '\
1345
- "gke-#{CLOUDFUNCTIONS_CLUSTER_NAME}-740fdafa\n"\
1346
- 'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
1347
- stub_metadata_request('instance/attributes/gcf_region',
1348
- CLOUDFUNCTIONS_REGION)
217
+ # Verify the number and the content of the log entries match the expectation.
218
+ # The caller can optionally provide a block which is called for each entry.
219
+ def verify_log_entries(n, params, payload_type = 'textPayload', &block)
220
+ verify_json_log_entries(n, params, payload_type, &block)
1349
221
  end
1350
222
 
1351
- def container_log_entry_with_metadata(log)
1352
- {
1353
- log: log,
1354
- stream: CONTAINER_STREAM,
1355
- time: CONTAINER_TIMESTAMP,
1356
- kubernetes: {
1357
- namespace_id: CONTAINER_NAMESPACE_ID,
1358
- namespace_name: CONTAINER_NAMESPACE_NAME,
1359
- pod_id: CONTAINER_POD_ID,
1360
- pod_name: CONTAINER_POD_NAME,
1361
- container_name: CONTAINER_CONTAINER_NAME,
1362
- labels: {
1363
- CONTAINER_LABEL_KEY => CONTAINER_LABEL_VALUE
1364
- }
1365
- }
1366
- }
223
+ # For an optional field with default values, Protobuf omits the field when it
224
+ # is deserialized to json. So we need to add an extra check for gRPC which
225
+ # uses Protobuf.
226
+ #
227
+ # An optional block can be passed in if we need to assert something other than
228
+ # a plain equal. e.g. assert_in_delta.
229
+ def assert_equal_with_default(field, expected_value, _default_value, entry)
230
+ if block_given?
231
+ yield
232
+ else
233
+ assert_equal expected_value, field, entry
234
+ end
1367
235
  end
1368
236
 
1369
- def container_log_entry(log, stream = CONTAINER_STREAM)
1370
- {
1371
- log: log,
1372
- stream: stream,
1373
- time: CONTAINER_TIMESTAMP
1374
- }
237
+ # A wrapper around the constant HTTP_REQUEST_MESSAGE, so the definition can be
238
+ # skipped in the shared module and defined here.
239
+ def http_request_message
240
+ HTTP_REQUEST_MESSAGE
1375
241
  end
1376
242
 
1377
- def cloudfunctions_log_entry(i)
1378
- {
1379
- stream: 'stdout',
1380
- log: '[D][2015-09-25T12:34:56.789Z][123-0] ' + log_entry(i)
1381
- }
243
+ # A wrapper around the constant HTTP_REQUEST_MESSAGE_WITHOUT_REFERER, so the
244
+ # definition can be skipped in the shared module and defined here.
245
+ def http_request_message_without_referer
246
+ HTTP_REQUEST_MESSAGE_WITHOUT_REFERER
1382
247
  end
1383
248
 
1384
- def cloudfunctions_log_entry_text_not_matched(i)
1385
- {
1386
- stream: 'stdout',
1387
- log: log_entry(i)
1388
- }
249
+ # Get the fields of the struct payload.
250
+ def get_fields(struct_payload)
251
+ struct_payload
1389
252
  end
1390
253
 
1391
- def log_entry(i)
1392
- 'test log entry ' + i.to_s
254
+ # Get the value of a struct field.
255
+ def get_struct(field)
256
+ field
1393
257
  end
1394
258
 
1395
- def check_labels(entry, common_labels, expected_labels)
1396
- # TODO(salty) test/handle overlap between common_labels and entry labels
1397
- all_labels ||= common_labels
1398
- all_labels.merge!(entry['metadata']['labels'] || {})
1399
- all_labels.each do |key, value|
1400
- assert value.is_a?(String), "Value #{value} for label #{key} " \
1401
- 'is not a string: ' + value.class.name
1402
- assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
1403
- assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
1404
- "#{expected_labels[key]} in #{key} => #{value}"
1405
- end
1406
- assert_equal expected_labels.length, all_labels.length, 'Expected ' \
1407
- "#{expected_labels.length} labels, got #{all_labels.length}"
259
+ # Get the value of a string field.
260
+ def get_string(field)
261
+ field
1408
262
  end
1409
263
 
1410
- # The caller can optionally provide a block which is called for each entry.
1411
- def verify_log_entries(n, params, payload_type = 'textPayload')
1412
- i = 0
1413
- @logs_sent.each do |batch|
1414
- batch['entries'].each do |entry|
1415
- unless payload_type.empty?
1416
- assert entry.key?(payload_type), 'Entry did not contain expected ' \
1417
- "#{payload_type} key: " + entry.to_s
1418
- # Check the payload for textPayload, otherwise it's up to the caller.
1419
- if payload_type == 'textPayload'
1420
- assert_equal "test log entry #{i}", entry['textPayload'], batch
1421
- end
1422
- end
1423
-
1424
- assert_equal params[:zone], entry['metadata']['zone']
1425
- assert_equal params[:service_name], entry['metadata']['serviceName']
1426
- check_labels entry, batch['commonLabels'], params[:labels]
1427
- yield(entry) if block_given?
1428
- i += 1
1429
- assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
1430
- end
1431
- end
1432
- assert i == n, "Number of entries #{i} does not match expected number #{n}"
264
+ # Get the value of a number field.
265
+ def get_number(field)
266
+ field
1433
267
  end
1434
268
  end