fluent-plugin-google-cloud 0.4.2 → 0.4.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +19 -4
- data/README.rdoc +27 -16
- data/Rakefile +10 -0
- data/fluent-plugin-google-cloud.gemspec +18 -10
- data/lib/fluent/plugin/out_google_cloud.rb +102 -103
- data/test/helper.rb +7 -9
- data/test/plugin/test_out_google_cloud.rb +190 -169
- metadata +40 -24
data/test/helper.rb
CHANGED
@@ -18,7 +18,7 @@ begin
|
|
18
18
|
Bundler.setup(:default, :development)
|
19
19
|
rescue Bundler::BundlerError => e
|
20
20
|
$stderr.puts e.message
|
21
|
-
$stderr.puts
|
21
|
+
$stderr.puts 'Run `bundle install` to install missing gems'
|
22
22
|
exit e.status_code
|
23
23
|
end
|
24
24
|
require 'test/unit'
|
@@ -26,17 +26,15 @@ require 'test/unit'
|
|
26
26
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
27
27
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
28
28
|
require 'fluent/test'
|
29
|
-
unless ENV.
|
29
|
+
unless ENV.key?('VERBOSE')
|
30
30
|
nulllogger = Object.new
|
31
|
-
nulllogger.instance_eval
|
32
|
-
def method_missing(
|
31
|
+
nulllogger.instance_eval do |_|
|
32
|
+
def method_missing(_method, *_args)
|
33
33
|
# pass
|
34
34
|
end
|
35
|
-
|
36
|
-
$log
|
35
|
+
end
|
36
|
+
# global $log variable is used by fluentd
|
37
|
+
$log = nulllogger # rubocop:disable Style/GlobalVars
|
37
38
|
end
|
38
39
|
|
39
40
|
require 'fluent/plugin/out_google_cloud'
|
40
|
-
|
41
|
-
class Test::Unit::TestCase
|
42
|
-
end
|
@@ -17,6 +17,10 @@ require 'json'
|
|
17
17
|
require 'mocha/test_unit'
|
18
18
|
require 'webmock/test_unit'
|
19
19
|
|
20
|
+
# This trips for many tests due to the use of conditionals.
|
21
|
+
# rubocop:disable Metrics/AbcSize
|
22
|
+
|
23
|
+
# Unit tests for Google Cloud Logging plugin
|
20
24
|
class GoogleCloudOutputTest < Test::Unit::TestCase
|
21
25
|
def setup
|
22
26
|
Fluent::Test.setup
|
@@ -45,11 +49,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
45
49
|
EC2_ACCOUNT_ID = '123456789012'
|
46
50
|
|
47
51
|
# The formatting here matches the format used on the VM.
|
48
|
-
EC2_IDENTITY_DOCUMENT = %
|
52
|
+
EC2_IDENTITY_DOCUMENT = %({
|
49
53
|
"accountId" : "#{EC2_ACCOUNT_ID}",
|
50
54
|
"availabilityZone" : "#{EC2_ZONE}",
|
51
55
|
"instanceId" : "#{EC2_VM_ID}"
|
52
|
-
}
|
56
|
+
})
|
53
57
|
|
54
58
|
# Managed VMs specific labels
|
55
59
|
MANAGED_VM_BACKEND_NAME = 'default'
|
@@ -60,56 +64,58 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
60
64
|
FAKE_AUTH_TOKEN = 'abc123'
|
61
65
|
|
62
66
|
# Configuration files for various test scenarios
|
63
|
-
APPLICATION_DEFAULT_CONFIG = %
|
64
|
-
|
67
|
+
APPLICATION_DEFAULT_CONFIG = %(
|
68
|
+
)
|
65
69
|
|
66
|
-
|
70
|
+
# rubocop:disable Metrics/LineLength
|
71
|
+
PRIVATE_KEY_CONFIG = %(
|
67
72
|
auth_method private_key
|
68
73
|
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
69
74
|
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
70
|
-
|
75
|
+
)
|
76
|
+
# rubocop:enable Metrics/LineLength
|
71
77
|
|
72
|
-
NO_METADATA_SERVICE_CONFIG = %
|
78
|
+
NO_METADATA_SERVICE_CONFIG = %(
|
73
79
|
use_metadata_service false
|
74
|
-
|
80
|
+
)
|
75
81
|
|
76
|
-
CUSTOM_METADATA_CONFIG = %
|
82
|
+
CUSTOM_METADATA_CONFIG = %(
|
77
83
|
project_id #{CUSTOM_PROJECT_ID}
|
78
84
|
zone #{CUSTOM_ZONE}
|
79
85
|
vm_id #{CUSTOM_VM_ID}
|
80
|
-
|
86
|
+
)
|
81
87
|
|
82
|
-
CONFIG_MISSING_PRIVATE_KEY_PATH = %
|
88
|
+
CONFIG_MISSING_PRIVATE_KEY_PATH = %(
|
83
89
|
auth_method private_key
|
84
90
|
private_key_email nobody@example.com
|
85
|
-
|
86
|
-
CONFIG_MISSING_PRIVATE_KEY_EMAIL = %
|
91
|
+
)
|
92
|
+
CONFIG_MISSING_PRIVATE_KEY_EMAIL = %(
|
87
93
|
auth_method private_key
|
88
94
|
private_key_path /fake/path/to/key
|
89
|
-
|
90
|
-
CONFIG_MISSING_METADATA_PROJECT_ID = %
|
95
|
+
)
|
96
|
+
CONFIG_MISSING_METADATA_PROJECT_ID = %(
|
91
97
|
zone #{CUSTOM_ZONE}
|
92
98
|
vm_id #{CUSTOM_VM_ID}
|
93
|
-
|
94
|
-
CONFIG_MISSING_METADATA_ZONE = %
|
99
|
+
)
|
100
|
+
CONFIG_MISSING_METADATA_ZONE = %(
|
95
101
|
project_id #{CUSTOM_PROJECT_ID}
|
96
102
|
vm_id #{CUSTOM_VM_ID}
|
97
|
-
|
98
|
-
CONFIG_MISSING_METADATA_VM_ID = %
|
103
|
+
)
|
104
|
+
CONFIG_MISSING_METADATA_VM_ID = %(
|
99
105
|
project_id #{CUSTOM_PROJECT_ID}
|
100
106
|
zone #{CUSTOM_ZONE}
|
101
|
-
|
102
|
-
CONFIG_MISSING_METADATA_ALL = %
|
103
|
-
|
107
|
+
)
|
108
|
+
CONFIG_MISSING_METADATA_ALL = %(
|
109
|
+
)
|
104
110
|
|
105
|
-
CONFIG_EC2_PROJECT_ID = %
|
111
|
+
CONFIG_EC2_PROJECT_ID = %(
|
106
112
|
project_id #{EC2_PROJECT_ID}
|
107
|
-
|
113
|
+
)
|
108
114
|
|
109
|
-
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %
|
115
|
+
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
|
110
116
|
project_id #{EC2_PROJECT_ID}
|
111
117
|
vm_id #{CUSTOM_VM_ID}
|
112
|
-
|
118
|
+
)
|
113
119
|
|
114
120
|
# Service configurations for various services
|
115
121
|
COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
|
@@ -163,9 +169,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
163
169
|
}
|
164
170
|
}
|
165
171
|
|
166
|
-
def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
|
172
|
+
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
167
173
|
Fluent::Test::BufferedOutputTestDriver.new(
|
168
|
-
|
174
|
+
Fluent::GoogleCloudOutput).configure(conf, use_v1_config: true)
|
169
175
|
end
|
170
176
|
|
171
177
|
def test_configure_service_account_application_default
|
@@ -188,19 +194,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
188
194
|
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
189
195
|
end
|
190
196
|
|
191
|
-
def
|
197
|
+
def test_configure_invalid_private_key_missing_path
|
192
198
|
exception_count = 0
|
193
199
|
begin
|
194
|
-
|
200
|
+
_d = create_driver(CONFIG_MISSING_PRIVATE_KEY_PATH)
|
195
201
|
rescue Fluent::ConfigError => error
|
196
202
|
assert error.message.include? 'private_key_path'
|
197
203
|
exception_count += 1
|
198
204
|
end
|
199
205
|
assert_equal 1, exception_count
|
206
|
+
end
|
200
207
|
|
208
|
+
def test_configure_invalid_private_key_missing_email
|
201
209
|
exception_count = 0
|
202
210
|
begin
|
203
|
-
|
211
|
+
_d = create_driver(CONFIG_MISSING_PRIVATE_KEY_EMAIL)
|
204
212
|
rescue Fluent::ConfigError => error
|
205
213
|
assert error.message.include? 'private_key_email'
|
206
214
|
exception_count += 1
|
@@ -208,41 +216,50 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
208
216
|
assert_equal 1, exception_count
|
209
217
|
end
|
210
218
|
|
211
|
-
def
|
219
|
+
def test_configure_invalid_metadata_missing_project_id_no_metadata_service
|
212
220
|
setup_no_metadata_service_stubs
|
213
221
|
exception_count = 0
|
214
222
|
begin
|
215
|
-
|
223
|
+
_d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
|
216
224
|
rescue Fluent::ConfigError => error
|
217
225
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
218
226
|
assert error.message.include? 'project_id'
|
219
227
|
exception_count += 1
|
220
228
|
end
|
221
229
|
assert_equal 1, exception_count
|
230
|
+
end
|
222
231
|
|
232
|
+
def test_configure_invalid_metadata_missing_zone_no_metadata_service
|
233
|
+
setup_no_metadata_service_stubs
|
223
234
|
exception_count = 0
|
224
235
|
begin
|
225
|
-
|
236
|
+
_d = create_driver(CONFIG_MISSING_METADATA_ZONE)
|
226
237
|
rescue Fluent::ConfigError => error
|
227
238
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
228
239
|
assert error.message.include? 'zone'
|
229
240
|
exception_count += 1
|
230
241
|
end
|
231
242
|
assert_equal 1, exception_count
|
243
|
+
end
|
232
244
|
|
245
|
+
def test_configure_invalid_metadata_missing_vm_id_no_metadata_service
|
246
|
+
setup_no_metadata_service_stubs
|
233
247
|
exception_count = 0
|
234
248
|
begin
|
235
|
-
|
249
|
+
_d = create_driver(CONFIG_MISSING_METADATA_VM_ID)
|
236
250
|
rescue Fluent::ConfigError => error
|
237
251
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
238
252
|
assert error.message.include? 'vm_id'
|
239
253
|
exception_count += 1
|
240
254
|
end
|
241
255
|
assert_equal 1, exception_count
|
256
|
+
end
|
242
257
|
|
258
|
+
def test_configure_invalid_metadata_missing_all_no_metadata_service
|
259
|
+
setup_no_metadata_service_stubs
|
243
260
|
exception_count = 0
|
244
261
|
begin
|
245
|
-
|
262
|
+
_d = create_driver(CONFIG_MISSING_METADATA_ALL)
|
246
263
|
rescue Fluent::ConfigError => error
|
247
264
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
248
265
|
assert error.message.include? 'project_id'
|
@@ -255,7 +272,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
255
272
|
|
256
273
|
def test_metadata_loading
|
257
274
|
setup_gce_metadata_stubs
|
258
|
-
d = create_driver
|
275
|
+
d = create_driver
|
259
276
|
d.run
|
260
277
|
assert_equal PROJECT_ID, d.instance.project_id
|
261
278
|
assert_equal ZONE, d.instance.zone
|
@@ -266,7 +283,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
266
283
|
def test_managed_vm_metadata_loading
|
267
284
|
setup_gce_metadata_stubs
|
268
285
|
setup_managed_vm_metadata_stubs
|
269
|
-
d = create_driver
|
286
|
+
d = create_driver
|
270
287
|
d.run
|
271
288
|
assert_equal PROJECT_ID, d.instance.project_id
|
272
289
|
assert_equal ZONE, d.instance.zone
|
@@ -334,7 +351,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
334
351
|
setup_ec2_metadata_stubs
|
335
352
|
exception_count = 0
|
336
353
|
begin
|
337
|
-
|
354
|
+
_d = create_driver
|
338
355
|
rescue Fluent::ConfigError => error
|
339
356
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
340
357
|
assert error.message.include? 'project_id'
|
@@ -346,8 +363,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
346
363
|
def test_one_log
|
347
364
|
setup_gce_metadata_stubs
|
348
365
|
setup_logging_stubs
|
349
|
-
d = create_driver
|
350
|
-
d.emit(
|
366
|
+
d = create_driver
|
367
|
+
d.emit('message' => log_entry(0))
|
351
368
|
d.run
|
352
369
|
verify_log_entries(1, COMPUTE_PARAMS)
|
353
370
|
end
|
@@ -356,8 +373,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
356
373
|
setup_gce_metadata_stubs
|
357
374
|
setup_logging_stubs
|
358
375
|
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
359
|
-
d = create_driver
|
360
|
-
d.emit(
|
376
|
+
d = create_driver
|
377
|
+
d.emit('message' => log_entry(0))
|
361
378
|
d.run
|
362
379
|
verify_log_entries(1, COMPUTE_PARAMS)
|
363
380
|
end
|
@@ -365,9 +382,10 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
365
382
|
def test_one_log_with_invalid_json_credentials
|
366
383
|
setup_gce_metadata_stubs
|
367
384
|
setup_logging_stubs
|
368
|
-
ENV['GOOGLE_APPLICATION_CREDENTIALS'] =
|
369
|
-
|
370
|
-
d
|
385
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = \
|
386
|
+
'test/plugin/data/invalid_credentials.json'
|
387
|
+
d = create_driver
|
388
|
+
d.emit('message' => log_entry(0))
|
371
389
|
exception_count = 0
|
372
390
|
begin
|
373
391
|
d.run
|
@@ -382,7 +400,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
382
400
|
setup_gce_metadata_stubs
|
383
401
|
setup_logging_stubs
|
384
402
|
d = create_driver(PRIVATE_KEY_CONFIG)
|
385
|
-
d.emit(
|
403
|
+
d.emit('message' => log_entry(0))
|
386
404
|
d.run
|
387
405
|
verify_log_entries(1, COMPUTE_PARAMS)
|
388
406
|
end
|
@@ -394,16 +412,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
394
412
|
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
395
413
|
setup_logging_stubs
|
396
414
|
d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
|
397
|
-
d.emit(
|
415
|
+
d.emit('message' => log_entry(0))
|
398
416
|
d.run
|
399
417
|
verify_log_entries(1, CUSTOM_PARAMS)
|
400
418
|
end
|
401
419
|
|
402
420
|
def test_one_log_ec2
|
421
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
403
422
|
setup_ec2_metadata_stubs
|
404
423
|
setup_logging_stubs
|
405
424
|
d = create_driver(CONFIG_EC2_PROJECT_ID)
|
406
|
-
d.emit(
|
425
|
+
d.emit('message' => log_entry(0))
|
407
426
|
d.run
|
408
427
|
verify_log_entries(1, EC2_PARAMS)
|
409
428
|
end
|
@@ -411,12 +430,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
411
430
|
def test_struct_payload_log
|
412
431
|
setup_gce_metadata_stubs
|
413
432
|
setup_logging_stubs
|
414
|
-
d = create_driver
|
415
|
-
d.emit(
|
433
|
+
d = create_driver
|
434
|
+
d.emit('msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000)
|
416
435
|
d.run
|
417
436
|
verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
|
418
437
|
assert_equal 3, entry['structPayload'].size, entry
|
419
|
-
assert_equal
|
438
|
+
assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
|
420
439
|
assert_equal 'test', entry['structPayload']['tag2'], entry
|
421
440
|
assert_equal 5000, entry['structPayload']['data'], entry
|
422
441
|
end
|
@@ -425,26 +444,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
425
444
|
def test_timestamps
|
426
445
|
setup_gce_metadata_stubs
|
427
446
|
setup_logging_stubs
|
428
|
-
d = create_driver
|
447
|
+
d = create_driver
|
429
448
|
expected_ts = []
|
430
449
|
emit_index = 0
|
431
|
-
[Time.at(
|
450
|
+
[Time.at(123_456.789), Time.at(0), Time.now].each do |ts|
|
432
451
|
# Test the "native" fluentd timestamp as well as our nanosecond tags.
|
433
|
-
d.emit({'message' => log_entry(emit_index)}, ts.to_f)
|
452
|
+
d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
|
434
453
|
# The native timestamp currently only supports second granularity
|
435
454
|
# (fluentd issue #461), so strip nanoseconds from the expected value.
|
436
455
|
expected_ts.push(Time.at(ts.tv_sec))
|
437
456
|
emit_index += 1
|
438
|
-
d.emit(
|
439
|
-
|
457
|
+
d.emit('message' => log_entry(emit_index),
|
458
|
+
'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
|
440
459
|
expected_ts.push(ts)
|
441
460
|
emit_index += 1
|
442
|
-
d.emit(
|
443
|
-
|
461
|
+
d.emit('message' => log_entry(emit_index),
|
462
|
+
'timestamp' => { 'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec })
|
444
463
|
expected_ts.push(ts)
|
445
464
|
emit_index += 1
|
446
|
-
d.emit(
|
447
|
-
|
465
|
+
d.emit('message' => log_entry(emit_index),
|
466
|
+
'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec)
|
448
467
|
expected_ts.push(ts)
|
449
468
|
emit_index += 1
|
450
469
|
end
|
@@ -452,9 +471,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
452
471
|
verify_index = 0
|
453
472
|
verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
|
454
473
|
assert_equal expected_ts[verify_index].tv_sec,
|
455
|
-
|
474
|
+
entry['metadata']['timestamp']['seconds'], entry
|
456
475
|
assert_equal expected_ts[verify_index].tv_nsec,
|
457
|
-
|
476
|
+
entry['metadata']['timestamp']['nanos'], entry
|
458
477
|
verify_index += 1
|
459
478
|
end
|
460
479
|
end
|
@@ -462,27 +481,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
462
481
|
def test_malformed_timestamp
|
463
482
|
setup_gce_metadata_stubs
|
464
483
|
setup_logging_stubs
|
465
|
-
d = create_driver
|
484
|
+
d = create_driver
|
466
485
|
# if timestamp is not a hash it is passed through to the struct payload.
|
467
|
-
d.emit(
|
486
|
+
d.emit('message' => log_entry(0), 'timestamp' => 'not-a-hash')
|
468
487
|
d.run
|
469
|
-
verify_index = 0
|
470
488
|
verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
|
471
489
|
assert_equal 2, entry['structPayload'].size, entry
|
472
|
-
assert_equal
|
490
|
+
assert_equal 'not-a-hash', entry['structPayload']['timestamp'], entry
|
473
491
|
end
|
474
492
|
end
|
475
493
|
|
476
494
|
def test_severities
|
477
495
|
setup_gce_metadata_stubs
|
478
496
|
setup_logging_stubs
|
479
|
-
d = create_driver
|
497
|
+
d = create_driver
|
480
498
|
expected_severity = []
|
481
499
|
emit_index = 0
|
482
500
|
# Array of pairs of [parsed_severity, expected_severity]
|
483
|
-
[
|
484
|
-
['
|
485
|
-
d.emit(
|
501
|
+
[%w(INFO INFO), %w(warn WARNING), %w(E ERROR), %w(BLAH DEFAULT),
|
502
|
+
['105', 100], ['', 'DEFAULT']].each do |sev|
|
503
|
+
d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
|
486
504
|
expected_severity.push(sev[1])
|
487
505
|
emit_index += 1
|
488
506
|
end
|
@@ -490,7 +508,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
490
508
|
verify_index = 0
|
491
509
|
verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
|
492
510
|
assert_equal expected_severity[verify_index],
|
493
|
-
|
511
|
+
entry['metadata']['severity'], entry
|
494
512
|
verify_index += 1
|
495
513
|
end
|
496
514
|
end
|
@@ -498,9 +516,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
498
516
|
def test_label_map_without_field_present
|
499
517
|
setup_gce_metadata_stubs
|
500
518
|
setup_logging_stubs
|
501
|
-
config = %
|
519
|
+
config = %(label_map { "label_field": "sent_label" })
|
502
520
|
d = create_driver(config)
|
503
|
-
d.emit(
|
521
|
+
d.emit('message' => log_entry(0))
|
504
522
|
d.run
|
505
523
|
# No additional labels should be present
|
506
524
|
verify_log_entries(1, COMPUTE_PARAMS)
|
@@ -509,9 +527,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
509
527
|
def test_label_map_with_field_present
|
510
528
|
setup_gce_metadata_stubs
|
511
529
|
setup_logging_stubs
|
512
|
-
config = %
|
530
|
+
config = %(label_map { "label_field": "sent_label" })
|
513
531
|
d = create_driver(config)
|
514
|
-
d.emit(
|
532
|
+
d.emit('message' => log_entry(0), 'label_field' => 'label_value')
|
515
533
|
d.run
|
516
534
|
# make a deep copy of COMPUTE_PARAMS and add the parsed label.
|
517
535
|
params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
|
@@ -522,19 +540,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
522
540
|
def test_label_map_with_multiple_fields
|
523
541
|
setup_gce_metadata_stubs
|
524
542
|
setup_logging_stubs
|
525
|
-
config = %
|
543
|
+
config = %(
|
526
544
|
label_map {
|
527
545
|
"label1": "sent_label_1",
|
528
546
|
"label_number_two": "foo.googleapis.com/bar",
|
529
547
|
"label3": "label3"
|
530
|
-
}
|
548
|
+
}
|
549
|
+
)
|
531
550
|
d = create_driver(config)
|
532
551
|
# not_a_label passes through to the struct payload
|
533
|
-
d.emit(
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
552
|
+
d.emit('message' => log_entry(0),
|
553
|
+
'label1' => 'value1',
|
554
|
+
'label_number_two' => 'value2',
|
555
|
+
'not_a_label' => 'value4',
|
556
|
+
'label3' => 'value3')
|
538
557
|
d.run
|
539
558
|
# make a deep copy of COMPUTE_PARAMS and add the parsed labels.
|
540
559
|
params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
|
@@ -543,7 +562,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
543
562
|
params['labels']['label3'] = 'value3'
|
544
563
|
verify_log_entries(1, params, 'structPayload') do |entry|
|
545
564
|
assert_equal 2, entry['structPayload'].size, entry
|
546
|
-
assert_equal
|
565
|
+
assert_equal 'test log entry 0', entry['structPayload']['message'], entry
|
547
566
|
assert_equal 'value4', entry['structPayload']['not_a_label'], entry
|
548
567
|
end
|
549
568
|
end
|
@@ -551,14 +570,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
551
570
|
def test_multiple_logs
|
552
571
|
setup_gce_metadata_stubs
|
553
572
|
setup_logging_stubs
|
554
|
-
d = create_driver
|
573
|
+
d = create_driver
|
555
574
|
# Only test a few values because otherwise the test can take minutes.
|
556
575
|
[2, 3, 5, 11, 50].each do |n|
|
557
576
|
# The test driver doesn't clear its buffer of entries after running, so
|
558
577
|
# do it manually here.
|
559
578
|
d.instance_variable_get('@entries').clear
|
560
579
|
@logs_sent = []
|
561
|
-
n.times { |i| d.emit(
|
580
|
+
n.times { |i| d.emit('message' => log_entry(i)) }
|
562
581
|
d.run
|
563
582
|
verify_log_entries(n, COMPUTE_PARAMS)
|
564
583
|
end
|
@@ -567,9 +586,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
567
586
|
def test_malformed_log
|
568
587
|
setup_gce_metadata_stubs
|
569
588
|
setup_logging_stubs
|
570
|
-
d = create_driver
|
589
|
+
d = create_driver
|
571
590
|
# if the entry is not a hash, the plugin should silently drop it.
|
572
|
-
d.emit(
|
591
|
+
d.emit('a string is not a valid message')
|
573
592
|
d.run
|
574
593
|
assert @logs_sent.empty?
|
575
594
|
end
|
@@ -578,21 +597,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
578
597
|
setup_gce_metadata_stubs
|
579
598
|
# The API Client should not retry this and the plugin should consume
|
580
599
|
# the exception.
|
581
|
-
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
582
|
-
|
583
|
-
d = create_driver
|
584
|
-
d.emit(
|
600
|
+
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
601
|
+
.to_return(status: 400, body: 'Bad Request')
|
602
|
+
d = create_driver
|
603
|
+
d.emit('message' => log_entry(0))
|
585
604
|
d.run
|
586
|
-
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :
|
605
|
+
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 1)
|
587
606
|
end
|
588
607
|
|
589
608
|
# helper for the ClientError retriable special cases below.
|
590
609
|
def client_error_helper(message)
|
591
610
|
setup_gce_metadata_stubs
|
592
|
-
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
593
|
-
|
594
|
-
d = create_driver
|
595
|
-
d.emit(
|
611
|
+
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
612
|
+
.to_return(status: 401, body: message)
|
613
|
+
d = create_driver
|
614
|
+
d.emit('message' => log_entry(0))
|
596
615
|
exception_count = 0
|
597
616
|
begin
|
598
617
|
d.run
|
@@ -600,38 +619,39 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
600
619
|
assert_equal message, error.message
|
601
620
|
exception_count += 1
|
602
621
|
end
|
603
|
-
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :
|
622
|
+
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 2)
|
604
623
|
assert_equal 1, exception_count
|
605
624
|
end
|
606
625
|
|
607
626
|
def test_client_error_invalid_credentials
|
608
|
-
client_error_helper(
|
627
|
+
client_error_helper('Invalid Credentials')
|
609
628
|
end
|
610
629
|
|
611
630
|
def test_client_error_caller_does_not_have_permission
|
612
|
-
client_error_helper(
|
631
|
+
client_error_helper('The caller does not have permission')
|
613
632
|
end
|
614
633
|
|
615
634
|
def test_client_error_request_had_invalid_credentials
|
616
|
-
client_error_helper(
|
635
|
+
client_error_helper('Request had invalid credentials.')
|
617
636
|
end
|
618
637
|
|
619
638
|
def test_client_error_project_has_not_enabled_the_api
|
620
|
-
client_error_helper(
|
639
|
+
client_error_helper('Project has not enabled the API. Please use ' \
|
640
|
+
'Google Developers Console to activate the API for your project.')
|
621
641
|
end
|
622
642
|
|
623
643
|
def test_client_error_unable_to_fetch_accesss_token
|
624
|
-
client_error_helper(
|
644
|
+
client_error_helper('Unable to fetch access token (no scopes configured?)')
|
625
645
|
end
|
626
646
|
|
627
647
|
def test_server_error
|
628
648
|
setup_gce_metadata_stubs
|
629
649
|
# The API client should retry this once, then throw an exception which
|
630
650
|
# gets propagated through the plugin.
|
631
|
-
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
632
|
-
|
633
|
-
d = create_driver
|
634
|
-
d.emit(
|
651
|
+
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
652
|
+
.to_return(status: 500, body: 'Server Error')
|
653
|
+
d = create_driver
|
654
|
+
d.emit('message' => log_entry(0))
|
635
655
|
exception_count = 0
|
636
656
|
begin
|
637
657
|
d.run
|
@@ -639,7 +659,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
639
659
|
assert_equal 'Server Error', error.message
|
640
660
|
exception_count += 1
|
641
661
|
end
|
642
|
-
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :
|
662
|
+
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 2)
|
643
663
|
assert_equal 1, exception_count
|
644
664
|
end
|
645
665
|
|
@@ -647,8 +667,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
647
667
|
setup_gce_metadata_stubs
|
648
668
|
setup_managed_vm_metadata_stubs
|
649
669
|
setup_logging_stubs
|
650
|
-
d = create_driver
|
651
|
-
d.emit(
|
670
|
+
d = create_driver
|
671
|
+
d.emit('message' => log_entry(0))
|
652
672
|
d.run
|
653
673
|
verify_log_entries(1, VMENGINE_PARAMS)
|
654
674
|
end
|
@@ -657,20 +677,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
657
677
|
setup_gce_metadata_stubs
|
658
678
|
setup_managed_vm_metadata_stubs
|
659
679
|
setup_logging_stubs
|
660
|
-
d = create_driver
|
680
|
+
d = create_driver
|
661
681
|
[2, 3, 5, 11, 50].each do |n|
|
662
682
|
# The test driver doesn't clear its buffer of entries after running, so
|
663
683
|
# do it manually here.
|
664
684
|
d.instance_variable_get('@entries').clear
|
665
685
|
@logs_sent = []
|
666
|
-
n.times { |i| d.emit(
|
686
|
+
n.times { |i| d.emit('message' => log_entry(i)) }
|
667
687
|
d.run
|
668
688
|
verify_log_entries(n, VMENGINE_PARAMS)
|
669
689
|
end
|
670
690
|
end
|
671
691
|
|
672
692
|
# Make parse_severity public so we can test it.
|
673
|
-
class Fluent::GoogleCloudOutput
|
693
|
+
class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
|
674
694
|
public :parse_severity
|
675
695
|
end
|
676
696
|
|
@@ -678,8 +698,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
678
698
|
test_obj = Fluent::GoogleCloudOutput.new
|
679
699
|
|
680
700
|
# known severities should translate to themselves, regardless of case
|
681
|
-
|
682
|
-
|
701
|
+
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY).each \
|
702
|
+
do |severity|
|
683
703
|
assert_equal(severity, test_obj.parse_severity(severity))
|
684
704
|
assert_equal(severity, test_obj.parse_severity(severity.downcase))
|
685
705
|
assert_equal(severity, test_obj.parse_severity(severity.capitalize))
|
@@ -751,25 +771,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
751
771
|
|
752
772
|
def uri_for_log(config)
|
753
773
|
'https://logging.googleapis.com/v1beta3/projects/' + config['project_id'] +
|
754
|
-
|
774
|
+
'/logs/' + config['log_name'] + '/entries:write'
|
755
775
|
end
|
756
776
|
|
757
777
|
def stub_metadata_request(metadata_path, response_body)
|
758
|
-
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
|
759
|
-
|
760
|
-
|
778
|
+
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
|
779
|
+
metadata_path)
|
780
|
+
.to_return(body: response_body, status: 200,
|
781
|
+
headers: { 'Content-Length' => response_body.length })
|
761
782
|
end
|
762
783
|
|
763
784
|
def setup_no_metadata_service_stubs
|
764
785
|
# Simulate a machine with no metadata service present
|
765
|
-
stub_request(:any,
|
766
|
-
to_raise
|
786
|
+
stub_request(:any, %r{http://169.254.169.254/.*})
|
787
|
+
.to_raise(Errno::EHOSTUNREACH)
|
767
788
|
end
|
768
789
|
|
769
790
|
def setup_gce_metadata_stubs
|
770
791
|
# Stub the root, used for platform detection by the plugin and 'googleauth'.
|
771
|
-
stub_request(:get, 'http://169.254.169.254')
|
772
|
-
to_return(:
|
792
|
+
stub_request(:get, 'http://169.254.169.254')
|
793
|
+
.to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
|
773
794
|
|
774
795
|
# Create stubs for all the GCE metadata lookups the agent needs to make.
|
775
796
|
stub_metadata_request('project/project-id', PROJECT_ID)
|
@@ -779,57 +800,60 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
779
800
|
"attribute1\nattribute2\nattribute3")
|
780
801
|
|
781
802
|
# Used by 'googleauth' to fetch the default service account credentials.
|
782
|
-
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/
|
783
|
-
|
784
|
-
|
785
|
-
|
786
|
-
|
803
|
+
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
|
804
|
+
'instance/service-accounts/default/token')
|
805
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
806
|
+
status: 200,
|
807
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
808
|
+
'Content-Type' => 'application/json' })
|
787
809
|
end
|
788
810
|
|
789
811
|
def setup_ec2_metadata_stubs
|
790
812
|
# Stub the root, used for platform detection
|
791
|
-
stub_request(:get, 'http://169.254.169.254')
|
792
|
-
to_return(:
|
813
|
+
stub_request(:get, 'http://169.254.169.254')
|
814
|
+
.to_return(status: 200, headers: { 'Server' => 'EC2ws' })
|
793
815
|
|
794
816
|
# Stub the identity document lookup made by the agent.
|
795
|
-
stub_request(:get, 'http://169.254.169.254/latest/dynamic/
|
796
|
-
|
797
|
-
|
817
|
+
stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
|
818
|
+
'instance-identity/document')
|
819
|
+
.to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
|
820
|
+
headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
|
798
821
|
end
|
799
822
|
|
800
823
|
def setup_logging_stubs
|
801
|
-
[COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS]
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
824
|
+
[COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS]
|
825
|
+
.each do |params|
|
826
|
+
stub_request(:post, uri_for_log(params))
|
827
|
+
.to_return do |request|
|
828
|
+
@logs_sent << JSON.parse(request.body)
|
829
|
+
{ body: '' }
|
830
|
+
end
|
806
831
|
end
|
807
|
-
end
|
808
832
|
end
|
809
833
|
|
810
834
|
def setup_auth_stubs
|
811
835
|
# Used when loading credentials from a JSON file.
|
812
|
-
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
813
|
-
with(:
|
814
|
-
to_return(:
|
815
|
-
|
816
|
-
|
817
|
-
|
818
|
-
|
819
|
-
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
820
|
-
with(:
|
821
|
-
to_return(:
|
822
|
-
|
823
|
-
|
824
|
-
|
836
|
+
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
837
|
+
.with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
|
838
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
839
|
+
status: 200,
|
840
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
841
|
+
'Content-Type' => 'application/json' })
|
842
|
+
|
843
|
+
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
844
|
+
.with(body: hash_including(grant_type: 'refresh_token'))
|
845
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
846
|
+
status: 200,
|
847
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
848
|
+
'Content-Type' => 'application/json' })
|
825
849
|
|
826
850
|
# Used for 'private_key' auth.
|
827
|
-
stub_request(:post, 'https://accounts.google.com/o/oauth2/token')
|
828
|
-
with(:
|
829
|
-
to_return(:
|
830
|
-
|
831
|
-
|
832
|
-
|
851
|
+
stub_request(:post, 'https://accounts.google.com/o/oauth2/token')
|
852
|
+
.with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
|
853
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
854
|
+
status: 200,
|
855
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
856
|
+
'Content-Type' => 'application/json' })
|
833
857
|
end
|
834
858
|
|
835
859
|
def setup_managed_vm_metadata_stubs
|
@@ -850,21 +874,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
850
874
|
all_labels ||= common_labels
|
851
875
|
all_labels.merge!(entry['metadata']['labels'] || {})
|
852
876
|
all_labels.each do |key, value|
|
853
|
-
assert expected_labels.
|
854
|
-
assert_equal value, expected_labels[key],
|
855
|
-
|
877
|
+
assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
|
878
|
+
assert_equal value, expected_labels[key], 'Value mismatch - expected ' \
|
879
|
+
"#{expected_labels[key]} in #{key} => #{value}"
|
856
880
|
end
|
857
|
-
assert_equal expected_labels.length, all_labels.length,
|
858
|
-
|
859
|
-
"#{all_labels.length}")
|
881
|
+
assert_equal expected_labels.length, all_labels.length, 'Expected ' \
|
882
|
+
"#{expected_labels.length} labels, got #{all_labels.length}"
|
860
883
|
end
|
861
884
|
|
862
885
|
# The caller can optionally provide a block which is called for each entry.
|
863
|
-
def verify_log_entries(n, params, payload_type='textPayload')
|
886
|
+
def verify_log_entries(n, params, payload_type = 'textPayload')
|
864
887
|
i = 0
|
865
888
|
@logs_sent.each do |batch|
|
866
889
|
batch['entries'].each do |entry|
|
867
|
-
assert entry.
|
890
|
+
assert entry.key?(payload_type)
|
868
891
|
if (payload_type == 'textPayload')
|
869
892
|
# Check the payload for textPayload, otherwise it is up to the caller.
|
870
893
|
assert_equal "test log entry #{i}", entry['textPayload'], batch
|
@@ -873,9 +896,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
873
896
|
assert_equal params['zone'], entry['metadata']['zone']
|
874
897
|
assert_equal params['service_name'], entry['metadata']['serviceName']
|
875
898
|
check_labels entry, batch['commonLabels'], params['labels']
|
876
|
-
if
|
877
|
-
yield(entry)
|
878
|
-
end
|
899
|
+
yield(entry) if block_given?
|
879
900
|
i += 1
|
880
901
|
assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
|
881
902
|
end
|