fluent-plugin-google-cloud 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +19 -4
- data/README.rdoc +27 -16
- data/Rakefile +10 -0
- data/fluent-plugin-google-cloud.gemspec +18 -10
- data/lib/fluent/plugin/out_google_cloud.rb +102 -103
- data/test/helper.rb +7 -9
- data/test/plugin/test_out_google_cloud.rb +190 -169
- metadata +40 -24
data/test/helper.rb
CHANGED
@@ -18,7 +18,7 @@ begin
|
|
18
18
|
Bundler.setup(:default, :development)
|
19
19
|
rescue Bundler::BundlerError => e
|
20
20
|
$stderr.puts e.message
|
21
|
-
$stderr.puts
|
21
|
+
$stderr.puts 'Run `bundle install` to install missing gems'
|
22
22
|
exit e.status_code
|
23
23
|
end
|
24
24
|
require 'test/unit'
|
@@ -26,17 +26,15 @@ require 'test/unit'
|
|
26
26
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
27
27
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
28
28
|
require 'fluent/test'
|
29
|
-
unless ENV.
|
29
|
+
unless ENV.key?('VERBOSE')
|
30
30
|
nulllogger = Object.new
|
31
|
-
nulllogger.instance_eval
|
32
|
-
def method_missing(
|
31
|
+
nulllogger.instance_eval do |_|
|
32
|
+
def method_missing(_method, *_args)
|
33
33
|
# pass
|
34
34
|
end
|
35
|
-
|
36
|
-
$log
|
35
|
+
end
|
36
|
+
# global $log variable is used by fluentd
|
37
|
+
$log = nulllogger # rubocop:disable Style/GlobalVars
|
37
38
|
end
|
38
39
|
|
39
40
|
require 'fluent/plugin/out_google_cloud'
|
40
|
-
|
41
|
-
class Test::Unit::TestCase
|
42
|
-
end
|
@@ -17,6 +17,10 @@ require 'json'
|
|
17
17
|
require 'mocha/test_unit'
|
18
18
|
require 'webmock/test_unit'
|
19
19
|
|
20
|
+
# This trips for many tests due to the use of conditionals.
|
21
|
+
# rubocop:disable Metrics/AbcSize
|
22
|
+
|
23
|
+
# Unit tests for Google Cloud Logging plugin
|
20
24
|
class GoogleCloudOutputTest < Test::Unit::TestCase
|
21
25
|
def setup
|
22
26
|
Fluent::Test.setup
|
@@ -45,11 +49,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
45
49
|
EC2_ACCOUNT_ID = '123456789012'
|
46
50
|
|
47
51
|
# The formatting here matches the format used on the VM.
|
48
|
-
EC2_IDENTITY_DOCUMENT = %
|
52
|
+
EC2_IDENTITY_DOCUMENT = %({
|
49
53
|
"accountId" : "#{EC2_ACCOUNT_ID}",
|
50
54
|
"availabilityZone" : "#{EC2_ZONE}",
|
51
55
|
"instanceId" : "#{EC2_VM_ID}"
|
52
|
-
}
|
56
|
+
})
|
53
57
|
|
54
58
|
# Managed VMs specific labels
|
55
59
|
MANAGED_VM_BACKEND_NAME = 'default'
|
@@ -60,56 +64,58 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
60
64
|
FAKE_AUTH_TOKEN = 'abc123'
|
61
65
|
|
62
66
|
# Configuration files for various test scenarios
|
63
|
-
APPLICATION_DEFAULT_CONFIG = %
|
64
|
-
|
67
|
+
APPLICATION_DEFAULT_CONFIG = %(
|
68
|
+
)
|
65
69
|
|
66
|
-
|
70
|
+
# rubocop:disable Metrics/LineLength
|
71
|
+
PRIVATE_KEY_CONFIG = %(
|
67
72
|
auth_method private_key
|
68
73
|
private_key_email 271661262351-ft99kc9kjro9rrihq3k2n3s2inbplu0q@developer.gserviceaccount.com
|
69
74
|
private_key_path test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
|
70
|
-
|
75
|
+
)
|
76
|
+
# rubocop:enable Metrics/LineLength
|
71
77
|
|
72
|
-
NO_METADATA_SERVICE_CONFIG = %
|
78
|
+
NO_METADATA_SERVICE_CONFIG = %(
|
73
79
|
use_metadata_service false
|
74
|
-
|
80
|
+
)
|
75
81
|
|
76
|
-
CUSTOM_METADATA_CONFIG = %
|
82
|
+
CUSTOM_METADATA_CONFIG = %(
|
77
83
|
project_id #{CUSTOM_PROJECT_ID}
|
78
84
|
zone #{CUSTOM_ZONE}
|
79
85
|
vm_id #{CUSTOM_VM_ID}
|
80
|
-
|
86
|
+
)
|
81
87
|
|
82
|
-
CONFIG_MISSING_PRIVATE_KEY_PATH = %
|
88
|
+
CONFIG_MISSING_PRIVATE_KEY_PATH = %(
|
83
89
|
auth_method private_key
|
84
90
|
private_key_email nobody@example.com
|
85
|
-
|
86
|
-
CONFIG_MISSING_PRIVATE_KEY_EMAIL = %
|
91
|
+
)
|
92
|
+
CONFIG_MISSING_PRIVATE_KEY_EMAIL = %(
|
87
93
|
auth_method private_key
|
88
94
|
private_key_path /fake/path/to/key
|
89
|
-
|
90
|
-
CONFIG_MISSING_METADATA_PROJECT_ID = %
|
95
|
+
)
|
96
|
+
CONFIG_MISSING_METADATA_PROJECT_ID = %(
|
91
97
|
zone #{CUSTOM_ZONE}
|
92
98
|
vm_id #{CUSTOM_VM_ID}
|
93
|
-
|
94
|
-
CONFIG_MISSING_METADATA_ZONE = %
|
99
|
+
)
|
100
|
+
CONFIG_MISSING_METADATA_ZONE = %(
|
95
101
|
project_id #{CUSTOM_PROJECT_ID}
|
96
102
|
vm_id #{CUSTOM_VM_ID}
|
97
|
-
|
98
|
-
CONFIG_MISSING_METADATA_VM_ID = %
|
103
|
+
)
|
104
|
+
CONFIG_MISSING_METADATA_VM_ID = %(
|
99
105
|
project_id #{CUSTOM_PROJECT_ID}
|
100
106
|
zone #{CUSTOM_ZONE}
|
101
|
-
|
102
|
-
CONFIG_MISSING_METADATA_ALL = %
|
103
|
-
|
107
|
+
)
|
108
|
+
CONFIG_MISSING_METADATA_ALL = %(
|
109
|
+
)
|
104
110
|
|
105
|
-
CONFIG_EC2_PROJECT_ID = %
|
111
|
+
CONFIG_EC2_PROJECT_ID = %(
|
106
112
|
project_id #{EC2_PROJECT_ID}
|
107
|
-
|
113
|
+
)
|
108
114
|
|
109
|
-
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %
|
115
|
+
CONFIG_EC2_PROJECT_ID_AND_CUSTOM_VM_ID = %(
|
110
116
|
project_id #{EC2_PROJECT_ID}
|
111
117
|
vm_id #{CUSTOM_VM_ID}
|
112
|
-
|
118
|
+
)
|
113
119
|
|
114
120
|
# Service configurations for various services
|
115
121
|
COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
|
@@ -163,9 +169,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
163
169
|
}
|
164
170
|
}
|
165
171
|
|
166
|
-
def create_driver(conf=APPLICATION_DEFAULT_CONFIG)
|
172
|
+
def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
|
167
173
|
Fluent::Test::BufferedOutputTestDriver.new(
|
168
|
-
|
174
|
+
Fluent::GoogleCloudOutput).configure(conf, use_v1_config: true)
|
169
175
|
end
|
170
176
|
|
171
177
|
def test_configure_service_account_application_default
|
@@ -188,19 +194,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
188
194
|
assert_equal CUSTOM_VM_ID, d.instance.vm_id
|
189
195
|
end
|
190
196
|
|
191
|
-
def
|
197
|
+
def test_configure_invalid_private_key_missing_path
|
192
198
|
exception_count = 0
|
193
199
|
begin
|
194
|
-
|
200
|
+
_d = create_driver(CONFIG_MISSING_PRIVATE_KEY_PATH)
|
195
201
|
rescue Fluent::ConfigError => error
|
196
202
|
assert error.message.include? 'private_key_path'
|
197
203
|
exception_count += 1
|
198
204
|
end
|
199
205
|
assert_equal 1, exception_count
|
206
|
+
end
|
200
207
|
|
208
|
+
def test_configure_invalid_private_key_missing_email
|
201
209
|
exception_count = 0
|
202
210
|
begin
|
203
|
-
|
211
|
+
_d = create_driver(CONFIG_MISSING_PRIVATE_KEY_EMAIL)
|
204
212
|
rescue Fluent::ConfigError => error
|
205
213
|
assert error.message.include? 'private_key_email'
|
206
214
|
exception_count += 1
|
@@ -208,41 +216,50 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
208
216
|
assert_equal 1, exception_count
|
209
217
|
end
|
210
218
|
|
211
|
-
def
|
219
|
+
def test_configure_invalid_metadata_missing_project_id_no_metadata_service
|
212
220
|
setup_no_metadata_service_stubs
|
213
221
|
exception_count = 0
|
214
222
|
begin
|
215
|
-
|
223
|
+
_d = create_driver(CONFIG_MISSING_METADATA_PROJECT_ID)
|
216
224
|
rescue Fluent::ConfigError => error
|
217
225
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
218
226
|
assert error.message.include? 'project_id'
|
219
227
|
exception_count += 1
|
220
228
|
end
|
221
229
|
assert_equal 1, exception_count
|
230
|
+
end
|
222
231
|
|
232
|
+
def test_configure_invalid_metadata_missing_zone_no_metadata_service
|
233
|
+
setup_no_metadata_service_stubs
|
223
234
|
exception_count = 0
|
224
235
|
begin
|
225
|
-
|
236
|
+
_d = create_driver(CONFIG_MISSING_METADATA_ZONE)
|
226
237
|
rescue Fluent::ConfigError => error
|
227
238
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
228
239
|
assert error.message.include? 'zone'
|
229
240
|
exception_count += 1
|
230
241
|
end
|
231
242
|
assert_equal 1, exception_count
|
243
|
+
end
|
232
244
|
|
245
|
+
def test_configure_invalid_metadata_missing_vm_id_no_metadata_service
|
246
|
+
setup_no_metadata_service_stubs
|
233
247
|
exception_count = 0
|
234
248
|
begin
|
235
|
-
|
249
|
+
_d = create_driver(CONFIG_MISSING_METADATA_VM_ID)
|
236
250
|
rescue Fluent::ConfigError => error
|
237
251
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
238
252
|
assert error.message.include? 'vm_id'
|
239
253
|
exception_count += 1
|
240
254
|
end
|
241
255
|
assert_equal 1, exception_count
|
256
|
+
end
|
242
257
|
|
258
|
+
def test_configure_invalid_metadata_missing_all_no_metadata_service
|
259
|
+
setup_no_metadata_service_stubs
|
243
260
|
exception_count = 0
|
244
261
|
begin
|
245
|
-
|
262
|
+
_d = create_driver(CONFIG_MISSING_METADATA_ALL)
|
246
263
|
rescue Fluent::ConfigError => error
|
247
264
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
248
265
|
assert error.message.include? 'project_id'
|
@@ -255,7 +272,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
255
272
|
|
256
273
|
def test_metadata_loading
|
257
274
|
setup_gce_metadata_stubs
|
258
|
-
d = create_driver
|
275
|
+
d = create_driver
|
259
276
|
d.run
|
260
277
|
assert_equal PROJECT_ID, d.instance.project_id
|
261
278
|
assert_equal ZONE, d.instance.zone
|
@@ -266,7 +283,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
266
283
|
def test_managed_vm_metadata_loading
|
267
284
|
setup_gce_metadata_stubs
|
268
285
|
setup_managed_vm_metadata_stubs
|
269
|
-
d = create_driver
|
286
|
+
d = create_driver
|
270
287
|
d.run
|
271
288
|
assert_equal PROJECT_ID, d.instance.project_id
|
272
289
|
assert_equal ZONE, d.instance.zone
|
@@ -334,7 +351,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
334
351
|
setup_ec2_metadata_stubs
|
335
352
|
exception_count = 0
|
336
353
|
begin
|
337
|
-
|
354
|
+
_d = create_driver
|
338
355
|
rescue Fluent::ConfigError => error
|
339
356
|
assert error.message.include? 'Unable to obtain metadata parameters:'
|
340
357
|
assert error.message.include? 'project_id'
|
@@ -346,8 +363,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
346
363
|
def test_one_log
|
347
364
|
setup_gce_metadata_stubs
|
348
365
|
setup_logging_stubs
|
349
|
-
d = create_driver
|
350
|
-
d.emit(
|
366
|
+
d = create_driver
|
367
|
+
d.emit('message' => log_entry(0))
|
351
368
|
d.run
|
352
369
|
verify_log_entries(1, COMPUTE_PARAMS)
|
353
370
|
end
|
@@ -356,8 +373,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
356
373
|
setup_gce_metadata_stubs
|
357
374
|
setup_logging_stubs
|
358
375
|
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
359
|
-
d = create_driver
|
360
|
-
d.emit(
|
376
|
+
d = create_driver
|
377
|
+
d.emit('message' => log_entry(0))
|
361
378
|
d.run
|
362
379
|
verify_log_entries(1, COMPUTE_PARAMS)
|
363
380
|
end
|
@@ -365,9 +382,10 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
365
382
|
def test_one_log_with_invalid_json_credentials
|
366
383
|
setup_gce_metadata_stubs
|
367
384
|
setup_logging_stubs
|
368
|
-
ENV['GOOGLE_APPLICATION_CREDENTIALS'] =
|
369
|
-
|
370
|
-
d
|
385
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = \
|
386
|
+
'test/plugin/data/invalid_credentials.json'
|
387
|
+
d = create_driver
|
388
|
+
d.emit('message' => log_entry(0))
|
371
389
|
exception_count = 0
|
372
390
|
begin
|
373
391
|
d.run
|
@@ -382,7 +400,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
382
400
|
setup_gce_metadata_stubs
|
383
401
|
setup_logging_stubs
|
384
402
|
d = create_driver(PRIVATE_KEY_CONFIG)
|
385
|
-
d.emit(
|
403
|
+
d.emit('message' => log_entry(0))
|
386
404
|
d.run
|
387
405
|
verify_log_entries(1, COMPUTE_PARAMS)
|
388
406
|
end
|
@@ -394,16 +412,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
394
412
|
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
395
413
|
setup_logging_stubs
|
396
414
|
d = create_driver(NO_METADATA_SERVICE_CONFIG + CUSTOM_METADATA_CONFIG)
|
397
|
-
d.emit(
|
415
|
+
d.emit('message' => log_entry(0))
|
398
416
|
d.run
|
399
417
|
verify_log_entries(1, CUSTOM_PARAMS)
|
400
418
|
end
|
401
419
|
|
402
420
|
def test_one_log_ec2
|
421
|
+
ENV['GOOGLE_APPLICATION_CREDENTIALS'] = 'test/plugin/data/credentials.json'
|
403
422
|
setup_ec2_metadata_stubs
|
404
423
|
setup_logging_stubs
|
405
424
|
d = create_driver(CONFIG_EC2_PROJECT_ID)
|
406
|
-
d.emit(
|
425
|
+
d.emit('message' => log_entry(0))
|
407
426
|
d.run
|
408
427
|
verify_log_entries(1, EC2_PARAMS)
|
409
428
|
end
|
@@ -411,12 +430,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
411
430
|
def test_struct_payload_log
|
412
431
|
setup_gce_metadata_stubs
|
413
432
|
setup_logging_stubs
|
414
|
-
d = create_driver
|
415
|
-
d.emit(
|
433
|
+
d = create_driver
|
434
|
+
d.emit('msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000)
|
416
435
|
d.run
|
417
436
|
verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
|
418
437
|
assert_equal 3, entry['structPayload'].size, entry
|
419
|
-
assert_equal
|
438
|
+
assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
|
420
439
|
assert_equal 'test', entry['structPayload']['tag2'], entry
|
421
440
|
assert_equal 5000, entry['structPayload']['data'], entry
|
422
441
|
end
|
@@ -425,26 +444,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
425
444
|
def test_timestamps
|
426
445
|
setup_gce_metadata_stubs
|
427
446
|
setup_logging_stubs
|
428
|
-
d = create_driver
|
447
|
+
d = create_driver
|
429
448
|
expected_ts = []
|
430
449
|
emit_index = 0
|
431
|
-
[Time.at(
|
450
|
+
[Time.at(123_456.789), Time.at(0), Time.now].each do |ts|
|
432
451
|
# Test the "native" fluentd timestamp as well as our nanosecond tags.
|
433
|
-
d.emit({'message' => log_entry(emit_index)}, ts.to_f)
|
452
|
+
d.emit({ 'message' => log_entry(emit_index) }, ts.to_f)
|
434
453
|
# The native timestamp currently only supports second granularity
|
435
454
|
# (fluentd issue #461), so strip nanoseconds from the expected value.
|
436
455
|
expected_ts.push(Time.at(ts.tv_sec))
|
437
456
|
emit_index += 1
|
438
|
-
d.emit(
|
439
|
-
|
457
|
+
d.emit('message' => log_entry(emit_index),
|
458
|
+
'timeNanos' => ts.tv_sec * 1_000_000_000 + ts.tv_nsec)
|
440
459
|
expected_ts.push(ts)
|
441
460
|
emit_index += 1
|
442
|
-
d.emit(
|
443
|
-
|
461
|
+
d.emit('message' => log_entry(emit_index),
|
462
|
+
'timestamp' => { 'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec })
|
444
463
|
expected_ts.push(ts)
|
445
464
|
emit_index += 1
|
446
|
-
d.emit(
|
447
|
-
|
465
|
+
d.emit('message' => log_entry(emit_index),
|
466
|
+
'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec)
|
448
467
|
expected_ts.push(ts)
|
449
468
|
emit_index += 1
|
450
469
|
end
|
@@ -452,9 +471,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
452
471
|
verify_index = 0
|
453
472
|
verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
|
454
473
|
assert_equal expected_ts[verify_index].tv_sec,
|
455
|
-
|
474
|
+
entry['metadata']['timestamp']['seconds'], entry
|
456
475
|
assert_equal expected_ts[verify_index].tv_nsec,
|
457
|
-
|
476
|
+
entry['metadata']['timestamp']['nanos'], entry
|
458
477
|
verify_index += 1
|
459
478
|
end
|
460
479
|
end
|
@@ -462,27 +481,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
462
481
|
def test_malformed_timestamp
|
463
482
|
setup_gce_metadata_stubs
|
464
483
|
setup_logging_stubs
|
465
|
-
d = create_driver
|
484
|
+
d = create_driver
|
466
485
|
# if timestamp is not a hash it is passed through to the struct payload.
|
467
|
-
d.emit(
|
486
|
+
d.emit('message' => log_entry(0), 'timestamp' => 'not-a-hash')
|
468
487
|
d.run
|
469
|
-
verify_index = 0
|
470
488
|
verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
|
471
489
|
assert_equal 2, entry['structPayload'].size, entry
|
472
|
-
assert_equal
|
490
|
+
assert_equal 'not-a-hash', entry['structPayload']['timestamp'], entry
|
473
491
|
end
|
474
492
|
end
|
475
493
|
|
476
494
|
def test_severities
|
477
495
|
setup_gce_metadata_stubs
|
478
496
|
setup_logging_stubs
|
479
|
-
d = create_driver
|
497
|
+
d = create_driver
|
480
498
|
expected_severity = []
|
481
499
|
emit_index = 0
|
482
500
|
# Array of pairs of [parsed_severity, expected_severity]
|
483
|
-
[
|
484
|
-
['
|
485
|
-
d.emit(
|
501
|
+
[%w(INFO INFO), %w(warn WARNING), %w(E ERROR), %w(BLAH DEFAULT),
|
502
|
+
['105', 100], ['', 'DEFAULT']].each do |sev|
|
503
|
+
d.emit('message' => log_entry(emit_index), 'severity' => sev[0])
|
486
504
|
expected_severity.push(sev[1])
|
487
505
|
emit_index += 1
|
488
506
|
end
|
@@ -490,7 +508,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
490
508
|
verify_index = 0
|
491
509
|
verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
|
492
510
|
assert_equal expected_severity[verify_index],
|
493
|
-
|
511
|
+
entry['metadata']['severity'], entry
|
494
512
|
verify_index += 1
|
495
513
|
end
|
496
514
|
end
|
@@ -498,9 +516,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
498
516
|
def test_label_map_without_field_present
|
499
517
|
setup_gce_metadata_stubs
|
500
518
|
setup_logging_stubs
|
501
|
-
config = %
|
519
|
+
config = %(label_map { "label_field": "sent_label" })
|
502
520
|
d = create_driver(config)
|
503
|
-
d.emit(
|
521
|
+
d.emit('message' => log_entry(0))
|
504
522
|
d.run
|
505
523
|
# No additional labels should be present
|
506
524
|
verify_log_entries(1, COMPUTE_PARAMS)
|
@@ -509,9 +527,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
509
527
|
def test_label_map_with_field_present
|
510
528
|
setup_gce_metadata_stubs
|
511
529
|
setup_logging_stubs
|
512
|
-
config = %
|
530
|
+
config = %(label_map { "label_field": "sent_label" })
|
513
531
|
d = create_driver(config)
|
514
|
-
d.emit(
|
532
|
+
d.emit('message' => log_entry(0), 'label_field' => 'label_value')
|
515
533
|
d.run
|
516
534
|
# make a deep copy of COMPUTE_PARAMS and add the parsed label.
|
517
535
|
params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
|
@@ -522,19 +540,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
522
540
|
def test_label_map_with_multiple_fields
|
523
541
|
setup_gce_metadata_stubs
|
524
542
|
setup_logging_stubs
|
525
|
-
config = %
|
543
|
+
config = %(
|
526
544
|
label_map {
|
527
545
|
"label1": "sent_label_1",
|
528
546
|
"label_number_two": "foo.googleapis.com/bar",
|
529
547
|
"label3": "label3"
|
530
|
-
}
|
548
|
+
}
|
549
|
+
)
|
531
550
|
d = create_driver(config)
|
532
551
|
# not_a_label passes through to the struct payload
|
533
|
-
d.emit(
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
552
|
+
d.emit('message' => log_entry(0),
|
553
|
+
'label1' => 'value1',
|
554
|
+
'label_number_two' => 'value2',
|
555
|
+
'not_a_label' => 'value4',
|
556
|
+
'label3' => 'value3')
|
538
557
|
d.run
|
539
558
|
# make a deep copy of COMPUTE_PARAMS and add the parsed labels.
|
540
559
|
params = Marshal.load(Marshal.dump(COMPUTE_PARAMS))
|
@@ -543,7 +562,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
543
562
|
params['labels']['label3'] = 'value3'
|
544
563
|
verify_log_entries(1, params, 'structPayload') do |entry|
|
545
564
|
assert_equal 2, entry['structPayload'].size, entry
|
546
|
-
assert_equal
|
565
|
+
assert_equal 'test log entry 0', entry['structPayload']['message'], entry
|
547
566
|
assert_equal 'value4', entry['structPayload']['not_a_label'], entry
|
548
567
|
end
|
549
568
|
end
|
@@ -551,14 +570,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
551
570
|
def test_multiple_logs
|
552
571
|
setup_gce_metadata_stubs
|
553
572
|
setup_logging_stubs
|
554
|
-
d = create_driver
|
573
|
+
d = create_driver
|
555
574
|
# Only test a few values because otherwise the test can take minutes.
|
556
575
|
[2, 3, 5, 11, 50].each do |n|
|
557
576
|
# The test driver doesn't clear its buffer of entries after running, so
|
558
577
|
# do it manually here.
|
559
578
|
d.instance_variable_get('@entries').clear
|
560
579
|
@logs_sent = []
|
561
|
-
n.times { |i| d.emit(
|
580
|
+
n.times { |i| d.emit('message' => log_entry(i)) }
|
562
581
|
d.run
|
563
582
|
verify_log_entries(n, COMPUTE_PARAMS)
|
564
583
|
end
|
@@ -567,9 +586,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
567
586
|
def test_malformed_log
|
568
587
|
setup_gce_metadata_stubs
|
569
588
|
setup_logging_stubs
|
570
|
-
d = create_driver
|
589
|
+
d = create_driver
|
571
590
|
# if the entry is not a hash, the plugin should silently drop it.
|
572
|
-
d.emit(
|
591
|
+
d.emit('a string is not a valid message')
|
573
592
|
d.run
|
574
593
|
assert @logs_sent.empty?
|
575
594
|
end
|
@@ -578,21 +597,21 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
578
597
|
setup_gce_metadata_stubs
|
579
598
|
# The API Client should not retry this and the plugin should consume
|
580
599
|
# the exception.
|
581
|
-
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
582
|
-
|
583
|
-
d = create_driver
|
584
|
-
d.emit(
|
600
|
+
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
601
|
+
.to_return(status: 400, body: 'Bad Request')
|
602
|
+
d = create_driver
|
603
|
+
d.emit('message' => log_entry(0))
|
585
604
|
d.run
|
586
|
-
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :
|
605
|
+
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 1)
|
587
606
|
end
|
588
607
|
|
589
608
|
# helper for the ClientError retriable special cases below.
|
590
609
|
def client_error_helper(message)
|
591
610
|
setup_gce_metadata_stubs
|
592
|
-
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
593
|
-
|
594
|
-
d = create_driver
|
595
|
-
d.emit(
|
611
|
+
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
612
|
+
.to_return(status: 401, body: message)
|
613
|
+
d = create_driver
|
614
|
+
d.emit('message' => log_entry(0))
|
596
615
|
exception_count = 0
|
597
616
|
begin
|
598
617
|
d.run
|
@@ -600,38 +619,39 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
600
619
|
assert_equal message, error.message
|
601
620
|
exception_count += 1
|
602
621
|
end
|
603
|
-
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :
|
622
|
+
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 2)
|
604
623
|
assert_equal 1, exception_count
|
605
624
|
end
|
606
625
|
|
607
626
|
def test_client_error_invalid_credentials
|
608
|
-
client_error_helper(
|
627
|
+
client_error_helper('Invalid Credentials')
|
609
628
|
end
|
610
629
|
|
611
630
|
def test_client_error_caller_does_not_have_permission
|
612
|
-
client_error_helper(
|
631
|
+
client_error_helper('The caller does not have permission')
|
613
632
|
end
|
614
633
|
|
615
634
|
def test_client_error_request_had_invalid_credentials
|
616
|
-
client_error_helper(
|
635
|
+
client_error_helper('Request had invalid credentials.')
|
617
636
|
end
|
618
637
|
|
619
638
|
def test_client_error_project_has_not_enabled_the_api
|
620
|
-
client_error_helper(
|
639
|
+
client_error_helper('Project has not enabled the API. Please use ' \
|
640
|
+
'Google Developers Console to activate the API for your project.')
|
621
641
|
end
|
622
642
|
|
623
643
|
def test_client_error_unable_to_fetch_accesss_token
|
624
|
-
client_error_helper(
|
644
|
+
client_error_helper('Unable to fetch access token (no scopes configured?)')
|
625
645
|
end
|
626
646
|
|
627
647
|
def test_server_error
|
628
648
|
setup_gce_metadata_stubs
|
629
649
|
# The API client should retry this once, then throw an exception which
|
630
650
|
# gets propagated through the plugin.
|
631
|
-
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
632
|
-
|
633
|
-
d = create_driver
|
634
|
-
d.emit(
|
651
|
+
stub_request(:post, uri_for_log(COMPUTE_PARAMS))
|
652
|
+
.to_return(status: 500, body: 'Server Error')
|
653
|
+
d = create_driver
|
654
|
+
d.emit('message' => log_entry(0))
|
635
655
|
exception_count = 0
|
636
656
|
begin
|
637
657
|
d.run
|
@@ -639,7 +659,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
639
659
|
assert_equal 'Server Error', error.message
|
640
660
|
exception_count += 1
|
641
661
|
end
|
642
|
-
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :
|
662
|
+
assert_requested(:post, uri_for_log(COMPUTE_PARAMS), times: 2)
|
643
663
|
assert_equal 1, exception_count
|
644
664
|
end
|
645
665
|
|
@@ -647,8 +667,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
647
667
|
setup_gce_metadata_stubs
|
648
668
|
setup_managed_vm_metadata_stubs
|
649
669
|
setup_logging_stubs
|
650
|
-
d = create_driver
|
651
|
-
d.emit(
|
670
|
+
d = create_driver
|
671
|
+
d.emit('message' => log_entry(0))
|
652
672
|
d.run
|
653
673
|
verify_log_entries(1, VMENGINE_PARAMS)
|
654
674
|
end
|
@@ -657,20 +677,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
657
677
|
setup_gce_metadata_stubs
|
658
678
|
setup_managed_vm_metadata_stubs
|
659
679
|
setup_logging_stubs
|
660
|
-
d = create_driver
|
680
|
+
d = create_driver
|
661
681
|
[2, 3, 5, 11, 50].each do |n|
|
662
682
|
# The test driver doesn't clear its buffer of entries after running, so
|
663
683
|
# do it manually here.
|
664
684
|
d.instance_variable_get('@entries').clear
|
665
685
|
@logs_sent = []
|
666
|
-
n.times { |i| d.emit(
|
686
|
+
n.times { |i| d.emit('message' => log_entry(i)) }
|
667
687
|
d.run
|
668
688
|
verify_log_entries(n, VMENGINE_PARAMS)
|
669
689
|
end
|
670
690
|
end
|
671
691
|
|
672
692
|
# Make parse_severity public so we can test it.
|
673
|
-
class Fluent::GoogleCloudOutput
|
693
|
+
class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
|
674
694
|
public :parse_severity
|
675
695
|
end
|
676
696
|
|
@@ -678,8 +698,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
678
698
|
test_obj = Fluent::GoogleCloudOutput.new
|
679
699
|
|
680
700
|
# known severities should translate to themselves, regardless of case
|
681
|
-
|
682
|
-
|
701
|
+
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY).each \
|
702
|
+
do |severity|
|
683
703
|
assert_equal(severity, test_obj.parse_severity(severity))
|
684
704
|
assert_equal(severity, test_obj.parse_severity(severity.downcase))
|
685
705
|
assert_equal(severity, test_obj.parse_severity(severity.capitalize))
|
@@ -751,25 +771,26 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
751
771
|
|
752
772
|
def uri_for_log(config)
|
753
773
|
'https://logging.googleapis.com/v1beta3/projects/' + config['project_id'] +
|
754
|
-
|
774
|
+
'/logs/' + config['log_name'] + '/entries:write'
|
755
775
|
end
|
756
776
|
|
757
777
|
def stub_metadata_request(metadata_path, response_body)
|
758
|
-
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
|
759
|
-
|
760
|
-
|
778
|
+
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' +
|
779
|
+
metadata_path)
|
780
|
+
.to_return(body: response_body, status: 200,
|
781
|
+
headers: { 'Content-Length' => response_body.length })
|
761
782
|
end
|
762
783
|
|
763
784
|
def setup_no_metadata_service_stubs
|
764
785
|
# Simulate a machine with no metadata service present
|
765
|
-
stub_request(:any,
|
766
|
-
to_raise
|
786
|
+
stub_request(:any, %r{http://169.254.169.254/.*})
|
787
|
+
.to_raise(Errno::EHOSTUNREACH)
|
767
788
|
end
|
768
789
|
|
769
790
|
def setup_gce_metadata_stubs
|
770
791
|
# Stub the root, used for platform detection by the plugin and 'googleauth'.
|
771
|
-
stub_request(:get, 'http://169.254.169.254')
|
772
|
-
to_return(:
|
792
|
+
stub_request(:get, 'http://169.254.169.254')
|
793
|
+
.to_return(status: 200, headers: { 'Metadata-Flavor' => 'Google' })
|
773
794
|
|
774
795
|
# Create stubs for all the GCE metadata lookups the agent needs to make.
|
775
796
|
stub_metadata_request('project/project-id', PROJECT_ID)
|
@@ -779,57 +800,60 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
779
800
|
"attribute1\nattribute2\nattribute3")
|
780
801
|
|
781
802
|
# Used by 'googleauth' to fetch the default service account credentials.
|
782
|
-
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/
|
783
|
-
|
784
|
-
|
785
|
-
|
786
|
-
|
803
|
+
stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' \
|
804
|
+
'instance/service-accounts/default/token')
|
805
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
806
|
+
status: 200,
|
807
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
808
|
+
'Content-Type' => 'application/json' })
|
787
809
|
end
|
788
810
|
|
789
811
|
def setup_ec2_metadata_stubs
|
790
812
|
# Stub the root, used for platform detection
|
791
|
-
stub_request(:get, 'http://169.254.169.254')
|
792
|
-
to_return(:
|
813
|
+
stub_request(:get, 'http://169.254.169.254')
|
814
|
+
.to_return(status: 200, headers: { 'Server' => 'EC2ws' })
|
793
815
|
|
794
816
|
# Stub the identity document lookup made by the agent.
|
795
|
-
stub_request(:get, 'http://169.254.169.254/latest/dynamic/
|
796
|
-
|
797
|
-
|
817
|
+
stub_request(:get, 'http://169.254.169.254/latest/dynamic/' \
|
818
|
+
'instance-identity/document')
|
819
|
+
.to_return(body: EC2_IDENTITY_DOCUMENT, status: 200,
|
820
|
+
headers: { 'Content-Length' => EC2_IDENTITY_DOCUMENT.length })
|
798
821
|
end
|
799
822
|
|
800
823
|
def setup_logging_stubs
|
801
|
-
[COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS]
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
|
824
|
+
[COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS]
|
825
|
+
.each do |params|
|
826
|
+
stub_request(:post, uri_for_log(params))
|
827
|
+
.to_return do |request|
|
828
|
+
@logs_sent << JSON.parse(request.body)
|
829
|
+
{ body: '' }
|
830
|
+
end
|
806
831
|
end
|
807
|
-
end
|
808
832
|
end
|
809
833
|
|
810
834
|
def setup_auth_stubs
|
811
835
|
# Used when loading credentials from a JSON file.
|
812
|
-
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
813
|
-
with(:
|
814
|
-
to_return(:
|
815
|
-
|
816
|
-
|
817
|
-
|
818
|
-
|
819
|
-
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
820
|
-
with(:
|
821
|
-
to_return(:
|
822
|
-
|
823
|
-
|
824
|
-
|
836
|
+
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
837
|
+
.with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
|
838
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
839
|
+
status: 200,
|
840
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
841
|
+
'Content-Type' => 'application/json' })
|
842
|
+
|
843
|
+
stub_request(:post, 'https://www.googleapis.com/oauth2/v3/token')
|
844
|
+
.with(body: hash_including(grant_type: 'refresh_token'))
|
845
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
846
|
+
status: 200,
|
847
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
848
|
+
'Content-Type' => 'application/json' })
|
825
849
|
|
826
850
|
# Used for 'private_key' auth.
|
827
|
-
stub_request(:post, 'https://accounts.google.com/o/oauth2/token')
|
828
|
-
with(:
|
829
|
-
to_return(:
|
830
|
-
|
831
|
-
|
832
|
-
|
851
|
+
stub_request(:post, 'https://accounts.google.com/o/oauth2/token')
|
852
|
+
.with(body: hash_including(grant_type: AUTH_GRANT_TYPE))
|
853
|
+
.to_return(body: %({"access_token": "#{FAKE_AUTH_TOKEN}"}),
|
854
|
+
status: 200,
|
855
|
+
headers: { 'Content-Length' => FAKE_AUTH_TOKEN.length,
|
856
|
+
'Content-Type' => 'application/json' })
|
833
857
|
end
|
834
858
|
|
835
859
|
def setup_managed_vm_metadata_stubs
|
@@ -850,21 +874,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
850
874
|
all_labels ||= common_labels
|
851
875
|
all_labels.merge!(entry['metadata']['labels'] || {})
|
852
876
|
all_labels.each do |key, value|
|
853
|
-
assert expected_labels.
|
854
|
-
assert_equal value, expected_labels[key],
|
855
|
-
|
877
|
+
assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
|
878
|
+
assert_equal value, expected_labels[key], 'Value mismatch - expected ' \
|
879
|
+
"#{expected_labels[key]} in #{key} => #{value}"
|
856
880
|
end
|
857
|
-
assert_equal expected_labels.length, all_labels.length,
|
858
|
-
|
859
|
-
"#{all_labels.length}")
|
881
|
+
assert_equal expected_labels.length, all_labels.length, 'Expected ' \
|
882
|
+
"#{expected_labels.length} labels, got #{all_labels.length}"
|
860
883
|
end
|
861
884
|
|
862
885
|
# The caller can optionally provide a block which is called for each entry.
|
863
|
-
def verify_log_entries(n, params, payload_type='textPayload')
|
886
|
+
def verify_log_entries(n, params, payload_type = 'textPayload')
|
864
887
|
i = 0
|
865
888
|
@logs_sent.each do |batch|
|
866
889
|
batch['entries'].each do |entry|
|
867
|
-
assert entry.
|
890
|
+
assert entry.key?(payload_type)
|
868
891
|
if (payload_type == 'textPayload')
|
869
892
|
# Check the payload for textPayload, otherwise it is up to the caller.
|
870
893
|
assert_equal "test log entry #{i}", entry['textPayload'], batch
|
@@ -873,9 +896,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
873
896
|
assert_equal params['zone'], entry['metadata']['zone']
|
874
897
|
assert_equal params['service_name'], entry['metadata']['serviceName']
|
875
898
|
check_labels entry, batch['commonLabels'], params['labels']
|
876
|
-
if
|
877
|
-
yield(entry)
|
878
|
-
end
|
899
|
+
yield(entry) if block_given?
|
879
900
|
i += 1
|
880
901
|
assert i <= n, "Number of entries #{i} exceeds expected number #{n}"
|
881
902
|
end
|