fluent-plugin-google-cloud 0.4.9 → 0.4.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +9 -6
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/out_google_cloud.rb +69 -31
- data/test/plugin/test_out_google_cloud.rb +45 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8625be40328ad3c14dd4596985a1017092c2884a
|
4
|
+
data.tar.gz: d39dc933fdc5c0db8830c83ed94bd19f1db147da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d60286af889814e55fa15012b8fcdd5d1114ea2e14c420fdce8d7942d33479378e26cf2074cb02b2ab08a1943052a1f4ce3fa5de75a4526eeb0974c17c3e4da3
|
7
|
+
data.tar.gz: 67d62b7f2292cdc4d1a98aa5a7591d7d838ce923764c5055c5da3a92c00edbb84558b1e73535e758f7c335b29074c743c48a6439de42db1cff29d9cdab63788c
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.4.
|
4
|
+
fluent-plugin-google-cloud (0.4.10)
|
5
5
|
fluentd (>= 0.10)
|
6
6
|
google-api-client (~> 0.8.6)
|
7
7
|
googleauth (~> 0.4)
|
@@ -24,13 +24,13 @@ GEM
|
|
24
24
|
addressable (>= 2.3.1)
|
25
25
|
extlib (>= 0.9.15)
|
26
26
|
multi_json (>= 1.0.0)
|
27
|
-
cool.io (1.4.
|
27
|
+
cool.io (1.4.1)
|
28
28
|
crack (0.4.2)
|
29
29
|
safe_yaml (~> 1.0.0)
|
30
30
|
extlib (0.9.16)
|
31
|
-
faraday (0.9.
|
31
|
+
faraday (0.9.2)
|
32
32
|
multipart-post (>= 1.2, < 3)
|
33
|
-
fluentd (0.12.
|
33
|
+
fluentd (0.12.16)
|
34
34
|
cool.io (>= 1.2.2, < 2.0.0)
|
35
35
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
36
36
|
json (>= 1.4.3)
|
@@ -70,7 +70,7 @@ GEM
|
|
70
70
|
multi_json (~> 1.10)
|
71
71
|
memoist (0.12.0)
|
72
72
|
metaclass (0.0.4)
|
73
|
-
minitest (5.8.
|
73
|
+
minitest (5.8.1)
|
74
74
|
mocha (1.1.0)
|
75
75
|
metaclass (~> 0.0.1)
|
76
76
|
msgpack (0.5.12)
|
@@ -104,7 +104,7 @@ GEM
|
|
104
104
|
thread_safe (0.3.5)
|
105
105
|
tzinfo (1.2.2)
|
106
106
|
thread_safe (~> 0.1)
|
107
|
-
tzinfo-data (1.2015.
|
107
|
+
tzinfo-data (1.2015.7)
|
108
108
|
tzinfo (>= 1.0.0)
|
109
109
|
webmock (1.21.0)
|
110
110
|
addressable (>= 2.3.6)
|
@@ -121,3 +121,6 @@ DEPENDENCIES
|
|
121
121
|
rubocop (~> 0.33.0)
|
122
122
|
test-unit (~> 3.0.2)
|
123
123
|
webmock (>= 1.17.0)
|
124
|
+
|
125
|
+
BUNDLED WITH
|
126
|
+
1.10.6
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache 2.0'
|
13
|
-
gem.version = '0.4.
|
13
|
+
gem.version = '0.4.10'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
|
@@ -156,7 +156,7 @@ module Fluent
|
|
156
156
|
end
|
157
157
|
|
158
158
|
@cloudfunctions_tag_regexp =
|
159
|
-
/\.(?<function_name
|
159
|
+
/\.(?<function_name>[^.]+)(?:\.\d+)?-[^-]+_default_worker$/
|
160
160
|
@cloudfunctions_log_regexp = /^
|
161
161
|
(?:\[(?<severity>.)\])?
|
162
162
|
\[(?<timestamp>.{24})\]
|
@@ -326,12 +326,54 @@ module Fluent
|
|
326
326
|
@service_name = CONTAINER_SERVICE
|
327
327
|
end
|
328
328
|
end
|
329
|
+
is_container_json = nil
|
329
330
|
arr.each do |time, record|
|
330
331
|
next unless record.is_a?(Hash)
|
332
|
+
|
333
|
+
entry = {
|
334
|
+
'metadata' => {
|
335
|
+
'serviceName' => @service_name,
|
336
|
+
'projectId' => @project_id,
|
337
|
+
'zone' => @zone,
|
338
|
+
'labels' => {}
|
339
|
+
}
|
340
|
+
}
|
341
|
+
|
331
342
|
if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
332
343
|
@cloudfunctions_log_match =
|
333
344
|
@cloudfunctions_log_regexp.match(record['log'])
|
334
345
|
end
|
346
|
+
if @service_name == CONTAINER_SERVICE
|
347
|
+
# Move the stdout/stderr annotation from the record into a label
|
348
|
+
field_to_label(record, 'stream', entry['metadata']['labels'],
|
349
|
+
"#{CONTAINER_SERVICE}/stream")
|
350
|
+
# If the record has been annotated by the kubernetes_metadata_filter
|
351
|
+
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
352
|
+
# populated at the grouped_entries level from the group's tag.
|
353
|
+
if record.key?('kubernetes')
|
354
|
+
handle_container_metadata(record, entry)
|
355
|
+
end
|
356
|
+
# If the log from the user container is json, we want to export it
|
357
|
+
# as a structured log. Now that we've pulled out all the
|
358
|
+
# container-specific metadata from the record, we can replace the
|
359
|
+
# record with the json that the user logged.
|
360
|
+
# To save CPU in the common case of unstructured logs, only check if
|
361
|
+
# the contents are parsable as json for the first entry of each
|
362
|
+
# batch.
|
363
|
+
if is_container_json.nil? && record.key?('log')
|
364
|
+
record_json = parse_json_or_nil(record['log'])
|
365
|
+
if record_json.nil?
|
366
|
+
is_container_json = false
|
367
|
+
else
|
368
|
+
record = record_json
|
369
|
+
is_container_json = true
|
370
|
+
end
|
371
|
+
elsif is_container_json && record.key?('log')
|
372
|
+
record_json = parse_json_or_nil(record['log'])
|
373
|
+
record = record_json unless record_json.nil?
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
335
377
|
if record.key?('timestamp') &&
|
336
378
|
record['timestamp'].is_a?(Hash) &&
|
337
379
|
record['timestamp'].key?('seconds') &&
|
@@ -367,33 +409,13 @@ module Fluent
|
|
367
409
|
ts_secs = timestamp.tv_sec
|
368
410
|
ts_nanos = timestamp.tv_nsec
|
369
411
|
end
|
370
|
-
entry = {
|
371
|
-
'
|
372
|
-
|
373
|
-
'projectId' => @project_id,
|
374
|
-
'zone' => @zone,
|
375
|
-
'timestamp' => {
|
376
|
-
'seconds' => ts_secs,
|
377
|
-
'nanos' => ts_nanos
|
378
|
-
},
|
379
|
-
'labels' => {}
|
380
|
-
}
|
412
|
+
entry['metadata']['timestamp'] = {
|
413
|
+
'seconds' => ts_secs,
|
414
|
+
'nanos' => ts_nanos
|
381
415
|
}
|
382
416
|
|
383
417
|
set_severity(record, entry)
|
384
418
|
|
385
|
-
if @service_name == CONTAINER_SERVICE
|
386
|
-
# Move the stdout/stderr annotation from the record into a label
|
387
|
-
field_to_label(record, 'stream', entry['metadata']['labels'],
|
388
|
-
"#{CONTAINER_SERVICE}/stream")
|
389
|
-
# If the record has been annotated by the kubernetes_metadata_filter
|
390
|
-
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
391
|
-
# populated at the grouped_entries level from the group's tag.
|
392
|
-
if record.key?('kubernetes')
|
393
|
-
handle_container_metadata(record, entry)
|
394
|
-
end
|
395
|
-
end
|
396
|
-
|
397
419
|
# If a field is present in the label_map, send its value as a label
|
398
420
|
# (mapping the field name to label name as specified in the config)
|
399
421
|
# and do not send that field as part of the payload.
|
@@ -410,7 +432,7 @@ module Fluent
|
|
410
432
|
@cloudfunctions_log_match['execution_id']
|
411
433
|
end
|
412
434
|
|
413
|
-
set_payload(record, entry)
|
435
|
+
set_payload(record, entry, is_container_json)
|
414
436
|
|
415
437
|
# Remove the labels metadata if we didn't populate it with anything.
|
416
438
|
if entry['metadata']['labels'].empty?
|
@@ -478,6 +500,17 @@ module Fluent
|
|
478
500
|
error_class: error.class.to_s, error: error.to_s
|
479
501
|
end
|
480
502
|
|
503
|
+
def parse_json_or_nil(input)
|
504
|
+
# Only here to please rubocop...
|
505
|
+
return nil if input.nil?
|
506
|
+
|
507
|
+
begin
|
508
|
+
return JSON.parse(input)
|
509
|
+
rescue JSON::ParserError
|
510
|
+
return nil
|
511
|
+
end
|
512
|
+
end
|
513
|
+
|
481
514
|
# "enum" of Platform values
|
482
515
|
module Platform
|
483
516
|
OTHER = 0 # Other/unkown platform
|
@@ -656,13 +689,18 @@ module Fluent
|
|
656
689
|
record.delete(field)
|
657
690
|
end
|
658
691
|
|
659
|
-
def set_payload(record, entry)
|
660
|
-
# Use textPayload if
|
661
|
-
#
|
692
|
+
def set_payload(record, entry, is_container_json)
|
693
|
+
# Use textPayload if
|
694
|
+
# 1. This is a Cloud Functions log that matched the expected regexp
|
695
|
+
# 2. This is a Cloud Functions log and the 'log' key is available
|
696
|
+
# 3. This is an unstructured Container log and the 'log' key is available
|
697
|
+
# 4. The only remaining key is 'message'
|
662
698
|
if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
|
663
699
|
entry['textPayload'] = @cloudfunctions_log_match['text']
|
664
|
-
elsif
|
665
|
-
|
700
|
+
elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
701
|
+
entry['textPayload'] = record['log']
|
702
|
+
elsif @service_name == CONTAINER_SERVICE && record.key?('log') &&
|
703
|
+
!is_container_json
|
666
704
|
entry['textPayload'] = record['log']
|
667
705
|
elsif record.size == 1 && record.key?('message')
|
668
706
|
entry['textPayload'] = record['message']
|
@@ -691,7 +729,7 @@ module Fluent
|
|
691
729
|
def init_api_client
|
692
730
|
@client = Google::APIClient.new(
|
693
731
|
application_name: 'Fluentd Google Cloud Logging plugin',
|
694
|
-
application_version: '0.4.
|
732
|
+
application_version: '0.4.10',
|
695
733
|
retries: 1)
|
696
734
|
|
697
735
|
if @auth_method == 'private_key'
|
@@ -75,7 +75,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
75
75
|
CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
|
76
76
|
CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
|
77
77
|
CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
|
78
|
-
CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}-c0l82"
|
78
|
+
CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}.987-c0l82"
|
79
79
|
CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
|
80
80
|
|
81
81
|
# Parameters used for authentication
|
@@ -854,7 +854,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
854
854
|
setup_container_metadata_stubs
|
855
855
|
setup_logging_stubs
|
856
856
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
857
|
-
d.emit(container_log_entry_with_metadata(0))
|
857
|
+
d.emit(container_log_entry_with_metadata(log_entry(0)))
|
858
858
|
d.run
|
859
859
|
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS)
|
860
860
|
end
|
@@ -869,7 +869,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
869
869
|
# do it manually here.
|
870
870
|
d.instance_variable_get('@entries').clear
|
871
871
|
@logs_sent = []
|
872
|
-
n.times { |i| d.emit(container_log_entry_with_metadata(i)) }
|
872
|
+
n.times { |i| d.emit(container_log_entry_with_metadata(log_entry(i))) }
|
873
873
|
d.run
|
874
874
|
verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS)
|
875
875
|
end
|
@@ -880,7 +880,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
880
880
|
setup_container_metadata_stubs
|
881
881
|
setup_logging_stubs
|
882
882
|
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
883
|
-
d.emit(container_log_entry(0))
|
883
|
+
d.emit(container_log_entry(log_entry(0)))
|
884
884
|
d.run
|
885
885
|
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS)
|
886
886
|
end
|
@@ -895,12 +895,46 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
895
895
|
# do it manually here.
|
896
896
|
d.instance_variable_get('@entries').clear
|
897
897
|
@logs_sent = []
|
898
|
-
n.times { |i| d.emit(container_log_entry(i)) }
|
898
|
+
n.times { |i| d.emit(container_log_entry(log_entry(i))) }
|
899
899
|
d.run
|
900
900
|
verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS)
|
901
901
|
end
|
902
902
|
end
|
903
903
|
|
904
|
+
def test_struct_container_log_metadata_from_plugin
|
905
|
+
setup_gce_metadata_stubs
|
906
|
+
setup_container_metadata_stubs
|
907
|
+
setup_logging_stubs
|
908
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
909
|
+
d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
|
910
|
+
'"tag2": "test", "data": 5000}'))
|
911
|
+
d.run
|
912
|
+
verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
|
913
|
+
'structPayload') do |entry|
|
914
|
+
assert_equal 3, entry['structPayload'].size, entry
|
915
|
+
assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
|
916
|
+
assert_equal 'test', entry['structPayload']['tag2'], entry
|
917
|
+
assert_equal 5000, entry['structPayload']['data'], entry
|
918
|
+
end
|
919
|
+
end
|
920
|
+
|
921
|
+
def test_struct_container_log_metadata_from_tag
|
922
|
+
setup_gce_metadata_stubs
|
923
|
+
setup_container_metadata_stubs
|
924
|
+
setup_logging_stubs
|
925
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
|
926
|
+
d.emit(container_log_entry('{"msg": "test log entry 0", ' \
|
927
|
+
'"tag2": "test", "data": 5000}'))
|
928
|
+
d.run
|
929
|
+
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
|
930
|
+
'structPayload') do |entry|
|
931
|
+
assert_equal 3, entry['structPayload'].size, entry
|
932
|
+
assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
|
933
|
+
assert_equal 'test', entry['structPayload']['tag2'], entry
|
934
|
+
assert_equal 5000, entry['structPayload']['data'], entry
|
935
|
+
end
|
936
|
+
end
|
937
|
+
|
904
938
|
def test_one_cloudfunctions_log
|
905
939
|
setup_gce_metadata_stubs
|
906
940
|
setup_cloudfunctions_metadata_stubs
|
@@ -1193,9 +1227,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1193
1227
|
stub_metadata_request('instance/attributes/gcf_region', 'us-central1')
|
1194
1228
|
end
|
1195
1229
|
|
1196
|
-
def container_log_entry_with_metadata(
|
1230
|
+
def container_log_entry_with_metadata(log)
|
1197
1231
|
{
|
1198
|
-
log:
|
1232
|
+
log: log,
|
1199
1233
|
stream: 'stdout',
|
1200
1234
|
kubernetes: {
|
1201
1235
|
namespace_id: CONTAINER_NAMESPACE_ID,
|
@@ -1207,9 +1241,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1207
1241
|
}
|
1208
1242
|
end
|
1209
1243
|
|
1210
|
-
def container_log_entry(
|
1244
|
+
def container_log_entry(log)
|
1211
1245
|
{
|
1212
|
-
log:
|
1246
|
+
log: log,
|
1213
1247
|
stream: 'stdout'
|
1214
1248
|
}
|
1215
1249
|
end
|
@@ -1253,7 +1287,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1253
1287
|
@logs_sent.each do |batch|
|
1254
1288
|
batch['entries'].each do |entry|
|
1255
1289
|
unless payload_type.empty?
|
1256
|
-
assert entry.key?(payload_type)
|
1290
|
+
assert entry.key?(payload_type), 'Entry did not contain expected ' \
|
1291
|
+
"#{payload_type} key: " + entry.to_s
|
1257
1292
|
# Check the payload for textPayload, otherwise it's up to the caller.
|
1258
1293
|
if (payload_type == 'textPayload')
|
1259
1294
|
assert_equal "test log entry #{i}", entry['textPayload'], batch
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Derr
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-10-
|
12
|
+
date: 2015-10-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|