fluent-plugin-google-cloud 0.4.7 → 0.4.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/out_google_cloud.rb +118 -22
- data/test/plugin/test_out_google_cloud.rb +187 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e6d35bf9167d8e4847b5e54cc16e0f78dc41310d
|
4
|
+
data.tar.gz: 3481166160940b98b9433cbe80811dfb71f98434
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0db9e3e069795d1965bbe72c49e2c7d9c8efce96087b099045dfb2308d21c55697bed09c6e6758aaab37d7ae267d1300970203df78f78db5a753f756b40717b9
|
7
|
+
data.tar.gz: 6006bd9033b7d82aa7a720eb8c330f852487e03de9f2816e18795704df69cedfe362fe210741059fe83ca26bd5e61bf83a5499cb761bc154233e11f633cc1516
|
data/Gemfile.lock
CHANGED
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache 2.0'
|
13
|
-
gem.version = '0.4.
|
13
|
+
gem.version = '0.4.8'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
|
@@ -19,6 +19,7 @@ module Fluent
|
|
19
19
|
|
20
20
|
# Constants for service names.
|
21
21
|
APPENGINE_SERVICE = 'appengine.googleapis.com'
|
22
|
+
CLOUDFUNCTIONS_SERVICE = 'cloudfunctions.googleapis.com'
|
22
23
|
COMPUTE_SERVICE = 'compute.googleapis.com'
|
23
24
|
CONTAINER_SERVICE = 'container.googleapis.com'
|
24
25
|
DATAFLOW_SERVICE = 'dataflow.googleapis.com'
|
@@ -70,6 +71,12 @@ module Fluent
|
|
70
71
|
# component (Docker, Kubelet, etc.) logs from container logs.
|
71
72
|
config_param :detect_subservice, :bool, :default => true
|
72
73
|
|
74
|
+
# The regular expression to use on Kubernetes logs to extract some basic
|
75
|
+
# information about the log source. The regex must contain capture groups
|
76
|
+
# for pod_name, namespace_name, and container_name.
|
77
|
+
config_param :kubernetes_tag_regexp, :string, :default =>
|
78
|
+
'\.(?<pod_name>[^_]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$'
|
79
|
+
|
73
80
|
# label_map (specified as a JSON object) is an unordered set of fluent
|
74
81
|
# field names whose values are sent as labels rather than as part of the
|
75
82
|
# struct payload.
|
@@ -142,8 +149,19 @@ module Fluent
|
|
142
149
|
|
143
150
|
# TODO: Send instance tags as labels as well?
|
144
151
|
@common_labels = {}
|
145
|
-
|
146
|
-
|
152
|
+
|
153
|
+
@compiled_kubernetes_tag_regexp = nil
|
154
|
+
if @kubernetes_tag_regexp
|
155
|
+
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
|
156
|
+
end
|
157
|
+
|
158
|
+
@cloudfunctions_tag_regexp =
|
159
|
+
/\.(?<function_name>.+)-[^-]+_default_worker$/
|
160
|
+
@cloudfunctions_log_regexp = /^
|
161
|
+
(?:\[(?<severity>.)\])?
|
162
|
+
\[(?<timestamp>.{24})\]
|
163
|
+
(?:\[(?<execution_id>[^\]]+)\])?
|
164
|
+
[ ](?<text>.*)$/x
|
147
165
|
|
148
166
|
# set attributes from metadata (unless overriden by static config)
|
149
167
|
@vm_name = Socket.gethostname if @vm_name.nil?
|
@@ -190,12 +208,16 @@ module Fluent
|
|
190
208
|
# Default this to false; it is only overwritten if we detect Managed VM.
|
191
209
|
@running_on_managed_vm = false
|
192
210
|
|
211
|
+
# Default this to false; it is only overwritten if we detect Cloud
|
212
|
+
# Functions.
|
213
|
+
@running_cloudfunctions = false
|
214
|
+
|
193
215
|
# Set labels, etc. based on the config
|
194
216
|
case @platform
|
195
217
|
when Platform::GCE
|
196
218
|
@service_name = COMPUTE_SERVICE
|
197
219
|
if @detect_subservice
|
198
|
-
# Check for specialized GCE environments
|
220
|
+
# Check for specialized GCE environments.
|
199
221
|
# TODO: Add config options for these to allow for running outside GCE?
|
200
222
|
attributes = fetch_gce_metadata('instance/attributes/').split
|
201
223
|
# Do nothing, just don't populate other service's labels.
|
@@ -224,6 +246,7 @@ module Fluent
|
|
224
246
|
@kube_env = YAML.load(@raw_kube_env)
|
225
247
|
common_labels["#{CONTAINER_SERVICE}/cluster_name"] =
|
226
248
|
cluster_name_from_kube_env(@kube_env)
|
249
|
+
detect_cloudfunctions(attributes)
|
227
250
|
end
|
228
251
|
end
|
229
252
|
common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
|
@@ -273,11 +296,11 @@ module Fluent
|
|
273
296
|
'commonLabels' => @common_labels,
|
274
297
|
'entries' => []
|
275
298
|
}
|
276
|
-
if @service_name == CONTAINER_SERVICE
|
299
|
+
if @service_name == CONTAINER_SERVICE && @compiled_kubernetes_tag_regexp
|
277
300
|
# Container logs in Kubernetes are tagged based on where they came
|
278
301
|
# from, so we can extract useful metadata from the tag.
|
279
302
|
# Do this here to avoid having to repeat it for each record.
|
280
|
-
match_data = @
|
303
|
+
match_data = @compiled_kubernetes_tag_regexp.match(tag)
|
281
304
|
if match_data
|
282
305
|
labels = write_log_entries_request['commonLabels']
|
283
306
|
%w(namespace_name pod_name container_name).each do |field|
|
@@ -285,8 +308,30 @@ module Fluent
|
|
285
308
|
end
|
286
309
|
end
|
287
310
|
end
|
311
|
+
if @running_cloudfunctions
|
312
|
+
# If the current group of entries is coming from a Cloud Functions
|
313
|
+
# function, the function name can be extracted from the tag.
|
314
|
+
match_data = @cloudfunctions_tag_regexp.match(tag)
|
315
|
+
if match_data
|
316
|
+
# Service name is set to Cloud Functions only for logs actually
|
317
|
+
# coming from a function.
|
318
|
+
@service_name = CLOUDFUNCTIONS_SERVICE
|
319
|
+
labels = write_log_entries_request['commonLabels']
|
320
|
+
labels["#{CLOUDFUNCTIONS_SERVICE}/region"] = @gcf_region
|
321
|
+
labels["#{CLOUDFUNCTIONS_SERVICE}/function_name"] =
|
322
|
+
match_data['function_name']
|
323
|
+
else
|
324
|
+
# Other logs are considered as coming from the Container Engine
|
325
|
+
# service.
|
326
|
+
@service_name = CONTAINER_SERVICE
|
327
|
+
end
|
328
|
+
end
|
288
329
|
arr.each do |time, record|
|
289
330
|
next unless record.is_a?(Hash)
|
331
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
332
|
+
@cloudfunctions_log_match =
|
333
|
+
@cloudfunctions_log_regexp.match(record['log'])
|
334
|
+
end
|
290
335
|
if record.key?('timestamp') &&
|
291
336
|
record['timestamp'].is_a?(Hash) &&
|
292
337
|
record['timestamp'].key?('seconds') &&
|
@@ -312,6 +357,11 @@ module Fluent
|
|
312
357
|
@log.warn 'timeNanos is deprecated - please use ' \
|
313
358
|
'timestampSeconds and timestampNanos instead.'
|
314
359
|
end
|
360
|
+
elsif @service_name == CLOUDFUNCTIONS_SERVICE &&
|
361
|
+
@cloudfunctions_log_match
|
362
|
+
timestamp = DateTime.parse(@cloudfunctions_log_match['timestamp'])
|
363
|
+
ts_secs = timestamp.strftime('%s')
|
364
|
+
ts_nanos = timestamp.strftime('%N')
|
315
365
|
else
|
316
366
|
timestamp = Time.at(time)
|
317
367
|
ts_secs = timestamp.tv_sec
|
@@ -329,12 +379,8 @@ module Fluent
|
|
329
379
|
'labels' => {}
|
330
380
|
}
|
331
381
|
}
|
332
|
-
|
333
|
-
|
334
|
-
record.delete('severity')
|
335
|
-
else
|
336
|
-
entry['metadata']['severity'] = 'DEFAULT'
|
337
|
-
end
|
382
|
+
|
383
|
+
set_severity(record, entry)
|
338
384
|
|
339
385
|
# If the record has been annotated by the kubernetes_metadata_filter
|
340
386
|
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
@@ -352,14 +398,15 @@ module Fluent
|
|
352
398
|
end
|
353
399
|
end
|
354
400
|
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
entry['
|
359
|
-
|
360
|
-
entry['structPayload'] = record
|
401
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE &&
|
402
|
+
@cloudfunctions_log_match &&
|
403
|
+
@cloudfunctions_log_match['execution_id']
|
404
|
+
entry['metadata']['labels']['execution_id'] =
|
405
|
+
@cloudfunctions_log_match['execution_id']
|
361
406
|
end
|
362
407
|
|
408
|
+
set_payload(record, entry)
|
409
|
+
|
363
410
|
# Remove the labels metadata if we didn't populate it with anything.
|
364
411
|
if entry['metadata']['labels'].empty?
|
365
412
|
entry['metadata'].delete('labels')
|
@@ -370,10 +417,7 @@ module Fluent
|
|
370
417
|
# Don't send an empty request if we rejected all the entries.
|
371
418
|
next if write_log_entries_request['entries'].empty?
|
372
419
|
|
373
|
-
|
374
|
-
# and also escape the log name.
|
375
|
-
log_name = CGI.escape(
|
376
|
-
@running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag)
|
420
|
+
log_name = CGI.escape(log_name(tag))
|
377
421
|
url = 'https://logging.googleapis.com/v1beta3/projects/' \
|
378
422
|
"#{@project_id}/logs/#{log_name}/entries:write"
|
379
423
|
begin
|
@@ -483,6 +527,13 @@ module Fluent
|
|
483
527
|
end
|
484
528
|
end
|
485
529
|
|
530
|
+
def detect_cloudfunctions(attributes)
|
531
|
+
return unless attributes.include?('gcf_region')
|
532
|
+
# Cloud Functions detected
|
533
|
+
@running_cloudfunctions = true
|
534
|
+
@gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
|
535
|
+
end
|
536
|
+
|
486
537
|
def cluster_name_from_kube_env(kube_env)
|
487
538
|
return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
|
488
539
|
instance_prefix = kube_env['INSTANCE_PREFIX']
|
@@ -492,6 +543,28 @@ module Fluent
|
|
492
543
|
instance_prefix
|
493
544
|
end
|
494
545
|
|
546
|
+
def set_severity(record, entry)
|
547
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE
|
548
|
+
if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
|
549
|
+
entry['metadata']['severity'] =
|
550
|
+
parse_severity(@cloudfunctions_log_match['severity'])
|
551
|
+
elsif record.key?('stream') && record['stream'] == 'stdout'
|
552
|
+
entry['metadata']['severity'] = 'INFO'
|
553
|
+
record.delete('stream')
|
554
|
+
elsif record.key?('stream') && record['stream'] == 'stderr'
|
555
|
+
entry['metadata']['severity'] = 'ERROR'
|
556
|
+
record.delete('stream')
|
557
|
+
else
|
558
|
+
entry['metadata']['severity'] = 'DEFAULT'
|
559
|
+
end
|
560
|
+
elsif record.key?('severity')
|
561
|
+
entry['metadata']['severity'] = parse_severity(record['severity'])
|
562
|
+
record.delete('severity')
|
563
|
+
else
|
564
|
+
entry['metadata']['severity'] = 'DEFAULT'
|
565
|
+
end
|
566
|
+
end
|
567
|
+
|
495
568
|
# Values permitted by the API for 'severity' (which is an enum).
|
496
569
|
VALID_SEVERITIES = Set.new(
|
497
570
|
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
|
@@ -577,10 +650,33 @@ module Fluent
|
|
577
650
|
record.delete(field)
|
578
651
|
end
|
579
652
|
|
653
|
+
def set_payload(record, entry)
|
654
|
+
# Use textPayload if this is the Cloud Functions service and 'log' key is
|
655
|
+
# available, or if the only remainaing key is 'message'.
|
656
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
|
657
|
+
entry['textPayload'] = @cloudfunctions_log_match['text']
|
658
|
+
elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
659
|
+
entry['textPayload'] = record['log']
|
660
|
+
elsif record.size == 1 && record.key?('message')
|
661
|
+
entry['textPayload'] = record['message']
|
662
|
+
else
|
663
|
+
entry['structPayload'] = record
|
664
|
+
end
|
665
|
+
end
|
666
|
+
|
667
|
+
def log_name(tag)
|
668
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE
|
669
|
+
return 'cloud-functions'
|
670
|
+
else
|
671
|
+
# Add a prefix to VMEngines logs to prevent namespace collisions.
|
672
|
+
return @running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag
|
673
|
+
end
|
674
|
+
end
|
675
|
+
|
580
676
|
def init_api_client
|
581
677
|
@client = Google::APIClient.new(
|
582
678
|
application_name: 'Fluentd Google Cloud Logging plugin',
|
583
|
-
application_version: '0.4.
|
679
|
+
application_version: '0.4.8',
|
584
680
|
retries: 1)
|
585
681
|
|
586
682
|
if @auth_method == 'private_key'
|
@@ -65,9 +65,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
65
65
|
CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
|
66
66
|
CONTAINER_NAMESPACE_NAME = 'kube-system'
|
67
67
|
CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
|
68
|
-
CONTAINER_POD_NAME = 'redis-master-c0l82'
|
68
|
+
CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
|
69
69
|
CONTAINER_CONTAINER_NAME = 'redis'
|
70
70
|
|
71
|
+
# Cloud Functions specific labels
|
72
|
+
CLOUDFUNCTIONS_FUNCTION_NAME = 'function-1'
|
73
|
+
CLOUDFUNCTIONS_REGION = 'us-central1'
|
74
|
+
CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
|
75
|
+
CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
|
76
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
|
77
|
+
CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}-c0l82"
|
78
|
+
CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
|
79
|
+
|
71
80
|
# Parameters used for authentication
|
72
81
|
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
73
82
|
FAKE_AUTH_TOKEN = 'abc123'
|
@@ -135,6 +144,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
135
144
|
COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
|
136
145
|
APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
|
137
146
|
CONTAINER_SERVICE_NAME = 'container.googleapis.com'
|
147
|
+
CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
|
138
148
|
EC2_SERVICE_NAME = 'ec2.amazonaws.com'
|
139
149
|
|
140
150
|
COMPUTE_PARAMS = {
|
@@ -203,6 +213,55 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
203
213
|
}
|
204
214
|
}
|
205
215
|
|
216
|
+
CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
|
217
|
+
"#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
|
218
|
+
"#{CLOUDFUNCTIONS_CONTAINER_NAME}"
|
219
|
+
|
220
|
+
CLOUDFUNCTIONS_PARAMS = {
|
221
|
+
'service_name' => CLOUDFUNCTIONS_SERVICE_NAME,
|
222
|
+
'log_name' => 'cloud-functions',
|
223
|
+
'project_id' => PROJECT_ID,
|
224
|
+
'zone' => ZONE,
|
225
|
+
'labels' => {
|
226
|
+
'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
|
227
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
|
228
|
+
CLOUDFUNCTIONS_FUNCTION_NAME,
|
229
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
|
230
|
+
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
231
|
+
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
|
232
|
+
"#{CONTAINER_SERVICE_NAME}/namespace_name" =>
|
233
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME,
|
234
|
+
"#{CONTAINER_SERVICE_NAME}/pod_name" => CLOUDFUNCTIONS_POD_NAME,
|
235
|
+
"#{CONTAINER_SERVICE_NAME}/container_name" =>
|
236
|
+
CLOUDFUNCTIONS_CONTAINER_NAME,
|
237
|
+
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
238
|
+
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
239
|
+
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
240
|
+
}
|
241
|
+
}
|
242
|
+
|
243
|
+
CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
|
244
|
+
'service_name' => CLOUDFUNCTIONS_SERVICE_NAME,
|
245
|
+
'log_name' => 'cloud-functions',
|
246
|
+
'project_id' => PROJECT_ID,
|
247
|
+
'zone' => ZONE,
|
248
|
+
'labels' => {
|
249
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
|
250
|
+
CLOUDFUNCTIONS_FUNCTION_NAME,
|
251
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
|
252
|
+
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
253
|
+
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
|
254
|
+
"#{CONTAINER_SERVICE_NAME}/namespace_name" =>
|
255
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME,
|
256
|
+
"#{CONTAINER_SERVICE_NAME}/pod_name" => CLOUDFUNCTIONS_POD_NAME,
|
257
|
+
"#{CONTAINER_SERVICE_NAME}/container_name" =>
|
258
|
+
CLOUDFUNCTIONS_CONTAINER_NAME,
|
259
|
+
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
260
|
+
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
261
|
+
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
262
|
+
}
|
263
|
+
}
|
264
|
+
|
206
265
|
CUSTOM_PARAMS = {
|
207
266
|
'service_name' => COMPUTE_SERVICE_NAME,
|
208
267
|
'log_name' => 'test',
|
@@ -839,6 +898,105 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
839
898
|
end
|
840
899
|
end
|
841
900
|
|
901
|
+
def test_one_cloudfunctions_log
|
902
|
+
setup_gce_metadata_stubs
|
903
|
+
setup_cloudfunctions_metadata_stubs
|
904
|
+
setup_logging_stubs
|
905
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
906
|
+
d.emit(cloudfunctions_log_entry(0))
|
907
|
+
d.run
|
908
|
+
verify_log_entries(1, CLOUDFUNCTIONS_PARAMS) do |entry|
|
909
|
+
assert_equal 'DEBUG', entry['metadata']['severity'], entry
|
910
|
+
end
|
911
|
+
end
|
912
|
+
|
913
|
+
def test_multiple_cloudfunctions_logs
|
914
|
+
setup_gce_metadata_stubs
|
915
|
+
setup_cloudfunctions_metadata_stubs
|
916
|
+
setup_logging_stubs
|
917
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
918
|
+
[2, 3, 5, 11, 50].each do |n|
|
919
|
+
# The test driver doesn't clear its buffer of entries after running, so
|
920
|
+
# do it manually here.
|
921
|
+
d.instance_variable_get('@entries').clear
|
922
|
+
@logs_sent = []
|
923
|
+
n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
|
924
|
+
d.run
|
925
|
+
verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
|
926
|
+
assert_equal 'DEBUG', entry['metadata']['severity'], entry
|
927
|
+
end
|
928
|
+
end
|
929
|
+
end
|
930
|
+
|
931
|
+
def test_one_cloudfunctions_log_text_not_matched
|
932
|
+
setup_gce_metadata_stubs
|
933
|
+
setup_cloudfunctions_metadata_stubs
|
934
|
+
setup_logging_stubs
|
935
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
936
|
+
d.emit(cloudfunctions_log_entry_text_not_matched(0))
|
937
|
+
d.run
|
938
|
+
verify_log_entries(1, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
|
939
|
+
assert_equal 'INFO', entry['metadata']['severity'], entry
|
940
|
+
end
|
941
|
+
end
|
942
|
+
|
943
|
+
def test_multiple_cloudfunctions_logs_text_not_matched
|
944
|
+
setup_gce_metadata_stubs
|
945
|
+
setup_cloudfunctions_metadata_stubs
|
946
|
+
setup_logging_stubs
|
947
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
948
|
+
[2, 3, 5, 11, 50].each do |n|
|
949
|
+
# The test driver doesn't clear its buffer of entries after running, so
|
950
|
+
# do it manually here.
|
951
|
+
d.instance_variable_get('@entries').clear
|
952
|
+
@logs_sent = []
|
953
|
+
n.times { |i| d.emit(cloudfunctions_log_entry_text_not_matched(i)) }
|
954
|
+
d.run
|
955
|
+
verify_log_entries(n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
|
956
|
+
assert_equal 'INFO', entry['metadata']['severity'], entry
|
957
|
+
end
|
958
|
+
end
|
959
|
+
end
|
960
|
+
|
961
|
+
def test_one_cloudfunctions_log_tag_not_matched
|
962
|
+
setup_gce_metadata_stubs
|
963
|
+
setup_cloudfunctions_metadata_stubs
|
964
|
+
setup_logging_stubs
|
965
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
|
966
|
+
d.emit(cloudfunctions_log_entry(0))
|
967
|
+
d.run
|
968
|
+
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS, 'structPayload') do |entry|
|
969
|
+
assert_equal 2, entry['structPayload'].size, entry
|
970
|
+
assert_equal '[D][2015-09-25T12:34:56.789Z][123-0] test log entry 0',
|
971
|
+
entry['structPayload']['log'], entry
|
972
|
+
assert_equal 'stdout', entry['structPayload']['stream'], entry
|
973
|
+
end
|
974
|
+
end
|
975
|
+
|
976
|
+
def test_multiple_cloudfunctions_logs_tag_not_matched
|
977
|
+
setup_gce_metadata_stubs
|
978
|
+
setup_cloudfunctions_metadata_stubs
|
979
|
+
setup_logging_stubs
|
980
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
|
981
|
+
[2, 3, 5, 11, 50].each do |n|
|
982
|
+
# The test driver doesn't clear its buffer of entries after running, so
|
983
|
+
# do it manually here.
|
984
|
+
d.instance_variable_get('@entries').clear
|
985
|
+
@logs_sent = []
|
986
|
+
n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
|
987
|
+
d.run
|
988
|
+
i = 0
|
989
|
+
params = CONTAINER_FROM_TAG_PARAMS
|
990
|
+
verify_log_entries(n, params, 'structPayload') do |entry|
|
991
|
+
assert_equal 2, entry['structPayload'].size, entry
|
992
|
+
assert_equal "[D][2015-09-25T12:34:56.789Z][123-0] test log entry #{i}",
|
993
|
+
entry['structPayload']['log'], entry
|
994
|
+
assert_equal 'stdout', entry['structPayload']['stream'], entry
|
995
|
+
i += 1
|
996
|
+
end
|
997
|
+
end
|
998
|
+
end
|
999
|
+
|
842
1000
|
# Make parse_severity public so we can test it.
|
843
1001
|
class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
|
844
1002
|
public :parse_severity
|
@@ -972,7 +1130,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
972
1130
|
|
973
1131
|
def setup_logging_stubs
|
974
1132
|
[COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
|
975
|
-
CONTAINER_FROM_METADATA_PARAMS, CUSTOM_PARAMS,
|
1133
|
+
CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS, CUSTOM_PARAMS,
|
1134
|
+
EC2_PARAMS].each do |params|
|
976
1135
|
stub_request(:post, uri_for_log(params)).to_return do |request|
|
977
1136
|
@logs_sent << JSON.parse(request.body)
|
978
1137
|
{ body: '' }
|
@@ -1024,6 +1183,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1024
1183
|
'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
|
1025
1184
|
end
|
1026
1185
|
|
1186
|
+
def setup_cloudfunctions_metadata_stubs
|
1187
|
+
stub_metadata_request(
|
1188
|
+
'instance/attributes/',
|
1189
|
+
"attribute1\nkube-env\ngcf_region\nlast_attribute")
|
1190
|
+
stub_metadata_request('instance/attributes/kube-env',
|
1191
|
+
"ENABLE_NODE_LOGGING: \"true\"\n"\
|
1192
|
+
"INSTANCE_PREFIX: gke-cluster-1-740fdafa\n"\
|
1193
|
+
'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
|
1194
|
+
stub_metadata_request('instance/attributes/gcf_region', 'us-central1')
|
1195
|
+
end
|
1196
|
+
|
1027
1197
|
def container_log_entry_with_metadata(i)
|
1028
1198
|
{
|
1029
1199
|
message: log_entry(i),
|
@@ -1037,6 +1207,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1037
1207
|
}
|
1038
1208
|
end
|
1039
1209
|
|
1210
|
+
def cloudfunctions_log_entry(i)
|
1211
|
+
{
|
1212
|
+
stream: 'stdout',
|
1213
|
+
log: '[D][2015-09-25T12:34:56.789Z][123-0] ' + log_entry(i)
|
1214
|
+
}
|
1215
|
+
end
|
1216
|
+
|
1217
|
+
def cloudfunctions_log_entry_text_not_matched(i)
|
1218
|
+
{
|
1219
|
+
stream: 'stdout',
|
1220
|
+
log: log_entry(i)
|
1221
|
+
}
|
1222
|
+
end
|
1223
|
+
|
1040
1224
|
def log_entry(i)
|
1041
1225
|
'test log entry ' + i.to_s
|
1042
1226
|
end
|
@@ -1049,7 +1233,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1049
1233
|
assert value.is_a?(String), "Value #{value} for label #{key} " \
|
1050
1234
|
'is not a string: ' + value.class.name
|
1051
1235
|
assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
|
1052
|
-
assert_equal
|
1236
|
+
assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
|
1053
1237
|
"#{expected_labels[key]} in #{key} => #{value}"
|
1054
1238
|
end
|
1055
1239
|
assert_equal expected_labels.length, all_labels.length, 'Expected ' \
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Derr
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-09-
|
12
|
+
date: 2015-09-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|