fluent-plugin-google-cloud 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/out_google_cloud.rb +118 -22
- data/test/plugin/test_out_google_cloud.rb +187 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e6d35bf9167d8e4847b5e54cc16e0f78dc41310d
|
4
|
+
data.tar.gz: 3481166160940b98b9433cbe80811dfb71f98434
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0db9e3e069795d1965bbe72c49e2c7d9c8efce96087b099045dfb2308d21c55697bed09c6e6758aaab37d7ae267d1300970203df78f78db5a753f756b40717b9
|
7
|
+
data.tar.gz: 6006bd9033b7d82aa7a720eb8c330f852487e03de9f2816e18795704df69cedfe362fe210741059fe83ca26bd5e61bf83a5499cb761bc154233e11f633cc1516
|
data/Gemfile.lock
CHANGED
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache 2.0'
|
13
|
-
gem.version = '0.4.
|
13
|
+
gem.version = '0.4.8'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
|
@@ -19,6 +19,7 @@ module Fluent
|
|
19
19
|
|
20
20
|
# Constants for service names.
|
21
21
|
APPENGINE_SERVICE = 'appengine.googleapis.com'
|
22
|
+
CLOUDFUNCTIONS_SERVICE = 'cloudfunctions.googleapis.com'
|
22
23
|
COMPUTE_SERVICE = 'compute.googleapis.com'
|
23
24
|
CONTAINER_SERVICE = 'container.googleapis.com'
|
24
25
|
DATAFLOW_SERVICE = 'dataflow.googleapis.com'
|
@@ -70,6 +71,12 @@ module Fluent
|
|
70
71
|
# component (Docker, Kubelet, etc.) logs from container logs.
|
71
72
|
config_param :detect_subservice, :bool, :default => true
|
72
73
|
|
74
|
+
# The regular expression to use on Kubernetes logs to extract some basic
|
75
|
+
# information about the log source. The regex must contain capture groups
|
76
|
+
# for pod_name, namespace_name, and container_name.
|
77
|
+
config_param :kubernetes_tag_regexp, :string, :default =>
|
78
|
+
'\.(?<pod_name>[^_]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$'
|
79
|
+
|
73
80
|
# label_map (specified as a JSON object) is an unordered set of fluent
|
74
81
|
# field names whose values are sent as labels rather than as part of the
|
75
82
|
# struct payload.
|
@@ -142,8 +149,19 @@ module Fluent
|
|
142
149
|
|
143
150
|
# TODO: Send instance tags as labels as well?
|
144
151
|
@common_labels = {}
|
145
|
-
|
146
|
-
|
152
|
+
|
153
|
+
@compiled_kubernetes_tag_regexp = nil
|
154
|
+
if @kubernetes_tag_regexp
|
155
|
+
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
|
156
|
+
end
|
157
|
+
|
158
|
+
@cloudfunctions_tag_regexp =
|
159
|
+
/\.(?<function_name>.+)-[^-]+_default_worker$/
|
160
|
+
@cloudfunctions_log_regexp = /^
|
161
|
+
(?:\[(?<severity>.)\])?
|
162
|
+
\[(?<timestamp>.{24})\]
|
163
|
+
(?:\[(?<execution_id>[^\]]+)\])?
|
164
|
+
[ ](?<text>.*)$/x
|
147
165
|
|
148
166
|
# set attributes from metadata (unless overriden by static config)
|
149
167
|
@vm_name = Socket.gethostname if @vm_name.nil?
|
@@ -190,12 +208,16 @@ module Fluent
|
|
190
208
|
# Default this to false; it is only overwritten if we detect Managed VM.
|
191
209
|
@running_on_managed_vm = false
|
192
210
|
|
211
|
+
# Default this to false; it is only overwritten if we detect Cloud
|
212
|
+
# Functions.
|
213
|
+
@running_cloudfunctions = false
|
214
|
+
|
193
215
|
# Set labels, etc. based on the config
|
194
216
|
case @platform
|
195
217
|
when Platform::GCE
|
196
218
|
@service_name = COMPUTE_SERVICE
|
197
219
|
if @detect_subservice
|
198
|
-
# Check for specialized GCE environments
|
220
|
+
# Check for specialized GCE environments.
|
199
221
|
# TODO: Add config options for these to allow for running outside GCE?
|
200
222
|
attributes = fetch_gce_metadata('instance/attributes/').split
|
201
223
|
# Do nothing, just don't populate other service's labels.
|
@@ -224,6 +246,7 @@ module Fluent
|
|
224
246
|
@kube_env = YAML.load(@raw_kube_env)
|
225
247
|
common_labels["#{CONTAINER_SERVICE}/cluster_name"] =
|
226
248
|
cluster_name_from_kube_env(@kube_env)
|
249
|
+
detect_cloudfunctions(attributes)
|
227
250
|
end
|
228
251
|
end
|
229
252
|
common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
|
@@ -273,11 +296,11 @@ module Fluent
|
|
273
296
|
'commonLabels' => @common_labels,
|
274
297
|
'entries' => []
|
275
298
|
}
|
276
|
-
if @service_name == CONTAINER_SERVICE
|
299
|
+
if @service_name == CONTAINER_SERVICE && @compiled_kubernetes_tag_regexp
|
277
300
|
# Container logs in Kubernetes are tagged based on where they came
|
278
301
|
# from, so we can extract useful metadata from the tag.
|
279
302
|
# Do this here to avoid having to repeat it for each record.
|
280
|
-
match_data = @
|
303
|
+
match_data = @compiled_kubernetes_tag_regexp.match(tag)
|
281
304
|
if match_data
|
282
305
|
labels = write_log_entries_request['commonLabels']
|
283
306
|
%w(namespace_name pod_name container_name).each do |field|
|
@@ -285,8 +308,30 @@ module Fluent
|
|
285
308
|
end
|
286
309
|
end
|
287
310
|
end
|
311
|
+
if @running_cloudfunctions
|
312
|
+
# If the current group of entries is coming from a Cloud Functions
|
313
|
+
# function, the function name can be extracted from the tag.
|
314
|
+
match_data = @cloudfunctions_tag_regexp.match(tag)
|
315
|
+
if match_data
|
316
|
+
# Service name is set to Cloud Functions only for logs actually
|
317
|
+
# coming from a function.
|
318
|
+
@service_name = CLOUDFUNCTIONS_SERVICE
|
319
|
+
labels = write_log_entries_request['commonLabels']
|
320
|
+
labels["#{CLOUDFUNCTIONS_SERVICE}/region"] = @gcf_region
|
321
|
+
labels["#{CLOUDFUNCTIONS_SERVICE}/function_name"] =
|
322
|
+
match_data['function_name']
|
323
|
+
else
|
324
|
+
# Other logs are considered as coming from the Container Engine
|
325
|
+
# service.
|
326
|
+
@service_name = CONTAINER_SERVICE
|
327
|
+
end
|
328
|
+
end
|
288
329
|
arr.each do |time, record|
|
289
330
|
next unless record.is_a?(Hash)
|
331
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
332
|
+
@cloudfunctions_log_match =
|
333
|
+
@cloudfunctions_log_regexp.match(record['log'])
|
334
|
+
end
|
290
335
|
if record.key?('timestamp') &&
|
291
336
|
record['timestamp'].is_a?(Hash) &&
|
292
337
|
record['timestamp'].key?('seconds') &&
|
@@ -312,6 +357,11 @@ module Fluent
|
|
312
357
|
@log.warn 'timeNanos is deprecated - please use ' \
|
313
358
|
'timestampSeconds and timestampNanos instead.'
|
314
359
|
end
|
360
|
+
elsif @service_name == CLOUDFUNCTIONS_SERVICE &&
|
361
|
+
@cloudfunctions_log_match
|
362
|
+
timestamp = DateTime.parse(@cloudfunctions_log_match['timestamp'])
|
363
|
+
ts_secs = timestamp.strftime('%s')
|
364
|
+
ts_nanos = timestamp.strftime('%N')
|
315
365
|
else
|
316
366
|
timestamp = Time.at(time)
|
317
367
|
ts_secs = timestamp.tv_sec
|
@@ -329,12 +379,8 @@ module Fluent
|
|
329
379
|
'labels' => {}
|
330
380
|
}
|
331
381
|
}
|
332
|
-
|
333
|
-
|
334
|
-
record.delete('severity')
|
335
|
-
else
|
336
|
-
entry['metadata']['severity'] = 'DEFAULT'
|
337
|
-
end
|
382
|
+
|
383
|
+
set_severity(record, entry)
|
338
384
|
|
339
385
|
# If the record has been annotated by the kubernetes_metadata_filter
|
340
386
|
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
@@ -352,14 +398,15 @@ module Fluent
|
|
352
398
|
end
|
353
399
|
end
|
354
400
|
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
entry['
|
359
|
-
|
360
|
-
entry['structPayload'] = record
|
401
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE &&
|
402
|
+
@cloudfunctions_log_match &&
|
403
|
+
@cloudfunctions_log_match['execution_id']
|
404
|
+
entry['metadata']['labels']['execution_id'] =
|
405
|
+
@cloudfunctions_log_match['execution_id']
|
361
406
|
end
|
362
407
|
|
408
|
+
set_payload(record, entry)
|
409
|
+
|
363
410
|
# Remove the labels metadata if we didn't populate it with anything.
|
364
411
|
if entry['metadata']['labels'].empty?
|
365
412
|
entry['metadata'].delete('labels')
|
@@ -370,10 +417,7 @@ module Fluent
|
|
370
417
|
# Don't send an empty request if we rejected all the entries.
|
371
418
|
next if write_log_entries_request['entries'].empty?
|
372
419
|
|
373
|
-
|
374
|
-
# and also escape the log name.
|
375
|
-
log_name = CGI.escape(
|
376
|
-
@running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag)
|
420
|
+
log_name = CGI.escape(log_name(tag))
|
377
421
|
url = 'https://logging.googleapis.com/v1beta3/projects/' \
|
378
422
|
"#{@project_id}/logs/#{log_name}/entries:write"
|
379
423
|
begin
|
@@ -483,6 +527,13 @@ module Fluent
|
|
483
527
|
end
|
484
528
|
end
|
485
529
|
|
530
|
+
def detect_cloudfunctions(attributes)
|
531
|
+
return unless attributes.include?('gcf_region')
|
532
|
+
# Cloud Functions detected
|
533
|
+
@running_cloudfunctions = true
|
534
|
+
@gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
|
535
|
+
end
|
536
|
+
|
486
537
|
def cluster_name_from_kube_env(kube_env)
|
487
538
|
return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
|
488
539
|
instance_prefix = kube_env['INSTANCE_PREFIX']
|
@@ -492,6 +543,28 @@ module Fluent
|
|
492
543
|
instance_prefix
|
493
544
|
end
|
494
545
|
|
546
|
+
def set_severity(record, entry)
|
547
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE
|
548
|
+
if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
|
549
|
+
entry['metadata']['severity'] =
|
550
|
+
parse_severity(@cloudfunctions_log_match['severity'])
|
551
|
+
elsif record.key?('stream') && record['stream'] == 'stdout'
|
552
|
+
entry['metadata']['severity'] = 'INFO'
|
553
|
+
record.delete('stream')
|
554
|
+
elsif record.key?('stream') && record['stream'] == 'stderr'
|
555
|
+
entry['metadata']['severity'] = 'ERROR'
|
556
|
+
record.delete('stream')
|
557
|
+
else
|
558
|
+
entry['metadata']['severity'] = 'DEFAULT'
|
559
|
+
end
|
560
|
+
elsif record.key?('severity')
|
561
|
+
entry['metadata']['severity'] = parse_severity(record['severity'])
|
562
|
+
record.delete('severity')
|
563
|
+
else
|
564
|
+
entry['metadata']['severity'] = 'DEFAULT'
|
565
|
+
end
|
566
|
+
end
|
567
|
+
|
495
568
|
# Values permitted by the API for 'severity' (which is an enum).
|
496
569
|
VALID_SEVERITIES = Set.new(
|
497
570
|
%w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
|
@@ -577,10 +650,33 @@ module Fluent
|
|
577
650
|
record.delete(field)
|
578
651
|
end
|
579
652
|
|
653
|
+
def set_payload(record, entry)
|
654
|
+
# Use textPayload if this is the Cloud Functions service and 'log' key is
|
655
|
+
# available, or if the only remainaing key is 'message'.
|
656
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
|
657
|
+
entry['textPayload'] = @cloudfunctions_log_match['text']
|
658
|
+
elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
659
|
+
entry['textPayload'] = record['log']
|
660
|
+
elsif record.size == 1 && record.key?('message')
|
661
|
+
entry['textPayload'] = record['message']
|
662
|
+
else
|
663
|
+
entry['structPayload'] = record
|
664
|
+
end
|
665
|
+
end
|
666
|
+
|
667
|
+
def log_name(tag)
|
668
|
+
if @service_name == CLOUDFUNCTIONS_SERVICE
|
669
|
+
return 'cloud-functions'
|
670
|
+
else
|
671
|
+
# Add a prefix to VMEngines logs to prevent namespace collisions.
|
672
|
+
return @running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag
|
673
|
+
end
|
674
|
+
end
|
675
|
+
|
580
676
|
def init_api_client
|
581
677
|
@client = Google::APIClient.new(
|
582
678
|
application_name: 'Fluentd Google Cloud Logging plugin',
|
583
|
-
application_version: '0.4.
|
679
|
+
application_version: '0.4.8',
|
584
680
|
retries: 1)
|
585
681
|
|
586
682
|
if @auth_method == 'private_key'
|
@@ -65,9 +65,18 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
65
65
|
CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
|
66
66
|
CONTAINER_NAMESPACE_NAME = 'kube-system'
|
67
67
|
CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
|
68
|
-
CONTAINER_POD_NAME = 'redis-master-c0l82'
|
68
|
+
CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
|
69
69
|
CONTAINER_CONTAINER_NAME = 'redis'
|
70
70
|
|
71
|
+
# Cloud Functions specific labels
|
72
|
+
CLOUDFUNCTIONS_FUNCTION_NAME = 'function-1'
|
73
|
+
CLOUDFUNCTIONS_REGION = 'us-central1'
|
74
|
+
CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
|
75
|
+
CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
|
76
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
|
77
|
+
CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}-c0l82"
|
78
|
+
CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
|
79
|
+
|
71
80
|
# Parameters used for authentication
|
72
81
|
AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
73
82
|
FAKE_AUTH_TOKEN = 'abc123'
|
@@ -135,6 +144,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
135
144
|
COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
|
136
145
|
APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
|
137
146
|
CONTAINER_SERVICE_NAME = 'container.googleapis.com'
|
147
|
+
CLOUDFUNCTIONS_SERVICE_NAME = 'cloudfunctions.googleapis.com'
|
138
148
|
EC2_SERVICE_NAME = 'ec2.amazonaws.com'
|
139
149
|
|
140
150
|
COMPUTE_PARAMS = {
|
@@ -203,6 +213,55 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
203
213
|
}
|
204
214
|
}
|
205
215
|
|
216
|
+
CLOUDFUNCTIONS_TAG = "kubernetes.#{CLOUDFUNCTIONS_POD_NAME}_" \
|
217
|
+
"#{CLOUDFUNCTIONS_NAMESPACE_NAME}_" \
|
218
|
+
"#{CLOUDFUNCTIONS_CONTAINER_NAME}"
|
219
|
+
|
220
|
+
CLOUDFUNCTIONS_PARAMS = {
|
221
|
+
'service_name' => CLOUDFUNCTIONS_SERVICE_NAME,
|
222
|
+
'log_name' => 'cloud-functions',
|
223
|
+
'project_id' => PROJECT_ID,
|
224
|
+
'zone' => ZONE,
|
225
|
+
'labels' => {
|
226
|
+
'execution_id' => CLOUDFUNCTIONS_EXECUTION_ID,
|
227
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
|
228
|
+
CLOUDFUNCTIONS_FUNCTION_NAME,
|
229
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
|
230
|
+
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
231
|
+
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
|
232
|
+
"#{CONTAINER_SERVICE_NAME}/namespace_name" =>
|
233
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME,
|
234
|
+
"#{CONTAINER_SERVICE_NAME}/pod_name" => CLOUDFUNCTIONS_POD_NAME,
|
235
|
+
"#{CONTAINER_SERVICE_NAME}/container_name" =>
|
236
|
+
CLOUDFUNCTIONS_CONTAINER_NAME,
|
237
|
+
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
238
|
+
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
239
|
+
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
240
|
+
}
|
241
|
+
}
|
242
|
+
|
243
|
+
CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS = {
|
244
|
+
'service_name' => CLOUDFUNCTIONS_SERVICE_NAME,
|
245
|
+
'log_name' => 'cloud-functions',
|
246
|
+
'project_id' => PROJECT_ID,
|
247
|
+
'zone' => ZONE,
|
248
|
+
'labels' => {
|
249
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/function_name" =>
|
250
|
+
CLOUDFUNCTIONS_FUNCTION_NAME,
|
251
|
+
"#{CLOUDFUNCTIONS_SERVICE_NAME}/region" => CLOUDFUNCTIONS_REGION,
|
252
|
+
"#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
|
253
|
+
"#{CONTAINER_SERVICE_NAME}/cluster_name" => CLOUDFUNCTIONS_CLUSTER_NAME,
|
254
|
+
"#{CONTAINER_SERVICE_NAME}/namespace_name" =>
|
255
|
+
CLOUDFUNCTIONS_NAMESPACE_NAME,
|
256
|
+
"#{CONTAINER_SERVICE_NAME}/pod_name" => CLOUDFUNCTIONS_POD_NAME,
|
257
|
+
"#{CONTAINER_SERVICE_NAME}/container_name" =>
|
258
|
+
CLOUDFUNCTIONS_CONTAINER_NAME,
|
259
|
+
"#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
|
260
|
+
"#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
|
261
|
+
"#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
|
262
|
+
}
|
263
|
+
}
|
264
|
+
|
206
265
|
CUSTOM_PARAMS = {
|
207
266
|
'service_name' => COMPUTE_SERVICE_NAME,
|
208
267
|
'log_name' => 'test',
|
@@ -839,6 +898,105 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
839
898
|
end
|
840
899
|
end
|
841
900
|
|
901
|
+
def test_one_cloudfunctions_log
|
902
|
+
setup_gce_metadata_stubs
|
903
|
+
setup_cloudfunctions_metadata_stubs
|
904
|
+
setup_logging_stubs
|
905
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
906
|
+
d.emit(cloudfunctions_log_entry(0))
|
907
|
+
d.run
|
908
|
+
verify_log_entries(1, CLOUDFUNCTIONS_PARAMS) do |entry|
|
909
|
+
assert_equal 'DEBUG', entry['metadata']['severity'], entry
|
910
|
+
end
|
911
|
+
end
|
912
|
+
|
913
|
+
def test_multiple_cloudfunctions_logs
|
914
|
+
setup_gce_metadata_stubs
|
915
|
+
setup_cloudfunctions_metadata_stubs
|
916
|
+
setup_logging_stubs
|
917
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
918
|
+
[2, 3, 5, 11, 50].each do |n|
|
919
|
+
# The test driver doesn't clear its buffer of entries after running, so
|
920
|
+
# do it manually here.
|
921
|
+
d.instance_variable_get('@entries').clear
|
922
|
+
@logs_sent = []
|
923
|
+
n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
|
924
|
+
d.run
|
925
|
+
verify_log_entries(n, CLOUDFUNCTIONS_PARAMS) do |entry|
|
926
|
+
assert_equal 'DEBUG', entry['metadata']['severity'], entry
|
927
|
+
end
|
928
|
+
end
|
929
|
+
end
|
930
|
+
|
931
|
+
def test_one_cloudfunctions_log_text_not_matched
|
932
|
+
setup_gce_metadata_stubs
|
933
|
+
setup_cloudfunctions_metadata_stubs
|
934
|
+
setup_logging_stubs
|
935
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
936
|
+
d.emit(cloudfunctions_log_entry_text_not_matched(0))
|
937
|
+
d.run
|
938
|
+
verify_log_entries(1, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
|
939
|
+
assert_equal 'INFO', entry['metadata']['severity'], entry
|
940
|
+
end
|
941
|
+
end
|
942
|
+
|
943
|
+
def test_multiple_cloudfunctions_logs_text_not_matched
|
944
|
+
setup_gce_metadata_stubs
|
945
|
+
setup_cloudfunctions_metadata_stubs
|
946
|
+
setup_logging_stubs
|
947
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CLOUDFUNCTIONS_TAG)
|
948
|
+
[2, 3, 5, 11, 50].each do |n|
|
949
|
+
# The test driver doesn't clear its buffer of entries after running, so
|
950
|
+
# do it manually here.
|
951
|
+
d.instance_variable_get('@entries').clear
|
952
|
+
@logs_sent = []
|
953
|
+
n.times { |i| d.emit(cloudfunctions_log_entry_text_not_matched(i)) }
|
954
|
+
d.run
|
955
|
+
verify_log_entries(n, CLOUDFUNCTIONS_TEXT_NOT_MATCHED_PARAMS) do |entry|
|
956
|
+
assert_equal 'INFO', entry['metadata']['severity'], entry
|
957
|
+
end
|
958
|
+
end
|
959
|
+
end
|
960
|
+
|
961
|
+
def test_one_cloudfunctions_log_tag_not_matched
|
962
|
+
setup_gce_metadata_stubs
|
963
|
+
setup_cloudfunctions_metadata_stubs
|
964
|
+
setup_logging_stubs
|
965
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
|
966
|
+
d.emit(cloudfunctions_log_entry(0))
|
967
|
+
d.run
|
968
|
+
verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS, 'structPayload') do |entry|
|
969
|
+
assert_equal 2, entry['structPayload'].size, entry
|
970
|
+
assert_equal '[D][2015-09-25T12:34:56.789Z][123-0] test log entry 0',
|
971
|
+
entry['structPayload']['log'], entry
|
972
|
+
assert_equal 'stdout', entry['structPayload']['stream'], entry
|
973
|
+
end
|
974
|
+
end
|
975
|
+
|
976
|
+
def test_multiple_cloudfunctions_logs_tag_not_matched
|
977
|
+
setup_gce_metadata_stubs
|
978
|
+
setup_cloudfunctions_metadata_stubs
|
979
|
+
setup_logging_stubs
|
980
|
+
d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
|
981
|
+
[2, 3, 5, 11, 50].each do |n|
|
982
|
+
# The test driver doesn't clear its buffer of entries after running, so
|
983
|
+
# do it manually here.
|
984
|
+
d.instance_variable_get('@entries').clear
|
985
|
+
@logs_sent = []
|
986
|
+
n.times { |i| d.emit(cloudfunctions_log_entry(i)) }
|
987
|
+
d.run
|
988
|
+
i = 0
|
989
|
+
params = CONTAINER_FROM_TAG_PARAMS
|
990
|
+
verify_log_entries(n, params, 'structPayload') do |entry|
|
991
|
+
assert_equal 2, entry['structPayload'].size, entry
|
992
|
+
assert_equal "[D][2015-09-25T12:34:56.789Z][123-0] test log entry #{i}",
|
993
|
+
entry['structPayload']['log'], entry
|
994
|
+
assert_equal 'stdout', entry['structPayload']['stream'], entry
|
995
|
+
i += 1
|
996
|
+
end
|
997
|
+
end
|
998
|
+
end
|
999
|
+
|
842
1000
|
# Make parse_severity public so we can test it.
|
843
1001
|
class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
|
844
1002
|
public :parse_severity
|
@@ -972,7 +1130,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
972
1130
|
|
973
1131
|
def setup_logging_stubs
|
974
1132
|
[COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
|
975
|
-
CONTAINER_FROM_METADATA_PARAMS, CUSTOM_PARAMS,
|
1133
|
+
CONTAINER_FROM_METADATA_PARAMS, CLOUDFUNCTIONS_PARAMS, CUSTOM_PARAMS,
|
1134
|
+
EC2_PARAMS].each do |params|
|
976
1135
|
stub_request(:post, uri_for_log(params)).to_return do |request|
|
977
1136
|
@logs_sent << JSON.parse(request.body)
|
978
1137
|
{ body: '' }
|
@@ -1024,6 +1183,17 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1024
1183
|
'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
|
1025
1184
|
end
|
1026
1185
|
|
1186
|
+
def setup_cloudfunctions_metadata_stubs
|
1187
|
+
stub_metadata_request(
|
1188
|
+
'instance/attributes/',
|
1189
|
+
"attribute1\nkube-env\ngcf_region\nlast_attribute")
|
1190
|
+
stub_metadata_request('instance/attributes/kube-env',
|
1191
|
+
"ENABLE_NODE_LOGGING: \"true\"\n"\
|
1192
|
+
"INSTANCE_PREFIX: gke-cluster-1-740fdafa\n"\
|
1193
|
+
'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
|
1194
|
+
stub_metadata_request('instance/attributes/gcf_region', 'us-central1')
|
1195
|
+
end
|
1196
|
+
|
1027
1197
|
def container_log_entry_with_metadata(i)
|
1028
1198
|
{
|
1029
1199
|
message: log_entry(i),
|
@@ -1037,6 +1207,20 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1037
1207
|
}
|
1038
1208
|
end
|
1039
1209
|
|
1210
|
+
def cloudfunctions_log_entry(i)
|
1211
|
+
{
|
1212
|
+
stream: 'stdout',
|
1213
|
+
log: '[D][2015-09-25T12:34:56.789Z][123-0] ' + log_entry(i)
|
1214
|
+
}
|
1215
|
+
end
|
1216
|
+
|
1217
|
+
def cloudfunctions_log_entry_text_not_matched(i)
|
1218
|
+
{
|
1219
|
+
stream: 'stdout',
|
1220
|
+
log: log_entry(i)
|
1221
|
+
}
|
1222
|
+
end
|
1223
|
+
|
1040
1224
|
def log_entry(i)
|
1041
1225
|
'test log entry ' + i.to_s
|
1042
1226
|
end
|
@@ -1049,7 +1233,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
|
|
1049
1233
|
assert value.is_a?(String), "Value #{value} for label #{key} " \
|
1050
1234
|
'is not a string: ' + value.class.name
|
1051
1235
|
assert expected_labels.key?(key), "Unexpected label #{key} => #{value}"
|
1052
|
-
assert_equal
|
1236
|
+
assert_equal expected_labels[key], value, 'Value mismatch - expected ' \
|
1053
1237
|
"#{expected_labels[key]} in #{key} => #{value}"
|
1054
1238
|
end
|
1055
1239
|
assert_equal expected_labels.length, all_labels.length, 'Expected ' \
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-google-cloud
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Todd Derr
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-09-
|
12
|
+
date: 2015-09-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|