fluent-plugin-google-cloud 0.5.6 → 0.6.0.v2.alpha.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +20 -1
- data/fluent-plugin-google-cloud.gemspec +2 -1
- data/lib/fluent/plugin/out_google_cloud.rb +355 -205
- data/test/plugin/base_test.rb +301 -211
- data/test/plugin/test_out_google_cloud.rb +17 -33
- data/test/plugin/test_out_google_cloud_grpc.rb +26 -33
- metadata +18 -9
- data/lib/google/logging/type/http_request_pb.rb +0 -30
- data/lib/google/logging/type/log_severity_pb.rb +0 -26
- data/lib/google/logging/v1/log_entry_pb.rb +0 -52
- data/lib/google/logging/v1/logging_pb.rb +0 -84
- data/lib/google/logging/v1/logging_services_pb.rb +0 -150
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46c9cecb544f4eccc43900d70e03f2e8b50620e8
|
4
|
+
data.tar.gz: d31204662ee282da827325657026fdc426d974bb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a058638489313253b731770cad0f296090ca9d43aca84c2514f0f1916aee1a0f9c86cce8128239fd10e40a38501768d7e2b3913313433cf4dc32f8a870782f3
|
7
|
+
data.tar.gz: d227ba85e626fc49e62e9554e8bae5283acdc5fa778a3521b116d6dc3b27959b0467e02f12922ccbb39805c0f574c23cb53e4c6a166d689ffa71d6c684960dc7
|
data/Gemfile.lock
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-google-cloud (0.
|
4
|
+
fluent-plugin-google-cloud (0.6.0.v2.alpha.1)
|
5
5
|
fluentd (~> 0.10)
|
6
6
|
google-api-client (~> 0.9.0)
|
7
|
+
google-cloud-logging (~> 0.23.2)
|
7
8
|
googleapis-common-protos (~> 1.3)
|
8
9
|
googleauth (~> 0.4)
|
9
10
|
grpc (~> 1.0)
|
@@ -41,6 +42,21 @@ GEM
|
|
41
42
|
mime-types (>= 1.6)
|
42
43
|
representable (~> 2.3.0)
|
43
44
|
retriable (~> 2.0)
|
45
|
+
google-cloud-core (0.21.1)
|
46
|
+
googleauth (~> 0.5.1)
|
47
|
+
google-cloud-logging (0.23.2)
|
48
|
+
google-cloud-core (~> 0.21.1)
|
49
|
+
google-gax (~> 0.6.0)
|
50
|
+
google-protobuf (~> 3.0)
|
51
|
+
googleapis-common-protos (~> 1.3)
|
52
|
+
grpc (~> 1.0)
|
53
|
+
orderedhash (= 0.0.6)
|
54
|
+
stackdriver-core (~> 0.21.0)
|
55
|
+
google-gax (0.6.0)
|
56
|
+
googleapis-common-protos (~> 1.3.1)
|
57
|
+
googleauth (~> 0.5.1)
|
58
|
+
grpc (~> 1.0)
|
59
|
+
rly (~> 0.2.3)
|
44
60
|
google-protobuf (3.2.0)
|
45
61
|
googleapis-common-protos (1.3.4)
|
46
62
|
google-protobuf (~> 3.0)
|
@@ -76,6 +92,7 @@ GEM
|
|
76
92
|
msgpack (1.0.3)
|
77
93
|
multi_json (1.12.1)
|
78
94
|
multipart-post (2.0.0)
|
95
|
+
orderedhash (0.0.6)
|
79
96
|
os (0.9.6)
|
80
97
|
parser (2.4.0.0)
|
81
98
|
ast (~> 2.2)
|
@@ -87,6 +104,7 @@ GEM
|
|
87
104
|
representable (2.3.0)
|
88
105
|
uber (~> 0.0.7)
|
89
106
|
retriable (2.1.0)
|
107
|
+
rly (0.2.3)
|
90
108
|
rubocop (0.35.1)
|
91
109
|
astrolabe (~> 1.3)
|
92
110
|
parser (>= 2.2.3.0, < 3.0)
|
@@ -104,6 +122,7 @@ GEM
|
|
104
122
|
faraday (~> 0.9)
|
105
123
|
jwt (~> 1.5)
|
106
124
|
multi_json (~> 1.10)
|
125
|
+
stackdriver-core (0.21.0)
|
107
126
|
strptime (0.1.9)
|
108
127
|
test-unit (3.2.3)
|
109
128
|
power_assert
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.
|
13
|
+
gem.version = '0.6.0.v2.alpha.1'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -22,6 +22,7 @@ eos
|
|
22
22
|
gem.add_runtime_dependency 'fluentd', '~> 0.10'
|
23
23
|
gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
|
24
24
|
gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'
|
25
|
+
gem.add_runtime_dependency 'google-cloud-logging', '~> 0.23.2'
|
25
26
|
gem.add_runtime_dependency 'googleauth', '~> 0.4'
|
26
27
|
gem.add_runtime_dependency 'grpc', '~> 1.0'
|
27
28
|
gem.add_runtime_dependency 'json', '~> 1.8'
|
@@ -18,10 +18,10 @@ require 'socket'
|
|
18
18
|
require 'time'
|
19
19
|
require 'yaml'
|
20
20
|
require 'google/apis'
|
21
|
-
require 'google/apis/
|
22
|
-
require 'google/logging/
|
23
|
-
require 'google/logging/
|
24
|
-
require 'google/logging/
|
21
|
+
require 'google/apis/logging_v2beta1'
|
22
|
+
require 'google/logging/v2/logging_pb'
|
23
|
+
require 'google/logging/v2/logging_services_pb'
|
24
|
+
require 'google/logging/v2/log_entry_pb'
|
25
25
|
require 'googleauth'
|
26
26
|
|
27
27
|
module Google
|
@@ -36,17 +36,44 @@ end
|
|
36
36
|
module Fluent
|
37
37
|
# fluentd output plugin for the Stackdriver Logging API
|
38
38
|
class GoogleCloudOutput < BufferedOutput
|
39
|
+
# Constants for service names and resource types.
|
40
|
+
module Constants
|
41
|
+
APPENGINE_CONSTANTS = {
|
42
|
+
service: 'appengine.googleapis.com',
|
43
|
+
resource_type: 'gae_app'
|
44
|
+
}
|
45
|
+
CLOUDFUNCTIONS_CONSTANTS = {
|
46
|
+
service: 'cloudfunctions.googleapis.com',
|
47
|
+
resource_type: 'cloud_function'
|
48
|
+
}
|
49
|
+
COMPUTE_CONSTANTS = {
|
50
|
+
service: 'compute.googleapis.com',
|
51
|
+
resource_type: 'gce_instance'
|
52
|
+
}
|
53
|
+
CONTAINER_CONSTANTS = {
|
54
|
+
service: 'container.googleapis.com',
|
55
|
+
resource_type: 'container'
|
56
|
+
}
|
57
|
+
DATAFLOW_CONSTANTS = {
|
58
|
+
service: 'dataflow.googleapis.com',
|
59
|
+
resource_type: 'dataflow_step'
|
60
|
+
}
|
61
|
+
EC2_CONSTANTS = {
|
62
|
+
service: 'ec2.amazonaws.com',
|
63
|
+
resource_type: 'aws_ec2_instance'
|
64
|
+
}
|
65
|
+
ML_CONSTANTS = {
|
66
|
+
service: 'ml.googleapis.com',
|
67
|
+
resource_type: 'ml_job'
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
include self::Constants
|
72
|
+
|
39
73
|
Fluent::Plugin.register_output('google_cloud', self)
|
40
74
|
|
41
75
|
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
|
42
|
-
PLUGIN_VERSION = '0.
|
43
|
-
|
44
|
-
# Constants for service names.
|
45
|
-
APPENGINE_SERVICE = 'appengine.googleapis.com'
|
46
|
-
CLOUDFUNCTIONS_SERVICE = 'cloudfunctions.googleapis.com'
|
47
|
-
COMPUTE_SERVICE = 'compute.googleapis.com'
|
48
|
-
CONTAINER_SERVICE = 'container.googleapis.com'
|
49
|
-
EC2_SERVICE = 'ec2.amazonaws.com'
|
76
|
+
PLUGIN_VERSION = '0.6.0.v2.alpha.1'
|
50
77
|
|
51
78
|
# Name of the the Google cloud logging write scope.
|
52
79
|
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
|
@@ -167,7 +194,7 @@ module Fluent
|
|
167
194
|
attr_reader :running_on_managed_vm
|
168
195
|
attr_reader :gae_backend_name
|
169
196
|
attr_reader :gae_backend_version
|
170
|
-
attr_reader :
|
197
|
+
attr_reader :resource
|
171
198
|
attr_reader :common_labels
|
172
199
|
|
173
200
|
def initialize
|
@@ -198,6 +225,11 @@ module Fluent
|
|
198
225
|
@common_labels = {}
|
199
226
|
@common_labels.merge!(@labels) if @labels
|
200
227
|
|
228
|
+
# TODO: Construct Google::Api::MonitoredResource when @use_grpc is
|
229
|
+
# true after the protobuf map corruption issue is fixed.
|
230
|
+
@resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
231
|
+
labels: {})
|
232
|
+
|
201
233
|
@compiled_kubernetes_tag_regexp = nil
|
202
234
|
if @kubernetes_tag_regexp
|
203
235
|
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
|
@@ -235,7 +267,7 @@ module Fluent
|
|
235
267
|
@vm_id = metadata['instanceId']
|
236
268
|
end
|
237
269
|
if metadata.key?('accountId')
|
238
|
-
|
270
|
+
@resource.labels['aws_account'] = metadata['accountId']
|
239
271
|
end
|
240
272
|
when Platform::OTHER
|
241
273
|
# do nothing
|
@@ -268,12 +300,19 @@ module Fluent
|
|
268
300
|
# Functions.
|
269
301
|
@running_cloudfunctions = false
|
270
302
|
|
271
|
-
# Set labels, etc. based on the config
|
303
|
+
# Set up the MonitoredResource, labels, etc. based on the config.
|
272
304
|
case @platform
|
273
305
|
when Platform::GCE
|
274
|
-
@
|
306
|
+
@resource.type = COMPUTE_CONSTANTS[:resource_type]
|
307
|
+
# TODO: introduce a new MonitoredResource-centric configuration and
|
308
|
+
# deprecate subservice-name; for now, translate known uses.
|
275
309
|
if @subservice_name
|
276
|
-
|
310
|
+
# TODO: what should we do if we encounter an unknown value?
|
311
|
+
if @subservice_name == DATAFLOW_CONSTANTS[:service]
|
312
|
+
@resource.type = DATAFLOW_CONSTANTS[:resource_type]
|
313
|
+
elsif @subservice_name == ML_CONSTANTS[:service]
|
314
|
+
@resource.type = ML_CONSTANTS[:resource_type]
|
315
|
+
end
|
277
316
|
elsif @detect_subservice
|
278
317
|
# Check for specialized GCE environments.
|
279
318
|
# TODO: Add config options for these to allow for running outside GCE?
|
@@ -287,41 +326,56 @@ module Fluent
|
|
287
326
|
fetch_gce_metadata('instance/attributes/gae_backend_name')
|
288
327
|
@gae_backend_version =
|
289
328
|
fetch_gce_metadata('instance/attributes/gae_backend_version')
|
290
|
-
@
|
291
|
-
|
292
|
-
|
293
|
-
@gae_backend_version
|
329
|
+
@resource.type = APPENGINE_CONSTANTS[:resource_type]
|
330
|
+
@resource.labels['module_id'] = @gae_backend_name
|
331
|
+
@resource.labels['version_id'] = @gae_backend_version
|
294
332
|
elsif attributes.include?('kube-env')
|
295
333
|
# Kubernetes/Container Engine
|
296
|
-
@
|
297
|
-
common_labels["#{CONTAINER_SERVICE}/instance_id"] = @vm_id
|
334
|
+
@resource.type = CONTAINER_CONSTANTS[:resource_type]
|
298
335
|
@raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
|
299
336
|
@kube_env = YAML.load(@raw_kube_env)
|
300
|
-
|
337
|
+
@resource.labels['cluster_name'] =
|
301
338
|
cluster_name_from_kube_env(@kube_env)
|
302
339
|
detect_cloudfunctions(attributes)
|
303
340
|
end
|
304
341
|
end
|
305
|
-
|
306
|
-
|
307
|
-
|
342
|
+
# Some services have the GCE instance_id and zone as MonitoredResource
|
343
|
+
# labels; for other services we send them as entry labels.
|
344
|
+
if @resource.type == COMPUTE_CONSTANTS[:resource_type] ||
|
345
|
+
@resource.type == CONTAINER_CONSTANTS[:resource_type]
|
346
|
+
@resource.labels['instance_id'] = @vm_id
|
347
|
+
@resource.labels['zone'] = @zone
|
348
|
+
else
|
349
|
+
common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
|
350
|
+
common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
|
351
|
+
end
|
352
|
+
common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
|
308
353
|
when Platform::EC2
|
309
|
-
@
|
310
|
-
|
311
|
-
|
312
|
-
|
354
|
+
@resource.type = EC2_CONSTANTS[:resource_type]
|
355
|
+
@resource.labels['instance_id'] = @vm_id
|
356
|
+
@resource.labels['region'] = @zone
|
357
|
+
# the aws_account label is populated above.
|
358
|
+
common_labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
|
313
359
|
when Platform::OTHER
|
314
|
-
# Use
|
315
|
-
@
|
316
|
-
|
317
|
-
|
318
|
-
common_labels["#{
|
360
|
+
# Use GCE as the default environment.
|
361
|
+
@resource.type = COMPUTE_CONSTANTS[:resource_type]
|
362
|
+
@resource.labels['instance_id'] = @vm_id
|
363
|
+
@resource.labels['zone'] = @zone
|
364
|
+
common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
|
319
365
|
end
|
366
|
+
@resource.labels.merge!(
|
367
|
+
extract_resource_labels(@resource.type, common_labels))
|
368
|
+
|
369
|
+
# The resource and labels are now set up; ensure they can't be modified
|
370
|
+
# without first duping them.
|
371
|
+
@resource.freeze
|
372
|
+
@resource.labels.freeze
|
373
|
+
@common_labels.freeze
|
320
374
|
|
321
375
|
# Log an informational message containing the Logs viewer URL
|
322
|
-
@log.info 'Logs viewer address: ',
|
323
|
-
|
324
|
-
|
376
|
+
@log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
|
377
|
+
"viewer?project=#{@project_id}&resource=#{@resource_type}/",
|
378
|
+
"instance_id/#{@vm_id}"
|
325
379
|
end
|
326
380
|
|
327
381
|
def start
|
@@ -353,6 +407,111 @@ module Fluent
|
|
353
407
|
tag
|
354
408
|
end
|
355
409
|
|
410
|
+
# Compute the monitored resource and common labels shared by a collection of
|
411
|
+
# entries.
|
412
|
+
def compute_group_resource_and_labels(tag)
|
413
|
+
# Note that we assume that labels added to group_common_labels below are
|
414
|
+
# not 'service' labels (i.e. we do not call extract_resource_labels
|
415
|
+
# again).
|
416
|
+
group_resource = @resource.dup
|
417
|
+
group_common_labels = @common_labels.dup
|
418
|
+
|
419
|
+
if @running_cloudfunctions
|
420
|
+
# If the current group of entries is coming from a Cloud Functions
|
421
|
+
# function, the function name can be extracted from the tag.
|
422
|
+
match_data = @cloudfunctions_tag_regexp.match(tag)
|
423
|
+
if match_data
|
424
|
+
# Resource type is set to Cloud Functions only for logs actually
|
425
|
+
# coming from a function, otherwise we leave it as Container.
|
426
|
+
group_resource.type = CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
427
|
+
group_resource.labels['region'] = @gcf_region
|
428
|
+
group_resource.labels['function_name'] =
|
429
|
+
decode_cloudfunctions_function_name(
|
430
|
+
match_data['encoded_function_name'])
|
431
|
+
# Move GKE container labels from the MonitoredResource to the
|
432
|
+
# LogEntry.
|
433
|
+
instance_id = group_resource.labels.delete('instance_id')
|
434
|
+
group_common_labels["#{CONTAINER_CONSTANTS[:service]}/cluster_name"] =
|
435
|
+
group_resource.labels.delete('cluster_name')
|
436
|
+
group_common_labels["#{CONTAINER_CONSTANTS[:service]}/instance_id"] =
|
437
|
+
instance_id
|
438
|
+
group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
|
439
|
+
instance_id
|
440
|
+
group_common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] =
|
441
|
+
group_resource.labels.delete('zone')
|
442
|
+
end
|
443
|
+
end
|
444
|
+
if group_resource.type == CONTAINER_CONSTANTS[:resource_type] &&
|
445
|
+
@compiled_kubernetes_tag_regexp
|
446
|
+
# Container logs in Kubernetes are tagged based on where they came
|
447
|
+
# from, so we can extract useful metadata from the tag.
|
448
|
+
# Do this here to avoid having to repeat it for each record.
|
449
|
+
match_data = @compiled_kubernetes_tag_regexp.match(tag)
|
450
|
+
if match_data
|
451
|
+
group_resource.labels['container_name'] =
|
452
|
+
match_data['container_name']
|
453
|
+
%w(namespace_name pod_name).each do |field|
|
454
|
+
group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
|
455
|
+
match_data[field]
|
456
|
+
end
|
457
|
+
end
|
458
|
+
end
|
459
|
+
|
460
|
+
# Freeze the per-request state. Any further changes must be made on a
|
461
|
+
# per-entry basis.
|
462
|
+
group_resource.freeze
|
463
|
+
group_resource.labels.freeze
|
464
|
+
group_common_labels.freeze
|
465
|
+
|
466
|
+
[group_resource, group_common_labels]
|
467
|
+
end
|
468
|
+
|
469
|
+
# Extract entry resource and common labels that should be applied to
|
470
|
+
# individual entries from the group resource.
|
471
|
+
def extract_entry_labels(group_resource, record)
|
472
|
+
resource_labels = {}
|
473
|
+
common_labels = {}
|
474
|
+
|
475
|
+
if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
476
|
+
record.key?('log')
|
477
|
+
@cloudfunctions_log_match =
|
478
|
+
@cloudfunctions_log_regexp.match(record['log'])
|
479
|
+
end
|
480
|
+
|
481
|
+
if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
|
482
|
+
# Move the stdout/stderr annotation from the record into a label
|
483
|
+
common_labels.merge!(
|
484
|
+
fields_to_labels(
|
485
|
+
record, 'stream' => "#{CONTAINER_CONSTANTS[:service]}/stream"))
|
486
|
+
|
487
|
+
# If the record has been annotated by the kubernetes_metadata_filter
|
488
|
+
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
489
|
+
# populated at the grouped_entries level from the group's tag.
|
490
|
+
if record.key?('kubernetes')
|
491
|
+
extracted_resource_labels, extracted_common_labels = \
|
492
|
+
extract_container_metadata(record)
|
493
|
+
resource_labels.merge!(extracted_resource_labels)
|
494
|
+
common_labels.merge!(extracted_common_labels)
|
495
|
+
end
|
496
|
+
end
|
497
|
+
|
498
|
+
# If a field is present in the label_map, send its value as a label
|
499
|
+
# (mapping the field name to label name as specified in the config)
|
500
|
+
# and do not send that field as part of the payload.
|
501
|
+
common_labels.merge!(fields_to_labels(record, @label_map))
|
502
|
+
|
503
|
+
if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
504
|
+
@cloudfunctions_log_match &&
|
505
|
+
@cloudfunctions_log_match['execution_id']
|
506
|
+
common_labels['execution_id'] =
|
507
|
+
@cloudfunctions_log_match['execution_id']
|
508
|
+
end
|
509
|
+
resource_labels.merge!(
|
510
|
+
extract_resource_labels(group_resource.type, common_labels))
|
511
|
+
|
512
|
+
[resource_labels, common_labels]
|
513
|
+
end
|
514
|
+
|
356
515
|
def write(chunk)
|
357
516
|
# Group the entries since we have to make one call per tag.
|
358
517
|
grouped_entries = {}
|
@@ -369,74 +528,20 @@ module Fluent
|
|
369
528
|
|
370
529
|
grouped_entries.each do |tag, arr|
|
371
530
|
entries = []
|
372
|
-
|
373
|
-
|
374
|
-
if @running_cloudfunctions
|
375
|
-
# If the current group of entries is coming from a Cloud Functions
|
376
|
-
# function, the function name can be extracted from the tag.
|
377
|
-
match_data = @cloudfunctions_tag_regexp.match(tag)
|
378
|
-
if match_data
|
379
|
-
# Service name is set to Cloud Functions only for logs actually
|
380
|
-
# coming from a function.
|
381
|
-
@service_name = CLOUDFUNCTIONS_SERVICE
|
382
|
-
labels["#{CLOUDFUNCTIONS_SERVICE}/region"] = @gcf_region
|
383
|
-
labels["#{CLOUDFUNCTIONS_SERVICE}/function_name"] =
|
384
|
-
decode_cloudfunctions_function_name(
|
385
|
-
match_data['encoded_function_name'])
|
386
|
-
else
|
387
|
-
# Other logs are considered as coming from the Container Engine
|
388
|
-
# service.
|
389
|
-
@service_name = CONTAINER_SERVICE
|
390
|
-
end
|
391
|
-
end
|
392
|
-
if @service_name == CONTAINER_SERVICE && @compiled_kubernetes_tag_regexp
|
393
|
-
# Container logs in Kubernetes are tagged based on where they came
|
394
|
-
# from, so we can extract useful metadata from the tag.
|
395
|
-
# Do this here to avoid having to repeat it for each record.
|
396
|
-
match_data = @compiled_kubernetes_tag_regexp.match(tag)
|
397
|
-
if match_data
|
398
|
-
%w(namespace_name pod_name container_name).each do |field|
|
399
|
-
labels["#{CONTAINER_SERVICE}/#{field}"] = match_data[field]
|
400
|
-
end
|
401
|
-
end
|
402
|
-
end
|
531
|
+
group_resource, group_common_labels = compute_group_resource_and_labels(
|
532
|
+
tag)
|
403
533
|
|
404
534
|
arr.each do |time, record|
|
405
535
|
next unless record.is_a?(Hash)
|
406
536
|
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
labels: {}
|
414
|
-
))
|
415
|
-
else
|
416
|
-
entry = Google::Apis::LoggingV1beta3::LogEntry.new(
|
417
|
-
metadata: Google::Apis::LoggingV1beta3::LogEntryMetadata.new(
|
418
|
-
service_name: @service_name,
|
419
|
-
project_id: @project_id,
|
420
|
-
zone: @zone,
|
421
|
-
labels: {}
|
422
|
-
))
|
423
|
-
end
|
424
|
-
|
425
|
-
if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
|
426
|
-
@cloudfunctions_log_match =
|
427
|
-
@cloudfunctions_log_regexp.match(record['log'])
|
428
|
-
end
|
429
|
-
if @service_name == CONTAINER_SERVICE
|
430
|
-
# Move the stdout/stderr annotation from the record into a label.
|
431
|
-
field_to_label(record, 'stream', entry.metadata.labels,
|
432
|
-
"#{CONTAINER_SERVICE}/stream")
|
433
|
-
# If the record has been annotated by the kubernetes_metadata_filter
|
434
|
-
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
435
|
-
# populated at the grouped_entries level from the group's tag.
|
436
|
-
if record.key?('kubernetes')
|
437
|
-
handle_container_metadata(record, entry)
|
438
|
-
end
|
537
|
+
extracted_resource_labels, extracted_common_labels = \
|
538
|
+
extract_entry_labels(group_resource, record)
|
539
|
+
entry_resource = group_resource.dup
|
540
|
+
entry_resource.labels.merge!(extracted_resource_labels)
|
541
|
+
entry_common_labels = \
|
542
|
+
group_common_labels.merge(extracted_common_labels)
|
439
543
|
|
544
|
+
if entry_resource.type == CONTAINER_CONSTANTS[:resource_type]
|
440
545
|
# Save the timestamp if available, then clear it out to allow for
|
441
546
|
# determining whether we should parse the log or message field.
|
442
547
|
timestamp = record.key?('time') ? record['time'] : nil
|
@@ -460,57 +565,47 @@ module Fluent
|
|
460
565
|
end
|
461
566
|
end
|
462
567
|
|
463
|
-
ts_secs, ts_nanos = compute_timestamp(
|
568
|
+
ts_secs, ts_nanos = compute_timestamp(
|
569
|
+
entry_resource.type, record, time)
|
570
|
+
severity = compute_severity(
|
571
|
+
entry_resource.type, record, entry_common_labels)
|
572
|
+
|
464
573
|
if @use_grpc
|
574
|
+
entry = Google::Logging::V2::LogEntry.new(
|
575
|
+
labels: entry_common_labels,
|
576
|
+
resource: Google::Api::MonitoredResource.new(
|
577
|
+
type: entry_resource.type,
|
578
|
+
labels: entry_resource.labels.to_h
|
579
|
+
),
|
580
|
+
severity: grpc_severity(severity)
|
581
|
+
)
|
465
582
|
# If "seconds" is null or not an integer, we will omit the timestamp
|
466
583
|
# field and defer the decision on how to handle it to the downstream
|
467
584
|
# Logging API. If "nanos" is null or not an integer, it will be set
|
468
585
|
# to 0.
|
469
586
|
if ts_secs.is_a?(Integer)
|
470
587
|
ts_nanos = 0 unless ts_nanos.is_a?(Integer)
|
471
|
-
entry.
|
588
|
+
entry.timestamp = Google::Protobuf::Timestamp.new(
|
472
589
|
seconds: ts_secs,
|
473
590
|
nanos: ts_nanos
|
474
591
|
)
|
475
592
|
end
|
476
|
-
|
477
|
-
|
478
|
-
grpc_severity(compute_severity(record, entry))
|
479
|
-
|
480
|
-
set_http_request_grpc(record, entry) # FIXME
|
593
|
+
set_http_request_grpc(record, entry)
|
594
|
+
set_payload_grpc(entry_resource.type, record, entry, is_json)
|
481
595
|
else
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
596
|
+
# Remove the labels if we didn't populate them with anything.
|
597
|
+
entry_resource.labels = nil if entry_resource.labels.empty?
|
598
|
+
entry = Google::Apis::LoggingV2beta1::LogEntry.new(
|
599
|
+
labels: entry_common_labels,
|
600
|
+
resource: entry_resource,
|
601
|
+
severity: severity,
|
602
|
+
timestamp: {
|
603
|
+
seconds: ts_secs,
|
604
|
+
nanos: ts_nanos
|
605
|
+
}
|
606
|
+
)
|
490
607
|
set_http_request(record, entry)
|
491
|
-
|
492
|
-
|
493
|
-
# If a field is present in the label_map, send its value as a label
|
494
|
-
# (mapping the field name to label name as specified in the config)
|
495
|
-
# and do not send that field as part of the payload.
|
496
|
-
if @label_map
|
497
|
-
@label_map.each do |field, label|
|
498
|
-
field_to_label(record, field, entry.metadata.labels, label)
|
499
|
-
end
|
500
|
-
end
|
501
|
-
|
502
|
-
if @service_name == CLOUDFUNCTIONS_SERVICE &&
|
503
|
-
@cloudfunctions_log_match &&
|
504
|
-
@cloudfunctions_log_match['execution_id']
|
505
|
-
entry.metadata.labels['execution_id'] =
|
506
|
-
@cloudfunctions_log_match['execution_id']
|
507
|
-
end
|
508
|
-
|
509
|
-
if @use_grpc
|
510
|
-
set_payload_grpc(record, entry, is_json)
|
511
|
-
else
|
512
|
-
set_payload(record, entry, is_json)
|
513
|
-
entry.metadata.labels = nil if entry.metadata.labels.empty?
|
608
|
+
set_payload(entry_resource.type, record, entry, is_json)
|
514
609
|
end
|
515
610
|
|
516
611
|
entries.push(entry)
|
@@ -518,21 +613,24 @@ module Fluent
|
|
518
613
|
# Don't send an empty request if we rejected all the entries.
|
519
614
|
next if entries.empty?
|
520
615
|
|
521
|
-
log_name = log_name(
|
616
|
+
log_name = "projects/#{@project_id}/logs/#{log_name(
|
617
|
+
tag, group_resource)}"
|
522
618
|
|
619
|
+
# Does the actual write to the cloud logging api.
|
620
|
+
client = api_client
|
523
621
|
if @use_grpc
|
524
622
|
begin
|
525
|
-
|
526
|
-
|
527
|
-
client = api_client
|
528
|
-
|
529
|
-
labels_utf8_pairs = labels.map do |k, v|
|
623
|
+
labels_utf8_pairs = group_common_labels.map do |k, v|
|
530
624
|
[k.encode('utf-8'), convert_to_utf8(v)]
|
531
625
|
end
|
532
626
|
|
533
|
-
write_request = Google::Logging::
|
534
|
-
log_name:
|
535
|
-
|
627
|
+
write_request = Google::Logging::V2::WriteLogEntriesRequest.new(
|
628
|
+
log_name: log_name,
|
629
|
+
resource: Google::Api::MonitoredResource.new(
|
630
|
+
type: group_resource.type,
|
631
|
+
labels: group_resource.labels.to_h
|
632
|
+
),
|
633
|
+
labels: labels_utf8_pairs.to_h,
|
536
634
|
entries: entries
|
537
635
|
)
|
538
636
|
|
@@ -584,21 +682,15 @@ module Fluent
|
|
584
682
|
end
|
585
683
|
else
|
586
684
|
begin
|
587
|
-
# Does the actual write to the cloud logging api.
|
588
|
-
|
589
|
-
client = api_client
|
590
|
-
|
591
|
-
# The URI of the write is constructed by the Google::Api request;
|
592
|
-
# it is equivalent to this URL:
|
593
|
-
# 'https://logging.googleapis.com/v1beta3/projects/' \
|
594
|
-
# "#{@project_id}/logs/#{log_name}/entries:write"
|
595
685
|
write_request = \
|
596
|
-
Google::Apis::
|
597
|
-
|
686
|
+
Google::Apis::LoggingV2beta1::WriteLogEntriesRequest.new(
|
687
|
+
log_name: log_name,
|
688
|
+
resource: group_resource,
|
689
|
+
labels: group_common_labels,
|
598
690
|
entries: entries)
|
599
691
|
|
600
692
|
# TODO: RequestOptions
|
601
|
-
client.
|
693
|
+
client.write_entry_log_entries(write_request)
|
602
694
|
|
603
695
|
# Let the user explicitly know when the first call succeeded,
|
604
696
|
# to aid with verification and troubleshooting.
|
@@ -759,7 +851,7 @@ module Fluent
|
|
759
851
|
instance_prefix
|
760
852
|
end
|
761
853
|
|
762
|
-
def compute_timestamp(record, time)
|
854
|
+
def compute_timestamp(resource_type, record, time)
|
763
855
|
if record.key?('timestamp') &&
|
764
856
|
record['timestamp'].is_a?(Hash) &&
|
765
857
|
record['timestamp'].key?('seconds') &&
|
@@ -783,7 +875,7 @@ module Fluent
|
|
783
875
|
@log.warn 'timeNanos is deprecated - please use ' \
|
784
876
|
'timestampSeconds and timestampNanos instead.'
|
785
877
|
end
|
786
|
-
elsif
|
878
|
+
elsif resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
787
879
|
@cloudfunctions_log_match
|
788
880
|
timestamp = DateTime.parse(@cloudfunctions_log_match['timestamp'])
|
789
881
|
ts_secs = timestamp.strftime('%s').to_i
|
@@ -805,8 +897,8 @@ module Fluent
|
|
805
897
|
[ts_secs, ts_nanos]
|
806
898
|
end
|
807
899
|
|
808
|
-
def compute_severity(record,
|
809
|
-
if
|
900
|
+
def compute_severity(resource_type, record, entry_common_labels)
|
901
|
+
if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
810
902
|
if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
|
811
903
|
return parse_severity(@cloudfunctions_log_match['severity'])
|
812
904
|
elsif record.key?('stream') && record['stream'] == 'stdout'
|
@@ -820,9 +912,9 @@ module Fluent
|
|
820
912
|
end
|
821
913
|
elsif record.key?('severity')
|
822
914
|
return parse_severity(record.delete('severity'))
|
823
|
-
elsif
|
824
|
-
|
825
|
-
stream =
|
915
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
916
|
+
entry_common_labels.key?("#{CONTAINER_CONSTANTS[:service]}/stream")
|
917
|
+
stream = entry_common_labels["#{CONTAINER_CONSTANTS[:service]}/stream"]
|
826
918
|
if stream == 'stdout'
|
827
919
|
return 'INFO'
|
828
920
|
elsif stream == 'stderr'
|
@@ -838,7 +930,7 @@ module Fluent
|
|
838
930
|
def set_http_request(record, entry)
|
839
931
|
return nil unless record['httpRequest'].is_a?(Hash)
|
840
932
|
input = record['httpRequest']
|
841
|
-
output = Google::Apis::
|
933
|
+
output = Google::Apis::LoggingV2beta1::HttpRequest.new
|
842
934
|
output.request_method = input.delete('requestMethod')
|
843
935
|
output.request_url = input.delete('requestUrl')
|
844
936
|
output.request_size = input.delete('requestSize')
|
@@ -848,8 +940,8 @@ module Fluent
|
|
848
940
|
output.remote_ip = input.delete('remoteIp')
|
849
941
|
output.referer = input.delete('referer')
|
850
942
|
output.cache_hit = input.delete('cacheHit')
|
851
|
-
output.
|
852
|
-
input.delete('
|
943
|
+
output.cache_validated_with_origin_server = \
|
944
|
+
input.delete('cacheValidatedWithOriginServer')
|
853
945
|
record.delete('httpRequest') if input.empty?
|
854
946
|
entry.http_request = output
|
855
947
|
end
|
@@ -989,16 +1081,23 @@ module Fluent
|
|
989
1081
|
end
|
990
1082
|
|
991
1083
|
# Requires that record has a 'kubernetes' field.
|
992
|
-
def
|
993
|
-
|
994
|
-
|
995
|
-
|
996
|
-
|
1084
|
+
def extract_container_metadata(record)
|
1085
|
+
resource_labels = {}
|
1086
|
+
common_labels = {}
|
1087
|
+
%w(namespace_id pod_id container_name).each do |field|
|
1088
|
+
resource_labels.merge!(
|
1089
|
+
fields_to_labels(record['kubernetes'], field => field))
|
1090
|
+
end
|
1091
|
+
%w(namespace_name pod_name).each do |field|
|
1092
|
+
common_labels.merge!(
|
1093
|
+
fields_to_labels(
|
1094
|
+
record['kubernetes'],
|
1095
|
+
field => "#{CONTAINER_CONSTANTS[:service]}/#{field}"))
|
997
1096
|
end
|
998
1097
|
# Prepend label/ to all user-defined labels' keys.
|
999
1098
|
if record['kubernetes'].key?('labels')
|
1000
1099
|
record['kubernetes']['labels'].each do |key, value|
|
1001
|
-
|
1100
|
+
common_labels["label/#{key}"] = value
|
1002
1101
|
end
|
1003
1102
|
end
|
1004
1103
|
# We've explicitly consumed all the fields we care about -- don't litter
|
@@ -1006,33 +1105,43 @@ module Fluent
|
|
1006
1105
|
# filter plugin includes (or an empty 'kubernetes' field).
|
1007
1106
|
record.delete('kubernetes')
|
1008
1107
|
record.delete('docker')
|
1108
|
+
[resource_labels, common_labels]
|
1009
1109
|
end
|
1010
1110
|
|
1011
|
-
|
1012
|
-
|
1013
|
-
|
1014
|
-
|
1111
|
+
# For every original_label => new_label pair in the label_map, delete the
|
1112
|
+
# original_label from the record if it exists, and extract the value to form
|
1113
|
+
# a map with the new_label as the key.
|
1114
|
+
def fields_to_labels(record, label_map)
|
1115
|
+
return {} if label_map.nil? || !label_map.is_a?(Hash)
|
1116
|
+
label_map.each_with_object({}) \
|
1117
|
+
do |(original_label, new_label), extracted_labels|
|
1118
|
+
extracted_labels[new_label] = convert_to_utf8(
|
1119
|
+
record.delete(original_label).to_s) if record.key?(original_label)
|
1120
|
+
end
|
1015
1121
|
end
|
1016
1122
|
|
1017
|
-
def set_payload(record, entry, is_json)
|
1123
|
+
def set_payload(resource_type, record, entry, is_json)
|
1018
1124
|
# If this is a Cloud Functions log that matched the expected regexp,
|
1019
1125
|
# use text payload. Otherwise, use JSON if we found valid JSON, or text
|
1020
1126
|
# payload in the following cases:
|
1021
1127
|
# 1. This is a Cloud Functions log and the 'log' key is available
|
1022
1128
|
# 2. This is an unstructured Container log and the 'log' key is available
|
1023
1129
|
# 3. The only remaining key is 'message'
|
1024
|
-
if
|
1130
|
+
if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
1131
|
+
@cloudfunctions_log_match
|
1025
1132
|
entry.text_payload = @cloudfunctions_log_match['text']
|
1026
|
-
elsif
|
1133
|
+
elsif resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
1134
|
+
record.key?('log')
|
1027
1135
|
entry.text_payload = record['log']
|
1028
1136
|
elsif is_json
|
1029
|
-
entry.
|
1030
|
-
elsif
|
1137
|
+
entry.json_payload = record
|
1138
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1139
|
+
record.key?('log')
|
1031
1140
|
entry.text_payload = record['log']
|
1032
1141
|
elsif record.size == 1 && record.key?('message')
|
1033
1142
|
entry.text_payload = record['message']
|
1034
1143
|
else
|
1035
|
-
entry.
|
1144
|
+
entry.json_payload = record
|
1036
1145
|
end
|
1037
1146
|
end
|
1038
1147
|
|
@@ -1080,56 +1189,82 @@ module Fluent
|
|
1080
1189
|
ret
|
1081
1190
|
end
|
1082
1191
|
|
1083
|
-
def set_payload_grpc(record, entry, is_json)
|
1192
|
+
def set_payload_grpc(resource_type, record, entry, is_json)
|
1084
1193
|
# If this is a Cloud Functions log that matched the expected regexp,
|
1085
1194
|
# use text payload. Otherwise, use JSON if we found valid JSON, or text
|
1086
1195
|
# payload in the following cases:
|
1087
1196
|
# 1. This is a Cloud Functions log and the 'log' key is available
|
1088
1197
|
# 2. This is an unstructured Container log and the 'log' key is available
|
1089
1198
|
# 3. The only remaining key is 'message'
|
1090
|
-
if
|
1199
|
+
if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
1200
|
+
@cloudfunctions_log_match
|
1091
1201
|
entry.text_payload = convert_to_utf8(
|
1092
1202
|
@cloudfunctions_log_match['text'])
|
1093
|
-
elsif
|
1203
|
+
elsif resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
1204
|
+
record.key?('log')
|
1094
1205
|
entry.text_payload = convert_to_utf8(record['log'])
|
1095
1206
|
elsif is_json
|
1096
|
-
entry.
|
1097
|
-
elsif
|
1207
|
+
entry.json_payload = struct_from_ruby(record)
|
1208
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1209
|
+
record.key?('log')
|
1098
1210
|
entry.text_payload = convert_to_utf8(record['log'])
|
1099
1211
|
elsif record.size == 1 && record.key?('message')
|
1100
1212
|
entry.text_payload = convert_to_utf8(record['message'])
|
1101
1213
|
else
|
1102
|
-
entry.
|
1214
|
+
entry.json_payload = struct_from_ruby(record)
|
1103
1215
|
end
|
1104
1216
|
end
|
1105
1217
|
|
1106
|
-
def log_name(tag,
|
1107
|
-
if
|
1218
|
+
def log_name(tag, resource)
|
1219
|
+
if resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
1108
1220
|
tag = 'cloud-functions'
|
1109
1221
|
elsif @running_on_managed_vm
|
1110
1222
|
# Add a prefix to Managed VM logs to prevent namespace collisions.
|
1111
|
-
tag = "#{
|
1112
|
-
elsif
|
1223
|
+
tag = "#{APPENGINE_CONSTANTS[:service]}/#{tag}"
|
1224
|
+
elsif resource.type == CONTAINER_CONSTANTS[:resource_type]
|
1113
1225
|
# For Kubernetes logs, use just the container name as the log name
|
1114
1226
|
# if we have it.
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
tag = sanitized_log_name unless sanitized_log_name.nil?
|
1227
|
+
if resource.labels && resource.labels.key?('container_name')
|
1228
|
+
sanitized_tag = sanitize_tag(resource.labels['container_name'])
|
1229
|
+
tag = sanitized_tag unless sanitized_tag.nil?
|
1119
1230
|
end
|
1120
1231
|
end
|
1121
|
-
|
1122
|
-
# lib already handles encoding.
|
1123
|
-
tag = ERB::Util.url_encode(tag) if @use_grpc
|
1232
|
+
tag = ERB::Util.url_encode(tag)
|
1124
1233
|
tag
|
1125
1234
|
end
|
1126
1235
|
|
1236
|
+
# Some services set labels (via configuring 'labels' or 'label_map') which
|
1237
|
+
# are now MonitoredResource labels in v2.
|
1238
|
+
# For these services, remove resource labels from 'labels' and return a
|
1239
|
+
# Hash of labels to be merged into the MonitoredResource labels.
|
1240
|
+
# Otherwise, return an empty hash and leave 'labels' unmodified.
|
1241
|
+
def extract_resource_labels(resource_type, labels)
|
1242
|
+
extracted_labels = {}
|
1243
|
+
return extracted_labels if labels.nil? || !labels.is_a?(Hash)
|
1244
|
+
|
1245
|
+
if resource_type == DATAFLOW_CONSTANTS[:resource_type]
|
1246
|
+
label_prefix = DATAFLOW_CONSTANTS[:service]
|
1247
|
+
labels_to_extract = %w(region job_name job_id step_id)
|
1248
|
+
elsif resource_type == ML_CONSTANTS[:resource_type]
|
1249
|
+
label_prefix = ML_CONSTANTS[:service]
|
1250
|
+
labels_to_extract = %w(job_id task_name)
|
1251
|
+
else
|
1252
|
+
return extracted_labels
|
1253
|
+
end
|
1254
|
+
|
1255
|
+
labels_to_extract.each do |label|
|
1256
|
+
extracted_labels[label] = labels.delete("#{label_prefix}/#{label}") if
|
1257
|
+
labels.key?("#{label_prefix}/#{label}")
|
1258
|
+
end
|
1259
|
+
extracted_labels
|
1260
|
+
end
|
1261
|
+
|
1127
1262
|
def init_api_client
|
1128
1263
|
return if @use_grpc
|
1129
1264
|
# TODO: Use a non-default ClientOptions object.
|
1130
1265
|
Google::Apis::ClientOptions.default.application_name = PLUGIN_NAME
|
1131
1266
|
Google::Apis::ClientOptions.default.application_version = PLUGIN_VERSION
|
1132
|
-
@client = Google::Apis::
|
1267
|
+
@client = Google::Apis::LoggingV2beta1::LoggingService.new
|
1133
1268
|
@client.authorization = Google::Auth.get_application_default(
|
1134
1269
|
LOGGING_SCOPE)
|
1135
1270
|
end
|
@@ -1140,7 +1275,7 @@ module Fluent
|
|
1140
1275
|
authentication = Google::Auth.get_application_default
|
1141
1276
|
creds = GRPC::Core::CallCredentials.new(authentication.updater_proc)
|
1142
1277
|
creds = ssl_creds.compose(creds)
|
1143
|
-
@client = Google::Logging::
|
1278
|
+
@client = Google::Logging::V2::LoggingServiceV2::Stub.new(
|
1144
1279
|
'logging.googleapis.com', creds)
|
1145
1280
|
else
|
1146
1281
|
unless @client.authorization.expired?
|
@@ -1182,3 +1317,18 @@ module Fluent
|
|
1182
1317
|
end
|
1183
1318
|
end
|
1184
1319
|
end
|
1320
|
+
|
1321
|
+
module Google
|
1322
|
+
module Apis
|
1323
|
+
module LoggingV2beta1
|
1324
|
+
# Override MonitoredResource::dup to make a deep copy.
|
1325
|
+
class MonitoredResource
|
1326
|
+
def dup
|
1327
|
+
ret = super
|
1328
|
+
ret.labels = labels.dup
|
1329
|
+
ret
|
1330
|
+
end
|
1331
|
+
end
|
1332
|
+
end
|
1333
|
+
end
|
1334
|
+
end
|