fluent-plugin-google-cloud 0.6.7.pre.1 → 0.6.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/Rakefile +2 -1
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/out_google_cloud.rb +189 -287
- data/test/plugin/base_test.rb +2 -254
- data/test/plugin/constants.rb +13 -148
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f19f3c278abefebea1b068850a3e7d09a31835e2
|
4
|
+
data.tar.gz: e4588beb45b2dba23d3901a6a527091134c1f5a6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 86369bce68f370907f35d589830ea73330032590ae43bc75b22bc7ff99157997740fdfeac46079c96fd832b90ca7bc36da9e66f8187e4f1c566277e028b38b0f
|
7
|
+
data.tar.gz: 9c8189c85d4980c160403b09bd2ccb58c3bcd264f36062e994ea7bf63aa3fc70aa79cf357f54a83ff255ae9ceb7d9bee9243866f2a7d00af6b6b2c147001e549
|
data/Gemfile.lock
CHANGED
data/Rakefile
CHANGED
@@ -22,6 +22,7 @@ desc 'Fix file permissions'
|
|
22
22
|
task :fix_perms do
|
23
23
|
files = [
|
24
24
|
'lib/fluent/plugin/out_google_cloud.rb',
|
25
|
+
'lib/fluent/plugin/monitoring.rb',
|
25
26
|
'lib/google/**/*.rb'
|
26
27
|
].flat_map do |file|
|
27
28
|
file.include?('*') ? Dir.glob(file) : [file]
|
@@ -37,6 +38,6 @@ task :fix_perms do
|
|
37
38
|
end
|
38
39
|
|
39
40
|
desc 'Run unit tests and RuboCop to check for style violations'
|
40
|
-
task all: [:
|
41
|
+
task all: [:rubocop, :test, :fix_perms]
|
41
42
|
|
42
43
|
task default: :all
|
@@ -10,7 +10,7 @@ eos
|
|
10
10
|
gem.homepage = \
|
11
11
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
|
12
12
|
gem.license = 'Apache-2.0'
|
13
|
-
gem.version = '0.6.7
|
13
|
+
gem.version = '0.6.7'
|
14
14
|
gem.authors = ['Todd Derr', 'Alex Robinson']
|
15
15
|
gem.email = ['salty@google.com']
|
16
16
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -38,12 +38,12 @@ end
|
|
38
38
|
module Fluent
|
39
39
|
# fluentd output plugin for the Stackdriver Logging API
|
40
40
|
class GoogleCloudOutput < BufferedOutput
|
41
|
-
# Constants for service names
|
42
|
-
module
|
41
|
+
# Constants for service names and resource types.
|
42
|
+
module Constants
|
43
43
|
APPENGINE_CONSTANTS = {
|
44
44
|
service: 'appengine.googleapis.com',
|
45
45
|
resource_type: 'gae_app',
|
46
|
-
metadata_attributes: %w(gae_backend_name gae_backend_version)
|
46
|
+
metadata_attributes: %w(gae_backend_name gae_backend_version).to_set
|
47
47
|
}
|
48
48
|
CLOUDFUNCTIONS_CONSTANTS = {
|
49
49
|
service: 'cloudfunctions.googleapis.com',
|
@@ -53,16 +53,12 @@ module Fluent
|
|
53
53
|
service: 'compute.googleapis.com',
|
54
54
|
resource_type: 'gce_instance'
|
55
55
|
}
|
56
|
-
|
56
|
+
CONTAINER_CONSTANTS = {
|
57
57
|
service: 'container.googleapis.com',
|
58
58
|
resource_type: 'container',
|
59
59
|
extra_resource_labels: %w(namespace_id pod_id container_name),
|
60
60
|
extra_common_labels: %w(namespace_name pod_name),
|
61
|
-
metadata_attributes: %w(kube-env)
|
62
|
-
}
|
63
|
-
DOCKER_CONSTANTS = {
|
64
|
-
service: 'docker.googleapis.com',
|
65
|
-
resource_type: 'docker_container'
|
61
|
+
metadata_attributes: %w(kube-env).to_set
|
66
62
|
}
|
67
63
|
DATAFLOW_CONSTANTS = {
|
68
64
|
service: 'dataflow.googleapis.com',
|
@@ -72,7 +68,8 @@ module Fluent
|
|
72
68
|
DATAPROC_CONSTANTS = {
|
73
69
|
service: 'cluster.dataproc.googleapis.com',
|
74
70
|
resource_type: 'cloud_dataproc_cluster',
|
75
|
-
metadata_attributes:
|
71
|
+
metadata_attributes:
|
72
|
+
%w(dataproc-cluster-uuid dataproc-cluster-name).to_set
|
76
73
|
}
|
77
74
|
EC2_CONSTANTS = {
|
78
75
|
service: 'ec2.amazonaws.com',
|
@@ -86,7 +83,7 @@ module Fluent
|
|
86
83
|
|
87
84
|
# The map between a subservice name and a resource type.
|
88
85
|
SUBSERVICE_MAP = \
|
89
|
-
[APPENGINE_CONSTANTS,
|
86
|
+
[APPENGINE_CONSTANTS, CONTAINER_CONSTANTS, DATAFLOW_CONSTANTS,
|
90
87
|
DATAPROC_CONSTANTS, ML_CONSTANTS]
|
91
88
|
.map { |consts| [consts[:service], consts[:resource_type]] }.to_h
|
92
89
|
# Default back to GCE if invalid value is detected.
|
@@ -94,35 +91,18 @@ module Fluent
|
|
94
91
|
|
95
92
|
# The map between a resource type and expected subservice attributes.
|
96
93
|
SUBSERVICE_METADATA_ATTRIBUTES = \
|
97
|
-
[APPENGINE_CONSTANTS,
|
98
|
-
.map
|
99
|
-
|
100
|
-
end.to_h
|
101
|
-
end
|
94
|
+
[APPENGINE_CONSTANTS, CONTAINER_CONSTANTS, DATAPROC_CONSTANTS]
|
95
|
+
.map { |consts| [consts[:resource_type], consts[:metadata_attributes]] }
|
96
|
+
.to_h
|
102
97
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
# "operation", "sourceLocation", "trace" fields in the LogEntry.
|
98
|
+
# Default values for JSON payload keys to set the "trace",
|
99
|
+
# "sourceLocation", "operation" and "labels" fields in the LogEntry.
|
100
|
+
DEFAULT_PAYLOAD_KEY_PREFIX = 'logging.googleapis.com'
|
107
101
|
DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'
|
108
|
-
DEFAULT_OPERATION_KEY =
|
109
|
-
DEFAULT_SOURCE_LOCATION_KEY =
|
110
|
-
|
111
|
-
|
112
|
-
DEFAULT_METADATA_AGENT_URL =
|
113
|
-
'http://local-metadata-agent.stackdriver.com:8000'
|
114
|
-
end
|
115
|
-
|
116
|
-
# Constants for log entry field extraction.
|
117
|
-
module InternalConstants
|
118
|
-
# Use empty string as request path when the local_resource_id of monitored
|
119
|
-
# resource can be implicitly inferred by Metadata Agent.
|
120
|
-
IMPLICIT_LOCAL_RESOURCE_ID = ''
|
121
|
-
|
122
|
-
# The label name of local_resource_id in the json payload. When a record
|
123
|
-
# has this field in the payload, we will use the value to retrieve
|
124
|
-
# monitored resource from Stackdriver Metadata agent.
|
125
|
-
LOCAL_RESOURCE_ID_KEY = 'logging.googleapis.com/local_resource_id'
|
102
|
+
DEFAULT_OPERATION_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/operation"
|
103
|
+
DEFAULT_SOURCE_LOCATION_KEY =
|
104
|
+
"#{DEFAULT_PAYLOAD_KEY_PREFIX}/sourceLocation"
|
105
|
+
DEFAULT_TRACE_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/trace"
|
126
106
|
|
127
107
|
# Map from each field name under LogEntry to corresponding variables
|
128
108
|
# required to perform field value extraction from the log record.
|
@@ -175,14 +155,12 @@ module Fluent
|
|
175
155
|
}
|
176
156
|
end
|
177
157
|
|
178
|
-
include self::
|
179
|
-
include self::ConfigConstants
|
180
|
-
include self::InternalConstants
|
158
|
+
include self::Constants
|
181
159
|
|
182
160
|
Fluent::Plugin.register_output('google_cloud', self)
|
183
161
|
|
184
162
|
PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
|
185
|
-
PLUGIN_VERSION = '0.6.7
|
163
|
+
PLUGIN_VERSION = '0.6.7'
|
186
164
|
|
187
165
|
# Name of the the Google cloud logging write scope.
|
188
166
|
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
|
@@ -317,11 +295,6 @@ module Fluent
|
|
317
295
|
config_param :monitoring_type, :string,
|
318
296
|
:default => Monitoring::PrometheusMonitoringRegistry.name
|
319
297
|
|
320
|
-
# Whether to call metadata agent to retrieve monitored resource.
|
321
|
-
config_param :enable_metadata_agent, :bool, :default => false
|
322
|
-
config_param :metadata_agent_url, :string,
|
323
|
-
:default => DEFAULT_METADATA_AGENT_URL
|
324
|
-
|
325
298
|
# rubocop:enable Style/HashSyntax
|
326
299
|
|
327
300
|
# TODO: Add a log_name config option rather than just using the tag?
|
@@ -381,30 +354,28 @@ module Fluent
|
|
381
354
|
|
382
355
|
@platform = detect_platform
|
383
356
|
|
384
|
-
# Set
|
385
|
-
#
|
386
|
-
#
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
end
|
394
|
-
|
395
|
-
# Set required variables: @project_id, @vm_id, @vm_name and @zone.
|
357
|
+
# Set required variables: @project_id, @vm_id, @vm_name and @zone by
|
358
|
+
# making some requests to metadata server.
|
359
|
+
#
|
360
|
+
# Note: Once we support metadata injection at Logging API side, we might
|
361
|
+
# no longer need to require all these metadata in logging agent. But for
|
362
|
+
# now, they are still required.
|
363
|
+
#
|
364
|
+
# TODO(qingling128): After Metadata Agent support is added, try extracting
|
365
|
+
# these info from responses from Metadata Agent first.
|
396
366
|
set_required_metadata_variables
|
397
367
|
|
398
368
|
# Retrieve monitored resource.
|
399
|
-
#
|
400
|
-
#
|
401
|
-
|
369
|
+
#
|
370
|
+
# TODO(qingling128): After Metadata Agent support is added, try retrieving
|
371
|
+
# the monitored resource from Metadata Agent first.
|
372
|
+
@resource = determine_agent_level_monitored_resource_via_legacy
|
402
373
|
|
403
374
|
# Set regexp that we should match tags against later on. Using a list
|
404
375
|
# instead of a map to ensure order. For example, tags will be matched
|
405
376
|
# against Cloud Functions first, then GKE.
|
406
377
|
@tag_regexp_list = []
|
407
|
-
if @resource.type ==
|
378
|
+
if @resource.type == CONTAINER_CONSTANTS[:resource_type]
|
408
379
|
# We only support Cloud Functions logs for GKE right now.
|
409
380
|
if fetch_gce_metadata('instance/attributes/'
|
410
381
|
).split.include?('gcf_region')
|
@@ -417,7 +388,7 @@ module Fluent
|
|
417
388
|
]
|
418
389
|
end
|
419
390
|
@tag_regexp_list << [
|
420
|
-
|
391
|
+
CONTAINER_CONSTANTS[:resource_type], @compiled_kubernetes_tag_regexp
|
421
392
|
]
|
422
393
|
end
|
423
394
|
|
@@ -433,7 +404,7 @@ module Fluent
|
|
433
404
|
|
434
405
|
# Log an informational message containing the Logs viewer URL
|
435
406
|
@log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
|
436
|
-
"viewer?project=#{@project_id}&resource=#{@
|
407
|
+
"viewer?project=#{@project_id}&resource=#{@resource_type}/",
|
437
408
|
"instance_id/#{@vm_id}"
|
438
409
|
end
|
439
410
|
|
@@ -449,27 +420,39 @@ module Fluent
|
|
449
420
|
end
|
450
421
|
|
451
422
|
def write(chunk)
|
452
|
-
|
423
|
+
# Group the entries since we have to make one call per tag.
|
424
|
+
grouped_entries = {}
|
425
|
+
chunk.msgpack_each do |tag, *arr|
|
426
|
+
sanitized_tag = sanitize_tag(tag)
|
427
|
+
if sanitized_tag.nil?
|
428
|
+
@log.warn "Dropping log entries with invalid tag: '#{tag}'. " \
|
429
|
+
'A tag should be a string with utf8 characters.'
|
430
|
+
next
|
431
|
+
end
|
432
|
+
grouped_entries[sanitized_tag] ||= []
|
433
|
+
grouped_entries[sanitized_tag].push(arr)
|
434
|
+
end
|
453
435
|
|
454
|
-
grouped_entries.each do |
|
436
|
+
grouped_entries.each do |tag, arr|
|
455
437
|
entries = []
|
456
|
-
|
457
|
-
determine_group_level_monitored_resource_and_labels(
|
458
|
-
tag, local_resource_id)
|
438
|
+
group_resource, group_common_labels =
|
439
|
+
determine_group_level_monitored_resource_and_labels(tag)
|
459
440
|
|
460
441
|
arr.each do |time, record|
|
461
|
-
|
462
|
-
|
463
|
-
|
442
|
+
next unless record.is_a?(Hash)
|
443
|
+
|
444
|
+
extracted_resource_labels, extracted_common_labels = \
|
445
|
+
determine_entry_level_labels(group_resource, record)
|
446
|
+
entry_resource = group_resource.dup
|
447
|
+
entry_resource.labels.merge!(extracted_resource_labels)
|
448
|
+
entry_common_labels = \
|
449
|
+
group_common_labels.merge(extracted_common_labels)
|
464
450
|
|
465
451
|
is_json = false
|
466
452
|
if @detect_json
|
467
|
-
# Save the timestamp
|
468
|
-
#
|
469
|
-
# field.
|
453
|
+
# Save the timestamp if available, then clear it out to allow for
|
454
|
+
# determining whether we should parse the log or message field.
|
470
455
|
timestamp = record.delete('time')
|
471
|
-
severity = record.delete('severity')
|
472
|
-
|
473
456
|
# If the log is json, we want to export it as a structured log
|
474
457
|
# unless there is additional metadata that would be lost.
|
475
458
|
record_json = nil
|
@@ -484,17 +467,16 @@ module Fluent
|
|
484
467
|
record = record_json
|
485
468
|
is_json = true
|
486
469
|
end
|
487
|
-
# Restore timestamp
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
record['severity'] ||= severity if severity
|
470
|
+
# Restore timestamp if necessary.
|
471
|
+
unless record.key?('time') || timestamp.nil?
|
472
|
+
record['time'] = timestamp
|
473
|
+
end
|
492
474
|
end
|
493
475
|
|
494
476
|
ts_secs, ts_nanos = compute_timestamp(
|
495
|
-
|
477
|
+
entry_resource.type, record, time)
|
496
478
|
severity = compute_severity(
|
497
|
-
|
479
|
+
entry_resource.type, record, entry_common_labels)
|
498
480
|
|
499
481
|
ts_secs = begin
|
500
482
|
Integer ts_secs
|
@@ -506,13 +488,12 @@ module Fluent
|
|
506
488
|
rescue ArgumentError, TypeError
|
507
489
|
ts_nanos
|
508
490
|
end
|
509
|
-
|
510
491
|
if @use_grpc
|
511
492
|
entry = Google::Logging::V2::LogEntry.new(
|
512
|
-
labels:
|
493
|
+
labels: entry_common_labels,
|
513
494
|
resource: Google::Api::MonitoredResource.new(
|
514
|
-
type:
|
515
|
-
labels:
|
495
|
+
type: entry_resource.type,
|
496
|
+
labels: entry_resource.labels.to_h
|
516
497
|
),
|
517
498
|
severity: grpc_severity(severity)
|
518
499
|
)
|
@@ -529,11 +510,10 @@ module Fluent
|
|
529
510
|
end
|
530
511
|
else
|
531
512
|
# Remove the labels if we didn't populate them with anything.
|
532
|
-
|
533
|
-
entry_level_resource.labels.empty?
|
513
|
+
entry_resource.labels = nil if entry_resource.labels.empty?
|
534
514
|
entry = Google::Apis::LoggingV2beta1::LogEntry.new(
|
535
|
-
labels:
|
536
|
-
resource:
|
515
|
+
labels: entry_common_labels,
|
516
|
+
resource: entry_resource,
|
537
517
|
severity: severity,
|
538
518
|
timestamp: {
|
539
519
|
seconds: ts_secs,
|
@@ -542,12 +522,13 @@ module Fluent
|
|
542
522
|
)
|
543
523
|
end
|
544
524
|
|
545
|
-
# Get fully-qualified trace id for LogEntry "trace" field.
|
525
|
+
# Get fully-qualified trace id for LogEntry "trace" field per config.
|
546
526
|
fq_trace_id = record.delete(@trace_key)
|
547
527
|
entry.trace = fq_trace_id if fq_trace_id
|
548
528
|
|
549
529
|
set_log_entry_fields(record, entry)
|
550
|
-
|
530
|
+
|
531
|
+
set_payload(entry_resource.type, record, entry, is_json)
|
551
532
|
|
552
533
|
entries.push(entry)
|
553
534
|
end
|
@@ -555,21 +536,21 @@ module Fluent
|
|
555
536
|
next if entries.empty?
|
556
537
|
|
557
538
|
log_name = "projects/#{@project_id}/logs/#{log_name(
|
558
|
-
tag,
|
539
|
+
tag, group_resource)}"
|
559
540
|
|
560
541
|
# Does the actual write to the cloud logging api.
|
561
542
|
client = api_client
|
562
543
|
if @use_grpc
|
563
544
|
begin
|
564
|
-
labels_utf8_pairs =
|
545
|
+
labels_utf8_pairs = group_common_labels.map do |k, v|
|
565
546
|
[k.encode('utf-8'), convert_to_utf8(v)]
|
566
547
|
end
|
567
548
|
|
568
549
|
write_request = Google::Logging::V2::WriteLogEntriesRequest.new(
|
569
550
|
log_name: log_name,
|
570
551
|
resource: Google::Api::MonitoredResource.new(
|
571
|
-
type:
|
572
|
-
labels:
|
552
|
+
type: group_resource.type,
|
553
|
+
labels: group_resource.labels.to_h
|
573
554
|
),
|
574
555
|
labels: labels_utf8_pairs.to_h,
|
575
556
|
entries: entries
|
@@ -579,8 +560,8 @@ module Fluent
|
|
579
560
|
increment_successful_requests_count
|
580
561
|
increment_ingested_entries_count(entries.length)
|
581
562
|
|
582
|
-
# Let the user explicitly know when the first call succeeded,
|
583
|
-
# with verification and troubleshooting.
|
563
|
+
# Let the user explicitly know when the first call succeeded,
|
564
|
+
# to aid with verification and troubleshooting.
|
584
565
|
unless @successful_call
|
585
566
|
@successful_call = true
|
586
567
|
@log.info 'Successfully sent gRPC to Stackdriver Logging API.'
|
@@ -619,8 +600,8 @@ module Fluent
|
|
619
600
|
@log.warn "Dropping #{dropped} log message(s)",
|
620
601
|
error: error.to_s, error_code: error.code.to_s
|
621
602
|
else
|
622
|
-
# Assume this is a problem with the request itself
|
623
|
-
# retry.
|
603
|
+
# Assume this is a problem with the request itself
|
604
|
+
# and don't retry.
|
624
605
|
dropped = entries.length
|
625
606
|
increment_dropped_entries_count(dropped)
|
626
607
|
@log.error "Unknown response code #{error.code} from the "\
|
@@ -633,8 +614,8 @@ module Fluent
|
|
633
614
|
write_request = \
|
634
615
|
Google::Apis::LoggingV2beta1::WriteLogEntriesRequest.new(
|
635
616
|
log_name: log_name,
|
636
|
-
resource:
|
637
|
-
labels:
|
617
|
+
resource: group_resource,
|
618
|
+
labels: group_common_labels,
|
638
619
|
entries: entries)
|
639
620
|
|
640
621
|
# TODO: RequestOptions
|
@@ -647,8 +628,8 @@ module Fluent
|
|
647
628
|
increment_successful_requests_count
|
648
629
|
increment_ingested_entries_count(entries.length)
|
649
630
|
|
650
|
-
# Let the user explicitly know when the first call succeeded,
|
651
|
-
# with verification and troubleshooting.
|
631
|
+
# Let the user explicitly know when the first call succeeded,
|
632
|
+
# to aid with verification and troubleshooting.
|
652
633
|
unless @successful_call
|
653
634
|
@successful_call = true
|
654
635
|
@log.info 'Successfully sent to Stackdriver Logging API.'
|
@@ -839,9 +820,8 @@ module Fluent
|
|
839
820
|
|
840
821
|
# Retrieve monitored resource via the legacy way.
|
841
822
|
#
|
842
|
-
#
|
843
|
-
#
|
844
|
-
# returns.
|
823
|
+
# TODO(qingling128): Use this as only a fallback plan after Metadata Agent
|
824
|
+
# support is added.
|
845
825
|
def determine_agent_level_monitored_resource_via_legacy
|
846
826
|
resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
847
827
|
labels: {})
|
@@ -903,7 +883,7 @@ module Fluent
|
|
903
883
|
}
|
904
884
|
|
905
885
|
# GKE container.
|
906
|
-
when
|
886
|
+
when CONTAINER_CONSTANTS[:resource_type]
|
907
887
|
raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
|
908
888
|
kube_env = YAML.load(raw_kube_env)
|
909
889
|
return {
|
@@ -938,7 +918,7 @@ module Fluent
|
|
938
918
|
rescue StandardError => e
|
939
919
|
@log.error "Failed to set monitored resource labels for #{type}: ",
|
940
920
|
error: e
|
941
|
-
{}
|
921
|
+
return {}
|
942
922
|
end
|
943
923
|
|
944
924
|
# Determine the common labels that should be added to all log entries
|
@@ -962,7 +942,7 @@ module Fluent
|
|
962
942
|
|
963
943
|
# GCE instance and GKE container.
|
964
944
|
when COMPUTE_CONSTANTS[:resource_type],
|
965
|
-
|
945
|
+
CONTAINER_CONSTANTS[:resource_type]
|
966
946
|
labels.merge!(
|
967
947
|
"#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name)
|
968
948
|
|
@@ -974,175 +954,138 @@ module Fluent
|
|
974
954
|
labels
|
975
955
|
end
|
976
956
|
|
977
|
-
# Group the log entries by tag and local_resource_id pairs.
|
978
|
-
def group_log_entries_by_tag_and_local_resource_id(chunk)
|
979
|
-
groups = {}
|
980
|
-
chunk.msgpack_each do |tag, time, record|
|
981
|
-
unless record.is_a?(Hash)
|
982
|
-
@log.warn 'Dropping log entries with malformed record: ' \
|
983
|
-
"'#{record.inspect}'. " \
|
984
|
-
'A log record should be in JSON format.'
|
985
|
-
next
|
986
|
-
end
|
987
|
-
sanitized_tag = sanitize_tag(tag)
|
988
|
-
if sanitized_tag.nil?
|
989
|
-
@log.warn "Dropping log entries with invalid tag: '#{tag.inspect}'." \
|
990
|
-
' A tag should be a string with utf8 characters.'
|
991
|
-
next
|
992
|
-
end
|
993
|
-
local_resource_id = record.delete(LOCAL_RESOURCE_ID_KEY)
|
994
|
-
# A nil local_resource_id means "fall back to legacy".
|
995
|
-
hash_key = [sanitized_tag, local_resource_id].freeze
|
996
|
-
groups[hash_key] ||= []
|
997
|
-
groups[hash_key].push([time, record])
|
998
|
-
end
|
999
|
-
groups
|
1000
|
-
end
|
1001
|
-
|
1002
957
|
# Determine the group level monitored resource and common labels shared by a
|
1003
958
|
# collection of entries.
|
1004
|
-
def determine_group_level_monitored_resource_and_labels(tag
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1008
|
-
|
1009
|
-
|
1010
|
-
|
1011
|
-
#
|
1012
|
-
|
959
|
+
def determine_group_level_monitored_resource_and_labels(tag)
|
960
|
+
# Determine group level monitored resource type. For certain types,
|
961
|
+
# extract useful info from the tag and store those in
|
962
|
+
# matched_regex_group.
|
963
|
+
group_resource_type, matched_regex_group =
|
964
|
+
determine_group_level_monitored_resource_type(tag)
|
965
|
+
|
966
|
+
# Determine group level monitored resource labels and common labels.
|
967
|
+
group_resource_labels, group_common_labels =
|
968
|
+
determine_group_level_labels(group_resource_type, matched_regex_group)
|
969
|
+
|
970
|
+
group_resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
971
|
+
type: group_resource_type,
|
972
|
+
labels: group_resource_labels.to_h
|
973
|
+
)
|
974
|
+
|
975
|
+
# Freeze the per-request state. Any further changes must be made on a
|
976
|
+
# per-entry basis.
|
977
|
+
group_resource.freeze
|
978
|
+
group_resource.labels.freeze
|
979
|
+
group_common_labels.freeze
|
980
|
+
|
981
|
+
[group_resource, group_common_labels]
|
982
|
+
end
|
983
|
+
|
984
|
+
# Determine group level monitored resource type shared by a collection of
|
985
|
+
# entries.
|
986
|
+
# Return the resource type and tag regexp matched groups. The matched groups
|
987
|
+
# only apply to some resource types. Return nil if not applicable or if
|
988
|
+
# there is no match.
|
989
|
+
def determine_group_level_monitored_resource_type(tag)
|
1013
990
|
@tag_regexp_list.each do |derived_type, tag_regexp|
|
1014
|
-
|
1015
|
-
if
|
1016
|
-
|
1017
|
-
break
|
1018
|
-
end
|
991
|
+
matched_regex_group = tag_regexp.match(tag)
|
992
|
+
return [derived_type, matched_regex_group] if
|
993
|
+
matched_regex_group
|
1019
994
|
end
|
995
|
+
[@resource.type, nil]
|
996
|
+
end
|
1020
997
|
|
1021
|
-
|
1022
|
-
|
1023
|
-
|
1024
|
-
|
1025
|
-
|
1026
|
-
# Metadata Agent with this key.
|
1027
|
-
#
|
1028
|
-
# Docker container:
|
1029
|
-
# "container.<container_id>"
|
1030
|
-
# "containerName.<container_name>"
|
1031
|
-
# GKE container:
|
1032
|
-
# "gke_containerName.<namespace_id>.<pod_name>.<container_name>"
|
1033
|
-
if @enable_metadata_agent && local_resource_id
|
1034
|
-
@log.debug 'Calling metadata agent with local_resource_id: ' \
|
1035
|
-
"#{local_resource_id}."
|
1036
|
-
retrieved_resource = call_metadata_agent_for_monitored_resource(
|
1037
|
-
local_resource_id)
|
1038
|
-
@log.debug 'Retrieved monitored resource from metadata agent: ' \
|
1039
|
-
"#{retrieved_resource.inspect}."
|
1040
|
-
unless retrieved_resource.nil?
|
1041
|
-
# TODO(qingling128): Fix this temporary renaming from 'gke_container'
|
1042
|
-
# to 'container'.
|
1043
|
-
retrieved_resource.type = 'container' if
|
1044
|
-
retrieved_resource.type == 'gke_container'
|
1045
|
-
resource = retrieved_resource
|
1046
|
-
end
|
1047
|
-
end
|
998
|
+
# Determine group level monitored resource labels and common labels. These
|
999
|
+
# labels will be shared by a collection of entries.
|
1000
|
+
def determine_group_level_labels(group_resource_type, matched_regex_group)
|
1001
|
+
group_resource_labels = @resource.labels.dup
|
1002
|
+
group_common_labels = @common_labels.dup
|
1048
1003
|
|
1049
|
-
|
1050
|
-
case resource.type
|
1004
|
+
case group_resource_type
|
1051
1005
|
# Cloud Functions.
|
1052
1006
|
when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
1053
|
-
|
1007
|
+
group_resource_labels.merge!(
|
1054
1008
|
'region' => @gcf_region,
|
1055
1009
|
'function_name' => decode_cloudfunctions_function_name(
|
1056
|
-
|
1010
|
+
matched_regex_group['encoded_function_name'])
|
1057
1011
|
)
|
1058
|
-
|
1059
|
-
|
1060
|
-
|
1012
|
+
|
1013
|
+
instance_id = group_resource_labels.delete('instance_id')
|
1014
|
+
group_common_labels.merge!(
|
1015
|
+
"#{CONTAINER_CONSTANTS[:service]}/instance_id" => instance_id,
|
1061
1016
|
"#{COMPUTE_CONSTANTS[:service]}/resource_id" => instance_id,
|
1062
|
-
"#{
|
1063
|
-
|
1017
|
+
"#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
|
1018
|
+
group_resource_labels.delete('cluster_name'),
|
1064
1019
|
"#{COMPUTE_CONSTANTS[:service]}/zone" =>
|
1065
|
-
|
1020
|
+
group_resource_labels.delete('zone')
|
1066
1021
|
)
|
1067
1022
|
|
1068
1023
|
# GKE container.
|
1069
|
-
when
|
1070
|
-
if
|
1024
|
+
when CONTAINER_CONSTANTS[:resource_type]
|
1025
|
+
if matched_regex_group
|
1071
1026
|
# We only expect one occurrence of each key in the match group.
|
1072
1027
|
resource_labels_candidates =
|
1073
|
-
|
1074
|
-
common_labels_candidates =
|
1075
|
-
|
1028
|
+
matched_regex_group.names.zip(matched_regex_group.captures).to_h
|
1029
|
+
common_labels_candidates =
|
1030
|
+
resource_labels_candidates.dup
|
1031
|
+
group_resource_labels.merge!(
|
1076
1032
|
delete_and_extract_labels(
|
1077
1033
|
resource_labels_candidates,
|
1078
1034
|
# The kubernetes_tag_regexp is poorly named. 'namespace_name' is
|
1079
1035
|
# in fact 'namespace_id'. 'pod_name' is in fact 'pod_id'.
|
1080
1036
|
# TODO(qingling128): Figure out how to put this map into
|
1081
|
-
# constants like
|
1037
|
+
# constants like CONTAINER_CONSTANTS[:extra_resource_labels].
|
1082
1038
|
'container_name' => 'container_name',
|
1083
1039
|
'namespace_name' => 'namespace_id',
|
1084
1040
|
'pod_name' => 'pod_id'))
|
1085
1041
|
|
1086
|
-
|
1042
|
+
group_common_labels.merge!(
|
1087
1043
|
delete_and_extract_labels(
|
1088
1044
|
common_labels_candidates,
|
1089
|
-
|
1090
|
-
.map { |l| [l, "#{
|
1045
|
+
CONTAINER_CONSTANTS[:extra_common_labels]
|
1046
|
+
.map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
|
1091
1047
|
end
|
1092
|
-
|
1093
|
-
# Docker container.
|
1094
|
-
# TODO(qingling128): Remove this logic once the resource is retrieved at a
|
1095
|
-
# proper time (b/65175256).
|
1096
|
-
when DOCKER_CONSTANTS[:resource_type]
|
1097
|
-
common_labels.delete("#{COMPUTE_CONSTANTS[:service]}/resource_name")
|
1098
1048
|
end
|
1099
1049
|
|
1100
|
-
|
1101
|
-
resource.labels.freeze
|
1102
|
-
common_labels.freeze
|
1103
|
-
|
1104
|
-
[resource, common_labels]
|
1050
|
+
[group_resource_labels, group_common_labels]
|
1105
1051
|
end
|
1106
1052
|
|
1107
|
-
# Extract entry
|
1108
|
-
#
|
1109
|
-
def
|
1110
|
-
|
1111
|
-
|
1112
|
-
resource.labels = group_level_resource.labels.dup
|
1113
|
-
common_labels = group_level_common_labels.dup
|
1053
|
+
# Extract entry resource and common labels that should be applied to
|
1054
|
+
# individual entries from the group resource.
|
1055
|
+
def determine_entry_level_labels(group_resource, record)
|
1056
|
+
resource_labels = {}
|
1057
|
+
common_labels = {}
|
1114
1058
|
|
1115
|
-
case resource.type
|
1116
1059
|
# Cloud Functions.
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1060
|
+
if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
|
1061
|
+
record.key?('log')
|
1062
|
+
@cloudfunctions_log_match =
|
1063
|
+
@compiled_cloudfunctions_log_regexp.match(record['log'])
|
1064
|
+
common_labels['execution_id'] =
|
1065
|
+
@cloudfunctions_log_match['execution_id'] if \
|
1066
|
+
@cloudfunctions_log_match &&
|
1067
|
+
@cloudfunctions_log_match['execution_id']
|
1068
|
+
end
|
1126
1069
|
|
1127
|
-
# GKE
|
1128
|
-
|
1070
|
+
# GKE containers.
|
1071
|
+
if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
|
1129
1072
|
# Move the stdout/stderr annotation from the record into a label.
|
1130
1073
|
common_labels.merge!(
|
1131
1074
|
delete_and_extract_labels(
|
1132
|
-
record, 'stream' => "#{
|
1075
|
+
record, 'stream' => "#{CONTAINER_CONSTANTS[:service]}/stream"))
|
1133
1076
|
|
1134
1077
|
# If the record has been annotated by the kubernetes_metadata_filter
|
1135
1078
|
# plugin, then use that metadata. Otherwise, rely on commonLabels
|
1136
|
-
# populated from the group's tag.
|
1079
|
+
# populated at the grouped_entries level from the group's tag.
|
1137
1080
|
if record.key?('kubernetes')
|
1138
|
-
|
1081
|
+
resource_labels.merge!(
|
1139
1082
|
delete_and_extract_labels(
|
1140
|
-
record['kubernetes'],
|
1083
|
+
record['kubernetes'], CONTAINER_CONSTANTS[:extra_resource_labels]
|
1141
1084
|
.map { |l| [l, l] }.to_h))
|
1142
1085
|
common_labels.merge!(
|
1143
1086
|
delete_and_extract_labels(
|
1144
|
-
record['kubernetes'],
|
1145
|
-
.map { |l| [l, "#{
|
1087
|
+
record['kubernetes'], CONTAINER_CONSTANTS[:extra_common_labels]
|
1088
|
+
.map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
|
1146
1089
|
# Prepend label/ to all user-defined labels' keys.
|
1147
1090
|
if record['kubernetes'].key?('labels')
|
1148
1091
|
common_labels.merge!(
|
@@ -1168,56 +1111,14 @@ module Fluent
|
|
1168
1111
|
# Report them as monitored resource labels instead of common labels.
|
1169
1112
|
# e.g. "dataflow.googleapis.com/job_id" => "job_id"
|
1170
1113
|
[DATAFLOW_CONSTANTS, ML_CONSTANTS].each do |service_constants|
|
1171
|
-
next unless
|
1172
|
-
|
1114
|
+
next unless group_resource.type == service_constants[:resource_type]
|
1115
|
+
resource_labels.merge!(
|
1173
1116
|
delete_and_extract_labels(
|
1174
1117
|
common_labels, service_constants[:extra_common_labels]
|
1175
1118
|
.map { |l| ["#{service_constants[:service]}/#{l}", l] }.to_h))
|
1176
1119
|
end
|
1177
1120
|
|
1178
|
-
[
|
1179
|
-
end
|
1180
|
-
|
1181
|
-
# Call Metadata Agent to get monitored resource information and parse
|
1182
|
-
# response to Google::Api::MonitoredResource.
|
1183
|
-
def call_metadata_agent_for_monitored_resource(local_resource_id)
|
1184
|
-
response = query_metadata_agent("monitoredResource/#{local_resource_id}")
|
1185
|
-
return nil if response.nil?
|
1186
|
-
begin
|
1187
|
-
resource = Google::Api::MonitoredResource.decode_json(response.to_json)
|
1188
|
-
rescue Google::Protobuf::ParseError, ArgumentError => e
|
1189
|
-
@log.error 'Error paring monitored resource from Metadata Agent. ' \
|
1190
|
-
"response: #{response.inspect}", error: e
|
1191
|
-
return nil
|
1192
|
-
end
|
1193
|
-
|
1194
|
-
# TODO(qingling128): Use Google::Api::MonitoredResource directly after we
|
1195
|
-
# upgrade gRPC version to include the fix for the protobuf map
|
1196
|
-
# corruption issue.
|
1197
|
-
Google::Apis::LoggingV2beta1::MonitoredResource.new(
|
1198
|
-
type: resource.type,
|
1199
|
-
labels: resource.labels.to_h
|
1200
|
-
)
|
1201
|
-
end
|
1202
|
-
|
1203
|
-
# Issue a request to the Metadata Agent's local API and parse the response
|
1204
|
-
# to JSON. Return nil in case of failure.
|
1205
|
-
def query_metadata_agent(path)
|
1206
|
-
url = "#{@metadata_agent_url}/#{path}"
|
1207
|
-
@log.debug("Calling Metadata Agent: #{url}")
|
1208
|
-
open(url) do |f|
|
1209
|
-
response = f.read
|
1210
|
-
parsed_hash = parse_json_or_nil(response)
|
1211
|
-
if parsed_hash.nil?
|
1212
|
-
@log.error 'Response from Metadata Agent is not in valid json ' \
|
1213
|
-
"format: '#{response.inspect}'."
|
1214
|
-
return nil
|
1215
|
-
end
|
1216
|
-
@log.debug "Response from Metadata Agent: #{parsed_hash}"
|
1217
|
-
return parsed_hash
|
1218
|
-
end
|
1219
|
-
rescue StandardError => e
|
1220
|
-
@log.error 'Error calling Metadata Agent.', error: e
|
1121
|
+
[resource_labels, common_labels]
|
1221
1122
|
end
|
1222
1123
|
|
1223
1124
|
# TODO: This functionality should eventually be available in another
|
@@ -1310,7 +1211,7 @@ module Fluent
|
|
1310
1211
|
[ts_secs, ts_nanos]
|
1311
1212
|
end
|
1312
1213
|
|
1313
|
-
def compute_severity(resource_type, record,
|
1214
|
+
def compute_severity(resource_type, record, entry_common_labels)
|
1314
1215
|
if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
|
1315
1216
|
if @cloudfunctions_log_match && @cloudfunctions_log_match['severity']
|
1316
1217
|
return parse_severity(@cloudfunctions_log_match['severity'])
|
@@ -1325,8 +1226,9 @@ module Fluent
|
|
1325
1226
|
end
|
1326
1227
|
elsif record.key?('severity')
|
1327
1228
|
return parse_severity(record.delete('severity'))
|
1328
|
-
elsif resource_type ==
|
1329
|
-
|
1229
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1230
|
+
entry_common_labels.key?("#{CONTAINER_CONSTANTS[:service]}/stream")
|
1231
|
+
stream = entry_common_labels["#{CONTAINER_CONSTANTS[:service]}/stream"]
|
1330
1232
|
if stream == 'stdout'
|
1331
1233
|
return 'INFO'
|
1332
1234
|
elsif stream == 'stderr'
|
@@ -1550,8 +1452,9 @@ module Fluent
|
|
1550
1452
|
hash.nil? || !hash.is_a?(Hash)
|
1551
1453
|
label_map.each_with_object({}) \
|
1552
1454
|
do |(original_label, new_label), extracted_labels|
|
1553
|
-
|
1554
|
-
|
1455
|
+
extracted_labels[new_label] =
|
1456
|
+
convert_to_utf8(hash.delete(original_label).to_s) if
|
1457
|
+
hash.key?(original_label)
|
1555
1458
|
end
|
1556
1459
|
end
|
1557
1460
|
|
@@ -1617,8 +1520,7 @@ module Fluent
|
|
1617
1520
|
text_payload = record['log']
|
1618
1521
|
elsif is_json
|
1619
1522
|
json_payload = record
|
1620
|
-
elsif [
|
1621
|
-
DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
|
1523
|
+
elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
|
1622
1524
|
record.key?('log')
|
1623
1525
|
text_payload = record['log']
|
1624
1526
|
elsif record.size == 1 && record.key?('message')
|
@@ -1648,7 +1550,7 @@ module Fluent
|
|
1648
1550
|
elsif resource.type == APPENGINE_CONSTANTS[:resource_type]
|
1649
1551
|
# Add a prefix to Managed VM logs to prevent namespace collisions.
|
1650
1552
|
tag = "#{APPENGINE_CONSTANTS[:service]}/#{tag}"
|
1651
|
-
elsif resource.type ==
|
1553
|
+
elsif resource.type == CONTAINER_CONSTANTS[:resource_type]
|
1652
1554
|
# For Kubernetes logs, use just the container name as the log name
|
1653
1555
|
# if we have it.
|
1654
1556
|
if resource.labels && resource.labels.key?('container_name')
|