fluent-plugin-google-cloud 0.8.2 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +14 -14
- data/fluent-plugin-google-cloud.gemspec +1 -1
- data/lib/fluent/plugin/common.rb +381 -0
- data/lib/fluent/plugin/filter_analyze_config.rb +306 -0
- data/lib/fluent/plugin/out_google_cloud.rb +29 -358
- data/test/plugin/asserts.rb +75 -0
- data/test/plugin/base_test.rb +3 -60
- data/test/plugin/constants.rb +9 -1
- data/test/plugin/data/google-fluentd-baseline.conf +24 -0
- data/test/plugin/data/google-fluentd-custom.conf +40 -0
- data/test/plugin/test_filter_analyze_config.rb +187 -0
- metadata +13 -3
@@ -0,0 +1,306 @@
|
|
1
|
+
# Copyright 2020 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'fileutils'
|
16
|
+
require 'fluent/config'
|
17
|
+
require 'fluent/config/v1_parser'
|
18
|
+
require 'set'
|
19
|
+
|
20
|
+
require_relative 'monitoring'
|
21
|
+
|
22
|
+
module Fluent
|
23
|
+
# Fluentd filter plugin to analyze configuration usage.
|
24
|
+
#
|
25
|
+
# For documentation on inspecting parsed configuration elements, see
|
26
|
+
# https://www.rubydoc.info/github/fluent/fluentd/Fluent/Config/Element
|
27
|
+
class AnalyzeConfigFilter < Filter
|
28
|
+
include Fluent::Config
|
29
|
+
Fluent::Plugin.register_filter('analyze_config', self)
|
30
|
+
|
31
|
+
module Constants
|
32
|
+
# Built-in plugins that are ok to reference in metrics.
|
33
|
+
KNOWN_PLUGINS = {
|
34
|
+
'filter' => Set[
|
35
|
+
'geoip',
|
36
|
+
'grep',
|
37
|
+
'parser',
|
38
|
+
'record_transformer',
|
39
|
+
'stdout',
|
40
|
+
],
|
41
|
+
'match' => Set[
|
42
|
+
'copy',
|
43
|
+
'elasticsearch',
|
44
|
+
'exec',
|
45
|
+
'exec_filter',
|
46
|
+
'file',
|
47
|
+
'forward',
|
48
|
+
'http',
|
49
|
+
'kafka',
|
50
|
+
'mongo',
|
51
|
+
'mongo_replset',
|
52
|
+
'null',
|
53
|
+
'relabel',
|
54
|
+
'rewrite_tag_filter',
|
55
|
+
'roundrobin',
|
56
|
+
's3',
|
57
|
+
'secondary_file',
|
58
|
+
'stdout',
|
59
|
+
'webhdfs',
|
60
|
+
],
|
61
|
+
'source' => Set[
|
62
|
+
'dummy',
|
63
|
+
'exec',
|
64
|
+
'forward',
|
65
|
+
'http',
|
66
|
+
'monitor_agent',
|
67
|
+
'syslog',
|
68
|
+
'tail',
|
69
|
+
'tcp',
|
70
|
+
'udp',
|
71
|
+
'unix',
|
72
|
+
'windows_eventlog',
|
73
|
+
]
|
74
|
+
}.freeze
|
75
|
+
|
76
|
+
# For Google plugins, we collect metrics on the params listed here.
|
77
|
+
GOOGLE_PLUGIN_PARAMS = {
|
78
|
+
'google_cloud' => %w(
|
79
|
+
adjust_invalid_timestamps
|
80
|
+
auth_method
|
81
|
+
autoformat_stackdriver_trace
|
82
|
+
coerce_to_utf8
|
83
|
+
detect_json
|
84
|
+
enable_monitoring
|
85
|
+
gcm_service_address
|
86
|
+
grpc_compression_algorithm
|
87
|
+
http_request_key
|
88
|
+
insert_id_key
|
89
|
+
label_map
|
90
|
+
labels
|
91
|
+
labels_key
|
92
|
+
logging_api_url
|
93
|
+
monitoring_type
|
94
|
+
non_utf8_replacement_string
|
95
|
+
operation_key
|
96
|
+
private_key_email
|
97
|
+
private_key_passphrase
|
98
|
+
private_key_path
|
99
|
+
project_id
|
100
|
+
source_location_key
|
101
|
+
span_id_key
|
102
|
+
statusz_port
|
103
|
+
trace_key
|
104
|
+
trace_sampled_key
|
105
|
+
use_grpc
|
106
|
+
use_metadata_service
|
107
|
+
vm_id
|
108
|
+
vm_name
|
109
|
+
zone
|
110
|
+
),
|
111
|
+
'detect_exceptions' => %w(
|
112
|
+
languages
|
113
|
+
max_bytes
|
114
|
+
max_lines
|
115
|
+
message
|
116
|
+
multiline_flush_interval
|
117
|
+
remove_tag_prefix
|
118
|
+
stream
|
119
|
+
)
|
120
|
+
}.freeze
|
121
|
+
end
|
122
|
+
|
123
|
+
include self::Constants
|
124
|
+
|
125
|
+
# The root configuration file of google-fluentd package.
|
126
|
+
# This only applies to Linux.
|
127
|
+
config_param :google_fluentd_config_path,
|
128
|
+
:string,
|
129
|
+
default: '/etc/google-fluentd/google-fluentd.conf'
|
130
|
+
# Baseline configuration for comparing with local
|
131
|
+
# customizations.
|
132
|
+
config_param :google_fluentd_baseline_config_path,
|
133
|
+
:string,
|
134
|
+
default: '/etc/google-fluentd/baseline/google-fluentd.conf'
|
135
|
+
|
136
|
+
def start
|
137
|
+
super
|
138
|
+
@log = $log # rubocop:disable Style/GlobalVars
|
139
|
+
|
140
|
+
# Initialize the insertID.
|
141
|
+
@log.info 'Started the analyze_config plugin to analyze configuration.'
|
142
|
+
end
|
143
|
+
|
144
|
+
def parse_config(path)
|
145
|
+
data = File.open(path, 'r', &:read)
|
146
|
+
fname = File.basename(path)
|
147
|
+
basepath = File.dirname(path)
|
148
|
+
eval_context = Kernel.binding
|
149
|
+
# Override instance_eval so that LiteralParser does not actually
|
150
|
+
# evaluate the embedded Ruby, but instead just returns the
|
151
|
+
# source string. See
|
152
|
+
# https://github.com/fluent/fluentd/blob/master/lib/fluent/config/literal_parser.rb
|
153
|
+
def eval_context.instance_eval(code)
|
154
|
+
code
|
155
|
+
end
|
156
|
+
Fluent::Config::V1Parser.parse(data, fname, basepath, eval_context)
|
157
|
+
end
|
158
|
+
|
159
|
+
# Returns a name for identifying plugins we ship by default.
|
160
|
+
def default_plugin_name(e)
|
161
|
+
case e['@type']
|
162
|
+
when 'syslog'
|
163
|
+
"#{e.name}/syslog/#{e['protocol_type']}"
|
164
|
+
when 'tail'
|
165
|
+
"#{e.name}/tail/#{File.basename(e['pos_file'], '.pos')}"
|
166
|
+
else
|
167
|
+
"#{e.name}/#{e['@type']}"
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
# Returns a name for identifying plugins not in our default
|
172
|
+
# config. This should not contain arbitrary user-supplied data.
|
173
|
+
def custom_plugin_name(e)
|
174
|
+
if KNOWN_PLUGINS.key?(e.name) &&
|
175
|
+
KNOWN_PLUGINS[e.name].include?(e['@type'])
|
176
|
+
"#{e.name}/#{e['@type']}"
|
177
|
+
else
|
178
|
+
e.name.to_s
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def embedded_ruby?(e)
|
183
|
+
(e.arg.include?('#{') ||
|
184
|
+
e.any? { |_, v| v.include?('#{') } ||
|
185
|
+
e.elements.any? { |ee| embedded_ruby?(ee) })
|
186
|
+
end
|
187
|
+
|
188
|
+
def configure(conf)
|
189
|
+
super
|
190
|
+
if File.file?(@google_fluentd_config_path) &&
|
191
|
+
File.file?(@google_fluentd_baseline_config_path)
|
192
|
+
@log.info(
|
193
|
+
'google-fluentd configuration file found at' \
|
194
|
+
" #{@google_fluentd_config_path}. " \
|
195
|
+
'google-fluentd baseline configuration file found at' \
|
196
|
+
" #{@google_fluentd_baseline_config_path}. " \
|
197
|
+
'google-fluentd Analyzing configuration.')
|
198
|
+
|
199
|
+
# TODO: Add OpenCensus support.
|
200
|
+
registry = Monitoring::MonitoringRegistryFactory.create(
|
201
|
+
Monitoring::PrometheusMonitoringRegistry.name, nil, nil, nil)
|
202
|
+
|
203
|
+
plugin_usage = registry.counter(
|
204
|
+
:stackdriver_enabled_plugins,
|
205
|
+
[:plugin_name, :is_default_plugin, :has_default_value],
|
206
|
+
'Enabled plugins')
|
207
|
+
config_usage = registry.counter(
|
208
|
+
:stackdriver_config_usage,
|
209
|
+
[:plugin_name, :param, :is_present, :has_default_value],
|
210
|
+
'Parameter usage for Google Cloud plugins')
|
211
|
+
config_bool_values = registry.counter(
|
212
|
+
:stackdriver_config_bool_values,
|
213
|
+
[:plugin_name, :param, :value],
|
214
|
+
'Values for bool parameters in Google Cloud plugins')
|
215
|
+
|
216
|
+
config = parse_config(@google_fluentd_config_path)
|
217
|
+
baseline_config = parse_config(@google_fluentd_baseline_config_path)
|
218
|
+
|
219
|
+
# Create hash of all baseline elements by their plugin names.
|
220
|
+
baseline_elements = Hash[baseline_config.elements.collect do |e|
|
221
|
+
[default_plugin_name(e), e]
|
222
|
+
end]
|
223
|
+
baseline_google_element = baseline_config.elements.find do |e|
|
224
|
+
e['@type'] == 'google_cloud'
|
225
|
+
end
|
226
|
+
|
227
|
+
# Look at each top-level config element and see whether it
|
228
|
+
# matches the baseline value.
|
229
|
+
#
|
230
|
+
# Note on custom configurations: If the plugin has a custom
|
231
|
+
# value (e.g. if a tail plugin has pos_file
|
232
|
+
# /var/lib/google-fluentd/pos/my-custom-value.pos), then the
|
233
|
+
# default_plugin_name (e.g. source/tail/my-custom-value) won't
|
234
|
+
# be a key in baseline_elements below, so it won't be
|
235
|
+
# used. Instead it will use the custom_plugin_name
|
236
|
+
# (e.g. source/tail).
|
237
|
+
config.elements.each do |e|
|
238
|
+
plugin_name = default_plugin_name(e)
|
239
|
+
if baseline_elements.key?(plugin_name)
|
240
|
+
is_default_plugin = true
|
241
|
+
has_default_value = (baseline_elements[plugin_name] == e)
|
242
|
+
else
|
243
|
+
plugin_name = custom_plugin_name(e)
|
244
|
+
is_default_plugin = false
|
245
|
+
has_default_value = false
|
246
|
+
end
|
247
|
+
plugin_usage.increment(
|
248
|
+
labels: {
|
249
|
+
plugin_name: plugin_name,
|
250
|
+
is_default_plugin: is_default_plugin,
|
251
|
+
has_default_value: has_default_value,
|
252
|
+
has_ruby_snippet: embedded_ruby?(e)
|
253
|
+
},
|
254
|
+
by: 1)
|
255
|
+
|
256
|
+
# Additional metric for Google plugins (google_cloud and
|
257
|
+
# detect_exceptions).
|
258
|
+
next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
|
259
|
+
GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
|
260
|
+
config_usage.increment(
|
261
|
+
labels: {
|
262
|
+
plugin_name: e['@type'],
|
263
|
+
param: p,
|
264
|
+
is_present: e.key?(p),
|
265
|
+
has_default_value: (e.key?(p) &&
|
266
|
+
baseline_google_element.key?(p) &&
|
267
|
+
e[p] == baseline_google_element[p])
|
268
|
+
},
|
269
|
+
by: 1)
|
270
|
+
next unless e.key?(p) && %w(true false).include?(e[p])
|
271
|
+
config_bool_values.increment(
|
272
|
+
labels: {
|
273
|
+
plugin_name: e['@type'],
|
274
|
+
param: p,
|
275
|
+
value: e[p] == 'true'
|
276
|
+
},
|
277
|
+
by: 1)
|
278
|
+
end
|
279
|
+
end
|
280
|
+
else
|
281
|
+
@log.info(
|
282
|
+
'google-fluentd configuration file does not exist at' \
|
283
|
+
" #{@google_fluentd_config_path} or " \
|
284
|
+
'google-fluentd baseline configuration file does not exist at' \
|
285
|
+
" #{@google_fluentd_baseline_config_path} or " \
|
286
|
+
'. Skipping configuration analysis.')
|
287
|
+
end
|
288
|
+
rescue => e
|
289
|
+
# Do not crash the agent due to configuration analysis failures.
|
290
|
+
@log.warn(
|
291
|
+
'Failed to optionally analyze the google-fluentd configuration' \
|
292
|
+
" file. Proceeding anyway. Error: #{e}")
|
293
|
+
end
|
294
|
+
|
295
|
+
def shutdown
|
296
|
+
super
|
297
|
+
end
|
298
|
+
|
299
|
+
# rubocop:disable Lint/UnusedMethodArgument
|
300
|
+
def filter(tag, time, record)
|
301
|
+
# Skip the actual filtering process.
|
302
|
+
record
|
303
|
+
end
|
304
|
+
# rubocop:enable Lint/UnusedMethodArgument
|
305
|
+
end
|
306
|
+
end
|
@@ -28,6 +28,7 @@ require 'google/logging/v2/logging_services_pb'
|
|
28
28
|
require 'google/logging/v2/log_entry_pb'
|
29
29
|
require 'googleauth'
|
30
30
|
|
31
|
+
require_relative 'common'
|
31
32
|
require_relative 'monitoring'
|
32
33
|
require_relative 'statusz'
|
33
34
|
|
@@ -90,73 +91,6 @@ end
|
|
90
91
|
module Fluent
|
91
92
|
# fluentd output plugin for the Stackdriver Logging API
|
92
93
|
class GoogleCloudOutput < BufferedOutput
|
93
|
-
# Constants for service names, resource types and etc.
|
94
|
-
module ServiceConstants
|
95
|
-
APPENGINE_CONSTANTS = {
|
96
|
-
service: 'appengine.googleapis.com',
|
97
|
-
resource_type: 'gae_app',
|
98
|
-
metadata_attributes: %w(gae_backend_name gae_backend_version)
|
99
|
-
}.freeze
|
100
|
-
COMPUTE_CONSTANTS = {
|
101
|
-
service: 'compute.googleapis.com',
|
102
|
-
resource_type: 'gce_instance'
|
103
|
-
}.freeze
|
104
|
-
GKE_CONSTANTS = {
|
105
|
-
service: 'container.googleapis.com',
|
106
|
-
resource_type: 'container',
|
107
|
-
extra_resource_labels: %w(namespace_id pod_id container_name),
|
108
|
-
extra_common_labels: %w(namespace_name pod_name),
|
109
|
-
metadata_attributes: %w(cluster-name cluster-location),
|
110
|
-
stream_severity_map: {
|
111
|
-
'stdout' => 'INFO',
|
112
|
-
'stderr' => 'ERROR'
|
113
|
-
}
|
114
|
-
}.freeze
|
115
|
-
K8S_CONTAINER_CONSTANTS = {
|
116
|
-
resource_type: 'k8s_container'
|
117
|
-
}.freeze
|
118
|
-
K8S_POD_CONSTANTS = {
|
119
|
-
resource_type: 'k8s_pod'
|
120
|
-
}.freeze
|
121
|
-
K8S_NODE_CONSTANTS = {
|
122
|
-
resource_type: 'k8s_node'
|
123
|
-
}.freeze
|
124
|
-
DATAFLOW_CONSTANTS = {
|
125
|
-
service: 'dataflow.googleapis.com',
|
126
|
-
resource_type: 'dataflow_step',
|
127
|
-
extra_resource_labels: %w(region job_name job_id step_id)
|
128
|
-
}.freeze
|
129
|
-
DATAPROC_CONSTANTS = {
|
130
|
-
service: 'cluster.dataproc.googleapis.com',
|
131
|
-
resource_type: 'cloud_dataproc_cluster',
|
132
|
-
metadata_attributes: %w(dataproc-cluster-uuid dataproc-cluster-name)
|
133
|
-
}.freeze
|
134
|
-
EC2_CONSTANTS = {
|
135
|
-
service: 'ec2.amazonaws.com',
|
136
|
-
resource_type: 'aws_ec2_instance'
|
137
|
-
}.freeze
|
138
|
-
ML_CONSTANTS = {
|
139
|
-
service: 'ml.googleapis.com',
|
140
|
-
resource_type: 'ml_job',
|
141
|
-
extra_resource_labels: %w(job_id task_name)
|
142
|
-
}.freeze
|
143
|
-
|
144
|
-
# The map between a subservice name and a resource type.
|
145
|
-
SUBSERVICE_MAP =
|
146
|
-
[APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAFLOW_CONSTANTS,
|
147
|
-
DATAPROC_CONSTANTS, ML_CONSTANTS]
|
148
|
-
.map { |consts| [consts[:service], consts[:resource_type]] }.to_h
|
149
|
-
# Default back to GCE if invalid value is detected.
|
150
|
-
SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
|
151
|
-
SUBSERVICE_MAP.freeze
|
152
|
-
|
153
|
-
# The map between a resource type and expected subservice attributes.
|
154
|
-
SUBSERVICE_METADATA_ATTRIBUTES =
|
155
|
-
[APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAPROC_CONSTANTS].map do |consts|
|
156
|
-
[consts[:resource_type], consts[:metadata_attributes].to_set]
|
157
|
-
end.to_h.freeze
|
158
|
-
end
|
159
|
-
|
160
94
|
# Constants for configuration.
|
161
95
|
module ConfigConstants
|
162
96
|
# Default values for JSON payload keys to set the "httpRequest",
|
@@ -247,7 +181,7 @@ module Fluent
|
|
247
181
|
.freeze
|
248
182
|
end
|
249
183
|
|
250
|
-
include
|
184
|
+
include Common::ServiceConstants
|
251
185
|
include self::ConfigConstants
|
252
186
|
include self::InternalConstants
|
253
187
|
|
@@ -278,12 +212,6 @@ module Fluent
|
|
278
212
|
end
|
279
213
|
end.freeze
|
280
214
|
|
281
|
-
# Name of the the Google cloud logging write scope.
|
282
|
-
LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
|
283
|
-
|
284
|
-
# Address of the metadata service.
|
285
|
-
METADATA_SERVICE_ADDR = '169.254.169.254'.freeze
|
286
|
-
|
287
215
|
# Disable this warning to conform to fluentd config_param conventions.
|
288
216
|
# rubocop:disable Style/HashSyntax
|
289
217
|
|
@@ -539,7 +467,9 @@ module Fluent
|
|
539
467
|
|
540
468
|
set_regexp_patterns
|
541
469
|
|
542
|
-
@
|
470
|
+
@utils = Common::Utils.new(@log)
|
471
|
+
|
472
|
+
@platform = @utils.detect_platform(@use_metadata_service)
|
543
473
|
|
544
474
|
# Treat an empty setting of the credentials file path environment variable
|
545
475
|
# as unset. This way the googleauth lib could fetch the credentials
|
@@ -548,12 +478,20 @@ module Fluent
|
|
548
478
|
ENV[CREDENTIALS_PATH_ENV_VAR] == ''
|
549
479
|
|
550
480
|
# Set required variables: @project_id, @vm_id, @vm_name and @zone.
|
551
|
-
|
481
|
+
@project_id = @utils.get_project_id(@platform, @project_id)
|
482
|
+
@vm_id = @utils.get_vm_id(@platform, @vm_id)
|
483
|
+
@vm_name = @utils.get_vm_name(@vm_name)
|
484
|
+
@zone = @utils.get_location(@platform, @zone, @use_aws_availability_zone)
|
485
|
+
|
486
|
+
# All metadata parameters must now be set.
|
487
|
+
@utils.check_required_metadata_variables(
|
488
|
+
@platform, @project_id, @zone, @vm_id)
|
552
489
|
|
553
490
|
# Retrieve monitored resource.
|
554
491
|
# Fail over to retrieve monitored resource via the legacy path if we fail
|
555
492
|
# to get it from Metadata Agent.
|
556
|
-
@resource ||= determine_agent_level_monitored_resource_via_legacy
|
493
|
+
@resource ||= @utils.determine_agent_level_monitored_resource_via_legacy(
|
494
|
+
@platform, @subservice_name, @detect_subservice, @vm_id, @zone)
|
557
495
|
|
558
496
|
# If monitoring is enabled, register metrics in the default registry
|
559
497
|
# and store metric objects for future use.
|
@@ -611,7 +549,7 @@ module Fluent
|
|
611
549
|
|
612
550
|
# Determine the common labels that should be added to all log entries
|
613
551
|
# processed by this logging agent.
|
614
|
-
@common_labels = determine_agent_level_common_labels
|
552
|
+
@common_labels = determine_agent_level_common_labels(@resource)
|
615
553
|
|
616
554
|
# The resource and labels are now set up; ensure they can't be modified
|
617
555
|
# without first duping them.
|
@@ -627,7 +565,7 @@ module Fluent
|
|
627
565
|
@write_request = method(:write_request_via_rest)
|
628
566
|
end
|
629
567
|
|
630
|
-
if [Platform::GCE, Platform::EC2].include?(@platform)
|
568
|
+
if [Common::Platform::GCE, Common::Platform::EC2].include?(@platform)
|
631
569
|
# Log an informational message containing the Logs viewer URL
|
632
570
|
@log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
|
633
571
|
"viewer?project=#{@project_id}&resource=#{@resource.type}/",
|
@@ -1094,66 +1032,6 @@ module Fluent
|
|
1094
1032
|
nil
|
1095
1033
|
end
|
1096
1034
|
|
1097
|
-
# "enum" of Platform values
|
1098
|
-
module Platform
|
1099
|
-
OTHER = 0 # Other/unkown platform
|
1100
|
-
GCE = 1 # Google Compute Engine
|
1101
|
-
EC2 = 2 # Amazon EC2
|
1102
|
-
end
|
1103
|
-
|
1104
|
-
# Determine what platform we are running on by consulting the metadata
|
1105
|
-
# service (unless the user has explicitly disabled using that).
|
1106
|
-
def detect_platform
|
1107
|
-
unless @use_metadata_service
|
1108
|
-
@log.info 'use_metadata_service is false; not detecting platform'
|
1109
|
-
return Platform::OTHER
|
1110
|
-
end
|
1111
|
-
|
1112
|
-
begin
|
1113
|
-
open('http://' + METADATA_SERVICE_ADDR, proxy: false) do |f|
|
1114
|
-
if f.meta['metadata-flavor'] == 'Google'
|
1115
|
-
@log.info 'Detected GCE platform'
|
1116
|
-
return Platform::GCE
|
1117
|
-
end
|
1118
|
-
if f.meta['server'] == 'EC2ws'
|
1119
|
-
@log.info 'Detected EC2 platform'
|
1120
|
-
return Platform::EC2
|
1121
|
-
end
|
1122
|
-
end
|
1123
|
-
rescue StandardError => e
|
1124
|
-
@log.error 'Failed to access metadata service: ', error: e
|
1125
|
-
end
|
1126
|
-
|
1127
|
-
@log.info 'Unable to determine platform'
|
1128
|
-
Platform::OTHER
|
1129
|
-
end
|
1130
|
-
|
1131
|
-
def fetch_gce_metadata(metadata_path)
|
1132
|
-
raise "Called fetch_gce_metadata with platform=#{@platform}" unless
|
1133
|
-
@platform == Platform::GCE
|
1134
|
-
# See https://cloud.google.com/compute/docs/metadata
|
1135
|
-
open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
|
1136
|
-
metadata_path, 'Metadata-Flavor' => 'Google', :proxy => false,
|
1137
|
-
&:read)
|
1138
|
-
end
|
1139
|
-
|
1140
|
-
# EC2 Metadata server returns everything in one call. Store it after the
|
1141
|
-
# first fetch to avoid making multiple calls.
|
1142
|
-
def ec2_metadata
|
1143
|
-
raise "Called ec2_metadata with platform=#{@platform}" unless
|
1144
|
-
@platform == Platform::EC2
|
1145
|
-
unless @ec2_metadata
|
1146
|
-
# See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
|
1147
|
-
open('http://' + METADATA_SERVICE_ADDR +
|
1148
|
-
'/latest/dynamic/instance-identity/document', proxy: false) do |f|
|
1149
|
-
contents = f.read
|
1150
|
-
@ec2_metadata = JSON.parse(contents)
|
1151
|
-
end
|
1152
|
-
end
|
1153
|
-
|
1154
|
-
@ec2_metadata
|
1155
|
-
end
|
1156
|
-
|
1157
1035
|
# Set regexp patterns to parse tags and logs.
|
1158
1036
|
def set_regexp_patterns
|
1159
1037
|
@compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp) if
|
@@ -1163,187 +1041,14 @@ module Fluent
|
|
1163
1041
|
/^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
|
1164
1042
|
end
|
1165
1043
|
|
1166
|
-
# Set required variables like @project_id, @vm_id, @vm_name and @zone.
|
1167
|
-
def set_required_metadata_variables
|
1168
|
-
set_project_id
|
1169
|
-
set_vm_id
|
1170
|
-
set_vm_name
|
1171
|
-
set_location
|
1172
|
-
|
1173
|
-
# All metadata parameters must now be set.
|
1174
|
-
missing = []
|
1175
|
-
missing << 'project_id' unless @project_id
|
1176
|
-
if @platform != Platform::OTHER
|
1177
|
-
missing << 'zone' unless @zone
|
1178
|
-
missing << 'vm_id' unless @vm_id
|
1179
|
-
end
|
1180
|
-
return if missing.empty?
|
1181
|
-
raise Fluent::ConfigError,
|
1182
|
-
"Unable to obtain metadata parameters: #{missing.join(' ')}"
|
1183
|
-
end
|
1184
|
-
|
1185
|
-
# 1. Return the value if it is explicitly set in the config already.
|
1186
|
-
# 2. If not, try to retrieve it by calling metadata server directly.
|
1187
|
-
# 3. If still not set, try to obtain it from the credentials.
|
1188
|
-
def set_project_id
|
1189
|
-
@project_id ||= CredentialsInfo.project_id
|
1190
|
-
@project_id ||= fetch_gce_metadata('project/project-id') if
|
1191
|
-
@platform == Platform::GCE
|
1192
|
-
end
|
1193
|
-
|
1194
|
-
# 1. Return the value if it is explicitly set in the config already.
|
1195
|
-
# 2. If not, try to retrieve it by calling metadata servers directly.
|
1196
|
-
def set_vm_id
|
1197
|
-
@vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
|
1198
|
-
@vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
|
1199
|
-
rescue StandardError => e
|
1200
|
-
@log.error 'Failed to obtain vm_id: ', error: e
|
1201
|
-
end
|
1202
|
-
|
1203
|
-
# 1. Return the value if it is explicitly set in the config already.
|
1204
|
-
# 2. If not, try to retrieve it locally.
|
1205
|
-
def set_vm_name
|
1206
|
-
@vm_name ||= Socket.gethostname
|
1207
|
-
rescue StandardError => e
|
1208
|
-
@log.error 'Failed to obtain vm name: ', error: e
|
1209
|
-
end
|
1210
|
-
|
1211
|
-
# 1. Return the value if it is explicitly set in the config already.
|
1212
|
-
# 2. If not, try to retrieve it locally.
|
1213
|
-
def set_location
|
1214
|
-
# Response format: "projects/<number>/zones/<zone>"
|
1215
|
-
@zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
|
1216
|
-
@platform == Platform::GCE
|
1217
|
-
aws_location_key = if @use_aws_availability_zone
|
1218
|
-
'availabilityZone'
|
1219
|
-
else
|
1220
|
-
'region'
|
1221
|
-
end
|
1222
|
-
@zone ||= 'aws:' + ec2_metadata[aws_location_key] if
|
1223
|
-
@platform == Platform::EC2 && ec2_metadata.key?(aws_location_key)
|
1224
|
-
rescue StandardError => e
|
1225
|
-
@log.error 'Failed to obtain location: ', error: e
|
1226
|
-
end
|
1227
|
-
|
1228
|
-
# Retrieve monitored resource via the legacy way.
|
1229
|
-
#
|
1230
|
-
# Note: This is just a failover plan if we fail to get metadata from
|
1231
|
-
# Metadata Agent. Thus it should be equivalent to what Metadata Agent
|
1232
|
-
# returns.
|
1233
|
-
def determine_agent_level_monitored_resource_via_legacy
|
1234
|
-
resource = Google::Apis::LoggingV2::MonitoredResource.new(
|
1235
|
-
labels: {})
|
1236
|
-
resource.type = determine_agent_level_monitored_resource_type
|
1237
|
-
resource.labels = determine_agent_level_monitored_resource_labels(
|
1238
|
-
resource.type)
|
1239
|
-
resource
|
1240
|
-
end
|
1241
|
-
|
1242
|
-
# Determine agent level monitored resource type.
|
1243
|
-
def determine_agent_level_monitored_resource_type
|
1244
|
-
case @platform
|
1245
|
-
when Platform::OTHER
|
1246
|
-
# Unknown platform will be defaulted to GCE instance.
|
1247
|
-
return COMPUTE_CONSTANTS[:resource_type]
|
1248
|
-
|
1249
|
-
when Platform::EC2
|
1250
|
-
return EC2_CONSTANTS[:resource_type]
|
1251
|
-
|
1252
|
-
when Platform::GCE
|
1253
|
-
# Resource types determined by @subservice_name config.
|
1254
|
-
return SUBSERVICE_MAP[@subservice_name] if @subservice_name
|
1255
|
-
|
1256
|
-
# Resource types determined by @detect_subservice config.
|
1257
|
-
if @detect_subservice
|
1258
|
-
begin
|
1259
|
-
attributes = fetch_gce_metadata('instance/attributes/').split.to_set
|
1260
|
-
SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
|
1261
|
-
return resource_type if attributes.superset?(expected)
|
1262
|
-
end
|
1263
|
-
rescue StandardError => e
|
1264
|
-
@log.error 'Failed to detect subservice: ', error: e
|
1265
|
-
end
|
1266
|
-
end
|
1267
|
-
|
1268
|
-
# GCE instance.
|
1269
|
-
return COMPUTE_CONSTANTS[:resource_type]
|
1270
|
-
end
|
1271
|
-
end
|
1272
|
-
|
1273
|
-
# Determine agent level monitored resource labels based on the resource
|
1274
|
-
# type. Each resource type has its own labels that need to be filled in.
|
1275
|
-
def determine_agent_level_monitored_resource_labels(type)
|
1276
|
-
case type
|
1277
|
-
# GAE app.
|
1278
|
-
when APPENGINE_CONSTANTS[:resource_type]
|
1279
|
-
return {
|
1280
|
-
'module_id' =>
|
1281
|
-
fetch_gce_metadata('instance/attributes/gae_backend_name'),
|
1282
|
-
'version_id' =>
|
1283
|
-
fetch_gce_metadata('instance/attributes/gae_backend_version')
|
1284
|
-
}
|
1285
|
-
|
1286
|
-
# GCE.
|
1287
|
-
when COMPUTE_CONSTANTS[:resource_type]
|
1288
|
-
raise "Cannot construct a #{type} resource without vm_id and zone" \
|
1289
|
-
unless @vm_id && @zone
|
1290
|
-
return {
|
1291
|
-
'instance_id' => @vm_id,
|
1292
|
-
'zone' => @zone
|
1293
|
-
}
|
1294
|
-
|
1295
|
-
# GKE container.
|
1296
|
-
when GKE_CONSTANTS[:resource_type]
|
1297
|
-
raise "Cannot construct a #{type} resource without vm_id and zone" \
|
1298
|
-
unless @vm_id && @zone
|
1299
|
-
return {
|
1300
|
-
'instance_id' => @vm_id,
|
1301
|
-
'zone' => @zone,
|
1302
|
-
'cluster_name' =>
|
1303
|
-
fetch_gce_metadata('instance/attributes/cluster-name')
|
1304
|
-
}
|
1305
|
-
|
1306
|
-
# Cloud Dataproc.
|
1307
|
-
when DATAPROC_CONSTANTS[:resource_type]
|
1308
|
-
return {
|
1309
|
-
'cluster_uuid' =>
|
1310
|
-
fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid'),
|
1311
|
-
'cluster_name' =>
|
1312
|
-
fetch_gce_metadata('instance/attributes/dataproc-cluster-name'),
|
1313
|
-
'region' =>
|
1314
|
-
fetch_gce_metadata('instance/attributes/dataproc-region')
|
1315
|
-
}
|
1316
|
-
|
1317
|
-
# EC2.
|
1318
|
-
when EC2_CONSTANTS[:resource_type]
|
1319
|
-
raise "Cannot construct a #{type} resource without vm_id and zone" \
|
1320
|
-
unless @vm_id && @zone
|
1321
|
-
labels = {
|
1322
|
-
'instance_id' => @vm_id,
|
1323
|
-
'region' => @zone
|
1324
|
-
}
|
1325
|
-
labels['aws_account'] = ec2_metadata['accountId'] if
|
1326
|
-
ec2_metadata.key?('accountId')
|
1327
|
-
return labels
|
1328
|
-
end
|
1329
|
-
|
1330
|
-
{}
|
1331
|
-
rescue StandardError => e
|
1332
|
-
if [Platform::GCE, Platform::EC2].include?(@platform)
|
1333
|
-
@log.error "Failed to set monitored resource labels for #{type}: ",
|
1334
|
-
error: e
|
1335
|
-
end
|
1336
|
-
{}
|
1337
|
-
end
|
1338
|
-
|
1339
1044
|
# Determine the common labels that should be added to all log entries
|
1340
1045
|
# processed by this logging agent.
|
1341
|
-
def determine_agent_level_common_labels
|
1046
|
+
def determine_agent_level_common_labels(resource)
|
1342
1047
|
labels = {}
|
1343
1048
|
# User can specify labels via config. We want to capture those as well.
|
1344
1049
|
labels.merge!(@labels) if @labels
|
1345
1050
|
|
1346
|
-
case
|
1051
|
+
case resource.type
|
1347
1052
|
# GAE, Cloud Dataflow, Cloud Dataproc and Cloud ML.
|
1348
1053
|
when APPENGINE_CONSTANTS[:resource_type],
|
1349
1054
|
DATAFLOW_CONSTANTS[:resource_type],
|
@@ -1548,44 +1253,6 @@ module Fluent
|
|
1548
1253
|
[resource, common_labels]
|
1549
1254
|
end
|
1550
1255
|
|
1551
|
-
# TODO: This functionality should eventually be available in another
|
1552
|
-
# library, but implement it ourselves for now.
|
1553
|
-
module CredentialsInfo
|
1554
|
-
# Determine the project ID from the credentials, if possible.
|
1555
|
-
# Returns the project ID (as a string) on success, or nil on failure.
|
1556
|
-
def self.project_id
|
1557
|
-
creds = Google::Auth.get_application_default(LOGGING_SCOPE)
|
1558
|
-
if creds.respond_to?(:project_id)
|
1559
|
-
return creds.project_id if creds.project_id
|
1560
|
-
end
|
1561
|
-
if creds.issuer
|
1562
|
-
id = extract_project_id(creds.issuer)
|
1563
|
-
return id unless id.nil?
|
1564
|
-
end
|
1565
|
-
if creds.client_id
|
1566
|
-
id = extract_project_id(creds.client_id)
|
1567
|
-
return id unless id.nil?
|
1568
|
-
end
|
1569
|
-
nil
|
1570
|
-
end
|
1571
|
-
|
1572
|
-
# Extracts the project id (either name or number) from str and returns
|
1573
|
-
# it (as a string) on success, or nil on failure.
|
1574
|
-
#
|
1575
|
-
# Recognizes IAM format (account@project-name.iam.gserviceaccount.com)
|
1576
|
-
# as well as the legacy format with a project number at the front of the
|
1577
|
-
# string, terminated by a dash (-) which is not part of the ID, i.e.:
|
1578
|
-
# 270694816269-1l1r2hb813leuppurdeik0apglbs80sv.apps.googleusercontent.com
|
1579
|
-
def self.extract_project_id(str)
|
1580
|
-
[/^.*@(?<project_id>.+)\.iam\.gserviceaccount\.com/,
|
1581
|
-
/^(?<project_id>\d+)-/].each do |exp|
|
1582
|
-
match_data = exp.match(str)
|
1583
|
-
return match_data['project_id'] unless match_data.nil?
|
1584
|
-
end
|
1585
|
-
nil
|
1586
|
-
end
|
1587
|
-
end
|
1588
|
-
|
1589
1256
|
def time_or_nil(ts_secs, ts_nanos)
|
1590
1257
|
Time.at((Integer ts_secs), (Integer ts_nanos) / 1_000.0)
|
1591
1258
|
rescue ArgumentError, TypeError
|
@@ -1909,7 +1576,11 @@ module Fluent
|
|
1909
1576
|
end
|
1910
1577
|
|
1911
1578
|
def format(tag, time, record)
|
1912
|
-
Fluent::
|
1579
|
+
Fluent::MessagePackFactory
|
1580
|
+
.engine_factory
|
1581
|
+
.packer
|
1582
|
+
.write([tag, time, record])
|
1583
|
+
.to_s
|
1913
1584
|
end
|
1914
1585
|
|
1915
1586
|
# Given a tag, returns the corresponding valid tag if possible, or nil if
|
@@ -2077,7 +1748,7 @@ module Fluent
|
|
2077
1748
|
Google::Apis::ClientOptions.default.application_version = PLUGIN_VERSION
|
2078
1749
|
@client = Google::Apis::LoggingV2::LoggingService.new
|
2079
1750
|
@client.authorization = Google::Auth.get_application_default(
|
2080
|
-
LOGGING_SCOPE)
|
1751
|
+
Common::LOGGING_SCOPE)
|
2081
1752
|
end
|
2082
1753
|
end
|
2083
1754
|
|
@@ -2306,10 +1977,10 @@ module Fluent
|
|
2306
1977
|
@k8s_cluster_location = nil if @k8s_cluster_location == ''
|
2307
1978
|
|
2308
1979
|
begin
|
2309
|
-
@k8s_cluster_name ||= fetch_gce_metadata(
|
2310
|
-
'instance/attributes/cluster-name')
|
2311
|
-
@k8s_cluster_location ||= fetch_gce_metadata(
|
2312
|
-
'instance/attributes/cluster-location')
|
1980
|
+
@k8s_cluster_name ||= @utils.fetch_gce_metadata(
|
1981
|
+
@platform, 'instance/attributes/cluster-name')
|
1982
|
+
@k8s_cluster_location ||= @utils.fetch_gce_metadata(
|
1983
|
+
@platform, 'instance/attributes/cluster-location')
|
2313
1984
|
rescue StandardError => e
|
2314
1985
|
@log.error 'Failed to retrieve k8s cluster name and location.', \
|
2315
1986
|
error: e
|