fluent-plugin-google-cloud 0.8.2 → 0.8.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,306 @@
1
+ # Copyright 2020 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'fileutils'
16
+ require 'fluent/config'
17
+ require 'fluent/config/v1_parser'
18
+ require 'set'
19
+
20
+ require_relative 'monitoring'
21
+
22
+ module Fluent
23
+ # Fluentd filter plugin to analyze configuration usage.
24
+ #
25
+ # For documentation on inspecting parsed configuration elements, see
26
+ # https://www.rubydoc.info/github/fluent/fluentd/Fluent/Config/Element
27
+ class AnalyzeConfigFilter < Filter
28
+ include Fluent::Config
29
+ Fluent::Plugin.register_filter('analyze_config', self)
30
+
31
+ module Constants
32
+ # Built-in plugins that are ok to reference in metrics.
33
+ KNOWN_PLUGINS = {
34
+ 'filter' => Set[
35
+ 'geoip',
36
+ 'grep',
37
+ 'parser',
38
+ 'record_transformer',
39
+ 'stdout',
40
+ ],
41
+ 'match' => Set[
42
+ 'copy',
43
+ 'elasticsearch',
44
+ 'exec',
45
+ 'exec_filter',
46
+ 'file',
47
+ 'forward',
48
+ 'http',
49
+ 'kafka',
50
+ 'mongo',
51
+ 'mongo_replset',
52
+ 'null',
53
+ 'relabel',
54
+ 'rewrite_tag_filter',
55
+ 'roundrobin',
56
+ 's3',
57
+ 'secondary_file',
58
+ 'stdout',
59
+ 'webhdfs',
60
+ ],
61
+ 'source' => Set[
62
+ 'dummy',
63
+ 'exec',
64
+ 'forward',
65
+ 'http',
66
+ 'monitor_agent',
67
+ 'syslog',
68
+ 'tail',
69
+ 'tcp',
70
+ 'udp',
71
+ 'unix',
72
+ 'windows_eventlog',
73
+ ]
74
+ }.freeze
75
+
76
+ # For Google plugins, we collect metrics on the params listed here.
77
+ GOOGLE_PLUGIN_PARAMS = {
78
+ 'google_cloud' => %w(
79
+ adjust_invalid_timestamps
80
+ auth_method
81
+ autoformat_stackdriver_trace
82
+ coerce_to_utf8
83
+ detect_json
84
+ enable_monitoring
85
+ gcm_service_address
86
+ grpc_compression_algorithm
87
+ http_request_key
88
+ insert_id_key
89
+ label_map
90
+ labels
91
+ labels_key
92
+ logging_api_url
93
+ monitoring_type
94
+ non_utf8_replacement_string
95
+ operation_key
96
+ private_key_email
97
+ private_key_passphrase
98
+ private_key_path
99
+ project_id
100
+ source_location_key
101
+ span_id_key
102
+ statusz_port
103
+ trace_key
104
+ trace_sampled_key
105
+ use_grpc
106
+ use_metadata_service
107
+ vm_id
108
+ vm_name
109
+ zone
110
+ ),
111
+ 'detect_exceptions' => %w(
112
+ languages
113
+ max_bytes
114
+ max_lines
115
+ message
116
+ multiline_flush_interval
117
+ remove_tag_prefix
118
+ stream
119
+ )
120
+ }.freeze
121
+ end
122
+
123
+ include self::Constants
124
+
125
+ # The root configuration file of google-fluentd package.
126
+ # This only applies to Linux.
127
+ config_param :google_fluentd_config_path,
128
+ :string,
129
+ default: '/etc/google-fluentd/google-fluentd.conf'
130
+ # Baseline configuration for comparing with local
131
+ # customizations.
132
+ config_param :google_fluentd_baseline_config_path,
133
+ :string,
134
+ default: '/etc/google-fluentd/baseline/google-fluentd.conf'
135
+
136
+ def start
137
+ super
138
+ @log = $log # rubocop:disable Style/GlobalVars
139
+
140
+ # Initialize the insertID.
141
+ @log.info 'Started the analyze_config plugin to analyze configuration.'
142
+ end
143
+
144
+ def parse_config(path)
145
+ data = File.open(path, 'r', &:read)
146
+ fname = File.basename(path)
147
+ basepath = File.dirname(path)
148
+ eval_context = Kernel.binding
149
+ # Override instance_eval so that LiteralParser does not actually
150
+ # evaluate the embedded Ruby, but instead just returns the
151
+ # source string. See
152
+ # https://github.com/fluent/fluentd/blob/master/lib/fluent/config/literal_parser.rb
153
+ def eval_context.instance_eval(code)
154
+ code
155
+ end
156
+ Fluent::Config::V1Parser.parse(data, fname, basepath, eval_context)
157
+ end
158
+
159
+ # Returns a name for identifying plugins we ship by default.
160
+ def default_plugin_name(e)
161
+ case e['@type']
162
+ when 'syslog'
163
+ "#{e.name}/syslog/#{e['protocol_type']}"
164
+ when 'tail'
165
+ "#{e.name}/tail/#{File.basename(e['pos_file'], '.pos')}"
166
+ else
167
+ "#{e.name}/#{e['@type']}"
168
+ end
169
+ end
170
+
171
+ # Returns a name for identifying plugins not in our default
172
+ # config. This should not contain arbitrary user-supplied data.
173
+ def custom_plugin_name(e)
174
+ if KNOWN_PLUGINS.key?(e.name) &&
175
+ KNOWN_PLUGINS[e.name].include?(e['@type'])
176
+ "#{e.name}/#{e['@type']}"
177
+ else
178
+ e.name.to_s
179
+ end
180
+ end
181
+
182
+ def embedded_ruby?(e)
183
+ (e.arg.include?('#{') ||
184
+ e.any? { |_, v| v.include?('#{') } ||
185
+ e.elements.any? { |ee| embedded_ruby?(ee) })
186
+ end
187
+
188
+ def configure(conf)
189
+ super
190
+ if File.file?(@google_fluentd_config_path) &&
191
+ File.file?(@google_fluentd_baseline_config_path)
192
+ @log.info(
193
+ 'google-fluentd configuration file found at' \
194
+ " #{@google_fluentd_config_path}. " \
195
+ 'google-fluentd baseline configuration file found at' \
196
+ " #{@google_fluentd_baseline_config_path}. " \
197
+ 'google-fluentd Analyzing configuration.')
198
+
199
+ # TODO: Add OpenCensus support.
200
+ registry = Monitoring::MonitoringRegistryFactory.create(
201
+ Monitoring::PrometheusMonitoringRegistry.name, nil, nil, nil)
202
+
203
+ plugin_usage = registry.counter(
204
+ :stackdriver_enabled_plugins,
205
+ [:plugin_name, :is_default_plugin, :has_default_value],
206
+ 'Enabled plugins')
207
+ config_usage = registry.counter(
208
+ :stackdriver_config_usage,
209
+ [:plugin_name, :param, :is_present, :has_default_value],
210
+ 'Parameter usage for Google Cloud plugins')
211
+ config_bool_values = registry.counter(
212
+ :stackdriver_config_bool_values,
213
+ [:plugin_name, :param, :value],
214
+ 'Values for bool parameters in Google Cloud plugins')
215
+
216
+ config = parse_config(@google_fluentd_config_path)
217
+ baseline_config = parse_config(@google_fluentd_baseline_config_path)
218
+
219
+ # Create hash of all baseline elements by their plugin names.
220
+ baseline_elements = Hash[baseline_config.elements.collect do |e|
221
+ [default_plugin_name(e), e]
222
+ end]
223
+ baseline_google_element = baseline_config.elements.find do |e|
224
+ e['@type'] == 'google_cloud'
225
+ end
226
+
227
+ # Look at each top-level config element and see whether it
228
+ # matches the baseline value.
229
+ #
230
+ # Note on custom configurations: If the plugin has a custom
231
+ # value (e.g. if a tail plugin has pos_file
232
+ # /var/lib/google-fluentd/pos/my-custom-value.pos), then the
233
+ # default_plugin_name (e.g. source/tail/my-custom-value) won't
234
+ # be a key in baseline_elements below, so it won't be
235
+ # used. Instead it will use the custom_plugin_name
236
+ # (e.g. source/tail).
237
+ config.elements.each do |e|
238
+ plugin_name = default_plugin_name(e)
239
+ if baseline_elements.key?(plugin_name)
240
+ is_default_plugin = true
241
+ has_default_value = (baseline_elements[plugin_name] == e)
242
+ else
243
+ plugin_name = custom_plugin_name(e)
244
+ is_default_plugin = false
245
+ has_default_value = false
246
+ end
247
+ plugin_usage.increment(
248
+ labels: {
249
+ plugin_name: plugin_name,
250
+ is_default_plugin: is_default_plugin,
251
+ has_default_value: has_default_value,
252
+ has_ruby_snippet: embedded_ruby?(e)
253
+ },
254
+ by: 1)
255
+
256
+ # Additional metric for Google plugins (google_cloud and
257
+ # detect_exceptions).
258
+ next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
259
+ GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
260
+ config_usage.increment(
261
+ labels: {
262
+ plugin_name: e['@type'],
263
+ param: p,
264
+ is_present: e.key?(p),
265
+ has_default_value: (e.key?(p) &&
266
+ baseline_google_element.key?(p) &&
267
+ e[p] == baseline_google_element[p])
268
+ },
269
+ by: 1)
270
+ next unless e.key?(p) && %w(true false).include?(e[p])
271
+ config_bool_values.increment(
272
+ labels: {
273
+ plugin_name: e['@type'],
274
+ param: p,
275
+ value: e[p] == 'true'
276
+ },
277
+ by: 1)
278
+ end
279
+ end
280
+ else
281
+ @log.info(
282
+ 'google-fluentd configuration file does not exist at' \
283
+ " #{@google_fluentd_config_path} or " \
284
+ 'google-fluentd baseline configuration file does not exist at' \
285
+ " #{@google_fluentd_baseline_config_path} or " \
286
+ '. Skipping configuration analysis.')
287
+ end
288
+ rescue => e
289
+ # Do not crash the agent due to configuration analysis failures.
290
+ @log.warn(
291
+ 'Failed to optionally analyze the google-fluentd configuration' \
292
+ " file. Proceeding anyway. Error: #{e}")
293
+ end
294
+
295
+ def shutdown
296
+ super
297
+ end
298
+
299
+ # rubocop:disable Lint/UnusedMethodArgument
300
+ def filter(tag, time, record)
301
+ # Skip the actual filtering process.
302
+ record
303
+ end
304
+ # rubocop:enable Lint/UnusedMethodArgument
305
+ end
306
+ end
@@ -28,6 +28,7 @@ require 'google/logging/v2/logging_services_pb'
28
28
  require 'google/logging/v2/log_entry_pb'
29
29
  require 'googleauth'
30
30
 
31
+ require_relative 'common'
31
32
  require_relative 'monitoring'
32
33
  require_relative 'statusz'
33
34
 
@@ -90,73 +91,6 @@ end
90
91
  module Fluent
91
92
  # fluentd output plugin for the Stackdriver Logging API
92
93
  class GoogleCloudOutput < BufferedOutput
93
- # Constants for service names, resource types and etc.
94
- module ServiceConstants
95
- APPENGINE_CONSTANTS = {
96
- service: 'appengine.googleapis.com',
97
- resource_type: 'gae_app',
98
- metadata_attributes: %w(gae_backend_name gae_backend_version)
99
- }.freeze
100
- COMPUTE_CONSTANTS = {
101
- service: 'compute.googleapis.com',
102
- resource_type: 'gce_instance'
103
- }.freeze
104
- GKE_CONSTANTS = {
105
- service: 'container.googleapis.com',
106
- resource_type: 'container',
107
- extra_resource_labels: %w(namespace_id pod_id container_name),
108
- extra_common_labels: %w(namespace_name pod_name),
109
- metadata_attributes: %w(cluster-name cluster-location),
110
- stream_severity_map: {
111
- 'stdout' => 'INFO',
112
- 'stderr' => 'ERROR'
113
- }
114
- }.freeze
115
- K8S_CONTAINER_CONSTANTS = {
116
- resource_type: 'k8s_container'
117
- }.freeze
118
- K8S_POD_CONSTANTS = {
119
- resource_type: 'k8s_pod'
120
- }.freeze
121
- K8S_NODE_CONSTANTS = {
122
- resource_type: 'k8s_node'
123
- }.freeze
124
- DATAFLOW_CONSTANTS = {
125
- service: 'dataflow.googleapis.com',
126
- resource_type: 'dataflow_step',
127
- extra_resource_labels: %w(region job_name job_id step_id)
128
- }.freeze
129
- DATAPROC_CONSTANTS = {
130
- service: 'cluster.dataproc.googleapis.com',
131
- resource_type: 'cloud_dataproc_cluster',
132
- metadata_attributes: %w(dataproc-cluster-uuid dataproc-cluster-name)
133
- }.freeze
134
- EC2_CONSTANTS = {
135
- service: 'ec2.amazonaws.com',
136
- resource_type: 'aws_ec2_instance'
137
- }.freeze
138
- ML_CONSTANTS = {
139
- service: 'ml.googleapis.com',
140
- resource_type: 'ml_job',
141
- extra_resource_labels: %w(job_id task_name)
142
- }.freeze
143
-
144
- # The map between a subservice name and a resource type.
145
- SUBSERVICE_MAP =
146
- [APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAFLOW_CONSTANTS,
147
- DATAPROC_CONSTANTS, ML_CONSTANTS]
148
- .map { |consts| [consts[:service], consts[:resource_type]] }.to_h
149
- # Default back to GCE if invalid value is detected.
150
- SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
151
- SUBSERVICE_MAP.freeze
152
-
153
- # The map between a resource type and expected subservice attributes.
154
- SUBSERVICE_METADATA_ATTRIBUTES =
155
- [APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAPROC_CONSTANTS].map do |consts|
156
- [consts[:resource_type], consts[:metadata_attributes].to_set]
157
- end.to_h.freeze
158
- end
159
-
160
94
  # Constants for configuration.
161
95
  module ConfigConstants
162
96
  # Default values for JSON payload keys to set the "httpRequest",
@@ -247,7 +181,7 @@ module Fluent
247
181
  .freeze
248
182
  end
249
183
 
250
- include self::ServiceConstants
184
+ include Common::ServiceConstants
251
185
  include self::ConfigConstants
252
186
  include self::InternalConstants
253
187
 
@@ -278,12 +212,6 @@ module Fluent
278
212
  end
279
213
  end.freeze
280
214
 
281
- # Name of the the Google cloud logging write scope.
282
- LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
283
-
284
- # Address of the metadata service.
285
- METADATA_SERVICE_ADDR = '169.254.169.254'.freeze
286
-
287
215
  # Disable this warning to conform to fluentd config_param conventions.
288
216
  # rubocop:disable Style/HashSyntax
289
217
 
@@ -539,7 +467,9 @@ module Fluent
539
467
 
540
468
  set_regexp_patterns
541
469
 
542
- @platform = detect_platform
470
+ @utils = Common::Utils.new(@log)
471
+
472
+ @platform = @utils.detect_platform(@use_metadata_service)
543
473
 
544
474
  # Treat an empty setting of the credentials file path environment variable
545
475
  # as unset. This way the googleauth lib could fetch the credentials
@@ -548,12 +478,20 @@ module Fluent
548
478
  ENV[CREDENTIALS_PATH_ENV_VAR] == ''
549
479
 
550
480
  # Set required variables: @project_id, @vm_id, @vm_name and @zone.
551
- set_required_metadata_variables
481
+ @project_id = @utils.get_project_id(@platform, @project_id)
482
+ @vm_id = @utils.get_vm_id(@platform, @vm_id)
483
+ @vm_name = @utils.get_vm_name(@vm_name)
484
+ @zone = @utils.get_location(@platform, @zone, @use_aws_availability_zone)
485
+
486
+ # All metadata parameters must now be set.
487
+ @utils.check_required_metadata_variables(
488
+ @platform, @project_id, @zone, @vm_id)
552
489
 
553
490
  # Retrieve monitored resource.
554
491
  # Fail over to retrieve monitored resource via the legacy path if we fail
555
492
  # to get it from Metadata Agent.
556
- @resource ||= determine_agent_level_monitored_resource_via_legacy
493
+ @resource ||= @utils.determine_agent_level_monitored_resource_via_legacy(
494
+ @platform, @subservice_name, @detect_subservice, @vm_id, @zone)
557
495
 
558
496
  # If monitoring is enabled, register metrics in the default registry
559
497
  # and store metric objects for future use.
@@ -611,7 +549,7 @@ module Fluent
611
549
 
612
550
  # Determine the common labels that should be added to all log entries
613
551
  # processed by this logging agent.
614
- @common_labels = determine_agent_level_common_labels
552
+ @common_labels = determine_agent_level_common_labels(@resource)
615
553
 
616
554
  # The resource and labels are now set up; ensure they can't be modified
617
555
  # without first duping them.
@@ -627,7 +565,7 @@ module Fluent
627
565
  @write_request = method(:write_request_via_rest)
628
566
  end
629
567
 
630
- if [Platform::GCE, Platform::EC2].include?(@platform)
568
+ if [Common::Platform::GCE, Common::Platform::EC2].include?(@platform)
631
569
  # Log an informational message containing the Logs viewer URL
632
570
  @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
633
571
  "viewer?project=#{@project_id}&resource=#{@resource.type}/",
@@ -1094,66 +1032,6 @@ module Fluent
1094
1032
  nil
1095
1033
  end
1096
1034
 
1097
- # "enum" of Platform values
1098
- module Platform
1099
- OTHER = 0 # Other/unkown platform
1100
- GCE = 1 # Google Compute Engine
1101
- EC2 = 2 # Amazon EC2
1102
- end
1103
-
1104
- # Determine what platform we are running on by consulting the metadata
1105
- # service (unless the user has explicitly disabled using that).
1106
- def detect_platform
1107
- unless @use_metadata_service
1108
- @log.info 'use_metadata_service is false; not detecting platform'
1109
- return Platform::OTHER
1110
- end
1111
-
1112
- begin
1113
- open('http://' + METADATA_SERVICE_ADDR, proxy: false) do |f|
1114
- if f.meta['metadata-flavor'] == 'Google'
1115
- @log.info 'Detected GCE platform'
1116
- return Platform::GCE
1117
- end
1118
- if f.meta['server'] == 'EC2ws'
1119
- @log.info 'Detected EC2 platform'
1120
- return Platform::EC2
1121
- end
1122
- end
1123
- rescue StandardError => e
1124
- @log.error 'Failed to access metadata service: ', error: e
1125
- end
1126
-
1127
- @log.info 'Unable to determine platform'
1128
- Platform::OTHER
1129
- end
1130
-
1131
- def fetch_gce_metadata(metadata_path)
1132
- raise "Called fetch_gce_metadata with platform=#{@platform}" unless
1133
- @platform == Platform::GCE
1134
- # See https://cloud.google.com/compute/docs/metadata
1135
- open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
1136
- metadata_path, 'Metadata-Flavor' => 'Google', :proxy => false,
1137
- &:read)
1138
- end
1139
-
1140
- # EC2 Metadata server returns everything in one call. Store it after the
1141
- # first fetch to avoid making multiple calls.
1142
- def ec2_metadata
1143
- raise "Called ec2_metadata with platform=#{@platform}" unless
1144
- @platform == Platform::EC2
1145
- unless @ec2_metadata
1146
- # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
1147
- open('http://' + METADATA_SERVICE_ADDR +
1148
- '/latest/dynamic/instance-identity/document', proxy: false) do |f|
1149
- contents = f.read
1150
- @ec2_metadata = JSON.parse(contents)
1151
- end
1152
- end
1153
-
1154
- @ec2_metadata
1155
- end
1156
-
1157
1035
  # Set regexp patterns to parse tags and logs.
1158
1036
  def set_regexp_patterns
1159
1037
  @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp) if
@@ -1163,187 +1041,14 @@ module Fluent
1163
1041
  /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
1164
1042
  end
1165
1043
 
1166
- # Set required variables like @project_id, @vm_id, @vm_name and @zone.
1167
- def set_required_metadata_variables
1168
- set_project_id
1169
- set_vm_id
1170
- set_vm_name
1171
- set_location
1172
-
1173
- # All metadata parameters must now be set.
1174
- missing = []
1175
- missing << 'project_id' unless @project_id
1176
- if @platform != Platform::OTHER
1177
- missing << 'zone' unless @zone
1178
- missing << 'vm_id' unless @vm_id
1179
- end
1180
- return if missing.empty?
1181
- raise Fluent::ConfigError,
1182
- "Unable to obtain metadata parameters: #{missing.join(' ')}"
1183
- end
1184
-
1185
- # 1. Return the value if it is explicitly set in the config already.
1186
- # 2. If not, try to retrieve it by calling metadata server directly.
1187
- # 3. If still not set, try to obtain it from the credentials.
1188
- def set_project_id
1189
- @project_id ||= CredentialsInfo.project_id
1190
- @project_id ||= fetch_gce_metadata('project/project-id') if
1191
- @platform == Platform::GCE
1192
- end
1193
-
1194
- # 1. Return the value if it is explicitly set in the config already.
1195
- # 2. If not, try to retrieve it by calling metadata servers directly.
1196
- def set_vm_id
1197
- @vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
1198
- @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
1199
- rescue StandardError => e
1200
- @log.error 'Failed to obtain vm_id: ', error: e
1201
- end
1202
-
1203
- # 1. Return the value if it is explicitly set in the config already.
1204
- # 2. If not, try to retrieve it locally.
1205
- def set_vm_name
1206
- @vm_name ||= Socket.gethostname
1207
- rescue StandardError => e
1208
- @log.error 'Failed to obtain vm name: ', error: e
1209
- end
1210
-
1211
- # 1. Return the value if it is explicitly set in the config already.
1212
- # 2. If not, try to retrieve it locally.
1213
- def set_location
1214
- # Response format: "projects/<number>/zones/<zone>"
1215
- @zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
1216
- @platform == Platform::GCE
1217
- aws_location_key = if @use_aws_availability_zone
1218
- 'availabilityZone'
1219
- else
1220
- 'region'
1221
- end
1222
- @zone ||= 'aws:' + ec2_metadata[aws_location_key] if
1223
- @platform == Platform::EC2 && ec2_metadata.key?(aws_location_key)
1224
- rescue StandardError => e
1225
- @log.error 'Failed to obtain location: ', error: e
1226
- end
1227
-
1228
- # Retrieve monitored resource via the legacy way.
1229
- #
1230
- # Note: This is just a failover plan if we fail to get metadata from
1231
- # Metadata Agent. Thus it should be equivalent to what Metadata Agent
1232
- # returns.
1233
- def determine_agent_level_monitored_resource_via_legacy
1234
- resource = Google::Apis::LoggingV2::MonitoredResource.new(
1235
- labels: {})
1236
- resource.type = determine_agent_level_monitored_resource_type
1237
- resource.labels = determine_agent_level_monitored_resource_labels(
1238
- resource.type)
1239
- resource
1240
- end
1241
-
1242
- # Determine agent level monitored resource type.
1243
- def determine_agent_level_monitored_resource_type
1244
- case @platform
1245
- when Platform::OTHER
1246
- # Unknown platform will be defaulted to GCE instance.
1247
- return COMPUTE_CONSTANTS[:resource_type]
1248
-
1249
- when Platform::EC2
1250
- return EC2_CONSTANTS[:resource_type]
1251
-
1252
- when Platform::GCE
1253
- # Resource types determined by @subservice_name config.
1254
- return SUBSERVICE_MAP[@subservice_name] if @subservice_name
1255
-
1256
- # Resource types determined by @detect_subservice config.
1257
- if @detect_subservice
1258
- begin
1259
- attributes = fetch_gce_metadata('instance/attributes/').split.to_set
1260
- SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
1261
- return resource_type if attributes.superset?(expected)
1262
- end
1263
- rescue StandardError => e
1264
- @log.error 'Failed to detect subservice: ', error: e
1265
- end
1266
- end
1267
-
1268
- # GCE instance.
1269
- return COMPUTE_CONSTANTS[:resource_type]
1270
- end
1271
- end
1272
-
1273
- # Determine agent level monitored resource labels based on the resource
1274
- # type. Each resource type has its own labels that need to be filled in.
1275
- def determine_agent_level_monitored_resource_labels(type)
1276
- case type
1277
- # GAE app.
1278
- when APPENGINE_CONSTANTS[:resource_type]
1279
- return {
1280
- 'module_id' =>
1281
- fetch_gce_metadata('instance/attributes/gae_backend_name'),
1282
- 'version_id' =>
1283
- fetch_gce_metadata('instance/attributes/gae_backend_version')
1284
- }
1285
-
1286
- # GCE.
1287
- when COMPUTE_CONSTANTS[:resource_type]
1288
- raise "Cannot construct a #{type} resource without vm_id and zone" \
1289
- unless @vm_id && @zone
1290
- return {
1291
- 'instance_id' => @vm_id,
1292
- 'zone' => @zone
1293
- }
1294
-
1295
- # GKE container.
1296
- when GKE_CONSTANTS[:resource_type]
1297
- raise "Cannot construct a #{type} resource without vm_id and zone" \
1298
- unless @vm_id && @zone
1299
- return {
1300
- 'instance_id' => @vm_id,
1301
- 'zone' => @zone,
1302
- 'cluster_name' =>
1303
- fetch_gce_metadata('instance/attributes/cluster-name')
1304
- }
1305
-
1306
- # Cloud Dataproc.
1307
- when DATAPROC_CONSTANTS[:resource_type]
1308
- return {
1309
- 'cluster_uuid' =>
1310
- fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid'),
1311
- 'cluster_name' =>
1312
- fetch_gce_metadata('instance/attributes/dataproc-cluster-name'),
1313
- 'region' =>
1314
- fetch_gce_metadata('instance/attributes/dataproc-region')
1315
- }
1316
-
1317
- # EC2.
1318
- when EC2_CONSTANTS[:resource_type]
1319
- raise "Cannot construct a #{type} resource without vm_id and zone" \
1320
- unless @vm_id && @zone
1321
- labels = {
1322
- 'instance_id' => @vm_id,
1323
- 'region' => @zone
1324
- }
1325
- labels['aws_account'] = ec2_metadata['accountId'] if
1326
- ec2_metadata.key?('accountId')
1327
- return labels
1328
- end
1329
-
1330
- {}
1331
- rescue StandardError => e
1332
- if [Platform::GCE, Platform::EC2].include?(@platform)
1333
- @log.error "Failed to set monitored resource labels for #{type}: ",
1334
- error: e
1335
- end
1336
- {}
1337
- end
1338
-
1339
1044
  # Determine the common labels that should be added to all log entries
1340
1045
  # processed by this logging agent.
1341
- def determine_agent_level_common_labels
1046
+ def determine_agent_level_common_labels(resource)
1342
1047
  labels = {}
1343
1048
  # User can specify labels via config. We want to capture those as well.
1344
1049
  labels.merge!(@labels) if @labels
1345
1050
 
1346
- case @resource.type
1051
+ case resource.type
1347
1052
  # GAE, Cloud Dataflow, Cloud Dataproc and Cloud ML.
1348
1053
  when APPENGINE_CONSTANTS[:resource_type],
1349
1054
  DATAFLOW_CONSTANTS[:resource_type],
@@ -1548,44 +1253,6 @@ module Fluent
1548
1253
  [resource, common_labels]
1549
1254
  end
1550
1255
 
1551
- # TODO: This functionality should eventually be available in another
1552
- # library, but implement it ourselves for now.
1553
- module CredentialsInfo
1554
- # Determine the project ID from the credentials, if possible.
1555
- # Returns the project ID (as a string) on success, or nil on failure.
1556
- def self.project_id
1557
- creds = Google::Auth.get_application_default(LOGGING_SCOPE)
1558
- if creds.respond_to?(:project_id)
1559
- return creds.project_id if creds.project_id
1560
- end
1561
- if creds.issuer
1562
- id = extract_project_id(creds.issuer)
1563
- return id unless id.nil?
1564
- end
1565
- if creds.client_id
1566
- id = extract_project_id(creds.client_id)
1567
- return id unless id.nil?
1568
- end
1569
- nil
1570
- end
1571
-
1572
- # Extracts the project id (either name or number) from str and returns
1573
- # it (as a string) on success, or nil on failure.
1574
- #
1575
- # Recognizes IAM format (account@project-name.iam.gserviceaccount.com)
1576
- # as well as the legacy format with a project number at the front of the
1577
- # string, terminated by a dash (-) which is not part of the ID, i.e.:
1578
- # 270694816269-1l1r2hb813leuppurdeik0apglbs80sv.apps.googleusercontent.com
1579
- def self.extract_project_id(str)
1580
- [/^.*@(?<project_id>.+)\.iam\.gserviceaccount\.com/,
1581
- /^(?<project_id>\d+)-/].each do |exp|
1582
- match_data = exp.match(str)
1583
- return match_data['project_id'] unless match_data.nil?
1584
- end
1585
- nil
1586
- end
1587
- end
1588
-
1589
1256
  def time_or_nil(ts_secs, ts_nanos)
1590
1257
  Time.at((Integer ts_secs), (Integer ts_nanos) / 1_000.0)
1591
1258
  rescue ArgumentError, TypeError
@@ -1909,7 +1576,11 @@ module Fluent
1909
1576
  end
1910
1577
 
1911
1578
  def format(tag, time, record)
1912
- Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
1579
+ Fluent::MessagePackFactory
1580
+ .engine_factory
1581
+ .packer
1582
+ .write([tag, time, record])
1583
+ .to_s
1913
1584
  end
1914
1585
 
1915
1586
  # Given a tag, returns the corresponding valid tag if possible, or nil if
@@ -2077,7 +1748,7 @@ module Fluent
2077
1748
  Google::Apis::ClientOptions.default.application_version = PLUGIN_VERSION
2078
1749
  @client = Google::Apis::LoggingV2::LoggingService.new
2079
1750
  @client.authorization = Google::Auth.get_application_default(
2080
- LOGGING_SCOPE)
1751
+ Common::LOGGING_SCOPE)
2081
1752
  end
2082
1753
  end
2083
1754
 
@@ -2306,10 +1977,10 @@ module Fluent
2306
1977
  @k8s_cluster_location = nil if @k8s_cluster_location == ''
2307
1978
 
2308
1979
  begin
2309
- @k8s_cluster_name ||= fetch_gce_metadata(
2310
- 'instance/attributes/cluster-name')
2311
- @k8s_cluster_location ||= fetch_gce_metadata(
2312
- 'instance/attributes/cluster-location')
1980
+ @k8s_cluster_name ||= @utils.fetch_gce_metadata(
1981
+ @platform, 'instance/attributes/cluster-name')
1982
+ @k8s_cluster_location ||= @utils.fetch_gce_metadata(
1983
+ @platform, 'instance/attributes/cluster-location')
2313
1984
  rescue StandardError => e
2314
1985
  @log.error 'Failed to retrieve k8s cluster name and location.', \
2315
1986
  error: e