fluent-plugin-google-cloud 0.8.6 → 0.10.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -17,6 +17,7 @@ require 'fluent/config'
17
17
  require 'fluent/config/v1_parser'
18
18
  require 'set'
19
19
 
20
+ require_relative 'common'
20
21
  require_relative 'monitoring'
21
22
 
22
23
  module Fluent
@@ -122,16 +123,33 @@ module Fluent
122
123
 
123
124
  include self::Constants
124
125
 
126
+ # Disable this warning to conform to fluentd config_param conventions.
127
+ # rubocop:disable Style/HashSyntax
128
+
125
129
  # The root configuration file of google-fluentd package.
126
130
  # This only applies to Linux.
127
131
  config_param :google_fluentd_config_path,
128
132
  :string,
129
- default: '/etc/google-fluentd/google-fluentd.conf'
133
+ :default => '/etc/google-fluentd/google-fluentd.conf'
130
134
  # Baseline configuration for comparing with local
131
135
  # customizations.
132
136
  config_param :google_fluentd_baseline_config_path,
133
137
  :string,
134
- default: '/etc/google-fluentd/baseline/google-fluentd.conf'
138
+ :default => '/etc/google-fluentd/baseline/google-fluentd.conf'
139
+
140
+ # What system to use when collecting metrics. Possible values are:
141
+ # - 'prometheus', in this case default registry in the Prometheus
142
+ # client library is used, without actually exposing the endpoint
143
+ # to serve metrics in the Prometheus format.
144
+ # - any other value will result in the absence of metrics.
145
+ config_param :monitoring_type, :string,
146
+ :default => Monitoring::PrometheusMonitoringRegistry.name
147
+
148
+ # Override for the Google Cloud Monitoring service hostname, or
149
+ # `nil` to leave as the default.
150
+ config_param :gcm_service_address, :string, :default => nil
151
+
152
+ # rubocop:enable Style/HashSyntax
135
153
 
136
154
  def start
137
155
  super
@@ -196,9 +214,29 @@ module Fluent
196
214
  " #{@google_fluentd_baseline_config_path}. " \
197
215
  'google-fluentd Analyzing configuration.')
198
216
 
199
- # TODO: Add OpenCensus support.
217
+ utils = Common::Utils.new(@log)
218
+ platform = utils.detect_platform(true)
219
+ project_id = utils.get_project_id(platform, nil)
220
+ vm_id = utils.get_vm_id(platform, nil)
221
+ zone = utils.get_location(platform, nil, true)
222
+
223
+ # All metadata parameters must now be set.
224
+ utils.check_required_metadata_variables(
225
+ platform, project_id, zone, vm_id)
226
+
227
+ # Retrieve monitored resource.
228
+ # Fail over to retrieve monitored resource via the legacy path if we
229
+ # fail to get it from Metadata Agent.
230
+ resource = utils.determine_agent_level_monitored_resource_via_legacy(
231
+ platform, nil, false, vm_id, zone)
232
+
233
+ unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
234
+ @monitoring_type)
235
+ @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
236
+ 'there will be no metrics'
237
+ end
200
238
  registry = Monitoring::MonitoringRegistryFactory.create(
201
- Monitoring::PrometheusMonitoringRegistry.name, nil, nil, nil)
239
+ @monitoring_type, project_id, resource, @gcm_service_address)
202
240
 
203
241
  plugin_usage = registry.counter(
204
242
  :stackdriver_enabled_plugins,
@@ -28,6 +28,7 @@ require 'google/logging/v2/logging_services_pb'
28
28
  require 'google/logging/v2/log_entry_pb'
29
29
  require 'googleauth'
30
30
 
31
+ require_relative 'common'
31
32
  require_relative 'monitoring'
32
33
  require_relative 'statusz'
33
34
 
@@ -90,73 +91,6 @@ end
90
91
  module Fluent
91
92
  # fluentd output plugin for the Stackdriver Logging API
92
93
  class GoogleCloudOutput < BufferedOutput
93
- # Constants for service names, resource types and etc.
94
- module ServiceConstants
95
- APPENGINE_CONSTANTS = {
96
- service: 'appengine.googleapis.com',
97
- resource_type: 'gae_app',
98
- metadata_attributes: %w(gae_backend_name gae_backend_version)
99
- }.freeze
100
- COMPUTE_CONSTANTS = {
101
- service: 'compute.googleapis.com',
102
- resource_type: 'gce_instance'
103
- }.freeze
104
- GKE_CONSTANTS = {
105
- service: 'container.googleapis.com',
106
- resource_type: 'gke_container',
107
- extra_resource_labels: %w(namespace_id pod_id container_name),
108
- extra_common_labels: %w(namespace_name pod_name),
109
- metadata_attributes: %w(cluster-name cluster-location),
110
- stream_severity_map: {
111
- 'stdout' => 'INFO',
112
- 'stderr' => 'ERROR'
113
- }
114
- }.freeze
115
- K8S_CONTAINER_CONSTANTS = {
116
- resource_type: 'k8s_container'
117
- }.freeze
118
- K8S_POD_CONSTANTS = {
119
- resource_type: 'k8s_pod'
120
- }.freeze
121
- K8S_NODE_CONSTANTS = {
122
- resource_type: 'k8s_node'
123
- }.freeze
124
- DATAFLOW_CONSTANTS = {
125
- service: 'dataflow.googleapis.com',
126
- resource_type: 'dataflow_step',
127
- extra_resource_labels: %w(region job_name job_id step_id)
128
- }.freeze
129
- DATAPROC_CONSTANTS = {
130
- service: 'cluster.dataproc.googleapis.com',
131
- resource_type: 'cloud_dataproc_cluster',
132
- metadata_attributes: %w(dataproc-cluster-uuid dataproc-cluster-name)
133
- }.freeze
134
- EC2_CONSTANTS = {
135
- service: 'ec2.amazonaws.com',
136
- resource_type: 'aws_ec2_instance'
137
- }.freeze
138
- ML_CONSTANTS = {
139
- service: 'ml.googleapis.com',
140
- resource_type: 'ml_job',
141
- extra_resource_labels: %w(job_id task_name)
142
- }.freeze
143
-
144
- # The map between a subservice name and a resource type.
145
- SUBSERVICE_MAP =
146
- [APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAFLOW_CONSTANTS,
147
- DATAPROC_CONSTANTS, ML_CONSTANTS]
148
- .map { |consts| [consts[:service], consts[:resource_type]] }.to_h
149
- # Default back to GCE if invalid value is detected.
150
- SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
151
- SUBSERVICE_MAP.freeze
152
-
153
- # The map between a resource type and expected subservice attributes.
154
- SUBSERVICE_METADATA_ATTRIBUTES =
155
- [APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAPROC_CONSTANTS].map do |consts|
156
- [consts[:resource_type], consts[:metadata_attributes].to_set]
157
- end.to_h.freeze
158
- end
159
-
160
94
  # Constants for configuration.
161
95
  module ConfigConstants
162
96
  # Default values for JSON payload keys to set the "httpRequest",
@@ -247,7 +181,7 @@ module Fluent
247
181
  .freeze
248
182
  end
249
183
 
250
- include self::ServiceConstants
184
+ include Common::ServiceConstants
251
185
  include self::ConfigConstants
252
186
  include self::InternalConstants
253
187
 
@@ -278,12 +212,6 @@ module Fluent
278
212
  end
279
213
  end.freeze
280
214
 
281
- # Name of the the Google cloud logging write scope.
282
- LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
283
-
284
- # Address of the metadata service.
285
- METADATA_SERVICE_ADDR = '169.254.169.254'.freeze
286
-
287
215
  # Disable this warning to conform to fluentd config_param conventions.
288
216
  # rubocop:disable Style/HashSyntax
289
217
 
@@ -442,10 +370,20 @@ module Fluent
442
370
  # - 'prometheus', in this case default registry in the Prometheus
443
371
  # client library is used, without actually exposing the endpoint
444
372
  # to serve metrics in the Prometheus format.
445
- # - any other value will result in the absence of metrics.
373
+ # - 'opencensus', in this case the OpenCensus implementation is
374
+ # used to send metrics directly to Google Cloud Monitoring.
375
+ # - any other value will result in the absence of metrics.
446
376
  config_param :monitoring_type, :string,
447
377
  :default => Monitoring::PrometheusMonitoringRegistry.name
448
378
 
379
+ # The monitored resource to use for OpenCensus metrics. Only valid
380
+ # when monitoring_type is set to 'opencensus'. This value is a hash in
381
+ # the form:
382
+ # {"type":"gce_instance","labels":{"instance_id":"aaa","zone":"bbb"} (JSON)
383
+ # or type:gce_instance,labels.instance_id:aaa,labels.zone:bbb (Hash)
384
+ config_param :metrics_resource, :hash,
385
+ :symbolize_keys => true, :default => nil
386
+
449
387
  # Whether to call metadata agent to retrieve monitored resource. This flag
450
388
  # is kept for backwards compatibility, and is no longer used.
451
389
  # TODO: Breaking change. Remove this flag in Logging Agent 2.0.0 release.
@@ -495,6 +433,7 @@ module Fluent
495
433
  attr_reader :vm_id
496
434
  attr_reader :resource
497
435
  attr_reader :common_labels
436
+ attr_reader :monitoring_resource
498
437
 
499
438
  def initialize
500
439
  super
@@ -539,7 +478,9 @@ module Fluent
539
478
 
540
479
  set_regexp_patterns
541
480
 
542
- @platform = detect_platform
481
+ @utils = Common::Utils.new(@log)
482
+
483
+ @platform = @utils.detect_platform(@use_metadata_service)
543
484
 
544
485
  # Treat an empty setting of the credentials file path environment variable
545
486
  # as unset. This way the googleauth lib could fetch the credentials
@@ -548,12 +489,57 @@ module Fluent
548
489
  ENV[CREDENTIALS_PATH_ENV_VAR] == ''
549
490
 
550
491
  # Set required variables: @project_id, @vm_id, @vm_name and @zone.
551
- set_required_metadata_variables
492
+ @project_id = @utils.get_project_id(@platform, @project_id)
493
+ @vm_id = @utils.get_vm_id(@platform, @vm_id)
494
+ @vm_name = @utils.get_vm_name(@vm_name)
495
+ @zone = @utils.get_location(@platform, @zone, @use_aws_availability_zone)
496
+
497
+ # All metadata parameters must now be set.
498
+ @utils.check_required_metadata_variables(
499
+ @platform, @project_id, @zone, @vm_id)
552
500
 
553
501
  # Retrieve monitored resource.
554
502
  # Fail over to retrieve monitored resource via the legacy path if we fail
555
503
  # to get it from Metadata Agent.
556
- @resource ||= determine_agent_level_monitored_resource_via_legacy
504
+ @resource ||= @utils.determine_agent_level_monitored_resource_via_legacy(
505
+ @platform, @subservice_name, @detect_subservice, @vm_id, @zone)
506
+
507
+ if @metrics_resource
508
+ unless @metrics_resource[:type].is_a?(String)
509
+ raise Fluent::ConfigError,
510
+ 'metrics_resource.type must be a string:' \
511
+ " #{@metrics_resource}."
512
+ end
513
+ if @metrics_resource.key?(:labels)
514
+ unless @metrics_resource[:labels].is_a?(Hash)
515
+ raise Fluent::ConfigError,
516
+ 'metrics_resource.labels must be a hash:' \
517
+ " #{@metrics_resource}."
518
+ end
519
+ extra_keys = @metrics_resource.reject do |k, _|
520
+ k == :type || k == :labels
521
+ end
522
+ unless extra_keys.empty?
523
+ raise Fluent::ConfigError,
524
+ "metrics_resource has unrecognized keys: #{extra_keys.keys}."
525
+ end
526
+ else
527
+ extra_keys = @metrics_resource.reject do |k, _|
528
+ k == :type || k.to_s.start_with?('labels.')
529
+ end
530
+ unless extra_keys.empty?
531
+ raise Fluent::ConfigError,
532
+ "metrics_resource has unrecognized keys: #{extra_keys.keys}."
533
+ end
534
+ # Transform the Hash form of the metrics_resource config if necessary.
535
+ resource_type = @metrics_resource[:type]
536
+ resource_labels = @metrics_resource.each_with_object({}) \
537
+ do |(k, v), h|
538
+ h[k.to_s.sub('labels.', '')] = v if k.to_s.start_with? 'labels.'
539
+ end
540
+ @metrics_resource = { type: resource_type, labels: resource_labels }
541
+ end
542
+ end
557
543
 
558
544
  # If monitoring is enabled, register metrics in the default registry
559
545
  # and store metric objects for future use.
@@ -563,9 +549,15 @@ module Fluent
563
549
  @log.warn "monitoring_type '#{@monitoring_type}' is unknown; "\
564
550
  'there will be no metrics'
565
551
  end
552
+ if @metrics_resource
553
+ @monitoring_resource = @utils.create_monitored_resource(
554
+ @metrics_resource[:type], @metrics_resource[:labels])
555
+ else
556
+ @monitoring_resource = @resource
557
+ end
566
558
  @registry = Monitoring::MonitoringRegistryFactory
567
- .create(@monitoring_type, @project_id, @resource,
568
- @gcm_service_address)
559
+ .create(@monitoring_type, @project_id,
560
+ @monitoring_resource, @gcm_service_address)
569
561
  # Export metrics every 60 seconds.
570
562
  timer_execute(:export_metrics, 60) { @registry.export }
571
563
  # Uptime should be a gauge, but the metric definition is a counter and
@@ -611,7 +603,7 @@ module Fluent
611
603
 
612
604
  # Determine the common labels that should be added to all log entries
613
605
  # processed by this logging agent.
614
- @common_labels = determine_agent_level_common_labels
606
+ @common_labels = determine_agent_level_common_labels(@resource)
615
607
 
616
608
  # The resource and labels are now set up; ensure they can't be modified
617
609
  # without first duping them.
@@ -627,7 +619,7 @@ module Fluent
627
619
  @write_request = method(:write_request_via_rest)
628
620
  end
629
621
 
630
- if [Platform::GCE, Platform::EC2].include?(@platform)
622
+ if [Common::Platform::GCE, Common::Platform::EC2].include?(@platform)
631
623
  # Log an informational message containing the Logs viewer URL
632
624
  @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
633
625
  "viewer?project=#{@project_id}&resource=#{@resource.type}/",
@@ -1094,66 +1086,6 @@ module Fluent
1094
1086
  nil
1095
1087
  end
1096
1088
 
1097
- # "enum" of Platform values
1098
- module Platform
1099
- OTHER = 0 # Other/unkown platform
1100
- GCE = 1 # Google Compute Engine
1101
- EC2 = 2 # Amazon EC2
1102
- end
1103
-
1104
- # Determine what platform we are running on by consulting the metadata
1105
- # service (unless the user has explicitly disabled using that).
1106
- def detect_platform
1107
- unless @use_metadata_service
1108
- @log.info 'use_metadata_service is false; not detecting platform'
1109
- return Platform::OTHER
1110
- end
1111
-
1112
- begin
1113
- open('http://' + METADATA_SERVICE_ADDR, proxy: false) do |f|
1114
- if f.meta['metadata-flavor'] == 'Google'
1115
- @log.info 'Detected GCE platform'
1116
- return Platform::GCE
1117
- end
1118
- if f.meta['server'] == 'EC2ws'
1119
- @log.info 'Detected EC2 platform'
1120
- return Platform::EC2
1121
- end
1122
- end
1123
- rescue StandardError => e
1124
- @log.error 'Failed to access metadata service: ', error: e
1125
- end
1126
-
1127
- @log.info 'Unable to determine platform'
1128
- Platform::OTHER
1129
- end
1130
-
1131
- def fetch_gce_metadata(metadata_path)
1132
- raise "Called fetch_gce_metadata with platform=#{@platform}" unless
1133
- @platform == Platform::GCE
1134
- # See https://cloud.google.com/compute/docs/metadata
1135
- open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
1136
- metadata_path, 'Metadata-Flavor' => 'Google', :proxy => false,
1137
- &:read)
1138
- end
1139
-
1140
- # EC2 Metadata server returns everything in one call. Store it after the
1141
- # first fetch to avoid making multiple calls.
1142
- def ec2_metadata
1143
- raise "Called ec2_metadata with platform=#{@platform}" unless
1144
- @platform == Platform::EC2
1145
- unless @ec2_metadata
1146
- # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
1147
- open('http://' + METADATA_SERVICE_ADDR +
1148
- '/latest/dynamic/instance-identity/document', proxy: false) do |f|
1149
- contents = f.read
1150
- @ec2_metadata = JSON.parse(contents)
1151
- end
1152
- end
1153
-
1154
- @ec2_metadata
1155
- end
1156
-
1157
1089
  # Set regexp patterns to parse tags and logs.
1158
1090
  def set_regexp_patterns
1159
1091
  @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp) if
@@ -1163,187 +1095,14 @@ module Fluent
1163
1095
  /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
1164
1096
  end
1165
1097
 
1166
- # Set required variables like @project_id, @vm_id, @vm_name and @zone.
1167
- def set_required_metadata_variables
1168
- set_project_id
1169
- set_vm_id
1170
- set_vm_name
1171
- set_location
1172
-
1173
- # All metadata parameters must now be set.
1174
- missing = []
1175
- missing << 'project_id' unless @project_id
1176
- if @platform != Platform::OTHER
1177
- missing << 'zone' unless @zone
1178
- missing << 'vm_id' unless @vm_id
1179
- end
1180
- return if missing.empty?
1181
- raise Fluent::ConfigError,
1182
- "Unable to obtain metadata parameters: #{missing.join(' ')}"
1183
- end
1184
-
1185
- # 1. Return the value if it is explicitly set in the config already.
1186
- # 2. If not, try to retrieve it by calling metadata server directly.
1187
- # 3. If still not set, try to obtain it from the credentials.
1188
- def set_project_id
1189
- @project_id ||= CredentialsInfo.project_id
1190
- @project_id ||= fetch_gce_metadata('project/project-id') if
1191
- @platform == Platform::GCE
1192
- end
1193
-
1194
- # 1. Return the value if it is explicitly set in the config already.
1195
- # 2. If not, try to retrieve it by calling metadata servers directly.
1196
- def set_vm_id
1197
- @vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
1198
- @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
1199
- rescue StandardError => e
1200
- @log.error 'Failed to obtain vm_id: ', error: e
1201
- end
1202
-
1203
- # 1. Return the value if it is explicitly set in the config already.
1204
- # 2. If not, try to retrieve it locally.
1205
- def set_vm_name
1206
- @vm_name ||= Socket.gethostname
1207
- rescue StandardError => e
1208
- @log.error 'Failed to obtain vm name: ', error: e
1209
- end
1210
-
1211
- # 1. Return the value if it is explicitly set in the config already.
1212
- # 2. If not, try to retrieve it locally.
1213
- def set_location
1214
- # Response format: "projects/<number>/zones/<zone>"
1215
- @zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
1216
- @platform == Platform::GCE
1217
- aws_location_key = if @use_aws_availability_zone
1218
- 'availabilityZone'
1219
- else
1220
- 'region'
1221
- end
1222
- @zone ||= 'aws:' + ec2_metadata[aws_location_key] if
1223
- @platform == Platform::EC2 && ec2_metadata.key?(aws_location_key)
1224
- rescue StandardError => e
1225
- @log.error 'Failed to obtain location: ', error: e
1226
- end
1227
-
1228
- # Retrieve monitored resource via the legacy way.
1229
- #
1230
- # Note: This is just a failover plan if we fail to get metadata from
1231
- # Metadata Agent. Thus it should be equivalent to what Metadata Agent
1232
- # returns.
1233
- def determine_agent_level_monitored_resource_via_legacy
1234
- resource = Google::Apis::LoggingV2::MonitoredResource.new(
1235
- labels: {})
1236
- resource.type = determine_agent_level_monitored_resource_type
1237
- resource.labels = determine_agent_level_monitored_resource_labels(
1238
- resource.type)
1239
- resource
1240
- end
1241
-
1242
- # Determine agent level monitored resource type.
1243
- def determine_agent_level_monitored_resource_type
1244
- case @platform
1245
- when Platform::OTHER
1246
- # Unknown platform will be defaulted to GCE instance.
1247
- return COMPUTE_CONSTANTS[:resource_type]
1248
-
1249
- when Platform::EC2
1250
- return EC2_CONSTANTS[:resource_type]
1251
-
1252
- when Platform::GCE
1253
- # Resource types determined by @subservice_name config.
1254
- return SUBSERVICE_MAP[@subservice_name] if @subservice_name
1255
-
1256
- # Resource types determined by @detect_subservice config.
1257
- if @detect_subservice
1258
- begin
1259
- attributes = fetch_gce_metadata('instance/attributes/').split.to_set
1260
- SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
1261
- return resource_type if attributes.superset?(expected)
1262
- end
1263
- rescue StandardError => e
1264
- @log.error 'Failed to detect subservice: ', error: e
1265
- end
1266
- end
1267
-
1268
- # GCE instance.
1269
- return COMPUTE_CONSTANTS[:resource_type]
1270
- end
1271
- end
1272
-
1273
- # Determine agent level monitored resource labels based on the resource
1274
- # type. Each resource type has its own labels that need to be filled in.
1275
- def determine_agent_level_monitored_resource_labels(type)
1276
- case type
1277
- # GAE app.
1278
- when APPENGINE_CONSTANTS[:resource_type]
1279
- return {
1280
- 'module_id' =>
1281
- fetch_gce_metadata('instance/attributes/gae_backend_name'),
1282
- 'version_id' =>
1283
- fetch_gce_metadata('instance/attributes/gae_backend_version')
1284
- }
1285
-
1286
- # GCE.
1287
- when COMPUTE_CONSTANTS[:resource_type]
1288
- raise "Cannot construct a #{type} resource without vm_id and zone" \
1289
- unless @vm_id && @zone
1290
- return {
1291
- 'instance_id' => @vm_id,
1292
- 'zone' => @zone
1293
- }
1294
-
1295
- # GKE container.
1296
- when GKE_CONSTANTS[:resource_type]
1297
- raise "Cannot construct a #{type} resource without vm_id and zone" \
1298
- unless @vm_id && @zone
1299
- return {
1300
- 'instance_id' => @vm_id,
1301
- 'zone' => @zone,
1302
- 'cluster_name' =>
1303
- fetch_gce_metadata('instance/attributes/cluster-name')
1304
- }
1305
-
1306
- # Cloud Dataproc.
1307
- when DATAPROC_CONSTANTS[:resource_type]
1308
- return {
1309
- 'cluster_uuid' =>
1310
- fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid'),
1311
- 'cluster_name' =>
1312
- fetch_gce_metadata('instance/attributes/dataproc-cluster-name'),
1313
- 'region' =>
1314
- fetch_gce_metadata('instance/attributes/dataproc-region')
1315
- }
1316
-
1317
- # EC2.
1318
- when EC2_CONSTANTS[:resource_type]
1319
- raise "Cannot construct a #{type} resource without vm_id and zone" \
1320
- unless @vm_id && @zone
1321
- labels = {
1322
- 'instance_id' => @vm_id,
1323
- 'region' => @zone
1324
- }
1325
- labels['aws_account'] = ec2_metadata['accountId'] if
1326
- ec2_metadata.key?('accountId')
1327
- return labels
1328
- end
1329
-
1330
- {}
1331
- rescue StandardError => e
1332
- if [Platform::GCE, Platform::EC2].include?(@platform)
1333
- @log.error "Failed to set monitored resource labels for #{type}: ",
1334
- error: e
1335
- end
1336
- {}
1337
- end
1338
-
1339
1098
  # Determine the common labels that should be added to all log entries
1340
1099
  # processed by this logging agent.
1341
- def determine_agent_level_common_labels
1100
+ def determine_agent_level_common_labels(resource)
1342
1101
  labels = {}
1343
1102
  # User can specify labels via config. We want to capture those as well.
1344
1103
  labels.merge!(@labels) if @labels
1345
1104
 
1346
- case @resource.type
1105
+ case resource.type
1347
1106
  # GAE, Cloud Dataflow, Cloud Dataproc and Cloud ML.
1348
1107
  when APPENGINE_CONSTANTS[:resource_type],
1349
1108
  DATAFLOW_CONSTANTS[:resource_type],
@@ -1548,44 +1307,6 @@ module Fluent
1548
1307
  [resource, common_labels]
1549
1308
  end
1550
1309
 
1551
- # TODO: This functionality should eventually be available in another
1552
- # library, but implement it ourselves for now.
1553
- module CredentialsInfo
1554
- # Determine the project ID from the credentials, if possible.
1555
- # Returns the project ID (as a string) on success, or nil on failure.
1556
- def self.project_id
1557
- creds = Google::Auth.get_application_default(LOGGING_SCOPE)
1558
- if creds.respond_to?(:project_id)
1559
- return creds.project_id if creds.project_id
1560
- end
1561
- if creds.issuer
1562
- id = extract_project_id(creds.issuer)
1563
- return id unless id.nil?
1564
- end
1565
- if creds.client_id
1566
- id = extract_project_id(creds.client_id)
1567
- return id unless id.nil?
1568
- end
1569
- nil
1570
- end
1571
-
1572
- # Extracts the project id (either name or number) from str and returns
1573
- # it (as a string) on success, or nil on failure.
1574
- #
1575
- # Recognizes IAM format (account@project-name.iam.gserviceaccount.com)
1576
- # as well as the legacy format with a project number at the front of the
1577
- # string, terminated by a dash (-) which is not part of the ID, i.e.:
1578
- # <PROJECT_ID>-<OTHER_PARTS>.apps.googleusercontent.com
1579
- def self.extract_project_id(str)
1580
- [/^.*@(?<project_id>.+)\.iam\.gserviceaccount\.com/,
1581
- /^(?<project_id>\d+)-/].each do |exp|
1582
- match_data = exp.match(str)
1583
- return match_data['project_id'] unless match_data.nil?
1584
- end
1585
- nil
1586
- end
1587
- end
1588
-
1589
1310
  def time_or_nil(ts_secs, ts_nanos)
1590
1311
  Time.at((Integer ts_secs), (Integer ts_nanos) / 1_000.0)
1591
1312
  rescue ArgumentError, TypeError
@@ -2081,7 +1802,7 @@ module Fluent
2081
1802
  Google::Apis::ClientOptions.default.application_version = PLUGIN_VERSION
2082
1803
  @client = Google::Apis::LoggingV2::LoggingService.new
2083
1804
  @client.authorization = Google::Auth.get_application_default(
2084
- LOGGING_SCOPE)
1805
+ Common::LOGGING_SCOPE)
2085
1806
  end
2086
1807
  end
2087
1808
 
@@ -2310,10 +2031,10 @@ module Fluent
2310
2031
  @k8s_cluster_location = nil if @k8s_cluster_location == ''
2311
2032
 
2312
2033
  begin
2313
- @k8s_cluster_name ||= fetch_gce_metadata(
2314
- 'instance/attributes/cluster-name')
2315
- @k8s_cluster_location ||= fetch_gce_metadata(
2316
- 'instance/attributes/cluster-location')
2034
+ @k8s_cluster_name ||= @utils.fetch_gce_metadata(
2035
+ @platform, 'instance/attributes/cluster-name')
2036
+ @k8s_cluster_location ||= @utils.fetch_gce_metadata(
2037
+ @platform, 'instance/attributes/cluster-location')
2317
2038
  rescue StandardError => e
2318
2039
  @log.error 'Failed to retrieve k8s cluster name and location.', \
2319
2040
  error: e