fluent-plugin-google-cloud 0.8.5 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97dc07c8c96a6f2901e76e22b62da38e809a843591df62536cfeddba7311775c
-  data.tar.gz: 4ae7ec58daa1232a7c9a4c8d7462da636801059c21d3e958e305432133bb267b
+  metadata.gz: fcb1205d0b6d5f6043f92e06e092a071b34904dc4c02f6860ada3e47714eb840
+  data.tar.gz: 4e7571c46da5eaf1ab6c7a53ea49b7b9fd2e1e71756837ddc22d67d700e70c7a
 SHA512:
-  metadata.gz: '089da19a00151839f4c0cd037cd446e8a34591a0cb739aa658d7aa1364d582ba236ce5a5c0433e84660b18410cda81bbc166596486f530cd8c0c302513f26e57'
-  data.tar.gz: 2ec8ca875f52f62bec8003cd458cf3b009061faf8c72965f3d9d4ae2b2b09be00c223a673013ae10750ab7bfff20b30b31a3d6af3f2be62d2f310a9040aea9f9
+  metadata.gz: f649b399e71371b43585bc23d74c0d21ba909201ffd648d03fa3668692560f10e5cf6972a30ce05c369742a6c6227d7a0dd2b0d4b9cfe21301835110a1b928d2
+  data.tar.gz: 59508e973cede65ed3e5db26fa0ecca549492bbb000b8df3eb0ec53d0eab75c55caceca7d30486e74bfc45788728338da995f699c73824bc4d489a4b9ab0e521

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.8.5)
-      fluentd (= 1.7.4)
+    fluent-plugin-google-cloud (0.10.0)
+      fluentd (= 1.11.2)
       google-api-client (= 0.30.8)
       google-cloud-logging (= 1.6.6)
-      google-protobuf (= 3.9.0)
+      google-protobuf (= 3.12.2)
       googleapis-common-protos (= 1.3.9)
       googleauth (= 0.9.0)
-      grpc (= 1.22.0)
+      grpc (= 1.30.2)
       json (= 2.2.0)
       opencensus (= 0.5.0)
       opencensus-stackdriver (= 0.3.2)
@@ -18,8 +18,8 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.0)
-    concurrent-ruby (1.1.6)
+    ast (2.4.1)
+    concurrent-ruby (1.1.7)
     cool.io (1.6.0)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
@@ -29,23 +29,27 @@ GEM
       tins (~> 1.6)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    declarative (0.0.10)
+    declarative (0.0.20)
     declarative-option (0.1.0)
-    dig_rb (1.0.1)
     docile (1.3.2)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
-    fluentd (1.7.4)
+    fluentd (1.11.2)
       cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.2.0, < 2.0.0)
+      msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (~> 2.0)
+      tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
+    gapic-common (0.3.4)
+      google-protobuf (~> 3.12, >= 3.12.2)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.25)
     google-api-client (0.30.8)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.5, < 0.10.0)
@@ -57,36 +61,44 @@ GEM
     google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
-    google-cloud-env (1.3.1)
+    google-cloud-env (1.3.3)
       faraday (>= 0.17.3, < 2.0)
-    google-cloud-errors (1.0.0)
+    google-cloud-errors (1.0.1)
     google-cloud-logging (1.6.6)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.2)
       google-gax (~> 1.7)
       googleapis-common-protos-types (>= 1.0.2)
       stackdriver-core (~> 1.3)
-    google-cloud-monitoring (0.32.0)
-      google-gax (~> 1.7)
-      googleapis-common-protos-types (>= 1.0.2)
-    google-cloud-trace (0.35.0)
+    google-cloud-monitoring (0.38.0)
+      google-gax (~> 1.8)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+    google-cloud-trace (0.40.0)
       concurrent-ruby (~> 1.1)
-      google-cloud-core (~> 1.2)
-      google-gax (~> 1.7)
+      google-cloud-core (~> 1.5)
+      google-cloud-trace-v1 (~> 0.0)
+      google-cloud-trace-v2 (~> 0.0)
       stackdriver-core (~> 1.3)
-    google-gax (1.7.1)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
+    google-cloud-trace-v1 (0.1.2)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+    google-cloud-trace-v2 (0.1.2)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+    google-gax (1.8.1)
+      google-protobuf (~> 3.9)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.24)
       rly (~> 0.2.3)
-    google-protobuf (3.9.0)
+    google-protobuf (3.12.2)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
       grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.4)
-      google-protobuf (~> 3.0)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
     googleauth (0.9.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
@@ -94,8 +106,8 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    grpc (1.22.0)
-      google-protobuf (~> 3.8)
+    grpc (1.30.2)
+      google-protobuf (~> 3.12)
       googleapis-common-protos-types (~> 1.0)
     hashdiff (1.0.1)
     http_parser.rb (0.6.0)
@@ -108,7 +120,7 @@ GEM
     mocha (1.9.0)
       metaclass (~> 0.0.1)
     msgpack (1.3.3)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     opencensus (0.5.0)
     opencensus-stackdriver (0.3.2)
@@ -116,9 +128,9 @@ GEM
       google-cloud-monitoring (~> 0.32)
       google-cloud-trace (~> 0.35)
       opencensus (~> 0.5)
-    os (1.1.0)
-    parser (2.7.1.2)
-      ast (~> 2.4.0)
+    os (1.1.1)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
     power_assert (1.2.0)
     powerpack (0.1.2)
     prometheus-client (0.9.0)
@@ -164,7 +176,7 @@ GEM
     test-unit (3.3.3)
       power_assert
     thor (1.0.1)
-    tins (1.24.1)
+    tins (1.25.0)
       sync
     tzinfo (2.0.2)
       concurrent-ruby (~> 1.0)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.8.5'
+  gem.version       = '0.10.0'
   gem.authors       = ['Stackdriver Agents Team']
   gem.email         = ['stackdriver-agents@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.2')
@@ -19,13 +19,13 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '1.7.4'
+  gem.add_runtime_dependency 'fluentd', '1.11.2'
   gem.add_runtime_dependency 'googleapis-common-protos', '1.3.9'
   gem.add_runtime_dependency 'googleauth', '0.9.0'
   gem.add_runtime_dependency 'google-api-client', '0.30.8'
   gem.add_runtime_dependency 'google-cloud-logging', '1.6.6'
-  gem.add_runtime_dependency 'google-protobuf', '3.9.0'
-  gem.add_runtime_dependency 'grpc', '1.22.0'
+  gem.add_runtime_dependency 'google-protobuf', '3.12.2'
+  gem.add_runtime_dependency 'grpc', '1.30.2'
   gem.add_runtime_dependency 'json', '2.2.0'
   gem.add_runtime_dependency 'opencensus', '0.5.0'
   gem.add_runtime_dependency 'opencensus-stackdriver', '0.3.2'

data/lib/fluent/plugin/common.rb

@@ -0,0 +1,386 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Common
+  # Constants for service names, resource types and etc.
+  module ServiceConstants
+    APPENGINE_CONSTANTS = {
+      service: 'appengine.googleapis.com',
+      resource_type: 'gae_app',
+      metadata_attributes: %w(gae_backend_name gae_backend_version)
+    }.freeze
+    COMPUTE_CONSTANTS = {
+      service: 'compute.googleapis.com',
+      resource_type: 'gce_instance'
+    }.freeze
+    GKE_CONSTANTS = {
+      service: 'container.googleapis.com',
+      resource_type: 'gke_container',
+      extra_resource_labels: %w(namespace_id pod_id container_name),
+      extra_common_labels: %w(namespace_name pod_name),
+      metadata_attributes: %w(cluster-name cluster-location),
+      stream_severity_map: {
+        'stdout' => 'INFO',
+        'stderr' => 'ERROR'
+      }
+    }.freeze
+    K8S_CONTAINER_CONSTANTS = {
+      resource_type: 'k8s_container'
+    }.freeze
+    K8S_POD_CONSTANTS = {
+      resource_type: 'k8s_pod'
+    }.freeze
+    K8S_NODE_CONSTANTS = {
+      resource_type: 'k8s_node'
+    }.freeze
+    DATAFLOW_CONSTANTS = {
+      service: 'dataflow.googleapis.com',
+      resource_type: 'dataflow_step',
+      extra_resource_labels: %w(region job_name job_id step_id)
+    }.freeze
+    DATAPROC_CONSTANTS = {
+      service: 'cluster.dataproc.googleapis.com',
+      resource_type: 'cloud_dataproc_cluster',
+      metadata_attributes: %w(dataproc-cluster-uuid dataproc-cluster-name)
+    }.freeze
+    EC2_CONSTANTS = {
+      service: 'ec2.amazonaws.com',
+      resource_type: 'aws_ec2_instance'
+    }.freeze
+    ML_CONSTANTS = {
+      service: 'ml.googleapis.com',
+      resource_type: 'ml_job',
+      extra_resource_labels: %w(job_id task_name)
+    }.freeze
+
+    # The map between a subservice name and a resource type.
+    SUBSERVICE_MAP =
+      [APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAFLOW_CONSTANTS,
+       DATAPROC_CONSTANTS, ML_CONSTANTS]
+      .map { |consts| [consts[:service], consts[:resource_type]] }.to_h
+    # Default back to GCE if invalid value is detected.
+    SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
+    SUBSERVICE_MAP.freeze
+
+    # The map between a resource type and expected subservice attributes.
+    SUBSERVICE_METADATA_ATTRIBUTES =
+      [APPENGINE_CONSTANTS, GKE_CONSTANTS, DATAPROC_CONSTANTS].map do |consts|
+        [consts[:resource_type], consts[:metadata_attributes].to_set]
+      end.to_h.freeze
+  end
+
+  # Name of the the Google cloud logging write scope.
+  LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
+
+  # Address of the metadata service.
+  METADATA_SERVICE_ADDR = '169.254.169.254'.freeze
+
+  # "enum" of Platform values
+  module Platform
+    OTHER = 0  # Other/unkown platform
+    GCE = 1    # Google Compute Engine
+    EC2 = 2    # Amazon EC2
+  end
+
+  # Utilities for managing the resource used when writing to the
+  # Google API.
+  class Utils
+    include Common::ServiceConstants
+
+    def initialize(log)
+      @log = log
+    end
+
+    # Determine what platform we are running on by consulting the metadata
+    # service (unless the user has explicitly disabled using that).
+    def detect_platform(use_metadata_service)
+      unless use_metadata_service
+        @log.info 'use_metadata_service is false; not detecting platform'
+        return Platform::OTHER
+      end
+
+      begin
+        open('http://' + METADATA_SERVICE_ADDR, proxy: false) do |f|
+          if f.meta['metadata-flavor'] == 'Google'
+            @log.info 'Detected GCE platform'
+            return Platform::GCE
+          end
+          if f.meta['server'] == 'EC2ws'
+            @log.info 'Detected EC2 platform'
+            return Platform::EC2
+          end
+        end
+      rescue StandardError => e
+        @log.error 'Failed to access metadata service: ', error: e
+      end
+
+      @log.info 'Unable to determine platform'
+      Platform::OTHER
+    end
+
+    def fetch_gce_metadata(platform, metadata_path)
+      raise "Called fetch_gce_metadata with platform=#{platform}" unless
+        platform == Platform::GCE
+      # See https://cloud.google.com/compute/docs/metadata
+      open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
+           metadata_path, 'Metadata-Flavor' => 'Google', :proxy => false,
+           &:read)
+    end
+
+    # EC2 Metadata server returns everything in one call. Store it after the
+    # first fetch to avoid making multiple calls.
+    def ec2_metadata(platform)
+      raise "Called ec2_metadata with platform=#{platform}" unless
+        platform == Platform::EC2
+      unless @ec2_metadata
+        # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+        open('http://' + METADATA_SERVICE_ADDR +
+             '/latest/dynamic/instance-identity/document', proxy: false) do |f|
+          contents = f.read
+          @ec2_metadata = JSON.parse(contents)
+        end
+      end
+
+      @ec2_metadata
+    end
+
+    # Check required variables like @project_id, @vm_id, @vm_name and @zone.
+    def check_required_metadata_variables(platform, project_id, zone, vm_id)
+      missing = []
+      missing << 'project_id' unless project_id
+      if platform != Platform::OTHER
+        missing << 'zone' unless zone
+        missing << 'vm_id' unless vm_id
+      end
+      return if missing.empty?
+      raise Fluent::ConfigError,
+            "Unable to obtain metadata parameters: #{missing.join(' ')}"
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it by calling metadata server directly.
+    # 3. If still not set, try to obtain it from the credentials.
+    def get_project_id(platform, project_id)
+      project_id ||= CredentialsInfo.project_id
+      project_id ||= fetch_gce_metadata(platform, 'project/project-id') if
+        platform == Platform::GCE
+      project_id
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it by calling metadata servers directly.
+    def get_vm_id(platform, vm_id)
+      vm_id ||= fetch_gce_metadata(platform, 'instance/id') if
+        platform == Platform::GCE
+      vm_id ||= ec2_metadata(platform)['instanceId'] if
+        platform == Platform::EC2
+      vm_id
+    rescue StandardError => e
+      @log.error 'Failed to obtain vm_id: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it locally.
+    def get_vm_name(vm_name)
+      vm_name ||= Socket.gethostname
+      vm_name
+    rescue StandardError => e
+      @log.error 'Failed to obtain vm name: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it locally.
+    def get_location(platform, zone, use_aws_availability_zone)
+      # Response format: "projects/<number>/zones/<zone>"
+      zone ||= fetch_gce_metadata(platform,
+                                  'instance/zone').rpartition('/')[2] if
+        platform == Platform::GCE
+      aws_location_key = if use_aws_availability_zone
+                           'availabilityZone'
+                         else
+                           'region'
+                         end
+      zone ||= 'aws:' + ec2_metadata(platform)[aws_location_key] if
+        platform == Platform::EC2 &&
+        ec2_metadata(platform).key?(aws_location_key)
+      zone
+    rescue StandardError => e
+      @log.error 'Failed to obtain location: ', error: e
+    end
+
+    # Create a monitored resource from type and labels.
+    def create_monitored_resource(type, labels)
+      Google::Apis::LoggingV2::MonitoredResource.new(
+        type: type, labels: labels.to_h)
+    end
+
+    # Retrieve monitored resource via the legacy way.
+    #
+    # Note: This is just a failover plan if we fail to get metadata from
+    # Metadata Agent. Thus it should be equivalent to what Metadata Agent
+    # returns.
+    def determine_agent_level_monitored_resource_via_legacy(
+          platform, subservice_name, detect_subservice, vm_id, zone)
+      resource_type = determine_agent_level_monitored_resource_type(
+        platform, subservice_name, detect_subservice)
+      create_monitored_resource(
+        resource_type,
+        determine_agent_level_monitored_resource_labels(
+          platform, resource_type, vm_id, zone))
+    end
+
+    # Determine agent level monitored resource type.
+    def determine_agent_level_monitored_resource_type(
+          platform, subservice_name, detect_subservice)
+      case platform
+      when Platform::OTHER
+        # Unknown platform will be defaulted to GCE instance.
+        return COMPUTE_CONSTANTS[:resource_type]
+
+      when Platform::EC2
+        return EC2_CONSTANTS[:resource_type]
+
+      when Platform::GCE
+        # Resource types determined by subservice_name config.
+        return SUBSERVICE_MAP[subservice_name] if subservice_name
+
+        # Resource types determined by detect_subservice config.
+        if detect_subservice
+          begin
+            attributes = fetch_gce_metadata(platform,
+                                            'instance/attributes/').split.to_set
+            SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
+              return resource_type if attributes.superset?(expected)
+            end
+          rescue StandardError => e
+            @log.error 'Failed to detect subservice: ', error: e
+          end
+        end
+
+        # GCE instance.
+        return COMPUTE_CONSTANTS[:resource_type]
+      end
+    end
+
+    # Determine agent level monitored resource labels based on the resource
+    # type. Each resource type has its own labels that need to be filled in.
+    def determine_agent_level_monitored_resource_labels(
+          platform, type, vm_id, zone)
+      case type
+      # GAE app.
+      when APPENGINE_CONSTANTS[:resource_type]
+        return {
+          'module_id' =>
+            fetch_gce_metadata(platform,
+                               'instance/attributes/gae_backend_name'),
+          'version_id' =>
+            fetch_gce_metadata(platform,
+                               'instance/attributes/gae_backend_version')
+        }
+
+      # GCE.
+      when COMPUTE_CONSTANTS[:resource_type]
+        raise "Cannot construct a #{type} resource without vm_id and zone" \
+          unless vm_id && zone
+        return {
+          'instance_id' => vm_id,
+          'zone' => zone
+        }
+
+      # GKE container.
+      when GKE_CONSTANTS[:resource_type]
+        raise "Cannot construct a #{type} resource without vm_id and zone" \
+          unless vm_id && zone
+        return {
+          'instance_id' => vm_id,
+          'zone' => zone,
+          'cluster_name' =>
+            fetch_gce_metadata(platform, 'instance/attributes/cluster-name')
+        }
+
+      # Cloud Dataproc.
+      when DATAPROC_CONSTANTS[:resource_type]
+        return {
+          'cluster_uuid' =>
+            fetch_gce_metadata(platform,
+                               'instance/attributes/dataproc-cluster-uuid'),
+          'cluster_name' =>
+            fetch_gce_metadata(platform,
+                               'instance/attributes/dataproc-cluster-name'),
+          'region' =>
+            fetch_gce_metadata(platform,
+                               'instance/attributes/dataproc-region')
+        }
+
+      # EC2.
+      when EC2_CONSTANTS[:resource_type]
+        raise "Cannot construct a #{type} resource without vm_id and zone" \
+          unless vm_id && zone
+        labels = {
+          'instance_id' => vm_id,
+          'region' => zone
+        }
+        labels['aws_account'] = ec2_metadata(platform)['accountId'] if
+          ec2_metadata(platform).key?('accountId')
+        return labels
+      end
+
+      {}
+    rescue StandardError => e
+      if [Platform::GCE, Platform::EC2].include?(platform)
+        @log.error "Failed to set monitored resource labels for #{type}: ",
+                   error: e
+      end
+      {}
+    end
+
+    # TODO: This functionality should eventually be available in another
+    # library, but implement it ourselves for now.
+    module CredentialsInfo
+      # Determine the project ID from the credentials, if possible.
+      # Returns the project ID (as a string) on success, or nil on failure.
+      def self.project_id
+        creds = Google::Auth.get_application_default(LOGGING_SCOPE)
+        if creds.respond_to?(:project_id)
+          return creds.project_id if creds.project_id
+        end
+        if creds.issuer
+          id = extract_project_id(creds.issuer)
+          return id unless id.nil?
+        end
+        if creds.client_id
+          id = extract_project_id(creds.client_id)
+          return id unless id.nil?
+        end
+        nil
+      end
+
+      # Extracts the project id (either name or number) from str and returns
+      # it (as a string) on success, or nil on failure.
+      #
+      # Recognizes IAM format (account@project-name.iam.gserviceaccount.com)
+      # as well as the legacy format with a project number at the front of the
+      # string, terminated by a dash (-) which is not part of the ID, i.e.:
+      # <PROJECT_ID>-<OTHER_PARTS>.apps.googleusercontent.com
+      def self.extract_project_id(str)
+        [/^.*@(?<project_id>.+)\.iam\.gserviceaccount\.com/,
+         /^(?<project_id>\d+)-/].each do |exp|
+          match_data = exp.match(str)
+          return match_data['project_id'] unless match_data.nil?
+        end
+        nil
+      end
+    end
+  end
+end