fluent-plugin-google-cloud 0.6.4.pre.3 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dacc67fcefff073a113c1c73ef7644ae802916e9
-  data.tar.gz: b2caa333b793ae00c8541e8b0c475fd0e71824a3
+  metadata.gz: ab431f0283f1c6bc88669159be387f21bb1e908f
+  data.tar.gz: 77d320fd8b6e4c5370542142fd296581fcd9186e
 SHA512:
-  metadata.gz: ee282c69851a0bf8dee3e2110f7a6635016c573af24f1b0757cf8d0a541b2396560976e8a064950087fc601a90441983dabb6312465d7873f90509236047717c
-  data.tar.gz: 5142fb67ba45c13759dfdd79d15f7e1cbfdf1a354da1423c2442bffc21003a1177410cbd8cb6c85495b9a501a3a74ea70a8dd1c87397199d38c1b7d74810640a
+  metadata.gz: 15fc471919366c52a612cb2b1941d0e9ffdf5e1d5c94ad6f3395c1860ca2f03edd888959a0ed267375bf80133458251bc1555c7416ef81f8be4f4aa83fd5c405
+  data.tar.gz: 4aed5e4e2686eb4776d32f3b44f61fc86849f4f02a64bd12399d0e0ac4a6cc1a8752489513fa5e482fdb7cac432d517ed5b7b26e71365e344e2dcf2ca8e96832

data/Gemfile.lock

@@ -0,0 +1,159 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-google-cloud (0.6.4)
+      fluentd (~> 0.10)
+      google-api-client (~> 0.9.0)
+      google-cloud-logging (~> 0.23.2)
+      googleapis-common-protos (~> 1.3)
+      googleauth (~> 0.4)
+      grpc (~> 1.0, < 1.3)
+      json (~> 1.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.1)
+      public_suffix (~> 2.0, >= 2.0.2)
+    ast (2.3.0)
+    astrolabe (1.3.1)
+      parser (~> 2.2)
+    cool.io (1.5.0)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    faraday (0.12.1)
+      multipart-post (>= 1.2, < 3)
+    fluentd (0.14.18)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    google-api-client (0.9.28)
+      addressable (~> 2.3)
+      googleauth (~> 0.5)
+      httpclient (~> 2.7)
+      hurley (~> 0.1)
+      memoist (~> 0.11)
+      mime-types (>= 1.6)
+      representable (~> 2.3.0)
+      retriable (~> 2.0)
+    google-cloud-core (0.21.1)
+      googleauth (~> 0.5.1)
+    google-cloud-logging (0.23.2)
+      google-cloud-core (~> 0.21.1)
+      google-gax (~> 0.6.0)
+      google-protobuf (~> 3.0)
+      googleapis-common-protos (~> 1.3)
+      grpc (~> 1.0)
+      orderedhash (= 0.0.6)
+      stackdriver-core (~> 0.21.0)
+    google-gax (0.6.0)
+      googleapis-common-protos (~> 1.3.1)
+      googleauth (~> 0.5.1)
+      grpc (~> 1.0)
+      rly (~> 0.2.3)
+    google-protobuf (3.3.0)
+    googleapis-common-protos (1.3.5)
+      google-protobuf (~> 3.2)
+      grpc (~> 1.0)
+    googleauth (0.5.1)
+      faraday (~> 0.9)
+      jwt (~> 1.4)
+      logging (~> 2.0)
+      memoist (~> 0.12)
+      multi_json (~> 1.11)
+      os (~> 0.9)
+      signet (~> 0.7)
+    grpc (1.2.5)
+      google-protobuf (~> 3.1)
+      googleauth (~> 0.5.1)
+    hashdiff (0.3.4)
+    http_parser.rb (0.6.0)
+    httpclient (2.8.3)
+    hurley (0.2)
+    json (1.8.6)
+    jwt (1.5.6)
+    little-plugger (1.1.4)
+    logging (2.2.2)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
+    memoist (0.16.0)
+    metaclass (0.0.4)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mocha (1.2.1)
+      metaclass (~> 0.0.1)
+    msgpack (1.1.0)
+    multi_json (1.12.1)
+    multipart-post (2.0.0)
+    orderedhash (0.0.6)
+    os (0.9.6)
+    parser (2.4.0.0)
+      ast (~> 2.2)
+    power_assert (1.0.2)
+    powerpack (0.1.1)
+    prometheus-client (0.7.1)
+      quantile (~> 0.2.0)
+    public_suffix (2.0.5)
+    quantile (0.2.0)
+    rainbow (2.2.2)
+      rake
+    rake (10.5.0)
+    representable (2.3.0)
+      uber (~> 0.0.7)
+    retriable (2.1.0)
+    rly (0.2.3)
+    rubocop (0.35.1)
+      astrolabe (~> 1.3)
+      parser (>= 2.2.3.0, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      tins (<= 1.6.0)
+    ruby-progressbar (1.8.1)
+    safe_yaml (1.0.4)
+    serverengine (2.0.5)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    signet (0.7.3)
+      addressable (~> 2.3)
+      faraday (~> 0.9)
+      jwt (~> 1.5)
+      multi_json (~> 1.10)
+    stackdriver-core (0.21.0)
+    strptime (0.1.9)
+    test-unit (3.2.5)
+      power_assert
+    thread_safe (0.3.6)
+    tins (1.6.0)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2017.2)
+      tzinfo (>= 1.0.0)
+    uber (0.0.15)
+    webmock (1.24.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    yajl-ruby (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-google-cloud!
+  mocha (~> 1.1)
+  prometheus-client (~> 0.7.1)
+  rake (~> 10.3)
+  rubocop (~> 0.35.0)
+  test-unit (~> 3.0)
+  webmock (~> 1.17)
+
+BUNDLED WITH
+   1.15.0

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.4.pre.3'
+  gem.version       = '0.6.4'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -19,7 +19,6 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'excon', '~> 0.57.1'
   gem.add_runtime_dependency 'fluentd', '~> 0.10'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
   gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'
@@ -33,4 +32,5 @@ eos
   gem.add_development_dependency 'rubocop', '~> 0.35.0'
   gem.add_development_dependency 'webmock', '~> 1.17'
   gem.add_development_dependency 'test-unit', '~> 3.0'
+  gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
 end

data/lib/fluent/plugin/monitoring.rb

@@ -0,0 +1,55 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Monitoring
+  # Base class for the monitoring registry.
+  class BaseMonitoringRegistry
+    def counter(_name, _desc)
+      _undefined
+    end
+  end
+
+  # Prometheus implementation of the monitoring registry, that uses the default
+  # registry in the official Prometheus client library.
+  class PrometheusMonitoringRegistry < BaseMonitoringRegistry
+    def self.name
+      'prometheus'
+    end
+
+    def initialize
+      require 'prometheus/client'
+      @registry = Prometheus::Client.registry
+    end
+
+    # Exception-driven behavior to avoid synchronization errors.
+    def counter(name, desc)
+      return @registry.counter(name, desc)
+    rescue Prometheus::Client::Registry::AlreadyRegisteredError
+      return @registry.get(name)
+    end
+  end
+
+  # Factory that is used to create a monitoring registry based on
+  # the monitoring solution name.
+  class MonitoringRegistryFactory
+    @known_registry_types = {
+      PrometheusMonitoringRegistry.name =>
+        PrometheusMonitoringRegistry
+    }
+
+    def self.create(name)
+      (@known_registry_types[name] || BaseMonitoringRegistry).new
+    end
+  end
+end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -11,11 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-require 'excon'
 require 'grpc'
 require 'json'
 require 'open-uri'
-require 'rubygems'
 require 'socket'
 require 'time'
 require 'yaml'
@@ -26,6 +24,8 @@ require 'google/logging/v2/logging_services_pb'
 require 'google/logging/v2/log_entry_pb'
 require 'googleauth'
 
+require_relative 'monitoring'
+
 module Google
   module Protobuf
     # Alias the has_key? method to have the same interface as a regular map.
@@ -38,10 +38,8 @@ end
 module Fluent
   # fluentd output plugin for the Stackdriver Logging API
   class GoogleCloudOutput < BufferedOutput
-    # Constants.
+    # Constants for service names and resource types.
     module Constants
-      # Service names and resource types.
-
       APPENGINE_CONSTANTS = {
         service: 'appengine.googleapis.com',
         resource_type: 'gae_app'
@@ -58,10 +56,6 @@ module Fluent
         service: 'container.googleapis.com',
         resource_type: 'container'
       }
-      DOCKER_CONSTANTS = {
-        service: 'dockercontainer.googleapis.com',
-        resource_type: 'docker_container'
-      }
       DATAFLOW_CONSTANTS = {
         service: 'dataflow.googleapis.com',
         resource_type: 'dataflow_step'
@@ -78,20 +72,6 @@ module Fluent
         service: 'ml.googleapis.com',
         resource_type: 'ml_job'
       }
-
-      # Metadata Agent support.
-
-      # Use empty string as request path when locally-unique key of monitored
-      # resource can be implicitly inferred by Metadata Agent.
-      IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY = ''
-
-      # The label name of locally unique id in the json payload. When a record
-      # has this field in the payload, we will use the value to retrieve
-      # monitored resource from Stackdriver Metadata agent.
-      LOCALLY_UNIQUE_ID_LABEL_NAME = 'logging.googleapis.com/locally_unique_id'
-
-      # Docker container support.
-      DEFAULT_DOCKER_API_SOCKET_PATH = '/var/run/docker.sock'
     end
 
     include self::Constants
@@ -99,7 +79,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.4.pre.3'
+    PLUGIN_VERSION = '0.6.3'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -146,7 +126,7 @@ module Fluent
     config_param :require_valid_tags, :bool, :default => false
 
     # The regular expression to use on Kubernetes logs to extract some basic
-    # information about the log source. The regexp must contain capture groups
+    # information about the log source. The regex must contain capture groups
     # for pod_name, namespace_name, and container_name.
     config_param :kubernetes_tag_regexp, :string, :default =>
       '\.(?<pod_name>[^_]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$'
@@ -208,17 +188,18 @@ module Fluent
                  :default => nil,
                  :secret => true
 
-    # Whether to call metadata agent to retrieve monitored resource.
-    config_param :enable_metadata_agent, :bool, :default => false
-    config_param :metadata_agent_url, :string,
-                 :default => 'http://local-metadata-agent.stackdriver.com:8000'
+    # Whether to collect metrics about the plugin usage. The mechanism for
+    # collecting and exposing metrics is controlled by the monitoring_type
+    # parameter.
+    config_param :enable_monitoring, :bool, :default => false
 
-    # Whether to call Docker Remote API locally when Metadata Agent is not
-    # enabled or if the request fails.
-    config_param :call_docker_api_locally, :bool, :default => true
-    # Docker Remote API unix socket path.
-    config_param :docker_remote_api_socket_path, :string,
-                 :default => DEFAULT_DOCKER_API_SOCKET_PATH
+    # What system to use when collecting metrics. Possible values are:
+    #   - 'prometheus', in this case default registry in the Prometheus
+    #     client library is used, without actually exposing the endpoint
+    #     to serve metrics in the Prometheus format.
+    #    - any other value will result in the absence of metrics.
+    config_param :monitoring_type, :string,
+                 :default => Monitoring::PrometheusMonitoringRegistry.name
 
     # rubocop:enable Style/HashSyntax
 
@@ -229,6 +210,9 @@ module Fluent
     attr_reader :project_id
     attr_reader :zone
     attr_reader :vm_id
+    attr_reader :running_on_managed_vm
+    attr_reader :gae_backend_name
+    attr_reader :gae_backend_version
     attr_reader :resource
     attr_reader :common_labels
 
@@ -238,38 +222,28 @@ module Fluent
       @log = $log # rubocop:disable Style/GlobalVars
     end
 
-    # Set up regex patterns used to parse tags and logs.
-    def setup_regex_patterns
-      @compiled_kubernetes_tag_regexp = nil
-      if @kubernetes_tag_regexp
-        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
-      end
-
-      @cloudfunctions_tag_regexp =
-        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
-      @cloudfunctions_log_regexp = /^
-        (?:\[(?<severity>.)\])?
-        \[(?<timestamp>.{24})\]
-        (?:\[(?<execution_id>[^\]]+)\])?
-        [ ](?<text>.*)$/x
-
-      # Docker container tag format:
-      # "container.<container_id>.<container_name>".
-      @dockercontainer_tag_regexp =
-        /^container\.(?<container_id>[a-zA-Z0-9]+)\.
-        (?<container_name>[a-zA-Z0-9_.-]+)$/x
-      # Docker container with application tag format:
-      # "application-container.<container_name>.<additional_tag>".
-      @dockercontainer_tag_with_application_regexp =
-        /^application-container\.(?<container_name>[a-zA-Z0-9_.-]+)\.
-        (?<additional_tag>.+)$/x
-
-      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
-    end
-
     def configure(conf)
       super
 
+      # If monitoring is enabled, register metrics in the default registry
+      # and store metric objects for future use.
+      if @enable_monitoring
+        registry = Monitoring::MonitoringRegistryFactory.create @monitoring_type
+        @successful_requests_count = registry.counter(
+          :stackdriver_successful_requests_count,
+          'A number of successful requests to the Stackdriver Logging API')
+        @failed_requests_count = registry.counter(
+          :stackdriver_failed_requests_count,
+          'A number of failed requests to the Stackdriver Logging API,'\
+            ' broken down by the error code')
+        @ingested_entries_count = registry.counter(
+          :stackdriver_ingested_entries_count,
+          'A number of log entries ingested by Stackdriver Logging')
+        @dropped_entries_count = registry.counter(
+          :stackdriver_dropped_entries_count,
+          'A number of log entries dropped by the Stackdriver output plugin')
+      end
+
       # Alert on old authentication configuration.
       unless @auth_method.nil? && @private_key_email.nil? &&
              @private_key_path.nil? && @private_key_passphrase.nil?
@@ -285,57 +259,160 @@ module Fluent
                extra.join(' ')
       end
 
-      setup_regex_patterns
+      # TODO: Send instance tags as labels as well?
+      @common_labels = {}
+      @common_labels.merge!(@labels) if @labels
+
+      # TODO: Construct Google::Api::MonitoredResource when @use_grpc is
+      # true after the protobuf map corruption issue is fixed.
+      @resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
+        labels: {})
+
+      @compiled_kubernetes_tag_regexp = nil
+      if @kubernetes_tag_regexp
+        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
+      end
 
+      @cloudfunctions_tag_regexp =
+        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
+      @cloudfunctions_log_regexp = /^
+        (?:\[(?<severity>.)\])?
+        \[(?<timestamp>.{24})\]
+        (?:\[(?<execution_id>[^\]]+)\])?
+        [ ](?<text>.*)$/x
+
+      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
+
+      # set attributes from metadata (unless overriden by static config)
+      @vm_name = Socket.gethostname if @vm_name.nil?
       @platform = detect_platform
+      case @platform
+      when Platform::GCE
+        if @project_id.nil?
+          @project_id = fetch_gce_metadata('project/project-id')
+        end
+        if @zone.nil?
+          # this returns "projects/<number>/zones/<zone>"; we only want
+          # the part after the final slash.
+          fully_qualified_zone = fetch_gce_metadata('instance/zone')
+          @zone = fully_qualified_zone.rpartition('/')[2]
+        end
+        @vm_id = fetch_gce_metadata('instance/id') if @vm_id.nil?
+      when Platform::EC2
+        metadata = fetch_ec2_metadata
+        if @zone.nil? && metadata.key?('availabilityZone')
+          @zone = 'aws:' + metadata['availabilityZone']
+        end
+        if @vm_id.nil? && metadata.key?('instanceId')
+          @vm_id = metadata['instanceId']
+        end
+        if metadata.key?('accountId')
+          @resource.labels['aws_account'] = metadata['accountId']
+        end
+      when Platform::OTHER
+        # do nothing
+      else
+        fail Fluent::ConfigError, 'Unknown platform ' + @platform
+      end
 
-      # Set agent-level monitored resource. This monitored resource is initiated
-      # as the logging agent starts up. It will be inherited by all log entries
-      # processed by this agent. First try to retrieve it via Metadata Agent.
-      if @enable_metadata_agent
-        # The locally-unique key for this should be the instance id. Since this
-        # can be implicitly inferred by Metadata Agent, we do not need to
-        # explicitly send the key.
-        @resource = call_metadata_agent_for_monitored_resource(
-          IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
+      # If we still don't have a project ID, try to obtain it from the
+      # credentials.
+      if @project_id.nil?
+        @project_id = CredentialsInfo.project_id
+        @log.info 'Set Project ID from credentials: ', @project_id unless
+          @project_id.nil?
       end
 
-      # Set required variables: @project_id, @vm_id, @vm_name and @zone.
-      # If any info above is included in the response from Metadata Agent, make
-      # use of that. Otherwise make some additional requests to metadata server.
-      #
-      # Note: Once we support metadata injection on the Logging API side, we
-      # might no longer need to require all these metadata in logging agent. But
-      # for now, they are still required.
-      set_required_metadata_variables
-
-      # Fail over to retrieve monitored resource via the legacy path if we fail
-      # to get it from Metadata Agent.
-      @resource ||= determine_agent_level_monitored_resource_via_legacy
-
-      # Set variables specific to CLoud Functions. This has to be called after
-      # we have determined the resource type. The purpose is to avoid repeated
-      # calls to metadata server.
-      @running_cloudfunctions = false
-      # We only support Cloud Functions logs for GKE right now.
-      if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
-         fetch_gce_metadata('instance/attributes/').split.include?('gcf_region')
-        # We are not setting resource type as Cloud Functions here because
-        # whether a log entry is truly coming from a Cloud Functions function
-        # depends on the log tag. Only when @running_cloudfunctions is true will
-        # we try to match log tags against Cloud Functions tag regexp when
-        # processing log entries.
-        @running_cloudfunctions = true
-        # Fetch this info and store it to avoid recurring metadata server calls.
-        @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
+      # all metadata parameters must now be set
+      unless @project_id && @zone && @vm_id
+        missing = []
+        missing << 'project_id' unless @project_id
+        missing << 'zone' unless @zone
+        missing << 'vm_id' unless @vm_id
+        fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
+          missing.join(' ')
       end
 
-      # Determine the common labels that should be added to all log entries
-      # processed by this logging agent.
-      @common_labels = determine_agent_level_common_labels
+      # Default this to false; it is only overwritten if we detect Managed VM.
+      @running_on_managed_vm = false
 
-      # For each resource type, there is a list of labels that we want to report
-      # as monitored resource instead of metadata labels. Move them if present.
+      # Default this to false; it is only overwritten if we detect Cloud
+      # Functions.
+      @running_cloudfunctions = false
+
+      # Set up the MonitoredResource, labels, etc. based on the config.
+      case @platform
+      when Platform::GCE
+        @resource.type = COMPUTE_CONSTANTS[:resource_type]
+        # TODO: introduce a new MonitoredResource-centric configuration and
+        # deprecate subservice-name; for now, translate known uses.
+        if @subservice_name
+          # TODO: what should we do if we encounter an unknown value?
+          if @subservice_name == DATAFLOW_CONSTANTS[:service]
+            @resource.type = DATAFLOW_CONSTANTS[:resource_type]
+          elsif @subservice_name == ML_CONSTANTS[:service]
+            @resource.type = ML_CONSTANTS[:resource_type]
+          end
+        elsif @detect_subservice
+          # Check for specialized GCE environments.
+          # TODO: Add config options for these to allow for running outside GCE?
+          attributes = fetch_gce_metadata('instance/attributes/').split
+          # Do nothing, just don't populate other service's labels.
+          if attributes.include?('gae_backend_name') &&
+             attributes.include?('gae_backend_version')
+            # Managed VM
+            @running_on_managed_vm = true
+            @gae_backend_name =
+                fetch_gce_metadata('instance/attributes/gae_backend_name')
+            @gae_backend_version =
+                fetch_gce_metadata('instance/attributes/gae_backend_version')
+            @resource.type = APPENGINE_CONSTANTS[:resource_type]
+            @resource.labels['module_id'] = @gae_backend_name
+            @resource.labels['version_id'] = @gae_backend_version
+          elsif attributes.include?('kube-env')
+            # Kubernetes/Container Engine
+            @resource.type = CONTAINER_CONSTANTS[:resource_type]
+            @raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
+            @kube_env = YAML.load(@raw_kube_env)
+            @resource.labels['cluster_name'] =
+              cluster_name_from_kube_env(@kube_env)
+            detect_cloudfunctions(attributes)
+          elsif attributes.include?('dataproc-cluster-uuid') &&
+                attributes.include?('dataproc-cluster-name')
+            # Dataproc
+            @resource.type = DATAPROC_CONSTANTS[:resource_type]
+            @resource.labels['cluster_uuid'] =
+              fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid')
+            @resource.labels['cluster_name'] =
+              fetch_gce_metadata('instance/attributes/dataproc-cluster-name')
+            @resource.labels['region'] =
+              fetch_gce_metadata('instance/attributes/dataproc-region')
+          end
+        end
+        # Some services have the GCE instance_id and zone as MonitoredResource
+        # labels; for other services we send them as entry labels.
+        if @resource.type == COMPUTE_CONSTANTS[:resource_type] ||
+           @resource.type == CONTAINER_CONSTANTS[:resource_type]
+          @resource.labels['instance_id'] = @vm_id
+          @resource.labels['zone'] = @zone
+        else
+          common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
+          common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
+        end
+        common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+      when Platform::EC2
+        @resource.type = EC2_CONSTANTS[:resource_type]
+        @resource.labels['instance_id'] = @vm_id
+        @resource.labels['region'] = @zone
+        # the aws_account label is populated above.
+        common_labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
+      when Platform::OTHER
+        # Use GCE as the default environment.
+        @resource.type = COMPUTE_CONSTANTS[:resource_type]
+        @resource.labels['instance_id'] = @vm_id
+        @resource.labels['zone'] = @zone
+        common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+      end
       @resource.labels.merge!(
         extract_resource_labels(@resource.type, common_labels))
 
@@ -347,7 +424,7 @@ module Fluent
 
       # Log an informational message containing the Logs viewer URL
       @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
-                "viewer?project=#{@project_id}&resource=#{@resource.type}/",
+                "viewer?project=#{@project_id}&resource=#{@resource_type}/",
                 "instance_id/#{@vm_id}"
     end
 
@@ -362,15 +439,135 @@ module Fluent
       super
     end
 
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # Given a tag, returns the corresponding valid tag if possible, or nil if
+    # the tag should be rejected. If 'require_valid_tags' is false, non-string
+    # tags are converted to strings, and invalid characters are sanitized;
+    # otherwise such tags are rejected.
+    def sanitize_tag(tag)
+      if @require_valid_tags &&
+         (!tag.is_a?(String) || tag == '' || convert_to_utf8(tag) != tag)
+        return nil
+      end
+      tag = convert_to_utf8(tag.to_s)
+      tag = '_' if tag == ''
+      tag
+    end
+
+    # Compute the monitored resource and common labels shared by a collection of
+    # entries.
+    def compute_group_resource_and_labels(tag)
+      # Note that we assume that labels added to group_common_labels below are
+      # not 'service' labels (i.e. we do not call extract_resource_labels
+      # again).
+      group_resource = @resource.dup
+      group_common_labels = @common_labels.dup
+
+      if @running_cloudfunctions
+        # If the current group of entries is coming from a Cloud Functions
+        # function, the function name can be extracted from the tag.
+        match_data = @cloudfunctions_tag_regexp.match(tag)
+        if match_data
+          # Resource type is set to Cloud Functions only for logs actually
+          # coming from a function, otherwise we leave it as Container.
+          group_resource.type = CLOUDFUNCTIONS_CONSTANTS[:resource_type]
+          group_resource.labels['region'] = @gcf_region
+          group_resource.labels['function_name'] =
+            decode_cloudfunctions_function_name(
+              match_data['encoded_function_name'])
+          # Move GKE container labels from the MonitoredResource to the
+          # LogEntry.
+          instance_id = group_resource.labels.delete('instance_id')
+          group_common_labels["#{CONTAINER_CONSTANTS[:service]}/cluster_name"] =
+            group_resource.labels.delete('cluster_name')
+          group_common_labels["#{CONTAINER_CONSTANTS[:service]}/instance_id"] =
+            instance_id
+          group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
+            instance_id
+          group_common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] =
+            group_resource.labels.delete('zone')
+        end
+      end
+      if group_resource.type == CONTAINER_CONSTANTS[:resource_type] &&
+         @compiled_kubernetes_tag_regexp
+        # Container logs in Kubernetes are tagged based on where they came
+        # from, so we can extract useful metadata from the tag.
+        # Do this here to avoid having to repeat it for each record.
+        match_data = @compiled_kubernetes_tag_regexp.match(tag)
+        if match_data
+          group_resource.labels['container_name'] = match_data['container_name']
+          group_resource.labels['namespace_id'] = match_data['namespace_name']
+          group_resource.labels['pod_id'] = match_data['pod_name']
+          %w(namespace_name pod_name).each do |field|
+            group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
+              match_data[field]
+          end
+        end
+      end
+
+      # Freeze the per-request state. Any further changes must be made on a
+      # per-entry basis.
+      group_resource.freeze
+      group_resource.labels.freeze
+      group_common_labels.freeze
+
+      [group_resource, group_common_labels]
+    end
+
+    # Extract entry resource and common labels that should be applied to
+    # individual entries from the group resource.
+    def extract_entry_labels(group_resource, record)
+      resource_labels = {}
+      common_labels = {}
+
+      if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
+         record.key?('log')
+        @cloudfunctions_log_match =
+          @cloudfunctions_log_regexp.match(record['log'])
+      end
+
+      if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
+        # Move the stdout/stderr annotation from the record into a label
+        common_labels.merge!(
+          fields_to_labels(
+            record, 'stream' => "#{CONTAINER_CONSTANTS[:service]}/stream"))
+
+        # If the record has been annotated by the kubernetes_metadata_filter
+        # plugin, then use that metadata. Otherwise, rely on commonLabels
+        # populated at the grouped_entries level from the group's tag.
+        if record.key?('kubernetes')
+          extracted_resource_labels, extracted_common_labels = \
+            extract_container_metadata(record)
+          resource_labels.merge!(extracted_resource_labels)
+          common_labels.merge!(extracted_common_labels)
+        end
+      end
+
+      # If a field is present in the label_map, send its value as a label
+      # (mapping the field name to label name as specified in the config)
+      # and do not send that field as part of the payload.
+      common_labels.merge!(fields_to_labels(record, @label_map))
+
+      if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
+         @cloudfunctions_log_match &&
+         @cloudfunctions_log_match['execution_id']
+        common_labels['execution_id'] =
+          @cloudfunctions_log_match['execution_id']
+      end
+      resource_labels.merge!(
+        extract_resource_labels(group_resource.type, common_labels))
+
+      [resource_labels, common_labels]
+    end
+
     def write(chunk)
       # Group the entries since we have to make one call per tag.
       grouped_entries = {}
       chunk.msgpack_each do |tag, *arr|
         sanitized_tag = sanitize_tag(tag)
-        @log.error("tag is: ")
-        @log.error(tag.inspect)
-        @log.error("arr is: ")
-        @log.error(arr.inspect)
         if sanitized_tag.nil?
           @log.warn "Dropping log entries with invalid tag: '#{tag}'. " \
                     'A tag should be a string with utf8 characters.'
@@ -382,22 +579,20 @@ module Fluent
 
       grouped_entries.each do |tag, arr|
         entries = []
-        group_resource, group_common_labels =
-          determine_group_level_monitored_resource_and_labels(tag)
+        group_resource, group_common_labels = compute_group_resource_and_labels(
+          tag)
 
         arr.each do |time, record|
           next unless record.is_a?(Hash)
 
-          resource_type, extracted_resource_labels, extracted_common_labels = \
-            determine_entry_level_labels(group_resource, record)
+          extracted_resource_labels, extracted_common_labels = \
+            extract_entry_labels(group_resource, record)
           entry_resource = group_resource.dup
-          entry_resource.type = resource_type
           entry_resource.labels.merge!(extracted_resource_labels)
           entry_common_labels = \
             group_common_labels.merge(extracted_common_labels)
 
-          if [CONTAINER_CONSTANTS[:resource_type],
-              DOCKER_CONSTANTS[:resource_type]].include?(entry_resource.type)
+          if entry_resource.type == CONTAINER_CONSTANTS[:resource_type]
             # Save the timestamp if available, then clear it out to allow for
             # determining whether we should parse the log or message field.
             timestamp = record.key?('time') ? record['time'] : nil
@@ -491,6 +686,8 @@ module Fluent
             )
 
             client.write_log_entries(write_request)
+            increment_successful_requests_count
+            increment_ingested_entries_count(entries.length)
 
             # Let the user explicitly know when the first call succeeded,
             # to aid with verification and troubleshooting.
@@ -500,10 +697,12 @@ module Fluent
             end
 
           rescue GRPC::Cancelled => error
+            increment_failed_requests_count(GRPC::Core::StatusCodes::CANCELLED)
             # RPC cancelled, so retry via re-raising the error.
             raise error
 
           rescue GRPC::BadStatus => error
+            increment_failed_requests_count(error.code)
             case error.code
             when GRPC::Core::StatusCodes::CANCELLED,
                  GRPC::Core::StatusCodes::UNAVAILABLE,
@@ -518,6 +717,7 @@ module Fluent
               # Most client errors indicate a problem with the request itself
               # and should not be retried.
               dropped = entries.length
+              increment_dropped_entries_count(dropped)
               @log.warn "Dropping #{dropped} log message(s)",
                         error: error.to_s, error_code: error.code.to_s
             when GRPC::Core::StatusCodes::UNAUTHENTICATED
@@ -525,12 +725,14 @@ module Fluent
               # These are usually solved via a `gcloud auth` call, or by
               # modifying the permissions on the Google Cloud project.
               dropped = entries.length
+              increment_dropped_entries_count(dropped)
               @log.warn "Dropping #{dropped} log message(s)",
                         error: error.to_s, error_code: error.code.to_s
             else
               # Assume this is a problem with the request itself
               # and don't retry.
               dropped = entries.length
+              increment_dropped_entries_count(dropped)
               @log.error "Unknown response code #{error.code} from the "\
                          "server, dropping #{dropped} log message(s)",
                          error: error.to_s, error_code: error.code.to_s
@@ -546,7 +748,14 @@ module Fluent
                 entries: entries)
 
             # TODO: RequestOptions
-            client.write_entry_log_entries(write_request)
+            begin
+              client.write_entry_log_entries(write_request)
+            rescue Google::Apis::Error => error
+              increment_failed_requests_count(error.status_code)
+              raise error
+            end
+            increment_successful_requests_count
+            increment_ingested_entries_count(entries.length)
 
             # Let the user explicitly know when the first call succeeded,
             # to aid with verification and troubleshooting.
@@ -564,6 +773,7 @@ module Fluent
             # These are usually solved via a `gcloud auth` call, or by modifying
             # the permissions on the Google Cloud project.
             dropped = entries.length
+            increment_dropped_entries_count(dropped)
             @log.warn "Dropping #{dropped} log message(s)",
                       error_class: error.class.to_s, error: error.to_s
 
@@ -571,6 +781,7 @@ module Fluent
             # Most ClientErrors indicate a problem with the request itself and
             # should not be retried.
             dropped = entries.length
+            increment_dropped_entries_count(dropped)
             @log.warn "Dropping #{dropped} log message(s)",
                       error_class: error.class.to_s, error: error.to_s
           end
@@ -630,7 +841,7 @@ module Fluent
           end
         end
       rescue StandardError => e
-        @log.error 'Failed to access metadata service: ', error: e
+        @log.debug 'Failed to access metadata service: ', error: e
       end
 
       @log.info 'Unable to determine platform'
@@ -645,600 +856,15 @@ module Fluent
            metadata_path, 'Metadata-Flavor' => 'Google', &:read)
     end
 
-    # EC2 Metadata server returns everything in one call. Store it after the
-    # first fetch to avoid making multiple calls.
-    def ec2_metadata
-      fail "Called ec2_metadata with platform=#{@platform}" unless
+    def fetch_ec2_metadata
+      fail "Called fetch_ec2_metadata with platform=#{@platform}" unless
         @platform == Platform::EC2
-      unless @ec2_metadata
-        # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
-        open('http://' + METADATA_SERVICE_ADDR +
-             '/latest/dynamic/instance-identity/document') do |f|
-          contents = f.read
-          @ec2_metadata = JSON.parse(contents)
-        end
-      end
-
-      @ec2_metadata
-    end
-
-    # Set regexp patterns to parse tags and logs.
-    def set_regexp_patterns
-      @compiled_kubernetes_tag_regexp = nil
-      if @kubernetes_tag_regexp
-        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
+      # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+      open('http://' + METADATA_SERVICE_ADDR +
+           '/latest/dynamic/instance-identity/document') do |f|
+        contents = f.read
+        return JSON.parse(contents)
       end
-
-      @cloudfunctions_tag_regexp =
-        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
-      @cloudfunctions_log_regexp = /^
-        (?:\[(?<severity>.)\])?
-        \[(?<timestamp>.{24})\]
-        (?:\[(?<execution_id>[^\]]+)\])?
-        [ ](?<text>.*)$/x
-
-      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
-    end
-
-    # Set required variables like @project_id, @vm_id, @vm_name and @zone.
-    def set_required_metadata_variables
-      set_project_id
-      set_vm_id
-      set_vm_name
-      set_location
-
-      # All metadata parameters must now be set.
-      return if @project_id && @zone && @vm_id
-      missing = []
-      missing << 'project_id' unless @project_id
-      missing << 'zone' unless @zone
-      missing << 'vm_id' unless @vm_id
-      fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
-        missing.join(' ')
-    end
-
-    # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, try to retrieve it by calling metadata server directly.
-    # 3. If still not set, try to obtain it from the credentials.
-    def set_project_id
-      @project_id ||= fetch_gce_metadata('project/project-id') if
-        @platform == Platform::GCE
-      @project_id ||= CredentialsInfo.project_id
-    rescue StandardError => e
-      @log.error 'Failed to obtain project id: ', error: e
-    end
-
-    # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, check if the response from Metadata Agent includes this info.
-    # 3. If not, try to retrieve it by calling metadata servers directly.
-    def set_vm_id
-      @vm_id ||= @resource.labels['instance_id'] if
-        !@resource.nil? && @resource.labels.key?('instance_id')
-      @vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
-      @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
-    rescue StandardError => e
-      @log.error 'Failed to obtain vm_id: ', error: e
-    end
-
-    # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, check if the response from Metadata Agent includes this info.
-    # 3. If not, try to retrieve it locally.
-    def set_vm_name
-      @vm_name ||= @resource.labels['instance_name'] if
-        !@resource.nil? && @resource.labels.key?('instance_name')
-      @vm_name ||= Socket.gethostname
-    rescue StandardError => e
-      @log.error 'Failed to obtain vm name: ', error: e
-    end
-
-    # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, check if the response from Metadata Agent includes this info.
-    # 3. If not, try to retrieve it locally.
-    def set_location
-      unless @resource.nil?
-        @zone ||= @resource.labels['location'] if
-          @resource.type == DOCKER_CONSTANTS[:resource_type] &&
-          @resource.labels.key?('location')
-        @zone ||= @resource.labels['zone'] if
-          @platform == Platform::GCE && @resource.labels.key?('zone')
-        @zone ||= @resource.labels['region'] if
-          @platform == Platform::EC2 && @resource.labels.key?('region')
-      end
-      # Response format: "projects/<number>/zones/<zone>"
-      @zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
-        @platform == Platform::GCE
-      @zone ||= 'aws:' + ec2_metadata['availabilityZone'] if
-        @platform == Platform::EC2 && ec2_metadata.key?('availabilityZone')
-    rescue StandardError => e
-      @log.error 'Failed to obtain location: ', error: e
-    end
-
-    # Retrieve monitored resource via the legacy way.
-    #
-    # Note: This is just a failover plan if we fail to get metadata from
-    # Metadata Agent. Thus it should be equivalent to what Metadata Agent
-    # returns.
-    def determine_agent_level_monitored_resource_via_legacy
-      resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
-        labels: {})
-      resource.type = determine_agent_level_monitored_resource_type
-      resource.labels = determine_agent_level_monitored_resource_labels(
-        resource.type)
-      resource
-    end
-
-    # Determine agent level monitored resource type.
-    def determine_agent_level_monitored_resource_type
-      # EC2 instance.
-      return EC2_CONSTANTS[:resource_type] if
-        @platform == Platform::EC2
-
-      # Unknown platform will be defaulted to GCE instance..
-      return COMPUTE_CONSTANTS[:resource_type] if
-        @platform == Platform::OTHER
-
-      # Resource types determined by @subservice_name config.
-      # Cloud Dataflow.
-      return DATAFLOW_CONSTANTS[:resource_type] if
-        @subservice_name == DATAFLOW_CONSTANTS[:service]
-      # Cloud ML.
-      return ML_CONSTANTS[:resource_type] if
-        @subservice_name == ML_CONSTANTS[:service]
-      # Default back to GCE if invalid value is detected.
-      return COMPUTE_CONSTANTS[:resource_type] if
-        @subservice_name
-
-      # Resource types determined by @detect_subservice config.
-      if @detect_subservice
-        begin
-          attributes = fetch_gce_metadata('instance/attributes/').split
-        rescue StandardError => e
-          @log.error 'Failed to detect subservice: ', error: e
-        end
-        # GAE app.
-        return APPENGINE_CONSTANTS[:resource_type] if
-          attributes.include?('gae_backend_name') &&
-          attributes.include?('gae_backend_version')
-        # GKE container.
-        return CONTAINER_CONSTANTS[:resource_type] if
-          attributes.include?('kube-env')
-        # Cloud Dataproc.
-        return DATAPROC_CONSTANTS[:resource_type] if
-          attributes.include?('dataproc-cluster-uuid') &&
-          attributes.include?('dataproc-cluster-name')
-      end
-      # GCE instance.
-      COMPUTE_CONSTANTS[:resource_type]
-    end
-
-    # Determine agent level monitored resource labels based on the resource
-    # type. Each resource type has its own labels that need to be filled in.
-    def determine_agent_level_monitored_resource_labels(type)
-      labels = {}
-
-      case type
-
-      # GAE app.
-      when APPENGINE_CONSTANTS[:resource_type]
-        begin
-          labels['module_id'] = fetch_gce_metadata(
-            'instance/attributes/gae_backend_name')
-          labels['version_id'] = fetch_gce_metadata(
-            'instance/attributes/gae_backend_version')
-        rescue StandardError => e
-          @log.error 'Failed to set monitored resource labels for GAE: ',
-                     error: e
-        end
-
-      # GCE.
-      when COMPUTE_CONSTANTS[:resource_type]
-        labels['instance_id'] = @vm_id
-        labels['zone'] = @zone
-
-      # GKE container.
-      when CONTAINER_CONSTANTS[:resource_type]
-        labels['instance_id'] = @vm_id
-        labels['zone'] = @zone
-        begin
-          raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
-          kube_env = YAML.load(raw_kube_env)
-          labels['cluster_name'] =
-            cluster_name_from_kube_env(kube_env)
-        rescue StandardError => e
-          @log.error 'Failed to set monitored resource labels for GKE: ',
-                     error: e
-        end
-
-      # Cloud Dataproc.
-      when DATAPROC_CONSTANTS[:resource_type]
-        begin
-          labels['cluster_uuid'] =
-            fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid')
-          labels['cluster_name'] =
-            fetch_gce_metadata('instance/attributes/dataproc-cluster-name')
-          labels['region'] =
-            fetch_gce_metadata('instance/attributes/dataproc-region')
-        rescue StandardError => e
-          @log.error 'Failed to set monitored resource labels for Cloud ' \
-                     'Dataproc: ', error: e
-        end
-
-      # EC2.
-      when EC2_CONSTANTS[:resource_type]
-        labels['instance_id'] = @vm_id
-        labels['region'] = @zone
-        labels['aws_account'] = ec2_metadata['accountId'] if
-          ec2_metadata.key?('accountId')
-      end
-      labels
-    end
-
-    # Determine the common labels that should be added to all log entries
-    # processed by this logging agent.
-    def determine_agent_level_common_labels
-      labels = {}
-      # User can specify labels via config. We want to capture those as well.
-      # TODO: Send instance tags as labels as well?
-      labels.merge!(@labels) if @labels
-
-      case @resource.type
-
-      # GAE app.
-      when APPENGINE_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
-
-      # GCE and GKE container.
-      when COMPUTE_CONSTANTS[:resource_type],
-           CONTAINER_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-
-      # Cloud Dataflow and Cloud Dataproc.
-      when DATAFLOW_CONSTANTS[:resource_type],
-           DATAPROC_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
-
-      # EC2.
-      when EC2_CONSTANTS[:resource_type]
-        labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
-
-      # Cloud ML.
-      when ML_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
-      end
-      labels
-    end
-
-    # Determine the group level monitored resource and common labels shared by a
-    # collection of entries.
-    def determine_group_level_monitored_resource_and_labels(tag)
-      # Determine group level monitored resource type. For certain types,
-      # extract useful info from the tag and store those in
-      # matched_regexp_group.
-      group_resource_type, matched_regexp_group =
-        determine_group_level_monitored_resource_type(tag)
-
-      # Determine group level monitored resource labels and common labels.
-      group_resource_type, group_resource_labels, group_common_labels = \
-        determine_group_level_labels_and_adjust_type(
-          group_resource_type, matched_regexp_group)
-
-      group_resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
-        type: group_resource_type,
-        labels: group_resource_labels.to_h
-      )
-
-      # Freeze the per-request state. Any further changes must be made on a
-      # per-entry basis.
-      group_resource.freeze
-      group_resource.labels.freeze
-      group_common_labels.freeze
-
-      [group_resource, group_common_labels]
-    end
-
-    # Determine group level monitored resource type shared by a collection of
-    # entries.
-    # Returns the resource type and tag regexp matched groups. The matched
-    # groups only apply to some resource types. Return nil if not applicable or
-    # if there is no match.
-    def determine_group_level_monitored_resource_type(tag)
-      # Match tag against Cloud Functions format.
-      if @running_cloudfunctions
-        matched_regexp_group = @cloudfunctions_tag_regexp.match(tag)
-        return [CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-                matched_regexp_group] if matched_regexp_group
-      end
-
-      # Match tag against Docker container stderr / stdout log format and
-      # Docker container application log format.
-      matched_regexp_group =
-        # Format: "container.<container_id>.<container_name>"
-        @dockercontainer_tag_regexp.match(tag) ||
-        # Format: "application-container.<container_name>.<additional_tag>"
-        @dockercontainer_tag_with_application_regexp.match(tag)
-      return [DOCKER_CONSTANTS[:resource_type], matched_regexp_group] if
-        matched_regexp_group
-
-      # Match tag against GKE Container format.
-      if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
-         @compiled_kubernetes_tag_regexp
-        # Container logs in Kubernetes are tagged based on where they came from,
-        # so we can extract useful metadata from the tag. Do this here to avoid
-        # having to repeat it for each record.
-        matched_regexp_group = @compiled_kubernetes_tag_regexp.match(tag)
-        return [@resource.type, matched_regexp_group] if matched_regexp_group
-      end
-
-      # Otherwise, return the original type.
-      [@resource.type, nil]
-    end
-
-    # Determine group level monitored resource labels and common labels. These
-    # labels will be shared by a collection of entries. In certain cases, we
-    # might also adjust the resource type.
-    def determine_group_level_labels_and_adjust_type(group_resource_type,
-                                                     matched_regexp_group)
-      group_resource_labels = @resource.labels.dup
-      group_common_labels = @common_labels.dup
-
-      case group_resource_type
-
-      # Cloud Functions.
-      when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-        group_resource_labels['region'] = @gcf_region
-        group_resource_labels['function_name'] =
-          decode_cloudfunctions_function_name(
-            matched_regexp_group['encoded_function_name'])
-        instance_id = group_resource_labels.delete('instance_id')
-        group_common_labels["#{CONTAINER_CONSTANTS[:service]}/cluster_name"] =
-          group_resource_labels.delete('cluster_name')
-        group_common_labels["#{CONTAINER_CONSTANTS[:service]}/instance_id"] =
-          instance_id
-        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
-          instance_id
-        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] =
-          group_resource_labels.delete('zone')
-
-      # GKE container.
-      when CONTAINER_CONSTANTS[:resource_type]
-        if matched_regexp_group
-          group_resource_labels['container_name'] =
-            matched_regexp_group['container_name']
-          group_resource_labels['namespace_id'] =
-            matched_regexp_group['namespace_name']
-          group_resource_labels['pod_id'] =
-            matched_regexp_group['pod_name']
-          %w(namespace_name pod_name).each do |field|
-            group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
-              matched_regexp_group[field]
-          end
-        end
-
-      # Docker container.
-      when DOCKER_CONSTANTS[:resource_type]
-        # For Docker container stderr / stdout logs generated by Docker Fluentd
-        # Logging Driver, tags are in the format of "container.<container_id>.
-        # <container_name>", thus they include 'container_id' info.
-        # For logs generated by applications running in Docker containers,
-        # tags are in the format of "application-container.<container_name>.
-        # <additional_tag>", thus 'container_id' info is unknown yet.
-        # 'container_name' info on the other hand is always available.
-        container_id = matched_regexp_group['container_id'] if
-          matched_regexp_group.names.include? 'container_id'
-        container_name = matched_regexp_group['container_name']
-
-        if @enable_metadata_agent
-          # Call Metadata Agent with "container.<container_id>" or
-          # "application-container.<container_name>" as the locally-unique key
-          # to retrieve monitored resource. This should be different from the
-          # original @resource value that got initiated when the agent starts up
-          # because that one is always at the VM level.
-          if container_id
-            locally_unique_id = "container.#{container_id}"
-          else
-            locally_unique_id = "containerName.#{container_name}"
-          end
-          retrieved_resource = call_metadata_agent_for_monitored_resource(
-            locally_unique_id)
-        end
-
-        if !retrieved_resource.nil?
-          # If we successfully get a monitored resource from Metadata Agent,
-          # use this one instead of the original instance monitored resource.
-          group_resource_labels = retrieved_resource.labels.dup
-          @log.debug 'Retrieved monitored resource from Metadata Agent: ' \
-                     "#{retrieved_resource.inspect}."
-        else
-          # If Metadata Agent is not enabled, or we failed to get a monitored
-          # resource, we need to have some backup plan.
-          @log.debug 'Metadata Agent not enabled or failed to retrieve ' \
-                     'docker container monitored resource from Metadata ' \
-                     'Agent.'
-
-          # 1. Check if 'container_id' is set already. It should be available
-          #    for stdout / stderr). If so, use that.
-          # 2. If not, call Docker Remote API to retrieve the container ID from
-          #    container name, but only if @call_docker_api_locally is true.
-          container_id ||= retrieve_container_id_by_name_locally(
-            container_name) if @call_docker_api_locally
-          unless container_id
-            @log.debug 'No docker container id retrieved. Falling back to
-                        instance monitored resource.'
-            # If a container id is not available, fall back to the instance
-            # monitored resource.
-            return [COMPUTE_CONSTANTS[:resource_type], group_resource_labels,
-                    group_common_labels]
-          end
-          group_resource_labels['container_id'] = container_id
-          # 'zone' for GCP and 'region' for EC2 must have been set at this
-          # point. Rename them to 'location'.
-          group_resource_labels['location'] = @zone
-          if @platform == Platform::EC2
-            group_resource_labels.delete('region')
-          else
-            group_resource_labels.delete('zone')
-          end
-          # vm id info should be reported as a metadata label instead.
-          group_resource_labels.delete('instance_id')
-
-        end
-        # Set metadata labels.
-        group_common_labels["#{DOCKER_CONSTANTS[:service]}/container_name"] =
-          matched_regexp_group['container_name']
-        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
-          @vm_id
-      end
-
-      [group_resource_type, group_resource_labels, group_common_labels]
-    end
-
-    # Extract entry resource and common labels that should be applied to
-    # individual entries from the group resource.
-    def determine_entry_level_labels(group_resource, record)
-      resource_type = group_resource.type
-      resource_labels = {}
-      common_labels = {}
-
-      # The format of the locally unique key varies by monitored resource.
-      #
-      # Docker container:
-      #   "container.<container_id>"
-      #   "containerName.<container_name>"
-      # GKE container:
-      #   "gke_containerName.<namespace_id>.<pod_name>.<container_name>"
-      if @enable_metadata_agent && record.key?(LOCALLY_UNIQUE_ID_LABEL_NAME)
-        # Call Metadata Agent with "gke_containerName.<namespace_id>.
-        # <pod_name>.<container_name>" as the locally-unique key to retrieve
-        # monitored resource.
-        locally_unique_id = record[LOCALLY_UNIQUE_ID_LABEL_NAME]
-        @log.debug 'Calling metadata agent with locally unique id: ' \
-                  "#{locally_unique_id}."
-        retrieved_resource = call_metadata_agent_for_monitored_resource(
-          locally_unique_id)
-        @log.debug 'Retrieved monitored resource from metadata agent: ' \
-                  "#{retrieved_resource.inspect}."
-        unless retrieved_resource.nil?
-          resource_type = retrieved_resource.type
-          # Temporarily renaming 'gke_container' to 'container'.
-          resource_type = 'container' if resource_type == 'gke_container'
-          # If we successfully get a monitored resource from Metadata Agent,
-          # use this one instead of the original VM-level monitored resource.
-          resource_labels = retrieved_resource.labels.dup
-          record.delete(LOCALLY_UNIQUE_ID_LABEL_NAME)
-          @log.debug 'Retrieved gke_container monitored resource from' \
-                     'Stackdriver Metadata agent: ' \
-                     "#{retrieved_resource.inspect}."
-        end
-      end
-
-      # Cloud Functions.
-      if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
-         record.key?('log')
-        @cloudfunctions_log_match =
-          @cloudfunctions_log_regexp.match(record['log'])
-        common_labels['execution_id'] =
-          @cloudfunctions_log_match['execution_id'] if \
-            @cloudfunctions_log_match &&
-            @cloudfunctions_log_match['execution_id']
-      end
-
-      # GKE containers.
-      if resource_type == CONTAINER_CONSTANTS[:resource_type]
-        # Move the stdout/stderr annotation from the record into a label.
-        common_labels.merge!(
-          fields_to_labels(
-            record, 'stream' => "#{CONTAINER_CONSTANTS[:service]}/stream"))
-
-        # If the record has been annotated by the kubernetes_metadata_filter
-        # plugin, then use that metadata. Otherwise, rely on commonLabels
-        # populated at the grouped_entries level from the group's tag.
-        if record.key?('kubernetes')
-          extracted_resource_labels, extracted_common_labels = \
-            extract_container_metadata(record)
-          resource_labels.merge!(extracted_resource_labels)
-          common_labels.merge!(extracted_common_labels)
-        end
-      end
-
-      # Docker containers.
-      if resource_type == DOCKER_CONSTANTS[:resource_type]
-        # For logs coming from Docker Fluentd Logging Driver, the log record
-        # has 4 fields: 'container_id', 'container_name', 'source' and 'log'.
-        # Extract 'container_id', 'container_name' and 'source' from json
-        # record, set corresponding labels, and remove these fields from record.
-        {
-          'container_name' => 'container_name',
-          'source' => 'stream'
-        }.each do |field_name, label_name|
-          common_labels.merge!(
-            fields_to_labels(
-              record,
-              field_name => "#{DOCKER_CONSTANTS[:service]}/#{label_name}"
-            )
-          )
-        end
-        resource_labels.merge!(
-          fields_to_labels(record, 'container_id' => 'container_id'))
-      end
-
-      # If the name of a field in the record is present in the @label_map
-      # configured by users, report its value as a label and do not send that
-      # field as part of the payload.
-      common_labels.merge!(fields_to_labels(record, @label_map))
-
-      resource_labels.merge!(
-        extract_resource_labels(resource_type, common_labels))
-
-      [resource_type, resource_labels, common_labels]
-    end
-
-    # Call Metadata Agent to get monitored resource information and parse
-    # response to Google::Api::MonitoredResource.
-    def call_metadata_agent_for_monitored_resource(unique_key)
-      response = call_metadata_agent("monitoredResource/#{unique_key}")
-      return nil if response.nil?
-      begin
-        resource = Google::Api::MonitoredResource.decode_json(response.to_json)
-      rescue Google::Protobuf::ParseError, ArgumentError => e
-        @log.error 'Error paring monitored resource from Metadata Agent. ' \
-                   "response: #{response.inspect}", error: e
-        return nil
-      end
-
-      # TODO(qingling128): Use Google::Api::MonitoredResource directly after we
-      # upgrade gRPC version to include the fix for the protobuf map
-      # corruption issue.
-      Google::Apis::LoggingV2beta1::MonitoredResource.new(
-        type: resource.type,
-        labels: resource.labels.to_h
-      )
-    end
-
-    # Call Metadata Agent and parse response to json. Return nil in case of any
-    # error / failure.
-    def call_metadata_agent(path)
-      url = "#{@metadata_agent_url}/#{path}"
-      @log.debug("Calling Metadata Agent: #{url}")
-      open(url) do |f|
-        response = f.read
-        parsed_hash = parse_json_or_nil(response)
-        if parsed_hash.nil?
-          @log.error 'Response from Metadata Agent is not in valid json ' \
-                     "format: '#{response.inspect}'."
-          return nil
-        end
-        @log.debug "Response from Metadata Agent: #{parsed_hash}"
-        return parsed_hash
-      end
-    rescue StandardError => e
-      @log.error 'Error calling Metadata Agent.', error: e
-      return nil
     end
 
     # TODO: This functionality should eventually be available in another
@@ -1276,28 +902,11 @@ module Fluent
       end
     end
 
-    # Calling Docker Remote API to get container id by name.
-    def retrieve_container_id_by_name_locally(container_name)
-      response = Excon.get(
-        "unix:///containers/#{container_name}/json",
-        socket: @docker_remote_api_socket_path)
-      @log.debug "Response from Docker API with name '#{container_name}': " \
-                 "#{response.inspect}."
-      return parse_container_id_from_docker_api_response(response)
-    rescue StandardError => e
-      @log.error 'Error calling Docker API to get container id.', error: e
-      return nil
-    end
-
-    # Parse the container id from Docker Remote API response.
-    # TODO(qingling128) Add a config for Docker API version to support parsing
-    # different versions of Docker Remote API when the format varies.
-    def parse_container_id_from_docker_api_response(response)
-      JSON.parse(response.data[:body])['Id']
-    rescue StandardError => e
-      @log.error 'Error parsing Docker API response to get container id.',
-                 error: e
-      return nil
+    def detect_cloudfunctions(attributes)
+      return unless attributes.include?('gcf_region')
+      # Cloud Functions detected
+      @running_cloudfunctions = true
+      @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
     end
 
     def cluster_name_from_kube_env(kube_env)
@@ -1370,14 +979,9 @@ module Fluent
         end
       elsif record.key?('severity')
         return parse_severity(record.delete('severity'))
-      elsif [CONTAINER_CONSTANTS[:resource_type],
-             DOCKER_CONSTANTS[:resource_type]].include?(resource_type)
-        stream = entry_common_labels[
-          "#{CONTAINER_CONSTANTS[:service]}/stream"] if
-           resource_type == CONTAINER_CONSTANTS[:resource_type]
-        stream = entry_common_labels[
-          "#{DOCKER_CONSTANTS[:service]}/stream"] if
-           resource_type == DOCKER_CONSTANTS[:resource_type]
+      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
+            entry_common_labels.key?("#{CONTAINER_CONSTANTS[:service]}/stream")
+        stream = entry_common_labels["#{CONTAINER_CONSTANTS[:service]}/stream"]
         if stream == 'stdout'
           return 'INFO'
         elsif stream == 'stderr'
@@ -1585,24 +1189,6 @@ module Fluent
       [resource_labels, common_labels]
     end
 
-    def format(tag, time, record)
-      [tag, time, record].to_msgpack
-    end
-
-    # Given a tag, returns the corresponding valid tag if possible, or nil if
-    # the tag should be rejected. If 'require_valid_tags' is false, non-string
-    # tags are converted to strings, and invalid characters are sanitized;
-    # otherwise such tags are rejected.
-    def sanitize_tag(tag)
-      if @require_valid_tags &&
-         (!tag.is_a?(String) || tag == '' || convert_to_utf8(tag) != tag)
-        return nil
-      end
-      tag = convert_to_utf8(tag.to_s)
-      tag = '_' if tag == ''
-      tag
-    end
-
     # For every original_label => new_label pair in the label_map, delete the
     # original_label from the record if it exists, and extract the value to form
     # a map with the new_label as the key.
@@ -1630,8 +1216,7 @@ module Fluent
         entry.text_payload = record['log']
       elsif is_json
         entry.json_payload = record
-      elsif [CONTAINER_CONSTANTS[:resource_type],
-             DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
+      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
             record.key?('log')
         entry.text_payload = record['log']
       elsif record.size == 1 && record.key?('message')
@@ -1701,8 +1286,7 @@ module Fluent
         entry.text_payload = convert_to_utf8(record['log'])
       elsif is_json
         entry.json_payload = struct_from_ruby(record)
-      elsif [CONTAINER_CONSTANTS[:resource_type],
-             DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
+      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
             record.key?('log')
         entry.text_payload = convert_to_utf8(record['log'])
       elsif record.size == 1 && record.key?('message')
@@ -1715,7 +1299,7 @@ module Fluent
     def log_name(tag, resource)
       if resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
         tag = 'cloud-functions'
-      elsif resource.type == APPENGINE_CONSTANTS[:resource_type]
+      elsif @running_on_managed_vm
         # Add a prefix to Managed VM logs to prevent namespace collisions.
         tag = "#{APPENGINE_CONSTANTS[:service]}/#{tag}"
       elsif resource.type == CONTAINER_CONSTANTS[:resource_type]
@@ -1745,9 +1329,6 @@ module Fluent
       elsif resource_type == ML_CONSTANTS[:resource_type]
         label_prefix = ML_CONSTANTS[:service]
         labels_to_extract = %w(job_id task_name)
-      elsif resource_type == DOCKER_CONSTANTS[:resource_type]
-        label_prefix = DOCKER_CONSTANTS[:service]
-        labels_to_extract = %w(container_id)
       else
         return extracted_labels
       end
@@ -1815,6 +1396,33 @@ module Fluent
         end
       end
     end
+
+    # Increment the metric for the number of successful requests.
+    def increment_successful_requests_count
+      return unless @successful_requests_count
+      @successful_requests_count.increment(grpc: @use_grpc)
+    end
+
+    # Increment the metric for the number of failed requests, labeled by
+    # the provided status code.
+    def increment_failed_requests_count(code)
+      return unless @failed_requests_count
+      @failed_requests_count.increment(grpc: @use_grpc, code: code)
+    end
+
+    # Increment the metric for the number of log entries, successfully
+    # ingested by the Stackdriver Logging API.
+    def increment_ingested_entries_count(count)
+      return unless @ingested_entries_count
+      @ingested_entries_count.increment({}, count)
+    end
+
+    # Increment the metric for the number of log entries that were dropped
+    # and not ingested by the Stackdriver Logging API.
+    def increment_dropped_entries_count(count)
+      return unless @dropped_entries_count
+      @dropped_entries_count.increment({}, count)
+    end
   end
 end