fluent-plugin-google-cloud 0.6.4 → 0.6.5.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab431f0283f1c6bc88669159be387f21bb1e908f
-  data.tar.gz: 77d320fd8b6e4c5370542142fd296581fcd9186e
+  metadata.gz: 7b1a26eab783c87890f78500848a29048b4ab615
+  data.tar.gz: 81711bc2e00bcf222b7249bde80d78c6bab34957
 SHA512:
-  metadata.gz: 15fc471919366c52a612cb2b1941d0e9ffdf5e1d5c94ad6f3395c1860ca2f03edd888959a0ed267375bf80133458251bc1555c7416ef81f8be4f4aa83fd5c405
-  data.tar.gz: 4aed5e4e2686eb4776d32f3b44f61fc86849f4f02a64bd12399d0e0ac4a6cc1a8752489513fa5e482fdb7cac432d517ed5b7b26e71365e344e2dcf2ca8e96832
+  metadata.gz: 7d5c1ca8f704998cd36c5a040dc4fa10989d2cbb7717dea8fa67032c3c42b428a5f3ef37f04581caea41bf991232688d3245ebe285e7fb72f9f0b783712d399f
+  data.tar.gz: 8a95c809c56fcc6351f08bd81d3124cd31b6529c54afb3c72d109519c4aecd5764d26514b4dc8a99417510c3883451ae54189e65a1e10d81465143888bf68b11

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.4)
+    fluent-plugin-google-cloud (0.6.5.pre.1)
+      excon (~> 0.57.1)
       fluentd (~> 0.10)
       google-api-client (~> 0.9.0)
-      google-cloud-logging (~> 0.23.2)
+      google-cloud-logging (= 0.24.1)
       googleapis-common-protos (~> 1.3)
-      googleauth (~> 0.4)
+      googleauth (~> 0.4, < 0.5.2)
       grpc (~> 1.0, < 1.3)
       json (~> 1.8)
 
@@ -18,15 +19,17 @@ GEM
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    cool.io (1.5.0)
+    cool.io (1.5.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    faraday (0.12.1)
+    excon (0.57.1)
+    faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.14.18)
+    fluentd (0.14.20)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
+      ruby_dig (~> 0.0.2)
       serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (~> 0.1.7)
@@ -44,20 +47,17 @@ GEM
       retriable (~> 2.0)
     google-cloud-core (0.21.1)
       googleauth (~> 0.5.1)
-    google-cloud-logging (0.23.2)
+    google-cloud-logging (0.24.1)
       google-cloud-core (~> 0.21.1)
-      google-gax (~> 0.6.0)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos (~> 1.3)
-      grpc (~> 1.0)
-      orderedhash (= 0.0.6)
+      google-gax (~> 0.8.0)
       stackdriver-core (~> 0.21.0)
-    google-gax (0.6.0)
-      googleapis-common-protos (~> 1.3.1)
+    google-gax (0.8.5)
+      google-protobuf (~> 3.2)
+      googleapis-common-protos (~> 1.3.5)
       googleauth (~> 0.5.1)
       grpc (~> 1.0)
       rly (~> 0.2.3)
-    google-protobuf (3.3.0)
+    google-protobuf (3.3.0-x86_64-linux)
     googleapis-common-protos (1.3.5)
       google-protobuf (~> 3.2)
       grpc (~> 1.0)
@@ -69,10 +69,10 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.2.5)
+    grpc (1.2.5-x86_64-linux)
       google-protobuf (~> 3.1)
       googleauth (~> 0.5.1)
-    hashdiff (0.3.4)
+    hashdiff (0.3.5)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
     hurley (0.2)
@@ -92,7 +92,6 @@ GEM
     msgpack (1.1.0)
     multi_json (1.12.1)
     multipart-post (2.0.0)
-    orderedhash (0.0.6)
     os (0.9.6)
     parser (2.4.0.0)
       ast (~> 2.2)
@@ -117,6 +116,7 @@ GEM
       ruby-progressbar (~> 1.7)
       tins (<= 1.6.0)
     ruby-progressbar (1.8.1)
+    ruby_dig (0.0.2)
     safe_yaml (1.0.4)
     serverengine (2.0.5)
       sigdump (~> 0.2.2)
@@ -137,7 +137,7 @@ GEM
     tzinfo-data (1.2017.2)
       tzinfo (>= 1.0.0)
     uber (0.0.15)
-    webmock (1.24.6)
+    webmock (2.3.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -153,7 +153,7 @@ DEPENDENCIES
   rake (~> 10.3)
   rubocop (~> 0.35.0)
   test-unit (~> 3.0)
-  webmock (~> 1.17)
+  webmock (~> 2.3.1)
 
 BUNDLED WITH
-   1.15.0
+   1.15.3

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.4'
+  gem.version       = '0.6.5.pre.1'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -19,18 +19,19 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
+  gem.add_runtime_dependency 'excon', '~> 0.57.1'
   gem.add_runtime_dependency 'fluentd', '~> 0.10'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
   gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'
-  gem.add_runtime_dependency 'google-cloud-logging', '~> 0.23.2'
-  gem.add_runtime_dependency 'googleauth', '~> 0.4'
+  gem.add_runtime_dependency 'google-cloud-logging', '0.24.1'
+  gem.add_runtime_dependency 'googleauth', '~> 0.4', '< 0.5.2'
   gem.add_runtime_dependency 'grpc', '~> 1.0', '< 1.3'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
   gem.add_development_dependency 'mocha', '~> 1.1'
   gem.add_development_dependency 'rake', '~> 10.3'
   gem.add_development_dependency 'rubocop', '~> 0.35.0'
-  gem.add_development_dependency 'webmock', '~> 1.17'
+  gem.add_development_dependency 'webmock', '~> 2.3.1'
   gem.add_development_dependency 'test-unit', '~> 3.0'
   gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
 end

data/lib/fluent/plugin/out_google_cloud.rb

@@ -11,9 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+require 'excon'
 require 'grpc'
 require 'json'
 require 'open-uri'
+require 'rubygems'
 require 'socket'
 require 'time'
 require 'yaml'
@@ -38,8 +40,10 @@ end
 module Fluent
   # fluentd output plugin for the Stackdriver Logging API
   class GoogleCloudOutput < BufferedOutput
-    # Constants for service names and resource types.
+    # Constants.
     module Constants
+      # Service names and resource types.
+
       APPENGINE_CONSTANTS = {
         service: 'appengine.googleapis.com',
         resource_type: 'gae_app'
@@ -56,6 +60,10 @@ module Fluent
         service: 'container.googleapis.com',
         resource_type: 'container'
       }
+      DOCKER_CONSTANTS = {
+        service: 'dockercontainer.googleapis.com',
+        resource_type: 'docker_container'
+      }
       DATAFLOW_CONSTANTS = {
         service: 'dataflow.googleapis.com',
         resource_type: 'dataflow_step'
@@ -72,6 +80,23 @@ module Fluent
         service: 'ml.googleapis.com',
         resource_type: 'ml_job'
       }
+
+      # Default value for trace_key config param to set "trace" LogEntry field.
+      DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'
+
+      # Metadata Agent support.
+
+      # Use empty string as request path when locally-unique key of monitored
+      # resource can be implicitly inferred by Metadata Agent.
+      IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY = ''
+
+      # The label name of locally unique id in the json payload. When a record
+      # has this field in the payload, we will use the value to retrieve
+      # monitored resource from Stackdriver Metadata agent.
+      LOCALLY_UNIQUE_ID_LABEL_NAME = 'logging.googleapis.com/locally_unique_id'
+
+      # Docker container support.
+      DEFAULT_DOCKER_API_SOCKET_PATH = '/var/run/docker.sock'
     end
 
     include self::Constants
@@ -79,7 +104,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.3'
+    PLUGIN_VERSION = '0.6.5.pre.1'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -109,6 +134,11 @@ module Fluent
     config_param :vm_id, :string, :default => nil
     config_param :vm_name, :string, :default => nil
 
+    # Set values from JSON payload with this key to the "trace" LogEntry field.
+    config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
+    # Whether to also keep the trace key/value in the payload.
+    config_param :keep_trace_key, :bool, :default => false
+
     # Whether to try to detect if the VM is owned by a "subservice" such as App
     # Engine of Kubernetes, rather than just associating the logs with the
     # compute service of the platform. This currently only has any effect when
@@ -126,7 +156,7 @@ module Fluent
     config_param :require_valid_tags, :bool, :default => false
 
     # The regular expression to use on Kubernetes logs to extract some basic
-    # information about the log source. The regex must contain capture groups
+    # information about the log source. The regexp must contain capture groups
     # for pod_name, namespace_name, and container_name.
     config_param :kubernetes_tag_regexp, :string, :default =>
       '\.(?<pod_name>[^_]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$'
@@ -201,6 +231,18 @@ module Fluent
     config_param :monitoring_type, :string,
                  :default => Monitoring::PrometheusMonitoringRegistry.name
 
+    # Whether to call metadata agent to retrieve monitored resource.
+    config_param :enable_metadata_agent, :bool, :default => false
+    config_param :metadata_agent_url, :string,
+                 :default => 'http://local-metadata-agent.stackdriver.com:8000'
+
+    # Whether to call Docker Remote API locally when Metadata Agent is not
+    # enabled or if the request fails.
+    config_param :call_docker_api_locally, :bool, :default => true
+    # Docker Remote API unix socket path.
+    config_param :docker_remote_api_socket_path, :string,
+                 :default => DEFAULT_DOCKER_API_SOCKET_PATH
+
     # rubocop:enable Style/HashSyntax
 
     # TODO: Add a log_name config option rather than just using the tag?
@@ -210,9 +252,6 @@ module Fluent
     attr_reader :project_id
     attr_reader :zone
     attr_reader :vm_id
-    attr_reader :running_on_managed_vm
-    attr_reader :gae_backend_name
-    attr_reader :gae_backend_version
     attr_reader :resource
     attr_reader :common_labels
 
@@ -222,6 +261,35 @@ module Fluent
       @log = $log # rubocop:disable Style/GlobalVars
     end
 
+    # Set up regex patterns used to parse tags and logs.
+    def setup_regex_patterns
+      @compiled_kubernetes_tag_regexp = nil
+      if @kubernetes_tag_regexp
+        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
+      end
+
+      @cloudfunctions_tag_regexp =
+        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
+      @cloudfunctions_log_regexp = /^
+        (?:\[(?<severity>.)\])?
+        \[(?<timestamp>.{24})\]
+        (?:\[(?<execution_id>[^\]]+)\])?
+        [ ](?<text>.*)$/x
+
+      # Docker container tag format:
+      # "container.<container_id>.<container_name>".
+      @dockercontainer_tag_regexp =
+        /^container\.(?<container_id>[a-zA-Z0-9]+)\.
+        (?<container_name>[a-zA-Z0-9_.-]+)$/x
+      # Docker container with application tag format:
+      # "application-container.<container_name>.<additional_tag>".
+      @dockercontainer_tag_with_application_regexp =
+        /^application-container\.(?<container_name>[a-zA-Z0-9_.-]+)\.
+        (?<additional_tag>.+)$/x
+
+      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
+    end
+
     def configure(conf)
       super
 
@@ -259,162 +327,54 @@ module Fluent
                extra.join(' ')
       end
 
-      # TODO: Send instance tags as labels as well?
-      @common_labels = {}
-      @common_labels.merge!(@labels) if @labels
-
-      # TODO: Construct Google::Api::MonitoredResource when @use_grpc is
-      # true after the protobuf map corruption issue is fixed.
-      @resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
-        labels: {})
-
-      @compiled_kubernetes_tag_regexp = nil
-      if @kubernetes_tag_regexp
-        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
-      end
+      setup_regex_patterns
 
-      @cloudfunctions_tag_regexp =
-        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
-      @cloudfunctions_log_regexp = /^
-        (?:\[(?<severity>.)\])?
-        \[(?<timestamp>.{24})\]
-        (?:\[(?<execution_id>[^\]]+)\])?
-        [ ](?<text>.*)$/x
-
-      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
-
-      # set attributes from metadata (unless overriden by static config)
-      @vm_name = Socket.gethostname if @vm_name.nil?
       @platform = detect_platform
-      case @platform
-      when Platform::GCE
-        if @project_id.nil?
-          @project_id = fetch_gce_metadata('project/project-id')
-        end
-        if @zone.nil?
-          # this returns "projects/<number>/zones/<zone>"; we only want
-          # the part after the final slash.
-          fully_qualified_zone = fetch_gce_metadata('instance/zone')
-          @zone = fully_qualified_zone.rpartition('/')[2]
-        end
-        @vm_id = fetch_gce_metadata('instance/id') if @vm_id.nil?
-      when Platform::EC2
-        metadata = fetch_ec2_metadata
-        if @zone.nil? && metadata.key?('availabilityZone')
-          @zone = 'aws:' + metadata['availabilityZone']
-        end
-        if @vm_id.nil? && metadata.key?('instanceId')
-          @vm_id = metadata['instanceId']
-        end
-        if metadata.key?('accountId')
-          @resource.labels['aws_account'] = metadata['accountId']
-        end
-      when Platform::OTHER
-        # do nothing
-      else
-        fail Fluent::ConfigError, 'Unknown platform ' + @platform
-      end
 
-      # If we still don't have a project ID, try to obtain it from the
-      # credentials.
-      if @project_id.nil?
-        @project_id = CredentialsInfo.project_id
-        @log.info 'Set Project ID from credentials: ', @project_id unless
-          @project_id.nil?
+      # Set agent-level monitored resource. This monitored resource is initiated
+      # as the logging agent starts up. It will be inherited by all log entries
+      # processed by this agent. First try to retrieve it via Metadata Agent.
+      if @enable_metadata_agent
+        # The locally-unique key for this should be the instance id. Since this
+        # can be implicitly inferred by Metadata Agent, we do not need to
+        # explicitly send the key.
+        @resource = call_metadata_agent_for_monitored_resource(
+          IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
       end
 
-      # all metadata parameters must now be set
-      unless @project_id && @zone && @vm_id
-        missing = []
-        missing << 'project_id' unless @project_id
-        missing << 'zone' unless @zone
-        missing << 'vm_id' unless @vm_id
-        fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
-          missing.join(' ')
-      end
-
-      # Default this to false; it is only overwritten if we detect Managed VM.
-      @running_on_managed_vm = false
-
-      # Default this to false; it is only overwritten if we detect Cloud
-      # Functions.
+      # Set required variables: @project_id, @vm_id, @vm_name and @zone.
+      # If any info above is included in the response from Metadata Agent, make
+      # use of that. Otherwise make some additional requests to metadata server.
+      #
+      # Note: Once we support metadata injection on the Logging API side, we
+      # might no longer need to require all these metadata in logging agent. But
+      # for now, they are still required.
+      set_required_metadata_variables
+
+      # Fail over to retrieve monitored resource via the legacy path if we fail
+      # to get it from Metadata Agent.
+      @resource ||= determine_agent_level_monitored_resource_via_legacy
+
+      # Set variables specific to CLoud Functions. This has to be called after
+      # we have determined the resource type. The purpose is to avoid repeated
+      # calls to metadata server.
       @running_cloudfunctions = false
-
-      # Set up the MonitoredResource, labels, etc. based on the config.
-      case @platform
-      when Platform::GCE
-        @resource.type = COMPUTE_CONSTANTS[:resource_type]
-        # TODO: introduce a new MonitoredResource-centric configuration and
-        # deprecate subservice-name; for now, translate known uses.
-        if @subservice_name
-          # TODO: what should we do if we encounter an unknown value?
-          if @subservice_name == DATAFLOW_CONSTANTS[:service]
-            @resource.type = DATAFLOW_CONSTANTS[:resource_type]
-          elsif @subservice_name == ML_CONSTANTS[:service]
-            @resource.type = ML_CONSTANTS[:resource_type]
-          end
-        elsif @detect_subservice
-          # Check for specialized GCE environments.
-          # TODO: Add config options for these to allow for running outside GCE?
-          attributes = fetch_gce_metadata('instance/attributes/').split
-          # Do nothing, just don't populate other service's labels.
-          if attributes.include?('gae_backend_name') &&
-             attributes.include?('gae_backend_version')
-            # Managed VM
-            @running_on_managed_vm = true
-            @gae_backend_name =
-                fetch_gce_metadata('instance/attributes/gae_backend_name')
-            @gae_backend_version =
-                fetch_gce_metadata('instance/attributes/gae_backend_version')
-            @resource.type = APPENGINE_CONSTANTS[:resource_type]
-            @resource.labels['module_id'] = @gae_backend_name
-            @resource.labels['version_id'] = @gae_backend_version
-          elsif attributes.include?('kube-env')
-            # Kubernetes/Container Engine
-            @resource.type = CONTAINER_CONSTANTS[:resource_type]
-            @raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
-            @kube_env = YAML.load(@raw_kube_env)
-            @resource.labels['cluster_name'] =
-              cluster_name_from_kube_env(@kube_env)
-            detect_cloudfunctions(attributes)
-          elsif attributes.include?('dataproc-cluster-uuid') &&
-                attributes.include?('dataproc-cluster-name')
-            # Dataproc
-            @resource.type = DATAPROC_CONSTANTS[:resource_type]
-            @resource.labels['cluster_uuid'] =
-              fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid')
-            @resource.labels['cluster_name'] =
-              fetch_gce_metadata('instance/attributes/dataproc-cluster-name')
-            @resource.labels['region'] =
-              fetch_gce_metadata('instance/attributes/dataproc-region')
-          end
-        end
-        # Some services have the GCE instance_id and zone as MonitoredResource
-        # labels; for other services we send them as entry labels.
-        if @resource.type == COMPUTE_CONSTANTS[:resource_type] ||
-           @resource.type == CONTAINER_CONSTANTS[:resource_type]
-          @resource.labels['instance_id'] = @vm_id
-          @resource.labels['zone'] = @zone
-        else
-          common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-          common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
-        end
-        common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-      when Platform::EC2
-        @resource.type = EC2_CONSTANTS[:resource_type]
-        @resource.labels['instance_id'] = @vm_id
-        @resource.labels['region'] = @zone
-        # the aws_account label is populated above.
-        common_labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
-      when Platform::OTHER
-        # Use GCE as the default environment.
-        @resource.type = COMPUTE_CONSTANTS[:resource_type]
-        @resource.labels['instance_id'] = @vm_id
-        @resource.labels['zone'] = @zone
-        common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+      # We only support Cloud Functions logs for GKE right now.
+      if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
+         fetch_gce_metadata('instance/attributes/').split.include?('gcf_region')
+        # We are not setting resource type as Cloud Functions here because
+        # whether a log entry is truly coming from a Cloud Functions function
+        # depends on the log tag. Only when @running_cloudfunctions is true will
+        # we try to match log tags against Cloud Functions tag regexp when
+        # processing log entries.
+        @running_cloudfunctions = true
+        # Fetch this info and store it to avoid recurring metadata server calls.
+        @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
       end
-      @resource.labels.merge!(
-        extract_resource_labels(@resource.type, common_labels))
+
+      # Determine the common labels that should be added to all log entries
+      # processed by this logging agent.
+      @common_labels = determine_agent_level_common_labels
 
       # The resource and labels are now set up; ensure they can't be modified
       # without first duping them.
@@ -424,7 +384,7 @@ module Fluent
 
       # Log an informational message containing the Logs viewer URL
       @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
-                "viewer?project=#{@project_id}&resource=#{@resource_type}/",
+                "viewer?project=#{@project_id}&resource=#{@resource.type}/",
                 "instance_id/#{@vm_id}"
     end
 
@@ -439,130 +399,6 @@ module Fluent
       super
     end
 
-    def format(tag, time, record)
-      [tag, time, record].to_msgpack
-    end
-
-    # Given a tag, returns the corresponding valid tag if possible, or nil if
-    # the tag should be rejected. If 'require_valid_tags' is false, non-string
-    # tags are converted to strings, and invalid characters are sanitized;
-    # otherwise such tags are rejected.
-    def sanitize_tag(tag)
-      if @require_valid_tags &&
-         (!tag.is_a?(String) || tag == '' || convert_to_utf8(tag) != tag)
-        return nil
-      end
-      tag = convert_to_utf8(tag.to_s)
-      tag = '_' if tag == ''
-      tag
-    end
-
-    # Compute the monitored resource and common labels shared by a collection of
-    # entries.
-    def compute_group_resource_and_labels(tag)
-      # Note that we assume that labels added to group_common_labels below are
-      # not 'service' labels (i.e. we do not call extract_resource_labels
-      # again).
-      group_resource = @resource.dup
-      group_common_labels = @common_labels.dup
-
-      if @running_cloudfunctions
-        # If the current group of entries is coming from a Cloud Functions
-        # function, the function name can be extracted from the tag.
-        match_data = @cloudfunctions_tag_regexp.match(tag)
-        if match_data
-          # Resource type is set to Cloud Functions only for logs actually
-          # coming from a function, otherwise we leave it as Container.
-          group_resource.type = CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-          group_resource.labels['region'] = @gcf_region
-          group_resource.labels['function_name'] =
-            decode_cloudfunctions_function_name(
-              match_data['encoded_function_name'])
-          # Move GKE container labels from the MonitoredResource to the
-          # LogEntry.
-          instance_id = group_resource.labels.delete('instance_id')
-          group_common_labels["#{CONTAINER_CONSTANTS[:service]}/cluster_name"] =
-            group_resource.labels.delete('cluster_name')
-          group_common_labels["#{CONTAINER_CONSTANTS[:service]}/instance_id"] =
-            instance_id
-          group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
-            instance_id
-          group_common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] =
-            group_resource.labels.delete('zone')
-        end
-      end
-      if group_resource.type == CONTAINER_CONSTANTS[:resource_type] &&
-         @compiled_kubernetes_tag_regexp
-        # Container logs in Kubernetes are tagged based on where they came
-        # from, so we can extract useful metadata from the tag.
-        # Do this here to avoid having to repeat it for each record.
-        match_data = @compiled_kubernetes_tag_regexp.match(tag)
-        if match_data
-          group_resource.labels['container_name'] = match_data['container_name']
-          group_resource.labels['namespace_id'] = match_data['namespace_name']
-          group_resource.labels['pod_id'] = match_data['pod_name']
-          %w(namespace_name pod_name).each do |field|
-            group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
-              match_data[field]
-          end
-        end
-      end
-
-      # Freeze the per-request state. Any further changes must be made on a
-      # per-entry basis.
-      group_resource.freeze
-      group_resource.labels.freeze
-      group_common_labels.freeze
-
-      [group_resource, group_common_labels]
-    end
-
-    # Extract entry resource and common labels that should be applied to
-    # individual entries from the group resource.
-    def extract_entry_labels(group_resource, record)
-      resource_labels = {}
-      common_labels = {}
-
-      if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
-         record.key?('log')
-        @cloudfunctions_log_match =
-          @cloudfunctions_log_regexp.match(record['log'])
-      end
-
-      if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
-        # Move the stdout/stderr annotation from the record into a label
-        common_labels.merge!(
-          fields_to_labels(
-            record, 'stream' => "#{CONTAINER_CONSTANTS[:service]}/stream"))
-
-        # If the record has been annotated by the kubernetes_metadata_filter
-        # plugin, then use that metadata. Otherwise, rely on commonLabels
-        # populated at the grouped_entries level from the group's tag.
-        if record.key?('kubernetes')
-          extracted_resource_labels, extracted_common_labels = \
-            extract_container_metadata(record)
-          resource_labels.merge!(extracted_resource_labels)
-          common_labels.merge!(extracted_common_labels)
-        end
-      end
-
-      # If a field is present in the label_map, send its value as a label
-      # (mapping the field name to label name as specified in the config)
-      # and do not send that field as part of the payload.
-      common_labels.merge!(fields_to_labels(record, @label_map))
-
-      if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
-         @cloudfunctions_log_match &&
-         @cloudfunctions_log_match['execution_id']
-        common_labels['execution_id'] =
-          @cloudfunctions_log_match['execution_id']
-      end
-      resource_labels.merge!(
-        extract_resource_labels(group_resource.type, common_labels))
-
-      [resource_labels, common_labels]
-    end
-
     def write(chunk)
       # Group the entries since we have to make one call per tag.
       grouped_entries = {}
@@ -579,20 +415,22 @@ module Fluent
 
       grouped_entries.each do |tag, arr|
         entries = []
-        group_resource, group_common_labels = compute_group_resource_and_labels(
-          tag)
+        group_resource, group_common_labels =
+          determine_group_level_monitored_resource_and_labels(tag)
 
         arr.each do |time, record|
           next unless record.is_a?(Hash)
 
-          extracted_resource_labels, extracted_common_labels = \
-            extract_entry_labels(group_resource, record)
+          resource_type, extracted_resource_labels, extracted_common_labels = \
+            determine_entry_level_labels(group_resource, record)
           entry_resource = group_resource.dup
+          entry_resource.type = resource_type
           entry_resource.labels.merge!(extracted_resource_labels)
           entry_common_labels = \
             group_common_labels.merge(extracted_common_labels)
 
-          if entry_resource.type == CONTAINER_CONSTANTS[:resource_type]
+          if [CONTAINER_CONSTANTS[:resource_type],
+              DOCKER_CONSTANTS[:resource_type]].include?(entry_resource.type)
             # Save the timestamp if available, then clear it out to allow for
             # determining whether we should parse the log or message field.
             timestamp = record.key?('time') ? record['time'] : nil
@@ -621,6 +459,13 @@ module Fluent
           severity = compute_severity(
             entry_resource.type, record, entry_common_labels)
 
+          # Get fully-qualified trace id for LogEntry "trace" field per config.
+          fq_trace_id = if @keep_trace_key
+                          record[@trace_key]
+                        else
+                          record.delete(@trace_key)
+                        end
+
           if @use_grpc
             entry = Google::Logging::V2::LogEntry.new(
               labels: entry_common_labels,
@@ -630,6 +475,7 @@ module Fluent
               ),
               severity: grpc_severity(severity)
             )
+            entry.trace = fq_trace_id if fq_trace_id
             # If "seconds" is null or not an integer, we will omit the timestamp
             # field and defer the decision on how to handle it to the downstream
             # Logging API. If "nanos" is null or not an integer, it will be set
@@ -655,6 +501,7 @@ module Fluent
                 nanos: ts_nanos
               }
             )
+            entry.trace = fq_trace_id if fq_trace_id
             set_http_request(record, entry)
             set_payload(entry_resource.type, record, entry, is_json)
           end
@@ -841,7 +688,7 @@ module Fluent
           end
         end
       rescue StandardError => e
-        @log.debug 'Failed to access metadata service: ', error: e
+        @log.error 'Failed to access metadata service: ', error: e
       end
 
       @log.info 'Unable to determine platform'
@@ -856,15 +703,630 @@ module Fluent
            metadata_path, 'Metadata-Flavor' => 'Google', &:read)
     end
 
-    def fetch_ec2_metadata
-      fail "Called fetch_ec2_metadata with platform=#{@platform}" unless
+    # EC2 Metadata server returns everything in one call. Store it after the
+    # first fetch to avoid making multiple calls.
+    def ec2_metadata
+      fail "Called ec2_metadata with platform=#{@platform}" unless
         @platform == Platform::EC2
-      # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
-      open('http://' + METADATA_SERVICE_ADDR +
-           '/latest/dynamic/instance-identity/document') do |f|
-        contents = f.read
-        return JSON.parse(contents)
+      unless @ec2_metadata
+        # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+        open('http://' + METADATA_SERVICE_ADDR +
+             '/latest/dynamic/instance-identity/document') do |f|
+          contents = f.read
+          @ec2_metadata = JSON.parse(contents)
+        end
+      end
+
+      @ec2_metadata
+    end
+
+    # Set regexp patterns to parse tags and logs.
+    def set_regexp_patterns
+      @compiled_kubernetes_tag_regexp = nil
+      if @kubernetes_tag_regexp
+        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
       end
+
+      @cloudfunctions_tag_regexp =
+        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
+      @cloudfunctions_log_regexp = /^
+        (?:\[(?<severity>.)\])?
+        \[(?<timestamp>.{24})\]
+        (?:\[(?<execution_id>[^\]]+)\])?
+        [ ](?<text>.*)$/x
+
+      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
+    end
+
+    # Set required variables like @project_id, @vm_id, @vm_name and @zone.
+    def set_required_metadata_variables
+      set_project_id
+      set_vm_id
+      set_vm_name
+      set_location
+
+      # All metadata parameters must now be set.
+      return if @project_id && @zone && @vm_id
+      missing = []
+      missing << 'project_id' unless @project_id
+      missing << 'zone' unless @zone
+      missing << 'vm_id' unless @vm_id
+      fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
+        missing.join(' ')
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it by calling metadata server directly.
+    # 3. If still not set, try to obtain it from the credentials.
+    def set_project_id
+      @project_id ||= fetch_gce_metadata('project/project-id') if
+        @platform == Platform::GCE
+      @project_id ||= CredentialsInfo.project_id
+    rescue StandardError => e
+      @log.error 'Failed to obtain project id: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, check if the response from Metadata Agent includes this info.
+    # 3. If not, try to retrieve it by calling metadata servers directly.
+    def set_vm_id
+      @vm_id ||= @resource.labels['instance_id'] if
+        !@resource.nil? && @resource.labels.key?('instance_id')
+      @vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
+      @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
+    rescue StandardError => e
+      @log.error 'Failed to obtain vm_id: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, check if the response from Metadata Agent includes this info.
+    # 3. If not, try to retrieve it locally.
+    def set_vm_name
+      @vm_name ||= @resource.labels['instance_name'] if
+        !@resource.nil? && @resource.labels.key?('instance_name')
+      @vm_name ||= Socket.gethostname
+    rescue StandardError => e
+      @log.error 'Failed to obtain vm name: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, check if the response from Metadata Agent includes this info.
+    # 3. If not, try to retrieve it locally.
+    def set_location
+      unless @resource.nil?
+        @zone ||= @resource.labels['location'] if
+          @resource.type == DOCKER_CONSTANTS[:resource_type] &&
+          @resource.labels.key?('location')
+        @zone ||= @resource.labels['zone'] if
+          @platform == Platform::GCE && @resource.labels.key?('zone')
+        @zone ||= @resource.labels['region'] if
+          @platform == Platform::EC2 && @resource.labels.key?('region')
+      end
+      # Response format: "projects/<number>/zones/<zone>"
+      @zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
+        @platform == Platform::GCE
+      @zone ||= 'aws:' + ec2_metadata['availabilityZone'] if
+        @platform == Platform::EC2 && ec2_metadata.key?('availabilityZone')
+    rescue StandardError => e
+      @log.error 'Failed to obtain location: ', error: e
+    end
+
+    # Retrieve monitored resource via the legacy way.
+    #
+    # Note: This is just a failover plan if we fail to get metadata from
+    # Metadata Agent. Thus it should be equivalent to what Metadata Agent
+    # returns.
+    def determine_agent_level_monitored_resource_via_legacy
+      resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
+        labels: {})
+      resource.type = determine_agent_level_monitored_resource_type
+      resource.labels = determine_agent_level_monitored_resource_labels(
+        resource.type)
+      resource
+    end
+
+    # Determine agent level monitored resource type.
+    def determine_agent_level_monitored_resource_type
+      # EC2 instance.
+      return EC2_CONSTANTS[:resource_type] if
+        @platform == Platform::EC2
+
+      # Unknown platform will be defaulted to GCE instance..
+      return COMPUTE_CONSTANTS[:resource_type] if
+        @platform == Platform::OTHER
+
+      # Resource types determined by @subservice_name config.
+      # Cloud Dataflow.
+      return DATAFLOW_CONSTANTS[:resource_type] if
+        @subservice_name == DATAFLOW_CONSTANTS[:service]
+      # Cloud ML.
+      return ML_CONSTANTS[:resource_type] if
+        @subservice_name == ML_CONSTANTS[:service]
+      # Default back to GCE if invalid value is detected.
+      return COMPUTE_CONSTANTS[:resource_type] if
+        @subservice_name
+
+      # Resource types determined by @detect_subservice config.
+      if @detect_subservice
+        begin
+          attributes = fetch_gce_metadata('instance/attributes/').split
+        rescue StandardError => e
+          @log.error 'Failed to detect subservice: ', error: e
+        end
+        # GAE app.
+        return APPENGINE_CONSTANTS[:resource_type] if
+          attributes.include?('gae_backend_name') &&
+          attributes.include?('gae_backend_version')
+        # GKE container.
+        return CONTAINER_CONSTANTS[:resource_type] if
+          attributes.include?('kube-env')
+        # Cloud Dataproc.
+        return DATAPROC_CONSTANTS[:resource_type] if
+          attributes.include?('dataproc-cluster-uuid') &&
+          attributes.include?('dataproc-cluster-name')
+      end
+      # GCE instance.
+      COMPUTE_CONSTANTS[:resource_type]
+    end
+
+    # Determine agent level monitored resource labels based on the resource
+    # type. Each resource type has its own labels that need to be filled in.
+    def determine_agent_level_monitored_resource_labels(type)
+      labels = {}
+
+      case type
+
+      # GAE app.
+      when APPENGINE_CONSTANTS[:resource_type]
+        begin
+          labels['module_id'] = fetch_gce_metadata(
+            'instance/attributes/gae_backend_name')
+          labels['version_id'] = fetch_gce_metadata(
+            'instance/attributes/gae_backend_version')
+        rescue StandardError => e
+          @log.error 'Failed to set monitored resource labels for GAE: ',
+                     error: e
+        end
+
+      # GCE.
+      when COMPUTE_CONSTANTS[:resource_type]
+        labels['instance_id'] = @vm_id
+        labels['zone'] = @zone
+
+      # GKE container.
+      when CONTAINER_CONSTANTS[:resource_type]
+        labels['instance_id'] = @vm_id
+        labels['zone'] = @zone
+        begin
+          raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
+          kube_env = YAML.load(raw_kube_env)
+          labels['cluster_name'] =
+            cluster_name_from_kube_env(kube_env)
+        rescue StandardError => e
+          @log.error 'Failed to set monitored resource labels for GKE: ',
+                     error: e
+        end
+
+      # Cloud Dataproc.
+      when DATAPROC_CONSTANTS[:resource_type]
+        begin
+          labels['cluster_uuid'] =
+            fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid')
+          labels['cluster_name'] =
+            fetch_gce_metadata('instance/attributes/dataproc-cluster-name')
+          labels['region'] =
+            fetch_gce_metadata('instance/attributes/dataproc-region')
+        rescue StandardError => e
+          @log.error 'Failed to set monitored resource labels for Cloud ' \
+                     'Dataproc: ', error: e
+        end
+
+      # EC2.
+      when EC2_CONSTANTS[:resource_type]
+        labels['instance_id'] = @vm_id
+        labels['region'] = @zone
+        labels['aws_account'] = ec2_metadata['accountId'] if
+          ec2_metadata.key?('accountId')
+      end
+      labels
+    end
+
+    # Determine the common labels that should be added to all log entries
+    # processed by this logging agent.
+    def determine_agent_level_common_labels
+      labels = {}
+      # User can specify labels via config. We want to capture those as well.
+      # TODO: Send instance tags as labels as well?
+      labels.merge!(@labels) if @labels
+
+      case @resource.type
+
+      # GAE app.
+      when APPENGINE_CONSTANTS[:resource_type]
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
+
+      # GCE and GKE container.
+      when COMPUTE_CONSTANTS[:resource_type],
+           CONTAINER_CONSTANTS[:resource_type]
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+
+      # Cloud Dataflow and Cloud Dataproc.
+      when DATAFLOW_CONSTANTS[:resource_type],
+           DATAPROC_CONSTANTS[:resource_type]
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
+
+      # EC2.
+      when EC2_CONSTANTS[:resource_type]
+        labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
+
+      # Cloud ML.
+      when ML_CONSTANTS[:resource_type]
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
+        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
+        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
+      end
+      labels
+    end
+
+    # Determine the group level monitored resource and common labels shared by a
+    # collection of entries.
+    def determine_group_level_monitored_resource_and_labels(tag)
+      # Determine group level monitored resource type. For certain types,
+      # extract useful info from the tag and store those in
+      # matched_regexp_group.
+      group_resource_type, matched_regexp_group =
+        determine_group_level_monitored_resource_type(tag)
+
+      # Determine group level monitored resource labels and common labels.
+      group_resource_type, group_resource_labels, group_common_labels = \
+        determine_group_level_labels_and_adjust_type(
+          group_resource_type, matched_regexp_group)
+
+      group_resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
+        type: group_resource_type,
+        labels: group_resource_labels.to_h
+      )
+
+      # Freeze the per-request state. Any further changes must be made on a
+      # per-entry basis.
+      group_resource.freeze
+      group_resource.labels.freeze
+      group_common_labels.freeze
+
+      [group_resource, group_common_labels]
+    end
+
+    # Determine group level monitored resource type shared by a collection of
+    # entries.
+    # Returns the resource type and tag regexp matched groups. The matched
+    # groups only apply to some resource types. Return nil if not applicable or
+    # if there is no match.
+    def determine_group_level_monitored_resource_type(tag)
+      # Match tag against Cloud Functions format.
+      if @running_cloudfunctions
+        matched_regexp_group = @cloudfunctions_tag_regexp.match(tag)
+        return [CLOUDFUNCTIONS_CONSTANTS[:resource_type],
+                matched_regexp_group] if matched_regexp_group
+      end
+
+      # Match tag against Docker container stderr / stdout log format and
+      # Docker container application log format.
+      matched_regexp_group =
+        # Format: "container.<container_id>.<container_name>"
+        @dockercontainer_tag_regexp.match(tag) ||
+        # Format: "application-container.<container_name>.<additional_tag>"
+        @dockercontainer_tag_with_application_regexp.match(tag)
+      return [DOCKER_CONSTANTS[:resource_type], matched_regexp_group] if
+        matched_regexp_group
+
+      # Match tag against GKE Container format.
+      if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
+         @compiled_kubernetes_tag_regexp
+        # Container logs in Kubernetes are tagged based on where they came from,
+        # so we can extract useful metadata from the tag. Do this here to avoid
+        # having to repeat it for each record.
+        matched_regexp_group = @compiled_kubernetes_tag_regexp.match(tag)
+        return [@resource.type, matched_regexp_group] if matched_regexp_group
+      end
+
+      # Otherwise, return the original type.
+      [@resource.type, nil]
+    end
+
+    # Determine group level monitored resource labels and common labels. These
+    # labels will be shared by a collection of entries. In certain cases, we
+    # might also adjust the resource type.
+    def determine_group_level_labels_and_adjust_type(group_resource_type,
+                                                     matched_regexp_group)
+      group_resource_labels = @resource.labels.dup
+      group_common_labels = @common_labels.dup
+
+      case group_resource_type
+
+      # Cloud Functions.
+      when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
+        group_resource_labels['region'] = @gcf_region
+        group_resource_labels['function_name'] =
+          decode_cloudfunctions_function_name(
+            matched_regexp_group['encoded_function_name'])
+        instance_id = group_resource_labels.delete('instance_id')
+        group_common_labels["#{CONTAINER_CONSTANTS[:service]}/cluster_name"] =
+          group_resource_labels.delete('cluster_name')
+        group_common_labels["#{CONTAINER_CONSTANTS[:service]}/instance_id"] =
+          instance_id
+        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
+          instance_id
+        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] =
+          group_resource_labels.delete('zone')
+
+      # GKE container.
+      when CONTAINER_CONSTANTS[:resource_type]
+        if matched_regexp_group
+          group_resource_labels['container_name'] =
+            matched_regexp_group['container_name']
+          # The kubernetes_tag_regexp is poorly named. 'namespace_name' is in
+          # fact 'namespace_id'. 'pod_name' is in fact 'pod_id'.
+          group_resource_labels['namespace_id'] =
+            matched_regexp_group['namespace_name']
+          group_resource_labels['pod_id'] =
+            matched_regexp_group['pod_name']
+          %w(namespace_name pod_name).each do |field|
+            group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
+              matched_regexp_group[field]
+          end
+        end
+
+      # Docker container.
+      when DOCKER_CONSTANTS[:resource_type]
+        # For Docker container stderr / stdout logs generated by Docker Fluentd
+        # Logging Driver, tags are in the format of "container.<container_id>.
+        # <container_name>", thus they include 'container_id' info.
+        # For logs generated by applications running in Docker containers,
+        # tags are in the format of "application-container.<container_name>.
+        # <additional_tag>", thus 'container_id' info is unknown yet.
+        # 'container_name' info on the other hand is always available.
+        container_id = matched_regexp_group['container_id'] if
+          matched_regexp_group.names.include? 'container_id'
+        container_name = matched_regexp_group['container_name']
+
+        if @enable_metadata_agent
+          # Call Metadata Agent with "container.<container_id>" or
+          # "application-container.<container_name>" as the locally-unique key
+          # to retrieve monitored resource. This should be different from the
+          # original @resource value that got initiated when the agent starts up
+          # because that one is always at the VM level.
+          if container_id
+            locally_unique_id = "container.#{container_id}"
+          else
+            locally_unique_id = "containerName.#{container_name}"
+          end
+          retrieved_resource = call_metadata_agent_for_monitored_resource(
+            locally_unique_id)
+        end
+
+        if !retrieved_resource.nil?
+          # If we successfully get a monitored resource from Metadata Agent,
+          # use this one instead of the original instance monitored resource.
+          group_resource_labels = retrieved_resource.labels.dup
+          @log.debug 'Retrieved monitored resource from Metadata Agent: ' \
+                     "#{retrieved_resource.inspect}."
+        else
+          # If Metadata Agent is not enabled, or we failed to get a monitored
+          # resource, we need to have some backup plan.
+          @log.debug 'Metadata Agent not enabled or failed to retrieve ' \
+                     'docker container monitored resource from Metadata ' \
+                     'Agent.'
+
+          # 1. Check if 'container_id' is set already. It should be available
+          #    for stdout / stderr). If so, use that.
+          # 2. If not, call Docker Remote API to retrieve the container ID from
+          #    container name, but only if @call_docker_api_locally is true.
+          container_id ||= retrieve_container_id_by_name_locally(
+            container_name) if @call_docker_api_locally
+          unless container_id
+            @log.debug 'No docker container id retrieved. Falling back to
+                        instance monitored resource.'
+            # If a container id is not available, fall back to the instance
+            # monitored resource.
+            return [COMPUTE_CONSTANTS[:resource_type], group_resource_labels,
+                    group_common_labels]
+          end
+          group_resource_labels['container_id'] = container_id
+          # 'zone' for GCP and 'region' for EC2 must have been set at this
+          # point. Rename them to 'location'.
+          group_resource_labels['location'] = @zone
+          if @platform == Platform::EC2
+            group_resource_labels.delete('region')
+          else
+            group_resource_labels.delete('zone')
+          end
+          # vm id info should be reported as a metadata label instead.
+          group_resource_labels.delete('instance_id')
+
+        end
+        # Set metadata labels.
+        group_common_labels["#{DOCKER_CONSTANTS[:service]}/container_name"] =
+          matched_regexp_group['container_name']
+        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
+          @vm_id
+      end
+
+      [group_resource_type, group_resource_labels, group_common_labels]
+    end
+
+    # Extract entry resource and common labels that should be applied to
+    # individual entries from the group resource.
+    def determine_entry_level_labels(group_resource, record)
+      resource_type = group_resource.type
+      resource_labels = {}
+      common_labels = {}
+
+      # The format of the locally unique key varies by monitored resource.
+      #
+      # Docker container:
+      #   "container.<container_id>"
+      #   "containerName.<container_name>"
+      # GKE container:
+      #   "gke_containerName.<namespace_id>.<pod_name>.<container_name>"
+      if @enable_metadata_agent && record.key?(LOCALLY_UNIQUE_ID_LABEL_NAME)
+        locally_unique_id = record.delete(LOCALLY_UNIQUE_ID_LABEL_NAME)
+        @log.debug 'Calling metadata agent with locally unique id: ' \
+                  "#{locally_unique_id}."
+        retrieved_resource = call_metadata_agent_for_monitored_resource(
+          locally_unique_id)
+        @log.debug 'Retrieved monitored resource from metadata agent: ' \
+                  "#{retrieved_resource.inspect}."
+        unless retrieved_resource.nil?
+          resource_type = retrieved_resource.type
+          # Temporarily renaming 'gke_container' to 'container'.
+          resource_type = 'container' if resource_type == 'gke_container'
+          # If we successfully get a monitored resource from Metadata Agent,
+          # use this one instead of the original VM-level monitored resource.
+          resource_labels = retrieved_resource.labels.dup
+          @log.debug 'Retrieved gke_container monitored resource from' \
+                     'Stackdriver Metadata agent: ' \
+                     "#{retrieved_resource.inspect}."
+        end
+      end
+
+      # Cloud Functions.
+      if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
+         record.key?('log')
+        @cloudfunctions_log_match =
+          @cloudfunctions_log_regexp.match(record['log'])
+        common_labels['execution_id'] =
+          @cloudfunctions_log_match['execution_id'] if \
+            @cloudfunctions_log_match &&
+            @cloudfunctions_log_match['execution_id']
+      end
+
+      # GKE containers.
+      if resource_type == CONTAINER_CONSTANTS[:resource_type]
+        # Move the stdout/stderr annotation from the record into a label.
+        common_labels.merge!(
+          delete_and_extract_labels(
+            record, 'stream' => "#{CONTAINER_CONSTANTS[:service]}/stream"))
+
+        # If the record has been annotated by the kubernetes_metadata_filter
+        # plugin, then use that metadata. Otherwise, rely on commonLabels
+        # populated at the grouped_entries level from the group's tag.
+        if record.key?('kubernetes')
+          resource_labels.merge!(
+            delete_and_extract_labels(
+              record['kubernetes'], %w(namespace_id pod_id container_name)
+                .map { |l| [l, l] }.to_h))
+          common_labels.merge!(
+            delete_and_extract_labels(
+              record['kubernetes'], %w(namespace_name pod_name)
+                .map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
+          # Prepend label/ to all user-defined labels' keys.
+          if record['kubernetes'].key?('labels')
+            record['kubernetes']['labels'].each do |key, value|
+              common_labels["label/#{key}"] = value
+            end
+          end
+          # We've explicitly consumed all the fields we care about -- don't
+          # litter the log entries with the remaining fields that the kubernetes
+          # metadata filter plugin includes (or an empty 'kubernetes' field).
+          record.delete('kubernetes')
+          record.delete('docker')
+        end
+      end
+
+      # Docker containers.
+      if resource_type == DOCKER_CONSTANTS[:resource_type]
+        # For logs coming from Docker Fluentd Logging Driver, the log record
+        # has 4 fields: 'container_id', 'container_name', 'source' and 'log'.
+        # Extract 'container_id', 'container_name' and 'source' from json
+        # record, set corresponding labels, and remove these fields from record.
+        {
+          'container_name' => 'container_name',
+          'source' => 'stream'
+        }.each do |field_name, label_name|
+          common_labels.merge!(
+            delete_and_extract_labels(
+              record,
+              field_name => "#{DOCKER_CONSTANTS[:service]}/#{label_name}"
+            )
+          )
+        end
+        resource_labels.merge!(
+          delete_and_extract_labels(record, 'container_id' => 'container_id'))
+      end
+
+      # If the name of a field in the record is present in the @label_map
+      # configured by users, report its value as a label and do not send that
+      # field as part of the payload.
+      common_labels.merge!(delete_and_extract_labels(record, @label_map))
+
+      # Cloud Dataflow.
+      # These labels can be set via configuring 'labels' or 'label_map'.
+      # Report them as monitored resource labels instead of common labels.
+      if group_resource.type == DATAFLOW_CONSTANTS[:resource_type]
+        resource_labels.merge!(
+          delete_and_extract_labels(
+            common_labels, %w(region job_name job_id step_id)
+              .map { |l| ["#{DATAFLOW_CONSTANTS[:service]}/#{l}", l] }.to_h))
+      end
+
+      # Cloud ML.
+      # These labels can be set via configuring 'labels' or 'label_map'.
+      # Report them as monitored resource labels instead of common labels.
+      if group_resource.type == ML_CONSTANTS[:resource_type]
+        resource_labels.merge!(
+          delete_and_extract_labels(
+            common_labels, %w(job_id task_name)
+              .map { |l| ["#{ML_CONSTANTS[:service]}/#{l}", l] }.to_h))
+      end
+
+      [resource_type, resource_labels, common_labels]
+    end
+
+    # Call Metadata Agent to get monitored resource information and parse
+    # response to Google::Api::MonitoredResource.
+    def call_metadata_agent_for_monitored_resource(unique_key)
+      response = call_metadata_agent("monitoredResource/#{unique_key}")
+      return nil if response.nil?
+      begin
+        resource = Google::Api::MonitoredResource.decode_json(response.to_json)
+      rescue Google::Protobuf::ParseError, ArgumentError => e
+        @log.error 'Error paring monitored resource from Metadata Agent. ' \
+                   "response: #{response.inspect}", error: e
+        return nil
+      end
+
+      # TODO(qingling128): Use Google::Api::MonitoredResource directly after we
+      # upgrade gRPC version to include the fix for the protobuf map
+      # corruption issue.
+      Google::Apis::LoggingV2beta1::MonitoredResource.new(
+        type: resource.type,
+        labels: resource.labels.to_h
+      )
+    end
+
+    # Call Metadata Agent and parse response to json. Return nil in case of any
+    # error / failure.
+    def call_metadata_agent(path)
+      url = "#{@metadata_agent_url}/#{path}"
+      @log.debug("Calling Metadata Agent: #{url}")
+      open(url) do |f|
+        response = f.read
+        parsed_hash = parse_json_or_nil(response)
+        if parsed_hash.nil?
+          @log.error 'Response from Metadata Agent is not in valid json ' \
+                     "format: '#{response.inspect}'."
+          return nil
+        end
+        @log.debug "Response from Metadata Agent: #{parsed_hash}"
+        return parsed_hash
+      end
+    rescue StandardError => e
+      @log.error 'Error calling Metadata Agent.', error: e
+      return nil
     end
 
     # TODO: This functionality should eventually be available in another
@@ -902,11 +1364,28 @@ module Fluent
       end
     end
 
-    def detect_cloudfunctions(attributes)
-      return unless attributes.include?('gcf_region')
-      # Cloud Functions detected
-      @running_cloudfunctions = true
-      @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
+    # Calling Docker Remote API to get container id by name.
+    def retrieve_container_id_by_name_locally(container_name)
+      response = Excon.get(
+        "unix:///containers/#{container_name}/json",
+        socket: @docker_remote_api_socket_path)
+      @log.debug "Response from Docker API with name '#{container_name}': " \
+                 "#{response.inspect}."
+      return parse_container_id_from_docker_api_response(response)
+    rescue StandardError => e
+      @log.error 'Error calling Docker API to get container id.', error: e
+      return nil
+    end
+
+    # Parse the container id from Docker Remote API response.
+    # TODO(qingling128) Add a config for Docker API version to support parsing
+    # different versions of Docker Remote API when the format varies.
+    def parse_container_id_from_docker_api_response(response)
+      JSON.parse(response.data[:body])['Id']
+    rescue StandardError => e
+      @log.error 'Error parsing Docker API response to get container id.',
+                 error: e
+      return nil
     end
 
     def cluster_name_from_kube_env(kube_env)
@@ -979,9 +1458,14 @@ module Fluent
         end
       elsif record.key?('severity')
         return parse_severity(record.delete('severity'))
-      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
-            entry_common_labels.key?("#{CONTAINER_CONSTANTS[:service]}/stream")
-        stream = entry_common_labels["#{CONTAINER_CONSTANTS[:service]}/stream"]
+      elsif [CONTAINER_CONSTANTS[:resource_type],
+             DOCKER_CONSTANTS[:resource_type]].include?(resource_type)
+        stream = entry_common_labels[
+          "#{CONTAINER_CONSTANTS[:service]}/stream"] if
+           resource_type == CONTAINER_CONSTANTS[:resource_type]
+        stream = entry_common_labels[
+          "#{DOCKER_CONSTANTS[:service]}/stream"] if
+           resource_type == DOCKER_CONSTANTS[:resource_type]
         if stream == 'stdout'
           return 'INFO'
         elsif stream == 'stderr'
@@ -1161,43 +1645,35 @@ module Fluent
         .gsub('u.u', '_').gsub('d.d', '$').gsub('a.a', '@').gsub('p.p', '.')
     end
 
-    # Requires that record has a 'kubernetes' field.
-    def extract_container_metadata(record)
-      resource_labels = {}
-      common_labels = {}
-      %w(namespace_id pod_id container_name).each do |field|
-        resource_labels.merge!(
-          fields_to_labels(record['kubernetes'], field => field))
-      end
-      %w(namespace_name pod_name).each do |field|
-        common_labels.merge!(
-          fields_to_labels(
-            record['kubernetes'],
-            field => "#{CONTAINER_CONSTANTS[:service]}/#{field}"))
-      end
-      # Prepend label/ to all user-defined labels' keys.
-      if record['kubernetes'].key?('labels')
-        record['kubernetes']['labels'].each do |key, value|
-          common_labels["label/#{key}"] = value
-        end
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    # Given a tag, returns the corresponding valid tag if possible, or nil if
+    # the tag should be rejected. If 'require_valid_tags' is false, non-string
+    # tags are converted to strings, and invalid characters are sanitized;
+    # otherwise such tags are rejected.
+    def sanitize_tag(tag)
+      if @require_valid_tags &&
+         (!tag.is_a?(String) || tag == '' || convert_to_utf8(tag) != tag)
+        return nil
       end
-      # We've explicitly consumed all the fields we care about -- don't litter
-      # the log entries with the remaining fields that the kubernetes metadata
-      # filter plugin includes (or an empty 'kubernetes' field).
-      record.delete('kubernetes')
-      record.delete('docker')
-      [resource_labels, common_labels]
+      tag = convert_to_utf8(tag.to_s)
+      tag = '_' if tag == ''
+      tag
     end
 
     # For every original_label => new_label pair in the label_map, delete the
-    # original_label from the record if it exists, and extract the value to form
-    # a map with the new_label as the key.
-    def fields_to_labels(record, label_map)
-      return {} if label_map.nil? || !label_map.is_a?(Hash)
+    # original_label from the original_resource map if it exists, and extract
+    # the value to form a map with the new_label as the key.
+    def delete_and_extract_labels(original_resource, label_map)
+      return {} if label_map.nil? || !label_map.is_a?(Hash) ||
+                   original_resource.nil? || !original_resource.is_a?(Hash)
       label_map.each_with_object({}) \
         do |(original_label, new_label), extracted_labels|
         extracted_labels[new_label] = convert_to_utf8(
-          record.delete(original_label).to_s) if record.key?(original_label)
+          original_resource.delete(original_label).to_s) if
+            original_resource.key?(original_label)
       end
     end
 
@@ -1216,7 +1692,8 @@ module Fluent
         entry.text_payload = record['log']
       elsif is_json
         entry.json_payload = record
-      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
+      elsif [CONTAINER_CONSTANTS[:resource_type],
+             DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
             record.key?('log')
         entry.text_payload = record['log']
       elsif record.size == 1 && record.key?('message')
@@ -1286,7 +1763,8 @@ module Fluent
         entry.text_payload = convert_to_utf8(record['log'])
       elsif is_json
         entry.json_payload = struct_from_ruby(record)
-      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
+      elsif [CONTAINER_CONSTANTS[:resource_type],
+             DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
             record.key?('log')
         entry.text_payload = convert_to_utf8(record['log'])
       elsif record.size == 1 && record.key?('message')
@@ -1299,7 +1777,7 @@ module Fluent
     def log_name(tag, resource)
       if resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type]
         tag = 'cloud-functions'
-      elsif @running_on_managed_vm
+      elsif resource.type == APPENGINE_CONSTANTS[:resource_type]
         # Add a prefix to Managed VM logs to prevent namespace collisions.
         tag = "#{APPENGINE_CONSTANTS[:service]}/#{tag}"
       elsif resource.type == CONTAINER_CONSTANTS[:resource_type]
@@ -1314,32 +1792,6 @@ module Fluent
       tag
     end
 
-    # Some services set labels (via configuring 'labels' or 'label_map') which
-    # are now MonitoredResource labels in v2.
-    # For these services, remove resource labels from 'labels' and return a
-    # Hash of labels to be merged into the MonitoredResource labels.
-    # Otherwise, return an empty hash and leave 'labels' unmodified.
-    def extract_resource_labels(resource_type, labels)
-      extracted_labels = {}
-      return extracted_labels if labels.nil? || !labels.is_a?(Hash)
-
-      if resource_type == DATAFLOW_CONSTANTS[:resource_type]
-        label_prefix = DATAFLOW_CONSTANTS[:service]
-        labels_to_extract = %w(region job_name job_id step_id)
-      elsif resource_type == ML_CONSTANTS[:resource_type]
-        label_prefix = ML_CONSTANTS[:service]
-        labels_to_extract = %w(job_id task_name)
-      else
-        return extracted_labels
-      end
-
-      labels_to_extract.each do |label|
-        extracted_labels[label] = labels.delete("#{label_prefix}/#{label}") if
-          labels.key?("#{label_prefix}/#{label}")
-      end
-      extracted_labels
-    end
-
     def init_api_client
       return if @use_grpc
       # TODO: Use a non-default ClientOptions object.