fluent-plugin-google-cloud 0.6.5.pre.1 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b1a26eab783c87890f78500848a29048b4ab615
-  data.tar.gz: 81711bc2e00bcf222b7249bde80d78c6bab34957
+  metadata.gz: d7a29c9e8d247a3ca399605a375529a371856aa2
+  data.tar.gz: 49254ff8d84d7ab097ae2087f830882c7c990f8d
 SHA512:
-  metadata.gz: 7d5c1ca8f704998cd36c5a040dc4fa10989d2cbb7717dea8fa67032c3c42b428a5f3ef37f04581caea41bf991232688d3245ebe285e7fb72f9f0b783712d399f
-  data.tar.gz: 8a95c809c56fcc6351f08bd81d3124cd31b6529c54afb3c72d109519c4aecd5764d26514b4dc8a99417510c3883451ae54189e65a1e10d81465143888bf68b11
+  metadata.gz: 582f2b49c2a5fbbefdeef47e124b6de5ac197028630c8ee2fe784cfa00ae914158c358ac04e88e06bab539c3969b29ba8b700d7048b0d5436a54733cba229808
+  data.tar.gz: 00decd87d0b8b4b4f1d1be8a77ec332d2b276f0c29edb7b0d6e34e0f950eb72e48b32948db3be40edfd8e3483cfe5ff166f5b51fef96128c296e0a867483f418

data/Gemfile.lock

@@ -1,8 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.5.pre.1)
-      excon (~> 0.57.1)
+    fluent-plugin-google-cloud (0.6.5)
       fluentd (~> 0.10)
       google-api-client (~> 0.9.0)
       google-cloud-logging (= 0.24.1)
@@ -22,8 +21,7 @@ GEM
     cool.io (1.5.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    excon (0.57.1)
-    faraday (0.12.2)
+    faraday (0.13.1)
       multipart-post (>= 1.2, < 3)
     fluentd (0.14.20)
       cool.io (>= 1.4.5, < 2.0.0)
@@ -51,13 +49,13 @@ GEM
       google-cloud-core (~> 0.21.1)
       google-gax (~> 0.8.0)
       stackdriver-core (~> 0.21.0)
-    google-gax (0.8.5)
+    google-gax (0.8.6)
       google-protobuf (~> 3.2)
       googleapis-common-protos (~> 1.3.5)
       googleauth (~> 0.5.1)
       grpc (~> 1.0)
       rly (~> 0.2.3)
-    google-protobuf (3.3.0-x86_64-linux)
+    google-protobuf (3.4.0.2)
     googleapis-common-protos (1.3.5)
       google-protobuf (~> 3.2)
       grpc (~> 1.0)
@@ -69,10 +67,10 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
-    grpc (1.2.5-x86_64-linux)
+    grpc (1.2.5)
       google-protobuf (~> 3.1)
       googleauth (~> 0.5.1)
-    hashdiff (0.3.5)
+    hashdiff (0.3.6)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
     hurley (0.2)
@@ -95,7 +93,7 @@ GEM
     os (0.9.6)
     parser (2.4.0.0)
       ast (~> 2.2)
-    power_assert (1.0.2)
+    power_assert (1.1.0)
     powerpack (0.1.1)
     prometheus-client (0.7.1)
       quantile (~> 0.2.0)
@@ -156,4 +154,4 @@ DEPENDENCIES
   webmock (~> 2.3.1)
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.5.pre.1'
+  gem.version       = '0.6.5'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -19,7 +19,6 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'excon', '~> 0.57.1'
   gem.add_runtime_dependency 'fluentd', '~> 0.10'
   gem.add_runtime_dependency 'googleapis-common-protos', '~> 1.3'
   gem.add_runtime_dependency 'google-api-client', '~> 0.9.0'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -11,11 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-require 'excon'
 require 'grpc'
 require 'json'
 require 'open-uri'
-require 'rubygems'
 require 'socket'
 require 'time'
 require 'yaml'
@@ -40,13 +38,12 @@ end
 module Fluent
   # fluentd output plugin for the Stackdriver Logging API
   class GoogleCloudOutput < BufferedOutput
-    # Constants.
+    # Constants for service names and resource types.
     module Constants
-      # Service names and resource types.
-
       APPENGINE_CONSTANTS = {
         service: 'appengine.googleapis.com',
-        resource_type: 'gae_app'
+        resource_type: 'gae_app',
+        metadata_attributes: %w(gae_backend_name gae_backend_version).to_set
       }
       CLOUDFUNCTIONS_CONSTANTS = {
         service: 'cloudfunctions.googleapis.com',
@@ -58,19 +55,21 @@ module Fluent
       }
       CONTAINER_CONSTANTS = {
         service: 'container.googleapis.com',
-        resource_type: 'container'
-      }
-      DOCKER_CONSTANTS = {
-        service: 'dockercontainer.googleapis.com',
-        resource_type: 'docker_container'
+        resource_type: 'container',
+        extra_resource_labels: %w(namespace_id pod_id container_name),
+        extra_common_labels: %w(namespace_name pod_name),
+        metadata_attributes: %w(kube-env).to_set
       }
       DATAFLOW_CONSTANTS = {
         service: 'dataflow.googleapis.com',
-        resource_type: 'dataflow_step'
+        resource_type: 'dataflow_step',
+        extra_common_labels: %w(region job_name job_id step_id)
       }
       DATAPROC_CONSTANTS = {
         service: 'cluster.dataproc.googleapis.com',
-        resource_type: 'cloud_dataproc_cluster'
+        resource_type: 'cloud_dataproc_cluster',
+        metadata_attributes:
+          %w(dataproc-cluster-uuid dataproc-cluster-name).to_set
       }
       EC2_CONSTANTS = {
         service: 'ec2.amazonaws.com',
@@ -78,25 +77,83 @@ module Fluent
       }
       ML_CONSTANTS = {
         service: 'ml.googleapis.com',
-        resource_type: 'ml_job'
+        resource_type: 'ml_job',
+        extra_common_labels: %w(job_id task_name)
       }
 
-      # Default value for trace_key config param to set "trace" LogEntry field.
-      DEFAULT_TRACE_KEY = 'logging.googleapis.com/trace'
-
-      # Metadata Agent support.
-
-      # Use empty string as request path when locally-unique key of monitored
-      # resource can be implicitly inferred by Metadata Agent.
-      IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY = ''
-
-      # The label name of locally unique id in the json payload. When a record
-      # has this field in the payload, we will use the value to retrieve
-      # monitored resource from Stackdriver Metadata agent.
-      LOCALLY_UNIQUE_ID_LABEL_NAME = 'logging.googleapis.com/locally_unique_id'
-
-      # Docker container support.
-      DEFAULT_DOCKER_API_SOCKET_PATH = '/var/run/docker.sock'
+      # The map between a subservice name and a resource type.
+      SUBSERVICE_MAP = \
+        [APPENGINE_CONSTANTS, CONTAINER_CONSTANTS, DATAFLOW_CONSTANTS,
+         DATAPROC_CONSTANTS, ML_CONSTANTS]
+        .map { |consts| [consts[:service], consts[:resource_type]] }.to_h
+      # Default back to GCE if invalid value is detected.
+      SUBSERVICE_MAP.default = COMPUTE_CONSTANTS[:resource_type]
+
+      # The map between a resource type and expected subservice attributes.
+      SUBSERVICE_METADATA_ATTRIBUTES = \
+        [APPENGINE_CONSTANTS, CONTAINER_CONSTANTS, DATAPROC_CONSTANTS]
+        .map { |consts| [consts[:resource_type], consts[:metadata_attributes]] }
+        .to_h
+
+      # Default values for JSON payload keys to set the "trace",
+      # "sourceLocation", "operation" and "labels" fields in the LogEntry.
+      DEFAULT_PAYLOAD_KEY_PREFIX = 'logging.googleapis.com'
+      DEFAULT_LABELS_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/labels"
+      DEFAULT_HTTP_REQUEST_KEY = 'httpRequest'
+      DEFAULT_OPERATION_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/operation"
+      DEFAULT_SOURCE_LOCATION_KEY =
+        "#{DEFAULT_PAYLOAD_KEY_PREFIX}/sourceLocation"
+      DEFAULT_TRACE_KEY = "#{DEFAULT_PAYLOAD_KEY_PREFIX}/trace"
+
+      # Map from each field name under LogEntry to corresponding variables
+      # required to perform field value extraction from the log record.
+      LOG_ENTRY_FIELDS_MAP = {
+        'http_request' => [
+          # The config to specify label name for field extraction from record.
+          '@http_request_key',
+          # Map from subfields' names to their types.
+          [
+            # subfield key in the payload, destination key, cast lambda (opt)
+            %w(requestMethod request_method parse_string),
+            %w(requestUrl request_url parse_string),
+            %w(requestSize request_size parse_int),
+            %w(status status parse_int),
+            %w(responseSize response_size parse_int),
+            %w(userAgent user_agent parse_string),
+            %w(remoteIp remote_ip parse_string),
+            %w(referer referer parse_string),
+            %w(cacheHit cache_hit parse_bool),
+            %w(cacheValidatedWithOriginServer
+               cache_validated_with_origin_server parse_bool),
+            %w(latency latency parse_latency)
+          ],
+          # The grpc version class name.
+          'Google::Logging::Type::HttpRequest',
+          # The non-grpc version class name.
+          'Google::Apis::LoggingV2beta1::HttpRequest'
+        ],
+        'source_location' => [
+          '@source_location_key',
+          [
+            %w(file file parse_string),
+            %w(function function parse_string),
+            %w(line line parse_int)
+          ],
+          'Google::Logging::V2::LogEntrySourceLocation',
+          'Google::Apis::LoggingV2beta1::LogEntrySourceLocation'
+        ],
+        'operation' => [
+          '@operation_key',
+          [
+            %w(id id parse_string),
+            %w(producer producer parse_string),
+            %w(first first parse_bool),
+            %w(last last parse_bool)
+          ],
+          'Google::Logging::V2::LogEntryOperation',
+          'Google::Apis::LoggingV2beta1::LogEntryOperation'
+        ]
+      }
     end
 
     include self::Constants
@@ -104,7 +161,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.5.pre.1'
+    PLUGIN_VERSION = '0.6.5'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -134,10 +191,14 @@ module Fluent
     config_param :vm_id, :string, :default => nil
     config_param :vm_name, :string, :default => nil
 
-    # Set values from JSON payload with this key to the "trace" LogEntry field.
+    # Map keys from a JSON payload to corresponding LogEntry fields.
+    config_param :labels_key, :string, :default => DEFAULT_LABELS_KEY
+    config_param :http_request_key, :string, :default =>
+      DEFAULT_HTTP_REQUEST_KEY
+    config_param :operation_key, :string, :default => DEFAULT_OPERATION_KEY
+    config_param :source_location_key, :string, :default =>
+      DEFAULT_SOURCE_LOCATION_KEY
     config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
-    # Whether to also keep the trace key/value in the payload.
-    config_param :keep_trace_key, :bool, :default => false
 
     # Whether to try to detect if the VM is owned by a "subservice" such as App
     # Engine of Kubernetes, rather than just associating the logs with the
@@ -231,18 +292,6 @@ module Fluent
     config_param :monitoring_type, :string,
                  :default => Monitoring::PrometheusMonitoringRegistry.name
 
-    # Whether to call metadata agent to retrieve monitored resource.
-    config_param :enable_metadata_agent, :bool, :default => false
-    config_param :metadata_agent_url, :string,
-                 :default => 'http://local-metadata-agent.stackdriver.com:8000'
-
-    # Whether to call Docker Remote API locally when Metadata Agent is not
-    # enabled or if the request fails.
-    config_param :call_docker_api_locally, :bool, :default => true
-    # Docker Remote API unix socket path.
-    config_param :docker_remote_api_socket_path, :string,
-                 :default => DEFAULT_DOCKER_API_SOCKET_PATH
-
     # rubocop:enable Style/HashSyntax
 
     # TODO: Add a log_name config option rather than just using the tag?
@@ -261,35 +310,6 @@ module Fluent
       @log = $log # rubocop:disable Style/GlobalVars
     end
 
-    # Set up regex patterns used to parse tags and logs.
-    def setup_regex_patterns
-      @compiled_kubernetes_tag_regexp = nil
-      if @kubernetes_tag_regexp
-        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
-      end
-
-      @cloudfunctions_tag_regexp =
-        /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
-      @cloudfunctions_log_regexp = /^
-        (?:\[(?<severity>.)\])?
-        \[(?<timestamp>.{24})\]
-        (?:\[(?<execution_id>[^\]]+)\])?
-        [ ](?<text>.*)$/x
-
-      # Docker container tag format:
-      # "container.<container_id>.<container_name>".
-      @dockercontainer_tag_regexp =
-        /^container\.(?<container_id>[a-zA-Z0-9]+)\.
-        (?<container_name>[a-zA-Z0-9_.-]+)$/x
-      # Docker container with application tag format:
-      # "application-container.<container_name>.<additional_tag>".
-      @dockercontainer_tag_with_application_regexp =
-        /^application-container\.(?<container_name>[a-zA-Z0-9_.-]+)\.
-        (?<additional_tag>.+)$/x
-
-      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
-    end
-
     def configure(conf)
       super
 
@@ -327,49 +347,46 @@ module Fluent
                extra.join(' ')
       end
 
-      setup_regex_patterns
+      set_regexp_patterns
 
       @platform = detect_platform
 
-      # Set agent-level monitored resource. This monitored resource is initiated
-      # as the logging agent starts up. It will be inherited by all log entries
-      # processed by this agent. First try to retrieve it via Metadata Agent.
-      if @enable_metadata_agent
-        # The locally-unique key for this should be the instance id. Since this
-        # can be implicitly inferred by Metadata Agent, we do not need to
-        # explicitly send the key.
-        @resource = call_metadata_agent_for_monitored_resource(
-          IMPLICIT_MONITORED_RESOURCE_UNIQUE_KEY)
-      end
-
-      # Set required variables: @project_id, @vm_id, @vm_name and @zone.
-      # If any info above is included in the response from Metadata Agent, make
-      # use of that. Otherwise make some additional requests to metadata server.
+      # Set required variables: @project_id, @vm_id, @vm_name and @zone by
+      # making some requests to metadata server.
+      #
+      # Note: Once we support metadata injection at Logging API side, we might
+      # no longer need to require all these metadata in logging agent. But for
+      # now, they are still required.
       #
-      # Note: Once we support metadata injection on the Logging API side, we
-      # might no longer need to require all these metadata in logging agent. But
-      # for now, they are still required.
+      # TODO(qingling128): After Metadata Agent support is added, try extracting
+      # these info from responses from Metadata Agent first.
       set_required_metadata_variables
 
-      # Fail over to retrieve monitored resource via the legacy path if we fail
-      # to get it from Metadata Agent.
-      @resource ||= determine_agent_level_monitored_resource_via_legacy
-
-      # Set variables specific to CLoud Functions. This has to be called after
-      # we have determined the resource type. The purpose is to avoid repeated
-      # calls to metadata server.
-      @running_cloudfunctions = false
-      # We only support Cloud Functions logs for GKE right now.
-      if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
-         fetch_gce_metadata('instance/attributes/').split.include?('gcf_region')
-        # We are not setting resource type as Cloud Functions here because
-        # whether a log entry is truly coming from a Cloud Functions function
-        # depends on the log tag. Only when @running_cloudfunctions is true will
-        # we try to match log tags against Cloud Functions tag regexp when
-        # processing log entries.
-        @running_cloudfunctions = true
-        # Fetch this info and store it to avoid recurring metadata server calls.
-        @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
+      # Retrieve monitored resource.
+      #
+      # TODO(qingling128): After Metadata Agent support is added, try retrieving
+      # the monitored resource from Metadata Agent first.
+      @resource = determine_agent_level_monitored_resource_via_legacy
+
+      # Set regexp that we should match tags against later on. Using a list
+      # instead of a map to ensure order. For example, tags will be matched
+      # against Cloud Functions first, then GKE.
+      @tag_regexp_list = []
+      if @resource.type == CONTAINER_CONSTANTS[:resource_type]
+        # We only support Cloud Functions logs for GKE right now.
+        if fetch_gce_metadata('instance/attributes/'
+                             ).split.include?('gcf_region')
+          # Fetch this info and store it to avoid recurring
+          # metadata server calls.
+          @gcf_region = fetch_gce_metadata('instance/attributes/gcf_region')
+          @tag_regexp_list << [
+            CLOUDFUNCTIONS_CONSTANTS[:resource_type],
+            @compiled_cloudfunctions_tag_regexp
+          ]
+        end
+        @tag_regexp_list << [
+          CONTAINER_CONSTANTS[:resource_type], @compiled_kubernetes_tag_regexp
+        ]
       end
 
       # Determine the common labels that should be added to all log entries
@@ -384,7 +401,7 @@ module Fluent
 
       # Log an informational message containing the Logs viewer URL
       @log.info 'Logs viewer address: https://console.cloud.google.com/logs/',
-                "viewer?project=#{@project_id}&resource=#{@resource.type}/",
+                "viewer?project=#{@project_id}&resource=#{@resource_type}/",
                 "instance_id/#{@vm_id}"
     end
 
@@ -421,16 +438,14 @@ module Fluent
         arr.each do |time, record|
           next unless record.is_a?(Hash)
 
-          resource_type, extracted_resource_labels, extracted_common_labels = \
+          extracted_resource_labels, extracted_common_labels = \
             determine_entry_level_labels(group_resource, record)
           entry_resource = group_resource.dup
-          entry_resource.type = resource_type
           entry_resource.labels.merge!(extracted_resource_labels)
           entry_common_labels = \
             group_common_labels.merge(extracted_common_labels)
 
-          if [CONTAINER_CONSTANTS[:resource_type],
-              DOCKER_CONSTANTS[:resource_type]].include?(entry_resource.type)
+          if entry_resource.type == CONTAINER_CONSTANTS[:resource_type]
             # Save the timestamp if available, then clear it out to allow for
             # determining whether we should parse the log or message field.
             timestamp = record.key?('time') ? record['time'] : nil
@@ -459,13 +474,16 @@ module Fluent
           severity = compute_severity(
             entry_resource.type, record, entry_common_labels)
 
-          # Get fully-qualified trace id for LogEntry "trace" field per config.
-          fq_trace_id = if @keep_trace_key
-                          record[@trace_key]
-                        else
-                          record.delete(@trace_key)
-                        end
-
+          ts_secs = begin
+                      Integer ts_secs
+                    rescue ArgumentError, TypeError
+                      ts_secs
+                    end
+          ts_nanos = begin
+                       Integer ts_nanos
+                     rescue ArgumentError, TypeError
+                       ts_nanos
+                     end
           if @use_grpc
             entry = Google::Logging::V2::LogEntry.new(
               labels: entry_common_labels,
@@ -475,7 +493,6 @@ module Fluent
               ),
               severity: grpc_severity(severity)
             )
-            entry.trace = fq_trace_id if fq_trace_id
             # If "seconds" is null or not an integer, we will omit the timestamp
             # field and defer the decision on how to handle it to the downstream
             # Logging API. If "nanos" is null or not an integer, it will be set
@@ -487,8 +504,6 @@ module Fluent
                 nanos: ts_nanos
               )
             end
-            set_http_request(record, entry)
-            set_payload_grpc(entry_resource.type, record, entry, is_json)
           else
             # Remove the labels if we didn't populate them with anything.
             entry_resource.labels = nil if entry_resource.labels.empty?
@@ -501,8 +516,18 @@ module Fluent
                 nanos: ts_nanos
               }
             )
-            entry.trace = fq_trace_id if fq_trace_id
-            set_http_request(record, entry)
+          end
+
+          # Get fully-qualified trace id for LogEntry "trace" field per config.
+          fq_trace_id = record.delete(@trace_key)
+          entry.trace = fq_trace_id if fq_trace_id
+
+          set_log_entry_fields(record, entry)
+          set_labels(record, entry)
+
+          if @use_grpc
+            set_payload_grpc(entry_resource.type, record, entry, is_json)
+          else
             set_payload(entry_resource.type, record, entry, is_json)
           end
 
@@ -722,20 +747,19 @@ module Fluent
 
     # Set regexp patterns to parse tags and logs.
     def set_regexp_patterns
-      @compiled_kubernetes_tag_regexp = nil
-      if @kubernetes_tag_regexp
-        @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp)
-      end
+      @compiled_kubernetes_tag_regexp = Regexp.new(@kubernetes_tag_regexp) if
+        @kubernetes_tag_regexp
 
-      @cloudfunctions_tag_regexp =
+      @compiled_cloudfunctions_tag_regexp =
         /\.(?<encoded_function_name>.+)\.\d+-[^-]+_default_worker$/
-      @cloudfunctions_log_regexp = /^
+      @compiled_cloudfunctions_log_regexp = /^
         (?:\[(?<severity>.)\])?
         \[(?<timestamp>.{24})\]
         (?:\[(?<execution_id>[^\]]+)\])?
         [ ](?<text>.*)$/x
 
-      @http_latency_regexp = /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
+      @compiled_http_latency_regexp =
+        /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
     end
 
     # Set required variables like @project_id, @vm_id, @vm_name and @zone.
@@ -746,11 +770,11 @@ module Fluent
       set_location
 
       # All metadata parameters must now be set.
-      return if @project_id && @zone && @vm_id
       missing = []
       missing << 'project_id' unless @project_id
       missing << 'zone' unless @zone
       missing << 'vm_id' unless @vm_id
+      return if missing.empty?
       fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
         missing.join(' ')
     end
@@ -767,11 +791,8 @@ module Fluent
     end
 
     # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, check if the response from Metadata Agent includes this info.
-    # 3. If not, try to retrieve it by calling metadata servers directly.
+    # 2. If not, try to retrieve it by calling metadata servers directly.
     def set_vm_id
-      @vm_id ||= @resource.labels['instance_id'] if
-        !@resource.nil? && @resource.labels.key?('instance_id')
       @vm_id ||= fetch_gce_metadata('instance/id') if @platform == Platform::GCE
       @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
     rescue StandardError => e
@@ -779,29 +800,16 @@ module Fluent
     end
 
     # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, check if the response from Metadata Agent includes this info.
-    # 3. If not, try to retrieve it locally.
+    # 2. If not, try to retrieve it locally.
     def set_vm_name
-      @vm_name ||= @resource.labels['instance_name'] if
-        !@resource.nil? && @resource.labels.key?('instance_name')
       @vm_name ||= Socket.gethostname
     rescue StandardError => e
       @log.error 'Failed to obtain vm name: ', error: e
     end
 
     # 1. Return the value if it is explicitly set in the config already.
-    # 2. If not, check if the response from Metadata Agent includes this info.
-    # 3. If not, try to retrieve it locally.
+    # 2. If not, try to retrieve it locally.
     def set_location
-      unless @resource.nil?
-        @zone ||= @resource.labels['location'] if
-          @resource.type == DOCKER_CONSTANTS[:resource_type] &&
-          @resource.labels.key?('location')
-        @zone ||= @resource.labels['zone'] if
-          @platform == Platform::GCE && @resource.labels.key?('zone')
-        @zone ||= @resource.labels['region'] if
-          @platform == Platform::EC2 && @resource.labels.key?('region')
-      end
       # Response format: "projects/<number>/zones/<zone>"
       @zone ||= fetch_gce_metadata('instance/zone').rpartition('/')[2] if
         @platform == Platform::GCE
@@ -813,9 +821,8 @@ module Fluent
 
     # Retrieve monitored resource via the legacy way.
     #
-    # Note: This is just a failover plan if we fail to get metadata from
-    # Metadata Agent. Thus it should be equivalent to what Metadata Agent
-    # returns.
+    # TODO(qingling128): Use this as only a fallback plan after Metadata Agent
+    # support is added.
     def determine_agent_level_monitored_resource_via_legacy
       resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
         labels: {})
@@ -827,108 +834,92 @@ module Fluent
 
     # Determine agent level monitored resource type.
     def determine_agent_level_monitored_resource_type
-      # EC2 instance.
-      return EC2_CONSTANTS[:resource_type] if
-        @platform == Platform::EC2
+      case @platform
+      when Platform::OTHER
+        # Unknown platform will be defaulted to GCE instance.
+        return COMPUTE_CONSTANTS[:resource_type]
 
-      # Unknown platform will be defaulted to GCE instance..
-      return COMPUTE_CONSTANTS[:resource_type] if
-        @platform == Platform::OTHER
-
-      # Resource types determined by @subservice_name config.
-      # Cloud Dataflow.
-      return DATAFLOW_CONSTANTS[:resource_type] if
-        @subservice_name == DATAFLOW_CONSTANTS[:service]
-      # Cloud ML.
-      return ML_CONSTANTS[:resource_type] if
-        @subservice_name == ML_CONSTANTS[:service]
-      # Default back to GCE if invalid value is detected.
-      return COMPUTE_CONSTANTS[:resource_type] if
-        @subservice_name
+      when Platform::EC2
+        return EC2_CONSTANTS[:resource_type]
 
-      # Resource types determined by @detect_subservice config.
-      if @detect_subservice
-        begin
-          attributes = fetch_gce_metadata('instance/attributes/').split
-        rescue StandardError => e
-          @log.error 'Failed to detect subservice: ', error: e
+      when Platform::GCE
+        # Resource types determined by @subservice_name config.
+        return SUBSERVICE_MAP[@subservice_name] if @subservice_name
+
+        # Resource types determined by @detect_subservice config.
+        if @detect_subservice
+          begin
+            attributes = fetch_gce_metadata('instance/attributes/').split.to_set
+            SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
+              return resource_type if attributes.superset?(expected)
+            end
+          rescue StandardError => e
+            @log.error 'Failed to detect subservice: ', error: e
+          end
         end
-        # GAE app.
-        return APPENGINE_CONSTANTS[:resource_type] if
-          attributes.include?('gae_backend_name') &&
-          attributes.include?('gae_backend_version')
-        # GKE container.
-        return CONTAINER_CONSTANTS[:resource_type] if
-          attributes.include?('kube-env')
-        # Cloud Dataproc.
-        return DATAPROC_CONSTANTS[:resource_type] if
-          attributes.include?('dataproc-cluster-uuid') &&
-          attributes.include?('dataproc-cluster-name')
+
+        # GCE instance.
+        return COMPUTE_CONSTANTS[:resource_type]
       end
-      # GCE instance.
-      COMPUTE_CONSTANTS[:resource_type]
     end
 
     # Determine agent level monitored resource labels based on the resource
     # type. Each resource type has its own labels that need to be filled in.
     def determine_agent_level_monitored_resource_labels(type)
-      labels = {}
-
       case type
-
       # GAE app.
       when APPENGINE_CONSTANTS[:resource_type]
-        begin
-          labels['module_id'] = fetch_gce_metadata(
-            'instance/attributes/gae_backend_name')
-          labels['version_id'] = fetch_gce_metadata(
-            'instance/attributes/gae_backend_version')
-        rescue StandardError => e
-          @log.error 'Failed to set monitored resource labels for GAE: ',
-                     error: e
-        end
+        return {
+          'module_id' =>
+            fetch_gce_metadata('instance/attributes/gae_backend_name'),
+          'version_id' =>
+            fetch_gce_metadata('instance/attributes/gae_backend_version')
+        }
 
       # GCE.
       when COMPUTE_CONSTANTS[:resource_type]
-        labels['instance_id'] = @vm_id
-        labels['zone'] = @zone
+        return {
+          'instance_id' => @vm_id,
+          'zone' => @zone
+        }
 
       # GKE container.
       when CONTAINER_CONSTANTS[:resource_type]
-        labels['instance_id'] = @vm_id
-        labels['zone'] = @zone
-        begin
-          raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
-          kube_env = YAML.load(raw_kube_env)
-          labels['cluster_name'] =
-            cluster_name_from_kube_env(kube_env)
-        rescue StandardError => e
-          @log.error 'Failed to set monitored resource labels for GKE: ',
-                     error: e
-        end
+        raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
+        kube_env = YAML.load(raw_kube_env)
+        return {
+          'instance_id' => @vm_id,
+          'zone' => @zone,
+          'cluster_name' => cluster_name_from_kube_env(kube_env)
+        }
 
       # Cloud Dataproc.
       when DATAPROC_CONSTANTS[:resource_type]
-        begin
-          labels['cluster_uuid'] =
-            fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid')
-          labels['cluster_name'] =
-            fetch_gce_metadata('instance/attributes/dataproc-cluster-name')
-          labels['region'] =
+        return {
+          'cluster_uuid' =>
+            fetch_gce_metadata('instance/attributes/dataproc-cluster-uuid'),
+          'cluster_name' =>
+            fetch_gce_metadata('instance/attributes/dataproc-cluster-name'),
+          'region' =>
             fetch_gce_metadata('instance/attributes/dataproc-region')
-        rescue StandardError => e
-          @log.error 'Failed to set monitored resource labels for Cloud ' \
-                     'Dataproc: ', error: e
-        end
+        }
 
       # EC2.
       when EC2_CONSTANTS[:resource_type]
-        labels['instance_id'] = @vm_id
-        labels['region'] = @zone
+        labels = {
+          'instance_id' => @vm_id,
+          'region' => @zone
+        }
         labels['aws_account'] = ec2_metadata['accountId'] if
           ec2_metadata.key?('accountId')
+        return labels
       end
-      labels
+
+      {}
+    rescue StandardError => e
+      @log.error "Failed to set monitored resource labels for #{type}: ",
+                 error: e
+      return {}
     end
 
     # Determine the common labels that should be added to all log entries
@@ -936,38 +927,30 @@ module Fluent
     def determine_agent_level_common_labels
       labels = {}
       # User can specify labels via config. We want to capture those as well.
-      # TODO: Send instance tags as labels as well?
       labels.merge!(@labels) if @labels
 
       case @resource.type
-
-      # GAE app.
-      when APPENGINE_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
-
-      # GCE and GKE container.
+      # GAE, Cloud Dataflow, Cloud Dataproc and Cloud ML.
+      when APPENGINE_CONSTANTS[:resource_type],
+           DATAFLOW_CONSTANTS[:resource_type],
+           DATAPROC_CONSTANTS[:resource_type],
+           ML_CONSTANTS[:resource_type]
+        labels.merge!(
+          "#{COMPUTE_CONSTANTS[:service]}/resource_id" => @vm_id,
+          "#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name,
+          "#{COMPUTE_CONSTANTS[:service]}/zone" => @zone
+        )
+
+      # GCE instance and GKE container.
       when COMPUTE_CONSTANTS[:resource_type],
            CONTAINER_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-
-      # Cloud Dataflow and Cloud Dataproc.
-      when DATAFLOW_CONSTANTS[:resource_type],
-           DATAPROC_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
+        labels.merge!(
+          "#{COMPUTE_CONSTANTS[:service]}/resource_name" => @vm_name)
 
       # EC2.
       when EC2_CONSTANTS[:resource_type]
-        labels["#{EC2_CONSTANTS[:service]}/resource_name"] = @vm_name
-
-      # Cloud ML.
-      when ML_CONSTANTS[:resource_type]
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] = @vm_id
-        labels["#{COMPUTE_CONSTANTS[:service]}/resource_name"] = @vm_name
-        labels["#{COMPUTE_CONSTANTS[:service]}/zone"] = @zone
+        labels.merge!(
+          "#{EC2_CONSTANTS[:service]}/resource_name" => @vm_name)
       end
       labels
     end
@@ -977,14 +960,13 @@ module Fluent
     def determine_group_level_monitored_resource_and_labels(tag)
       # Determine group level monitored resource type. For certain types,
       # extract useful info from the tag and store those in
-      # matched_regexp_group.
-      group_resource_type, matched_regexp_group =
+      # matched_regex_group.
+      group_resource_type, matched_regex_group =
         determine_group_level_monitored_resource_type(tag)
 
       # Determine group level monitored resource labels and common labels.
-      group_resource_type, group_resource_labels, group_common_labels = \
-        determine_group_level_labels_and_adjust_type(
-          group_resource_type, matched_regexp_group)
+      group_resource_labels, group_common_labels =
+        determine_group_level_labels(group_resource_type, matched_regex_group)
 
       group_resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
         type: group_resource_type,
@@ -1002,202 +984,84 @@ module Fluent
 
     # Determine group level monitored resource type shared by a collection of
     # entries.
-    # Returns the resource type and tag regexp matched groups. The matched
-    # groups only apply to some resource types. Return nil if not applicable or
-    # if there is no match.
+    # Return the resource type and tag regexp matched groups. The matched groups
+    # only apply to some resource types. Return nil if not applicable or if
+    # there is no match.
     def determine_group_level_monitored_resource_type(tag)
-      # Match tag against Cloud Functions format.
-      if @running_cloudfunctions
-        matched_regexp_group = @cloudfunctions_tag_regexp.match(tag)
-        return [CLOUDFUNCTIONS_CONSTANTS[:resource_type],
-                matched_regexp_group] if matched_regexp_group
+      @tag_regexp_list.each do |derived_type, tag_regexp|
+        matched_regex_group = tag_regexp.match(tag)
+        return [derived_type, matched_regex_group] if
+          matched_regex_group
       end
-
-      # Match tag against Docker container stderr / stdout log format and
-      # Docker container application log format.
-      matched_regexp_group =
-        # Format: "container.<container_id>.<container_name>"
-        @dockercontainer_tag_regexp.match(tag) ||
-        # Format: "application-container.<container_name>.<additional_tag>"
-        @dockercontainer_tag_with_application_regexp.match(tag)
-      return [DOCKER_CONSTANTS[:resource_type], matched_regexp_group] if
-        matched_regexp_group
-
-      # Match tag against GKE Container format.
-      if @resource.type == CONTAINER_CONSTANTS[:resource_type] &&
-         @compiled_kubernetes_tag_regexp
-        # Container logs in Kubernetes are tagged based on where they came from,
-        # so we can extract useful metadata from the tag. Do this here to avoid
-        # having to repeat it for each record.
-        matched_regexp_group = @compiled_kubernetes_tag_regexp.match(tag)
-        return [@resource.type, matched_regexp_group] if matched_regexp_group
-      end
-
-      # Otherwise, return the original type.
       [@resource.type, nil]
     end
 
     # Determine group level monitored resource labels and common labels. These
-    # labels will be shared by a collection of entries. In certain cases, we
-    # might also adjust the resource type.
-    def determine_group_level_labels_and_adjust_type(group_resource_type,
-                                                     matched_regexp_group)
+    # labels will be shared by a collection of entries.
+    def determine_group_level_labels(group_resource_type, matched_regex_group)
       group_resource_labels = @resource.labels.dup
       group_common_labels = @common_labels.dup
 
       case group_resource_type
-
       # Cloud Functions.
       when CLOUDFUNCTIONS_CONSTANTS[:resource_type]
-        group_resource_labels['region'] = @gcf_region
-        group_resource_labels['function_name'] =
-          decode_cloudfunctions_function_name(
-            matched_regexp_group['encoded_function_name'])
+        group_resource_labels.merge!(
+          'region' => @gcf_region,
+          'function_name' => decode_cloudfunctions_function_name(
+            matched_regex_group['encoded_function_name'])
+        )
+
         instance_id = group_resource_labels.delete('instance_id')
-        group_common_labels["#{CONTAINER_CONSTANTS[:service]}/cluster_name"] =
-          group_resource_labels.delete('cluster_name')
-        group_common_labels["#{CONTAINER_CONSTANTS[:service]}/instance_id"] =
-          instance_id
-        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
-          instance_id
-        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/zone"] =
-          group_resource_labels.delete('zone')
+        group_common_labels.merge!(
+          "#{CONTAINER_CONSTANTS[:service]}/instance_id" => instance_id,
+          "#{COMPUTE_CONSTANTS[:service]}/resource_id" => instance_id,
+          "#{CONTAINER_CONSTANTS[:service]}/cluster_name" =>
+            group_resource_labels.delete('cluster_name'),
+          "#{COMPUTE_CONSTANTS[:service]}/zone" =>
+            group_resource_labels.delete('zone')
+        )
 
       # GKE container.
       when CONTAINER_CONSTANTS[:resource_type]
-        if matched_regexp_group
-          group_resource_labels['container_name'] =
-            matched_regexp_group['container_name']
-          # The kubernetes_tag_regexp is poorly named. 'namespace_name' is in
-          # fact 'namespace_id'. 'pod_name' is in fact 'pod_id'.
-          group_resource_labels['namespace_id'] =
-            matched_regexp_group['namespace_name']
-          group_resource_labels['pod_id'] =
-            matched_regexp_group['pod_name']
-          %w(namespace_name pod_name).each do |field|
-            group_common_labels["#{CONTAINER_CONSTANTS[:service]}/#{field}"] =
-              matched_regexp_group[field]
-          end
-        end
-
-      # Docker container.
-      when DOCKER_CONSTANTS[:resource_type]
-        # For Docker container stderr / stdout logs generated by Docker Fluentd
-        # Logging Driver, tags are in the format of "container.<container_id>.
-        # <container_name>", thus they include 'container_id' info.
-        # For logs generated by applications running in Docker containers,
-        # tags are in the format of "application-container.<container_name>.
-        # <additional_tag>", thus 'container_id' info is unknown yet.
-        # 'container_name' info on the other hand is always available.
-        container_id = matched_regexp_group['container_id'] if
-          matched_regexp_group.names.include? 'container_id'
-        container_name = matched_regexp_group['container_name']
-
-        if @enable_metadata_agent
-          # Call Metadata Agent with "container.<container_id>" or
-          # "application-container.<container_name>" as the locally-unique key
-          # to retrieve monitored resource. This should be different from the
-          # original @resource value that got initiated when the agent starts up
-          # because that one is always at the VM level.
-          if container_id
-            locally_unique_id = "container.#{container_id}"
-          else
-            locally_unique_id = "containerName.#{container_name}"
-          end
-          retrieved_resource = call_metadata_agent_for_monitored_resource(
-            locally_unique_id)
-        end
-
-        if !retrieved_resource.nil?
-          # If we successfully get a monitored resource from Metadata Agent,
-          # use this one instead of the original instance monitored resource.
-          group_resource_labels = retrieved_resource.labels.dup
-          @log.debug 'Retrieved monitored resource from Metadata Agent: ' \
-                     "#{retrieved_resource.inspect}."
-        else
-          # If Metadata Agent is not enabled, or we failed to get a monitored
-          # resource, we need to have some backup plan.
-          @log.debug 'Metadata Agent not enabled or failed to retrieve ' \
-                     'docker container monitored resource from Metadata ' \
-                     'Agent.'
-
-          # 1. Check if 'container_id' is set already. It should be available
-          #    for stdout / stderr). If so, use that.
-          # 2. If not, call Docker Remote API to retrieve the container ID from
-          #    container name, but only if @call_docker_api_locally is true.
-          container_id ||= retrieve_container_id_by_name_locally(
-            container_name) if @call_docker_api_locally
-          unless container_id
-            @log.debug 'No docker container id retrieved. Falling back to
-                        instance monitored resource.'
-            # If a container id is not available, fall back to the instance
-            # monitored resource.
-            return [COMPUTE_CONSTANTS[:resource_type], group_resource_labels,
-                    group_common_labels]
-          end
-          group_resource_labels['container_id'] = container_id
-          # 'zone' for GCP and 'region' for EC2 must have been set at this
-          # point. Rename them to 'location'.
-          group_resource_labels['location'] = @zone
-          if @platform == Platform::EC2
-            group_resource_labels.delete('region')
-          else
-            group_resource_labels.delete('zone')
-          end
-          # vm id info should be reported as a metadata label instead.
-          group_resource_labels.delete('instance_id')
-
+        if matched_regex_group
+          # We only expect one occurrence of each key in the match group.
+          resource_labels_candidates =
+            matched_regex_group.names.zip(matched_regex_group.captures).to_h
+          common_labels_candidates =
+            resource_labels_candidates.dup
+          group_resource_labels.merge!(
+            delete_and_extract_labels(
+              resource_labels_candidates,
+              # The kubernetes_tag_regexp is poorly named. 'namespace_name' is
+              # in fact 'namespace_id'. 'pod_name' is in fact 'pod_id'.
+              # TODO(qingling128): Figure out how to put this map into
+              # constants like CONTAINER_CONSTANTS[:extra_resource_labels].
+              'container_name' => 'container_name',
+              'namespace_name' => 'namespace_id',
+              'pod_name' => 'pod_id'))
+
+          group_common_labels.merge!(
+            delete_and_extract_labels(
+              common_labels_candidates,
+              CONTAINER_CONSTANTS[:extra_common_labels]
+                .map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
         end
-        # Set metadata labels.
-        group_common_labels["#{DOCKER_CONSTANTS[:service]}/container_name"] =
-          matched_regexp_group['container_name']
-        group_common_labels["#{COMPUTE_CONSTANTS[:service]}/resource_id"] =
-          @vm_id
       end
 
-      [group_resource_type, group_resource_labels, group_common_labels]
+      [group_resource_labels, group_common_labels]
     end
 
     # Extract entry resource and common labels that should be applied to
     # individual entries from the group resource.
     def determine_entry_level_labels(group_resource, record)
-      resource_type = group_resource.type
       resource_labels = {}
       common_labels = {}
 
-      # The format of the locally unique key varies by monitored resource.
-      #
-      # Docker container:
-      #   "container.<container_id>"
-      #   "containerName.<container_name>"
-      # GKE container:
-      #   "gke_containerName.<namespace_id>.<pod_name>.<container_name>"
-      if @enable_metadata_agent && record.key?(LOCALLY_UNIQUE_ID_LABEL_NAME)
-        locally_unique_id = record.delete(LOCALLY_UNIQUE_ID_LABEL_NAME)
-        @log.debug 'Calling metadata agent with locally unique id: ' \
-                  "#{locally_unique_id}."
-        retrieved_resource = call_metadata_agent_for_monitored_resource(
-          locally_unique_id)
-        @log.debug 'Retrieved monitored resource from metadata agent: ' \
-                  "#{retrieved_resource.inspect}."
-        unless retrieved_resource.nil?
-          resource_type = retrieved_resource.type
-          # Temporarily renaming 'gke_container' to 'container'.
-          resource_type = 'container' if resource_type == 'gke_container'
-          # If we successfully get a monitored resource from Metadata Agent,
-          # use this one instead of the original VM-level monitored resource.
-          resource_labels = retrieved_resource.labels.dup
-          @log.debug 'Retrieved gke_container monitored resource from' \
-                     'Stackdriver Metadata agent: ' \
-                     "#{retrieved_resource.inspect}."
-        end
-      end
-
       # Cloud Functions.
-      if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
+      if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
          record.key?('log')
         @cloudfunctions_log_match =
-          @cloudfunctions_log_regexp.match(record['log'])
+          @compiled_cloudfunctions_log_regexp.match(record['log'])
         common_labels['execution_id'] =
           @cloudfunctions_log_match['execution_id'] if \
             @cloudfunctions_log_match &&
@@ -1205,7 +1069,7 @@ module Fluent
       end
 
       # GKE containers.
-      if resource_type == CONTAINER_CONSTANTS[:resource_type]
+      if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
         # Move the stdout/stderr annotation from the record into a label.
         common_labels.merge!(
           delete_and_extract_labels(
@@ -1217,17 +1081,18 @@ module Fluent
         if record.key?('kubernetes')
           resource_labels.merge!(
             delete_and_extract_labels(
-              record['kubernetes'], %w(namespace_id pod_id container_name)
+              record['kubernetes'], CONTAINER_CONSTANTS[:extra_resource_labels]
                 .map { |l| [l, l] }.to_h))
           common_labels.merge!(
             delete_and_extract_labels(
-              record['kubernetes'], %w(namespace_name pod_name)
+              record['kubernetes'], CONTAINER_CONSTANTS[:extra_common_labels]
                 .map { |l| [l, "#{CONTAINER_CONSTANTS[:service]}/#{l}"] }.to_h))
           # Prepend label/ to all user-defined labels' keys.
           if record['kubernetes'].key?('labels')
-            record['kubernetes']['labels'].each do |key, value|
-              common_labels["label/#{key}"] = value
-            end
+            common_labels.merge!(
+              delete_and_extract_labels(
+                record['kubernetes']['labels'], record['kubernetes']['labels']
+                  .map { |key, _| [key, "label/#{key}"] }.to_h))
           end
           # We've explicitly consumed all the fields we care about -- don't
           # litter the log entries with the remaining fields that the kubernetes
@@ -1237,96 +1102,24 @@ module Fluent
         end
       end
 
-      # Docker containers.
-      if resource_type == DOCKER_CONSTANTS[:resource_type]
-        # For logs coming from Docker Fluentd Logging Driver, the log record
-        # has 4 fields: 'container_id', 'container_name', 'source' and 'log'.
-        # Extract 'container_id', 'container_name' and 'source' from json
-        # record, set corresponding labels, and remove these fields from record.
-        {
-          'container_name' => 'container_name',
-          'source' => 'stream'
-        }.each do |field_name, label_name|
-          common_labels.merge!(
-            delete_and_extract_labels(
-              record,
-              field_name => "#{DOCKER_CONSTANTS[:service]}/#{label_name}"
-            )
-          )
-        end
-        resource_labels.merge!(
-          delete_and_extract_labels(record, 'container_id' => 'container_id'))
-      end
-
       # If the name of a field in the record is present in the @label_map
       # configured by users, report its value as a label and do not send that
       # field as part of the payload.
       common_labels.merge!(delete_and_extract_labels(record, @label_map))
 
-      # Cloud Dataflow.
-      # These labels can be set via configuring 'labels' or 'label_map'.
-      # Report them as monitored resource labels instead of common labels.
-      if group_resource.type == DATAFLOW_CONSTANTS[:resource_type]
-        resource_labels.merge!(
-          delete_and_extract_labels(
-            common_labels, %w(region job_name job_id step_id)
-              .map { |l| ["#{DATAFLOW_CONSTANTS[:service]}/#{l}", l] }.to_h))
-      end
-
-      # Cloud ML.
+      # Cloud Dataflow and Cloud ML.
       # These labels can be set via configuring 'labels' or 'label_map'.
       # Report them as monitored resource labels instead of common labels.
-      if group_resource.type == ML_CONSTANTS[:resource_type]
+      # e.g. "dataflow.googleapis.com/job_id" => "job_id"
+      [DATAFLOW_CONSTANTS, ML_CONSTANTS].each do |service_constants|
+        next unless group_resource.type == service_constants[:resource_type]
         resource_labels.merge!(
           delete_and_extract_labels(
-            common_labels, %w(job_id task_name)
-              .map { |l| ["#{ML_CONSTANTS[:service]}/#{l}", l] }.to_h))
+            common_labels, service_constants[:extra_common_labels]
+              .map { |l| ["#{service_constants[:service]}/#{l}", l] }.to_h))
       end
 
-      [resource_type, resource_labels, common_labels]
-    end
-
-    # Call Metadata Agent to get monitored resource information and parse
-    # response to Google::Api::MonitoredResource.
-    def call_metadata_agent_for_monitored_resource(unique_key)
-      response = call_metadata_agent("monitoredResource/#{unique_key}")
-      return nil if response.nil?
-      begin
-        resource = Google::Api::MonitoredResource.decode_json(response.to_json)
-      rescue Google::Protobuf::ParseError, ArgumentError => e
-        @log.error 'Error paring monitored resource from Metadata Agent. ' \
-                   "response: #{response.inspect}", error: e
-        return nil
-      end
-
-      # TODO(qingling128): Use Google::Api::MonitoredResource directly after we
-      # upgrade gRPC version to include the fix for the protobuf map
-      # corruption issue.
-      Google::Apis::LoggingV2beta1::MonitoredResource.new(
-        type: resource.type,
-        labels: resource.labels.to_h
-      )
-    end
-
-    # Call Metadata Agent and parse response to json. Return nil in case of any
-    # error / failure.
-    def call_metadata_agent(path)
-      url = "#{@metadata_agent_url}/#{path}"
-      @log.debug("Calling Metadata Agent: #{url}")
-      open(url) do |f|
-        response = f.read
-        parsed_hash = parse_json_or_nil(response)
-        if parsed_hash.nil?
-          @log.error 'Response from Metadata Agent is not in valid json ' \
-                     "format: '#{response.inspect}'."
-          return nil
-        end
-        @log.debug "Response from Metadata Agent: #{parsed_hash}"
-        return parsed_hash
-      end
-    rescue StandardError => e
-      @log.error 'Error calling Metadata Agent.', error: e
-      return nil
+      [resource_labels, common_labels]
     end
 
     # TODO: This functionality should eventually be available in another
@@ -1364,30 +1157,6 @@ module Fluent
       end
     end
 
-    # Calling Docker Remote API to get container id by name.
-    def retrieve_container_id_by_name_locally(container_name)
-      response = Excon.get(
-        "unix:///containers/#{container_name}/json",
-        socket: @docker_remote_api_socket_path)
-      @log.debug "Response from Docker API with name '#{container_name}': " \
-                 "#{response.inspect}."
-      return parse_container_id_from_docker_api_response(response)
-    rescue StandardError => e
-      @log.error 'Error calling Docker API to get container id.', error: e
-      return nil
-    end
-
-    # Parse the container id from Docker Remote API response.
-    # TODO(qingling128) Add a config for Docker API version to support parsing
-    # different versions of Docker Remote API when the format varies.
-    def parse_container_id_from_docker_api_response(response)
-      JSON.parse(response.data[:body])['Id']
-    rescue StandardError => e
-      @log.error 'Error parsing Docker API response to get container id.',
-                 error: e
-      return nil
-    end
-
     def cluster_name_from_kube_env(kube_env)
       return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
       instance_prefix = kube_env['INSTANCE_PREFIX']
@@ -1458,14 +1227,9 @@ module Fluent
         end
       elsif record.key?('severity')
         return parse_severity(record.delete('severity'))
-      elsif [CONTAINER_CONSTANTS[:resource_type],
-             DOCKER_CONSTANTS[:resource_type]].include?(resource_type)
-        stream = entry_common_labels[
-          "#{CONTAINER_CONSTANTS[:service]}/stream"] if
-           resource_type == CONTAINER_CONSTANTS[:resource_type]
-        stream = entry_common_labels[
-          "#{DOCKER_CONSTANTS[:service]}/stream"] if
-           resource_type == DOCKER_CONSTANTS[:resource_type]
+      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
+            entry_common_labels.key?("#{CONTAINER_CONSTANTS[:service]}/stream")
+        stream = entry_common_labels["#{CONTAINER_CONSTANTS[:service]}/stream"]
         if stream == 'stdout'
           return 'INFO'
         elsif stream == 'stderr'
@@ -1478,71 +1242,61 @@ module Fluent
       end
     end
 
-    NANOS_IN_A_SECOND = 1000 * 1000 * 1000
+    def set_log_entry_fields(record, entry)
+      LOG_ENTRY_FIELDS_MAP.each do |field_name, config|
+        payload_key, subfields, grpc_class, non_grpc_class = config
+        begin
+          payload_key = instance_variable_get(payload_key)
+          fields = record[payload_key]
+          next unless fields.is_a?(Hash)
+
+          extracted_subfields = subfields.each_with_object({}) \
+            do |(original_key, destination_key, cast_fn), extracted_fields|
+            value = fields.delete(original_key)
+            next if value.nil?
+            begin
+              casted_value = send(cast_fn, value)
+            rescue TypeError
+              @log.error "Failed to #{cast_fn} for #{field_name}." \
+                         "#{original_key} with value #{value.inspect}.", err
+              next
+            end
+            next if casted_value.nil?
+            extracted_fields[destination_key] = casted_value
+          end
+
+          next unless extracted_subfields
 
-    def set_http_request(record, entry)
-      return nil unless record['httpRequest'].is_a?(Hash)
-      input = record['httpRequest']
-      if @use_grpc
-        output = Google::Logging::Type::HttpRequest.new
-      else
-        output = Google::Apis::LoggingV2beta1::HttpRequest.new
-      end
-      # We need to delete each field from 'httpRequest' even if its value is
-      # nil. However we do not want to assign this nil value to the constructed
-      # json or proto.
-      request_method = input.delete('requestMethod')
-      output.request_method = request_method unless request_method.nil?
-      request_url = input.delete('requestUrl')
-      output.request_url = request_url unless request_url.nil?
-      request_size = input.delete('requestSize')
-      output.request_size = request_size.to_i unless request_size.nil?
-      status = input.delete('status')
-      output.status = status.to_i unless status.nil?
-      response_size = input.delete('responseSize')
-      output.response_size = response_size.to_i unless response_size.nil?
-      user_agent = input.delete('userAgent')
-      output.user_agent = user_agent unless user_agent.nil?
-      remote_ip = input.delete('remoteIp')
-      output.remote_ip = remote_ip unless remote_ip.nil?
-      referer = input.delete('referer')
-      output.referer = referer unless referer.nil?
-      cache_hit = input.delete('cacheHit')
-      output.cache_hit = cache_hit unless cache_hit.nil?
-      cache_validated_with_origin_server = \
-        input.delete('cacheValidatedWithOriginServer')
-      output.cache_validated_with_origin_server = \
-        cache_validated_with_origin_server \
-        unless cache_validated_with_origin_server.nil?
-
-      latency = input.delete('latency')
-      unless latency.nil?
-        # Parse latency. If no valid format is detected, skip setting latency.
-        # Format: whitespace (optional) + integer + point & decimal (optional)
-        #       + whitespace (optional) + "s" + whitespace (optional)
-        # e.g.: "1.42 s"
-        match = @http_latency_regexp.match(latency)
-        if match
-          # Split the integer and decimal parts in order to calculate seconds
-          # and nanos.
-          latency_seconds = match['seconds'].to_i
-          latency_nanos = (match['decimal'].to_f * NANOS_IN_A_SECOND).round
           if @use_grpc
-            output.latency = Google::Protobuf::Duration.new(
-              seconds: latency_seconds,
-              nanos: latency_nanos
-            )
+            output = Object.const_get(grpc_class).new
           else
-            output.latency = {
-              seconds: latency_seconds,
-              nanos: latency_nanos
-            }.delete_if { |_, v| v == 0 }
+            output = Object.const_get(non_grpc_class).new
           end
+          extracted_subfields.each do |key, value|
+            output.send("#{key}=", value)
+          end
+
+          record.delete(payload_key) if fields.empty?
+
+          entry.send("#{field_name}=", output)
+        rescue StandardError => err
+          @log.error "Failed to set log entry field for #{field_name}.", err
+        end
+      end
+    end
+
+    def set_labels(record, entry)
+      record_labels = record[@labels_key]
+      return nil unless record_labels.is_a?(Hash)
+
+      record_labels.each do |key, value|
+        unless entry.labels.key?(key)
+          record_labels.delete(key)
+          entry.labels[key] = value
         end
       end
 
-      record.delete('httpRequest') if input.empty?
-      entry.http_request = output
+      record.delete(@labels_key) if record_labels.empty?
     end
 
     # Values permitted by the API for 'severity' (which is an enum).
@@ -1640,6 +1394,45 @@ module Fluent
       severity
     end
 
+    def parse_string(value)
+      value.to_s
+    end
+
+    def parse_int(value)
+      value.to_i
+    end
+
+    def parse_bool(value)
+      [true, 'true', 1].include?(value)
+    end
+
+    def parse_latency(latency)
+      # Parse latency.
+      # If no valid format is detected, return nil so we can later skip
+      # setting latency.
+      # Format: whitespace (opt.) + integer + point & decimal (opt.)
+      #       + whitespace (opt.) + "s" + whitespace (opt.)
+      # e.g.: "1.42 s"
+      match = @compiled_http_latency_regexp.match(latency)
+      return nil unless match
+
+      # Split the integer and decimal parts in order to calculate
+      # seconds and nanos.
+      seconds = match['seconds'].to_i
+      nanos = (match['decimal'].to_f * 1000 * 1000 * 1000).round
+      if @use_grpc
+        return Google::Protobuf::Duration.new(
+          seconds: seconds,
+          nanos: nanos
+        )
+      else
+        return {
+          seconds: seconds,
+          nanos: nanos
+        }.delete_if { |_, v| v == 0 }
+      end
+    end
+
     def decode_cloudfunctions_function_name(function_name)
       function_name.gsub(/c\.[a-z]/) { |s| s.upcase[-1] }
         .gsub('u.u', '_').gsub('d.d', '$').gsub('a.a', '@').gsub('p.p', '.')
@@ -1664,16 +1457,16 @@ module Fluent
     end
 
     # For every original_label => new_label pair in the label_map, delete the
-    # original_label from the original_resource map if it exists, and extract
-    # the value to form a map with the new_label as the key.
-    def delete_and_extract_labels(original_resource, label_map)
+    # original_label from the hash map if it exists, and extract the value to
+    # form a map with the new_label as the key.
+    def delete_and_extract_labels(hash, label_map)
       return {} if label_map.nil? || !label_map.is_a?(Hash) ||
-                   original_resource.nil? || !original_resource.is_a?(Hash)
+                   hash.nil? || !hash.is_a?(Hash)
       label_map.each_with_object({}) \
         do |(original_label, new_label), extracted_labels|
-        extracted_labels[new_label] = convert_to_utf8(
-          original_resource.delete(original_label).to_s) if
-            original_resource.key?(original_label)
+        extracted_labels[new_label] =
+          convert_to_utf8(hash.delete(original_label).to_s) if
+            hash.key?(original_label)
       end
     end
 
@@ -1692,8 +1485,7 @@ module Fluent
         entry.text_payload = record['log']
       elsif is_json
         entry.json_payload = record
-      elsif [CONTAINER_CONSTANTS[:resource_type],
-             DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
+      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
             record.key?('log')
         entry.text_payload = record['log']
       elsif record.size == 1 && record.key?('message')
@@ -1763,8 +1555,7 @@ module Fluent
         entry.text_payload = convert_to_utf8(record['log'])
       elsif is_json
         entry.json_payload = struct_from_ruby(record)
-      elsif [CONTAINER_CONSTANTS[:resource_type],
-             DOCKER_CONSTANTS[:resource_type]].include?(resource_type) &&
+      elsif resource_type == CONTAINER_CONSTANTS[:resource_type] &&
             record.key?('log')
         entry.text_payload = convert_to_utf8(record['log'])
       elsif record.size == 1 && record.key?('message')