fluent-plugin-google-cloud 0.6.18.pre.1 → 0.6.18.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96a5943e560b95ee0e8427efb2eef06d17218f75
-  data.tar.gz: 944c1f9d6391f286a00024ad69a433725b82bdd5
+  metadata.gz: 22363c75dc8601301e8ce3e6a759b7f83eb02c45
+  data.tar.gz: db92ec8ea70313b48bbf654aea9a5cc88998b8f2
 SHA512:
-  metadata.gz: 75e217bea7703bddf67cba284afb6cab2c16b70e18ae1f7e87e6e8e0e7a8fdfd434c1a6dd8be91f299f5e6dac9643ebb9d2c240ca6369f744b9837bd13d9ec59
-  data.tar.gz: b3f7a0aa1b2df536bff44bdcb1272a4d8a70b7ce4c8737ad79f28b1615945b698a9683b7d291c22566a22f9ffa98d382a82a518c3238a28cdce5c7b6bd605f8f
+  metadata.gz: faca80ef319660243f9ffb4607e19a60b1400f12eb4efce959b80de5e77e8c519530403869fc1725f9cd53afb54fe230739f57d02f2bcf950d7f83855c642084
+  data.tar.gz: 77e7f853e24625b14eb18a55a7459d35a2135873b200d58ae537beefef937bb352552d76adbdc21ae92a8dd9a0e18f3cccb6d2a52b0a60d5a14d5d9d5ae3996f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.17)
+    fluent-plugin-google-cloud (0.6.18.pre.2)
       fluentd (~> 0.10)
       google-api-client (~> 0.17)
       google-cloud-logging (~> 1.3, >= 1.3.2)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      =
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.18.pre.1'
+  gem.version       = '0.6.18.pre.2'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -226,7 +226,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'.freeze
-    PLUGIN_VERSION = '0.6.18.pre.1'.freeze
+    PLUGIN_VERSION = '0.6.18.pre.2'.freeze
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'.freeze
@@ -980,16 +980,6 @@ module Fluent
 
       @compiled_http_latency_regexp =
         /^\s*(?<seconds>\d+)(?<decimal>\.\d+)?\s*s\s*$/
-
-      # TODO(qingling128): Temporary fallback for metadata agent restarts.
-      @compiled_k8s_container_local_resource_id_regexp = /^
-        (?<resource_type>k8s_container)
-        \.(?<namespace_name>[0-9a-z-]+)
-        \.(?<pod_name>[0-9a-z-.]+)
-        \.(?<container_name>[0-9a-z-]+)$/x
-      @compiled_k8s_node_local_resource_id_regexp = /^
-        (?<resource_type>k8s_node)
-        \.(?<node_name>[0-9a-z-]+)$/x
     end
 
     # Set required variables like @project_id, @vm_id, @vm_name and @zone.
@@ -1257,10 +1247,8 @@ module Fluent
         else
           # TODO(qingling128): This entire else clause is temporary before we
           # implement buffering and caching.
-          @log.warn('Failing to retrieve monitored resource from Metadata' \
-                     " Agent with local_resource_id #{local_resource_id}." \
-                     ' Trying to construct resource locally if it is a k8s' \
-                     ' resource.')
+          @log.warn('Failed to retrieve monitored resource from Metadata' \
+                     " Agent with local_resource_id #{local_resource_id}.")
           constructed_k8s_resource = construct_k8s_resource_locally(
             local_resource_id)
           resource = constructed_k8s_resource if constructed_k8s_resource
@@ -1314,12 +1302,15 @@ module Fluent
       # Docker container.
       # TODO(qingling128): Remove this logic once the resource is retrieved at a
       # proper time (b/65175256).
-      when DOCKER_CONSTANTS[:resource_type],
-           # TODO(qingling128): Temporary fallback for metadata agent restarts.
-           K8S_CONTAINER_CONSTANTS[:resource_type],
-           K8S_NODE_CONSTANTS[:resource_type]
+      when DOCKER_CONSTANTS[:resource_type]
+        common_labels.delete("#{COMPUTE_CONSTANTS[:service]}/resource_name")
 
+      # TODO(qingling128): Temporary fallback for metadata agent restarts.
+      # K8s resources.
+      when K8S_CONTAINER_CONSTANTS[:resource_type],
+           K8S_NODE_CONSTANTS[:resource_type]
         common_labels.delete("#{COMPUTE_CONSTANTS[:service]}/resource_name")
+
       end
 
       # Cloud Dataflow and Cloud ML.
@@ -2177,85 +2168,55 @@ module Fluent
 
     # Construct monitored resource locally for k8s resources.
     def construct_k8s_resource_locally(local_resource_id)
-      matched_regexp_group = nil
-      # Regex match the local_resource_id to make sure we only do
-      # this for 'k8s_container' and 'k8s_node'.
-      [@compiled_k8s_node_local_resource_id_regexp,
-       @compiled_k8s_container_local_resource_id_regexp].each do |id_regexp|
-        matched_regexp_group ||= id_regexp.match(local_resource_id)
-      end
-      return unless matched_regexp_group
-      case matched_regexp_group['resource_type']
-      when 'k8s_container'
-        @log.debug('Detected local_resource_id in k8s_container format.' \
-                   "Constructing resource locally with #{local_resource_id}.")
-        return construct_k8s_container_resource(matched_regexp_group)
-      when 'k8s_node'
-        @log.debug('Detected local_resource_id in k8s_node format.' \
-                   "Constructing resource locally with #{local_resource_id}.")
-        return construct_k8s_node_resource(matched_regexp_group)
-      end
-    end
+      return unless
+        /^
+          (?<resource_type>k8s_container)
+          \.(?<namespace_name>[0-9a-z-]+)
+          \.(?<pod_name>[.0-9a-z-]+)
+          \.(?<container_name>[0-9a-z-]+)$/x =~ local_resource_id ||
+        /^
+          (?<resource_type>k8s_node)
+          \.(?<node_name>[0-9a-z-]+)$/x =~ local_resource_id
 
-    # Construct the k8s_container monitored resource locally. In case of any
-    # exception, return nil.
-    def construct_k8s_container_resource(matched_regexp_group)
-      constructed_resource = Google::Apis::LoggingV2::MonitoredResource.new(
-        type: 'k8s_container',
-        labels: {
-          'namespace_name' => matched_regexp_group['namespace_name'],
-          'pod_name' => matched_regexp_group['pod_name'],
-          'container_name' => matched_regexp_group['container_name'],
-          'cluster_name' => retrieve_k8s_cluster_name,
-          'location' => retrieve_k8s_location
+      begin
+        @k8s_cluster_name ||= fetch_gce_metadata(
+          'instance/attributes/cluster-name')
+        @k8s_location ||= fetch_gce_metadata(
+          'instance/attributes/cluster-location')
+      rescue StandardError => e
+        @log.error 'Failed to retrieve k8s cluster name and location.', \
+                   error: e
+      end
+      case resource_type
+      when K8S_CONTAINER_CONSTANTS[:resource_type]
+        labels = {
+          'namespace_name' => namespace_name,
+          'pod_name' => pod_name,
+          'container_name' => container_name,
+          'cluster_name' => @k8s_cluster_name,
+          'location' => @k8s_location
         }
-      )
-      @log.debug('Constructed k8s_container resource locally:\n' \
-                 "#{constructed_resource.inspect}")
-      constructed_resource
-    rescue StandardError => e
-      @log.error 'Failed to construct "k8s_container" resource locally.' \
-                 ' Falling back to writing logs against instance resource.',
-                 error: e
-      # Fall back to legacy gke resource or default instance resource.
-      return
-    end
-
-    # Construct the k8s_node monitored resource locally. In case of any
-    # exception, return nil.
-    def construct_k8s_node_resource(matched_regexp_group)
-      constructed_resource = Google::Apis::LoggingV2::MonitoredResource.new(
-        type: 'k8s_node',
-        labels: {
-          'node_name' => matched_regexp_group['node_name'],
-          'cluster_name' => retrieve_k8s_cluster_name,
-          'location' => retrieve_k8s_location
+        fallback_resource = GKE_CONSTANTS[:resource_type]
+      when K8S_NODE_CONSTANTS[:resource_type]
+        labels = {
+          'node_name' => node_name,
+          'cluster_name' => @k8s_cluster_name,
+          'location' => @k8s_location
         }
-      )
-      @log.debug('Constructed k8s_node resource locally:\n' \
-                 "#{constructed_resource.inspect}")
+        fallback_resource = COMPUTE_CONSTANTS[:resource_type]
+      end
+      unless @k8s_cluster_name && @k8s_location
+        @log.error "Failed to construct #{resource_type} resource locally." \
+                   ' Falling back to writing logs against' \
+                   " #{fallback_resource} resource.", error: e
+        return
+      end
+      constructed_resource = Google::Apis::LoggingV2::MonitoredResource.new(
+        type: resource_type,
+        labels: labels)
+      @log.debug("Constructed #{resource_type} resource locally:\n" \
+               "#{constructed_resource.inspect}")
       constructed_resource
-    rescue StandardError => e
-      @log.error 'Failed to construct "k8s_node" resource locally. Falling' \
-                 ' back to writing logs against instance resource.', error: e
-      # Fall back to default instance resource.
-      return
-    end
-
-    # Only retrieve cluster name once since it's not changing for this
-    # agent instance's life time.
-    # Throw an error if failed to retrieve the cluster name.
-    def retrieve_k8s_cluster_name
-      @k8s_cluster_name ||= fetch_gce_metadata(
-        'instance/attributes/cluster-name')
-    end
-
-    # Only retrieve location once since it is the location of K8s Master which
-    # is not changing for this agent instance's life time.
-    # Throw an error if neither endpoint is available.
-    def retrieve_k8s_location
-      @k8s_location ||= fetch_gce_metadata(
-        'instance/attributes/cluster-location')
     end
 
     def ensure_array(value)

data/test/plugin/base_test.rb

@@ -1311,113 +1311,115 @@ module BaseTest
       # When enable_metadata_agent is false.
       {
         enable_metadata_agent: false,
-        set_up_metadata_agent_stub: false,
-        set_up_k8s_stub: false,
+        setup_metadata_agent_stub: false,
+        setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: COMPUTE_PARAMS
       },
       {
         enable_metadata_agent: false,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: false,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: COMPUTE_PARAMS
       },
       {
         enable_metadata_agent: false,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: COMPUTE_PARAMS
       },
       {
         enable_metadata_agent: false,
-        set_up_metadata_agent_stub: false,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: false,
+        setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: COMPUTE_PARAMS
       },
       # When enable_metadata_agent is true.
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: false,
-        set_up_k8s_stub: false,
+        setup_metadata_agent_stub: false,
+        setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: COMPUTE_PARAMS
       },
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: false,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: false,
+        setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: K8S_CONTAINER_PARAMS_FROM_LOCAL
       },
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: false,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: false,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: K8S_CONTAINER_PARAMS
       },
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(log_entry(0)),
         expected_params: K8S_CONTAINER_PARAMS
       },
       # When local_resource_id is not present or does not match k8s regexes.
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(
           log_entry(0)).reject { |k, _| k == LOCAL_RESOURCE_ID_KEY },
         expected_params: COMPUTE_PARAMS
       },
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_container_log_entry(
           log_entry(0),
           local_resource_id: RANDOM_LOCAL_RESOURCE_ID),
+        # When 'kube-env' is present, "compute.googleapis.com/resource_name" is
+        # not added.
         expected_params: COMPUTE_PARAMS
       },
       # Specific cases for k8s_node.
       {
         enable_metadata_agent: false,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_node_log_entry(log_entry(0)),
         expected_params: COMPUTE_PARAMS
       },
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_node_log_entry(log_entry(0)),
         expected_params: K8S_NODE_PARAMS
       },
       {
         enable_metadata_agent: true,
-        set_up_metadata_agent_stub: true,
-        set_up_k8s_stub: true,
+        setup_metadata_agent_stub: true,
+        setup_k8s_stub: true,
         log_entry: k8s_node_log_entry(log_entry(0)),
         expected_params: K8S_NODE_PARAMS
       }
     ].each do |test_params|
       new_stub_context do
         setup_gce_metadata_stubs
-        if test_params[:set_up_metadata_agent_stub]
+        if test_params[:setup_metadata_agent_stub]
           setup_metadata_agent_stubs
         else
           setup_no_metadata_agent_stubs
         end
-        if test_params[:set_up_k8s_stub]
-          set_up_k8s_metadata_stubs
+        if test_params[:setup_k8s_stub]
+          setup_k8s_metadata_stubs
         else
-          set_up_no_k8s_metadata_stubs
+          setup_no_k8s_metadata_stubs
         end
         setup_logging_stubs do
           config = if test_params[:enable_metadata_agent]
@@ -1603,7 +1605,7 @@ module BaseTest
                           'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
   end
 
-  def set_up_k8s_metadata_stubs
+  def setup_k8s_metadata_stubs
     stub_metadata_request(
       'instance/attributes/',
       "attribute1\ncluster-name\ncluster-location\nlast_attribute")
@@ -1611,12 +1613,14 @@ module BaseTest
     stub_metadata_request('instance/attributes/cluster-name', K8S_CLUSTER_NAME)
   end
 
-  def set_up_no_k8s_metadata_stubs
-    # Simulate an environment with no new k8s endpoints present.
-    stub_request(:get, %r{.*instance/attributes/cluster-location.*})
-      .to_raise(Errno::EHOSTUNREACH)
-    stub_request(:get, %r{.*instance/attributes/cluster-name.*})
-      .to_raise(Errno::EHOSTUNREACH)
+  def setup_no_k8s_metadata_stubs
+    ['cluster-location', 'cluster-name'].each do |metadata_name|
+      stub_request(:get, %r{.*instance/attributes/#{metadata_name}.*})
+        .to_return(status: 404,
+                   body: 'The requested URL /computeMetadata/v1/instance/' \
+                         "attributes/#{metadata_name} was not found on this" \
+                         ' server.')
+    end
   end
 
   def setup_cloudfunctions_metadata_stubs
@@ -1750,6 +1754,7 @@ module BaseTest
     }
   end
 
+  # TODO(qingling128): Temporary fallback for metadata agent restarts.
   # k8s resources.
 
   def k8s_container_log_entry(log,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.18.pre.1
+  version: 0.6.18.pre.2
 platform: ruby
 authors:
 - Todd Derr