fluent-plugin-google-cloud 0.6.4.pre.1 → 0.6.4.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8b2b5d197dc8da098e69040d3c1e33cf1d13390
-  data.tar.gz: d01388ebf1188dcd27061bdc3466a65be62856e9
+  metadata.gz: e87c18a3faae1b0b748c947487bac806f9af22ff
+  data.tar.gz: 7aa3fb16a6625f55e86dd7a1b2b983ce090b0803
 SHA512:
-  metadata.gz: b6720df5c417423746b423fd34cebbcb3d417cdbc1cdfe91307970841661cf8d18edd987e0f622680bcc51b4503b5e3458c3029774362396eea769f2a2378dc5
-  data.tar.gz: 0ebf298b06bc970370d4a7640096f2fd14a2646339c9db6e93f29dacbf640140e52d1016c89cc63811b8248c314e945e54f8e47088b58a4384cd6082da233d59
+  metadata.gz: 16e3e4c2db4dc5dccc26c403bd1a64802afa7fd97fa0a9bdc3109a583d865c594958efeb7e769a7aa45987d237f4842ecadb18280c6c6adc87ee6bee2a1a34b6
+  data.tar.gz: 07a36c3637028e394f84f01baa05650f6d8db4260db6b64dae92ff7d1f4b6af6bca3173162bcbb588dffbc765e9426c24588ef30523d1dd29e0f6f1ca13d502c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.6.4.pre.1)
+    fluent-plugin-google-cloud (0.6.4.pre.2)
       excon (~> 0.56.0)
       fluentd (~> 0.10)
       google-api-client (~> 0.9.0)
@@ -25,7 +25,7 @@ GEM
     excon (0.56.0)
     faraday (0.12.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.14.18)
+    fluentd (0.14.19)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
@@ -127,7 +127,7 @@ GEM
       multi_json (~> 1.10)
     stackdriver-core (0.21.0)
     strptime (0.1.9)
-    test-unit (3.2.4)
+    test-unit (3.2.5)
       power_assert
     thread_safe (0.3.6)
     tins (1.6.0)
@@ -154,4 +154,4 @@ DEPENDENCIES
   webmock (~> 1.17)
 
 BUNDLED WITH
-   1.15.1
+   1.15.2

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.6.4.pre.1'
+  gem.version       = '0.6.4.pre.2'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_google_cloud.rb

@@ -79,7 +79,7 @@ module Fluent
         resource_type: 'ml_job'
       }
 
-      # Metadata agent support.
+      # Metadata Agent support.
 
       # Use empty string as request path when locally-unique key of monitored
       # resource can be implicitly inferred by Metadata Agent.
@@ -94,7 +94,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.6.4.pre.1'
+    PLUGIN_VERSION = '0.6.4.pre.2'
 
     # Name of the the Google cloud logging write scope.
     LOGGING_SCOPE = 'https://www.googleapis.com/auth/logging.write'
@@ -213,7 +213,7 @@ module Fluent
     config_param :call_docker_api_locally, :bool, :default => true
     # Docker Remote API unix socket path.
     config_param :docker_remote_api_socket_path, :string,
-                 :default => '/var/run/docker.sock'
+                 :default => DEFAULT_DOCKER_API_SOCKET_PATH
 
     # rubocop:enable Style/HashSyntax
 
@@ -379,9 +379,10 @@ module Fluent
         arr.each do |time, record|
           next unless record.is_a?(Hash)
 
-          extracted_resource_labels, extracted_common_labels = \
+          resource_type, extracted_resource_labels, extracted_common_labels = \
             determine_entry_level_labels(group_resource, record)
           entry_resource = group_resource.dup
+          entry_resource.type = resource_type
           entry_resource.labels.merge!(extracted_resource_labels)
           entry_common_labels = \
             group_common_labels.merge(extracted_common_labels)
@@ -746,7 +747,7 @@ module Fluent
     # Retrieve monitored resource via the legacy way.
     #
     # Note: This is just a failover plan if we fail to get metadata from
-    # Metadata agent. Thus it should be equivalent to what Metadata Agent
+    # Metadata Agent. Thus it should be equivalent to what Metadata Agent
     # returns.
     def determine_agent_level_monitored_resource_via_legacy
       resource = Google::Apis::LoggingV2beta1::MonitoredResource.new(
@@ -1041,22 +1042,22 @@ module Fluent
             locally_unique_id)
         end
 
-        if @enable_metadata_agent && !retrieved_resource.nil?
+        if !retrieved_resource.nil?
           # If we successfully get a monitored resource from Metadata Agent,
-          # use this one instead of the original VM-level monitored resource.
+          # use this one instead of the original instance monitored resource.
           group_resource_labels = retrieved_resource.labels.dup
           @log.debug 'Retrieved monitored resource from Metadata Agent: ' \
                      "#{retrieved_resource.inspect}."
         else
           # If Metadata Agent is not enabled, or we failed to get a monitored
           # resource, we need to have some backup plan.
-          @log.debug 'Metadata agent not enabled or failed to retrieve ' \
+          @log.debug 'Metadata Agent not enabled or failed to retrieve ' \
                      'docker container monitored resource from Metadata ' \
                      'Agent.'
 
           # 1. Check if 'container_id' is set already. It should be available
           #    for stdout / stderr). If so, use that.
-          # 2. If not, call Docker Remote API to calculate container id from
+          # 2. If not, call Docker Remote API to retrieve the container ID from
           #    container name, but only if @call_docker_api_locally is true.
           container_id ||= retrieve_container_id_by_name_locally(
             container_name) if @call_docker_api_locally
@@ -1094,11 +1095,45 @@ module Fluent
     # Extract entry resource and common labels that should be applied to
     # individual entries from the group resource.
     def determine_entry_level_labels(group_resource, record)
+      resource_type = group_resource.type
       resource_labels = {}
       common_labels = {}
 
+      if record.key?("#{CONTAINER_CONSTANTS[:service]}/namespace_id") &&
+         record.key?("#{CONTAINER_CONSTANTS[:service]}/pod_name") &&
+         record.key?("#{CONTAINER_CONSTANTS[:service]}/container_name")
+        if @enable_metadata_agent
+          # Call Metadata Agent with "gke_containerName.<namespace_id>.
+          # <pod_name>.<container_name>" as the locally-unique key to retrieve
+          # monitored resource.
+          locally_unique_id = 'gke_containerName.' \
+            "#{record["#{CONTAINER_CONSTANTS[:service]}/namespace_id"]}." \
+            "#{record["#{CONTAINER_CONSTANTS[:service]}/pod_name"]}." \
+            "#{record["#{CONTAINER_CONSTANTS[:service]}/container_name"]}"
+          @log.debug 'Calling metadata agent with locally unique id: ' \
+                    "#{locally_unique_id}."
+          retrieved_resource = call_metadata_agent_for_monitored_resource(
+            locally_unique_id)
+          @log.debug 'Retrieved monitored resource from metadata agent: ' \
+                    "#{retrieved_resource.inspect}."
+          unless retrieved_resource.nil?
+            # Temporarily renaming 'gke_container' to 'container'.
+            # resource_type = retrieved_resource.type
+            resource_type = 'container'
+            # If we successfully get a monitored resource from Metadata Agent,
+            # use this one instead of the original VM-level monitored resource.
+            resource_labels = retrieved_resource.labels.dup
+            record.delete("#{CONTAINER_CONSTANTS[:service]}/namespace_id")
+            record.delete("#{CONTAINER_CONSTANTS[:service]}/pod_name")
+            record.delete("#{CONTAINER_CONSTANTS[:service]}/container_name")
+            @log.debug 'Retrieved gke monitored resource from Metadata ' \
+                       "Agent: ' #{retrieved_resource.inspect}."
+          end
+        end
+      end
+
       # Cloud Functions.
-      if group_resource.type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
+      if resource_type == CLOUDFUNCTIONS_CONSTANTS[:resource_type] &&
          record.key?('log')
         @cloudfunctions_log_match =
           @cloudfunctions_log_regexp.match(record['log'])
@@ -1109,7 +1144,7 @@ module Fluent
       end
 
       # GKE containers.
-      if group_resource.type == CONTAINER_CONSTANTS[:resource_type]
+      if resource_type == CONTAINER_CONSTANTS[:resource_type]
         # Move the stdout/stderr annotation from the record into a label.
         common_labels.merge!(
           fields_to_labels(
@@ -1127,7 +1162,7 @@ module Fluent
       end
 
       # Docker containers.
-      if group_resource.type == DOCKER_CONSTANTS[:resource_type]
+      if resource_type == DOCKER_CONSTANTS[:resource_type]
         # For logs coming from Docker Fluentd Logging Driver, the log record
         # has 4 fields: 'container_id', 'container_name', 'source' and 'log'.
         # Extract 'container_id', 'container_name' and 'source' from json
@@ -1153,9 +1188,9 @@ module Fluent
       common_labels.merge!(fields_to_labels(record, @label_map))
 
       resource_labels.merge!(
-        extract_resource_labels(group_resource.type, common_labels))
+        extract_resource_labels(resource_type, common_labels))
 
-      [resource_labels, common_labels]
+      [resource_type, resource_labels, common_labels]
     end
 
     # Call Metadata Agent to get monitored resource information and parse
@@ -1171,7 +1206,7 @@ module Fluent
         return nil
       end
 
-      # TODO(lingshi): Use Google::Api::MonitoredResource directly after we
+      # TODO(qingling128): Use Google::Api::MonitoredResource directly after we
       # upgrade gRPC version to include the fix for the protobuf map
       # corruption issue.
       Google::Apis::LoggingV2beta1::MonitoredResource.new(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.6.4.pre.1
+  version: 0.6.4.pre.2
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-22 00:00:00.000000000 Z
+date: 2017-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: excon