fluent-plugin-google-cloud 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66d74a0b5f7bb8a69f0b229fab8db399efe33a5e
-  data.tar.gz: fa1f657fbbe617ab9ef5159c1577caba6f09b44f
+  metadata.gz: ef4989cf52201417aaa16cd77abc9680b55ddab7
+  data.tar.gz: 4c313c6fbe55b6c17c2eaec8b5c756c947135cab
 SHA512:
-  metadata.gz: 0091443b2d9a27ad9cde62f200d0c11d2ded76633ac126649680a5eb1b54bc914b19bc8148ec6fb3af083e4889c652e0f7f56289dff09cec4eb77ba56d22eaa2
-  data.tar.gz: c35ee5aa129a3420e83e17d753573c1c6a64fce215d6a783a692efe26f33c91516f8ad2cff2a6104827bf89e642df306bff7980671b513f6b80ebc9952fb8b60
+  metadata.gz: f50bd833b6b65548f439a5493f802fe99b266479d077a68abc4fec52e1f04b6223967344eabed170eb1a846525a5ab16e0dbc3b60df6638eaafa1f172c1c76e1
+  data.tar.gz: 4744b04e0a82268535afc934cf2c3a8af71b409fe9ca45f2b4f88f6777e1e80d72e65860ac7a48288c58809120dab1715cbd46287609a4832eea4f68d0287cd8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.4.5)
+    fluent-plugin-google-cloud (0.4.6)
       fluentd (>= 0.10)
       google-api-client (~> 0.8.6)
       googleauth (~> 0.4)
@@ -121,6 +121,3 @@ DEPENDENCIES
   rubocop (~> 0.33.0)
   test-unit (~> 3.0.2)
   webmock (>= 1.17.0)
-
-BUNDLED WITH
-   1.10.6

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.4.5'
+  gem.version       = '0.4.6'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -20,6 +20,7 @@ module Fluent
     # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
     COMPUTE_SERVICE = 'compute.googleapis.com'
+    CONTAINER_SERVICE = 'container.googleapis.com'
     DATAFLOW_SERVICE = 'dataflow.googleapis.com'
     EC2_SERVICE = 'ec2.amazonaws.com'
 
@@ -42,7 +43,9 @@ module Fluent
     # DEPRECATED: Parameters necessary to use the private_key auth_method.
     config_param :private_key_email, :string, :default => nil
     config_param :private_key_path, :string, :default => nil
-    config_param :private_key_passphrase, :string, :default => 'notasecret'
+    config_param :private_key_passphrase, :string,
+                 :default => 'notasecret',
+                 :secret => true
 
     # Specify project/instance metadata.
     #
@@ -103,6 +106,7 @@ module Fluent
       require 'json'
       require 'open-uri'
       require 'socket'
+      require 'yaml'
 
       # use the global logger
       @log = $log # rubocop:disable Style/GlobalVars
@@ -128,8 +132,10 @@ module Fluent
         end
       end
 
-      # TODO: Send instance tags and/or hostname as labels as well?
+      # TODO: Send instance tags as labels as well?
       @common_labels = {}
+      @tag_to_kubernetes_labels_regexp =
+        /\.(?<pod_name>[^\._]+)_(?<namespace_name>[^_]+)_(?<container_name>.+)$/
 
       # set attributes from metadata (unless overriden by static config)
       @vm_name = Socket.gethostname if @vm_name.nil?
@@ -200,14 +206,18 @@ module Fluent
           @service_name = DATAFLOW_SERVICE
           @dataflow_job_id = fetch_gce_metadata('instance/attributes/job_id')
           common_labels["#{DATAFLOW_SERVICE}/job_id"] = @dataflow_job_id
+        elsif attributes.include?('kube-env')
+          # Kubernetes/Container Engine
+          @service_name = CONTAINER_SERVICE
+          common_labels["#{CONTAINER_SERVICE}/instance_id"] = @vm_id
+          @raw_kube_env = fetch_gce_metadata('instance/attributes/kube-env')
+          @kube_env = YAML.load(@raw_kube_env)
+          common_labels["#{CONTAINER_SERVICE}/cluster_name"] =
+            cluster_name_from_kube_env(@kube_env)
         end
-        # include GCE labels unless we're running on dataflow, which
-        # uses their own labels exclusively.
-        if (@service_name != DATAFLOW_SERVICE)
-          common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
-          common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
-          common_labels["#{COMPUTE_SERVICE}/resource_name"] = @vm_name
-        end
+        common_labels["#{COMPUTE_SERVICE}/resource_type"] = 'instance'
+        common_labels["#{COMPUTE_SERVICE}/resource_id"] = @vm_id
+        common_labels["#{COMPUTE_SERVICE}/resource_name"] = @vm_name
       when Platform::EC2
         @service_name = EC2_SERVICE
         common_labels["#{EC2_SERVICE}/resource_type"] = 'instance'
@@ -252,6 +262,18 @@ module Fluent
           'commonLabels' => @common_labels,
           'entries' => []
         }
+        if @service_name == CONTAINER_SERVICE
+          # Container logs in Kubernetes are tagged based on where they came
+          # from, so we can extract useful metadata from the tag.
+          # Do this here to avoid having to repeat it for each record.
+          match_data = @tag_to_kubernetes_labels_regexp.match(tag)
+          if match_data
+            labels = write_log_entries_request['commonLabels']
+            %w(namespace_name pod_name container_name).each do |field|
+              labels["#{CONTAINER_SERVICE}/#{field}"] = match_data[field]
+            end
+          end
+        end
         arr.each do |time, record|
           next unless record.is_a?(Hash)
           if record.key?('timestamp') &&
@@ -292,7 +314,8 @@ module Fluent
               'timestamp' => {
                 'seconds' => ts_secs,
                 'nanos' => ts_nanos
-              }
+              },
+              'labels' => {}
             }
           }
           if record.key?('severity')
@@ -302,18 +325,20 @@ module Fluent
             entry['metadata']['severity'] = 'DEFAULT'
           end
 
+          # If the record has been annotated by the kubernetes_metadata_filter
+          # plugin, then use that metadata. Otherwise, rely on commonLabels
+          # populated at the grouped_entries level from the group's tag.
+          if @service_name == CONTAINER_SERVICE && record['kubernetes']
+            handle_container_metadata(record, entry)
+          end
+
           # If a field is present in the label_map, send its value as a label
           # (mapping the field name to label name as specified in the config)
           # and do not send that field as part of the payload.
-          unless label_map.nil?
-            labels = {}
+          if @label_map
             @label_map.each do |field, label|
-              next unless record.key?(field)
-              # label values are required to be strings.
-              labels[label] = record[field].to_s
-              record.delete(field)
+              field_to_label(record, field, entry['metadata']['labels'], label)
             end
-            entry['metadata']['labels'] = labels unless labels.empty?
           end
 
           # use textPayload if the only remainaing key is 'message',
@@ -323,6 +348,12 @@ module Fluent
           else
             entry['structPayload'] = record
           end
+
+          # Remove the labels metadata if we didn't populate it with anything.
+          if entry['metadata']['labels'].empty?
+            entry['metadata'].delete('labels')
+          end
+
           write_log_entries_request['entries'].push(entry)
         end
         # Don't send an empty request if we rejected all the entries.
@@ -441,6 +472,15 @@ module Fluent
       end
     end
 
+    def cluster_name_from_kube_env(kube_env)
+      return kube_env['CLUSTER_NAME'] if kube_env.key?('CLUSTER_NAME')
+      instance_prefix = kube_env['INSTANCE_PREFIX']
+      gke_name_match = /^gke-(.+)-[0-9a-f]{8}$/.match(instance_prefix)
+      return gke_name_match.captures[0] if gke_name_match &&
+                                           !gke_name_match.captures.empty?
+      instance_prefix
+    end
+
     # Values permitted by the API for 'severity' (which is an enum).
     VALID_SEVERITIES = Set.new(
       %w(DEFAULT DEBUG INFO NOTICE WARNING ERROR CRITICAL ALERT EMERGENCY))
@@ -500,10 +540,36 @@ module Fluent
       'DEFAULT'
     end
 
+    # Requires that record has a 'kubernetes' field.
+    def handle_container_metadata(record, entry)
+      fields = %w(namespace_id namespace_name pod_id pod_name container_name)
+      fields.each do |field|
+        field_to_label(record['kubernetes'], field, entry['metadata']['labels'],
+                       "#{CONTAINER_SERVICE}/#{field}")
+      end
+      # Prepend label/ to all user-defined labels' keys.
+      if record.key?('labels')
+        record['kubernetes']['labels'].each do |key, value|
+          entry['metadata']['labels']["label/#{key}"] = value
+        end
+      end
+      # We've explicitly consumed all the fields we care about -- don't litter
+      # the log entries with the remaining fields that the kubernetes metadata
+      # filter plugin includes (or an empty 'kubernetes' field).
+      record.delete('kubernetes')
+      record.delete('docker')
+    end
+
+    def field_to_label(record, field, labels, label)
+      return unless record.key?(field)
+      labels[label] = record[field].to_s
+      record.delete(field)
+    end
+
     def init_api_client
       @client = Google::APIClient.new(
         application_name: 'Fluentd Google Cloud Logging plugin',
-        application_version: '0.4.5',
+        application_version: '0.4.6',
         retries: 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -60,6 +60,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   MANAGED_VM_BACKEND_NAME = 'default'
   MANAGED_VM_BACKEND_VERSION = 'guestbook2.0'
 
+  # Container Engine / Kubernetes specific labels
+  CONTAINER_CLUSTER_NAME = 'cluster-1'
+  CONTAINER_NAMESPACE_ID = '898268c8-4a36-11e5-9d81-42010af0194c'
+  CONTAINER_NAMESPACE_NAME = 'kube-system'
+  CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
+  CONTAINER_POD_NAME = 'redis-master-c0l82'
+  CONTAINER_CONTAINER_NAME = 'redis'
+
   # Parameters used for authentication
   AUTH_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
   FAKE_AUTH_TOKEN = 'abc123'
@@ -122,6 +130,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   # Service configurations for various services
   COMPUTE_SERVICE_NAME = 'compute.googleapis.com'
   APPENGINE_SERVICE_NAME = 'appengine.googleapis.com'
+  CONTAINER_SERVICE_NAME = 'container.googleapis.com'
   EC2_SERVICE_NAME = 'ec2.amazonaws.com'
 
   COMPUTE_PARAMS = {
@@ -150,6 +159,46 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   }
 
+  CONTAINER_LOG_NAME = "kubernetes.#{CONTAINER_POD_NAME}_" \
+                       "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
+
+  CONTAINER_FROM_METADATA_PARAMS = {
+    'service_name' => CONTAINER_SERVICE_NAME,
+    'log_name' => CONTAINER_LOG_NAME,
+    'project_id' => PROJECT_ID,
+    'zone' => ZONE,
+    'labels' => {
+      "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
+      "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
+      "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
+      "#{CONTAINER_SERVICE_NAME}/namespace_id" => CONTAINER_NAMESPACE_ID,
+      "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
+      "#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
+      "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
+      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
+      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
+      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
+    }
+  }
+
+  # Almost the same as from metadata, but missing namespace_id and pod_id.
+  CONTAINER_FROM_TAG_PARAMS = {
+    'service_name' => CONTAINER_SERVICE_NAME,
+    'log_name' => CONTAINER_LOG_NAME,
+    'project_id' => PROJECT_ID,
+    'zone' => ZONE,
+    'labels' => {
+      "#{CONTAINER_SERVICE_NAME}/instance_id" => VM_ID,
+      "#{CONTAINER_SERVICE_NAME}/cluster_name" => CONTAINER_CLUSTER_NAME,
+      "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
+      "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
+      "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
+      "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
+      "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
+      "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
+    }
+  }
+
   CUSTOM_PARAMS = {
     'service_name' => COMPUTE_SERVICE_NAME,
     'log_name' => 'test',
@@ -175,9 +224,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   }
 
-  def create_driver(conf = APPLICATION_DEFAULT_CONFIG)
+  def create_driver(conf = APPLICATION_DEFAULT_CONFIG, tag = 'test')
     Fluent::Test::BufferedOutputTestDriver.new(
-      Fluent::GoogleCloudOutput).configure(conf, use_v1_config: true)
+      Fluent::GoogleCloudOutput, tag).configure(conf, use_v1_config: true)
   end
 
   def test_configure_service_account_application_default
@@ -724,6 +773,58 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_one_container_log_metadata_from_plugin
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d.emit(container_log_entry_with_metadata(0))
+    d.run
+    verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS)
+  end
+
+  def test_multiple_container_logs_metadata_from_plugin
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    [2, 3, 5, 11, 50].each do |n|
+      # The test driver doesn't clear its buffer of entries after running, so
+      # do it manually here.
+      d.instance_variable_get('@entries').clear
+      @logs_sent = []
+      n.times { |i| d.emit(container_log_entry_with_metadata(i)) }
+      d.run
+      verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS)
+    end
+  end
+
+  def test_one_container_log_metadata_from_tag
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d.emit('message' => log_entry(0))
+    d.run
+    verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS)
+  end
+
+  def test_multiple_container_logs_metadata_from_tag
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    [2, 3, 5, 11, 50].each do |n|
+      # The test driver doesn't clear its buffer of entries after running, so
+      # do it manually here.
+      d.instance_variable_get('@entries').clear
+      @logs_sent = []
+      n.times { |i| d.emit('message' => log_entry(i)) }
+      d.run
+      verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS)
+    end
+  end
+
   # Make parse_severity public so we can test it.
   class Fluent::GoogleCloudOutput # rubocop:disable Style/ClassAndModuleChildren
     public :parse_severity
@@ -856,14 +957,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def setup_logging_stubs
-    [COMPUTE_PARAMS, VMENGINE_PARAMS, CUSTOM_PARAMS, EC2_PARAMS]
-      .each do |params|
-        stub_request(:post, uri_for_log(params))
-          .to_return do |request|
-            @logs_sent << JSON.parse(request.body)
-            { body: '' }
-          end
+    [COMPUTE_PARAMS, VMENGINE_PARAMS, CONTAINER_FROM_TAG_PARAMS,
+     CONTAINER_FROM_METADATA_PARAMS, CUSTOM_PARAMS, EC2_PARAMS].each do |params|
+      stub_request(:post, uri_for_log(params)).to_return do |request|
+        @logs_sent << JSON.parse(request.body)
+        { body: '' }
       end
+    end
   end
 
   def setup_auth_stubs
@@ -900,6 +1000,29 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
                           'guestbook2.0')
   end
 
+  def setup_container_metadata_stubs
+    stub_metadata_request(
+      'instance/attributes/',
+      "attribute1\nkube-env\nlast_attribute")
+    stub_metadata_request('instance/attributes/kube-env',
+                          "ENABLE_NODE_LOGGING: \"true\"\n"\
+                          "INSTANCE_PREFIX: gke-cluster-1-740fdafa\n"\
+                          'KUBE_BEARER_TOKEN: AoQiMuwkNP2BMT0S')
+  end
+
+  def container_log_entry_with_metadata(i)
+    {
+      message: log_entry(i),
+      kubernetes: {
+        namespace_id: CONTAINER_NAMESPACE_ID,
+        namespace_name: CONTAINER_NAMESPACE_NAME,
+        pod_id: CONTAINER_POD_ID,
+        pod_name: CONTAINER_POD_NAME,
+        container_name: CONTAINER_CONTAINER_NAME
+      }
+    }
+  end
+
   def log_entry(i)
     'test log entry ' + i.to_s
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-17 00:00:00.000000000 Z
+date: 2015-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -181,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: fluentd output plugin for the Google Cloud Logging API