fluent-plugin-google-cloud 0.4.8 → 0.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6d35bf9167d8e4847b5e54cc16e0f78dc41310d
-  data.tar.gz: 3481166160940b98b9433cbe80811dfb71f98434
+  metadata.gz: 2e57b5b102aadda45f4cbcab3f47d4a01ee863cd
+  data.tar.gz: 16f62d33c58b8f6fef5385f7f8f0e4958b57de8d
 SHA512:
-  metadata.gz: 0db9e3e069795d1965bbe72c49e2c7d9c8efce96087b099045dfb2308d21c55697bed09c6e6758aaab37d7ae267d1300970203df78f78db5a753f756b40717b9
-  data.tar.gz: 6006bd9033b7d82aa7a720eb8c330f852487e03de9f2816e18795704df69cedfe362fe210741059fe83ca26bd5e61bf83a5499cb761bc154233e11f633cc1516
+  metadata.gz: d1b6ccd7dc730e086f26f09820a8ba32f32872fb746a9e514bef5ae0b03971548ddcab6663470ee89c9700add1b8d663eb59a6838abc67445d4a2e8a2dbd159e
+  data.tar.gz: a5dde71268839c39338e6257c29c3a77cebeea51c6b69c6ee0b56bb26fdeb4b8eb363197c508c36e34be4c237aef0baf6a908ec5358ac47ba35d7ea6e9aa3aaa

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.4.7)
+    fluent-plugin-google-cloud (0.4.8)
       fluentd (>= 0.10)
       google-api-client (~> 0.8.6)
       googleauth (~> 0.4)

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.4.8'
+  gem.version       = '0.4.9'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -382,11 +382,16 @@ module Fluent
 
           set_severity(record, entry)
 
-          # If the record has been annotated by the kubernetes_metadata_filter
-          # plugin, then use that metadata. Otherwise, rely on commonLabels
-          # populated at the grouped_entries level from the group's tag.
-          if @service_name == CONTAINER_SERVICE && record['kubernetes']
-            handle_container_metadata(record, entry)
+          if @service_name == CONTAINER_SERVICE
+            # Move the stdout/stderr annotation from the record into a label
+            field_to_label(record, 'stream', entry['metadata']['labels'],
+                           "#{CONTAINER_SERVICE}/stream")
+            # If the record has been annotated by the kubernetes_metadata_filter
+            # plugin, then use that metadata. Otherwise, rely on commonLabels
+            # populated at the grouped_entries level from the group's tag.
+            if record.key?('kubernetes')
+              handle_container_metadata(record, entry)
+            end
           end
 
           # If a field is present in the label_map, send its value as a label
@@ -417,7 +422,8 @@ module Fluent
         # Don't send an empty request if we rejected all the entries.
         next if write_log_entries_request['entries'].empty?
 
-        log_name = CGI.escape(log_name(tag))
+        log_name = CGI.escape(
+          log_name(tag, write_log_entries_request['commonLabels']))
         url = 'https://logging.googleapis.com/v1beta3/projects/' \
           "#{@project_id}/logs/#{log_name}/entries:write"
         begin
@@ -655,7 +661,8 @@ module Fluent
       # available, or if the only remainaing key is 'message'.
       if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
         entry['textPayload'] = @cloudfunctions_log_match['text']
-      elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
+      elsif (@service_name == CLOUDFUNCTIONS_SERVICE ||
+             @service_name == CONTAINER_SERVICE) && record.key?('log')
         entry['textPayload'] = record['log']
       elsif record.size == 1 && record.key?('message')
         entry['textPayload'] = record['message']
@@ -664,19 +671,27 @@ module Fluent
       end
     end
 
-    def log_name(tag)
+    def log_name(tag, commonLabels)
       if @service_name == CLOUDFUNCTIONS_SERVICE
         return 'cloud-functions'
-      else
-        # Add a prefix to VMEngines logs to prevent namespace collisions.
-        return @running_on_managed_vm ? "#{APPENGINE_SERVICE}/#{tag}" : tag
+      elsif @running_on_managed_vm
+        # Add a prefix to Managed VM logs to prevent namespace collisions.
+        return "#{APPENGINE_SERVICE}/#{tag}"
+      elsif @service_name == CONTAINER_SERVICE
+        # For Kubernetes logs, use just the container name as the log name
+        # if we have it.
+        container_name_key = "#{CONTAINER_SERVICE}/container_name"
+        if commonLabels && commonLabels.key?(container_name_key)
+          return commonLabels[container_name_key]
+        end
       end
+      tag
     end
 
     def init_api_client
       @client = Google::APIClient.new(
         application_name: 'Fluentd Google Cloud Logging plugin',
-        application_version: '0.4.8',
+        application_version: '0.4.9',
         retries: 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -67,6 +67,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   CONTAINER_POD_ID = 'cad3c3c4-4b9c-11e5-9d81-42010af0194c'
   CONTAINER_POD_NAME = 'redis-master-c0l82.foo.bar'
   CONTAINER_CONTAINER_NAME = 'redis'
+  CONTAINER_STREAM = 'stdout'
 
   # Cloud Functions specific labels
   CLOUDFUNCTIONS_FUNCTION_NAME = 'function-1'
@@ -173,12 +174,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   }
 
-  CONTAINER_LOG_NAME = "kubernetes.#{CONTAINER_POD_NAME}_" \
-                       "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
+  CONTAINER_TAG = "kubernetes.#{CONTAINER_POD_NAME}_" \
+                  "#{CONTAINER_NAMESPACE_NAME}_#{CONTAINER_CONTAINER_NAME}"
 
   CONTAINER_FROM_METADATA_PARAMS = {
     'service_name' => CONTAINER_SERVICE_NAME,
-    'log_name' => CONTAINER_LOG_NAME,
+    'log_name' => CONTAINER_CONTAINER_NAME,
     'project_id' => PROJECT_ID,
     'zone' => ZONE,
     'labels' => {
@@ -189,6 +190,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
       "#{CONTAINER_SERVICE_NAME}/pod_id" => CONTAINER_POD_ID,
       "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
+      "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
       "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
       "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
       "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
@@ -198,7 +200,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   # Almost the same as from metadata, but missing namespace_id and pod_id.
   CONTAINER_FROM_TAG_PARAMS = {
     'service_name' => CONTAINER_SERVICE_NAME,
-    'log_name' => CONTAINER_LOG_NAME,
+    'log_name' => CONTAINER_CONTAINER_NAME,
     'project_id' => PROJECT_ID,
     'zone' => ZONE,
     'labels' => {
@@ -207,6 +209,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       "#{CONTAINER_SERVICE_NAME}/namespace_name" => CONTAINER_NAMESPACE_NAME,
       "#{CONTAINER_SERVICE_NAME}/pod_name" => CONTAINER_POD_NAME,
       "#{CONTAINER_SERVICE_NAME}/container_name" => CONTAINER_CONTAINER_NAME,
+      "#{CONTAINER_SERVICE_NAME}/stream" => CONTAINER_STREAM,
       "#{COMPUTE_SERVICE_NAME}/resource_type" => 'instance',
       "#{COMPUTE_SERVICE_NAME}/resource_id" => VM_ID,
       "#{COMPUTE_SERVICE_NAME}/resource_name" => HOSTNAME
@@ -850,7 +853,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs
-    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     d.emit(container_log_entry_with_metadata(0))
     d.run
     verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS)
@@ -860,7 +863,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs
-    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     [2, 3, 5, 11, 50].each do |n|
       # The test driver doesn't clear its buffer of entries after running, so
       # do it manually here.
@@ -876,8 +879,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs
-    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
-    d.emit('message' => log_entry(0))
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
+    d.emit(container_log_entry(0))
     d.run
     verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS)
   end
@@ -886,13 +889,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     setup_container_metadata_stubs
     setup_logging_stubs
-    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     [2, 3, 5, 11, 50].each do |n|
       # The test driver doesn't clear its buffer of entries after running, so
       # do it manually here.
       d.instance_variable_get('@entries').clear
       @logs_sent = []
-      n.times { |i| d.emit('message' => log_entry(i)) }
+      n.times { |i| d.emit(container_log_entry(i)) }
       d.run
       verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS)
     end
@@ -962,14 +965,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     setup_cloudfunctions_metadata_stubs
     setup_logging_stubs
-    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     d.emit(cloudfunctions_log_entry(0))
     d.run
-    verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS, 'structPayload') do |entry|
-      assert_equal 2, entry['structPayload'].size, entry
+    verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS, '') do |entry|
       assert_equal '[D][2015-09-25T12:34:56.789Z][123-0] test log entry 0',
-                   entry['structPayload']['log'], entry
-      assert_equal 'stdout', entry['structPayload']['stream'], entry
+                   entry['textPayload'], entry
     end
   end
 
@@ -977,7 +978,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_gce_metadata_stubs
     setup_cloudfunctions_metadata_stubs
     setup_logging_stubs
-    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_LOG_NAME)
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     [2, 3, 5, 11, 50].each do |n|
       # The test driver doesn't clear its buffer of entries after running, so
       # do it manually here.
@@ -987,11 +988,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       d.run
       i = 0
       params = CONTAINER_FROM_TAG_PARAMS
-      verify_log_entries(n, params, 'structPayload') do |entry|
-        assert_equal 2, entry['structPayload'].size, entry
+      verify_log_entries(n, params, '') do |entry|
         assert_equal "[D][2015-09-25T12:34:56.789Z][123-0] test log entry #{i}",
-                     entry['structPayload']['log'], entry
-        assert_equal 'stdout', entry['structPayload']['stream'], entry
+                     entry['textPayload'], entry
         i += 1
       end
     end
@@ -1196,7 +1195,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
 
   def container_log_entry_with_metadata(i)
     {
-      message: log_entry(i),
+      log: log_entry(i),
+      stream: 'stdout',
       kubernetes: {
         namespace_id: CONTAINER_NAMESPACE_ID,
         namespace_name: CONTAINER_NAMESPACE_NAME,
@@ -1207,6 +1207,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   end
 
+  def container_log_entry(i)
+    {
+      log: log_entry(i),
+      stream: 'stdout'
+    }
+  end
+
   def cloudfunctions_log_entry(i)
     {
       stream: 'stdout',
@@ -1245,10 +1252,12 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     i = 0
     @logs_sent.each do |batch|
       batch['entries'].each do |entry|
-        assert entry.key?(payload_type)
-        if (payload_type == 'textPayload')
-          # Check the payload for textPayload, otherwise it is up to the caller.
-          assert_equal "test log entry #{i}", entry['textPayload'], batch
+        unless payload_type.empty?
+          assert entry.key?(payload_type)
+          # Check the payload for textPayload, otherwise it's up to the caller.
+          if (payload_type == 'textPayload')
+            assert_equal "test log entry #{i}", entry['textPayload'], batch
+          end
         end
 
         assert_equal params['zone'], entry['metadata']['zone']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.4.8
+  version: 0.4.9
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-30 00:00:00.000000000 Z
+date: 2015-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd