fluent-plugin-google-cloud 0.4.9 → 0.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e57b5b102aadda45f4cbcab3f47d4a01ee863cd
-  data.tar.gz: 16f62d33c58b8f6fef5385f7f8f0e4958b57de8d
+  metadata.gz: 8625be40328ad3c14dd4596985a1017092c2884a
+  data.tar.gz: d39dc933fdc5c0db8830c83ed94bd19f1db147da
 SHA512:
-  metadata.gz: d1b6ccd7dc730e086f26f09820a8ba32f32872fb746a9e514bef5ae0b03971548ddcab6663470ee89c9700add1b8d663eb59a6838abc67445d4a2e8a2dbd159e
-  data.tar.gz: a5dde71268839c39338e6257c29c3a77cebeea51c6b69c6ee0b56bb26fdeb4b8eb363197c508c36e34be4c237aef0baf6a908ec5358ac47ba35d7ea6e9aa3aaa
+  metadata.gz: d60286af889814e55fa15012b8fcdd5d1114ea2e14c420fdce8d7942d33479378e26cf2074cb02b2ab08a1943052a1f4ce3fa5de75a4526eeb0974c17c3e4da3
+  data.tar.gz: 67d62b7f2292cdc4d1a98aa5a7591d7d838ce923764c5055c5da3a92c00edbb84558b1e73535e758f7c335b29074c743c48a6439de42db1cff29d9cdab63788c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.4.8)
+    fluent-plugin-google-cloud (0.4.10)
       fluentd (>= 0.10)
       google-api-client (~> 0.8.6)
       googleauth (~> 0.4)
@@ -24,13 +24,13 @@ GEM
       addressable (>= 2.3.1)
       extlib (>= 0.9.15)
       multi_json (>= 1.0.0)
-    cool.io (1.4.0)
+    cool.io (1.4.1)
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     extlib (0.9.16)
-    faraday (0.9.1)
+    faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.12.15)
+    fluentd (0.12.16)
       cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
@@ -70,7 +70,7 @@ GEM
       multi_json (~> 1.10)
     memoist (0.12.0)
     metaclass (0.0.4)
-    minitest (5.8.0)
+    minitest (5.8.1)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
     msgpack (0.5.12)
@@ -104,7 +104,7 @@ GEM
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2015.6)
+    tzinfo-data (1.2015.7)
       tzinfo (>= 1.0.0)
     webmock (1.21.0)
       addressable (>= 2.3.6)
@@ -121,3 +121,6 @@ DEPENDENCIES
   rubocop (~> 0.33.0)
   test-unit (~> 3.0.2)
   webmock (>= 1.17.0)
+
+BUNDLED WITH
+   1.10.6

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.4.9'
+  gem.version       = '0.4.10'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -156,7 +156,7 @@ module Fluent
       end
 
       @cloudfunctions_tag_regexp =
-        /\.(?<function_name>.+)-[^-]+_default_worker$/
+        /\.(?<function_name>[^.]+)(?:\.\d+)?-[^-]+_default_worker$/
       @cloudfunctions_log_regexp = /^
         (?:\[(?<severity>.)\])?
         \[(?<timestamp>.{24})\]
@@ -326,12 +326,54 @@ module Fluent
             @service_name = CONTAINER_SERVICE
           end
         end
+        is_container_json = nil
         arr.each do |time, record|
           next unless record.is_a?(Hash)
+
+          entry = {
+            'metadata' => {
+              'serviceName' => @service_name,
+              'projectId' => @project_id,
+              'zone' => @zone,
+              'labels' => {}
+            }
+          }
+
           if @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
             @cloudfunctions_log_match =
               @cloudfunctions_log_regexp.match(record['log'])
           end
+          if @service_name == CONTAINER_SERVICE
+            # Move the stdout/stderr annotation from the record into a label
+            field_to_label(record, 'stream', entry['metadata']['labels'],
+                           "#{CONTAINER_SERVICE}/stream")
+            # If the record has been annotated by the kubernetes_metadata_filter
+            # plugin, then use that metadata. Otherwise, rely on commonLabels
+            # populated at the grouped_entries level from the group's tag.
+            if record.key?('kubernetes')
+              handle_container_metadata(record, entry)
+            end
+            # If the log from the user container is json, we want to export it
+            # as a structured log. Now that we've pulled out all the
+            # container-specific metadata from the record, we can replace the
+            # record with the json that the user logged.
+            # To save CPU in the common case of unstructured logs, only check if
+            # the contents are parsable as json for the first entry of each
+            # batch.
+            if is_container_json.nil? && record.key?('log')
+              record_json = parse_json_or_nil(record['log'])
+              if record_json.nil?
+                is_container_json = false
+              else
+                record = record_json
+                is_container_json = true
+              end
+            elsif is_container_json && record.key?('log')
+              record_json = parse_json_or_nil(record['log'])
+              record = record_json unless record_json.nil?
+            end
+          end
+
           if record.key?('timestamp') &&
              record['timestamp'].is_a?(Hash) &&
              record['timestamp'].key?('seconds') &&
@@ -367,33 +409,13 @@ module Fluent
             ts_secs = timestamp.tv_sec
             ts_nanos = timestamp.tv_nsec
           end
-          entry = {
-            'metadata' => {
-              'serviceName' => @service_name,
-              'projectId' => @project_id,
-              'zone' => @zone,
-              'timestamp' => {
-                'seconds' => ts_secs,
-                'nanos' => ts_nanos
-              },
-              'labels' => {}
-            }
+          entry['metadata']['timestamp'] = {
+            'seconds' => ts_secs,
+            'nanos' => ts_nanos
           }
 
           set_severity(record, entry)
 
-          if @service_name == CONTAINER_SERVICE
-            # Move the stdout/stderr annotation from the record into a label
-            field_to_label(record, 'stream', entry['metadata']['labels'],
-                           "#{CONTAINER_SERVICE}/stream")
-            # If the record has been annotated by the kubernetes_metadata_filter
-            # plugin, then use that metadata. Otherwise, rely on commonLabels
-            # populated at the grouped_entries level from the group's tag.
-            if record.key?('kubernetes')
-              handle_container_metadata(record, entry)
-            end
-          end
-
           # If a field is present in the label_map, send its value as a label
           # (mapping the field name to label name as specified in the config)
           # and do not send that field as part of the payload.
@@ -410,7 +432,7 @@ module Fluent
               @cloudfunctions_log_match['execution_id']
           end
 
-          set_payload(record, entry)
+          set_payload(record, entry, is_container_json)
 
           # Remove the labels metadata if we didn't populate it with anything.
           if entry['metadata']['labels'].empty?
@@ -478,6 +500,17 @@ module Fluent
                 error_class: error.class.to_s, error: error.to_s
     end
 
+    def parse_json_or_nil(input)
+      # Only here to please rubocop...
+      return nil if input.nil?
+
+      begin
+        return JSON.parse(input)
+      rescue JSON::ParserError
+        return nil
+      end
+    end
+
     # "enum" of Platform values
     module Platform
       OTHER = 0  # Other/unkown platform
@@ -656,13 +689,18 @@ module Fluent
       record.delete(field)
     end
 
-    def set_payload(record, entry)
-      # Use textPayload if this is the Cloud Functions service and 'log' key is
-      # available, or if the only remainaing key is 'message'.
+    def set_payload(record, entry, is_container_json)
+      # Use textPayload if
+      # 1. This is a Cloud Functions log that matched the expected regexp
+      # 2. This is a Cloud Functions log and the 'log' key is available
+      # 3. This is an unstructured Container log and the 'log' key is available
+      # 4. The only remaining key is 'message'
       if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
         entry['textPayload'] = @cloudfunctions_log_match['text']
-      elsif (@service_name == CLOUDFUNCTIONS_SERVICE ||
-             @service_name == CONTAINER_SERVICE) && record.key?('log')
+      elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
+        entry['textPayload'] = record['log']
+      elsif @service_name == CONTAINER_SERVICE && record.key?('log') &&
+            !is_container_json
         entry['textPayload'] = record['log']
       elsif record.size == 1 && record.key?('message')
         entry['textPayload'] = record['message']
@@ -691,7 +729,7 @@ module Fluent
     def init_api_client
       @client = Google::APIClient.new(
         application_name: 'Fluentd Google Cloud Logging plugin',
-        application_version: '0.4.9',
+        application_version: '0.4.10',
         retries: 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -75,7 +75,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   CLOUDFUNCTIONS_EXECUTION_ID = '123-0'
   CLOUDFUNCTIONS_CLUSTER_NAME = 'cluster-1'
   CLOUDFUNCTIONS_NAMESPACE_NAME = 'default'
-  CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}-c0l82"
+  CLOUDFUNCTIONS_POD_NAME = "#{CLOUDFUNCTIONS_FUNCTION_NAME}.987-c0l82"
   CLOUDFUNCTIONS_CONTAINER_NAME = 'worker'
 
   # Parameters used for authentication
@@ -854,7 +854,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_container_metadata_stubs
     setup_logging_stubs
     d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
-    d.emit(container_log_entry_with_metadata(0))
+    d.emit(container_log_entry_with_metadata(log_entry(0)))
     d.run
     verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS)
   end
@@ -869,7 +869,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       # do it manually here.
       d.instance_variable_get('@entries').clear
       @logs_sent = []
-      n.times { |i| d.emit(container_log_entry_with_metadata(i)) }
+      n.times { |i| d.emit(container_log_entry_with_metadata(log_entry(i))) }
       d.run
       verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS)
     end
@@ -880,7 +880,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_container_metadata_stubs
     setup_logging_stubs
     d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
-    d.emit(container_log_entry(0))
+    d.emit(container_log_entry(log_entry(0)))
     d.run
     verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS)
   end
@@ -895,12 +895,46 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       # do it manually here.
       d.instance_variable_get('@entries').clear
       @logs_sent = []
-      n.times { |i| d.emit(container_log_entry(i)) }
+      n.times { |i| d.emit(container_log_entry(log_entry(i))) }
       d.run
       verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS)
     end
   end
 
+  def test_struct_container_log_metadata_from_plugin
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
+    d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
+                                             '"tag2": "test", "data": 5000}'))
+    d.run
+    verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
+                       'structPayload') do |entry|
+      assert_equal 3, entry['structPayload'].size, entry
+      assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
+      assert_equal 'test', entry['structPayload']['tag2'], entry
+      assert_equal 5000, entry['structPayload']['data'], entry
+    end
+  end
+
+  def test_struct_container_log_metadata_from_tag
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
+    d.emit(container_log_entry('{"msg": "test log entry 0", ' \
+                               '"tag2": "test", "data": 5000}'))
+    d.run
+    verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
+                       'structPayload') do |entry|
+      assert_equal 3, entry['structPayload'].size, entry
+      assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
+      assert_equal 'test', entry['structPayload']['tag2'], entry
+      assert_equal 5000, entry['structPayload']['data'], entry
+    end
+  end
+
   def test_one_cloudfunctions_log
     setup_gce_metadata_stubs
     setup_cloudfunctions_metadata_stubs
@@ -1193,9 +1227,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     stub_metadata_request('instance/attributes/gcf_region', 'us-central1')
   end
 
-  def container_log_entry_with_metadata(i)
+  def container_log_entry_with_metadata(log)
     {
-      log: log_entry(i),
+      log: log,
       stream: 'stdout',
       kubernetes: {
         namespace_id: CONTAINER_NAMESPACE_ID,
@@ -1207,9 +1241,9 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   end
 
-  def container_log_entry(i)
+  def container_log_entry(log)
     {
-      log: log_entry(i),
+      log: log,
       stream: 'stdout'
     }
   end
@@ -1253,7 +1287,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     @logs_sent.each do |batch|
       batch['entries'].each do |entry|
         unless payload_type.empty?
-          assert entry.key?(payload_type)
+          assert entry.key?(payload_type), 'Entry did not contain expected ' \
+            "#{payload_type} key: " + entry.to_s
           # Check the payload for textPayload, otherwise it's up to the caller.
           if (payload_type == 'textPayload')
             assert_equal "test log entry #{i}", entry['textPayload'], batch

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.4.10
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd