fluent-plugin-google-cloud 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bae4e3edae194459b6bffe522aa070d6c96f4c01
-  data.tar.gz: 290eaab1b92a6cea5c9204f5892fb008c6334d57
+  metadata.gz: 8109f72f6f16a0756b3a30c911e9343392051b96
+  data.tar.gz: e76c44a9c3e8cb63c6125f1b621e7985e3050918
 SHA512:
-  metadata.gz: c8644923f32740b2aee18e0b71047c8dc6d1c881c7f590a88a7a5e136088c9eccd428bfcbe53aadaeedf6b301ecc5f6e29d9a7ec5545635b3f7a1021a2371aba
-  data.tar.gz: 8d101639abc752dd42a7c0dcf29bdc8834d20bbde8686e4eb3b41b6ca6c09b830e62b2a8ecc4551657eac3ef9baa5d779148358b6326dddd23749ea108a60dda
+  metadata.gz: aad0d625ddb42768a112e887bae953d601bd646d9003d7a7308a19a3e9e4afdbd61725bd56b844cefc7626a99dad067ea3ef6d68e44f06bc324ab7abe937bcd2
+  data.tar.gz: c880a81905b4ce2535f247a7bd1938e46082461f8c9f48a03716908219e6fe95234397a09ee4975ed21dc5c196b782d1efd1ebf499617c7fb989aa234efbf60e

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.5.0)
-      fluentd (~> 0.10)
+    fluent-plugin-google-cloud (0.5.2)
+      fluentd (~> 0.10, <= 0.13)
       google-api-client (> 0.9)
       googleauth (~> 0.4)
       json (~> 1.8)
@@ -11,25 +11,25 @@ GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.4.0)
-    ast (2.2.0)
+    ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    cool.io (1.4.3)
+    cool.io (1.4.4)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.12.22)
+    fluentd (0.12.26)
       cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
       msgpack (>= 0.5.11, < 0.6.0)
       sigdump (~> 0.2.2)
-      string-scrub (>= 0.0.3)
+      string-scrub (>= 0.0.3, <= 0.0.5)
       tzinfo (>= 1.0.0)
       tzinfo-data (>= 1.0.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.9.5)
+    google-api-client (0.9.9)
       addressable (~> 2.3)
       googleauth (~> 0.5)
       httpclient (~> 2.7)
@@ -49,7 +49,7 @@ GEM
       signet (~> 0.7)
     hashdiff (0.3.0)
     http_parser.rb (0.6.0)
-    httpclient (2.7.1)
+    httpclient (2.8.0)
     hurley (0.2)
     json (1.8.3)
     jwt (1.5.4)
@@ -59,18 +59,18 @@ GEM
       multi_json (~> 1.10)
     memoist (0.14.0)
     metaclass (0.0.4)
-    mime-types (3.0)
+    mime-types (3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0221)
+    mime-types-data (3.2016.0521)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
     msgpack (0.5.12)
-    multi_json (1.11.2)
+    multi_json (1.12.1)
     multipart-post (2.0.0)
     os (0.9.6)
-    parser (2.3.0.7)
+    parser (2.3.1.2)
       ast (~> 2.2)
-    power_assert (0.2.7)
+    power_assert (0.3.0)
     powerpack (0.1.1)
     rainbow (2.1.0)
     rake (10.5.0)
@@ -83,25 +83,25 @@ GEM
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
-    ruby-progressbar (1.7.5)
+    ruby-progressbar (1.8.1)
     safe_yaml (1.0.4)
     sigdump (0.2.4)
-    signet (0.7.2)
+    signet (0.7.3)
       addressable (~> 2.3)
       faraday (~> 0.9)
       jwt (~> 1.5)
       multi_json (~> 1.10)
     string-scrub (0.0.5)
-    test-unit (3.1.8)
+    test-unit (3.2.0)
       power_assert
     thor (0.19.1)
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2016.4)
+    tzinfo-data (1.2016.5)
       tzinfo (>= 1.0.0)
     uber (0.0.15)
-    webmock (1.24.3)
+    webmock (1.24.6)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -119,4 +119,4 @@ DEPENDENCIES
   webmock (~> 1.17)
 
 BUNDLED WITH
-   1.11.2
+   1.12.5

data/fluent-plugin-google-cloud.gemspec

@@ -10,7 +10,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.5.1'
+  gem.version       = '0.5.2'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -19,7 +19,7 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '~> 0.10'
+  gem.add_runtime_dependency 'fluentd', '~> 0.10', '<= 0.13'
   gem.add_runtime_dependency 'google-api-client', '> 0.9'
   gem.add_runtime_dependency 'googleauth', '~> 0.4'
   gem.add_runtime_dependency 'json', '~> 1.8'

data/lib/fluent/plugin/out_google_cloud.rb

@@ -14,6 +14,7 @@
 require 'json'
 require 'open-uri'
 require 'socket'
+require 'time'
 require 'yaml'
 require 'google/apis'
 require 'google/apis/logging_v1beta3'
@@ -25,7 +26,7 @@ module Fluent
     Fluent::Plugin.register_output('google_cloud', self)
 
     PLUGIN_NAME = 'Fluentd Google Cloud Logging plugin'
-    PLUGIN_VERSION = '0.5.1'
+    PLUGIN_VERSION = '0.5.2'
 
     # Constants for service names.
     APPENGINE_SERVICE = 'appengine.googleapis.com'
@@ -347,7 +348,6 @@ module Fluent
             end
           end
         end
-        is_container_json = nil
         arr.each do |time, record|
           next unless record.is_a?(Hash)
 
@@ -373,24 +373,27 @@ module Fluent
             if record.key?('kubernetes')
               handle_container_metadata(record, entry)
             end
-            # If the log from the user container is json, we want to export it
-            # as a structured log. Now that we've pulled out all the
-            # container-specific metadata from the record, we can replace the
-            # record with the json that the user logged.
-            # To save CPU in the common case of unstructured logs, only check if
-            # the contents are parsable as json for the first entry of each
-            # batch.
-            if is_container_json.nil? && record.key?('log')
-              record_json = parse_json_or_nil(record['log'])
-              if record_json.nil?
-                is_container_json = false
-              else
-                record = record_json
-                is_container_json = true
-              end
-            elsif is_container_json && record.key?('log')
+
+            # Save the timestamp if available, then clear it out to allow for
+            # determining whether we should parse the log or message field.
+            timestamp = record.key?('time') ? record['time'] : nil
+            record.delete('time')
+            # If the log is json, we want to export it as a structured log
+            # unless there is additional metadata that would be lost.
+            is_json = false
+            if record.length == 1 && record.key?('log')
               record_json = parse_json_or_nil(record['log'])
-              record = record_json unless record_json.nil?
+            end
+            if record.length == 1 && record.key?('message')
+              record_json = parse_json_or_nil(record['message'])
+            end
+            unless record_json.nil?
+              record = record_json
+              is_json = true
+            end
+            # Restore timestamp if necessary.
+            unless record.key?('time') || timestamp.nil?
+              record['time'] = timestamp
             end
           end
 
@@ -414,7 +417,7 @@ module Fluent
               @cloudfunctions_log_match['execution_id']
           end
 
-          set_payload(record, entry, is_container_json)
+          set_payload(record, entry, is_json)
           entry.metadata.labels = nil if entry.metadata.labels.empty?
 
           entries.push(entry)
@@ -476,11 +479,23 @@ module Fluent
       # Only here to please rubocop...
       return nil if input.nil?
 
-      begin
-        return JSON.parse(input)
-      rescue JSON::ParserError
-        return nil
-      end
+      input.each_codepoint do |c|
+        if c == 123
+          # left curly bracket (U+007B)
+          begin
+            return JSON.parse(input)
+          rescue JSON::ParserError
+            return nil
+          end
+        else
+          # Break (and return nil) unless the current character is whitespace,
+          # in which case we continue to look for a left curly bracket.
+          # Whitespace as per the JSON spec are: tabulation (U+0009),
+          # line feed (U+000A), carriage return (U+000D), and space (U+0020).
+          break unless c == 9 || c == 10 || c == 13 || c == 32
+        end # case
+      end # do
+      nil
     end
 
     # "enum" of Platform values
@@ -616,6 +631,15 @@ module Fluent
         timestamp = DateTime.parse(@cloudfunctions_log_match['timestamp'])
         ts_secs = timestamp.strftime('%s')
         ts_nanos = timestamp.strftime('%N')
+      elsif record.key?('time')
+        # k8s ISO8601 timestamp
+        begin
+          timestamp = Time.iso8601(record.delete('time'))
+        rescue
+          timestamp = Time.at(time)
+        end
+        ts_secs = timestamp.tv_sec
+        ts_nanos = timestamp.tv_nsec
       else
         timestamp = Time.at(time)
         ts_secs = timestamp.tv_sec
@@ -644,6 +668,16 @@ module Fluent
       elsif record.key?('severity')
         entry.metadata.severity = parse_severity(record['severity'])
         record.delete('severity')
+      elsif @service_name == CONTAINER_SERVICE && \
+            entry.metadata.labels.key?("#{CONTAINER_SERVICE}/stream")
+        stream = entry.metadata.labels["#{CONTAINER_SERVICE}/stream"]
+        if stream == 'stdout'
+          entry.metadata.severity = 'INFO'
+        elsif stream == 'stderr'
+          entry.metadata.severity = 'ERROR'
+        else
+          entry.metadata.severity = 'DEFAULT'
+        end
       else
         entry.metadata.severity = 'DEFAULT'
       end
@@ -759,18 +793,20 @@ module Fluent
       record.delete(field)
     end
 
-    def set_payload(record, entry, is_container_json)
-      # Use textPayload if
-      # 1. This is a Cloud Functions log that matched the expected regexp
-      # 2. This is a Cloud Functions log and the 'log' key is available
-      # 3. This is an unstructured Container log and the 'log' key is available
-      # 4. The only remaining key is 'message'
+    def set_payload(record, entry, is_json)
+      # If this is a Cloud Functions log that matched the expected regexp,
+      # use text payload. Otherwise, use JSON if we found valid JSON, or text
+      # payload in the following cases:
+      # 1. This is a Cloud Functions log and the 'log' key is available
+      # 2. This is an unstructured Container log and the 'log' key is available
+      # 3. The only remaining key is 'message'
       if @service_name == CLOUDFUNCTIONS_SERVICE && @cloudfunctions_log_match
         entry.text_payload = @cloudfunctions_log_match['text']
       elsif @service_name == CLOUDFUNCTIONS_SERVICE && record.key?('log')
         entry.text_payload = record['log']
-      elsif @service_name == CONTAINER_SERVICE && record.key?('log') &&
-            !is_container_json
+      elsif is_json
+        entry.struct_payload = record
+      elsif @service_name == CONTAINER_SERVICE && record.key?('log')
         entry.text_payload = record['log']
       elsif record.size == 1 && record.key?('message')
         entry.text_payload = record['message']

data/test/plugin/test_out_google_cloud.rb

@@ -14,6 +14,7 @@
 
 require 'helper'
 require 'json'
+require 'time'
 require 'mocha/test_unit'
 require 'webmock/test_unit'
 require 'google/apis'
@@ -82,6 +83,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   CONTAINER_LABEL_KEY = 'component'
   CONTAINER_LABEL_VALUE = 'redis-component'
   CONTAINER_STREAM = 'stdout'
+  CONTAINER_SEVERITY = 'INFO'
+  # Timestamp for 1234567890 seconds and 987654321 nanoseconds since epoch
+  CONTAINER_TIMESTAMP = '2009-02-13T23:31:30.987654321Z'
+  CONTAINER_SECONDS_EPOCH = 1_234_567_890
+  CONTAINER_NANOS = 987_654_321
 
   # Cloud Functions specific labels
   CLOUDFUNCTIONS_FUNCTION_NAME = '$My_Function.Name-@1'
@@ -583,6 +589,46 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_struct_payload_json_log
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    d = create_driver
+    json_string = '{"msg": "test log entry 0", "tag2": "test", "data": 5000}'
+    d.emit('message' => 'notJSON ' + json_string)
+    d.emit('message' => json_string)
+    d.emit('message' => "\t" + json_string)
+    d.emit('message' => '  ' + json_string)
+    d.run
+    verify_log_entries(4, COMPUTE_PARAMS, '') do |entry|
+      assert entry.key?('textPayload'), 'Entry did not have textPayload'
+    end
+  end
+
+  def test_struct_payload_json_container_log
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
+    json_string = '{"msg": "test log entry 0", "tag2": "test", "data": 5000}'
+    d.emit(container_log_entry_with_metadata('notJSON' + json_string))
+    d.emit(container_log_entry_with_metadata(json_string))
+    d.emit(container_log_entry_with_metadata("  \r\n \t" + json_string))
+    d.run
+    log_index = 0
+    verify_log_entries(3, CONTAINER_FROM_METADATA_PARAMS, '') do |entry|
+      log_index += 1
+      if log_index == 1
+        assert entry.key?('textPayload'), 'Entry did not have textPayload'
+      else
+        assert entry.key?('structPayload'), 'Entry did not have structPayload'
+        assert_equal 3, entry['structPayload'].size, entry
+        assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
+        assert_equal 'test', entry['structPayload']['tag2'], entry
+        assert_equal 5000, entry['structPayload']['data'], entry
+      end
+    end
+  end
+
   def test_timestamps
     setup_gce_metadata_stubs
     setup_logging_stubs
@@ -843,7 +889,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     d.emit(container_log_entry_with_metadata(log_entry(0)))
     d.run
-    verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS)
+    verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS) do |entry|
+      assert_equal CONTAINER_SECONDS_EPOCH, \
+                   entry['metadata']['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, \
+                   entry['metadata']['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+    end
   end
 
   def test_multiple_container_logs_metadata_from_plugin
@@ -858,7 +910,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       @logs_sent = []
       n.times { |i| d.emit(container_log_entry_with_metadata(log_entry(i))) }
       d.run
-      verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS)
+      verify_log_entries(n, CONTAINER_FROM_METADATA_PARAMS) do |entry|
+        assert_equal CONTAINER_SECONDS_EPOCH, \
+                     entry['metadata']['timestamp']['seconds'], entry
+        assert_equal CONTAINER_NANOS, \
+                     entry['metadata']['timestamp']['nanos'], entry
+        assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+      end
     end
   end
 
@@ -869,7 +927,13 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     d.emit(container_log_entry(log_entry(0)))
     d.run
-    verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS)
+    verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS) do |entry|
+      assert_equal CONTAINER_SECONDS_EPOCH, \
+                   entry['metadata']['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, \
+                   entry['metadata']['timestamp']['nanos'], entry
+      assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+    end
   end
 
   def test_multiple_container_logs_metadata_from_tag
@@ -884,7 +948,32 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       @logs_sent = []
       n.times { |i| d.emit(container_log_entry(log_entry(i))) }
       d.run
-      verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS)
+      verify_log_entries(n, CONTAINER_FROM_TAG_PARAMS) do |entry|
+        assert_equal CONTAINER_SECONDS_EPOCH, \
+                     entry['metadata']['timestamp']['seconds'], entry
+        assert_equal CONTAINER_NANOS, \
+                     entry['metadata']['timestamp']['nanos'], entry
+        assert_equal CONTAINER_SEVERITY, entry['metadata']['severity'], entry
+      end
+    end
+  end
+
+  def test_one_container_log_from_tag_stderr
+    setup_gce_metadata_stubs
+    setup_container_metadata_stubs
+    setup_logging_stubs
+    d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
+    d.emit(container_log_entry(log_entry(0), 'stderr'))
+    d.run
+    expected_params = CONTAINER_FROM_TAG_PARAMS.merge(
+      labels: { "#{CONTAINER_SERVICE_NAME}/stream" => 'stderr' }
+    ) { |_, oldval, newval| oldval.merge(newval) }
+    verify_log_entries(1, expected_params) do |entry|
+      assert_equal CONTAINER_SECONDS_EPOCH, \
+                   entry['metadata']['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, \
+                   entry['metadata']['timestamp']['nanos'], entry
+      assert_equal 'ERROR', entry['metadata']['severity'], entry
     end
   end
 
@@ -894,7 +983,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_logging_stubs
     d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     d.emit(container_log_entry_with_metadata('{"msg": "test log entry 0", ' \
-                                             '"tag2": "test", "data": 5000}'))
+                                             '"tag2": "test", "data": 5000, ' \
+                                             '"severity": "WARNING"}'))
     d.run
     verify_log_entries(1, CONTAINER_FROM_METADATA_PARAMS,
                        'structPayload') do |entry|
@@ -902,6 +992,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
       assert_equal 'test', entry['structPayload']['tag2'], entry
       assert_equal 5000, entry['structPayload']['data'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, \
+                   entry['metadata']['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, \
+                   entry['metadata']['timestamp']['nanos'], entry
+      assert_equal 'WARNING', entry['metadata']['severity'], entry
     end
   end
 
@@ -911,7 +1006,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     setup_logging_stubs
     d = create_driver(APPLICATION_DEFAULT_CONFIG, CONTAINER_TAG)
     d.emit(container_log_entry('{"msg": "test log entry 0", ' \
-                               '"tag2": "test", "data": 5000}'))
+                               '"tag2": "test", "data": 5000, ' \
+                               '"severity": "W"}'))
     d.run
     verify_log_entries(1, CONTAINER_FROM_TAG_PARAMS,
                        'structPayload') do |entry|
@@ -919,6 +1015,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
       assert_equal 'test log entry 0', entry['structPayload']['msg'], entry
       assert_equal 'test', entry['structPayload']['tag2'], entry
       assert_equal 5000, entry['structPayload']['data'], entry
+      assert_equal CONTAINER_SECONDS_EPOCH, \
+                   entry['metadata']['timestamp']['seconds'], entry
+      assert_equal CONTAINER_NANOS, \
+                   entry['metadata']['timestamp']['nanos'], entry
+      assert_equal 'WARNING', entry['metadata']['severity'], entry
     end
   end
 
@@ -1250,7 +1351,8 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   def container_log_entry_with_metadata(log)
     {
       log: log,
-      stream: 'stdout',
+      stream: CONTAINER_STREAM,
+      time: CONTAINER_TIMESTAMP,
       kubernetes: {
         namespace_id: CONTAINER_NAMESPACE_ID,
         namespace_name: CONTAINER_NAMESPACE_NAME,
@@ -1264,10 +1366,11 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     }
   end
 
-  def container_log_entry(log)
+  def container_log_entry(log, stream = CONTAINER_STREAM)
     {
       log: log,
-      stream: 'stdout'
+      stream: stream,
+      time: CONTAINER_TIMESTAMP
     }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Todd Derr
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-11 00:00:00.000000000 Z
+date: 2016-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -18,6 +18,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '0.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -25,6 +28,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '0.13'
 - !ruby/object:Gem::Dependency
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement