fluent-plugin-google-cloud 0.2.2 → 0.2.3

data/CONTRIBUTING

@@ -0,0 +1,24 @@
+Want to contribute? Great! First, read this page (including the small print at the end).
+
+### Before you contribute
+Before we can use your code, you must sign the
+[Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1)
+(CLA), which you can do online. The CLA is necessary mainly because you own the
+copyright to your changes, even after your contribution becomes part of our
+codebase, so we need your permission to use and distribute your code. We also
+need to be sure of various other things—for instance that you'll tell us if you
+know that your code infringes on other people's patents. You don't have to sign
+the CLA until after you've submitted your code for review and a member has
+approved it, but you must do it before we can put your code into our codebase.
+Before you start working on a larger contribution, you should get in touch with
+us first through the issue tracker with your idea so that we can help out and
+possibly guide you. Coordinating up front makes it much easier to avoid
+frustration later on.
+
+### Code reviews
+All submissions, including submissions by project members, require review. We
+use Github pull requests for this purpose.
+
+### The small print
+Contributions made by corporations are covered by a different agreement than
+the one above, the Software Grant and Corporate Contributor License Agreement.

data/Gemfile.lock

@@ -1,16 +1,19 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-google-cloud (0.2.2)
+    fluent-plugin-google-cloud (0.2.3)
       fluentd (>= 0.10)
       google-api-client (>= 0.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (3.2.21)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
     addressable (2.3.8)
     autoparse (0.3.3)
       addressable (>= 2.3.1)
@@ -22,7 +25,7 @@ GEM
     extlib (0.9.16)
     faraday (0.9.1)
       multipart-post (>= 1.2, < 3)
-    fluentd (0.12.7)
+    fluentd (0.12.8)
       cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
@@ -32,8 +35,8 @@ GEM
       tzinfo (>= 1.0.0)
       tzinfo-data (>= 1.0.0)
       yajl-ruby (~> 1.0)
-    google-api-client (0.8.4)
-      activesupport (~> 3.2)
+    google-api-client (0.8.6)
+      activesupport (>= 3.2)
       addressable (~> 2.3)
       autoparse (~> 0.3)
       extlib (~> 0.9)
@@ -43,11 +46,11 @@ GEM
       multi_json (~> 1.10)
       retriable (~> 1.4)
       signet (~> 0.6)
-    googleauth (0.4.0)
+    googleauth (0.4.1)
       faraday (~> 0.9)
       jwt (~> 1.4)
-      logging (~> 1.8)
-      memoist (~> 0.11)
+      logging (~> 2.0)
+      memoist (~> 0.12)
       multi_json (= 1.11)
       signet (~> 0.6)
     http_parser.rb (0.6.0)
@@ -57,10 +60,11 @@ GEM
     launchy (2.4.3)
       addressable (~> 2.3)
     little-plugger (1.1.3)
-    logging (1.8.2)
-      little-plugger (>= 1.1.3)
-      multi_json (>= 1.8.4)
-    memoist (0.11.0)
+    logging (2.0.0)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
+    memoist (0.12.0)
+    minitest (5.6.1)
     msgpack (0.5.11)
     multi_json (1.11.0)
     multipart-post (2.0.0)
@@ -78,7 +82,7 @@ GEM
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2015.2)
+    tzinfo-data (1.2015.4)
       tzinfo (>= 1.0.0)
     webmock (1.21.0)
       addressable (>= 2.3.6)

data/fluent-plugin-google-cloud.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
   gem.homepage      = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.2.2'
+  gem.version       = '0.2.3'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -86,6 +86,8 @@ module Fluent
 
       init_api_client()
 
+      @successful_call = false
+
       # Grab metadata about the Google Compute Engine instance that we're on.
       @project_id = fetch_metadata('project/project-id')
       fully_qualified_zone = fetch_metadata('instance/zone')
@@ -152,6 +154,7 @@ module Fluent
           if (record.has_key?('timeNanos'))
             ts_secs = (record['timeNanos'] / 1000000000).to_i
             ts_nanos = record['timeNanos'] % 1000000000
+            record.delete('timeNanos')
           else
             timestamp = Time.at(time)
             ts_secs = timestamp.tv_sec
@@ -167,11 +170,20 @@ module Fluent
                 'nanos' => ts_nanos
               },
             },
-            'textPayload' => record['message']
-            # TODO: default severity?
           }
           if record.has_key?('severity')
-            entry['metadata']['severity'] = record['severity']
+            entry['metadata']['severity'] = parse_severity(record['severity'])
+            record.delete('severity')
+          else
+            entry['metadata']['severity'] = 'DEFAULT'
+          end
+
+          # use textPayload if the only remainaing key is 'message',
+          # otherwise use a struct.
+          if (record.size == 1 && record.has_key?('message'))
+            entry['textPayload'] = record['message']
+          else
+            entry['structPayload'] = record
           end
           write_log_entries_request['entries'].push(entry)
         end
@@ -191,6 +203,12 @@ module Fluent
             :authenticated => true
           })
           client.execute!(request)
+          # Let the user explicitly know when the first call succeeded,
+          # to aid with verification and troubleshooting.
+          if (!@successful_call)
+            @successful_call = true
+            $log.info "Successfully sent to Google Cloud Logging API."
+          end
         # Allow most exceptions to propagate, which will cause fluentd to
         # retry (with backoff), but in some cases we catch the error and
         # drop the request (we will emit a log message in those cases).
@@ -205,7 +223,6 @@ module Fluent
         rescue JSON::GeneratorError => error
           # This happens if the request contains illegal characters;
           # do not retry it because it will fail repeatedly.
-          dropped = write_log_entries_request['entries'].length
           log_write_failure(write_log_entries_request, error)
         end
       end
@@ -237,10 +254,72 @@ module Fluent
       end
     end
 
+    # Values permitted by the API for 'severity' (which is an enum).
+    VALID_SEVERITIES = Set.new [
+      'DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
+      'ALERT', 'EMERGENCY']
+
+    # Translates other severity strings to one of the valid values above.
+    SEVERITY_TRANSLATIONS = {
+      # log4j levels (both current and obsolete).
+      'WARN' => 'WARNING',
+      'FATAL' => 'CRITICAL',
+      'TRACE' => 'DEBUG',
+      'TRACE_INT' => 'DEBUG',
+      'FINE' => 'DEBUG',
+      'FINER' => 'DEBUG',
+      'FINEST' => 'DEBUG',
+      # single-letter levels.  Note E->ERROR and D->DEBUG.
+      'D' => 'DEBUG',
+      'I' => 'INFO',
+      'N' => 'NOTICE',
+      'W' => 'WARNING',
+      'E' => 'ERROR',
+      'C' => 'CRITICAL',
+      'A' => 'ALERT',
+      # other misc. translations.
+      'ERR' => 'ERROR',
+    }
+
+    def parse_severity(severity_str)
+      # The API is case insensitive, but uppercase to make things simpler.
+      severity = severity_str.upcase.strip
+
+      # If the severity is already valid, just return it.
+      if (VALID_SEVERITIES.include?(severity))
+        return severity
+      end
+
+      # If the severity is an integer (string) return it as an integer,
+      # truncated to the closest valid value (multiples of 100 between 0-800).
+      if (/\A\d+\z/.match(severity))
+        begin
+          numeric_severity = (severity.to_i / 100) * 100
+          if (numeric_severity < 0)
+            return 0
+          elsif (numeric_severity > 800)
+            return 800
+          else
+            return numeric_severity
+          end
+        rescue
+          return 'DEFAULT'
+        end
+      end
+
+      # Try to translate the severity.
+      if (SEVERITY_TRANSLATIONS.has_key?(severity))
+        return SEVERITY_TRANSLATIONS[severity]
+      end
+
+      # If all else fails, use 'DEFAULT'.
+      return 'DEFAULT'
+    end
+
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',
-        :application_version => '0.2.2',
+        :application_version => '0.2.3',
         :retries => 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -119,19 +119,19 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   def test_configure_invalid_configs
     begin
       d = create_driver(INVALID_CONFIG1)
-      assert_false
+      assert false
     rescue Fluent::ConfigError => error
       assert error.message.include? 'private_key_path'
     end
     begin
       d = create_driver(INVALID_CONFIG2)
-      assert_false
+      assert false
     rescue Fluent::ConfigError => error
       assert error.message.include? 'private_key_email'
     end
     begin
       d = create_driver(INVALID_CONFIG3)
-      assert_false
+      assert false
     rescue Fluent::ConfigError => error
       assert error.message.include? 'auth_method'
     end
@@ -166,6 +166,19 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     verify_log_entries(1, COMPUTE_PARAMS)
   end
 
+  def test_struct_payload_log
+    setup_logging_stubs
+    d = create_driver(PRIVATE_KEY_CONFIG)
+    d.emit({'msg' => log_entry(0), 'tag2' => 'test', 'data' => 5000})
+    d.run
+    verify_log_entries(1, COMPUTE_PARAMS, 'structPayload') do |entry|
+      assert_equal 3, entry['structPayload'].size, entry
+      assert_equal "test log entry 0", entry['structPayload']['msg'], entry
+      assert_equal 'test', entry['structPayload']['tag2'], entry
+      assert_equal 5000, entry['structPayload']['data'], entry
+    end
+  end
+
   def test_timestamps
     setup_logging_stubs
     d = create_driver(PRIVATE_KEY_CONFIG)
@@ -194,6 +207,27 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_severities
+    setup_logging_stubs
+    d = create_driver(PRIVATE_KEY_CONFIG)
+    expected_severity = []
+    emit_index = 0
+    # Array of pairs of [parsed_severity, expected_severity]
+    [['INFO', 'INFO'], ['warn', 'WARNING'], ['E', 'ERROR'],
+     ['BLAH', 'DEFAULT'], ['105', 100], ['', 'DEFAULT']].each do |sev|
+      d.emit({'message' => log_entry(emit_index), 'severity' => sev[0]})
+      expected_severity.push(sev[1])
+      emit_index += 1
+    end
+    d.run
+    verify_index = 0
+    verify_log_entries(emit_index, COMPUTE_PARAMS) do |entry|
+      assert_equal expected_severity[verify_index],
+        entry['metadata']['severity'], entry
+      verify_index += 1
+    end
+  end
+
   def test_multiple_logs
     setup_logging_stubs
     d = create_driver(PRIVATE_KEY_CONFIG)
@@ -299,6 +333,84 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  # Make parse_severity public so we can test it.
+  class Fluent::GoogleCloudOutput
+    public :parse_severity
+  end
+
+  def test_parse_severity
+    test_obj = Fluent::GoogleCloudOutput.new
+
+    # known severities should translate to themselves, regardless of case
+    ['DEFAULT', 'DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL',
+     'ALERT', 'EMERGENCY'].each do |severity|
+      assert_equal(severity, test_obj.parse_severity(severity))
+      assert_equal(severity, test_obj.parse_severity(severity.downcase))
+      assert_equal(severity, test_obj.parse_severity(severity.capitalize))
+    end
+
+    # numeric levels
+    assert_equal(0, test_obj.parse_severity('0'))
+    assert_equal(100, test_obj.parse_severity('100'))
+    assert_equal(200, test_obj.parse_severity('200'))
+    assert_equal(300, test_obj.parse_severity('300'))
+    assert_equal(400, test_obj.parse_severity('400'))
+    assert_equal(500, test_obj.parse_severity('500'))
+    assert_equal(600, test_obj.parse_severity('600'))
+    assert_equal(700, test_obj.parse_severity('700'))
+    assert_equal(800, test_obj.parse_severity('800'))
+
+    assert_equal(800, test_obj.parse_severity('900'))
+    assert_equal(0, test_obj.parse_severity('1'))
+    assert_equal(100, test_obj.parse_severity('105'))
+    assert_equal(400, test_obj.parse_severity('420'))
+    assert_equal(700, test_obj.parse_severity('799'))
+
+    assert_equal(100, test_obj.parse_severity('105 '))
+    assert_equal(100, test_obj.parse_severity('     105'))
+    assert_equal(100, test_obj.parse_severity('     105    '))
+
+    assert_equal('DEFAULT', test_obj.parse_severity('-100'))
+    assert_equal('DEFAULT', test_obj.parse_severity('105 100'))
+
+    # synonyms for existing log levels
+    assert_equal('ERROR', test_obj.parse_severity('ERR'))
+    assert_equal('WARNING', test_obj.parse_severity('WARN'))
+    assert_equal('CRITICAL', test_obj.parse_severity('FATAL'))
+    assert_equal('DEBUG', test_obj.parse_severity('TRACE'))
+    assert_equal('DEBUG', test_obj.parse_severity('TRACE_INT'))
+    assert_equal('DEBUG', test_obj.parse_severity('FINE'))
+    assert_equal('DEBUG', test_obj.parse_severity('FINER'))
+    assert_equal('DEBUG', test_obj.parse_severity('FINEST'))
+
+    # single letters.
+    assert_equal('DEBUG', test_obj.parse_severity('D'))
+    assert_equal('INFO', test_obj.parse_severity('I'))
+    assert_equal('NOTICE', test_obj.parse_severity('N'))
+    assert_equal('WARNING', test_obj.parse_severity('W'))
+    assert_equal('ERROR', test_obj.parse_severity('E'))
+    assert_equal('CRITICAL', test_obj.parse_severity('C'))
+    assert_equal('ALERT', test_obj.parse_severity('A'))
+    assert_equal('ERROR', test_obj.parse_severity('e'))
+
+    assert_equal('DEFAULT', test_obj.parse_severity('x'))
+    assert_equal('DEFAULT', test_obj.parse_severity('-'))
+
+    # leading/trailing whitespace should be stripped
+    assert_equal('ERROR', test_obj.parse_severity('  ERROR'))
+    assert_equal('ERROR', test_obj.parse_severity('ERROR  '))
+    assert_equal('ERROR', test_obj.parse_severity('   ERROR  '))
+    assert_equal('ERROR', test_obj.parse_severity("\t  ERROR  "))
+
+    # space in the middle should not be stripped.
+    assert_equal('DEFAULT', test_obj.parse_severity('ER ROR'))
+
+    # anything else should translate to 'DEFAULT'
+    assert_equal('DEFAULT', test_obj.parse_severity(''))
+    assert_equal('DEFAULT', test_obj.parse_severity('garbage'))
+    assert_equal('DEFAULT', test_obj.parse_severity('er'))
+  end
+
   private
 
   def uri_for_log(config)
@@ -340,11 +452,16 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   # The caller can optionally provide a block which is called for each entry.
-  def verify_log_entries(n, params)
+  def verify_log_entries(n, params, payload_type='textPayload')
     i = 0
     @logs_sent.each do |batch|
       batch['entries'].each do |entry|
-        assert_equal "test log entry #{i}", entry['textPayload'], batch
+        assert entry.has_key?(payload_type)
+        if (payload_type == 'textPayload')
+          # Check the payload for textPayload, otherwise it is up to the caller.
+          assert_equal "test log entry #{i}", entry['textPayload'], batch
+        end
+
         assert_equal ZONE, entry['metadata']['zone']
         assert_equal params['service_name'], entry['metadata']['serviceName']
         check_labels entry, batch['commonLabels'], params['labels']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-30 00:00:00.000000000 Z
+date: 2015-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -86,16 +86,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/fluent/plugin/out_google_cloud.rb
+- Gemfile.lock
+- CONTRIBUTING
+- README.rdoc
 - test/helper.rb
-- test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/test_out_google_cloud.rb
+- test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - LICENSE
 - Rakefile
-- fluent-plugin-google-cloud.gemspec
-- lib/fluent/plugin/out_google_cloud.rb
-- Gemfile.lock
 - Gemfile
-- README.rdoc
+- fluent-plugin-google-cloud.gemspec
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud
 licenses:
 - Apache 2.0
@@ -109,18 +110,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2434444153919230667
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2434444153919230667
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -129,5 +124,5 @@ specification_version: 3
 summary: Fluentd plugin to stream logs to the Google Cloud Platform's logging API
 test_files:
 - test/helper.rb
-- test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12
 - test/plugin/test_out_google_cloud.rb
+- test/plugin/data/c31e573fd7f62ed495c9ca3821a5a85cb036dee1-privatekey.p12