fluent-plugin-google-cloud 0.3.0 → 0.3.1

data/Gemfile.lock

@@ -0,0 +1,107 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-google-cloud (0.3.0)
+      fluentd (>= 0.10)
+      google-api-client (>= 0.8)
+      googleauth (~> 0.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.3.8)
+    autoparse (0.3.3)
+      addressable (>= 2.3.1)
+      extlib (>= 0.9.15)
+      multi_json (>= 1.0.0)
+    cool.io (1.3.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    extlib (0.9.16)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    fluentd (0.12.8)
+      cool.io (>= 1.2.2, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.5.11, < 0.6.0)
+      sigdump (~> 0.2.2)
+      string-scrub (>= 0.0.3)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    google-api-client (0.8.6)
+      activesupport (>= 3.2)
+      addressable (~> 2.3)
+      autoparse (~> 0.3)
+      extlib (~> 0.9)
+      faraday (~> 0.9)
+      googleauth (~> 0.3)
+      launchy (~> 2.4)
+      multi_json (~> 1.10)
+      retriable (~> 1.4)
+      signet (~> 0.6)
+    googleauth (0.4.1)
+      faraday (~> 0.9)
+      jwt (~> 1.4)
+      logging (~> 2.0)
+      memoist (~> 0.12)
+      multi_json (= 1.11)
+      signet (~> 0.6)
+    http_parser.rb (0.6.0)
+    i18n (0.7.0)
+    json (1.8.2)
+    jwt (1.4.1)
+    launchy (2.4.3)
+      addressable (~> 2.3)
+    little-plugger (1.1.3)
+    logging (2.0.0)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
+    memoist (0.12.0)
+    metaclass (0.0.4)
+    minitest (5.6.1)
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
+    msgpack (0.5.11)
+    multi_json (1.11.0)
+    multipart-post (2.0.0)
+    power_assert (0.2.3)
+    rake (10.4.2)
+    retriable (1.4.1)
+    safe_yaml (1.0.4)
+    sigdump (0.2.2)
+    signet (0.6.0)
+      addressable (~> 2.3)
+      extlib (~> 0.9)
+      faraday (~> 0.9)
+      jwt (~> 1.0)
+      multi_json (~> 1.10)
+    string-scrub (0.0.5)
+    test-unit (3.0.9)
+      power_assert
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2015.4)
+      tzinfo (>= 1.0.0)
+    webmock (1.21.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-google-cloud!
+  mocha (~> 1.1)
+  rake (>= 10.3.2)
+  test-unit (~> 3.0.2)
+  webmock (>= 1.17.0)

data/fluent-plugin-google-cloud.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
   gem.homepage      = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.3.0'
+  gem.version       = '0.3.1'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -100,6 +100,7 @@ module Fluent
       init_api_client()
 
       @successful_call = false
+      @timenanos_warning = false
 
       if @use_metadata_service
         # Grab metadata about the Google Compute Engine instance that we're on.
@@ -167,10 +168,31 @@ module Fluent
           'entries' => [],
         }
         arr.each do |time, record|
-          if (record.has_key?('timeNanos'))
+          next unless record.is_a? Hash
+          if (record.has_key?('timestamp') &&
+              record['timestamp'].has_key?('seconds') &&
+              record['timestamp'].has_key?('nanos'))
+            ts_secs = record['timestamp']['seconds']
+            ts_nanos = record['timestamp']['nanos']
+            record.delete('timestamp')
+          elsif (record.has_key?('timestampSeconds') &&
+                 record.has_key?('timestampNanos'))
+            ts_secs = record['timestampSeconds']
+            ts_nanos = record['timestampNanos']
+            record.delete('timestampSeconds')
+            record.delete('timestampNanos')
+          elsif (record.has_key?('timeNanos'))
+            # This is deprecated since the precision is insufficient.
+            # Use timestampSeconds/timestampNanos instead
             ts_secs = (record['timeNanos'] / 1000000000).to_i
             ts_nanos = record['timeNanos'] % 1000000000
             record.delete('timeNanos')
+            if (!@timenanos_warning)
+              # Warn the user this is deprecated, but only once to avoid spam.
+              @timenanos_warning = true
+              $log.warn ("timeNanos is deprecated - please use " +
+                         "timestampSeconds and timestampNanos instead.")
+            end
           else
             timestamp = Time.at(time)
             ts_secs = timestamp.tv_sec
@@ -203,6 +225,8 @@ module Fluent
           end
           write_log_entries_request['entries'].push(entry)
         end
+        # Don't send an empty request if we rejected all the entries.
+        next if write_log_entries_request['entries'].empty?
 
         # Add a prefix to VMEngines logs to prevent namespace collisions,
         # and also escape the log name.
@@ -264,7 +288,8 @@ module Fluent
     end
 
     def fetch_metadata(metadata_path)
-      open('http://metadata/computeMetadata/v1/' + metadata_path,
+      # Fetch GCE metadata - see https://cloud.google.com/compute/docs/metadata
+      open('http://169.254.169.254/computeMetadata/v1/' + metadata_path,
            {'Metadata-Flavor' => 'Google'}) do |f|
         f.read
       end
@@ -335,7 +360,7 @@ module Fluent
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',
-        :application_version => '0.3.0',
+        :application_version => '0.3.1',
         :retries => 1)
 
       if @auth_method == 'private_key'

data/test/plugin/test_out_google_cloud.rb

@@ -270,7 +270,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     expected_ts = []
     emit_index = 0
     [Time.at(123456.789), Time.at(0), Time.now].each do |ts|
-      # Test both the "native" fluentd timestamp and timeNanos.
+      # Test the "native" fluentd timestamp as well as our nanosecond tags.
       d.emit({'message' => log_entry(emit_index)}, ts.to_f)
       # The native timestamp currently only supports second granularity
       # (fluentd issue #461), so strip nanoseconds from the expected value.
@@ -280,6 +280,14 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
               'timeNanos' => ts.tv_sec * 1000000000 + ts.tv_nsec})
       expected_ts.push(ts)
       emit_index += 1
+      d.emit({'message' => log_entry(emit_index),
+              'timestamp' => {'seconds' => ts.tv_sec, 'nanos' => ts.tv_nsec}})
+      expected_ts.push(ts)
+      emit_index += 1
+      d.emit({'message' => log_entry(emit_index),
+              'timestampSeconds' => ts.tv_sec, 'timestampNanos' => ts.tv_nsec})
+      expected_ts.push(ts)
+      emit_index += 1
     end
     d.run
     verify_index = 0
@@ -330,6 +338,16 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_malformed_log
+    setup_gce_metadata_stubs
+    setup_logging_stubs
+    d = create_driver()
+    # if the entry is not a hash, the plugin should silently drop it.
+    d.emit("a string is not a valid message")
+    d.run
+    assert @logs_sent.empty?
+  end
+
   def test_client_error
     setup_gce_metadata_stubs
     # The API Client should not retry this and the plugin should consume
@@ -511,7 +529,7 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
   end
 
   def stub_metadata_request(metadata_path, response_body)
-    stub_request(:get, 'http://metadata/computeMetadata/v1/' + metadata_path).
+    stub_request(:get, 'http://169.254.169.254/computeMetadata/v1/' + metadata_path).
       to_return(:body => response_body, :status => 200,
                 :headers => {'Content-Length' => response_body.length})
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-08 00:00:00.000000000 Z
+date: 2015-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -144,6 +144,7 @@ files:
 - fluent-plugin-google-cloud.gemspec
 - lib/fluent/plugin/out_google_cloud.rb
 - CONTRIBUTING
+- Gemfile.lock
 - Gemfile
 - README.rdoc
 homepage: https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud