fluent-plugin-google-cloud 0.2.0 → 0.2.1

data/fluent-plugin-google-cloud.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Fluentd plugin to stream logs to the Google Cloud Platform's logging API}
   gem.homepage      = 'https://github.com/GoogleCloudPlatform/fluent-plugin-google-cloud'
   gem.license       = 'Apache 2.0'
-  gem.version       = '0.2.0'
+  gem.version       = '0.2.1'
   gem.authors       = ['Todd Derr', 'Alex Robinson']
   gem.email         = ['salty@google.com']
 

data/lib/fluent/plugin/out_google_cloud.rb

@@ -192,24 +192,44 @@ module Fluent
           })
           client.execute!(request)
         # Allow most exceptions to propagate, which will cause fluentd to
-        # retry (with backoff). However, most ClientErrors indicate a problem
-        # with the request itself and should not be retried - the exception
-        # is 'Invalid Credentials' which we can retry.
+        # retry (with backoff), but in some cases we catch the error and
+        # drop the request (we will emit a log message in those cases).
         rescue Google::APIClient::ClientError => error
-          if (error.message == 'Invalid Credentials')
+          # Most ClientErrors indicate a problem with the request itself and
+          # should not be retried, unless it is an authentication issue, in
+          # which case we will retry the request via re-raising the exception.
+          if (is_retriable_client_error(error))
             raise error
           end
-          # fall through to next rescue clause
+          log_write_failure(write_log_entries_request, error)
         rescue JSON::GeneratorError => error
+          # This happens if the request contains illegal characters;
+          # do not retry it because it will fail repeatedly.
           dropped = write_log_entries_request['entries'].length
-          $log.warn "Dropping #{dropped} log message(s)",
-              :error_class=>error.class.to_s, :error=>error.to_s
+          log_write_failure(write_log_entries_request, error)
         end
       end
     end
 
     private
 
+    RETRIABLE_CLIENT_ERRORS = Set.new [
+      'Invalid Credentials',
+      'Request had invalid credentials.',
+      'The caller does not have permission',
+      'Project has not enabled the API. Please use Google Developers Console to activate the API for your project.',
+      'Unable to fetch access token (no scopes configured?)']
+
+    def is_retriable_client_error(error)
+      return RETRIABLE_CLIENT_ERRORS.include?(error.message)
+    end
+
+    def log_write_failure(request, error)
+      dropped = request['entries'].length
+      $log.warn "Dropping #{dropped} log message(s)",
+        :error_class=>error.class.to_s, :error=>error.to_s
+    end
+
     def fetch_metadata(metadata_path)
       open('http://metadata/computeMetadata/v1/' + metadata_path,
            {'Metadata-Flavor' => 'Google'}) do |f|
@@ -220,7 +240,7 @@ module Fluent
     def init_api_client
       @client = Google::APIClient.new(
         :application_name => 'Fluentd Google Cloud Logging plugin',
-        :application_version => '0.2.0',
+        :application_version => '0.2.1',
         :retries => 1)
 
       if @auth_method == 'private_key'
@@ -238,7 +258,14 @@ module Fluent
 
     def api_client
       if !@client.authorization.expired?
-        @client.authorization.fetch_access_token!
+        begin
+          @client.authorization.fetch_access_token!
+        rescue MultiJson::ParseError
+          # Workaround an issue in the API client; just re-raise a more
+          # descriptive error for the user (which will still cause a retry).
+          raise Google::APIClient::ClientError,
+            'Unable to fetch access token (no scopes configured?)'
+        end
       end
       return @client
     end

data/test/plugin/test_out_google_cloud.rb

@@ -223,23 +223,43 @@ class GoogleCloudOutputTest < Test::Unit::TestCase
     assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :times => 2)
   end
 
-  def test_client_error_invalid_credentials
-    # we expect this to retry once, then throw the error.
+  # helper for the ClientError retriable special cases below.
+  def client_error_helper(message)
     stub_request(:post, uri_for_log(COMPUTE_PARAMS)).to_return(
-        :status => 401, :body => "Invalid Credentials")
+        :status => 401, :body => message)
     d = create_driver(PRIVATE_KEY_CONFIG)
     d.emit({'message' => log_entry(0)})
     exception_count = 0
     begin
       d.run
     rescue Google::APIClient::ClientError => error
-      assert_equal 'Invalid Credentials', error.message
+      assert_equal message, error.message
       exception_count += 1
     end
     assert_requested(:post, uri_for_log(COMPUTE_PARAMS), :times => 2)
     assert_equal 1, exception_count
   end
 
+  def test_client_error_invalid_credentials
+    client_error_helper("Invalid Credentials")
+  end
+
+  def test_client_error_caller_does_not_have_permission
+    client_error_helper("The caller does not have permission")
+  end
+
+  def test_client_error_request_had_invalid_credentials
+    client_error_helper("Request had invalid credentials.")
+  end
+
+  def test_client_error_project_has_not_enabled_the_api
+    client_error_helper("Project has not enabled the API. Please use Google Developers Console to activate the API for your project.")
+  end
+
+  def test_client_error_unable_to_fetch_accesss_token
+    client_error_helper("Unable to fetch access token (no scopes configured?)")
+  end
+
   def test_server_error
     # The API client should retry this once, then throw an exception which
     # gets propagated through the plugin.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-23 00:00:00.000000000 Z
+date: 2015-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd