fluent-plugin-gcloud-pubsub-custom 0.3.4.pre → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87e0242aec786f1963f7c1591779834c6e9aa24b
-  data.tar.gz: 9d79282d546582ff6f7f3a764411b8932c9965a0
+  metadata.gz: dea78067ca4b8f4a2859ed280ad3e577fee82767
+  data.tar.gz: 3aadcbc365f28e5ec3e80037ebbb8ae7a79e5162
 SHA512:
-  metadata.gz: bac13ad67cced2f8284b588e8cab7b1cf93444cce759c3d1d24c55d80fcccb3ad29804998502dea02d237ff9fde4a030e47766eb144854526cbffb24d7a069b2
-  data.tar.gz: a7bf1b6f604783e5013ab9dc309e7c7f0100c2395cd1668d94c2a208ea121c84bf9168d9a3263005ae098efa8ab4077b9ee8d26fc755cd6d092a125f768d0805
+  metadata.gz: 9b9f6e9c9b3afb9e407f22d5cec7429e03bf196478bd513f74200da3c2449143c17997f49cf38e9bf32234ae75063cd4086c95ee828735b0108f5299132cdabe
+  data.tar.gz: f2af751a0a0c1abc468ad7d675d9d4960c856f4278e87a86e4d19fd96a824a49f6976fd3e2879586d0f42b1fbcd47dabb22008d0d6a912f6e83b116cab33c1f9

data/.travis.yml

@@ -1,12 +1,11 @@
 language: ruby
 
 rvm:
-  - 2.0
   - 2.1
   - 2.2.5
-  - 2.3.1
+  - 2.3.3
+  - 2.4.0
   - ruby-head
-  - rbx
 
 gemfile:
  - Gemfile
@@ -23,8 +22,5 @@ sudo: false
 
 matrix:
   allow_failures:
+    - rvm: 2.4.0
     - rvm: ruby-head
-    - rvm: rbx
-  exclude:
-    - rvm: 2.0
-      gemfile: Gemfile

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## ChangeLog
 
+### Release 0.3.4 - 2017/01/03
+
+- Output plugin
+  - Rescue 50x errors
+- Input plugin
+  - Guard emit to be called with multi-threading
+  - Rescue 50x errors
+  - Enabled to select whether to raise an exception if message processing failed
+
 ### Release 0.3.3 - 2016/12/03
 
 - Input plugin

data/README.md

@@ -7,10 +7,10 @@ This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
 
 ## Overview
 
-[Cloud Pub/Sub](https://cloud.google.com/pubsub/) Input/Output(BufferedOutput) plugin for [Fluentd](http://www.fluentd.org/) with [gcloud](https://googlecloudplatform.github.io/gcloud-ruby/) gem
+[Google Cloud Pub/Sub](https://cloud.google.com/pubsub/) Input/Output(BufferedOutput) plugin for [Fluentd](http://www.fluentd.org/) with [google-cloud](https://googlecloudplatform.github.io/google-cloud-ruby/) gem
 
-- [Publish](https://googlecloudplatform.github.io/gcloud-ruby/docs/v0.12.2/Gcloud/Pubsub/Topic.html#publish-instance_method) messages to Cloud Pub/Sub
-- [Pull](https://googlecloudplatform.github.io/gcloud-ruby/docs/v0.12.2/Gcloud/Pubsub/Subscription.html#pull-instance_method) messages from Cloud Pub/Sub
+- Publish messages to Google Cloud Pub/Sub
+- Pull messages from Google Cloud Pub/Sub
 
 ## Preparation
 
@@ -96,6 +96,7 @@ Use `gcloud_pubsub` input plugin.
   pull_interval 0.5
   pull_threads 2
   format json
+  parse_error_action exception
   enable_rpc true
   rpc_bind 0.0.0.0
   rpc_port 24680
@@ -127,6 +128,10 @@ Use `gcloud_pubsub` input plugin.
   - Set number of threads to pull messages.
 - `format` (optional, default: `json`)
   - Set input format. See format section in http://docs.fluentd.org/articles/in_tail
+- `parse_error_action` (optional, default: `exception`)
+  - Set error type when parsing messages fails.
+    - `exception`: Raise exception. Messages are not acknowledged.
+    - `warning`: Only logging as warning.
 - `enable_rpc` (optional, default: `false`)
   - If `true` is specified, HTTP RPC to stop or start pulling message is enabled.
 - `rpc_bind` (optional, default: `0.0.0.0`)

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.3.4.pre"
+  gem.version     = "0.3.4"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -21,7 +21,7 @@ module Fluent
             batch.publish m
           end
         end
-      rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError => ex
+      rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
         raise RetryableError.new "Google api returns error:#{ex.class.to_s} message:#{ex.to_s}"
       end
     end
@@ -36,6 +36,8 @@ module Fluent
 
       def pull(immediate, max)
         @client.pull immediate: immediate, max: max
+      rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
+        raise RetryableError.new "Google api returns error:#{ex.class.to_s} message:#{ex.to_s}"
       end
 
       def acknowledge(messages)

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -40,6 +40,8 @@ module Fluent
     config_param :pull_threads,       :integer, default: 1
     desc 'Set input format.'
     config_param :format,             :string,  default: 'json'
+    desc 'Set error type when parsing messages fails.'
+    config_param :parse_error_action, :enum,    default: :exception, list: [:exception, :warning]
     # for HTTP RPC
     desc 'If `true` is specified, HTTP RPC to stop or start pulling message is enabled.'
     config_param :enable_rpc,         :bool,    default: false
@@ -191,6 +193,8 @@ module Fluent
       @subscriber.acknowledge messages
 
       log.debug "#{messages.length} message(s) processed"
+    rescue Fluent::GcloudPubSub::RetryableError => ex
+      log.warn "Retryable error occurs. Fluentd will retry.", error_message: ex.to_s, error_class: ex.class.to_s
     rescue => ex
       log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
       log.error_backtrace ex.backtrace
@@ -204,7 +208,12 @@ module Fluent
           if time && record
             es.add(time, record)
           else
-            raise FailedParseError.new "pattern not match: #{line.inspect}"
+            case @parse_error_action
+            when :exception
+              raise FailedParseError.new "pattern not match: #{line.inspect}"
+            else
+              log.warn 'pattern not match', record: line.inspect
+            end
           end
         end
       end

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -16,6 +16,17 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
   DEFAULT_HOST = '127.0.0.1'
   DEFAULT_PORT = 24680
 
+  class DummyInvalidMsgData
+    def data
+      return 'foo:bar'
+    end
+  end
+  class DummyInvalidMessage
+    def message
+      DummyInvalidMsgData.new
+    end
+  end
+
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::GcloudPubSubInput).configure(conf)
   end
@@ -143,18 +154,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       end
     end
 
-    test 'invalid messages' do
-      class DummyInvalidMsgData
-        def data
-          return 'foo:bar'
-        end
-      end
-      class DummyInvalidMessage
-        def message
-          DummyInvalidMsgData.new
-        end
-      end
-
+    test 'invalid messages with parse_error_action exception ' do
       messages = Array.new(1, DummyInvalidMessage.new)
       @subscriber.pull(immediate: true, max: 100).once { messages }
       @subscriber.acknowledge.times(0)
@@ -166,6 +166,18 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(true, d.emits.empty?)
     end
 
+    test 'invalid messages with parse_error_action warning' do
+      messages = Array.new(1, DummyInvalidMessage.new)
+      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.acknowledge(messages).once
+
+      d = create_driver("#{CONFIG}\nparse_error_action warning")
+      d.run {
+        # d.run sleeps 0.5 sec
+      }
+      assert_equal(true, d.emits.empty?)
+    end
+
     test 'retry if raised error' do
       class UnknownError < StandardError
       end
@@ -181,6 +193,19 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(true, d.emits.empty?)
     end
 
+    test 'retry if raised RetryableError' do
+      @subscriber.pull(immediate: true, max: 100).twice { raise Google::Cloud::UnavailableError.new('TEST') }
+      @subscriber.acknowledge.times(0)
+
+      d = create_driver(CONFIG + 'pull_interval 0.5')
+      d.run {
+        sleep 0.1 # + 0.5s
+      }
+
+      assert_equal(0.5, d.instance.pull_interval)
+      assert_equal(true, d.emits.empty?)
+    end
+
     test 'stop by http rpc' do
       messages = Array.new(1, DummyMessage.new)
       @subscriber.pull(immediate: true, max: 100).once { messages }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.3.4.pre
+  version: 0.3.4
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-06 00:00:00.000000000 Z
+date: 2017-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -136,12 +136,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector