fluent-plugin-gcloud-pubsub-custom 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6537f709d3f3e8849ae455fad4ee15a5f8b36d36
-  data.tar.gz: 72ac72b492b01d4ac1301837f9faeaa185689f72
+  metadata.gz: fca9651c5d6e9f497579b5ee6ba6a42f2bb4e69a
+  data.tar.gz: d3a2a04f0e645a04a9828188715a4fab827cd4c6
 SHA512:
-  metadata.gz: 21dfe776b5de4edfedc3cd71c47c050934db1b09ddb05f43a6d7c2fff0f7586e450bb669d7b5c9e5b7b5569dec176b6d1fed05f0aee12f3f01091c697f6cfe88
-  data.tar.gz: 0c388a5d3eaf2edd941c81ba76676fc21ca9950b12c3fd6229f8e6981d6affc68ca33dc26d76a151fa4e7e752fbac33ff48cb0b3843b04542b2dd90605ac52d3
+  metadata.gz: 14806e97d5234bf531850e890b53066e65deb222e9eb602d509102f09496d38a8082568ae228a9ff74cfb544323327529422de22a39d1ac5c5977ea6208e27ea
+  data.tar.gz: e874e411b67cfaac3e0f13635661206fd50a812104d2ac0bb2cf68eb3d17f3c74d9b05797af1a4b14d9bd4f646602823631f96641640e61f6a2e1ce053a42698

data/CHANGELOG.md

@@ -1,26 +1,44 @@
 ## ChangeLog
 
-- Release 0.3.0 - 2016/10/30
-  - Bump up google-cloud-pubsub to v0.21
-  - Input plugin
-    - Add multithreaded pulling feature
-- Release 0.2.0 - 2016/10/15
-  - Input plugin
-    - Add HTTP RPC feature
-- Release 0.1.4 - 2016/09/19
-  - Input plugin
-    - `pull_interval` can be specified float value
-    - `topic` must be specified
-- Release 0.1.3 - 2016/09/17
-  - Input plugin
-    - Fix error handling and add debug logging
-- Release 0.1.2 - 2016/09/11
-  - Output plugin
-    - Change default max message size and add debug message
-- Release 0.1.1 - 2016/08/27
-  - Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
-- Release 0.1.0 - 2016/08/22
-  - Use formatter / parser plugin and add format configuration
-  - Bump up gcloud-ruby to 0.12
-  - Remove dependency on lightening buffer
-  - Fix error caused by Pub/Sub quotas
+### Release 0.3.1 - 2016/11/03
+
+- Output plugin
+- Improve error handling
+
+### Release 0.3.0 - 2016/10/30
+
+- Bump up google-cloud-pubsub to v0.21
+- Input plugin
+  - Add multithreaded pulling feature
+
+### Release 0.2.0 - 2016/10/15
+
+- Input plugin
+  - Add HTTP RPC feature
+
+### Release 0.1.4 - 2016/09/19
+
+- Input plugin
+  - `pull_interval` can be specified float value
+  - `topic` must be specified
+
+### Release 0.1.3 - 2016/09/17
+
+- Input plugin
+  - Fix error handling and add debug logging
+
+### Release 0.1.2 - 2016/09/11
+
+- Output plugin
+  - Change default max message size and add debug message
+
+### Release 0.1.1 - 2016/08/27
+
+- Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
+
+### Release 0.1.0 - 2016/08/22
+
+- Use formatter / parser plugin and add format configuration
+- Bump up gcloud-ruby to 0.12
+- Remove dependency on lightening buffer
+- Fix error caused by Pub/Sub quotas

data/README.md

@@ -88,6 +88,7 @@ Use `gcloud_pubsub` input plugin.
   max_messages 1000
   return_immediately true
   pull_interval 0.5
+  pull_threads 2
   format json
   enable_rpc true
   rpc_bind 0.0.0.0
@@ -116,6 +117,8 @@ Use `gcloud_pubsub` input plugin.
   - If `return_immediately` is `true` and pulling message is stopped by HTTP RPC, this plugin wait `pull_interval` each pull.
 - `pull_interval` (optional, default: `5.0`)
   - Pulling messages by intervals of specified seconds.
+- `pull_threads` (optional, default: `1`)
+  - Set number of threads to pull messages.
 - `format` (optional, default: `json`)
   - Set input format. See format section in http://docs.fluentd.org/articles/in_tail
 - `enable_rpc` (optional, default: `false`)

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.3.0"
+  gem.version     = "0.3.1"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -2,14 +2,17 @@ require 'google/cloud/pubsub'
 
 module Fluent
   module GcloudPubSub
-    class Error < StandardError; end
+    class Error < StandardError
+    end
+    class RetryableError < Error
+    end
 
     class Publisher
       def initialize(project, key, topic, autocreate_topic)
         pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
 
         @client = pubsub.topic topic, autocreate: autocreate_topic
-        raise Fluent::GcloudPubSub::Error.new "topic:#{topic} does not exist." if @client.nil?
+        raise Error.new "topic:#{topic} does not exist." if @client.nil?
       end
 
       def publish(messages)
@@ -18,6 +21,8 @@ module Fluent
             batch.publish m
           end
         end
+      rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError => ex
+        raise RetryableError.new "Google api returns error:#{ex.class.to_s} message:#{ex.to_s}"
       end
     end
 
@@ -26,7 +31,7 @@ module Fluent
         pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
         topic = pubsub.topic topic
         @client = topic.subscription subscription
-        raise Fluent::GcloudPubSub::Error.new "subscription:#{subscription} does not exist." if @client.nil?
+        raise Error.new "subscription:#{subscription} does not exist." if @client.nil?
       end
 
       def pull(immediate, max)

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -35,15 +35,14 @@ module Fluent
     end
 
     def format(tag, time, record)
-      [tag, time, record].to_msgpack
+      @formatter.format(tag, time, record).to_msgpack
     end
 
     def write(chunk)
       messages = []
       size = 0
 
-      chunk.msgpack_each do |tag, time, record|
-        msg = @formatter.format(tag, time, record)
+      chunk.msgpack_each do |msg|
         if messages.length + 1 > @max_messages || size + msg.bytesize > @max_total_size
           publish messages
           messages = []
@@ -56,10 +55,13 @@ module Fluent
       if messages.length > 0
         publish messages
       end
-    rescue => e
-      log.error "unexpected error", :error=>$!.to_s
+    rescue Fluent::GcloudPubSub::RetryableError => ex
+      log.warn "Retryable error occurs. Fluentd will retry.", error_message: ex.to_s, error_class: ex.class.to_s
+      raise ex
+    rescue => ex
+      log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
       log.error_backtrace
-      raise e
+      raise ex
     end
 
     private

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -120,7 +120,7 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       d.run
     end
 
-    test 'reraise errors' do
+    test 'reraise unexpected errors' do
       d = create_driver
       @publisher.publish.once { raise ReRaisedError }
       assert_raises ReRaisedError do
@@ -128,5 +128,14 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
         d.run
       end
     end
+
+    test 'reraise RetryableError' do
+      d = create_driver
+      @publisher.publish.once { raise Google::Cloud::UnavailableError.new('TEST') }
+      assert_raises Fluent::GcloudPubSub::RetryableError do
+        d.emit([{'a' => 1, 'b' => 2}])
+        d.run
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-30 00:00:00.000000000 Z
+date: 2016-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd