fluent-plugin-gcloud-pubsub-custom 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2fd80971b144486ed5200c6e8a3680275f3b2af0
-  data.tar.gz: fcddbba7b68cb1ad6ce4fc72b9bce3df3fce2688
+  metadata.gz: 1c24f666eacd9d35bb07d5eb1aad175fd0c549b3
+  data.tar.gz: 888acd31eb8ca1f74c689c39b21f915f8b9034ab
 SHA512:
-  metadata.gz: bb4c1f75828bb55177f1a35a159e61925c45afb6cb6ecc07d5d6c29dd4215b002289beb293dd2cdf94255444f44469fecd4ede53571b291e0c5d0d732460b964
-  data.tar.gz: 4526515a2b03759ca0c306288d7940a0fdac4b12f80809fed493e10a08c64d9edd43f7ae29a8ff41788dfcc5d9aaafc45a28fb420e54038e02c4efa4e376ff0c
+  metadata.gz: f17fc4b9578f08f4181501a0be689db36dcad82b0474b06646f99edc34f1850eb2273efce945ce098a90257cdd9ded87449a93dea5fc0ca5449b30ba3ddc843f
+  data.tar.gz: 8a937780624f4c02ff5f5d39bc6950a28d8561b1b5494cffb19d0669a713f82f8b7b4cf2789c727ca450d933c9a483275781301aa210d6549e4cb485523505f1

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+## ChangeLog
+
+- Release 0.1.3 - 2016/09/17
+  - Fix error handling and add debug logging in input plugin
+- Release 0.1.2 - 2016/09/11
+  - Change default max message size and add debug message
+- Release 0.1.1 - 2016/08/27
+  - Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
+- Release 0.1.0 - 2016/08/22
+  - Use formatter / parser plugin and add format configuration
+  - Bump up gcloud-ruby to 0.12
+  - Remove dependency on lightening buffer
+  - Fix error caused by Pub/Sub quotas

data/README.md

@@ -41,7 +41,7 @@ Use `gcloud_pubsub` output plugin.
   topic <YOUR TOPIC>
   autocreate_topic false
   max_messages 1000
-  max_total_size 10000000
+  max_total_size 9800000
   buffer_type file
   buffer_path /path/to/your/buffer
   flush_interval 1s
@@ -110,22 +110,12 @@ Use `gcloud_pubsub` input plugin.
   - See maxMessages on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
 - `return_immediately` (optional, default: `true`)
   - See returnImmediately on https://cloud.google.com/pubsub/subscriber#receiving-pull-messages
-  - If `return_immediately` is `true`, this plugin ignore `pull_interval`.
+  - If `return_immediately` is `false`, this plugin ignore `pull_interval`.
 - `pull_interval` (optional, default: `5`)
   - Pulling messages by intervals of specified seconds.
 - `format` (optional, default: `json`)
   - Set input format. See format section in http://docs.fluentd.org/articles/in_tail
 
-## ChangeLog
-
-- Release 0.1.1 - 2016/08/27
-  - Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
-- Release 0.1.0 - 2016/08/22
-  - Use formatter / parser plugin and add format configuration
-  - Bump up gcloud-ruby to 0.12
-  - Remove dependency on lightening buffer
-  - Fix error caused by Pub/Sub quotas
-
 ## Contributing
 
 1. Fork it

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.1.2"
+  gem.version     = "0.1.3"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -49,28 +49,40 @@ module Fluent
 
     def subscribe
       until @stop_subscribing
-        messages = @subscriber.pull @return_immediately, @max_messages
-
-        if messages.length > 0
-          es = parse_messages(messages)
-          unless es.empty?
-            begin
-              router.emit_stream(@tag, es)
-            rescue
-              # ignore errors. Engine shows logs and backtraces.
-            end
-            @subscriber.acknowledge messages
-            log.debug "#{messages.length} message(s) processed"
-          end
-        end
+        _subscribe
 
         if @return_immediately
           sleep @pull_interval
         end
       end
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
-      log.error_backtrace
+    rescue => ex
+      log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
+      log.error_backtrace ex.backtrace
+    end
+
+    def _subscribe
+      messages = @subscriber.pull @return_immediately, @max_messages
+      if messages.length == 0
+        log.debug "no messages are pulled"
+        return
+      end
+
+      es = parse_messages(messages)
+      if es.empty?
+        log.warn "#{messages.length} message(s) are pulled, but no messages are parsed"
+        return
+      end
+
+      begin
+        router.emit_stream(@tag, es)
+      rescue
+        # ignore errors. Engine shows logs and backtraces.
+      end
+      @subscriber.acknowledge messages
+      log.debug "#{messages.length} message(s) processed"
+    rescue => ex
+      log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
+      log.error_backtrace ex.backtrace
     end
 
     def parse_messages(messages)
@@ -90,9 +102,9 @@ module Fluent
           log.warn "pattern not match: #{line.inspect}"
         end
       }
-    rescue => e
-      log.warn line.dump, :error => e.to_s
-      log.debug_backtrace(e.backtrace)
+    rescue => ex
+      log.warn line.dump, error_message: ex.to_s, error_class: ex.class.to_s
+      log.warn_backtrace ex.backtrace
     end
   end
 end

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -4,6 +4,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
   CONFIG = %[
       tag test
       project project-test
+      topic topic-test
       subscription subscription-test
       key key-test
       format json
@@ -44,11 +45,98 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
     test 'default values are configured' do
       d = create_driver
-      assert_equal(nil, d.instance.topic)
       assert_equal(5, d.instance.pull_interval)
       assert_equal(100, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal('json', d.instance.format)
     end
   end
+
+  sub_test_case 'emit' do
+    class DummyMsgData
+      def data
+        return '{"foo": "bar"}'
+      end
+    end
+    class DummyMessage
+      def message
+        DummyMsgData.new
+      end
+    end
+
+    setup do
+      @subscriber = mock!
+      @topic_mock = mock!.subscription('subscription-test') { @subscriber }
+      @pubsub_mock = mock!.topic('topic-test') { @topic_mock }
+      @gcloud_mock = mock!.pubsub { @pubsub_mock }
+      stub(Google::Cloud).new { @gcloud_mock }
+    end
+
+    test 'empty' do
+      @subscriber.pull(immediate: true, max: 100).once { [] }
+      @subscriber.acknowledge.times(0)
+
+      d = create_driver
+      d.run {
+        # d.run sleeps 0.5 sec
+      }
+
+      assert_equal(true, d.emits.empty?)
+    end
+
+    test 'simple' do
+      messages = Array.new(1, DummyMessage.new)
+      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.acknowledge(messages).once
+
+      d = create_driver
+      d.run {
+        # d.run sleeps 0.5 sec
+      }
+      emits = d.emits
+
+      assert_equal(1, emits.length)
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar"}, record)
+      end
+    end
+
+    test 'invalid messages' do
+      class DummyInvalidMsgData
+        def data
+          return 'foo:bar'
+        end
+      end
+      class DummyInvalidMessage
+        def message
+          DummyInvalidMsgData.new
+        end
+      end
+
+      messages = Array.new(1, DummyInvalidMessage.new)
+      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.acknowledge.times(0)
+
+      d = create_driver
+      d.run {
+        # d.run sleeps 0.5 sec
+      }
+      assert_equal(true, d.emits.empty?)
+    end
+
+    test 'retry if raised error' do
+      class UnknownError < StandardError
+      end
+      @subscriber.pull(immediate: true, max: 100).twice { raise UnknownError.new('test') }
+      @subscriber.acknowledge.times(0)
+
+      d = create_driver(CONFIG + 'pull_interval 1')
+      d.run {
+        sleep 1 # + 0.5s
+      }
+
+      assert_equal(true, d.emits.empty?)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-11 00:00:00.000000000 Z
+date: 2016-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -108,6 +108,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.v0.12
 - LICENSE