fluent-plugin-gcloud-pubsub-custom 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ce4671f1f14d950d6c5eed9b5c86fab28c0c0e4e
-  data.tar.gz: d6b615131cdf7f4a3e64d4f37918f22aafe18c57
+  metadata.gz: b6a3dc4854e4b3d760150f6ba6e46513fd432d82
+  data.tar.gz: e79d3538b84e949c8754c4c40f429af5ccfb9b42
 SHA512:
-  metadata.gz: 5e0d49b85278cf8f9faa5fbf9f490930131e769130fb7124aa8cf6991eedb0fc68c5ece236c54047f117ce698de75fa225279e760f1a33e1d7be2fd9261aa766
-  data.tar.gz: a83690434bc3b5ee2a45c2bd0a244fbb1f21ec0d81516fe42661a33f0d1e08452b4df5ead99201bf2b27ba51125b5fe15f9eeec23e344d05ef06ae3a10eae704
+  metadata.gz: 611d5fc134b67c6f51740beeab19fa2ee0fe1a001399fb3cbddb64b71a0187b5bcab8000dac1a007d0758de0d9923f838b424a8e95098e4c95783d6a192141e5
+  data.tar.gz: 8cd2edd9f32f283ba06686a102ee66af5a7c9d86811a86fe38491d336d431f2c85fc1caa21b387cc9328678897c380b1a00d7fca9610b2d412bf5c97542beb22

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## ChangeLog
 
+### Release 0.4.2 - 2017/02/03
+
+- Make retry to get topic/subscription when Pub/Sub API returns 50x code
+
 ### Release 0.4.1 - 2017/02/02
 
 - Bump up google-cloud-pubsub to v0.22.x

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.4.1"
+  gem.version     = "0.4.2"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false
@@ -18,6 +18,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.22.0"
+  gem.add_runtime_dependency "retryable", "~> 2.0"
 
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -1,4 +1,5 @@
 require 'google/cloud/pubsub'
+require 'retryable'
 
 module Fluent
   module GcloudPubSub
@@ -8,11 +9,15 @@ module Fluent
     end
 
     class Publisher
-      def initialize(project, key, topic, autocreate_topic)
-        pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
+      RETRY_COUNT = 5
+      RETRYABLE_ERRORS = [Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError]
 
-        @client = pubsub.topic topic, autocreate: autocreate_topic
-        raise Error.new "topic:#{topic} does not exist." if @client.nil?
+      def initialize(project, key, topic_name, autocreate_topic)
+        Retryable.retryable(tries: RETRY_COUNT, on: RETRYABLE_ERRORS) do
+          pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
+          @client = pubsub.topic topic_name, autocreate: autocreate_topic
+        end
+        raise Error.new "topic:#{topic_name} does not exist." if @client.nil?
       end
 
       def publish(messages)
@@ -27,11 +32,16 @@ module Fluent
     end
 
     class Subscriber
-      def initialize(project, key, topic, subscription)
-        pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
-        topic = pubsub.topic topic
-        @client = topic.subscription subscription
-        raise Error.new "subscription:#{subscription} does not exist." if @client.nil?
+      RETRY_COUNT = 5
+      RETRYABLE_ERRORS = [Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError]
+
+      def initialize(project, key, topic_name, subscription_name)
+        Retryable.retryable(tries: RETRY_COUNT, on: RETRYABLE_ERRORS) do
+          pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
+          topic = pubsub.topic topic_name
+          @client = topic.subscription subscription_name
+        end
+        raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
       end
 
       def pull(immediate, max)

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -87,6 +87,45 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'start' do
+    setup do
+      @topic_mock = mock!
+      @pubsub_mock = mock!.topic('topic-test').at_least(1) { @topic_mock }
+      stub(Google::Cloud::Pubsub).new { @pubsub_mock }
+    end
+
+    test '40x error occurred on connecting to Pub/Sub' do
+      @topic_mock.subscription('subscription-test').once do
+        raise Google::Cloud::NotFoundError.new('TEST')
+      end
+
+      d = create_driver
+      assert_raise Google::Cloud::NotFoundError do
+        d.run {}
+      end
+    end
+
+    test '50x error occurred on connecting to Pub/Sub' do
+      @topic_mock.subscription('subscription-test').times(5) do
+        raise Google::Cloud::UnavailableError.new('TEST')
+      end
+
+      d = create_driver
+      assert_raise Google::Cloud::UnavailableError do
+        d.run {}
+      end
+    end
+
+    test 'subscription is nil' do
+      @topic_mock.subscription('subscription-test').once { nil }
+
+      d = create_driver
+      assert_raise Fluent::GcloudPubSub::Error do
+        d.run {}
+      end
+    end
+  end
+
   sub_test_case 'emit' do
     class DummyMsgData
       def data

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -72,6 +72,40 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       @pubsub_mock.topic("topic-test", autocreate: true).once { @publisher }
       d.run
     end
+
+    test '40x error occurred on connecting to Pub/Sub' do
+      d = create_driver
+
+      @pubsub_mock.topic('topic-test', autocreate: false).once do
+        raise Google::Cloud::NotFoundError.new('TEST')
+      end
+
+      assert_raise Google::Cloud::NotFoundError do
+        d.run {}
+      end
+    end
+
+    test '50x error occurred on connecting to Pub/Sub' do
+      d = create_driver
+
+      @pubsub_mock.topic('topic-test', autocreate: false).times(5) do
+        raise Google::Cloud::UnavailableError.new('TEST')
+      end
+
+      assert_raise Google::Cloud::UnavailableError do
+        d.run {}
+      end
+    end
+
+    test 'topic is nil' do
+      d = create_driver
+
+      @pubsub_mock.topic('topic-test', autocreate: false).once { nil }
+
+      assert_raise Fluent::GcloudPubSub::Error do
+        d.run {}
+      end
+    end
   end
 
   sub_test_case 'publish' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-02 00:00:00.000000000 Z
+date: 2017-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -44,6 +44,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.22.0
+- !ruby/object:Gem::Dependency
+  name: retryable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement