fluent-plugin-gcloud-pubsub-custom-subscriber 1.3.2 → 1.3.3Z

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94cef955f491241979aa15a1deea1e3adf54c246b1085dee0bf4e7c8cfded20b
-  data.tar.gz: 939bb509e4376aa1dce659e77013fcec4520e74b65a0a97f94761fa579bf2264
+  metadata.gz: 4d98636b923d8f1e55b85770bf3c950590c994edea4e0232851ae843503a0a84
+  data.tar.gz: d9610eab7e90766acdcd9a19860e192e4b18240fc9f11ca789d532efa39c1827
 SHA512:
-  metadata.gz: a41456349ce256842625645c3103c414b0b4be3897f235252f895eb9384d1e3e5a7142c7a72129b0f27de318ca95ef46a1a4dc398fbc44b3c66e6d7fe678634c
-  data.tar.gz: b86f5aba05bd90071bd9f87fec73c6f6e91c712a415a066285de31201a968758b0279b1631241460148ec75745ee55927c667c04e67c47a5938c0c1c31a4ad66
+  metadata.gz: 57fce3fcc4d9d76e5df6b48563e1145c7313b9e3240fdc832ed2b53d04a29650b36fb3bf88db26b78e0369acfe5721f2cfd6b4bf4f9b2fffc8daa1043956e7a3
+  data.tar.gz: e846defa23a5de13179171221a4b124a3059ce9c5230e38cd184454626c4000ffd0c1e0f92f472393fec5f992b81de172056d23e9c8fce4e2719778993f29565

data/.travis.yml

@@ -2,10 +2,9 @@ language: ruby
 
 rvm:
   - 2.1
-  - 2.2
-  - 2.3.6
-  - 2.4.3
-  - 2.5.0
+  - 2.2.5
+  - 2.3.3
+  - 2.4.0
   - ruby-head
 
 gemfile:

data/CHANGELOG.md

@@ -1,33 +1,5 @@
 ## ChangeLog
 
-### Release 1.3.1 - 2018/06/08
-
-- Input plugin
-  - Dumps raw string to logs for parsing errors
-
-### Release 1.3.0 - 2018/05/18
-
-- Make attributes available 
-
-### Release 1.2.0 - 2018/04/24
-
-- Output plugin
-  - Support inject section 
-
-### Release 1.1.0 - 2018/03/30
-
-- Output plugin
-  - The placeholder is now available in topic param
-
-### Release 1.0.3 - 2018/03/29
-
-- Bump up google-cloud-pubsub to v0.30.x
-- Make config file Fluentd v1 compatible
-
-### Release 1.0.2 - 2017/09/11
-
-- Bump up google-cloud-pubsub to v0.27.x
-
 ### Release 1.0.1 - 2017/09/05
 
 - Bump up google-cloud-pubsub to v0.26.x

data/README.md

@@ -19,10 +19,6 @@ This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
 - Add a pull style subscription to the topic
 - Download your credential (json) or [set scope on GCE instance](https://cloud.google.com/compute/docs/api/how-tos/authorization)
 
-When using output plugin, you need to grant Pub/Sub Publisher and Pub/Sub Viewer role to IAM.
-
-Also, when using input plugin, you need to grant Pub/Sub Subscriber and Pub/Sub Viewer role to IAM.
-
 ## Requirements
 
 | fluent-plugin-gcloud-pubsub-custom | fluentd | ruby |
@@ -60,13 +56,11 @@ Use `gcloud_pubsub` output plugin.
   max_messages 1000
   max_total_size 9800000
   max_message_size 4000000
-  <buffer>
-    @type memory
-    flush_interval 1s
-  </buffer>
-  <format>
-    @type json
-  </format>
+  buffer_type file
+  buffer_path /path/to/your/buffer
+  flush_interval 1s
+  try_flush_interval 0.1
+  format json
 </match>
 ```
 
@@ -80,7 +74,6 @@ Use `gcloud_pubsub` output plugin.
   - You can also use environment variable such as `GCLOUD_KEYFILE`.
 - `topic` (required)
   - Set topic name to publish.
-  - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
 - `autocreate_topic` (optional, default: `false`)
   - If set to `true`, specified topic will be created when it doesn't exist.
 - `max_messages` (optional, default: `1000`)
@@ -91,8 +84,10 @@ Use `gcloud_pubsub` output plugin.
     - See https://cloud.google.com/pubsub/quotas#other_limits
 - `max_message_size` (optional, default: `4000000` = `4MB`)
   - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
-- `attribute_keys` (optional, default: `[]`)
-  - Publishing the set fields as attributes.
+- `buffer_type`, `buffer_path`, `flush_interval`, `try_flush_interval`
+  - These are fluentd buffer configuration. See http://docs.fluentd.org/articles/buffer-plugin-overview
+- `format` (optional, default: `json`)
+  - Set output format. See http://docs.fluentd.org/articles/out_file#format
 
 ### Pull messages
 
@@ -110,13 +105,11 @@ Use `gcloud_pubsub` input plugin.
   return_immediately true
   pull_interval 0.5
   pull_threads 2
+  format json
   parse_error_action exception
   enable_rpc true
   rpc_bind 0.0.0.0
   rpc_port 24680
-  <parse>
-    @type json
-  </parse>
 </source>
 ```
 
@@ -146,8 +139,8 @@ Use `gcloud_pubsub` input plugin.
   - Pulling messages by intervals of specified seconds.
 - `pull_threads` (optional, default: `1`)
   - Set number of threads to pull messages.
-- `attribute_keys` (optional, default: `[]`)
-  - Specify the key of the attribute to be emitted as the field of record.
+- `format` (optional, default: `json`)
+  - Set input format. See format section in http://docs.fluentd.org/articles/in_tail
 - `parse_error_action` (optional, default: `exception`)
   - Set error type when parsing messages fails.
     - `exception`: Raise exception. Messages are not acknowledged.

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -3,12 +3,11 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-gcloud-pubsub-custom-subscriber"
-  gem.description = "Google Cloud Pub/Sub input/output plugin for Fluentd event collector without auto-create topic requiring only Pub/Sub subscriber ACL"
   gem.license     = "MIT"
-  gem.homepage    = "https://github.com/KMK-ONLINE/fluent-plugin-gcloud-pubsub-custom/tree/v1.0.1-kmk"
-  gem.summary     = gem.description
-  gem.version     = "1.3.2"
-  gem.authors     = ["Yoshihiro MIYAI"]
+  gem.description = "Google Cloud Pub/Sub input/output plugin for Fluentd event collector without auto-create topic requiring only Pub/Sub subscriber ACL"
+  gem.summary     = "Google Cloud Pub/Sub input/output plugin for Fluentd event collector"
+  gem.version     = "1.3.3Z"
+  gem.authors     = ["Yoshihiro MIYAI", "mjchuable"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false
   gem.files       = `git ls-files`.split("\n")
@@ -16,11 +15,15 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
-  gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.30.0"
+  # fluentd version matrix against td-agent
+  # https://support.treasuredata.com/hc/en-us/articles/360001479187-The-td-agent-ChangeLog
+  gem.add_runtime_dependency "fluentd", "0.14.25" # for td-agent v3.1.0
+
+  gem.add_runtime_dependency "google-cloud-pubsub", "0.30.2"
+  gem.add_runtime_dependency "retryable", "2.0.4"
 
-  gem.add_development_dependency "bundler"
-  gem.add_development_dependency "rake"
-  gem.add_development_dependency "test-unit"
-  gem.add_development_dependency "test-unit-rr"
+  gem.add_development_dependency "bundler", "1.16.6"
+  gem.add_development_dependency "rake", "12.1.0"
+  gem.add_development_dependency "test-unit", "3.2.5"
+  gem.add_development_dependency "test-unit-rr", "1.0.5"
 end

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -1,4 +1,5 @@
 require 'google/cloud/pubsub'
+require 'retryable'
 
 module Fluent
   module GcloudPubSub
@@ -7,48 +8,23 @@ module Fluent
     class RetryableError < Error
     end
 
-    class Message
-      attr_reader :message, :attributes
-      def initialize(message, attributes={})
-        @message = message
-        @attributes = attributes
-      end
-
-      def bytesize()
-        attr_size = 0
-        @attributes.each do |key, val|
-          attr_size += key.bytesize + val.bytesize
-        end
-        @message.bytesize + attr_size
-      end
-    end
-
     class Publisher
-      def initialize(project, key, autocreate_topic)
-        @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
-        @autocreate_topic = autocreate_topic
-        @topics = {}
-      end
-
-      def topic(topic_name)
-        return @topics[topic_name] if @topics.has_key? topic_name
-
-        client = @pubsub.topic topic_name
-        if client.nil? && @autocreate_topic
-          client = @pubsub.create_topic topic_name
-        end
-        if client.nil?
-          raise Error.new "topic:#{topic_name} does not exist."
+      RETRY_COUNT = 5
+      RETRYABLE_ERRORS = [Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError]
+
+      # autocreate_topic is unused
+      def initialize(project, key, topic_name, skip_lookup)
+        Retryable.retryable(tries: RETRY_COUNT, on: RETRYABLE_ERRORS) do
+          pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
+          @client = pubsub.topic topic_name, skip_lookup: skip_lookup
         end
-
-        @topics[topic_name] = client
-        client
+        raise Error.new "topic:#{topic_name} does not exist." if @client.nil?
       end
 
-      def publish(topic_name, messages)
-        topic(topic_name).publish do |batch|
+      def publish(messages)
+        @client.publish do |batch|
           messages.each do |m|
-            batch.publish m.message, m.attributes
+            batch.publish m
           end
         end
       rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
@@ -57,10 +33,15 @@ module Fluent
     end
 
     class Subscriber
+      RETRY_COUNT = 5
+      RETRYABLE_ERRORS = [Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError]
+
       def initialize(project, key, topic_name, subscription_name)
-        pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
-        topic = pubsub.topic topic_name
-        @client = topic.subscription subscription_name
+        Retryable.retryable(tries: RETRY_COUNT, on: RETRYABLE_ERRORS) do
+          pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
+          topic = pubsub.topic topic_name
+          @client = topic.subscription subscription_name
+        end
         raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
       end
 

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -37,8 +37,8 @@ module Fluent::Plugin
     config_param :return_immediately, :bool,    default: true
     desc 'Set number of threads to pull messages.'
     config_param :pull_threads,       :integer, default: 1
-    desc 'Specify the key of the attribute to be acquired as a record'
-    config_param :attribute_keys,     :array,   default: []
+    desc 'Set input format.'
+    config_param :format,             :string,  default: DEFAULT_PARSER_TYPE
     desc 'Set error type when parsing messages fails.'
     config_param :parse_error_action, :enum,    default: :exception, list: [:exception, :warning]
     # for HTTP RPC
@@ -222,20 +222,15 @@ module Fluent::Plugin
 
       messages.each do |m|
         line = m.message.data.chomp
-        attributes = m.attributes
         @parser.parse(line) do |time, record|
           if time && record
-            @attribute_keys.each do |key|
-              record[key] = attributes[key]
-            end
-
             event_streams[@extract_tag.call(record)].add(time, record)
           else
             case @parse_error_action
             when :exception
-              raise FailedParseError.new "pattern not match: #{line}"
+              raise FailedParseError.new "pattern not match: #{line.inspect}"
             else
-              log.warn 'pattern not match', record: line
+              log.warn 'pattern not match', record: line.inspect
             end
           end
         end

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -1,11 +1,9 @@
 require 'fluent/plugin/output'
+
 require 'fluent/plugin/gcloud_pubsub/client'
-require 'fluent/plugin_helper/inject'
 
 module Fluent::Plugin
   class GcloudPubSubOutput < Output
-    include Fluent::PluginHelper::Inject
-
     Fluent::Plugin.register_output('gcloud_pubsub', self)
 
     helpers :compat_parameters, :formatter
@@ -21,14 +19,16 @@ module Fluent::Plugin
     config_param :topic,              :string
     desc "If set to `true`, specified topic will be created when it doesn't exist."
     config_param :autocreate_topic,   :bool,    :default => false
+    desc "Custom param - skips topic retrieval; service accounts can just solely have PubSub Publishing roles when set to true"
+    config_param :skip_lookup,   :bool,    :default => true
     desc 'Publishing messages count per request to Cloud Pub/Sub.'
     config_param :max_messages,       :integer, :default => 1000
     desc 'Publishing messages bytesize per request to Cloud Pub/Sub.'
     config_param :max_total_size,     :integer, :default => 9800000  # 9.8MB
     desc 'Limit bytesize per message.'
     config_param :max_message_size,   :integer, :default => 4000000  # 4MB
-    desc 'Publishing the set field as an attribute'
-    config_param :attribute_keys,     :array,   :default => []
+    desc 'Set output format.'
+    config_param :format,             :string,  :default => 'json'
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -41,22 +41,17 @@ module Fluent::Plugin
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :formatter)
       super
-      placeholder_validate!(:topic, @topic)
       @formatter = formatter_create
     end
 
     def start
       super
-      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
+      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @topic, @skip_lookup
+      log.debug "connected topic:#{@topic} in project #{@project}"
     end
 
     def format(tag, time, record)
-      record = inject_values_to_record(tag, time, record)
-      attributes = {}
-      @attribute_keys.each do |key|
-        attributes[key] = record.delete(key)
-      end
-      [@formatter.format(tag, time, record), attributes].to_msgpack
+      @formatter.format(tag, time, record).to_msgpack
     end
 
     def formatted_to_msgpack_binary?
@@ -68,19 +63,16 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      topic = extract_placeholders(@topic, chunk.metadata)
-
       messages = []
       size = 0
 
-      chunk.msgpack_each do |msg, attr|
-        msg = Fluent::GcloudPubSub::Message.new(msg, attr)
+      chunk.msgpack_each do |msg|
         if msg.bytesize > @max_message_size
           log.warn 'Drop a message because its size exceeds `max_message_size`', size: msg.bytesize
           next
         end
         if messages.length + 1 > @max_messages || size + msg.bytesize > @max_total_size
-          publish(topic, messages)
+          publish messages
           messages = []
           size = 0
         end
@@ -89,11 +81,15 @@ module Fluent::Plugin
       end
 
       if messages.length > 0
-        publish(topic, messages)
+        publish messages
       end
     rescue Fluent::GcloudPubSub::RetryableError => ex
       log.warn "Retryable error occurs. Fluentd will retry.", error_message: ex.to_s, error_class: ex.class.to_s
       raise ex
+    rescue Google::Cloud::UnauthenticatedError => ex
+      log.warn "Encountered UnauthenticatedError, renewing @publisher instance", error_message: ex.to_s, error_class: ex.class.to_s
+      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @topic, @skip_lookup
+      raise ex
     rescue => ex
       log.error "unexpected error", error_message: ex.to_s, error_class: ex.class.to_s
       log.error_backtrace
@@ -102,9 +98,9 @@ module Fluent::Plugin
 
     private
 
-    def publish(topic, messages)
-      log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
-      @publisher.publish(topic, messages)
+    def publish(messages)
+      log.debug "send message topic:#{@topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
+      @publisher.publish messages
     end
   end
 end

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -11,6 +11,8 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       topic topic-test
       subscription subscription-test
       key key-test
+      format json
+      json_parser yajl
   ]
 
   DEFAULT_HOST = '127.0.0.1'
@@ -18,16 +20,13 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
   class DummyInvalidMsgData
     def data
-      'foo:bar'
+      return 'foo:bar'
     end
   end
   class DummyInvalidMessage
     def message
       DummyInvalidMsgData.new
     end
-    def attributes
-      {"attr_1" => "a", "attr_2" => "b"}
-    end
   end
 
   def create_driver(conf=CONFIG)
@@ -56,7 +55,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         return_immediately true
         pull_interval 2
         pull_threads 3
-        attribute_keys attr-test
+        format ltsv
         enable_rpc true
         rpc_bind 127.0.0.1
         rpc_port 24681
@@ -71,7 +70,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(1000, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal(3, d.instance.pull_threads)
-      assert_equal(['attr-test'], d.instance.attribute_keys)
+      assert_equal('ltsv', d.instance.format)
       assert_equal(true, d.instance.enable_rpc)
       assert_equal('127.0.0.1', d.instance.rpc_bind)
       assert_equal(24681, d.instance.rpc_port)
@@ -83,7 +82,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(100, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
       assert_equal(1, d.instance.pull_threads)
-      assert_equal([], d.instance.attribute_keys)
+      assert_equal('json', d.instance.format)
       assert_equal(false, d.instance.enable_rpc)
       assert_equal('0.0.0.0', d.instance.rpc_bind)
       assert_equal(24680, d.instance.rpc_port)
@@ -109,7 +108,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
     end
 
     test '50x error occurred on connecting to Pub/Sub' do
-      @topic_mock.subscription('subscription-test').once do
+      @topic_mock.subscription('subscription-test').times(5) do
         raise Google::Cloud::UnavailableError.new('TEST')
       end
 
@@ -132,16 +131,13 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
   sub_test_case 'emit' do
     class DummyMsgData
       def data
-        '{"foo": "bar"}'
+        return '{"foo": "bar"}'
       end
     end
     class DummyMessage
       def message
         DummyMsgData.new
       end
-      def attributes
-        {"attr_1" => "a", "attr_2" => "b"}
-      end
     end
 
     class DummyMsgDataWithTagKey
@@ -149,7 +145,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         @tag = tag
       end
       def data
-        '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
+        return '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
       end
     end
     class DummyMessageWithTagKey
@@ -159,9 +155,6 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       def message
         DummyMsgDataWithTagKey.new @tag
       end
-      def attributes
-        {"attr_1" => "a", "attr_2" => "b"}
-      end
     end
 
     setup do
@@ -184,13 +177,13 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
     test 'simple' do
       messages = Array.new(1, DummyMessage.new)
       @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
-      @subscriber.acknowledge(messages).at_least(1)
+      @subscriber.acknowledge(messages).once
 
       d = create_driver
       d.run(expect_emits: 1, timeout: 3)
       emits = d.events
 
-      assert(1 <= emits.length)
+      assert_equal(1, emits.length)
       emits.each do |tag, time, record|
         assert_equal("test", tag)
         assert_equal({"foo" => "bar"}, record)
@@ -220,13 +213,13 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         DummyMessage.new
       ]
       @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
-      @subscriber.acknowledge(messages).at_least(1)
+      @subscriber.acknowledge(messages).once
 
       d = create_driver("#{CONFIG}\ntag_key test_tag_key")
       d.run(expect_emits: 1, timeout: 3)
       emits = d.events
 
-      assert(3 <= emits.length)
+      assert_equal(3, emits.length)
       # test tag
       assert_equal("tag1", emits[0][0])
       assert_equal("tag2", emits[1][0])
@@ -247,22 +240,6 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_true d.events.empty?
     end
 
-    test 'with attributes' do
-      messages = Array.new(1, DummyMessage.new)
-      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
-      @subscriber.acknowledge(messages).at_least(1)
-
-      d = create_driver("#{CONFIG}\nattribute_keys attr_1")
-      d.run(expect_emits: 1, timeout: 3)
-      emits = d.events
-
-      assert(1 <= emits.length)
-      emits.each do |tag, time, record|
-        assert_equal("test", tag)
-        assert_equal({"foo" => "bar", "attr_1" => "a"}, record)
-      end
-    end
-
     test 'invalid messages with parse_error_action warning' do
       messages = Array.new(1, DummyInvalidMessage.new)
       @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
@@ -300,7 +277,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
     test 'retry if raised RetryableError on acknowledge' do
       messages = Array.new(1, DummyMessage.new)
       @subscriber.pull(immediate: true, max: 100).at_least(2) { messages }
-      @subscriber.acknowledge(messages).at_least(2) { raise Google::Cloud::UnavailableError.new('TEST') }
+      @subscriber.acknowledge(messages).twice { raise Google::Cloud::UnavailableError.new('TEST') }
 
       d = create_driver("#{CONFIG}\npull_interval 0.5")
       d.run(expect_emits: 2, timeout: 3)

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -22,10 +22,6 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  setup do
-    @time = event_time('2016-07-09 11:12:13 UTC')
-  end
-
   sub_test_case 'configure' do
     test 'default values are configured' do
       d = create_driver(%[
@@ -79,65 +75,41 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
         autocreate_topic true
       ])
 
-      @publisher.publish.once
-      @pubsub_mock.topic("topic-test").once { nil }
-      @pubsub_mock.create_topic("topic-test").once { @publisher }
-      d.run(default_tag: "test") do
-        d.feed(@time, {"a" => "b"})
-      end
+      @pubsub_mock.topic("topic-test", skip_lookup: true).once { @publisher }
+      d.run
     end
 
     test '40x error occurred on connecting to Pub/Sub' do
       d = create_driver
 
-      @pubsub_mock.topic('topic-test').once do
+      @pubsub_mock.topic('topic-test', skip_lookup: true).once do
         raise Google::Cloud::NotFoundError.new('TEST')
       end
 
       assert_raise Google::Cloud::NotFoundError do
-        d.run(default_tag: "test") do
-          d.feed(@time, {"a" => "b"})
-        end
+        d.run {}
       end
     end
 
     test '50x error occurred on connecting to Pub/Sub' do
       d = create_driver
 
-      @pubsub_mock.topic('topic-test').once do
+      @pubsub_mock.topic('topic-test', skip_lookup: true).times(5) do
         raise Google::Cloud::UnavailableError.new('TEST')
       end
 
-      assert_raise Fluent::GcloudPubSub::RetryableError do
-        d.run(default_tag: "test") do
-          d.feed(@time, {"a" => "b"})
-        end
+      assert_raise Google::Cloud::UnavailableError do
+        d.run {}
       end
     end
 
     test 'topic is nil' do
       d = create_driver
 
-      @pubsub_mock.topic('topic-test').once { nil }
+      @pubsub_mock.topic('topic-test', skip_lookup: true).once { nil }
 
       assert_raise Fluent::GcloudPubSub::Error do
-        d.run(default_tag: "test") do
-          d.feed(@time, {"a" => "b"})
-        end
-      end
-    end
-
-    test 'messages exceeding "max_message_size" are not published' do
-      d = create_driver(%[
-        project project-test
-        topic topic-test
-        key key-test
-        max_message_size 1000
-      ])
-
-      @publisher.publish.times(0)
-      d.run(default_tag: "test") do
-        d.feed(@time, {"a" => "a" * 1000})
+        d.run {}
       end
     end
   end
@@ -145,10 +117,14 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
   sub_test_case 'publish' do
     setup do
       @publisher = mock!
-      @pubsub_mock = mock!.topic(anything) { @publisher }
+      @pubsub_mock = mock!.topic(anything, anything) { @publisher }
       stub(Google::Cloud::Pubsub).new { @pubsub_mock }
     end
 
+    setup do
+      @time = event_time('2016-07-09 11:12:13 UTC')
+    end
+
     test 'messages are divided into "max_messages"' do
       d = create_driver
       @publisher.publish.times(2)
@@ -178,6 +154,20 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       end
     end
 
+    test 'messages exceeding "max_message_size" are not published' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        max_message_size 1000
+      ])
+
+      @publisher.publish.times(0)
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "a" * 1000})
+      end
+    end
+
     test 'accept "ASCII-8BIT" encoded multibyte strings' do
       # on fluentd v0.14, all strings treated as "ASCII-8BIT" except specified encoding.
       d = create_driver
@@ -206,21 +196,50 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
         end
       end
     end
+  end
 
-    test 'inject section' do
-      d = create_driver(%[
-        project project-test
-        topic topic-test
-        key key-test
-        <inject>
-          tag_key tag
-        </inject>
-      ])
-      @publisher.publish.once
-      d.run(default_tag: 'test') do
-        d.feed({"foo" => "bar"})
+  sub_test_case 'NotFoundError' do
+    setup do
+      @publisher = mock!
+      @pubsub_mock = mock!
+      @google_cloud_pubsub_stub = stub(Google::Cloud::Pubsub)
+    end
+
+    test 'raises NotFoundError on publishing to non-existing topic' do
+      d = create_driver
+
+      @google_cloud_pubsub_stub.new.once { @pubsub_mock }
+      @pubsub_mock.topic("topic-test", skip_lookup: true).once { @publisher }
+      @publisher.publish.once { raise Google::Cloud::NotFoundError.new('5:Resource not found (resource=marc-sg-publish-something)') }
+
+      assert_raises Google::Cloud::NotFoundError do
+        d.run(default_tag: "test") do
+          d.feed([{'a' => 1, 'b' => 2}])
+        end
       end
-      assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
     end
   end
+
+  sub_test_case 'UnauthenticatedError' do
+    setup do
+      @publisher = mock!
+      @pubsub_mock = mock!
+      @google_cloud_pubsub_stub = stub(Google::Cloud::Pubsub)
+    end
+
+    test 'renew publisher on UnauthenticatedError' do
+      d = create_driver
+
+      @google_cloud_pubsub_stub.new.twice { @pubsub_mock }
+      @pubsub_mock.topic("topic-test", skip_lookup: true).twice { @publisher }
+      @publisher.publish.once { raise Google::Cloud::UnauthenticatedError.new('16:Deadline Exceeded') }
+
+      assert_raises Google::Cloud::UnauthenticatedError do
+        d.run(default_tag: "test") do
+          d.feed([{'a' => 1, 'b' => 2}])
+        end
+      end
+    end
+
+  end
 end

metadata

@@ -1,105 +1,114 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom-subscriber
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.3.3Z
 platform: ruby
 authors:
 - Yoshihiro MIYAI
+- mjchuable
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-18 00:00:00.000000000 Z
+date: 2019-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.14.15
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '2'
+        version: 0.14.25
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.14.15
-    - - "<"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 0.14.25
 - !ruby/object:Gem::Dependency
   name: google-cloud-pubsub
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.30.0
+        version: 0.30.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.30.0
+        version: 0.30.2
+- !ruby/object:Gem::Dependency
+  name: retryable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.4
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.16.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.16.6
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 12.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 12.1.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.5
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.5
 description: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
   without auto-create topic requiring only Pub/Sub subscriber ACL
 email: msparrow17@gmail.com
@@ -121,7 +130,7 @@ files:
 - test/plugin/test_in_gcloud_pubsub.rb
 - test/plugin/test_out_gcloud_pubsub.rb
 - test/test_helper.rb
-homepage: https://github.com/KMK-ONLINE/fluent-plugin-gcloud-pubsub-custom/tree/v1.0.1-kmk
+homepage: 
 licenses:
 - MIT
 metadata: {}
@@ -136,13 +145,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
-summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector without
-  auto-create topic requiring only Pub/Sub subscriber ACL
+summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
 test_files: []