fluent-plugin-gcloud-pubsub-custom 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cff8f14b950f9ad31d4ff53e99eca09abb58f9e408da1e39d47c09b339b97fff
-  data.tar.gz: 8a95a2fe872bf460727d20d6061616919de0aec557d5383347cdc43a1f261e6e
+  metadata.gz: a55a4279110490b7ac03da92af18e269d92d5d75af86d294a130acd4fb440364
+  data.tar.gz: cfa6ba338f70ae9826fc71e3733dfb6c42175f55ae7015db38fc1703c643ed33
 SHA512:
-  metadata.gz: 3aa9e0201dc0d387bfb3d051ea9e3a5ac797d9452658ede73e063e0ea75db14ec9614abc28e0b2088f49dd2cd2aabbdc45a6c50fa4e8c2dea7059a76ae6855e9
-  data.tar.gz: b044538a4c7e5c20d5b4ab521088d3e44c5f6f1ea318e7b000ebc511e8ca2ae76e9b095fb07db8cc8a86e1c244c7aed736e8ad0405ede26e309cd037555eaa9f
+  metadata.gz: 00aedffe49e51e556d3b8ef86965b67614aab97d7bb06ef1032d8b826ff8d33ca85383d822430ff8c30f6638372bd2a7f2544d86d6745b9b4f89395af384e226
+  data.tar.gz: 0b4c66aa2418626728eebb048a4e57f1f49915d609bbf028c7979ca0ce79168e36a3d413d79301faf5ce4fb998162ae75e508b784f6764a0d08aad76a6af6d45

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## ChangeLog
 
+### Release 1.1.0 - 2018/03/30
+
+- The placeholder is now available in topic param
+
 ### Release 1.0.3 - 2018/03/29
 
 - Bump up google-cloud-pubsub to v0.30.x

data/README.md

@@ -80,6 +80,7 @@ Use `gcloud_pubsub` output plugin.
   - You can also use environment variable such as `GCLOUD_KEYFILE`.
 - `topic` (required)
   - Set topic name to publish.
+  - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
 - `autocreate_topic` (optional, default: `false`)
   - If set to `true`, specified topic will be created when it doesn't exist.
 - `max_messages` (optional, default: `1000`)

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "1.0.3"
+  gem.version     = "1.1.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -8,20 +8,29 @@ module Fluent
     end
 
     class Publisher
-      def initialize(project, key, topic_name, autocreate_topic)
-        pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
-        @client = pubsub.topic topic_name
-        if @client.nil?
-          if autocreate_topic
-            @client = pubsub.create_topic topic_name
-          else
-            raise Error.new "topic:#{topic_name} does not exist."
-          end
+      def initialize(project, key, autocreate_topic)
+        @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
+        @autocreate_topic = autocreate_topic
+        @topics = {}
+      end
+
+      def topic(topic_name)
+        return @topics[topic_name] if @topics.has_key? topic_name
+
+        client = @pubsub.topic topic_name
+        if client.nil? && @autocreate_topic
+          client = @pubsub.create_topic topic_name
+        end
+        if client.nil?
+          raise Error.new "topic:#{topic_name} does not exist."
         end
+
+        @topics[topic_name] = client
+        client
       end
 
-      def publish(messages)
-        @client.publish do |batch|
+      def publish(topic_name, messages)
+        topic(topic_name).publish do |batch|
           messages.each do |m|
             batch.publish m
           end

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -37,13 +37,13 @@ module Fluent::Plugin
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :formatter)
       super
+      placeholder_validate!(:topic, @topic)
       @formatter = formatter_create
     end
 
     def start
       super
-      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @topic, @autocreate_topic
-      log.debug "connected topic:#{@topic} in project #{@project}"
+      @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
     end
 
     def format(tag, time, record)
@@ -59,6 +59,8 @@ module Fluent::Plugin
     end
 
     def write(chunk)
+      topic = extract_placeholders(@topic, chunk.metadata)
+
       messages = []
       size = 0
 
@@ -68,7 +70,7 @@ module Fluent::Plugin
           next
         end
         if messages.length + 1 > @max_messages || size + msg.bytesize > @max_total_size
-          publish messages
+          publish(topic, messages)
           messages = []
           size = 0
         end
@@ -77,7 +79,7 @@ module Fluent::Plugin
       end
 
       if messages.length > 0
-        publish messages
+        publish(topic, messages)
       end
     rescue Fluent::GcloudPubSub::RetryableError => ex
       log.warn "Retryable error occurs. Fluentd will retry.", error_message: ex.to_s, error_class: ex.class.to_s
@@ -90,9 +92,9 @@ module Fluent::Plugin
 
     private
 
-    def publish(messages)
-      log.debug "send message topic:#{@topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
-      @publisher.publish messages
+    def publish(topic, messages)
+      log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
+      @publisher.publish(topic, messages)
     end
   end
 end

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -22,6 +22,10 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  setup do
+    @time = event_time('2016-07-09 11:12:13 UTC')
+  end
+
   sub_test_case 'configure' do
     test 'default values are configured' do
       d = create_driver(%[
@@ -75,9 +79,12 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
         autocreate_topic true
       ])
 
+      @publisher.publish.once
       @pubsub_mock.topic("topic-test").once { nil }
       @pubsub_mock.create_topic("topic-test").once { @publisher }
-      d.run
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "b"})
+      end
     end
 
     test '40x error occurred on connecting to Pub/Sub' do
@@ -88,7 +95,9 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       end
 
       assert_raise Google::Cloud::NotFoundError do
-        d.run {}
+        d.run(default_tag: "test") do
+          d.feed(@time, {"a" => "b"})
+        end
       end
     end
 
@@ -99,8 +108,10 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
         raise Google::Cloud::UnavailableError.new('TEST')
       end
 
-      assert_raise Google::Cloud::UnavailableError do
-        d.run {}
+      assert_raise Fluent::GcloudPubSub::RetryableError do
+        d.run(default_tag: "test") do
+          d.feed(@time, {"a" => "b"})
+        end
       end
     end
 
@@ -110,7 +121,23 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       @pubsub_mock.topic('topic-test').once { nil }
 
       assert_raise Fluent::GcloudPubSub::Error do
-        d.run {}
+        d.run(default_tag: "test") do
+          d.feed(@time, {"a" => "b"})
+        end
+      end
+    end
+
+    test 'messages exceeding "max_message_size" are not published' do
+      d = create_driver(%[
+        project project-test
+        topic topic-test
+        key key-test
+        max_message_size 1000
+      ])
+
+      @publisher.publish.times(0)
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "a" * 1000})
       end
     end
   end
@@ -122,10 +149,6 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       stub(Google::Cloud::Pubsub).new { @pubsub_mock }
     end
 
-    setup do
-      @time = event_time('2016-07-09 11:12:13 UTC')
-    end
-
     test 'messages are divided into "max_messages"' do
       d = create_driver
       @publisher.publish.times(2)
@@ -155,20 +178,6 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       end
     end
 
-    test 'messages exceeding "max_message_size" are not published' do
-      d = create_driver(%[
-        project project-test
-        topic topic-test
-        key key-test
-        max_message_size 1000
-      ])
-
-      @publisher.publish.times(0)
-      d.run(default_tag: "test") do
-        d.feed(@time, {"a" => "a" * 1000})
-      end
-    end
-
     test 'accept "ASCII-8BIT" encoded multibyte strings' do
       # on fluentd v0.14, all strings treated as "ASCII-8BIT" except specified encoding.
       d = create_driver

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-29 00:00:00.000000000 Z
+date: 2018-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd