fluent-plugin-gcloud-pubsub-custom 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08c5555e386df16b94ce1a8db279f952893b3f3b
-  data.tar.gz: e6eee873e191628778e9793a4545baf4a9d90344
+  metadata.gz: 6537f709d3f3e8849ae455fad4ee15a5f8b36d36
+  data.tar.gz: 72ac72b492b01d4ac1301837f9faeaa185689f72
 SHA512:
-  metadata.gz: eb8869520add948a27084fd46c58740455432f35bc29210efc77e4578901f5b10762c2b24a56eede4df04bfa54b57eaa210de5d022a11bcd5555e925939849aa
-  data.tar.gz: 3d2ef099388aa1a0260061e3cc134d84c9326c29202414643a2fe3612ca27fed9697b763ddd2ead9c3f68aa3ef3adb85d55b1e00de77b1b6d79e035b16e452a7
+  metadata.gz: 21dfe776b5de4edfedc3cd71c47c050934db1b09ddb05f43a6d7c2fff0f7586e450bb669d7b5c9e5b7b5569dec176b6d1fed05f0aee12f3f01091c697f6cfe88
+  data.tar.gz: 0c388a5d3eaf2edd941c81ba76676fc21ca9950b12c3fd6229f8e6981d6affc68ca33dc26d76a151fa4e7e752fbac33ff48cb0b3843b04542b2dd90605ac52d3

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## ChangeLog
 
+- Release 0.3.0 - 2016/10/30
+  - Bump up google-cloud-pubsub to v0.21
+  - Input plugin
+    - Add multithreaded pulling feature
 - Release 0.2.0 - 2016/10/15
   - Input plugin
     - Add HTTP RPC feature

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.2.0"
+  gem.version     = "0.3.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
-  gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.20.0"
+  gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.21.0"
 
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"

data/lib/fluent/plugin/gcloud_pubsub/client.rb

@@ -1,4 +1,4 @@
-require 'google/cloud'
+require 'google/cloud/pubsub'
 
 module Fluent
   module GcloudPubSub
@@ -6,7 +6,7 @@ module Fluent
 
     class Publisher
       def initialize(project, key, topic, autocreate_topic)
-        pubsub = (Google::Cloud.new project, key).pubsub
+        pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
 
         @client = pubsub.topic topic, autocreate: autocreate_topic
         raise Fluent::GcloudPubSub::Error.new "topic:#{topic} does not exist." if @client.nil?
@@ -23,7 +23,7 @@ module Fluent
 
     class Subscriber
       def initialize(project, key, topic, subscription)
-        pubsub = (Google::Cloud.new project, key).pubsub
+        pubsub = Google::Cloud::Pubsub.new project: project, keyfile: key
         topic = pubsub.topic topic
         @client = topic.subscription subscription
         raise Fluent::GcloudPubSub::Error.new "subscription:#{subscription} does not exist." if @client.nil?

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -18,6 +18,7 @@ module Fluent
     config_param :pull_interval,      :float,   default: 5.0
     config_param :max_messages,       :integer, default: 100
     config_param :return_immediately, :bool,    default: true
+    config_param :pull_threads,       :integer, default: 1
     config_param :format,             :string,  default: 'json'
     # for HTTP RPC
     config_param :enable_rpc,         :bool,    default: false
@@ -94,7 +95,10 @@ module Fluent
       log.debug "connected subscription:#{@subscription} in project #{@project}"
 
       @stop_subscribing = false
-      @subscribe_thread = Thread.new(&method(:subscribe))
+      @subscribe_threads = []
+      @pull_threads.times do
+        @subscribe_threads.push Thread.new(&method(:subscribe))
+      end
     end
 
     def shutdown
@@ -108,7 +112,7 @@ module Fluent
         @rpc_thread = nil
       end
       @stop_subscribing = true
-      @subscribe_thread.join
+      @subscribe_threads.each(&:join)
     end
 
     def stop_pull

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -40,6 +40,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         max_messages 1000
         return_immediately true
         pull_interval 2
+        pull_threads 3
         format ltsv
         enable_rpc true
         rpc_bind 127.0.0.1
@@ -54,6 +55,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(2.0, d.instance.pull_interval)
       assert_equal(1000, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
+      assert_equal(3, d.instance.pull_threads)
       assert_equal('ltsv', d.instance.format)
       assert_equal(true, d.instance.enable_rpc)
       assert_equal('127.0.0.1', d.instance.rpc_bind)
@@ -65,6 +67,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       assert_equal(5.0, d.instance.pull_interval)
       assert_equal(100, d.instance.max_messages)
       assert_equal(true, d.instance.return_immediately)
+      assert_equal(1, d.instance.pull_threads)
       assert_equal('json', d.instance.format)
       assert_equal(false, d.instance.enable_rpc)
       assert_equal('0.0.0.0', d.instance.rpc_bind)
@@ -88,8 +91,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       @subscriber = mock!
       @topic_mock = mock!.subscription('subscription-test') { @subscriber }
       @pubsub_mock = mock!.topic('topic-test') { @topic_mock }
-      @gcloud_mock = mock!.pubsub { @pubsub_mock }
-      stub(Google::Cloud).new { @gcloud_mock }
+      stub(Google::Cloud::Pubsub).new { @pubsub_mock }
     end
 
     test 'empty' do
@@ -122,6 +124,24 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       end
     end
 
+    test 'multithread' do
+      messages = Array.new(1, DummyMessage.new)
+      @subscriber.pull(immediate: true, max: 100).twice { messages }
+      @subscriber.acknowledge(messages).twice
+
+      d = create_driver("#{CONFIG}\npull_threads 2")
+      d.run {
+        # d.run sleeps 0.5 sec
+      }
+      emits = d.emits
+
+      assert_equal(2, emits.length)
+      emits.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"foo" => "bar"}, record)
+      end
+    end
+
     test 'invalid messages' do
       class DummyInvalidMsgData
         def data

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -58,8 +58,7 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
     setup do
       @publisher = mock!
       @pubsub_mock = mock!
-      @gcloud_mock = mock!.pubsub { @pubsub_mock }
-      stub(Google::Cloud).new { @gcloud_mock }
+      stub(Google::Cloud::Pubsub).new { @pubsub_mock }
     end
 
     test '"autocreate_topic" is enabled' do
@@ -79,8 +78,7 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
     setup do
       @publisher = mock!
       @pubsub_mock = mock!.topic(anything, anything) { @publisher }
-      @gcloud_mock = mock!.pubsub { @pubsub_mock }
-      stub(Google::Cloud).new { @gcloud_mock }
+      stub(Google::Cloud::Pubsub).new { @pubsub_mock }
     end
 
     setup do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-18 00:00:00.000000000 Z
+date: 2016-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: 0.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: 0.21.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement