fluent-plugin-gcloud-pubsub-custom 0.4.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9db89e553b8ed2af172483421345f062fb457bdb
-  data.tar.gz: 6a48713f2def51581fcd5745b513a8b5d2f17970
+  metadata.gz: 576bd9856edc014e5c0927f90e615c0405df6651
+  data.tar.gz: 9b1ba5307adcc19b8e996925e0fc0b2b967b2465
 SHA512:
-  metadata.gz: d57966e03358dd7d660f3448ac9106a36496643f9f7109fa75f87f0294bd596775c4ccb7703f689e03de844d7826dc0376858c910b9fa4d02d86af7b4a58f645
-  data.tar.gz: 757ee54ba5d8ccb9872535819cb27242642541cb0b2cb18f7802b8e2f95037ee191cfbaf10616ec7e7a22913c1ba7fb9fa7afb442b92d09c42e9ebced80ac1e8
+  metadata.gz: 3c2998193aa348ffa5bc5b5a1f77d5965e91e2dca4df536e69e4554a976071dfe65d74aa324587b09febd984aa8daf8f13bf28bffcec95d1bf6003cb96a91bbc
+  data.tar.gz: 2a9b00f9aadc38cf51af96c49812eaacee60f43f2f15b5e11eb719f04795a818013c81948970df206b22356f95268b59b6046ca01e74696ddef4469fd0578b64

data/.travis.yml

@@ -9,7 +9,6 @@ rvm:
 
 gemfile:
  - Gemfile
- - Gemfile.v0.12
 
 branches:
   only:

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## ChangeLog
 
+### Release 1.0.0 - 2017/06/23
+
+- Fluentd v0.14 ready
+  - Fluentd v0.12 is not supported in the later version
+
 ### Release 0.4.6 - 2017/05/14
 
 - Output plugin

data/fluent-plugin-gcloud-pubsub-custom.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.license     = "MIT"
   gem.homepage    = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
   gem.summary     = gem.description
-  gem.version     = "0.4.6"
+  gem.version     = "1.0.0"
   gem.authors     = ["Yoshihiro MIYAI"]
   gem.email       = "msparrow17@gmail.com"
   gem.has_rdoc    = false
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
   gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.24.0"
   gem.add_runtime_dependency "retryable", "~> 2.0"
 

data/lib/fluent/plugin/in_gcloud_pubsub.rb

@@ -1,21 +1,18 @@
 require 'json'
 require 'webrick'
 
-require 'fluent/input'
-require 'fluent/parser'
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser'
 
 require 'fluent/plugin/gcloud_pubsub/client'
 
-module Fluent
+module Fluent::Plugin
   class GcloudPubSubInput < Input
     Fluent::Plugin.register_input('gcloud_pubsub', self)
 
-    class << self
-      unless method_defined?(:desc)
-        def desc(description)
-        end
-      end
-    end
+    helpers :compat_parameters, :parser, :thread
+
+    DEFAULT_PARSER_TYPE = 'json'
 
     class FailedParseError < StandardError
     end
@@ -41,7 +38,7 @@ module Fluent
     desc 'Set number of threads to pull messages.'
     config_param :pull_threads,       :integer, default: 1
     desc 'Set input format.'
-    config_param :format,             :string,  default: 'json'
+    config_param :format,             :string,  default: DEFAULT_PARSER_TYPE
     desc 'Set error type when parsing messages fails.'
     config_param :parse_error_action, :enum,    default: :exception, list: [:exception, :warning]
     # for HTTP RPC
@@ -52,12 +49,8 @@ module Fluent
     desc 'Port for HTTP RPC.'
     config_param :rpc_port,           :integer, default: 24680
 
-    unless method_defined?(:log)
-      define_method("log") { $log }
-    end
-
-    unless method_defined?(:router)
-      define_method("router") { Fluent::Engine }
+    config_section :parse do
+      config_set_default :@type, DEFAULT_PARSER_TYPE
     end
 
     class RPCServlet < WEBrick::HTTPServlet::AbstractServlet
@@ -108,6 +101,7 @@ module Fluent
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :parser)
       super
       @rpc_srv = nil
       @rpc_thread = nil
@@ -119,8 +113,7 @@ module Fluent
                        method(:dynamic_tag)
                      end
 
-      @parser = Plugin.new_parser(@format)
-      @parser.configure(conf)
+      @parser = parser_create
     end
 
     def start
@@ -133,23 +126,22 @@ module Fluent
       @emit_guard = Mutex.new
       @stop_subscribing = false
       @subscribe_threads = []
-      @pull_threads.times do
-        @subscribe_threads.push Thread.new(&method(:subscribe))
+      @pull_threads.times do |idx|
+        @subscribe_threads.push thread_create("in_gcloud_pubsub_subscribe_#{idx}".to_sym, &method(:subscribe))
       end
     end
 
     def shutdown
-      super
       if @rpc_srv
         @rpc_srv.shutdown
         @rpc_srv = nil
       end
       if @rpc_thread
-        @rpc_thread.join
         @rpc_thread = nil
       end
       @stop_subscribing = true
       @subscribe_threads.each(&:join)
+      super
     end
 
     def stop_pull
@@ -187,7 +179,7 @@ module Fluent
         }
       )
       @rpc_srv.mount('/api/in_gcloud_pubsub/pull/', RPCServlet, self)
-      @rpc_thread = Thread.new {
+      @rpc_thread = thread_create(:in_gcloud_pubsub_rpc_thread){
         @rpc_srv.start
       }
     end
@@ -225,7 +217,7 @@ module Fluent
 
     def process(messages)
       event_streams = Hash.new do |hsh, key|
-        hsh[key] = MultiEventStream.new
+        hsh[key] = Fluent::MultiEventStream.new
       end
 
       messages.each do |m|

data/lib/fluent/plugin/out_gcloud_pubsub.rb

@@ -1,17 +1,15 @@
-require 'fluent/output'
+require 'fluent/plugin/output'
 
 require 'fluent/plugin/gcloud_pubsub/client'
 
-module Fluent
-  class GcloudPubSubOutput < BufferedOutput
+module Fluent::Plugin
+  class GcloudPubSubOutput < Output
     Fluent::Plugin.register_output('gcloud_pubsub', self)
 
-    class << self
-      unless method_defined?(:desc)
-        def desc(description)
-        end
-      end
-    end
+    helpers :compat_parameters, :formatter
+
+    DEFAULT_BUFFER_TYPE = "memory"
+    DEFAULT_FORMATTER_TYPE = "json"
 
     desc 'Set your GCP project.'
     config_param :project,            :string,  :default => nil
@@ -30,18 +28,18 @@ module Fluent
     desc 'Set output format.'
     config_param :format,             :string,  :default => 'json'
 
-    unless method_defined?(:log)
-      define_method("log") { $log }
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
     end
 
-    unless method_defined?(:router)
-      define_method("router") { Fluent::Engine }
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMATTER_TYPE
     end
 
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :formatter)
       super
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
     end
 
     def start
@@ -54,6 +52,14 @@ module Fluent
       @formatter.format(tag, time, record).to_msgpack
     end
 
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
       messages = []
       size = 0

data/test/plugin/test_in_gcloud_pubsub.rb

@@ -2,6 +2,7 @@ require 'net/http'
 require 'json'
 
 require_relative "../test_helper"
+require "fluent/test/driver/input"
 
 class GcloudPubSubInputTest < Test::Unit::TestCase
   CONFIG = %[
@@ -29,7 +30,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
   end
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::GcloudPubSubInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::GcloudPubSubInput).configure(conf)
   end
 
   def http_get(path)
@@ -164,27 +165,23 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
     end
 
     test 'empty' do
-      @subscriber.pull(immediate: true, max: 100).once { [] }
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { [] }
       @subscriber.acknowledge.times(0)
 
       d = create_driver
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
+      d.run(expect_emits: 1, timeout: 3)
 
-      assert_equal(true, d.emits.empty?)
+      assert_true d.events.empty?
     end
 
     test 'simple' do
       messages = Array.new(1, DummyMessage.new)
-      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
       @subscriber.acknowledge(messages).once
 
       d = create_driver
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      emits = d.emits
+      d.run(expect_emits: 1, timeout: 3)
+      emits = d.events
 
       assert_equal(1, emits.length)
       emits.each do |tag, time, record|
@@ -195,16 +192,14 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
     test 'multithread' do
       messages = Array.new(1, DummyMessage.new)
-      @subscriber.pull(immediate: true, max: 100).twice { messages }
-      @subscriber.acknowledge(messages).twice
+      @subscriber.pull(immediate: true, max: 100).at_least(2) { messages }
+      @subscriber.acknowledge(messages).at_least(2)
 
       d = create_driver("#{CONFIG}\npull_threads 2")
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      emits = d.emits
+      d.run(expect_emits: 2, timeout: 1)
+      emits = d.events
 
-      assert_equal(2, emits.length)
+      assert(2 <= emits.length)
       emits.each do |tag, time, record|
         assert_equal("test", tag)
         assert_equal({"foo" => "bar"}, record)
@@ -217,14 +212,12 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         DummyMessageWithTagKey.new('tag2'),
         DummyMessage.new
       ]
-      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
       @subscriber.acknowledge(messages).once
 
       d = create_driver("#{CONFIG}\ntag_key test_tag_key")
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      emits = d.emits
+      d.run(expect_emits: 1, timeout: 3)
+      emits = d.events
 
       assert_equal(3, emits.length)
       # test tag
@@ -239,69 +232,59 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
     test 'invalid messages with parse_error_action exception ' do
       messages = Array.new(1, DummyInvalidMessage.new)
-      @subscriber.pull(immediate: true, max: 100).once { messages }
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
       @subscriber.acknowledge.times(0)
 
       d = create_driver
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      assert_equal(true, d.emits.empty?)
+      d.run(expect_emits: 1, timeout: 3)
+      assert_true d.events.empty?
     end
 
     test 'invalid messages with parse_error_action warning' do
       messages = Array.new(1, DummyInvalidMessage.new)
-      @subscriber.pull(immediate: true, max: 100).once { messages }
-      @subscriber.acknowledge(messages).once
+      @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
+      @subscriber.acknowledge(messages).at_least(1)
 
       d = create_driver("#{CONFIG}\nparse_error_action warning")
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      assert_equal(true, d.emits.empty?)
+      d.run(expect_emits: 1, timeout: 3)
+      assert_true d.events.empty?
     end
 
     test 'retry if raised error' do
       class UnknownError < StandardError
       end
-      @subscriber.pull(immediate: true, max: 100).twice { raise UnknownError.new('test') }
+      @subscriber.pull(immediate: true, max: 100).at_least(2) { raise UnknownError.new('test') }
       @subscriber.acknowledge.times(0)
 
       d = create_driver(CONFIG + 'pull_interval 0.5')
-      d.run {
-        sleep 0.1 # + 0.5s
-      }
+      d.run(expect_emits: 1, timeout: 0.8)
 
       assert_equal(0.5, d.instance.pull_interval)
-      assert_equal(true, d.emits.empty?)
+      assert_true d.events.empty?
     end
 
     test 'retry if raised RetryableError on pull' do
-      @subscriber.pull(immediate: true, max: 100).twice { raise Google::Cloud::UnavailableError.new('TEST') }
+      @subscriber.pull(immediate: true, max: 100).at_least(2) { raise Google::Cloud::UnavailableError.new('TEST') }
       @subscriber.acknowledge.times(0)
 
       d = create_driver("#{CONFIG}\npull_interval 0.5")
-      d.run {
-        sleep 0.1 # + 0.5s
-      }
+      d.run(expect_emits: 1, timeout: 0.8)
 
       assert_equal(0.5, d.instance.pull_interval)
-      assert_equal(true, d.emits.empty?)
+      assert_true d.events.empty?
     end
 
     test 'retry if raised RetryableError on acknowledge' do
       messages = Array.new(1, DummyMessage.new)
-      @subscriber.pull(immediate: true, max: 100).twice { messages }
+      @subscriber.pull(immediate: true, max: 100).at_least(2) { messages }
       @subscriber.acknowledge(messages).twice { raise Google::Cloud::UnavailableError.new('TEST') }
 
       d = create_driver("#{CONFIG}\npull_interval 0.5")
-      d.run {
-        sleep 0.1 # + 0.5s
-      }
-      emits = d.emits
+      d.run(expect_emits: 2, timeout: 3)
+      emits = d.events
 
       # not acknowledged, but already emitted to engine.
-      assert_equal(2, emits.length)
+      assert(2 <= emits.length)
       emits.each do |tag, time, record|
         assert_equal("test", tag)
         assert_equal({"foo" => "bar"}, record)
@@ -321,10 +304,10 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
         sleep 0.75
         # d.run sleeps 0.5 sec
       }
-      emits = d.emits
+      emits = d.events
 
       assert_equal(1, emits.length)
-      assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
+      assert_true d.instance.instance_variable_get(:@stop_pull)
 
       emits.each do |tag, time, record|
         assert_equal("test", tag)
@@ -341,15 +324,15 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
       d.instance.stop_pull
       assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
 
-      d.run {
+      d.run(expect_emits: 1, timeout: 3) {
         http_get('/api/in_gcloud_pubsub/pull/start')
         sleep 0.75
         # d.run sleeps 0.5 sec
       }
-      emits = d.emits
+      emits = d.events
 
       assert_equal(true, emits.length > 0)
-      assert_equal(false, d.instance.instance_variable_get(:@stop_pull))
+      assert_false d.instance.instance_variable_get(:@stop_pull)
 
       emits.each do |tag, time, record|
         assert_equal("test", tag)
@@ -359,7 +342,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
 
     test 'get status by http rpc when started' do
       d = create_driver("#{CONFIG}\npull_interval 1.0\nenable_rpc true")
-      assert_equal(false, d.instance.instance_variable_get(:@stop_pull))
+      assert_false d.instance.instance_variable_get(:@stop_pull)
 
       d.run {
         res = http_get('/api/in_gcloud_pubsub/pull/status')
@@ -370,7 +353,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
     test 'get status by http rpc when stopped' do
       d = create_driver("#{CONFIG}\npull_interval 1.0\nenable_rpc true")
       d.instance.stop_pull
-      assert_equal(true, d.instance.instance_variable_get(:@stop_pull))
+      assert_true d.instance.instance_variable_get(:@stop_pull)
 
       d.run {
         res = http_get('/api/in_gcloud_pubsub/pull/status')

data/test/plugin/test_out_gcloud_pubsub.rb

@@ -1,6 +1,11 @@
+# coding: utf-8
 require_relative "../test_helper"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
 
 class GcloudPubSubOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   CONFIG = %[
     project project-test
     topic topic-test
@@ -10,7 +15,7 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
   ReRaisedError = Class.new(RuntimeError)
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::GcloudPubSubOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::GcloudPubSubOutput).configure(conf)
   end
 
   setup do
@@ -117,17 +122,18 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
     end
 
     setup do
-      @time = Time.parse('2016-07-09 11:12:13 UTC').to_i
+      @time = event_time('2016-07-09 11:12:13 UTC')
     end
 
     test 'messages are divided into "max_messages"' do
       d = create_driver
       @publisher.publish.times(2)
-      # max_messages is default 1000
-      1001.times do |i|
-        d.emit({"a" => i}, @time)
+      d.run(default_tag: "test") do
+        # max_messages is default 1000
+        1001.times do |i|
+          d.feed(@time, {"a" => i})
+        end
       end
-      d.run
     end
 
     test 'messages are divided into "max_total_size"' do
@@ -140,11 +146,12 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       ])
 
       @publisher.publish.times(2)
-      # 400 * 4 / max_total_size = twice
-      4.times do
-        d.emit({"a" => "a" * 400}, @time)
+      d.run(default_tag: "test") do
+        # 400 * 4 / max_total_size = twice
+        4.times do
+          d.feed(@time, {"a" => "a" * 400})
+        end
       end
-      d.run
     end
 
     test 'messages exceeding "max_message_size" are not published' do
@@ -156,24 +163,27 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       ])
 
       @publisher.publish.times(0)
-      d.emit({"a" => "a" * 1000}, @time)
-      d.run
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "a" * 1000})
+      end
     end
 
     test 'accept "ASCII-8BIT" encoded multibyte strings' do
       # on fluentd v0.14, all strings treated as "ASCII-8BIT" except specified encoding.
       d = create_driver
       @publisher.publish.once
-      d.emit({"a" => "あああ".force_encoding("ASCII-8BIT")}, @time)
-      d.run
+      d.run(default_tag: "test") do
+        d.feed(@time, {"a" => "あああ".force_encoding("ASCII-8BIT")})
+      end
     end
 
     test 'reraise unexpected errors' do
       d = create_driver
       @publisher.publish.once { raise ReRaisedError }
       assert_raises ReRaisedError do
-        d.emit([{'a' => 1, 'b' => 2}])
-        d.run
+        d.run(default_tag: "test") do
+          d.feed([{'a' => 1, 'b' => 2}])
+        end
       end
     end
 
@@ -181,8 +191,9 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
       d = create_driver
       @publisher.publish.once { raise Google::Cloud::UnavailableError.new('TEST') }
       assert_raises Fluent::GcloudPubSub::RetryableError do
-        d.emit([{'a' => 1, 'b' => 2}])
-        d.run
+        d.run(default_tag: "test") do
+          d.feed([{'a' => 1, 'b' => 2}])
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-gcloud-pubsub-custom
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 1.0.0
 platform: ruby
 authors:
 - Yoshihiro MIYAI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-14 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.15
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -124,7 +124,6 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
-- Gemfile.v0.12
 - LICENSE
 - README.md
 - Rakefile

data/Gemfile.v0.12

@@ -1,5 +0,0 @@
-source 'https://rubygems.org'
-
-gem 'fluentd', '~> 0.12.0'
-
-gemspec