fluent-plugin-mongo 0.8.1 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bf1813aa5586654b52813282aa11f01c7dba05f
-  data.tar.gz: 240c7befe5e653491b962753b250134ab48d2ee6
+  metadata.gz: 130e2e8712112dcda7b7d44b19448f5cd2177ccc
+  data.tar.gz: 79e4d7512049f24a830ccfef2b973084f9847b21
 SHA512:
-  metadata.gz: 9ab2a0129a3bd0b2fa56d520d58a67d1bf545981d3b68ed45b577453a9f29d942b65fb3a126b07e004619b239b643653866e7b31a47b0a8b88e7a625b7644721
-  data.tar.gz: 1e2387e3218da98c9afae121da65f1dd70fff92addef623f2c0d23d4d0054061f51c4c8635d8c1668638abdfad10f78932f49544fb02ccb27c34c97b3c66778e
+  metadata.gz: 635611797a2fae046e0961801e8a7f1fee7d436fcc2356406f5227b78916c6e7feb954b2d1d4d129e4573dffdf9c2627fbada20b35af35663e6ca33237cfe1ff
+  data.tar.gz: 642e6babffd923a6426d0bc4bb85d64608b45e372f8bb3d30de755488737d8c0d4d25eb0266a66ad226c2b9c506115b24d2d07bd08f4032f9c5d49ef33f9e200

data/.travis.yml

@@ -1,14 +1,12 @@
 rvm:
- - 1.9.3
- - 2.0.0
  - 2.1
  - 2.2.4
  - 2.3.1
+ - 2.4.0
  - ruby-head
 
 gemfile:
  - Gemfile
- - Gemfile.v0.12
 
 services:
   - mongodb
@@ -22,10 +20,4 @@ script: bundle exec rake test
 
 matrix:
   allow_failures:
-    - rvm: 1.9.3
     - rvm: ruby-head
-  exclude:
-    - rvm: 1.9.3
-      gemfile: Gemfile
-    - rvm: 2.0.0
-      gemfile: Gemfile

data/ChangeLog

@@ -1,14 +1,3 @@
-Release 0.8.1 - 2017/03/23
-
-	* Add connection_string parameter for MongDB URI
-	* out_mongo_replset: Add nodes parameter
-
-
-Release 0.8.0 - 2017/02/10
-
-	* Update mongo gem dependency to v2.0 or upper
-
-
 Release 0.7.16 - 2016/10/05
 
 	* out_mongo: Log warn / deprecated message for invalid record handling

data/README.rdoc

@@ -2,6 +2,13 @@
 
 fluent-plugin-mongo provides input and output plugins for {Fluentd}[http://fluentd.org] ({GitHub}[http://github.com/fluent/fluentd])
 
+= Requirements
+
+    |fluent-plugin-mongo|   fluentd  |  ruby  |
+    |-------------------|------------|--------|
+    |     >= 1.0.0      | >= 0.14.12 | >= 2.1 |
+    |     <  1.0.0      | ~> 0.12.0  | >= 1.9 |
+
 = Installation
 
 == Gems

data/VERSION

@@ -1 +1 @@
-0.8.1
+1.0.0.rc1

data/fluent-plugin-mongo.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"
   gem.add_development_dependency "test-unit", ">= 3.0.0"
+  gem.add_development_dependency "timecop", "~> 0.8.0"
 end

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -1,23 +1,19 @@
-require 'fluent/input'
-
-module Fluent
+# coding: utf-8
+require 'mongo'
+require 'bson'
+require 'fluent/plugin/input'
+require 'fluent/plugin/mongo_auth'
+require 'fluent/plugin/logger_support'
+
+module Fluent::Plugin
   class MongoTailInput < Input
-    Plugin.register_input('mongo_tail', self)
-
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
+    Fluent::Plugin.register_input('mongo_tail', self)
 
-    # Define `router` method of v0.12 to support v0.10 or earlier
-    unless method_defined?(:router)
-      define_method("router") { ::Fluent::Engine }
-    end
+    helpers :timer
 
-    require 'fluent/plugin/mongo_auth'
-    include MongoAuthParams
-    include MongoAuth
-    require 'fluent/plugin/logger_support'
-    include LoggerSupport
+    include Fluent::MongoAuthParams
+    include Fluent::MongoAuth
+    include Fluent::LoggerSupport
 
     desc "MongoDB database"
     config_param :database, :string, default: nil
@@ -50,8 +46,6 @@ module Fluent
 
     def initialize
       super
-      require 'mongo'
-      require 'bson'
 
       @client_options = {}
       @connection_options = {}
@@ -61,15 +55,15 @@ module Fluent
       super
 
       if !@tag and !@tag_key
-        raise ConfigError, "'tag' or 'tag_key' option is required on mongo_tail input"
+        raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on mongo_tail input"
       end
 
       if @database && @url
-        raise ConfigError, "Both 'database' and 'url' can not be set"
+        raise Fluent::ConfigError, "Both 'database' and 'url' can not be set"
       end
 
       if !@database && !@url
-        raise ConfigError, "One of 'database' or 'url' must be specified"
+        raise Fluent::ConfigError, "One of 'database' or 'url' must be specified"
       end
 
       @last_id = @id_store_file ? get_last_id : nil
@@ -86,7 +80,7 @@ module Fluent
       # Resume tailing from last inserted id.
       # Because tailable option is obsoleted since mongo driver 2.0.
       @last_id = get_last_inserted_id if !@id_store_file and get_last_inserted_id
-      @thread = Thread.new(&method(:run))
+      timer_execute(:in_mongo_tail_watcher, @wait_time, &method(:run))
     end
 
     def shutdown
@@ -95,32 +89,22 @@ module Fluent
         @file.close
       end
 
-      @stop = true
-      @thread.join
       @client.close
 
       super
     end
 
     def run
-      loop {
-        option = {}
-        begin
-          loop {
-            return if @stop
-
-            option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
-            documents = @collection.find(option)
-            if documents.count >= 1
-              process_documents(documents)
-            else
-              sleep @wait_time
-            end
-          }
-        rescue
-          # ignore Exceptions
+      option = {}
+      begin
+        option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
+        documents = @collection.find(option)
+        if documents.count >= 1
+          process_documents(documents)
         end
-      }
+      rescue
+        # ignore Exceptions
+      end
     end
 
     private
@@ -148,13 +132,13 @@ module Fluent
     end
 
     def process_documents(documents)
-      es = MultiEventStream.new
+      es = Fluent::MultiEventStream.new
       documents.each {|doc|
         time = if @time_key
                  t = doc.delete(@time_key)
-                 t.nil? ? Engine.now : t.to_i
+                 t.nil? ? Fluent::Engine.now : t.to_i
                else
-                 Engine.now
+                 Fluent::Engine.now
                end
         @tag = if @tag_key
                 t = doc.delete(@tag_key)

data/lib/fluent/plugin/out_mongo.rb

@@ -1,29 +1,26 @@
-require 'fluent/output'
+require 'mongo'
+require 'msgpack'
+require 'fluent/plugin/output'
+require 'fluent/plugin/mongo_auth'
+require 'fluent/plugin/logger_support'
 
-module Fluent
-  class MongoOutput < BufferedOutput
-    Plugin.register_output('mongo', self)
+module Fluent::Plugin
+  class MongoOutput < Output
+    Fluent::Plugin.register_output('mongo', self)
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
+    helpers :event_emitter, :inject, :compat_parameters
 
-    require 'fluent/plugin/mongo_auth'
-    include MongoAuthParams
-    include MongoAuth
-    require 'fluent/plugin/logger_support'
-    include LoggerSupport
+    include Fluent::MongoAuthParams
+    include Fluent::MongoAuth
+    include Fluent::LoggerSupport
 
-    include SetTagKeyMixin
-    config_set_default :include_tag_key, false
+    DEFAULT_BUFFER_TYPE = "memory"
 
-    include SetTimeKeyMixin
+    config_set_default :include_tag_key, false
     config_set_default :include_time_key, true
 
-    desc "MongoDB connection string"
-    config_param :connection_string, :default => nil
     desc "MongoDB database"
-    config_param :database, :string, :default => nil
+    config_param :database, :string
     desc "MongoDB collection"
     config_param :collection, :string, default: 'untagged'
     desc "MongoDB host"
@@ -53,15 +50,16 @@ module Fluent
     config_param :ssl_verify, :bool, default: false
     config_param :ssl_ca_cert, :string, default: nil
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ['tag']
+    end
+
     attr_reader :client_options, :collection_options
 
     def initialize
       super
 
-      require 'mongo'
-      require 'msgpack'
-
-      @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
     end
@@ -72,10 +70,10 @@ module Fluent
 
     def configure(conf)
       if conf.has_key?('buffer_chunk_limit')
-        configured_chunk_limit_size = Config.size_value(conf['buffer_chunk_limit'])
+        configured_chunk_limit_size = Fluent::Config.size_value(conf['buffer_chunk_limit'])
         estimated_limit_size = LIMIT_AFTER_v1_8
         estimated_limit_size_conf = '8m'
-        if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
+        if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
           estimated_limit_size = LIMIT_BEFORE_v1_8
           estimated_limit_size_conf = '2m'
         end
@@ -84,33 +82,31 @@ module Fluent
           conf['buffer_chunk_limit'] = estimated_limit_size_conf
         end
       else
-        if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
+        if conf.has_key?('mongodb_smaller_bson_limit') && Fluent::Config.bool_value(conf['mongodb_smaller_bson_limit'])
           conf['buffer_chunk_limit'] = '2m'
         else
           conf['buffer_chunk_limit'] = '8m'
         end
       end
+      compat_parameters_convert(conf, :inject)
 
       super
 
-      if @connection_string.nil? && @database.nil?
-        raise Fluent::ConfigError,  "connection_string or database parameter is required"
-      end
-
       unless @ignore_invalid_record
         log.warn "Since v0.8, invalid record detection will be removed because mongo driver v2.x and API spec don't provide it. You may lose invalid records, so you should not send such records to mongo plugin"
       end
 
       if conf.has_key?('tag_mapped')
-        @tag_mapped = true
+        log.warn "'tag_mapped' feature is replaced with built-in config placeholder. Please consider to use 'collection ${tag}'."
+        @collection = '${tag}'
       end
-      raise ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
+      raise Fluent::ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
 
       if conf.has_key?('capped')
-        raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
+        raise Fluent::ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
         @collection_options[:capped] = true
-        @collection_options[:size] = Config.size_value(conf['capped_size'])
-        @collection_options[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
+        @collection_options[:size] = Fluent::Config.size_value(conf['capped_size'])
+        @collection_options[:max] = Fluent::Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
       end
 
       if remove_tag_prefix = conf['remove_tag_prefix']
@@ -128,12 +124,6 @@ module Fluent
         @client_options[:ssl_verify] = @ssl_verify
         @client_options[:ssl_ca_cert] = @ssl_ca_cert
       end
-      @nodes = ["#{@host}:#{@port}"] if @nodes.nil?
-
-      # MongoDB uses BSON's Date for time.
-      def @timef.format_nocache(time)
-        time
-      end
 
       configure_logger(@mongo_log_level)
 
@@ -163,28 +153,36 @@ module Fluent
       [time, record].to_msgpack
     end
 
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
-      collection_name = @tag_mapped ? chunk.key : @collection
+      collection_name = extract_placeholders(@collection, chunk.metadata)
       operate(format_collection_name(collection_name), collect_records(chunk))
     end
 
     private
 
     def client
-      if @connection_string
-        Mongo::Client.new(@connection_string)
-      else
-        @client_options[:database] = @database
-        @client_options[:user] = @user if @user
-        @client_options[:password] = @password if @password
-        Mongo::Client.new(@nodes, @client_options)
-      end
+      @client_options[:database] = @database
+      @client_options[:user] = @user if @user
+      @client_options[:password] = @password if @password
+      Mongo::Client.new(["#{@host}:#{@port}"], @client_options)
     end
 
     def collect_records(chunk)
       records = []
+      time_key = @inject_config.time_key
+      tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
-        record[@time_key] = Time.at(time || record[@time_key]) if @include_time_key
+        record = inject_values_to_record(tag, time, record)
+        # MongoDB uses BSON's Date for time.
+        record[time_key] = Time.at(time || record[time_key]) if time_key
         records << record
       }
       records

data/lib/fluent/plugin/out_mongo_replset.rb

@@ -1,18 +1,12 @@
 require 'fluent/plugin/out_mongo'
 
-module Fluent
+module Fluent::Plugin
   class MongoOutputReplset < MongoOutput
-    Plugin.register_output('mongo_replset', self)
-
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
+    Fluent::Plugin.register_output('mongo_replset', self)
 
     config_set_default :include_tag_key, false
     config_set_default :include_time_key, true
 
-    desc "Nodes of replica set"
-    config_param :nodes, :array, value_type: :string, :default => nil
     desc "Replica set name"
     config_param :replica_set, :string
     desc "Read from specified role"
@@ -20,10 +14,6 @@ module Fluent
     desc "Retry number"
     config_param :num_retries, :integer, :default => 60
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
-
     def configure(conf)
       super
 
@@ -37,6 +27,14 @@ module Fluent
       log.debug "Setup replica set configuration: #{conf['replica_set']}"
     end
 
+    def format(tag, time, record)
+      super
+    end
+
+    def write(chunk)
+      super
+    end
+
     private
 
     def operate(client, records)

data/test/plugin/test_in_mongo_tail.rb

@@ -1,4 +1,7 @@
 require "helper"
+require "fluent/test/driver/input"
+require "fluent/test/helpers"
+require "timecop"
 
 class MongoTailInputTest < Test::Unit::TestCase
   def setup
@@ -44,7 +47,7 @@ class MongoTailInputTest < Test::Unit::TestCase
   end
 
   def create_driver(conf=default_config)
-    Fluent::Test::InputTestDriver.new(Fluent::MongoTailInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MongoTailInput).configure(conf)
   end
 
   def test_configure
@@ -67,6 +70,78 @@ class MongoTailInputTest < Test::Unit::TestCase
     assert_equal(expected, d.instance.mongo_log_level)
   end
 
+  class TailInputTest < self
+    include Fluent::Test::Helpers
+
+    def setup_mongod
+      options = {}
+      options[:database] = database_name
+      @client = ::Mongo::Client.new(["localhost:#{port}"], options)
+      @time = Time.now
+      Timecop.freeze(@time)
+    end
+
+    def teardown_mongod
+      @client[collection_name].drop
+      Timecop.return
+    end
+
+    def test_emit
+      d = create_driver(%[
+        @type mongo_tail
+        database #{database_name}
+        collection #{collection_name}
+        tag input.mongo
+        time_key time
+      ])
+      d.run(expect_records: 1, timeout: 5) do
+        @client[collection_name].insert_one({message: "test"})
+      end
+      events = d.events
+      assert_equal "input.mongo", events[0][0]
+      assert_equal event_time(@time.to_s), events[0][1]
+      assert_equal "test", events[0][2]["message"]
+    end
+
+    def test_emit_with_tag_time_keys
+      d = create_driver(%[
+        @type mongo_tail
+        database #{database_name}
+        collection #{collection_name}
+        tag input.mongo
+        tag_key tag
+        time_key time
+      ])
+      d.run(expect_records: 1, timeout: 5) do
+        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Fluent::Engine.now})
+      end
+      events = d.events
+      assert_equal "user.defined", events[0][0]
+      assert_equal event_time(@time.to_s), events[0][1]
+      assert_equal "test", events[0][2]["message"]
+    end
+
+    def test_emit_after_last_id
+      d = create_driver(%[
+        @type mongo_tail
+        database #{database_name}
+        collection #{collection_name}
+        tag input.mongo.last_id
+        time_key time
+      ])
+      @client[collection_name].insert_one({message: "can't obtain"})
+
+      d.run(expect_records: 1, timeout: 5) do
+        @client[collection_name].insert_one({message: "can obtain"})
+      end
+      events = d.events
+      assert_equal 1, events.size
+      assert_equal "input.mongo.last_id", events[0][0]
+      assert_equal event_time(@time.to_s), events[0][1]
+      assert_equal "can obtain", events[0][2]["message"]
+    end
+  end
+
   class MongoAuthenticateTest < self
     require 'fluent/plugin/mongo_auth'
     include ::Fluent::MongoAuth

data/test/plugin/test_out_mongo.rb

@@ -1,7 +1,11 @@
 # coding: utf-8
 require "helper"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
 
 class MongoOutputTest < ::Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
     setup_mongod
@@ -38,17 +42,17 @@ class MongoOutputTest < ::Test::Unit::TestCase
     @client = ::Mongo::Client.new(["localhost:#{port}"], options)
   end
 
-  def teardown_mongod
-    @client[collection_name].drop
+  def teardown_mongod(collection = collection_name)
+    @client[collection].drop
   end
 
-  def create_driver(conf=default_config, tag='test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput, tag).configure(conf, true)
+  def create_driver(conf=default_config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::MongoOutput).configure(conf)
   end
 
   def test_configure
     d = create_driver(%[
-      @type mongo
+      type mongo
       database fluent_test
       collection test_collection
 
@@ -62,30 +66,6 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal(port, d.instance.port)
     assert_equal({capped: true, size: 100}, d.instance.collection_options)
     assert_equal({ssl: false, write: {j: false}}, d.instance.client_options)
-    assert_nil d.instance.connection_string
-  end
-
-  def test_configure_with_connection_string
-    d = create_driver(%[
-      @type mongo
-      connection_string mongodb://localhost/fluent_test
-      collection test_collection
-      capped
-      capped_size 100
-    ])
-    assert_equal('mongodb://localhost/fluent_test', d.instance.connection_string)
-    assert_nil d.instance.database
-  end
-
-  def test_configure_without_connection_string_or_database
-    assert_raise Fluent::ConfigError do
-      d = create_driver(%[
-        @type mongo
-        collection test_collection
-        capped
-        capped_size 100
-      ])
-    end
   end
 
   def test_configure_with_ssl
@@ -115,6 +95,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
     d = create_driver(conf)
     assert_true(d.instance.tag_mapped)
     assert_equal(/^raw\./, d.instance.remove_tag_prefix)
+    assert_equal('${tag}', d.instance.collection)
   end
 
   def test_configure_with_write_concern
@@ -155,61 +136,88 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal(expected, d.instance.mongo_log_level)
   end
 
-  def get_documents
-    @client[collection_name].find.to_a.map {|e| e.delete('_id'); e}
+  def get_documents(collection = collection_name)
+    @client[collection].find.to_a.map {|e| e.delete('_id'); e}
   end
 
   def emit_documents(d)
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 1}, time)
-    d.emit({'a' => 2}, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.feed(time, {'a' => 1})
+    d.feed(time, {'a' => 2})
     time
   end
 
   def test_format
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 1}, time)
-    d.emit({'a' => 2}, time)
-    d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
-    d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
-    d.run
-
-    documents = get_documents
-    assert_equal(2, documents.size)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {'a' => 1})
+      d.feed(time, {'a' => 2})
+    end
+    assert_equal([time, {'a' => 1}].to_msgpack, d.formatted[0])
+    assert_equal([time, {'a' => 2}].to_msgpack, d.formatted[1])
+    assert_equal(2, d.formatted.size)
   end
 
   def test_write
     d = create_driver
-    t = emit_documents(d)
-
-    d.run
+    d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
     actual_documents = get_documents
-    time = Time.parse("2011-01-02 13:14:15 UTC")
-    expected = [{'a' => 1, d.instance.time_key => time},
-                {'a' => 2, d.instance.time_key => time}]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
     assert_equal(expected, actual_documents)
   end
 
+  class WriteWithCollectionPlaceholder < self
+    def setup
+      @tag = 'custom'
+      setup_mongod
+    end
+
+    def teardown
+      teardown_mongod(@tag)
+    end
+
+    def test_write_with_collection_placeholder
+      d = create_driver(%[
+      type mongo
+      database #{database_name}
+      collection ${tag}
+      include_time_key true
+    ])
+      d.run(default_tag: @tag) do
+        emit_documents(d)
+      end
+      actual_documents = get_documents(@tag)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                  {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
+      assert_equal(expected, actual_documents)
+    end
+  end
+
   def test_write_at_enable_tag
     d = create_driver(default_config + %[
       include_tag_key true
       include_time_key false
     ])
-    t = emit_documents(d)
-
-    d.run
+    d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
     actual_documents = get_documents
-    expected = [{'a' => 1, d.instance.tag_key => 'test'},
-                {'a' => 2, d.instance.tag_key => 'test'}]
+    expected = [{'a' => 1, d.instance.inject_config.tag_key => 'test'},
+                {'a' => 2, d.instance.inject_config.tag_key => 'test'}]
     assert_equal(expected, actual_documents)
   end
 
   def emit_invalid_documents(d)
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 3, 'b' => "c", '$last' => '石動'}, time)
-    d.emit({'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')}, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.feed(time, {'a' => 3, 'b' => "c", '$last' => '石動'})
+    d.feed(time, {'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')})
     time
   end
 
@@ -219,19 +227,19 @@ class MongoOutputTest < ::Test::Unit::TestCase
       replace_dollar_in_key_with _dollar_
     ])
 
-    original_time = "2016-02-01 13:14:15 UTC"
-    time = Time.parse(original_time).to_i
-    d.emit({
-      "foo.bar1" => {
-        "$foo$bar" => "baz"
-      },
-      "foo.bar2" => [
-        {
-          "$foo$bar" => "baz"
-        }
-      ],
-    }, time)
-    d.run
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {
+        "foo.bar1" => {
+           "$foo$bar" => "baz"
+        },
+        "foo.bar2" => [
+          {
+            "$foo$bar" => "baz"
+          }
+        ],
+             })
+    end
 
     documents = get_documents
     expected = {"foo_dot_bar1" => {
@@ -241,7 +249,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
                   {
                     "_dollar_foo$bar"=>"baz"
                   },
-                ], "time" => Time.parse(original_time)
+                ], "time" => Time.at(time).localtime
                }
     assert_equal(1, documents.size)
     assert_equal(expected, documents[0])
@@ -266,13 +274,13 @@ class MongoOutputTest < ::Test::Unit::TestCase
         user fluent
         password password
       ])
-      t = emit_documents(d)
-
-      d.run
+      d.run(default_tag: 'test') do
+        emit_documents(d)
+      end
       actual_documents = get_documents
-      time = Time.parse("2011-01-02 13:14:15 UTC")
-      expected = [{'a' => 1, d.instance.time_key => time},
-                  {'a' => 2, d.instance.time_key => time}]
+      time = event_time("2011-01-02 13:14:15 UTC")
+      expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                  {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
       assert_equal(expected, actual_documents)
     end
   end

data/test/plugin/test_out_mongo_replset.rb

@@ -1,6 +1,11 @@
 require "helper"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+require 'fluent/mixin' # for TimeFormatter
 
 class MongoReplsetOutputTest < ::Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
   end
@@ -16,18 +21,14 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
     'fluent_test'
   end
 
-  def nodes
-    ["localhost:#{port}"]
-  end
-
   def port
     27018
   end
 
   def default_config
     %[
-      @type mongo_replset
-      nodes localhost:27018
+      type mongo
+      port 27018
       database #{database_name}
       collection #{collection_name}
       include_time_key true
@@ -35,39 +36,24 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
     ]
   end
 
-  def create_driver(conf=default_config, tag='test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset, tag).configure(conf, true)
+  def create_driver(conf=default_config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::MongoOutputReplset).configure(conf)
   end
 
   def test_configure
     d = create_driver(%[
-      @type mongo_replset
+      type mongo
       port 27018
       database fluent_test
       collection test_collection
-      replica_set rs0
-    ])
 
-    assert_equal('fluent_test', d.instance.database)
-    assert_equal('test_collection', d.instance.collection)
-    assert_equal('localhost', d.instance.host)
-    assert_equal(27018, d.instance.port)
-    assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
-                 d.instance.client_options)
-  end
-
-  def test_configure_with_nodes
-    d = create_driver(%[
-      @type mongo_replset
-      nodes localhost:27018,localhost:27019
-      database fluent_test
-      collection test_collection
       replica_set rs0
     ])
 
     assert_equal('fluent_test', d.instance.database)
     assert_equal('test_collection', d.instance.collection)
-    assert_equal(['localhost:27018', 'localhost:27019'], d.instance.nodes)
+    assert_equal('localhost', d.instance.host)
+    assert_equal(port, d.instance.port)
     assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
                  d.instance.client_options)
   end
@@ -97,7 +83,7 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
     def setup_mongod
       options = {}
       options[:database] = database_name
-      @client = ::Mongo::Client.new(nodes, options)
+      @client = ::Mongo::Client.new(["localhost:#{port}"], options)
     end
 
     def teardown_mongod
@@ -109,35 +95,34 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
     end
 
     def emit_documents(d)
-      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-      d.emit({'a' => 1}, time)
-      d.emit({'a' => 2}, time)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1})
+      d.feed(time, {'a' => 2})
       time
     end
 
     def test_format
       d = create_driver
 
-      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-      d.emit({'a' => 1}, time)
-      d.emit({'a' => 2}, time)
-      d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
-      d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
-      d.run
-
-      documents = get_documents
-      assert_equal(2, documents.size)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, {'a' => 1})
+        d.feed(time, {'a' => 2})
+      end
+      assert_equal([time, {'a' => 1}].to_msgpack, d.formatted[0])
+      assert_equal([time, {'a' => 2}].to_msgpack, d.formatted[1])
+      assert_equal(2, d.formatted.size)
     end
 
     def test_write
       d = create_driver
-      t = emit_documents(d)
-
-      d.run
+      d.run(default_tag: 'test') do
+        emit_documents(d)
+      end
       actual_documents = get_documents
-      time = Time.parse("2011-01-02 13:14:15 UTC")
-      expected = [{'a' => 1, d.instance.time_key => time},
-                  {'a' => 2, d.instance.time_key => time}]
+      time = event_time("2011-01-02 13:14:15 UTC")
+      expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                  {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
       assert_equal(expected, actual_documents)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-22 00:00:00.000000000 Z
+date: 2017-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -100,6 +100,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 description: MongoDB plugin for Fluentd
 email: repeatedly@gmail.com
 executables:
@@ -112,7 +126,6 @@ files:
 - AUTHORS
 - ChangeLog
 - Gemfile
-- Gemfile.v0.12
 - README.rdoc
 - Rakefile
 - VERSION
@@ -142,9 +155,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.2

data/Gemfile.v0.12

@@ -1,6 +0,0 @@
-source "http://rubygems.org"
-
-gem 'json', '= 1.8.3'
-gem 'fluentd', '~> 0.12.0'
-
-gemspec