fluent-plugin-mongo 0.7.12 → 0.7.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 306a5a645b491a0c8edb197064336662588aab89
-  data.tar.gz: a80cc84c0e68319cd68a891678a08d4def516d22
+  metadata.gz: c00835bf3c3c44d2055e59c9d38d8654c438f83e
+  data.tar.gz: 21f1d514533a48beddf1391b68a2467e6131c73c
 SHA512:
-  metadata.gz: 15dc38b80bca19d1c7d52914d46f4d7688bdcfe2e5355ed0fdbe2b835d802bba2ed8436b9ed0b4565cdb51cc4cbbf9295daedff78a9fb115fe650da37d58e5e4
-  data.tar.gz: 23047da16921b3f30a3bc16e6f1881255010952f2542b921b92608aef6ecc7f73ff85c15ff2a27ab3bf05ad04b41f6cadec006eccb3436ee58c4e4c0057c0c4f
+  metadata.gz: 83fcb6008050f5eb175764dfaf1ab96836c41168947cd24ced230257fe60d17d04d7de60acfc9c18fe08eda59b103ad39b15b5b1ccbb76ac6f54d2e5c02f04ac
+  data.tar.gz: ca4144e0884ee794f8cdd1ef68f65a5e83748bd37b45a1c5a037f9a03cbad9f828292ab8200ab02e70dc3a41e66471e57a646c7855fd8cf7572f25dd97596eb9

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.7.13 - 2016/06/03
+
+	* Add mongodb_smaller_bson_limit parameter to disable MongoDB v1.7 or earlier versions by default.
+
+
 Release 0.7.12 - 2016/02/09
 
 	* Support saving last_id to mongod instead of local file

data/Rakefile

@@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks
 require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
+  test.libs << 'test'
   test.test_files = FileList['test/plugin/*.rb']
   test.verbose = true
 end

data/VERSION

@@ -1 +1 @@
-0.7.12
+0.7.13

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -1,3 +1,5 @@
+require 'fluent/input'
+
 module Fluent
   class MongoTailInput < Input
     Plugin.register_input('mongo_tail', self)

data/lib/fluent/plugin/out_mongo.rb

@@ -1,3 +1,5 @@
+require 'fluent/output'
+
 module Fluent
   class MongoOutput < BufferedOutput
     Plugin.register_output('mongo', self)
@@ -36,6 +38,9 @@ module Fluent
     config_param :ssl_verify, :bool, :default => false
     config_param :ssl_ca_cert, :string, :default => nil
 
+    # For older (1.7 or earlier) MongoDB versions
+    config_param :mongodb_smaller_bson_limit, :bool, :default => false
+
     attr_reader :collection_options, :connection_options
 
     unless method_defined?(:log)
@@ -52,7 +57,31 @@ module Fluent
       @collection_options = {:capped => false}
     end
 
+    # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
+    LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024  # 2MB = 4MB  / 2
+    LIMIT_AFTER_v1_8 =  8 * 1024 * 1024  # 8MB = 16MB / 2
+
     def configure(conf)
+      if conf.has_key?('buffer_chunk_limit')
+        configured_chunk_limit_size = Config.size_value(conf['buffer_chunk_limit'])
+        estimated_limit_size = LIMIT_AFTER_v1_8
+        estimated_limit_size_conf = '8m'
+        if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
+          estimated_limit_size = LIMIT_BEFORE_v1_8
+          estimated_limit_size_conf = '2m'
+        end
+        if configured_chunk_limit_size > estimated_limit_size
+          log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{estimated_limit_size_conf}"
+          conf['buffer_chunk_limit'] = estimated_limit_size_conf
+        end
+      else
+        if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
+          conf['buffer_chunk_limit'] = '2m'
+        else
+          conf['buffer_chunk_limit'] = '8m'
+        end
+      end
+
       super
 
       if conf.has_key?('tag_mapped')
@@ -102,9 +131,6 @@ module Fluent
       # Non tag mapped mode, we can check collection configuration before server start.
       get_or_create_collection(@collection) unless @tag_mapped
 
-      # From configure for avoding complex method dependency...
-      @buffer.buffer_chunk_limit = available_buffer_chunk_limit
-
       super
     end
 
@@ -232,45 +258,6 @@ module Fluent
       authenticate(db)
     end
 
-    # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
-    LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024  # 2MB = 4MB  / 2
-    LIMIT_AFTER_v1_8 =  8 * 1024 * 1024  # 8MB = 16MB / 2
-
-    def available_buffer_chunk_limit
-      begin
-        limit = mongod_version >= "1.8.0" ? LIMIT_AFTER_v1_8 : LIMIT_BEFORE_v1_8
-      rescue Mongo::ConnectionFailure => e
-        log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
-        exit!
-      rescue Mongo::OperationFailure => e
-        log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
-        exit!
-      rescue Exception => e
-        log.warn "mongo unknown error #{e}, set #{LIMIT_BEFORE_v1_8} to chunk limit"
-        limit = LIMIT_BEFORE_v1_8
-      end
-
-      if @buffer.buffer_chunk_limit > limit
-        log.warn ":buffer_chunk_limit(#{@buffer.buffer_chunk_limit}) is large. Reset :buffer_chunk_limit with #{limit}"
-        limit
-      else
-        @buffer.buffer_chunk_limit
-      end
-    end
-
-    def mongod_version
-      version = nil
-
-      begin
-        version = get_connection.command('buildInfo' => 1)['version']
-      rescue Mongo::OperationFailure
-        # fallback for buggy mongod version support
-        version = authenticate(Mongo::MongoClient.new(@host, @port, @connection_options).db('admin')).command('buildInfo' => 1)['version']
-      end
-
-      version
-    end
-
     def replace_key_of_hash(hash_or_array, pattern, replacement)
       case hash_or_array
       when Array

data/test/plugin/in_mongo_tail.rb

@@ -1,9 +1,9 @@
 require 'test_helper'
+require 'fluent/plugin/in_mongo_tail'
 
 class MongoTailInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/in_mongo_tail'
   end
 
   CONFIG = %[

data/test/plugin/out_mongo.rb

@@ -1,63 +1,12 @@
 # -*- coding: utf-8 -*-
 require 'tools/rs_test_helper'
+require 'fluent/plugin/out_mongo'
 
-class MongoOutputTest < Test::Unit::TestCase
-  include MongoTestHelper
-
-  def setup
-    Fluent::Test.setup
-    require 'fluent/plugin/out_mongo'
-
-    setup_mongod
-  end
-
-  def teardown
-    @db.collection(collection_name).drop
-    teardown_mongod
-  end
-
+module MongoOutputTestCases
   def collection_name
     'test'
   end
 
-  def default_config
-    %[
-      type mongo
-      database #{MONGO_DB_DB}
-      collection #{collection_name}
-      include_time_key true # TestDriver ignore config_set_default?
-    ]
-  end
-
-  def create_driver(conf = default_config)
-    conf = conf + %[
-      port #{@@mongod_port}
-    ]
-    @db = Mongo::MongoClient.new('localhost', @@mongod_port).db(MONGO_DB_DB)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver(%[
-      type mongo
-      database fluent_test
-      collection test_collection
-
-      capped
-      capped_size 100
-    ])
-
-    assert_equal('fluent_test', d.instance.database)
-    assert_equal('test_collection', d.instance.collection)
-    assert_equal('localhost', d.instance.host)
-    assert_equal(@@mongod_port, d.instance.port)
-    assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
-    assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
-    # buffer_chunk_limit moved from configure to start
-    # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
-    # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
-  end
-
   def test_configure_with_write_concern
     d = create_driver(default_config + %[
       write_concern 2
@@ -223,16 +172,86 @@ class MongoOutputTest < Test::Unit::TestCase
   end
 end
 
-class MongoReplOutputTest < MongoOutputTest
+class MongoOutputTest < Test::Unit::TestCase
+  include MongoOutputTestCases
+
+  class << self
+    def startup
+      MongoTestHelper.setup_mongod
+    end
+
+    def shutdown
+      MongoTestHelper.teardown_mongod
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+    @db.collection(collection_name).drop
+  end
+
+  def default_config
+    %[
+      type mongo
+      database #{MONGO_DB_DB}
+      collection #{collection_name}
+      include_time_key true # TestDriver ignore config_set_default?
+    ]
+  end
+
+  def create_driver(conf = default_config)
+    conf = conf + %[
+      port #{MongoTestHelper.mongod_port}
+    ]
+    @db = Mongo::MongoClient.new('localhost', MongoTestHelper.mongod_port).db(MONGO_DB_DB)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      type mongo
+      database fluent_test
+      collection test_collection
+
+      capped
+      capped_size 100
+    ])
+
+    assert_equal('fluent_test', d.instance.database)
+    assert_equal('test_collection', d.instance.collection)
+    assert_equal('localhost', d.instance.host)
+    assert_equal(MongoTestHelper.mongod_port, d.instance.port)
+    assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
+    assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
+    # buffer_chunk_limit moved from configure to start
+    # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
+    # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
+  end
+end
+
+class MongoReplOutputTest < Test::Unit::TestCase
+  include MongoOutputTestCases
+
+  class << self
+    def startup
+      setup_rs
+    end
+
+    def shutdown
+      teardown_rs
+    end
+  end
+
   def setup
     Fluent::Test.setup
     require 'fluent/plugin/out_mongo_replset'
-
-    ensure_rs
   end
 
   def teardown
-    @rs.restart_killed_nodes
+    @@rs.restart_killed_nodes
     if defined?(@db) && @db
       @db.collection(collection_name).drop
       @db.connection.close
@@ -251,7 +270,7 @@ class MongoReplOutputTest < MongoOutputTest
   end
 
   def create_driver(conf = default_config)
-    @db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @rs.name).db(MONGO_DB_DB)
+    @db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @@rs.name).db(MONGO_DB_DB)
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset).configure(conf)
   end
 

data/test/plugin/out_mongo_tag_mapped.rb

@@ -1,9 +1,9 @@
 require 'test_helper'
+require 'fluent/plugin/out_mongo'
 
 class MongoTagCollectionTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/out_mongo'
   end
 
   CONFIG = %[

data/test/test_helper.rb

@@ -33,33 +33,27 @@ MONGO_DB_DB = 'fluent_test'
 MONGO_DB_PATH = File.join(File.dirname(__FILE__), 'plugin', 'data')
 
 module MongoTestHelper
-  @@setup_count = 0
-
-  def cleanup_mongod_env
-    system("killall mongod")
+  def self.cleanup_mongod_env
+    Process.kill "TERM", @@pid
+    Process.waitpid @@pid
     system("rm -rf #{MONGO_DB_PATH}")
-    system("mkdir -p #{MONGO_DB_PATH}")
   end
 
-  def setup_mongod
-    unless defined?(@@current_mongo_test_class) and @@current_mongo_test_class == self.class
-      cleanup_mongod_env
+  def self.setup_mongod
+    system("rm -rf #{MONGO_DB_PATH}")
+    system("mkdir -p #{MONGO_DB_PATH}")
 
-      @@current_mongo_test_class = self.class
-      @@mongod_port = unused_port
-      @@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
-      sleep 3
-    end
+    @@mongod_port = unused_port
+    @@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
+    sleep 3
+  end
 
-    @@setup_count += 1;
+  def self.teardown_mongod
+    Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
+    cleanup_mongod_env
   end
 
-  def teardown_mongod
-    if defined?(@@current_mongo_test_class)
-      Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
-    end
-    if @@setup_count == self.class.methods.size
-      cleanup_mongod_env
-    end
+  def self.mongod_port
+    @@mongod_port
   end
 end

data/test/tools/repl_set_manager.rb

@@ -45,8 +45,9 @@ class ReplSetManager
   end
 
   def start_set
-    system("killall mongod")
-    sleep(1)
+    @mongods.keys.each do |node|
+      kill(node)
+    end
     should_start = true
     puts "** Starting a replica set with #{@count} nodes"
 
@@ -79,7 +80,9 @@ class ReplSetManager
   end
 
   def cleanup_set
-    system("killall mongod")
+    @mongods.keys.each do |node|
+      kill(node)
+    end
     @count.times do |n|
       system("rm -rf #{@mongods[n]['db_path']}")
     end
@@ -193,9 +196,11 @@ class ReplSetManager
   end
 
   def kill(node, signal=2)
+    return unless @mongods[node]['up']
+
     pid = @mongods[node]['pid']
     puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
-    system("kill #{pid}")
+    Process.kill signal, pid
     dead = wait_for_death(pid)
     @mongods[node]['up'] = false if dead
   end
@@ -254,7 +259,7 @@ class ReplSetManager
     system(@mongods[node]['start'])
     @mongods[node]['up'] = true
     sleep(0.5)
-    @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
+    @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip.to_i
   end
   alias :restart :start
 

data/test/tools/rs_test_helper.rb

@@ -5,13 +5,13 @@ require 'tools/repl_set_manager'
 class Test::Unit::TestCase
   # Ensure replica set is available as an instance variable and that
   # a new set is spun up for each TestCase class
-  def ensure_rs
-    unless defined?(@@current_class) and @@current_class == self.class
-      @@current_class = self.class 
-      @@rs = ReplSetManager.new
-      @@rs.start_set
-    end
-    @rs = @@rs
+  def self.setup_rs
+    @@rs = ReplSetManager.new
+    @@rs.start_set
+  end
+
+  def self.teardown_rs
+    @@rs.cleanup_set
   end
 
   # Generic code for rescuing connection failures and retrying operations.
@@ -32,7 +32,7 @@ class Test::Unit::TestCase
   def build_seeds(num_hosts)
     seeds = []
     num_hosts.times do |n|
-      seeds << "#{@rs.host}:#{@rs.ports[n]}"
+      seeds << "#{@@rs.host}:#{@@rs.ports[n]}"
     end
     seeds
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.7.12
+  version: 0.7.13
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-09 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: MongoDB plugin for Fluentd