fluent-plugin-mongo 0.6.13 → 0.7.0

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.7.0 - 2012/03/20
+
+	* Upgrade mongo gem least version to 1.8
+	* Upgrade fluentd gem least version to 0.10.9
+	* Use new classes with mongo gem version 1.8
+	* Replace safe with write_concern option in output plugins
+	* Change buffer_chunk_limit to 8MB when mongod version is 1.8 or later.
+
+
 Release 0.6.13 - 2012/01/15
 
 	* Add exclude_broken_fields config to output plugins

data/README.rdoc

@@ -208,7 +208,7 @@ MongoDB returns error or mongo-ruby-driver raises an exception.
 
 So, Mongo plugin resets _buffer_chunk_limit_ if configurated value is larger than above limitation:
 - Before v1.8, max of _buffer_chunk_limit_ is 2MB
-- After v1.8, max of _buffer_chunk_limit_ is 10MB
+- After  v1.8, max of _buffer_chunk_limit_ is 8MB
 
 == Disable collection check
 

data/VERSION

@@ -1 +1 @@
-0.6.13
+0.7.0

data/bin/mongo-tail

@@ -29,10 +29,10 @@ OptionParser.new { |opt|
 }
 
 def get_capped_collection(conf)
-  db = Mongo::Connection.new(conf[:h], conf[:p]).db(conf[:d])
+  db = Mongo::MongoClient.new(conf[:h], conf[:p]).db(conf[:d])
   if db.collection_names.include?(conf[:c])
     collection = db.collection(conf[:c])
-    if [1, true].include?(collection.stats['capped']) # older version(2.0.x or earlier) returns 1 
+    if collection.capped?
       collection
     else
       puts "#{conf[:c]} is not capped. mongo-tail can not tail normal collection."

data/fluent-plugin-mongo.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", "~> 0.10.7"
-  gem.add_dependency "mongo", "~> 1.6.0"
+  gem.add_dependency "fluentd", "~> 0.10.9"
+  gem.add_dependency "mongo", "~> 1.8.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -70,7 +70,7 @@ class MongoTailInput < Input
       db = authenticate(Mongo::Connection.new(@host, @port).db(@database))
       raise ConfigError, "'#{@database}.#{@collection}' not found: node = #{@host}:#{@port}" unless db.collection_names.include?(@collection)
       collection = db.collection(@collection)
-      raise ConfigError, "'#{@database}.#{@collection}' is not capped: node = #{@host}:#{@port}" unless [1, true].include?(collection.stats['capped']) # older version(2.0.x or earlier) returns 1
+      raise ConfigError, "'#{@database}.#{@collection}' is not capped: node = #{@host}:#{@port}" unless collection.capped?
       collection
     rescue Mongo::ConnectionFailure => e
       $log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"

data/lib/fluent/plugin/out_mongo.rb

@@ -19,14 +19,14 @@ class MongoOutput < BufferedOutput
   config_param :port, :integer, :default => 27017
   config_param :ignore_invalid_record, :bool, :default => false
   config_param :disable_collection_check, :bool, :default => nil
-  config_param :safe, :bool, :default => true
   config_param :exclude_broken_fields, :string, :default => nil
+  config_param :write_concern, :integer, :default => nil
 
   # tag mapping mode
   config_param :tag_mapped, :bool, :default => false
   config_param :remove_tag_prefix, :string, :default => nil
 
-  attr_reader :collection_options
+  attr_reader :collection_options, :connection_options
 
   def initialize
     super
@@ -55,7 +55,6 @@ class MongoOutput < BufferedOutput
 
     @exclude_broken_fields = @exclude_broken_fields.split(',') if @exclude_broken_fields
 
-    # capped configuration
     if conf.has_key?('capped')
       raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
       @collection_options[:capped] = true
@@ -63,7 +62,7 @@ class MongoOutput < BufferedOutput
       @collection_options[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
     end
 
-    @connection_options[:safe] = @safe
+    @connection_options[:w] = @write_concern unless @write_concern.nil?
 
     # MongoDB uses BSON's Date for time.
     def @timef.format_nocache(time)
@@ -78,11 +77,7 @@ class MongoOutput < BufferedOutput
     get_or_create_collection(@collection) unless @tag_mapped
 
     # From configure for avoding complex method dependency...
-    if @buffer.respond_to?(:buffer_chunk_limit)
-      @buffer.buffer_chunk_limit = available_buffer_chunk_limit
-    else
-      $log.warn "#{Fluent::VERSION} does not have :buffer_chunk_limit. Be careful when insert large documents to MongoDB"
-    end
+    @buffer.buffer_chunk_limit = available_buffer_chunk_limit
 
     super
   end
@@ -147,7 +142,7 @@ class MongoOutput < BufferedOutput
       new_record[BROKEN_DATA_KEY] = BSON::Binary.new(Marshal.dump(record))
       new_record
     }
-    collection.insert(converted_records) # Should create another collection like name_broken?
+    collection.insert(converted_records)
   end
 
   def collect_records(chunk)
@@ -177,7 +172,7 @@ class MongoOutput < BufferedOutput
     if @db.collection_names.include?(collection_name)
       collection = @db.collection(collection_name)
       unless @disable_collection_check
-        capped = [1, true].include?(collection.stats['capped']) # TODO: Remove this check after mongo gem upgrade to 1.7.x
+        capped = collection.capped?
         unless @collection_options[:capped] == capped # TODO: Verify capped configuration
           new_mode = format_collection_mode(@collection_options[:capped])
           old_mode = format_collection_mode(capped)
@@ -196,17 +191,17 @@ class MongoOutput < BufferedOutput
   end
 
   def get_connection
-    db = Mongo::Connection.new(@host, @port, @connection_options).db(@database)
+    db = Mongo::MongoClient.new(@host, @port, @connection_options).db(@database)
     authenticate(db)
   end
 
   # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
-  LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024  # 2MB  = 4MB  / 2
-  LIMIT_AFTER_v1_8 = 10 * 1024 * 1024  # 10MB = 16MB / 2 + alpha
+  LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024  # 2MB = 4MB  / 2
+  LIMIT_AFTER_v1_8 =  8 * 1024 * 1024  # 8MB = 16MB / 2
 
   def available_buffer_chunk_limit
     begin
-      limit = mongod_version >= "1.8.0" ? LIMIT_AFTER_v1_8 : LIMIT_BEFORE_v1_8  # TODO: each version comparison
+      limit = mongod_version >= "1.8.0" ? LIMIT_AFTER_v1_8 : LIMIT_BEFORE_v1_8
     rescue Mongo::ConnectionFailure => e
       $log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
       exit!
@@ -214,7 +209,6 @@ class MongoOutput < BufferedOutput
       $log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
       exit!
     rescue Exception => e
-      # TODO: should do exit?
       $log.warn "mongo unknown error #{e}, set #{LIMIT_BEFORE_v1_8} to chunk limit"
       limit = LIMIT_BEFORE_v1_8
     end

data/lib/fluent/plugin/out_mongo_replset.rb

@@ -50,7 +50,7 @@ class MongoOutputReplset < MongoOutput
   end
 
   def get_connection
-    db = Mongo::ReplSetConnection.new(@nodes, @connection_options).db(@database)
+    db = Mongo::MongoReplicaSetClient.new(@nodes, @connection_options).db(@database)
     authenticate(db)
   end
 

data/test/plugin/out_mongo.rb

@@ -32,7 +32,7 @@ class MongoOutputTest < Test::Unit::TestCase
     conf = conf + %[
       port #{@@mongod_port}
     ]
-    @db = Mongo::Connection.new('localhost', @@mongod_port).db(MONGO_DB_DB)
+    @db = Mongo::MongoClient.new('localhost', @@mongod_port).db(MONGO_DB_DB)
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
   end
 
@@ -51,11 +51,20 @@ class MongoOutputTest < Test::Unit::TestCase
     assert_equal('localhost', d.instance.host)
     assert_equal(@@mongod_port, d.instance.port)
     assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
+    assert(d.instance.connection_options.empty?)
     # buffer_chunk_limit moved from configure to start
     # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
     # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
   end
 
+  def test_configure_with_write_concern
+    d = create_driver(default_config + %[
+      write_concern 2
+    ])
+
+    assert_equal({:w => 2}, d.instance.connection_options)
+  end
+
   def test_format
     d = create_driver
 
@@ -198,7 +207,7 @@ class MongoReplOutputTest < MongoOutputTest
   end
 
   def create_driver(conf = default_config)
-    @db = Mongo::ReplSetConnection.new(build_seeds(3), :name => @rs.name).db(MONGO_DB_DB)
+    @db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @rs.name).db(MONGO_DB_DB)
     Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset).configure(conf)
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.6.13
+  version: 0.7.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-15 00:00:00.000000000 Z
+date: 2013-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.7
+        version: 0.10.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.7
+        version: 0.10.9
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1797874939389521588
+      hash: -2812294622738366735
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1797874939389521588
+      hash: -2812294622738366735
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23