fluent-plugin-mongo 0.6.6 → 0.6.7

data/ChangeLog

@@ -0,0 +1,108 @@
+
+Release 0.6.6 - 2012/03/01
+
+	* Update mongo gem 1.5.2 -> 1.6.0 or later
+	* Move buffer_chunk_limit checking from configure to start
+
+
+Release 0.6.5 - 2012/02/27
+
+	* fix "mongo_replset unexpectedly requires 'host' in configuration"
+	  https://github.com/fluent/fluent-plugin-mongo/issues/9
+
+
+Release 0.6.4 - 2012/02/16
+
+	* Add 'disable_collection_check' parameter
+	* Fix mongod_version bug
+
+
+Release 0.6.3 - 2012/02/08
+
+	* Add authentication support to input / output plugins
+	  https://github.com/fluent/fluent-plugin-mongo/pull/8
+
+
+Release 0.6.2 - 2012/01/23
+
+	* Add :safe to Connection#new options
+	  https://github.com/fluent/fluent-plugin-mongo/issues/7
+	* out_mongo_tag_collection renamed to out_mongo_tag_mapped
+
+
+Release 0.6.1 - 2012/01/17
+
+	* Add mongo_replset explanation to README
+
+
+Release 0.6.0 - 2012/01/16
+
+	* Add mongo_replset for Replica Set
+	* out_mongo_tag_collection merged into out_mongo.
+	  Please use tag_mapped mode.
+	* Support invalid documets handling
+
+
+Release 0.5.3 - 2011/12/20
+
+	* Fix "use format time argument when record to mongodb"
+	  https://github.com/fluent/fluent-plugin-mongo/pull/6
+
+
+Release 0.5.2 - 2011/11/29
+
+	* Fix mongod_version
+	* Fix "configure of ouput plugins raises an exception when mongod is down"
+	  https://github.com/fluent/fluent-plugin-mongo/issues/4
+
+
+Release 0.5.1 - 2011/11/26
+
+	* Fix typo
+
+
+Release 0.5.0 - 2011/11/26
+
+	* Jeweler to Bundler
+	* Add in_mongo_tail
+	* Add out_mongo_tag_collection
+	* Add default 'collection' configuration to mongo_tag_collection
+	* Update the version of dependency modules
+	* Fix "MongoDB and Ruby-Driver have a size limit of insert operation."
+	  https://github.com/fluent/fluent-plugin-mongo/issues/3
+
+
+Release 0.4.0 - 2011/10/16
+
+	* Support fluentd 0.10.1
+	* Add out_mongo.rb test
+
+
+Release 0.3.1 - 2011/10/05
+
+	* Add mongo-tail tool
+	  https://github.com/fluent/fluent-plugin-mongo/issues/1
+
+
+Release 0.3.0 - 2011/10/03
+
+	* Add mongo_backup
+
+
+Release 0.2.1 - 2011/10/02
+
+	* Fix mongo dependency
+	* Fix typo
+	* Fix configuration parsing
+	* Replace MongoBuffer with MongoOutput's <store>
+	* Fix duplicated insert to backup
+
+
+Release 0.2.0 - 2011/09/28
+
+	* MongoOutput becomes BufferedOutput
+
+
+Release 0.1.0 - 2011/09/28
+
+	* Add out_mongo

data/README.rdoc

@@ -146,7 +146,7 @@ Use _mongo_tail_ type in source.
 
 Fluentd event sometimes has an invalid record as a BSON.
 In such case, Mongo plugin marshals an invalid record using Marshal.dump
-and re-inserts its to same collection.
+and re-inserts its to same collection as a binary.
 
 If passed following invalid record:
 
@@ -154,11 +154,18 @@ If passed following invalid record:
 
 then Mongo plugin converts this record to following format:
 
-    {"__broken_data": Marshal.dump result of {"key1": "invalid value", "key2": "valid value"}, "time": ISODate("2012-01-15T21:09:53Z") }
+    {"__broken_data": BinData(0, Marshal.dump result of {"key1": "invalid value", "key2": "valid value"}), "time": ISODate("2012-01-15T21:09:53Z") }
 
 Mongo-Ruby-Driver cannot detect an invalid attribute,
 so Mongo plugin marshals all attributes excluding Fluentd keys("tag_key" and "time_key").
 
+You can deserialize broken data using Mongo and Marshal.load. Sample code is below:
+
+    # _collection_ is an instance of Mongo::Collection
+    collection.find({'__broken_data' => {'$exists' => true}}).each do |doc|
+      p Marshal.load(doc['__broken_data'].to_s) #=> {"key1": "invalid value", "key2": "valid value"}
+    end
+
 If you want to ignore an invalid record, set _true_ to _ignore_invalid_record_ parameter in match.
 
     <match forward.*>

data/VERSION

@@ -1 +1 @@
-0.6.6
+0.6.7

data/lib/fluent/plugin/out_mongo.rb

@@ -18,7 +18,7 @@ class MongoOutput < BufferedOutput
   config_param :host, :string, :default => 'localhost'
   config_param :port, :integer, :default => 27017
   config_param :ignore_invalid_record, :bool, :default => false
-  config_param :disable_collection_check, :bool, :default => false
+  config_param :disable_collection_check, :bool, :default => nil
   config_param :safe, :bool, :default => true
 
   # tag mapping mode
@@ -40,7 +40,12 @@ class MongoOutput < BufferedOutput
   def configure(conf)
     super
 
-    @tag_mapped = true if conf.has_key?('tag_mapped')
+    if conf.has_key?('tag_mapped')
+      @tag_mapped = true
+      @disable_collection_check = true if @disable_collection_check.nil?
+    else
+      @disable_collection_check = false if @disable_collection_check.nil?
+    end
     raise ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
 
     if remove_tag_prefix = conf['remove_tag_prefix']
@@ -128,7 +133,7 @@ class MongoOutput < BufferedOutput
       new_record = {}
       new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
       new_record[@time_key] = record.delete(@time_key)
-      new_record[BROKEN_DATA_KEY] = Marshal.dump(record) # Should use BSON::ByteBuffer
+      new_record[BROKEN_DATA_KEY] = BSON::Binary.new(Marshal.dump(record))
       new_record
     }
     collection.insert(converted_records) # Should create another collection like name_broken?
@@ -158,11 +163,12 @@ class MongoOutput < BufferedOutput
     return @clients[collection_name] if @clients[collection_name]
 
     @db ||= get_connection
-    if @db.collection_names.include?(collection_name) and !@disable_collection_check
+    if @db.collection_names.include?(collection_name)
       collection = @db.collection(collection_name)
-      unless @collection_options[:capped] == collection.capped? # TODO: Verify capped configuration
-        # raise Exception if old collection does not match lastest configuration
-        raise ConfigError, "New configuration is different from existing collection"
+      unless @disable_collection_check
+        unless @collection_options[:capped] == collection.capped? # TODO: Verify capped configuration
+          raise ConfigError, "New configuration is different from existing collection"
+        end
       end
     else
       collection = @db.create_collection(collection_name, @collection_options)

data/test/plugin/out_mongo.rb

@@ -54,7 +54,10 @@ class MongoOutputTest < Test::Unit::TestCase
     assert_equal('fluenter', d.instance.host)
     assert_equal(27018, d.instance.port)
     assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
-    assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
+    assert_equal(33554432, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
+    # buffer_chunk_limit moved from configure to start
+    # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
+    # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
   end
 
   def test_format

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.6.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-01 00:00:00.000000000Z
+date: 2012-03-30 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2152542640 !ruby/object:Gem::Requirement
+  requirement: &2153123160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 0.10.7
   type: :runtime
   prerelease: false
-  version_requirements: *2152542640
+  version_requirements: *2153123160
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2152542020 !ruby/object:Gem::Requirement
+  requirement: &2153105660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152542020
+  version_requirements: *2153105660
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2152541400 !ruby/object:Gem::Requirement
+  requirement: &2153104360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2152541400
+  version_requirements: *2153104360
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2152540780 !ruby/object:Gem::Requirement
+  requirement: &2153103260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *2152540780
+  version_requirements: *2153103260
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &2152539800 !ruby/object:Gem::Requirement
+  requirement: &2153102160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2152539800
+  version_requirements: *2153102160
 description: MongoDB plugin for Fluent event collector
 email: repeatedly@gmail.com
 executables:
@@ -77,6 +77,7 @@ files:
 - .gitmodules
 - .travis.yml
 - AUTHORS
+- ChangeLog
 - Gemfile
 - README.rdoc
 - Rakefile
@@ -107,7 +108,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1733988723759885390
+      hash: 3355529483381375280
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -116,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1733988723759885390
+      hash: 3355529483381375280
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10