fluent-plugin-mongo 0.6.12 → 0.6.13

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.6.13 - 2012/01/15
+
+	* Add exclude_broken_fields config to output plugins
+
+
 Release 0.6.12 - 2012/12/28
 
 	* Fix mongo 2.2 capped? problem in mongo_tail

data/README.rdoc

@@ -169,6 +169,8 @@ You can deserialize broken data using Mongo and Marshal.load. Sample code is bel
       p Marshal.load(doc['__broken_data'].to_s) #=> {"key1": "invalid value", "key2": "valid value"}
     end
 
+=== ignore_invalid_record
+
 If you want to ignore an invalid record, set _true_ to _ignore_invalid_record_ parameter in match.
 
     <match forward.*>
@@ -180,6 +182,23 @@ If you want to ignore an invalid record, set _true_ to _ignore_invalid_record_ p
       ...
     </match>
 
+=== exclude_broken_fields
+
+If you want to exclude some fields from broken data marshaling, use _exclude_broken_fields_ to specfiy the keys.
+
+    <match forward.*>
+      ...
+
+      # key2 is excluded from __broken_data.
+      # e.g. {"__broken_data": BinData(0, Marshal.dump result of {"key1": "invalid value"}), "key2": "valid value", "time": ISODate("2012-01-15T21:09:53Z")
+      exclude_broken_fields key2
+
+      ...
+    </match>
+
+Specified value is a comma separated keys(e.g. key1,key2,key3).
+This parameter is useful for excluding shard keys in shard environment.
+
 == Buffer size limitation
 
 Mongo plugin has the limitation of buffer size.

data/VERSION

@@ -1 +1 @@
-0.6.12
+0.6.13

data/lib/fluent/plugin/out_mongo.rb

@@ -20,6 +20,7 @@ class MongoOutput < BufferedOutput
   config_param :ignore_invalid_record, :bool, :default => false
   config_param :disable_collection_check, :bool, :default => nil
   config_param :safe, :bool, :default => true
+  config_param :exclude_broken_fields, :string, :default => nil
 
   # tag mapping mode
   config_param :tag_mapped, :bool, :default => false
@@ -52,6 +53,8 @@ class MongoOutput < BufferedOutput
       @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
     end
 
+    @exclude_broken_fields = @exclude_broken_fields.split(',') if @exclude_broken_fields
+
     # capped configuration
     if conf.has_key?('capped')
       raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
@@ -136,6 +139,11 @@ class MongoOutput < BufferedOutput
       new_record = {}
       new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
       new_record[@time_key] = record.delete(@time_key)
+      if @exclude_broken_fields
+        @exclude_broken_fields.each { |key|
+          new_record[key] = record.delete(key)
+        }
+      end
       new_record[BROKEN_DATA_KEY] = BSON::Binary.new(Marshal.dump(record))
       new_record
     }

data/test/plugin/out_mongo.rb

@@ -106,8 +106,8 @@ class MongoOutputTest < Test::Unit::TestCase
 
   def emit_invalid_documents(d)
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 3, '$last' => '石動'}, time)
-    d.emit({'a' => 4, 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')}, time)
+    d.emit({'a' => 3, 'b' => "c", '$last' => '石動'}, time)
+    d.emit({'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')}, time)
     time
   end
 
@@ -126,6 +126,36 @@ class MongoOutputTest < Test::Unit::TestCase
     }.sort)
   end
 
+  def test_write_with_invalid_recoreds_with_exclude_one_broken_fields
+    d = create_driver(default_config + %[
+      exclude_broken_fields a
+    ])
+    t = emit_documents(d)
+    t = emit_invalid_documents(d)
+
+    d.run
+    documents = get_documents
+    assert_equal(4, documents.size)
+    assert_equal(2, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY) }.size)
+    assert_equal([1, 2, 3, 4], documents.select { |e| e.has_key?('a') }.map { |e| e['a'] }.sort)
+    assert_equal(0, documents.select { |e| e.has_key?('b') }.size)
+  end
+
+  def test_write_with_invalid_recoreds_with_exclude_two_broken_fields
+    d = create_driver(default_config + %[
+      exclude_broken_fields a,b
+    ])
+    t = emit_documents(d)
+    t = emit_invalid_documents(d)
+
+    d.run
+    documents = get_documents
+    assert_equal(4, documents.size)
+    assert_equal(2, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY) }.size)
+    assert_equal([1, 2, 3, 4], documents.select { |e| e.has_key?('a') }.map { |e| e['a'] }.sort)
+    assert_equal(["c", "d"], documents.select { |e| e.has_key?('b') }.map { |e| e['b'] }.sort)
+  end
+
   def test_write_with_invalid_recoreds_at_ignore
     d = create_driver(default_config + %[
       ignore_invalid_record true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.6.12
+  version: 0.6.13
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-28 00:00:00.000000000 Z
+date: 2013-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -134,12 +134,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1797874939389521588
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1797874939389521588
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23