fluent-plugin-mongo 0.6.1 → 0.6.2

data/VERSION

@@ -1 +1 @@
-0.6.1
+0.6.2

data/fluent-plugin-mongo.gemspec

@@ -1,9 +1,6 @@
 # encoding: utf-8
 $:.push File.expand_path('../lib', __FILE__)
 
-# Fixup the YAML engine mismatch problem
-YAML::ENGINE.yamler = "syck"
-
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-mongo"
   gem.description = "MongoDB plugin for Fluent event collector"
@@ -20,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", ">= 0.10.7"
-  gem.add_dependency "mongo", "= 1.5.2"
+  gem.add_dependency "mongo", [">= 1.5.2", "<= 1.5.2"]
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/out_mongo.rb

@@ -15,12 +15,13 @@ class MongoOutput < BufferedOutput
   config_param :host, :string, :default => 'localhost'
   config_param :port, :integer, :default => 27017
   config_param :ignore_invalid_record, :bool, :default => false
+  config_param :safe, :bool, :default => true
 
   # tag mapping mode
   config_param :tag_mapped, :bool, :default => false
   config_param :remove_tag_prefix, :string, :default => nil
 
-  attr_reader :argument
+  attr_reader :collection_options
 
   def initialize
     super
@@ -29,7 +30,8 @@ class MongoOutput < BufferedOutput
     require 'msgpack'
 
     @clients = {}
-    @argument = {:capped => false}
+    @connection_options = {}
+    @collection_options = {:capped => false}
   end
 
   def configure(conf)
@@ -45,9 +47,9 @@ class MongoOutput < BufferedOutput
     # capped configuration
     if conf.has_key?('capped')
       raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
-      @argument[:capped] = true
-      @argument[:size] = Config.size_value(conf['capped_size'])
-      @argument[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
+      @collection_options[:capped] = true
+      @collection_options[:size] = Config.size_value(conf['capped_size'])
+      @collection_options[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
     end
 
     if @buffer.respond_to?(:buffer_chunk_limit)
@@ -56,6 +58,8 @@ class MongoOutput < BufferedOutput
       $log.warn "#{Fluent::VERSION} does not have :buffer_chunk_limit. Be careful when insert large documents to MongoDB"
     end
 
+    @connection_options[:safe] = @safe
+
     # MongoDB uses BSON's Date for time.
     def @timef.format_nocache(time)
       time
@@ -90,7 +94,7 @@ class MongoOutput < BufferedOutput
   def write(chunk)
     # TODO: See emit comment
     collection_name = @tag_mapped ? chunk.key : @collection
-    operate(collection_name, collect_records(chunk))
+    operate(get_or_create_collection(collection_name), collect_records(chunk))
   end
 
   private
@@ -98,27 +102,34 @@ class MongoOutput < BufferedOutput
   INSERT_ARGUMENT = {:collect_on_error => true}
   BROKEN_DATA_KEY = '__broken_data'
 
-  def operate(collection_name, records)
-    collection = get_or_create_collection(collection_name)
-    record_ids, error_records = collection.insert(records, INSERT_ARGUMENT)
-    if error_records
-      if @ignore_invalid_record
-        $log.warn "Ignore #{error_records.size} documents"
+  def operate(collection, records)
+    begin
+      record_ids, error_records = collection.insert(records, INSERT_ARGUMENT)
+      if !@ignore_invalid_record and error_records.size > 0
+        operate_invalid_records(collection, error_records)
+      end
+    rescue Mongo::OperationFailure => e
+      # Probably, all records of _records_ are broken...
+      if e.error_code == 13066  # 13066 means "Message contains no documents"
+        operate_invalid_records(collection, records) unless @ignore_invalid_record
       else
-        # Should create another collection like name_broken?
-        converted_records = error_records.map { |record|
-          new_record = {}
-          new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
-          new_record[@time_key] = record.delete(@time_key)
-          new_record[BROKEN_DATA_KEY] = Marshal.dump(record) # Should use BSON::ByteBuffer
-          new_record
-        }
-        collection.insert(converted_records)
+        raise e
       end
     end
     records
   end
 
+  def operate_invalid_records(collection, records)
+    converted_records = records.map { |record|
+      new_record = {}
+      new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
+      new_record[@time_key] = record.delete(@time_key)
+      new_record[BROKEN_DATA_KEY] = Marshal.dump(record) # Should use BSON::ByteBuffer
+      new_record
+    }
+    collection.insert(converted_records) # Should create another collection like name_broken?
+  end
+
   def collect_records(chunk)
     records = []
     chunk.msgpack_each { |time, record|
@@ -145,19 +156,19 @@ class MongoOutput < BufferedOutput
     @db ||= get_connection
     if @db.collection_names.include?(collection_name)
       collection = @db.collection(collection_name)
-      unless @argument[:capped] == collection.capped? # TODO: Verify capped configuration
+      unless @collection_options[:capped] == collection.capped? # TODO: Verify capped configuration
         # raise Exception if old collection does not match lastest configuration
         raise ConfigError, "New configuration is different from existing collection"
       end
     else
-      collection = @db.create_collection(collection_name, @argument)
+      collection = @db.create_collection(collection_name, @collection_options)
     end
 
     @clients[collection_name] = collection
   end
 
   def get_connection
-    Mongo::Connection.new(@host, @port).db(@database)
+    Mongo::Connection.new(@host, @port, @connection_options).db(@database)
   end
 
   # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.

data/lib/fluent/plugin/out_mongo_replset.rb

@@ -17,18 +17,17 @@ class MongoOutputReplset < MongoOutput
     super
 
     @nodes = parse_nodes(conf['nodes'])
-    @rs_argument = {}
     if name = conf['name']
-      @rs_argument[:name] = conf['name']
+      @connection_options[:name] = conf['name']
     end
     if read = conf['read']
-      @rs_argument[:read] = read.to_sym
+      @connection_options[:read] = read.to_sym
     end
     if refresh_mode = conf['refresh_mode']
-      @rs_argument[:refresh_mode] = refresh_mode.to_sym
+      @connection_options[:refresh_mode] = refresh_mode.to_sym
     end
     if refresh_interval = conf['refresh_interval']
-      @rs_argument[:refresh_interval] = refresh_interval
+      @connection_options[:refresh_interval] = refresh_interval
     end
 
     $log.debug "Setup replica set configuration: nodes = #{conf['nodes']}"
@@ -36,10 +35,9 @@ class MongoOutputReplset < MongoOutput
 
   private
 
-  def operate(collection_name, records)
-    collection = get_or_create_collection(collection_name)
+  def operate(collection, records)
     rescue_connection_failure do
-      collection.insert(records)
+      super(collection, records)
     end
   end
 
@@ -51,7 +49,7 @@ class MongoOutputReplset < MongoOutput
   end
 
   def get_connection
-    Mongo::ReplSetConnection.new(*@nodes, @rs_argument).db(@database)
+    Mongo::ReplSetConnection.new(*@nodes, @connection_options).db(@database)
   end
 
   def rescue_connection_failure

data/test/plugin/out_mongo.rb

@@ -22,8 +22,12 @@ class MongoOutputTest < Test::Unit::TestCase
         super
       end
 
-      def operate(collection_name, records)
-        [format_collection_name(collection_name), records]
+      def operate(collection, records)
+        [format_collection_name(collection), records]
+      end
+
+      def get_or_create_collection(collection_name)
+        collection_name
       end
 
       def mongod_version
@@ -49,7 +53,7 @@ class MongoOutputTest < Test::Unit::TestCase
     assert_equal('test_collection', d.instance.collection)
     assert_equal('fluenter', d.instance.host)
     assert_equal(27018, d.instance.port)
-    assert_equal({:capped => true, :size => 100}, d.instance.argument)
+    assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
     assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
   end
 

data/test/plugin/{out_mongo_tag_collection.rb → out_mongo_tag_mapped.rb}

@@ -23,8 +23,12 @@ class MongoTagCollectionTest < Test::Unit::TestCase
         super
       end
 
-      def operate(collection_name, records)
-        [format_collection_name(collection_name), records]
+      def operate(collection, records)
+        [format_collection_name(collection), records]
+      end
+
+      def get_or_create_collection(collection_name)
+        collection_name
       end
     }.configure(conf)
   end

metadata

@@ -1,81 +1,81 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.6.2
   prerelease: 
-  version: 0.6.1
 platform: ruby
-authors: 
+authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-01-17 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-01-23 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2152067980 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.10.7
   type: :runtime
   prerelease: false
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+  version_requirements: *2152067980
+- !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &2152067040 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+    - - <=
+      - !ruby/object:Gem::Version
         version: 1.5.2
   type: :runtime
   prerelease: false
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
+  version_requirements: *2152067040
+- !ruby/object:Gem::Dependency
   name: rake
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &2152065820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
+  version_requirements: *2152065820
+- !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &2152064740 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
+  version_requirements: *2152064740
+- !ruby/object:Gem::Dependency
   name: rr
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &2152042360 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id005
+  version_requirements: *2152042360
 description: MongoDB plugin for Fluent event collector
 email: repeatedly@gmail.com
-executables: 
+executables:
 - mongo-tail
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - .gitmodules
 - .travis.yml
@@ -94,43 +94,40 @@ files:
 - lib/fluent/plugin/out_mongo_tag_collection.rb
 - test/plugin/in_mongo_tail.rb
 - test/plugin/out_mongo.rb
-- test/plugin/out_mongo_tag_collection.rb
+- test/plugin/out_mongo_tag_mapped.rb
 - test/test_helper.rb
 homepage: https://github.com/fluent/fluent-plugin-mongo
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: -2592226509051705447
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: 2968858870877082029
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: -2592226509051705447
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
+      hash: 2968858870877082029
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: MongoDB plugin for Fluent event collector
-test_files: 
+test_files:
 - test/plugin/in_mongo_tail.rb
 - test/plugin/out_mongo.rb
-- test/plugin/out_mongo_tag_collection.rb
+- test/plugin/out_mongo_tag_mapped.rb
 - test/test_helper.rb