fluent-plugin-mongo 0.5.3 → 0.6.0

data/README.rdoc

@@ -7,11 +7,11 @@
 Store fluent-event as MongoDB Document to MongoDB database.
 
 MongoOutput set 'time' field to a document by default.
-You set -false- to 'include_time_key' parameter if you disable this behaivor.
+You set _false_ to 'include_time_key' parameter if you disable this behaivor.
 
-=== MongoOutputTagCollection
+==== tag mapped mode
 
-Tag mapping to MongoDB collection automatically.
+Tag mapped to MongoDB collection automatically.
 
 === MongoBackupOutput
 
@@ -35,41 +35,67 @@ Tail capped collection to input data.
       port 10000
 
       # You can use 'capped' if you want to use capped collection
-      capped true
+      capped
       capped_size 100m
 
       # Other buffer configurations here
     </match>
 
-==== NOTE
+==== Tag mapped mode
 
-MongoDB's output plugins have the limitation of buffer size.
-Because MongoDB and Ruby-Driver checks the total object size at each insertion.
-If total object size gets over the size limitation,
-MongoDB returns error or Ruby-Driver raises an Exception.
-
-So, MongoDB's output plugins reset :buffer_chunk_limit if configurated value is larger than above limitation.
-- Before v1.8, max of :buffer_chunk_limit is 2MB
-- After v1.8, max of :buffer_chunk_limit is 10MB
-
-=== MongoOutputTagCollection
-
-Use mongo_tag_collection type in match.
+Use 'tag_mapped' parameter.
 
 If tag name is "foo.bar", auto create collection "foo.bar" and insert data.
 
     <match forward.*>
-      type mongo_tag_collection
+      type mongo
       database fluent
 
-      # This configuration is used if tag not found. Default is 'untagged'
+      # If You use 'tag_mapped', then tag mapped mode enabled.
+      tag_mapped
+
+      # If tag is "forward.foo.bar", then prefix "forward." is removed.
+      # Collection name to insert is "foo.bar".
+      remove_tag_prefix forward.
+
+      # This configuration is used if tag not found. Default is 'untagged'.
       collection misc
 
-      # Tag "forward.foo.bar" remove prefix "forward.".
-      # Collection Mapping name is "foo.bar".
-      remove_prefix_collection forward.
+      # Other configurations here
     </match>
 
+==== NOTE
+
+===== Broken data as a BSON
+
+Fluentd event sometimes has an invalid record as a BSON.
+In such case, mongo plugin marshals an invalid record using Marshal.dump
+and re-inserts its to same collection.
+
+Example:
+
+    {"key1": "invalid value", "key2": "valid value", "time": ISODate("2012-01-15T21:09:53Z") }
+
+to
+
+    {"__broken_data": "Marshal.dump result", "time": ISODate("2012-01-15T21:09:53Z") }
+
+mongo-ruby-driver cannot detect invalid attribute,
+so mongo plugin marshals all attributes excluding Fluentd keys(tag_key and time_key).
+
+If you ignores an invalid record, please set 'ignore_invalid_record true' in match.
+
+===== Buffer size limitation
+
+MongoDB's output plugins have the limitation of buffer size.
+Because MongoDB and Ruby-Driver checks the total object size at each insertion.
+If total object size gets over the size limitation,
+MongoDB returns error or Ruby-Driver raises an Exception.
+
+So, MongoDB's output plugins reset :buffer_chunk_limit if configurated value is larger than above limitation.
+- Before v1.8, max of :buffer_chunk_limit is 2MB
+- After v1.8, max of :buffer_chunk_limit is 10MB
+
 === MongoBackupOutput
 
 Use mongo_backup type in match. mongo_backup alwalys use capped collection.
@@ -107,7 +133,7 @@ Use mongo_tail type in source.
 
 === More configuration
 
-- Create Index
+- Authentication
 - Select insert or update
 - etc
 

data/VERSION

@@ -1 +1 @@
-0.5.3
+0.6.0

data/fluent-plugin-mongo.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", "~> 0.10.6"
-  gem.add_dependency "mongo", ">= 1.4.0"
+  gem.add_dependency "fluentd", ">= 0.10.7"
+  gem.add_dependency "mongo", "= 1.5.2"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -31,6 +31,8 @@ class MongoTailInput < Input
     end
 
     @last_id = @id_store_file ? get_last_id : nil
+
+    $log.debug "Setup mongo_tail configuration: mode = #{@id_store_file ? 'persistent' : 'non-persistent'}"
   end
 
   def start
@@ -61,9 +63,9 @@ class MongoTailInput < Input
 
   def get_capped_collection
     db = Mongo::Connection.new(@host, @port).db(@database)
-    raise ConfigError, "'#{@database}.#{@collection}' not found: server = #{@host}:#{@port}" unless db.collection_names.include?(@collection)
+    raise ConfigError, "'#{@database}.#{@collection}' not found: node = #{@host}:#{@port}" unless db.collection_names.include?(@collection)
     collection = db.collection(@collection)
-    raise ConfigError, "'#{@database}.#{@collection}' is not capped: server = #{@host}:#{@port}" unless collection.capped?
+    raise ConfigError, "'#{@database}.#{@collection}' is not capped: node = #{@host}:#{@port}" unless collection.capped?
     collection
   end
 

data/lib/fluent/plugin/mongo_ext.rb

@@ -0,0 +1,55 @@
+require 'mongo'
+
+module Mongo
+  class Collection
+    # Temporary fix.
+    # See pull request 82: https://github.com/mongodb/mongo-ruby-driver/pull/82
+    def insert_documents(documents, collection_name=@name, check_keys=true, safe=false, flags={})
+      if flags[:continue_on_error]
+        message = BSON::ByteBuffer.new
+        message.put_int(1)
+      else
+        message = BSON::ByteBuffer.new("\0\0\0\0")
+      end
+
+      collect_on_error = !!flags[:collect_on_error]
+      error_docs = [] if collect_on_error
+
+      BSON::BSON_RUBY.serialize_cstr(message, "#{@db.name}.#{collection_name}")
+      documents =
+        if collect_on_error
+          documents.select do |doc|
+            begin
+              message.put_binary(BSON::BSON_CODER.serialize(doc, check_keys, true, @connection.max_bson_size).to_s)
+              true
+            rescue StandardError => e  # StandardError will be replaced with BSONError
+              doc.delete(:_id)
+              error_docs << doc
+              false
+            end
+          end
+        else
+          documents.each do |doc|
+            message.put_binary(BSON::BSON_CODER.serialize(doc, check_keys, true, @connection.max_bson_size).to_s)
+          end
+        end
+      raise InvalidOperation, "Exceded maximum insert size of 16,000,000 bytes" if message.size > 16_000_000
+
+      instrument(:insert, :database => @db.name, :collection => collection_name, :documents => documents) do
+        if safe
+          @connection.send_message_with_safe_check(Mongo::Constants::OP_INSERT, message, @db.name, nil, safe)
+        else
+          @connection.send_message(Mongo::Constants::OP_INSERT, message)
+        end
+      end
+
+      doc_ids = documents.collect { |o| o[:_id] || o['_id'] }
+      if collect_on_error
+        return doc_ids, error_docs
+      else
+        doc_ids
+      end
+    end
+  end
+end
+

data/lib/fluent/plugin/out_mongo.rb

@@ -11,15 +11,21 @@ class MongoOutput < BufferedOutput
   config_set_default :include_time_key, true
 
   config_param :database, :string
-  config_param :collection, :string
+  config_param :collection, :string, :default => 'untagged'
   config_param :host, :string, :default => 'localhost'
   config_param :port, :integer, :default => 27017
+  config_param :ignore_invalid_record, :bool, :default => false
+
+  # tag mapping mode
+  config_param :tag_mapped, :bool, :default => false
+  config_param :remove_tag_prefix, :string, :default => nil
 
   attr_reader :argument
 
   def initialize
     super
     require 'mongo'
+    require 'fluent/plugin/mongo_ext'
     require 'msgpack'
 
     @clients = {}
@@ -29,6 +35,13 @@ class MongoOutput < BufferedOutput
   def configure(conf)
     super
 
+    @tag_mapped = true if conf.has_key?('tag_mapped')
+    raise ConfigError, "normal mode requires collection parameter" if !@tag_mapped and !conf.has_key?('collection')
+
+    if remove_tag_prefix = conf['remove_tag_prefix']
+      @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+    end
+
     # capped configuration
     if conf.has_key?('capped')
       raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
@@ -47,6 +60,8 @@ class MongoOutput < BufferedOutput
     def @timef.format_nocache(time)
       time
     end
+
+    $log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
   end
 
   def start
@@ -63,22 +78,45 @@ class MongoOutput < BufferedOutput
     [time, record].to_msgpack
   end
 
-  def write(chunk)
-    operate(@collection, collect_records(chunk))
+  def emit(tag, es, chain)
+    # TODO: Should replacement using eval in configure?
+    if @tag_mapped
+      super(tag, es, chain, tag)
+    else
+      super(tag, es, chain)
+    end
   end
 
-  def format_collection_name(collection_name)
-    formatted = collection_name
-    formatted = formatted.gsub(@remove_prefix_collection, '') if @remove_prefix_collection
-    formatted = formatted.gsub(/(^\.+)|(\.+$)/, '')
-    formatted = @collection if formatted.size == 0 # set default for nil tag
-    formatted
+  def write(chunk)
+    # TODO: See emit comment
+    collection_name = @tag_mapped ? chunk.key : @collection
+    operate(collection_name, collect_records(chunk))
   end
 
   private
 
+  INSERT_ARGUMENT = {:collect_on_error => true}
+  BROKEN_DATA_KEY = '__broken_data'
+
   def operate(collection_name, records)
-    get_or_create_collection(collection_name).insert(records)
+    collection = get_or_create_collection(collection_name)
+    record_ids, error_records = collection.insert(records, INSERT_ARGUMENT)
+    if error_records
+      if @ignore_invalid_record
+        $log.warn "Ignore #{error_records.size} documents"
+      else
+        # Should create another collection like name_broken?
+        converted_records = error_records.map { |record|
+          new_record = {}
+          new_record[@tag_key] = record.delete(@tag_key) if @include_tag_key
+          new_record[@time_key] = record.delete(@time_key)
+          new_record[BROKEN_DATA_KEY] = Marshal.dump(record) # Should use BSON::ByteBuffer
+          new_record
+        }
+        collection.insert(converted_records)
+      end
+    end
+    records
   end
 
   def collect_records(chunk)
@@ -90,11 +128,21 @@ class MongoOutput < BufferedOutput
     records
   end
 
+  FORMAT_COLLECTION_NAME_RE = /(^\.+)|(\.+$)/
+
+  def format_collection_name(collection_name)
+    formatted = collection_name
+    formatted = formatted.gsub(@remove_tag_prefix, '') if @remove_tag_prefix
+    formatted = formatted.gsub(FORMAT_COLLECTION_NAME_RE, '')
+    formatted = @collection if formatted.size == 0 # set default for nil tag
+    formatted
+  end
+
   def get_or_create_collection(collection_name)
     collection_name = format_collection_name(collection_name)
     return @clients[collection_name] if @clients[collection_name]
 
-    @db ||= Mongo::Connection.new(@host, @port).db(@database)
+    @db ||= get_connection
     if @db.collection_names.include?(collection_name)
       collection = @db.collection(collection_name)
       unless @argument[:capped] == collection.capped? # TODO: Verify capped configuration
@@ -108,6 +156,10 @@ class MongoOutput < BufferedOutput
     @clients[collection_name] = collection
   end
 
+  def get_connection
+    Mongo::Connection.new(@host, @port).db(@database)
+  end
+
   # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
   LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024  # 2MB  = 4MB  / 2
   LIMIT_AFTER_v1_8 = 10 * 1024 * 1024  # 10MB = 16MB / 2 + alpha

data/lib/fluent/plugin/out_mongo_replset.rb

@@ -0,0 +1,73 @@
+require 'fluent/plugin/out_mongo'
+
+module Fluent
+
+
+class MongoOutputReplset < MongoOutput
+  Fluent::Plugin.register_output('mongo_replset', self)
+
+  config_param :nodes, :string
+  config_param :name, :string, :default => nil
+  config_param :read, :string, :default => nil
+  config_param :refresh_mode, :string, :default => nil
+  config_param :refresh_interval, :integer, :default => nil
+  config_param :num_retries, :integer, :default => 60
+
+  def configure(conf)
+    super
+
+    @nodes = parse_nodes(conf['nodes'])
+    @rs_argument = {}
+    if name = conf['name']
+      @rs_argument[:name] = conf['name']
+    end
+    if read = conf['read']
+      @rs_argument[:read] = read.to_sym
+    end
+    if refresh_mode = conf['refresh_mode']
+      @rs_argument[:refresh_mode] = refresh_mode.to_sym
+    end
+    if refresh_interval = conf['refresh_interval']
+      @rs_argument[:refresh_interval] = refresh_interval
+    end
+
+    $log.debug "Setup replica set configuration: nodes = #{conf['nodes']}"
+  end
+
+  private
+
+  def operate(collection_name, records)
+    collection = get_or_create_collection(collection_name)
+    rescue_connection_failure do
+      collection.insert(records)
+    end
+  end
+
+  def parse_nodes(nodes)
+    nodes.split(',').map { |node|
+      host, port = node.split(':')
+      [host, Integer(port)]
+    }
+  end
+
+  def get_connection
+    Mongo::ReplSetConnection.new(*@nodes, @rs_argument).db(@database)
+  end
+
+  def rescue_connection_failure
+    retries = 0
+    begin
+      yield
+    rescue Mongo::ConnectionFailure => e
+      retries += 1
+      raise e if retries > @num_retries
+
+      $log.warn "Failed to connect to Replica Set. Try to retry: retry number = #{retries}"
+      sleep 0.5
+      retry
+    end
+  end
+end
+
+
+end

data/lib/fluent/plugin/out_mongo_tag_collection.rb

@@ -11,17 +11,12 @@ class MongoOutputTagCollection < MongoOutput
   def configure(conf)
     super
 
+    @tag_mapped = true
     if remove_prefix_collection = conf['remove_prefix_collection']
-      @remove_prefix_collection = Regexp.new('^' + Regexp.escape(remove_prefix_collection))
+      @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_prefix_collection))
     end
-  end
-
-  def emit(tag, es, chain)
-    super(tag, es, chain, tag)
-  end
 
-  def write(chunk)
-    operate(chunk.key, collect_records(chunk))
+    $log.warn "'mongo_tag_collection' deprecated. Please use 'mongo' type with 'tag_mapped' parameter"
   end
 end
 

data/test/plugin/out_mongo.rb

@@ -82,6 +82,10 @@ class MongoOutputTest < Test::Unit::TestCase
     assert_equal('test', collection_name)
   end
 
+  def test_write_with_invalid_recoreds
+    skip('Implement this test using BSON directory later')
+  end
+
   def test_write_at_enable_tag
     d = create_driver(CONFIG + %[
       include_tag_key true

data/test/plugin/out_mongo_tag_collection.rb

@@ -1,20 +1,20 @@
-
 require 'test_helper'
 
 class MongoTagCollectionTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/out_mongo_tag_collection'
+    require 'fluent/plugin/out_mongo'
   end
 
   CONFIG = %[
-    type mongo_tag_collection
+    type mongo
     database fluent
-    remove_prefix_collection should.remove.
+    tag_mapped
+    remove_tag_prefix should.remove.
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputTagCollection) {
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput) {
       def start
         super
       end
@@ -31,7 +31,7 @@ class MongoTagCollectionTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver(CONFIG)
-    assert_equal(/^should\.remove\./, d.instance.instance_variable_get(:@remove_prefix_collection))
+    assert_equal(/^should\.remove\./, d.instance.instance_variable_get(:@remove_tag_prefix))
   end
 
   def emit_documents(d)
@@ -54,8 +54,8 @@ class MongoTagCollectionTest < Test::Unit::TestCase
 
   def test_remove_prefix_collection
     d = create_driver(CONFIG)
-    assert_equal('prefix', d.instance.format_collection_name('should.remove.prefix'))
-    assert_equal('test', d.instance.format_collection_name('..test..'))
-    assert_equal('test.foo', d.instance.format_collection_name('..test.foo.'))
+    assert_equal('prefix', d.instance.__send__(:format_collection_name, 'should.remove.prefix'))
+    assert_equal('test', d.instance.__send__(:format_collection_name, '..test..'))
+    assert_equal('test.foo', d.instance.__send__(:format_collection_name, '..test.foo.'))
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,33 +9,33 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-20 00:00:00.000000000Z
+date: 2012-01-15 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2160411800 !ruby/object:Gem::Requirement
+  requirement: &2155986740 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.10.6
+        version: 0.10.7
   type: :runtime
   prerelease: false
-  version_requirements: *2160411800
+  version_requirements: *2155986740
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2160411080 !ruby/object:Gem::Requirement
+  requirement: &2155985080 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - =
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.2
   type: :runtime
   prerelease: false
-  version_requirements: *2160411080
+  version_requirements: *2155985080
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2160410520 !ruby/object:Gem::Requirement
+  requirement: &2155982420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2160410520
+  version_requirements: *2155982420
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2160409960 !ruby/object:Gem::Requirement
+  requirement: &2155978740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *2160409960
+  version_requirements: *2155978740
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &2160409500 !ruby/object:Gem::Requirement
+  requirement: &2155976020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2160409500
+  version_requirements: *2155976020
 description: MongoDB plugin for Fluent event collector
 email: repeatedly@gmail.com
 executables:
@@ -84,8 +84,10 @@ files:
 - bin/mongo-tail
 - fluent-plugin-mongo.gemspec
 - lib/fluent/plugin/in_mongo_tail.rb
+- lib/fluent/plugin/mongo_ext.rb
 - lib/fluent/plugin/out_mongo.rb
 - lib/fluent/plugin/out_mongo_backup.rb
+- lib/fluent/plugin/out_mongo_replset.rb
 - lib/fluent/plugin/out_mongo_tag_collection.rb
 - test/plugin/in_mongo_tail.rb
 - test/plugin/out_mongo.rb
@@ -105,7 +107,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -744088123008270238
+      hash: -1464110601700401475
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -114,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -744088123008270238
+      hash: -1464110601700401475
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10