fluent-plugin-mongo 0.6.2 → 0.6.3

data/README.rdoc

@@ -1,33 +1,26 @@
-= MongoDB plugin for Fluent
+= MongoDB plugin for Fluentd
 
-== Component
+fluent-plugin-mongo provides input and output plugins for {Fluentd}[http://fluentd.org/].
 
-=== MongoOutput
+= Installation
 
-Store Fluentd event as a MongoDB Document to MongoDB database.
+== Gems
 
-MongoOutput set "time" field to a document by default.
-You set _false_ to _include_time_key_ parameter if you disable this behaivor.
+The gem is hosted at {Rubygems.org}[http://rubygems.org]. You can install the gem as follows:
 
-==== tag mapped mode
+    $ fluent-gem install fluent-plugin-mongo
 
-Tag mapped to MongoDB collection automatically.
-
-=== MongoReplSetOutput
-
-Replica Set version of MongoOutput
+= Plugins
 
-=== MongoBackupOutput
+== Output plugin
 
-Store fluent-event to local capped collection for backup.
+=== mongo
 
-=== MongoTailInput
+Store Fluentd event to MongoDB database.
 
-Tail capped collection to input data.
-
-== Configuration
+==== Configuration
 
-=== MongoOutput
+Use _mongo_ type in match.
 
     <match mongo.**>
       type mongo
@@ -42,12 +35,20 @@ Tail capped collection to input data.
       capped
       capped_size 100m
 
+      # Set 'user' and 'password' for authentication
+      user handa
+      password shinobu
+
       # Other buffer configurations here
     </match>
 
-==== Tag mapped mode
+=== mongo(tag mapped mode)
 
-Use _tag_mapped_ parameter.
+Tag mapped to MongoDB collection automatically.
+
+==== Configuration
+
+Use _tag_mapped_ parameter in match of _mongo_ type.
 
 If tag name is "foo.bar", auto create collection "foo.bar" and insert data.
 
@@ -68,48 +69,11 @@ If tag name is "foo.bar", auto create collection "foo.bar" and insert data.
       # Other configurations here
     </match>
 
-==== NOTE
+=== mongo_replset
 
-===== Broken data as a BSON
+Replica Set version of mongo.
 
-Fluentd event sometimes has an invalid record as a BSON.
-In such case, Mongo plugin marshals an invalid record using Marshal.dump
-and re-inserts its to same collection.
-
-If passed following invalid record:
-
-    {"key1": "invalid value", "key2": "valid value", "time": ISODate("2012-01-15T21:09:53Z") }
-
-then Mongo plugin converts this record to following format:
-
-    {"__broken_data": Marshal.dump result of {"key1": "invalid value", "key2": "valid value"}, "time": ISODate("2012-01-15T21:09:53Z") }
-
-Mongo-Ruby-Driver cannot detect an invalid attribute,
-so Mongo plugin marshals all attributes excluding Fluentd keys("tag_key" and "time_key").
-
-If you want to ignore an invalid record, set _true_ to _ignore_invalid_record_ parameter in match.
-
-    <match forward.*>
-      ...
-
-      # ignore invalid documents at write operation
-      ignore_invalid_document true
-
-      ...
-    </match>
-
-===== Buffer size limitation
-
-Mongo plugin has the limitation of buffer size.
-Because MongoDB and mongo-ruby-driver checks the total object size at each insertion.
-If total object size gets over the size limitation, then
-MongoDB returns error or mongo-ruby-driver raises an exception.
-
-So, Mongo plugin resets _buffer_chunk_limit_ if configurated value is larger than above limitation:
-- Before v1.8, max of _buffer_chunk_limit_ is 2MB
-- After v1.8, max of _buffer_chunk_limit_ is 10MB
-
-=== MongoReplSetOutput
+==== Configuration
 
 Use _mongo_replset_ type in match.
 
@@ -133,7 +97,11 @@ Use _mongo_replset_ type in match.
       #refresh_interval 60
     </match>
 
-=== MongoBackupOutput
+=== mongo_backup
+
+Store Fluentd event to local capped collection for backup.
+
+==== Configuration
 
 Use _mongo_backup_ type in match. _mongo_backup_ alwalys use capped collection.
 
@@ -148,7 +116,13 @@ Use _mongo_backup_ type in match. _mongo_backup_ alwalys use capped collection.
       </store>
     </match>
 
-=== Tail capped collection
+== Input plugin
+
+=== mongo_tail
+
+Tail capped collection to input data.
+
+==== Configuration
 
 Use _mongo_tail_ type in source.
 
@@ -166,21 +140,61 @@ Use _mongo_tail_ type in source.
       id_store_file /Users/repeatedly/devel/fluent-plugin-mongo/last_id
     </source>
 
-== TODO
+= NOTE
 
-=== More configuration
+== Broken data as a BSON
 
-- Authentication
-- Select insert or update
-- etc
+Fluentd event sometimes has an invalid record as a BSON.
+In such case, Mongo plugin marshals an invalid record using Marshal.dump
+and re-inserts its to same collection.
+
+If passed following invalid record:
+
+    {"key1": "invalid value", "key2": "valid value", "time": ISODate("2012-01-15T21:09:53Z") }
+
+then Mongo plugin converts this record to following format:
+
+    {"__broken_data": Marshal.dump result of {"key1": "invalid value", "key2": "valid value"}, "time": ISODate("2012-01-15T21:09:53Z") }
+
+Mongo-Ruby-Driver cannot detect an invalid attribute,
+so Mongo plugin marshals all attributes excluding Fluentd keys("tag_key" and "time_key").
+
+If you want to ignore an invalid record, set _true_ to _ignore_invalid_record_ parameter in match.
 
-== Tool
+    <match forward.*>
+      ...
+
+      # ignore invalid documents at write operation
+      ignore_invalid_record true
+
+      ...
+    </match>
+
+== Buffer size limitation
+
+Mongo plugin has the limitation of buffer size.
+Because MongoDB and mongo-ruby-driver checks the total object size at each insertion.
+If total object size gets over the size limitation, then
+MongoDB returns error or mongo-ruby-driver raises an exception.
+
+So, Mongo plugin resets _buffer_chunk_limit_ if configurated value is larger than above limitation:
+- Before v1.8, max of _buffer_chunk_limit_ is 2MB
+- After v1.8, max of _buffer_chunk_limit_ is 10MB
+
+= Tool
 
 You can tail mongo capped collection.
 
-    mongo-tail -f
+    $ mongo-tail -f
+
+= TODO
+
+== More configuration
+
+- Multi process
+- etc
 
-== Copyright
+= Copyright
 
 Copyright:: Copyright (c) 2011- Masahiro Nakagawa
 License::   Apache License, Version 2.0

data/VERSION

@@ -1 +1 @@
-0.6.2
+0.6.3

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -4,6 +4,9 @@ module Fluent
 class MongoTailInput < Input
   Plugin.register_input('mongo_tail', self)
 
+  require 'fluent/plugin/mongo_util'
+  include MongoUtil
+
   config_param :database, :string
   config_param :collection, :string
   config_param :host, :string, :default => 'localhost'
@@ -18,9 +21,9 @@ class MongoTailInput < Input
   config_param :id_store_file, :string, :default => nil
 
   def initialize
+    super
     require 'mongo'
     require 'bson'
-    super
   end
 
   def configure(conf)
@@ -62,11 +65,19 @@ class MongoTailInput < Input
   private
 
   def get_capped_collection
-    db = Mongo::Connection.new(@host, @port).db(@database)
-    raise ConfigError, "'#{@database}.#{@collection}' not found: node = #{@host}:#{@port}" unless db.collection_names.include?(@collection)
-    collection = db.collection(@collection)
-    raise ConfigError, "'#{@database}.#{@collection}' is not capped: node = #{@host}:#{@port}" unless collection.capped?
-    collection
+    begin
+      db = authenticate(Mongo::Connection.new(@host, @port).db(@database))
+      raise ConfigError, "'#{@database}.#{@collection}' not found: node = #{@host}:#{@port}" unless db.collection_names.include?(@collection)
+      collection = db.collection(@collection)
+      raise ConfigError, "'#{@database}.#{@collection}' is not capped: node = #{@host}:#{@port}" unless collection.capped?
+      collection
+    rescue Mongo::ConnectionFailure => e
+      $log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
+      exit!
+    rescue Mongo::OperationFailure => e
+      $log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
+      exit!
+    end
   end
 
   def tailoop(cursor)

data/lib/fluent/plugin/mongo_util.rb

@@ -0,0 +1,27 @@
+module Fluent
+
+
+module MongoUtil
+  def self.included(klass)
+    klass.instance_eval {
+      config_param :user, :string, :default => nil
+      config_param :password, :string, :default => nil
+    }
+  end
+
+  def authenticate(db)
+    unless @user.nil? || @password.nil?
+      begin
+        db.authenticate(@user, @password)
+      rescue Mongo::AuthenticationError => e
+        $log.fatal e
+        exit!
+      end
+    end
+
+    db
+  end
+end
+
+
+end

data/lib/fluent/plugin/out_mongo.rb

@@ -4,6 +4,9 @@ module Fluent
 class MongoOutput < BufferedOutput
   Fluent::Plugin.register_output('mongo', self)
 
+  require 'fluent/plugin/mongo_util'
+  include MongoUtil
+
   include SetTagKeyMixin
   config_set_default :include_tag_key, false
 
@@ -26,8 +29,8 @@ class MongoOutput < BufferedOutput
   def initialize
     super
     require 'mongo'
-    require 'fluent/plugin/mongo_ext'
     require 'msgpack'
+    require 'fluent/plugin/mongo_ext'
 
     @clients = {}
     @connection_options = {}
@@ -168,7 +171,8 @@ class MongoOutput < BufferedOutput
   end
 
   def get_connection
-    Mongo::Connection.new(@host, @port, @connection_options).db(@database)
+    db = Mongo::Connection.new(@host, @port, @connection_options).db(@database)
+    authenticate(db)
   end
 
   # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -179,9 +183,13 @@ class MongoOutput < BufferedOutput
     begin
       limit = mongod_version >= "1.8.0" ? LIMIT_AFTER_v1_8 : LIMIT_BEFORE_v1_8  # TODO: each version comparison
     rescue Mongo::ConnectionFailure => e
-      $log.warn "mongo connection failed, set #{LIMIT_BEFORE_v1_8} to chunk limit"
-      limit = LIMIT_BEFORE_v1_8
+      $log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
+      exit!
+    rescue Mongo::OperationFailure => e
+      $log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
+      exit!
     rescue Exception => e
+      # TODO: should do exit?
       $log.warn "mongo unknown error #{e}, set #{LIMIT_BEFORE_v1_8} to chunk limit"
       limit = LIMIT_BEFORE_v1_8
     end
@@ -195,7 +203,8 @@ class MongoOutput < BufferedOutput
   end
 
   def mongod_version
-    Mongo::Connection.new(@host, @port).db('admin').command('serverStatus' => 1)['version']
+    db = authenticate(Mongo::Connection.new(@host, @port).db('admin'))
+    db.command('serverStatus' => 1)['version']
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-23 00:00:00.000000000Z
+date: 2012-02-07 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2152067980 !ruby/object:Gem::Requirement
+  requirement: &2152356120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 0.10.7
   type: :runtime
   prerelease: false
-  version_requirements: *2152067980
+  version_requirements: *2152356120
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &2152067040 !ruby/object:Gem::Requirement
+  requirement: &2152354440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -35,10 +35,10 @@ dependencies:
         version: 1.5.2
   type: :runtime
   prerelease: false
-  version_requirements: *2152067040
+  version_requirements: *2152354440
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2152065820 !ruby/object:Gem::Requirement
+  requirement: &2152352640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -46,10 +46,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2152065820
+  version_requirements: *2152352640
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2152064740 !ruby/object:Gem::Requirement
+  requirement: &2152351520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -57,10 +57,10 @@ dependencies:
         version: 0.5.4
   type: :development
   prerelease: false
-  version_requirements: *2152064740
+  version_requirements: *2152351520
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &2152042360 !ruby/object:Gem::Requirement
+  requirement: &2152350480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -68,7 +68,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *2152042360
+  version_requirements: *2152350480
 description: MongoDB plugin for Fluent event collector
 email: repeatedly@gmail.com
 executables:
@@ -88,6 +88,7 @@ files:
 - fluent-plugin-mongo.gemspec
 - lib/fluent/plugin/in_mongo_tail.rb
 - lib/fluent/plugin/mongo_ext.rb
+- lib/fluent/plugin/mongo_util.rb
 - lib/fluent/plugin/out_mongo.rb
 - lib/fluent/plugin/out_mongo_backup.rb
 - lib/fluent/plugin/out_mongo_replset.rb
@@ -110,7 +111,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2968858870877082029
+      hash: 688925002882805851
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -119,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2968858870877082029
+      hash: 688925002882805851
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10