fluent-plugin-mongo 0.3.1 → 0.4.0

data/README.rdoc

@@ -6,6 +6,9 @@
 
 Store fluent-event as MongoDB Document to MongoDB database.
 
+MongoOutput set 'time' field to a document by default.
+You set -false- to 'include_time_key' parameter if you disable this behaivor.
+
 === MongoBackup
 
 Store fluent-event to capped collection for backup.

data/Rakefile

@@ -12,7 +12,7 @@ begin
     gemspec.homepage = "http://github.com/fluent"
     gemspec.has_rdoc = false
     gemspec.require_paths = ["lib"]
-    gemspec.add_dependency "fluent", "~> 0.9.14"
+    gemspec.add_dependency "fluentd", "~> 0.10.1"
     gemspec.add_dependency "mongo", ">= 1.2.0"
     gemspec.test_files = Dir["test/**/*.rb"]
     gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] + %w[VERSION AUTHORS Rakefile]

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

data/lib/fluent/plugin/out_mongo.rb

@@ -4,81 +4,85 @@ module Fluent
 class MongoOutput < BufferedOutput
   Fluent::Plugin.register_output('mongo', self)
 
+  include SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  include SetTimeKeyMixin
+  config_set_default :include_time_key, true
+
+  config_param :database, :string
+  config_param :collection, :string
+  config_param :host, :string, :default => 'localhost'
+  config_param :port, :integer, :default => 27017
+
+  attr_reader :argument
+
   def initialize
     super
     require 'mongo'
     require 'msgpack'
 
-    # Sub-class can overwrite following parameters
-    @database_name = nil
-    @collection_name = nil
+    @argument = {:capped => false}
   end
 
   def configure(conf)
     super
 
-    @database_name = conf['database'] if conf.has_key?('database')
-    @collection_name = conf['collection'] if conf.has_key?('collection')
-    raise ConfigError, "'database' and 'collection' parameter is required on mongo output" if @database_name.nil? || @collection_name.nil?
-    @host, @port = host_and_port(conf)
-
     # capped configuration
-    @argument = {:capped => false}
-    if conf['capped']
+    if conf.has_key?('capped')
       raise ConfigError, "'capped_size' parameter is required on <store> of Mongo output" unless conf.has_key?('capped_size')
       @argument[:capped] = true
       @argument[:size] = Config.size_value(conf['capped_size'])
       @argument[:max] = Config.size_value(conf['capped_max']) if conf.has_key?('capped_max')
     end
+
+    # MongoDB uses BSON's Date for time.
+    def @timef.format_nocache(time)
+      time
+    end
   end
 
   def start
     super
-    @collection = get_or_create_collection #Mongo::Connection.new(@host, @port).db(@database_name).collection(@collection_name)
+    @client = get_or_create_collection
   end
 
   def shutdown
     # Mongo::Connection checks alive or closed myself
-    @collection.db.connection.close
+    @client.db.connection.close
     super
   end
 
-  def format(tag, event)
-    event.record.to_msgpack
+  def format(tag, time, record)
+    record.to_msgpack
   end
 
   def write(chunk)
     records = []
-    chunk.open { |io|
-      begin
-        MessagePack::Unpacker.new(io).each { |record| records << record }
-      rescue EOFError
-        # EOFError always occured when reached end of chunk.
-      end
+    chunk.msgpack_each { |record|
+      record[@time_key] = Time.at(record[@time_key]) if @include_time_key
+      records << record
     }
-
-    @collection.insert(records)
+    operate(records)
   end
 
   private
 
-  def host_and_port(conf)
-    host = conf['host'] || 'localhost'
-    port = conf['port'] || 27017
-    [host, Integer(port)]
-  end
-
   def get_or_create_collection
-    db = Mongo::Connection.new(@host, @port).db(@database_name)
-    if db.collection_names.include?(@collection_name)
-      collection = db.collection(@collection_name)
+    db = Mongo::Connection.new(@host, @port).db(@database)
+    if db.collection_names.include?(@collection)
+      collection = db.collection(@collection)
       return collection if @argument[:capped] == collection.capped? # TODO: Verify capped configuration
 
       # raise Exception if old collection does not match lastest configuration
       raise ConfigError, "New configuration is different from existing collection"
     end
 
-    db.create_collection(@collection_name, @argument)
+    db.create_collection(@collection, @argument)
+  end
+
+  def operate(records)
+    @client.insert(records)
   end
 end
 

data/lib/fluent/plugin/out_mongo_backup.rb

@@ -9,13 +9,10 @@ class MongoBackupOutput < CopyOutput
   Fluent::Plugin.register_output('mongo_backup', self)
 
   class MongoOutputForBackup < MongoOutput
-    def initialize
-      super
+    config_param :database, :string, :default => 'fluent'
+    config_param :collection, :string, :default => 'out_mongo_backup'
 
-      # default parameters
-      @database_name = 'fluent'
-      @collection_name = 'out_mongo_backup'
-    end
+    # TODO: optimize
   end
 
   def configure(conf)

data/test/plugin/out_mongo.rb

@@ -0,0 +1,91 @@
+require 'test/unit'
+require 'fluent/test'
+
+class MongoOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    require 'fluent/plugin/out_mongo'
+  end
+
+  CONFIG = %[
+    type mongo
+    database fluent
+    collection test
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput) {
+      def start
+        super
+      end
+
+      def shutdown
+        super
+      end
+
+      def operate(records)
+        records
+      end
+    }.configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      type mongo
+      database fluent_test
+      collection test_collection
+
+      host fluenter
+      port 27018
+
+      capped
+      capped_size 100
+    ])
+
+    assert_equal('fluent_test', d.instance.database)
+    assert_equal('test_collection', d.instance.collection)
+    assert_equal('fluenter', d.instance.host)
+    assert_equal(27018, d.instance.port)
+    assert_equal({:capped => true, :size => 100}, d.instance.argument)
+  end
+
+  def test_format
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({'a' => 1}, time)
+    d.emit({'a' => 2}, time)
+    d.expect_format({'a' => 1, d.instance.time_key => time}.to_msgpack)
+    d.expect_format({'a' => 2, d.instance.time_key => time}.to_msgpack)
+
+    d.run
+  end
+
+  def emit_documents(d)
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({'a' => 1}, time)
+    d.emit({'a' => 2}, time)
+    time
+  end
+
+  def test_write
+    d = create_driver
+    t = emit_documents(d)
+
+    documents = d.run
+    assert_equal([{'a' => 1, d.instance.time_key => Time.at(t)},
+                  {'a' => 2, d.instance.time_key => Time.at(t)}], documents)
+  end
+
+  def test_write_at_enable_tag
+    d = create_driver(CONFIG + %[
+        include_tag_key true
+        include_time_key false
+    ])
+    t = emit_documents(d)
+
+    documents = d.run
+    assert_equal([{'a' => 1, d.instance.tag_key => 'test'},
+                  {'a' => 2, d.instance.tag_key => 'test'}], documents)
+  end
+end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 3
-  - 1
-  version: 0.3.1
+  - 4
+  - 0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Masahiro Nakagawa
@@ -14,11 +14,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-05 00:00:00 +09:00
+date: 2011-10-16 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: fluent
+  name: fluentd
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -27,9 +27,9 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        - 9
-        - 14
-        version: 0.9.14
+        - 10
+        - 1
+        version: 0.10.1
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -62,6 +62,7 @@ files:
 - bin/mongo-tail
 - lib/fluent/plugin/out_mongo.rb
 - lib/fluent/plugin/out_mongo_backup.rb
+- test/plugin/out_mongo.rb
 - README.rdoc
 has_rdoc: true
 homepage: http://github.com/fluent
@@ -95,5 +96,5 @@ rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: MongoDB output plugin for Fluent event collector
-test_files: []
-
+test_files: 
+- test/plugin/out_mongo.rb