fluent-plugin-mongo 0.7.16 → 0.8.0.rc1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.7.16
+  version: 0.8.0.rc1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-06 00:00:00.000000000 Z
+date: 2016-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -36,14 +36,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -119,17 +119,14 @@ files:
 - bin/mongo-tail
 - fluent-plugin-mongo.gemspec
 - lib/fluent/plugin/in_mongo_tail.rb
-- lib/fluent/plugin/mongo_util.rb
+- lib/fluent/plugin/logger_support.rb
+- lib/fluent/plugin/mongo_auth.rb
 - lib/fluent/plugin/out_mongo.rb
 - lib/fluent/plugin/out_mongo_replset.rb
-- lib/fluent/plugin/out_mongo_tag_collection.rb
-- test/plugin/in_mongo_tail.rb
-- test/plugin/out_mongo.rb
-- test/plugin/out_mongo_tag_mapped.rb
-- test/test_helper.rb
-- test/tools/auth_repl_set_manager.rb
-- test/tools/repl_set_manager.rb
-- test/tools/rs_test_helper.rb
+- test/helper.rb
+- test/plugin/test_in_mongo_tail.rb
+- test/plugin/test_out_mongo.rb
+- test/plugin/test_out_mongo_replset.rb
 homepage: https://github.com/fluent/fluent-plugin-mongo
 licenses:
 - Apache-2.0
@@ -145,9 +142,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1
@@ -155,10 +152,7 @@ signing_key:
 specification_version: 4
 summary: MongoDB plugin for Fluentd
 test_files:
-- test/plugin/in_mongo_tail.rb
-- test/plugin/out_mongo.rb
-- test/plugin/out_mongo_tag_mapped.rb
-- test/test_helper.rb
-- test/tools/auth_repl_set_manager.rb
-- test/tools/repl_set_manager.rb
-- test/tools/rs_test_helper.rb
+- test/helper.rb
+- test/plugin/test_in_mongo_tail.rb
+- test/plugin/test_out_mongo.rb
+- test/plugin/test_out_mongo_replset.rb

data/lib/fluent/plugin/mongo_util.rb

@@ -1,27 +0,0 @@
-module Fluent
-
-
-module MongoUtil
-  def self.included(klass)
-    klass.instance_eval {
-      config_param :user, :string, :default => nil
-      config_param :password, :string, :default => nil, :secret => true
-    }
-  end
-
-  def authenticate(db)
-    unless @user.nil? || @password.nil?
-      begin
-        db.authenticate(@user, @password)
-      rescue Mongo::AuthenticationError => e
-        log.fatal e
-        exit!
-      end
-    end
-
-    db
-  end
-end
-
-
-end

data/lib/fluent/plugin/out_mongo_tag_collection.rb

@@ -1,20 +0,0 @@
-require 'fluent/plugin/out_mongo'
-
-module Fluent
-  class MongoOutputTagCollection < MongoOutput
-    Plugin.register_output('mongo_tag_collection', self)
-
-    config_param :collection, :string, :default => 'untagged'
-
-    def configure(conf)
-      super
-
-      @tag_mapped = true
-      if remove_prefix_collection = conf['remove_prefix_collection']
-        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_prefix_collection))
-      end
-
-      log.warn "'mongo_tag_collection' deprecated. Please use 'mongo' type with 'tag_mapped' parameter"
-    end
-  end
-end

data/test/plugin/in_mongo_tail.rb

@@ -1,73 +0,0 @@
-require 'test_helper'
-require 'fluent/plugin/in_mongo_tail'
-
-class MongoTailInputTest < Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  CONFIG = %[
-    type mongo_tail
-    database test
-    collection log
-    tag_key tag
-    time_key time
-    id_store_file /tmp/fluent_mongo_last_id
-    id_store_collection test_last_id
-  ]
-
-  def create_driver(conf = CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::MongoTailInput).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal('localhost', d.instance.host)
-    assert_equal(27017, d.instance.port)
-    assert_equal('test', d.instance.database)
-    assert_equal('log', d.instance.collection)
-    assert_equal('tag', d.instance.tag_key)
-    assert_equal('time', d.instance.time_key)
-    assert_equal('/tmp/fluent_mongo_last_id', d.instance.id_store_file)
-    assert_equal('test_last_id', d.instance.id_store_collection)
-  end
-  
-  def test_url_configration
-    config = %[
-    type mongo_tail
-    url mongodb://localhost:27017/test
-    collection log
-    tag_key tag
-    time_key time
-    id_store_file /tmp/fluent_mongo_last_id
-    ]
-    
-    d = create_driver(config)
-    assert_equal("mongodb://localhost:27017/test", d.instance.url)
-    assert_nil(d.instance.database)
-    assert_equal('log', d.instance.collection)
-    assert_equal('tag', d.instance.tag_key)
-    assert_equal('time', d.instance.time_key)
-    assert_equal('/tmp/fluent_mongo_last_id', d.instance.id_store_file)
-  end
-
-  def test_url_and_database_can_not_exist
-    config = %[
-    type mongo_tail
-    url mongodb://localhost:27017/test
-    database test2
-    collection log
-    tag_key tag
-    time_key time
-    id_store_file /tmp/fluent_mongo_last_id
-    ]
-
-    assert_raises Fluent::ConfigError do
-      create_driver(config)
-    end
-  end
-
-  def test_emit
-    # TODO: write actual code
-  end
-end

data/test/plugin/out_mongo.rb

@@ -1,298 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'tools/rs_test_helper'
-require 'fluent/plugin/out_mongo'
-
-module MongoOutputTestCases
-  def collection_name
-    'test'
-  end
-
-  def test_configure_with_write_concern
-    d = create_driver(default_config + %[
-      write_concern 2
-    ])
-
-    assert_equal({:w => 2, :pool_size => 1, :ssl => false, :j => false}, d.instance.connection_options)
-  end
-
-  def test_configure_with_journaled
-    d = create_driver(default_config + %[
-      journaled true
-    ])
-
-    assert_equal({:ssl => false, :pool_size => 1, :j => true}, d.instance.connection_options)
-  end
-
-  def test_configure_with_ssl
-    d = create_driver(default_config + %[
-      ssl true
-    ])
-
-    assert_equal({:ssl => true, :pool_size => 1, :j => false, :ssl_cert=>nil, :ssl_key=>nil, :ssl_key_pass_phrase=>nil, :ssl_verify=>false, :ssl_ca_cert=>nil}, d.instance.connection_options)
-  end
-
-  def test_format
-    d = create_driver
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 1}, time)
-    d.emit({'a' => 2}, time)
-    d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
-    d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
-    d.run
-
-    assert_equal(2, @db.collection(collection_name).count)
-  end
-
-  def emit_documents(d)
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 1}, time)
-    d.emit({'a' => 2}, time)
-    time
-  end
-
-  def get_documents
-    @db.collection(collection_name).find().to_a.map { |e| e.delete('_id'); e }
-  end
-
-  def test_write
-    d = create_driver
-    t = emit_documents(d)
-
-    d.run
-    documents = get_documents.map { |e| e['a'] }.sort
-    assert_equal([1, 2], documents)
-    assert_equal(2, documents.size)
-  end
-
-  def test_write_at_enable_tag
-    d = create_driver(default_config + %[
-      include_tag_key true
-      include_time_key false
-    ])
-    t = emit_documents(d)
-
-    d.run
-    documents = get_documents.sort_by { |e| e['a'] }
-    assert_equal([{'a' => 1, d.instance.tag_key => 'test'},
-                  {'a' => 2, d.instance.tag_key => 'test'}], documents)
-    assert_equal(2, documents.size)
-  end
-
-  def emit_invalid_documents(d)
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({'a' => 3, 'b' => "c", '$last' => '石動'}, time)
-    d.emit({'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')}, time)
-    time
-  end
-
-  def test_write_with_invalid_recoreds_with_keys_containing_dot_and_dollar
-    d = create_driver(default_config + %[
-      replace_dot_in_key_with _dot_
-      replace_dollar_in_key_with _dollar_
-    ])
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({
-      "foo.bar1" => {
-        "$foo$bar" => "baz"
-      },
-      "foo.bar2" => [
-        {
-          "$foo$bar" => "baz"
-        }
-      ],
-    }, time)
-    d.run
-
-    documents = get_documents
-    assert_equal(1, documents.size)
-    assert_equal("baz", documents[0]["foo_dot_bar1"]["_dollar_foo$bar"])
-    assert_equal("baz", documents[0]["foo_dot_bar2"][0]["_dollar_foo$bar"])
-    assert_equal(0, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY)}.size)
-  end
-
-  def test_write_with_invalid_recoreds
-    d = create_driver
-    t = emit_documents(d)
-    t = emit_invalid_documents(d)
-
-    d.run
-    documents = get_documents
-    assert_equal(4, documents.size)
-    assert_equal([1, 2], documents.select { |e| e.has_key?('a') }.map { |e| e['a'] }.sort)
-    assert_equal(2, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY)}.size)
-    assert_equal([3, 4], @db.collection(collection_name).find({Fluent::MongoOutput::BROKEN_DATA_KEY => {'$exists' => true}}).map { |doc|
-      Marshal.load(doc[Fluent::MongoOutput::BROKEN_DATA_KEY].to_s)['a']
-    }.sort)
-  end
-
-  def test_write_with_invalid_recoreds_with_exclude_one_broken_fields
-    d = create_driver(default_config + %[
-      exclude_broken_fields a
-    ])
-    t = emit_documents(d)
-    t = emit_invalid_documents(d)
-
-    d.run
-    documents = get_documents
-    assert_equal(4, documents.size)
-    assert_equal(2, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY) }.size)
-    assert_equal([1, 2, 3, 4], documents.select { |e| e.has_key?('a') }.map { |e| e['a'] }.sort)
-    assert_equal(0, documents.select { |e| e.has_key?('b') }.size)
-  end
-
-  def test_write_with_invalid_recoreds_with_exclude_two_broken_fields
-    d = create_driver(default_config + %[
-      exclude_broken_fields a,b
-    ])
-    t = emit_documents(d)
-    t = emit_invalid_documents(d)
-
-    d.run
-    documents = get_documents
-    assert_equal(4, documents.size)
-    assert_equal(2, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY) }.size)
-    assert_equal([1, 2, 3, 4], documents.select { |e| e.has_key?('a') }.map { |e| e['a'] }.sort)
-    assert_equal(["c", "d"], documents.select { |e| e.has_key?('b') }.map { |e| e['b'] }.sort)
-  end
-
-  def test_write_with_invalid_recoreds_at_ignore
-    d = create_driver(default_config + %[
-      ignore_invalid_record true
-    ])
-    t = emit_documents(d)
-    t = emit_invalid_documents(d)
-
-    d.run
-    documents = get_documents
-    assert_equal(2, documents.size)
-    assert_equal([1, 2], documents.select { |e| e.has_key?('a') }.map { |e| e['a'] }.sort)
-    assert_equal(true, @db.collection(collection_name).find({Fluent::MongoOutput::BROKEN_DATA_KEY => {'$exists' => true}}).count.zero?)
-  end
-end
-
-class MongoOutputTest < Test::Unit::TestCase
-  include MongoOutputTestCases
-
-  class << self
-    def startup
-      MongoTestHelper.setup_mongod
-    end
-
-    def shutdown
-      MongoTestHelper.teardown_mongod
-    end
-  end
-
-  def setup
-    Fluent::Test.setup
-  end
-
-  def teardown
-    @db.collection(collection_name).drop
-  end
-
-  def default_config
-    %[
-      type mongo
-      database #{MONGO_DB_DB}
-      collection #{collection_name}
-      include_time_key true # TestDriver ignore config_set_default?
-    ]
-  end
-
-  def create_driver(conf = default_config)
-    conf = conf + %[
-      port #{MongoTestHelper.mongod_port}
-    ]
-    @db = Mongo::MongoClient.new('localhost', MongoTestHelper.mongod_port).db(MONGO_DB_DB)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver(%[
-      type mongo
-      database fluent_test
-      collection test_collection
-
-      capped
-      capped_size 100
-    ])
-
-    assert_equal('fluent_test', d.instance.database)
-    assert_equal('test_collection', d.instance.collection)
-    assert_equal('localhost', d.instance.host)
-    assert_equal(MongoTestHelper.mongod_port, d.instance.port)
-    assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
-    assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
-    # buffer_chunk_limit moved from configure to start
-    # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
-    # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
-  end
-end
-
-class MongoReplOutputTest < Test::Unit::TestCase
-  include MongoOutputTestCases
-
-  class << self
-    def startup
-      setup_rs
-    end
-
-    def shutdown
-      teardown_rs
-    end
-  end
-
-  def setup
-    Fluent::Test.setup
-    require 'fluent/plugin/out_mongo_replset'
-  end
-
-  def teardown
-    @@rs.restart_killed_nodes
-    if defined?(@db) && @db
-      @db.collection(collection_name).drop
-      @db.connection.close
-    end
-  end
-
-  def default_config
-    %[
-      type mongo_replset
-      database #{MONGO_DB_DB}
-      collection #{collection_name}
-      nodes #{build_seeds(3).join(',')}
-      num_retries 30
-      include_time_key true # TestDriver ignore config_set_default?
-    ]
-  end
-
-  def create_driver(conf = default_config)
-    @db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @@rs.name).db(MONGO_DB_DB)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset).configure(conf)
-  end
-
-  def test_configure
-    d = create_driver(%[
-      type mongo_replset
-
-      database fluent_test
-      collection test_collection
-      nodes #{build_seeds(3).join(',')}
-      num_retries 45
-
-      capped
-      capped_size 100
-    ])
-
-    assert_equal('fluent_test', d.instance.database)
-    assert_equal('test_collection', d.instance.collection)
-    assert_equal(build_seeds(3), d.instance.nodes)
-    assert_equal(45, d.instance.num_retries)
-    assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
-    assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
-  end
-end
-