fluent-plugin-mongo 1.0.0.rc1 → 1.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 130e2e8712112dcda7b7d44b19448f5cd2177ccc
-  data.tar.gz: 79e4d7512049f24a830ccfef2b973084f9847b21
+  metadata.gz: 0374d7478e773ffdeba70eba9f7ba7df050ee22a
+  data.tar.gz: a6c4e10042582db77666352673e1a7ee7a812f74
 SHA512:
-  metadata.gz: 635611797a2fae046e0961801e8a7f1fee7d436fcc2356406f5227b78916c6e7feb954b2d1d4d129e4573dffdf9c2627fbada20b35af35663e6ca33237cfe1ff
-  data.tar.gz: 642e6babffd923a6426d0bc4bb85d64608b45e372f8bb3d30de755488737d8c0d4d25eb0266a66ad226c2b9c506115b24d2d07bd08f4032f9c5d49ef33f9e200
+  metadata.gz: 8d0877102caad677c07d60a9df53585c8ce1693f6de16fc96764305fa47caad39bdfbefd6d56d40a183bf76bebcd9e771595347870f1de1e8dcda1e138be02fc
+  data.tar.gz: 29061649d0edf590b18ad8192141e14980517dc07a80fa5544504a1bff02d64a33a9feeb435791797142f79d47fe9f28befa3dd2f06766bf8e70aaa93d228471

data/README.rdoc

@@ -7,7 +7,7 @@ fluent-plugin-mongo provides input and output plugins for {Fluentd}[http://fluen
     |fluent-plugin-mongo|   fluentd  |  ruby  |
     |-------------------|------------|--------|
     |     >= 1.0.0      | >= 0.14.12 | >= 2.1 |
-    |     <  1.0.0      | ~> 0.12.0  | >= 1.9 |
+    |     <  1.0.0      | >= 0.12.0  | >= 1.9 |
 
 = Installation
 
@@ -82,45 +82,47 @@ Replica Set version of mongo.
 
 ==== Configuration
 
-Use _mongo_replset_ type in match.
+===== v0.8 or later
 
     <match mongo.**>
       type mongo_replset
       database fluent
       collection logs
 
-      # each node separated by ','
-      nodes localhost:27017,localhost:27018,localhost:27019
+      host localhost
+      port 27017
+
+      # The replica set name
+      replica_set myapp
 
       # num_retries is threshold at failover, default is 60.
       # If retry count reached this threshold, mongo plugin raises an exception.
       num_retries 30
 
-      # following optional parameters passed to ReplSetConnection of mongo-ruby-driver.
-      # See mongo-ruby-driver docs for more detail.
-      #name replset_name
+      # following optional parameters passed to mongo-ruby-driver.
+      # See mongo-ruby-driver docs for more detail: https://docs.mongodb.com/ruby-driver/master/tutorials/ruby-driver-create-client/
+      # Specifies the read preference mode
       #read secondary
-      #refresh_mode sync
-      #refresh_interval 60
     </match>
 
-=== mongo_backup
-
-Store Fluentd event to local capped collection for backup.
+===== v0.7 or ealier
 
-==== Configuration
+Use _mongo_replset_ type in match.
 
-Use _mongo_backup_ type in match. _mongo_backup_ alwalys use capped collection.
+    <match mongo.**>
+      type mongo_replset
+      database fluent
+      collection logs
 
-    <match ...>
-      type mongo_backup
-      capped_size 100m
+      # each node separated by ','
+      nodes localhost:27017,localhost:27018,localhost:27019
 
-      <store>
-        type tcp
-        host 192.168.0.13
-        ...
-      </store>
+      # following optional parameters passed to mongo-ruby-driver.
+      #name replset_name
+      #read secondary
+      #refresh_mode sync
+      #refresh_interval 60
+      #num_retries 60
     </match>
 
 == Input plugin
@@ -181,6 +183,21 @@ Make sure the collection is capped. The plugin inserts records but does not remo
 
 = NOTE
 
+== replace_dot_in_key_with and replace_dollar_in_key_with
+
+BSON records which include '.' or start with '$' are invalid and they will be stored as broken data to MongoDB. If you want to sanitize keys, you can use _replace_dot_in_key_with_ and _replace_dollar_in_key_with_.
+
+    <match forward.*>
+      ...
+      # replace '.' in keys with '__dot__'
+      replace_dot_in_key_with __dot__
+    
+      # replace '$' in keys with '__dollar__'
+      # Note: This replaces '$' only on first character
+      replace_dollar_in_key_with __dollar__
+      ...
+    </match>
+
 == Broken data as a BSON
 
 NOTE: This feature will be removed since v0.8
@@ -237,21 +254,6 @@ If you want to exclude some fields from broken data marshaling, use _exclude_bro
 Specified value is a comma separated keys(e.g. key1,key2,key3).
 This parameter is useful for excluding shard keys in shard environment.
 
-=== replace_dot_in_key_with and replace_dollar_in_key_with
-
-BSON records which include '.' or start with '$' are invalid and they will be stored as broken data to MongoDB. If you want to sanitize keys, you can use _replace_dot_in_key_with_ and _replace_dollar_in_key_with_.
-
-    <match forward.*>
-      ...
-      # replace '.' in keys with '__dot__'
-      replace_dot_in_key_with __dot__
-    
-      # replace '$' in keys with '__dollar__'
-      # Note: This replaces '$' only on first character
-      replace_dollar_in_key_with __dollar__
-      ...
-    </match>
-
 == Buffer size limitation
 
 Mongo plugin has the limitation of buffer size.
@@ -263,21 +265,6 @@ So, Mongo plugin resets _buffer_chunk_limit_ if configurated value is larger tha
 - Before v1.8, max of _buffer_chunk_limit_ is 2MB
 - After  v1.8, max of _buffer_chunk_limit_ is 8MB
 
-== Disable collection check
-
-Mongo plugin checks a collection's configuration to prevent unexpected insertion to existing collection.
-In tag mapped mode, Mongo plugin accesses many collections. In this case, collection checking is not usable
-because current configuration format cannot write multiple values in one parameter.
-So, if you disable this checking, put ```disable_collection_check true``` in match.
-
-    <match forward.*>
-      ...
-
-      disable_collection_check true
-
-      ...
-    </match>
-
 = Tool
 
 You can tail mongo capped collection.
@@ -296,13 +283,6 @@ You can use 'mongod' environment variable for specified mongod:
 
 Note that source code in test/tools are from mongo-ruby-driver.
 
-= TODO
-
-== More configuration
-
-- Multi process
-- etc
-
 = Copyright
 
 Copyright:: Copyright (c) 2011- Masahiro Nakagawa

data/VERSION

@@ -1 +1 @@
-1.0.0.rc1
+1.0.0.rc2

data/fluent-plugin-mongo.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.12", "< 2"]
   gem.add_runtime_dependency "mongo", "~> 2.2.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"

data/lib/fluent/plugin/out_mongo.rb

@@ -19,8 +19,10 @@ module Fluent::Plugin
     config_set_default :include_tag_key, false
     config_set_default :include_time_key, true
 
+    desc "MongoDB connection string"
+    config_param :connection_string, :default => nil
     desc "MongoDB database"
-    config_param :database, :string
+    config_param :database, :string, :default => nil
     desc "MongoDB collection"
     config_param :collection, :string, default: 'untagged'
     desc "MongoDB host"
@@ -38,9 +40,11 @@ module Fluent::Plugin
 
     # tag mapping mode
     desc "Use tag_mapped mode"
-    config_param :tag_mapped, :bool, default: false
+    config_param :tag_mapped, :bool, default: false,
+                 deprecated: "use '${tag}' placeholder in collection parameter."
     desc "Remove tag prefix"
-    config_param :remove_tag_prefix, :string, default: nil
+    config_param :remove_tag_prefix, :string, default: nil,
+                 deprecated: "use @label instead for event routing."
 
     # SSL connection
     config_param :ssl, :bool, default: false
@@ -60,6 +64,7 @@ module Fluent::Plugin
     def initialize
       super
 
+      @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
     end
@@ -92,6 +97,10 @@ module Fluent::Plugin
 
       super
 
+      if @connection_string.nil? && @database.nil?
+        raise Fluent::ConfigError,  "connection_string or database parameter is required"
+      end
+
       unless @ignore_invalid_record
         log.warn "Since v0.8, invalid record detection will be removed because mongo driver v2.x and API spec don't provide it. You may lose invalid records, so you should not send such records to mongo plugin"
       end
@@ -124,6 +133,7 @@ module Fluent::Plugin
         @client_options[:ssl_verify] = @ssl_verify
         @client_options[:ssl_ca_cert] = @ssl_ca_cert
       end
+      @nodes = ["#{@host}:#{@port}"] if @nodes.nil?
 
       configure_logger(@mongo_log_level)
 
@@ -169,10 +179,14 @@ module Fluent::Plugin
     private
 
     def client
-      @client_options[:database] = @database
-      @client_options[:user] = @user if @user
-      @client_options[:password] = @password if @password
-      Mongo::Client.new(["#{@host}:#{@port}"], @client_options)
+      if @connection_string
+        Mongo::Client.new(@connection_string)
+      else
+        @client_options[:database] = @database
+        @client_options[:user] = @user if @user
+        @client_options[:password] = @password if @password
+        Mongo::Client.new(@nodes, @client_options)
+      end
     end
 
     def collect_records(chunk)

data/lib/fluent/plugin/out_mongo_replset.rb

@@ -7,6 +7,8 @@ module Fluent::Plugin
     config_set_default :include_tag_key, false
     config_set_default :include_time_key, true
 
+    desc "Nodes of replica set"
+    config_param :nodes, :array, value_type: :string, :default => nil
     desc "Replica set name"
     config_param :replica_set, :string
     desc "Read from specified role"

data/test/plugin/test_out_mongo.rb

@@ -52,7 +52,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
 
   def test_configure
     d = create_driver(%[
-      type mongo
+      @type mongo
       database fluent_test
       collection test_collection
 
@@ -66,6 +66,30 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal(port, d.instance.port)
     assert_equal({capped: true, size: 100}, d.instance.collection_options)
     assert_equal({ssl: false, write: {j: false}}, d.instance.client_options)
+    assert_nil d.instance.connection_string
+  end
+
+  def test_configure_with_connection_string
+    d = create_driver(%[
+      @type mongo
+      connection_string mongodb://localhost/fluent_test
+      collection test_collection
+      capped
+      capped_size 100
+    ])
+    assert_equal('mongodb://localhost/fluent_test', d.instance.connection_string)
+    assert_nil d.instance.database
+  end
+
+  def test_configure_without_connection_string_or_database
+    assert_raise Fluent::ConfigError do
+      d = create_driver(%[
+        @type mongo
+        collection test_collection
+        capped
+        capped_size 100
+      ])
+    end
   end
 
   def test_configure_with_ssl
@@ -184,11 +208,11 @@ class MongoOutputTest < ::Test::Unit::TestCase
 
     def test_write_with_collection_placeholder
       d = create_driver(%[
-      type mongo
-      database #{database_name}
-      collection ${tag}
-      include_time_key true
-    ])
+        @type mongo
+        database #{database_name}
+        collection ${tag}
+        include_time_key true
+      ])
       d.run(default_tag: @tag) do
         emit_documents(d)
       end

data/test/plugin/test_out_mongo_replset.rb

@@ -21,14 +21,18 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
     'fluent_test'
   end
 
+  def nodes
+    ["localhost:#{port}"]
+  end
+
   def port
     27018
   end
 
   def default_config
     %[
-      type mongo
-      port 27018
+      @type mongo_replset
+      nodes localhost:27018
       database #{database_name}
       collection #{collection_name}
       include_time_key true
@@ -42,18 +46,33 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
 
   def test_configure
     d = create_driver(%[
-      type mongo
+      @type mongo_replset
       port 27018
       database fluent_test
       collection test_collection
-
       replica_set rs0
     ])
 
     assert_equal('fluent_test', d.instance.database)
     assert_equal('test_collection', d.instance.collection)
     assert_equal('localhost', d.instance.host)
-    assert_equal(port, d.instance.port)
+    assert_equal(27018, d.instance.port)
+    assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
+                 d.instance.client_options)
+  end
+
+  def test_configure_with_nodes
+    d = create_driver(%[
+      @type mongo_replset
+      nodes localhost:27018,localhost:27019
+      database fluent_test
+      collection test_collection
+      replica_set rs0
+    ])
+
+    assert_equal('fluent_test', d.instance.database)
+    assert_equal('test_collection', d.instance.collection)
+    assert_equal(['localhost:27018', 'localhost:27019'], d.instance.nodes)
     assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
                  d.instance.client_options)
   end
@@ -83,7 +102,7 @@ class MongoReplsetOutputTest < ::Test::Unit::TestCase
     def setup_mongod
       options = {}
       options[:database] = database_name
-      @client = ::Mongo::Client.new(["localhost:#{port}"], options)
+      @client = ::Mongo::Client.new(nodes, options)
     end
 
     def teardown_mongod

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc1
+  version: 1.0.0.rc2
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-09 00:00:00.000000000 Z
+date: 2017-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.12
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.14.12
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'