fluent-plugin-mongo 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 704336957f5f5ebdec8351e961f38555db15ce16
-  data.tar.gz: f8dc945869ddf4c3e8125b517757371b86067c3d
+  metadata.gz: b4c03c163ad52a2f48b8fa88d69370d4c1ed0810
+  data.tar.gz: cf0f26a570dcfaed4c24bde308446b36ba0c2cfd
 SHA512:
-  metadata.gz: aad06123bf520927bba1e718efaf297f379c98419ce7dc6d1068d2da9a0b86b40d291b017b2d0dfd1b1cb9fc72e766d7f82884e5f7248790fbce47197612b703
-  data.tar.gz: 9b18985959ec858e77680ca8bf21037cfb2b62461f427889c03d5166e156617492d8ceda492a1d2598e44f18c30e14d745708c1136d6fe0260808e90ca300959
+  metadata.gz: 871f0d9f4880db300354aff08ba51f1a9f07718e01e9b3527ee22c5be107bda3f4ce9e96d7d5ed431a7ed704eb90fe421ba178a357216a1e9070630ff33847dd
+  data.tar.gz: 182460bff1536f1997c238537292cbb0a048af93ef74a2393e275b79b0ae969707336d4c3243cdfe50a8f717b3929ac8ab1cffd4b91a0e01c51499985dc035ee

data/.travis.yml

@@ -1,9 +1,15 @@
 rvm:
  - 1.9.2
  - 1.9.3
- - ruby-head
+ - 2.0.0
+ - 2.1.0
+ - rbx
 
 before_script:
   - git submodule update -i
 
-script: bundle exec rake test
+script: mongod=mongod bundle exec rake test
+
+matrix:
+  allow_failures:
+    - rvm: rbx

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.7.3 - 2014/03/09
+
+	* Add replace_dot_in_key_with and replace_dollar_in_key_with parameters to sanitize invalid key
+	* Add ssl parameter to enable SSL connection
+	* Relax gem version dependency
+
+
 Release 0.7.2 - 2014/02/05
 
 	* Support log_level option

data/README.rdoc

@@ -1,6 +1,6 @@
-= MongoDB plugin for Fluentd
+= MongoDB plugin for {Fluentd}[http://github.com/fluent/fluentd]
 
-fluent-plugin-mongo provides input and output plugins for {Fluentd}[http://fluentd.org/].
+fluent-plugin-mongo provides input and output plugins for {Fluentd}[http://fluentd.org] ({GitHub}[http://github.com/fluent/fluentd])
 
 = Installation
 
@@ -199,6 +199,21 @@ If you want to exclude some fields from broken data marshaling, use _exclude_bro
 Specified value is a comma separated keys(e.g. key1,key2,key3).
 This parameter is useful for excluding shard keys in shard environment.
 
+=== replace_dot_in_key_with and replace_dollar_in_key_with
+
+BSON records which include '.' or start with '$' are invalid and they will be stored as broken data to MongoDB. If you want to sanitize keys, you can use _replace_dot_in_key_with_ and _replace_dollar_in_key_with_.
+
+    <match forward.*>
+      ...
+      # replace '.' in keys with '__dot__'
+      replace_dot_in_key_with __dot__
+    
+      # replace '$' in keys with '__dollar__'
+      # Note: This replaces '$' only on first character
+      replace_dollar_in_key_with __dollar__
+      ...
+    </match>
+
 == Buffer size limitation
 
 Mongo plugin has the limitation of buffer size.

data/VERSION

@@ -1 +1 @@
-0.7.2
+0.7.3

data/fluent-plugin-mongo.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-mongo"
-  gem.description = "MongoDB plugin for Fluent event collector"
+  gem.description = "MongoDB plugin for Fluentd"
   gem.homepage    = "https://github.com/fluent/fluent-plugin-mongo"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", "~> 0.10.9"
-  gem.add_dependency "mongo", "~> 1.8.0"
+  gem.add_dependency "mongo", ">= 1.8.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -19,6 +19,9 @@ module Fluent
     # To store last ObjectID
     config_param :id_store_file, :string, :default => nil
 
+    # SSL connection
+    config_param :ssl, :bool, :default => false
+
     unless method_defined?(:log)
       define_method(:log) { $log }
     end
@@ -27,6 +30,8 @@ module Fluent
       super
       require 'mongo'
       require 'bson'
+
+      @connection_options = {}
     end
 
     def configure(conf)
@@ -37,6 +42,7 @@ module Fluent
       end
 
       @last_id = @id_store_file ? get_last_id : nil
+      @connection_options[:ssl] = @ssl
 
       $log.debug "Setup mongo_tail configuration: mode = #{@id_store_file ? 'persistent' : 'non-persistent'}"
     end
@@ -69,7 +75,7 @@ module Fluent
 
     def get_capped_collection
       begin
-        db = authenticate(Mongo::Connection.new(@host, @port).db(@database))
+        db = authenticate(Mongo::Connection.new(@host, @port, @connection_options).db(@database))
         raise ConfigError, "'#{@database}.#{@collection}' not found: node = #{@host}:#{@port}" unless db.collection_names.include?(@collection)
         collection = db.collection(@collection)
         raise ConfigError, "'#{@database}.#{@collection}' is not capped: node = #{@host}:#{@port}" unless collection.capped?

data/lib/fluent/plugin/out_mongo.rb

@@ -19,11 +19,16 @@ module Fluent
     config_param :disable_collection_check, :bool, :default => nil
     config_param :exclude_broken_fields, :string, :default => nil
     config_param :write_concern, :integer, :default => nil
+    config_param :replace_dot_in_key_with, :string, :default => nil
+    config_param :replace_dollar_in_key_with, :string, :default => nil
 
     # tag mapping mode
     config_param :tag_mapped, :bool, :default => false
     config_param :remove_tag_prefix, :string, :default => nil
 
+    # SSL connection
+    config_param :ssl, :bool, :default => false
+
     attr_reader :collection_options, :connection_options
 
     unless method_defined?(:log)
@@ -65,6 +70,7 @@ module Fluent
       end
 
       @connection_options[:w] = @write_concern unless @write_concern.nil?
+      @connection_options[:ssl] = @ssl
 
       # MongoDB uses BSON's Date for time.
       def @timef.format_nocache(time)
@@ -116,6 +122,17 @@ module Fluent
 
     def operate(collection, records)
       begin
+        if @replace_dot_in_key_with
+          records.map! do |r|
+            replace_key_of_hash(r, ".", @replace_dot_in_key_with)
+          end
+        end
+        if @replace_dollar_in_key_with
+          records.map! do |r|
+            replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
+          end
+        end
+
         record_ids, error_records = collection.insert(records, INSERT_ARGUMENT)
         if !@ignore_invalid_record and error_records.size > 0
           operate_invalid_records(collection, error_records)
@@ -235,5 +252,19 @@ module Fluent
 
       version
     end
+
+    def replace_key_of_hash(hash, pattern, replacement)
+      result = Hash.new
+      hash.each_pair do |k, v|
+        k = k.gsub(pattern, replacement)
+
+        if v.is_a?(Hash)
+          result[k] = replace_key_of_hash(v, pattern, replacement)
+        else
+          result[k] = (v.dup rescue v)
+        end
+      end
+      result
+    end
   end
 end

data/test/plugin/out_mongo.rb

@@ -51,7 +51,7 @@ class MongoOutputTest < Test::Unit::TestCase
     assert_equal('localhost', d.instance.host)
     assert_equal(@@mongod_port, d.instance.port)
     assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
-    assert(d.instance.connection_options.empty?)
+    assert_equal({:ssl => false}, d.instance.connection_options)
     # buffer_chunk_limit moved from configure to start
     # I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
     # assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
@@ -62,7 +62,15 @@ class MongoOutputTest < Test::Unit::TestCase
       write_concern 2
     ])
 
-    assert_equal({:w => 2}, d.instance.connection_options)
+    assert_equal({:w => 2, :ssl => false}, d.instance.connection_options)
+  end
+
+  def test_configure_with_ssl
+    d = create_driver(default_config + %[
+      ssl true
+    ])
+
+    assert_equal({:ssl => true}, d.instance.connection_options)
   end
 
   def test_format
@@ -120,6 +128,26 @@ class MongoOutputTest < Test::Unit::TestCase
     time
   end
 
+  def test_write_with_invalid_recoreds_with_keys_containing_dot_and_dollar
+    d = create_driver(default_config + %[
+      replace_dot_in_key_with _dot_
+      replace_dollar_in_key_with _dollar_
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({
+      "foo.bar" => {
+        "$foo$bar" => "baz"
+      }
+    }, time)
+    d.run
+
+    documents = get_documents
+    assert_equal(1, documents.size)
+    assert_equal("baz", documents[0]["foo_dot_bar"]["_dollar_foo$bar"])
+    assert_equal(0, documents.select { |e| e.has_key?(Fluent::MongoOutput::BROKEN_DATA_KEY)}.size)
+  end
+
   def test_write_with_invalid_recoreds
     d = create_driver
     t = emit_documents(d)
@@ -229,5 +257,6 @@ class MongoReplOutputTest < MongoOutputTest
     assert_equal(build_seeds(3), d.instance.nodes)
     assert_equal(45, d.instance.num_retries)
     assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
+    assert_equal({:ssl => false}, d.instance.connection_options)
   end
 end

metadata

@@ -1,95 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.9
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
-description: MongoDB plugin for Fluent event collector
+description: MongoDB plugin for Fluentd
 email: repeatedly@gmail.com
 executables:
 - mongo-tail
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .gitmodules
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -119,20 +118,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
-summary: MongoDB plugin for Fluent event collector
+summary: MongoDB plugin for Fluentd
 test_files:
 - test/plugin/in_mongo_tail.rb
 - test/plugin/out_mongo.rb

data/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "vendor/fluentd"]
-	path = vendor/fluentd
-	url = git://github.com/fluent/fluentd.git