fluent-plugin-mongo 1.2.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 892fbaf245b427435db3ebb4edaa6e4681d5fd07
-  data.tar.gz: de0b7e9b1b054e02819f5242888361bb4ebbf8a5
+SHA256:
+  metadata.gz: 6208365e5862e84db04e62061f822324bc6ce1f82ebb8b9b0c3cb6ed230d983e
+  data.tar.gz: 2c45c015dd0fc2cf98e3b309ec4d96865762ed6bf6e534d316f5c6b214c55fcb
 SHA512:
-  metadata.gz: 9193319af1591a3b21b5ceeb1c98155434f41cb01ad5125fe6bb3dfaea30bd40d4fbea868421f38f85fccb30f8eea1480f912c84ddeea3f2415b3b2f6520e45a
-  data.tar.gz: c486200f457c322e6c0350dfba0191776dac3909b85cf5e46547c705680ccae5c9b19be2857e7076c00f195e7d93528698bebd8aa7e76d81f1fa27d78ef536ca
+  metadata.gz: 046b7463bad7a1be9aae5f2a20991656392e32a4693de753aa0c5010f269d8708e0a2c085b5809408b17c5114aeff534e5cafe5b2527453a07f6707227bcbd15
+  data.tar.gz: 42d7e9a95afeb58709422a13e0e90a911a6e32b8d972e26982496b5e90c106db83eb7d5b06d6c775a4f16a92569f10f2e82764a26d20c1c061408ad2bee89cb9

data/.travis.yml

@@ -1,9 +1,8 @@
 rvm:
- - 2.1
- - 2.2.4
- - 2.3.1
- - 2.4.3
- - 2.5.0
+ - 2.4.10
+ - 2.5
+ - 2.6
+ - 2.7
  - ruby-head
 
 gemfile:
@@ -12,11 +11,6 @@ gemfile:
 services:
   - mongodb
 
-before_install:
-  - gem update --system
-before_script:
-  - git submodule update -i
-
 script: bundle exec rake test
 
 matrix:

data/ChangeLog

@@ -1,3 +1,24 @@
+Release 1.5.0 - 2020/11/26
+
+	* out_mongo: Support nested fields in date_keys
+
+Release 1.4.1 - 2020/08/21
+
+	* out_mongo: Add expire_after parameter
+
+Release 1.4.0 - 2020/04/28
+
+	* out_mongo: Add date_keys parameter to support MongoDB Date object in record fields.
+
+Release 1.3.0 - 2019/04/24
+
+	* out_mongo: Support auth_mech parameter to allow other authentication
+	* in_mongo_tail: Fix replicat set issue
+
+Release 1.2.2 - 2019/04/01
+
+	* out_mongo: Don't handle database placholders when specifying connection_string parameter
+
 Release 1.2.1 - 2018/12/18
 
 	* out_mongo_replset: Fix internal signature mismatch

data/README.rdoc

@@ -53,6 +53,12 @@ Use _mongo_ type in match.
       capped
       capped_size 100m
 
+      # Specify date fields in record to use MongoDB's Date object (Optional) default: nil
+      # Supported data types are String/Integer/Float/Fuentd EventTime.
+      # For Integer type, milliseconds epoch and seconds epoch are supported.
+      # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
+      date_keys updated_at
+
       # Other buffer configurations here
     </match>
 
@@ -223,7 +229,7 @@ BSON records which include '.' or start with '$' are invalid and they will be st
       ...
       # replace '.' in keys with '__dot__'
       replace_dot_in_key_with __dot__
-    
+
       # replace '$' in keys with '__dollar__'
       # Note: This replaces '$' only on first character
       replace_dollar_in_key_with __dollar__

data/VERSION

@@ -1 +1 @@
-1.2.1
+1.5.0

data/fluent-plugin-mongo.gemspec

@@ -10,14 +10,13 @@ Gem::Specification.new do |gem|
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Masahiro Nakagawa"]
   gem.email       = "repeatedly@gmail.com"
-  gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.14.12", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
   gem.add_runtime_dependency "mongo", "~> 2.6.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -118,10 +118,10 @@ module Fluent::Plugin
     private
 
     def client
-      @client_options[:database] = @database
+      @client_options[:database] = @database if @database
       @client_options[:user] = @user if @user
       @client_options[:password] = @password if @password
-      Mongo::Client.new(["#{node_string}"], @client_options)
+      Mongo::Client.new(node_string, @client_options)
     end
 
     def get_collection
@@ -133,7 +133,7 @@ module Fluent::Plugin
     def node_string
       case
       when @database
-        "#{@host}:#{@port}"
+        ["#{@host}:#{@port}"]
       when @url
         @url
       end

data/lib/fluent/plugin/mongo_auth.rb

@@ -8,23 +8,25 @@ module Fluent
         config_param :password, :string, default: nil, secret: true
         desc "MongoDB authentication database"
         config_param :auth_source, :string, default: nil
+        desc "MongoDB authentication mechanism"
+        config_param :auth_mech, :string, default: nil
       }
     end
   end
 
   module MongoAuth
     def authenticate(client)
-      unless @user.nil? || @password.nil?
-        begin
-          if @auth_source.nil?
-            client = client.with(user: @user, password: @password)
-          else
-            client = client.with(user: @user, password: @password, auth_source: @auth_source)
-          end
-        rescue Mongo::Auth::Unauthorized => e
-          log.fatal e
-          exit!
+      begin
+        if [@user, @password, @auth_source].all?
+          client = client.with(user: @user, password: @password, auth_source: @auth_source)
+        elsif [@user, @password].all?
+          client = client.with(user: @user, password: @password)
+        elsif [@user, @auth_source, @auth_mech].all?
+          client = client.with(user: @user, auth_source: @auth_source, auth_mech: @auth_mech.to_sym)
         end
+      rescue Mongo::Auth::Unauthorized => e
+        log.fatal e
+        exit!
       end
       client
     end

data/lib/fluent/plugin/out_mongo.rb

@@ -8,7 +8,7 @@ module Fluent::Plugin
   class MongoOutput < Output
     Fluent::Plugin.register_output('mongo', self)
 
-    helpers :event_emitter, :inject, :compat_parameters
+    helpers :event_emitter, :inject, :compat_parameters, :record_accessor
 
     include Fluent::MongoAuthParams
     include Fluent::MongoAuth
@@ -38,6 +38,13 @@ module Fluent::Plugin
     desc "Replace dollar with specified string"
     config_param :replace_dollar_in_key_with, :string, default: nil
 
+    # Additional date field to be used to Date object
+    desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
+    config_param :date_keys, :array, default: nil
+    desc "Specify if the fields in date_keys are of type Integer or Float"
+    config_param :parse_string_number_date, :bool, default: false
+
+
     # tag mapping mode
     desc "Use tag_mapped mode"
     config_param :tag_mapped, :bool, default: false,
@@ -45,6 +52,9 @@ module Fluent::Plugin
     desc "Remove tag prefix"
     config_param :remove_tag_prefix, :string, default: nil,
                  deprecated: "use @label instead for event routing."
+    # expire indexes
+    desc "Specify expire after seconds"
+    config_param :expire_after, :time, default: 0
 
     # SSL connection
     config_param :ssl, :bool, default: false
@@ -54,6 +64,7 @@ module Fluent::Plugin
     config_param :ssl_verify, :bool, default: false
     config_param :ssl_ca_cert, :string, default: nil
 
+
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
       config_set_default :chunk_keys, ['tag']
@@ -67,6 +78,7 @@ module Fluent::Plugin
       @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
+      @accessors = {}
     end
 
     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -108,12 +120,12 @@ module Fluent::Plugin
 
       super
 
-      if @connection_string.nil? && @database.nil?
-        raise Fluent::ConfigError,  "connection_string or database parameter is required"
+      if @auth_mech && !Mongo::Auth::SOURCES.has_key?(@auth_mech.to_sym)
+        raise Fluent::ConfigError, Mongo::Auth::InvalidMechanism.new(@auth_mech.to_sym)
       end
 
-      unless @ignore_invalid_record
-        log.warn "Since v0.8, invalid record detection will be removed because mongo driver v2.x and API spec don't provide it. You may lose invalid records, so you should not send such records to mongo plugin"
+      if @connection_string.nil? && @database.nil?
+        raise Fluent::ConfigError,  "connection_string or database parameter is required"
       end
 
       if conf.has_key?('tag_mapped')
@@ -149,6 +161,13 @@ module Fluent::Plugin
       configure_logger(@mongo_log_level)
 
       log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
+
+      if @date_keys
+        @date_keys.each { |field_name|
+          @accessors[field_name.to_s] = record_accessor_create(field_name)
+        }
+        log.debug "Setup record accessor for every date key"
+      end
     end
 
     def start
@@ -172,8 +191,9 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      collection_name = extract_placeholders(@collection, chunk.metadata)
-      database_name = extract_placeholders(@database, chunk.metadata)
+      collection_name = extract_placeholders(@collection, chunk)
+      # In connection_string case, we shouldn't handle extract_placeholers for @database.
+      database_name = extract_placeholders(@database, chunk) unless @connection_string
       operate(database_name, format_collection_name(collection_name), collect_records(chunk))
     end
 
@@ -193,11 +213,58 @@ module Fluent::Plugin
     def collect_records(chunk)
       records = []
       time_key = @inject_config.time_key if @inject_config
+      date_keys = @date_keys
+
       tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
         record = inject_values_to_record(tag, time, record)
         # MongoDB uses BSON's Date for time.
         record[time_key] = Time.at(time || record[time_key]) if time_key
+
+        if date_keys
+          @accessors.each_pair { |date_key, date_key_accessor|
+            begin
+              date_value = date_key_accessor.call(record)
+              case date_value
+              when Fluent::EventTime
+                value_to_set = date_value.to_time
+              when Integer
+                value_to_set = if date_value > 9999999999
+                                 # epoch with milliseconds: e.g. javascript
+                                 Time.at(date_value / 1000.0)
+                               else
+                                 # epoch with seconds: e.g. ruby
+                                 Time.at(date_value)
+                               end
+              when Float
+                value_to_set = Time.at(date_value)
+              else
+                if @parse_string_number_date
+                  if date_value.to_i.to_s == date_value
+                    date_value = date_value.to_i
+                    value_to_set = if date_value > 9999999999
+                                     # epoch with milliseconds: e.g. javascript
+                                     date_value / 1000.0
+                                   else
+                                     # epoch with seconds: e.g. ruby
+                                     date_value
+                                   end
+                  elsif date_value.to_f.to_s == date_value
+                    date_value = date_value.to_f
+                  end
+                  value_to_set = date_value.is_a?(String) ? Time.parse(date_value) : Time.at(date_value)
+                else
+                  value_to_set = Time.parse(date_value)
+                end
+              end
+
+              date_key_accessor.set(record, value_to_set)
+            rescue ArgumentError
+              log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{date_value}"
+              date_key_accessor.set(record, nil)
+            end
+          }
+        end
         records << record
       }
       records
@@ -235,6 +302,13 @@ module Fluent::Plugin
       unless collection_exists?(name)
         log.trace "Create collection #{name} with options #{options}"
         @client[name, options].create
+        if @expire_after > 0 && @inject_config
+          log.trace "Create expiring index with key: \"#{@inject_config.time_key}\" and seconds: \"#{@expire_after}\""
+          @client[name].indexes.create_one(
+            {"#{@inject_config.time_key}": 1},
+            expire_after: @expire_after
+          )
+        end
       end
       @collections[name] = true
       @client[name]

data/test/plugin/test_in_mongo_tail.rb

@@ -100,7 +100,7 @@ class MongoTailInputTest < Test::Unit::TestCase
       end
       events = d.events
       assert_equal "input.mongo", events[0][0]
-      assert_equal event_time(@time.to_s), events[0][1]
+      assert events[0][1].is_a?(Fluent::EventTime)
       assert_equal "test", events[0][2]["message"]
     end
 
@@ -114,7 +114,7 @@ class MongoTailInputTest < Test::Unit::TestCase
         time_key time
       ])
       d.run(expect_records: 1, timeout: 5) do
-        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Fluent::Engine.now})
+        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Time.at(Fluent::Engine.now)})
       end
       events = d.events
       assert_equal "user.defined", events[0][0]
@@ -138,7 +138,7 @@ class MongoTailInputTest < Test::Unit::TestCase
       events = d.events
       assert_equal 1, events.size
       assert_equal "input.mongo.last_id", events[0][0]
-      assert_equal event_time(@time.to_s), events[0][1]
+      assert events[0][1].is_a?(Fluent::EventTime)
       assert_equal "can obtain", events[0][2]["message"]
     end
   end

data/test/plugin/test_out_mongo.rb

@@ -92,6 +92,22 @@ class MongoOutputTest < ::Test::Unit::TestCase
     end
   end
 
+  def test_configure_auth_mechanism
+    Mongo::Auth::SOURCES.each do |key, value|
+      conf = default_config + %[
+        auth_mech #{key}
+      ]
+      d = create_driver(conf)
+      assert_equal(key.to_s, d.instance.auth_mech)
+    end
+    assert_raise Fluent::ConfigError do
+      conf = default_config + %[
+        auth_mech invalid
+      ]
+      d = create_driver(conf)
+    end
+  end
+
   def test_configure_with_ssl
     conf = default_config + %[
       ssl true
@@ -164,6 +180,10 @@ class MongoOutputTest < ::Test::Unit::TestCase
     @client[collection].find.to_a.map {|e| e.delete('_id'); e}
   end
 
+  def get_indexes(collection = collection_name)
+    @client[collection].indexes
+  end
+
   def emit_documents(d)
     time = event_time("2011-01-02 13:14:15 UTC")
     d.feed(time, {'a' => 1})
@@ -183,6 +203,52 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal(expected, actual_documents)
   end
 
+  def test_write_with_connection_string
+    d = create_driver(%[
+      @type mongo
+      connection_string mongodb://localhost:#{port}/#{database_name}
+      collection #{collection_name}
+      capped
+      capped_size 100
+    ])
+    assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
+    assert_nil d.instance.database
+
+    d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
+    actual_documents = get_documents
+    time = event_time("2011-01-02 13:14:15 UTC")
+    expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
+    assert_equal(expected, actual_documents)
+  end
+
+  def test_write_with_expire_index
+    d = create_driver(%[
+      @type mongo
+      connection_string mongodb://localhost:#{port}/#{database_name}
+      collection #{collection_name}
+      capped
+      capped_size 100
+      expire_after 120
+    ])
+    assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
+    assert_nil d.instance.database
+        d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
+    actual_documents = get_documents
+    time = event_time("2011-01-02 13:14:15 UTC")
+    expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
+    assert_equal(expected, actual_documents)
+
+    indexes = get_indexes()
+    expire_after_hash = indexes.map {|e| e.select{|k, v| k == "expireAfterSeconds"} }.reject{|e| e.empty?}.first
+    assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
+  end
+
   class WriteWithCollectionPlaceholder < self
     def setup
       @tag = 'custom'
@@ -278,7 +344,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
             "$foo$bar" => "baz"
           }
         ],
-             })
+      })
     end
 
     documents = get_documents
@@ -351,4 +417,104 @@ class MongoOutputTest < ::Test::Unit::TestCase
       assert authenticate(@client)
     end
   end
+
+  sub_test_case 'date_keys' do
+    setup do
+      @updated_at_str = "2020-02-01T08:22:23.780Z"
+      @updated_at_t = Time.parse(@updated_at_str)
+    end
+
+    def emit_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: @updated_at_str})
+      d.feed(time, {'a' => 2, updated_at: @updated_at_t.to_f})
+      d.feed(time, {'a' => 3, updated_at: @updated_at_t.to_i})
+      time
+    end
+
+    def emit_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: "Invalid Date String"})
+      time
+    end
+
+    def emit_nested_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: { 'time': @updated_at_str}})
+      d.feed(time, {'a' => 2, updated_at: { 'time': @updated_at_t.to_f}})
+      d.feed(time, {'a' => 3, updated_at: { 'time': @updated_at_t.to_i}})
+      time
+    end
+
+    def emit_nested_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, 'updated_at': { 'time': "Invalid Date String"}})
+      time
+    end
+
+    def test_write_with_date_keys
+      d = create_driver(default_config + %[
+        date_keys updated_at
+        time_key created_at
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      date_key = d.instance.date_keys.first
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc[date_key].is_a?(Time)
+      }
+    end
+
+    def test_write_with_parsed_date_key_invalid_string
+      d = create_driver(default_config + %[
+            date_keys updated_at
+            time_key created_at
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['updated_at']
+    end
+
+    def test_write_with_date_nested_keys
+      d = create_driver(default_config + %[
+        replace_dot_in_key_with _
+        replace_dollar_in_key_with _
+        date_keys $.updated_at.time
+        time_key created_at
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc['updated_at']['time'].is_a?(Time)
+      }
+    end
+
+    def test_write_with_parsed_date_nested_key_invalid_string
+      d = create_driver(default_config + %[
+            replace_dot_in_key_with _
+            replace_dollar_in_key_with _
+            date_keys $.updated_at.time
+            time_key created_at
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['updated_at']['time']
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2020-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.12
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.12
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -159,8 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: MongoDB plugin for Fluentd