fluent-plugin-mongo 1.4.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 709f3068ce7f97bcad4737b3c7a9db60bdea998b184949afa7f3fa1794831474
-  data.tar.gz: 53d8b989b2d30d8b0df0e37701af671713460dd46fe920435016ececcaef6c36
+  metadata.gz: fc300a11be780d8b3ea62e71db08393171fad3625d1d9259cb65a7abe11f9e5e
+  data.tar.gz: db12f19c83ae360e9720c41a2d64c3ce8bd65487d554ed2e416277d0f158b311
 SHA512:
-  metadata.gz: 6ce121ba68754c2d6687a7d1ed8cedf8c9fe94652f85476b03aacddf92bede132aa7f45444c8cb4153ff5509143bf00688cffc5c872c15e76c754e0cd6f5ff8f
-  data.tar.gz: 4391ce8e22ca8173f38630a6f22024acdccae01d48a3bf5c86f255bbdbe057cdbbb9039b9f55b209bfaf377188124a5b1dd83fcaf8b58db80785e16348e8caa0
+  metadata.gz: c105cf4d712af0524a554b072f86a2af2cfc5485825db24f47dcd47cc850628cfeb08af8f7db4337aefc7342b2325e35698707db22aa0e595d786a6d72317320
+  data.tar.gz: '08454ce8638ee2bf64d1f6ae2d6adfd0632ed43ad47667558b9ee00dccb54a12d8fbac31c0b8032bd69598752a442bac02e7425e2ccd649754d7c48df1723b8b'

data/.github/workflows/linux.yml

@@ -0,0 +1,49 @@
+name: Test
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.1', '3.0', '2.7', '2.6' ]
+        mongodb-version: ['5.0', '4.4', '4.2', '4.0']
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.0'
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.2'
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.4'
+
+    name: Ruby ${{ matrix.ruby }} and MongoDB ${{ matrix.mongodb-version }} on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Start MongoDB
+      uses: supercharge/mongodb-github-action@1.3.0
+      with:
+        mongodb-version: ${{ matrix.mongodb-version }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 1.6.0 - 2022/07/15
+
+	* Update mongo gem dependency
+	* out_mongo: Add object_id_keys parameter to convert string to Mongo ObjectId
+
+Release 1.5.0 - 2020/11/26
+
+	* out_mongo: Support nested fields in date_keys
+
+Release 1.4.1 - 2020/08/21
+
+	* out_mongo: Add expire_after parameter
+
 Release 1.4.0 - 2020/04/28
 
 	* out_mongo: Add date_keys parameter to support MongoDB Date object in record fields.

data/README.rdoc

@@ -59,6 +59,10 @@ Use _mongo_ type in match.
       # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
       date_keys updated_at
 
+      # Specify id fields in record to use MongoDB's BSON ObjectID (Optional) default: nil
+      # eg: my_id: "507f1f77bcf86cd799439011"
+      object_id_keys my_id
+
       # Other buffer configurations here
     </match>
 

data/VERSION

@@ -1 +1 @@
-1.4.0
+1.6.0

data/fluent-plugin-mongo.gemspec

@@ -17,10 +17,11 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
-  gem.add_runtime_dependency "mongo", "~> 2.6.0"
+  gem.add_runtime_dependency "mongo", ">= 2.15.0", "< 2.19.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"
   gem.add_development_dependency "test-unit", ">= 3.0.0"
-  gem.add_development_dependency "timecop", "~> 0.8.0"
+  gem.add_development_dependency "timecop", "~> 0.9.4"
+  gem.add_development_dependency "webrick", ">= 1.7.0"
 end

data/lib/fluent/plugin/out_mongo.rb

@@ -8,7 +8,7 @@ module Fluent::Plugin
   class MongoOutput < Output
     Fluent::Plugin.register_output('mongo', self)
 
-    helpers :event_emitter, :inject, :compat_parameters
+    helpers :event_emitter, :inject, :compat_parameters, :record_accessor
 
     include Fluent::MongoAuthParams
     include Fluent::MongoAuth
@@ -41,6 +41,10 @@ module Fluent::Plugin
     # Additional date field to be used to Date object
     desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
     config_param :date_keys, :array, default: nil
+    desc "Specify if the fields in date_keys are of type Integer or Float"
+    config_param :parse_string_number_date, :bool, default: false
+    desc "Specify keys to use MongoDB's ObjectId"
+    config_param :object_id_keys, :array, default: nil
 
     # tag mapping mode
     desc "Use tag_mapped mode"
@@ -49,6 +53,9 @@ module Fluent::Plugin
     desc "Remove tag prefix"
     config_param :remove_tag_prefix, :string, default: nil,
                  deprecated: "use @label instead for event routing."
+    # expire indexes
+    desc "Specify expire after seconds"
+    config_param :expire_after, :time, default: 0
 
     # SSL connection
     config_param :ssl, :bool, default: false
@@ -72,6 +79,8 @@ module Fluent::Plugin
       @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
+      @date_accessors = {}
+      @object_id_accessors = {}
     end
 
     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -121,10 +130,6 @@ module Fluent::Plugin
         raise Fluent::ConfigError,  "connection_string or database parameter is required"
       end
 
-      unless @ignore_invalid_record
-        log.warn "Since v0.8, invalid record detection will be removed because mongo driver v2.x and API spec don't provide it. You may lose invalid records, so you should not send such records to mongo plugin"
-      end
-
       if conf.has_key?('tag_mapped')
         log.warn "'tag_mapped' feature is replaced with built-in config placeholder. Please consider to use 'collection ${tag}'."
         @collection = '${tag}'
@@ -158,6 +163,19 @@ module Fluent::Plugin
       configure_logger(@mongo_log_level)
 
       log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
+
+      if @date_keys
+        @date_keys.each { |field_name|
+          @date_accessors[field_name.to_s] = record_accessor_create(field_name)
+        }
+        log.debug "Setup record accessor for every date key"
+      end
+      if @object_id_keys
+        @object_id_keys.each { |field_name|
+          @object_id_accessors[field_name.to_s] = record_accessor_create(field_name)
+        }
+        log.debug "Setup record accessor for every object_id key"
+      end
     end
 
     def start
@@ -204,6 +222,7 @@ module Fluent::Plugin
       records = []
       time_key = @inject_config.time_key if @inject_config
       date_keys = @date_keys
+      object_id_keys = @object_id_keys
 
       tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
@@ -212,28 +231,58 @@ module Fluent::Plugin
         record[time_key] = Time.at(time || record[time_key]) if time_key
 
         if date_keys
-          date_keys.each { |date_key|
+          @date_accessors.each_pair { |date_key, date_key_accessor|
             begin
-              date_value = record[date_key]
+              date_value = date_key_accessor.call(record)
               case date_value
               when Fluent::EventTime
-                record[date_key] = date_value.to_time
+                value_to_set = date_value.to_time
               when Integer
-                record[date_key] = if date_value > 9999999999
+                value_to_set = if date_value > 9999999999
+                                 # epoch with milliseconds: e.g. javascript
+                                 Time.at(date_value / 1000.0)
+                               else
+                                 # epoch with seconds: e.g. ruby
+                                 Time.at(date_value)
+                               end
+              when Float
+                value_to_set = Time.at(date_value)
+              else
+                if @parse_string_number_date
+                  if date_value.to_i.to_s == date_value
+                    date_value = date_value.to_i
+                    value_to_set = if date_value > 9999999999
                                      # epoch with milliseconds: e.g. javascript
-                                     Time.at(date_value / 1000.0)
+                                     date_value / 1000.0
                                    else
                                      # epoch with seconds: e.g. ruby
-                                     Time.at(date_value)
+                                     date_value
                                    end
-              when Float
-                record[date_key] = Time.at(date_value)
-              else
-                record[date_key] = Time.parse(date_value)
+                  elsif date_value.to_f.to_s == date_value
+                    date_value = date_value.to_f
+                  end
+                  value_to_set = date_value.is_a?(String) ? Time.parse(date_value) : Time.at(date_value)
+                else
+                  value_to_set = Time.parse(date_value)
+                end
               end
+
+              date_key_accessor.set(record, value_to_set)
             rescue ArgumentError
-              log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{record[date_key]}"
-              record[date_key] = nil
+              log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{date_value}"
+              date_key_accessor.set(record, nil)
+            end
+          }
+        end
+        if object_id_keys
+          @object_id_accessors.each_pair { |object_id_key, object_id_key_accessor|
+            begin
+              object_id_value = object_id_key_accessor.call(record)
+              value_to_set = BSON::ObjectId(object_id_value)
+              object_id_key_accessor.set(record, value_to_set)
+            rescue BSON::ObjectId::Invalid
+              log.warn "Failed to parse '#{object_id_key}' field. Expected object_id types are String: #{object_id_value}"
+              object_id_key_accessor.set(record, nil)
             end
           }
         end
@@ -274,6 +323,13 @@ module Fluent::Plugin
       unless collection_exists?(name)
         log.trace "Create collection #{name} with options #{options}"
         @client[name, options].create
+        if @expire_after > 0 && @inject_config
+          log.trace "Create expiring index with key: \"#{@inject_config.time_key}\" and seconds: \"#{@expire_after}\""
+          @client[name].indexes.create_one(
+            {"#{@inject_config.time_key}": 1},
+            expire_after: @expire_after
+          )
+        end
       end
       @collections[name] = true
       @client[name]

data/test/plugin/test_in_mongo_tail.rb

@@ -114,7 +114,7 @@ class MongoTailInputTest < Test::Unit::TestCase
         time_key time
       ])
       d.run(expect_records: 1, timeout: 5) do
-        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Fluent::Engine.now})
+        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Time.at(Fluent::Engine.now)})
       end
       events = d.events
       assert_equal "user.defined", events[0][0]

data/test/plugin/test_out_mongo.rb

@@ -180,6 +180,10 @@ class MongoOutputTest < ::Test::Unit::TestCase
     @client[collection].find.to_a.map {|e| e.delete('_id'); e}
   end
 
+  def get_indexes(collection = collection_name)
+    @client[collection].indexes
+  end
+
   def emit_documents(d)
     time = event_time("2011-01-02 13:14:15 UTC")
     d.feed(time, {'a' => 1})
@@ -220,6 +224,31 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal(expected, actual_documents)
   end
 
+  def test_write_with_expire_index
+    d = create_driver(%[
+      @type mongo
+      connection_string mongodb://localhost:#{port}/#{database_name}
+      collection #{collection_name}
+      capped
+      capped_size 100
+      expire_after 120
+    ])
+    assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
+    assert_nil d.instance.database
+        d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
+    actual_documents = get_documents
+    time = event_time("2011-01-02 13:14:15 UTC")
+    expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
+    assert_equal(expected, actual_documents)
+
+    indexes = get_indexes()
+    expire_after_hash = indexes.map {|e| e.select{|k, v| k == "expireAfterSeconds"} }.reject{|e| e.empty?}.first
+    assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
+  end
+
   class WriteWithCollectionPlaceholder < self
     def setup
       @tag = 'custom'
@@ -409,6 +438,20 @@ class MongoOutputTest < ::Test::Unit::TestCase
       time
     end
 
+    def emit_nested_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: { 'time': @updated_at_str}})
+      d.feed(time, {'a' => 2, updated_at: { 'time': @updated_at_t.to_f}})
+      d.feed(time, {'a' => 3, updated_at: { 'time': @updated_at_t.to_i}})
+      time
+    end
+
+    def emit_nested_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, 'updated_at': { 'time': "Invalid Date String"}})
+      time
+    end
+
     def test_write_with_date_keys
       d = create_driver(default_config + %[
         date_keys updated_at
@@ -439,5 +482,130 @@ class MongoOutputTest < ::Test::Unit::TestCase
       actual_documents = get_documents
       assert_nil actual_documents.first['updated_at']
     end
+
+    def test_write_with_date_nested_keys
+      d = create_driver(default_config + %[
+        replace_dot_in_key_with _
+        replace_dollar_in_key_with _
+        date_keys $.updated_at.time
+        time_key created_at
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc['updated_at']['time'].is_a?(Time)
+      }
+    end
+
+    def test_write_with_parsed_date_nested_key_invalid_string
+      d = create_driver(default_config + %[
+            replace_dot_in_key_with _
+            replace_dollar_in_key_with _
+            date_keys $.updated_at.time
+            time_key created_at
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['updated_at']['time']
+    end
+  end
+
+  sub_test_case 'object_id_keys' do
+    setup do
+      @my_id_str = "507f1f77bcf86cd799439011"
+    end
+
+    def emit_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: @my_id_str})
+      time
+    end
+
+    def emit_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: "Invalid ObjectId String"})
+      time
+    end
+
+    def emit_nested_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: { 'id': @my_id_str}})
+      time
+    end
+
+    def emit_nested_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, 'my_id': { 'id': "Invalid ObjectId String"}})
+      time
+    end
+
+    def test_write_with_object_id_keys
+      d = create_driver(default_config + %[
+        object_id_keys my_id
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      object_id_key = d.instance.object_id_keys.first
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc[object_id_key].is_a?(BSON::ObjectId)
+      }
+    end
+
+    def test_write_with_parsed_object_id_key_invalid_string
+      d = create_driver(default_config + %[
+            object_id_keys my_id
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['my_id']
+    end
+
+    def test_write_with_date_nested_keys
+      d = create_driver(default_config + %[
+        replace_dot_in_key_with _
+        replace_dollar_in_key_with _
+        object_id_keys $.my_id.id
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc['my_id']['id'].is_a?(BSON::ObjectId)
+      }
+    end
+
+    def test_write_with_parsed_date_nested_key_invalid_string
+      d = create_driver(default_config + %[
+            replace_dot_in_key_with _
+            replace_dollar_in_key_with _
+            object_id_keys $.my_id.id
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['my_id']['id']
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-28 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,22 @@ dependencies:
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.15.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.19.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.15.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +112,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.4
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 description: MongoDB plugin for Fluentd
 email: repeatedly@gmail.com
 executables:
@@ -121,6 +141,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".travis.yml"
 - AUTHORS
@@ -159,12 +180,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: MongoDB plugin for Fluentd
-test_files:
-- test/helper.rb
-- test/plugin/test_in_mongo_tail.rb
-- test/plugin/test_out_mongo.rb
-- test/plugin/test_out_mongo_replset.rb
+test_files: []