fluent-plugin-mongo 1.5.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6208365e5862e84db04e62061f822324bc6ce1f82ebb8b9b0c3cb6ed230d983e
-  data.tar.gz: 2c45c015dd0fc2cf98e3b309ec4d96865762ed6bf6e534d316f5c6b214c55fcb
+  metadata.gz: 0a7505d5d82e1fcf8aa0625efa3bd19f68d7ddd6a9ec8ef75050d5aba1b663a1
+  data.tar.gz: a0efb583886eb8e8880881d3727dd61e6e2d19c3d2a3d34e4e5d94f0d843b3c3
 SHA512:
-  metadata.gz: 046b7463bad7a1be9aae5f2a20991656392e32a4693de753aa0c5010f269d8708e0a2c085b5809408b17c5114aeff534e5cafe5b2527453a07f6707227bcbd15
-  data.tar.gz: 42d7e9a95afeb58709422a13e0e90a911a6e32b8d972e26982496b5e90c106db83eb7d5b06d6c775a4f16a92569f10f2e82764a26d20c1c061408ad2bee89cb9
+  metadata.gz: c26d8de1f5c46fbeec69a871fcd26ac83969b58b49990a458f8a25eff830a2b35197267a002d657d3ae751aced56d5637ed1fbc394635e2c59886b4ba2fbe82f
+  data.tar.gz: 4a33374071ca219f56bb3be9132c1e05d3c2bf1c4018d769fbf2cb7ea61b4eb8f0cabbb6e85baceea9dfcd44c80bc11e7fce75356abe7846fdb6b6a150d082b3

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

data/.github/workflows/linux.yml

@@ -0,0 +1,48 @@
+name: Test
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '4.0', '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
+        mongodb-version: ['5.0', '4.4', '4.2']
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.0'
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.2'
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.4'
+
+    name: Ruby ${{ matrix.ruby }} and MongoDB ${{ matrix.mongodb-version }} on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Start MongoDB
+      uses: supercharge/mongodb-github-action@1.8.0
+      with:
+        mongodb-version: ${{ matrix.mongodb-version }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/ChangeLog

@@ -1,3 +1,14 @@
+Release 1.6.1 - 2026/02/12
+
+	* out_mongo: Fix RangeError by converting large integers to BSON::Decimal128
+	* gem: add bigdecimal as dependency for Ruby 3.4
+	* Drop support for MongoDB 4.0.
+
+Release 1.6.0 - 2022/07/15
+
+	* Update mongo gem dependency
+	* out_mongo: Add object_id_keys parameter to convert string to Mongo ObjectId
+
 Release 1.5.0 - 2020/11/26
 
 	* out_mongo: Support nested fields in date_keys

data/README.rdoc

@@ -59,6 +59,10 @@ Use _mongo_ type in match.
       # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
       date_keys updated_at
 
+      # Specify id fields in record to use MongoDB's BSON ObjectID (Optional) default: nil
+      # eg: my_id: "507f1f77bcf86cd799439011"
+      object_id_keys my_id
+
       # Other buffer configurations here
     </match>
 

data/VERSION

@@ -1 +1 @@
-1.5.0
+1.6.1

data/fluent-plugin-mongo.gemspec

@@ -17,10 +17,12 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
-  gem.add_runtime_dependency "mongo", "~> 2.6.0"
+  gem.add_dependency "bigdecimal", "~> 4.0"
+  gem.add_runtime_dependency "mongo", ">= 2.15.0", "< 2.19.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"
   gem.add_development_dependency "test-unit", ">= 3.0.0"
-  gem.add_development_dependency "timecop", "~> 0.8.0"
+  gem.add_development_dependency "timecop", "~> 0.9.4"
+  gem.add_development_dependency "webrick", ">= 1.7.0"
 end

data/lib/fluent/plugin/out_mongo.rb

@@ -43,7 +43,8 @@ module Fluent::Plugin
     config_param :date_keys, :array, default: nil
     desc "Specify if the fields in date_keys are of type Integer or Float"
     config_param :parse_string_number_date, :bool, default: false
-
+    desc "Specify keys to use MongoDB's ObjectId"
+    config_param :object_id_keys, :array, default: nil
 
     # tag mapping mode
     desc "Use tag_mapped mode"
@@ -78,7 +79,8 @@ module Fluent::Plugin
       @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
-      @accessors = {}
+      @date_accessors = {}
+      @object_id_accessors = {}
     end
 
     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -164,10 +166,16 @@ module Fluent::Plugin
 
       if @date_keys
         @date_keys.each { |field_name|
-          @accessors[field_name.to_s] = record_accessor_create(field_name)
+          @date_accessors[field_name.to_s] = record_accessor_create(field_name)
         }
         log.debug "Setup record accessor for every date key"
       end
+      if @object_id_keys
+        @object_id_keys.each { |field_name|
+          @object_id_accessors[field_name.to_s] = record_accessor_create(field_name)
+        }
+        log.debug "Setup record accessor for every object_id key"
+      end
     end
 
     def start
@@ -214,6 +222,7 @@ module Fluent::Plugin
       records = []
       time_key = @inject_config.time_key if @inject_config
       date_keys = @date_keys
+      object_id_keys = @object_id_keys
 
       tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
@@ -222,7 +231,7 @@ module Fluent::Plugin
         record[time_key] = Time.at(time || record[time_key]) if time_key
 
         if date_keys
-          @accessors.each_pair { |date_key, date_key_accessor|
+          @date_accessors.each_pair { |date_key, date_key_accessor|
             begin
               date_value = date_key_accessor.call(record)
               case date_value
@@ -265,6 +274,18 @@ module Fluent::Plugin
             end
           }
         end
+        if object_id_keys
+          @object_id_accessors.each_pair { |object_id_key, object_id_key_accessor|
+            begin
+              object_id_value = object_id_key_accessor.call(record)
+              value_to_set = BSON::ObjectId(object_id_value)
+              object_id_key_accessor.set(record, value_to_set)
+            rescue BSON::ObjectId::Invalid
+              log.warn "Failed to parse '#{object_id_key}' field. Expected object_id types are String: #{object_id_value}"
+              object_id_key_accessor.set(record, nil)
+            end
+          }
+        end
         records << record
       }
       records
@@ -330,6 +351,9 @@ module Fluent::Plugin
             replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
           end
         end
+        records.map! do |r|
+          replace_value_of_hash(r)
+        end
 
         get_collection(database, collection, @collection_options).insert_many(records)
       rescue Mongo::Error::BulkWriteError => e
@@ -363,5 +387,23 @@ module Fluent::Plugin
         hash_or_array
       end
     end
+
+    INT64_MAX = (2 ** 63) - 1
+    def replace_value_of_hash(hash_or_array)
+      case hash_or_array
+      when Array
+        hash_or_array.map { |v| replace_value_of_hash(v) }
+      when Hash
+        hash_or_array.each_pair { |k, v| hash_or_array[k] = replace_value_of_hash(v) }
+      when Integer
+        if hash_or_array > INT64_MAX
+          BSON::Decimal128.new(hash_or_array.to_s)
+        else
+          hash_or_array
+        end
+      else
+        hash_or_array
+      end
+    end
   end
 end

data/test/plugin/test_out_mongo.rb

@@ -249,6 +249,20 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
   end
 
+  def test_overflow_integer_value
+    d = create_driver
+    d.run(default_tag: 'test') do
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'overflow' => (2 ** 63)})
+      d.feed(time, {'nested' => {'overflow' => (2 ** 63)}})
+      d.feed(time, {'array' => [(2 ** 63)]})
+    end
+    documents = get_documents
+    assert_equal(BSON::Decimal128.new((2 ** 63).to_s), documents[0]['overflow'])
+    assert_equal(BSON::Decimal128.new((2 ** 63).to_s), documents[1]['nested']['overflow'])
+    assert_equal(BSON::Decimal128.new((2 ** 63).to_s), documents[2]['array'][0])
+  end
+
   class WriteWithCollectionPlaceholder < self
     def setup
       @tag = 'custom'
@@ -517,4 +531,95 @@ class MongoOutputTest < ::Test::Unit::TestCase
       assert_nil actual_documents.first['updated_at']['time']
     end
   end
+
+  sub_test_case 'object_id_keys' do
+    setup do
+      @my_id_str = "507f1f77bcf86cd799439011"
+    end
+
+    def emit_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: @my_id_str})
+      time
+    end
+
+    def emit_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: "Invalid ObjectId String"})
+      time
+    end
+
+    def emit_nested_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: { 'id': @my_id_str}})
+      time
+    end
+
+    def emit_nested_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, 'my_id': { 'id': "Invalid ObjectId String"}})
+      time
+    end
+
+    def test_write_with_object_id_keys
+      d = create_driver(default_config + %[
+        object_id_keys my_id
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      object_id_key = d.instance.object_id_keys.first
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc[object_id_key].is_a?(BSON::ObjectId)
+      }
+    end
+
+    def test_write_with_parsed_object_id_key_invalid_string
+      d = create_driver(default_config + %[
+            object_id_keys my_id
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['my_id']
+    end
+
+    def test_write_with_date_nested_keys
+      d = create_driver(default_config + %[
+        replace_dot_in_key_with _
+        replace_dollar_in_key_with _
+        object_id_keys $.my_id.id
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc['my_id']['id'].is_a?(BSON::ObjectId)
+      }
+    end
+
+    def test_write_with_parsed_date_nested_key_invalid_string
+      d = create_driver(default_config + %[
+            replace_dot_in_key_with _
+            replace_dollar_in_key_with _
+            object_id_keys $.my_id.id
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['my_id']['id']
+    end
+  end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Masahiro Nakagawa
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-26 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -31,19 +30,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: bigdecimal
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.15.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.15.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +125,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.4
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 description: MongoDB plugin for Fluentd
 email: repeatedly@gmail.com
 executables:
@@ -121,8 +154,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/linux.yml"
 - ".gitignore"
-- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -144,7 +178,6 @@ homepage: https://github.com/fluent/fluent-plugin-mongo
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -159,8 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 4.0.3
 specification_version: 4
 summary: MongoDB plugin for Fluentd
 test_files:

data/.travis.yml

@@ -1,18 +0,0 @@
-rvm:
- - 2.4.10
- - 2.5
- - 2.6
- - 2.7
- - ruby-head
-
-gemfile:
- - Gemfile
-
-services:
-  - mongodb
-
-script: bundle exec rake test
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head