fluent-plugin-mongo 1.6.0 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc300a11be780d8b3ea62e71db08393171fad3625d1d9259cb65a7abe11f9e5e
-  data.tar.gz: db12f19c83ae360e9720c41a2d64c3ce8bd65487d554ed2e416277d0f158b311
+  metadata.gz: '09b007f64939a3ad6f8476d7845c8932ee122d035902a5ca75c22574fededde0'
+  data.tar.gz: 3f18cdb797b67bfadc7f6a26236ab5eb1d47af5b271c9820dacb65b684eacc08
 SHA512:
-  metadata.gz: c105cf4d712af0524a554b072f86a2af2cfc5485825db24f47dcd47cc850628cfeb08af8f7db4337aefc7342b2325e35698707db22aa0e595d786a6d72317320
-  data.tar.gz: '08454ce8638ee2bf64d1f6ae2d6adfd0632ed43ad47667558b9ee00dccb54a12d8fbac31c0b8032bd69598752a442bac02e7425e2ccd649754d7c48df1723b8b'
+  metadata.gz: 05d1cb4a3bc5c31f8318a25d04f80ded83f57b18e1ab2d4c1e7b43c85c644d7b1db2fd6ed2ea4ca24f3f9f8d3d03cc811566cd73e71f1bcda0a0cd7aee1f74d0
+  data.tar.gz: 8d76f125209e1d7bfcf890d445302df48988ea63e4e24e252e60b9fee8ecb38e9f46af0ab6c1faf9d627f8a166617b7b1a4aa772304ad730b583586846b88878

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please check the [README](https://github.com/fluent/fluent-plugin-mongo/blob/master/README.rdoc) first. To help us investigate the problem, please provide the following information.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent/Fluent Package version:
+        - fluent-plugin-mongo version:
+        - mongo version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-mongo. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please check the [README](https://github.com/fluent/fluent-plugin-mongo/blob/master/README.rdoc) first. To help us understand the request, please provide the following information.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'

data/.github/workflows/add-to-project.yml

@@ -0,0 +1,24 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        with:
+          app-id: ${{ secrets.PROJECT_APP_ID }}
+          private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
+          owner: fluent
+      - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ steps.generate_token.outputs.token }}
+          labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -4,46 +4,39 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
+    continue-on-error: ${{ matrix.ruby == 'head' }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.1', '3.0', '2.7', '2.6' ]
-        mongodb-version: ['5.0', '4.4', '4.2', '4.0']
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        mongodb-version: ['8.0', '7.0', '6.0', '5.0', '4.4', '4.2']
         os:
           - ubuntu-latest
-        experimental: [false]
-        include:
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-            mongodb-version: '4.0'
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-            mongodb-version: '4.2'
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-            mongodb-version: '4.4'
 
     name: Ruby ${{ matrix.ruby }} and MongoDB ${{ matrix.mongodb-version }} on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@3ff19f5e2baf30647122352b96108b1fbe250c64 # v1.299.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Start MongoDB
-      uses: supercharge/mongodb-github-action@1.3.0
+      uses: supercharge/mongodb-github-action@315db7fe45ac2880b7758f1933e6e5d59afd5e94 # v1.12.1
       with:
         mongodb-version: ${{ matrix.mongodb-version }}
     - name: unit testing
       env:
         CI: true
       run: |
-        gem install bundler rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/ChangeLog

@@ -1,3 +1,15 @@
+Release 1.6.2 - 2026/04/23
+
+	* out_mongo: prevent infinite retry on MaxBSONSize error (#206)
+	* out_mongo: fix NoMethodError during gracefulReload (#205)
+	* Add support for MongoDB 6.0, 7.0, 8.0 (#177)
+
+Release 1.6.1 - 2026/02/12
+
+	* out_mongo: Fix RangeError by converting large integers to BSON::Decimal128
+	* gem: add bigdecimal as dependency for Ruby 3.4
+	* Drop support for MongoDB 4.0.
+
 Release 1.6.0 - 2022/07/15
 
 	* Update mongo gem dependency

data/README.rdoc

@@ -17,6 +17,17 @@ The gem is hosted at {Rubygems.org}[http://rubygems.org]. You can install the ge
 
     $ fluent-gem install fluent-plugin-mongo
 
+=== For Windows users
+
+This plugin depends on the C extension library (such as `bson` gem) which requires a C compiler to install.
+
+If you are using Windows, you must install the {MSYS2}[https://www.msys2.org/] development toolchain before installing this plugin:
+
+1. Run the following command in your command prompt:
+    $ ridk install
+2. Select option `3` (MSYS2 and MINGW development toolchain).
+3. Once the compiler installation is complete, run the <tt>ridk exec fluent-gem install fluent-plugin-mongo</tt> command.
+
 = Plugins
 
 == Output plugin

data/VERSION

@@ -1 +1 @@
-1.6.0
+1.6.2

data/fluent-plugin-mongo.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
+  gem.add_dependency "bigdecimal", "~> 4.0"
   gem.add_runtime_dependency "mongo", ">= 2.15.0", "< 2.19.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"

data/lib/fluent/plugin/out_mongo.rb

@@ -186,7 +186,7 @@ module Fluent::Plugin
     end
 
     def shutdown
-      @client.close
+      @client&.close
       super
     end
 
@@ -351,11 +351,17 @@ module Fluent::Plugin
             replace_key_of_hash(r, /^\$/, @replace_dollar_in_key_with)
           end
         end
+        records.map! do |r|
+          replace_value_of_hash(r)
+        end
 
         get_collection(database, collection, @collection_options).insert_many(records)
       rescue Mongo::Error::BulkWriteError => e
         log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
         forget_collection(collection)
+      rescue Mongo::Error::MaxBSONSize => e
+        log.warn e
+        raise Fluent::UnrecoverableError, e
       rescue ArgumentError => e
         log.warn e
       end
@@ -384,5 +390,23 @@ module Fluent::Plugin
         hash_or_array
       end
     end
+
+    INT64_MAX = (2 ** 63) - 1
+    def replace_value_of_hash(hash_or_array)
+      case hash_or_array
+      when Array
+        hash_or_array.map { |v| replace_value_of_hash(v) }
+      when Hash
+        hash_or_array.each_pair { |k, v| hash_or_array[k] = replace_value_of_hash(v) }
+      when Integer
+        if hash_or_array > INT64_MAX
+          BSON::Decimal128.new(hash_or_array.to_s)
+        else
+          hash_or_array
+        end
+      else
+        hash_or_array
+      end
+    end
   end
 end

data/test/plugin/test_out_mongo.rb

@@ -229,8 +229,6 @@ class MongoOutputTest < ::Test::Unit::TestCase
       @type mongo
       connection_string mongodb://localhost:#{port}/#{database_name}
       collection #{collection_name}
-      capped
-      capped_size 100
       expire_after 120
     ])
     assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
@@ -249,6 +247,20 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
   end
 
+  def test_overflow_integer_value
+    d = create_driver
+    d.run(default_tag: 'test') do
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'overflow' => (2 ** 63)})
+      d.feed(time, {'nested' => {'overflow' => (2 ** 63)}})
+      d.feed(time, {'array' => [(2 ** 63)]})
+    end
+    documents = get_documents
+    assert_equal(BSON::Decimal128.new((2 ** 63).to_s), documents[0]['overflow'])
+    assert_equal(BSON::Decimal128.new((2 ** 63).to_s), documents[1]['nested']['overflow'])
+    assert_equal(BSON::Decimal128.new((2 ** 63).to_s), documents[2]['array'][0])
+  end
+
   class WriteWithCollectionPlaceholder < self
     def setup
       @tag = 'custom'
@@ -608,4 +620,25 @@ class MongoOutputTest < ::Test::Unit::TestCase
       assert_nil actual_documents.first['my_id']['id']
     end
   end
+
+  def test_unrecoverable_error_on_huge_document
+    d = create_driver(%[
+      @type mongo
+      database test
+      collection test_unrecoverable
+      <buffer>
+        chunk_limit_size 30m
+      </buffer>
+    ])
+
+    d.run(default_tag: 'test') do
+      time = event_time("2011-01-02 13:14:15 UTC")
+      # The document size exceeds the 16MB limit of BSON, so it should raise an UnrecoverableError.
+      d.feed(time, {'msg' => 'x' * (20 * 1024 * 1024)})
+    end
+
+    logs = d.logs.join("\n")
+    assert_match(/got unrecoverable error.*Fluent::UnrecoverableError/, logs)
+    assert_match(/MaxBSONSize/, logs)
+  end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.2
 platform: ruby
 authors:
 - Masahiro Nakagawa
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,6 +29,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
@@ -141,9 +154,13 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/dependabot.yml"
+- ".github/workflows/add-to-project.yml"
 - ".github/workflows/linux.yml"
 - ".gitignore"
-- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -165,7 +182,6 @@ homepage: https://github.com/fluent/fluent-plugin-mongo
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -180,8 +196,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
-signing_key: 
+rubygems_version: 4.0.6
 specification_version: 4
 summary: MongoDB plugin for Fluentd
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_in_mongo_tail.rb
+- test/plugin/test_out_mongo.rb
+- test/plugin/test_out_mongo_replset.rb

data/.travis.yml

@@ -1,18 +0,0 @@
-rvm:
- - 2.4.10
- - 2.5
- - 2.6
- - 2.7
- - ruby-head
-
-gemfile:
- - Gemfile
-
-services:
-  - mongodb
-
-script: bundle exec rake test
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head