fluent-plugin-mongo 1.6.1 → 1.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a7505d5d82e1fcf8aa0625efa3bd19f68d7ddd6a9ec8ef75050d5aba1b663a1
-  data.tar.gz: a0efb583886eb8e8880881d3727dd61e6e2d19c3d2a3d34e4e5d94f0d843b3c3
+  metadata.gz: 4b6ccaa707cc9ac56e75260f5482ae641a07b9492bfdd897e37c4f28b3cb6d9a
+  data.tar.gz: abfa43c69e8565a83681f68dc6b390b84895e5939a985cb9403cb749f1275b55
 SHA512:
-  metadata.gz: c26d8de1f5c46fbeec69a871fcd26ac83969b58b49990a458f8a25eff830a2b35197267a002d657d3ae751aced56d5637ed1fbc394635e2c59886b4ba2fbe82f
-  data.tar.gz: 4a33374071ca219f56bb3be9132c1e05d3c2bf1c4018d769fbf2cb7ea61b4eb8f0cabbb6e85baceea9dfcd44c80bc11e7fce75356abe7846fdb6b6a150d082b3
+  metadata.gz: c881203d25fbbbec76d3dbbbc3ed9c92484552e0ea9869827b54d96a1ce7fb303cb49b1162f6fc66e3aa284fc6a07895202efe9a3cb3e811be3476c2dfe051cc
+  data.tar.gz: f447183d0f51b61d31f6b913242df7d6719d0f32efa79b23dfd4ea4f404fb30468ce29c8de68be0d45250360cc98829b341bc9040c15ae6fff9f21618b4f59cc

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please check the [README](https://github.com/fluent/fluent-plugin-mongo/blob/master/README.rdoc) first. To help us investigate the problem, please provide the following information.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent/Fluent Package version:
+        - fluent-plugin-mongo version:
+        - mongo version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-mongo. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please check the [README](https://github.com/fluent/fluent-plugin-mongo/blob/master/README.rdoc) first. To help us understand the request, please provide the following information.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

data/.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:
-      interval: 'weekly'
+      interval: 'monthly'

data/.github/workflows/add-to-project.yml

@@ -0,0 +1,24 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
+        with:
+          app-id: ${{ secrets.PROJECT_APP_ID }}
+          private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
+          owner: fluent
+      - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ steps.generate_token.outputs.token }}
+          labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -4,40 +4,34 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
+    continue-on-error: ${{ matrix.ruby == 'head' }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '4.0', '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
-        mongodb-version: ['5.0', '4.4', '4.2']
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        mongodb-version: ['8.0', '7.0', '6.0', '5.0', '4.4', '4.2']
         os:
           - ubuntu-latest
-        experimental: [false]
-        include:
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-            mongodb-version: '4.0'
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-            mongodb-version: '4.2'
-          - ruby: head
-            os: ubuntu-latest
-            experimental: true
-            mongodb-version: '4.4'
 
     name: Ruby ${{ matrix.ruby }} and MongoDB ${{ matrix.mongodb-version }} on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@3ff19f5e2baf30647122352b96108b1fbe250c64 # v1.299.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Start MongoDB
-      uses: supercharge/mongodb-github-action@1.8.0
+      uses: supercharge/mongodb-github-action@315db7fe45ac2880b7758f1933e6e5d59afd5e94 # v1.12.1
       with:
         mongodb-version: ${{ matrix.mongodb-version }}
     - name: unit testing

data/ChangeLog

@@ -1,3 +1,13 @@
+Release 1.6.3 - 2026/04/24
+
+    * out_mongo: Add `ordered` parameter (#210)
+
+Release 1.6.2 - 2026/04/23
+
+	* out_mongo: prevent infinite retry on MaxBSONSize error (#206)
+	* out_mongo: fix NoMethodError during gracefulReload (#205)
+	* Add support for MongoDB 6.0, 7.0, 8.0 (#177)
+
 Release 1.6.1 - 2026/02/12
 
 	* out_mongo: Fix RangeError by converting large integers to BSON::Decimal128

data/README.rdoc

@@ -17,6 +17,17 @@ The gem is hosted at {Rubygems.org}[http://rubygems.org]. You can install the ge
 
     $ fluent-gem install fluent-plugin-mongo
 
+=== For Windows users
+
+This plugin depends on the C extension library (such as `bson` gem) which requires a C compiler to install.
+
+If you are using Windows, you must install the {MSYS2}[https://www.msys2.org/] development toolchain before installing this plugin:
+
+1. Run the following command in your command prompt:
+    $ ridk install
+2. Select option `3` (MSYS2 and MINGW development toolchain).
+3. Once the compiler installation is complete, run the <tt>ridk exec fluent-gem install fluent-plugin-mongo</tt> command.
+
 = Plugins
 
 == Output plugin
@@ -63,6 +74,10 @@ Use _mongo_ type in match.
       # eg: my_id: "507f1f77bcf86cd799439011"
       object_id_keys my_id
 
+      # Specify whether the operations should be executed in order (default: true).
+      # If false, continues to insert remaining documents even if some inserts fail.
+      ordered false
+
       # Other buffer configurations here
     </match>
 
@@ -90,7 +105,7 @@ Here is an example to use built-in placeholders:
       </buffer>
     </match>
 
-In more detail, please refer to the officilal document for built-in placeholders: https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders
+In more detail, please refer to the officilal document for built-in placeholders: https://docs.fluentd.org/configuration/buffer-section#placeholders
 
 === mongo(tag mapped mode)
 

data/VERSION

@@ -1 +1 @@
-1.6.1
+1.6.3

data/fluent-plugin-mongo.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"
   gem.add_development_dependency "test-unit", ">= 3.0.0"
+  gem.add_development_dependency "test-unit-rr", ">= 1.0.0"
   gem.add_development_dependency "timecop", "~> 0.9.4"
   gem.add_development_dependency "webrick", ">= 1.7.0"
 end

data/lib/fluent/plugin/out_mongo.rb

@@ -65,6 +65,8 @@ module Fluent::Plugin
     config_param :ssl_verify, :bool, default: false
     config_param :ssl_ca_cert, :string, default: nil
 
+    desc "Whether the operations should be executed in order. If false, continues to insert remaining documents even if some inserts fail."
+    config_param :ordered, :bool, default: true
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -186,7 +188,7 @@ module Fluent::Plugin
     end
 
     def shutdown
-      @client.close
+      @client&.close
       super
     end
 
@@ -355,10 +357,13 @@ module Fluent::Plugin
           replace_value_of_hash(r)
         end
 
-        get_collection(database, collection, @collection_options).insert_many(records)
+        get_collection(database, collection, @collection_options).insert_many(records, ordered: @ordered)
       rescue Mongo::Error::BulkWriteError => e
         log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
         forget_collection(collection)
+      rescue Mongo::Error::MaxBSONSize => e
+        log.warn e
+        raise Fluent::UnrecoverableError, e
       rescue ArgumentError => e
         log.warn e
       end

data/test/helper.rb

@@ -4,3 +4,5 @@ require 'mongo'
 require 'fluent/plugin/out_mongo'
 require 'fluent/plugin/out_mongo_replset'
 require 'fluent/plugin/in_mongo_tail'
+require 'rr'
+require 'test/unit/rr'

data/test/plugin/test_in_mongo_tail.rb

@@ -77,7 +77,7 @@ class MongoTailInputTest < Test::Unit::TestCase
       options = {}
       options[:database] = database_name
       @client = ::Mongo::Client.new(["localhost:#{port}"], options)
-      @time = Time.now
+      @time = Time.at(Time.now.to_i)
       Timecop.freeze(@time)
     end
 
@@ -114,7 +114,7 @@ class MongoTailInputTest < Test::Unit::TestCase
         time_key time
       ])
       d.run(expect_records: 1, timeout: 5) do
-        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Time.at(Fluent::Engine.now)})
+        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Time.at(Fluent::Engine.now.to_i)})
       end
       events = d.events
       assert_equal "user.defined", events[0][0]

data/test/plugin/test_out_mongo.rb

@@ -67,6 +67,23 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal({capped: true, size: 100}, d.instance.collection_options)
     assert_equal({ssl: false, write: {j: false}}, d.instance.client_options)
     assert_nil d.instance.connection_string
+    assert_true d.instance.ordered
+  end
+
+  def test_configure_with_disabled_ordered
+    d = create_driver(%[
+      @type mongo
+      database fluent_test
+      collection test_collection
+
+      ordered false
+    ])
+
+    assert_equal('fluent_test', d.instance.database)
+    assert_equal('test_collection', d.instance.collection)
+    assert_equal('localhost', d.instance.host)
+    assert_equal(port, d.instance.port)
+    assert_false d.instance.ordered
   end
 
   def test_configure_with_connection_string
@@ -229,8 +246,6 @@ class MongoOutputTest < ::Test::Unit::TestCase
       @type mongo
       connection_string mongodb://localhost:#{port}/#{database_name}
       collection #{collection_name}
-      capped
-      capped_size 100
       expire_after 120
     ])
     assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
@@ -249,6 +264,24 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
   end
 
+  def test_write_with_disabled_ordered
+    d = create_driver(%[
+      @type mongo
+      connection_string mongodb://localhost:#{port}/#{database_name}
+      collection #{collection_name}
+      ordered false
+    ])
+
+    mock_collection = Object.new
+    # Check the expected value is given as ordered parameter
+    mock(mock_collection).insert_many(anything, ordered: false)
+    stub(d.instance).get_collection { mock_collection }
+
+    d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
+  end
+
   def test_overflow_integer_value
     d = create_driver
     d.run(default_tag: 'test') do
@@ -622,4 +655,25 @@ class MongoOutputTest < ::Test::Unit::TestCase
       assert_nil actual_documents.first['my_id']['id']
     end
   end
+
+  def test_unrecoverable_error_on_huge_document
+    d = create_driver(%[
+      @type mongo
+      database test
+      collection test_unrecoverable
+      <buffer>
+        chunk_limit_size 30m
+      </buffer>
+    ])
+
+    d.run(default_tag: 'test') do
+      time = event_time("2011-01-02 13:14:15 UTC")
+      # The document size exceeds the 16MB limit of BSON, so it should raise an UnrecoverableError.
+      d.feed(time, {'msg' => 'x' * (20 * 1024 * 1024)})
+    end
+
+    logs = d.logs.join("\n")
+    assert_match(/got unrecoverable error.*Fluent::UnrecoverableError/, logs)
+    assert_match(/MaxBSONSize/, logs)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.6.3
 platform: ruby
 authors:
 - Masahiro Nakagawa
@@ -119,6 +119,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -154,7 +168,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/dependabot.yml"
+- ".github/workflows/add-to-project.yml"
 - ".github/workflows/linux.yml"
 - ".gitignore"
 - AUTHORS
@@ -192,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: MongoDB plugin for Fluentd
 test_files: