fluent-plugin-mongo 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6208365e5862e84db04e62061f822324bc6ce1f82ebb8b9b0c3cb6ed230d983e
-  data.tar.gz: 2c45c015dd0fc2cf98e3b309ec4d96865762ed6bf6e534d316f5c6b214c55fcb
+  metadata.gz: fc300a11be780d8b3ea62e71db08393171fad3625d1d9259cb65a7abe11f9e5e
+  data.tar.gz: db12f19c83ae360e9720c41a2d64c3ce8bd65487d554ed2e416277d0f158b311
 SHA512:
-  metadata.gz: 046b7463bad7a1be9aae5f2a20991656392e32a4693de753aa0c5010f269d8708e0a2c085b5809408b17c5114aeff534e5cafe5b2527453a07f6707227bcbd15
-  data.tar.gz: 42d7e9a95afeb58709422a13e0e90a911a6e32b8d972e26982496b5e90c106db83eb7d5b06d6c775a4f16a92569f10f2e82764a26d20c1c061408ad2bee89cb9
+  metadata.gz: c105cf4d712af0524a554b072f86a2af2cfc5485825db24f47dcd47cc850628cfeb08af8f7db4337aefc7342b2325e35698707db22aa0e595d786a6d72317320
+  data.tar.gz: '08454ce8638ee2bf64d1f6ae2d6adfd0632ed43ad47667558b9ee00dccb54a12d8fbac31c0b8032bd69598752a442bac02e7425e2ccd649754d7c48df1723b8b'

data/.github/workflows/linux.yml

@@ -0,0 +1,49 @@
+name: Test
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.1', '3.0', '2.7', '2.6' ]
+        mongodb-version: ['5.0', '4.4', '4.2', '4.0']
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.0'
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.2'
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+            mongodb-version: '4.4'
+
+    name: Ruby ${{ matrix.ruby }} and MongoDB ${{ matrix.mongodb-version }} on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Start MongoDB
+      uses: supercharge/mongodb-github-action@1.3.0
+      with:
+        mongodb-version: ${{ matrix.mongodb-version }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 1.6.0 - 2022/07/15
+
+	* Update mongo gem dependency
+	* out_mongo: Add object_id_keys parameter to convert string to Mongo ObjectId
+
 Release 1.5.0 - 2020/11/26
 
 	* out_mongo: Support nested fields in date_keys

data/README.rdoc

@@ -59,6 +59,10 @@ Use _mongo_ type in match.
       # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
       date_keys updated_at
 
+      # Specify id fields in record to use MongoDB's BSON ObjectID (Optional) default: nil
+      # eg: my_id: "507f1f77bcf86cd799439011"
+      object_id_keys my_id
+
       # Other buffer configurations here
     </match>
 

data/VERSION

@@ -1 +1 @@
-1.5.0
+1.6.0

data/fluent-plugin-mongo.gemspec

@@ -17,10 +17,11 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
-  gem.add_runtime_dependency "mongo", "~> 2.6.0"
+  gem.add_runtime_dependency "mongo", ">= 2.15.0", "< 2.19.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "simplecov", ">= 0.5.4"
   gem.add_development_dependency "rr", ">= 1.0.0"
   gem.add_development_dependency "test-unit", ">= 3.0.0"
-  gem.add_development_dependency "timecop", "~> 0.8.0"
+  gem.add_development_dependency "timecop", "~> 0.9.4"
+  gem.add_development_dependency "webrick", ">= 1.7.0"
 end

data/lib/fluent/plugin/out_mongo.rb

@@ -43,7 +43,8 @@ module Fluent::Plugin
     config_param :date_keys, :array, default: nil
     desc "Specify if the fields in date_keys are of type Integer or Float"
     config_param :parse_string_number_date, :bool, default: false
-
+    desc "Specify keys to use MongoDB's ObjectId"
+    config_param :object_id_keys, :array, default: nil
 
     # tag mapping mode
     desc "Use tag_mapped mode"
@@ -78,7 +79,8 @@ module Fluent::Plugin
       @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
-      @accessors = {}
+      @date_accessors = {}
+      @object_id_accessors = {}
     end
 
     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -164,10 +166,16 @@ module Fluent::Plugin
 
       if @date_keys
         @date_keys.each { |field_name|
-          @accessors[field_name.to_s] = record_accessor_create(field_name)
+          @date_accessors[field_name.to_s] = record_accessor_create(field_name)
         }
         log.debug "Setup record accessor for every date key"
       end
+      if @object_id_keys
+        @object_id_keys.each { |field_name|
+          @object_id_accessors[field_name.to_s] = record_accessor_create(field_name)
+        }
+        log.debug "Setup record accessor for every object_id key"
+      end
     end
 
     def start
@@ -214,6 +222,7 @@ module Fluent::Plugin
       records = []
       time_key = @inject_config.time_key if @inject_config
       date_keys = @date_keys
+      object_id_keys = @object_id_keys
 
       tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
@@ -222,7 +231,7 @@ module Fluent::Plugin
         record[time_key] = Time.at(time || record[time_key]) if time_key
 
         if date_keys
-          @accessors.each_pair { |date_key, date_key_accessor|
+          @date_accessors.each_pair { |date_key, date_key_accessor|
             begin
               date_value = date_key_accessor.call(record)
               case date_value
@@ -265,6 +274,18 @@ module Fluent::Plugin
             end
           }
         end
+        if object_id_keys
+          @object_id_accessors.each_pair { |object_id_key, object_id_key_accessor|
+            begin
+              object_id_value = object_id_key_accessor.call(record)
+              value_to_set = BSON::ObjectId(object_id_value)
+              object_id_key_accessor.set(record, value_to_set)
+            rescue BSON::ObjectId::Invalid
+              log.warn "Failed to parse '#{object_id_key}' field. Expected object_id types are String: #{object_id_value}"
+              object_id_key_accessor.set(record, nil)
+            end
+          }
+        end
         records << record
       }
       records

data/test/plugin/test_out_mongo.rb

@@ -517,4 +517,95 @@ class MongoOutputTest < ::Test::Unit::TestCase
       assert_nil actual_documents.first['updated_at']['time']
     end
   end
+
+  sub_test_case 'object_id_keys' do
+    setup do
+      @my_id_str = "507f1f77bcf86cd799439011"
+    end
+
+    def emit_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: @my_id_str})
+      time
+    end
+
+    def emit_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: "Invalid ObjectId String"})
+      time
+    end
+
+    def emit_nested_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, my_id: { 'id': @my_id_str}})
+      time
+    end
+
+    def emit_nested_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, 'my_id': { 'id': "Invalid ObjectId String"}})
+      time
+    end
+
+    def test_write_with_object_id_keys
+      d = create_driver(default_config + %[
+        object_id_keys my_id
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      object_id_key = d.instance.object_id_keys.first
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc[object_id_key].is_a?(BSON::ObjectId)
+      }
+    end
+
+    def test_write_with_parsed_object_id_key_invalid_string
+      d = create_driver(default_config + %[
+            object_id_keys my_id
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['my_id']
+    end
+
+    def test_write_with_date_nested_keys
+      d = create_driver(default_config + %[
+        replace_dot_in_key_with _
+        replace_dollar_in_key_with _
+        object_id_keys $.my_id.id
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc['my_id']['id'].is_a?(BSON::ObjectId)
+      }
+    end
+
+    def test_write_with_parsed_date_nested_key_invalid_string
+      d = create_driver(default_config + %[
+            replace_dot_in_key_with _
+            replace_dollar_in_key_with _
+            object_id_keys $.my_id.id
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['my_id']['id']
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-26 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,22 @@ dependencies:
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.15.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.19.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.15.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.19.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -106,14 +112,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.4
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 description: MongoDB plugin for Fluentd
 email: repeatedly@gmail.com
 executables:
@@ -121,6 +141,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".travis.yml"
 - AUTHORS
@@ -159,12 +180,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: MongoDB plugin for Fluentd
-test_files:
-- test/helper.rb
-- test/plugin/test_in_mongo_tail.rb
-- test/plugin/test_out_mongo.rb
-- test/plugin/test_out_mongo_replset.rb
+test_files: []