fluent-plugin-mongo 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 709f3068ce7f97bcad4737b3c7a9db60bdea998b184949afa7f3fa1794831474
-  data.tar.gz: 53d8b989b2d30d8b0df0e37701af671713460dd46fe920435016ececcaef6c36
+  metadata.gz: baad354dcd89624e9e6ed224718d569de6a37329e7a0339d2b8f41011ba0b970
+  data.tar.gz: 15c2c50509c70a325fe6c053a09062e5701e61003e550d316ffa491e9cc99b75
 SHA512:
-  metadata.gz: 6ce121ba68754c2d6687a7d1ed8cedf8c9fe94652f85476b03aacddf92bede132aa7f45444c8cb4153ff5509143bf00688cffc5c872c15e76c754e0cd6f5ff8f
-  data.tar.gz: 4391ce8e22ca8173f38630a6f22024acdccae01d48a3bf5c86f255bbdbe057cdbbb9039b9f55b209bfaf377188124a5b1dd83fcaf8b58db80785e16348e8caa0
+  metadata.gz: 59aeae091430fae5bb089245f24d85e8e204a11f7d886f0b53b7cde6af054c0e5b9c2fc94f17f2b0dfdc7fb79c562be7112adfafae71583e8fcf8b18aa8766ab
+  data.tar.gz: 242c5d3d7a58f0be87514909749b066942118d90c24e9990f719eff5956919556a86ae24c427c8d21e55b70b39d03b8796d10c4493d00d7c41a7be635009f738

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.4.1 - 2020/08/21
+
+	* out_mongo: Add expire_after parameter
+
 Release 1.4.0 - 2020/04/28
 
 	* out_mongo: Add date_keys parameter to support MongoDB Date object in record fields.

data/VERSION

@@ -1 +1 @@
-1.4.0
+1.4.1

data/lib/fluent/plugin/out_mongo.rb

@@ -49,6 +49,9 @@ module Fluent::Plugin
     desc "Remove tag prefix"
     config_param :remove_tag_prefix, :string, default: nil,
                  deprecated: "use @label instead for event routing."
+    # expire indexes
+    desc "Specify expire after seconds"
+    config_param :expire_after, :time, default: 0
 
     # SSL connection
     config_param :ssl, :bool, default: false
@@ -121,10 +124,6 @@ module Fluent::Plugin
         raise Fluent::ConfigError,  "connection_string or database parameter is required"
       end
 
-      unless @ignore_invalid_record
-        log.warn "Since v0.8, invalid record detection will be removed because mongo driver v2.x and API spec don't provide it. You may lose invalid records, so you should not send such records to mongo plugin"
-      end
-
       if conf.has_key?('tag_mapped')
         log.warn "'tag_mapped' feature is replaced with built-in config placeholder. Please consider to use 'collection ${tag}'."
         @collection = '${tag}'
@@ -274,6 +273,13 @@ module Fluent::Plugin
       unless collection_exists?(name)
         log.trace "Create collection #{name} with options #{options}"
         @client[name, options].create
+        if @expire_after > 0 && @inject_config
+          log.trace "Create expiring index with key: \"#{@inject_config.time_key}\" and seconds: \"#{@expire_after}\""
+          @client[name].indexes.create_one(
+            {"#{@inject_config.time_key}": 1},
+            expire_after: @expire_after
+          )
+        end
       end
       @collections[name] = true
       @client[name]

data/test/plugin/test_in_mongo_tail.rb

@@ -114,7 +114,7 @@ class MongoTailInputTest < Test::Unit::TestCase
         time_key time
       ])
       d.run(expect_records: 1, timeout: 5) do
-        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Fluent::Engine.now})
+        @client[collection_name].insert_one({message: "test", tag: "user.defined", time: Time.at(Fluent::Engine.now)})
       end
       events = d.events
       assert_equal "user.defined", events[0][0]

data/test/plugin/test_out_mongo.rb

@@ -180,6 +180,10 @@ class MongoOutputTest < ::Test::Unit::TestCase
     @client[collection].find.to_a.map {|e| e.delete('_id'); e}
   end
 
+  def get_indexes(collection = collection_name)
+    @client[collection].indexes
+  end
+
   def emit_documents(d)
     time = event_time("2011-01-02 13:14:15 UTC")
     d.feed(time, {'a' => 1})
@@ -220,6 +224,31 @@ class MongoOutputTest < ::Test::Unit::TestCase
     assert_equal(expected, actual_documents)
   end
 
+  def test_write_with_expire_index
+    d = create_driver(%[
+      @type mongo
+      connection_string mongodb://localhost:#{port}/#{database_name}
+      collection #{collection_name}
+      capped
+      capped_size 100
+      expire_after 120
+    ])
+    assert_equal("mongodb://localhost:#{port}/#{database_name}", d.instance.connection_string)
+    assert_nil d.instance.database
+        d.run(default_tag: 'test') do
+      emit_documents(d)
+    end
+    actual_documents = get_documents
+    time = event_time("2011-01-02 13:14:15 UTC")
+    expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
+                {'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
+    assert_equal(expected, actual_documents)
+
+    indexes = get_indexes()
+    expire_after_hash = indexes.map {|e| e.select{|k, v| k == "expireAfterSeconds"} }.reject{|e| e.empty?}.first
+    assert_equal({"expireAfterSeconds"=>120.0}, expire_after_hash)
+  end
+
   class WriteWithCollectionPlaceholder < self
     def setup
       @tag = 'custom'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-28 00:00:00.000000000 Z
+date: 2020-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd