fluent-plugin-mongo 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: baad354dcd89624e9e6ed224718d569de6a37329e7a0339d2b8f41011ba0b970
-  data.tar.gz: 15c2c50509c70a325fe6c053a09062e5701e61003e550d316ffa491e9cc99b75
+  metadata.gz: 6208365e5862e84db04e62061f822324bc6ce1f82ebb8b9b0c3cb6ed230d983e
+  data.tar.gz: 2c45c015dd0fc2cf98e3b309ec4d96865762ed6bf6e534d316f5c6b214c55fcb
 SHA512:
-  metadata.gz: 59aeae091430fae5bb089245f24d85e8e204a11f7d886f0b53b7cde6af054c0e5b9c2fc94f17f2b0dfdc7fb79c562be7112adfafae71583e8fcf8b18aa8766ab
-  data.tar.gz: 242c5d3d7a58f0be87514909749b066942118d90c24e9990f719eff5956919556a86ae24c427c8d21e55b70b39d03b8796d10c4493d00d7c41a7be635009f738
+  metadata.gz: 046b7463bad7a1be9aae5f2a20991656392e32a4693de753aa0c5010f269d8708e0a2c085b5809408b17c5114aeff534e5cafe5b2527453a07f6707227bcbd15
+  data.tar.gz: 42d7e9a95afeb58709422a13e0e90a911a6e32b8d972e26982496b5e90c106db83eb7d5b06d6c775a4f16a92569f10f2e82764a26d20c1c061408ad2bee89cb9

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.5.0 - 2020/11/26
+
+	* out_mongo: Support nested fields in date_keys
+
 Release 1.4.1 - 2020/08/21
 
 	* out_mongo: Add expire_after parameter

data/VERSION

@@ -1 +1 @@
-1.4.1
+1.5.0

data/lib/fluent/plugin/out_mongo.rb

@@ -8,7 +8,7 @@ module Fluent::Plugin
   class MongoOutput < Output
     Fluent::Plugin.register_output('mongo', self)
 
-    helpers :event_emitter, :inject, :compat_parameters
+    helpers :event_emitter, :inject, :compat_parameters, :record_accessor
 
     include Fluent::MongoAuthParams
     include Fluent::MongoAuth
@@ -41,6 +41,9 @@ module Fluent::Plugin
     # Additional date field to be used to Date object
     desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
     config_param :date_keys, :array, default: nil
+    desc "Specify if the fields in date_keys are of type Integer or Float"
+    config_param :parse_string_number_date, :bool, default: false
+
 
     # tag mapping mode
     desc "Use tag_mapped mode"
@@ -75,6 +78,7 @@ module Fluent::Plugin
       @nodes = nil
       @client_options = {}
       @collection_options = {capped: false}
+      @accessors = {}
     end
 
     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -157,6 +161,13 @@ module Fluent::Plugin
       configure_logger(@mongo_log_level)
 
       log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
+
+      if @date_keys
+        @date_keys.each { |field_name|
+          @accessors[field_name.to_s] = record_accessor_create(field_name)
+        }
+        log.debug "Setup record accessor for every date key"
+      end
     end
 
     def start
@@ -211,28 +222,46 @@ module Fluent::Plugin
         record[time_key] = Time.at(time || record[time_key]) if time_key
 
         if date_keys
-          date_keys.each { |date_key|
+          @accessors.each_pair { |date_key, date_key_accessor|
             begin
-              date_value = record[date_key]
+              date_value = date_key_accessor.call(record)
               case date_value
               when Fluent::EventTime
-                record[date_key] = date_value.to_time
+                value_to_set = date_value.to_time
               when Integer
-                record[date_key] = if date_value > 9999999999
+                value_to_set = if date_value > 9999999999
+                                 # epoch with milliseconds: e.g. javascript
+                                 Time.at(date_value / 1000.0)
+                               else
+                                 # epoch with seconds: e.g. ruby
+                                 Time.at(date_value)
+                               end
+              when Float
+                value_to_set = Time.at(date_value)
+              else
+                if @parse_string_number_date
+                  if date_value.to_i.to_s == date_value
+                    date_value = date_value.to_i
+                    value_to_set = if date_value > 9999999999
                                      # epoch with milliseconds: e.g. javascript
-                                     Time.at(date_value / 1000.0)
+                                     date_value / 1000.0
                                    else
                                      # epoch with seconds: e.g. ruby
-                                     Time.at(date_value)
+                                     date_value
                                    end
-              when Float
-                record[date_key] = Time.at(date_value)
-              else
-                record[date_key] = Time.parse(date_value)
+                  elsif date_value.to_f.to_s == date_value
+                    date_value = date_value.to_f
+                  end
+                  value_to_set = date_value.is_a?(String) ? Time.parse(date_value) : Time.at(date_value)
+                else
+                  value_to_set = Time.parse(date_value)
+                end
               end
+
+              date_key_accessor.set(record, value_to_set)
             rescue ArgumentError
-              log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{record[date_key]}"
-              record[date_key] = nil
+              log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{date_value}"
+              date_key_accessor.set(record, nil)
             end
           }
         end

data/test/plugin/test_out_mongo.rb

@@ -438,6 +438,20 @@ class MongoOutputTest < ::Test::Unit::TestCase
       time
     end
 
+    def emit_nested_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: { 'time': @updated_at_str}})
+      d.feed(time, {'a' => 2, updated_at: { 'time': @updated_at_t.to_f}})
+      d.feed(time, {'a' => 3, updated_at: { 'time': @updated_at_t.to_i}})
+      time
+    end
+
+    def emit_nested_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, 'updated_at': { 'time': "Invalid Date String"}})
+      time
+    end
+
     def test_write_with_date_keys
       d = create_driver(default_config + %[
         date_keys updated_at
@@ -468,5 +482,39 @@ class MongoOutputTest < ::Test::Unit::TestCase
       actual_documents = get_documents
       assert_nil actual_documents.first['updated_at']
     end
+
+    def test_write_with_date_nested_keys
+      d = create_driver(default_config + %[
+        replace_dot_in_key_with _
+        replace_dollar_in_key_with _
+        date_keys $.updated_at.time
+        time_key created_at
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc['updated_at']['time'].is_a?(Time)
+      }
+    end
+
+    def test_write_with_parsed_date_nested_key_invalid_string
+      d = create_driver(default_config + %[
+            replace_dot_in_key_with _
+            replace_dollar_in_key_with _
+            date_keys $.updated_at.time
+            time_key created_at
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_nested_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['updated_at']['time']
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-21 00:00:00.000000000 Z
+date: 2020-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd