fluent-plugin-mongo 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fb5c56a40c5a058bd8b710af7fe2b065b96b2e0d
-  data.tar.gz: 761f54a14e8dde03a9c669ff89a0dda9d07b84ce
+SHA256:
+  metadata.gz: 709f3068ce7f97bcad4737b3c7a9db60bdea998b184949afa7f3fa1794831474
+  data.tar.gz: 53d8b989b2d30d8b0df0e37701af671713460dd46fe920435016ececcaef6c36
 SHA512:
-  metadata.gz: b9b5368add232fe67d644a95d28e5adbe8f4584f63e67c347effb36b50188b1321f1628d16c4494c1aac404db120edc91ff09558f85ad3a7439310909fc4e4d9
-  data.tar.gz: cd1d6a83f39438d464829baab15d6e9e490fa1b693947111b9ffc7176ff2c8c73673f5c109a562a9d17be447b99cbf59fbb9ebf98ebc04f6ba39cdaafcdb4684
+  metadata.gz: 6ce121ba68754c2d6687a7d1ed8cedf8c9fe94652f85476b03aacddf92bede132aa7f45444c8cb4153ff5509143bf00688cffc5c872c15e76c754e0cd6f5ff8f
+  data.tar.gz: 4391ce8e22ca8173f38630a6f22024acdccae01d48a3bf5c86f255bbdbe057cdbbb9039b9f55b209bfaf377188124a5b1dd83fcaf8b58db80785e16348e8caa0

data/.travis.yml

@@ -1,9 +1,8 @@
 rvm:
- - 2.1
- - 2.2.4
- - 2.3.1
- - 2.4.3
- - 2.5.0
+ - 2.4.10
+ - 2.5
+ - 2.6
+ - 2.7
  - ruby-head
 
 gemfile:
@@ -12,12 +11,6 @@ gemfile:
 services:
   - mongodb
 
-before_install:
-  - gem update --system=2.7.8
-
-before_script:
-  - git submodule update -i
-
 script: bundle exec rake test
 
 matrix:

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.4.0 - 2020/04/28
+
+	* out_mongo: Add date_keys parameter to support MongoDB Date object in record fields.
+
 Release 1.3.0 - 2019/04/24
 
 	* out_mongo: Support auth_mech parameter to allow other authentication

data/README.rdoc

@@ -53,6 +53,12 @@ Use _mongo_ type in match.
       capped
       capped_size 100m
 
+      # Specify date fields in record to use MongoDB's Date object (Optional) default: nil
+      # Supported data types are String/Integer/Float/Fuentd EventTime.
+      # For Integer type, milliseconds epoch and seconds epoch are supported.
+      # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
+      date_keys updated_at
+
       # Other buffer configurations here
     </match>
 
@@ -223,7 +229,7 @@ BSON records which include '.' or start with '$' are invalid and they will be st
       ...
       # replace '.' in keys with '__dot__'
       replace_dot_in_key_with __dot__
-    
+
       # replace '$' in keys with '__dollar__'
       # Note: This replaces '$' only on first character
       replace_dollar_in_key_with __dollar__

data/VERSION

@@ -1 +1 @@
-1.3.0
+1.4.0

data/fluent-plugin-mongo.gemspec

@@ -10,7 +10,6 @@ Gem::Specification.new do |gem|
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Masahiro Nakagawa"]
   gem.email       = "repeatedly@gmail.com"
-  gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/out_mongo.rb

@@ -38,6 +38,10 @@ module Fluent::Plugin
     desc "Replace dollar with specified string"
     config_param :replace_dollar_in_key_with, :string, default: nil
 
+    # Additional date field to be used to Date object
+    desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
+    config_param :date_keys, :array, default: nil
+
     # tag mapping mode
     desc "Use tag_mapped mode"
     config_param :tag_mapped, :bool, default: false,
@@ -54,6 +58,7 @@ module Fluent::Plugin
     config_param :ssl_verify, :bool, default: false
     config_param :ssl_ca_cert, :string, default: nil
 
+
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
       config_set_default :chunk_keys, ['tag']
@@ -198,11 +203,40 @@ module Fluent::Plugin
     def collect_records(chunk)
       records = []
       time_key = @inject_config.time_key if @inject_config
+      date_keys = @date_keys
+
       tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
         record = inject_values_to_record(tag, time, record)
         # MongoDB uses BSON's Date for time.
         record[time_key] = Time.at(time || record[time_key]) if time_key
+
+        if date_keys
+          date_keys.each { |date_key|
+            begin
+              date_value = record[date_key]
+              case date_value
+              when Fluent::EventTime
+                record[date_key] = date_value.to_time
+              when Integer
+                record[date_key] = if date_value > 9999999999
+                                     # epoch with milliseconds: e.g. javascript
+                                     Time.at(date_value / 1000.0)
+                                   else
+                                     # epoch with seconds: e.g. ruby
+                                     Time.at(date_value)
+                                   end
+              when Float
+                record[date_key] = Time.at(date_value)
+              else
+                record[date_key] = Time.parse(date_value)
+              end
+            rescue ArgumentError
+              log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{record[date_key]}"
+              record[date_key] = nil
+            end
+          }
+        end
         records << record
       }
       records

data/test/plugin/test_in_mongo_tail.rb

@@ -100,7 +100,7 @@ class MongoTailInputTest < Test::Unit::TestCase
       end
       events = d.events
       assert_equal "input.mongo", events[0][0]
-      assert_equal event_time(@time.to_s), events[0][1]
+      assert events[0][1].is_a?(Fluent::EventTime)
       assert_equal "test", events[0][2]["message"]
     end
 
@@ -138,7 +138,7 @@ class MongoTailInputTest < Test::Unit::TestCase
       events = d.events
       assert_equal 1, events.size
       assert_equal "input.mongo.last_id", events[0][0]
-      assert_equal event_time(@time.to_s), events[0][1]
+      assert events[0][1].is_a?(Fluent::EventTime)
       assert_equal "can obtain", events[0][2]["message"]
     end
   end

data/test/plugin/test_out_mongo.rb

@@ -315,7 +315,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
             "$foo$bar" => "baz"
           }
         ],
-             })
+      })
     end
 
     documents = get_documents
@@ -388,4 +388,56 @@ class MongoOutputTest < ::Test::Unit::TestCase
       assert authenticate(@client)
     end
   end
+
+  sub_test_case 'date_keys' do
+    setup do
+      @updated_at_str = "2020-02-01T08:22:23.780Z"
+      @updated_at_t = Time.parse(@updated_at_str)
+    end
+
+    def emit_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: @updated_at_str})
+      d.feed(time, {'a' => 2, updated_at: @updated_at_t.to_f})
+      d.feed(time, {'a' => 3, updated_at: @updated_at_t.to_i})
+      time
+    end
+
+    def emit_invalid_date_documents(d)
+      time = event_time("2011-01-02 13:14:15 UTC")
+      d.feed(time, {'a' => 1, updated_at: "Invalid Date String"})
+      time
+    end
+
+    def test_write_with_date_keys
+      d = create_driver(default_config + %[
+        date_keys updated_at
+        time_key created_at
+      ])
+
+      d.run(default_tag: 'test') do
+        emit_date_documents(d)
+      end
+
+      actual_documents = get_documents
+      date_key = d.instance.date_keys.first
+      actual_documents.each_with_index { |doc, i|
+        assert_equal(i + 1, doc['a'])
+        assert doc[date_key].is_a?(Time)
+      }
+    end
+
+    def test_write_with_parsed_date_key_invalid_string
+      d = create_driver(default_config + %[
+            date_keys updated_at
+            time_key created_at
+          ])
+
+      d.run(default_tag: 'test') do
+        emit_invalid_date_documents(d)
+      end
+      actual_documents = get_documents
+      assert_nil actual_documents.first['updated_at']
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-24 00:00:00.000000000 Z
+date: 2020-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -159,8 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: MongoDB plugin for Fluentd