fluent-plugin-mongo 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6da21defe0a09c2330001d3c0839d4b2f090aa57
-  data.tar.gz: 301c0fd93b3f10f7a4fca779e67e14663e46d106
+  metadata.gz: 0ad7f90a08c0090846884b95f124df0b6257b420
+  data.tar.gz: '094e060574195842fd49c42d5b0fc433c6d16970'
 SHA512:
-  metadata.gz: cc0dd8f45510a50c0bcf8f9d31c8f1ecb2ed6be94072106192d22d5e77ea452c0ac9085884f8271d08cdbbdd6c1938eed3c5936dc2d216b26c1fb2d53aa31754
-  data.tar.gz: 71aeffbf5e5ac00dda3fe4eb89a2f42e85f203d12cad4de5cc092be3cf32fd738da727c099e3a5d2866bacf765875b777ba1bb1de47979e66aa8e9914c53a446
+  metadata.gz: 508cf655a0807b681f0e7e0229c510d023dd5f0ffe55c98a80d397e9b92caa97566e31ce7b7250900973fc43c4956e51f84f76b9d0a4d48607d050b0ec3fdb4a
+  data.tar.gz: ab536cea004d73000c19dc6d743c224f955d30ab6cfd14f232f046e6cdaba775a2adb446ad70c00b4b6cd341992e91f06e5aac1aadeb7cd02eedd05ca977f170

data/.travis.yml

@@ -2,7 +2,8 @@ rvm:
  - 2.1
  - 2.2.4
  - 2.3.1
- - 2.4.0
+ - 2.4.3
+ - 2.5.0
  - ruby-head
 
 gemfile:
@@ -12,7 +13,7 @@ services:
   - mongodb
 
 before_install:
-  - gem update bundler
+  - gem update --system
 before_script:
   - git submodule update -i
 

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 1.1.0 - 2018/01/18
+
+	* in_mongo: Add batch_size parameter
+	* out_mongo: Handle collection options correctly
+
+
 Release 1.0.0 - 2017/11/26
 
 	* Use new Plugin API

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

data/lib/fluent/plugin/in_mongo_tail.rb

@@ -44,6 +44,9 @@ module Fluent::Plugin
     desc "SSL connection"
     config_param :ssl, :bool, default: false
 
+    desc "Batch size for each find"
+    config_param :batch_size, :integer, default: nil
+
     def initialize
       super
 
@@ -69,6 +72,10 @@ module Fluent::Plugin
       @last_id = @id_store_file ? get_last_id : nil
       @connection_options[:ssl] = @ssl
 
+      if @batch_size && @batch_size <= 0
+        raise Fluent::ConfigError, "Batch size must be positive."
+      end
+
       configure_logger(@mongo_log_level)
     end
 
@@ -99,6 +106,7 @@ module Fluent::Plugin
       begin
         option['_id'] = {'$gt' => BSON::ObjectId(@last_id)} if @last_id
         documents = @collection.find(option)
+        documents = documents.limit(@batch_size) if @batch_size
         if documents.count >= 1
           process_documents(documents)
         end

data/lib/fluent/plugin/out_mongo.rb

@@ -154,6 +154,7 @@ module Fluent::Plugin
     def start
       @client = client
       @client = authenticate(@client)
+      @collections = {}
       super
     end
 
@@ -202,7 +203,7 @@ module Fluent::Plugin
 
     def collect_records(chunk)
       records = []
-      time_key = @inject_config.time_key
+      time_key = @inject_config.time_key if @inject_config
       tag = chunk.metadata.tag
       chunk.msgpack_each {|time, record|
         record = inject_values_to_record(tag, time, record)
@@ -223,6 +224,36 @@ module Fluent::Plugin
       formatted
     end
 
+    def list_collections_enabled?
+      @client.cluster.next_primary(false).features.list_collections_enabled?
+    end
+
+    def collection_exists?(name)
+      if list_collections_enabled?
+        r = @client.database.command(
+          { :listCollections => 1, :filter => { :name => name } }
+        ).first
+        r[:ok] && r[:cursor][:firstBatch].size == 1
+      else
+        @client.database.collection_names.include?(name)
+      end
+    end
+
+    def get_collection(name, options)
+      return @client[name] if @collections[name]
+
+      unless collection_exists?(name)
+        log.trace "Create collection #{name} with options #{options}"
+        @client[name, options].create
+      end
+      @collections[name] = true
+      @client[name]
+    end
+
+    def forget_collection(name)
+      @collections.delete(name)
+    end
+
     def operate(collection, records)
       begin
         if @replace_dot_in_key_with
@@ -236,9 +267,10 @@ module Fluent::Plugin
           end
         end
 
-        @client[collection, @collection_options].insert_many(records)
+        get_collection(collection, @collection_options).insert_many(records)
       rescue Mongo::Error::BulkWriteError => e
         log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
+        forget_collection(collection)
       rescue ArgumentError => e
         log.warn e
       end

data/test/plugin/test_in_mongo_tail.rb

@@ -93,6 +93,7 @@ class MongoTailInputTest < Test::Unit::TestCase
         collection #{collection_name}
         tag input.mongo
         time_key time
+        batch_size 10000
       ])
       d.run(expect_records: 1, timeout: 5) do
         @client[collection_name].insert_one({message: "test"})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mongo
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-27 00:00:00.000000000 Z
+date: 2018-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd