logstash-output-mongodb 3.0.1 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4ffa90de651ecb6cdded946ccb3df2e6b4ac6f40
-  data.tar.gz: 763433ec998c5c463eaead7a6253310fd41d2fe4
+  metadata.gz: 37c6554062844f26b2e5523500973e725f290ff0
+  data.tar.gz: 7a23d57d1bce56f1382eaf175d2513bcec133085
 SHA512:
-  metadata.gz: 3638c30d6d3eeaab6ab73916f08679847b348888749b059a0e83a9624ac99f57598243aaa7c29b8256dfd6dbc960531e4b4fe0042ed89b7de78f5eed2128c9ff
-  data.tar.gz: 90c011a9f9f71fb0d93cce2e16b1653f29d2675319090f0ed6b5ce4026ed9c0b2b98d5b940d75b5d52fc0abaa0799f95d73ded649a06bea9dcbfe743b686d42f
+  metadata.gz: 8bedf6b5a8e973d5a3a940b5ad9606719c4457a23be1947311a4910cd00a512f1e10503038d710ae4dd4ff19562b4e7fddd5ee7878a939e40f532627226c5572
+  data.tar.gz: aef06d938a3f2c6722c583ad8465ebb7563666c3af6f946061b9197bf5207e41960a377be30e28a3a543fb7e0cdf96c397584594a07087772baa2103a53807af

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.0
+ - Add support for bulk inserts to improve performance.
+
 ## 3.0.1
  - Docs: Fix doc generation issue by removing extraneous comments and adding a short description of the plugin
 

data/lib/logstash/outputs/mongodb.rb

@@ -34,11 +34,41 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
   # "_id" field in the event.
   config :generateId, :validate => :boolean, :default => false
 
+
+  # Bulk insert flag, set to true to allow bulk insertion, else it will insert events one by one.
+  config :bulk, :validate => :boolean, :default => false
+  # Bulk interval, Used to insert events periodically if the "bulk" flag is activated.
+  config :bulk_interval, :validate => :number, :default => 2
+  # Bulk events number, if the number of events to insert into a collection raise that limit, it will be bulk inserted
+  # whatever the bulk interval value (mongodb hard limit is 1000).
+  config :bulk_size, :validate => :number, :default => 900, :maximum => 999, :min => 2
+
+  # Mutex used to synchronize access to 'documents'
+  @@mutex = Mutex.new
+
   public
   def register
     Mongo::Logger.logger = @logger
     conn = Mongo::Client.new(@uri)
     @db = conn.use(@database)
+
+    if @bulk_size > 1000
+      raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
+    end
+    @documents = {}
+    Thread.new do
+      loop do
+        sleep @bulk_interval
+        @@mutex.synchronize do
+          @documents.each do |collection, values|
+            if values.length > 0
+              @db[collection].insert_many(values)
+              @documents.delete(collection)
+            end
+          end
+        end
+      end
+    end
   end # def register
 
   def receive(event)
@@ -53,7 +83,23 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
       if @generateId
         document["_id"] = BSON::ObjectId.new(nil, event.timestamp)
       end
-      @db[event.sprintf(@collection)].insert_one(document)
+      if @bulk
+        @@mutex.synchronize do
+          collection = event.sprintf(@collection)
+          if(!@documents[collection])
+            @documents[collection] = []
+          end
+          @documents[collection].push(document)
+
+          if(@documents[collection].length >= @bulk_size)
+            @db[collection].insert_many(@documents[collection])
+            @documents.delete(collection)
+          end
+        end
+      else
+        @db[event.sprintf(@collection)].insert_one(document)
+      end
+
     rescue => e
       @logger.warn("Failed to send event to MongoDB", :event => event, :exception => e,
                    :backtrace => e.backtrace)

data/logstash-output-mongodb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-mongodb'
-  s.version         = '3.0.1'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Store events into MongoDB"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-mongodb
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-15 00:00:00.000000000 Z
+date: 2017-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement