logstash-output-mongodb 3.1.4 → 3.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: baf5b3247ad538b7b06f0a74a9f1bfd0e1628c30a19a7645331c052d7e05be2a
-  data.tar.gz: f375e7f1c918a0222261aac759a425c45a0d4302acf8760639acaadcab29d0e4
+  metadata.gz: a80fe32d5f3500dadd616df40cbf18fdb09678d4c2501d7a5514662cf6934b87
+  data.tar.gz: ca57807a9e2a897d6b2d4177735c58a7a2ac8db5f156148393f5e8ae6694584f
 SHA512:
-  metadata.gz: acb5751495b30c5fef2192e9e3f5dad862c32f0af6178b0855b3d268db860bdf467388852c87494e097815193ce1e81652e682200dd48c63f04524c43639669c
-  data.tar.gz: a6a3b1e2b5ec9663e4a2f13a2e1c059866497e971a971dc89b87df16a5ca488c3bea958dee9d0ebf7aac6ec8a08458bbefa2c76638e8569e97abfbbfc41ab6a7
+  metadata.gz: 338736bbad3f0f3f7bea52c42afbf820490308f4c66de9918c30ab87ec65f3a8dc143cf2466de117c5ed747a604fe7ea5d34255914ecb47cd235593aaa7d8fce
+  data.tar.gz: 44f24d65baec8b345730066a17fa0c329d8b6d8109f50c5f50351abb4902d0c68f4d8af54d9f42556c65b0724975bf2426f16e59150f3fa3f0971d78c133206d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.5
+  - Fixed @timestamp handling, BSON::ObjectId generation, close method [#57](https://github.com/logstash-plugins/logstash-output-mongodb/pull/57)
+
 ## 3.1.4
   - Docs: Set the default_codec doc attribute.
 

data/CONTRIBUTORS

@@ -15,6 +15,7 @@ Contributors:
 * Richard Pijnenburg (electrical)
 * bitsofinfo (bitsofinfo)
 * Guy Boertje (guyboertje)
+* Colin Surprenant (colinsurprenant)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/outputs/mongodb.rb

@@ -46,19 +46,22 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
   # Mutex used to synchronize access to 'documents'
   @@mutex = Mutex.new
 
-  public
   def register
+    if @bulk_size > 1000
+      raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
+    end
+
     Mongo::Logger.logger = @logger
     conn = Mongo::Client.new(@uri)
     @db = conn.use(@database)
 
-    if @bulk_size > 1000
-      raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
-    end
+    @closed = Concurrent::AtomicBoolean.new(false)
     @documents = {}
-    Thread.new do
-      loop do
-        sleep @bulk_interval
+
+    @bulk_thread = Thread.new(@bulk_interval) do |bulk_interval|
+      while @closed.false? do
+        sleep(bulk_interval)
+
         @@mutex.synchronize do
           @documents.each do |collection, values|
             if values.length > 0
@@ -69,23 +72,31 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
         end
       end
     end
-  end # def register
+  end
 
   def receive(event)
     begin
       # Our timestamp object now has a to_bson method, using it here
       # {}.merge(other) so we don't taint the event hash innards
       document = {}.merge(event.to_hash)
+
       if !@isodate
-        # not using timestamp.to_bson
-        document["@timestamp"] = event.timestamp.to_json
+        timestamp = event.timestamp
+        if timestamp
+          # not using timestamp.to_bson
+          document["@timestamp"] = timestamp.to_json
+        else
+          @logger.warn("Cannot set MongoDB document `@timestamp` field because it does not exist in the event", :event => event)
+        end
       end
+
       if @generateId
-        document["_id"] = BSON::ObjectId.new(nil, event.timestamp)
+        document["_id"] = BSON::ObjectId.new
       end
+
       if @bulk
+        collection = event.sprintf(@collection)
         @@mutex.synchronize do
-          collection = event.sprintf(@collection)
           if(!@documents[collection])
             @documents[collection] = []
           end
@@ -99,20 +110,25 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
       else
         @db[event.sprintf(@collection)].insert_one(document)
       end
-
     rescue => e
-      @logger.warn("Failed to send event to MongoDB", :event => event, :exception => e,
-                   :backtrace => e.backtrace)
       if e.message =~ /^E11000/
-          # On a duplicate key error, skip the insert.
-          # We could check if the duplicate key err is the _id key
-          # and generate a new primary key.
-          # If the duplicate key error is on another field, we have no way
-          # to fix the issue.
+        # On a duplicate key error, skip the insert.
+        # We could check if the duplicate key err is the _id key
+        # and generate a new primary key.
+        # If the duplicate key error is on another field, we have no way
+        # to fix the issue.
+        @logger.warn("Skipping insert because of a duplicate key error", :event => event, :exception => e)
       else
-        sleep @retry_delay
+        @logger.warn("Failed to send event to MongoDB, retrying in #{@retry_delay.to_s} seconds", :event => event, :exception => e)
+        sleep(@retry_delay)
         retry
       end
     end
-  end # def receive
-end # class LogStash::Outputs::Mongodb
+  end
+
+  def close
+    @closed.make_true
+    @bulk_thread.wakeup
+    @bulk_thread.join
+  end
+end

data/logstash-output-mongodb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-mongodb'
-  s.version         = '3.1.4'
+  s.version         = '3.1.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Writes events to MongoDB"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/mongodb_spec.rb

@@ -8,23 +8,21 @@ describe LogStash::Outputs::Mongodb do
   let(:database)   { 'logstash' }
   let(:collection) { 'logs' }
 
-  let(:config) do
-    { "uri" => uri, "database" => database, "collection" => collection }
-  end
+  let(:config) {{
+    "uri" => uri,
+    "database" => database,
+    "collection" => collection
+  }}
 
-  it "should register" do
+  it "should register and close" do
     plugin = LogStash::Plugin.lookup("output", "mongodb").new(config)
     expect {plugin.register}.to_not raise_error
+    plugin.close
   end
 
-  describe "#send" do
-
+  describe "receive" do
     subject! { LogStash::Outputs::Mongodb.new(config) }
 
-    let(:properties) { { "message" => "This is a message!",
-                         "uuid" => SecureRandom.uuid,
-                         "number" => BigDecimal.new("4321.1234"),
-                         "utf8" => "żółć"} }
     let(:event)      { LogStash::Event.new(properties) }
     let(:connection) { double("connection") }
     let(:client)     { double("client") }
@@ -38,10 +36,51 @@ describe LogStash::Outputs::Mongodb do
       subject.register
     end
 
-    it "should send the event to the database" do
-      expect(collection).to receive(:insert_one)
-      subject.receive(event)
+    after(:each) do
+      subject.close
+    end
+
+    describe "#send" do
+      let(:properties) {{
+        "message" => "This is a message!",
+        "uuid" => SecureRandom.uuid,
+        "number" => BigDecimal.new("4321.1234"),
+        "utf8" => "żółć"
+      }}
+
+      it "should send the event to the database" do
+        expect(collection).to receive(:insert_one)
+        subject.receive(event)
+      end
     end
-  end
 
+    describe "no event @timestamp" do
+      let(:properties) { { "message" => "foo" } }
+
+      it "should not contain a @timestamp field in the mongo document" do
+        expect(event).to receive(:timestamp).and_return(nil)
+        expect(event).to receive(:to_hash).and_return(properties)
+        expect(collection).to receive(:insert_one).with(properties)
+        subject.receive(event)
+      end
+    end
+
+    describe "generateId" do
+      let(:properties) { { "message" => "foo" } }
+      let(:config) {{
+          "uri" => uri,
+          "database" => database,
+          "collection" => collection,
+          "generateId" => true
+      }}
+
+      it "should contain a BSON::ObjectId as _id" do
+        expect(BSON::ObjectId).to receive(:new).and_return("BSON::ObjectId")
+        expect(event).to receive(:timestamp).and_return(nil)
+        expect(event).to receive(:to_hash).and_return(properties)
+        expect(collection).to receive(:insert_one).with(properties.merge("_id" => "BSON::ObjectId"))
+        subject.receive(event)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-mongodb
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.1.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2019-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -119,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Writes events to MongoDB