logstash-output-mongodb 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a44c64b7edc0e6bced48bd938482682554d4a35f
-  data.tar.gz: 0d550c878ad41f7a29e832438c4b6296aa0bc314
+  metadata.gz: 0e05acac3ccb551768dae1439fef3580a37b6f7a
+  data.tar.gz: 8d240b38a49fe08b48794b93ab4f30b9e8696cf9
 SHA512:
-  metadata.gz: 308e535f7cd35a4eb37208a4f60bbafd5f8bdc9fa935d7799869a93d7d61b936eb77d752bc6adadb58026a3d798c6a656f1e559d38540bfdcea5cd5ba9559a04
-  data.tar.gz: 7fa9612680aec15ab2cf73bd125db5120b04b59892fa3c35f8359f9ea1857e369c65ae8e898bdb1d0ef38aa171a03cf64e64bfc7f141161476ac11b2c2ce5dbe
+  metadata.gz: 385ae9c6b958f15c1d040fa3cfd840347f77135e610134c99370ae183cb349c8aaa5f48ae8095499090e02787a83a072fcd83dfd9d6ba0ea3762d8769fc8d6a9
+  data.tar.gz: fb929fa63ab767c9441bbd753bfd7ef65bc41ebf46719b2163ab4d873d9afc4da007571fded4c4698660ad050645cef0b481f90061ade750e1483beb779cb142

data/CHANGELOG.md

@@ -1,13 +1,16 @@
+## 2.0.3
+ - Patch Timestamp and BigDecimal with to_bson method and register with BSON.
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 
-# 1.0.0
-  - Fixes the plugin to be in the 2.0 series
-  - Add integration and unit test to the project
-  - Adapt the codebase to be 2.0 compatible
-  - Make the internal logger in mongo to report to LS logger
+## 1.0.0
+ - Fixes the plugin to be in the 2.0 series
+ - Add integration and unit test to the project
+ - Adapt the codebase to be 2.0 compatible
+ - Make the internal logger in mongo to report to LS logger
 
-# 0.2.0
-  - Add basic registration test to the project
+## 0.2.0
+ - Add basic registration test to the project

data/CONTRIBUTORS

@@ -14,6 +14,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * bitsofinfo (bitsofinfo)
+* Guy Boertje (guyboertje)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-mongodb-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-mongodb-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/outputs/bson/big_decimal.rb

@@ -0,0 +1,66 @@
+# Copyright (C) 2009-2014 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Modified 2015 Elastic
+
+module BSON
+
+  # Injects behaviour for encoding and decoding BigDecimal values
+  # to and from # raw bytes as specified by the BSON spec.
+  #
+  # @see http://bsonspec.org/#/specification
+  module BigDecimal
+
+    # A floating point is type 0x01 in the BSON spec.
+    BSON_TYPE = 1.chr.force_encoding(BINARY).freeze
+
+    # The pack directive is for 8 byte floating points.
+    PACK = "E".freeze
+
+    # Get the floating point as encoded BSON.
+    # @example Get the floating point as encoded BSON.
+    #   1.221311.to_bson
+    # @return [ String ] The encoded string.
+    # @see http://bsonspec.org/#/specification
+    def to_bson(encoded = ''.force_encoding(BINARY))
+      encoded << [ self ].pack(PACK)
+    end
+
+    module ClassMethods
+
+      # Deserialize an instance of a BigDecimal from a BSON double.
+      # @param [ BSON ] bson object from Mongo.
+      # @return [ BigDecimal ] The decoded BigDecimal.
+      # @see http://bsonspec.org/#/specification
+      def from_bson(bson)
+        from_bson_double(bson.read(8))
+      end
+
+      private
+
+      def from_bson_double(double)
+        new(double.unpack(PACK).first.to_s)
+      end
+    end
+
+    # Register this type when the module is loaded.
+    Registry.register(BSON_TYPE, ::BigDecimal)
+  end
+
+  # Enrich the core BigDecimal class with this module.
+  #
+  # @since 2.0.0
+  ::BigDecimal.send(:include, BigDecimal)
+  ::BigDecimal.send(:extend, BigDecimal::ClassMethods)
+end

data/lib/logstash/outputs/bson/logstash_event.rb

@@ -0,0 +1,76 @@
+# Copyright (C) 2009-2014 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Modified 2015 Elastic
+
+module BSON
+
+  # Injects behaviour for encoding and decoding time values to
+  # and from raw bytes as specified by the BSON spec.
+  #
+  # @see http://bsonspec.org/#/specification
+  module LogStashEvent
+
+    # An Event is an embedded document is type 0x03 in the BSON spec..
+    BSON_TYPE = 3.chr.force_encoding(BINARY).freeze
+
+    # Get the event as encoded BSON.
+    # @example Get the hash as encoded BSON.
+    #   Event.new("field" => "value").to_bson
+    # @return [ String ] The encoded string.
+    # @see http://bsonspec.org/#/specification
+     def to_bson(buffer = ByteBuffer.new)
+      position = buffer.length
+      buffer.put_int32(0)
+      to_hash.each do |field, value|
+        buffer.put_byte(value.bson_type)
+        buffer.put_cstring(field.to_bson_key)
+        value.to_bson(buffer)
+      end
+      buffer.put_byte(NULL_BYTE)
+      buffer.replace_int32(position, buffer.length - position)
+    end
+
+    # Converts the event to a normalized value in a BSON document.
+    # @example Convert the event to a normalized value.
+    #   event.to_bson_normalized_value
+    # @return [ BSON::Document ] The normalized event.
+    def to_bson_normalized_value
+      Document.new(self)
+    end
+
+    module ClassMethods
+      # Deserialize the Event from BSON.
+      # @param [ ByteBuffer ] buffer The byte buffer.
+      # @return [ Event ] The decoded bson document.
+      # @see http://bsonspec.org/#/specification
+      def from_bson(buffer)
+        hash = Hash.new
+        buffer.get_int32 # Throw away the size.
+        while (type = buffer.get_byte) != NULL_BYTE
+          field = buffer.get_cstring
+          hash.store(field, BSON::Registry.get(type).from_bson(buffer))
+        end
+        new(hash)
+      end
+    end
+
+    # Register this type when the module is loaded.
+    Registry.register(BSON_TYPE, ::LogStash::Event)
+  end
+
+  # Enrich the core LogStash::Event class with this module.
+  ::LogStash::Event.send(:include, ::LogStashEvent)
+  ::LogStash::Event.send(:extend, ::LogStashEvent::ClassMethods)
+end

data/lib/logstash/outputs/bson/logstash_timestamp.rb

@@ -0,0 +1,50 @@
+# Copyright (C) 2009-2014 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Modified 2015 Elastic
+
+module BSON
+
+  # Injects behaviour for encoding and decoding time values to
+  # and from raw bytes as specified by the BSON spec.
+  #
+  # @see http://bsonspec.org/#/specification
+  module LogStashTimestamp
+
+    # A time is type 0x09 in the BSON spec.
+    BSON_TYPE = 9.chr.force_encoding(BINARY).freeze
+
+    def to_bson(encoded = ''.force_encoding(BINARY))
+      time.to_bson(encoded)
+    end
+
+    module ClassMethods
+      # Deserialize UTC time from BSON.
+      # @param [ BSON ] bson encoded time.
+      # @return [ ::LogStash::Timestamp ] The decoded UTC time as a ::LogStash::Timestamp.
+      # @see http://bsonspec.org/#/specification
+      def from_bson(bson)
+        seconds, fragment = BSON::Int64.from_bson(bson).divmod(1000)
+        new(::Time.at(seconds, fragment * 1000).utc)
+      end
+    end
+
+    # Register this type when the module is loaded.
+    Registry.register(BSON_TYPE, ::LogStash::Timestamp)
+  end
+
+  # Enrich the core LogStash::Timestamp class with this module.
+  ::LogStash::Timestamp.send(:include, LogStashTimestamp)
+  ::LogStash::Timestamp.send(:extend, LogStashTimestamp::ClassMethods)
+end

data/lib/logstash/outputs/mongodb.rb

@@ -2,6 +2,9 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "mongo"
+require_relative "bson/big_decimal"
+require_relative "bson/logstash_timestamp"
+# require_relative "bson/logstash_event"
 
 class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
 
@@ -38,20 +41,17 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
     @db = conn.use(@database)
   end # def register
 
-  public
   def receive(event)
-    
-
     begin
-      if @isodate
-        # the mongodb driver wants time values as a ruby Time object.
-        # set the @timestamp value of the document to a ruby Time object, then.
-        document = event.to_hash
-      else
-        document = event.to_hash.merge("@timestamp" => event["@timestamp"].to_json)
+      # Our timestamp object now has a to_bson method, using it here
+      # {}.merge(other) so we don't taint the event hash innards
+      document = {}.merge(event.to_hash)
+      if !@isodate
+        # not using timestamp.to_bson
+        document["@timestamp"] = event["@timestamp"].to_json
       end
       if @generateId
-        document['_id'] = BSON::ObjectId.new(nil, event["@timestamp"])
+        document["_id"] = BSON::ObjectId.new(nil, event["@timestamp"])
       end
       @db[event.sprintf(@collection)].insert_one(document)
     rescue => e

data/logstash-output-mongodb.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-mongodb'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Store events into MongoDB"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/bson/big_decimal_spec.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+require 'bigdecimal'
+require_relative "../spec_helper"
+require 'stringio'
+
+describe ::BigDecimal do
+  let(:a_number) { "4321.1234" }
+  let(:bson_number) { 4321.1234.to_bson }
+
+  subject { described_class.new(a_number) }
+
+  it "responds to to_bson" do
+    expect(subject).to respond_to(:to_bson)
+  end
+
+  it "to_bson returns a binary encoded number" do
+    expect(subject.to_bson).to eq(4321.1234.to_bson)
+  end
+
+  it "bson_type returns a binary encoded 1" do
+    expect(subject.bson_type).to eq(12.34.bson_type)
+  end
+
+  describe "class methods" do
+    it "builds a new BigDecimal from BSON" do
+      decoded = described_class.from_bson(StringIO.new(4321.1234.to_bson))
+      expect(decoded).to eql(BigDecimal.new(a_number))
+    end
+  end
+end

data/spec/bson/logstash_timestamp_spec.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require 'stringio'
+
+describe ::LogStash::Timestamp do
+  let(:time_array) { [1918,11,11,11,0,0, "+00:00"] }
+  let(:a_time) { Time.utc(*time_array) }
+  let(:bson_time) { Time.utc(*time_array).to_bson }
+
+  subject(:timestamp) { described_class.new(a_time) }
+
+  it "responds to to_bson" do
+    expect(subject).to respond_to(:to_bson)
+  end
+
+  it "to_bson returns a binary encoded timestamp" do
+    expect(timestamp.to_bson).to eq(bson_time)
+  end
+
+  it "bson_type returns a binary encoded 9" do
+    expect(subject.bson_type).to eq(a_time.bson_type)
+  end
+
+  describe "class methods" do
+    it "builds a new Timestamp from BSON" do
+      expected = ::LogStash::Timestamp.new(a_time)
+      decoded = ::LogStash::Timestamp.from_bson(StringIO.new(bson_time))
+      expect(decoded <=> expected).to eq(0)
+    end
+  end
+end

data/spec/integration/mongodb_spec.rb

@@ -6,16 +6,21 @@ describe LogStash::Outputs::Mongodb, :integration => true do
   let(:uri)        { 'mongodb://localhost:27017' }
   let(:database)   { 'logstash' }
   let(:collection) { 'logs' }
+  let(:uuid)       { SecureRandom.uuid }
 
   let(:config) do
-    { "uri" => uri, "database" => database, "collection" => collection }
+    { "uri" => uri, "database" => database,
+      "collection" => collection, "isodate" => true }
   end
 
   describe "#send" do
 
     subject { LogStash::Outputs::Mongodb.new(config) }
 
-    let(:properties) { { "message" => "This is a message!"} }
+    let(:properties) { { "message" => "This is a message!",
+                         "uuid" => uuid, "number" => BigDecimal.new("4321.1234"),
+                         "utf8" => "żółć", "int" => 42,
+                         "arry" => [42, "string", 4321.1234]} }
     let(:event)      { LogStash::Event.new(properties) }
 
     before(:each) do

data/spec/outputs/mongodb_spec.rb

@@ -21,7 +21,10 @@ describe LogStash::Outputs::Mongodb do
 
     subject { LogStash::Outputs::Mongodb.new(config) }
 
-    let(:properties) { { "message" => "This is a message!"} }
+    let(:properties) { { "message" => "This is a message!",
+                         "uuid" => SecureRandom.uuid,
+                         "number" => BigDecimal.new("4321.1234"),
+                         "utf8" => "żółć"} }
     let(:event)      { LogStash::Event.new(properties) }
     let(:connection) { double("connection") }
     let(:client)     { double("client") }

data/spec/spec_helper.rb

@@ -6,6 +6,6 @@ RSpec::Matchers.define :have_received do |event|
   match do |subject|
     client     = subject.instance_variable_get("@db")
     collection = subject.instance_variable_get("@collection")
-    client["#{collection}"].find("@timestamp" => event["@timestamp"].to_json).count > 0
+    client["#{collection}"].find("uuid" => event["uuid"]).count > 0
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-mongodb
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,8 +84,13 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- lib/logstash/outputs/bson/big_decimal.rb
+- lib/logstash/outputs/bson/logstash_event.rb
+- lib/logstash/outputs/bson/logstash_timestamp.rb
 - lib/logstash/outputs/mongodb.rb
 - logstash-output-mongodb.gemspec
+- spec/bson/big_decimal_spec.rb
+- spec/bson/logstash_timestamp_spec.rb
 - spec/integration/mongodb_spec.rb
 - spec/outputs/mongodb_spec.rb
 - spec/spec_helper.rb
@@ -111,11 +116,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: Store events into MongoDB
 test_files:
+- spec/bson/big_decimal_spec.rb
+- spec/bson/logstash_timestamp_spec.rb
 - spec/integration/mongodb_spec.rb
 - spec/outputs/mongodb_spec.rb
 - spec/spec_helper.rb