logstash-input-mongodb 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 388af0f33d8e7e99f421dfb5f178b213b394fbb8
-  data.tar.gz: a5275546392e3c9c588f266e9d84f5c8124af36b
+  metadata.gz: 72644c97133f7c613f5b48174648a9ea546db026
+  data.tar.gz: 15fe10c80d301bc8dad93d934406a0965326fbf7
 SHA512:
-  metadata.gz: e61433c0e8d75fcb990c7353d5ba5edb3af08a5ae924696b40f8ad2b0e0e4a2edef4d71a204caf2b6d7dc439b85db24983edade05f1725c3fa713284974583d5
-  data.tar.gz: 45321ea60084617ea0e917632b072cd12eae3a6098de2712dc3a8eb4b15738cb164821e17daee8b92ec29c04c8f17c686767726338c6c131476055ca38245862
+  metadata.gz: 7ae54fbae2c1399e5c33063dccb1af2ca54f9a1ac2570f69ba09a5b3bcee0cd8498ccfbbb0ea8d78141cc5bb3e2f9b0c907fa37fccdc8d885bb6a64845216b32
+  data.tar.gz: ef0e8f79fe5324b55a6b7f9c443727c34c6d7869d90ad7c9b297c818b2900ced3cb73ec9016a326f42385a7593ac55b2b18114960fa31ebee26910db6275e6d3

data/DEVELOPER.md

@@ -1,2 +1,46 @@
 # logstash-input-example
 Example input plugin. This should help bootstrap your effort to write your own input plugin!
+
+## Build new gem
+
+Change version in logstash-input-mongodb.gemspec
+
+And build new gem:
+
+```bash
+$ gem build logstash-input-mongodb.gemspec
+```
+
+## Developing
+
+### Use rvm on Ubuntu
+
+1. Install rvm https://rvm.io/
+
+2. Install jruby
+
+```bash
+$ rvm install jruby 1.7.25
+```
+
+3. Use jruby from rvm
+
+```bash
+$ rvm alias create default jruby-1.7.25
+$ rvm use default
+$ /bin/bash --login
+$ gem install bundler
+$ bundle install
+```
+
+4. Run tests
+
+* Unit
+```bash
+$ ruby test/*
+```
+
+* Rspec (but it needs mongo on localhost, it would be nice to have it is not depended on local mongo)
+```bash
+$ ./bin/rspec
+```

data/lib/logstash/inputs/mongodb.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/timestamp"
@@ -31,6 +32,12 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
 
   config :since_table, :validate => :string, :default => "logstash_since"
 
+  # This allows you to select the column you would like compare the since info
+  config :since_column, :validate => :string, :default => "_id"
+
+  # This allows you to select the type of since info, like "id", "date"
+  config :since_type,   :validate => :string, :default => "id"
+
   # The collection to use. Is turned into a regex so 'events' will match 'events_20150227'
   # Example collection: events_20150227 or events_
   config :collection, :validate => :string, :required => true
@@ -85,9 +92,16 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     @logger.debug("init placeholder for #{since_table}_#{mongo_collection_name}")
     since = sqlitedb[SINCE_TABLE]
     mongo_collection = mongodb.collection(mongo_collection_name)
-    first_entry = mongo_collection.find({}).sort('_id' => 1).limit(1).first
-    first_entry_id = first_entry['_id'].to_s
+
+    first_entry = mongo_collection.find({}).sort(since_column => 1).limit(1).first
+    first_entry_id = ''
+    if since_type == 'id'
+      first_entry_id = first_entry[since_column].to_s
+    else
+      first_entry_id = first_entry[since_column].to_i
+    end
     since.insert(:table => "#{since_table}_#{mongo_collection_name}", :place => first_entry_id)
+    @logger.info("init placeholder for #{since_table}_#{mongo_collection_name}: #{first_entry}")
     return first_entry_id
   end
 
@@ -134,7 +148,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     collection = mongodb.collection(mongo_collection_name)
     # Need to make this sort by date in object id then get the first of the series
     # db.events_20150320.find().limit(1).sort({ts:1})
-    return collection.find({:_id => {:$gt => last_id_object}}).limit(batch_size)
+    return collection.find({:_id => {:$gte => last_id_object}}).limit(batch_size)
   end
 
   public
@@ -224,17 +238,26 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
           last_id = @collection_data[index][:last_id]
           #@logger.debug("last_id is #{last_id}", :index => index, :collection => collection_name)
           # get batch of events starting at the last_place if it is set
-          last_id_object = BSON::ObjectId(last_id)
+
+
+          last_id_object = last_id
+          if since_type == 'id'
+            last_id_object = BSON::ObjectId(last_id)
+          elsif since_type == 'time'
+            if last_id != ''
+              last_id_object = Time.at(last_id)
+            end
+          end
           cursor = get_cursor_for_collection(@mongodb, collection_name, last_id_object, batch_size)
           cursor.each do |doc|
             logdate = DateTime.parse(doc['_id'].generation_time.to_s)
             event = LogStash::Event.new("host" => @host)
             decorate(event)
-            event["logdate"] = logdate.iso8601
+            event.set("logdate",logdate.iso8601.force_encoding(Encoding::UTF_8))
             log_entry = doc.to_h.to_s
             log_entry['_id'] = log_entry['_id'].to_s
-            event["log_entry"] = log_entry
-            event["mongo_id"] = doc['_id'].to_s
+            event.set("log_entry",log_entry.force_encoding(Encoding::UTF_8))
+            event.set("mongo_id",doc['_id'].to_s)
             @logger.debug("mongo_id: "+doc['_id'].to_s)
             #@logger.debug("EVENT looks like: "+event.to_s)
             #@logger.debug("Sent message: "+doc.to_h.to_s)
@@ -245,8 +268,8 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
               doc_obj_bin = doc_hex_bytes.pack("C*").unpack("a4 a3 a2 a3")
               host_id = doc_obj_bin[1].unpack("S")
               process_id = doc_obj_bin[2].unpack("S")
-              event['host_id'] = host_id.first.to_i
-              event['process_id'] = process_id.first.to_i
+              event.set('host_id',host_id.first.to_i)
+              event.set('process_id',process_id.first.to_i)
             end
 
             if @parse_method == 'flatten'
@@ -265,21 +288,26 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
                 # Check for an integer
                 @logger.debug("key: #{k.to_s} value: #{v.to_s}")
                 if v.is_a? Numeric
-                  event[k.to_s] = v
+                  event.set(k.to_s,v)
+                elsif v.is_a? Time
+                  event.set(k.to_s,v.iso8601)
+
                 elsif v.is_a? String
                   if v == "NaN"
-                    event[k.to_s] = Float::NAN
+                    event.set(k.to_s, Float::NAN)
                   elsif /\A[-+]?\d+[.][\d]+\z/ == v
-                    event[k.to_s] = v.to_f
+                    event.set(k.to_s, v.to_f)
                   elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
-                    event[k.to_s] = v.to_i
+                    event.set(k.to_s, v.to_i)
                   else
-                    event[k.to_s] = v
+                    event.set(k.to_s, v)
                   end
                 else
-                  event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
+                  if k.to_s  == "_id" || k.to_s == "tags"
+                    event.set(k.to_s, v.to_s )
+                  end
                   if (k.to_s == "tags") && (v.is_a? Array)
-                    event['tags'] = v
+                    event.set('tags',v)
                   end
                 end
               end
@@ -292,24 +320,24 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
                       if (@dig_dig_fields.include? kk) && (vv.respond_to? :each)
                         vv.each do |kkk, vvv|
                           if /\A[-+]?\d+\z/ === vvv
-                            event["#{k}_#{kk}_#{kkk}"] = vvv.to_i
+                            event.set("#{k}_#{kk}_#{kkk}",vvv.to_i)
                           else
-                            event["#{k}_#{kk}_#{kkk}"] = vvv.to_s
+                            event.set("#{k}_#{kk}_#{kkk}", vvv.to_s)
                           end
                         end
                       else
                         if /\A[-+]?\d+\z/ === vv
-                          event["#{k}_#{kk}"] = vv.to_i
+                          event.set("#{k}_#{kk}", vv.to_i)
                         else
-                          event["#{k}_#{kk}"] = vv.to_s
+                          event.set("#{k}_#{kk}",vv.to_s)
                         end
                       end
                     end
                   else
                     if /\A[-+]?\d+\z/ === v
-                      event[k] = v.to_i
+                      event.set(k,v.to_i)
                     else
-                      event[k] = v.to_s
+                      event.set(k,v.to_s)
                     end
                   end
                 end
@@ -317,19 +345,27 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
             elsif @parse_method == 'simple'
               doc.each do |k, v|
                   if v.is_a? Numeric
-                    event[k] = v.abs
+                    event.set(k, v.abs)
                   elsif v.is_a? Array
-                    event[k] = v
+                    event.set(k, v)
                   elsif v == "NaN"
-                    event[k] = Float::NAN
+                    event.set(k, Float::NAN)
                   else
-                    event[k] = v.to_s
+                    event.set(k, v.to_s)
                   end
               end
             end
 
             queue << event
-            @collection_data[index][:last_id] = doc['_id'].to_s
+
+            since_id = doc[since_column]
+            if since_type == 'id'
+              since_id = doc[since_column].to_s
+            elsif since_type == 'time'
+              since_id = doc[since_column].to_i
+            end
+
+            @collection_data[index][:last_id] = since_id
           end
           # Store the last-seen doc in the database
           update_placeholder(@sqlitedb, since_table, collection_name, @collection_data[index][:last_id])

data/logstash-input-mongodb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-mongodb'
-  s.version = '0.3.3'
+  s.version = '0.4.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This takes entries from mongodb as an input to logstash."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -28,7 +28,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core', ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency 'logstash-core', ">= 5.0"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud'
   s.add_runtime_dependency 'jdbc-sqlite3', '3.8.10.1'

data/spec/inputs/mongodb_spec.rb

@@ -4,10 +4,12 @@ require "logstash/devutils/rspec/spec_helper"
 require "tempfile"
 require "stud/temporary"
 require "logstash/inputs/mongodb"
+require 'mongo'
+include Mongo
 
 FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
 
-describe LogStash::Inputs::Mongodb do
+describe LogStash::Inputs::MongoDB do
   before(:all) do
     @abort_on_exception = Thread.abort_on_exception
     Thread.abort_on_exception = true
@@ -18,24 +20,31 @@ describe LogStash::Inputs::Mongodb do
   end
 
   it_behaves_like "an interruptible input plugin" do
+    sqlite_db_file = Stud::Temporary.file
+    placeholder_db_dir = File.dirname sqlite_db_file
+    placeholder_db_name = File.basename sqlite_db_file
+    collection = 'logstash-input-mongodb_test'
     let(:config) do
       {
-        placeholder_db_dir => Stud::Temporary.pathname,
-        placeholder_db_bame => Stud::Temporary.file,
-        collection => 'logstash-input-mongodb_test'
+        "uri" => 'mongodb://localhost/logstash-input-mongodb_test',
+        "placeholder_db_dir" => "#{placeholder_db_dir}",
+        "collection" => "#{collection}"
       }
     end
   end
 
   it "should start at the beginning of a collection when no sincedb data exists" do
-    placeholder_db_dir = Stud::Temporary.pathname
-    placeholder_db_name = Stud::Temporary.pathname
+    mongo_uri = 'mongodb://localhost/logstash-input-mongodb_test'
+    sqlite_db_file = Stud::Temporary.file
+    placeholder_db_dir = File.dirname sqlite_db_file
+    placeholder_db_name = File.basename sqlite_db_file
+
     collection = 'logstash-input-mongodb_test'
 
     conf = <<-CONFIG
       input {
         mongodb {
-          uri => 'mongodb://localhost/logstash-input-mongodb_test',
+          uri => "#{mongo_uri}"
           placeholder_db_dir => "#{placeholder_db_dir}"
           placeholder_db_name => "#{placeholder_db_name}"
           collection => "#{collection}"
@@ -44,19 +53,23 @@ describe LogStash::Inputs::Mongodb do
     CONFIG
 
     # Create the test DB and populate it with some data
-    # add "first message"
-    # add "second message"
 
-    events = input(conf) do |pipeline, queue|
+    db = Mongo::Client.new(mongo_uri).database
+    coll = db.collection(collection)
+    coll.drop
+    coll = db.collection(collection)
+    coll.insert_one({:message => "first message"})
+    coll.insert_one({:message => "second message"})
+    coll.insert_one({:message => "third message"})
+    coll.insert_one({:message => "fourth message"})
 
-      events = []
+    events = input(conf) do |pipeline, queue|
 
       retries = 0
       while retries < 20
         # Add some new entries to the database
-        # add "third message"
-        # add "fourth message"
 
+        events = []
         if queue.size >= 4
           events = 4.times.collect { queue.pop }
           break
@@ -75,7 +88,8 @@ describe LogStash::Inputs::Mongodb do
     insist { events[3]["message"] } == "fourth message"
   end
 
-  it "should start where it left off in a collection when it has sincedb data" do
+  xit "should start where it left off in a collection when it has sincedb data" do
 
   end
+
 end

metadata

@@ -1,122 +1,114 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Philip Hutchins
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-18 00:00:00.000000000 Z
+date: 2017-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  type: :runtime
+        version: '5.0'
+  name: logstash-core
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: stud
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: jdbc-sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.8.10.1
-  type: :runtime
+  name: jdbc-sqlite3
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.8.10.1
 - !ruby/object:Gem::Dependency
-  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: sequel
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
-  type: :runtime
+  name: mongo
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the
-  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
-  a stand-alone program
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: flipture@gmail.com
 executables: []
 extensions: []
@@ -135,7 +127,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -150,9 +142,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.6
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
 specification_version: 4
 summary: This takes entries from mongodb as an input to logstash.
 test_files: