logstash-input-mongodb 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82148f9ed66d81ea6da480e90ed78600c6a92e42
-  data.tar.gz: 328b9ec0aa5f3de550f70d37b84063dbbf85fb15
+  metadata.gz: 499524d8f98c5acf27c05c4b61a2fdac415fb673
+  data.tar.gz: 2a9bf33c6e3291af1c85aa3b006e49e65a090f23
 SHA512:
-  metadata.gz: a6d157dbd02e4c914a9c3e4caf02ae5bd281b92f233456c8fcb575b8fd01629235c606d4054cc357c060ff743773378f7f5b816da1943926430ee5a7c100c83d
-  data.tar.gz: 00510814687f7a723260e436b9c5396d9148730ba81f324aee5d6cf859d43846fb9893a38d4fb1bfb6034df8322dd889cca7c77c171d4d2668964587e1162825
+  metadata.gz: 8e1f6eaf62f02701343976545138dce5876bb3ea0a4b46d4b4dc838b43e14aeadcfcbd0d872e2da2e4eb146534f878152bcfe5f3f6f298c335e6955f7ed9b3f1
+  data.tar.gz: 17a76549b8456d50baaa768ba7104c600cd3f18b81f65673f570e1248137a5746d43e28d87d8ce775522b0daf766ac780fca0715f3c39d8e1e7fc3cefa1470e9

data/README.md

@@ -6,6 +6,34 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
 
 ## Documentation
 
+This is a logstash plugin for pulling data out of mongodb and processing with logstash. It will connect to the database specified in `uri`, use the `collection` attribute to find collections to pull documents from, start at the first collection it finds and pull the number of documents specified in `batch_size`, save it's progress in an sqlite database who's location is specified by `placeholder_db_dir` and `placeholder_db_name` and repeat. It will continue this until it no longer finds documents newer than ones that it has processed, sleep for a moment, then continue to loop over the collections.
+
+This was designed for parsing logs that were written into mongodb. This means that it may not re-parse db entries that were changed and already parsed.
+
+
+### Installation
+
++ Logstash installed from ZIP | TGZ
+  + bin/plugin install /path/to/logstash-input-mongodb-0.1.3.gem
+
++ Logstash from GIT
+  + git clone https://github.com/elastic/logstash.git
+  + cd logstash
+  + (ensure that the correct jruby is installed for the version of logstash you are installing)
+  + rake test:install-core
+  + bin/plugin install /path/to/logstash-input-mongodb-0.1.3.gem
+  + bin/plugin install --development
+
+### Configuration Options
+
+  uri: A MongoDB URI for your database or cluster (check the MongoDB documentation for further info on this) [No Default, Required]
+  placeholder_db_dir: Path where the place holder database will be stored locally to disk [No Default, Required]
+    This gets created by the plugin so the directory needs to be writeable by the user that logstash is running as
+  placeholder_db_name: Name of the database file that will be created [Default: logstash_sqlite.db]
+  collection: A regex that will be used to find desired collecitons. [No Default, Required]
+  batch_size: Size of the batch of mongo documents to pull at a time [Default: 30]
+
+
 ### Configuration
 
 Example
@@ -13,9 +41,9 @@ Example
 input {
   mongodb {
     uri => 'mongodb://10.0.0.30/my-logs?ssl=true'
-    path => '/opt/logstash-mongodb/logstash_sqlite.db'
+    placeholder_db_dir => '/opt/logstash-mongodb/'
+    placeholder_db_name => 'logstash_sqlite.db'
     collection => 'events_'
-    unpack_mongo_id => true
     batch_size => 5000
   }
 }

data/lib/logstash/inputs/mongodb.rb

@@ -18,8 +18,11 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
   # Example URI: mongodb://mydb.host:27017/mydbname?ssl=true
   config :uri, :validate => :string, :required => true
 
-  # The path to the sqlite database file.
-  config :path, :validate => :string, :required => true
+  # The directory that will contain the sqlite database file.
+  config :placeholder_db_dir, :validate => :string, :required => true
+
+  # The name of the sqlite databse file
+  config :placeholder_db_name, :validate => :string, :default => "logstash_sqlite.db"
 
   # Any table to exclude by name
   config :exclude_tables, :validate => :array, :default => []
@@ -152,6 +155,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
   def register
     require "jdbc/sqlite3"
     require "sequel"
+    placeholder_db_path = File.join(@placeholder_db_dir, @placeholder_db_name)
     mongo_uri = Mongo::URI.new(@uri)
     hosts_array = mongo_uri.servers
     db_name = mongo_uri.database
@@ -168,10 +172,10 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     end
 
     @host = Socket.gethostname
-    @logger.info("Registering MongoDB input", :database => @path)
+    @logger.info("Registering MongoDB input")
 
     @mongodb = conn.database
-    @sqlitedb = Sequel.connect("jdbc:sqlite:#{@path}")
+    @sqlitedb = Sequel.connect("jdbc:sqlite:#{placeholder_db_path}")
 
     # Should check to see if there are new matching tables at a predefined interval or on some trigger
     @collection_data = update_watched_collections(@mongodb, @collection, @sqlitedb)
@@ -223,7 +227,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     sleeptime = sleep_min
 
     begin
-      @logger.debug("Tailing MongoDB", :path => @path)
+      @logger.debug("Tailing MongoDB")
       @logger.debug("Collection data is: #{@collection_data}")
       loop do
         @collection_data.each do |index, collection|
@@ -281,6 +285,8 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
                     event[k.to_s] = v.to_f
                   elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
                     event[k.to_s] = v.to_i
+                  else
+                    event[k.to_s] = v
                   end
                 else
                   event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"

data/logstash-input-mongodb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-mongodb'
-  s.version = '0.1.3'
+  s.version = '0.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This takes entries from mongodb as an input to logstash."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/test/flattener_test.rb

@@ -6,13 +6,20 @@ require 'bson'
 class TestThis < MiniTest::Test
   def test_that_this_works
     input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
+    input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
+    input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
+    input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
+
     expected = "your mom"
-    assert_equal flatten(input), expected
+    assert_equal flatten(input4), expected
   end
   def test_the_flat_parser
     input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
+    input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
+    input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
+    input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
     expected = "your mom"
-    assert_equal flat_doc(flatten(input)), expected
+    assert_equal flat_doc(flatten(input4)), expected
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Philip Hutchins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-13 00:00:00.000000000 Z
+date: 2015-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -123,6 +123,7 @@ files:
 - README.md
 - Rakefile
 - lib/logstash/inputs/mongodb.rb
+- logstash-input-mongodb-0.1.3.gem
 - logstash-input-mongodb.gemspec
 - spec/inputs/example_spec.rb
 - test/flattener_test.rb