logstash-input-mongodb 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 82148f9ed66d81ea6da480e90ed78600c6a92e42
4
- data.tar.gz: 328b9ec0aa5f3de550f70d37b84063dbbf85fb15
3
+ metadata.gz: 499524d8f98c5acf27c05c4b61a2fdac415fb673
4
+ data.tar.gz: 2a9bf33c6e3291af1c85aa3b006e49e65a090f23
5
5
  SHA512:
6
- metadata.gz: a6d157dbd02e4c914a9c3e4caf02ae5bd281b92f233456c8fcb575b8fd01629235c606d4054cc357c060ff743773378f7f5b816da1943926430ee5a7c100c83d
7
- data.tar.gz: 00510814687f7a723260e436b9c5396d9148730ba81f324aee5d6cf859d43846fb9893a38d4fb1bfb6034df8322dd889cca7c77c171d4d2668964587e1162825
6
+ metadata.gz: 8e1f6eaf62f02701343976545138dce5876bb3ea0a4b46d4b4dc838b43e14aeadcfcbd0d872e2da2e4eb146534f878152bcfe5f3f6f298c335e6955f7ed9b3f1
7
+ data.tar.gz: 17a76549b8456d50baaa768ba7104c600cd3f18b81f65673f570e1248137a5746d43e28d87d8ce775522b0daf766ac780fca0715f3c39d8e1e7fc3cefa1470e9
data/README.md CHANGED
@@ -6,6 +6,34 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
6
6
 
7
7
  ## Documentation
8
8
 
9
+ This is a logstash plugin for pulling data out of mongodb and processing with logstash. It will connect to the database specified in `uri`, use the `collection` attribute to find collections to pull documents from, start at the first collection it finds and pull the number of documents specified in `batch_size`, save it's progress in an sqlite database who's location is specified by `placeholder_db_dir` and `placeholder_db_name` and repeat. It will continue this until it no longer finds documents newer than ones that it has processed, sleep for a moment, then continue to loop over the collections.
10
+
11
+ This was designed for parsing logs that were written into mongodb. This means that it may not re-parse db entries that were changed and already parsed.
12
+
13
+
14
+ ### Installation
15
+
16
+ + Logstash installed from ZIP | TGZ
17
+ + bin/plugin install /path/to/logstash-input-mongodb-0.1.3.gem
18
+
19
+ + Logstash from GIT
20
+ + git clone https://github.com/elastic/logstash.git
21
+ + cd logstash
22
+ + (ensure that the correct jruby is installed for the version of logstash you are installing)
23
+ + rake test:install-core
24
+ + bin/plugin install /path/to/logstash-input-mongodb-0.1.3.gem
25
+ + bin/plugin install --development
26
+
27
+ ### Configuration Options
28
+
29
+ uri: A MongoDB URI for your database or cluster (check the MongoDB documentation for further info on this) [No Default, Required]
30
+ placeholder_db_dir: Path where the place holder database will be stored locally to disk [No Default, Required]
31
+ This gets created by the plugin so the directory needs to be writeable by the user that logstash is running as
32
+ placeholder_db_name: Name of the database file that will be created [Default: logstash_sqlite.db]
33
+ collection: A regex that will be used to find desired collecitons. [No Default, Required]
34
+ batch_size: Size of the batch of mongo documents to pull at a time [Default: 30]
35
+
36
+
9
37
  ### Configuration
10
38
 
11
39
  Example
@@ -13,9 +41,9 @@ Example
13
41
  input {
14
42
  mongodb {
15
43
  uri => 'mongodb://10.0.0.30/my-logs?ssl=true'
16
- path => '/opt/logstash-mongodb/logstash_sqlite.db'
44
+ placeholder_db_dir => '/opt/logstash-mongodb/'
45
+ placeholder_db_name => 'logstash_sqlite.db'
17
46
  collection => 'events_'
18
- unpack_mongo_id => true
19
47
  batch_size => 5000
20
48
  }
21
49
  }
@@ -18,8 +18,11 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
18
18
  # Example URI: mongodb://mydb.host:27017/mydbname?ssl=true
19
19
  config :uri, :validate => :string, :required => true
20
20
 
21
- # The path to the sqlite database file.
22
- config :path, :validate => :string, :required => true
21
+ # The directory that will contain the sqlite database file.
22
+ config :placeholder_db_dir, :validate => :string, :required => true
23
+
24
+ # The name of the sqlite databse file
25
+ config :placeholder_db_name, :validate => :string, :default => "logstash_sqlite.db"
23
26
 
24
27
  # Any table to exclude by name
25
28
  config :exclude_tables, :validate => :array, :default => []
@@ -152,6 +155,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
152
155
  def register
153
156
  require "jdbc/sqlite3"
154
157
  require "sequel"
158
+ placeholder_db_path = File.join(@placeholder_db_dir, @placeholder_db_name)
155
159
  mongo_uri = Mongo::URI.new(@uri)
156
160
  hosts_array = mongo_uri.servers
157
161
  db_name = mongo_uri.database
@@ -168,10 +172,10 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
168
172
  end
169
173
 
170
174
  @host = Socket.gethostname
171
- @logger.info("Registering MongoDB input", :database => @path)
175
+ @logger.info("Registering MongoDB input")
172
176
 
173
177
  @mongodb = conn.database
174
- @sqlitedb = Sequel.connect("jdbc:sqlite:#{@path}")
178
+ @sqlitedb = Sequel.connect("jdbc:sqlite:#{placeholder_db_path}")
175
179
 
176
180
  # Should check to see if there are new matching tables at a predefined interval or on some trigger
177
181
  @collection_data = update_watched_collections(@mongodb, @collection, @sqlitedb)
@@ -223,7 +227,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
223
227
  sleeptime = sleep_min
224
228
 
225
229
  begin
226
- @logger.debug("Tailing MongoDB", :path => @path)
230
+ @logger.debug("Tailing MongoDB")
227
231
  @logger.debug("Collection data is: #{@collection_data}")
228
232
  loop do
229
233
  @collection_data.each do |index, collection|
@@ -281,6 +285,8 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
281
285
  event[k.to_s] = v.to_f
282
286
  elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
283
287
  event[k.to_s] = v.to_i
288
+ else
289
+ event[k.to_s] = v
284
290
  end
285
291
  else
286
292
  event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-input-mongodb'
3
- s.version = '0.1.3'
3
+ s.version = '0.2.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "This takes entries from mongodb as an input to logstash."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -6,13 +6,20 @@ require 'bson'
6
6
  class TestThis < MiniTest::Test
7
7
  def test_that_this_works
8
8
  input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
9
+ input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
10
+ input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
11
+ input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
12
+
9
13
  expected = "your mom"
10
- assert_equal flatten(input), expected
14
+ assert_equal flatten(input4), expected
11
15
  end
12
16
  def test_the_flat_parser
13
17
  input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
18
+ input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
19
+ input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
20
+ input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
14
21
  expected = "your mom"
15
- assert_equal flat_doc(flatten(input)), expected
22
+ assert_equal flat_doc(flatten(input4)), expected
16
23
  end
17
24
  end
18
25
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-mongodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Philip Hutchins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-13 00:00:00.000000000 Z
11
+ date: 2015-08-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
@@ -123,6 +123,7 @@ files:
123
123
  - README.md
124
124
  - Rakefile
125
125
  - lib/logstash/inputs/mongodb.rb
126
+ - logstash-input-mongodb-0.1.3.gem
126
127
  - logstash-input-mongodb.gemspec
127
128
  - spec/inputs/example_spec.rb
128
129
  - test/flattener_test.rb