logstash-input-mongodb 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +30 -2
- data/lib/logstash/inputs/mongodb.rb +11 -5
- data/logstash-input-mongodb-0.1.3.gem +0 -0
- data/logstash-input-mongodb.gemspec +1 -1
- data/test/flattener_test.rb +9 -2
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 499524d8f98c5acf27c05c4b61a2fdac415fb673
|
4
|
+
data.tar.gz: 2a9bf33c6e3291af1c85aa3b006e49e65a090f23
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8e1f6eaf62f02701343976545138dce5876bb3ea0a4b46d4b4dc838b43e14aeadcfcbd0d872e2da2e4eb146534f878152bcfe5f3f6f298c335e6955f7ed9b3f1
|
7
|
+
data.tar.gz: 17a76549b8456d50baaa768ba7104c600cd3f18b81f65673f570e1248137a5746d43e28d87d8ce775522b0daf766ac780fca0715f3c39d8e1e7fc3cefa1470e9
|
data/README.md
CHANGED
@@ -6,6 +6,34 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
|
|
6
6
|
|
7
7
|
## Documentation
|
8
8
|
|
9
|
+
This is a logstash plugin for pulling data out of mongodb and processing with logstash. It will connect to the database specified in `uri`, use the `collection` attribute to find collections to pull documents from, start at the first collection it finds and pull the number of documents specified in `batch_size`, save it's progress in an sqlite database who's location is specified by `placeholder_db_dir` and `placeholder_db_name` and repeat. It will continue this until it no longer finds documents newer than ones that it has processed, sleep for a moment, then continue to loop over the collections.
|
10
|
+
|
11
|
+
This was designed for parsing logs that were written into mongodb. This means that it may not re-parse db entries that were changed and already parsed.
|
12
|
+
|
13
|
+
|
14
|
+
### Installation
|
15
|
+
|
16
|
+
+ Logstash installed from ZIP | TGZ
|
17
|
+
+ bin/plugin install /path/to/logstash-input-mongodb-0.1.3.gem
|
18
|
+
|
19
|
+
+ Logstash from GIT
|
20
|
+
+ git clone https://github.com/elastic/logstash.git
|
21
|
+
+ cd logstash
|
22
|
+
+ (ensure that the correct jruby is installed for the version of logstash you are installing)
|
23
|
+
+ rake test:install-core
|
24
|
+
+ bin/plugin install /path/to/logstash-input-mongodb-0.1.3.gem
|
25
|
+
+ bin/plugin install --development
|
26
|
+
|
27
|
+
### Configuration Options
|
28
|
+
|
29
|
+
uri: A MongoDB URI for your database or cluster (check the MongoDB documentation for further info on this) [No Default, Required]
|
30
|
+
placeholder_db_dir: Path where the place holder database will be stored locally to disk [No Default, Required]
|
31
|
+
This gets created by the plugin so the directory needs to be writeable by the user that logstash is running as
|
32
|
+
placeholder_db_name: Name of the database file that will be created [Default: logstash_sqlite.db]
|
33
|
+
collection: A regex that will be used to find desired collecitons. [No Default, Required]
|
34
|
+
batch_size: Size of the batch of mongo documents to pull at a time [Default: 30]
|
35
|
+
|
36
|
+
|
9
37
|
### Configuration
|
10
38
|
|
11
39
|
Example
|
@@ -13,9 +41,9 @@ Example
|
|
13
41
|
input {
|
14
42
|
mongodb {
|
15
43
|
uri => 'mongodb://10.0.0.30/my-logs?ssl=true'
|
16
|
-
|
44
|
+
placeholder_db_dir => '/opt/logstash-mongodb/'
|
45
|
+
placeholder_db_name => 'logstash_sqlite.db'
|
17
46
|
collection => 'events_'
|
18
|
-
unpack_mongo_id => true
|
19
47
|
batch_size => 5000
|
20
48
|
}
|
21
49
|
}
|
@@ -18,8 +18,11 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
|
|
18
18
|
# Example URI: mongodb://mydb.host:27017/mydbname?ssl=true
|
19
19
|
config :uri, :validate => :string, :required => true
|
20
20
|
|
21
|
-
# The
|
22
|
-
config :
|
21
|
+
# The directory that will contain the sqlite database file.
|
22
|
+
config :placeholder_db_dir, :validate => :string, :required => true
|
23
|
+
|
24
|
+
# The name of the sqlite databse file
|
25
|
+
config :placeholder_db_name, :validate => :string, :default => "logstash_sqlite.db"
|
23
26
|
|
24
27
|
# Any table to exclude by name
|
25
28
|
config :exclude_tables, :validate => :array, :default => []
|
@@ -152,6 +155,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
|
|
152
155
|
def register
|
153
156
|
require "jdbc/sqlite3"
|
154
157
|
require "sequel"
|
158
|
+
placeholder_db_path = File.join(@placeholder_db_dir, @placeholder_db_name)
|
155
159
|
mongo_uri = Mongo::URI.new(@uri)
|
156
160
|
hosts_array = mongo_uri.servers
|
157
161
|
db_name = mongo_uri.database
|
@@ -168,10 +172,10 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
|
|
168
172
|
end
|
169
173
|
|
170
174
|
@host = Socket.gethostname
|
171
|
-
@logger.info("Registering MongoDB input"
|
175
|
+
@logger.info("Registering MongoDB input")
|
172
176
|
|
173
177
|
@mongodb = conn.database
|
174
|
-
@sqlitedb = Sequel.connect("jdbc:sqlite:#{
|
178
|
+
@sqlitedb = Sequel.connect("jdbc:sqlite:#{placeholder_db_path}")
|
175
179
|
|
176
180
|
# Should check to see if there are new matching tables at a predefined interval or on some trigger
|
177
181
|
@collection_data = update_watched_collections(@mongodb, @collection, @sqlitedb)
|
@@ -223,7 +227,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
|
|
223
227
|
sleeptime = sleep_min
|
224
228
|
|
225
229
|
begin
|
226
|
-
@logger.debug("Tailing MongoDB"
|
230
|
+
@logger.debug("Tailing MongoDB")
|
227
231
|
@logger.debug("Collection data is: #{@collection_data}")
|
228
232
|
loop do
|
229
233
|
@collection_data.each do |index, collection|
|
@@ -281,6 +285,8 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
|
|
281
285
|
event[k.to_s] = v.to_f
|
282
286
|
elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
|
283
287
|
event[k.to_s] = v.to_i
|
288
|
+
else
|
289
|
+
event[k.to_s] = v
|
284
290
|
end
|
285
291
|
else
|
286
292
|
event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-mongodb'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.2.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "This takes entries from mongodb as an input to logstash."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
data/test/flattener_test.rb
CHANGED
@@ -6,13 +6,20 @@ require 'bson'
|
|
6
6
|
class TestThis < MiniTest::Test
|
7
7
|
def test_that_this_works
|
8
8
|
input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
|
9
|
+
input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
|
10
|
+
input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
|
11
|
+
input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
|
12
|
+
|
9
13
|
expected = "your mom"
|
10
|
-
assert_equal flatten(
|
14
|
+
assert_equal flatten(input4), expected
|
11
15
|
end
|
12
16
|
def test_the_flat_parser
|
13
17
|
input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
|
18
|
+
input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
|
19
|
+
input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
|
20
|
+
input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
|
14
21
|
expected = "your mom"
|
15
|
-
assert_equal flat_doc(flatten(
|
22
|
+
assert_equal flat_doc(flatten(input4)), expected
|
16
23
|
end
|
17
24
|
end
|
18
25
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-mongodb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Philip Hutchins
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
@@ -123,6 +123,7 @@ files:
|
|
123
123
|
- README.md
|
124
124
|
- Rakefile
|
125
125
|
- lib/logstash/inputs/mongodb.rb
|
126
|
+
- logstash-input-mongodb-0.1.3.gem
|
126
127
|
- logstash-input-mongodb.gemspec
|
127
128
|
- spec/inputs/example_spec.rb
|
128
129
|
- test/flattener_test.rb
|