logstash-input-mongodb 0.3.0 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. checksums.yaml +5 -13
  2. data/DEVELOPER.md +2 -2
  3. data/Gemfile +2 -2
  4. data/LICENSE +13 -13
  5. data/README.md +64 -64
  6. data/lib/logstash/inputs/mongodb.rb +356 -356
  7. data/logstash-input-mongodb.gemspec +38 -28
  8. data/spec/inputs/example_spec.rb +0 -0
  9. metadata +25 -25
  10. data/Gemfile.lock +0 -106
  11. data/Rakefile +0 -1
  12. data/logstash-input-mongodb-0.1.3.gem +0 -0
  13. data/logstash-input-mongodb-0.2.0.gem +0 -0
  14. data/test/flattener_test.rb +0 -73
data/logstash-input-mongodb.gemspec CHANGED
@@ -1,28 +1,38 @@
1
- Gem::Specification.new do |s|
2
- s.name = 'logstash-input-mongodb'
3
- s.version = '0.3.0'
4
- s.licenses = ['Apache License (2.0)']
5
- s.summary = "This takes entries from mongodb as an input to logstash."
6
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
- s.authors = ["Philip Hutchins"]
8
- s.email = 'flipture@gmail.com'
9
- s.homepage = "http://www.phutchins.com"
10
- s.require_paths = ["lib"]
11
-
12
- # Files
13
- s.files = `git ls-files`.split($\)
14
- # Tests
15
- s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
-
17
- # Special flag to let us know this is actually a logstash plugin
18
- s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
19
-
20
- # Gem dependencies
21
- s.add_runtime_dependency 'logstash-core'
22
- s.add_runtime_dependency 'logstash-codec-plain'
23
- s.add_runtime_dependency 'stud'
24
- s.add_runtime_dependency 'jdbc-sqlite3', '3.8.10.1'
25
- s.add_runtime_dependency 'sequel'
26
- s.add_runtime_dependency 'mongo', '>= 2.0.0'
27
- s.add_development_dependency 'logstash-devutils'
28
- end
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-input-mongodb'
3
+ s.version = '0.3.1'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = "This takes entries from mongodb as an input to logstash."
6
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Philip Hutchins"]
8
+ s.email = 'flipture@gmail.com'
9
+ s.homepage = "http://www.phutchins.com"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir[
14
+ 'lib/**/*',
15
+ 'spec/**/*',
16
+ 'vendor/**/*',
17
+ '*.gemspec',
18
+ '*.md',
19
+ 'CONTRIBUTORS',
20
+ 'Gemfile',
21
+ 'LICENSE',
22
+ 'NOTICE.TXT'
23
+ ]
24
+ # Tests
25
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
26
+
27
+ # Special flag to let us know this is actually a logstash plugin
28
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
29
+
30
+ # Gem dependencies
31
+ s.add_runtime_dependency 'logstash-core', ">= 2.0.0.beta2", "< 3.0.0"
32
+ s.add_runtime_dependency 'logstash-codec-plain'
33
+ s.add_runtime_dependency 'stud'
34
+ s.add_runtime_dependency 'jdbc-sqlite3', '3.8.10.1'
35
+ s.add_runtime_dependency 'sequel'
36
+ s.add_runtime_dependency 'mongo', '>= 2.0.0'
37
+ s.add_development_dependency 'logstash-devutils'
38
+ end
data/spec/inputs/example_spec.rb CHANGED
File without changes
metadata CHANGED
@@ -1,55 +1,61 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-mongodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Philip Hutchins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-06 00:00:00.000000000 Z
11
+ date: 2015-11-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ! '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: 2.0.0.beta2
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: 3.0.0
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - ! '>='
27
+ - - ">="
25
28
  - !ruby/object:Gem::Version
26
- version: '0'
29
+ version: 2.0.0.beta2
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: 3.0.0
27
33
  - !ruby/object:Gem::Dependency
28
34
  name: logstash-codec-plain
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
- - - ! '>='
37
+ - - ">="
32
38
  - !ruby/object:Gem::Version
33
39
  version: '0'
34
40
  type: :runtime
35
41
  prerelease: false
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
- - - ! '>='
44
+ - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: '0'
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: stud
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
- - - ! '>='
51
+ - - ">="
46
52
  - !ruby/object:Gem::Version
47
53
  version: '0'
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
- - - ! '>='
58
+ - - ">="
53
59
  - !ruby/object:Gem::Version
54
60
  version: '0'
55
61
  - !ruby/object:Gem::Dependency
@@ -70,42 +76,42 @@ dependencies:
70
76
  name: sequel
71
77
  requirement: !ruby/object:Gem::Requirement
72
78
  requirements:
73
- - - ! '>='
79
+ - - ">="
74
80
  - !ruby/object:Gem::Version
75
81
  version: '0'
76
82
  type: :runtime
77
83
  prerelease: false
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
- - - ! '>='
86
+ - - ">="
81
87
  - !ruby/object:Gem::Version
82
88
  version: '0'
83
89
  - !ruby/object:Gem::Dependency
84
90
  name: mongo
85
91
  requirement: !ruby/object:Gem::Requirement
86
92
  requirements:
87
- - - ! '>='
93
+ - - ">="
88
94
  - !ruby/object:Gem::Version
89
95
  version: 2.0.0
90
96
  type: :runtime
91
97
  prerelease: false
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
- - - ! '>='
100
+ - - ">="
95
101
  - !ruby/object:Gem::Version
96
102
  version: 2.0.0
97
103
  - !ruby/object:Gem::Dependency
98
104
  name: logstash-devutils
99
105
  requirement: !ruby/object:Gem::Requirement
100
106
  requirements:
101
- - - ! '>='
107
+ - - ">="
102
108
  - !ruby/object:Gem::Version
103
109
  version: '0'
104
110
  type: :development
105
111
  prerelease: false
106
112
  version_requirements: !ruby/object:Gem::Requirement
107
113
  requirements:
108
- - - ! '>='
114
+ - - ">="
109
115
  - !ruby/object:Gem::Version
110
116
  version: '0'
111
117
  description: This gem is a logstash plugin required to be installed on top of the
@@ -118,16 +124,11 @@ extra_rdoc_files: []
118
124
  files:
119
125
  - DEVELOPER.md
120
126
  - Gemfile
121
- - Gemfile.lock
122
127
  - LICENSE
123
128
  - README.md
124
- - Rakefile
125
129
  - lib/logstash/inputs/mongodb.rb
126
- - logstash-input-mongodb-0.1.3.gem
127
- - logstash-input-mongodb-0.2.0.gem
128
130
  - logstash-input-mongodb.gemspec
129
131
  - spec/inputs/example_spec.rb
130
- - test/flattener_test.rb
131
132
  homepage: http://www.phutchins.com
132
133
  licenses:
133
134
  - Apache License (2.0)
@@ -140,20 +141,19 @@ require_paths:
140
141
  - lib
141
142
  required_ruby_version: !ruby/object:Gem::Requirement
142
143
  requirements:
143
- - - ! '>='
144
+ - - ">="
144
145
  - !ruby/object:Gem::Version
145
146
  version: '0'
146
147
  required_rubygems_version: !ruby/object:Gem::Requirement
147
148
  requirements:
148
- - - ! '>='
149
+ - - ">="
149
150
  - !ruby/object:Gem::Version
150
151
  version: '0'
151
152
  requirements: []
152
153
  rubyforge_project:
153
- rubygems_version: 2.4.8
154
+ rubygems_version: 2.4.5.1
154
155
  signing_key:
155
156
  specification_version: 4
156
157
  summary: This takes entries from mongodb as an input to logstash.
157
158
  test_files:
158
159
  - spec/inputs/example_spec.rb
159
- - test/flattener_test.rb
data/Gemfile.lock DELETED
@@ -1,106 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- logstash-input-mongodb (0.1.1)
5
- jdbc-sqlite3 (= 3.8.10.1)
6
- logstash-codec-plain (~> 0)
7
- logstash-core (>= 1.4.0, < 2.0.0)
8
- mongo (>= 1.12.2, < 2.0.0)
9
- sequel (~> 0)
10
- stud (~> 0)
11
-
12
- GEM
13
- remote: https://rubygems.org/
14
- specs:
15
- ParseTree (3.0.9)
16
- RubyInline (~> 3.9.0)
17
- sexp_processor (~> 3.2.0)
18
- RubyInline (3.9.0)
19
- ZenTest (~> 4.3)
20
- ZenTest (4.11.0)
21
- bson (1.12.3-java)
22
- cabin (0.7.1)
23
- clamp (0.6.5)
24
- coderay (1.1.0)
25
- diff-lcs (1.2.5)
26
- ffi (1.9.10-java)
27
- filesize (0.0.4)
28
- gem_publisher (1.5.0)
29
- gems (0.8.3)
30
- i18n (0.6.9)
31
- insist (1.0.0)
32
- jdbc-sqlite3 (3.8.10.1)
33
- jrjackson (0.2.9)
34
- kramdown (1.8.0)
35
- logstash-codec-plain (0.1.6)
36
- logstash-core (>= 1.4.0, < 2.0.0)
37
- logstash-core (1.5.3-java)
38
- cabin (~> 0.7.0)
39
- clamp (~> 0.6.5)
40
- filesize (= 0.0.4)
41
- gems (~> 0.8.3)
42
- i18n (= 0.6.9)
43
- jrjackson (~> 0.2.9)
44
- minitar (~> 0.5.4)
45
- pry (~> 0.10.1)
46
- stud (~> 0.0.19)
47
- thread_safe (~> 0.3.5)
48
- treetop (< 1.5.0)
49
- logstash-devutils (0.0.15-java)
50
- gem_publisher
51
- insist (= 1.0.0)
52
- kramdown
53
- minitar
54
- rake
55
- rspec (~> 3.1.0)
56
- stud (>= 0.0.20)
57
- metaid (1.0)
58
- method_source (0.8.2)
59
- minitar (0.5.4)
60
- mongo (1.12.3)
61
- bson (= 1.12.3)
62
- polyglot (0.3.5)
63
- pry (0.10.1-java)
64
- coderay (~> 1.1.0)
65
- method_source (~> 0.8.1)
66
- slop (~> 3.4)
67
- spoon (~> 0.0)
68
- rake (10.4.2)
69
- rspec (3.1.0)
70
- rspec-core (~> 3.1.0)
71
- rspec-expectations (~> 3.1.0)
72
- rspec-mocks (~> 3.1.0)
73
- rspec-core (3.1.7)
74
- rspec-support (~> 3.1.0)
75
- rspec-expectations (3.1.2)
76
- diff-lcs (>= 1.2.0, < 2.0)
77
- rspec-support (~> 3.1.0)
78
- rspec-mocks (3.1.3)
79
- rspec-support (~> 3.1.0)
80
- rspec-support (3.1.2)
81
- ruby2ruby (1.3.1)
82
- ruby_parser (~> 2.0)
83
- sexp_processor (~> 3.0)
84
- ruby_parser (2.3.1)
85
- sexp_processor (~> 3.0)
86
- sequel (0.5.0.2)
87
- ParseTree (>= 2.1.1)
88
- RubyInline (>= 3.6.6)
89
- metaid
90
- ruby2ruby
91
- sexp_processor (3.2.0)
92
- slop (3.6.0)
93
- spoon (0.0.4)
94
- ffi
95
- stud (0.0.20)
96
- thread_safe (0.3.5-java)
97
- treetop (1.4.15)
98
- polyglot
99
- polyglot (>= 0.3.1)
100
-
101
- PLATFORMS
102
- java
103
-
104
- DEPENDENCIES
105
- logstash-devutils (~> 0)
106
- logstash-input-mongodb!
data/Rakefile DELETED
@@ -1 +0,0 @@
1
- require "logstash/devutils/rake"
data/test/flattener_test.rb DELETED
@@ -1,73 +0,0 @@
1
- require 'minitest/autorun'
2
- require 'json'
3
- require 'mongo'
4
- require 'bson'
5
-
6
- class TestThis < MiniTest::Test
7
- def test_that_this_works
8
- input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
9
- input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
10
- input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
11
- input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
12
-
13
- expected = "your mom"
14
- assert_equal flatten(input4), expected
15
- end
16
- def test_the_flat_parser
17
- input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
18
- input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
19
- input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
20
- input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
21
- expected = "your mom"
22
- assert_equal flat_doc(flatten(input4)), expected
23
- end
24
- end
25
-
26
- def flatten(my_hash)
27
- new_hash = {}
28
- if my_hash.respond_to? :each
29
- my_hash.each do |k1,v1|
30
- if v1.is_a?(Hash)
31
- v1.each do |k2,v2|
32
- if v2.is_a?(Hash)
33
- # puts "Found a nested hash"
34
- result = flatten(v2)
35
- result.each do |k3,v3|
36
- new_hash[k1.to_s+"_"+k2.to_s+"_"+k3.to_s] = v3
37
- end
38
- # puts "result: "+result.to_s+" k2: "+k2.to_s+" v2: "+v2.to_s
39
- else
40
- new_hash[k1.to_s+"_"+k2.to_s] = v2
41
- end
42
- end
43
- else
44
- # puts "Key: "+k1.to_s+" is not a hash"
45
- new_hash[k1.to_s] = v1
46
- end
47
- end
48
- end
49
- return new_hash
50
- end
51
-
52
- def flat_doc(flat_doc)
53
- event = {}
54
- flat_doc.each do |k,v|
55
- # Check for an integer
56
- if v.is_a? Numeric
57
- event[k.to_s] = v
58
- elsif v.is_a? String
59
- if v == "NaN"
60
- event[k.to_s] = Float::NAN
61
- elsif /\A[-+]?\d+[.][\d]+\z/ == v
62
- event[k.to_s] = v.to_f
63
- elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
64
- event[k.to_s] = v.to_i
65
- end
66
- else
67
- event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
68
- if (k.to_s == "tags") && (v.is_a? Array)
69
- event['tags'] = v
70
- end
71
- end
72
- end
73
- end