logstash-input-mongodb 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. checksums.yaml +5 -13
  2. data/DEVELOPER.md +2 -2
  3. data/Gemfile +2 -2
  4. data/LICENSE +13 -13
  5. data/README.md +64 -64
  6. data/lib/logstash/inputs/mongodb.rb +356 -356
  7. data/logstash-input-mongodb.gemspec +38 -28
  8. data/spec/inputs/example_spec.rb +0 -0
  9. metadata +25 -25
  10. data/Gemfile.lock +0 -106
  11. data/Rakefile +0 -1
  12. data/logstash-input-mongodb-0.1.3.gem +0 -0
  13. data/logstash-input-mongodb-0.2.0.gem +0 -0
  14. data/test/flattener_test.rb +0 -73
data/logstash-input-mongodb.gemspec CHANGED
@@ -1,28 +1,38 @@
1
- Gem::Specification.new do |s|
2
- s.name = 'logstash-input-mongodb'
3
- s.version = '0.3.0'
4
- s.licenses = ['Apache License (2.0)']
5
- s.summary = "This takes entries from mongodb as an input to logstash."
6
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
- s.authors = ["Philip Hutchins"]
8
- s.email = 'flipture@gmail.com'
9
- s.homepage = "http://www.phutchins.com"
10
- s.require_paths = ["lib"]
11
-
12
- # Files
13
- s.files = `git ls-files`.split($\)
14
- # Tests
15
- s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
-
17
- # Special flag to let us know this is actually a logstash plugin
18
- s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
19
-
20
- # Gem dependencies
21
- s.add_runtime_dependency 'logstash-core'
22
- s.add_runtime_dependency 'logstash-codec-plain'
23
- s.add_runtime_dependency 'stud'
24
- s.add_runtime_dependency 'jdbc-sqlite3', '3.8.10.1'
25
- s.add_runtime_dependency 'sequel'
26
- s.add_runtime_dependency 'mongo', '>= 2.0.0'
27
- s.add_development_dependency 'logstash-devutils'
28
- end
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-input-mongodb'
3
+ s.version = '0.3.1'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = "This takes entries from mongodb as an input to logstash."
6
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Philip Hutchins"]
8
+ s.email = 'flipture@gmail.com'
9
+ s.homepage = "http://www.phutchins.com"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir[
14
+ 'lib/**/*',
15
+ 'spec/**/*',
16
+ 'vendor/**/*',
17
+ '*.gemspec',
18
+ '*.md',
19
+ 'CONTRIBUTORS',
20
+ 'Gemfile',
21
+ 'LICENSE',
22
+ 'NOTICE.TXT'
23
+ ]
24
+ # Tests
25
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
26
+
27
+ # Special flag to let us know this is actually a logstash plugin
28
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
29
+
30
+ # Gem dependencies
31
+ s.add_runtime_dependency 'logstash-core', ">= 2.0.0.beta2", "< 3.0.0"
32
+ s.add_runtime_dependency 'logstash-codec-plain'
33
+ s.add_runtime_dependency 'stud'
34
+ s.add_runtime_dependency 'jdbc-sqlite3', '3.8.10.1'
35
+ s.add_runtime_dependency 'sequel'
36
+ s.add_runtime_dependency 'mongo', '>= 2.0.0'
37
+ s.add_development_dependency 'logstash-devutils'
38
+ end
data/spec/inputs/example_spec.rb CHANGED
File without changes
metadata CHANGED
@@ -1,55 +1,61 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-mongodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Philip Hutchins
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-06 00:00:00.000000000 Z
11
+ date: 2015-11-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ! '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: 2.0.0.beta2
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: 3.0.0
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
- - - ! '>='
27
+ - - ">="
25
28
  - !ruby/object:Gem::Version
26
- version: '0'
29
+ version: 2.0.0.beta2
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: 3.0.0
27
33
  - !ruby/object:Gem::Dependency
28
34
  name: logstash-codec-plain
29
35
  requirement: !ruby/object:Gem::Requirement
30
36
  requirements:
31
- - - ! '>='
37
+ - - ">="
32
38
  - !ruby/object:Gem::Version
33
39
  version: '0'
34
40
  type: :runtime
35
41
  prerelease: false
36
42
  version_requirements: !ruby/object:Gem::Requirement
37
43
  requirements:
38
- - - ! '>='
44
+ - - ">="
39
45
  - !ruby/object:Gem::Version
40
46
  version: '0'
41
47
  - !ruby/object:Gem::Dependency
42
48
  name: stud
43
49
  requirement: !ruby/object:Gem::Requirement
44
50
  requirements:
45
- - - ! '>='
51
+ - - ">="
46
52
  - !ruby/object:Gem::Version
47
53
  version: '0'
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
51
57
  requirements:
52
- - - ! '>='
58
+ - - ">="
53
59
  - !ruby/object:Gem::Version
54
60
  version: '0'
55
61
  - !ruby/object:Gem::Dependency
@@ -70,42 +76,42 @@ dependencies:
70
76
  name: sequel
71
77
  requirement: !ruby/object:Gem::Requirement
72
78
  requirements:
73
- - - ! '>='
79
+ - - ">="
74
80
  - !ruby/object:Gem::Version
75
81
  version: '0'
76
82
  type: :runtime
77
83
  prerelease: false
78
84
  version_requirements: !ruby/object:Gem::Requirement
79
85
  requirements:
80
- - - ! '>='
86
+ - - ">="
81
87
  - !ruby/object:Gem::Version
82
88
  version: '0'
83
89
  - !ruby/object:Gem::Dependency
84
90
  name: mongo
85
91
  requirement: !ruby/object:Gem::Requirement
86
92
  requirements:
87
- - - ! '>='
93
+ - - ">="
88
94
  - !ruby/object:Gem::Version
89
95
  version: 2.0.0
90
96
  type: :runtime
91
97
  prerelease: false
92
98
  version_requirements: !ruby/object:Gem::Requirement
93
99
  requirements:
94
- - - ! '>='
100
+ - - ">="
95
101
  - !ruby/object:Gem::Version
96
102
  version: 2.0.0
97
103
  - !ruby/object:Gem::Dependency
98
104
  name: logstash-devutils
99
105
  requirement: !ruby/object:Gem::Requirement
100
106
  requirements:
101
- - - ! '>='
107
+ - - ">="
102
108
  - !ruby/object:Gem::Version
103
109
  version: '0'
104
110
  type: :development
105
111
  prerelease: false
106
112
  version_requirements: !ruby/object:Gem::Requirement
107
113
  requirements:
108
- - - ! '>='
114
+ - - ">="
109
115
  - !ruby/object:Gem::Version
110
116
  version: '0'
111
117
  description: This gem is a logstash plugin required to be installed on top of the
@@ -118,16 +124,11 @@ extra_rdoc_files: []
118
124
  files:
119
125
  - DEVELOPER.md
120
126
  - Gemfile
121
- - Gemfile.lock
122
127
  - LICENSE
123
128
  - README.md
124
- - Rakefile
125
129
  - lib/logstash/inputs/mongodb.rb
126
- - logstash-input-mongodb-0.1.3.gem
127
- - logstash-input-mongodb-0.2.0.gem
128
130
  - logstash-input-mongodb.gemspec
129
131
  - spec/inputs/example_spec.rb
130
- - test/flattener_test.rb
131
132
  homepage: http://www.phutchins.com
132
133
  licenses:
133
134
  - Apache License (2.0)
@@ -140,20 +141,19 @@ require_paths:
140
141
  - lib
141
142
  required_ruby_version: !ruby/object:Gem::Requirement
142
143
  requirements:
143
- - - ! '>='
144
+ - - ">="
144
145
  - !ruby/object:Gem::Version
145
146
  version: '0'
146
147
  required_rubygems_version: !ruby/object:Gem::Requirement
147
148
  requirements:
148
- - - ! '>='
149
+ - - ">="
149
150
  - !ruby/object:Gem::Version
150
151
  version: '0'
151
152
  requirements: []
152
153
  rubyforge_project:
153
- rubygems_version: 2.4.8
154
+ rubygems_version: 2.4.5.1
154
155
  signing_key:
155
156
  specification_version: 4
156
157
  summary: This takes entries from mongodb as an input to logstash.
157
158
  test_files:
158
159
  - spec/inputs/example_spec.rb
159
- - test/flattener_test.rb
data/Gemfile.lock DELETED
@@ -1,106 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- logstash-input-mongodb (0.1.1)
5
- jdbc-sqlite3 (= 3.8.10.1)
6
- logstash-codec-plain (~> 0)
7
- logstash-core (>= 1.4.0, < 2.0.0)
8
- mongo (>= 1.12.2, < 2.0.0)
9
- sequel (~> 0)
10
- stud (~> 0)
11
-
12
- GEM
13
- remote: https://rubygems.org/
14
- specs:
15
- ParseTree (3.0.9)
16
- RubyInline (~> 3.9.0)
17
- sexp_processor (~> 3.2.0)
18
- RubyInline (3.9.0)
19
- ZenTest (~> 4.3)
20
- ZenTest (4.11.0)
21
- bson (1.12.3-java)
22
- cabin (0.7.1)
23
- clamp (0.6.5)
24
- coderay (1.1.0)
25
- diff-lcs (1.2.5)
26
- ffi (1.9.10-java)
27
- filesize (0.0.4)
28
- gem_publisher (1.5.0)
29
- gems (0.8.3)
30
- i18n (0.6.9)
31
- insist (1.0.0)
32
- jdbc-sqlite3 (3.8.10.1)
33
- jrjackson (0.2.9)
34
- kramdown (1.8.0)
35
- logstash-codec-plain (0.1.6)
36
- logstash-core (>= 1.4.0, < 2.0.0)
37
- logstash-core (1.5.3-java)
38
- cabin (~> 0.7.0)
39
- clamp (~> 0.6.5)
40
- filesize (= 0.0.4)
41
- gems (~> 0.8.3)
42
- i18n (= 0.6.9)
43
- jrjackson (~> 0.2.9)
44
- minitar (~> 0.5.4)
45
- pry (~> 0.10.1)
46
- stud (~> 0.0.19)
47
- thread_safe (~> 0.3.5)
48
- treetop (< 1.5.0)
49
- logstash-devutils (0.0.15-java)
50
- gem_publisher
51
- insist (= 1.0.0)
52
- kramdown
53
- minitar
54
- rake
55
- rspec (~> 3.1.0)
56
- stud (>= 0.0.20)
57
- metaid (1.0)
58
- method_source (0.8.2)
59
- minitar (0.5.4)
60
- mongo (1.12.3)
61
- bson (= 1.12.3)
62
- polyglot (0.3.5)
63
- pry (0.10.1-java)
64
- coderay (~> 1.1.0)
65
- method_source (~> 0.8.1)
66
- slop (~> 3.4)
67
- spoon (~> 0.0)
68
- rake (10.4.2)
69
- rspec (3.1.0)
70
- rspec-core (~> 3.1.0)
71
- rspec-expectations (~> 3.1.0)
72
- rspec-mocks (~> 3.1.0)
73
- rspec-core (3.1.7)
74
- rspec-support (~> 3.1.0)
75
- rspec-expectations (3.1.2)
76
- diff-lcs (>= 1.2.0, < 2.0)
77
- rspec-support (~> 3.1.0)
78
- rspec-mocks (3.1.3)
79
- rspec-support (~> 3.1.0)
80
- rspec-support (3.1.2)
81
- ruby2ruby (1.3.1)
82
- ruby_parser (~> 2.0)
83
- sexp_processor (~> 3.0)
84
- ruby_parser (2.3.1)
85
- sexp_processor (~> 3.0)
86
- sequel (0.5.0.2)
87
- ParseTree (>= 2.1.1)
88
- RubyInline (>= 3.6.6)
89
- metaid
90
- ruby2ruby
91
- sexp_processor (3.2.0)
92
- slop (3.6.0)
93
- spoon (0.0.4)
94
- ffi
95
- stud (0.0.20)
96
- thread_safe (0.3.5-java)
97
- treetop (1.4.15)
98
- polyglot
99
- polyglot (>= 0.3.1)
100
-
101
- PLATFORMS
102
- java
103
-
104
- DEPENDENCIES
105
- logstash-devutils (~> 0)
106
- logstash-input-mongodb!
data/Rakefile DELETED
@@ -1 +0,0 @@
1
- require "logstash/devutils/rake"
data/test/flattener_test.rb DELETED
@@ -1,73 +0,0 @@
1
- require 'minitest/autorun'
2
- require 'json'
3
- require 'mongo'
4
- require 'bson'
5
-
6
- class TestThis < MiniTest::Test
7
- def test_that_this_works
8
- input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
9
- input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
10
- input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
11
- input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
12
-
13
- expected = "your mom"
14
- assert_equal flatten(input4), expected
15
- end
16
- def test_the_flat_parser
17
- input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
18
- input2 = {"_id"=>BSON::ObjectId('55cade9a5ef4000e2df9403e'), "tags"=>["info", "api"], "info"=>{"message"=>"api request completed: (%(ptype)s: %(pdata)s) %(facade)s.%(method)s init: %(id)sms facade: %(fd)sms total: %(total)sms params: %(params)s", "data"=>{"ptype"=>"no policy", "pdata"=>"none", "facade"=>"public", "method"=>"getInvoice", "id"=>0, "fd"=>5, "total"=>5, "params"=>"{\"id\":\"EwkJYnFj2bFE9e6xf3aq\"}", "raddr"=>"104.10.38.126"}}}
19
- input3 = {"_id"=>BSON::ObjectId('55ccdccf5d36005258e35972'), "tags"=>["info", "api"], "info"=>{"message"=>"API v1 request %(url)s", "data"=>{"path"=>"/rates/:currencyCode", "url"=>"/api/rates/eur", "query"=>{}, "raddr"=>"54.85.231.179"}}}
20
- input4 = {"_id"=>BSON::ObjectId('55cdfede6d2800b42c8b8c6f'), "tags"=>["http"], "info"=>{"method"=>"GET", "rtime"=>"17", "url"=>"/api", "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"81.171.50.83", "ver"=>"1.1", "ua"=>nil, "rlen"=>"214457", "rlocation"=>nil, "query"=>{}}}
21
- expected = "your mom"
22
- assert_equal flat_doc(flatten(input4)), expected
23
- end
24
- end
25
-
26
- def flatten(my_hash)
27
- new_hash = {}
28
- if my_hash.respond_to? :each
29
- my_hash.each do |k1,v1|
30
- if v1.is_a?(Hash)
31
- v1.each do |k2,v2|
32
- if v2.is_a?(Hash)
33
- # puts "Found a nested hash"
34
- result = flatten(v2)
35
- result.each do |k3,v3|
36
- new_hash[k1.to_s+"_"+k2.to_s+"_"+k3.to_s] = v3
37
- end
38
- # puts "result: "+result.to_s+" k2: "+k2.to_s+" v2: "+v2.to_s
39
- else
40
- new_hash[k1.to_s+"_"+k2.to_s] = v2
41
- end
42
- end
43
- else
44
- # puts "Key: "+k1.to_s+" is not a hash"
45
- new_hash[k1.to_s] = v1
46
- end
47
- end
48
- end
49
- return new_hash
50
- end
51
-
52
- def flat_doc(flat_doc)
53
- event = {}
54
- flat_doc.each do |k,v|
55
- # Check for an integer
56
- if v.is_a? Numeric
57
- event[k.to_s] = v
58
- elsif v.is_a? String
59
- if v == "NaN"
60
- event[k.to_s] = Float::NAN
61
- elsif /\A[-+]?\d+[.][\d]+\z/ == v
62
- event[k.to_s] = v.to_f
63
- elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
64
- event[k.to_s] = v.to_i
65
- end
66
- else
67
- event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
68
- if (k.to_s == "tags") && (v.is_a? Array)
69
- event['tags'] = v
70
- end
71
- end
72
- end
73
- end