logstash-output-mongodb 0.1.4 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/NOTICE.TXT +5 -0
- data/README.md +5 -5
- data/lib/logstash/outputs/mongodb.rb +8 -17
- data/logstash-output-mongodb.gemspec +3 -3
- data/spec/integration/mongodb_spec.rb +30 -0
- data/spec/outputs/mongodb_spec.rb +44 -1
- data/spec/spec_helper.rb +11 -0
- metadata +35 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb9ed79815c1d301d6cc48cb73c4596968a6fe40
|
4
|
+
data.tar.gz: 585099e5e7e29e254f84d6545173ebbbd06d71b6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3a44cf5562ece64bf632ce109e6e637537da9cf9b439c1e705879a2afadd6b2c672126f2cc5564cea812edcd679500a5bea47001837515e01b1a3f3eaff7fdca
|
7
|
+
data.tar.gz: 9730532c09f17fecb9c1b13edf495e12c7adc6fc3035039ea2faaa6f3112f58d2e557dd3a731a38b3d8b09200ee5a554d6fcf171ff3a47835436d6bf1ee97515
|
data/CHANGELOG.md
ADDED
data/NOTICE.TXT
ADDED
data/README.md
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
This is a plugin for [Logstash](https://github.com/
|
3
|
+
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
4
|
|
5
5
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
6
6
|
|
7
7
|
## Documentation
|
8
8
|
|
9
|
-
Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.
|
9
|
+
Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
|
10
10
|
|
11
11
|
- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
|
12
|
-
- For more asciidoc formatting tips, see the excellent reference here https://github.com/
|
12
|
+
- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
|
13
13
|
|
14
14
|
## Need Help?
|
15
15
|
|
16
|
-
Need help? Try #logstash on freenode IRC or the logstash
|
16
|
+
Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
|
17
17
|
|
18
18
|
## Developing
|
19
19
|
|
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
83
83
|
|
84
84
|
It is more important to the community that you are able to contribute.
|
85
85
|
|
86
|
-
For more information about contributing, see the [CONTRIBUTING](https://github.com/
|
86
|
+
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/outputs/base"
|
3
3
|
require "logstash/namespace"
|
4
|
-
|
4
|
+
require "mongo"
|
5
5
|
|
6
6
|
class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
|
7
7
|
|
@@ -10,10 +10,10 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
|
|
10
10
|
# a MongoDB URI to connect to
|
11
11
|
# See http://docs.mongodb.org/manual/reference/connection-string/
|
12
12
|
config :uri, :validate => :string, :required => true
|
13
|
-
|
13
|
+
|
14
14
|
# The database to use
|
15
15
|
config :database, :validate => :string, :required => true
|
16
|
-
|
16
|
+
|
17
17
|
# The collection to use. This value can use `%{foo}` values to dynamically
|
18
18
|
# select a collection based on data in the event.
|
19
19
|
config :collection, :validate => :string, :required => true
|
@@ -33,18 +33,9 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
|
|
33
33
|
|
34
34
|
public
|
35
35
|
def register
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
if uriParsed.auths.length > 0
|
40
|
-
uriParsed.auths.each do |auth|
|
41
|
-
if !auth['db_name'].nil?
|
42
|
-
conn.add_auth(auth['db_name'], auth['username'], auth['password'], nil)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
conn.apply_saved_authentication()
|
46
|
-
end
|
47
|
-
@db = conn.db(@database)
|
36
|
+
Mongo::Logger.logger = @logger
|
37
|
+
conn = Mongo::Client.new(@uri)
|
38
|
+
@db = conn.use(@database)
|
48
39
|
end # def register
|
49
40
|
|
50
41
|
public
|
@@ -62,11 +53,11 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
|
|
62
53
|
if @generateId
|
63
54
|
document['_id'] = BSON::ObjectId.new(nil, event["@timestamp"])
|
64
55
|
end
|
65
|
-
@db
|
56
|
+
@db[event.sprintf(@collection)].insert_one(document)
|
66
57
|
rescue => e
|
67
58
|
@logger.warn("Failed to send event to MongoDB", :event => event, :exception => e,
|
68
59
|
:backtrace => e.backtrace)
|
69
|
-
if e.
|
60
|
+
if e.message =~ /^E11000/
|
70
61
|
# On a duplicate key error, skip the insert.
|
71
62
|
# We could check if the duplicate key err is the _id key
|
72
63
|
# and generate a new primary key.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-mongodb'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '1.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Store events into MongoDB"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -21,8 +21,8 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
|
24
|
-
|
25
|
-
s.add_runtime_dependency 'mongo'
|
24
|
+
s.add_runtime_dependency 'logstash-codec-plain'
|
25
|
+
s.add_runtime_dependency 'mongo', '~> 2.0.6'
|
26
26
|
|
27
27
|
s.add_development_dependency 'logstash-devutils'
|
28
28
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative "../spec_helper"
|
3
|
+
|
4
|
+
describe LogStash::Outputs::Mongodb, :integration => true do
|
5
|
+
|
6
|
+
let(:uri) { 'mongodb://localhost:27017' }
|
7
|
+
let(:database) { 'logstash' }
|
8
|
+
let(:collection) { 'logs' }
|
9
|
+
|
10
|
+
let(:config) do
|
11
|
+
{ "uri" => uri, "database" => database, "collection" => collection }
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "#send" do
|
15
|
+
|
16
|
+
subject { LogStash::Outputs::Mongodb.new(config) }
|
17
|
+
|
18
|
+
let(:properties) { { "message" => "This is a message!"} }
|
19
|
+
let(:event) { LogStash::Event.new(properties) }
|
20
|
+
|
21
|
+
before(:each) do
|
22
|
+
subject.register
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should send the event to the database" do
|
26
|
+
subject.receive(event)
|
27
|
+
expect(subject).to have_received(event)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -1 +1,44 @@
|
|
1
|
-
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative "../spec_helper"
|
3
|
+
require "logstash/plugin"
|
4
|
+
|
5
|
+
describe LogStash::Outputs::Mongodb do
|
6
|
+
|
7
|
+
let(:uri) { 'mongodb://localhost:27017' }
|
8
|
+
let(:database) { 'logstash' }
|
9
|
+
let(:collection) { 'logs' }
|
10
|
+
|
11
|
+
let(:config) do
|
12
|
+
{ "uri" => uri, "database" => database, "collection" => collection }
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should register" do
|
16
|
+
plugin = LogStash::Plugin.lookup("output", "mongodb").new(config)
|
17
|
+
expect {plugin.register}.to_not raise_error
|
18
|
+
end
|
19
|
+
|
20
|
+
describe "#send" do
|
21
|
+
|
22
|
+
subject { LogStash::Outputs::Mongodb.new(config) }
|
23
|
+
|
24
|
+
let(:properties) { { "message" => "This is a message!"} }
|
25
|
+
let(:event) { LogStash::Event.new(properties) }
|
26
|
+
let(:connection) { double("connection") }
|
27
|
+
let(:client) { double("client") }
|
28
|
+
let(:collection) { double("collection") }
|
29
|
+
|
30
|
+
before(:each) do
|
31
|
+
allow(Mongo::Client).to receive(:new).and_return(connection)
|
32
|
+
allow(connection).to receive(:use).and_return(client)
|
33
|
+
allow(client).to receive(:[]).and_return(collection)
|
34
|
+
allow(collection).to receive(:insert_one)
|
35
|
+
subject.register
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should send the event to the database" do
|
39
|
+
expect(collection).to receive(:insert_one)
|
40
|
+
subject.receive(event)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/mongodb"
|
4
|
+
|
5
|
+
RSpec::Matchers.define :have_received do |event|
|
6
|
+
match do |subject|
|
7
|
+
client = subject.instance_variable_get("@db")
|
8
|
+
collection = subject.instance_variable_get("@collection")
|
9
|
+
client["#{collection}"].find("@timestamp" => event["@timestamp"].to_json).count > 0
|
10
|
+
end
|
11
|
+
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-mongodb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 2.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,34 +28,50 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 2.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-plain
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
45
|
+
prerelease: false
|
46
|
+
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
39
48
|
name: mongo
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 2.0.6
|
54
|
+
requirement: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - ~>
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: 2.0.6
|
40
59
|
prerelease: false
|
41
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-devutils
|
42
63
|
version_requirements: !ruby/object:Gem::Requirement
|
43
64
|
requirements:
|
44
65
|
- - '>='
|
45
66
|
- !ruby/object:Gem::Version
|
46
67
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
68
|
requirement: !ruby/object:Gem::Requirement
|
49
69
|
requirements:
|
50
70
|
- - '>='
|
51
71
|
- !ruby/object:Gem::Version
|
52
72
|
version: '0'
|
53
|
-
name: logstash-devutils
|
54
73
|
prerelease: false
|
55
74
|
type: :development
|
56
|
-
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
requirements:
|
58
|
-
- - '>='
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: '0'
|
61
75
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
62
76
|
email: info@elastic.co
|
63
77
|
executables: []
|
@@ -65,14 +79,18 @@ extensions: []
|
|
65
79
|
extra_rdoc_files: []
|
66
80
|
files:
|
67
81
|
- .gitignore
|
82
|
+
- CHANGELOG.md
|
68
83
|
- CONTRIBUTORS
|
69
84
|
- Gemfile
|
70
85
|
- LICENSE
|
86
|
+
- NOTICE.TXT
|
71
87
|
- README.md
|
72
88
|
- Rakefile
|
73
89
|
- lib/logstash/outputs/mongodb.rb
|
74
90
|
- logstash-output-mongodb.gemspec
|
91
|
+
- spec/integration/mongodb_spec.rb
|
75
92
|
- spec/outputs/mongodb_spec.rb
|
93
|
+
- spec/spec_helper.rb
|
76
94
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
77
95
|
licenses:
|
78
96
|
- Apache License (2.0)
|
@@ -95,9 +113,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
113
|
version: '0'
|
96
114
|
requirements: []
|
97
115
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
116
|
+
rubygems_version: 2.4.6
|
99
117
|
signing_key:
|
100
118
|
specification_version: 4
|
101
119
|
summary: Store events into MongoDB
|
102
120
|
test_files:
|
121
|
+
- spec/integration/mongodb_spec.rb
|
103
122
|
- spec/outputs/mongodb_spec.rb
|
123
|
+
- spec/spec_helper.rb
|