logstash-output-mongodb 2.0.2 → 2.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -8
- data/CONTRIBUTORS +1 -0
- data/README.md +3 -0
- data/lib/logstash/outputs/bson/big_decimal.rb +66 -0
- data/lib/logstash/outputs/bson/logstash_event.rb +76 -0
- data/lib/logstash/outputs/bson/logstash_timestamp.rb +50 -0
- data/lib/logstash/outputs/mongodb.rb +10 -10
- data/logstash-output-mongodb.gemspec +1 -1
- data/spec/bson/big_decimal_spec.rb +30 -0
- data/spec/bson/logstash_timestamp_spec.rb +31 -0
- data/spec/integration/mongodb_spec.rb +7 -2
- data/spec/outputs/mongodb_spec.rb +4 -1
- data/spec/spec_helper.rb +1 -1
- metadata +10 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0e05acac3ccb551768dae1439fef3580a37b6f7a
|
4
|
+
data.tar.gz: 8d240b38a49fe08b48794b93ab4f30b9e8696cf9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 385ae9c6b958f15c1d040fa3cfd840347f77135e610134c99370ae183cb349c8aaa5f48ae8095499090e02787a83a072fcd83dfd9d6ba0ea3762d8769fc8d6a9
|
7
|
+
data.tar.gz: fb929fa63ab767c9441bbd753bfd7ef65bc41ebf46719b2163ab4d873d9afc4da007571fded4c4698660ad050645cef0b481f90061ade750e1483beb779cb142
|
data/CHANGELOG.md
CHANGED
@@ -1,13 +1,16 @@
|
|
1
|
+
## 2.0.3
|
2
|
+
- Patch Timestamp and BigDecimal with to_bson method and register with BSON.
|
3
|
+
|
1
4
|
## 2.0.0
|
2
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
5
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
6
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
7
|
- Dependency on logstash-core update to 2.0
|
5
8
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
9
|
+
## 1.0.0
|
10
|
+
- Fixes the plugin to be in the 2.0 series
|
11
|
+
- Add integration and unit test to the project
|
12
|
+
- Adapt the codebase to be 2.0 compatible
|
13
|
+
- Make the internal logger in mongo to report to LS logger
|
11
14
|
|
12
|
-
|
13
|
-
|
15
|
+
## 0.2.0
|
16
|
+
- Add basic registration test to the project
|
data/CONTRIBUTORS
CHANGED
@@ -14,6 +14,7 @@ Contributors:
|
|
14
14
|
* Pier-Hugues Pellerin (ph)
|
15
15
|
* Richard Pijnenburg (electrical)
|
16
16
|
* bitsofinfo (bitsofinfo)
|
17
|
+
* Guy Boertje (guyboertje)
|
17
18
|
|
18
19
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
19
20
|
Logstash, and you aren't on the list above and want to be, please let us know
|
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
[![Build
|
4
|
+
Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-mongodb-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-mongodb-unit/)
|
5
|
+
|
3
6
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
7
|
|
5
8
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# Copyright (C) 2009-2014 MongoDB Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
# Modified 2015 Elastic
|
16
|
+
|
17
|
+
module BSON
|
18
|
+
|
19
|
+
# Injects behaviour for encoding and decoding BigDecimal values
|
20
|
+
# to and from # raw bytes as specified by the BSON spec.
|
21
|
+
#
|
22
|
+
# @see http://bsonspec.org/#/specification
|
23
|
+
module BigDecimal
|
24
|
+
|
25
|
+
# A floating point is type 0x01 in the BSON spec.
|
26
|
+
BSON_TYPE = 1.chr.force_encoding(BINARY).freeze
|
27
|
+
|
28
|
+
# The pack directive is for 8 byte floating points.
|
29
|
+
PACK = "E".freeze
|
30
|
+
|
31
|
+
# Get the floating point as encoded BSON.
|
32
|
+
# @example Get the floating point as encoded BSON.
|
33
|
+
# 1.221311.to_bson
|
34
|
+
# @return [ String ] The encoded string.
|
35
|
+
# @see http://bsonspec.org/#/specification
|
36
|
+
def to_bson(encoded = ''.force_encoding(BINARY))
|
37
|
+
encoded << [ self ].pack(PACK)
|
38
|
+
end
|
39
|
+
|
40
|
+
module ClassMethods
|
41
|
+
|
42
|
+
# Deserialize an instance of a BigDecimal from a BSON double.
|
43
|
+
# @param [ BSON ] bson object from Mongo.
|
44
|
+
# @return [ BigDecimal ] The decoded BigDecimal.
|
45
|
+
# @see http://bsonspec.org/#/specification
|
46
|
+
def from_bson(bson)
|
47
|
+
from_bson_double(bson.read(8))
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def from_bson_double(double)
|
53
|
+
new(double.unpack(PACK).first.to_s)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Register this type when the module is loaded.
|
58
|
+
Registry.register(BSON_TYPE, ::BigDecimal)
|
59
|
+
end
|
60
|
+
|
61
|
+
# Enrich the core BigDecimal class with this module.
|
62
|
+
#
|
63
|
+
# @since 2.0.0
|
64
|
+
::BigDecimal.send(:include, BigDecimal)
|
65
|
+
::BigDecimal.send(:extend, BigDecimal::ClassMethods)
|
66
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
# Copyright (C) 2009-2014 MongoDB Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
# Modified 2015 Elastic
|
16
|
+
|
17
|
+
module BSON
|
18
|
+
|
19
|
+
# Injects behaviour for encoding and decoding time values to
|
20
|
+
# and from raw bytes as specified by the BSON spec.
|
21
|
+
#
|
22
|
+
# @see http://bsonspec.org/#/specification
|
23
|
+
module LogStashEvent
|
24
|
+
|
25
|
+
# An Event is an embedded document is type 0x03 in the BSON spec..
|
26
|
+
BSON_TYPE = 3.chr.force_encoding(BINARY).freeze
|
27
|
+
|
28
|
+
# Get the event as encoded BSON.
|
29
|
+
# @example Get the hash as encoded BSON.
|
30
|
+
# Event.new("field" => "value").to_bson
|
31
|
+
# @return [ String ] The encoded string.
|
32
|
+
# @see http://bsonspec.org/#/specification
|
33
|
+
def to_bson(buffer = ByteBuffer.new)
|
34
|
+
position = buffer.length
|
35
|
+
buffer.put_int32(0)
|
36
|
+
to_hash.each do |field, value|
|
37
|
+
buffer.put_byte(value.bson_type)
|
38
|
+
buffer.put_cstring(field.to_bson_key)
|
39
|
+
value.to_bson(buffer)
|
40
|
+
end
|
41
|
+
buffer.put_byte(NULL_BYTE)
|
42
|
+
buffer.replace_int32(position, buffer.length - position)
|
43
|
+
end
|
44
|
+
|
45
|
+
# Converts the event to a normalized value in a BSON document.
|
46
|
+
# @example Convert the event to a normalized value.
|
47
|
+
# event.to_bson_normalized_value
|
48
|
+
# @return [ BSON::Document ] The normalized event.
|
49
|
+
def to_bson_normalized_value
|
50
|
+
Document.new(self)
|
51
|
+
end
|
52
|
+
|
53
|
+
module ClassMethods
|
54
|
+
# Deserialize the Event from BSON.
|
55
|
+
# @param [ ByteBuffer ] buffer The byte buffer.
|
56
|
+
# @return [ Event ] The decoded bson document.
|
57
|
+
# @see http://bsonspec.org/#/specification
|
58
|
+
def from_bson(buffer)
|
59
|
+
hash = Hash.new
|
60
|
+
buffer.get_int32 # Throw away the size.
|
61
|
+
while (type = buffer.get_byte) != NULL_BYTE
|
62
|
+
field = buffer.get_cstring
|
63
|
+
hash.store(field, BSON::Registry.get(type).from_bson(buffer))
|
64
|
+
end
|
65
|
+
new(hash)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Register this type when the module is loaded.
|
70
|
+
Registry.register(BSON_TYPE, ::LogStash::Event)
|
71
|
+
end
|
72
|
+
|
73
|
+
# Enrich the core LogStash::Event class with this module.
|
74
|
+
::LogStash::Event.send(:include, ::LogStashEvent)
|
75
|
+
::LogStash::Event.send(:extend, ::LogStashEvent::ClassMethods)
|
76
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# Copyright (C) 2009-2014 MongoDB Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
# Modified 2015 Elastic
|
16
|
+
|
17
|
+
module BSON
|
18
|
+
|
19
|
+
# Injects behaviour for encoding and decoding time values to
|
20
|
+
# and from raw bytes as specified by the BSON spec.
|
21
|
+
#
|
22
|
+
# @see http://bsonspec.org/#/specification
|
23
|
+
module LogStashTimestamp
|
24
|
+
|
25
|
+
# A time is type 0x09 in the BSON spec.
|
26
|
+
BSON_TYPE = 9.chr.force_encoding(BINARY).freeze
|
27
|
+
|
28
|
+
def to_bson(encoded = ''.force_encoding(BINARY))
|
29
|
+
time.to_bson(encoded)
|
30
|
+
end
|
31
|
+
|
32
|
+
module ClassMethods
|
33
|
+
# Deserialize UTC time from BSON.
|
34
|
+
# @param [ BSON ] bson encoded time.
|
35
|
+
# @return [ ::LogStash::Timestamp ] The decoded UTC time as a ::LogStash::Timestamp.
|
36
|
+
# @see http://bsonspec.org/#/specification
|
37
|
+
def from_bson(bson)
|
38
|
+
seconds, fragment = BSON::Int64.from_bson(bson).divmod(1000)
|
39
|
+
new(::Time.at(seconds, fragment * 1000).utc)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# Register this type when the module is loaded.
|
44
|
+
Registry.register(BSON_TYPE, ::LogStash::Timestamp)
|
45
|
+
end
|
46
|
+
|
47
|
+
# Enrich the core LogStash::Timestamp class with this module.
|
48
|
+
::LogStash::Timestamp.send(:include, LogStashTimestamp)
|
49
|
+
::LogStash::Timestamp.send(:extend, LogStashTimestamp::ClassMethods)
|
50
|
+
end
|
@@ -2,6 +2,9 @@
|
|
2
2
|
require "logstash/outputs/base"
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "mongo"
|
5
|
+
require_relative "bson/big_decimal"
|
6
|
+
require_relative "bson/logstash_timestamp"
|
7
|
+
# require_relative "bson/logstash_event"
|
5
8
|
|
6
9
|
class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
|
7
10
|
|
@@ -38,20 +41,17 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
|
|
38
41
|
@db = conn.use(@database)
|
39
42
|
end # def register
|
40
43
|
|
41
|
-
public
|
42
44
|
def receive(event)
|
43
|
-
|
44
|
-
|
45
45
|
begin
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
document
|
46
|
+
# Our timestamp object now has a to_bson method, using it here
|
47
|
+
# {}.merge(other) so we don't taint the event hash innards
|
48
|
+
document = {}.merge(event.to_hash)
|
49
|
+
if !@isodate
|
50
|
+
# not using timestamp.to_bson
|
51
|
+
document["@timestamp"] = event["@timestamp"].to_json
|
52
52
|
end
|
53
53
|
if @generateId
|
54
|
-
document[
|
54
|
+
document["_id"] = BSON::ObjectId.new(nil, event["@timestamp"])
|
55
55
|
end
|
56
56
|
@db[event.sprintf(@collection)].insert_one(document)
|
57
57
|
rescue => e
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-mongodb'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Store events into MongoDB"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'bigdecimal'
|
3
|
+
require_relative "../spec_helper"
|
4
|
+
require 'stringio'
|
5
|
+
|
6
|
+
describe ::BigDecimal do
|
7
|
+
let(:a_number) { "4321.1234" }
|
8
|
+
let(:bson_number) { 4321.1234.to_bson }
|
9
|
+
|
10
|
+
subject { described_class.new(a_number) }
|
11
|
+
|
12
|
+
it "responds to to_bson" do
|
13
|
+
expect(subject).to respond_to(:to_bson)
|
14
|
+
end
|
15
|
+
|
16
|
+
it "to_bson returns a binary encoded number" do
|
17
|
+
expect(subject.to_bson).to eq(4321.1234.to_bson)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "bson_type returns a binary encoded 1" do
|
21
|
+
expect(subject.bson_type).to eq(12.34.bson_type)
|
22
|
+
end
|
23
|
+
|
24
|
+
describe "class methods" do
|
25
|
+
it "builds a new BigDecimal from BSON" do
|
26
|
+
decoded = described_class.from_bson(StringIO.new(4321.1234.to_bson))
|
27
|
+
expect(decoded).to eql(BigDecimal.new(a_number))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative "../spec_helper"
|
3
|
+
require 'stringio'
|
4
|
+
|
5
|
+
describe ::LogStash::Timestamp do
|
6
|
+
let(:time_array) { [1918,11,11,11,0,0, "+00:00"] }
|
7
|
+
let(:a_time) { Time.utc(*time_array) }
|
8
|
+
let(:bson_time) { Time.utc(*time_array).to_bson }
|
9
|
+
|
10
|
+
subject(:timestamp) { described_class.new(a_time) }
|
11
|
+
|
12
|
+
it "responds to to_bson" do
|
13
|
+
expect(subject).to respond_to(:to_bson)
|
14
|
+
end
|
15
|
+
|
16
|
+
it "to_bson returns a binary encoded timestamp" do
|
17
|
+
expect(timestamp.to_bson).to eq(bson_time)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "bson_type returns a binary encoded 9" do
|
21
|
+
expect(subject.bson_type).to eq(a_time.bson_type)
|
22
|
+
end
|
23
|
+
|
24
|
+
describe "class methods" do
|
25
|
+
it "builds a new Timestamp from BSON" do
|
26
|
+
expected = ::LogStash::Timestamp.new(a_time)
|
27
|
+
decoded = ::LogStash::Timestamp.from_bson(StringIO.new(bson_time))
|
28
|
+
expect(decoded <=> expected).to eq(0)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -6,16 +6,21 @@ describe LogStash::Outputs::Mongodb, :integration => true do
|
|
6
6
|
let(:uri) { 'mongodb://localhost:27017' }
|
7
7
|
let(:database) { 'logstash' }
|
8
8
|
let(:collection) { 'logs' }
|
9
|
+
let(:uuid) { SecureRandom.uuid }
|
9
10
|
|
10
11
|
let(:config) do
|
11
|
-
{ "uri" => uri, "database" => database,
|
12
|
+
{ "uri" => uri, "database" => database,
|
13
|
+
"collection" => collection, "isodate" => true }
|
12
14
|
end
|
13
15
|
|
14
16
|
describe "#send" do
|
15
17
|
|
16
18
|
subject { LogStash::Outputs::Mongodb.new(config) }
|
17
19
|
|
18
|
-
let(:properties) { { "message" => "This is a message!"
|
20
|
+
let(:properties) { { "message" => "This is a message!",
|
21
|
+
"uuid" => uuid, "number" => BigDecimal.new("4321.1234"),
|
22
|
+
"utf8" => "żółć", "int" => 42,
|
23
|
+
"arry" => [42, "string", 4321.1234]} }
|
19
24
|
let(:event) { LogStash::Event.new(properties) }
|
20
25
|
|
21
26
|
before(:each) do
|
@@ -21,7 +21,10 @@ describe LogStash::Outputs::Mongodb do
|
|
21
21
|
|
22
22
|
subject { LogStash::Outputs::Mongodb.new(config) }
|
23
23
|
|
24
|
-
let(:properties) { { "message" => "This is a message!"
|
24
|
+
let(:properties) { { "message" => "This is a message!",
|
25
|
+
"uuid" => SecureRandom.uuid,
|
26
|
+
"number" => BigDecimal.new("4321.1234"),
|
27
|
+
"utf8" => "żółć"} }
|
25
28
|
let(:event) { LogStash::Event.new(properties) }
|
26
29
|
let(:connection) { double("connection") }
|
27
30
|
let(:client) { double("client") }
|
data/spec/spec_helper.rb
CHANGED
@@ -6,6 +6,6 @@ RSpec::Matchers.define :have_received do |event|
|
|
6
6
|
match do |subject|
|
7
7
|
client = subject.instance_variable_get("@db")
|
8
8
|
collection = subject.instance_variable_get("@collection")
|
9
|
-
client["#{collection}"].find("
|
9
|
+
client["#{collection}"].find("uuid" => event["uuid"]).count > 0
|
10
10
|
end
|
11
11
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-mongodb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10
|
11
|
+
date: 2015-12-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,8 +84,13 @@ files:
|
|
84
84
|
- LICENSE
|
85
85
|
- NOTICE.TXT
|
86
86
|
- README.md
|
87
|
+
- lib/logstash/outputs/bson/big_decimal.rb
|
88
|
+
- lib/logstash/outputs/bson/logstash_event.rb
|
89
|
+
- lib/logstash/outputs/bson/logstash_timestamp.rb
|
87
90
|
- lib/logstash/outputs/mongodb.rb
|
88
91
|
- logstash-output-mongodb.gemspec
|
92
|
+
- spec/bson/big_decimal_spec.rb
|
93
|
+
- spec/bson/logstash_timestamp_spec.rb
|
89
94
|
- spec/integration/mongodb_spec.rb
|
90
95
|
- spec/outputs/mongodb_spec.rb
|
91
96
|
- spec/spec_helper.rb
|
@@ -111,11 +116,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
111
116
|
version: '0'
|
112
117
|
requirements: []
|
113
118
|
rubyforge_project:
|
114
|
-
rubygems_version: 2.4.
|
119
|
+
rubygems_version: 2.4.5
|
115
120
|
signing_key:
|
116
121
|
specification_version: 4
|
117
122
|
summary: Store events into MongoDB
|
118
123
|
test_files:
|
124
|
+
- spec/bson/big_decimal_spec.rb
|
125
|
+
- spec/bson/logstash_timestamp_spec.rb
|
119
126
|
- spec/integration/mongodb_spec.rb
|
120
127
|
- spec/outputs/mongodb_spec.rb
|
121
128
|
- spec/spec_helper.rb
|