logstash-output-mongodb 3.1.4 → 3.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: baf5b3247ad538b7b06f0a74a9f1bfd0e1628c30a19a7645331c052d7e05be2a
4
- data.tar.gz: f375e7f1c918a0222261aac759a425c45a0d4302acf8760639acaadcab29d0e4
3
+ metadata.gz: a80fe32d5f3500dadd616df40cbf18fdb09678d4c2501d7a5514662cf6934b87
4
+ data.tar.gz: ca57807a9e2a897d6b2d4177735c58a7a2ac8db5f156148393f5e8ae6694584f
5
5
  SHA512:
6
- metadata.gz: acb5751495b30c5fef2192e9e3f5dad862c32f0af6178b0855b3d268db860bdf467388852c87494e097815193ce1e81652e682200dd48c63f04524c43639669c
7
- data.tar.gz: a6a3b1e2b5ec9663e4a2f13a2e1c059866497e971a971dc89b87df16a5ca488c3bea958dee9d0ebf7aac6ec8a08458bbefa2c76638e8569e97abfbbfc41ab6a7
6
+ metadata.gz: 338736bbad3f0f3f7bea52c42afbf820490308f4c66de9918c30ab87ec65f3a8dc143cf2466de117c5ed747a604fe7ea5d34255914ecb47cd235593aaa7d8fce
7
+ data.tar.gz: 44f24d65baec8b345730066a17fa0c329d8b6d8109f50c5f50351abb4902d0c68f4d8af54d9f42556c65b0724975bf2426f16e59150f3fa3f0971d78c133206d
@@ -1,3 +1,6 @@
1
+ ## 3.1.5
2
+ - Fixed @timestamp handling, BSON::ObjectId generation, close method [#57](https://github.com/logstash-plugins/logstash-output-mongodb/pull/57)
3
+
1
4
  ## 3.1.4
2
5
  - Docs: Set the default_codec doc attribute.
3
6
 
@@ -15,6 +15,7 @@ Contributors:
15
15
  * Richard Pijnenburg (electrical)
16
16
  * bitsofinfo (bitsofinfo)
17
17
  * Guy Boertje (guyboertje)
18
+ * Colin Surprenant (colinsurprenant)
18
19
 
19
20
  Note: If you've sent us patches, bug reports, or otherwise contributed to
20
21
  Logstash, and you aren't on the list above and want to be, please let us know
@@ -46,19 +46,22 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
46
46
  # Mutex used to synchronize access to 'documents'
47
47
  @@mutex = Mutex.new
48
48
 
49
- public
50
49
  def register
50
+ if @bulk_size > 1000
51
+ raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
52
+ end
53
+
51
54
  Mongo::Logger.logger = @logger
52
55
  conn = Mongo::Client.new(@uri)
53
56
  @db = conn.use(@database)
54
57
 
55
- if @bulk_size > 1000
56
- raise LogStash::ConfigurationError, "Bulk size must be lower than '1000', currently '#{@bulk_size}'"
57
- end
58
+ @closed = Concurrent::AtomicBoolean.new(false)
58
59
  @documents = {}
59
- Thread.new do
60
- loop do
61
- sleep @bulk_interval
60
+
61
+ @bulk_thread = Thread.new(@bulk_interval) do |bulk_interval|
62
+ while @closed.false? do
63
+ sleep(bulk_interval)
64
+
62
65
  @@mutex.synchronize do
63
66
  @documents.each do |collection, values|
64
67
  if values.length > 0
@@ -69,23 +72,31 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
69
72
  end
70
73
  end
71
74
  end
72
- end # def register
75
+ end
73
76
 
74
77
  def receive(event)
75
78
  begin
76
79
  # Our timestamp object now has a to_bson method, using it here
77
80
  # {}.merge(other) so we don't taint the event hash innards
78
81
  document = {}.merge(event.to_hash)
82
+
79
83
  if !@isodate
80
- # not using timestamp.to_bson
81
- document["@timestamp"] = event.timestamp.to_json
84
+ timestamp = event.timestamp
85
+ if timestamp
86
+ # not using timestamp.to_bson
87
+ document["@timestamp"] = timestamp.to_json
88
+ else
89
+ @logger.warn("Cannot set MongoDB document `@timestamp` field because it does not exist in the event", :event => event)
90
+ end
82
91
  end
92
+
83
93
  if @generateId
84
- document["_id"] = BSON::ObjectId.new(nil, event.timestamp)
94
+ document["_id"] = BSON::ObjectId.new
85
95
  end
96
+
86
97
  if @bulk
98
+ collection = event.sprintf(@collection)
87
99
  @@mutex.synchronize do
88
- collection = event.sprintf(@collection)
89
100
  if(!@documents[collection])
90
101
  @documents[collection] = []
91
102
  end
@@ -99,20 +110,25 @@ class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
99
110
  else
100
111
  @db[event.sprintf(@collection)].insert_one(document)
101
112
  end
102
-
103
113
  rescue => e
104
- @logger.warn("Failed to send event to MongoDB", :event => event, :exception => e,
105
- :backtrace => e.backtrace)
106
114
  if e.message =~ /^E11000/
107
- # On a duplicate key error, skip the insert.
108
- # We could check if the duplicate key err is the _id key
109
- # and generate a new primary key.
110
- # If the duplicate key error is on another field, we have no way
111
- # to fix the issue.
115
+ # On a duplicate key error, skip the insert.
116
+ # We could check if the duplicate key err is the _id key
117
+ # and generate a new primary key.
118
+ # If the duplicate key error is on another field, we have no way
119
+ # to fix the issue.
120
+ @logger.warn("Skipping insert because of a duplicate key error", :event => event, :exception => e)
112
121
  else
113
- sleep @retry_delay
122
+ @logger.warn("Failed to send event to MongoDB, retrying in #{@retry_delay.to_s} seconds", :event => event, :exception => e)
123
+ sleep(@retry_delay)
114
124
  retry
115
125
  end
116
126
  end
117
- end # def receive
118
- end # class LogStash::Outputs::Mongodb
127
+ end
128
+
129
+ def close
130
+ @closed.make_true
131
+ @bulk_thread.wakeup
132
+ @bulk_thread.join
133
+ end
134
+ end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-mongodb'
3
- s.version = '3.1.4'
3
+ s.version = '3.1.5'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Writes events to MongoDB"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -8,23 +8,21 @@ describe LogStash::Outputs::Mongodb do
8
8
  let(:database) { 'logstash' }
9
9
  let(:collection) { 'logs' }
10
10
 
11
- let(:config) do
12
- { "uri" => uri, "database" => database, "collection" => collection }
13
- end
11
+ let(:config) {{
12
+ "uri" => uri,
13
+ "database" => database,
14
+ "collection" => collection
15
+ }}
14
16
 
15
- it "should register" do
17
+ it "should register and close" do
16
18
  plugin = LogStash::Plugin.lookup("output", "mongodb").new(config)
17
19
  expect {plugin.register}.to_not raise_error
20
+ plugin.close
18
21
  end
19
22
 
20
- describe "#send" do
21
-
23
+ describe "receive" do
22
24
  subject! { LogStash::Outputs::Mongodb.new(config) }
23
25
 
24
- let(:properties) { { "message" => "This is a message!",
25
- "uuid" => SecureRandom.uuid,
26
- "number" => BigDecimal.new("4321.1234"),
27
- "utf8" => "żółć"} }
28
26
  let(:event) { LogStash::Event.new(properties) }
29
27
  let(:connection) { double("connection") }
30
28
  let(:client) { double("client") }
@@ -38,10 +36,51 @@ describe LogStash::Outputs::Mongodb do
38
36
  subject.register
39
37
  end
40
38
 
41
- it "should send the event to the database" do
42
- expect(collection).to receive(:insert_one)
43
- subject.receive(event)
39
+ after(:each) do
40
+ subject.close
41
+ end
42
+
43
+ describe "#send" do
44
+ let(:properties) {{
45
+ "message" => "This is a message!",
46
+ "uuid" => SecureRandom.uuid,
47
+ "number" => BigDecimal.new("4321.1234"),
48
+ "utf8" => "żółć"
49
+ }}
50
+
51
+ it "should send the event to the database" do
52
+ expect(collection).to receive(:insert_one)
53
+ subject.receive(event)
54
+ end
44
55
  end
45
- end
46
56
 
57
+ describe "no event @timestamp" do
58
+ let(:properties) { { "message" => "foo" } }
59
+
60
+ it "should not contain a @timestamp field in the mongo document" do
61
+ expect(event).to receive(:timestamp).and_return(nil)
62
+ expect(event).to receive(:to_hash).and_return(properties)
63
+ expect(collection).to receive(:insert_one).with(properties)
64
+ subject.receive(event)
65
+ end
66
+ end
67
+
68
+ describe "generateId" do
69
+ let(:properties) { { "message" => "foo" } }
70
+ let(:config) {{
71
+ "uri" => uri,
72
+ "database" => database,
73
+ "collection" => collection,
74
+ "generateId" => true
75
+ }}
76
+
77
+ it "should contain a BSON::ObjectId as _id" do
78
+ expect(BSON::ObjectId).to receive(:new).and_return("BSON::ObjectId")
79
+ expect(event).to receive(:timestamp).and_return(nil)
80
+ expect(event).to receive(:to_hash).and_return(properties)
81
+ expect(collection).to receive(:insert_one).with(properties.merge("_id" => "BSON::ObjectId"))
82
+ subject.receive(event)
83
+ end
84
+ end
85
+ end
47
86
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-mongodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.4
4
+ version: 3.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-06 00:00:00.000000000 Z
11
+ date: 2019-03-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -119,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
119
119
  version: '0'
120
120
  requirements: []
121
121
  rubyforge_project:
122
- rubygems_version: 2.6.11
122
+ rubygems_version: 2.6.13
123
123
  signing_key:
124
124
  specification_version: 4
125
125
  summary: Writes events to MongoDB