logstash-input-rabbitmq 3.1.5 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/logstash/inputs/rabbitmq.rb +159 -6
- data/logstash-input-rabbitmq.gemspec +1 -1
- data/spec/inputs/rabbitmq_spec.rb +74 -10
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5caab21a9bb09f1d4af8660f26558c9bde6e9968
|
4
|
+
data.tar.gz: 522d2d66e0c768a06dc14f7dea3e38cc3e4419c0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d8a4404784a9de832e23c4b1c9272a87997fbeb4fc5984380c27e6c52a1e8f3661f170e8909eb59bfe7fad15c7af4303e327e44da23ce89632e65a737d1b47b
|
7
|
+
data.tar.gz: 5f01768900d95ddba5259bff24c01856297684c3a46642d78643c96b16d2f508eb37052cb309bc13d8829a6ca3154d97275d85ab96e0c4e22240be6fce2039d7
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 3.2.0
|
2
|
+
- The properties and headers of the messages are now saved in the [@metadata][rabbitmq_headers] and [@metadata][rabbitmq_properties] fields.
|
3
|
+
- Logstash now shuts down if the server sends a basic.cancel method.
|
4
|
+
- Reinstating the overview documentation that was lost in 3.0.0 and updating it to be clearer.
|
5
|
+
- Internal: Various spec improvements that decrease flakiness and eliminate the queue littering of the integration test RabbitMQ instance.
|
6
|
+
|
1
7
|
## 3.1.5
|
2
8
|
- Fix a bug where when reconnecting a duplicate consumer would be created
|
3
9
|
|
@@ -4,15 +4,101 @@ require 'logstash/inputs/threadable'
|
|
4
4
|
|
5
5
|
module LogStash
|
6
6
|
module Inputs
|
7
|
+
# Pull events from a http://www.rabbitmq.com/[RabbitMQ] queue.
|
8
|
+
#
|
9
|
+
# The default settings will create an entirely transient queue and listen for all messages by default.
|
10
|
+
# If you need durability or any other advanced settings, please set the appropriate options
|
11
|
+
#
|
12
|
+
# This plugin uses the http://rubymarchhare.info/[March Hare] library
|
13
|
+
# for interacting with the RabbitMQ server. Most configuration options
|
14
|
+
# map directly to standard RabbitMQ and AMQP concepts. The
|
15
|
+
# https://www.rabbitmq.com/amqp-0-9-1-reference.html[AMQP 0-9-1 reference guide]
|
16
|
+
# and other parts of the RabbitMQ documentation are useful for deeper
|
17
|
+
# understanding.
|
18
|
+
#
|
19
|
+
# The properties of messages received will be stored in the
|
20
|
+
# `[@metadata][rabbitmq_properties]` field. The following
|
21
|
+
# properties may be available (in most cases dependent on whether
|
22
|
+
# they were set by the sender):
|
23
|
+
#
|
24
|
+
# * app-id
|
25
|
+
# * cluster-id
|
26
|
+
# * consumer-tag
|
27
|
+
# * content-encoding
|
28
|
+
# * content-type
|
29
|
+
# * correlation-id
|
30
|
+
# * delivery-mode
|
31
|
+
# * exchange
|
32
|
+
# * expiration
|
33
|
+
# * message-id
|
34
|
+
# * priority
|
35
|
+
# * redeliver
|
36
|
+
# * reply-to
|
37
|
+
# * routing-key
|
38
|
+
# * timestamp
|
39
|
+
# * type
|
40
|
+
# * user-id
|
41
|
+
#
|
42
|
+
# For example, to get the RabbitMQ message's timestamp property
|
43
|
+
# into the Logstash event's `@timestamp` field, use the date
|
44
|
+
# filter to parse the `[@metadata][rabbitmq_properties][timestamp]`
|
45
|
+
# field:
|
46
|
+
# [source,ruby]
|
47
|
+
# filter {
|
48
|
+
# if [@metadata][rabbitmq_properties][timestamp] {
|
49
|
+
# date {
|
50
|
+
# match => ["[@metadata][rabbitmq_properties][timestamp]", "UNIX"]
|
51
|
+
# }
|
52
|
+
# }
|
53
|
+
# }
|
54
|
+
#
|
55
|
+
# Additionally, any message headers will be saved in the
|
56
|
+
# `[@metadata][rabbitmq_headers]` field.
|
7
57
|
class RabbitMQ < LogStash::Inputs::Threadable
|
8
58
|
include ::LogStash::PluginMixins::RabbitMQConnection
|
9
59
|
|
60
|
+
# The properties to extract from each message and store in a
|
61
|
+
# @metadata field.
|
62
|
+
#
|
63
|
+
# Technically the exchange, redeliver, and routing-key
|
64
|
+
# properties belong to the envelope and not the message but we
|
65
|
+
# ignore that distinction here. However, we extract the
|
66
|
+
# headers separately via get_headers even though the header
|
67
|
+
# table technically is a message property.
|
68
|
+
#
|
69
|
+
# Freezing all strings so that code modifying the event's
|
70
|
+
# @metadata field can't touch them.
|
71
|
+
#
|
72
|
+
# If updating this list, remember to update the documentation
|
73
|
+
# above too.
|
74
|
+
MESSAGE_PROPERTIES = [
|
75
|
+
"app-id",
|
76
|
+
"cluster-id",
|
77
|
+
"consumer-tag",
|
78
|
+
"content-encoding",
|
79
|
+
"content-type",
|
80
|
+
"correlation-id",
|
81
|
+
"delivery-mode",
|
82
|
+
"exchange",
|
83
|
+
"expiration",
|
84
|
+
"message-id",
|
85
|
+
"priority",
|
86
|
+
"redeliver",
|
87
|
+
"reply-to",
|
88
|
+
"routing-key",
|
89
|
+
"timestamp",
|
90
|
+
"type",
|
91
|
+
"user-id",
|
92
|
+
].map { |s| s.freeze }.freeze
|
93
|
+
|
10
94
|
config_name "rabbitmq"
|
11
95
|
|
12
96
|
# The default codec for this plugin is JSON. You can override this to suit your particular needs however.
|
13
97
|
default :codec, "json"
|
14
98
|
|
15
|
-
# The name of the queue Logstash will consume events from.
|
99
|
+
# The name of the queue Logstash will consume events from. If
|
100
|
+
# left empty, a transient queue with an randomly chosen name
|
101
|
+
# will be created.
|
16
102
|
config :queue, :validate => :string, :default => ""
|
17
103
|
|
18
104
|
# Is this queue durable? (aka; Should it survive a broker restart?)
|
@@ -33,13 +119,24 @@ module LogStash
|
|
33
119
|
# To make a RabbitMQ queue mirrored, use: `{"x-ha-policy" => "all"}`
|
34
120
|
config :arguments, :validate => :array, :default => {}
|
35
121
|
|
36
|
-
# Prefetch count.
|
122
|
+
# Prefetch count. If acknowledgements are enabled with the `ack`
|
123
|
+
# option, specifies the number of outstanding unacknowledged
|
124
|
+
# messages allowed. With acknowledgemnts disabled this setting
|
125
|
+
# has no effect.
|
37
126
|
config :prefetch_count, :validate => :number, :default => 256
|
38
127
|
|
39
|
-
# Enable message
|
128
|
+
# Enable message acknowledgements. With acknowledgements
|
129
|
+
# messages fetched by Logstash but not yet sent into the
|
130
|
+
# Logstash pipeline will be requeued by the server if Logstash
|
131
|
+
# shuts down. Acknowledgements will however hurt the message
|
132
|
+
# throughput.
|
40
133
|
config :ack, :validate => :boolean, :default => true
|
41
134
|
|
42
|
-
#
|
135
|
+
# If true the queue will be passively declared, meaning it must
|
136
|
+
# already exist on the server. To have Logstash create the queue
|
137
|
+
# if necessary leave this option as false. If actively declaring
|
138
|
+
# a queue that already exists, the queue options for this plugin
|
139
|
+
# (durable etc) must match those of the existing queue.
|
43
140
|
config :passive, :validate => :boolean, :default => false
|
44
141
|
|
45
142
|
# The name of the exchange to bind the queue to.
|
@@ -53,7 +150,7 @@ module LogStash
|
|
53
150
|
config :key, :validate => :string, :default => "logstash"
|
54
151
|
|
55
152
|
# Amount of time in seconds to wait after a failed subscription request
|
56
|
-
# before retrying. Subscribes can fail if the server goes away and then comes back
|
153
|
+
# before retrying. Subscribes can fail if the server goes away and then comes back.
|
57
154
|
config :subscription_retry_interval_seconds, :validate => :number, :required => true, :default => 5
|
58
155
|
|
59
156
|
def register
|
@@ -102,9 +199,12 @@ module LogStash
|
|
102
199
|
# that we rely on MarchHare to do the reconnection for us with auto_reconnect.
|
103
200
|
# Unfortunately, while MarchHare does the reconnection work it won't re-subscribe the consumer
|
104
201
|
# hence the logic below.
|
105
|
-
@consumer = @hare_info.queue.build_consumer(
|
202
|
+
@consumer = @hare_info.queue.build_consumer(:block => true,
|
203
|
+
:on_cancellation => Proc.new { on_cancellation }) do |metadata, data|
|
106
204
|
@codec.decode(data) do |event|
|
107
205
|
decorate(event)
|
206
|
+
event["@metadata"]["rabbitmq_headers"] = get_headers(metadata)
|
207
|
+
event["@metadata"]["rabbitmq_properties"] = get_properties(metadata)
|
108
208
|
@output_queue << event if event
|
109
209
|
end
|
110
210
|
@hare_info.channel.ack(metadata.delivery_tag) if @ack
|
@@ -135,6 +235,59 @@ module LogStash
|
|
135
235
|
@consumer.gracefully_shut_down
|
136
236
|
end
|
137
237
|
|
238
|
+
def on_cancellation
|
239
|
+
@logger.info("Received basic.cancel from #{rabbitmq_settings[:host]}, shutting down.")
|
240
|
+
stop
|
241
|
+
end
|
242
|
+
|
243
|
+
private
|
244
|
+
|
245
|
+
# ByteArrayLongString is a private static inner class which
|
246
|
+
# can't be access via the regular Java::SomeNameSpace::Classname
|
247
|
+
# notation. See https://github.com/jruby/jruby/issues/3333.
|
248
|
+
ByteArrayLongString = JavaUtilities::get_proxy_class('com.rabbitmq.client.impl.LongStringHelper$ByteArrayLongString')
|
249
|
+
|
250
|
+
def get_header_value(value)
|
251
|
+
# Two kinds of values require exceptional treatment:
|
252
|
+
#
|
253
|
+
# String values are instances of
|
254
|
+
# com.rabbitmq.client.impl.LongStringHelper.ByteArrayLongString
|
255
|
+
# and we don't want to propagate those.
|
256
|
+
#
|
257
|
+
# List values are java.util.ArrayList objects and we need to
|
258
|
+
# recurse into them to convert any nested strings values.
|
259
|
+
if value.class == Java::JavaUtil::ArrayList
|
260
|
+
value.map{|item| get_header_value(item) }
|
261
|
+
elsif value.class == ByteArrayLongString
|
262
|
+
value.toString
|
263
|
+
else
|
264
|
+
value
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
private
|
269
|
+
def get_headers(metadata)
|
270
|
+
if !metadata.headers.nil?
|
271
|
+
Hash[metadata.headers.map {|k, v| [k, get_header_value(v)]}]
|
272
|
+
else
|
273
|
+
{}
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
private
|
278
|
+
def get_properties(metadata)
|
279
|
+
MESSAGE_PROPERTIES.reduce({}) do |acc, name|
|
280
|
+
# The method names obviously can't contain hyphens.
|
281
|
+
value = metadata.send(name.gsub("-", "_"))
|
282
|
+
if value
|
283
|
+
# The AMQP 0.9.1 timestamp field only has second resolution
|
284
|
+
# so storing milliseconds serves no purpose and might give
|
285
|
+
# the incorrect impression of a higher resolution.
|
286
|
+
acc[name] = name != "timestamp" ? value : value.getTime / 1000
|
287
|
+
end
|
288
|
+
acc
|
289
|
+
end
|
290
|
+
end
|
138
291
|
end
|
139
292
|
end
|
140
293
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-rabbitmq'
|
3
|
-
s.version = '3.
|
3
|
+
s.version = '3.2.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Pull events from a RabbitMQ exchange."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -112,7 +112,7 @@ end
|
|
112
112
|
|
113
113
|
describe "with a live server", :integration => true do
|
114
114
|
let(:klass) { LogStash::Inputs::RabbitMQ }
|
115
|
-
let(:config) { {"host" => "127.0.0.1"} }
|
115
|
+
let(:config) { {"host" => "127.0.0.1", "auto_delete" => true, "codec" => "plain" } }
|
116
116
|
let(:instance) { klass.new(config) }
|
117
117
|
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
118
118
|
let(:output_queue) { Queue.new }
|
@@ -134,7 +134,7 @@ describe "with a live server", :integration => true do
|
|
134
134
|
# Extra time to make sure the consumer can attach
|
135
135
|
# Without this there's a chance the shutdown code will execute
|
136
136
|
# before consumption begins. This is tricky to do more elegantly
|
137
|
-
sleep
|
137
|
+
sleep 4
|
138
138
|
end
|
139
139
|
|
140
140
|
let(:test_connection) { MarchHare.connect(instance.send(:rabbitmq_settings)) }
|
@@ -166,21 +166,85 @@ describe "with a live server", :integration => true do
|
|
166
166
|
end
|
167
167
|
|
168
168
|
describe "receiving a message with a queue specified" do
|
169
|
-
let(:queue_name) { "foo_queue" }
|
170
169
|
let(:config) { super.merge("queue" => queue_name) }
|
170
|
+
let(:event) { output_queue.pop }
|
171
|
+
let(:queue) { test_channel.queue(queue_name, :auto_delete => true) }
|
172
|
+
let(:queue_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
|
171
173
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
174
|
+
context "when the message has a payload but no message headers" do
|
175
|
+
before do
|
176
|
+
queue.publish(message)
|
177
|
+
end
|
178
|
+
|
179
|
+
let(:message) { "Foo Message" }
|
180
|
+
|
181
|
+
it "should process the message and store the payload" do
|
182
|
+
expect(event["message"]).to eql(message)
|
183
|
+
end
|
176
184
|
|
177
|
-
|
178
|
-
|
185
|
+
it "should save an empty message header hash" do
|
186
|
+
expect(event).to include("@metadata")
|
187
|
+
expect(event["@metadata"]).to include("rabbitmq_headers")
|
188
|
+
expect(event["@metadata"]["rabbitmq_headers"]).to eq({})
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
context "when message properties are available" do
|
193
|
+
before do
|
194
|
+
# Don't test every single property but select a few with
|
195
|
+
# different characteristics to get sufficient coverage.
|
196
|
+
queue.publish("",
|
197
|
+
:properties => {
|
198
|
+
:app_id => app_id,
|
199
|
+
:timestamp => Java::JavaUtil::Date.new(epoch * 1000),
|
200
|
+
:priority => priority,
|
201
|
+
})
|
202
|
+
end
|
203
|
+
|
204
|
+
let(:app_id) { "myapplication" }
|
205
|
+
# Randomize the epoch we test with but limit its range to signed
|
206
|
+
# ints to not assume all protocols and libraries involved use
|
207
|
+
# unsigned ints for epoch values.
|
208
|
+
let(:epoch) { rand(0x7FFFFFFF) }
|
209
|
+
let(:priority) { 5 }
|
210
|
+
|
211
|
+
it "should save message properties into a @metadata field" do
|
212
|
+
expect(event).to include("@metadata")
|
213
|
+
expect(event["@metadata"]).to include("rabbitmq_properties")
|
214
|
+
|
215
|
+
props = event["@metadata"]["rabbitmq_properties"]
|
216
|
+
expect(props["app-id"]).to eq(app_id)
|
217
|
+
expect(props["delivery-mode"]).to eq(1)
|
218
|
+
expect(props["exchange"]).to eq("")
|
219
|
+
expect(props["priority"]).to eq(priority)
|
220
|
+
expect(props["routing-key"]).to eq(queue_name)
|
221
|
+
expect(props["timestamp"]).to eq(epoch)
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
context "when message headers are available" do
|
226
|
+
before do
|
227
|
+
queue.publish("", :properties => { :headers => headers })
|
228
|
+
end
|
229
|
+
|
230
|
+
let (:headers) {
|
231
|
+
{
|
232
|
+
"arrayvalue" => [true, 123, "foo"],
|
233
|
+
"boolvalue" => true,
|
234
|
+
"intvalue" => 123,
|
235
|
+
"stringvalue" => "foo",
|
236
|
+
}
|
237
|
+
}
|
238
|
+
|
239
|
+
it "should save message headers into a @metadata field" do
|
240
|
+
expect(event).to include("@metadata")
|
241
|
+
expect(event["@metadata"]).to include("rabbitmq_headers")
|
242
|
+
expect(event["@metadata"]["rabbitmq_headers"]).to include(headers)
|
243
|
+
end
|
179
244
|
end
|
180
245
|
end
|
181
246
|
|
182
247
|
describe LogStash::Inputs::RabbitMQ do
|
183
|
-
let(:config) { super.merge("queue" => "foo_queue") }
|
184
248
|
it_behaves_like "an interruptible input plugin" do
|
185
249
|
|
186
250
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-rabbitmq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|