logstash-integration-rabbitmq 7.0.2-java → 7.3.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -0
- data/LICENSE +199 -10
- data/README.md +1 -1
- data/docs/input-rabbitmq.asciidoc +13 -4
- data/docs/output-rabbitmq.asciidoc +4 -2
- data/lib/logstash/inputs/rabbitmq.rb +72 -16
- data/lib/logstash/outputs/rabbitmq.rb +71 -10
- data/lib/logstash/plugin_mixins/rabbitmq_connection.rb +75 -45
- data/logstash-integration-rabbitmq.gemspec +2 -2
- data/spec/inputs/rabbitmq_spec.rb +228 -9
- data/spec/outputs/rabbitmq_spec.rb +61 -9
- data/spec/plugin_mixins/rabbitmq_connection_spec.rb +48 -23
- metadata +7 -7
@@ -1,14 +1,12 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
require "logstash/pipeline"
|
3
3
|
require_relative '../plugin_mixins/rabbitmq_connection'
|
4
|
-
java_import java.util.concurrent.TimeoutException
|
5
|
-
java_import com.rabbitmq.client.AlreadyClosedException
|
6
4
|
|
7
5
|
require 'back_pressure'
|
8
6
|
|
9
7
|
# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
|
10
8
|
# or later version (3.x is recommended).
|
11
|
-
#
|
9
|
+
#
|
12
10
|
# Relevant links:
|
13
11
|
#
|
14
12
|
# * http://www.rabbitmq.com/[RabbitMQ]
|
@@ -16,10 +14,14 @@ require 'back_pressure'
|
|
16
14
|
module LogStash
|
17
15
|
module Outputs
|
18
16
|
class RabbitMQ < LogStash::Outputs::Base
|
17
|
+
|
18
|
+
java_import java.util.concurrent.TimeoutException
|
19
|
+
java_import com.rabbitmq.client.AlreadyClosedException
|
20
|
+
|
19
21
|
include LogStash::PluginMixins::RabbitMQConnection
|
20
22
|
|
21
23
|
config_name "rabbitmq"
|
22
|
-
|
24
|
+
|
23
25
|
concurrency :shared
|
24
26
|
|
25
27
|
# The default codec for this plugin is JSON. You can override this to suit your particular needs however.
|
@@ -46,6 +48,8 @@ module LogStash
|
|
46
48
|
config :message_properties, :validate => :hash, :default => {}
|
47
49
|
|
48
50
|
def register
|
51
|
+
@message_properties_template = MessagePropertiesTemplate.new(symbolize(@message_properties).merge(:persistent => @persistent))
|
52
|
+
|
49
53
|
connect!
|
50
54
|
@hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
|
51
55
|
# The connection close should close all channels, so it is safe to store thread locals here without closing them
|
@@ -67,14 +71,13 @@ module LogStash
|
|
67
71
|
|
68
72
|
def publish(event, message)
|
69
73
|
raise ArgumentError, "No exchange set in HareInfo!!!" unless @hare_info.exchange
|
74
|
+
routing_key = event.sprintf(@key)
|
75
|
+
message_properties = @message_properties_template.build(event)
|
70
76
|
@gated_executor.execute do
|
71
|
-
local_exchange.publish(message, :routing_key =>
|
77
|
+
local_exchange.publish(message, :routing_key => routing_key, :properties => message_properties)
|
72
78
|
end
|
73
79
|
rescue MarchHare::Exception, IOError, AlreadyClosedException, TimeoutException => e
|
74
|
-
@logger.error("Error while publishing
|
75
|
-
:message => e.message,
|
76
|
-
:exception => e.class,
|
77
|
-
:backtrace => e.backtrace)
|
80
|
+
@logger.error("Error while publishing, will retry", error_details(e, backtrace: true))
|
78
81
|
|
79
82
|
sleep_for_retry
|
80
83
|
retry
|
@@ -126,6 +129,64 @@ module LogStash
|
|
126
129
|
end
|
127
130
|
end
|
128
131
|
end
|
132
|
+
|
133
|
+
##
|
134
|
+
# A `MessagePropertiesTemplate` efficiently produces per-event message properties from the
|
135
|
+
# provided template Hash.
|
136
|
+
#
|
137
|
+
# In order to efficiently reuse constant-value objects, returned values may be frozen.
|
138
|
+
class MessagePropertiesTemplate
|
139
|
+
##
|
140
|
+
# Creates a new `MessagePropertiesTemplate` from the provided `template`
|
141
|
+
# @param template [Hash{Symbol=>Object}]
|
142
|
+
def initialize(template)
|
143
|
+
constant_properties = template.reject { |_,v| templated?(v) }
|
144
|
+
variable_properties = template.select { |_,v| templated?(v) }
|
145
|
+
|
146
|
+
@constant_properties = normalize(constant_properties).freeze
|
147
|
+
@variable_properties = variable_properties
|
148
|
+
end
|
149
|
+
|
150
|
+
##
|
151
|
+
# Builds a property mapping for the given `event`, including templated values.
|
152
|
+
#
|
153
|
+
# @param event [LogStash::Event]: the event with which to populated templated values, if any.
|
154
|
+
# @return [Hash{Symbol=>Object}] a possibly-frozen properties hash for the provided `event`.
|
155
|
+
def build(event)
|
156
|
+
return @constant_properties if @variable_properties.empty?
|
157
|
+
|
158
|
+
properties = @variable_properties.each_with_object(@constant_properties.dup) do |(k,v), memo|
|
159
|
+
memo.store(k, event.sprintf(v))
|
160
|
+
end
|
161
|
+
|
162
|
+
return normalize(properties)
|
163
|
+
end
|
164
|
+
|
165
|
+
private
|
166
|
+
|
167
|
+
##
|
168
|
+
# Normalize the provided property mapping with respect to the value types the underlying
|
169
|
+
# client expects.
|
170
|
+
#
|
171
|
+
# @api private
|
172
|
+
# @param properties [Hash{Symbol=>Object}]: a possibly-frozen Hash whose values may need type-coercion.
|
173
|
+
# @return [Hash{Symbol=>Object}]
|
174
|
+
def normalize(properties)
|
175
|
+
if properties[:priority] && properties[:priority].kind_of?(String)
|
176
|
+
properties = properties.merge(:priority => properties[:priority].to_i)
|
177
|
+
end
|
178
|
+
|
179
|
+
properties
|
180
|
+
end
|
181
|
+
|
182
|
+
##
|
183
|
+
# @api private
|
184
|
+
# @param [Object]: an object, which may or may not be a template `String`
|
185
|
+
# @return [Boolean]: returns `true` IFF `value` is a template `String`
|
186
|
+
def templated?(value)
|
187
|
+
value.kind_of?(String) && value.include?('%{')
|
188
|
+
end
|
189
|
+
end
|
129
190
|
end
|
130
191
|
end
|
131
|
-
end
|
192
|
+
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
require "logstash/namespace"
|
3
3
|
require "march_hare"
|
4
4
|
require "java"
|
5
|
+
require "stud/interval"
|
5
6
|
|
6
7
|
# Common functionality for the rabbitmq input/output
|
7
8
|
module LogStash
|
@@ -104,17 +105,17 @@ module LogStash
|
|
104
105
|
def rabbitmq_settings
|
105
106
|
return @rabbitmq_settings if @rabbitmq_settings
|
106
107
|
|
108
|
+
|
107
109
|
s = {
|
108
110
|
:vhost => @vhost,
|
109
|
-
:
|
110
|
-
:
|
111
|
-
:user => @user,
|
111
|
+
:addresses => addresses_from_hosts_and_port(@host, @port),
|
112
|
+
:username => @user,
|
112
113
|
:automatic_recovery => @automatic_recovery,
|
113
|
-
:
|
114
|
+
:password => @password ? @password.value : "guest",
|
114
115
|
}
|
115
116
|
|
116
|
-
s[:
|
117
|
-
s[:
|
117
|
+
s[:connection_timeout] = @connection_timeout || 0
|
118
|
+
s[:requested_heartbeat] = @heartbeat || 0
|
118
119
|
|
119
120
|
if @ssl
|
120
121
|
s[:tls] = @ssl_version
|
@@ -133,10 +134,15 @@ module LogStash
|
|
133
134
|
@rabbitmq_settings = s
|
134
135
|
end
|
135
136
|
|
137
|
+
def addresses_from_hosts_and_port(hosts, port)
|
138
|
+
hosts.map {|host| host.include?(':') ? host : "#{host}:#{port}"}
|
139
|
+
end
|
140
|
+
|
141
|
+
|
136
142
|
def connect!
|
137
143
|
@hare_info = connect() unless @hare_info # Don't duplicate the conn!
|
138
144
|
rescue MarchHare::Exception, java.io.IOException => e
|
139
|
-
|
145
|
+
message = if e.message.empty? && e.is_a?(java.io.IOException)
|
140
146
|
# IOException with an empty message is probably an instance of
|
141
147
|
# these problems:
|
142
148
|
# https://github.com/logstash-plugins/logstash-output-rabbitmq/issues/52
|
@@ -145,21 +151,12 @@ module LogStash
|
|
145
151
|
# Best guess is to help the user understand that there is probably
|
146
152
|
# some kind of configuration problem causing the error, but we
|
147
153
|
# can't really offer any more detailed hints :\
|
148
|
-
"An unknown error occurred
|
154
|
+
"An unknown RabbitMQ error occurred, maybe this is a configuration error (invalid vhost, for example) - please check the RabbitMQ server logs for clues about this failure"
|
149
155
|
else
|
150
|
-
|
156
|
+
"RabbitMQ connection error, will retry"
|
151
157
|
end
|
152
158
|
|
153
|
-
|
154
|
-
@logger.error("RabbitMQ connection error, will retry.",
|
155
|
-
:error_message => error_message,
|
156
|
-
:exception => e.class.name,
|
157
|
-
:backtrace => e.backtrace)
|
158
|
-
else
|
159
|
-
@logger.error("RabbitMQ connection error, will retry.",
|
160
|
-
:error_message => error_message,
|
161
|
-
:exception => e.class.name)
|
162
|
-
end
|
159
|
+
@logger.error(message, error_details(e))
|
163
160
|
|
164
161
|
sleep_for_retry
|
165
162
|
retry
|
@@ -173,48 +170,43 @@ module LogStash
|
|
173
170
|
@hare_info && @hare_info.connection && @hare_info.connection.open?
|
174
171
|
end
|
175
172
|
|
176
|
-
def connected?
|
177
|
-
return nil unless @hare_info && @hare_info.connection
|
178
|
-
@hare_info.connection.connected?
|
179
|
-
end
|
180
|
-
|
181
173
|
private
|
182
174
|
|
183
175
|
def declare_exchange!(channel, exchange, exchange_type, durable)
|
184
|
-
@logger.debug? && @logger.debug("Declaring an exchange", :name => exchange,
|
185
|
-
|
186
|
-
|
187
|
-
@logger.
|
188
|
-
|
189
|
-
rescue StandardError => e
|
190
|
-
@logger.error("Could not declare exchange!",
|
191
|
-
:exchange => exchange, :type => exchange_type,
|
192
|
-
:durable => durable, :error_class => e.class.name,
|
193
|
-
:error_message => e.message, :backtrace => e.backtrace)
|
176
|
+
@logger.debug? && @logger.debug("Declaring an exchange", :name => exchange, :type => exchange_type, :durable => durable)
|
177
|
+
channel.exchange(exchange, :type => exchange_type.to_sym, :durable => durable)
|
178
|
+
rescue => e
|
179
|
+
@logger.error("Could not declare exchange", error_details(e, :exchange => exchange, :type => exchange_type, :durable => durable))
|
180
|
+
|
194
181
|
raise e
|
195
182
|
end
|
196
183
|
|
197
184
|
def connect
|
198
|
-
@logger.debug? && @logger.debug("Connecting to RabbitMQ
|
199
|
-
|
200
|
-
|
185
|
+
@logger.debug? && @logger.debug("Connecting to RabbitMQ", rabbitmq_settings)
|
186
|
+
|
187
|
+
# disable MarchHare's attempt to provide a "better" exception logging experience:
|
188
|
+
settings = rabbitmq_settings.merge :exception_handler => com.rabbitmq.client.impl.ForgivingExceptionHandler.new
|
189
|
+
connection = MarchHare.connect(settings) # MarchHare::Session.connect
|
190
|
+
# we could pass down the :logger => logger but that adds an extra:
|
191
|
+
# `logger.info("Using TLS/SSL version #{tls}")` which isn't useful
|
192
|
+
# the rest of MH::Session logging is mostly debug level details
|
193
|
+
#
|
194
|
+
# NOTE: effectively redirects MarchHare's default std-out logging to LS
|
195
|
+
# (MARCH_HARE_LOG_LEVEL=debug no longer has an effect)
|
196
|
+
connection.instance_variable_set(:@logger, LoggerAdapter.new(logger))
|
201
197
|
|
202
198
|
connection.on_shutdown do |conn, cause|
|
203
|
-
@logger.warn("RabbitMQ connection was closed
|
204
|
-
:url => connection_url(conn),
|
205
|
-
:automatic_recovery => @automatic_recovery,
|
206
|
-
:cause => cause)
|
199
|
+
@logger.warn("RabbitMQ connection was closed", url: connection_url(conn), automatic_recovery: @automatic_recovery, cause: cause)
|
207
200
|
end
|
208
201
|
connection.on_blocked do
|
209
|
-
@logger.warn("RabbitMQ connection blocked
|
210
|
-
:url => connection_url(connection))
|
202
|
+
@logger.warn("RabbitMQ connection blocked - please check the RabbitMQ server logs", url: connection_url(connection))
|
211
203
|
end
|
212
204
|
connection.on_unblocked do
|
213
|
-
@logger.warn("RabbitMQ connection unblocked
|
205
|
+
@logger.warn("RabbitMQ connection unblocked", url: connection_url(connection))
|
214
206
|
end
|
215
207
|
|
216
208
|
channel = connection.create_channel
|
217
|
-
@logger.info("Connected to RabbitMQ
|
209
|
+
@logger.info("Connected to RabbitMQ", url: connection_url(connection))
|
218
210
|
|
219
211
|
HareInfo.new(connection, channel)
|
220
212
|
end
|
@@ -229,6 +221,44 @@ module LogStash
|
|
229
221
|
def sleep_for_retry
|
230
222
|
Stud.stoppable_sleep(@connect_retry_interval) { @rabbitmq_connection_stopping }
|
231
223
|
end
|
224
|
+
|
225
|
+
def error_details(e, info = {})
|
226
|
+
details = info.merge(:exception => e.class, :message => e.message)
|
227
|
+
if e.is_a?(MarchHare::Exception) && e.cause
|
228
|
+
details[:cause] = e.cause # likely a Java exception
|
229
|
+
end
|
230
|
+
details[:backtrace] = e.backtrace if @logger.debug? || info[:backtrace] == true
|
231
|
+
details
|
232
|
+
end
|
233
|
+
|
234
|
+
# @private adapting MarchHare's Ruby Logger assumptions
|
235
|
+
class LoggerAdapter < SimpleDelegator
|
236
|
+
|
237
|
+
java_import java.lang.Throwable
|
238
|
+
|
239
|
+
[:trace, :debug, :info, :warn, :error, :fatal].each do |level|
|
240
|
+
# sample logging used by MarchHare that we're after:
|
241
|
+
#
|
242
|
+
# rescue Exception => e
|
243
|
+
# logger.error("Caught exception when recovering queue #{q.name}")
|
244
|
+
# logger.error(e)
|
245
|
+
# end
|
246
|
+
class_eval <<-RUBY, __FILE__, __LINE__
|
247
|
+
def #{level}(arg)
|
248
|
+
if arg.is_a?(Exception) || arg.is_a?(Throwable)
|
249
|
+
details = { :exception => arg.class }
|
250
|
+
details[:cause] = arg.cause if arg.cause
|
251
|
+
details[:backtrace] = arg.backtrace
|
252
|
+
__getobj__.#{level}(arg.message.to_s, details)
|
253
|
+
else
|
254
|
+
__getobj__.#{level}(arg) # String
|
255
|
+
end
|
256
|
+
end
|
257
|
+
RUBY
|
258
|
+
end
|
259
|
+
|
260
|
+
end
|
261
|
+
|
232
262
|
end
|
233
263
|
end
|
234
264
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-rabbitmq'
|
3
|
-
s.version = '7.0
|
3
|
+
s.version = '7.3.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Integration with RabbitMQ - input and output plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
|
@@ -47,7 +47,7 @@ Gem::Specification.new do |s|
|
|
47
47
|
s.add_runtime_dependency 'stud', '~> 0.0.22'
|
48
48
|
s.add_runtime_dependency 'back_pressure', '~> 1.0'
|
49
49
|
|
50
|
-
s.add_development_dependency 'logstash-devutils'
|
50
|
+
s.add_development_dependency 'logstash-devutils', '~>2.0'
|
51
51
|
s.add_development_dependency 'logstash-input-generator'
|
52
52
|
s.add_development_dependency 'logstash-codec-plain'
|
53
53
|
end
|
@@ -21,8 +21,13 @@ describe LogStash::Inputs::RabbitMQ do
|
|
21
21
|
"prefetch_count" => 123
|
22
22
|
}
|
23
23
|
}
|
24
|
-
|
24
|
+
subject(:instance) { klass.new(rabbitmq_settings) }
|
25
25
|
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
26
|
+
let(:instance_logger) { double("Logger").as_null_object }
|
27
|
+
|
28
|
+
before do
|
29
|
+
allow_any_instance_of(klass).to receive(:logger).and_return(instance_logger)
|
30
|
+
end
|
26
31
|
|
27
32
|
context "when connected" do
|
28
33
|
let(:connection) { double("MarchHare Connection") }
|
@@ -39,6 +44,11 @@ describe LogStash::Inputs::RabbitMQ do
|
|
39
44
|
allow(connection).to receive(:on_shutdown)
|
40
45
|
allow(connection).to receive(:on_blocked)
|
41
46
|
allow(connection).to receive(:on_unblocked)
|
47
|
+
allow(connection).to receive(:close)
|
48
|
+
allow(connection).to receive(:host).and_return host
|
49
|
+
allow(connection).to receive(:port).and_return port
|
50
|
+
allow(connection).to receive(:vhost).and_return nil
|
51
|
+
allow(connection).to receive(:user).and_return 'guest'
|
42
52
|
allow(channel).to receive(:exchange).and_return(exchange)
|
43
53
|
allow(channel).to receive(:queue).and_return(queue)
|
44
54
|
allow(channel).to receive(:prefetch=)
|
@@ -74,7 +84,7 @@ describe LogStash::Inputs::RabbitMQ do
|
|
74
84
|
context "with an exchange declared" do
|
75
85
|
let(:exchange) { "exchange" }
|
76
86
|
let(:key) { "routing key" }
|
77
|
-
let(:rabbitmq_settings) { super.merge("exchange" => exchange, "key" => key, "exchange_type" => "fanout") }
|
87
|
+
let(:rabbitmq_settings) { super().merge("exchange" => exchange, "key" => key, "exchange_type" => "fanout") }
|
78
88
|
|
79
89
|
before do
|
80
90
|
allow(instance).to receive(:declare_exchange!)
|
@@ -127,11 +137,193 @@ describe LogStash::Inputs::RabbitMQ do
|
|
127
137
|
end
|
128
138
|
end
|
129
139
|
end
|
140
|
+
|
141
|
+
context '#register' do
|
142
|
+
let(:rabbitmq_settings) { super().merge(metadata_enabled_override) }
|
143
|
+
let(:metadata_enabled_override) { { "metadata_enabled" => metadata_enabled } }
|
144
|
+
before do
|
145
|
+
instance.register
|
146
|
+
end
|
147
|
+
|
148
|
+
shared_examples('`metadata_enabled => none`') do
|
149
|
+
context 'metadata_level' do
|
150
|
+
subject(:metadata_level) { instance.metadata_level }
|
151
|
+
it { is_expected.to be_empty }
|
152
|
+
it { is_expected.to be_frozen }
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
shared_examples('`metadata_enabled => basic`') do
|
157
|
+
context 'metadata_level' do
|
158
|
+
subject(:metadata_level) { instance.metadata_level }
|
159
|
+
it { is_expected.to include :headers }
|
160
|
+
it { is_expected.to include :properties }
|
161
|
+
it { is_expected.to_not include :payload }
|
162
|
+
it { is_expected.to be_frozen }
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
shared_examples("deprecated `metadata_enabled` setting") do |deprecated_value|
|
167
|
+
context 'the logger' do
|
168
|
+
subject(:logger) { instance_logger }
|
169
|
+
it 'receives a useful deprecation warning' do
|
170
|
+
expect(logger).to have_received(:warn).with(/Deprecated value `#{Regexp.escape(deprecated_value)}`/)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
context 'when `metadata_enabled` is `true`' do
|
176
|
+
let(:metadata_enabled) { "true" }
|
177
|
+
it_behaves_like '`metadata_enabled => basic`'
|
178
|
+
include_examples "deprecated `metadata_enabled` setting", "true"
|
179
|
+
end
|
180
|
+
|
181
|
+
context 'when `metadata_enabled` is `false`' do
|
182
|
+
let(:metadata_enabled) { "false" }
|
183
|
+
it_behaves_like '`metadata_enabled => none`'
|
184
|
+
include_examples "deprecated `metadata_enabled` setting", "false"
|
185
|
+
end
|
186
|
+
|
187
|
+
context 'when `metadata_enabled` is not provided' do
|
188
|
+
let(:metadata_enabled_override) { Hash.new }
|
189
|
+
it_behaves_like '`metadata_enabled => none`'
|
190
|
+
end
|
191
|
+
|
192
|
+
context 'when `metadata_enabled` is `basic`' do
|
193
|
+
let(:metadata_enabled) { "basic" }
|
194
|
+
include_examples '`metadata_enabled => basic`'
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'when `metadata_enabled` is `none`' do
|
198
|
+
let(:metadata_enabled) { "none" }
|
199
|
+
include_examples '`metadata_enabled => none`'
|
200
|
+
end
|
201
|
+
|
202
|
+
context 'when `metadata_enabled` is `extended`' do
|
203
|
+
let(:metadata_enabled) { "extended" }
|
204
|
+
context 'metadata_level' do
|
205
|
+
subject(:metadata_level) { instance.metadata_level }
|
206
|
+
it { is_expected.to include :headers }
|
207
|
+
it { is_expected.to include :properties }
|
208
|
+
it { is_expected.to include :payload }
|
209
|
+
it { is_expected.to be_frozen }
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
describe "#decorate(event, metadata, data)" do
|
215
|
+
let(:rabbitmq_settings) do
|
216
|
+
super().merge("metadata_enabled" => metadata_enabled)
|
217
|
+
end
|
218
|
+
before(:each) { instance.register }
|
219
|
+
|
220
|
+
let(:metadata) { double("METADATA") }
|
221
|
+
let(:headers) { Hash("header_key"=>"header_value") }
|
222
|
+
let(:properties) { Hash("property_key"=>"property_value") }
|
223
|
+
|
224
|
+
let(:data) { %Q({"message"=>"fubar"}\n) }
|
225
|
+
|
226
|
+
before do
|
227
|
+
allow(instance).to receive(:get_headers).with(metadata).and_return(headers)
|
228
|
+
allow(instance).to receive(:get_properties).with(metadata).and_return(properties)
|
229
|
+
end
|
230
|
+
|
231
|
+
describe 'the decorated event' do
|
232
|
+
subject(:decorated_event) do
|
233
|
+
LogStash::Event.new("message"=>"fubar").tap do |e|
|
234
|
+
instance.decorate(e, metadata, data)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
matcher :include_field do |fieldref|
|
239
|
+
match do |event|
|
240
|
+
# setting `@actual` makes failure messages clearer
|
241
|
+
@actual = event.to_hash_with_metadata
|
242
|
+
|
243
|
+
break false unless event.include?(fieldref)
|
244
|
+
break true unless @specific_value
|
245
|
+
|
246
|
+
values_match?(@expected_value, event.get(fieldref))
|
247
|
+
end
|
248
|
+
chain :with_value do |expected_value|
|
249
|
+
@specific_value = true
|
250
|
+
@expected_value = expected_value
|
251
|
+
end
|
252
|
+
description do
|
253
|
+
desc = "include field `#{fieldref}`"
|
254
|
+
desc += " with value matching `#{@expected_value.inspect}`" if @specific_value
|
255
|
+
desc
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
shared_examples('core decoration features') do
|
260
|
+
let(:rabbitmq_settings) do
|
261
|
+
super().merge("type" => "decorated_type",
|
262
|
+
"add_field" => {"added_field" => "field_value"})
|
263
|
+
end
|
264
|
+
it 'has been decorated with core decoration features' do
|
265
|
+
expect(decorated_event).to include_field("added_field").with_value("field_value")
|
266
|
+
expect(decorated_event).to include_field("type").with_value("decorated_type")
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
let(:headers_fieldref) { "[@metadata][rabbitmq_headers]" }
|
271
|
+
let(:properties_fieldref) { "[@metadata][rabbitmq_properties]" }
|
272
|
+
let(:payload_fieldref) { "[@metadata][rabbitmq_payload]" }
|
273
|
+
|
274
|
+
shared_examples('`metadata_enabled => none`') do
|
275
|
+
it { is_expected.to_not include_field(headers_fieldref) }
|
276
|
+
it { is_expected.to_not include_field(properties_fieldref) }
|
277
|
+
it { is_expected.to_not include_field(payload_fieldref) }
|
278
|
+
|
279
|
+
include_examples 'core decoration features'
|
280
|
+
end
|
281
|
+
|
282
|
+
shared_examples('`metadata_enabled => basic`') do
|
283
|
+
it { is_expected.to include_field(headers_fieldref).with_value(headers) }
|
284
|
+
it { is_expected.to include_field(properties_fieldref).with_value(properties) }
|
285
|
+
it { is_expected.to_not include_field(payload_fieldref) }
|
286
|
+
|
287
|
+
include_examples 'core decoration features'
|
288
|
+
end
|
289
|
+
|
290
|
+
context "with `metadata_enabled => none`" do
|
291
|
+
let(:metadata_enabled) { "none" }
|
292
|
+
include_examples '`metadata_enabled => none`'
|
293
|
+
end
|
294
|
+
|
295
|
+
context "with `metadata_enabled => basic`" do
|
296
|
+
let(:metadata_enabled) { "basic" }
|
297
|
+
include_examples '`metadata_enabled => basic`'
|
298
|
+
end
|
299
|
+
|
300
|
+
context 'with `metadata_enabled => extended`' do
|
301
|
+
let(:metadata_enabled) { "extended" }
|
302
|
+
it { is_expected.to include_field(headers_fieldref).with_value(headers) }
|
303
|
+
it { is_expected.to include_field(properties_fieldref).with_value(properties) }
|
304
|
+
it { is_expected.to include_field(payload_fieldref).with_value(data) }
|
305
|
+
include_examples 'core decoration features'
|
306
|
+
end
|
307
|
+
|
308
|
+
# Deprecated alias: false -> none
|
309
|
+
context "with `metadata_enabled => false`" do
|
310
|
+
let(:metadata_enabled) { "false" }
|
311
|
+
it_behaves_like '`metadata_enabled => none`'
|
312
|
+
end
|
313
|
+
|
314
|
+
# Deprecated alias: true -> basic
|
315
|
+
context "with `metadata_enabled => true`" do
|
316
|
+
let(:metadata_enabled) { "true" }
|
317
|
+
it_behaves_like '`metadata_enabled => basic`'
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
130
321
|
end
|
131
322
|
|
132
|
-
describe "with a live server", :integration => true do
|
323
|
+
describe "LogStash::Inputs::RabbitMQ with a live server", :integration => true do
|
133
324
|
let(:klass) { LogStash::Inputs::RabbitMQ }
|
134
|
-
let(:
|
325
|
+
let(:rabbitmq_host) { ENV["RABBITMQ_HOST"] || "127.0.0.1" }
|
326
|
+
let(:config) { {"host" => rabbitmq_host, "auto_delete" => true, "codec" => "plain", "add_field" => {"[@metadata][foo]" => "bar"} } }
|
135
327
|
let(:instance) { klass.new(config) }
|
136
328
|
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
137
329
|
let(:output_queue) { Queue.new }
|
@@ -147,7 +339,7 @@ describe "with a live server", :integration => true do
|
|
147
339
|
}
|
148
340
|
|
149
341
|
20.times do
|
150
|
-
instance.
|
342
|
+
instance.send(:connection_open?) ? break : sleep(0.1)
|
151
343
|
end
|
152
344
|
|
153
345
|
# Extra time to make sure the consumer can attach
|
@@ -176,16 +368,16 @@ describe "with a live server", :integration => true do
|
|
176
368
|
|
177
369
|
context "using defaults" do
|
178
370
|
it "should start, connect, and stop cleanly" do
|
179
|
-
expect(instance.
|
371
|
+
expect(instance.send(:connection_open?)).to be_truthy
|
180
372
|
end
|
181
373
|
end
|
182
374
|
|
183
375
|
it "should have the correct prefetch value" do
|
184
|
-
expect(
|
376
|
+
expect(hare_info.channel.prefetch).to eql(256)
|
185
377
|
end
|
186
378
|
|
187
379
|
describe "receiving a message with a queue + exchange specified" do
|
188
|
-
let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => true) }
|
380
|
+
let(:config) { super().merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => "true") }
|
189
381
|
let(:event) { output_queue.pop }
|
190
382
|
let(:exchange) { test_channel.exchange(exchange_name, :type => "fanout") }
|
191
383
|
let(:exchange_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
|
@@ -237,7 +429,7 @@ describe "with a live server", :integration => true do
|
|
237
429
|
expect(event).to include("@metadata")
|
238
430
|
expect(event.get("@metadata")).to include("rabbitmq_properties")
|
239
431
|
|
240
|
-
props = event.get("[@metadata][rabbitmq_properties")
|
432
|
+
props = event.get("[@metadata][rabbitmq_properties]")
|
241
433
|
expect(props["app-id"]).to eq(app_id)
|
242
434
|
expect(props["delivery-mode"]).to eq(1)
|
243
435
|
expect(props["exchange"]).to eq(exchange_name)
|
@@ -273,7 +465,34 @@ describe "with a live server", :integration => true do
|
|
273
465
|
end
|
274
466
|
end
|
275
467
|
|
468
|
+
context "(MarchHare) error logging" do
|
469
|
+
|
470
|
+
let(:error) do
|
471
|
+
MarchHare::Exception.new('TEST ERROR').tap do |error|
|
472
|
+
allow( error ).to receive(:cause).and_return(error_cause)
|
473
|
+
end
|
474
|
+
end
|
475
|
+
let(:error_cause) { java.io.IOException.new('TEST CAUSE') }
|
476
|
+
let(:logger) { instance.logger }
|
477
|
+
|
478
|
+
before do
|
479
|
+
queues = hare_info.channel.instance_variable_get(:@queues)
|
480
|
+
expect( queue = queues.values.first ).to_not be nil
|
481
|
+
# emulate an issue during recovery (to trigger logger.error calls)
|
482
|
+
allow( queue ).to receive(:recover_from_network_failure).and_raise(error)
|
483
|
+
allow( logger ).to receive(:error)
|
484
|
+
end
|
485
|
+
|
486
|
+
it "gets redirected to plugin logger" do
|
487
|
+
hare_info.channel.recover_queues
|
488
|
+
expect( logger ).to have_received(:error).with(/Caught exception when recovering queue/i)
|
489
|
+
expect( logger ).to have_received(:error).with('TEST ERROR', hash_including(exception: MarchHare::Exception, cause: error_cause))
|
490
|
+
end
|
491
|
+
|
492
|
+
end
|
493
|
+
|
276
494
|
describe LogStash::Inputs::RabbitMQ do
|
495
|
+
require "logstash/devutils/rspec/shared_examples"
|
277
496
|
it_behaves_like "an interruptible input plugin"
|
278
497
|
end
|
279
498
|
end
|