logstash-integration-rabbitmq 7.0.2-java → 7.3.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -0
- data/LICENSE +199 -10
- data/README.md +1 -1
- data/docs/input-rabbitmq.asciidoc +13 -4
- data/docs/output-rabbitmq.asciidoc +4 -2
- data/lib/logstash/inputs/rabbitmq.rb +72 -16
- data/lib/logstash/outputs/rabbitmq.rb +71 -10
- data/lib/logstash/plugin_mixins/rabbitmq_connection.rb +75 -45
- data/logstash-integration-rabbitmq.gemspec +2 -2
- data/spec/inputs/rabbitmq_spec.rb +228 -9
- data/spec/outputs/rabbitmq_spec.rb +61 -9
- data/spec/plugin_mixins/rabbitmq_connection_spec.rb +48 -23
- metadata +7 -7
@@ -1,14 +1,12 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
require "logstash/pipeline"
|
3
3
|
require_relative '../plugin_mixins/rabbitmq_connection'
|
4
|
-
java_import java.util.concurrent.TimeoutException
|
5
|
-
java_import com.rabbitmq.client.AlreadyClosedException
|
6
4
|
|
7
5
|
require 'back_pressure'
|
8
6
|
|
9
7
|
# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
|
10
8
|
# or later version (3.x is recommended).
|
11
|
-
#
|
9
|
+
#
|
12
10
|
# Relevant links:
|
13
11
|
#
|
14
12
|
# * http://www.rabbitmq.com/[RabbitMQ]
|
@@ -16,10 +14,14 @@ require 'back_pressure'
|
|
16
14
|
module LogStash
|
17
15
|
module Outputs
|
18
16
|
class RabbitMQ < LogStash::Outputs::Base
|
17
|
+
|
18
|
+
java_import java.util.concurrent.TimeoutException
|
19
|
+
java_import com.rabbitmq.client.AlreadyClosedException
|
20
|
+
|
19
21
|
include LogStash::PluginMixins::RabbitMQConnection
|
20
22
|
|
21
23
|
config_name "rabbitmq"
|
22
|
-
|
24
|
+
|
23
25
|
concurrency :shared
|
24
26
|
|
25
27
|
# The default codec for this plugin is JSON. You can override this to suit your particular needs however.
|
@@ -46,6 +48,8 @@ module LogStash
|
|
46
48
|
config :message_properties, :validate => :hash, :default => {}
|
47
49
|
|
48
50
|
def register
|
51
|
+
@message_properties_template = MessagePropertiesTemplate.new(symbolize(@message_properties).merge(:persistent => @persistent))
|
52
|
+
|
49
53
|
connect!
|
50
54
|
@hare_info.exchange = declare_exchange!(@hare_info.channel, @exchange, @exchange_type, @durable)
|
51
55
|
# The connection close should close all channels, so it is safe to store thread locals here without closing them
|
@@ -67,14 +71,13 @@ module LogStash
|
|
67
71
|
|
68
72
|
def publish(event, message)
|
69
73
|
raise ArgumentError, "No exchange set in HareInfo!!!" unless @hare_info.exchange
|
74
|
+
routing_key = event.sprintf(@key)
|
75
|
+
message_properties = @message_properties_template.build(event)
|
70
76
|
@gated_executor.execute do
|
71
|
-
local_exchange.publish(message, :routing_key =>
|
77
|
+
local_exchange.publish(message, :routing_key => routing_key, :properties => message_properties)
|
72
78
|
end
|
73
79
|
rescue MarchHare::Exception, IOError, AlreadyClosedException, TimeoutException => e
|
74
|
-
@logger.error("Error while publishing
|
75
|
-
:message => e.message,
|
76
|
-
:exception => e.class,
|
77
|
-
:backtrace => e.backtrace)
|
80
|
+
@logger.error("Error while publishing, will retry", error_details(e, backtrace: true))
|
78
81
|
|
79
82
|
sleep_for_retry
|
80
83
|
retry
|
@@ -126,6 +129,64 @@ module LogStash
|
|
126
129
|
end
|
127
130
|
end
|
128
131
|
end
|
132
|
+
|
133
|
+
##
|
134
|
+
# A `MessagePropertiesTemplate` efficiently produces per-event message properties from the
|
135
|
+
# provided template Hash.
|
136
|
+
#
|
137
|
+
# In order to efficiently reuse constant-value objects, returned values may be frozen.
|
138
|
+
class MessagePropertiesTemplate
|
139
|
+
##
|
140
|
+
# Creates a new `MessagePropertiesTemplate` from the provided `template`
|
141
|
+
# @param template [Hash{Symbol=>Object}]
|
142
|
+
def initialize(template)
|
143
|
+
constant_properties = template.reject { |_,v| templated?(v) }
|
144
|
+
variable_properties = template.select { |_,v| templated?(v) }
|
145
|
+
|
146
|
+
@constant_properties = normalize(constant_properties).freeze
|
147
|
+
@variable_properties = variable_properties
|
148
|
+
end
|
149
|
+
|
150
|
+
##
|
151
|
+
# Builds a property mapping for the given `event`, including templated values.
|
152
|
+
#
|
153
|
+
# @param event [LogStash::Event]: the event with which to populated templated values, if any.
|
154
|
+
# @return [Hash{Symbol=>Object}] a possibly-frozen properties hash for the provided `event`.
|
155
|
+
def build(event)
|
156
|
+
return @constant_properties if @variable_properties.empty?
|
157
|
+
|
158
|
+
properties = @variable_properties.each_with_object(@constant_properties.dup) do |(k,v), memo|
|
159
|
+
memo.store(k, event.sprintf(v))
|
160
|
+
end
|
161
|
+
|
162
|
+
return normalize(properties)
|
163
|
+
end
|
164
|
+
|
165
|
+
private
|
166
|
+
|
167
|
+
##
|
168
|
+
# Normalize the provided property mapping with respect to the value types the underlying
|
169
|
+
# client expects.
|
170
|
+
#
|
171
|
+
# @api private
|
172
|
+
# @param properties [Hash{Symbol=>Object}]: a possibly-frozen Hash whose values may need type-coercion.
|
173
|
+
# @return [Hash{Symbol=>Object}]
|
174
|
+
def normalize(properties)
|
175
|
+
if properties[:priority] && properties[:priority].kind_of?(String)
|
176
|
+
properties = properties.merge(:priority => properties[:priority].to_i)
|
177
|
+
end
|
178
|
+
|
179
|
+
properties
|
180
|
+
end
|
181
|
+
|
182
|
+
##
|
183
|
+
# @api private
|
184
|
+
# @param [Object]: an object, which may or may not be a template `String`
|
185
|
+
# @return [Boolean]: returns `true` IFF `value` is a template `String`
|
186
|
+
def templated?(value)
|
187
|
+
value.kind_of?(String) && value.include?('%{')
|
188
|
+
end
|
189
|
+
end
|
129
190
|
end
|
130
191
|
end
|
131
|
-
end
|
192
|
+
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
require "logstash/namespace"
|
3
3
|
require "march_hare"
|
4
4
|
require "java"
|
5
|
+
require "stud/interval"
|
5
6
|
|
6
7
|
# Common functionality for the rabbitmq input/output
|
7
8
|
module LogStash
|
@@ -104,17 +105,17 @@ module LogStash
|
|
104
105
|
def rabbitmq_settings
|
105
106
|
return @rabbitmq_settings if @rabbitmq_settings
|
106
107
|
|
108
|
+
|
107
109
|
s = {
|
108
110
|
:vhost => @vhost,
|
109
|
-
:
|
110
|
-
:
|
111
|
-
:user => @user,
|
111
|
+
:addresses => addresses_from_hosts_and_port(@host, @port),
|
112
|
+
:username => @user,
|
112
113
|
:automatic_recovery => @automatic_recovery,
|
113
|
-
:
|
114
|
+
:password => @password ? @password.value : "guest",
|
114
115
|
}
|
115
116
|
|
116
|
-
s[:
|
117
|
-
s[:
|
117
|
+
s[:connection_timeout] = @connection_timeout || 0
|
118
|
+
s[:requested_heartbeat] = @heartbeat || 0
|
118
119
|
|
119
120
|
if @ssl
|
120
121
|
s[:tls] = @ssl_version
|
@@ -133,10 +134,15 @@ module LogStash
|
|
133
134
|
@rabbitmq_settings = s
|
134
135
|
end
|
135
136
|
|
137
|
+
def addresses_from_hosts_and_port(hosts, port)
|
138
|
+
hosts.map {|host| host.include?(':') ? host : "#{host}:#{port}"}
|
139
|
+
end
|
140
|
+
|
141
|
+
|
136
142
|
def connect!
|
137
143
|
@hare_info = connect() unless @hare_info # Don't duplicate the conn!
|
138
144
|
rescue MarchHare::Exception, java.io.IOException => e
|
139
|
-
|
145
|
+
message = if e.message.empty? && e.is_a?(java.io.IOException)
|
140
146
|
# IOException with an empty message is probably an instance of
|
141
147
|
# these problems:
|
142
148
|
# https://github.com/logstash-plugins/logstash-output-rabbitmq/issues/52
|
@@ -145,21 +151,12 @@ module LogStash
|
|
145
151
|
# Best guess is to help the user understand that there is probably
|
146
152
|
# some kind of configuration problem causing the error, but we
|
147
153
|
# can't really offer any more detailed hints :\
|
148
|
-
"An unknown error occurred
|
154
|
+
"An unknown RabbitMQ error occurred, maybe this is a configuration error (invalid vhost, for example) - please check the RabbitMQ server logs for clues about this failure"
|
149
155
|
else
|
150
|
-
|
156
|
+
"RabbitMQ connection error, will retry"
|
151
157
|
end
|
152
158
|
|
153
|
-
|
154
|
-
@logger.error("RabbitMQ connection error, will retry.",
|
155
|
-
:error_message => error_message,
|
156
|
-
:exception => e.class.name,
|
157
|
-
:backtrace => e.backtrace)
|
158
|
-
else
|
159
|
-
@logger.error("RabbitMQ connection error, will retry.",
|
160
|
-
:error_message => error_message,
|
161
|
-
:exception => e.class.name)
|
162
|
-
end
|
159
|
+
@logger.error(message, error_details(e))
|
163
160
|
|
164
161
|
sleep_for_retry
|
165
162
|
retry
|
@@ -173,48 +170,43 @@ module LogStash
|
|
173
170
|
@hare_info && @hare_info.connection && @hare_info.connection.open?
|
174
171
|
end
|
175
172
|
|
176
|
-
def connected?
|
177
|
-
return nil unless @hare_info && @hare_info.connection
|
178
|
-
@hare_info.connection.connected?
|
179
|
-
end
|
180
|
-
|
181
173
|
private
|
182
174
|
|
183
175
|
def declare_exchange!(channel, exchange, exchange_type, durable)
|
184
|
-
@logger.debug? && @logger.debug("Declaring an exchange", :name => exchange,
|
185
|
-
|
186
|
-
|
187
|
-
@logger.
|
188
|
-
|
189
|
-
rescue StandardError => e
|
190
|
-
@logger.error("Could not declare exchange!",
|
191
|
-
:exchange => exchange, :type => exchange_type,
|
192
|
-
:durable => durable, :error_class => e.class.name,
|
193
|
-
:error_message => e.message, :backtrace => e.backtrace)
|
176
|
+
@logger.debug? && @logger.debug("Declaring an exchange", :name => exchange, :type => exchange_type, :durable => durable)
|
177
|
+
channel.exchange(exchange, :type => exchange_type.to_sym, :durable => durable)
|
178
|
+
rescue => e
|
179
|
+
@logger.error("Could not declare exchange", error_details(e, :exchange => exchange, :type => exchange_type, :durable => durable))
|
180
|
+
|
194
181
|
raise e
|
195
182
|
end
|
196
183
|
|
197
184
|
def connect
|
198
|
-
@logger.debug? && @logger.debug("Connecting to RabbitMQ
|
199
|
-
|
200
|
-
|
185
|
+
@logger.debug? && @logger.debug("Connecting to RabbitMQ", rabbitmq_settings)
|
186
|
+
|
187
|
+
# disable MarchHare's attempt to provide a "better" exception logging experience:
|
188
|
+
settings = rabbitmq_settings.merge :exception_handler => com.rabbitmq.client.impl.ForgivingExceptionHandler.new
|
189
|
+
connection = MarchHare.connect(settings) # MarchHare::Session.connect
|
190
|
+
# we could pass down the :logger => logger but that adds an extra:
|
191
|
+
# `logger.info("Using TLS/SSL version #{tls}")` which isn't useful
|
192
|
+
# the rest of MH::Session logging is mostly debug level details
|
193
|
+
#
|
194
|
+
# NOTE: effectively redirects MarchHare's default std-out logging to LS
|
195
|
+
# (MARCH_HARE_LOG_LEVEL=debug no longer has an effect)
|
196
|
+
connection.instance_variable_set(:@logger, LoggerAdapter.new(logger))
|
201
197
|
|
202
198
|
connection.on_shutdown do |conn, cause|
|
203
|
-
@logger.warn("RabbitMQ connection was closed
|
204
|
-
:url => connection_url(conn),
|
205
|
-
:automatic_recovery => @automatic_recovery,
|
206
|
-
:cause => cause)
|
199
|
+
@logger.warn("RabbitMQ connection was closed", url: connection_url(conn), automatic_recovery: @automatic_recovery, cause: cause)
|
207
200
|
end
|
208
201
|
connection.on_blocked do
|
209
|
-
@logger.warn("RabbitMQ connection blocked
|
210
|
-
:url => connection_url(connection))
|
202
|
+
@logger.warn("RabbitMQ connection blocked - please check the RabbitMQ server logs", url: connection_url(connection))
|
211
203
|
end
|
212
204
|
connection.on_unblocked do
|
213
|
-
@logger.warn("RabbitMQ connection unblocked
|
205
|
+
@logger.warn("RabbitMQ connection unblocked", url: connection_url(connection))
|
214
206
|
end
|
215
207
|
|
216
208
|
channel = connection.create_channel
|
217
|
-
@logger.info("Connected to RabbitMQ
|
209
|
+
@logger.info("Connected to RabbitMQ", url: connection_url(connection))
|
218
210
|
|
219
211
|
HareInfo.new(connection, channel)
|
220
212
|
end
|
@@ -229,6 +221,44 @@ module LogStash
|
|
229
221
|
def sleep_for_retry
|
230
222
|
Stud.stoppable_sleep(@connect_retry_interval) { @rabbitmq_connection_stopping }
|
231
223
|
end
|
224
|
+
|
225
|
+
def error_details(e, info = {})
|
226
|
+
details = info.merge(:exception => e.class, :message => e.message)
|
227
|
+
if e.is_a?(MarchHare::Exception) && e.cause
|
228
|
+
details[:cause] = e.cause # likely a Java exception
|
229
|
+
end
|
230
|
+
details[:backtrace] = e.backtrace if @logger.debug? || info[:backtrace] == true
|
231
|
+
details
|
232
|
+
end
|
233
|
+
|
234
|
+
# @private adapting MarchHare's Ruby Logger assumptions
|
235
|
+
class LoggerAdapter < SimpleDelegator
|
236
|
+
|
237
|
+
java_import java.lang.Throwable
|
238
|
+
|
239
|
+
[:trace, :debug, :info, :warn, :error, :fatal].each do |level|
|
240
|
+
# sample logging used by MarchHare that we're after:
|
241
|
+
#
|
242
|
+
# rescue Exception => e
|
243
|
+
# logger.error("Caught exception when recovering queue #{q.name}")
|
244
|
+
# logger.error(e)
|
245
|
+
# end
|
246
|
+
class_eval <<-RUBY, __FILE__, __LINE__
|
247
|
+
def #{level}(arg)
|
248
|
+
if arg.is_a?(Exception) || arg.is_a?(Throwable)
|
249
|
+
details = { :exception => arg.class }
|
250
|
+
details[:cause] = arg.cause if arg.cause
|
251
|
+
details[:backtrace] = arg.backtrace
|
252
|
+
__getobj__.#{level}(arg.message.to_s, details)
|
253
|
+
else
|
254
|
+
__getobj__.#{level}(arg) # String
|
255
|
+
end
|
256
|
+
end
|
257
|
+
RUBY
|
258
|
+
end
|
259
|
+
|
260
|
+
end
|
261
|
+
|
232
262
|
end
|
233
263
|
end
|
234
264
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-rabbitmq'
|
3
|
-
s.version = '7.0
|
3
|
+
s.version = '7.3.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Integration with RabbitMQ - input and output plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
|
@@ -47,7 +47,7 @@ Gem::Specification.new do |s|
|
|
47
47
|
s.add_runtime_dependency 'stud', '~> 0.0.22'
|
48
48
|
s.add_runtime_dependency 'back_pressure', '~> 1.0'
|
49
49
|
|
50
|
-
s.add_development_dependency 'logstash-devutils'
|
50
|
+
s.add_development_dependency 'logstash-devutils', '~>2.0'
|
51
51
|
s.add_development_dependency 'logstash-input-generator'
|
52
52
|
s.add_development_dependency 'logstash-codec-plain'
|
53
53
|
end
|
@@ -21,8 +21,13 @@ describe LogStash::Inputs::RabbitMQ do
|
|
21
21
|
"prefetch_count" => 123
|
22
22
|
}
|
23
23
|
}
|
24
|
-
|
24
|
+
subject(:instance) { klass.new(rabbitmq_settings) }
|
25
25
|
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
26
|
+
let(:instance_logger) { double("Logger").as_null_object }
|
27
|
+
|
28
|
+
before do
|
29
|
+
allow_any_instance_of(klass).to receive(:logger).and_return(instance_logger)
|
30
|
+
end
|
26
31
|
|
27
32
|
context "when connected" do
|
28
33
|
let(:connection) { double("MarchHare Connection") }
|
@@ -39,6 +44,11 @@ describe LogStash::Inputs::RabbitMQ do
|
|
39
44
|
allow(connection).to receive(:on_shutdown)
|
40
45
|
allow(connection).to receive(:on_blocked)
|
41
46
|
allow(connection).to receive(:on_unblocked)
|
47
|
+
allow(connection).to receive(:close)
|
48
|
+
allow(connection).to receive(:host).and_return host
|
49
|
+
allow(connection).to receive(:port).and_return port
|
50
|
+
allow(connection).to receive(:vhost).and_return nil
|
51
|
+
allow(connection).to receive(:user).and_return 'guest'
|
42
52
|
allow(channel).to receive(:exchange).and_return(exchange)
|
43
53
|
allow(channel).to receive(:queue).and_return(queue)
|
44
54
|
allow(channel).to receive(:prefetch=)
|
@@ -74,7 +84,7 @@ describe LogStash::Inputs::RabbitMQ do
|
|
74
84
|
context "with an exchange declared" do
|
75
85
|
let(:exchange) { "exchange" }
|
76
86
|
let(:key) { "routing key" }
|
77
|
-
let(:rabbitmq_settings) { super.merge("exchange" => exchange, "key" => key, "exchange_type" => "fanout") }
|
87
|
+
let(:rabbitmq_settings) { super().merge("exchange" => exchange, "key" => key, "exchange_type" => "fanout") }
|
78
88
|
|
79
89
|
before do
|
80
90
|
allow(instance).to receive(:declare_exchange!)
|
@@ -127,11 +137,193 @@ describe LogStash::Inputs::RabbitMQ do
|
|
127
137
|
end
|
128
138
|
end
|
129
139
|
end
|
140
|
+
|
141
|
+
context '#register' do
|
142
|
+
let(:rabbitmq_settings) { super().merge(metadata_enabled_override) }
|
143
|
+
let(:metadata_enabled_override) { { "metadata_enabled" => metadata_enabled } }
|
144
|
+
before do
|
145
|
+
instance.register
|
146
|
+
end
|
147
|
+
|
148
|
+
shared_examples('`metadata_enabled => none`') do
|
149
|
+
context 'metadata_level' do
|
150
|
+
subject(:metadata_level) { instance.metadata_level }
|
151
|
+
it { is_expected.to be_empty }
|
152
|
+
it { is_expected.to be_frozen }
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
shared_examples('`metadata_enabled => basic`') do
|
157
|
+
context 'metadata_level' do
|
158
|
+
subject(:metadata_level) { instance.metadata_level }
|
159
|
+
it { is_expected.to include :headers }
|
160
|
+
it { is_expected.to include :properties }
|
161
|
+
it { is_expected.to_not include :payload }
|
162
|
+
it { is_expected.to be_frozen }
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
shared_examples("deprecated `metadata_enabled` setting") do |deprecated_value|
|
167
|
+
context 'the logger' do
|
168
|
+
subject(:logger) { instance_logger }
|
169
|
+
it 'receives a useful deprecation warning' do
|
170
|
+
expect(logger).to have_received(:warn).with(/Deprecated value `#{Regexp.escape(deprecated_value)}`/)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
context 'when `metadata_enabled` is `true`' do
|
176
|
+
let(:metadata_enabled) { "true" }
|
177
|
+
it_behaves_like '`metadata_enabled => basic`'
|
178
|
+
include_examples "deprecated `metadata_enabled` setting", "true"
|
179
|
+
end
|
180
|
+
|
181
|
+
context 'when `metadata_enabled` is `false`' do
|
182
|
+
let(:metadata_enabled) { "false" }
|
183
|
+
it_behaves_like '`metadata_enabled => none`'
|
184
|
+
include_examples "deprecated `metadata_enabled` setting", "false"
|
185
|
+
end
|
186
|
+
|
187
|
+
context 'when `metadata_enabled` is not provided' do
|
188
|
+
let(:metadata_enabled_override) { Hash.new }
|
189
|
+
it_behaves_like '`metadata_enabled => none`'
|
190
|
+
end
|
191
|
+
|
192
|
+
context 'when `metadata_enabled` is `basic`' do
|
193
|
+
let(:metadata_enabled) { "basic" }
|
194
|
+
include_examples '`metadata_enabled => basic`'
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'when `metadata_enabled` is `none`' do
|
198
|
+
let(:metadata_enabled) { "none" }
|
199
|
+
include_examples '`metadata_enabled => none`'
|
200
|
+
end
|
201
|
+
|
202
|
+
context 'when `metadata_enabled` is `extended`' do
|
203
|
+
let(:metadata_enabled) { "extended" }
|
204
|
+
context 'metadata_level' do
|
205
|
+
subject(:metadata_level) { instance.metadata_level }
|
206
|
+
it { is_expected.to include :headers }
|
207
|
+
it { is_expected.to include :properties }
|
208
|
+
it { is_expected.to include :payload }
|
209
|
+
it { is_expected.to be_frozen }
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
describe "#decorate(event, metadata, data)" do
|
215
|
+
let(:rabbitmq_settings) do
|
216
|
+
super().merge("metadata_enabled" => metadata_enabled)
|
217
|
+
end
|
218
|
+
before(:each) { instance.register }
|
219
|
+
|
220
|
+
let(:metadata) { double("METADATA") }
|
221
|
+
let(:headers) { Hash("header_key"=>"header_value") }
|
222
|
+
let(:properties) { Hash("property_key"=>"property_value") }
|
223
|
+
|
224
|
+
let(:data) { %Q({"message"=>"fubar"}\n) }
|
225
|
+
|
226
|
+
before do
|
227
|
+
allow(instance).to receive(:get_headers).with(metadata).and_return(headers)
|
228
|
+
allow(instance).to receive(:get_properties).with(metadata).and_return(properties)
|
229
|
+
end
|
230
|
+
|
231
|
+
describe 'the decorated event' do
|
232
|
+
subject(:decorated_event) do
|
233
|
+
LogStash::Event.new("message"=>"fubar").tap do |e|
|
234
|
+
instance.decorate(e, metadata, data)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
matcher :include_field do |fieldref|
|
239
|
+
match do |event|
|
240
|
+
# setting `@actual` makes failure messages clearer
|
241
|
+
@actual = event.to_hash_with_metadata
|
242
|
+
|
243
|
+
break false unless event.include?(fieldref)
|
244
|
+
break true unless @specific_value
|
245
|
+
|
246
|
+
values_match?(@expected_value, event.get(fieldref))
|
247
|
+
end
|
248
|
+
chain :with_value do |expected_value|
|
249
|
+
@specific_value = true
|
250
|
+
@expected_value = expected_value
|
251
|
+
end
|
252
|
+
description do
|
253
|
+
desc = "include field `#{fieldref}`"
|
254
|
+
desc += " with value matching `#{@expected_value.inspect}`" if @specific_value
|
255
|
+
desc
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
shared_examples('core decoration features') do
|
260
|
+
let(:rabbitmq_settings) do
|
261
|
+
super().merge("type" => "decorated_type",
|
262
|
+
"add_field" => {"added_field" => "field_value"})
|
263
|
+
end
|
264
|
+
it 'has been decorated with core decoration features' do
|
265
|
+
expect(decorated_event).to include_field("added_field").with_value("field_value")
|
266
|
+
expect(decorated_event).to include_field("type").with_value("decorated_type")
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
let(:headers_fieldref) { "[@metadata][rabbitmq_headers]" }
|
271
|
+
let(:properties_fieldref) { "[@metadata][rabbitmq_properties]" }
|
272
|
+
let(:payload_fieldref) { "[@metadata][rabbitmq_payload]" }
|
273
|
+
|
274
|
+
shared_examples('`metadata_enabled => none`') do
|
275
|
+
it { is_expected.to_not include_field(headers_fieldref) }
|
276
|
+
it { is_expected.to_not include_field(properties_fieldref) }
|
277
|
+
it { is_expected.to_not include_field(payload_fieldref) }
|
278
|
+
|
279
|
+
include_examples 'core decoration features'
|
280
|
+
end
|
281
|
+
|
282
|
+
shared_examples('`metadata_enabled => basic`') do
|
283
|
+
it { is_expected.to include_field(headers_fieldref).with_value(headers) }
|
284
|
+
it { is_expected.to include_field(properties_fieldref).with_value(properties) }
|
285
|
+
it { is_expected.to_not include_field(payload_fieldref) }
|
286
|
+
|
287
|
+
include_examples 'core decoration features'
|
288
|
+
end
|
289
|
+
|
290
|
+
context "with `metadata_enabled => none`" do
|
291
|
+
let(:metadata_enabled) { "none" }
|
292
|
+
include_examples '`metadata_enabled => none`'
|
293
|
+
end
|
294
|
+
|
295
|
+
context "with `metadata_enabled => basic`" do
|
296
|
+
let(:metadata_enabled) { "basic" }
|
297
|
+
include_examples '`metadata_enabled => basic`'
|
298
|
+
end
|
299
|
+
|
300
|
+
context 'with `metadata_enabled => extended`' do
|
301
|
+
let(:metadata_enabled) { "extended" }
|
302
|
+
it { is_expected.to include_field(headers_fieldref).with_value(headers) }
|
303
|
+
it { is_expected.to include_field(properties_fieldref).with_value(properties) }
|
304
|
+
it { is_expected.to include_field(payload_fieldref).with_value(data) }
|
305
|
+
include_examples 'core decoration features'
|
306
|
+
end
|
307
|
+
|
308
|
+
# Deprecated alias: false -> none
|
309
|
+
context "with `metadata_enabled => false`" do
|
310
|
+
let(:metadata_enabled) { "false" }
|
311
|
+
it_behaves_like '`metadata_enabled => none`'
|
312
|
+
end
|
313
|
+
|
314
|
+
# Deprecated alias: true -> basic
|
315
|
+
context "with `metadata_enabled => true`" do
|
316
|
+
let(:metadata_enabled) { "true" }
|
317
|
+
it_behaves_like '`metadata_enabled => basic`'
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
130
321
|
end
|
131
322
|
|
132
|
-
describe "with a live server", :integration => true do
|
323
|
+
describe "LogStash::Inputs::RabbitMQ with a live server", :integration => true do
|
133
324
|
let(:klass) { LogStash::Inputs::RabbitMQ }
|
134
|
-
let(:
|
325
|
+
let(:rabbitmq_host) { ENV["RABBITMQ_HOST"] || "127.0.0.1" }
|
326
|
+
let(:config) { {"host" => rabbitmq_host, "auto_delete" => true, "codec" => "plain", "add_field" => {"[@metadata][foo]" => "bar"} } }
|
135
327
|
let(:instance) { klass.new(config) }
|
136
328
|
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
137
329
|
let(:output_queue) { Queue.new }
|
@@ -147,7 +339,7 @@ describe "with a live server", :integration => true do
|
|
147
339
|
}
|
148
340
|
|
149
341
|
20.times do
|
150
|
-
instance.
|
342
|
+
instance.send(:connection_open?) ? break : sleep(0.1)
|
151
343
|
end
|
152
344
|
|
153
345
|
# Extra time to make sure the consumer can attach
|
@@ -176,16 +368,16 @@ describe "with a live server", :integration => true do
|
|
176
368
|
|
177
369
|
context "using defaults" do
|
178
370
|
it "should start, connect, and stop cleanly" do
|
179
|
-
expect(instance.
|
371
|
+
expect(instance.send(:connection_open?)).to be_truthy
|
180
372
|
end
|
181
373
|
end
|
182
374
|
|
183
375
|
it "should have the correct prefetch value" do
|
184
|
-
expect(
|
376
|
+
expect(hare_info.channel.prefetch).to eql(256)
|
185
377
|
end
|
186
378
|
|
187
379
|
describe "receiving a message with a queue + exchange specified" do
|
188
|
-
let(:config) { super.merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => true) }
|
380
|
+
let(:config) { super().merge("queue" => queue_name, "exchange" => exchange_name, "exchange_type" => "fanout", "metadata_enabled" => "true") }
|
189
381
|
let(:event) { output_queue.pop }
|
190
382
|
let(:exchange) { test_channel.exchange(exchange_name, :type => "fanout") }
|
191
383
|
let(:exchange_name) { "logstash-input-rabbitmq-#{rand(0xFFFFFFFF)}" }
|
@@ -237,7 +429,7 @@ describe "with a live server", :integration => true do
|
|
237
429
|
expect(event).to include("@metadata")
|
238
430
|
expect(event.get("@metadata")).to include("rabbitmq_properties")
|
239
431
|
|
240
|
-
props = event.get("[@metadata][rabbitmq_properties")
|
432
|
+
props = event.get("[@metadata][rabbitmq_properties]")
|
241
433
|
expect(props["app-id"]).to eq(app_id)
|
242
434
|
expect(props["delivery-mode"]).to eq(1)
|
243
435
|
expect(props["exchange"]).to eq(exchange_name)
|
@@ -273,7 +465,34 @@ describe "with a live server", :integration => true do
|
|
273
465
|
end
|
274
466
|
end
|
275
467
|
|
468
|
+
context "(MarchHare) error logging" do
|
469
|
+
|
470
|
+
let(:error) do
|
471
|
+
MarchHare::Exception.new('TEST ERROR').tap do |error|
|
472
|
+
allow( error ).to receive(:cause).and_return(error_cause)
|
473
|
+
end
|
474
|
+
end
|
475
|
+
let(:error_cause) { java.io.IOException.new('TEST CAUSE') }
|
476
|
+
let(:logger) { instance.logger }
|
477
|
+
|
478
|
+
before do
|
479
|
+
queues = hare_info.channel.instance_variable_get(:@queues)
|
480
|
+
expect( queue = queues.values.first ).to_not be nil
|
481
|
+
# emulate an issue during recovery (to trigger logger.error calls)
|
482
|
+
allow( queue ).to receive(:recover_from_network_failure).and_raise(error)
|
483
|
+
allow( logger ).to receive(:error)
|
484
|
+
end
|
485
|
+
|
486
|
+
it "gets redirected to plugin logger" do
|
487
|
+
hare_info.channel.recover_queues
|
488
|
+
expect( logger ).to have_received(:error).with(/Caught exception when recovering queue/i)
|
489
|
+
expect( logger ).to have_received(:error).with('TEST ERROR', hash_including(exception: MarchHare::Exception, cause: error_cause))
|
490
|
+
end
|
491
|
+
|
492
|
+
end
|
493
|
+
|
276
494
|
describe LogStash::Inputs::RabbitMQ do
|
495
|
+
require "logstash/devutils/rspec/shared_examples"
|
277
496
|
it_behaves_like "an interruptible input plugin"
|
278
497
|
end
|
279
498
|
end
|