logstash-output-rabbitmq 1.0.1-java → 1.1.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/CONTRIBUTORS +1 -0
- data/examples/sample.conf +11 -0
- data/lib/logstash/outputs/rabbitmq.rb +110 -27
- data/logstash-output-rabbitmq.gemspec +3 -7
- data/spec/outputs/rabbitmq_spec.rb +133 -102
- metadata +5 -7
- data/lib/logstash/outputs/rabbitmq/bunny.rb +0 -138
- data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb +0 -1
- data/lib/logstash/outputs/rabbitmq/march_hare.rb +0 -142
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89989010433ce0b5c2eb56006cefbb38eb71e465
|
4
|
+
data.tar.gz: 004f589a86bbbcb901167a38976c411c2b1d7dab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9cce7f5bd69bd54c440b21f0cb1c89879fec275b58c63c17cc30536e9d2c61646e74e33f968f247e7d44dcd99606704168251aa4f54a31cafc4a08c62483d06d
|
7
|
+
data.tar.gz: de07d8b894eb6c93bbeb2ed82d70587a5ffd764a2dc924021134e6dae629718c085b496ecfe58f31fce16fd9892e58bfa057c7f9d6c2714b8dbc3e650b0b3138
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
@@ -13,6 +13,7 @@ Contributors:
|
|
13
13
|
* Pier-Hugues Pellerin (ph)
|
14
14
|
* Richard Pijnenburg (electrical)
|
15
15
|
* avleen
|
16
|
+
* ssc-
|
16
17
|
|
17
18
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
18
19
|
Logstash, and you aren't on the list above and want to be, please let us know
|
@@ -1,6 +1,8 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/outputs/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require "march_hare"
|
5
|
+
require "java"
|
4
6
|
|
5
7
|
# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
|
6
8
|
# or later version (3.x is recommended).
|
@@ -9,16 +11,12 @@ require "logstash/namespace"
|
|
9
11
|
#
|
10
12
|
# * http://www.rabbitmq.com/[RabbitMQ]
|
11
13
|
# * http://rubymarchhare.info[March Hare]
|
12
|
-
# * http://rubybunny.info[Bunny]
|
13
14
|
class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
14
15
|
EXCHANGE_TYPES = ["fanout", "direct", "topic"]
|
15
16
|
|
16
|
-
|
17
|
-
|
17
|
+
HareInfo = Struct.new(:connection, :channel, :exchange)
|
18
18
|
|
19
|
-
|
20
|
-
# Connection
|
21
|
-
#
|
19
|
+
config_name "rabbitmq"
|
22
20
|
|
23
21
|
# RabbitMQ server address
|
24
22
|
config :host, :validate => :string, :required => true
|
@@ -44,12 +42,8 @@ class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
|
44
42
|
# Enable or disable logging
|
45
43
|
config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
|
46
44
|
|
47
|
-
|
48
|
-
|
49
|
-
#
|
50
|
-
# Exchange
|
51
|
-
#
|
52
|
-
|
45
|
+
# Try to automatically recovery from broken connections. You almost certainly don't want to override this!!!
|
46
|
+
config :automatic_recovery, :validate => :boolean, :default => true
|
53
47
|
|
54
48
|
# The exchange type (fanout, topic, direct)
|
55
49
|
config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
|
@@ -68,7 +62,8 @@ class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
|
68
62
|
# Should RabbitMQ persist messages to disk?
|
69
63
|
config :persistent, :validate => :boolean, :default => true
|
70
64
|
|
71
|
-
|
65
|
+
# Time in seconds to wait before retrying a connection
|
66
|
+
config :connect_retry_interval, :validate => :number, :default => 1
|
72
67
|
|
73
68
|
def initialize(params)
|
74
69
|
params["codec"] = "json" if !params["codec"]
|
@@ -76,20 +71,108 @@ class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
|
76
71
|
super
|
77
72
|
end
|
78
73
|
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
74
|
+
public
|
75
|
+
def register
|
76
|
+
connect!
|
77
|
+
@codec.on_event(&method(:publish))
|
78
|
+
end
|
79
|
+
|
80
|
+
public
|
81
|
+
def receive(event)
|
82
|
+
return unless output?(event)
|
83
|
+
|
84
|
+
@codec.encode(event)
|
85
|
+
rescue StandardError => e
|
86
|
+
@logger.warn("Error encoding event", :exception => e, :event => event)
|
87
|
+
end
|
88
|
+
|
89
|
+
private
|
90
|
+
def publish(event, message)
|
91
|
+
@hare_info.exchange.publish(message, :routing_key => event.sprintf(@key), :properties => { :persistent => @persistent })
|
92
|
+
rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
|
93
|
+
return if terminating?
|
94
|
+
|
95
|
+
@logger.error("Error while publishing. Will retry.",
|
96
|
+
:message => e.message,
|
97
|
+
:exception => e.class,
|
98
|
+
:backtrace => e.backtrace)
|
99
|
+
|
100
|
+
sleep_for_retry
|
101
|
+
connect!
|
102
|
+
retry
|
103
|
+
end
|
88
104
|
|
89
|
-
|
90
|
-
|
91
|
-
|
105
|
+
public
|
106
|
+
def to_s
|
107
|
+
return "<LogStash::RabbitMQ::Output: amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}>"
|
108
|
+
end
|
109
|
+
|
110
|
+
public
|
111
|
+
def teardown
|
112
|
+
@hare_info.connection.close if connection_open?
|
113
|
+
|
114
|
+
finished
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
def settings
|
119
|
+
return @settings if @settings
|
120
|
+
|
121
|
+
s = {
|
122
|
+
:vhost => @vhost,
|
123
|
+
:host => @host,
|
124
|
+
:port => @port,
|
125
|
+
:user => @user,
|
126
|
+
:automatic_recovery => @automatic_recovery,
|
127
|
+
:pass => @password ? @password.value : "guest",
|
128
|
+
}
|
129
|
+
s[:tls] = @ssl if @ssl
|
130
|
+
@settings = s
|
131
|
+
end
|
132
|
+
|
133
|
+
private
|
134
|
+
def connect
|
135
|
+
@logger.debug("Connecting to RabbitMQ. Settings: #{settings.inspect}, queue: #{@queue.inspect}")
|
136
|
+
|
137
|
+
|
138
|
+
connection = MarchHare.connect(settings)
|
139
|
+
connection.on_blocked { @logger.warn("RabbitMQ output blocked! Check your RabbitMQ instance!") }
|
140
|
+
connection.on_unblocked { @logger.warn("RabbitMQ output unblocked!") }
|
141
|
+
|
142
|
+
channel = connection.create_channel
|
143
|
+
@logger.info("Connected to RabbitMQ at #{settings[:host]}")
|
144
|
+
|
145
|
+
@logger.debug("Declaring an exchange", :name => @exchange,
|
146
|
+
:type => @exchange_type, :durable => @durable)
|
147
|
+
|
148
|
+
exchange = channel.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
|
149
|
+
@logger.debug("Exchange declared")
|
150
|
+
|
151
|
+
HareInfo.new(connection, channel, exchange)
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
def connect!
|
156
|
+
@hare_info = connect() unless connection_open?
|
157
|
+
rescue MarchHare::Exception => e
|
158
|
+
return if terminating?
|
159
|
+
|
160
|
+
@logger.error("RabbitMQ connection error, will retry.",
|
161
|
+
:message => e.message,
|
162
|
+
:exception => e.class.name,
|
163
|
+
:backtrace => e.backtrace)
|
164
|
+
|
165
|
+
sleep_for_retry
|
166
|
+
retry
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
def connection_open?
|
171
|
+
@hare_info && @hare_info.connection && @hare_info.connection.open?
|
172
|
+
end
|
92
173
|
|
93
|
-
|
174
|
+
private
|
175
|
+
def sleep_for_retry
|
176
|
+
sleep @connect_retry_interval
|
94
177
|
end
|
95
|
-
end
|
178
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-rabbitmq'
|
4
|
-
s.version = '1.0
|
4
|
+
s.version = '1.1.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Push events to a RabbitMQ exchange"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -22,12 +22,8 @@ Gem::Specification.new do |s|
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
s.add_runtime_dependency 'march_hare', ['~> 2.5.1'] #(MIT license)
|
28
|
-
else
|
29
|
-
s.add_runtime_dependency 'bunny', ['>= 1.6.0'] #(MIT license)
|
30
|
-
end
|
25
|
+
s.platform = RUBY_PLATFORM
|
26
|
+
s.add_runtime_dependency 'march_hare', ['~> 2.11.0'] #(MIT license)
|
31
27
|
|
32
28
|
s.add_development_dependency 'logstash-devutils'
|
33
29
|
s.add_development_dependency 'logstash-input-generator'
|
@@ -4,129 +4,160 @@ require "logstash/pipeline"
|
|
4
4
|
require "logstash/outputs/rabbitmq"
|
5
5
|
|
6
6
|
describe LogStash::Outputs::RabbitMQ do
|
7
|
+
let(:klass) { LogStash::Outputs::RabbitMQ }
|
8
|
+
let(:host) { "localhost" }
|
9
|
+
let(:port) { 5672 }
|
10
|
+
let(:exchange_type) { "topic" }
|
11
|
+
let(:exchange) { "myexchange" }
|
12
|
+
let(:key) { "mykey" }
|
13
|
+
let(:persistent) { true }
|
14
|
+
let(:settings) {
|
15
|
+
{
|
16
|
+
"host" => host,
|
17
|
+
"port" => port,
|
18
|
+
"exchange_type" => exchange_type,
|
19
|
+
"exchange" => exchange,
|
20
|
+
"key" => key,
|
21
|
+
"persistent" => persistent
|
22
|
+
}
|
23
|
+
}
|
24
|
+
let(:instance) { klass.new(settings) }
|
25
|
+
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
7
26
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
count => 1
|
13
|
-
}
|
14
|
-
}
|
15
|
-
output {
|
16
|
-
rabbitmq {
|
17
|
-
host => "localhost"
|
18
|
-
exchange_type => "topic"
|
19
|
-
exchange => "foo"
|
20
|
-
key => "bar"
|
21
|
-
}
|
22
|
-
}
|
23
|
-
END
|
24
|
-
|
25
|
-
it "should use defined key" do
|
26
|
-
exchange = double("exchange")
|
27
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:connect).and_return(nil)
|
28
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:declare_exchange).and_return(exchange)
|
29
|
-
|
30
|
-
expect(exchange).to receive(:publish).with(an_instance_of(String), {:routing_key => "bar", :properties => {:persistent => true}})
|
31
|
-
|
32
|
-
# we need to set expectations before running the pipeline, this is why we cannot use the
|
33
|
-
# "agent" spec construct here so we do it manually
|
34
|
-
pipeline = LogStash::Pipeline.new(config)
|
35
|
-
pipeline.run
|
36
|
-
end
|
37
|
-
|
38
|
-
end
|
27
|
+
context "when connected" do
|
28
|
+
let(:connection) { double("MarchHare Connection") }
|
29
|
+
let(:channel) { double("Channel") }
|
30
|
+
let(:exchange) { double("Exchange") }
|
39
31
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
rabbitmq {
|
50
|
-
host => "localhost"
|
51
|
-
exchange_type => "topic"
|
52
|
-
exchange => "foo"
|
53
|
-
key => "%{foo}"
|
54
|
-
}
|
55
|
-
}
|
56
|
-
END
|
57
|
-
|
58
|
-
it "should populate the key with the content of the event foo field" do
|
59
|
-
exchange = double("exchange")
|
60
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:connect).and_return(nil)
|
61
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:declare_exchange).and_return(exchange)
|
62
|
-
|
63
|
-
expect(exchange).to receive(:publish).with(an_instance_of(String), {:routing_key => "bar", :properties => {:persistent => true}})
|
64
|
-
|
65
|
-
# we need to set expectations before running the pipeline, this is why we cannot use the
|
66
|
-
# "agent" spec construct here so we do it manually
|
67
|
-
pipeline = LogStash::Pipeline.new(config)
|
68
|
-
pipeline.run
|
32
|
+
before do
|
33
|
+
allow(instance).to receive(:connect!).and_call_original
|
34
|
+
allow(::MarchHare).to receive(:connect).and_return(connection)
|
35
|
+
allow(connection).to receive(:create_channel).and_return(channel)
|
36
|
+
allow(connection).to receive(:on_blocked)
|
37
|
+
allow(connection).to receive(:on_unblocked)
|
38
|
+
allow(channel).to receive(:exchange).and_return(exchange)
|
39
|
+
|
40
|
+
instance.register
|
69
41
|
end
|
70
42
|
|
71
|
-
|
43
|
+
describe "#register" do
|
44
|
+
subject { instance }
|
72
45
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# This should be a small number of lines, but until we refactor we must jump through hoops
|
77
|
-
# to put the plugin in the proper state and extract the proper state to test
|
78
|
-
describe "retrying a publish" do
|
79
|
-
let(:settings) {
|
80
|
-
{
|
81
|
-
"host" => "localhost",
|
82
|
-
"exchange_type" => "topic",
|
83
|
-
"exchange" => "foo",
|
84
|
-
"key" => "%{foo}"
|
85
|
-
}
|
86
|
-
}
|
87
|
-
let(:event) { LogStash::Event.new("foo" => "bar")}
|
46
|
+
it "should create cleanly" do
|
47
|
+
expect(subject).to be_a(klass)
|
48
|
+
end
|
88
49
|
|
89
|
-
|
50
|
+
it "should connect" do
|
51
|
+
expect(subject).to have_received(:connect!).once
|
52
|
+
end
|
53
|
+
end
|
90
54
|
|
91
|
-
|
92
|
-
|
93
|
-
subject.instance_variable_get(:@connected).set(true)
|
94
|
-
}
|
55
|
+
describe "#connect!" do
|
56
|
+
subject { hare_info }
|
95
57
|
|
96
|
-
|
58
|
+
it "should set @hare_info correctly" do
|
59
|
+
expect(subject).to be_a(LogStash::Outputs::RabbitMQ::HareInfo)
|
60
|
+
end
|
97
61
|
|
98
|
-
@
|
62
|
+
it "should set @connection correctly" do
|
63
|
+
expect(subject.connection).to eql(connection)
|
64
|
+
end
|
99
65
|
|
100
|
-
|
101
|
-
|
102
|
-
@most_recent_exchange = double("exchange #{exchange_invocation}")
|
66
|
+
it "should set the channel correctly" do
|
67
|
+
expect(subject.channel).to eql(channel)
|
103
68
|
end
|
104
69
|
|
105
|
-
|
70
|
+
it "should set the exchange correctly" do
|
71
|
+
expect(subject.exchange).to eql(exchange)
|
72
|
+
end
|
106
73
|
end
|
107
74
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
75
|
+
describe "#publish_encoded" do
|
76
|
+
let(:event) { LogStash::Event.new("foo" => "bar") }
|
77
|
+
let(:sprinted_key) { double("sprinted key") }
|
78
|
+
let(:encoded_event) { LogStash::Json.dump(event) }
|
79
|
+
|
80
|
+
describe "issuing the publish" do
|
81
|
+
before do
|
82
|
+
allow(exchange).to receive(:publish).with(any_args)
|
83
|
+
allow(event).to receive(:sprintf).with(key).and_return(sprinted_key)
|
84
|
+
instance.send(:publish, event, encoded_event)
|
118
85
|
end
|
119
86
|
|
120
|
-
|
87
|
+
it "should send the correct message" do
|
88
|
+
expect(exchange).to have_received(:publish).with(encoded_event, anything)
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should send the correct metadata" do
|
92
|
+
expected_metadata = {:routing_key => sprinted_key, :properties => {:persistent => persistent }}
|
93
|
+
|
94
|
+
expect(exchange).to have_received(:publish).with(anything, expected_metadata)
|
95
|
+
end
|
121
96
|
end
|
122
97
|
|
123
|
-
|
124
|
-
|
98
|
+
context "when a MarchHare::Exception is encountered" do
|
99
|
+
before do
|
100
|
+
i = 0
|
101
|
+
allow(instance).to receive(:connect!)
|
102
|
+
allow(instance).to receive(:sleep_for_retry)
|
103
|
+
allow(exchange).to receive(:publish).with(any_args) do
|
104
|
+
i += 1
|
105
|
+
raise(MarchHare::Exception, "foo") if i == 1
|
106
|
+
end
|
107
|
+
|
108
|
+
instance.send(:publish, event, encoded_event)
|
109
|
+
end
|
110
|
+
|
111
|
+
it "should execute publish twice due to a retry" do
|
112
|
+
expect(exchange).to have_received(:publish).twice
|
113
|
+
end
|
114
|
+
|
115
|
+
it "should sleep for the retry" do
|
116
|
+
expect(instance).to have_received(:sleep_for_retry).once
|
117
|
+
end
|
118
|
+
|
119
|
+
it "should send the correct message (twice)" do
|
120
|
+
expect(exchange).to have_received(:publish).with(encoded_event, anything).twice
|
121
|
+
end
|
122
|
+
|
123
|
+
it "should send the correct metadata (twice)" do
|
124
|
+
expected_metadata = {:routing_key => event.sprintf(key), :properties => {:persistent => persistent }}
|
125
|
+
|
126
|
+
expect(exchange).to have_received(:publish).with(anything, expected_metadata).twice
|
127
|
+
end
|
125
128
|
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
# If the connection encounters an exception during its initial
|
133
|
+
# connection attempt we must handle that. Subsequent errors should be
|
134
|
+
# handled by the automatic retry mechanism built-in to MarchHare
|
135
|
+
describe "initial connection exceptions" do
|
136
|
+
subject { instance }
|
137
|
+
|
138
|
+
before do
|
139
|
+
allow(subject).to receive(:sleep_for_retry)
|
140
|
+
|
126
141
|
|
127
|
-
|
128
|
-
|
142
|
+
i = 0
|
143
|
+
allow(subject).to receive(:connect) do
|
144
|
+
i += 1
|
145
|
+
if i == 1
|
146
|
+
raise(MarchHare::ConnectionRefused, "Error!")
|
147
|
+
else
|
148
|
+
double("connection")
|
149
|
+
end
|
129
150
|
end
|
151
|
+
|
152
|
+
subject.send(:connect!)
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should retry its connection when conn fails" do
|
156
|
+
expect(subject).to have_received(:connect).twice
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should sleep between retries" do
|
160
|
+
expect(subject).to have_received(:sleep_for_retry).once
|
130
161
|
end
|
131
162
|
end
|
132
163
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-rabbitmq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
@@ -36,12 +36,12 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - ~>
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: 2.
|
39
|
+
version: 2.11.0
|
40
40
|
requirement: !ruby/object:Gem::Requirement
|
41
41
|
requirements:
|
42
42
|
- - ~>
|
43
43
|
- !ruby/object:Gem::Version
|
44
|
-
version: 2.
|
44
|
+
version: 2.11.0
|
45
45
|
prerelease: false
|
46
46
|
type: :runtime
|
47
47
|
- !ruby/object:Gem::Dependency
|
@@ -100,10 +100,8 @@ files:
|
|
100
100
|
- NOTICE.TXT
|
101
101
|
- README.md
|
102
102
|
- Rakefile
|
103
|
+
- examples/sample.conf
|
103
104
|
- lib/logstash/outputs/rabbitmq.rb
|
104
|
-
- lib/logstash/outputs/rabbitmq/bunny.rb
|
105
|
-
- lib/logstash/outputs/rabbitmq/hot_bunnies.rb
|
106
|
-
- lib/logstash/outputs/rabbitmq/march_hare.rb
|
107
105
|
- logstash-output-rabbitmq.gemspec
|
108
106
|
- spec/outputs/rabbitmq_spec.rb
|
109
107
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
@@ -1,138 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
|
-
require "logstash/json"
|
4
|
-
|
5
|
-
class LogStash::Outputs::RabbitMQ
|
6
|
-
module BunnyImpl
|
7
|
-
|
8
|
-
#
|
9
|
-
# API
|
10
|
-
#
|
11
|
-
|
12
|
-
def register
|
13
|
-
require "bunny"
|
14
|
-
|
15
|
-
@logger.info("Registering output", :plugin => self)
|
16
|
-
|
17
|
-
connect
|
18
|
-
declare_exchange
|
19
|
-
end # def register
|
20
|
-
|
21
|
-
|
22
|
-
def receive(event)
|
23
|
-
return unless output?(event)
|
24
|
-
|
25
|
-
@logger.debug("Sending event", :destination => to_s, :event => event, :key => key)
|
26
|
-
key = event.sprintf(@key) if @key
|
27
|
-
|
28
|
-
begin
|
29
|
-
publish_serialized(event.to_json, key)
|
30
|
-
rescue LogStash::Json::GeneratorError => e
|
31
|
-
@logger.warn("Trouble converting event to JSON", :exception => e,
|
32
|
-
:event => event)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def publish_serialized(message, key = @key)
|
37
|
-
begin
|
38
|
-
if @x
|
39
|
-
@x.publish(message, :persistent => @persistent, :routing_key => key)
|
40
|
-
else
|
41
|
-
@logger.warn("Tried to send a message, but not connected to RabbitMQ yet.")
|
42
|
-
end
|
43
|
-
rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
|
44
|
-
n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
|
45
|
-
|
46
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
47
|
-
:exception => e,
|
48
|
-
:backtrace => e.backtrace)
|
49
|
-
return if terminating?
|
50
|
-
|
51
|
-
sleep n
|
52
|
-
connect
|
53
|
-
declare_exchange
|
54
|
-
retry
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def to_s
|
59
|
-
return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
|
60
|
-
end
|
61
|
-
|
62
|
-
def teardown
|
63
|
-
@conn.close if @conn && @conn.open?
|
64
|
-
@conn = nil
|
65
|
-
|
66
|
-
finished
|
67
|
-
end
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
#
|
72
|
-
# Implementation
|
73
|
-
#
|
74
|
-
|
75
|
-
def connect
|
76
|
-
@vhost ||= Bunny::DEFAULT_HOST
|
77
|
-
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
|
78
|
-
@port ||= AMQ::Protocol::DEFAULT_PORT
|
79
|
-
@routing_key ||= "#"
|
80
|
-
|
81
|
-
@settings = {
|
82
|
-
:vhost => @vhost,
|
83
|
-
:host => @host,
|
84
|
-
:port => @port,
|
85
|
-
:automatically_recover => false
|
86
|
-
}
|
87
|
-
@settings[:user] = @user || Bunny::DEFAULT_USER
|
88
|
-
@settings[:pass] = if @password
|
89
|
-
@password.value
|
90
|
-
else
|
91
|
-
Bunny::DEFAULT_PASSWORD
|
92
|
-
end
|
93
|
-
|
94
|
-
@settings[:log_level] = if @debug || @logger.debug?
|
95
|
-
:debug
|
96
|
-
else
|
97
|
-
:error
|
98
|
-
end
|
99
|
-
|
100
|
-
@settings[:tls] = @ssl if @ssl
|
101
|
-
@settings[:verify_ssl] = @verify_ssl if @verify_ssl
|
102
|
-
|
103
|
-
proto = if @ssl
|
104
|
-
"amqp"
|
105
|
-
else
|
106
|
-
"amqps"
|
107
|
-
end
|
108
|
-
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
|
109
|
-
|
110
|
-
begin
|
111
|
-
@conn = Bunny.new(@settings)
|
112
|
-
|
113
|
-
@logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
|
114
|
-
return if terminating?
|
115
|
-
@conn.start
|
116
|
-
|
117
|
-
@ch = @conn.create_channel
|
118
|
-
@logger.info("Connected to RabbitMQ at #{@settings[:host]}")
|
119
|
-
rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
|
120
|
-
n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
|
121
|
-
|
122
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
123
|
-
:exception => e,
|
124
|
-
:backtrace => e.backtrace)
|
125
|
-
return if terminating?
|
126
|
-
|
127
|
-
sleep n
|
128
|
-
retry
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
def declare_exchange
|
133
|
-
@logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
|
134
|
-
:durable => @durable)
|
135
|
-
@x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
|
136
|
-
end
|
137
|
-
end # BunnyImpl
|
138
|
-
end # LogStash::Outputs::RabbitMQ
|
@@ -1 +0,0 @@
|
|
1
|
-
require "logstash/outputs/rabbitmq/march_hare"
|
@@ -1,142 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
class LogStash::Outputs::RabbitMQ
|
3
|
-
module MarchHareImpl
|
4
|
-
|
5
|
-
|
6
|
-
#
|
7
|
-
# API
|
8
|
-
#
|
9
|
-
|
10
|
-
def register
|
11
|
-
require "march_hare"
|
12
|
-
require "java"
|
13
|
-
|
14
|
-
@logger.info("Registering output", :plugin => self)
|
15
|
-
|
16
|
-
@connected = java.util.concurrent.atomic.AtomicBoolean.new
|
17
|
-
|
18
|
-
connect
|
19
|
-
@x = declare_exchange
|
20
|
-
|
21
|
-
@connected.set(true)
|
22
|
-
|
23
|
-
@codec.on_event(&method(:publish_serialized))
|
24
|
-
end
|
25
|
-
|
26
|
-
|
27
|
-
def receive(event)
|
28
|
-
return unless output?(event)
|
29
|
-
|
30
|
-
begin
|
31
|
-
@codec.encode(event)
|
32
|
-
rescue => e
|
33
|
-
@logger.warn("Error encoding event", :exception => e, :event => event)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def publish_serialized(event, message)
|
38
|
-
attempt_publish_serialized(event, message)
|
39
|
-
rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
|
40
|
-
@connected.set(false)
|
41
|
-
n = 10
|
42
|
-
|
43
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
44
|
-
:exception => e,
|
45
|
-
:backtrace => e.backtrace)
|
46
|
-
return if terminating?
|
47
|
-
|
48
|
-
sleep n
|
49
|
-
|
50
|
-
connect
|
51
|
-
@x = declare_exchange
|
52
|
-
retry
|
53
|
-
end
|
54
|
-
|
55
|
-
def attempt_publish_serialized(event, message)
|
56
|
-
if @connected.get
|
57
|
-
@x.publish(message, :routing_key => event.sprintf(@key), :properties => { :persistent => @persistent })
|
58
|
-
else
|
59
|
-
@logger.warn("Tried to send a message, but not connected to RabbitMQ.")
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
def to_s
|
64
|
-
return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
|
65
|
-
end
|
66
|
-
|
67
|
-
def teardown
|
68
|
-
@connected.set(false)
|
69
|
-
@conn.close if @conn && @conn.open?
|
70
|
-
@conn = nil
|
71
|
-
|
72
|
-
finished
|
73
|
-
end
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
#
|
78
|
-
# Implementation
|
79
|
-
#
|
80
|
-
|
81
|
-
def connect
|
82
|
-
return if terminating?
|
83
|
-
|
84
|
-
@vhost ||= "127.0.0.1"
|
85
|
-
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
|
86
|
-
@port ||= 5672
|
87
|
-
|
88
|
-
@settings = {
|
89
|
-
:vhost => @vhost,
|
90
|
-
:host => @host,
|
91
|
-
:port => @port,
|
92
|
-
:user => @user,
|
93
|
-
:automatic_recovery => false
|
94
|
-
}
|
95
|
-
@settings[:pass] = if @password
|
96
|
-
@password.value
|
97
|
-
else
|
98
|
-
"guest"
|
99
|
-
end
|
100
|
-
|
101
|
-
@settings[:tls] = @ssl if @ssl
|
102
|
-
proto = if @ssl
|
103
|
-
"amqp"
|
104
|
-
else
|
105
|
-
"amqps"
|
106
|
-
end
|
107
|
-
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
|
108
|
-
|
109
|
-
begin
|
110
|
-
@conn = MarchHare.connect(@settings) unless @conn && @conn.open?
|
111
|
-
|
112
|
-
@logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
|
113
|
-
|
114
|
-
@ch = @conn.create_channel
|
115
|
-
@logger.info("Connected to RabbitMQ at #{@settings[:host]}")
|
116
|
-
rescue MarchHare::Exception => e
|
117
|
-
@connected.set(false)
|
118
|
-
n = 10
|
119
|
-
|
120
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
121
|
-
:exception => e,
|
122
|
-
:backtrace => e.backtrace)
|
123
|
-
return if terminating?
|
124
|
-
|
125
|
-
sleep n
|
126
|
-
retry
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
def declare_exchange
|
131
|
-
@logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
|
132
|
-
:durable => @durable)
|
133
|
-
x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
|
134
|
-
|
135
|
-
# sets @connected to true during recovery. MK.
|
136
|
-
@connected.set(true)
|
137
|
-
|
138
|
-
x
|
139
|
-
end
|
140
|
-
|
141
|
-
end # MarchHareImpl
|
142
|
-
end
|