logstash-output-rabbitmq 1.0.1-java → 1.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/CONTRIBUTORS +1 -0
- data/examples/sample.conf +11 -0
- data/lib/logstash/outputs/rabbitmq.rb +110 -27
- data/logstash-output-rabbitmq.gemspec +3 -7
- data/spec/outputs/rabbitmq_spec.rb +133 -102
- metadata +5 -7
- data/lib/logstash/outputs/rabbitmq/bunny.rb +0 -138
- data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb +0 -1
- data/lib/logstash/outputs/rabbitmq/march_hare.rb +0 -142
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89989010433ce0b5c2eb56006cefbb38eb71e465
|
4
|
+
data.tar.gz: 004f589a86bbbcb901167a38976c411c2b1d7dab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9cce7f5bd69bd54c440b21f0cb1c89879fec275b58c63c17cc30536e9d2c61646e74e33f968f247e7d44dcd99606704168251aa4f54a31cafc4a08c62483d06d
|
7
|
+
data.tar.gz: de07d8b894eb6c93bbeb2ed82d70587a5ffd764a2dc924021134e6dae629718c085b496ecfe58f31fce16fd9892e58bfa057c7f9d6c2714b8dbc3e650b0b3138
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
@@ -13,6 +13,7 @@ Contributors:
|
|
13
13
|
* Pier-Hugues Pellerin (ph)
|
14
14
|
* Richard Pijnenburg (electrical)
|
15
15
|
* avleen
|
16
|
+
* ssc-
|
16
17
|
|
17
18
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
18
19
|
Logstash, and you aren't on the list above and want to be, please let us know
|
@@ -1,6 +1,8 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/outputs/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require "march_hare"
|
5
|
+
require "java"
|
4
6
|
|
5
7
|
# Push events to a RabbitMQ exchange. Requires RabbitMQ 2.x
|
6
8
|
# or later version (3.x is recommended).
|
@@ -9,16 +11,12 @@ require "logstash/namespace"
|
|
9
11
|
#
|
10
12
|
# * http://www.rabbitmq.com/[RabbitMQ]
|
11
13
|
# * http://rubymarchhare.info[March Hare]
|
12
|
-
# * http://rubybunny.info[Bunny]
|
13
14
|
class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
14
15
|
EXCHANGE_TYPES = ["fanout", "direct", "topic"]
|
15
16
|
|
16
|
-
|
17
|
-
|
17
|
+
HareInfo = Struct.new(:connection, :channel, :exchange)
|
18
18
|
|
19
|
-
|
20
|
-
# Connection
|
21
|
-
#
|
19
|
+
config_name "rabbitmq"
|
22
20
|
|
23
21
|
# RabbitMQ server address
|
24
22
|
config :host, :validate => :string, :required => true
|
@@ -44,12 +42,8 @@ class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
|
44
42
|
# Enable or disable logging
|
45
43
|
config :debug, :validate => :boolean, :default => false, :deprecated => "Use the logstash --debug flag for this instead."
|
46
44
|
|
47
|
-
|
48
|
-
|
49
|
-
#
|
50
|
-
# Exchange
|
51
|
-
#
|
52
|
-
|
45
|
+
# Try to automatically recovery from broken connections. You almost certainly don't want to override this!!!
|
46
|
+
config :automatic_recovery, :validate => :boolean, :default => true
|
53
47
|
|
54
48
|
# The exchange type (fanout, topic, direct)
|
55
49
|
config :exchange_type, :validate => EXCHANGE_TYPES, :required => true
|
@@ -68,7 +62,8 @@ class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
|
68
62
|
# Should RabbitMQ persist messages to disk?
|
69
63
|
config :persistent, :validate => :boolean, :default => true
|
70
64
|
|
71
|
-
|
65
|
+
# Time in seconds to wait before retrying a connection
|
66
|
+
config :connect_retry_interval, :validate => :number, :default => 1
|
72
67
|
|
73
68
|
def initialize(params)
|
74
69
|
params["codec"] = "json" if !params["codec"]
|
@@ -76,20 +71,108 @@ class LogStash::Outputs::RabbitMQ < LogStash::Outputs::Base
|
|
76
71
|
super
|
77
72
|
end
|
78
73
|
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
74
|
+
public
|
75
|
+
def register
|
76
|
+
connect!
|
77
|
+
@codec.on_event(&method(:publish))
|
78
|
+
end
|
79
|
+
|
80
|
+
public
|
81
|
+
def receive(event)
|
82
|
+
return unless output?(event)
|
83
|
+
|
84
|
+
@codec.encode(event)
|
85
|
+
rescue StandardError => e
|
86
|
+
@logger.warn("Error encoding event", :exception => e, :event => event)
|
87
|
+
end
|
88
|
+
|
89
|
+
private
|
90
|
+
def publish(event, message)
|
91
|
+
@hare_info.exchange.publish(message, :routing_key => event.sprintf(@key), :properties => { :persistent => @persistent })
|
92
|
+
rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
|
93
|
+
return if terminating?
|
94
|
+
|
95
|
+
@logger.error("Error while publishing. Will retry.",
|
96
|
+
:message => e.message,
|
97
|
+
:exception => e.class,
|
98
|
+
:backtrace => e.backtrace)
|
99
|
+
|
100
|
+
sleep_for_retry
|
101
|
+
connect!
|
102
|
+
retry
|
103
|
+
end
|
88
104
|
|
89
|
-
|
90
|
-
|
91
|
-
|
105
|
+
public
|
106
|
+
def to_s
|
107
|
+
return "<LogStash::RabbitMQ::Output: amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}>"
|
108
|
+
end
|
109
|
+
|
110
|
+
public
|
111
|
+
def teardown
|
112
|
+
@hare_info.connection.close if connection_open?
|
113
|
+
|
114
|
+
finished
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
def settings
|
119
|
+
return @settings if @settings
|
120
|
+
|
121
|
+
s = {
|
122
|
+
:vhost => @vhost,
|
123
|
+
:host => @host,
|
124
|
+
:port => @port,
|
125
|
+
:user => @user,
|
126
|
+
:automatic_recovery => @automatic_recovery,
|
127
|
+
:pass => @password ? @password.value : "guest",
|
128
|
+
}
|
129
|
+
s[:tls] = @ssl if @ssl
|
130
|
+
@settings = s
|
131
|
+
end
|
132
|
+
|
133
|
+
private
|
134
|
+
def connect
|
135
|
+
@logger.debug("Connecting to RabbitMQ. Settings: #{settings.inspect}, queue: #{@queue.inspect}")
|
136
|
+
|
137
|
+
|
138
|
+
connection = MarchHare.connect(settings)
|
139
|
+
connection.on_blocked { @logger.warn("RabbitMQ output blocked! Check your RabbitMQ instance!") }
|
140
|
+
connection.on_unblocked { @logger.warn("RabbitMQ output unblocked!") }
|
141
|
+
|
142
|
+
channel = connection.create_channel
|
143
|
+
@logger.info("Connected to RabbitMQ at #{settings[:host]}")
|
144
|
+
|
145
|
+
@logger.debug("Declaring an exchange", :name => @exchange,
|
146
|
+
:type => @exchange_type, :durable => @durable)
|
147
|
+
|
148
|
+
exchange = channel.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
|
149
|
+
@logger.debug("Exchange declared")
|
150
|
+
|
151
|
+
HareInfo.new(connection, channel, exchange)
|
152
|
+
end
|
153
|
+
|
154
|
+
private
|
155
|
+
def connect!
|
156
|
+
@hare_info = connect() unless connection_open?
|
157
|
+
rescue MarchHare::Exception => e
|
158
|
+
return if terminating?
|
159
|
+
|
160
|
+
@logger.error("RabbitMQ connection error, will retry.",
|
161
|
+
:message => e.message,
|
162
|
+
:exception => e.class.name,
|
163
|
+
:backtrace => e.backtrace)
|
164
|
+
|
165
|
+
sleep_for_retry
|
166
|
+
retry
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
def connection_open?
|
171
|
+
@hare_info && @hare_info.connection && @hare_info.connection.open?
|
172
|
+
end
|
92
173
|
|
93
|
-
|
174
|
+
private
|
175
|
+
def sleep_for_retry
|
176
|
+
sleep @connect_retry_interval
|
94
177
|
end
|
95
|
-
end
|
178
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-rabbitmq'
|
4
|
-
s.version = '1.0
|
4
|
+
s.version = '1.1.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Push events to a RabbitMQ exchange"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -22,12 +22,8 @@ Gem::Specification.new do |s|
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
s.add_runtime_dependency 'march_hare', ['~> 2.5.1'] #(MIT license)
|
28
|
-
else
|
29
|
-
s.add_runtime_dependency 'bunny', ['>= 1.6.0'] #(MIT license)
|
30
|
-
end
|
25
|
+
s.platform = RUBY_PLATFORM
|
26
|
+
s.add_runtime_dependency 'march_hare', ['~> 2.11.0'] #(MIT license)
|
31
27
|
|
32
28
|
s.add_development_dependency 'logstash-devutils'
|
33
29
|
s.add_development_dependency 'logstash-input-generator'
|
@@ -4,129 +4,160 @@ require "logstash/pipeline"
|
|
4
4
|
require "logstash/outputs/rabbitmq"
|
5
5
|
|
6
6
|
describe LogStash::Outputs::RabbitMQ do
|
7
|
+
let(:klass) { LogStash::Outputs::RabbitMQ }
|
8
|
+
let(:host) { "localhost" }
|
9
|
+
let(:port) { 5672 }
|
10
|
+
let(:exchange_type) { "topic" }
|
11
|
+
let(:exchange) { "myexchange" }
|
12
|
+
let(:key) { "mykey" }
|
13
|
+
let(:persistent) { true }
|
14
|
+
let(:settings) {
|
15
|
+
{
|
16
|
+
"host" => host,
|
17
|
+
"port" => port,
|
18
|
+
"exchange_type" => exchange_type,
|
19
|
+
"exchange" => exchange,
|
20
|
+
"key" => key,
|
21
|
+
"persistent" => persistent
|
22
|
+
}
|
23
|
+
}
|
24
|
+
let(:instance) { klass.new(settings) }
|
25
|
+
let(:hare_info) { instance.instance_variable_get(:@hare_info) }
|
7
26
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
count => 1
|
13
|
-
}
|
14
|
-
}
|
15
|
-
output {
|
16
|
-
rabbitmq {
|
17
|
-
host => "localhost"
|
18
|
-
exchange_type => "topic"
|
19
|
-
exchange => "foo"
|
20
|
-
key => "bar"
|
21
|
-
}
|
22
|
-
}
|
23
|
-
END
|
24
|
-
|
25
|
-
it "should use defined key" do
|
26
|
-
exchange = double("exchange")
|
27
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:connect).and_return(nil)
|
28
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:declare_exchange).and_return(exchange)
|
29
|
-
|
30
|
-
expect(exchange).to receive(:publish).with(an_instance_of(String), {:routing_key => "bar", :properties => {:persistent => true}})
|
31
|
-
|
32
|
-
# we need to set expectations before running the pipeline, this is why we cannot use the
|
33
|
-
# "agent" spec construct here so we do it manually
|
34
|
-
pipeline = LogStash::Pipeline.new(config)
|
35
|
-
pipeline.run
|
36
|
-
end
|
37
|
-
|
38
|
-
end
|
27
|
+
context "when connected" do
|
28
|
+
let(:connection) { double("MarchHare Connection") }
|
29
|
+
let(:channel) { double("Channel") }
|
30
|
+
let(:exchange) { double("Exchange") }
|
39
31
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
rabbitmq {
|
50
|
-
host => "localhost"
|
51
|
-
exchange_type => "topic"
|
52
|
-
exchange => "foo"
|
53
|
-
key => "%{foo}"
|
54
|
-
}
|
55
|
-
}
|
56
|
-
END
|
57
|
-
|
58
|
-
it "should populate the key with the content of the event foo field" do
|
59
|
-
exchange = double("exchange")
|
60
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:connect).and_return(nil)
|
61
|
-
expect_any_instance_of(LogStash::Outputs::RabbitMQ).to receive(:declare_exchange).and_return(exchange)
|
62
|
-
|
63
|
-
expect(exchange).to receive(:publish).with(an_instance_of(String), {:routing_key => "bar", :properties => {:persistent => true}})
|
64
|
-
|
65
|
-
# we need to set expectations before running the pipeline, this is why we cannot use the
|
66
|
-
# "agent" spec construct here so we do it manually
|
67
|
-
pipeline = LogStash::Pipeline.new(config)
|
68
|
-
pipeline.run
|
32
|
+
before do
|
33
|
+
allow(instance).to receive(:connect!).and_call_original
|
34
|
+
allow(::MarchHare).to receive(:connect).and_return(connection)
|
35
|
+
allow(connection).to receive(:create_channel).and_return(channel)
|
36
|
+
allow(connection).to receive(:on_blocked)
|
37
|
+
allow(connection).to receive(:on_unblocked)
|
38
|
+
allow(channel).to receive(:exchange).and_return(exchange)
|
39
|
+
|
40
|
+
instance.register
|
69
41
|
end
|
70
42
|
|
71
|
-
|
43
|
+
describe "#register" do
|
44
|
+
subject { instance }
|
72
45
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# This should be a small number of lines, but until we refactor we must jump through hoops
|
77
|
-
# to put the plugin in the proper state and extract the proper state to test
|
78
|
-
describe "retrying a publish" do
|
79
|
-
let(:settings) {
|
80
|
-
{
|
81
|
-
"host" => "localhost",
|
82
|
-
"exchange_type" => "topic",
|
83
|
-
"exchange" => "foo",
|
84
|
-
"key" => "%{foo}"
|
85
|
-
}
|
86
|
-
}
|
87
|
-
let(:event) { LogStash::Event.new("foo" => "bar")}
|
46
|
+
it "should create cleanly" do
|
47
|
+
expect(subject).to be_a(klass)
|
48
|
+
end
|
88
49
|
|
89
|
-
|
50
|
+
it "should connect" do
|
51
|
+
expect(subject).to have_received(:connect!).once
|
52
|
+
end
|
53
|
+
end
|
90
54
|
|
91
|
-
|
92
|
-
|
93
|
-
subject.instance_variable_get(:@connected).set(true)
|
94
|
-
}
|
55
|
+
describe "#connect!" do
|
56
|
+
subject { hare_info }
|
95
57
|
|
96
|
-
|
58
|
+
it "should set @hare_info correctly" do
|
59
|
+
expect(subject).to be_a(LogStash::Outputs::RabbitMQ::HareInfo)
|
60
|
+
end
|
97
61
|
|
98
|
-
@
|
62
|
+
it "should set @connection correctly" do
|
63
|
+
expect(subject.connection).to eql(connection)
|
64
|
+
end
|
99
65
|
|
100
|
-
|
101
|
-
|
102
|
-
@most_recent_exchange = double("exchange #{exchange_invocation}")
|
66
|
+
it "should set the channel correctly" do
|
67
|
+
expect(subject.channel).to eql(channel)
|
103
68
|
end
|
104
69
|
|
105
|
-
|
70
|
+
it "should set the exchange correctly" do
|
71
|
+
expect(subject.exchange).to eql(exchange)
|
72
|
+
end
|
106
73
|
end
|
107
74
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
75
|
+
describe "#publish_encoded" do
|
76
|
+
let(:event) { LogStash::Event.new("foo" => "bar") }
|
77
|
+
let(:sprinted_key) { double("sprinted key") }
|
78
|
+
let(:encoded_event) { LogStash::Json.dump(event) }
|
79
|
+
|
80
|
+
describe "issuing the publish" do
|
81
|
+
before do
|
82
|
+
allow(exchange).to receive(:publish).with(any_args)
|
83
|
+
allow(event).to receive(:sprintf).with(key).and_return(sprinted_key)
|
84
|
+
instance.send(:publish, event, encoded_event)
|
118
85
|
end
|
119
86
|
|
120
|
-
|
87
|
+
it "should send the correct message" do
|
88
|
+
expect(exchange).to have_received(:publish).with(encoded_event, anything)
|
89
|
+
end
|
90
|
+
|
91
|
+
it "should send the correct metadata" do
|
92
|
+
expected_metadata = {:routing_key => sprinted_key, :properties => {:persistent => persistent }}
|
93
|
+
|
94
|
+
expect(exchange).to have_received(:publish).with(anything, expected_metadata)
|
95
|
+
end
|
121
96
|
end
|
122
97
|
|
123
|
-
|
124
|
-
|
98
|
+
context "when a MarchHare::Exception is encountered" do
|
99
|
+
before do
|
100
|
+
i = 0
|
101
|
+
allow(instance).to receive(:connect!)
|
102
|
+
allow(instance).to receive(:sleep_for_retry)
|
103
|
+
allow(exchange).to receive(:publish).with(any_args) do
|
104
|
+
i += 1
|
105
|
+
raise(MarchHare::Exception, "foo") if i == 1
|
106
|
+
end
|
107
|
+
|
108
|
+
instance.send(:publish, event, encoded_event)
|
109
|
+
end
|
110
|
+
|
111
|
+
it "should execute publish twice due to a retry" do
|
112
|
+
expect(exchange).to have_received(:publish).twice
|
113
|
+
end
|
114
|
+
|
115
|
+
it "should sleep for the retry" do
|
116
|
+
expect(instance).to have_received(:sleep_for_retry).once
|
117
|
+
end
|
118
|
+
|
119
|
+
it "should send the correct message (twice)" do
|
120
|
+
expect(exchange).to have_received(:publish).with(encoded_event, anything).twice
|
121
|
+
end
|
122
|
+
|
123
|
+
it "should send the correct metadata (twice)" do
|
124
|
+
expected_metadata = {:routing_key => event.sprintf(key), :properties => {:persistent => persistent }}
|
125
|
+
|
126
|
+
expect(exchange).to have_received(:publish).with(anything, expected_metadata).twice
|
127
|
+
end
|
125
128
|
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
# If the connection encounters an exception during its initial
|
133
|
+
# connection attempt we must handle that. Subsequent errors should be
|
134
|
+
# handled by the automatic retry mechanism built-in to MarchHare
|
135
|
+
describe "initial connection exceptions" do
|
136
|
+
subject { instance }
|
137
|
+
|
138
|
+
before do
|
139
|
+
allow(subject).to receive(:sleep_for_retry)
|
140
|
+
|
126
141
|
|
127
|
-
|
128
|
-
|
142
|
+
i = 0
|
143
|
+
allow(subject).to receive(:connect) do
|
144
|
+
i += 1
|
145
|
+
if i == 1
|
146
|
+
raise(MarchHare::ConnectionRefused, "Error!")
|
147
|
+
else
|
148
|
+
double("connection")
|
149
|
+
end
|
129
150
|
end
|
151
|
+
|
152
|
+
subject.send(:connect!)
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should retry its connection when conn fails" do
|
156
|
+
expect(subject).to have_received(:connect).twice
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should sleep between retries" do
|
160
|
+
expect(subject).to have_received(:sleep_for_retry).once
|
130
161
|
end
|
131
162
|
end
|
132
163
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-rabbitmq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
@@ -36,12 +36,12 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - ~>
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: 2.
|
39
|
+
version: 2.11.0
|
40
40
|
requirement: !ruby/object:Gem::Requirement
|
41
41
|
requirements:
|
42
42
|
- - ~>
|
43
43
|
- !ruby/object:Gem::Version
|
44
|
-
version: 2.
|
44
|
+
version: 2.11.0
|
45
45
|
prerelease: false
|
46
46
|
type: :runtime
|
47
47
|
- !ruby/object:Gem::Dependency
|
@@ -100,10 +100,8 @@ files:
|
|
100
100
|
- NOTICE.TXT
|
101
101
|
- README.md
|
102
102
|
- Rakefile
|
103
|
+
- examples/sample.conf
|
103
104
|
- lib/logstash/outputs/rabbitmq.rb
|
104
|
-
- lib/logstash/outputs/rabbitmq/bunny.rb
|
105
|
-
- lib/logstash/outputs/rabbitmq/hot_bunnies.rb
|
106
|
-
- lib/logstash/outputs/rabbitmq/march_hare.rb
|
107
105
|
- logstash-output-rabbitmq.gemspec
|
108
106
|
- spec/outputs/rabbitmq_spec.rb
|
109
107
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
@@ -1,138 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
|
-
require "logstash/json"
|
4
|
-
|
5
|
-
class LogStash::Outputs::RabbitMQ
|
6
|
-
module BunnyImpl
|
7
|
-
|
8
|
-
#
|
9
|
-
# API
|
10
|
-
#
|
11
|
-
|
12
|
-
def register
|
13
|
-
require "bunny"
|
14
|
-
|
15
|
-
@logger.info("Registering output", :plugin => self)
|
16
|
-
|
17
|
-
connect
|
18
|
-
declare_exchange
|
19
|
-
end # def register
|
20
|
-
|
21
|
-
|
22
|
-
def receive(event)
|
23
|
-
return unless output?(event)
|
24
|
-
|
25
|
-
@logger.debug("Sending event", :destination => to_s, :event => event, :key => key)
|
26
|
-
key = event.sprintf(@key) if @key
|
27
|
-
|
28
|
-
begin
|
29
|
-
publish_serialized(event.to_json, key)
|
30
|
-
rescue LogStash::Json::GeneratorError => e
|
31
|
-
@logger.warn("Trouble converting event to JSON", :exception => e,
|
32
|
-
:event => event)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def publish_serialized(message, key = @key)
|
37
|
-
begin
|
38
|
-
if @x
|
39
|
-
@x.publish(message, :persistent => @persistent, :routing_key => key)
|
40
|
-
else
|
41
|
-
@logger.warn("Tried to send a message, but not connected to RabbitMQ yet.")
|
42
|
-
end
|
43
|
-
rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
|
44
|
-
n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
|
45
|
-
|
46
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
47
|
-
:exception => e,
|
48
|
-
:backtrace => e.backtrace)
|
49
|
-
return if terminating?
|
50
|
-
|
51
|
-
sleep n
|
52
|
-
connect
|
53
|
-
declare_exchange
|
54
|
-
retry
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def to_s
|
59
|
-
return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
|
60
|
-
end
|
61
|
-
|
62
|
-
def teardown
|
63
|
-
@conn.close if @conn && @conn.open?
|
64
|
-
@conn = nil
|
65
|
-
|
66
|
-
finished
|
67
|
-
end
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
#
|
72
|
-
# Implementation
|
73
|
-
#
|
74
|
-
|
75
|
-
def connect
|
76
|
-
@vhost ||= Bunny::DEFAULT_HOST
|
77
|
-
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
|
78
|
-
@port ||= AMQ::Protocol::DEFAULT_PORT
|
79
|
-
@routing_key ||= "#"
|
80
|
-
|
81
|
-
@settings = {
|
82
|
-
:vhost => @vhost,
|
83
|
-
:host => @host,
|
84
|
-
:port => @port,
|
85
|
-
:automatically_recover => false
|
86
|
-
}
|
87
|
-
@settings[:user] = @user || Bunny::DEFAULT_USER
|
88
|
-
@settings[:pass] = if @password
|
89
|
-
@password.value
|
90
|
-
else
|
91
|
-
Bunny::DEFAULT_PASSWORD
|
92
|
-
end
|
93
|
-
|
94
|
-
@settings[:log_level] = if @debug || @logger.debug?
|
95
|
-
:debug
|
96
|
-
else
|
97
|
-
:error
|
98
|
-
end
|
99
|
-
|
100
|
-
@settings[:tls] = @ssl if @ssl
|
101
|
-
@settings[:verify_ssl] = @verify_ssl if @verify_ssl
|
102
|
-
|
103
|
-
proto = if @ssl
|
104
|
-
"amqp"
|
105
|
-
else
|
106
|
-
"amqps"
|
107
|
-
end
|
108
|
-
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
|
109
|
-
|
110
|
-
begin
|
111
|
-
@conn = Bunny.new(@settings)
|
112
|
-
|
113
|
-
@logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
|
114
|
-
return if terminating?
|
115
|
-
@conn.start
|
116
|
-
|
117
|
-
@ch = @conn.create_channel
|
118
|
-
@logger.info("Connected to RabbitMQ at #{@settings[:host]}")
|
119
|
-
rescue Bunny::NetworkFailure, Bunny::ConnectionClosedError, Bunny::ConnectionLevelException, Bunny::TCPConnectionFailed => e
|
120
|
-
n = Bunny::Session::DEFAULT_NETWORK_RECOVERY_INTERVAL * 2
|
121
|
-
|
122
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
123
|
-
:exception => e,
|
124
|
-
:backtrace => e.backtrace)
|
125
|
-
return if terminating?
|
126
|
-
|
127
|
-
sleep n
|
128
|
-
retry
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
def declare_exchange
|
133
|
-
@logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
|
134
|
-
:durable => @durable)
|
135
|
-
@x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
|
136
|
-
end
|
137
|
-
end # BunnyImpl
|
138
|
-
end # LogStash::Outputs::RabbitMQ
|
@@ -1 +0,0 @@
|
|
1
|
-
require "logstash/outputs/rabbitmq/march_hare"
|
@@ -1,142 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
class LogStash::Outputs::RabbitMQ
|
3
|
-
module MarchHareImpl
|
4
|
-
|
5
|
-
|
6
|
-
#
|
7
|
-
# API
|
8
|
-
#
|
9
|
-
|
10
|
-
def register
|
11
|
-
require "march_hare"
|
12
|
-
require "java"
|
13
|
-
|
14
|
-
@logger.info("Registering output", :plugin => self)
|
15
|
-
|
16
|
-
@connected = java.util.concurrent.atomic.AtomicBoolean.new
|
17
|
-
|
18
|
-
connect
|
19
|
-
@x = declare_exchange
|
20
|
-
|
21
|
-
@connected.set(true)
|
22
|
-
|
23
|
-
@codec.on_event(&method(:publish_serialized))
|
24
|
-
end
|
25
|
-
|
26
|
-
|
27
|
-
def receive(event)
|
28
|
-
return unless output?(event)
|
29
|
-
|
30
|
-
begin
|
31
|
-
@codec.encode(event)
|
32
|
-
rescue => e
|
33
|
-
@logger.warn("Error encoding event", :exception => e, :event => event)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def publish_serialized(event, message)
|
38
|
-
attempt_publish_serialized(event, message)
|
39
|
-
rescue MarchHare::Exception, IOError, com.rabbitmq.client.AlreadyClosedException => e
|
40
|
-
@connected.set(false)
|
41
|
-
n = 10
|
42
|
-
|
43
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
44
|
-
:exception => e,
|
45
|
-
:backtrace => e.backtrace)
|
46
|
-
return if terminating?
|
47
|
-
|
48
|
-
sleep n
|
49
|
-
|
50
|
-
connect
|
51
|
-
@x = declare_exchange
|
52
|
-
retry
|
53
|
-
end
|
54
|
-
|
55
|
-
def attempt_publish_serialized(event, message)
|
56
|
-
if @connected.get
|
57
|
-
@x.publish(message, :routing_key => event.sprintf(@key), :properties => { :persistent => @persistent })
|
58
|
-
else
|
59
|
-
@logger.warn("Tried to send a message, but not connected to RabbitMQ.")
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
def to_s
|
64
|
-
return "amqp://#{@user}@#{@host}:#{@port}#{@vhost}/#{@exchange_type}/#{@exchange}\##{@key}"
|
65
|
-
end
|
66
|
-
|
67
|
-
def teardown
|
68
|
-
@connected.set(false)
|
69
|
-
@conn.close if @conn && @conn.open?
|
70
|
-
@conn = nil
|
71
|
-
|
72
|
-
finished
|
73
|
-
end
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
#
|
78
|
-
# Implementation
|
79
|
-
#
|
80
|
-
|
81
|
-
def connect
|
82
|
-
return if terminating?
|
83
|
-
|
84
|
-
@vhost ||= "127.0.0.1"
|
85
|
-
# 5672. Will be switched to 5671 by Bunny if TLS is enabled.
|
86
|
-
@port ||= 5672
|
87
|
-
|
88
|
-
@settings = {
|
89
|
-
:vhost => @vhost,
|
90
|
-
:host => @host,
|
91
|
-
:port => @port,
|
92
|
-
:user => @user,
|
93
|
-
:automatic_recovery => false
|
94
|
-
}
|
95
|
-
@settings[:pass] = if @password
|
96
|
-
@password.value
|
97
|
-
else
|
98
|
-
"guest"
|
99
|
-
end
|
100
|
-
|
101
|
-
@settings[:tls] = @ssl if @ssl
|
102
|
-
proto = if @ssl
|
103
|
-
"amqp"
|
104
|
-
else
|
105
|
-
"amqps"
|
106
|
-
end
|
107
|
-
@connection_url = "#{proto}://#{@user}@#{@host}:#{@port}#{vhost}/#{@queue}"
|
108
|
-
|
109
|
-
begin
|
110
|
-
@conn = MarchHare.connect(@settings) unless @conn && @conn.open?
|
111
|
-
|
112
|
-
@logger.debug("Connecting to RabbitMQ. Settings: #{@settings.inspect}, queue: #{@queue.inspect}")
|
113
|
-
|
114
|
-
@ch = @conn.create_channel
|
115
|
-
@logger.info("Connected to RabbitMQ at #{@settings[:host]}")
|
116
|
-
rescue MarchHare::Exception => e
|
117
|
-
@connected.set(false)
|
118
|
-
n = 10
|
119
|
-
|
120
|
-
@logger.error("RabbitMQ connection error: #{e.message}. Will attempt to reconnect in #{n} seconds...",
|
121
|
-
:exception => e,
|
122
|
-
:backtrace => e.backtrace)
|
123
|
-
return if terminating?
|
124
|
-
|
125
|
-
sleep n
|
126
|
-
retry
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
def declare_exchange
|
131
|
-
@logger.debug("Declaring an exchange", :name => @exchange, :type => @exchange_type,
|
132
|
-
:durable => @durable)
|
133
|
-
x = @ch.exchange(@exchange, :type => @exchange_type.to_sym, :durable => @durable)
|
134
|
-
|
135
|
-
# sets @connected to true during recovery. MK.
|
136
|
-
@connected.set(true)
|
137
|
-
|
138
|
-
x
|
139
|
-
end
|
140
|
-
|
141
|
-
end # MarchHareImpl
|
142
|
-
end
|