logstash-input-relp 2.0.1 → 2.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/inputs/relp.rb +58 -28
- data/lib/logstash/util/relp.rb +4 -5
- data/logstash-input-relp.gemspec +1 -1
- data/spec/inputs/relp_spec.rb +58 -36
- data/spec/spec_helper.rb +5 -41
- data/spec/support/client.rb +22 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 38af08ca585650125a700ef56813a35c18c2f947
|
4
|
+
data.tar.gz: ee7e0b9bbc59fa5d1cde1d1aee4fa9240b6f4410
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 86c0e5ea580ebd2dbcaa8a37a76809d5540a3e23cde0a673c2d3936fd348368aaac0d956e0f31a0923f2096fe82940d0b645ce8dc3116f08de10f827a98eb91a
|
7
|
+
data.tar.gz: ac4ac922af4bb5de2b03c2b9245c15f9f2f025d5a7146120bb04965b11734cc0d83014105be1c9345add663544ecff33c421dc9830f6bdde861c4f1ae9e86908
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 2.0.2
|
2
|
+
- Update the test to play nice within the context of the default
|
3
|
+
plugins LS core integration test.
|
4
|
+
|
1
5
|
## 2.0.0
|
2
6
|
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
7
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
data/lib/logstash/inputs/relp.rb
CHANGED
@@ -3,6 +3,7 @@ require "logstash/inputs/base"
|
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "logstash/util/relp"
|
5
5
|
require "logstash/util/socket_peer"
|
6
|
+
require "openssl"
|
6
7
|
|
7
8
|
# Read RELP events over a TCP socket.
|
8
9
|
#
|
@@ -47,11 +48,16 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
|
|
47
48
|
def initialize(*args)
|
48
49
|
super(*args)
|
49
50
|
@relp_server = nil
|
51
|
+
|
52
|
+
# monkey patch TCPSocket and SSLSocket to include socket peer
|
53
|
+
TCPSocket.module_eval{include ::LogStash::Util::SocketPeer}
|
54
|
+
OpenSSL::SSL::SSLSocket.module_eval{include ::LogStash::Util::SocketPeer}
|
50
55
|
end # def initialize
|
51
56
|
|
52
57
|
public
|
53
58
|
def register
|
54
59
|
@logger.info("Starting relp input listener", :address => "#{@host}:#{@port}")
|
60
|
+
|
55
61
|
if @ssl_enable
|
56
62
|
initialize_ssl_context
|
57
63
|
if @ssl_verify == false
|
@@ -62,13 +68,17 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
|
|
62
68
|
].join("\n")
|
63
69
|
end
|
64
70
|
end
|
71
|
+
|
72
|
+
# note that since we are opening a socket (through @relp_server) in register,
|
73
|
+
# we must also make sure we close it in the close method even if we also close
|
74
|
+
# it in the stop method since we could have a situation where register is called
|
75
|
+
# but not run & stop.
|
76
|
+
|
65
77
|
@relp_server = RelpServer.new(@host, @port,['syslog'], @ssl_context)
|
66
78
|
end # def register
|
67
79
|
|
68
80
|
private
|
69
81
|
def initialize_ssl_context
|
70
|
-
require "openssl"
|
71
|
-
|
72
82
|
@ssl_context = OpenSSL::SSL::SSLContext.new
|
73
83
|
@ssl_context.cert = OpenSSL::X509::Certificate.new(File.read(@ssl_cert))
|
74
84
|
@ssl_context.key = OpenSSL::PKey::RSA.new(File.read(@ssl_key),@ssl_key_passphrase.value)
|
@@ -110,28 +120,8 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
|
|
110
120
|
def run(output_queue)
|
111
121
|
while !stop?
|
112
122
|
begin
|
113
|
-
|
114
|
-
|
115
|
-
rs = client[0]
|
116
|
-
socket = client[1]
|
117
|
-
begin
|
118
|
-
rs.relp_setup_connection(socket)
|
119
|
-
# monkeypatch a 'peer' method onto the socket.
|
120
|
-
socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end }
|
121
|
-
peer = socket.peer
|
122
|
-
@logger.debug("Relp Connection to #{peer} created")
|
123
|
-
relp_stream(rs,socket, output_queue, peer)
|
124
|
-
rescue Relp::ConnectionClosed => e
|
125
|
-
@logger.debug("Relp Connection to #{peer} Closed")
|
126
|
-
rescue Relp::RelpError => e
|
127
|
-
@logger.warn('Relp error: '+e.class.to_s+' '+e.message)
|
128
|
-
#TODO: Still not happy with this, are they all warn level?
|
129
|
-
#Will this catch everything I want it to?
|
130
|
-
#Relp spec says to close connection on error, ensure this is the case
|
131
|
-
ensure
|
132
|
-
socket.close rescue nil
|
133
|
-
end
|
134
|
-
end # Thread.start
|
123
|
+
server, socket = @relp_server.accept
|
124
|
+
connection_thread(server, socket, output_queue)
|
135
125
|
rescue Relp::InvalidCommand,Relp::InappropriateCommand => e
|
136
126
|
@logger.warn('Relp client trying to open connection with something other than open:'+e.message)
|
137
127
|
rescue Relp::InsufficientCommands
|
@@ -140,18 +130,58 @@ class LogStash::Inputs::Relp < LogStash::Inputs::Base
|
|
140
130
|
@logger.debug('Relp Connection closed')
|
141
131
|
rescue OpenSSL::SSL::SSLError => ssle
|
142
132
|
# NOTE(mrichar1): This doesn't return a useful error message for some reason
|
143
|
-
@logger.error("SSL Error", :exception => ssle,
|
144
|
-
|
145
|
-
|
146
|
-
|
133
|
+
@logger.error("SSL Error", :exception => ssle, :backtrace => ssle.backtrace)
|
134
|
+
rescue => e
|
135
|
+
# if this exception occured while the plugin is stopping
|
136
|
+
# just ignore and exit
|
137
|
+
raise e unless stop?
|
138
|
+
end
|
139
|
+
end
|
147
140
|
end # def run
|
148
141
|
|
149
142
|
def stop
|
143
|
+
# force close @relp_server which will escape any blocking read with a IO exception
|
144
|
+
# and any thread using them will exit.
|
145
|
+
# catch all rescue nil to discard any close errors or invalid socket
|
146
|
+
if @relp_server
|
147
|
+
@relp_server.shutdown rescue nil
|
148
|
+
@relp_server = nil
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def close
|
153
|
+
# see related comment in register: we must make sure to close the @relp_server here
|
154
|
+
# because it is created in the register method and we could be in the context of having
|
155
|
+
# register called but never run & stop, only close.
|
150
156
|
if @relp_server
|
151
157
|
@relp_server.shutdown rescue nil
|
152
158
|
@relp_server = nil
|
153
159
|
end
|
154
160
|
end
|
161
|
+
|
162
|
+
private
|
163
|
+
|
164
|
+
def connection_thread(server, socket, output_queue)
|
165
|
+
Thread.start(server, socket, output_queue) do |server, socket, output_queue|
|
166
|
+
begin
|
167
|
+
server.relp_setup_connection(socket)
|
168
|
+
peer = socket.peer
|
169
|
+
@logger.debug("Relp Connection to #{peer} created")
|
170
|
+
relp_stream(server,socket, output_queue, peer)
|
171
|
+
rescue Relp::ConnectionClosed => e
|
172
|
+
@logger.debug("Relp Connection to #{peer} Closed")
|
173
|
+
rescue Relp::RelpError => e
|
174
|
+
@logger.warn('Relp error: '+e.class.to_s+' '+e.message)
|
175
|
+
#TODO: Still not happy with this, are they all warn level?
|
176
|
+
#Will this catch everything I want it to?
|
177
|
+
#Relp spec says to close connection on error, ensure this is the case
|
178
|
+
rescue => e
|
179
|
+
# ignore other exceptions
|
180
|
+
ensure
|
181
|
+
socket.close rescue nil
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
155
185
|
end # class LogStash::Inputs::Relp
|
156
186
|
|
157
187
|
#TODO: structured error logging
|
data/lib/logstash/util/relp.rb
CHANGED
@@ -15,7 +15,7 @@ class Relp#This isn't much use on its own, but gives RelpServer and RelpClient t
|
|
15
15
|
|
16
16
|
def valid_command?(command)
|
17
17
|
valid_commands = Array.new
|
18
|
-
|
18
|
+
|
19
19
|
#Allow anything in the basic protocol for both directions
|
20
20
|
valid_commands << 'open'
|
21
21
|
valid_commands << 'close'
|
@@ -102,7 +102,7 @@ class RelpServer < Relp
|
|
102
102
|
|
103
103
|
def initialize(host,port,required_commands=[],ssl_context=nil)
|
104
104
|
@logger = Cabin::Channel.get(LogStash)
|
105
|
-
|
105
|
+
|
106
106
|
@server=true
|
107
107
|
|
108
108
|
#These are things that are part of the basic protocol, but only valid in one direction (rsp, close etc.)
|
@@ -197,14 +197,13 @@ class RelpServer < Relp
|
|
197
197
|
frame['command'] = 'serverclose'
|
198
198
|
begin
|
199
199
|
self.frame_write(socket,frame)
|
200
|
-
socket.close
|
200
|
+
socket.close rescue nil
|
201
201
|
rescue ConnectionClosed
|
202
202
|
end
|
203
203
|
end
|
204
204
|
|
205
205
|
def shutdown
|
206
|
-
@server.close
|
207
|
-
rescue Exception#@server might already be down
|
206
|
+
@server.close rescue nil
|
208
207
|
end
|
209
208
|
|
210
209
|
def ack(socket, txnr)
|
data/logstash-input-relp.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-relp'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.2'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Read RELP events over a TCP socket."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/relp_spec.rb
CHANGED
@@ -4,6 +4,8 @@ require_relative "../support/ssl"
|
|
4
4
|
|
5
5
|
describe LogStash::Inputs::Relp do
|
6
6
|
|
7
|
+
let!(:helper) { RelpHelpers.new }
|
8
|
+
|
7
9
|
before do
|
8
10
|
srand(RSpec.configuration.seed)
|
9
11
|
end
|
@@ -13,12 +15,11 @@ describe LogStash::Inputs::Relp do
|
|
13
15
|
it "should register without errors" do
|
14
16
|
input = LogStash::Plugin.lookup("input", "relp").new("port" => 1234)
|
15
17
|
expect {input.register}.to_not raise_error
|
18
|
+
input.close rescue nil
|
16
19
|
end
|
17
|
-
|
18
20
|
end
|
19
21
|
|
20
22
|
describe "when interrupting the plugin" do
|
21
|
-
|
22
23
|
let(:port) { rand(1024..65532) }
|
23
24
|
|
24
25
|
it_behaves_like "an interruptible input plugin" do
|
@@ -28,9 +29,13 @@ describe LogStash::Inputs::Relp do
|
|
28
29
|
|
29
30
|
describe "multiple client connections" do
|
30
31
|
|
31
|
-
|
32
|
+
# (colinsurprenant) don't put number of simultaneous clients too high,
|
33
|
+
# it seems to lock no sure why. the test client code needs a very serious,
|
34
|
+
# refactoring, its too complex and very hard to debug :P
|
35
|
+
let(:nclients) { rand(10) }
|
36
|
+
|
32
37
|
let(:nevents) { 100 }
|
33
|
-
let(:port)
|
38
|
+
let(:port) { rand(1024..65532) }
|
34
39
|
|
35
40
|
let(:conf) do
|
36
41
|
<<-CONFIG
|
@@ -43,15 +48,22 @@ describe LogStash::Inputs::Relp do
|
|
43
48
|
CONFIG
|
44
49
|
end
|
45
50
|
|
46
|
-
let(:clients) { setup_clients(nclients, port) }
|
51
|
+
let(:clients) { RelpHelpers.setup_clients(nclients, port) }
|
47
52
|
|
48
53
|
let(:events) do
|
49
|
-
input(conf
|
54
|
+
input(conf) do |pipeline, queue|
|
50
55
|
nevents.times do |value|
|
51
56
|
clients.each_with_index do |client, index|
|
52
57
|
client.syslog_write("Hello from client#{index}")
|
53
58
|
end
|
54
59
|
end
|
60
|
+
(nevents * nclients).times.collect { queue.pop }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
after :each do
|
65
|
+
clients.each do |client|
|
66
|
+
client.close rescue nil
|
55
67
|
end
|
56
68
|
end
|
57
69
|
|
@@ -64,50 +76,60 @@ describe LogStash::Inputs::Relp do
|
|
64
76
|
|
65
77
|
describe "SSL support" do
|
66
78
|
|
67
|
-
let(:nevents) { 100 }
|
68
79
|
let(:certificate) { RelpTest.certificate }
|
69
|
-
let(:port)
|
80
|
+
let(:port) { rand(1024..65532) }
|
70
81
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
82
|
+
context "events reading" do
|
83
|
+
|
84
|
+
let(:nevents) { 100 }
|
85
|
+
|
86
|
+
let(:conf) do
|
87
|
+
<<-CONFIG
|
88
|
+
input {
|
89
|
+
relp {
|
90
|
+
type => "blah"
|
91
|
+
port => #{port}
|
92
|
+
ssl_enable => true
|
93
|
+
ssl_verify => false
|
94
|
+
ssl_cert => "#{certificate.ssl_cert}"
|
95
|
+
ssl_key => "#{certificate.ssl_key}"
|
96
|
+
}
|
81
97
|
}
|
82
|
-
|
83
|
-
|
84
|
-
end
|
98
|
+
CONFIG
|
99
|
+
end
|
85
100
|
|
86
|
-
|
101
|
+
let(:client) { RelpClient.new("0.0.0.0", port, ["syslog"], {:ssl => true}) }
|
87
102
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
103
|
+
let!(:events) do
|
104
|
+
input(conf) do |pipeline, queue|
|
105
|
+
nevents.times do
|
106
|
+
client.syslog_write("Hello from client")
|
107
|
+
end
|
108
|
+
nevents.times.collect { queue.pop }
|
92
109
|
end
|
93
110
|
end
|
111
|
+
|
112
|
+
after :each do
|
113
|
+
client.close rescue nil
|
114
|
+
end
|
115
|
+
|
116
|
+
it "should generated the events as expected" do
|
117
|
+
expect(events).to have(nevents).with("Hello from client")
|
118
|
+
end
|
94
119
|
end
|
95
120
|
|
96
121
|
context "registration and close" do
|
97
122
|
|
98
123
|
it "should register without errors" do
|
99
|
-
input = LogStash::Plugin.lookup("input", "relp").new(
|
100
|
-
|
101
|
-
|
124
|
+
input = LogStash::Plugin.lookup("input", "relp").new(
|
125
|
+
"port" => port,
|
126
|
+
"ssl_enable" => true,
|
127
|
+
"ssl_cert" => certificate.ssl_cert,
|
128
|
+
"ssl_key" => certificate.ssl_key
|
129
|
+
)
|
102
130
|
expect {input.register}.to_not raise_error
|
131
|
+
input.close rescue nil
|
103
132
|
end
|
104
|
-
|
105
133
|
end
|
106
|
-
|
107
|
-
it "should generated the events as expected" do
|
108
|
-
expect(events).to have(nevents).with("Hello from client")
|
109
|
-
end
|
110
|
-
|
111
134
|
end
|
112
|
-
|
113
135
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -5,62 +5,26 @@ require "logstash/util/relp"
|
|
5
5
|
require "socket"
|
6
6
|
require "support/client"
|
7
7
|
|
8
|
-
|
8
|
+
class RelpHelpers
|
9
9
|
|
10
|
-
def setup_clients(number_of_clients, port)
|
10
|
+
def self.setup_clients(number_of_clients, port)
|
11
11
|
number_of_clients.times.inject([]) do |clients|
|
12
12
|
clients << RelpClient.new("0.0.0.0", port, ["syslog"])
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
-
def filter(events, message)
|
16
|
+
def self.filter(events, message)
|
17
17
|
events.select{|event| event["message"] == message }
|
18
18
|
end
|
19
|
-
|
20
|
-
def input(config, size, &block)
|
21
|
-
pipeline = LogStash::Pipeline.new(config)
|
22
|
-
queue = Queue.new
|
23
|
-
|
24
|
-
pipeline.instance_eval do
|
25
|
-
# create closure to capture queue
|
26
|
-
@output_func = lambda { |event| queue << event }
|
27
|
-
|
28
|
-
# output_func is now a method, call closure
|
29
|
-
def output_func(event)
|
30
|
-
@output_func.call(event)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
pipeline_thread = Thread.new { pipeline.run }
|
35
|
-
sleep 0.1 while !pipeline.ready?
|
36
|
-
|
37
|
-
block.call
|
38
|
-
sleep 0.1 while queue.size != size
|
39
|
-
|
40
|
-
result = size.times.inject([]) do |acc|
|
41
|
-
acc << queue.pop
|
42
|
-
end
|
43
|
-
|
44
|
-
pipeline.shutdown
|
45
|
-
pipeline_thread.join
|
46
|
-
|
47
|
-
result
|
48
|
-
end # def input
|
49
|
-
|
50
|
-
end
|
51
|
-
|
52
|
-
RSpec.configure do |c|
|
53
|
-
c.include RelpHelpers
|
54
19
|
end
|
55
20
|
|
56
21
|
RSpec::Matchers.define :have do |nevents|
|
57
22
|
|
58
23
|
match do |events|
|
59
|
-
filter(events, @pattern).size == nevents
|
24
|
+
RelpHelpers.filter(events, @pattern).size == nevents
|
60
25
|
end
|
61
26
|
|
62
27
|
chain :with do |pattern|
|
63
28
|
@pattern = pattern
|
64
29
|
end
|
65
|
-
|
66
|
-
end
|
30
|
+
end
|
data/spec/support/client.rb
CHANGED
@@ -33,7 +33,7 @@ class RelpClient < Relp
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
-
#This'll start the automatic frame numbering
|
36
|
+
#This'll start the automatic frame numbering
|
37
37
|
@lasttxnr = 0
|
38
38
|
|
39
39
|
offer=Hash.new
|
@@ -56,7 +56,7 @@ class RelpClient < Relp
|
|
56
56
|
|
57
57
|
#subtracting one array from the other checks to see if all elements in @required_relp_commands are present in the offer
|
58
58
|
elsif ! (@required_relp_commands - response['commands'].split(',')).empty?
|
59
|
-
#if it can't receive syslog it's useless to us; close the connection
|
59
|
+
#if it can't receive syslog it's useless to us; close the connection
|
60
60
|
self.close()
|
61
61
|
raise InsufficientCommands, response['commands'] + ' offered, require '
|
62
62
|
+ @required_relp_commands.join(',')
|
@@ -67,7 +67,12 @@ class RelpClient < Relp
|
|
67
67
|
#This thread deals with responses that come back
|
68
68
|
reader = Thread.start do
|
69
69
|
loop do
|
70
|
-
|
70
|
+
begin
|
71
|
+
f = self.frame_read(@socket)
|
72
|
+
rescue
|
73
|
+
# ignore exceptions and quit thread
|
74
|
+
break
|
75
|
+
end
|
71
76
|
if f['command'] == 'rsp' && f['message'] == '200 OK'
|
72
77
|
@buffer.delete(f['txnr'])
|
73
78
|
elsif f['command'] == 'rsp' && f['message'][0,1] == '5'
|
@@ -88,14 +93,19 @@ class RelpClient < Relp
|
|
88
93
|
Thread.start do
|
89
94
|
old_buffer = Hash.new
|
90
95
|
loop do
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
+
begin
|
97
|
+
#This returns old txnrs that are still present
|
98
|
+
(@buffer.keys & old_buffer.keys).each do |txnr|
|
99
|
+
new_txnr = self.frame_write(@socket, @buffer[txnr])
|
100
|
+
@buffer[new_txnr] = @buffer[txnr]
|
101
|
+
@buffer.delete(txnr)
|
102
|
+
end
|
103
|
+
old_buffer = @buffer
|
104
|
+
sleep @retransmission_timeout
|
105
|
+
rescue
|
106
|
+
# ignore exceptions and quit thread
|
107
|
+
break
|
96
108
|
end
|
97
|
-
old_buffer = @buffer
|
98
|
-
sleep @retransmission_timeout
|
99
109
|
end
|
100
110
|
end
|
101
111
|
end
|
@@ -107,8 +117,9 @@ class RelpClient < Relp
|
|
107
117
|
@close_txnr=self.frame_write(@socket, frame)
|
108
118
|
#TODO: ought to properly wait for a reply etc. The serverclose will make it work though
|
109
119
|
sleep @retransmission_timeout
|
110
|
-
@socket.close#TODO: shutdown?
|
111
120
|
return @buffer
|
121
|
+
ensure
|
122
|
+
@socket.close
|
112
123
|
end
|
113
124
|
|
114
125
|
def syslog_write(logline)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-relp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|