log-courier 1.3 → 1.6.pre.40.g3efb33c
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/log-courier/client.rb +214 -119
- data/lib/log-courier/{client_tls.rb → client_tcp.rb} +59 -34
- data/lib/log-courier/event_queue.rb +12 -4
- data/lib/log-courier/server.rb +4 -4
- data/lib/log-courier/server_tcp.rb +16 -2
- data/lib/log-courier/server_zmq.rb +7 -3
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3d3903076cfa365c6fc3b3732c4990d84b7dcb13
|
4
|
+
data.tar.gz: 391ef46aae7fa0d1da6719a8fa6913b61fa4d6e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db9e733ccac3f49b2a16658ebe60783e1249831f1bfb3225520ac6d325e6a99e4aa33c419276cb27ceb4598720bbf7caea8b65eb5a9f56d7fc35386d8655c6ab
|
7
|
+
data.tar.gz: a1be65639cbd57b7c0c1e503d122d1498bcb70c90cb015b9e74cbf7a6d756932c2019e2b381977711c8840f5f5fff30d508f8c57cd8b6c5984ac4f5dc7cd4254
|
data/lib/log-courier/client.rb
CHANGED
@@ -25,6 +25,7 @@ require 'zlib'
|
|
25
25
|
class NativeException; end
|
26
26
|
|
27
27
|
module LogCourier
|
28
|
+
class TimeoutError < StandardError; end
|
28
29
|
class ShutdownSignal < StandardError; end
|
29
30
|
class ProtocolError < StandardError; end
|
30
31
|
|
@@ -77,12 +78,14 @@ module LogCourier
|
|
77
78
|
lines = @sequence_len - @last_sequence
|
78
79
|
@last_sequence = sequence
|
79
80
|
@payload = nil
|
81
|
+
@events = []
|
80
82
|
return lines, true
|
81
83
|
end
|
82
84
|
|
83
85
|
lines = sequence - @last_sequence
|
84
86
|
@last_sequence = sequence
|
85
87
|
@payload = nil
|
88
|
+
@events.shift(lines)
|
86
89
|
return lines, false
|
87
90
|
end
|
88
91
|
end
|
@@ -92,15 +95,26 @@ module LogCourier
|
|
92
95
|
def initialize(options = {})
|
93
96
|
@options = {
|
94
97
|
logger: nil,
|
98
|
+
transport: 'tls',
|
95
99
|
spool_size: 1024,
|
96
|
-
idle_timeout: 5
|
100
|
+
idle_timeout: 5,
|
101
|
+
port: nil,
|
102
|
+
addresses: [],
|
97
103
|
}.merge!(options)
|
98
104
|
|
99
105
|
@logger = @options[:logger]
|
100
|
-
@logger['plugin'] = 'output/courier'
|
106
|
+
@logger['plugin'] = 'output/courier' unless @logger.nil?
|
101
107
|
|
102
|
-
|
103
|
-
|
108
|
+
case @options[:transport]
|
109
|
+
when 'tcp', 'tls'
|
110
|
+
require 'log-courier/client_tcp'
|
111
|
+
@client = ClientTcp.new(@options)
|
112
|
+
else
|
113
|
+
fail 'output/courier: \'transport\' must be tcp or tls'
|
114
|
+
end
|
115
|
+
|
116
|
+
fail 'output/courier: \'addresses\' must contain at least one address' if @options[:addresses].empty?
|
117
|
+
fail 'output/courier: \'addresses\' only supports a single address at this time' if @options[:addresses].length > 1
|
104
118
|
|
105
119
|
@event_queue = EventQueue.new @options[:spool_size]
|
106
120
|
@pending_payloads = {}
|
@@ -115,6 +129,16 @@ module LogCourier
|
|
115
129
|
run_spooler
|
116
130
|
end
|
117
131
|
|
132
|
+
# TODO: Make these configurable?
|
133
|
+
@keepalive_timeout = 1800
|
134
|
+
@network_timeout = 30
|
135
|
+
|
136
|
+
# TODO: Make pending payload max configurable?
|
137
|
+
@max_pending_payloads = 100
|
138
|
+
|
139
|
+
@retry_payload = nil
|
140
|
+
@received_payloads = Queue.new
|
141
|
+
|
118
142
|
@pending_ping = false
|
119
143
|
|
120
144
|
# Start the IO thread
|
@@ -130,11 +154,17 @@ module LogCourier
|
|
130
154
|
return
|
131
155
|
end
|
132
156
|
|
133
|
-
def shutdown
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
157
|
+
def shutdown(force=false)
|
158
|
+
if force
|
159
|
+
# Raise a shutdown signal in the spooler and wait for it
|
160
|
+
@spooler_thread.raise ShutdownSignal
|
161
|
+
@spooler_thread.join
|
162
|
+
@io_thread.raise ShutdownSignal
|
163
|
+
else
|
164
|
+
@event_queue.push nil
|
165
|
+
@spooler_thread.join
|
166
|
+
@io_control << ['!', nil]
|
167
|
+
end
|
138
168
|
@io_thread.join
|
139
169
|
return @pending_payloads.length == 0
|
140
170
|
end
|
@@ -150,7 +180,13 @@ module LogCourier
|
|
150
180
|
begin
|
151
181
|
loop do
|
152
182
|
event = @event_queue.pop next_flush - Time.now.to_i
|
183
|
+
|
184
|
+
if event.nil?
|
185
|
+
raise ShutdownSignal
|
186
|
+
end
|
187
|
+
|
153
188
|
spooled.push(event)
|
189
|
+
|
154
190
|
break if spooled.length >= @options[:spool_size]
|
155
191
|
end
|
156
192
|
rescue TimeoutError
|
@@ -158,12 +194,19 @@ module LogCourier
|
|
158
194
|
next if spooled.length == 0
|
159
195
|
end
|
160
196
|
|
197
|
+
if spooled.length >= @options[:spool_size]
|
198
|
+
@logger.debug 'Flushing full spool', :events => spooled.length unless @logger.nil?
|
199
|
+
else
|
200
|
+
@logger.debug 'Flushing spool due to timeout', :events => spooled.length unless @logger.nil?
|
201
|
+
end
|
202
|
+
|
161
203
|
# Pass through to io_control but only if we're ready to send
|
162
204
|
@send_mutex.synchronize do
|
163
|
-
@send_cond.wait(@send_mutex)
|
205
|
+
@send_cond.wait(@send_mutex) until @send_ready
|
164
206
|
@send_ready = false
|
165
|
-
@io_control << ['E', spooled]
|
166
207
|
end
|
208
|
+
|
209
|
+
@io_control << ['E', spooled]
|
167
210
|
end
|
168
211
|
return
|
169
212
|
rescue ShutdownSignal
|
@@ -171,118 +214,17 @@ module LogCourier
|
|
171
214
|
end
|
172
215
|
|
173
216
|
def run_io
|
174
|
-
# TODO: Make keepalive configurable?
|
175
|
-
@keepalive_timeout = 1800
|
176
|
-
|
177
|
-
# TODO: Make pending payload max configurable?
|
178
|
-
max_pending_payloads = 100
|
179
|
-
|
180
|
-
retry_payload = nil
|
181
|
-
|
182
|
-
can_send = true
|
183
|
-
|
184
217
|
loop do
|
185
218
|
# Reconnect loop
|
186
219
|
@client.connect @io_control
|
187
220
|
|
188
|
-
|
221
|
+
@timeout = Time.now.to_i + @keepalive_timeout
|
189
222
|
|
190
|
-
|
191
|
-
begin
|
192
|
-
# IO loop
|
193
|
-
loop do
|
194
|
-
catch :keepalive do
|
195
|
-
begin
|
196
|
-
action = @io_control.pop @keepalive_next - Time.now.to_i
|
197
|
-
|
198
|
-
# Process the action
|
199
|
-
case action[0]
|
200
|
-
when 'S'
|
201
|
-
# If we're flushing through the pending, pick from there
|
202
|
-
unless retry_payload.nil?
|
203
|
-
# Regenerate data if we need to
|
204
|
-
retry_payload.data = buffer_jdat_data(retry_payload.events, retry_payload.nonce) if retry_payload.data == nil
|
205
|
-
|
206
|
-
# Send and move onto next
|
207
|
-
@client.send 'JDAT', retry_payload.data
|
208
|
-
|
209
|
-
retry_payload = retry_payload.next
|
210
|
-
throw :keepalive
|
211
|
-
end
|
212
|
-
|
213
|
-
# Ready to send, allow spooler to pass us something
|
214
|
-
@send_mutex.synchronize do
|
215
|
-
@send_ready = true
|
216
|
-
@send_cond.signal
|
217
|
-
end
|
218
|
-
|
219
|
-
can_send = true
|
220
|
-
when 'E'
|
221
|
-
# If we have too many pending payloads, pause the IO
|
222
|
-
if @pending_payloads.length + 1 >= max_pending_payloads
|
223
|
-
@client.pause_send
|
224
|
-
end
|
225
|
-
|
226
|
-
# Received some events - send them
|
227
|
-
send_jdat action[1]
|
228
|
-
|
229
|
-
# The send action will trigger another "S" if we have more send buffer
|
230
|
-
can_send = false
|
231
|
-
when 'R'
|
232
|
-
# Received a message
|
233
|
-
signature, message = action[1..2]
|
234
|
-
case signature
|
235
|
-
when 'PONG'
|
236
|
-
process_pong message
|
237
|
-
when 'ACKN'
|
238
|
-
process_ackn message
|
239
|
-
else
|
240
|
-
# Unknown message - only listener is allowed to respond with a "????" message
|
241
|
-
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
242
|
-
end
|
243
|
-
when 'F'
|
244
|
-
# Reconnect, an error occurred
|
245
|
-
break
|
246
|
-
end
|
247
|
-
rescue TimeoutError
|
248
|
-
# Keepalive timeout hit, send a PING unless we were awaiting a PONG
|
249
|
-
if @pending_ping
|
250
|
-
# Timed out, break into reconnect
|
251
|
-
fail TimeoutError
|
252
|
-
end
|
253
|
-
|
254
|
-
# Is send full? can_send will be false if so
|
255
|
-
# We should've started receiving ACK by now so time out
|
256
|
-
fail TimeoutError unless can_send
|
257
|
-
|
258
|
-
# Send PING
|
259
|
-
send_ping
|
260
|
-
|
261
|
-
# We may have filled send buffer
|
262
|
-
can_send = false
|
263
|
-
end
|
264
|
-
end
|
265
|
-
|
266
|
-
# Reset keepalive timeout
|
267
|
-
reset_keepalive
|
268
|
-
end
|
269
|
-
rescue ProtocolError => e
|
270
|
-
# Reconnect required due to a protocol error
|
271
|
-
@logger.warn 'Protocol error', :error => e.message unless @logger.nil?
|
272
|
-
rescue TimeoutError
|
273
|
-
# Reconnect due to timeout
|
274
|
-
@logger.warn 'Timeout occurred' unless @logger.nil?
|
275
|
-
rescue ShutdownSignal
|
276
|
-
# Shutdown, break out
|
277
|
-
break
|
278
|
-
rescue StandardError, NativeException => e
|
279
|
-
# Unknown error occurred
|
280
|
-
@logger.warn e, :hint => 'Unknown error' unless @logger.nil?
|
281
|
-
end
|
223
|
+
run_io_loop
|
282
224
|
|
283
225
|
# Disconnect and retry payloads
|
284
226
|
@client.disconnect
|
285
|
-
retry_payload = @first_payload
|
227
|
+
@retry_payload = @first_payload
|
286
228
|
|
287
229
|
# TODO: Make reconnect time configurable?
|
288
230
|
sleep 5
|
@@ -290,11 +232,164 @@ module LogCourier
|
|
290
232
|
|
291
233
|
@client.disconnect
|
292
234
|
return
|
235
|
+
rescue ShutdownSignal
|
236
|
+
# Ensure disconnected
|
237
|
+
@client.disconnect
|
293
238
|
end
|
294
239
|
|
295
|
-
def
|
296
|
-
|
297
|
-
|
240
|
+
def run_io_loop()
|
241
|
+
io_stop = false
|
242
|
+
can_send = false
|
243
|
+
|
244
|
+
# IO loop
|
245
|
+
loop do
|
246
|
+
begin
|
247
|
+
action = @io_control.pop @timeout - Time.now.to_i
|
248
|
+
|
249
|
+
# Process the action
|
250
|
+
case action[0]
|
251
|
+
when 'S'
|
252
|
+
# If we're flushing through the pending, pick from there
|
253
|
+
unless @retry_payload.nil?
|
254
|
+
@logger.debug 'Send is ready, retrying previous payload' unless @logger.nil?
|
255
|
+
|
256
|
+
# Regenerate data if we need to
|
257
|
+
@retry_payload.generate if @retry_payload.payload.nil?
|
258
|
+
|
259
|
+
# Send and move onto next
|
260
|
+
@client.send 'JDAT', @retry_payload.payload
|
261
|
+
|
262
|
+
@retry_payload = @retry_payload.next
|
263
|
+
|
264
|
+
# If first send, exit idle mode
|
265
|
+
if @retry_payload == @first_payload
|
266
|
+
@timeout = Time.now.to_i + @network_timeout
|
267
|
+
end
|
268
|
+
next
|
269
|
+
end
|
270
|
+
|
271
|
+
# Ready to send, allow spooler to pass us something if we don't
|
272
|
+
# have something already
|
273
|
+
if @received_payloads.length != 0
|
274
|
+
@logger.debug 'Send is ready, using events from backlog' unless @logger.nil?
|
275
|
+
send_payload @received_payloads.pop()
|
276
|
+
else
|
277
|
+
@logger.debug 'Send is ready, requesting events' unless @logger.nil?
|
278
|
+
|
279
|
+
can_send = true
|
280
|
+
|
281
|
+
@send_mutex.synchronize do
|
282
|
+
@send_ready = true
|
283
|
+
@send_cond.signal
|
284
|
+
end
|
285
|
+
end
|
286
|
+
when 'E'
|
287
|
+
# Were we expecting a payload? Store it if not
|
288
|
+
if can_send
|
289
|
+
@logger.debug 'Sending events', :events => action[1].length unless @logger.nil?
|
290
|
+
send_payload action[1]
|
291
|
+
can_send = false
|
292
|
+
else
|
293
|
+
@logger.debug 'Events received when not ready; saved to backlog' unless @logger.nil?
|
294
|
+
@received_payloads.push action[1]
|
295
|
+
end
|
296
|
+
when 'R'
|
297
|
+
# Received a message
|
298
|
+
signature, message = action[1..2]
|
299
|
+
case signature
|
300
|
+
when 'PONG'
|
301
|
+
process_pong message
|
302
|
+
when 'ACKN'
|
303
|
+
process_ackn message
|
304
|
+
else
|
305
|
+
# Unknown message - only listener is allowed to respond with a "????" message
|
306
|
+
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
307
|
+
end
|
308
|
+
|
309
|
+
# Any pending payloads left?
|
310
|
+
if @pending_payloads.length == 0
|
311
|
+
# Handle shutdown
|
312
|
+
if io_stop
|
313
|
+
raise ShutdownSignal
|
314
|
+
end
|
315
|
+
|
316
|
+
# Enter idle mode
|
317
|
+
@timeout = Time.now.to_i + @keepalive_timeout
|
318
|
+
else
|
319
|
+
# Set network timeout
|
320
|
+
@timeout = Time.now.to_i + @network_timeout
|
321
|
+
end
|
322
|
+
when 'F'
|
323
|
+
# Reconnect, an error occurred
|
324
|
+
break
|
325
|
+
when '!'
|
326
|
+
@logger.debug 'Shutdown request received' unless @logger.nil?
|
327
|
+
|
328
|
+
# Shutdown request received
|
329
|
+
if @pending_payloads.length == 0
|
330
|
+
raise ShutdownSignal
|
331
|
+
end
|
332
|
+
|
333
|
+
@logger.debug 'Delaying shutdown due to pending payloads', :payloads => @pending_payloads.length unless @logger.nil?
|
334
|
+
|
335
|
+
io_stop = true
|
336
|
+
|
337
|
+
# Stop spooler sending
|
338
|
+
can_send = false
|
339
|
+
@send_mutex.synchronize do
|
340
|
+
@send_ready = false
|
341
|
+
end
|
342
|
+
end
|
343
|
+
rescue TimeoutError
|
344
|
+
if @pending_payloads != 0
|
345
|
+
# Network timeout
|
346
|
+
fail TimeoutError
|
347
|
+
end
|
348
|
+
|
349
|
+
# Keepalive timeout hit, send a PING unless we were awaiting a PONG
|
350
|
+
if @pending_ping
|
351
|
+
# Timed out, break into reconnect
|
352
|
+
fail TimeoutError
|
353
|
+
end
|
354
|
+
|
355
|
+
# Stop spooler sending
|
356
|
+
can_send = false
|
357
|
+
@send_mutex.synchronize do
|
358
|
+
@send_ready = false
|
359
|
+
end
|
360
|
+
|
361
|
+
# Send PING
|
362
|
+
send_ping
|
363
|
+
|
364
|
+
@timeout = Time.now.to_i + @network_timeout
|
365
|
+
end
|
366
|
+
end
|
367
|
+
rescue ProtocolError => e
|
368
|
+
# Reconnect required due to a protocol error
|
369
|
+
@logger.warn 'Protocol error', :error => e.message unless @logger.nil?
|
370
|
+
rescue TimeoutError
|
371
|
+
# Reconnect due to timeout
|
372
|
+
@logger.warn 'Timeout occurred' unless @logger.nil?
|
373
|
+
rescue ShutdownSignal => e
|
374
|
+
raise
|
375
|
+
rescue StandardError, NativeException => e
|
376
|
+
# Unknown error occurred
|
377
|
+
@logger.warn e, :hint => 'Unknown error' unless @logger.nil?
|
378
|
+
end
|
379
|
+
|
380
|
+
def send_payload(payload)
|
381
|
+
# If we have too many pending payloads, pause the IO
|
382
|
+
if @pending_payloads.length + 1 >= @max_pending_payloads
|
383
|
+
@client.pause_send
|
384
|
+
end
|
385
|
+
|
386
|
+
# Received some events - send them
|
387
|
+
send_jdat payload
|
388
|
+
|
389
|
+
# Leave idle mode if this is the first payload after idle
|
390
|
+
if @pending_payloads.length == 1
|
391
|
+
@timeout = Time.now.to_i + @network_timeout
|
392
|
+
end
|
298
393
|
end
|
299
394
|
|
300
395
|
def generate_nonce
|
@@ -345,7 +440,7 @@ module LogCourier
|
|
345
440
|
fail ProtocolError, "ACKN message size invalid (#{message.length})" if message.length != 20
|
346
441
|
|
347
442
|
# Grab nonce
|
348
|
-
nonce, sequence = message.unpack('
|
443
|
+
nonce, sequence = message.unpack('a16N')
|
349
444
|
|
350
445
|
if !@logger.nil? && @logger.debug?
|
351
446
|
nonce_str = nonce.each_byte.map do |b|
|
@@ -23,16 +23,15 @@ require 'thread'
|
|
23
23
|
|
24
24
|
module LogCourier
|
25
25
|
# TLS transport implementation
|
26
|
-
class
|
26
|
+
class ClientTcp
|
27
27
|
def initialize(options = {})
|
28
28
|
@options = {
|
29
29
|
logger: nil,
|
30
|
-
|
31
|
-
addresses: [],
|
30
|
+
transport: 'tls',
|
32
31
|
ssl_ca: nil,
|
33
32
|
ssl_certificate: nil,
|
34
33
|
ssl_key: nil,
|
35
|
-
ssl_key_passphrase: nil
|
34
|
+
ssl_key_passphrase: nil,
|
36
35
|
}.merge!(options)
|
37
36
|
|
38
37
|
@logger = @options[:logger]
|
@@ -41,14 +40,13 @@ module LogCourier
|
|
41
40
|
fail "output/courier: '#{k}' is required" if @options[k].nil?
|
42
41
|
end
|
43
42
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
43
|
+
if @options[:transport] == 'tls'
|
44
|
+
c = 0
|
45
|
+
[:ssl_certificate, :ssl_key].each do
|
46
|
+
c += 1
|
47
|
+
end
|
48
|
+
fail 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
|
49
49
|
end
|
50
|
-
|
51
|
-
fail 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
|
52
50
|
end
|
53
51
|
|
54
52
|
def connect(io_control)
|
@@ -67,10 +65,16 @@ module LogCourier
|
|
67
65
|
@send_paused = false
|
68
66
|
|
69
67
|
@send_thread = Thread.new do
|
70
|
-
|
68
|
+
begin
|
69
|
+
run_send io_control
|
70
|
+
rescue ShutdownSignal
|
71
|
+
end
|
71
72
|
end
|
72
73
|
@recv_thread = Thread.new do
|
73
|
-
|
74
|
+
begin
|
75
|
+
run_recv io_control
|
76
|
+
rescue ShutdownSignal
|
77
|
+
end
|
74
78
|
end
|
75
79
|
return
|
76
80
|
end
|
@@ -139,14 +143,18 @@ module LogCourier
|
|
139
143
|
end
|
140
144
|
end
|
141
145
|
return
|
142
|
-
rescue OpenSSL::SSL::SSLError
|
146
|
+
rescue OpenSSL::SSL::SSLError => e
|
143
147
|
@logger.warn 'SSL write error', :error => e.message unless @logger.nil?
|
144
148
|
io_control << ['F']
|
145
149
|
return
|
146
|
-
rescue
|
150
|
+
rescue IOError, Errno::ECONNRESET => e
|
151
|
+
@logger.warn 'Write error', :error => e.message unless @logger.nil?
|
152
|
+
io_control << ['F']
|
147
153
|
return
|
154
|
+
rescue ShutdownSignal
|
155
|
+
raise
|
148
156
|
rescue StandardError, NativeException => e
|
149
|
-
@logger.warn e, :hint => 'Unknown
|
157
|
+
@logger.warn e, :hint => 'Unknown write error' unless @logger.nil?
|
150
158
|
io_control << ['F']
|
151
159
|
return
|
152
160
|
end
|
@@ -174,18 +182,22 @@ module LogCourier
|
|
174
182
|
io_control << ['R', signature, message]
|
175
183
|
end
|
176
184
|
return
|
177
|
-
rescue OpenSSL::SSL::SSLError
|
185
|
+
rescue OpenSSL::SSL::SSLError => e
|
178
186
|
@logger.warn 'SSL read error', :error => e.message unless @logger.nil?
|
179
187
|
io_control << ['F']
|
180
188
|
return
|
189
|
+
rescue IOError, Errno::ECONNRESET => e
|
190
|
+
@logger.warn 'Read error', :error => e.message unless @logger.nil?
|
191
|
+
io_control << ['F']
|
192
|
+
return
|
181
193
|
rescue EOFError
|
182
194
|
@logger.warn 'Connection closed by server' unless @logger.nil?
|
183
195
|
io_control << ['F']
|
184
196
|
return
|
185
197
|
rescue ShutdownSignal
|
186
|
-
|
187
|
-
rescue => e
|
188
|
-
@logger.warn e, :hint => 'Unknown
|
198
|
+
raise
|
199
|
+
rescue StandardError, NativeException => e
|
200
|
+
@logger.warn e, :hint => 'Unknown read error' unless @logger.nil?
|
189
201
|
io_control << ['F']
|
190
202
|
return
|
191
203
|
end
|
@@ -200,24 +212,37 @@ module LogCourier
|
|
200
212
|
begin
|
201
213
|
tcp_socket = TCPSocket.new(address, port)
|
202
214
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
215
|
+
if @options[:transport] == 'tls'
|
216
|
+
ssl = OpenSSL::SSL::SSLContext.new
|
217
|
+
|
218
|
+
# Disable SSLv2 and SSLv3
|
219
|
+
# Call set_params first to ensure options attribute is there (hmmmm?)
|
220
|
+
ssl.set_params
|
221
|
+
# Modify the default options to ensure SSLv2 and SSLv3 is disabled
|
222
|
+
# This retains any beneficial options set by default in the current Ruby implementation
|
223
|
+
ssl.options |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2)
|
224
|
+
ssl.options |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3)
|
225
|
+
|
226
|
+
# Set the certificate file
|
227
|
+
unless @options[:ssl_certificate].nil?
|
228
|
+
ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
|
229
|
+
ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase])
|
230
|
+
end
|
209
231
|
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
232
|
+
cert_store = OpenSSL::X509::Store.new
|
233
|
+
cert_store.add_file(@options[:ssl_ca])
|
234
|
+
ssl.cert_store = cert_store
|
235
|
+
ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
|
214
236
|
|
215
|
-
|
237
|
+
@ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket)
|
216
238
|
|
217
|
-
|
239
|
+
socket = @ssl_client.connect
|
218
240
|
|
219
|
-
|
220
|
-
|
241
|
+
# Verify certificate
|
242
|
+
socket.post_connection_check(address)
|
243
|
+
else
|
244
|
+
socket = tcp_socket.connect
|
245
|
+
end
|
221
246
|
|
222
247
|
# Add extra logging data now we're connected
|
223
248
|
@logger['address'] = address
|
@@ -135,14 +135,18 @@ module LogCourier
|
|
135
135
|
# Returns +true+ if the queue is empty.
|
136
136
|
#
|
137
137
|
def empty?
|
138
|
-
@
|
138
|
+
@mutex.synchronize do
|
139
|
+
return @que.empty?
|
140
|
+
end
|
139
141
|
end
|
140
142
|
|
141
143
|
#
|
142
144
|
# Removes all objects from the queue.
|
143
145
|
#
|
144
146
|
def clear
|
145
|
-
@
|
147
|
+
@mutex.synchronize do
|
148
|
+
@que.clear
|
149
|
+
end
|
146
150
|
self
|
147
151
|
end
|
148
152
|
|
@@ -150,7 +154,9 @@ module LogCourier
|
|
150
154
|
# Returns the length of the queue.
|
151
155
|
#
|
152
156
|
def length
|
153
|
-
@
|
157
|
+
@mutex.synchronize do
|
158
|
+
return @que.length
|
159
|
+
end
|
154
160
|
end
|
155
161
|
|
156
162
|
#
|
@@ -162,7 +168,9 @@ module LogCourier
|
|
162
168
|
# Returns the number of threads waiting on the queue.
|
163
169
|
#
|
164
170
|
def num_waiting
|
165
|
-
@
|
171
|
+
@mutex.synchronize do
|
172
|
+
return @num_waiting + @num_enqueue_waiting
|
173
|
+
end
|
166
174
|
end
|
167
175
|
|
168
176
|
private
|
data/lib/log-courier/server.rb
CHANGED
@@ -36,11 +36,11 @@ module LogCourier
|
|
36
36
|
def initialize(options = {})
|
37
37
|
@options = {
|
38
38
|
logger: nil,
|
39
|
-
transport: 'tls'
|
39
|
+
transport: 'tls',
|
40
40
|
}.merge!(options)
|
41
41
|
|
42
42
|
@logger = @options[:logger]
|
43
|
-
@logger['plugin'] = 'input/courier'
|
43
|
+
@logger['plugin'] = 'input/courier' unless @logger.nil?
|
44
44
|
|
45
45
|
case @options[:transport]
|
46
46
|
when 'tcp', 'tls'
|
@@ -59,7 +59,7 @@ module LogCourier
|
|
59
59
|
|
60
60
|
# Load the json adapter
|
61
61
|
@json_adapter = MultiJson.adapter.instance
|
62
|
-
@json_options = { raw: true
|
62
|
+
@json_options = { raw: true }
|
63
63
|
end
|
64
64
|
|
65
65
|
def run(&block)
|
@@ -191,7 +191,7 @@ module LogCourier
|
|
191
191
|
# Full pipeline, partial ack
|
192
192
|
# NOTE: comm.send can raise a Timeout::Error of its own
|
193
193
|
@logger.debug 'Partially acknowledging message', :nonce => nonce_str.join, :sequence => sequence if !@logger.nil? && @logger.debug?
|
194
|
-
comm.send 'ACKN', [nonce, sequence].pack('
|
194
|
+
comm.send 'ACKN', [nonce, sequence].pack('a*N')
|
195
195
|
ack_timeout = Time.now.to_i + 5
|
196
196
|
retry
|
197
197
|
end
|
@@ -87,6 +87,16 @@ module LogCourier
|
|
87
87
|
|
88
88
|
if @options[:transport] == 'tls'
|
89
89
|
ssl = OpenSSL::SSL::SSLContext.new
|
90
|
+
|
91
|
+
# Disable SSLv2 and SSLv3
|
92
|
+
# Call set_params first to ensure options attribute is there (hmmmm?)
|
93
|
+
ssl.set_params
|
94
|
+
# Modify the default options to ensure SSLv2 and SSLv3 is disabled
|
95
|
+
# This retains any beneficial options set by default in the current Ruby implementation
|
96
|
+
ssl.options |= OpenSSL::SSL::OP_NO_SSLv2 if defined?(OpenSSL::SSL::OP_NO_SSLv2)
|
97
|
+
ssl.options |= OpenSSL::SSL::OP_NO_SSLv3 if defined?(OpenSSL::SSL::OP_NO_SSLv3)
|
98
|
+
|
99
|
+
# Set the certificate file
|
90
100
|
ssl.cert = OpenSSL::X509::Certificate.new(File.read(@options[:ssl_certificate]))
|
91
101
|
ssl.key = OpenSSL::PKey::RSA.new(File.read(@options[:ssl_key]), @options[:ssl_key_passphrase])
|
92
102
|
|
@@ -134,7 +144,7 @@ module LogCourier
|
|
134
144
|
client = @server.accept
|
135
145
|
rescue EOFError, OpenSSL::SSL::SSLError, IOError => e
|
136
146
|
# Accept failure or other issue
|
137
|
-
@logger.warn 'Connection failed to accept', :error => e.message, :peer => @tcp_server.peer unless @logger.nil
|
147
|
+
@logger.warn 'Connection failed to accept', :error => e.message, :peer => @tcp_server.peer unless @logger.nil?
|
138
148
|
client.close rescue nil unless client.nil?
|
139
149
|
next
|
140
150
|
end
|
@@ -254,10 +264,14 @@ module LogCourier
|
|
254
264
|
@logger.info 'Connection closed', :peer => @peer unless @logger.nil?
|
255
265
|
end
|
256
266
|
return
|
257
|
-
rescue OpenSSL::SSL::SSLError
|
267
|
+
rescue OpenSSL::SSL::SSLError => e
|
258
268
|
# Read errors, only action is to shutdown which we'll do in ensure
|
259
269
|
@logger.warn 'SSL error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
|
260
270
|
return
|
271
|
+
rescue IOError, Errno::ECONNRESET => e
|
272
|
+
# Read errors, only action is to shutdown which we'll do in ensure
|
273
|
+
@logger.warn 'Connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
|
274
|
+
return
|
261
275
|
rescue ProtocolError => e
|
262
276
|
# Connection abort request due to a protocol error
|
263
277
|
@logger.warn 'Protocol error, connection aborted', :error => e.message, :peer => @peer unless @logger.nil?
|
@@ -85,7 +85,7 @@ module LogCourier
|
|
85
85
|
|
86
86
|
bind = 'tcp://' + @options[:address] + (@options[:port] == 0 ? ':*' : ':' + @options[:port].to_s)
|
87
87
|
rc = @socket.bind(bind)
|
88
|
-
fail 'failed to bind at ' + bind + ': ' +
|
88
|
+
fail 'failed to bind at ' + bind + ': ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
89
89
|
|
90
90
|
# Lookup port number that was allocated in case it was set to 0
|
91
91
|
endpoint = ''
|
@@ -273,8 +273,12 @@ module LogCourier
|
|
273
273
|
}
|
274
274
|
end
|
275
275
|
|
276
|
-
# Existing thread, throw on the queue, if not enough room drop the message
|
277
|
-
|
276
|
+
# Existing thread, throw on the queue, if not enough room (timeout) drop the message
|
277
|
+
begin
|
278
|
+
index['']['client'].push data, 0
|
279
|
+
rescue LogCourier::TimeoutError
|
280
|
+
# TODO: Log a warning about this?
|
281
|
+
end
|
278
282
|
end
|
279
283
|
return
|
280
284
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log-courier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.6.pre.40.g3efb33c
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Woods
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cabin
|
@@ -60,7 +60,7 @@ extensions: []
|
|
60
60
|
extra_rdoc_files: []
|
61
61
|
files:
|
62
62
|
- lib/log-courier/client.rb
|
63
|
-
- lib/log-courier/
|
63
|
+
- lib/log-courier/client_tcp.rb
|
64
64
|
- lib/log-courier/event_queue.rb
|
65
65
|
- lib/log-courier/server.rb
|
66
66
|
- lib/log-courier/server_tcp.rb
|
@@ -81,9 +81,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
81
81
|
version: '0'
|
82
82
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
83
|
requirements:
|
84
|
-
- - '
|
84
|
+
- - '>'
|
85
85
|
- !ruby/object:Gem::Version
|
86
|
-
version:
|
86
|
+
version: 1.3.1
|
87
87
|
requirements: []
|
88
88
|
rubyforge_project: nowarning
|
89
89
|
rubygems_version: 2.4.2
|