log-courier 1.8.3 → 2.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/log-courier/client.rb +142 -182
- data/lib/log-courier/client_tcp.rb +117 -87
- data/lib/log-courier/event_queue.rb +18 -23
- data/lib/log-courier/protocol.rb +58 -0
- data/lib/log-courier/server.rb +33 -59
- data/lib/log-courier/server_tcp.rb +174 -120
- metadata +11 -27
- data/lib/log-courier/server_zmq.rb +0 -400
- data/lib/log-courier/zmq_qpoll.rb +0 -324
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: a3ef8d232349a4aeaabfab1481b1675ccd57fa9e10b965d87d7824c23d6f7791
|
4
|
+
data.tar.gz: 64942e87ac4da08392b7f48bdf0170914617696bebf41a3cb21a87596f8506b8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e38df8efda4dcf97cac9bfee072c2c8dc46a4af6a4ee09233c0cc5562b3ee389f8e99b9ecbbdf3347f42bd498c44bf649ddfb86676ddd5e1ebd4db0cf4be1a94
|
7
|
+
data.tar.gz: fef73ad4bbc70488a2d6dbed07ae5c0ba3ec89c42806fa836ab25781ea995e5fd2783dbaf1b0f483a6c528da0f2f8e0dd2c3fd97668e94dd6827d1f128de3f8e
|
data/lib/log-courier/client.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
# Copyright 2014 Jason Woods.
|
1
|
+
# Copyright 2014-2021 Jason Woods and Contributors.
|
4
2
|
#
|
5
3
|
# This file is a modification of code from Logstash Forwarder.
|
6
4
|
# Copyright 2012-2013 Jordan Sissel and contributors.
|
@@ -18,43 +16,25 @@
|
|
18
16
|
# limitations under the License.
|
19
17
|
|
20
18
|
require 'log-courier/event_queue'
|
19
|
+
require 'log-courier/protocol'
|
21
20
|
require 'multi_json'
|
22
|
-
require 'thread'
|
23
21
|
require 'zlib'
|
24
22
|
|
25
|
-
|
23
|
+
# Dummy until its no longer needed
|
24
|
+
class NativeException
|
25
|
+
def dummy; end
|
26
|
+
end
|
26
27
|
|
27
28
|
module LogCourier
|
28
29
|
class TimeoutError < StandardError; end
|
30
|
+
|
29
31
|
class ShutdownSignal < StandardError; end
|
32
|
+
|
30
33
|
class ProtocolError < StandardError; end
|
31
34
|
|
32
35
|
# Describes a pending payload
|
33
36
|
class PendingPayload
|
34
|
-
|
35
|
-
class << self
|
36
|
-
@json_adapter
|
37
|
-
@json_parseerror
|
38
|
-
|
39
|
-
def get_json_adapter
|
40
|
-
@json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
|
41
|
-
@json_adapter
|
42
|
-
end
|
43
|
-
|
44
|
-
def get_json_parseerror
|
45
|
-
if @json_parseerror.nil?
|
46
|
-
@json_parseerror = get_json_adapter.class::ParseError
|
47
|
-
end
|
48
|
-
@json_parseerror
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
attr_accessor :next
|
53
|
-
attr_accessor :nonce
|
54
|
-
attr_accessor :events
|
55
|
-
attr_accessor :last_sequence
|
56
|
-
attr_accessor :sequence_len
|
57
|
-
attr_accessor :payload
|
37
|
+
attr_accessor :next, :nonce, :events, :last_sequence, :sequence_len, :payload
|
58
38
|
|
59
39
|
def initialize(events, nonce)
|
60
40
|
@events = events
|
@@ -64,27 +44,27 @@ module LogCourier
|
|
64
44
|
end
|
65
45
|
|
66
46
|
def generate
|
67
|
-
|
47
|
+
raise ArgumentError, 'Corrupt payload' if @events.length.zero?
|
68
48
|
|
69
49
|
buffer = Zlib::Deflate.new
|
70
50
|
|
71
51
|
# Write each event in JSON format
|
72
52
|
events.each do |event|
|
73
|
-
json_data =
|
53
|
+
json_data = MultiJson.dump(event)
|
74
54
|
# Add length and then the data
|
75
|
-
buffer << [json_data.
|
55
|
+
buffer << [json_data.bytesize].pack('N') << json_data
|
76
56
|
end
|
77
57
|
|
78
58
|
# Generate and store the payload
|
79
|
-
@payload = nonce + buffer.finish
|
59
|
+
@payload = nonce + buffer.finish
|
80
60
|
@last_sequence = 0
|
81
61
|
@sequence_len = @events.length
|
82
62
|
end
|
83
63
|
|
84
64
|
def ack(sequence)
|
85
|
-
if sequence <= @last_sequence
|
86
|
-
|
87
|
-
|
65
|
+
return 0, false if sequence <= @last_sequence
|
66
|
+
|
67
|
+
if sequence >= @sequence_len
|
88
68
|
lines = @sequence_len - @last_sequence
|
89
69
|
@last_sequence = sequence
|
90
70
|
@payload = nil
|
@@ -96,7 +76,7 @@ module LogCourier
|
|
96
76
|
@last_sequence = sequence
|
97
77
|
@payload = nil
|
98
78
|
@events.shift(lines)
|
99
|
-
|
79
|
+
[lines, false]
|
100
80
|
end
|
101
81
|
end
|
102
82
|
|
@@ -104,12 +84,14 @@ module LogCourier
|
|
104
84
|
class Client
|
105
85
|
def initialize(options = {})
|
106
86
|
@options = {
|
107
|
-
logger:
|
108
|
-
transport:
|
109
|
-
spool_size:
|
87
|
+
logger: nil,
|
88
|
+
transport: 'tls',
|
89
|
+
spool_size: 1024,
|
110
90
|
idle_timeout: 5,
|
111
|
-
port:
|
112
|
-
addresses:
|
91
|
+
port: nil,
|
92
|
+
addresses: [],
|
93
|
+
min_tls_version: 1.2,
|
94
|
+
disable_handshake: false,
|
113
95
|
}.merge!(options)
|
114
96
|
|
115
97
|
@logger = @options[:logger]
|
@@ -119,11 +101,11 @@ module LogCourier
|
|
119
101
|
require 'log-courier/client_tcp'
|
120
102
|
@client = ClientTcp.new(@options)
|
121
103
|
else
|
122
|
-
|
104
|
+
raise 'output/courier: \'transport\' must be tcp or tls'
|
123
105
|
end
|
124
106
|
|
125
|
-
|
126
|
-
|
107
|
+
raise 'output/courier: \'addresses\' must contain at least one address' if @options[:addresses].empty?
|
108
|
+
raise 'output/courier: \'addresses\' only supports a single address at this time' if @options[:addresses].length > 1
|
127
109
|
|
128
110
|
@event_queue = EventQueue.new @options[:spool_size]
|
129
111
|
@pending_payloads = {}
|
@@ -160,10 +142,10 @@ module LogCourier
|
|
160
142
|
def publish(event)
|
161
143
|
# Pass the event into the spooler
|
162
144
|
@event_queue << event
|
163
|
-
|
145
|
+
nil
|
164
146
|
end
|
165
147
|
|
166
|
-
def shutdown(force=false)
|
148
|
+
def shutdown(force = false) # rubocop:disable Style/OptionalBooleanParameter
|
167
149
|
if force
|
168
150
|
# Raise a shutdown signal in the spooler and wait for it
|
169
151
|
@spooler_thread.raise ShutdownSignal
|
@@ -175,7 +157,7 @@ module LogCourier
|
|
175
157
|
@io_control << ['!', nil]
|
176
158
|
end
|
177
159
|
@io_thread.join
|
178
|
-
|
160
|
+
@pending_payloads.length.zero?
|
179
161
|
end
|
180
162
|
|
181
163
|
private
|
@@ -190,9 +172,7 @@ module LogCourier
|
|
190
172
|
loop do
|
191
173
|
event = @event_queue.pop next_flush - Time.now.to_i
|
192
174
|
|
193
|
-
if event.nil?
|
194
|
-
raise ShutdownSignal
|
195
|
-
end
|
175
|
+
raise ShutdownSignal if event.nil?
|
196
176
|
|
197
177
|
spooled.push(event)
|
198
178
|
|
@@ -200,13 +180,13 @@ module LogCourier
|
|
200
180
|
end
|
201
181
|
rescue TimeoutError
|
202
182
|
# Hit timeout but no events, keep waiting
|
203
|
-
next if spooled.length
|
183
|
+
next if spooled.length.zero?
|
204
184
|
end
|
205
185
|
|
206
186
|
if spooled.length >= @options[:spool_size]
|
207
|
-
@logger
|
187
|
+
@logger&.debug 'Flushing full spool', events: spooled.length
|
208
188
|
else
|
209
|
-
@logger
|
189
|
+
@logger&.debug 'Flushing spool due to timeout', events: spooled.length
|
210
190
|
end
|
211
191
|
|
212
192
|
# Pass through to io_control but only if we're ready to send
|
@@ -217,9 +197,8 @@ module LogCourier
|
|
217
197
|
|
218
198
|
@io_control << ['E', spooled]
|
219
199
|
end
|
220
|
-
return
|
221
200
|
rescue ShutdownSignal
|
222
|
-
|
201
|
+
# Shutdown
|
223
202
|
end
|
224
203
|
|
225
204
|
def run_io
|
@@ -240,175 +219,159 @@ module LogCourier
|
|
240
219
|
end
|
241
220
|
|
242
221
|
@client.disconnect
|
243
|
-
return
|
244
222
|
rescue ShutdownSignal
|
245
223
|
# Ensure disconnected
|
246
224
|
@client.disconnect
|
247
225
|
end
|
248
226
|
|
249
|
-
def run_io_loop
|
227
|
+
def run_io_loop
|
250
228
|
io_stop = false
|
251
229
|
can_send = false
|
252
230
|
|
253
231
|
# IO loop
|
254
232
|
loop do
|
255
|
-
|
256
|
-
action = @io_control.pop @timeout - Time.now.to_i
|
257
|
-
|
258
|
-
# Process the action
|
259
|
-
case action[0]
|
260
|
-
when 'S'
|
261
|
-
# If we're flushing through the pending, pick from there
|
262
|
-
unless @retry_payload.nil?
|
263
|
-
@logger.debug 'Send is ready, retrying previous payload' unless @logger.nil?
|
233
|
+
action = @io_control.pop @timeout - Time.now.to_i
|
264
234
|
|
265
|
-
|
266
|
-
|
235
|
+
# Process the action
|
236
|
+
case action[0]
|
237
|
+
when 'S'
|
238
|
+
# If we're flushing through the pending, pick from there
|
239
|
+
unless @retry_payload.nil?
|
240
|
+
@logger&.debug 'Send is ready, retrying previous payload'
|
267
241
|
|
268
|
-
|
269
|
-
|
242
|
+
# Regenerate data if we need to
|
243
|
+
@retry_payload.generate if @retry_payload.payload.nil?
|
270
244
|
|
271
|
-
|
245
|
+
# Send and move onto next
|
246
|
+
@client.send 'JDAT', @retry_payload.payload
|
272
247
|
|
273
|
-
|
274
|
-
if @retry_payload == @first_payload
|
275
|
-
@timeout = Time.now.to_i + @network_timeout
|
276
|
-
end
|
277
|
-
next
|
278
|
-
end
|
279
|
-
|
280
|
-
# Ready to send, allow spooler to pass us something if we don't
|
281
|
-
# have something already
|
282
|
-
if @received_payloads.length != 0
|
283
|
-
@logger.debug 'Send is ready, using events from backlog' unless @logger.nil?
|
284
|
-
send_payload @received_payloads.pop()
|
285
|
-
else
|
286
|
-
@logger.debug 'Send is ready, requesting events' unless @logger.nil?
|
248
|
+
@retry_payload = @retry_payload.next
|
287
249
|
|
288
|
-
|
250
|
+
# If first send, exit idle mode
|
251
|
+
@timeout = Time.now.to_i + @network_timeout if @retry_payload == @first_payload
|
289
252
|
|
290
|
-
|
291
|
-
|
292
|
-
@send_cond.signal
|
293
|
-
end
|
294
|
-
end
|
295
|
-
when 'E'
|
296
|
-
# Were we expecting a payload? Store it if not
|
297
|
-
if can_send
|
298
|
-
@logger.debug 'Sending events', :events => action[1].length unless @logger.nil?
|
299
|
-
send_payload action[1]
|
300
|
-
can_send = false
|
301
|
-
else
|
302
|
-
@logger.debug 'Events received when not ready; saved to backlog' unless @logger.nil?
|
303
|
-
@received_payloads.push action[1]
|
304
|
-
end
|
305
|
-
when 'R'
|
306
|
-
# Received a message
|
307
|
-
signature, message = action[1..2]
|
308
|
-
case signature
|
309
|
-
when 'PONG'
|
310
|
-
process_pong message
|
311
|
-
when 'ACKN'
|
312
|
-
process_ackn message
|
313
|
-
else
|
314
|
-
# Unknown message - only listener is allowed to respond with a "????" message
|
315
|
-
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
316
|
-
end
|
317
|
-
|
318
|
-
# Any pending payloads left?
|
319
|
-
if @pending_payloads.length == 0
|
320
|
-
# Handle shutdown
|
321
|
-
if io_stop
|
322
|
-
raise ShutdownSignal
|
323
|
-
end
|
324
|
-
|
325
|
-
# Enter idle mode
|
326
|
-
@timeout = Time.now.to_i + @keepalive_timeout
|
327
|
-
else
|
328
|
-
# Set network timeout
|
329
|
-
@timeout = Time.now.to_i + @network_timeout
|
330
|
-
end
|
331
|
-
when 'F'
|
332
|
-
# Reconnect, an error occurred
|
333
|
-
break
|
334
|
-
when '!'
|
335
|
-
@logger.debug 'Shutdown request received' unless @logger.nil?
|
336
|
-
|
337
|
-
# Shutdown request received
|
338
|
-
if @pending_payloads.length == 0
|
339
|
-
raise ShutdownSignal
|
340
|
-
end
|
253
|
+
next
|
254
|
+
end
|
341
255
|
|
342
|
-
|
256
|
+
# Ready to send, allow spooler to pass us something if we don't
|
257
|
+
# have something already
|
258
|
+
if @received_payloads.length.zero?
|
259
|
+
@logger&.debug 'Send is ready, requesting events'
|
343
260
|
|
344
|
-
|
261
|
+
can_send = true
|
345
262
|
|
346
|
-
# Stop spooler sending
|
347
|
-
can_send = false
|
348
263
|
@send_mutex.synchronize do
|
349
|
-
@send_ready =
|
264
|
+
@send_ready = true
|
265
|
+
@send_cond.signal
|
350
266
|
end
|
267
|
+
else
|
268
|
+
@logger&.debug 'Send is ready, using events from backlog'
|
269
|
+
send_payload @received_payloads.pop
|
351
270
|
end
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
271
|
+
when 'E'
|
272
|
+
# Were we expecting a payload? Store it if not
|
273
|
+
if can_send
|
274
|
+
@logger&.debug 'Sending events', events: action[1].length
|
275
|
+
send_payload action[1]
|
276
|
+
can_send = false
|
277
|
+
else
|
278
|
+
@logger&.debug 'Events received when not ready; saved to backlog'
|
279
|
+
@received_payloads.push action[1]
|
356
280
|
end
|
357
|
-
|
358
|
-
#
|
359
|
-
|
360
|
-
|
361
|
-
|
281
|
+
when 'R'
|
282
|
+
# Received a message
|
283
|
+
signature, message = action[1..2]
|
284
|
+
case signature
|
285
|
+
when 'PONG'
|
286
|
+
process_pong message
|
287
|
+
when 'ACKN'
|
288
|
+
process_ackn message
|
289
|
+
end
|
290
|
+
# else
|
291
|
+
# Unknown message - only listener is allowed to respond with a "????" message
|
292
|
+
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
293
|
+
|
294
|
+
# Any pending payloads left?
|
295
|
+
if @pending_payloads.length.zero?
|
296
|
+
# Handle shutdown
|
297
|
+
raise ShutdownSignal if io_stop
|
298
|
+
|
299
|
+
# Enter idle mode
|
300
|
+
@timeout = Time.now.to_i + @keepalive_timeout
|
301
|
+
else
|
302
|
+
# Set network timeout
|
303
|
+
@timeout = Time.now.to_i + @network_timeout
|
362
304
|
end
|
305
|
+
when 'F'
|
306
|
+
# Reconnect, an error occurred
|
307
|
+
break
|
308
|
+
when '!'
|
309
|
+
@logger&.debug 'Shutdown request received'
|
310
|
+
|
311
|
+
# Shutdown request received
|
312
|
+
raise ShutdownSignal if @pending_payloads.length.zero?
|
313
|
+
|
314
|
+
@logger&.debug 'Delaying shutdown due to pending payloads', payloads: @pending_payloads.length
|
315
|
+
|
316
|
+
io_stop = true
|
363
317
|
|
364
318
|
# Stop spooler sending
|
365
319
|
can_send = false
|
366
320
|
@send_mutex.synchronize do
|
367
321
|
@send_ready = false
|
368
322
|
end
|
323
|
+
end
|
324
|
+
rescue TimeoutError
|
325
|
+
# Handle network timeout
|
326
|
+
raise TimeoutError if @pending_payloads != 0
|
369
327
|
|
370
|
-
|
371
|
-
|
328
|
+
# Keepalive timeout hit, timeout if we were waiting for a pong
|
329
|
+
raise TimeoutError if @pending_ping
|
372
330
|
|
373
|
-
|
331
|
+
# Stop spooler sending
|
332
|
+
can_send = false
|
333
|
+
@send_mutex.synchronize do
|
334
|
+
@send_ready = false
|
374
335
|
end
|
336
|
+
|
337
|
+
# Send PING
|
338
|
+
send_ping
|
339
|
+
|
340
|
+
@timeout = Time.now.to_i + @network_timeout
|
341
|
+
|
342
|
+
next
|
375
343
|
end
|
376
344
|
rescue ProtocolError => e
|
377
345
|
# Reconnect required due to a protocol error
|
378
|
-
@logger
|
346
|
+
@logger&.warn 'Protocol error', error: e.message
|
379
347
|
rescue TimeoutError
|
380
348
|
# Reconnect due to timeout
|
381
|
-
@logger
|
382
|
-
rescue ShutdownSignal
|
349
|
+
@logger&.warn 'Timeout occurred'
|
350
|
+
rescue ShutdownSignal
|
383
351
|
raise
|
384
352
|
rescue StandardError, NativeException => e
|
385
353
|
# Unknown error occurred
|
386
|
-
@logger
|
354
|
+
@logger&.warn e, hint: 'Unknown error'
|
387
355
|
end
|
388
356
|
|
389
357
|
def send_payload(payload)
|
390
358
|
# If we have too many pending payloads, pause the IO
|
391
|
-
if @pending_payloads.length + 1 >= @max_pending_payloads
|
392
|
-
@client.pause_send
|
393
|
-
end
|
359
|
+
@client.pause_send if @pending_payloads.length + 1 >= @max_pending_payloads
|
394
360
|
|
395
361
|
# Received some events - send them
|
396
362
|
send_jdat payload
|
397
363
|
|
398
364
|
# Leave idle mode if this is the first payload after idle
|
399
|
-
if @pending_payloads.length == 1
|
400
|
-
@timeout = Time.now.to_i + @network_timeout
|
401
|
-
end
|
365
|
+
@timeout = Time.now.to_i + @network_timeout if @pending_payloads.length == 1
|
402
366
|
end
|
403
367
|
|
404
368
|
def generate_nonce
|
405
|
-
(0...16).map { rand(256).chr }.join
|
369
|
+
(0...16).map { rand(256).chr }.join
|
406
370
|
end
|
407
371
|
|
408
372
|
def send_ping
|
409
373
|
# Send it
|
410
374
|
@client.send 'PING', ''
|
411
|
-
return
|
412
375
|
end
|
413
376
|
|
414
377
|
def send_jdat(events)
|
@@ -422,54 +385,51 @@ module LogCourier
|
|
422
385
|
|
423
386
|
if @first_payload.nil?
|
424
387
|
@first_payload = payload
|
425
|
-
@last_payload = payload
|
426
388
|
else
|
427
389
|
@last_payload.next = payload
|
428
|
-
@last_payload = payload
|
429
390
|
end
|
391
|
+
@last_payload = payload
|
430
392
|
|
431
393
|
# Send it
|
432
394
|
@client.send 'JDAT', payload.payload
|
433
|
-
return
|
434
395
|
end
|
435
396
|
|
436
397
|
def process_pong(message)
|
437
398
|
# Sanity
|
438
|
-
|
399
|
+
raise ProtocolError, "Unexpected data attached to pong message (#{message.bytesize})" if message.bytesize != 0
|
439
400
|
|
440
|
-
@logger
|
401
|
+
@logger&.debug 'PONG message received' || !@logger&.debug?
|
441
402
|
|
442
403
|
# No longer pending a PONG
|
443
404
|
@ping_pending = false
|
444
|
-
return
|
445
405
|
end
|
446
406
|
|
447
407
|
def process_ackn(message)
|
448
408
|
# Sanity
|
449
|
-
|
409
|
+
raise ProtocolError, "ACKN message size invalid (#{message.bytesize})" if message.bytesize != 20
|
450
410
|
|
451
411
|
# Grab nonce
|
452
412
|
nonce, sequence = message.unpack('a16N')
|
453
413
|
|
454
|
-
if
|
414
|
+
if @logger&.debug?
|
455
415
|
nonce_str = nonce.each_byte.map do |b|
|
456
416
|
b.to_s(16).rjust(2, '0')
|
457
417
|
end
|
458
418
|
|
459
|
-
@logger
|
419
|
+
@logger&.debug 'ACKN message received', nonce: nonce_str.join, sequence: sequence
|
460
420
|
end
|
461
421
|
|
462
422
|
# Find the payload - skip if we couldn't as it will just a duplicated ACK
|
463
423
|
return unless @pending_payloads.key?(nonce)
|
464
424
|
|
465
425
|
payload = @pending_payloads[nonce]
|
466
|
-
|
426
|
+
_, complete = payload.ack(sequence)
|
467
427
|
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
428
|
+
return unless complete
|
429
|
+
|
430
|
+
@pending_payloads.delete nonce
|
431
|
+
@first_payload = payload.next
|
432
|
+
@client.resume_send if @client.send_paused?
|
473
433
|
end
|
474
434
|
end
|
475
435
|
end
|