log-courier 1.10.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/log-courier/client.rb +141 -181
- data/lib/log-courier/client_tcp.rb +117 -87
- data/lib/log-courier/event_queue.rb +18 -23
- data/lib/log-courier/server.rb +32 -58
- data/lib/log-courier/server_tcp.rb +174 -120
- metadata +9 -26
- data/lib/log-courier/server_zmq.rb +0 -400
- data/lib/log-courier/zmq_qpoll.rb +0 -324
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eeea3f53b4849932879f09791e5f0f842e210546916797791c89426bcbffff61
|
4
|
+
data.tar.gz: 8c0d77a0dfd6dbc1b48e511dee3ff8d91954f33610ddaebbcc64cf07fbd02f79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f4c1a27c6024bf6787bd56f60b2af15a0518e4ba250dcb1b3ff7da70ffe961006cff3c4bc6d0595dea0af43307a652457d8c6d715b765b88c1641250d8e327b5
|
7
|
+
data.tar.gz: 9c83a0e1f3bdf516bab3917ccaea04e33bb1ee8902708820cad651ed905be765e84b120da79798cc68ffe2f5b2a8f03fac0c7193b3179b13e9b4c251a0a78636
|
data/lib/log-courier/client.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
# Copyright 2014-2019 Jason Woods and Contributors.
|
1
|
+
# Copyright 2014-2021 Jason Woods and Contributors.
|
4
2
|
#
|
5
3
|
# This file is a modification of code from Logstash Forwarder.
|
6
4
|
# Copyright 2012-2013 Jordan Sissel and contributors.
|
@@ -18,43 +16,25 @@
|
|
18
16
|
# limitations under the License.
|
19
17
|
|
20
18
|
require 'log-courier/event_queue'
|
19
|
+
require 'log-courier/protocol'
|
21
20
|
require 'multi_json'
|
22
|
-
require 'thread'
|
23
21
|
require 'zlib'
|
24
22
|
|
25
|
-
|
23
|
+
# Dummy until its no longer needed
|
24
|
+
class NativeException
|
25
|
+
def dummy; end
|
26
|
+
end
|
26
27
|
|
27
28
|
module LogCourier
|
28
29
|
class TimeoutError < StandardError; end
|
30
|
+
|
29
31
|
class ShutdownSignal < StandardError; end
|
32
|
+
|
30
33
|
class ProtocolError < StandardError; end
|
31
34
|
|
32
35
|
# Describes a pending payload
|
33
36
|
class PendingPayload
|
34
|
-
|
35
|
-
class << self
|
36
|
-
@json_adapter
|
37
|
-
@json_parseerror
|
38
|
-
|
39
|
-
def get_json_adapter
|
40
|
-
@json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
|
41
|
-
@json_adapter
|
42
|
-
end
|
43
|
-
|
44
|
-
def get_json_parseerror
|
45
|
-
if @json_parseerror.nil?
|
46
|
-
@json_parseerror = get_json_adapter.class::ParseError
|
47
|
-
end
|
48
|
-
@json_parseerror
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
attr_accessor :next
|
53
|
-
attr_accessor :nonce
|
54
|
-
attr_accessor :events
|
55
|
-
attr_accessor :last_sequence
|
56
|
-
attr_accessor :sequence_len
|
57
|
-
attr_accessor :payload
|
37
|
+
attr_accessor :next, :nonce, :events, :last_sequence, :sequence_len, :payload
|
58
38
|
|
59
39
|
def initialize(events, nonce)
|
60
40
|
@events = events
|
@@ -64,27 +44,27 @@ module LogCourier
|
|
64
44
|
end
|
65
45
|
|
66
46
|
def generate
|
67
|
-
|
47
|
+
raise ArgumentError, 'Corrupt payload' if @events.length.zero?
|
68
48
|
|
69
49
|
buffer = Zlib::Deflate.new
|
70
50
|
|
71
51
|
# Write each event in JSON format
|
72
52
|
events.each do |event|
|
73
|
-
json_data =
|
53
|
+
json_data = MultiJson.dump(event)
|
74
54
|
# Add length and then the data
|
75
55
|
buffer << [json_data.bytesize].pack('N') << json_data
|
76
56
|
end
|
77
57
|
|
78
58
|
# Generate and store the payload
|
79
|
-
@payload = nonce + buffer.finish
|
59
|
+
@payload = nonce + buffer.finish
|
80
60
|
@last_sequence = 0
|
81
61
|
@sequence_len = @events.length
|
82
62
|
end
|
83
63
|
|
84
64
|
def ack(sequence)
|
85
|
-
if sequence <= @last_sequence
|
86
|
-
|
87
|
-
|
65
|
+
return 0, false if sequence <= @last_sequence
|
66
|
+
|
67
|
+
if sequence >= @sequence_len
|
88
68
|
lines = @sequence_len - @last_sequence
|
89
69
|
@last_sequence = sequence
|
90
70
|
@payload = nil
|
@@ -96,7 +76,7 @@ module LogCourier
|
|
96
76
|
@last_sequence = sequence
|
97
77
|
@payload = nil
|
98
78
|
@events.shift(lines)
|
99
|
-
|
79
|
+
[lines, false]
|
100
80
|
end
|
101
81
|
end
|
102
82
|
|
@@ -104,12 +84,14 @@ module LogCourier
|
|
104
84
|
class Client
|
105
85
|
def initialize(options = {})
|
106
86
|
@options = {
|
107
|
-
logger:
|
108
|
-
transport:
|
109
|
-
spool_size:
|
87
|
+
logger: nil,
|
88
|
+
transport: 'tls',
|
89
|
+
spool_size: 1024,
|
110
90
|
idle_timeout: 5,
|
111
|
-
port:
|
112
|
-
addresses:
|
91
|
+
port: nil,
|
92
|
+
addresses: [],
|
93
|
+
min_tls_version: 1.2,
|
94
|
+
disable_handshake: false,
|
113
95
|
}.merge!(options)
|
114
96
|
|
115
97
|
@logger = @options[:logger]
|
@@ -119,11 +101,11 @@ module LogCourier
|
|
119
101
|
require 'log-courier/client_tcp'
|
120
102
|
@client = ClientTcp.new(@options)
|
121
103
|
else
|
122
|
-
|
104
|
+
raise 'output/courier: \'transport\' must be tcp or tls'
|
123
105
|
end
|
124
106
|
|
125
|
-
|
126
|
-
|
107
|
+
raise 'output/courier: \'addresses\' must contain at least one address' if @options[:addresses].empty?
|
108
|
+
raise 'output/courier: \'addresses\' only supports a single address at this time' if @options[:addresses].length > 1
|
127
109
|
|
128
110
|
@event_queue = EventQueue.new @options[:spool_size]
|
129
111
|
@pending_payloads = {}
|
@@ -160,10 +142,10 @@ module LogCourier
|
|
160
142
|
def publish(event)
|
161
143
|
# Pass the event into the spooler
|
162
144
|
@event_queue << event
|
163
|
-
|
145
|
+
nil
|
164
146
|
end
|
165
147
|
|
166
|
-
def shutdown(force=false)
|
148
|
+
def shutdown(force = false) # rubocop:disable Style/OptionalBooleanParameter
|
167
149
|
if force
|
168
150
|
# Raise a shutdown signal in the spooler and wait for it
|
169
151
|
@spooler_thread.raise ShutdownSignal
|
@@ -175,7 +157,7 @@ module LogCourier
|
|
175
157
|
@io_control << ['!', nil]
|
176
158
|
end
|
177
159
|
@io_thread.join
|
178
|
-
|
160
|
+
@pending_payloads.length.zero?
|
179
161
|
end
|
180
162
|
|
181
163
|
private
|
@@ -190,9 +172,7 @@ module LogCourier
|
|
190
172
|
loop do
|
191
173
|
event = @event_queue.pop next_flush - Time.now.to_i
|
192
174
|
|
193
|
-
if event.nil?
|
194
|
-
raise ShutdownSignal
|
195
|
-
end
|
175
|
+
raise ShutdownSignal if event.nil?
|
196
176
|
|
197
177
|
spooled.push(event)
|
198
178
|
|
@@ -200,13 +180,13 @@ module LogCourier
|
|
200
180
|
end
|
201
181
|
rescue TimeoutError
|
202
182
|
# Hit timeout but no events, keep waiting
|
203
|
-
next if spooled.length
|
183
|
+
next if spooled.length.zero?
|
204
184
|
end
|
205
185
|
|
206
186
|
if spooled.length >= @options[:spool_size]
|
207
|
-
@logger
|
187
|
+
@logger&.debug 'Flushing full spool', events: spooled.length
|
208
188
|
else
|
209
|
-
@logger
|
189
|
+
@logger&.debug 'Flushing spool due to timeout', events: spooled.length
|
210
190
|
end
|
211
191
|
|
212
192
|
# Pass through to io_control but only if we're ready to send
|
@@ -217,9 +197,8 @@ module LogCourier
|
|
217
197
|
|
218
198
|
@io_control << ['E', spooled]
|
219
199
|
end
|
220
|
-
return
|
221
200
|
rescue ShutdownSignal
|
222
|
-
|
201
|
+
# Shutdown
|
223
202
|
end
|
224
203
|
|
225
204
|
def run_io
|
@@ -240,175 +219,159 @@ module LogCourier
|
|
240
219
|
end
|
241
220
|
|
242
221
|
@client.disconnect
|
243
|
-
return
|
244
222
|
rescue ShutdownSignal
|
245
223
|
# Ensure disconnected
|
246
224
|
@client.disconnect
|
247
225
|
end
|
248
226
|
|
249
|
-
def run_io_loop
|
227
|
+
def run_io_loop
|
250
228
|
io_stop = false
|
251
229
|
can_send = false
|
252
230
|
|
253
231
|
# IO loop
|
254
232
|
loop do
|
255
|
-
|
256
|
-
action = @io_control.pop @timeout - Time.now.to_i
|
257
|
-
|
258
|
-
# Process the action
|
259
|
-
case action[0]
|
260
|
-
when 'S'
|
261
|
-
# If we're flushing through the pending, pick from there
|
262
|
-
unless @retry_payload.nil?
|
263
|
-
@logger.debug 'Send is ready, retrying previous payload' unless @logger.nil?
|
233
|
+
action = @io_control.pop @timeout - Time.now.to_i
|
264
234
|
|
265
|
-
|
266
|
-
|
235
|
+
# Process the action
|
236
|
+
case action[0]
|
237
|
+
when 'S'
|
238
|
+
# If we're flushing through the pending, pick from there
|
239
|
+
unless @retry_payload.nil?
|
240
|
+
@logger&.debug 'Send is ready, retrying previous payload'
|
267
241
|
|
268
|
-
|
269
|
-
|
242
|
+
# Regenerate data if we need to
|
243
|
+
@retry_payload.generate if @retry_payload.payload.nil?
|
270
244
|
|
271
|
-
|
245
|
+
# Send and move onto next
|
246
|
+
@client.send 'JDAT', @retry_payload.payload
|
272
247
|
|
273
|
-
|
274
|
-
if @retry_payload == @first_payload
|
275
|
-
@timeout = Time.now.to_i + @network_timeout
|
276
|
-
end
|
277
|
-
next
|
278
|
-
end
|
279
|
-
|
280
|
-
# Ready to send, allow spooler to pass us something if we don't
|
281
|
-
# have something already
|
282
|
-
if @received_payloads.length != 0
|
283
|
-
@logger.debug 'Send is ready, using events from backlog' unless @logger.nil?
|
284
|
-
send_payload @received_payloads.pop()
|
285
|
-
else
|
286
|
-
@logger.debug 'Send is ready, requesting events' unless @logger.nil?
|
248
|
+
@retry_payload = @retry_payload.next
|
287
249
|
|
288
|
-
|
250
|
+
# If first send, exit idle mode
|
251
|
+
@timeout = Time.now.to_i + @network_timeout if @retry_payload == @first_payload
|
289
252
|
|
290
|
-
|
291
|
-
|
292
|
-
@send_cond.signal
|
293
|
-
end
|
294
|
-
end
|
295
|
-
when 'E'
|
296
|
-
# Were we expecting a payload? Store it if not
|
297
|
-
if can_send
|
298
|
-
@logger.debug 'Sending events', :events => action[1].length unless @logger.nil?
|
299
|
-
send_payload action[1]
|
300
|
-
can_send = false
|
301
|
-
else
|
302
|
-
@logger.debug 'Events received when not ready; saved to backlog' unless @logger.nil?
|
303
|
-
@received_payloads.push action[1]
|
304
|
-
end
|
305
|
-
when 'R'
|
306
|
-
# Received a message
|
307
|
-
signature, message = action[1..2]
|
308
|
-
case signature
|
309
|
-
when 'PONG'
|
310
|
-
process_pong message
|
311
|
-
when 'ACKN'
|
312
|
-
process_ackn message
|
313
|
-
else
|
314
|
-
# Unknown message - only listener is allowed to respond with a "????" message
|
315
|
-
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
316
|
-
end
|
317
|
-
|
318
|
-
# Any pending payloads left?
|
319
|
-
if @pending_payloads.length == 0
|
320
|
-
# Handle shutdown
|
321
|
-
if io_stop
|
322
|
-
raise ShutdownSignal
|
323
|
-
end
|
324
|
-
|
325
|
-
# Enter idle mode
|
326
|
-
@timeout = Time.now.to_i + @keepalive_timeout
|
327
|
-
else
|
328
|
-
# Set network timeout
|
329
|
-
@timeout = Time.now.to_i + @network_timeout
|
330
|
-
end
|
331
|
-
when 'F'
|
332
|
-
# Reconnect, an error occurred
|
333
|
-
break
|
334
|
-
when '!'
|
335
|
-
@logger.debug 'Shutdown request received' unless @logger.nil?
|
336
|
-
|
337
|
-
# Shutdown request received
|
338
|
-
if @pending_payloads.length == 0
|
339
|
-
raise ShutdownSignal
|
340
|
-
end
|
253
|
+
next
|
254
|
+
end
|
341
255
|
|
342
|
-
|
256
|
+
# Ready to send, allow spooler to pass us something if we don't
|
257
|
+
# have something already
|
258
|
+
if @received_payloads.length.zero?
|
259
|
+
@logger&.debug 'Send is ready, requesting events'
|
343
260
|
|
344
|
-
|
261
|
+
can_send = true
|
345
262
|
|
346
|
-
# Stop spooler sending
|
347
|
-
can_send = false
|
348
263
|
@send_mutex.synchronize do
|
349
|
-
@send_ready =
|
264
|
+
@send_ready = true
|
265
|
+
@send_cond.signal
|
350
266
|
end
|
267
|
+
else
|
268
|
+
@logger&.debug 'Send is ready, using events from backlog'
|
269
|
+
send_payload @received_payloads.pop
|
351
270
|
end
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
271
|
+
when 'E'
|
272
|
+
# Were we expecting a payload? Store it if not
|
273
|
+
if can_send
|
274
|
+
@logger&.debug 'Sending events', events: action[1].length
|
275
|
+
send_payload action[1]
|
276
|
+
can_send = false
|
277
|
+
else
|
278
|
+
@logger&.debug 'Events received when not ready; saved to backlog'
|
279
|
+
@received_payloads.push action[1]
|
356
280
|
end
|
357
|
-
|
358
|
-
#
|
359
|
-
|
360
|
-
|
361
|
-
|
281
|
+
when 'R'
|
282
|
+
# Received a message
|
283
|
+
signature, message = action[1..2]
|
284
|
+
case signature
|
285
|
+
when 'PONG'
|
286
|
+
process_pong message
|
287
|
+
when 'ACKN'
|
288
|
+
process_ackn message
|
289
|
+
end
|
290
|
+
# else
|
291
|
+
# Unknown message - only listener is allowed to respond with a "????" message
|
292
|
+
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
293
|
+
|
294
|
+
# Any pending payloads left?
|
295
|
+
if @pending_payloads.length.zero?
|
296
|
+
# Handle shutdown
|
297
|
+
raise ShutdownSignal if io_stop
|
298
|
+
|
299
|
+
# Enter idle mode
|
300
|
+
@timeout = Time.now.to_i + @keepalive_timeout
|
301
|
+
else
|
302
|
+
# Set network timeout
|
303
|
+
@timeout = Time.now.to_i + @network_timeout
|
362
304
|
end
|
305
|
+
when 'F'
|
306
|
+
# Reconnect, an error occurred
|
307
|
+
break
|
308
|
+
when '!'
|
309
|
+
@logger&.debug 'Shutdown request received'
|
310
|
+
|
311
|
+
# Shutdown request received
|
312
|
+
raise ShutdownSignal if @pending_payloads.length.zero?
|
313
|
+
|
314
|
+
@logger&.debug 'Delaying shutdown due to pending payloads', payloads: @pending_payloads.length
|
315
|
+
|
316
|
+
io_stop = true
|
363
317
|
|
364
318
|
# Stop spooler sending
|
365
319
|
can_send = false
|
366
320
|
@send_mutex.synchronize do
|
367
321
|
@send_ready = false
|
368
322
|
end
|
323
|
+
end
|
324
|
+
rescue TimeoutError
|
325
|
+
# Handle network timeout
|
326
|
+
raise TimeoutError if @pending_payloads != 0
|
369
327
|
|
370
|
-
|
371
|
-
|
328
|
+
# Keepalive timeout hit, timeout if we were waiting for a pong
|
329
|
+
raise TimeoutError if @pending_ping
|
372
330
|
|
373
|
-
|
331
|
+
# Stop spooler sending
|
332
|
+
can_send = false
|
333
|
+
@send_mutex.synchronize do
|
334
|
+
@send_ready = false
|
374
335
|
end
|
336
|
+
|
337
|
+
# Send PING
|
338
|
+
send_ping
|
339
|
+
|
340
|
+
@timeout = Time.now.to_i + @network_timeout
|
341
|
+
|
342
|
+
next
|
375
343
|
end
|
376
344
|
rescue ProtocolError => e
|
377
345
|
# Reconnect required due to a protocol error
|
378
|
-
@logger
|
346
|
+
@logger&.warn 'Protocol error', error: e.message
|
379
347
|
rescue TimeoutError
|
380
348
|
# Reconnect due to timeout
|
381
|
-
@logger
|
382
|
-
rescue ShutdownSignal
|
349
|
+
@logger&.warn 'Timeout occurred'
|
350
|
+
rescue ShutdownSignal
|
383
351
|
raise
|
384
352
|
rescue StandardError, NativeException => e
|
385
353
|
# Unknown error occurred
|
386
|
-
@logger
|
354
|
+
@logger&.warn e, hint: 'Unknown error'
|
387
355
|
end
|
388
356
|
|
389
357
|
def send_payload(payload)
|
390
358
|
# If we have too many pending payloads, pause the IO
|
391
|
-
if @pending_payloads.length + 1 >= @max_pending_payloads
|
392
|
-
@client.pause_send
|
393
|
-
end
|
359
|
+
@client.pause_send if @pending_payloads.length + 1 >= @max_pending_payloads
|
394
360
|
|
395
361
|
# Received some events - send them
|
396
362
|
send_jdat payload
|
397
363
|
|
398
364
|
# Leave idle mode if this is the first payload after idle
|
399
|
-
if @pending_payloads.length == 1
|
400
|
-
@timeout = Time.now.to_i + @network_timeout
|
401
|
-
end
|
365
|
+
@timeout = Time.now.to_i + @network_timeout if @pending_payloads.length == 1
|
402
366
|
end
|
403
367
|
|
404
368
|
def generate_nonce
|
405
|
-
(0...16).map { rand(256).chr }.join
|
369
|
+
(0...16).map { rand(256).chr }.join
|
406
370
|
end
|
407
371
|
|
408
372
|
def send_ping
|
409
373
|
# Send it
|
410
374
|
@client.send 'PING', ''
|
411
|
-
return
|
412
375
|
end
|
413
376
|
|
414
377
|
def send_jdat(events)
|
@@ -422,54 +385,51 @@ module LogCourier
|
|
422
385
|
|
423
386
|
if @first_payload.nil?
|
424
387
|
@first_payload = payload
|
425
|
-
@last_payload = payload
|
426
388
|
else
|
427
389
|
@last_payload.next = payload
|
428
|
-
@last_payload = payload
|
429
390
|
end
|
391
|
+
@last_payload = payload
|
430
392
|
|
431
393
|
# Send it
|
432
394
|
@client.send 'JDAT', payload.payload
|
433
|
-
return
|
434
395
|
end
|
435
396
|
|
436
397
|
def process_pong(message)
|
437
398
|
# Sanity
|
438
|
-
|
399
|
+
raise ProtocolError, "Unexpected data attached to pong message (#{message.bytesize})" if message.bytesize != 0
|
439
400
|
|
440
|
-
@logger
|
401
|
+
@logger&.debug 'PONG message received' || !@logger&.debug?
|
441
402
|
|
442
403
|
# No longer pending a PONG
|
443
404
|
@ping_pending = false
|
444
|
-
return
|
445
405
|
end
|
446
406
|
|
447
407
|
def process_ackn(message)
|
448
408
|
# Sanity
|
449
|
-
|
409
|
+
raise ProtocolError, "ACKN message size invalid (#{message.bytesize})" if message.bytesize != 20
|
450
410
|
|
451
411
|
# Grab nonce
|
452
412
|
nonce, sequence = message.unpack('a16N')
|
453
413
|
|
454
|
-
if
|
414
|
+
if @logger&.debug?
|
455
415
|
nonce_str = nonce.each_byte.map do |b|
|
456
416
|
b.to_s(16).rjust(2, '0')
|
457
417
|
end
|
458
418
|
|
459
|
-
@logger
|
419
|
+
@logger&.debug 'ACKN message received', nonce: nonce_str.join, sequence: sequence
|
460
420
|
end
|
461
421
|
|
462
422
|
# Find the payload - skip if we couldn't as it will just a duplicated ACK
|
463
423
|
return unless @pending_payloads.key?(nonce)
|
464
424
|
|
465
425
|
payload = @pending_payloads[nonce]
|
466
|
-
|
426
|
+
_, complete = payload.ack(sequence)
|
467
427
|
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
428
|
+
return unless complete
|
429
|
+
|
430
|
+
@pending_payloads.delete nonce
|
431
|
+
@first_payload = payload.next
|
432
|
+
@client.resume_send if @client.send_paused?
|
473
433
|
end
|
474
434
|
end
|
475
435
|
end
|