log-courier 1.10.0 → 2.7.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/log-courier/client.rb +139 -183
- data/lib/log-courier/client_tcp.rb +121 -95
- data/lib/log-courier/event_queue.rb +18 -23
- data/lib/log-courier/protocol.rb +58 -0
- data/lib/log-courier/rspec/spec_helper.rb +217 -0
- data/lib/log-courier/server.rb +29 -59
- data/lib/log-courier/server_tcp.rb +175 -121
- metadata +12 -27
- data/lib/log-courier/server_zmq.rb +0 -400
- data/lib/log-courier/zmq_qpoll.rb +0 -324
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3aa4bc695758102c35a9bc74cb856ec456b92715cb6f133592bfdeaa0a8a6a30
|
4
|
+
data.tar.gz: c5f67b639fca236f884f01f3054dc6e4b3292826051f1696cff324f80e6878ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e0826718c6e075b731d04f7f8c825aa0fd59b9d0045b58183009f42516e1f1a7b8465cd72af7d04603d8946e3240acba91101fe41cdb6bcc609856a8b50d171d
|
7
|
+
data.tar.gz: 372bbefcfd75d6de6139fc6b0d1901a77cb77609c104fe897d156b29c83edca6994abb9bf8a44a718d963e0b20a3127fe8d91615f0ae4aaff05cea1de31bbbf8
|
data/lib/log-courier/client.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
# Copyright 2014-2019 Jason Woods and Contributors.
|
1
|
+
# Copyright 2014-2021 Jason Woods and Contributors.
|
4
2
|
#
|
5
3
|
# This file is a modification of code from Logstash Forwarder.
|
6
4
|
# Copyright 2012-2013 Jordan Sissel and contributors.
|
@@ -18,43 +16,21 @@
|
|
18
16
|
# limitations under the License.
|
19
17
|
|
20
18
|
require 'log-courier/event_queue'
|
19
|
+
require 'log-courier/protocol'
|
20
|
+
require 'log-courier/version'
|
21
21
|
require 'multi_json'
|
22
|
-
require 'thread'
|
23
22
|
require 'zlib'
|
24
23
|
|
25
|
-
class NativeException; end
|
26
|
-
|
27
24
|
module LogCourier
|
28
25
|
class TimeoutError < StandardError; end
|
26
|
+
|
29
27
|
class ShutdownSignal < StandardError; end
|
28
|
+
|
30
29
|
class ProtocolError < StandardError; end
|
31
30
|
|
32
31
|
# Describes a pending payload
|
33
32
|
class PendingPayload
|
34
|
-
|
35
|
-
class << self
|
36
|
-
@json_adapter
|
37
|
-
@json_parseerror
|
38
|
-
|
39
|
-
def get_json_adapter
|
40
|
-
@json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
|
41
|
-
@json_adapter
|
42
|
-
end
|
43
|
-
|
44
|
-
def get_json_parseerror
|
45
|
-
if @json_parseerror.nil?
|
46
|
-
@json_parseerror = get_json_adapter.class::ParseError
|
47
|
-
end
|
48
|
-
@json_parseerror
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
attr_accessor :next
|
53
|
-
attr_accessor :nonce
|
54
|
-
attr_accessor :events
|
55
|
-
attr_accessor :last_sequence
|
56
|
-
attr_accessor :sequence_len
|
57
|
-
attr_accessor :payload
|
33
|
+
attr_accessor :next, :nonce, :events, :last_sequence, :sequence_len, :payload
|
58
34
|
|
59
35
|
def initialize(events, nonce)
|
60
36
|
@events = events
|
@@ -64,27 +40,27 @@ module LogCourier
|
|
64
40
|
end
|
65
41
|
|
66
42
|
def generate
|
67
|
-
|
43
|
+
raise ArgumentError, 'Corrupt payload' if @events.length.zero?
|
68
44
|
|
69
45
|
buffer = Zlib::Deflate.new
|
70
46
|
|
71
47
|
# Write each event in JSON format
|
72
48
|
events.each do |event|
|
73
|
-
json_data =
|
49
|
+
json_data = MultiJson.dump(event)
|
74
50
|
# Add length and then the data
|
75
51
|
buffer << [json_data.bytesize].pack('N') << json_data
|
76
52
|
end
|
77
53
|
|
78
54
|
# Generate and store the payload
|
79
|
-
@payload = nonce + buffer.finish
|
55
|
+
@payload = nonce + buffer.finish
|
80
56
|
@last_sequence = 0
|
81
57
|
@sequence_len = @events.length
|
82
58
|
end
|
83
59
|
|
84
60
|
def ack(sequence)
|
85
|
-
if sequence <= @last_sequence
|
86
|
-
|
87
|
-
|
61
|
+
return 0, false if sequence <= @last_sequence
|
62
|
+
|
63
|
+
if sequence >= @sequence_len
|
88
64
|
lines = @sequence_len - @last_sequence
|
89
65
|
@last_sequence = sequence
|
90
66
|
@payload = nil
|
@@ -96,7 +72,7 @@ module LogCourier
|
|
96
72
|
@last_sequence = sequence
|
97
73
|
@payload = nil
|
98
74
|
@events.shift(lines)
|
99
|
-
|
75
|
+
[lines, false]
|
100
76
|
end
|
101
77
|
end
|
102
78
|
|
@@ -104,12 +80,14 @@ module LogCourier
|
|
104
80
|
class Client
|
105
81
|
def initialize(options = {})
|
106
82
|
@options = {
|
107
|
-
logger:
|
108
|
-
transport:
|
109
|
-
spool_size:
|
83
|
+
logger: nil,
|
84
|
+
transport: 'tls',
|
85
|
+
spool_size: 1024,
|
110
86
|
idle_timeout: 5,
|
111
|
-
port:
|
112
|
-
addresses:
|
87
|
+
port: nil,
|
88
|
+
addresses: [],
|
89
|
+
min_tls_version: 1.2,
|
90
|
+
disable_handshake: false,
|
113
91
|
}.merge!(options)
|
114
92
|
|
115
93
|
@logger = @options[:logger]
|
@@ -119,11 +97,11 @@ module LogCourier
|
|
119
97
|
require 'log-courier/client_tcp'
|
120
98
|
@client = ClientTcp.new(@options)
|
121
99
|
else
|
122
|
-
|
100
|
+
raise 'output/courier: \'transport\' must be tcp or tls'
|
123
101
|
end
|
124
102
|
|
125
|
-
|
126
|
-
|
103
|
+
raise 'output/courier: \'addresses\' must contain at least one address' if @options[:addresses].empty?
|
104
|
+
raise 'output/courier: \'addresses\' only supports a single address at this time' if @options[:addresses].length > 1
|
127
105
|
|
128
106
|
@event_queue = EventQueue.new @options[:spool_size]
|
129
107
|
@pending_payloads = {}
|
@@ -160,10 +138,10 @@ module LogCourier
|
|
160
138
|
def publish(event)
|
161
139
|
# Pass the event into the spooler
|
162
140
|
@event_queue << event
|
163
|
-
|
141
|
+
nil
|
164
142
|
end
|
165
143
|
|
166
|
-
def shutdown(force=false)
|
144
|
+
def shutdown(force = false) # rubocop:disable Style/OptionalBooleanParameter
|
167
145
|
if force
|
168
146
|
# Raise a shutdown signal in the spooler and wait for it
|
169
147
|
@spooler_thread.raise ShutdownSignal
|
@@ -175,7 +153,7 @@ module LogCourier
|
|
175
153
|
@io_control << ['!', nil]
|
176
154
|
end
|
177
155
|
@io_thread.join
|
178
|
-
|
156
|
+
@pending_payloads.length.zero?
|
179
157
|
end
|
180
158
|
|
181
159
|
private
|
@@ -190,9 +168,7 @@ module LogCourier
|
|
190
168
|
loop do
|
191
169
|
event = @event_queue.pop next_flush - Time.now.to_i
|
192
170
|
|
193
|
-
if event.nil?
|
194
|
-
raise ShutdownSignal
|
195
|
-
end
|
171
|
+
raise ShutdownSignal if event.nil?
|
196
172
|
|
197
173
|
spooled.push(event)
|
198
174
|
|
@@ -200,13 +176,13 @@ module LogCourier
|
|
200
176
|
end
|
201
177
|
rescue TimeoutError
|
202
178
|
# Hit timeout but no events, keep waiting
|
203
|
-
next if spooled.length
|
179
|
+
next if spooled.length.zero?
|
204
180
|
end
|
205
181
|
|
206
182
|
if spooled.length >= @options[:spool_size]
|
207
|
-
@logger
|
183
|
+
@logger&.debug 'Flushing full spool', events: spooled.length
|
208
184
|
else
|
209
|
-
@logger
|
185
|
+
@logger&.debug 'Flushing spool due to timeout', events: spooled.length
|
210
186
|
end
|
211
187
|
|
212
188
|
# Pass through to io_control but only if we're ready to send
|
@@ -217,9 +193,8 @@ module LogCourier
|
|
217
193
|
|
218
194
|
@io_control << ['E', spooled]
|
219
195
|
end
|
220
|
-
return
|
221
196
|
rescue ShutdownSignal
|
222
|
-
|
197
|
+
# Shutdown
|
223
198
|
end
|
224
199
|
|
225
200
|
def run_io
|
@@ -240,175 +215,159 @@ module LogCourier
|
|
240
215
|
end
|
241
216
|
|
242
217
|
@client.disconnect
|
243
|
-
return
|
244
218
|
rescue ShutdownSignal
|
245
219
|
# Ensure disconnected
|
246
220
|
@client.disconnect
|
247
221
|
end
|
248
222
|
|
249
|
-
def run_io_loop
|
223
|
+
def run_io_loop
|
250
224
|
io_stop = false
|
251
225
|
can_send = false
|
252
226
|
|
253
227
|
# IO loop
|
254
228
|
loop do
|
255
|
-
|
256
|
-
action = @io_control.pop @timeout - Time.now.to_i
|
257
|
-
|
258
|
-
# Process the action
|
259
|
-
case action[0]
|
260
|
-
when 'S'
|
261
|
-
# If we're flushing through the pending, pick from there
|
262
|
-
unless @retry_payload.nil?
|
263
|
-
@logger.debug 'Send is ready, retrying previous payload' unless @logger.nil?
|
229
|
+
action = @io_control.pop @timeout - Time.now.to_i
|
264
230
|
|
265
|
-
|
266
|
-
|
231
|
+
# Process the action
|
232
|
+
case action[0]
|
233
|
+
when 'S'
|
234
|
+
# If we're flushing through the pending, pick from there
|
235
|
+
unless @retry_payload.nil?
|
236
|
+
@logger&.debug 'Send is ready, retrying previous payload'
|
267
237
|
|
268
|
-
|
269
|
-
|
238
|
+
# Regenerate data if we need to
|
239
|
+
@retry_payload.generate if @retry_payload.payload.nil?
|
270
240
|
|
271
|
-
|
241
|
+
# Send and move onto next
|
242
|
+
@client.send 'JDAT', @retry_payload.payload
|
272
243
|
|
273
|
-
|
274
|
-
if @retry_payload == @first_payload
|
275
|
-
@timeout = Time.now.to_i + @network_timeout
|
276
|
-
end
|
277
|
-
next
|
278
|
-
end
|
279
|
-
|
280
|
-
# Ready to send, allow spooler to pass us something if we don't
|
281
|
-
# have something already
|
282
|
-
if @received_payloads.length != 0
|
283
|
-
@logger.debug 'Send is ready, using events from backlog' unless @logger.nil?
|
284
|
-
send_payload @received_payloads.pop()
|
285
|
-
else
|
286
|
-
@logger.debug 'Send is ready, requesting events' unless @logger.nil?
|
244
|
+
@retry_payload = @retry_payload.next
|
287
245
|
|
288
|
-
|
246
|
+
# If first send, exit idle mode
|
247
|
+
@timeout = Time.now.to_i + @network_timeout if @retry_payload == @first_payload
|
289
248
|
|
290
|
-
|
291
|
-
|
292
|
-
@send_cond.signal
|
293
|
-
end
|
294
|
-
end
|
295
|
-
when 'E'
|
296
|
-
# Were we expecting a payload? Store it if not
|
297
|
-
if can_send
|
298
|
-
@logger.debug 'Sending events', :events => action[1].length unless @logger.nil?
|
299
|
-
send_payload action[1]
|
300
|
-
can_send = false
|
301
|
-
else
|
302
|
-
@logger.debug 'Events received when not ready; saved to backlog' unless @logger.nil?
|
303
|
-
@received_payloads.push action[1]
|
304
|
-
end
|
305
|
-
when 'R'
|
306
|
-
# Received a message
|
307
|
-
signature, message = action[1..2]
|
308
|
-
case signature
|
309
|
-
when 'PONG'
|
310
|
-
process_pong message
|
311
|
-
when 'ACKN'
|
312
|
-
process_ackn message
|
313
|
-
else
|
314
|
-
# Unknown message - only listener is allowed to respond with a "????" message
|
315
|
-
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
316
|
-
end
|
317
|
-
|
318
|
-
# Any pending payloads left?
|
319
|
-
if @pending_payloads.length == 0
|
320
|
-
# Handle shutdown
|
321
|
-
if io_stop
|
322
|
-
raise ShutdownSignal
|
323
|
-
end
|
324
|
-
|
325
|
-
# Enter idle mode
|
326
|
-
@timeout = Time.now.to_i + @keepalive_timeout
|
327
|
-
else
|
328
|
-
# Set network timeout
|
329
|
-
@timeout = Time.now.to_i + @network_timeout
|
330
|
-
end
|
331
|
-
when 'F'
|
332
|
-
# Reconnect, an error occurred
|
333
|
-
break
|
334
|
-
when '!'
|
335
|
-
@logger.debug 'Shutdown request received' unless @logger.nil?
|
336
|
-
|
337
|
-
# Shutdown request received
|
338
|
-
if @pending_payloads.length == 0
|
339
|
-
raise ShutdownSignal
|
340
|
-
end
|
249
|
+
next
|
250
|
+
end
|
341
251
|
|
342
|
-
|
252
|
+
# Ready to send, allow spooler to pass us something if we don't
|
253
|
+
# have something already
|
254
|
+
if @received_payloads.length.zero?
|
255
|
+
@logger&.debug 'Send is ready, requesting events'
|
343
256
|
|
344
|
-
|
257
|
+
can_send = true
|
345
258
|
|
346
|
-
# Stop spooler sending
|
347
|
-
can_send = false
|
348
259
|
@send_mutex.synchronize do
|
349
|
-
@send_ready =
|
260
|
+
@send_ready = true
|
261
|
+
@send_cond.signal
|
350
262
|
end
|
263
|
+
else
|
264
|
+
@logger&.debug 'Send is ready, using events from backlog'
|
265
|
+
send_payload @received_payloads.pop
|
351
266
|
end
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
267
|
+
when 'E'
|
268
|
+
# Were we expecting a payload? Store it if not
|
269
|
+
if can_send
|
270
|
+
@logger&.debug 'Sending events', events: action[1].length
|
271
|
+
send_payload action[1]
|
272
|
+
can_send = false
|
273
|
+
else
|
274
|
+
@logger&.debug 'Events received when not ready; saved to backlog'
|
275
|
+
@received_payloads.push action[1]
|
356
276
|
end
|
357
|
-
|
358
|
-
#
|
359
|
-
|
360
|
-
|
361
|
-
|
277
|
+
when 'R'
|
278
|
+
# Received a message
|
279
|
+
signature, message = action[1..2]
|
280
|
+
case signature
|
281
|
+
when 'PONG'
|
282
|
+
process_pong message
|
283
|
+
when 'ACKN'
|
284
|
+
process_ackn message
|
285
|
+
end
|
286
|
+
# else
|
287
|
+
# Unknown message - only listener is allowed to respond with a "????" message
|
288
|
+
# TODO: What should we do? Just ignore for now and let timeouts conquer
|
289
|
+
|
290
|
+
# Any pending payloads left?
|
291
|
+
if @pending_payloads.length.zero?
|
292
|
+
# Handle shutdown
|
293
|
+
raise ShutdownSignal if io_stop
|
294
|
+
|
295
|
+
# Enter idle mode
|
296
|
+
@timeout = Time.now.to_i + @keepalive_timeout
|
297
|
+
else
|
298
|
+
# Set network timeout
|
299
|
+
@timeout = Time.now.to_i + @network_timeout
|
362
300
|
end
|
301
|
+
when 'F'
|
302
|
+
# Reconnect, an error occurred
|
303
|
+
break
|
304
|
+
when '!'
|
305
|
+
@logger&.debug 'Shutdown request received'
|
306
|
+
|
307
|
+
# Shutdown request received
|
308
|
+
raise ShutdownSignal if @pending_payloads.length.zero?
|
309
|
+
|
310
|
+
@logger&.debug 'Delaying shutdown due to pending payloads', payloads: @pending_payloads.length
|
311
|
+
|
312
|
+
io_stop = true
|
363
313
|
|
364
314
|
# Stop spooler sending
|
365
315
|
can_send = false
|
366
316
|
@send_mutex.synchronize do
|
367
317
|
@send_ready = false
|
368
318
|
end
|
319
|
+
end
|
320
|
+
rescue TimeoutError
|
321
|
+
# Handle network timeout
|
322
|
+
raise TimeoutError if @pending_payloads != 0
|
369
323
|
|
370
|
-
|
371
|
-
|
324
|
+
# Keepalive timeout hit, timeout if we were waiting for a pong
|
325
|
+
raise TimeoutError if @pending_ping
|
372
326
|
|
373
|
-
|
327
|
+
# Stop spooler sending
|
328
|
+
can_send = false
|
329
|
+
@send_mutex.synchronize do
|
330
|
+
@send_ready = false
|
374
331
|
end
|
332
|
+
|
333
|
+
# Send PING
|
334
|
+
send_ping
|
335
|
+
|
336
|
+
@timeout = Time.now.to_i + @network_timeout
|
337
|
+
|
338
|
+
next
|
375
339
|
end
|
376
340
|
rescue ProtocolError => e
|
377
341
|
# Reconnect required due to a protocol error
|
378
|
-
@logger
|
342
|
+
@logger&.warn 'Protocol error', error: e.message
|
379
343
|
rescue TimeoutError
|
380
344
|
# Reconnect due to timeout
|
381
|
-
@logger
|
382
|
-
rescue ShutdownSignal
|
345
|
+
@logger&.warn 'Timeout occurred'
|
346
|
+
rescue ShutdownSignal
|
383
347
|
raise
|
384
|
-
rescue StandardError
|
348
|
+
rescue StandardError => e
|
385
349
|
# Unknown error occurred
|
386
|
-
@logger
|
350
|
+
@logger&.warn e, hint: 'Unknown error'
|
387
351
|
end
|
388
352
|
|
389
353
|
def send_payload(payload)
|
390
354
|
# If we have too many pending payloads, pause the IO
|
391
|
-
if @pending_payloads.length + 1 >= @max_pending_payloads
|
392
|
-
@client.pause_send
|
393
|
-
end
|
355
|
+
@client.pause_send if @pending_payloads.length + 1 >= @max_pending_payloads
|
394
356
|
|
395
357
|
# Received some events - send them
|
396
358
|
send_jdat payload
|
397
359
|
|
398
360
|
# Leave idle mode if this is the first payload after idle
|
399
|
-
if @pending_payloads.length == 1
|
400
|
-
@timeout = Time.now.to_i + @network_timeout
|
401
|
-
end
|
361
|
+
@timeout = Time.now.to_i + @network_timeout if @pending_payloads.length == 1
|
402
362
|
end
|
403
363
|
|
404
364
|
def generate_nonce
|
405
|
-
(0...16).map { rand(256).chr }.join
|
365
|
+
(0...16).map { rand(256).chr }.join
|
406
366
|
end
|
407
367
|
|
408
368
|
def send_ping
|
409
369
|
# Send it
|
410
370
|
@client.send 'PING', ''
|
411
|
-
return
|
412
371
|
end
|
413
372
|
|
414
373
|
def send_jdat(events)
|
@@ -422,54 +381,51 @@ module LogCourier
|
|
422
381
|
|
423
382
|
if @first_payload.nil?
|
424
383
|
@first_payload = payload
|
425
|
-
@last_payload = payload
|
426
384
|
else
|
427
385
|
@last_payload.next = payload
|
428
|
-
@last_payload = payload
|
429
386
|
end
|
387
|
+
@last_payload = payload
|
430
388
|
|
431
389
|
# Send it
|
432
390
|
@client.send 'JDAT', payload.payload
|
433
|
-
return
|
434
391
|
end
|
435
392
|
|
436
393
|
def process_pong(message)
|
437
394
|
# Sanity
|
438
|
-
|
395
|
+
raise ProtocolError, "Unexpected data attached to pong message (#{message.bytesize})" if message.bytesize != 0
|
439
396
|
|
440
|
-
@logger
|
397
|
+
@logger&.debug 'PONG message received' || !@logger&.debug?
|
441
398
|
|
442
399
|
# No longer pending a PONG
|
443
400
|
@ping_pending = false
|
444
|
-
return
|
445
401
|
end
|
446
402
|
|
447
403
|
def process_ackn(message)
|
448
404
|
# Sanity
|
449
|
-
|
405
|
+
raise ProtocolError, "ACKN message size invalid (#{message.bytesize})" if message.bytesize != 20
|
450
406
|
|
451
407
|
# Grab nonce
|
452
408
|
nonce, sequence = message.unpack('a16N')
|
453
409
|
|
454
|
-
if
|
410
|
+
if @logger&.debug?
|
455
411
|
nonce_str = nonce.each_byte.map do |b|
|
456
412
|
b.to_s(16).rjust(2, '0')
|
457
413
|
end
|
458
414
|
|
459
|
-
@logger
|
415
|
+
@logger&.debug 'ACKN message received', nonce: nonce_str.join, sequence: sequence
|
460
416
|
end
|
461
417
|
|
462
418
|
# Find the payload - skip if we couldn't as it will just a duplicated ACK
|
463
419
|
return unless @pending_payloads.key?(nonce)
|
464
420
|
|
465
421
|
payload = @pending_payloads[nonce]
|
466
|
-
|
422
|
+
_, complete = payload.ack(sequence)
|
467
423
|
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
424
|
+
return unless complete
|
425
|
+
|
426
|
+
@pending_payloads.delete nonce
|
427
|
+
@first_payload = payload.next
|
428
|
+
@client.resume_send if @client.send_paused?
|
473
429
|
end
|
474
430
|
end
|
475
431
|
end
|