log-courier 1.8.2 → 2.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/log-courier/client.rb +142 -182
- data/lib/log-courier/client_tcp.rb +117 -87
- data/lib/log-courier/event_queue.rb +18 -23
- data/lib/log-courier/server.rb +32 -57
- data/lib/log-courier/server_tcp.rb +174 -120
- metadata +11 -28
- data/lib/log-courier/server_zmq.rb +0 -400
- data/lib/log-courier/zmq_qpoll.rb +0 -324
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
# Copyright 2014 Jason Woods.
|
1
|
+
# Copyright 2014-2021 Jason Woods and Contributors.
|
4
2
|
#
|
5
3
|
# This file is a modification of code from Logstash Forwarder.
|
6
4
|
# Copyright 2012-2013 Jordan Sissel and contributors.
|
@@ -19,42 +17,47 @@
|
|
19
17
|
|
20
18
|
require 'openssl'
|
21
19
|
require 'socket'
|
22
|
-
require 'thread'
|
23
20
|
|
24
21
|
module LogCourier
|
25
22
|
# TLS transport implementation
|
26
23
|
class ClientTcp
|
27
24
|
def initialize(options = {})
|
28
25
|
@options = {
|
29
|
-
logger:
|
30
|
-
transport:
|
31
|
-
ssl_ca:
|
32
|
-
ssl_certificate:
|
33
|
-
ssl_key:
|
26
|
+
logger: nil,
|
27
|
+
transport: 'tls',
|
28
|
+
ssl_ca: nil,
|
29
|
+
ssl_certificate: nil,
|
30
|
+
ssl_key: nil,
|
34
31
|
ssl_key_passphrase: nil,
|
32
|
+
min_tls_version: 1.2,
|
33
|
+
disable_handshake: false,
|
35
34
|
}.merge!(options)
|
36
35
|
|
37
36
|
@logger = @options[:logger]
|
38
37
|
|
39
38
|
[:port, :ssl_ca].each do |k|
|
40
|
-
|
39
|
+
raise "output/courier: '#{k}' is required" if @options[k].nil?
|
41
40
|
end
|
42
41
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
fail 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
|
42
|
+
return unless @options[:transport] == 'tls'
|
43
|
+
|
44
|
+
c = 0
|
45
|
+
[:ssl_certificate, :ssl_key].each do
|
46
|
+
c += 1
|
49
47
|
end
|
48
|
+
raise 'output/courier: \'ssl_certificate\' and \'ssl_key\' must be specified together' if c == 1
|
50
49
|
end
|
51
50
|
|
52
51
|
def connect(io_control)
|
53
52
|
loop do
|
54
53
|
begin
|
55
|
-
|
54
|
+
if tls_connect
|
55
|
+
return unless handshake(io_control)
|
56
|
+
|
57
|
+
break
|
58
|
+
end
|
56
59
|
rescue ShutdownSignal
|
57
|
-
|
60
|
+
return
|
58
61
|
end
|
59
62
|
|
60
63
|
# TODO: Make this configurable
|
@@ -65,47 +68,44 @@ module LogCourier
|
|
65
68
|
@send_paused = false
|
66
69
|
|
67
70
|
@send_thread = Thread.new do
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
return
|
75
|
-
end
|
71
|
+
run_send io_control
|
72
|
+
rescue ShutdownSignal
|
73
|
+
# Shutdown
|
74
|
+
rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
|
75
|
+
@logger&.warn e, hint: 'Unknown write error'
|
76
|
+
io_control << ['F']
|
76
77
|
end
|
77
78
|
@recv_thread = Thread.new do
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
return
|
85
|
-
end
|
79
|
+
run_recv io_control
|
80
|
+
rescue ShutdownSignal
|
81
|
+
# Shutdown
|
82
|
+
rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
|
83
|
+
@logger&.warn e, hint: 'Unknown read error'
|
84
|
+
io_control << ['F']
|
86
85
|
end
|
87
|
-
|
86
|
+
nil
|
88
87
|
end
|
89
88
|
|
90
89
|
def disconnect
|
91
|
-
@send_thread
|
92
|
-
@send_thread
|
93
|
-
@recv_thread
|
94
|
-
@recv_thread
|
95
|
-
|
90
|
+
@send_thread&.raise ShutdownSignal
|
91
|
+
@send_thread&.join
|
92
|
+
@recv_thread&.raise ShutdownSignal
|
93
|
+
@recv_thread&.join
|
94
|
+
nil
|
96
95
|
end
|
97
96
|
|
98
97
|
def send(signature, message)
|
99
98
|
# Add to send queue
|
100
|
-
@send_q << [signature, message.length].pack('A4N') + message
|
101
|
-
|
99
|
+
@send_q << ([signature, message.length].pack('A4N') + message)
|
100
|
+
nil
|
102
101
|
end
|
103
102
|
|
104
103
|
def pause_send
|
105
104
|
return if @send_paused
|
105
|
+
|
106
106
|
@send_paused = true
|
107
107
|
@send_q << nil
|
108
|
-
|
108
|
+
nil
|
109
109
|
end
|
110
110
|
|
111
111
|
def send_paused?
|
@@ -117,11 +117,35 @@ module LogCourier
|
|
117
117
|
@send_paused = false
|
118
118
|
@send_q << nil
|
119
119
|
end
|
120
|
-
|
120
|
+
nil
|
121
121
|
end
|
122
122
|
|
123
123
|
private
|
124
124
|
|
125
|
+
def handshake(io_control)
|
126
|
+
return true if @options[:disable_handshake]
|
127
|
+
|
128
|
+
@socket.write ['HELO', 8, 0, 2, 7, 0, 'RYLC'].pack('A4NCCCCA4')
|
129
|
+
|
130
|
+
signature, data = receive
|
131
|
+
if signature != 'VERS'
|
132
|
+
raise "Unexpected message during handshake: #{signature}" if signature != '????'
|
133
|
+
|
134
|
+
@vers = Protocol.parse_helo_vers('')
|
135
|
+
@logger&.info 'Remote does not support protocol handshake', server_version: @vers[:client_version]
|
136
|
+
return true
|
137
|
+
end
|
138
|
+
|
139
|
+
@vers = Protocol.parse_helo_vers(data)
|
140
|
+
@logger&.info 'Remote identified', server_version: @vers[:client_version]
|
141
|
+
|
142
|
+
true
|
143
|
+
rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
|
144
|
+
@logger&.warn e, hint: 'Unknown write error'
|
145
|
+
io_control << ['F']
|
146
|
+
false
|
147
|
+
end
|
148
|
+
|
125
149
|
def run_send(io_control)
|
126
150
|
# Ask for something to send
|
127
151
|
io_control << ['S']
|
@@ -147,55 +171,52 @@ module LogCourier
|
|
147
171
|
# Ask for more to send while we send this one
|
148
172
|
io_control << ['S'] unless paused
|
149
173
|
|
150
|
-
@
|
174
|
+
@socket.write message
|
151
175
|
end
|
152
176
|
end
|
153
|
-
return
|
154
177
|
rescue OpenSSL::SSL::SSLError => e
|
155
|
-
@logger
|
178
|
+
@logger&.warn 'SSL write error', error: e.message
|
156
179
|
io_control << ['F']
|
157
|
-
return
|
158
180
|
rescue IOError, Errno::ECONNRESET => e
|
159
|
-
@logger
|
181
|
+
@logger&.warn 'Write error', error: e.message
|
160
182
|
io_control << ['F']
|
161
|
-
return
|
162
183
|
end
|
163
184
|
|
164
185
|
def run_recv(io_control)
|
165
186
|
loop do
|
166
|
-
|
167
|
-
header = @ssl_client.read(8)
|
168
|
-
fail EOFError if header.nil?
|
169
|
-
|
170
|
-
# Decode signature and length
|
171
|
-
signature, length = header.unpack('A4N')
|
172
|
-
|
173
|
-
if length > 1048576
|
174
|
-
# Too big raise error
|
175
|
-
@logger.warn 'Invalid message: data too big', :data_length => length unless @logger.nil?
|
176
|
-
io_control << ['F']
|
177
|
-
break
|
178
|
-
end
|
179
|
-
|
180
|
-
# Read remainder
|
181
|
-
message = @ssl_client.read(length)
|
187
|
+
signature, message = receive
|
182
188
|
|
183
189
|
# Pass through to receive
|
184
190
|
io_control << ['R', signature, message]
|
185
191
|
end
|
186
|
-
return
|
187
192
|
rescue OpenSSL::SSL::SSLError => e
|
188
|
-
@logger
|
189
|
-
io_control << ['F']
|
190
|
-
return
|
191
|
-
rescue IOError, Errno::ECONNRESET => e
|
192
|
-
@logger.warn 'Read error', :error => e.message unless @logger.nil?
|
193
|
+
@logger&.warn 'SSL read error', error: e.message
|
193
194
|
io_control << ['F']
|
194
|
-
return
|
195
195
|
rescue EOFError
|
196
|
-
@logger
|
196
|
+
@logger&.warn 'Connection closed by server'
|
197
|
+
io_control << ['F']
|
198
|
+
rescue IOError, Errno::ECONNRESET => e
|
199
|
+
@logger&.warn 'Read error', error: e.message
|
197
200
|
io_control << ['F']
|
198
|
-
|
201
|
+
end
|
202
|
+
|
203
|
+
def receive
|
204
|
+
# Grab a header
|
205
|
+
header = @socket.read(8)
|
206
|
+
raise EOFError if header.nil?
|
207
|
+
|
208
|
+
# Decode signature and length
|
209
|
+
signature, length = header.unpack('A4N')
|
210
|
+
|
211
|
+
if length > 1_048_576
|
212
|
+
# Too big raise error
|
213
|
+
raise IOError, 'Invalid message: data too big'
|
214
|
+
end
|
215
|
+
|
216
|
+
# Read remainder
|
217
|
+
message = @socket.read(length)
|
218
|
+
|
219
|
+
[signature, message]
|
199
220
|
end
|
200
221
|
|
201
222
|
def tls_connect
|
@@ -203,7 +224,7 @@ module LogCourier
|
|
203
224
|
address = @options[:addresses][0]
|
204
225
|
port = @options[:port]
|
205
226
|
|
206
|
-
@logger
|
227
|
+
@logger&.info 'Connecting', address: address, port: port
|
207
228
|
|
208
229
|
begin
|
209
230
|
tcp_socket = TCPSocket.new(address, port)
|
@@ -216,8 +237,15 @@ module LogCourier
|
|
216
237
|
ssl.set_params
|
217
238
|
# Modify the default options to ensure SSLv2 and SSLv3 is disabled
|
218
239
|
# This retains any beneficial options set by default in the current Ruby implementation
|
219
|
-
|
220
|
-
|
240
|
+
# TODO: https://github.com/jruby/jruby-openssl/pull/215 is fixed in JRuby 9.3.0.0
|
241
|
+
# As of 7.15 Logstash, JRuby version is still 9.2
|
242
|
+
# Once 9.3 is in use we can switch to using min_version and max_version
|
243
|
+
ssl.options |= OpenSSL::SSL::OP_NO_SSLv2
|
244
|
+
ssl.options |= OpenSSL::SSL::OP_NO_SSLv3
|
245
|
+
ssl.options |= OpenSSL::SSL::OP_NO_TLSv1 if @options[:min_tls_version] > 1
|
246
|
+
ssl.options |= OpenSSL::SSL::OP_NO_TLSv1_1 if @options[:min_tls_version] > 1.1
|
247
|
+
ssl.options |= OpenSSL::SSL::OP_NO_TLSv1_2 if @options[:min_tls_version] > 1.2
|
248
|
+
raise 'Invalid min_tls_version - max is 1.3' if @options[:min_tls_version] > 1.3
|
221
249
|
|
222
250
|
# Set the certificate file
|
223
251
|
unless @options[:ssl_certificate].nil?
|
@@ -230,26 +258,28 @@ module LogCourier
|
|
230
258
|
ssl.cert_store = cert_store
|
231
259
|
ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
|
232
260
|
|
233
|
-
@
|
234
|
-
|
235
|
-
socket = @ssl_client.connect
|
261
|
+
@socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl)
|
262
|
+
@socket.connect
|
236
263
|
|
237
264
|
# Verify certificate
|
238
|
-
socket.post_connection_check(address)
|
265
|
+
@socket.post_connection_check(address)
|
266
|
+
|
267
|
+
@logger&.info 'Connected successfully', address: address, port: port, ssl_version: @socket.ssl_version
|
239
268
|
else
|
240
|
-
socket = tcp_socket
|
269
|
+
@socket = tcp_socket
|
270
|
+
|
271
|
+
@logger&.info 'Connected successfully', address: address, port: port
|
241
272
|
end
|
242
273
|
|
243
274
|
# Add extra logging data now we're connected
|
244
275
|
@logger['address'] = address
|
245
276
|
@logger['port'] = port
|
246
277
|
|
247
|
-
@logger.info 'Connected successfully' unless @logger.nil?
|
248
278
|
return true
|
249
279
|
rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
|
250
|
-
@logger
|
251
|
-
rescue StandardError, NativeException => e
|
252
|
-
@logger
|
280
|
+
@logger&.warn 'Connection failed', error: e.message, address: address, port: port
|
281
|
+
rescue StandardError, NativeException => e # Can remove NativeException after 9.2.14.0 JRuby
|
282
|
+
@logger&.warn e, hint: 'Unknown connection failure', address: address, port: port
|
253
283
|
end
|
254
284
|
|
255
285
|
false
|
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
# Copyright 2014 Jason Woods.
|
1
|
+
# Copyright 2014-2021 Jason Woods and Contributors.
|
4
2
|
#
|
5
3
|
# This file is a modification of code from Ruby.
|
6
4
|
# Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
|
@@ -28,23 +26,24 @@
|
|
28
26
|
# The majority of the code is taken from Ruby's SizedQueue<Queue implementation.
|
29
27
|
#
|
30
28
|
module LogCourier
|
29
|
+
# EventQueue
|
31
30
|
class EventQueue
|
32
31
|
#
|
33
32
|
# Creates a fixed-length queue with a maximum size of +max+.
|
34
33
|
#
|
35
34
|
def initialize(max)
|
36
|
-
|
35
|
+
raise ArgumentError, 'queue size must be positive' unless max.positive?
|
36
|
+
|
37
37
|
@max = max
|
38
38
|
@enque_cond = ConditionVariable.new
|
39
39
|
@num_enqueue_waiting = 0
|
40
40
|
|
41
41
|
@que = []
|
42
|
-
@que.taint
|
42
|
+
@que.taint # enable tainted communication
|
43
43
|
@num_waiting = 0
|
44
|
-
|
44
|
+
taint
|
45
45
|
@mutex = Mutex.new
|
46
46
|
@cond = ConditionVariable.new
|
47
|
-
return
|
48
47
|
end
|
49
48
|
|
50
49
|
#
|
@@ -56,7 +55,7 @@ module LogCourier
|
|
56
55
|
# Sets the maximum size of the queue.
|
57
56
|
#
|
58
57
|
def max=(max)
|
59
|
-
|
58
|
+
raise ArgumentError, 'queue size must be positive' unless max.positive?
|
60
59
|
|
61
60
|
@mutex.synchronize do
|
62
61
|
if max <= @max
|
@@ -69,7 +68,6 @@ module LogCourier
|
|
69
68
|
end
|
70
69
|
end
|
71
70
|
end
|
72
|
-
max
|
73
71
|
end
|
74
72
|
|
75
73
|
#
|
@@ -77,19 +75,19 @@ module LogCourier
|
|
77
75
|
# until space becomes available, up to a maximum of +timeout+ seconds.
|
78
76
|
#
|
79
77
|
def push(obj, timeout = nil)
|
80
|
-
unless timeout.nil?
|
81
|
-
start = Time.now
|
82
|
-
end
|
78
|
+
start = Time.now unless timeout.nil?
|
83
79
|
@mutex.synchronize do
|
84
80
|
loop do
|
85
81
|
break if @que.length < @max
|
82
|
+
|
86
83
|
@num_enqueue_waiting += 1
|
87
84
|
begin
|
88
85
|
@enque_cond.wait @mutex, timeout
|
89
86
|
ensure
|
90
87
|
@num_enqueue_waiting -= 1
|
91
88
|
end
|
92
|
-
|
89
|
+
|
90
|
+
raise TimeoutError if !timeout.nil? && Time.now - start >= timeout
|
93
91
|
end
|
94
92
|
|
95
93
|
@que.push obj
|
@@ -112,11 +110,9 @@ module LogCourier
|
|
112
110
|
# Retrieves data from the queue and runs a waiting thread, if any.
|
113
111
|
#
|
114
112
|
def pop(*args)
|
115
|
-
retval = pop_timeout
|
113
|
+
retval = pop_timeout(*args)
|
116
114
|
@mutex.synchronize do
|
117
|
-
if @que.length < @max
|
118
|
-
@enque_cond.signal
|
119
|
-
end
|
115
|
+
@enque_cond.signal if @que.length < @max
|
120
116
|
end
|
121
117
|
retval
|
122
118
|
end
|
@@ -182,23 +178,22 @@ module LogCourier
|
|
182
178
|
# raised.
|
183
179
|
#
|
184
180
|
def pop_timeout(timeout = nil)
|
185
|
-
unless timeout.nil?
|
186
|
-
start = Time.now
|
187
|
-
end
|
181
|
+
start = Time.now unless timeout.nil?
|
188
182
|
@mutex.synchronize do
|
189
183
|
loop do
|
190
184
|
return @que.shift unless @que.empty?
|
191
|
-
|
185
|
+
raise TimeoutError if !timeout.nil? && timeout.zero?
|
186
|
+
|
192
187
|
begin
|
193
188
|
@num_waiting += 1
|
194
189
|
@cond.wait @mutex, timeout
|
195
190
|
ensure
|
196
191
|
@num_waiting -= 1
|
197
192
|
end
|
198
|
-
|
193
|
+
raise TimeoutError if !timeout.nil? && Time.now - start >= timeout
|
199
194
|
end
|
200
195
|
end
|
201
|
-
|
196
|
+
nil
|
202
197
|
end
|
203
198
|
end
|
204
199
|
end
|
data/lib/log-courier/server.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
# Copyright 2014 Jason Woods.
|
1
|
+
# Copyright 2014-2021 Jason Woods and Contributors.
|
4
2
|
#
|
5
3
|
# This file is a modification of code from Logstash Forwarder.
|
6
4
|
# Copyright 2012-2013 Jordan Sissel and contributors.
|
@@ -18,43 +16,31 @@
|
|
18
16
|
# limitations under the License.
|
19
17
|
|
20
18
|
require 'log-courier/event_queue'
|
19
|
+
require 'log-courier/protocol'
|
21
20
|
require 'multi_json'
|
22
|
-
require 'thread'
|
23
21
|
require 'zlib'
|
24
22
|
|
25
|
-
|
23
|
+
# NativeException in case it is missing
|
24
|
+
class NativeException
|
25
|
+
def dummy; end
|
26
|
+
end
|
26
27
|
|
27
28
|
module LogCourier
|
28
29
|
class TimeoutError < StandardError; end
|
30
|
+
|
29
31
|
class ShutdownSignal < StandardError; end
|
32
|
+
|
30
33
|
class ProtocolError < StandardError; end
|
31
34
|
|
32
35
|
# Implementation of the server
|
33
36
|
class Server
|
34
37
|
attr_reader :port
|
35
38
|
|
36
|
-
# TODO(driskell): Consolidate singleton into another file
|
37
|
-
class << self
|
38
|
-
@json_adapter
|
39
|
-
@json_parseerror
|
40
|
-
|
41
|
-
def get_json_adapter
|
42
|
-
@json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
|
43
|
-
@json_adapter
|
44
|
-
end
|
45
|
-
|
46
|
-
def get_json_parseerror
|
47
|
-
if @json_parseerror.nil?
|
48
|
-
@json_parseerror = get_json_adapter.class::ParseError
|
49
|
-
end
|
50
|
-
@json_parseerror
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
39
|
def initialize(options = {})
|
55
40
|
@options = {
|
56
|
-
logger:
|
41
|
+
logger: nil,
|
57
42
|
transport: 'tls',
|
43
|
+
disable_handshake: false,
|
58
44
|
}.merge!(options)
|
59
45
|
|
60
46
|
@logger = @options[:logger]
|
@@ -63,11 +49,8 @@ module LogCourier
|
|
63
49
|
when 'tcp', 'tls'
|
64
50
|
require 'log-courier/server_tcp'
|
65
51
|
@server = ServerTcp.new(@options)
|
66
|
-
when 'plainzmq', 'zmq'
|
67
|
-
require 'log-courier/server_zmq'
|
68
|
-
@server = ServerZmq.new(@options)
|
69
52
|
else
|
70
|
-
|
53
|
+
raise 'input/courier: \'transport\' must be tcp or tls'
|
71
54
|
end
|
72
55
|
|
73
56
|
# Grab the port back and update the logger context
|
@@ -90,9 +73,9 @@ module LogCourier
|
|
90
73
|
process_jdat message, comm, @event_queue
|
91
74
|
else
|
92
75
|
if comm.peer.nil?
|
93
|
-
@logger
|
76
|
+
@logger&.warn 'Unknown message received', from: 'unknown'
|
94
77
|
else
|
95
|
-
@logger
|
78
|
+
@logger&.warn 'Unknown message received', from: comm.peer
|
96
79
|
end
|
97
80
|
# Don't kill a client that sends a bad message
|
98
81
|
# Just reject it and let it send it again, potentially to another server
|
@@ -104,6 +87,7 @@ module LogCourier
|
|
104
87
|
loop do
|
105
88
|
event = @event_queue.pop
|
106
89
|
break if event.nil?
|
90
|
+
|
107
91
|
block.call event
|
108
92
|
end
|
109
93
|
ensure
|
@@ -113,25 +97,23 @@ module LogCourier
|
|
113
97
|
server_thread.join
|
114
98
|
end
|
115
99
|
end
|
116
|
-
|
100
|
+
nil
|
117
101
|
end
|
118
102
|
|
119
103
|
def stop
|
120
104
|
@event_queue << nil
|
105
|
+
nil
|
121
106
|
end
|
122
107
|
|
123
108
|
private
|
124
109
|
|
125
110
|
def process_ping(message, comm)
|
126
111
|
# Size of message should be 0
|
127
|
-
|
128
|
-
fail ProtocolError, "unexpected data attached to ping message (#{message.length})"
|
129
|
-
end
|
112
|
+
raise ProtocolError, "unexpected data attached to ping message (#{message.bytesize})" unless message.bytesize.zero?
|
130
113
|
|
131
114
|
# PONG!
|
132
115
|
# NOTE: comm.send can raise a Timeout::Error of its own
|
133
116
|
comm.send 'PONG', ''
|
134
|
-
return
|
135
117
|
end
|
136
118
|
|
137
119
|
def process_jdat(message, comm, event_queue)
|
@@ -141,45 +123,39 @@ module LogCourier
|
|
141
123
|
# OK - first is a nonce - we send this back with sequence acks
|
142
124
|
# This allows the client to know what is being acknowledged
|
143
125
|
# Nonce is 16 so check we have enough
|
144
|
-
if message.
|
145
|
-
fail ProtocolError, "JDAT message too small (#{message.length})"
|
146
|
-
end
|
126
|
+
raise ProtocolError, "JDAT message too small (#{message.bytesize})" if message.bytesize < 17
|
147
127
|
|
148
128
|
nonce = message[0...16]
|
149
129
|
|
150
|
-
if
|
130
|
+
if @logger&.debug?
|
151
131
|
nonce_str = nonce.each_byte.map do |b|
|
152
132
|
b.to_s(16).rjust(2, '0')
|
153
133
|
end
|
154
134
|
end
|
155
135
|
|
156
136
|
# The remainder of the message is the compressed data block
|
157
|
-
message = StringIO.new Zlib::Inflate.inflate(message
|
137
|
+
message = StringIO.new Zlib::Inflate.inflate(message.byteslice(16, message.bytesize))
|
158
138
|
|
159
139
|
# Message now contains JSON encoded events
|
160
140
|
# They are aligned as [length][event]... so on
|
161
141
|
# We acknowledge them by their 1-index position in the stream
|
162
142
|
# A 0 sequence acknowledgement means we haven't processed any yet
|
163
143
|
sequence = 0
|
164
|
-
events = []
|
165
144
|
length_buf = ''
|
166
145
|
data_buf = ''
|
167
146
|
loop do
|
168
147
|
ret = message.read 4, length_buf
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
elsif length_buf.length < 4
|
173
|
-
fail ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.length})"
|
174
|
-
end
|
148
|
+
# Finished?
|
149
|
+
break if ret.nil?
|
150
|
+
raise ProtocolError, "JDAT length extraction failed (#{ret} #{length_buf.bytesize})" if length_buf.bytesize < 4
|
175
151
|
|
176
|
-
length = length_buf.
|
152
|
+
length = length_buf.unpack1('N')
|
177
153
|
|
178
154
|
# Extract message
|
179
155
|
ret = message.read length, data_buf
|
180
|
-
if ret.nil?
|
181
|
-
@logger
|
182
|
-
|
156
|
+
if ret.nil? || data_buf.bytesize < length
|
157
|
+
@logger&.warn()
|
158
|
+
raise ProtocolError, "JDAT message extraction failed #{ret} #{data_buf.bytesize}"
|
183
159
|
end
|
184
160
|
|
185
161
|
data_buf.force_encoding('utf-8')
|
@@ -199,9 +175,9 @@ module LogCourier
|
|
199
175
|
|
200
176
|
# Decode the JSON
|
201
177
|
begin
|
202
|
-
event =
|
203
|
-
rescue
|
204
|
-
@logger
|
178
|
+
event = MultiJson.load(data_buf)
|
179
|
+
rescue MultiJson::ParseError => e
|
180
|
+
@logger&.warn e, invalid_encodings: invalid_encodings, hint: 'JSON parse failure, falling back to plain-text'
|
205
181
|
event = { 'message' => data_buf }
|
206
182
|
end
|
207
183
|
|
@@ -214,7 +190,7 @@ module LogCourier
|
|
214
190
|
rescue TimeoutError
|
215
191
|
# Full pipeline, partial ack
|
216
192
|
# NOTE: comm.send can raise a Timeout::Error of its own
|
217
|
-
@logger
|
193
|
+
@logger&.debug 'Partially acknowledging message', nonce: nonce_str.join, sequence: sequence if @logger&.debug?
|
218
194
|
comm.send 'ACKN', [nonce, sequence].pack('a*N')
|
219
195
|
ack_timeout = Time.now.to_i + 5
|
220
196
|
retry
|
@@ -225,9 +201,8 @@ module LogCourier
|
|
225
201
|
|
226
202
|
# Acknowledge the full message
|
227
203
|
# NOTE: comm.send can raise a Timeout::Error
|
228
|
-
@logger
|
204
|
+
@logger&.debug 'Acknowledging message', nonce: nonce_str.join, sequence: sequence if @logger&.debug?
|
229
205
|
comm.send 'ACKN', [nonce, sequence].pack('A*N')
|
230
|
-
return
|
231
206
|
end
|
232
207
|
end
|
233
208
|
end
|