log-courier 1.2 → 1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/log-courier/client.rb +67 -56
- data/lib/log-courier/client_tls.rb +30 -31
- data/lib/log-courier/server.rb +3 -0
- data/lib/log-courier/server_tcp.rb +21 -1
- data/lib/log-courier/server_zmq.rb +18 -16
- data/lib/log-courier/zmq_qpoll.rb +324 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 645bd508ce891c3fb1fd69c4a414758fdc55c867
|
4
|
+
data.tar.gz: c4b14fb6f9cc1b75ac2c886d0162b543246ef5c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2f512338b7e7a11fff7293a9006ef5a924596f5b7f40863e5f12a68297588e1969628aa6b3730b30bac9464071d9da63186aac9c460bfbe8f92219f42f6731e2
|
7
|
+
data.tar.gz: 736be3625601a89409beee281f432e6450af0a76aa040eb916e848512cfcb195b8bd6b0c6ef013e0a0757fe5237f35acdf9bd273ffe34a9c765e5f81ac4ab841
|
data/lib/log-courier/client.rb
CHANGED
@@ -30,21 +30,60 @@ module LogCourier
|
|
30
30
|
|
31
31
|
# Describes a pending payload
|
32
32
|
class PendingPayload
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
33
|
+
class << self
|
34
|
+
@json_adapter
|
35
|
+
def get_json_adapter
|
36
|
+
@json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
|
37
|
+
return @json_adapter
|
38
|
+
end
|
39
|
+
end
|
37
40
|
|
38
|
-
attr_accessor :previous
|
39
41
|
attr_accessor :next
|
42
|
+
attr_accessor :nonce
|
43
|
+
attr_accessor :events
|
44
|
+
attr_accessor :last_sequence
|
45
|
+
attr_accessor :sequence_len
|
46
|
+
attr_accessor :payload
|
40
47
|
|
41
|
-
def initialize(
|
42
|
-
@
|
48
|
+
def initialize(events, nonce)
|
49
|
+
@events = events
|
50
|
+
@nonce = nonce
|
51
|
+
|
52
|
+
generate
|
53
|
+
end
|
54
|
+
|
55
|
+
def generate
|
56
|
+
fail ArgumentError, 'Corrupt payload' if @events.length == 0
|
57
|
+
|
58
|
+
buffer = Zlib::Deflate.new
|
43
59
|
|
44
|
-
|
45
|
-
|
46
|
-
|
60
|
+
# Write each event in JSON format
|
61
|
+
events.each do |event|
|
62
|
+
json_data = self.class.get_json_adapter.dump(event)
|
63
|
+
# Add length and then the data
|
64
|
+
buffer << [json_data.length].pack('N') << json_data
|
47
65
|
end
|
66
|
+
|
67
|
+
# Generate and store the payload
|
68
|
+
@payload = nonce + buffer.flush(Zlib::FINISH)
|
69
|
+
@last_sequence = 0
|
70
|
+
@sequence_len = @events.length
|
71
|
+
end
|
72
|
+
|
73
|
+
def ack(sequence)
|
74
|
+
if sequence <= @last_sequence
|
75
|
+
return 0, false
|
76
|
+
elsif sequence >= @sequence_len
|
77
|
+
lines = @sequence_len - @last_sequence
|
78
|
+
@last_sequence = sequence
|
79
|
+
@payload = nil
|
80
|
+
return lines, true
|
81
|
+
end
|
82
|
+
|
83
|
+
lines = sequence - @last_sequence
|
84
|
+
@last_sequence = sequence
|
85
|
+
@payload = nil
|
86
|
+
return lines, false
|
48
87
|
end
|
49
88
|
end
|
50
89
|
|
@@ -63,9 +102,6 @@ module LogCourier
|
|
63
102
|
require 'log-courier/client_tls'
|
64
103
|
@client = ClientTls.new(@options)
|
65
104
|
|
66
|
-
# Load the json adapter
|
67
|
-
@json_adapter = MultiJson.adapter.instance
|
68
|
-
|
69
105
|
@event_queue = EventQueue.new @options[:spool_size]
|
70
106
|
@pending_payloads = {}
|
71
107
|
@first_payload = nil
|
@@ -100,7 +136,7 @@ module LogCourier
|
|
100
136
|
@io_thread.raise ShutdownSignal
|
101
137
|
@spooler_thread.join
|
102
138
|
@io_thread.join
|
103
|
-
return
|
139
|
+
return @pending_payloads.length == 0
|
104
140
|
end
|
105
141
|
|
106
142
|
private
|
@@ -274,14 +310,9 @@ module LogCourier
|
|
274
310
|
def send_jdat(events)
|
275
311
|
# Generate the JSON payload and compress it
|
276
312
|
nonce = generate_nonce
|
277
|
-
data = buffer_jdat_data(events, nonce)
|
278
313
|
|
279
314
|
# Save the pending payload
|
280
|
-
payload = PendingPayload.new(
|
281
|
-
:events => events,
|
282
|
-
:nonce => nonce,
|
283
|
-
:data => data
|
284
|
-
)
|
315
|
+
payload = PendingPayload.new(events, nonce)
|
285
316
|
|
286
317
|
@pending_payloads[nonce] = payload
|
287
318
|
|
@@ -294,27 +325,7 @@ module LogCourier
|
|
294
325
|
end
|
295
326
|
|
296
327
|
# Send it
|
297
|
-
@client.send 'JDAT', payload.
|
298
|
-
return
|
299
|
-
end
|
300
|
-
|
301
|
-
def buffer_jdat_data(events, nonce)
|
302
|
-
buffer = Zlib::Deflate.new
|
303
|
-
|
304
|
-
# Write each event in JSON format
|
305
|
-
events.each do |event|
|
306
|
-
buffer_jdat_data_event(buffer, event)
|
307
|
-
end
|
308
|
-
|
309
|
-
# Generate and return the message
|
310
|
-
nonce + buffer.flush(Zlib::FINISH)
|
311
|
-
end
|
312
|
-
|
313
|
-
def buffer_jdat_data_event(buffer, event)
|
314
|
-
json_data = @json_adapter.dump(event)
|
315
|
-
|
316
|
-
# Add length and then the data
|
317
|
-
buffer << [json_data.length].pack('N') << json_data
|
328
|
+
@client.send 'JDAT', payload.payload
|
318
329
|
return
|
319
330
|
end
|
320
331
|
|
@@ -322,6 +333,8 @@ module LogCourier
|
|
322
333
|
# Sanity
|
323
334
|
fail ProtocolError, "Unexpected data attached to pong message (#{message.length})" if message.length != 0
|
324
335
|
|
336
|
+
@logger.debug 'PONG message received' unless @logger.nil? || !@logger.debug?
|
337
|
+
|
325
338
|
# No longer pending a PONG
|
326
339
|
@ping_pending = false
|
327
340
|
return
|
@@ -332,29 +345,27 @@ module LogCourier
|
|
332
345
|
fail ProtocolError, "ACKN message size invalid (#{message.length})" if message.length != 20
|
333
346
|
|
334
347
|
# Grab nonce
|
335
|
-
|
348
|
+
nonce, sequence = message.unpack('A16N')
|
349
|
+
|
350
|
+
if !@logger.nil? && @logger.debug?
|
351
|
+
nonce_str = nonce.each_byte.map do |b|
|
352
|
+
b.to_s(16).rjust(2, '0')
|
353
|
+
end
|
354
|
+
|
355
|
+
@logger.debug 'ACKN message received', :nonce => nonce_str.join, :sequence => sequence
|
356
|
+
end
|
336
357
|
|
337
358
|
# Find the payload - skip if we couldn't as it will just a duplicated ACK
|
338
359
|
return unless @pending_payloads.key?(nonce)
|
339
360
|
|
340
361
|
payload = @pending_payloads[nonce]
|
362
|
+
lines, complete = payload.ack(sequence)
|
341
363
|
|
342
|
-
|
343
|
-
# TODO: protocol error if sequence too large?
|
344
|
-
if sequence >= payload.events.length
|
345
|
-
@client.resume_send if @client.send_paused?
|
346
|
-
|
364
|
+
if complete
|
347
365
|
@pending_payloads.delete nonce
|
348
|
-
|
349
|
-
|
350
|
-
# Partial ACK - only process if something was actually processed
|
351
|
-
if sequence > payload.ack_events
|
352
|
-
payload.ack_events = sequence
|
353
|
-
payload.events = payload.events[0...sequence]
|
354
|
-
payload.data = nil
|
355
|
-
end
|
366
|
+
@first_payload = payload.next
|
367
|
+
@client.resume_send if @client.send_paused?
|
356
368
|
end
|
357
|
-
return
|
358
369
|
end
|
359
370
|
end
|
360
371
|
end
|
@@ -52,14 +52,15 @@ module LogCourier
|
|
52
52
|
end
|
53
53
|
|
54
54
|
def connect(io_control)
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
loop do
|
56
|
+
begin
|
57
|
+
break if tls_connect
|
58
|
+
rescue ShutdownSignal
|
59
|
+
raise
|
60
|
+
end
|
61
|
+
|
60
62
|
# TODO: Make this configurable
|
61
63
|
sleep 5
|
62
|
-
retry
|
63
64
|
end
|
64
65
|
|
65
66
|
@send_q = SizedQueue.new 1
|
@@ -88,6 +89,25 @@ module LogCourier
|
|
88
89
|
return
|
89
90
|
end
|
90
91
|
|
92
|
+
def pause_send
|
93
|
+
return if @send_paused
|
94
|
+
@send_paused = true
|
95
|
+
@send_q << nil
|
96
|
+
return
|
97
|
+
end
|
98
|
+
|
99
|
+
def send_paused?
|
100
|
+
@send_paused
|
101
|
+
end
|
102
|
+
|
103
|
+
def resume_send
|
104
|
+
if @send_paused
|
105
|
+
@send_paused = false
|
106
|
+
@send_q << nil
|
107
|
+
end
|
108
|
+
return
|
109
|
+
end
|
110
|
+
|
91
111
|
private
|
92
112
|
|
93
113
|
def run_send(io_control)
|
@@ -170,25 +190,6 @@ module LogCourier
|
|
170
190
|
return
|
171
191
|
end
|
172
192
|
|
173
|
-
def pause_send
|
174
|
-
return if @send_paused
|
175
|
-
@send_paused = true
|
176
|
-
@send_q << nil
|
177
|
-
return
|
178
|
-
end
|
179
|
-
|
180
|
-
def send_paused
|
181
|
-
@send_paused
|
182
|
-
end
|
183
|
-
|
184
|
-
def resume_send
|
185
|
-
if @send_paused
|
186
|
-
@send_paused = false
|
187
|
-
@send_q << nil
|
188
|
-
end
|
189
|
-
return
|
190
|
-
end
|
191
|
-
|
192
193
|
def tls_connect
|
193
194
|
# TODO: Implement random selection - and don't use separate :port - remember to update post_connection_check too
|
194
195
|
address = @options[:addresses][0]
|
@@ -208,7 +209,7 @@ module LogCourier
|
|
208
209
|
|
209
210
|
cert_store = OpenSSL::X509::Store.new
|
210
211
|
cert_store.add_file(@options[:ssl_ca])
|
211
|
-
|
212
|
+
ssl.cert_store = cert_store
|
212
213
|
ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
|
213
214
|
|
214
215
|
@ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket)
|
@@ -223,16 +224,14 @@ module LogCourier
|
|
223
224
|
@logger['port'] = port
|
224
225
|
|
225
226
|
@logger.info 'Connected successfully' unless @logger.nil?
|
226
|
-
return
|
227
|
+
return true
|
227
228
|
rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
|
228
229
|
@logger.warn 'Connection failed', :error => e.message, :address => address, :port => port unless @logger.nil?
|
229
|
-
return
|
230
|
-
rescue ShutdownSignal
|
231
|
-
return
|
232
230
|
rescue StandardError, NativeException => e
|
233
231
|
@logger.warn e, :hint => 'Unknown connection failure', :address => address, :port => port unless @logger.nil?
|
234
|
-
raise e
|
235
232
|
end
|
233
|
+
|
234
|
+
false
|
236
235
|
end
|
237
236
|
end
|
238
237
|
end
|
data/lib/log-courier/server.rb
CHANGED
@@ -63,6 +63,7 @@ module LogCourier
|
|
63
63
|
ssl_verify_default_ca: false,
|
64
64
|
ssl_verify_ca: nil,
|
65
65
|
max_packet_size: 10_485_760,
|
66
|
+
add_peer_fields: false,
|
66
67
|
}.merge!(options)
|
67
68
|
|
68
69
|
@logger = @options[:logger]
|
@@ -156,7 +157,7 @@ module LogCourier
|
|
156
157
|
rescue StandardError, NativeException => e
|
157
158
|
# Some other unknown problem
|
158
159
|
@logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
|
159
|
-
|
160
|
+
return
|
160
161
|
ensure
|
161
162
|
# Raise shutdown in all client threads and join then
|
162
163
|
client_threads.each do |_, thr|
|
@@ -195,8 +196,20 @@ module LogCourier
|
|
195
196
|
@logger = logger
|
196
197
|
@fd = fd
|
197
198
|
@peer = peer
|
199
|
+
@peer_fields = {}
|
198
200
|
@in_progress = false
|
199
201
|
@options = options
|
202
|
+
|
203
|
+
if @options[:add_peer_fields]
|
204
|
+
@peer_fields['peer'] = peer
|
205
|
+
if @options[:transport] == 'tls' && !@fd.peer_cert.nil?
|
206
|
+
@peer_fields['peer_ssl_cn'] = get_cn(@fd.peer_cert)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
def add_fields(event)
|
212
|
+
event.merge! @peer_fields if @peer_fields.length != 0
|
200
213
|
end
|
201
214
|
|
202
215
|
def run
|
@@ -284,6 +297,13 @@ module LogCourier
|
|
284
297
|
|
285
298
|
private
|
286
299
|
|
300
|
+
def get_cn(cert)
|
301
|
+
cert.subject.to_a.find do |oid, value|
|
302
|
+
return value if oid == "CN"
|
303
|
+
end
|
304
|
+
nil
|
305
|
+
end
|
306
|
+
|
287
307
|
def recv(need)
|
288
308
|
reset_timeout
|
289
309
|
have = ''
|
@@ -66,8 +66,14 @@ module LogCourier
|
|
66
66
|
|
67
67
|
begin
|
68
68
|
@context = ZMQ::Context.new
|
69
|
+
fail ZMQError, 'context creation error: ' + ZMQ::Util.error_string if @context.nil?
|
70
|
+
|
69
71
|
# Router so we can send multiple responses
|
70
72
|
@socket = @context.socket(ZMQ::ROUTER)
|
73
|
+
fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if @socket.nil?
|
74
|
+
|
75
|
+
rc = @socket.setsockopt(ZMQ::LINGER, 0)
|
76
|
+
fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
71
77
|
|
72
78
|
if @options[:transport] == 'zmq'
|
73
79
|
rc = @socket.setsockopt(ZMQ::CURVE_SERVER, 1)
|
@@ -104,20 +110,10 @@ module LogCourier
|
|
104
110
|
@poller = ZMQPoll::ZMQPoll.new(@context)
|
105
111
|
@poller.register_socket @socket, ZMQ::POLLIN
|
106
112
|
@poller.register_queue_to_socket @send_queue, @socket
|
107
|
-
|
108
|
-
# Register a finaliser that sets @context to nil
|
109
|
-
# This allows us to detect the JRuby bug where during "exit!" finalisers
|
110
|
-
# are run but threads are not killed - which leaves us in a situation of
|
111
|
-
# a terminated @context (it has a terminate finalizer) and an IO thread
|
112
|
-
# looping retries
|
113
|
-
# JRuby will still crash and burn, but at least we don't spam STDOUT with
|
114
|
-
# errors
|
115
|
-
ObjectSpace.define_finalizer(self, Proc.new do
|
116
|
-
@context = nil
|
117
|
-
end)
|
118
113
|
end
|
119
114
|
|
120
115
|
def run(&block)
|
116
|
+
errors = 0
|
121
117
|
loop do
|
122
118
|
begin
|
123
119
|
@poller.poll(5_000) do |socket, r, w|
|
@@ -126,11 +122,14 @@ module LogCourier
|
|
126
122
|
|
127
123
|
receive &block
|
128
124
|
end
|
125
|
+
errors = 0
|
129
126
|
rescue ZMQPoll::ZMQError => e
|
130
|
-
|
131
|
-
fail e if @context.nil?
|
132
|
-
@logger.warn e, :hint => 'ZMQ recv_string failure' unless @logger.nil?
|
127
|
+
@logger.warn e, :hint => 'ZMQ poll failure' unless @logger.nil?
|
133
128
|
next
|
129
|
+
rescue ZMQPoll::ZMQTerm
|
130
|
+
# Fall into shutdown signal, context was terminated
|
131
|
+
# This can happen in JRuby - it seems to run finalisers too early
|
132
|
+
fail ShutdownSignal
|
134
133
|
rescue ZMQPoll::TimeoutError
|
135
134
|
# We'll let ZeroMQ manage reconnections and new connections
|
136
135
|
# There is no point in us doing any form of reconnect ourselves
|
@@ -145,7 +144,7 @@ module LogCourier
|
|
145
144
|
rescue StandardError, NativeException => e
|
146
145
|
# Some other unknown problem
|
147
146
|
@logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
|
148
|
-
|
147
|
+
return
|
149
148
|
ensure
|
150
149
|
@poller.shutdown
|
151
150
|
@factory.shutdown
|
@@ -259,7 +258,7 @@ module LogCourier
|
|
259
258
|
|
260
259
|
# Create the client and associated thread
|
261
260
|
client = ClientZmq.new(self, source, source_str) do
|
262
|
-
try_drop
|
261
|
+
try_drop source, source_str
|
263
262
|
end
|
264
263
|
|
265
264
|
thread = Thread.new do
|
@@ -366,6 +365,9 @@ module LogCourier
|
|
366
365
|
return
|
367
366
|
end
|
368
367
|
|
368
|
+
def add_fields(event)
|
369
|
+
end
|
370
|
+
|
369
371
|
private
|
370
372
|
|
371
373
|
def recv(data)
|
@@ -0,0 +1,324 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# Copyright 2014 Jason Woods.
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
begin
|
18
|
+
require 'ffi-rzmq'
|
19
|
+
require 'ffi-rzmq/version'
|
20
|
+
require 'ffi-rzmq-core/version'
|
21
|
+
rescue LoadError => e
|
22
|
+
raise "ZMQPoll could not initialise: #{e}"
|
23
|
+
end
|
24
|
+
|
25
|
+
module ZMQPoll
|
26
|
+
class ZMQError < StandardError; end
|
27
|
+
class ZMQTerm < StandardError; end
|
28
|
+
class TimeoutError < StandardError; end
|
29
|
+
|
30
|
+
class ZMQPoll
|
31
|
+
def initialize(context, logger=nil)
|
32
|
+
@logger = logger
|
33
|
+
@context = context
|
34
|
+
@poller = ZMQ::Poller.new
|
35
|
+
@sockets = []
|
36
|
+
@socket_to_socket = []
|
37
|
+
@handlers = {}
|
38
|
+
@queues = {}
|
39
|
+
end
|
40
|
+
|
41
|
+
def readables
|
42
|
+
@poller.readables
|
43
|
+
end
|
44
|
+
|
45
|
+
def writables
|
46
|
+
@poller.writables
|
47
|
+
end
|
48
|
+
|
49
|
+
def shutdown
|
50
|
+
@queues.each_key do |queue|
|
51
|
+
deregister_queue queue
|
52
|
+
end
|
53
|
+
|
54
|
+
@socket_to_socket.each do |socket|
|
55
|
+
_close_socket_to_socket socket
|
56
|
+
end
|
57
|
+
|
58
|
+
@sockets.each do |socket|
|
59
|
+
socket.close
|
60
|
+
end
|
61
|
+
return
|
62
|
+
end
|
63
|
+
|
64
|
+
def register_socket(socket, flags)
|
65
|
+
@poller.register socket, flags
|
66
|
+
return
|
67
|
+
end
|
68
|
+
|
69
|
+
def deregister_socket(socket)
|
70
|
+
return if @handlers.key?(socket)
|
71
|
+
|
72
|
+
@poller.delete socket
|
73
|
+
return
|
74
|
+
end
|
75
|
+
|
76
|
+
def register_queue_to_socket(queue, socket)
|
77
|
+
s2s_state = _create_socket_to_socket(socket)
|
78
|
+
|
79
|
+
state = {
|
80
|
+
state: s2s_state,
|
81
|
+
mutex: Mutex.new,
|
82
|
+
shutdown: false,
|
83
|
+
}
|
84
|
+
|
85
|
+
state[:thread] = Thread.new do
|
86
|
+
loop do
|
87
|
+
data = queue.pop
|
88
|
+
break if data.nil?
|
89
|
+
begin
|
90
|
+
send s2s_state[:sender], data
|
91
|
+
rescue TimeoutError
|
92
|
+
state[:mutex].synchronize do
|
93
|
+
break if state[:shutdown]
|
94
|
+
end
|
95
|
+
retry
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@queues[queue] = state
|
101
|
+
return
|
102
|
+
end
|
103
|
+
|
104
|
+
def deregister_queue(queue)
|
105
|
+
return if !@queues.key?(queue)
|
106
|
+
|
107
|
+
# Push nil so if we're idle we jump into action and exit
|
108
|
+
# But also set shutdown to try so if we're mid-send and timeout, we exit
|
109
|
+
@queues[queue][:mutex].synchronize do
|
110
|
+
queue.push nil
|
111
|
+
@queues[queue][:shutdown] = true
|
112
|
+
end
|
113
|
+
@queues[queue][:thread].join
|
114
|
+
|
115
|
+
_close_socket_to_socket @queues[queue][:state]
|
116
|
+
@queues.delete queue
|
117
|
+
return
|
118
|
+
end
|
119
|
+
|
120
|
+
def create_socket_to_socket(socket)
|
121
|
+
state = _create_socket_to_socket(socket)
|
122
|
+
@socket_to_socket[state[:sender]] = state
|
123
|
+
state[:sender]
|
124
|
+
end
|
125
|
+
|
126
|
+
def close_socket_to_socket(socket)
|
127
|
+
return if !@socket_to_socket.include?(socket)
|
128
|
+
state = @socket_to_socket[socket]
|
129
|
+
@socket_to_socket.delete socket
|
130
|
+
_close_socket_to_socket(state)
|
131
|
+
return
|
132
|
+
end
|
133
|
+
|
134
|
+
def poll(timeout)
|
135
|
+
if @poller.size == 0
|
136
|
+
fail ZMQError, 'poll run called with zero socket/queues'
|
137
|
+
end
|
138
|
+
|
139
|
+
rc = @poller.poll(timeout)
|
140
|
+
unless ZMQ::Util.resultcode_ok?(rc)
|
141
|
+
# If we get ETERM or ENOTSOCK - we may have hit JRuby bug where finalisers are called too early
|
142
|
+
# We throw it so the caller knows NOT to retry and hit a bad loop
|
143
|
+
# The downside is - this Jruby bug will cause an inevitable deadlock
|
144
|
+
# The finaliser for the context will have run before finalisers for sockets (random order?)
|
145
|
+
# The resulting ZMQ context termination will block, waiting for sockets to be closed
|
146
|
+
# However, when closing a socket, ffi-rzmq will attempt to remove it's finaliser
|
147
|
+
# This removal will deadlock as it attempts to lock the finaliser list, which is locked by the
|
148
|
+
# thread that is running the ZMQ context termination, which is blocked...
|
149
|
+
# TODO: Raise issue in JRuby github and try to track down why finalisers run while code is running
|
150
|
+
# if the exit! call is made
|
151
|
+
fail ZMQTerm, 'poll error: ' + ZMQ::Util.error_string if ZMQ::Util.errno == ZMQ::ETERM || ZMQ::Util.errno == ZMQ::ENOTSOCK
|
152
|
+
fail ZMQError, 'poll error: ' + ZMQ::Util.error_string
|
153
|
+
end
|
154
|
+
|
155
|
+
return if rc == 0
|
156
|
+
|
157
|
+
ready = (@poller.readables|@poller.writables)
|
158
|
+
|
159
|
+
ready.each do |socket|
|
160
|
+
if @handlers.key?(socket)
|
161
|
+
__send__ @handlers[socket][:callback], @handlers[socket]
|
162
|
+
end
|
163
|
+
|
164
|
+
yield socket, @poller.readables.include?(socket), @poller.writables.include?(socket)
|
165
|
+
end
|
166
|
+
|
167
|
+
return
|
168
|
+
end
|
169
|
+
|
170
|
+
private
|
171
|
+
|
172
|
+
def _create_socket_to_socket(socket)
|
173
|
+
receiver = @context.socket(ZMQ::PULL)
|
174
|
+
fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if receiver.nil?
|
175
|
+
|
176
|
+
rc = receiver.setsockopt(ZMQ::LINGER, 0)
|
177
|
+
fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
178
|
+
|
179
|
+
rc = receiver.bind("inproc://zmqpollreceiver-#{receiver.hash}")
|
180
|
+
fail ZMQError, 'bind error: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
181
|
+
|
182
|
+
sender = @context.socket(ZMQ::PUSH)
|
183
|
+
fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if sender.nil?
|
184
|
+
|
185
|
+
rc = sender.setsockopt(ZMQ::LINGER, 0)
|
186
|
+
fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
187
|
+
|
188
|
+
rc = sender.connect("inproc://zmqpollreceiver-#{receiver.hash}")
|
189
|
+
fail ZMQError, 'bind error: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
190
|
+
|
191
|
+
state = {
|
192
|
+
:callback => :handle_socket_to_socket,
|
193
|
+
:sender => sender,
|
194
|
+
:receiver => receiver,
|
195
|
+
:socket => socket,
|
196
|
+
:buffer => nil,
|
197
|
+
:send_ok => false,
|
198
|
+
:recv_ok => false,
|
199
|
+
}
|
200
|
+
|
201
|
+
@poller.register receiver, ZMQ::POLLIN
|
202
|
+
@poller.register socket, ZMQ::POLLOUT
|
203
|
+
@handlers[receiver] = state
|
204
|
+
@handlers[socket] = state
|
205
|
+
|
206
|
+
@sockets.push sender
|
207
|
+
|
208
|
+
state
|
209
|
+
end
|
210
|
+
|
211
|
+
def _close_socket_to_socket(state)
|
212
|
+
@sockets.delete state[:sender]
|
213
|
+
|
214
|
+
@poller.delete state[:receiver]
|
215
|
+
@poller.delete state[:socket]
|
216
|
+
|
217
|
+
state[:sender].close
|
218
|
+
state[:receiver].close
|
219
|
+
|
220
|
+
@handlers.delete state[:receiver]
|
221
|
+
@handlers.delete state[:socket]
|
222
|
+
|
223
|
+
return
|
224
|
+
end
|
225
|
+
|
226
|
+
def handle_socket_to_socket(state)
|
227
|
+
state[:recv_ok] = @poller.readables.include?(state[:receiver]) || state[:recv_ok]
|
228
|
+
state[:send_ok] = @poller.writables.include?(state[:socket]) || state[:send_ok]
|
229
|
+
|
230
|
+
loop do
|
231
|
+
if state[:send_ok] && !state[:buffer].nil?
|
232
|
+
begin
|
233
|
+
send state[:socket], state[:buffer]
|
234
|
+
rescue TimeoutError
|
235
|
+
end
|
236
|
+
state[:buffer] = nil if state[:buffer].length == 0
|
237
|
+
state[:send_ok] = false
|
238
|
+
end
|
239
|
+
|
240
|
+
break if !state[:recv_ok]
|
241
|
+
|
242
|
+
if state[:recv_ok] && state[:buffer].nil?
|
243
|
+
begin
|
244
|
+
state[:buffer] = recv(state[:receiver])
|
245
|
+
rescue TimeoutError
|
246
|
+
end
|
247
|
+
state[:recv_ok] = false
|
248
|
+
end
|
249
|
+
|
250
|
+
break if !state[:send_ok]
|
251
|
+
end
|
252
|
+
|
253
|
+
if state[:recv_ok]
|
254
|
+
@poller.deregister state[:receiver], ZMQ::POLLIN
|
255
|
+
else
|
256
|
+
@poller.register state[:receiver], ZMQ::POLLIN
|
257
|
+
end
|
258
|
+
|
259
|
+
if state[:send_ok]
|
260
|
+
@poller.deregister state[:socket], ZMQ::POLLOUT
|
261
|
+
else
|
262
|
+
@poller.register state[:socket], ZMQ::POLLOUT
|
263
|
+
end
|
264
|
+
|
265
|
+
return
|
266
|
+
end
|
267
|
+
|
268
|
+
def recv(socket)
|
269
|
+
data = []
|
270
|
+
|
271
|
+
poll_eagain(socket, ZMQ::POLLIN, 5) do
|
272
|
+
# recv_strings appears to be safe, ZMQ documents that a client will either
|
273
|
+
# receive 0 parts or all parts
|
274
|
+
socket.recv_strings(data, ZMQ::DONTWAIT)
|
275
|
+
end
|
276
|
+
|
277
|
+
data
|
278
|
+
end
|
279
|
+
|
280
|
+
def send(socket, data)
|
281
|
+
while data.length != 1
|
282
|
+
send_part socket, data.shift, true
|
283
|
+
end
|
284
|
+
send_part socket, data.shift
|
285
|
+
return
|
286
|
+
end
|
287
|
+
|
288
|
+
def send_part(socket, data, more=false)
|
289
|
+
poll_eagain(socket, ZMQ::POLLOUT, 5) do
|
290
|
+
# Try to send a message but never block
|
291
|
+
# We could use send_strings but it is vague on if ZMQ can return an
|
292
|
+
# error midway through sending parts...
|
293
|
+
socket.send_string(data, (more ? ZMQ::SNDMORE : 0) | ZMQ::DONTWAIT)
|
294
|
+
end
|
295
|
+
|
296
|
+
return
|
297
|
+
end
|
298
|
+
|
299
|
+
def poll_eagain(socket, flag, timeout, &block)
|
300
|
+
poller = nil
|
301
|
+
timeout = Time.now.to_i + timeout
|
302
|
+
loop do
|
303
|
+
rc = block.call()
|
304
|
+
break if ZMQ::Util.resultcode_ok?(rc)
|
305
|
+
if ZMQ::Util.errno != ZMQ::EAGAIN
|
306
|
+
fail ZMQError, 'message receive failed: ' + ZMQ::Util.error_string if flag == ZMQ::POLLIN
|
307
|
+
fail ZMQError, 'message send failed: ' + ZMQ::Util.error_string
|
308
|
+
end
|
309
|
+
|
310
|
+
# Wait for send to become available, handling timeouts
|
311
|
+
if poller.nil?
|
312
|
+
poller = ZMQ::Poller.new
|
313
|
+
poller.register socket, flag
|
314
|
+
end
|
315
|
+
|
316
|
+
while poller.poll(1_000) == 0
|
317
|
+
# Using this inner while triggers pollThreadEvents in JRuby which checks for Thread.raise immediately
|
318
|
+
fail TimeoutError while Time.now.to_i >= timeout
|
319
|
+
end
|
320
|
+
end
|
321
|
+
return
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log-courier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.3'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Woods
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cabin
|
@@ -65,6 +65,7 @@ files:
|
|
65
65
|
- lib/log-courier/server.rb
|
66
66
|
- lib/log-courier/server_tcp.rb
|
67
67
|
- lib/log-courier/server_zmq.rb
|
68
|
+
- lib/log-courier/zmq_qpoll.rb
|
68
69
|
homepage: https://github.com/driskell/log-courier
|
69
70
|
licenses:
|
70
71
|
- Apache
|