log-courier 1.2 → 1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/log-courier/client.rb +67 -56
- data/lib/log-courier/client_tls.rb +30 -31
- data/lib/log-courier/server.rb +3 -0
- data/lib/log-courier/server_tcp.rb +21 -1
- data/lib/log-courier/server_zmq.rb +18 -16
- data/lib/log-courier/zmq_qpoll.rb +324 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 645bd508ce891c3fb1fd69c4a414758fdc55c867
|
4
|
+
data.tar.gz: c4b14fb6f9cc1b75ac2c886d0162b543246ef5c5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2f512338b7e7a11fff7293a9006ef5a924596f5b7f40863e5f12a68297588e1969628aa6b3730b30bac9464071d9da63186aac9c460bfbe8f92219f42f6731e2
|
7
|
+
data.tar.gz: 736be3625601a89409beee281f432e6450af0a76aa040eb916e848512cfcb195b8bd6b0c6ef013e0a0757fe5237f35acdf9bd273ffe34a9c765e5f81ac4ab841
|
data/lib/log-courier/client.rb
CHANGED
@@ -30,21 +30,60 @@ module LogCourier
|
|
30
30
|
|
31
31
|
# Describes a pending payload
|
32
32
|
class PendingPayload
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
33
|
+
class << self
|
34
|
+
@json_adapter
|
35
|
+
def get_json_adapter
|
36
|
+
@json_adapter = MultiJson.adapter.instance if @json_adapter.nil?
|
37
|
+
return @json_adapter
|
38
|
+
end
|
39
|
+
end
|
37
40
|
|
38
|
-
attr_accessor :previous
|
39
41
|
attr_accessor :next
|
42
|
+
attr_accessor :nonce
|
43
|
+
attr_accessor :events
|
44
|
+
attr_accessor :last_sequence
|
45
|
+
attr_accessor :sequence_len
|
46
|
+
attr_accessor :payload
|
40
47
|
|
41
|
-
def initialize(
|
42
|
-
@
|
48
|
+
def initialize(events, nonce)
|
49
|
+
@events = events
|
50
|
+
@nonce = nonce
|
51
|
+
|
52
|
+
generate
|
53
|
+
end
|
54
|
+
|
55
|
+
def generate
|
56
|
+
fail ArgumentError, 'Corrupt payload' if @events.length == 0
|
57
|
+
|
58
|
+
buffer = Zlib::Deflate.new
|
43
59
|
|
44
|
-
|
45
|
-
|
46
|
-
|
60
|
+
# Write each event in JSON format
|
61
|
+
events.each do |event|
|
62
|
+
json_data = self.class.get_json_adapter.dump(event)
|
63
|
+
# Add length and then the data
|
64
|
+
buffer << [json_data.length].pack('N') << json_data
|
47
65
|
end
|
66
|
+
|
67
|
+
# Generate and store the payload
|
68
|
+
@payload = nonce + buffer.flush(Zlib::FINISH)
|
69
|
+
@last_sequence = 0
|
70
|
+
@sequence_len = @events.length
|
71
|
+
end
|
72
|
+
|
73
|
+
def ack(sequence)
|
74
|
+
if sequence <= @last_sequence
|
75
|
+
return 0, false
|
76
|
+
elsif sequence >= @sequence_len
|
77
|
+
lines = @sequence_len - @last_sequence
|
78
|
+
@last_sequence = sequence
|
79
|
+
@payload = nil
|
80
|
+
return lines, true
|
81
|
+
end
|
82
|
+
|
83
|
+
lines = sequence - @last_sequence
|
84
|
+
@last_sequence = sequence
|
85
|
+
@payload = nil
|
86
|
+
return lines, false
|
48
87
|
end
|
49
88
|
end
|
50
89
|
|
@@ -63,9 +102,6 @@ module LogCourier
|
|
63
102
|
require 'log-courier/client_tls'
|
64
103
|
@client = ClientTls.new(@options)
|
65
104
|
|
66
|
-
# Load the json adapter
|
67
|
-
@json_adapter = MultiJson.adapter.instance
|
68
|
-
|
69
105
|
@event_queue = EventQueue.new @options[:spool_size]
|
70
106
|
@pending_payloads = {}
|
71
107
|
@first_payload = nil
|
@@ -100,7 +136,7 @@ module LogCourier
|
|
100
136
|
@io_thread.raise ShutdownSignal
|
101
137
|
@spooler_thread.join
|
102
138
|
@io_thread.join
|
103
|
-
return
|
139
|
+
return @pending_payloads.length == 0
|
104
140
|
end
|
105
141
|
|
106
142
|
private
|
@@ -274,14 +310,9 @@ module LogCourier
|
|
274
310
|
def send_jdat(events)
|
275
311
|
# Generate the JSON payload and compress it
|
276
312
|
nonce = generate_nonce
|
277
|
-
data = buffer_jdat_data(events, nonce)
|
278
313
|
|
279
314
|
# Save the pending payload
|
280
|
-
payload = PendingPayload.new(
|
281
|
-
:events => events,
|
282
|
-
:nonce => nonce,
|
283
|
-
:data => data
|
284
|
-
)
|
315
|
+
payload = PendingPayload.new(events, nonce)
|
285
316
|
|
286
317
|
@pending_payloads[nonce] = payload
|
287
318
|
|
@@ -294,27 +325,7 @@ module LogCourier
|
|
294
325
|
end
|
295
326
|
|
296
327
|
# Send it
|
297
|
-
@client.send 'JDAT', payload.
|
298
|
-
return
|
299
|
-
end
|
300
|
-
|
301
|
-
def buffer_jdat_data(events, nonce)
|
302
|
-
buffer = Zlib::Deflate.new
|
303
|
-
|
304
|
-
# Write each event in JSON format
|
305
|
-
events.each do |event|
|
306
|
-
buffer_jdat_data_event(buffer, event)
|
307
|
-
end
|
308
|
-
|
309
|
-
# Generate and return the message
|
310
|
-
nonce + buffer.flush(Zlib::FINISH)
|
311
|
-
end
|
312
|
-
|
313
|
-
def buffer_jdat_data_event(buffer, event)
|
314
|
-
json_data = @json_adapter.dump(event)
|
315
|
-
|
316
|
-
# Add length and then the data
|
317
|
-
buffer << [json_data.length].pack('N') << json_data
|
328
|
+
@client.send 'JDAT', payload.payload
|
318
329
|
return
|
319
330
|
end
|
320
331
|
|
@@ -322,6 +333,8 @@ module LogCourier
|
|
322
333
|
# Sanity
|
323
334
|
fail ProtocolError, "Unexpected data attached to pong message (#{message.length})" if message.length != 0
|
324
335
|
|
336
|
+
@logger.debug 'PONG message received' unless @logger.nil? || !@logger.debug?
|
337
|
+
|
325
338
|
# No longer pending a PONG
|
326
339
|
@ping_pending = false
|
327
340
|
return
|
@@ -332,29 +345,27 @@ module LogCourier
|
|
332
345
|
fail ProtocolError, "ACKN message size invalid (#{message.length})" if message.length != 20
|
333
346
|
|
334
347
|
# Grab nonce
|
335
|
-
|
348
|
+
nonce, sequence = message.unpack('A16N')
|
349
|
+
|
350
|
+
if !@logger.nil? && @logger.debug?
|
351
|
+
nonce_str = nonce.each_byte.map do |b|
|
352
|
+
b.to_s(16).rjust(2, '0')
|
353
|
+
end
|
354
|
+
|
355
|
+
@logger.debug 'ACKN message received', :nonce => nonce_str.join, :sequence => sequence
|
356
|
+
end
|
336
357
|
|
337
358
|
# Find the payload - skip if we couldn't as it will just a duplicated ACK
|
338
359
|
return unless @pending_payloads.key?(nonce)
|
339
360
|
|
340
361
|
payload = @pending_payloads[nonce]
|
362
|
+
lines, complete = payload.ack(sequence)
|
341
363
|
|
342
|
-
|
343
|
-
# TODO: protocol error if sequence too large?
|
344
|
-
if sequence >= payload.events.length
|
345
|
-
@client.resume_send if @client.send_paused?
|
346
|
-
|
364
|
+
if complete
|
347
365
|
@pending_payloads.delete nonce
|
348
|
-
|
349
|
-
|
350
|
-
# Partial ACK - only process if something was actually processed
|
351
|
-
if sequence > payload.ack_events
|
352
|
-
payload.ack_events = sequence
|
353
|
-
payload.events = payload.events[0...sequence]
|
354
|
-
payload.data = nil
|
355
|
-
end
|
366
|
+
@first_payload = payload.next
|
367
|
+
@client.resume_send if @client.send_paused?
|
356
368
|
end
|
357
|
-
return
|
358
369
|
end
|
359
370
|
end
|
360
371
|
end
|
@@ -52,14 +52,15 @@ module LogCourier
|
|
52
52
|
end
|
53
53
|
|
54
54
|
def connect(io_control)
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
55
|
+
loop do
|
56
|
+
begin
|
57
|
+
break if tls_connect
|
58
|
+
rescue ShutdownSignal
|
59
|
+
raise
|
60
|
+
end
|
61
|
+
|
60
62
|
# TODO: Make this configurable
|
61
63
|
sleep 5
|
62
|
-
retry
|
63
64
|
end
|
64
65
|
|
65
66
|
@send_q = SizedQueue.new 1
|
@@ -88,6 +89,25 @@ module LogCourier
|
|
88
89
|
return
|
89
90
|
end
|
90
91
|
|
92
|
+
def pause_send
|
93
|
+
return if @send_paused
|
94
|
+
@send_paused = true
|
95
|
+
@send_q << nil
|
96
|
+
return
|
97
|
+
end
|
98
|
+
|
99
|
+
def send_paused?
|
100
|
+
@send_paused
|
101
|
+
end
|
102
|
+
|
103
|
+
def resume_send
|
104
|
+
if @send_paused
|
105
|
+
@send_paused = false
|
106
|
+
@send_q << nil
|
107
|
+
end
|
108
|
+
return
|
109
|
+
end
|
110
|
+
|
91
111
|
private
|
92
112
|
|
93
113
|
def run_send(io_control)
|
@@ -170,25 +190,6 @@ module LogCourier
|
|
170
190
|
return
|
171
191
|
end
|
172
192
|
|
173
|
-
def pause_send
|
174
|
-
return if @send_paused
|
175
|
-
@send_paused = true
|
176
|
-
@send_q << nil
|
177
|
-
return
|
178
|
-
end
|
179
|
-
|
180
|
-
def send_paused
|
181
|
-
@send_paused
|
182
|
-
end
|
183
|
-
|
184
|
-
def resume_send
|
185
|
-
if @send_paused
|
186
|
-
@send_paused = false
|
187
|
-
@send_q << nil
|
188
|
-
end
|
189
|
-
return
|
190
|
-
end
|
191
|
-
|
192
193
|
def tls_connect
|
193
194
|
# TODO: Implement random selection - and don't use separate :port - remember to update post_connection_check too
|
194
195
|
address = @options[:addresses][0]
|
@@ -208,7 +209,7 @@ module LogCourier
|
|
208
209
|
|
209
210
|
cert_store = OpenSSL::X509::Store.new
|
210
211
|
cert_store.add_file(@options[:ssl_ca])
|
211
|
-
|
212
|
+
ssl.cert_store = cert_store
|
212
213
|
ssl.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
|
213
214
|
|
214
215
|
@ssl_client = OpenSSL::SSL::SSLSocket.new(tcp_socket)
|
@@ -223,16 +224,14 @@ module LogCourier
|
|
223
224
|
@logger['port'] = port
|
224
225
|
|
225
226
|
@logger.info 'Connected successfully' unless @logger.nil?
|
226
|
-
return
|
227
|
+
return true
|
227
228
|
rescue OpenSSL::SSL::SSLError, IOError, Errno::ECONNRESET => e
|
228
229
|
@logger.warn 'Connection failed', :error => e.message, :address => address, :port => port unless @logger.nil?
|
229
|
-
return
|
230
|
-
rescue ShutdownSignal
|
231
|
-
return
|
232
230
|
rescue StandardError, NativeException => e
|
233
231
|
@logger.warn e, :hint => 'Unknown connection failure', :address => address, :port => port unless @logger.nil?
|
234
|
-
raise e
|
235
232
|
end
|
233
|
+
|
234
|
+
false
|
236
235
|
end
|
237
236
|
end
|
238
237
|
end
|
data/lib/log-courier/server.rb
CHANGED
@@ -63,6 +63,7 @@ module LogCourier
|
|
63
63
|
ssl_verify_default_ca: false,
|
64
64
|
ssl_verify_ca: nil,
|
65
65
|
max_packet_size: 10_485_760,
|
66
|
+
add_peer_fields: false,
|
66
67
|
}.merge!(options)
|
67
68
|
|
68
69
|
@logger = @options[:logger]
|
@@ -156,7 +157,7 @@ module LogCourier
|
|
156
157
|
rescue StandardError, NativeException => e
|
157
158
|
# Some other unknown problem
|
158
159
|
@logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
|
159
|
-
|
160
|
+
return
|
160
161
|
ensure
|
161
162
|
# Raise shutdown in all client threads and join then
|
162
163
|
client_threads.each do |_, thr|
|
@@ -195,8 +196,20 @@ module LogCourier
|
|
195
196
|
@logger = logger
|
196
197
|
@fd = fd
|
197
198
|
@peer = peer
|
199
|
+
@peer_fields = {}
|
198
200
|
@in_progress = false
|
199
201
|
@options = options
|
202
|
+
|
203
|
+
if @options[:add_peer_fields]
|
204
|
+
@peer_fields['peer'] = peer
|
205
|
+
if @options[:transport] == 'tls' && !@fd.peer_cert.nil?
|
206
|
+
@peer_fields['peer_ssl_cn'] = get_cn(@fd.peer_cert)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
def add_fields(event)
|
212
|
+
event.merge! @peer_fields if @peer_fields.length != 0
|
200
213
|
end
|
201
214
|
|
202
215
|
def run
|
@@ -284,6 +297,13 @@ module LogCourier
|
|
284
297
|
|
285
298
|
private
|
286
299
|
|
300
|
+
def get_cn(cert)
|
301
|
+
cert.subject.to_a.find do |oid, value|
|
302
|
+
return value if oid == "CN"
|
303
|
+
end
|
304
|
+
nil
|
305
|
+
end
|
306
|
+
|
287
307
|
def recv(need)
|
288
308
|
reset_timeout
|
289
309
|
have = ''
|
@@ -66,8 +66,14 @@ module LogCourier
|
|
66
66
|
|
67
67
|
begin
|
68
68
|
@context = ZMQ::Context.new
|
69
|
+
fail ZMQError, 'context creation error: ' + ZMQ::Util.error_string if @context.nil?
|
70
|
+
|
69
71
|
# Router so we can send multiple responses
|
70
72
|
@socket = @context.socket(ZMQ::ROUTER)
|
73
|
+
fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if @socket.nil?
|
74
|
+
|
75
|
+
rc = @socket.setsockopt(ZMQ::LINGER, 0)
|
76
|
+
fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
71
77
|
|
72
78
|
if @options[:transport] == 'zmq'
|
73
79
|
rc = @socket.setsockopt(ZMQ::CURVE_SERVER, 1)
|
@@ -104,20 +110,10 @@ module LogCourier
|
|
104
110
|
@poller = ZMQPoll::ZMQPoll.new(@context)
|
105
111
|
@poller.register_socket @socket, ZMQ::POLLIN
|
106
112
|
@poller.register_queue_to_socket @send_queue, @socket
|
107
|
-
|
108
|
-
# Register a finaliser that sets @context to nil
|
109
|
-
# This allows us to detect the JRuby bug where during "exit!" finalisers
|
110
|
-
# are run but threads are not killed - which leaves us in a situation of
|
111
|
-
# a terminated @context (it has a terminate finalizer) and an IO thread
|
112
|
-
# looping retries
|
113
|
-
# JRuby will still crash and burn, but at least we don't spam STDOUT with
|
114
|
-
# errors
|
115
|
-
ObjectSpace.define_finalizer(self, Proc.new do
|
116
|
-
@context = nil
|
117
|
-
end)
|
118
113
|
end
|
119
114
|
|
120
115
|
def run(&block)
|
116
|
+
errors = 0
|
121
117
|
loop do
|
122
118
|
begin
|
123
119
|
@poller.poll(5_000) do |socket, r, w|
|
@@ -126,11 +122,14 @@ module LogCourier
|
|
126
122
|
|
127
123
|
receive &block
|
128
124
|
end
|
125
|
+
errors = 0
|
129
126
|
rescue ZMQPoll::ZMQError => e
|
130
|
-
|
131
|
-
fail e if @context.nil?
|
132
|
-
@logger.warn e, :hint => 'ZMQ recv_string failure' unless @logger.nil?
|
127
|
+
@logger.warn e, :hint => 'ZMQ poll failure' unless @logger.nil?
|
133
128
|
next
|
129
|
+
rescue ZMQPoll::ZMQTerm
|
130
|
+
# Fall into shutdown signal, context was terminated
|
131
|
+
# This can happen in JRuby - it seems to run finalisers too early
|
132
|
+
fail ShutdownSignal
|
134
133
|
rescue ZMQPoll::TimeoutError
|
135
134
|
# We'll let ZeroMQ manage reconnections and new connections
|
136
135
|
# There is no point in us doing any form of reconnect ourselves
|
@@ -145,7 +144,7 @@ module LogCourier
|
|
145
144
|
rescue StandardError, NativeException => e
|
146
145
|
# Some other unknown problem
|
147
146
|
@logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
|
148
|
-
|
147
|
+
return
|
149
148
|
ensure
|
150
149
|
@poller.shutdown
|
151
150
|
@factory.shutdown
|
@@ -259,7 +258,7 @@ module LogCourier
|
|
259
258
|
|
260
259
|
# Create the client and associated thread
|
261
260
|
client = ClientZmq.new(self, source, source_str) do
|
262
|
-
try_drop
|
261
|
+
try_drop source, source_str
|
263
262
|
end
|
264
263
|
|
265
264
|
thread = Thread.new do
|
@@ -366,6 +365,9 @@ module LogCourier
|
|
366
365
|
return
|
367
366
|
end
|
368
367
|
|
368
|
+
def add_fields(event)
|
369
|
+
end
|
370
|
+
|
369
371
|
private
|
370
372
|
|
371
373
|
def recv(data)
|
@@ -0,0 +1,324 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# Copyright 2014 Jason Woods.
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
begin
|
18
|
+
require 'ffi-rzmq'
|
19
|
+
require 'ffi-rzmq/version'
|
20
|
+
require 'ffi-rzmq-core/version'
|
21
|
+
rescue LoadError => e
|
22
|
+
raise "ZMQPoll could not initialise: #{e}"
|
23
|
+
end
|
24
|
+
|
25
|
+
module ZMQPoll
|
26
|
+
class ZMQError < StandardError; end
|
27
|
+
class ZMQTerm < StandardError; end
|
28
|
+
class TimeoutError < StandardError; end
|
29
|
+
|
30
|
+
class ZMQPoll
|
31
|
+
def initialize(context, logger=nil)
|
32
|
+
@logger = logger
|
33
|
+
@context = context
|
34
|
+
@poller = ZMQ::Poller.new
|
35
|
+
@sockets = []
|
36
|
+
@socket_to_socket = []
|
37
|
+
@handlers = {}
|
38
|
+
@queues = {}
|
39
|
+
end
|
40
|
+
|
41
|
+
def readables
|
42
|
+
@poller.readables
|
43
|
+
end
|
44
|
+
|
45
|
+
def writables
|
46
|
+
@poller.writables
|
47
|
+
end
|
48
|
+
|
49
|
+
def shutdown
|
50
|
+
@queues.each_key do |queue|
|
51
|
+
deregister_queue queue
|
52
|
+
end
|
53
|
+
|
54
|
+
@socket_to_socket.each do |socket|
|
55
|
+
_close_socket_to_socket socket
|
56
|
+
end
|
57
|
+
|
58
|
+
@sockets.each do |socket|
|
59
|
+
socket.close
|
60
|
+
end
|
61
|
+
return
|
62
|
+
end
|
63
|
+
|
64
|
+
def register_socket(socket, flags)
|
65
|
+
@poller.register socket, flags
|
66
|
+
return
|
67
|
+
end
|
68
|
+
|
69
|
+
def deregister_socket(socket)
|
70
|
+
return if @handlers.key?(socket)
|
71
|
+
|
72
|
+
@poller.delete socket
|
73
|
+
return
|
74
|
+
end
|
75
|
+
|
76
|
+
def register_queue_to_socket(queue, socket)
|
77
|
+
s2s_state = _create_socket_to_socket(socket)
|
78
|
+
|
79
|
+
state = {
|
80
|
+
state: s2s_state,
|
81
|
+
mutex: Mutex.new,
|
82
|
+
shutdown: false,
|
83
|
+
}
|
84
|
+
|
85
|
+
state[:thread] = Thread.new do
|
86
|
+
loop do
|
87
|
+
data = queue.pop
|
88
|
+
break if data.nil?
|
89
|
+
begin
|
90
|
+
send s2s_state[:sender], data
|
91
|
+
rescue TimeoutError
|
92
|
+
state[:mutex].synchronize do
|
93
|
+
break if state[:shutdown]
|
94
|
+
end
|
95
|
+
retry
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@queues[queue] = state
|
101
|
+
return
|
102
|
+
end
|
103
|
+
|
104
|
+
def deregister_queue(queue)
|
105
|
+
return if !@queues.key?(queue)
|
106
|
+
|
107
|
+
# Push nil so if we're idle we jump into action and exit
|
108
|
+
# But also set shutdown to try so if we're mid-send and timeout, we exit
|
109
|
+
@queues[queue][:mutex].synchronize do
|
110
|
+
queue.push nil
|
111
|
+
@queues[queue][:shutdown] = true
|
112
|
+
end
|
113
|
+
@queues[queue][:thread].join
|
114
|
+
|
115
|
+
_close_socket_to_socket @queues[queue][:state]
|
116
|
+
@queues.delete queue
|
117
|
+
return
|
118
|
+
end
|
119
|
+
|
120
|
+
def create_socket_to_socket(socket)
|
121
|
+
state = _create_socket_to_socket(socket)
|
122
|
+
@socket_to_socket[state[:sender]] = state
|
123
|
+
state[:sender]
|
124
|
+
end
|
125
|
+
|
126
|
+
def close_socket_to_socket(socket)
|
127
|
+
return if !@socket_to_socket.include?(socket)
|
128
|
+
state = @socket_to_socket[socket]
|
129
|
+
@socket_to_socket.delete socket
|
130
|
+
_close_socket_to_socket(state)
|
131
|
+
return
|
132
|
+
end
|
133
|
+
|
134
|
+
def poll(timeout)
|
135
|
+
if @poller.size == 0
|
136
|
+
fail ZMQError, 'poll run called with zero socket/queues'
|
137
|
+
end
|
138
|
+
|
139
|
+
rc = @poller.poll(timeout)
|
140
|
+
unless ZMQ::Util.resultcode_ok?(rc)
|
141
|
+
# If we get ETERM or ENOTSOCK - we may have hit JRuby bug where finalisers are called too early
|
142
|
+
# We throw it so the caller knows NOT to retry and hit a bad loop
|
143
|
+
# The downside is - this Jruby bug will cause an inevitable deadlock
|
144
|
+
# The finaliser for the context will have run before finalisers for sockets (random order?)
|
145
|
+
# The resulting ZMQ context termination will block, waiting for sockets to be closed
|
146
|
+
# However, when closing a socket, ffi-rzmq will attempt to remove it's finaliser
|
147
|
+
# This removal will deadlock as it attempts to lock the finaliser list, which is locked by the
|
148
|
+
# thread that is running the ZMQ context termination, which is blocked...
|
149
|
+
# TODO: Raise issue in JRuby github and try to track down why finalisers run while code is running
|
150
|
+
# if the exit! call is made
|
151
|
+
fail ZMQTerm, 'poll error: ' + ZMQ::Util.error_string if ZMQ::Util.errno == ZMQ::ETERM || ZMQ::Util.errno == ZMQ::ENOTSOCK
|
152
|
+
fail ZMQError, 'poll error: ' + ZMQ::Util.error_string
|
153
|
+
end
|
154
|
+
|
155
|
+
return if rc == 0
|
156
|
+
|
157
|
+
ready = (@poller.readables|@poller.writables)
|
158
|
+
|
159
|
+
ready.each do |socket|
|
160
|
+
if @handlers.key?(socket)
|
161
|
+
__send__ @handlers[socket][:callback], @handlers[socket]
|
162
|
+
end
|
163
|
+
|
164
|
+
yield socket, @poller.readables.include?(socket), @poller.writables.include?(socket)
|
165
|
+
end
|
166
|
+
|
167
|
+
return
|
168
|
+
end
|
169
|
+
|
170
|
+
private
|
171
|
+
|
172
|
+
def _create_socket_to_socket(socket)
|
173
|
+
receiver = @context.socket(ZMQ::PULL)
|
174
|
+
fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if receiver.nil?
|
175
|
+
|
176
|
+
rc = receiver.setsockopt(ZMQ::LINGER, 0)
|
177
|
+
fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
178
|
+
|
179
|
+
rc = receiver.bind("inproc://zmqpollreceiver-#{receiver.hash}")
|
180
|
+
fail ZMQError, 'bind error: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
181
|
+
|
182
|
+
sender = @context.socket(ZMQ::PUSH)
|
183
|
+
fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if sender.nil?
|
184
|
+
|
185
|
+
rc = sender.setsockopt(ZMQ::LINGER, 0)
|
186
|
+
fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
187
|
+
|
188
|
+
rc = sender.connect("inproc://zmqpollreceiver-#{receiver.hash}")
|
189
|
+
fail ZMQError, 'bind error: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
|
190
|
+
|
191
|
+
state = {
|
192
|
+
:callback => :handle_socket_to_socket,
|
193
|
+
:sender => sender,
|
194
|
+
:receiver => receiver,
|
195
|
+
:socket => socket,
|
196
|
+
:buffer => nil,
|
197
|
+
:send_ok => false,
|
198
|
+
:recv_ok => false,
|
199
|
+
}
|
200
|
+
|
201
|
+
@poller.register receiver, ZMQ::POLLIN
|
202
|
+
@poller.register socket, ZMQ::POLLOUT
|
203
|
+
@handlers[receiver] = state
|
204
|
+
@handlers[socket] = state
|
205
|
+
|
206
|
+
@sockets.push sender
|
207
|
+
|
208
|
+
state
|
209
|
+
end
|
210
|
+
|
211
|
+
def _close_socket_to_socket(state)
|
212
|
+
@sockets.delete state[:sender]
|
213
|
+
|
214
|
+
@poller.delete state[:receiver]
|
215
|
+
@poller.delete state[:socket]
|
216
|
+
|
217
|
+
state[:sender].close
|
218
|
+
state[:receiver].close
|
219
|
+
|
220
|
+
@handlers.delete state[:receiver]
|
221
|
+
@handlers.delete state[:socket]
|
222
|
+
|
223
|
+
return
|
224
|
+
end
|
225
|
+
|
226
|
+
def handle_socket_to_socket(state)
|
227
|
+
state[:recv_ok] = @poller.readables.include?(state[:receiver]) || state[:recv_ok]
|
228
|
+
state[:send_ok] = @poller.writables.include?(state[:socket]) || state[:send_ok]
|
229
|
+
|
230
|
+
loop do
|
231
|
+
if state[:send_ok] && !state[:buffer].nil?
|
232
|
+
begin
|
233
|
+
send state[:socket], state[:buffer]
|
234
|
+
rescue TimeoutError
|
235
|
+
end
|
236
|
+
state[:buffer] = nil if state[:buffer].length == 0
|
237
|
+
state[:send_ok] = false
|
238
|
+
end
|
239
|
+
|
240
|
+
break if !state[:recv_ok]
|
241
|
+
|
242
|
+
if state[:recv_ok] && state[:buffer].nil?
|
243
|
+
begin
|
244
|
+
state[:buffer] = recv(state[:receiver])
|
245
|
+
rescue TimeoutError
|
246
|
+
end
|
247
|
+
state[:recv_ok] = false
|
248
|
+
end
|
249
|
+
|
250
|
+
break if !state[:send_ok]
|
251
|
+
end
|
252
|
+
|
253
|
+
if state[:recv_ok]
|
254
|
+
@poller.deregister state[:receiver], ZMQ::POLLIN
|
255
|
+
else
|
256
|
+
@poller.register state[:receiver], ZMQ::POLLIN
|
257
|
+
end
|
258
|
+
|
259
|
+
if state[:send_ok]
|
260
|
+
@poller.deregister state[:socket], ZMQ::POLLOUT
|
261
|
+
else
|
262
|
+
@poller.register state[:socket], ZMQ::POLLOUT
|
263
|
+
end
|
264
|
+
|
265
|
+
return
|
266
|
+
end
|
267
|
+
|
268
|
+
def recv(socket)
|
269
|
+
data = []
|
270
|
+
|
271
|
+
poll_eagain(socket, ZMQ::POLLIN, 5) do
|
272
|
+
# recv_strings appears to be safe, ZMQ documents that a client will either
|
273
|
+
# receive 0 parts or all parts
|
274
|
+
socket.recv_strings(data, ZMQ::DONTWAIT)
|
275
|
+
end
|
276
|
+
|
277
|
+
data
|
278
|
+
end
|
279
|
+
|
280
|
+
def send(socket, data)
|
281
|
+
while data.length != 1
|
282
|
+
send_part socket, data.shift, true
|
283
|
+
end
|
284
|
+
send_part socket, data.shift
|
285
|
+
return
|
286
|
+
end
|
287
|
+
|
288
|
+
def send_part(socket, data, more=false)
|
289
|
+
poll_eagain(socket, ZMQ::POLLOUT, 5) do
|
290
|
+
# Try to send a message but never block
|
291
|
+
# We could use send_strings but it is vague on if ZMQ can return an
|
292
|
+
# error midway through sending parts...
|
293
|
+
socket.send_string(data, (more ? ZMQ::SNDMORE : 0) | ZMQ::DONTWAIT)
|
294
|
+
end
|
295
|
+
|
296
|
+
return
|
297
|
+
end
|
298
|
+
|
299
|
+
def poll_eagain(socket, flag, timeout, &block)
|
300
|
+
poller = nil
|
301
|
+
timeout = Time.now.to_i + timeout
|
302
|
+
loop do
|
303
|
+
rc = block.call()
|
304
|
+
break if ZMQ::Util.resultcode_ok?(rc)
|
305
|
+
if ZMQ::Util.errno != ZMQ::EAGAIN
|
306
|
+
fail ZMQError, 'message receive failed: ' + ZMQ::Util.error_string if flag == ZMQ::POLLIN
|
307
|
+
fail ZMQError, 'message send failed: ' + ZMQ::Util.error_string
|
308
|
+
end
|
309
|
+
|
310
|
+
# Wait for send to become available, handling timeouts
|
311
|
+
if poller.nil?
|
312
|
+
poller = ZMQ::Poller.new
|
313
|
+
poller.register socket, flag
|
314
|
+
end
|
315
|
+
|
316
|
+
while poller.poll(1_000) == 0
|
317
|
+
# Using this inner while triggers pollThreadEvents in JRuby which checks for Thread.raise immediately
|
318
|
+
fail TimeoutError while Time.now.to_i >= timeout
|
319
|
+
end
|
320
|
+
end
|
321
|
+
return
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log-courier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.3'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Woods
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: cabin
|
@@ -65,6 +65,7 @@ files:
|
|
65
65
|
- lib/log-courier/server.rb
|
66
66
|
- lib/log-courier/server_tcp.rb
|
67
67
|
- lib/log-courier/server_zmq.rb
|
68
|
+
- lib/log-courier/zmq_qpoll.rb
|
68
69
|
homepage: https://github.com/driskell/log-courier
|
69
70
|
licenses:
|
70
71
|
- Apache
|