newque 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +54 -0
- data/Gemfile +3 -0
- data/LICENSE +373 -0
- data/README.md +340 -0
- data/Rakefile +5 -0
- data/conf/channels/example.json +12 -0
- data/conf/channels/example_fifo.json +18 -0
- data/conf/channels/example_plaintext.json +15 -0
- data/conf/channels/example_pubsub.json +15 -0
- data/conf/newque.json +26 -0
- data/lib/newque.rb +16 -0
- data/lib/newque/clients/client.rb +23 -0
- data/lib/newque/clients/fifo_client.rb +109 -0
- data/lib/newque/clients/pubsub_client.rb +101 -0
- data/lib/newque/errors.rb +6 -0
- data/lib/newque/future.rb +33 -0
- data/lib/newque/http/http_json.rb +49 -0
- data/lib/newque/http/http_plaintext.rb +66 -0
- data/lib/newque/http/newque_http.rb +116 -0
- data/lib/newque/responses.rb +219 -0
- data/lib/newque/util.rb +42 -0
- data/lib/newque/version.rb +3 -0
- data/lib/newque/zmq/newque_zmq.rb +121 -0
- data/lib/newque/zmq/protobuf.rb +96 -0
- data/lib/newque/zmq/zmq_tools.rb +52 -0
- data/newque.gemspec +35 -0
- data/spec/client_spec.rb +124 -0
- data/spec/fifo_client_spec.rb +130 -0
- data/spec/helpers.rb +12 -0
- data/spec/pubsub_client_spec.rb +142 -0
- metadata +164 -0
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
{
|
2
|
+
"listeners": ["http8000", "zmq8005"],
|
3
|
+
"backend": "fifo",
|
4
|
+
"backendSettings": {
|
5
|
+
"host": "0.0.0.0",
|
6
|
+
"port": 8007,
|
7
|
+
"timeout": 1000,
|
8
|
+
"healthTimeLimit": 500
|
9
|
+
},
|
10
|
+
"acknowledgement": "saved",
|
11
|
+
"readSettings": {
|
12
|
+
},
|
13
|
+
"writeSettings": {
|
14
|
+
"forward": []
|
15
|
+
},
|
16
|
+
"raw": true,
|
17
|
+
"emptiable": true
|
18
|
+
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
{
|
2
|
+
"listeners": ["http8000", "zmq8005"],
|
3
|
+
"backend": "disk",
|
4
|
+
"acknowledgement": "saved",
|
5
|
+
"readSettings": {
|
6
|
+
"httpFormat": "plaintext"
|
7
|
+
},
|
8
|
+
"writeSettings": {
|
9
|
+
"forward": [],
|
10
|
+
"httpFormat": "plaintext"
|
11
|
+
},
|
12
|
+
"raw": true,
|
13
|
+
"emptiable": true,
|
14
|
+
"maxRead": 5000
|
15
|
+
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
{
|
2
|
+
"listeners": ["http8000", "zmq8005"],
|
3
|
+
"backend": "pubsub",
|
4
|
+
"backendSettings": {
|
5
|
+
"host": "0.0.0.0",
|
6
|
+
"port": 8006
|
7
|
+
},
|
8
|
+
"acknowledgement": "saved",
|
9
|
+
"readSettings": null,
|
10
|
+
"writeSettings": {
|
11
|
+
"forward": []
|
12
|
+
},
|
13
|
+
"raw": true,
|
14
|
+
"emptiable": false
|
15
|
+
}
|
data/conf/newque.json
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
{
|
2
|
+
"logLevel": "debug",
|
3
|
+
"environment": "development",
|
4
|
+
"admin": {
|
5
|
+
"host": "0.0.0.0",
|
6
|
+
"port": 8001
|
7
|
+
},
|
8
|
+
"listeners": [
|
9
|
+
{
|
10
|
+
"protocol": "http",
|
11
|
+
"name": "http8000",
|
12
|
+
"host": "0.0.0.0",
|
13
|
+
"port": 8000,
|
14
|
+
"protocolSettings": {}
|
15
|
+
},
|
16
|
+
{
|
17
|
+
"protocol": "zmq",
|
18
|
+
"name": "zmq8005",
|
19
|
+
"host": "0.0.0.0",
|
20
|
+
"port": 8005,
|
21
|
+
"protocolSettings": {
|
22
|
+
"concurrency": 20
|
23
|
+
}
|
24
|
+
}
|
25
|
+
]
|
26
|
+
}
|
data/lib/newque.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'newque/errors'
|
2
|
+
require 'newque/util'
|
3
|
+
require 'newque/responses'
|
4
|
+
require 'newque/future'
|
5
|
+
|
6
|
+
require 'newque/zmq/protobuf'
|
7
|
+
require 'newque/zmq/zmq_tools'
|
8
|
+
require 'newque/zmq/newque_zmq'
|
9
|
+
|
10
|
+
require 'newque/http/http_json'
|
11
|
+
require 'newque/http/http_plaintext'
|
12
|
+
require 'newque/http/newque_http'
|
13
|
+
|
14
|
+
require 'newque/clients/client'
|
15
|
+
require 'newque/clients/pubsub_client'
|
16
|
+
require 'newque/clients/fifo_client'
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Newque
|
2
|
+
|
3
|
+
class Client
|
4
|
+
extend Forwardable
|
5
|
+
|
6
|
+
def_delegators :@instance, :write, :read, :read_stream, :count, :delete, :health
|
7
|
+
|
8
|
+
attr_reader :protocol
|
9
|
+
|
10
|
+
def initialize protocol, host, port, protocol_options:nil, timeout:10000
|
11
|
+
@protocol = protocol
|
12
|
+
@instance = if protocol == :zmq
|
13
|
+
Newque_zmq.new host, port, (protocol_options || {}), timeout
|
14
|
+
elsif protocol == :http
|
15
|
+
Newque_http.new host, port, (protocol_options || {}), timeout
|
16
|
+
else
|
17
|
+
raise NewqueError.new "Invalid protocol [#{protocol}]. Must be either :zmq or :http"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'ffi-rzmq'
|
2
|
+
|
3
|
+
module Newque
|
4
|
+
|
5
|
+
class Fifo_client
|
6
|
+
|
7
|
+
STATES = [:NEW, :CONNECTING, :RUNNING, :DISCONNECTING, :CLOSING, :CLOSED]
|
8
|
+
|
9
|
+
def initialize host, port, protocol_options:{}, socket_wait:100
|
10
|
+
@ctx = ZMQ::Context.new
|
11
|
+
@addr = "tcp://#{host}:#{port}"
|
12
|
+
@options = Util.compute_options Zmq_tools::BASE_OPTIONS, protocol_options
|
13
|
+
@socket_wait = socket_wait
|
14
|
+
@state = 0
|
15
|
+
end
|
16
|
+
|
17
|
+
def connect &block
|
18
|
+
raise NewqueError.new "Cannot connect because this client is #{current_state}" unless current_state == :NEW
|
19
|
+
next_state :CONNECTING
|
20
|
+
@handler = block
|
21
|
+
@sock = @ctx.socket ZMQ::DEALER
|
22
|
+
Zmq_tools.set_zmq_sock_options @sock, @options
|
23
|
+
|
24
|
+
@poller = ZMQ::Poller.new
|
25
|
+
@poller.register_readable @sock
|
26
|
+
|
27
|
+
@sock.connect @addr
|
28
|
+
|
29
|
+
ready = Util.wait_t
|
30
|
+
@thread = Thread.new do
|
31
|
+
next_state :RUNNING
|
32
|
+
@poller.poll(@socket_wait)
|
33
|
+
Util.resolve_t ready, ''
|
34
|
+
loop do
|
35
|
+
if current_state == :DISCONNECTING
|
36
|
+
@sock.disconnect @addr
|
37
|
+
@socket_wait = 0
|
38
|
+
next_state :CLOSING
|
39
|
+
end
|
40
|
+
if @poller.poll(@socket_wait) == 0
|
41
|
+
if current_state == :CLOSING
|
42
|
+
@sock.close
|
43
|
+
break
|
44
|
+
else
|
45
|
+
next
|
46
|
+
end
|
47
|
+
end
|
48
|
+
# RECEIVING INCOMING MESSAGE
|
49
|
+
buffers = []
|
50
|
+
@sock.recv_strings buffers, ZMQ::DONTWAIT
|
51
|
+
id, *frames = buffers
|
52
|
+
parsed = Zmq_tools.parse_input frames
|
53
|
+
|
54
|
+
response = begin
|
55
|
+
returned = @handler.(parsed)
|
56
|
+
unless returned.respond_to?(:serialize)
|
57
|
+
raise NewqueError.new "Block given to Fifo_client.connect returned #{returned.class} which is not a valid response object"
|
58
|
+
end
|
59
|
+
serialized = returned.serialize
|
60
|
+
messages = serialized[:messages]
|
61
|
+
serialized.delete :messages
|
62
|
+
{
|
63
|
+
output: Output.new(serialized.merge!(errors: [])),
|
64
|
+
messages: messages
|
65
|
+
}
|
66
|
+
rescue => handler_error
|
67
|
+
{
|
68
|
+
output: Output.new(errors: [handler_error.to_s], error_output: Error_Output.new),
|
69
|
+
messages: []
|
70
|
+
}
|
71
|
+
end
|
72
|
+
@sock.send_strings ([id, response[:output].encode.to_s] + response[:messages]), ZMQ::DONTWAIT
|
73
|
+
|
74
|
+
end
|
75
|
+
next_state :CLOSED
|
76
|
+
end
|
77
|
+
@thread.abort_on_exception = true
|
78
|
+
Future.new ready, 10
|
79
|
+
end
|
80
|
+
|
81
|
+
def disconnect
|
82
|
+
state = current_state
|
83
|
+
if state == :NEW
|
84
|
+
nil
|
85
|
+
elsif state == :RUNNING
|
86
|
+
next_state :DISCONNECTING
|
87
|
+
else
|
88
|
+
raise NewqueError.new "Can't disconnect since the Fifo_client is currently #{state}"
|
89
|
+
end
|
90
|
+
nil
|
91
|
+
end
|
92
|
+
|
93
|
+
private
|
94
|
+
|
95
|
+
def current_state
|
96
|
+
STATES[@state]
|
97
|
+
end
|
98
|
+
|
99
|
+
def next_state should_be
|
100
|
+
goes_to = STATES[@state + 1]
|
101
|
+
unless goes_to == should_be
|
102
|
+
raise NewqueError.new "Inconsistent state in Fifo_client. #{goes_to} should be #{should_be}"
|
103
|
+
end
|
104
|
+
@state = @state + 1
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'ffi-rzmq'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
module Newque
|
5
|
+
|
6
|
+
class Pubsub_client
|
7
|
+
|
8
|
+
def initialize host, port, protocol_options:{}, socket_wait:100
|
9
|
+
@ctx = ZMQ::Context.new
|
10
|
+
@addr = "tcp://#{host}:#{port}"
|
11
|
+
@options = Util.compute_options Zmq_tools::BASE_OPTIONS, protocol_options
|
12
|
+
@socket_wait = socket_wait
|
13
|
+
@disconnect = false
|
14
|
+
@ready
|
15
|
+
|
16
|
+
@listeners = {}
|
17
|
+
@error_handlers = []
|
18
|
+
end
|
19
|
+
|
20
|
+
def add_error_handler &block
|
21
|
+
@error_handlers << block
|
22
|
+
end
|
23
|
+
|
24
|
+
def subscribe &block
|
25
|
+
@disconnect = false
|
26
|
+
id = SecureRandom.uuid
|
27
|
+
@listeners[id] = block
|
28
|
+
start_loop unless is_looping?
|
29
|
+
thread = Thread.new do
|
30
|
+
@ready.join(1)
|
31
|
+
id
|
32
|
+
end
|
33
|
+
Future.new thread, 10
|
34
|
+
end
|
35
|
+
|
36
|
+
def unsubscribe id
|
37
|
+
@listeners.delete id
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
|
41
|
+
def disconnect
|
42
|
+
@disconnect = true
|
43
|
+
nil
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
# The socket connection happens here so that no network traffic occurs while not subscribed
|
49
|
+
def start_loop
|
50
|
+
@disconnect = false
|
51
|
+
@sock = @ctx.socket ZMQ::SUB
|
52
|
+
Zmq_tools.set_zmq_sock_options @sock, @options
|
53
|
+
|
54
|
+
@poller = ZMQ::Poller.new
|
55
|
+
@poller.register_readable @sock
|
56
|
+
|
57
|
+
@sock.connect @addr
|
58
|
+
@sock.setsockopt(ZMQ::SUBSCRIBE, '')
|
59
|
+
|
60
|
+
@ready = Util.wait_t
|
61
|
+
@thread = Thread.new do
|
62
|
+
@poller.poll(@socket_wait)
|
63
|
+
Util.resolve_t @ready, ''
|
64
|
+
loop do
|
65
|
+
break if @disconnect
|
66
|
+
next if @poller.poll(@socket_wait) == 0
|
67
|
+
buffers = []
|
68
|
+
@sock.recv_strings buffers, ZMQ::DONTWAIT
|
69
|
+
input = Zmq_tools.parse_input buffers
|
70
|
+
@listeners.values.each do |listener|
|
71
|
+
begin
|
72
|
+
listener.(input)
|
73
|
+
rescue => listener_error
|
74
|
+
print_uncaught_exception(listener_error, 'subscribe') if @error_handlers.size == 0
|
75
|
+
@error_handlers.each do |err_handler|
|
76
|
+
begin
|
77
|
+
err_handler.(listener_error)
|
78
|
+
rescue => uncaught_err
|
79
|
+
print_uncaught_exception uncaught_err, 'add_error_handler'
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
@sock.disconnect @addr
|
86
|
+
@sock.close
|
87
|
+
end
|
88
|
+
@thread.abort_on_exception = true
|
89
|
+
end
|
90
|
+
|
91
|
+
def is_looping?
|
92
|
+
!@thread.nil? && @thread.alive?
|
93
|
+
end
|
94
|
+
|
95
|
+
def print_uncaught_exception err, block_name
|
96
|
+
puts "Uncaught exception in Pubsub_client.#{block_name} block: #{err.to_s} #{JSON.pretty_generate(err.backtrace)}"
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Newque
|
2
|
+
|
3
|
+
class Future
|
4
|
+
|
5
|
+
attr_reader :thread
|
6
|
+
|
7
|
+
def initialize thread, timeout
|
8
|
+
@thread = thread
|
9
|
+
@timeout = timeout
|
10
|
+
end
|
11
|
+
|
12
|
+
def get limit=@timeout
|
13
|
+
result = @thread.join(limit)
|
14
|
+
if result.nil?
|
15
|
+
# Timeout exceeded
|
16
|
+
@thread.kill
|
17
|
+
raise Timeout::Error
|
18
|
+
end
|
19
|
+
result.value
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
def to_s
|
24
|
+
"<NewqueFuture timeout: #{@timeout} status: #{@thread.status}>"
|
25
|
+
end
|
26
|
+
|
27
|
+
def inspect
|
28
|
+
to_s
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
|
3
|
+
module Newque
|
4
|
+
|
5
|
+
class Http_json
|
6
|
+
def initialize http
|
7
|
+
@http = http
|
8
|
+
end
|
9
|
+
|
10
|
+
def write channel, atomic, msgs, ids=nil
|
11
|
+
thread = Thread.new do
|
12
|
+
body = {
|
13
|
+
'atomic' => false,
|
14
|
+
'messages' => msgs
|
15
|
+
}
|
16
|
+
body["ids"] = ids unless ids.nil?
|
17
|
+
res = @http.conn.post do |req|
|
18
|
+
@http.send :set_req_options, req
|
19
|
+
req.url "/v1/#{channel}"
|
20
|
+
req.body = body.to_json
|
21
|
+
end
|
22
|
+
parsed = @http.send :parse_json_response, res.body
|
23
|
+
Write_response.new parsed['saved']
|
24
|
+
end
|
25
|
+
Future.new thread, @http.timeout
|
26
|
+
end
|
27
|
+
|
28
|
+
def read channel, mode, limit=nil
|
29
|
+
thread = Thread.new do
|
30
|
+
res = @http.conn.get do |req|
|
31
|
+
@http.send :set_req_options, req
|
32
|
+
req.url "/v1/#{channel}"
|
33
|
+
req.headers['newque-mode'] = mode
|
34
|
+
req.headers['newque-read-max'] = limit unless limit.nil?
|
35
|
+
end
|
36
|
+
parsed = @http.send :parse_json_response, res.body
|
37
|
+
Read_response.new(
|
38
|
+
res.headers['newque-response-length'].to_i,
|
39
|
+
res.headers['newque-response-last-id'],
|
40
|
+
res.headers['newque-response-last-ts'].to_i,
|
41
|
+
parsed['messages']
|
42
|
+
)
|
43
|
+
end
|
44
|
+
Future.new thread, @http.timeout
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|