newque 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +54 -0
- data/Gemfile +3 -0
- data/LICENSE +373 -0
- data/README.md +340 -0
- data/Rakefile +5 -0
- data/conf/channels/example.json +12 -0
- data/conf/channels/example_fifo.json +18 -0
- data/conf/channels/example_plaintext.json +15 -0
- data/conf/channels/example_pubsub.json +15 -0
- data/conf/newque.json +26 -0
- data/lib/newque.rb +16 -0
- data/lib/newque/clients/client.rb +23 -0
- data/lib/newque/clients/fifo_client.rb +109 -0
- data/lib/newque/clients/pubsub_client.rb +101 -0
- data/lib/newque/errors.rb +6 -0
- data/lib/newque/future.rb +33 -0
- data/lib/newque/http/http_json.rb +49 -0
- data/lib/newque/http/http_plaintext.rb +66 -0
- data/lib/newque/http/newque_http.rb +116 -0
- data/lib/newque/responses.rb +219 -0
- data/lib/newque/util.rb +42 -0
- data/lib/newque/version.rb +3 -0
- data/lib/newque/zmq/newque_zmq.rb +121 -0
- data/lib/newque/zmq/protobuf.rb +96 -0
- data/lib/newque/zmq/zmq_tools.rb +52 -0
- data/newque.gemspec +35 -0
- data/spec/client_spec.rb +124 -0
- data/spec/fifo_client_spec.rb +130 -0
- data/spec/helpers.rb +12 -0
- data/spec/pubsub_client_spec.rb +142 -0
- metadata +164 -0
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
|
3
|
+
module Newque
|
4
|
+
|
5
|
+
class Http_plaintext
|
6
|
+
def initialize http
|
7
|
+
@http = http
|
8
|
+
end
|
9
|
+
|
10
|
+
def write channel, atomic, msgs, ids=nil
|
11
|
+
head, *tail = msgs
|
12
|
+
raise NewqueError.new("No messages given") if head.nil?
|
13
|
+
|
14
|
+
stream = StringIO.new
|
15
|
+
stream.write head
|
16
|
+
if msgs.size > 1
|
17
|
+
tail.each do |msg|
|
18
|
+
stream.write @http.options[:separator]
|
19
|
+
stream.write msg
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
thread = Thread.new do
|
24
|
+
res = @http.conn.post do |req|
|
25
|
+
@http.send :set_req_options, req
|
26
|
+
req.url "/v1/#{channel}"
|
27
|
+
req.body = stream.string
|
28
|
+
|
29
|
+
req.headers['newque-mode'] = if atomic then 'atomic'
|
30
|
+
elsif msgs.size == 1 then 'single'
|
31
|
+
else 'multiple'
|
32
|
+
end
|
33
|
+
req.headers['newque-msg-id'] = ids unless ids.nil?
|
34
|
+
end
|
35
|
+
|
36
|
+
parsed = @http.send :parse_json_response, res.body
|
37
|
+
Write_response.new parsed['saved']
|
38
|
+
end
|
39
|
+
Future.new thread, @http.timeout
|
40
|
+
end
|
41
|
+
|
42
|
+
def read channel, mode, limit=nil
|
43
|
+
thread = Thread.new do
|
44
|
+
res = @http.conn.get do |req|
|
45
|
+
@http.send :set_req_options, req
|
46
|
+
req.url "/v1/#{channel}"
|
47
|
+
req.headers['newque-mode'] = mode
|
48
|
+
req.headers['newque-read-max'] = limit unless limit.nil?
|
49
|
+
end
|
50
|
+
if res.status == 200
|
51
|
+
Read_response.new(
|
52
|
+
res.headers['newque-response-length'].to_i,
|
53
|
+
res.headers['newque-response-last-id'],
|
54
|
+
res.headers['newque-response-last-ts'].to_i,
|
55
|
+
res.body.split(@http.options[:separator])
|
56
|
+
)
|
57
|
+
else
|
58
|
+
@http.send :parse_json_response, res.body
|
59
|
+
end
|
60
|
+
end
|
61
|
+
Future.new thread, @http.timeout
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Newque
|
5
|
+
|
6
|
+
class Newque_http
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
BASE_OPTIONS = {
|
10
|
+
https: false,
|
11
|
+
http_format: :json,
|
12
|
+
separator: "\n"
|
13
|
+
}
|
14
|
+
|
15
|
+
def_delegators :@instance, :write, :read
|
16
|
+
attr_reader :conn, :options, :timeout
|
17
|
+
|
18
|
+
def initialize host, port, options, timeout
|
19
|
+
@host = host
|
20
|
+
@port = port
|
21
|
+
@options = Util.compute_options BASE_OPTIONS, options
|
22
|
+
@timeout = timeout / 1000.0
|
23
|
+
|
24
|
+
@conn = Faraday.new({ url: "#{@options[:https] ? "https" : "http"}://#{host}:#{port}" })
|
25
|
+
|
26
|
+
@instance = if @options[:http_format] == :json
|
27
|
+
Http_json.new self
|
28
|
+
elsif @options[:http_format] == :plaintext
|
29
|
+
Http_plaintext.new self
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def read_stream channel, mode, limit=nil
|
34
|
+
Enumerator.new do |generator|
|
35
|
+
Net::HTTP.start(@host, @port) do |http|
|
36
|
+
req = Net::HTTP::Get.new "/v1/#{channel}"
|
37
|
+
req['newque-mode'] = mode
|
38
|
+
req['newque-read-max'] = limit unless limit.nil?
|
39
|
+
req['Transfer-Encoding'] = 'chunked'
|
40
|
+
http.open_timeout = @timeout
|
41
|
+
http.read_timeout = @timeout
|
42
|
+
|
43
|
+
http.request req do |res|
|
44
|
+
if res.code != '200'
|
45
|
+
generator << parse_json_response(res.body) # Will throw
|
46
|
+
else
|
47
|
+
leftover = nil
|
48
|
+
res.read_body do |chunk|
|
49
|
+
lines = "#{leftover || ''}#{chunk}".split(@options[:separator], -1)
|
50
|
+
*fulls, partial = lines
|
51
|
+
fulls.each do |msg|
|
52
|
+
generator << msg
|
53
|
+
end
|
54
|
+
leftover = partial
|
55
|
+
end
|
56
|
+
generator << leftover unless leftover.nil?
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def count channel
|
64
|
+
thread = Thread.new do
|
65
|
+
res = @conn.get do |req|
|
66
|
+
set_req_options req
|
67
|
+
req.url "/v1/#{channel}/count"
|
68
|
+
end
|
69
|
+
parsed = parse_json_response res.body
|
70
|
+
Count_response.new parsed['count']
|
71
|
+
end
|
72
|
+
Future.new thread, @timeout
|
73
|
+
end
|
74
|
+
|
75
|
+
def delete channel
|
76
|
+
thread = Thread.new do
|
77
|
+
res = @conn.delete do |req|
|
78
|
+
set_req_options req
|
79
|
+
req.url "/v1/#{channel}"
|
80
|
+
end
|
81
|
+
parsed = parse_json_response res.body
|
82
|
+
Delete_response.new
|
83
|
+
end
|
84
|
+
Future.new thread, @timeout
|
85
|
+
end
|
86
|
+
|
87
|
+
def health channel, global=false
|
88
|
+
thread = Thread.new do
|
89
|
+
res = @conn.get do |req|
|
90
|
+
set_req_options req
|
91
|
+
req.url "/v1#{global ? '' : '/' + channel}/health"
|
92
|
+
end
|
93
|
+
parsed = parse_json_response res.body
|
94
|
+
Health_response.new
|
95
|
+
end
|
96
|
+
Future.new thread, @timeout
|
97
|
+
end
|
98
|
+
|
99
|
+
private
|
100
|
+
|
101
|
+
def set_req_options req
|
102
|
+
req.options.open_timeout = @timeout
|
103
|
+
req.options.timeout = @timeout
|
104
|
+
end
|
105
|
+
|
106
|
+
def parse_json_response body
|
107
|
+
parsed = JSON.parse body
|
108
|
+
if parsed['errors'].size > 0
|
109
|
+
raise Util.newque_error parsed['errors']
|
110
|
+
end
|
111
|
+
parsed
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
@@ -0,0 +1,219 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Newque
|
4
|
+
|
5
|
+
# ------------------------------------
|
6
|
+
# REQUESTS
|
7
|
+
# ------------------------------------
|
8
|
+
class Write_request
|
9
|
+
attr_reader :atomic, :ids
|
10
|
+
|
11
|
+
def initialize atomic, ids
|
12
|
+
@atomic = atomic
|
13
|
+
@ids = ids
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_s
|
17
|
+
"<Write atomic: #{@atomic.to_json} ids: #{@ids.to_json}>"
|
18
|
+
end
|
19
|
+
|
20
|
+
def inspect
|
21
|
+
to_s
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class Read_request
|
26
|
+
attr_reader :mode, :limit
|
27
|
+
|
28
|
+
def initialize mode, limit
|
29
|
+
@mode = mode
|
30
|
+
@limit = limit
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_s
|
34
|
+
"<Read mode: #{@mode.to_json} limit: #{@limit.to_json}>"
|
35
|
+
end
|
36
|
+
|
37
|
+
def inspect
|
38
|
+
to_s
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class Count_request
|
43
|
+
def initialize
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_s
|
47
|
+
"<Count >"
|
48
|
+
end
|
49
|
+
|
50
|
+
def inspect
|
51
|
+
to_s
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class Delete_request
|
56
|
+
def initialize
|
57
|
+
end
|
58
|
+
|
59
|
+
def to_s
|
60
|
+
"<Delete >"
|
61
|
+
end
|
62
|
+
|
63
|
+
def inspect
|
64
|
+
to_s
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class Health_request
|
69
|
+
attr_reader :global
|
70
|
+
|
71
|
+
def initialize global
|
72
|
+
@global = global
|
73
|
+
end
|
74
|
+
|
75
|
+
def to_s
|
76
|
+
"<Health global: #{@global.to_json}>"
|
77
|
+
end
|
78
|
+
|
79
|
+
def inspect
|
80
|
+
to_s
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
class Input_request
|
85
|
+
attr_reader :channel, :action, :messages
|
86
|
+
|
87
|
+
def initialize channel, action, messages
|
88
|
+
@channel = channel
|
89
|
+
@action = action
|
90
|
+
@messages = messages
|
91
|
+
end
|
92
|
+
|
93
|
+
def to_s
|
94
|
+
"<Newque_input channel: #{channel.to_json} action: #{@action} #{@messages.size > 0 ? 'messages: ' + @messages.to_json : ''}>"
|
95
|
+
end
|
96
|
+
|
97
|
+
def inspect
|
98
|
+
to_s
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# ------------------------------------
|
103
|
+
# RESPONSES
|
104
|
+
# ------------------------------------
|
105
|
+
class Write_response
|
106
|
+
attr_reader :saved
|
107
|
+
|
108
|
+
def initialize saved
|
109
|
+
@saved = saved
|
110
|
+
end
|
111
|
+
|
112
|
+
def to_s
|
113
|
+
"<Newque_write saved: #{saved.to_json}>"
|
114
|
+
end
|
115
|
+
|
116
|
+
def inspect
|
117
|
+
to_s
|
118
|
+
end
|
119
|
+
|
120
|
+
def serialize
|
121
|
+
{
|
122
|
+
write_output: Write_Output.new(saved: @saved),
|
123
|
+
messages: []
|
124
|
+
}
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
class Read_response
|
129
|
+
attr_reader :length, :last_id, :last_timens, :messages
|
130
|
+
|
131
|
+
def initialize length, last_id, last_timens, messages
|
132
|
+
@length = length
|
133
|
+
@last_id = last_id
|
134
|
+
@last_timens = last_timens
|
135
|
+
@messages = messages
|
136
|
+
end
|
137
|
+
|
138
|
+
def to_s
|
139
|
+
"<Newque_read length: #{length.to_json} last_id: #{last_id.to_json} last_timens: #{last_timens.to_json} messages: #{messages}>"
|
140
|
+
end
|
141
|
+
|
142
|
+
def inspect
|
143
|
+
to_s
|
144
|
+
end
|
145
|
+
|
146
|
+
def serialize
|
147
|
+
{
|
148
|
+
read_output: Read_Output.new(length: @length, last_id: @last_id, last_timens: @last_timens),
|
149
|
+
messages: @messages
|
150
|
+
}
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
class Count_response
|
155
|
+
attr_reader :count
|
156
|
+
|
157
|
+
def initialize count
|
158
|
+
@count = count
|
159
|
+
end
|
160
|
+
|
161
|
+
def to_s
|
162
|
+
"<Newque_count count: #{count.to_json}>"
|
163
|
+
end
|
164
|
+
|
165
|
+
def inspect
|
166
|
+
to_s
|
167
|
+
end
|
168
|
+
|
169
|
+
def serialize
|
170
|
+
{
|
171
|
+
count_output: Count_Output.new(count: @count),
|
172
|
+
messages: []
|
173
|
+
}
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
class Delete_response
|
178
|
+
|
179
|
+
def initialize
|
180
|
+
end
|
181
|
+
|
182
|
+
def to_s
|
183
|
+
"<Newque_delete >"
|
184
|
+
end
|
185
|
+
|
186
|
+
def inspect
|
187
|
+
to_s
|
188
|
+
end
|
189
|
+
|
190
|
+
def serialize
|
191
|
+
{
|
192
|
+
delete_output: Delete_Output.new,
|
193
|
+
messages: []
|
194
|
+
}
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
class Health_response
|
199
|
+
|
200
|
+
def initialize
|
201
|
+
end
|
202
|
+
|
203
|
+
def to_s
|
204
|
+
"<Newque_health >"
|
205
|
+
end
|
206
|
+
|
207
|
+
def inspect
|
208
|
+
to_s
|
209
|
+
end
|
210
|
+
|
211
|
+
def serialize
|
212
|
+
{
|
213
|
+
health_output: Health_Output.new,
|
214
|
+
messages: []
|
215
|
+
}
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
data/lib/newque/util.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
module Newque
|
2
|
+
|
3
|
+
class Util
|
4
|
+
|
5
|
+
def self.compute_options base_options, options
|
6
|
+
Hash[
|
7
|
+
base_options.map do |name, default_value|
|
8
|
+
[name, (options[name].nil? ? default_value : options[name])]
|
9
|
+
end
|
10
|
+
]
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.time_ms
|
14
|
+
(Time.now.to_f * 1000).to_i
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.newque_error errors
|
18
|
+
NewqueError.new "Client Error#{errors.size > 1 ? 's' : ''}: #{errors.join(', ')}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.zmq_error error
|
22
|
+
NewqueError.new "Network Error: #{error}"
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.wait_t
|
26
|
+
t = Thread.new do
|
27
|
+
while Thread.current.thread_variable_get(:result).nil?
|
28
|
+
sleep 0
|
29
|
+
end
|
30
|
+
Thread.current.thread_variable_get(:result)
|
31
|
+
end
|
32
|
+
t.priority = -1
|
33
|
+
t
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.resolve_t thread, result
|
37
|
+
thread.thread_variable_set(:result, result)
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|