newque 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +54 -0
- data/Gemfile +3 -0
- data/LICENSE +373 -0
- data/README.md +340 -0
- data/Rakefile +5 -0
- data/conf/channels/example.json +12 -0
- data/conf/channels/example_fifo.json +18 -0
- data/conf/channels/example_plaintext.json +15 -0
- data/conf/channels/example_pubsub.json +15 -0
- data/conf/newque.json +26 -0
- data/lib/newque.rb +16 -0
- data/lib/newque/clients/client.rb +23 -0
- data/lib/newque/clients/fifo_client.rb +109 -0
- data/lib/newque/clients/pubsub_client.rb +101 -0
- data/lib/newque/errors.rb +6 -0
- data/lib/newque/future.rb +33 -0
- data/lib/newque/http/http_json.rb +49 -0
- data/lib/newque/http/http_plaintext.rb +66 -0
- data/lib/newque/http/newque_http.rb +116 -0
- data/lib/newque/responses.rb +219 -0
- data/lib/newque/util.rb +42 -0
- data/lib/newque/version.rb +3 -0
- data/lib/newque/zmq/newque_zmq.rb +121 -0
- data/lib/newque/zmq/protobuf.rb +96 -0
- data/lib/newque/zmq/zmq_tools.rb +52 -0
- data/newque.gemspec +35 -0
- data/spec/client_spec.rb +124 -0
- data/spec/fifo_client_spec.rb +130 -0
- data/spec/helpers.rb +12 -0
- data/spec/pubsub_client_spec.rb +142 -0
- metadata +164 -0
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
|
3
|
+
module Newque
|
4
|
+
|
5
|
+
class Http_plaintext
|
6
|
+
def initialize http
|
7
|
+
@http = http
|
8
|
+
end
|
9
|
+
|
10
|
+
def write channel, atomic, msgs, ids=nil
|
11
|
+
head, *tail = msgs
|
12
|
+
raise NewqueError.new("No messages given") if head.nil?
|
13
|
+
|
14
|
+
stream = StringIO.new
|
15
|
+
stream.write head
|
16
|
+
if msgs.size > 1
|
17
|
+
tail.each do |msg|
|
18
|
+
stream.write @http.options[:separator]
|
19
|
+
stream.write msg
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
thread = Thread.new do
|
24
|
+
res = @http.conn.post do |req|
|
25
|
+
@http.send :set_req_options, req
|
26
|
+
req.url "/v1/#{channel}"
|
27
|
+
req.body = stream.string
|
28
|
+
|
29
|
+
req.headers['newque-mode'] = if atomic then 'atomic'
|
30
|
+
elsif msgs.size == 1 then 'single'
|
31
|
+
else 'multiple'
|
32
|
+
end
|
33
|
+
req.headers['newque-msg-id'] = ids unless ids.nil?
|
34
|
+
end
|
35
|
+
|
36
|
+
parsed = @http.send :parse_json_response, res.body
|
37
|
+
Write_response.new parsed['saved']
|
38
|
+
end
|
39
|
+
Future.new thread, @http.timeout
|
40
|
+
end
|
41
|
+
|
42
|
+
def read channel, mode, limit=nil
|
43
|
+
thread = Thread.new do
|
44
|
+
res = @http.conn.get do |req|
|
45
|
+
@http.send :set_req_options, req
|
46
|
+
req.url "/v1/#{channel}"
|
47
|
+
req.headers['newque-mode'] = mode
|
48
|
+
req.headers['newque-read-max'] = limit unless limit.nil?
|
49
|
+
end
|
50
|
+
if res.status == 200
|
51
|
+
Read_response.new(
|
52
|
+
res.headers['newque-response-length'].to_i,
|
53
|
+
res.headers['newque-response-last-id'],
|
54
|
+
res.headers['newque-response-last-ts'].to_i,
|
55
|
+
res.body.split(@http.options[:separator])
|
56
|
+
)
|
57
|
+
else
|
58
|
+
@http.send :parse_json_response, res.body
|
59
|
+
end
|
60
|
+
end
|
61
|
+
Future.new thread, @http.timeout
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Newque
|
5
|
+
|
6
|
+
class Newque_http
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
BASE_OPTIONS = {
|
10
|
+
https: false,
|
11
|
+
http_format: :json,
|
12
|
+
separator: "\n"
|
13
|
+
}
|
14
|
+
|
15
|
+
def_delegators :@instance, :write, :read
|
16
|
+
attr_reader :conn, :options, :timeout
|
17
|
+
|
18
|
+
def initialize host, port, options, timeout
|
19
|
+
@host = host
|
20
|
+
@port = port
|
21
|
+
@options = Util.compute_options BASE_OPTIONS, options
|
22
|
+
@timeout = timeout / 1000.0
|
23
|
+
|
24
|
+
@conn = Faraday.new({ url: "#{@options[:https] ? "https" : "http"}://#{host}:#{port}" })
|
25
|
+
|
26
|
+
@instance = if @options[:http_format] == :json
|
27
|
+
Http_json.new self
|
28
|
+
elsif @options[:http_format] == :plaintext
|
29
|
+
Http_plaintext.new self
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def read_stream channel, mode, limit=nil
|
34
|
+
Enumerator.new do |generator|
|
35
|
+
Net::HTTP.start(@host, @port) do |http|
|
36
|
+
req = Net::HTTP::Get.new "/v1/#{channel}"
|
37
|
+
req['newque-mode'] = mode
|
38
|
+
req['newque-read-max'] = limit unless limit.nil?
|
39
|
+
req['Transfer-Encoding'] = 'chunked'
|
40
|
+
http.open_timeout = @timeout
|
41
|
+
http.read_timeout = @timeout
|
42
|
+
|
43
|
+
http.request req do |res|
|
44
|
+
if res.code != '200'
|
45
|
+
generator << parse_json_response(res.body) # Will throw
|
46
|
+
else
|
47
|
+
leftover = nil
|
48
|
+
res.read_body do |chunk|
|
49
|
+
lines = "#{leftover || ''}#{chunk}".split(@options[:separator], -1)
|
50
|
+
*fulls, partial = lines
|
51
|
+
fulls.each do |msg|
|
52
|
+
generator << msg
|
53
|
+
end
|
54
|
+
leftover = partial
|
55
|
+
end
|
56
|
+
generator << leftover unless leftover.nil?
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def count channel
|
64
|
+
thread = Thread.new do
|
65
|
+
res = @conn.get do |req|
|
66
|
+
set_req_options req
|
67
|
+
req.url "/v1/#{channel}/count"
|
68
|
+
end
|
69
|
+
parsed = parse_json_response res.body
|
70
|
+
Count_response.new parsed['count']
|
71
|
+
end
|
72
|
+
Future.new thread, @timeout
|
73
|
+
end
|
74
|
+
|
75
|
+
def delete channel
|
76
|
+
thread = Thread.new do
|
77
|
+
res = @conn.delete do |req|
|
78
|
+
set_req_options req
|
79
|
+
req.url "/v1/#{channel}"
|
80
|
+
end
|
81
|
+
parsed = parse_json_response res.body
|
82
|
+
Delete_response.new
|
83
|
+
end
|
84
|
+
Future.new thread, @timeout
|
85
|
+
end
|
86
|
+
|
87
|
+
def health channel, global=false
|
88
|
+
thread = Thread.new do
|
89
|
+
res = @conn.get do |req|
|
90
|
+
set_req_options req
|
91
|
+
req.url "/v1#{global ? '' : '/' + channel}/health"
|
92
|
+
end
|
93
|
+
parsed = parse_json_response res.body
|
94
|
+
Health_response.new
|
95
|
+
end
|
96
|
+
Future.new thread, @timeout
|
97
|
+
end
|
98
|
+
|
99
|
+
private
|
100
|
+
|
101
|
+
def set_req_options req
|
102
|
+
req.options.open_timeout = @timeout
|
103
|
+
req.options.timeout = @timeout
|
104
|
+
end
|
105
|
+
|
106
|
+
def parse_json_response body
|
107
|
+
parsed = JSON.parse body
|
108
|
+
if parsed['errors'].size > 0
|
109
|
+
raise Util.newque_error parsed['errors']
|
110
|
+
end
|
111
|
+
parsed
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
@@ -0,0 +1,219 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Newque
|
4
|
+
|
5
|
+
# ------------------------------------
|
6
|
+
# REQUESTS
|
7
|
+
# ------------------------------------
|
8
|
+
class Write_request
|
9
|
+
attr_reader :atomic, :ids
|
10
|
+
|
11
|
+
def initialize atomic, ids
|
12
|
+
@atomic = atomic
|
13
|
+
@ids = ids
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_s
|
17
|
+
"<Write atomic: #{@atomic.to_json} ids: #{@ids.to_json}>"
|
18
|
+
end
|
19
|
+
|
20
|
+
def inspect
|
21
|
+
to_s
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class Read_request
|
26
|
+
attr_reader :mode, :limit
|
27
|
+
|
28
|
+
def initialize mode, limit
|
29
|
+
@mode = mode
|
30
|
+
@limit = limit
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_s
|
34
|
+
"<Read mode: #{@mode.to_json} limit: #{@limit.to_json}>"
|
35
|
+
end
|
36
|
+
|
37
|
+
def inspect
|
38
|
+
to_s
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class Count_request
|
43
|
+
def initialize
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_s
|
47
|
+
"<Count >"
|
48
|
+
end
|
49
|
+
|
50
|
+
def inspect
|
51
|
+
to_s
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class Delete_request
|
56
|
+
def initialize
|
57
|
+
end
|
58
|
+
|
59
|
+
def to_s
|
60
|
+
"<Delete >"
|
61
|
+
end
|
62
|
+
|
63
|
+
def inspect
|
64
|
+
to_s
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class Health_request
|
69
|
+
attr_reader :global
|
70
|
+
|
71
|
+
def initialize global
|
72
|
+
@global = global
|
73
|
+
end
|
74
|
+
|
75
|
+
def to_s
|
76
|
+
"<Health global: #{@global.to_json}>"
|
77
|
+
end
|
78
|
+
|
79
|
+
def inspect
|
80
|
+
to_s
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
class Input_request
|
85
|
+
attr_reader :channel, :action, :messages
|
86
|
+
|
87
|
+
def initialize channel, action, messages
|
88
|
+
@channel = channel
|
89
|
+
@action = action
|
90
|
+
@messages = messages
|
91
|
+
end
|
92
|
+
|
93
|
+
def to_s
|
94
|
+
"<Newque_input channel: #{channel.to_json} action: #{@action} #{@messages.size > 0 ? 'messages: ' + @messages.to_json : ''}>"
|
95
|
+
end
|
96
|
+
|
97
|
+
def inspect
|
98
|
+
to_s
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# ------------------------------------
|
103
|
+
# RESPONSES
|
104
|
+
# ------------------------------------
|
105
|
+
class Write_response
|
106
|
+
attr_reader :saved
|
107
|
+
|
108
|
+
def initialize saved
|
109
|
+
@saved = saved
|
110
|
+
end
|
111
|
+
|
112
|
+
def to_s
|
113
|
+
"<Newque_write saved: #{saved.to_json}>"
|
114
|
+
end
|
115
|
+
|
116
|
+
def inspect
|
117
|
+
to_s
|
118
|
+
end
|
119
|
+
|
120
|
+
def serialize
|
121
|
+
{
|
122
|
+
write_output: Write_Output.new(saved: @saved),
|
123
|
+
messages: []
|
124
|
+
}
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
class Read_response
|
129
|
+
attr_reader :length, :last_id, :last_timens, :messages
|
130
|
+
|
131
|
+
def initialize length, last_id, last_timens, messages
|
132
|
+
@length = length
|
133
|
+
@last_id = last_id
|
134
|
+
@last_timens = last_timens
|
135
|
+
@messages = messages
|
136
|
+
end
|
137
|
+
|
138
|
+
def to_s
|
139
|
+
"<Newque_read length: #{length.to_json} last_id: #{last_id.to_json} last_timens: #{last_timens.to_json} messages: #{messages}>"
|
140
|
+
end
|
141
|
+
|
142
|
+
def inspect
|
143
|
+
to_s
|
144
|
+
end
|
145
|
+
|
146
|
+
def serialize
|
147
|
+
{
|
148
|
+
read_output: Read_Output.new(length: @length, last_id: @last_id, last_timens: @last_timens),
|
149
|
+
messages: @messages
|
150
|
+
}
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
class Count_response
|
155
|
+
attr_reader :count
|
156
|
+
|
157
|
+
def initialize count
|
158
|
+
@count = count
|
159
|
+
end
|
160
|
+
|
161
|
+
def to_s
|
162
|
+
"<Newque_count count: #{count.to_json}>"
|
163
|
+
end
|
164
|
+
|
165
|
+
def inspect
|
166
|
+
to_s
|
167
|
+
end
|
168
|
+
|
169
|
+
def serialize
|
170
|
+
{
|
171
|
+
count_output: Count_Output.new(count: @count),
|
172
|
+
messages: []
|
173
|
+
}
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
class Delete_response
|
178
|
+
|
179
|
+
def initialize
|
180
|
+
end
|
181
|
+
|
182
|
+
def to_s
|
183
|
+
"<Newque_delete >"
|
184
|
+
end
|
185
|
+
|
186
|
+
def inspect
|
187
|
+
to_s
|
188
|
+
end
|
189
|
+
|
190
|
+
def serialize
|
191
|
+
{
|
192
|
+
delete_output: Delete_Output.new,
|
193
|
+
messages: []
|
194
|
+
}
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
class Health_response
|
199
|
+
|
200
|
+
def initialize
|
201
|
+
end
|
202
|
+
|
203
|
+
def to_s
|
204
|
+
"<Newque_health >"
|
205
|
+
end
|
206
|
+
|
207
|
+
def inspect
|
208
|
+
to_s
|
209
|
+
end
|
210
|
+
|
211
|
+
def serialize
|
212
|
+
{
|
213
|
+
health_output: Health_Output.new,
|
214
|
+
messages: []
|
215
|
+
}
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
data/lib/newque/util.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
module Newque
|
2
|
+
|
3
|
+
class Util
|
4
|
+
|
5
|
+
def self.compute_options base_options, options
|
6
|
+
Hash[
|
7
|
+
base_options.map do |name, default_value|
|
8
|
+
[name, (options[name].nil? ? default_value : options[name])]
|
9
|
+
end
|
10
|
+
]
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.time_ms
|
14
|
+
(Time.now.to_f * 1000).to_i
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.newque_error errors
|
18
|
+
NewqueError.new "Client Error#{errors.size > 1 ? 's' : ''}: #{errors.join(', ')}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.zmq_error error
|
22
|
+
NewqueError.new "Network Error: #{error}"
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.wait_t
|
26
|
+
t = Thread.new do
|
27
|
+
while Thread.current.thread_variable_get(:result).nil?
|
28
|
+
sleep 0
|
29
|
+
end
|
30
|
+
Thread.current.thread_variable_get(:result)
|
31
|
+
end
|
32
|
+
t.priority = -1
|
33
|
+
t
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.resolve_t thread, result
|
37
|
+
thread.thread_variable_set(:result, result)
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|