iron_mq 3.1.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile.lock +10 -8
- data/README.md +278 -53
- data/iron_mq.gemspec +3 -2
- data/lib/iron_mq/client.rb +32 -9
- data/lib/iron_mq/messages.rb +21 -104
- data/lib/iron_mq/queues.rb +149 -142
- data/lib/iron_mq/response.rb +16 -15
- data/lib/iron_mq/subscribers.rb +12 -21
- data/lib/iron_mq/version.rb +1 -1
- data/test/quick_run.rb +69 -39
- data/test/test_base.rb +13 -5
- data/test/test_beanstalkd.rb +26 -4
- data/test/test_bulk.rb +7 -3
- data/test/test_iron_mq.rb +370 -159
- data/test/test_push_queues.rb +21 -7
- metadata +41 -45
data/lib/iron_mq/queues.rb
CHANGED
@@ -2,210 +2,217 @@ require 'cgi'
|
|
2
2
|
require 'iron_mq/subscribers'
|
3
3
|
|
4
4
|
module IronMQ
|
5
|
-
class Queues
|
6
5
|
|
7
|
-
|
6
|
+
class Queue < ResponseBase
|
7
|
+
attr_reader :name
|
8
8
|
|
9
|
-
def initialize(client)
|
9
|
+
def initialize(client, queue_name)
|
10
10
|
@client = client
|
11
|
+
@name = queue_name
|
12
|
+
super({"name" => queue_name})
|
11
13
|
end
|
12
14
|
|
13
|
-
def
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
15
|
+
def info
|
16
|
+
info = raw
|
17
|
+
begin
|
18
|
+
# Do not instantiate response
|
19
|
+
info = call_api_and_parse_response(:get, '', {}, false)
|
20
|
+
rescue Rest::HttpError
|
18
21
|
end
|
19
|
-
path
|
20
|
-
end
|
21
22
|
|
22
|
-
|
23
|
-
options[:project_id] = @client.project_id
|
24
|
-
Queues.path(options)
|
23
|
+
ResponseBase.new(info)
|
25
24
|
end
|
26
25
|
|
27
|
-
def
|
28
|
-
|
29
|
-
r1 = @client.get("#{path(options)}", options)
|
30
|
-
#p r1
|
31
|
-
res = @client.parse_response(r1)
|
32
|
-
res.each do |q|
|
33
|
-
#p q
|
34
|
-
q = Queue.new(@client, q)
|
35
|
-
ret << q
|
36
|
-
end
|
37
|
-
ret
|
26
|
+
def size
|
27
|
+
info.size.to_i
|
38
28
|
end
|
39
29
|
|
40
|
-
def
|
41
|
-
|
42
|
-
r1 = @client.post("#{path(options)}/clear", options)
|
43
|
-
@client.logger.debug "Clear result: #{r1}"
|
44
|
-
r1
|
30
|
+
def total_messages
|
31
|
+
info.total_messages.to_i
|
45
32
|
end
|
46
33
|
|
47
|
-
def
|
48
|
-
|
49
|
-
r1 = @client.delete("#{path(options)}", options)
|
50
|
-
@client.logger.debug "Delete result: #{r1}"
|
51
|
-
r1
|
34
|
+
def new?
|
35
|
+
id.nil?
|
52
36
|
end
|
53
37
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
#puts "HEADERS"
|
60
|
-
#p r.headers
|
61
|
-
res = @client.parse_response(r)
|
62
|
-
return Queue.new(@client, res)
|
38
|
+
def push_queue?
|
39
|
+
# FIXME: `push_type` parameter in not guaranted it's push queue.
|
40
|
+
# When the parameter absent it is not guaranted that queue is not push queue.
|
41
|
+
ptype = push_type
|
42
|
+
not (ptype.nil? || ptype.empty?)
|
63
43
|
end
|
64
44
|
|
65
|
-
|
66
|
-
|
67
|
-
# :name => if not specified, will use global queue name
|
68
|
-
# :subscribers => url's to subscribe to
|
69
|
-
# :push_type => multicast (default) or unicast.
|
70
|
-
def post(options={})
|
71
|
-
options[:name] ||= @client.queue_name
|
72
|
-
res = @client.parse_response(@client.post(path(options), options))
|
73
|
-
#p res
|
74
|
-
res
|
45
|
+
def update(options)
|
46
|
+
call_api_and_parse_response(:post, "", options)
|
75
47
|
end
|
76
48
|
|
49
|
+
alias_method :update_queue, :update
|
77
50
|
|
78
|
-
|
79
|
-
|
80
|
-
class Queue
|
81
|
-
|
82
|
-
attr_reader :client
|
83
|
-
|
84
|
-
def initialize(client, res)
|
85
|
-
@client = client
|
86
|
-
@data = res
|
51
|
+
def clear
|
52
|
+
call_api_and_parse_response(:post, "/clear")
|
87
53
|
end
|
88
54
|
|
89
|
-
|
90
|
-
@data
|
91
|
-
end
|
55
|
+
alias_method :clear_queue, :clear
|
92
56
|
|
93
|
-
|
94
|
-
|
57
|
+
# Backward compatibility, better name is `delete`
|
58
|
+
def delete_queue
|
59
|
+
call_api_and_parse_response(:delete)
|
95
60
|
end
|
96
61
|
|
97
|
-
|
98
|
-
|
62
|
+
# Backward compatibility
|
63
|
+
def delete(message_id, options = {})
|
64
|
+
# API does not accept any options
|
65
|
+
Message.new(self, {"id" => message_id}).delete
|
99
66
|
end
|
100
67
|
|
101
|
-
def
|
102
|
-
|
68
|
+
def add_subscribers(subscribers)
|
69
|
+
call_api_and_parse_response(:post, "/subscribers", :subscribers => subscribers)
|
103
70
|
end
|
104
71
|
|
105
|
-
|
106
|
-
|
72
|
+
# `options` for backward compatibility
|
73
|
+
def add_subscriber(subscriber, options = {})
|
74
|
+
add_subscribers([subscriber])
|
107
75
|
end
|
108
76
|
|
109
|
-
def
|
110
|
-
|
77
|
+
def remove_subscribers(subscribers)
|
78
|
+
call_api_and_parse_response(:delete,
|
79
|
+
"/subscribers",
|
80
|
+
{
|
81
|
+
:subscribers => subscribers,
|
82
|
+
:headers => {"Content-Type" => @client.content_type}
|
83
|
+
})
|
111
84
|
end
|
112
85
|
|
113
|
-
|
114
|
-
|
115
|
-
return if @loaded && !options[:force]
|
116
|
-
q = @client.queues.get(:name => name)
|
117
|
-
@client.logger.debug "GOT Q: " + q.inspect
|
118
|
-
@data = q.raw
|
119
|
-
@loaded = true
|
120
|
-
q
|
86
|
+
def remove_subscriber(subscriber)
|
87
|
+
remove_subscribers([subscriber])
|
121
88
|
end
|
122
89
|
|
123
|
-
def
|
124
|
-
|
125
|
-
end
|
90
|
+
def post_messages(payload, options = {})
|
91
|
+
batch = false
|
126
92
|
|
127
|
-
|
128
|
-
|
129
|
-
|
93
|
+
msgs = if payload.is_a?(Array)
|
94
|
+
batch = true
|
95
|
+
# FIXME: This maybe better to process Array of Objects the same way as for single message.
|
96
|
+
#
|
97
|
+
# payload.each_with_object([]) do |msg, res|
|
98
|
+
# res << options.merge(:body => msg)
|
99
|
+
# end
|
100
|
+
#
|
101
|
+
# For now user must pass objects like `[{:body => msg1}, {:body => msg2}]`
|
102
|
+
payload.each_with_object([]) do |msg, res|
|
103
|
+
res << msg.merge(options)
|
104
|
+
end
|
105
|
+
else
|
106
|
+
[ options.merge(:body => payload) ]
|
107
|
+
end
|
130
108
|
|
131
|
-
|
132
|
-
|
133
|
-
# :push_type => multicast (default) or unicast.
|
134
|
-
def update_queue(options)
|
135
|
-
@client.queues.post(options.merge(:name => name))
|
136
|
-
end
|
109
|
+
# Do not instantiate response
|
110
|
+
res = call_api_and_parse_response(:post, "/messages", {:messages => msgs}, false)
|
137
111
|
|
138
|
-
|
139
|
-
|
140
|
-
|
112
|
+
if batch
|
113
|
+
# FIXME: Return Array of ResponsBase instead, it seems more clear than raw response
|
114
|
+
#
|
115
|
+
# res["ids"].each_with_object([]) do |id, responses|
|
116
|
+
# responses << ResponseBase.new({"id" => id, "msg" => res["msg"]})
|
117
|
+
# end
|
118
|
+
ResponseBase.new(res) # Backward capable
|
119
|
+
else
|
120
|
+
ResponseBase.new({"id" => res["ids"][0], "msg" => res["msg"]})
|
121
|
+
end
|
141
122
|
end
|
142
123
|
|
143
|
-
|
144
|
-
load_queue()
|
145
|
-
return raw["total_messages"]
|
146
|
-
end
|
124
|
+
alias_method :post, :post_messages
|
147
125
|
|
148
|
-
def
|
149
|
-
|
150
|
-
|
151
|
-
|
126
|
+
def get_messages(options = {})
|
127
|
+
if options.is_a?(String)
|
128
|
+
# assume it's an id
|
129
|
+
return Message.new(self, {"id" => options})
|
130
|
+
end
|
152
131
|
|
153
|
-
|
154
|
-
res = @client.post("#{@client.queues.path(:name => name)}/subscribers", :subscribers => [subscriber_hash])
|
155
|
-
res = @client.parse_response(res)
|
156
|
-
#p res
|
157
|
-
res
|
158
|
-
end
|
132
|
+
resp = call_api_and_parse_response(:get, "/messages", options)
|
159
133
|
|
160
|
-
|
161
|
-
res = @client.delete("#{@client.queues.path(:name => name)}/subscribers", {:subscribers => [subscriber_hash]}, {"Content-Type"=>@client.content_type})
|
162
|
-
res = @client.parse_response(res)
|
163
|
-
#p res
|
164
|
-
res
|
134
|
+
process_messages(resp["messages"], options)
|
165
135
|
end
|
166
136
|
|
167
|
-
|
168
|
-
@client.messages.post(body, options.merge(:queue_name => name))
|
169
|
-
end
|
137
|
+
alias_method :get, :get_messages
|
170
138
|
|
171
|
-
|
172
|
-
|
173
|
-
|
139
|
+
# Backward compatibility
|
140
|
+
def messages ; self; end
|
141
|
+
|
142
|
+
def peek_messages(options = {})
|
143
|
+
resp = call_api_and_parse_response(:get, "/messages/peek", options)
|
174
144
|
|
175
|
-
|
176
|
-
@client.messages.delete(id, options.merge(:queue_name => name))
|
145
|
+
process_messages(resp["messages"], options)
|
177
146
|
end
|
178
147
|
|
148
|
+
alias_method :peek, :peek_messages
|
179
149
|
|
180
|
-
|
181
|
-
#
|
182
|
-
# queue.poll { |msg| puts msg.body }
|
183
|
-
#
|
184
|
-
# options:
|
185
|
-
# - :sleep_duration=>seconds => time between polls if msg is nil. default 1.
|
186
|
-
# - :break_if_nil=>true/false => if true, will break if msg is nil (ie: queue is empty)
|
187
|
-
def poll(options={}, &blk)
|
150
|
+
def poll_messages(options = {}, &block)
|
188
151
|
sleep_duration = options[:sleep_duration] || 1
|
152
|
+
|
189
153
|
while true
|
190
|
-
|
191
|
-
msg = get(options)
|
154
|
+
msg = get_messages(options.merge(:n => 1))
|
192
155
|
if msg.nil?
|
193
|
-
|
194
|
-
break
|
195
|
-
else
|
196
|
-
sleep sleep_duration
|
197
|
-
end
|
156
|
+
options[:break_if_nil] ? break : sleep(sleep_duration)
|
198
157
|
else
|
199
158
|
yield msg
|
159
|
+
# Delete message after processing
|
200
160
|
msg.delete
|
201
161
|
end
|
202
162
|
end
|
203
163
|
end
|
204
164
|
|
205
|
-
|
206
|
-
|
165
|
+
alias_method :poll, :poll_messages
|
166
|
+
|
167
|
+
def call_api_and_parse_response(meth, ext_path = "", options = {}, instantiate = true)
|
168
|
+
response = if meth.to_s == "delete"
|
169
|
+
headers = options.delete(:headers) || options.delete("headers") || {}
|
170
|
+
|
171
|
+
@client.parse_response(@client.send(meth, "#{path(ext_path)}", options, headers))
|
172
|
+
else
|
173
|
+
@client.parse_response(@client.send(meth, "#{path(ext_path)}", options))
|
174
|
+
end
|
175
|
+
instantiate ? ResponseBase.new(response) : response
|
176
|
+
end
|
177
|
+
|
178
|
+
def method_missing(meth, *args)
|
179
|
+
# This is for reload queue info data when calling:
|
180
|
+
# queue.id
|
181
|
+
# queue.size
|
182
|
+
# etc.
|
183
|
+
if args.length == 0
|
184
|
+
res = info.send(meth)
|
185
|
+
res ? res : super
|
186
|
+
else
|
187
|
+
super
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
private
|
192
|
+
|
193
|
+
def path(ext_path)
|
194
|
+
"/#{CGI::escape(@name).gsub('+', '%20')}#{ext_path}"
|
195
|
+
end
|
196
|
+
|
197
|
+
def process_messages(messages, options)
|
198
|
+
multiple = wait_for_multiple?(options)
|
199
|
+
|
200
|
+
if messages.is_a?(Array) && messages.size > 0
|
201
|
+
if multiple
|
202
|
+
messages.each_with_object([]) do |m, msgs|
|
203
|
+
msgs << Message.new(self, m)
|
204
|
+
end
|
205
|
+
else
|
206
|
+
Message.new(self, messages[0])
|
207
|
+
end
|
208
|
+
else
|
209
|
+
multiple ? [] : nil
|
210
|
+
end
|
207
211
|
end
|
208
212
|
|
213
|
+
def wait_for_multiple?(options)
|
214
|
+
options.is_a?(Hash) && ((options[:n] || options['n']).to_i > 1)
|
215
|
+
end
|
209
216
|
end
|
210
217
|
|
211
218
|
end
|
data/lib/iron_mq/response.rb
CHANGED
@@ -1,28 +1,29 @@
|
|
1
|
+
require 'ostruct'
|
1
2
|
|
2
3
|
module IronMQ
|
3
4
|
|
4
|
-
class ResponseBase
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
def initialize(raw, code=200)
|
9
|
-
@raw = raw
|
10
|
-
@code = code
|
5
|
+
class ResponseBase < OpenStruct
|
6
|
+
def initialize(data, code = 200)
|
7
|
+
super(data.merge(:code => code.to_i))
|
11
8
|
end
|
12
9
|
|
13
10
|
def [](key)
|
14
|
-
|
11
|
+
send(key.to_s)
|
15
12
|
end
|
16
13
|
|
17
|
-
def
|
18
|
-
|
19
|
-
|
14
|
+
def raw
|
15
|
+
res = stringify_keys(marshal_dump)
|
16
|
+
# `code` is not part of response body
|
17
|
+
res.delete("code")
|
20
18
|
|
21
|
-
|
22
|
-
raw["msg"]
|
19
|
+
res
|
23
20
|
end
|
24
21
|
|
25
|
-
|
22
|
+
private
|
26
23
|
|
24
|
+
def stringify_keys(hash)
|
25
|
+
hash.keys.each_with_object({}) { |k, res| res[k.to_s] = hash[k] }
|
26
|
+
end
|
27
|
+
end
|
27
28
|
|
28
|
-
end
|
29
|
+
end
|
data/lib/iron_mq/subscribers.rb
CHANGED
@@ -1,34 +1,25 @@
|
|
1
1
|
module IronMQ
|
2
|
-
class Subscribers
|
3
|
-
def self.path(options)
|
4
|
-
path = "#{Messages.path(options)}/subscribers"
|
5
|
-
if options[:subscriber_id]
|
6
|
-
path << "/#{options[:subscriber_id]}"
|
7
|
-
end
|
8
|
-
path
|
9
|
-
end
|
10
|
-
end
|
11
2
|
|
12
3
|
class Subscriber < ResponseBase
|
4
|
+
# `options` was kept for backward compatibility
|
13
5
|
attr_accessor :options
|
14
6
|
|
15
|
-
def initialize(
|
16
|
-
super(
|
7
|
+
def initialize(data, message, options = {})
|
8
|
+
super(data, 200)
|
17
9
|
@message = message
|
18
10
|
@options = options
|
19
11
|
end
|
20
12
|
|
21
|
-
|
22
|
-
|
13
|
+
# `options` was kept for backward compatibility
|
14
|
+
def delete(options = {})
|
15
|
+
@message.call_api_and_parse_response(:delete, path)
|
16
|
+
end
|
23
17
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
path = Subscribers.path(options)
|
29
|
-
raw = client.delete(path)
|
30
|
-
res = client.parse_response(raw)
|
31
|
-
return ResponseBase.new(res)
|
18
|
+
private
|
19
|
+
|
20
|
+
def path
|
21
|
+
"/subscribers/#{id}"
|
32
22
|
end
|
33
23
|
end
|
24
|
+
|
34
25
|
end
|
data/lib/iron_mq/version.rb
CHANGED
data/test/quick_run.rb
CHANGED
@@ -1,56 +1,86 @@
|
|
1
|
-
require '
|
1
|
+
require 'quicky'
|
2
|
+
require File.expand_path('test_base.rb', File.dirname(__FILE__))
|
2
3
|
|
3
|
-
TIMES =
|
4
|
+
TIMES = 10
|
4
5
|
|
5
6
|
class QuickRun < TestBase
|
6
7
|
|
7
8
|
def setup
|
8
9
|
super
|
9
|
-
@client.queue_name = 'ironmq-gem_quick'
|
10
|
-
clear_queue
|
11
10
|
end
|
12
11
|
|
13
|
-
def
|
12
|
+
def test_quick
|
13
|
+
queue_name = 'ironmq-gem_quick'
|
14
|
+
clear_queue(queue_name)
|
15
|
+
queue = @client.queue(queue_name)
|
14
16
|
|
15
|
-
|
17
|
+
quicky = Quicky::Timer.new
|
18
|
+
|
19
|
+
# make connection
|
20
|
+
res2 = queue.get
|
21
|
+
# p res2
|
22
|
+
|
23
|
+
quicky.loop(:test_quick, TIMES, :warmup => 2) do |i|
|
16
24
|
puts "==== LOOP #{i} =================================="
|
17
25
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
p
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
26
|
+
post_id = nil
|
27
|
+
quicky.time(:post, :warmup => 2) do
|
28
|
+
res = queue.post("hello world!")
|
29
|
+
# p res
|
30
|
+
assert_not_nil res
|
31
|
+
assert_not_nil res.id
|
32
|
+
post_id = res.id
|
33
|
+
assert !(res.msg.nil? || res.msg.empty?)
|
34
|
+
end
|
35
|
+
|
36
|
+
quicky.time(:get, :warmup => 2) do
|
37
|
+
msg = queue.get
|
38
|
+
# p res
|
39
|
+
puts "post_id=" + post_id.inspect
|
40
|
+
assert_not_nil msg.id
|
41
|
+
assert_equal msg.id, post_id
|
42
|
+
assert !(msg.body.nil? || msg.body.empty?)
|
43
|
+
end
|
44
|
+
|
45
|
+
quicky.time(:delete, :warmup => 2) do
|
46
|
+
res = queue.delete(post_id)
|
47
|
+
# p res
|
48
|
+
assert_not_nil res
|
49
|
+
assert !(res.msg.nil? || res.msg.empty?)
|
50
|
+
end
|
51
|
+
|
52
|
+
msg = queue.get
|
53
|
+
# p msg
|
54
|
+
assert_nil res
|
55
|
+
|
56
|
+
|
57
|
+
q = @client.queue('test2')
|
58
|
+
res = q.post("hello world!")
|
59
|
+
# p res
|
60
|
+
assert_not_nil res.id
|
61
|
+
assert_not_nil res.msg
|
62
|
+
|
63
|
+
msg = q.get
|
64
|
+
# p res
|
65
|
+
assert_not_nil msg
|
66
|
+
assert_not_nil msg.id
|
67
|
+
assert_not_nil msg.body
|
68
|
+
|
69
|
+
res = msg.delete
|
70
|
+
# p res
|
71
|
+
assert_equal 200, res.code, "API must delete message and response with HTTP 200 status, but returned HTTP #{res.code}"
|
51
72
|
end
|
73
|
+
puts "count: #{quicky.results(:post).count}"
|
74
|
+
puts "avg post: #{quicky.results(:post).duration}"
|
75
|
+
puts "avg get: #{quicky.results(:get).duration}"
|
76
|
+
puts "avg delete: #{quicky.results(:delete).duration}"
|
52
77
|
|
78
|
+
# delete queue on test complete
|
79
|
+
resp = queue.delete_queue
|
80
|
+
assert_equal 200, resp.code, "API must response with HTTP 200 status, but returned HTTP #{resp.code}"
|
53
81
|
|
82
|
+
resp = @client.queue('test2').delete_queue
|
83
|
+
assert_equal 200, resp.code, "API must response with HTTP 200 status, but returned HTTP #{resp.code}"
|
54
84
|
end
|
55
85
|
|
56
86
|
|
data/test/test_base.rb
CHANGED
@@ -30,26 +30,34 @@ class TestBase < Test::Unit::TestCase
|
|
30
30
|
|
31
31
|
config = @config['iron']
|
32
32
|
@host = "#{config['host'] || "mq-aws-us-east-1.iron.io"}"
|
33
|
+
|
33
34
|
@client = IronMQ::Client.new(@config['iron'])
|
35
|
+
|
34
36
|
Rest.logger.level = Logger::DEBUG # this doesn't work for some reason?
|
35
37
|
IronCore::Logger.logger.level = Logger::DEBUG
|
36
|
-
@client.queue_name = 'ironmq-ruby-tests'
|
37
38
|
|
39
|
+
@queue_name = 'ironmq-ruby-tests' # default queue for tests
|
38
40
|
end
|
39
41
|
|
40
42
|
|
41
43
|
def clear_queue(queue_name=nil)
|
42
|
-
queue_name ||= @
|
44
|
+
queue_name ||= @queue_name
|
45
|
+
|
46
|
+
queue = @client.queue(queue_name)
|
47
|
+
queue.post("test")
|
48
|
+
|
43
49
|
puts "clearing queue #{queue_name}"
|
44
|
-
|
45
|
-
@client.queue(queue_name).clear
|
50
|
+
queue.clear
|
46
51
|
puts 'cleared.'
|
47
52
|
end
|
48
53
|
|
49
54
|
def assert_performance(time)
|
50
55
|
start_time = Time.now
|
56
|
+
|
51
57
|
yield
|
52
|
-
|
58
|
+
|
59
|
+
execution_time = Time.now - start_time
|
60
|
+
|
53
61
|
assert execution_time < time, "Execution time too big #{execution_time.round(2)}, should be #{time}"
|
54
62
|
end
|
55
63
|
|