iron_mq 4.0.3 → 4.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/Gemfile.lock +23 -15
- data/README.md +20 -7
- data/iron_mq.gemspec +2 -2
- data/lib/iron_mq.rb +1 -0
- data/lib/iron_mq/alert.rb +53 -0
- data/lib/iron_mq/client.rb +2 -2
- data/lib/iron_mq/messages.rb +21 -1
- data/lib/iron_mq/queues.rb +85 -37
- data/lib/iron_mq/response.rb +26 -9
- data/lib/iron_mq/subscribers.rb +1 -1
- data/lib/iron_mq/version.rb +1 -2
- data/test/quick_run.rb +22 -39
- data/test/quick_run2.rb +114 -0
- data/test/test_alerts.rb +142 -0
- data/test/test_base.rb +13 -11
- data/test/test_beanstalkd.rb +1 -1
- data/test/test_iron_mq.rb +175 -141
- data/test/test_mq_worker_subscribers.rb +71 -0
- data/test/test_push_queues.rb +123 -3
- data/test/tmp.rb +14 -0
- metadata +29 -6
data/lib/iron_mq/response.rb
CHANGED
@@ -2,22 +2,39 @@ require 'ostruct'
|
|
2
2
|
|
3
3
|
module IronMQ
|
4
4
|
|
5
|
-
|
5
|
+
|
6
|
+
class ResponseBase
|
7
|
+
attr_reader :raw, :code
|
8
|
+
|
6
9
|
def initialize(data, code = 200)
|
7
|
-
|
10
|
+
@raw = data
|
11
|
+
@code = code
|
12
|
+
#super(data.merge(:code => code.to_i))
|
8
13
|
end
|
9
14
|
|
10
|
-
def
|
11
|
-
|
15
|
+
def id
|
16
|
+
@raw["id"]
|
12
17
|
end
|
13
18
|
|
14
|
-
def
|
15
|
-
|
16
|
-
|
17
|
-
res.delete("code")
|
19
|
+
def [](key)
|
20
|
+
@raw[key]
|
21
|
+
end
|
18
22
|
|
19
|
-
|
23
|
+
def msg
|
24
|
+
@raw["msg"]
|
20
25
|
end
|
26
|
+
#
|
27
|
+
#def raw
|
28
|
+
# if @raw.nil?
|
29
|
+
# @raw = call_api_and_parse_response(:get, "", {}, false)
|
30
|
+
# end
|
31
|
+
# #res = stringify_keys(marshal_dump)
|
32
|
+
# ## `code` is not part of response body
|
33
|
+
# #res.delete("code")
|
34
|
+
# #
|
35
|
+
# #res
|
36
|
+
# @raw
|
37
|
+
#end
|
21
38
|
|
22
39
|
private
|
23
40
|
|
data/lib/iron_mq/subscribers.rb
CHANGED
data/lib/iron_mq/version.rb
CHANGED
data/test/quick_run.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
require 'quicky'
|
2
|
+
require 'go'
|
2
3
|
require File.expand_path('test_base.rb', File.dirname(__FILE__))
|
3
4
|
|
4
|
-
|
5
|
+
TIMES_PER_THREAD = 10
|
5
6
|
|
6
7
|
class QuickRun < TestBase
|
7
8
|
|
@@ -10,22 +11,21 @@ class QuickRun < TestBase
|
|
10
11
|
end
|
11
12
|
|
12
13
|
def test_quick
|
13
|
-
|
14
|
+
|
15
|
+
queue_name = "ironmq_gem_quick_#{rand(100)}"
|
14
16
|
clear_queue(queue_name)
|
15
17
|
queue = @client.queue(queue_name)
|
16
18
|
|
17
19
|
quicky = Quicky::Timer.new
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
quicky.loop(:test_quick, TIMES, :warmup => 2) do |i|
|
24
|
-
puts "==== LOOP #{i} =================================="
|
25
|
-
|
20
|
+
j = 0
|
21
|
+
quicky.loop(:test_quick, TIMES_PER_THREAD) do |i|
|
22
|
+
puts "==== LOOP #{i || j} =================================="
|
23
|
+
j += 1
|
24
|
+
body = "hello world!"
|
26
25
|
post_id = nil
|
27
|
-
quicky.time(:post
|
28
|
-
|
26
|
+
quicky.time(:post) do
|
27
|
+
puts "post"
|
28
|
+
res = queue.post(body)
|
29
29
|
# p res
|
30
30
|
assert_not_nil res
|
31
31
|
assert_not_nil res.id
|
@@ -33,16 +33,17 @@ class QuickRun < TestBase
|
|
33
33
|
assert !(res.msg.nil? || res.msg.empty?)
|
34
34
|
end
|
35
35
|
|
36
|
-
quicky.time(:get
|
36
|
+
quicky.time(:get) do
|
37
|
+
puts "get"
|
37
38
|
msg = queue.get
|
38
|
-
# p res
|
39
|
-
puts "post_id=" + post_id.inspect
|
40
39
|
assert_not_nil msg.id
|
41
40
|
assert_equal msg.id, post_id
|
42
41
|
assert !(msg.body.nil? || msg.body.empty?)
|
42
|
+
assert_equal body, msg.body
|
43
43
|
end
|
44
44
|
|
45
|
-
quicky.time(:delete
|
45
|
+
quicky.time(:delete) do
|
46
|
+
puts "delete"
|
46
47
|
res = queue.delete(post_id)
|
47
48
|
# p res
|
48
49
|
assert_not_nil res
|
@@ -51,38 +52,20 @@ class QuickRun < TestBase
|
|
51
52
|
|
52
53
|
msg = queue.get
|
53
54
|
# p msg
|
54
|
-
assert_nil
|
55
|
-
|
55
|
+
assert_nil msg
|
56
56
|
|
57
|
-
q = @client.queue('test2')
|
58
|
-
res = q.post("hello world!")
|
59
|
-
# p res
|
60
|
-
assert_not_nil res.id
|
61
|
-
assert_not_nil res.msg
|
62
|
-
|
63
|
-
msg = q.get
|
64
|
-
# p res
|
65
|
-
assert_not_nil msg
|
66
|
-
assert_not_nil msg.id
|
67
|
-
assert_not_nil msg.body
|
68
|
-
|
69
|
-
res = msg.delete
|
70
|
-
# p res
|
71
|
-
assert_equal 200, res.code, "API must delete message and response with HTTP 200 status, but returned HTTP #{res.code}"
|
72
57
|
end
|
73
58
|
puts "count: #{quicky.results(:post).count}"
|
74
59
|
puts "avg post: #{quicky.results(:post).duration}"
|
75
60
|
puts "avg get: #{quicky.results(:get).duration}"
|
76
61
|
puts "avg delete: #{quicky.results(:delete).duration}"
|
77
|
-
|
78
|
-
|
62
|
+
puts "queue size: #{queue.reload.size}"
|
63
|
+
assert_equal 0, queue.size
|
79
64
|
resp = queue.delete_queue
|
80
|
-
assert_equal 200, resp.code, "API must
|
65
|
+
assert_equal 200, resp.code, "API must respond with HTTP 200 status, but returned HTTP #{resp.code}"
|
81
66
|
|
82
|
-
resp = @client.queue('test2').delete_queue
|
83
|
-
assert_equal 200, resp.code, "API must response with HTTP 200 status, but returned HTTP #{resp.code}"
|
84
|
-
end
|
85
67
|
|
68
|
+
end
|
86
69
|
|
87
70
|
end
|
88
71
|
|
data/test/quick_run2.rb
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'quicky'
|
2
|
+
require 'go'
|
3
|
+
require 'pp'
|
4
|
+
require File.expand_path('test_base.rb', File.dirname(__FILE__))
|
5
|
+
|
6
|
+
THREADS = 10
|
7
|
+
TIMES_PER_THREAD = 1000
|
8
|
+
|
9
|
+
class QuickRun < TestBase
|
10
|
+
|
11
|
+
def setup
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_quick
|
16
|
+
Go.logger.level = Logger::DEBUG
|
17
|
+
Concur.logger.level = Logger::DEBUG
|
18
|
+
ch = Go::Channel.new
|
19
|
+
results = nil
|
20
|
+
|
21
|
+
(THREADS-1).times do |ti|
|
22
|
+
|
23
|
+
go do
|
24
|
+
do_it(ch, ti)
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
ch.each do |r|
|
30
|
+
puts "results: #{r.inspect}"
|
31
|
+
if results
|
32
|
+
results.merge!(r)
|
33
|
+
else
|
34
|
+
results = r
|
35
|
+
end
|
36
|
+
end
|
37
|
+
pp results.to_hash
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
def do_it(ch, ti)
|
42
|
+
|
43
|
+
begin
|
44
|
+
|
45
|
+
queue_name = "ironmq_gem_quick_#{ti}_#{rand(1000)}"
|
46
|
+
clear_queue(queue_name)
|
47
|
+
queue = @client.queue(queue_name)
|
48
|
+
|
49
|
+
quicky = Quicky::Timer.new
|
50
|
+
consumer_ch = Go::Channel.new
|
51
|
+
quicky.loop(:test_quick, TIMES_PER_THREAD) do |i|
|
52
|
+
puts "==== LOOP t#{ti} - #{i} =================================="
|
53
|
+
|
54
|
+
if i == 50 || i == TIMES_PER_THREAD-1
|
55
|
+
start_consumer(quicky, queue, consumer_ch)
|
56
|
+
end
|
57
|
+
|
58
|
+
post_id = nil
|
59
|
+
quicky.time(:post) do
|
60
|
+
res = queue.post("hello world!")
|
61
|
+
# p res
|
62
|
+
assert_not_nil res
|
63
|
+
assert_not_nil res.id
|
64
|
+
post_id = res.id
|
65
|
+
assert !(res.msg.nil? || res.msg.empty?)
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
# wait for consumer to end
|
70
|
+
#i = 0
|
71
|
+
#consumer_ch.each do |r|
|
72
|
+
# i+=1
|
73
|
+
# puts "consumer #{r}"
|
74
|
+
# consumer_ch.close if i == TIMES_PER_THREAD
|
75
|
+
#end
|
76
|
+
sleep TIMES_PER_THREAD / 2
|
77
|
+
|
78
|
+
puts "count: #{quicky.results(:post).count}"
|
79
|
+
puts "avg post: #{quicky.results(:post).duration}"
|
80
|
+
puts "avg get: #{quicky.results(:get).duration}"
|
81
|
+
puts "avg delete: #{quicky.results(:delete).duration}"
|
82
|
+
puts "queue size: #{queue.reload.size}"
|
83
|
+
resp = queue.delete_queue
|
84
|
+
assert_equal 200, resp.code, "API must respond with HTTP 200 status, but returned HTTP #{resp.code}"
|
85
|
+
|
86
|
+
ch << quicky.results
|
87
|
+
|
88
|
+
rescue Exception => ex
|
89
|
+
p ex
|
90
|
+
p ex.backtrace
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
def start_consumer(quicky, queue, channel)
|
96
|
+
go do
|
97
|
+
while true do
|
98
|
+
msg = nil
|
99
|
+
quicky.time(:get) do
|
100
|
+
msg = queue.get
|
101
|
+
end
|
102
|
+
if msg.nil?
|
103
|
+
channel << "done"
|
104
|
+
break
|
105
|
+
end
|
106
|
+
quicky.time(:delete) do
|
107
|
+
msg.delete
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
data/test/test_alerts.rb
ADDED
@@ -0,0 +1,142 @@
|
|
1
|
+
# Put config.yml file in ~/Dropbox/configs/ironmq_gem/test/config.yml
|
2
|
+
require File.expand_path('test_base.rb', File.dirname(__FILE__))
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
class TestAlerts < TestBase
|
6
|
+
|
7
|
+
def setup
|
8
|
+
super
|
9
|
+
@skip = @host.include? 'rackspace'
|
10
|
+
return if @skip # bypass these tests if rackspace
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_size_alerts
|
14
|
+
return if @skip
|
15
|
+
|
16
|
+
# Add a normal size alert
|
17
|
+
# Post x messages
|
18
|
+
# check to ensure alert was triggered
|
19
|
+
|
20
|
+
x = rand(1000)
|
21
|
+
qname = "alert-queue-#{x}"
|
22
|
+
clear_queue(qname)
|
23
|
+
queue = @client.queue(qname)
|
24
|
+
puts "queue: #{queue}"
|
25
|
+
trigger_value = 10
|
26
|
+
# todo: should :queue be called something else, like target_queue? or url and have to use ironmq:// url?
|
27
|
+
target_queue_name = "alert-target-queue-#{x}"
|
28
|
+
r = queue.add_alert({:type => "size", :trigger => trigger_value, :queue => target_queue_name})
|
29
|
+
p r
|
30
|
+
|
31
|
+
alerts = queue.alerts
|
32
|
+
p alerts
|
33
|
+
assert_equal 1, alerts.size
|
34
|
+
alert = alerts[0]
|
35
|
+
assert_equal "size", alert.type
|
36
|
+
assert_equal trigger_value, alert.trigger
|
37
|
+
assert_equal target_queue_name, alert.queue
|
38
|
+
|
39
|
+
target_queue = @client.queue(target_queue_name)
|
40
|
+
assert_equal 0, target_queue.size
|
41
|
+
|
42
|
+
(trigger_value - 1).times do |i|
|
43
|
+
puts "posting #{i}"
|
44
|
+
queue.post("message #{i}")
|
45
|
+
end
|
46
|
+
sleep 0.5
|
47
|
+
assert_equal 0, target_queue.size
|
48
|
+
queue.post("message #{trigger_value}")
|
49
|
+
sleep 0.5
|
50
|
+
assert_equal trigger_value, queue.size
|
51
|
+
assert_equal 1, target_queue.size
|
52
|
+
|
53
|
+
# now let's get it down the reset point and trigger it again
|
54
|
+
(trigger_value / 2).times do |i|
|
55
|
+
m = queue.get
|
56
|
+
m.delete
|
57
|
+
end
|
58
|
+
assert_equal trigger_value/2, queue.size
|
59
|
+
assert_equal 1, target_queue.size
|
60
|
+
|
61
|
+
# once it's at half, it should reset so let's get it back up to trigger_value again
|
62
|
+
(trigger_value / 2).times do |i|
|
63
|
+
queue.post("second set: message #{i}")
|
64
|
+
end
|
65
|
+
sleep 0.5
|
66
|
+
assert_equal trigger_value, queue.size
|
67
|
+
assert_equal 2, target_queue.size
|
68
|
+
|
69
|
+
queue.delete_queue
|
70
|
+
target_queue.delete_queue
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
def test_progressive_alerts
|
76
|
+
return if @skip
|
77
|
+
|
78
|
+
# Add a normal size alert
|
79
|
+
# Post x messages
|
80
|
+
# check to ensure alert was triggered
|
81
|
+
|
82
|
+
x = rand(1000)
|
83
|
+
qname = "alert-queue-#{x}"
|
84
|
+
queue = @client.queue(qname)
|
85
|
+
trigger_value = 10
|
86
|
+
# todo: should :queue be called something else, like target_queue? or url and have to use ironmq:// url?
|
87
|
+
target_queue_name = "alert-target-queue-#{x}"
|
88
|
+
r = queue.add_alert({:type => "progressive", :trigger => trigger_value, :queue => target_queue_name})
|
89
|
+
p r
|
90
|
+
|
91
|
+
alerts = queue.alerts
|
92
|
+
assert_equal 1, alerts.size
|
93
|
+
alert = alerts[0]
|
94
|
+
assert_equal "progressive", alert.type
|
95
|
+
assert_equal trigger_value, alert.trigger
|
96
|
+
assert_equal target_queue_name, alert.queue
|
97
|
+
|
98
|
+
target_queue = @client.queue(target_queue_name)
|
99
|
+
assert_equal 0, target_queue.size
|
100
|
+
|
101
|
+
post_messages(queue, trigger_value)
|
102
|
+
assert_equal trigger_value, queue.size
|
103
|
+
assert_equal 1, target_queue.size
|
104
|
+
|
105
|
+
# now let's do it again and see if it tiggers again
|
106
|
+
post_messages(queue, trigger_value)
|
107
|
+
assert_equal trigger_value * 2, queue.size
|
108
|
+
assert_equal 2, target_queue.size
|
109
|
+
|
110
|
+
# now let's do it once again and see if it triggers again
|
111
|
+
post_messages(queue, trigger_value)
|
112
|
+
assert_equal trigger_value * 3, queue.size
|
113
|
+
assert_equal 3, target_queue.size
|
114
|
+
|
115
|
+
# now let's get it down to the reset point and trigger it again
|
116
|
+
(trigger_value).times do |i|
|
117
|
+
m = queue.get
|
118
|
+
m.delete
|
119
|
+
end
|
120
|
+
sleep 0.5
|
121
|
+
assert_equal trigger_value*2, queue.size
|
122
|
+
assert_equal 3, target_queue.size
|
123
|
+
|
124
|
+
# once it's at half, it should reset so let's get it back up to trigger_value again
|
125
|
+
post_messages(queue, trigger_value)
|
126
|
+
assert_equal trigger_value*3, queue.size
|
127
|
+
assert_equal 4, target_queue.size
|
128
|
+
|
129
|
+
queue.delete_queue
|
130
|
+
target_queue.delete_queue
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
def post_messages(queue, num)
|
135
|
+
(num).times do |i|
|
136
|
+
queue.post("message #{i}")
|
137
|
+
end
|
138
|
+
sleep 0.5
|
139
|
+
end
|
140
|
+
|
141
|
+
|
142
|
+
end
|
data/test/test_base.rb
CHANGED
@@ -3,13 +3,6 @@ require 'test/unit'
|
|
3
3
|
require 'yaml'
|
4
4
|
require 'uber_config'
|
5
5
|
|
6
|
-
unless Hash.instance_methods.include?(:default_proc=)
|
7
|
-
class Hash
|
8
|
-
def default_proc=(proc)
|
9
|
-
end
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
6
|
begin
|
14
7
|
require File.expand_path('../lib/iron_mq', File.dirname(__FILE__))
|
15
8
|
rescue Exception => ex
|
@@ -18,7 +11,8 @@ rescue Exception => ex
|
|
18
11
|
end
|
19
12
|
|
20
13
|
LOG = Logger.new(STDOUT)
|
21
|
-
|
14
|
+
LOG.level = Logger::INFO
|
15
|
+
MAX_TRIES = 10
|
22
16
|
|
23
17
|
class TestBase < Test::Unit::TestCase
|
24
18
|
|
@@ -44,11 +38,19 @@ class TestBase < Test::Unit::TestCase
|
|
44
38
|
queue_name ||= @queue_name
|
45
39
|
|
46
40
|
queue = @client.queue(queue_name)
|
47
|
-
queue.post("test")
|
48
41
|
|
49
42
|
puts "clearing queue #{queue_name}"
|
50
|
-
|
51
|
-
|
43
|
+
begin
|
44
|
+
queue.clear
|
45
|
+
puts 'cleared.'
|
46
|
+
rescue Rest::HttpError => ex
|
47
|
+
if ex.code == 404
|
48
|
+
# this is fine
|
49
|
+
else
|
50
|
+
raise ex
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
52
54
|
end
|
53
55
|
|
54
56
|
def assert_performance(time)
|