iron 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +7 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +47 -0
- data/README.md +60 -0
- data/Rakefile +21 -0
- data/iron_mq.gemspec +31 -0
- data/lib/iron_mq.rb +5 -0
- data/lib/iron_mq/client.rb +52 -0
- data/lib/iron_mq/messages.rb +123 -0
- data/lib/iron_mq/queues.rb +169 -0
- data/lib/iron_mq/version.rb +3 -0
- data/test/long_run.rb +31 -0
- data/test/long_run_worker.rb +81 -0
- data/test/quick_run.rb +52 -0
- data/test/schedule_abt.rb +27 -0
- data/test/test_base.rb +44 -0
- data/test/test_beanstalkd.rb +222 -0
- data/test/test_bulk.rb +33 -0
- data/test/test_iron_mq.rb +356 -0
- data/test/tmp.rb +45 -0
- metadata +218 -0
data/test/long_run.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'concur'
|
3
|
+
require 'uber_config'
|
4
|
+
begin
|
5
|
+
require File.join(File.dirname(__FILE__), '..', 'lib', 'iron_mq')
|
6
|
+
rescue Exception => ex
|
7
|
+
puts "Could NOT load current iron_mq: " + ex.message
|
8
|
+
raise ex
|
9
|
+
end
|
10
|
+
|
11
|
+
require_relative 'long_run_worker'
|
12
|
+
|
13
|
+
@config = UberConfig.load
|
14
|
+
@num_to_add = 10000
|
15
|
+
|
16
|
+
p @config
|
17
|
+
|
18
|
+
worker = LongRunWorker.new
|
19
|
+
worker.config = @config
|
20
|
+
worker.queue_name = "concur8"
|
21
|
+
worker.num_threads = 100
|
22
|
+
worker.num_to_add = @num_to_add
|
23
|
+
worker.skip_get_and_delete = false
|
24
|
+
worker.run
|
25
|
+
#worker.queue
|
26
|
+
#status = worker.wait_until_complete
|
27
|
+
#p status
|
28
|
+
#puts worker.get_log
|
29
|
+
|
30
|
+
|
31
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'concur'
|
2
|
+
require 'iron_mq'
|
3
|
+
|
4
|
+
class LongRunWorker
|
5
|
+
|
6
|
+
attr_accessor :config, :num_to_add, :queue_name, :skip_get_and_delete, :num_threads
|
7
|
+
|
8
|
+
def run
|
9
|
+
|
10
|
+
@client = IronMQ::Client.new(@config['iron'])
|
11
|
+
queue = @client.queue(queue_name)
|
12
|
+
|
13
|
+
@error_count = 0
|
14
|
+
|
15
|
+
start = Time.now
|
16
|
+
puts "Queuing #{@num_to_add} items at #{start}..."
|
17
|
+
executor = Concur::Executor.new_thread_pool_executor(num_threads || 20)
|
18
|
+
@num_to_add.times do |i|
|
19
|
+
task = executor.execute do
|
20
|
+
begin
|
21
|
+
puts "POST #{i}..."
|
22
|
+
res = queue.post("hello world! #{i}")
|
23
|
+
rescue => ex
|
24
|
+
puts "ERROR! #{ex.class.name}: #{ex.message} -- #{ex.backtrace.inspect}"
|
25
|
+
@error_count += 1
|
26
|
+
raise ex
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
i = 0
|
32
|
+
while executor.queue_size > 0 do
|
33
|
+
i += 1
|
34
|
+
puts "waiting #{i}, queue size=#{executor.queue_size}"
|
35
|
+
sleep 2
|
36
|
+
end
|
37
|
+
put_time = (Time.now.to_f - start.to_f)
|
38
|
+
sleep 1
|
39
|
+
puts "Finished pushing in #{put_time} seconds"
|
40
|
+
|
41
|
+
queue = @client.queue(queue_name)
|
42
|
+
queue_size_after_push = queue.size
|
43
|
+
puts "QUEUE SIZE: #{queue_size_after_push}"
|
44
|
+
|
45
|
+
#exit if true
|
46
|
+
|
47
|
+
if skip_get_and_delete
|
48
|
+
start = Time.now
|
49
|
+
puts "Getting and deleting #{@num_to_add} items at #{start}..."
|
50
|
+
@num_to_add.times do |i|
|
51
|
+
task = executor.execute do
|
52
|
+
puts "GET #{i}..."
|
53
|
+
res = queue.get()
|
54
|
+
p res
|
55
|
+
puts "DELETE #{i}..."
|
56
|
+
res = queue.delete(res.id)
|
57
|
+
p res
|
58
|
+
end
|
59
|
+
end
|
60
|
+
i = 0
|
61
|
+
while executor.queue_size > 0 do
|
62
|
+
i += 1
|
63
|
+
puts "waiting #{i}, queue size=#{executor.queue_size}"
|
64
|
+
sleep 2
|
65
|
+
end
|
66
|
+
sleep 1
|
67
|
+
end
|
68
|
+
|
69
|
+
queue = @client.queue(queue_name)
|
70
|
+
|
71
|
+
puts "Finished pushing #{@num_to_add} items in #{put_time} seconds."
|
72
|
+
puts "QUEUE SIZE after push: #{queue_size_after_push}"
|
73
|
+
puts "Finished getting and deleting #{@num_to_add} items in #{(Time.now.to_f - start.to_f)} seconds."
|
74
|
+
puts "QUEUE SIZE after delete: #{queue.size}"
|
75
|
+
puts "Errors: #{@error_count}"
|
76
|
+
|
77
|
+
executor.shutdown
|
78
|
+
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
data/test/quick_run.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require_relative 'test_base'
|
2
|
+
|
3
|
+
class QuickRun < TestBase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
super
|
7
|
+
@client.queue_name = 'ironmq-gem_quick'
|
8
|
+
clear_queue
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_basics
|
12
|
+
|
13
|
+
res = @client.messages.post("hello world!")
|
14
|
+
p res
|
15
|
+
assert res.id
|
16
|
+
post_id = res.id
|
17
|
+
assert res.msg
|
18
|
+
|
19
|
+
res = @client.messages.get()
|
20
|
+
p res
|
21
|
+
puts "post_id=" + post_id.inspect
|
22
|
+
assert res.id
|
23
|
+
assert_equal res.id, post_id
|
24
|
+
assert res.body
|
25
|
+
|
26
|
+
res = @client.messages.delete(res["id"])
|
27
|
+
p res
|
28
|
+
assert res.msg
|
29
|
+
|
30
|
+
res = @client.messages.get()
|
31
|
+
p res
|
32
|
+
assert res.nil?
|
33
|
+
|
34
|
+
res = @client.messages.post("hello world!", :queue_name=>'test2')
|
35
|
+
p res
|
36
|
+
assert res.id
|
37
|
+
assert res.msg
|
38
|
+
|
39
|
+
res = @client.messages.get(:queue_name=>'test2')
|
40
|
+
p res
|
41
|
+
assert res.id
|
42
|
+
assert res.body
|
43
|
+
|
44
|
+
res = res.delete
|
45
|
+
p res
|
46
|
+
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
end
|
52
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'abt'
|
2
|
+
require 'time'
|
3
|
+
require_relative 'test_base'
|
4
|
+
|
5
|
+
# Config for abt tests to run on IronWorker
|
6
|
+
@abt_config = YAML::load_file(File.expand_path(File.join("~", "Dropbox", "configs", "abt", "test", "config.yml")))
|
7
|
+
IronWorker.configure do |c|
|
8
|
+
c.token = @abt_config['iron']['token']
|
9
|
+
c.project_id = @abt_config['iron']['project_id']
|
10
|
+
end
|
11
|
+
|
12
|
+
# IronWorker.logger.level = Logger::DEBUG
|
13
|
+
# Config to run iron_mq_ruby tests
|
14
|
+
@test_config = TestBase.load_config
|
15
|
+
|
16
|
+
worker = Abt::AbtWorker.new
|
17
|
+
worker.git_url = "git://github.com/iron-io/iron_mq_ruby.git"
|
18
|
+
worker.test_config = @test_config
|
19
|
+
worker.add_notifier(:hip_chat_notifier, :config=>@abt_config["hipchat"])
|
20
|
+
worker.upload # Must upload after adding notifier to ensure it's merged
|
21
|
+
#worker.run_local
|
22
|
+
#worker.queue
|
23
|
+
#status = worker.wait_until_complete
|
24
|
+
#p status
|
25
|
+
#puts "LOG:"
|
26
|
+
#puts worker.get_log
|
27
|
+
#worker.schedule(:start_at=>Time.now.iso8601, :run_every=>3600)
|
data/test/test_base.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
gem 'test-unit'
|
2
|
+
require 'test/unit'
|
3
|
+
require 'yaml'
|
4
|
+
require 'uber_config'
|
5
|
+
begin
|
6
|
+
require File.join(File.dirname(__FILE__), '../lib/iron_mq')
|
7
|
+
rescue Exception => ex
|
8
|
+
puts "Could NOT load current iron_mq: " + ex.message
|
9
|
+
raise ex
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
class TestBase < Test::Unit::TestCase
|
14
|
+
def setup
|
15
|
+
puts 'setup'
|
16
|
+
# check multiple config locations
|
17
|
+
@config = UberConfig.load
|
18
|
+
puts "config=" + @config.inspect
|
19
|
+
@client = IronMQ::Client.new(@config['iron'])
|
20
|
+
IronCore::Logger.logger.level = Logger::DEBUG
|
21
|
+
@client.queue_name = 'ironmq-ruby-tests'
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def clear_queue(queue_name=nil)
|
27
|
+
queue_name ||= @client.queue_name
|
28
|
+
puts "clearing queue #{queue_name}"
|
29
|
+
while res = @client.messages.get(:queue_name=>queue_name)
|
30
|
+
p res
|
31
|
+
puts res.body.to_s
|
32
|
+
res.delete
|
33
|
+
end
|
34
|
+
puts 'cleared.'
|
35
|
+
end
|
36
|
+
|
37
|
+
def assert_performance(time)
|
38
|
+
start_time = Time.now
|
39
|
+
yield
|
40
|
+
execution_time = Time.now - start_time
|
41
|
+
assert execution_time < time, "Execution time too big #{execution_time.round(2)}, should be #{time}"
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,222 @@
|
|
1
|
+
gem 'test-unit'
|
2
|
+
require 'test/unit'
|
3
|
+
require 'beanstalk-client'
|
4
|
+
require 'yaml'
|
5
|
+
require_relative 'test_base'
|
6
|
+
|
7
|
+
class BeanstalkTests < TestBase
|
8
|
+
def setup
|
9
|
+
super
|
10
|
+
|
11
|
+
config = @config['iron']
|
12
|
+
@host = "#{config['host'] || "mq-aws-us-east-1.iron.io"}:#{config['beanstalkd_port'] || 11300}"
|
13
|
+
puts "beanstalkd url: #{@host}"
|
14
|
+
@skip = @host.include? 'rackspace'
|
15
|
+
return if @skip # bypass these tests if rackspace
|
16
|
+
@beanstalk = Beanstalk::Connection.new(@host)
|
17
|
+
@beanstalk.put("oauth #{config['token']} #{config['project_id']}")
|
18
|
+
|
19
|
+
clear_tube('default')
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_basics
|
23
|
+
omit_if @skip # bypass this test if rackspace
|
24
|
+
puts 'test_basics3'
|
25
|
+
|
26
|
+
queue_name = "beanstalk_test"
|
27
|
+
clear_queue(queue_name)
|
28
|
+
@beanstalk.use(queue_name)
|
29
|
+
@beanstalk.watch(queue_name)
|
30
|
+
|
31
|
+
#puts 'reserving...'
|
32
|
+
#job = beanstalk.reserve(1)
|
33
|
+
#p job
|
34
|
+
#exit
|
35
|
+
|
36
|
+
msg = "hello #{Time.now}"
|
37
|
+
puts "msg=" + msg
|
38
|
+
@beanstalk.put(msg)
|
39
|
+
job = @beanstalk.reserve
|
40
|
+
p job
|
41
|
+
p job.body
|
42
|
+
assert_equal msg, job.body, "job.body #{job.body} not the same as the one we put on #{msg}."
|
43
|
+
job.delete
|
44
|
+
job = assert_raise(Beanstalk::TimedOut) {
|
45
|
+
@beanstalk.reserve(1)
|
46
|
+
}
|
47
|
+
|
48
|
+
hasher = {:x => 1, :y => "hello", "yo" => "scooby doo"}
|
49
|
+
@beanstalk.put(hasher.to_json)
|
50
|
+
job = @beanstalk.reserve(1)
|
51
|
+
got = JSON.parse(job.body)
|
52
|
+
assert got.is_a?(Hash)
|
53
|
+
assert_equal hasher[:x], got['x']
|
54
|
+
job.delete
|
55
|
+
|
56
|
+
msg = "hello there\nthis is a new line"
|
57
|
+
@beanstalk.put(msg)
|
58
|
+
job = @beanstalk.reserve(1)
|
59
|
+
assert_equal msg, job.body, "#{job.body} does not equal #{msg}"
|
60
|
+
job.delete
|
61
|
+
end
|
62
|
+
|
63
|
+
def clear_tube(tube)
|
64
|
+
omit_if @skip # bypass this test if rackspace
|
65
|
+
watched = @beanstalk.list_tubes_watched(true)
|
66
|
+
puts 'watched: ' + watched.inspect
|
67
|
+
@beanstalk.watch(tube)
|
68
|
+
puts "clear #{tube}"
|
69
|
+
# clean up anything in queue
|
70
|
+
while x = reserve(0) do
|
71
|
+
puts 'deleting ' + x.inspect
|
72
|
+
x.delete
|
73
|
+
end
|
74
|
+
puts 'done clearing'
|
75
|
+
@beanstalk.ignore(tube) if not watched.include?(tube)
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_basics2
|
79
|
+
omit_if @skip # bypass this test if rackspace
|
80
|
+
puts 'test_basics'
|
81
|
+
msg = "hello #{Time.now}"
|
82
|
+
@beanstalk.put(msg)
|
83
|
+
job = reserve
|
84
|
+
puts 'first job: ' + job.inspect
|
85
|
+
puts "body=" + job.body # prints "hello"
|
86
|
+
assert_equal msg, job.body, "body not the same as message."
|
87
|
+
job.delete
|
88
|
+
puts 'deleted'
|
89
|
+
job = reserve(1)
|
90
|
+
puts 'second job = ' + job.inspect
|
91
|
+
assert job.nil?, "second job was not nil " + job.inspect
|
92
|
+
|
93
|
+
hasher = {:x => 1, :y => "hello", "yo" => "scooby doo"}
|
94
|
+
@beanstalk.put(hasher.to_json)
|
95
|
+
job = reserve(1)
|
96
|
+
got = JSON.parse(job.body)
|
97
|
+
puts 'got=' + got.inspect
|
98
|
+
assert got.is_a?(Hash)
|
99
|
+
assert_equal hasher[:x], got['x']
|
100
|
+
job.delete
|
101
|
+
|
102
|
+
msg = "hello there\nthis is a new line"
|
103
|
+
@beanstalk.put(msg)
|
104
|
+
job = reserve(1)
|
105
|
+
assert_equal msg, job.body, "#{job.body} does not equal #{msg}"
|
106
|
+
job.delete
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_timeout
|
110
|
+
omit_if @skip # bypass this test if rackspace
|
111
|
+
puts 'test_timeout'
|
112
|
+
msg = "timeout message #{Time.now}"
|
113
|
+
# timeout of 10 seconds
|
114
|
+
res = @beanstalk.put(msg, 65536, 0, 10)
|
115
|
+
puts 'result: ' + res.inspect
|
116
|
+
job = reserve(0)
|
117
|
+
puts 'first timeout job: ' + job.inspect
|
118
|
+
assert_equal msg, job.body, "body not the same as message."
|
119
|
+
|
120
|
+
niljob = reserve(0)
|
121
|
+
assert niljob.nil?, "job is not nil! #{niljob.inspect}"
|
122
|
+
|
123
|
+
# let it time out
|
124
|
+
sleep 10
|
125
|
+
job = reserve(0)
|
126
|
+
puts 'second delayed job: ' + job.inspect
|
127
|
+
assert_not_nil job, "job is nil"
|
128
|
+
assert_equal msg, job.body, "body not the same as message."
|
129
|
+
job.delete
|
130
|
+
end
|
131
|
+
|
132
|
+
def test_delay
|
133
|
+
omit_if @skip # bypass this test if rackspace
|
134
|
+
puts 'test_delay'
|
135
|
+
msg = "delayed message #{Time.now}"
|
136
|
+
# delay of 2 seconds
|
137
|
+
@beanstalk.put(msg, 65536, 2)
|
138
|
+
|
139
|
+
# delay should still be in effect, so job is nil
|
140
|
+
job = reserve(0)
|
141
|
+
assert job.nil?, "job is not nil"
|
142
|
+
|
143
|
+
# wait for delay to expire
|
144
|
+
sleep 2
|
145
|
+
job = reserve(0)
|
146
|
+
assert_not_nil job, "job is nil"
|
147
|
+
assert_equal msg, job.body, "body not the same as message."
|
148
|
+
job.delete
|
149
|
+
end
|
150
|
+
|
151
|
+
def tube_message(tube)
|
152
|
+
omit_if @skip # bypass this test if rackspace
|
153
|
+
"hello #{tube}! #{Time.now}"
|
154
|
+
end
|
155
|
+
|
156
|
+
def reserve(timeout=nil)
|
157
|
+
omit_if @skip # bypass this test if rackspace
|
158
|
+
begin
|
159
|
+
job = @beanstalk.reserve(timeout)
|
160
|
+
puts 'got job: ' + job.inspect
|
161
|
+
return job
|
162
|
+
rescue Beanstalk::TimedOut => ex
|
163
|
+
puts "Timed out: #{ex.message}"
|
164
|
+
return nil
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def test_tubes
|
169
|
+
omit_if @skip # bypass this test if rackspace
|
170
|
+
clear_tube('youtube')
|
171
|
+
tube1 = 'default'
|
172
|
+
msg1 = tube_message(tube1)
|
173
|
+
@beanstalk.put(msg1)
|
174
|
+
tube2 = 'youtube'
|
175
|
+
@beanstalk.use(tube2) # switch tubes to put messages on
|
176
|
+
msg2 = tube_message(tube2)
|
177
|
+
@beanstalk.put(msg2)
|
178
|
+
# now we have a single message in two different tubes.
|
179
|
+
# let's first try to ensure we only get the one off the default tube.
|
180
|
+
job = reserve(1)
|
181
|
+
assert_equal msg1, job.body, "body #{job.body.to_s} does not equal #{msg1}"
|
182
|
+
job.delete
|
183
|
+
# second message should not come off since we're not watching that tube.
|
184
|
+
job = reserve(1)
|
185
|
+
assert job.nil?, "second job was not nil " + job.inspect
|
186
|
+
|
187
|
+
@beanstalk.watch(tube2)
|
188
|
+
job = reserve(1)
|
189
|
+
assert_equal msg2, job.body, "body #{job.body.to_s} does not equal #{msg2}"
|
190
|
+
job.delete
|
191
|
+
|
192
|
+
# Now that we're watching both tubes we should get messages put into either.
|
193
|
+
@beanstalk.use(tube1)
|
194
|
+
msg3 = tube_message(tube1)
|
195
|
+
@beanstalk.put(msg3)
|
196
|
+
job = reserve(1)
|
197
|
+
assert_equal msg3, job.body
|
198
|
+
job.delete
|
199
|
+
|
200
|
+
# now put one in default and one in tube2, then we'll ignore default
|
201
|
+
msg1 = tube_message(tube1)
|
202
|
+
@beanstalk.put(msg1)
|
203
|
+
@beanstalk.use(tube2) # switch tubes to put messages on
|
204
|
+
msg2 = tube_message(tube2)
|
205
|
+
@beanstalk.put(msg2)
|
206
|
+
|
207
|
+
@beanstalk.ignore(tube1)
|
208
|
+
job = reserve(1)
|
209
|
+
assert_equal msg2, job.body, "body #{job.body.to_s} does not equal #{msg2}"
|
210
|
+
job.delete
|
211
|
+
job = reserve(1)
|
212
|
+
assert job.nil?
|
213
|
+
|
214
|
+
# clean up the last message from default
|
215
|
+
@beanstalk.watch(tube1)
|
216
|
+
job = reserve(1)
|
217
|
+
assert_equal msg1, job.body, "body #{job.body.to_s} does not equal #{msg1}"
|
218
|
+
job.delete
|
219
|
+
end
|
220
|
+
|
221
|
+
|
222
|
+
end
|