iron 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ module IronMQ
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,31 @@
1
+ require 'yaml'
2
+ require 'concur'
3
+ require 'uber_config'
4
+ begin
5
+ require File.join(File.dirname(__FILE__), '..', 'lib', 'iron_mq')
6
+ rescue Exception => ex
7
+ puts "Could NOT load current iron_mq: " + ex.message
8
+ raise ex
9
+ end
10
+
11
+ require_relative 'long_run_worker'
12
+
13
+ @config = UberConfig.load
14
+ @num_to_add = 10000
15
+
16
+ p @config
17
+
18
+ worker = LongRunWorker.new
19
+ worker.config = @config
20
+ worker.queue_name = "concur8"
21
+ worker.num_threads = 100
22
+ worker.num_to_add = @num_to_add
23
+ worker.skip_get_and_delete = false
24
+ worker.run
25
+ #worker.queue
26
+ #status = worker.wait_until_complete
27
+ #p status
28
+ #puts worker.get_log
29
+
30
+
31
+
@@ -0,0 +1,81 @@
1
+ require 'concur'
2
+ require 'iron_mq'
3
+
4
+ class LongRunWorker
5
+
6
+ attr_accessor :config, :num_to_add, :queue_name, :skip_get_and_delete, :num_threads
7
+
8
+ def run
9
+
10
+ @client = IronMQ::Client.new(@config['iron'])
11
+ queue = @client.queue(queue_name)
12
+
13
+ @error_count = 0
14
+
15
+ start = Time.now
16
+ puts "Queuing #{@num_to_add} items at #{start}..."
17
+ executor = Concur::Executor.new_thread_pool_executor(num_threads || 20)
18
+ @num_to_add.times do |i|
19
+ task = executor.execute do
20
+ begin
21
+ puts "POST #{i}..."
22
+ res = queue.post("hello world! #{i}")
23
+ rescue => ex
24
+ puts "ERROR! #{ex.class.name}: #{ex.message} -- #{ex.backtrace.inspect}"
25
+ @error_count += 1
26
+ raise ex
27
+ end
28
+ end
29
+ end
30
+
31
+ i = 0
32
+ while executor.queue_size > 0 do
33
+ i += 1
34
+ puts "waiting #{i}, queue size=#{executor.queue_size}"
35
+ sleep 2
36
+ end
37
+ put_time = (Time.now.to_f - start.to_f)
38
+ sleep 1
39
+ puts "Finished pushing in #{put_time} seconds"
40
+
41
+ queue = @client.queue(queue_name)
42
+ queue_size_after_push = queue.size
43
+ puts "QUEUE SIZE: #{queue_size_after_push}"
44
+
45
+ #exit if true
46
+
47
+ if skip_get_and_delete
48
+ start = Time.now
49
+ puts "Getting and deleting #{@num_to_add} items at #{start}..."
50
+ @num_to_add.times do |i|
51
+ task = executor.execute do
52
+ puts "GET #{i}..."
53
+ res = queue.get()
54
+ p res
55
+ puts "DELETE #{i}..."
56
+ res = queue.delete(res.id)
57
+ p res
58
+ end
59
+ end
60
+ i = 0
61
+ while executor.queue_size > 0 do
62
+ i += 1
63
+ puts "waiting #{i}, queue size=#{executor.queue_size}"
64
+ sleep 2
65
+ end
66
+ sleep 1
67
+ end
68
+
69
+ queue = @client.queue(queue_name)
70
+
71
+ puts "Finished pushing #{@num_to_add} items in #{put_time} seconds."
72
+ puts "QUEUE SIZE after push: #{queue_size_after_push}"
73
+ puts "Finished getting and deleting #{@num_to_add} items in #{(Time.now.to_f - start.to_f)} seconds."
74
+ puts "QUEUE SIZE after delete: #{queue.size}"
75
+ puts "Errors: #{@error_count}"
76
+
77
+ executor.shutdown
78
+
79
+ end
80
+
81
+ end
@@ -0,0 +1,52 @@
1
+ require_relative 'test_base'
2
+
3
+ class QuickRun < TestBase
4
+
5
+ def setup
6
+ super
7
+ @client.queue_name = 'ironmq-gem_quick'
8
+ clear_queue
9
+ end
10
+
11
+ def test_basics
12
+
13
+ res = @client.messages.post("hello world!")
14
+ p res
15
+ assert res.id
16
+ post_id = res.id
17
+ assert res.msg
18
+
19
+ res = @client.messages.get()
20
+ p res
21
+ puts "post_id=" + post_id.inspect
22
+ assert res.id
23
+ assert_equal res.id, post_id
24
+ assert res.body
25
+
26
+ res = @client.messages.delete(res["id"])
27
+ p res
28
+ assert res.msg
29
+
30
+ res = @client.messages.get()
31
+ p res
32
+ assert res.nil?
33
+
34
+ res = @client.messages.post("hello world!", :queue_name=>'test2')
35
+ p res
36
+ assert res.id
37
+ assert res.msg
38
+
39
+ res = @client.messages.get(:queue_name=>'test2')
40
+ p res
41
+ assert res.id
42
+ assert res.body
43
+
44
+ res = res.delete
45
+ p res
46
+
47
+
48
+ end
49
+
50
+
51
+ end
52
+
@@ -0,0 +1,27 @@
1
+ require 'abt'
2
+ require 'time'
3
+ require_relative 'test_base'
4
+
5
+ # Config for abt tests to run on IronWorker
6
+ @abt_config = YAML::load_file(File.expand_path(File.join("~", "Dropbox", "configs", "abt", "test", "config.yml")))
7
+ IronWorker.configure do |c|
8
+ c.token = @abt_config['iron']['token']
9
+ c.project_id = @abt_config['iron']['project_id']
10
+ end
11
+
12
+ # IronWorker.logger.level = Logger::DEBUG
13
+ # Config to run iron_mq_ruby tests
14
+ @test_config = TestBase.load_config
15
+
16
+ worker = Abt::AbtWorker.new
17
+ worker.git_url = "git://github.com/iron-io/iron_mq_ruby.git"
18
+ worker.test_config = @test_config
19
+ worker.add_notifier(:hip_chat_notifier, :config=>@abt_config["hipchat"])
20
+ worker.upload # Must upload after adding notifier to ensure it's merged
21
+ #worker.run_local
22
+ #worker.queue
23
+ #status = worker.wait_until_complete
24
+ #p status
25
+ #puts "LOG:"
26
+ #puts worker.get_log
27
+ #worker.schedule(:start_at=>Time.now.iso8601, :run_every=>3600)
@@ -0,0 +1,44 @@
1
+ gem 'test-unit'
2
+ require 'test/unit'
3
+ require 'yaml'
4
+ require 'uber_config'
5
+ begin
6
+ require File.join(File.dirname(__FILE__), '../lib/iron_mq')
7
+ rescue Exception => ex
8
+ puts "Could NOT load current iron_mq: " + ex.message
9
+ raise ex
10
+ end
11
+
12
+
13
+ class TestBase < Test::Unit::TestCase
14
+ def setup
15
+ puts 'setup'
16
+ # check multiple config locations
17
+ @config = UberConfig.load
18
+ puts "config=" + @config.inspect
19
+ @client = IronMQ::Client.new(@config['iron'])
20
+ IronCore::Logger.logger.level = Logger::DEBUG
21
+ @client.queue_name = 'ironmq-ruby-tests'
22
+
23
+ end
24
+
25
+
26
+ def clear_queue(queue_name=nil)
27
+ queue_name ||= @client.queue_name
28
+ puts "clearing queue #{queue_name}"
29
+ while res = @client.messages.get(:queue_name=>queue_name)
30
+ p res
31
+ puts res.body.to_s
32
+ res.delete
33
+ end
34
+ puts 'cleared.'
35
+ end
36
+
37
+ def assert_performance(time)
38
+ start_time = Time.now
39
+ yield
40
+ execution_time = Time.now - start_time
41
+ assert execution_time < time, "Execution time too big #{execution_time.round(2)}, should be #{time}"
42
+ end
43
+
44
+ end
@@ -0,0 +1,222 @@
1
+ gem 'test-unit'
2
+ require 'test/unit'
3
+ require 'beanstalk-client'
4
+ require 'yaml'
5
+ require_relative 'test_base'
6
+
7
+ class BeanstalkTests < TestBase
8
+ def setup
9
+ super
10
+
11
+ config = @config['iron']
12
+ @host = "#{config['host'] || "mq-aws-us-east-1.iron.io"}:#{config['beanstalkd_port'] || 11300}"
13
+ puts "beanstalkd url: #{@host}"
14
+ @skip = @host.include? 'rackspace'
15
+ return if @skip # bypass these tests if rackspace
16
+ @beanstalk = Beanstalk::Connection.new(@host)
17
+ @beanstalk.put("oauth #{config['token']} #{config['project_id']}")
18
+
19
+ clear_tube('default')
20
+ end
21
+
22
+ def test_basics
23
+ omit_if @skip # bypass this test if rackspace
24
+ puts 'test_basics3'
25
+
26
+ queue_name = "beanstalk_test"
27
+ clear_queue(queue_name)
28
+ @beanstalk.use(queue_name)
29
+ @beanstalk.watch(queue_name)
30
+
31
+ #puts 'reserving...'
32
+ #job = beanstalk.reserve(1)
33
+ #p job
34
+ #exit
35
+
36
+ msg = "hello #{Time.now}"
37
+ puts "msg=" + msg
38
+ @beanstalk.put(msg)
39
+ job = @beanstalk.reserve
40
+ p job
41
+ p job.body
42
+ assert_equal msg, job.body, "job.body #{job.body} not the same as the one we put on #{msg}."
43
+ job.delete
44
+ job = assert_raise(Beanstalk::TimedOut) {
45
+ @beanstalk.reserve(1)
46
+ }
47
+
48
+ hasher = {:x => 1, :y => "hello", "yo" => "scooby doo"}
49
+ @beanstalk.put(hasher.to_json)
50
+ job = @beanstalk.reserve(1)
51
+ got = JSON.parse(job.body)
52
+ assert got.is_a?(Hash)
53
+ assert_equal hasher[:x], got['x']
54
+ job.delete
55
+
56
+ msg = "hello there\nthis is a new line"
57
+ @beanstalk.put(msg)
58
+ job = @beanstalk.reserve(1)
59
+ assert_equal msg, job.body, "#{job.body} does not equal #{msg}"
60
+ job.delete
61
+ end
62
+
63
+ def clear_tube(tube)
64
+ omit_if @skip # bypass this test if rackspace
65
+ watched = @beanstalk.list_tubes_watched(true)
66
+ puts 'watched: ' + watched.inspect
67
+ @beanstalk.watch(tube)
68
+ puts "clear #{tube}"
69
+ # clean up anything in queue
70
+ while x = reserve(0) do
71
+ puts 'deleting ' + x.inspect
72
+ x.delete
73
+ end
74
+ puts 'done clearing'
75
+ @beanstalk.ignore(tube) if not watched.include?(tube)
76
+ end
77
+
78
+ def test_basics2
79
+ omit_if @skip # bypass this test if rackspace
80
+ puts 'test_basics'
81
+ msg = "hello #{Time.now}"
82
+ @beanstalk.put(msg)
83
+ job = reserve
84
+ puts 'first job: ' + job.inspect
85
+ puts "body=" + job.body # prints "hello"
86
+ assert_equal msg, job.body, "body not the same as message."
87
+ job.delete
88
+ puts 'deleted'
89
+ job = reserve(1)
90
+ puts 'second job = ' + job.inspect
91
+ assert job.nil?, "second job was not nil " + job.inspect
92
+
93
+ hasher = {:x => 1, :y => "hello", "yo" => "scooby doo"}
94
+ @beanstalk.put(hasher.to_json)
95
+ job = reserve(1)
96
+ got = JSON.parse(job.body)
97
+ puts 'got=' + got.inspect
98
+ assert got.is_a?(Hash)
99
+ assert_equal hasher[:x], got['x']
100
+ job.delete
101
+
102
+ msg = "hello there\nthis is a new line"
103
+ @beanstalk.put(msg)
104
+ job = reserve(1)
105
+ assert_equal msg, job.body, "#{job.body} does not equal #{msg}"
106
+ job.delete
107
+ end
108
+
109
+ def test_timeout
110
+ omit_if @skip # bypass this test if rackspace
111
+ puts 'test_timeout'
112
+ msg = "timeout message #{Time.now}"
113
+ # timeout of 10 seconds
114
+ res = @beanstalk.put(msg, 65536, 0, 10)
115
+ puts 'result: ' + res.inspect
116
+ job = reserve(0)
117
+ puts 'first timeout job: ' + job.inspect
118
+ assert_equal msg, job.body, "body not the same as message."
119
+
120
+ niljob = reserve(0)
121
+ assert niljob.nil?, "job is not nil! #{niljob.inspect}"
122
+
123
+ # let it time out
124
+ sleep 10
125
+ job = reserve(0)
126
+ puts 'second delayed job: ' + job.inspect
127
+ assert_not_nil job, "job is nil"
128
+ assert_equal msg, job.body, "body not the same as message."
129
+ job.delete
130
+ end
131
+
132
+ def test_delay
133
+ omit_if @skip # bypass this test if rackspace
134
+ puts 'test_delay'
135
+ msg = "delayed message #{Time.now}"
136
+ # delay of 2 seconds
137
+ @beanstalk.put(msg, 65536, 2)
138
+
139
+ # delay should still be in effect, so job is nil
140
+ job = reserve(0)
141
+ assert job.nil?, "job is not nil"
142
+
143
+ # wait for delay to expire
144
+ sleep 2
145
+ job = reserve(0)
146
+ assert_not_nil job, "job is nil"
147
+ assert_equal msg, job.body, "body not the same as message."
148
+ job.delete
149
+ end
150
+
151
+ def tube_message(tube)
152
+ omit_if @skip # bypass this test if rackspace
153
+ "hello #{tube}! #{Time.now}"
154
+ end
155
+
156
+ def reserve(timeout=nil)
157
+ omit_if @skip # bypass this test if rackspace
158
+ begin
159
+ job = @beanstalk.reserve(timeout)
160
+ puts 'got job: ' + job.inspect
161
+ return job
162
+ rescue Beanstalk::TimedOut => ex
163
+ puts "Timed out: #{ex.message}"
164
+ return nil
165
+ end
166
+ end
167
+
168
+ def test_tubes
169
+ omit_if @skip # bypass this test if rackspace
170
+ clear_tube('youtube')
171
+ tube1 = 'default'
172
+ msg1 = tube_message(tube1)
173
+ @beanstalk.put(msg1)
174
+ tube2 = 'youtube'
175
+ @beanstalk.use(tube2) # switch tubes to put messages on
176
+ msg2 = tube_message(tube2)
177
+ @beanstalk.put(msg2)
178
+ # now we have a single message in two different tubes.
179
+ # let's first try to ensure we only get the one off the default tube.
180
+ job = reserve(1)
181
+ assert_equal msg1, job.body, "body #{job.body.to_s} does not equal #{msg1}"
182
+ job.delete
183
+ # second message should not come off since we're not watching that tube.
184
+ job = reserve(1)
185
+ assert job.nil?, "second job was not nil " + job.inspect
186
+
187
+ @beanstalk.watch(tube2)
188
+ job = reserve(1)
189
+ assert_equal msg2, job.body, "body #{job.body.to_s} does not equal #{msg2}"
190
+ job.delete
191
+
192
+ # Now that we're watching both tubes we should get messages put into either.
193
+ @beanstalk.use(tube1)
194
+ msg3 = tube_message(tube1)
195
+ @beanstalk.put(msg3)
196
+ job = reserve(1)
197
+ assert_equal msg3, job.body
198
+ job.delete
199
+
200
+ # now put one in default and one in tube2, then we'll ignore default
201
+ msg1 = tube_message(tube1)
202
+ @beanstalk.put(msg1)
203
+ @beanstalk.use(tube2) # switch tubes to put messages on
204
+ msg2 = tube_message(tube2)
205
+ @beanstalk.put(msg2)
206
+
207
+ @beanstalk.ignore(tube1)
208
+ job = reserve(1)
209
+ assert_equal msg2, job.body, "body #{job.body.to_s} does not equal #{msg2}"
210
+ job.delete
211
+ job = reserve(1)
212
+ assert job.nil?
213
+
214
+ # clean up the last message from default
215
+ @beanstalk.watch(tube1)
216
+ job = reserve(1)
217
+ assert_equal msg1, job.body, "body #{job.body.to_s} does not equal #{msg1}"
218
+ job.delete
219
+ end
220
+
221
+
222
+ end