backburner-allq 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.travis.yml +29 -0
  4. data/CHANGELOG.md +133 -0
  5. data/CONTRIBUTING.md +37 -0
  6. data/Gemfile +4 -0
  7. data/HOOKS.md +99 -0
  8. data/LICENSE +22 -0
  9. data/README.md +658 -0
  10. data/Rakefile +17 -0
  11. data/TODO +4 -0
  12. data/backburner-allq.gemspec +26 -0
  13. data/bin/backburner +7 -0
  14. data/circle.yml +3 -0
  15. data/deploy.sh +3 -0
  16. data/examples/custom.rb +25 -0
  17. data/examples/demo.rb +60 -0
  18. data/examples/god.rb +46 -0
  19. data/examples/hooked.rb +87 -0
  20. data/examples/retried.rb +31 -0
  21. data/examples/simple.rb +43 -0
  22. data/examples/stress.rb +31 -0
  23. data/lib/backburner.rb +75 -0
  24. data/lib/backburner/allq_wrapper.rb +317 -0
  25. data/lib/backburner/async_proxy.rb +25 -0
  26. data/lib/backburner/cli.rb +53 -0
  27. data/lib/backburner/configuration.rb +48 -0
  28. data/lib/backburner/connection.rb +157 -0
  29. data/lib/backburner/helpers.rb +193 -0
  30. data/lib/backburner/hooks.rb +53 -0
  31. data/lib/backburner/job.rb +118 -0
  32. data/lib/backburner/logger.rb +53 -0
  33. data/lib/backburner/performable.rb +95 -0
  34. data/lib/backburner/queue.rb +145 -0
  35. data/lib/backburner/tasks.rb +54 -0
  36. data/lib/backburner/version.rb +3 -0
  37. data/lib/backburner/worker.rb +221 -0
  38. data/lib/backburner/workers/forking.rb +52 -0
  39. data/lib/backburner/workers/simple.rb +29 -0
  40. data/lib/backburner/workers/threading.rb +163 -0
  41. data/lib/backburner/workers/threads_on_fork.rb +263 -0
  42. data/test/async_proxy_test.rb +36 -0
  43. data/test/back_burner_test.rb +88 -0
  44. data/test/connection_test.rb +179 -0
  45. data/test/fixtures/hooked.rb +122 -0
  46. data/test/fixtures/test_fork_jobs.rb +72 -0
  47. data/test/fixtures/test_forking_jobs.rb +56 -0
  48. data/test/fixtures/test_jobs.rb +87 -0
  49. data/test/fixtures/test_queue_settings.rb +14 -0
  50. data/test/helpers/templogger.rb +22 -0
  51. data/test/helpers_test.rb +278 -0
  52. data/test/hooks_test.rb +112 -0
  53. data/test/job_test.rb +185 -0
  54. data/test/logger_test.rb +44 -0
  55. data/test/performable_test.rb +88 -0
  56. data/test/queue_test.rb +69 -0
  57. data/test/test_helper.rb +128 -0
  58. data/test/worker_test.rb +157 -0
  59. data/test/workers/forking_worker_test.rb +181 -0
  60. data/test/workers/simple_worker_test.rb +350 -0
  61. data/test/workers/threading_worker_test.rb +104 -0
  62. data/test/workers/threads_on_fork_worker_test.rb +484 -0
  63. metadata +217 -0
@@ -0,0 +1,36 @@
1
+ require File.expand_path('../test_helper', __FILE__)
2
+
3
+ class AsyncUser; end
4
+
5
+ describe "Backburner::AsyncProxy class" do
6
+ before do
7
+ Backburner.default_queues.clear
8
+ clear_jobs!(Backburner.configuration.primary_queue)
9
+ end
10
+
11
+ describe "for method_missing enqueue" do
12
+ should "enqueue job onto worker with no args" do
13
+ @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
14
+ @async.foo
15
+ pop_one_job do |job, body|
16
+ assert_equal "AsyncUser", body["class"]
17
+ assert_equal [10, "foo"], body["args"]
18
+ assert_equal 100, job.ttr
19
+ assert_equal 1000, job.pri
20
+ job.delete
21
+ end
22
+ end
23
+
24
+ should "enqueue job onto worker with args" do
25
+ @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
26
+ @async.bar(1, 2, 3)
27
+ pop_one_job do |job, body|
28
+ assert_equal "AsyncUser", body["class"]
29
+ assert_equal [10, "bar", 1, 2, 3], body["args"]
30
+ assert_equal 100, job.ttr
31
+ assert_equal 1000, job.pri
32
+ job.delete
33
+ end
34
+ end
35
+ end # method_missing
36
+ end # AsyncProxy
@@ -0,0 +1,88 @@
1
+ require File.expand_path('../test_helper', __FILE__)
2
+
3
+ $backburner_sum = 0
4
+ $backburner_numbers = []
5
+
6
+ class TestBackburnerJob
7
+ include Backburner::Queue
8
+ queue "test.jobber"
9
+
10
+ def self.perform(value, number)
11
+ $backburner_sum += value
12
+ $backburner_numbers << number
13
+ end
14
+ end
15
+
16
+ class TestWorker < Backburner::Worker; end
17
+
18
+ describe "Backburner module" do
19
+ before { Backburner.default_queues.clear }
20
+
21
+ describe "for enqueue method" do
22
+ before do
23
+ Backburner.enqueue TestBackburnerJob, 5, 6
24
+ Backburner.enqueue TestBackburnerJob, 15, 10
25
+ silenced(2) do
26
+ worker = Backburner::Workers::Simple.new('test.jobber')
27
+ worker.prepare
28
+ 2.times { worker.work_one_job }
29
+ end
30
+ end
31
+
32
+ it "can run jobs using #run method" do
33
+ assert_equal 20, $backburner_sum
34
+ assert_same_elements [6, 10], $backburner_numbers
35
+ end
36
+ end # enqueue
37
+
38
+ describe "for work method" do
39
+ it "invokes worker simple start" do
40
+ Backburner::Workers::Simple.expects(:start).with(["foo", "bar"])
41
+ Backburner.work("foo", "bar")
42
+ end
43
+
44
+ it "invokes other worker if specified in configuration" do
45
+ Backburner.configure { |config| config.default_worker = TestWorker }
46
+ TestWorker.expects(:start).with(["foo", "bar"])
47
+ Backburner.work("foo", "bar")
48
+ end
49
+
50
+ it "invokes other worker if specified in work method as options" do
51
+ TestWorker.expects(:start).with(["foo", "bar"])
52
+ Backburner.work("foo", "bar", :worker => TestWorker)
53
+ end
54
+
55
+ it "invokes worker start with no args" do
56
+ Backburner::Workers::Simple.expects(:start).with([])
57
+ Backburner.work
58
+ end
59
+ end # work!
60
+
61
+ describe "for configuration" do
62
+ it "remembers the tube_namespace" do
63
+ assert_equal "demo.test", Backburner.configuration.tube_namespace
64
+ end
65
+
66
+ it "remembers the namespace_separator" do
67
+ assert_equal ".", Backburner.configuration.namespace_separator
68
+ end
69
+
70
+ it "disallows a reserved separator" do
71
+ assert_raises RuntimeError do
72
+ Backburner.configuration.namespace_separator = ':'
73
+ end
74
+ end
75
+ end # configuration
76
+
77
+ describe "for default_queues" do
78
+ it "supports assignment" do
79
+ Backburner.default_queues << "foo"
80
+ Backburner.default_queues << "bar"
81
+ assert_same_elements ["foo", "bar"], Backburner.default_queues
82
+ end
83
+ end
84
+
85
+ after do
86
+ Backburner.configure { |config| config.default_worker = Backburner::Workers::Simple }
87
+ end
88
+ end # Backburner
@@ -0,0 +1,179 @@
1
+ require File.expand_path('../test_helper', __FILE__)
2
+
3
+ describe "Backburner::Connection class" do
4
+ describe "for initialize with single url" do
5
+ before do
6
+ @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
7
+ end
8
+
9
+ it "should store url in accessor" do
10
+ assert_equal "beanstalk://127.0.0.1", @connection.url
11
+ end
12
+
13
+ it "should setup beanstalk connection" do
14
+ assert_kind_of Beaneater, @connection.beanstalk
15
+ end
16
+ end # initialize single connection
17
+
18
+ describe "for initialize with url" do
19
+ it "should delegate the address url correctly" do
20
+ @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
21
+ connection = @connection.beanstalk.connection
22
+ assert_equal '127.0.0.1:11300', connection.address
23
+ end
24
+ end # initialize
25
+
26
+ describe "for bad uri" do
27
+ it "should raise a BadUrl" do
28
+ assert_raises(Backburner::Connection::BadURL) {
29
+ @connection = Backburner::Connection.new("fake://foo")
30
+ }
31
+ end
32
+ end
33
+
34
+ describe "for initialize with on_reconnect block" do
35
+ it "should store the block for use upon reconnect" do
36
+ callback = proc {}
37
+ connection = Backburner::Connection.new('beanstalk://127.0.0.1', &callback)
38
+ assert_equal callback, connection.on_reconnect
39
+ end
40
+ end
41
+
42
+ describe "dealing with connecting and reconnecting" do
43
+ before do
44
+ @connection = Backburner::Connection.new('beanstalk://127.0.0.1')
45
+ end
46
+
47
+ it "should know if its connection is open" do
48
+ assert_equal true, @connection.connected?
49
+ @connection.close
50
+ assert_equal false, @connection.connected?
51
+ end
52
+
53
+ it "should be able to attempt reconnecting to beanstalk" do
54
+ @connection.close
55
+ assert_equal false, @connection.connected?
56
+ @connection.reconnect!
57
+ assert_equal true, @connection.connected?
58
+ end
59
+
60
+ it "should allow for retryable commands" do
61
+ @result = false
62
+ @connection.close
63
+ @connection.retryable { @result = true }
64
+ assert_equal true, @result
65
+ end
66
+
67
+ it "should provide a hook when a retryable command successfully retries" do
68
+ @result = false
69
+ @retried = false
70
+ @connection.close
71
+ callback = proc { @result = true }
72
+ @connection.retryable(:on_retry => callback) do
73
+ unless @retried
74
+ @retried = true
75
+ raise Beaneater::NotConnected.new
76
+ end
77
+ end
78
+ assert_equal true, @result
79
+ end
80
+
81
+ it "should provide a hook when the connection successfully reconnects" do
82
+ reconnected = false
83
+ retried = false
84
+ @connection.close
85
+ @connection.on_reconnect = proc { reconnected = true }
86
+ @connection.retryable do
87
+ unless retried
88
+ retried = true
89
+ raise Beaneater::NotConnected.new
90
+ end
91
+ end
92
+ assert_equal true, reconnected
93
+ end
94
+
95
+ it "should call the on_reconnect hook before the on_retry hook" do
96
+ @result = []
97
+ @retried = false
98
+ @connection.close
99
+ @connection.on_reconnect = proc { @result << "reconnect" }
100
+ on_retry = proc { @result << "retry" }
101
+ @connection.retryable(:on_retry => on_retry) do
102
+ unless @retried
103
+ @retried = true
104
+ raise Beaneater::NotConnected.new
105
+ end
106
+ end
107
+ assert_equal %w(reconnect retry), @result
108
+ end
109
+
110
+ describe "ensuring the connection is open" do
111
+ it "should reattempt the connection to beanstalk several times" do
112
+ stats = @connection.stats
113
+ simulate_disconnect(@connection)
114
+ new_connection = Beaneater.new('127.0.0.1:11300')
115
+ Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(new_connection)
116
+ @connection.tubes
117
+ assert_equal true, @connection.connected?
118
+ end
119
+
120
+ it "should not attempt reconnecting if the current connection is open" do
121
+ assert_equal true, @connection.connected?
122
+ Beaneater.expects(:new).never
123
+ @connection.tubes
124
+ end
125
+
126
+ describe "when reconnecting is successful" do
127
+ it "should allow for a callback" do
128
+ @result = false
129
+ simulate_disconnect(@connection)
130
+ @connection.on_reconnect = proc { @result = true }
131
+ @connection.tubes
132
+ assert_equal true, @result
133
+ end
134
+
135
+ it "should pass self to the callback" do
136
+ result = nil
137
+ simulate_disconnect(@connection)
138
+ @connection.on_reconnect = lambda { |conn| result = conn }
139
+ @connection.tubes
140
+ assert_equal result, @connection
141
+ end
142
+ end
143
+ end
144
+
145
+ describe "when unable to ensure its connected" do
146
+ it "should raise Beaneater::NotConnected" do
147
+ Beaneater.stubs(:new).raises(Beaneater::NotConnected)
148
+ simulate_disconnect(@connection, 1) # since we're stubbing Beaneater.new above we only to simlulate the disconnect of our current connection
149
+ assert_raises Beaneater::NotConnected do
150
+ @connection.tubes
151
+ end
152
+ end
153
+ end
154
+
155
+ describe "when using the retryable method" do
156
+ it "should yield to the block multiple times" do
157
+ expected = 2
158
+ retry_count = 0
159
+ @connection.retryable(max_retries: expected) do
160
+ if retry_count < 2
161
+ retry_count += 1
162
+ raise Beaneater::NotConnected
163
+ end
164
+ end
165
+ assert_equal expected, retry_count
166
+ end
167
+ end
168
+ end
169
+
170
+ describe "for delegated methods" do
171
+ before do
172
+ @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
173
+ end
174
+
175
+ it "delegate methods to beanstalk connection" do
176
+ assert_equal "127.0.0.1", @connection.connection.host
177
+ end
178
+ end # delegator
179
+ end # Connection
@@ -0,0 +1,122 @@
1
+ $hooked_fail_count = 0
2
+ class HookFailError < RuntimeError; end
3
+
4
+ class HookedObjectBeforeEnqueueFail
5
+ include Backburner::Performable
6
+
7
+ def self.before_enqueue_abe(*args)
8
+ puts "!!before_enqueue_foo!! #{args.inspect}"
9
+ end
10
+
11
+ def self.before_enqueue_bar(*args)
12
+ return false
13
+ end
14
+ end
15
+
16
+
17
+ class HookedObjectAfterEnqueueFail
18
+ def self.after_enqueue_abe(*args)
19
+ puts "!!after_enqueue_foo!! #{args.inspect}"
20
+ end
21
+
22
+ def self.after_enqueue_bar(*args)
23
+ raise HookFailError, "Fail HookedObjectAfterEnqueueFail"
24
+ end
25
+ end
26
+
27
+ class HookedObjectBeforePerformFail
28
+ include Backburner::Performable
29
+
30
+ def self.before_perform_abe(*args)
31
+ puts "!!before_perform_foo!! #{args.inspect}"
32
+ end
33
+
34
+ def self.before_perform_foo(*args)
35
+ return false
36
+ end
37
+
38
+ def self.foo(x)
39
+ puts "Fail ran!!"
40
+ raise HookFailError, "HookedObjectJobFailure on foo!"
41
+ end
42
+ end
43
+
44
+ class HookedObjectAfterPerformFail
45
+ def self.after_perform_abe(*args)
46
+ puts "!!after_perform_foo!! #{args.inspect}"
47
+ end
48
+
49
+ def self.after_perform_bar(*args)
50
+ raise HookFailError, "Fail HookedObjectAfterEnqueueFail"
51
+ end
52
+ end
53
+
54
+ class HookedObjectJobFailure
55
+ def self.foo(x)
56
+ raise HookFailError, "HookedObjectJobFailure on foo!"
57
+ end
58
+ end
59
+
60
+ class HookedObjectSuccess
61
+ include Backburner::Performable
62
+
63
+ def self.before_enqueue_foo(*args)
64
+ puts "!!before_enqueue_foo!! #{args.inspect}"
65
+ end
66
+
67
+ def self.before_enqueue_bar(*args)
68
+ puts "!!before_enqueue_bar!! #{args.inspect}"
69
+ end
70
+
71
+ def self.after_enqueue_foo(*args)
72
+ puts "!!after_enqueue_foo!! #{args.inspect}"
73
+ end
74
+
75
+ def self.after_enqueue_bar(*args)
76
+ puts "!!after_enqueue_bar!! #{args.inspect}"
77
+ end
78
+
79
+ def self.before_perform_foo(*args)
80
+ puts "!!before_perform_foo!! #{args.inspect}"
81
+ end
82
+
83
+ def self.after_perform_foo(*args)
84
+ puts "!!after_perform_foo!! #{args.inspect}"
85
+ end
86
+
87
+ def self.around_perform_bar(*args)
88
+ puts "!!BEGIN around_perform_bar!! #{args.inspect}"
89
+ yield
90
+ puts "!!END around_perform_bar!! #{args.inspect}"
91
+ end
92
+
93
+ def self.around_perform_cat(*args)
94
+ puts "!!BEGIN around_perform_cat!! #{args.inspect}"
95
+ yield
96
+ puts "!!END around_perform_cat!! #{args.inspect}"
97
+ end
98
+
99
+ def self.on_failure_foo(ex, *args)
100
+ puts "!!on_failure_foo!! #{ex.inspect} #{args.inspect}"
101
+ end
102
+
103
+ def self.on_bury_foo(*args)
104
+ puts "!!on_bury_foo!! #{args.inspect}"
105
+ end
106
+
107
+ def self.on_retry_foo(retry_count, delay, *args)
108
+ puts "!!on_retry_foo!! #{retry_count} #{delay} #{args.inspect}"
109
+ end
110
+
111
+ def self.foo(x)
112
+ $hooked_fail_count += 1
113
+ raise HookFailError, "Fail!" if $hooked_fail_count == 1
114
+ puts "This is the job running successfully!! #{x.inspect}"
115
+ end
116
+ end # HookedObjectSuccess
117
+
118
+ class HookedWorker < Backburner::Worker
119
+ def on_reconnect
120
+ puts "!!on_reconnect!!"
121
+ end
122
+ end
@@ -0,0 +1,72 @@
1
+ class ResponseJob
2
+ include Backburner::Queue
3
+ queue_priority 1000
4
+ def self.perform(data)
5
+ $worker_test_count += data['worker_test_count'].to_i if data['worker_test_count']
6
+ $worker_success = data['worker_success'] if data['worker_success']
7
+ $worker_test_count = data['worker_test_count_set'].to_i if data['worker_test_count_set']
8
+ $worker_raise = data['worker_raise'] if data['worker_raise']
9
+ end
10
+ end
11
+
12
+ class TestJobFork
13
+ include Backburner::Queue
14
+ queue "test-job-fork"
15
+ queue_priority 1000
16
+ def self.perform(x, y)
17
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
18
+ :worker_test_count_set => x + y
19
+ }], :queue => 'response'
20
+ end
21
+ end
22
+
23
+ class TestFailJobFork
24
+ include Backburner::Queue
25
+ queue "test-fail-job-fork"
26
+ def self.perform(x, y)
27
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
28
+ :worker_raise => true
29
+ }], :queue => 'response'
30
+ end
31
+ end
32
+
33
+ class TestRetryJobFork
34
+ include Backburner::Queue
35
+ def self.perform(x, y)
36
+ $worker_test_count += 1
37
+
38
+ if $worker_test_count <= 2
39
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
40
+ :worker_test_count => 1
41
+ }], :queue => 'response'
42
+
43
+ raise RuntimeError
44
+ else # succeeds
45
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
46
+ :worker_test_count => 1,
47
+ :worker_success => true
48
+ }], :queue => 'response'
49
+ end
50
+ end
51
+ end
52
+
53
+ class TestAsyncJobFork
54
+ include Backburner::Performable
55
+ def self.foo(x, y)
56
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
57
+ :worker_test_count_set => x * y
58
+ }], :queue => 'response'
59
+ end
60
+ end
61
+
62
+ class TestJobMultithreadFork
63
+ include Backburner::Queue
64
+ queue "test-job-multithread-fork"
65
+ queue_priority 1000
66
+ def self.perform(x, y)
67
+ sleep 1 # simluate work
68
+ Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
69
+ :worker_test_count_set => x + y
70
+ }], :queue => 'response'
71
+ end
72
+ end