backburner 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +205 -0
- data/Rakefile +15 -0
- data/TODO +5 -0
- data/backburner.gemspec +25 -0
- data/bin/backburner +10 -0
- data/examples/custom.rb +25 -0
- data/examples/demo.rb +60 -0
- data/examples/simple.rb +43 -0
- data/lib/backburner.rb +51 -0
- data/lib/backburner/async_proxy.rb +22 -0
- data/lib/backburner/configuration.rb +19 -0
- data/lib/backburner/connection.rb +43 -0
- data/lib/backburner/helpers.rb +99 -0
- data/lib/backburner/logger.rb +44 -0
- data/lib/backburner/performable.rb +40 -0
- data/lib/backburner/queue.rb +22 -0
- data/lib/backburner/tasks.rb +11 -0
- data/lib/backburner/version.rb +3 -0
- data/lib/backburner/worker.rb +140 -0
- data/test/back_burner_test.rb +61 -0
- data/test/connection_test.rb +35 -0
- data/test/helpers_test.rb +79 -0
- data/test/logger_test.rb +19 -0
- data/test/performable_test.rb +38 -0
- data/test/queue_test.rb +28 -0
- data/test/test_helper.rb +93 -0
- data/test/worker_test.rb +163 -0
- metadata +181 -0
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Nathan Esquenazi
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
# Backburner
|
2
|
+
|
3
|
+
Backburner is a beanstalkd-powered job queue designed to be as simple and easy to use as possible.
|
4
|
+
You can create background jobs, place those on specialized queues and process them later.
|
5
|
+
|
6
|
+
Processing background jobs reliably has never been easier. Backburner works with any ruby-based
|
7
|
+
web framework but is well-suited for use with [Sinatra](http://sinatrarb.com) and [Padrino](http://padrinorb.com).
|
8
|
+
|
9
|
+
If you want to use beanstalk for job processing, consider using Backburner. Backburner is heavily inspired by Resque and DelayedJob.
|
10
|
+
Backburner can be a persistent queue if the beanstalk persistence mode is enabled, supports priority, delays, and timeouts.
|
11
|
+
Backburner stores jobs as simple JSON payloads.
|
12
|
+
|
13
|
+
## Installation
|
14
|
+
|
15
|
+
Add this line to your application's Gemfile:
|
16
|
+
|
17
|
+
gem 'backburner'
|
18
|
+
|
19
|
+
And then execute:
|
20
|
+
|
21
|
+
$ bundle
|
22
|
+
|
23
|
+
Or install it yourself as:
|
24
|
+
|
25
|
+
$ gem install backburner
|
26
|
+
|
27
|
+
## Configuration ##
|
28
|
+
|
29
|
+
Backburner is extremely simple to setup. Just configure basic settings for backburner:
|
30
|
+
|
31
|
+
```ruby
|
32
|
+
Backburner.configure do |config|
|
33
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
34
|
+
config.tube_namespace = "some.app.production"
|
35
|
+
config.on_error = lambda { |e| puts e }
|
36
|
+
config.default_priority = 65536
|
37
|
+
config.respond_timeout = 120
|
38
|
+
end
|
39
|
+
```
|
40
|
+
|
41
|
+
## Usage
|
42
|
+
|
43
|
+
Backburner allows you to create jobs and place them on a beanstalk queue, and later pull those jobs off the queue and
|
44
|
+
process them asynchronously.
|
45
|
+
|
46
|
+
### Enqueuing Jobs ###
|
47
|
+
|
48
|
+
At the core, Backburner is about jobs that can be processed. Jobs are simple ruby objects with a method defined named `perform`.
|
49
|
+
|
50
|
+
Any object which responds to `perform` can be queued as a job. Job objects are queued as JSON to be later processed by a task runner.
|
51
|
+
Here's an example:
|
52
|
+
|
53
|
+
```ruby
|
54
|
+
class NewsletterJob
|
55
|
+
include Backburner::Queue
|
56
|
+
queue "newsletter"
|
57
|
+
|
58
|
+
def self.perform(email, body)
|
59
|
+
NewsletterMailer.deliver_text_to_email(email, body)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
```
|
63
|
+
|
64
|
+
Notice that you must include the `Backburner::Queue` module and that you can set a `queue` name within the job automatically.
|
65
|
+
Jobs can then be enqueued using:
|
66
|
+
|
67
|
+
```ruby
|
68
|
+
Backburner.enqueue NewsletterJob, 'lorem ipsum...', 5
|
69
|
+
```
|
70
|
+
|
71
|
+
`Backburner.enqueue` accepts first a ruby object that supports `perform` and then a series of parameters
|
72
|
+
to that object's `perform` method. The queue name used by default is the normalized class name (i.e `{namespace}.newsletter-job`)
|
73
|
+
if not otherwise specified.
|
74
|
+
|
75
|
+
### Simple Async Jobs ###
|
76
|
+
|
77
|
+
In addition to defining custom jobs, a job can also be enqueued by invoking the `async` method on any object which
|
78
|
+
includes `Backburner::Performable`.
|
79
|
+
|
80
|
+
```ruby
|
81
|
+
class User
|
82
|
+
include Backburner::Performable
|
83
|
+
|
84
|
+
def activate(device_id)
|
85
|
+
@device = Device.find(device_id)
|
86
|
+
# ...
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
@user = User.first
|
91
|
+
@user.async(:pri => 1000, :ttr => 100, :queue => "user.activate").activate(@device.id)
|
92
|
+
```
|
93
|
+
|
94
|
+
This will automatically enqueue a job that will run `activate` with the specified argument for that user record.
|
95
|
+
The queue name used by default is the normalized class name (i.e `{namespace}.user`) if not otherwise specified.
|
96
|
+
Note you are able to pass `pri`, `ttr`, `delay` and `queue` directly as options into `async`.
|
97
|
+
|
98
|
+
### Working Jobs
|
99
|
+
|
100
|
+
Backburner workers are processes that run forever handling jobs that get reserved. Starting a worker in ruby code is simple:
|
101
|
+
|
102
|
+
```ruby
|
103
|
+
Backburner.work
|
104
|
+
```
|
105
|
+
|
106
|
+
This will process jobs in all queues but you can also restrict processing to specific queues:
|
107
|
+
|
108
|
+
```ruby
|
109
|
+
Backburner.work('newsletter_sender')
|
110
|
+
```
|
111
|
+
|
112
|
+
The Backburner worker also exists as a rake task:
|
113
|
+
|
114
|
+
```ruby
|
115
|
+
require 'backburner/tasks'
|
116
|
+
```
|
117
|
+
|
118
|
+
so you can run:
|
119
|
+
|
120
|
+
```
|
121
|
+
$ QUEUES=newsletter-sender,push-message rake backburner:work
|
122
|
+
```
|
123
|
+
|
124
|
+
You can also run the backburner binary for a convenient worker:
|
125
|
+
|
126
|
+
```
|
127
|
+
bundle exec backburner newsletter-sender,push-message -d -P /var/run/backburner.pid -l /var/log/backburner.log
|
128
|
+
```
|
129
|
+
|
130
|
+
This will daemonize the worker and store the pid and logs automatically.
|
131
|
+
|
132
|
+
### Default Queues
|
133
|
+
|
134
|
+
Workers can be easily restricted to processing only a specific set of queues as shown above. However, if you want a worker to
|
135
|
+
process **all** queues instead, then you can leave the queue list blank.
|
136
|
+
|
137
|
+
When you execute a worker without queues specified, any queue for a known job queue class with `include Backburner::Queue` will be processed. To access the list of known
|
138
|
+
queue classes, you can use:
|
139
|
+
|
140
|
+
```ruby
|
141
|
+
Backburner::Worker.known_queue_classes
|
142
|
+
# => [NewsletterJob, SomeOtherJob]
|
143
|
+
```
|
144
|
+
|
145
|
+
Dynamic queues created by passing queue options **will not be processed** by a default worker. For this reason, you may want to take control over the default list of
|
146
|
+
queues processed when none are specified. To do this, you can use the `default_queues` class method:
|
147
|
+
|
148
|
+
```ruby
|
149
|
+
Backburner.default_queues.concat(["foo", "bar"])
|
150
|
+
```
|
151
|
+
|
152
|
+
This will ensure that the _foo_ and _bar_ queues are processed by default. You can also add job queue names:
|
153
|
+
|
154
|
+
```ruby
|
155
|
+
Backburner.default_queues << NewsletterJob.queue
|
156
|
+
```
|
157
|
+
|
158
|
+
The `default_queues` stores the specific list of queues that should be processed by default by a worker.
|
159
|
+
|
160
|
+
### Failures
|
161
|
+
|
162
|
+
You can setup the error handler for jobs using configure:
|
163
|
+
|
164
|
+
```ruby
|
165
|
+
Backburner.configure do |config|
|
166
|
+
config.on_error = lambda { |ex| Airbrake.notify(ex) }
|
167
|
+
end
|
168
|
+
```
|
169
|
+
|
170
|
+
Now all beanstalk queue errors will show up on airbrake.
|
171
|
+
If a job fails in beanstalk, the job is automatically buried and must be 'kicked' later.
|
172
|
+
|
173
|
+
### Logging
|
174
|
+
|
175
|
+
Right now, all logging happens to standard out and can be piped to a file or any other output manually. More on logging coming later.
|
176
|
+
|
177
|
+
### Front-end Monitoring
|
178
|
+
|
179
|
+
To be completed is an admin dashboard that provides insight into beanstalk jobs via a simple Sinatra front-end. Coming soon.
|
180
|
+
|
181
|
+
## Why Backburner?
|
182
|
+
|
183
|
+
To be filled in. DelayedJob, Resque, Stalker, et al.
|
184
|
+
|
185
|
+
## Acknowledgements
|
186
|
+
|
187
|
+
* Nathan Esquenazi - Project maintainer
|
188
|
+
* Kristen Tucker - Coming up with the gem name
|
189
|
+
* [Tim Lee](https://github.com/timothy1ee), [Josh Hull](https://github.com/joshbuddy), [Nico Taing](https://github.com/Nico-Taing) - Helping me work through the idea
|
190
|
+
* [Miso](http://gomiso.com) - Open-source friendly place to work
|
191
|
+
|
192
|
+
## Contributing
|
193
|
+
|
194
|
+
1. Fork it
|
195
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
196
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
197
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
198
|
+
5. Create new Pull Request
|
199
|
+
|
200
|
+
## References
|
201
|
+
|
202
|
+
The code in this project has been adapted from a few excellent projects:
|
203
|
+
|
204
|
+
* [DelayedJob](https://github.com/collectiveidea/delayed_job)
|
205
|
+
* [Stalker](https://github.com/han/stalker)
|
data/Rakefile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rake/testtask'
|
4
|
+
# require 'yard'
|
5
|
+
|
6
|
+
task :test do
|
7
|
+
Rake::TestTask.new do |t|
|
8
|
+
t.libs.push "lib"
|
9
|
+
t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# task :doc do
|
15
|
+
# YARD::CLI::Yardoc.new.run
|
data/TODO
ADDED
@@ -0,0 +1,5 @@
|
|
1
|
+
- Better way to enummerate job names in worker (right now using the tubes that exist but they may not exist even tho they will)
|
2
|
+
- Use a better way to aggregate tube names for different jobs
|
3
|
+
- Front-end in sinatra for viewing beanstalk jobs
|
4
|
+
- Better logging
|
5
|
+
- Better error handling (failures, retries)
|
data/backburner.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/backburner/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.authors = ["Nathan Esquenazi"]
|
6
|
+
s.email = ["nesquena@gmail.com"]
|
7
|
+
s.description = %q{Beanstalk background job processing made easy}
|
8
|
+
s.summary = %q{Reliable beanstalk background job processing made easy for Ruby and Sinatra}
|
9
|
+
s.homepage = "http://github.com/nesquena/backburner"
|
10
|
+
|
11
|
+
s.files = `git ls-files`.split($\)
|
12
|
+
s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
14
|
+
s.name = "backburner"
|
15
|
+
s.require_paths = ["lib"]
|
16
|
+
s.version = Backburner::VERSION
|
17
|
+
|
18
|
+
s.add_runtime_dependency(%q<beanstalk-client>, [">= 0"])
|
19
|
+
s.add_runtime_dependency(%q<json_pure>, [">= 0"])
|
20
|
+
s.add_runtime_dependency(%q<dante>)
|
21
|
+
|
22
|
+
s.add_development_dependency 'rake'
|
23
|
+
s.add_development_dependency 'minitest'
|
24
|
+
s.add_development_dependency 'mocha'
|
25
|
+
end
|
data/bin/backburner
ADDED
data/examples/custom.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
# Define ruby job
|
5
|
+
class TestJob
|
6
|
+
include Backburner::Queue
|
7
|
+
# queue "test-job"
|
8
|
+
|
9
|
+
def self.perform(value, user)
|
10
|
+
puts "[TestJob] Running perform with args: [#{value}, #{user}]"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Configure Backburner
|
15
|
+
Backburner.configure do |config|
|
16
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
17
|
+
config.tube_namespace = "demo.production"
|
18
|
+
end
|
19
|
+
|
20
|
+
# Enqueue tasks
|
21
|
+
Backburner.enqueue TestJob, 5, 3
|
22
|
+
Backburner.enqueue TestJob, 10, 6
|
23
|
+
|
24
|
+
# Work tasks
|
25
|
+
Backburner.work("test-job")
|
data/examples/demo.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
module Tester
|
5
|
+
class TestJob
|
6
|
+
include Backburner::Queue
|
7
|
+
queue "test.job"
|
8
|
+
|
9
|
+
def self.perform(value, user)
|
10
|
+
p [value, user]
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class UserModel
|
15
|
+
include Backburner::Performable
|
16
|
+
|
17
|
+
attr_accessor :id, :name
|
18
|
+
|
19
|
+
def self.first
|
20
|
+
self.find(3, "John")
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.find(id, name="Fetched")
|
24
|
+
self.new(id, name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize(id, name)
|
28
|
+
@id, @name = id, name
|
29
|
+
end
|
30
|
+
|
31
|
+
def hello(x, y)
|
32
|
+
puts "Instance #{x} and #{y} and my id is #{id}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.foo(x, y)
|
36
|
+
puts "Class #{x} and #{y}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# connection = Backburner::Connection.new("beanstalk://localhost")
|
42
|
+
|
43
|
+
Backburner.configure do |config|
|
44
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
45
|
+
config.tube_namespace = "myblog.production"
|
46
|
+
end
|
47
|
+
|
48
|
+
# p Backburner.configuration.beanstalk_url
|
49
|
+
# p Backburner::Worker.connection
|
50
|
+
|
51
|
+
Backburner.enqueue Tester::TestJob, 5, 3
|
52
|
+
Backburner.enqueue Tester::TestJob, 10, 6
|
53
|
+
@user = Tester::UserModel.first
|
54
|
+
@user.async.hello("foo", "bar")
|
55
|
+
Tester::UserModel.async.foo("bar", "baz")
|
56
|
+
|
57
|
+
Backburner.default_queues.concat([Tester::TestJob.queue, Tester::UserModel.queue])
|
58
|
+
Backburner.work
|
59
|
+
# Backburner.work("test.job")
|
60
|
+
# Backburner.work("tester/user-model")
|
data/examples/simple.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
class User
|
5
|
+
include Backburner::Performable
|
6
|
+
attr_accessor :id, :name
|
7
|
+
|
8
|
+
def self.first
|
9
|
+
User.find(3, "John")
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.find(id, name="Fetched")
|
13
|
+
User.new(id, name)
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(id, name)
|
17
|
+
@id, @name = id, name
|
18
|
+
end
|
19
|
+
|
20
|
+
def hello(x, y)
|
21
|
+
puts "User(id=#{id}) #hello args: [#{x}, #{y}] (Instance method)"
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.foo(x, y)
|
25
|
+
puts "User #foo args [#{x}, #{y}] (Class method)"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Configure Backburner
|
30
|
+
Backburner.configure do |config|
|
31
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
32
|
+
config.tube_namespace = "demo.production"
|
33
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
34
|
+
end
|
35
|
+
|
36
|
+
# Enqueue tasks
|
37
|
+
@user = User.first
|
38
|
+
@user.async(:pri => 1000).hello("foo", "bar")
|
39
|
+
User.async.foo("bar", "baz")
|
40
|
+
|
41
|
+
# Run work
|
42
|
+
# Backburner.default_queues << "user"
|
43
|
+
Backburner.work
|