backburner-allq 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.travis.yml +29 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +37 -0
- data/Gemfile +4 -0
- data/HOOKS.md +99 -0
- data/LICENSE +22 -0
- data/README.md +658 -0
- data/Rakefile +17 -0
- data/TODO +4 -0
- data/backburner-allq.gemspec +26 -0
- data/bin/backburner +7 -0
- data/circle.yml +3 -0
- data/deploy.sh +3 -0
- data/examples/custom.rb +25 -0
- data/examples/demo.rb +60 -0
- data/examples/god.rb +46 -0
- data/examples/hooked.rb +87 -0
- data/examples/retried.rb +31 -0
- data/examples/simple.rb +43 -0
- data/examples/stress.rb +31 -0
- data/lib/backburner.rb +75 -0
- data/lib/backburner/allq_wrapper.rb +317 -0
- data/lib/backburner/async_proxy.rb +25 -0
- data/lib/backburner/cli.rb +53 -0
- data/lib/backburner/configuration.rb +48 -0
- data/lib/backburner/connection.rb +157 -0
- data/lib/backburner/helpers.rb +193 -0
- data/lib/backburner/hooks.rb +53 -0
- data/lib/backburner/job.rb +118 -0
- data/lib/backburner/logger.rb +53 -0
- data/lib/backburner/performable.rb +95 -0
- data/lib/backburner/queue.rb +145 -0
- data/lib/backburner/tasks.rb +54 -0
- data/lib/backburner/version.rb +3 -0
- data/lib/backburner/worker.rb +221 -0
- data/lib/backburner/workers/forking.rb +52 -0
- data/lib/backburner/workers/simple.rb +29 -0
- data/lib/backburner/workers/threading.rb +163 -0
- data/lib/backburner/workers/threads_on_fork.rb +263 -0
- data/test/async_proxy_test.rb +36 -0
- data/test/back_burner_test.rb +88 -0
- data/test/connection_test.rb +179 -0
- data/test/fixtures/hooked.rb +122 -0
- data/test/fixtures/test_fork_jobs.rb +72 -0
- data/test/fixtures/test_forking_jobs.rb +56 -0
- data/test/fixtures/test_jobs.rb +87 -0
- data/test/fixtures/test_queue_settings.rb +14 -0
- data/test/helpers/templogger.rb +22 -0
- data/test/helpers_test.rb +278 -0
- data/test/hooks_test.rb +112 -0
- data/test/job_test.rb +185 -0
- data/test/logger_test.rb +44 -0
- data/test/performable_test.rb +88 -0
- data/test/queue_test.rb +69 -0
- data/test/test_helper.rb +128 -0
- data/test/worker_test.rb +157 -0
- data/test/workers/forking_worker_test.rb +181 -0
- data/test/workers/simple_worker_test.rb +350 -0
- data/test/workers/threading_worker_test.rb +104 -0
- data/test/workers/threads_on_fork_worker_test.rb +484 -0
- metadata +217 -0
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rake/testtask'
|
4
|
+
# require 'yard'
|
5
|
+
|
6
|
+
task :test do
|
7
|
+
Rake::TestTask.new do |t|
|
8
|
+
t.libs.push "lib"
|
9
|
+
t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
task :default => :test
|
15
|
+
|
16
|
+
# task :doc do
|
17
|
+
# YARD::CLI::Yardoc.new.run
|
data/TODO
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/backburner/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.authors = ["Jason Malcolm"]
|
6
|
+
s.email = ["jason@blitline.com"]
|
7
|
+
s.description = %q{Beanstalk background job processing made easy}
|
8
|
+
s.summary = %q{Reliable allq background job processing made easy for Ruby and Sinatra}
|
9
|
+
s.homepage = "http://github.com/nesquena/backburner"
|
10
|
+
|
11
|
+
s.files = `git ls-files`.split($\)
|
12
|
+
s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
14
|
+
s.name = "backburner-allq"
|
15
|
+
s.require_paths = ["lib"]
|
16
|
+
s.version = Backburner::VERSION
|
17
|
+
s.license = 'MIT'
|
18
|
+
|
19
|
+
s.add_runtime_dependency 'allq_rest', '~> 1.0'
|
20
|
+
s.add_runtime_dependency 'dante', '> 0.1.5'
|
21
|
+
s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
|
22
|
+
|
23
|
+
s.add_development_dependency 'rake'
|
24
|
+
s.add_development_dependency 'minitest', '3.2.0'
|
25
|
+
s.add_development_dependency 'mocha'
|
26
|
+
end
|
data/bin/backburner
ADDED
data/circle.yml
ADDED
data/deploy.sh
ADDED
data/examples/custom.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
# Define ruby job
|
5
|
+
class TestJob
|
6
|
+
include Backburner::Queue
|
7
|
+
# queue "test-job"
|
8
|
+
|
9
|
+
def self.perform(value, user)
|
10
|
+
puts "[TestJob] Running perform with args: [#{value}, #{user}]"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Configure Backburner
|
15
|
+
Backburner.configure do |config|
|
16
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
17
|
+
config.tube_namespace = "demo.production"
|
18
|
+
end
|
19
|
+
|
20
|
+
# Enqueue tasks
|
21
|
+
Backburner.enqueue TestJob, 5, 3
|
22
|
+
Backburner.enqueue TestJob, 10, 6
|
23
|
+
|
24
|
+
# Work tasks using threaded worker
|
25
|
+
Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)
|
data/examples/demo.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
module Tester
|
5
|
+
class TestJob
|
6
|
+
include Backburner::Queue
|
7
|
+
queue "test.job"
|
8
|
+
|
9
|
+
def self.perform(value, user)
|
10
|
+
p [value, user]
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class UserModel
|
15
|
+
include Backburner::Performable
|
16
|
+
|
17
|
+
attr_accessor :id, :name
|
18
|
+
|
19
|
+
def self.first
|
20
|
+
self.find(3, "John")
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.find(id, name="Fetched")
|
24
|
+
self.new(id, name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize(id, name)
|
28
|
+
@id, @name = id, name
|
29
|
+
end
|
30
|
+
|
31
|
+
def hello(x, y)
|
32
|
+
puts "Instance #{x} and #{y} and my id is #{id}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.foo(x, y)
|
36
|
+
puts "Class #{x} and #{y}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# connection = Backburner::Connection.new("beanstalk://127.0.0.1")
|
42
|
+
|
43
|
+
Backburner.configure do |config|
|
44
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
45
|
+
config.tube_namespace = "myblog.production"
|
46
|
+
end
|
47
|
+
|
48
|
+
# p Backburner.configuration.beanstalk_url
|
49
|
+
# p Backburner::Worker.connection
|
50
|
+
|
51
|
+
Backburner.enqueue Tester::TestJob, 5, 3
|
52
|
+
Backburner.enqueue Tester::TestJob, 10, 6
|
53
|
+
@user = Tester::UserModel.first
|
54
|
+
@user.async.hello("foo", "bar")
|
55
|
+
Tester::UserModel.async.foo("bar", "baz")
|
56
|
+
|
57
|
+
Backburner.default_queues.concat([Tester::TestJob.queue, Tester::UserModel.queue])
|
58
|
+
Backburner.work
|
59
|
+
# Backburner.work("test.job")
|
60
|
+
# Backburner.work("tester/user-model")
|
data/examples/god.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
God.watch do |w|
|
2
|
+
w.name = "backburner-worker-1"
|
3
|
+
w.dir = '/path/to/app/dir'
|
4
|
+
w.env = { 'PADRINO_ENV' => 'production', 'QUEUES' => 'newsletter-sender,push-message' }
|
5
|
+
w.group = 'backburner-workers'
|
6
|
+
w.interval = 30.seconds
|
7
|
+
w.start = "bundle exec rake -f Rakefile backburner:work"
|
8
|
+
w.log = "/var/log/god/backburner-worker-1.log"
|
9
|
+
|
10
|
+
# restart if memory gets too high
|
11
|
+
w.transition(:up, :restart) do |on|
|
12
|
+
on.condition(:memory_usage) do |c|
|
13
|
+
c.above = 50.megabytes
|
14
|
+
c.times = 3
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# determine the state on startup
|
19
|
+
w.transition(:init, { true => :up, false => :start }) do |on|
|
20
|
+
on.condition(:process_running) do |c|
|
21
|
+
c.running = true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# determine when process has finished starting
|
26
|
+
w.transition([:start, :restart], :up) do |on|
|
27
|
+
on.condition(:process_running) do |c|
|
28
|
+
c.running = true
|
29
|
+
c.interval = 5.seconds
|
30
|
+
end
|
31
|
+
|
32
|
+
# failsafe
|
33
|
+
on.condition(:tries) do |c|
|
34
|
+
c.times = 5
|
35
|
+
c.transition = :start
|
36
|
+
c.interval = 5.seconds
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# start if process is not running
|
41
|
+
w.transition(:up, :start) do |on|
|
42
|
+
on.condition(:process_running) do |c|
|
43
|
+
c.running = false
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/examples/hooked.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
$fail = 0
|
5
|
+
class User
|
6
|
+
include Backburner::Performable
|
7
|
+
|
8
|
+
# Called with the job args before a job is placed on the queue.
|
9
|
+
# !! If the hook returns `false`, the job will not be placed on the queue.
|
10
|
+
def self.before_enqueue_foo(*args)
|
11
|
+
puts "[before_enqueue] Just about to enqueue #{self} with #{args.inspect}"
|
12
|
+
end
|
13
|
+
|
14
|
+
# Called with the job args after a job is placed on the queue.
|
15
|
+
# !! Any exception raised propagates up to the code which queued the job.
|
16
|
+
def self.after_enqueue_foo(*args)
|
17
|
+
puts "[after_enqueue] Finished enqueuing #{self} with #{args.inspect}"
|
18
|
+
end
|
19
|
+
|
20
|
+
# Called with the job args before perform. If it raises
|
21
|
+
# `Backburner::Job::DontPerform`, the job is aborted. Other exceptions
|
22
|
+
# are treated like regular job exceptions.
|
23
|
+
def self.before_perform_foo(*args)
|
24
|
+
puts "[before_perform] Just about to perform #{self} with #{args.inspect}"
|
25
|
+
end
|
26
|
+
|
27
|
+
# Called with the job args after it performs. Uncaught
|
28
|
+
# exceptions will be treated like regular job exceptions.
|
29
|
+
def self.after_perform_foo(*args)
|
30
|
+
puts "[after_perform] Just finished performing #{self} with #{args.inspect}"
|
31
|
+
end
|
32
|
+
|
33
|
+
# Called with the job args. It is expected to yield in order
|
34
|
+
# to perform the job (but is not required to do so). It may handle exceptions
|
35
|
+
# thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
|
36
|
+
def self.around_perform_bar(*args)
|
37
|
+
puts "[around_perform_bar before] About to perform #{self} with #{args.inspect}"
|
38
|
+
yield
|
39
|
+
puts "[around_perform_bar after] Just after performing #{self} with #{args.inspect}"
|
40
|
+
end
|
41
|
+
|
42
|
+
# Called with the job args. It is expected to yield in order
|
43
|
+
# to perform the job (but is not required to do so). It may handle exceptions
|
44
|
+
# thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
|
45
|
+
def self.around_perform_cat(*args)
|
46
|
+
puts "[around_perform_cat before] About to perform #{self} with #{args.inspect}"
|
47
|
+
yield
|
48
|
+
puts "[around_perform_cat after] Just after performing #{self} with #{args.inspect}"
|
49
|
+
end
|
50
|
+
|
51
|
+
# Called with the job args. It is expected to yield in order
|
52
|
+
# to perform the job (but is not required to do so). It may handle exceptions
|
53
|
+
# thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
|
54
|
+
def self.around_perform_foo(*args)
|
55
|
+
puts "[around_perform_foo before] About to perform #{self} with #{args.inspect}"
|
56
|
+
yield
|
57
|
+
puts "[around_perform_foo after] Just after performing #{self} with #{args.inspect}"
|
58
|
+
end
|
59
|
+
|
60
|
+
# Called with the exception and job args if any exception occurs
|
61
|
+
# while performing the job (or hooks).
|
62
|
+
def self.on_failure_foo(ex, *args)
|
63
|
+
puts "[on_failure] Failure #{ex.inspect} occurred for job #{self} with #{args.inspect}"
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.foo
|
67
|
+
$fail += 1
|
68
|
+
raise "Fail!" if $fail == 1
|
69
|
+
puts "This is the job running successfully!!"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# Configure Backburner
|
74
|
+
Backburner.configure do |config|
|
75
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
76
|
+
config.tube_namespace = "demo.production"
|
77
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
78
|
+
config.max_job_retries = 1
|
79
|
+
config.retry_delay = 0
|
80
|
+
end
|
81
|
+
|
82
|
+
# Enqueue tasks
|
83
|
+
User.async.foo
|
84
|
+
|
85
|
+
# Run work
|
86
|
+
# Backburner.default_queues << "user"
|
87
|
+
Backburner.work
|
data/examples/retried.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
$error = 0
|
5
|
+
|
6
|
+
class User
|
7
|
+
include Backburner::Performable
|
8
|
+
attr_accessor :id, :name
|
9
|
+
|
10
|
+
def self.foo(x, y)
|
11
|
+
$error += 1
|
12
|
+
raise "fail #{$error}" unless $error > 3
|
13
|
+
puts "User #foo args [#{x}, #{y}] Success!!"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# Configure Backburner
|
18
|
+
Backburner.configure do |config|
|
19
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
20
|
+
config.tube_namespace = "demo.production"
|
21
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
22
|
+
config.max_job_retries = 3
|
23
|
+
config.retry_delay = 0
|
24
|
+
end
|
25
|
+
|
26
|
+
# Enqueue tasks
|
27
|
+
User.async(:queue => "retried").foo("bar", "baz")
|
28
|
+
|
29
|
+
# Run work
|
30
|
+
# Backburner.default_queues << "user"
|
31
|
+
Backburner.work
|
data/examples/simple.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
class User
|
5
|
+
include Backburner::Performable
|
6
|
+
attr_accessor :id, :name
|
7
|
+
|
8
|
+
def self.first
|
9
|
+
User.find(3, "John")
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.find(id, name="Fetched")
|
13
|
+
User.new(id, name)
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(id, name)
|
17
|
+
@id, @name = id, name
|
18
|
+
end
|
19
|
+
|
20
|
+
def hello(x, y)
|
21
|
+
puts "User(id=#{id}) #hello args: [#{x}, #{y}] (Instance method)"
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.foo(x, y)
|
25
|
+
puts "User #foo args [#{x}, #{y}] (Class method)"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Configure Backburner
|
30
|
+
Backburner.configure do |config|
|
31
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
32
|
+
config.tube_namespace = "demo.production"
|
33
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
34
|
+
end
|
35
|
+
|
36
|
+
# Enqueue tasks
|
37
|
+
@user = User.first
|
38
|
+
@user.async(:pri => 1000).hello("foo", "bar")
|
39
|
+
User.async.foo("bar", "baz")
|
40
|
+
|
41
|
+
# Run work
|
42
|
+
# Backburner.default_queues << "user"
|
43
|
+
Backburner.work
|
data/examples/stress.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
$values = []
|
5
|
+
|
6
|
+
# Define ruby job
|
7
|
+
class TestJob
|
8
|
+
include Backburner::Queue
|
9
|
+
queue "test-job"
|
10
|
+
|
11
|
+
def self.perform(value)
|
12
|
+
puts "[TestJob] Running perform with args: [#{value}]"
|
13
|
+
$values << value
|
14
|
+
puts "#{$values.size} total jobs processed"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Configure Backburner
|
19
|
+
Backburner.configure do |config|
|
20
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
21
|
+
config.tube_namespace = "demo.production"
|
22
|
+
end
|
23
|
+
|
24
|
+
# Enqueue tasks
|
25
|
+
1.upto(1000) do |i|
|
26
|
+
Backburner.enqueue TestJob, i
|
27
|
+
end
|
28
|
+
|
29
|
+
# Work tasks using threads_on_fork worker
|
30
|
+
# twitter tube will have 10 threads, garbage after 1000 executions and retry jobs 1 times.
|
31
|
+
Backburner.work("test-job:10:100:1", :worker => Backburner::Workers::ThreadsOnFork)
|
data/lib/backburner.rb
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'allq_rest'
|
2
|
+
require 'json'
|
3
|
+
require 'uri'
|
4
|
+
require 'timeout'
|
5
|
+
require 'backburner/version'
|
6
|
+
require 'backburner/allq_wrapper'
|
7
|
+
require 'backburner/helpers'
|
8
|
+
require 'backburner/configuration'
|
9
|
+
require 'backburner/logger'
|
10
|
+
require 'backburner/connection'
|
11
|
+
require 'backburner/hooks'
|
12
|
+
require 'backburner/performable'
|
13
|
+
require 'backburner/worker'
|
14
|
+
require 'backburner/workers/simple'
|
15
|
+
require 'backburner/workers/forking'
|
16
|
+
require 'backburner/workers/threads_on_fork'
|
17
|
+
require 'backburner/workers/threading'
|
18
|
+
require 'backburner/queue'
|
19
|
+
|
20
|
+
module Backburner
|
21
|
+
class << self
|
22
|
+
|
23
|
+
# Enqueues a job to be performed with given arguments.
|
24
|
+
#
|
25
|
+
# @example
|
26
|
+
# Backburner.enqueue NewsletterSender, self.id, user.id
|
27
|
+
#
|
28
|
+
def enqueue(job_class, *args)
|
29
|
+
Backburner::Worker.enqueue(job_class, args, {})
|
30
|
+
end
|
31
|
+
|
32
|
+
# Begins working on jobs enqueued with optional tubes specified
|
33
|
+
#
|
34
|
+
# @example
|
35
|
+
# Backburner.work('newsletter_sender', 'test_job')
|
36
|
+
# Backburner.work('newsletter_sender', 'test_job', :worker => NotSimpleWorker)
|
37
|
+
#
|
38
|
+
def work(*tubes)
|
39
|
+
options = tubes.last.is_a?(Hash) ? tubes.pop : {}
|
40
|
+
worker_class = options[:worker] || configuration.default_worker
|
41
|
+
worker_class.start(tubes)
|
42
|
+
end
|
43
|
+
|
44
|
+
# Yields a configuration block
|
45
|
+
#
|
46
|
+
# @example
|
47
|
+
# Backburner.configure do |config|
|
48
|
+
# config.beanstalk_url = "beanstalk://..."
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
def configure(&block)
|
52
|
+
yield(configuration)
|
53
|
+
configuration
|
54
|
+
end
|
55
|
+
|
56
|
+
# Returns the configuration options set for Backburner
|
57
|
+
#
|
58
|
+
# @example
|
59
|
+
# Backburner.configuration.beanstalk_url => false
|
60
|
+
#
|
61
|
+
def configuration
|
62
|
+
@_configuration ||= Configuration.new
|
63
|
+
end
|
64
|
+
|
65
|
+
# Returns the queues that are processed by default if none are specified
|
66
|
+
#
|
67
|
+
# @example
|
68
|
+
# Backburner.default_queues << "foo"
|
69
|
+
# Backburner.default_queues => ["foo", "bar"]
|
70
|
+
#
|
71
|
+
def default_queues
|
72
|
+
configuration.default_queues
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|