backburner-allq 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.travis.yml +29 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +37 -0
- data/Gemfile +4 -0
- data/HOOKS.md +99 -0
- data/LICENSE +22 -0
- data/README.md +658 -0
- data/Rakefile +17 -0
- data/TODO +4 -0
- data/backburner-allq.gemspec +26 -0
- data/bin/backburner +7 -0
- data/circle.yml +3 -0
- data/deploy.sh +3 -0
- data/examples/custom.rb +25 -0
- data/examples/demo.rb +60 -0
- data/examples/god.rb +46 -0
- data/examples/hooked.rb +87 -0
- data/examples/retried.rb +31 -0
- data/examples/simple.rb +43 -0
- data/examples/stress.rb +31 -0
- data/lib/backburner.rb +75 -0
- data/lib/backburner/allq_wrapper.rb +317 -0
- data/lib/backburner/async_proxy.rb +25 -0
- data/lib/backburner/cli.rb +53 -0
- data/lib/backburner/configuration.rb +48 -0
- data/lib/backburner/connection.rb +157 -0
- data/lib/backburner/helpers.rb +193 -0
- data/lib/backburner/hooks.rb +53 -0
- data/lib/backburner/job.rb +118 -0
- data/lib/backburner/logger.rb +53 -0
- data/lib/backburner/performable.rb +95 -0
- data/lib/backburner/queue.rb +145 -0
- data/lib/backburner/tasks.rb +54 -0
- data/lib/backburner/version.rb +3 -0
- data/lib/backburner/worker.rb +221 -0
- data/lib/backburner/workers/forking.rb +52 -0
- data/lib/backburner/workers/simple.rb +29 -0
- data/lib/backburner/workers/threading.rb +163 -0
- data/lib/backburner/workers/threads_on_fork.rb +263 -0
- data/test/async_proxy_test.rb +36 -0
- data/test/back_burner_test.rb +88 -0
- data/test/connection_test.rb +179 -0
- data/test/fixtures/hooked.rb +122 -0
- data/test/fixtures/test_fork_jobs.rb +72 -0
- data/test/fixtures/test_forking_jobs.rb +56 -0
- data/test/fixtures/test_jobs.rb +87 -0
- data/test/fixtures/test_queue_settings.rb +14 -0
- data/test/helpers/templogger.rb +22 -0
- data/test/helpers_test.rb +278 -0
- data/test/hooks_test.rb +112 -0
- data/test/job_test.rb +185 -0
- data/test/logger_test.rb +44 -0
- data/test/performable_test.rb +88 -0
- data/test/queue_test.rb +69 -0
- data/test/test_helper.rb +128 -0
- data/test/worker_test.rb +157 -0
- data/test/workers/forking_worker_test.rb +181 -0
- data/test/workers/simple_worker_test.rb +350 -0
- data/test/workers/threading_worker_test.rb +104 -0
- data/test/workers/threads_on_fork_worker_test.rb +484 -0
- metadata +217 -0
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require 'rake/testtask'
|
4
|
+
# require 'yard'
|
5
|
+
|
6
|
+
task :test do
|
7
|
+
Rake::TestTask.new do |t|
|
8
|
+
t.libs.push "lib"
|
9
|
+
t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
|
10
|
+
t.verbose = true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
task :default => :test
|
15
|
+
|
16
|
+
# task :doc do
|
17
|
+
# YARD::CLI::Yardoc.new.run
|
data/TODO
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/backburner/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.authors = ["Jason Malcolm"]
|
6
|
+
s.email = ["jason@blitline.com"]
|
7
|
+
s.description = %q{Beanstalk background job processing made easy}
|
8
|
+
s.summary = %q{Reliable allq background job processing made easy for Ruby and Sinatra}
|
9
|
+
s.homepage = "http://github.com/nesquena/backburner"
|
10
|
+
|
11
|
+
s.files = `git ls-files`.split($\)
|
12
|
+
s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
14
|
+
s.name = "backburner-allq"
|
15
|
+
s.require_paths = ["lib"]
|
16
|
+
s.version = Backburner::VERSION
|
17
|
+
s.license = 'MIT'
|
18
|
+
|
19
|
+
s.add_runtime_dependency 'allq_rest', '~> 1.0'
|
20
|
+
s.add_runtime_dependency 'dante', '> 0.1.5'
|
21
|
+
s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
|
22
|
+
|
23
|
+
s.add_development_dependency 'rake'
|
24
|
+
s.add_development_dependency 'minitest', '3.2.0'
|
25
|
+
s.add_development_dependency 'mocha'
|
26
|
+
end
|
data/bin/backburner
ADDED
data/circle.yml
ADDED
data/deploy.sh
ADDED
data/examples/custom.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
# Define ruby job
|
5
|
+
class TestJob
|
6
|
+
include Backburner::Queue
|
7
|
+
# queue "test-job"
|
8
|
+
|
9
|
+
def self.perform(value, user)
|
10
|
+
puts "[TestJob] Running perform with args: [#{value}, #{user}]"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Configure Backburner
|
15
|
+
Backburner.configure do |config|
|
16
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
17
|
+
config.tube_namespace = "demo.production"
|
18
|
+
end
|
19
|
+
|
20
|
+
# Enqueue tasks
|
21
|
+
Backburner.enqueue TestJob, 5, 3
|
22
|
+
Backburner.enqueue TestJob, 10, 6
|
23
|
+
|
24
|
+
# Work tasks using threaded worker
|
25
|
+
Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)
|
data/examples/demo.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
module Tester
|
5
|
+
class TestJob
|
6
|
+
include Backburner::Queue
|
7
|
+
queue "test.job"
|
8
|
+
|
9
|
+
def self.perform(value, user)
|
10
|
+
p [value, user]
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class UserModel
|
15
|
+
include Backburner::Performable
|
16
|
+
|
17
|
+
attr_accessor :id, :name
|
18
|
+
|
19
|
+
def self.first
|
20
|
+
self.find(3, "John")
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.find(id, name="Fetched")
|
24
|
+
self.new(id, name)
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize(id, name)
|
28
|
+
@id, @name = id, name
|
29
|
+
end
|
30
|
+
|
31
|
+
def hello(x, y)
|
32
|
+
puts "Instance #{x} and #{y} and my id is #{id}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.foo(x, y)
|
36
|
+
puts "Class #{x} and #{y}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# connection = Backburner::Connection.new("beanstalk://127.0.0.1")
|
42
|
+
|
43
|
+
Backburner.configure do |config|
|
44
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
45
|
+
config.tube_namespace = "myblog.production"
|
46
|
+
end
|
47
|
+
|
48
|
+
# p Backburner.configuration.beanstalk_url
|
49
|
+
# p Backburner::Worker.connection
|
50
|
+
|
51
|
+
Backburner.enqueue Tester::TestJob, 5, 3
|
52
|
+
Backburner.enqueue Tester::TestJob, 10, 6
|
53
|
+
@user = Tester::UserModel.first
|
54
|
+
@user.async.hello("foo", "bar")
|
55
|
+
Tester::UserModel.async.foo("bar", "baz")
|
56
|
+
|
57
|
+
Backburner.default_queues.concat([Tester::TestJob.queue, Tester::UserModel.queue])
|
58
|
+
Backburner.work
|
59
|
+
# Backburner.work("test.job")
|
60
|
+
# Backburner.work("tester/user-model")
|
data/examples/god.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
God.watch do |w|
|
2
|
+
w.name = "backburner-worker-1"
|
3
|
+
w.dir = '/path/to/app/dir'
|
4
|
+
w.env = { 'PADRINO_ENV' => 'production', 'QUEUES' => 'newsletter-sender,push-message' }
|
5
|
+
w.group = 'backburner-workers'
|
6
|
+
w.interval = 30.seconds
|
7
|
+
w.start = "bundle exec rake -f Rakefile backburner:work"
|
8
|
+
w.log = "/var/log/god/backburner-worker-1.log"
|
9
|
+
|
10
|
+
# restart if memory gets too high
|
11
|
+
w.transition(:up, :restart) do |on|
|
12
|
+
on.condition(:memory_usage) do |c|
|
13
|
+
c.above = 50.megabytes
|
14
|
+
c.times = 3
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# determine the state on startup
|
19
|
+
w.transition(:init, { true => :up, false => :start }) do |on|
|
20
|
+
on.condition(:process_running) do |c|
|
21
|
+
c.running = true
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# determine when process has finished starting
|
26
|
+
w.transition([:start, :restart], :up) do |on|
|
27
|
+
on.condition(:process_running) do |c|
|
28
|
+
c.running = true
|
29
|
+
c.interval = 5.seconds
|
30
|
+
end
|
31
|
+
|
32
|
+
# failsafe
|
33
|
+
on.condition(:tries) do |c|
|
34
|
+
c.times = 5
|
35
|
+
c.transition = :start
|
36
|
+
c.interval = 5.seconds
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# start if process is not running
|
41
|
+
w.transition(:up, :start) do |on|
|
42
|
+
on.condition(:process_running) do |c|
|
43
|
+
c.running = false
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/examples/hooked.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
$fail = 0
|
5
|
+
class User
|
6
|
+
include Backburner::Performable
|
7
|
+
|
8
|
+
# Called with the job args before a job is placed on the queue.
|
9
|
+
# !! If the hook returns `false`, the job will not be placed on the queue.
|
10
|
+
def self.before_enqueue_foo(*args)
|
11
|
+
puts "[before_enqueue] Just about to enqueue #{self} with #{args.inspect}"
|
12
|
+
end
|
13
|
+
|
14
|
+
# Called with the job args after a job is placed on the queue.
|
15
|
+
# !! Any exception raised propagates up to the code which queued the job.
|
16
|
+
def self.after_enqueue_foo(*args)
|
17
|
+
puts "[after_enqueue] Finished enqueuing #{self} with #{args.inspect}"
|
18
|
+
end
|
19
|
+
|
20
|
+
# Called with the job args before perform. If it raises
|
21
|
+
# `Backburner::Job::DontPerform`, the job is aborted. Other exceptions
|
22
|
+
# are treated like regular job exceptions.
|
23
|
+
def self.before_perform_foo(*args)
|
24
|
+
puts "[before_perform] Just about to perform #{self} with #{args.inspect}"
|
25
|
+
end
|
26
|
+
|
27
|
+
# Called with the job args after it performs. Uncaught
|
28
|
+
# exceptions will be treated like regular job exceptions.
|
29
|
+
def self.after_perform_foo(*args)
|
30
|
+
puts "[after_perform] Just finished performing #{self} with #{args.inspect}"
|
31
|
+
end
|
32
|
+
|
33
|
+
# Called with the job args. It is expected to yield in order
|
34
|
+
# to perform the job (but is not required to do so). It may handle exceptions
|
35
|
+
# thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
|
36
|
+
def self.around_perform_bar(*args)
|
37
|
+
puts "[around_perform_bar before] About to perform #{self} with #{args.inspect}"
|
38
|
+
yield
|
39
|
+
puts "[around_perform_bar after] Just after performing #{self} with #{args.inspect}"
|
40
|
+
end
|
41
|
+
|
42
|
+
# Called with the job args. It is expected to yield in order
|
43
|
+
# to perform the job (but is not required to do so). It may handle exceptions
|
44
|
+
# thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
|
45
|
+
def self.around_perform_cat(*args)
|
46
|
+
puts "[around_perform_cat before] About to perform #{self} with #{args.inspect}"
|
47
|
+
yield
|
48
|
+
puts "[around_perform_cat after] Just after performing #{self} with #{args.inspect}"
|
49
|
+
end
|
50
|
+
|
51
|
+
# Called with the job args. It is expected to yield in order
|
52
|
+
# to perform the job (but is not required to do so). It may handle exceptions
|
53
|
+
# thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
|
54
|
+
def self.around_perform_foo(*args)
|
55
|
+
puts "[around_perform_foo before] About to perform #{self} with #{args.inspect}"
|
56
|
+
yield
|
57
|
+
puts "[around_perform_foo after] Just after performing #{self} with #{args.inspect}"
|
58
|
+
end
|
59
|
+
|
60
|
+
# Called with the exception and job args if any exception occurs
|
61
|
+
# while performing the job (or hooks).
|
62
|
+
def self.on_failure_foo(ex, *args)
|
63
|
+
puts "[on_failure] Failure #{ex.inspect} occurred for job #{self} with #{args.inspect}"
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.foo
|
67
|
+
$fail += 1
|
68
|
+
raise "Fail!" if $fail == 1
|
69
|
+
puts "This is the job running successfully!!"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# Configure Backburner
|
74
|
+
Backburner.configure do |config|
|
75
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
76
|
+
config.tube_namespace = "demo.production"
|
77
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
78
|
+
config.max_job_retries = 1
|
79
|
+
config.retry_delay = 0
|
80
|
+
end
|
81
|
+
|
82
|
+
# Enqueue tasks
|
83
|
+
User.async.foo
|
84
|
+
|
85
|
+
# Run work
|
86
|
+
# Backburner.default_queues << "user"
|
87
|
+
Backburner.work
|
data/examples/retried.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
$error = 0
|
5
|
+
|
6
|
+
class User
|
7
|
+
include Backburner::Performable
|
8
|
+
attr_accessor :id, :name
|
9
|
+
|
10
|
+
def self.foo(x, y)
|
11
|
+
$error += 1
|
12
|
+
raise "fail #{$error}" unless $error > 3
|
13
|
+
puts "User #foo args [#{x}, #{y}] Success!!"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# Configure Backburner
|
18
|
+
Backburner.configure do |config|
|
19
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
20
|
+
config.tube_namespace = "demo.production"
|
21
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
22
|
+
config.max_job_retries = 3
|
23
|
+
config.retry_delay = 0
|
24
|
+
end
|
25
|
+
|
26
|
+
# Enqueue tasks
|
27
|
+
User.async(:queue => "retried").foo("bar", "baz")
|
28
|
+
|
29
|
+
# Run work
|
30
|
+
# Backburner.default_queues << "user"
|
31
|
+
Backburner.work
|
data/examples/simple.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
class User
|
5
|
+
include Backburner::Performable
|
6
|
+
attr_accessor :id, :name
|
7
|
+
|
8
|
+
def self.first
|
9
|
+
User.find(3, "John")
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.find(id, name="Fetched")
|
13
|
+
User.new(id, name)
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(id, name)
|
17
|
+
@id, @name = id, name
|
18
|
+
end
|
19
|
+
|
20
|
+
def hello(x, y)
|
21
|
+
puts "User(id=#{id}) #hello args: [#{x}, #{y}] (Instance method)"
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.foo(x, y)
|
25
|
+
puts "User #foo args [#{x}, #{y}] (Class method)"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# Configure Backburner
|
30
|
+
Backburner.configure do |config|
|
31
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
32
|
+
config.tube_namespace = "demo.production"
|
33
|
+
config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
|
34
|
+
end
|
35
|
+
|
36
|
+
# Enqueue tasks
|
37
|
+
@user = User.first
|
38
|
+
@user.async(:pri => 1000).hello("foo", "bar")
|
39
|
+
User.async.foo("bar", "baz")
|
40
|
+
|
41
|
+
# Run work
|
42
|
+
# Backburner.default_queues << "user"
|
43
|
+
Backburner.work
|
data/examples/stress.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
$:.unshift "lib"
|
2
|
+
require 'backburner'
|
3
|
+
|
4
|
+
$values = []
|
5
|
+
|
6
|
+
# Define ruby job
|
7
|
+
class TestJob
|
8
|
+
include Backburner::Queue
|
9
|
+
queue "test-job"
|
10
|
+
|
11
|
+
def self.perform(value)
|
12
|
+
puts "[TestJob] Running perform with args: [#{value}]"
|
13
|
+
$values << value
|
14
|
+
puts "#{$values.size} total jobs processed"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Configure Backburner
|
19
|
+
Backburner.configure do |config|
|
20
|
+
config.beanstalk_url = "beanstalk://127.0.0.1"
|
21
|
+
config.tube_namespace = "demo.production"
|
22
|
+
end
|
23
|
+
|
24
|
+
# Enqueue tasks
|
25
|
+
1.upto(1000) do |i|
|
26
|
+
Backburner.enqueue TestJob, i
|
27
|
+
end
|
28
|
+
|
29
|
+
# Work tasks using threads_on_fork worker
|
30
|
+
# twitter tube will have 10 threads, garbage after 1000 executions and retry jobs 1 times.
|
31
|
+
Backburner.work("test-job:10:100:1", :worker => Backburner::Workers::ThreadsOnFork)
|
data/lib/backburner.rb
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'allq_rest'
|
2
|
+
require 'json'
|
3
|
+
require 'uri'
|
4
|
+
require 'timeout'
|
5
|
+
require 'backburner/version'
|
6
|
+
require 'backburner/allq_wrapper'
|
7
|
+
require 'backburner/helpers'
|
8
|
+
require 'backburner/configuration'
|
9
|
+
require 'backburner/logger'
|
10
|
+
require 'backburner/connection'
|
11
|
+
require 'backburner/hooks'
|
12
|
+
require 'backburner/performable'
|
13
|
+
require 'backburner/worker'
|
14
|
+
require 'backburner/workers/simple'
|
15
|
+
require 'backburner/workers/forking'
|
16
|
+
require 'backburner/workers/threads_on_fork'
|
17
|
+
require 'backburner/workers/threading'
|
18
|
+
require 'backburner/queue'
|
19
|
+
|
20
|
+
module Backburner
|
21
|
+
class << self
|
22
|
+
|
23
|
+
# Enqueues a job to be performed with given arguments.
|
24
|
+
#
|
25
|
+
# @example
|
26
|
+
# Backburner.enqueue NewsletterSender, self.id, user.id
|
27
|
+
#
|
28
|
+
def enqueue(job_class, *args)
|
29
|
+
Backburner::Worker.enqueue(job_class, args, {})
|
30
|
+
end
|
31
|
+
|
32
|
+
# Begins working on jobs enqueued with optional tubes specified
|
33
|
+
#
|
34
|
+
# @example
|
35
|
+
# Backburner.work('newsletter_sender', 'test_job')
|
36
|
+
# Backburner.work('newsletter_sender', 'test_job', :worker => NotSimpleWorker)
|
37
|
+
#
|
38
|
+
def work(*tubes)
|
39
|
+
options = tubes.last.is_a?(Hash) ? tubes.pop : {}
|
40
|
+
worker_class = options[:worker] || configuration.default_worker
|
41
|
+
worker_class.start(tubes)
|
42
|
+
end
|
43
|
+
|
44
|
+
# Yields a configuration block
|
45
|
+
#
|
46
|
+
# @example
|
47
|
+
# Backburner.configure do |config|
|
48
|
+
# config.beanstalk_url = "beanstalk://..."
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
def configure(&block)
|
52
|
+
yield(configuration)
|
53
|
+
configuration
|
54
|
+
end
|
55
|
+
|
56
|
+
# Returns the configuration options set for Backburner
|
57
|
+
#
|
58
|
+
# @example
|
59
|
+
# Backburner.configuration.beanstalk_url => false
|
60
|
+
#
|
61
|
+
def configuration
|
62
|
+
@_configuration ||= Configuration.new
|
63
|
+
end
|
64
|
+
|
65
|
+
# Returns the queues that are processed by default if none are specified
|
66
|
+
#
|
67
|
+
# @example
|
68
|
+
# Backburner.default_queues << "foo"
|
69
|
+
# Backburner.default_queues => ["foo", "bar"]
|
70
|
+
#
|
71
|
+
def default_queues
|
72
|
+
configuration.default_queues
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|