backburner-allq 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.travis.yml +29 -0
  4. data/CHANGELOG.md +133 -0
  5. data/CONTRIBUTING.md +37 -0
  6. data/Gemfile +4 -0
  7. data/HOOKS.md +99 -0
  8. data/LICENSE +22 -0
  9. data/README.md +658 -0
  10. data/Rakefile +17 -0
  11. data/TODO +4 -0
  12. data/backburner-allq.gemspec +26 -0
  13. data/bin/backburner +7 -0
  14. data/circle.yml +3 -0
  15. data/deploy.sh +3 -0
  16. data/examples/custom.rb +25 -0
  17. data/examples/demo.rb +60 -0
  18. data/examples/god.rb +46 -0
  19. data/examples/hooked.rb +87 -0
  20. data/examples/retried.rb +31 -0
  21. data/examples/simple.rb +43 -0
  22. data/examples/stress.rb +31 -0
  23. data/lib/backburner.rb +75 -0
  24. data/lib/backburner/allq_wrapper.rb +317 -0
  25. data/lib/backburner/async_proxy.rb +25 -0
  26. data/lib/backburner/cli.rb +53 -0
  27. data/lib/backburner/configuration.rb +48 -0
  28. data/lib/backburner/connection.rb +157 -0
  29. data/lib/backburner/helpers.rb +193 -0
  30. data/lib/backburner/hooks.rb +53 -0
  31. data/lib/backburner/job.rb +118 -0
  32. data/lib/backburner/logger.rb +53 -0
  33. data/lib/backburner/performable.rb +95 -0
  34. data/lib/backburner/queue.rb +145 -0
  35. data/lib/backburner/tasks.rb +54 -0
  36. data/lib/backburner/version.rb +3 -0
  37. data/lib/backburner/worker.rb +221 -0
  38. data/lib/backburner/workers/forking.rb +52 -0
  39. data/lib/backburner/workers/simple.rb +29 -0
  40. data/lib/backburner/workers/threading.rb +163 -0
  41. data/lib/backburner/workers/threads_on_fork.rb +263 -0
  42. data/test/async_proxy_test.rb +36 -0
  43. data/test/back_burner_test.rb +88 -0
  44. data/test/connection_test.rb +179 -0
  45. data/test/fixtures/hooked.rb +122 -0
  46. data/test/fixtures/test_fork_jobs.rb +72 -0
  47. data/test/fixtures/test_forking_jobs.rb +56 -0
  48. data/test/fixtures/test_jobs.rb +87 -0
  49. data/test/fixtures/test_queue_settings.rb +14 -0
  50. data/test/helpers/templogger.rb +22 -0
  51. data/test/helpers_test.rb +278 -0
  52. data/test/hooks_test.rb +112 -0
  53. data/test/job_test.rb +185 -0
  54. data/test/logger_test.rb +44 -0
  55. data/test/performable_test.rb +88 -0
  56. data/test/queue_test.rb +69 -0
  57. data/test/test_helper.rb +128 -0
  58. data/test/worker_test.rb +157 -0
  59. data/test/workers/forking_worker_test.rb +181 -0
  60. data/test/workers/simple_worker_test.rb +350 -0
  61. data/test/workers/threading_worker_test.rb +104 -0
  62. data/test/workers/threads_on_fork_worker_test.rb +484 -0
  63. metadata +217 -0
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+ require 'rake/testtask'
4
+ # require 'yard'
5
+
6
+ task :test do
7
+ Rake::TestTask.new do |t|
8
+ t.libs.push "lib"
9
+ t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
10
+ t.verbose = true
11
+ end
12
+ end
13
+
14
+ task :default => :test
15
+
16
+ # task :doc do
17
+ # YARD::CLI::Yardoc.new.run
data/TODO ADDED
@@ -0,0 +1,4 @@
1
+ - Custom front-end in sinatra for viewing beanstalk jobs
2
+ - Refer to https://github.com/denniskuczynski/beanstalkd_view
3
+ - Fork jobs to control memory
4
+ - https://github.com/michaeldwan/stalker/commit/386267690a7c03e11d1a8b7b6f08b7c9c7cd2c0d
@@ -0,0 +1,26 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/backburner/version', __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.authors = ["Jason Malcolm"]
6
+ s.email = ["jason@blitline.com"]
7
+ s.description = %q{Beanstalk background job processing made easy}
8
+ s.summary = %q{Reliable allq background job processing made easy for Ruby and Sinatra}
9
+ s.homepage = "http://github.com/nesquena/backburner"
10
+
11
+ s.files = `git ls-files`.split($\)
12
+ s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
14
+ s.name = "backburner-allq"
15
+ s.require_paths = ["lib"]
16
+ s.version = Backburner::VERSION
17
+ s.license = 'MIT'
18
+
19
+ s.add_runtime_dependency 'allq_rest', '~> 1.0'
20
+ s.add_runtime_dependency 'dante', '> 0.1.5'
21
+ s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
22
+
23
+ s.add_development_dependency 'rake'
24
+ s.add_development_dependency 'minitest', '3.2.0'
25
+ s.add_development_dependency 'mocha'
26
+ end
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'backburner'
4
+ require 'backburner/cli'
5
+
6
+ # bundle exec backburner foo,bar
7
+ Backburner::CLI.start(ARGV)
@@ -0,0 +1,3 @@
1
+ machine:
2
+ services:
3
+ - beanstalkd
@@ -0,0 +1,3 @@
1
+ echo "Did you update the version?"
2
+ gem build backburner-allq.gemspec
3
+ gem push backburner-allq-1.0.0.gem
@@ -0,0 +1,25 @@
1
+ $:.unshift "lib"
2
+ require 'backburner'
3
+
4
+ # Define ruby job
5
+ class TestJob
6
+ include Backburner::Queue
7
+ # queue "test-job"
8
+
9
+ def self.perform(value, user)
10
+ puts "[TestJob] Running perform with args: [#{value}, #{user}]"
11
+ end
12
+ end
13
+
14
+ # Configure Backburner
15
+ Backburner.configure do |config|
16
+ config.beanstalk_url = "beanstalk://127.0.0.1"
17
+ config.tube_namespace = "demo.production"
18
+ end
19
+
20
+ # Enqueue tasks
21
+ Backburner.enqueue TestJob, 5, 3
22
+ Backburner.enqueue TestJob, 10, 6
23
+
24
+ # Work tasks using threaded worker
25
+ Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)
@@ -0,0 +1,60 @@
1
+ $:.unshift "lib"
2
+ require 'backburner'
3
+
4
+ module Tester
5
+ class TestJob
6
+ include Backburner::Queue
7
+ queue "test.job"
8
+
9
+ def self.perform(value, user)
10
+ p [value, user]
11
+ end
12
+ end
13
+
14
+ class UserModel
15
+ include Backburner::Performable
16
+
17
+ attr_accessor :id, :name
18
+
19
+ def self.first
20
+ self.find(3, "John")
21
+ end
22
+
23
+ def self.find(id, name="Fetched")
24
+ self.new(id, name)
25
+ end
26
+
27
+ def initialize(id, name)
28
+ @id, @name = id, name
29
+ end
30
+
31
+ def hello(x, y)
32
+ puts "Instance #{x} and #{y} and my id is #{id}"
33
+ end
34
+
35
+ def self.foo(x, y)
36
+ puts "Class #{x} and #{y}"
37
+ end
38
+ end
39
+ end
40
+
41
+ # connection = Backburner::Connection.new("beanstalk://127.0.0.1")
42
+
43
+ Backburner.configure do |config|
44
+ config.beanstalk_url = "beanstalk://127.0.0.1"
45
+ config.tube_namespace = "myblog.production"
46
+ end
47
+
48
+ # p Backburner.configuration.beanstalk_url
49
+ # p Backburner::Worker.connection
50
+
51
+ Backburner.enqueue Tester::TestJob, 5, 3
52
+ Backburner.enqueue Tester::TestJob, 10, 6
53
+ @user = Tester::UserModel.first
54
+ @user.async.hello("foo", "bar")
55
+ Tester::UserModel.async.foo("bar", "baz")
56
+
57
+ Backburner.default_queues.concat([Tester::TestJob.queue, Tester::UserModel.queue])
58
+ Backburner.work
59
+ # Backburner.work("test.job")
60
+ # Backburner.work("tester/user-model")
@@ -0,0 +1,46 @@
1
+ God.watch do |w|
2
+ w.name = "backburner-worker-1"
3
+ w.dir = '/path/to/app/dir'
4
+ w.env = { 'PADRINO_ENV' => 'production', 'QUEUES' => 'newsletter-sender,push-message' }
5
+ w.group = 'backburner-workers'
6
+ w.interval = 30.seconds
7
+ w.start = "bundle exec rake -f Rakefile backburner:work"
8
+ w.log = "/var/log/god/backburner-worker-1.log"
9
+
10
+ # restart if memory gets too high
11
+ w.transition(:up, :restart) do |on|
12
+ on.condition(:memory_usage) do |c|
13
+ c.above = 50.megabytes
14
+ c.times = 3
15
+ end
16
+ end
17
+
18
+ # determine the state on startup
19
+ w.transition(:init, { true => :up, false => :start }) do |on|
20
+ on.condition(:process_running) do |c|
21
+ c.running = true
22
+ end
23
+ end
24
+
25
+ # determine when process has finished starting
26
+ w.transition([:start, :restart], :up) do |on|
27
+ on.condition(:process_running) do |c|
28
+ c.running = true
29
+ c.interval = 5.seconds
30
+ end
31
+
32
+ # failsafe
33
+ on.condition(:tries) do |c|
34
+ c.times = 5
35
+ c.transition = :start
36
+ c.interval = 5.seconds
37
+ end
38
+ end
39
+
40
+ # start if process is not running
41
+ w.transition(:up, :start) do |on|
42
+ on.condition(:process_running) do |c|
43
+ c.running = false
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,87 @@
1
+ $:.unshift "lib"
2
+ require 'backburner'
3
+
4
+ $fail = 0
5
+ class User
6
+ include Backburner::Performable
7
+
8
+ # Called with the job args before a job is placed on the queue.
9
+ # !! If the hook returns `false`, the job will not be placed on the queue.
10
+ def self.before_enqueue_foo(*args)
11
+ puts "[before_enqueue] Just about to enqueue #{self} with #{args.inspect}"
12
+ end
13
+
14
+ # Called with the job args after a job is placed on the queue.
15
+ # !! Any exception raised propagates up to the code which queued the job.
16
+ def self.after_enqueue_foo(*args)
17
+ puts "[after_enqueue] Finished enqueuing #{self} with #{args.inspect}"
18
+ end
19
+
20
+ # Called with the job args before perform. If it raises
21
+ # `Backburner::Job::DontPerform`, the job is aborted. Other exceptions
22
+ # are treated like regular job exceptions.
23
+ def self.before_perform_foo(*args)
24
+ puts "[before_perform] Just about to perform #{self} with #{args.inspect}"
25
+ end
26
+
27
+ # Called with the job args after it performs. Uncaught
28
+ # exceptions will be treated like regular job exceptions.
29
+ def self.after_perform_foo(*args)
30
+ puts "[after_perform] Just finished performing #{self} with #{args.inspect}"
31
+ end
32
+
33
+ # Called with the job args. It is expected to yield in order
34
+ # to perform the job (but is not required to do so). It may handle exceptions
35
+ # thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
36
+ def self.around_perform_bar(*args)
37
+ puts "[around_perform_bar before] About to perform #{self} with #{args.inspect}"
38
+ yield
39
+ puts "[around_perform_bar after] Just after performing #{self} with #{args.inspect}"
40
+ end
41
+
42
+ # Called with the job args. It is expected to yield in order
43
+ # to perform the job (but is not required to do so). It may handle exceptions
44
+ # thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
45
+ def self.around_perform_cat(*args)
46
+ puts "[around_perform_cat before] About to perform #{self} with #{args.inspect}"
47
+ yield
48
+ puts "[around_perform_cat after] Just after performing #{self} with #{args.inspect}"
49
+ end
50
+
51
+ # Called with the job args. It is expected to yield in order
52
+ # to perform the job (but is not required to do so). It may handle exceptions
53
+ # thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
54
+ def self.around_perform_foo(*args)
55
+ puts "[around_perform_foo before] About to perform #{self} with #{args.inspect}"
56
+ yield
57
+ puts "[around_perform_foo after] Just after performing #{self} with #{args.inspect}"
58
+ end
59
+
60
+ # Called with the exception and job args if any exception occurs
61
+ # while performing the job (or hooks).
62
+ def self.on_failure_foo(ex, *args)
63
+ puts "[on_failure] Failure #{ex.inspect} occurred for job #{self} with #{args.inspect}"
64
+ end
65
+
66
+ def self.foo
67
+ $fail += 1
68
+ raise "Fail!" if $fail == 1
69
+ puts "This is the job running successfully!!"
70
+ end
71
+ end
72
+
73
+ # Configure Backburner
74
+ Backburner.configure do |config|
75
+ config.beanstalk_url = "beanstalk://127.0.0.1"
76
+ config.tube_namespace = "demo.production"
77
+ config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
78
+ config.max_job_retries = 1
79
+ config.retry_delay = 0
80
+ end
81
+
82
+ # Enqueue tasks
83
+ User.async.foo
84
+
85
+ # Run work
86
+ # Backburner.default_queues << "user"
87
+ Backburner.work
@@ -0,0 +1,31 @@
1
+ $:.unshift "lib"
2
+ require 'backburner'
3
+
4
+ $error = 0
5
+
6
+ class User
7
+ include Backburner::Performable
8
+ attr_accessor :id, :name
9
+
10
+ def self.foo(x, y)
11
+ $error += 1
12
+ raise "fail #{$error}" unless $error > 3
13
+ puts "User #foo args [#{x}, #{y}] Success!!"
14
+ end
15
+ end
16
+
17
+ # Configure Backburner
18
+ Backburner.configure do |config|
19
+ config.beanstalk_url = "beanstalk://127.0.0.1"
20
+ config.tube_namespace = "demo.production"
21
+ config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
22
+ config.max_job_retries = 3
23
+ config.retry_delay = 0
24
+ end
25
+
26
+ # Enqueue tasks
27
+ User.async(:queue => "retried").foo("bar", "baz")
28
+
29
+ # Run work
30
+ # Backburner.default_queues << "user"
31
+ Backburner.work
@@ -0,0 +1,43 @@
1
+ $:.unshift "lib"
2
+ require 'backburner'
3
+
4
+ class User
5
+ include Backburner::Performable
6
+ attr_accessor :id, :name
7
+
8
+ def self.first
9
+ User.find(3, "John")
10
+ end
11
+
12
+ def self.find(id, name="Fetched")
13
+ User.new(id, name)
14
+ end
15
+
16
+ def initialize(id, name)
17
+ @id, @name = id, name
18
+ end
19
+
20
+ def hello(x, y)
21
+ puts "User(id=#{id}) #hello args: [#{x}, #{y}] (Instance method)"
22
+ end
23
+
24
+ def self.foo(x, y)
25
+ puts "User #foo args [#{x}, #{y}] (Class method)"
26
+ end
27
+ end
28
+
29
+ # Configure Backburner
30
+ Backburner.configure do |config|
31
+ config.beanstalk_url = "beanstalk://127.0.0.1"
32
+ config.tube_namespace = "demo.production"
33
+ config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
34
+ end
35
+
36
+ # Enqueue tasks
37
+ @user = User.first
38
+ @user.async(:pri => 1000).hello("foo", "bar")
39
+ User.async.foo("bar", "baz")
40
+
41
+ # Run work
42
+ # Backburner.default_queues << "user"
43
+ Backburner.work
@@ -0,0 +1,31 @@
1
+ $:.unshift "lib"
2
+ require 'backburner'
3
+
4
+ $values = []
5
+
6
+ # Define ruby job
7
+ class TestJob
8
+ include Backburner::Queue
9
+ queue "test-job"
10
+
11
+ def self.perform(value)
12
+ puts "[TestJob] Running perform with args: [#{value}]"
13
+ $values << value
14
+ puts "#{$values.size} total jobs processed"
15
+ end
16
+ end
17
+
18
+ # Configure Backburner
19
+ Backburner.configure do |config|
20
+ config.beanstalk_url = "beanstalk://127.0.0.1"
21
+ config.tube_namespace = "demo.production"
22
+ end
23
+
24
+ # Enqueue tasks
25
+ 1.upto(1000) do |i|
26
+ Backburner.enqueue TestJob, i
27
+ end
28
+
29
+ # Work tasks using threads_on_fork worker
30
+ # twitter tube will have 10 threads, garbage after 1000 executions and retry jobs 1 times.
31
+ Backburner.work("test-job:10:100:1", :worker => Backburner::Workers::ThreadsOnFork)
@@ -0,0 +1,75 @@
1
+ require 'allq_rest'
2
+ require 'json'
3
+ require 'uri'
4
+ require 'timeout'
5
+ require 'backburner/version'
6
+ require 'backburner/allq_wrapper'
7
+ require 'backburner/helpers'
8
+ require 'backburner/configuration'
9
+ require 'backburner/logger'
10
+ require 'backburner/connection'
11
+ require 'backburner/hooks'
12
+ require 'backburner/performable'
13
+ require 'backburner/worker'
14
+ require 'backburner/workers/simple'
15
+ require 'backburner/workers/forking'
16
+ require 'backburner/workers/threads_on_fork'
17
+ require 'backburner/workers/threading'
18
+ require 'backburner/queue'
19
+
20
+ module Backburner
21
+ class << self
22
+
23
+ # Enqueues a job to be performed with given arguments.
24
+ #
25
+ # @example
26
+ # Backburner.enqueue NewsletterSender, self.id, user.id
27
+ #
28
+ def enqueue(job_class, *args)
29
+ Backburner::Worker.enqueue(job_class, args, {})
30
+ end
31
+
32
+ # Begins working on jobs enqueued with optional tubes specified
33
+ #
34
+ # @example
35
+ # Backburner.work('newsletter_sender', 'test_job')
36
+ # Backburner.work('newsletter_sender', 'test_job', :worker => NotSimpleWorker)
37
+ #
38
+ def work(*tubes)
39
+ options = tubes.last.is_a?(Hash) ? tubes.pop : {}
40
+ worker_class = options[:worker] || configuration.default_worker
41
+ worker_class.start(tubes)
42
+ end
43
+
44
+ # Yields a configuration block
45
+ #
46
+ # @example
47
+ # Backburner.configure do |config|
48
+ # config.beanstalk_url = "beanstalk://..."
49
+ # end
50
+ #
51
+ def configure(&block)
52
+ yield(configuration)
53
+ configuration
54
+ end
55
+
56
+ # Returns the configuration options set for Backburner
57
+ #
58
+ # @example
59
+ # Backburner.configuration.beanstalk_url => false
60
+ #
61
+ def configuration
62
+ @_configuration ||= Configuration.new
63
+ end
64
+
65
+ # Returns the queues that are processed by default if none are specified
66
+ #
67
+ # @example
68
+ # Backburner.default_queues << "foo"
69
+ # Backburner.default_queues => ["foo", "bar"]
70
+ #
71
+ def default_queues
72
+ configuration.default_queues
73
+ end
74
+ end
75
+ end