backburner-allq 1.0.0

data/Rakefile

@@ -0,0 +1,17 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+# require 'yard'
+
+task :test do
+  Rake::TestTask.new do |t|
+    t.libs.push "lib"
+    t.test_files = FileList[File.expand_path('../test/**/*_test.rb', __FILE__)]
+    t.verbose = true
+  end
+end
+
+task :default => :test
+
+# task :doc do
+#  YARD::CLI::Yardoc.new.run

data/TODO

@@ -0,0 +1,4 @@
+- Custom front-end in sinatra for viewing beanstalk jobs
+  - Refer to https://github.com/denniskuczynski/beanstalkd_view
+- Fork jobs to control memory
+  - https://github.com/michaeldwan/stalker/commit/386267690a7c03e11d1a8b7b6f08b7c9c7cd2c0d

data/backburner-allq.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/backburner/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.authors       = ["Jason Malcolm"]
+  s.email         = ["jason@blitline.com"]
+  s.description   = %q{Beanstalk background job processing made easy}
+  s.summary       = %q{Reliable allq background job processing made easy for Ruby and Sinatra}
+  s.homepage      = "http://github.com/nesquena/backburner"
+
+  s.files         = `git ls-files`.split($\)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.name          = "backburner-allq"
+  s.require_paths = ["lib"]
+  s.version       = Backburner::VERSION
+  s.license       = 'MIT'
+
+  s.add_runtime_dependency 'allq_rest', '~> 1.0'
+  s.add_runtime_dependency 'dante', '> 0.1.5'
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'minitest', '3.2.0'
+  s.add_development_dependency 'mocha'
+end

data/bin/backburner

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'backburner'
+require 'backburner/cli'
+
+# bundle exec backburner foo,bar
+Backburner::CLI.start(ARGV)

data/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  services:
+    - beanstalkd

data/deploy.sh

@@ -0,0 +1,3 @@
+echo "Did you update the version?"
+gem build backburner-allq.gemspec
+gem push backburner-allq-1.0.0.gem

data/examples/custom.rb

@@ -0,0 +1,25 @@
+$:.unshift "lib"
+require 'backburner'
+
+# Define ruby job
+class TestJob
+  include Backburner::Queue
+  # queue "test-job"
+
+  def self.perform(value, user)
+    puts "[TestJob] Running perform with args: [#{value}, #{user}]"
+  end
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.production"
+end
+
+# Enqueue tasks
+Backburner.enqueue TestJob, 5, 3
+Backburner.enqueue TestJob, 10, 6
+
+# Work tasks using threaded worker
+Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)

data/examples/demo.rb

@@ -0,0 +1,60 @@
+$:.unshift "lib"
+require 'backburner'
+
+module Tester
+  class TestJob
+    include Backburner::Queue
+    queue "test.job"
+
+    def self.perform(value, user)
+      p [value, user]
+    end
+  end
+
+  class UserModel
+    include Backburner::Performable
+
+    attr_accessor :id, :name
+
+    def self.first
+      self.find(3, "John")
+    end
+
+    def self.find(id, name="Fetched")
+      self.new(id, name)
+    end
+
+    def initialize(id, name)
+      @id, @name = id, name
+    end
+
+    def hello(x, y)
+      puts "Instance #{x} and #{y} and my id is #{id}"
+    end
+
+    def self.foo(x, y)
+      puts "Class #{x} and #{y}"
+    end
+  end
+end
+
+# connection = Backburner::Connection.new("beanstalk://127.0.0.1")
+
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "myblog.production"
+end
+
+# p Backburner.configuration.beanstalk_url
+# p Backburner::Worker.connection
+
+Backburner.enqueue Tester::TestJob, 5, 3
+Backburner.enqueue Tester::TestJob, 10, 6
+@user = Tester::UserModel.first
+@user.async.hello("foo", "bar")
+Tester::UserModel.async.foo("bar", "baz")
+
+Backburner.default_queues.concat([Tester::TestJob.queue, Tester::UserModel.queue])
+Backburner.work
+# Backburner.work("test.job")
+# Backburner.work("tester/user-model")

data/examples/god.rb

@@ -0,0 +1,46 @@
+God.watch do |w|
+  w.name   = "backburner-worker-1"
+  w.dir    = '/path/to/app/dir'
+  w.env = { 'PADRINO_ENV' => 'production', 'QUEUES' => 'newsletter-sender,push-message' }
+  w.group    = 'backburner-workers'
+  w.interval = 30.seconds
+  w.start = "bundle exec rake -f Rakefile backburner:work"
+  w.log   = "/var/log/god/backburner-worker-1.log"
+
+  # restart if memory gets too high
+  w.transition(:up, :restart) do |on|
+    on.condition(:memory_usage) do |c|
+      c.above = 50.megabytes
+      c.times = 3
+    end
+  end
+
+  # determine the state on startup
+  w.transition(:init, { true => :up, false => :start }) do |on|
+    on.condition(:process_running) do |c|
+      c.running = true
+    end
+  end
+
+  # determine when process has finished starting
+  w.transition([:start, :restart], :up) do |on|
+    on.condition(:process_running) do |c|
+      c.running = true
+      c.interval = 5.seconds
+    end
+
+    # failsafe
+    on.condition(:tries) do |c|
+      c.times = 5
+      c.transition = :start
+      c.interval = 5.seconds
+    end
+  end
+
+  # start if process is not running
+  w.transition(:up, :start) do |on|
+    on.condition(:process_running) do |c|
+      c.running = false
+    end
+  end
+end

data/examples/hooked.rb

@@ -0,0 +1,87 @@
+$:.unshift "lib"
+require 'backburner'
+
+$fail = 0
+class User
+  include Backburner::Performable
+
+  # Called with the job args before a job is placed on the queue.
+  # !! If the hook returns `false`, the job will not be placed on the queue.
+  def self.before_enqueue_foo(*args)
+    puts "[before_enqueue] Just about to enqueue #{self} with #{args.inspect}"
+  end
+
+  # Called with the job args after a job is placed on the queue.
+  # !! Any exception raised propagates up to the code which queued the job.
+  def self.after_enqueue_foo(*args)
+    puts "[after_enqueue] Finished enqueuing #{self} with #{args.inspect}"
+  end
+
+  # Called with the job args before perform. If it raises
+  #  `Backburner::Job::DontPerform`, the job is aborted. Other exceptions
+  #  are treated like regular job exceptions.
+  def self.before_perform_foo(*args)
+    puts "[before_perform] Just about to perform #{self} with #{args.inspect}"
+  end
+
+  # Called with the job args after it performs. Uncaught
+  #   exceptions will be treated like regular job exceptions.
+  def self.after_perform_foo(*args)
+    puts "[after_perform] Just finished performing #{self} with #{args.inspect}"
+  end
+
+  # Called with the job args. It is expected to yield in order
+  #   to perform the job (but is not required to do so). It may handle exceptions
+  #   thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
+  def self.around_perform_bar(*args)
+    puts "[around_perform_bar before] About to perform #{self} with #{args.inspect}"
+    yield
+    puts "[around_perform_bar after] Just after performing #{self} with #{args.inspect}"
+  end
+
+  # Called with the job args. It is expected to yield in order
+  #   to perform the job (but is not required to do so). It may handle exceptions
+  #   thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
+  def self.around_perform_cat(*args)
+    puts "[around_perform_cat before] About to perform #{self} with #{args.inspect}"
+    yield
+    puts "[around_perform_cat after] Just after performing #{self} with #{args.inspect}"
+  end
+
+  # Called with the job args. It is expected to yield in order
+  #   to perform the job (but is not required to do so). It may handle exceptions
+  #   thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
+  def self.around_perform_foo(*args)
+    puts "[around_perform_foo before] About to perform #{self} with #{args.inspect}"
+    yield
+    puts "[around_perform_foo after] Just after performing #{self} with #{args.inspect}"
+  end
+
+  # Called with the exception and job args if any exception occurs
+  #  while performing the job (or hooks).
+  def self.on_failure_foo(ex, *args)
+    puts "[on_failure] Failure #{ex.inspect} occurred for job #{self} with #{args.inspect}"
+  end
+
+  def self.foo
+    $fail += 1
+    raise "Fail!" if $fail == 1
+    puts "This is the job running successfully!!"
+  end
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.production"
+  config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
+  config.max_job_retries = 1
+  config.retry_delay     = 0
+end
+
+# Enqueue tasks
+User.async.foo
+
+# Run work
+# Backburner.default_queues << "user"
+Backburner.work

data/examples/retried.rb

@@ -0,0 +1,31 @@
+$:.unshift "lib"
+require 'backburner'
+
+$error = 0
+
+class User
+  include Backburner::Performable
+  attr_accessor :id, :name
+
+  def self.foo(x, y)
+    $error += 1
+    raise "fail #{$error}" unless $error > 3
+    puts "User #foo args [#{x}, #{y}] Success!!"
+  end
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.production"
+  config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
+  config.max_job_retries = 3
+  config.retry_delay     = 0
+end
+
+# Enqueue tasks
+User.async(:queue => "retried").foo("bar", "baz")
+
+# Run work
+# Backburner.default_queues << "user"
+Backburner.work

data/examples/simple.rb

@@ -0,0 +1,43 @@
+$:.unshift "lib"
+require 'backburner'
+
+class User
+  include Backburner::Performable
+  attr_accessor :id, :name
+
+  def self.first
+    User.find(3, "John")
+  end
+
+  def self.find(id, name="Fetched")
+    User.new(id, name)
+  end
+
+  def initialize(id, name)
+    @id, @name = id, name
+  end
+
+  def hello(x, y)
+    puts "User(id=#{id}) #hello args: [#{x}, #{y}] (Instance method)"
+  end
+
+  def self.foo(x, y)
+    puts "User #foo args [#{x}, #{y}] (Class method)"
+  end
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.production"
+  config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
+end
+
+# Enqueue tasks
+@user = User.first
+@user.async(:pri => 1000).hello("foo", "bar")
+User.async.foo("bar", "baz")
+
+# Run work
+# Backburner.default_queues << "user"
+Backburner.work

data/examples/stress.rb

@@ -0,0 +1,31 @@
+$:.unshift "lib"
+require 'backburner'
+
+$values = []
+
+# Define ruby job
+class TestJob
+  include Backburner::Queue
+  queue "test-job"
+
+  def self.perform(value)
+    puts "[TestJob] Running perform with args: [#{value}]"
+    $values << value
+    puts "#{$values.size} total jobs processed"
+  end
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.production"
+end
+
+# Enqueue tasks
+1.upto(1000) do |i|
+  Backburner.enqueue TestJob, i
+end
+
+# Work tasks using threads_on_fork worker
+# twitter tube will have 10 threads, garbage after 1000 executions and retry jobs 1 times.
+Backburner.work("test-job:10:100:1", :worker => Backburner::Workers::ThreadsOnFork)

data/lib/backburner.rb

@@ -0,0 +1,75 @@
+require 'allq_rest'
+require 'json'
+require 'uri'
+require 'timeout'
+require 'backburner/version'
+require 'backburner/allq_wrapper'
+require 'backburner/helpers'
+require 'backburner/configuration'
+require 'backburner/logger'
+require 'backburner/connection'
+require 'backburner/hooks'
+require 'backburner/performable'
+require 'backburner/worker'
+require 'backburner/workers/simple'
+require 'backburner/workers/forking'
+require 'backburner/workers/threads_on_fork'
+require 'backburner/workers/threading'
+require 'backburner/queue'
+
+module Backburner
+  class << self
+
+    # Enqueues a job to be performed with given arguments.
+    #
+    # @example
+    #   Backburner.enqueue NewsletterSender, self.id, user.id
+    #
+    def enqueue(job_class, *args)
+      Backburner::Worker.enqueue(job_class, args, {})
+    end
+
+    # Begins working on jobs enqueued with optional tubes specified
+    #
+    # @example
+    #   Backburner.work('newsletter_sender', 'test_job')
+    #   Backburner.work('newsletter_sender', 'test_job', :worker => NotSimpleWorker)
+    #
+    def work(*tubes)
+      options = tubes.last.is_a?(Hash) ? tubes.pop : {}
+      worker_class = options[:worker] || configuration.default_worker
+      worker_class.start(tubes)
+    end
+
+    # Yields a configuration block
+    #
+    # @example
+    #   Backburner.configure do |config|
+    #     config.beanstalk_url = "beanstalk://..."
+    #   end
+    #
+    def configure(&block)
+      yield(configuration)
+      configuration
+    end
+
+    # Returns the configuration options set for Backburner
+    #
+    # @example
+    #   Backburner.configuration.beanstalk_url => false
+    #
+    def configuration
+      @_configuration ||= Configuration.new
+    end
+
+    # Returns the queues that are processed by default if none are specified
+    #
+    # @example
+    #   Backburner.default_queues << "foo"
+    #   Backburner.default_queues => ["foo", "bar"]
+    #
+    def default_queues
+      configuration.default_queues
+    end
+  end
+end