backburner-allq 1.0.0

data/lib/backburner/queue.rb

@@ -0,0 +1,145 @@
+module Backburner
+  module Queue
+    def self.included(base)
+      base.instance_variable_set(:@queue_name, nil)
+      base.instance_variable_set(:@queue_priority, nil)
+      base.instance_variable_set(:@queue_respond_timeout, nil)
+      base.instance_variable_set(:@queue_max_job_retries, nil)
+      base.instance_variable_set(:@queue_retry_delay, nil)
+      base.instance_variable_set(:@queue_retry_delay_proc, nil)
+      base.instance_variable_set(:@queue_jobs_limit, nil)
+      base.instance_variable_set(:@queue_garbage_limit, nil)
+      base.instance_variable_set(:@queue_retry_limit, nil)
+      base.extend ClassMethods
+      Backburner::Worker.known_queue_classes << base
+    end
+
+    module ClassMethods
+      # Returns or assigns queue name for this job.
+      #
+      # @example
+      #   queue "some.task.name"
+      #   @klass.queue # => "some.task.name"
+      #
+      def queue(name=nil)
+        if name
+          @queue_name = name
+        else # accessor
+          (@queue_name.is_a?(Proc) ? @queue_name.call(self) : @queue_name) || Backburner.configuration.primary_queue
+        end
+      end
+
+      # Returns or assigns queue priority for this job
+      #
+      # @example
+      #   queue_priority 120
+      #   @klass.queue_priority # => 120
+      #
+      def queue_priority(pri=nil)
+        if pri
+          @queue_priority = pri
+        else # accessor
+          @queue_priority
+        end
+      end
+
+      # Returns or assigns queue respond_timeout for this job
+      #
+      # @example
+      #   queue_respond_timeout 120
+      #   @klass.queue_respond_timeout # => 120
+      #
+      def queue_respond_timeout(ttr=nil)
+        if ttr
+          @queue_respond_timeout = ttr
+        else # accessor
+          @queue_respond_timeout
+        end
+      end
+
+      # Returns or assigns queue max_job_retries for this job
+      #
+      # @example
+      #   queue_max_job_retries 120
+      #   @klass.queue_max_job_retries # => 120
+      #
+      def queue_max_job_retries(delay=nil)
+        if delay
+          @queue_max_job_retries = delay
+        else # accessor
+          @queue_max_job_retries
+        end
+      end
+
+      # Returns or assigns queue retry_delay for this job
+      #
+      # @example
+      #   queue_retry_delay 120
+      #   @klass.queue_retry_delay # => 120
+      #
+      def queue_retry_delay(delay=nil)
+        if delay
+          @queue_retry_delay = delay
+        else # accessor
+          @queue_retry_delay
+        end
+      end
+
+      # Returns or assigns queue retry_delay_proc for this job
+      #
+      # @example
+      #   queue_retry_delay_proc lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      #   @klass.queue_retry_delay_proc # => lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      #
+      def queue_retry_delay_proc(proc=nil)
+        if proc
+          @queue_retry_delay_proc = proc
+        else # accessor
+          @queue_retry_delay_proc
+        end
+      end
+
+      # Returns or assigns queue parallel active jobs limit (only ThreadsOnFork and Threading workers)
+      #
+      # @example
+      #   queue_jobs_limit 5
+      #   @klass.queue_jobs_limit # => 5
+      #
+      def queue_jobs_limit(limit=nil)
+        if limit
+          @queue_jobs_limit = limit
+        else #accessor
+          @queue_jobs_limit
+        end
+      end
+
+      # Returns or assigns queue jobs garbage limit (only ThreadsOnFork Worker)
+      #
+      # @example
+      #   queue_garbage_limit 1000
+      #   @klass.queue_garbage_limit # => 1000
+      #
+      def queue_garbage_limit(limit=nil)
+        if limit
+          @queue_garbage_limit = limit
+        else #accessor
+          @queue_garbage_limit
+        end
+      end
+
+      # Returns or assigns queue retry limit (only ThreadsOnFork worker)
+      #
+      # @example
+      #   queue_retry_limit 6
+      #   @klass.queue_retry_limit # => 6
+      #
+      def queue_retry_limit(limit=nil)
+        if limit
+          @queue_retry_limit = limit
+        else #accessor
+          @queue_retry_limit
+        end
+      end
+    end # ClassMethods
+  end # Queue
+end # Backburner

data/lib/backburner/tasks.rb

@@ -0,0 +1,54 @@
+# require 'backburner/tasks'
+# will give you the backburner tasks
+
+namespace :backburner do
+  # QUEUE=foo,bar,baz rake backburner:work
+  desc "Start backburner worker using default worker"
+  task :work => :environment do
+    Backburner.work get_queues
+  end
+
+  namespace :simple do
+    # QUEUE=foo,bar,baz rake backburner:simple:work
+    desc "Starts backburner worker using simple processing"
+    task :work => :environment do
+      Backburner.work get_queues, :worker => Backburner::Workers::Simple
+    end
+  end # simple
+
+  namespace :forking do
+    # QUEUE=foo,bar,baz rake backburner:forking:work
+    desc "Starts backburner worker using fork processing"
+    task :work => :environment do
+      Backburner.work get_queues, :worker => Backburner::Workers::Forking
+    end
+  end # forking
+
+  namespace :threads_on_fork do
+    # QUEUE=twitter:10:5000:5,parse_page,send_mail,verify_bithday THREADS=2 GARBAGE=1000 rake backburner:threads_on_fork:work
+    # twitter tube will have 10 threads, garbage after 5k executions and retry 5 times.
+    desc "Starts backburner worker using threads_on_fork processing"
+    task :work => :environment do
+      threads = ENV['THREADS'].to_i
+      garbage = ENV['GARBAGE'].to_i
+      Backburner::Workers::ThreadsOnFork.threads_number = threads if threads > 0
+      Backburner::Workers::ThreadsOnFork.garbage_after  = garbage if garbage > 0
+      Backburner.work get_queues, :worker => Backburner::Workers::ThreadsOnFork
+    end
+  end # threads_on_fork
+
+  namespace :threading do
+    # QUEUE=twitter:10,parse_page,send_mail,verify_bithday THREADS=2 rake backburner:threading:work
+    # twitter tube will have 10 threads
+    desc "Starts backburner worker using threading processing"
+    task :work => :environment do
+      threads = ENV['THREADS'].to_i
+      Backburner::Workers::Threading.threads_number = threads if threads > 0
+      Backburner.work get_queues, :worker => Backburner::Workers::Threading
+    end
+  end # threads_on_fork
+
+  def get_queues
+    (ENV["QUEUE"] ? ENV["QUEUE"].split(',') : nil) rescue nil
+  end
+end

data/lib/backburner/version.rb

@@ -0,0 +1,3 @@
+module Backburner
+  VERSION = "1.0.0"
+end

data/lib/backburner/worker.rb

@@ -0,0 +1,221 @@
+require 'backburner/job'
+
+module Backburner
+  #
+  # @abstract Subclass and override {#process_tube_names}, {#prepare} and {#start} to implement
+  #   a custom Worker class.
+  #
+  class Worker
+    include Backburner::Helpers
+    include Backburner::Logger
+
+    # Backburner::Worker.known_queue_classes
+    # List of known_queue_classes
+    class << self
+      attr_writer :known_queue_classes
+      def known_queue_classes; @known_queue_classes ||= []; end
+    end
+
+    # Enqueues a job to be processed later by a worker.
+    # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+    #
+    # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
+    # @example
+    #   Backburner::Worker.enqueue NewsletterSender, [self.id, user.id], :ttr => 1000
+    #
+    def self.enqueue(job_class, args=[], opts={})
+      pri   = resolve_priority(opts[:pri] || job_class)
+      delay = [0, opts[:delay].to_i].max
+      ttr   = resolve_respond_timeout(opts[:ttr] || job_class)
+      res   = Backburner::Hooks.invoke_hook_events(job_class, :before_enqueue, *args)
+
+      return nil unless res # stop if hook is false
+
+      data = { :class => job_class.name, :args => args }
+      queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
+
+      begin
+        response = nil
+        connection = Backburner::Connection.new(Backburner.configuration.allq_url)
+        connection.retryable do
+          tube_name = expand_tube_name(queue || job_class)
+          serialized_data = Backburner.configuration.job_serializer_proc.call(data)
+          response = connection.put(serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
+        end
+        return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
+      ensure
+        connection.close if connection
+      end
+
+      response
+    end
+
+    # Starts processing jobs with the specified tube_names.
+    #
+    # @example
+    #   Backburner::Worker.start(["foo.tube.name"])
+    #
+    def self.start(tube_names=nil)
+      begin
+        self.new(tube_names).start
+      rescue SystemExit
+        # do nothing
+      end
+    end
+
+    # List of tube names to be watched and processed
+    attr_accessor :tube_names, :connection
+
+    # Constructs a new worker for processing jobs within specified tubes.
+    #
+    # @example
+    #   Worker.new(['test.job'])
+    def initialize(tube_names=nil)
+      @connection = new_connection
+      @tube_names = self.process_tube_names(tube_names)
+      register_signal_handlers!
+    end
+
+    # Starts processing ready jobs indefinitely.
+    # Primary way to consume and process jobs in specified tubes.
+    #
+    # @example
+    #   @worker.start
+    #
+    def start
+      raise NotImplementedError
+    end
+
+    # Used to prepare the job queues before job processing is initiated.
+    #
+    # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
+    # @example
+    #   @worker.prepare
+    #
+    # @abstract Define this in your worker subclass
+    # to be run once before processing. Recommended to watch tubes
+    # or print a message to the logs with 'log_info'
+    #
+    def prepare
+      raise NotImplementedError
+    end
+
+    # Triggers this worker to shutdown
+    def shutdown
+      Thread.new do
+        log_info 'Worker exiting...'
+      end
+      Kernel.exit
+    end
+
+    # Processes tube_names given tube_names array.
+    # Should return normalized tube_names as an array of strings.
+    #
+    # @example
+    #   process_tube_names([['foo'], ['bar']])
+    #   => ['foo', 'bar', 'baz']
+    #
+    # @note This method can be overridden in inherited workers
+    # to add more complex tube name processing.
+    def process_tube_names(tube_names)
+      compact_tube_names(tube_names)
+    end
+
+    # Performs a job by reserving a job from beanstalk and processing it
+    #
+    # @example
+    #   @worker.work_one_job
+    # @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
+    def work_one_job(conn = connection)
+      begin
+        job = reserve_job(conn)
+      rescue Beaneater::TimedOutError => e
+        return
+      end
+
+      self.log_job_begin(job.name, job.args)
+      job.process
+      self.log_job_end(job.name)
+
+    rescue Backburner::Job::JobFormatInvalid => e
+      self.log_error self.exception_message(e)
+    rescue => e # Error occurred processing job
+      self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
+
+      unless job
+        self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
+        return
+      end
+
+      # NB: There's a slight chance here that the connection to beanstalkd has
+      # gone down between the time we reserved / processed the job and here.
+      num_retries = job.stats.releases
+      max_job_retries = resolve_max_job_retries(job.job_class)
+      retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+      if num_retries < max_job_retries # retry again
+        retry_delay = resolve_retry_delay(job.job_class)
+        delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
+        job.retry(num_retries + 1, delay)
+        self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
+      else # retries failed, bury
+        job.bury
+        self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
+      end
+
+      handle_error(e, job.name, job.args, job)
+    end
+
+
+    protected
+
+    # Return a new connection instance
+    def new_connection
+      Connection.new(Backburner.configuration.beanstalk_url) { |conn| Backburner::Hooks.invoke_hook_events(self, :on_reconnect, conn) }
+    end
+
+    # Reserve a job from the watched queues
+    def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
+      Backburner::Job.new(conn.get(@tube_names.sample))
+    end
+
+    # Returns a list of all tubes known within the system
+    # Filtered for tubes that match the known prefix
+    def all_existing_queues
+      known_queues    = Backburner::Worker.known_queue_classes.map(&:queue)
+      existing_tubes  = self.connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
+      existing_tubes + known_queues + [queue_config.primary_queue]
+    end
+
+
+    # Handles an error according to custom definition
+    # Used when processing a job that errors out
+    def handle_error(e, name, args, job)
+      if error_handler = Backburner.configuration.on_error
+        if error_handler.arity == 1
+          error_handler.call(e)
+        elsif error_handler.arity == 3
+          error_handler.call(e, name, args)
+        else
+          error_handler.call(e, name, args, job)
+        end
+      end
+    end
+
+    # Normalizes tube names given array of tube_names
+    # Compacts nil items, flattens arrays, sets tubes to nil if no valid names
+    # Loads default tubes when no tubes given.
+    def compact_tube_names(tube_names)
+      tube_names = tube_names.first if tube_names && tube_names.size == 1 && tube_names.first.is_a?(Array)
+      tube_names = Array(tube_names).compact if tube_names && Array(tube_names).compact.size > 0
+      tube_names = nil if tube_names && tube_names.compact.empty?
+      tube_names ||= Backburner.default_queues.any? ? Backburner.default_queues : all_existing_queues
+      Array(tube_names).uniq
+    end
+
+    # Registers signal handlers TERM and INT to trigger
+    def register_signal_handlers!
+      trap('TERM') { shutdown  }
+      trap('INT')  { shutdown  }
+    end
+  end # Worker
+end # Backburner

data/lib/backburner/workers/forking.rb

@@ -0,0 +1,52 @@
+module Backburner
+  module Workers
+    class Forking < Worker
+      # Used to prepare job queues before processing jobs.
+      # Setup beanstalk tube_names and watch all specified tubes for jobs.
+      #
+      # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
+      # @example
+      #   @worker.prepare
+      #
+      def prepare
+        self.tube_names.map! { |name| expand_tube_name(name)  }.uniq!
+        log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
+        self.connection.tubes.watch!(*self.tube_names)
+      end
+
+      # Starts processing new jobs indefinitely.
+      # Primary way to consume and process jobs in specified tubes.
+      #
+      # @example
+      #   @worker.start
+      #
+      def start
+        prepare
+        loop { fork_one_job }
+      end
+
+      # Need to re-establish the connection to the server(s) after forking
+      # Waits for a job, works the job, and exits
+      def fork_one_job
+        pid = Process.fork do
+          work_one_job
+          coolest_exit
+        end
+        Process.wait(pid)
+      end
+
+      def on_reconnect(conn)
+        @connection = conn
+        prepare
+      end
+
+      # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
+      # parent process
+      # We will use exitcode 99 that means the fork reached the garbage number
+      def coolest_exit
+        Kernel.exit! 99
+      end
+
+    end
+  end
+end