resqueue 1.0.0

data/lib/resque/data_store.rb

@@ -0,0 +1,326 @@
+module Resque
+  # An interface between Resque's persistence and the actual
+  # implementation.
+  class DataStore
+    extend Forwardable
+
+    HEARTBEAT_KEY = "workers:heartbeat"
+
+    def initialize(redis)
+      @redis               = redis
+      @queue_access        = QueueAccess.new(@redis)
+      @failed_queue_access = FailedQueueAccess.new(@redis)
+      @workers             = Workers.new(@redis)
+      @stats_access        = StatsAccess.new(@redis)
+    end
+
+    def_delegators :@queue_access, :push_to_queue,
+                                   :pop_from_queue,
+                                   :queue_size,
+                                   :peek_in_queue,
+                                   :queue_names,
+                                   :remove_queue,
+                                   :everything_in_queue,
+                                   :remove_from_queue,
+                                   :watch_queue,
+                                   :list_range
+
+    def_delegators :@failed_queue_access, :add_failed_queue,
+                                          :remove_failed_queue,
+                                          :num_failed,
+                                          :failed_queue_names,
+                                          :push_to_failed_queue,
+                                          :clear_failed_queue,
+                                          :update_item_in_failed_queue,
+                                          :remove_from_failed_queue
+    def_delegators :@workers, :worker_ids,
+                              :workers_map,
+                              :get_worker_payload,
+                              :worker_exists?,
+                              :register_worker,
+                              :worker_started,
+                              :unregister_worker,
+                              :heartbeat,
+                              :heartbeat!,
+                              :remove_heartbeat,
+                              :all_heartbeats,
+                              :set_worker_payload,
+                              :worker_start_time,
+                              :worker_done_working
+
+      def_delegators :@stats_access, :clear_stat,
+                                     :decremet_stat,
+                                     :increment_stat,
+                                     :stat
+
+    # Compatibility with any non-Resque classes that were using Resque.redis as a way to access Redis
+    def method_missing(sym,*args,&block)
+      # TODO: deprecation warning?
+      @redis.send(sym,*args,&block)
+    end
+
+    # make use respond like redis
+    def respond_to?(method,include_all=false)
+      @redis.respond_to?(method,include_all)
+    end
+
+    # Get a string identifying the underlying server.
+    # Probably should be private, but was public so must stay public
+    def identifier
+      # support 1.x versions of redis-rb
+      if @redis.respond_to?(:server)
+        @redis.server
+      elsif @redis.respond_to?(:nodes) # distributed
+        @redis.nodes.map { |n| n.id }.join(', ')
+      else
+        @redis.client.id
+      end
+    end
+
+    # Force a reconnect to Redis.
+    def reconnect
+      @redis.client.reconnect
+    end
+
+    # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
+    # is O(N) for the keyspace, so be careful - this can be slow for big databases.
+    def all_resque_keys
+      @redis.keys("*").map do |key|
+        key.sub("#{@redis.namespace}:", '')
+      end
+    end
+
+    def server_time
+      time, _ = @redis.time
+      Time.at(time)
+    end
+
+    class QueueAccess
+      def initialize(redis)
+        @redis = redis
+      end
+      def push_to_queue(queue,encoded_item)
+        @redis.pipelined do
+          watch_queue(queue)
+          @redis.rpush redis_key_for_queue(queue), encoded_item
+        end
+      end
+
+      # Pop whatever is on queue
+      def pop_from_queue(queue)
+        @redis.lpop(redis_key_for_queue(queue))
+      end
+
+      # Get the number of items in the queue
+      def queue_size(queue)
+        @redis.llen(redis_key_for_queue(queue)).to_i
+      end
+
+      # Examine items in the queue.
+      #
+      # NOTE: if count is 1, you will get back an object, otherwise you will
+      #       get an Array.  I'm not making this up.
+      def peek_in_queue(queue, start = 0, count = 1)
+        list_range(redis_key_for_queue(queue), start, count)
+      end
+
+      def queue_names
+        Array(@redis.smembers(:queues))
+      end
+
+      def remove_queue(queue)
+        @redis.pipelined do
+          @redis.srem(:queues, queue.to_s)
+          @redis.del(redis_key_for_queue(queue))
+        end
+      end
+
+      def everything_in_queue(queue)
+        @redis.lrange(redis_key_for_queue(queue), 0, -1)
+      end
+
+      # Remove data from the queue, if it's there, returning the number of removed elements
+      def remove_from_queue(queue,data)
+        @redis.lrem(redis_key_for_queue(queue), 0, data)
+      end
+
+      # Private: do not call
+      def watch_queue(queue)
+        @redis.sadd(:queues, queue.to_s)
+      end
+
+      # Private: do not call
+      def list_range(key, start = 0, count = 1)
+        if count == 1
+          @redis.lindex(key, start)
+        else
+          Array(@redis.lrange(key, start, start+count-1))
+        end
+      end
+
+    private
+
+      def redis_key_for_queue(queue)
+        "queue:#{queue}"
+      end
+
+    end
+
+    class FailedQueueAccess
+      def initialize(redis)
+        @redis = redis
+      end
+
+      def add_failed_queue(failed_queue_name)
+        @redis.sadd(:failed_queues, failed_queue_name)
+      end
+
+      def remove_failed_queue(failed_queue_name=:failed)
+        @redis.del(failed_queue_name)
+      end
+
+      def num_failed(failed_queue_name=:failed)
+        @redis.llen(failed_queue_name).to_i
+      end
+
+      def failed_queue_names(find_queue_names_in_key=nil)
+        if find_queue_names_in_key.nil?
+          [:failed]
+        else
+          Array(@redis.smembers(find_queue_names_in_key))
+        end
+      end
+
+      def push_to_failed_queue(data,failed_queue_name=:failed)
+        @redis.rpush(failed_queue_name,data)
+      end
+
+      def clear_failed_queue(failed_queue_name=:failed)
+        @redis.del(failed_queue_name)
+      end
+
+      def update_item_in_failed_queue(index_in_failed_queue,new_item_data,failed_queue_name=:failed)
+        @redis.lset(failed_queue_name, index_in_failed_queue, new_item_data)
+      end
+
+      def remove_from_failed_queue(index_in_failed_queue,failed_queue_name=nil)
+        failed_queue_name ||= :failed
+        hopefully_unique_value_we_can_use_to_delete_job = ""
+        @redis.lset(failed_queue_name, index_in_failed_queue, hopefully_unique_value_we_can_use_to_delete_job)
+        @redis.lrem(failed_queue_name, 1,                     hopefully_unique_value_we_can_use_to_delete_job)
+      end
+    end
+
+    class Workers
+      def initialize(redis)
+        @redis = redis
+      end
+
+      def worker_ids
+        Array(@redis.smembers(:workers))
+      end
+
+      # Given a list of worker ids, returns a map of those ids to the worker's value
+      # in redis, even if that value maps to nil
+      def workers_map(worker_ids)
+        redis_keys = worker_ids.map { |id| "worker:#{id}" }
+        @redis.mapped_mget(*redis_keys)
+      end
+
+      # return the worker's payload i.e. job
+      def get_worker_payload(worker_id)
+        @redis.get("worker:#{worker_id}")
+      end
+
+      def worker_exists?(worker_id)
+        @redis.sismember(:workers, worker_id)
+      end
+
+      def register_worker(worker)
+        @redis.pipelined do
+          @redis.sadd(:workers, worker)
+          worker_started(worker)
+        end
+      end
+
+      def worker_started(worker)
+        @redis.set(redis_key_for_worker_start_time(worker), Time.now.to_s)
+      end
+
+      def unregister_worker(worker, &block)
+        @redis.pipelined do
+          @redis.srem(:workers, worker)
+          @redis.del(redis_key_for_worker(worker))
+          @redis.del(redis_key_for_worker_start_time(worker))
+          @redis.hdel(HEARTBEAT_KEY, worker.to_s)
+
+          block.call
+        end
+      end
+
+      def remove_heartbeat(worker)
+        @redis.hdel(HEARTBEAT_KEY, worker.to_s)
+      end
+
+      def heartbeat(worker)
+        heartbeat = @redis.hget(HEARTBEAT_KEY, worker.to_s)
+        heartbeat && Time.parse(heartbeat)
+      end
+
+      def heartbeat!(worker, time)
+        @redis.hset(HEARTBEAT_KEY, worker.to_s, time.iso8601)
+      end
+
+      def all_heartbeats
+        @redis.hgetall(HEARTBEAT_KEY)
+      end
+
+      def set_worker_payload(worker, data)
+        @redis.set(redis_key_for_worker(worker), data)
+      end
+
+      def worker_start_time(worker)
+        @redis.get(redis_key_for_worker_start_time(worker))
+      end
+
+      def worker_done_working(worker, &block)
+        @redis.pipelined do
+          @redis.del(redis_key_for_worker(worker))
+          block.call
+        end
+      end
+
+    private
+
+      def redis_key_for_worker(worker)
+        "worker:#{worker}"
+      end
+
+      def redis_key_for_worker_start_time(worker)
+        "#{redis_key_for_worker(worker)}:started"
+      end
+
+    end
+
+    class StatsAccess
+      def initialize(redis)
+        @redis = redis
+      end
+      def stat(stat)
+        @redis.get("stat:#{stat}").to_i
+      end
+
+      def increment_stat(stat, by = 1)
+        @redis.incrby("stat:#{stat}", by)
+      end
+
+      def decremet_stat(stat, by = 1)
+        @redis.decrby("stat:#{stat}", by)
+      end
+
+      def clear_stat(stat)
+        @redis.del("stat:#{stat}")
+      end
+    end
+  end
+end

data/lib/resque/errors.rb

@@ -0,0 +1,21 @@
+module Resque
+  # Raised whenever we need a queue but none is provided.
+  class NoQueueError < RuntimeError; end
+
+  # Raised when trying to create a job without a class
+  class NoClassError < RuntimeError; end
+
+  # Raised when a worker was killed while processing a job.
+  class DirtyExit < RuntimeError
+    attr_reader :process_status
+
+    def initialize(message=nil, process_status=nil)
+      @process_status = process_status
+      super message
+    end
+  end
+  class PruneDeadWorkerDirtyExit < DirtyExit; end
+
+  # Raised when child process is TERM'd so job can rescue this to do shutdown work.
+  class TermException < SignalException; end
+end

data/lib/resque/failure.rb

@@ -0,0 +1,119 @@
+module Resque
+  # The Failure module provides an interface for working with different
+  # failure backends.
+  #
+  # You can use it to query the failure backend without knowing which specific
+  # backend is being used. For instance, the Resque web app uses it to display
+  # stats and other information.
+  module Failure
+    # Creates a new failure, which is delegated to the appropriate backend.
+    #
+    # Expects a hash with the following keys:
+    #   :exception - The Exception object
+    #   :worker    - The Worker object who is reporting the failure
+    #   :queue     - The string name of the queue from which the job was pulled
+    #   :payload   - The job's payload
+    def self.create(options = {})
+      backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
+    end
+
+    #
+    # Sets the current backend. Expects a class descendent of
+    # `Resque::Failure::Base`.
+    #
+    # Example use:
+    #   require 'resque/failure/airbrake'
+    #   Resque::Failure.backend = Resque::Failure::Airbrake
+    def self.backend=(backend)
+      @backend = backend
+    end
+
+    # Returns the current backend class. If none has been set, falls
+    # back to `Resque::Failure::Redis`
+    def self.backend
+      return @backend if @backend
+
+      case ENV['FAILURE_BACKEND']
+      when 'redis_multi_queue'
+        require 'resque/failure/redis_multi_queue'
+        @backend = Failure::RedisMultiQueue
+      when 'redis', nil
+        require 'resque/failure/redis'
+        @backend = Failure::Redis
+      else
+        raise ArgumentError, "invalid failure backend: #{FAILURE_BACKEND}"
+      end
+    end
+
+    # Obtain the failure queue name for a given job queue
+    def self.failure_queue_name(job_queue_name)
+      name = "#{job_queue_name}_failed"
+      Resque.data_store.add_failed_queue(name)
+      name
+    end
+
+    # Obtain the job queue name for a given failure queue
+    def self.job_queue_name(failure_queue_name)
+      failure_queue_name.sub(/_failed$/, '')
+    end
+
+    # Returns an array of all the failed queues in the system
+    def self.queues
+      backend.queues
+    end
+
+    # Returns the int count of how many failures we have seen.
+    def self.count(queue = nil, class_name = nil)
+      backend.count(queue, class_name)
+    end
+
+    # Returns an array of all the failures, paginated.
+    #
+    # `offset` is the int of the first item in the page, `limit` is the
+    # number of items to return.
+    def self.all(offset = 0, limit = 1, queue = nil)
+      backend.all(offset, limit, queue)
+    end
+
+    # Iterate across all failures with the given options
+    def self.each(offset = 0, limit = self.count, queue = nil, class_name = nil, order = 'desc', &block)
+      backend.each(offset, limit, queue, class_name, order, &block)
+    end
+
+    # The string url of the backend's web interface, if any.
+    def self.url
+      backend.url
+    end
+
+    # Clear all failure jobs
+    def self.clear(queue = nil)
+      backend.clear(queue)
+    end
+
+    def self.requeue(id, queue = nil)
+      backend.requeue(id, queue)
+    end
+
+    def self.remove(id, queue = nil)
+      backend.remove(id, queue)
+    end
+
+    # Requeues all failed jobs in a specific queue.
+    # Queue name should be a string.
+    def self.requeue_queue(queue)
+      backend.requeue_queue(queue)
+    end
+
+    # Requeues all failed jobs
+    def self.requeue_all
+      backend.requeue_all
+    end
+
+    # Removes all failed jobs in a specific queue.
+    # Queue name should be a string.
+    def self.remove_queue(queue)
+      backend.remove_queue(queue)
+    end
+
+  end
+end