grockit-resque 1.5.0

data/examples/simple.rb

@@ -0,0 +1,30 @@
+# This is a simple Resque job.
+class Archive
+  @queue = :file_serve
+
+  def self.perform(repo_id, branch = 'master')
+    repo = Repository.find(repo_id)
+    repo.create_archive(branch)
+  end
+end
+
+# This is in our app code
+class Repository < Model
+  # ... stuff ...
+
+  def async_create_archive(branch)
+    Resque.enqueue(Archive, self.id, branch)
+  end
+
+  # ... more stuff ...
+end
+
+# Calling this code:
+repo = Repository.find(22)
+repo.async_create_archive('homebrew')
+
+# Will return immediately and create a Resque job which is later
+# processed.
+
+# Essentially, this code is run by the worker when processing:
+Archive.perform(22, 'homebrew')

data/init.rb

@@ -0,0 +1 @@
+require 'resque'

data/lib/resque.rb

@@ -0,0 +1,247 @@
+require 'redis/namespace'
+
+begin
+  require 'yajl'
+rescue LoadError
+  require 'json'
+end
+
+require 'resque/version'
+
+require 'resque/errors'
+
+require 'resque/failure'
+require 'resque/failure/base'
+
+require 'resque/helpers'
+require 'resque/stat'
+require 'resque/job'
+require 'resque/worker'
+
+module Resque
+  include Helpers
+  extend self
+
+  # Accepts:
+  #   1. A 'hostname:port' string
+  #   2. A 'hostname:port:db' string (to select the Redis db)
+  #   3. An instance of `Redis`
+  def redis=(server)
+    case server
+    when String
+      host, port, db = server.split(':')
+      redis = Redis.new(:host => host, :port => port,
+        :thread_safe => true, :db => db)
+      @redis = Redis::Namespace.new(:resque, :redis => redis)
+    when Redis
+      @redis = Redis::Namespace.new(:resque, :redis => server)
+    else
+      raise "I don't know what to do with #{server.inspect}"
+    end
+  end
+
+  # Returns the current Redis connection. If none has been created, will
+  # create a new one.
+  def redis
+    return @redis if @redis
+    self.redis = 'localhost:6379'
+    self.redis
+  end
+
+  def to_s
+    "Resque Client connected to #{redis.server}"
+  end
+
+
+  #
+  # queue manipulation
+  #
+
+  # Pushes a job onto a queue. Queue name should be a string and the
+  # item should be any JSON-able Ruby object.
+  def push(queue, item)
+    watch_queue(queue)
+    queue_key = "queue:#{queue}"
+    if item[:unique]
+      unless redis.lrange(queue_key, 0, -1).include?(encode(item))
+        redis.rpush queue_key, encode(item)
+      end
+    else
+      redis.rpush queue_key, encode(item)
+    end
+  end
+
+  # Pops a job off a queue. Queue name should be a string.
+  #
+  # Returns a Ruby object.
+  def pop(queue)
+    decode redis.lpop("queue:#{queue}")
+  end
+
+  # Returns an int representing the size of a queue.
+  # Queue name should be a string.
+  def size(queue)
+    redis.llen("queue:#{queue}").to_i
+  end
+
+  # Returns an array of items currently queued. Queue name should be
+  # a string.
+  #
+  # start and count should be integer and can be used for pagination.
+  # start is the item to begin, count is how many items to return.
+  #
+  # To get the 3rd page of a 30 item, paginatied list one would use:
+  #   Resque.peek('my_list', 59, 30)
+  def peek(queue, start = 0, count = 1)
+    list_range("queue:#{queue}", start, count)
+  end
+
+  # Does the dirty work of fetching a range of items from a Redis list
+  # and converting them into Ruby objects.
+  def list_range(key, start = 0, count = 1)
+    if count == 1
+      decode redis.lindex(key, start)
+    else
+      Array(redis.lrange(key, start, start+count-1)).map do |item|
+        decode item
+      end
+    end
+  end
+
+  # Returns an array of all known Resque queues as strings.
+  def queues
+    redis.smembers(:queues)
+  end
+
+  # Given a queue name, completely deletes the queue.
+  def remove_queue(queue)
+    redis.srem(:queues, queue.to_s)
+    redis.del("queue:#{queue}")
+  end
+
+  # Used internally to keep track of which queues we've created.
+  # Don't call this directly.
+  def watch_queue(queue)
+    redis.sadd(:queues, queue.to_s)
+  end
+
+
+  #
+  # job shortcuts
+  #
+
+  # This method can be used to conveniently add a job to a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `Resque::NoQueueError`
+  #
+  # This method is considered part of the `stable` API.
+  def enqueue(klass, *args)
+    Job.create(queue_from_class(klass), klass, *args)
+  end
+  
+  def enqueue_unique(klass, *args)
+    Job.create_unique(queue_from_class(klass), klass, *args)
+  end
+
+  # This method can be used to conveniently remove a job from a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `Resque::NoQueueError`
+  #
+  # If no args are given, this method will dequeue *all* jobs matching
+  # the provided class. See `Resque::Job.destroy` for more
+  # information.
+  #
+  # Returns the number of jobs destroyed.
+  #
+  # Example:
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph`
+  #   Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph)
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph` with matching args.
+  #   Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph, 'repo:135325')
+  #
+  # This method is considered part of the `stable` API.
+  def dequeue(klass, *args)
+    Job.destroy(queue_from_class(klass), klass, *args)
+  end
+
+  # Given a class, try to extrapolate an appropriate queue based on a
+  # class instance variable or `queue` method.
+  def queue_from_class(klass)
+    klass.instance_variable_get(:@queue) ||
+      (klass.respond_to?(:queue) and klass.queue)
+  end
+
+  # This method will return a `Resque::Job` object or a non-true value
+  # depending on whether a job can be obtained. You should pass it the
+  # precise name of a queue: case matters.
+  #
+  # This method is considered part of the `stable` API.
+  def reserve(queue)
+    Job.reserve(queue)
+  end
+
+
+  #
+  # worker shortcuts
+  #
+
+  # A shortcut to Worker.all
+  def workers
+    Worker.all
+  end
+
+  # A shortcut to Worker.working
+  def working
+    Worker.working
+  end
+
+  # A shortcut to unregister_worker
+  # useful for command line tool
+  def remove_worker(worker_id)
+    worker = Resque::Worker.find(worker_id)
+    worker.unregister_worker
+  end
+
+  #
+  # stats
+  #
+
+  # Returns a hash, similar to redis-rb's #info, of interesting stats.
+  def info
+    return {
+      :pending   => queues.inject(0) { |m,k| m + size(k) },
+      :processed => Stat[:processed],
+      :queues    => queues.size,
+      :workers   => workers.size.to_i,
+      :working   => working.size,
+      :failed    => Stat[:failed],
+      :servers   => [redis.server]
+    }
+  end
+
+  # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
+  # is O(N) for the keyspace, so be careful - this can be slow for big databases.
+  def keys
+    redis.keys("*").map do |key|
+      key.sub('resque:', '')
+    end
+  end
+end

data/lib/resque/errors.rb

@@ -0,0 +1,7 @@
+module Resque
+  # Raised whenever we need a queue but none is provided.
+  class NoQueueError < RuntimeError; end
+
+  # Raised when trying to create a job without a class
+  class NoClassError < RuntimeError; end
+end

data/lib/resque/failure.rb

@@ -0,0 +1,63 @@
+module Resque
+  # The Failure module provides an interface for working with different
+  # failure backends.
+  #
+  # You can use it to query the failure backend without knowing which specific
+  # backend is being used. For instance, the Resque web app uses it to display
+  # stats and other information.
+  module Failure
+    # Creates a new failure, which is delegated to the appropriate backend.
+    #
+    # Expects a hash with the following keys:
+    #   :exception - The Exception object
+    #   :worker    - The Worker object who is reporting the failure
+    #   :queue     - The string name of the queue from which the job was pulled
+    #   :payload   - The job's payload
+    def self.create(options = {})
+      backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
+    end
+
+    #
+    # Sets the current backend. Expects a class descendent of
+    # `Resque::Failure::Base`.
+    #
+    # Example use:
+    #   require 'resque/failure/hoptoad'
+    #   Resque::Failure.backend = Resque::Failure::Hoptoad
+    def self.backend=(backend)
+      @backend = backend
+    end
+
+    # Returns the current backend class. If none has been set, falls
+    # back to `Resque::Failure::Redis`
+    def self.backend
+      return @backend if @backend
+      require 'resque/failure/redis'
+      @backend = Failure::Redis
+    end
+
+    # Returns the int count of how many failures we have seen.
+    def self.count
+      backend.count
+    end
+
+    # Returns an array of all the failures, paginated.
+    #
+    # `start` is the int of the first item in the page, `count` is the
+    # number of items to return.
+    def self.all(start = 0, count = 1)
+      backend.all(start, count)
+    end
+
+    # The string url of the backend's web interface, if any.
+    def self.url
+      backend.url
+    end
+    
+    # Clear all failure jobs
+    def self.clear
+      backend.clear
+    end
+    
+  end
+end

data/lib/resque/failure/base.rb

@@ -0,0 +1,58 @@
+module Resque
+  module Failure
+    # All Failure classes are expected to subclass Base.
+    #
+    # When a job fails, a new instance of your Failure backend is created
+    # and #save is called.
+    class Base
+      # The exception object raised by the failed job
+      attr_accessor :exception
+
+      # The worker object who detected the failure
+      attr_accessor :worker
+
+      # The string name of the queue from which the failed job was pulled
+      attr_accessor :queue
+
+      # The payload object associated with the failed job
+      attr_accessor :payload
+
+      def initialize(exception, worker, queue, payload)
+        @exception = exception
+        @worker    = worker
+        @queue     = queue
+        @payload   = payload
+      end
+
+      # When a job fails, a new instance of your Failure backend is created
+      # and #save is called.
+      #
+      # This is where you POST or PUT or whatever to your Failure service.
+      def save
+      end
+
+      # The number of failures.
+      def self.count
+        0
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        []
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+      end
+
+      # Logging!
+      def log(message)
+        @worker.log(message)
+      end
+    end
+  end
+end

data/lib/resque/failure/hoptoad.rb

@@ -0,0 +1,122 @@
+require 'net/http'
+require 'builder'
+
+module Resque
+  module Failure
+    # A Failure backend that sends exceptions raised by jobs to Hoptoad.
+    #
+    # To use it, put this code in an initializer, Rake task, or wherever:
+    #
+    #   Resque::Failure::Hoptoad.configure do |config|
+    #     config.api_key = 'blah'
+    #     config.secure = true
+    #     config.subdomain = 'your_hoptoad_subdomain'
+    #   end
+    class Hoptoad < Base
+      #from the hoptoad plugin
+      INPUT_FORMAT = %r{^([^:]+):(\d+)(?::in `([^']+)')?$}.freeze
+      
+      class << self
+        attr_accessor :secure, :api_key, :subdomain
+      end
+
+      def self.url
+        "http://#{subdomain}.hoptoadapp.com/" if subdomain
+      end
+
+      def self.count
+        # We can't get the total # of errors from Hoptoad so we fake it
+        # by asking Resque how many errors it has seen.
+        Stat[:failed]
+      end
+
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+
+      
+
+      def save
+        http = use_ssl? ? :https : :http
+        url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
+
+        http = Net::HTTP.new(url.host, url.port)
+        headers = {
+          'Content-type' => 'text/xml',
+          'Accept' => 'text/xml, application/xml'
+        }
+
+        http.read_timeout = 5 # seconds
+        http.open_timeout = 2 # seconds
+        
+        http.use_ssl = use_ssl?
+
+        begin
+          response = http.post(url.path, xml, headers)
+        rescue TimeoutError => e
+          log "Timeout while contacting the Hoptoad server."
+        end
+
+        case response
+        when Net::HTTPSuccess then
+          log "Hoptoad Success: #{response.class}"
+        else
+          body = response.body if response.respond_to? :body
+          log "Hoptoad Failure: #{response.class}\n#{body}"
+        end
+      end
+      
+      def xml
+        x = Builder::XmlMarkup.new
+        x.instruct!
+        x.notice :version=>"2.0" do
+          x.tag! "api-key", api_key
+          x.notifier do
+            x.name "Resqueue"
+            x.version "0.1"
+            x.url "http://github.com/defunkt/resque"
+          end
+          x.error do
+            x.class exception.class.name
+            x.message "#{exception.class.name}: #{exception.message}"
+            x.backtrace do
+              fill_in_backtrace_lines(x)
+            end
+          end
+          x.request do
+            x.url queue.to_s
+            x.component worker.to_s
+            x.params do
+              x.var :key=>"payload_class" do
+                x.text! payload["class"].to_s
+              end
+              x.var :key=>"payload_args" do
+                x.text! payload["args"].to_s
+              end
+            end
+          end
+          x.tag!("server-environment") do
+            x.tag!("environment-name",RAILS_ENV)
+          end
+          
+        end
+      end
+      
+      def fill_in_backtrace_lines(x)
+        exception.backtrace.each do |unparsed_line|
+          _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
+          x.line :file=>file,:number=>number
+        end
+      end
+
+      def use_ssl?
+        self.class.secure
+      end
+
+      def api_key
+        self.class.api_key
+      end
+    end
+  end
+end