jerefrer-resque 1.1.0

data/Rakefile

@@ -0,0 +1,61 @@
+eval File.read('tasks/redis.rake')
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/lib'
+require 'resque/tasks'
+
+task :default => :test
+
+desc "Run tests"
+task :test do
+  # Don't use the rake/testtask because it loads a new
+  # Ruby interpreter - we want to run tests with the current
+  # `rake` so our library manager still works
+  Dir['test/*_test.rb'].each do |f|
+    require f
+  end
+end
+
+desc "Activate kicker - gem install kicker"
+task :kick do
+  exec "kicker -e rake lib test"
+end
+
+task :install => [ 'redis:install', 'dtach:install' ]
+
+desc "Build a gem"
+task :gem => [ :test, :gemspec, :build ]
+
+begin
+  require 'jeweler'
+  $LOAD_PATH.unshift 'lib'
+  require 'resque/version'
+
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "jerefrer-resque"
+    gemspec.summary = ""
+    gemspec.description = ""
+    gemspec.email = "frere.jeremy@gmail.com"
+    gemspec.homepage = "http://github.com/jerefrer/resque"
+    gemspec.authors = ["Chris Wanstrath", "Jérémy Frere"]
+    gemspec.version = Resque::Version 
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: "
+  puts "gem install jeweler"
+end
+
+begin
+  require 'sdoc_helpers'
+rescue LoadError
+  puts "sdoc support not enabled. Please gem install sdoc-helpers."
+end
+
+desc "Push a new version to Gemcutter"
+task :publish => [ :test, :gemspec, :build ] do
+  system "git tag v#{Resque::Version}"
+  system "git push origin v#{Resque::Version}"
+  system "git push origin master"
+  system "gem push pkg/resque-#{Resque::Version}.gem"
+  system "git clean -fd"
+  exec "rake pages"
+end

data/bin/resque

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'resque'
+
+def kill(worker)
+  abort "** resque kill WORKER_ID" if worker.nil?
+  pid = worker.split(':')[1].to_i
+
+  begin
+    Process.kill("KILL", pid)
+    puts "** killed #{worker}"
+  rescue Errno::ESRCH
+    puts "** worker #{worker} not running"
+  end
+
+  remove worker
+end
+
+def killall
+  abort "** resque killall " if Resque.workers.empty?
+  Resque.workers.each do |worker|
+    pid = worker.to_s.split(':')[1].to_i
+    Process.kill("KILL", pid)
+    puts "** killed #{worker}"
+  end
+end
+
+def remove(worker)
+  abort "** resque remove WORKER_ID" if worker.nil?
+
+  Resque.remove_worker(worker)
+  puts "** removed #{worker}"
+end
+
+def list
+  if Resque.workers.any?
+    Resque.workers.each do |worker|
+      puts "#{worker} (#{worker.state})"
+    end
+  else
+    puts "None"
+  end
+end
+
+if (i = ARGV.index('-r')) && ARGV[i+1]
+  Resque.redis = ARGV[i+1]
+  ARGV.delete_at(i)
+  ARGV.delete_at(i+1)
+end
+
+case ARGV[0]
+when 'kill'
+  kill ARGV[1]
+when 'killall'
+  killall
+when 'remove'
+  remove ARGV[1]
+when 'list'
+  list
+else
+  puts "Usage: resque [-r redis_host:redis_port] COMMAND [option]"
+  puts
+  puts "Commands:"
+  puts "  remove WORKER   Removes a worker"
+  puts "  kill WORKER     Kills a worker"
+  puts "  killall         Kill all workers"
+  puts "  list            Lists known workers"
+end

data/bin/resque-web

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+__DIR__ = File.expand_path(File.dirname(__FILE__))
+
+if `which rackup`.empty?
+  abort "** Can't find `rackup` in PATH."
+end
+
+if ARGV.include?('-h')
+  puts <<-usage
+Usage: resque-web [ruby options] [rack options] [resque config]
+
+Starts a Resque front-end by way of `rackup`.
+
+Ruby options:
+  -e, --eval LINE          evaluate a LINE of code
+  -d, --debug              set debugging flags (set $DEBUG to true)
+  -w, --warn               turn warnings on for your script
+  -I, --include PATH       specify $LOAD_PATH (may be used more than once)
+  -r, --require LIBRARY    require the library, before executing your script
+Rack options:
+  -s, --server SERVER      serve using SERVER (webrick/mongrel)
+  -o, --host HOST          listen on HOST (default: 0.0.0.0)
+  -p, --port PORT          use PORT (default: 9292)
+  -E, --env ENVIRONMENT    use ENVIRONMENT for defaults (default: development)
+  -D, --daemonize          run daemonized in the background
+  -P, --pid FILE           file to store PID (default: rack.pid)
+Common options:
+  -h, --help               Show this message
+      --version            Show version
+usage
+else
+  if !ENV['CONFIG']&&ARGV[-1]&&ARGV[-1][0]!=?-&&(ARGV[-2]?ARGV[-2][0]!=?-:true)
+    if File.file?(file = File.expand_path(ARGV[-1]))
+      ARGV.delete_at(-1)
+      ENV['CONFIG'] = file
+    else
+      abort "** Can't find #{file}"
+    end
+  end
+
+  args = ARGV
+  ENV['RUBYLIB'] = ENV['RUBYLIB'].to_s + ':' + __DIR__ + '/../lib'
+  args.unshift '-e', 'require "resque";load ENV["CONFIG"] if ENV["CONFIG"]'
+  args.push File.expand_path(File.dirname(__FILE__) + "/../config.ru")
+  exec "rackup", *args
+end

data/config.ru

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+require 'logger'
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'resque/server'
+
+use Rack::ShowExceptions
+run Resque::Server.new

data/deps.rip

@@ -0,0 +1,5 @@
+git://github.com/ezmobius/redis-rb.git eed200ad
+git://github.com/brianmario/yajl-ruby.git 0.6.3
+git://github.com/sinatra/sinatra.git 0.9.4
+git://github.com/rack/rack.git 1.0
+rake

data/examples/async_helper.rb

@@ -0,0 +1,31 @@
+# If you want to just call a method on an object in the background,
+# we can easily add that functionality to Resque.
+#
+# This is similar to DelayedJob's `send_later`.
+#
+# Keep in mind that, unlike DelayedJob, only simple Ruby objects
+# can be persisted.
+#
+# If it can be represented in JSON, it can be stored in a job.
+
+# Here's our ActiveRecord class
+class Repository < ActiveRecord::Base
+  # This will be called by a worker when a job needs to be processed
+  def self.perform(id, method, *args)
+    find(id).send(method, *args)
+  end
+
+  # We can pass this any Repository instance method that we want to
+  # run later.
+  def async(method, *args)
+    Resque.enqueue(Repository, id, method, *args)
+  end
+end
+
+# Now we can call any method and have it execute later:
+
+@repo.async(:update_disk_usage)
+
+# or
+
+@repo.async(:update_network_source_id, 34)

data/examples/demo/README.markdown

@@ -0,0 +1,71 @@
+Resque Demo
+-----------
+
+This is a dirt simple Resque setup for you to play with.
+
+
+### Starting the Demo App
+
+Here's how to run the Sinatra app:
+
+    $ git clone git://github.com/defunkt/resque.git
+    $ cd resque/examples/demo
+    $ rackup config.ru
+    $ open http://localhost:9292/
+
+Click 'Create New Job' a few times. You should see the number of
+pending jobs rising.
+  
+
+### Starting the Demo Worker
+
+Now in another shell terminal start the worker:
+
+    $ cd resque/examples/demo
+    $ VERBOSE=true QUEUE=default rake resque:work
+
+You should see the following output:
+
+    *** Starting worker hostname:90185:default
+    *** got: (Job{default} | Demo::Job | [{}])
+    Processed a job!
+    *** done: (Job{default} | Demo::Job | [{}])
+
+You can also use `VVERBOSE` (very verbose) if you want to see more:
+
+    $ VERBOSE=true QUEUE=default rake resque:work
+    *** Starting worker hostname:90399:default
+    ** [05:55:09 2009-09-16] 90399: Registered signals
+    ** [05:55:09 2009-09-16] 90399: Checking default
+    ** [05:55:09 2009-09-16] 90399: Found job on default
+    ** [05:55:09 2009-09-16] 90399: got: (Job{default} | Demo::Job | [{}])
+    ** [05:55:09 2009-09-16] 90399: resque: Forked 90401 at 1253141709
+    ** [05:55:09 2009-09-16] 90401: resque: Processing default since 1253141709
+    Processed a job!
+    ** [05:55:10 2009-09-16] 90401: done: (Job{default} | Demo::Job | [{}])
+
+Notice that our workers `require 'job'` in our `Rakefile`. This
+ensures they have our app loaded and can access the job classes.
+
+
+### Starting the Resque frontend
+
+Great, now let's check out the Resque frontend. Either click on 'View
+Resque' in your web browser or run:
+
+    $ open http://localhost:9292/resque/
+    
+You should see the Resque web frontend. 404 page? Don't forget the
+trailing slash!
+
+
+### config.ru
+
+The `config.ru` shows you how to mount multiple Rack apps. Resque
+should work fine on a subpath - feel free to load it up in your
+Passenger app and protect it with some basic auth.
+
+
+### That's it!
+
+Click around, add some more queues, add more jobs, do whatever, have fun.

data/examples/demo/Rakefile

@@ -0,0 +1,3 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'resque/tasks'
+require 'job'

data/examples/demo/app.rb

@@ -0,0 +1,27 @@
+require 'sinatra/base'
+require 'resque'
+require 'job'
+
+module Demo
+  class App < Sinatra::Base
+    get '/' do
+      info = Resque.info
+      out = "<html><head><title>Resque Demo</title></head><body>"
+      out << "<p>"
+      out << "There are #{info[:pending]} pending and "
+      out << "#{info[:processed]} processed jobs across #{info[:queues]} queues."
+      out << "</p>"
+      out << '<form method="POST">'
+      out << '<input type="submit" value="Create New Job"/>'
+      out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+      out << '</form>'
+      out << "</body></html>"
+      out
+    end
+
+    post '/' do
+      Resque.enqueue(Job, params)
+      redirect "/"
+    end
+  end
+end

data/examples/demo/config.ru

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'logger'
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'app'
+require 'resque/server'
+
+use Rack::ShowExceptions
+
+# Set the AUTH env variable to your basic auth password to protect Resque.
+AUTH_PASSWORD = ENV['AUTH']
+if AUTH_PASSWORD
+  Resque::Server.use Rack::Auth::Basic do |username, password|
+    password == AUTH_PASSWORD
+  end
+end
+
+run Rack::URLMap.new \
+  "/"       => Demo::App.new,
+  "/resque" => Resque::Server.new

data/examples/demo/job.rb

@@ -0,0 +1,12 @@
+require 'resque'
+
+module Demo
+  module Job
+    @queue = :default
+
+    def self.perform(params)
+      sleep 1
+      puts "Processed a job!"
+    end
+  end
+end

data/examples/god/resque.god

@@ -0,0 +1,52 @@
+rails_env   = ENV['RAILS_ENV']  || "production"
+rails_root  = ENV['RAILS_ROOT'] || "/data/github/current"
+num_workers = rails_env == 'production' ? 5 : 2
+
+num_workers.times do |num|
+  God.watch do |w|
+    w.name     = "resque-#{num}"
+    w.group    = 'resque'
+    w.interval = 30.seconds
+    w.start    = "env QUEUE=critical,high,low /usr/bin/rake -f #{rails_root}/Rakefile #{rails_env} resque:work"
+
+    w.uid = 'git'
+    w.gid = 'git'
+
+    # retart if memory gets too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.above = 350.megabytes
+        c.times = 2
+      end
+    end
+
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+  end
+end

data/examples/god/stale.god

@@ -0,0 +1,26 @@
+# This will ride alongside god and kill any rogue stale worker
+# processes. Their sacrifice is for the greater good.
+
+WORKER_TIMEOUT = 60 * 10 # 10 minutes
+
+Thread.new do
+  loop do
+    begin
+      `ps -e -o pid,command | grep [r]esque`.split("\n").each do |line|
+        parts   = line.split(' ')
+        next if parts[-2] != "at"
+        started = parts[-1].to_i
+        elapsed = Time.now - Time.at(started)
+
+        if elapsed >= WORKER_TIMEOUT
+          ::Process.kill('USR1', parts[0].to_i)
+        end
+      end
+    rescue
+      # don't die because of stupid exceptions
+      nil
+    end
+
+    sleep 30
+  end
+end

data/examples/instance.rb

@@ -0,0 +1,11 @@
+# DelayedJob wants you to create instances. No problem.
+
+class Archive < Struct.new(:repo_id, :branch)
+  def self.perform(*args)
+    new(*args).perform
+  end
+
+  def perform
+    # do work!
+  end
+end

data/examples/simple.rb

@@ -0,0 +1,30 @@
+# This is a simple Resque job.
+class Archive
+  @queue = :file_serve
+
+  def self.perform(repo_id, branch = 'master')
+    repo = Repository.find(repo_id)
+    repo.create_archive(branch)
+  end
+end
+
+# This is in our app code
+class Repository < Model
+  # ... stuff ...
+
+  def async_create_archive(branch)
+    Resque.enqueue(Archive, self.id, branch)
+  end
+
+  # ... more stuff ...
+end
+
+# Calling this code:
+repo = Repository.find(22)
+repo.async_create_archive('homebrew')
+
+# Will return immediately and create a Resque job which is later
+# processed.
+
+# Essentially, this code is run by the worker when processing:
+Archive.perform(22, 'homebrew')

data/init.rb

@@ -0,0 +1 @@
+require 'resque'

data/lib/resque/errors.rb

@@ -0,0 +1,7 @@
+module Resque
+  # Raised whenever we need a queue but none is provided.
+  class NoQueueError < RuntimeError; end
+
+  # Raised when trying to create a job without a class
+  class NoClassError < RuntimeError; end
+end

data/lib/resque/failure/base.rb

@@ -0,0 +1,58 @@
+module Resque
+  module Failure
+    # All Failure classes are expected to subclass Base.
+    #
+    # When a job fails, a new instance of your Failure backend is created
+    # and #save is called.
+    class Base
+      # The exception object raised by the failed job
+      attr_accessor :exception
+
+      # The worker object who detected the failure
+      attr_accessor :worker
+
+      # The string name of the queue from which the failed job was pulled
+      attr_accessor :queue
+
+      # The payload object associated with the failed job
+      attr_accessor :payload
+
+      def initialize(exception, worker, queue, payload)
+        @exception = exception
+        @worker    = worker
+        @queue     = queue
+        @payload   = payload
+      end
+
+      # When a job fails, a new instance of your Failure backend is created
+      # and #save is called.
+      #
+      # This is where you POST or PUT or whatever to your Failure service.
+      def save
+      end
+
+      # The number of failures.
+      def self.count
+        0
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        []
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+      end
+
+      # Logging!
+      def log(message)
+        @worker.log(message)
+      end
+    end
+  end
+end

data/lib/resque/failure/hoptoad.rb

@@ -0,0 +1,88 @@
+require 'net/http'
+
+module Resque
+  module Failure
+    # A Failure backend that sends exceptions raised by jobs to Hoptoad.
+    #
+    # To use it, put this code in an initializer, Rake task, or wherever:
+    #
+    #   Resque::Failure::Hoptoad.configure do |config|
+    #     config.api_key = 'blah'
+    #     config.secure = true
+    #     config.subdomain = 'your_hoptoad_subdomain'
+    #   end
+    class Hoptoad < Base
+      class << self
+        attr_accessor :secure, :api_key, :subdomain
+      end
+
+      def self.url
+        "http://#{subdomain}.hoptoadapp.com/" if subdomain
+      end
+
+      def self.count
+        # We can't get the total # of errors from Hoptoad so we fake it
+        # by asking Resque how many errors it has seen.
+        Stat[:failed]
+      end
+
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+
+      def save
+        data = {
+          :api_key       => api_key,
+          :error_class   => exception.class.name,
+          :error_message => "#{exception.class.name}: #{exception.message}",
+          :backtrace     => exception.backtrace,
+          :environment   => {},
+          :session       => {},
+          :request       => {
+            :params => payload.merge(:worker => worker.to_s, :queue => queue.to_s)
+          }
+        }
+
+        send_to_hoptoad(:notice => data)
+      end
+
+      def send_to_hoptoad(data)
+        http = use_ssl? ? :https : :http
+        url = URI.parse("#{http}://hoptoadapp.com/notices/")
+
+        http = Net::HTTP.new(url.host, url.port)
+        headers = {
+          'Content-type' => 'application/json',
+          'Accept' => 'text/xml, application/xml'
+        }
+
+        http.read_timeout = 5 # seconds
+        http.open_timeout = 2 # seconds
+        http.use_ssl = use_ssl?
+
+        begin
+          response = http.post(url.path, Resque.encode(data), headers)
+        rescue TimeoutError => e
+          log "Timeout while contacting the Hoptoad server."
+        end
+
+        case response
+        when Net::HTTPSuccess then
+          log "Hoptoad Success: #{response.class}"
+        else
+          body = response.body if response.respond_to? :body
+          log "Hoptoad Failure: #{response.class}\n#{body}"
+        end
+      end
+
+      def use_ssl?
+        self.class.secure
+      end
+
+      def api_key
+        self.class.api_key
+      end
+    end
+  end
+end

data/lib/resque/failure/redis.rb

@@ -0,0 +1,33 @@
+module Resque
+  module Failure
+    # A Failure backend that stores exceptions in Redis. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class Redis < Base
+      def save
+        data = {
+          :failed_at => Time.now.to_s,
+          :payload   => payload,
+          :error     => exception.to_s,
+          :backtrace => exception.backtrace,
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        data = Resque.encode(data)
+        Resque.redis.rpush(:failed, data)
+      end
+
+      def self.count
+        Resque.redis.llen(:failed).to_i
+      end
+
+      def self.all(start = 0, count = 1)
+        Resque.list_range(:failed, start, count)
+      end
+      
+      def self.clear
+        Resque.redis.delete('resque:failed')
+      end
+      
+    end
+  end
+end