scotttam-resque 0.0.1

data/Rakefile

@@ -0,0 +1,66 @@
+load 'tasks/redis.rake'
+
+$LOAD_PATH.unshift 'lib'
+require 'resque/tasks'
+
+task :default => :test
+
+desc "Run tests"
+task :test do
+  # Don't use the rake/testtask because it loads a new
+  # Ruby interpreter - we want to run tests with the current
+  # `rake` so our library manager still works
+  Dir['test/*_test.rb'].each do |f|
+    require f
+  end
+end
+
+desc "Activate kicker - gem install kicker"
+task :kick do
+  exec "kicker -e rake lib test"
+end
+
+task :install => [ 'redis:install', 'dtach:install' ]
+
+desc "Build a gem"
+task :gem => [ :test, :gemspec, :build ]
+
+begin
+  require 'jeweler'
+  require 'resque/version'
+
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "scotttam-resque"
+    gemspec.summary = %Q{scotttam-resque is an extension to the resque queue system that has pre-fork hooks and ability to turn off forking.}
+    gemspec.description = %Q{scotttam-resque is an extension to the resque queue system that has pre-fork hooks and ability to turn off forking. Add more description here.}
+    gemspec.email = "tamosunas@gmail.com"
+    gemspec.homepage = "http://github.com/scotttam/resque"
+    gemspec.authors = ["Chris Wanstrath, Scott Tamosunas"]
+    gemspec.version = Resque::Version
+
+    gemspec.add_dependency "redis"
+    gemspec.add_dependency "redis-namespace"
+    gemspec.add_dependency "vegas", ">=0.1.2"
+    gemspec.add_dependency "sinatra", ">=0.9.2"
+    gemspec.add_development_dependency "jeweler"
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: "
+  puts "gem install jeweler"
+end
+
+begin
+  require 'sdoc_helpers'
+rescue LoadError
+  puts "sdoc support not enabled. Please gem install sdoc-helpers."
+end
+
+desc "Push a new version to Gemcutter"
+task :publish => [ :test, :gemspec, :build ] do
+  system "git tag v#{Resque::Version}"
+  system "git push origin v#{Resque::Version}"
+  system "git push origin master"
+  system "gem push pkg/scotttam-resque-#{Resque::Version}.gem"
+  system "git clean -fd"
+  exec "rake pages"
+end

data/bin/resque

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'resque'
+
+def kill(worker)
+  abort "** resque kill WORKER_ID" if worker.nil?
+  pid = worker.split(':')[1].to_i
+
+  begin
+    Process.kill("KILL", pid)
+    puts "** killed #{worker}"
+  rescue Errno::ESRCH
+    puts "** worker #{worker} not running"
+  end
+
+  remove worker
+end
+
+def remove(worker)
+  abort "** resque remove WORKER_ID" if worker.nil?
+
+  Resque.remove_worker(worker)
+  puts "** removed #{worker}"
+end
+
+def list
+  if Resque.workers.any?
+    Resque.workers.each do |worker|
+      puts "#{worker} (#{worker.state})"
+    end
+  else
+    puts "None"
+  end
+end
+
+if (i = ARGV.index('-r')) && ARGV[i+1]
+  Resque.redis = ARGV[i+1]
+  ARGV.delete_at(i)
+  ARGV.delete_at(i+1)
+end
+
+case ARGV[0]
+when 'kill'
+  kill ARGV[1]
+when 'remove'
+  remove ARGV[1]
+when 'list'
+  list
+else
+  puts "Usage: resque [-r redis_host:redis_port] COMMAND [option]"
+  puts
+  puts "Commands:"
+  puts "  remove WORKER   Removes a worker"
+  puts "  kill WORKER     Kills a worker"
+  puts "  list            Lists known workers"
+end

data/bin/resque-web

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'vegas'
+rescue LoadError
+  require 'rubygems'
+  require 'vegas'
+end  
+require 'resque/server'
+
+
+Vegas::Runner.new(Resque::Server, 'resque-web', {
+  :before_run => lambda {|v|
+    path = (ENV['RESQUECONFIG'] || v.args.first)
+    load path.to_s.strip if path
+  }
+})

data/config.ru

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require 'logger'
+
+$LOAD_PATH.unshift ::File.expand_path(::File.dirname(__FILE__) + '/lib')
+require 'resque/server'
+
+# Set the RESQUECONFIG env variable if you've a `resque.rb` or similar
+# config file you want loaded on boot.
+if ENV['RESQUECONFIG'] && ::File.exists?(::File.expand_path(ENV['RESQUECONFIG']))
+  load ::File.expand_path(ENV['RESQUECONFIG'])
+end
+
+use Rack::ShowExceptions
+run Resque::Server.new

data/deps.rip

@@ -0,0 +1,6 @@
+git://github.com/ezmobius/redis-rb.git eed200ad
+git://github.com/brianmario/yajl-ruby.git 0.6.3
+git://github.com/sinatra/sinatra.git 0.9.4
+git://github.com/rack/rack.git 1.0
+git://github.com/quirkey/vegas.git v0.1.2
+rake

data/examples/async_helper.rb

@@ -0,0 +1,31 @@
+# If you want to just call a method on an object in the background,
+# we can easily add that functionality to Resque.
+#
+# This is similar to DelayedJob's `send_later`.
+#
+# Keep in mind that, unlike DelayedJob, only simple Ruby objects
+# can be persisted.
+#
+# If it can be represented in JSON, it can be stored in a job.
+
+# Here's our ActiveRecord class
+class Repository < ActiveRecord::Base
+  # This will be called by a worker when a job needs to be processed
+  def self.perform(id, method, *args)
+    find(id).send(method, *args)
+  end
+
+  # We can pass this any Repository instance method that we want to
+  # run later.
+  def async(method, *args)
+    Resque.enqueue(Repository, id, method, *args)
+  end
+end
+
+# Now we can call any method and have it execute later:
+
+@repo.async(:update_disk_usage)
+
+# or
+
+@repo.async(:update_network_source_id, 34)

data/examples/demo/README.markdown

@@ -0,0 +1,71 @@
+Resque Demo
+-----------
+
+This is a dirt simple Resque setup for you to play with.
+
+
+### Starting the Demo App
+
+Here's how to run the Sinatra app:
+
+    $ git clone git://github.com/defunkt/resque.git
+    $ cd resque/examples/demo
+    $ rackup config.ru
+    $ open http://localhost:9292/
+
+Click 'Create New Job' a few times. You should see the number of
+pending jobs rising.
+  
+
+### Starting the Demo Worker
+
+Now in another shell terminal start the worker:
+
+    $ cd resque/examples/demo
+    $ VERBOSE=true QUEUE=default rake resque:work
+
+You should see the following output:
+
+    *** Starting worker hostname:90185:default
+    *** got: (Job{default} | Demo::Job | [{}])
+    Processed a job!
+    *** done: (Job{default} | Demo::Job | [{}])
+
+You can also use `VVERBOSE` (very verbose) if you want to see more:
+
+    $ VERBOSE=true QUEUE=default rake resque:work
+    *** Starting worker hostname:90399:default
+    ** [05:55:09 2009-09-16] 90399: Registered signals
+    ** [05:55:09 2009-09-16] 90399: Checking default
+    ** [05:55:09 2009-09-16] 90399: Found job on default
+    ** [05:55:09 2009-09-16] 90399: got: (Job{default} | Demo::Job | [{}])
+    ** [05:55:09 2009-09-16] 90399: resque: Forked 90401 at 1253141709
+    ** [05:55:09 2009-09-16] 90401: resque: Processing default since 1253141709
+    Processed a job!
+    ** [05:55:10 2009-09-16] 90401: done: (Job{default} | Demo::Job | [{}])
+
+Notice that our workers `require 'job'` in our `Rakefile`. This
+ensures they have our app loaded and can access the job classes.
+
+
+### Starting the Resque frontend
+
+Great, now let's check out the Resque frontend. Either click on 'View
+Resque' in your web browser or run:
+
+    $ open http://localhost:9292/resque/
+    
+You should see the Resque web frontend. 404 page? Don't forget the
+trailing slash!
+
+
+### config.ru
+
+The `config.ru` shows you how to mount multiple Rack apps. Resque
+should work fine on a subpath - feel free to load it up in your
+Passenger app and protect it with some basic auth.
+
+
+### That's it!
+
+Click around, add some more queues, add more jobs, do whatever, have fun.

data/examples/demo/Rakefile

@@ -0,0 +1,3 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'resque/tasks'
+require 'job'

data/examples/demo/app.rb

@@ -0,0 +1,38 @@
+require 'sinatra/base'
+require 'resque'
+require 'job'
+
+module Demo
+  class App < Sinatra::Base
+    get '/' do
+      info = Resque.info
+      out = "<html><head><title>Resque Demo</title></head><body>"
+      out << "<p>"
+      out << "There are #{info[:pending]} pending and "
+      out << "#{info[:processed]} processed jobs across #{info[:queues]} queues."
+      out << "</p>"
+      out << '<form method="POST">'
+      out << '<input type="submit" value="Create New Job"/>'
+      out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+      out << '</form>'
+      
+       out << "<form action='/failing' method='POST''>"
+       out << '<input type="submit" value="Create Failing New Job"/>'
+       out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+       out << '</form>'
+      
+      out << "</body></html>"
+      out
+    end
+
+    post '/' do
+      Resque.enqueue(Job, params)
+      redirect "/"
+    end
+    
+    post '/failing' do 
+      Resque.enqueue(FailingJob, params)
+      redirect "/"
+    end
+  end
+end

data/examples/demo/config.ru

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'logger'
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'app'
+require 'resque/server'
+
+use Rack::ShowExceptions
+
+# Set the AUTH env variable to your basic auth password to protect Resque.
+AUTH_PASSWORD = ENV['AUTH']
+if AUTH_PASSWORD
+  Resque::Server.use Rack::Auth::Basic do |username, password|
+    password == AUTH_PASSWORD
+  end
+end
+
+run Rack::URLMap.new \
+  "/"       => Demo::App.new,
+  "/resque" => Resque::Server.new

data/examples/demo/job.rb

@@ -0,0 +1,22 @@
+require 'resque'
+
+module Demo
+  module Job
+    @queue = :default
+
+    def self.perform(params)
+      sleep 1
+      puts "Processed a job!"
+    end
+  end
+  
+  module FailingJob
+    @queue = :failing
+
+    def self.perform(params)
+      sleep 1
+      raise 'not processable!'
+      puts "Processed a job!"
+    end
+  end
+end

data/examples/god/resque.god

@@ -0,0 +1,53 @@
+rails_env   = ENV['RAILS_ENV']  || "production"
+rails_root  = ENV['RAILS_ROOT'] || "/data/github/current"
+num_workers = rails_env == 'production' ? 5 : 2
+
+num_workers.times do |num|
+  God.watch do |w|
+    w.name     = "resque-#{num}"
+    w.group    = 'resque'
+    w.interval = 30.seconds
+    w.env      = {"QUEUE"=>"critical,high,low", "RAILS_ENV"=>rails_env}
+    w.start    = "/usr/bin/rake -f #{rails_root}/Rakefile environment resque:work"
+
+    w.uid = 'git'
+    w.gid = 'git'
+
+    # retart if memory gets too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.above = 350.megabytes
+        c.times = 2
+      end
+    end
+
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+  end
+end

data/examples/god/stale.god

@@ -0,0 +1,26 @@
+# This will ride alongside god and kill any rogue stale worker
+# processes. Their sacrifice is for the greater good.
+
+WORKER_TIMEOUT = 60 * 10 # 10 minutes
+
+Thread.new do
+  loop do
+    begin
+      `ps -e -o pid,command | grep [r]esque`.split("\n").each do |line|
+        parts   = line.split(' ')
+        next if parts[-2] != "at"
+        started = parts[-1].to_i
+        elapsed = Time.now - Time.at(started)
+
+        if elapsed >= WORKER_TIMEOUT
+          ::Process.kill('USR1', parts[0].to_i)
+        end
+      end
+    rescue
+      # don't die because of stupid exceptions
+      nil
+    end
+
+    sleep 30
+  end
+end

data/examples/instance.rb

@@ -0,0 +1,11 @@
+# DelayedJob wants you to create instances. No problem.
+
+class Archive < Struct.new(:repo_id, :branch)
+  def self.perform(*args)
+    new(*args).perform
+  end
+
+  def perform
+    # do work!
+  end
+end

data/examples/simple.rb

@@ -0,0 +1,30 @@
+# This is a simple Resque job.
+class Archive
+  @queue = :file_serve
+
+  def self.perform(repo_id, branch = 'master')
+    repo = Repository.find(repo_id)
+    repo.create_archive(branch)
+  end
+end
+
+# This is in our app code
+class Repository < Model
+  # ... stuff ...
+
+  def async_create_archive(branch)
+    Resque.enqueue(Archive, self.id, branch)
+  end
+
+  # ... more stuff ...
+end
+
+# Calling this code:
+repo = Repository.find(22)
+repo.async_create_archive('homebrew')
+
+# Will return immediately and create a Resque job which is later
+# processed.
+
+# Essentially, this code is run by the worker when processing:
+Archive.perform(22, 'homebrew')

data/init.rb

@@ -0,0 +1 @@
+require 'resque'

data/lib/resque/errors.rb

@@ -0,0 +1,7 @@
+module Resque
+  # Raised whenever we need a queue but none is provided.
+  class NoQueueError < RuntimeError; end
+
+  # Raised when trying to create a job without a class
+  class NoClassError < RuntimeError; end
+end

data/lib/resque/failure/base.rb

@@ -0,0 +1,58 @@
+module Resque
+  module Failure
+    # All Failure classes are expected to subclass Base.
+    #
+    # When a job fails, a new instance of your Failure backend is created
+    # and #save is called.
+    class Base
+      # The exception object raised by the failed job
+      attr_accessor :exception
+
+      # The worker object who detected the failure
+      attr_accessor :worker
+
+      # The string name of the queue from which the failed job was pulled
+      attr_accessor :queue
+
+      # The payload object associated with the failed job
+      attr_accessor :payload
+
+      def initialize(exception, worker, queue, payload)
+        @exception = exception
+        @worker    = worker
+        @queue     = queue
+        @payload   = payload
+      end
+
+      # When a job fails, a new instance of your Failure backend is created
+      # and #save is called.
+      #
+      # This is where you POST or PUT or whatever to your Failure service.
+      def save
+      end
+
+      # The number of failures.
+      def self.count
+        0
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        []
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+      end
+
+      # Logging!
+      def log(message)
+        @worker.log(message)
+      end
+    end
+  end
+end

data/lib/resque/failure/hoptoad.rb

@@ -0,0 +1,121 @@
+require 'net/https'
+require 'builder'
+require 'uri'
+
+module Resque
+  module Failure
+    # A Failure backend that sends exceptions raised by jobs to Hoptoad.
+    #
+    # To use it, put this code in an initializer, Rake task, or wherever:
+    #
+    #   Resque::Failure::Hoptoad.configure do |config|
+    #     config.api_key = 'blah'
+    #     config.secure = true
+    #
+    #     # optional proxy support
+    #     config.proxy_host = 'x.y.z.t'
+    #     config.proxy_port = 8080
+    #   end
+    class Hoptoad < Base
+      # From the hoptoad plugin
+      INPUT_FORMAT = /^([^:]+):(\d+)(?::in `([^']+)')?$/
+
+      class << self
+        attr_accessor :secure, :api_key, :proxy_host, :proxy_port
+      end
+
+      def self.count
+        # We can't get the total # of errors from Hoptoad so we fake it
+        # by asking Resque how many errors it has seen.
+        Stat[:failed]
+      end
+
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+
+      def save
+        http = use_ssl? ? :https : :http
+        url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
+
+        request = Net::HTTP::Proxy(self.class.proxy_host, self.class.proxy_port)
+        http = request.new(url.host, url.port)
+        headers = {
+          'Content-type' => 'text/xml',
+          'Accept' => 'text/xml, application/xml'
+        }
+
+        http.read_timeout = 5 # seconds
+        http.open_timeout = 2 # seconds
+
+        http.use_ssl = use_ssl?
+
+        begin
+          response = http.post(url.path, xml, headers)
+        rescue TimeoutError => e
+          log "Timeout while contacting the Hoptoad server."
+        end
+
+        case response
+        when Net::HTTPSuccess then
+          log "Hoptoad Success: #{response.class}"
+        else
+          body = response.body if response.respond_to? :body
+          log "Hoptoad Failure: #{response.class}\n#{body}"
+        end
+      end
+
+      def xml
+        x = Builder::XmlMarkup.new
+        x.instruct!
+        x.notice :version=>"2.0" do
+          x.tag! "api-key", api_key
+          x.notifier do
+            x.name "Resqueue"
+            x.version "0.1"
+            x.url "http://github.com/defunkt/resque"
+          end
+          x.error do
+            x.class exception.class.name
+            x.message "#{exception.class.name}: #{exception.message}"
+            x.backtrace do
+              fill_in_backtrace_lines(x)
+            end
+          end
+          x.request do
+            x.url queue.to_s
+            x.component worker.to_s
+            x.params do
+              x.var :key=>"payload_class" do
+                x.text! payload["class"].to_s
+              end
+              x.var :key=>"payload_args" do
+                x.text! payload["args"].to_s
+              end
+            end
+          end
+          x.tag!("server-environment") do
+            x.tag!("environment-name",RAILS_ENV)
+          end
+
+        end
+      end
+
+      def fill_in_backtrace_lines(x)
+        exception.backtrace.each do |unparsed_line|
+          _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
+          x.line :file=>file,:number=>number
+        end
+      end
+
+      def use_ssl?
+        self.class.secure
+      end
+
+      def api_key
+        self.class.api_key
+      end
+    end
+  end
+end