resque 1.3.1 → 1.4.0

data/HISTORY.md

@@ -1,3 +1,17 @@
+## 1.4.0 (2010-02-11)
+
+* Fallback when unable to bind QUIT and USR1 for Windows and JRuby.
+* Fallback when no `Kernel.fork` is provided (for IronRuby).
+* Web: Rounded corners in Firefox
+* Cut down system calls in `Worker#prune_dead_workers`
+* Enable switching DB in a Redis server from config
+* Support USR2 and CONT to stop and start job processing.
+* Web: Add example failing job
+* Bugfix: `Worker#unregister_worker` shouldn't call `done_working`
+* Bugfix: Example god config now restarts Resque properly.
+* Multiple failure backends now permitted.
+* Hoptoad failure backend updated to new API
+
 ## 1.3.1 (2010-01-11)
 
 * Vegas bugfix: Don't error without a config

data/README.markdown

@@ -365,6 +365,8 @@ Resque workers respond to a few different signals:
 * `QUIT` - Wait for child to finish processing then exit
 * `TERM` / `INT` - Immediately kill child then exit
 * `USR1` - Immediately kill child but don't exit
+* `USR2` - Don't start to process any new jobs
+* `CONT` - Start to process new jobs again after a USR2
 
 If you want to gracefully shutdown a Resque worker, use `QUIT`.
 
@@ -374,6 +376,10 @@ Resque assumes the parent process is in a bad state and shuts down.
 
 If you want to kill a stale or stuck child and shutdown, use `TERM`
 
+If you want to stop processing jobs, but want to leave the worker running
+(for example, to temporarily alleviate load), use `USR2` to stop processing,
+then `CONT` to start it again.
+
 ### Mysql::Error: MySQL server has gone away
 
 If your workers remain idle for too long they may lose their MySQL
@@ -570,7 +576,7 @@ Now start your application:
 
 That's it! You can now create Resque jobs from within your app.
 
-To start a worker, add this to your Rakefile in RAILS_ROOT:
+To start a worker, add this to your Rakefile in `RAILS_ROOT`:
 
     require 'resque/tasks'
 
@@ -607,9 +613,9 @@ Resque has a `redis` setter which can be given a string or a Redis
 object. This means if you're already using Redis in your app, Resque
 can re-use the existing connection.
 
-String: `Resque.redis = 'localhost:6379'
+String: `Resque.redis = 'localhost:6379'`
 
-Redis: `Redus.redis = $redis`
+Redis: `Resque.redis = $redis`
 
 For our rails app we have a `config/initializers/resque.rb` file where
 we load `config/resque.yml` by hand and set the Redis information
@@ -634,11 +640,30 @@ And our initializer:
 Easy peasy! Why not just use `RAILS_ROOT` and `RAILS_ENV`? Because
 this way we can tell our Sinatra app about the config file:
 
-   $ RAILS_ENV=production resque-web rails_root/config/initializers/resque.rb
+    $ RAILS_ENV=production resque-web rails_root/config/initializers/resque.rb
 
 Now everyone is on the same page.
 
 
+Namespaces
+----------
+
+If you're running multiple, separate instances of Resque you may want
+to namespace the keyspaces so they do not overlap. This is not unlike
+the approach taken by many memcached clients.
+
+This feature is provided by the [redis-namespace][rs] library, which
+Resque uses by default to separate the keys it manages from other keys
+in your Redis server.
+
+Simply use the `Resque.redis.namespace` accessor:
+
+    Resque.redis.namespace = "resque:GitHub"
+
+We recommend sticking this in your initializer somewhere after Redis
+is configured.
+
+
 Demo
 ----
 
@@ -734,3 +759,4 @@ Chris Wanstrath :: chris@ozmm.org :: @defunkt
 [1]: http://help.github.com/forking/
 [2]: http://github.com/defunkt/resque/issues
 [sv]: http://semver.org/
+[rs]: http://github.com/defunkt/redis-namespace

data/examples/demo/app.rb

@@ -15,6 +15,12 @@ module Demo
       out << '<input type="submit" value="Create New Job"/>'
       out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
       out << '</form>'
+      
+       out << "<form action='/failing' method='POST''>"
+       out << '<input type="submit" value="Create Failing New Job"/>'
+       out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+       out << '</form>'
+      
       out << "</body></html>"
       out
     end
@@ -23,5 +29,10 @@ module Demo
       Resque.enqueue(Job, params)
       redirect "/"
     end
+    
+    post '/failing' do 
+      Resque.enqueue(FailingJob, params)
+      redirect "/"
+    end
   end
 end

data/examples/demo/job.rb

@@ -9,4 +9,14 @@ module Demo
       puts "Processed a job!"
     end
   end
+  
+  module FailingJob
+    @queue = :failing
+
+    def self.perform(params)
+      sleep 1
+      raise 'not processable!'
+      puts "Processed a job!"
+    end
+  end
 end

data/examples/god/resque.god

@@ -7,7 +7,8 @@ num_workers.times do |num|
     w.name     = "resque-#{num}"
     w.group    = 'resque'
     w.interval = 30.seconds
-    w.start    = "env QUEUE=critical,high,low /usr/bin/rake -f #{rails_root}/Rakefile #{rails_env} resque:work"
+    w.env      = {"QUEUE"=>"critical,high,low", "RAILS_ENV"=>rails_env}
+    w.start    = "/usr/bin/rake -f #{rails_root}/Rakefile environment resque:work"
 
     w.uid = 'git'
     w.gid = 'git'

data/lib/resque.rb

@@ -20,12 +20,16 @@ module Resque
   include Helpers
   extend self
 
-  # Accepts a 'hostname:port' string or a Redis server.
+  # Accepts:
+  #   1. A 'hostname:port' string
+  #   2. A 'hostname:port:db' string (to select the Redis db)
+  #   3. An instance of `Redis`
   def redis=(server)
     case server
     when String
-      host, port = server.split(':')
-      redis = Redis.new(:host => host, :port => port, :thread_safe => true)
+      host, port, db = server.split(':')
+      redis = Redis.new(:host => host, :port => port,
+        :thread_safe => true, :db => db)
       @redis = Redis::Namespace.new(:resque, :redis => redis)
     when Redis
       @redis = Redis::Namespace.new(:resque, :redis => server)
@@ -160,6 +164,12 @@ module Resque
     Worker.working
   end
 
+  # A shortcut to unregister_worker
+  # useful for command line tool
+  def remove_worker(worker_id)
+    worker = Resque::Worker.find(worker_id)
+    worker.unregister_worker
+  end
 
   #
   # stats

data/lib/resque/failure/hoptoad.rb

@@ -1,4 +1,5 @@
 require 'net/http'
+require 'builder'
 
 module Resque
   module Failure
@@ -12,6 +13,9 @@ module Resque
     #     config.subdomain = 'your_hoptoad_subdomain'
     #   end
     class Hoptoad < Base
+      #from the hoptoad plugin
+      INPUT_FORMAT = %r{^([^:]+):(\d+)(?::in `([^']+)')?$}.freeze
+      
       class << self
         attr_accessor :secure, :api_key, :subdomain
       end
@@ -31,38 +35,25 @@ module Resque
         Resque::Failure.backend = self
       end
 
-      def save
-        data = {
-          :api_key       => api_key,
-          :error_class   => exception.class.name,
-          :error_message => "#{exception.class.name}: #{exception.message}",
-          :backtrace     => exception.backtrace,
-          :environment   => {},
-          :session       => {},
-          :request       => {
-            :params => payload.merge(:worker => worker.to_s, :queue => queue.to_s)
-          }
-        }
+      
 
-        send_to_hoptoad(:notice => data)
-      end
-
-      def send_to_hoptoad(data)
+      def save
         http = use_ssl? ? :https : :http
-        url = URI.parse("#{http}://hoptoadapp.com/notices/")
+        url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
 
         http = Net::HTTP.new(url.host, url.port)
         headers = {
-          'Content-type' => 'application/json',
+          'Content-type' => 'text/xml',
           'Accept' => 'text/xml, application/xml'
         }
 
         http.read_timeout = 5 # seconds
         http.open_timeout = 2 # seconds
+        
         http.use_ssl = use_ssl?
 
         begin
-          response = http.post(url.path, Resque.encode(data), headers)
+          response = http.post(url.path, xml, headers)
         rescue TimeoutError => e
           log "Timeout while contacting the Hoptoad server."
         end
@@ -75,6 +66,49 @@ module Resque
           log "Hoptoad Failure: #{response.class}\n#{body}"
         end
       end
+      
+      def xml
+        x = Builder::XmlMarkup.new
+        x.instruct!
+        x.notice :version=>"2.0" do
+          x.tag! "api-key", api_key
+          x.notifier do
+            x.name "Resqueue"
+            x.version "0.1"
+            x.url "http://github.com/defunkt/resque"
+          end
+          x.error do
+            x.class exception.class.name
+            x.message "#{exception.class.name}: #{exception.message}"
+            x.backtrace do
+              fill_in_backtrace_lines(x)
+            end
+          end
+          x.request do
+            x.url queue.to_s
+            x.component worker.to_s
+            x.params do
+              x.var :key=>"payload_class" do
+                x.text! payload["class"].to_s
+              end
+              x.var :key=>"payload_args" do
+                x.text! payload["args"].to_s
+              end
+            end
+          end
+          x.tag!("server-environment") do
+            x.tag!("environment-name",RAILS_ENV)
+          end
+          
+        end
+      end
+      
+      def fill_in_backtrace_lines(x)
+        exception.backtrace.each do |unparsed_line|
+          _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
+          x.line :file=>file,:number=>number
+        end
+      end
 
       def use_ssl?
         self.class.secure

data/lib/resque/failure/multiple.rb

@@ -0,0 +1,44 @@
+module Resque
+  module Failure
+    # A Failure backend that uses multiple backends
+    # delegates all queries to the first backend
+    class Multiple < Base
+      
+      class << self
+        attr_accessor :classes
+      end
+      
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+      
+      def initialize(*args)
+        @backends = self.class.classes.map {|klass| klass.new(*args)}
+      end
+      def save
+        @backends.each(&:save)
+      end
+      
+      # The number of failures.
+      def self.count
+        classes.first.count
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        classes.first.all(start,count)
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+        classes.first.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+        classes.first.clear
+      end
+    end
+  end
+end

data/lib/resque/server/public/style.css

@@ -4,7 +4,7 @@ body { padding:0; margin:0; }
 .header { background:#000; padding:8px 5% 0 5%; border-bottom:1px solid #444;border-bottom:5px solid #ce1212;}
 .header h1 { color:#333; font-size:90%; font-weight:bold; margin-bottom:6px;}
 .header ul li { display:inline;}
-.header ul li a { color:#fff; text-decoration:none; margin-right:10px; display:inline-block;  padding:8px; -webkit-border-top-right-radius:6px; -webkit-border-top-left-radius:6px; }
+.header ul li a { color:#fff; text-decoration:none; margin-right:10px; display:inline-block;  padding:8px; -webkit-border-top-right-radius:6px; -webkit-border-top-left-radius:6px; -moz-border-radius-topleft:6px; -moz-border-radius-topright:6px; }
 .header ul li a:hover { background:#333;}
 .header ul li.current a { background:#ce1212; font-weight:bold; color:#fff;}
 
@@ -72,4 +72,4 @@ body { padding:0; margin:0; }
 #main p.pagination a.less { float:left;}
 #main p.pagination a.more { float:right;}
 
-#main form.clear-failed {float:right; margin-top:-10px;}
+#main form.clear-failed {float:right; margin-top:-10px;}

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = '1.3.1'
+  Version = '1.4.0'
 end

data/lib/resque/worker.rb

@@ -92,14 +92,14 @@ module Resque
     #
     # The following events occur during a worker's life cycle:
     #
-    # 1. startup: Signals are registered, dead workers are pruned,
-    #             and this worker is registered.
-    # 2. work loop: Jobs are pulled from a queue and processed
-    # 3. teardown: This worker is unregistered.
+    # 1. Startup:   Signals are registered, dead workers are pruned,
+    #               and this worker is registered.
+    # 2. Work loop: Jobs are pulled from a queue and processed.
+    # 3. Teardown:  This worker is unregistered.
     #
-    # Can be passed an integered representing the polling
-    # frequency. The default is 5 seconds, but for a semi-active site
-    # you may want to use a smaller value.
+    # Can be passed an integer representing the polling frequency.
+    # The default is 5 seconds, but for a semi-active site you may
+    # want to use a smaller value.
     #
     # Also accepts a block which will be passed the job as soon as it
     # has completed processing. Useful for testing.
@@ -110,7 +110,7 @@ module Resque
       loop do
         break if @shutdown
 
-        if job = reserve
+        if not @paused and job = reserve
           log "got: #{job.inspect}"
 
           if @child = fork
@@ -131,7 +131,7 @@ module Resque
         else
           break if interval.to_i == 0
           log! "Sleeping for #{interval.to_i}"
-          $0 = "resque: Waiting for #{@queues.join(',')}"
+          $0 = @paused ? "resque: Paused" : "resque: Waiting for #{@queues.join(',')}"
           sleep interval.to_i
         end
       end
@@ -189,7 +189,12 @@ module Resque
       return if @cant_fork
 
       begin
-        Kernel.fork
+        # IronRuby doesn't support `Kernel.fork` yet
+        if Kernel.respond_to?(:fork)
+          Kernel.fork
+        else
+          raise NotImplementedError
+        end
       rescue NotImplementedError
         @cant_fork = true
         nil
@@ -218,13 +223,21 @@ module Resque
     #  INT: Shutdown immediately, stop processing jobs.
     # QUIT: Shutdown after the current job has finished processing.
     # USR1: Kill the forked child immediately, continue processing jobs.
+    # USR2: Don't process any new jobs
+    # CONT: Start processing jobs again after a USR2
     def register_signal_handlers
       trap('TERM') { shutdown!  }
       trap('INT')  { shutdown!  }
-      unless defined? JRUBY_VERSION
+
+      begin
         trap('QUIT') { shutdown   }
         trap('USR1') { kill_child }
+        trap('USR2') { pause_processing }
+        trap('CONT') { unpause_processing }
+      rescue ArgumentError
+        warn "Signals QUIT, USR1, USR2, and/or CONT not supported."
       end
+
       log! "Registered signals"
     end
 
@@ -255,21 +268,36 @@ module Resque
       end
     end
 
+    # Stop processing jobs after the current one has completed (if we're
+    # currently running one).
+    def pause_processing
+      log "USR2 received; pausing job processing"
+      @paused = true
+    end
+
+    # Start processing jobs again after a pause
+    def unpause_processing
+      log "CONT received; resuming job processing"
+      @paused = false
+    end
+
     # Looks for any workers which should be running on this server
     # and, if they're not, removes them from Redis.
     #
     # This is a form of garbage collection. If a server is killed by a
     # hard shutdown, power failure, or something else beyond our
-    # control, the Resque workers will not die gracefully and therefor
+    # control, the Resque workers will not die gracefully and therefore
     # will leave stale state information in Redis.
     #
     # By checking the current Redis state against the actual
     # environment, we can determine if Redis is old and clean it up a bit.
     def prune_dead_workers
-      Worker.all.each do |worker|
+      all_workers = Worker.all
+      known_workers = worker_pids unless all_workers.empty?
+      all_workers.each do |worker|
         host, pid, queues = worker.id.split(':')
         next unless host == hostname
-        next if worker_pids.include?(pid)
+        next if known_workers.include?(pid)
         log! "Pruning dead worker: #{worker}"
         worker.unregister_worker
       end
@@ -284,8 +312,6 @@ module Resque
 
     # Unregisters ourself as a worker. Useful when shutting down.
     def unregister_worker
-      done_working
-
       redis.srem(:workers, self)
       redis.del("worker:#{self}:started")
 

data/tasks/redis.rake

@@ -29,7 +29,8 @@ class RedisRunner
   def self.start
     puts 'Detach with Ctrl+\  Re-attach with rake redis:attach'
     sleep 1
-    exec "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
+    command = "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
+    sh command
   end
 
   def self.attach
@@ -80,8 +81,9 @@ namespace :redis do
     puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
 
     ENV['PREFIX'] and conf_dir = "#{ENV['PREFIX']}/etc" or conf_dir = '/etc'
-    unless File.exists?("#{conf_dir}")
-      sh "cp /tmp/redis/redis.conf #{conf_dir}"
+    unless File.exists?("#{conf_dir}/redis.conf")
+      sh "mkdir #{conf_dir}" unless File.exists?("#{conf_dir}")
+      sh "cp /tmp/redis/redis.conf #{conf_dir}/redis.conf"
       puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
     end
   end

data/test/worker_test.rb

@@ -114,6 +114,14 @@ context "Resque::Worker" do
     assert_equal [], Resque.workers
   end
 
+  test "removes worker with stringified id" do
+    @worker.work(0) do
+      worker_id = Resque.workers[0].to_s
+      Resque.remove_worker(worker_id)
+      assert_equal [], Resque.workers
+   end
+  end
+
   test "records what it is working on" do
     @worker.work(0) do
       task = @worker.job
@@ -226,4 +234,9 @@ context "Resque::Worker" do
       assert_equal 1, Resque.workers.size
     end
   end
+
+  test "Processed jobs count" do
+    @worker.work(0)
+    assert_equal 1, Resque.info[:processed]
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: resque
 version: !ruby/object:Gem::Version 
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors: 
 - Chris Wanstrath
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-11 00:00:00 -08:00
+date: 2010-02-11 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -100,6 +100,7 @@ files:
 - lib/resque/failure.rb
 - lib/resque/failure/base.rb
 - lib/resque/failure/hoptoad.rb
+- lib/resque/failure/multiple.rb
 - lib/resque/failure/redis.rb
 - lib/resque/helpers.rb
 - lib/resque/job.rb