resque-mongo 1.3.1 → 1.4.0

data/HISTORY.md

@@ -1,3 +1,17 @@
+## 1.4.0 (2010-02-11)
+
+* Fallback when unable to bind QUIT and USR1 for Windows and JRuby.
+* Fallback when no `Kernel.fork` is provided (for IronRuby).
+* Web: Rounded corners in Firefox
+* Cut down system calls in `Worker#prune_dead_workers`
+* Enable switching DB in a Redis server from config
+* Support USR2 and CONT to stop and start job processing.
+* Web: Add example failing job
+* Bugfix: `Worker#unregister_worker` shouldn't call `done_working`
+* Bugfix: Example god config now restarts Resque properly.
+* Multiple failure backends now permitted.
+* Hoptoad failure backend updated to new API
+
 ## 1.3.1 (2010-01-11)
 
 * Vegas bugfix: Don't error without a config

data/README.markdown

@@ -383,6 +383,8 @@ Resque workers respond to a few different signals:
 * `QUIT` - Wait for child to finish processing then exit
 * `TERM` / `INT` - Immediately kill child then exit
 * `USR1` - Immediately kill child but don't exit
+* `USR2` - Don't start to process any new jobs
+* `CONT` - Start to process new jobs again after a USR2
 
 If you want to gracefully shutdown a Resque worker, use `QUIT`.
 
@@ -392,6 +394,10 @@ Resque assumes the parent process is in a bad state and shuts down.
 
 If you want to kill a stale or stuck child and shutdown, use `TERM`
 
+If you want to stop processing jobs, but want to leave the worker running
+(for example, to temporarily alleviate load), use `USR2` to stop processing,
+then `CONT` to start it again.
+
 ### Mysql::Error: MySQL server has gone away
 
 If your workers remain idle for too long they may lose their MySQL
@@ -627,7 +633,7 @@ can re-use the existing connection.
 
 String: `Resque.redis = 'localhost:6379'`
 
-Redis: `Redus.redis = $redis`
+Redis: `Resque.redis = $redis`
 
 For our rails app we have a `config/initializers/resque.rb` file where
 we load `config/resque.yml` by hand and set the Redis information
@@ -657,6 +663,25 @@ this way we can tell our Sinatra app about the config file:
 Now everyone is on the same page.
 
 
+Namespaces
+----------
+
+If you're running multiple, separate instances of Resque you may want
+to namespace the keyspaces so they do not overlap. This is not unlike
+the approach taken by many memcached clients.
+
+This feature is provided by the [redis-namespace][rs] library, which
+Resque uses by default to separate the keys it manages from other keys
+in your Redis server.
+
+Simply use the `Resque.redis.namespace` accessor:
+
+    Resque.redis.namespace = "resque:GitHub"
+
+We recommend sticking this in your initializer somewhere after Redis
+is configured.
+
+
 Demo
 ----
 
@@ -752,4 +777,4 @@ Chris Wanstrath :: chris@ozmm.org :: @defunkt
 [1]: http://help.github.com/forking/
 [2]: http://github.com/defunkt/resque/issues
 [sv]: http://semver.org/
-[resque]: http://github.com/defunct/resque
+[rs]: http://github.com/defunkt/redis-namespace

data/examples/demo/app.rb

@@ -15,6 +15,12 @@ module Demo
       out << '<input type="submit" value="Create New Job"/>'
       out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
       out << '</form>'
+      
+       out << "<form action='/failing' method='POST''>"
+       out << '<input type="submit" value="Create Failing New Job"/>'
+       out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+       out << '</form>'
+      
       out << "</body></html>"
       out
     end
@@ -23,5 +29,10 @@ module Demo
       Resque.enqueue(Job, params)
       redirect "/"
     end
+    
+    post '/failing' do 
+      Resque.enqueue(FailingJob, params)
+      redirect "/"
+    end
   end
 end

data/examples/demo/job.rb

@@ -9,4 +9,14 @@ module Demo
       puts "Processed a job!"
     end
   end
+  
+  module FailingJob
+    @queue = :failing
+
+    def self.perform(params)
+      sleep 1
+      raise 'not processable!'
+      puts "Processed a job!"
+    end
+  end
 end

data/examples/god/resque.god

@@ -7,7 +7,8 @@ num_workers.times do |num|
     w.name     = "resque-#{num}"
     w.group    = 'resque'
     w.interval = 30.seconds
-    w.start    = "env QUEUE=critical,high,low /usr/bin/rake -f #{rails_root}/Rakefile #{rails_env} resque:work"
+    w.env      = {"QUEUE"=>"critical,high,low", "RAILS_ENV"=>rails_env}
+    w.start    = "/usr/bin/rake -f #{rails_root}/Rakefile environment resque:work"
 
     w.uid = 'git'
     w.gid = 'git'

data/lib/resque.rb

@@ -198,6 +198,12 @@ module Resque
     Worker.working
   end
 
+  # A shortcut to unregister_worker
+  # useful for command line tool
+  def remove_worker(worker_id)
+    worker = Resque::Worker.find(worker_id)
+    worker.unregister_worker
+  end
 
   #
   # stats

data/lib/resque/failure/hoptoad.rb

@@ -1,4 +1,5 @@
 require 'net/http'
+require 'builder'
 
 module Resque
   module Failure
@@ -12,6 +13,9 @@ module Resque
     #     config.subdomain = 'your_hoptoad_subdomain'
     #   end
     class Hoptoad < Base
+      #from the hoptoad plugin
+      INPUT_FORMAT = %r{^([^:]+):(\d+)(?::in `([^']+)')?$}.freeze
+      
       class << self
         attr_accessor :secure, :api_key, :subdomain
       end
@@ -31,38 +35,25 @@ module Resque
         Resque::Failure.backend = self
       end
 
-      def save
-        data = {
-          :api_key       => api_key,
-          :error_class   => exception.class.name,
-          :error_message => "#{exception.class.name}: #{exception.message}",
-          :backtrace     => exception.backtrace,
-          :environment   => {},
-          :session       => {},
-          :request       => {
-            :params => payload.merge(:worker => worker.to_s, :queue => queue.to_s)
-          }
-        }
+      
 
-        send_to_hoptoad(:notice => data)
-      end
-
-      def send_to_hoptoad(data)
+      def save
         http = use_ssl? ? :https : :http
-        url = URI.parse("#{http}://hoptoadapp.com/notices/")
+        url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
 
         http = Net::HTTP.new(url.host, url.port)
         headers = {
-          'Content-type' => 'application/json',
+          'Content-type' => 'text/xml',
           'Accept' => 'text/xml, application/xml'
         }
 
         http.read_timeout = 5 # seconds
         http.open_timeout = 2 # seconds
+        
         http.use_ssl = use_ssl?
 
         begin
-          response = http.post(url.path, Resque.encode(data), headers)
+          response = http.post(url.path, xml, headers)
         rescue TimeoutError => e
           log "Timeout while contacting the Hoptoad server."
         end
@@ -75,6 +66,49 @@ module Resque
           log "Hoptoad Failure: #{response.class}\n#{body}"
         end
       end
+      
+      def xml
+        x = Builder::XmlMarkup.new
+        x.instruct!
+        x.notice :version=>"2.0" do
+          x.tag! "api-key", api_key
+          x.notifier do
+            x.name "Resqueue"
+            x.version "0.1"
+            x.url "http://github.com/defunkt/resque"
+          end
+          x.error do
+            x.class exception.class.name
+            x.message "#{exception.class.name}: #{exception.message}"
+            x.backtrace do
+              fill_in_backtrace_lines(x)
+            end
+          end
+          x.request do
+            x.url queue.to_s
+            x.component worker.to_s
+            x.params do
+              x.var :key=>"payload_class" do
+                x.text! payload["class"].to_s
+              end
+              x.var :key=>"payload_args" do
+                x.text! payload["args"].to_s
+              end
+            end
+          end
+          x.tag!("server-environment") do
+            x.tag!("environment-name",RAILS_ENV)
+          end
+          
+        end
+      end
+      
+      def fill_in_backtrace_lines(x)
+        exception.backtrace.each do |unparsed_line|
+          _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
+          x.line :file=>file,:number=>number
+        end
+      end
 
       def use_ssl?
         self.class.secure

data/lib/resque/failure/multiple.rb

@@ -0,0 +1,44 @@
+module Resque
+  module Failure
+    # A Failure backend that uses multiple backends
+    # delegates all queries to the first backend
+    class Multiple < Base
+      
+      class << self
+        attr_accessor :classes
+      end
+      
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+      
+      def initialize(*args)
+        @backends = self.class.classes.map {|klass| klass.new(*args)}
+      end
+      def save
+        @backends.each(&:save)
+      end
+      
+      # The number of failures.
+      def self.count
+        classes.first.count
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        classes.first.all(start,count)
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+        classes.first.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+        classes.first.clear
+      end
+    end
+  end
+end

data/lib/resque/server/public/style.css

@@ -1,23 +1,23 @@
 html { background:#efefef; font-family:Arial, Verdana, sans-serif; font-size:13px; }
 body { padding:0; margin:0; }
 
-.header { background:#000; padding:8px 5% 0 5%; border-bottom:1px solid #444;border-bottom:5px solid #ce1212;}
+.header { background:#000; padding:8px 5% 0 5%; border-bottom:1px solid #444;border-bottom:5px solid #429234;}
 .header h1 { color:#333; font-size:90%; font-weight:bold; margin-bottom:6px;}
 .header ul li { display:inline;}
-.header ul li a { color:#fff; text-decoration:none; margin-right:10px; display:inline-block;  padding:8px; -webkit-border-top-right-radius:6px; -webkit-border-top-left-radius:6px; }
+.header ul li a { color:#fff; text-decoration:none; margin-right:10px; display:inline-block;  padding:8px; -webkit-border-top-right-radius:6px; -webkit-border-top-left-radius:6px; -moz-border-radius-topleft:6px; -moz-border-radius-topright:6px; }
 .header ul li a:hover { background:#333;}
-.header ul li.current a { background:#ce1212; font-weight:bold; color:#fff;}
+.header ul li.current a { background:#429234; font-weight:bold; color:#fff;}
 
-.subnav { padding:2px 5% 7px 5%; background:#ce1212; font-size:90%;}
+.subnav { padding:2px 5% 7px 5%; background:#429234; font-size:90%;}
 .subnav li { display:inline;}
-.subnav li a { color:#fff; text-decoration:none; margin-right:10px; display:inline-block; background:#dd5b5b; padding:5px; -webkit-border-radius:3px; -moz-border-radius:3px;}
-.subnav li.current a { background:#fff; font-weight:bold; color:#ce1212;}
+.subnav li a { color:#fff; text-decoration:none; margin-right:10px; display:inline-block; background:#55ad46; padding:5px; -webkit-border-radius:3px; -moz-border-radius:3px;}
+.subnav li.current a { background:#fff; font-weight:bold; color:#429234;}
 .subnav li a:active { background:#b00909;}
 
 #main { padding:10px 5%; background:#fff; overflow:hidden; }
 #main .logo { float:right; margin:10px;}
 #main span.hl { background:#efefef; padding:2px;}
-#main h1 { margin:10px 0; font-size:190%; font-weight:bold; color:#ce1212;}
+#main h1 { margin:10px 0; font-size:190%; font-weight:bold; color:#429234;}
 #main h2 { margin:10px 0; font-size:130%;}
 #main table { width:100%; margin:10px 0;}
 #main table tr td, #main table tr th { border:1px solid #ccc; padding:6px;}
@@ -31,8 +31,8 @@ body { padding:0; margin:0; }
 
 #main table.queues { width:40%;}
 #main table.queues td.queue { font-weight:bold; width:50%;}
-#main table.queues tr.failed td { background:#ffecec; border-top:2px solid #d37474; font-size:90%; color:#d37474;}
-#main table.queues tr.failed td a{  color:#d37474;}
+#main table.queues tr.failed td { background:#ebffed; border-top:2px solid #6fd380; font-size:90%; color:#6fd380;}
+#main table.queues tr.failed td a{  color:#6fd380;}
 
 #main table.jobs td.class { font-family:Monaco, "Courier New", monospace; font-size:90%; width:50%;}
 #main table.jobs td.args{ width:50%;}
@@ -72,4 +72,4 @@ body { padding:0; margin:0; }
 #main p.pagination a.less { float:left;}
 #main p.pagination a.more { float:right;}
 
-#main form.clear-failed {float:right; margin-top:-10px;}
+#main form.clear-failed {float:right; margin-top:-10px;}

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = '1.3.1'
+  Version = '1.4.0'
 end

data/lib/resque/worker.rb

@@ -89,14 +89,14 @@ module Resque
     #
     # The following events occur during a worker's life cycle:
     #
-    # 1. startup: Signals are registered, dead workers are pruned,
-    #             and this worker is registered.
-    # 2. work loop: Jobs are pulled from a queue and processed
-    # 3. teardown: This worker is unregistered.
+    # 1. Startup:   Signals are registered, dead workers are pruned,
+    #               and this worker is registered.
+    # 2. Work loop: Jobs are pulled from a queue and processed.
+    # 3. Teardown:  This worker is unregistered.
     #
-    # Can be passed an integered representing the polling
-    # frequency. The default is 5 seconds, but for a semi-active site
-    # you may want to use a smaller value.
+    # Can be passed an integer representing the polling frequency.
+    # The default is 5 seconds, but for a semi-active site you may
+    # want to use a smaller value.
     #
     # Also accepts a block which will be passed the job as soon as it
     # has completed processing. Useful for testing.
@@ -107,7 +107,7 @@ module Resque
       loop do
         break if @shutdown
 
-        if job = reserve
+        if not @paused and job = reserve
           log "got: #{job.inspect}"
 
           if @child = fork
@@ -128,7 +128,7 @@ module Resque
         else
           break if interval.to_i == 0
           log! "Sleeping for #{interval.to_i}"
-          $0 = "resque: Waiting for #{@queues.join(',')}"
+          $0 = @paused ? "resque: Paused" : "resque: Waiting for #{@queues.join(',')}"
           sleep interval.to_i
         end
       end
@@ -186,7 +186,12 @@ module Resque
       return if @cant_fork
 
       begin
-        Kernel.fork
+        # IronRuby doesn't support `Kernel.fork` yet
+        if Kernel.respond_to?(:fork)
+          Kernel.fork
+        else
+          raise NotImplementedError
+        end
       rescue NotImplementedError
         @cant_fork = true
         nil
@@ -215,13 +220,21 @@ module Resque
     #  INT: Shutdown immediately, stop processing jobs.
     # QUIT: Shutdown after the current job has finished processing.
     # USR1: Kill the forked child immediately, continue processing jobs.
+    # USR2: Don't process any new jobs
+    # CONT: Start processing jobs again after a USR2
     def register_signal_handlers
       trap('TERM') { shutdown!  }
       trap('INT')  { shutdown!  }
-      unless defined? JRUBY_VERSION
+
+      begin
         trap('QUIT') { shutdown   }
         trap('USR1') { kill_child }
+        trap('USR2') { pause_processing }
+        trap('CONT') { unpause_processing }
+      rescue ArgumentError
+        warn "Signals QUIT, USR1, USR2, and/or CONT not supported."
       end
+
       log! "Registered signals"
     end
 
@@ -252,21 +265,36 @@ module Resque
       end
     end
 
+    # Stop processing jobs after the current one has completed (if we're
+    # currently running one).
+    def pause_processing
+      log "USR2 received; pausing job processing"
+      @paused = true
+    end
+
+    # Start processing jobs again after a pause
+    def unpause_processing
+      log "CONT received; resuming job processing"
+      @paused = false
+    end
+
     # Looks for any workers which should be running on this server
     # and, if they're not, removes them from Redis.
     #
     # This is a form of garbage collection. If a server is killed by a
     # hard shutdown, power failure, or something else beyond our
-    # control, the Resque workers will not die gracefully and therefor
+    # control, the Resque workers will not die gracefully and therefore
     # will leave stale state information in Redis.
     #
     # By checking the current Redis state against the actual
     # environment, we can determine if Redis is old and clean it up a bit.
     def prune_dead_workers
-      Worker.all.each do |worker|
+      all_workers = Worker.all
+      known_workers = worker_pids unless all_workers.empty?
+      all_workers.each do |worker|
         host, pid, queues = worker.to_s.split(':')
         next unless host == hostname
-        next if worker_pids.include?(pid)
+        next if known_workers.include?(pid)
         log! "Pruning dead worker: #{worker}"
         worker.unregister_worker
       end
@@ -281,9 +309,8 @@ module Resque
 
     # Unregisters ourself as a worker. Useful when shutting down.
     def unregister_worker
-      done_working
-
       mongo_workers.remove(:worker => self.to_s)
+
       Stat.clear("processed:#{self}")
       Stat.clear("failed:#{self}")
     end

data/tasks/redis.rake

@@ -29,7 +29,8 @@ class RedisRunner
   def self.start
     puts 'Detach with Ctrl+\  Re-attach with rake redis:attach'
     sleep 1
-    exec "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
+    command = "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
+    sh command
   end
 
   def self.attach
@@ -80,8 +81,9 @@ namespace :redis do
     puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
 
     ENV['PREFIX'] and conf_dir = "#{ENV['PREFIX']}/etc" or conf_dir = '/etc'
-    unless File.exists?("#{conf_dir}")
-      sh "cp /tmp/redis/redis.conf #{conf_dir}"
+    unless File.exists?("#{conf_dir}/redis.conf")
+      sh "mkdir #{conf_dir}" unless File.exists?("#{conf_dir}")
+      sh "cp /tmp/redis/redis.conf #{conf_dir}/redis.conf"
       puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
     end
   end

data/test/worker_test.rb

@@ -114,6 +114,14 @@ context "Resque::Worker" do
     assert_equal [], Resque.workers
   end
 
+  test "removes worker with stringified id" do
+    @worker.work(0) do
+      worker_id = Resque.workers[0].to_s
+      Resque.remove_worker(worker_id)
+      assert_equal [], Resque.workers
+   end
+  end
+
   test "records what it is working on" do
     @worker.work(0) do
       task = @worker.job
@@ -226,4 +234,9 @@ context "Resque::Worker" do
       assert_equal 1, Resque.workers.size
     end
   end
+
+  test "Processed jobs count" do
+    @worker.work(0)
+    assert_equal 1, Resque.info[:processed]
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: resque-mongo
 version: !ruby/object:Gem::Version 
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors: 
 - Christos Trochalakis
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-28 00:00:00 +02:00
+date: 2010-02-12 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -91,6 +91,7 @@ files:
 - lib/resque/failure/base.rb
 - lib/resque/failure/hoptoad.rb
 - lib/resque/failure/mongo.rb
+- lib/resque/failure/multiple.rb
 - lib/resque/helpers.rb
 - lib/resque/job.rb
 - lib/resque/server.rb