mongo-resque 1.17.1 → 1.18.2

data/HISTORY.md

@@ -1,3 +1,21 @@
+## 1.18.2 (2011-08-19)
+
+* Fix RAILS_ROOT deprecation warning
+
+## 1.18.1 (2011-08-19)
+
+* Bugfix: Use RAILS_ROOT in preload task
+
+## 1.18.0 (2011-08-18)
+
+* Added before_enqueue hook.
+* Resque workers now preload files under app/ in Rails
+* Switch to MultiJSON
+* Bugfix: Finding worker pids on Solaris
+* Web UI: Fix NaN days ago for worker screens
+* Web UI: Add Cache-Control header to prevent proxy caching
+* Web UI: Update Resque.redis_id so it can be used in a distributed ring.
+
 ## 1.17.1 (2011-05-30)
 
 * More monit typo fixes

data/README.markdown

@@ -29,19 +29,30 @@ the change was for the better or not (we avoid dependency troubles this way).
 Delayed Jobs
 ------------
 
-If your job class indicates that @delayed_jobs = true, you can queue
+If your job class indicates that @delayed = true, you can queue
 delayed jobs.  These jobs will not be popped off the queue until the
 Time indicated in arg[0][:delay_until] has come.  Note that you must
 call Resque.enable_delay(:queue) before enququing any delayed jobs, to
 ensure that the performance impact on other queues is minimal.
 
+Bundling
+========
+
+Make sure you use the right require in your Gemfile.
+
+``` ruby
+gem 'mongo-resque', :require => 'resque'
+```
+
 Configuration
 =============
 
 Resque.redis= has been replaced with Resque.mongo= and expects a Mongo::DB
 object as an argument.
 
-    Resque.mongo = Mongo::Connection.new.db("my_awesome_queue")
+``` ruby
+Resque.mongo = Mongo::Connection.new.db("my_awesome_queue")
+```
 
 Stern Warnings
 ==============

data/docs/HOOKS.md

@@ -66,6 +66,9 @@ An unnamed hook (`before_perform`) will be executed first.
 
 The available hooks are:
 
+* `before_enqueue`: Called with the job args before a job is placed on the queue.
+  If the hook returns `false`, the job will not be placed on the queue.
+
 * `after_enqueue`: Called with the job args after a job is placed on the queue.
   Any exception raised propagates up to the code which queued the job.
 

data/lib/resque.rb

@@ -1,10 +1,3 @@
-
-begin
-  require 'yajl'
-rescue LoadError
-  require 'json'
-end
-
 require 'mongo'
 
 require 'resque/version'
@@ -23,7 +16,7 @@ require 'resque/plugin'
 module Resque
   include Helpers
   extend self
-  @delay_allowed = []
+  @delayed_queues = []
   
   # Set the queue database. Expects a Mongo::DB object.
   def mongo=(database)
@@ -46,8 +39,8 @@ module Resque
   def initialize_mongo
     mongo_workers.create_index :worker
     mongo_stats.create_index :stat
-    delay_allowed = mongo_stats.find_one({ :stat => 'Delayable Queues'}, { :fields => ['value']})
-    @delay_allowed = delay_allowed['value'].map{ |queue| queue.to_sym} if delay_allowed    
+    delayed_queues = mongo_stats.find_one(:stat => 'Delayed Queues')
+    @delayed_queues = delayed_queues['value'] if delayed_queues
   end
 
   def mongo_workers
@@ -114,21 +107,20 @@ module Resque
     "Resque Client connected to #{connection_info.host}:#{connection_info.port}/#{mongo.name}"
   end
 
-  def allows_delayed_jobs(klass)
-    klass.instance_variable_get(:@delayed_jobs) ||
-      (klass.respond_to?(:delayed_jobs) and klass.delayed_jobs)
+  def delayed_job?(klass)
+    klass.instance_variable_get(:@delayed) ||
+      (klass.respond_to?(:delayed) and klass.delayed)
   end
 
-  def queue_allows_delayed(queue)
-    queue = namespace_queue(queue)
-    @delay_allowed.include?(queue.to_sym) || @delay_allowed.include?(queue.to_s)
+  def delayed_queue?(queue)
+    @delayed_queues.include? namespace_queue(queue)
   end
 
   def enable_delay(queue)
     queue = namespace_queue(queue)
-    unless queue_allows_delayed queue
-      @delay_allowed << queue
-      mongo_stats.update({:stat => 'Delayable Queues'}, { '$addToSet' => { 'value' => queue}}, { :upsert => true})
+    unless delayed_queue? queue
+      @delayed_queues << queue
+      mongo_stats.update({:stat => 'Delayed Queues'}, {'$addToSet' => {'value' => queue}}, {:upsert => true})
     end
   end
   
@@ -175,11 +167,9 @@ module Resque
   def pop(queue)
     queue = namespace_queue(queue)
     query = {}
-    if queue_allows_delayed queue
-      query['delay_until'] = { '$lt' => Time.now }
-    end
+    query['delay_until'] = { '$lt' => Time.now } if delayed_queue?(queue)
     #sorting will result in significant performance penalties for large queues, you have been warned.
-    item = mongo[queue].find_and_modify(:query => query, :remove => true, :sort => [[:_id, :asc]] )
+    item = mongo[queue].find_and_modify(:query => query, :remove => true, :sort => [[:_id, :asc]])
   rescue Mongo::OperationFailure => e
     return nil if e.message =~ /No matching object/
     raise e
@@ -194,7 +184,7 @@ module Resque
 
   def delayed_size(queue)
     queue = namespace_queue(queue)
-    if queue_allows_delayed queue
+    if delayed_queue? queue
       mongo[queue].find({'delay_until' => { '$gt' => Time.now }}).count
     else
       mongo[queue].count
@@ -203,7 +193,7 @@ module Resque
 
   def ready_size(queue)
     queue = namespace_queue(queue)
-    if queue_allows_delayed queue
+    if delayed_queue? queue
       mongo[queue].find({'delay_until' => { '$lt' => Time.now }}).count
     else
       mongo[queue].count
@@ -228,7 +218,7 @@ module Resque
   def list_range(key, start = 0, count = 1, mode = :ready)
     query = { }
     sort = []
-    if queue_allows_delayed(key)
+    if delayed_queue? key
       if mode == :ready
         query['delay_until'] = { '$not' => { '$gt' => Time.new}}
       elsif mode == :delayed
@@ -283,12 +273,22 @@ module Resque
   #
   # This method is considered part of the `stable` API.
   def enqueue(klass, *args)
+    # Perform before_enqueue hooks. Don't perform enqueue if any hook returns false
+    before_hooks = Plugin.before_enqueue_hooks(klass).collect do |hook|
+      klass.send(hook, *args)
+    end
+    return if before_hooks.any? { |result| result == false }
+
     Job.create(queue_from_class(klass), klass, *args)
     
     Plugin.after_enqueue_hooks(klass).each do |hook|
       klass.send(hook, *args)
     end
   end
+  
+  def enqueue_delayed(klass, *args)
+    
+  end
 
   # This method can be used to conveniently remove a job from a queue.
   # It assumes the class you're passing it is a real Ruby class (not

data/lib/resque/delayed.rb

@@ -0,0 +1,68 @@
+module Resque
+  module DelayedJob
+    def before_enqueue_with_delay(*args)
+      unless Resque.delayed_queue?(self)
+        raise DelayedQueueError.new 'trying to insert a delayed job into a non-delayed queue' 
+      end
+      unless args[0].is_a?(Hash) && args[0].has_key?(:delay_until)
+        raise DelayedQueueError.new 'trying to insert delayed job without delay_until'
+      end
+    end
+
+    def before_create_with_delay(item, *args)
+      item[:delay_until] = args[0][:delay_until]
+    end
+
+    def before_pop_with_delay(query)
+      query['delay_until'] = {'$lt' => Time.now } if delayed_queue?(queue)
+    end
+  end
+  
+  module Delayed
+    def self.extended(base)
+      base.class_eval { @delayed_queues = [] }
+    end
+
+    def initialize_delayed
+      delayed_queues = mongo_stats.find_one(:stat => 'Delayed Queues')
+      @delayed_queues = delayed_queues['value'] if delayed_queues
+    end
+
+    def delayed_job?(klass)
+      klass.instance_variable_get(:@delayed) ||
+        (klass.respond_to?(:delayed) and klass.delayed)
+    end
+
+    def delayed_queue?(queue)
+      @delayed_queues.include? namespace_queue(queue)
+    end
+
+    def enable_delay(queue)
+      queue = namespace_queue(queue)
+      unless delayed_queue? queue
+        @delayed_queues << queue
+        mongo_stats.update({:stat => 'Delayed Queues'}, {'$addToSet' => {'value' => queue}}, {:upsert => true})
+      end
+    end
+
+    def delayed_size(queue)
+      queue = namespace_queue(queue)
+      if delayed_queue? queue
+        mongo[queue].find({'delay_until' => { '$gt' => Time.now }}).count
+      else
+        mongo[queue].count
+      end
+    end
+
+    def ready_size(queue)
+      queue = namespace_queue(queue)
+      if delayed_queue? queue
+        mongo[queue].find({'delay_until' => { '$lt' => Time.now }}).count
+      else
+        mongo[queue].count
+      end
+    end
+  end
+  
+  class DelayedQueueError < RuntimeError; end
+end

data/lib/resque/failure/redis.rb

@@ -5,11 +5,11 @@ module Resque
     class Redis < Base
       def save
         data = {
-          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
+          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
           :payload   => payload,
           :exception => exception.class.to_s,
           :error     => exception.to_s,
-          :backtrace => Array(exception.backtrace),
+          :backtrace => filter_backtrace(Array(exception.backtrace)),
           :worker    => worker.to_s,
           :queue     => queue
         }
@@ -41,6 +41,11 @@ module Resque
         Resque.redis.lset(:failed, index, id)
         Resque.redis.lrem(:failed, 1, id)
       end
+
+      def filter_backtrace(backtrace)
+        index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
+        backtrace.first(index.to_i)
+      end
     end
   end
 end

data/lib/resque/helpers.rb

@@ -1,3 +1,11 @@
+require 'multi_json'
+
+# OkJson won't work because it doesn't serialize symbols
+# in the same way yajl and json do.
+if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson'
+  raise "Please install the yajl-ruby or json gem"
+end
+
 module Resque
   # Methods used by various classes in Resque.
   module Helpers
@@ -19,29 +27,17 @@ module Resque
     # Given a Ruby object, returns a string suitable for storage in a
     # queue.
     def encode(object)
-      if defined? Yajl
-        Yajl::Encoder.encode(object)
-      else
-        object.to_json
-      end
+      ::MultiJson.encode(object)
     end
     
     # Given a string, returns a Ruby object.
     def decode(object)
       return unless object
 
-      if defined? Yajl
-        begin
-          Yajl::Parser.parse(object, :check_utf8 => false)
-        rescue Yajl::ParseError => e
-          raise DecodeException, e
-        end
-      else
-        begin
-          JSON.parse(object)
-        rescue JSON::ParserError => e
-          raise DecodeException, e
-        end
+      begin
+        ::MultiJson.decode(object)
+      rescue ::MultiJson::DecodeError => e
+        raise DecodeException, e
       end
     end
 

data/lib/resque/job.rb

@@ -45,8 +45,8 @@ module Resque
       item = { :class => klass.to_s, :args => args}
 
       #are we trying to put a non-delayed job into a delayed queue?
-      if Resque.queue_allows_delayed(queue)
-        if Resque.allows_delayed_jobs(klass)
+      if Resque.delayed_queue?(queue)
+        if Resque.delayed_job?(klass)
           if args[0].is_a?(Hash) && args[0].has_key?(:delay_until)
             item[:delay_until] = args[0][:delay_until]
           else
@@ -56,7 +56,7 @@ module Resque
           raise QueueError.new 'trying to insert non-delayed job into delayed queue'
         end
       else
-        if Resque.allows_delayed_jobs(klass)
+        if Resque.delayed_job?(klass)
           raise QueueError.new 'trying to insert a delayed job into a non-delayed queue'
         end
       end

data/lib/resque/plugin.rb

@@ -47,5 +47,10 @@ module Resque
     def after_enqueue_hooks(job)
       job.methods.grep(/^after_enqueue/).sort
     end
+
+    # Given an object, returns a list `before_enqueue` hook names.
+    def before_enqueue_hooks(job)
+      job.methods.grep(/^before_enqueue/).sort
+    end
   end
 end

data/lib/resque/server.rb

@@ -97,13 +97,14 @@ module Resque
     end
 
     def show(page, layout = true)
+      response["Cache-Control"] = "max-age=0, private, must-revalidate"
       begin
         erb page.to_sym, {:layout => layout}, :resque => Resque
-      rescue Errno::ECONNREFUSED
-        erb :error, {:layout => false}, :error => "Can't connect to Redis! (#{Resque.redis_id})"
+      rescue Mongo::ConnectionError, Mongo::ConnectionFailure
+        erb :error, {:layout => false}, :error => "Can't connect to MongoDB!"
       end
     end
-    
+
     def show_for_polling(page)
       content_type "text/html"
       @polling = true
@@ -174,12 +175,12 @@ module Resque
     get "/?" do
       redirect url_path(:overview)
     end
-    
+
     %w( overview workers ).each do |page|
       get "/#{page}.poll" do
         show_for_polling(page)
       end
-      
+
       get "/#{page}/:id.poll" do
         show_for_polling(page)
       end

data/lib/resque/server/views/queues.erb

@@ -5,7 +5,7 @@
   <form method="POST" action="<%=u "/queues/#{queue}/remove" %>" class='remove-queue'>
     <input type='submit' name='' value='Remove Queue' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
   </form>
-  <% if !resque.queue_allows_delayed queue.to_sym %>
+  <% unless resque.delayed_queue? queue %>
     <p class='sub'>Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%=size = resque.size(queue.to_sym)%></b> jobs</p><table class='jobs'>
       <tr>
         <th>Class</th>
@@ -57,7 +57,7 @@
     <tr>
       <td class='queue'><a class="queue" href="<%= u "queues/#{queue}" %>"><%= queue %></a></td>
       <td class='size'><%= resque.ready_size queue.to_sym %></td>
-      <td class='size'><%= resque.delayed_size(queue.to_sym) if resque.queue_allows_delayed(queue.to_sym) %></td>
+      <td class='size'><%= resque.delayed_size(queue.to_sym) if resque.delayed_queue?(queue) %></td>
     </tr>
     <% end %>
     <tr class="<%= Resque::Failure.count.zero? ? "failed" : "failure" %>">

data/lib/resque/tasks.rb

@@ -5,7 +5,7 @@ namespace :resque do
   task :setup
 
   desc "Start a Resque worker"
-  task :work => :setup do
+  task :work => [ :preload, :setup ] do
     require 'resque'
 
     queues = (ENV['QUEUES'] || ENV['QUEUE']).to_s.split(',')
@@ -39,4 +39,13 @@ namespace :resque do
 
     threads.each { |thread| thread.join }
   end
+
+  # Preload app files if this is Rails
+  task :preload do
+    if defined? Rails
+      Dir["#{Rails.root}/app/**/*.rb"].each do |file|
+        require file
+      end
+    end
+  end
 end

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = VERSION = '1.17.1'
+  Version = VERSION = '1.18.2'
 end

data/lib/resque/worker.rb

@@ -292,7 +292,7 @@ module Resque
     def paused?
       @paused
     end
-    
+
     # Stop processing jobs after the current one has completed (if we're
     # currently running one).
     def pause_processing
@@ -367,10 +367,9 @@ module Resque
     # what workers are doing and when.
     def working_on(job)
       job.worker = self
-      data = #encode \
-        { :queue   => job.queue,
-        :run_at  => Time.now.to_s,
-        :payload => job.payload}
+      data = { :queue   => job.queue,
+               :run_at  => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+               :payload => job.payload }
       mongo_workers.update({:worker => self.to_s}, { '$set' => { 'working_on' => data}}, :upsert => true)
     end
 
@@ -463,14 +462,40 @@ module Resque
       @pid ||= to_s.split(":")[1].to_i
     end
 
-    # Returns an array of string pids of all the other workers on this
+    # Returns an Array of string pids of all the other workers on this
     # machine. Useful when pruning dead workers on startup.
     def worker_pids
+      if RUBY_PLATFORM =~ /solaris/
+        solaris_worker_pids
+      else
+        linux_worker_pids
+      end
+    end
+
+    # Find Resque worker pids on Linux and OS X.
+    #
+    # Returns an Array of string pids of all the other workers on this
+    # machine. Useful when pruning dead workers on startup.
+    def linux_worker_pids
       `ps -A -o pid,command | grep [r]esque | grep -v "resque-web"`.split("\n").map do |line|
         line.split(' ')[0]
       end
     end
 
+    # Find Resque worker pids on Solaris.
+    #
+    # Returns an Array of string pids of all the other workers on this
+    # machine. Useful when pruning dead workers on startup.
+    def solaris_worker_pids
+      `ps -A -o pid,comm | grep [r]uby | grep -v "resque-web"`.split("\n").map do |line|
+        real_pid = line.split(' ')[0]
+        pargs_command = `pargs -a #{real_pid} 2>/dev/null | grep [r]esque | grep -v "resque-web"`
+        if pargs_command.split(':')[1] == " resque-#{Resque::Version}"
+          real_pid
+        end
+      end.compact
+    end
+
     # Given a string, sets the procline ($0) and logs.
     # Procline is always in the format of:
     #   resque-VERSION: STRING

metadata

@@ -1,76 +1,75 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mongo-resque
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 1.18.2
   prerelease: 
-  version: 1.17.1
 platform: ruby
-authors: 
+authors:
 - David Backeus
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-07-19 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: vegas
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+date: 2011-08-22 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: &2157276880 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: 0.1.2
+      - !ruby/object:Gem::Version
+        version: 1.3.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: sinatra
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2157276880
+- !ruby/object:Gem::Dependency
+  name: vegas
+  requirement: &2157276380 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.9.2
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: json
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *2157276380
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: &2157275680 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.4.6
-    - - <
-      - !ruby/object:Gem::Version 
-        version: 1.6.0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: mongo
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *2157275680
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &2157275100 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: 1.3.0
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :runtime
-  version_requirements: *id004
-description: "    Resque is a Redis-backed Ruby library for creating background jobs,\n    placing those jobs on multiple queues, and processing them later.\n\n    Mongo-Resque is the same thing, but for mongo. It would not exist\n    without the work of defunkt and ctrochalakis on github.\n"
-email: 
+  prerelease: false
+  version_requirements: *2157275100
+description: ! "    Resque is a Redis-backed Ruby library for creating background
+  jobs,\n    placing those jobs on multiple queues, and processing them later.\n\n
+  \   Mongo-Resque is the same thing, but for mongo. It would not exist\n    without
+  the work of defunkt and ctrochalakis on github.\n"
+email:
 - david@streamio.se
-executables: 
+executables:
 - resque
 - resque-web
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE
 - README.markdown
-files: 
+files:
+- lib/resque/delayed.rb
 - lib/resque/errors.rb
 - lib/resque/failure/base.rb
 - lib/resque/failure/hoptoad.rb
@@ -116,33 +115,28 @@ files:
 - README.markdown
 - LICENSE
 - HISTORY.md
-homepage: http://github.com/dbackeus/mongo-resque
+homepage: https://github.com/streamio/mongo-resque
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.7.2
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: Mongo-Resque is a mongo-backed queueing system
 test_files: []
-
-has_rdoc: