resque 1.26.0 → 1.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f654ccc902abcf33c2340b1e7a718224404880aa
-  data.tar.gz: bb83a2fabd7698745fcde434567c85f4852b48a4
+  metadata.gz: b5f0cb00378f55254a8f68c68fed023ea569e1a8
+  data.tar.gz: bf7219b392370ef84f16ab57ff3646c64544a17d
 SHA512:
-  metadata.gz: b4c51510f6ce42a7c4b2b31bda3c5758d91e1503fbdc7fac70ae976ef02aa3d519258e38da5a74def54db4487eb673df309b16b3b09b9bc9870beb844f474dd5
-  data.tar.gz: bda8063a0120e1a9e12c483b0a8c0aac1c472104f129f36d1867d2b75663ab2d712c6c965d1735edc1b3f436cc4272f9d1f9037cc3af1e5608c9ada00592949e
+  metadata.gz: fd8e4634086ea6c5f5147799d35596924abb49a71fa1c99223017c4ade6b24b5afba32f2e70985b18af6493530fd554ecd76947a7f7ff04f32e5aeb43c707477
+  data.tar.gz: 484db5ed7b0b3be036c9523a2720a16edaa43748a65d34a2add2932784fc37440e0922596529b229d471177c03599df65ab7d57b980b99850c39ab98dfe26e4f

data/HISTORY.md

@@ -1,6 +1,14 @@
-## 1.27.0 (TBD)
+## 1.28.0 (TBD)
 
-None yet!
+
+## 1.27.0 (2017-02-08)
+
+* Update jQuery from 1.3.2 to 1.12.4
+
+* Fix issue where calling Worker.find, Worker.all, or Worker.working from withing
+  a running job would rewrite the PID file with the PID of the forked worker.
+  This causes a change to the Worker#new API that may affect plugin
+  implementations. See Worker#new and Worker#prepare for details. (@jeremywadsack)
 
 ## 1.26.0 (2016-03-10)
 

data/README.markdown

@@ -524,8 +524,8 @@ or set the Redis connection string if you need to do something like select a dif
 Using Passenger? Resque ships with a `config.ru` you can use. See
 Phusion's guide:
 
-Apache: <http://www.modrails.com/documentation/Users%20guide%20Apache.html#_deploying_a_rack_based_ruby_application>
-Nginx: <http://www.modrails.com/documentation/Users%20guide%20Nginx.html#deploying_a_rack_app>
+Apache: <https://www.phusionpassenger.com/library/deploy/apache/deploy/ruby/>
+Nginx: <https://www.phusionpassenger.com/library/deploy/nginx/deploy/ruby/>
 
 ### Rack::URLMap
 
@@ -687,7 +687,7 @@ Don't forget you can define a `resque:setup` hook in
 `lib/tasks/whatever.rake` that loads the `environment` task every time.
 
 
-### In a Rails 3 app, as a gem
+### In a Rails 3.x or 4.x app, as a gem
 
 First include it in your Gemfile.
 

data/lib/resque.rb

@@ -18,6 +18,8 @@ require 'resque/log_formatters/very_verbose_formatter'
 require 'resque/job'
 require 'resque/worker'
 require 'resque/plugin'
+require 'resque/data_store'
+require 'resque/thread_signal'
 
 require 'resque/vendor/utf8_util'
 
@@ -120,33 +122,29 @@ module Resque
       end
       namespace ||= :resque
 
-      @redis = Redis::Namespace.new(namespace, :redis => redis)
+      @data_store = Resque::DataStore.new(Redis::Namespace.new(namespace, :redis => redis))
     when Redis::Namespace
-      @redis = server
+      @data_store = Resque::DataStore.new(server)
+    when Resque::DataStore
+      @data_store = server
     when Hash
-      @redis = Redis::Namespace.new(:resque, :redis => Redis.new(server))
+      @data_store = Resque::DataStore.new(Redis::Namespace.new(:resque, :redis => Redis.new(server)))
     else
-      @redis = Redis::Namespace.new(:resque, :redis => server)
+      @data_store = Resque::DataStore.new(Redis::Namespace.new(:resque, :redis => server))
     end
   end
 
   # Returns the current Redis connection. If none has been created, will
   # create a new one.
   def redis
-    return @redis if @redis
+    return @data_store if @data_store
     self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
     self.redis
   end
+  alias :data_store :redis
 
   def redis_id
-    # support 1.x versions of redis-rb
-    if redis.respond_to?(:server)
-      redis.server
-    elsif redis.respond_to?(:nodes) # distributed
-      redis.nodes.map { |n| n.id }.join(', ')
-    else
-      redis.client.id
-    end
+    data_store.identifier
   end
 
   # Set or retrieve the current logger object
@@ -165,6 +163,12 @@ module Resque
     @prune_interval || DEFAULT_PRUNE_INTERVAL
   end
 
+  attr_writer :enqueue_front
+  def enqueue_front
+    return @enqueue_front unless @enqueue_front.nil?
+    @enqueue_front = false
+  end
+
   # The `before_first_fork` hook will be run in the **parent** process
   # only once, before forking to run the first job. Be careful- any
   # changes you make will be permanent for the lifespan of the
@@ -264,24 +268,20 @@ module Resque
   #
   # Returns nothing
   def push(queue, item)
-    encoded = encode(item)
-    redis.pipelined do
-      watch_queue(queue)
-      redis.rpush "queue:#{queue}", encoded
-    end
+    data_store.push_to_queue(queue,encode(item))
   end
 
   # Pops a job off a queue. Queue name should be a string.
   #
   # Returns a Ruby object.
   def pop(queue)
-    decode redis.lpop("queue:#{queue}")
+    decode(data_store.pop_from_queue(queue))
   end
 
   # Returns an integer representing the size of a queue.
   # Queue name should be a string.
   def size(queue)
-    redis.llen("queue:#{queue}").to_i
+    data_store.queue_size(queue)
   end
 
   # Returns an array of items currently queued. Queue name should be
@@ -293,38 +293,39 @@ module Resque
   # To get the 3rd page of a 30 item, paginatied list one would use:
   #   Resque.peek('my_list', 59, 30)
   def peek(queue, start = 0, count = 1)
-    list_range("queue:#{queue}", start, count)
+    results = data_store.peek_in_queue(queue,start,count)
+    if count == 1
+      decode(results)
+    else
+      results.map { |result| decode(result) }
+    end
   end
 
   # Does the dirty work of fetching a range of items from a Redis list
   # and converting them into Ruby objects.
   def list_range(key, start = 0, count = 1)
+    results = data_store.list_range(key, start, count)
     if count == 1
-      decode redis.lindex(key, start)
+      decode(results)
     else
-      Array(redis.lrange(key, start, start+count-1)).map do |item|
-        decode item
-      end
+      results.map { |result| decode(result) }
     end
   end
 
   # Returns an array of all known Resque queues as strings.
   def queues
-    Array(redis.smembers(:queues))
+    data_store.queue_names
   end
 
   # Given a queue name, completely deletes the queue.
   def remove_queue(queue)
-    redis.pipelined do
-      redis.srem(:queues, queue.to_s)
-      redis.del("queue:#{queue}")
-    end
+    data_store.remove_queue(queue)
   end
 
   # Used internally to keep track of which queues we've created.
   # Don't call this directly.
   def watch_queue(queue)
-    redis.sadd(:queues, queue.to_s)
+    data_store.watch_queue(queue)
   end
 
 
@@ -487,7 +488,7 @@ module Resque
       :queues    => queues.size,
       :workers   => workers.size.to_i,
       :working   => working.size,
-      :failed    => Resque.redis.llen(:failed).to_i,
+      :failed    => data_store.num_failed,
       :servers   => [redis_id],
       :environment  => ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
     }
@@ -496,9 +497,7 @@ module Resque
   # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
   # is O(N) for the keyspace, so be careful - this can be slow for big databases.
   def keys
-    redis.keys("*").map do |key|
-      key.sub("#{redis.namespace}:", '')
-    end
+    data_store.all_resque_keys
   end
 
   # Returns a hash, mapping queue names to queue sizes

data/lib/resque/data_store.rb

@@ -0,0 +1,326 @@
+module Resque
+  # An interface between Resque's persistence and the actual
+  # implementation.
+  class DataStore
+    extend Forwardable
+
+    HEARTBEAT_KEY = "workers:heartbeat"
+
+    def initialize(redis)
+      @redis               = redis
+      @queue_access        = QueueAccess.new(@redis)
+      @failed_queue_access = FailedQueueAccess.new(@redis)
+      @workers             = Workers.new(@redis)
+      @stats_access        = StatsAccess.new(@redis)
+    end
+
+    def_delegators :@queue_access, :push_to_queue,
+                                   :pop_from_queue,
+                                   :queue_size,
+                                   :peek_in_queue,
+                                   :queue_names,
+                                   :remove_queue,
+                                   :everything_in_queue,
+                                   :remove_from_queue,
+                                   :watch_queue,
+                                   :list_range
+
+    def_delegators :@failed_queue_access, :add_failed_queue,
+                                          :remove_failed_queue,
+                                          :num_failed,
+                                          :failed_queue_names,
+                                          :push_to_failed_queue,
+                                          :clear_failed_queue,
+                                          :update_item_in_failed_queue,
+                                          :remove_from_failed_queue
+    def_delegators :@workers, :worker_ids,
+                              :workers_map,
+                              :get_worker_payload,
+                              :worker_exists?,
+                              :register_worker,
+                              :worker_started,
+                              :unregister_worker,
+                              :heartbeat,
+                              :heartbeat!,
+                              :remove_heartbeat,
+                              :all_heartbeats,
+                              :set_worker_payload,
+                              :worker_start_time,
+                              :worker_done_working
+
+      def_delegators :@stats_access, :clear_stat,
+                                     :decremet_stat,
+                                     :increment_stat,
+                                     :stat
+
+    # Compatibility with any non-Resque classes that were using Resque.redis as a way to access Redis
+    def method_missing(sym,*args,&block)
+      # TODO: deprecation warning?
+      @redis.send(sym,*args,&block)
+    end
+
+    # make use respond like redis
+    def respond_to?(method,include_all=false)
+      @redis.respond_to?(method,include_all)
+    end
+
+    # Get a string identifying the underlying server.
+    # Probably should be private, but was public so must stay public
+    def identifier
+      # support 1.x versions of redis-rb
+      if @redis.respond_to?(:server)
+        @redis.server
+      elsif @redis.respond_to?(:nodes) # distributed
+        @redis.nodes.map { |n| n.id }.join(', ')
+      else
+        @redis.client.id
+      end
+    end
+
+    # Force a reconnect to Redis.
+    def reconnect
+      @redis.client.reconnect
+    end
+
+    # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
+    # is O(N) for the keyspace, so be careful - this can be slow for big databases.
+    def all_resque_keys
+      @redis.keys("*").map do |key|
+        key.sub("#{@redis.namespace}:", '')
+      end
+    end
+
+    def server_time
+      time, _ = @redis.time
+      Time.at(time)
+    end
+
+    class QueueAccess
+      def initialize(redis)
+        @redis = redis
+      end
+      def push_to_queue(queue,encoded_item)
+        @redis.pipelined do
+          watch_queue(queue)
+          @redis.rpush redis_key_for_queue(queue), encoded_item
+        end
+      end
+
+      # Pop whatever is on queue
+      def pop_from_queue(queue)
+        @redis.lpop(redis_key_for_queue(queue))
+      end
+
+      # Get the number of items in the queue
+      def queue_size(queue)
+        @redis.llen(redis_key_for_queue(queue)).to_i
+      end
+
+      # Examine items in the queue.
+      #
+      # NOTE: if count is 1, you will get back an object, otherwise you will
+      #       get an Array.  I'm not making this up.
+      def peek_in_queue(queue, start = 0, count = 1)
+        list_range(redis_key_for_queue(queue), start, count)
+      end
+
+      def queue_names
+        Array(@redis.smembers(:queues))
+      end
+
+      def remove_queue(queue)
+        @redis.pipelined do
+          @redis.srem(:queues, queue.to_s)
+          @redis.del(redis_key_for_queue(queue))
+        end
+      end
+
+      def everything_in_queue(queue)
+        @redis.lrange(redis_key_for_queue(queue), 0, -1)
+      end
+
+      # Remove data from the queue, if it's there, returning the number of removed elements
+      def remove_from_queue(queue,data)
+        @redis.lrem(redis_key_for_queue(queue), 0, data)
+      end
+
+      # Private: do not call
+      def watch_queue(queue)
+        @redis.sadd(:queues, queue.to_s)
+      end
+
+      # Private: do not call
+      def list_range(key, start = 0, count = 1)
+        if count == 1
+          @redis.lindex(key, start)
+        else
+          Array(@redis.lrange(key, start, start+count-1))
+        end
+      end
+
+    private
+
+      def redis_key_for_queue(queue)
+        "queue:#{queue}"
+      end
+
+    end
+
+    class FailedQueueAccess
+      def initialize(redis)
+        @redis = redis
+      end
+
+      def add_failed_queue(failed_queue_name)
+        @redis.sadd(:failed_queues, failed_queue_name)
+      end
+
+      def remove_failed_queue(failed_queue_name=:failed)
+        @redis.del(failed_queue_name)
+      end
+
+      def num_failed(failed_queue_name=:failed)
+        @redis.llen(failed_queue_name).to_i
+      end
+
+      def failed_queue_names(find_queue_names_in_key=nil)
+        if find_queue_names_in_key.nil?
+          [:failed]
+        else
+          Array(@redis.smembers(find_queue_names_in_key))
+        end
+      end
+
+      def push_to_failed_queue(data,failed_queue_name=:failed)
+        @redis.rpush(failed_queue_name,data)
+      end
+
+      def clear_failed_queue(failed_queue_name=:failed)
+        @redis.del(failed_queue_name)
+      end
+
+      def update_item_in_failed_queue(index_in_failed_queue,new_item_data,failed_queue_name=:failed)
+        @redis.lset(failed_queue_name, index_in_failed_queue, new_item_data)
+      end
+
+      def remove_from_failed_queue(index_in_failed_queue,failed_queue_name=nil)
+        failed_queue_name ||= :failed
+        hopefully_unique_value_we_can_use_to_delete_job = ""
+        @redis.lset(failed_queue_name, index_in_failed_queue, hopefully_unique_value_we_can_use_to_delete_job)
+        @redis.lrem(failed_queue_name, 1,                     hopefully_unique_value_we_can_use_to_delete_job)
+      end
+    end
+
+    class Workers
+      def initialize(redis)
+        @redis = redis
+      end
+
+      def worker_ids
+        Array(@redis.smembers(:workers))
+      end
+
+      # Given a list of worker ids, returns a map of those ids to the worker's value
+      # in redis, even if that value maps to nil
+      def workers_map(worker_ids)
+        redis_keys = worker_ids.map { |id| "worker:#{id}" }
+        @redis.mapped_mget(*redis_keys)
+      end
+
+      # return the worker's payload i.e. job
+      def get_worker_payload(worker_id)
+        @redis.get("worker:#{worker_id}")
+      end
+
+      def worker_exists?(worker_id)
+        @redis.sismember(:workers, worker_id)
+      end
+
+      def register_worker(worker)
+        @redis.pipelined do
+          @redis.sadd(:workers, worker)
+          worker_started(worker)
+        end
+      end
+
+      def worker_started(worker)
+        @redis.set(redis_key_for_worker_start_time(worker), Time.now.to_s)
+      end
+
+      def unregister_worker(worker, &block)
+        @redis.pipelined do
+          @redis.srem(:workers, worker)
+          @redis.del(redis_key_for_worker(worker))
+          @redis.del(redis_key_for_worker_start_time(worker))
+          @redis.hdel(HEARTBEAT_KEY, worker.to_s)
+
+          block.call
+        end
+      end
+
+      def remove_heartbeat(worker)
+        @redis.hdel(HEARTBEAT_KEY, worker.to_s)
+      end
+
+      def heartbeat(worker)
+        heartbeat = @redis.hget(HEARTBEAT_KEY, worker.to_s)
+        heartbeat && Time.parse(heartbeat)
+      end
+
+      def heartbeat!(worker, time)
+        @redis.hset(HEARTBEAT_KEY, worker.to_s, time.iso8601)
+      end
+
+      def all_heartbeats
+        @redis.hgetall(HEARTBEAT_KEY)
+      end
+
+      def set_worker_payload(worker, data)
+        @redis.set(redis_key_for_worker(worker), data)
+      end
+
+      def worker_start_time(worker)
+        @redis.get(redis_key_for_worker_start_time(worker))
+      end
+
+      def worker_done_working(worker, &block)
+        @redis.pipelined do
+          @redis.del(redis_key_for_worker(worker))
+          block.call
+        end
+      end
+
+    private
+
+      def redis_key_for_worker(worker)
+        "worker:#{worker}"
+      end
+
+      def redis_key_for_worker_start_time(worker)
+        "#{redis_key_for_worker(worker)}:started"
+      end
+
+    end
+
+    class StatsAccess
+      def initialize(redis)
+        @redis = redis
+      end
+      def stat(stat)
+        @redis.get("stat:#{stat}").to_i
+      end
+
+      def increment_stat(stat, by = 1)
+        @redis.incrby("stat:#{stat}", by)
+      end
+
+      def decremet_stat(stat, by = 1)
+        @redis.decrby("stat:#{stat}", by)
+      end
+
+      def clear_stat(stat)
+        @redis.del("stat:#{stat}")
+      end
+    end
+  end
+end