resqued 0.8.4 → 0.10.1

data/lib/resqued/master_state.rb

@@ -0,0 +1,73 @@
+module Resqued
+  class MasterState
+    def initialize
+      @listeners_created = 0
+      @listener_states = {}
+    end
+
+    # Public: When starting fresh, from command-line options, assign the initial values.
+    def init(options)
+      @config_paths = options.fetch(:config_paths)
+      @exec_on_hup  = options.fetch(:exec_on_hup, false)
+      @fast_exit    = options.fetch(:fast_exit, false)
+      @pidfile      = options.fetch(:master_pidfile, nil)
+    end
+
+    # Public: Restore state from a serialized form.
+    def restore(data)
+      @config_paths = data[:config_paths]
+      @current_listener_pid = data[:current_listener_pid]
+      @exec_on_hup = data[:exec_on_hup]
+      @fast_exit = data[:fast_exit]
+      @last_good_listener_pid = data[:last_good_listener_pid]
+      @listeners_created = data[:listeners_created]
+      data[:listener_states].each do |lsh|
+        @listener_states[lsh[:pid]] = ListenerState.new.tap do |ls|
+          ls.master_socket = lsh[:master_socket] && Socket.for_fd(lsh[:master_socket])
+          ls.options = lsh[:options]
+          ls.pid = lsh[:pid]
+          ls.worker_pids = lsh[:worker_pids]
+        end
+      end
+      @paused = data[:paused]
+      @pidfile = data[:pidfile]
+    end
+
+    # Public: Return this state so that it can be serialized.
+    def to_h
+      {
+        config_paths: @config_paths,
+        current_listener_pid: @current_listener_pid,
+        exec_on_hup: @exec_on_hup,
+        fast_exit: @fast_exit,
+        last_good_listener_pid: @last_good_listener_pid,
+        listeners_created: @listeners_created,
+        listener_states: @listener_states.values.map { |ls|
+          {
+            master_socket: ls.master_socket&.to_i,
+            options: ls.options,
+            pid: ls.pid,
+            worker_pids: ls.worker_pids,
+          }
+        },
+        paused: @paused,
+        pidfile: @pidfile,
+      }
+    end
+
+    # Public: Return an array of open sockets or other file handles that should be forwarded to a new master.
+    def sockets
+      @listener_states.values.map { |l| l.master_socket }.compact
+    end
+
+    attr_reader :config_paths
+    attr_accessor :current_listener_pid
+    attr_reader :exec_on_hup
+    attr_reader :fast_exit
+    attr_accessor :last_good_listener_pid
+    attr_accessor :listeners_created
+    attr_reader :listener_states
+    attr_accessor :paused
+    attr_reader :pidfile
+  end
+end

data/lib/resqued/procline_version.rb

@@ -1,4 +1,4 @@
-require 'resqued/version'
+require "resqued/version"
 
 module Resqued
   module ProclineVersion
@@ -6,7 +6,7 @@ module Resqued
       @version ||=
         begin
           # If we've built a custom version, this should show the custom version.
-          Gem.loaded_specs['resqued'].version.to_s
+          Gem.loaded_specs["resqued"].version.to_s
         rescue Object
           # If this isn't a gem, fall back to the version in resqued/version.rb.
           Resqued::VERSION

data/lib/resqued/sleepy.rb

@@ -1,9 +1,11 @@
-require 'fcntl'
-require 'kgio'
+require "fcntl"
+require "kgio"
 
 module Resqued
   module Sleepy
-    # Public: Like sleep, but the sleep is interrupted if input is detected on one of the provided IO objects, or if `awake` is called (e.g. from a signal handler).
+    # Public: Like sleep, but the sleep is interrupted if input is
+    # detected on one of the provided IO objects, or if `awake` is
+    # called (e.g. from a signal handler).
     def yawn(duration, *inputs)
       if duration > 0
         inputs = [self_pipe[0]] + [inputs].flatten.compact
@@ -14,7 +16,7 @@ module Resqued
 
     # Public: Break out of `yawn`.
     def awake
-      self_pipe[1].kgio_trywrite('.')
+      self_pipe[1].kgio_trywrite(".")
     end
 
     # Private.

data/lib/resqued/test_case.rb

@@ -1,5 +1,5 @@
-require 'resqued/config'
-require 'resqued/runtime_info'
+require "resqued/config"
+require "resqued/runtime_info"
 
 module Resqued
   module TestCase
@@ -17,7 +17,7 @@ module Resqued
         config = Resqued::Config.new(paths)
         config.before_fork(RuntimeInfo.new)
         config.build_workers
-        config.after_fork(Resque::Worker.new('*'))
+        config.after_fork(Resque::Worker.new("*"))
       end
     end
 

data/lib/resqued/version.rb

@@ -1,3 +1,3 @@
 module Resqued
-  VERSION = '0.8.4'
+  VERSION = '0.10.1'
 end

data/lib/resqued/worker.rb

@@ -1,18 +1,28 @@
-require 'resque'
+require "resque"
+require "digest"
 
-require 'resqued/backoff'
-require 'resqued/logging'
+require "resqued/backoff"
+require "resqued/logging"
 
 module Resqued
   # Models a worker process.
   class Worker
     include Resqued::Logging
 
+    DEFAULT_WORKER_FACTORY = lambda { |queues|
+      resque_worker = Resque::Worker.new(*queues)
+      resque_worker.term_child = true if resque_worker.respond_to?("term_child=")
+      redis_client = Resque.redis.respond_to?(:_client) ? Resque.redis._client : Resque.redis.client
+      redis_client.reconnect
+      resque_worker
+    }
+
     def initialize(options)
       @queues = options.fetch(:queues)
       @config = options.fetch(:config)
       @interval = options[:interval]
       @backoff = Backoff.new
+      @worker_factory = options.fetch(:worker_factory, DEFAULT_WORKER_FACTORY)
       @pids = []
     end
 
@@ -34,12 +44,13 @@ module Resqued
 
     # Public: A string that compares if this worker is equivalent to a worker in another Resqued::Listener.
     def queue_key
-      queues.sort.join(';')
+      Digest::SHA256.hexdigest(queues.sort.join(";"))
     end
 
     # Public: Claim this worker for another listener's worker.
     def wait_for(pid)
       raise "Already running #{@pid} (can't wait for #{pid})" if @pid
+
       @self_started = false
       @pids << pid
       @pid = pid
@@ -47,16 +58,17 @@ module Resqued
 
     # Public: The old worker process finished!
     def finished!(process_status)
-      if process_status.nil? && ! @self_started
-        log :debug, "(#{@pid}/#{@pids.inspect}/self_started=#{@self_started}/killed=#{@killed}) I am no longer blocked."
+      summary = "(#{@pid}/#{@pids.inspect}/self_started=#{@self_started}/killed=#{@killed})"
+      if process_status.nil? && !@self_started
+        log :debug, "#{summary} I am no longer blocked."
         @pid = nil
         @backoff.died unless @killed
-      elsif ! process_status.nil? && @self_started
-        log :debug, "(#{@pid}/#{@pids.inspect}/self_started=#{@self_started}/killed=#{@killed}) I exited: #{process_status}"
+      elsif !process_status.nil? && @self_started
+        log :debug, "#{summary} I exited: #{process_status}"
         @pid = nil
         @backoff.died unless @killed
       else
-        log :debug, "(#{@pid}/#{@pids.inspect}/self_started=#{@self_started}/killed=#{@killed}) Reports of my death are highly exaggerated (#{process_status.inspect})"
+        log :debug, "#{summary} Reports of my death are highly exaggerated (#{process_status.inspect})"
       end
     end
 
@@ -68,6 +80,7 @@ module Resqued
     # Public: Start a job, if there's one waiting in one of my queues.
     def try_start
       return if @backoff.wait?
+
       @backoff.started
       @self_started = true
       @killed = false
@@ -77,12 +90,10 @@ module Resqued
         log "Forked worker #{@pid}"
       else
         # In case we get a signal before resque is ready for it.
-        Resqued::Listener::ALL_SIGNALS.each { |signal| trap(signal, 'DEFAULT') }
+        Resqued::Listener::ALL_SIGNALS.each { |signal| trap(signal, "DEFAULT") }
         trap(:QUIT) { exit! 0 } # If we get a QUIT during boot, just spin back down.
         $0 = "STARTING RESQUE FOR #{queues.join(',')}"
-        resque_worker = Resque::Worker.new(*queues)
-        resque_worker.term_child = true if resque_worker.respond_to?('term_child=')
-        Resque.redis.client.reconnect
+        resque_worker = @worker_factory.call(queues)
         @config.after_fork(resque_worker)
         resque_worker.work(@interval || 5)
         exit 0

data/spec/fixtures/test_case_after_fork_raises.rb

@@ -1,3 +1,6 @@
-after_fork { raise 'boom' }
+after_fork do
+  raise "boom"
+end
+
 worker_pool 100
-queue 'test'
+queue "test"

data/spec/fixtures/test_case_before_fork_raises.rb

@@ -1,2 +1,5 @@
-before_fork { raise 'boom' }
+before_fork do
+  raise "boom"
+end
+
 worker "test"

data/spec/fixtures/test_case_environment.rb

@@ -1 +1,3 @@
-before_fork { Resque.redis = Redis.new(:host => 'localhost', :port => ENV['RESQUED_TEST_REDIS_PORT'].to_i) }
+before_fork do
+  Resque.redis = Redis.new(host: "localhost", port: ENV["RESQUED_TEST_REDIS_PORT"].to_i)
+end

data/spec/resqued/backoff_spec.rb

@@ -1,61 +1,61 @@
-require 'spec_helper'
+require "spec_helper"
 
-require 'resqued/backoff'
+require "resqued/backoff"
 
 describe Resqued::Backoff do
-  let(:backoff) { described_class.new(:min => 0.5, :max => 64.0) }
+  let(:backoff) { described_class.new(min: 0.5, max: 64.0) }
 
-  it 'can start on the first try' do
-    expect(backoff.wait?).to be_false
+  it "can start on the first try" do
+    expect(backoff.wait?).to be_falsey
   end
 
-  it 'has no waiting at first' do
+  it "has no waiting at first" do
     expect(backoff.how_long?).to be_nil
   end
 
-  context 'after expected exits' do
+  context "after expected exits" do
     before { 3.times { backoff.started } }
-    it { expect(backoff.wait?).to be_true }
+    it { expect(backoff.wait?).to be true }
     it { expect(backoff.how_long?).to be_close_to(0.5) }
   end
 
-  context 'after one quick exit' do
-    before { 1.times { backoff.started ; backoff.died } }
-    it { expect(backoff.wait?).to be_true }
+  context "after one quick exit" do
+    before { 1.times { backoff.started; backoff.died } }
+    it { expect(backoff.wait?).to be true }
     it { expect(backoff.how_long?).to be_close_to(1.0) }
   end
 
-  context 'after two quick starts' do
-    before { 2.times { backoff.started ; backoff.died } }
-    it { expect(backoff.wait?).to be_true }
+  context "after two quick starts" do
+    before { 2.times { backoff.started; backoff.died } }
+    it { expect(backoff.wait?).to be true }
     it { expect(backoff.how_long?).to be_close_to(2.0) }
   end
 
-  context 'after five quick starts' do
-    before { 6.times { backoff.started ; backoff.died } }
-    it { expect(backoff.wait?).to be_true }
+  context "after five quick starts" do
+    before { 6.times { backoff.started; backoff.died } }
+    it { expect(backoff.wait?).to be true }
     it { expect(backoff.how_long?).to be_close_to(32.0) }
   end
 
-  context 'after six quick starts' do
-    before { 7.times { backoff.started ; backoff.died } }
-    it { expect(backoff.wait?).to be_true }
+  context "after six quick starts" do
+    before { 7.times { backoff.started; backoff.died } }
+    it { expect(backoff.wait?).to be true }
     it { expect(backoff.how_long?).to be_close_to(64.0) }
   end
 
-  context 'does not wait longer than 64s' do
-    before { 8.times { backoff.started ; backoff.died } }
-    it { expect(backoff.wait?).to be_true }
+  context "does not wait longer than 64s" do
+    before { 8.times { backoff.started; backoff.died } }
+    it { expect(backoff.wait?).to be true }
     it { expect(backoff.how_long?).to be_close_to(64.0) }
-    it 'and resets after an expected exit' do
+    it "and resets after an expected exit" do
       backoff.started
       backoff.started
-      expect(backoff.wait?).to be_true
+      expect(backoff.wait?).to be true
       expect(backoff.how_long?).to be_close_to(0.5)
     end
   end
 
-  def be_close_to(x)
-    be_within(0.005).of(x)
+  def be_close_to(number)
+    be_within(0.005).of(number)
   end
 end

data/spec/resqued/config/fork_event_spec.rb

@@ -1,12 +1,12 @@
-require 'spec_helper'
-require 'resqued/config/after_fork'
-require 'resqued/config/before_fork'
-require 'resqued/runtime_info'
+require "spec_helper"
+require "resqued/config/after_fork"
+require "resqued/config/before_fork"
+require "resqued/runtime_info"
 
 describe do
   before { evaluator.apply(config) }
 
-  context 'after_fork' do
+  context "after_fork" do
     # Run the after_fork block.
     #
     #    after_fork do |resque_worker|
@@ -26,13 +26,13 @@ describe do
       end
     END_CONFIG
 
-    let(:evaluator) { Resqued::Config::AfterFork.new(:worker => worker) }
+    let(:evaluator) { Resqued::Config::AfterFork.new(worker: worker) }
     let(:worker) { FakeResqueWorker.new }
 
     it { expect(worker.token).to eq(:called) }
   end
 
-  context 'before_fork' do
+  context "before_fork" do
     # Run the before_fork block.
     #
     #    before_fork do
@@ -56,7 +56,7 @@ describe do
       end
     END_CONFIG
 
-    let(:evaluator) { $before_fork_called = false ; Resqued::Config::BeforeFork.new(:resqued => resqued) }
+    let(:evaluator) { $before_fork_called = false; Resqued::Config::BeforeFork.new(resqued: resqued) }
     let(:resqued) { Resqued::RuntimeInfo.new }
 
     it { expect($before_fork_called).to eq(true) }

data/spec/resqued/config/worker_spec.rb

@@ -1,5 +1,5 @@
-require 'spec_helper'
-require 'resqued/config/worker'
+require "spec_helper"
+require "resqued/config/worker"
 
 describe Resqued::Config::Worker do
   # Create a bunch of Resqued::Worker objects from
@@ -15,7 +15,7 @@ describe Resqued::Config::Worker do
   #
   # ignore calls to any other top-level method.
 
-  let(:evaluator) { described_class.new(:worker_class => FakeWorker) }
+  let(:evaluator) { described_class.new(worker_class: FakeWorker) }
   let(:result) { evaluator.apply(config) }
   module FakeWorker
     def self.new(options)
@@ -23,7 +23,7 @@ describe Resqued::Config::Worker do
     end
   end
 
-  context 'individual' do
+  context "individual" do
     let(:config) { <<-END_CONFIG }
       before_fork { }
       after_fork { }
@@ -35,25 +35,38 @@ describe Resqued::Config::Worker do
       after_fork { } # So that we don't rely on `workers`'s result falling through.
     END_CONFIG
     it { expect(result.size).to eq(6) }
-    it { expect(result[0]).to eq(:queues => ['a']) }
-    it { expect(result[1]).to eq(:queues => ['a']) }
-    it { expect(result[2]).to eq(:queues => ['b']) }
-    it { expect(result[3]).to eq(:queues => ['c', 'd']) }
-    it { expect(result[4]).to eq(:queues => ['d', 'c'], :interval => 3) }
-    it { expect(result[5]).to eq(:queues => ['*']) }
+    it { expect(result[0]).to eq(queues: ["a"]) }
+    it { expect(result[1]).to eq(queues: ["a"]) }
+    it { expect(result[2]).to eq(queues: ["b"]) }
+    it { expect(result[3]).to eq(queues: ["c", "d"]) }
+    it { expect(result[4]).to eq(queues: ["d", "c"], interval: 3) }
+    it { expect(result[5]).to eq(queues: ["*"]) }
   end
 
-  context 'concise pool' do
+  context "concise pool" do
     let(:config) { <<-END_CONFIG }
       worker_pool 2, 'a', 'b', 'c', :interval => 1
     END_CONFIG
-    it { expect(result).to eq([
-      { :queues => ['a', 'b', 'c'], :interval => 1 },
-      { :queues => ['a', 'b', 'c'], :interval => 1 },
-    ]) }
+    it do
+      expect(result).to eq([
+                             { queues: ["a", "b", "c"], interval: 1 },
+                             { queues: ["a", "b", "c"], interval: 1 },
+                           ])
+    end
   end
 
-  context 'pool (hash for concurrency)' do
+  context "small pool with percent" do
+    let(:config) { <<-END_CONFIG }
+      worker_pool 2
+      queue "a"
+      queue "b", :percent => 45
+    END_CONFIG
+    it { expect(result.size).to eq(2) }
+    it { expect(result[0]).to eq(queues: ["a", "b"]) }
+    it { expect(result[1]).to eq(queues: ["a"]) }
+  end
+
+  context "pool (hash for concurrency)" do
     let(:config) { <<-END_CONFIG }
       before_fork { }
       after_fork { }
@@ -64,15 +77,15 @@ describe Resqued::Config::Worker do
       after_fork { } # So that we don't rely on `worker_pool`'s result falling through.
     END_CONFIG
     it { expect(result.size).to eq(20) }
-    it { expect(result[0]).to eq(:queues => ['a', 'b1', 'b2', 'c'], :interval => 1) }
-    it { expect(result[3]).to eq(:queues => ['a', 'b1', 'b2', 'c'], :interval => 1) }
-    it { expect(result[4]).to eq(:queues => ['b1', 'b2', 'c'], :interval => 1) }
-    it { expect(result[9]).to eq(:queues => ['b1', 'b2', 'c'], :interval => 1) }
-    it { expect(result[10]).to eq(:queues => ['c'], :interval => 1) }
-    it { expect(result[19]).to eq(:queues => ['c'], :interval => 1) }
+    it { expect(result[0]).to eq(queues: ["a", "b1", "b2", "c"], interval: 1) }
+    it { expect(result[3]).to eq(queues: ["a", "b1", "b2", "c"], interval: 1) }
+    it { expect(result[4]).to eq(queues: ["b1", "b2", "c"], interval: 1) }
+    it { expect(result[9]).to eq(queues: ["b1", "b2", "c"], interval: 1) }
+    it { expect(result[10]).to eq(queues: ["c"], interval: 1) }
+    it { expect(result[19]).to eq(queues: ["c"], interval: 1) }
   end
 
-  context 'pool, with implied queue' do
+  context "pool, with implied queue" do
     let(:config) { <<-END_CONFIG }
       before_fork { }
       after_fork { }
@@ -80,11 +93,11 @@ describe Resqued::Config::Worker do
       after_fork { } # So that we don't rely on `worker_pool`'s result falling through.
     END_CONFIG
     it { expect(result.size).to eq(20) }
-    it { expect(result[0]).to eq(:queues => ['*']) }
-    it { expect(result[19]).to eq(:queues => ['*']) }
+    it { expect(result[0]).to eq(queues: ["*"]) }
+    it { expect(result[19]).to eq(queues: ["*"]) }
   end
 
-  context 'pool, with fewer queues than workers' do
+  context "pool, with fewer queues than workers" do
     let(:config) { <<-END_CONFIG }
       before_fork { }
       after_fork { }
@@ -93,13 +106,13 @@ describe Resqued::Config::Worker do
       after_fork { } # So that we don't rely on `worker_pool`'s result falling through.
     END_CONFIG
     it { expect(result.size).to eq(20) }
-    it { expect(result[0]).to eq(:queues => ['a']) }
-    it { expect(result[9]).to eq(:queues => ['a']) }
-    it { expect(result[10]).to eq(:queues => ['*']) }
-    it { expect(result[19]).to eq(:queues => ['*']) }
+    it { expect(result[0]).to eq(queues: ["a"]) }
+    it { expect(result[9]).to eq(queues: ["a"]) }
+    it { expect(result[10]).to eq(queues: ["*"]) }
+    it { expect(result[19]).to eq(queues: ["*"]) }
   end
 
-  context 'pool, with more queues than workers' do
+  context "pool, with more queues than workers" do
     let(:config) { <<-END_CONFIG }
       before_fork { }
       after_fork { }
@@ -110,41 +123,64 @@ describe Resqued::Config::Worker do
     it { expect(result.size).to eq(20) }
   end
 
-  context 'pool, with shuffled queues' do
+  context "pool, with shuffled queues" do
     let(:config) { <<-END_CONFIG }
       worker_pool 20, :shuffle_queues => true
       queue 'a', :count => 10
       queue 'b', :count => 15
     END_CONFIG
     it { expect(result.size).to eq(20) }
-    it { (0..9).each { |i| expect(result[i][:queues].sort).to eq(['a', 'b']) } }
-    it { (10..14).each { |i| expect(result[i][:queues]).to eq(['b']) } }
-    it { (15..19).each { |i| expect(result[i][:queues]).to eq(['*']) } }
+    it { (0..9).each { |i| expect(result[i][:queues].sort).to eq(["a", "b"]) } }
+    it { (10..14).each { |i| expect(result[i][:queues]).to eq(["b"]) } }
+    it { (15..19).each { |i| expect(result[i][:queues]).to eq(["*"]) } }
     it { result.each { |x| expect(x).not_to have_key(:shuffle_queues) } }
     it do
       shuffled_queues = result.take(10).map { |x| x[:queues] }
-      expect(shuffled_queues.sort.uniq).to eq([ ['a','b'], ['b','a'] ]) # Some of the queues should be shuffled
+      expect(shuffled_queues.sort.uniq).to eq([["a", "b"], ["b", "a"]]) # Some of the queues should be shuffled
     end
   end
 
-  context 'multiple worker configs' do
+  context "multiple worker configs" do
     let(:config) { <<-END_CONFIG }
       worker 'one'
       worker 'two'
       worker_pool 2
     END_CONFIG
     it { expect(result.size).to eq(4) }
-    it { expect(result[0]).to eq(:queues => ['one']) }
-    it { expect(result[1]).to eq(:queues => ['two']) }
-    it { expect(result[2]).to eq(:queues => ['*']) }
-    it { expect(result[3]).to eq(:queues => ['*']) }
+    it { expect(result[0]).to eq(queues: ["one"]) }
+    it { expect(result[1]).to eq(queues: ["two"]) }
+    it { expect(result[2]).to eq(queues: ["*"]) }
+    it { expect(result[3]).to eq(queues: ["*"]) }
+  end
+
+  context "worker factory" do
+    let(:config) { <<-END_CONFIG }
+      worker_factory { |queues| queues }
+      worker 'a'
+    END_CONFIG
+
+    it { expect(result.size).to eq(1) }
+    it { expect(result[0].reject { |k, _| k == :worker_factory }).to eq(queues: ["a"]) }
+    it { expect(result[0][:worker_factory].call(result[0][:queues])).to eq(["a"]) }
+  end
+
+  context "worker factory with pool" do
+    let(:config) { <<-END_CONFIG }
+      worker_factory { |queues| queues }
+      worker_pool 1
+      queue 'a'
+    END_CONFIG
+
+    it { expect(result.size).to eq(1) }
+    it { expect(result[0].reject { |k, _| k == :worker_factory }).to eq(queues: ["a"]) }
+    it { expect(result[0][:worker_factory].call(result[0][:queues])).to eq(["a"]) }
   end
 
-  context 'with default options' do
-    let(:evaluator) { described_class.new(:worker_class => FakeWorker, :config => 'something') }
+  context "with default options" do
+    let(:evaluator) { described_class.new(worker_class: FakeWorker, config: "something") }
     let(:config) { <<-END_CONFIG }
       worker 'a', :interval => 1
     END_CONFIG
-    it { expect(result[0]).to eq(:queues => ['a'], :interval => 1, :config => 'something') }
+    it { expect(result[0]).to eq(queues: ["a"], interval: 1, config: "something") }
   end
 end