inst-jobs 0.12.3 → 0.13.0

data/lib/delayed/work_queue/parent_process/client.rb

@@ -0,0 +1,54 @@
+module Delayed
+module WorkQueue
+class ParentProcess
+  class Client
+    attr_reader :addrinfo
+
+    include Delayed::Logging
+
+    def initialize(addrinfo, config: Settings.parent_process)
+      @addrinfo = addrinfo
+      @connect_timeout = config['client_connect_timeout'] || 2
+      @receive_timeout = config['client_receive_timeout'] || 10
+    end
+
+    def get_and_lock_next_available(worker_name, worker_config)
+      Marshal.dump([worker_name, worker_config], socket)
+
+      # We're assuming there won't ever be a partial write here so we only need
+      # to wait for anything to be available on the 'wire', this is a valid
+      # assumption because we control the server and it's a Unix domain socket,
+      # not TCP.
+      if socket.wait_readable(@receive_timeout)
+        return reset_connection if socket.eof? # Other end closed gracefully, so should we
+        Marshal.load(socket).tap do |response|
+          unless response.nil? || (response.is_a?(Delayed::Job) && response.locked_by == worker_name)
+            raise(ProtocolError, "response is not a locked job: #{response.inspect}")
+          end
+        end
+      else
+        reset_connection
+      end
+    rescue SystemCallError, IOError => ex
+      logger.error("Work queue connection lost, reestablishing on next poll. (#{ex})")
+      # The work queue process died. Return nil to signal the worker
+      # process should sleep as if no job was found, and then retry.
+      reset_connection
+    end
+
+    private
+
+    def socket
+      @socket ||= @addrinfo.connect(timeout: @connect_timeout)
+    end
+
+    def reset_connection
+      if @socket
+        @socket.close
+        @socket = nil
+      end
+    end
+  end
+end
+end
+end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -0,0 +1,200 @@
+module Delayed
+module WorkQueue
+class ParentProcess
+  class Server
+    attr_reader :clients, :listen_socket
+
+    include Delayed::Logging
+
+    def initialize(listen_socket, parent_pid: nil, config: Settings.parent_process)
+      @listen_socket = listen_socket
+      @parent_pid = parent_pid
+      @clients = {}
+      @waiting_clients = {}
+      @pending_work = {}
+
+      @config = config
+      @client_timeout = config['server_socket_timeout'] || 10.0 # left for backwards compat
+      @receive_timeout = config['server_receive_timeout'] || 10.0
+    end
+
+    def connected_clients
+      @clients.size
+    end
+
+    def all_workers_idle?
+      !@clients.any? { |_, c| c.working }
+    end
+
+    # run the server queue worker
+    # this method does not return, only exits or raises an exception
+    def run
+      logger.debug "Starting work queue process"
+
+      last_orphaned_pending_jobs_purge = Job.db_time_now - rand(15 * 60)
+      while !exit?
+        run_once
+        if last_orphaned_pending_jobs_purge + 15 * 60 < Job.db_time_now
+          Job.unlock_orphaned_pending_jobs
+          last_orphaned_pending_jobs_purge = Job.db_time_now
+        end
+      end
+
+    rescue => e
+      logger.debug "WorkQueue Server died: #{e.inspect}", :error
+      raise
+    ensure
+      purge_all_pending_work
+    end
+
+    def run_once
+      handles = @clients.keys + [@listen_socket]
+      timeout = Settings.sleep_delay + (rand * Settings.sleep_delay_stagger)
+      readable, _, _ = IO.select(handles, nil, nil, timeout)
+      if readable
+        readable.each { |s| handle_read(s) }
+      end
+      check_for_work
+      purge_extra_pending_work
+    end
+
+    def handle_read(socket)
+      if socket == @listen_socket
+        handle_accept
+      else
+        handle_request(socket)
+      end
+    end
+
+    # Any error on the listen socket other than WaitReadable will bubble up
+    # and terminate the work queue process, to be restarted by the parent daemon.
+    def handle_accept
+      socket, _addr = @listen_socket.accept_nonblock
+      if socket
+        @clients[socket] = ClientState.new(false, socket)
+      end
+    rescue IO::WaitReadable
+      logger.error("Server attempted to read listen_socket but failed with IO::WaitReadable")
+      # ignore and just try accepting again next time through the loop
+    end
+
+    def handle_request(socket)
+      # There is an assumption here that the client will never send a partial
+      # request and then leave the socket open. Doing so would leave us hanging
+      # in Marshal.load forever. This is only a reasonable assumption because we
+      # control the client. Also, in theory, we shouldn't need to call
+      # #wait_readable to ensure there is data available since we just used
+      # select(2) to get this handle but it's better to be safe than sorry.
+      if socket.wait_readable(@receive_timeout)
+        worker_name, worker_config = Marshal.load(socket)
+        client = @clients[socket]
+        client.name = worker_name
+        client.working = false
+        (@waiting_clients[worker_config] ||= []) << client
+      else
+        drop_socket(socket)
+      end
+
+    rescue SystemCallError, IOError => ex
+      logger.error("Receiving message from client (#{socket}) failed: #{ex.inspect}")
+      drop_socket(socket)
+    end
+
+    def check_for_work
+      @waiting_clients.each do |(worker_config, workers)|
+        pending_work = @pending_work[worker_config] ||= []
+        logger.debug("I have #{pending_work.length} jobs for #{workers.length} waiting workers")
+        while !pending_work.empty? && !workers.empty?
+          job = pending_work.shift
+          client = workers.shift
+          # couldn't re-lock it for some reason
+          unless job.transfer_lock!(from: pending_jobs_owner, to: client.name)
+            workers.unshift(client)
+            next
+          end
+          begin
+            client_timeout { Marshal.dump(job, client.socket) }
+          rescue SystemCallError, IOError, Timeout::Error
+            drop_socket(client.socket)
+          end
+        end
+
+        next if workers.empty?
+
+        Delayed::Worker.lifecycle.run_callbacks(:work_queue_pop, self, worker_config) do
+          recipients = workers.map(&:name)
+
+          response = Delayed::Job.get_and_lock_next_available(
+              recipients,
+              worker_config[:queue],
+              worker_config[:min_priority],
+              worker_config[:max_priority],
+              extra_jobs: Settings.fetch_batch_size * (worker_config[:workers] || 1) - recipients.length,
+              extra_jobs_owner: pending_jobs_owner)
+          response.each do |(worker_name, job)|
+            if worker_name == pending_jobs_owner
+              # it's actually an array of all the extra jobs
+              pending_work.concat(job)
+              next
+            end
+            client = workers.find { |worker| worker.name == worker_name }
+            client.working = true
+            @waiting_clients[worker_config].delete(client)
+            begin
+              client_timeout { Marshal.dump(job, client.socket) }
+            rescue SystemCallError, IOError, Timeout::Error
+              drop_socket(client.socket)
+            end
+          end
+        end
+      end
+    end
+
+    def purge_extra_pending_work
+      @pending_work.each do |(worker_config, jobs)|
+        next if jobs.empty?
+        if jobs.first.locked_at < Time.now.utc - Settings.parent_process[:pending_jobs_idle_timeout]
+          Delayed::Job.unlock(jobs)
+          @pending_work[worker_config] = []
+        end
+      end
+    end
+
+    def purge_all_pending_work
+      @pending_work.each do |(_worker_config, jobs)|
+        next if jobs.empty?
+        Delayed::Job.unlock(jobs)
+      end
+      @pending_work = {}
+    end
+
+    def drop_socket(socket)
+      # this socket went away
+      begin
+        socket.close
+      rescue IOError
+      end
+      @clients.delete(socket)
+    end
+
+    def exit?
+      parent_exited?
+    end
+
+    def pending_jobs_owner
+      "work_queue:#{Socket.gethostname rescue 'X'}"
+    end
+
+    def parent_exited?
+      @parent_pid && @parent_pid != Process.ppid
+    end
+
+    def client_timeout
+      Timeout.timeout(@client_timeout) { yield }
+    end
+
+    ClientState = Struct.new(:working, :socket, :name)
+  end
+end
+end
+end

data/lib/delayed_job.rb

@@ -29,6 +29,7 @@ require 'delayed/daemon'
 require 'delayed/job_tracking'
 require 'delayed/lifecycle'
 require 'delayed/log_tailer'
+require 'delayed/logging'
 require 'delayed/message_sending'
 require 'delayed/performable_method'
 require 'delayed/periodic'

data/spec/active_record_job_spec.rb

@@ -244,6 +244,22 @@ describe 'Delayed::Backed::ActiveRecord::Job' do
     end
   end
 
+  it "unlocks orphaned jobs from work queue" do
+    job1 = Delayed::Job.new(:tag => 'tag')
+    job2 = Delayed::Job.new(:tag => 'tag')
+
+    job1.create_and_lock!("work_queue:a")
+    job1.locked_at = Delayed::Job.db_time_now - 15 * 60
+    job1.save!
+    job2.create_and_lock!("work_queue:a")
+
+    expect(Delayed::Job.unlock_orphaned_pending_jobs).to eq 1
+    expect(Delayed::Job.unlock_orphaned_pending_jobs).to eq 0
+
+    expect(Delayed::Job.find(job1.id).locked_by).to be_nil
+    expect(Delayed::Job.find(job2.id).locked_by).to eq 'work_queue:a'
+  end
+
   it "allows fetching multiple jobs at once" do
     jobs = 3.times.map { Delayed::Job.create :payload_object => SimpleJob.new }
     locked_jobs = Delayed::Job.get_and_lock_next_available(['worker1', 'worker2'])
@@ -252,4 +268,16 @@ describe 'Delayed::Backed::ActiveRecord::Job' do
     locked_jobs.values.should == jobs[0..1]
     jobs.map(&:reload).map(&:locked_by).should == ['worker1', 'worker2', nil]
   end
+
+  it "allows fetching extra jobs" do
+    jobs = 5.times.map { Delayed::Job.create :payload_object => SimpleJob.new }
+    locked_jobs = Delayed::Job.get_and_lock_next_available(['worker1'],
+                                                           extra_jobs: 2,
+                                                           extra_jobs_owner: 'work_queue')
+    expect(locked_jobs.length).to eq 2
+    expect(locked_jobs.keys).to eq ['worker1', 'work_queue']
+    expect(locked_jobs['worker1']).to eq jobs[0]
+    expect(locked_jobs['work_queue']).to eq jobs[1..2]
+    jobs.map(&:reload).map(&:locked_by).should == ['worker1', 'work_queue', 'work_queue', nil, nil]
+  end
 end

data/spec/delayed/settings_spec.rb

@@ -29,4 +29,11 @@ default:
       described_class.apply_worker_config!('last_ditch_logfile' => true)
     end
   end
+
+  describe '.parent_process_client_timeout=' do
+    it 'must update the value in the parent_process settings hash' do
+      Delayed::Settings.parent_process_client_timeout = 42
+      expect(Delayed::Settings.parent_process['server_socket_timeout']).to eq 42
+    end
+  end
 end

data/spec/delayed/work_queue/parent_process/client_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
+  let(:subject) { described_class.new(addrinfo) }
+  let(:addrinfo) { double('Addrinfo') }
+  let(:connection) { double('Socket') }
+  let(:job) { Delayed::Job.new(locked_by: "worker_name") }
+  let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
+  let(:args) { ["worker_name", worker_config] }
+  let(:job_args) { [["worker_name"], "queue_name", 1, 2] }
+
+  before :all do
+    FileUtils.mkdir_p(Delayed::Settings.expand_rails_path('tmp'))
+    Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+  end
+
+  after :all do
+    Delayed.send(:remove_const, :Job)
+  end
+
+  it 'marshals the given arguments to the server and returns the response' do
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(connection)
+    expect(connection).to receive(:eof?).and_return(false)
+    expect(Marshal).to receive(:dump).with(args, connection).ordered
+    expect(Marshal).to receive(:load).with(connection).and_return(job).ordered
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to eq(job)
+  end
+
+  it 'returns nil and then reconnects on receive timeout' do
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(nil)
+    expect(Marshal).to receive(:dump).with(args, connection).ordered
+    expect(connection).to receive(:close)
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to be_nil
+
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(Marshal).to receive(:dump).with(args, connection)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(connection)
+    expect(connection).to receive(:eof?).and_return(false)
+    expect(Marshal).to receive(:load).with(connection).and_return(job)
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to eq(job)
+  end
+
+  it 'returns nil and then reconnects on socket write error' do
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(Marshal).to receive(:dump).and_raise(SystemCallError.new("failure"))
+    expect(connection).to receive(:close)
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to be_nil
+
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(Marshal).to receive(:dump).with(args, connection)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(connection)
+    expect(connection).to receive(:eof?).and_return(false)
+    expect(Marshal).to receive(:load).with(connection).and_return(job)
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to eq(job)
+  end
+
+  it 'returns nil and then reconnects when the socket indicates eof' do
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(true)
+    expect(connection).to receive(:eof?).and_return(true)
+    expect(Marshal).to receive(:dump).with(args, connection).ordered
+    expect(connection).to receive(:close)
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to be_nil
+
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(Marshal).to receive(:dump).with(args, connection)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(connection)
+    expect(connection).to receive(:eof?).and_return(false)
+    expect(Marshal).to receive(:load).with(connection).and_return(job)
+    response = subject.get_and_lock_next_available(*args)
+    expect(response).to eq(job)
+  end
+
+  it 'errors if the response is not a locked job' do
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(Marshal).to receive(:dump).with(args, connection)
+    expect(Marshal).to receive(:load).with(connection).and_return(:not_a_job)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(connection)
+    expect(connection).to receive(:eof?).and_return(false)
+
+    expect { subject.get_and_lock_next_available(*args) }.to raise_error(Delayed::WorkQueue::ParentProcess::ProtocolError)
+  end
+
+  it 'errors if the response is a job not locked by this worker' do
+    expect(addrinfo).to receive(:connect).once.and_return(connection)
+    expect(Marshal).to receive(:dump).with(args, connection)
+    job.locked_by = "somebody_else"
+    expect(Marshal).to receive(:load).with(connection).and_return(job)
+    expect(connection).to receive(:wait_readable).with(10.0).and_return(connection)
+    expect(connection).to receive(:eof?).and_return(false)
+
+    expect { subject.get_and_lock_next_available(*args) }.to raise_error(Delayed::WorkQueue::ParentProcess::ProtocolError)
+  end
+end

data/spec/delayed/work_queue/parent_process/server_spec.rb

@@ -0,0 +1,162 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
+  let(:parent) { Delayed::WorkQueue::ParentProcess.new }
+  let(:subject) { described_class.new(listen_socket) }
+  let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
+  let(:job) { :a_job }
+  let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
+  let(:args) { ["worker_name", worker_config] }
+  let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(extra_jobs: 4)] }
+
+  before :all do
+    Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+    Delayed::Settings.parent_process = {
+      'server_address' => '/tmp/inst-jobs-test.sock'
+    }
+  end
+
+  after :all do
+    Delayed.send(:remove_const, :Job)
+    Delayed::Settings.parent_process = {}
+  end
+
+  after :each do
+    File.unlink('/tmp/inst-jobs-test.sock') if File.exist?('/tmp/inst-jobs-test.sock')
+  end
+
+  it 'accepts new clients' do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    expect { subject.run_once }.to change(subject, :connected_clients).by(1)
+  end
+
+  it 'queries the queue on client request' do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return('worker_name' => job)
+    Marshal.dump(args, client)
+    subject.run_once
+    expect(client).to be_ready
+    expect(Marshal.load(client)).to eq(job)
+  end
+
+  it 'can pop multiple jobs at once' do
+    client1 = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+    client2 = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    job_args = [["worker_name1", "worker_name2"], "queue_name", 1, 2, hash_including(extra_jobs: 3)]
+    jobs = { 'worker_name1' => :job1, 'worker_name2' => :job2 }
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return(jobs)
+    Marshal.dump(["worker_name1", worker_config], client1)
+    Marshal.dump(["worker_name2", worker_config], client2)
+    subject.run_once
+    expect(Marshal.load(client1)).to eq(:job1)
+    expect(Marshal.load(client2)).to eq(:job2)
+  end
+
+  it 'will fetch and use extra jobs' do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    allow(subject).to receive(:pending_jobs_owner).and_return('work_queue:X')
+    job_args = [["worker_name1"], "queue_name", 1, 2, extra_jobs: 4, extra_jobs_owner: 'work_queue:X']
+    job2 = Delayed::Job.new(:tag => 'tag')
+    job2.create_and_lock!('work_queue:X')
+    job3 = Delayed::Job.new(:tag => 'tag')
+    job3.create_and_lock!('work_queue:X')
+    jobs = { 'worker_name1' => :job1, 'work_queue:X' => [job2, job3]}
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).once.with(*job_args).and_return(jobs)
+    Marshal.dump(["worker_name1", worker_config], client)
+    subject.run_once
+    expect(Marshal.load(client)).to eq(:job1)
+    Marshal.dump(["worker_name1", worker_config], client)
+    subject.run_once
+    expect(Marshal.load(client)).to eq(job2)
+  end
+
+  it "doesn't respond immediately if there are no jobs available" do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return({}).ordered
+    Marshal.dump(args, client)
+    subject.run_once
+    expect(client).not_to be_ready
+
+    # next time around, return the result
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return('worker_name' => job).ordered
+    allow(Delayed::Settings).to receive(:sleep_delay).and_return(0)
+    allow(Delayed::Settings).to receive(:sleep_delay_stagger).and_return(0)
+    subject.run_once
+    expect(client).to be_ready
+    expect(Marshal.load(client)).to eq(job)
+  end
+
+  it 'drops the client on i/o error' do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    Marshal.dump(args, client)
+
+    expect(Marshal).to receive(:load).and_raise(IOError.new("socket went away"))
+    expect { subject.run_once }.to change(subject, :connected_clients).by(-1)
+  end
+
+  it 'drops the client on timeout' do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    Marshal.dump(args, client)
+
+    server_client_socket = subject.clients.keys.first
+
+    expect(server_client_socket).to receive(:wait_readable).and_return(false)
+    expect { subject.run_once }.to change(subject, :connected_clients).by(-1)
+  end
+
+  it 'tracks when clients are idle' do
+    expect(subject.all_workers_idle?).to be(true)
+
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+    expect(subject.all_workers_idle?).to be(true)
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return('worker_name' => job)
+    Marshal.dump(args, client)
+    subject.run_once
+    expect(subject.all_workers_idle?).to be(false)
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return({})
+    Marshal.dump(args, client)
+    subject.run_once
+    expect(subject.all_workers_idle?).to be(true)
+  end
+
+  it 'triggers the lifecycle event around the pop' do
+    called = false
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    Delayed::Worker.lifecycle.around(:work_queue_pop) do |queue, &cb|
+      expect(subject.all_workers_idle?).to be(true)
+      expect(queue).to eq(subject)
+      expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return('worker_name' => job)
+      called = true
+      res = cb.call(queue)
+      expect(subject.all_workers_idle?).to be(false)
+      res
+    end
+
+    Marshal.dump(args, client)
+    subject.run_once
+
+    expect(Marshal.load(client)).to eq(job)
+    expect(called).to eq(true)
+  end
+end
+