inst-jobs 0.13.2 → 0.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b0d5ed365c051a94b8b26f7fc43b66ddf472221
-  data.tar.gz: 53d13170b1a9f85ea4e9064932e674b6059063ba
+  metadata.gz: 87a4b51b71b02dd1ab9e4817dffbc6b4780c4224
+  data.tar.gz: e6a9a26a33f424950c37c83238085233be136e72
 SHA512:
-  metadata.gz: d911c43ce7aadb8d1373c96d82829ea2ecfc4ffecb1b8ea1243b713cbcb30aa57b6721fa77eea5446b72bb741db128dbf4ba7da9c4dcf57b6e34fa87261483d7
-  data.tar.gz: 02ba0072c4d9c1335fcf8d70e91368a47e34b9ae2d2748a2749def1f0f0e4d4ce76114ad0baf7da5bdf23e09abab0079466f1430076dec959adb096fd72698c8
+  metadata.gz: 21bb8f405b710a970a8e70570934d6344ebdb25d91b811542f6d8cc4fc1008d8550570960497a1f15d8a27c110a61e2dba1a819cf1c7982980f5269cb49bedfe
+  data.tar.gz: 2eff9e07b7b083bee75d4536d50434f92b12f860c6390160ab90ff1737164fd83917b4d59fe64495278be2f75affe3cf63684d64b3a3930dfeb15a7d9a96feee

data/lib/delayed/backend/active_record.rb

@@ -202,8 +202,8 @@ module Delayed
                                              queue = Delayed::Settings.queue,
                                              min_priority = nil,
                                              max_priority = nil,
-                                             extra_jobs: 0,
-                                             extra_jobs_owner: nil)
+                                             prefetch: 0,
+                                             prefetch_owner: nil)
 
           check_queue(queue)
           check_priorities(min_priority, max_priority)
@@ -218,7 +218,7 @@ module Delayed
                 effective_worker_names = Array(worker_names)
 
                 target_jobs = all_available(queue, min_priority, max_priority).
-                    limit(effective_worker_names.length + extra_jobs).
+                    limit(effective_worker_names.length + prefetch).
                     lock
                 jobs_with_row_number = all.from(target_jobs).
                     select("id, ROW_NUMBER() OVER () AS row_number")
@@ -226,8 +226,8 @@ module Delayed
                 effective_worker_names.each_with_index do |worker, i|
                   updates << "WHEN #{i + 1} THEN #{connection.quote(worker)} "
                 end
-                if extra_jobs_owner
-                  updates << "ELSE #{connection.quote(extra_jobs_owner)} "
+                if prefetch_owner
+                  updates << "ELSE #{connection.quote(prefetch_owner)} "
                 end
                 updates << "END, locked_at = #{connection.quote(db_time_now)}"
                 # joins and returning in an update! just bypass AR
@@ -238,8 +238,8 @@ module Delayed
                 # all of the jobs we tried to lock had already been locked by someone else
                 if worker_names.is_a?(Array)
                   result = jobs.index_by(&:locked_by)
-                  # all of the extras can come back as an array
-                  result[extra_jobs_owner] = jobs.select { |j| j.locked_by == extra_jobs_owner } if extra_jobs_owner
+                  # all of the prefetched jobs can come back as an array
+                  result[prefetch_owner] = jobs.select { |j| j.locked_by == prefetch_owner } if prefetch_owner
                   return result
                 else
                   return jobs.first
@@ -326,6 +326,11 @@ module Delayed
           end
         end
 
+        def self.unlock_orphaned_prefetched_jobs
+          horizon = db_time_now - Settings.parent_process[:prefetched_jobs_timeout] * 4
+          where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+        end
+
         def self.unlock(jobs)
           unlocked = where(id: jobs).update_all(locked_at: nil, locked_by: nil)
           jobs.each(&:unlock)

data/lib/delayed/backend/base.rb

@@ -118,9 +118,9 @@ module Delayed
           Time.now.utc
         end
 
-        def unlock_orphaned_pending_jobs
-          horizon = db_time_now - Settings.parent_process[:pending_jobs_idle_timeout] * 4
-          orphaned_jobs = running_jobs.select { |job| job.locked_by.start_with?('work_queue:') && job.locked_at < horizon }
+        def unlock_orphaned_prefetched_jobs
+          horizon = db_time_now - Settings.parent_process[:prefetched_jobs_timeout] * 4
+          orphaned_jobs = running_jobs.select { |job| job.locked_by.start_with?('prefetch:') && job.locked_at < horizon }
           return 0 if orphaned_jobs.empty?
           unlock(orphaned_jobs)
         end

data/lib/delayed/backend/redis/job.rb

@@ -222,8 +222,8 @@ class Job
       queue = Delayed::Settings.queue,
       min_priority = Delayed::MIN_PRIORITY,
       max_priority = Delayed::MAX_PRIORITY,
-      extra_jobs: nil,
-      extra_jobs_owner: nil)
+      prefetch: nil,
+      prefetch_owner: nil)
 
     check_queue(queue)
     check_priorities(min_priority, max_priority)

data/lib/delayed/settings.rb

@@ -34,7 +34,7 @@ module Delayed
 
     PARENT_PROCESS_DEFAULTS = {
       server_socket_timeout: 10.0,
-      pending_jobs_idle_timeout: 30.0,
+      prefetched_jobs_timeout: 30.0,
 
       client_connect_timeout: 2.0,
 

data/lib/delayed/version.rb

@@ -1,3 +1,3 @@
 module Delayed
-  VERSION = "0.13.2"
+  VERSION = "0.13.3"
 end

data/lib/delayed/work_queue/parent_process/client.rb

@@ -18,11 +18,17 @@ class ParentProcess
       # to wait for anything to be available on the 'wire', this is a valid
       # assumption because we control the server and it's a Unix domain socket,
       # not TCP.
-      return reset_connection if socket.eof? # Other end closed gracefully, so should we
+      if socket.eof?
+        # Other end closed gracefully, so should we
+        logger.debug("server closed connection")
+        return reset_connection
+      end
+
       Marshal.load(socket).tap do |response|
         unless response.nil? || (response.is_a?(Delayed::Job) && response.locked_by == worker_name)
           raise(ProtocolError, "response is not a locked job: #{response.inspect}")
         end
+        logger.debug("Received job #{response.id}")
       end
     rescue SystemCallError, IOError => ex
       logger.error("Work queue connection lost, reestablishing on next poll. (#{ex})")

data/lib/delayed/work_queue/parent_process/server.rb

@@ -11,7 +11,7 @@ class ParentProcess
       @parent_pid = parent_pid
       @clients = {}
       @waiting_clients = {}
-      @pending_work = {}
+      @prefetched_jobs = {}
 
       @config = config
       @client_timeout = config['server_socket_timeout'] || 10.0 # left for backwards compat
@@ -30,12 +30,12 @@ class ParentProcess
     def run
       logger.debug "Starting work queue process"
 
-      last_orphaned_pending_jobs_purge = Job.db_time_now - rand(15 * 60)
+      last_orphaned_prefetched_jobs_purge = Job.db_time_now - rand(15 * 60)
       while !exit?
         run_once
-        if last_orphaned_pending_jobs_purge + 15 * 60 < Job.db_time_now
-          Job.unlock_orphaned_pending_jobs
-          last_orphaned_pending_jobs_purge = Job.db_time_now
+        if last_orphaned_prefetched_jobs_purge + 15 * 60 < Job.db_time_now
+          Job.unlock_orphaned_prefetched_jobs
+          last_orphaned_prefetched_jobs_purge = Job.db_time_now
         end
       end
 
@@ -43,7 +43,7 @@ class ParentProcess
       logger.debug "WorkQueue Server died: #{e.inspect}", :error
       raise
     ensure
-      purge_all_pending_work
+      unlock_all_prefetched_jobs
     end
 
     def run_once
@@ -54,7 +54,7 @@ class ParentProcess
         readable.each { |s| handle_read(s) }
       end
       check_for_work
-      purge_extra_pending_work
+      unlock_timed_out_prefetched_jobs
     end
 
     def handle_read(socket)
@@ -82,9 +82,12 @@ class ParentProcess
       # request and then leave the socket open. Doing so would leave us hanging
       # in Marshal.load forever. This is only a reasonable assumption because we
       # control the client.
-      return drop_socket(socket) if socket.eof?
-      worker_name, worker_config = Marshal.load(socket)
       client = @clients[socket]
+      if socket.eof?
+        logger.debug("Client #{client.name} closed connection")
+        return drop_socket(socket)
+      end
+      worker_name, worker_config = Marshal.load(socket)
       client.name = worker_name
       client.working = false
       (@waiting_clients[worker_config] ||= []) << client
@@ -95,17 +98,18 @@ class ParentProcess
 
     def check_for_work
       @waiting_clients.each do |(worker_config, workers)|
-        pending_work = @pending_work[worker_config] ||= []
-        logger.debug("I have #{pending_work.length} jobs for #{workers.length} waiting workers")
-        while !pending_work.empty? && !workers.empty?
-          job = pending_work.shift
+        prefetched_jobs = @prefetched_jobs[worker_config] ||= []
+        logger.debug("I have #{prefetched_jobs.length} jobs for #{workers.length} waiting workers")
+        while !prefetched_jobs.empty? && !workers.empty?
+          job = prefetched_jobs.shift
           client = workers.shift
           # couldn't re-lock it for some reason
-          unless job.transfer_lock!(from: pending_jobs_owner, to: client.name)
+          unless job.transfer_lock!(from: prefetch_owner, to: client.name)
             workers.unshift(client)
             next
           end
           begin
+            logger.debug("Sending prefetched job #{job.id} to #{client.name}")
             client_timeout { Marshal.dump(job, client.socket) }
           rescue SystemCallError, IOError, Timeout::Error => ex
             logger.error("Failed to send pre-fetched job to #{client.name}: #{ex.inspect}")
@@ -124,18 +128,19 @@ class ParentProcess
               worker_config[:queue],
               worker_config[:min_priority],
               worker_config[:max_priority],
-              extra_jobs: Settings.fetch_batch_size * (worker_config[:workers] || 1) - recipients.length,
-              extra_jobs_owner: pending_jobs_owner)
+              prefetch: Settings.fetch_batch_size * (worker_config[:workers] || 1) - recipients.length,
+              prefetch_owner: prefetch_owner)
           response.each do |(worker_name, job)|
-            if worker_name == pending_jobs_owner
+            if worker_name == prefetch_owner
               # it's actually an array of all the extra jobs
-              pending_work.concat(job)
+              prefetched_jobs.concat(job)
               next
             end
             client = workers.find { |worker| worker.name == worker_name }
             client.working = true
             @waiting_clients[worker_config].delete(client)
             begin
+              logger.debug("Sending job #{job.id} to #{client.name}")
               client_timeout { Marshal.dump(job, client.socket) }
             rescue SystemCallError, IOError, Timeout::Error => ex
               logger.error("Failed to send job to #{client.name}: #{ex.inspect}")
@@ -147,22 +152,22 @@ class ParentProcess
       end
     end
 
-    def purge_extra_pending_work
-      @pending_work.each do |(worker_config, jobs)|
+    def unlock_timed_out_prefetched_jobs
+      @prefetched_jobs.each do |(worker_config, jobs)|
         next if jobs.empty?
-        if jobs.first.locked_at < Time.now.utc - Settings.parent_process[:pending_jobs_idle_timeout]
+        if jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
           Delayed::Job.unlock(jobs)
-          @pending_work[worker_config] = []
+          @prefetched_jobs[worker_config] = []
         end
       end
     end
 
-    def purge_all_pending_work
-      @pending_work.each do |(_worker_config, jobs)|
+    def unlock_all_prefetched_jobs
+      @prefetched_jobs.each do |(_worker_config, jobs)|
         next if jobs.empty?
         Delayed::Job.unlock(jobs)
       end
-      @pending_work = {}
+      @prefetched_jobs = {}
     end
 
     def drop_socket(socket)
@@ -171,15 +176,19 @@ class ParentProcess
         socket.close
       rescue IOError
       end
+      client = @clients[socket]
       @clients.delete(socket)
+      @waiting_clients.each do |(_config, workers)|
+        workers.delete(client)
+      end
     end
 
     def exit?
       parent_exited?
     end
 
-    def pending_jobs_owner
-      "work_queue:#{Socket.gethostname rescue 'X'}"
+    def prefetch_owner
+      "prefetch:#{Socket.gethostname rescue 'X'}"
     end
 
     def parent_exited?

data/spec/active_record_job_spec.rb

@@ -244,20 +244,20 @@ describe 'Delayed::Backed::ActiveRecord::Job' do
     end
   end
 
-  it "unlocks orphaned jobs from work queue" do
+  it "unlocks orphaned prefetched_jobs" do
     job1 = Delayed::Job.new(:tag => 'tag')
     job2 = Delayed::Job.new(:tag => 'tag')
 
-    job1.create_and_lock!("work_queue:a")
+    job1.create_and_lock!("prefetch:a")
     job1.locked_at = Delayed::Job.db_time_now - 15 * 60
     job1.save!
-    job2.create_and_lock!("work_queue:a")
+    job2.create_and_lock!("prefetch:a")
 
-    expect(Delayed::Job.unlock_orphaned_pending_jobs).to eq 1
-    expect(Delayed::Job.unlock_orphaned_pending_jobs).to eq 0
+    expect(Delayed::Job.unlock_orphaned_prefetched_jobs).to eq 1
+    expect(Delayed::Job.unlock_orphaned_prefetched_jobs).to eq 0
 
     expect(Delayed::Job.find(job1.id).locked_by).to be_nil
-    expect(Delayed::Job.find(job2.id).locked_by).to eq 'work_queue:a'
+    expect(Delayed::Job.find(job2.id).locked_by).to eq 'prefetch:a'
   end
 
   it "allows fetching multiple jobs at once" do
@@ -272,8 +272,8 @@ describe 'Delayed::Backed::ActiveRecord::Job' do
   it "allows fetching extra jobs" do
     jobs = 5.times.map { Delayed::Job.create :payload_object => SimpleJob.new }
     locked_jobs = Delayed::Job.get_and_lock_next_available(['worker1'],
-                                                           extra_jobs: 2,
-                                                           extra_jobs_owner: 'work_queue')
+                                                           prefetch: 2,
+                                                           prefetch_owner: 'work_queue')
     expect(locked_jobs.length).to eq 2
     expect(locked_jobs.keys).to eq ['worker1', 'work_queue']
     expect(locked_jobs['worker1']).to eq jobs[0]

data/spec/delayed/work_queue/parent_process/server_spec.rb

@@ -1,13 +1,25 @@
 require 'spec_helper'
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
+  class JobClass
+    attr_reader :id
+
+    def initialize
+      @id = rand
+    end
+
+    def ==(other)
+      self.id == other.id
+    end
+  end
+
   let(:parent) { Delayed::WorkQueue::ParentProcess.new }
   let(:subject) { described_class.new(listen_socket) }
   let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
-  let(:job) { :a_job }
+  let(:job) { JobClass.new }
   let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
   let(:args) { ["worker_name", worker_config] }
-  let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(extra_jobs: 4)] }
+  let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(prefetch: 4)] }
 
   before :all do
     Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
@@ -47,33 +59,35 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
     client2 = Socket.unix(subject.listen_socket.local_address.unix_path)
     subject.run_once
 
-    job_args = [["worker_name1", "worker_name2"], "queue_name", 1, 2, hash_including(extra_jobs: 3)]
-    jobs = { 'worker_name1' => :job1, 'worker_name2' => :job2 }
+    job1 = JobClass.new
+    job2 = JobClass.new
+    job_args = [["worker_name1", "worker_name2"], "queue_name", 1, 2, hash_including(prefetch: 3)]
+    jobs = { 'worker_name1' => job1, 'worker_name2' => job2 }
 
     expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return(jobs)
     Marshal.dump(["worker_name1", worker_config], client1)
     Marshal.dump(["worker_name2", worker_config], client2)
     subject.run_once
-    expect(Marshal.load(client1)).to eq(:job1)
-    expect(Marshal.load(client2)).to eq(:job2)
+    expect(Marshal.load(client1)).to eq(job1)
+    expect(Marshal.load(client2)).to eq(job2)
   end
 
-  it 'will fetch and use extra jobs' do
+  it 'will prefetch and use jobs' do
     client = Socket.unix(subject.listen_socket.local_address.unix_path)
     subject.run_once
 
-    allow(subject).to receive(:pending_jobs_owner).and_return('work_queue:X')
-    job_args = [["worker_name1"], "queue_name", 1, 2, extra_jobs: 4, extra_jobs_owner: 'work_queue:X']
+    allow(subject).to receive(:prefetch_owner).and_return('work_queue:X')
+    job_args = [["worker_name1"], "queue_name", 1, 2, prefetch: 4, prefetch_owner: 'work_queue:X']
     job2 = Delayed::Job.new(:tag => 'tag')
     job2.create_and_lock!('work_queue:X')
     job3 = Delayed::Job.new(:tag => 'tag')
     job3.create_and_lock!('work_queue:X')
-    jobs = { 'worker_name1' => :job1, 'work_queue:X' => [job2, job3]}
+    jobs = { 'worker_name1' => job, 'work_queue:X' => [job2, job3]}
 
     expect(Delayed::Job).to receive(:get_and_lock_next_available).once.with(*job_args).and_return(jobs)
     Marshal.dump(["worker_name1", worker_config], client)
     subject.run_once
-    expect(Marshal.load(client)).to eq(:job1)
+    expect(Marshal.load(client)).to eq(job)
     Marshal.dump(["worker_name1", worker_config], client)
     subject.run_once
     expect(Marshal.load(client)).to eq(job2)
@@ -112,11 +126,35 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
     subject.run_once
 
     Marshal.dump(args, client)
+    # make sure the server knows the client is waiting for a job
+    subject.run_once
+
+    client.close
+    expect { subject.run_once }.to change(subject, :connected_clients).by(-1)
+    expect(subject.instance_variable_get(:@waiting_clients).first.last).to eq []
+  end
+
+  it 'drops the client when a write fails' do
+    client = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+
+    Marshal.dump(args, client)
+    subject.run_once
+
+    client.close
 
     server_client_socket = subject.clients.keys.first
+    # don't let the server see the close and process it there; we want to check a failure later
+    expect(subject).to receive(:handle_request).with(server_client_socket)
 
-    expect(server_client_socket).to receive(:eof?).and_return(true)
-    expect { subject.run_once }.to change(subject, :connected_clients).by(-1)
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*job_args).and_return('worker_name' => job)
+    # the job gets unlocked
+    expect(Delayed::Job).to receive(:unlock).with([job])
+    subject.run_once
+
+    # and the server removes the client from both of its internal state arrays
+    expect(subject.connected_clients).to eq 0
+    expect(subject.instance_variable_get(:@waiting_clients).first.last).to eq []
   end
 
   it 'tracks when clients are idle' do

data/spec/spec_helper.rb

@@ -41,7 +41,7 @@ Delayed::Backend::Redis::Job.redis.select ENV['TEST_ENV_NUMBER']
 
 connection_config = {
   adapter: :postgresql,
-  host: ENV['TEST_DB_HOST'],
+  host: ENV['TEST_DB_HOST'].presence,
   encoding: 'utf8',
   username: ENV['TEST_DB_USERNAME'],
   database: ENV['TEST_DB_DATABASE'],

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 0.13.3
 platform: ruby
 authors:
 - Tobias Luetke
@@ -45,14 +45,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.2
+        version: '3.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.2
+        version: '3.4'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement