inst-jobs 2.4.8 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b810ed7504a4de6c0338c2b6f0b2303e72e172225ca57d8ed3ebf8a9f05c6111
-  data.tar.gz: 7fff2151aa908f846af19401a39390beee21ca2fdd9b9317425fdcf0345970fb
+  metadata.gz: c9d86b7b6161b9397c885a82d687f85d84e7a17db5261bfc5ef1bb7e9f25e2ac
+  data.tar.gz: 4a75f8ac73b5f0a0b19d8073bcbfe87f654bbbdb90e147ddf02066916653d4e0
 SHA512:
-  metadata.gz: 1a98557f9c875df6e9961b849dd8d6fec4564e247eed2d68a717dbaef94ed5af4b2a61642c0c17a4bf7f1eaac102b9607a2da5b4fb9b2c40ef3cca2b6bafcef5
-  data.tar.gz: 1fcaa3bc4d1191d2a8d56156e40755d032f635dcaef7f05fbaf19d197b6456d552578ae7fcf376d4fe90ab70c3406a81d8d959ff87b714b669defccb8d9b04d0
+  metadata.gz: cd3f9df169146f34f7366e02830575bf27e2fd33e97d190bed34dd8be44e34431261111eef133368cd342668cabeb529af6e16c8985fcf15c99807ea7a07fbab
+  data.tar.gz: 3924f7b2f6f4eee37c4d92bfedfd683304dfd0c035a65f47ea9dae607c0e81b03c35860420f39a7dadf890d203c85fe24de68c90c80a48839f23b6193311b3e3

data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class UpdateConflictingSingletonFunctionToUseIndex < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
+      BEGIN
+        DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton AND locked_by IS NULL;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
+      BEGIN
+        IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
+          DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -62,14 +62,9 @@ module Delayed
             _write_attribute(column, current_time) unless attribute_present?(column)
           end
 
-          if Rails.version >= "6"
-            attribute_names = attribute_names_for_partial_writes
-            attribute_names = attributes_for_create(attribute_names)
-            values = attributes_with_values(attribute_names)
-          else
-            attribute_names = partial_writes? ? keys_for_partial_write : self.attribute_names
-            values = attributes_with_values_for_create(attribute_names)
-          end
+          attribute_names = attribute_names_for_partial_writes
+          attribute_names = attributes_for_create(attribute_names)
+          values = attributes_with_values(attribute_names)
 
           im = self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
 

data/lib/delayed/backend/base.rb

@@ -176,7 +176,9 @@ module Delayed
           ignores = []
           loop do
             batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
-            batch = batch_scope.to_a
+            # if we don't reload this it's possible to keep getting the
+            # same array each loop even after the jobs have been deleted.
+            batch = batch_scope.reload.to_a
             break if batch.empty?
 
             batch.each do |job|

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "2.4.8"
+  VERSION = "3.0.1"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "activerecord-pg-extensions"
+
 module Delayed
   module WorkQueue
     class ParentProcess
@@ -179,7 +181,7 @@ module Delayed
             end
 
             jobs_to_send.each do |(recipient, job_to_send)|
-              @waiting_clients[worker_config].delete(client)
+              @waiting_clients[worker_config].delete(recipient)
               begin
                 logger.debug("Sending job #{job_to_send.id} to #{recipient.name}")
                 client_timeout { Marshal.dump(job_to_send, recipient.socket) }
@@ -192,29 +194,54 @@ module Delayed
           end
         end
 
-        def unlock_timed_out_prefetched_jobs
+        def unlock_prefetched_jobs
           @prefetched_jobs.each do |(worker_config, jobs)|
             next if jobs.empty?
-            next unless jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
+            next if block_given? && !yield(jobs)
 
-            Delayed::Job.transaction do
+            connection = Delayed::Job.connection
+            connection.transaction do
+              # make absolutely sure we don't get hung up and leave things
+              # locked in the database
+              if connection.postgresql_version >= 9_06_00 # rubocop:disable Style/NumericLiterals
+                connection.idle_in_transaction_session_timeout = 5
+              end
+              # relatively short timeout for acquiring the lock
+              connection.statement_timeout = Settings.sleep_delay
               Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
+
+              # this query might take longer, and we really want to get it
+              # done if we got the lock, but still don't want an inadvertent
+              # hang
+              connection.statement_timeout = 30
               Delayed::Job.unlock(jobs)
+              @prefetched_jobs[worker_config] = []
             end
-            @prefetched_jobs[worker_config] = []
+          rescue ActiveRecord::QueryCanceled
+            # ignore; we'll retry anyway
+            logger.warn("unable to unlock prefetched jobs; skipping for now")
+          rescue ActiveRecord::StatementInvalid
+            # see if we dropped the connection
+            raise if connection.active?
+
+            # otherwise just reconnect and let it retry
+            logger.warn("failed to unlock prefetched jobs - connection terminated; skipping for now")
+            Delayed::Job.clear_all_connections!
           end
         end
 
-        def unlock_all_prefetched_jobs
-          @prefetched_jobs.each do |(_worker_config, jobs)|
-            next if jobs.empty?
+        def unlock_timed_out_prefetched_jobs
+          unlock_prefetched_jobs do |jobs|
+            jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
+          end
+        end
 
-            Delayed::Job.transaction do
-              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
-              Delayed::Job.unlock(jobs)
-            end
+        def unlock_all_prefetched_jobs
+          # we try really hard; it may not have done any work if it timed out
+          10.times do
+            unlock_prefetched_jobs
+            break if @prefetched_jobs.each_value.all?(&:empty?)
           end
-          @prefetched_jobs = {}
         end
 
         def drop_socket(socket)

data/spec/active_record_job_spec.rb

@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
       expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
     end
 
-    it "doesn't  allow a second worker to get exclusive access if failed to be processed by worker1 and
-        run_at time is now in future (due to backing off behaviour)" do
+    it "doesn't allow a second worker to get exclusive access if failed to be " \
+       "processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
       @job.update(attempts: 1, run_at: 1.day.from_now)
       expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
     end

data/spec/delayed/daemon_spec.rb

@@ -3,9 +3,10 @@
 require "spec_helper"
 
 RSpec.describe Delayed::Daemon do
+  subject { described_class.new(pid_folder) }
+
   let(:pid_folder) { "/test/pid/folder" }
   let(:pid) { 9999 }
-  let(:subject) { described_class.new(pid_folder) }
 
   before do
     allow(subject).to receive(:pid).and_return(pid)

data/spec/delayed/work_queue/in_process_spec.rb

@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
     Delayed::Worker.lifecycle.reset!
   end
 
-  let(:subject) { described_class.new }
   let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
   let(:args) { ["worker_name", worker_config] }
 

data/spec/delayed/work_queue/parent_process/client_spec.rb

@@ -3,7 +3,8 @@
 require "spec_helper"
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
-  let(:subject) { described_class.new(addrinfo).tap(&:init) }
+  subject { described_class.new(addrinfo).tap(&:init) }
+
   let(:addrinfo) { double("Addrinfo") }
   let(:connection) { double("Socket") }
   let(:job) { Delayed::Job.new(locked_by: "worker_name") }

data/spec/delayed/work_queue/parent_process/server_spec.rb

@@ -15,14 +15,19 @@ class JobClass
 end
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
+  subject { described_class.new(listen_socket) }
+
   let(:parent) { Delayed::WorkQueue::ParentProcess.new }
-  let(:subject) { described_class.new(listen_socket) }
   let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
   let(:job) { JobClass.new }
   let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
   let(:args) { ["worker_name", worker_config] }
   let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(prefetch: 4)] }
 
+  before do
+    Delayed::Worker.lifecycle.reset!
+  end
+
   before :all do
     Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
     Delayed::Settings.parent_process = {
@@ -36,6 +41,7 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
 
   after do
     File.unlink("/tmp/inst-jobs-test.sock") if File.exist?("/tmp/inst-jobs-test.sock")
+    Delayed::Worker.lifecycle.reset!
   end
 
   it "accepts new clients" do
@@ -98,6 +104,53 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
     expect(Marshal.load(client)).to eq(job2)
   end
 
+  context "prefetched job unlocking" do
+    let(:job_args) do
+      [["worker_name1"], "queue_name", 1, 2,
+       { prefetch: 4, prefetch_owner: "prefetch:work_queue:X", forced_latency: 6.0 }]
+    end
+    let(:job2) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
+    let(:job3) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
+
+    before do
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+
+      jobs = { "worker_name1" => job, "prefetch:work_queue:X" => [job2, job3] }
+      allow(subject).to receive(:prefetch_owner).and_return("prefetch:work_queue:X")
+      allow(Delayed::Job).to receive(:get_and_lock_next_available).once.with(*job_args).and_return(jobs)
+      Marshal.dump(["worker_name1", worker_config], client)
+      subject.run_once
+    end
+
+    it "doesn't unlock anything if nothing is timed out" do
+      expect(Delayed::Job).not_to receive(:advisory_lock)
+      expect(Delayed::Job).not_to receive(:unlock)
+      subject.unlock_timed_out_prefetched_jobs
+    end
+
+    it "unlocks timed out prefetched jobs" do
+      allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
+      expect(Delayed::Job).to receive(:unlock).with([job2, job3])
+      subject.unlock_timed_out_prefetched_jobs
+      expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
+    end
+
+    it "fails gracefully if the lock times out" do
+      allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
+      expect(Delayed::Job).not_to receive(:unlock)
+      expect(Delayed::Job).to receive(:advisory_lock).and_raise(ActiveRecord::QueryCanceled)
+      subject.unlock_timed_out_prefetched_jobs
+      expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 2
+    end
+
+    it "unlocks all jobs" do
+      expect(Delayed::Job).to receive(:unlock).with([job2, job3])
+      subject.unlock_all_prefetched_jobs
+      expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
+    end
+  end
+
   it "doesn't respond immediately if there are no jobs available" do
     client = Socket.unix(subject.listen_socket.local_address.unix_path)
     subject.run_once
@@ -204,4 +257,24 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
     expect(Marshal.load(client)).to eq(job)
     expect(called).to eq(true)
   end
+
+  it "deletes the correct worker when transferring jobs" do
+    client1 = Socket.unix(subject.listen_socket.local_address.unix_path)
+    client2 = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+    subject.run_once
+
+    Marshal.dump(args, client1)
+    Marshal.dump(["worker_name2", worker_config], client2)
+    subject.run_once
+    subject.run_once
+
+    waiting_clients = subject.instance_variable_get(:@waiting_clients)
+    expect(waiting_clients.first.last.length).to eq 2
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).and_return("worker_name" => job,
+                                                                             "worker_name2" => job)
+    subject.run_once
+    expect(waiting_clients.first.last).to be_empty
+  end
 end

data/spec/delayed/work_queue/parent_process_spec.rb

@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
     Delayed::Worker.lifecycle.reset!
   end
 
-  let(:subject) { described_class.new }
-
   describe "#initalize(config = Settings.parent_process)" do
     it "must expand a relative path to be within the Rails root" do
       queue = described_class.new("server_address" => "tmp/foo.sock")

data/spec/shared/shared_backend.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "timeout"
+
 module InDelayedJobTest
   def self.check_in_job
     Delayed::Job.in_delayed_job?.should == true
@@ -960,6 +962,21 @@ shared_examples_for "a backend" do
     end
   end
 
+  it "removes an un-reschedulable job" do
+    change_setting(Delayed::Settings, :max_attempts, -1) do
+      job = Delayed::Job.new(tag: "tag")
+      `echo ''`
+      child_pid = $?.pid
+      job.create_and_lock!("Jobworker:#{child_pid}")
+      Timeout.timeout(1) do
+        # if this takes longer than a second it's hung
+        # in an infinite loop, which would be bad.
+        expect(Delayed::Job.unlock_orphaned_jobs(nil, "Jobworker")).to eq(1)
+      end
+      expect { Delayed::Job.find(job.id) }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+
   it "unlocks orphaned jobs given a pid" do
     change_setting(Delayed::Settings, :max_attempts, 2) do
       job1 = Delayed::Job.new(tag: "tag")

data/spec/spec_helper.rb

@@ -58,11 +58,7 @@ connection_config = {
 }
 
 def migrate(file)
-  if ::Rails.version >= "6"
-    ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
-  else
-    ActiveRecord::MigrationContext.new(file).migrate
-  end
+  ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
 end
 
 # create the test db if it does not exist, to help out wwtd
@@ -73,6 +69,11 @@ rescue ActiveRecord::StatementInvalid
   nil
 end
 ActiveRecord::Base.establish_connection(connection_config)
+
+# we need to ensure this callback is called for activerecord-pg-extensions,
+# which isn't running because we're not using Rails to setup the database
+ActiveRecord::PGExtensions::Railtie.run_initializers
+
 # TODO: reset db and migrate again, to test migrations
 
 migrate("db/migrate")

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 2.4.8
+  version: 3.0.1
 platform: ruby
 authors:
-- Tobias Luetke
-- Brian Palmer
-autorequire:
+- Cody Cutrer
+- Ethan Vizitei
+- Jacob Burroughs
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-21 00:00:00.000000000 Z
+date: 2021-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -17,28 +18,42 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-pg-extensions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: after_transaction_commit
   requirement: !ruby/object:Gem::Requirement
@@ -93,14 +108,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -395,9 +410,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
-description:
+description: 
 email:
-- brianp@instructure.com
+- cody@instructure.com
+- evizitei@instructure.com
+- jburroughs@instructure.com
 executables:
 - inst_jobs
 extensions: []
@@ -432,6 +449,8 @@ files:
 - db/migrate/20210809145804_add_n_strand_index.rb
 - db/migrate/20210812210128_add_singleton_column.rb
 - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
+- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
+- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
@@ -497,7 +516,7 @@ files:
 homepage: https://github.com/instructure/inst-jobs
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -512,8 +531,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.24
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files: