inst-jobs 3.0.1 → 3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9d86b7b6161b9397c885a82d687f85d84e7a17db5261bfc5ef1bb7e9f25e2ac
-  data.tar.gz: 4a75f8ac73b5f0a0b19d8073bcbfe87f654bbbdb90e147ddf02066916653d4e0
+  metadata.gz: 55850c386085b90bbb9df7ff954d40ed7f2035268860ec8b47dfc11f11ec7e51
+  data.tar.gz: 53065a1f09e8cd30271bc9b57ab58d3bcfff8cf1b9590568379427093de9cbe9
 SHA512:
-  metadata.gz: cd3f9df169146f34f7366e02830575bf27e2fd33e97d190bed34dd8be44e34431261111eef133368cd342668cabeb529af6e16c8985fcf15c99807ea7a07fbab
-  data.tar.gz: 3924f7b2f6f4eee37c4d92bfedfd683304dfd0c035a65f47ea9dae607c0e81b03c35860420f39a7dadf890d203c85fe24de68c90c80a48839f23b6193311b3e3
+  metadata.gz: 8f7d579cddb80890c6158a1384f6828ae91272a864308528ad84e6dc47b3e4769062f91de7ebf9641f4b766c21c571a419385ee57bec9b6e5531f39be30a50a8
+  data.tar.gz: 887648362dec636b3e1f6938e66dd25b97f42a3e606c5d8466768d5b2ed5137ea50b67c68ce9ec817df86f117ef0660f6ddd3456c86e4871965f89704cc161f3

data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonIndex < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -309,7 +309,7 @@ module Delayed
           if Settings.silence_periodic_log
             ::ActiveRecord::Base.logger.silence(&block)
           else
-            block.call
+            yield
           end
         end
 
@@ -548,7 +548,7 @@ module Delayed
 
         def fail!
           attrs = attributes
-          attrs["original_job_id"] = attrs.delete("id")
+          attrs["original_job_id"] = attrs.delete("id") if Failed.columns_hash.key?("original_job_id")
           attrs["failed_at"] ||= self.class.db_time_now
           attrs.delete("next_in_strand")
           attrs.delete("max_concurrent")

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "3.0.1"
+  VERSION = "3.0.2"
 end

data/lib/delayed/worker/process_helper.rb

@@ -5,10 +5,10 @@ module Delayed
     module ProcessHelper
       STAT_LINUX = "stat --format=%%Y /proc/$WORKER_PID"
       STAT_MAC = "ps -o lstart -p $WORKER_PID"
-      STAT = RUBY_PLATFORM =~ /darwin/ ? STAT_MAC : STAT_LINUX
+      STAT = RUBY_PLATFORM.include?("darwin") ? STAT_MAC : STAT_LINUX
       ALIVE_CHECK_LINUX = '[ -d "/proc/$WORKER_PID" ]'
       ALIVE_CHECK_MAC = "ps -p $WORKER_PID > /dev/null"
-      ALIVE_CHECK = RUBY_PLATFORM =~ /darwin/ ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
+      ALIVE_CHECK = RUBY_PLATFORM.include?("darwin") ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
       SCRIPT_TEMPLATE = <<-BASH
         WORKER_PID="%<pid>d" # an example, filled from ruby when the check is created
         ORIGINAL_MTIME="%<mtime>s" # an example, filled from ruby when the check is created
@@ -29,7 +29,7 @@ module Delayed
       BASH
 
       def self.mtime(pid)
-        if RUBY_PLATFORM =~ /darwin/
+        if RUBY_PLATFORM.include?("darwin")
           `ps -o lstart -p #{pid}`.sub(/\n$/, "").presence
         else
           File::Stat.new("/proc/#{pid}").mtime.to_i.to_s rescue nil

data/spec/active_record_job_spec.rb

@@ -219,14 +219,14 @@ describe "Delayed::Backed::ActiveRecord::Job" do
   end
 
   it "gets process ids from locked_by" do
-    3.times.map { Delayed::Job.create payload_object: SimpleJob.new }
+    Array.new(3) { Delayed::Job.create payload_object: SimpleJob.new }
     Delayed::Job.get_and_lock_next_available(["job42:2", "job42:9001"])
     expect(Delayed::Job.processes_locked_locally(name: "job42").sort).to eq [2, 9001]
     expect(Delayed::Job.processes_locked_locally(name: "jobnotme")).to be_empty
   end
 
   it "allows fetching multiple jobs at once" do
-    jobs = 3.times.map { Delayed::Job.create payload_object: SimpleJob.new }
+    jobs = Array.new(3) { Delayed::Job.create payload_object: SimpleJob.new }
     locked_jobs = Delayed::Job.get_and_lock_next_available(%w[worker1 worker2])
     expect(locked_jobs.length).to eq(2)
     expect(locked_jobs.keys).to eq(%w[worker1 worker2])
@@ -235,7 +235,7 @@ describe "Delayed::Backed::ActiveRecord::Job" do
   end
 
   it "allows fetching extra jobs" do
-    jobs = 5.times.map { Delayed::Job.create payload_object: SimpleJob.new }
+    jobs = Array.new(5) { Delayed::Job.create payload_object: SimpleJob.new }
     locked_jobs = Delayed::Job.get_and_lock_next_available(["worker1"],
                                                            prefetch: 2,
                                                            prefetch_owner: "work_queue")

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.2
 platform: ruby
 authors:
 - Cody Cutrer
 - Ethan Vizitei
 - Jacob Burroughs
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2021-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -298,6 +298,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
@@ -410,7 +424,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
-description: 
+description:
 email:
 - cody@instructure.com
 - evizitei@instructure.com
@@ -451,6 +465,7 @@ files:
 - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
 - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
 - db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
+- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
@@ -516,7 +531,7 @@ files:
 homepage: https://github.com/instructure/inst-jobs
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -531,8 +546,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.24
+signing_key:
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files: