inst-jobs 0.15.5 → 0.15.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec568aafa04aa958061f78e677b5db73d5053d251f18b2cd6ffc87e032c3c656
-  data.tar.gz: e42b99cfa96f1eba59e86589ad05db68b1ae5be18c90caf2baa1ec4f878b5ff1
+  metadata.gz: 221663b9f3e3a49a829dd63a7f25b6eb8f558e3bdd594daf2e66e2adee996723
+  data.tar.gz: 14dd96fbad1a13b259761f4822e9f107537b6d16c92a9f1ceb5db0bb1f003e21
 SHA512:
-  metadata.gz: 4d7d973fac64918cea16eb1a7b21009f39910bb87d2fb848014f6c9cf53effe439a1aa0d13cab62518f0239481e440fc92c7bc3f4246454ecb155e939517f50c
-  data.tar.gz: 565e5474ed3825f3c2f152ed46674c0c346fbeae73c0a5ebec1d6b0041a95973ed07f348d8f8aec8484bb8525e97c5b683df13fb2fa82a8fce72071ed04d0d83
+  metadata.gz: 509766d3ede131621c056be6e95939d077dbdcdec2796217d4d53fdf8cb165d0e5a21a21a75b6bd3457d7de2b804b4786fa1b143377430825ddfd3a24fa2124b
+  data.tar.gz: eb31db5a091c2ff87e3d283a679830a0b77bbf85fc9b592ab988be7ca389e5c84a564d1ce021e7d87db9127d4f9b3885f4a17da24161d7949647e9da9a2dd480

data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb

@@ -0,0 +1,95 @@
+class SpeedUpMaxConcurrentTriggers  < ActiveRecord::Migration[4.2]
+  def connection
+    Delayed::Job.connection
+  end
+
+  def up
+    if connection.adapter_name == 'PostgreSQL'
+      # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
+      # don't get the count every single time - it's usually safe to just set the next one in line
+      # since the max_concurrent doesn't change all that often for a strand
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          IF OLD.id % 20 = 0 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            IF running_count < OLD.max_concurrent THEN
+              UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+              );
+            END IF;
+          ELSE
+            UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+              (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+          END IF;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+
+      # don't need the full count on insert
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        IF NEW.strand IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+          IF (SELECT COUNT(*) FROM (
+              SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
+            ) subquery_for_count) = NEW.max_concurrent THEN
+            NEW.next_in_strand := 'f';
+          END IF;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+    end
+  end
+
+  def down
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
+          IF running_count < OLD.max_concurrent THEN
+            UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+              SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+              j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+            );
+          END IF;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+
+      execute(<<-CODE)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+      BEGIN
+        IF NEW.strand IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+          IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
+            NEW.next_in_strand := 'f';
+          END IF;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+      CODE
+    end
+  end
+end

data/lib/delayed/version.rb

@@ -1,3 +1,3 @@
 module Delayed
-  VERSION = "0.15.5"
+  VERSION = "0.15.6"
 end

data/lib/delayed/worker/health_check.rb

@@ -32,11 +32,15 @@ module Delayed
             # prefetched jobs have their own way of automatically unlocking themselves
             next if job.locked_by.start_with?("prefetch:")
             unless live_workers.include?(job.locked_by)
-              Delayed::Job.transaction do
-                # double check that the job is still there. locked_by will immediately be reset
-                # to nil in this transaction by Job#reschedule
-                next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                job.reschedule
+              begin
+                Delayed::Job.transaction do
+                  # double check that the job is still there. locked_by will immediately be reset
+                  # to nil in this transaction by Job#reschedule
+                  next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
+                  job.reschedule
+                end
+              rescue
+                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
             end
           end

data/spec/active_record_job_spec.rb

@@ -208,40 +208,6 @@ describe 'Delayed::Backed::ActiveRecord::Job' do
           job3.next_in_strand.should == true
         end
       end
-
-      it "should set multiple jobs as next_in_strand at once if needed" do
-        change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
-          case strand_name
-          when "njobs"; 2
-          else nil
-          end
-        }) do
-          job1 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
-          job1.reload
-          job1.next_in_strand.should == true
-          job2 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
-          job2.reload
-          job2.next_in_strand.should == true
-
-          job3 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
-          job3.reload
-          job3.next_in_strand.should == false
-
-          # manually unset next_in_strand
-          Delayed::Job.where(:id => job2).update_all(:next_in_strand => false)
-          job2.reload
-          job2.next_in_strand.should == false
-
-          run_job(job1) # should update both jobs
-
-          job3.reload
-          job3.next_in_strand.should == true
-
-          job2.reload
-          job2.next_in_strand.should == true
-
-        end
-      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 0.15.5
+  version: 0.15.6
 platform: ruby
 authors:
 - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-26 00:00:00.000000000 Z
+date: 2019-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -346,6 +346,7 @@ files:
 - db/migrate/20151123210429_add_expires_at_to_jobs.rb
 - db/migrate/20151210162949_improve_max_concurrent.rb
 - db/migrate/20161206323555_add_back_default_string_limits_jobs.rb
+- db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb