inst-jobs 2.4.11 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 207f92edc1bf044fe040fac47e1eb3eff01d8d11ff66f3a5fa3ff3429ffca748
-  data.tar.gz: 0aa8bb0e7c81fe22eca78171b6827ba84f325e2981cda075691072f4305a355d
+  metadata.gz: 8a9b37f7e43ec441a00056cbacce6f749e93de4496f4cbdff2c765015056ac71
+  data.tar.gz: ccbb0ed96932e436e109db2ae5c3076ce64245bc98a04424f5f9858f045d7a81
 SHA512:
-  metadata.gz: d9278910b958ca2854c7b3d5e10a0b3911ee14938067897abb166929db2888f8577d8507622e372b12a5e5db53fbb09d50d5bdcea591364ca5b40ab7a46d1a27
-  data.tar.gz: 9b4b76b471d442e102a217d16d426dc0eed0e3e2a9bded47cc5d8505806264f189629cea337d084f0300398b96c52f9036a6670aa4b5ef426fff9f56775001a0
+  metadata.gz: 560e60386a4ea4e2851b3c6304ab38982134be7141c6d84671259f744c1d1fbb2dbb7af099ecbe2606fe35364b0d20dd54a6a6dccc6cea95ed02dc56c840d8d7
+  data.tar.gz: 9aceb6dc65083dc71a7f360d6a7e3fafbb8992f17b9a080e53e190d5a7772b417ef7caecd05f3c6f9fe8462e7ea9f67f788e9423d45dd251979547e3329924a0

data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class UpdateConflictingSingletonFunctionToUseIndex < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
+      BEGIN
+        DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton AND locked_by IS NULL;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
+      BEGIN
+        IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
+          DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/exe/inst_jobs

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require File.expand_path("config/environment")
+require_relative "config/environment"
 Delayed::CLI.new.run

data/lib/delayed/backend/active_record.rb

@@ -62,14 +62,9 @@ module Delayed
             _write_attribute(column, current_time) unless attribute_present?(column)
           end
 
-          if Rails.version >= "6"
-            attribute_names = attribute_names_for_partial_writes
-            attribute_names = attributes_for_create(attribute_names)
-            values = attributes_with_values(attribute_names)
-          else
-            attribute_names = partial_writes? ? keys_for_partial_write : self.attribute_names
-            values = attributes_with_values_for_create(attribute_names)
-          end
+          attribute_names = attribute_names_for_partial_writes
+          attribute_names = attributes_for_create(attribute_names)
+          values = attributes_with_values(attribute_names)
 
           im = self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
 

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "2.4.11"
+  VERSION = "3.0.0"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "activerecord-pg-extensions"
+
 module Delayed
   module WorkQueue
     class ParentProcess
@@ -192,29 +194,54 @@ module Delayed
           end
         end
 
-        def unlock_timed_out_prefetched_jobs
+        def unlock_prefetched_jobs
           @prefetched_jobs.each do |(worker_config, jobs)|
             next if jobs.empty?
-            next unless jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
+            next if block_given? && !yield(jobs)
 
-            Delayed::Job.transaction do
+            connection = Delayed::Job.connection
+            connection.transaction do
+              # make absolutely sure we don't get hung up and leave things
+              # locked in the database
+              if connection.postgresql_version >= 9_06_00 # rubocop:disable Style/NumericLiterals
+                connection.idle_in_transaction_session_timeout = 5
+              end
+              # relatively short timeout for acquiring the lock
+              connection.statement_timeout = Settings.sleep_delay
               Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
+
+              # this query might take longer, and we really want to get it
+              # done if we got the lock, but still don't want an inadvertent
+              # hang
+              connection.statement_timeout = 30
               Delayed::Job.unlock(jobs)
+              @prefetched_jobs[worker_config] = []
             end
-            @prefetched_jobs[worker_config] = []
+          rescue ActiveRecord::QueryCanceled
+            # ignore; we'll retry anyway
+            logger.warn("unable to unlock prefetched jobs; skipping for now")
+          rescue ActiveRecord::StatementInvalid
+            # see if we dropped the connection
+            raise if connection.active?
+
+            # otherwise just reconnect and let it retry
+            logger.warn("failed to unlock prefetched jobs - connection terminated; skipping for now")
+            Delayed::Job.clear_all_connections!
           end
         end
 
-        def unlock_all_prefetched_jobs
-          @prefetched_jobs.each do |(_worker_config, jobs)|
-            next if jobs.empty?
+        def unlock_timed_out_prefetched_jobs
+          unlock_prefetched_jobs do |jobs|
+            jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
+          end
+        end
 
-            Delayed::Job.transaction do
-              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
-              Delayed::Job.unlock(jobs)
-            end
+        def unlock_all_prefetched_jobs
+          # we try really hard; it may not have done any work if it timed out
+          10.times do
+            unlock_prefetched_jobs
+            break if @prefetched_jobs.each_value.all?(&:empty?)
           end
-          @prefetched_jobs = {}
         end
 
         def drop_socket(socket)

data/spec/delayed/work_queue/parent_process/server_spec.rb

@@ -104,6 +104,53 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
     expect(Marshal.load(client)).to eq(job2)
   end
 
+  context "prefetched job unlocking" do
+    let(:job_args) do
+      [["worker_name1"], "queue_name", 1, 2,
+       { prefetch: 4, prefetch_owner: "prefetch:work_queue:X", forced_latency: 6.0 }]
+    end
+    let(:job2) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
+    let(:job3) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
+
+    before do
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+
+      jobs = { "worker_name1" => job, "prefetch:work_queue:X" => [job2, job3] }
+      allow(subject).to receive(:prefetch_owner).and_return("prefetch:work_queue:X")
+      allow(Delayed::Job).to receive(:get_and_lock_next_available).once.with(*job_args).and_return(jobs)
+      Marshal.dump(["worker_name1", worker_config], client)
+      subject.run_once
+    end
+
+    it "doesn't unlock anything if nothing is timed out" do
+      expect(Delayed::Job).not_to receive(:advisory_lock)
+      expect(Delayed::Job).not_to receive(:unlock)
+      subject.unlock_timed_out_prefetched_jobs
+    end
+
+    it "unlocks timed out prefetched jobs" do
+      allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
+      expect(Delayed::Job).to receive(:unlock).with([job2, job3])
+      subject.unlock_timed_out_prefetched_jobs
+      expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
+    end
+
+    it "fails gracefully if the lock times out" do
+      allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
+      expect(Delayed::Job).not_to receive(:unlock)
+      expect(Delayed::Job).to receive(:advisory_lock).and_raise(ActiveRecord::QueryCanceled)
+      subject.unlock_timed_out_prefetched_jobs
+      expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 2
+    end
+
+    it "unlocks all jobs" do
+      expect(Delayed::Job).to receive(:unlock).with([job2, job3])
+      subject.unlock_all_prefetched_jobs
+      expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
+    end
+  end
+
   it "doesn't respond immediately if there are no jobs available" do
     client = Socket.unix(subject.listen_socket.local_address.unix_path)
     subject.run_once

data/spec/spec_helper.rb

@@ -58,11 +58,7 @@ connection_config = {
 }
 
 def migrate(file)
-  if ::Rails.version >= "6"
-    ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
-  else
-    ActiveRecord::MigrationContext.new(file).migrate
-  end
+  ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
 end
 
 # create the test db if it does not exist, to help out wwtd
@@ -73,6 +69,11 @@ rescue ActiveRecord::StatementInvalid
   nil
 end
 ActiveRecord::Base.establish_connection(connection_config)
+
+# we need to ensure this callback is called for activerecord-pg-extensions,
+# which isn't running because we're not using Rails to setup the database
+ActiveRecord::PGExtensions::Railtie.run_initializers
+
 # TODO: reset db and migrate again, to test migrations
 
 migrate("db/migrate")

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 2.4.11
+  version: 3.0.0
 platform: ruby
 authors:
 - Tobias Luetke
 - Brian Palmer
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-22 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -17,28 +17,42 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-pg-extensions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: after_transaction_commit
   requirement: !ruby/object:Gem::Requirement
@@ -93,14 +107,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -395,7 +409,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
-description: 
+description:
 email:
 - brianp@instructure.com
 executables:
@@ -433,6 +447,7 @@ files:
 - db/migrate/20210812210128_add_singleton_column.rb
 - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
 - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
+- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
@@ -498,7 +513,7 @@ files:
 homepage: https://github.com/instructure/inst-jobs
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -513,8 +528,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.2.24
+signing_key:
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files: