inst-jobs 3.3.2 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66e746a245b7b6285d8be89085060e790b35b1890f886bce7ccd8fe69aba6123
-  data.tar.gz: 2e537eab47f0c65a4670f3a5230274d20068687a968739d88f01eb5cfb1b0037
+  metadata.gz: b27bc2f28f4906ac69cc6982188b528bdad5897c0bed2a6a5c6445f294eb9973
+  data.tar.gz: cddeff5c83f6ed73d63fee9ae7ffe0df2a4bc18636b11c0fd4cc80e04c8bf835
 SHA512:
-  metadata.gz: daae66931ebd4d17f2c578f2a85ee793fad01cdf048ae43332c5884b2596d800d250c503ca4431d036cc4ff95edd7b885394c8a7745a46ea6861260e1437b98c
-  data.tar.gz: 787c0db4e6dc7e14188dfbe3be2b9603275e89a50372ca93b49d302573cf6cec2e67ff9982e472107dc6b4510d7d631686781839e3f60433c1088d9dfcaec52d
+  metadata.gz: 612d7681460b68f02c11b059b09504544969123ac1f70fbb89203b69f8002a768dae8c281cd498cb610cd316ee962f58d1d8bba3b13680aff94c807931e1c68c
+  data.tar.gz: fc8126210f011236d8ca57e6a938e6e6e33c0e88f79f27cf9e612cb2bae3f743066b6527c728da3b668cf22f27c13dcc0a04764ddd02fe3c3dbc4b1ad1c2c61e

data/db/migrate/20261012154653_add_cancelled_at_to_delayed_jobs.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddCancelledAtToDelayedJobs < ActiveRecord::Migration[7.1]
+  def change
+    add_column :delayed_jobs, :cancelled_at, :datetime
+    add_column :failed_jobs, :cancelled_at, :datetime
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -215,6 +215,12 @@ module Delayed
           where(locked_by: worker_name).update_all(locked_by: nil, locked_at: nil)
         end
 
+        def self.cancel!
+          deleted = where("locked_by IS NULL OR locked_by=?", ON_HOLD_LOCKED_BY).delete_all
+          marked = update_all(cancelled_at: db_time_now)
+          [deleted, marked]
+        end
+
         def self.strand_size(strand)
           where(strand:).count
         end
@@ -613,6 +619,18 @@ module Delayed
           single_step_create
         end
 
+        # Delete unlocked jobs; if the job is locked mark it for the job worker
+        # to cancel within a reasonable amount of time
+        def cancel!
+          if self.class.where(id:).where("locked_by IS NULL OR locked_by=?", ON_HOLD_LOCKED_BY).delete_all == 1
+            @destroyed = true
+            return true
+          end
+
+          update!(cancelled_at: self.class.db_time_now)
+          false
+        end
+
         def fail!
           attrs = attributes
           attrs["original_job_id"] = attrs.delete("id") if Failed.columns_hash.key?("original_job_id")
@@ -646,6 +664,7 @@ module Delayed
                                       "last_error",
                                       "locked_at",
                                       "failed_at",
+                                      "cancelled_at",
                                       "locked_by",
                                       "original_job_id",
                                       "requeued_job_id")

data/lib/delayed/backend/base.rb

@@ -234,6 +234,10 @@ module Delayed
         expires_at && (self.class.db_time_now >= expires_at)
       end
 
+      def cancelled?
+        cancelled_at && (self.class.db_time_now >= cancelled_at)
+      end
+
       def inferred_max_attempts
         max_attempts || Delayed::Settings.max_attempts
       end

data/lib/delayed/settings.rb

@@ -25,6 +25,7 @@ module Delayed
       :silence_periodic_log,
       :sleep_delay,
       :sleep_delay_stagger,
+      :cancellation_check_interval,
       :slow_exit_timeout,
       :stranded_run_at_grace_period,
       :worker_health_check_type,
@@ -130,6 +131,7 @@ module Delayed
     self.max_attempts = 1
     self.sleep_delay = 2.0
     self.sleep_delay_stagger = 2.0
+    self.cancellation_check_interval = 10.0
     self.fetch_batch_size = 5
     self.select_random_from_batch = false
     self.silence_periodic_log = false

data/lib/delayed/testing.rb

@@ -3,7 +3,9 @@
 module Delayed
   module Testing
     def self.run_job(job)
-      Delayed::Worker.new.perform(job)
+      Delayed::Worker.running_job(job) do
+        Delayed::Worker.new.perform(job)
+      end
     end
 
     def self.drain

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "3.3.2"
+  VERSION = "3.5.0"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -24,6 +24,7 @@ module Delayed
 
           @exit = false
           @self_pipe = IO.pipe
+          @last_cancellation_check = Delayed::Job.db_time_now
         end
 
         def connected_clients
@@ -79,6 +80,11 @@ module Delayed
             check_for_work(forced_latency:)
           end
           unlock_timed_out_prefetched_jobs
+
+          return if @last_cancellation_check + Settings.cancellation_check_interval > Delayed::Job.db_time_now
+
+          @last_cancellation_check = Delayed::Job.db_time_now
+          check_for_cancellations
         end
 
         def handle_read(socket)
@@ -97,7 +103,7 @@ module Delayed
         # and terminate the work queue process, to be restarted by the parent daemon.
         def handle_accept
           socket, _addr = @listen_socket.accept_nonblock
-          @clients[socket] = ClientState.new(false, socket) if socket
+          @clients[socket] = ClientState.new(false, nil, socket) if socket
         rescue IO::WaitReadable
           logger.error("Server attempted to read listen_socket but failed with IO::WaitReadable")
           # ignore and just try accepting again next time through the loop
@@ -116,6 +122,7 @@ module Delayed
           worker_name, worker_config = Marshal.load(socket)
           client.name = worker_name
           client.working = false
+          client.active_job_id = nil
           (@waiting_clients[worker_config] ||= []) << client
         rescue SystemCallError, IOError => e
           logger.error("Receiving message from client (#{socket}) failed: #{e.inspect}")
@@ -136,6 +143,7 @@ module Delayed
                 next
               end
               client.working = true
+              client.active_job_id = job.id
               begin
                 logger.debug("Sending prefetched job #{job.id} to #{client.name}")
                 client_timeout { Marshal.dump(job, client.socket) }
@@ -177,6 +185,7 @@ module Delayed
                 end
                 client = workers.find { |worker| worker.name == worker_name }
                 client.working = true
+                client.active_job_id = locked_jobs.id
                 jobs_to_send << [client, locked_jobs]
               end
             end
@@ -245,6 +254,36 @@ module Delayed
           end
         end
 
+        def check_for_cancellations
+          job_map = @clients.values.select(&:active_job_id).index_by(&:active_job_id)
+          prefetched_ids = @prefetched_jobs.values.flatten.map(&:id)
+          return if job_map.empty? && prefetched_ids.empty?
+
+          canceled_jobs = Delayed::Job.where(id: job_map.keys + prefetched_ids)
+                                      .where(cancelled_at: ..Delayed::Job.db_time_now)
+                                      .pluck(:id)
+          return if canceled_jobs.empty?
+
+          canceled_prefetched = canceled_jobs & prefetched_ids
+          unless canceled_prefetched.empty?
+            logger.debug("Deleting cancelled prefetched jobs #{canceled_prefetched.join(", ")}")
+            Delayed::Job.where(id: canceled_prefetched, locked_by: prefetch_owner).delete_all
+            @prefetched_jobs.each_value { |jobs| jobs.reject! { |job| canceled_prefetched.include?(job.id) } }
+          end
+
+          (canceled_jobs & job_map.keys).each do |job_id|
+            client = job_map[job_id]
+            pid = client.name.split(":").last.to_i
+            next unless pid.positive?
+
+            begin
+              Process.kill("USR1", pid)
+            rescue Errno::ESRCH
+              # ignore exited process
+            end
+          end
+        end
+
         def drop_socket(socket)
           # this socket went away
           begin
@@ -275,7 +314,7 @@ module Delayed
           Timeout.timeout(@client_timeout, &)
         end
 
-        ClientState = Struct.new(:working, :socket, :name)
+        ClientState = Struct.new(:working, :active_job_id, :socket, :name)
       end
     end
   end

data/lib/delayed/worker/consul_health_check.rb

@@ -7,11 +7,13 @@ require "socket"
 module Delayed
   class Worker
     class ConsulHealthCheck < HealthCheck
+      include Delayed::Logging
+
       self.type_name = :consul
 
       CONSUL_CONFIG_KEYS = %w[url acl_token].map(&:freeze).freeze
       DEFAULT_SERVICE_NAME = "inst-jobs_worker"
-      attr_reader :service_client, :health_client
+      attr_reader :service_client, :health_client, :check_client
 
       def initialize(*, **)
         super
@@ -27,21 +29,23 @@ module Delayed
           end
           @service_client = Diplomat::Service.new(configuration: consul_config)
           @health_client = Diplomat::Health.new(configuration: consul_config)
+          @check_client = Diplomat::Check.new(configuration: consul_config)
         else
           @service_client = Diplomat::Service.new
           @health_client = Diplomat::Health.new
+          @check_client = Diplomat::Check.new
         end
       end
 
       def start
-        @service_client.register({
-                                   id: worker_name,
-                                   name: service_name,
-                                   check: check_attributes
-                                 })
+        registered = register
+        start_heartbeat if registered && ttl_check?
+        registered
       end
 
       def stop
+        @heartbeat_thread&.kill
+        @heartbeat_thread = nil
         @service_client.deregister(worker_name)
       end
 
@@ -56,13 +60,56 @@ module Delayed
 
       private
 
+      def register
+        @service_client.register({
+                                   id: worker_name,
+                                   name: service_name,
+                                   check: check_attributes
+                                 })
+      end
+
+      def ttl_check?
+        @config.fetch(:check_type, "script").to_s == "ttl"
+      end
+
+      def check_id
+        "service:#{worker_name}"
+      end
+
+      # in seconds
+      def ttl
+        Integer(@config.fetch(:ttl, 60))
+      end
+
+      def start_heartbeat
+        @heartbeat_thread = Thread.new do
+          loop do
+            sleep(ttl / 3.0)
+            heartbeat
+          rescue => e
+            logger.warn("Consul TTL update failed for #{worker_name}: #{e.class}: #{e.message}")
+          end
+        end
+      end
+
+      def heartbeat
+        @check_client.pass(check_id)
+      rescue Faraday::ResourceNotFound
+        logger.warn("Consul forgot #{worker_name}, re-registering")
+        register
+      end
+
       def check_attributes
-        {
-          args: ["bash", "-c", check_script],
+        base = {
           status: "passing",
-          interval: @config.fetch(:check_interval, "5m"),
           deregister_critical_service_after: @config.fetch(:deregister_service_delay, "10m")
-        }.tap do |h|
+        }
+        return base.merge(ttl: "#{ttl}s") if ttl_check?
+
+        base.merge(
+          args: ["bash", "-c", check_script],
+          interval: @config.fetch(:check_interval, "5m")
+        ).tap do |h|
           h[:docker_container_id] = docker_container_id if @config["docker"]
         end
       end

data/lib/delayed/worker.rb

@@ -3,6 +3,11 @@
 module Delayed
   class TimeoutError < RuntimeError; end
 
+  # Use Exception so that user code that rescues regular errors won't accidentally catch this
+  # rubocop:disable Lint/InheritException
+  class CancelledError < Exception; end
+  # rubocop:enable Lint/InheritException
+
   class RetriableError < RuntimeError
     # this error is a special case.  You _should_ raise
     # it from inside the rescue block for another error,
@@ -21,7 +26,7 @@ module Delayed
   class Worker
     include Delayed::Logging
 
-    SIGNALS = %i[INT TERM QUIT].freeze
+    SIGNALS = %i[INT TERM QUIT USR1].freeze
 
     attr_reader :config, :queue_name, :min_priority, :max_priority, :work_queue
 
@@ -69,6 +74,7 @@ module Delayed
       @health_check_config = Settings.worker_health_check_config
       @config = options
       @job_count = 0
+      @active_job_mutex = Mutex.new
 
       @signal_queue = []
 
@@ -129,14 +135,29 @@ module Delayed
             break
           when :QUIT
             @exit = true
+          when :USR1
+            # Use the explicit check in case the job changed since the signal was sent
+            # Use dedicated thread to avoid blocking the signal handler while making DB queries
+            Thread.new { cancel_active_job_if_cancelled(work_thread) }
           else
             logger.error "Unknown signal '#{sig}' received"
           end
         end
       end
 
-      self.class.lifecycle.run_callbacks(:execute, self) do
-        run until exit?
+      if work_queue.is_a?(WorkQueue::InProcess)
+        cancellation_checker = Thread.new do
+          loop do
+            sleep(Settings.cancellation_check_interval)
+            cancel_active_job_if_cancelled(work_thread)
+          end
+        end
+      end
+
+      Thread.handle_interrupt(CancelledError => :never) do
+        self.class.lifecycle.run_callbacks(:execute, self) do
+          run until exit?
+        end
       end
 
       logger.info "Stopping worker"
@@ -151,6 +172,11 @@ module Delayed
         signal_processor.join
       end
 
+      if cancellation_checker
+        cancellation_checker.kill
+        cancellation_checker.join
+      end
+
       @self_pipe&.each(&:close)
       @self_pipe = nil
     end
@@ -180,8 +206,11 @@ module Delayed
         end
 
         if job
+          @active_job_mutex.synchronize { @active_root_job = job }
           configure_for_job(job) do
             @job_count += perform(job)
+            @active_job_mutex.synchronize { @active_root_job = nil }
+            discard_pending_cancellation
 
             if @max_job_count.positive? && @job_count >= @max_job_count
               logger.debug "Max job count of #{@max_job_count} exceeded, dying"
@@ -206,21 +235,26 @@ module Delayed
       end
     end
 
-    def perform(job)
+    def perform(job, in_batch: false)
       begin
         count = 1
         raise Delayed::Backend::JobExpired, "job expired at #{job.expires_at}" if job.expired?
+        raise CancelledError, "job #{job.id} cancelled at #{job.cancelled_at}" if job.cancelled?
 
         self.class.lifecycle.run_callbacks(:perform, self, job) do
           self.process_name = "run:#{Settings.worker_procname_prefix}#{job.id}:#{job.name}"
           logger.info("Processing #{log_job(job, :long)}")
           runtime = Benchmark.realtime do
-            if job.batch?
-              # each job in the batch will have perform called on it, so we don't
-              # need a timeout around this
-              count = perform_batch(job)
-            else
-              job.invoke_job
+            # cancellation is only delivered while the job's own code is running, so it can't
+            # interrupt our bookkeeping (destroying the job, or failing it in a rescue below)
+            Thread.handle_interrupt(CancelledError => :immediate) do
+              if job.batch?
+                # each job in the batch will have perform called on it, so we don't
+                # need a timeout around this
+                count = perform_batch(job)
+              else
+                job.invoke_job
+              end
             end
             job.destroy
           end
@@ -236,6 +270,16 @@ module Delayed
         # There wasn't really a failure here so no callbacks and whatnot needed,
         # still reschedule the job though.
         job.reschedule(e)
+      rescue CancelledError => e
+        logger.debug "Job #{job.id} was cancelled"
+        self.class.lifecycle.run_callbacks(:error, self, job, e) do
+          job.last_error = e.message
+          # Do not call on_failure, as a cancelled job must not be simply unlocked
+          # and thus the behavior switching that method supports must be ignored
+          job.permanent_failure(e)
+        end
+        # cancellation applies to the whole batch, so don't move on to the next job in it
+        raise if in_batch
       rescue Exception => e # rubocop:disable Lint/RescueException
         if e.is_a?(NoMemoryError)
           GC.start # try and free up some memory before reporting the error
@@ -257,12 +301,49 @@ module Delayed
         job.source = parent_job.source
         job.create_and_lock!(name)
         configure_for_job(job) do
-          perform(job)
+          # we're inside the batch's :immediate block; mask again so the sub-job's own bookkeeping
+          # is protected. a cancellation that arrives during it is raised once this block exits,
+          # which cancels the rest of the batch
+          Thread.handle_interrupt(CancelledError => :never) do
+            perform(job, in_batch: true)
+          end
         end
       end
       batch.items.size
     end
 
+    def cancel_active_job_if_cancelled(work_thread)
+      job = @active_root_job
+      return unless job
+
+      # this runs on a background thread; don't hold a connection from the pool between checks
+      cancelled_at = Delayed::Job.connection_pool.with_connection do
+        Delayed::Job.where(id: job.id).pick(:cancelled_at)
+      end
+      return unless cancelled_at && cancelled_at <= Delayed::Job.db_time_now
+
+      @active_job_mutex.synchronize do
+        return unless @active_root_job.equal?(job)
+
+        logger.debug "Active job #{job.id} has been cancelled"
+        work_thread.raise(CancelledError, "job #{job.id} cancelled at #{cancelled_at}")
+      end
+    rescue => e
+      logger.error("Error while checking for job cancellation: #{e.inspect}")
+    end
+
+    # A cancellation that arrives after a job finishes is deferred by handle_interrupt,
+    # and would otherwise be raised as soon as the next job starts.
+    def discard_pending_cancellation
+      while Thread.pending_interrupt?(CancelledError)
+        begin
+          Thread.handle_interrupt(CancelledError => :immediate) { nil }
+        rescue CancelledError => e
+          logger.debug "Discarding cancellation for a job that already finished: #{e.message}"
+        end
+      end
+    end
+
     def handle_failed_job(job, error)
       job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
       logger.error("Failed with #{error.class} [#{error.message}] (#{job.attempts} attempts)")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.5.0
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-10-20 00:00:00.000000000 Z
+date: 2026-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -167,6 +167,7 @@ files:
 - db/migrate/20220203063200_remove_old_singleton_index.rb
 - db/migrate/20220328152900_add_failed_jobs_indicies.rb
 - db/migrate/20220519204546_add_requeued_job_id_to_failed_jobs.rb
+- db/migrate/20261012154653_add_cancelled_at_to_delayed_jobs.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb