riverqueue 0.12.0 → 0.13.0

data/lib/client_runtime.rb

@@ -6,6 +6,21 @@ module River
   class ClientRuntime
     class Interrupted < StandardError; end
 
+    # Keep wall time for queue latency and error timestamps, and monotonic time
+    # for execution and completion durations. Externally executed jobs start
+    # with no locally observed execution time. Snapshot queue latency before
+    # retries, snoozes, or application code change the job's schedule.
+    class JobTiming
+      attr_accessor :finished_monotonic
+      attr_reader :queue_wait_duration, :started_at, :started_monotonic
+
+      def initialize(started_at, started_monotonic, scheduled_at)
+        @queue_wait_duration = started_at - scheduled_at
+        @started_at = started_at
+        @finished_monotonic = @started_monotonic = started_monotonic
+      end
+    end
+
     attr_reader :periodic_jobs
 
     def initialize(client, driver, config)
@@ -14,7 +29,6 @@ module River
       @config = config
       @driver = driver
       @mutex = Mutex.new
-      @periodic_jobs = PeriodicJobBundle.new(config.periodic_jobs, wake: method(:wake))
       @producer_threads = {}
       @queue_configs = config.queues.dup
       @removed_queues = {}
@@ -23,17 +37,37 @@ module River
       @stopped = true
       @subscriptions = []
       @threads = []
+      @periodic_jobs = PeriodicJobBundle.new(config.periodic_jobs,
+        enabled: !config.leader_election_disabled, wake: method(:periodic_jobs_changed))
     end
 
     def finish_claimed(row, error = nil)
-      started_at = Time.now.utc
+      timing = JobTiming.new(row.attempted_at || Time.now.utc, monotonic_now, row.scheduled_at)
       job = Job.new(@client, row)
-      if error
-        finish_failed(row, job, error, started_at)
-      else
+      case error
+      when nil
         now = Time.now.utc
-        completed = @driver.job_set_state_if_running(id: row.id, finalized_at: now, now: now, state: JOB_STATE_COMPLETED)
-        publish(EVENT_JOB_COMPLETED, completed, started_at) if completed
+        completed = @driver.job_complete(id: row.id, finalized_at: now, now: now)
+        case completed
+        when :cancelled
+          finish_failed(row, job, JobCancelError.new, timing, cancelled: true)
+        else
+          publish(EVENT_JOB_COMPLETED, completed, timing) if completed
+        end
+      when JobCancelError
+        finish_failed(row, job, error, timing, cancelled: true)
+      when JobSnoozeError
+        finish_snoozed(row, job, error, timing)
+      when Interrupted
+        finish_interrupted(row, job, timing)
+      else
+        worker = begin
+          resolve_worker(row.kind) if !row.__decode_error && @config.workers.include?(row.kind)
+        rescue => worker_error
+          @config.logger.error("River worker initialization failed during finalization: #{worker_error.full_message}")
+          nil
+        end
+        finish_failed(row, job, error, timing, worker: worker)
       end
     end
 
@@ -115,6 +149,9 @@ module River
       name = name.to_s
       producer = @mutex.synchronize do
         raise NotFoundError, "queue is not configured: #{name}" unless @queue_configs.key?(name)
+        if @producer_threads[name] == Thread.current || @running.any? { |_id, entry| entry[:queue] == name && entry[:thread] == Thread.current }
+          raise ThreadError, "cannot remove a queue from its own worker or producer"
+        end
 
         @removed_queues[name] = true
         @condition.broadcast
@@ -148,7 +185,7 @@ module River
       begin
         queues = @mutex.synchronize { @queue_configs.dup }
         queues.each { |name, queue_config| start_producer(name, queue_config) }
-        start_maintenance unless @queue_configs.empty?
+        start_maintenance unless @queue_configs.empty? && @periodic_jobs.empty? && @config.maintenance_services.empty?
 
         self
       rescue
@@ -164,6 +201,9 @@ module River
     def stop(cancel: false, wait: true)
       threads = @mutex.synchronize do
         return self if @stopped
+        if wait && (@threads.include?(Thread.current) || @running.any? { |_id, entry| entry[:thread] == Thread.current })
+          raise ThreadError, "cannot wait for client stop from its own runtime thread; use wait: false"
+        end
 
         @stop_requested = true
         @condition.broadcast
@@ -246,18 +286,20 @@ module River
     end
 
     private def execute(row)
-      started_at = Time.now.utc
+      timing = JobTiming.new(Time.now.utc, monotonic_now, row.scheduled_at)
       job = Job.new(@client, row)
+      worker = nil #: untyped
       begin
-        raise row.__decode_error if row.__decode_error
-
-        worker = resolve_worker(row.kind)
-        begin_work(row.id)
         begin
+          raise row.__decode_error if row.__decode_error
+
+          worker = resolve_worker(row.kind)
+          begin_work(row.id)
           Thread.handle_interrupt(Interrupted => :immediate, JobCancelError => :immediate) do
             invoke_worker(worker, job)
           end
         ensure
+          timing.finished_monotonic = monotonic_now
           finish_work(row.id)
         end
 
@@ -266,46 +308,33 @@ module River
           raise JobCancelError if @driver.job_get_cancelled_ids([row.id]).include?(row.id)
 
           if invoke_plugins(:job_finalize, job, JOB_STATE_COMPLETED).include?(:delete)
-            @driver.job_delete_if_running(row.id)
+            raise JobCancelError if @driver.job_delete_if_running(row.id) == :cancelled
+
             return [:deleted, nil]
           end
         end
 
         completed_at = Time.now.utc
-        completed = if finalize_hooks
-          @driver.job_set_state_if_running(id: row.id, finalized_at: completed_at, metadata: job.metadata_updates, now: completed_at, state: JOB_STATE_COMPLETED)
-        else
-          result = @driver.job_complete(id: row.id, finalized_at: completed_at, metadata: job.metadata_updates, now: completed_at)
-          case result
-          when :cancelled then raise JobCancelError
-          else result
-          end
+        completed = @driver.job_complete(id: row.id, finalized_at: completed_at, metadata: job.metadata_updates, now: completed_at)
+        case completed
+        when :cancelled then raise JobCancelError
+        else publish(EVENT_JOB_COMPLETED, completed, timing) if completed
         end
-        publish(EVENT_JOB_COMPLETED, completed, started_at) if completed
 
         [:completed, nil]
       rescue JobSnoozeError => error
         job.__capture_resumable_metadata!
-        [finish_snoozed(row, job, error, started_at), error]
+        [finish_snoozed(row, job, error, timing), error]
       rescue JobCancelError => error
         job.__capture_resumable_metadata!
-        finish_failed(row, job, error, started_at, cancelled: true)
+        finish_failed(row, job, error, timing, cancelled: true)
         [:cancelled, error]
       rescue Interrupted => error
         job.__capture_resumable_metadata!
-        interrupted = @driver.job_set_state_if_running(
-          id: row.id,
-          attempt: [row.attempt - 1, 0].max,
-          metadata: job.metadata_updates,
-          scheduled_at: Time.now.utc,
-          state: JOB_STATE_AVAILABLE
-        )
-        publish(EVENT_JOB_INTERRUPTED, interrupted, started_at) if interrupted
-
-        [(interrupted&.state == JOB_STATE_CANCELLED) ? :cancelled : :interrupted, error]
+        [finish_interrupted(row, job, timing), error]
       rescue => error
         job.__capture_resumable_metadata!
-        [finish_failed(row, job, error, started_at, worker: worker), error]
+        [finish_failed(row, job, error, timing, worker: worker), error]
       ensure
         @mutex.synchronize do
           @running.delete(row.id)
@@ -314,11 +343,11 @@ module River
       end
     end
 
-    private def finish_failed(row, job, error, started_at, cancelled: false, worker: nil)
+    private def finish_failed(row, job, error, timing, cancelled: false, worker: nil)
       cancelled ||= error_handler_cancel?(error, job)
       now = Time.now.utc
       attempt_error = AttemptError.new(
-        at: started_at,
+        at: timing.started_at,
         attempt: row.attempt,
         error: error.message,
         trace: Array(error.backtrace).join("\n")
@@ -344,7 +373,7 @@ module River
         state: state
       )
       event = cancelled ? EVENT_JOB_CANCELLED : EVENT_JOB_FAILED
-      publish(event, updated, started_at) if updated
+      publish(event, updated, timing) if updated
 
       case updated&.state || state
       when JOB_STATE_CANCELLED then :cancelled
@@ -353,10 +382,23 @@ module River
       end
     end
 
-    private def finish_snoozed(row, job, error, started_at)
+    private def finish_interrupted(row, job, timing)
+      interrupted = @driver.job_set_state_if_running(
+        id: row.id,
+        attempt: [row.attempt - 1, 0].max,
+        metadata: job.metadata_updates,
+        scheduled_at: Time.now.utc,
+        state: JOB_STATE_AVAILABLE
+      )
+      publish(EVENT_JOB_INTERRUPTED, interrupted, timing) if interrupted
+
+      (interrupted&.state == JOB_STATE_CANCELLED) ? :cancelled : :interrupted
+    end
+
+    private def finish_snoozed(row, job, error, timing)
       scheduled_at = Time.now.utc + error.duration
       state = (error.duration <= 5) ? JOB_STATE_AVAILABLE : JOB_STATE_SCHEDULED
-      metadata = job.metadata_updates.merge("snoozes" => row.metadata.fetch("snoozes", 0).to_i + 1)
+      metadata = job.metadata_updates.merge("snoozes" => next_snooze_count(row.metadata["snoozes"]))
       updated = @driver.job_set_state_if_running(
         id: row.id,
         attempt: [row.attempt - 1, 0].max,
@@ -364,7 +406,7 @@ module River
         scheduled_at: scheduled_at,
         state: state
       )
-      publish(EVENT_JOB_SNOOZED, updated, started_at) if updated
+      publish(EVENT_JOB_SNOOZED, updated, timing) if updated
       (updated&.state == JOB_STATE_CANCELLED) ? :cancelled : :snoozed
     end
 
@@ -429,12 +471,17 @@ module River
           if now >= next_schedule
             @driver.job_schedule(now: now)
             run_periodic(now)
-            @config.maintenance_services.each { |service| service.run(@client, @driver, now) }
+            @config.maintenance_services.each do |service|
+              service.run(@client, @driver, now)
+            rescue => error
+              @config.logger.error("River maintenance service failed (#{service.class}): #{error.full_message}")
+            end
             next_schedule = now + 5
           end
 
           if now >= next_rescue
-            @driver.job_rescue_stuck(horizon: now - 3_600, now: now, retry_policy: @config.retry_policy)
+            @driver.job_rescue_stuck(horizon: now - 3_600, logger: @config.logger, now: now,
+              rescue_if: method(:rescue_job?), retry_policy: @config.retry_policy)
             next_rescue = now + 30
           end
 
@@ -474,17 +521,33 @@ module River
       DefaultClientRetryPolicy.new.next_retry(row, error, now: now)
     end
 
-    private def perform_work(worker, job)
-      timeout = worker.respond_to?(:timeout) ? worker.timeout(job) : @config.job_timeout
-      timeout = Float(timeout) unless timeout.nil?
-      timeout = @config.job_timeout if timeout == 0
-      raise ArgumentError, "worker timeout must be finite and nonnegative, or nil" if timeout && (!timeout.finite? || timeout.negative?)
+    # Preserve Ruby's lenient counter conversion without calling to_i on
+    # booleans/collections, or accepting only a prefix of a numeric string.
+    # Only canonical non-negative integers are part of the shared protocol.
+    private def next_snooze_count(value)
+      count = case value
+      when Integer then value
+      when Float then value.finite? ? value.to_i : 0
+      when String then /\A-?[0-9]+\z/.match?(value) ? value.to_i : 0
+      when true then 1
+      else 0
+      end
+      # Go stores and increments a signed 64-bit integer.
+      ((count + 1 + (1 << 63)) % (1 << 64)) - (1 << 63)
+    end
 
+    private def perform_work(worker, job)
+      timeout = worker_timeout(worker, job)
       result = timeout ? Timeout.timeout(timeout) { worker.work(job) } : worker.work(job)
       job.__finish_resumable_work!
       result
     end
 
+    private def periodic_jobs_changed
+      start_maintenance
+      wake
+    end
+
     private def producer_loop(queue, queue_config)
       cooldown = queue_config.resolved_fetch_cooldown(@config)
       # Capture registrations (including aliases) at startup, not construction.
@@ -493,9 +556,15 @@ module River
       last_fetch = 0.0
       poll_interval = queue_config.resolved_fetch_poll_interval(@config)
       loop do
-        break if queue_stopping?(queue)
+        draining = queue_stopping?(queue)
+        break if draining && running_count(queue).zero?
 
         check_remote_cancellations(queue)
+        if draining
+          wait_for_running_jobs(queue, poll_interval)
+          next
+        end
+
         capacity = queue_config.max_workers - running_count(queue)
         if capacity.positive?
           # Inserts, completions, and spurious condition wakes must not bypass
@@ -503,7 +572,7 @@ module River
           while (sleep_for = cooldown - (monotonic_now - last_fetch)).positive? && !queue_stopping?(queue)
             wait(sleep_for)
           end
-          break if queue_stopping?(queue)
+          next if queue_stopping?(queue)
 
           # A pause may arrive during cooldown; only read the queue immediately
           # before fetching, and avoid the read entirely when all slots are busy.
@@ -519,18 +588,25 @@ module River
       end
     rescue => error
       @config.logger.error("River producer for #{queue.inspect} stopped: #{error.full_message}")
-      wait(poll_interval || @config.fetch_poll_interval)
-      retry unless queue_stopping?(queue)
+      if queue_stopping?(queue)
+        wait_for_running_jobs(queue, poll_interval || @config.fetch_poll_interval)
+      else
+        wait(poll_interval || @config.fetch_poll_interval)
+      end
+      retry unless queue_stopping?(queue) && running_count(queue).zero?
     end
 
-    private def publish(kind, job, started_at)
+    private def publish(kind, job, timing)
       # Another actor may have changed the row after our update. Pending,
       # running, and unknown states do not represent a settled attempt.
       return unless %w[available cancelled completed discarded retryable scheduled].include?(job.state)
 
       kind = EVENT_JOB_CANCELLED if job.state == JOB_STATE_CANCELLED
-      completed_at = Time.now.utc
-      stats = JobStatistics.new(0, started_at - job.scheduled_at, completed_at - started_at)
+      stats = JobStatistics.new(
+        monotonic_now - timing.finished_monotonic,
+        timing.queue_wait_duration,
+        timing.finished_monotonic - timing.started_monotonic
+      )
       event = Event.new(kind, job, nil, stats)
       @mutex.synchronize { @subscriptions.dup }.each { |subscription| subscription.publish(event) }
     end
@@ -543,6 +619,19 @@ module River
       @mutex.synchronize { @subscriptions.delete(subscription) }
     end
 
+    private def rescue_job?(row, now)
+      return true if row.__decode_error
+
+      worker = @config.workers[row.kind]
+      worker = worker.new if worker.is_a?(Class)
+      timeout = worker_timeout(worker, Job.new(@client, row))
+      attempted_at = row.attempted_at #: Time
+      !timeout.nil? && now - attempted_at >= timeout
+    rescue => error
+      @config.logger.error("River rescue timeout check failed; rescuing attempt: #{error.full_message}")
+      true
+    end
+
     private def resolve_worker(kind)
       worker = @config.workers[kind]
       raise UnknownJobKindError, kind unless worker
@@ -565,7 +654,10 @@ module River
         next unless value
 
         args, opts = value.is_a?(Array) ? value : [value, nil]
-        @client.insert(args, insert_opts: opts || InsertOpts.new)
+        opts = (opts || InsertOpts.new).dup
+        opts.metadata = (opts.metadata || {}).merge("periodic" => true)
+        opts.metadata["river:periodic_job_id"] = periodic_job.id if periodic_job.id && !periodic_job.id.empty?
+        @client.insert(args, insert_opts: opts)
       rescue => error
         @config.logger.error("River periodic job failed to insert: #{error.full_message}")
       end
@@ -579,7 +671,7 @@ module River
       return if @config.leader_election_disabled
 
       @mutex.synchronize do
-        return if @stop_requested
+        return unless @started && !@stop_requested
         return if @maintenance_thread&.alive?
 
         thread = Thread.new { maintenance_loop }
@@ -608,5 +700,22 @@ module River
     private def wait(duration)
       @mutex.synchronize { @condition.wait(@mutex, duration) unless @stop_requested }
     end
+
+    private def wait_for_running_jobs(queue, duration)
+      @mutex.synchronize do
+        # Draining still needs a poll interval after stop is requested. Check
+        # under the same mutex as completion so the last job's wake isn't lost.
+        @condition.wait(@mutex, duration) if @running.any? { |_id, entry| entry[:queue] == queue }
+      end
+    end
+
+    private def worker_timeout(worker, job)
+      timeout = worker.respond_to?(:timeout) ? worker.timeout(job) : @config.job_timeout
+      timeout = Float(timeout) unless timeout.nil?
+      timeout = @config.job_timeout if timeout == 0
+      raise ArgumentError, "worker timeout must be finite and nonnegative, or nil" if timeout && (!timeout.finite? || timeout.negative?)
+
+      timeout
+    end
   end
 end