riverqueue 0.12.0 → 0.14.0

data/lib/client_runtime.rb

@@ -6,6 +6,21 @@ module River
   class ClientRuntime
     class Interrupted < StandardError; end
 
+    # Keep wall time for queue latency and error timestamps, and monotonic time
+    # for execution and completion durations. Externally executed jobs start
+    # with no locally observed execution time. Snapshot queue latency before
+    # retries, snoozes, or application code change the job's schedule.
+    class JobTiming
+      attr_accessor :finished_monotonic
+      attr_reader :queue_wait_duration, :started_at, :started_monotonic
+
+      def initialize(started_at, started_monotonic, scheduled_at)
+        @queue_wait_duration = started_at - scheduled_at
+        @started_at = started_at
+        @finished_monotonic = @started_monotonic = started_monotonic
+      end
+    end
+
     attr_reader :periodic_jobs
 
     def initialize(client, driver, config)
@@ -13,27 +28,54 @@ module River
       @condition = ConditionVariable.new
       @config = config
       @driver = driver
+      @fetching_queues = {}
+      @leader = false
+      @maintenance_condition = ConditionVariable.new
+      @maintenance_generation = 0
       @mutex = Mutex.new
-      @periodic_jobs = PeriodicJobBundle.new(config.periodic_jobs, wake: method(:wake))
+      @notifications_disabled = false
       @producer_threads = {}
       @queue_configs = config.queues.dup
+      @queue_generations = Hash.new(0)
+      @queue_paused = {}
       @removed_queues = {}
+      @resign_requested = false
       @running = {}
       @started = false
       @stopped = true
       @subscriptions = []
       @threads = []
+      @periodic_jobs = PeriodicJobBundle.new(config.periodic_jobs,
+        enabled: !config.leader_election_disabled, wake: method(:periodic_jobs_changed))
     end
 
     def finish_claimed(row, error = nil)
-      started_at = Time.now.utc
+      timing = JobTiming.new(row.attempted_at || Time.now.utc, monotonic_now, row.scheduled_at)
       job = Job.new(@client, row)
-      if error
-        finish_failed(row, job, error, started_at)
-      else
+      case error
+      when nil
         now = Time.now.utc
-        completed = @driver.job_set_state_if_running(id: row.id, finalized_at: now, now: now, state: JOB_STATE_COMPLETED)
-        publish(EVENT_JOB_COMPLETED, completed, started_at) if completed
+        completed = @driver.job_complete(id: row.id, finalized_at: now, now: now)
+        case completed
+        when :cancelled
+          finish_failed(row, job, JobCancelError.new, timing, cancelled: true)
+        else
+          publish(EVENT_JOB_COMPLETED, completed, timing) if completed
+        end
+      when JobCancelError
+        finish_failed(row, job, error, timing, cancelled: true)
+      when JobSnoozeError
+        finish_snoozed(row, job, error, timing)
+      when Interrupted
+        finish_interrupted(row, job, timing)
+      else
+        worker = begin
+          resolve_worker(row.kind) if !row.__decode_error && @config.workers.include?(row.kind)
+        rescue => worker_error
+          @config.logger.error("River worker initialization failed during finalization: #{worker_error.full_message}")
+          nil
+        end
+        finish_failed(row, job, error, timing, worker: worker)
       end
     end
 
@@ -64,12 +106,21 @@ module River
 
     def publish_queue(kind, queue)
       event = Event.new(kind, nil, queue, nil)
-      @mutex.synchronize { @subscriptions.dup }.each { |subscription| subscription.publish(event) }
+      subscriptions = @mutex.synchronize do
+        paused = kind == EVENT_QUEUE_PAUSED
+        return if @queue_paused[queue.name] == paused
+
+        @queue_paused[queue.name] = paused
+        @subscriptions.dup
+      end
+      subscriptions.each { |subscription| subscription.publish(event) }
     end
 
     def healthy?
       @mutex.synchronize do
-        @stop_requested || (@producer_threads.all? { |name, thread| @removed_queues[name] || thread.alive? } && (!@maintenance_thread || @maintenance_thread.alive?))
+        @stop_requested || (@producer_threads.all? { |name, thread| @removed_queues[name] || thread.alive? } &&
+          (!@maintenance_thread || @maintenance_thread.alive?) &&
+          (!@notification_thread || @notifications_disabled || @notification_thread.alive?))
       end
     end
 
@@ -115,6 +166,9 @@ module River
       name = name.to_s
       producer = @mutex.synchronize do
         raise NotFoundError, "queue is not configured: #{name}" unless @queue_configs.key?(name)
+        if @producer_threads[name] == Thread.current || @running.any? { |_id, entry| entry[:queue] == name && entry[:thread] == Thread.current }
+          raise ThreadError, "cannot remove a queue from its own worker or producer"
+        end
 
         @removed_queues[name] = true
         @condition.broadcast
@@ -146,9 +200,10 @@ module River
       end
 
       begin
+        start_notifications
         queues = @mutex.synchronize { @queue_configs.dup }
         queues.each { |name, queue_config| start_producer(name, queue_config) }
-        start_maintenance unless @queue_configs.empty?
+        start_maintenance unless @queue_configs.empty? && @periodic_jobs.empty? && @config.maintenance_services.empty?
 
         self
       rescue
@@ -164,9 +219,13 @@ module River
     def stop(cancel: false, wait: true)
       threads = @mutex.synchronize do
         return self if @stopped
+        if wait && (@threads.include?(Thread.current) || @running.any? { |_id, entry| entry[:thread] == Thread.current })
+          raise ThreadError, "cannot wait for client stop from its own runtime thread; use wait: false"
+        end
 
         @stop_requested = true
         @condition.broadcast
+        @maintenance_condition.broadcast
         @running.values.each { |entry| entry[:thread].raise(Interrupted) if cancel && entry[:working] }
         @threads.dup
       end
@@ -185,6 +244,9 @@ module River
         @threads.clear
         @producer_threads.clear
         @maintenance_thread = nil
+        @notification_thread = nil
+        @leader = false
+        @resign_requested = false
       end
 
       self
@@ -205,21 +267,23 @@ module River
     end
 
     def wake
-      @mutex.synchronize { @condition.broadcast }
+      wake_queue("*")
     end
 
     private def begin_work(id)
-      should_interrupt = @mutex.synchronize do
+      interruption = @mutex.synchronize do
         entry = @running.fetch(id)
         if @stop_requested
-          true
+          Interrupted
+        elsif entry[:cancelled]
+          JobCancelError
         else
           entry[:working] = true
-          false
+          nil
         end
       end
 
-      raise Interrupted if should_interrupt
+      raise interruption if interruption
     end
 
     private def check_remote_cancellations(queue)
@@ -246,79 +310,74 @@ module River
     end
 
     private def execute(row)
-      started_at = Time.now.utc
+      timing = JobTiming.new(Time.now.utc, monotonic_now, row.scheduled_at)
       job = Job.new(@client, row)
+      worker = nil #: untyped
       begin
-        raise row.__decode_error if row.__decode_error
-
-        worker = resolve_worker(row.kind)
-        begin_work(row.id)
         begin
+          raise row.__decode_error if row.__decode_error
+
+          worker = resolve_worker(row.kind)
+          begin_work(row.id)
           Thread.handle_interrupt(Interrupted => :immediate, JobCancelError => :immediate) do
             invoke_worker(worker, job)
           end
         ensure
+          timing.finished_monotonic = monotonic_now
           finish_work(row.id)
         end
 
+        return [:completed, nil] if finish_transactional_completion(job, timing)
+
         finalize_hooks = @config.plugins.any? { |plugin| plugin.respond_to?(:job_finalize) }
         if finalize_hooks
           raise JobCancelError if @driver.job_get_cancelled_ids([row.id]).include?(row.id)
 
           if invoke_plugins(:job_finalize, job, JOB_STATE_COMPLETED).include?(:delete)
-            @driver.job_delete_if_running(row.id)
+            raise JobCancelError if @driver.job_delete_if_running(row.id) == :cancelled
+
             return [:deleted, nil]
           end
         end
 
         completed_at = Time.now.utc
-        completed = if finalize_hooks
-          @driver.job_set_state_if_running(id: row.id, finalized_at: completed_at, metadata: job.metadata_updates, now: completed_at, state: JOB_STATE_COMPLETED)
-        else
-          result = @driver.job_complete(id: row.id, finalized_at: completed_at, metadata: job.metadata_updates, now: completed_at)
-          case result
-          when :cancelled then raise JobCancelError
-          else result
-          end
+        completed = @driver.job_complete(id: row.id, finalized_at: completed_at, metadata: job.metadata_updates, now: completed_at)
+        case completed
+        when :cancelled then raise JobCancelError
+        else publish(EVENT_JOB_COMPLETED, completed, timing) if completed
         end
-        publish(EVENT_JOB_COMPLETED, completed, started_at) if completed
 
         [:completed, nil]
-      rescue JobSnoozeError => error
-        job.__capture_resumable_metadata!
-        [finish_snoozed(row, job, error, started_at), error]
-      rescue JobCancelError => error
-        job.__capture_resumable_metadata!
-        finish_failed(row, job, error, started_at, cancelled: true)
-        [:cancelled, error]
-      rescue Interrupted => error
-        job.__capture_resumable_metadata!
-        interrupted = @driver.job_set_state_if_running(
-          id: row.id,
-          attempt: [row.attempt - 1, 0].max,
-          metadata: job.metadata_updates,
-          scheduled_at: Time.now.utc,
-          state: JOB_STATE_AVAILABLE
-        )
-        publish(EVENT_JOB_INTERRUPTED, interrupted, started_at) if interrupted
-
-        [(interrupted&.state == JOB_STATE_CANCELLED) ? :cancelled : :interrupted, error]
       rescue => error
+        return [:completed, nil] if finish_transactional_completion(job, timing)
+
         job.__capture_resumable_metadata!
-        [finish_failed(row, job, error, started_at, worker: worker), error]
+        outcome = case error
+        when JobSnoozeError
+          finish_snoozed(row, job, error, timing)
+        when JobCancelError
+          finish_failed(row, job, error, timing, cancelled: true)
+          :cancelled
+        when Interrupted
+          finish_interrupted(row, job, timing)
+        else
+          finish_failed(row, job, error, timing, worker: worker)
+        end
+        [outcome, error]
       ensure
         @mutex.synchronize do
           @running.delete(row.id)
+          @queue_generations[row.queue] += 1
           @condition.broadcast
         end
       end
     end
 
-    private def finish_failed(row, job, error, started_at, cancelled: false, worker: nil)
+    private def finish_failed(row, job, error, timing, cancelled: false, worker: nil)
       cancelled ||= error_handler_cancel?(error, job)
       now = Time.now.utc
       attempt_error = AttemptError.new(
-        at: started_at,
+        at: timing.started_at,
         attempt: row.attempt,
         error: error.message,
         trace: Array(error.backtrace).join("\n")
@@ -344,7 +403,7 @@ module River
         state: state
       )
       event = cancelled ? EVENT_JOB_CANCELLED : EVENT_JOB_FAILED
-      publish(event, updated, started_at) if updated
+      publish(event, updated, timing) if updated
 
       case updated&.state || state
       when JOB_STATE_CANCELLED then :cancelled
@@ -353,10 +412,23 @@ module River
       end
     end
 
-    private def finish_snoozed(row, job, error, started_at)
+    private def finish_interrupted(row, job, timing)
+      interrupted = @driver.job_set_state_if_running(
+        id: row.id,
+        attempt: [row.attempt - 1, 0].max,
+        metadata: job.metadata_updates,
+        scheduled_at: Time.now.utc,
+        state: JOB_STATE_AVAILABLE
+      )
+      publish(EVENT_JOB_INTERRUPTED, interrupted, timing) if interrupted
+
+      (interrupted&.state == JOB_STATE_CANCELLED) ? :cancelled : :interrupted
+    end
+
+    private def finish_snoozed(row, job, error, timing)
       scheduled_at = Time.now.utc + error.duration
       state = (error.duration <= 5) ? JOB_STATE_AVAILABLE : JOB_STATE_SCHEDULED
-      metadata = job.metadata_updates.merge("snoozes" => row.metadata.fetch("snoozes", 0).to_i + 1)
+      metadata = job.metadata_updates.merge("snoozes" => next_snooze_count(row.metadata["snoozes"]))
       updated = @driver.job_set_state_if_running(
         id: row.id,
         attempt: [row.attempt - 1, 0].max,
@@ -364,10 +436,39 @@ module River
         scheduled_at: scheduled_at,
         state: state
       )
-      publish(EVENT_JOB_SNOOZED, updated, started_at) if updated
+      publish(EVENT_JOB_SNOOZED, updated, timing) if updated
       (updated&.state == JOB_STATE_CANCELLED) ? :cancelled : :snoozed
     end
 
+    private def finish_transactional_completion(job, timing)
+      return false unless job.__completion_attempted
+
+      completed = @driver.job_get_by_id(job.row.id)
+      unless completed
+        snapshot = job.__completion_snapshot
+        return false unless snapshot
+
+        publish(EVENT_JOB_COMPLETED, snapshot, timing)
+        return true
+      end
+
+      # A retry can replace a committed attempt before its worker or middleware
+      # returns. Only a still-running original attempt may have rolled back and
+      # need normal finalization; never finalize or publish a replacement.
+      return true unless completed.attempt == job.row.attempt && completed.attempted_at == job.row.attempted_at
+      return false if completed.state == JOB_STATE_RUNNING
+      return true unless completed.state == JOB_STATE_COMPLETED
+
+      # Work middleware (including persisted logging) can add metadata after the
+      # worker commits. Save it without changing the committed state or timestamp.
+      # Guard the write too: a retry can happen after the read above.
+      metadata = job.metadata_updates
+      completed = @driver.job_metadata_merge(job.row.id, metadata, completed) || completed unless metadata.empty?
+
+      publish(EVENT_JOB_COMPLETED, completed, timing)
+      true
+    end
+
     private def finish_work(id)
       @mutex.synchronize do
         entry = @running[id]
@@ -383,15 +484,22 @@ module River
       return perform_work(worker, job) if @config.plugins.empty?
 
       operation = -> do
+        invoke_plugins(:work_begin, job)
+
         error = nil
-        begin
-          invoke_plugins(:work_begin, job)
+        result = begin
           perform_work(worker, job)
-        rescue => error
-          raise
-        ensure
-          invoke_plugins(:work_end, job, error)
+        rescue => work_error
+          error = work_error
+          nil
+        end
+
+        @config.plugins.each do |plugin|
+          error = plugin.work_end(job, error) if plugin.respond_to?(:work_end)
         end
+        raise error if error
+
+        result
       end
       @config.plugins.reverse_each do |plugin|
         next unless plugin.respond_to?(:work)
@@ -415,26 +523,54 @@ module River
         end
       end
 
-      @mutex.synchronize { @running[row.id] = {queue: row.queue, thread: thread, working: false} }
+      @mutex.synchronize do
+        @running[row.id] = {cancelled: @fetching_queues.fetch(row.queue).include?(row.id), queue: row.queue, thread: thread, working: false}
+      end
       gate.push(true)
     end
 
     private def maintenance_loop
       leader = false
+      next_election = 0.0
       next_schedule = next_rescue = next_cleanup = Time.at(0)
       until stopping?
+        generation, resign = @mutex.synchronize { [@maintenance_generation, @resign_requested] }
+        if resign && leader
+          @driver.leader_release(@config.id)
+          leader = false
+          @mutex.synchronize do
+            @leader = false
+            @resign_requested = false
+          end
+          # Give followers a chance to acquire the term we just released.
+          next_election = monotonic_now + 5
+        end
+        if !leader && (delay = next_election - monotonic_now).positive?
+          wait_for_maintenance(delay, generation)
+          next
+        end
+
         now = Time.now.utc
         leader = leader ? @driver.leader_renew(@config.id, now: now) : @driver.leader_acquire(@config.id, now: now)
+        @mutex.synchronize do
+          @leader = leader
+          @resign_requested = false unless leader
+        end
         if leader
           if now >= next_schedule
             @driver.job_schedule(now: now)
             run_periodic(now)
-            @config.maintenance_services.each { |service| service.run(@client, @driver, now) }
+            @config.maintenance_services.each do |service|
+              service.run(@client, @driver, now)
+            rescue => error
+              @config.logger.error("River maintenance service failed (#{service.class}): #{error.full_message}")
+            end
             next_schedule = now + 5
           end
 
           if now >= next_rescue
-            @driver.job_rescue_stuck(horizon: now - 3_600, now: now, retry_policy: @config.retry_policy)
+            @driver.job_rescue_stuck(horizon: now - 3_600, logger: @config.logger, now: now,
+              rescue_if: method(:rescue_job?), retry_policy: @config.retry_policy)
             next_rescue = now + 30
           end
 
@@ -449,7 +585,7 @@ module River
           end
         end
 
-        wait(5)
+        wait_for_maintenance(5, generation)
       end
     rescue => error
       @config.logger.error("River maintenance stopped: #{error.full_message}")
@@ -474,17 +610,82 @@ module River
       DefaultClientRetryPolicy.new.next_retry(row, error, now: now)
     end
 
-    private def perform_work(worker, job)
-      timeout = worker.respond_to?(:timeout) ? worker.timeout(job) : @config.job_timeout
-      timeout = Float(timeout) unless timeout.nil?
-      timeout = @config.job_timeout if timeout == 0
-      raise ArgumentError, "worker timeout must be finite and nonnegative, or nil" if timeout && (!timeout.finite? || timeout.negative?)
+    # Preserve Ruby's lenient counter conversion without calling to_i on
+    # booleans/collections, or accepting only a prefix of a numeric string.
+    # Only canonical non-negative integers are part of the shared protocol.
+    private def next_snooze_count(value)
+      count = case value
+      when Integer then value
+      when Float then value.finite? ? value.to_i : 0
+      when String then /\A-?[0-9]+\z/.match?(value) ? value.to_i : 0
+      when true then 1
+      else 0
+      end
+      # Go stores and increments a signed 64-bit integer.
+      ((count + 1 + (1 << 63)) % (1 << 64)) - (1 << 63)
+    end
+
+    private def notification_loop(ready)
+      listener = nil #: untyped
+      retry_delay = 0.1
+      begin
+        until notifications_stopping?
+          begin
+            unless listener
+              listener = @driver.notification_listener
+              @mutex.synchronize { @notifications_disabled = listener.nil? }
+              ready << true if ready
+              ready = nil
+              return unless listener
+
+              # Poll durable state after reconnecting: Postgres messages sent
+              # while disconnected cannot be replayed.
+              wake
+              wake_maintenance
+            end
+            listener.poll(0.1).each { |topic, payload| receive_notification(topic, payload) }
+            retry_delay = 0.1
+          rescue => error
+            if ready
+              ready << error
+              ready = nil
+              return
+            end
+            @config.logger.error("River notification receiver failed: #{error.full_message}")
+            # SQLite's cursor survives temporary read errors. Reopening it would
+            # skip requests committed while the database was unavailable.
+            unless listener.is_a?(Driver::NotificationListener::SQLite)
+              listener&.close
+              listener = nil
+            end
+            sleep(retry_delay) unless notifications_stopping?
+            retry_delay = [retry_delay * 2, 1.0].min
+          end
+        end
+      ensure
+        ready << true if ready
+        listener&.close
+      end
+    end
+
+    private def notifications_stopping?
+      @mutex.synchronize do
+        @stop_requested && @running.empty? && @producer_threads.values.none?(&:alive?)
+      end
+    end
 
+    private def perform_work(worker, job)
+      timeout = worker_timeout(worker, job)
       result = timeout ? Timeout.timeout(timeout) { worker.work(job) } : worker.work(job)
       job.__finish_resumable_work!
       result
     end
 
+    private def periodic_jobs_changed
+      start_maintenance
+      wake_maintenance
+    end
+
     private def producer_loop(queue, queue_config)
       cooldown = queue_config.resolved_fetch_cooldown(@config)
       # Capture registrations (including aliases) at startup, not construction.
@@ -493,9 +694,16 @@ module River
       last_fetch = 0.0
       poll_interval = queue_config.resolved_fetch_poll_interval(@config)
       loop do
-        break if queue_stopping?(queue)
+        generation = @mutex.synchronize { @queue_generations[queue] }
+        draining = queue_stopping?(queue)
+        break if draining && running_count(queue).zero?
 
         check_remote_cancellations(queue)
+        if draining
+          wait_for_running_jobs(queue, poll_interval)
+          next
+        end
+
         capacity = queue_config.max_workers - running_count(queue)
         if capacity.positive?
           # Inserts, completions, and spurious condition wakes must not bypass
@@ -503,34 +711,48 @@ module River
           while (sleep_for = cooldown - (monotonic_now - last_fetch)).positive? && !queue_stopping?(queue)
             wait(sleep_for)
           end
-          break if queue_stopping?(queue)
+          next if queue_stopping?(queue)
+        end
 
-          # A pause may arrive during cooldown; only read the queue immediately
-          # before fetching, and avoid the read entirely when all slots are busy.
-          unless @driver.queue_get(queue)&.paused_at
+        # Refresh controls even when every worker slot is occupied. A pause
+        # can also arrive during cooldown, so read immediately before fetching.
+        queue_state = @driver.queue_get(queue)
+        publish_queue(queue_state.paused_at ? EVENT_QUEUE_PAUSED : EVENT_QUEUE_RESUMED, queue_state) if queue_state
+        if capacity.positive? && !queue_state&.paused_at
+          @mutex.synchronize { @fetching_queues[queue] = [] }
+          begin
             jobs = @driver.job_get_available(attempted_by: @config.id, max: capacity, queue: queue, **fetch_options)
             last_fetch = monotonic_now
             jobs.each { |job| launch(job) }
             next unless jobs.empty?
+          ensure
+            @mutex.synchronize { @fetching_queues.delete(queue) }
           end
         end
 
-        wait(poll_interval)
+        wait_for_queue(queue, poll_interval, generation)
       end
     rescue => error
       @config.logger.error("River producer for #{queue.inspect} stopped: #{error.full_message}")
-      wait(poll_interval || @config.fetch_poll_interval)
-      retry unless queue_stopping?(queue)
+      if queue_stopping?(queue)
+        wait_for_running_jobs(queue, poll_interval || @config.fetch_poll_interval)
+      else
+        wait(poll_interval || @config.fetch_poll_interval)
+      end
+      retry unless queue_stopping?(queue) && running_count(queue).zero?
     end
 
-    private def publish(kind, job, started_at)
+    private def publish(kind, job, timing)
       # Another actor may have changed the row after our update. Pending,
       # running, and unknown states do not represent a settled attempt.
       return unless %w[available cancelled completed discarded retryable scheduled].include?(job.state)
 
       kind = EVENT_JOB_CANCELLED if job.state == JOB_STATE_CANCELLED
-      completed_at = Time.now.utc
-      stats = JobStatistics.new(0, started_at - job.scheduled_at, completed_at - started_at)
+      stats = JobStatistics.new(
+        monotonic_now - timing.finished_monotonic,
+        timing.queue_wait_duration,
+        timing.finished_monotonic - timing.started_monotonic
+      )
       event = Event.new(kind, job, nil, stats)
       @mutex.synchronize { @subscriptions.dup }.each { |subscription| subscription.publish(event) }
     end
@@ -539,10 +761,69 @@ module River
       @mutex.synchronize { @stop_requested || @removed_queues[queue] }
     end
 
+    private def receive_notification(topic, payload)
+      message = JSON.parse(payload)
+      return unless message.is_a?(Hash)
+
+      case topic
+      when "river_insert"
+        wake_queue(message["queue"]) if message["queue"].is_a?(String)
+      when "river_control"
+        queue = message["queue"]
+        return unless queue.is_a?(String)
+
+        case message["action"]
+        when "cancel"
+          return unless message["job_id"].is_a?(Integer)
+
+          @mutex.synchronize do
+            entry = @running[message["job_id"]]
+            if entry && entry[:queue] == queue
+              entry[:cancelled] = true
+              entry[:thread].raise(JobCancelError) if entry[:working]
+            elsif (pending = @fetching_queues[queue])
+              # A claim may have committed before its workers are registered.
+              pending << message["job_id"]
+            end
+          end
+        when "metadata_changed", "pause", "resume"
+          wake_queue(queue)
+        end
+      when "river_leadership"
+        case message["action"]
+        when "request_resign"
+          @mutex.synchronize do
+            if @leader
+              @resign_requested = true
+              @maintenance_generation += 1
+              @maintenance_condition.broadcast
+            end
+          end
+        when "resigned"
+          wake_maintenance if message["leader_id"].is_a?(String) && message["leader_id"] != @config.id
+        end
+      end
+    rescue JSON::ParserError, TypeError => error
+      @config.logger.warn("River ignored invalid notification: #{error.message}")
+    end
+
     private def remove_subscription(subscription)
       @mutex.synchronize { @subscriptions.delete(subscription) }
     end
 
+    private def rescue_job?(row, now)
+      return true if row.__decode_error
+
+      worker = @config.workers[row.kind]
+      worker = worker.new if worker.is_a?(Class)
+      timeout = worker_timeout(worker, Job.new(@client, row))
+      attempted_at = row.attempted_at #: Time
+      !timeout.nil? && now - attempted_at >= timeout
+    rescue => error
+      @config.logger.error("River rescue timeout check failed; rescuing attempt: #{error.full_message}")
+      true
+    end
+
     private def resolve_worker(kind)
       worker = @config.workers[kind]
       raise UnknownJobKindError, kind unless worker
@@ -557,15 +838,18 @@ module River
     end
 
     private def run_periodic(now)
-      due = @periodic_jobs.due(now) do |job, error|
+      due = @periodic_jobs.__due_with_times(now) do |job, error|
         @config.logger.error("River periodic job schedule failed (id=#{job.id.inspect}): #{error.full_message}")
       end
-      due.each do |periodic_job|
+      due.each do |periodic_job, scheduled_at|
         value = periodic_job.constructor.call
         next unless value
 
         args, opts = value.is_a?(Array) ? value : [value, nil]
-        @client.insert(args, insert_opts: opts || InsertOpts.new)
+        opts = (opts || InsertOpts.new).dup
+        opts.metadata = (opts.metadata || {}).merge("periodic" => true)
+        opts.metadata["river:periodic_job_id"] = periodic_job.id if periodic_job.id && !periodic_job.id.empty?
+        @client.__insert_with_default_scheduled_at(args, opts, scheduled_at)
       rescue => error
         @config.logger.error("River periodic job failed to insert: #{error.full_message}")
       end
@@ -579,7 +863,7 @@ module River
       return if @config.leader_election_disabled
 
       @mutex.synchronize do
-        return if @stop_requested
+        return unless @started && !@stop_requested
         return if @maintenance_thread&.alive?
 
         thread = Thread.new { maintenance_loop }
@@ -588,11 +872,28 @@ module River
       end
     end
 
+    private def start_notifications
+      return if @config.poll_only || !@driver.respond_to?(:notification_listener)
+
+      ready = ::Queue.new
+      @mutex.synchronize do
+        return if @stop_requested
+
+        thread = Thread.new { notification_loop(ready) }
+        @notification_thread = thread
+        @threads << thread
+      end
+      result = ready.pop
+      raise result if result.is_a?(Exception)
+    end
+
     private def start_producer(name, queue_config)
-      @driver.queue_upsert(name)
+      queue = @driver.queue_upsert(name)
       @mutex.synchronize do
         return if @stop_requested || @removed_queues[name] || @producer_threads.key?(name)
 
+        @queue_paused[name] = !queue.paused_at.nil? if queue
+
         # Shutdown must see every launched thread, including a producer whose
         # database setup overlapped a stop or queue removal.
         thread = Thread.new { producer_loop(name, queue_config) }
@@ -608,5 +909,51 @@ module River
     private def wait(duration)
       @mutex.synchronize { @condition.wait(@mutex, duration) unless @stop_requested }
     end
+
+    private def wait_for_maintenance(duration, generation)
+      @mutex.synchronize do
+        @maintenance_condition.wait(@mutex, duration) if !@stop_requested && @maintenance_generation == generation
+      end
+    end
+
+    private def wait_for_queue(queue, duration, generation)
+      deadline = monotonic_now + duration
+      @mutex.synchronize do
+        while !@stop_requested && !@removed_queues[queue] && @queue_generations[queue] == generation && (remaining = deadline - monotonic_now).positive?
+          @condition.wait(@mutex, remaining)
+        end
+      end
+    end
+
+    private def wait_for_running_jobs(queue, duration)
+      @mutex.synchronize do
+        # Draining still needs a poll interval after stop is requested. Check
+        # under the same mutex as completion so the last job's wake isn't lost.
+        @condition.wait(@mutex, duration) if @running.any? { |_id, entry| entry[:queue] == queue }
+      end
+    end
+
+    private def wake_maintenance
+      @mutex.synchronize do
+        @maintenance_generation += 1
+        @maintenance_condition.broadcast
+      end
+    end
+
+    private def wake_queue(queue)
+      @mutex.synchronize do
+        @queue_configs.each_key { |name| @queue_generations[name] += 1 if queue == "*" || queue == name }
+        @condition.broadcast
+      end
+    end
+
+    private def worker_timeout(worker, job)
+      timeout = worker.respond_to?(:timeout) ? worker.timeout(job) : @config.job_timeout
+      timeout = Float(timeout) unless timeout.nil?
+      timeout = @config.job_timeout if timeout == 0
+      raise ArgumentError, "worker timeout must be finite and nonnegative, or nil" if timeout && (!timeout.finite? || timeout.negative?)
+
+      timeout
+    end
   end
 end