riverqueue 0.12.0 → 0.14.0

data/lib/driver/runtime.rb

@@ -5,12 +5,19 @@ require "time"
 module River::Driver
   # Database operations used by River's worker runtime. Driver gems implement a
   # small set of raw SQL primitives and include this module so the state machine
-  # stays identical across ActiveRecord and Sequel.
+  # stays identical across ActiveRecord and Sequel. Those private primitives
+  # provide execution, quoting, row access, and connection management inside
+  # driver modules; callers use the driver's public operations and Ruby values.
   module Runtime
     def init_driver
       postgres_capabilities if runtime_postgres?
     end
 
+    # Returns the UTC timestamp that writing and reading +time+ preserves.
+    def normalize_time(time)
+      runtime_postgres? ? time.getutc.floor(6) : time.getutc.round(3)
+    end
+
     def postgres_capabilities
       @postgres_capabilities_cache.fetch(runtime_connection_pool) do
         row = runtime_query_rows(<<~SQL).first
@@ -27,7 +34,7 @@ module River::Driver
     end
 
     # Notifications and rows must commit together. The insertion drivers call
-    # this inside their transaction; PostgreSQL defers delivery until commit.
+    # this inside their transaction; Postgres defers delivery until commit.
     private def postgres_notify_insert(params)
       queues = params.select { |param| param.state == River::JOB_STATE_AVAILABLE }.map(&:queue).uniq
       return if queues.empty?
@@ -65,13 +72,26 @@ module River::Driver
       runtime_claim_jobs(predicate, 1, attempted_by, now).first
     end
 
-    # Drivers may combine the cancellation probe and completion atomically.
+    # The cancellation predicate must be part of the completion update so it is
+    # checked against the row we lock, including concurrent cancellations.
     # :cancelled asks the runtime to apply its normal cancellation/error hooks.
-    def job_complete(id:, finalized_at:, metadata: nil, now: Time.now.utc)
+    def job_complete(id:, finalized_at:, expected_job: nil, metadata: nil, now: Time.now.utc)
       id = Integer(id)
-      return :cancelled if job_get_cancelled_ids([id]).include?(id)
+      predicate = "id = #{id}"
+      predicate += " AND #{runtime_job_attempt_predicate(expected_job)}" if expected_job
+      assignments = ["state = 'completed'", "finalized_at = #{runtime_time(finalized_at)}"]
+      assignments << "metadata = #{runtime_merge_metadata(metadata)}" unless metadata.nil? || metadata.empty?
 
-      job_set_state_if_running(id: id, finalized_at: finalized_at, metadata: metadata, now: now, state: "completed")
+      transaction do
+        updated_id = runtime_returning_ids(<<~SQL).first
+          UPDATE river_job SET #{assignments.join(", ")}
+          WHERE #{predicate} AND state = 'running' AND NOT (#{runtime_cancel_attempted})
+          RETURNING id
+        SQL
+        next runtime_read_job(updated_id) if updated_id
+
+        :cancelled if runtime_returning_ids("SELECT id FROM river_job WHERE #{predicate} AND (#{runtime_cancel_attempted})").any?
+      end
     end
 
     def job_delete(id)
@@ -104,13 +124,24 @@ module River::Driver
       SQL
     end
 
+    # A finalization hook's deletion must honor the same cancellation marker as
+    # completion. Return :cancelled so the runtime can record the cancellation.
     def job_delete_if_running(id)
-      runtime_returning_ids("DELETE FROM river_job WHERE id = #{Integer(id)} AND state = 'running' RETURNING id").any?
+      id = Integer(id)
+      deleted = runtime_returning_ids(<<~SQL).any?
+        DELETE FROM river_job
+        WHERE id = #{id} AND state = 'running' AND NOT (#{runtime_cancel_attempted})
+        RETURNING id
+      SQL
+      return true if deleted
+      return :cancelled if job_get_cancelled_ids([id]).include?(id)
+
+      false
     end
 
     def job_delete_many(params)
       transaction do
-        jobs = job_list(params).reject { |job| job.state == River::JOB_STATE_RUNNING }
+        jobs = runtime_job_list(params, for_delete: true)
         next [] if jobs.empty?
 
         ids = jobs.map(&:id)
@@ -142,133 +173,129 @@ module River::Driver
       params ||= River::JobListParams.new
       return runtime_job_list_without_params if params == :all
 
-      clauses = [] #: Array[String]
-      clauses << runtime_cursor_clause(params) if params.after
-      clauses << "id #{(params.sort_order == :asc) ? ">" : "<"} #{Integer(params.after_id)}" if params.after_id
-      clauses << runtime_in_clause("id", params.ids.map { |value| Integer(value) }) if params.ids&.any?
-      clauses << runtime_in_clause("kind", params.kinds) if params.kinds&.any?
-      clauses << runtime_in_clause("priority", params.priorities.map { |value| Integer(value) }) if params.priorities&.any?
-      clauses << runtime_in_clause("queue", params.queues) if params.queues&.any?
-      clauses << runtime_in_clause("state", params.states) if params.states&.any?
-
-      finalized = params.sort_by == :finalized_at && params.states&.length == 1 &&
-        %w[cancelled completed discarded].include?(params.states.first)
-      # Schemas require finalized timestamps for terminal states. Spell this out
-      # so PostgreSQL can use the partial (state, finalized_at) index.
-      clauses << "finalized_at IS NOT NULL" if finalized
-      # Explicit NULLS LAST prevents a backward index scan, even when there are
-      # no nulls. Only request it for timestamps that can actually be null.
-      null_order = (params.sort_by == :finalized_at && !finalized) ? " NULLS LAST" : ""
+      runtime_job_list(params)
+    end
 
-      params.metadata&.each { |key, value| clauses << runtime_metadata_equals(key, value) }
-      Array(params.tags_all).each { |tag| clauses << runtime_tag_contains(tag) }
-      if params.tags_any&.any?
-        clauses << "(" + params.tags_any.map { |tag| runtime_tag_contains(tag) }.join(" OR ") + ")"
+    # When supplied, expected_job prevents metadata from crossing into another
+    # attempt or state. Keep the update and returned snapshot in one transaction.
+    def job_metadata_merge(id, metadata, expected_job = nil)
+      predicate = "id = #{Integer(id)}"
+      if expected_job
+        predicate += " AND #{runtime_job_attempt_predicate(expected_job)}"
+        predicate += " AND state = #{runtime_quote(expected_job.state)}"
       end
 
-      where = clauses.empty? ? "" : "WHERE #{clauses.join(" AND ")}"
-      runtime_job_rows(<<~SQL)
-        #{where}
-        ORDER BY #{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase}
-        LIMIT #{params.limit}
-      SQL
-    end
-
-    def job_metadata_merge(id, metadata)
-      updated_id = runtime_returning_ids(<<~SQL).first
-        UPDATE river_job
-        SET metadata = #{runtime_merge_metadata(metadata)}
-        WHERE id = #{Integer(id)}
-        RETURNING id
-      SQL
-      updated_id ? job_get_by_id(updated_id) : nil
+      transaction do
+        updated_id = runtime_returning_ids(<<~SQL).first
+          UPDATE river_job
+          SET metadata = #{runtime_merge_metadata(metadata)}
+          WHERE #{predicate}
+          RETURNING id
+        SQL
+        updated_id ? job_get_by_id(updated_id) : nil
+      end
     end
 
-    def job_rescue_stuck(horizon:, retry_policy:, now: Time.now.utc, max: 1_000)
+    def job_rescue_stuck(horizon:, retry_policy:, now: Time.now.utc, max: 1_000, logger: nil, rescue_if: nil)
+      max = Integer(max)
       transaction do
-        # Select only stuck jobs before applying the limit, and hold their locks
-        # until rescue finishes so a newer attempt cannot be rescued by mistake.
+        # Lock each candidate until its transition finishes. Continue past jobs
+        # that are still within their timeouts without consuming the rescue limit.
+        after_id = rescued = 0
         lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : ""
-        ids = runtime_returning_ids(<<~SQL)
-          SELECT id FROM river_job
-          WHERE state = 'running' AND attempted_at < #{runtime_time(horizon)}
-          ORDER BY id LIMIT #{Integer(max)} #{lock}
-        SQL
-        jobs = ids.map { |id| runtime_read_job(id) }
-        jobs.each do |job|
-          cancelled = job.metadata.key?("cancel_attempted_at")
-          final = cancelled || job.attempt >= job.max_attempts
-          state = if cancelled
-            River::JOB_STATE_CANCELLED
-          elsif final
-            River::JOB_STATE_DISCARDED
-          else
-            River::JOB_STATE_RETRYABLE
+        while rescued < max
+          ids = runtime_returning_ids(<<~SQL)
+            SELECT id FROM river_job
+            WHERE state = 'running' AND attempted_at < #{runtime_time(horizon)} AND id > #{after_id}
+            ORDER BY id LIMIT #{max - rescued} #{lock}
+          SQL
+          break if ids.empty?
+
+          after_id = ids.last
+          ids.each do |id|
+            job = runtime_read_job(id)
+            cancelled = job.metadata.key?("cancel_attempted_at")
+            next if !cancelled && rescue_if && !rescue_if.call(job, now)
+            final = cancelled || job.attempt >= job.max_attempts
+            state = if cancelled
+              River::JOB_STATE_CANCELLED
+            elsif final
+              River::JOB_STATE_DISCARDED
+            else
+              River::JOB_STATE_RETRYABLE
+            end
+            error = River::AttemptError.new(at: now, attempt: job.attempt, error: "Stuck job rescued by River", trace: "")
+            rescue_count = job.metadata["river:rescue_count"]
+            rescue_count = case rescue_count
+            when Integer then rescue_count
+            when Float then rescue_count.finite? ? rescue_count.to_i : 0
+            else 0
+            end
+            job_set_state_if_running(
+              id: job.id,
+              error: error,
+              finalized_at: final ? now : nil,
+              metadata: {"river:rescue_count" => rescue_count + 1},
+              now: now,
+              scheduled_at: final ? nil : runtime_rescue_retry(job, error, retry_policy, now, logger),
+              state: state
+            )
+            rescued += 1
           end
-          error = River::AttemptError.new(at: now, attempt: job.attempt, error: "Stuck job rescued by River", trace: "")
-          job_set_state_if_running(
-            id: job.id,
-            error: error,
-            finalized_at: final ? now : nil,
-            metadata: {"river:rescue_count" => job.metadata.fetch("river:rescue_count", 0).to_i + 1},
-            now: now,
-            scheduled_at: final ? nil : retry_policy.next_retry(job, error, now: now),
-            state: state
-          )
         end
 
-        jobs.length
+        rescued
       end
     end
 
     def job_retry(id, now: Time.now.utc)
-      updated_id = runtime_returning_ids(<<~SQL).first
-        UPDATE river_job
-        SET state = 'available',
-            max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END,
-            finalized_at = NULL,
-            scheduled_at = #{runtime_time(now)}
-        WHERE id = #{Integer(id)}
-          AND state != 'running'
-          AND (state != 'available' OR scheduled_at > #{runtime_time(now)})
-        RETURNING id
-      SQL
-      job_get_by_id(updated_id || id)
+      updated_id, job = transaction do
+        # Check the counter in the write itself so a concurrent claim cannot
+        # advance it between validation and the update.
+        updated_id = runtime_returning_ids(<<~SQL).first
+          UPDATE river_job
+          SET state = 'available',
+              max_attempts = CASE WHEN attempt = max_attempts THEN max_attempts + 1 ELSE max_attempts END,
+              metadata = #{runtime_postgres? ? "metadata - 'cancel_attempted_at'" : "jsonb_remove(metadata, '$.cancel_attempted_at')"},
+              finalized_at = NULL,
+              scheduled_at = #{runtime_time(now)}
+          WHERE id = #{Integer(id)}
+            AND state != 'running'
+            AND (state != 'available' OR scheduled_at > #{runtime_time(now)})
+            AND attempt < #{River::MAX_ATTEMPTS_LIMIT}
+          RETURNING id
+        SQL
+        job = job_get_by_id(updated_id || id)
+        runtime_notify("river_insert", queue: job.queue) if updated_id && job
+        [updated_id, job]
+      end
+      if !updated_id && job && job.state != "running" && (job.state != "available" || job.scheduled_at > now) && job.attempt >= River::MAX_ATTEMPTS_LIMIT
+        raise ArgumentError, "cannot retry a job with #{River::MAX_ATTEMPTS_LIMIT} or more attempts"
+      end
+      job
     end
 
+    # Makes due scheduled and retryable jobs available. A job is discarded
+    # instead if another row already holds its unique key, or if an earlier
+    # job in the same pass shares that key. The queries find these conflicts
+    # before they write, so the pass never needs a savepoint to recover from a
+    # failed write.
     def job_schedule(now: Time.now.utc, max: 1_000)
       transaction do
-        # Hold each selected row until its transition (including uniqueness
-        # conflict handling) finishes. A concurrent retry may change its due time.
-        lock = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : ""
-        ids = runtime_query_rows(<<~SQL).map { |row| runtime_value(row, :id).to_i }
-          SELECT id FROM river_job
-          WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= #{runtime_time(now)}
-          ORDER BY priority, scheduled_at, id
-          LIMIT #{Integer(max)} #{lock}
-        SQL
-        ids.each do |id|
-          transaction do
-            runtime_execute("UPDATE river_job SET state = 'available' WHERE id = #{id} AND state IN ('retryable', 'scheduled')")
-          end
-        rescue runtime_unique_violation_class
-          runtime_execute(<<~SQL)
-            UPDATE river_job
-            SET state = 'discarded', finalized_at = #{runtime_time(now)},
-                metadata = #{runtime_merge_metadata("unique_key_conflict" => "scheduler_discarded")}
-            WHERE id = #{id}
-          SQL
-        end
-
-        ids.length
+        queues = runtime_postgres? ? runtime_schedule_postgres(now, Integer(max)) : runtime_schedule_sqlite(now, Integer(max))
+        # Notify each changed queue, including queues with only discarded
+        # jobs, as Go does.
+        queues.uniq.each { |queue| runtime_notify("river_insert", queue: queue) }
+        queues.length
       end
     end
 
     def job_set_state_if_running(id:, state:, now: Time.now.utc, attempt: nil,
       error: nil, finalized_at: nil, metadata: nil, scheduled_at: nil)
       state = state.to_s
-      retrying = [River::JOB_STATE_AVAILABLE, River::JOB_STATE_RETRYABLE, River::JOB_STATE_SCHEDULED].include?(state)
-      cancel_path = retrying ? runtime_cancel_attempted : "false"
+      # Cancellation wins over every attempt outcome, including exhausted
+      # retries, direct completion, and transitions requested by extensions.
+      cancel_path = runtime_cancel_attempted
 
       assignments = [] #: Array[String]
       assignments << "attempt = CASE WHEN NOT (#{cancel_path}) THEN #{Integer(attempt)} ELSE attempt END" unless attempt.nil?
@@ -319,7 +346,10 @@ module River::Driver
     end
 
     def leader_release(id)
-      runtime_execute("DELETE FROM river_leader WHERE leader_id = #{runtime_quote(id)}")
+      transaction do
+        deleted = runtime_query_rows("DELETE FROM river_leader WHERE leader_id = #{runtime_quote(id)} RETURNING leader_id")
+        runtime_notify("river_leadership", action: "resigned", leader_id: id) unless deleted.empty?
+      end
     end
 
     def leader_renew(id, ttl: 30, now: Time.now.utc)
@@ -345,6 +375,16 @@ module River::Driver
       SQL
     end
 
+    # Opens a receiver independent of application transactions. Postgres uses
+    # one dedicated connection; SQLite keeps a cursor in its shared outbox.
+    def notification_listener
+      return NotificationListener::SQLite.new(method(:notification_query)) unless runtime_postgres?
+      return unless postgres_capabilities.supports_listen_notify
+
+      connection, schema = runtime_notification_connection
+      NotificationListener::Postgres.new(connection, schema)
+    end
+
     def queue_get(name)
       row = runtime_query_rows("SELECT #{runtime_queue_columns} FROM river_queue WHERE name = #{runtime_quote(name)}").first
       runtime_queue_from_row(row)
@@ -356,34 +396,53 @@ module River::Driver
     end
 
     def queue_pause(name, now: Time.now.utc)
-      filter = (name == "*") ? "true" : "name = #{runtime_quote(name)}"
-      runtime_execute(<<~SQL)
-        UPDATE river_queue
-        SET paused_at = CASE WHEN paused_at IS NULL THEN #{runtime_time(now)} ELSE paused_at END,
-            updated_at = CASE WHEN paused_at IS NULL THEN #{runtime_time(now)} ELSE updated_at END
-        WHERE #{filter}
-      SQL
+      transaction do
+        filter = (name == "*") ? "true" : "name = #{runtime_quote(name)}"
+        changed = runtime_query_rows(<<~SQL)
+          UPDATE river_queue
+          SET paused_at = CASE WHEN paused_at IS NULL THEN #{runtime_time(now)} ELSE paused_at END,
+              updated_at = CASE WHEN paused_at IS NULL THEN #{runtime_time(now)} ELSE updated_at END
+          WHERE #{filter}
+          RETURNING #{runtime_queue_columns}
+        SQL
+        runtime_notify("river_control", action: "pause", queue: name) unless changed.empty?
+        changed.map { |row| runtime_queue_from_row(row) }
+      end
     end
 
     def queue_resume(name, now: Time.now.utc)
-      filter = (name == "*") ? "true" : "name = #{runtime_quote(name)}"
-      runtime_execute(<<~SQL)
-        UPDATE river_queue
-        SET updated_at = CASE WHEN paused_at IS NOT NULL THEN #{runtime_time(now)} ELSE updated_at END,
-            paused_at = NULL
-        WHERE #{filter}
-      SQL
+      transaction do
+        filter = (name == "*") ? "true" : "name = #{runtime_quote(name)}"
+        changed = runtime_query_rows(<<~SQL)
+          UPDATE river_queue
+          SET updated_at = CASE WHEN paused_at IS NOT NULL THEN #{runtime_time(now)} ELSE updated_at END,
+              paused_at = NULL
+          WHERE #{filter}
+          RETURNING #{runtime_queue_columns}
+        SQL
+        runtime_notify("river_control", action: "resume", queue: name) unless changed.empty?
+        changed.map { |row| runtime_queue_from_row(row) }
+      end
     end
 
     def queue_update(name, metadata:, now: Time.now.utc)
-      id = runtime_query_rows(<<~SQL).first
-        UPDATE river_queue SET metadata = #{runtime_json(metadata)}, updated_at = #{runtime_time(now)}
-        WHERE name = #{runtime_quote(name)} RETURNING name
-      SQL
-      id ? queue_get(name) : nil
+      raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash)
+
+      transaction do
+        row = runtime_query_rows(<<~SQL).first
+          UPDATE river_queue SET metadata = #{runtime_json(metadata)}, updated_at = #{runtime_time(now)}
+          WHERE name = #{runtime_quote(name)} RETURNING name
+        SQL
+        if row
+          runtime_notify("river_control", action: "metadata_changed", queue: name, metadata: metadata)
+          queue_get(name)
+        end
+      end
     end
 
     def queue_upsert(name, metadata: {}, now: Time.now.utc)
+      raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash)
+
       runtime_execute(<<~SQL)
         INSERT INTO river_queue (name, created_at, metadata, updated_at)
         VALUES (#{runtime_quote(name)}, #{runtime_time(now)}, #{runtime_json(metadata)}, #{runtime_time(now)})
@@ -392,6 +451,21 @@ module River::Driver
       queue_get(name)
     end
 
+    # Delivery follows the caller's transaction, just like job insertion.
+    def request_resign
+      transaction { runtime_notify("river_leadership", action: "request_resign", leader_id: "") }
+      nil
+    end
+
+    private def notification_query(sql)
+      runtime_query_rows(sql)
+    end
+
+    # Smallest timestamp increment preserved by this driver, in seconds.
+    def time_precision
+      runtime_postgres? ? Rational(1, 1_000_000) : Rational(1, 1_000)
+    end
+
     private def runtime_append_error(error)
       value = error.respond_to?(:to_h) ? error.to_h : error
       if runtime_postgres?
@@ -419,17 +493,27 @@ module River::Driver
           "array_append(CASE WHEN cardinality(attempted_by) >= 100 THEN attempted_by[(cardinality(attempted_by) - 98):] ELSE attempted_by END, #{runtime_quote(attempted_by)})"
         else
           # Preserve corrupt history so decoding can fail this attempt instead
-          # of silently repairing it or aborting the entire claim batch.
+          # of silently repairing it or aborting the entire claim batch. Trim
+          # to the newest 99 entries before appending, preserving JSON types.
           <<~SQL
             CASE WHEN NOT json_valid(attempted_by, 10) AND attempted_by IS NOT NULL THEN attempted_by
-                 ELSE jsonb(json_insert(json(coalesce(attempted_by, jsonb('[]'))), '$[#]', #{runtime_quote(attempted_by)})) END
+                 ELSE jsonb(json_insert(json(
+                   CASE WHEN json_array_length(attempted_by) >= 100 THEN (
+                     SELECT json_group_array(json(value)) FROM (
+                       SELECT attempted_by -> ('$[' || key || ']') AS value FROM json_each(attempted_by)
+                       WHERE key >= json_array_length(attempted_by) - 99 ORDER BY key
+                     )
+                   ) ELSE coalesce(attempted_by, jsonb('[]')) END
+                 ), '$[#]', #{runtime_quote(attempted_by)})) END
           SQL
         end
 
         lock_clause = runtime_postgres? ? "FOR UPDATE SKIP LOCKED" : ""
+        # An administratively requeued job may already be at the counter limit.
+        # Saturate instead of letting that row abort the entire claim batch.
         ids = runtime_returning_ids(<<~SQL)
           UPDATE river_job
-          SET attempt = attempt + 1,
+          SET attempt = CASE WHEN attempt < #{River::MAX_ATTEMPTS_LIMIT} THEN attempt + 1 ELSE attempt END,
               attempted_at = #{runtime_time(now)},
               attempted_by = #{attempted_by_sql},
               state = 'running'
@@ -509,7 +593,7 @@ module River::Driver
       params.map do |param|
         key = param.unique_key
         if key && !key.empty? && River::UniqueBitmask.to_states((param.unique_states || "0").to_i(2)).include?(param.state)
-          # PostgreSQL rejects batches that upsert the same indexed row twice.
+          # Postgres rejects batches that upsert the same indexed row twice.
           raise ArgumentError, "unique key appears more than once in batch" if keys[key]
 
           keys[key] = true
@@ -531,21 +615,77 @@ module River::Driver
       "(#{column} IS NULL OR #{column} #{comparison} #{value} OR (#{column} = #{value} AND #{id_clause}))"
     end
 
+    private def runtime_job_attempt_predicate(job)
+      "attempt = #{Integer(job.attempt)} AND attempted_at IS NOT DISTINCT FROM #{runtime_nullable_time(job.attempted_at)}"
+    end
+
+    private def runtime_job_list(params, for_delete: false)
+      clauses = [] #: Array[String]
+      clauses << "state != 'running'" if for_delete
+      clauses << runtime_cursor_clause(params) if params.after
+      clauses << "id #{(params.sort_order == :asc) ? ">" : "<"} #{Integer(params.after_id)}" if params.after_id
+      clauses << runtime_in_clause("id", params.ids.map { |value| Integer(value) }) if params.ids&.any?
+      clauses << runtime_in_clause("kind", params.kinds) if params.kinds&.any?
+      clauses << runtime_in_clause("priority", params.priorities.map { |value| Integer(value) }) if params.priorities&.any?
+      clauses << runtime_in_clause("queue", params.queues) if params.queues&.any?
+      clauses << runtime_in_clause("state", params.states) if params.states&.any?
+
+      finalized = params.sort_by == :finalized_at && params.states&.length == 1 &&
+        %w[cancelled completed discarded].include?(params.states.first)
+      # Schemas require finalized timestamps for terminal states. Spell this out
+      # so Postgres can use the partial (state, finalized_at) index.
+      clauses << "finalized_at IS NOT NULL" if finalized
+      # Explicit NULLS LAST prevents a backward index scan, even when there are
+      # no nulls. Only request it for timestamps that can actually be null.
+      null_order = (params.sort_by == :finalized_at && !finalized) ? " NULLS LAST" : ""
+
+      params.metadata&.each { |key, value| clauses << runtime_metadata_equals(key, value) }
+      Array(params.tags_all).each { |tag| clauses << runtime_tag_contains(tag) }
+      if params.tags_any&.any?
+        clauses << "(" + params.tags_any.map { |tag| runtime_tag_contains(tag) }.join(" OR ") + ")"
+      end
+
+      where = clauses.empty? ? "" : "WHERE #{clauses.join(" AND ")}"
+      runtime_job_rows(<<~SQL)
+        #{where}
+        ORDER BY #{params.sort_by} #{params.sort_order.to_s.upcase}#{null_order}, id #{params.sort_order.to_s.upcase}
+        LIMIT #{params.limit}
+        #{"FOR UPDATE SKIP LOCKED" if for_delete && runtime_postgres?}
+      SQL
+    end
+
     private def runtime_json(value)
       encoded = value.is_a?(String) ? value : JSON.generate(value)
       runtime_postgres? ? "#{runtime_quote(encoded)}::jsonb" : "jsonb(#{runtime_quote(encoded)})"
     end
 
+    # Temporary fallback: when true, transaction helpers open a savepoint
+    # inside an open transaction instead of joining it.
+    private def runtime_legacy_subtransactions?
+      %w[1 true].include?(ENV.fetch("RIVER_USE_LEGACY_SUBTRANSACTIONS", ""))
+    end
+
     private def runtime_merge_metadata(metadata)
+      raise ArgumentError, "metadata must be a Hash" unless metadata.is_a?(Hash)
+
       if runtime_postgres?
         "metadata || #{runtime_json(metadata)}"
       else
-        # PostgreSQL's || replaces top-level values, including JSON null.
-        # JSON Merge Patch would recursively merge objects and delete nulls.
-        merged = metadata.reduce("metadata") do |expression, (key, value)|
-          path = runtime_quote("$.#{JSON.generate(key.to_s)}")
-          "jsonb_set(#{expression}, #{path}, jsonb(#{runtime_quote(JSON.generate(value))}))"
-        end
+        # Keep the SQL shape fixed regardless of the number of updated keys.
+        # -> preserves JSON booleans and large numeric tokens during aggregation.
+        merged = <<~SQL.strip
+          (WITH updates(doc) AS (SELECT #{runtime_json(metadata)})
+              SELECT CASE WHEN json_type(metadata) = 'object' THEN (
+                  SELECT jsonb_group_object(key, jsonb(doc -> fullkey))
+                  FROM (
+                      SELECT key, fullkey, metadata AS doc FROM json_each(metadata)
+                      WHERE key NOT IN (SELECT key FROM updates, json_each(doc))
+                      UNION ALL
+                      SELECT key, fullkey, doc FROM updates, json_each(doc)
+                  )
+              ) WHEN json_type(metadata) = 'array' THEN jsonb_insert(metadata, '$[#]', doc)
+              ELSE jsonb_array(json(metadata), doc) END FROM updates)
+        SQL
         "CASE WHEN json_valid(metadata, 10) THEN #{merged} ELSE metadata END"
       end
     end
@@ -611,10 +751,129 @@ module River::Driver
       )
     end
 
+    private def runtime_rescue_retry(job, error, retry_policy, now, logger)
+      retry_at = retry_policy.next_retry(job, error, now: now)
+      raise ArgumentError, "next_retry must return a Time" unless retry_at.is_a?(Time)
+      raise ArgumentError, "next_retry must not return a past Time" if retry_at < now
+
+      retry_at
+    rescue => retry_error
+      logger&.error("River rescue retry scheduling failed; using default backoff: #{retry_error.full_message}")
+      River::DefaultClientRetryPolicy.new.next_retry(job, error, now: now)
+    end
+
     private def runtime_returning_ids(sql)
       runtime_query_rows(sql).map { |row| runtime_value(row, :id).to_i }
     end
 
+    # Finds conflicts and changes the whole pass in one statement.
+    private def runtime_schedule_postgres(now, max)
+      now = runtime_time(now)
+      runtime_query_rows(<<~SQL).map { |row| runtime_value(row, :queue) }
+        WITH jobs_to_schedule AS (
+            SELECT id, unique_key, unique_states, priority, scheduled_at
+            FROM river_job
+            WHERE state IN ('retryable', 'scheduled')
+                AND priority >= 0
+                AND queue IS NOT NULL
+                AND scheduled_at <= #{now}
+            ORDER BY priority, scheduled_at, id
+            LIMIT #{max}
+            FOR UPDATE
+        ),
+        jobs_with_rownum AS (
+            SELECT
+                *,
+                CASE
+                    WHEN unique_key IS NOT NULL AND unique_states IS NOT NULL THEN
+                        ROW_NUMBER() OVER (PARTITION BY unique_key ORDER BY priority, scheduled_at, id)
+                END AS row_num
+            FROM jobs_to_schedule
+        ),
+        unique_conflicts AS (
+            SELECT river_job.unique_key
+            FROM river_job
+            JOIN jobs_with_rownum
+                ON river_job.unique_key = jobs_with_rownum.unique_key
+                AND river_job.id != jobs_with_rownum.id
+            WHERE river_job.unique_key IS NOT NULL
+                AND river_job.unique_states IS NOT NULL
+                AND river_job_state_in_bitmask(river_job.unique_states, river_job.state)
+        ),
+        job_updates AS (
+            SELECT
+                job.id,
+                job.row_num IS NOT NULL AND (uc.unique_key IS NOT NULL OR job.row_num > 1) AS discard
+            FROM jobs_with_rownum job
+            LEFT JOIN unique_conflicts uc ON job.unique_key = uc.unique_key
+        )
+        UPDATE river_job
+        SET state = CASE WHEN job_updates.discard THEN 'discarded'::river_job_state ELSE 'available'::river_job_state END,
+            finalized_at = CASE WHEN job_updates.discard THEN #{now} ELSE river_job.finalized_at END,
+            metadata = CASE WHEN job_updates.discard THEN river_job.metadata || '{"unique_key_conflict": "scheduler_discarded"}'::jsonb
+                            ELSE river_job.metadata END
+        FROM job_updates
+        WHERE river_job.id = job_updates.id
+        RETURNING river_job.queue
+      SQL
+    end
+
+    # Follows Go's SQLite driver: check each unique job in order, and make it
+    # available before checking the next one, so a later job with the same
+    # key sees it as a conflict. SQLite allows one writer at a time. Once this
+    # pass writes, other connections cannot write until it ends. If another
+    # connection writes first, this pass's next write fails instead of using
+    # stale rows.
+    private def runtime_schedule_sqlite(now, max)
+      rows = runtime_query_rows(<<~SQL)
+        SELECT id, queue, unique_key IS NOT NULL AS has_unique_key
+        FROM river_job
+        WHERE state IN ('retryable', 'scheduled') AND scheduled_at <= #{runtime_time(now)}
+        ORDER BY priority, scheduled_at, id
+        LIMIT #{max}
+      SQL
+      available = [] #: Array[Integer]
+      discarded = [] #: Array[Integer]
+      rows.each do |row|
+        id = runtime_value(row, :id).to_i
+        next available << id if runtime_value(row, :has_unique_key).to_i.zero?
+
+        conflict = runtime_query_rows(<<~SQL).any?
+          SELECT 1 FROM river_job
+          WHERE id <> #{id}
+            AND unique_key = (SELECT job.unique_key FROM river_job AS job WHERE job.id = #{id})
+            AND unique_states IS NOT NULL
+            AND CASE state
+                  WHEN 'available' THEN unique_states & (1 << 0)
+                  WHEN 'cancelled' THEN unique_states & (1 << 1)
+                  WHEN 'completed' THEN unique_states & (1 << 2)
+                  WHEN 'discarded' THEN unique_states & (1 << 3)
+                  WHEN 'pending'   THEN unique_states & (1 << 4)
+                  WHEN 'retryable' THEN unique_states & (1 << 5)
+                  WHEN 'running'   THEN unique_states & (1 << 6)
+                  WHEN 'scheduled' THEN unique_states & (1 << 7)
+                  ELSE 0
+                END >= 1
+          LIMIT 1
+        SQL
+        next discarded << id if conflict
+
+        runtime_execute("UPDATE river_job SET state = 'available' WHERE id = #{id}")
+      end
+
+      unless discarded.empty?
+        runtime_execute(<<~SQL)
+          UPDATE river_job
+          SET state = 'discarded', finalized_at = #{runtime_time(now)},
+              metadata = #{runtime_merge_metadata("unique_key_conflict" => "scheduler_discarded")}
+          WHERE id IN (#{discarded.join(",")})
+        SQL
+      end
+      runtime_execute("UPDATE river_job SET state = 'available' WHERE id IN (#{available.join(",")})") unless available.empty?
+
+      rows.map { |row| runtime_value(row, :queue) }
+    end
+
     private def runtime_state(value)
       runtime_postgres? ? "#{runtime_quote(value)}::river_job_state" : runtime_quote(value)
     end
@@ -632,26 +891,40 @@ module River::Driver
 
       cast = runtime_postgres? ? "::timestamptz" : ""
       encoded = if runtime_postgres?
-        value.getutc.iso8601(6)
+        normalize_time(value).iso8601(6)
       else
-        value.getutc.round(3).strftime("%Y-%m-%d %H:%M:%S.%3N")
+        normalize_time(value).strftime("%Y-%m-%d %H:%M:%S.%3N")
       end
       "#{runtime_quote(encoded)}#{cast}"
     end
 
     private def runtime_update_value(field, value)
       case field
-      when :attempt, :max_attempts
-        Integer(value).to_s
+      when :attempt
+        attempt = Integer(value)
+        raise ArgumentError, "attempt must be between 0 and #{River::MAX_ATTEMPTS_LIMIT}" unless (0..River::MAX_ATTEMPTS_LIMIT).cover?(attempt)
+
+        attempt.to_s
       when :attempted_at, :finalized_at
         value ? runtime_time(value) : "NULL"
       when :attempted_by
-        runtime_postgres? ? "ARRAY[#{Array(value).map { |item| runtime_quote(item) }.join(",")}]::text[]" : runtime_json(Array(value))
+        values = Array(value) #: Array[untyped]
+        raise ArgumentError, "attempted_by must contain only Strings" unless values.all? { |item| item.is_a?(String) }
+
+        runtime_postgres? ? "ARRAY[#{values.map { |item| runtime_quote(item) }.join(",")}]::text[]" : runtime_json(values)
       when :errors
         input_values = Array(value) #: Array[untyped]
         values = input_values.map { |error| error.respond_to?(:to_h) ? error.to_h : error }
         runtime_postgres? ? "ARRAY[#{values.map { |item| runtime_json(item) }.join(",")}]::jsonb[]" : runtime_json(values)
+      when :max_attempts
+        max_attempts = Integer(value)
+        raise ArgumentError, "max_attempts must be greater than zero" unless max_attempts > 0
+        raise ArgumentError, "max_attempts must not exceed #{River::MAX_ATTEMPTS_LIMIT}" if max_attempts > River::MAX_ATTEMPTS_LIMIT
+
+        max_attempts.to_s
       when :metadata
+        raise ArgumentError, "metadata must be a Hash" unless value.is_a?(Hash)
+
         runtime_json(value)
       when :state
         runtime_state(value)