solid_loop 0.0.4

data/app/services/solid_loop/middlewares/response_parsing.rb

@@ -0,0 +1,348 @@
+# frozen_string_literal: true
+
+module SolidLoop
+  module Middlewares
+    class ResponseParsing
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        # We always call the inner app first (network). ResponseParsing acts on the way back (Upstream).
+        @app.call(env)
+
+        if env.response&.success?
+          process_result(env)
+        elsif env.response
+          handle_error(env)
+        end
+        # Exceptions are left to ErrorHandling middleware
+      end
+
+      private
+
+      def process_result(env)
+        # Use normalized data from the adapter/dialect if available
+        data = if env.normalized_data.present?
+                 env.normalized_data
+        elsif env.agent.streaming?
+                 # Fallback for old aggregator usage (should not happen with new Native adapter)
+                 SolidLoop::Dialects::OpenAi.new.normalize_response(env.aggregator.result)
+        else
+                 # Fallback for non-streaming old-style Faraday responses
+                 raw_json = JSON.parse(env.response.body) rescue {}
+                 SolidLoop::Dialects::OpenAi.new.normalize_response(raw_json)
+        end
+
+        content = data[:content]
+        reasoning_content = data[:reasoning]
+        tool_calls = (data[:tool_calls] || []).dup
+        usage = data[:usage] || {}
+        metadata = (data[:metadata] || {}).dup
+
+        # Stamp the model that served this turn onto the message metadata. Prefer
+        # the model the provider echoed back in its response (surfaced by the
+        # dialect's normalize_response as metadata["model"]) and fall back to the
+        # model we requested for this turn (always present via model_config). This
+        # is a per-message, immutable record: the same loop may have turns served
+        # by different models over its lifetime.
+        served_model = metadata["model"].presence || env.model_config&.dig(:model)
+        metadata["model"] = served_model if served_model.present?
+
+        ttft = env.first_token_at ? (env.first_token_at - env.request_started_at).to_f : 0.0
+        duration_thinking = if env.content_started_at && env.first_token_at
+                              (env.content_started_at - env.first_token_at).to_f
+        else
+                              0.0
+        end
+
+        # Calculate duration based on request start
+        full_duration = env.duration || (env.request_started_at ? (Time.current - env.request_started_at).to_f : 0.0)
+
+        # usage is already a normalized hash from the dialect
+        usage_data = usage
+
+        msg_params = {
+          content: content,
+          reasoning_content: reasoning_content,
+          status: "success",
+          # The canonical commit is the ONLY place a born-hidden streaming shell
+          # becomes visible history. Also correct for the
+          # non-streaming path, where we create the assistant row fresh here.
+          is_hidden: false,
+          tokens_prompt: usage_data[:tokens_prompt],
+          tokens_completion: usage_data[:tokens_completion],
+          tokens_total: usage_data[:tokens_total],
+          tokens_prompt_cached: usage_data[:tokens_prompt_cached],
+          tps: usage_data[:tps],
+          ttft: ttft,
+          duration_thinking: duration_thinking,
+          duration_generation: full_duration,
+          tool_calls_raw: tool_calls,
+          metadata: metadata,
+          cost: usage_data[:cost] || 0.0
+        }
+
+        # If the background heartbeat has LOST the lease (a renew
+        # touched 0 rows because the generation rotated away or the reaper
+        # reclaimed us), raise BEFORE any canonical write. The with_generation
+        # token fence below would also reject a stale commit, but raising here
+        # keeps the "no canonical write after a lost lease" contract explicit and
+        # surfaces the lost turn to the job's LostLease rescue.
+        env.heartbeat&.check!
+
+        # The heartbeat's job ENDS the instant we decide to commit — stop it
+        # BEFORE the canonical commit so no renew can race the lease-clear. On the
+        # tool-phase success path the commit leaves the loop `running` + same
+        # token + lease NULL ("tool phase, not an LLM turn"); a renew firing after
+        # that commit but before the outer `ensure` stop would STILL match
+        # (running + our token) and re-populate the lease, falsely reclaiming a
+        # healthy tool phase. Stopping here removes that window entirely; the outer
+        # `ensure` stop remains (idempotent) for every non-commit exit path.
+        env.heartbeat&.stop
+
+        # Every generation-owned write (turn persistence, message states,
+        # counters, tool-call rows and their successor-job enqueues) happens
+        # under the loop row lock AFTER re-validating that this generation
+        # still owns the loop. A stale worker performs zero mutations.
+        msg = nil
+        owned = env.loop.with_generation(env.execution_token) do
+          # The canonical commit ends the in-flight LLM turn: clear the lease so
+          # the loop leaves the reclaimable LLM-turn state. In the tool-phase
+          # branch below the loop stays `running` but with lease NULL (= "tool
+          # phase, not an LLM turn"); terminal/queued branches re-clear it too.
+          env.loop.update!(lease_expires_at: nil) if env.loop.lease_expires_at.present?
+
+          env.triggering_message&.update!(status: "success")
+
+          if env.assistant_message
+            msg_params[:metadata] = (env.assistant_message.metadata || {}).merge(msg_params[:metadata] || {})
+            env.assistant_message.update!(msg_params)
+            msg = env.assistant_message
+          else
+            begin
+              msg = env.loop.messages.create!(msg_params.merge(role: "assistant"))
+            rescue ActiveRecord::RecordInvalid => e
+              Rails.logger.error "Failed to create message: #{e.record.errors.full_messages.join(', ')} Params: #{msg_params}"
+              raise e
+            end
+          end
+
+          counters_to_add = {
+            step_count: 1,
+            tokens_prompt: msg.tokens_prompt || 0,
+            tokens_completion: msg.tokens_completion || 0,
+            tokens_total: msg.tokens_total || 0,
+            tokens_prompt_cached: msg.tokens_prompt_cached || 0,
+            duration_generation: full_duration,
+            cost: msg.cost || 0.0
+          }.delete_if { |_, v| v == 0 }
+
+          env.loop.class.update_counters(env.loop.id, counters_to_add) if counters_to_add.any?
+
+          if tool_calls&.any?
+            process_tool_calls(env, msg, tool_calls)
+          elsif env.loop.pending_user_messages.exists?
+            # Steering that arrived while this final (no-tool-call) request was in
+            # flight is a hidden, still-pending user message. Completing here would
+            # strand it forever. Instead requeue for another turn — the next
+            # MessageBuilding#build_messages drains and delivers it. Ownership is
+            # already validated under the held lock, and the enqueue commits inside
+            # this same transaction (a failed enqueue rolls the requeue back).
+            env.loop.update!(status: :queued, execution_token: nil, lease_expires_at: nil)
+            SolidLoop.enqueue!(SolidLoop::LlmCompletionJob, env.loop.id)
+          else
+            # Ownership is already validated under the held lock.
+            env.loop.update!(status: :completed, execution_token: nil, lease_expires_at: nil)
+          end
+        end
+        return unless owned
+
+        env.assistant_message&.broadcast_update
+
+        env.agent.on_message_created(msg) if env.agent.respond_to?(:on_message_created)
+
+        if env.loop.observe_enabled?
+          if defined?(Turbo::StreamsChannel)
+            SolidLoop::ObserveBroadcastJob.perform_later(msg.id)
+          else
+            Rails.logger.warn(
+              "[SolidLoop] observe_enabled is on for loop #{env.loop.id} but turbo-rails is not " \
+              "available; skipping live broadcast. Add `turbo-rails` to your host application to " \
+              "enable live observation of agent loops."
+            )
+          end
+        end
+      end
+
+      def process_tool_calls(env, msg, tool_calls)
+        invalid_tool_calls = []
+
+        # Persistence and successor-job dispatch commit atomically: the
+        # enqueue happens INSIDE the same transaction as the tool-call rows
+        # (SolidLoop requires a same-database transactional job backend), so a
+        # crash or rollback leaves neither the state change nor a dangling job.
+        msg.transaction do
+          tool_calls.each do |tc|
+            begin
+              args = JSON.parse(tc.dig("function", "arguments") || "{}")
+              msg.tool_calls.create!(
+                tool_call_id: tc["id"],
+                function_name: tc.dig("function", "name"),
+                arguments: args
+              )
+            rescue JSON::ParserError => e
+              error_msg = build_invalid_tool_call_message(tc, e)
+              Rails.logger.error error_msg
+              env.loop.messages.create!(
+                role: "tool",
+                tool_call_id: tc["id"],
+                content: error_msg,
+                status: "success"
+              )
+              invalid_tool_calls << tc
+            end
+          end
+
+          persisted_tool_calls = msg.tool_calls.ordered.to_a
+
+          # Parallel (default): enqueue every call in the turn so workers run
+          # them concurrently. Sequential: enqueue only the first;
+          # ResponseCreation chains the next one after each result lands.
+          calls_to_dispatch =
+            if env.agent.sequential_tool_calls?
+              persisted_tool_calls.first(1)
+            else
+              persisted_tool_calls
+            end
+
+          calls_to_dispatch.each do |tool_call|
+            SolidLoop.enqueue!(SolidLoop::ToolExecutionJob, tool_call.id, env.execution_token)
+          end
+
+          if persisted_tool_calls.empty? && invalid_tool_calls.any?
+            transitioned = env.loop.transition_status(
+              from: :running,
+              to: :queued,
+              expected_execution_token: env.execution_token,
+              execution_token: nil,
+              lease_expires_at: nil
+            )
+            SolidLoop.enqueue!(SolidLoop::LlmCompletionJob, env.loop.id) if transitioned
+          end
+        end
+      end
+
+      def build_invalid_tool_call_message(tc, error)
+        function_name = tc.dig("function", "name") || "unknown_tool"
+        raw_arguments = tc.dig("function", "arguments").to_s
+        pointer = json_error_pointer(raw_arguments, error)
+
+        <<~MSG.strip
+          Tool call rejected: `#{function_name}`
+
+          I could not use this tool because its arguments were not valid JSON.
+          Please send the same tool call again with valid JSON only.
+
+          What went wrong: #{error.message}
+          Raw arguments:
+          #{raw_arguments.presence || "(empty)"}
+          #{pointer}
+        MSG
+      end
+
+      def json_error_pointer(raw_arguments, error)
+        return nil if raw_arguments.blank?
+
+        offset = error.message[/column (\d+)/, 1]&.to_i
+        return nil unless offset && offset > 0
+
+        "#{' ' * (offset - 1)}^"
+      end
+
+      # Cap for the failing-response body slice that lands in error_message and
+      # the persisted Event, so a runaway provider error page can't bloat rows.
+      MAX_ERROR_BODY = 8_192
+
+      def handle_error(env)
+        response = env.response
+        status = response.status
+        body = error_response_body(env, response)
+
+        # Expose the failing response envelope so EventLogging can persist the
+        # provider's status/headers/body in the wire-log Event (undebuggable
+        # otherwise — especially streaming, where response.body is drained).
+        env.error_status = status
+        env.error_headers = safe_response_headers(response)
+        env.error_body = body
+
+        error_msg = "HTTP #{status}: #{body}"
+
+        is_context_limit = (status == 400) && (
+          body.include?("maximum context length") ||
+          body.include?("reduce the length of the input messages") ||
+          body.include?("context_length_exceeded")
+        )
+
+        # Duration fallback
+        full_duration = env.duration || (env.request_started_at ? (Time.current - env.request_started_at).to_f : 0.0)
+
+        # Message state and counters are generation-owned: they run inside the
+        # status CAS (under the loop lock), so a stale worker whose token lost
+        # performs zero mutations.
+        transitioned =
+          if is_context_limit
+            env.loop.transition_status(
+              from: :running,
+              to: :completed,
+              expected_execution_token: env.execution_token,
+              error_message: "Context limit reached: #{error_msg}",
+              execution_token: nil,
+              lease_expires_at: nil
+            ) do
+              env.triggering_message&.update!(status: "success")
+              env.loop.class.update_counters(env.loop.id, step_count: 1, duration_generation: full_duration)
+            end
+          else
+            env.loop.transition_status(
+              from: :running,
+              to: :failed,
+              expected_execution_token: env.execution_token,
+              error_message: error_msg,
+              execution_token: nil,
+              lease_expires_at: nil
+            ) do
+              env.triggering_message&.update!(status: "failed", error_message: error_msg)
+              env.loop.class.update_counters(env.loop.id, step_count: 1, duration_generation: full_duration)
+            end
+          end
+
+        env.triggering_message&.broadcast_update if transitioned
+      end
+
+      # Faraday's non-2xx response body is usually intact for sync calls, but a
+      # streaming request drains it through on_data into env.debug_output,
+      # leaving response.body empty (this is the "HTTP 403: " with-nothing-after
+      # bug). Fall back to the raw stream buffer so the provider's error text is
+      # never lost. Bounded to MAX_ERROR_BODY.
+      def error_response_body(env, response)
+        raw = response.body.is_a?(Hash) ? response.body.to_json : response.body.to_s
+        raw = env.debug_output.string.to_s if raw.blank? && env.debug_output
+        raw = raw.to_s
+        raw.length > MAX_ERROR_BODY ? "#{raw[0, MAX_ERROR_BODY]}… (truncated)" : raw
+      end
+
+      # Provider response headers are exactly what we need for debugging (rate
+      # limits, request ids, cf-ray, etc.), but we still scrub credential-ish
+      # ones defensively so nothing sensitive is echoed back into a persisted row.
+      def safe_response_headers(response)
+        headers = response.respond_to?(:headers) ? response.headers : nil
+        return {} if headers.blank?
+
+        headers.to_h.each_with_object({}) do |(key, value), acc|
+          acc[key] = key.to_s.match?(/\A(authorization|x-api-key|api-key)\z/i) ? "[REDACTED]" : value
+        end
+      end
+    end
+  end
+end

data/app/services/solid_loop/middlewares/tool_call_xml_parser.rb

@@ -0,0 +1,117 @@
+module SolidLoop
+  module Middlewares
+    class ToolCallXmlParser
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        # 1. Let the downstream pipeline run (ResponseParsing)
+        @app.call(env)
+
+        # 2. On the way back up, ResponseParsing has already created the message.
+        # We check if there's a message, if it has content, and NO tool calls.
+        msg = env.assistant_message
+        return unless msg && msg.content.present? && msg.tool_calls.empty?
+
+        extracted = extract_from_legacy_xml(msg.content, env.payload[:model])
+
+        if extracted.any?
+          # Revival, tool-call persistence and successor-job dispatch commit
+          # atomically (same-DB transactional job backend): a crash or
+          # rollback leaves neither a revived loop without jobs nor jobs for
+          # rows that never committed.
+          env.loop.transaction do
+            revived = env.loop.transition_status(
+              from: :completed,
+              to: :running,
+              expected_execution_token: nil,
+              execution_token: env.execution_token
+            )
+            next unless revived
+
+            # Save raw JSON representation just in case it's needed
+            msg.update!(tool_calls_raw: extracted)
+
+            extracted.each do |tc|
+              begin
+                args = JSON.parse(tc.dig("function", "arguments") || "{}")
+                msg.tool_calls.create!(
+                  tool_call_id: tc["id"],
+                  function_name: tc.dig("function", "name"),
+                  arguments: args
+                )
+              rescue JSON::ParserError => e
+                error_msg = "Invalid JSON in tool call arguments: #{e.message}. Raw: #{tc.dig('function', 'arguments')}"
+                Rails.logger.error error_msg
+                env.loop.messages.create!(
+                  role: "tool",
+                  tool_call_id: tc["id"],
+                  content: error_msg,
+                  status: "success"
+                )
+              end
+            end
+
+            # Explicit ordered scope — never rely on association default order.
+            persisted_tool_calls = msg.tool_calls.ordered.to_a
+
+            # Same emission policy as ResponseParsing: parallel agents dispatch
+            # every call at once; sequential agents dispatch only the first and
+            # let ResponseCreation chain the rest.
+            calls_to_dispatch =
+              if env.agent.sequential_tool_calls?
+                persisted_tool_calls.first(1)
+              else
+                persisted_tool_calls
+              end
+
+            calls_to_dispatch.each do |tool_call|
+              SolidLoop.enqueue!(SolidLoop::ToolExecutionJob, tool_call.id, env.execution_token)
+            end
+          end
+        end
+      end
+
+      private
+
+      def extract_from_legacy_xml(content, model_name)
+        tool_calls = []
+
+        content.scan(/<tool_call>(.*?)<\/tool_call>/m).each do |match|
+          begin
+            tc_data = JSON.parse(match[0].strip)
+            tool_calls << {
+              "id" => "xml_#{SecureRandom.hex(4)}",
+              "type" => "function",
+              "function" => { "name" => tc_data["name"], "arguments" => tc_data["arguments"].to_json }
+            }
+          rescue JSON::ParserError
+            next
+          end
+        end
+
+        tool_calls.uniq! { |tc| tc.dig("function", "name") + tc.dig("function", "arguments").to_s }
+
+        if tool_calls.empty? && model_name&.downcase&.include?("deepseek")
+          content.scan(/```json\n(.*?)\n```/m).each do |match|
+            begin
+              tc_data = JSON.parse(match[0].strip)
+              if tc_data["action"] && tc_data["action_input"]
+                tool_calls << {
+                  "id" => "json_#{SecureRandom.hex(4)}",
+                  "type" => "function",
+                  "function" => { "name" => tc_data["action"], "arguments" => tc_data["action_input"].to_json }
+                }
+              end
+            rescue JSON::ParserError
+              next
+            end
+          end
+        end
+
+        tool_calls
+      end
+    end
+  end
+end

data/app/services/solid_loop/sse_stream_aggregator.rb

@@ -0,0 +1,105 @@
+require "event_stream_parser"
+
+module SolidLoop
+  class SseStreamAggregator
+    attr_reader :result
+
+    def initialize
+      @result = {}
+      @done = false
+      @parser = EventStreamParser::Parser.new
+    end
+
+    def ingest(chunk)
+      return if chunk.blank?
+
+      # The block must be passed to feed in event_stream_parser v1.x
+      @parser.feed(chunk) do |type, data, id, reconnection_time|
+        process_event(type, data)
+      end
+    end
+
+    CONCAT_KEYS = %w[content arguments reasoning_content reasoning thought].freeze
+
+    def done?
+      return true if @done
+      choices = @result["choices"] || []
+      return false if choices.empty?
+
+      choices.any? { |c| c["finish_reason"].present? }
+    end
+
+    private
+
+    def process_event(type, data)
+      return if data.blank?
+
+      if data == "[DONE]"
+        @done = true
+        return
+      end
+
+      begin
+        json_data = JSON.parse(data)
+        @result = deep_merge_sse(@result, json_data)
+      rescue JSON::ParserError
+        # Ignore partial or invalid JSON payloads
+      end
+    end
+
+    def deep_merge_sse(target, source)
+      source.each do |key, value|
+        next if value.nil?
+
+        target_key = (key == "delta") ? "message" : key
+
+        if target[target_key].is_a?(Hash) && value.is_a?(Hash)
+          deep_merge_sse(target[target_key], value)
+        elsif target[target_key].is_a?(Array) && value.is_a?(Array)
+          target[target_key] = merge_arrays(target[target_key], value)
+        elsif target[target_key].is_a?(String) && value.is_a?(String)
+          if CONCAT_KEYS.include?(target_key)
+            target[target_key] += value
+          else
+            target[target_key] = value
+          end
+        else
+          target[target_key] = deep_clone_and_map_sse(value)
+        end
+      end
+      target
+    end
+
+    def deep_clone_and_map_sse(value)
+      if value.is_a?(Hash)
+        new_hash = {}
+        value.each do |k, v|
+          nk = (k == "delta") ? "message" : k
+          new_hash[nk] = deep_clone_and_map_sse(v)
+        end
+        new_hash
+      elsif value.is_a?(Array)
+        value.map { |v| deep_clone_and_map_sse(v) }
+      else
+        value
+      end
+    end
+
+    def merge_arrays(target_arr, source_arr)
+      if source_arr.all? { |el| el.is_a?(Hash) && el["index"].is_a?(Integer) }
+        source_arr.each do |source_el|
+          idx = source_el["index"]
+          target_el = target_arr.find { |el| el["index"] == idx }
+          if target_el
+            deep_merge_sse(target_el, source_el)
+          else
+            target_arr << deep_clone_and_map_sse(source_el)
+          end
+        end
+        target_arr.sort_by { |el| el["index"] }
+      else
+        source_arr
+      end
+    end
+  end
+end

data/app/services/solid_loop/tool_failure_reconciler.rb

@@ -0,0 +1,122 @@
+module SolidLoop
+  # ONE shared fenced-transition helper for the tool path's TWO failure
+  # reconcilers (the tool-write failure path) so their semantics cannot drift:
+  #
+  #   * `ToolExecutionJob#reconcile_failure!` — a non-graceful StandardError that
+  #     escaped the pipeline.
+  #   * `ToolMiddlewares::ErrorHandling#handle_exception` — a graceful error
+  #     (McpError) the middleware terminalizes in-line.
+  #
+  # WHY it cannot use `Loop#transition_status`'s block to gate ownership: that
+  # method performs the loop `update!` AFTER yielding REGARDLESS — a `next` inside
+  # the block aborts only the block, not the status write. So a stale worker whose
+  # per-tool lease was rotated away (its lease expired while it was still alive and
+  # the reaper's case 2 reclaimed the ToolCall as a fresh lease, siblings unfenced)
+  # would still terminalize a loop it no longer owns: `loop.execution_token` is the
+  # SHARED generation token and still matches, so the transition fires, the token
+  # is cleared, and the CURRENT owner (worker B) is killed. The nested
+  # `lease_held_by?` check only skips the ToolCall update, never the loop write.
+  #
+  # The fix: do the ownership gate OURSELVES under the loop→tool_call row locks and
+  # only terminalize when BOTH tokens validate. Three post-error shapes:
+  #
+  #   POST-CLAIM error (`lease_token` present, tool NOT executed): the classic
+  #     both-token case. Acquire loop → tool_call (the established lock order),
+  #     validate `loop.execution_token == mine` AND `tool_call.lease_held_by?(mine)`
+  #     BEFORE either write, then do the ToolCall failure mark AND the loop
+  #     terminalization in that one locked transaction. A ROTATED lease (A holds
+  #     LA, reaper reclaimed as LB) → stale/no-op, the loop is NOT failed.
+  #
+  #   EXECUTED continuation, no lease (`lease_token` nil, `executed_at` present):
+  #     the tool result is already the terminal exactly-once checkpoint; the error
+  #     is in the continuation (canonical response / successor). Fail the loop under
+  #     the GENERATION fence only — never dirty the already-committed ToolCall.
+  #
+  #   PRE-LEASE setup error (`lease_token` nil, NOT executed): no tool lease was
+  #     ever acquired (agent resolution / sequential-mode eval / builder /
+  #     configure_tool_middlewares). Generation-only fenced failure keeps the
+  #     job-level reconciliation contract intact. No ToolCall write (none is owned).
+  #
+  # Returns TRUE when reconciliation was handled here (the caller swallows the
+  # error — the failure is durable OR we are a proven-stale no-op that must not
+  # retry); FALSE only means "not our concern" (the caller re-raises).
+  module ToolFailureReconciler
+    module_function
+
+    def reconcile!(loop_model:, tool_call:, execution_token:, lease_token:, error:)
+      # A cancellation fires precisely when this generation already lost the token
+      # (pause/stop): the new generation owns both loop and call. Nothing to fail
+      # or dirty — swallow (handled) without touching the DB.
+      return true if error.is_a?(SolidLoop::CancellationError)
+
+      error_msg = "#{error.class}: #{error.message}"
+      Rails.logger.error(
+        "ToolFailureReconciler: #{error_msg}\n#{error.backtrace&.take(10)&.join("\n")}"
+      )
+
+      if lease_token.present?
+        reconcile_post_claim(loop_model, tool_call, execution_token, lease_token, error_msg)
+      elsif tool_call&.reload&.executed_at?
+        # Executed continuation with no lease: terminal checkpoint already written;
+        # fail the loop under the generation fence, leave the ToolCall untouched.
+        fail_loop_generation_fenced(loop_model, execution_token, error_msg)
+      else
+        # Pre-lease setup error: generation-only fenced failure.
+        fail_loop_generation_fenced(loop_model, execution_token, error_msg)
+      end
+
+      # Reconciliation was handled on the tool path in every branch above — either
+      # we terminalized under a validated fence, or we proved ourselves stale and
+      # must NOT retry (a stale worker that keeps retrying is worse than a no-op).
+      true
+    end
+
+    # POST-CLAIM: the BOTH-token gate, done under the loop→tool_call row locks so
+    # `loop.execution_token` and the ToolCall lease columns are frozen between the
+    # validation and the writes. If EITHER token fails to match → no-op: we do NOT
+    # terminalize a loop we no longer own (the rotated-lease stale-worker case).
+    def reconcile_post_claim(loop_model, tool_call, execution_token, lease_token, error_msg)
+      loop_model.with_lock do
+        # Loop-generation fence (first token) under the loop row lock.
+        next unless loop_model.running? && loop_model.execution_token == execution_token
+
+        tool_call.with_lock do
+          now = Time.current
+          # Per-tool lease fence (second token) under the tool_call row lock.
+          # BOTH must validate BEFORE either write — a rotated/expired lease means
+          # a live successor (worker B) owns this call; failing the loop would kill
+          # it. Stale → no-op. Note: an EXPIRED lease with no rotation yet (still
+          # our generation, no worker B) also no-ops here — an over-TTL worker is
+          # by definition no longer authoritative, so its failure surfaces via the
+          # reaper's case 2 re-invoke rather than terminalizing the loop directly.
+          next unless tool_call.lease_held_by?(lease_token, now: now)
+
+          # Both tokens validated under the locks: do BOTH writes in this one
+          # locked transaction.
+          tool_call.update!(is_success: false, error_message: error_msg, lease_token: nil, leased_until: nil)
+          loop_model.update!(
+            status: :failed,
+            error_message: error_msg,
+            execution_token: nil,
+            lease_expires_at: nil
+          )
+        end
+      end
+    end
+
+    # Generation-only fenced loop failure (pre-lease setup / executed continuation).
+    # No ToolCall write — no per-tool lease is owned here. Reuses the loop's status
+    # CAS: the block is empty, so `transition_status`'s "update after yield"
+    # behaviour is exactly what we want (a bare generation-fenced transition).
+    def fail_loop_generation_fenced(loop_model, execution_token, error_msg)
+      loop_model.transition_status(
+        from: :running,
+        to: :failed,
+        expected_execution_token: execution_token,
+        error_message: error_msg,
+        execution_token: nil,
+        lease_expires_at: nil
+      )
+    end
+  end
+end