solid_loop 0.0.4

data/app/helpers/solid_loop/application_helper.rb

@@ -0,0 +1,125 @@
+module SolidLoop
+  module ApplicationHelper
+    # ---- Money -------------------------------------------------------------
+    # One money formatter used everywhere in the admin so the same cost never
+    # reads as "$0.09" in one place, "$0.0248" in another and "$0.00029" in a
+    # third. Shows 2 dp for "normal" amounts but keeps enough significant
+    # figures that a sub-cent run never collapses to "$0.00".
+    #
+    #   1.2345   -> "$1.23"
+    #   0.0248   -> "$0.025"
+    #   0.00029  -> "$0.00029"
+    #   0.0      -> "$0.00"
+    #  -0.005    -> "-$0.0050"
+    def sl_money(amount)
+      value = amount.to_f
+      return "$0.00" if value.zero?
+
+      abs = value.abs
+      # Default to 2 dp so "normal" money reads $0.09 / $1.23. But when 2 dp
+      # would lose precision (0.0248 -> 0.02) OR round a real amount away to
+      # zero (0.00029 -> 0.00), extend precision to ~2 significant figures so
+      # sub-cent runs stay legible instead of collapsing.
+      precision = 2
+      if abs < 0.1 && (abs - abs.round(2)).abs > Float::EPSILON
+        precision = [(-Math.log10(abs)).floor + 2, 8].min
+      end
+
+      # Sign goes before the "$" (never "$-0.0050").
+      "#{'-' if value.negative?}$#{number_with_precision(abs, precision: precision)}"
+    end
+
+    # ---- Failure classification -------------------------------------------
+    # A loop's terminal error is captured only as a free-text `error_message`
+    # (e.g. "HTTP 403: ...", "Faraday::ConnectionFailed: ...", or an agent-side
+    # exception). Classify without a schema change by pattern-matching that
+    # string: provider/transport problems (HTTP status, Faraday, JSON parse of
+    # the wire response, PII-redaction rejection) are INFRA failures — not agent
+    # bugs. Everything else that reached `failed` is an AGENT failure.
+    #
+    # Returns :infra, :agent, or nil (loop is not failed / has no message).
+    def sl_failure_kind(loop)
+      return nil unless loop.status == "failed"
+
+      SolidLoop::ApplicationHelper.classify_failure(loop.error_message)
+    end
+
+    # Module-level so read models / KPIs can reuse the same rules as the view.
+    INFRA_ERROR_PATTERNS = [
+      /\AHTTP \d{3}\b/,               # provider returned a non-2xx envelope
+      /\bFaraday::/,                  # transport / connection error class
+      /\bJSON::ParserError\b/,        # unparseable provider response body
+      /invalid_json_after_redaction/, # OpenRouter PII-redaction mangled the body
+      /PII detected/i,
+      /\b(?:timed out|timeout|connection (?:refused|reset))\b/i
+    ].freeze
+
+    # Agent-side code bugs — even when the text happens to mention "timeout" or
+    # "connection" — must NOT be mislabelled infra (e.g. a NoMethodError whose
+    # message references a `timeout` method). Checked before the infra patterns.
+    CODE_ERROR_PREFIX = /\A(?:NoMethodError|NameError|ArgumentError|TypeError|KeyError|IndexError|RuntimeError|NotImplementedError|FrozenError|ZeroDivisionError|NoMatchingPatternError)\b/
+
+    def self.classify_failure(error_message)
+      msg = error_message.to_s.strip
+      return :agent if msg.blank?
+      return :agent if msg.match?(CODE_ERROR_PREFIX)
+
+      INFRA_ERROR_PATTERNS.any? { |re| msg.match?(re) } ? :infra : :agent
+    end
+
+    # A short, human failure reason for inline display / tooltips. Trims the
+    # noisy body so a table cell stays readable; full text is on the loop page.
+    def sl_failure_reason(loop, limit: 140)
+      msg = loop.error_message.to_s.strip
+      return nil if msg.empty?
+
+      msg = msg.gsub(/\s+/, " ")
+      msg.length > limit ? "#{msg[0, limit - 1]}…" : msg
+    end
+
+    # ---- Subject linking ---------------------------------------------------
+    # A loop's polymorphic subject ("User #2") is an app record the engine
+    # can't route to. Host apps register a resolver returning a URL (and,
+    # optionally, a richer label) so the admin can deep-link to it. When no
+    # resolver is configured we degrade to the plain "Type #id" text.
+    #
+    #   SolidLoop.subject_resolver = ->(loop) {
+    #     { url: main_app.user_path(loop.subject_id), label: loop.subject&.name }
+    #   }
+    def sl_subject_link(loop)
+      return nil if loop.subject_type.blank? && loop.subject_id.blank?
+
+      resolved = sl_resolve_subject(loop)
+      label = resolved[:label].presence || "#{loop.subject_type} ##{loop.subject_id}"
+      url = sl_safe_subject_url(resolved[:url])
+
+      url ? link_to(label, url) : label
+    end
+
+    private
+
+    # The subject_resolver is a host-app hook — treat its URL as untrusted.
+    # Allow only relative paths or http(s); drop javascript:/data:/other schemes
+    # and protocol-relative "//host" so a resolver can't inject a clickable XSS
+    # link into the admin. Unsafe -> nil (renders the label as plain text).
+    def sl_safe_subject_url(url)
+      url = url.to_s.strip
+      return nil if url.empty?
+      return url if url.start_with?("/") && !url.start_with?("//")
+      return url if url.match?(%r{\Ahttps?://}i)
+
+      nil
+    end
+
+    def sl_resolve_subject(loop)
+      resolver = SolidLoop.respond_to?(:subject_resolver) ? SolidLoop.subject_resolver : nil
+      return {} unless resolver
+
+      result = resolver.call(loop)
+      result.is_a?(Hash) ? result : { url: result }
+    rescue StandardError => e
+      Rails.logger.warn("SolidLoop subject_resolver raised: #{e.class}: #{e.message}")
+      {}
+    end
+  end
+end

data/app/jobs/solid_loop/application_job.rb

@@ -0,0 +1,13 @@
+module SolidLoop
+  class ApplicationJob < ActiveJob::Base
+    # SolidLoop requires a same-database transactional job backend (Solid
+    # Queue / GoodJob in the app database). Successor jobs are enqueued INSIDE
+    # the transaction that advances loop state, so state + dispatch commit
+    # atomically — a crash or rollback leaves neither. The enqueue must
+    # therefore not be deferred to after-commit. (Rails 7.1 has no such
+    # deferral; the attribute exists from 7.2.)
+    if respond_to?(:enqueue_after_transaction_commit=)
+      self.enqueue_after_transaction_commit = false
+    end
+  end
+end

data/app/jobs/solid_loop/llm_completion_job.rb

@@ -0,0 +1,139 @@
+module SolidLoop
+  class LlmCompletionJob < ApplicationJob
+    queue_as :solid_loop_llm
+
+    READ_TIMEOUT = 600 # 10 minutes
+    EVENTS_SAVE_BEFORE_REQUEST = true # two phase event save, good for slow models debugging
+    HTTP_STREAMING_UPDATE_MODEL_INTERVAL = 2 # seconds - update model content every N seconds
+
+    def perform(loop_id)
+      execution_token = SecureRandom.uuid
+
+      # Derive the lease from the agent's own read_timeout (+ margin) so it is
+      # guaranteed to outlive the HTTP client — a healthy in-flight turn can then
+      # never be reclaimed by the reaper. Resolved before the CAS so the same
+      # atomic transition to `running` also stamps `lease_expires_at`.
+      loop_for_lease = SolidLoop::Loop.find(loop_id)
+      begin
+        agent_for_lease = loop_for_lease.agent
+      rescue NameError => e
+        # A renamed/missing agent class: fail the still-`queued` loop
+        # VISIBLY instead of raising forever on every retry. No token/lease is
+        # held yet, so this is a plain queued→failed transition.
+        Rails.logger.error "LlmCompletionJob: cannot resolve agent for loop #{loop_id}: #{e.class}: #{e.message}"
+        # from: :queued only — a missing agent fails BEFORE any CAS claim, so no
+        # worker can ever hold this loop in `running`; fencing to :queued keeps
+        # this consistent with every other transition and can't stomp a sibling.
+        loop_for_lease.transition_status(
+          from: :queued,
+          to: :failed,
+          error_message: "#{e.class}: #{e.message}",
+          execution_token: nil,
+          lease_expires_at: nil
+        )
+        return
+      end
+      return unless agent_for_lease # nil agent_class_name: nothing to run
+
+      lease_duration   = SolidLoop.config.llm_lease_duration(agent_for_lease)
+      lease_expires_at = Time.current + lease_duration
+
+      claimed = SolidLoop::Loop
+        .where(id: loop_id, status: :queued)
+        .update_all(
+          status: :running,
+          execution_token: execution_token,
+          lease_expires_at: lease_expires_at,
+          updated_at: Time.current
+        )
+      return if claimed == 0
+
+      loop_record = nil
+
+      # Reconciliation wrapper: wrap the ENTIRE post-claim job body —
+      # the Context build, the second `agent` resolution, middleware config, and
+      # the whole pipeline — so a non-graceful error (a NoMethodError from a
+      # renamed agent, a bug in a middleware) fails the loop VISIBLY instead of
+      # stranding it `running`. Hard kills still fall through to the reaper. The
+      # inner ErrorHandling middleware still terminalizes its graceful allowlist
+      # first; this only catches what escapes it.
+      begin
+        # Heartbeat the lease for the ENTIRE claimed interval.
+        # The heartbeat wrapper opens IMMEDIATELY after the successful claim, so
+        # it covers the CAS→`Context.new`→second-`agent`→config window too — making
+        # "the entire claimed interval is heartbeated" literally true (previously
+        # Context.new/agent/config ran BEFORE the heartbeat started, an unheartbeated
+        # gap). It covers MCP init (AgentInitialization), the whole stream incl.
+        # silent prefill (NetworkCalling), and finalization. `read_timeout` is an
+        # inactivity timeout, not a total-duration bound, so the static claim-time
+        # lease can expire on a HEALTHY worker; the heartbeat renews it at ~ttl/4
+        # keyed on loop_id + execution_token, and flips a lost-lease flag if the
+        # generation rotated away. The heartbeat still stops in `run`'s ensure
+        # BEFORE any canonical completion.
+        SolidLoop::LeaseHeartbeat.run(
+          loop_id: loop_id,
+          execution_token: execution_token,
+          lease_duration: lease_duration,
+          # The agent's own legal turn bound sizes the renewer's leak ceiling
+          # (`max_duration + lease_leak_grace`), so a legitimate turn — capped at
+          # max_duration by the agent's contract — is never dropped, while a genuine
+          # leak self-expires shortly after max_duration.
+          max_duration: agent_for_lease.max_duration,
+          logger: Rails.logger
+        ) do |heartbeat|
+          context = SolidLoop::Pipeline::Context.new(loop_id: loop_id, execution_token: execution_token)
+          loop_record = context.loop
+          agent = loop_record.agent
+          context.heartbeat = heartbeat
+
+          builder = SolidLoop::Pipeline::Builder.new(SolidLoop.llm_middlewares.middlewares)
+          agent.configure_llm_middlewares(builder)
+
+          pipeline = SolidLoop::Pipeline.new(builder.middlewares)
+          pipeline.call(context)
+        end
+      rescue SolidLoop::LostLease => e
+        # The heartbeat lost the lease mid-turn (generation rotated away or the
+        # reaper reclaimed us). We DON'T own the loop, so we write nothing and do
+        # NOT fail it — recovery belongs to whoever now owns the generation (the
+        # resume, or the reaper's re-enqueue). Swallow so ActiveJob doesn't retry.
+        Rails.logger.info "LlmCompletionJob: lease lost for loop #{loop_id}, yielding turn: #{e.message}"
+      rescue StandardError => e
+        # If we owned the loop we reconciled it to `failed` (durable) — swallow so
+        # ActiveJob does not retry endlessly (the CAS would no-op anyway). If we
+        # did NOT own it (a concurrent pause/stop/reclaim already rotated the
+        # token), re-raise so the genuinely-unhandled error stays visible.
+        raise unless reconcile_failure!(loop_record, execution_token, e)
+      end
+    end
+
+    private
+
+    # Fence-checked reconciliation: only mutate if THIS generation still owns the
+    # loop (running + our token) under the row lock. A stale worker (pause/stop/
+    # reclaim already rotated the token) writes nothing. Marks any processing
+    # shell failed+hidden, fails the loop, and clears the token + lease atomically.
+    def reconcile_failure!(loop_record, execution_token, error)
+      # Context build itself failed before we could load the loop: nothing to
+      # reconcile against here. Re-raise so the error stays visible (a missing
+      # loop row is not a stale-worker no-op).
+      return false unless loop_record
+
+      error_msg = "#{error.class}: #{error.message}"
+      Rails.logger.error "LlmCompletionJob reconciling uncaught error: #{error_msg}\n#{error.backtrace&.take(10)&.join("\n")}"
+
+      loop_record.transition_status(
+        from: :running,
+        to: :failed,
+        expected_execution_token: execution_token,
+        error_message: error_msg,
+        execution_token: nil,
+        lease_expires_at: nil
+      ) do
+        loop_record.messages.where(role: "assistant", status: "processing").find_each do |shell|
+          shell.update!(status: "failed", is_hidden: true)
+        end
+      end
+    end
+  end
+end

data/app/jobs/solid_loop/observe_broadcast_job.rb

@@ -0,0 +1,15 @@
+module SolidLoop
+  class ObserveBroadcastJob < ApplicationJob
+    def perform(message_id)
+      message = SolidLoop::Message.find_by(id: message_id)
+      return unless message
+
+      Turbo::StreamsChannel.broadcast_append_to(
+        "solid_loop:loop:#{message.loop_id}",
+        target: "sl-messages",
+        partial: "solid_loop/messages/message",
+        locals: { message: message }
+      )
+    end
+  end
+end

data/app/jobs/solid_loop/reaper_job.rb

@@ -0,0 +1,20 @@
+module SolidLoop
+  # A plain entry point for `SolidLoop.reap!`, so a host can install the
+  # recurrence with whatever scheduler it already runs (good_job cron,
+  # solid_queue recurring, sidekiq-scheduler, a clock process, …). The gem does
+  # NOT hardcode a scheduler — see docs/decisions/durable_attempt_lease.md.
+  #
+  #   # good_job (config/initializers or good_job.rb):
+  #   config.good_job.cron = {
+  #     solid_loop_reaper: { cron: "* * * * *", class: "SolidLoop::ReaperJob" }
+  #   }
+  #
+  # Idempotent and safe to run concurrently.
+  class ReaperJob < ApplicationJob
+    queue_as :solid_loop_maintenance
+
+    def perform
+      SolidLoop.reap!
+    end
+  end
+end

data/app/jobs/solid_loop/tool_execution_job.rb

@@ -0,0 +1,200 @@
+module SolidLoop
+  class ToolExecutionJob < ApplicationJob
+    queue_as :solid_loop_tool
+
+    # `reaper_lease_token` (optional): when the reaper's case 2 re-enqueues a
+    # stalled tool call it FIRST CAS-claims the per-tool lease under the loop
+    # lock and hands its token here, so two racing reapers resolve to exactly one
+    # re-enqueue (the loser's CAS gets 0 rows). A normal dispatch omits it and the
+    # job self-claims a fresh lease below.
+    def perform(tool_call_id, execution_token = nil, reaper_lease_token = nil)
+      # A tool job must carry the generation token it was dispatched with.
+      # A job without one (legacy/host-enqueued) is not part of any current
+      # generation: fail closed instead of adopting the loop's token.
+      return if execution_token.nil?
+
+      # Pre-fetch basic objects to determine the agent class. The token fence
+      # here is a plain generation check (no lease acquired yet); a stale job
+      # no-ops silently — there is nothing to reconcile.
+      tool_call = SolidLoop::ToolCall.find(tool_call_id)
+      message = tool_call.message
+      loop_model = message.loop
+      loop_model.reload
+      return unless loop_model.running? && loop_model.execution_token == execution_token
+
+      # Reconciliation wrapper (findings 4/5/9): begin reconciliation immediately
+      # after loading the loop/tool IDs and INCLUDE everything that used to sit
+      # OUTSIDE the begin — agent resolution (a renamed/missing agent method),
+      # sequential-mode evaluation, the lease claim aftermath, builder creation,
+      # and `configure_tool_middlewares`. A non-graceful error at ANY of these
+      # steps fails the loop VISIBLY under the fence instead of stranding it
+      # `running` / reaping-and-retrying it forever. The context is built up
+      # front (with lease_token nil until claimed) so a PRE-LEASE error still has
+      # a generation-fenced failure path: `reconcile_failure!` transitions the
+      # loop to failed under the execution_token even when no tool lease was held.
+      # Hard kills (SIGKILL / an Exception outside StandardError) still fall
+      # through to the reaper's case 2, which re-invokes with the SAME stable key.
+      context = SolidLoop::Pipeline::ToolContext.new(tool_call_id: tool_call_id)
+      context.tool_call = tool_call
+      context.message = message
+      context.loop = loop_model
+      context.execution_token = execution_token
+      context.lease_token = nil
+
+      begin
+        agent = loop_model.agent
+
+        # Token fencing + the executed_at checkpoint keep every mode safe against
+        # stale/duplicate jobs. The first-unresolved boundary is the *ordering*
+        # invariant: it only applies when the agent opts into sequential
+        # execution (parallel mode deliberately runs the turn's jobs in any
+        # order). The check runs under the parent assistant message lock — the
+        # same lock ResponseCreation resolves calls under — so it cannot race a
+        # result landing.
+        if agent.sequential_tool_calls?
+          first_unresolved_id = nil
+          message.with_lock do
+            first_unresolved_id = loop_model.unresolved_tool_calls.first&.id
+          end
+
+          unless first_unresolved_id == tool_call.id
+            # A stray/duplicate/out-of-order job must not stall the turn:
+            # no-op, but re-dispatch the actual head so ordering self-heals. Route
+            # through enqueue! so a silently-aborted re-dispatch surfaces (job
+            # retries) instead of leaving the head-of-line call unrun.
+            SolidLoop.enqueue!(self.class, first_unresolved_id, execution_token) if first_unresolved_id
+            return
+          end
+        end
+
+        # The per-tool_call lease (the SECOND fence token).
+        # `execution_token` above is the generation fence (shared by every tool of
+        # the turn); the `lease_token` here is per-tool, so the reaper can reclaim
+        # ONE dead parallel tool without fencing its healthy siblings. The lease is
+        # derived to strictly outlive the tool client timeout, so a healthy
+        # long-running tool can never lose its lease to a reaper.
+        lease_token =
+          if tool_call.executed_at?
+            # Already checkpointed but re-dispatched: the tool BODY must not run
+            # again (executed_at is the terminal exactly-once guard). This path
+            # exists so a call whose CONTINUATION (canonical response + successor)
+            # was dropped — a crash after executed_at, reclaimed by reaper case 2 —
+            # is completed: ToolExecution restores the checkpoint and ResponseCreation
+            # commits the continuation (its own idempotent `response_message` check
+            # makes a duplicate delivery harmless). No lease is needed or held.
+            nil
+          elsif reaper_lease_token
+            # Reaper path: the lease was already CAS-claimed under the loop lock and
+            # handed here. ADOPT it only if we STILL hold it (a worker that took over
+            # after the reaper enqueued would have rotated it). No-op otherwise.
+            return unless tool_call.lease_held_by?(reaper_lease_token)
+
+            reaper_lease_token
+          else
+            # Normal dispatch: self-claim UNDER THE LOOP LOCK.
+            #
+            # SECONDARY race (clean-resume violation): the generation was validated
+            # by the `reload` at the top of `perform`, but claiming the ToolCall
+            # lease independently leaves a window in which a pause/resume can rotate
+            # G1→G2 and revoke old leases BETWEEN the two. A stale worker would then
+            # install a FRESH lease on the ToolCall without re-checking the loop,
+            # and the legitimate G2 job could not claim until that lease expired (a
+            # full tool-TTL delay before reaper case 2 repairs it). Fix: re-validate
+            # `running + execution_token` under the loop row lock and claim the lease
+            # in the SAME locked scope (loop → tool_call order). A rotation that
+            # commits before we take the lock is now seen here → no-op; one that
+            # arrives after blocks on the loop lock and rotates cleanly afterward.
+            minted = claim_lease_under_loop_lock(loop_model, tool_call, execution_token, agent)
+            if minted.nil?
+              # Observability: the CAS minted nothing — either the
+              # generation rotated away under us (stale worker no-op) or a DIFFERENT
+              # live per-tool lease still holds this call (e.g. a host-side
+              # paused→running resume that re-enqueued this job WITHOUT revoking the
+              # gated tool_call's unexecuted lease — see SolidLoop::Base#resume!,
+              # which NULLs lease_token/leased_until before re-enqueue). Without this
+              # log the strand is invisible: no error, no event, the loop just waits
+              # out the lease TTL / reaper. Reload so the logged lease reflects the
+              # holder we lost to, not our stale in-memory copy — guarded so a
+              # concurrent teardown of the row can't turn this pure-observability
+              # path into a raise (a log must never change control flow).
+              begin
+                tool_call.reload
+              rescue ActiveRecord::RecordNotFound
+                nil
+              end
+              Rails.logger.warn(
+                "ToolExecutionJob: tool lease claim lost — another live lease holds " \
+                "this tool_call; job no-op (loop may strand until lease TTL/reaper). " \
+                "tool_call_id=#{tool_call.id} loop_id=#{loop_model.id} " \
+                "lease_token=#{tool_call.lease_token.inspect} " \
+                "leased_until=#{tool_call.leased_until.inspect} " \
+                "executed_at=#{tool_call.executed_at.inspect}"
+              )
+              return
+            end
+
+            minted
+          end
+
+        tool_call.reload
+        context.tool_call = tool_call
+        context.lease_token = lease_token
+
+        builder = SolidLoop::Pipeline::Builder.new(SolidLoop.tool_middlewares.middlewares)
+        agent.configure_tool_middlewares(builder)
+        pipeline = SolidLoop::Pipeline.new(builder.middlewares)
+
+        pipeline.call(context)
+      rescue StandardError => e
+        # If this generation (+ lease, when one was held) still owns the call under
+        # the row lock we reconciled it to `failed` (durable) — swallow so ActiveJob
+        # does not retry endlessly. If ownership was already rotated away
+        # (pause/stop/reclaim), re-raise so a genuinely-unhandled error stays
+        # visible. A pre-lease error takes the no-lease, generation-fenced path.
+        raise unless reconcile_failure!(loop_model, tool_call, context, e)
+      end
+    end
+
+    private
+
+    # Claim the per-tool lease UNDER the loop row lock, after re-validating that
+    # THIS generation still owns the loop (SECONDARY claim-under-lock race). Lock
+    # order loop → tool_call. Returns the minted lease token, or nil when either
+    # the generation rotated away (stale worker — no-op) or the CAS claimed 0 rows
+    # (a live sibling lease, or the row became executed under us). No lease is
+    # installed on a loop we no longer own, so the legitimate next-generation job
+    # never waits out a stale lease's TTL.
+    def claim_lease_under_loop_lock(loop_model, tool_call, execution_token, agent)
+      minted = nil
+      loop_model.with_lock do
+        next unless loop_model.running? && loop_model.execution_token == execution_token
+
+        candidate = SecureRandom.uuid
+        now = Time.current
+        claimed = SolidLoop::ToolCall
+          .where(id: tool_call.id, executed_at: nil)
+          .where("lease_token IS NULL OR leased_until < ?", now)
+          .update_all(lease_token: candidate, leased_until: now + SolidLoop.config.tool_lease_duration(agent), updated_at: now)
+        minted = candidate if claimed.positive?
+      end
+      minted
+    end
+
+    # Fence-checked reconciliation for the tool job (findings 4/9). Delegates to
+    # the ONE shared helper so the job's and the middleware's semantics cannot
+    # drift. The helper does the BOTH-token ownership gate ITSELF under the
+    # loop→tool_call row locks (NOT via `transition_status`'s block, which cannot
+    # abort the loop write) and only terminalizes when both tokens validate — so a
+    # stale worker whose per-tool lease was rotated away never fails the loop the
+    # current owner is running. Returns true ⇒ handled (swallow); false ⇒ re-raise.
+    def reconcile_failure!(loop_model, tool_call, context, error)
+      SolidLoop::ToolFailureReconciler.reconcile!(
+        loop_model:      loop_model,
+        tool_call:       tool_call,
+        execution_token: context.execution_token,
+        lease_token:     context.lease_token,
+        error:           error
+      )
+    end
+  end
+end

data/app/models/solid_loop/application_record.rb

@@ -0,0 +1,5 @@
+module SolidLoop
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end