solid_loop 0.0.4

data/lib/solid_loop/configuration.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module SolidLoop
+  # Durable-lease + reaper tunables. See docs/decisions/durable_attempt_lease.md.
+  #
+  # The load-bearing invariant is: an LLM turn's lease MUST outlive the HTTP
+  # client's read timeout, so a healthy in-flight turn can never be reclaimed by
+  # the reaper (its provider client raises first). The lease duration is derived
+  # PER CLAIM from the agent's actual `read_timeout` plus `lease_margin`, so the
+  # invariant holds by construction rather than by a fragile global-TTL constant.
+  class Configuration
+    # Seconds added on top of the agent's HTTP read_timeout when deriving a
+    # per-claim LLM lease. Must be > 0 so the lease strictly outlives the client
+    # timeout. Also acts as the floor when an agent exposes no read_timeout.
+    attr_reader :lease_margin
+
+    # A loop sitting in `queued` for longer than this (seconds) is treated by the
+    # reaper (case 4) as having lost its enqueued job, and is re-enqueued. Must
+    # comfortably exceed normal queue latency so a merely-backlogged loop is not
+    # redundantly re-enqueued (the claim CAS dedupes duplicates harmlessly, so a
+    # low value is safe-but-wasteful, not incorrect).
+    attr_reader :queued_reap_threshold
+
+    # Fallback read_timeout (seconds) used to derive a lease when the agent's
+    # `llm_provider` hash carries no `:read_timeout`. Kept in sync with the
+    # adapter default (LlmCompletionJob::READ_TIMEOUT).
+    attr_reader :default_read_timeout
+
+    # Fallback tool client timeout (seconds) used to derive a per-tool_call lease.
+    # Tools do not expose a single uniform read_timeout the way the LLM provider
+    # does (a loop can mix in-process toolsets and remote MCP servers, each with
+    # its own Faraday timeout — default 60s in Mcp::Client), so the tool lease is
+    # derived from THIS configurable ceiling on the slowest tool a healthy worker
+    # could still be blocked in. The lease (this + `lease_margin`) must strictly
+    # outlive that ceiling so a live, long-running tool is never reclaimed.
+    attr_reader :default_tool_timeout
+
+    # Grace period (seconds) added on top of the OWNING AGENT'S `max_duration` to
+    # form the lease renewer's per-registration leak ceiling. Once a registration has
+    # been renewed for longer than `max_duration + lease_leak_grace` — only possible
+    # if a turn NEVER deregistered (a job that skipped its `ensure`, an unreachable
+    # death path) — the renewer DROPS it and stops renewing, so its lease lapses and
+    # the reaper's case-1 reclaim reclaims the loop; renewer liveness no longer
+    # depends on `ensure` ever running.
+    #
+    # This replaces the old `lease_renew_ceiling_multiplier` (a MULTIPLE of the lease
+    # duration), which was WRONG: a healthy turn stays registered for its WHOLE life,
+    # so any legitimate stream longer than K lease-widths (e.g. a small `read_timeout`
+    # with steady chunks) was dropped and reaper-reclaimed mid-flight, and no long
+    # turn could ever complete. Tying the ceiling to the agent's OWN legal turn bound
+    # (`max_duration`, its own contract) means a legitimate turn always finishes
+    # before the ceiling and is never dropped, while a genuine leak still self-expires
+    # shortly after `max_duration`. Must be > 0.
+    attr_reader :lease_leak_grace
+
+    # Fallback `max_duration` (seconds) used to size the leak ceiling when a
+    # registration is created without one (an out-of-band caller of
+    # `LeaseRenewer#register`). Mirrors `SolidLoop::Base#max_duration`'s default (2h).
+    attr_reader :default_max_duration
+
+    def initialize
+      @lease_margin          = 60
+      @queued_reap_threshold = 300
+      @default_read_timeout  = SolidLoop::LlmCompletionJob::READ_TIMEOUT
+      @default_tool_timeout  = 600
+      # 5 minutes past the agent's own legal turn bound: comfortably absorbs clock
+      # skew / a straggling final chunk, yet a genuine leak self-expires soon after
+      # `max_duration` rather than never.
+      @lease_leak_grace      = 300
+      @default_max_duration  = 2.hours.to_i
+    end
+
+    def lease_margin=(seconds)
+      seconds = Integer(seconds)
+      raise ArgumentError, "lease_margin must be > 0 (the lease must outlive the client timeout)" unless seconds.positive?
+
+      @lease_margin = seconds
+    end
+
+    def queued_reap_threshold=(seconds)
+      seconds = Integer(seconds)
+      raise ArgumentError, "queued_reap_threshold must be > 0" unless seconds.positive?
+
+      @queued_reap_threshold = seconds
+    end
+
+    def default_read_timeout=(seconds)
+      seconds = Integer(seconds)
+      raise ArgumentError, "default_read_timeout must be > 0" unless seconds.positive?
+
+      @default_read_timeout = seconds
+    end
+
+    def default_tool_timeout=(seconds)
+      seconds = Integer(seconds)
+      raise ArgumentError, "default_tool_timeout must be > 0" unless seconds.positive?
+
+      @default_tool_timeout = seconds
+    end
+
+    def lease_leak_grace=(seconds)
+      seconds = Integer(seconds)
+      raise ArgumentError, "lease_leak_grace must be > 0" unless seconds.positive?
+
+      @lease_leak_grace = seconds
+    end
+
+    def default_max_duration=(seconds)
+      seconds = Integer(seconds)
+      raise ArgumentError, "default_max_duration must be > 0" unless seconds.positive?
+
+      @default_max_duration = seconds
+    end
+
+    # Derive an LLM lease duration (seconds) from an agent's configured HTTP read
+    # timeout plus the safety margin. The margin GUARANTEES lease > read_timeout,
+    # so a live worker never holds an expired lease. A missing/invalid read
+    # timeout falls back to `default_read_timeout`.
+    def llm_lease_duration(agent)
+      read_timeout =
+        begin
+          Integer(agent.llm_provider[:read_timeout])
+        rescue StandardError, TypeError
+          nil
+        end
+      read_timeout = default_read_timeout unless read_timeout&.positive?
+      read_timeout + lease_margin
+    end
+
+    # Derive a per-tool_call lease duration (seconds) for `agent`. Mirrors the LLM
+    # lease so the invariant holds BY CONSTRUCTION: the lease strictly outlives the
+    # SLOWEST client an in-flight tool could be blocked in, so a healthy
+    # long-running tool can never be reclaimed by the reaper (case 2).
+    #
+    #   max(default_tool_timeout, slowest resolved HTTP MCP client timeout) + margin
+    #
+    # HTTP MCP tools use `mcp_config[:timeout] || 60` (Mcp::ClientFactory) — an
+    # operator can set this well above `default_tool_timeout`, so it must be folded
+    # in. In-process/custom tools expose NO client timeout to derive from, so
+    # `default_tool_timeout` is their ceiling. `lease_margin` (validated > 0)
+    # guarantees lease > the resolved timeout.
+    #
+    # Residual (documented): an in-process/custom tool that legitimately runs
+    # longer than `default_tool_timeout` MAY be reclaimed → a duplicate invocation.
+    # That is harmless — the DB fence yields exactly one canonical result and tools
+    # carry the stable `solid_loop:tool_call:#{id}` idempotency key — but operators
+    # who run a slow in-process tool must raise `default_tool_timeout` above it.
+    def tool_lease_duration(agent = nil)
+      max_http = max_http_tool_timeout(agent)
+      [ default_tool_timeout, max_http ].compact.max + lease_margin
+    end
+
+    private
+
+    # The slowest resolved HTTP MCP client timeout across the agent's `mcps`, or
+    # nil if the agent exposes none / cannot be inspected. Matches the resolution
+    # in Mcp::ClientFactory#transport_for (`mcp_config[:timeout] || 60` for a url:
+    # entry). Defensive: a raising/oddly-shaped `mcps` yields nil (fall back to
+    # `default_tool_timeout`) rather than breaking the claim.
+    def max_http_tool_timeout(agent)
+      return nil unless agent.respond_to?(:mcps)
+
+      timeouts = Array(agent.mcps).filter_map do |mcp_config|
+        next unless mcp_config.is_a?(Hash)
+        next unless mcp_config[:url] # only HTTP-sugar entries carry a client timeout
+        next if mcp_config[:toolset] || mcp_config[:transport]
+
+        Integer(mcp_config[:timeout] || 60)
+      end
+      timeouts.max
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/solid_loop/engine.rb

@@ -0,0 +1,14 @@
+require_relative "pipeline/context"
+require_relative "pipeline/tool_context"
+require_relative "pipeline"
+require_relative "llm_metrics"
+
+module SolidLoop
+  class Engine < ::Rails::Engine
+    isolate_namespace SolidLoop
+
+    initializer "solid_loop.mime_types" do
+      Mime::Type.register "text/vnd.turbo-stream.html", :turbo_stream unless Mime[:turbo_stream]
+    end
+  end
+end

data/lib/solid_loop/lease_heartbeat.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module SolidLoop
+  # A thin per-turn HANDLE over the process-wide {SolidLoop::LeaseRenewer}
+  # (docs/decisions/durable_attempt_lease.md). It preserves the
+  # exact public surface the LLM path relies on (`run`, `start`, `stop`, `lost?`,
+  # `check!`) so callers (`LlmCompletionJob`, `ResponseParsing`) are unchanged.
+  #
+  # WHY it no longer owns a thread/connection: the old design spawned ONE renewer
+  # thread per in-flight turn and each renew tick checked out its own pool
+  # connection. At full worker-pool occupancy (pool size == worker concurrency, the
+  # standard Rails shape) every connection is held by a busy worker, so every
+  # heartbeat blocked in `with_connection` and — after `checkout_timeout` — treated
+  # pool exhaustion as a lost lease, yielding healthy turns → the reaper requeued
+  # them into the same saturated pool → LIVELOCK. Renewal is now O(1) in
+  # connections: the shared `LeaseRenewer` renews ALL registered leases from ONE
+  # dedicated connection, so a per-turn renew never competes with workers again.
+  #
+  # This handle registers the turn on `start` and deregisters on `stop`. The
+  # ownership contract is UNCHANGED and now lives in the renewer: each renew is a
+  # conditional UPDATE keyed on `loop_id + execution_token`, gated by
+  # `lease_expires_at IS NOT NULL` (never resurrects a NULLed tool-phase lease); a
+  # genuine 0-row loss (rotated away / reclaimed) flips a per-registration lost
+  # flag; the main path checks `lost?`/`check!` before any canonical write; `stop`
+  # deregisters BEFORE the canonical commit and again in `ensure`.
+  class LeaseHeartbeat
+    # Kept for source/back-compat; the cadence now lives in the renewer.
+    MIN_INTERVAL = SolidLoop::LeaseRenewer::MIN_INTERVAL
+
+    def self.run(loop_id:, execution_token:, lease_duration:, max_duration: nil, logger: nil)
+      hb = new(loop_id: loop_id, execution_token: execution_token,
+               lease_duration: lease_duration, max_duration: max_duration, logger: logger)
+      hb.start
+      begin
+        yield hb
+      ensure
+        # Deregister BEFORE the caller's canonical completion so the two never race
+        # on lease_expires_at. Idempotent.
+        hb.stop
+      end
+    end
+
+    attr_reader :loop_id, :execution_token, :lease_duration, :max_duration
+
+    def initialize(loop_id:, execution_token:, lease_duration:, max_duration: nil, logger: nil)
+      @loop_id         = loop_id
+      @execution_token = execution_token
+      @lease_duration  = lease_duration.to_f
+      # The owning agent's legal turn bound, forwarded to the renewer to size the
+      # leak ceiling (max_duration + grace). A legitimate turn — capped at
+      # max_duration by the agent's own contract — is therefore never dropped.
+      @max_duration    = max_duration
+      @logger          = logger || (defined?(Rails) ? Rails.logger : nil)
+      @renewer         = SolidLoop::LeaseRenewer.instance
+      @key             = nil
+    end
+
+    # Register this turn with the shared renewer. Idempotent.
+    def start
+      return self if @key
+
+      @key = @renewer.register(
+        loop_id: @loop_id, execution_token: @execution_token,
+        lease_duration: @lease_duration, max_duration: @max_duration
+      )
+      self
+    end
+
+    # True once a conditional renew for THIS turn touched 0 rows (the lease is no
+    # longer ours). Thread-safe; readable from the main path between chunks.
+    def lost?
+      return false unless @key
+
+      @renewer.lost?(@key)
+    end
+
+    # Raise if the heartbeat has lost the lease. Call this on the main path
+    # immediately before any canonical write/checkpoint.
+    def check!
+      raise SolidLoop::LostLease, "lease for loop #{@loop_id} lost (generation rotated away)" if lost?
+    end
+
+    # Idempotent. Deregisters this turn from the shared renewer so no further renew
+    # fires for it. The renewer thread and its connection are process-scoped and
+    # outlive individual turns by design (that is the whole point of the shared
+    # renewer), so `stop` here does NOT tear down the thread — it only unregisters.
+    def stop
+      key  = @key
+      @key = nil
+      @renewer.deregister(key) if key
+      nil
+    end
+  end
+end

data/lib/solid_loop/lease_renewer.rb

@@ -0,0 +1,347 @@
+# frozen_string_literal: true
+
+module SolidLoop
+  # A SINGLE per-process background service that renews ALL in-flight LLM-turn
+  # leases from ONE dedicated DB connection
+  # (docs/decisions/durable_attempt_lease.md).
+  #
+  # WHY a shared renewer (not a heartbeat-per-turn): the previous design spawned
+  # one background thread PER in-flight turn, and each renew tick checked out its
+  # own connection via `with_connection`. At full worker-pool occupancy — the
+  # STANDARD Rails deployment shape, where the DB pool size == worker concurrency —
+  # every pooled connection is held by a busy worker, so EVERY heartbeat blocks in
+  # `with_connection` waiting for a connection that never frees. After
+  # `checkout_timeout` each heartbeat treats pool exhaustion as a lost lease → the
+  # turn yields uncommitted → the reaper requeues into the same saturated pool →
+  # livelock. No generation could complete while saturated.
+  #
+  # The fix makes renewal O(1) in connections, independent of worker concurrency:
+  # ONE renewer thread owns ONE dedicated connection (checked out for the process
+  # lifetime, NOT returned to the shared pool between ticks) and batch-renews every
+  # registered lease each tick. A `LlmCompletionJob` REGISTERS its
+  # `(loop_id, execution_token, lease_duration)` on start and DEREGISTERS in
+  # `ensure`; renewal never competes with workers for a connection again.
+  #
+  # Leak safety (does NOT depend on `ensure` running): every registration also
+  # carries a HARD CEILING derived from the OWNING AGENT'S legal turn duration —
+  # once it has been renewed for longer than `max_duration + config.lease_leak_grace`,
+  # the renewer DROPS it and stops renewing. A leaked registration (a turn that
+  # somehow skipped its `ensure`) therefore self-expires shortly after `max_duration`,
+  # its lease lapses, and the reaper reclaims the loop — so a leaked registration can
+  # never strand a loop forever. Crucially, a LEGITIMATE long turn is NEVER dropped:
+  # the agent already caps a legal turn at `max_duration` (its own contract), so any
+  # turn within that bound completes before the ceiling. (The earlier
+  # `lease_duration × K` ceiling was WRONG: a healthy turn stays registered for its
+  # whole life, so any legitimate stream longer than K lease-widths — e.g. a tiny
+  # `read_timeout` with steady chunks — was dropped and reaper-reclaimed mid-flight,
+  # and the replacement hit the same ceiling, so no long turn could ever complete.)
+  #
+  # Ownership safety is unchanged from the old per-turn heartbeat: each registered
+  # lease is renewed with the SAME conditional UPDATE keyed on
+  # `loop_id + execution_token`, gated by `lease_expires_at IS NOT NULL` so a
+  # canonical commit that NULLed the lease (tool phase) is never resurrected. A
+  # 0-row renew that is genuinely lost (rotated away / reclaimed) flips a per-
+  # registration lost flag; a 0-row renew because the lease was legitimately
+  # cleared to NULL is NOT a loss. The main path checks the per-registration flag
+  # (`check!`) before any canonical write, exactly as before.
+  #
+  # Isolation & lifecycle: the renewer connection is checked out from the SHARED
+  # pool ONCE (one extra long-lived connection for the whole process — size the
+  # pool as `worker_concurrency + 1` so it never contends with a full worker set;
+  # documented in the README) and released only when the renewer stops. Drawing
+  # from the shared pool (rather than a separate one) keeps the renewer's view of
+  # the data identical to the workers' — including under transactional test
+  # fixtures, which tie every pooled connection to one test transaction. Each renew
+  # runs inside the Rails executor so it participates in query-cache/reload/
+  # connection management. The thread is lazily started on the first registration
+  # and INTENTIONALLY runs for the whole process lifetime (there is nothing to tear
+  # down between turns — registrations come and go, the one thread + one connection
+  # persist). Its single connection is released back to the pool at process exit
+  # when the thread unwinds. `stop!`/`reset_instance!` (used by tests to prove a
+  # clean exit) are idempotent, join the thread, and let the held connection return
+  # to the pool — no thread or connection is leaked (verified by the specs'
+  # clean-exit / baseline-thread-count assertions).
+  class LeaseRenewer
+    # Never renew slower than this, even for a tiny ttl, so a pathologically small
+    # lease does not spin the DB. Mirrors the old LeaseHeartbeat::MIN_INTERVAL.
+    MIN_INTERVAL = 1.0
+
+    # A single registered in-flight lease. `lost` is the exact per-turn signal the
+    # old heartbeat exposed: set true iff a conditional renew for THIS registration
+    # touched 0 rows AND the loop is no longer ours (rotated away / reclaimed). A
+    # renew error also flips it (a dead renewer must never let a dead lease look
+    # alive). It is a `Concurrent::AtomicBoolean` so the write (on the renewer
+    # thread) and the read (on the main path) carry a proper memory barrier on
+    # every Ruby engine, not only under MRI's GVL.
+    #
+    # `registered_at` stamps when this turn registered, used to enforce a HARD
+    # CEILING (see `#ceiling_expired?`): a leaked registration (a job that never
+    # deregistered) is dropped once it exceeds the OWNING AGENT'S OWN legal turn
+    # bound — `max_duration + config.lease_leak_grace` — so its lease finally lapses
+    # and the reaper reclaims the loop, and renewer liveness no longer depends on
+    # `ensure` ever running.
+    #
+    # `max_duration` is the agent's already-enforced legal turn duration (see
+    # `SolidLoop::Base#max_duration`, default 2h): a turn longer than that is
+    # illegitimate BY THE AGENT'S OWN CONTRACT. Deriving the ceiling from it (not
+    # from `lease_duration × K`) means a LEGITIMATE turn — bounded by `max_duration`
+    # — always finishes before the ceiling and is NEVER dropped, while a genuine leak
+    # still self-expires shortly after `max_duration`. (The old `lease_duration × K`
+    # ceiling dropped healthy long turns too: an agent with a tiny `read_timeout` and
+    # steady chunks streams healthily for far longer than K lease-widths.)
+    Registration = Struct.new(:loop_id, :execution_token, :lease_duration, :max_duration, :registered_at, :lost, keyword_init: true) do
+      def lost?
+        lost.true?
+      end
+
+      def mark_lost!
+        lost.make_true
+      end
+    end
+
+    # Guards singleton creation AND teardown so two concurrent first-turns can never
+    # construct DISTINCT renewers (BLOCKER — the class-var `||=` is not atomic: two
+    # threads could both see `@instance == nil`, each `new` a renewer, each start a
+    # thread and try to check out the one process-lifetime connection; whichever lost
+    # the class-var assignment would be unreachable from `instance`/`reset_instance!`
+    # yet keep its thread — and possibly the ONLY reserved connection — running
+    # forever, reintroducing the livelock). All access to `@instance` goes through
+    # this constant mutex; the actual `stop!` runs OUTSIDE the lock so we never hold
+    # it across a thread join.
+    MUTEX = Mutex.new
+
+    class << self
+      def instance
+        MUTEX.synchronize { @instance ||= new }
+      end
+
+      # Reset the process singleton (tests only): stop the running renewer and drop
+      # the memoized instance so the next registration starts a fresh one. Idempotent.
+      # The swap-to-nil is atomic under MUTEX; the (blocking, thread-joining) `stop!`
+      # runs OUTSIDE the lock so a concurrent `instance` never waits on a join.
+      def reset_instance!
+        inst = MUTEX.synchronize do
+          taken = @instance
+          @instance = nil
+          taken
+        end
+        inst&.stop!
+      end
+    end
+
+    def initialize(logger: nil)
+      @logger        = logger || (defined?(Rails) ? Rails.logger : nil)
+      @registrations = {} # key => Registration
+      @mutex         = Mutex.new
+      @wake          = Concurrent::Event.new
+      @stopping      = Concurrent::AtomicBoolean.new(false)
+      @thread        = nil
+      @tick_interval = MIN_INTERVAL
+    end
+
+    # Register an in-flight LLM turn. Returns an opaque key used to query the
+    # per-registration lost flag and to deregister. Starts the shared renewer
+    # thread on first use. The cadence is driven by the SMALLEST registered lease
+    # (~ttl/4), so a short-lived lease is still renewed in time.
+    #
+    # `max_duration` (the owning agent's already-enforced legal turn bound) sizes the
+    # per-registration leak ceiling (`max_duration + config.lease_leak_grace`). A
+    # legitimate turn — bounded by `max_duration` — always finishes before it; a
+    # genuine leak self-expires shortly after. Defaults to `config.default_max_duration`
+    # so an out-of-band caller that omits it still gets a bounded ceiling.
+    def register(loop_id:, execution_token:, lease_duration:, max_duration: nil)
+      key = [ loop_id, execution_token ]
+      reg = Registration.new(
+        loop_id: loop_id, execution_token: execution_token,
+        lease_duration: lease_duration.to_f,
+        max_duration: (max_duration || SolidLoop.config.default_max_duration).to_f,
+        registered_at: Time.current,
+        lost: Concurrent::AtomicBoolean.new(false)
+      )
+      @mutex.synchronize do
+        @registrations[key] = reg
+        recompute_interval
+      end
+      ensure_running
+      @wake.set # renew promptly so a just-registered lease is refreshed without waiting a full tick
+      key
+    end
+
+    # Deregister an in-flight turn (called from the job's `ensure`). Idempotent.
+    def deregister(key)
+      @mutex.synchronize do
+        @registrations.delete(key)
+        recompute_interval
+      end
+    end
+
+    # True once a conditional renew for this registration touched 0 rows and the
+    # lease is no longer ours. Thread-safe; readable from the main path between
+    # chunks. Missing key (already deregistered / ceiling-dropped) reads as not-lost.
+    def lost?(key)
+      @mutex.synchronize { @registrations[key]&.lost? } || false
+    end
+
+    # Whether `key` is still actively registered (renewed each tick). Drops to
+    # false on deregister OR once the hard ceiling drops a leaked registration.
+    # Thread-safe; used by specs to assert the ceiling behavior.
+    def registered?(key)
+      @mutex.synchronize { @registrations.key?(key) }
+    end
+
+    # Idempotent. Stops the renewer thread, joins it, and releases the dedicated
+    # connection. Called at process exit; also used by tests to prove a clean exit.
+    def stop!
+      @stopping.make_true
+      @wake.set
+      thread = nil
+      @mutex.synchronize do
+        thread   = @thread
+        @thread  = nil
+      end
+      begin
+        thread&.join
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        @logger&.error("[SolidLoop] lease renewer died: #{e.class}: #{e.message}")
+      end
+      # The held connection is released back to the shared pool when the renewer
+      # thread's `with_connection` block unwinds on stop — no connection leaks.
+      nil
+    end
+
+    private
+
+    def ensure_running
+      return if @stopping.true?
+
+      @mutex.synchronize do
+        return if @thread&.alive?
+
+        @stopping.make_false
+        @thread = Thread.new do
+          Thread.current.name = "solid-loop-lease-renewer"
+          run_loop
+        end
+      end
+    end
+
+    # Recompute the tick cadence from the smallest registered lease (~ttl/4),
+    # floored at MIN_INTERVAL. Caller holds @mutex.
+    def recompute_interval
+      smallest = @registrations.values.map(&:lease_duration).min
+      @tick_interval = smallest ? [ smallest / 4.0, MIN_INTERVAL ].max : MIN_INTERVAL
+    end
+
+    def run_loop
+      # ONE connection for the WHOLE renewer lifetime. Checked out ONCE here (not
+      # per tick) and held until the renewer stops. This is the BLOCKER fix:
+      # renewal's connection demand is now O(1) — a SINGLE checkout for ALL
+      # in-flight leases — instead of O(worker_concurrency) (the old design's
+      # one-thread-and-one-checkout-per-turn, which at full pool occupancy had
+      # every heartbeat blocking in `with_connection` → treating pool exhaustion as
+      # a lost lease → livelock). Because it is drawn from the SAME pool as the
+      # workers, it correctly participates in transactional test fixtures (which
+      # tie all pooled connections to one test transaction) and sees the same
+      # committed data any worker does. Operators size the pool as
+      # `worker_concurrency + 1` so this one long-lived connection never contends
+      # with a full worker set (documented in the README durability section).
+      SolidLoop::Loop.connection_pool.with_connection do
+        until @stopping.true?
+          interval = @mutex.synchronize { @tick_interval }
+          # Wait up to one tick; a register/deregister/stop wakes us immediately.
+          # Reset BEFORE the stop-check so a set that arrives during renew_all is
+          # not lost: worst case an extra (harmless, idempotent) renew pass.
+          @wake.wait(interval)
+          @wake.reset
+          break if @stopping.true?
+
+          renew_all
+        end
+      end
+    rescue StandardError => e
+      # A renewer-thread death must NEVER let dead leases look alive: mark every
+      # registered turn lost so each main path fails visibly.
+      @logger&.error("[SolidLoop] lease renewer loop error: #{e.class}: #{e.message}")
+      @mutex.synchronize { @registrations.each_value(&:mark_lost!) }
+    end
+
+    # Batch-renew every currently registered lease on the renewer's held
+    # connection, inside the Rails executor. A snapshot of the registrations is
+    # taken under the mutex so a concurrent register/deregister never mutates the
+    # list mid-iteration; the lost flag is written back onto the live Registration
+    # object (safe — a deregistered key's write is simply never read).
+    def renew_all
+      executor = defined?(Rails) ? Rails.application&.executor : nil
+      if executor
+        executor.wrap { renew_each }
+      else
+        renew_each
+      end
+    end
+
+    def renew_each
+      now = Time.current
+
+      # HARD CEILING (finding-1 at this layer): drop any registration older than the
+      # owning agent's legal turn bound (`max_duration + config.lease_leak_grace`)
+      # even if it was never deregistered. A dropped registration is no longer
+      # renewed, so its lease lapses and the reaper (case 1) reclaims the loop —
+      # renewer liveness never depends on `ensure` running. The drop happens under the
+      # mutex; the snapshot renewed below excludes the dropped entries. A LEGITIMATE
+      # turn (bounded by its agent's `max_duration`) always finishes before the
+      # ceiling, so it is never dropped.
+      snapshot = @mutex.synchronize do
+        leaked = @registrations.select { |_key, reg| ceiling_expired?(reg, now) }
+        if leaked.any?
+          leaked.each_key { |key| @registrations.delete(key) }
+          recompute_interval
+          @logger&.warn(
+            "[SolidLoop] lease renewer dropped #{leaked.size} leaked registration(s) " \
+            "past the renew ceiling (loops #{leaked.values.map(&:loop_id).inspect}); " \
+            "their leases will lapse and be reaped."
+          )
+        end
+        @registrations.values.dup
+      end
+
+      snapshot.each { |reg| renew_one(reg, now) }
+    end
+
+    # True once a registration has been renewed for longer than the owning agent's
+    # OWN legal turn duration allows (`max_duration + config.lease_leak_grace`). Only
+    # a LEAKED registration (never deregistered) can reach this — a legitimate turn is
+    # bounded by `max_duration` and deregisters in its `ensure` far sooner.
+    def ceiling_expired?(reg, now)
+      ceiling = reg.max_duration + SolidLoop.config.lease_leak_grace
+      (now - reg.registered_at) > ceiling
+    end
+
+    # One conditional renew for a single registration — byte-for-byte the same
+    # ownership contract as the old per-turn heartbeat. Keyed on
+    # loop_id + execution_token, gated by `lease_expires_at IS NOT NULL` so a NULLed
+    # (tool-phase / ended) lease is never resurrected. A genuine 0-row loss flips
+    # the per-registration lost flag.
+    def renew_one(reg, now)
+      rows = SolidLoop::Loop
+        .where(id: reg.loop_id, status: "running", execution_token: reg.execution_token)
+        .where.not(lease_expires_at: nil)
+        .update_all(lease_expires_at: now + reg.lease_duration, updated_at: now)
+
+      return if rows.positive?
+
+      # 0 rows: distinguish "lost" (rotated away / reclaimed) from "already ended"
+      # (our own commit NULLed the lease). Only the former is a loss. Re-check: still
+      # running on our token ⇒ the 0-row renew was just the NULL-lease guard (a
+      # normal end-of-turn), NOT a lost lease.
+      still_ours = SolidLoop::Loop
+        .where(id: reg.loop_id, status: "running", execution_token: reg.execution_token)
+        .exists?
+      reg.mark_lost! unless still_ours
+    rescue StandardError => e
+      # A per-lease renew error is treated as a loss for THAT turn only (the others
+      # keep renewing). The main path will see the flag and fail visibly.
+      @logger&.error("[SolidLoop] lease renew error for loop #{reg.loop_id}: #{e.class}: #{e.message}")
+      reg.mark_lost!
+    end
+  end
+end

data/lib/solid_loop/llm_metrics.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SolidLoop
+  # Value object to hold raw timing and token metrics from the network layer.
+  LlmMetrics = Struct.new(
+    :ttft,                # Time to First Token
+    :duration_thinking,   # Time spent in reasoning/thought
+    :duration_generation, # Total time from request start
+    :tokens_completion,   # Estimated tokens during stream
+    keyword_init: true
+  ) do
+    def to_h
+      super.to_h.compact
+    end
+  end
+end

data/lib/solid_loop/mcp/call_context.rb

@@ -0,0 +1,19 @@
+module SolidLoop
+  module Mcp
+    # Caller identity handed to Transport#deliver. HTTP transports typically
+    # ignore it (auth travels in headers); in-process transports use it to
+    # reach domain state without a network hop.
+    #
+    # Two caller shapes (see docs/decisions/mcp-server.md):
+    #   in-process agent      — agent/loop set, principal from Base#mcp_principal
+    #   external via Server   — agent/loop nil, principal = authorizer's return
+    # Handlers must not assume agent/loop are present.
+    #
+    # idempotency_key: stable per-ToolCall key for tools/call dispatches (nil
+    # for non-tool traffic such as initialize/list). It also travels on the
+    # wire as `_meta["solidloop/idempotencyKey"]`; handlers of irreversible
+    # tools should dedupe on it because a crash between a remote side effect
+    # and SolidLoop's executed_at checkpoint can re-deliver the invocation.
+    CallContext = Struct.new(:agent, :loop, :principal, :idempotency_key, keyword_init: true)
+  end
+end