solid_loop 0.0.4

data/lib/solid_loop/pipeline/tool_context.rb

@@ -0,0 +1,53 @@
+module SolidLoop
+  class Pipeline
+    class ToolContext
+      # Inputs
+      attr_accessor :tool_call_id
+
+      # Loaded Context
+      attr_accessor :tool_call
+      attr_accessor :message
+      attr_accessor :loop
+      attr_accessor :agent
+      # Working State
+      attr_accessor :result
+      attr_accessor :duration
+      attr_accessor :is_success
+      attr_accessor :exception
+      attr_accessor :metadata # Hash for middlewares to share custom state
+      attr_accessor :execution_token
+      # The per-tool_call lease token this worker claimed. The SECOND
+      # fence, alongside execution_token: the canonical completion write
+      # re-validates BOTH under the row lock, so a stale worker whose tool lease
+      # was revoked by pause/reclaim writes nothing.
+      attr_accessor :lease_token
+
+      def initialize(tool_call_id:)
+        @tool_call_id = tool_call_id
+        @metadata = {}
+      end
+
+      def loop_active?
+        return true unless @loop
+        # Fail closed: a context without a generation token never owns the loop.
+        @loop.running? && @execution_token.present? && @loop.execution_token == @execution_token
+      end
+
+      def loop_active_fresh?
+        @loop.reload
+        loop_active?
+      end
+
+      def skip_execution!(result:, success: false, duration: 0.0)
+        @skip_execution = true
+        @result = result
+        @is_success = success
+        @duration = duration
+      end
+
+      def skip_execution?
+        @skip_execution == true
+      end
+    end
+  end
+end

data/lib/solid_loop/pipeline.rb

@@ -0,0 +1,40 @@
+require_relative "pipeline/builder"
+
+module SolidLoop
+  class Pipeline
+    attr_reader :middlewares
+
+    def initialize(middlewares = [])
+      @middlewares = middlewares
+    end
+
+    def call(env)
+      runner = build_runner(@middlewares.dup)
+      runner.call(env)
+    end
+
+    private
+
+    def build_runner(middlewares)
+      # Base case: the innermost block does nothing (or could raise NotImplementedError if expected to be handled)
+      runner = ->(env) { }
+
+      # Wrap from inside out so the first middleware wraps the second, etc.
+      middlewares.reverse_each do |middleware_class|
+        next_runner = runner
+        runner = ->(env) do
+          # Instantiate the middleware passing the next step in the chain
+          instance = middleware_class.new(next_runner)
+
+          # We check loop_active? at every jump to short-circuit if the UI cancelled the loop
+          active = env.respond_to?(:loop_active_fresh?) ? env.loop_active_fresh? : env.loop_active?
+          return unless active
+
+          instance.call(env)
+        end
+      end
+
+      runner
+    end
+  end
+end

data/lib/solid_loop/reaper.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+module SolidLoop
+  # Idempotent recovery service for durable LLM turns and tool calls
+  # (docs/decisions/durable_attempt_lease.md). Host installs the recurrence via
+  # SolidLoop::ReaperJob (the gem does NOT own a scheduler).
+  #
+  # Each case below is a short, row-locked transaction that RE-checks its
+  # triggering condition under the lock, so two racing reapers on distinct
+  # backends resolve to exactly one acting on any given row.
+  #
+  # Cases 1, 3, 4 cover the LLM path; case 2 (per-tool_call lease reclaim) covers
+  # the tool path via `reap_stalled_tool_calls`.
+  class Reaper
+    # DIAGNOSTIC return value: the counters below are a
+    # best-effort summary of what one `reap!` pass did, intended for logging /
+    # liveness metrics — NOT a stable, precise public API. In particular
+    # `cleaned_shell_loops` counts LOOPS whose orphaned `processing` shells were
+    # cleaned (incremented once per loop), not the number of shell ROWS cleaned;
+    # the name says so to avoid the earlier "shells" mis-read (a single loop can
+    # own several shells). Treat the struct as a diagnostic snapshot; the field set
+    # may grow across releases.
+    Result = Struct.new(
+      :requeued_dead_llm,   # case 1: dead LLM turns re-enqueued
+      :cleaned_shell_loops, # case 1 + backstop: LOOPS whose processing shells were failed+hidden (per-loop count)
+      :requeued_stalled_tools, # case 2: dead tool calls re-enqueued
+      :repaired_gaps,       # case 3: dropped-enqueue repairs
+      :requeued_stale_queue, # case 4: lost queue rows re-enqueued
+      keyword_init: true
+    ) do
+      def total
+        requeued_dead_llm + cleaned_shell_loops + requeued_stalled_tools + repaired_gaps + requeued_stale_queue
+      end
+    end
+
+    def initialize(now: Time.current)
+      @now = now
+      @result = Result.new(
+        requeued_dead_llm: 0,
+        cleaned_shell_loops: 0,
+        requeued_stalled_tools: 0,
+        repaired_gaps: 0,
+        requeued_stale_queue: 0
+      )
+    end
+
+    def call
+      reap_expired_llm_leases    # case 1
+      reap_stalled_tool_calls    # case 2
+      reap_dropped_enqueues      # case 3
+      reap_lost_queue_rows       # case 4
+      reap_orphaned_shells       # cleanup 6 backstop (paused/terminal loops)
+
+      SolidLoop.last_reaped_at = @now
+      @result
+    end
+
+    private
+
+    # Case 1 — `running` with an EXPIRED lease: the worker that held this LLM
+    # turn is presumed dead (a healthy worker's HTTP client would have raised
+    # before the lease — derived to outlive read_timeout — expired). Requeue a
+    # fresh LLM turn; if the loop is no longer a live LLM turn (paused/terminal/
+    # frozen/token gone), just clean up its orphaned processing shell.
+    def reap_expired_llm_leases
+      candidate_ids(status: "running").each do |loop_id|
+        loop = SolidLoop::Loop.find(loop_id)
+        loop.with_lock do
+          # with_lock reloads the row under FOR UPDATE; re-check the expiry so a
+          # second reaper (or a resume that already moved the loop on) does not
+          # double-act.
+          next unless loop.running?
+          next unless loop.lease_expires_at.present? && loop.lease_expires_at < @now
+
+          fail_and_hide_processing_shells!(loop)
+
+          if loop.admin_frozen?
+            # Frozen: leave status alone (admin intervention pending); the shell
+            # cleanup above is all that is safe. Clearing the stale lease avoids
+            # re-selecting this row every sweep.
+            loop.update!(lease_expires_at: nil)
+            @result.cleaned_shell_loops += 1
+          else
+            # Rotate the generation token AWAY (fences any late commit from the
+            # dead turn) and hand the loop back to a fresh LLM turn. The enqueue
+            # commits inside this same transaction (verified perform_later).
+            loop.update!(status: :queued, execution_token: nil, lease_expires_at: nil, error_message: nil)
+            SolidLoop.enqueue!(SolidLoop::LlmCompletionJob, loop.id)
+            @result.requeued_dead_llm += 1
+          end
+        end
+      end
+    end
+
+    # Case 2 — `running`, lease_expires_at NULL (tool phase, NOT an LLM turn),
+    # with an unresolved tool_call (executed_at IS NULL) whose per-tool lease is
+    # expired or absent: the worker that held that tool is presumed dead. Re-enqueue
+    # a ToolExecutionJob for it. HEALTHY SIBLINGS with a live lease are left
+    # untouched — that is the whole reason the per-tool lease exists (parallel
+    # tools are the shipped default).
+    #
+    # Reaper-vs-reaper dedup: under the loop row lock, CAS-claim the dead call's
+    # lease and hand the token to the job. Two reapers on distinct backends
+    # serialize on the loop lock; the loser's CAS gets 0 rows and does NOT
+    # re-enqueue — exactly one acts. The job adopts the reaper's lease iff it
+    # still holds it, and re-invokes with the SAME stable ToolCall key.
+    def reap_stalled_tool_calls
+      candidate_tool_phase_loop_ids.each do |loop_id|
+        loop = SolidLoop::Loop.find(loop_id)
+        loop.with_lock do
+          # Re-check the tool-phase condition under the lock: a resume/LLM claim
+          # that moved the loop on (lease set again, or left `running`) is not
+          # case 2 territory. A frozen loop awaits admin intervention.
+          next unless loop.running? && loop.lease_expires_at.nil?
+          next if loop.admin_frozen?
+
+          # Mirror the emission policy (resume! / ResponseCreation): a SEQUENTIAL
+          # agent has only the HEAD call in flight — reclaiming a non-head call
+          # too would stamp a live lease on it, making the head's later
+          # ResponseCreation chain-enqueue no-op (a stall). Parallel agents run
+          # every call at once, so reclaim each dead one independently. A missing
+          # agent falls back to reclaiming all (safe: the job ordering guard
+          # self-heals sequential order anyway).
+          agent = begin
+            loop.agent
+          rescue StandardError
+            nil
+          end
+          sequential = begin
+            agent&.sequential_tool_calls?
+          rescue StandardError
+            false
+          end
+          candidates = loop.unresolved_tool_calls.to_a
+          candidates = candidates.first(1) if sequential
+
+          candidates.each do |tool_call|
+            if tool_call.executed_at?
+              # Executed but its CONTINUATION was dropped (crash after the
+              # executed_at checkpoint, before the canonical response + successor
+              # committed). The tool BODY must NOT re-run; re-enqueue to finish the
+              # continuation. No lease to claim (executed calls carry none), so the
+              # dedup is ResponseCreation's idempotent `response_message` check —
+              # a duplicate re-enqueue is harmless.
+              SolidLoop.enqueue!(SolidLoop::ToolExecutionJob, tool_call.id, loop.execution_token)
+              @result.requeued_stalled_tools += 1
+            else
+              # Unexecuted: CAS-claim ONLY a lease that is free or expired; a
+              # live-leased (healthy, in-flight) sibling returns 0 rows and is
+              # skipped. Two racing reapers ⇒ the loser's CAS gets 0 rows ⇒ exactly
+              # one re-enqueue. Hand the reaper's token to the job so it adopts it.
+              reaper_token = SecureRandom.uuid
+              claimed = SolidLoop::ToolCall
+                .where(id: tool_call.id, executed_at: nil)
+                .where("lease_token IS NULL OR leased_until < ?", @now)
+                .update_all(lease_token: reaper_token, leased_until: @now + SolidLoop.config.tool_lease_duration(agent), updated_at: @now)
+              next if claimed.zero?
+
+              SolidLoop.enqueue!(SolidLoop::ToolExecutionJob, tool_call.id, loop.execution_token, reaper_token)
+              @result.requeued_stalled_tools += 1
+            end
+          end
+        end
+      end
+    end
+
+    # Loops that MIGHT hold a stalled tool: `running`, in tool phase (lease NULL),
+    # with at least one tool_call that is EITHER unexecuted with a dead/absent
+    # lease OR executed-but-not-yet-continued (executed_at set but no tool response
+    # message committed). Both disjuncts exclude calls that already have a response
+    # message, so a fully-resolved loop is NOT selected. The precise per-call
+    # decision (and whether the call is in the current unresolved tail) is still
+    # made under the loop lock in `reap_stalled_tool_calls`; this only narrows the
+    # scan. Explicit parens around the whole (dead-lease OR dropped-continuation)
+    # disjunction so operator precedence can't rot into a bug.
+    def candidate_tool_phase_loop_ids
+      no_response = <<~SQL.squish
+        NOT EXISTS (
+          SELECT 1 FROM solid_loop_messages r
+          WHERE r.loop_id = solid_loop_loops.id
+            AND r.role = 'tool'
+            AND r.tool_call_id = solid_loop_tool_calls.tool_call_id
+            AND r.id > solid_loop_tool_calls.message_id
+        )
+      SQL
+
+      SolidLoop::ToolCall
+        .joins(message: :loop)
+        .where(solid_loop_loops: { status: "running", lease_expires_at: nil })
+        .where(no_response)
+        .where(
+          "(solid_loop_tool_calls.executed_at IS NULL " \
+          "  AND (solid_loop_tool_calls.lease_token IS NULL OR solid_loop_tool_calls.leased_until < :now)) " \
+          "OR (solid_loop_tool_calls.executed_at IS NOT NULL)",
+          now: @now
+        )
+        .distinct
+        .pluck("solid_loop_loops.id")
+    end
+
+    # Case 3 — `running`, lease NULL (so NOT an in-flight LLM turn), and ZERO
+    # unresolved tool calls: the loop committed its LLM turn (cleared the lease)
+    # but the successor enqueue was lost, OR it landed in tool-phase with nothing
+    # left to do. Repair by handing it a fresh LLM turn.
+    #
+    # GUARD: a tool-phase loop that legitimately has unresolved tool calls is
+    # LEFT ALONE — it is waiting on a ToolExecutionJob and there is no
+    # tool lease yet, so reclaiming it is case 2's job. This method must only
+    # ever act when there are genuinely no unresolved tools.
+    def reap_dropped_enqueues
+      SolidLoop::Loop
+        .where(status: "running", lease_expires_at: nil)
+        .pluck(:id)
+        .each do |loop_id|
+          loop = SolidLoop::Loop.find(loop_id)
+          loop.with_lock do
+            next unless loop.running? && loop.lease_expires_at.nil?
+            # Never resurrect a frozen loop (admin intervention pending). This
+            # also stops case 1's frozen-clean branch — which nulls the lease —
+            # from being picked up here and requeued a sweep later.
+            next if loop.admin_frozen?
+            # Under the lock, re-confirm there is nothing in flight and nothing
+            # to resolve. A non-empty set => tool phase => case 2's territory.
+            next if loop.unresolved_tool_calls.exists?
+
+            loop.update!(status: :queued, execution_token: nil, error_message: nil, lease_expires_at: nil)
+            SolidLoop.enqueue!(SolidLoop::LlmCompletionJob, loop.id)
+            @result.repaired_gaps += 1
+          end
+        end
+    end
+
+    # Case 4 — `queued` longer than the configured threshold: its enqueued
+    # LlmCompletionJob was probably lost. Re-enqueue; the claim CAS makes a
+    # duplicate delivery a harmless no-op.
+    #
+    # Cleanup 7 — "two racing reapers ⇒ exactly one acts" made true for case 4:
+    # BUMP `updated_at` to now inside the same row-locked txn BEFORE the enqueue.
+    # The second reaper serializes on the loop lock and then re-reads
+    # `updated_at < threshold`, which is now false, so it does NOT re-enqueue.
+    # (The claim CAS would already dedupe the LlmCompletionJob delivery, but the
+    # bump makes the reaper-vs-reaper resolution hold at the reaper layer too, so
+    # the ADR's "one winner" statement is literally true — not merely
+    # "duplicate-but-harmless".)
+    def reap_lost_queue_rows
+      threshold = @now - SolidLoop.config.queued_reap_threshold
+
+      SolidLoop::Loop
+        .where(status: "queued")
+        .where("updated_at < ?", threshold)
+        .pluck(:id)
+        .each do |loop_id|
+          loop = SolidLoop::Loop.find(loop_id)
+          loop.with_lock do
+            next unless loop.queued? && loop.updated_at < threshold
+
+            # Marker bump first: fences the losing reaper's re-check above.
+            loop.update_column(:updated_at, @now)
+            SolidLoop.enqueue!(SolidLoop::LlmCompletionJob, loop.id)
+            @result.requeued_stale_queue += 1
+          end
+        end
+    end
+
+    # Cleanup 6 backstop — pause / admin-freeze / force-stop already SYNCHRONOUSLY
+    # fail+hide their `processing` assistant shells, but a hard kill between the
+    # status transition and that cleanup (or a legacy row) could leave a paused/
+    # terminal loop holding a hidden `processing` shell forever. Cases 1–4 only
+    # scan `running` loops, so this sweep catches the paused/terminal ones. It is
+    # idempotent (a shell already failed is not re-selected) and row-locked so two
+    # reapers don't double-write.
+    def reap_orphaned_shells
+      loop_ids = SolidLoop::Message
+        .joins(:loop)
+        .where(role: "assistant", status: "processing")
+        .where(solid_loop_loops: { status: %w[paused completed failed] })
+        .distinct
+        .pluck("solid_loop_loops.id")
+
+      loop_ids.each do |loop_id|
+        loop = SolidLoop::Loop.find(loop_id)
+        loop.with_lock do
+          # Re-check under the lock: a resume could have moved the loop back to
+          # `running` (its shell is legitimately in flight again) between the scan
+          # and the lock — leave those alone.
+          next if loop.running? || loop.queued? || loop.init?
+
+          shells = loop.messages.where(role: "assistant", status: "processing")
+          next unless shells.exists?
+
+          shells.find_each { |shell| shell.update!(status: "failed", is_hidden: true) }
+          @result.cleaned_shell_loops += 1
+        end
+      end
+    end
+
+    # Fail + hide any `processing` assistant/streaming shell for the loop,
+    # preserving partial content for audit. Force-stop/reclaim must never mark a
+    # partial `success` (born-hidden shell protocol).
+    def fail_and_hide_processing_shells!(loop)
+      loop.messages.where(role: "assistant", status: "processing").find_each do |shell|
+        shell.update!(status: "failed", is_hidden: true)
+      end
+    end
+
+    def candidate_ids(status:)
+      SolidLoop::Loop
+        .where(status: status)
+        .where("lease_expires_at < ?", @now)
+        .pluck(:id)
+    end
+  end
+end

data/lib/solid_loop/version.rb

@@ -0,0 +1,3 @@
+module SolidLoop
+  VERSION = "0.0.4"
+end

data/lib/solid_loop.rb

@@ -0,0 +1,94 @@
+require "solid_loop/version"
+require "solid_loop/engine"
+require "solid_loop/mcp_client"
+require "solid_loop/configuration"
+require "solid_loop/lease_renewer"
+require "solid_loop/lease_heartbeat"
+require "solid_loop/reaper"
+
+module SolidLoop
+  class CancellationError < StandardError; end
+  class LimitExceededError < StandardError; end
+  # Raised on the main LLM-turn path when the background lease heartbeat has
+  # LOST the lease (a conditional renew touched 0 rows — the generation token
+  # rotated away via pause/stop/reclaim, or the reaper already reclaimed the
+  # turn). The main path must raise this BEFORE any canonical write so a worker
+  # that no longer owns the loop never commits a turn. See LeaseHeartbeat.
+  class LostLease < StandardError; end
+  # Raised when a steering operation is invalid: enqueue on a terminal loop, or
+  # edit/cancel of an already-delivered (or non-steering) message.
+  class SteeringError < StandardError; end
+  # Raised when a successor job could not be enqueued (ActiveJob `perform_later`
+  # returned a non-`successfully_enqueued?` job — e.g. a `before_enqueue`
+  # callback did `throw :abort`). Every dispatch site runs inside a DB
+  # transaction, so raising here rolls back the enclosing state change instead
+  # of committing it with no successor job to carry the loop forward.
+  class EnqueueError < StandardError; end
+
+  class << self
+    # Enqueue a successor job, raising unless ActiveJob confirms it was actually
+    # enqueued. `perform_later` returns `false` when an enqueue callback aborts;
+    # left unchecked, callers would commit state (a queued loop, a persisted
+    # tool-call row) with no job to advance it. Returns the enqueued job.
+    def enqueue!(job_class, *args)
+      job = job_class.perform_later(*args)
+
+      unless job.is_a?(ActiveJob::Base) && job.successfully_enqueued?
+        raise EnqueueError,
+          "Failed to enqueue #{job_class.name}(#{args.map(&:inspect).join(', ')}): " \
+          "perform_later returned #{job.inspect} (enqueue aborted?)"
+      end
+
+      job
+    end
+
+    # Optional host-app hook for the admin dashboard to deep-link a loop's
+    # polymorphic subject (which the engine can't route to on its own). A
+    # callable taking the loop and returning a URL string, or a Hash
+    # { url:, label: }. Left nil by default — the admin degrades to plain text.
+    attr_accessor :subject_resolver
+
+    # Durable-lease + reaper configuration (see docs/decisions/durable_attempt_lease.md).
+    def config
+      @config ||= Configuration.new
+    end
+
+    # Reap dead/lost LLM turns and repair lost queue rows. Idempotent; safe to
+    # run concurrently (each case re-checks its condition under a row lock).
+    # Returns the SolidLoop::Reaper::Result. Host installs the recurrence via
+    # SolidLoop::ReaperJob — the gem does NOT own a scheduler.
+    def reap!(now: Time.current)
+      Reaper.new(now: now).call
+    end
+
+    # Timestamp of the last successful `reap!` — a liveness health signal for
+    # hosts to alert on ("has the reaper run recently?"). nil until the first
+    # reap completes. Process-local (each worker tracks its own last run).
+    attr_accessor :last_reaped_at
+
+    def llm_middlewares
+      @llm_middlewares ||= Pipeline::Builder.new([
+        Middlewares::ErrorHandling,
+        Middlewares::AgentInitialization,
+        Middlewares::MessageBuilding,
+        Middlewares::EventLogging,
+        Middlewares::NetworkCalling,
+        Middlewares::ResponseParsing
+      ])
+    end
+
+    def tool_middlewares
+      @tool_middlewares ||= Pipeline::Builder.new([
+        ToolMiddlewares::ErrorHandling,
+        ToolMiddlewares::AgentInitialization,
+        ToolMiddlewares::EventLogging,
+        ToolMiddlewares::ToolExecution,
+        ToolMiddlewares::ResponseCreation
+      ])
+    end
+
+    def configure
+      yield self
+    end
+  end
+end