solid_loop 0.0.4

data/app/services/solid_loop/tool_middlewares/agent_initialization.rb

@@ -0,0 +1,17 @@
+module SolidLoop
+  module ToolMiddlewares
+    class AgentInitialization
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        env.agent = env.loop.agent
+
+        SolidLoop::McpSessionInitializer.call(env.loop, env.agent, execution_token: env.execution_token)
+
+        @app.call(env)
+      end
+    end
+  end
+end

data/app/services/solid_loop/tool_middlewares/error_handling.rb

@@ -0,0 +1,66 @@
+module SolidLoop
+  module ToolMiddlewares
+    class ErrorHandling
+      def initialize(app)
+        @app = app
+      end
+
+      # Errors the tool path terminalizes in-line (loop → failed) and then
+      # SWALLOWS (never re-raised), so ActiveJob does not retry a durable,
+      # already-recorded failure. Everything else re-raises to the job-level
+      # reconciliation wrapper (findings 4/9), which fails the loop under the
+      # BOTH-token fence — closing the "stranded running" gap for a renamed
+      # agent method or a middleware bug. A CancellationError is fenceless: it
+      # fires precisely when this generation already lost the token (pause/stop).
+      GRACEFUL_ERRORS = [
+        SolidLoop::CancellationError,
+        SolidLoop::McpError
+      ].freeze
+
+      def call(env)
+        begin
+          @app.call(env)
+          # Re-raise if exception was caught by inner layer but left in env
+          raise env.exception if env.exception
+        rescue StandardError => e
+          raise e unless graceful_error?(e)
+
+          handle_exception(env, e)
+          # Swallowed: expose it for outer inspection (tests) but do NOT re-raise
+          # — the failure is already durable in the DB.
+          env.exception = e
+        end
+      end
+
+      private
+
+      def graceful_error?(e)
+        GRACEFUL_ERRORS.any? { |klass| e.is_a?(klass) }
+      end
+
+      def handle_exception(env, e)
+        # A cancellation fires when this generation lost the token (pause/stop):
+        # the reconciler short-circuits it (the new generation owns both loop and
+        # call). Everything else terminalizes through the ONE shared helper so this
+        # graceful path and the job-level reconciler share EXACTLY the semantics.
+        # Finding 9, failure path: the helper does the BOTH-token gate ITSELF under
+        # the loop→tool_call row locks and only fails the loop when both tokens
+        # validate, so a stale worker whose per-tool lease was rotated away (its
+        # lease expired while alive, reaper case 2 reclaimed the ToolCall) never
+        # terminalizes the loop the CURRENT owner is running. `transition_status`'s
+        # block CANNOT do this — it performs the loop update after yielding
+        # regardless, so a `next` on lease mismatch skipped only the ToolCall write.
+        return if e.is_a?(SolidLoop::CancellationError)
+        return unless env.loop
+
+        SolidLoop::ToolFailureReconciler.reconcile!(
+          loop_model:      env.loop,
+          tool_call:       env.tool_call,
+          execution_token: env.execution_token,
+          lease_token:     env.lease_token,
+          error:           e
+        )
+      end
+    end
+  end
+end

data/app/services/solid_loop/tool_middlewares/event_logging.rb

@@ -0,0 +1,27 @@
+module SolidLoop
+  module ToolMiddlewares
+    class EventLogging
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        @app.call(env)
+
+        if env.metadata[:mcp_log].present? || env.metadata[:mcp_request_data].present?
+          SolidLoop::Event.create!(
+            loop_id: env.loop.id,
+            eventable: env.tool_call,
+            name: "mcp_tool_call",
+            http_method: "POST", # Simplified
+            url: env.metadata[:mcp_url],
+            request_data: env.metadata[:mcp_request_data],
+            response_data: env.metadata[:mcp_response_data],
+            duration: env.duration,
+            log: env.metadata[:mcp_log]
+          )
+        end
+      end
+    end
+  end
+end

data/app/services/solid_loop/tool_middlewares/response_creation.rb

@@ -0,0 +1,110 @@
+module SolidLoop
+  module ToolMiddlewares
+    class ResponseCreation
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        created_message = nil
+
+        # LOCK ORDER (the tool canonical-write fence): loop row → tool_call → message,
+        # continuing the order ToolExecution began. The LOOP row lock is held
+        # across BOTH the both-token validation AND the canonical response +
+        # continuation, so a pause/resume/stop (which only takes the loop lock)
+        # cannot slip between validation and the write: with the loop FOR
+        # UPDATE-locked, `loop.execution_token` and `tool_call.lease_token` are
+        # frozen for the whole write. A stale/revoked worker writes NOTHING —
+        # no tool response message, no completion transition, no successor job.
+        #
+        # Response persistence, state advancement and successor-job dispatch
+        # commit atomically inside this transaction (SolidLoop requires a
+        # same-database transactional job backend): a crash or rollback leaves
+        # neither the state change nor a dangling job.
+        env.loop.with_lock do
+          env.tool_call.with_lock do
+          env.message.with_lock do
+          # Finding 9 — the canonical tool RESPONSE message + continuation is the
+          # second half of the tool completion write; fence it with BOTH tokens
+          # too. A worker whose generation was rotated or whose per-tool lease was
+          # revoked between ToolExecution and here must write no canonical
+          # response and dispatch no successor. The `executed_at` checkpoint
+          # (written under the tool_call lock in ToolExecution, itself fenced)
+          # remains the terminal exactly-once guard; this guards the message +
+          # continuation half against the same stale-worker race.
+          return unless canonical_continuation_owned?(env)
+
+          unless env.tool_call.response_message
+            created_message = env.loop.messages.create!(
+              role: "tool",
+              content: env.result,
+              tool_call_id: env.tool_call.tool_call_id,
+              status: "success"
+            )
+          end
+
+          # Explicit ordered scope — never rely on association default order.
+          tool_calls = env.message.tool_calls.ordered.to_a
+          response_ids = env.loop.messages
+            .where(role: :tool, tool_call_id: tool_calls.map(&:tool_call_id))
+            .where("id > ?", env.message.id)
+            .distinct
+            .pluck(:tool_call_id)
+
+          next_tool_call = tool_calls.find { |tool_call| !response_ids.include?(tool_call.tool_call_id) }
+          all_resolved = next_tool_call.nil?
+
+          # Sequential agents chain the next unresolved call here; parallel
+          # agents already enqueued every call up front, so each job only
+          # advances the loop once *all* results have landed. The completion
+          # gate is identical in both modes and fenced by the execution token,
+          # so whichever job commits the final result queues the next LLM turn.
+          sequential = (env.agent || env.loop.agent)&.sequential_tool_calls?
+
+          if sequential && next_tool_call
+            SolidLoop.enqueue!(SolidLoop::ToolExecutionJob, next_tool_call.id, env.execution_token)
+          elsif all_resolved && env.loop.transition_status(
+            from: :running,
+            to: :queued,
+            expected_execution_token: env.execution_token,
+            execution_token: nil,
+            lease_expires_at: nil
+          )
+            SolidLoop.enqueue!(SolidLoop::LlmCompletionJob, env.loop.id)
+          end
+          end # env.message.with_lock
+          end # env.tool_call.with_lock
+        end # env.loop.with_lock
+
+        if created_message && env.loop.observe_enabled?
+          SolidLoop::ObserveBroadcastJob.perform_later(created_message.id)
+        end
+
+        @app.call(env)
+      end
+
+      private
+
+      # BOTH-token ownership check for the canonical response + continuation.
+      # The loop row and tool_call row are already FOR
+      # UPDATE-locked by the enclosing `with_lock`s, so their columns are fresh
+      # AND frozen for the whole write — no `reload` (a reload is not a fence).
+      # Ownership requires the loop still `running` on THIS generation. The tool
+      # lease is validated as terminal-OR-mine: by the time control reaches here
+      # ToolExecution has cleared the lease on the exactly-once path (executed_at
+      # set, lease NULL) — that is a legitimately owned completion. A NON-terminal
+      # call whose lease is neither NULL nor ours means a competing worker holds
+      # it: bail.
+      def canonical_continuation_owned?(env)
+        return false if env.execution_token.blank?
+
+        loop_record = env.loop
+        return false unless loop_record.running? && loop_record.execution_token == env.execution_token
+
+        return true if env.tool_call.executed_at? && env.tool_call.lease_token.nil?
+
+        env.tool_call.lease_held_by?(env.lease_token)
+      end
+    end
+  end
+end

data/app/services/solid_loop/tool_middlewares/tool_execution.rb

@@ -0,0 +1,206 @@
+module SolidLoop
+  module ToolMiddlewares
+    class ToolExecution
+      def initialize(app)
+        @app = app
+      end
+
+      # LOCK ORDER (the tool canonical-write fence): loop row → tool_call → message,
+      # deterministic ascending id where several rows of one kind are locked.
+      # ToolExecution takes the first two rungs (loop, tool_call); ResponseCreation
+      # continues onto the message rung under the SAME loop lock. The both-token
+      # validation AND every canonical write are held under the LOOP row lock, so a
+      # pause/resume/stop (which only takes the loop lock) cannot slip between
+      # validation and the write: with the loop FOR UPDATE-locked,
+      # `loop.execution_token` and `tool_call.lease_token` cannot change between
+      # validation and mutation, so a stale/revoked worker writes NOTHING.
+      #
+      # CRITICAL: the tool BODY runs OUTSIDE the loop lock. A long-running tool
+      # holding the loop lock would serialize pause/stop/resume behind it — the
+      # very thing the fence exists to make safe. The side effect (at-least-once)
+      # may run even for a worker that ultimately loses the fence — that discarded
+      # effect is the documented at-least-once boundary (the stable ToolCall key
+      # dedupes). Only the validate-then-write step is locked.
+      def call(env)
+        # Terminal (executed_at) OR already has a canonical response (a crash
+        # between the response commit and the executed_at checkpoint): the tool
+        # BODY must NOT re-run. Take the fenced backfill+restore path only.
+        if env.tool_call.executed_at? || env.tool_call.response_message
+          resolve_from_checkpoint(env)
+          return @app.call(env)
+        end
+
+        # Run the tool body WITHOUT any loop lock held (a long tool must never
+        # serialize pause/stop/resume behind the loop lock). The side effect is
+        # at-least-once; the canonical write below is what the fence guards.
+        execute_once(env)
+
+        # Test-only failpoint (no-op in production): runs AFTER the tool body and
+        # BEFORE the loop lock is acquired, so a concurrency spec can commit a
+        # racing pause/stop in exactly this window and prove the subsequent
+        # under-lock validation fences the now-stale worker out.
+        env.metadata[:before_lock_acquire]&.call
+
+        env.loop.with_lock do
+          env.tool_call.with_lock do
+            # Re-check under the fence: a concurrent worker may have written the
+            # response/checkpoint while our body ran. If so, restore instead of
+            # writing a second canonical result.
+            backfill_checkpoint_from_response(env)
+
+            if env.tool_call.executed_at?
+              restore_checkpoint(env)
+              next
+            end
+
+            # The canonical `executed_at` + result write
+            # is fenced with BOTH tokens while the LOOP row is still locked, so a
+            # worker whose per-tool lease was revoked (pause/reclaim) or whose
+            # generation was rotated away writes NOTHING canonical. `executed_at`
+            # stays the terminal exactly-once guard.
+            unless canonical_write_owned?(env)
+              env.metadata[:tool_write_fenced_out] = true
+              return
+            end
+
+            # Test-only failpoint (no-op in production): a callable placed here
+            # runs INSIDE the loop lock, AFTER the both-token validation and
+            # BEFORE the canonical write, so a concurrency spec can attempt a
+            # racing pause/stop at exactly that interleaving and prove the loop
+            # row lock serializes it. Harmless when absent.
+            env.metadata[:after_validate_before_write]&.call
+
+            env.tool_call.update!({
+              is_success: env.is_success,
+              result: env.result,
+              duration: env.duration || 0.0,
+              executed_at: Time.current,
+              lease_token: nil,
+              leased_until: nil
+            }.merge(env.metadata.key?(:mcp_session_id) ? { mcp_session_id: env.metadata[:mcp_session_id] } : {}))
+            env.loop.increment!(:duration_tools, env.duration) if env.duration.to_f.nonzero?
+          end
+        end
+
+        @app.call(env)
+      end
+
+      private
+
+      # Terminal / already-responded path: no tool body ran. Backfill the
+      # executed_at checkpoint (fenced) if a response exists without one, then
+      # restore env.result/duration/is_success from the checkpoint. Under the same
+      # loop→tool_call lock order as the write path.
+      def resolve_from_checkpoint(env)
+        env.loop.with_lock do
+          env.tool_call.with_lock do
+            backfill_checkpoint_from_response(env)
+            restore_checkpoint(env)
+          end
+        end
+      end
+
+      # BOTH-token ownership check for the canonical tool write.
+      # Both the loop row and the tool_call row are FOR UPDATE-locked
+      # by the enclosing `with_lock`s, so `loop.execution_token` and the tool
+      # lease columns are fresh AND frozen for the duration of the write. Ownership
+      # requires: the loop is still `running` on THIS generation AND this worker
+      # still holds a live per-tool lease. No `reload` needed — the row is locked,
+      # not merely re-read (a reload is not a fence).
+      def canonical_write_owned?(env)
+        return false if env.execution_token.blank? || env.lease_token.blank?
+
+        loop_record = env.loop
+        return false unless loop_record.running? && loop_record.execution_token == env.execution_token
+
+        env.tool_call.lease_held_by?(env.lease_token)
+      end
+
+      # Cleanup 3 — the executed_at backfill is a canonical ToolCall mutation, so
+      # it runs under the SAME loop+tool_call locks and only when this generation
+      # still owns the loop. A stale worker must not backfill a checkpoint either.
+      # (It is a genuine no-op recovery — re-deriving the checkpoint from an
+      # already-committed response — but keeping it fenced means a revoked worker
+      # writes NOTHING to the ToolCall.)
+      def backfill_checkpoint_from_response(env)
+        return if env.tool_call.executed_at?
+        # Generation fence only — NO per-tool lease check here (both-token exempt):
+        # this backfill derives the checkpoint from an ALREADY-CANONICAL response
+        # message and invokes NO tool, so there is no at-least-once side effect to
+        # fence and nothing a stale lease could double-write. Requiring the loop to
+        # still own THIS generation is sufficient to keep a revoked worker from
+        # writing to the ToolCall; a live-lease requirement would only wrongly block
+        # the legitimate recovery of a response committed under an earlier lease.
+        return unless env.execution_token.present? &&
+                      env.loop.running? && env.loop.execution_token == env.execution_token
+
+        response = env.tool_call.response_message
+        return unless response
+
+        env.tool_call.update!(
+          executed_at: response.created_at,
+          result: env.tool_call.result.presence || response.content,
+          is_success: env.tool_call.is_success.nil? ? true : env.tool_call.is_success
+        )
+      end
+
+      def execute_once(env)
+        return if env.skip_execution?
+
+        mcp_tool = env.loop.mcp_tools.find_by(name: env.tool_call.function_name)
+        if mcp_tool
+          execute_mcp_tool(env, mcp_tool)
+        else
+          env.result = build_unknown_tool_message(env)
+          env.is_success = false
+          env.duration = 0.0
+        end
+      end
+
+      def restore_checkpoint(env)
+        env.result = env.tool_call.result
+        env.duration = env.tool_call.duration
+        env.is_success = env.tool_call.is_success
+      end
+
+      def execute_mcp_tool(env, mcp_tool)
+        result_hash = mcp_tool.call(
+          agent: env.agent,
+          arguments: env.tool_call.arguments,
+          tool_call: env.tool_call
+        )
+
+        env.result = result_hash[:result]
+        env.duration = result_hash[:duration]
+        env.is_success = result_hash[:success]
+
+        # Cleanup 3 — mcp_session_id is telemetry (which session served the call),
+        # not a canonical checkpoint. Do NOT write it here (unfenced): a stale
+        # worker whose lease was revoked would otherwise overwrite it. Stash it
+        # and let the FENCED canonical update below persist it, so only the owning
+        # worker records it.
+        env.metadata[:mcp_session_id] = mcp_tool.mcp_session_id
+
+        env.metadata[:mcp_request_data] = result_hash[:request_data]
+        env.metadata[:mcp_response_data] = result_hash[:response_data]
+        env.metadata[:mcp_log] = result_hash[:log]
+        env.metadata[:mcp_url] = "mcp://#{mcp_tool.mcp_session.mcp_name}/#{mcp_tool.name}"
+      end
+
+      def build_unknown_tool_message(env)
+        available_tools = env.loop.mcp_tools.pluck(:name).uniq.sort
+        tools_list = available_tools.any? ? available_tools.map { |name| "- #{name}" }.join("\n") : "- none"
+
+        <<~MSG.strip
+          Tool call rejected: `#{env.tool_call.function_name}`
+
+          I could not find a tool with that name in this loop.
+          Please pick one of the available tools below and try again.
+
+          Available tools:
+          #{tools_list}
+        MSG
+      end
+    end
+  end
+end