silas 0.1.1

data/lib/silas/ledger.rb

@@ -0,0 +1,178 @@
+module Silas
+  # The three-state tool executor — the exactly-once machinery the spike proved.
+  #
+  # settle!(step, resolver:) walks the step's invocations in creation order and
+  # drives each to a terminal state. Returns :completed when every invocation
+  # settled, or :parked when one is awaiting a human (approval or in-doubt).
+  #
+  # State machine per invocation:
+  #   pending  --approval fires--> pending + approval_state=required  (PARK)
+  #   pending  --transactional-->  completed        (tool + row in ONE txn)
+  #   pending  --at_most_once/idempotent--> started --> completed
+  #   started  --found on replay, at_most_once--> in_doubt            (PARK)
+  #   started  --found on replay, idempotent-->  re-run --> completed
+  #   completed --> no-op (replay feeds the persisted result forward)
+  #
+  # Concurrency: claims are compare-and-swap UPDATEs (status='pending' in the
+  # WHERE clause), so two executions racing the same invocation cannot both run
+  # the tool. The unique (step_id, tool_call_id) index backstops row creation.
+  module Ledger
+    GUARD_KEY = :silas_ledger_transaction
+
+    class << self
+      # True while a ledger transaction is open on this thread. A continuation
+      # checkpoint inside would raise Interrupt and roll back committed-looking
+      # progress (spike finding #5) — AgentLoopJob asserts against this.
+      def in_transaction? = Thread.current[GUARD_KEY] == true
+
+      def assert_no_checkpoint!
+        return unless in_transaction?
+
+        raise CheckpointInLedgerError,
+              "A continuation checkpoint was attempted inside a ledger transaction. " \
+              "Checkpoints raise Interrupt, which would roll back tool side effects " \
+              "the continuation believes are committed."
+      end
+
+      def settle!(step, resolver:)
+        step.tool_invocations.order(:id).each do |invocation|
+          outcome = settle_invocation!(invocation, resolver)
+          return :parked if outcome == :parked
+        end
+        :completed
+      end
+
+      # Drive a SINGLE freshly-created invocation to a terminal state — the
+      # :agent_sdk MCP endpoint creates one invocation per tools/call and needs
+      # exactly the same exactly-once/effect-mode machinery as settle!. Returns
+      # :done or :parked; the invocation carries its .result.
+      def execute_invocation!(invocation, resolver:)
+        settle_invocation!(invocation, resolver)
+      end
+
+      private
+
+      def settle_invocation!(invocation, resolver)
+        case invocation.status
+        when "completed", "failed" then :done
+        # approve! resets in_doubt -> pending+approved (re-execute); decline!
+        # resolves it to failed with an operator-supplied outcome. So a row
+        # still in_doubt here is by definition awaiting its human.
+        when "in_doubt" then :parked
+        when "started" then handle_in_doubt(invocation, resolver)
+        when "pending" then execute_pending(invocation, resolver)
+        end
+      end
+
+      def execute_pending(invocation, resolver)
+        return :parked if invocation.awaiting_approval?
+
+        tool = resolver.call(invocation.tool_name)
+
+        unless invocation.approval_state == "approved"
+          verdict = approval_verdict(tool, invocation)
+          case verdict
+          when :user_approval
+            invocation.update!(approval_state: "required",
+                               approval_expires_at: Silas.config.approval_ttl.from_now)
+            return :parked
+          when Hash # {denied: "reason"} — eve's shape
+            invocation.update!(status: "failed", result: { "denied" => verdict[:denied] })
+            return :done
+          end
+          # :not_applicable / :approved / :once-satisfied fall through
+        end
+
+        execute!(invocation, tool)
+      end
+
+      def execute!(invocation, tool)
+        tool.session = invocation.turn.session if tool.respond_to?(:session=)
+        args = invocation.arguments.symbolize_keys
+
+        if invocation.effect_mode == "transactional"
+          # Tool side effects + ledger completion commit or roll back together:
+          # exactly-once for DB-backed tools (spike cell C).
+          guarded_transaction do
+            next :done unless claim!(invocation, from: "pending", to: "completed")
+
+            result = tool.call(**args)
+            invocation.update!(result: wrap_result(result))
+          end
+        else
+          # External effects: mark started (committed), call, mark completed.
+          # A crash between the two commits is the in-doubt window.
+          return :done unless claim!(invocation, from: "pending", to: "started")
+
+          result = tool.call(**args)
+          invocation.update!(status: "completed", result: wrap_result(result))
+        end
+        :done
+      rescue StandardError => e
+        raise if e.is_a?(Silas::Error)
+
+        invocation.update!(status: "failed", error: "#{e.class}: #{e.message}")
+        :done
+      end
+
+      # started + replay = the tool may or may not have run (spike's in-doubt).
+      def handle_in_doubt(invocation, resolver)
+        if invocation.effect_mode == "idempotent"
+          tool = resolver.call(invocation.tool_name)
+          tool.session = invocation.turn.session if tool.respond_to?(:session=)
+          result = tool.call(**invocation.arguments.symbolize_keys)
+          invocation.update!(status: "completed", result: wrap_result(result))
+          :done
+        else
+          # at_most_once (default): park for a human verdict via the approval
+          # machinery — approve! = "it did not run, re-execute";
+          # decline! = "it ran / abandon", operator supplies the outcome.
+          invocation.update!(status: "in_doubt", approval_state: "required",
+                             approval_expires_at: Silas.config.approval_ttl.from_now)
+          :parked
+        end
+      end
+
+      def approval_verdict(tool, invocation)
+        policy = tool.respond_to?(:approval_policy) ? tool.approval_policy : :never
+        case policy
+        when :never then :not_applicable
+        when :always then :user_approval
+        when :once
+          previously_approved?(invocation) ? :approved : :user_approval
+        when Proc
+          policy.call(session: invocation.turn.session, input: invocation.arguments)
+        else
+          :not_applicable
+        end
+      end
+
+      def previously_approved?(invocation)
+        ToolInvocation.joins(:turn)
+                      .where(silas_turns: { session_id: invocation.turn.session_id },
+                             tool_name: invocation.tool_name, approval_state: "approved")
+                      .where.not(id: invocation.id)
+                      .exists?
+      end
+
+      # Compare-and-swap claim: only one racing execution wins.
+      def claim!(invocation, from:, to:)
+        claimed = ToolInvocation.where(id: invocation.id, status: from)
+                                .update_all(status: to, updated_at: Time.current) == 1
+        invocation.reload if claimed
+        claimed
+      end
+
+      def guarded_transaction
+        Thread.current[GUARD_KEY] = true
+        ApplicationRecord.transaction { yield }
+      ensure
+        Thread.current[GUARD_KEY] = false
+      end
+
+      def wrap_result(result)
+        result.is_a?(Hash) ? result : { "value" => result }
+      end
+    end
+  end
+end

data/lib/silas/mcp/client.rb

@@ -0,0 +1,86 @@
+require "json"
+require "net/http"
+require "securerandom"
+
+module Silas
+  module Mcp
+    # JSON-RPC 2.0 over MCP Streamable HTTP — the CLIENT half of Handler/Server,
+    # used by connections to call remote MCP servers. Stateless per operation
+    # (fresh initialize each call) so it's replay-safe. Net::HTTP only.
+    class Client
+      PROTOCOL_VERSION = "2025-06-18".freeze
+
+      def initialize(url:, headers: {}, open_timeout: 5, read_timeout: 30)
+        @uri = URI(url)
+        @headers = headers
+        @open_timeout = open_timeout
+        @read_timeout = read_timeout
+      end
+
+      def list_tools
+        with_session { |sid| rpc("tools/list", {}, session: sid).fetch("tools") }
+      end
+
+      # An MCP application error comes back as a normal result with isError=true
+      # (the remote tool ran and errored) — returned so the model can react. Only
+      # JSON-RPC/transport failures raise.
+      def call_tool(name, arguments)
+        with_session { |sid| rpc("tools/call", { "name" => name, "arguments" => arguments }, session: sid) }
+      end
+
+      private
+
+      def with_session
+        _, headers = post(rpc_body("initialize",
+                                   "protocolVersion" => PROTOCOL_VERSION, "capabilities" => {},
+                                   "clientInfo" => { "name" => "silas", "version" => Silas::VERSION }))
+        sid = headers["mcp-session-id"]
+        post(JSON.generate("jsonrpc" => "2.0", "method" => "notifications/initialized"), session: sid)
+        yield sid
+      end
+
+      def rpc(method, params, session:)
+        parsed, = post(rpc_body(method, params), session: session)
+        raise Error, "MCP #{method} error: #{parsed['error']}" if parsed && parsed["error"]
+
+        parsed && parsed["result"]
+      end
+
+      def rpc_body(method, params)
+        JSON.generate("jsonrpc" => "2.0", "id" => SecureRandom.uuid, "method" => method, "params" => params)
+      end
+
+      def post(body, session: nil)
+        req = Net::HTTP::Post.new(@uri)
+        req["content-type"] = "application/json"
+        req["accept"] = "application/json, text/event-stream"
+        req["mcp-protocol-version"] = PROTOCOL_VERSION
+        req["mcp-session-id"] = session if session
+        @headers.each { |k, v| req[k] = v } # secret injection lives here
+        req.body = body
+        res = http.request(req)
+        raise Error, "MCP HTTP #{res.code}" unless res.code.start_with?("2")
+
+        [ parse_payload(res), res.each_header.to_h ]
+      end
+
+      def parse_payload(res)
+        return nil if res.body.to_s.empty?
+
+        if res["content-type"].to_s.include?("text/event-stream")
+          data = res.body.each_line.select { |l| l.start_with?("data:") }.last
+          return data && JSON.parse(data.delete_prefix("data:").strip)
+        end
+        JSON.parse(res.body)
+      end
+
+      def http
+        @http ||= Net::HTTP.new(@uri.host, @uri.port).tap do |h|
+          h.use_ssl = @uri.scheme == "https"
+          h.open_timeout = @open_timeout
+          h.read_timeout = @read_timeout
+        end
+      end
+    end
+  end
+end

data/lib/silas/mcp/handler.rb

@@ -0,0 +1,89 @@
+require "json"
+require "securerandom"
+
+module Silas
+  module Mcp
+    # JSON-RPC handler for the hosted MCP endpoint. tools/call runs the tool
+    # THROUGH the Ledger, so the :agent_sdk path gets the same exactly-once and
+    # effect-mode semantics as :ruby_llm. Closes over one turn + its anchor step;
+    # authenticated by a per-turn bearer token in the URL query.
+    class Handler
+      TOOL_PREFIX = "mcp__silas__".freeze
+
+      def initialize(turn:, step:, tools:, resolver:, token:)
+        @turn = turn
+        @step = step
+        @tools = tools
+        @resolver = resolver
+        @token = token
+      end
+
+      # Returns [status, json_string_or_nil]. path/query already parsed by the server.
+      def call(path:, query_token:, body:)
+        return [ 404, error_body("not found") ] unless path == "/mcp/#{@turn.id}"
+        return [ 403, error_body("forbidden") ] unless ActiveSupport::SecurityUtils.secure_compare(query_token.to_s, @token)
+
+        msg = JSON.parse(body)
+        case msg["method"]
+        when "initialize"                then ok(msg["id"], initialize_result(msg))
+        when "notifications/initialized" then [ 202, nil ]
+        when "tools/list"                then ok(msg["id"], { "tools" => @tools.map { |d| mcp_tool(d) } })
+        when "tools/call"                then ok(msg["id"], call_tool(msg))
+        else rpc_error(msg["id"], -32601, "method not found: #{msg['method']}")
+        end
+      rescue JSON::ParserError
+        [ 400, error_body("bad json") ]
+      end
+
+      private
+
+      def call_tool(msg)
+        name = msg.dig("params", "name").to_s.sub(/\A#{Regexp.escape(TOOL_PREFIX)}/, "")
+        args = msg.dig("params", "arguments") || {}
+
+        invocation = Silas::ToolInvocation.create!(
+          step: @step, turn: @turn,
+          tool_call_id: SecureRandom.uuid, # Claude's tool_use.id isn't on the JSON-RPC msg; synth is fine (never fed to a provider)
+          tool_name: name, arguments: args, effect_mode: effect_mode_for(name)
+        )
+        outcome = Silas::Ledger.execute_invocation!(invocation, resolver: @resolver)
+        invocation.reload
+
+        if outcome == :parked
+          # v1 excludes approval-gated tools; if one slips through, fail loud.
+          { "isError" => true, "content" => [ text_content("approval-gated tools are not supported by :agent_sdk") ] }
+        else
+          { "content" => [ text_content(JSON.generate(invocation.result)) ] }
+        end
+      end
+
+      def effect_mode_for(name)
+        tool = @resolver.call(name)
+        tool.respond_to?(:effect_mode) ? tool.effect_mode.to_s : "at_most_once"
+      end
+
+      def mcp_tool(definition)
+        { "name" => definition["name"], "description" => definition["description"],
+          "inputSchema" => definition["input_schema"] }
+      end
+
+      def initialize_result(msg)
+        { "protocolVersion" => msg.dig("params", "protocolVersion") || "2025-06-18",
+          "capabilities" => { "tools" => {} },
+          "serverInfo" => { "name" => "silas", "version" => Silas::VERSION } }
+      end
+
+      def text_content(text) = { "type" => "text", "text" => text }
+
+      def ok(id, result)
+        [ 200, JSON.generate("jsonrpc" => "2.0", "id" => id, "result" => result) ]
+      end
+
+      def rpc_error(id, code, message)
+        [ 200, JSON.generate("jsonrpc" => "2.0", "id" => id, "error" => { "code" => code, "message" => message }) ]
+      end
+
+      def error_body(message) = JSON.generate("error" => message)
+    end
+  end
+end

data/lib/silas/mcp/server.rb

@@ -0,0 +1,134 @@
+require "socket"
+
+module Silas
+  module Mcp
+    # A minimal HTTP/1.1 server hosting the MCP Handler for the lifetime of one
+    # claude -p subprocess. Deliberately NOT Puma/Rack: the transport claude's
+    # MCP client needs (verified in the spike) is plain request/response JSON —
+    # one request per connection, application/json, no SSE — so a raw threaded
+    # TCPServer is fewer moving parts than embedding an app server in a worker.
+    #
+    # In-process: the Handler has direct access to the Ledger/models, so no
+    # cross-service call is needed and it works on any worker box.
+    class Server
+      attr_reader :port
+
+      def self.start(turn:, step:, tools:, resolver:, host: Silas.config.agent_sdk_mcp_host)
+        token = SecureRandom.hex(16)
+        turn.update_columns(mcp_token: token)
+        handler = Handler.new(turn: turn, step: step, tools: tools, resolver: resolver, token: token)
+        new(handler: handler, turn: turn, token: token, host: host).tap(&:boot)
+      end
+
+      def initialize(handler:, turn:, token:, host:)
+        @handler = handler
+        @turn = turn
+        @token = token
+        @host = host
+      end
+
+      def boot
+        @socket = TCPServer.new(@host, 0)
+        @port = @socket.addr[1]
+        @thread = Thread.new { accept_loop }
+        @thread.report_on_exception = false
+      end
+
+      def mcp_url = "http://#{@host}:#{@port}/mcp/#{@turn.id}?t=#{@token}"
+
+      # Health-check before spawning claude: a booting server that refuses the
+      # first connection gets marked "pending" by the CLI with no retry (spike trap).
+      def await_ready!(timeout: 5.0)
+        deadline = clock + timeout
+        loop do
+          TCPSocket.new(@host, @port).close
+          return true
+        rescue SystemCallError
+          raise Silas::Error, "MCP server did not become ready" if clock > deadline
+
+          sleep 0.02
+        end
+      end
+
+      def stop
+        @stopping = true
+        @socket&.close
+        @thread&.kill
+      rescue IOError
+        # already closed
+      end
+
+      private
+
+      def accept_loop
+        loop do
+          client = @socket.accept
+          Thread.new(client) { |c| handle_connection(c) }
+        rescue IOError, Errno::EBADF
+          break # socket closed by #stop
+        end
+      end
+
+      def handle_connection(client)
+        request_line = client.gets
+        return unless request_line
+
+        method, target, = request_line.split(" ")
+        path, query = target.split("?", 2)
+        token = query_param(query, "t")
+
+        headers = read_headers(client)
+        length = headers["content-length"].to_i
+        body = length.positive? ? client.read(length) : ""
+
+        status, payload = dispatch(method, path, token, body)
+        write_response(client, status, payload)
+      rescue StandardError
+        write_response(client, 500, nil) rescue nil
+      ensure
+        client.close rescue nil
+      end
+
+      def dispatch(method, path, token, body)
+        return [ 405, nil ] unless method == "POST"
+
+        # ActiveRecord connection per handler thread; released after the call.
+        ActiveRecord::Base.connection_pool.with_connection do
+          @handler.call(path: path, query_token: token, body: body)
+        end
+      end
+
+      def read_headers(client)
+        headers = {}
+        while (line = client.gets) && line != "\r\n"
+          key, value = line.split(":", 2)
+          headers[key.strip.downcase] = value.to_s.strip if value
+        end
+        headers
+      end
+
+      def query_param(query, key)
+        return nil unless query
+
+        query.split("&").each do |pair|
+          k, v = pair.split("=", 2)
+          return v if k == key
+        end
+        nil
+      end
+
+      def write_response(client, status, body)
+        reason = { 200 => "OK", 202 => "Accepted", 400 => "Bad Request", 403 => "Forbidden",
+                   404 => "Not Found", 405 => "Method Not Allowed", 500 => "Internal Server Error" }[status] || "OK"
+        body = body.to_s
+        client.write("HTTP/1.1 #{status} #{reason}\r\n")
+        client.write("content-type: application/json\r\n")
+        client.write("content-length: #{body.bytesize}\r\n")
+        client.write("connection: close\r\n\r\n")
+        client.write(body)
+      end
+
+      def clock = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/silas/message_builder.rb

@@ -0,0 +1,63 @@
+module Silas
+  # Rebuilds the provider conversation deterministically from persisted rows —
+  # the rows ARE the transcript (no separate events table in v0.1). Replayed
+  # executions must produce byte-identical message arrays, so nothing here may
+  # read mutable state or the clock.
+  module MessageBuilder
+    module_function
+
+    # Canonical provider-agnostic shape; adapters map it to their wire format.
+    #   { role: "user", content: "..." }
+    #   { role: "assistant", content: [ {type:, ...blocks} ] }
+    #   { role: "tool", tool_call_id:, content: {...} }
+    # NOTE: always fresh queries, never cached associations — this runs inside
+    # a live loop where rows were created moments ago on the same objects, and
+    # a memoized empty `turn.steps` silently erases the model's own history.
+    def call(turn, upto_index:)
+      messages = []
+
+      Turn.where(session_id: turn.session_id).order(:index).each do |prior|
+        break if prior.index >= turn.index
+
+        messages << { role: "user", content: prior.input }
+        messages.concat(step_messages(prior, upto_index: nil))
+      end
+
+      messages << { role: "user", content: turn.input }
+      messages.concat(step_messages(turn, upto_index: upto_index))
+      messages
+    end
+
+    def step_messages(turn, upto_index:)
+      Step.where(turn_id: turn.id).order(:index).each_with_object([]) do |step, acc|
+        next unless step.completed?
+        next if upto_index && step.index >= upto_index
+
+        # The settled invocations are the ledger's source of truth for what the
+        # model actually invoked. Reconstruct the assistant's tool_use blocks
+        # from them (not from step.response_blocks, which can drift when the
+        # model emits parallel tool calls) so every tool_result replayed below
+        # has a matching tool_use by construction — the provider requires it.
+        settled = ToolInvocation.where(step_id: step.id).order(:id)
+                                .select { |inv| inv.status == "completed" || inv.status == "failed" }
+
+        acc << { role: "assistant", content: assistant_blocks(step, settled) }
+        settled.each do |inv|
+          acc << { role: "tool", tool_call_id: inv.tool_call_id, content: inv.result }
+        end
+      end
+    end
+
+    # Text comes from the model's own blocks; tool_use blocks are rebuilt from
+    # the settled invocations so the assistant message and the tool results that
+    # follow are always a matched set (same ids, same count).
+    def assistant_blocks(step, settled)
+      text = Array(step.response_blocks).select { |b| b["type"] == "text" }
+      tools = settled.map do |inv|
+        { "type" => "tool_call", "id" => inv.tool_call_id,
+          "name" => inv.tool_name, "arguments" => inv.arguments || {} }
+      end
+      text + tools
+    end
+  end
+end

data/lib/silas/nested_runner.rb

@@ -0,0 +1,41 @@
+module Silas
+  # Drives a nested subagent Session's turn to terminal INLINE (no AgentLoopJob),
+  # inside the parent's delegate #call. Swaps the subagent's scope in as the
+  # active globals for the duration, then restores — so the existing loop
+  # machinery (StepRunner/Instructions/Budget/Ledger) runs unchanged against the
+  # subagent's tools/instructions. Synchronous, depth-1 (v1).
+  #
+  # Durability of the nested run = its persisted rows; its exactly-once BOUNDARY
+  # is the parent's at_most_once delegate invocation (a crash parks it in-doubt).
+  module NestedRunner
+    module_function
+
+    def run(session, input:)
+      scope = Silas.subagent_scope(session.agent_name)
+      turn = session.turns.create!(index: 0, input: input)
+
+      Silas.with_agent_scope(scope) do
+        agent = scope.agent
+        turn.update!(status: "running", started_at: Time.current)
+        Instructions.snapshot!(turn) # renders the SUBAGENT's instructions + snapshots ITS digest
+
+        index = 0
+        loop do
+          case StepRunner.call(turn, index)
+          when :parked   then return turn.finish!(:failed, reason: "subagent_parked") && turn.reload
+          when :terminal then return turn.finish!(:completed) && turn.reload
+          end
+          if (reason = Budget.exceeded_reason(turn, agent: agent))
+            return turn.finish!(:failed, reason: reason) && turn.reload
+          end
+
+          index += 1
+          if index >= agent.max_steps
+            turn.finish!(:failed, reason: "max_steps")
+            return turn.reload
+          end
+        end
+      end
+    end
+  end
+end