silas 0.1.1

data/lib/silas/eval/assertions.rb

@@ -0,0 +1,107 @@
+module Silas
+  module Eval
+    module Assertions
+      # Assertions read ONLY the transcript rows; the block runs in this context.
+      # They collect failures rather than raising, so one eval reports all misses.
+      class Context
+        EXECUTED = %w[completed started in_doubt].freeze
+        MONEY = /(?<cur>[£$€])\s?(?<amt>\d{1,3}(?:,\d{3})*(?:\.\d+)?|\d+(?:\.\d+)?)/
+
+        attr_reader :failures, :skips
+
+        def initialize(transcript)
+          @t = transcript
+          @failures = []
+          @skips = []
+        end
+
+        def assert_tool_called(name, times: nil)
+          n = execed(name).size
+          check(times ? n == times : n.positive?,
+                "expected #{name} called#{" #{times}x" if times}, saw #{n} (#{summary})")
+        end
+
+        def assert_no_tool_called(name = nil)
+          bad = name ? execed(name) : @t.invocations.select { |i| EXECUTED.include?(i.status) }
+          check(bad.empty?, "expected no#{" #{name}" if name} tool execution, saw #{bad.map(&:tool_name)}")
+        end
+
+        def assert_tool_arg(name, key, value = :__unset, &pred)
+          inv = execed(name).last || @t.invocations_for(name).last
+          return check(false, "#{name} was never called") unless inv
+
+          got = inv.arguments[key.to_s]
+          ok = pred ? pred.call(got) : got == value
+          check(ok, "#{name}.#{key} expected #{pred ? 'predicate' : value.inspect}, got #{got.inspect}")
+        end
+
+        def assert_final_matches(matcher)
+          check(matcher === @t.final_text, "final answer #{@t.final_text.inspect} does not match #{matcher.inspect}")
+        end
+
+        # No-hallucinated-price guard: every money amount in the final answer must
+        # trace to a number the agent actually saw (tool results or the user input),
+        # allowing pence<->pounds scaling.
+        def assert_no_hallucinated_price(allowed: [], scale: [ 1, 100, 0.01 ])
+          stated = @t.final_text.scan(MONEY).map { |_c, a| a.delete(",").to_f }
+          grounded = grounded_numbers + Array(allowed).map(&:to_f)
+          bad = stated.reject { |n| grounded.any? { |g| scale.any? { |s| (g * s - n).abs < 0.005 } } }
+          check(bad.empty?, "final answer states ungrounded amount(s) #{bad.inspect}; grounded=#{grounded.sort.inspect}")
+        end
+
+        def assert_approved(tool: nil)
+          inv = tool ? @t.invocations_for(tool).last : @t.invocations.last
+          check(inv&.approval_state == "approved", "expected #{tool || 'an invocation'} approved, state=#{inv&.approval_state.inspect}")
+        end
+
+        def assert_parked(tool: nil)
+          parked = @t.parked?
+          parked &&= @t.invocations_for(tool).any? { |i| i.approval_state == "required" || i.status == "in_doubt" } if tool
+          check(parked, "expected turn parked#{" on #{tool}" if tool}, status=#{@t.status}")
+        end
+
+        def assert_turn_completed
+          check(@t.completed?, "turn not completed (#{@t.status}/#{@t.turn.failure_reason})")
+        end
+
+        def assert_turn_failed(reason: nil)
+          ok = @t.status == "failed" && (reason.nil? || @t.turn.failure_reason == reason)
+          check(ok, "expected failed#{"/#{reason}" if reason}, got #{@t.status}/#{@t.turn.failure_reason}")
+        end
+
+        # LLM-graded — opt-in; SKIPS offline (never fails the gate unless SILAS_EVAL_STRICT=1).
+        def assert_rubric(criteria)
+          unless Grader.available?
+            @skips << "assert_rubric skipped (no grader / offline)"
+            return check(false, "assert_rubric strict skip (SILAS_EVAL_STRICT)") if ENV["SILAS_EVAL_STRICT"] == "1"
+
+            return
+          end
+          verdict = Grader.grade(Grader.prompt(@t, criteria))
+          check(verdict.start_with?("PASS"), "rubric FAIL: #{verdict}")
+        end
+
+        private
+
+        def execed(name) = @t.invocations_for(name).select { |i| EXECUTED.include?(i.status) }
+        def summary = @t.invocations.map { |i| "#{i.tool_name}:#{i.status}" }.join(",")
+        def check(cond, msg) = (@failures << msg unless cond)
+
+        def grounded_numbers
+          nums = []
+          walk = lambda do |v|
+            case v
+            when Numeric then nums << v.to_f
+            when Hash    then v.each_value(&walk)
+            when Array   then v.each(&walk)
+            when String  then v.scan(/\d+(?:\.\d+)?/) { |d| nums << d.to_f }
+            end
+          end
+          @t.results.each(&walk)
+          walk.call(@t.input)
+          nums
+        end
+      end
+    end
+  end
+end

data/lib/silas/eval/driver.rb

@@ -0,0 +1,47 @@
+module Silas
+  module Eval
+    # Drives ONE scenario to its resting state through the REAL framework loop
+    # (Silas.agent.start + AgentLoopJob.perform_now inline), then hands back a
+    # Transcript. In fake mode the ScriptedEngine supplies the model's decisions;
+    # the real Ledger runs the real tools.
+    module Driver
+      module_function
+
+      def drive(scenario, root: Rails.root)
+        prior_adapter = ActiveJob::Base.queue_adapter
+        ActiveJob::Base.queue_adapter = :test
+        configure!(scenario, root)
+
+        session = Silas.agent.start(input: scenario.input, metadata: scenario.metadata)
+        turn = session.turns.order(:index).last
+        Silas::AgentLoopJob.perform_now(turn.id)
+
+        scenario.approvals.each do |tool|
+          inv = turn.reload.tool_invocations.find { |i| i.tool_name == tool && i.approval_state == "required" }
+          next unless inv
+
+          inv.approve!(by: "eval")
+          Silas::AgentLoopJob.perform_now(turn.id)
+        end
+
+        Transcript.new(turn.reload, session.reload)
+      ensure
+        ActiveJob::Base.queue_adapter = prior_adapter
+      end
+
+      def configure!(scenario, root)
+        Silas::Registry.install!(root: root)
+        engine = scenario.real? ? nil : ScriptedEngine.new(scenario.steps)
+        base_resolver = Silas.config.tool_resolver
+        Silas.configure do |c|
+          c.engine = engine if engine
+          c.isolate_steps = false
+          c.max_steps = scenario.max_steps if scenario.max_steps
+          if scenario.stubs.any?
+            c.tool_resolver = ->(name) { scenario.stubs[name] || base_resolver.call(name) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/silas/eval/dsl.rb

@@ -0,0 +1,55 @@
+module Silas
+  module Eval
+    # Define-time recorder for the scenario DSL.
+    class DSL
+      def initialize(name, tags)
+        @name = name
+        @tags = tags
+        @mode = :fake
+        @steps = {}
+        @stubs = {}
+        @approvals = []
+        @metadata = {}
+        @max_steps = nil
+      end
+
+      def input(text)  = (@input = text)
+      def metadata(h)  = (@metadata = h)
+      def mode(m)      = (@mode = m)
+      def max_steps(n) = (@max_steps = n)
+      def approve(tool:) = (@approvals << tool.to_s)
+
+      # on_step(0, text:, call: {name:, arguments:}, calls: [ {…}, … ])
+      def on_step(index, text: nil, call: nil, calls: [])
+        tcs = (calls + [ call ].compact).each_with_index.map do |c, n|
+          Silas::Engines::ToolCall.new(id: "eval_s#{index}_#{n}", name: c[:name].to_s,
+                                       arguments: (c[:arguments] || {}).stringify_keys)
+        end
+        blocks = []
+        blocks << { "type" => "text", "text" => text } if text
+        @steps[index] = { blocks: blocks, tool_calls: tcs }
+      end
+
+      # Override a real tool with a stub for a side-effect-free eval.
+      def stub_tool(name, effect_mode: :at_most_once, approval: :never, &body)
+        obj = Object.new
+        obj.define_singleton_method(:effect_mode) { effect_mode }
+        obj.define_singleton_method(:approval_policy) { approval }
+        obj.singleton_class.attr_accessor :session
+        obj.define_singleton_method(:call, &body)
+        @stubs[name.to_s] = obj
+      end
+
+      def expect(&block) = (@assertions = block)
+
+      def to_scenario
+        raise Silas::Error, "eval #{@name.inspect} needs an input" unless @input
+        raise Silas::Error, "eval #{@name.inspect} needs an expect block" unless @assertions
+
+        Scenario.new(name: @name, input: @input, mode: @mode, steps: @steps,
+                     stubs: @stubs, approvals: @approvals, metadata: @metadata,
+                     max_steps: @max_steps, tags: @tags, assertions: @assertions)
+      end
+    end
+  end
+end

data/lib/silas/eval/grader.rb

@@ -0,0 +1,37 @@
+module Silas
+  module Eval
+    # Opt-in LLM judge for assert_rubric. Offline-skippable so the deterministic
+    # gate never depends on a live model unless you ask for it.
+    module Grader
+      module_function
+
+      def available? = ENV["SILAS_EVAL_OFFLINE"] != "1" && !grader.nil?
+      def grade(prompt) = grader.call(prompt).to_s.strip
+
+      def grader
+        Silas.config.eval_grader || builtin
+      end
+
+      def builtin
+        return nil if ENV["ANTHROPIC_API_KEY"].blank?
+
+        lambda do |prompt|
+          ::RubyLLM.chat(model: ENV["SILAS_EVAL_GRADER_MODEL"] || "claude-haiku-4-5-20251001")
+                   .ask(prompt).content.to_s
+        end
+      end
+
+      def prompt(transcript, criteria)
+        calls = transcript.invocations.map { |i| "#{i.tool_name}(#{i.arguments}) => #{i.result}" }.join("; ")
+        <<~PROMPT
+          You are grading an AI agent transcript. Reply with exactly one line: "PASS" or "FAIL: <reason>".
+          Criteria: #{criteria}
+
+          User: #{transcript.input}
+          Tool calls: #{calls}
+          Final answer: #{transcript.final_text}
+        PROMPT
+      end
+    end
+  end
+end

data/lib/silas/eval/result.rb

@@ -0,0 +1,28 @@
+module Silas
+  module Eval
+    class Result
+      attr_reader :scenario, :failures, :skips
+
+      def initialize(scenario, failures = [], skips = [], skipped: false)
+        @scenario = scenario
+        @failures = failures
+        @skips = skips
+        @skipped = skipped
+      end
+
+      def self.skipped(scenario, why) = new(scenario, [], [ why ], skipped: true)
+      def self.errored(scenario, error) = new(scenario, [ "ERROR: #{error.class}: #{error.message}" ])
+
+      def pass? = @failures.empty? && !@skipped
+      def skipped? = @skipped
+
+      def line
+        mark = skipped? ? "SKIP" : (pass? ? "PASS" : "FAIL")
+        out = "  [#{mark}] #{@scenario.name}"
+        out += "\n        - #{@failures.join("\n        - ")}" unless @failures.empty?
+        out += "\n        (#{@skips.join('; ')})" unless @skips.empty?
+        out
+      end
+    end
+  end
+end

data/lib/silas/eval/runner.rb

@@ -0,0 +1,38 @@
+module Silas
+  module Eval
+    # Discovers *_eval.rb files, runs each scenario in isolation against the real
+    # loop, reports, and returns a pass/fail boolean (the rake task exits 1 on false).
+    module Runner
+      module_function
+
+      def run(dir:, pattern: nil, mode: nil, root: Rails.root)
+        Eval.reset!
+        Dir[File.join(dir, "**/*_eval.rb")].sort.each { |f| load f }
+        picked = Eval.scenarios.select { |s| pattern.nil? || s.name.match?(pattern) }
+
+        results = picked.map { |s| run_one(s.with_mode(mode), root) }
+        report(results)
+        results.all? { |r| r.pass? || r.skipped? }
+      end
+
+      def run_one(scenario, root = Rails.root)
+        if scenario.real? && ENV["ANTHROPIC_API_KEY"].to_s.empty?
+          return Result.skipped(scenario, "real mode, no ANTHROPIC_API_KEY")
+        end
+
+        transcript = Driver.drive(scenario, root: root)
+        ctx = Assertions::Context.new(transcript)
+        ctx.instance_exec(&scenario.assertions)
+        Result.new(scenario, ctx.failures, ctx.skips)
+      rescue StandardError => e
+        Result.errored(scenario, e)
+      end
+
+      def report(results)
+        results.each { |r| puts r.line }
+        failing = results.count { |r| !(r.pass? || r.skipped?) }
+        puts "\nsilas eval: #{results.size} scenarios, #{failing} failing, #{results.count(&:skipped?)} skipped"
+      end
+    end
+  end
+end

data/lib/silas/eval/scripted_engine.rb

@@ -0,0 +1,39 @@
+module Silas
+  module Eval
+    # The productized FakeEngine: a pure function of context[:index] that lets an
+    # eval script the MODEL's decisions while the REAL Ledger runs the REAL tools —
+    # so assertions see a genuine transcript.
+    class ScriptedEngine < Silas::Engines::Base
+      def self.loop_ownership = :framework
+
+      attr_reader :calls
+
+      def initialize(steps)
+        @steps = steps
+        @calls = []
+      end
+
+      def execute_step(context, &_on_event)
+        i = context[:index]
+        @calls << { index: i }
+        spec = @steps[i]
+        return terminal("OK.") unless spec
+
+        Silas::Engines::Result.new(
+          blocks: spec[:blocks],
+          tool_calls: spec[:tool_calls],
+          stop_reason: spec[:tool_calls].empty? ? "end_turn" : "tool_use",
+          usage: { input_tokens: 10, output_tokens: 5 }
+        )
+      end
+
+      private
+
+      def terminal(text)
+        Silas::Engines::Result.new(blocks: [ { "type" => "text", "text" => text } ],
+                                   tool_calls: [], stop_reason: "end_turn",
+                                   usage: { input_tokens: 1, output_tokens: 1 })
+      end
+    end
+  end
+end

data/lib/silas/eval/transcript.rb

@@ -0,0 +1,22 @@
+module Silas
+  module Eval
+    # Read-only view over a turn at its resting state — the surface assertions read.
+    class Transcript
+      attr_reader :turn, :session
+
+      def initialize(turn, session)
+        @turn = turn
+        @session = session
+      end
+
+      def input      = @turn.input
+      def status     = @turn.status
+      def completed? = @turn.completed?
+      def parked?    = @turn.parked?
+      def final_text = @turn.answer_text.to_s
+      def invocations = @turn.tool_invocations.order(:id).to_a
+      def invocations_for(name) = invocations.select { |i| i.tool_name == name.to_s }
+      def results = invocations.map(&:result).compact
+    end
+  end
+end

data/lib/silas/eval.rb

@@ -0,0 +1,35 @@
+require "silas/eval/scripted_engine"
+require "silas/eval/dsl"
+require "silas/eval/transcript"
+require "silas/eval/assertions"
+require "silas/eval/grader"
+require "silas/eval/driver"
+require "silas/eval/result"
+require "silas/eval/runner"
+
+module Silas
+  # Evals: fixtures are scenarios, assertions run against the DURABLE transcript
+  # (the real Session/Turn/Step/ToolInvocation rows). The harness only observes
+  # the exactly-once loop; it never touches it. Written like Rails tests, run as
+  # a deploy gate (non-zero exit on failure). The launch differentiator.
+  module Eval
+    class AssertionError < Silas::Error; end
+
+    Scenario = Struct.new(:name, :input, :mode, :steps, :stubs, :approvals,
+                          :metadata, :max_steps, :tags, :assertions, keyword_init: true) do
+      def real? = mode == :real
+      def with_mode(m) = m ? dup.tap { |s| s.mode = m } : self
+    end
+
+    class << self
+      def scenarios = @scenarios ||= []
+      def reset!    = (@scenarios = [])
+
+      def scenario(name, tags: [], &block)
+        dsl = DSL.new(name, tags)
+        dsl.instance_exec(&block)
+        scenarios << dsl.to_scenario
+      end
+    end
+  end
+end

data/lib/silas/inbox/cost.rb

@@ -0,0 +1,58 @@
+module Silas
+  module Inbox
+    # cost_microcents is declared on silas_turns but never populated (only
+    # silas_steps carry real token counts). So the inbox derives cost at read
+    # time from step tokens x a host-supplied price map — and honestly reports
+    # `unpriced` when a model isn't in the map rather than a lying £0.00.
+    module Cost
+      module_function
+
+      def for_session(session)
+        rows = Silas::Step.joins(:turn)
+                          .where(silas_turns: { session_id: session.id })
+                          .group(:model)
+                          .pluck(:model, Arel.sql("SUM(silas_steps.input_tokens)"), Arel.sql("SUM(silas_steps.output_tokens)"))
+        aggregate(rows)
+      end
+
+      def for_turn(turn)
+        rows = Silas::Step.where(turn_id: turn.id)
+                          .group(:model)
+                          .pluck(:model, Arel.sql("SUM(silas_steps.input_tokens)"), Arel.sql("SUM(silas_steps.output_tokens)"))
+        aggregate(rows)
+      end
+
+      def for_agent(agent_name)
+        rows = Silas::Step.joins(turn: :session)
+                          .where(silas_sessions: { agent_name: agent_name })
+                          .group(:model)
+                          .pluck(:model, Arel.sql("SUM(silas_steps.input_tokens)"), Arel.sql("SUM(silas_steps.output_tokens)"))
+        aggregate(rows)
+      end
+
+      def aggregate(rows)
+        input = output = microcents = 0
+        unpriced = false
+        rows.each do |model, in_tok, out_tok|
+          in_tok = in_tok.to_i
+          out_tok = out_tok.to_i
+          input += in_tok
+          output += out_tok
+          if (price = Silas.config.model_prices[model])
+            microcents += (in_tok * price[:in] + out_tok * price[:out]) / 1000
+          else
+            unpriced = true
+          end
+        end
+        { input_tokens: input, output_tokens: output, microcents: microcents, unpriced: unpriced }
+      end
+
+      # microcents -> "$0.0123" (or nil when unpriced with no priced tokens)
+      def format(cents)
+        return nil if cents.nil?
+
+        "$%.4f" % (cents / 1_000_000.0)
+      end
+    end
+  end
+end

data/lib/silas/inbox.rb

@@ -0,0 +1,23 @@
+module Silas
+  # The mountable inbox UI (at /silas/inbox). It renders the existing rows
+  # (Session -> Turn -> Step -> ToolInvocation) as a live trace — no events
+  # table — and streams changes over Turbo when the host has turbo-rails.
+  module Inbox
+    module_function
+
+    # turbo-rails is NOT a gem dependency (the core stays lean, like the
+    # solid_queue guard). Broadcasting activates only when the host bundles it.
+    def streaming_available?
+      defined?(Turbo::StreamsChannel)
+    end
+
+    # Broadcasting is on when Turbo is present. A host can hard-disable it.
+    def streaming?
+      streaming_available? && Silas.config.inbox_streaming != false
+    end
+
+    def stream_name(session_id)
+      "silas:inbox:session:#{session_id}"
+    end
+  end
+end

data/lib/silas/instructions.rb

@@ -0,0 +1,55 @@
+require "erb"
+
+module Silas
+  # Renders instructions.md ONCE per turn into an immutable snapshot — the
+  # determinism anchor. (Skill-index injection and loaded-skill inlining land
+  # with the skills work; the snapshot-once mechanics are load-bearing now.)
+  module Instructions
+    module_function
+
+    def snapshot!(turn)
+      return if turn.instructions_snapshot.present? # idempotent: replay-safe
+
+      turn.update!(
+        instructions_snapshot: render(turn),
+        definitions_digest: Silas.config.definitions_digest&.call.to_s
+      )
+    end
+
+    def render(turn)
+      [ base_instructions(turn), skill_index_block(turn.session), loaded_skills_block(turn.session) ]
+        .compact.join("\n\n")
+    end
+
+    def base_instructions(turn)
+      # config.instructions_dir points at the active agent's directory (root or a
+      # subagent, swapped during a nested run).
+      dir = Silas.config.instructions_dir || Rails.root.join("app/agent")
+      path = Pathname(dir).join("instructions.md")
+      return default_instructions unless path.exist?
+
+      ERB.new(path.read).result_with_hash(session: turn.session, agent_name: turn.session.agent_name)
+    end
+
+    # Advertise skill descriptions (eve's routing hint); bodies load on demand.
+    def skill_index_block(session)
+      advertised = Silas.skills.reject { |s| session.loaded_skills.include?(s.name) }
+      return nil if advertised.empty?
+
+      lines = advertised.map { |s| "- #{s.name}: #{s.description}" }
+      "## Available skills\n\nLoad with the load_skill tool when relevant:\n#{lines.join("\n")}"
+    end
+
+    # Skills loaded in prior turns are inlined "for the rest of the session".
+    def loaded_skills_block(session)
+      loaded = Silas.skills.select { |s| session.loaded_skills.include?(s.name) }
+      return nil if loaded.empty?
+
+      loaded.map { |s| "## Skill: #{s.name}\n\n#{s.body.strip}" }.join("\n\n")
+    end
+
+    def default_instructions
+      "You are a helpful agent."
+    end
+  end
+end