insika 0.3.0 → 0.7.0

data/lib/insika/evals/persona.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Insika
+  module Evals
+    # A SIMULATED CUSTOMER — the data that turns a scripted case into a
+    # generated conversation. Pure data: goal, style, the opening
+    # message, the ONLY facts the persona may assert, and a hard turn cap. The
+    # persona is played by a model (the cheap utility_model) with this as its whole
+    # instruction; the anti-invention rule below is the soul of the feature — a
+    # simulator that invents an order number produces a conversation the agent
+    # could never have had, and a case that tests nothing.
+    Persona = Struct.new(:goal, :style, :opens_with, :knows, :max_turns, keyword_init: true) do
+      def to_h
+        { "goal" => goal, "style" => style, "opens_with" => opens_with,
+          "knows" => knows, "max_turns" => max_turns }.compact
+      end
+
+      # The persona as a whole instruction. The `knows` facts are the ONLY
+      # assertions the persona may make; anything else is answered with
+      # ignorance — "não sei", "não tenho isso aqui" — exactly like a real
+      # customer who does not have the fact. `transcript` is the conversation
+      # so far, in order, as [{ role: "user"|"assistant", text: }].
+      def prompt(transcript)
+        facts = knows.map { |k, v| "- #{k}: #{v}" }.join("\n")
+        <<~PROMPT
+          You are simulating a customer in a chat with a store's virtual assistant.
+          Stay in character. You are not helping the assistant; you are the person
+          it serves.
+
+          GOAL: #{goal}
+          STYLE: #{style || "short, natural messages; answers what is asked"}
+
+          FACTS YOU KNOW — the ONLY facts you may assert:
+          #{facts}
+
+          RULES:
+          1. You may ONLY assert the facts above. Asked about anything else, you do
+             not know it — answer with ignorance, like a real customer without that
+             fact (you have no order number, no name, no date, no price beyond the
+             facts above). Never invent an order number, a name, a date, a price or
+             any other detail that is not in FACTS YOU KNOW.
+          2. Reply with ONLY the customer's next message.
+          3. When your goal has been met, end the message with the marker
+             <<goal_met>>. When you give up (the assistant cannot get you there),
+             end the message with the marker <<gave_up>>. Otherwise end with no
+             marker.
+
+          CONVERSATION SO FAR:
+          #{transcript.map { |m| "#{m[:role]}: #{m[:text]}" }.join("\n")}
+
+          Your next message:
+        PROMPT
+      end
+    end
+
+    # Loads + validates a persona mapping (the `persona:` key of a golden, or the
+    # `--persona` file of the simulate CLI). Fails LOUD on a malformed persona —
+    # a silently relaxed max_turns or a missing knows would produce a simulation
+    # that tests nothing.
+    module PersonaLoader
+      class InvalidPersona < StandardError; end
+
+      module_function
+
+      def build(raw, source: "(inline)")
+        raise InvalidPersona, "#{source}: persona must be a mapping" unless raw.is_a?(Hash)
+
+        goal = presence(raw["goal"])
+        raise InvalidPersona, "#{source}: persona needs a non-empty 'goal'" if goal.nil?
+
+        knows = raw["knows"]
+        unless knows.is_a?(Hash) && !knows.empty?
+          raise InvalidPersona, "#{source}: persona needs a non-empty 'knows' mapping (the only facts it may assert)"
+        end
+
+        opens = presence(raw["opens_with"])
+        raise InvalidPersona, "#{source}: persona needs a non-empty 'opens_with'" if opens.nil?
+
+        max = raw["max_turns"]
+        unless max.is_a?(Integer) && max.positive?
+          raise InvalidPersona, "#{source}: persona needs 'max_turns' as a positive integer"
+        end
+
+        Persona.new(
+          goal: goal, style: presence(raw["style"]),
+          opens_with: opens,
+          knows: knows.transform_keys(&:to_s).transform_values(&:to_s),
+          max_turns: max
+        )
+      end
+
+      def presence(v)
+        s = v.to_s.strip
+        s.empty? ? nil : s
+      end
+    end
+  end
+end

data/lib/insika/evals/runner.rb

@@ -50,6 +50,15 @@ module Insika
       end
 
       def run_case(golden)
+        # A persona case is GENERATED, not replayed: the turns do not exist until a
+        # Simulator drives the conversation. The replay Runner cannot run it, and a
+        # silent no-op would read as a pass — so it is SKIPPED with the reason, and
+        # the Simulator (the simulate CLI) is the only driver.
+        if golden.simulated?
+          return RunCase.new(result: Assertions.skip(golden, "simulated case (persona) — drive it with `insika evals:simulate`"),
+                             timings: [])
+        end
+
         skip = skip_reason(golden)
         return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
 

data/lib/insika/evals/simulator.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module Insika
+  module Evals
+    # SIMULATED USERS. Two models talking: the target agent
+    # (reached through the same Transport seam as the Runner) and the simulated
+    # customer (a cheap model — the platform utility_model — playing a `Persona`).
+    #
+    # Pure over its seams, like the Runner: the Transport is injected (HttpTransport
+    # for a remote deployment, GraphTransport for the own graph, A2ATransport for an
+    # agent that only speaks A2A) and the persona model is an injected `ask`
+    # (prompt -> raw text), so the loop is unit-testable offline.
+    #
+    # Termination is recorded, never guessed: the persona's `max_turns`, the persona
+    # emitting `<<goal_met>>` (its goal is served) or `<<gave_up>>` (it abandons),
+    # or an errored agent turn. "Gave up at turn 3" is a finding, not a detail.
+    #
+    # SAFETY (rule fixed in the spec): a simulated conversation must not write for
+    # real. The `Safety` gate refuses the run unless the target is declared
+    # `staging` or the run uses an eval profile where the agent's side-effect tools
+    # are swapped for fakes — and the swap list is DERIVED from the tool registry
+    # (the engine marks `side_effect` on tools; see `EvalProfile`), never
+    # hand-maintained. Every run is `simulated: true`, so a report never mixes a
+    # generated conversation with real traffic.
+    class Simulator
+      STOP_GOAL_MET = "<<goal_met>>"
+      STOP_GAVE_UP = "<<gave_up>>"
+      STOPS = { STOP_GOAL_MET => :goal_met, STOP_GAVE_UP => :gave_up }.freeze
+
+      # A generated conversation. `transcript` is [{ role: "user"|"assistant",
+      # text:, tools: [names] }] in order; `stop` is one of :goal_met | :gave_up |
+      # :max_turns | :error; `simulated` is ALWAYS true — the flag that keeps a
+      # report from mixing generated traffic with real conversations (rule D).
+      SimulatedRun = Struct.new(:transcript, :stop, :turns, :error, keyword_init: true) do
+        def simulated = true
+        def simulated? = true
+
+        def to_h
+          { "simulated" => true, "stop" => stop.to_s, "turns" => turns, "error" => error,
+            "transcript" => transcript }.compact
+        end
+      end
+
+      # The target-safety gate. A run is allowed when:
+      #   · `staging`         — the operator declares the target is a staging
+      #                         deployment (real tools, staging data), or
+      #   · the derived `side_effect_tools` is EMPTY — the agent has nothing
+      #     that can write, or
+      #   · `eval_profile` AND `swapped_tools` covers EVERY derived
+      #     side-effect tool — the eval profile swaps them all for dry-runs.
+      # Anything else is refused with the offending tool names. Refusals are loud:
+      # `UnsafeTarget` is raised BEFORE a single model call.
+      #
+      # `side_effect_tools` is the DERIVED list (see EvalProfile) — never a
+      # hand-typed claim; `swapped_tools` is what the run's eval profile declares
+      # it swaps. A bare `eval_profile: true` with a known side-effect list is
+      # REFUSED: an eval profile that leaves a write-capable tool unswapped is a
+      # trust-me flag wearing a safety's clothes.
+      Safety = Struct.new(:staging, :eval_profile, :side_effect_tools, :swapped_tools, keyword_init: true) do
+        def self.staging = new(staging: true)
+
+        # -> reason to refuse, or nil. `side_effect_tools` is the DERIVED list (the
+        # target's reachable tools the registry marks `side_effect`).
+        def refusal
+          return nil if staging
+
+          tools = Array(side_effect_tools).map(&:to_s).reject(&:empty?)
+          return nil if tools.empty?
+
+          if eval_profile
+            swapped = Array(swapped_tools).map(&:to_s)
+            uncovered = tools - swapped
+            return nil if uncovered.empty?
+
+            return "eval profile declares swapped tool(s) (#{swapped.join(', ')}) but the target " \
+                   "also exposes side-effect tool(s) (#{uncovered.join(', ')}) — an eval profile " \
+                   "must swap EVERY side-effect tool"
+          end
+
+          "the target agent exposes side-effect tool(s) (#{tools.join(', ')}) — a simulated " \
+            "conversation could write for real. Run against staging (--staging) or against an " \
+            "eval profile where these tools are swapped for fakes (--eval-profile --eval-tools ...)."
+        end
+      end
+
+      # Raised before any model call when the Safety gate refuses.
+      class UnsafeTarget < StandardError; end
+
+      # transport: the target agent seam (TurnOutcome per turn, like the Runner).
+      # ask:       ->(prompt) { text } — the simulated customer (the cheap model).
+      # safety:    a Safety — the gate evaluated before every run.
+      def initialize(transport:, ask:, safety:)
+        @transport = transport
+        @ask = ask
+        @safety = safety
+      end
+
+      # Runs one simulated conversation. -> SimulatedRun.
+      def run(persona:, agent:, conv:)
+        reason = @safety.refusal
+        raise UnsafeTarget, reason if reason
+
+        transcript = []
+        message = persona.opens_with
+        stop = :max_turns
+
+        1.upto(persona.max_turns) do |turn|
+          transcript << { role: "user", text: message }
+          outcome = @transport.turn(agent: agent, conv: conv, message: message)
+          if outcome.result.error
+            transcript << { role: "assistant", text: "", tools: [] }
+            return SimulatedRun.new(transcript: transcript, stop: :error, turns: turn,
+                                    error: outcome.result.error)
+          end
+          transcript << { role: "assistant", text: outcome.result.output_text,
+                          tools: outcome.result.tool_names }
+          break if turn == persona.max_turns # the persona's budget is spent
+
+          reply = @ask.call(persona.prompt(transcript)).to_s
+          marker, text = strip_stop(reply)
+          if marker
+            transcript << { role: "user", text: text } unless text.empty?
+            return SimulatedRun.new(transcript: transcript, stop: marker, turns: turn)
+          end
+          if text.empty?
+            return SimulatedRun.new(transcript: transcript, stop: :error, turns: turn,
+                                    error: "the persona produced an empty message")
+          end
+
+          message = text
+        end
+
+        SimulatedRun.new(transcript: transcript, stop: stop, turns: persona.max_turns)
+      end
+
+      private
+
+      # -> [stop_reason | nil, text_without_marker]. A trailing stop marker ends
+      # the conversation; the text before it is the customer's final message.
+      def strip_stop(reply)
+        text = reply.strip
+        STOPS.each do |marker, reason|
+          return [reason, text.delete_suffix(marker).strip] if text.end_with?(marker)
+        end
+        [nil, text]
+      end
+    end
+
+    # The DERIVED eval profile: which of an agent's reachable tools can write for
+    # real, computed from the tool registry — the engine marks `side_effect` on
+    # tools (a data-tool's non-GET method, the MCP ingestor's `tools/call`), so the
+    # swap list is a fact of the deployment, never a hand-maintained list.
+    module EvalProfile
+      module_function
+
+      # profile + registry (answers #names and #side_effect?) ->
+      # [tool names] the agent can reach that are marked side-effect, sorted.
+      def side_effect_tools(profile, registry)
+        allowed = if profile.tools_allow.nil? then Array(registry.names)
+                  else Array(profile.tools_allow).map(&:to_s)
+                  end
+        denied = Array(profile.tools_deny).map(&:to_s)
+        (allowed - denied).select { |name| registry.side_effect?(name) }.sort
+      end
+
+      # -> bool: can a simulated run touch this agent without a staging
+      # declaration (no reachable side-effect tool)?
+      def safe?(profile, registry)
+        side_effect_tools(profile, registry).empty?
+      end
+
+      # A registry overlay that answers the swapped names with a DRY-RUN tool and
+      # delegates everything else to the base — the "side_effect -> fake" half of
+      # the eval profile, derived from the base registry (nothing hand-maintained).
+      # The overlay is a drop-in for the Executor's registry: same
+      # entries/resolve/side_effect? surface.
+      def registry(base, side_effect_tools:, dry_run: nil)
+        swapped = Array(side_effect_tools).map(&:to_s)
+        fake = dry_run || ->(name) { Simulator::DryRunTool.new(name) }
+        OverlayRegistry.new(base: base, swapped: swapped, fake: fake)
+      end
+    end
+
+    # The overlay behind `EvalProfile.registry`. Kept as a named class so the
+    # constant is assigned once at load time, not inside the method.
+    class EvalProfile::OverlayRegistry
+      def initialize(base:, swapped:, fake:)
+        @base = base
+        @swapped = swapped
+        @fake = fake
+      end
+
+      def names = @base.names
+      def entries = @base.entries
+      def side_effect?(name) = @swapped.include?(name.to_s) ? false : @base.side_effect?(name)
+
+      def resolve(name)
+        key = name.to_s
+        return @fake.call(key) if @swapped.include?(key)
+
+        @base.resolve(key)
+      end
+    end
+  end
+end
+
+# The dry-run fake lives with the Simulator (it is its "side_effect -> fake"
+# convention). It answers the same surface as a RubyLLM tool (`call`) and never
+# performs the real side effect. Its result carries `dry_run: true` so a reader
+# can tell a swapped call from a real one in the transcript's tool trace.
+class Insika::Evals::Simulator::DryRunTool
+  def initialize(name, description: nil)
+    @name = name.to_s
+    @description = description ||
+                   "DRY-RUN of #{@name} — disabled for this simulated run, returns a canned envelope"
+  end
+
+  def name = @name
+  def description = @description
+
+  def call(args)
+    { "dry_run" => true, "tool" => @name, "simulated" => true,
+      "note" => "side-effect tool disabled by the eval profile — the real call was NOT performed" }
+  end
+end

data/lib/insika/evals/transport.rb

@@ -108,7 +108,8 @@ module Insika
         return nil unless res.code.to_i == 200
 
         body = JSON.parse(res.body)
-        { "tools" => body["tools"], "capabilities" => Array(body["capabilities"]) }
+        { "tools" => body["tools"], "capabilities" => Array(body["capabilities"]),
+          "side_effect_tools" => body["side_effect_tools"] }
       rescue StandardError, JSON::ParserError
         nil
       end
@@ -174,5 +175,86 @@ module Insika
                         ttfb: nil, total: (mono - t0) * 1000.0)
       end
     end
+
+    # A transport over the deployment's OWN graph, in-process — no HTTP. This is
+    # how a simulated conversation (or a replay) exercises the local agent the
+    # way a customer would reach it, tools and guardrails included, without a
+    # server in between. `runtime` is anything answering the DSL Runtime contract
+    # (#chat(message, session_id:, agent:) -> text, raising on failure) — the DSL
+    # Definition/System runtime, or a test double.
+    #
+    # Tool activity is captured from the graph's event stream (the `:tool_call`
+    # events the ChatBuilder emits), so an in-process transcript records the same
+    # tool names an HTTP replay would — the Simulator's transcript is not blind to
+    # what the local agent called. `event_stream` is optional; when omitted it is
+    # read off the runtime's graph when one is reachable.
+    class GraphTransport
+      def initialize(runtime:, event_stream: nil)
+        @runtime = runtime
+        @event_stream = event_stream ||
+                      (runtime.graph.event_stream if runtime.respond_to?(:graph) && runtime.graph)
+      end
+
+      def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def turn(agent:, conv:, message:)
+        t0 = mono
+        sub = @event_stream&.subscribe(types: [:tool_call])
+        begin
+          text = @runtime.chat(message, session_id: conv, agent: agent)
+          TurnOutcome.new(
+            result: TurnResult.new(output_text: text.to_s, tool_calls: drain_tools(sub), error: nil),
+            ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
+          )
+        rescue Insika::Error => e
+          TurnOutcome.new(
+            result: TurnResult.new(output_text: "", tool_calls: drain_tools(sub), error: e.message),
+            ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
+          )
+        ensure
+          sub&.close
+        end
+      end
+
+      private
+
+      # The same shape an HTTP replay produces: [{ "name" =>, "status" => nil }]
+      # (the stream carries no per-tool status — that lives in the trace store).
+      def drain_tools(sub)
+        return [] if sub.nil?
+
+        sub.drain_nonblocking.map { |ev| { "name" => ev.data[:name].to_s, "status" => nil } }
+      end
+    end
+
+    # Thin A2A transport: drives a REMOTE A2A agent (an agent that only speaks
+    # A2A) through the same `turn` seam the Simulator uses. The outbound A2A
+    # client already does the send+poll; this is the wrapper that makes it a
+    # Transport. `client` answers `#call(url, text, context_id:)` -> {text:} or
+    # {error:} — the `Server::A2A::Client` shape.
+    class A2ATransport
+      def initialize(client:, url:)
+        @client = client
+        @url = url
+      end
+
+      def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def turn(agent:, conv:, message:)
+        t0 = mono
+        result = @client.call(@url, message.to_s, context_id: conv)
+        if result[:error]
+          TurnOutcome.new(
+            result: TurnResult.new(output_text: "", tool_calls: [], error: result[:error].to_s),
+            ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
+          )
+        else
+          TurnOutcome.new(
+            result: TurnResult.new(output_text: result[:text].to_s, tool_calls: [], error: nil),
+            ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
+          )
+        end
+      end
+    end
   end
 end

data/lib/insika/event_stream.rb

@@ -85,6 +85,16 @@ module Insika
         end
       end
 
+      # Drains whatever is ALREADY queued without ever blocking. Safe in a
+      # cooperative reactor: between the `empty?` check and the `dequeue` no other
+      # fiber runs, so a non-empty dequeue never waits. The eval transports use it
+      # to collect the events a turn already emitted, AFTER the turn returned.
+      def drain_nonblocking
+        drained = []
+        drained << @queue.dequeue until @queue.empty?
+        drained
+      end
+
       # Idempotent: a second CLOSED is harmless (the `each` stops at the first).
       # `@on_close` fires only once (avoids removing the subscription twice).
       def close