insika 0.0.1 → 0.2.0

data/lib/insika/evals/assertions.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+# the PII/secret patterns live in the RUNTIME (single source of
+# truth) — the eval consumes them rather than keeping a divergent copy. Since the
+# module moved under `lib/`, `Safety::Detectors` is loaded by `insika.rb` before this
+# file; the explicit climb out of `evals/` that used to be here is gone.
+
+module Insika
+  module Evals
+    # What the runner extracts from ONE replayed conversation, entirely from the
+    # public SSE stream of POST /v1/responses (no store reads — the eval stays a
+    # client). The assertion engine is pure over this value, so it's unit-testable
+    # offline without a server.
+    #
+    #   output_text: the final assistant text (for content checks)
+    #   tool_calls:  [{ "name" =>, "status" => }] captured from the stream's tool events
+    #   error:       transport/turn error string, or nil on a clean turn
+    TurnResult = Struct.new(:output_text, :tool_calls, :error, keyword_init: true) do
+      def tool_names = Array(tool_calls).map { |t| (t["name"] || t[:name]).to_s }
+
+      # A tool call whose status is anything but a success ("ok"/2xx/"success").
+      def errored_tools
+        Array(tool_calls).reject { |t| Assertions.ok_status?(t["status"] || t[:status]) }
+      end
+    end
+
+    # A single check within a case (e.g. "tool:shipping_quote", "must_not:pii_leak").
+    Check = Struct.new(:name, :pass, :detail, keyword_init: true)
+
+    # The verdict for one golden case. `judge` (a Judge::Verdict) is attached AFTER
+    # the deterministic pass when the case has a rubric and a judge is configured;
+    # until then a rubric'd case is `judge_pending?` — it reads as "not fully
+    # evaluated", never a silent pass. A case passes only if the deterministic checks
+    # pass AND (there's no judge verdict OR it passed).
+    #
+    # `skipped` (a reason, nil = it ran) is the THIRD outcome: the
+    # deployment lacks something the case declared it needs, so there was nothing to
+    # assert. It is never a pass and never a failure — a suite of 40 cases where 12
+    # are skipped says something true, where 40 cases with 12 failures on capability
+    # grounds says nothing and gets ignored.
+    # `pairwise` (a Pairwise::Verdict) is attached when the case
+    # carries a `reference:` and a panel is configured. It DELIBERATELY does not enter
+    # `pass?`: "worse than the incumbent" is a judgement about a replacement decision,
+    # not a regression in the suite, and letting it fail a case would put an opinion
+    # about two conversations in the pre-merge gate.
+    CaseResult = Struct.new(:id, :agent, :checks, :error, :rubric, :judge, :skipped, :pairwise,
+                            keyword_init: true) do
+      def skipped? = !skipped.nil?
+      def pass? = !skipped? && error.nil? && checks.all?(&:pass) && (judge.nil? || judge.pass)
+      def failures = checks.reject(&:pass)
+      # Has a rubric to score but no verdict yet (judge disabled / not run). A skipped
+      # case is not pending anything — nobody is going to judge a turn that never ran.
+      def judge_pending? = !skipped? && !rubric.to_s.strip.empty? && judge.nil?
+    end
+
+    # Deterministic evaluation — cheap, zero-token, zero-flakiness. It's the
+    # layer that catches the gross regressions (a tool stopped being called, a secret
+    # leaked, the turn errored). Subjective scoring is the LLM-judge in.
+    module Assertions
+      # Named negative detectors for `must_not` now live in the runtime. Kept as
+      # an alias so any external reference to Evals::Assertions::PII_DETECTORS still
+      # resolves; the values ARE the runtime's, never a fork.
+      PII_DETECTORS = Insika::Safety::Detectors::PII
+
+      # HOW MUCH THE AGENT SHOULD ASK BEFORE ACTING. Declared per
+      # case because it is a per-STORE decision, not a universal rule: sometimes the
+      # agent should establish the objective before searching ("energia, treino ou
+      # sono?" — a good store agent does this well), and sometimes asking again is the
+      # failure and it should just search. A global assertion would be wrong half
+      # the time; the judge is TOLD the policy (Judge#build_prompt) and this layer
+      # checks the half that needs no reader.
+      #
+      # Each rule is stated as the CUSTOMER-VISIBLE fact it checks. phrased
+      # this as "questions before the first tool call", written before: text a
+      # model emits before calling a tool never reaches the customer now (it rides
+      # `:intermediate`), and the eval is a client of `/v1/responses`, so what it can
+      # observe per turn is the published answer plus the tools that turn called.
+      # That is also the honest scope — a question nobody received is not a question.
+      POLICIES = {
+        # "UMA PERGUNTA POR VEZ" — the rule Insika broke twice under a 28 KB prompt.
+        "ask_once" => "at most one question per reply",
+        # Establish the objective before acting on a vague opener.
+        "investigate_first" => "asks before calling a tool, on the first turn",
+        # Act on the first plausible reading; refine after.
+        "act_fast" => "calls a tool on the first turn instead of asking"
+      }.freeze
+
+      module_function
+
+      # WHAT THIS DEPLOYMENT LACKS for the case to be worth running.
+      # -> [reason]; empty = run it.
+      #
+      # `available` is the deployment's answer for this agent:
+      #   { "tools" => [names] | nil, "capabilities" => [names] }
+      # `tools` nil means an OPEN allowlist — the agent may call every registered tool,
+      # so no tool requirement can be judged missing and the case runs. That is the
+      # deliberate reading: "I could not rule it out" must not become a skip, or a
+      # permissive agent would quietly stop being tested.
+      def unmet_requirements(golden, available)
+        tools = available["tools"]
+        declared = Array(available["capabilities"]).map(&:to_s)
+
+        missing_tools = tools.nil? ? [] : golden.required_tools - Array(tools).map(&:to_s)
+        missing_caps = golden.required_capabilities - declared
+
+        reasons = []
+        reasons << "tool not available: #{missing_tools.join(', ')}" unless missing_tools.empty?
+        reasons << "capability not declared: #{missing_caps.join(', ')}" unless missing_caps.empty?
+        reasons
+      end
+
+      # The case did not run and MUST NOT read as either a pass or a failure.
+      def skip(golden, reason)
+        CaseResult.new(id: golden.id, agent: golden.agent, checks: [], error: nil,
+                       rubric: nil, judge: nil, skipped: reason)
+      end
+
+      # A tool status counts as success when it's blank/"ok"/"success" or a 2xx code.
+      def ok_status?(status)
+        return true if status.nil?
+
+        s = status.to_s.strip.downcase
+        return true if s.empty? || %w[ok success succeeded done].include?(s)
+
+        code = Integer(s, exception: false)
+        code ? code.between?(200, 299) : false
+      end
+
+      # Golden + TurnResult -> CaseResult. A turn that failed to run yields a single
+      # failing check (there's nothing to assert on a turn that never produced output).
+      #
+      # `turns` is every turn of the conversation, in order; `result` is the last one
+      # (what the tool/content assertions have always run on). The policy checks need
+      # all of them — "one question per reply" is a rule about every reply, and the
+      # violation that motivated this was on the FIRST turn. Defaults to the single
+      # result so existing callers keep working.
+      def evaluate(golden, result, turns: nil)
+        if result.error
+          return CaseResult.new(id: golden.id, agent: golden.agent, error: result.error, rubric: nil, judge: nil,
+                                checks: [Check.new(name: "turn", pass: false, detail: "turn error: #{result.error}")])
+        end
+
+        # NOT `Array(turns)`: TurnResult is a Struct, so Array() would explode a single
+        # one into its members and hand the policy checks three strings.
+        conversation = turns.nil? || turns.empty? ? [result] : turns
+        checks = tool_checks(golden, result) + must_not_checks(golden, result) +
+                 policy_checks(golden, conversation)
+        CaseResult.new(id: golden.id, agent: golden.agent, error: nil, checks: checks,
+                       rubric: golden.rubric, judge: nil)
+      end
+
+      # Each REQUIRED expected tool must appear in the turn's tool calls. Optional
+      # ("name?") tools are informational — present or not, they never fail.
+      def tool_checks(golden, result)
+        names = result.tool_names
+        golden.tools_called.filter_map do |t|
+          next if t[:optional]
+
+          present = names.include?(t[:name])
+          Check.new(name: "tool:#{t[:name]}", pass: present,
+                    detail: present ? "called" : "expected but not called (saw: #{names.join(', ')})")
+        end
+      end
+
+      # `must_not` detectors. "tool_error" is special (inspects statuses); the rest
+      # are content detectors over the output text.
+      def must_not_checks(golden, result)
+        golden.must_not.map do |name|
+          if name == "tool_error"
+            bad = result.errored_tools
+            Check.new(name: "must_not:tool_error", pass: bad.empty?,
+                      detail: bad.empty? ? "no tool errors" : "errored: #{bad.map { |t| t['name'] || t[:name] }.join(', ')}")
+          else
+            hit = detect(name, result.output_text.to_s)
+            Check.new(name: "must_not:#{name}", pass: hit.nil?,
+                      detail: hit ? "matched #{hit.inspect}" : "clean")
+          end
+        end
+      end
+
+      # The declared `policy`, checked deterministically over the conversation. No
+      # policy -> no check (and nothing to explain in the report).
+      def policy_checks(golden, turns)
+        name = golden.policy
+        return [] if name.nil?
+
+        # Exhaustive on purpose: a policy added to POLICIES without a rule here would
+        # otherwise fall into whichever branch was last and check the wrong thing.
+        pass, detail = case name
+                       when "ask_once" then ask_once(turns)
+                       when "investigate_first" then investigate_first(turns.first)
+                       when "act_fast" then act_fast(turns.first)
+                       else raise ArgumentError, "policy #{name.inspect} has no rule"
+                       end
+        [Check.new(name: "policy:#{name}", pass: pass, detail: detail)]
+      end
+
+      # Every reply asks at most one question. Reported with the offending turn and
+      # the reply itself — "2 questions" alone sends the reader digging.
+      def ask_once(turns)
+        offender = turns.each_with_index.find { |t, _| count_questions(t.output_text) > 1 }
+        return [true, "at most one question per reply"] unless offender
+
+        turn, i = offender
+        [false, "turn #{i + 1} asked #{count_questions(turn.output_text)} questions: " \
+                "#{turn.output_text.to_s.strip[0, 160].inspect}"]
+      end
+
+      def investigate_first(turn)
+        return [false, "no turn to check"] if turn.nil?
+
+        tools = turn.tool_names
+        return [false, "called #{tools.join(', ')} before asking anything"] unless tools.empty?
+        return [false, "answered without asking: #{turn.output_text.to_s.strip[0, 160].inspect}"] if
+          count_questions(turn.output_text).zero?
+
+        [true, "asked before acting"]
+      end
+
+      def act_fast(turn)
+        return [false, "no turn to check"] if turn.nil?
+
+        tools = turn.tool_names
+        return [true, "acted: called #{tools.join(', ')}"] unless tools.empty?
+
+        [false, "asked instead of acting: #{turn.output_text.to_s.strip[0, 160].inspect}"]
+      end
+
+      # Questions in ONE reply. Deliberately crude and deliberately documented: a run
+      # of "?" counts once ("já pensou??" is one question), and URLs are dropped first
+      # so a tracking link's query string is not read as the agent asking something.
+      # It is a policy signal, not grammar — and it already caught a real violation
+      # ("é pra você ou tá pensando em presentear alguém? E qual seu tamanho?").
+      def count_questions(text)
+        text.to_s.gsub(%r{https?://\S+}, " ").scan(/\?+/).size
+      end
+
+      # Runs a named detector over the text. "pii_leak" = union of all PII detectors;
+      # otherwise a single named pattern. Delegates to the runtime's single source
+      # which itself fails loud on an unknown name (a typo'd assertion must not
+      # silently pass).
+      def detect(name, text)
+        Insika::Safety::Detectors.detect(name, text)
+      end
+    end
+  end
+end

data/lib/insika/evals/baseline.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # gating. A baseline is the accepted state of the golden
+    # set — `{ cases: { id => { pass, score } } }`. A gated run compares against it and
+    # blocks only on a REGRESSION, so known-failing cases don't wedge the gate while a
+    # real drop (a passing case that now fails, or a judge score that fell past the
+    # tolerance) does. That's the pre-merge gate for prompt/tool/model changes.
+    module Baseline
+      Regression = Struct.new(:id, :kind, :detail, keyword_init: true)
+
+      module_function
+
+      # [CaseResult] -> baseline hash. `at` is stamped by the caller (kept out of here
+      # so the module stays deterministic/testable).
+      # A SKIPPED case is left out entirely: writing it as `pass:
+      # false` would accept "this deployment cannot run it" as the accepted state,
+      # and the case would never block anywhere again.
+      def snapshot(results, at:)
+        {
+          "at" => at,
+          "cases" => results.reject(&:skipped?).each_with_object({}) do |r, h|
+            h[r.id] = { "pass" => r.pass?, "score" => (r.judge&.score) }
+          end
+        }
+      end
+
+      def load(path) = JSON.parse(File.read(path))
+
+      def write(path, results, at:)
+        File.write(path, JSON.pretty_generate(snapshot(results, at: at)))
+      end
+
+      # Compares a run against a loaded baseline. -> [Regression]. Only cases present
+      # in BOTH are compared: a new case (no baseline entry) never blocks the gate (it
+      # shows in the report as ❌ but is not a "regression"); document this in README.
+      #   • pass→fail   : baseline pass, now failing (hard regression).
+      #   • pass→skipped: baseline pass, now unrunnable HERE. says the
+      #                   gate never blocks ON a skip, and it does not: a case that was
+      #                   already skipped or unknown stays silent. But a case that used
+      #                   to run on this deployment and no longer can means the agent
+      #                   lost a tool or a declaration — a suite shrinking in silence is
+      #                   exactly what this outcome was added to prevent. Re-baseline if
+      #                   the shrink is intended.
+      #   • score-drop  : baseline score - current judge score > tolerance (quality
+      #                   drift, even if the case still technically passes).
+      def compare(results, baseline, tolerance:)
+        base = baseline["cases"] || {}
+        results.filter_map do |r|
+          b = base[r.id]
+          next unless b
+
+          if b["pass"] && r.skipped?
+            Regression.new(id: r.id, kind: "pass→skipped",
+                           detail: "was passing, now unrunnable here: #{r.skipped}")
+          elsif b["pass"] && !r.pass?
+            Regression.new(id: r.id, kind: "pass→fail", detail: "was passing, now failing")
+          elsif b["score"] && r.judge && (b["score"] - r.judge.score) > tolerance
+            Regression.new(id: r.id, kind: "score-drop",
+                           detail: "judge #{b['score']} -> #{r.judge.score} (> #{tolerance})")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insika/evals/golden.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+# Evals — the quality harness. It lives in `lib/` so the engine itself can
+# call it (the refinement gate of needs to score a candidate agent, and a
+# second copy of the judge would be the worst possible outcome), but it stays a
+# CLIENT: it reaches a running deployment over HTTP through `HttpTransport` and never
+# reads a store directly. `evals/run.rb` is a thin CLI over this module.
+module Insika
+  module Evals
+    # A curated behavior case, loaded from a data file (evals/golden/<agent>/*.yml).
+    # Data, not code — same spirit as tools-as-data. See evals/README.md for the format.
+    Golden = Struct.new(:id, :agent, :turns, :expect, :requires, :reference, :source, keyword_init: true) do
+      # The user messages to replay, in order.
+      def user_turns = turns.map { |t| t["user"] }
+
+      # Tool refs the case expects; a trailing "?" marks OPTIONAL (never fails).
+      # -> [{ name:, optional: }]
+      def tools_called
+        Array(expect["tools_called"]).map do |ref|
+          s = ref.to_s
+          optional = s.end_with?("?")
+          { name: optional ? s[0..-2] : s, optional: optional }
+        end
+      end
+
+      # Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
+      def must_not = Array(expect["must_not"]).map(&:to_s)
+
+      # How much the agent should ask before acting. nil = the store
+      # has no opinion and only the rubric decides.
+      def policy = GoldenLoader.presence(expect["policy"])
+
+      # What the DEPLOYMENT must have for this case to mean anything.
+      # Empty = runs everywhere.
+      def required_tools = Array(requires["tools"]).map(&:to_s)
+      def required_capabilities = Array(requires["capabilities"]).map(&:to_s)
+      def requirements? = !(required_tools + required_capabilities).empty?
+
+      # THE INCUMBENT'S CONVERSATION for the same opening — the other
+      # half of a pairwise comparison. Data in the case, not a store read: the eval is
+      # a client, and a pair that lives in one reviewable file cannot go stale against
+      # a database nobody looked at.
+      def reference_messages = Array(reference["messages"])
+      def reference_source = GoldenLoader.presence(reference["source"])
+      def reference? = !reference_messages.empty?
+
+      # Did a PERSON type part of the reference half? After a handoff the operator's
+      # words are stored as `role: assistant`, and comparing a model to a human
+      # and calling it a win is a lie in both directions — so the pair is LABELLED and
+      # the report never prints the outcome without it.
+      def human_assisted?
+        reference_messages.any? { |m| MessageOrigin.origin_of(m) == MessageOrigin::OPERATOR }
+      end
+
+      # LLM-judge rubric + threshold (consumed in — deferred here).
+      def rubric = expect["rubric"]
+      def min_score = expect["min_score"]
+    end
+
+    # Loads + validates golden files. Fails LOUD on a malformed case — a silently
+    # dropped golden is a hole in the safety net.
+    module GoldenLoader
+      class InvalidGolden < StandardError; end
+
+      module_function
+
+      # Loads every *.yml/*.yaml under `dir` (recursive), sorted by path for a stable
+      # run order. -> [Golden].
+      def load_dir(dir)
+        Dir.glob(File.join(dir, "**", "*.{yml,yaml}")).sort.map { |f| load_file(f) }
+      end
+
+      def load_file(path)
+        raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: false) || {}
+        build(raw, source: path)
+      rescue Psych::SyntaxError => e
+        raise InvalidGolden, "#{path}: invalid YAML — #{e.message}"
+      end
+
+      # hash (string keys) -> validated Golden. `source` is only for error messages.
+      def build(raw, source: "(inline)")
+        raise InvalidGolden, "#{source}: golden must be a mapping" unless raw.is_a?(Hash)
+
+        id = presence(raw["id"]) || (raise InvalidGolden, "#{source}: 'id' is required")
+        agent = presence(raw["agent"]) || (raise InvalidGolden, "#{source}: 'agent' is required (case '#{id}')")
+        turns = normalize_turns(raw["turns"], id: id, source: source)
+        expect = raw["expect"] || {}
+        raise InvalidGolden, "#{source}: 'expect' must be a mapping (case '#{id}')" unless expect.is_a?(Hash)
+
+        validate_policy!(expect["policy"], id: id, source: source)
+        requires = raw["requires"] || {}
+        unless requires.is_a?(Hash)
+          raise InvalidGolden, "#{source}: 'requires' must be a mapping (case '#{id}')"
+        end
+
+        reference = normalize_reference(raw["reference"], id: id, source: source)
+
+        Golden.new(id: id, agent: agent, turns: turns, expect: expect,
+                   requires: requires, reference: reference, source: source)
+      end
+
+      # reference: { "source" => String?, "messages" => [{ "role" =>, "text" =>,
+      # "origin" => }] }. Absent -> {}, and the case simply has nothing to compare
+      # against. Malformed is REFUSED: a reference that half-loads would produce a
+      # pairwise verdict about a transcript nobody wrote.
+      def normalize_reference(raw, id:, source:)
+        return {} if raw.nil?
+        raise InvalidGolden, "#{source}: 'reference' must be a mapping (case '#{id}')" unless raw.is_a?(Hash)
+
+        messages = raw["messages"]
+        unless messages.is_a?(Array) && !messages.empty?
+          raise InvalidGolden, "#{source}: reference needs a non-empty 'messages' array (case '#{id}')"
+        end
+
+        { "source" => presence(raw["source"]),
+          "messages" => messages.each_with_index.map { |m, i| reference_message(m, i, id: id, source: source) } }.compact
+      end
+
+      def reference_message(raw, index, id:, source:)
+        where = "#{source}: reference.messages[#{index}] (case '#{id}')"
+        raise InvalidGolden, "#{where} must be a mapping" unless raw.is_a?(Hash)
+
+        role = presence(raw["role"])
+        raise InvalidGolden, "#{where} needs a 'role' of user or assistant" unless %w[user assistant].include?(role)
+
+        text = presence(raw["text"]) || (raise InvalidGolden, "#{where} needs a non-empty 'text'")
+        # The SAME closed vocabulary the engine stamps. A typo'd marker would
+        # read as "absent" downstream, which is how a human turn gets scored as the
+        # incumbent's model.
+        origin = begin
+          MessageOrigin.parse!(raw["origin"])
+        rescue Insika::ValidationError => e
+          raise InvalidGolden, "#{where}: #{e.message}"
+        end
+        { "role" => role, "text" => text }.merge(origin ? { "origin" => origin } : {})
+      end
+
+      # A typo'd policy must not silently mean "no policy" — the case would go on
+      # passing while the rule it was written for stopped being checked. The
+      # `Assertions` constant is resolved at CALL time (this file loads first, and
+      # assertions.rb touches `Safety::Detectors` at load time).
+      def validate_policy!(value, id:, source:)
+        name = presence(value)
+        return if name.nil? || Assertions::POLICIES.key?(name)
+
+        raise InvalidGolden, "#{source}: unknown policy #{name.inspect} (case '#{id}') — " \
+                             "known: #{Assertions::POLICIES.keys.join(', ')}"
+      end
+
+      # turns: a non-empty array of { "user" => String }. Rejects anything else so a
+      # typo (e.g. `users:`) surfaces at load time, not as an empty replay.
+      def normalize_turns(turns, id:, source:)
+        unless turns.is_a?(Array) && !turns.empty?
+          raise InvalidGolden, "#{source}: 'turns' must be a non-empty array (case '#{id}')"
+        end
+
+        turns.each_with_index.map do |t, i|
+          user = t.is_a?(Hash) ? presence(t["user"]) : nil
+          user || (raise InvalidGolden, "#{source}: turns[#{i}] needs a non-empty 'user' (case '#{id}')")
+          { "user" => user }
+        end
+      end
+
+      def presence(v)
+        s = v.to_s.strip
+        s.empty? ? nil : s
+      end
+    end
+  end
+end