@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/evaluate.ts

@@ -0,0 +1,333 @@
+/**
+ * scripts/evaluate.ts — EV-048 `*evaluate` (GATE verdict + variance view).
+ * ---------------------------------------------------------------------------
+ * Runs the built suite vs a subject → per-criterion binary+confidence → a
+ * severity-gated GATE verdict (binary) + the agent-variance view (EV-054).
+ *
+ *   GATE          — binary, severity-gated: a FAILING criterion whose severity
+ *                   is in the gating set (default {CRIT, HIGH}) fails the gate.
+ *                   MED/LOW failures are reported but do not gate.
+ *   variance view — eval-SCORE variance across reruns (does the verdict flap?)
+ *                   + TRAJECTORY variance (does the agent take different tool
+ *                   paths across reruns?). A trustworthy eval is consistent
+ *                   BEFORE it is accurate (variance control precedes accuracy).
+ *
+ * PURE rollup — no clock/random/network; same inputs → identical verdict.
+ * Reuses the v1 Severity enum (same package, not sealed-sibling).
+ */
+import {
+  OutcomeVerdict,
+  RunVerdict,
+  type CriterionVerdict,
+  type OutcomeVerdictValue,
+  type RunVerdictValue,
+} from "./contracts/eval-types.ts";
+import { Severity, type SeverityValue } from "./contracts/types.ts";
+import {
+  EMPTY_EXPECTED_FLOW,
+  type ExpectedFlow,
+  type ExpectedFlowEdge,
+} from "./contracts/flow-graph.ts";
+
+/** One criterion's single-run verdict plus its gating severity. */
+export interface GradedCriterion {
+  criterionId: string;
+  severity: string;
+  verdict: CriterionVerdict;
+}
+
+export interface GateInput {
+  criteria: GradedCriterion[];
+  /** failing criteria at these severities fail the gate (default CRIT+HIGH). */
+  gatingSeverities?: string[];
+}
+
+export interface GateRef {
+  criterionId: string;
+  severity: string;
+}
+
+export interface GateResult {
+  /** back-compat: true iff `runVerdict === "pass"` (no CRIT/HIGH fail OR indeterminate). */
+  passed: boolean;
+  /**
+   * GA — the ternary RUN verdict `fail ▸ incomplete ▸ pass`:
+   *   - fail        — ≥1 CRIT/HIGH criterion FAILED.
+   *   - incomplete  — no CRIT/HIGH fail, but ≥1 CRIT/HIGH was INDETERMINATE
+   *                   (`uncertain`). The latent-false-green killer — a run that
+   *                   could not be certified is NOT a pass.
+   *   - pass        — every CRIT/HIGH criterion passed.
+   */
+  runVerdict: RunVerdictValue;
+  /** every failing criterion (any severity). */
+  failedCriteria: GateRef[];
+  /** the subset that actually GATED (FAILED + severity in the gating set). */
+  gatedBy: GateRef[];
+  /** GA — CRIT/HIGH criteria that were INDETERMINATE (drive `incomplete`). */
+  indeterminateBy: GateRef[];
+  total: number;
+  passCount: number;
+}
+
+const DEFAULT_GATING: readonly SeverityValue[] = [Severity.Crit, Severity.High];
+
+/**
+ * Severity-gated TERNARY GATE (GA). A CRIT/HIGH FAIL fails the run; a CRIT/HIGH
+ * INDETERMINATE (uncertain) — with no CRIT/HIGH fail — makes the run INCOMPLETE
+ * (never a silent pass); else pass. `failedCriteria` keeps the legacy "any
+ * non-pass at any severity" set for back-compat reporting. PURE.
+ */
+export function evaluateGate(input: GateInput): GateResult {
+  const gating = new Set<string>(input.gatingSeverities ?? DEFAULT_GATING);
+  const failedCriteria: GateRef[] = [];
+  const gatedBy: GateRef[] = [];
+  const indeterminateBy: GateRef[] = [];
+  let passCount = 0;
+
+  for (const c of input.criteria) {
+    const result = c.verdict.result;
+    if (result === OutcomeVerdict.Pass) {
+      passCount += 1;
+      continue;
+    }
+    const ref: GateRef = { criterionId: c.criterionId, severity: c.severity };
+    if (result === OutcomeVerdict.Fail) {
+      // a hard fail (legacy failedCriteria + the real gate).
+      failedCriteria.push(ref);
+      if (gating.has(c.severity)) gatedBy.push(ref);
+    } else {
+      // uncertain / indeterminate — counted in legacy failedCriteria, but it
+      // drives INCOMPLETE (not fail) at gating severity.
+      failedCriteria.push(ref);
+      if (gating.has(c.severity)) indeterminateBy.push(ref);
+    }
+  }
+
+  const runVerdict: RunVerdictValue =
+    gatedBy.length > 0
+      ? RunVerdict.Fail
+      : indeterminateBy.length > 0
+        ? RunVerdict.Incomplete
+        : RunVerdict.Pass;
+
+  return {
+    passed: runVerdict === RunVerdict.Pass,
+    runVerdict,
+    failedCriteria,
+    gatedBy,
+    indeterminateBy,
+    total: input.criteria.length,
+    passCount,
+  };
+}
+
+// ── Variance view (EV-054) ──────────────────────────────────────────────────
+
+/** Map a verdict to a numeric score: pass=1, fail=0, uncertain=0.5. */
+function scoreOf(v: OutcomeVerdictValue): number {
+  if (v === OutcomeVerdict.Pass) return 1;
+  if (v === OutcomeVerdict.Fail) return 0;
+  return 0.5;
+}
+
+export interface ScoreVariance {
+  mean: number;
+  variance: number;
+  /** consistent iff variance is exactly 0 (every rerun agreed). */
+  consistent: boolean;
+}
+
+/** Population variance of the eval scores across reruns of one criterion. PURE. */
+export function evalScoreVariance(results: OutcomeVerdictValue[]): ScoreVariance {
+  if (results.length === 0) return { mean: 0, variance: 0, consistent: true };
+  const scores = results.map(scoreOf);
+  const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
+  const variance = scores.reduce((a, s) => a + (s - mean) * (s - mean), 0) / scores.length;
+  return { mean, variance, consistent: variance === 0 };
+}
+
+export interface TrajectoryVariance {
+  distinct: number;
+  total: number;
+  /** stable iff every rerun took the identical tool trajectory. */
+  stable: boolean;
+}
+
+/** Distinct-trajectory count across reruns (a trajectory = ordered tool names). PURE. */
+export function trajectoryVariance(trajectories: string[][]): TrajectoryVariance {
+  const keys = trajectories.map((t) => t.join("␟")); // unit-separator join
+  const distinct = new Set(keys).size;
+  return {
+    distinct,
+    total: trajectories.length,
+    stable: distinct <= 1,
+  };
+}
+
+// ── rollupScorecard — the *evaluate output ──────────────────────────────────
+
+export interface CriterionReruns {
+  scores: OutcomeVerdictValue[];
+  trajectories: string[][];
+}
+
+export interface ScorecardInput {
+  criteria: GradedCriterion[];
+  gatingSeverities?: string[];
+  /** criterionId -> rerun observations (for the variance view). */
+  reruns?: Record<string, CriterionReruns>;
+}
+
+export interface CriterionVarianceView {
+  score: ScoreVariance;
+  trajectory: TrajectoryVariance;
+}
+
+export interface Scorecard {
+  gate: GateResult;
+  variance: Record<string, CriterionVarianceView>;
+}
+
+/** Combine the severity-gated GATE with the per-criterion variance view. PURE. */
+export function rollupScorecard(input: ScorecardInput): Scorecard {
+  const gate = evaluateGate({
+    criteria: input.criteria,
+    ...(input.gatingSeverities !== undefined ? { gatingSeverities: input.gatingSeverities } : {}),
+  });
+  const variance: Record<string, CriterionVarianceView> = {};
+  const reruns = input.reruns ?? {};
+  for (const [criterionId, obs] of Object.entries(reruns)) {
+    variance[criterionId] = {
+      score: evalScoreVariance(obs.scores),
+      trajectory: trajectoryVariance(obs.trajectories),
+    };
+  }
+  return { gate, variance };
+}
+
+// ── EV-054 deepened: agent-appropriate variance (supersedes blind 15-dim) ─────
+//
+// For an AGENT, "variance" is not a fixed 15-dimension determinism scorecard — it
+// is (a) does the eval VERDICT flap across reruns, (b) does the agent's
+// TRAJECTORY flap in shape (tool path / turn count / sub-agent count), and
+// (c) does the trajectory CONFORM to the subject's expected information-flow
+// (EV-037). A trustworthy agent eval is consistent in all three BEFORE it is
+// accurate. All PURE — no clock/random/network.
+
+/** Spread of a list of integers (min/max/distinct) — turn/sub-agent counts. */
+export interface IntSpread {
+  min: number;
+  max: number;
+  distinct: number;
+}
+
+function intSpread(values: number[]): IntSpread {
+  if (values.length === 0) return { min: 0, max: 0, distinct: 0 };
+  return {
+    min: Math.min(...values),
+    max: Math.max(...values),
+    distinct: new Set(values).size,
+  };
+}
+
+/** Deepened trajectory variance: tool-sequence + turn-count + sub-agent-count. PURE. */
+export interface TrajectoryShapeVariance {
+  /** distinct ordered tool sequences across reruns. */
+  distinctToolSeqs: number;
+  /** spread of turn counts (trajectory lengths). */
+  turnCounts: IntSpread;
+  /** spread of sub-agent dispatch counts (tools ∈ dispatchToolNames). */
+  subAgentCounts: IntSpread;
+  /** stable iff every rerun took the identical tool trajectory. */
+  stable: boolean;
+}
+
+/**
+ * Trajectory shape variance across reruns. `dispatchToolNames` (subject-supplied,
+ * EV-037) lets the sub-agent-count dimension be subject-agnostic — when omitted,
+ * sub-agent counts are all 0. PURE.
+ */
+export function trajectoryShapeVariance(
+  trajectories: string[][],
+  dispatchToolNames: string[] = [],
+): TrajectoryShapeVariance {
+  const dispatch = new Set(dispatchToolNames);
+  const distinctToolSeqs = new Set(trajectories.map((t) => t.join("␟"))).size;
+  const turnCounts = intSpread(trajectories.map((t) => t.length));
+  const subAgentCounts = intSpread(
+    trajectories.map((t) => t.filter((name) => dispatch.has(name)).length),
+  );
+  return {
+    distinctToolSeqs,
+    turnCounts,
+    subAgentCounts,
+    stable: distinctToolSeqs <= 1,
+  };
+}
+
+/** Trajectory-vs-expected-flow divergence (EV-054 · EV-037). */
+export interface FlowDivergence {
+  expected: number;
+  covered: number;
+  /** expected edges whose `fromTool → toTool` order is NOT respected in the path. */
+  uncovered: ExpectedFlowEdge[];
+  /** conformant iff every expected edge is covered (fromTool precedes toTool). */
+  conformant: boolean;
+}
+
+/**
+ * Does one observed trajectory (ordered tool names) respect the subject's
+ * expected information-flow? An expected edge `fromTool → toTool` is COVERED when
+ * `fromTool` occurs in the path and `toTool` occurs AFTER its first occurrence.
+ * Uncovered edges = the agent skipped/reordered an expected handoff. PURE +
+ * deterministic; SUBJECT-AGNOSTIC (the expected-flow is supplied).
+ */
+export function trajectoryFlowDivergence(
+  trajectory: string[],
+  expected: ExpectedFlow = EMPTY_EXPECTED_FLOW,
+): FlowDivergence {
+  const uncovered: ExpectedFlowEdge[] = [];
+  let covered = 0;
+  for (const edge of expected.edges) {
+    const fromIdx = trajectory.indexOf(edge.fromTool);
+    const toIdx = trajectory.indexOf(edge.toTool, fromIdx + 1);
+    if (fromIdx !== -1 && toIdx !== -1) covered += 1;
+    else uncovered.push(edge);
+  }
+  return {
+    expected: expected.edges.length,
+    covered,
+    uncovered,
+    conformant: uncovered.length === 0,
+  };
+}
+
+/** The agent-appropriate variance view for one criterion across reruns. */
+export interface AgentVarianceView {
+  score: ScoreVariance;
+  shape: TrajectoryShapeVariance;
+  /** per-rerun trajectory conformance to the expected-flow. */
+  flow: FlowDivergence[];
+}
+
+export interface AgentVarianceInput {
+  scores: OutcomeVerdictValue[];
+  trajectories: string[][];
+  expectedFlow?: ExpectedFlow;
+  dispatchToolNames?: string[];
+}
+
+/**
+ * The deepened EV-054 view: score variance + trajectory-shape variance + per-rerun
+ * expected-flow conformance. This is the agent-appropriate replacement for the
+ * blind 15-dim determinism scorecard. PURE.
+ */
+export function agentVarianceView(input: AgentVarianceInput): AgentVarianceView {
+  const expectedFlow = input.expectedFlow ?? EMPTY_EXPECTED_FLOW;
+  const dispatch = input.dispatchToolNames ?? expectedFlow.dispatchToolNames;
+  return {
+    score: evalScoreVariance(input.scores),
+    shape: trajectoryShapeVariance(input.trajectories, dispatch),
+    flow: input.trajectories.map((t) => trajectoryFlowDivergence(t, expectedFlow)),
+  };
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/flow-graph.ts

@@ -0,0 +1,230 @@
+/**
+ * scripts/flow-graph.ts — EV-032 trace → information-flow graph (THE FOUNDATION).
+ * ---------------------------------------------------------------------------
+ * Deterministic adapter: an `EvalTrace` (events + TOOL observations + sub-agent
+ * dispatches) → a subject-agnostic information-flow graph. Nodes are the trace's
+ * observations viewed as producers/consumers of named data slots; edges are the
+ * data handoffs (a producer's output value that reaches a later consumer's input).
+ *
+ * This is what "lets the evaluator SEE an agent's context-flow": the context-flow
+ * audit (EV-028 tool-result threading · EV-029 sub-agent handoff completeness)
+ * reasons over THIS structured graph instead of the raw trace, and the
+ * deterministic `diffExpectedFlow` flags which expected threadings are missing
+ * (the Code-only half of the Hybrid; the judge interprets severity).
+ *
+ * SUBJECT-AGNOSTIC: producers/consumers are named by the trace's OWN observation
+ * names; the only subject vocabulary (which tool names are sub-agent dispatches)
+ * is SUPPLIED via opts (auto-gen EV-049 / authored EV-037), never a constant.
+ *
+ * PURE + deterministic: no clock / random / network; same trace → same graph.
+ * Threading is a deterministic CONTENT-OVERLAP signal (a producer's emitted
+ * value string appears verbatim in a later consumer's serialized input).
+ */
+import type { EvalTrace, TraceObservation } from "./contracts/eval-types.ts";
+import {
+  EMPTY_EXPECTED_FLOW,
+  FlowEdgeKind,
+  TEXT_SLOT,
+  VALUE_SLOT,
+  type ExpectedFlow,
+  type ExpectedFlowDiff,
+  type ExpectedFlowEdge,
+  type FlowEdge,
+  type FlowGraph,
+  type FlowNode,
+  type UnthreadedOutput,
+} from "./contracts/flow-graph.ts";
+
+/** Options for `buildFlowGraph`. All optional → standalone-pure default. */
+export interface FlowGraphOptions {
+  /**
+   * Tool/step names that are sub-agent DISPATCHES (EV-029). A node with one of
+   * these names produces a context HANDOFF rather than a plain tool result.
+   * SUPPLIED by the subject profile — never a module constant.
+   */
+  dispatchToolNames?: string[];
+  /**
+   * Minimum length of a producer slot's value string for it to count as a
+   * threadable token (avoids trivial `true`/`1`/`""` matches). Default 12.
+   */
+  minThreadLen?: number;
+}
+
+const DEFAULT_MIN_THREAD_LEN = 12;
+
+/** Stable node id by position. */
+function nodeId(index: number): string {
+  return `n${index}`;
+}
+
+/** Top-level slot keys of a payload. Primitive → a single sentinel slot. */
+function slotKeysOf(payload: unknown, primitiveSlot: string): string[] {
+  if (payload === undefined || payload === null) return [];
+  if (typeof payload === "object" && !Array.isArray(payload)) {
+    return Object.keys(payload as Record<string, unknown>).sort();
+  }
+  // primitive or array → one opaque value slot
+  return [primitiveSlot];
+}
+
+/** The string value of a producer slot (for content-overlap threading). */
+function slotValueString(payload: unknown, slot: string): string {
+  if (slot === VALUE_SLOT || slot === TEXT_SLOT) {
+    return typeof payload === "string" ? payload : safeJson(payload);
+  }
+  if (payload !== null && typeof payload === "object") {
+    const v = (payload as Record<string, unknown>)[slot];
+    return typeof v === "string" ? v : safeJson(v);
+  }
+  return "";
+}
+
+/** Deterministic JSON serialization (sorted keys), tolerant of cycles. */
+function safeJson(value: unknown): string {
+  try {
+    return JSON.stringify(value, (_k, v) =>
+      v !== null && typeof v === "object" && !Array.isArray(v)
+        ? sortObject(v as Record<string, unknown>)
+        : v,
+    );
+  } catch {
+    return "";
+  }
+}
+
+function sortObject(o: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const k of Object.keys(o).sort()) out[k] = o[k];
+  return out;
+}
+
+/** The serialized consumer-input string for a node (what later context carries). */
+function consumerText(o: TraceObservation): string {
+  if (o.input === undefined) return "";
+  return typeof o.input === "string" ? o.input : safeJson(o.input);
+}
+
+/** Build one node from an observation. */
+function toNode(o: TraceObservation, index: number, dispatch: Set<string>): FlowNode {
+  const name = typeof o.name === "string" ? o.name : "";
+  return {
+    id: nodeId(index),
+    index,
+    type: typeof o.type === "string" ? o.type : "UNKNOWN",
+    name,
+    produces: slotKeysOf(o.output, VALUE_SLOT),
+    consumes: slotKeysOf(o.input, TEXT_SLOT),
+    isHandoff: name.length > 0 && dispatch.has(name),
+  };
+}
+
+/**
+ * Build the information-flow graph for a trace. PURE + deterministic.
+ *
+ * Edges are emitted when a producer slot's value string (length ≥ minThreadLen)
+ * appears verbatim in a LATER node's serialized input — the deterministic
+ * "this output was threaded into that context" signal. A producer slot that
+ * matched NO downstream consumer is recorded in `unthreadedOutputs` (the EV-028
+ * candidate the judge then adjudicates).
+ */
+export function buildFlowGraph(
+  trace: EvalTrace,
+  opts: FlowGraphOptions = {},
+): FlowGraph {
+  const dispatch = new Set(opts.dispatchToolNames ?? []);
+  const minLen = opts.minThreadLen ?? DEFAULT_MIN_THREAD_LEN;
+  const obs = trace.observations ?? [];
+  const nodes = obs.map((o, i) => toNode(o, i, dispatch));
+
+  // Pre-serialize every node's consumer-input once (O(n) not O(n²·serialize)).
+  const consumerTexts = obs.map((o) => consumerText(o));
+
+  const edges: FlowEdge[] = [];
+  const unthreaded: UnthreadedOutput[] = [];
+
+  for (let p = 0; p < nodes.length; p += 1) {
+    const producer = nodes[p];
+    for (const slot of producer.produces) {
+      const value = slotValueString(obs[p].output, slot).trim();
+      if (value.length < minLen) continue; // trivial value → not a threadable token
+      let threaded = false;
+      for (let c = p + 1; c < nodes.length; c += 1) {
+        if (consumerTexts[c].includes(value)) {
+          edges.push({
+            from: producer.id,
+            to: nodes[c].id,
+            slot,
+            kind: producer.isHandoff ? FlowEdgeKind.Handoff : FlowEdgeKind.Threaded,
+          });
+          threaded = true;
+        }
+      }
+      if (!threaded) {
+        unthreaded.push({ node: producer.id, name: producer.name, slot });
+      }
+    }
+  }
+
+  return {
+    traceId: trace.id,
+    nodes,
+    edges,
+    unthreadedOutputs: unthreaded,
+  };
+}
+
+/**
+ * Diff an actual flow-graph against an expected-flow (EV-037). DETERMINISTIC: an
+ * expected edge `fromTool → toTool [slot]` is SATISFIED iff the graph has an edge
+ * from a node named `fromTool` to a node named `toTool` (matching `slot` when the
+ * expected edge pins one). Missing edges are the lossy-handoff / dropped-thread
+ * candidates the context-flow judge then adjudicates (EV-028/029). This is the
+ * Code-only half of the Hybrid — it decides nothing about severity.
+ */
+export function diffExpectedFlow(
+  graph: FlowGraph,
+  expected: ExpectedFlow = EMPTY_EXPECTED_FLOW,
+): ExpectedFlowDiff {
+  const nameById = new Map(graph.nodes.map((n) => [n.id, n.name]));
+  const satisfied: ExpectedFlowEdge[] = [];
+  const missing: ExpectedFlowEdge[] = [];
+
+  for (const exp of expected.edges) {
+    const hit = graph.edges.some((e) => {
+      const fromName = nameById.get(e.from);
+      const toName = nameById.get(e.to);
+      if (fromName !== exp.fromTool || toName !== exp.toTool) return false;
+      return exp.slot === undefined || e.slot === exp.slot;
+    });
+    if (hit) satisfied.push(exp);
+    else missing.push(exp);
+  }
+
+  return { satisfied, missing };
+}
+
+/**
+ * The deterministic context-flow CANDIDATE leaks the EV-028/029 judge then
+ * adjudicates (Code-only PREP, the deterministic half of the Hybrid):
+ *   - `unthreadedOutputs` — producer slots whose value never reached a consumer
+ *     (EV-028 tool-result-threading candidates).
+ *   - `lossyHandoffs`     — expected-flow edges ABSENT from the actual graph
+ *     (EV-029 sub-agent-handoff-completeness candidates: the dispatch brief did
+ *     not carry the context the expected-flow says the child needs).
+ * This DECIDES NOTHING about severity or whether a candidate is a true leak —
+ * that is the context-flow lens' judgment over this structured bundle.
+ */
+export interface ContextFlowCandidates {
+  unthreadedOutputs: UnthreadedOutput[];
+  lossyHandoffs: ExpectedFlowEdge[];
+}
+
+export function contextFlowCandidates(
+  graph: FlowGraph,
+  expected: ExpectedFlow = EMPTY_EXPECTED_FLOW,
+): ContextFlowCandidates {
+  return {
+    unthreadedOutputs: graph.unthreadedOutputs,
+    lossyHandoffs: diffExpectedFlow(graph, expected).missing,
+  };
+}