@mutagent/evaluator 0.1.0-alpha.2

package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts

@@ -0,0 +1,320 @@
+/**
+ * scripts/route-failures.ts — EV-051 route-failures (judge-only handoff).
+ * ---------------------------------------------------------------------------
+ * "A judge is only a judge." The evaluator FLAGS failures and ROUTES them to
+ * diagnostics — it NEVER fixes. This emits a HandoverBundle for the diagnose
+ * stage: the failing criteria become the acceptance criteria the downstream
+ * diagnose stage must root-cause, and the failing traces/scorecard become the
+ * enumerated `inputs` that cross the boundary.
+ *
+ * Sealed-sibling: the HandoverBundle shape is RE-IMPLEMENTED here (mirroring
+ * the orchestrator's handover-contract.ts SHAPE), NEVER imported across the
+ * package boundary. This is a one-way EMIT (the evaluator produces it; some
+ * diagnose stage consumes it), not a shared frozen contract co-owned by both
+ * packages — so it stays in-package rather than escalating to templates/.
+ *
+ * PURE + deterministic: provenance (produced_by/at) is INJECTED, never a
+ * self-read clock, so the same input yields a byte-identical bundle. The
+ * bundle is a CLOSED object (additionalProperties:false at every level) so a
+ * smuggled undeclared field is caught by validateHandoverBundle.
+ */
+import { type Static, Type } from "@sinclair/typebox";
+import { TypeCompiler } from "@sinclair/typebox/compiler";
+import {
+  OutcomeVerdict,
+  type CriterionFlagValue,
+  type OutcomeVerdictValue,
+  type VerdictBlock,
+} from "./contracts/eval-types.ts";
+import { maskedCanonicalJson } from "./mask.ts";
+
+/** FROZEN contract version (in-package mirror; bump only via reviewed migration). */
+export const HANDOVER_BUNDLE_VERSION = "0.1.0" as const;
+
+export const AdlStage = {
+  Build: "build",
+  Evaluate: "evaluate",
+  Diagnose: "diagnose",
+  Improve: "improve",
+  Audit: "audit",
+} as const;
+export type AdlStageValue = (typeof AdlStage)[keyof typeof AdlStage];
+
+export const SubjectKind = { Skill: "skill", Agent: "agent" } as const;
+export type SubjectKindValue = (typeof SubjectKind)[keyof typeof SubjectKind];
+
+export const ArtifactKind = {
+  Trace: "trace",
+  Dataset: "dataset",
+  Verdict: "verdict",
+  Scorecard: "scorecard",
+  Findings: "findings",
+  Report: "report",
+  Source: "source",
+  Spec: "spec",
+  Config: "config",
+} as const;
+export type ArtifactKindValue = (typeof ArtifactKind)[keyof typeof ArtifactKind];
+
+export const EscalationPolicy = {
+  Escalate: "escalate",
+  Abort: "abort",
+  Proceed: "proceed",
+} as const;
+export type EscalationPolicyValue =
+  (typeof EscalationPolicy)[keyof typeof EscalationPolicy];
+
+// ── TypeBox schemas (closed objects — auditable boundary) ───────────────────
+const SubjectSchema = Type.Object(
+  {
+    kind: Type.Union([Type.Literal(SubjectKind.Skill), Type.Literal(SubjectKind.Agent)]),
+    name: Type.String({ minLength: 1 }),
+    path: Type.String({ minLength: 1 }),
+  },
+  { additionalProperties: false },
+);
+
+const IntentSchema = Type.Object(
+  { command: Type.String({ minLength: 1 }), utterance: Type.Optional(Type.String()) },
+  { additionalProperties: false },
+);
+
+const ArtifactRefSchema = Type.Object(
+  {
+    id: Type.String({ minLength: 1 }),
+    kind: Type.Union([
+      Type.Literal(ArtifactKind.Trace),
+      Type.Literal(ArtifactKind.Dataset),
+      Type.Literal(ArtifactKind.Verdict),
+      Type.Literal(ArtifactKind.Scorecard),
+      Type.Literal(ArtifactKind.Findings),
+      Type.Literal(ArtifactKind.Report),
+      Type.Literal(ArtifactKind.Source),
+      Type.Literal(ArtifactKind.Spec),
+      Type.Literal(ArtifactKind.Config),
+    ]),
+    path: Type.Optional(Type.String({ minLength: 1 })),
+    uri: Type.Optional(Type.String({ minLength: 1 })),
+    sha: Type.Optional(Type.String({ minLength: 1 })),
+    bytes: Type.Optional(Type.Integer({ minimum: 0 })),
+  },
+  { additionalProperties: false },
+);
+
+const PartialLoadSchema = Type.Object(
+  { path: Type.String({ minLength: 1 }), reason: Type.String({ minLength: 1 }) },
+  { additionalProperties: false },
+);
+
+const ContextPackSchema = Type.Object(
+  {
+    rules: Type.Array(Type.String()),
+    memory: Type.Array(Type.String()),
+    partial_loads: Type.Array(PartialLoadSchema),
+  },
+  { additionalProperties: false },
+);
+
+const AcceptanceSchema = Type.Object(
+  { goal: Type.String({ minLength: 1 }), criteria: Type.Array(Type.String()) },
+  { additionalProperties: false },
+);
+
+const ProvenanceSchema = Type.Object(
+  { produced_by: Type.String({ minLength: 1 }), produced_at: Type.String({ minLength: 1 }) },
+  { additionalProperties: false },
+);
+
+const HandoverBundleSchema = Type.Object(
+  {
+    bundle_version: Type.Literal(HANDOVER_BUNDLE_VERSION),
+    adl_stage: Type.Union([
+      Type.Literal(AdlStage.Build),
+      Type.Literal(AdlStage.Evaluate),
+      Type.Literal(AdlStage.Diagnose),
+      Type.Literal(AdlStage.Improve),
+      Type.Literal(AdlStage.Audit),
+    ]),
+    subject: SubjectSchema,
+    intent: IntentSchema,
+    inputs: Type.Array(ArtifactRefSchema),
+    context_pack: ContextPackSchema,
+    acceptance: AcceptanceSchema,
+    provenance: ProvenanceSchema,
+    escalation_policy: Type.Union([
+      Type.Literal(EscalationPolicy.Escalate),
+      Type.Literal(EscalationPolicy.Abort),
+      Type.Literal(EscalationPolicy.Proceed),
+    ]),
+  },
+  { additionalProperties: false },
+);
+export type HandoverBundle = Static<typeof HandoverBundleSchema>;
+export type ArtifactRef = Static<typeof ArtifactRefSchema>;
+
+const BundleChecker = TypeCompiler.Compile(HandoverBundleSchema);
+
+// ── Input ───────────────────────────────────────────────────────────────────
+
+/**
+ * One routed failure. NOTE: there is deliberately NO `fix`/`remediation`/
+ * `patch` field — the evaluator is judge-only. The downstream diagnose stage
+ * decides remedies; the evaluator only states WHAT failed and WHY (critique).
+ */
+export interface FailureRef {
+  criterionId: string;
+  severity: string;
+  flag: CriterionFlagValue;
+  traceId: string;
+  result: OutcomeVerdictValue;
+  critique: string;
+  /** GA — set iff `result === uncertain` (indeterminate); types the calibration route. */
+  blockedBy?: VerdictBlock;
+}
+
+/**
+ * GA — partition routed rows by verdict: a true FAIL routes to diagnostics
+ * (root-cause it), an INDETERMINATE (`uncertain`) routes to the CALIBRATION loop
+ * (re-ground / operator-ratify / re-scope by `blockedBy.kind`) — NEVER to
+ * diagnostics, because there is no confirmed defect to root-cause yet. PURE.
+ */
+export interface RoutingPartition {
+  toDiagnostics: FailureRef[];
+  toCalibrate: FailureRef[];
+}
+export function partitionRouting(failures: FailureRef[]): RoutingPartition {
+  const toDiagnostics: FailureRef[] = [];
+  const toCalibrate: FailureRef[] = [];
+  for (const f of failures) {
+    if (f.result === OutcomeVerdict.Uncertain) toCalibrate.push(f);
+    else if (f.result === OutcomeVerdict.Fail) toDiagnostics.push(f);
+    // pass rows never appear in a failure list; ignore defensively.
+  }
+  return { toDiagnostics, toCalibrate };
+}
+
+/** GA — one calibration-loop item for an indeterminate verdict. The route is the
+ *  `blockedBy.kind`: factual-intent→re-ground · normative→operator · scope→re-scope. */
+export interface CalibrationItem {
+  criterionId: string;
+  severity: string;
+  traceId: string;
+  blockedBy: VerdictBlock;
+  route: "re-ground" | "operator" | "re-scope";
+}
+const KIND_ROUTE: Record<string, CalibrationItem["route"]> = {
+  "factual-intent": "re-ground",
+  normative: "operator",
+  scope: "re-scope",
+};
+export function calibrationItems(failures: FailureRef[]): CalibrationItem[] {
+  return partitionRouting(failures).toCalibrate.map((f) => {
+    const blockedBy: VerdictBlock = f.blockedBy ?? {
+      kind: "factual-intent",
+      text: f.critique,
+    };
+    return {
+      criterionId: f.criterionId,
+      severity: f.severity,
+      traceId: f.traceId,
+      blockedBy,
+      route: KIND_ROUTE[blockedBy.kind] ?? "re-ground",
+    };
+  });
+}
+
+export interface RouteFailuresInput {
+  subject: { kind: SubjectKindValue; name: string; path: string };
+  failures: FailureRef[];
+  artifacts: ArtifactRef[];
+  /** INJECTED provenance — never a self-read clock (determinism). */
+  producedBy: string;
+  producedAt: string;
+  escalationPolicy?: EscalationPolicyValue;
+  /** optional curated context that crossed the boundary. */
+  contextRules?: string[];
+  contextMemory?: string[];
+  partialLoads?: { path: string; reason: string }[];
+}
+
+/**
+ * Build the diagnose-stage HandoverBundle for the routed failures. Judge-only:
+ * no remedy is emitted — the failing criteria become the acceptance criteria
+ * the diagnose stage must root-cause. PURE + deterministic.
+ */
+export function routeFailures(input: RouteFailuresInput): HandoverBundle {
+  // GA — only true FAILs are root-causable defects. Indeterminates route to the
+  // calibration loop (see `calibrationItems`), never into a diagnose handoff.
+  const criteria = partitionRouting(input.failures).toDiagnostics.map(
+    (f) =>
+      `${f.criterionId} [${f.severity}/${f.flag}] FAILED on trace ${f.traceId}: ${f.critique}`,
+  );
+  return {
+    bundle_version: HANDOVER_BUNDLE_VERSION,
+    adl_stage: AdlStage.Diagnose,
+    subject: { kind: input.subject.kind, name: input.subject.name, path: input.subject.path },
+    intent: { command: "*diagnose" },
+    inputs: input.artifacts,
+    context_pack: {
+      rules: input.contextRules ?? [],
+      memory: input.contextMemory ?? [],
+      partial_loads: input.partialLoads ?? [],
+    },
+    acceptance: {
+      goal:
+        "Root-cause the routed eval failures. JUDGE-ONLY handoff: the evaluator " +
+        "flags + routes, it does not repair — the diagnose stage owns the " +
+        "downstream repair.",
+      criteria,
+    },
+    provenance: { produced_by: input.producedBy, produced_at: input.producedAt },
+    escalation_policy: input.escalationPolicy ?? EscalationPolicy.Escalate,
+  };
+}
+
+/**
+ * MASK-ON-HANDOFF (carry). Serialize the handover bundle for EMIT through the
+ * byte-identity masker (`mask.ts`): `provenance.produced_at` (an injected
+ * timestamp) → `<TS>` and any absolute home path that slipped into an artifact
+ * locator → `<ABSPATH>`. RELATIVE artifact paths survive untouched (they are
+ * portable, not a leak).
+ *
+ * WHY: the evaluator's OWN diagnose handoff is itself a run-bundle the data-leak
+ * audit inspects. An un-masked `produced_at` + abs home path is exactly the
+ * operational leak the audit flags — so the evaluator dogfoods its own audit by
+ * masking on emit. It also preserves C-PIN: two runs (or two machines) that
+ * differ ONLY in timestamp / home-root serialize to a BYTE-IDENTICAL handoff.
+ *
+ * The in-memory `routeFailures` bundle keeps its real values (the structured
+ * object is unchanged); ONLY the emitted serialization is masked.
+ */
+export function serializeHandoverBundle(bundle: HandoverBundle): string {
+  return maskedCanonicalJson(bundle);
+}
+
+export interface ValidationResult {
+  ok: boolean;
+  errors: string[];
+}
+
+/**
+ * Validate a candidate bundle against the closed contract + the "every
+ * artifact has at least one locator (path|uri)" rule TypeBox alone can't
+ * express. A smuggled undeclared field fails (additionalProperties:false).
+ */
+export function validateHandoverBundle(candidate: unknown): ValidationResult {
+  const errors: string[] = [];
+  if (!BundleChecker.Check(candidate)) {
+    for (const e of BundleChecker.Errors(candidate)) {
+      errors.push(`${e.path}: ${e.message}`);
+    }
+    return { ok: false, errors };
+  }
+  const bundle = candidate as HandoverBundle;
+  bundle.inputs.forEach((a, i) => {
+    if (a.path === undefined && a.uri === undefined) {
+      errors.push(`inputs/${i}: artifact '${a.id}' has neither path nor uri`);
+    }
+  });
+  return { ok: errors.length === 0, errors };
+}

package/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts

@@ -0,0 +1,271 @@
+/**
+ * scripts/run-deterministic.ts
+ * ---------------------------------------------------------------------------
+ * The deterministic executor — Tab-1 deterministic rows (decision #4). Walks the
+ * subject eval-matrix; for every criterion whose checkMethod is in the
+ * DETERMINISTIC track (deterministic-script | typebox-schema | gate) it runs a
+ * binary pass/fail check with NO model. Judge-track rows (trace-cross-ref |
+ * trajectory-diff) are emitted as `track:judge, result:skip` placeholders that
+ * the pinned-judge seam (run-judge.ts) fills in.
+ *
+ * The deterministic checks are REAL and grounded in the loaded run-bundle:
+ *  - typebox-schema  : the relevant produced artifact must be present + parse as
+ *                      a non-empty object/array (the schema-conformance proxy the
+ *                      bundle can answer offline).
+ *  - gate            : the gate's evidence must be present (e.g. an evidence/
+ *                      file set, a wave6 stamp) — absence => fail-loud.
+ *  - deterministic-script : the artifact the script produces must be present and
+ *                      non-empty in the bundle.
+ *
+ * Where a criterion's deep semantic check genuinely needs to re-execute the
+ * subject's own script against live data (beyond presence/shape), that is an
+ * EXPLICIT, documented integration seam (see `evaluateDeterministic` -> the
+ * `needsLiveReexec` path) rather than a silent pass. Such rows return result
+ * `skip` with a detail explaining the seam, so they never FALSE-PASS.
+ *
+ * Pure + deterministic: components/criteria are processed in matrix order; no
+ * clock/random/network.
+ */
+import {
+  type Component,
+  type Criterion,
+  type EvalMatrix,
+  type RowResultValue,
+  type ScorecardCriterion,
+  RowResult,
+  Track,
+  trackForCheckMethod,
+} from "./contracts/types.ts";
+import { type RunBundle } from "./contracts/types.ts";
+
+export interface DeterministicRowResult {
+  componentId: string;
+  criterion: ScorecardCriterion;
+}
+
+/**
+ * Map a criterion to the artifact whose presence/shape proves it. Subject-
+ * agnostic heuristic: the bundle's well-known artifacts cover the produced
+ * outputs an audit can verify offline. A criterion that references no bundle
+ * artifact is a live-reexec seam.
+ */
+function evidenceArtifactFor(
+  criterion: Criterion,
+  bundle: RunBundle,
+): string | null {
+  // Gate rows about evidence/aggregate require the evidence dir.
+  const s = criterion.statement.toLowerCase();
+  if (s.includes("evidence file") || s.includes("evidence/")) {
+    return bundle.data.evidence ? "evidence" : null;
+  }
+  if (s.includes("wave-6") || s.includes("wave6") || s.includes("stamp")) {
+    return bundle.data.wave6 ? "wave6" : null;
+  }
+  if (s.includes("runmeta") || s.includes("runmeta.")) {
+    return bundle.data.runMeta ? "runMeta" : null;
+  }
+  if (s.includes("entity") || s.includes("diagnosedentity")) {
+    return bundle.data.entityContext ? "entityContext" : null;
+  }
+  if (s.includes("renderinput") || s.includes("render input") || s.includes("heatmap")) {
+    return bundle.data.renderInput ? "renderInput" : null;
+  }
+  if (s.includes("traces-metadata") || s.includes("traces metadata")) {
+    return bundle.data.tracesMetadata ? "tracesMetadata" : null;
+  }
+  return null;
+}
+
+function isNonEmpty(value: unknown): boolean {
+  if (value == null) return false;
+  if (Array.isArray(value)) return value.length > 0;
+  if (typeof value === "object") return Object.keys(value).length > 0;
+  if (typeof value === "string") return value.length > 0;
+  return true;
+}
+
+export interface EvaluateOptions {
+  /**
+   * When false (default), criteria that need a LIVE re-exec of the subject's own
+   * script (beyond bundle presence/shape) return `skip` with a documented seam
+   * note instead of false-passing. A future integration may set this true and
+   * provide a re-exec callback.
+   */
+  liveReexec?: (criterion: Criterion) => RowResultValue | null;
+}
+
+/**
+ * Evaluate ONE deterministic criterion against the bundle. Never throws; returns
+ * a binary pass/fail, or `skip` with a documented reason for a live-reexec seam.
+ */
+export function evaluateDeterministic(
+  criterion: Criterion,
+  bundle: RunBundle,
+  opts: EvaluateOptions = {},
+): ScorecardCriterion {
+  const track = Track.Deterministic;
+  const artifactKey = evidenceArtifactFor(criterion, bundle);
+
+  if (artifactKey == null) {
+    // No bundle artifact answers this row offline -> live-reexec seam.
+    const live = opts.liveReexec?.(criterion) ?? null;
+    if (live) {
+      return {
+        dimension: criterion.dimension,
+        severity: criterion.severity,
+        checkMethod: criterion.checkMethod,
+        track,
+        result: live,
+        detail: "evaluated via live-reexec callback",
+      };
+    }
+    return {
+      dimension: criterion.dimension,
+      severity: criterion.severity,
+      checkMethod: criterion.checkMethod,
+      track,
+      result: RowResult.Skip,
+      detail:
+        "INTEGRATION SEAM: requires live re-exec of the subject's own script (no bundle artifact answers this row offline); skipped rather than false-passed",
+    };
+  }
+
+  const present = isNonEmpty(bundle.data[artifactKey]);
+  return {
+    dimension: criterion.dimension,
+    severity: criterion.severity,
+    checkMethod: criterion.checkMethod,
+    track,
+    result: present ? RowResult.Pass : RowResult.Fail,
+    detail: present
+      ? `evidence artifact '${artifactKey}' present + non-empty`
+      : `evidence artifact '${artifactKey}' MISSING/empty in bundle (fail-loud)`,
+  };
+}
+
+/** Judge-track placeholder — filled in by the pinned-judge seam. */
+export function judgePlaceholder(criterion: Criterion): ScorecardCriterion {
+  return {
+    dimension: criterion.dimension,
+    severity: criterion.severity,
+    checkMethod: criterion.checkMethod,
+    track: Track.Judge,
+    result: RowResult.Skip,
+    detail:
+      "PINNED-JUDGE SEAM: requires pinned model (id + temp=0) reading transcript vs behavior-tree; deferred to run-judge.ts",
+  };
+}
+
+export interface RunDeterministicResult {
+  rows: DeterministicRowResult[];
+  deterministicCount: number;
+  judgeCount: number;
+}
+
+// ── Coverage honesty (EV-OUT-002) ──────────────────────────────────────────
+/**
+ * Default skip-rate at/above which a coverage WARNING is raised. Conservative
+ * (0.5) so a run where more rows were skipped than graded can no longer claim a
+ * silent PASS. This is a WARNING threshold ONLY — it never flips gate pass/fail.
+ */
+export const DEFAULT_SKIP_RATE_WARN_THRESHOLD = 0.5;
+
+/** Fixed precision for skipRate so two audits serialize byte-identically. */
+const SKIP_RATE_PRECISION = 4;
+
+/** The pass/fail/skip tally `assembleScorecard` derives from the graded rows. */
+export interface CoverageTotals {
+  pass: number;
+  fail: number;
+  skip: number;
+}
+
+export interface CoverageOptions {
+  /**
+   * Skip-rate STRICTLY ABOVE which `coverageWarning` is set. Default
+   * {@link DEFAULT_SKIP_RATE_WARN_THRESHOLD}. Warning-only — this never alters
+   * the GATE's pass/fail decision; the honesty mechanism is the surfaced
+   * warning, not a gate flip.
+   */
+  skipRateWarnThreshold?: number;
+}
+
+export interface Coverage {
+  /** rows actually graded (pass + fail) — the non-vacuous denominator. */
+  graded: number;
+  /** every criterion (pass + fail + skip). */
+  total: number;
+  /** rows skipped (seam / no-bundle-artifact / judge placeholder). */
+  skipped: number;
+  /** skipped / total, rounded to fixed precision (0 when total is 0). */
+  skipRate: number;
+  /** the threshold that was applied. */
+  skipRateWarnThreshold: number;
+  /** true iff skipRate exceeds the threshold — surfaces a near-vacuous PASS. */
+  coverageWarning: boolean;
+}
+
+/**
+ * Derive the coverage-honesty stat from a pass/fail/skip tally. Pure +
+ * deterministic: integer counts in, fixed-precision ratio out, no clock/random.
+ *
+ * A high skip-rate means most criteria were never graded — so a `gateRunPass`
+ * of true is near-vacuous. `coverageWarning` makes that LOUD without changing
+ * pass/fail semantics (the gate is decided entirely upstream by fail counts).
+ */
+export function computeCoverage(
+  totals: CoverageTotals,
+  opts: CoverageOptions = {},
+): Coverage {
+  const skipRateWarnThreshold =
+    opts.skipRateWarnThreshold ?? DEFAULT_SKIP_RATE_WARN_THRESHOLD;
+  const graded = totals.pass + totals.fail;
+  const skipped = totals.skip;
+  const total = graded + skipped;
+  const rawRate = total === 0 ? 0 : skipped / total;
+  const factor = 10 ** SKIP_RATE_PRECISION;
+  const skipRate = Math.round(rawRate * factor) / factor;
+  return {
+    graded,
+    total,
+    skipped,
+    skipRate,
+    skipRateWarnThreshold,
+    coverageWarning: skipRate > skipRateWarnThreshold,
+  };
+}
+
+/**
+ * Run the full deterministic pass over a matrix + bundle. Emits one row per
+ * criterion: deterministic rows are graded now; judge rows are placeholders.
+ */
+export function runDeterministic(
+  matrix: EvalMatrix,
+  bundle: RunBundle,
+  opts: EvaluateOptions = {},
+): RunDeterministicResult {
+  const rows: DeterministicRowResult[] = [];
+  let deterministicCount = 0;
+  let judgeCount = 0;
+
+  for (const component of matrix.components as Component[]) {
+    for (const criterion of component.criteria) {
+      const track = trackForCheckMethod(criterion.checkMethod);
+      if (track === Track.Deterministic) {
+        deterministicCount += 1;
+        rows.push({
+          componentId: component.componentId,
+          criterion: evaluateDeterministic(criterion, bundle, opts),
+        });
+      } else {
+        judgeCount += 1;
+        rows.push({
+          componentId: component.componentId,
+          criterion: judgePlaceholder(criterion),
+        });
+      }
+    }
+  }
+
+  return { rows, deterministicCount, judgeCount };
+}