@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/artifact-paths.ts

@@ -0,0 +1,115 @@
+/**
+ * scripts/artifact-paths.ts — P8 (EV-REQ-058): the localized artifact hierarchy resolver.
+ * ---------------------------------------------------------------------------
+ * The EVALUATOR reference implementation of the operator's NO-SPILLOVER goal:
+ * every durable artifact the engine writes resolves under ONE namespaced root
+ * — `<workspace>/.mutagent/evaluator/` — the evaluator's stage sub-dir of the
+ * unified LOCAL `.mutagent/` (sibling to the diagnostics `.mutagent/diagnostics/`
+ * precedent; pattern, not import). A published skill
+ * running in a CONSUMER's repo therefore never litters their tracked source tree:
+ * the root is auto-gitignored (parity with `.mutagent/`).
+ *
+ * Hierarchy (deterministic):
+ *   .mutagent/evaluator/
+ *     runs/<runId>/{ingest,tasks,verdicts,packets,source-map,scorecard}/
+ *     reports/<runId>/
+ *     living-suite/
+ *     datasets/
+ *
+ * PURE + deterministic: `runId` + `cwd` are PASSED IN — there is NO clock/random in
+ * path construction (a runId is never `Date.now()` here), so the same inputs always
+ * yield the same paths (C-PIN-safe: P8 changes WHERE artifacts land, never WHAT).
+ *
+ * NO-SPILLOVER guard: every resolver output flows through `assertUnderRoot`, which
+ * THROWS if a path escapes the root (a `..` traversal or an absolute path outside)
+ * — so even a malicious path-traversal `runId` fails loud instead of writing
+ * outside the namespace.
+ */
+import { isAbsolute, join, relative, resolve } from "node:path";
+
+/**
+ * The single namespaced root every durable artifact lives under — the evaluator's
+ * stage sub-dir of the unified LOCAL `.mutagent/` (operator decision 2026-06-29;
+ * relocated from the former per-stage dot-root).
+ */
+export const ARTIFACT_ROOT_NAME = ".mutagent/evaluator";
+
+/** Resolve the artifact root: `<cwd>/.mutagent/evaluator`. `cwd` passed in (pure). */
+export function artifactRoot(cwd: string = process.cwd()): string {
+  return resolve(cwd, ARTIFACT_ROOT_NAME);
+}
+
+/**
+ * NO-SPILLOVER guard. Resolve `path` and assert it is INSIDE `artifactRoot(cwd)`;
+ * THROW otherwise (a `..` escape or an absolute path outside the root). Returns the
+ * input path unchanged on success (so resolvers can `return assertUnderRoot(...)`).
+ */
+export function assertUnderRoot(path: string, cwd?: string): string {
+  const root = artifactRoot(cwd);
+  const rel = relative(root, resolve(path));
+  if (rel === ".." || rel.startsWith(".." + "/") || rel.startsWith("..\\") || isAbsolute(rel)) {
+    throw new Error(
+      `assertUnderRoot: path '${path}' ESCAPES the artifact root '${root}' — ` +
+        "no-spillover invariant (EV-REQ-058): the engine writes ONLY under its " +
+        "namespaced dot-root, never the consumer's tracked source tree.",
+    );
+  }
+  return path;
+}
+
+/** Internal: join under the root + assert no escape (catches a traversal runId). */
+function underRoot(cwd: string | undefined, ...segments: string[]): string {
+  return assertUnderRoot(join(artifactRoot(cwd), ...segments), cwd);
+}
+
+// ── per-run hierarchy ───────────────────────────────────────────────────────
+
+export function runDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId);
+}
+export function ingestDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId, "ingest");
+}
+export function tasksDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId, "tasks");
+}
+export function verdictsDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId, "verdicts");
+}
+export function packetsDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId, "packets");
+}
+export function sourceMapDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId, "source-map");
+}
+export function scorecardDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "runs", runId, "scorecard");
+}
+
+// ── cross-run hierarchy ─────────────────────────────────────────────────────
+
+export function reportDir(runId: string, cwd?: string): string {
+  return underRoot(cwd, "reports", runId);
+}
+
+// ── the three named report artifacts (evaluator-reports PRD R-1/R-3) ──────────
+// Distinct names under ONE run folder — no `report.html` collision, and *audit no longer
+// writes under `.mutagent/diagnostics/`. Single resolver = single source of truth.
+/** `*evaluate` → the 5-tab judge report. */
+export function evaluationReportPath(runId: string, cwd?: string): string {
+  return underRoot(cwd, "reports", runId, "evaluation-report.html");
+}
+/** `*discover-evals` → the mined-criteria report. */
+export function discoveryReportPath(runId: string, cwd?: string): string {
+  return underRoot(cwd, "reports", runId, "discovery-report.html");
+}
+/** `*audit` → the 4-tab master-audit (now under the evaluator root, not diagnostics). */
+export function masterAuditPath(runId: string, cwd?: string): string {
+  return underRoot(cwd, "reports", runId, "master-audit.html");
+}
+export function livingSuiteDir(cwd?: string): string {
+  return underRoot(cwd, "living-suite");
+}
+export function datasetsDir(cwd?: string): string {
+  return underRoot(cwd, "datasets");
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/assemble-scorecard.ts

@@ -0,0 +1,172 @@
+/**
+ * scripts/assemble-scorecard.ts
+ * ---------------------------------------------------------------------------
+ * The two-track rollup (decision #5). GATE and TREND are assembled SEPARATELY
+ * and NEVER merged.
+ *
+ *   Track-1 GATE  — binary, severity-gated. Component PASS iff 0 CRIT/HIGH rows
+ *                   FAIL. Run PASS iff every component passes. (skip != fail.)
+ *   Track-2 TREND — the manual's 15-dim variance score, carried verbatim from
+ *                   the variance comparator (or an empty/zeroed frame on a
+ *                   single-run audit).
+ *
+ * Pure + deterministic: rows are grouped in stable order; no clock/random. The
+ * generatedAt is injected by the caller so byte-identity masking is satisfiable.
+ */
+import {
+  type CoverageOptions,
+  type DeterministicRowResult,
+  computeCoverage,
+} from "./run-deterministic.ts";
+import {
+  type GateComponent,
+  type RunVerdictValue,
+  type Scorecard,
+  type ScorecardCriterion,
+  type TrendDimension,
+  RowResult,
+  RunVerdict,
+  Severity,
+} from "./contracts/types.ts";
+
+const GATING_SEVERITIES: ReadonlySet<string> = new Set([
+  Severity.Crit,
+  Severity.High,
+]);
+
+export interface AssembleInput {
+  subject: string;
+  runId: string;
+  generatedAt: string;
+  rows: DeterministicRowResult[];
+  /** Track-2 trend dimensions (from variance-compare). Empty on single-run. */
+  trendDimensions?: TrendDimension[];
+  trendRunPair?: { a: string; b: string };
+  /**
+   * Coverage-honesty options (EV-OUT-002). Controls the skip-rate WARNING
+   * threshold only — never alters gate pass/fail. Defaults preserve behavior.
+   */
+  coverageOptions?: CoverageOptions;
+}
+
+/** Group rows by componentId, preserving first-seen order. */
+function groupByComponent(
+  rows: DeterministicRowResult[],
+): { componentId: string; criteria: ScorecardCriterion[] }[] {
+  const order: string[] = [];
+  const map = new Map<string, ScorecardCriterion[]>();
+  for (const r of rows) {
+    if (!map.has(r.componentId)) {
+      map.set(r.componentId, []);
+      order.push(r.componentId);
+    }
+    map.get(r.componentId)!.push(r.criterion);
+  }
+  return order.map((componentId) => ({
+    componentId,
+    criteria: map.get(componentId)!,
+  }));
+}
+
+/** Track-1: a component passes iff no CRIT/HIGH criterion FAILED (skip != fail). */
+function componentPass(criteria: ScorecardCriterion[]): boolean {
+  return !criteria.some(
+    (c) => c.result === RowResult.Fail && GATING_SEVERITIES.has(c.severity),
+  );
+}
+
+/**
+ * GA — the component TERNARY verdict `fail ▸ incomplete ▸ pass`:
+ *   fail        iff a CRIT/HIGH criterion FAILED.
+ *   incomplete  iff (no CRIT/HIGH fail) AND a CRIT/HIGH criterion was INDETERMINATE.
+ *   pass        otherwise.
+ * The v1 audit path never emits `incomplete` rows ⇒ this returns pass/fail only
+ * there (byte-compatible with `componentPass`).
+ */
+function componentVerdict(criteria: ScorecardCriterion[]): RunVerdictValue {
+  const gatingFail = criteria.some(
+    (c) => c.result === RowResult.Fail && GATING_SEVERITIES.has(c.severity),
+  );
+  if (gatingFail) return RunVerdict.Fail;
+  const gatingIndeterminate = criteria.some(
+    (c) => c.result === RowResult.Incomplete && GATING_SEVERITIES.has(c.severity),
+  );
+  return gatingIndeterminate ? RunVerdict.Incomplete : RunVerdict.Pass;
+}
+
+export function assembleScorecard(input: AssembleInput): Scorecard {
+  const grouped = groupByComponent(input.rows);
+
+  const gateComponents: GateComponent[] = grouped.map((g) => ({
+    componentId: g.componentId,
+    pass: componentPass(g.criteria),
+    verdict: componentVerdict(g.criteria),
+    criteria: g.criteria,
+  }));
+
+  // Totals across ALL criteria (both tracks).
+  let pass = 0;
+  let fail = 0;
+  let skip = 0;
+  let critFail = 0;
+  let highFail = 0;
+  let incomplete = 0;
+  for (const g of gateComponents) {
+    for (const c of g.criteria) {
+      if (c.result === RowResult.Pass) pass += 1;
+      else if (c.result === RowResult.Fail) {
+        fail += 1;
+        if (c.severity === Severity.Crit) critFail += 1;
+        else if (c.severity === Severity.High) highFail += 1;
+      } else if (c.result === RowResult.Incomplete) incomplete += 1;
+      else skip += 1;
+    }
+  }
+
+  // GA — the ternary run verdict: fail ▸ incomplete ▸ pass.
+  const runVerdict: RunVerdictValue = gateComponents.some(
+    (g) => g.verdict === RunVerdict.Fail,
+  )
+    ? RunVerdict.Fail
+    : gateComponents.some((g) => g.verdict === RunVerdict.Incomplete)
+      ? RunVerdict.Incomplete
+      : RunVerdict.Pass;
+  // back-compat: runPass is now exactly `runVerdict === "pass"`. For the v1 audit
+  // path (which never emits `incomplete` rows) this is byte-identical to the old
+  // `every(component.pass)`; GA's only behavior delta is incomplete ⇒ runPass=false.
+  const runPass = runVerdict === RunVerdict.Pass;
+
+  // Coverage honesty (EV-OUT-002): graded-vs-total + skip-rate warning, derived
+  // from the SAME totals the gate uses. Warning-only — `runPass` above is the
+  // sole pass/fail authority; coverage NEVER mutates it.
+  const coverage = computeCoverage(
+    { pass, fail, skip },
+    input.coverageOptions,
+  );
+
+  // Track-2 TREND: assembled SEPARATELY. varianceScore = count of dimensions
+  // that diverged beyond target.
+  const dimensions = input.trendDimensions ?? [];
+  const varianceScore = dimensions.filter(
+    (d) => d.divergence === "diverged",
+  ).length;
+
+  const scorecard: Scorecard = {
+    subject: input.subject,
+    runId: input.runId,
+    generatedAt: input.generatedAt,
+    coverage,
+    gate: {
+      runPass,
+      runVerdict,
+      components: gateComponents,
+      totals: { pass, fail, skip, critFail, highFail, incomplete },
+    },
+    trend: {
+      ...(input.trendRunPair ? { runPair: input.trendRunPair } : {}),
+      dimensions,
+      varianceScore,
+    },
+  };
+  return scorecard;
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-dataset.ts

@@ -0,0 +1,186 @@
+/**
+ * scripts/build-dataset.ts — EV-046 `*build-dataset` deterministic half (Type A — DATA only).
+ * ---------------------------------------------------------------------------
+ * The Code-only piece of the 3-way `*build-dataset` Hybrid (operation-inventory):
+ *   - HITL seed interview (~10 tuples)        → AskUserQuestion / chat gate
+ *   - tuple gen · NL query · quality judgment  → `assets/agents/dataset-builder.md`
+ *   - cartesian expand · dedup · near-dup · id · append  → THIS FILE
+ *
+ * Holds NO prompt prose and makes NO realism judgment (that is the agent's). It
+ * only does the deterministic shape: enumerate the dimension cartesian product,
+ * drop exact + near-duplicate cases, assign a content-derived id, and merge into
+ * the growing dataset MONOTONICALLY (existing cases are never dropped). The
+ * near-duplicate REMOVAL is deterministic token-Jaccard — the agent PROPOSES
+ * realism, the script ENFORCES non-redundancy.
+ *
+ * PURE — no clock / random / network. Same (input) → same output (reproducible
+ * datasets, C-PIN-adjacent). Subject-agnostic (dimensions are DATA).
+ */
+import {
+  type Dataset,
+  type DatasetCase,
+  type DatasetTuple,
+  type Dimension,
+  type CaseSourceValue,
+  tupleKey,
+} from "./contracts/dataset.ts";
+import { appendOnly, assertMonotonicGrowth } from "./living-suite.ts";
+
+// ── cartesian tuples (generate-synthetic-data Step 2 mechanics) ──────────────
+
+/** Guard so a pathological dimension set can't blow up memory. */
+const DEFAULT_MAX_TUPLES = 5000;
+
+/**
+ * The full cartesian product of the dimensions' values, in a DETERMINISTIC
+ * order (dimensions in given order, values in given order). THROWS if the
+ * product would exceed `maxTuples` (fail-loud, not silent truncation). PURE.
+ */
+export function cartesianTuples(
+  dimensions: Dimension[],
+  maxTuples = DEFAULT_MAX_TUPLES,
+): DatasetTuple[] {
+  if (dimensions.length === 0) return [];
+  const total = dimensions.reduce((n, d) => n * d.values.length, 1);
+  if (total > maxTuples) {
+    throw new Error(
+      `cartesianTuples: ${total} combinations exceeds maxTuples=${maxTuples}. ` +
+        "Reduce dimensions/values or raise the cap explicitly (no silent truncation).",
+    );
+  }
+  let acc: DatasetTuple[] = [{}];
+  for (const dim of dimensions) {
+    const next: DatasetTuple[] = [];
+    for (const partial of acc) {
+      for (const v of dim.values) next.push({ ...partial, [dim.name]: v });
+    }
+    acc = next;
+  }
+  return acc;
+}
+
+/** Remove exact-duplicate tuples by canonical key, preserving first-seen order. PURE. */
+export function dedupTuples(tuples: DatasetTuple[]): DatasetTuple[] {
+  const seen = new Set<string>();
+  const out: DatasetTuple[] = [];
+  for (const t of tuples) {
+    const k = tupleKey(t);
+    if (!seen.has(k)) {
+      seen.add(k);
+      out.push(t);
+    }
+  }
+  return out;
+}
+
+// ── near-duplicate query filter (generate-synthetic-data Step 5) ─────────────
+
+/** Normalize a query to a token set (lowercase word tokens). PURE. */
+export function queryTokens(q: string): Set<string> {
+  return new Set(
+    q
+      .toLowerCase()
+      .split(/[^a-z0-9]+/)
+      .filter((t) => t.length > 0),
+  );
+}
+
+/** Token-set Jaccard similarity in [0,1]. Two empty sets → 1 (identical). PURE. */
+export function jaccard(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 && b.size === 0) return 1;
+  let inter = 0;
+  for (const t of a) if (b.has(t)) inter++;
+  const union = a.size + b.size - inter;
+  return union === 0 ? 0 : inter / union;
+}
+
+/**
+ * Drop cases whose query is ≥ `threshold` Jaccard-similar to an already-kept
+ * case (the "queries too similar to each other" filter). DETERMINISTIC — first
+ * occurrence in input order is kept. PURE.
+ */
+export function dedupNearQueries(cases: DatasetCase[], threshold = 0.8): DatasetCase[] {
+  const keptTokens: Set<string>[] = [];
+  const out: DatasetCase[] = [];
+  for (const c of cases) {
+    const toks = queryTokens(c.query);
+    const dup = keptTokens.some((k) => jaccard(toks, k) >= threshold);
+    if (!dup) {
+      keptTokens.push(toks);
+      out.push(c);
+    }
+  }
+  return out;
+}
+
+// ── deterministic content-derived case id (no clock/random) ──────────────────
+
+/** FNV-1a 32-bit hex of a string — a stable, dependency-free content hash. PURE. */
+export function fnv1aHex(s: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(16).padStart(8, "0");
+}
+
+/** A stable id derived from the tuple + query (so re-builds collide → dedup). PURE. */
+export function caseId(tuple: DatasetTuple, query: string): string {
+  return `case-${fnv1aHex(`${tupleKey(tuple)}::${query}`)}`;
+}
+
+/** Assemble a DatasetCase with a content-derived id. PURE. */
+export function buildCase(
+  tuple: DatasetTuple,
+  query: string,
+  source: CaseSourceValue,
+  originTraceId?: string,
+  extra?: { selectedBy?: string; rationale?: string },
+): DatasetCase {
+  const c: DatasetCase = { id: caseId(tuple, query), tuple, query, source };
+  if (originTraceId !== undefined) c.originTraceId = originTraceId;
+  if (extra?.selectedBy !== undefined) c.selectedBy = extra.selectedBy;
+  if (extra?.rationale !== undefined) c.rationale = extra.rationale;
+  return c;
+}
+
+// ── monotonic merge into the growing dataset (EV-053 co-develop) ─────────────
+
+/**
+ * Merge incoming cases into existing, deduped by id AND near-duplicate query,
+ * MONOTONICALLY — every existing case is retained (the suite never shrinks); only
+ * genuinely-novel, non-near-duplicate incoming cases are added. DETERMINISTIC.
+ * Routes the append through `living-suite.ts` (`appendOnly` for the id-dedup +
+ * `assertMonotonicGrowth` for the EV-053 invariant); the near-duplicate-query
+ * filter is the dataset-specific specialization layered on top.
+ */
+export function mergeCases(
+  existing: DatasetCase[],
+  incoming: DatasetCase[],
+  threshold = 0.8,
+): DatasetCase[] {
+  const byId = new Set(existing.map((c) => c.id));
+  // near-dup filter the novel cases against the existing queries first, then each other.
+  const keptTokens: Set<string>[] = existing.map((c) => queryTokens(c.query));
+  const novel: DatasetCase[] = [];
+  for (const c of incoming) {
+    if (byId.has(c.id)) continue;
+    const toks = queryTokens(c.query);
+    if (keptTokens.some((k) => jaccard(toks, k) >= threshold)) continue;
+    keptTokens.push(toks);
+    novel.push(c);
+  }
+  const out = appendOnly(existing, novel, (c) => c.id); // monotonic append via living-suite
+  assertMonotonicGrowth(existing, out, (c) => c.id); // EV-053 invariant (fail-loud)
+  return out;
+}
+
+/**
+ * Append cases into a Dataset, bumping the version. MONOTONIC: `cases` only
+ * grows. Returns a NEW Dataset (no mutation). DETERMINISTIC.
+ */
+export function appendToDataset(dataset: Dataset, incoming: DatasetCase[], threshold = 0.8): Dataset {
+  const cases = mergeCases(dataset.cases, incoming, threshold);
+  return { ...dataset, cases, version: dataset.version + 1 };
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-evals.ts

@@ -0,0 +1,93 @@
+/**
+ * scripts/build-evals.ts — EV-043 `*build-evals` judge-SPEC builder (Type A — DATA only).
+ * ---------------------------------------------------------------------------
+ * Assembles the DATA a per-criterion judge needs (the 4-component judge SPEC) +
+ * the held-out TRAIN/EVAL split + the data-leakage guard. The judging itself —
+ * rendering the prompt + producing the verdict — is NOT here:
+ *   - under the DEFAULT agent-dispatch substrate the verdict comes from a
+ *     dispatched `eval-judge` subagent (the rubric is in `eval-judge.md`);
+ *   - the OPTIONAL in-house/export prompt rendering + run-wrapper (`runJudge`)
+ *     live in `judge-prompt-template.ts` (the EV-050 exception).
+ *
+ * Script austerity (operator directive): this script holds NO judge prompt and
+ * NO LLM-reasoning logic. It keeps only the Type-A pieces:
+ *   1. the held-out split (`splitTrainEval`) — deterministic, reproducible;
+ *   2. the data-leakage guard (`assertExemplarsFromTrain`) — few-shot from TRAIN
+ *      only, or THROW;
+ *   3. the 4-component judge SPEC (`buildJudgeSpec`) — DATA (criterion statement,
+ *      binary Pass/Fail definitions, few-shot), NO Likert.
+ * PURE — no clock/random/network.
+ */
+import type { LabeledTrace } from "./sample-traces.ts";
+import {
+  type DiscoveredCriterion,
+  type JudgeExemplar,
+  type JudgeSpec,
+} from "./contracts/eval-types.ts";
+
+/** A pinned judge declaration. Pinned iff modelId present AND temperature===0. */
+export interface JudgePin {
+  modelId: string;
+  temperature: number;
+}
+
+export interface TrainEvalSplit {
+  /** the small TRAIN split — few-shot exemplars are drawn ONLY from here. */
+  train: LabeledTrace[];
+  /** the held-out EVAL split — judges are validated/run against this. */
+  evalSet: LabeledTrace[];
+}
+
+/**
+ * Set aside a TRAIN split (default 15%, the 10-20% band) for few-shot, holding
+ * out the rest for eval. Deterministic: orders by traceId, takes the first
+ * fraction as train (no random, no clock) so splits are reproducible.
+ */
+export function splitTrainEval(
+  labeled: LabeledTrace[],
+  trainFraction = 0.15,
+): TrainEvalSplit {
+  const ordered = [...labeled].sort((a, b) => a.trace.id.localeCompare(b.trace.id));
+  const n = Math.max(0, Math.min(ordered.length, Math.round(ordered.length * trainFraction)));
+  return { train: ordered.slice(0, n), evalSet: ordered.slice(n) };
+}
+
+/**
+ * Guard against data leakage: every few-shot exemplar MUST come from the train
+ * split. A dev/test exemplar used as few-shot is leakage — THROW.
+ */
+export function assertExemplarsFromTrain(
+  exemplars: JudgeExemplar[],
+  trainIds: ReadonlySet<string>,
+): void {
+  for (const ex of exemplars) {
+    if (!trainIds.has(ex.traceId)) {
+      throw new Error(
+        `assertExemplarsFromTrain: exemplar '${ex.traceId}' is NOT in the train ` +
+          "split — using a dev/test trace as a few-shot example is DATA LEAKAGE. " +
+          "Draw few-shot only from splitTrainEval().train.",
+      );
+    }
+  }
+}
+
+/**
+ * Build the judge spec (4 components) for one criterion. Pass def = the
+ * criterion statement; fail def = its negation. Few-shot is validated against
+ * the train split (leakage guard). PURE.
+ */
+export function buildJudgeSpec(
+  criterion: DiscoveredCriterion,
+  fewShot: JudgeExemplar[],
+  trainIds: ReadonlySet<string>,
+): JudgeSpec {
+  assertExemplarsFromTrain(fewShot, trainIds);
+  return {
+    criterionId: criterion.id,
+    statement: criterion.statement,
+    passDefinition: `Pass = ${criterion.statement}.`,
+    failDefinition: `Fail = the agent fails to satisfy: ${criterion.statement}.`,
+    fewShot,
+    judgeKind: criterion.judgeKind,
+  };
+}