@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts

@@ -0,0 +1,182 @@
+/**
+ * scripts/cli/audit-run.ts — Mode A entrypoint.
+ * ---------------------------------------------------------------------------
+ * The harness spine: load the named subject profile + the run-bundle, run the
+ * deterministic pass, assemble the two-track scorecard, render the 4-tab report.
+ *
+ *   audit-run.ts <runId> --subject <name> [--bundle-dir <dir>]
+ *                [--subjects-root <dir>] [--out-dir <dir>]
+ *                [--generated-at <iso>]
+ *
+ * The pinned-judge rows (trace-cross-ref / trajectory-diff) and the data-leak /
+ * variance tabs are JUDGE-EXECUTION SEAMS: this deterministic spine runs fully
+ * offline and emits the scorecard + report; the judge-filled tabs are composed
+ * by the workflow harness (workflows/audit.workflow.js) which dispatches the
+ * pinned judge. This CLI never silently fabricates judge verdicts.
+ *
+ * Deterministic spine: generatedAt is injected (default: a fixed sentinel so a
+ * bare run is reproducible; pass --generated-at for a real stamp).
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  countCriteria,
+  loadEvalMatrix,
+  resolveProfilePaths,
+} from "../load-profile.ts";
+import { loadBundle } from "../load-bundle.ts";
+import { runDeterministic, reexecDeterministicCheck } from "../run-deterministic.ts";
+import { assembleScorecard } from "../assemble-scorecard.ts";
+import { renderReport } from "../render-report.ts";
+import { maskedCanonicalJson } from "../mask.ts";
+import { reportDir } from "../artifact-paths.ts";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const PKG_ROOT = join(HERE, "..", "..");
+
+/** Fixed sentinel so a bare audit-run is byte-reproducible without a clock. */
+export const DEFAULT_GENERATED_AT = "1970-01-01T00:00:00.000Z";
+
+interface Args {
+  runId: string;
+  subject: string;
+  bundleDir: string;
+  subjectsRoot: string;
+  outDir?: string;
+  generatedAt: string;
+  emitMasked: boolean;
+}
+
+function parseArgs(argv: string[]): Args {
+  const positional: string[] = [];
+  const flags: Record<string, string | boolean> = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith("--")) {
+      const key = a.slice(2);
+      const next = argv[i + 1];
+      if (next && !next.startsWith("--")) {
+        flags[key] = next;
+        i++;
+      } else {
+        flags[key] = true;
+      }
+    } else {
+      positional.push(a);
+    }
+  }
+  const subject = (flags.subject as string) || "";
+  const runId = positional[0] || "";
+  if (!runId || !subject) {
+    throw new Error(
+      "usage: audit-run.ts <runId> --subject <name> [--bundle-dir <dir>] [--subjects-root <dir>] [--out-dir <dir>] [--generated-at <iso>] [--emit-masked]",
+    );
+  }
+  return {
+    runId,
+    subject,
+    bundleDir:
+      (flags["bundle-dir"] as string) ||
+      join(process.cwd(), ".mutagent/diagnostics", runId),
+    subjectsRoot:
+      (flags["subjects-root"] as string) || join(PKG_ROOT, "subjects"),
+    outDir: (flags["out-dir"] as string) || undefined,
+    generatedAt: (flags["generated-at"] as string) || DEFAULT_GENERATED_AT,
+    emitMasked: flags["emit-masked"] === true,
+  };
+}
+
+export function main(argvIn?: string[]): number {
+  const argv =
+    argvIn ??
+    (typeof Bun !== "undefined" ? Bun.argv.slice(2) : process.argv.slice(2));
+  const args = parseArgs(argv);
+
+  // 1. Subject profile
+  const paths = resolveProfilePaths(args.subjectsRoot, args.subject);
+  const { matrix, errors } = loadEvalMatrix(paths.evalMatrix);
+  if (errors.length > 0) {
+    console.error(
+      `audit-run: subject '${args.subject}' eval-matrix has ${errors.length} schema error(s):`,
+    );
+    for (const e of errors) console.error(`  - ${e}`);
+    return 1;
+  }
+
+  // 2. Run-bundle
+  const { bundle } = loadBundle(args.bundleDir, args.runId);
+
+  // 3. Deterministic pass — WIRE the real live re-exec (Item #4): every mapped
+  // deterministic row runs an actual checkMethod-keyed check over the produced
+  // artifact instead of a presence proxy. In-cell, no sub-agents (PR-ORCH-01),
+  // judge-only (EV-051), C-PIN byte-stable.
+  const det = runDeterministic(matrix, bundle, { liveReexec: reexecDeterministicCheck });
+
+  // 4. Two-track scorecard
+  const scorecard = assembleScorecard({
+    subject: matrix.subject,
+    runId: bundle.runId,
+    generatedAt: args.generatedAt,
+    rows: det.rows,
+  });
+
+  // 5. Render
+  const html = renderReport(scorecard);
+
+  // Output → the UNIFIED evaluator reports root (PRD R-3): reports/<runId>/master-audit.html
+  // (was under .mutagent/diagnostics/<runId>/audit/ — the wrong stage root). bundleDir stays the
+  // INPUT (the run-bundle being audited); the report is written to the evaluator namespace.
+  const outDir = args.outDir ?? reportDir(args.runId);
+  mkdirSync(outDir, { recursive: true });
+  const scPath = join(outDir, "master-audit.scorecard.json");
+  const htmlPath = join(outDir, "master-audit.html");
+  const scJson = args.emitMasked
+    ? maskedCanonicalJson(scorecard)
+    : JSON.stringify(scorecard, null, 2);
+  writeFileSync(scPath, scJson + "\n");
+  writeFileSync(htmlPath, html);
+
+  const coverage = scorecard.coverage;
+
+  console.info(
+    JSON.stringify(
+      {
+        subject: matrix.subject,
+        runId: bundle.runId,
+        criteria: countCriteria(matrix),
+        deterministicRows: det.deterministicCount,
+        judgeRows: det.judgeCount,
+        gateRunPass: scorecard.gate.runPass,
+        coverage,
+        scorecard: scPath,
+        report: htmlPath,
+        bundleWarnings: bundle.warnings.length,
+      },
+      null,
+      2,
+    ),
+  );
+
+  // Coverage honesty (EV-OUT-002): a near-vacuous PASS must never go out
+  // SILENTLY. Surface the skip-rate WARNING loudly on stderr — this is a
+  // warning ONLY; it does NOT change the exit code / gate decision.
+  if (coverage?.coverageWarning) {
+    console.warn(
+      `audit-run: COVERAGE WARNING — only ${coverage.graded}/${coverage.total} ` +
+        `criteria graded (skip-rate ${(coverage.skipRate * 100).toFixed(1)}% > ` +
+        `${(coverage.skipRateWarnThreshold * 100).toFixed(1)}% threshold). ` +
+        `gateRunPass=${scorecard.gate.runPass} is near-vacuous — most criteria were skipped.`,
+    );
+  }
+
+  return scorecard.gate.runPass ? 0 : 1;
+}
+
+declare const Bun: { argv: string[] } | undefined;
+const isMain =
+  typeof import.meta !== "undefined" &&
+  (import.meta as unknown as { main?: boolean }).main === true;
+if (isMain) {
+  process.exit(main());
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/doctor.ts

@@ -0,0 +1,211 @@
+/**
+ * scripts/cli/doctor.ts
+ * Runtime probe + environment validator + JSON health report.
+ * Type A — Pure Script (deterministic environment inspection; READ-ONLY).
+ *
+ * Detects the JS runtime (bun | pnpm-tsx | npx-tsx | node), checks required
+ * environment variables (name-level presence only — values are NEVER read or
+ * surfaced), reports SKILL.md version, lock-file presence, and git availability.
+ * Exits 0 with a structured JSON report — it mutates NOTHING (mirrors the
+ * diagnostics doctor; the evaluator is a reviewer, never an executor).
+ *
+ * Usage: scripts/cli/run.sh scripts/cli/doctor.ts
+ *
+ * Output shape:
+ *   { runtime, env: { present[], missing[] }, version, lockFile, gitAvailable, errors[] }
+ */
+
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join, resolve } from "node:path";
+import { resolveCredential, readFileOrNull } from "../resolve-credential.ts";
+
+// ── Types ─────────────────────────────────────────────────────────────────────
+
+/**
+ * JS runtime executing this script. Corresponds to the runtime selector in
+ * scripts/cli/run.sh: bun → `bun run`, pnpm-tsx → `pnpm exec tsx`,
+ * npx-tsx → `npx tsx`, node → bare node.
+ */
+export type DoctorRuntime = "bun" | "pnpm-tsx" | "npx-tsx" | "node";
+
+export interface DoctorEnvReport {
+  /** Env var names that are SET (non-empty). Values are never surfaced. */
+  present: string[];
+  /** Env var names that are ABSENT or empty. */
+  missing: string[];
+}
+
+export interface DoctorReport {
+  /** JS runtime executing this skill's scripts. */
+  runtime: DoctorRuntime;
+  /** Presence/absence of required env vars (name-level only). */
+  env: DoctorEnvReport;
+  /** Skill version string from SKILL.md frontmatter, or null if unreadable. */
+  version: string | null;
+  /** Lock-file detected in the skill root, or null. */
+  lockFile: string | null;
+  /** Whether `git` is available in PATH (branch-hygiene operations need it). */
+  gitAvailable: boolean;
+  /** Non-fatal warnings or missing-env notices. Exit code is still 0. */
+  errors: string[];
+}
+
+// ── Required env vars ──────────────────────────────────────────────────────────
+//
+// The evaluator's DEFAULT transport is host-runtime agent-dispatch (no provider
+// SDK call), so none of these are hard requirements — they are surfaced as
+// non-fatal notices. LANGFUSE_* enable a live trace SOURCE; GOOGLE_API_KEY is
+// only needed on the OPTIONAL in-house AI-SDK judge substrate.
+const REQUIRED_ENV_VARS: string[] = [
+  "LANGFUSE_PUBLIC_KEY",
+  "LANGFUSE_SECRET_KEY",
+  "LANGFUSE_BASEURL",
+  "GOOGLE_API_KEY",
+];
+
+// ── Runtime detection ─────────────────────────────────────────────────────────
+
+/**
+ * Detect the JS runtime executing this script.
+ * Order: Bun (process.versions.bun) → pnpm (npm_config_user_agent) →
+ * npx/npm (npm_execpath) → bare node.
+ *
+ * NOTE: this detects the SCRIPT RUNNER, not the coding-agent runtime. For the
+ * coding-agent runtime (claude-code, codex, cursor…) see cli/init.ts.
+ */
+export function detectJsRuntime(): DoctorRuntime {
+  const versions = process.versions as Record<string, string | undefined>;
+  if (versions.bun) return "bun";
+
+  const userAgent = process.env.npm_config_user_agent ?? "";
+  if (userAgent.startsWith("pnpm")) return "pnpm-tsx";
+
+  if (process.env.npm_execpath) return "npx-tsx";
+
+  return "node";
+}
+
+// ── Env var check ─────────────────────────────────────────────────────────────
+
+/**
+ * Check which required env vars are present vs missing.
+ * Values are never read or returned — only name-level presence/absence.
+ */
+export function checkEnvVars(
+  required: string[] = REQUIRED_ENV_VARS,
+  projectRoot: string = process.cwd(),
+): DoctorEnvReport {
+  const present: string[] = [];
+  const missing: string[] = [];
+  // Presence = resolvable via the 3-tier resolver (process.env → .env →
+  // .mutagentrc); a creds-store-only secret counts as PRESENT. Value read into
+  // memory only for the presence decision — NEVER surfaced (secret-safety).
+  for (const name of required) {
+    const res = resolveCredential(name, projectRoot, {
+      env: process.env,
+      readFile: readFileOrNull,
+    });
+    if (res.found) present.push(name);
+    else missing.push(name);
+  }
+  return { present, missing };
+}
+
+// ── SKILL.md version ──────────────────────────────────────────────────────────
+
+/**
+ * Read the skill version from SKILL.md frontmatter (the `version:` key, which is
+ * nested under `metadata:` in the evaluator's frontmatter). Returns null if
+ * SKILL.md is absent or the version field is missing.
+ *
+ * SKILL.md lives at <skill-root>/SKILL.md; this script is at
+ * <skill-root>/scripts/cli/doctor.ts → two directories up.
+ */
+export function readSkillVersion(scriptDir: string): string | null {
+  const skillMd = resolve(scriptDir, "../..", "SKILL.md");
+  if (!existsSync(skillMd)) return null;
+  try {
+    const content = readFileSync(skillMd, "utf8");
+    // Match `version: "0.1.0"` or `  version: "0.1.0"` (indented in metadata: block)
+    const match = content.match(/^\s*version:\s*["']?([^"'\n\r]+?)["']?\s*$/m);
+    return match ? match[1].trim() : null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Lock file detection ───────────────────────────────────────────────────────
+
+/**
+ * Detect the first lock file present in the given directory.
+ * Checks: bun.lockb, bun.lock, pnpm-lock.yaml, package-lock.json, yarn.lock.
+ */
+export function detectLockFile(dir: string): string | null {
+  const candidates = ["bun.lockb", "bun.lock", "pnpm-lock.yaml", "package-lock.json", "yarn.lock"];
+  for (const name of candidates) {
+    if (existsSync(join(dir, name))) return name;
+  }
+  return null;
+}
+
+// ── Git availability ──────────────────────────────────────────────────────────
+
+/**
+ * Check whether `git` is available in PATH. Uses execSync with stdio suppressed —
+ * no shell expansion of user input.
+ */
+export function checkGitAvailable(): boolean {
+  try {
+    execSync("git --version", { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ── Main doctor function ──────────────────────────────────────────────────────
+
+/**
+ * Run the full doctor probe and return a structured report. Never throws —
+ * errors are collected in report.errors[]. READ-ONLY: mutates nothing on disk.
+ */
+export function runDoctor(): DoctorReport {
+  const errors: string[] = [];
+
+  // Canonical script dir: realpath-resolve through any symlink chain (the skill
+  // is often dev-symlinked into a worktree; mirrors init.ts cwd-leak safety).
+  const scriptDir = realpathSync(
+    import.meta.dirname ?? import.meta.url.replace(/^file:\/\//, "").replace(/\/[^/]+$/, "")
+  );
+
+  const runtime = detectJsRuntime();
+  const env = checkEnvVars();
+  const version = readSkillVersion(scriptDir);
+  const lockFile = detectLockFile(resolve(scriptDir, "../.."));
+  const gitAvailable = checkGitAvailable();
+
+  if (env.missing.length > 0) {
+    errors.push(
+      `Missing env vars: ${env.missing.join(", ")} ` +
+        `(non-fatal — the DEFAULT agent-dispatch transport needs none; LANGFUSE_* enable a live ` +
+        `trace source, GOOGLE_API_KEY only the OPTIONAL in-house judge substrate)`
+    );
+  }
+  if (!gitAvailable) {
+    errors.push("git not found in PATH — branch-hygiene / route-to-diagnostics operations will fail");
+  }
+  if (version === null) {
+    errors.push("Could not read SKILL.md version");
+  }
+
+  return { runtime, env, version, lockFile, gitAvailable, errors };
+}
+
+// ── CLI entrypoint ────────────────────────────────────────────────────────────
+
+if (import.meta.main) {
+  const report = runDoctor();
+  process.stdout.write(JSON.stringify(report, null, 2) + "\n");
+  process.exit(0);
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/dogfood.ts

@@ -0,0 +1,133 @@
+/**
+ * scripts/cli/dogfood.ts — EV-041..052 AGGREGATE runner (the #7 RUN entrypoint).
+ * ---------------------------------------------------------------------------
+ * Wires the W1 engine cores against a real trace export. This is the I/O shell —
+ * it is reached ONLY from the CLI, never from the `bun test` gate (which
+ * exercises the pure cores + a stub judge). The composition it calls
+ * (run-pipeline.ts) is fully unit-tested.
+ *
+ *   # DEFAULT — agent-dispatch: AGGREGATE the verdict files dispatched subagents
+ *   # wrote on the host runtime (PREP + dispatch happen first — see the protocol).
+ *   dogfood.ts --traces <handover.jsonl|.gz> --criteria <criteria.json>
+ *              --substrate agent-dispatch --verdict-dir <dir> --model <pinned> [--out <report.json>]
+ *
+ *   # OPTIONAL — in-house: run the LIVE google-genai judge (CI/code-based export).
+ *   dogfood.ts --traces <f> --criteria <criteria.json> --substrate in-house --model <gemini-*>
+ *
+ * The DEFAULT substrate (agent-dispatch) calls NO provider — it reads the
+ * verdict files. The in-house substrate THROWS on an unsupported model / missing
+ * GOOGLE_API_KEY (blocked, never a silent swap — model intent sacred). The
+ * criteria come from `*discover-evals` — this CLI does NOT fabricate them; it reads
+ * them from --criteria so the run is grounded, never invented.
+ */
+import { execSync } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+import { parseUnitfJsonl } from "../read-unitf-traces.ts";
+import { runEvalPipeline } from "../run-pipeline.ts";
+import { resolveJudgeModel } from "../judge-provider.ts";
+import { resolveJudgeRuntime, judgeForRuntime, describeSubstrate } from "../substrate.ts";
+import type { JudgeInvoke } from "../determine-outcome.ts";
+import { Substrate, type DiscoveredCriterion } from "../contracts/eval-types.ts";
+import { SubjectKind } from "../route-failures.ts";
+
+function flag(argv: string[], name: string): string | undefined {
+  const i = argv.indexOf(`--${name}`);
+  return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined;
+}
+
+/**
+ * Load the handed-over UniTF trace file as text (`mutagent-cli trace fetch
+ * --export` writes one `UnifiedTrace` record per line). `.gz` → gunzip | jq -c;
+ * else read text. `parseUnitfJsonl` projects each UniTF record → EvalTrace. The
+ * skill NEVER fetches — Helix runs `mutagent-cli trace fetch` before dispatch.
+ */
+function loadNdjson(file: string, limit?: number): string {
+  const headN = limit !== undefined ? ` | head -n ${limit}` : "";
+  if (file.endsWith(".gz")) {
+    // jq -c normalizes the multi-line JSON stream to one compact record per line.
+    return execSync(`gunzip -c ${JSON.stringify(file)} | jq -c '.'${headN}`, {
+      maxBuffer: 1024 * 1024 * 512,
+      encoding: "utf8",
+    });
+  }
+  const text = readFileSync(file, "utf8");
+  if (limit === undefined) return text;
+  return text.split("\n").slice(0, limit).join("\n");
+}
+
+async function main(): Promise<void> {
+  const argv = process.argv.slice(2);
+  const tracesFile = flag(argv, "traces");
+  const criteriaFile = flag(argv, "criteria");
+  const model = flag(argv, "model");
+  const limit = flag(argv, "limit");
+  const out = flag(argv, "out");
+
+  if (tracesFile === undefined || criteriaFile === undefined) {
+    throw new Error("dogfood: --traces <file> and --criteria <criteria.json> are required");
+  }
+
+  // EV-050 judge-runtime (DEFAULT = agent-dispatch). Model intent sacred: resolve
+  // the pinned model exactly (no swap). --model is authoritative for the dogfood.
+  // `--judge-runtime` is the v0.2.0 flag; `--substrate` stays a back-compat alias.
+  const substrate = resolveJudgeRuntime(flag(argv, "judge-runtime") ?? flag(argv, "substrate"));
+  const decision = resolveJudgeModel({ ...(model !== undefined ? { model } : {}) });
+
+  let judge: JudgeInvoke;
+  if (substrate === Substrate.AgentDispatch) {
+    const verdictDir = flag(argv, "verdict-dir");
+    if (verdictDir === undefined) {
+      throw new Error(
+        "dogfood --judge-runtime agent-dispatch: --verdict-dir <dir> is required (the dir the " +
+          "dispatched eval-judge/error-analyst subagents wrote their verdict files into). " +
+          "Run PREP + dispatch first (references/workflows/orchestrator-protocol.md).",
+      );
+    }
+    judge = judgeForRuntime({ substrate, verdictDir }); // reads verdict files — NO provider
+  } else {
+    judge = judgeForRuntime({ substrate, model: decision.model }); // in-house: THROWS if creds absent
+  }
+
+  const ndjson = loadNdjson(tracesFile, limit !== undefined ? Number.parseInt(limit, 10) : undefined);
+  const { traces, skipped } = parseUnitfJsonl(ndjson);
+  const criteria = JSON.parse(readFileSync(criteriaFile, "utf8")) as DiscoveredCriterion[];
+
+  const subjectName = traces[0]?.name ?? "unknown-subject";
+  const result = await runEvalPipeline(traces, judge, {
+    criteria,
+    pin: { modelId: decision.model, temperature: 0 },
+    subject: { kind: SubjectKind.Agent, name: subjectName, path: `subjects/${subjectName}` },
+    producedBy: "mutagent-evaluator/dogfood",
+    // a real run stamp is acceptable here (CLI side); the cores stay clock-free.
+    producedAt: new Date().toISOString(),
+    artifacts: [{ id: "traces", kind: "trace", path: tracesFile }],
+  });
+
+  const substrateDesc = describeSubstrate(substrate);
+  const summary = {
+    substrate,
+    judgeTransport: substrateDesc.transport,
+    callsProvider: substrateDesc.callsProvider,
+    model: decision.model,
+    modelSource: decision.source,
+    tracesLoaded: traces.length,
+    tracesSkipped: skipped,
+    profile: result.profile,
+    gate: result.scorecard.gate,
+    verdicts: result.verdicts,
+    routedFailures: result.handoff.acceptance.criteria,
+  };
+
+  const json = JSON.stringify(summary, null, 2);
+  if (out !== undefined) {
+    writeFileSync(out, json);
+    process.stdout.write(`dogfood: wrote ${out} (gate.passed=${result.scorecard.gate.passed})\n`);
+  } else {
+    process.stdout.write(json + "\n");
+  }
+}
+
+main().catch((err: unknown) => {
+  process.stderr.write(`dogfood FAILED: ${err instanceof Error ? err.message : String(err)}\n`);
+  process.exit(1);
+});