@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-judge.ts

@@ -0,0 +1,155 @@
+/**
+ * scripts/run-judge.ts
+ * ---------------------------------------------------------------------------
+ * The pinned-judge applicator seam (EV-REQ-024 / GAP-JUDGE-SEAM).
+ *
+ * This is the INNER, deterministic, testable HALF of the judge track. The OTHER
+ * half — the live LLM call that PRODUCES verdicts by reading a transcript vs the
+ * behavior-tree — is a documented integration seam at the WORKFLOW layer, never
+ * in this script. Keeping the engine PURE is what makes the judge track
+ * reproducible and unit-testable.
+ *
+ * `applyJudgeVerdicts(rows, verdictMap, judgePin?)` consumes the scorecard rows
+ * (incl. the `judgePlaceholder()` skip rows emitted by run-deterministic.ts —
+ * we CONSUME that placeholder, we do NOT redefine it) plus a CALLER-SUPPLIED
+ * verdict map keyed by a stable per-row id, and resolves each judge placeholder:
+ *
+ *   - no verdict in the map            -> STAYS skip (the judge never fabricates)
+ *   - a non-pinned verdict (temp != 0
+ *     OR missing modelId)              -> REJECTED -> stays skip + records a
+ *                                         determinism-defect detail. A non-pinned
+ *                                         judge can NEVER produce a pass (C-PIN).
+ *   - a pinned pass/fail verdict       -> applies (sets result + records modelId)
+ *   - deterministic-track rows         -> passed through UNTOUCHED
+ *
+ * ── Row-keying convention (needed so the verdictMap key is unambiguous) ───────
+ * ScorecardCriterion carries no stable per-row id. The applicator receives the
+ * flat row list in MATRIX ORDER (the order run-deterministic emits), so the
+ * stable key is `componentId#<indexWithinComponent>` — i.e. the Nth row of a
+ * given component. `rowKey(componentId, indexInComponent)` builds it; callers
+ * that produce verdicts MUST key the map with this same convention.
+ *
+ * Pure + deterministic: a fresh array is returned; inputs are never mutated; no
+ * clock/random/network. Same (rows, verdictMap) -> deep-equal output, always.
+ */
+import { type DeterministicRowResult } from "./run-deterministic.ts";
+import {
+  type RowResultValue,
+  type ScorecardCriterion,
+  RowResult,
+  Track,
+} from "./contracts/types.ts";
+
+/**
+ * A caller-supplied judge verdict for one row. The live judge layer fills these
+ * in; this engine only APPLIES them under the C-PIN invariant. A verdict is
+ * "pinned" iff it carries a modelId AND temperature === 0.
+ */
+export interface JudgeVerdict {
+  /** The pinned model id the judge ran under. Absent => not pinned => rejected. */
+  modelId?: string;
+  /** Sampling temperature. Anything other than 0 => not pinned => rejected. */
+  temperature?: number;
+  /** The judge's claimed result. Only `pass`/`fail` are meaningful; a pinned
+   *  verdict applies it. (A `skip` verdict is treated as no decision.) */
+  result: RowResultValue;
+  /** Optional free-text rationale from the judge (masked downstream). */
+  detail?: string;
+}
+
+/** verdictMap: stable rowKey -> verdict. Keyed by `rowKey(componentId, idx)`. */
+export type JudgeVerdictMap = Record<string, JudgeVerdict>;
+
+/**
+ * Stable per-row key: the Nth criterion of a given component. This is the
+ * documented row-keying convention the verdictMap is keyed by.
+ */
+export function rowKey(componentId: string, indexInComponent: number): string {
+  return `${componentId}#${indexInComponent}`;
+}
+
+/** A verdict is pinned iff it declares a modelId AND temperature === 0. */
+function isPinned(verdict: JudgeVerdict): boolean {
+  return (
+    typeof verdict.modelId === "string" &&
+    verdict.modelId.length > 0 &&
+    verdict.temperature === 0
+  );
+}
+
+/** Build the resolved criterion for a pinned, applicable verdict. */
+function resolvePinned(
+  placeholder: ScorecardCriterion,
+  verdict: JudgeVerdict,
+): ScorecardCriterion {
+  const judged =
+    verdict.result === RowResult.Pass || verdict.result === RowResult.Fail
+      ? verdict.result
+      : RowResult.Skip;
+  const base = `pinned judge '${verdict.modelId}' (temp=0) -> ${judged}`;
+  return {
+    ...placeholder,
+    result: judged,
+    detail: verdict.detail ? `${base}; ${verdict.detail}` : base,
+  };
+}
+
+/** Build the skip+defect criterion for a rejected (non-pinned) verdict. */
+function rejectNonPinned(
+  placeholder: ScorecardCriterion,
+  verdict: JudgeVerdict,
+): ScorecardCriterion {
+  const reason =
+    verdict.modelId == null || verdict.modelId.length === 0
+      ? "missing modelId"
+      : `temperature=${String(verdict.temperature)} (!= 0)`;
+  return {
+    ...placeholder,
+    // C-PIN: a non-pinned judge can NEVER produce a pass — force skip.
+    result: RowResult.Skip,
+    detail:
+      `DETERMINISM DEFECT: verdict is not pinned (${reason}); ` +
+      "a non-pinned judge can never produce a pass — staying skip",
+  };
+}
+
+/**
+ * Apply caller-supplied judge verdicts to the scorecard rows.
+ *
+ * @param rows       scorecard rows in matrix order (deterministic + judge).
+ * @param verdictMap rowKey -> verdict, keyed via `rowKey(componentId, idx)`.
+ * @param _judgePin  reserved: a future global pin assertion hook. Per-verdict
+ *                   pinning (modelId + temp=0) is the enforced contract today.
+ */
+export function applyJudgeVerdicts(
+  rows: DeterministicRowResult[],
+  verdictMap: JudgeVerdictMap,
+  _judgePin?: { modelId: string; temperature: number },
+): DeterministicRowResult[] {
+  // Per-component running index -> the stable rowKey for each judge row.
+  const perComponentIndex = new Map<string, number>();
+
+  return rows.map((r) => {
+    const idx = perComponentIndex.get(r.componentId) ?? 0;
+    perComponentIndex.set(r.componentId, idx + 1);
+
+    // Deterministic-track rows pass through UNTOUCHED.
+    if (r.criterion.track !== Track.Judge) {
+      return { componentId: r.componentId, criterion: { ...r.criterion } };
+    }
+
+    const key = rowKey(r.componentId, idx);
+    const verdict = verdictMap[key];
+
+    // No verdict -> the judge never fabricates -> stay skip (untouched).
+    if (verdict == null) {
+      return { componentId: r.componentId, criterion: { ...r.criterion } };
+    }
+
+    const criterion = isPinned(verdict)
+      ? resolvePinned(r.criterion, verdict)
+      : rejectNonPinned(r.criterion, verdict);
+
+    return { componentId: r.componentId, criterion };
+  });
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts

@@ -0,0 +1,175 @@
+/**
+ * scripts/run-pipeline.ts — the W1 engine composed end-to-end.
+ * ---------------------------------------------------------------------------
+ * Wires the eight engine cores into one pass so the dogfood (#7) is a pure
+ * RUN with no new logic:
+ *
+ *   profileSubject (EV-049)
+ *     → determineOutcome per trace (EV-042, via the DI judge seam)
+ *     → label + balancedSample with a held-out TRAIN/EVAL split (EV-052)
+ *     → build ONE judge per EVAL-WORTHY criterion (EV-043; few-shot from TRAIN
+ *       only, judged on the EVAL split — held-out discipline)
+ *     → severity-gated GATE + variance (EV-048)
+ *     → route ALL fixable + any failing criteria to diagnostics (EV-051)
+ *
+ * Substrate (EV-050) is chosen by the CALLER and handed in as the `judge` seam
+ * (in-house = createInHouseJudge; the gate passes a stub). FIXABLE criteria are
+ * NEVER judged — they route straight to diagnostics (a judge is only a judge).
+ *
+ * PURE composition: no clock/random/network here; determinism comes from the
+ * cores + the injected judge + injected provenance.
+ */
+import type { JudgeInvoke } from "./determine-outcome.ts";
+import { determineOutcome, runJudge } from "./judge-prompt-template.ts";
+import { profileSubject, type SubjectProfile } from "./profile-subject.ts";
+import { splitTrainEval, buildJudgeSpec, type JudgePin } from "./build-evals.ts";
+import { rollupScorecard, type GradedCriterion, type Scorecard } from "./evaluate.ts";
+import {
+  routeFailures,
+  type ArtifactRef,
+  type FailureRef,
+  type HandoverBundle,
+  type SubjectKindValue,
+} from "./route-failures.ts";
+import type { LabeledTrace } from "./sample-traces.ts";
+import {
+  CriterionFlag,
+  OutcomeVerdict,
+  type CriterionVerdict,
+  type DiscoveredCriterion,
+  type EvalTrace,
+  type JudgeExemplar,
+  type OutcomeResult,
+  type SubjectVocab,
+} from "./contracts/eval-types.ts";
+import { Severity } from "./contracts/types.ts";
+
+export interface PipelineOptions {
+  criteria: DiscoveredCriterion[];
+  pin: JudgePin;
+  subject: { kind: SubjectKindValue; name: string; path: string };
+  producedBy: string;
+  /** INJECTED provenance stamp (no self-read clock). */
+  producedAt: string;
+  /**
+   * AUTHORED subject vocab (EV-002 / EV-049). When supplied it is stamped onto
+   * the profile and injected into the determiner; when omitted the profiler
+   * auto-infers a best-effort vocab from the traces. Either way the engine reads
+   * the vocab off the profile, never a module constant.
+   */
+  vocab?: SubjectVocab;
+  /** sample budget (default = all traces). */
+  sampleSize?: number;
+  /** per-criterion gating severity (default HIGH). */
+  severityById?: Record<string, string>;
+  /** artifact refs to enumerate on the handoff (caller-injected locators). */
+  artifacts?: ArtifactRef[];
+}
+
+export interface PipelineResult {
+  profile: SubjectProfile;
+  outcomes: OutcomeResult[];
+  sample: LabeledTrace[];
+  verdicts: CriterionVerdict[];
+  scorecard: Scorecard;
+  handoff: HandoverBundle;
+}
+
+/** Map a determiner outcome → a sampling LabeledTrace. */
+function toLabeled(trace: EvalTrace, outcome: OutcomeResult): LabeledTrace {
+  return { trace, label: outcome.reached, confidence: outcome.confidence };
+}
+
+/** Build few-shot exemplars from the TRAIN split (label from the determiner). */
+function exemplarsFromTrain(train: LabeledTrace[], limit = 2): JudgeExemplar[] {
+  return train.slice(0, limit).map((l) => ({
+    traceId: l.trace.id,
+    label: l.label,
+    why: `determiner labeled this ${l.label} (confidence ${l.confidence})`,
+  }));
+}
+
+/**
+ * Run the full W1 engine over a trace set. The `judge` seam is the substrate's
+ * judge (live in CLI, stub in the gate). Deterministic given its inputs.
+ */
+export async function runEvalPipeline(
+  traces: EvalTrace[],
+  judge: JudgeInvoke,
+  opts: PipelineOptions,
+): Promise<PipelineResult> {
+  // EV-049 — subject profile (carries the subject vocab the engine reads).
+  const profile = profileSubject(traces, opts.vocab);
+
+  // EV-042 — determine an outcome per trace (sequential → deterministic order).
+  // The determiner reads the subject vocab off the profile (EV-002).
+  const outcomes: OutcomeResult[] = [];
+  for (const t of traces) {
+    outcomes.push(await determineOutcome(t, judge, profile.vocab));
+  }
+  const labeled = traces.map((t, i) => toLabeled(t, outcomes[i]));
+
+  // EV-052 — sample + held-out TRAIN/EVAL split.
+  const size = opts.sampleSize ?? labeled.length;
+  const sample = labeled.slice(0, size);
+  const { train, evalSet } = splitTrainEval(sample, 0.34);
+  const trainIds = new Set(train.map((l) => l.trace.id));
+  // judge on the held-out split (fall back to the whole sample if eval is empty).
+  const judgeTargets = evalSet.length > 0 ? evalSet : sample;
+
+  // EV-043 — one judge per EVAL-WORTHY criterion (fixables are NEVER judged).
+  const evalWorthy = opts.criteria.filter((c) => c.flag === CriterionFlag.EvalWorthy);
+  const verdicts: CriterionVerdict[] = [];
+  const graded: GradedCriterion[] = [];
+  for (const criterion of evalWorthy) {
+    const spec = buildJudgeSpec(criterion, exemplarsFromTrain(train), trainIds);
+    // judge a representative held-out subject trace (first eval target).
+    const subjectTrace = judgeTargets[0]?.trace ?? sample[0]?.trace ?? traces[0];
+    const verdict = await runJudge(spec, subjectTrace, judge, opts.pin);
+    verdicts.push(verdict);
+    graded.push({
+      criterionId: criterion.id,
+      severity: opts.severityById?.[criterion.id] ?? Severity.High,
+      verdict,
+    });
+  }
+
+  // EV-048 — severity-gated GATE + variance.
+  const scorecard = rollupScorecard({ criteria: graded });
+
+  // EV-051 — route ALL fixable criteria + any failing eval-worthy verdicts.
+  const failures: FailureRef[] = [];
+  for (const criterion of opts.criteria) {
+    if (criterion.flag === CriterionFlag.Fixable) {
+      failures.push({
+        criterionId: criterion.id,
+        severity: opts.severityById?.[criterion.id] ?? Severity.High,
+        flag: CriterionFlag.Fixable,
+        traceId: "(suite)",
+        result: OutcomeVerdict.Fail,
+        critique: `infra/dependency criterion — routed to diagnostics: ${criterion.statement}`,
+      });
+    }
+  }
+  for (const v of verdicts) {
+    if (v.result !== OutcomeVerdict.Pass) {
+      failures.push({
+        criterionId: v.criterionId,
+        severity: opts.severityById?.[v.criterionId] ?? Severity.High,
+        flag: CriterionFlag.EvalWorthy,
+        traceId: v.traceId,
+        result: v.result,
+        critique: v.critique,
+      });
+    }
+  }
+  const handoff = routeFailures({
+    subject: opts.subject,
+    failures,
+    artifacts: opts.artifacts ?? [],
+    producedBy: opts.producedBy,
+    producedAt: opts.producedAt,
+  });
+
+  return { profile, outcomes, sample, verdicts, scorecard, handoff };
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sample-traces.ts

@@ -0,0 +1,210 @@
+/**
+ * scripts/sample-traces.ts — EV-052 trace filtering + sampling.
+ * ---------------------------------------------------------------------------
+ * The 5 sampling strategies from the error-analysis CORE source, as one
+ * deterministic mix:
+ *   - Random        — uniform spread (a stable seeded STRIDE, not Math.random)
+ *   - Outlier       — extremes by response length / latency / tool-call count
+ *   - Failure-driven— prioritize ✗-labeled traces
+ *   - Uncertainty   — lowest-confidence labels (judge disagreement proxy)
+ *   - Stratified    — even coverage across the pass/fail strata (validate-
+ *                     evaluator's ~50✓/50✗ balance)
+ *
+ * Sealed-sibling: this RE-IMPLEMENTS the diagnostics filtering *pattern* — it
+ * NEVER imports the diagnostics package. PURE + deterministic: no Math.random,
+ * no clock, no network. The "random" strategy is a fixed stride over the
+ * stable input order, so the same (traces, opts) always yields the identical
+ * sample (reproducible eval datasets — C-PIN-adjacent).
+ */
+import { OutcomeVerdict, type EvalTrace, type OutcomeVerdictValue } from "./contracts/eval-types.ts";
+
+/** A trace plus its determiner label (from EV-042) — the unit `*discover-evals` mines. */
+export interface LabeledTrace {
+  trace: EvalTrace;
+  label: OutcomeVerdictValue;
+  /** the determiner's confidence (drives uncertainty sampling). */
+  confidence: number;
+}
+
+export const SampleStrategy = {
+  Random: "random",
+  Outlier: "outlier",
+  FailureDriven: "failure-driven",
+  Uncertainty: "uncertainty",
+  Stratified: "stratified",
+} as const;
+export type SampleStrategyValue =
+  (typeof SampleStrategy)[keyof typeof SampleStrategy];
+
+/** All five, in a stable priority order (the default mix). */
+const ALL_STRATEGIES: readonly SampleStrategyValue[] = [
+  SampleStrategy.Stratified,
+  SampleStrategy.FailureDriven,
+  SampleStrategy.Outlier,
+  SampleStrategy.Uncertainty,
+  SampleStrategy.Random,
+];
+
+// ── filterTraces (the re-implemented filtering pattern) ─────────────────────
+
+export interface TraceFilter {
+  /** exact trace.name match. */
+  name?: string;
+  /** minimum tool-call count. */
+  minTools?: number;
+  /** maximum tool-call count. */
+  maxTools?: number;
+}
+
+function toolCount(t: EvalTrace): number {
+  return t.observations.filter((o) => o.type === "TOOL").length;
+}
+
+/** Apply a filter (re-implemented diagnostics pattern; no cross-package import). */
+export function filterTraces(traces: EvalTrace[], filter: TraceFilter): EvalTrace[] {
+  return traces.filter((t) => {
+    if (filter.name !== undefined && t.name !== filter.name) return false;
+    const n = toolCount(t);
+    if (filter.minTools !== undefined && n < filter.minTools) return false;
+    if (filter.maxTools !== undefined && n > filter.maxTools) return false;
+    return true;
+  });
+}
+
+// ── outlierScore ─────────────────────────────────────────────────────────────
+
+/** A monotone "how unusual" score: tools + latency + response length, scaled. */
+export function outlierScore(t: EvalTrace): number {
+  const tools = toolCount(t);
+  const latency = typeof t.latencyMs === "number" ? t.latencyMs : 0;
+  const respLen =
+    typeof t.output?.response === "string" ? t.output.response.length : 0;
+  // weights chosen so each axis can dominate at its extreme; pure arithmetic.
+  return tools * 100 + latency + respLen;
+}
+
+// ── balancedSample — ~50/50 ✓/✗, never fabricated ───────────────────────────
+
+/**
+ * Pick up to `size` labeled traces with an even pass/fail balance. NEVER
+ * invents items: if one class is scarcer, all of it is included and the other
+ * class is capped to match (so the result is <= size and as balanced as the
+ * data allows). Deterministic — preserves input order within each class.
+ */
+export function balancedSample(labeled: LabeledTrace[], size: number): LabeledTrace[] {
+  const passes = labeled.filter((l) => l.label === OutcomeVerdict.Pass);
+  const fails = labeled.filter((l) => l.label === OutcomeVerdict.Fail);
+  const perSide = Math.floor(size / 2);
+  // balance to the scarcer side so neither class is over-represented vs the other.
+  const take = Math.min(perSide, passes.length, fails.length);
+  const out: LabeledTrace[] = [...passes.slice(0, take), ...fails.slice(0, take)];
+
+  // if balanced take didn't fill the budget AND one side has leftovers, top up
+  // from the larger side up to `size` (still never fabricating).
+  if (out.length < size) {
+    const remaining = size - out.length;
+    const leftover = [...passes.slice(take), ...fails.slice(take)];
+    out.push(...leftover.slice(0, remaining));
+  }
+  return out;
+}
+
+// ── sampleTraces — the mixed strategy sample ────────────────────────────────
+
+export interface SampleOptions {
+  size: number;
+  /** which strategies to mix (default: all five). */
+  strategies?: SampleStrategyValue[];
+}
+
+/** Stable stride indices over [0,n) yielding ~count evenly-spread positions. */
+function strideIndices(n: number, count: number): number[] {
+  if (n <= 0 || count <= 0) return [];
+  if (count >= n) return Array.from({ length: n }, (_v, i) => i);
+  const step = n / count;
+  const out: number[] = [];
+  for (let k = 0; k < count; k++) out.push(Math.floor(k * step));
+  return out;
+}
+
+/** A selected trace plus the DETERMINISTIC strategy that nominated it ("data link"). */
+export interface AttributedTrace {
+  trace: LabeledTrace;
+  /** which strategy's queue this trace was first picked from. */
+  selectedBy: SampleStrategyValue;
+}
+
+/**
+ * Build a deduped sample (<= size) mixing the requested strategies. Each
+ * strategy contributes candidates in a fixed order; we round-robin across the
+ * strategies and add the next not-yet-picked candidate from each until the
+ * budget is met. Deterministic for a given (labeled, opts).
+ */
+export function sampleTraces(labeled: LabeledTrace[], opts: SampleOptions): LabeledTrace[] {
+  return sampleTracesAttributed(labeled, opts).map((a) => a.trace);
+}
+
+/**
+ * Like {@link sampleTraces}, but each result carries `selectedBy` — the strategy
+ * whose queue first nominated it. This is the deterministic provenance ("data
+ * link") surfaced on derived dataset cases; the human "why high-value" rationale
+ * is authored separately by the selecting agent. Deterministic for (labeled, opts).
+ */
+export function sampleTracesAttributed(labeled: LabeledTrace[], opts: SampleOptions): AttributedTrace[] {
+  const strategies = opts.strategies ?? ALL_STRATEGIES;
+  const byId = new Map(labeled.map((l) => [l.trace.id, l]));
+
+  // Per-strategy ordered candidate id lists (pure, stable).
+  const queues: string[][] = strategies.map((s) => {
+    switch (s) {
+      case SampleStrategy.FailureDriven:
+        return labeled.filter((l) => l.label === OutcomeVerdict.Fail).map((l) => l.trace.id);
+      case SampleStrategy.Outlier:
+        return [...labeled]
+          .sort((a, b) => outlierScore(b.trace) - outlierScore(a.trace))
+          .map((l) => l.trace.id);
+      case SampleStrategy.Uncertainty:
+        return [...labeled]
+          .sort((a, b) => a.confidence - b.confidence)
+          .map((l) => l.trace.id);
+      case SampleStrategy.Stratified: {
+        // interleave pass/fail so coverage spans both strata.
+        const p = labeled.filter((l) => l.label === OutcomeVerdict.Pass).map((l) => l.trace.id);
+        const f = labeled.filter((l) => l.label === OutcomeVerdict.Fail).map((l) => l.trace.id);
+        const out: string[] = [];
+        for (let i = 0; i < Math.max(p.length, f.length); i++) {
+          if (i < f.length) out.push(f[i]);
+          if (i < p.length) out.push(p[i]);
+        }
+        return out;
+      }
+      case SampleStrategy.Random:
+      default:
+        return strideIndices(labeled.length, labeled.length).map((i) => labeled[i].trace.id);
+    }
+  });
+
+  const picked = new Set<string>();
+  const result: AttributedTrace[] = [];
+  const cursors = new Array<number>(queues.length).fill(0);
+
+  // round-robin across strategies until budget met or all queues exhausted.
+  let progressed = true;
+  while (result.length < opts.size && progressed) {
+    progressed = false;
+    for (let qi = 0; qi < queues.length && result.length < opts.size; qi++) {
+      const q = queues[qi];
+      while (cursors[qi] < q.length) {
+        const id = q[cursors[qi]++];
+        if (!picked.has(id)) {
+          picked.add(id);
+          const lt = byId.get(id);
+          if (lt !== undefined) result.push({ trace: lt, selectedBy: strategies[qi] });
+          progressed = true;
+          break;
+        }
+      }
+    }
+  }
+  return result;
+}