@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/derive-dataset.ts

@@ -0,0 +1,152 @@
+/**
+ * scripts/derive-dataset.ts — EV-047 `*discover-dataset` (Type A — Code-only).
+ * ---------------------------------------------------------------------------
+ * Distill a LIVING REGRESSION SET from past ✓/✗ traces: pick the cases worth
+ * keeping forever (catastrophes, edges, outliers, low-confidence calls), turn
+ * each into a `DatasetCase` whose query IS the real input that produced the
+ * outcome, and append them MONOTONICALLY into the growing dataset.
+ *
+ * Reuses (does NOT re-implement) the EV-052 selectors in `sample-traces.ts`
+ * (failure-driven = catastrophe · outlier = edge · uncertainty = low-confidence)
+ * and the EV-046 monotonic merge in `build-dataset.ts`. The NEW work here is the
+ * distillation: trace → regression case + the append-only living set. Holds NO
+ * LLM reasoning and makes NO pass/fail decision (the labels are already on the
+ * traces, from `*discover-evals`). PURE — no clock / random / network; same input →
+ * same regression set (reproducible).
+ */
+import { OutcomeVerdict, type OutcomeVerdictValue } from "./contracts/eval-types.ts";
+import {
+  sampleTracesAttributed,
+  SampleStrategy,
+  type LabeledTrace,
+  type AttributedTrace,
+  type SampleStrategyValue,
+} from "./sample-traces.ts";
+import { buildCase, mergeCases } from "./build-dataset.ts";
+import { CaseSource, type DatasetCase } from "./contracts/dataset.ts";
+
+/**
+ * The regression-worthy selection strategies, in priority order:
+ *   - failure-driven (catastrophes — known ✗ we must never regress on)
+ *   - outlier        (edges — unusual length / latency / tool-call count)
+ *   - uncertainty    (low-confidence determinations — the brittle boundary)
+ * Stratified is appended so a balanced ✓/✗ spread is preserved (a regression set
+ * needs known-GOOD anchors too, not only failures).
+ */
+const REGRESSION_STRATEGIES = [
+  SampleStrategy.FailureDriven,
+  SampleStrategy.Outlier,
+  SampleStrategy.Uncertainty,
+  SampleStrategy.Stratified,
+] as const;
+
+/** Map a determiner verdict onto the outcome tag carried in a derived tuple. */
+function outcomeTag(label: OutcomeVerdictValue): string {
+  return label === OutcomeVerdict.Pass ? "pass" : label === OutcomeVerdict.Fail ? "fail" : "uncertain";
+}
+
+/**
+ * Human-readable DETERMINISTIC provenance label for the selecting strategy — the
+ * "data link" half of the why (what objective signal nominated this trace). The
+ * JUDGMENT half (why it's high-value as a test) is authored separately. PURE.
+ */
+export function selectedByLabel(s: SampleStrategyValue): string {
+  switch (s) {
+    case SampleStrategy.FailureDriven:
+      return "failure-driven — a confirmed ✗ catastrophe the suite must never regress on";
+    case SampleStrategy.Outlier:
+      return "outlier — an edge trajectory (unusual length / latency / tool-call count)";
+    case SampleStrategy.Uncertainty:
+      return "uncertainty — a low-confidence boundary call (the brittle decision edge)";
+    case SampleStrategy.Stratified:
+      return "stratified — a known-✓ anchor (keeps the regression set ✓/✗ balanced)";
+    default:
+      return String(s);
+  }
+}
+
+/**
+ * Authors the human "why high-value as a test" rationale for a selected trace.
+ * This is the JUDGMENT — supplied by whoever SELECTS the case (the discover/judge
+ * agent reasoning over the trace). Receives the attributed trace; returns the
+ * rationale string, or undefined to leave it unauthored (renderer shows the
+ * deterministic provenance + a "rationale pending" note).
+ */
+export type RationaleAuthor = (a: AttributedTrace) => string | undefined;
+
+/**
+ * Convert one labeled trace into a regression `DatasetCase`. The query IS the
+ * trace's real input prompt (the input that produced the ✓/✗); the tuple records
+ * the salient axis (the known outcome) so the case is self-describing; the source
+ * is `derived` with `originTraceId` for provenance. `selectedBy` carries the
+ * DETERMINISTIC strategy label; `rationale` (when `author` supplies one) carries
+ * the LLM JUDGMENT of why this is high-value. Returns null when the trace has no
+ * input prompt (a regression case needs a replayable input).
+ */
+export function traceToRegressionCase(a: AttributedTrace, author?: RationaleAuthor): DatasetCase | null {
+  const lt = a.trace;
+  const prompt = lt.trace.input?.prompt;
+  if (typeof prompt !== "string" || prompt.length === 0) return null;
+  return buildCase({ outcome: outcomeTag(lt.label) }, prompt, CaseSource.Derived, lt.trace.id, {
+    selectedBy: selectedByLabel(a.selectedBy),
+    rationale: author?.(a),
+  });
+}
+
+export interface DeriveOptions {
+  /**
+   * GA — exclude INDETERMINATE (`uncertain`) traces from the held-out set
+   * (DEFAULT true). A held-out regression case needs a CONFIRMED ✓/✗ anchor; an
+   * indeterminate outcome is unresolved (it belongs in the calibration loop, not
+   * the gate-eligible regression set). Set false to keep them tagged
+   * `needs-resolution` instead of dropping.
+   */
+  excludeIndeterminate?: boolean;
+  /**
+   * Authors the per-case "why high-value as a test" rationale (the JUDGMENT). The
+   * selecting agent supplies it; when omitted the cases carry only the
+   * deterministic `selectedBy` provenance.
+   */
+  authorRationale?: RationaleAuthor;
+}
+
+/**
+ * Select the regression-worthy traces (reusing the EV-052 selectors) and distill
+ * each into a `DatasetCase`. Traces without a replayable input are dropped, and
+ * (GA, default) INDETERMINATE traces are excluded from the held-out set. The
+ * result is deduped (content-id + near-duplicate query) so the same brittle case
+ * isn't kept twice. DETERMINISTIC for a given (labeled, size).
+ */
+export function deriveRegressionCases(
+  labeled: LabeledTrace[],
+  size: number,
+  opts: DeriveOptions = {},
+): DatasetCase[] {
+  const excludeIndeterminate = opts.excludeIndeterminate ?? true;
+  const pool = excludeIndeterminate
+    ? labeled.filter((lt) => lt.label !== OutcomeVerdict.Uncertain)
+    : labeled;
+  const selected = sampleTracesAttributed(pool, { size, strategies: [...REGRESSION_STRATEGIES] });
+  const cases: DatasetCase[] = [];
+  for (const a of selected) {
+    const c = traceToRegressionCase(a, opts.authorRationale);
+    if (c !== null) cases.push(c);
+  }
+  // dedup within the freshly-derived batch (id + near-dup) via the shared merge.
+  return mergeCases([], cases);
+}
+
+/**
+ * Grow an existing regression set with newly-derived cases — APPEND-ONLY: every
+ * existing case is retained, only genuinely-novel (non-near-duplicate) derived
+ * cases are added. The living regression set never shrinks (EV-053). DETERMINISTIC.
+ */
+export function growRegressionSet(
+  existing: DatasetCase[],
+  labeled: LabeledTrace[],
+  size: number,
+  opts: DeriveOptions = {},
+): DatasetCase[] {
+  const derived = deriveRegressionCases(labeled, size, opts);
+  return mergeCases(existing, derived);
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts

@@ -0,0 +1,219 @@
+/**
+ * scripts/determine-outcome.ts — EV-042 success/failure determiner (Type A — DATA only).
+ * ---------------------------------------------------------------------------
+ * The deterministic FOUNDATION of the W1 eval engine. sample traces are UNLABELED
+ * (0 scores / 0 tags), so `*discover-evals` has no ✓/✗ to mine until a trace is
+ * deep-read and labeled. The DEEP-READ + the success/failure DECISION is an
+ * LLM-only operation → it lives in the `error-analyst` subagent def (the host
+ * runtime reasons; "inaction can be success" is the rubric encoded THERE).
+ *
+ * Script austerity (operator directive): this script holds NO judge prompt and
+ * NO LLM-reasoning logic. It keeps only the Type-A pieces:
+ *   - deterministic SIGNAL extraction (event kind · tool count · guard count · …)
+ *     fed to the judge — NEVER a verdict (toolCount is a signal, never a proxy);
+ *   - the verdict PARSER (critique-before-verdict schema enforcement);
+ *   - the deterministic OUTCOME ASSEMBLER (parsed verdict + signals → result).
+ * The judge-prompt rendering + the in-house/export run-wrapper (`determineOutcome`)
+ * live in `judge-prompt-template.ts` (the EV-050 in-house/export exception). The
+ * DEFAULT agent-dispatch path reads a verdict file the subagent wrote — no
+ * prompt is rendered here, and no provider is ever constructed.
+ */
+import {
+  OutcomeVerdict,
+  UNCLASSIFIED_EVENT,
+  type CritiqueVerdict,
+  type EvalTrace,
+  type OutcomeResult,
+  type OutcomeSignals,
+  type SubjectVocab,
+  type TraceObservation,
+} from "./contracts/eval-types.ts";
+
+/**
+ * The judge DI seam — `(systemPrompt, userPrompt) => raw text`. Under the DEFAULT
+ * agent-dispatch substrate the verdict comes from a dispatched subagent (a
+ * verdict file); under the OPTIONAL in-house substrate it is a provider call.
+ * This module never builds a provider and never renders a prompt — it only
+ * declares the seam type the in-house/export wrapper consumes.
+ */
+export type JudgeInvoke = (
+  systemPrompt: string,
+  userPrompt: string,
+) => Promise<string>;
+
+// ── Event classification (the intended-goal half of the read) ───────────────
+//
+// SUBJECT-AGNOSTIC: the tag→kind rules come from the injected `vocab` (EV-002 /
+// EV-049). This module holds NO subject tag names — only the generic matching
+// machinery + the `UNCLASSIFIED_EVENT` sentinel.
+
+/** Escape a vocab token so it is matched literally inside a RegExp. */
+function escapeRegExp(token: string): string {
+  return token.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Classify the event block → the intended goal, using the subject's tag rules.
+ * Rules are tried in order; the first whose stem appears as an opening tag
+ * (`<{tag}`, case-insensitive, prefix match) wins. No match → UNCLASSIFIED_EVENT.
+ */
+export function classifyEvent(prompt: string, vocab: SubjectVocab): string {
+  for (const rule of vocab.eventTags) {
+    if (new RegExp(`<${escapeRegExp(rule.tag)}`, "i").test(prompt)) return rule.kind;
+  }
+  return UNCLASSIFIED_EVENT;
+}
+
+/**
+ * Parse a guard's consecutive-action count (`{attr}="N"`) from the prompt, using
+ * the subject's `guardCounterAttr`. Returns null when the subject declares no
+ * guard counter (`attr === null`) or the attribute is absent/non-numeric.
+ */
+export function extractGuardConsecutive(prompt: string, attr: string | null): number | null {
+  if (attr === null) return null;
+  const m = new RegExp(`${escapeRegExp(attr)}\\s*=\\s*"(\\d+)"`, "i").exec(prompt);
+  if (m === null) return null;
+  const n = Number.parseInt(m[1], 10);
+  return Number.isNaN(n) ? null : n;
+}
+
+// ── Tool-trajectory inspection ──────────────────────────────────────────────
+
+function isTool(obs: TraceObservation): boolean {
+  return obs.type === "TOOL";
+}
+
+/** Read `output.success` (boolean) off a tool observation, else null. */
+function toolSuccess(obs: TraceObservation): boolean | null {
+  const out = obs.output;
+  if (out !== null && typeof out === "object" && "success" in out) {
+    const s = (out as { success: unknown }).success;
+    if (typeof s === "boolean") return s;
+  }
+  return null;
+}
+
+function promptOf(trace: EvalTrace): string {
+  return typeof trace.input?.prompt === "string" ? trace.input.prompt : "";
+}
+
+/**
+ * Extract the deterministic signals. PURE — reads only what's in the trace, and
+ * the subject NAMES (send tool · recovery tools · event tags · guard attr) from
+ * the injected `vocab` (EV-002 / EV-049 — no module-level subject constants).
+ * Decides NOTHING about goal-attainment (that is the judge's job over these
+ * signals). In particular it does not map toolCount → pass/fail.
+ */
+export function extractOutcomeSignals(trace: EvalTrace, vocab: SubjectVocab): OutcomeSignals {
+  const prompt = promptOf(trace);
+  const tools = trace.observations.filter(isTool);
+
+  // EV-049 HONEST-NULL: when the subject's send tool is unset/uninferred, we
+  // CANNOT identify which observations are sends — so `sentMessage`/`sendSucceeded`
+  // are UNKNOWN (null), NOT a false `false`. A bare `false` here lies to the
+  // determiner ("Sent a message: false") even on traces that DID send. Only when
+  // the send tool IS named do we count and report a real true/false.
+  const sendToolKnown = vocab.sendTool.length > 0;
+  const sends = sendToolKnown
+    ? tools.filter((t) => t.name === vocab.sendTool)
+    : [];
+
+  let sentMessage: boolean | null = null;
+  let sendSucceeded: boolean | null = null;
+  if (sendToolKnown) {
+    sentMessage = sends.length > 0;
+    if (sends.length > 0) {
+      // true if ANY send succeeded; false if every send failed/unknown-false.
+      sendSucceeded = sends.some((t) => toolSuccess(t) === true);
+    }
+  }
+
+  const recoverySet = new Set(vocab.recoveryTools);
+  const recoveryPresent = tools.some(
+    (t) => t.name !== undefined && recoverySet.has(t.name),
+  );
+
+  return {
+    eventKind: classifyEvent(prompt, vocab),
+    toolCount: tools.length,
+    sentMessage,
+    sendSucceeded,
+    guardConsecutive: extractGuardConsecutive(prompt, vocab.guardCounterAttr),
+    recoveryPresent,
+  };
+}
+
+// ── Verdict parsing (critique-before-verdict enforced) ──────────────────────
+
+/** Strip a ```json … ``` (or bare ```) fence if present. */
+function stripFence(raw: string): string {
+  const fenced = /```(?:json)?\s*([\s\S]*?)\s*```/i.exec(raw);
+  return fenced !== null ? fenced[1] : raw;
+}
+
+const VALID_RESULTS: ReadonlySet<string> = new Set([
+  OutcomeVerdict.Pass,
+  OutcomeVerdict.Fail,
+  OutcomeVerdict.Uncertain,
+]);
+
+/**
+ * Parse a judge's structured answer, ENFORCING critique-before-verdict: a
+ * `critique` string MUST be present and non-empty (a bare verdict with no
+ * reasoning is rejected), and `result` must be in the closed set. This is the
+ * schema gate applied to a verdict file the subagent wrote (default path) OR the
+ * in-house provider's raw text — same enforcement either way.
+ */
+export function parseCritiqueVerdict(raw: string): CritiqueVerdict {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stripFence(raw).trim());
+  } catch {
+    throw new Error(`parseCritiqueVerdict: not valid JSON: ${raw.slice(0, 120)}`);
+  }
+  if (parsed === null || typeof parsed !== "object") {
+    throw new Error("parseCritiqueVerdict: expected a JSON object");
+  }
+  const o = parsed as Record<string, unknown>;
+  if (typeof o.critique !== "string" || o.critique.trim().length === 0) {
+    throw new Error(
+      "parseCritiqueVerdict: missing/empty `critique` — critique-before-verdict " +
+        "is mandatory (a bare verdict with no reasoning is rejected)",
+    );
+  }
+  if (typeof o.result !== "string" || !VALID_RESULTS.has(o.result)) {
+    throw new Error(
+      `parseCritiqueVerdict: result '${String(o.result)}' not in ` +
+        `{pass,fail,uncertain}`,
+    );
+  }
+  const confidence = typeof o.confidence === "number" ? o.confidence : 0;
+  return {
+    critique: o.critique,
+    result: o.result as CritiqueVerdict["result"],
+    confidence: Math.max(0, Math.min(1, confidence)),
+  };
+}
+
+// ── Deterministic outcome assembler (no judge call) ─────────────────────────
+
+/**
+ * Assemble the determiner's `OutcomeResult` from a trace's signals + a PARSED
+ * verdict (read from a dispatched error-analyst's verdict file, default path).
+ * PURE — no judge call, no clock/random. The verdict is the subagent's (never
+ * derived from tool-count). This is the Type-A AGGREGATE half of EV-042; the
+ * LLM-only DECISION lives in `error-analyst.md`.
+ */
+export function assembleOutcome(
+  trace: EvalTrace,
+  verdict: CritiqueVerdict,
+  vocab: SubjectVocab,
+): OutcomeResult {
+  return {
+    traceId: trace.id,
+    reached: verdict.result as OutcomeResult["reached"],
+    confidence: verdict.confidence,
+    rationale: verdict.critique,
+    signals: extractOutcomeSignals(trace, vocab),
+  };
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/diff-discriminate.ts

@@ -0,0 +1,160 @@
+/**
+ * scripts/diff-discriminate.ts — GA-11 observed-eligibility via the broken∧¬healthy diff.
+ * ---------------------------------------------------------------------------
+ * A criterion earns the `observed` grounding tier ONLY if it DISCRIMINATES: it
+ * must fire on a BROKEN trace AND NOT fire on a HEALTHY companion trace. A check
+ * that fires on both is not detecting THIS failure — it is noise (or garbage-in)
+ * and must NOT be presented as an observed defect.
+ *
+ *   fires-on-broken ∧ ¬fires-on-healthy   → OBSERVED-eligible (discriminates)
+ *   fires-on-broken ∧  fires-on-healthy   → DEMOTE → inferred (does not discriminate)
+ *                                            · if FIXABLE infra → route→diagnostics
+ *   ¬fires-on-broken                       → DEMOTE → inferred (no observed failure)
+ *
+ * GRANDFATHER (regression-safety rule #2): when there is NO healthy companion
+ * trace, this NEVER hard-fails — it gracefully falls back to single-trace +
+ * honest prevalence, keeps the `observed` tier, and TAGS the decision
+ * `single-trace-fallback`. The diff is skipped, not enforced.
+ *
+ * PURE — a decision function over boolean fire-signals; no clock/random/network.
+ * The actual "does it fire?" judgement is produced upstream (the judge/determiner
+ * leaf, LLM); this module only DECIDES eligibility from those signals (code).
+ */
+import {
+  CriterionFlag,
+  Grounding,
+  type GroundingValue,
+  type MinedCriterion,
+} from "./contracts/eval-types.ts";
+
+/** The upstream fire-signals for one criterion (produced by the judge leaf). */
+export interface CriterionFireSignals {
+  criterionId: string;
+  /** did the criterion fire (detect its failure) on the BROKEN trace? */
+  firesOnBroken: boolean;
+  /** did it fire on the HEALTHY companion? (ignored when `hasHealthy` is false) */
+  firesOnHealthy: boolean;
+  /** is a healthy companion trace available at all? false ⇒ graceful fallback. */
+  hasHealthy: boolean;
+}
+
+export const DiffDecision = {
+  /** discriminates → keep `observed`. */
+  Observed: "observed-eligible",
+  /** no healthy companion → keep `observed`, tagged single-trace. */
+  SingleTraceFallback: "single-trace-fallback",
+  /** fires on both / not on broken → demote to `inferred`. */
+  DemoteInferred: "demote-inferred",
+  /** fires on healthy too AND fixable infra → route to diagnostics (garbage-in). */
+  RouteDiagnostics: "route-diagnostics",
+} as const;
+export type DiffDecisionValue = (typeof DiffDecision)[keyof typeof DiffDecision];
+
+export interface DiffResult {
+  criterionId: string;
+  decision: DiffDecisionValue;
+  /** the grounding the criterion should carry after the diff. */
+  grounding: GroundingValue;
+  /** true when the diff was skipped for lack of a healthy companion (tagged). */
+  tagged: boolean;
+  reason: string;
+}
+
+/**
+ * Decide observed-eligibility for ONE criterion from its fire-signals. PURE.
+ * `isFixable` lets a non-discriminating INFRA criterion route to diagnostics
+ * (garbage-in) rather than merely demote.
+ */
+export function discriminate(
+  signals: CriterionFireSignals,
+  isFixable = false,
+): DiffResult {
+  const { criterionId, firesOnBroken, firesOnHealthy, hasHealthy } = signals;
+
+  if (!hasHealthy) {
+    return {
+      criterionId,
+      decision: DiffDecision.SingleTraceFallback,
+      grounding: Grounding.Observed,
+      tagged: true,
+      reason:
+        "no healthy companion trace — graceful single-trace fallback (diff skipped, " +
+        "honest prevalence retained); observed tier kept, tagged single-trace.",
+    };
+  }
+  if (!firesOnBroken) {
+    return {
+      criterionId,
+      decision: DiffDecision.DemoteInferred,
+      grounding: Grounding.Inferred,
+      tagged: false,
+      reason: "did not fire on the broken trace — no observed failure; demoted to inferred.",
+    };
+  }
+  if (firesOnHealthy) {
+    return isFixable
+      ? {
+          criterionId,
+          decision: DiffDecision.RouteDiagnostics,
+          grounding: Grounding.Inferred,
+          tagged: false,
+          reason:
+            "fires on BOTH broken and healthy (does not discriminate) AND is fixable infra — " +
+            "garbage-in; route to diagnostics rather than judge.",
+        }
+      : {
+          criterionId,
+          decision: DiffDecision.DemoteInferred,
+          grounding: Grounding.Inferred,
+          tagged: false,
+          reason:
+            "fires on BOTH broken and healthy — does not discriminate this failure; demoted to inferred.",
+        };
+  }
+  return {
+    criterionId,
+    decision: DiffDecision.Observed,
+    grounding: Grounding.Observed,
+    tagged: false,
+    reason: "fires on broken ∧ NOT on healthy — discriminates; observed-eligible.",
+  };
+}
+
+/**
+ * Apply diff-discrimination to a batch of mined criteria, demoting grounding +
+ * tagging single-trace fallbacks. A criterion with NO matching fire-signal is
+ * left untouched (the diff was not run for it). Returns a NEW array; inputs are
+ * never mutated. PURE — same (criteria, signals) ⇒ identical output.
+ *
+ * When a criterion is demoted from `observed` → `inferred`, its evidence is
+ * updated honestly (grounding + seen_in_traces) and the decision is recorded in
+ * `discovery.reasoning` so the demotion is never silent (NO data loss — refs +
+ * assumptions are preserved verbatim).
+ */
+export function applyDiffDiscrimination(
+  criteria: MinedCriterion[],
+  signals: CriterionFireSignals[],
+): MinedCriterion[] {
+  const byId = new Map(signals.map((s) => [s.criterionId, s]));
+  return criteria.map((c) => {
+    const sig = byId.get(c.id);
+    if (sig === undefined) return c; // diff not run for this criterion
+    if (c.discovery.evidence.grounding !== Grounding.Observed) return c; // only gate observed
+    const isFixable = c.flag === CriterionFlag.Fixable;
+    const res = discriminate(sig, isFixable);
+    if (res.grounding === Grounding.Observed && !res.tagged) return c; // unchanged
+    return {
+      ...c,
+      discovery: {
+        ...c.discovery,
+        evidence: {
+          ...c.discovery.evidence,
+          grounding: res.grounding,
+          seen_in_traces:
+            res.grounding === Grounding.Observed ? c.discovery.evidence.seen_in_traces : false,
+        },
+        reasoning: `${c.discovery.reasoning} | diff-discriminate: ${res.reason}`,
+      },
+    };
+  });
+}