@mutagent/helix 0.1.0-alpha.14

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/mask.ts

@@ -0,0 +1,138 @@
+/**
+ * scripts/mask.ts
+ * ---------------------------------------------------------------------------
+ * Byte-identity masking (decision #10 / R10). The audit is itself deterministic:
+ * runId / timestamps / absolute-paths are masked so two audits on one bundle
+ * (or two runs on byte-identical source) produce a BYTE-IDENTICAL scorecard.
+ *
+ * "byte-identical across runs" is only testable AFTER masking the declared
+ * injected fields (the byte-identity masking contract). This module IS that
+ * contract — a versioned, ordered set of deterministic substitutions.
+ *
+ * Pure, deterministic: no clock, no random, no network. Same input string ->
+ * byte-identical output string, always.
+ */
+
+/** Versioned masking set. Bump VERSION when adding/reordering patterns. */
+export const MASK_SET_VERSION = "v1";
+
+export interface MaskRule {
+  readonly id: string;
+  readonly pattern: RegExp;
+  readonly replacement: string;
+}
+
+/**
+ * The ordered masking rules. ORDER MATTERS — earlier rules run first. ISO
+ * timestamps are masked before bare dates so the longer pattern wins.
+ *
+ * NOTE: every pattern uses the global flag so replaceAll-style semantics apply.
+ */
+export const MASK_RULES: readonly MaskRule[] = [
+  // ISO-8601 timestamps (with optional ms + Z/offset) -> stable sentinel
+  {
+    id: "iso-timestamp",
+    pattern:
+      /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})/g,
+    replacement: "<TS>",
+  },
+  // epoch-millis (13-digit) and epoch-seconds (10-digit) -> sentinel
+  { id: "epoch-millis", pattern: /\b\d{13}\b/g, replacement: "<EPOCHMS>" },
+  { id: "epoch-seconds", pattern: /\b\d{10}\b/g, replacement: "<EPOCHS>" },
+  // runId of the canonical form name-YYYYMMDD-HHMMSS -> sentinel
+  {
+    id: "runid-dated",
+    pattern: /\b[a-z0-9][a-z0-9-]*-\d{8}-\d{6}\b/gi,
+    replacement: "<RUNID>",
+  },
+  // UUID v4-ish -> sentinel
+  {
+    id: "uuid",
+    pattern:
+      /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi,
+    replacement: "<UUID>",
+  },
+  // absolute POSIX home paths -> sentinel (strips machine-specific roots)
+  {
+    id: "abs-home-path",
+    pattern: /\/(?:Users|home)\/[^\s"'`]+/g,
+    replacement: "<ABSPATH>",
+  },
+  // bare YYYY-MM-DD dates -> sentinel (after iso-timestamp so it only hits
+  // standalone dates)
+  { id: "bare-date", pattern: /\b\d{4}-\d{2}-\d{2}\b/g, replacement: "<DATE>" },
+  // short git sha (7-12 hex) -> sentinel
+  { id: "git-sha", pattern: /\b[0-9a-f]{7,12}\b/g, replacement: "<SHA>" },
+];
+
+/**
+ * Mask a string deterministically. Applies every rule in order.
+ */
+export function maskString(input: string): string {
+  let out = input;
+  for (const rule of MASK_RULES) {
+    out = out.replace(rule.pattern, rule.replacement);
+  }
+  return out;
+}
+
+/**
+ * Deeply mask a JSON value: strings are masked, objects/arrays recursed. Object
+ * keys are sorted so key-ordering can never cause a spurious byte difference
+ * (the masked form is canonicalized).
+ */
+export function maskValue(value: unknown): unknown {
+  if (typeof value === "string") return maskString(value);
+  if (Array.isArray(value)) return value.map(maskValue);
+  if (value && typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      out[maskString(key)] = maskValue((value as Record<string, unknown>)[key]);
+    }
+    return out;
+  }
+  return value;
+}
+
+/**
+ * Canonical masked serialization of a JSON value. Two values that differ ONLY in
+ * masked fields (runId / timestamps / abs-paths) serialize byte-identically.
+ */
+export function maskedCanonicalJson(value: unknown): string {
+  return JSON.stringify(maskValue(value), null, 2);
+}
+
+// CLI entrypoint: `run.sh scripts/mask.ts <file.json|->` masks stdin or a file
+// and prints the canonical masked JSON. Documented integration point: this is
+// the masking spine the variance comparator calls.
+declare const Bun: { argv: string[] } | undefined;
+async function main(): Promise<void> {
+  const argv =
+    typeof Bun !== "undefined" ? Bun.argv.slice(2) : process.argv.slice(2);
+  const target = argv[0];
+  let raw: string;
+  if (!target || target === "-") {
+    const chunks: Buffer[] = [];
+    for await (const c of process.stdin) chunks.push(c as Buffer);
+    raw = Buffer.concat(chunks).toString("utf8");
+  } else {
+    const { readFileSync } = await import("node:fs");
+    raw = readFileSync(target, "utf8");
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+    process.stdout.write(maskedCanonicalJson(parsed) + "\n");
+  } catch {
+    // Not JSON — mask as a plain string.
+    process.stdout.write(maskString(raw));
+  }
+}
+
+// import.meta.main is true only when executed directly (not when imported).
+const isMain =
+  typeof import.meta !== "undefined" &&
+  (import.meta as unknown as { main?: boolean }).main === true;
+if (isMain) {
+  void main();
+}

package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts

@@ -0,0 +1,113 @@
+/**
+ * scripts/materialize-dataset.ts — F8: MATERIALIZE real dataset items from the
+ * agentspec (Type A — PURE). Seed → ACTUAL items, not just definitions.
+ * ---------------------------------------------------------------------------
+ * The ADL `*build` stage emits an agentspec whose `definition.evals` carries the
+ * dataset DEFINITION (dataset_categories[] × edge_cases[]) + scenarios[]. The old
+ * `*build-dataset` only had the dimensions/definitions; F8 requires it to
+ * MATERIALIZE real, runnable DatasetCases — ≥1 per category, plus one per declared
+ * edge_case — BEFORE the dataset-builder agent expands them synthetically.
+ *
+ * The materialized cases are the SEED layer (source: "seed"):
+ *   - one BASE item per category — from a category-tagged scenario when present,
+ *     else synthesized from the category description ("seed, don't duplicate").
+ *   - one EDGE item per declared edge_case — query = the edge_case phrase in the
+ *     category's context; flagged `edge_case: "true"` in its tuple.
+ *
+ * These are REAL queries (non-empty NL), not category definitions. The
+ * dataset-builder agent then expands them; build-dataset.ts's deterministic
+ * id/dedup/merge dedups any overlap. Subject-agnostic. DETERMINISTIC — stable
+ * content-derived ids, scenarios/categories/edge_cases consumed in given order;
+ * no clock / random / network → re-materializing is byte-identical (C-PIN).
+ */
+import {
+  type Dataset,
+  type DatasetCase,
+  type DatasetTuple,
+  type Dimension,
+} from "./contracts/dataset.ts";
+import type { AgentspecEvals, AgentspecScenario } from "./contracts/agentspec-evals.ts";
+import { buildCase, appendToDataset } from "./build-dataset.ts";
+import { CaseSource } from "./contracts/dataset.ts";
+
+/** The literal tuple values for the boolean edge_case dimension (DatasetTuple is string-valued). */
+const EDGE_TRUE = "true";
+const EDGE_FALSE = "false";
+
+/**
+ * The dimensions the materialized dataset varies over: `category` (values = the
+ * category ids) + `edge_case` (boolean). Subject-agnostic — derived from the
+ * agentspec, never hard-coded. PURE.
+ */
+export function dimensionsFromAgentspec(evals: AgentspecEvals): Dimension[] {
+  const categories = evals.dataset_categories.map((c) => c.id);
+  const dims: Dimension[] = [];
+  if (categories.length > 0) {
+    dims.push({ name: "category", description: "the dataset-category slice", values: categories });
+  }
+  dims.push({ name: "edge_case", description: "is this an edge/adversarial case", values: [EDGE_FALSE, EDGE_TRUE] });
+  return dims;
+}
+
+/** The first scenario tagged into `categoryId` (given order), if any. PURE. */
+function baseScenarioFor(evals: AgentspecEvals, categoryId: string): AgentspecScenario | undefined {
+  return evals.scenarios.find((s) => s.category === categoryId && s.edge_case !== true);
+}
+
+/** The base query for a category — the scenario text when tagged, else the description. PURE. */
+function baseQueryFor(evals: AgentspecEvals, categoryId: string, description: string): string {
+  const scenario = baseScenarioFor(evals, categoryId);
+  return scenario !== undefined ? scenario.description : description;
+}
+
+/** The edge query for an edge_case phrase in a category context. PURE. */
+function edgeQueryFor(categoryId: string, edgePhrase: string): string {
+  // A real NL query expressing the edge condition in the category's context.
+  return `[${categoryId}] handle the edge case: ${edgePhrase}`;
+}
+
+/**
+ * MATERIALIZE real DatasetCases from the agentspec.evals seed (F8). For EACH
+ * category: one base item (≥1 per category — the success gate) + one item per
+ * declared edge_case (flagged edge_case). DETERMINISTIC, deduped by content id.
+ * Returns [] for an empty evals block (no categories → no items). PURE.
+ */
+export function materializeFromAgentspec(evals: AgentspecEvals): DatasetCase[] {
+  const out: DatasetCase[] = [];
+  const seen = new Set<string>();
+  const push = (tuple: DatasetTuple, query: string): void => {
+    const c = buildCase(tuple, query, CaseSource.Seed);
+    if (seen.has(c.id)) return; // content-id dedup (re-materialize collides → drop)
+    seen.add(c.id);
+    out.push(c);
+  };
+  for (const cat of evals.dataset_categories) {
+    // base item — one real query per category.
+    push(
+      { category: cat.id, edge_case: EDGE_FALSE },
+      baseQueryFor(evals, cat.id, cat.description),
+    );
+    // edge items — one real query per declared edge_case.
+    for (const edge of cat.edge_cases) {
+      if (edge.length === 0) continue;
+      push({ category: cat.id, edge_case: EDGE_TRUE }, edgeQueryFor(cat.id, edge));
+    }
+  }
+  return out;
+}
+
+/**
+ * Assemble (or extend) a Dataset seeded with the materialized real items. When
+ * `existing` is supplied, the materialized cases are merged MONOTONICALLY (no
+ * duplicates, version bumps). Subject-agnostic. DETERMINISTIC. PURE.
+ */
+export function materializeToDataset(
+  subject: string,
+  evals: AgentspecEvals,
+  existing?: Dataset,
+): Dataset {
+  const dimensions = dimensionsFromAgentspec(evals);
+  const cases = materializeFromAgentspec(evals);
+  const base: Dataset = existing ?? { subject, dimensions, cases: [], version: 0 };
+  return appendToDataset(base, cases);
+}