@hassangameryt/freeflow 0.8.0 → 0.9.0

package/skills/evaluate-skill/scripts/lib/review.mjs

@@ -1,196 +0,0 @@
-import { randomInt } from "node:crypto";
-import { mkdir, readFile, rm } from "node:fs/promises";
-import path from "node:path";
-
-import { sha256, writeJson, writeText } from "./evidence.mjs";
-import { baseProcessEnvironment, runPiProcess } from "./pi.mjs";
-import { resolveResult } from "./view.mjs";
-
-const ANSWERS = new Set(["yes", "no", "unclear"]);
-const RESPONSE_LIMIT = 8000;
-const ARGUMENT_LIMIT = 300;
-
-/**
- * Answers each group's review questions with a separate model that sees two unlabeled runs.
- * The result is advisory: it is written beside, never into, canonical run and grade evidence.
- *
- * @param {string} target
- * @param {{group: string | null}} selectors
- * @param {{root?: string, signal?: AbortSignal, model: string, thinking?: string | null}} options
- */
-export async function reviewResult(target, selectors, { root = process.cwd(), signal, model, thinking = null }) {
-  const resultDirectory = await resolveResult(target, root);
-  const summary = JSON.parse(await readFile(path.join(resultDirectory, "summary.json"), "utf8"));
-  const groups = selectGroups(summary, selectors.group);
-  const outcomes = [];
-  for (const group of groups) {
-    if (signal?.aborted) break;
-    outcomes.push(
-      await reviewGroup(path.join(resultDirectory, "groups", group.id), group.id, { model, thinking, signal }),
-    );
-  }
-  return { resultDirectory, groups: outcomes };
-}
-
-function selectGroups(summary, selector) {
-  const groups = Array.isArray(summary.groups) ? summary.groups : [];
-  if (selector === null) return groups;
-  if (summary.definitionKind === "group") throw new Error("--group cannot be used with a direct group result");
-  const position = /^\d+$/.test(selector) ? Number(selector) : null;
-  const selected = groups.filter((group) => (position === null ? group.id === selector : group.position === position));
-  if (selected.length === 0) throw new Error(`Unknown group selector: ${selector}`);
-  return selected;
-}
-
-async function reviewGroup(groupDirectory, id, { model, thinking, signal }) {
-  const definition = JSON.parse(await readFile(path.join(groupDirectory, "definition.json"), "utf8"));
-  const questions = Array.isArray(definition.review_questions) ? definition.review_questions : [];
-  if (questions.length === 0) return { id, state: "skipped", reason: "no review questions" };
-  const runs = {};
-  for (const variant of ["baseline", "candidate"]) {
-    runs[variant] = JSON.parse(await readFile(path.join(groupDirectory, variant, "run.json"), "utf8"));
-    if (runs[variant].state !== "complete")
-      return { id, state: "skipped", reason: `${variant} run is ${runs[variant].state}` };
-  }
-
-  const labels = randomInt(2) === 0 ? { A: "baseline", B: "candidate" } : { A: "candidate", B: "baseline" };
-  const prompt = reviewPrompt(definition, questions, { A: runs[labels.A], B: runs[labels.B] });
-  const reviewDirectory = path.join(groupDirectory, "review");
-  await rm(reviewDirectory, { recursive: true, force: true });
-  await mkdir(reviewDirectory, { recursive: true });
-  await writeText(path.join(reviewDirectory, "prompt.md"), prompt);
-
-  const args = [
-    "--mode",
-    "json",
-    "-p",
-    "--no-session",
-    "--no-extensions",
-    "--no-skills",
-    "--no-prompt-templates",
-    "--no-themes",
-    "--no-context-files",
-    "--no-approve",
-    "--no-tools",
-    "--model",
-    model,
-  ];
-  if (thinking !== null) args.push("--thinking", thinking);
-  args.push(prompt);
-  const observation = await runPiProcess({
-    command: "pi",
-    args,
-    cwd: reviewDirectory,
-    eventsFile: path.join(reviewDirectory, "events.jsonl"),
-    stderrFile: path.join(reviewDirectory, "stderr.log"),
-    signal,
-    environment: { ...baseProcessEnvironment(), PWD: reviewDirectory, PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0" },
-  });
-  await writeText(path.join(reviewDirectory, "final.md"), `${observation.response}\n`);
-
-  const errors = [];
-  if (observation.exitCode !== 0 || !observation.settled || observation.assistantError !== null) {
-    errors.push(
-      `reviewer did not complete: exit=${observation.exitCode} settled=${observation.settled} error=${observation.assistantError}`,
-    );
-  }
-  const answers = errors.length === 0 ? parseAnswers(observation.response, questions, labels, errors) : [];
-  const state = errors.length === 0 ? "complete" : "review-error";
-  await writeJson(path.join(groupDirectory, "semantic-grade.json"), {
-    schema_version: 1,
-    kind: "advisory-review",
-    advisory: true,
-    state,
-    model,
-    thinking,
-    labels,
-    prompt: { file: "review/prompt.md", sha256: sha256(prompt) },
-    answers,
-    errors,
-    usage: observation.usage,
-    artifacts: { events: "review/events.jsonl", final: "review/final.md", stderr: "review/stderr.log" },
-    completedAt: new Date().toISOString(),
-  });
-  return { id, state, artifact: "semantic-grade.json" };
-}
-
-function reviewPrompt(definition, questions, runs) {
-  const task =
-    typeof definition.input?.prompt === "string" ? [definition.input.prompt] : (definition.input?.turns ?? []);
-  const lines = [
-    "You are reviewing two anonymous runs, Run A and Run B, of the same agent task.",
-    "Judge only the evidence below. Answer every question for both runs with yes, no, or unclear.",
-    "",
-    "Task turns:",
-    ...task.map((turn, index) => `${index + 1}. ${turn}`),
-    "",
-    "Questions:",
-    ...questions.map((question, index) => `${index + 1}. ${question}`),
-  ];
-  for (const label of ["A", "B"]) lines.push("", `## Run ${label}`, ...runEvidence(runs[label]));
-  lines.push(
-    "",
-    "Reply with only JSON in this shape:",
-    '{"answers":[{"question":1,"A":"yes|no|unclear","B":"yes|no|unclear","rationale":"one sentence"}]}',
-  );
-  return lines.join("\n");
-}
-
-function runEvidence(run) {
-  const turns = Array.isArray(run.turns)
-    ? run.turns
-    : [{ turn: 1, response: run.response, toolActivity: run.toolActivity }];
-  const lines = [];
-  for (const turn of turns) {
-    lines.push(`Turn ${turn.turn ?? 1} response:`, truncate(String(turn.response ?? ""), RESPONSE_LIMIT));
-    for (const call of turn.toolActivity ?? []) {
-      const status = call.completed ? (call.isError ? "failed" : "succeeded") : "incomplete";
-      lines.push(`- tool ${call.toolName} ${status}: ${truncate(JSON.stringify(call.args ?? null), ARGUMENT_LIMIT)}`);
-    }
-  }
-  const changes = run.effects?.changes;
-  if (changes) {
-    for (const kind of ["created", "modified", "deleted"]) {
-      lines.push(`Files ${kind}: ${(changes[kind] ?? []).join(", ") || "none"}`);
-    }
-  }
-  return lines;
-}
-
-function truncate(value, limit) {
-  return value.length <= limit ? value : `${value.slice(0, limit)}… [truncated ${value.length - limit} characters]`;
-}
-
-function parseAnswers(response, questions, labels, errors) {
-  const text = response
-    .trim()
-    .replace(/^```(?:json)?\s*/i, "")
-    .replace(/\s*```$/, "");
-  let parsed;
-  try {
-    parsed = JSON.parse(text);
-  } catch {
-    errors.push("reviewer response is not JSON");
-    return [];
-  }
-  const answers = Array.isArray(parsed?.answers) ? parsed.answers : null;
-  if (answers === null || answers.length !== questions.length) {
-    errors.push(`reviewer must answer exactly ${questions.length} question(s)`);
-    return [];
-  }
-  const mapped = [];
-  for (const [index, question] of questions.entries()) {
-    const answer = answers.find((entry) => entry?.question === index + 1);
-    if (!answer || !ANSWERS.has(answer.A) || !ANSWERS.has(answer.B) || typeof answer.rationale !== "string") {
-      errors.push(`reviewer answer for question ${index + 1} is missing or invalid`);
-      continue;
-    }
-    mapped.push({
-      question,
-      [labels.A]: answer.A,
-      [labels.B]: answer.B,
-      rationale: answer.rationale,
-    });
-  }
-  return errors.length === 0 ? mapped : [];
-}

package/skills/evaluate-skill/scripts/lib/runner.mjs

@@ -1,405 +0,0 @@
-import { mkdir, realpath } from "node:fs/promises";
-import path from "node:path";
-
-import { COMMAND_TOOL, loadDefinition, selectDefinition } from "./definitions.mjs";
-import { createInvocationId, fileIdentity, writeJson, writeText } from "./evidence.mjs";
-import { gradeDeterministic } from "./grade.mjs";
-import { runBody, runDescription, VariantSetupError } from "./pi.mjs";
-import { prepareFixture, prepareRuntime } from "./sandbox.mjs";
-
-const VARIANTS = ["baseline", "candidate"];
-const DESCRIPTION_TOOLS = new Set(["read"]);
-const BODY_TOOLS = new Set(["read", "write", "edit"]);
-// Native tools must be separated from extension tools before CLI tool selection is built.
-const NATIVE_TOOLS = new Set(["read", "bash", "powershell", "edit", "write", "grep", "find", "ls"]);
-const DEFAULT_RUNTIME = {
-  host: "pi",
-  session: false,
-  prompt: "isolated",
-  extensions: [],
-  environment: { literal: {}, inherit: [] },
-};
-
-/**
- * @param {string} definitionFile
- * @param {any} selectors
- * @param {{root?: string, signal?: AbortSignal, budget?: {maxCost: number, spent: number, exhausted: boolean} | null}} options
- */
-export async function runEvaluation(
-  definitionFile,
-  selectors = {},
-  { root = process.cwd(), signal, budget = null } = {},
-) {
-  const definitionPath = await realpath(path.resolve(root, definitionFile));
-  const definition = await loadDefinition(definitionPath, { root });
-  const selection = selectDefinition(definition, selectors);
-  assertSupportedSelection(selection);
-  const invocationId = createInvocationId();
-  const runsDirectory = path.join(root, ".skill-eval", "runs");
-  const resultDirectory = path.join(runsDirectory, invocationId);
-  await mkdir(runsDirectory, { recursive: true });
-  await mkdir(resultDirectory);
-  await writeJson(path.join(resultDirectory, "invocation.json"), {
-    schema_version: 1,
-    id: invocationId,
-    definition: definitionPath,
-    selection: {
-      group: selectors.group ?? null,
-      variant: selectors.variant ?? null,
-      groups: selection.groups.map((entry) => ({
-        id: entry.group.id,
-        position: entry.position,
-        variants: entry.variants,
-      })),
-    },
-    startedAt: new Date().toISOString(),
-  });
-
-  // Budget exhaustion cancels only queued subjects; a subject already running completes and is charged.
-  const controller = new AbortController();
-  const onParentAbort = () => controller.abort();
-  if (signal?.aborted) controller.abort();
-  else signal?.addEventListener("abort", onParentAbort, { once: true });
-  const charge = (run) => {
-    if (budget === null) return;
-    budget.spent += run.usage?.cost?.total ?? 0;
-    if (budget.spent >= budget.maxCost) {
-      budget.exhausted = true;
-      controller.abort();
-    }
-  };
-
-  const groups = [];
-  try {
-    for (const selected of selection.groups) {
-      groups.push(await runGroup({ selected, root, resultDirectory, signal: controller.signal, charge }));
-    }
-  } finally {
-    signal?.removeEventListener("abort", onParentAbort);
-  }
-  const state = batchState(groups);
-  const summary = {
-    schema_version: 1,
-    id: invocationId,
-    state,
-    definitionKind: selection.definitionKind,
-    groups,
-    completedAt: new Date().toISOString(),
-  };
-  if (budget !== null) summary.budget = budgetEvidence(budget);
-  await writeJson(path.join(resultDirectory, "summary.json"), summary);
-  return { id: invocationId, path: resultDirectory, state, summary };
-}
-
-function assertSupportedSelection(selection) {
-  for (const selected of selection.groups) {
-    const { group } = selected;
-    if (group.type !== "description" && group.type !== "body") {
-      throw new Error("skill-eval run currently supports description and body groups only");
-    }
-    if (group.expectations.some((expectation) => expectation.kind === "command")) {
-      throw new Error("command expectations are not supported");
-    }
-    const supportedTools = group.type === "body" ? BODY_TOOLS : DESCRIPTION_TOOLS;
-    const unsupportedNativeTools = group.tools.filter((tool) => NATIVE_TOOLS.has(tool) && !supportedTools.has(tool));
-    if (unsupportedNativeTools.length > 0) {
-      throw new Error(`${group.type} execution does not support native tools: ${unsupportedNativeTools.join(", ")}`);
-    }
-    // The evaluator supplies run_command itself when the definition declares commands.
-    const customTools = group.tools.filter(
-      (tool) => !NATIVE_TOOLS.has(tool) && !supportedTools.has(tool) && tool !== COMMAND_TOOL,
-    );
-    if (group.type === "description" && customTools.length > 0) {
-      throw new Error(`${group.type} execution does not support custom tools: ${customTools.join(", ")}`);
-    }
-    if (customTools.length > 0 && (group.runtime?.extensions?.length ?? 0) === 0) {
-      throw new Error(
-        `${group.type} execution requires declared extensions for custom tools: ${customTools.join(", ")}`,
-      );
-    }
-  }
-}
-
-export function budgetEvidence(budget) {
-  return { maxCost: budget.maxCost, spent: Number(budget.spent.toFixed(10)), exhausted: budget.exhausted };
-}
-
-async function runGroup({ selected, root, resultDirectory, signal, charge }) {
-  const { group } = selected;
-  const groupDirectory = path.join(resultDirectory, "groups", group.id);
-  await mkdir(groupDirectory, { recursive: true });
-  await writeJson(path.join(groupDirectory, "definition.json"), group);
-
-  let fixture = null;
-  let runtime = null;
-  let resourceError = null;
-  if (!signal?.aborted) {
-    try {
-      fixture = await prepareFixture({ declaredPath: group.fixture, root, groupDirectory });
-      runtime = await prepareRuntime({
-        runtime: group.runtime ?? DEFAULT_RUNTIME,
-        root,
-        groupDirectory,
-      });
-    } catch (error) {
-      resourceError = error instanceof Error ? error.message : String(error);
-    }
-  }
-
-  const runs = {};
-  const runFiles = {};
-  for (const variant of VARIANTS) {
-    const variantDirectory = path.join(groupDirectory, variant);
-    await mkdir(variantDirectory, { recursive: true });
-    if (selected.variants[variant] === "not-selected") {
-      runs[variant] = notSelectedRun(group, variant);
-    } else if (signal?.aborted) {
-      runs[variant] = cancelledRun(group, variant);
-    } else if (resourceError === null) {
-      runs[variant] = await executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal });
-      charge(runs[variant]);
-    } else {
-      runs[variant] = invalidRun(group, variant, resourceError);
-    }
-    const persisted = await persistRunOrFailure(variantDirectory, runs[variant]);
-    runs[variant] = persisted.run;
-    runFiles[variant] = persisted.file;
-  }
-
-  const evidence = {
-    baseline: await fileIdentity(runFiles.baseline),
-    candidate: await fileIdentity(runFiles.candidate),
-  };
-  const grade = await createGrade(group, runs, evidence);
-  const persistedGrade = await persistGrade(groupDirectory, grade);
-  const selectedRuns = VARIANTS.filter((variant) => selected.variants[variant] === "selected").map(
-    (variant) => runs[variant],
-  );
-  const selectedState = selectedGroupState(selectedRuns, persistedGrade.grade);
-  let groupState = selectedState;
-  if (selectedState !== "cancelled" && resourceError !== null) groupState = "invalid";
-  const groupResult = {
-    id: group.id,
-    position: selected.position,
-    state: groupState,
-    variants: Object.fromEntries(VARIANTS.map((variant) => [variant, runs[variant].state])),
-    grade: persistedGrade.grade.state,
-    artifacts: { grade: persistedGrade.artifact, group: "group.json" },
-    errors: persistedGrade.errors,
-  };
-  return persistGroupResult(groupDirectory, groupResult);
-}
-
-function batchState(groups) {
-  if (groups.some((group) => group.state === "cancelled")) return "cancelled";
-  if (groups.every((group) => group.state === "complete")) return "complete";
-  return "partially-complete";
-}
-
-function selectedGroupState(runs, grade) {
-  if (runs.some((run) => run.state === "cancelled")) return "cancelled";
-  if (runs.every((run) => run.state === "complete") && grade.state !== "grade-error") return "complete";
-  return "partially-complete";
-}
-
-async function createGrade(group, runs, evidence) {
-  try {
-    return await gradeDeterministic(group, runs, evidence);
-  } catch (error) {
-    return gradeErrorReport(evidence, `deterministic grading failed: ${errorMessage(error)}`);
-  }
-}
-
-async function persistGrade(groupDirectory, grade) {
-  const artifact = "deterministic-grade.json";
-  try {
-    await writeJson(path.join(groupDirectory, artifact), grade);
-    return { grade, artifact, errors: [] };
-  } catch (error) {
-    const reason = `failed to persist ${artifact}: ${errorMessage(error)}`;
-    const failed = gradeErrorReport(grade.evidence, reason, grade);
-    const fallback = "deterministic-grade-error.json";
-    await writeJson(path.join(groupDirectory, fallback), failed);
-    return {
-      grade: failed,
-      artifact: fallback,
-      errors: [{ kind: "persistence", artifact, message: reason }],
-    };
-  }
-}
-
-async function persistGroupResult(groupDirectory, groupResult) {
-  const artifact = "group.json";
-  try {
-    await writeJson(path.join(groupDirectory, artifact), groupResult);
-    return groupResult;
-  } catch (error) {
-    const message = `failed to persist ${artifact}: ${errorMessage(error)}`;
-    const fallback = "group-error.json";
-    const failed = {
-      ...groupResult,
-      state: groupResult.state === "complete" ? "partially-complete" : groupResult.state,
-      artifacts: { ...groupResult.artifacts, group: fallback },
-      errors: [...groupResult.errors, { kind: "persistence", artifact, message }],
-    };
-    await writeJson(path.join(groupDirectory, fallback), failed);
-    return failed;
-  }
-}
-
-function gradeErrorReport(evidence, reason, grade = null) {
-  return {
-    schema_version: 1,
-    state: "grade-error",
-    evidence,
-    checks: grade?.checks ?? [],
-    comparisons: grade?.comparisons ?? [],
-    errors: [...(grade?.errors ?? []), { id: null, reason }],
-  };
-}
-
-function errorMessage(error) {
-  return error instanceof Error ? error.message : String(error);
-}
-
-async function executeVariant({ group, variant, root, variantDirectory, fixture, runtime, signal }) {
-  try {
-    const run = group.type === "body" ? runBody : runDescription;
-    return await run({ group, variant, root, variantDirectory, fixture, runtime, signal });
-  } catch (error) {
-    if (error instanceof VariantSetupError) return invalidRun(group, variant, error.message);
-    return infrastructureFailedRun(group, variant, error);
-  }
-}
-
-async function persistRunOrFailure(variantDirectory, run) {
-  try {
-    return { run, file: await persistRun(variantDirectory, run) };
-  } catch (error) {
-    const failed = persistenceFailedRun(run, error);
-    const runFile = path.join(variantDirectory, "run.json");
-    await writeJson(runFile, {
-      ...failed,
-      transcript: failed.transcript ?? [],
-      artifacts: {
-        events: failed.process === undefined ? null : "events.jsonl",
-        final: null,
-        stderr: failed.process === undefined ? null : "stderr.log",
-      },
-    });
-    return { run: failed, file: runFile };
-  }
-}
-
-async function persistRun(variantDirectory, run) {
-  const transcript = run.transcript ?? [];
-  const response = run.response ?? "";
-  await writeJson(path.join(variantDirectory, "transcript.json"), transcript);
-  await writeText(path.join(variantDirectory, "final.md"), response === "" ? "" : `${response}\n`);
-  const runFile = path.join(variantDirectory, "run.json");
-  const hasProcessEvidence = run.process !== undefined;
-  const persisted = {
-    ...run,
-    transcript: "transcript.json",
-    artifacts: {
-      events: hasProcessEvidence ? "events.jsonl" : null,
-      final: "final.md",
-      stderr: hasProcessEvidence ? "stderr.log" : null,
-      context: run.contextObservationArtifact ? "context-observations.jsonl" : null,
-    },
-  };
-  if (Array.isArray(run.turns)) persisted.turns = externalizeTurnTranscripts(run.turns);
-  await writeJson(runFile, persisted);
-  return runFile;
-}
-
-function externalizeTurnTranscripts(turns) {
-  let offset = 0;
-  return turns.map((turn) => {
-    const length = Array.isArray(turn.transcript) ? turn.transcript.length : 0;
-    const transcript = { file: "transcript.json", start: offset, end: offset + length };
-    offset += length;
-    return { ...turn, transcript };
-  });
-}
-
-function notSelectedRun(group, variant) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "not-selected",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: null,
-  };
-}
-
-function cancelledRun(group, variant) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "cancelled",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: { kind: "cancellation", message: "Cancelled before subject execution" },
-  };
-}
-
-function invalidRun(group, variant, message) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "invalid",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: { kind: "setup", message },
-  };
-}
-
-function persistenceFailedRun(run, error) {
-  return {
-    ...run,
-    state: "infrastructure-failed",
-    error: {
-      kind: "persistence",
-      message: error instanceof Error ? error.message : String(error),
-      cause: run.error ?? null,
-    },
-  };
-}
-
-function infrastructureFailedRun(group, variant, error) {
-  return {
-    schema_version: 1,
-    variant,
-    state: "infrastructure-failed",
-    evaluationType: group.type,
-    prompt: group.input.prompt ?? null,
-    prompts: group.input.turns ?? null,
-    activation: unavailableActivation(),
-    response: "",
-    transcript: [],
-    error: { kind: "infrastructure", message: error instanceof Error ? error.message : String(error) },
-  };
-}
-
-function unavailableActivation() {
-  return {
-    targetRead: null,
-    firstReadTurn: null,
-    readTurns: [],
-    successfulReadPaths: [],
-  };
-}

package/skills/evaluate-skill/scripts/lib/trials.mjs

@@ -1,103 +0,0 @@
-import { mkdir, readFile } from "node:fs/promises";
-import path from "node:path";
-
-import { createInvocationId, writeJson } from "./evidence.mjs";
-import { budgetEvidence, runEvaluation } from "./runner.mjs";
-
-const CHECK_STATES = ["pass", "fail", "unavailable"];
-
-/**
- * Runs the selection as independent complete invocations and records an aggregate beside them.
- * Each trial remains an ordinary result; the aggregate only counts their deterministic grades.
- *
- * @param {string} definitionFile
- * @param {any} selectors
- * @param {{root?: string, signal?: AbortSignal, trials: number, maxCost: number | null, onTrial?: (result: any) => void}} options
- */
-export async function runTrials(definitionFile, selectors, { root = process.cwd(), signal, trials, maxCost, onTrial }) {
-  const budget = maxCost === null ? null : { maxCost, spent: 0, exhausted: false };
-  const results = [];
-  for (let trial = 0; trial < trials; trial += 1) {
-    if (signal?.aborted || budget?.exhausted) break;
-    const result = await runEvaluation(definitionFile, selectors, { root, signal, budget });
-    results.push(result);
-    onTrial?.(result);
-  }
-
-  const id = createInvocationId();
-  const directory = path.join(root, ".skill-eval", "runs", id);
-  await mkdir(directory, { recursive: true });
-  const aggregate = {
-    schema_version: 1,
-    kind: "aggregate",
-    id,
-    state: aggregateState(results, trials),
-    requestedTrials: trials,
-    trials: results.map((result) => ({ id: result.id, state: result.state })),
-    groups: await aggregateGroups(results),
-    completedAt: new Date().toISOString(),
-  };
-  if (budget !== null) aggregate.budget = budgetEvidence(budget);
-  await writeJson(path.join(directory, "aggregate.json"), aggregate);
-  return { id, path: directory, state: aggregate.state, aggregate };
-}
-
-function aggregateState(results, requested) {
-  if (results.length < requested || results.some((result) => result.state === "cancelled")) return "cancelled";
-  if (results.every((result) => result.state === "complete")) return "complete";
-  return "partially-complete";
-}
-
-async function aggregateGroups(results) {
-  /** @type {Map<string, any>} */
-  const groups = new Map();
-  for (const result of results) {
-    for (const group of result.summary.groups) {
-      const entry = groups.get(group.id) ?? { id: group.id, trials: 0, comparisons: new Map(), checks: new Map() };
-      groups.set(group.id, entry);
-      const gradeFile = path.join(
-        result.path,
-        "groups",
-        group.id,
-        group.artifacts?.grade ?? "deterministic-grade.json",
-      );
-      const grade = JSON.parse(await readFile(gradeFile, "utf8"));
-      entry.trials += 1;
-      for (const check of grade.checks ?? []) {
-        const counts = entry.checks.get(check.id) ?? {
-          id: check.id,
-          variant: check.variant,
-          kind: check.kind,
-          ...zeroCounts(),
-        };
-        entry.checks.set(check.id, counts);
-        if (CHECK_STATES.includes(check.state)) counts[check.state] += 1;
-      }
-      for (const comparison of grade.comparisons ?? []) {
-        const counts = entry.comparisons.get(comparison.id) ?? {
-          id: comparison.id,
-          kind: comparison.kind,
-          baseline: zeroCounts(),
-          candidate: zeroCounts(),
-          transitions: {},
-        };
-        entry.comparisons.set(comparison.id, counts);
-        for (const variant of ["baseline", "candidate"]) {
-          const state = comparison[variant]?.state;
-          if (CHECK_STATES.includes(state)) counts[variant][state] += 1;
-        }
-        counts.transitions[comparison.transition] = (counts.transitions[comparison.transition] ?? 0) + 1;
-      }
-    }
-  }
-  return [...groups.values()].map((entry) => ({
-    id: entry.id,
-    trials: entry.trials,
-    comparisons: [...entry.comparisons.values()],
-    checks: [...entry.checks.values()],
-  }));
-}
-
-function zeroCounts() {
-  return { pass: 0, fail: 0, unavailable: 0 };
-}