@mintlify/cli 4.0.1466 → 4.0.1468

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/cli",
-  "version": "4.0.1466",
+  "version": "4.0.1468",
   "description": "The Mintlify CLI",
   "engines": {
     "node": ">=18.0.0"
@@ -105,5 +105,5 @@
     "vitest": "2.1.9",
     "vitest-mock-process": "1.0.4"
   },
-  "gitHead": "ce376b725af9861a1342801490b421ba3e414136"
+  "gitHead": "8baca715ce0789cc49c2d73efad90254b89411ff"
 }

package/src/agent-harness/MINT_TEST_SYSTEM_DESIGN.md

@@ -43,15 +43,21 @@ flowchart TD
 
 The harness exports five types. Status unions and nested block, environment, summary, command, and task fields live on the contract that owns them.
 
-| Exported type           | Contract                                                                                                                                    |
-| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
-| `HarnessAgent`          | The supported runner names, `claude` and `codex`.                                                                                           |
-| `AgentRunner`           | One `prompt(prompt)` method returning the final response text. `ClaudeRunner` and `CodexRunner` implement it.                               |
-| `CodeBlockCollection`   | The docs root and fenced block locations grouped by markdown file. `collectCodeBlocks` returns it.                                          |
-| `GeneratedTestManifest` | The inferred output of the runtime `mint-test.json` schema, including generated files, block dispositions, environment needs, and commands. |
-| `CodeTestReport`        | The complete persisted `report.json` shape. Task, block, command, environment, summary, and status fields are nested here.                  |
-
-`GeneratedTestManifest` is inferred from the runtime schema in `packages/cli/src/agent-harness/types.ts`, so validation and TypeScript cannot drift apart. Contract changes still require a schema test and an update to this table.
+| Exported type           | Contract                                                                                                                                                                                                                  |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `HarnessAgent`          | The supported runner names, `claude` and `codex`.                                                                                                                                                                         |
+| `AgentRunner`           | One `prompt(prompt)` method returning the final response text. `ClaudeRunner` and `CodexRunner` implement it.                                                                                                             |
+| `CodeBlockCollection`   | The docs root and fenced block locations grouped by markdown file. `collectCodeBlocks` returns it.                                                                                                                        |
+| `GeneratedTestManifest` | The normalized output of the runtime `mint-test.json` schema: generated files plus setup and test commands. Command ids are optional in the file; `manifestSchema` assigns `setup-N` / `test-N` when an agent omits them. |
+| `CodeTestReport`        | The complete persisted `report.json` shape. Task, block, command, environment, summary, and status fields are nested here.                                                                                                |
+
+`GeneratedTestManifest` is produced by the runtime schema in `packages/cli/src/agent-harness/types.ts` (`manifestFileSchema.transform(assignCommandIds)`), so validation and TypeScript cannot drift apart. Contract changes still require a schema test and an update to this table.
+
+`packages/cli/src/agent-harness/manifest.ts` owns reading the manifest. It turns a missing file, invalid JSON, or a schema mismatch into one actionable message per problem; that message is what the UI, the non-interactive summary, and `report.json` show for the page. Each task directory also keeps `mint-test-agent.md` (the agent's final summary per attempt) and, after a failed generation, `mint-test-error.txt` (every attempt's failure reason).
+
+`runCodeTests` loads each selected agent SDK before any page is checked (`agentPreflight.ts`), so a missing optional dependency fails the run once with an install hint instead of failing every page minutes later.
+
+`runCodeTests` runs the pipeline in order: `setupFolders` picks the run id, creates the run directory, and builds one task per agent and page without creating task directories; `checkTestability` asks a small model whether each page is testable, running from the run directory, and only once a page is proven testable creates its task directory and writes `mint-test-check.json` there; `scaffoldTaskDirectories` then adds `tests/` and `package.json` to those same tasks; `generateTestCode` and `executeCodeBlocks` skip every other task. A page that is not testable leaves nothing on disk. A resumed run reuses the check file for testable pages, re-checks the others, and re-applies the scaffold, which is idempotent.
 
 `runCodeTests` declares its input object and progress events on the function. `executeCodeTestCommand` does the same for command input. Neither API has a separate options, request, progress, preflight, command-runner, or command-result type to keep in sync.
 

package/src/agent-harness/agentPreflight.ts

@@ -0,0 +1,13 @@
+import { loadClaudeAgentSdk, loadCodexSdk } from './loadAgentSdk.js';
+import type { HarnessAgent } from './types.js';
+
+/**
+ * Fail fast, before any page is checked, when an agent SDK is not installed. Without this every
+ * page reports the same missing-dependency error minutes later as "test could not be generated".
+ */
+export async function ensureAgentSdks(agents: Iterable<HarnessAgent>): Promise<void> {
+  for (const agent of new Set(agents)) {
+    if (agent === 'claude') await loadClaudeAgentSdk();
+    else await loadCodexSdk();
+  }
+}

package/src/agent-harness/buildTaskPrompt.ts

@@ -1,9 +1,11 @@
 import path from 'node:path';
 
+import { SAFE_EXECUTABLES } from './manifest.js';
+
 export const SYSTEM_PROMPT = `# Mint test generator
 
-You will be provided a markdown page guide that contians codeblocks. Follow the guide and implement
-the code as instrucuted by the guide into a given output directory.
+You will be given a markdown documentation page that contains code blocks. Follow the page and
+implement the code as the page instructs, inside the given output directory.
 
 Once finished, create a mint-test.json file in the current working directory with this shape:
 
@@ -13,12 +15,14 @@ Once finished, create a mint-test.json file in the current working directory wit
   "setupCommands": [],
   "testCommands": [
     {
+      "id": "node-tests",
       "executable": "node",
-      "args": ["--test", "tests/"],
+      "args": ["--test"],
       "cwd": ".",
       "timeoutMs": 120000
     },
     {
+      "id": "python-tests",
       "executable": "python3",
       "args": ["-m", "unittest", "discover", "-s", "tests"],
       "cwd": ".",
@@ -27,11 +31,18 @@ Once finished, create a mint-test.json file in the current working directory wit
   ]
 }
 
-Command cwd values must be relative to the current working directory. Finish with a short plain-text summary.
+Manifest rules:
+- "generatedFiles" lists every file you wrote, as paths relative to the current working directory. Each one must exist.
+- Every command has a unique "id" (letters, digits, dashes). Command "cwd" values are relative to the current working directory.
+- "executable" must be one of: ${[...SAFE_EXECUTABLES].join(', ')}, or a "./" script inside the current working directory. Do not use a shell, pipes, or "&&".
+- Do not add any other keys to the manifest.
+
+Finish with a short plain-text summary of what you tested and anything you could not test.
 
 Rules:
 - Read only the page named in the task. You may also read docs.json and an OpenAPI file referenced by that page.
-- The output directory is already scaffolded: package.json exists with "npm test" wired to "node --test tests/", and a tests/ directory is ready. Do not recreate or rewrite this scaffolding.
+- The output directory is already scaffolded: package.json exists with "npm test" wired to "node --test", and a tests/ directory is ready. Do not recreate or rewrite this scaffolding.
+- Run JavaScript tests with "node" and args ["--test"] from the task directory (no directory argument: on Node 21+ a path like "tests/" is treated as a glob and finds nothing). Node discovers tests/*.test.js on its own.
 - Write JavaScript and TypeScript tests as ES modules named tests/*.test.js using the Node built-in test runner (node:test and node:assert). Write Python tests as tests/test_*.py using the standard library unittest module, run with python3 -m unittest discover -s tests.
 - Prefer the standard library over installing dependencies; add a setup command only when a test truly cannot run without one.
 - Never write README, docs, or explanation files.
@@ -73,10 +84,10 @@ export const CHECK_JSON_INSTRUCTION = `
 Respond with only this JSON object and nothing else:
 { "version": 1, "testable": false, "reason": "expected output, not runnable code" }`;
 
-export function buildCheckPrompt(file: string, fileContnet: string): string {
+export function buildCheckPrompt(file: string, fileContent: string): string {
   return `Report if the file at filePath ${JSON.stringify(file)} is testable
 
 <file>
-${fileContnet}
+${fileContent}
 </file>`;
 }

package/src/agent-harness/countReport.ts

@@ -0,0 +1,17 @@
+import type { TaskResult } from './types.js';
+
+export interface ReportCounts {
+  passed: number;
+  failed: number;
+  cancelled: number;
+  agentErrors: number;
+}
+
+export function countReport(tasks: TaskResult[]): ReportCounts {
+  const counts: ReportCounts = { passed: 0, failed: 0, cancelled: 0, agentErrors: 0 };
+  for (const task of tasks) {
+    if (task.status === 'agent_error') counts.agentErrors++;
+    else counts[task.status]++;
+  }
+  return counts;
+}

package/src/agent-harness/executeCodeBlocks.ts

@@ -52,6 +52,55 @@ function outputCollector(): { append: (chunk: Buffer | string) => void; value: (
   };
 }
 
+const MAX_FAILURE_LINE_LENGTH = 200;
+const TAP_SUMMARY_PATTERN =
+  /^#\s*(tests|suites|pass|fail|cancelled|skipped|todo|duration_ms)\b|^1\.\.\d+$|^TAP version/;
+const ERROR_LINE_PATTERN = /\b(?:[A-Za-z]*Error|Traceback|FAILED|ModuleNotFoundError)\b/;
+const TAP_METADATA_PATTERN = /^(name|code|failureType|location|stack)\s*:/;
+
+/**
+ * The most useful single line from a failed command's output: the first thrown error, else the
+ * first failing TAP assertion, else the last line that is not a TAP summary.
+ */
+export function pickFailureLine(text: string): string {
+  const lines = text
+    .split(/\r?\n/)
+    .map((raw) => raw.trim())
+    .filter((raw) => raw.length > 0 && !raw.startsWith('[mint test:'))
+    .map((raw) => ({ raw, text: raw.replace(/^#\s?/, '').trim() }))
+    .filter((line) => line.text.length > 0);
+  const thrown = lines.find(
+    ({ text: candidate }) =>
+      ERROR_LINE_PATTERN.test(candidate) &&
+      !candidate.startsWith('at ') &&
+      !/^(ok|Traceback)\b/.test(candidate) &&
+      !TAP_METADATA_PATTERN.test(candidate)
+  );
+  const assertion = lines.find(({ text: candidate }) => candidate.startsWith('not ok'));
+  const informative = lines.filter(({ raw }) => !TAP_SUMMARY_PATTERN.test(raw));
+  const line =
+    thrown?.text ??
+    assertion?.text ??
+    informative[informative.length - 1]?.text ??
+    lines[lines.length - 1]?.text ??
+    '';
+  return line.length > MAX_FAILURE_LINE_LENGTH
+    ? `${line.slice(0, MAX_FAILURE_LINE_LENGTH - 1)}…`
+    : line;
+}
+
+/** One line explaining the first command that did not pass, for the UI and the report. */
+export function summarizeCommandFailure(commands: CommandResult[]): string | undefined {
+  const failed = commands.find((command) => command.status !== 'passed');
+  if (!failed) return undefined;
+  const invocation = [failed.executable, ...failed.args].join(' ');
+  const reason =
+    failed.error ??
+    (failed.exitCode === null ? 'did not exit cleanly' : `exited with code ${failed.exitCode}`);
+  const detail = pickFailureLine(failed.stderr) || pickFailureLine(failed.stdout);
+  return `${failed.phase} command \`${invocation}\` ${reason}${detail ? `: ${detail}` : ''}`;
+}
+
 function unrunCommandResult(
   command: ManifestCommand,
   phase: 'setup' | 'test',
@@ -184,11 +233,17 @@ export async function executeCodeBlocks({
   onTaskUpdate: (update: TaskUpdate) => void;
 }): Promise<TaskResult[]> {
   return mapWithConcurrency(tasks, concurrency, async (task) => {
+    const generatedFiles =
+      task.manifest?.generatedFiles.map((file) => path.resolve(task.directory, file)) ??
+      task.generatedFiles ??
+      [];
     const base = {
       id: task.id,
       agent: task.agent,
       file: task.file,
       directory: task.directory,
+      generatedFiles,
+      testable: task.testable === true,
       attempts: task.attempts,
       tokens: task.tokens,
     };
@@ -199,6 +254,8 @@ export async function executeCodeBlocks({
           id: task.id,
           agent: task.agent,
           file: task.file,
+          directory: task.directory,
+          generatedFiles,
           phase: 'not_testable',
           tokens: task.tokens,
         });
@@ -209,8 +266,11 @@ export async function executeCodeBlocks({
         id: task.id,
         agent: task.agent,
         file: task.file,
+        directory: task.directory,
+        generatedFiles,
         phase: status,
         tokens: task.tokens,
+        error: task.error,
       });
       return {
         ...base,
@@ -223,6 +283,8 @@ export async function executeCodeBlocks({
       id: task.id,
       agent: task.agent,
       file: task.file,
+      directory: task.directory,
+      generatedFiles,
       phase: 'running',
       tokens: task.tokens,
     });
@@ -273,13 +335,17 @@ export async function executeCodeBlocks({
       FAILURE_PRECEDENCE.find((candidate) =>
         commands.some((command) => command.status === candidate)
       ) ?? 'passed';
+    const error = status === 'passed' ? undefined : summarizeCommandFailure(commands);
     onTaskUpdate({
       id: task.id,
       agent: task.agent,
       file: task.file,
+      directory: task.directory,
+      generatedFiles,
       phase: status,
       tokens: task.tokens,
+      ...(error ? { error } : {}),
     });
-    return { ...base, status, commands };
+    return { ...base, status, ...(error ? { error } : {}), commands };
   });
 }

package/src/agent-harness/generateTestCode.ts

@@ -3,45 +3,19 @@ import path from 'node:path';
 
 import { SYSTEM_PROMPT, buildTaskPrompt } from './buildTaskPrompt.js';
 import { loadClaudeAgentSdk, loadCodexSdk } from './loadAgentSdk.js';
+import {
+  MANIFEST_FILENAME,
+  SAFE_EXECUTABLES,
+  appendAgentReply,
+  readManifest,
+  writeErrorDetails,
+} from './manifest.js';
 import { mapWithConcurrency } from './mapWithConcurrency.js';
 import { checkFileTestability } from './tasks/checkTestability.js';
-import {
-  manifestSchema,
-  type GeneratedTask,
-  type GeneratedTestManifest,
-  type TaskUpdate,
-} from './types.js';
+import type { GeneratedTask, GeneratedTestManifest, TaskUpdate } from './types.js';
 
-const MANIFEST_FILENAME = 'mint-test.json';
-const SAFE_EXECUTABLES = new Set([
-  'bun',
-  'bunx',
-  'bundle',
-  'cargo',
-  'composer',
-  'deno',
-  'dotnet',
-  'go',
-  'gradle',
-  'java',
-  'javac',
-  'mvn',
-  'node',
-  'npm',
-  'npx',
-  'php',
-  'pip',
-  'pip3',
-  'pnpm',
-  'python',
-  'python3',
-  'pytest',
-  'ruby',
-  'rustc',
-  'uv',
-  'vitest',
-  'yarn',
-]);
+const SCAFFOLD_FILENAMES = new Set([MANIFEST_FILENAME, 'package.json']);
+const IGNORED_GENERATED_DIRECTORIES = new Set(['.git', 'node_modules']);
 
 async function promptClaude({
   prompt,
@@ -187,9 +161,7 @@ async function validateManifest(
 
 async function readValidManifest(directory: string): Promise<GeneratedTestManifest | null> {
   try {
-    const manifest = manifestSchema.parse(
-      JSON.parse(await fs.readFile(path.join(directory, MANIFEST_FILENAME), 'utf8'))
-    );
+    const manifest = await readManifest(directory);
     const errors = await validateManifest(manifest, directory);
     return errors.length === 0 ? manifest : null;
   } catch {
@@ -197,14 +169,41 @@ async function readValidManifest(directory: string): Promise<GeneratedTestManife
   }
 }
 
+async function discoverGeneratedFiles(directory: string): Promise<string[]> {
+  const realDirectory = await fs.realpath(directory);
+  const files: string[] = [];
+
+  async function walk(currentDirectory: string): Promise<void> {
+    const entries = await fs.readdir(currentDirectory, { withFileTypes: true }).catch(() => []);
+    for (const entry of entries) {
+      if (files.length >= 1_000) return;
+      if (currentDirectory === directory && SCAFFOLD_FILENAMES.has(entry.name)) continue;
+      const candidate = path.join(currentDirectory, entry.name);
+      if (entry.isDirectory()) {
+        if (IGNORED_GENERATED_DIRECTORIES.has(entry.name)) continue;
+        await walk(candidate);
+        continue;
+      }
+      if (!entry.isFile()) continue;
+      const realPath = await fs.realpath(candidate).catch(() => undefined);
+      if (realPath && isPathInside(realDirectory, realPath)) files.push(path.resolve(candidate));
+    }
+  }
+
+  await walk(directory);
+  return files.sort((left, right) => left.localeCompare(right));
+}
+
 export const CHECK_CONCURRENCY = 20;
 
 export async function checkTestability({
   tasks,
+  runDirectory,
   signal,
   onTaskUpdate,
 }: {
   tasks: GeneratedTask[];
+  runDirectory: string;
   signal: AbortSignal;
   onTaskUpdate: (update: TaskUpdate) => void;
 }): Promise<GeneratedTask[]> {
@@ -218,6 +217,7 @@ export async function checkTestability({
   return mapWithConcurrency(folderOrdered, CHECK_CONCURRENCY, checkFileTestability, {
     signal,
     onTaskUpdate,
+    runDirectory,
   });
 }
 
@@ -238,15 +238,34 @@ export async function generateTestCode({
 }): Promise<GeneratedTask[]> {
   const generateTask = async (task: GeneratedTask): Promise<GeneratedTask> => {
     if (task.error || !task.testable) return task;
-    const emit = (phase: TaskUpdate['phase'], tokens: number) =>
-      onTaskUpdate({ id: task.id, agent: task.agent, file: task.file, phase, tokens });
+    const emit = (
+      phase: TaskUpdate['phase'],
+      tokens: number,
+      generatedFiles: string[] = [],
+      error?: string
+    ) =>
+      onTaskUpdate({
+        id: task.id,
+        agent: task.agent,
+        file: task.file,
+        directory: task.directory,
+        generatedFiles,
+        phase,
+        tokens,
+        ...(error ? { error } : {}),
+      });
     let attempts = task.attempts;
     let tokens = task.tokens;
     let lastError = 'run cancelled';
+    const attemptErrors: string[] = [];
 
     const existing = await readValidManifest(task.directory);
     if (existing) {
-      emit('generated', tokens);
+      emit(
+        'generated',
+        tokens,
+        existing.generatedFiles.map((file) => path.resolve(task.directory, file))
+      );
       return { ...task, manifest: existing };
     }
 
@@ -274,24 +293,35 @@ export async function generateTestCode({
                 signal,
               });
         tokens += reply.tokens;
-        const manifest = manifestSchema.parse(
-          JSON.parse(await fs.readFile(path.join(task.directory, MANIFEST_FILENAME), 'utf8'))
-        );
+        await appendAgentReply(task.directory, attempt, reply.result);
+        const manifest = await readManifest(task.directory);
         const errors = await validateManifest(manifest, task.directory);
-        if (errors.length > 0) throw new Error(errors.join('\n'));
+        if (errors.length > 0) {
+          const [first = '', ...rest] = errors;
+          throw new Error([`${MANIFEST_FILENAME} has problems: ${first}`, ...rest].join('\n'));
+        }
 
-        emit('generated', tokens);
+        emit(
+          'generated',
+          tokens,
+          manifest.generatedFiles.map((file) => path.resolve(task.directory, file))
+        );
         return { ...task, attempts, tokens, manifest };
       } catch (error) {
         lastError = error instanceof Error ? error.message : String(error);
+        attemptErrors.push(`Attempt ${attempt}: ${lastError}`);
       }
     }
     if (signal.aborted) {
-      emit('cancelled', tokens);
-      return { ...task, attempts, tokens, error: 'run cancelled' };
+      const generatedFiles = await discoverGeneratedFiles(task.directory);
+      emit('cancelled', tokens, generatedFiles, 'run cancelled');
+      return { ...task, attempts, tokens, generatedFiles, error: 'run cancelled' };
     }
-    emit('agent_error', tokens);
-    return { ...task, attempts, tokens, error: lastError };
+    const generatedFiles = await discoverGeneratedFiles(task.directory);
+    const detailsPath = await writeErrorDetails(task.directory, attemptErrors);
+    const error = detailsPath ? `${lastError}\nDetails: ${detailsPath}` : lastError;
+    emit('agent_error', tokens, generatedFiles, error);
+    return { ...task, attempts, tokens, generatedFiles, error };
   };
 
   return mapWithConcurrency(tasks, concurrency, generateTask);

package/src/agent-harness/index.ts

@@ -1,17 +1,21 @@
 import fs from 'node:fs/promises';
 
+import { ensureAgentSdks } from './agentPreflight.js';
 import { discoverCodeBlocks } from './discoverCodeBlocks.js';
 import { executeCodeBlocks } from './executeCodeBlocks.js';
 import { filterSelectedPages } from './filterSelectedPages.js';
 import { checkTestability, generateTestCode } from './generateTestCode.js';
+import { recordTestRun, testRunReportPath } from './runHistory.js';
 import { clearRunSave, writeRunSave } from './runSave.js';
-import { setupFolders } from './setupFolders.js';
+import { scaffoldTaskDirectories, setupFolders } from './setupFolders.js';
 import type { CodeTestReport, RunCodeTestsOptions } from './types.js';
 
 export const DEFAULT_CONCURRENCY = 5;
 export const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60 * 1_000;
 
 export { discoverCodeBlocks } from './discoverCodeBlocks.js';
+export { countReport } from './countReport.js';
+export { loadLatestTestRun, loadTestRuns } from './runHistory.js';
 export { clearRunSave, loadRunSave } from './runSave.js';
 export type {
   CodeTestReport,
@@ -26,6 +30,8 @@ export type {
   TaskResult,
   TaskStatus,
   TaskUpdate,
+  TestRunSummary,
+  TestRunsIndex,
 } from './types.js';
 
 export async function runCodeTests({
@@ -45,14 +51,21 @@ export async function runCodeTests({
 
   const { docsRoot, pages: allPages } = await discoverCodeBlocks(docsPath);
   const pages = filterSelectedPages(allPages, selectedFiles);
+  if (pages.length > 0) await ensureAgentSdks(uniqueAgents);
 
-  const { runId, runDirectory, reportPath, tasks } = await setupFolders({
+  const {
+    runId,
+    runDirectory,
+    reportPath: legacyReportPath,
+    tasks,
+  } = await setupFolders({
     pages,
     agents: uniqueAgents,
     docsRoot,
     outputDirectory,
     runId: savedRunId,
   });
+  const reportPath = testRunReportPath(docsRoot, runId);
   await writeRunSave(outputDirectory, {
     version: 1,
     runId,
@@ -62,7 +75,8 @@ export async function runCodeTests({
     createdAt: startedAt.toISOString(),
   });
 
-  const checked = await checkTestability({ tasks, signal, onTaskUpdate });
+  const checked = await checkTestability({ tasks, runDirectory, signal, onTaskUpdate });
+  await scaffoldTaskDirectories(checked.filter((task) => task.testable === true));
 
   const generated = await generateTestCode({
     tasks: checked,
@@ -97,13 +111,14 @@ export async function runCodeTests({
     outputDirectory: runDirectory,
     reportPath,
     agents: uniqueAgents,
+    model,
     selectedFiles: pages.map((page) => page.file),
     startedAt: startedAt.toISOString(),
     completedAt: completedAt.toISOString(),
     durationMs: completedAt.getTime() - startedAt.getTime(),
     tasks: results,
   };
-  await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+  await fs.writeFile(legacyReportPath, `${JSON.stringify(report, null, 2)}\n`);
   if (!signal.aborted) await clearRunSave(outputDirectory);
-  return report;
+  return recordTestRun(docsRoot, report, legacyReportPath);
 }

package/src/agent-harness/manifest.ts

@@ -0,0 +1,116 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { z } from 'zod';
+
+import { manifestSchema, type GeneratedTestManifest } from './types.js';
+
+export const MANIFEST_FILENAME = 'mint-test.json';
+export const AGENT_REPLY_FILENAME = 'mint-test-agent.md';
+export const ERROR_FILENAME = 'mint-test-error.txt';
+
+export const SAFE_EXECUTABLES = new Set([
+  'bun',
+  'bunx',
+  'bundle',
+  'cargo',
+  'composer',
+  'deno',
+  'dotnet',
+  'go',
+  'gradle',
+  'java',
+  'javac',
+  'mvn',
+  'node',
+  'npm',
+  'npx',
+  'php',
+  'pip',
+  'pip3',
+  'pnpm',
+  'python',
+  'python3',
+  'pytest',
+  'ruby',
+  'rustc',
+  'uv',
+  'vitest',
+  'yarn',
+]);
+
+function isMissingFile(error: unknown): boolean {
+  return (
+    typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'ENOENT'
+  );
+}
+
+/** Turn a schema failure into one readable line per problem, e.g. `testCommands.0.id: expected string`. */
+export function formatManifestIssues(error: z.ZodError): string {
+  return error.issues
+    .map((issue) => {
+      const location = issue.path.map(String).join('.');
+      return `${location || MANIFEST_FILENAME}: ${issue.message}`;
+    })
+    .join('\n');
+}
+
+/**
+ * Read and schema-check the manifest an agent wrote. Every failure mode has a message a
+ * documentation author can act on, because it is what `mint test` shows for the page.
+ */
+export async function readManifest(directory: string): Promise<GeneratedTestManifest> {
+  const manifestPath = path.join(directory, MANIFEST_FILENAME);
+  let raw: string;
+  try {
+    raw = await fs.readFile(manifestPath, 'utf8');
+  } catch (error) {
+    if (isMissingFile(error)) {
+      throw new Error(`The agent finished without writing ${MANIFEST_FILENAME}`);
+    }
+    throw error;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    throw new Error(`${MANIFEST_FILENAME} is not valid JSON: ${reason}`);
+  }
+
+  const result = manifestSchema.safeParse(parsed);
+  if (!result.success) {
+    const [first = '', ...rest] = formatManifestIssues(result.error).split('\n');
+    throw new Error(
+      [`${MANIFEST_FILENAME} does not match the expected shape: ${first}`, ...rest].join('\n')
+    );
+  }
+  return result.data;
+}
+
+/** Keep the agent's final message next to its output so a failure can be traced later. */
+export async function appendAgentReply(
+  directory: string,
+  attempt: number,
+  reply: string
+): Promise<void> {
+  const body = reply.trim() || '(no summary)';
+  await fs
+    .appendFile(path.join(directory, AGENT_REPLY_FILENAME), `## Attempt ${attempt}\n\n${body}\n\n`)
+    .catch(() => {});
+}
+
+/** Persist every attempt's failure reason. Returns the file path, or undefined if it could not be written. */
+export async function writeErrorDetails(
+  directory: string,
+  errors: string[]
+): Promise<string | undefined> {
+  if (errors.length === 0) return undefined;
+  const errorPath = path.join(directory, ERROR_FILENAME);
+  try {
+    await fs.writeFile(errorPath, `${errors.join('\n\n')}\n`);
+    return errorPath;
+  } catch {
+    return undefined;
+  }
+}