@mintlify/cli 4.0.1465 → 4.0.1467

package/src/agent-harness/manifest.ts

@@ -0,0 +1,116 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { z } from 'zod';
+
+import { manifestSchema, type GeneratedTestManifest } from './types.js';
+
+export const MANIFEST_FILENAME = 'mint-test.json';
+export const AGENT_REPLY_FILENAME = 'mint-test-agent.md';
+export const ERROR_FILENAME = 'mint-test-error.txt';
+
+export const SAFE_EXECUTABLES = new Set([
+  'bun',
+  'bunx',
+  'bundle',
+  'cargo',
+  'composer',
+  'deno',
+  'dotnet',
+  'go',
+  'gradle',
+  'java',
+  'javac',
+  'mvn',
+  'node',
+  'npm',
+  'npx',
+  'php',
+  'pip',
+  'pip3',
+  'pnpm',
+  'python',
+  'python3',
+  'pytest',
+  'ruby',
+  'rustc',
+  'uv',
+  'vitest',
+  'yarn',
+]);
+
+function isMissingFile(error: unknown): boolean {
+  return (
+    typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'ENOENT'
+  );
+}
+
+/** Turn a schema failure into one readable line per problem, e.g. `testCommands.0.id: expected string`. */
+export function formatManifestIssues(error: z.ZodError): string {
+  return error.issues
+    .map((issue) => {
+      const location = issue.path.map(String).join('.');
+      return `${location || MANIFEST_FILENAME}: ${issue.message}`;
+    })
+    .join('\n');
+}
+
+/**
+ * Read and schema-check the manifest an agent wrote. Every failure mode has a message a
+ * documentation author can act on, because it is what `mint test` shows for the page.
+ */
+export async function readManifest(directory: string): Promise<GeneratedTestManifest> {
+  const manifestPath = path.join(directory, MANIFEST_FILENAME);
+  let raw: string;
+  try {
+    raw = await fs.readFile(manifestPath, 'utf8');
+  } catch (error) {
+    if (isMissingFile(error)) {
+      throw new Error(`The agent finished without writing ${MANIFEST_FILENAME}`);
+    }
+    throw error;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    throw new Error(`${MANIFEST_FILENAME} is not valid JSON: ${reason}`);
+  }
+
+  const result = manifestSchema.safeParse(parsed);
+  if (!result.success) {
+    const [first = '', ...rest] = formatManifestIssues(result.error).split('\n');
+    throw new Error(
+      [`${MANIFEST_FILENAME} does not match the expected shape: ${first}`, ...rest].join('\n')
+    );
+  }
+  return result.data;
+}
+
+/** Keep the agent's final message next to its output so a failure can be traced later. */
+export async function appendAgentReply(
+  directory: string,
+  attempt: number,
+  reply: string
+): Promise<void> {
+  const body = reply.trim() || '(no summary)';
+  await fs
+    .appendFile(path.join(directory, AGENT_REPLY_FILENAME), `## Attempt ${attempt}\n\n${body}\n\n`)
+    .catch(() => {});
+}
+
+/** Persist every attempt's failure reason. Returns the file path, or undefined if it could not be written. */
+export async function writeErrorDetails(
+  directory: string,
+  errors: string[]
+): Promise<string | undefined> {
+  if (errors.length === 0) return undefined;
+  const errorPath = path.join(directory, ERROR_FILENAME);
+  try {
+    await fs.writeFile(errorPath, `${errors.join('\n\n')}\n`);
+    return errorPath;
+  } catch {
+    return undefined;
+  }
+}

package/src/agent-harness/runHistory.ts

@@ -0,0 +1,250 @@
+import { randomUUID } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { z } from 'zod';
+
+import { countReport } from './countReport.js';
+import type { CodeTestReport, TestRunSummary, TestRunsIndex } from './types.js';
+
+const RUN_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
+const INDEX_LOCK_FILENAME = 'testRuns.lock';
+const LOCK_RETRY_MS = 20;
+const STALE_LOCK_MS = 30_000;
+const LOCK_TIMEOUT_MS = STALE_LOCK_MS + 10_000;
+const commandResultSchema = z
+  .object({
+    id: z.string(),
+    phase: z.enum(['setup', 'test']),
+    status: z.enum(['passed', 'failed', 'cancelled']),
+    executable: z.string(),
+    args: z.array(z.string()),
+    cwd: z.string(),
+    durationMs: z.number().nonnegative(),
+    exitCode: z.number().int().nullable(),
+    stdout: z.string(),
+    stderr: z.string(),
+    error: z.string().optional(),
+  })
+  .strict();
+const taskResultSchema = z
+  .object({
+    id: z.string(),
+    agent: z.enum(['claude', 'codex']),
+    file: z.string(),
+    directory: z.string(),
+    generatedFiles: z.array(z.string()),
+    testable: z.boolean(),
+    attempts: z.number().int().nonnegative(),
+    tokens: z.number().nonnegative(),
+    status: z.enum(['passed', 'failed', 'agent_error', 'cancelled']),
+    error: z.string().optional(),
+    commands: z.array(commandResultSchema),
+  })
+  .strict();
+const codeTestReportSchema = z
+  .object({
+    version: z.literal(1),
+    runId: z.string().regex(RUN_ID_PATTERN),
+    status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
+    docsRoot: z.string(),
+    outputDirectory: z.string(),
+    reportPath: z.string(),
+    agents: z.array(z.enum(['claude', 'codex'])).min(1),
+    model: z.string().min(1),
+    selectedFiles: z.array(z.string()),
+    startedAt: z.string(),
+    completedAt: z.string(),
+    durationMs: z.number().nonnegative(),
+    tasks: z.array(taskResultSchema),
+    historyError: z.string().optional(),
+  })
+  .strict();
+const countsSchema = z
+  .object({
+    passed: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    cancelled: z.number().int().nonnegative(),
+    agentErrors: z.number().int().nonnegative(),
+  })
+  .strict();
+const testRunSummarySchema = z
+  .object({
+    runId: z.string().regex(RUN_ID_PATTERN),
+    status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
+    agents: z.array(z.enum(['claude', 'codex'])).min(1),
+    model: z.string().min(1),
+    selectedFiles: z.array(z.string()),
+    startedAt: z.string(),
+    completedAt: z.string(),
+    durationMs: z.number().nonnegative(),
+    reportPath: z.string(),
+    outputDirectory: z.string(),
+    counts: countsSchema,
+  })
+  .strict();
+const testRunsIndexSchema = z
+  .object({ version: z.literal(1), runs: z.array(testRunSummarySchema) })
+  .strict();
+
+function historyDirectory(docsRoot: string): string {
+  return path.join(docsRoot, '.mintlify', 'test');
+}
+
+function runsDirectory(docsRoot: string): string {
+  return path.join(historyDirectory(docsRoot), 'runs');
+}
+
+export function testRunReportPath(docsRoot: string, runId: string): string {
+  if (!RUN_ID_PATTERN.test(runId)) throw new Error(`Invalid mint test run ID: ${runId}`);
+  return path.join(runsDirectory(docsRoot), `${runId}.json`);
+}
+
+export function testRunsIndexPath(docsRoot: string): string {
+  return path.join(historyDirectory(docsRoot), 'testRuns.json');
+}
+
+function summarizeReport(report: CodeTestReport): TestRunSummary {
+  return {
+    runId: report.runId,
+    status: report.status,
+    agents: report.agents,
+    model: report.model,
+    selectedFiles: report.selectedFiles,
+    startedAt: report.startedAt,
+    completedAt: report.completedAt,
+    durationMs: report.durationMs,
+    reportPath: report.reportPath,
+    outputDirectory: report.outputDirectory,
+    counts: countReport(report.tasks),
+  };
+}
+
+function errorCode(error: unknown): string | undefined {
+  return typeof error === 'object' && error !== null && 'code' in error
+    ? String(error.code)
+    : undefined;
+}
+
+async function writeJsonAtomically(filePath: string, value: unknown): Promise<void> {
+  const temporaryPath = path.join(
+    path.dirname(filePath),
+    `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`
+  );
+  try {
+    await fs.writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx' });
+    await fs.rename(temporaryPath, filePath);
+  } finally {
+    await fs.unlink(temporaryPath).catch(() => undefined);
+  }
+}
+
+async function acquireIndexLock(docsRoot: string): Promise<() => Promise<void>> {
+  const lockPath = path.join(historyDirectory(docsRoot), INDEX_LOCK_FILENAME);
+  const token = `${process.pid}:${randomUUID()}`;
+  const deadline = Date.now() + LOCK_TIMEOUT_MS;
+
+  while (Date.now() < deadline) {
+    try {
+      const lock = await fs.open(lockPath, 'wx');
+      try {
+        await lock.writeFile(token);
+      } finally {
+        await lock.close();
+      }
+      return async () => {
+        const owner = await fs.readFile(lockPath, 'utf8').catch(() => undefined);
+        if (owner === token) await fs.unlink(lockPath).catch(() => undefined);
+      };
+    } catch (error) {
+      if (errorCode(error) !== 'EEXIST') throw error;
+    }
+
+    const stats = await fs.stat(lockPath).catch(() => undefined);
+    if (stats && Date.now() - stats.mtimeMs > STALE_LOCK_MS) {
+      await fs.unlink(lockPath).catch(() => undefined);
+      continue;
+    }
+    await delay(LOCK_RETRY_MS);
+  }
+  throw new Error('Timed out updating mint test run history');
+}
+
+async function readReport(filePath: string): Promise<CodeTestReport | null> {
+  try {
+    return codeTestReportSchema.parse(JSON.parse(await fs.readFile(filePath, 'utf8')));
+  } catch {
+    return null;
+  }
+}
+
+async function rebuildIndex(docsRoot: string): Promise<TestRunsIndex> {
+  const entries = await fs
+    .readdir(runsDirectory(docsRoot), { withFileTypes: true })
+    .catch(() => []);
+  const reports = await Promise.all(
+    entries
+      .filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
+      .map((entry) => readReport(path.join(runsDirectory(docsRoot), entry.name)))
+  );
+  return {
+    version: 1,
+    runs: reports
+      .filter((report): report is CodeTestReport => report !== null)
+      .map(summarizeReport)
+      .sort((left, right) => right.completedAt.localeCompare(left.completedAt)),
+  };
+}
+
+export async function loadTestRuns(docsRoot: string): Promise<TestRunsIndex> {
+  try {
+    return testRunsIndexSchema.parse(
+      JSON.parse(await fs.readFile(testRunsIndexPath(docsRoot), 'utf8'))
+    );
+  } catch {
+    return rebuildIndex(docsRoot);
+  }
+}
+
+export async function loadLatestTestRun(docsRoot: string): Promise<CodeTestReport | null> {
+  const index = await loadTestRuns(docsRoot);
+  for (const run of index.runs) {
+    const report = await readReport(testRunReportPath(docsRoot, run.runId));
+    if (report) return report;
+  }
+  return null;
+}
+
+export async function writeTestRunHistory(docsRoot: string, report: CodeTestReport): Promise<void> {
+  await fs.mkdir(runsDirectory(docsRoot), { recursive: true });
+  await writeJsonAtomically(report.reportPath, report);
+  const releaseLock = await acquireIndexLock(docsRoot);
+  try {
+    await writeJsonAtomically(testRunsIndexPath(docsRoot), await rebuildIndex(docsRoot));
+  } finally {
+    await releaseLock();
+  }
+}
+
+/**
+ * Saves a finished run to history without letting a history failure fail the run.
+ * On failure the returned report points at `fallbackReportPath` (the report that was
+ * already written) and carries the problem in `historyError`.
+ */
+export async function recordTestRun(
+  docsRoot: string,
+  report: CodeTestReport,
+  fallbackReportPath: string
+): Promise<CodeTestReport> {
+  try {
+    await writeTestRunHistory(docsRoot, report);
+    return report;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      ...report,
+      reportPath: fallbackReportPath,
+      historyError: `run history could not be saved: ${message}`,
+    };
+  }
+}

package/src/agent-harness/setupFolders.ts

@@ -9,17 +9,32 @@ const PACKAGE_JSON = `${JSON.stringify(
     name: 'mint-test-task',
     private: true,
     type: 'module',
-    scripts: { test: 'node --test tests/' },
+    // No positional path: Node 21+ treats --test arguments as globs, so a bare
+    // directory such as `tests/` fails with MODULE_NOT_FOUND. The default patterns
+    // pick up tests/*.test.js on every supported Node version.
+    scripts: { test: 'node --test' },
   },
   null,
   2
 )}\n`;
 
+function isExistingFile(error: unknown): boolean {
+  return (
+    typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'EEXIST'
+  );
+}
+
 async function scaffoldTaskDirectory(directory: string): Promise<void> {
   await fs.mkdir(path.join(directory, 'tests'), { recursive: true });
-  await fs
-    .writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' })
-    .catch(() => {});
+  try {
+    await fs.writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' });
+  } catch (error) {
+    if (!isExistingFile(error)) throw error;
+  }
+}
+
+export async function scaffoldTaskDirectories(tasks: GeneratedTask[]): Promise<void> {
+  for (const task of tasks) await scaffoldTaskDirectory(task.directory);
 }
 
 export async function setupFolders({
@@ -68,6 +83,5 @@ export async function setupFolders({
       });
     }
   }
-  for (const task of tasks) await scaffoldTaskDirectory(task.directory);
   return { runId, runDirectory, reportPath, tasks };
 }

package/src/agent-harness/tasks/checkTestability.ts

@@ -50,17 +50,16 @@ async function runTestabilityCheck({
   file,
   content,
   agent,
-  outputPath,
+  workingDirectory,
   signal,
 }: {
   file: string;
   content: string;
   agent: HarnessAgent;
-  outputPath: string;
+  workingDirectory: string;
   signal: AbortSignal;
 }): Promise<{ tokens: number; check: TestCheck }> {
   const prompt = buildCheckPrompt(file, content);
-  const workingDirectory = path.dirname(outputPath);
   let tokens = 0;
   let raw: unknown;
   if (agent === 'claude') {
@@ -81,19 +80,24 @@ async function runTestabilityCheck({
   const check = testCheckSchema.parse(raw);
   const errors = validateCheck(check);
   if (errors.length > 0) throw new Error(errors.join('\n'));
-  await fs.writeFile(outputPath, `${JSON.stringify(check, null, 2)}\n`);
   return { tokens, check };
 }
 
 export const checkFileTestability = async (
   task: GeneratedTask,
-  { onTaskUpdate, signal }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'>
+  {
+    onTaskUpdate,
+    signal,
+    runDirectory,
+  }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'> & { runDirectory: string }
 ): Promise<GeneratedTask> => {
   const emit = (phase: TaskUpdate['phase'], tokens: number) =>
     onTaskUpdate({
       id: task.id,
       agent: task.agent,
       file: task.file,
+      directory: task.directory,
+      generatedFiles: [],
       phase,
       tokens,
     });
@@ -112,7 +116,7 @@ export const checkFileTestability = async (
           file: task.file,
           content,
           agent: task.agent,
-          outputPath: path.join(task.directory, CHECK_FILENAME),
+          workingDirectory: runDirectory,
           signal,
         });
         tokens += reply.tokens;
@@ -135,6 +139,11 @@ export const checkFileTestability = async (
     emit('not_testable', tokens);
     return { ...task, attempts, tokens, testable: false };
   }
+  await fs.mkdir(task.directory, { recursive: true });
+  await fs.writeFile(
+    path.join(task.directory, CHECK_FILENAME),
+    `${JSON.stringify(check, null, 2)}\n`
+  );
   emit('checked', tokens);
   return { ...task, attempts, tokens, testable: true };
 };

package/src/agent-harness/types.ts

@@ -30,6 +30,8 @@ export interface TaskResult {
   agent: HarnessAgent;
   file: string;
   directory: string;
+  generatedFiles: string[];
+  testable: boolean;
   attempts: number;
   tokens: number;
   status: TaskStatus;
@@ -45,11 +47,38 @@ export interface CodeTestReport {
   outputDirectory: string;
   reportPath: string;
   agents: HarnessAgent[];
+  model: string;
   selectedFiles: string[];
   startedAt: string;
   completedAt: string;
   durationMs: number;
   tasks: TaskResult[];
+  /** Set when the run finished but could not be saved to `.mintlify/test` history. */
+  historyError?: string;
+}
+
+export interface TestRunSummary {
+  runId: string;
+  status: CodeTestReport['status'];
+  agents: HarnessAgent[];
+  model: string;
+  selectedFiles: string[];
+  startedAt: string;
+  completedAt: string;
+  durationMs: number;
+  reportPath: string;
+  outputDirectory: string;
+  counts: {
+    passed: number;
+    failed: number;
+    cancelled: number;
+    agentErrors: number;
+  };
+}
+
+export interface TestRunsIndex {
+  version: 1;
+  runs: TestRunSummary[];
 }
 
 export type TaskPhase =
@@ -68,8 +97,12 @@ export interface TaskUpdate {
   id: string;
   agent: HarnessAgent;
   file: string;
+  directory: string;
+  generatedFiles: string[];
   phase: TaskPhase;
   tokens: number;
+  /** Why the task ended in `agent_error` or `failed`; the first line is shown in the UI. */
+  error?: string;
 }
 
 export interface RunSave {
@@ -98,7 +131,7 @@ const identifier = z.string().trim().min(1).max(120);
 
 const commandSchema = z
   .object({
-    id: identifier,
+    id: identifier.optional(),
     executable: z.string().trim().min(1).max(512),
     args: z.array(z.string().max(10_000)).max(100),
     cwd: z.string().max(1_024).optional(),
@@ -111,7 +144,7 @@ const commandSchema = z
   })
   .strict();
 
-export const manifestSchema = z
+const manifestFileSchema = z
   .object({
     version: z.literal(1),
     generatedFiles: z.array(z.string().min(1).max(1_024)).max(1_000),
@@ -120,7 +153,51 @@ export const manifestSchema = z
   })
   .strict();
 
-export type GeneratedTestManifest = z.infer<typeof manifestSchema>;
+type ManifestFile = z.infer<typeof manifestFileSchema>;
+type ManifestFileCommand = ManifestFile['testCommands'][number];
+
+export type ManifestCommand = Omit<ManifestFileCommand, 'id'> & { id: string };
+
+export interface GeneratedTestManifest {
+  version: 1;
+  generatedFiles: string[];
+  setupCommands: ManifestCommand[];
+  testCommands: ManifestCommand[];
+}
+
+/**
+ * Agents often copy the manifest example verbatim and omit command ids. The id only needs to be
+ * unique within the manifest, so assign `setup-N` / `test-N` instead of rejecting the whole page.
+ */
+export function assignCommandIds(manifest: ManifestFile): GeneratedTestManifest {
+  const taken = new Set(
+    [...manifest.setupCommands, ...manifest.testCommands]
+      .map((command) => command.id)
+      .filter((id): id is string => id !== undefined)
+  );
+  const withIds = (commands: ManifestFileCommand[], phase: 'setup' | 'test'): ManifestCommand[] => {
+    let next = 1;
+    return commands.map((command) => {
+      if (command.id !== undefined) return { ...command, id: command.id };
+      let id = `${phase}-${next}`;
+      while (taken.has(id)) {
+        next += 1;
+        id = `${phase}-${next}`;
+      }
+      taken.add(id);
+      next += 1;
+      return { ...command, id };
+    });
+  };
+  return {
+    version: manifest.version,
+    generatedFiles: manifest.generatedFiles,
+    setupCommands: withIds(manifest.setupCommands, 'setup'),
+    testCommands: withIds(manifest.testCommands, 'test'),
+  };
+}
+
+export const manifestSchema = manifestFileSchema.transform(assignCommandIds);
 
 export const testCheckSchema = z
   .object({
@@ -136,8 +213,6 @@ export const testCheckSchema = z
 
 export type TestCheck = z.infer<typeof testCheckSchema>;
 
-export type ManifestCommand = GeneratedTestManifest['testCommands'][number];
-
 export interface GeneratedTask {
   id: string;
   agent: HarnessAgent;
@@ -148,5 +223,6 @@ export interface GeneratedTask {
   tokens: number;
   testable?: boolean;
   manifest?: GeneratedTestManifest;
+  generatedFiles?: string[];
   error?: string;
 }

package/src/constants.ts

@@ -29,7 +29,11 @@ const PROD_TOKEN_ENDPOINT =
   'https://api.stytch.com/v1/public/project-live-731b7a04-9ac3-4923-90b8-0806d4aa29d4/oauth2/token';
 const PROD_STYTCH_CLIENT_ID = 'connected-app-live-d813eedd-dbb0-434b-a1f9-2ce69e5efc49';
 
-export const TOKEN_ENDPOINT = IS_LOCAL_BUILD ? DEV_TOKEN_ENDPOINT : PROD_TOKEN_ENDPOINT;
-export const STYTCH_CLIENT_ID = IS_LOCAL_BUILD ? DEV_STYTCH_CLIENT_ID : PROD_STYTCH_CLIENT_ID;
+export const TOKEN_ENDPOINT =
+  process.env.MINTLIFY_TOKEN_ENDPOINT ??
+  (IS_LOCAL_BUILD ? DEV_TOKEN_ENDPOINT : PROD_TOKEN_ENDPOINT);
+export const STYTCH_CLIENT_ID =
+  process.env.MINTLIFY_STYTCH_CLIENT_ID ??
+  (IS_LOCAL_BUILD ? DEV_STYTCH_CLIENT_ID : PROD_STYTCH_CLIENT_ID);
 
 export const CUSTOM_DOMAIN_CNAME_TARGET = 'cname.mintlify.builders';