@mintlify/cli 4.0.1654 → 4.0.1656

package/src/agent-harness/runHistory.ts

@@ -1,255 +0,0 @@
-import { randomUUID } from 'node:crypto';
-import fs from 'node:fs/promises';
-import type { FileHandle } from 'node:fs/promises';
-import path from 'node:path';
-import { setTimeout as delay } from 'node:timers/promises';
-import { z } from 'zod';
-
-import { countReport } from './countReport.js';
-import type { CodeTestReport, TestRunSummary, TestRunsIndex } from './types.js';
-
-const RUN_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
-const INDEX_LOCK_FILENAME = 'testRuns.lock';
-const LOCK_RETRY_MS = 20;
-const STALE_LOCK_MS = 30_000;
-const LOCK_TIMEOUT_MS = STALE_LOCK_MS + 10_000;
-const commandResultSchema = z
-  .object({
-    id: z.string(),
-    phase: z.enum(['setup', 'test']),
-    status: z.enum(['passed', 'failed', 'cancelled']),
-    executable: z.string(),
-    args: z.array(z.string()),
-    cwd: z.string(),
-    durationMs: z.number().nonnegative(),
-    exitCode: z.number().int().nullable(),
-    stdout: z.string(),
-    stderr: z.string(),
-    error: z.string().optional(),
-  })
-  .strict();
-const taskResultSchema = z
-  .object({
-    id: z.string(),
-    agent: z.enum(['claude', 'codex']),
-    file: z.string(),
-    directory: z.string(),
-    generatedFiles: z.array(z.string()),
-    testable: z.boolean(),
-    attempts: z.number().int().nonnegative(),
-    tokens: z.number().nonnegative(),
-    status: z.enum(['passed', 'failed', 'agent_error', 'cancelled']),
-    error: z.string().optional(),
-    commands: z.array(commandResultSchema),
-  })
-  .strict();
-const codeTestReportSchema = z
-  .object({
-    version: z.literal(1),
-    runId: z.string().regex(RUN_ID_PATTERN),
-    status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
-    docsRoot: z.string(),
-    outputDirectory: z.string(),
-    reportPath: z.string(),
-    agents: z.array(z.enum(['claude', 'codex'])).min(1),
-    model: z.string().min(1),
-    selectedFiles: z.array(z.string()),
-    startedAt: z.string(),
-    completedAt: z.string(),
-    durationMs: z.number().nonnegative(),
-    tasks: z.array(taskResultSchema),
-    historyError: z.string().optional(),
-  })
-  .strict();
-const countsSchema = z
-  .object({
-    passed: z.number().int().nonnegative(),
-    failed: z.number().int().nonnegative(),
-    cancelled: z.number().int().nonnegative(),
-    agentErrors: z.number().int().nonnegative(),
-  })
-  .strict();
-const testRunSummarySchema = z
-  .object({
-    runId: z.string().regex(RUN_ID_PATTERN),
-    status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
-    agents: z.array(z.enum(['claude', 'codex'])).min(1),
-    model: z.string().min(1),
-    selectedFiles: z.array(z.string()),
-    startedAt: z.string(),
-    completedAt: z.string(),
-    durationMs: z.number().nonnegative(),
-    reportPath: z.string(),
-    outputDirectory: z.string(),
-    counts: countsSchema,
-  })
-  .strict();
-const testRunsIndexSchema = z
-  .object({ version: z.literal(1), runs: z.array(testRunSummarySchema) })
-  .strict();
-
-function historyDirectory(docsRoot: string): string {
-  return path.join(docsRoot, '.mintlify', 'test');
-}
-
-function runsDirectory(docsRoot: string): string {
-  return path.join(historyDirectory(docsRoot), 'runs');
-}
-
-export function testRunReportPath(docsRoot: string, runId: string): string {
-  if (!RUN_ID_PATTERN.test(runId)) throw new Error(`Invalid mint test run ID: ${runId}`);
-  return path.join(runsDirectory(docsRoot), `${runId}.json`);
-}
-
-export function testRunsIndexPath(docsRoot: string): string {
-  return path.join(historyDirectory(docsRoot), 'testRuns.json');
-}
-
-function summarizeReport(report: CodeTestReport): TestRunSummary {
-  return {
-    runId: report.runId,
-    status: report.status,
-    agents: report.agents,
-    model: report.model,
-    selectedFiles: report.selectedFiles,
-    startedAt: report.startedAt,
-    completedAt: report.completedAt,
-    durationMs: report.durationMs,
-    reportPath: report.reportPath,
-    outputDirectory: report.outputDirectory,
-    counts: countReport(report.tasks),
-  };
-}
-
-function errorCode(error: unknown): string | undefined {
-  return typeof error === 'object' && error !== null && 'code' in error
-    ? String(error.code)
-    : undefined;
-}
-
-async function writeJsonAtomically(filePath: string, value: unknown): Promise<void> {
-  const temporaryPath = path.join(
-    path.dirname(filePath),
-    `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`
-  );
-  try {
-    await fs.writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx' });
-    await fs.rename(temporaryPath, filePath);
-  } finally {
-    await fs.unlink(temporaryPath).catch(() => undefined);
-  }
-}
-
-async function acquireIndexLock(docsRoot: string): Promise<() => Promise<void>> {
-  const lockPath = path.join(historyDirectory(docsRoot), INDEX_LOCK_FILENAME);
-  const token = `${process.pid}:${randomUUID()}`;
-  const deadline = Date.now() + LOCK_TIMEOUT_MS;
-
-  while (Date.now() < deadline) {
-    let lock: FileHandle;
-    try {
-      lock = await fs.open(lockPath, 'wx');
-    } catch (error) {
-      const code = errorCode(error);
-      // Windows can deny opening a lock while another writer's deletion is still pending.
-      if (code !== 'EEXIST' && !(process.platform === 'win32' && code === 'EPERM')) throw error;
-
-      const stats = await fs.stat(lockPath).catch(() => undefined);
-      if (stats && Date.now() - stats.mtimeMs > STALE_LOCK_MS) {
-        await fs.unlink(lockPath).catch(() => undefined);
-        continue;
-      }
-      await delay(LOCK_RETRY_MS);
-      continue;
-    }
-    try {
-      await lock.writeFile(token);
-    } finally {
-      await lock.close();
-    }
-    return async () => {
-      const owner = await fs.readFile(lockPath, 'utf8').catch(() => undefined);
-      if (owner === token) await fs.unlink(lockPath).catch(() => undefined);
-    };
-  }
-  throw new Error('Timed out updating mint test run history');
-}
-
-async function readReport(filePath: string): Promise<CodeTestReport | null> {
-  try {
-    return codeTestReportSchema.parse(JSON.parse(await fs.readFile(filePath, 'utf8')));
-  } catch {
-    return null;
-  }
-}
-
-async function rebuildIndex(docsRoot: string): Promise<TestRunsIndex> {
-  const entries = await fs
-    .readdir(runsDirectory(docsRoot), { withFileTypes: true })
-    .catch(() => []);
-  const reports = await Promise.all(
-    entries
-      .filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
-      .map((entry) => readReport(path.join(runsDirectory(docsRoot), entry.name)))
-  );
-  return {
-    version: 1,
-    runs: reports
-      .filter((report): report is CodeTestReport => report !== null)
-      .map(summarizeReport)
-      .sort((left, right) => right.completedAt.localeCompare(left.completedAt)),
-  };
-}
-
-export async function loadTestRuns(docsRoot: string): Promise<TestRunsIndex> {
-  try {
-    return testRunsIndexSchema.parse(
-      JSON.parse(await fs.readFile(testRunsIndexPath(docsRoot), 'utf8'))
-    );
-  } catch {
-    return rebuildIndex(docsRoot);
-  }
-}
-
-export async function loadLatestTestRun(docsRoot: string): Promise<CodeTestReport | null> {
-  const index = await loadTestRuns(docsRoot);
-  for (const run of index.runs) {
-    const report = await readReport(testRunReportPath(docsRoot, run.runId));
-    if (report) return report;
-  }
-  return null;
-}
-
-export async function writeTestRunHistory(docsRoot: string, report: CodeTestReport): Promise<void> {
-  await fs.mkdir(runsDirectory(docsRoot), { recursive: true });
-  await writeJsonAtomically(report.reportPath, report);
-  const releaseLock = await acquireIndexLock(docsRoot);
-  try {
-    await writeJsonAtomically(testRunsIndexPath(docsRoot), await rebuildIndex(docsRoot));
-  } finally {
-    await releaseLock();
-  }
-}
-
-/**
- * Saves a finished run to history without letting a history failure fail the run.
- * On failure the returned report points at `fallbackReportPath` (the report that was
- * already written) and carries the problem in `historyError`.
- */
-export async function recordTestRun(
-  docsRoot: string,
-  report: CodeTestReport,
-  fallbackReportPath: string
-): Promise<CodeTestReport> {
-  try {
-    await writeTestRunHistory(docsRoot, report);
-    return report;
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return {
-      ...report,
-      reportPath: fallbackReportPath,
-      historyError: `run history could not be saved: ${message}`,
-    };
-  }
-}

package/src/agent-harness/runSave.ts

@@ -1,38 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { z } from 'zod';
-
-import type { RunSave } from './types.js';
-
-const runSaveSchema = z
-  .object({
-    version: z.literal(1),
-    runId: z.string().regex(/^[A-Za-z0-9._-]+$/),
-    agents: z.array(z.enum(['claude', 'codex'])).min(1),
-    model: z.string().min(1),
-    selectedFiles: z.array(z.string().min(1)),
-    createdAt: z.string(),
-  })
-  .strict();
-
-function savePath(outputDirectory: string): string {
-  return path.join(outputDirectory, 'save.json');
-}
-
-export async function loadRunSave(outputDirectory: string): Promise<RunSave | null> {
-  try {
-    const content = await fs.readFile(savePath(outputDirectory), 'utf8');
-    return runSaveSchema.parse(JSON.parse(content));
-  } catch {
-    return null;
-  }
-}
-
-export async function writeRunSave(outputDirectory: string, save: RunSave): Promise<void> {
-  await fs.mkdir(outputDirectory, { recursive: true });
-  await fs.writeFile(savePath(outputDirectory), `${JSON.stringify(save, null, 2)}\n`);
-}
-
-export async function clearRunSave(outputDirectory: string): Promise<void> {
-  await fs.unlink(savePath(outputDirectory)).catch(() => undefined);
-}

package/src/agent-harness/setupFolders.ts

@@ -1,87 +0,0 @@
-import { randomBytes } from 'node:crypto';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-
-import type { DocsPage, GeneratedTask, HarnessAgent } from './types.js';
-
-const PACKAGE_JSON = `${JSON.stringify(
-  {
-    name: 'mint-test-task',
-    private: true,
-    type: 'module',
-    // No positional path: Node 21+ treats --test arguments as globs, so a bare
-    // directory such as `tests/` fails with MODULE_NOT_FOUND. The default patterns
-    // pick up tests/*.test.js on every supported Node version.
-    scripts: { test: 'node --test' },
-  },
-  null,
-  2
-)}\n`;
-
-function isExistingFile(error: unknown): boolean {
-  return (
-    typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'EEXIST'
-  );
-}
-
-async function scaffoldTaskDirectory(directory: string): Promise<void> {
-  await fs.mkdir(path.join(directory, 'tests'), { recursive: true });
-  try {
-    await fs.writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' });
-  } catch (error) {
-    if (!isExistingFile(error)) throw error;
-  }
-}
-
-export async function scaffoldTaskDirectories(tasks: GeneratedTask[]): Promise<void> {
-  for (const task of tasks) await scaffoldTaskDirectory(task.directory);
-}
-
-export async function setupFolders({
-  pages,
-  agents,
-  docsRoot,
-  outputDirectory,
-  runId: savedRunId,
-}: {
-  pages: DocsPage[];
-  agents: HarnessAgent[];
-  docsRoot: string;
-  outputDirectory: string;
-  runId: string | null;
-}): Promise<{ runId: string; runDirectory: string; reportPath: string; tasks: GeneratedTask[] }> {
-  const runId =
-    savedRunId ??
-    `${new Date()
-      .toISOString()
-      .replace(/[-:TZ.]/g, '')
-      .slice(0, 14)}-${randomBytes(4).toString('hex')}`;
-  const runDirectory = path.join(path.resolve(outputDirectory), runId);
-  const reportPath = path.join(runDirectory, 'report.json');
-  await fs.mkdir(runDirectory, { recursive: true });
-
-  const tasks: GeneratedTask[] = [];
-  for (const agent of agents) {
-    for (const [pageIndex, page] of pages.entries()) {
-      const slug = page.file
-        .replace(/\.[^.]+$/, '')
-        .replace(/[^a-z0-9]+/gi, '-')
-        .replace(/^-|-$/g, '')
-        .slice(0, 56);
-      tasks.push({
-        id: `${agent}:${page.file}`,
-        agent,
-        docsRoot,
-        file: page.file,
-        directory: path.join(
-          runDirectory,
-          agent,
-          `${String(pageIndex + 1).padStart(3, '0')}-${slug || 'page'}`
-        ),
-        attempts: 0,
-        tokens: 0,
-      });
-    }
-  }
-  return { runId, runDirectory, reportPath, tasks };
-}

package/src/agent-harness/tasks/checkTestability.ts

@@ -1,256 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import z from 'zod';
-
-import {
-  buildCheckPrompt,
-  CHECK_JSON_INSTRUCTION,
-  CHECK_SYSTEM_PROMPT,
-  CHECK_TOOL_INSTRUCTION,
-} from '../buildTaskPrompt.js';
-import { loadClaudeAgentSdk, loadCodexSdk } from '../loadAgentSdk.js';
-import {
-  GeneratedTask,
-  HarnessAgent,
-  RunCodeTestsOptions,
-  TaskUpdate,
-  TestCheck,
-  testCheckSchema,
-} from '../types.js';
-
-const CHECK_FILENAME = 'mint-test-check.json';
-
-const CHECK_MODEL = 'claude-haiku-4-5-20251001';
-
-async function readValidCheck(directory: string): Promise<TestCheck | null> {
-  try {
-    const check = testCheckSchema.parse(
-      JSON.parse(await fs.readFile(path.join(directory, CHECK_FILENAME), 'utf8'))
-    );
-    return validateCheck(check).length === 0 ? check : null;
-  } catch {
-    return null;
-  }
-}
-
-function validateCheck(check: TestCheck): string[] {
-  return !check.testable && !check.reason ? ['Not-testable page needs a reason'] : [];
-}
-
-function parseJsonReply(reply: string): unknown {
-  const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(reply);
-  const raw = fenced?.[1] ?? reply;
-  const start = raw.indexOf('{');
-  const end = raw.lastIndexOf('}');
-  if (start === -1 || end === -1) throw new Error('Reply contained no JSON object');
-  return JSON.parse(raw.slice(start, end + 1));
-}
-
-async function runTestabilityCheck({
-  file,
-  content,
-  agent,
-  workingDirectory,
-  signal,
-}: {
-  file: string;
-  content: string;
-  agent: HarnessAgent;
-  workingDirectory: string;
-  signal: AbortSignal;
-}): Promise<{ tokens: number; check: TestCheck }> {
-  const prompt = buildCheckPrompt(file, content);
-  let tokens = 0;
-  let raw: unknown;
-  if (agent === 'claude') {
-    const reply = await promptClaudeCheck({ prompt, workingDirectory, signal });
-    tokens = reply.tokens;
-    if (reply.captured === null) throw new Error('agent never called report_testability');
-    raw = reply.captured;
-  } else {
-    const reply = await promptCodex({
-      prompt,
-      systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_JSON_INSTRUCTION,
-      workingDirectory,
-      signal,
-    });
-    tokens = reply.tokens;
-    raw = parseJsonReply(reply.result);
-  }
-  const check = testCheckSchema.parse(raw);
-  const errors = validateCheck(check);
-  if (errors.length > 0) throw new Error(errors.join('\n'));
-  return { tokens, check };
-}
-
-export const checkFileTestability = async (
-  task: GeneratedTask,
-  {
-    onTaskUpdate,
-    signal,
-    runDirectory,
-  }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'> & { runDirectory: string }
-): Promise<GeneratedTask> => {
-  const emit = (phase: TaskUpdate['phase'], tokens: number) =>
-    onTaskUpdate({
-      id: task.id,
-      agent: task.agent,
-      file: task.file,
-      directory: task.directory,
-      generatedFiles: [],
-      phase,
-      tokens,
-    });
-  let attempts = 0;
-  let tokens = 0;
-  let lastError = 'run cancelled';
-
-  let check = await readValidCheck(task.directory);
-  if (!check) {
-    emit('checking', tokens);
-    const content = await fs.readFile(path.join(task.docsRoot, task.file), 'utf8');
-    for (let attempt = 1; attempt <= 2 && !signal.aborted && !check; attempt++) {
-      attempts = attempt;
-      try {
-        const reply = await runTestabilityCheck({
-          file: task.file,
-          content,
-          agent: task.agent,
-          workingDirectory: runDirectory,
-          signal,
-        });
-        tokens += reply.tokens;
-        check = reply.check;
-      } catch (error) {
-        lastError = error instanceof Error ? error.message : String(error);
-      }
-    }
-    if (!check) {
-      if (signal.aborted) {
-        emit('cancelled', tokens);
-        return { ...task, attempts, tokens, error: 'run cancelled' };
-      }
-      emit('agent_error', tokens);
-      return { ...task, attempts, tokens, error: lastError };
-    }
-  }
-
-  if (!check.testable) {
-    emit('not_testable', tokens);
-    return { ...task, attempts, tokens, testable: false };
-  }
-  await fs.mkdir(task.directory, { recursive: true });
-  await fs.writeFile(
-    path.join(task.directory, CHECK_FILENAME),
-    `${JSON.stringify(check, null, 2)}\n`
-  );
-  emit('checked', tokens);
-  return { ...task, attempts, tokens, testable: true };
-};
-
-async function promptClaudeCheck({
-  prompt,
-  workingDirectory,
-  signal,
-}: {
-  prompt: string;
-  workingDirectory: string;
-  signal: AbortSignal;
-}): Promise<{ tokens: number; captured: unknown }> {
-  const { query, tool, createSdkMcpServer } = await loadClaudeAgentSdk();
-  const abortController = new AbortController();
-  const abort = () => abortController.abort(signal.reason);
-  if (signal.aborted) abort();
-  else signal.addEventListener('abort', abort, { once: true });
-
-  let captured: unknown = null;
-  const server = createSdkMcpServer({
-    name: 'mint-test',
-    tools: [
-      tool(
-        'report_testability',
-        'Report whether the documentation page is testable. Call exactly once.',
-        {
-          version: z.literal(1),
-          testable: z.boolean(),
-          reason: z.string().optional(),
-        },
-        async (input) => {
-          captured = input;
-          return { content: [{ type: 'text', text: 'recorded' }] };
-        }
-      ),
-    ],
-  });
-
-  const session = query({
-    prompt,
-    options: {
-      abortController,
-      allowedTools: ['mcp__mint-test__report_testability'],
-      cwd: workingDirectory,
-      mcpServers: { 'mint-test': server },
-      model: CHECK_MODEL,
-      permissionMode: 'dontAsk',
-      persistSession: false,
-      settingSources: [],
-      strictMcpConfig: true,
-      systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_TOOL_INSTRUCTION,
-      tools: [],
-    },
-  });
-
-  try {
-    for await (const message of session) {
-      if (message.type !== 'result') continue;
-      if (message.subtype !== 'success' || message.is_error) {
-        const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
-        throw new Error(details || `Claude stopped with ${message.subtype}`);
-      }
-      const usage = message.usage;
-      return {
-        tokens:
-          usage.input_tokens +
-          usage.output_tokens +
-          usage.cache_creation_input_tokens +
-          usage.cache_read_input_tokens,
-        captured,
-      };
-    }
-    throw new Error('Claude finished without returning a result');
-  } finally {
-    signal.removeEventListener('abort', abort);
-    session.close();
-  }
-}
-
-async function promptCodex({
-  prompt,
-  systemPrompt,
-  workingDirectory,
-  signal,
-}: {
-  prompt: string;
-  systemPrompt: string;
-  workingDirectory: string;
-  signal: AbortSignal;
-}): Promise<{ tokens: number; result: string }> {
-  const { Codex } = await loadCodexSdk();
-  const thread = new Codex().startThread({
-    approvalPolicy: 'never',
-    networkAccessEnabled: true,
-    sandboxMode: 'workspace-write',
-    skipGitRepoCheck: true,
-    workingDirectory,
-  });
-  const result = await thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
-  const usage = result.usage;
-  const tokens = usage
-    ? usage.input_tokens +
-      usage.cached_input_tokens +
-      usage.cache_write_input_tokens +
-      usage.output_tokens +
-      usage.reasoning_output_tokens
-    : 0;
-  return { tokens, result: result.finalResponse };
-}