@mintlify/cli 4.0.1654 → 4.0.1656

package/src/agent-harness/generateTestCode.ts

@@ -1,396 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-
-import { SYSTEM_PROMPT, buildTaskPrompt } from './buildTaskPrompt.js';
-import { loadClaudeAgentSdk, loadCodexSdk } from './loadAgentSdk.js';
-import {
-  MANIFEST_FILENAME,
-  SAFE_EXECUTABLES,
-  appendAgentReply,
-  readManifest,
-  writeErrorDetails,
-} from './manifest.js';
-import { mapWithConcurrency } from './mapWithConcurrency.js';
-import { checkFileTestability } from './tasks/checkTestability.js';
-import type {
-  GeneratedTask,
-  GeneratedTestManifest,
-  TaskUpdate,
-  TestGenerationResult,
-} from './types.js';
-
-const SCAFFOLD_FILENAMES = new Set([MANIFEST_FILENAME, 'package.json']);
-const IGNORED_GENERATED_DIRECTORIES = new Set(['.git', 'node_modules']);
-
-async function promptClaude({
-  prompt,
-  systemPrompt,
-  model,
-  workingDirectory,
-  docsRoot,
-  tools,
-  signal,
-}: {
-  prompt: string;
-  systemPrompt: string;
-  model: string;
-  workingDirectory: string;
-  docsRoot: string;
-  tools: string[];
-  signal: AbortSignal;
-}): Promise<{ tokens: number; result: string }> {
-  const { query } = await loadClaudeAgentSdk();
-  const abortController = new AbortController();
-  const abort = () => abortController.abort(signal.reason);
-  if (signal.aborted) abort();
-  else signal.addEventListener('abort', abort, { once: true });
-
-  const session = query({
-    prompt,
-    options: {
-      abortController,
-      additionalDirectories: [docsRoot],
-      allowedTools: tools,
-      cwd: workingDirectory,
-      model,
-      permissionMode: 'dontAsk',
-      persistSession: false,
-      settingSources: [],
-      strictMcpConfig: true,
-      systemPrompt,
-      tools,
-    },
-  });
-
-  try {
-    for await (const message of session) {
-      if (message.type !== 'result') continue;
-      if (message.subtype !== 'success' || message.is_error) {
-        const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
-        throw new Error(details || `Claude stopped with ${message.subtype}`);
-      }
-      const usage = message.usage;
-      return {
-        tokens:
-          usage.input_tokens +
-          usage.output_tokens +
-          usage.cache_creation_input_tokens +
-          usage.cache_read_input_tokens,
-        result: message.result,
-      };
-    }
-    throw new Error('Claude finished without returning a result');
-  } finally {
-    signal.removeEventListener('abort', abort);
-    session.close();
-  }
-}
-
-const AGENT_TOOLS = ['Read', 'Glob', 'Grep', 'Write', 'Edit', 'Bash'];
-
-function isPathInside(root: string, candidate: string): boolean {
-  const relative = path.relative(root, candidate);
-  return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
-}
-
-async function validateManifest(
-  manifest: GeneratedTestManifest,
-  directory: string
-): Promise<string[]> {
-  const errors: string[] = [];
-  const realDirectory = await fs.realpath(directory);
-
-  for (const file of manifest.generatedFiles) {
-    if (path.isAbsolute(file)) {
-      errors.push(`Generated file path must be relative: ${file}`);
-      continue;
-    }
-    const resolved = path.resolve(directory, file);
-    if (!isPathInside(directory, resolved)) {
-      errors.push(`Generated file leaves its task: ${file}`);
-      continue;
-    }
-    const realPath = await fs.realpath(resolved).catch(() => undefined);
-    if (!realPath || !isPathInside(realDirectory, realPath)) {
-      errors.push(`Generated file leaves its task through a link: ${file}`);
-      continue;
-    }
-    const stats = await fs.stat(realPath).catch(() => undefined);
-    if (!stats?.isFile()) errors.push(`Generated file does not exist: ${file}`);
-  }
-
-  const commandIds = new Set<string>();
-
-  for (const command of [...manifest.setupCommands, ...manifest.testCommands]) {
-    if (commandIds.has(command.id)) errors.push(`Duplicate command ID: ${command.id}`);
-    commandIds.add(command.id);
-    if (path.isAbsolute(command.executable) || command.executable.split(/[\\/]/).includes('..')) {
-      errors.push(`Unsafe command executable: ${command.executable}`);
-      continue;
-    }
-    if (!SAFE_EXECUTABLES.has(command.executable) && !command.executable.startsWith('./')) {
-      errors.push(`Unsupported command executable: ${command.executable}`);
-      continue;
-    }
-    if (command.executable.startsWith('./')) {
-      const executablePath = await fs
-        .realpath(path.resolve(directory, command.executable))
-        .catch(() => undefined);
-      const stats = executablePath
-        ? await fs.stat(executablePath).catch(() => undefined)
-        : undefined;
-      if (!executablePath || !isPathInside(realDirectory, executablePath) || !stats?.isFile()) {
-        errors.push(`Command executable leaves its task or is not a file: ${command.executable}`);
-      }
-    }
-    const cwd = command.cwd ?? '.';
-    if (path.isAbsolute(cwd)) {
-      errors.push(`Command cwd must be relative: ${cwd}`);
-      continue;
-    }
-    const resolvedCwd = path.resolve(directory, cwd);
-    const realCwd = await fs.realpath(resolvedCwd).catch(() => undefined);
-    const stats = realCwd ? await fs.stat(realCwd).catch(() => undefined) : undefined;
-    if (
-      !isPathInside(directory, resolvedCwd) ||
-      !realCwd ||
-      !isPathInside(realDirectory, realCwd) ||
-      !stats?.isDirectory()
-    ) {
-      errors.push(`Command cwd is missing or leaves its task: ${cwd}`);
-    }
-  }
-
-  return errors;
-}
-
-async function readValidManifest(directory: string): Promise<GeneratedTestManifest | null> {
-  try {
-    const manifest = await readManifest(directory);
-    const errors = await validateManifest(manifest, directory);
-    return errors.length === 0 ? manifest : null;
-  } catch {
-    return null;
-  }
-}
-
-async function discoverGeneratedFiles(directory: string): Promise<string[]> {
-  const realDirectory = await fs.realpath(directory);
-  const files: string[] = [];
-
-  async function walk(currentDirectory: string): Promise<void> {
-    const entries = await fs.readdir(currentDirectory, { withFileTypes: true }).catch(() => []);
-    for (const entry of entries) {
-      if (files.length >= 1_000) return;
-      if (currentDirectory === directory && SCAFFOLD_FILENAMES.has(entry.name)) continue;
-      const candidate = path.join(currentDirectory, entry.name);
-      if (entry.isDirectory()) {
-        if (IGNORED_GENERATED_DIRECTORIES.has(entry.name)) continue;
-        await walk(candidate);
-        continue;
-      }
-      if (!entry.isFile()) continue;
-      const realPath = await fs.realpath(candidate).catch(() => undefined);
-      if (realPath && isPathInside(realDirectory, realPath)) files.push(path.resolve(candidate));
-    }
-  }
-
-  await walk(directory);
-  return files.sort((left, right) => left.localeCompare(right));
-}
-
-export const CHECK_CONCURRENCY = 20;
-
-export async function checkTestability({
-  tasks,
-  runDirectory,
-  signal,
-  onTaskUpdate,
-}: {
-  tasks: GeneratedTask[];
-  runDirectory: string;
-  signal: AbortSignal;
-  onTaskUpdate: (update: TaskUpdate) => void;
-}): Promise<GeneratedTask[]> {
-  const folderOrdered = [...tasks].sort(
-    (left, right) =>
-      path.dirname(left.file).localeCompare(path.dirname(right.file)) ||
-      left.file.localeCompare(right.file) ||
-      left.agent.localeCompare(right.agent)
-  );
-
-  return mapWithConcurrency(folderOrdered, CHECK_CONCURRENCY, checkFileTestability, {
-    signal,
-    onTaskUpdate,
-    runDirectory,
-  });
-}
-
-export async function generateTestCode({
-  tasks,
-  docsRoot,
-  model,
-  concurrency,
-  signal,
-  onTaskUpdate,
-  onGenerationResult,
-}: {
-  tasks: GeneratedTask[];
-  docsRoot: string;
-  model: string;
-  concurrency: number;
-  signal: AbortSignal;
-  onTaskUpdate: (update: TaskUpdate) => void;
-  onGenerationResult?: (result: TestGenerationResult) => void;
-}): Promise<GeneratedTask[]> {
-  const generateTask = async (task: GeneratedTask): Promise<GeneratedTask> => {
-    if (task.error || !task.testable) return task;
-    const emit = (
-      phase: TaskUpdate['phase'],
-      tokens: number,
-      generatedFiles: string[] = [],
-      error?: string
-    ) =>
-      onTaskUpdate({
-        id: task.id,
-        agent: task.agent,
-        file: task.file,
-        directory: task.directory,
-        generatedFiles,
-        phase,
-        tokens,
-        ...(error ? { error } : {}),
-      });
-    let attempts = task.attempts;
-    let tokens = task.tokens;
-    let lastError = 'run cancelled';
-    const attemptErrors: string[] = [];
-
-    const existing = await readValidManifest(task.directory);
-    if (existing) {
-      emit(
-        'generated',
-        tokens,
-        existing.generatedFiles.map((file) => path.resolve(task.directory, file))
-      );
-      return { ...task, manifest: existing };
-    }
-
-    const startedAt = Date.now();
-    const startingTokens = tokens;
-    let generationAttempts = 0;
-    const emitGenerationResult = (result: TestGenerationResult) => {
-      try {
-        onGenerationResult?.(result);
-      } catch {}
-    };
-
-    for (let attempt = 1; attempt <= 2 && !signal.aborted; attempt++) {
-      attempts += 1;
-      generationAttempts += 1;
-      if (attempt === 1) emit('generating', tokens);
-      try {
-        const prompt = buildTaskPrompt(docsRoot, task.file, task.directory);
-        const reply =
-          task.agent === 'claude'
-            ? await promptClaude({
-                prompt,
-                systemPrompt: SYSTEM_PROMPT,
-                model,
-                workingDirectory: task.directory,
-                docsRoot,
-                tools: AGENT_TOOLS,
-                signal,
-              })
-            : await promptCodex({
-                prompt,
-                systemPrompt: SYSTEM_PROMPT,
-                workingDirectory: task.directory,
-                docsRoot,
-                signal,
-              });
-        tokens += reply.tokens;
-        await appendAgentReply(task.directory, attempt, reply.result);
-        const manifest = await readManifest(task.directory);
-        const errors = await validateManifest(manifest, task.directory);
-        if (errors.length > 0) {
-          const [first = '', ...rest] = errors;
-          throw new Error([`${MANIFEST_FILENAME} has problems: ${first}`, ...rest].join('\n'));
-        }
-
-        const generatedFiles = manifest.generatedFiles.map((file) =>
-          path.resolve(task.directory, file)
-        );
-        emit('generated', tokens, generatedFiles);
-        emitGenerationResult({
-          status: 'succeeded',
-          agent: task.agent,
-          model,
-          attemptCount: generationAttempts,
-          tokenCount: tokens - startingTokens,
-          durationMs: Date.now() - startedAt,
-          generatedFileCount: generatedFiles.length,
-        });
-        return { ...task, attempts, tokens, manifest };
-      } catch (error) {
-        lastError = error instanceof Error ? error.message : String(error);
-        attemptErrors.push(`Attempt ${attempt}: ${lastError}`);
-      }
-    }
-    if (signal.aborted) {
-      const generatedFiles = await discoverGeneratedFiles(task.directory);
-      emit('cancelled', tokens, generatedFiles, 'run cancelled');
-      return { ...task, attempts, tokens, generatedFiles, error: 'run cancelled' };
-    }
-    const generatedFiles = await discoverGeneratedFiles(task.directory);
-    emitGenerationResult({
-      status: 'failed',
-      agent: task.agent,
-      model,
-      attemptCount: generationAttempts,
-      tokenCount: tokens - startingTokens,
-      durationMs: Date.now() - startedAt,
-      generatedFileCount: generatedFiles.length,
-    });
-    const detailsPath = await writeErrorDetails(task.directory, attemptErrors);
-    const error = detailsPath ? `${lastError}\nDetails: ${detailsPath}` : lastError;
-    emit('agent_error', tokens, generatedFiles, error);
-    return { ...task, attempts, tokens, generatedFiles, error };
-  };
-
-  return mapWithConcurrency(tasks, concurrency, generateTask);
-}
-
-async function promptCodex({
-  prompt,
-  systemPrompt,
-  workingDirectory,
-  docsRoot,
-  signal,
-}: {
-  prompt: string;
-  systemPrompt: string;
-  workingDirectory: string;
-  docsRoot: string;
-  signal: AbortSignal;
-}): Promise<{ tokens: number; result: string }> {
-  const { Codex } = await loadCodexSdk();
-  const thread = new Codex().startThread({
-    additionalDirectories: [docsRoot],
-    approvalPolicy: 'never',
-    networkAccessEnabled: true,
-    sandboxMode: 'workspace-write',
-    skipGitRepoCheck: true,
-    workingDirectory,
-  });
-  const result = await thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
-  const usage = result.usage;
-  const tokens = usage
-    ? usage.input_tokens +
-      usage.cached_input_tokens +
-      usage.cache_write_input_tokens +
-      usage.output_tokens +
-      usage.reasoning_output_tokens
-    : 0;
-  return { tokens, result: result.finalResponse };
-}

package/src/agent-harness/index.ts

@@ -1,127 +0,0 @@
-import fs from 'node:fs/promises';
-
-import { ensureAgentSdks } from './agentPreflight.js';
-import { discoverCodeBlocks } from './discoverCodeBlocks.js';
-import { executeCodeBlocks } from './executeCodeBlocks.js';
-import { filterSelectedPages } from './filterSelectedPages.js';
-import { checkTestability, generateTestCode } from './generateTestCode.js';
-import { recordTestRun, testRunReportPath } from './runHistory.js';
-import { clearRunSave, writeRunSave } from './runSave.js';
-import { scaffoldTaskDirectories, setupFolders } from './setupFolders.js';
-import type { CodeTestReport, RunCodeTestsOptions } from './types.js';
-
-export const DEFAULT_CONCURRENCY = 5;
-export const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60 * 1_000;
-
-export { discoverCodeBlocks } from './discoverCodeBlocks.js';
-export { countReport } from './countReport.js';
-export { loadLatestTestRun, loadTestRuns } from './runHistory.js';
-export { clearRunSave, loadRunSave } from './runSave.js';
-export type {
-  CodeTestReport,
-  CommandResult,
-  CommandStatus,
-  DocsPage,
-  GeneratedTestManifest,
-  HarnessAgent,
-  RunCodeTestsOptions,
-  RunSave,
-  TaskPhase,
-  TaskResult,
-  TaskStatus,
-  TaskUpdate,
-  TestGenerationResult,
-  TestRunSummary,
-  TestRunsIndex,
-} from './types.js';
-
-export async function runCodeTests({
-  path: docsPath,
-  agents,
-  model,
-  selectedFiles,
-  outputDirectory,
-  runId: savedRunId,
-  concurrency,
-  commandTimeoutMs,
-  signal,
-  onTaskUpdate,
-  onGenerationResult,
-}: RunCodeTestsOptions): Promise<CodeTestReport> {
-  const startedAt = new Date();
-  const uniqueAgents = [...new Set(agents)];
-
-  const { docsRoot, pages: allPages } = await discoverCodeBlocks(docsPath);
-  const pages = filterSelectedPages(allPages, selectedFiles);
-  if (pages.length > 0) await ensureAgentSdks(uniqueAgents);
-
-  const {
-    runId,
-    runDirectory,
-    reportPath: legacyReportPath,
-    tasks,
-  } = await setupFolders({
-    pages,
-    agents: uniqueAgents,
-    docsRoot,
-    outputDirectory,
-    runId: savedRunId,
-  });
-  const reportPath = testRunReportPath(docsRoot, runId);
-  await writeRunSave(outputDirectory, {
-    version: 1,
-    runId,
-    agents: uniqueAgents,
-    model,
-    selectedFiles: pages.map((page) => page.file),
-    createdAt: startedAt.toISOString(),
-  });
-
-  const checked = await checkTestability({ tasks, runDirectory, signal, onTaskUpdate });
-  await scaffoldTaskDirectories(checked.filter((task) => task.testable === true));
-
-  const generated = await generateTestCode({
-    tasks: checked,
-    docsRoot,
-    model,
-    concurrency,
-    signal,
-    onTaskUpdate,
-    onGenerationResult,
-  });
-
-  const results = await executeCodeBlocks({
-    tasks: generated,
-    docsRoot,
-    concurrency,
-    commandTimeoutMs,
-    signal,
-    onTaskUpdate,
-  });
-
-  const completedAt = new Date();
-  const report: CodeTestReport = {
-    version: 1,
-    runId,
-    status: signal.aborted
-      ? 'cancelled'
-      : results.every((task) => task.status === 'passed')
-        ? 'passed'
-        : results.some((task) => task.status === 'passed')
-          ? 'completed'
-          : 'failed',
-    docsRoot,
-    outputDirectory: runDirectory,
-    reportPath,
-    agents: uniqueAgents,
-    model,
-    selectedFiles: pages.map((page) => page.file),
-    startedAt: startedAt.toISOString(),
-    completedAt: completedAt.toISOString(),
-    durationMs: completedAt.getTime() - startedAt.getTime(),
-    tasks: results,
-  };
-  await fs.writeFile(legacyReportPath, `${JSON.stringify(report, null, 2)}\n`);
-  if (!signal.aborted) await clearRunSave(outputDirectory);
-  return recordTestRun(docsRoot, report, legacyReportPath);
-}

package/src/agent-harness/loadAgentSdk.ts

@@ -1,35 +0,0 @@
-const INSTALL_HINTS: Record<string, string> = {
-  '@anthropic-ai/claude-agent-sdk':
-    'npm install @anthropic-ai/claude-agent-sdk @anthropic-ai/sdk @modelcontextprotocol/sdk',
-  '@openai/codex-sdk': 'npm install @openai/codex-sdk',
-};
-
-function isModuleNotFound(error: unknown): boolean {
-  if (typeof error !== 'object' || error === null) return false;
-  const { code } = error as { code?: unknown };
-  return code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND';
-}
-
-function missingSdkError(packageName: string): Error {
-  return new Error(
-    `mint test requires the optional dependency ${packageName}, which is not installed. Install it with: ${INSTALL_HINTS[packageName]}`
-  );
-}
-
-export async function loadClaudeAgentSdk() {
-  try {
-    return await import('@anthropic-ai/claude-agent-sdk');
-  } catch (error) {
-    if (isModuleNotFound(error)) throw missingSdkError('@anthropic-ai/claude-agent-sdk');
-    throw error;
-  }
-}
-
-export async function loadCodexSdk() {
-  try {
-    return await import('@openai/codex-sdk');
-  } catch (error) {
-    if (isModuleNotFound(error)) throw missingSdkError('@openai/codex-sdk');
-    throw error;
-  }
-}

package/src/agent-harness/manifest.ts

@@ -1,116 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import type { z } from 'zod';
-
-import { manifestSchema, type GeneratedTestManifest } from './types.js';
-
-export const MANIFEST_FILENAME = 'mint-test.json';
-export const AGENT_REPLY_FILENAME = 'mint-test-agent.md';
-export const ERROR_FILENAME = 'mint-test-error.txt';
-
-export const SAFE_EXECUTABLES = new Set([
-  'bun',
-  'bunx',
-  'bundle',
-  'cargo',
-  'composer',
-  'deno',
-  'dotnet',
-  'go',
-  'gradle',
-  'java',
-  'javac',
-  'mvn',
-  'node',
-  'npm',
-  'npx',
-  'php',
-  'pip',
-  'pip3',
-  'pnpm',
-  'python',
-  'python3',
-  'pytest',
-  'ruby',
-  'rustc',
-  'uv',
-  'vitest',
-  'yarn',
-]);
-
-function isMissingFile(error: unknown): boolean {
-  return (
-    typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'ENOENT'
-  );
-}
-
-/** Turn a schema failure into one readable line per problem, e.g. `testCommands.0.id: expected string`. */
-export function formatManifestIssues(error: z.ZodError): string {
-  return error.issues
-    .map((issue) => {
-      const location = issue.path.map(String).join('.');
-      return `${location || MANIFEST_FILENAME}: ${issue.message}`;
-    })
-    .join('\n');
-}
-
-/**
- * Read and schema-check the manifest an agent wrote. Every failure mode has a message a
- * documentation author can act on, because it is what `mint test` shows for the page.
- */
-export async function readManifest(directory: string): Promise<GeneratedTestManifest> {
-  const manifestPath = path.join(directory, MANIFEST_FILENAME);
-  let raw: string;
-  try {
-    raw = await fs.readFile(manifestPath, 'utf8');
-  } catch (error) {
-    if (isMissingFile(error)) {
-      throw new Error(`The agent finished without writing ${MANIFEST_FILENAME}`);
-    }
-    throw error;
-  }
-
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(raw);
-  } catch (error) {
-    const reason = error instanceof Error ? error.message : String(error);
-    throw new Error(`${MANIFEST_FILENAME} is not valid JSON: ${reason}`);
-  }
-
-  const result = manifestSchema.safeParse(parsed);
-  if (!result.success) {
-    const [first = '', ...rest] = formatManifestIssues(result.error).split('\n');
-    throw new Error(
-      [`${MANIFEST_FILENAME} does not match the expected shape: ${first}`, ...rest].join('\n')
-    );
-  }
-  return result.data;
-}
-
-/** Keep the agent's final message next to its output so a failure can be traced later. */
-export async function appendAgentReply(
-  directory: string,
-  attempt: number,
-  reply: string
-): Promise<void> {
-  const body = reply.trim() || '(no summary)';
-  await fs
-    .appendFile(path.join(directory, AGENT_REPLY_FILENAME), `## Attempt ${attempt}\n\n${body}\n\n`)
-    .catch(() => {});
-}
-
-/** Persist every attempt's failure reason. Returns the file path, or undefined if it could not be written. */
-export async function writeErrorDetails(
-  directory: string,
-  errors: string[]
-): Promise<string | undefined> {
-  if (errors.length === 0) return undefined;
-  const errorPath = path.join(directory, ERROR_FILENAME);
-  try {
-    await fs.writeFile(errorPath, `${errors.join('\n\n')}\n`);
-    return errorPath;
-  } catch {
-    return undefined;
-  }
-}

package/src/agent-harness/mapWithConcurrency.ts

@@ -1,21 +0,0 @@
-export async function mapWithConcurrency<T, R, A extends unknown[]>(
-  items: T[],
-  concurrency: number,
-  fn: (item: T, ...args: A) => Promise<R>,
-  ...args: A
-): Promise<R[]> {
-  const results: R[] = new Array(items.length);
-  let nextIndex = 0;
-  const worker = async (): Promise<void> => {
-    for (;;) {
-      const index = nextIndex++;
-      const item = items[index];
-      if (index >= items.length || item === undefined) return;
-      results[index] = await fn(item, ...args);
-    }
-  };
-  await Promise.all(
-    Array.from({ length: Math.min(Math.max(1, concurrency), items.length) }, () => worker())
-  );
-  return results;
-}