@mintlify/cli 4.0.1434 → 4.0.1438

package/src/agent-harness/generateTestCode.ts

@@ -0,0 +1,332 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { SYSTEM_PROMPT, buildTaskPrompt } from './buildTaskPrompt.js';
+import { loadClaudeAgentSdk, loadCodexSdk } from './loadAgentSdk.js';
+import { mapWithConcurrency } from './mapWithConcurrency.js';
+import { checkFileTestability } from './tasks/checkTestability.js';
+import {
+  manifestSchema,
+  type GeneratedTask,
+  type GeneratedTestManifest,
+  type TaskUpdate,
+} from './types.js';
+
+const MANIFEST_FILENAME = 'mint-test.json';
+const SAFE_EXECUTABLES = new Set([
+  'bun',
+  'bunx',
+  'bundle',
+  'cargo',
+  'composer',
+  'deno',
+  'dotnet',
+  'go',
+  'gradle',
+  'java',
+  'javac',
+  'mvn',
+  'node',
+  'npm',
+  'npx',
+  'php',
+  'pip',
+  'pip3',
+  'pnpm',
+  'python',
+  'python3',
+  'pytest',
+  'ruby',
+  'rustc',
+  'uv',
+  'vitest',
+  'yarn',
+]);
+
+async function promptClaude({
+  prompt,
+  systemPrompt,
+  model,
+  workingDirectory,
+  docsRoot,
+  tools,
+  signal,
+}: {
+  prompt: string;
+  systemPrompt: string;
+  model: string;
+  workingDirectory: string;
+  docsRoot: string;
+  tools: string[];
+  signal: AbortSignal;
+}): Promise<{ tokens: number; result: string }> {
+  const { query } = await loadClaudeAgentSdk();
+  const abortController = new AbortController();
+  const abort = () => abortController.abort(signal.reason);
+  if (signal.aborted) abort();
+  else signal.addEventListener('abort', abort, { once: true });
+
+  const session = query({
+    prompt,
+    options: {
+      abortController,
+      additionalDirectories: [docsRoot],
+      allowedTools: tools,
+      cwd: workingDirectory,
+      model,
+      permissionMode: 'dontAsk',
+      persistSession: false,
+      settingSources: [],
+      strictMcpConfig: true,
+      systemPrompt,
+      tools,
+    },
+  });
+
+  try {
+    for await (const message of session) {
+      if (message.type !== 'result') continue;
+      if (message.subtype !== 'success' || message.is_error) {
+        const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
+        throw new Error(details || `Claude stopped with ${message.subtype}`);
+      }
+      const usage = message.usage;
+      return {
+        tokens:
+          usage.input_tokens +
+          usage.output_tokens +
+          usage.cache_creation_input_tokens +
+          usage.cache_read_input_tokens,
+        result: message.result,
+      };
+    }
+    throw new Error('Claude finished without returning a result');
+  } finally {
+    signal.removeEventListener('abort', abort);
+    session.close();
+  }
+}
+
+const AGENT_TOOLS = ['Read', 'Glob', 'Grep', 'Write', 'Edit', 'Bash'];
+
+function isPathInside(root: string, candidate: string): boolean {
+  const relative = path.relative(root, candidate);
+  return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
+}
+
+async function validateManifest(
+  manifest: GeneratedTestManifest,
+  directory: string
+): Promise<string[]> {
+  const errors: string[] = [];
+  const realDirectory = await fs.realpath(directory);
+
+  for (const file of manifest.generatedFiles) {
+    if (path.isAbsolute(file)) {
+      errors.push(`Generated file path must be relative: ${file}`);
+      continue;
+    }
+    const resolved = path.resolve(directory, file);
+    if (!isPathInside(directory, resolved)) {
+      errors.push(`Generated file leaves its task: ${file}`);
+      continue;
+    }
+    const realPath = await fs.realpath(resolved).catch(() => undefined);
+    if (!realPath || !isPathInside(realDirectory, realPath)) {
+      errors.push(`Generated file leaves its task through a link: ${file}`);
+      continue;
+    }
+    const stats = await fs.stat(realPath).catch(() => undefined);
+    if (!stats?.isFile()) errors.push(`Generated file does not exist: ${file}`);
+  }
+
+  const commandIds = new Set<string>();
+
+  for (const command of [...manifest.setupCommands, ...manifest.testCommands]) {
+    if (commandIds.has(command.id)) errors.push(`Duplicate command ID: ${command.id}`);
+    commandIds.add(command.id);
+    if (path.isAbsolute(command.executable) || command.executable.split(/[\\/]/).includes('..')) {
+      errors.push(`Unsafe command executable: ${command.executable}`);
+      continue;
+    }
+    if (!SAFE_EXECUTABLES.has(command.executable) && !command.executable.startsWith('./')) {
+      errors.push(`Unsupported command executable: ${command.executable}`);
+      continue;
+    }
+    if (command.executable.startsWith('./')) {
+      const executablePath = await fs
+        .realpath(path.resolve(directory, command.executable))
+        .catch(() => undefined);
+      const stats = executablePath
+        ? await fs.stat(executablePath).catch(() => undefined)
+        : undefined;
+      if (!executablePath || !isPathInside(realDirectory, executablePath) || !stats?.isFile()) {
+        errors.push(`Command executable leaves its task or is not a file: ${command.executable}`);
+      }
+    }
+    const cwd = command.cwd ?? '.';
+    if (path.isAbsolute(cwd)) {
+      errors.push(`Command cwd must be relative: ${cwd}`);
+      continue;
+    }
+    const resolvedCwd = path.resolve(directory, cwd);
+    const realCwd = await fs.realpath(resolvedCwd).catch(() => undefined);
+    const stats = realCwd ? await fs.stat(realCwd).catch(() => undefined) : undefined;
+    if (
+      !isPathInside(directory, resolvedCwd) ||
+      !realCwd ||
+      !isPathInside(realDirectory, realCwd) ||
+      !stats?.isDirectory()
+    ) {
+      errors.push(`Command cwd is missing or leaves its task: ${cwd}`);
+    }
+  }
+
+  return errors;
+}
+
+async function readValidManifest(directory: string): Promise<GeneratedTestManifest | null> {
+  try {
+    const manifest = manifestSchema.parse(
+      JSON.parse(await fs.readFile(path.join(directory, MANIFEST_FILENAME), 'utf8'))
+    );
+    const errors = await validateManifest(manifest, directory);
+    return errors.length === 0 ? manifest : null;
+  } catch {
+    return null;
+  }
+}
+
+export const CHECK_CONCURRENCY = 20;
+
+export async function checkTestability({
+  tasks,
+  signal,
+  onTaskUpdate,
+}: {
+  tasks: GeneratedTask[];
+  signal: AbortSignal;
+  onTaskUpdate: (update: TaskUpdate) => void;
+}): Promise<GeneratedTask[]> {
+  const folderOrdered = [...tasks].sort(
+    (left, right) =>
+      path.dirname(left.file).localeCompare(path.dirname(right.file)) ||
+      left.file.localeCompare(right.file) ||
+      left.agent.localeCompare(right.agent)
+  );
+
+  return mapWithConcurrency(folderOrdered, CHECK_CONCURRENCY, checkFileTestability, {
+    signal,
+    onTaskUpdate,
+  });
+}
+
+export async function generateTestCode({
+  tasks,
+  docsRoot,
+  model,
+  concurrency,
+  signal,
+  onTaskUpdate,
+}: {
+  tasks: GeneratedTask[];
+  docsRoot: string;
+  model: string;
+  concurrency: number;
+  signal: AbortSignal;
+  onTaskUpdate: (update: TaskUpdate) => void;
+}): Promise<GeneratedTask[]> {
+  const generateTask = async (task: GeneratedTask): Promise<GeneratedTask> => {
+    if (task.error || !task.testable) return task;
+    const emit = (phase: TaskUpdate['phase'], tokens: number) =>
+      onTaskUpdate({ id: task.id, agent: task.agent, file: task.file, phase, tokens });
+    let attempts = task.attempts;
+    let tokens = task.tokens;
+    let lastError = 'run cancelled';
+
+    const existing = await readValidManifest(task.directory);
+    if (existing) {
+      emit('generated', tokens);
+      return { ...task, manifest: existing };
+    }
+
+    for (let attempt = 1; attempt <= 2 && !signal.aborted; attempt++) {
+      attempts += 1;
+      if (attempt === 1) emit('generating', tokens);
+      try {
+        const prompt = buildTaskPrompt(docsRoot, task.file, task.directory);
+        const reply =
+          task.agent === 'claude'
+            ? await promptClaude({
+                prompt,
+                systemPrompt: SYSTEM_PROMPT,
+                model,
+                workingDirectory: task.directory,
+                docsRoot,
+                tools: AGENT_TOOLS,
+                signal,
+              })
+            : await promptCodex({
+                prompt,
+                systemPrompt: SYSTEM_PROMPT,
+                workingDirectory: task.directory,
+                docsRoot,
+                signal,
+              });
+        tokens += reply.tokens;
+        const manifest = manifestSchema.parse(
+          JSON.parse(await fs.readFile(path.join(task.directory, MANIFEST_FILENAME), 'utf8'))
+        );
+        const errors = await validateManifest(manifest, task.directory);
+        if (errors.length > 0) throw new Error(errors.join('\n'));
+
+        emit('generated', tokens);
+        return { ...task, attempts, tokens, manifest };
+      } catch (error) {
+        lastError = error instanceof Error ? error.message : String(error);
+      }
+    }
+    if (signal.aborted) {
+      emit('cancelled', tokens);
+      return { ...task, attempts, tokens, error: 'run cancelled' };
+    }
+    emit('agent_error', tokens);
+    return { ...task, attempts, tokens, error: lastError };
+  };
+
+  return mapWithConcurrency(tasks, concurrency, generateTask);
+}
+
+async function promptCodex({
+  prompt,
+  systemPrompt,
+  workingDirectory,
+  docsRoot,
+  signal,
+}: {
+  prompt: string;
+  systemPrompt: string;
+  workingDirectory: string;
+  docsRoot: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; result: string }> {
+  const { Codex } = await loadCodexSdk();
+  const thread = new Codex().startThread({
+    additionalDirectories: [docsRoot],
+    approvalPolicy: 'never',
+    networkAccessEnabled: true,
+    sandboxMode: 'workspace-write',
+    skipGitRepoCheck: true,
+    workingDirectory,
+  });
+  const result = await thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
+  const usage = result.usage;
+  const tokens = usage
+    ? usage.input_tokens +
+      usage.cached_input_tokens +
+      usage.cache_write_input_tokens +
+      usage.output_tokens +
+      usage.reasoning_output_tokens
+    : 0;
+  return { tokens, result: result.finalResponse };
+}

package/src/agent-harness/index.ts

@@ -0,0 +1,109 @@
+import fs from 'node:fs/promises';
+
+import { discoverCodeBlocks } from './discoverCodeBlocks.js';
+import { executeCodeBlocks } from './executeCodeBlocks.js';
+import { filterSelectedPages } from './filterSelectedPages.js';
+import { checkTestability, generateTestCode } from './generateTestCode.js';
+import { clearRunSave, writeRunSave } from './runSave.js';
+import { setupFolders } from './setupFolders.js';
+import type { CodeTestReport, RunCodeTestsOptions } from './types.js';
+
+export const DEFAULT_CONCURRENCY = 5;
+export const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60 * 1_000;
+
+export { discoverCodeBlocks } from './discoverCodeBlocks.js';
+export { clearRunSave, loadRunSave } from './runSave.js';
+export type {
+  CodeTestReport,
+  CommandResult,
+  CommandStatus,
+  DocsPage,
+  GeneratedTestManifest,
+  HarnessAgent,
+  RunCodeTestsOptions,
+  RunSave,
+  TaskPhase,
+  TaskResult,
+  TaskStatus,
+  TaskUpdate,
+} from './types.js';
+
+export async function runCodeTests({
+  path: docsPath,
+  agents,
+  model,
+  selectedFiles,
+  outputDirectory,
+  runId: savedRunId,
+  concurrency,
+  commandTimeoutMs,
+  signal,
+  onTaskUpdate,
+}: RunCodeTestsOptions): Promise<CodeTestReport> {
+  const startedAt = new Date();
+  const uniqueAgents = [...new Set(agents)];
+
+  const { docsRoot, pages: allPages } = await discoverCodeBlocks(docsPath);
+  const pages = filterSelectedPages(allPages, selectedFiles);
+
+  const { runId, runDirectory, reportPath, tasks } = await setupFolders({
+    pages,
+    agents: uniqueAgents,
+    docsRoot,
+    outputDirectory,
+    runId: savedRunId,
+  });
+  await writeRunSave(outputDirectory, {
+    version: 1,
+    runId,
+    agents: uniqueAgents,
+    model,
+    selectedFiles: pages.map((page) => page.file),
+    createdAt: startedAt.toISOString(),
+  });
+
+  const checked = await checkTestability({ tasks, signal, onTaskUpdate });
+
+  const generated = await generateTestCode({
+    tasks: checked,
+    docsRoot,
+    model,
+    concurrency,
+    signal,
+    onTaskUpdate,
+  });
+
+  const results = await executeCodeBlocks({
+    tasks: generated,
+    docsRoot,
+    concurrency,
+    commandTimeoutMs,
+    signal,
+    onTaskUpdate,
+  });
+
+  const completedAt = new Date();
+  const report: CodeTestReport = {
+    version: 1,
+    runId,
+    status: signal.aborted
+      ? 'cancelled'
+      : results.every((task) => task.status === 'passed')
+        ? 'passed'
+        : results.some((task) => task.status === 'passed')
+          ? 'completed'
+          : 'failed',
+    docsRoot,
+    outputDirectory: runDirectory,
+    reportPath,
+    agents: uniqueAgents,
+    selectedFiles: pages.map((page) => page.file),
+    startedAt: startedAt.toISOString(),
+    completedAt: completedAt.toISOString(),
+    durationMs: completedAt.getTime() - startedAt.getTime(),
+    tasks: results,
+  };
+  await fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+  if (!signal.aborted) await clearRunSave(outputDirectory);
+  return report;
+}

package/src/agent-harness/loadAgentSdk.ts

@@ -0,0 +1,35 @@
+const INSTALL_HINTS: Record<string, string> = {
+  '@anthropic-ai/claude-agent-sdk':
+    'npm install @anthropic-ai/claude-agent-sdk @anthropic-ai/sdk @modelcontextprotocol/sdk',
+  '@openai/codex-sdk': 'npm install @openai/codex-sdk',
+};
+
+function isModuleNotFound(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null) return false;
+  const { code } = error as { code?: unknown };
+  return code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND';
+}
+
+function missingSdkError(packageName: string): Error {
+  return new Error(
+    `mint test requires the optional dependency ${packageName}, which is not installed. Install it with: ${INSTALL_HINTS[packageName]}`
+  );
+}
+
+export async function loadClaudeAgentSdk() {
+  try {
+    return await import('@anthropic-ai/claude-agent-sdk');
+  } catch (error) {
+    if (isModuleNotFound(error)) throw missingSdkError('@anthropic-ai/claude-agent-sdk');
+    throw error;
+  }
+}
+
+export async function loadCodexSdk() {
+  try {
+    return await import('@openai/codex-sdk');
+  } catch (error) {
+    if (isModuleNotFound(error)) throw missingSdkError('@openai/codex-sdk');
+    throw error;
+  }
+}

package/src/agent-harness/mapWithConcurrency.ts

@@ -0,0 +1,21 @@
+export async function mapWithConcurrency<T, R, A extends unknown[]>(
+  items: T[],
+  concurrency: number,
+  fn: (item: T, ...args: A) => Promise<R>,
+  ...args: A
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let nextIndex = 0;
+  const worker = async (): Promise<void> => {
+    for (;;) {
+      const index = nextIndex++;
+      const item = items[index];
+      if (index >= items.length || item === undefined) return;
+      results[index] = await fn(item, ...args);
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(Math.max(1, concurrency), items.length) }, () => worker())
+  );
+  return results;
+}

package/src/agent-harness/runSave.ts

@@ -0,0 +1,38 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+
+import type { RunSave } from './types.js';
+
+const runSaveSchema = z
+  .object({
+    version: z.literal(1),
+    runId: z.string().regex(/^[A-Za-z0-9._-]+$/),
+    agents: z.array(z.enum(['claude', 'codex'])).min(1),
+    model: z.string().min(1),
+    selectedFiles: z.array(z.string().min(1)),
+    createdAt: z.string(),
+  })
+  .strict();
+
+function savePath(outputDirectory: string): string {
+  return path.join(outputDirectory, 'save.json');
+}
+
+export async function loadRunSave(outputDirectory: string): Promise<RunSave | null> {
+  try {
+    const content = await fs.readFile(savePath(outputDirectory), 'utf8');
+    return runSaveSchema.parse(JSON.parse(content));
+  } catch {
+    return null;
+  }
+}
+
+export async function writeRunSave(outputDirectory: string, save: RunSave): Promise<void> {
+  await fs.mkdir(outputDirectory, { recursive: true });
+  await fs.writeFile(savePath(outputDirectory), `${JSON.stringify(save, null, 2)}\n`);
+}
+
+export async function clearRunSave(outputDirectory: string): Promise<void> {
+  await fs.unlink(savePath(outputDirectory)).catch(() => undefined);
+}

package/src/agent-harness/setupFolders.ts

@@ -0,0 +1,73 @@
+import { randomBytes } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { DocsPage, GeneratedTask, HarnessAgent } from './types.js';
+
+const PACKAGE_JSON = `${JSON.stringify(
+  {
+    name: 'mint-test-task',
+    private: true,
+    type: 'module',
+    scripts: { test: 'node --test tests/' },
+  },
+  null,
+  2
+)}\n`;
+
+async function scaffoldTaskDirectory(directory: string): Promise<void> {
+  await fs.mkdir(path.join(directory, 'tests'), { recursive: true });
+  await fs
+    .writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' })
+    .catch(() => {});
+}
+
+export async function setupFolders({
+  pages,
+  agents,
+  docsRoot,
+  outputDirectory,
+  runId: savedRunId,
+}: {
+  pages: DocsPage[];
+  agents: HarnessAgent[];
+  docsRoot: string;
+  outputDirectory: string;
+  runId: string | null;
+}): Promise<{ runId: string; runDirectory: string; reportPath: string; tasks: GeneratedTask[] }> {
+  const runId =
+    savedRunId ??
+    `${new Date()
+      .toISOString()
+      .replace(/[-:TZ.]/g, '')
+      .slice(0, 14)}-${randomBytes(4).toString('hex')}`;
+  const runDirectory = path.join(path.resolve(outputDirectory), runId);
+  const reportPath = path.join(runDirectory, 'report.json');
+  await fs.mkdir(runDirectory, { recursive: true });
+
+  const tasks: GeneratedTask[] = [];
+  for (const agent of agents) {
+    for (const [pageIndex, page] of pages.entries()) {
+      const slug = page.file
+        .replace(/\.[^.]+$/, '')
+        .replace(/[^a-z0-9]+/gi, '-')
+        .replace(/^-|-$/g, '')
+        .slice(0, 56);
+      tasks.push({
+        id: `${agent}:${page.file}`,
+        agent,
+        docsRoot,
+        file: page.file,
+        directory: path.join(
+          runDirectory,
+          agent,
+          `${String(pageIndex + 1).padStart(3, '0')}-${slug || 'page'}`
+        ),
+        attempts: 0,
+        tokens: 0,
+      });
+    }
+  }
+  for (const task of tasks) await scaffoldTaskDirectory(task.directory);
+  return { runId, runDirectory, reportPath, tasks };
+}