@mintlify/cli 4.0.1437 → 4.0.1438

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/cli",
-  "version": "4.0.1437",
+  "version": "4.0.1438",
   "description": "The Mintlify CLI",
   "engines": {
     "node": ">=18.0.0"
@@ -45,7 +45,6 @@
   },
   "dependencies": {
     "@inquirer/prompts": "7.9.0",
-    "@mintlify/agent-harness": "0.0.3",
     "@mintlify/common": "1.0.1111",
     "@mintlify/link-rot": "3.0.1313",
     "@mintlify/models": "0.0.351",
@@ -80,6 +79,10 @@
     "zod": "4.3.6"
   },
   "optionalDependencies": {
+    "@anthropic-ai/claude-agent-sdk": "0.3.241",
+    "@anthropic-ai/sdk": "0.120.0",
+    "@modelcontextprotocol/sdk": "1.30.0",
+    "@openai/codex-sdk": "0.149.1",
     "keytar": "7.9.0"
   },
   "devDependencies": {
@@ -102,5 +105,5 @@
     "vitest": "2.1.9",
     "vitest-mock-process": "1.0.4"
   },
-  "gitHead": "246494b8852335337e48cd0518b54abb2fb3f774"
+  "gitHead": "a7aa8658185853553564a04b1132bbd9dec768d5"
 }

package/src/agent-harness/MINT_TEST_SYSTEM_DESIGN.md

@@ -0,0 +1,58 @@
+# `mint test` system design
+
+`packages/cli` owns command parsing and terminal interaction. `packages/cli/src/agent-harness` collects code blocks, runs agents, validates generated projects, executes commands, and writes reports so other callers can use the same test engine without the CLI.
+
+```mermaid
+flowchart TD
+    User[Developer] --> Command["mint test [path]"]
+
+    subgraph CLI["packages/cli"]
+        Command --> Args["Parse path, agent, scope, and TUI flags"]
+        Mode{"Interactive TTY?"}
+        UI["Ink flow: agent, code blocks, scope, progress, results"]
+        Plain["Non-interactive runner with Claude default"]
+        Exit["Print summary and set exit code"]
+    end
+
+    subgraph Harness["packages/cli/src/agent-harness"]
+        Collect["collectCodeBlocks"]
+        Collection["CodeBlockCollection"]
+        Run["runCodeTests<br/>tasks and inline preflight, progress, and report writes"]
+        Prompt["AgentRunner.prompt()<br/>ClaudeRunner or CodexRunner"]
+        Validate["Manifest validation inside runCodeTests<br/>schema, paths, block coverage, and executables"]
+        Execute["executeCodeTestCommand<br/>setup and test commands without a shell"]
+        Report["CodeTestReport<br/>persisted as report.json"]
+    end
+
+    Docs["Documentation source, read only"] --> Collect
+    Args --> Collect
+    Collect --> Collection
+    Collection --> Mode
+    Mode -->|yes| UI
+    Mode -->|no| Plain
+    UI -->|selected agents and files| Run
+    Plain -->|default agent and selected files| Run
+    Collection -->|collected blocks| Run
+    Run --> Prompt --> Validate --> Execute --> Report
+    Docs -. "read-only context" .-> Prompt
+    Run -. "inline progress callbacks" .-> UI
+    Report --> Exit
+```
+
+## Type contracts
+
+The harness exports five types. Status unions and nested block, environment, summary, command, and task fields live on the contract that owns them.
+
+| Exported type           | Contract                                                                                                                                    |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| `HarnessAgent`          | The supported runner names, `claude` and `codex`.                                                                                           |
+| `AgentRunner`           | One `prompt(prompt)` method returning the final response text. `ClaudeRunner` and `CodexRunner` implement it.                               |
+| `CodeBlockCollection`   | The docs root and fenced block locations grouped by markdown file. `collectCodeBlocks` returns it.                                          |
+| `GeneratedTestManifest` | The inferred output of the runtime `mint-test.json` schema, including generated files, block dispositions, environment needs, and commands. |
+| `CodeTestReport`        | The complete persisted `report.json` shape. Task, block, command, environment, summary, and status fields are nested here.                  |
+
+`GeneratedTestManifest` is inferred from the runtime schema in `packages/cli/src/agent-harness/types.ts`, so validation and TypeScript cannot drift apart. Contract changes still require a schema test and an update to this table.
+
+`runCodeTests` declares its input object and progress events on the function. `executeCodeTestCommand` does the same for command input. Neither API has a separate options, request, progress, preflight, command-runner, or command-result type to keep in sync.
+
+The manifest is the boundary between generated agent output and trusted execution. The harness rejects commands outside the allowlist or task workspace, runs each command with `shell: false`, applies timeouts and cancellation, redacts secrets from captured output, and records every state change in the report.

package/src/agent-harness/buildTaskPrompt.ts

@@ -0,0 +1,82 @@
+import path from 'node:path';
+
+export const SYSTEM_PROMPT = `# Mint test generator
+
+You will be provided a markdown page guide that contians codeblocks. Follow the guide and implement
+the code as instrucuted by the guide into a given output directory.
+
+Once finished, create a mint-test.json file in the current working directory with this shape:
+
+{
+  "version": 1,
+  "generatedFiles": ["tests/example.test.js", "tests/test_example.py"],
+  "setupCommands": [],
+  "testCommands": [
+    {
+      "executable": "node",
+      "args": ["--test", "tests/"],
+      "cwd": ".",
+      "timeoutMs": 120000
+    },
+    {
+      "executable": "python3",
+      "args": ["-m", "unittest", "discover", "-s", "tests"],
+      "cwd": ".",
+      "timeoutMs": 120000
+    }
+  ]
+}
+
+Command cwd values must be relative to the current working directory. Finish with a short plain-text summary.
+
+Rules:
+- Read only the page named in the task. You may also read docs.json and an OpenAPI file referenced by that page.
+- The output directory is already scaffolded: package.json exists with "npm test" wired to "node --test tests/", and a tests/ directory is ready. Do not recreate or rewrite this scaffolding.
+- Write JavaScript and TypeScript tests as ES modules named tests/*.test.js using the Node built-in test runner (node:test and node:assert). Write Python tests as tests/test_*.py using the standard library unittest module, run with python3 -m unittest discover -s tests.
+- Prefer the standard library over installing dependencies; add a setup command only when a test truly cannot run without one.
+- Never write README, docs, or explanation files.
+- Keep helpers inline in the test file; do not create separate helper modules.
+- Write one test file per page unless the page genuinely needs more.
+- Create every additional source file and fixture inside the current working directory.
+- Never modify the documentation source. Do not commit, push, or open a pull request.
+- Cover fenced code blocks inside Mintlify components such as CodeGroup, Tabs, Steps, RequestExample, and ResponseExample.
+- If the page has openapi frontmatter, you may resolve its spec relative to the docs root and add tests for request snippets.
+- Do not install dependencies or run the generated tests. The Mint CLI handles setup and execution.
+- Never copy secrets into a file. In your code pull secrets from .env
+`;
+
+export function buildTaskPrompt(docsRoot: string, file: string, directory: string): string {
+  return `
+Use the guide at "${JSON.stringify(path.join(docsRoot, file))}"
+
+Write the required tests and mint-test.json output in the "${JSON.stringify(directory)}" directory.
+Record your report here "${JSON.stringify(file)}"`;
+}
+
+export const CHECK_SYSTEM_PROMPT = `# Mint test triage
+You are given a markdown page guide. Decide if the given file is able to be tested.
+
+Testable pages are pages that include any runnable code (python, typescript, bash) and any logic that could fail if the documented code was wrong
+
+Code blocks are not testable when it is used for formatting, showing logs, configuration, pseudo-code or a diff, an illustrative request or response shape, or a UI-only example.
+
+When unsure, mark it not testable.
+
+Include "reason" only when the page is not testable; omit it entirely when testable is true.`;
+
+export const CHECK_TOOL_INSTRUCTION = `
+
+Report your classification by calling the report_testability tool`;
+
+export const CHECK_JSON_INSTRUCTION = `
+
+Respond with only this JSON object and nothing else:
+{ "version": 1, "testable": false, "reason": "expected output, not runnable code" }`;
+
+export function buildCheckPrompt(file: string, fileContnet: string): string {
+  return `Report if the file at filePath ${JSON.stringify(file)} is testable
+
+<file>
+${fileContnet}
+</file>`;
+}

package/src/agent-harness/discoverCodeBlocks.ts

@@ -0,0 +1,90 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { DocsPage } from './types.js';
+
+const PAGE_EXTENSIONS = new Set(['.md', '.mdx']);
+const IGNORED_DIRECTORIES = new Set(['.git', '.next', 'build', 'dist', 'node_modules']);
+const OPENING_FENCE_PATTERN = /^\s*(`{3,}|~{3,})/;
+
+function countFencedBlocks(content: string): number {
+  const lines = content.split(/\r?\n/);
+  let count = 0;
+  for (let index = 0; index < lines.length; index++) {
+    const opening = OPENING_FENCE_PATTERN.exec(lines[index] ?? '');
+    const marker = opening?.[1];
+    if (!marker) continue;
+    const closingPattern = new RegExp(`^\\s*${marker[0]}{${marker.length},}\\s*$`);
+    let closingIndex = index + 1;
+    while (closingIndex < lines.length && !closingPattern.test(lines[closingIndex] ?? '')) {
+      closingIndex++;
+    }
+    if (closingIndex >= lines.length) break;
+    count++;
+    index = closingIndex;
+  }
+  return count;
+}
+
+async function findSiteRoot(startDirectory: string): Promise<string | null> {
+  const { root } = path.parse(startDirectory);
+  let directory = startDirectory;
+  let previous: string | null = null;
+  while (directory !== previous) {
+    for (const configName of ['docs.json', 'mint.json']) {
+      const stats = await fs.stat(path.join(directory, configName)).catch(() => null);
+      if (stats?.isFile()) return directory;
+    }
+    if (directory === root) return null;
+    previous = directory;
+    directory = path.dirname(directory);
+  }
+  return null;
+}
+
+export async function discoverCodeBlocks(
+  inputPath: string
+): Promise<{ docsRoot: string; pages: DocsPage[] }> {
+  const resolvedPath = path.resolve(inputPath);
+  const stats = await fs.stat(resolvedPath);
+  const baseDirectory = stats.isDirectory() ? resolvedPath : path.dirname(resolvedPath);
+  const docsRoot = (await findSiteRoot(baseDirectory)) ?? baseDirectory;
+
+  const filePaths: string[] = [];
+  if (stats.isDirectory()) {
+    const directories = [resolvedPath];
+    while (directories.length > 0) {
+      const directory = directories.pop();
+      if (!directory) break;
+      for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
+        const entryPath = path.join(directory, entry.name);
+        if (
+          entry.isDirectory() &&
+          !IGNORED_DIRECTORIES.has(entry.name) &&
+          !(entry.name === 'mint-test' && path.basename(directory) === 'tests')
+        ) {
+          directories.push(entryPath);
+        } else if (entry.isFile() && PAGE_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
+          filePaths.push(entryPath);
+        }
+      }
+    }
+  } else if (stats.isFile() && PAGE_EXTENSIONS.has(path.extname(resolvedPath).toLowerCase())) {
+    filePaths.push(resolvedPath);
+  } else {
+    throw new Error(`Expected a docs directory or an MD/MDX file: ${resolvedPath}`);
+  }
+
+  const pages: DocsPage[] = [];
+  for (const filePath of filePaths.sort()) {
+    const content = await fs.readFile(filePath, 'utf8');
+    const blockCount = countFencedBlocks(content);
+    if (blockCount > 0) {
+      pages.push({
+        file: path.relative(docsRoot, filePath).split(path.sep).join('/'),
+        blockCount,
+      });
+    }
+  }
+  return { docsRoot, pages };
+}

package/src/agent-harness/executeCodeBlocks.ts

@@ -0,0 +1,285 @@
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+
+import { mapWithConcurrency } from './mapWithConcurrency.js';
+import type {
+  CommandResult,
+  GeneratedTask,
+  TaskUpdate,
+  ManifestCommand,
+  TaskResult,
+} from './types.js';
+
+const MAX_STREAM_BYTES = 128 * 1024;
+const KILL_GRACE_MS = 2_000;
+const FAILURE_PRECEDENCE = ['failed', 'cancelled'] as const;
+const SECRET_NAME_PATTERN = /(TOKEN|SECRET|PASSWORD|API_KEY|PRIVATE_KEY|CREDENTIAL)/i;
+
+function redactSecrets(value: string): string {
+  let redacted = value;
+  const secrets = Object.entries(process.env)
+    .filter(
+      ([name, secret]) =>
+        SECRET_NAME_PATTERN.test(name) && typeof secret === 'string' && secret.length >= 8
+    )
+    .map(([, secret]) => secret as string)
+    .sort((left, right) => right.length - left.length);
+  for (const secret of secrets) redacted = redacted.split(secret).join('[REDACTED]');
+  return redacted
+    .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, 'Bearer [REDACTED]')
+    .replace(/\b(?:sk|sess)-[A-Za-z0-9._-]{12,}\b/gi, '[REDACTED]');
+}
+
+function outputCollector(): { append: (chunk: Buffer | string) => void; value: () => string } {
+  let output = '';
+  let bytes = 0;
+  let truncated = false;
+  return {
+    append(chunk) {
+      if (truncated) return;
+      const value = chunk.toString();
+      const remaining = MAX_STREAM_BYTES - bytes;
+      if (Buffer.byteLength(value) <= remaining) {
+        output += value;
+        bytes += Buffer.byteLength(value);
+        return;
+      }
+      output += Buffer.from(value).subarray(0, Math.max(0, remaining)).toString();
+      output += '\n[mint test: output truncated]\n';
+      truncated = true;
+    },
+    value: () => output,
+  };
+}
+
+function unrunCommandResult(
+  command: ManifestCommand,
+  phase: 'setup' | 'test',
+  cwd: string,
+  status: 'failed' | 'cancelled',
+  error: string
+): CommandResult {
+  return {
+    id: command.id,
+    phase,
+    status,
+    executable: command.executable,
+    args: command.args,
+    cwd,
+    durationMs: 0,
+    exitCode: null,
+    stdout: '',
+    stderr: '',
+    error,
+  };
+}
+
+function spawnCommand({
+  command,
+  phase,
+  cwd,
+  timeoutMs,
+  signal,
+  docsRootForCommands,
+}: {
+  command: ManifestCommand;
+  phase: 'setup' | 'test';
+  cwd: string;
+  timeoutMs: number;
+  signal: AbortSignal;
+  docsRootForCommands: string;
+}): Promise<CommandResult> {
+  const startedAt = Date.now();
+  const base = {
+    id: command.id,
+    phase,
+    executable: command.executable,
+    args: command.args,
+    cwd,
+  };
+  const commandTimeoutMs = command.timeoutMs ?? timeoutMs;
+
+  return new Promise((resolve) => {
+    const stdout = outputCollector();
+    const stderr = outputCollector();
+    let completed = false;
+    let timedOut = false;
+    let cancelled = false;
+    let forceKillTimer: NodeJS.Timeout | undefined;
+
+    const child = spawn(command.executable, command.args, {
+      cwd,
+      detached: true,
+      env: { ...process.env, MINT_DOCS_ROOT: docsRootForCommands },
+      shell: false,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    child.stdout.on('data', stdout.append);
+    child.stderr.on('data', stderr.append);
+
+    const killTree = (killSignal: NodeJS.Signals) => {
+      if (child.pid) {
+        try {
+          process.kill(-child.pid, killSignal);
+          return;
+        } catch {
+          /* noop */
+        }
+      }
+      child.kill(killSignal);
+    };
+    const kill = () => {
+      killTree('SIGTERM');
+      forceKillTimer = setTimeout(() => killTree('SIGKILL'), KILL_GRACE_MS);
+    };
+    const timeout = setTimeout(() => {
+      timedOut = true;
+      kill();
+    }, commandTimeoutMs);
+    const abort = () => {
+      cancelled = true;
+      kill();
+    };
+    signal.addEventListener('abort', abort, { once: true });
+
+    const finish = (exitCode: number | null, spawnError?: string) => {
+      if (completed) return;
+      completed = true;
+      clearTimeout(timeout);
+      if (forceKillTimer) clearTimeout(forceKillTimer);
+      signal.removeEventListener('abort', abort);
+      const error = cancelled
+        ? 'run cancelled'
+        : timedOut
+          ? `command timed out after ${commandTimeoutMs}ms`
+          : spawnError;
+      resolve({
+        ...base,
+        status: cancelled ? 'cancelled' : exitCode === 0 && !error ? 'passed' : 'failed',
+        durationMs: Date.now() - startedAt,
+        exitCode,
+        stdout: redactSecrets(stdout.value()),
+        stderr: redactSecrets(stderr.value()),
+        ...(error ? { error } : {}),
+      });
+    };
+    child.once('error', (error) => finish(null, error.message));
+    child.once('close', (exitCode) => finish(exitCode));
+  });
+}
+
+export async function executeCodeBlocks({
+  tasks,
+  docsRoot,
+  concurrency,
+  commandTimeoutMs,
+  signal,
+  onTaskUpdate,
+}: {
+  tasks: GeneratedTask[];
+  docsRoot: string;
+  concurrency: number;
+  commandTimeoutMs: number;
+  signal: AbortSignal;
+  onTaskUpdate: (update: TaskUpdate) => void;
+}): Promise<TaskResult[]> {
+  return mapWithConcurrency(tasks, concurrency, async (task) => {
+    const base = {
+      id: task.id,
+      agent: task.agent,
+      file: task.file,
+      directory: task.directory,
+      attempts: task.attempts,
+      tokens: task.tokens,
+    };
+    const manifest = task.manifest;
+    if (!manifest) {
+      if (!task.error) {
+        onTaskUpdate({
+          id: task.id,
+          agent: task.agent,
+          file: task.file,
+          phase: 'not_testable',
+          tokens: task.tokens,
+        });
+        return { ...base, status: 'passed' as const, commands: [] };
+      }
+      const status = signal.aborted ? ('cancelled' as const) : ('agent_error' as const);
+      onTaskUpdate({
+        id: task.id,
+        agent: task.agent,
+        file: task.file,
+        phase: status,
+        tokens: task.tokens,
+      });
+      return {
+        ...base,
+        status,
+        error: task.error,
+        commands: [],
+      };
+    }
+    onTaskUpdate({
+      id: task.id,
+      agent: task.agent,
+      file: task.file,
+      phase: 'running',
+      tokens: task.tokens,
+    });
+
+    const commands: CommandResult[] = [];
+    let setupFailure: 'failed' | 'cancelled' | undefined;
+
+    const run = async (
+      command: ManifestCommand,
+      phase: 'setup' | 'test'
+    ): Promise<CommandResult> => {
+      const cwd = path.resolve(task.directory, command.cwd ?? '.');
+      if (signal.aborted) {
+        return unrunCommandResult(command, phase, cwd, 'cancelled', 'run cancelled');
+      }
+      if (phase === 'test' && setupFailure) {
+        return unrunCommandResult(
+          command,
+          phase,
+          cwd,
+          setupFailure,
+          `a setup command ${setupFailure === 'cancelled' ? 'was cancelled' : 'failed'}`
+        );
+      }
+      return spawnCommand({
+        command,
+        phase,
+        cwd,
+        timeoutMs: commandTimeoutMs,
+        signal,
+        docsRootForCommands: docsRoot,
+      });
+    };
+
+    for (const command of manifest.setupCommands) {
+      const result = await run(command, 'setup');
+      commands.push(result);
+      if (result.status !== 'passed') {
+        setupFailure = result.status;
+        break;
+      }
+    }
+    for (const command of manifest.testCommands) {
+      commands.push(await run(command, 'test'));
+    }
+
+    const status =
+      FAILURE_PRECEDENCE.find((candidate) =>
+        commands.some((command) => command.status === candidate)
+      ) ?? 'passed';
+    onTaskUpdate({
+      id: task.id,
+      agent: task.agent,
+      file: task.file,
+      phase: status,
+      tokens: task.tokens,
+    });
+    return { ...base, status, commands };
+  });
+}

package/src/agent-harness/filterSelectedPages.ts

@@ -0,0 +1,14 @@
+import path from 'node:path';
+
+import type { DocsPage } from './types.js';
+
+export function filterSelectedPages(pages: DocsPage[], selectedFiles: string[]): DocsPage[] {
+  const selected = new Set(
+    selectedFiles.map((file) => file.split(path.sep).join('/').replace(/^\.\//, ''))
+  );
+  const known = new Set(pages.map((page) => page.file));
+  for (const file of selected) {
+    if (!known.has(file)) throw new Error(`Selected docs file was not found: ${file}`);
+  }
+  return pages.filter((page) => selected.has(page.file));
+}