@mintlify/cli 4.0.1434 → 4.0.1438

package/src/agent-harness/tasks/checkTestability.ts

@@ -0,0 +1,247 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import z from 'zod';
+
+import {
+  buildCheckPrompt,
+  CHECK_JSON_INSTRUCTION,
+  CHECK_SYSTEM_PROMPT,
+  CHECK_TOOL_INSTRUCTION,
+} from '../buildTaskPrompt.js';
+import { loadClaudeAgentSdk, loadCodexSdk } from '../loadAgentSdk.js';
+import {
+  GeneratedTask,
+  HarnessAgent,
+  RunCodeTestsOptions,
+  TaskUpdate,
+  TestCheck,
+  testCheckSchema,
+} from '../types.js';
+
+const CHECK_FILENAME = 'mint-test-check.json';
+
+const CHECK_MODEL = 'claude-haiku-4-5-20251001';
+
+async function readValidCheck(directory: string): Promise<TestCheck | null> {
+  try {
+    const check = testCheckSchema.parse(
+      JSON.parse(await fs.readFile(path.join(directory, CHECK_FILENAME), 'utf8'))
+    );
+    return validateCheck(check).length === 0 ? check : null;
+  } catch {
+    return null;
+  }
+}
+
+function validateCheck(check: TestCheck): string[] {
+  return !check.testable && !check.reason ? ['Not-testable page needs a reason'] : [];
+}
+
+function parseJsonReply(reply: string): unknown {
+  const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(reply);
+  const raw = fenced?.[1] ?? reply;
+  const start = raw.indexOf('{');
+  const end = raw.lastIndexOf('}');
+  if (start === -1 || end === -1) throw new Error('Reply contained no JSON object');
+  return JSON.parse(raw.slice(start, end + 1));
+}
+
+async function runTestabilityCheck({
+  file,
+  content,
+  agent,
+  outputPath,
+  signal,
+}: {
+  file: string;
+  content: string;
+  agent: HarnessAgent;
+  outputPath: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; check: TestCheck }> {
+  const prompt = buildCheckPrompt(file, content);
+  const workingDirectory = path.dirname(outputPath);
+  let tokens = 0;
+  let raw: unknown;
+  if (agent === 'claude') {
+    const reply = await promptClaudeCheck({ prompt, workingDirectory, signal });
+    tokens = reply.tokens;
+    if (reply.captured === null) throw new Error('agent never called report_testability');
+    raw = reply.captured;
+  } else {
+    const reply = await promptCodex({
+      prompt,
+      systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_JSON_INSTRUCTION,
+      workingDirectory,
+      signal,
+    });
+    tokens = reply.tokens;
+    raw = parseJsonReply(reply.result);
+  }
+  const check = testCheckSchema.parse(raw);
+  const errors = validateCheck(check);
+  if (errors.length > 0) throw new Error(errors.join('\n'));
+  await fs.writeFile(outputPath, `${JSON.stringify(check, null, 2)}\n`);
+  return { tokens, check };
+}
+
+export const checkFileTestability = async (
+  task: GeneratedTask,
+  { onTaskUpdate, signal }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'>
+): Promise<GeneratedTask> => {
+  const emit = (phase: TaskUpdate['phase'], tokens: number) =>
+    onTaskUpdate({
+      id: task.id,
+      agent: task.agent,
+      file: task.file,
+      phase,
+      tokens,
+    });
+  let attempts = 0;
+  let tokens = 0;
+  let lastError = 'run cancelled';
+
+  let check = await readValidCheck(task.directory);
+  if (!check) {
+    emit('checking', tokens);
+    const content = await fs.readFile(path.join(task.docsRoot, task.file), 'utf8');
+    for (let attempt = 1; attempt <= 2 && !signal.aborted && !check; attempt++) {
+      attempts = attempt;
+      try {
+        const reply = await runTestabilityCheck({
+          file: task.file,
+          content,
+          agent: task.agent,
+          outputPath: path.join(task.directory, CHECK_FILENAME),
+          signal,
+        });
+        tokens += reply.tokens;
+        check = reply.check;
+      } catch (error) {
+        lastError = error instanceof Error ? error.message : String(error);
+      }
+    }
+    if (!check) {
+      if (signal.aborted) {
+        emit('cancelled', tokens);
+        return { ...task, attempts, tokens, error: 'run cancelled' };
+      }
+      emit('agent_error', tokens);
+      return { ...task, attempts, tokens, error: lastError };
+    }
+  }
+
+  if (!check.testable) {
+    emit('not_testable', tokens);
+    return { ...task, attempts, tokens, testable: false };
+  }
+  emit('checked', tokens);
+  return { ...task, attempts, tokens, testable: true };
+};
+
+async function promptClaudeCheck({
+  prompt,
+  workingDirectory,
+  signal,
+}: {
+  prompt: string;
+  workingDirectory: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; captured: unknown }> {
+  const { query, tool, createSdkMcpServer } = await loadClaudeAgentSdk();
+  const abortController = new AbortController();
+  const abort = () => abortController.abort(signal.reason);
+  if (signal.aborted) abort();
+  else signal.addEventListener('abort', abort, { once: true });
+
+  let captured: unknown = null;
+  const server = createSdkMcpServer({
+    name: 'mint-test',
+    tools: [
+      tool(
+        'report_testability',
+        'Report whether the documentation page is testable. Call exactly once.',
+        {
+          version: z.literal(1),
+          testable: z.boolean(),
+          reason: z.string().optional(),
+        },
+        async (input) => {
+          captured = input;
+          return { content: [{ type: 'text', text: 'recorded' }] };
+        }
+      ),
+    ],
+  });
+
+  const session = query({
+    prompt,
+    options: {
+      abortController,
+      allowedTools: ['mcp__mint-test__report_testability'],
+      cwd: workingDirectory,
+      mcpServers: { 'mint-test': server },
+      model: CHECK_MODEL,
+      permissionMode: 'dontAsk',
+      persistSession: false,
+      settingSources: [],
+      strictMcpConfig: true,
+      systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_TOOL_INSTRUCTION,
+      tools: [],
+    },
+  });
+
+  try {
+    for await (const message of session) {
+      if (message.type !== 'result') continue;
+      if (message.subtype !== 'success' || message.is_error) {
+        const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
+        throw new Error(details || `Claude stopped with ${message.subtype}`);
+      }
+      const usage = message.usage;
+      return {
+        tokens:
+          usage.input_tokens +
+          usage.output_tokens +
+          usage.cache_creation_input_tokens +
+          usage.cache_read_input_tokens,
+        captured,
+      };
+    }
+    throw new Error('Claude finished without returning a result');
+  } finally {
+    signal.removeEventListener('abort', abort);
+    session.close();
+  }
+}
+
+async function promptCodex({
+  prompt,
+  systemPrompt,
+  workingDirectory,
+  signal,
+}: {
+  prompt: string;
+  systemPrompt: string;
+  workingDirectory: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; result: string }> {
+  const { Codex } = await loadCodexSdk();
+  const thread = new Codex().startThread({
+    approvalPolicy: 'never',
+    networkAccessEnabled: true,
+    sandboxMode: 'workspace-write',
+    skipGitRepoCheck: true,
+    workingDirectory,
+  });
+  const result = await thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
+  const usage = result.usage;
+  const tokens = usage
+    ? usage.input_tokens +
+      usage.cached_input_tokens +
+      usage.cache_write_input_tokens +
+      usage.output_tokens +
+      usage.reasoning_output_tokens
+    : 0;
+  return { tokens, result: result.finalResponse };
+}

package/src/agent-harness/types.ts

@@ -0,0 +1,152 @@
+import { z } from 'zod';
+
+export type HarnessAgent = 'claude' | 'codex';
+
+export interface DocsPage {
+  file: string;
+  blockCount: number;
+}
+
+export type CommandStatus = 'passed' | 'failed' | 'cancelled';
+
+export interface CommandResult {
+  id: string;
+  phase: 'setup' | 'test';
+  status: CommandStatus;
+  executable: string;
+  args: string[];
+  cwd: string;
+  durationMs: number;
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+  error?: string;
+}
+
+export type TaskStatus = 'passed' | 'failed' | 'agent_error' | 'cancelled';
+
+export interface TaskResult {
+  id: string;
+  agent: HarnessAgent;
+  file: string;
+  directory: string;
+  attempts: number;
+  tokens: number;
+  status: TaskStatus;
+  error?: string;
+  commands: CommandResult[];
+}
+
+export interface CodeTestReport {
+  version: 1;
+  runId: string;
+  status: 'passed' | 'completed' | 'failed' | 'cancelled';
+  docsRoot: string;
+  outputDirectory: string;
+  reportPath: string;
+  agents: HarnessAgent[];
+  selectedFiles: string[];
+  startedAt: string;
+  completedAt: string;
+  durationMs: number;
+  tasks: TaskResult[];
+}
+
+export type TaskPhase =
+  | 'checking'
+  | 'checked'
+  | 'not_testable'
+  | 'generating'
+  | 'generated'
+  | 'agent_error'
+  | 'running'
+  | 'passed'
+  | 'failed'
+  | 'cancelled';
+
+export interface TaskUpdate {
+  id: string;
+  agent: HarnessAgent;
+  file: string;
+  phase: TaskPhase;
+  tokens: number;
+}
+
+export interface RunSave {
+  version: 1;
+  runId: string;
+  agents: HarnessAgent[];
+  model: string;
+  selectedFiles: string[];
+  createdAt: string;
+}
+
+export interface RunCodeTestsOptions {
+  path: string;
+  agents: HarnessAgent[];
+  model: string;
+  selectedFiles: string[];
+  outputDirectory: string;
+  runId: string | null;
+  concurrency: number;
+  commandTimeoutMs: number;
+  signal: AbortSignal;
+  onTaskUpdate: (update: TaskUpdate) => void;
+}
+
+const identifier = z.string().trim().min(1).max(120);
+
+const commandSchema = z
+  .object({
+    id: identifier,
+    executable: z.string().trim().min(1).max(512),
+    args: z.array(z.string().max(10_000)).max(100),
+    cwd: z.string().max(1_024).optional(),
+    timeoutMs: z
+      .number()
+      .int()
+      .min(1_000)
+      .max(30 * 60 * 1_000)
+      .optional(),
+  })
+  .strict();
+
+export const manifestSchema = z
+  .object({
+    version: z.literal(1),
+    generatedFiles: z.array(z.string().min(1).max(1_024)).max(1_000),
+    setupCommands: z.array(commandSchema).max(50),
+    testCommands: z.array(commandSchema).max(200),
+  })
+  .strict();
+
+export type GeneratedTestManifest = z.infer<typeof manifestSchema>;
+
+export const testCheckSchema = z
+  .object({
+    version: z.literal(1),
+    testable: z.boolean(),
+    reason: z
+      .string()
+      .max(2_000)
+      .optional()
+      .transform((value) => (value?.trim() ? value.trim() : undefined)),
+  })
+  .strict();
+
+export type TestCheck = z.infer<typeof testCheckSchema>;
+
+export type ManifestCommand = GeneratedTestManifest['testCommands'][number];
+
+export interface GeneratedTask {
+  id: string;
+  agent: HarnessAgent;
+  docsRoot: string;
+  file: string;
+  directory: string;
+  attempts: number;
+  tokens: number;
+  testable?: boolean;
+  manifest?: GeneratedTestManifest;
+  error?: string;
+}

package/src/cli.tsx

@@ -44,6 +44,7 @@ import { login } from './login.js';
 import { logout } from './logout.js';
 import { mdxLinter } from './mdxLinter.js';
 import { createTelemetryMiddleware } from './middlewares/telemetryMiddleware.js';
+import { mintTestHandler } from './mintTest.js';
 import { checkOpenApiFile, getOpenApiFilenamesFromDocsConfig } from './openApiCheck.js';
 import { resolveExplicitFiles } from './resolveFiles.js';
 import { scoreHandler } from './score/index.js';
@@ -85,15 +86,18 @@ const showWelcome = async (packageName: string): Promise<void> => {
 };
 
 export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
-  if (shouldShowWelcome(hideBin(process.argv))) {
+  const commandArguments = hideBin(process.argv);
+  if (shouldShowWelcome(commandArguments)) {
     return showWelcome(packageName);
   }
 
   const telemetryMiddleware = createTelemetryMiddleware();
-  render(<Logs />);
+  if (commandArguments.find((argument) => !argument.startsWith('-')) !== 'test') {
+    render(<Logs />);
+  }
 
   return (
-    yargs(hideBin(process.argv))
+    yargs(commandArguments)
       .scriptName(packageName)
       .option('telemetry', {
         type: 'boolean',
@@ -239,6 +243,14 @@ export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
           });
         }
       )
+      .command(
+        'test',
+        'Test runnable code blocks with a local coding agent',
+        (yargs) => yargs.usage('usage: mint test'),
+        async () => {
+          await mintTestHandler();
+        }
+      )
       .command(
         'export',
         'Export a static site for air-gapped deployment',
@@ -762,12 +774,6 @@ export const cli = ({ packageName = 'mint' }: { packageName?: string }) => {
         () => undefined,
         comingSoon('ai', packageName)
       )
-      .command(
-        'test',
-        '[Coming soon] Test your documentation (run mint test to vote)',
-        () => undefined,
-        comingSoon('test', packageName)
-      )
       .command(
         'mcp',
         '[Coming soon] MCP server for documentation (run mint mcp to vote)',

package/src/mintTest.tsx

@@ -0,0 +1,102 @@
+import path from 'node:path';
+
+import {
+  DEFAULT_COMMAND_TIMEOUT_MS,
+  DEFAULT_CONCURRENCY,
+  clearRunSave,
+  discoverCodeBlocks,
+  loadRunSave,
+  runCodeTests,
+  type RunSave,
+} from './agent-harness/index.js';
+import { CMD_EXEC_PATH, findDocsRoot, isAI, terminate } from './helpers.js';
+import { getAccessToken } from './keyring.js';
+import { fileInNavPaths, loadDocsNavScope } from './mintTestNav.js';
+import { countReport, formatDuration, runMintTestUi } from './mintTestUi.js';
+import { getCliStatus } from './status.js';
+
+export async function mintTestHandler(): Promise<void> {
+  try {
+    const accessToken = await getAccessToken().catch(() => null);
+    if (!accessToken) {
+      process.stderr.write('not logged in. Run `mint login` to authenticate.\n');
+      await terminate(1);
+      return;
+    }
+    const cliStatus = await getCliStatus(accessToken);
+    if (cliStatus?.org.name.toLowerCase() !== 'mintlify') {
+      process.stderr.write('mint test is not available for your organization.\n');
+      await terminate(1);
+      return;
+    }
+    const targetPath = await findDocsRoot(CMD_EXEC_PATH);
+    const interactive =
+      process.stdin.isTTY === true &&
+      process.stdout.isTTY === true &&
+      !isAI() &&
+      !process.env.CI &&
+      process.env.CLI_TEST_MODE !== 'true';
+
+    const { docsRoot, pages: discoveredPages } = await discoverCodeBlocks(targetPath);
+    const scope = await loadDocsNavScope(docsRoot);
+    const pages = scope
+      ? discoveredPages.filter((page) => fileInNavPaths(page.file, scope.navPaths))
+      : discoveredPages;
+    const autoSelectedFiles = (
+      scope ? pages.filter((page) => fileInNavPaths(page.file, scope.autoSelectedPaths)) : pages
+    ).map((page) => page.file);
+    const totalPages = scope ? scope.navPaths.size : pages.length;
+    const outputDirectory = path.join(docsRoot, 'tests', 'mint-test');
+
+    let save: RunSave | null = await loadRunSave(outputDirectory);
+    if (save) {
+      const known = new Set(pages.map((page) => page.file));
+      const selectedFiles = save.selectedFiles.filter((file) => known.has(file));
+      if (selectedFiles.length === 0) {
+        await clearRunSave(outputDirectory);
+        save = null;
+      } else {
+        save = { ...save, selectedFiles };
+      }
+    }
+
+    if (interactive) {
+      const exitCode = await runMintTestUi({
+        targetPath,
+        docsRoot,
+        pages,
+        autoSelectedFiles,
+        totalPages,
+        save,
+      });
+      await terminate(exitCode);
+      return;
+    }
+
+    if (save) process.stdout.write(`resuming run ${save.runId}\n`);
+    const report = await runCodeTests({
+      path: targetPath,
+      agents: save?.agents ?? ['claude'],
+      model: save?.model ?? 'claude-opus-4-8',
+      selectedFiles: save?.selectedFiles ?? autoSelectedFiles,
+      outputDirectory,
+      runId: save?.runId ?? null,
+      concurrency: DEFAULT_CONCURRENCY,
+      commandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
+      signal: new AbortController().signal,
+      onTaskUpdate: () => {},
+    });
+    const counts = countReport(report.tasks);
+    process.stdout.write(
+      `mint test ${report.status}: ${counts.passed} passed, ${counts.failed} failed, ${counts.agentErrors} agent errors\n`
+    );
+    process.stdout.write(`finished in ${formatDuration(report.durationMs)}\n`);
+    process.stdout.write(`report: ${report.reportPath}\n`);
+    await terminate(report.status === 'passed' ? 0 : 1);
+  } catch (error) {
+    process.stderr.write(
+      `mint test failed: ${error instanceof Error ? error.message : 'unknown error'}\n`
+    );
+    await terminate(1);
+  }
+}

package/src/mintTestNav.ts

@@ -0,0 +1,31 @@
+import { getAllPathsInDocsNav } from '@mintlify/common';
+import { getConfigObj } from '@mintlify/prebuild';
+import type { DocsConfig, NavigationConfig } from '@mintlify/validation';
+
+export interface DocsNavScope {
+  navPaths: Set<string>;
+  autoSelectedPaths: Set<string>;
+}
+
+export async function loadDocsNavScope(docsRoot: string): Promise<DocsNavScope | null> {
+  const config = (await getConfigObj(docsRoot, 'docs').catch(() => null)) as DocsConfig | null;
+  const navigation = config?.navigation;
+  if (!navigation) return null;
+  const navPaths = new Set(getAllPathsInDocsNav(navigation));
+  if (navPaths.size === 0) return null;
+  const firstLanguage =
+    'languages' in navigation ? (navigation.languages[0] as NavigationConfig | undefined) : null;
+  const autoSelectedPaths = firstLanguage ? new Set(getAllPathsInDocsNav(firstLanguage)) : navPaths;
+  return { navPaths, autoSelectedPaths };
+}
+
+export function navCandidatesForFile(file: string): string[] {
+  const base = file.replace(/\.mdx?$/i, '');
+  const candidates = [base];
+  if (base.endsWith('/index')) candidates.push(base.slice(0, -'/index'.length));
+  return candidates;
+}
+
+export function fileInNavPaths(file: string, paths: Set<string>): boolean {
+  return navCandidatesForFile(file).some((candidate) => paths.has(candidate));
+}