@mintlify/cli 4.0.1437 → 4.0.1438

package/src/agent-harness/tasks/checkTestability.ts

@@ -0,0 +1,247 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import z from 'zod';
+
+import {
+  buildCheckPrompt,
+  CHECK_JSON_INSTRUCTION,
+  CHECK_SYSTEM_PROMPT,
+  CHECK_TOOL_INSTRUCTION,
+} from '../buildTaskPrompt.js';
+import { loadClaudeAgentSdk, loadCodexSdk } from '../loadAgentSdk.js';
+import {
+  GeneratedTask,
+  HarnessAgent,
+  RunCodeTestsOptions,
+  TaskUpdate,
+  TestCheck,
+  testCheckSchema,
+} from '../types.js';
+
+const CHECK_FILENAME = 'mint-test-check.json';
+
+const CHECK_MODEL = 'claude-haiku-4-5-20251001';
+
+async function readValidCheck(directory: string): Promise<TestCheck | null> {
+  try {
+    const check = testCheckSchema.parse(
+      JSON.parse(await fs.readFile(path.join(directory, CHECK_FILENAME), 'utf8'))
+    );
+    return validateCheck(check).length === 0 ? check : null;
+  } catch {
+    return null;
+  }
+}
+
+function validateCheck(check: TestCheck): string[] {
+  return !check.testable && !check.reason ? ['Not-testable page needs a reason'] : [];
+}
+
+function parseJsonReply(reply: string): unknown {
+  const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(reply);
+  const raw = fenced?.[1] ?? reply;
+  const start = raw.indexOf('{');
+  const end = raw.lastIndexOf('}');
+  if (start === -1 || end === -1) throw new Error('Reply contained no JSON object');
+  return JSON.parse(raw.slice(start, end + 1));
+}
+
+async function runTestabilityCheck({
+  file,
+  content,
+  agent,
+  outputPath,
+  signal,
+}: {
+  file: string;
+  content: string;
+  agent: HarnessAgent;
+  outputPath: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; check: TestCheck }> {
+  const prompt = buildCheckPrompt(file, content);
+  const workingDirectory = path.dirname(outputPath);
+  let tokens = 0;
+  let raw: unknown;
+  if (agent === 'claude') {
+    const reply = await promptClaudeCheck({ prompt, workingDirectory, signal });
+    tokens = reply.tokens;
+    if (reply.captured === null) throw new Error('agent never called report_testability');
+    raw = reply.captured;
+  } else {
+    const reply = await promptCodex({
+      prompt,
+      systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_JSON_INSTRUCTION,
+      workingDirectory,
+      signal,
+    });
+    tokens = reply.tokens;
+    raw = parseJsonReply(reply.result);
+  }
+  const check = testCheckSchema.parse(raw);
+  const errors = validateCheck(check);
+  if (errors.length > 0) throw new Error(errors.join('\n'));
+  await fs.writeFile(outputPath, `${JSON.stringify(check, null, 2)}\n`);
+  return { tokens, check };
+}
+
+export const checkFileTestability = async (
+  task: GeneratedTask,
+  { onTaskUpdate, signal }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'>
+): Promise<GeneratedTask> => {
+  const emit = (phase: TaskUpdate['phase'], tokens: number) =>
+    onTaskUpdate({
+      id: task.id,
+      agent: task.agent,
+      file: task.file,
+      phase,
+      tokens,
+    });
+  let attempts = 0;
+  let tokens = 0;
+  let lastError = 'run cancelled';
+
+  let check = await readValidCheck(task.directory);
+  if (!check) {
+    emit('checking', tokens);
+    const content = await fs.readFile(path.join(task.docsRoot, task.file), 'utf8');
+    for (let attempt = 1; attempt <= 2 && !signal.aborted && !check; attempt++) {
+      attempts = attempt;
+      try {
+        const reply = await runTestabilityCheck({
+          file: task.file,
+          content,
+          agent: task.agent,
+          outputPath: path.join(task.directory, CHECK_FILENAME),
+          signal,
+        });
+        tokens += reply.tokens;
+        check = reply.check;
+      } catch (error) {
+        lastError = error instanceof Error ? error.message : String(error);
+      }
+    }
+    if (!check) {
+      if (signal.aborted) {
+        emit('cancelled', tokens);
+        return { ...task, attempts, tokens, error: 'run cancelled' };
+      }
+      emit('agent_error', tokens);
+      return { ...task, attempts, tokens, error: lastError };
+    }
+  }
+
+  if (!check.testable) {
+    emit('not_testable', tokens);
+    return { ...task, attempts, tokens, testable: false };
+  }
+  emit('checked', tokens);
+  return { ...task, attempts, tokens, testable: true };
+};
+
+async function promptClaudeCheck({
+  prompt,
+  workingDirectory,
+  signal,
+}: {
+  prompt: string;
+  workingDirectory: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; captured: unknown }> {
+  const { query, tool, createSdkMcpServer } = await loadClaudeAgentSdk();
+  const abortController = new AbortController();
+  const abort = () => abortController.abort(signal.reason);
+  if (signal.aborted) abort();
+  else signal.addEventListener('abort', abort, { once: true });
+
+  let captured: unknown = null;
+  const server = createSdkMcpServer({
+    name: 'mint-test',
+    tools: [
+      tool(
+        'report_testability',
+        'Report whether the documentation page is testable. Call exactly once.',
+        {
+          version: z.literal(1),
+          testable: z.boolean(),
+          reason: z.string().optional(),
+        },
+        async (input) => {
+          captured = input;
+          return { content: [{ type: 'text', text: 'recorded' }] };
+        }
+      ),
+    ],
+  });
+
+  const session = query({
+    prompt,
+    options: {
+      abortController,
+      allowedTools: ['mcp__mint-test__report_testability'],
+      cwd: workingDirectory,
+      mcpServers: { 'mint-test': server },
+      model: CHECK_MODEL,
+      permissionMode: 'dontAsk',
+      persistSession: false,
+      settingSources: [],
+      strictMcpConfig: true,
+      systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_TOOL_INSTRUCTION,
+      tools: [],
+    },
+  });
+
+  try {
+    for await (const message of session) {
+      if (message.type !== 'result') continue;
+      if (message.subtype !== 'success' || message.is_error) {
+        const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
+        throw new Error(details || `Claude stopped with ${message.subtype}`);
+      }
+      const usage = message.usage;
+      return {
+        tokens:
+          usage.input_tokens +
+          usage.output_tokens +
+          usage.cache_creation_input_tokens +
+          usage.cache_read_input_tokens,
+        captured,
+      };
+    }
+    throw new Error('Claude finished without returning a result');
+  } finally {
+    signal.removeEventListener('abort', abort);
+    session.close();
+  }
+}
+
+async function promptCodex({
+  prompt,
+  systemPrompt,
+  workingDirectory,
+  signal,
+}: {
+  prompt: string;
+  systemPrompt: string;
+  workingDirectory: string;
+  signal: AbortSignal;
+}): Promise<{ tokens: number; result: string }> {
+  const { Codex } = await loadCodexSdk();
+  const thread = new Codex().startThread({
+    approvalPolicy: 'never',
+    networkAccessEnabled: true,
+    sandboxMode: 'workspace-write',
+    skipGitRepoCheck: true,
+    workingDirectory,
+  });
+  const result = await thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
+  const usage = result.usage;
+  const tokens = usage
+    ? usage.input_tokens +
+      usage.cached_input_tokens +
+      usage.cache_write_input_tokens +
+      usage.output_tokens +
+      usage.reasoning_output_tokens
+    : 0;
+  return { tokens, result: result.finalResponse };
+}

package/src/agent-harness/types.ts

@@ -0,0 +1,152 @@
+import { z } from 'zod';
+
+export type HarnessAgent = 'claude' | 'codex';
+
+export interface DocsPage {
+  file: string;
+  blockCount: number;
+}
+
+export type CommandStatus = 'passed' | 'failed' | 'cancelled';
+
+export interface CommandResult {
+  id: string;
+  phase: 'setup' | 'test';
+  status: CommandStatus;
+  executable: string;
+  args: string[];
+  cwd: string;
+  durationMs: number;
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+  error?: string;
+}
+
+export type TaskStatus = 'passed' | 'failed' | 'agent_error' | 'cancelled';
+
+export interface TaskResult {
+  id: string;
+  agent: HarnessAgent;
+  file: string;
+  directory: string;
+  attempts: number;
+  tokens: number;
+  status: TaskStatus;
+  error?: string;
+  commands: CommandResult[];
+}
+
+export interface CodeTestReport {
+  version: 1;
+  runId: string;
+  status: 'passed' | 'completed' | 'failed' | 'cancelled';
+  docsRoot: string;
+  outputDirectory: string;
+  reportPath: string;
+  agents: HarnessAgent[];
+  selectedFiles: string[];
+  startedAt: string;
+  completedAt: string;
+  durationMs: number;
+  tasks: TaskResult[];
+}
+
+export type TaskPhase =
+  | 'checking'
+  | 'checked'
+  | 'not_testable'
+  | 'generating'
+  | 'generated'
+  | 'agent_error'
+  | 'running'
+  | 'passed'
+  | 'failed'
+  | 'cancelled';
+
+export interface TaskUpdate {
+  id: string;
+  agent: HarnessAgent;
+  file: string;
+  phase: TaskPhase;
+  tokens: number;
+}
+
+export interface RunSave {
+  version: 1;
+  runId: string;
+  agents: HarnessAgent[];
+  model: string;
+  selectedFiles: string[];
+  createdAt: string;
+}
+
+export interface RunCodeTestsOptions {
+  path: string;
+  agents: HarnessAgent[];
+  model: string;
+  selectedFiles: string[];
+  outputDirectory: string;
+  runId: string | null;
+  concurrency: number;
+  commandTimeoutMs: number;
+  signal: AbortSignal;
+  onTaskUpdate: (update: TaskUpdate) => void;
+}
+
+const identifier = z.string().trim().min(1).max(120);
+
+const commandSchema = z
+  .object({
+    id: identifier,
+    executable: z.string().trim().min(1).max(512),
+    args: z.array(z.string().max(10_000)).max(100),
+    cwd: z.string().max(1_024).optional(),
+    timeoutMs: z
+      .number()
+      .int()
+      .min(1_000)
+      .max(30 * 60 * 1_000)
+      .optional(),
+  })
+  .strict();
+
+export const manifestSchema = z
+  .object({
+    version: z.literal(1),
+    generatedFiles: z.array(z.string().min(1).max(1_024)).max(1_000),
+    setupCommands: z.array(commandSchema).max(50),
+    testCommands: z.array(commandSchema).max(200),
+  })
+  .strict();
+
+export type GeneratedTestManifest = z.infer<typeof manifestSchema>;
+
+export const testCheckSchema = z
+  .object({
+    version: z.literal(1),
+    testable: z.boolean(),
+    reason: z
+      .string()
+      .max(2_000)
+      .optional()
+      .transform((value) => (value?.trim() ? value.trim() : undefined)),
+  })
+  .strict();
+
+export type TestCheck = z.infer<typeof testCheckSchema>;
+
+export type ManifestCommand = GeneratedTestManifest['testCommands'][number];
+
+export interface GeneratedTask {
+  id: string;
+  agent: HarnessAgent;
+  docsRoot: string;
+  file: string;
+  directory: string;
+  attempts: number;
+  tokens: number;
+  testable?: boolean;
+  manifest?: GeneratedTestManifest;
+  error?: string;
+}

package/src/mintTest.tsx

@@ -1,3 +1,5 @@
+import path from 'node:path';
+
 import {
   DEFAULT_COMMAND_TIMEOUT_MS,
   DEFAULT_CONCURRENCY,
@@ -6,9 +8,7 @@ import {
   loadRunSave,
   runCodeTests,
   type RunSave,
-} from '@mintlify/agent-harness';
-import path from 'node:path';
-
+} from './agent-harness/index.js';
 import { CMD_EXEC_PATH, findDocsRoot, isAI, terminate } from './helpers.js';
 import { getAccessToken } from './keyring.js';
 import { fileInNavPaths, loadDocsNavScope } from './mintTestNav.js';

package/src/mintTestTree.ts

@@ -1,4 +1,4 @@
-import type { DocsPage } from '@mintlify/agent-harness';
+import type { DocsPage } from './agent-harness/index.js';
 
 export type ScopeSelectionState = 'all' | 'some' | 'none';
 

package/src/mintTestUi.tsx

@@ -1,3 +1,7 @@
+import { Box, render, Text, useInput } from 'ink';
+import path from 'node:path';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
+
 import {
   DEFAULT_COMMAND_TIMEOUT_MS,
   DEFAULT_CONCURRENCY,
@@ -10,11 +14,7 @@ import {
   type TaskPhase,
   type TaskResult,
   type TaskUpdate,
-} from '@mintlify/agent-harness';
-import { Box, render, Text, useInput } from 'ink';
-import path from 'node:path';
-import React, { useEffect, useMemo, useRef, useState } from 'react';
-
+} from './agent-harness/index.js';
 import { toggleScopeSelection, visibleScopeRows, type ScopeTreeRow } from './mintTestTree.js';
 
 type Screen = 'resume' | 'harnesses' | 'scope' | 'running' | 'report' | 'error';