@mintlify/cli 4.0.1437 → 4.0.1439

package/bin/agent-harness/generateTestCode.js

@@ -0,0 +1,293 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { SYSTEM_PROMPT, buildTaskPrompt } from './buildTaskPrompt.js';
+import { loadClaudeAgentSdk, loadCodexSdk } from './loadAgentSdk.js';
+import { mapWithConcurrency } from './mapWithConcurrency.js';
+import { checkFileTestability } from './tasks/checkTestability.js';
+import { manifestSchema, } from './types.js';
+const MANIFEST_FILENAME = 'mint-test.json';
+const SAFE_EXECUTABLES = new Set([
+    'bun',
+    'bunx',
+    'bundle',
+    'cargo',
+    'composer',
+    'deno',
+    'dotnet',
+    'go',
+    'gradle',
+    'java',
+    'javac',
+    'mvn',
+    'node',
+    'npm',
+    'npx',
+    'php',
+    'pip',
+    'pip3',
+    'pnpm',
+    'python',
+    'python3',
+    'pytest',
+    'ruby',
+    'rustc',
+    'uv',
+    'vitest',
+    'yarn',
+]);
+function promptClaude(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ prompt, systemPrompt, model, workingDirectory, docsRoot, tools, signal, }) {
+        var _b, e_1, _c, _d;
+        const { query } = yield loadClaudeAgentSdk();
+        const abortController = new AbortController();
+        const abort = () => abortController.abort(signal.reason);
+        if (signal.aborted)
+            abort();
+        else
+            signal.addEventListener('abort', abort, { once: true });
+        const session = query({
+            prompt,
+            options: {
+                abortController,
+                additionalDirectories: [docsRoot],
+                allowedTools: tools,
+                cwd: workingDirectory,
+                model,
+                permissionMode: 'dontAsk',
+                persistSession: false,
+                settingSources: [],
+                strictMcpConfig: true,
+                systemPrompt,
+                tools,
+            },
+        });
+        try {
+            try {
+                for (var _e = true, session_1 = __asyncValues(session), session_1_1; session_1_1 = yield session_1.next(), _b = session_1_1.done, !_b; _e = true) {
+                    _d = session_1_1.value;
+                    _e = false;
+                    const message = _d;
+                    if (message.type !== 'result')
+                        continue;
+                    if (message.subtype !== 'success' || message.is_error) {
+                        const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
+                        throw new Error(details || `Claude stopped with ${message.subtype}`);
+                    }
+                    const usage = message.usage;
+                    return {
+                        tokens: usage.input_tokens +
+                            usage.output_tokens +
+                            usage.cache_creation_input_tokens +
+                            usage.cache_read_input_tokens,
+                        result: message.result,
+                    };
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_e && !_b && (_c = session_1.return)) yield _c.call(session_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            throw new Error('Claude finished without returning a result');
+        }
+        finally {
+            signal.removeEventListener('abort', abort);
+            session.close();
+        }
+    });
+}
+const AGENT_TOOLS = ['Read', 'Glob', 'Grep', 'Write', 'Edit', 'Bash'];
+function isPathInside(root, candidate) {
+    const relative = path.relative(root, candidate);
+    return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
+}
+function validateManifest(manifest, directory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        const errors = [];
+        const realDirectory = yield fs.realpath(directory);
+        for (const file of manifest.generatedFiles) {
+            if (path.isAbsolute(file)) {
+                errors.push(`Generated file path must be relative: ${file}`);
+                continue;
+            }
+            const resolved = path.resolve(directory, file);
+            if (!isPathInside(directory, resolved)) {
+                errors.push(`Generated file leaves its task: ${file}`);
+                continue;
+            }
+            const realPath = yield fs.realpath(resolved).catch(() => undefined);
+            if (!realPath || !isPathInside(realDirectory, realPath)) {
+                errors.push(`Generated file leaves its task through a link: ${file}`);
+                continue;
+            }
+            const stats = yield fs.stat(realPath).catch(() => undefined);
+            if (!(stats === null || stats === void 0 ? void 0 : stats.isFile()))
+                errors.push(`Generated file does not exist: ${file}`);
+        }
+        const commandIds = new Set();
+        for (const command of [...manifest.setupCommands, ...manifest.testCommands]) {
+            if (commandIds.has(command.id))
+                errors.push(`Duplicate command ID: ${command.id}`);
+            commandIds.add(command.id);
+            if (path.isAbsolute(command.executable) || command.executable.split(/[\\/]/).includes('..')) {
+                errors.push(`Unsafe command executable: ${command.executable}`);
+                continue;
+            }
+            if (!SAFE_EXECUTABLES.has(command.executable) && !command.executable.startsWith('./')) {
+                errors.push(`Unsupported command executable: ${command.executable}`);
+                continue;
+            }
+            if (command.executable.startsWith('./')) {
+                const executablePath = yield fs
+                    .realpath(path.resolve(directory, command.executable))
+                    .catch(() => undefined);
+                const stats = executablePath
+                    ? yield fs.stat(executablePath).catch(() => undefined)
+                    : undefined;
+                if (!executablePath || !isPathInside(realDirectory, executablePath) || !(stats === null || stats === void 0 ? void 0 : stats.isFile())) {
+                    errors.push(`Command executable leaves its task or is not a file: ${command.executable}`);
+                }
+            }
+            const cwd = (_a = command.cwd) !== null && _a !== void 0 ? _a : '.';
+            if (path.isAbsolute(cwd)) {
+                errors.push(`Command cwd must be relative: ${cwd}`);
+                continue;
+            }
+            const resolvedCwd = path.resolve(directory, cwd);
+            const realCwd = yield fs.realpath(resolvedCwd).catch(() => undefined);
+            const stats = realCwd ? yield fs.stat(realCwd).catch(() => undefined) : undefined;
+            if (!isPathInside(directory, resolvedCwd) ||
+                !realCwd ||
+                !isPathInside(realDirectory, realCwd) ||
+                !(stats === null || stats === void 0 ? void 0 : stats.isDirectory())) {
+                errors.push(`Command cwd is missing or leaves its task: ${cwd}`);
+            }
+        }
+        return errors;
+    });
+}
+function readValidManifest(directory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const manifest = manifestSchema.parse(JSON.parse(yield fs.readFile(path.join(directory, MANIFEST_FILENAME), 'utf8')));
+            const errors = yield validateManifest(manifest, directory);
+            return errors.length === 0 ? manifest : null;
+        }
+        catch (_a) {
+            return null;
+        }
+    });
+}
+export const CHECK_CONCURRENCY = 20;
+export function checkTestability(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ tasks, signal, onTaskUpdate, }) {
+        const folderOrdered = [...tasks].sort((left, right) => path.dirname(left.file).localeCompare(path.dirname(right.file)) ||
+            left.file.localeCompare(right.file) ||
+            left.agent.localeCompare(right.agent));
+        return mapWithConcurrency(folderOrdered, CHECK_CONCURRENCY, checkFileTestability, {
+            signal,
+            onTaskUpdate,
+        });
+    });
+}
+export function generateTestCode(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ tasks, docsRoot, model, concurrency, signal, onTaskUpdate, }) {
+        const generateTask = (task) => __awaiter(this, void 0, void 0, function* () {
+            if (task.error || !task.testable)
+                return task;
+            const emit = (phase, tokens) => onTaskUpdate({ id: task.id, agent: task.agent, file: task.file, phase, tokens });
+            let attempts = task.attempts;
+            let tokens = task.tokens;
+            let lastError = 'run cancelled';
+            const existing = yield readValidManifest(task.directory);
+            if (existing) {
+                emit('generated', tokens);
+                return Object.assign(Object.assign({}, task), { manifest: existing });
+            }
+            for (let attempt = 1; attempt <= 2 && !signal.aborted; attempt++) {
+                attempts += 1;
+                if (attempt === 1)
+                    emit('generating', tokens);
+                try {
+                    const prompt = buildTaskPrompt(docsRoot, task.file, task.directory);
+                    const reply = task.agent === 'claude'
+                        ? yield promptClaude({
+                            prompt,
+                            systemPrompt: SYSTEM_PROMPT,
+                            model,
+                            workingDirectory: task.directory,
+                            docsRoot,
+                            tools: AGENT_TOOLS,
+                            signal,
+                        })
+                        : yield promptCodex({
+                            prompt,
+                            systemPrompt: SYSTEM_PROMPT,
+                            workingDirectory: task.directory,
+                            docsRoot,
+                            signal,
+                        });
+                    tokens += reply.tokens;
+                    const manifest = manifestSchema.parse(JSON.parse(yield fs.readFile(path.join(task.directory, MANIFEST_FILENAME), 'utf8')));
+                    const errors = yield validateManifest(manifest, task.directory);
+                    if (errors.length > 0)
+                        throw new Error(errors.join('\n'));
+                    emit('generated', tokens);
+                    return Object.assign(Object.assign({}, task), { attempts, tokens, manifest });
+                }
+                catch (error) {
+                    lastError = error instanceof Error ? error.message : String(error);
+                }
+            }
+            if (signal.aborted) {
+                emit('cancelled', tokens);
+                return Object.assign(Object.assign({}, task), { attempts, tokens, error: 'run cancelled' });
+            }
+            emit('agent_error', tokens);
+            return Object.assign(Object.assign({}, task), { attempts, tokens, error: lastError });
+        });
+        return mapWithConcurrency(tasks, concurrency, generateTask);
+    });
+}
+function promptCodex(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ prompt, systemPrompt, workingDirectory, docsRoot, signal, }) {
+        const { Codex } = yield loadCodexSdk();
+        const thread = new Codex().startThread({
+            additionalDirectories: [docsRoot],
+            approvalPolicy: 'never',
+            networkAccessEnabled: true,
+            sandboxMode: 'workspace-write',
+            skipGitRepoCheck: true,
+            workingDirectory,
+        });
+        const result = yield thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
+        const usage = result.usage;
+        const tokens = usage
+            ? usage.input_tokens +
+                usage.cached_input_tokens +
+                usage.cache_write_input_tokens +
+                usage.output_tokens +
+                usage.reasoning_output_tokens
+            : 0;
+        return { tokens, result: result.finalResponse };
+    });
+}

package/bin/agent-harness/index.js

@@ -0,0 +1,85 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from 'node:fs/promises';
+import { discoverCodeBlocks } from './discoverCodeBlocks.js';
+import { executeCodeBlocks } from './executeCodeBlocks.js';
+import { filterSelectedPages } from './filterSelectedPages.js';
+import { checkTestability, generateTestCode } from './generateTestCode.js';
+import { clearRunSave, writeRunSave } from './runSave.js';
+import { setupFolders } from './setupFolders.js';
+export const DEFAULT_CONCURRENCY = 5;
+export const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60 * 1000;
+export { discoverCodeBlocks } from './discoverCodeBlocks.js';
+export { clearRunSave, loadRunSave } from './runSave.js';
+export function runCodeTests(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ path: docsPath, agents, model, selectedFiles, outputDirectory, runId: savedRunId, concurrency, commandTimeoutMs, signal, onTaskUpdate, }) {
+        const startedAt = new Date();
+        const uniqueAgents = [...new Set(agents)];
+        const { docsRoot, pages: allPages } = yield discoverCodeBlocks(docsPath);
+        const pages = filterSelectedPages(allPages, selectedFiles);
+        const { runId, runDirectory, reportPath, tasks } = yield setupFolders({
+            pages,
+            agents: uniqueAgents,
+            docsRoot,
+            outputDirectory,
+            runId: savedRunId,
+        });
+        yield writeRunSave(outputDirectory, {
+            version: 1,
+            runId,
+            agents: uniqueAgents,
+            model,
+            selectedFiles: pages.map((page) => page.file),
+            createdAt: startedAt.toISOString(),
+        });
+        const checked = yield checkTestability({ tasks, signal, onTaskUpdate });
+        const generated = yield generateTestCode({
+            tasks: checked,
+            docsRoot,
+            model,
+            concurrency,
+            signal,
+            onTaskUpdate,
+        });
+        const results = yield executeCodeBlocks({
+            tasks: generated,
+            docsRoot,
+            concurrency,
+            commandTimeoutMs,
+            signal,
+            onTaskUpdate,
+        });
+        const completedAt = new Date();
+        const report = {
+            version: 1,
+            runId,
+            status: signal.aborted
+                ? 'cancelled'
+                : results.every((task) => task.status === 'passed')
+                    ? 'passed'
+                    : results.some((task) => task.status === 'passed')
+                        ? 'completed'
+                        : 'failed',
+            docsRoot,
+            outputDirectory: runDirectory,
+            reportPath,
+            agents: uniqueAgents,
+            selectedFiles: pages.map((page) => page.file),
+            startedAt: startedAt.toISOString(),
+            completedAt: completedAt.toISOString(),
+            durationMs: completedAt.getTime() - startedAt.getTime(),
+            tasks: results,
+        };
+        yield fs.writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+        if (!signal.aborted)
+            yield clearRunSave(outputDirectory);
+        return report;
+    });
+}

package/bin/agent-harness/loadAgentSdk.js

@@ -0,0 +1,46 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const INSTALL_HINTS = {
+    '@anthropic-ai/claude-agent-sdk': 'npm install @anthropic-ai/claude-agent-sdk @anthropic-ai/sdk @modelcontextprotocol/sdk',
+    '@openai/codex-sdk': 'npm install @openai/codex-sdk',
+};
+function isModuleNotFound(error) {
+    if (typeof error !== 'object' || error === null)
+        return false;
+    const { code } = error;
+    return code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND';
+}
+function missingSdkError(packageName) {
+    return new Error(`mint test requires the optional dependency ${packageName}, which is not installed. Install it with: ${INSTALL_HINTS[packageName]}`);
+}
+export function loadClaudeAgentSdk() {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            return yield import('@anthropic-ai/claude-agent-sdk');
+        }
+        catch (error) {
+            if (isModuleNotFound(error))
+                throw missingSdkError('@anthropic-ai/claude-agent-sdk');
+            throw error;
+        }
+    });
+}
+export function loadCodexSdk() {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            return yield import('@openai/codex-sdk');
+        }
+        catch (error) {
+            if (isModuleNotFound(error))
+                throw missingSdkError('@openai/codex-sdk');
+            throw error;
+        }
+    });
+}

package/bin/agent-harness/mapWithConcurrency.js

@@ -0,0 +1,26 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+export function mapWithConcurrency(items, concurrency, fn, ...args) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const results = new Array(items.length);
+        let nextIndex = 0;
+        const worker = () => __awaiter(this, void 0, void 0, function* () {
+            for (;;) {
+                const index = nextIndex++;
+                const item = items[index];
+                if (index >= items.length || item === undefined)
+                    return;
+                results[index] = yield fn(item, ...args);
+            }
+        });
+        yield Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), items.length) }, () => worker()));
+        return results;
+    });
+}

package/bin/agent-harness/runSave.js

@@ -0,0 +1,47 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+const runSaveSchema = z
+    .object({
+    version: z.literal(1),
+    runId: z.string().regex(/^[A-Za-z0-9._-]+$/),
+    agents: z.array(z.enum(['claude', 'codex'])).min(1),
+    model: z.string().min(1),
+    selectedFiles: z.array(z.string().min(1)),
+    createdAt: z.string(),
+})
+    .strict();
+function savePath(outputDirectory) {
+    return path.join(outputDirectory, 'save.json');
+}
+export function loadRunSave(outputDirectory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            const content = yield fs.readFile(savePath(outputDirectory), 'utf8');
+            return runSaveSchema.parse(JSON.parse(content));
+        }
+        catch (_a) {
+            return null;
+        }
+    });
+}
+export function writeRunSave(outputDirectory, save) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield fs.mkdir(outputDirectory, { recursive: true });
+        yield fs.writeFile(savePath(outputDirectory), `${JSON.stringify(save, null, 2)}\n`);
+    });
+}
+export function clearRunSave(outputDirectory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield fs.unlink(savePath(outputDirectory)).catch(() => undefined);
+    });
+}

package/bin/agent-harness/setupFolders.js

@@ -0,0 +1,59 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { randomBytes } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+const PACKAGE_JSON = `${JSON.stringify({
+    name: 'mint-test-task',
+    private: true,
+    type: 'module',
+    scripts: { test: 'node --test tests/' },
+}, null, 2)}\n`;
+function scaffoldTaskDirectory(directory) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield fs.mkdir(path.join(directory, 'tests'), { recursive: true });
+        yield fs
+            .writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' })
+            .catch(() => { });
+    });
+}
+export function setupFolders(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ pages, agents, docsRoot, outputDirectory, runId: savedRunId, }) {
+        const runId = savedRunId !== null && savedRunId !== void 0 ? savedRunId : `${new Date()
+            .toISOString()
+            .replace(/[-:TZ.]/g, '')
+            .slice(0, 14)}-${randomBytes(4).toString('hex')}`;
+        const runDirectory = path.join(path.resolve(outputDirectory), runId);
+        const reportPath = path.join(runDirectory, 'report.json');
+        yield fs.mkdir(runDirectory, { recursive: true });
+        const tasks = [];
+        for (const agent of agents) {
+            for (const [pageIndex, page] of pages.entries()) {
+                const slug = page.file
+                    .replace(/\.[^.]+$/, '')
+                    .replace(/[^a-z0-9]+/gi, '-')
+                    .replace(/^-|-$/g, '')
+                    .slice(0, 56);
+                tasks.push({
+                    id: `${agent}:${page.file}`,
+                    agent,
+                    docsRoot,
+                    file: page.file,
+                    directory: path.join(runDirectory, agent, `${String(pageIndex + 1).padStart(3, '0')}-${slug || 'page'}`),
+                    attempts: 0,
+                    tokens: 0,
+                });
+            }
+        }
+        for (const task of tasks)
+            yield scaffoldTaskDirectory(task.directory);
+        return { runId, runDirectory, reportPath, tasks };
+    });
+}