@mintlify/cli 4.0.1469 → 4.0.1471

package/__test__/mintTestOutput.test.ts

@@ -79,6 +79,7 @@ it('includes partial generated files when generation fails', async () => {
     throw new Error('generation failed');
   });
   const updates: TaskUpdate[] = [];
+  const onGenerationResult = vi.fn();
 
   const generated = await generateTestCode({
     tasks: [generatedTask(directory)],
@@ -87,12 +88,22 @@ it('includes partial generated files when generation fails', async () => {
     concurrency: 1,
     signal: new AbortController().signal,
     onTaskUpdate: (update) => updates.push(update),
+    onGenerationResult,
   });
 
   expect(updates.at(-1)).toMatchObject({
     phase: 'agent_error',
     generatedFiles: [generatedFile],
   });
+  expect(onGenerationResult).toHaveBeenCalledWith({
+    status: 'failed',
+    agent: 'codex',
+    model: 'unused',
+    attemptCount: 2,
+    tokenCount: 0,
+    durationMs: expect.any(Number),
+    generatedFileCount: 1,
+  });
   const results = await executeCodeBlocks({
     tasks: generated,
     docsRoot: directory,
@@ -108,6 +119,57 @@ it('includes partial generated files when generation fails', async () => {
   });
 });
 
+it('reports a successful generation run', async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'mint-test-output-'));
+  temporaryDirectories.push(directory);
+  const generatedFile = path.join(directory, 'tests', 'generated.test.ts');
+  await fs.mkdir(path.dirname(generatedFile), { recursive: true });
+  await fs.writeFile(path.join(directory, 'package.json'), '{}\n');
+  agentMocks.codexRun.mockImplementation(async () => {
+    await fs.writeFile(generatedFile, 'export const generated = true;\n');
+    await fs.writeFile(
+      path.join(directory, 'mint-test.json'),
+      `${JSON.stringify({
+        version: 1,
+        generatedFiles: ['tests/generated.test.ts'],
+        setupCommands: [],
+        testCommands: [],
+      })}\n`
+    );
+    return {
+      finalResponse: 'done',
+      usage: {
+        input_tokens: 100,
+        cached_input_tokens: 25,
+        cache_write_input_tokens: 10,
+        output_tokens: 50,
+        reasoning_output_tokens: 15,
+      },
+    };
+  });
+  const onGenerationResult = vi.fn();
+
+  await generateTestCode({
+    tasks: [generatedTask(directory)],
+    docsRoot: directory,
+    model: 'gpt-test',
+    concurrency: 1,
+    signal: new AbortController().signal,
+    onTaskUpdate: () => {},
+    onGenerationResult,
+  });
+
+  expect(onGenerationResult).toHaveBeenCalledWith({
+    status: 'succeeded',
+    agent: 'codex',
+    model: 'gpt-test',
+    attemptCount: 1,
+    tokenCount: 200,
+    durationMs: expect.any(Number),
+    generatedFileCount: 1,
+  });
+});
+
 it('includes partial generated files when generation is cancelled', async () => {
   const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'mint-test-output-'));
   temporaryDirectories.push(directory);

package/bin/agent-harness/generateTestCode.js

@@ -214,7 +214,7 @@ export function checkTestability(_a) {
     });
 }
 export function generateTestCode(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ tasks, docsRoot, model, concurrency, signal, onTaskUpdate, }) {
+    return __awaiter(this, arguments, void 0, function* ({ tasks, docsRoot, model, concurrency, signal, onTaskUpdate, onGenerationResult, }) {
         const generateTask = (task) => __awaiter(this, void 0, void 0, function* () {
             if (task.error || !task.testable)
                 return task;
@@ -230,8 +230,18 @@ export function generateTestCode(_a) {
                 emit('generated', tokens, existing.generatedFiles.map((file) => path.resolve(task.directory, file)));
                 return Object.assign(Object.assign({}, task), { manifest: existing });
             }
+            const startedAt = Date.now();
+            const startingTokens = tokens;
+            let generationAttempts = 0;
+            const emitGenerationResult = (result) => {
+                try {
+                    onGenerationResult === null || onGenerationResult === void 0 ? void 0 : onGenerationResult(result);
+                }
+                catch (_a) { }
+            };
             for (let attempt = 1; attempt <= 2 && !signal.aborted; attempt++) {
                 attempts += 1;
+                generationAttempts += 1;
                 if (attempt === 1)
                     emit('generating', tokens);
                 try {
@@ -261,7 +271,17 @@ export function generateTestCode(_a) {
                         const [first = '', ...rest] = errors;
                         throw new Error([`${MANIFEST_FILENAME} has problems: ${first}`, ...rest].join('\n'));
                     }
-                    emit('generated', tokens, manifest.generatedFiles.map((file) => path.resolve(task.directory, file)));
+                    const generatedFiles = manifest.generatedFiles.map((file) => path.resolve(task.directory, file));
+                    emit('generated', tokens, generatedFiles);
+                    emitGenerationResult({
+                        status: 'succeeded',
+                        agent: task.agent,
+                        model,
+                        attemptCount: generationAttempts,
+                        tokenCount: tokens - startingTokens,
+                        durationMs: Date.now() - startedAt,
+                        generatedFileCount: generatedFiles.length,
+                    });
                     return Object.assign(Object.assign({}, task), { attempts, tokens, manifest });
                 }
                 catch (error) {
@@ -275,6 +295,15 @@ export function generateTestCode(_a) {
                 return Object.assign(Object.assign({}, task), { attempts, tokens, generatedFiles, error: 'run cancelled' });
             }
             const generatedFiles = yield discoverGeneratedFiles(task.directory);
+            emitGenerationResult({
+                status: 'failed',
+                agent: task.agent,
+                model,
+                attemptCount: generationAttempts,
+                tokenCount: tokens - startingTokens,
+                durationMs: Date.now() - startedAt,
+                generatedFileCount: generatedFiles.length,
+            });
             const detailsPath = yield writeErrorDetails(task.directory, attemptErrors);
             const error = detailsPath ? `${lastError}\nDetails: ${detailsPath}` : lastError;
             emit('agent_error', tokens, generatedFiles, error);

package/bin/agent-harness/index.js

@@ -23,7 +23,7 @@ export { countReport } from './countReport.js';
 export { loadLatestTestRun, loadTestRuns } from './runHistory.js';
 export { clearRunSave, loadRunSave } from './runSave.js';
 export function runCodeTests(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ path: docsPath, agents, model, selectedFiles, outputDirectory, runId: savedRunId, concurrency, commandTimeoutMs, signal, onTaskUpdate, }) {
+    return __awaiter(this, arguments, void 0, function* ({ path: docsPath, agents, model, selectedFiles, outputDirectory, runId: savedRunId, concurrency, commandTimeoutMs, signal, onTaskUpdate, onGenerationResult, }) {
         const startedAt = new Date();
         const uniqueAgents = [...new Set(agents)];
         const { docsRoot, pages: allPages } = yield discoverCodeBlocks(docsPath);
@@ -55,6 +55,7 @@ export function runCodeTests(_a) {
             concurrency,
             signal,
             onTaskUpdate,
+            onGenerationResult,
         });
         const results = yield executeCodeBlocks({
             tasks: generated,

package/bin/mintTest.js

@@ -12,6 +12,7 @@ import { DEFAULT_COMMAND_TIMEOUT_MS, DEFAULT_CONCURRENCY, clearRunSave, countRep
 import { CMD_EXEC_PATH, findDocsRoot, isAI, terminate } from './helpers.js';
 import { getAccessToken } from './keyring.js';
 import { fileInNavPaths, loadDocsNavScope } from './mintTestNav.js';
+import { trackMintTestGenerationResult, trackMintTestRunCompleted, trackMintTestRunStarted, } from './mintTestTelemetry.js';
 import { firstLine, taskProblemLabel } from './mintTestText.js';
 import { formatDuration, runMintTestUi } from './mintTestUi.js';
 export function mintTestHandler() {
@@ -36,6 +37,7 @@ export function mintTestHandler() {
                 ? discoveredPages.filter((page) => fileInNavPaths(page.file, scope.navPaths))
                 : discoveredPages;
             const autoSelectedFiles = (scope ? pages.filter((page) => fileInNavPaths(page.file, scope.autoSelectedPaths)) : pages).map((page) => page.file);
+            const totalPages = scope ? scope.navPaths.size : pages.length;
             const outputDirectory = path.join(docsRoot, 'tests', 'mint-test');
             let save = yield loadRunSave(outputDirectory);
             if (save) {
@@ -56,25 +58,46 @@ export function mintTestHandler() {
                     docsRoot,
                     pages,
                     autoSelectedFiles,
+                    totalPages,
                     save,
                     lastReport,
                 });
                 yield terminate(exitCode);
                 return;
             }
+            const agents = (_a = save === null || save === void 0 ? void 0 : save.agents) !== null && _a !== void 0 ? _a : ['claude'];
+            const model = (_b = save === null || save === void 0 ? void 0 : save.model) !== null && _b !== void 0 ? _b : 'claude-opus-4-8';
+            const selectedFiles = (_c = save === null || save === void 0 ? void 0 : save.selectedFiles) !== null && _c !== void 0 ? _c : autoSelectedFiles;
+            const resumed = save !== null;
             if (save)
                 process.stdout.write(`resuming run ${save.runId}\n`);
+            void trackMintTestRunStarted({
+                mode: 'non_interactive',
+                resumed,
+                agents,
+                model,
+                selectedPageCount: new Set(selectedFiles).size,
+                totalPageCount: totalPages,
+            });
             const report = yield runCodeTests({
                 path: targetPath,
-                agents: (_a = save === null || save === void 0 ? void 0 : save.agents) !== null && _a !== void 0 ? _a : ['claude'],
-                model: (_b = save === null || save === void 0 ? void 0 : save.model) !== null && _b !== void 0 ? _b : 'claude-opus-4-8',
-                selectedFiles: (_c = save === null || save === void 0 ? void 0 : save.selectedFiles) !== null && _c !== void 0 ? _c : autoSelectedFiles,
+                agents,
+                model,
+                selectedFiles,
                 outputDirectory,
                 runId: (_d = save === null || save === void 0 ? void 0 : save.runId) !== null && _d !== void 0 ? _d : null,
                 concurrency: DEFAULT_CONCURRENCY,
                 commandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
                 signal: new AbortController().signal,
                 onTaskUpdate: () => { },
+                onGenerationResult: (result) => {
+                    void trackMintTestGenerationResult(Object.assign(Object.assign({}, result), { mode: 'non_interactive', resumed }));
+                },
+            });
+            yield trackMintTestRunCompleted(report, {
+                mode: 'non_interactive',
+                resumed,
+                totalPageCount: totalPages,
             });
             const counts = countReport(report.tasks);
             process.stdout.write(`mint test ${report.status}: ${counts.passed} passed, ${counts.failed} failed, ${counts.agentErrors} agent errors\n`);

package/bin/mintTestTelemetry.js

@@ -0,0 +1,63 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { countReport, } from './agent-harness/index.js';
+import { trackEvent } from './telemetry/track.js';
+export function trackMintTestRunStarted(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ mode, resumed, agents, model, selectedPageCount, totalPageCount, }) {
+        yield trackEvent('cli.test.started', {
+            mode,
+            resumed,
+            agents,
+            model,
+            selected_page_count: selectedPageCount,
+            total_page_count: totalPageCount,
+        });
+    });
+}
+export function trackMintTestRunCompleted(report_1, _a) {
+    return __awaiter(this, arguments, void 0, function* (report, { mode, resumed, totalPageCount }) {
+        const counts = countReport(report.tasks);
+        const skippedCheckCount = report.tasks.filter((task) => !task.testable && task.status === 'passed').length;
+        const tokenCount = report.tasks.reduce((total, task) => total + task.tokens, 0);
+        const runnableCheckCount = report.tasks.filter((task) => task.testable).length;
+        yield trackEvent('cli.test.completed', {
+            mode,
+            resumed,
+            status: report.status,
+            agents: report.agents,
+            model: report.model,
+            selected_page_count: new Set(report.selectedFiles).size,
+            total_page_count: totalPageCount,
+            total_check_count: report.tasks.length,
+            runnable_check_count: runnableCheckCount,
+            passed_check_count: counts.passed - skippedCheckCount,
+            failed_check_count: counts.failed,
+            skipped_check_count: skippedCheckCount,
+            agent_error_check_count: counts.agentErrors,
+            cancelled_check_count: counts.cancelled,
+            token_count: tokenCount,
+            duration_ms: report.durationMs,
+        });
+    });
+}
+export function trackMintTestGenerationResult(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ status, mode, resumed, agent, model, attemptCount, tokenCount, durationMs, generatedFileCount, }) {
+        yield trackEvent(`cli.test.generation.${status}`, {
+            mode,
+            resumed,
+            agent,
+            model,
+            attempt_count: attemptCount,
+            token_count: tokenCount,
+            duration_ms: durationMs,
+            generated_file_count: generatedFileCount,
+        });
+    });
+}

package/bin/mintTestUi.js

@@ -13,6 +13,7 @@ import path from 'node:path';
 import { useEffect, useMemo, useRef, useState } from 'react';
 import { DEFAULT_COMMAND_TIMEOUT_MS, DEFAULT_CONCURRENCY, clearRunSave, runCodeTests, } from './agent-harness/index.js';
 import { readFilePreview } from './mintTestFilePreview.js';
+import { trackMintTestGenerationResult, trackMintTestRunCompleted, trackMintTestRunStarted, } from './mintTestTelemetry.js';
 import { firstLine, footerSegments, taskProblemLabel } from './mintTestText.js';
 import { toggleScopeSelection, visibleScopeRows } from './mintTestTree.js';
 const HARNESSES = [
@@ -375,7 +376,7 @@ function footerForScreen(screen, running, cancelled, activeRunStage, reviewingSa
     }
     return 'Enter exit';
 }
-function MintTestApp({ targetPath, docsRoot, pages, autoSelectedFiles, save, lastReport, onFinish, }) {
+function MintTestApp({ targetPath, docsRoot, pages, autoSelectedFiles, totalPages, save, lastReport, onFinish, }) {
     var _a;
     const size = useTerminalSize();
     const files = useMemo(() => pages.map((page) => page.file), [pages]);
@@ -515,6 +516,7 @@ function MintTestApp({ targetPath, docsRoot, pages, autoSelectedFiles, save, las
         var _a, _b;
         if (abortController.current)
             return;
+        const resumed = runId !== null;
         const controller = new AbortController();
         abortController.current = controller;
         setTaskPhases(new Map());
@@ -535,6 +537,14 @@ function MintTestApp({ targetPath, docsRoot, pages, autoSelectedFiles, save, las
         })));
         setScreen('running');
         setActiveRunStage('check');
+        void trackMintTestRunStarted({
+            mode: 'interactive',
+            resumed,
+            agents: runAgents,
+            model: runModel,
+            selectedPageCount: new Set(runFiles_).size,
+            totalPageCount: totalPages,
+        });
         void runCodeTests({
             path: targetPath,
             agents: runAgents,
@@ -545,6 +555,9 @@ function MintTestApp({ targetPath, docsRoot, pages, autoSelectedFiles, save, las
             concurrency: DEFAULT_CONCURRENCY,
             commandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
             signal: controller.signal,
+            onGenerationResult: (result) => {
+                void trackMintTestGenerationResult(Object.assign(Object.assign({}, result), { mode: 'interactive', resumed }));
+            },
             onTaskUpdate: (update) => {
                 const phaseStage = update.phase === 'running'
                     ? 'run'
@@ -566,6 +579,11 @@ function MintTestApp({ targetPath, docsRoot, pages, autoSelectedFiles, save, las
             },
         })
             .then((nextReport) => {
+            void trackMintTestRunCompleted(nextReport, {
+                mode: 'interactive',
+                resumed,
+                totalPageCount: totalPages,
+            });
             setReport(nextReport);
             const failedFiles = nextReport.tasks
                 .filter((task) => task.status === 'failed' || task.status === 'agent_error')