@mintlify/cli 4.0.1469 → 4.0.1471

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mintlify/cli",
-  "version": "4.0.1469",
+  "version": "4.0.1471",
   "description": "The Mintlify CLI",
   "engines": {
     "node": ">=18.0.0"
@@ -45,11 +45,11 @@
   },
   "dependencies": {
     "@inquirer/prompts": "7.9.0",
-    "@mintlify/common": "1.0.1125",
-    "@mintlify/link-rot": "3.0.1327",
+    "@mintlify/common": "1.0.1126",
+    "@mintlify/link-rot": "3.0.1328",
     "@mintlify/models": "0.0.351",
-    "@mintlify/prebuild": "1.0.1279",
-    "@mintlify/previewing": "4.0.1351",
+    "@mintlify/prebuild": "1.0.1280",
+    "@mintlify/previewing": "4.0.1352",
     "@mintlify/validation": "0.1.844",
     "adm-zip": "0.6.0",
     "chalk": "5.2.0",
@@ -86,7 +86,7 @@
     "keytar": "7.9.0"
   },
   "devDependencies": {
-    "@mintlify/editor": "0.0.355",
+    "@mintlify/editor": "0.0.356",
     "@mintlify/ts-config": "2.0.2",
     "@tsconfig/recommended": "1.0.2",
     "@types/detect-port": "1.3.2",
@@ -105,5 +105,5 @@
     "vitest": "2.1.9",
     "vitest-mock-process": "1.0.4"
   },
-  "gitHead": "38d9ac8a4f2425cb701131ef72de78da3ecc8677"
+  "gitHead": "c1d2437e2890e97075390d6160f6bcf72541e574"
 }

package/src/agent-harness/generateTestCode.ts

@@ -12,7 +12,12 @@ import {
 } from './manifest.js';
 import { mapWithConcurrency } from './mapWithConcurrency.js';
 import { checkFileTestability } from './tasks/checkTestability.js';
-import type { GeneratedTask, GeneratedTestManifest, TaskUpdate } from './types.js';
+import type {
+  GeneratedTask,
+  GeneratedTestManifest,
+  TaskUpdate,
+  TestGenerationResult,
+} from './types.js';
 
 const SCAFFOLD_FILENAMES = new Set([MANIFEST_FILENAME, 'package.json']);
 const IGNORED_GENERATED_DIRECTORIES = new Set(['.git', 'node_modules']);
@@ -228,6 +233,7 @@ export async function generateTestCode({
   concurrency,
   signal,
   onTaskUpdate,
+  onGenerationResult,
 }: {
   tasks: GeneratedTask[];
   docsRoot: string;
@@ -235,6 +241,7 @@ export async function generateTestCode({
   concurrency: number;
   signal: AbortSignal;
   onTaskUpdate: (update: TaskUpdate) => void;
+  onGenerationResult?: (result: TestGenerationResult) => void;
 }): Promise<GeneratedTask[]> {
   const generateTask = async (task: GeneratedTask): Promise<GeneratedTask> => {
     if (task.error || !task.testable) return task;
@@ -269,8 +276,18 @@ export async function generateTestCode({
       return { ...task, manifest: existing };
     }
 
+    const startedAt = Date.now();
+    const startingTokens = tokens;
+    let generationAttempts = 0;
+    const emitGenerationResult = (result: TestGenerationResult) => {
+      try {
+        onGenerationResult?.(result);
+      } catch {}
+    };
+
     for (let attempt = 1; attempt <= 2 && !signal.aborted; attempt++) {
       attempts += 1;
+      generationAttempts += 1;
       if (attempt === 1) emit('generating', tokens);
       try {
         const prompt = buildTaskPrompt(docsRoot, task.file, task.directory);
@@ -301,11 +318,19 @@ export async function generateTestCode({
           throw new Error([`${MANIFEST_FILENAME} has problems: ${first}`, ...rest].join('\n'));
         }
 
-        emit(
-          'generated',
-          tokens,
-          manifest.generatedFiles.map((file) => path.resolve(task.directory, file))
+        const generatedFiles = manifest.generatedFiles.map((file) =>
+          path.resolve(task.directory, file)
         );
+        emit('generated', tokens, generatedFiles);
+        emitGenerationResult({
+          status: 'succeeded',
+          agent: task.agent,
+          model,
+          attemptCount: generationAttempts,
+          tokenCount: tokens - startingTokens,
+          durationMs: Date.now() - startedAt,
+          generatedFileCount: generatedFiles.length,
+        });
         return { ...task, attempts, tokens, manifest };
       } catch (error) {
         lastError = error instanceof Error ? error.message : String(error);
@@ -318,6 +343,15 @@ export async function generateTestCode({
       return { ...task, attempts, tokens, generatedFiles, error: 'run cancelled' };
     }
     const generatedFiles = await discoverGeneratedFiles(task.directory);
+    emitGenerationResult({
+      status: 'failed',
+      agent: task.agent,
+      model,
+      attemptCount: generationAttempts,
+      tokenCount: tokens - startingTokens,
+      durationMs: Date.now() - startedAt,
+      generatedFileCount: generatedFiles.length,
+    });
     const detailsPath = await writeErrorDetails(task.directory, attemptErrors);
     const error = detailsPath ? `${lastError}\nDetails: ${detailsPath}` : lastError;
     emit('agent_error', tokens, generatedFiles, error);

package/src/agent-harness/index.ts

@@ -30,6 +30,7 @@ export type {
   TaskResult,
   TaskStatus,
   TaskUpdate,
+  TestGenerationResult,
   TestRunSummary,
   TestRunsIndex,
 } from './types.js';
@@ -45,6 +46,7 @@ export async function runCodeTests({
   commandTimeoutMs,
   signal,
   onTaskUpdate,
+  onGenerationResult,
 }: RunCodeTestsOptions): Promise<CodeTestReport> {
   const startedAt = new Date();
   const uniqueAgents = [...new Set(agents)];
@@ -85,6 +87,7 @@ export async function runCodeTests({
     concurrency,
     signal,
     onTaskUpdate,
+    onGenerationResult,
   });
 
   const results = await executeCodeBlocks({

package/src/agent-harness/types.ts

@@ -105,6 +105,16 @@ export interface TaskUpdate {
   error?: string;
 }
 
+export interface TestGenerationResult {
+  status: 'succeeded' | 'failed';
+  agent: HarnessAgent;
+  model: string;
+  attemptCount: number;
+  tokenCount: number;
+  durationMs: number;
+  generatedFileCount: number;
+}
+
 export interface RunSave {
   version: 1;
   runId: string;
@@ -125,6 +135,7 @@ export interface RunCodeTestsOptions {
   commandTimeoutMs: number;
   signal: AbortSignal;
   onTaskUpdate: (update: TaskUpdate) => void;
+  onGenerationResult?: (result: TestGenerationResult) => void;
 }
 
 const identifier = z.string().trim().min(1).max(120);

package/src/mintTest.tsx

@@ -14,6 +14,11 @@ import {
 import { CMD_EXEC_PATH, findDocsRoot, isAI, terminate } from './helpers.js';
 import { getAccessToken } from './keyring.js';
 import { fileInNavPaths, loadDocsNavScope } from './mintTestNav.js';
+import {
+  trackMintTestGenerationResult,
+  trackMintTestRunCompleted,
+  trackMintTestRunStarted,
+} from './mintTestTelemetry.js';
 import { firstLine, taskProblemLabel } from './mintTestText.js';
 import { formatDuration, runMintTestUi } from './mintTestUi.js';
 
@@ -41,6 +46,7 @@ export async function mintTestHandler(): Promise<void> {
     const autoSelectedFiles = (
       scope ? pages.filter((page) => fileInNavPaths(page.file, scope.autoSelectedPaths)) : pages
     ).map((page) => page.file);
+    const totalPages = scope ? scope.navPaths.size : pages.length;
     const outputDirectory = path.join(docsRoot, 'tests', 'mint-test');
 
     let save: RunSave | null = await loadRunSave(outputDirectory);
@@ -62,6 +68,7 @@ export async function mintTestHandler(): Promise<void> {
         docsRoot,
         pages,
         autoSelectedFiles,
+        totalPages,
         save,
         lastReport,
       });
@@ -69,18 +76,42 @@ export async function mintTestHandler(): Promise<void> {
       return;
     }
 
+    const agents = save?.agents ?? ['claude'];
+    const model = save?.model ?? 'claude-opus-4-8';
+    const selectedFiles = save?.selectedFiles ?? autoSelectedFiles;
+    const resumed = save !== null;
     if (save) process.stdout.write(`resuming run ${save.runId}\n`);
+    void trackMintTestRunStarted({
+      mode: 'non_interactive',
+      resumed,
+      agents,
+      model,
+      selectedPageCount: new Set(selectedFiles).size,
+      totalPageCount: totalPages,
+    });
     const report = await runCodeTests({
       path: targetPath,
-      agents: save?.agents ?? ['claude'],
-      model: save?.model ?? 'claude-opus-4-8',
-      selectedFiles: save?.selectedFiles ?? autoSelectedFiles,
+      agents,
+      model,
+      selectedFiles,
       outputDirectory,
       runId: save?.runId ?? null,
       concurrency: DEFAULT_CONCURRENCY,
       commandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
       signal: new AbortController().signal,
       onTaskUpdate: () => {},
+      onGenerationResult: (result) => {
+        void trackMintTestGenerationResult({
+          ...result,
+          mode: 'non_interactive',
+          resumed,
+        });
+      },
+    });
+    await trackMintTestRunCompleted(report, {
+      mode: 'non_interactive',
+      resumed,
+      totalPageCount: totalPages,
     });
     const counts = countReport(report.tasks);
     process.stdout.write(

package/src/mintTestTelemetry.ts

@@ -0,0 +1,101 @@
+import {
+  countReport,
+  type CodeTestReport,
+  type HarnessAgent,
+  type TestGenerationResult,
+} from './agent-harness/index.js';
+import { trackEvent } from './telemetry/track.js';
+
+export type MintTestMode = 'interactive' | 'non_interactive';
+
+export interface MintTestRunStartedOptions {
+  mode: MintTestMode;
+  resumed: boolean;
+  agents: HarnessAgent[];
+  model: string;
+  selectedPageCount: number;
+  totalPageCount: number;
+}
+
+export interface MintTestRunCompletedOptions {
+  mode: MintTestMode;
+  resumed: boolean;
+  totalPageCount: number;
+}
+
+export interface MintTestGenerationResultOptions extends TestGenerationResult {
+  mode: MintTestMode;
+  resumed: boolean;
+}
+
+export async function trackMintTestRunStarted({
+  mode,
+  resumed,
+  agents,
+  model,
+  selectedPageCount,
+  totalPageCount,
+}: MintTestRunStartedOptions): Promise<void> {
+  await trackEvent('cli.test.started', {
+    mode,
+    resumed,
+    agents,
+    model,
+    selected_page_count: selectedPageCount,
+    total_page_count: totalPageCount,
+  });
+}
+
+export async function trackMintTestRunCompleted(
+  report: CodeTestReport,
+  { mode, resumed, totalPageCount }: MintTestRunCompletedOptions
+): Promise<void> {
+  const counts = countReport(report.tasks);
+  const skippedCheckCount = report.tasks.filter(
+    (task) => !task.testable && task.status === 'passed'
+  ).length;
+  const tokenCount = report.tasks.reduce((total, task) => total + task.tokens, 0);
+  const runnableCheckCount = report.tasks.filter((task) => task.testable).length;
+
+  await trackEvent('cli.test.completed', {
+    mode,
+    resumed,
+    status: report.status,
+    agents: report.agents,
+    model: report.model,
+    selected_page_count: new Set(report.selectedFiles).size,
+    total_page_count: totalPageCount,
+    total_check_count: report.tasks.length,
+    runnable_check_count: runnableCheckCount,
+    passed_check_count: counts.passed - skippedCheckCount,
+    failed_check_count: counts.failed,
+    skipped_check_count: skippedCheckCount,
+    agent_error_check_count: counts.agentErrors,
+    cancelled_check_count: counts.cancelled,
+    token_count: tokenCount,
+    duration_ms: report.durationMs,
+  });
+}
+
+export async function trackMintTestGenerationResult({
+  status,
+  mode,
+  resumed,
+  agent,
+  model,
+  attemptCount,
+  tokenCount,
+  durationMs,
+  generatedFileCount,
+}: MintTestGenerationResultOptions): Promise<void> {
+  await trackEvent(`cli.test.generation.${status}`, {
+    mode,
+    resumed,
+    agent,
+    model,
+    attempt_count: attemptCount,
+    token_count: tokenCount,
+    duration_ms: durationMs,
+    generated_file_count: generatedFileCount,
+  });
+}

package/src/mintTestUi.tsx

@@ -15,6 +15,11 @@ import {
   type TaskUpdate,
 } from './agent-harness/index.js';
 import { readFilePreview } from './mintTestFilePreview.js';
+import {
+  trackMintTestGenerationResult,
+  trackMintTestRunCompleted,
+  trackMintTestRunStarted,
+} from './mintTestTelemetry.js';
 import { firstLine, footerSegments, taskProblemLabel } from './mintTestText.js';
 import { toggleScopeSelection, visibleScopeRows, type ScopeTreeRow } from './mintTestTree.js';
 
@@ -25,6 +30,7 @@ interface MintTestUiProps {
   docsRoot: string;
   pages: DocsPage[];
   autoSelectedFiles: string[];
+  totalPages: number;
   save: RunSave | null;
   lastReport: CodeTestReport | null;
   onFinish: (exitCode: number) => void;
@@ -827,6 +833,7 @@ function MintTestApp({
   docsRoot,
   pages,
   autoSelectedFiles,
+  totalPages,
   save,
   lastReport,
   onFinish,
@@ -998,6 +1005,7 @@ function MintTestApp({
     runFiles: string[];
   }) => {
     if (abortController.current) return;
+    const resumed = runId !== null;
     const controller = new AbortController();
     abortController.current = controller;
     setTaskPhases(new Map());
@@ -1025,6 +1033,14 @@ function MintTestApp({
     );
     setScreen('running');
     setActiveRunStage('check');
+    void trackMintTestRunStarted({
+      mode: 'interactive',
+      resumed,
+      agents: runAgents,
+      model: runModel,
+      selectedPageCount: new Set(runFiles_).size,
+      totalPageCount: totalPages,
+    });
     void runCodeTests({
       path: targetPath,
       agents: runAgents,
@@ -1035,6 +1051,13 @@ function MintTestApp({
       concurrency: DEFAULT_CONCURRENCY,
       commandTimeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
       signal: controller.signal,
+      onGenerationResult: (result) => {
+        void trackMintTestGenerationResult({
+          ...result,
+          mode: 'interactive',
+          resumed,
+        });
+      },
       onTaskUpdate: (update) => {
         const phaseStage =
           update.phase === 'running'
@@ -1060,6 +1083,11 @@ function MintTestApp({
       },
     })
       .then((nextReport) => {
+        void trackMintTestRunCompleted(nextReport, {
+          mode: 'interactive',
+          resumed,
+          totalPageCount: totalPages,
+        });
         setReport(nextReport);
         const failedFiles = nextReport.tasks
           .filter((task) => task.status === 'failed' || task.status === 'agent_error')