@mintlify/cli 4.0.1466 → 4.0.1467

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/__test__/mintTestFilePreview.test.ts +35 -0
  2. package/__test__/mintTestOutput.test.ts +194 -0
  3. package/__test__/runHistory.test.ts +164 -0
  4. package/bin/agent-harness/agentPreflight.js +24 -0
  5. package/bin/agent-harness/buildTaskPrompt.js +17 -7
  6. package/bin/agent-harness/countReport.js +10 -0
  7. package/bin/agent-harness/executeCodeBlocks.js +53 -10
  8. package/bin/agent-harness/generateTestCode.js +58 -43
  9. package/bin/agent-harness/index.js +14 -5
  10. package/bin/agent-harness/manifest.js +113 -0
  11. package/bin/agent-harness/runHistory.js +252 -0
  12. package/bin/agent-harness/setupFolders.js +20 -6
  13. package/bin/agent-harness/tasks/checkTestability.js +7 -5
  14. package/bin/agent-harness/types.js +33 -2
  15. package/bin/constants.js +3 -3
  16. package/bin/mintTest.js +31 -9
  17. package/bin/mintTestFilePreview.js +39 -0
  18. package/bin/mintTestText.js +31 -0
  19. package/bin/mintTestUi.js +295 -105
  20. package/bin/status.js +26 -0
  21. package/bin/tsconfig.build.tsbuildinfo +1 -1
  22. package/package.json +2 -2
  23. package/src/agent-harness/MINT_TEST_SYSTEM_DESIGN.md +15 -9
  24. package/src/agent-harness/agentPreflight.ts +13 -0
  25. package/src/agent-harness/buildTaskPrompt.ts +18 -7
  26. package/src/agent-harness/countReport.ts +17 -0
  27. package/src/agent-harness/executeCodeBlocks.ts +67 -1
  28. package/src/agent-harness/generateTestCode.ts +81 -51
  29. package/src/agent-harness/index.ts +20 -5
  30. package/src/agent-harness/manifest.ts +116 -0
  31. package/src/agent-harness/runHistory.ts +250 -0
  32. package/src/agent-harness/setupFolders.ts +19 -5
  33. package/src/agent-harness/tasks/checkTestability.ts +15 -6
  34. package/src/agent-harness/types.ts +81 -5
  35. package/src/constants.ts +6 -2
  36. package/src/mintTest.tsx +34 -8
  37. package/src/mintTestFilePreview.ts +32 -0
  38. package/src/mintTestText.ts +35 -0
  39. package/src/mintTestUi.tsx +586 -224
  40. package/src/status.tsx +24 -0
@@ -0,0 +1,250 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import fs from 'node:fs/promises';
3
+ import path from 'node:path';
4
+ import { setTimeout as delay } from 'node:timers/promises';
5
+ import { z } from 'zod';
6
+
7
+ import { countReport } from './countReport.js';
8
+ import type { CodeTestReport, TestRunSummary, TestRunsIndex } from './types.js';
9
+
10
+ const RUN_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
11
+ const INDEX_LOCK_FILENAME = 'testRuns.lock';
12
+ const LOCK_RETRY_MS = 20;
13
+ const STALE_LOCK_MS = 30_000;
14
+ const LOCK_TIMEOUT_MS = STALE_LOCK_MS + 10_000;
15
+ const commandResultSchema = z
16
+ .object({
17
+ id: z.string(),
18
+ phase: z.enum(['setup', 'test']),
19
+ status: z.enum(['passed', 'failed', 'cancelled']),
20
+ executable: z.string(),
21
+ args: z.array(z.string()),
22
+ cwd: z.string(),
23
+ durationMs: z.number().nonnegative(),
24
+ exitCode: z.number().int().nullable(),
25
+ stdout: z.string(),
26
+ stderr: z.string(),
27
+ error: z.string().optional(),
28
+ })
29
+ .strict();
30
+ const taskResultSchema = z
31
+ .object({
32
+ id: z.string(),
33
+ agent: z.enum(['claude', 'codex']),
34
+ file: z.string(),
35
+ directory: z.string(),
36
+ generatedFiles: z.array(z.string()),
37
+ testable: z.boolean(),
38
+ attempts: z.number().int().nonnegative(),
39
+ tokens: z.number().nonnegative(),
40
+ status: z.enum(['passed', 'failed', 'agent_error', 'cancelled']),
41
+ error: z.string().optional(),
42
+ commands: z.array(commandResultSchema),
43
+ })
44
+ .strict();
45
+ const codeTestReportSchema = z
46
+ .object({
47
+ version: z.literal(1),
48
+ runId: z.string().regex(RUN_ID_PATTERN),
49
+ status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
50
+ docsRoot: z.string(),
51
+ outputDirectory: z.string(),
52
+ reportPath: z.string(),
53
+ agents: z.array(z.enum(['claude', 'codex'])).min(1),
54
+ model: z.string().min(1),
55
+ selectedFiles: z.array(z.string()),
56
+ startedAt: z.string(),
57
+ completedAt: z.string(),
58
+ durationMs: z.number().nonnegative(),
59
+ tasks: z.array(taskResultSchema),
60
+ historyError: z.string().optional(),
61
+ })
62
+ .strict();
63
+ const countsSchema = z
64
+ .object({
65
+ passed: z.number().int().nonnegative(),
66
+ failed: z.number().int().nonnegative(),
67
+ cancelled: z.number().int().nonnegative(),
68
+ agentErrors: z.number().int().nonnegative(),
69
+ })
70
+ .strict();
71
+ const testRunSummarySchema = z
72
+ .object({
73
+ runId: z.string().regex(RUN_ID_PATTERN),
74
+ status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
75
+ agents: z.array(z.enum(['claude', 'codex'])).min(1),
76
+ model: z.string().min(1),
77
+ selectedFiles: z.array(z.string()),
78
+ startedAt: z.string(),
79
+ completedAt: z.string(),
80
+ durationMs: z.number().nonnegative(),
81
+ reportPath: z.string(),
82
+ outputDirectory: z.string(),
83
+ counts: countsSchema,
84
+ })
85
+ .strict();
86
+ const testRunsIndexSchema = z
87
+ .object({ version: z.literal(1), runs: z.array(testRunSummarySchema) })
88
+ .strict();
89
+
90
+ function historyDirectory(docsRoot: string): string {
91
+ return path.join(docsRoot, '.mintlify', 'test');
92
+ }
93
+
94
+ function runsDirectory(docsRoot: string): string {
95
+ return path.join(historyDirectory(docsRoot), 'runs');
96
+ }
97
+
98
+ export function testRunReportPath(docsRoot: string, runId: string): string {
99
+ if (!RUN_ID_PATTERN.test(runId)) throw new Error(`Invalid mint test run ID: ${runId}`);
100
+ return path.join(runsDirectory(docsRoot), `${runId}.json`);
101
+ }
102
+
103
+ export function testRunsIndexPath(docsRoot: string): string {
104
+ return path.join(historyDirectory(docsRoot), 'testRuns.json');
105
+ }
106
+
107
+ function summarizeReport(report: CodeTestReport): TestRunSummary {
108
+ return {
109
+ runId: report.runId,
110
+ status: report.status,
111
+ agents: report.agents,
112
+ model: report.model,
113
+ selectedFiles: report.selectedFiles,
114
+ startedAt: report.startedAt,
115
+ completedAt: report.completedAt,
116
+ durationMs: report.durationMs,
117
+ reportPath: report.reportPath,
118
+ outputDirectory: report.outputDirectory,
119
+ counts: countReport(report.tasks),
120
+ };
121
+ }
122
+
123
+ function errorCode(error: unknown): string | undefined {
124
+ return typeof error === 'object' && error !== null && 'code' in error
125
+ ? String(error.code)
126
+ : undefined;
127
+ }
128
+
129
+ async function writeJsonAtomically(filePath: string, value: unknown): Promise<void> {
130
+ const temporaryPath = path.join(
131
+ path.dirname(filePath),
132
+ `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`
133
+ );
134
+ try {
135
+ await fs.writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx' });
136
+ await fs.rename(temporaryPath, filePath);
137
+ } finally {
138
+ await fs.unlink(temporaryPath).catch(() => undefined);
139
+ }
140
+ }
141
+
142
+ async function acquireIndexLock(docsRoot: string): Promise<() => Promise<void>> {
143
+ const lockPath = path.join(historyDirectory(docsRoot), INDEX_LOCK_FILENAME);
144
+ const token = `${process.pid}:${randomUUID()}`;
145
+ const deadline = Date.now() + LOCK_TIMEOUT_MS;
146
+
147
+ while (Date.now() < deadline) {
148
+ try {
149
+ const lock = await fs.open(lockPath, 'wx');
150
+ try {
151
+ await lock.writeFile(token);
152
+ } finally {
153
+ await lock.close();
154
+ }
155
+ return async () => {
156
+ const owner = await fs.readFile(lockPath, 'utf8').catch(() => undefined);
157
+ if (owner === token) await fs.unlink(lockPath).catch(() => undefined);
158
+ };
159
+ } catch (error) {
160
+ if (errorCode(error) !== 'EEXIST') throw error;
161
+ }
162
+
163
+ const stats = await fs.stat(lockPath).catch(() => undefined);
164
+ if (stats && Date.now() - stats.mtimeMs > STALE_LOCK_MS) {
165
+ await fs.unlink(lockPath).catch(() => undefined);
166
+ continue;
167
+ }
168
+ await delay(LOCK_RETRY_MS);
169
+ }
170
+ throw new Error('Timed out updating mint test run history');
171
+ }
172
+
173
+ async function readReport(filePath: string): Promise<CodeTestReport | null> {
174
+ try {
175
+ return codeTestReportSchema.parse(JSON.parse(await fs.readFile(filePath, 'utf8')));
176
+ } catch {
177
+ return null;
178
+ }
179
+ }
180
+
181
+ async function rebuildIndex(docsRoot: string): Promise<TestRunsIndex> {
182
+ const entries = await fs
183
+ .readdir(runsDirectory(docsRoot), { withFileTypes: true })
184
+ .catch(() => []);
185
+ const reports = await Promise.all(
186
+ entries
187
+ .filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
188
+ .map((entry) => readReport(path.join(runsDirectory(docsRoot), entry.name)))
189
+ );
190
+ return {
191
+ version: 1,
192
+ runs: reports
193
+ .filter((report): report is CodeTestReport => report !== null)
194
+ .map(summarizeReport)
195
+ .sort((left, right) => right.completedAt.localeCompare(left.completedAt)),
196
+ };
197
+ }
198
+
199
+ export async function loadTestRuns(docsRoot: string): Promise<TestRunsIndex> {
200
+ try {
201
+ return testRunsIndexSchema.parse(
202
+ JSON.parse(await fs.readFile(testRunsIndexPath(docsRoot), 'utf8'))
203
+ );
204
+ } catch {
205
+ return rebuildIndex(docsRoot);
206
+ }
207
+ }
208
+
209
+ export async function loadLatestTestRun(docsRoot: string): Promise<CodeTestReport | null> {
210
+ const index = await loadTestRuns(docsRoot);
211
+ for (const run of index.runs) {
212
+ const report = await readReport(testRunReportPath(docsRoot, run.runId));
213
+ if (report) return report;
214
+ }
215
+ return null;
216
+ }
217
+
218
+ export async function writeTestRunHistory(docsRoot: string, report: CodeTestReport): Promise<void> {
219
+ await fs.mkdir(runsDirectory(docsRoot), { recursive: true });
220
+ await writeJsonAtomically(report.reportPath, report);
221
+ const releaseLock = await acquireIndexLock(docsRoot);
222
+ try {
223
+ await writeJsonAtomically(testRunsIndexPath(docsRoot), await rebuildIndex(docsRoot));
224
+ } finally {
225
+ await releaseLock();
226
+ }
227
+ }
228
+
229
+ /**
230
+ * Saves a finished run to history without letting a history failure fail the run.
231
+ * On failure the returned report points at `fallbackReportPath` (the report that was
232
+ * already written) and carries the problem in `historyError`.
233
+ */
234
+ export async function recordTestRun(
235
+ docsRoot: string,
236
+ report: CodeTestReport,
237
+ fallbackReportPath: string
238
+ ): Promise<CodeTestReport> {
239
+ try {
240
+ await writeTestRunHistory(docsRoot, report);
241
+ return report;
242
+ } catch (error) {
243
+ const message = error instanceof Error ? error.message : String(error);
244
+ return {
245
+ ...report,
246
+ reportPath: fallbackReportPath,
247
+ historyError: `run history could not be saved: ${message}`,
248
+ };
249
+ }
250
+ }
@@ -9,17 +9,32 @@ const PACKAGE_JSON = `${JSON.stringify(
9
9
  name: 'mint-test-task',
10
10
  private: true,
11
11
  type: 'module',
12
- scripts: { test: 'node --test tests/' },
12
+ // No positional path: Node 21+ treats --test arguments as globs, so a bare
13
+ // directory such as `tests/` fails with MODULE_NOT_FOUND. The default patterns
14
+ // pick up tests/*.test.js on every supported Node version.
15
+ scripts: { test: 'node --test' },
13
16
  },
14
17
  null,
15
18
  2
16
19
  )}\n`;
17
20
 
21
+ function isExistingFile(error: unknown): boolean {
22
+ return (
23
+ typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'EEXIST'
24
+ );
25
+ }
26
+
18
27
  async function scaffoldTaskDirectory(directory: string): Promise<void> {
19
28
  await fs.mkdir(path.join(directory, 'tests'), { recursive: true });
20
- await fs
21
- .writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' })
22
- .catch(() => {});
29
+ try {
30
+ await fs.writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' });
31
+ } catch (error) {
32
+ if (!isExistingFile(error)) throw error;
33
+ }
34
+ }
35
+
36
+ export async function scaffoldTaskDirectories(tasks: GeneratedTask[]): Promise<void> {
37
+ for (const task of tasks) await scaffoldTaskDirectory(task.directory);
23
38
  }
24
39
 
25
40
  export async function setupFolders({
@@ -68,6 +83,5 @@ export async function setupFolders({
68
83
  });
69
84
  }
70
85
  }
71
- for (const task of tasks) await scaffoldTaskDirectory(task.directory);
72
86
  return { runId, runDirectory, reportPath, tasks };
73
87
  }
@@ -50,17 +50,16 @@ async function runTestabilityCheck({
50
50
  file,
51
51
  content,
52
52
  agent,
53
- outputPath,
53
+ workingDirectory,
54
54
  signal,
55
55
  }: {
56
56
  file: string;
57
57
  content: string;
58
58
  agent: HarnessAgent;
59
- outputPath: string;
59
+ workingDirectory: string;
60
60
  signal: AbortSignal;
61
61
  }): Promise<{ tokens: number; check: TestCheck }> {
62
62
  const prompt = buildCheckPrompt(file, content);
63
- const workingDirectory = path.dirname(outputPath);
64
63
  let tokens = 0;
65
64
  let raw: unknown;
66
65
  if (agent === 'claude') {
@@ -81,19 +80,24 @@ async function runTestabilityCheck({
81
80
  const check = testCheckSchema.parse(raw);
82
81
  const errors = validateCheck(check);
83
82
  if (errors.length > 0) throw new Error(errors.join('\n'));
84
- await fs.writeFile(outputPath, `${JSON.stringify(check, null, 2)}\n`);
85
83
  return { tokens, check };
86
84
  }
87
85
 
88
86
  export const checkFileTestability = async (
89
87
  task: GeneratedTask,
90
- { onTaskUpdate, signal }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'>
88
+ {
89
+ onTaskUpdate,
90
+ signal,
91
+ runDirectory,
92
+ }: Pick<RunCodeTestsOptions, 'onTaskUpdate' | 'signal'> & { runDirectory: string }
91
93
  ): Promise<GeneratedTask> => {
92
94
  const emit = (phase: TaskUpdate['phase'], tokens: number) =>
93
95
  onTaskUpdate({
94
96
  id: task.id,
95
97
  agent: task.agent,
96
98
  file: task.file,
99
+ directory: task.directory,
100
+ generatedFiles: [],
97
101
  phase,
98
102
  tokens,
99
103
  });
@@ -112,7 +116,7 @@ export const checkFileTestability = async (
112
116
  file: task.file,
113
117
  content,
114
118
  agent: task.agent,
115
- outputPath: path.join(task.directory, CHECK_FILENAME),
119
+ workingDirectory: runDirectory,
116
120
  signal,
117
121
  });
118
122
  tokens += reply.tokens;
@@ -135,6 +139,11 @@ export const checkFileTestability = async (
135
139
  emit('not_testable', tokens);
136
140
  return { ...task, attempts, tokens, testable: false };
137
141
  }
142
+ await fs.mkdir(task.directory, { recursive: true });
143
+ await fs.writeFile(
144
+ path.join(task.directory, CHECK_FILENAME),
145
+ `${JSON.stringify(check, null, 2)}\n`
146
+ );
138
147
  emit('checked', tokens);
139
148
  return { ...task, attempts, tokens, testable: true };
140
149
  };
@@ -30,6 +30,8 @@ export interface TaskResult {
30
30
  agent: HarnessAgent;
31
31
  file: string;
32
32
  directory: string;
33
+ generatedFiles: string[];
34
+ testable: boolean;
33
35
  attempts: number;
34
36
  tokens: number;
35
37
  status: TaskStatus;
@@ -45,11 +47,38 @@ export interface CodeTestReport {
45
47
  outputDirectory: string;
46
48
  reportPath: string;
47
49
  agents: HarnessAgent[];
50
+ model: string;
48
51
  selectedFiles: string[];
49
52
  startedAt: string;
50
53
  completedAt: string;
51
54
  durationMs: number;
52
55
  tasks: TaskResult[];
56
+ /** Set when the run finished but could not be saved to `.mintlify/test` history. */
57
+ historyError?: string;
58
+ }
59
+
60
+ export interface TestRunSummary {
61
+ runId: string;
62
+ status: CodeTestReport['status'];
63
+ agents: HarnessAgent[];
64
+ model: string;
65
+ selectedFiles: string[];
66
+ startedAt: string;
67
+ completedAt: string;
68
+ durationMs: number;
69
+ reportPath: string;
70
+ outputDirectory: string;
71
+ counts: {
72
+ passed: number;
73
+ failed: number;
74
+ cancelled: number;
75
+ agentErrors: number;
76
+ };
77
+ }
78
+
79
+ export interface TestRunsIndex {
80
+ version: 1;
81
+ runs: TestRunSummary[];
53
82
  }
54
83
 
55
84
  export type TaskPhase =
@@ -68,8 +97,12 @@ export interface TaskUpdate {
68
97
  id: string;
69
98
  agent: HarnessAgent;
70
99
  file: string;
100
+ directory: string;
101
+ generatedFiles: string[];
71
102
  phase: TaskPhase;
72
103
  tokens: number;
104
+ /** Why the task ended in `agent_error` or `failed`; the first line is shown in the UI. */
105
+ error?: string;
73
106
  }
74
107
 
75
108
  export interface RunSave {
@@ -98,7 +131,7 @@ const identifier = z.string().trim().min(1).max(120);
98
131
 
99
132
  const commandSchema = z
100
133
  .object({
101
- id: identifier,
134
+ id: identifier.optional(),
102
135
  executable: z.string().trim().min(1).max(512),
103
136
  args: z.array(z.string().max(10_000)).max(100),
104
137
  cwd: z.string().max(1_024).optional(),
@@ -111,7 +144,7 @@ const commandSchema = z
111
144
  })
112
145
  .strict();
113
146
 
114
- export const manifestSchema = z
147
+ const manifestFileSchema = z
115
148
  .object({
116
149
  version: z.literal(1),
117
150
  generatedFiles: z.array(z.string().min(1).max(1_024)).max(1_000),
@@ -120,7 +153,51 @@ export const manifestSchema = z
120
153
  })
121
154
  .strict();
122
155
 
123
- export type GeneratedTestManifest = z.infer<typeof manifestSchema>;
156
+ type ManifestFile = z.infer<typeof manifestFileSchema>;
157
+ type ManifestFileCommand = ManifestFile['testCommands'][number];
158
+
159
+ export type ManifestCommand = Omit<ManifestFileCommand, 'id'> & { id: string };
160
+
161
+ export interface GeneratedTestManifest {
162
+ version: 1;
163
+ generatedFiles: string[];
164
+ setupCommands: ManifestCommand[];
165
+ testCommands: ManifestCommand[];
166
+ }
167
+
168
+ /**
169
+ * Agents often copy the manifest example verbatim and omit command ids. The id only needs to be
170
+ * unique within the manifest, so assign `setup-N` / `test-N` instead of rejecting the whole page.
171
+ */
172
+ export function assignCommandIds(manifest: ManifestFile): GeneratedTestManifest {
173
+ const taken = new Set(
174
+ [...manifest.setupCommands, ...manifest.testCommands]
175
+ .map((command) => command.id)
176
+ .filter((id): id is string => id !== undefined)
177
+ );
178
+ const withIds = (commands: ManifestFileCommand[], phase: 'setup' | 'test'): ManifestCommand[] => {
179
+ let next = 1;
180
+ return commands.map((command) => {
181
+ if (command.id !== undefined) return { ...command, id: command.id };
182
+ let id = `${phase}-${next}`;
183
+ while (taken.has(id)) {
184
+ next += 1;
185
+ id = `${phase}-${next}`;
186
+ }
187
+ taken.add(id);
188
+ next += 1;
189
+ return { ...command, id };
190
+ });
191
+ };
192
+ return {
193
+ version: manifest.version,
194
+ generatedFiles: manifest.generatedFiles,
195
+ setupCommands: withIds(manifest.setupCommands, 'setup'),
196
+ testCommands: withIds(manifest.testCommands, 'test'),
197
+ };
198
+ }
199
+
200
+ export const manifestSchema = manifestFileSchema.transform(assignCommandIds);
124
201
 
125
202
  export const testCheckSchema = z
126
203
  .object({
@@ -136,8 +213,6 @@ export const testCheckSchema = z
136
213
 
137
214
  export type TestCheck = z.infer<typeof testCheckSchema>;
138
215
 
139
- export type ManifestCommand = GeneratedTestManifest['testCommands'][number];
140
-
141
216
  export interface GeneratedTask {
142
217
  id: string;
143
218
  agent: HarnessAgent;
@@ -148,5 +223,6 @@ export interface GeneratedTask {
148
223
  tokens: number;
149
224
  testable?: boolean;
150
225
  manifest?: GeneratedTestManifest;
226
+ generatedFiles?: string[];
151
227
  error?: string;
152
228
  }
package/src/constants.ts CHANGED
@@ -29,7 +29,11 @@ const PROD_TOKEN_ENDPOINT =
29
29
  'https://api.stytch.com/v1/public/project-live-731b7a04-9ac3-4923-90b8-0806d4aa29d4/oauth2/token';
30
30
  const PROD_STYTCH_CLIENT_ID = 'connected-app-live-d813eedd-dbb0-434b-a1f9-2ce69e5efc49';
31
31
 
32
- export const TOKEN_ENDPOINT = IS_LOCAL_BUILD ? DEV_TOKEN_ENDPOINT : PROD_TOKEN_ENDPOINT;
33
- export const STYTCH_CLIENT_ID = IS_LOCAL_BUILD ? DEV_STYTCH_CLIENT_ID : PROD_STYTCH_CLIENT_ID;
32
+ export const TOKEN_ENDPOINT =
33
+ process.env.MINTLIFY_TOKEN_ENDPOINT ??
34
+ (IS_LOCAL_BUILD ? DEV_TOKEN_ENDPOINT : PROD_TOKEN_ENDPOINT);
35
+ export const STYTCH_CLIENT_ID =
36
+ process.env.MINTLIFY_STYTCH_CLIENT_ID ??
37
+ (IS_LOCAL_BUILD ? DEV_STYTCH_CLIENT_ID : PROD_STYTCH_CLIENT_ID);
34
38
 
35
39
  export const CUSTOM_DOMAIN_CNAME_TARGET = 'cname.mintlify.builders';
package/src/mintTest.tsx CHANGED
@@ -4,16 +4,22 @@ import {
4
4
  DEFAULT_COMMAND_TIMEOUT_MS,
5
5
  DEFAULT_CONCURRENCY,
6
6
  clearRunSave,
7
+ countReport,
7
8
  discoverCodeBlocks,
9
+ loadLatestTestRun,
8
10
  loadRunSave,
9
11
  runCodeTests,
10
12
  type RunSave,
11
13
  } from './agent-harness/index.js';
14
+ import { API_URL } from './constants.js';
12
15
  import { CMD_EXEC_PATH, findDocsRoot, isAI, terminate } from './helpers.js';
13
16
  import { getAccessToken } from './keyring.js';
14
17
  import { fileInNavPaths, loadDocsNavScope } from './mintTestNav.js';
15
- import { countReport, formatDuration, runMintTestUi } from './mintTestUi.js';
16
- import { getCliStatus } from './status.js';
18
+ import { firstLine, taskProblemLabel } from './mintTestText.js';
19
+ import { formatDuration, runMintTestUi } from './mintTestUi.js';
20
+ import { fetchCliStatus } from './status.js';
21
+
22
+ const ALLOWED_ORGS = new Set(['mintlify', 'mintlify internal']);
17
23
 
18
24
  export async function mintTestHandler(): Promise<void> {
19
25
  try {
@@ -23,10 +29,21 @@ export async function mintTestHandler(): Promise<void> {
23
29
  await terminate(1);
24
30
  return;
25
31
  }
26
- const cliStatus = await getCliStatus(accessToken);
27
- const orgName = cliStatus?.org.name.toLowerCase();
28
- if (orgName !== 'mintlify' && orgName !== 'mintlify internal') {
29
- process.stderr.write('mint test is not available for your organization.\n');
32
+ const cliStatus = await fetchCliStatus();
33
+ if (!cliStatus.ok) {
34
+ const message =
35
+ cliStatus.reason === 'unauthenticated'
36
+ ? 'your session has expired. Run `mint login` to authenticate.'
37
+ : cliStatus.reason === 'unreachable'
38
+ ? `could not reach Mintlify at ${API_URL}${cliStatus.detail ? ` (${cliStatus.detail})` : ''}.`
39
+ : 'unexpected response from Mintlify. Please try again.';
40
+ process.stderr.write(`${message}\n`);
41
+ await terminate(1);
42
+ return;
43
+ }
44
+ const orgName = cliStatus.status.org.name;
45
+ if (!ALLOWED_ORGS.has(orgName.toLowerCase())) {
46
+ process.stderr.write(`mint test is not available for your organization (${orgName}).\n`);
30
47
  await terminate(1);
31
48
  return;
32
49
  }
@@ -46,7 +63,6 @@ export async function mintTestHandler(): Promise<void> {
46
63
  const autoSelectedFiles = (
47
64
  scope ? pages.filter((page) => fileInNavPaths(page.file, scope.autoSelectedPaths)) : pages
48
65
  ).map((page) => page.file);
49
- const totalPages = scope ? scope.navPaths.size : pages.length;
50
66
  const outputDirectory = path.join(docsRoot, 'tests', 'mint-test');
51
67
 
52
68
  let save: RunSave | null = await loadRunSave(outputDirectory);
@@ -60,6 +76,7 @@ export async function mintTestHandler(): Promise<void> {
60
76
  save = { ...save, selectedFiles };
61
77
  }
62
78
  }
79
+ const lastReport = await loadLatestTestRun(docsRoot);
63
80
 
64
81
  if (interactive) {
65
82
  const exitCode = await runMintTestUi({
@@ -67,8 +84,8 @@ export async function mintTestHandler(): Promise<void> {
67
84
  docsRoot,
68
85
  pages,
69
86
  autoSelectedFiles,
70
- totalPages,
71
87
  save,
88
+ lastReport,
72
89
  });
73
90
  await terminate(exitCode);
74
91
  return;
@@ -93,6 +110,15 @@ export async function mintTestHandler(): Promise<void> {
93
110
  );
94
111
  process.stdout.write(`finished in ${formatDuration(report.durationMs)}\n`);
95
112
  process.stdout.write(`report: ${report.reportPath}\n`);
113
+ for (const task of report.tasks) {
114
+ if (task.status !== 'failed' && task.status !== 'agent_error') continue;
115
+ const reason = firstLine(task.error);
116
+ process.stdout.write(
117
+ ` ✗ ${task.file}: ${taskProblemLabel(task.status)}${reason ? ` (${reason})` : ''}\n`
118
+ );
119
+ process.stdout.write(` output: ${task.directory}\n`);
120
+ }
121
+ if (report.historyError) process.stderr.write(`warning: ${report.historyError}\n`);
96
122
  await terminate(report.status === 'passed' ? 0 : 1);
97
123
  } catch (error) {
98
124
  process.stderr.write(
@@ -0,0 +1,32 @@
1
+ import fs from 'node:fs/promises';
2
+
3
+ export const PREVIEW_BYTE_LIMIT = 64 * 1024;
4
+
5
+ export type FilePreview = { binary: true } | { binary: false; text: string; truncated: boolean };
6
+
7
+ export async function readFilePreview(filePath: string): Promise<FilePreview> {
8
+ const buffer = new Uint8Array(PREVIEW_BYTE_LIMIT + 1);
9
+ const file = await fs.open(filePath, 'r');
10
+ let bytesRead = 0;
11
+
12
+ try {
13
+ while (bytesRead < buffer.length) {
14
+ const result = await file.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead);
15
+ if (result.bytesRead === 0) break;
16
+ bytesRead += result.bytesRead;
17
+ }
18
+ } finally {
19
+ await file.close();
20
+ }
21
+
22
+ const contents = buffer.subarray(0, bytesRead);
23
+ if (contents.includes(0)) return { binary: true };
24
+
25
+ return {
26
+ binary: false,
27
+ text: new TextDecoder()
28
+ .decode(contents.subarray(0, PREVIEW_BYTE_LIMIT))
29
+ .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, '�'),
30
+ truncated: bytesRead > PREVIEW_BYTE_LIMIT,
31
+ };
32
+ }
@@ -0,0 +1,35 @@
1
+ export interface FooterSegment {
2
+ key: string;
3
+ action: string;
4
+ }
5
+
6
+ /**
7
+ * Split a footer hint such as `↑/↓/j/k move · Enter exit` into key/action pairs so the key can
8
+ * be rendered bold and the action plain.
9
+ */
10
+ export function footerSegments(text: string): FooterSegment[] {
11
+ return text
12
+ .split(' · ')
13
+ .map((segment) => segment.trim())
14
+ .filter((segment) => segment.length > 0)
15
+ .map((segment) => {
16
+ const split = segment.indexOf(' ');
17
+ if (split === -1) return { key: segment, action: '' };
18
+ return { key: segment.slice(0, split), action: segment.slice(split + 1).trim() };
19
+ });
20
+ }
21
+
22
+ /** First non-empty line of a multi-line message, trimmed to fit a terminal row. */
23
+ export function firstLine(text: string | undefined, maxLength = 160): string {
24
+ if (!text) return '';
25
+ const line = text
26
+ .split(/\r?\n/)
27
+ .map((candidate) => candidate.trim())
28
+ .find((candidate) => candidate.length > 0);
29
+ if (!line) return '';
30
+ return line.length > maxLength ? `${line.slice(0, maxLength - 1)}…` : line;
31
+ }
32
+
33
+ export function taskProblemLabel(status: 'failed' | 'agent_error'): string {
34
+ return status === 'agent_error' ? 'test could not be generated' : 'tests failed';
35
+ }