@mintlify/cli 4.0.1655 → 4.0.1657
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +1 -7
- package/bin/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +8 -12
- package/src/cli.tsx +1 -12
- package/__test__/mintTestFilePreview.test.ts +0 -35
- package/__test__/mintTestOutput.test.ts +0 -256
- package/__test__/runHistory.test.ts +0 -240
- package/bin/agent-harness/agentPreflight.js +0 -24
- package/bin/agent-harness/buildTaskPrompt.js +0 -86
- package/bin/agent-harness/countReport.js +0 -10
- package/bin/agent-harness/discoverCodeBlocks.js +0 -101
- package/bin/agent-harness/executeCodeBlocks.js +0 -263
- package/bin/agent-harness/filterSelectedPages.js +0 -10
- package/bin/agent-harness/generateTestCode.js +0 -337
- package/bin/agent-harness/index.js +0 -95
- package/bin/agent-harness/loadAgentSdk.js +0 -46
- package/bin/agent-harness/manifest.js +0 -113
- package/bin/agent-harness/mapWithConcurrency.js +0 -26
- package/bin/agent-harness/runHistory.js +0 -256
- package/bin/agent-harness/runSave.js +0 -47
- package/bin/agent-harness/setupFolders.js +0 -73
- package/bin/agent-harness/tasks/checkTestability.js +0 -228
- package/bin/agent-harness/types.js +0 -66
- package/bin/mintTest.js +0 -122
- package/bin/mintTestFilePreview.js +0 -39
- package/bin/mintTestNav.js +0 -35
- package/bin/mintTestTelemetry.js +0 -63
- package/bin/mintTestText.js +0 -31
- package/bin/mintTestTree.js +0 -94
- package/bin/mintTestUi.js +0 -818
- package/src/agent-harness/MINT_TEST_SYSTEM_DESIGN.md +0 -64
- package/src/agent-harness/agentPreflight.ts +0 -13
- package/src/agent-harness/buildTaskPrompt.ts +0 -93
- package/src/agent-harness/countReport.ts +0 -17
- package/src/agent-harness/discoverCodeBlocks.ts +0 -90
- package/src/agent-harness/executeCodeBlocks.ts +0 -351
- package/src/agent-harness/filterSelectedPages.ts +0 -14
- package/src/agent-harness/generateTestCode.ts +0 -396
- package/src/agent-harness/index.ts +0 -127
- package/src/agent-harness/loadAgentSdk.ts +0 -35
- package/src/agent-harness/manifest.ts +0 -116
- package/src/agent-harness/mapWithConcurrency.ts +0 -21
- package/src/agent-harness/runHistory.ts +0 -255
- package/src/agent-harness/runSave.ts +0 -38
- package/src/agent-harness/setupFolders.ts +0 -87
- package/src/agent-harness/tasks/checkTestability.ts +0 -256
- package/src/agent-harness/types.ts +0 -239
- package/src/mintTest.tsx +0 -138
- package/src/mintTestFilePreview.ts +0 -32
- package/src/mintTestNav.ts +0 -31
- package/src/mintTestTelemetry.ts +0 -101
- package/src/mintTestText.ts +0 -35
- package/src/mintTestTree.ts +0 -134
- package/src/mintTestUi.tsx +0 -1378
|
@@ -1,256 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { randomUUID } from 'node:crypto';
|
|
11
|
-
import fs from 'node:fs/promises';
|
|
12
|
-
import path from 'node:path';
|
|
13
|
-
import { setTimeout as delay } from 'node:timers/promises';
|
|
14
|
-
import { z } from 'zod';
|
|
15
|
-
import { countReport } from './countReport.js';
|
|
16
|
-
const RUN_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
|
|
17
|
-
const INDEX_LOCK_FILENAME = 'testRuns.lock';
|
|
18
|
-
const LOCK_RETRY_MS = 20;
|
|
19
|
-
const STALE_LOCK_MS = 30000;
|
|
20
|
-
const LOCK_TIMEOUT_MS = STALE_LOCK_MS + 10000;
|
|
21
|
-
const commandResultSchema = z
|
|
22
|
-
.object({
|
|
23
|
-
id: z.string(),
|
|
24
|
-
phase: z.enum(['setup', 'test']),
|
|
25
|
-
status: z.enum(['passed', 'failed', 'cancelled']),
|
|
26
|
-
executable: z.string(),
|
|
27
|
-
args: z.array(z.string()),
|
|
28
|
-
cwd: z.string(),
|
|
29
|
-
durationMs: z.number().nonnegative(),
|
|
30
|
-
exitCode: z.number().int().nullable(),
|
|
31
|
-
stdout: z.string(),
|
|
32
|
-
stderr: z.string(),
|
|
33
|
-
error: z.string().optional(),
|
|
34
|
-
})
|
|
35
|
-
.strict();
|
|
36
|
-
const taskResultSchema = z
|
|
37
|
-
.object({
|
|
38
|
-
id: z.string(),
|
|
39
|
-
agent: z.enum(['claude', 'codex']),
|
|
40
|
-
file: z.string(),
|
|
41
|
-
directory: z.string(),
|
|
42
|
-
generatedFiles: z.array(z.string()),
|
|
43
|
-
testable: z.boolean(),
|
|
44
|
-
attempts: z.number().int().nonnegative(),
|
|
45
|
-
tokens: z.number().nonnegative(),
|
|
46
|
-
status: z.enum(['passed', 'failed', 'agent_error', 'cancelled']),
|
|
47
|
-
error: z.string().optional(),
|
|
48
|
-
commands: z.array(commandResultSchema),
|
|
49
|
-
})
|
|
50
|
-
.strict();
|
|
51
|
-
const codeTestReportSchema = z
|
|
52
|
-
.object({
|
|
53
|
-
version: z.literal(1),
|
|
54
|
-
runId: z.string().regex(RUN_ID_PATTERN),
|
|
55
|
-
status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
|
|
56
|
-
docsRoot: z.string(),
|
|
57
|
-
outputDirectory: z.string(),
|
|
58
|
-
reportPath: z.string(),
|
|
59
|
-
agents: z.array(z.enum(['claude', 'codex'])).min(1),
|
|
60
|
-
model: z.string().min(1),
|
|
61
|
-
selectedFiles: z.array(z.string()),
|
|
62
|
-
startedAt: z.string(),
|
|
63
|
-
completedAt: z.string(),
|
|
64
|
-
durationMs: z.number().nonnegative(),
|
|
65
|
-
tasks: z.array(taskResultSchema),
|
|
66
|
-
historyError: z.string().optional(),
|
|
67
|
-
})
|
|
68
|
-
.strict();
|
|
69
|
-
const countsSchema = z
|
|
70
|
-
.object({
|
|
71
|
-
passed: z.number().int().nonnegative(),
|
|
72
|
-
failed: z.number().int().nonnegative(),
|
|
73
|
-
cancelled: z.number().int().nonnegative(),
|
|
74
|
-
agentErrors: z.number().int().nonnegative(),
|
|
75
|
-
})
|
|
76
|
-
.strict();
|
|
77
|
-
const testRunSummarySchema = z
|
|
78
|
-
.object({
|
|
79
|
-
runId: z.string().regex(RUN_ID_PATTERN),
|
|
80
|
-
status: z.enum(['passed', 'completed', 'failed', 'cancelled']),
|
|
81
|
-
agents: z.array(z.enum(['claude', 'codex'])).min(1),
|
|
82
|
-
model: z.string().min(1),
|
|
83
|
-
selectedFiles: z.array(z.string()),
|
|
84
|
-
startedAt: z.string(),
|
|
85
|
-
completedAt: z.string(),
|
|
86
|
-
durationMs: z.number().nonnegative(),
|
|
87
|
-
reportPath: z.string(),
|
|
88
|
-
outputDirectory: z.string(),
|
|
89
|
-
counts: countsSchema,
|
|
90
|
-
})
|
|
91
|
-
.strict();
|
|
92
|
-
const testRunsIndexSchema = z
|
|
93
|
-
.object({ version: z.literal(1), runs: z.array(testRunSummarySchema) })
|
|
94
|
-
.strict();
|
|
95
|
-
function historyDirectory(docsRoot) {
|
|
96
|
-
return path.join(docsRoot, '.mintlify', 'test');
|
|
97
|
-
}
|
|
98
|
-
function runsDirectory(docsRoot) {
|
|
99
|
-
return path.join(historyDirectory(docsRoot), 'runs');
|
|
100
|
-
}
|
|
101
|
-
export function testRunReportPath(docsRoot, runId) {
|
|
102
|
-
if (!RUN_ID_PATTERN.test(runId))
|
|
103
|
-
throw new Error(`Invalid mint test run ID: ${runId}`);
|
|
104
|
-
return path.join(runsDirectory(docsRoot), `${runId}.json`);
|
|
105
|
-
}
|
|
106
|
-
export function testRunsIndexPath(docsRoot) {
|
|
107
|
-
return path.join(historyDirectory(docsRoot), 'testRuns.json');
|
|
108
|
-
}
|
|
109
|
-
function summarizeReport(report) {
|
|
110
|
-
return {
|
|
111
|
-
runId: report.runId,
|
|
112
|
-
status: report.status,
|
|
113
|
-
agents: report.agents,
|
|
114
|
-
model: report.model,
|
|
115
|
-
selectedFiles: report.selectedFiles,
|
|
116
|
-
startedAt: report.startedAt,
|
|
117
|
-
completedAt: report.completedAt,
|
|
118
|
-
durationMs: report.durationMs,
|
|
119
|
-
reportPath: report.reportPath,
|
|
120
|
-
outputDirectory: report.outputDirectory,
|
|
121
|
-
counts: countReport(report.tasks),
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
function errorCode(error) {
|
|
125
|
-
return typeof error === 'object' && error !== null && 'code' in error
|
|
126
|
-
? String(error.code)
|
|
127
|
-
: undefined;
|
|
128
|
-
}
|
|
129
|
-
function writeJsonAtomically(filePath, value) {
|
|
130
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
131
|
-
const temporaryPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${randomUUID()}.tmp`);
|
|
132
|
-
try {
|
|
133
|
-
yield fs.writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx' });
|
|
134
|
-
yield fs.rename(temporaryPath, filePath);
|
|
135
|
-
}
|
|
136
|
-
finally {
|
|
137
|
-
yield fs.unlink(temporaryPath).catch(() => undefined);
|
|
138
|
-
}
|
|
139
|
-
});
|
|
140
|
-
}
|
|
141
|
-
function acquireIndexLock(docsRoot) {
|
|
142
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
143
|
-
const lockPath = path.join(historyDirectory(docsRoot), INDEX_LOCK_FILENAME);
|
|
144
|
-
const token = `${process.pid}:${randomUUID()}`;
|
|
145
|
-
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
146
|
-
while (Date.now() < deadline) {
|
|
147
|
-
let lock;
|
|
148
|
-
try {
|
|
149
|
-
lock = yield fs.open(lockPath, 'wx');
|
|
150
|
-
}
|
|
151
|
-
catch (error) {
|
|
152
|
-
const code = errorCode(error);
|
|
153
|
-
// Windows can deny opening a lock while another writer's deletion is still pending.
|
|
154
|
-
if (code !== 'EEXIST' && !(process.platform === 'win32' && code === 'EPERM'))
|
|
155
|
-
throw error;
|
|
156
|
-
const stats = yield fs.stat(lockPath).catch(() => undefined);
|
|
157
|
-
if (stats && Date.now() - stats.mtimeMs > STALE_LOCK_MS) {
|
|
158
|
-
yield fs.unlink(lockPath).catch(() => undefined);
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
yield delay(LOCK_RETRY_MS);
|
|
162
|
-
continue;
|
|
163
|
-
}
|
|
164
|
-
try {
|
|
165
|
-
yield lock.writeFile(token);
|
|
166
|
-
}
|
|
167
|
-
finally {
|
|
168
|
-
yield lock.close();
|
|
169
|
-
}
|
|
170
|
-
return () => __awaiter(this, void 0, void 0, function* () {
|
|
171
|
-
const owner = yield fs.readFile(lockPath, 'utf8').catch(() => undefined);
|
|
172
|
-
if (owner === token)
|
|
173
|
-
yield fs.unlink(lockPath).catch(() => undefined);
|
|
174
|
-
});
|
|
175
|
-
}
|
|
176
|
-
throw new Error('Timed out updating mint test run history');
|
|
177
|
-
});
|
|
178
|
-
}
|
|
179
|
-
function readReport(filePath) {
|
|
180
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
181
|
-
try {
|
|
182
|
-
return codeTestReportSchema.parse(JSON.parse(yield fs.readFile(filePath, 'utf8')));
|
|
183
|
-
}
|
|
184
|
-
catch (_a) {
|
|
185
|
-
return null;
|
|
186
|
-
}
|
|
187
|
-
});
|
|
188
|
-
}
|
|
189
|
-
function rebuildIndex(docsRoot) {
|
|
190
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
191
|
-
const entries = yield fs
|
|
192
|
-
.readdir(runsDirectory(docsRoot), { withFileTypes: true })
|
|
193
|
-
.catch(() => []);
|
|
194
|
-
const reports = yield Promise.all(entries
|
|
195
|
-
.filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
|
|
196
|
-
.map((entry) => readReport(path.join(runsDirectory(docsRoot), entry.name))));
|
|
197
|
-
return {
|
|
198
|
-
version: 1,
|
|
199
|
-
runs: reports
|
|
200
|
-
.filter((report) => report !== null)
|
|
201
|
-
.map(summarizeReport)
|
|
202
|
-
.sort((left, right) => right.completedAt.localeCompare(left.completedAt)),
|
|
203
|
-
};
|
|
204
|
-
});
|
|
205
|
-
}
|
|
206
|
-
export function loadTestRuns(docsRoot) {
|
|
207
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
208
|
-
try {
|
|
209
|
-
return testRunsIndexSchema.parse(JSON.parse(yield fs.readFile(testRunsIndexPath(docsRoot), 'utf8')));
|
|
210
|
-
}
|
|
211
|
-
catch (_a) {
|
|
212
|
-
return rebuildIndex(docsRoot);
|
|
213
|
-
}
|
|
214
|
-
});
|
|
215
|
-
}
|
|
216
|
-
export function loadLatestTestRun(docsRoot) {
|
|
217
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
218
|
-
const index = yield loadTestRuns(docsRoot);
|
|
219
|
-
for (const run of index.runs) {
|
|
220
|
-
const report = yield readReport(testRunReportPath(docsRoot, run.runId));
|
|
221
|
-
if (report)
|
|
222
|
-
return report;
|
|
223
|
-
}
|
|
224
|
-
return null;
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
|
-
export function writeTestRunHistory(docsRoot, report) {
|
|
228
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
229
|
-
yield fs.mkdir(runsDirectory(docsRoot), { recursive: true });
|
|
230
|
-
yield writeJsonAtomically(report.reportPath, report);
|
|
231
|
-
const releaseLock = yield acquireIndexLock(docsRoot);
|
|
232
|
-
try {
|
|
233
|
-
yield writeJsonAtomically(testRunsIndexPath(docsRoot), yield rebuildIndex(docsRoot));
|
|
234
|
-
}
|
|
235
|
-
finally {
|
|
236
|
-
yield releaseLock();
|
|
237
|
-
}
|
|
238
|
-
});
|
|
239
|
-
}
|
|
240
|
-
/**
|
|
241
|
-
* Saves a finished run to history without letting a history failure fail the run.
|
|
242
|
-
* On failure the returned report points at `fallbackReportPath` (the report that was
|
|
243
|
-
* already written) and carries the problem in `historyError`.
|
|
244
|
-
*/
|
|
245
|
-
export function recordTestRun(docsRoot, report, fallbackReportPath) {
|
|
246
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
247
|
-
try {
|
|
248
|
-
yield writeTestRunHistory(docsRoot, report);
|
|
249
|
-
return report;
|
|
250
|
-
}
|
|
251
|
-
catch (error) {
|
|
252
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
253
|
-
return Object.assign(Object.assign({}, report), { reportPath: fallbackReportPath, historyError: `run history could not be saved: ${message}` });
|
|
254
|
-
}
|
|
255
|
-
});
|
|
256
|
-
}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import fs from 'node:fs/promises';
|
|
11
|
-
import path from 'node:path';
|
|
12
|
-
import { z } from 'zod';
|
|
13
|
-
const runSaveSchema = z
|
|
14
|
-
.object({
|
|
15
|
-
version: z.literal(1),
|
|
16
|
-
runId: z.string().regex(/^[A-Za-z0-9._-]+$/),
|
|
17
|
-
agents: z.array(z.enum(['claude', 'codex'])).min(1),
|
|
18
|
-
model: z.string().min(1),
|
|
19
|
-
selectedFiles: z.array(z.string().min(1)),
|
|
20
|
-
createdAt: z.string(),
|
|
21
|
-
})
|
|
22
|
-
.strict();
|
|
23
|
-
function savePath(outputDirectory) {
|
|
24
|
-
return path.join(outputDirectory, 'save.json');
|
|
25
|
-
}
|
|
26
|
-
export function loadRunSave(outputDirectory) {
|
|
27
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
28
|
-
try {
|
|
29
|
-
const content = yield fs.readFile(savePath(outputDirectory), 'utf8');
|
|
30
|
-
return runSaveSchema.parse(JSON.parse(content));
|
|
31
|
-
}
|
|
32
|
-
catch (_a) {
|
|
33
|
-
return null;
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
export function writeRunSave(outputDirectory, save) {
|
|
38
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
39
|
-
yield fs.mkdir(outputDirectory, { recursive: true });
|
|
40
|
-
yield fs.writeFile(savePath(outputDirectory), `${JSON.stringify(save, null, 2)}\n`);
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
export function clearRunSave(outputDirectory) {
|
|
44
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
45
|
-
yield fs.unlink(savePath(outputDirectory)).catch(() => undefined);
|
|
46
|
-
});
|
|
47
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { randomBytes } from 'node:crypto';
|
|
11
|
-
import fs from 'node:fs/promises';
|
|
12
|
-
import path from 'node:path';
|
|
13
|
-
const PACKAGE_JSON = `${JSON.stringify({
|
|
14
|
-
name: 'mint-test-task',
|
|
15
|
-
private: true,
|
|
16
|
-
type: 'module',
|
|
17
|
-
// No positional path: Node 21+ treats --test arguments as globs, so a bare
|
|
18
|
-
// directory such as `tests/` fails with MODULE_NOT_FOUND. The default patterns
|
|
19
|
-
// pick up tests/*.test.js on every supported Node version.
|
|
20
|
-
scripts: { test: 'node --test' },
|
|
21
|
-
}, null, 2)}\n`;
|
|
22
|
-
function isExistingFile(error) {
|
|
23
|
-
return (typeof error === 'object' && error !== null && error.code === 'EEXIST');
|
|
24
|
-
}
|
|
25
|
-
function scaffoldTaskDirectory(directory) {
|
|
26
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
27
|
-
yield fs.mkdir(path.join(directory, 'tests'), { recursive: true });
|
|
28
|
-
try {
|
|
29
|
-
yield fs.writeFile(path.join(directory, 'package.json'), PACKAGE_JSON, { flag: 'wx' });
|
|
30
|
-
}
|
|
31
|
-
catch (error) {
|
|
32
|
-
if (!isExistingFile(error))
|
|
33
|
-
throw error;
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
export function scaffoldTaskDirectories(tasks) {
|
|
38
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
39
|
-
for (const task of tasks)
|
|
40
|
-
yield scaffoldTaskDirectory(task.directory);
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
export function setupFolders(_a) {
|
|
44
|
-
return __awaiter(this, arguments, void 0, function* ({ pages, agents, docsRoot, outputDirectory, runId: savedRunId, }) {
|
|
45
|
-
const runId = savedRunId !== null && savedRunId !== void 0 ? savedRunId : `${new Date()
|
|
46
|
-
.toISOString()
|
|
47
|
-
.replace(/[-:TZ.]/g, '')
|
|
48
|
-
.slice(0, 14)}-${randomBytes(4).toString('hex')}`;
|
|
49
|
-
const runDirectory = path.join(path.resolve(outputDirectory), runId);
|
|
50
|
-
const reportPath = path.join(runDirectory, 'report.json');
|
|
51
|
-
yield fs.mkdir(runDirectory, { recursive: true });
|
|
52
|
-
const tasks = [];
|
|
53
|
-
for (const agent of agents) {
|
|
54
|
-
for (const [pageIndex, page] of pages.entries()) {
|
|
55
|
-
const slug = page.file
|
|
56
|
-
.replace(/\.[^.]+$/, '')
|
|
57
|
-
.replace(/[^a-z0-9]+/gi, '-')
|
|
58
|
-
.replace(/^-|-$/g, '')
|
|
59
|
-
.slice(0, 56);
|
|
60
|
-
tasks.push({
|
|
61
|
-
id: `${agent}:${page.file}`,
|
|
62
|
-
agent,
|
|
63
|
-
docsRoot,
|
|
64
|
-
file: page.file,
|
|
65
|
-
directory: path.join(runDirectory, agent, `${String(pageIndex + 1).padStart(3, '0')}-${slug || 'page'}`),
|
|
66
|
-
attempts: 0,
|
|
67
|
-
tokens: 0,
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
return { runId, runDirectory, reportPath, tasks };
|
|
72
|
-
});
|
|
73
|
-
}
|
|
@@ -1,228 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
11
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
12
|
-
var m = o[Symbol.asyncIterator], i;
|
|
13
|
-
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
14
|
-
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
15
|
-
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
16
|
-
};
|
|
17
|
-
import fs from 'node:fs/promises';
|
|
18
|
-
import path from 'node:path';
|
|
19
|
-
import z from 'zod';
|
|
20
|
-
import { buildCheckPrompt, CHECK_JSON_INSTRUCTION, CHECK_SYSTEM_PROMPT, CHECK_TOOL_INSTRUCTION, } from '../buildTaskPrompt.js';
|
|
21
|
-
import { loadClaudeAgentSdk, loadCodexSdk } from '../loadAgentSdk.js';
|
|
22
|
-
import { testCheckSchema, } from '../types.js';
|
|
23
|
-
const CHECK_FILENAME = 'mint-test-check.json';
|
|
24
|
-
const CHECK_MODEL = 'claude-haiku-4-5-20251001';
|
|
25
|
-
function readValidCheck(directory) {
|
|
26
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
27
|
-
try {
|
|
28
|
-
const check = testCheckSchema.parse(JSON.parse(yield fs.readFile(path.join(directory, CHECK_FILENAME), 'utf8')));
|
|
29
|
-
return validateCheck(check).length === 0 ? check : null;
|
|
30
|
-
}
|
|
31
|
-
catch (_a) {
|
|
32
|
-
return null;
|
|
33
|
-
}
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
function validateCheck(check) {
|
|
37
|
-
return !check.testable && !check.reason ? ['Not-testable page needs a reason'] : [];
|
|
38
|
-
}
|
|
39
|
-
function parseJsonReply(reply) {
|
|
40
|
-
var _a;
|
|
41
|
-
const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(reply);
|
|
42
|
-
const raw = (_a = fenced === null || fenced === void 0 ? void 0 : fenced[1]) !== null && _a !== void 0 ? _a : reply;
|
|
43
|
-
const start = raw.indexOf('{');
|
|
44
|
-
const end = raw.lastIndexOf('}');
|
|
45
|
-
if (start === -1 || end === -1)
|
|
46
|
-
throw new Error('Reply contained no JSON object');
|
|
47
|
-
return JSON.parse(raw.slice(start, end + 1));
|
|
48
|
-
}
|
|
49
|
-
function runTestabilityCheck(_a) {
|
|
50
|
-
return __awaiter(this, arguments, void 0, function* ({ file, content, agent, workingDirectory, signal, }) {
|
|
51
|
-
const prompt = buildCheckPrompt(file, content);
|
|
52
|
-
let tokens = 0;
|
|
53
|
-
let raw;
|
|
54
|
-
if (agent === 'claude') {
|
|
55
|
-
const reply = yield promptClaudeCheck({ prompt, workingDirectory, signal });
|
|
56
|
-
tokens = reply.tokens;
|
|
57
|
-
if (reply.captured === null)
|
|
58
|
-
throw new Error('agent never called report_testability');
|
|
59
|
-
raw = reply.captured;
|
|
60
|
-
}
|
|
61
|
-
else {
|
|
62
|
-
const reply = yield promptCodex({
|
|
63
|
-
prompt,
|
|
64
|
-
systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_JSON_INSTRUCTION,
|
|
65
|
-
workingDirectory,
|
|
66
|
-
signal,
|
|
67
|
-
});
|
|
68
|
-
tokens = reply.tokens;
|
|
69
|
-
raw = parseJsonReply(reply.result);
|
|
70
|
-
}
|
|
71
|
-
const check = testCheckSchema.parse(raw);
|
|
72
|
-
const errors = validateCheck(check);
|
|
73
|
-
if (errors.length > 0)
|
|
74
|
-
throw new Error(errors.join('\n'));
|
|
75
|
-
return { tokens, check };
|
|
76
|
-
});
|
|
77
|
-
}
|
|
78
|
-
export const checkFileTestability = (task_1, _a) => __awaiter(void 0, [task_1, _a], void 0, function* (task, { onTaskUpdate, signal, runDirectory, }) {
|
|
79
|
-
const emit = (phase, tokens) => onTaskUpdate({
|
|
80
|
-
id: task.id,
|
|
81
|
-
agent: task.agent,
|
|
82
|
-
file: task.file,
|
|
83
|
-
directory: task.directory,
|
|
84
|
-
generatedFiles: [],
|
|
85
|
-
phase,
|
|
86
|
-
tokens,
|
|
87
|
-
});
|
|
88
|
-
let attempts = 0;
|
|
89
|
-
let tokens = 0;
|
|
90
|
-
let lastError = 'run cancelled';
|
|
91
|
-
let check = yield readValidCheck(task.directory);
|
|
92
|
-
if (!check) {
|
|
93
|
-
emit('checking', tokens);
|
|
94
|
-
const content = yield fs.readFile(path.join(task.docsRoot, task.file), 'utf8');
|
|
95
|
-
for (let attempt = 1; attempt <= 2 && !signal.aborted && !check; attempt++) {
|
|
96
|
-
attempts = attempt;
|
|
97
|
-
try {
|
|
98
|
-
const reply = yield runTestabilityCheck({
|
|
99
|
-
file: task.file,
|
|
100
|
-
content,
|
|
101
|
-
agent: task.agent,
|
|
102
|
-
workingDirectory: runDirectory,
|
|
103
|
-
signal,
|
|
104
|
-
});
|
|
105
|
-
tokens += reply.tokens;
|
|
106
|
-
check = reply.check;
|
|
107
|
-
}
|
|
108
|
-
catch (error) {
|
|
109
|
-
lastError = error instanceof Error ? error.message : String(error);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
if (!check) {
|
|
113
|
-
if (signal.aborted) {
|
|
114
|
-
emit('cancelled', tokens);
|
|
115
|
-
return Object.assign(Object.assign({}, task), { attempts, tokens, error: 'run cancelled' });
|
|
116
|
-
}
|
|
117
|
-
emit('agent_error', tokens);
|
|
118
|
-
return Object.assign(Object.assign({}, task), { attempts, tokens, error: lastError });
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
if (!check.testable) {
|
|
122
|
-
emit('not_testable', tokens);
|
|
123
|
-
return Object.assign(Object.assign({}, task), { attempts, tokens, testable: false });
|
|
124
|
-
}
|
|
125
|
-
yield fs.mkdir(task.directory, { recursive: true });
|
|
126
|
-
yield fs.writeFile(path.join(task.directory, CHECK_FILENAME), `${JSON.stringify(check, null, 2)}\n`);
|
|
127
|
-
emit('checked', tokens);
|
|
128
|
-
return Object.assign(Object.assign({}, task), { attempts, tokens, testable: true });
|
|
129
|
-
});
|
|
130
|
-
function promptClaudeCheck(_a) {
|
|
131
|
-
return __awaiter(this, arguments, void 0, function* ({ prompt, workingDirectory, signal, }) {
|
|
132
|
-
var _b, e_1, _c, _d;
|
|
133
|
-
const { query, tool, createSdkMcpServer } = yield loadClaudeAgentSdk();
|
|
134
|
-
const abortController = new AbortController();
|
|
135
|
-
const abort = () => abortController.abort(signal.reason);
|
|
136
|
-
if (signal.aborted)
|
|
137
|
-
abort();
|
|
138
|
-
else
|
|
139
|
-
signal.addEventListener('abort', abort, { once: true });
|
|
140
|
-
let captured = null;
|
|
141
|
-
const server = createSdkMcpServer({
|
|
142
|
-
name: 'mint-test',
|
|
143
|
-
tools: [
|
|
144
|
-
tool('report_testability', 'Report whether the documentation page is testable. Call exactly once.', {
|
|
145
|
-
version: z.literal(1),
|
|
146
|
-
testable: z.boolean(),
|
|
147
|
-
reason: z.string().optional(),
|
|
148
|
-
}, (input) => __awaiter(this, void 0, void 0, function* () {
|
|
149
|
-
captured = input;
|
|
150
|
-
return { content: [{ type: 'text', text: 'recorded' }] };
|
|
151
|
-
})),
|
|
152
|
-
],
|
|
153
|
-
});
|
|
154
|
-
const session = query({
|
|
155
|
-
prompt,
|
|
156
|
-
options: {
|
|
157
|
-
abortController,
|
|
158
|
-
allowedTools: ['mcp__mint-test__report_testability'],
|
|
159
|
-
cwd: workingDirectory,
|
|
160
|
-
mcpServers: { 'mint-test': server },
|
|
161
|
-
model: CHECK_MODEL,
|
|
162
|
-
permissionMode: 'dontAsk',
|
|
163
|
-
persistSession: false,
|
|
164
|
-
settingSources: [],
|
|
165
|
-
strictMcpConfig: true,
|
|
166
|
-
systemPrompt: CHECK_SYSTEM_PROMPT + CHECK_TOOL_INSTRUCTION,
|
|
167
|
-
tools: [],
|
|
168
|
-
},
|
|
169
|
-
});
|
|
170
|
-
try {
|
|
171
|
-
try {
|
|
172
|
-
for (var _e = true, session_1 = __asyncValues(session), session_1_1; session_1_1 = yield session_1.next(), _b = session_1_1.done, !_b; _e = true) {
|
|
173
|
-
_d = session_1_1.value;
|
|
174
|
-
_e = false;
|
|
175
|
-
const message = _d;
|
|
176
|
-
if (message.type !== 'result')
|
|
177
|
-
continue;
|
|
178
|
-
if (message.subtype !== 'success' || message.is_error) {
|
|
179
|
-
const details = message.subtype === 'success' ? message.result : message.errors.join('; ');
|
|
180
|
-
throw new Error(details || `Claude stopped with ${message.subtype}`);
|
|
181
|
-
}
|
|
182
|
-
const usage = message.usage;
|
|
183
|
-
return {
|
|
184
|
-
tokens: usage.input_tokens +
|
|
185
|
-
usage.output_tokens +
|
|
186
|
-
usage.cache_creation_input_tokens +
|
|
187
|
-
usage.cache_read_input_tokens,
|
|
188
|
-
captured,
|
|
189
|
-
};
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
193
|
-
finally {
|
|
194
|
-
try {
|
|
195
|
-
if (!_e && !_b && (_c = session_1.return)) yield _c.call(session_1);
|
|
196
|
-
}
|
|
197
|
-
finally { if (e_1) throw e_1.error; }
|
|
198
|
-
}
|
|
199
|
-
throw new Error('Claude finished without returning a result');
|
|
200
|
-
}
|
|
201
|
-
finally {
|
|
202
|
-
signal.removeEventListener('abort', abort);
|
|
203
|
-
session.close();
|
|
204
|
-
}
|
|
205
|
-
});
|
|
206
|
-
}
|
|
207
|
-
function promptCodex(_a) {
|
|
208
|
-
return __awaiter(this, arguments, void 0, function* ({ prompt, systemPrompt, workingDirectory, signal, }) {
|
|
209
|
-
const { Codex } = yield loadCodexSdk();
|
|
210
|
-
const thread = new Codex().startThread({
|
|
211
|
-
approvalPolicy: 'never',
|
|
212
|
-
networkAccessEnabled: true,
|
|
213
|
-
sandboxMode: 'workspace-write',
|
|
214
|
-
skipGitRepoCheck: true,
|
|
215
|
-
workingDirectory,
|
|
216
|
-
});
|
|
217
|
-
const result = yield thread.run(`${systemPrompt}\n\n${prompt}`, { signal });
|
|
218
|
-
const usage = result.usage;
|
|
219
|
-
const tokens = usage
|
|
220
|
-
? usage.input_tokens +
|
|
221
|
-
usage.cached_input_tokens +
|
|
222
|
-
usage.cache_write_input_tokens +
|
|
223
|
-
usage.output_tokens +
|
|
224
|
-
usage.reasoning_output_tokens
|
|
225
|
-
: 0;
|
|
226
|
-
return { tokens, result: result.finalResponse };
|
|
227
|
-
});
|
|
228
|
-
}
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
import { z } from 'zod';
|
|
2
|
-
const identifier = z.string().trim().min(1).max(120);
|
|
3
|
-
const commandSchema = z
|
|
4
|
-
.object({
|
|
5
|
-
id: identifier.optional(),
|
|
6
|
-
executable: z.string().trim().min(1).max(512),
|
|
7
|
-
args: z.array(z.string().max(10000)).max(100),
|
|
8
|
-
cwd: z.string().max(1024).optional(),
|
|
9
|
-
timeoutMs: z
|
|
10
|
-
.number()
|
|
11
|
-
.int()
|
|
12
|
-
.min(1000)
|
|
13
|
-
.max(30 * 60 * 1000)
|
|
14
|
-
.optional(),
|
|
15
|
-
})
|
|
16
|
-
.strict();
|
|
17
|
-
const manifestFileSchema = z
|
|
18
|
-
.object({
|
|
19
|
-
version: z.literal(1),
|
|
20
|
-
generatedFiles: z.array(z.string().min(1).max(1024)).max(1000),
|
|
21
|
-
setupCommands: z.array(commandSchema).max(50),
|
|
22
|
-
testCommands: z.array(commandSchema).max(200),
|
|
23
|
-
})
|
|
24
|
-
.strict();
|
|
25
|
-
/**
|
|
26
|
-
* Agents often copy the manifest example verbatim and omit command ids. The id only needs to be
|
|
27
|
-
* unique within the manifest, so assign `setup-N` / `test-N` instead of rejecting the whole page.
|
|
28
|
-
*/
|
|
29
|
-
export function assignCommandIds(manifest) {
|
|
30
|
-
const taken = new Set([...manifest.setupCommands, ...manifest.testCommands]
|
|
31
|
-
.map((command) => command.id)
|
|
32
|
-
.filter((id) => id !== undefined));
|
|
33
|
-
const withIds = (commands, phase) => {
|
|
34
|
-
let next = 1;
|
|
35
|
-
return commands.map((command) => {
|
|
36
|
-
if (command.id !== undefined)
|
|
37
|
-
return Object.assign(Object.assign({}, command), { id: command.id });
|
|
38
|
-
let id = `${phase}-${next}`;
|
|
39
|
-
while (taken.has(id)) {
|
|
40
|
-
next += 1;
|
|
41
|
-
id = `${phase}-${next}`;
|
|
42
|
-
}
|
|
43
|
-
taken.add(id);
|
|
44
|
-
next += 1;
|
|
45
|
-
return Object.assign(Object.assign({}, command), { id });
|
|
46
|
-
});
|
|
47
|
-
};
|
|
48
|
-
return {
|
|
49
|
-
version: manifest.version,
|
|
50
|
-
generatedFiles: manifest.generatedFiles,
|
|
51
|
-
setupCommands: withIds(manifest.setupCommands, 'setup'),
|
|
52
|
-
testCommands: withIds(manifest.testCommands, 'test'),
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
export const manifestSchema = manifestFileSchema.transform(assignCommandIds);
|
|
56
|
-
export const testCheckSchema = z
|
|
57
|
-
.object({
|
|
58
|
-
version: z.literal(1),
|
|
59
|
-
testable: z.boolean(),
|
|
60
|
-
reason: z
|
|
61
|
-
.string()
|
|
62
|
-
.max(2000)
|
|
63
|
-
.optional()
|
|
64
|
-
.transform((value) => ((value === null || value === void 0 ? void 0 : value.trim()) ? value.trim() : undefined)),
|
|
65
|
-
})
|
|
66
|
-
.strict();
|