@popoverai/dotrequirements 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -3
- package/dist/cli.js +1 -1
- package/dist/commands/browsertest.d.ts +0 -1
- package/dist/commands/browsertest.js +24 -6
- package/dist/commands/finalize.d.ts +1 -0
- package/dist/commands/finalize.js +1 -0
- package/dist/harness/cache.d.ts +22 -17
- package/dist/harness/cache.js +47 -10
- package/dist/harness/finalize.d.ts +5 -0
- package/dist/harness/finalize.js +64 -18
- package/dist/mcp/index.js +9 -11
- package/dist/schema/browser.d.ts +2 -0
- package/dist/schema/browser.js +2 -0
- package/dist/schema/index.d.ts +2 -0
- package/dist/schema/index.js +2 -0
- package/dist/schema/scenario.d.ts +91 -0
- package/dist/schema/scenario.js +82 -0
- package/dist/utils/project-settings.d.ts +4 -3
- package/dist/utils/project-settings.js +6 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -182,18 +182,21 @@ dotreq browsertest LOGIN-1 --json
|
|
|
182
182
|
|
|
183
183
|
**Configuration required:**
|
|
184
184
|
|
|
185
|
-
Browser testing
|
|
185
|
+
Browser testing needs a Stagehand model and the API key for that model's provider in `project-settings.json`:
|
|
186
186
|
|
|
187
187
|
```json
|
|
188
188
|
{
|
|
189
189
|
"defaultURL": "https://your-app.com",
|
|
190
190
|
"browserTest": {
|
|
191
|
-
"
|
|
191
|
+
"modelName": "gateway/anthropic/claude-haiku-4-5",
|
|
192
|
+
"modelApiKey": "your-api-key"
|
|
192
193
|
}
|
|
193
194
|
}
|
|
194
195
|
```
|
|
195
196
|
|
|
196
|
-
|
|
197
|
+
Pick any Stagehand-supported model. Example values: `gateway/anthropic/claude-haiku-4-5` (Vercel AI Gateway key), `google/gemini-3-flash-preview` (Gemini key).
|
|
198
|
+
|
|
199
|
+
Optional settings: `vercelBypassSecret`, `browserbaseApiKey`, `browserbaseProjectId`. Setting both Browserbase fields switches runs from LOCAL (spawns Playwright on your machine) to BROWSERBASE (managed cloud browsers).
|
|
197
200
|
|
|
198
201
|
### `dotreq mcp-setup`
|
|
199
202
|
|
|
@@ -276,6 +279,14 @@ await finalize({ showTestedList: true }); // Include tested requirements
|
|
|
276
279
|
await finalize({ showSummary: false, showUntestedList: false }); // Quiet mode
|
|
277
280
|
```
|
|
278
281
|
|
|
282
|
+
**Attribution:** tag cloud coverage rows with the name of the framework that produced them:
|
|
283
|
+
|
|
284
|
+
```typescript
|
|
285
|
+
await finalize({ context: 'Vitest' });
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
The Coverage tab uses this label to show which tool produced each result. Any string works — useful when you have multiple test suites (`"Vitest — unit"`, `"Vitest — integration"`).
|
|
289
|
+
|
|
279
290
|
By default, only untested requirements are shown. See the [test harness docs](https://docs.dotrequirements.io/tools/test-harness) for all options.
|
|
280
291
|
|
|
281
292
|
### Setup with Jest
|
package/dist/cli.js
CHANGED
|
@@ -78,7 +78,6 @@ program
|
|
|
78
78
|
.command('browsertest <requirement-key> [url]')
|
|
79
79
|
.description('Run browser-based acceptance test for a requirement')
|
|
80
80
|
.option('--json', 'Output results as JSON')
|
|
81
|
-
.option('--useAgent', 'Encourage Claude to use the Agent tool for multi-step tasks')
|
|
82
81
|
.action(wrapCommand(browserTestCommand));
|
|
83
82
|
program
|
|
84
83
|
.command('prepare')
|
|
@@ -90,6 +89,7 @@ program
|
|
|
90
89
|
.description('Aggregate test tracking data and generate coverage report')
|
|
91
90
|
.option('--push', 'Push coverage to DotRequirements Cloud')
|
|
92
91
|
.option('-q, --quiet', 'Output only coverage percentage (for scripting)')
|
|
92
|
+
.option('--context <label>', 'Attribution label identifying the reporter (e.g. "Vitest", "pytest")')
|
|
93
93
|
.action(wrapCommand(finalizeCommand));
|
|
94
94
|
program
|
|
95
95
|
.command('report')
|
|
@@ -44,11 +44,22 @@ export async function browserTestCommand(requirementKey, url, options) {
|
|
|
44
44
|
console.log('Either provide a URL argument or add "defaultURL" to .requirements/project-settings.json');
|
|
45
45
|
process.exit(1);
|
|
46
46
|
}
|
|
47
|
-
// 3.
|
|
47
|
+
// 3. Resolve model config
|
|
48
|
+
const modelName = settings?.browserTest?.modelName;
|
|
49
|
+
const modelApiKey = settings?.browserTest?.modelApiKey;
|
|
48
50
|
const geminiApiKey = settings?.browserTest?.geminiApiKey;
|
|
49
|
-
if (!geminiApiKey) {
|
|
50
|
-
console.error('Error:
|
|
51
|
-
console.log('Add
|
|
51
|
+
if (!modelName && !modelApiKey && !geminiApiKey) {
|
|
52
|
+
console.error('Error: no browser-test model configured in project-settings.json.');
|
|
53
|
+
console.log('Add a block like:\n' +
|
|
54
|
+
' "browserTest": {\n' +
|
|
55
|
+
' "modelName": "gateway/anthropic/claude-haiku-4-5",\n' +
|
|
56
|
+
' "modelApiKey": "..."\n' +
|
|
57
|
+
' }\n' +
|
|
58
|
+
'to .requirements/project-settings.json. See the docs for supported models.');
|
|
59
|
+
process.exit(1);
|
|
60
|
+
}
|
|
61
|
+
if ((modelName && !modelApiKey) || (modelApiKey && !modelName)) {
|
|
62
|
+
console.error('Error: browserTest.modelName and browserTest.modelApiKey must be set together.');
|
|
52
63
|
process.exit(1);
|
|
53
64
|
}
|
|
54
65
|
// 4. Load requirements
|
|
@@ -67,7 +78,13 @@ export async function browserTestCommand(requirementKey, url, options) {
|
|
|
67
78
|
const assertions = requirementTree.map((req) => formatRequirement(req));
|
|
68
79
|
// 7. Build environment with injected secrets
|
|
69
80
|
const env = { ...process.env };
|
|
70
|
-
env
|
|
81
|
+
// env var keeps the key out of process listings (not a CLI flag)
|
|
82
|
+
if (modelApiKey) {
|
|
83
|
+
env.MODEL_API_KEY = modelApiKey;
|
|
84
|
+
}
|
|
85
|
+
else if (geminiApiKey) {
|
|
86
|
+
env.GEMINI_API_KEY = geminiApiKey;
|
|
87
|
+
}
|
|
71
88
|
if (settings?.browserTest?.vercelBypassSecret) {
|
|
72
89
|
env.VERCEL_AUTOMATION_BYPASS_SECRET = settings.browserTest.vercelBypassSecret;
|
|
73
90
|
}
|
|
@@ -82,7 +99,8 @@ export async function browserTestCommand(requirementKey, url, options) {
|
|
|
82
99
|
console.log(`Testing ${requirementKey} against ${targetURL}...\n`);
|
|
83
100
|
}
|
|
84
101
|
try {
|
|
85
|
-
const
|
|
102
|
+
const modelFlags = modelName ? ['--modelName', modelName] : [];
|
|
103
|
+
const { stdout } = await execFileAsync('npx', ['@popoverai/browser-automation', 'test', ...modelFlags, targetURL, ...assertions], { env, maxBuffer: 10 * 1024 * 1024 });
|
|
86
104
|
// 9. Parse results
|
|
87
105
|
let results;
|
|
88
106
|
try {
|
|
@@ -31,6 +31,7 @@ export async function finalizeCommand(options) {
|
|
|
31
31
|
showTestedList: true,
|
|
32
32
|
showUntestedList: true,
|
|
33
33
|
showCloudStatus: !options.quiet,
|
|
34
|
+
context: options.context,
|
|
34
35
|
});
|
|
35
36
|
if (options.quiet) {
|
|
36
37
|
// In quiet mode, just output the coverage percent for scripting
|
package/dist/harness/cache.d.ts
CHANGED
|
@@ -39,17 +39,22 @@ export interface LookupCache {
|
|
|
39
39
|
/**
|
|
40
40
|
* Structure of the coverage.json cache (for cloud deduplication)
|
|
41
41
|
*/
|
|
42
|
+
/** Attribution tuple used for debounce keying (COVERAGE-DEBOUNCE-2). */
|
|
43
|
+
export interface CoverageTuple {
|
|
44
|
+
requirementKey: string;
|
|
45
|
+
context?: string;
|
|
46
|
+
branch?: string;
|
|
47
|
+
}
|
|
48
|
+
export interface CoverageCacheEntry {
|
|
49
|
+
testRunId: string;
|
|
50
|
+
timestamp: number;
|
|
51
|
+
tuples: CoverageTuple[];
|
|
52
|
+
/** Legacy field retained for backward-compat reads of pre-attribution caches. */
|
|
53
|
+
requirementKeys?: string[];
|
|
54
|
+
}
|
|
42
55
|
export interface CoverageCache {
|
|
43
|
-
current:
|
|
44
|
-
|
|
45
|
-
timestamp: number;
|
|
46
|
-
requirementKeys: string[];
|
|
47
|
-
} | null;
|
|
48
|
-
previous: {
|
|
49
|
-
testRunId: string;
|
|
50
|
-
timestamp: number;
|
|
51
|
-
requirementKeys: string[];
|
|
52
|
-
} | null;
|
|
56
|
+
current: CoverageCacheEntry | null;
|
|
57
|
+
previous: CoverageCacheEntry | null;
|
|
53
58
|
}
|
|
54
59
|
/**
|
|
55
60
|
* Find the nearest .requirements directory by walking up from startDir
|
|
@@ -123,16 +128,16 @@ export declare function readTrackingEntries(requirementsDir: string): TrackingEn
|
|
|
123
128
|
* Delete the tracking file after finalize
|
|
124
129
|
*/
|
|
125
130
|
export declare function deleteTrackingFile(requirementsDir: string): void;
|
|
126
|
-
/**
|
|
127
|
-
* Read the coverage cache for cloud deduplication
|
|
128
|
-
*/
|
|
129
131
|
export declare function readCoverageCache(requirementsDir: string): CoverageCache;
|
|
130
132
|
/**
|
|
131
|
-
* Update the coverage cache after reporting
|
|
133
|
+
* Update the coverage cache after reporting. Accepts either attribution tuples
|
|
134
|
+
* (new) or bare requirement keys (legacy; converted to tuples with no context/branch).
|
|
132
135
|
*/
|
|
133
|
-
export declare function updateCoverageCache(requirementsDir: string, testRunId: string,
|
|
136
|
+
export declare function updateCoverageCache(requirementsDir: string, testRunId: string, tuples: CoverageTuple[] | string[]): void;
|
|
137
|
+
export declare const COVERAGE_DEBOUNCE_WINDOW_MS: number;
|
|
134
138
|
/**
|
|
135
|
-
* Check if
|
|
139
|
+
* Check if an attribution tuple needs reporting based on the coverage cache.
|
|
140
|
+
* Returns true if no matching tuple was reported within the debounce window.
|
|
136
141
|
*/
|
|
137
|
-
export declare function needsReporting(
|
|
142
|
+
export declare function needsReporting(tupleOrKey: CoverageTuple | string, cache: CoverageCache): boolean;
|
|
138
143
|
//# sourceMappingURL=cache.d.ts.map
|
package/dist/harness/cache.js
CHANGED
|
@@ -288,6 +288,30 @@ export function deleteTrackingFile(requirementsDir) {
|
|
|
288
288
|
/**
|
|
289
289
|
* Read the coverage cache for cloud deduplication
|
|
290
290
|
*/
|
|
291
|
+
/**
|
|
292
|
+
* Normalize a cache entry read from disk. Migrates legacy `requirementKeys`
|
|
293
|
+
* arrays into the tuple shape (context/branch undefined) so downstream code
|
|
294
|
+
* can treat both shapes uniformly.
|
|
295
|
+
*/
|
|
296
|
+
function normalizeEntry(raw) {
|
|
297
|
+
if (!raw || typeof raw !== 'object')
|
|
298
|
+
return null;
|
|
299
|
+
if (Array.isArray(raw.tuples)) {
|
|
300
|
+
return {
|
|
301
|
+
testRunId: raw.testRunId,
|
|
302
|
+
timestamp: raw.timestamp,
|
|
303
|
+
tuples: raw.tuples,
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
if (Array.isArray(raw.requirementKeys)) {
|
|
307
|
+
return {
|
|
308
|
+
testRunId: raw.testRunId,
|
|
309
|
+
timestamp: raw.timestamp,
|
|
310
|
+
tuples: raw.requirementKeys.map((requirementKey) => ({ requirementKey })),
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
return null;
|
|
314
|
+
}
|
|
291
315
|
export function readCoverageCache(requirementsDir) {
|
|
292
316
|
const cacheDir = getCacheDir(requirementsDir);
|
|
293
317
|
const coveragePath = path.join(cacheDir, COVERAGE_FILE);
|
|
@@ -296,45 +320,58 @@ export function readCoverageCache(requirementsDir) {
|
|
|
296
320
|
}
|
|
297
321
|
try {
|
|
298
322
|
const content = fs.readFileSync(coveragePath, 'utf-8');
|
|
299
|
-
|
|
323
|
+
const raw = JSON.parse(content);
|
|
324
|
+
return {
|
|
325
|
+
current: normalizeEntry(raw?.current),
|
|
326
|
+
previous: normalizeEntry(raw?.previous),
|
|
327
|
+
};
|
|
300
328
|
}
|
|
301
329
|
catch {
|
|
302
330
|
return { current: null, previous: null };
|
|
303
331
|
}
|
|
304
332
|
}
|
|
305
333
|
/**
|
|
306
|
-
* Update the coverage cache after reporting
|
|
334
|
+
* Update the coverage cache after reporting. Accepts either attribution tuples
|
|
335
|
+
* (new) or bare requirement keys (legacy; converted to tuples with no context/branch).
|
|
307
336
|
*/
|
|
308
|
-
export function updateCoverageCache(requirementsDir, testRunId,
|
|
337
|
+
export function updateCoverageCache(requirementsDir, testRunId, tuples) {
|
|
309
338
|
const cacheDir = getCacheDir(requirementsDir, true);
|
|
310
339
|
const coveragePath = path.join(cacheDir, COVERAGE_FILE);
|
|
340
|
+
const normalized = tuples.map((t) => typeof t === 'string' ? { requirementKey: t } : t);
|
|
311
341
|
const existing = readCoverageCache(requirementsDir);
|
|
312
342
|
const newCache = {
|
|
313
343
|
current: {
|
|
314
344
|
testRunId,
|
|
315
345
|
timestamp: Date.now(),
|
|
316
|
-
|
|
346
|
+
tuples: normalized,
|
|
317
347
|
},
|
|
318
348
|
previous: existing.current,
|
|
319
349
|
};
|
|
320
350
|
fs.writeFileSync(coveragePath, JSON.stringify(newCache, null, 2));
|
|
321
351
|
}
|
|
352
|
+
export const COVERAGE_DEBOUNCE_WINDOW_MS = 4 * 60 * 60 * 1000; // 4 hours — must match server-side in testCoverage/mutations.ts
|
|
353
|
+
function tuplesMatch(a, b) {
|
|
354
|
+
return (a.requirementKey === b.requirementKey &&
|
|
355
|
+
a.context === b.context &&
|
|
356
|
+
a.branch === b.branch);
|
|
357
|
+
}
|
|
322
358
|
/**
|
|
323
|
-
* Check if
|
|
359
|
+
* Check if an attribution tuple needs reporting based on the coverage cache.
|
|
360
|
+
* Returns true if no matching tuple was reported within the debounce window.
|
|
324
361
|
*/
|
|
325
|
-
export function needsReporting(
|
|
326
|
-
const
|
|
362
|
+
export function needsReporting(tupleOrKey, cache) {
|
|
363
|
+
const tuple = typeof tupleOrKey === 'string' ? { requirementKey: tupleOrKey } : tupleOrKey;
|
|
327
364
|
// If no current run, everything needs reporting
|
|
328
365
|
if (!cache.current) {
|
|
329
366
|
return true;
|
|
330
367
|
}
|
|
331
|
-
// If
|
|
332
|
-
if (!cache.current.
|
|
368
|
+
// If the tuple wasn't in the current cached run, it needs reporting
|
|
369
|
+
if (!cache.current.tuples.some((t) => tuplesMatch(t, tuple))) {
|
|
333
370
|
return true;
|
|
334
371
|
}
|
|
335
372
|
// If current run is stale (>4 hours old), needs reporting
|
|
336
373
|
const age = Date.now() - cache.current.timestamp;
|
|
337
|
-
if (age >
|
|
374
|
+
if (age > COVERAGE_DEBOUNCE_WINDOW_MS) {
|
|
338
375
|
return true;
|
|
339
376
|
}
|
|
340
377
|
// Otherwise, skip reporting (already reported recently)
|
|
@@ -24,6 +24,11 @@ export interface FinalizeOptions {
|
|
|
24
24
|
showUntestedList?: boolean;
|
|
25
25
|
/** Whether to show cloud reporting status messages (defaults to true) */
|
|
26
26
|
showCloudStatus?: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* Attribution label identifying the reporter (e.g. "Vitest", "Jest", "pytest").
|
|
29
|
+
* When omitted, rows are recorded with context absent per COVERAGE-CONTEXT-1.2.
|
|
30
|
+
*/
|
|
31
|
+
context?: string;
|
|
27
32
|
}
|
|
28
33
|
export interface FinalizeResult {
|
|
29
34
|
/** Number of unique requirements exercised */
|
package/dist/harness/finalize.js
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* Implements: HARNESS-FINALIZE-1, HARNESS-FINALIZE-2, HARNESS-FINALIZE-3, HARNESS-FINALIZE-4
|
|
11
11
|
*/
|
|
12
12
|
import { execSync } from 'child_process';
|
|
13
|
+
import { randomUUID } from 'crypto';
|
|
13
14
|
import { findRequirementsDir, findProjectRoot, getTestRunId, cleanupTestRunId, readTrackingEntries, deleteTrackingFile, readLookupCache, readCoverageCache, updateCoverageCache, needsReporting, } from './cache.js';
|
|
14
15
|
import { getProjectInfo } from '../utils/project-settings.js';
|
|
15
16
|
/**
|
|
@@ -40,6 +41,26 @@ function getCurrentBranch(cwd) {
|
|
|
40
41
|
return 'unknown';
|
|
41
42
|
}
|
|
42
43
|
}
|
|
44
|
+
/**
|
|
45
|
+
* Return the git-blame author email for a specific file:line, or undefined
|
|
46
|
+
* if unavailable (not a git repo, file untracked, git unavailable, etc.).
|
|
47
|
+
*
|
|
48
|
+
* COVERAGE-CONTEXT-2: "last reported by" attribution.
|
|
49
|
+
*/
|
|
50
|
+
function getLineAuthor(cwd, file, line) {
|
|
51
|
+
try {
|
|
52
|
+
const output = execSync(`git blame -L ${line},${line} --porcelain -- "${file}"`, {
|
|
53
|
+
cwd,
|
|
54
|
+
encoding: 'utf-8',
|
|
55
|
+
stdio: ['pipe', 'pipe', 'ignore'],
|
|
56
|
+
});
|
|
57
|
+
const match = output.match(/^author-mail\s+<([^>]+)>/m);
|
|
58
|
+
return match?.[1];
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return undefined;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
43
64
|
/**
|
|
44
65
|
* Print local coverage report to console
|
|
45
66
|
*
|
|
@@ -112,7 +133,7 @@ const CONVEX_URL = 'https://data.dotrequirements.io';
|
|
|
112
133
|
* HARNESS-FINALIZE-2: Cloud reporting with error tolerance
|
|
113
134
|
* HARNESS-FINALIZE-3: Coverage records include requirement, file, line, branch
|
|
114
135
|
*/
|
|
115
|
-
async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatus) {
|
|
136
|
+
async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatus, context) {
|
|
116
137
|
try {
|
|
117
138
|
// Get project info from .requirements/project-settings.json
|
|
118
139
|
const projectInfo = getProjectInfo(projectRoot);
|
|
@@ -125,24 +146,34 @@ async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatu
|
|
|
125
146
|
}
|
|
126
147
|
const { projectId, projectSecret } = projectInfo.credentials;
|
|
127
148
|
const requirementsDir = findRequirementsDir(projectRoot);
|
|
128
|
-
|
|
149
|
+
const branch = getCurrentBranch(projectRoot);
|
|
150
|
+
// Build the full set of tuples for this run, one per tracked requirement
|
|
129
151
|
const requirementKeys = Array.from(aggregated.keys());
|
|
130
|
-
|
|
152
|
+
const allTuples = requirementKeys.map((key) => ({
|
|
153
|
+
requirementKey: key,
|
|
154
|
+
context,
|
|
155
|
+
branch,
|
|
156
|
+
}));
|
|
157
|
+
// Local debounce (COVERAGE-DEBOUNCE-1): only report tuples not recently cached.
|
|
158
|
+
// Server-side debounce is a backstop (COVERAGE-DEBOUNCE-4) — this layer just
|
|
159
|
+
// prevents unnecessary HTTP calls.
|
|
131
160
|
const coverageCache = readCoverageCache(requirementsDir);
|
|
132
|
-
const
|
|
133
|
-
if (
|
|
161
|
+
const tuplesToReport = allTuples.filter((tuple) => needsReporting(tuple, coverageCache));
|
|
162
|
+
if (tuplesToReport.length === 0) {
|
|
134
163
|
if (showCloudStatus) {
|
|
135
164
|
console.log('\n✓ Coverage unchanged since last run (skipping cloud report)');
|
|
136
165
|
}
|
|
137
|
-
// Still update the cache
|
|
138
|
-
updateCoverageCache(requirementsDir, testRunId,
|
|
166
|
+
// Still update the cache so the window continues tracking
|
|
167
|
+
updateCoverageCache(requirementsDir, testRunId, allTuples);
|
|
139
168
|
return { sent: false, count: 0 };
|
|
140
169
|
}
|
|
141
|
-
|
|
170
|
+
// Deterministic runId for this finalize invocation (COVERAGE-CONTEXT-4)
|
|
171
|
+
const runId = randomUUID();
|
|
172
|
+
// Cache git-blame lookups per file:line to avoid redundant subprocess spawns
|
|
173
|
+
const userCache = new Map();
|
|
142
174
|
// Build coverage payload
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const entries = aggregated.get(key);
|
|
175
|
+
const coverage = tuplesToReport.map((tuple) => {
|
|
176
|
+
const entries = aggregated.get(tuple.requirementKey);
|
|
146
177
|
// Use the first access location
|
|
147
178
|
const firstEntry = entries[0];
|
|
148
179
|
const location = firstEntry.callerLocation;
|
|
@@ -159,10 +190,23 @@ async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatu
|
|
|
159
190
|
testFile = location;
|
|
160
191
|
}
|
|
161
192
|
}
|
|
193
|
+
// COVERAGE-CONTEXT-2: git blame on the requirement() call line
|
|
194
|
+
let user;
|
|
195
|
+
if (testFile && testLine !== undefined) {
|
|
196
|
+
const cacheKey = `${testFile}:${testLine}`;
|
|
197
|
+
if (userCache.has(cacheKey)) {
|
|
198
|
+
user = userCache.get(cacheKey);
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
user = getLineAuthor(projectRoot, testFile, testLine);
|
|
202
|
+
userCache.set(cacheKey, user);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
162
205
|
return {
|
|
163
|
-
requirementKey:
|
|
206
|
+
requirementKey: tuple.requirementKey,
|
|
164
207
|
testFile,
|
|
165
208
|
testLine,
|
|
209
|
+
user,
|
|
166
210
|
};
|
|
167
211
|
});
|
|
168
212
|
// Send to Convex
|
|
@@ -185,6 +229,8 @@ async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatu
|
|
|
185
229
|
slug: projectId,
|
|
186
230
|
},
|
|
187
231
|
branch,
|
|
232
|
+
context,
|
|
233
|
+
runId,
|
|
188
234
|
coverage,
|
|
189
235
|
},
|
|
190
236
|
format: 'json',
|
|
@@ -218,11 +264,11 @@ async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatu
|
|
|
218
264
|
}
|
|
219
265
|
}
|
|
220
266
|
if (showCloudStatus) {
|
|
221
|
-
console.log(`\n✓ Reported ${
|
|
267
|
+
console.log(`\n✓ Reported ${tuplesToReport.length} requirement(s) to cloud (branch: ${branch})`);
|
|
222
268
|
}
|
|
223
|
-
// Update coverage cache
|
|
224
|
-
updateCoverageCache(requirementsDir, testRunId,
|
|
225
|
-
return { sent: true, count:
|
|
269
|
+
// Update coverage cache with the full tuple set (includes those not reported this run)
|
|
270
|
+
updateCoverageCache(requirementsDir, testRunId, allTuples);
|
|
271
|
+
return { sent: true, count: tuplesToReport.length };
|
|
226
272
|
}
|
|
227
273
|
catch (error) {
|
|
228
274
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -244,7 +290,7 @@ async function reportToConvex(projectRoot, testRunId, aggregated, showCloudStatu
|
|
|
244
290
|
* HARNESS-FINALIZE-4: Finalize cleans up after itself
|
|
245
291
|
*/
|
|
246
292
|
export async function finalize(options = {}) {
|
|
247
|
-
const { cwd = process.cwd(), reportToCloud: shouldReportToCloud = true, cleanup: shouldCleanup = true, showSummary = true, showTestedList = false, showUntestedList = true, showCloudStatus = true, } = options;
|
|
293
|
+
const { cwd = process.cwd(), reportToCloud: shouldReportToCloud = true, cleanup: shouldCleanup = true, showSummary = true, showTestedList = false, showUntestedList = true, showCloudStatus = true, context, } = options;
|
|
248
294
|
// Priority 1: Use environment variable (cross-process persistence from globalSetup)
|
|
249
295
|
let projectRoot = process.env.DOTREQUIREMENTS_PROJECT_ROOT || null;
|
|
250
296
|
// Priority 2: Find from cwd
|
|
@@ -309,7 +355,7 @@ export async function finalize(options = {}) {
|
|
|
309
355
|
// HARNESS-FINALIZE-2, HARNESS-FINALIZE-3: Report to cloud
|
|
310
356
|
let cloudResult = { sent: false, count: 0 };
|
|
311
357
|
if (shouldReportToCloud) {
|
|
312
|
-
cloudResult = await reportToConvex(projectRoot, testRunId, aggregated, showCloudStatus);
|
|
358
|
+
cloudResult = await reportToConvex(projectRoot, testRunId, aggregated, showCloudStatus, context);
|
|
313
359
|
}
|
|
314
360
|
// HARNESS-FINALIZE-4: Clean up tracking data
|
|
315
361
|
if (shouldCleanup && !cloudResult.error) {
|
package/dist/mcp/index.js
CHANGED
|
@@ -33,17 +33,16 @@ const PROJECT_PATHS = getProjectPathsFromEnv();
|
|
|
33
33
|
// Parse --auth-from-env flag for CI/CD environments
|
|
34
34
|
// When set, credentials are read from DOTREQ_PROJECT_ID and DOTREQ_PROJECT_SECRET env vars
|
|
35
35
|
const USE_ENV_AUTH = process.argv.includes('--auth-from-env');
|
|
36
|
-
// Cache for loaded requirements (refreshed on each tool call for now)
|
|
37
|
-
let cachedRequirements = new Map();
|
|
38
36
|
let cachedDiscoveryResult = null;
|
|
37
|
+
// Requirements are loaded fresh on every call. A long-lived in-memory cache
|
|
38
|
+
// silently served stale data when .requirements/*.md files were created or
|
|
39
|
+
// edited mid-session, which is the primary authoring workflow the MCP server
|
|
40
|
+
// is meant to support. If this ever becomes a measured performance concern,
|
|
41
|
+
// invalidate via directory mtime rather than reintroducing a lifetime cache.
|
|
39
42
|
async function getRequirements(projectId) {
|
|
40
43
|
const project = await getProjectFromDiscovery(projectId);
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
const { flattened } = await loadAllRequirements(project.path);
|
|
44
|
-
cachedRequirements.set(cacheKey, flattened);
|
|
45
|
-
}
|
|
46
|
-
return cachedRequirements.get(cacheKey);
|
|
44
|
+
const { flattened } = await loadAllRequirements(project.path);
|
|
45
|
+
return flattened;
|
|
47
46
|
}
|
|
48
47
|
async function getProjectFromDiscovery(projectId) {
|
|
49
48
|
const { isConfiguredProject } = await import('../utils/project-discovery.js');
|
|
@@ -92,10 +91,9 @@ async function getProjectFromDiscovery(projectId) {
|
|
|
92
91
|
}
|
|
93
92
|
return resolveProject(cachedDiscoveryResult, projectId);
|
|
94
93
|
}
|
|
95
|
-
// Invalidate
|
|
96
|
-
// Exported for testing
|
|
94
|
+
// Invalidate the project-discovery cache. Requirements are no longer cached,
|
|
95
|
+
// so this only resets discovery state. Exported for testing.
|
|
97
96
|
export function invalidateCache() {
|
|
98
|
-
cachedRequirements.clear();
|
|
99
97
|
cachedDiscoveryResult = null;
|
|
100
98
|
}
|
|
101
99
|
// Tool definitions
|
package/dist/schema/browser.d.ts
CHANGED
|
@@ -9,4 +9,6 @@ export { DEFAULT_DELIMITER, buildRequirementsMarkdown, buildRequirementMarkdown,
|
|
|
9
9
|
export type { ConvexRequirement, } from './conversions.js';
|
|
10
10
|
export { convexToRequirements, requirementsToConvex, buildMetadata, extractRequirementKeys, groupByRoot, constructKey, parseKey, } from './conversions.js';
|
|
11
11
|
export { DELIMITER_PATTERN, parseCriterionLine, parseRootLine, parseRequirementBlock, extractRequirementBlocks, parseRequirementBlocksFromMarkdown, flattenRequirementTree, findRequirementById, getAllRequirements, } from './parser-core.js';
|
|
12
|
+
export { buildScenarioFromRequirements, requirementTreeToScenario, } from './scenario.js';
|
|
13
|
+
export type { Scenario, ScenarioStep, ScenarioAssertionSource, BuildScenarioOptions, } from './scenario.js';
|
|
12
14
|
//# sourceMappingURL=browser.d.ts.map
|
package/dist/schema/browser.js
CHANGED
|
@@ -21,4 +21,6 @@ export { convexToRequirements, requirementsToConvex, buildMetadata, extractRequi
|
|
|
21
21
|
export { DELIMITER_PATTERN, parseCriterionLine, parseRootLine, parseRequirementBlock, extractRequirementBlocks, parseRequirementBlocksFromMarkdown, flattenRequirementTree, findRequirementById, getAllRequirements, } from './parser-core.js';
|
|
22
22
|
// NOTE: parser.ts and resolver.ts are excluded because they use Node.js 'fs' module.
|
|
23
23
|
// Use parser-core.ts functions above for browser/Convex environments.
|
|
24
|
+
// Scenario building (pure TypeScript - browser-safe, used by Convex Node actions)
|
|
25
|
+
export { buildScenarioFromRequirements, requirementTreeToScenario, } from './scenario.js';
|
|
24
26
|
//# sourceMappingURL=browser.js.map
|
package/dist/schema/index.d.ts
CHANGED
|
@@ -11,4 +11,6 @@ export { buildRequirementsMarkdown, buildRequirementMarkdown, buildRequirementsF
|
|
|
11
11
|
export { parseRequirementPath, parsePathSegment, findChildrenByLabel, resolvePathSegment, resolveRequirementPath, resolveToNumericPath, getAllLabelPaths, checkPathAmbiguity, } from './resolver.js';
|
|
12
12
|
export type { ConvexRequirement, } from './conversions.js';
|
|
13
13
|
export { convexToRequirements, requirementsToConvex, buildMetadata, extractRequirementKeys, groupByRoot, constructKey, parseKey, } from './conversions.js';
|
|
14
|
+
export { buildScenarioFromRequirements, requirementTreeToScenario, } from './scenario.js';
|
|
15
|
+
export type { Scenario, ScenarioStep, ScenarioAssertionSource, BuildScenarioOptions, } from './scenario.js';
|
|
14
16
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/schema/index.js
CHANGED
|
@@ -21,4 +21,6 @@ export { buildRequirementsMarkdown, buildRequirementMarkdown, buildRequirementsF
|
|
|
21
21
|
// Path resolution
|
|
22
22
|
export { parseRequirementPath, parsePathSegment, findChildrenByLabel, resolvePathSegment, resolveRequirementPath, resolveToNumericPath, getAllLabelPaths, checkPathAmbiguity, } from './resolver.js';
|
|
23
23
|
export { convexToRequirements, requirementsToConvex, buildMetadata, extractRequirementKeys, groupByRoot, constructKey, parseKey, } from './conversions.js';
|
|
24
|
+
// Scenario building (for browser-automation / runScenario)
|
|
25
|
+
export { buildScenarioFromRequirements, requirementTreeToScenario, } from './scenario.js';
|
|
24
26
|
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scenario builder — turns a requirement tree into a browser-automation Scenario.
|
|
3
|
+
*
|
|
4
|
+
* Used by both the CLI (via `dotreq browsertest`) and the Convex Node action
|
|
5
|
+
* (`testCoverage/browserRun:run`) so the two paths produce identical scenarios
|
|
6
|
+
* for the same input.
|
|
7
|
+
*
|
|
8
|
+
* Naive passthrough: every requirement node with non-empty content becomes
|
|
9
|
+
* exactly one assertion keyed by its numeric-path id (e.g. "LOGIN-1" for the
|
|
10
|
+
* root, "LOGIN-1.0" for the first criterion). No arrange/act steps are
|
|
11
|
+
* emitted; Stagehand's hybrid agent is expected to infer setup from the
|
|
12
|
+
* assertion list. See BROWSER-RUN-3 in
|
|
13
|
+
* .requirements/testing-tab-browser-run.requirements.md.
|
|
14
|
+
*
|
|
15
|
+
* Types here are defined locally to avoid adding `@popoverai/browser-automation`
|
|
16
|
+
* as a runtime dependency of this package. They are structurally compatible
|
|
17
|
+
* with browser-automation's exported `Scenario` / `Step` types and can be
|
|
18
|
+
* passed directly to `runScenario`.
|
|
19
|
+
*/
|
|
20
|
+
import type { RequirementNode } from "./schemas.js";
|
|
21
|
+
/**
|
|
22
|
+
* One step in a scenario. Structurally compatible with
|
|
23
|
+
* `@popoverai/browser-automation`'s `Step` type.
|
|
24
|
+
*/
|
|
25
|
+
export interface ScenarioStep {
|
|
26
|
+
step: "arrange" | "act" | "assert";
|
|
27
|
+
description: string;
|
|
28
|
+
url?: string;
|
|
29
|
+
key?: string;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* One entry in the `Scenario.variables` map. Structurally compatible with
|
|
33
|
+
* `@popoverai/browser-automation`'s `Variable` type — the `value` is what
|
|
34
|
+
* gets substituted into the scenario, and the optional `description` gives
|
|
35
|
+
* the agent extra context about what the substitution represents.
|
|
36
|
+
*/
|
|
37
|
+
export interface ScenarioVariable {
|
|
38
|
+
value: string;
|
|
39
|
+
description?: string;
|
|
40
|
+
}
|
|
41
|
+
export type ScenarioVariables = Record<string, ScenarioVariable>;
|
|
42
|
+
/**
|
|
43
|
+
* A scenario passed to `runScenario`. Structurally compatible with
|
|
44
|
+
* `@popoverai/browser-automation`'s `Scenario` type.
|
|
45
|
+
*/
|
|
46
|
+
export interface Scenario {
|
|
47
|
+
baseUrl: string;
|
|
48
|
+
steps: ScenarioStep[];
|
|
49
|
+
variables?: ScenarioVariables;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* One requirement contributing an assertion to a scenario. The minimum shape
|
|
53
|
+
* the builder needs so callers sourcing requirements from different backends
|
|
54
|
+
* (Markdown files via `RequirementNode`, Convex rows, etc.) can all use it.
|
|
55
|
+
*/
|
|
56
|
+
export interface ScenarioAssertionSource {
|
|
57
|
+
/** Full requirement key including path, e.g. "LOGIN-1" or "LOGIN-1.0.1". */
|
|
58
|
+
id: string;
|
|
59
|
+
/** Requirement text; becomes the assertion's `description`. */
|
|
60
|
+
content: string;
|
|
61
|
+
}
|
|
62
|
+
export interface BuildScenarioOptions {
|
|
63
|
+
/**
|
|
64
|
+
* The requirements contributing assertions, in the order they should be
|
|
65
|
+
* evaluated. Every entry with non-empty `content` becomes one `assert` step.
|
|
66
|
+
*/
|
|
67
|
+
assertions: ScenarioAssertionSource[];
|
|
68
|
+
/** Target URL the scenario runs against; becomes `Scenario.baseUrl`. */
|
|
69
|
+
baseUrl: string;
|
|
70
|
+
/**
|
|
71
|
+
* Optional Stagehand variable substitutions. Used to inject secrets
|
|
72
|
+
* (test-user credentials, bypass tokens) into assertion descriptions
|
|
73
|
+
* without exposing them to the language model in plaintext.
|
|
74
|
+
*/
|
|
75
|
+
variables?: Record<string, string>;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Build a `Scenario` from a list of requirement assertion sources.
|
|
79
|
+
*
|
|
80
|
+
* Throws if the resulting scenario would have zero assertions
|
|
81
|
+
* (browser-automation rejects scenarios with no asserts).
|
|
82
|
+
*/
|
|
83
|
+
export declare function buildScenarioFromRequirements(opts: BuildScenarioOptions): Scenario;
|
|
84
|
+
/**
|
|
85
|
+
* Convenience wrapper for callers that have a `RequirementNode` tree (root
|
|
86
|
+
* plus nested children). Flattens the tree to the order produced by
|
|
87
|
+
* `flattenRequirementTree` (pre-order: root first, then each subtree), then
|
|
88
|
+
* delegates to `buildScenarioFromRequirements`.
|
|
89
|
+
*/
|
|
90
|
+
export declare function requirementTreeToScenario(root: RequirementNode, opts: Omit<BuildScenarioOptions, "assertions">): Scenario;
|
|
91
|
+
//# sourceMappingURL=scenario.d.ts.map
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scenario builder — turns a requirement tree into a browser-automation Scenario.
|
|
3
|
+
*
|
|
4
|
+
* Used by both the CLI (via `dotreq browsertest`) and the Convex Node action
|
|
5
|
+
* (`testCoverage/browserRun:run`) so the two paths produce identical scenarios
|
|
6
|
+
* for the same input.
|
|
7
|
+
*
|
|
8
|
+
* Naive passthrough: every requirement node with non-empty content becomes
|
|
9
|
+
* exactly one assertion keyed by its numeric-path id (e.g. "LOGIN-1" for the
|
|
10
|
+
* root, "LOGIN-1.0" for the first criterion). No arrange/act steps are
|
|
11
|
+
* emitted; Stagehand's hybrid agent is expected to infer setup from the
|
|
12
|
+
* assertion list. See BROWSER-RUN-3 in
|
|
13
|
+
* .requirements/testing-tab-browser-run.requirements.md.
|
|
14
|
+
*
|
|
15
|
+
* Types here are defined locally to avoid adding `@popoverai/browser-automation`
|
|
16
|
+
* as a runtime dependency of this package. They are structurally compatible
|
|
17
|
+
* with browser-automation's exported `Scenario` / `Step` types and can be
|
|
18
|
+
* passed directly to `runScenario`.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Pre-order flatten: root first, then each subtree's nodes in order.
|
|
22
|
+
* Inlined here (rather than imported from `parser.ts`) so this module stays
|
|
23
|
+
* free of Node-only dependencies and can be re-exported from the browser
|
|
24
|
+
* entry point.
|
|
25
|
+
*/
|
|
26
|
+
function flattenTree(node) {
|
|
27
|
+
const result = [node];
|
|
28
|
+
for (const child of node.children) {
|
|
29
|
+
result.push(...flattenTree(child));
|
|
30
|
+
}
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Build a `Scenario` from a list of requirement assertion sources.
|
|
35
|
+
*
|
|
36
|
+
* Throws if the resulting scenario would have zero assertions
|
|
37
|
+
* (browser-automation rejects scenarios with no asserts).
|
|
38
|
+
*/
|
|
39
|
+
export function buildScenarioFromRequirements(opts) {
|
|
40
|
+
const { assertions, baseUrl, variables } = opts;
|
|
41
|
+
if (!baseUrl || baseUrl.trim().length === 0) {
|
|
42
|
+
throw new Error("buildScenarioFromRequirements: baseUrl is required");
|
|
43
|
+
}
|
|
44
|
+
const steps = [];
|
|
45
|
+
for (const source of assertions) {
|
|
46
|
+
const description = source.content.trim();
|
|
47
|
+
if (description.length === 0)
|
|
48
|
+
continue;
|
|
49
|
+
steps.push({
|
|
50
|
+
step: "assert",
|
|
51
|
+
description,
|
|
52
|
+
key: source.id,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
if (steps.length === 0) {
|
|
56
|
+
throw new Error("buildScenarioFromRequirements: requirement tree has no nodes with content; cannot build a scenario");
|
|
57
|
+
}
|
|
58
|
+
const scenario = { baseUrl, steps };
|
|
59
|
+
if (variables && Object.keys(variables).length > 0) {
|
|
60
|
+
// Callers pass a flat Record<string, string> for ergonomics; wrap each
|
|
61
|
+
// entry as `{ value }` so the scenario matches runScenario's expected
|
|
62
|
+
// Variables shape. Callers with richer values can set variables on the
|
|
63
|
+
// returned scenario directly before invocation.
|
|
64
|
+
scenario.variables = Object.fromEntries(Object.entries(variables).map(([key, value]) => [key, { value }]));
|
|
65
|
+
}
|
|
66
|
+
return scenario;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Convenience wrapper for callers that have a `RequirementNode` tree (root
|
|
70
|
+
* plus nested children). Flattens the tree to the order produced by
|
|
71
|
+
* `flattenRequirementTree` (pre-order: root first, then each subtree), then
|
|
72
|
+
* delegates to `buildScenarioFromRequirements`.
|
|
73
|
+
*/
|
|
74
|
+
export function requirementTreeToScenario(root, opts) {
|
|
75
|
+
const flat = flattenTree(root);
|
|
76
|
+
const assertions = flat.map((node) => ({
|
|
77
|
+
id: node.id,
|
|
78
|
+
content: node.content,
|
|
79
|
+
}));
|
|
80
|
+
return buildScenarioFromRequirements({ ...opts, assertions });
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=scenario.js.map
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Browser test configuration
|
|
3
|
-
*/
|
|
1
|
+
/** Browser test configuration. */
|
|
4
2
|
export interface BrowserTestSettings {
|
|
3
|
+
modelName?: string;
|
|
4
|
+
modelApiKey?: string;
|
|
5
|
+
/** @deprecated Backward-compat fallback; use modelName + modelApiKey instead. */
|
|
5
6
|
geminiApiKey?: string;
|
|
6
7
|
vercelBypassSecret?: string;
|
|
7
8
|
browserbaseApiKey?: string;
|
|
@@ -80,6 +80,12 @@ export function readProjectSettings(projectRoot) {
|
|
|
80
80
|
if (typeof record.browserTest === 'object' && record.browserTest !== null) {
|
|
81
81
|
const bt = record.browserTest;
|
|
82
82
|
settings.browserTest = {};
|
|
83
|
+
if (typeof bt.modelName === 'string') {
|
|
84
|
+
settings.browserTest.modelName = bt.modelName;
|
|
85
|
+
}
|
|
86
|
+
if (typeof bt.modelApiKey === 'string') {
|
|
87
|
+
settings.browserTest.modelApiKey = bt.modelApiKey;
|
|
88
|
+
}
|
|
83
89
|
if (typeof bt.geminiApiKey === 'string') {
|
|
84
90
|
settings.browserTest.geminiApiKey = bt.geminiApiKey;
|
|
85
91
|
}
|