@activate-core/test-forge 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,39 @@
1
+ /**
2
+ * What the AI cost: calls, wall time and an estimate of tokens, counted in
3
+ * one place every AI call goes through. Tokens are estimated as characters
4
+ * sent and received divided by four — close enough to compare journeys and
5
+ * runs, not a bill.
6
+ */
7
+ import { execFileSync } from 'node:child_process';
8
+ const total = { calls: 0, ms: 0, inChars: 0, outChars: 0 };
9
+ export function noteAiCall(inChars, outChars, ms) {
10
+ total.calls += 1;
11
+ total.ms += ms;
12
+ total.inChars += inChars;
13
+ total.outChars += outChars;
14
+ }
15
+ /** A snapshot of the running totals. */
16
+ export function aiUsage() {
17
+ return { ...total };
18
+ }
19
+ /** What was spent between two snapshots. */
20
+ export function usageSince(start, now = aiUsage()) {
21
+ return { calls: now.calls - start.calls, ms: now.ms - start.ms, inChars: now.inChars - start.inChars, outChars: now.outChars - start.outChars };
22
+ }
23
+ export const approxTokens = (u) => Math.round((u.inChars + u.outChars) / 4);
24
+ export function describeUsage(u) {
25
+ const min = u.ms / 60_000;
26
+ return `${u.calls} AI call(s), ${min >= 1 ? `${min.toFixed(1)} min` : `${Math.round(u.ms / 1000)} s`}, ~${approxTokens(u).toLocaleString('en-GB')} tokens`;
27
+ }
28
+ /** execFileSync for an AI CLI, counted. Same signature as the call it replaces. */
29
+ export function runAiSync(program, args, options) {
30
+ const started = Date.now();
31
+ let out = '';
32
+ try {
33
+ out = execFileSync(program, args, options);
34
+ return out;
35
+ }
36
+ finally {
37
+ noteAiCall(String(options.input ?? '').length, out.length, Date.now() - started);
38
+ }
39
+ }
@@ -16,12 +16,14 @@
16
16
  * what survives is reported as a defect; everything else is said for what
17
17
  * it is.
18
18
  */
19
- import { existsSync, readFileSync } from 'node:fs';
20
- import { relative } from 'node:path';
19
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
20
+ import { dirname, relative } from 'node:path';
21
21
  import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
22
- import { runWithStdin } from './ai-generator.js';
22
+ import { runWithStdin, DEFECT_MARK, FINDING_MARK } from './ai-generator.js';
23
23
  import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
24
- const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
24
+ const IMPACTS = ['blocker', 'major', 'minor'];
25
+ const asImpact = (v) => (IMPACTS.includes(v) ? v : undefined);
26
+ const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'environment', 'unproven'];
25
27
  /**
26
28
  * Group failing tests by the claim marked above them. A parametrised test
27
29
  * shares one declaration — and so one claim — across all its rows, and one
@@ -57,6 +59,10 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
57
59
  ``,
58
60
  `THE CLAIM (written by the tool that generated and repaired the test):`,
59
61
  ` ${group.claim}`,
62
+ ...(group.claim.includes(NOT_CLAIMED)
63
+ ? [` The generator did not claim a defect here: the claim is the test's own oracle.`,
64
+ ` Judge it the same way, and say "test-fault" or "environment" when that is what it is.`]
65
+ : []),
60
66
  ``,
61
67
  `FAILING TESTS that carry it (${group.tests.length}), numbered:`,
62
68
  ...group.tests.slice(0, MAX_LISTED).map((t, i) => ` ${i + 1}. ${relative(process.cwd(), t.file)}:${t.line} ${t.title}`),
@@ -89,8 +95,22 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
89
95
  `- "pending-decision": the behaviour is covered by an open decision or a known`,
90
96
  ` issue in the list below — whatever the tests say.`,
91
97
  `- "test-fault": the test, its stub, its precondition or its claim is wrong.`,
98
+ `- "environment": the failure comes from the environment the test ran in — a`,
99
+ ` missing variable or account, an unreachable or refusing outside service, seed`,
100
+ ` data that is not there, a rate limit, or data left behind by earlier runs —`,
101
+ ` not from the application or the test's logic.`,
92
102
  `- "unproven": none of the above can be shown.`,
93
103
  ``,
104
+ `For a "defect" or "finding", also grade its impact on the customer:`,
105
+ `- "blocker": no way through, or unsafe — security or data exposure, money taken`,
106
+ ` wrongly, or a core step (log in, sign up, book, pay) that cannot be completed`,
107
+ ` at all. A failure the test forced (a stubbed error, a cut connection, absurd`,
108
+ ` input) is never a blocker on its own.`,
109
+ `- "major": the task can be done but the customer is misled or left without a`,
110
+ ` reason — a missing or wrong message, wrong data shown, an error on input.`,
111
+ `- "minor": presentation, or another way works — keyboard, focus, Escape,`,
112
+ ` wording and labels, icons, layout.`,
113
+ ``,
94
114
  ...(decisions.text
95
115
  ? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet):`, decisions.text, ``]
96
116
  : decisions.link
@@ -106,8 +126,9 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
106
126
  `that fits most rows and list the others as exceptions, by number.`,
107
127
  ``,
108
128
  `Reply with ONLY this JSON object, nothing before or after it:`,
109
- `{"verdict": "defect|finding|pending-decision|test-fault|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
110
- ` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "..."}]}`,
129
+ `{"verdict": "defect|finding|pending-decision|test-fault|environment|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
130
+ ` "impact": "blocker|major|minor (defect or finding only, else empty)",`,
131
+ ` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "...", "impact": "..."}]}`,
111
132
  ].join('\n');
112
133
  }
113
134
  /** How many rows of one claim the reviewer is shown and may single out. */
@@ -126,14 +147,20 @@ export function parseReview(raw) {
126
147
  verdict: asVerdict(e?.verdict),
127
148
  reason: text(e?.reason) ?? '',
128
149
  evidence: text(e?.evidence),
150
+ ...(asImpact(e?.impact) ? { impact: asImpact(e?.impact) } : {}),
129
151
  }))
130
152
  .filter((e) => e.tests.length);
131
- return { verdict: asVerdict(o.verdict), reason: text(o.reason) ?? '', evidence: text(o.evidence), exceptions };
153
+ const impact = asImpact(o.impact);
154
+ return { verdict: asVerdict(o.verdict), reason: text(o.reason) ?? '', evidence: text(o.evidence), ...(impact ? { impact } : {}), exceptions };
132
155
  }
133
156
  catch {
134
157
  return { verdict: 'unproven', reason: 'the review could not be read', exceptions: [] };
135
158
  }
136
159
  }
160
+ /** A defect or finding always carries an impact; it defaults to major when the reviewer gave none. Other verdicts carry none. */
161
+ function withImpact(r, impact) {
162
+ return r.verdict === 'defect' || r.verdict === 'finding' ? { ...r, impact: impact ?? 'major' } : r;
163
+ }
137
164
  /** One review per verdict: the group's own, and one for each set of exceptions. */
138
165
  export function splitReview(group, parsed) {
139
166
  const taken = new Set();
@@ -142,14 +169,102 @@ export function splitReview(group, parsed) {
142
169
  const tests = e.tests.filter((n) => n <= group.tests.length && !taken.has(n));
143
170
  tests.forEach((n) => taken.add(n));
144
171
  if (tests.length) {
145
- out.push({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence });
172
+ out.push(withImpact({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence }, e.impact));
146
173
  }
147
174
  }
148
175
  const rest = group.tests.filter((_, i) => !taken.has(i + 1));
149
176
  if (rest.length)
150
- out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
177
+ out.unshift(withImpact({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence }, parsed.impact));
151
178
  return out;
152
179
  }
180
+ /** Marks a claim built from a test's oracle because the generator claimed nothing. */
181
+ export const NOT_CLAIMED = '(not claimed as a defect by the generator)';
182
+ /**
183
+ * The rule a failing test checks, read from its first "// oracle:" comment,
184
+ * so a failure nobody claimed can still be reviewed against its tier. Falls
185
+ * back to the test's title.
186
+ */
187
+ export function oracleClaim(t, content) {
188
+ const lines = content.split('\n');
189
+ for (let i = Math.max(0, t.line - 1), seen = 0; i < lines.length && seen < 200; i++, seen++) {
190
+ if (seen > 0 && /^\s*(?:test|it)(?:\.\w+)?\s*\(/.test(lines[i]))
191
+ break;
192
+ const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
193
+ if (m)
194
+ return `${m[1].trim()} ${NOT_CLAIMED}`;
195
+ }
196
+ return `${t.title} ${NOT_CLAIMED}`;
197
+ }
198
+ /** A [spec] line that carries the client's words, quoted or pointing at the quote above it ("source as above"). */
199
+ export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:?\s*(?:as|same)\b)|same source\b)/i;
200
+ /** Playwright's report of a locator that matched more than one element. */
201
+ const AMBIGUOUS = /strict mode violation|resolved to \d+ elements/i;
202
+ /** An application file:line, the only evidence a defect may rest on. */
203
+ const CODE_REF = /[\w@./-]+\.(?:tsx?|jsx?|mjs|cjs|php|py|rb|go|java|kt|cs|vue|svelte)(?::\d+)/i;
204
+ const alnum = (v) => v.toLowerCase().replace(/\\./g, '').replace(/[^a-z0-9]+/g, '');
205
+ /** The tier a claim is made against, read from its tag. */
206
+ export function claimTier(claim) {
207
+ return /\[(spec|baseline|observed)\]/i.exec(claim)?.[1]?.toLowerCase();
208
+ }
209
+ /** The journey line a [spec] claim quotes, if the doc has one. */
210
+ export function specLineFor(claim, journeyMarkdown) {
211
+ // Claims often shorten the rule with "..." anywhere in it: match on its longest unshortened piece.
212
+ const rule = claim.slice(claim.search(/\[spec\]/i) + '[spec]'.length).split(/\s[—-]\s/)[0];
213
+ const piece = rule.split(/\.\.\.|…/).map(alnum).sort((a, b) => b.length - a.length)[0] ?? '';
214
+ const key = piece.slice(0, 40);
215
+ if (key.length < 12)
216
+ return undefined;
217
+ return journeyMarkdown.split('\n').find((l) => /\[spec\]/i.test(l) && alnum(l).includes(key));
218
+ }
219
+ /**
220
+ * The reviewer's word is not the last one. A "defect" stands only when the
221
+ * claim is a [spec] rule the client wrote, the journey line carries its
222
+ * (source: "…") quote, and the reviewer names the application file:line.
223
+ * Without the code reference it is unproven; against a [baseline]/[observed]
224
+ * rule, or a [spec] line with no quoted source, it is at most a finding.
225
+ */
226
+ export function gradeReview(r, journeyMarkdown) {
227
+ // A locator that matched several elements never reached one behaviour: that is the test's fault.
228
+ if ((r.verdict === 'defect' || r.verdict === 'finding') && r.tests.some((t) => AMBIGUOUS.test(t.detail ?? ''))) {
229
+ return { ...r, verdict: 'test-fault', impact: undefined, reason: `${r.reason} [graded: the test's locator matched several elements (strict mode violation)]`.trim() };
230
+ }
231
+ if (r.verdict !== 'defect')
232
+ return r;
233
+ if (!r.evidence || !CODE_REF.test(r.evidence)) {
234
+ return { ...r, verdict: 'unproven', impact: undefined, reason: `${r.reason} [graded: no application file:line was given as evidence]`.trim() };
235
+ }
236
+ const tier = claimTier(r.claim);
237
+ if (tier !== 'spec') {
238
+ return { ...r, verdict: 'finding', reason: `${r.reason} [graded: the rule is ${tier ? `[${tier}]` : 'untagged'}, not a client requirement]`.trim() };
239
+ }
240
+ const line = specLineFor(r.claim, journeyMarkdown);
241
+ if (!line || !HAS_SOURCE.test(line)) {
242
+ return { ...r, verdict: 'finding', reason: `${r.reason} [graded: the journey doc has no quoted source for this [spec] rule]`.trim() };
243
+ }
244
+ return r;
245
+ }
246
+ /**
247
+ * A repair may claim a defect only against a [spec] rule the journey quotes
248
+ * with its source. Any other claim — [baseline], [observed], untagged, or a
249
+ * [spec] rule the doc does not quote — is rewritten to a finding mark: the
250
+ * test stays failing and protected from repair, but is never reported as a
251
+ * defect.
252
+ */
253
+ export function normaliseClaims(content, journeyMarkdown) {
254
+ let downgraded = 0;
255
+ const lines = content.split('\n').map((line) => {
256
+ const at = line.indexOf(DEFECT_MARK);
257
+ if (at === -1)
258
+ return line;
259
+ const claim = line.slice(at + DEFECT_MARK.length).trim();
260
+ const spec = claimTier(claim) === 'spec' ? specLineFor(claim, journeyMarkdown) : undefined;
261
+ if (spec && HAS_SOURCE.test(spec))
262
+ return line;
263
+ downgraded += 1;
264
+ return line.slice(0, at) + FINDING_MARK + line.slice(at + DEFECT_MARK.length);
265
+ });
266
+ return { content: lines.join('\n'), downgraded };
267
+ }
153
268
  /**
154
269
  * Review every claim group, a few at a time. Never throws.
155
270
  *
@@ -160,7 +275,26 @@ export function splitReview(group, parsed) {
160
275
  * reported 0 defects. So the group goes back on the queue, every worker
161
276
  * shares one wait, and the review is asked again once the AI answers.
162
277
  */
163
- export async function reviewDefects(journey, groups, passes, io) {
278
+ export async function reviewDefects(journey, groups, passes, io,
279
+ /** Where answered groups are kept, so an interrupted review asks only the rest again. */
280
+ checkpoint) {
281
+ const groupKey = (g) => `${g.claim}\u0000${g.tests.map((t) => `${t.file}:${t.line}:${t.title}`).join('|')}`;
282
+ let answered = {};
283
+ if (checkpoint) {
284
+ try {
285
+ answered = JSON.parse(readFileSync(checkpoint, 'utf-8'));
286
+ }
287
+ catch {
288
+ answered = {};
289
+ }
290
+ }
291
+ const keep = (g, rs) => {
292
+ if (!checkpoint)
293
+ return;
294
+ answered[groupKey(g)] = rs;
295
+ mkdirSync(dirname(checkpoint), { recursive: true });
296
+ writeFileSync(checkpoint, JSON.stringify(answered), 'utf-8');
297
+ };
164
298
  const config = loadConfig();
165
299
  const decisions = decisionsSource(config.decisions);
166
300
  // A linked list (an internal Notion page) needs the reference tools to be read.
@@ -174,6 +308,11 @@ export async function reviewDefects(journey, groups, passes, io) {
174
308
  let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
175
309
  const worker = async () => {
176
310
  for (let g = queue.shift(); g; g = queue.shift()) {
311
+ const earlier = answered[groupKey(g)];
312
+ if (earlier) {
313
+ out.push(...earlier);
314
+ continue;
315
+ }
177
316
  const files = new Set(g.tests.map((t) => t.file));
178
317
  const siblings = passes.filter((p) => files.has(p.file));
179
318
  let raw = '';
@@ -193,7 +332,11 @@ export async function reviewDefects(journey, groups, passes, io) {
193
332
  }
194
333
  // an unanswered review proves nothing
195
334
  }
196
- out.push(...splitReview(g, parseReview(raw)));
335
+ const rs = splitReview(g, parseReview(raw)).map((r) => gradeReview(r, journey.rawMarkdown));
336
+ // An unanswered group (the AI never replied) is not kept: it is asked again next time.
337
+ if (raw)
338
+ keep(g, rs);
339
+ out.push(...rs);
197
340
  }
198
341
  };
199
342
  await Promise.all(Array.from({ length: Math.max(1, Math.min(config.aiConcurrency ?? 4, groups.length)) }, worker));
@@ -11,11 +11,12 @@
11
11
  * auth (read-only tools), so it connects and seeds against the actual
12
12
  * schema — not a guess.
13
13
  */
14
- import { execFileSync, spawn } from 'node:child_process';
14
+ import { spawn } from 'node:child_process';
15
15
  import { existsSync, readFileSync } from 'node:fs';
16
16
  import { createHash } from 'node:crypto';
17
17
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
18
18
  import { cleanOutput } from './ai-generator.js';
19
+ import { runAiSync } from './ai-usage.js';
19
20
  const MAP_PATH = 'testforge.md';
20
21
  /**
21
22
  * The fixed contract every testkit exposes and every generated test
@@ -39,7 +40,16 @@ export const HARNESS_CONTRACT = [
39
40
  ` - REQUIRED_PACKAGES: string[] packages the testkit needs (see H6).`,
40
41
  ` - resetRateLimits(), bypassVerification(), suppressPopups(), and one`,
41
42
  ` resolver per seeded entity — see H7 and H3.`,
43
+ ` - unique(label?): string a value no other run produces (H8).`,
44
+ ` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
45
+ ` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
46
+ ` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
42
47
  `Plus, for Playwright suites:`,
48
+ ` - openStep(page, step, input?): Promise<true | string> open a later step`,
49
+ ` of a multi-step flow with what the earlier steps would have left (H9).`,
50
+ ` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
51
+ ` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
52
+ ` rewritten here.`,
43
53
  ` - apiContext(request): an authenticated API request context, created from`,
44
54
  ` the imported request MODULE (\`const { request: apiRequest } =`,
45
55
  ` require('@playwright/test'); apiRequest.newContext({ baseURL,`,
@@ -144,6 +154,40 @@ export const HARNESS_CONTRACT = [
144
154
  ` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
145
155
  ` answered with the real success shape too. Scope each to exactly that`,
146
156
  ` request; everything after it runs against the real backend.`,
157
+ `H8. WHAT A RUN CREATES. Runs share accounts and environments, so data one`,
158
+ ` run leaves behind must never change another run's result.`,
159
+ ` - Load the run-data runtime once:`,
160
+ ` const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
161
+ ` unique(label) returns runData.unique(label). created(kind, id, cleanup,`,
162
+ ` role) calls runData.record({ kind, id, role, cleanup }), where cleanup is`,
163
+ ` { method, url } of the product's own request that removes or cancels that`,
164
+ ` record (absolute url; read the routes), or undefined when there is none.`,
165
+ ` - Every testkit helper that creates a record through the API calls`,
166
+ ` created() with it.`,
167
+ ` - The "run" role. TEST_RUN_ACCOUNT_MODE picks the account:`,
168
+ ` fixture (default) — the "run" role IS the customer role;`,
169
+ ` pool — TEST_RUN_ACCOUNTS is a comma list of identifiers sharing`,
170
+ ` TEST_RUN_ACCOUNTS_PASSWORD; pick one by a stable hash of runData.runId();`,
171
+ ` signup — create a fresh account through the product's own sign-up, only`,
172
+ ` when this environment can complete it (no code it cannot read and`,
173
+ ` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
174
+ ` Mint and cache its token like any role.`,
175
+ `H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
176
+ ` booking or checkout wizard, a sign-up in stages) depends on what the`,
177
+ ` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
178
+ ` parameters, browser storage or the server. Opened by its URL alone it`,
179
+ ` often sends the visitor back to an earlier step without a word, and a`,
180
+ ` test then "fails" on the product's correct guard.`,
181
+ ` - Read the flow's step components, its router guard and its store, and`,
182
+ ` give openStep(page, step, input?) the way the product reaches that step:`,
183
+ ` the parameters a real entry link carries, or the stored state the`,
184
+ ` earlier steps write (pre-set with an init script before the app runs).`,
185
+ ` \`input\` overrides the defaults (which choice, which address).`,
186
+ ` - It waits until the step is really shown and resolves true; when the`,
187
+ ` product still sends the visitor elsewhere it resolves a string naming`,
188
+ ` the step it landed on, so the test can skip with that reason.`,
189
+ ` - Offer only the steps the flow has; when the project has no such flow,`,
190
+ ` export openStep as a function resolving 'this product has no multi-step flow'.`,
147
191
  ].join('\n');
148
192
  /**
149
193
  * Stamp written into every generated testkit: the contract it was built
@@ -239,7 +283,7 @@ function buildHarnessPrompt(harnessPath, framework) {
239
283
  export function generateHarness(harnessPath, framework) {
240
284
  const prompt = buildHarnessPrompt(harnessPath, framework);
241
285
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
242
- const output = execFileSync(program, args, {
286
+ const output = runAiSync(program, args, {
243
287
  input: prompt,
244
288
  encoding: 'utf-8',
245
289
  maxBuffer: 10 * 1024 * 1024,
@@ -295,6 +339,8 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
295
339
  `const { defineConfig } = require('@playwright/test');`,
296
340
  `const fs = require('fs');`,
297
341
  `const path = require('path');`,
342
+ `// One run id for this run, set here in the main process so every worker inherits it.`,
343
+ `try { require('@activate-core/test-forge/reporters/run-data.cjs').runId(); } catch (_) {}`,
298
344
  ``,
299
345
  `// Load the env file named in testforge.config.json, so a plain`,
300
346
  `// \`npx playwright test\` (or an editor's test runner) gets the same env as`,
@@ -352,6 +398,8 @@ export function jestConfig(testDirRelToPackage, setupRelToPackage, category = 'a
352
398
  return [
353
399
  `// @generated by testforge`,
354
400
  `// Install once: npm i -D jest`,
401
+ `// One run id for this run, set before the workers start so each inherits it.`,
402
+ `try { require('@activate-core/test-forge/reporters/run-data.cjs').runId(); } catch (_) {}`,
355
403
  `module.exports = {`,
356
404
  ` testEnvironment: 'node',`,
357
405
  ` setupFiles: ['<rootDir>/${setupRelToPackage}'],`,
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Where a journey's starting data comes from.
3
+ *
4
+ * A repository that owns its data store gets a generated seed that writes
5
+ * it. One that does not (a web client talking to someone else's API) cannot
6
+ * run such a seed, so it gets no seed code at all: the testkit sets up what
7
+ * the product's API can create, and REQUIRED-DATA.md lists, per journey, what
8
+ * a person has to provide — taken word for word from the journey's own
9
+ * "Test setup" section, never invented.
10
+ */
11
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ /** Packages that only a repository talking to its own data store depends on. */
14
+ const STORE_PACKAGES = /^(?:mysql2?|pg|postgres|mongoose|mongodb|prisma|@prisma\/client|typeorm|sequelize|knex|drizzle-orm|better-sqlite3|sqlite3|ioredis|redis|@mikro-orm\/core|objection)$/;
15
+ const STORE_LIBS = /\b(?:sqlalchemy|django|psycopg2?|pymongo|peewee|activerecord|mongoid|sequel|doctrine\/orm|illuminate\/database|laravel\/framework|gorm|sqlx)\b/i;
16
+ /**
17
+ * "db" when this repository talks to its own data store, so a seed it
18
+ * generates can run; else "api". The config's `dataAccess` decides when set.
19
+ * Otherwise the repository itself is read — its dependencies and whether it
20
+ * has migrations — never the env file, whose variable names can be left over.
21
+ */
22
+ export function dataAccess(config, root = '.') {
23
+ if (config.dataAccess === 'db' || config.dataAccess === 'api')
24
+ return config.dataAccess;
25
+ const read = (f) => { try {
26
+ return readFileSync(join(root, f), 'utf-8');
27
+ }
28
+ catch {
29
+ return '';
30
+ } };
31
+ try {
32
+ const pkg = JSON.parse(read('package.json') || '{}');
33
+ const deps = Object.keys({ ...pkg.dependencies, ...pkg.devDependencies });
34
+ if (deps.some((d) => STORE_PACKAGES.test(d)))
35
+ return 'db';
36
+ }
37
+ catch { /* not a node project, or unreadable package.json */ }
38
+ if (['composer.json', 'requirements.txt', 'pyproject.toml', 'Gemfile', 'go.mod'].some((f) => STORE_LIBS.test(read(f))))
39
+ return 'db';
40
+ if (['migrations', 'database/migrations', 'db/migrate', 'prisma/migrations'].some((d) => existsSync(join(root, d))))
41
+ return 'db';
42
+ return 'api';
43
+ }
44
+ /** The journey's "Test setup" section, heading excluded, or undefined when it has none. */
45
+ export function testSetupSection(markdown) {
46
+ const m = /^##\s+Test setup\s*$/im.exec(markdown);
47
+ if (!m)
48
+ return undefined;
49
+ const rest = markdown.slice(m.index + m[0].length);
50
+ const end = rest.search(/^##\s/m);
51
+ const body = (end === -1 ? rest : rest.slice(0, end)).trim();
52
+ return body || undefined;
53
+ }
54
+ const HEAD = [
55
+ '# Data these tests need',
56
+ '',
57
+ 'This repository does not own its data store, so TestForge writes no seed code for it.',
58
+ 'The testkit creates what the product\'s API can create. Everything below must exist',
59
+ 'in the environment the tests run against; each section is the journey\'s own "Test setup".',
60
+ 'Ids and values go in the env file named in testforge.config.json.',
61
+ '',
62
+ ].join('\n');
63
+ /** Write or replace one journey's section of <seedDir>/REQUIRED-DATA.md. Returns the file path. */
64
+ export function writeRequiredData(seedDir, journeyId, journeyName, markdown) {
65
+ const path = join(seedDir, 'REQUIRED-DATA.md');
66
+ const open = `<!-- journey: ${journeyId} -->`, close = `<!-- /journey: ${journeyId} -->`;
67
+ const body = testSetupSection(markdown) ?? '_The journey doc has no "Test setup" section: nothing is listed._';
68
+ const block = `${open}\n## ${journeyName}\n\n${body}\n${close}`;
69
+ let text = existsSync(path) ? readFileSync(path, 'utf-8') : HEAD;
70
+ const at = text.indexOf(open), end = text.indexOf(close);
71
+ text = at !== -1 && end > at ? text.slice(0, at) + block + text.slice(end + close.length) : `${text.trimEnd()}\n\n${block}\n`;
72
+ mkdirSync(seedDir, { recursive: true });
73
+ writeFileSync(path, text, 'utf-8');
74
+ return path;
75
+ }
@@ -0,0 +1,104 @@
1
+ /**
2
+ * `testforge review` — judge a journey's failures again, without the verify
3
+ * and repair pass that produced them.
4
+ *
5
+ * Verify keeps its last results per journey. Review reads them (or runs the
6
+ * journey once, without repairing anything, when there are none or --rerun
7
+ * asks for it), reviews every failure — a claimed defect against its claim,
8
+ * an unclaimed one against the test's own oracle — and writes the verdicts
9
+ * where the reporters pick them up for the dashboard.
10
+ */
11
+ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
12
+ import { basename, extname, join } from 'node:path';
13
+ import { STATE_DIR } from '../paths.js';
14
+ import { findMarkdownFiles } from '../find-files.js';
15
+ import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
16
+ import { loadConfig, testDirForType } from '../project/config.js';
17
+ import { matchesJourney } from './plan-cache.js';
18
+ import { claimedDefects, verifyRun } from './verify.js';
19
+ import { groupClaims, oracleClaim, reviewDefects } from './defect-review.js';
20
+ const RESULTS = join(STATE_DIR, 'verify-results');
21
+ const REVIEWS = join(STATE_DIR, 'defect-review');
22
+ export function saveVerifyResults(journeyId, failures, passes) {
23
+ mkdirSync(RESULTS, { recursive: true });
24
+ const saved = { at: new Date().toISOString(), failures, passes };
25
+ writeFileSync(join(RESULTS, `${journeyId}.json`), JSON.stringify(saved, null, 2), 'utf-8');
26
+ }
27
+ export function loadVerifyResults(journeyId) {
28
+ try {
29
+ return JSON.parse(readFileSync(join(RESULTS, `${journeyId}.json`), 'utf-8'));
30
+ }
31
+ catch {
32
+ return undefined;
33
+ }
34
+ }
35
+ /** The claim a failure is reviewed under: the generator's own, else the test's oracle. */
36
+ export function claimFor(t) {
37
+ let content = '';
38
+ try {
39
+ content = readFileSync(t.file, 'utf-8');
40
+ }
41
+ catch {
42
+ return oracleClaim(t, '');
43
+ }
44
+ return claimedDefects(content).get(t.line) ?? oracleClaim(t, content);
45
+ }
46
+ const testFiles = (dir) => {
47
+ const out = [];
48
+ const walk = (d) => {
49
+ for (const e of readdirSync(d, { withFileTypes: true })) {
50
+ const p = join(d, e.name);
51
+ if (e.isDirectory())
52
+ walk(p);
53
+ else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
54
+ out.push(p);
55
+ }
56
+ };
57
+ walk(dir);
58
+ return out;
59
+ };
60
+ export async function runReview(opts = {}) {
61
+ const config = loadConfig();
62
+ const docsDir = opts.docsDir ?? 'docs';
63
+ const out = [];
64
+ for (const file of findMarkdownFiles(docsDir)) {
65
+ const markdown = readFileSync(join(docsDir, file), 'utf-8');
66
+ if (notAJourneyDoc(file, markdown))
67
+ continue;
68
+ const area = /[\\/]/.test(file) ? file.split(/[\\/]/)[0] : undefined;
69
+ const journey = parseJourney(markdown, area, basename(file, extname(file)));
70
+ if (opts.journey && !matchesJourney(file, journey.id, opts.journey))
71
+ continue;
72
+ const testDir = join(testDirForType(config, journey.area), journey.id);
73
+ if (!existsSync(testDir))
74
+ continue;
75
+ let saved = opts.rerun ? undefined : loadVerifyResults(journey.id);
76
+ if (!saved) {
77
+ console.log(` ${journey.id}: running the journey once (no repair) to get its failures…`);
78
+ // A whole journey runs far longer than one file: give it the time a full run takes.
79
+ const files = testFiles(testDir);
80
+ const run = await verifyRun(config, journey.area, files, Math.max(config.verifyTimeoutMs ?? 600_000, files.length * 180_000));
81
+ if (!run.ran || run.infra) {
82
+ out.push({ journey: journey.id, failures: 0, verdicts: {}, skipped: run.infra ?? run.reason ?? 'the run produced no results' });
83
+ continue;
84
+ }
85
+ saveVerifyResults(journey.id, run.failures, run.passes);
86
+ saved = loadVerifyResults(journey.id);
87
+ }
88
+ if (!saved.failures.length) {
89
+ out.push({ journey: journey.id, failures: 0, verdicts: {} });
90
+ continue;
91
+ }
92
+ const groups = groupClaims(saved.failures, claimFor);
93
+ console.log(` ${journey.id}: reviewing ${saved.failures.length} failure(s) in ${groups.length} group(s)…`);
94
+ const reviews = await reviewDefects(journey, groups, saved.passes, undefined, join(REVIEWS, `${journey.id}.partial.json`));
95
+ mkdirSync(REVIEWS, { recursive: true });
96
+ writeFileSync(join(REVIEWS, `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
97
+ rmSync(join(REVIEWS, `${journey.id}.partial.json`), { force: true });
98
+ const verdicts = {};
99
+ for (const r of reviews)
100
+ verdicts[r.verdict] = (verdicts[r.verdict] ?? 0) + r.tests.length;
101
+ out.push({ journey: journey.id, failures: saved.failures.length, verdicts });
102
+ }
103
+ return out;
104
+ }
@@ -33,7 +33,9 @@ import { normalisePending } from './pending.js';
33
33
  import { packageRoot } from '../run/run-tests.js';
34
34
  import { groundJourney } from '../docs/grounding.js';
35
35
  import { generateSeed } from './seed-generator.js';
36
- import { isSyntaxValid, checkTestGates } from '../gates/validate.js';
36
+ import { dataAccess, writeRequiredData } from './required-data.js';
37
+ import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
38
+ import { isSyntaxValid, checkTestGates, duplicatedHelpers } from '../gates/validate.js';
37
39
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
38
40
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
39
41
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
@@ -409,7 +411,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
409
411
  // A freshly written file also may not hold placeholders; a repair is held to
410
412
  // the repair guard instead (it may not add a skip), so old files stay repairable.
411
413
  const genGate = (content, filePath) => testGate(content) ??
412
- checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
414
+ checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath, maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 4 });
413
415
  // Skip if unchanged — but NOT if the journey still has deferred stubs
414
416
  // to retry (a resume pass fills those in without touching real files).
415
417
  if (!force &&
@@ -427,6 +429,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
427
429
  continue;
428
430
  }
429
431
  const warnings = [];
432
+ const usageStart = aiUsage();
430
433
  const collision = collisionWarning(areaDir, journey.id, ext);
431
434
  if (collision)
432
435
  warnings.push(collision);
@@ -568,12 +571,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
568
571
  .join(', ')}${aiFailureNote()}`);
569
572
  }
570
573
  // 2. Seed file (only when using AI — basic mode has no seed concept).
574
+ // A repository without its own data store gets no seed code it could
575
+ // never run: its journey's "Test setup" goes into REQUIRED-DATA.md.
571
576
  let seedStatus = 'none';
572
- if (useAI && !force && existsSync(seedPath)) {
577
+ // Per area: in a monorepo the api package owns the store, the web package does not.
578
+ if (useAI && dataAccess(config, packageRoot(areaDir)) === 'api') {
579
+ writeRequiredData(seedDir, journey.id, journey.name, markdown);
580
+ }
581
+ else if (useAI && !force && existsSync(seedPath)) {
573
582
  seedStatus = 'ok'; // kept, like an existing test file
574
583
  }
575
584
  else if (useAI) {
576
- const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
585
+ let seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
586
+ // A usage limit is not a failed seed: wait for the AI like every other step, then try again.
587
+ while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
588
+ seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
589
+ }
577
590
  if (seed !== null) {
578
591
  mkdirSync(dirname(seedPath), { recursive: true });
579
592
  writeFileSync(seedPath, seed, 'utf-8');
@@ -605,6 +618,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
605
618
  // Only files testforge generated: a project's own config is never edited.
606
619
  ...[harnessPath, join(dirname(testDir), 'playwright.config.js')].filter((f) => !!f && existsSync(f) && readFileSync(f, 'utf-8').includes(GEN_MARKER)),
607
620
  ];
621
+ // One helper copied into several files diverges and fails them all at once: name each copy.
622
+ const dupes = duplicatedHelpers(testFiles.map((t) => (typeof t === 'string' ? t : t.path)).filter((p) => existsSync(p)).map((p) => ({ path: relative(process.cwd(), p), content: readFileSync(p, 'utf-8') })));
623
+ for (const d of dupes) {
624
+ warnings.push(`helper "${d.name}" is copied into ${d.files.length} files (${d.files.join(', ')}) — it belongs in the testkit`);
625
+ }
608
626
  const lint = lintGenerated(lintTargets);
609
627
  if (lint.fixed > 0) {
610
628
  console.log(` tidied ${lint.fixed} lint problem(s) in ${lint.checked} generated file(s)`);
@@ -668,6 +686,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
668
686
  stubbed.add(file);
669
687
  else
670
688
  stubbed.delete(file);
689
+ const ai = usageSince(usageStart);
690
+ if (ai.calls)
691
+ console.log(` ${journey.id}: ${describeUsage(ai)}`);
671
692
  results.push({
672
693
  journeyName: journey.name,
673
694
  testDir,
@@ -678,6 +699,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
678
699
  verified,
679
700
  warnings: warnings.length ? warnings : undefined,
680
701
  reviewed: reviewed.length ? reviewed : undefined,
702
+ ai,
681
703
  });
682
704
  }
683
705
  for (const [area, b] of blockedAreas) {