@activate-core/test-forge 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/cli/index.js +60 -2
- package/dist/core/docs/discover.js +2 -2
- package/dist/core/docs/grounding.js +2 -2
- package/dist/core/docs/reference.js +8 -2
- package/dist/core/docs/review.js +43 -0
- package/dist/core/docs/tickets.js +56 -0
- package/dist/core/gates/coverage.js +11 -3
- package/dist/core/gates/validate.js +109 -0
- package/dist/core/generate/ai-generator.js +52 -3
- package/dist/core/generate/ai-session.js +8 -0
- package/dist/core/generate/ai-usage.js +39 -0
- package/dist/core/generate/defect-review.js +154 -11
- package/dist/core/generate/harness.js +50 -2
- package/dist/core/generate/required-data.js +75 -0
- package/dist/core/generate/review-journey.js +104 -0
- package/dist/core/generate/run-generate.js +26 -4
- package/dist/core/generate/seed-generator.js +2 -2
- package/dist/core/generate/suite-planner.js +10 -2
- package/dist/core/generate/verify-journey.js +46 -5
- package/dist/core/generate/verify.js +9 -4
- package/dist/core/project/env-diff.js +122 -0
- package/dist/core/project/env-template.js +1 -1
- package/dist/core/project/init.js +2 -2
- package/dist/core/project/map.js +2 -2
- package/dist/core/run/ci.js +94 -0
- package/dist/core/run/clean.js +111 -0
- package/dist/core/run/report.js +2 -0
- package/package.json +1 -1
- package/reporters/_shared.cjs +24 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +3 -1
- package/reporters/run-data.cjs +66 -0
- package/reporters/ui-helpers.cjs +56 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the AI cost: calls, wall time and an estimate of tokens, counted in
|
|
3
|
+
* one place every AI call goes through. Tokens are estimated as characters
|
|
4
|
+
* sent and received divided by four — close enough to compare journeys and
|
|
5
|
+
* runs, not a bill.
|
|
6
|
+
*/
|
|
7
|
+
import { execFileSync } from 'node:child_process';
|
|
8
|
+
const total = { calls: 0, ms: 0, inChars: 0, outChars: 0 };
|
|
9
|
+
export function noteAiCall(inChars, outChars, ms) {
|
|
10
|
+
total.calls += 1;
|
|
11
|
+
total.ms += ms;
|
|
12
|
+
total.inChars += inChars;
|
|
13
|
+
total.outChars += outChars;
|
|
14
|
+
}
|
|
15
|
+
/** A snapshot of the running totals. */
|
|
16
|
+
export function aiUsage() {
|
|
17
|
+
return { ...total };
|
|
18
|
+
}
|
|
19
|
+
/** What was spent between two snapshots. */
|
|
20
|
+
export function usageSince(start, now = aiUsage()) {
|
|
21
|
+
return { calls: now.calls - start.calls, ms: now.ms - start.ms, inChars: now.inChars - start.inChars, outChars: now.outChars - start.outChars };
|
|
22
|
+
}
|
|
23
|
+
export const approxTokens = (u) => Math.round((u.inChars + u.outChars) / 4);
|
|
24
|
+
export function describeUsage(u) {
|
|
25
|
+
const min = u.ms / 60_000;
|
|
26
|
+
return `${u.calls} AI call(s), ${min >= 1 ? `${min.toFixed(1)} min` : `${Math.round(u.ms / 1000)} s`}, ~${approxTokens(u).toLocaleString('en-GB')} tokens`;
|
|
27
|
+
}
|
|
28
|
+
/** execFileSync for an AI CLI, counted. Same signature as the call it replaces. */
|
|
29
|
+
export function runAiSync(program, args, options) {
|
|
30
|
+
const started = Date.now();
|
|
31
|
+
let out = '';
|
|
32
|
+
try {
|
|
33
|
+
out = execFileSync(program, args, options);
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
finally {
|
|
37
|
+
noteAiCall(String(options.input ?? '').length, out.length, Date.now() - started);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -16,12 +16,14 @@
|
|
|
16
16
|
* what survives is reported as a defect; everything else is said for what
|
|
17
17
|
* it is.
|
|
18
18
|
*/
|
|
19
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
20
|
-
import { relative } from 'node:path';
|
|
19
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
20
|
+
import { dirname, relative } from 'node:path';
|
|
21
21
|
import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
|
|
22
|
-
import { runWithStdin } from './ai-generator.js';
|
|
22
|
+
import { runWithStdin, DEFECT_MARK, FINDING_MARK } from './ai-generator.js';
|
|
23
23
|
import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
|
|
24
|
-
const
|
|
24
|
+
const IMPACTS = ['blocker', 'major', 'minor'];
|
|
25
|
+
const asImpact = (v) => (IMPACTS.includes(v) ? v : undefined);
|
|
26
|
+
const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'environment', 'unproven'];
|
|
25
27
|
/**
|
|
26
28
|
* Group failing tests by the claim marked above them. A parametrised test
|
|
27
29
|
* shares one declaration — and so one claim — across all its rows, and one
|
|
@@ -57,6 +59,10 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
|
|
|
57
59
|
``,
|
|
58
60
|
`THE CLAIM (written by the tool that generated and repaired the test):`,
|
|
59
61
|
` ${group.claim}`,
|
|
62
|
+
...(group.claim.includes(NOT_CLAIMED)
|
|
63
|
+
? [` The generator did not claim a defect here: the claim is the test's own oracle.`,
|
|
64
|
+
` Judge it the same way, and say "test-fault" or "environment" when that is what it is.`]
|
|
65
|
+
: []),
|
|
60
66
|
``,
|
|
61
67
|
`FAILING TESTS that carry it (${group.tests.length}), numbered:`,
|
|
62
68
|
...group.tests.slice(0, MAX_LISTED).map((t, i) => ` ${i + 1}. ${relative(process.cwd(), t.file)}:${t.line} ${t.title}`),
|
|
@@ -89,8 +95,22 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
|
|
|
89
95
|
`- "pending-decision": the behaviour is covered by an open decision or a known`,
|
|
90
96
|
` issue in the list below — whatever the tests say.`,
|
|
91
97
|
`- "test-fault": the test, its stub, its precondition or its claim is wrong.`,
|
|
98
|
+
`- "environment": the failure comes from the environment the test ran in — a`,
|
|
99
|
+
` missing variable or account, an unreachable or refusing outside service, seed`,
|
|
100
|
+
` data that is not there, a rate limit, or data left behind by earlier runs —`,
|
|
101
|
+
` not from the application or the test's logic.`,
|
|
92
102
|
`- "unproven": none of the above can be shown.`,
|
|
93
103
|
``,
|
|
104
|
+
`For a "defect" or "finding", also grade its impact on the customer:`,
|
|
105
|
+
`- "blocker": no way through, or unsafe — security or data exposure, money taken`,
|
|
106
|
+
` wrongly, or a core step (log in, sign up, book, pay) that cannot be completed`,
|
|
107
|
+
` at all. A failure the test forced (a stubbed error, a cut connection, absurd`,
|
|
108
|
+
` input) is never a blocker on its own.`,
|
|
109
|
+
`- "major": the task can be done but the customer is misled or left without a`,
|
|
110
|
+
` reason — a missing or wrong message, wrong data shown, an error on input.`,
|
|
111
|
+
`- "minor": presentation, or another way works — keyboard, focus, Escape,`,
|
|
112
|
+
` wording and labels, icons, layout.`,
|
|
113
|
+
``,
|
|
94
114
|
...(decisions.text
|
|
95
115
|
? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet):`, decisions.text, ``]
|
|
96
116
|
: decisions.link
|
|
@@ -106,8 +126,9 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
|
|
|
106
126
|
`that fits most rows and list the others as exceptions, by number.`,
|
|
107
127
|
``,
|
|
108
128
|
`Reply with ONLY this JSON object, nothing before or after it:`,
|
|
109
|
-
`{"verdict": "defect|finding|pending-decision|test-fault|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
|
|
110
|
-
` "
|
|
129
|
+
`{"verdict": "defect|finding|pending-decision|test-fault|environment|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
|
|
130
|
+
` "impact": "blocker|major|minor (defect or finding only, else empty)",`,
|
|
131
|
+
` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "...", "impact": "..."}]}`,
|
|
111
132
|
].join('\n');
|
|
112
133
|
}
|
|
113
134
|
/** How many rows of one claim the reviewer is shown and may single out. */
|
|
@@ -126,14 +147,20 @@ export function parseReview(raw) {
|
|
|
126
147
|
verdict: asVerdict(e?.verdict),
|
|
127
148
|
reason: text(e?.reason) ?? '',
|
|
128
149
|
evidence: text(e?.evidence),
|
|
150
|
+
...(asImpact(e?.impact) ? { impact: asImpact(e?.impact) } : {}),
|
|
129
151
|
}))
|
|
130
152
|
.filter((e) => e.tests.length);
|
|
131
|
-
|
|
153
|
+
const impact = asImpact(o.impact);
|
|
154
|
+
return { verdict: asVerdict(o.verdict), reason: text(o.reason) ?? '', evidence: text(o.evidence), ...(impact ? { impact } : {}), exceptions };
|
|
132
155
|
}
|
|
133
156
|
catch {
|
|
134
157
|
return { verdict: 'unproven', reason: 'the review could not be read', exceptions: [] };
|
|
135
158
|
}
|
|
136
159
|
}
|
|
160
|
+
/** A defect or finding always carries an impact; it defaults to major when the reviewer gave none. Other verdicts carry none. */
|
|
161
|
+
function withImpact(r, impact) {
|
|
162
|
+
return r.verdict === 'defect' || r.verdict === 'finding' ? { ...r, impact: impact ?? 'major' } : r;
|
|
163
|
+
}
|
|
137
164
|
/** One review per verdict: the group's own, and one for each set of exceptions. */
|
|
138
165
|
export function splitReview(group, parsed) {
|
|
139
166
|
const taken = new Set();
|
|
@@ -142,14 +169,102 @@ export function splitReview(group, parsed) {
|
|
|
142
169
|
const tests = e.tests.filter((n) => n <= group.tests.length && !taken.has(n));
|
|
143
170
|
tests.forEach((n) => taken.add(n));
|
|
144
171
|
if (tests.length) {
|
|
145
|
-
out.push({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence });
|
|
172
|
+
out.push(withImpact({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence }, e.impact));
|
|
146
173
|
}
|
|
147
174
|
}
|
|
148
175
|
const rest = group.tests.filter((_, i) => !taken.has(i + 1));
|
|
149
176
|
if (rest.length)
|
|
150
|
-
out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
|
|
177
|
+
out.unshift(withImpact({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence }, parsed.impact));
|
|
151
178
|
return out;
|
|
152
179
|
}
|
|
180
|
+
/** Marks a claim built from a test's oracle because the generator claimed nothing. */
|
|
181
|
+
export const NOT_CLAIMED = '(not claimed as a defect by the generator)';
|
|
182
|
+
/**
|
|
183
|
+
* The rule a failing test checks, read from its first "// oracle:" comment,
|
|
184
|
+
* so a failure nobody claimed can still be reviewed against its tier. Falls
|
|
185
|
+
* back to the test's title.
|
|
186
|
+
*/
|
|
187
|
+
export function oracleClaim(t, content) {
|
|
188
|
+
const lines = content.split('\n');
|
|
189
|
+
for (let i = Math.max(0, t.line - 1), seen = 0; i < lines.length && seen < 200; i++, seen++) {
|
|
190
|
+
if (seen > 0 && /^\s*(?:test|it)(?:\.\w+)?\s*\(/.test(lines[i]))
|
|
191
|
+
break;
|
|
192
|
+
const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
|
|
193
|
+
if (m)
|
|
194
|
+
return `${m[1].trim()} ${NOT_CLAIMED}`;
|
|
195
|
+
}
|
|
196
|
+
return `${t.title} ${NOT_CLAIMED}`;
|
|
197
|
+
}
|
|
198
|
+
/** A [spec] line that carries the client's words, quoted or pointing at the quote above it ("source as above"). */
|
|
199
|
+
export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:?\s*(?:as|same)\b)|same source\b)/i;
|
|
200
|
+
/** Playwright's report of a locator that matched more than one element. */
|
|
201
|
+
const AMBIGUOUS = /strict mode violation|resolved to \d+ elements/i;
|
|
202
|
+
/** An application file:line, the only evidence a defect may rest on. */
|
|
203
|
+
const CODE_REF = /[\w@./-]+\.(?:tsx?|jsx?|mjs|cjs|php|py|rb|go|java|kt|cs|vue|svelte)(?::\d+)/i;
|
|
204
|
+
const alnum = (v) => v.toLowerCase().replace(/\\./g, '').replace(/[^a-z0-9]+/g, '');
|
|
205
|
+
/** The tier a claim is made against, read from its tag. */
|
|
206
|
+
export function claimTier(claim) {
|
|
207
|
+
return /\[(spec|baseline|observed)\]/i.exec(claim)?.[1]?.toLowerCase();
|
|
208
|
+
}
|
|
209
|
+
/** The journey line a [spec] claim quotes, if the doc has one. */
|
|
210
|
+
export function specLineFor(claim, journeyMarkdown) {
|
|
211
|
+
// Claims often shorten the rule with "..." anywhere in it: match on its longest unshortened piece.
|
|
212
|
+
const rule = claim.slice(claim.search(/\[spec\]/i) + '[spec]'.length).split(/\s[—-]\s/)[0];
|
|
213
|
+
const piece = rule.split(/\.\.\.|…/).map(alnum).sort((a, b) => b.length - a.length)[0] ?? '';
|
|
214
|
+
const key = piece.slice(0, 40);
|
|
215
|
+
if (key.length < 12)
|
|
216
|
+
return undefined;
|
|
217
|
+
return journeyMarkdown.split('\n').find((l) => /\[spec\]/i.test(l) && alnum(l).includes(key));
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* The reviewer's word is not the last one. A "defect" stands only when the
|
|
221
|
+
* claim is a [spec] rule the client wrote, the journey line carries its
|
|
222
|
+
* (source: "…") quote, and the reviewer names the application file:line.
|
|
223
|
+
* Without the code reference it is unproven; against a [baseline]/[observed]
|
|
224
|
+
* rule, or a [spec] line with no quoted source, it is at most a finding.
|
|
225
|
+
*/
|
|
226
|
+
export function gradeReview(r, journeyMarkdown) {
|
|
227
|
+
// A locator that matched several elements never reached one behaviour: that is the test's fault.
|
|
228
|
+
if ((r.verdict === 'defect' || r.verdict === 'finding') && r.tests.some((t) => AMBIGUOUS.test(t.detail ?? ''))) {
|
|
229
|
+
return { ...r, verdict: 'test-fault', impact: undefined, reason: `${r.reason} [graded: the test's locator matched several elements (strict mode violation)]`.trim() };
|
|
230
|
+
}
|
|
231
|
+
if (r.verdict !== 'defect')
|
|
232
|
+
return r;
|
|
233
|
+
if (!r.evidence || !CODE_REF.test(r.evidence)) {
|
|
234
|
+
return { ...r, verdict: 'unproven', impact: undefined, reason: `${r.reason} [graded: no application file:line was given as evidence]`.trim() };
|
|
235
|
+
}
|
|
236
|
+
const tier = claimTier(r.claim);
|
|
237
|
+
if (tier !== 'spec') {
|
|
238
|
+
return { ...r, verdict: 'finding', reason: `${r.reason} [graded: the rule is ${tier ? `[${tier}]` : 'untagged'}, not a client requirement]`.trim() };
|
|
239
|
+
}
|
|
240
|
+
const line = specLineFor(r.claim, journeyMarkdown);
|
|
241
|
+
if (!line || !HAS_SOURCE.test(line)) {
|
|
242
|
+
return { ...r, verdict: 'finding', reason: `${r.reason} [graded: the journey doc has no quoted source for this [spec] rule]`.trim() };
|
|
243
|
+
}
|
|
244
|
+
return r;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* A repair may claim a defect only against a [spec] rule the journey quotes
|
|
248
|
+
* with its source. Any other claim — [baseline], [observed], untagged, or a
|
|
249
|
+
* [spec] rule the doc does not quote — is rewritten to a finding mark: the
|
|
250
|
+
* test stays failing and protected from repair, but is never reported as a
|
|
251
|
+
* defect.
|
|
252
|
+
*/
|
|
253
|
+
export function normaliseClaims(content, journeyMarkdown) {
|
|
254
|
+
let downgraded = 0;
|
|
255
|
+
const lines = content.split('\n').map((line) => {
|
|
256
|
+
const at = line.indexOf(DEFECT_MARK);
|
|
257
|
+
if (at === -1)
|
|
258
|
+
return line;
|
|
259
|
+
const claim = line.slice(at + DEFECT_MARK.length).trim();
|
|
260
|
+
const spec = claimTier(claim) === 'spec' ? specLineFor(claim, journeyMarkdown) : undefined;
|
|
261
|
+
if (spec && HAS_SOURCE.test(spec))
|
|
262
|
+
return line;
|
|
263
|
+
downgraded += 1;
|
|
264
|
+
return line.slice(0, at) + FINDING_MARK + line.slice(at + DEFECT_MARK.length);
|
|
265
|
+
});
|
|
266
|
+
return { content: lines.join('\n'), downgraded };
|
|
267
|
+
}
|
|
153
268
|
/**
|
|
154
269
|
* Review every claim group, a few at a time. Never throws.
|
|
155
270
|
*
|
|
@@ -160,7 +275,26 @@ export function splitReview(group, parsed) {
|
|
|
160
275
|
* reported 0 defects. So the group goes back on the queue, every worker
|
|
161
276
|
* shares one wait, and the review is asked again once the AI answers.
|
|
162
277
|
*/
|
|
163
|
-
export async function reviewDefects(journey, groups, passes, io
|
|
278
|
+
export async function reviewDefects(journey, groups, passes, io,
|
|
279
|
+
/** Where answered groups are kept, so an interrupted review asks only the rest again. */
|
|
280
|
+
checkpoint) {
|
|
281
|
+
const groupKey = (g) => `${g.claim}\u0000${g.tests.map((t) => `${t.file}:${t.line}:${t.title}`).join('|')}`;
|
|
282
|
+
let answered = {};
|
|
283
|
+
if (checkpoint) {
|
|
284
|
+
try {
|
|
285
|
+
answered = JSON.parse(readFileSync(checkpoint, 'utf-8'));
|
|
286
|
+
}
|
|
287
|
+
catch {
|
|
288
|
+
answered = {};
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
const keep = (g, rs) => {
|
|
292
|
+
if (!checkpoint)
|
|
293
|
+
return;
|
|
294
|
+
answered[groupKey(g)] = rs;
|
|
295
|
+
mkdirSync(dirname(checkpoint), { recursive: true });
|
|
296
|
+
writeFileSync(checkpoint, JSON.stringify(answered), 'utf-8');
|
|
297
|
+
};
|
|
164
298
|
const config = loadConfig();
|
|
165
299
|
const decisions = decisionsSource(config.decisions);
|
|
166
300
|
// A linked list (an internal Notion page) needs the reference tools to be read.
|
|
@@ -174,6 +308,11 @@ export async function reviewDefects(journey, groups, passes, io) {
|
|
|
174
308
|
let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
|
|
175
309
|
const worker = async () => {
|
|
176
310
|
for (let g = queue.shift(); g; g = queue.shift()) {
|
|
311
|
+
const earlier = answered[groupKey(g)];
|
|
312
|
+
if (earlier) {
|
|
313
|
+
out.push(...earlier);
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
177
316
|
const files = new Set(g.tests.map((t) => t.file));
|
|
178
317
|
const siblings = passes.filter((p) => files.has(p.file));
|
|
179
318
|
let raw = '';
|
|
@@ -193,7 +332,11 @@ export async function reviewDefects(journey, groups, passes, io) {
|
|
|
193
332
|
}
|
|
194
333
|
// an unanswered review proves nothing
|
|
195
334
|
}
|
|
196
|
-
|
|
335
|
+
const rs = splitReview(g, parseReview(raw)).map((r) => gradeReview(r, journey.rawMarkdown));
|
|
336
|
+
// An unanswered group (the AI never replied) is not kept: it is asked again next time.
|
|
337
|
+
if (raw)
|
|
338
|
+
keep(g, rs);
|
|
339
|
+
out.push(...rs);
|
|
197
340
|
}
|
|
198
341
|
};
|
|
199
342
|
await Promise.all(Array.from({ length: Math.max(1, Math.min(config.aiConcurrency ?? 4, groups.length)) }, worker));
|
|
@@ -11,11 +11,12 @@
|
|
|
11
11
|
* auth (read-only tools), so it connects and seeds against the actual
|
|
12
12
|
* schema — not a guess.
|
|
13
13
|
*/
|
|
14
|
-
import {
|
|
14
|
+
import { spawn } from 'node:child_process';
|
|
15
15
|
import { existsSync, readFileSync } from 'node:fs';
|
|
16
16
|
import { createHash } from 'node:crypto';
|
|
17
17
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
18
18
|
import { cleanOutput } from './ai-generator.js';
|
|
19
|
+
import { runAiSync } from './ai-usage.js';
|
|
19
20
|
const MAP_PATH = 'testforge.md';
|
|
20
21
|
/**
|
|
21
22
|
* The fixed contract every testkit exposes and every generated test
|
|
@@ -39,7 +40,16 @@ export const HARNESS_CONTRACT = [
|
|
|
39
40
|
` - REQUIRED_PACKAGES: string[] packages the testkit needs (see H6).`,
|
|
40
41
|
` - resetRateLimits(), bypassVerification(), suppressPopups(), and one`,
|
|
41
42
|
` resolver per seeded entity — see H7 and H3.`,
|
|
43
|
+
` - unique(label?): string a value no other run produces (H8).`,
|
|
44
|
+
` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
|
|
45
|
+
` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
|
|
46
|
+
` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
|
|
42
47
|
`Plus, for Playwright suites:`,
|
|
48
|
+
` - openStep(page, step, input?): Promise<true | string> open a later step`,
|
|
49
|
+
` of a multi-step flow with what the earlier steps would have left (H9).`,
|
|
50
|
+
` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
|
|
51
|
+
` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
|
|
52
|
+
` rewritten here.`,
|
|
43
53
|
` - apiContext(request): an authenticated API request context, created from`,
|
|
44
54
|
` the imported request MODULE (\`const { request: apiRequest } =`,
|
|
45
55
|
` require('@playwright/test'); apiRequest.newContext({ baseURL,`,
|
|
@@ -144,6 +154,40 @@ export const HARNESS_CONTRACT = [
|
|
|
144
154
|
` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
|
|
145
155
|
` answered with the real success shape too. Scope each to exactly that`,
|
|
146
156
|
` request; everything after it runs against the real backend.`,
|
|
157
|
+
`H8. WHAT A RUN CREATES. Runs share accounts and environments, so data one`,
|
|
158
|
+
` run leaves behind must never change another run's result.`,
|
|
159
|
+
` - Load the run-data runtime once:`,
|
|
160
|
+
` const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
|
|
161
|
+
` unique(label) returns runData.unique(label). created(kind, id, cleanup,`,
|
|
162
|
+
` role) calls runData.record({ kind, id, role, cleanup }), where cleanup is`,
|
|
163
|
+
` { method, url } of the product's own request that removes or cancels that`,
|
|
164
|
+
` record (absolute url; read the routes), or undefined when there is none.`,
|
|
165
|
+
` - Every testkit helper that creates a record through the API calls`,
|
|
166
|
+
` created() with it.`,
|
|
167
|
+
` - The "run" role. TEST_RUN_ACCOUNT_MODE picks the account:`,
|
|
168
|
+
` fixture (default) — the "run" role IS the customer role;`,
|
|
169
|
+
` pool — TEST_RUN_ACCOUNTS is a comma list of identifiers sharing`,
|
|
170
|
+
` TEST_RUN_ACCOUNTS_PASSWORD; pick one by a stable hash of runData.runId();`,
|
|
171
|
+
` signup — create a fresh account through the product's own sign-up, only`,
|
|
172
|
+
` when this environment can complete it (no code it cannot read and`,
|
|
173
|
+
` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
|
|
174
|
+
` Mint and cache its token like any role.`,
|
|
175
|
+
`H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
|
|
176
|
+
` booking or checkout wizard, a sign-up in stages) depends on what the`,
|
|
177
|
+
` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
|
|
178
|
+
` parameters, browser storage or the server. Opened by its URL alone it`,
|
|
179
|
+
` often sends the visitor back to an earlier step without a word, and a`,
|
|
180
|
+
` test then "fails" on the product's correct guard.`,
|
|
181
|
+
` - Read the flow's step components, its router guard and its store, and`,
|
|
182
|
+
` give openStep(page, step, input?) the way the product reaches that step:`,
|
|
183
|
+
` the parameters a real entry link carries, or the stored state the`,
|
|
184
|
+
` earlier steps write (pre-set with an init script before the app runs).`,
|
|
185
|
+
` \`input\` overrides the defaults (which choice, which address).`,
|
|
186
|
+
` - It waits until the step is really shown and resolves true; when the`,
|
|
187
|
+
` product still sends the visitor elsewhere it resolves a string naming`,
|
|
188
|
+
` the step it landed on, so the test can skip with that reason.`,
|
|
189
|
+
` - Offer only the steps the flow has; when the project has no such flow,`,
|
|
190
|
+
` export openStep as a function resolving 'this product has no multi-step flow'.`,
|
|
147
191
|
].join('\n');
|
|
148
192
|
/**
|
|
149
193
|
* Stamp written into every generated testkit: the contract it was built
|
|
@@ -239,7 +283,7 @@ function buildHarnessPrompt(harnessPath, framework) {
|
|
|
239
283
|
export function generateHarness(harnessPath, framework) {
|
|
240
284
|
const prompt = buildHarnessPrompt(harnessPath, framework);
|
|
241
285
|
const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
|
|
242
|
-
const output =
|
|
286
|
+
const output = runAiSync(program, args, {
|
|
243
287
|
input: prompt,
|
|
244
288
|
encoding: 'utf-8',
|
|
245
289
|
maxBuffer: 10 * 1024 * 1024,
|
|
@@ -295,6 +339,8 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
|
|
|
295
339
|
`const { defineConfig } = require('@playwright/test');`,
|
|
296
340
|
`const fs = require('fs');`,
|
|
297
341
|
`const path = require('path');`,
|
|
342
|
+
`// One run id for this run, set here in the main process so every worker inherits it.`,
|
|
343
|
+
`try { require('@activate-core/test-forge/reporters/run-data.cjs').runId(); } catch (_) {}`,
|
|
298
344
|
``,
|
|
299
345
|
`// Load the env file named in testforge.config.json, so a plain`,
|
|
300
346
|
`// \`npx playwright test\` (or an editor's test runner) gets the same env as`,
|
|
@@ -352,6 +398,8 @@ export function jestConfig(testDirRelToPackage, setupRelToPackage, category = 'a
|
|
|
352
398
|
return [
|
|
353
399
|
`// @generated by testforge`,
|
|
354
400
|
`// Install once: npm i -D jest`,
|
|
401
|
+
`// One run id for this run, set before the workers start so each inherits it.`,
|
|
402
|
+
`try { require('@activate-core/test-forge/reporters/run-data.cjs').runId(); } catch (_) {}`,
|
|
355
403
|
`module.exports = {`,
|
|
356
404
|
` testEnvironment: 'node',`,
|
|
357
405
|
` setupFiles: ['<rootDir>/${setupRelToPackage}'],`,
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a journey's starting data comes from.
|
|
3
|
+
*
|
|
4
|
+
* A repository that owns its data store gets a generated seed that writes
|
|
5
|
+
* it. One that does not (a web client talking to someone else's API) cannot
|
|
6
|
+
* run such a seed, so it gets no seed code at all: the testkit sets up what
|
|
7
|
+
* the product's API can create, and REQUIRED-DATA.md lists, per journey, what
|
|
8
|
+
* a person has to provide — taken word for word from the journey's own
|
|
9
|
+
* "Test setup" section, never invented.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
/** Packages that only a repository talking to its own data store depends on. */
|
|
14
|
+
const STORE_PACKAGES = /^(?:mysql2?|pg|postgres|mongoose|mongodb|prisma|@prisma\/client|typeorm|sequelize|knex|drizzle-orm|better-sqlite3|sqlite3|ioredis|redis|@mikro-orm\/core|objection)$/;
|
|
15
|
+
const STORE_LIBS = /\b(?:sqlalchemy|django|psycopg2?|pymongo|peewee|activerecord|mongoid|sequel|doctrine\/orm|illuminate\/database|laravel\/framework|gorm|sqlx)\b/i;
|
|
16
|
+
/**
|
|
17
|
+
* "db" when this repository talks to its own data store, so a seed it
|
|
18
|
+
* generates can run; else "api". The config's `dataAccess` decides when set.
|
|
19
|
+
* Otherwise the repository itself is read — its dependencies and whether it
|
|
20
|
+
* has migrations — never the env file, whose variable names can be left over.
|
|
21
|
+
*/
|
|
22
|
+
export function dataAccess(config, root = '.') {
|
|
23
|
+
if (config.dataAccess === 'db' || config.dataAccess === 'api')
|
|
24
|
+
return config.dataAccess;
|
|
25
|
+
const read = (f) => { try {
|
|
26
|
+
return readFileSync(join(root, f), 'utf-8');
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return '';
|
|
30
|
+
} };
|
|
31
|
+
try {
|
|
32
|
+
const pkg = JSON.parse(read('package.json') || '{}');
|
|
33
|
+
const deps = Object.keys({ ...pkg.dependencies, ...pkg.devDependencies });
|
|
34
|
+
if (deps.some((d) => STORE_PACKAGES.test(d)))
|
|
35
|
+
return 'db';
|
|
36
|
+
}
|
|
37
|
+
catch { /* not a node project, or unreadable package.json */ }
|
|
38
|
+
if (['composer.json', 'requirements.txt', 'pyproject.toml', 'Gemfile', 'go.mod'].some((f) => STORE_LIBS.test(read(f))))
|
|
39
|
+
return 'db';
|
|
40
|
+
if (['migrations', 'database/migrations', 'db/migrate', 'prisma/migrations'].some((d) => existsSync(join(root, d))))
|
|
41
|
+
return 'db';
|
|
42
|
+
return 'api';
|
|
43
|
+
}
|
|
44
|
+
/** The journey's "Test setup" section, heading excluded, or undefined when it has none. */
|
|
45
|
+
export function testSetupSection(markdown) {
|
|
46
|
+
const m = /^##\s+Test setup\s*$/im.exec(markdown);
|
|
47
|
+
if (!m)
|
|
48
|
+
return undefined;
|
|
49
|
+
const rest = markdown.slice(m.index + m[0].length);
|
|
50
|
+
const end = rest.search(/^##\s/m);
|
|
51
|
+
const body = (end === -1 ? rest : rest.slice(0, end)).trim();
|
|
52
|
+
return body || undefined;
|
|
53
|
+
}
|
|
54
|
+
const HEAD = [
|
|
55
|
+
'# Data these tests need',
|
|
56
|
+
'',
|
|
57
|
+
'This repository does not own its data store, so TestForge writes no seed code for it.',
|
|
58
|
+
'The testkit creates what the product\'s API can create. Everything below must exist',
|
|
59
|
+
'in the environment the tests run against; each section is the journey\'s own "Test setup".',
|
|
60
|
+
'Ids and values go in the env file named in testforge.config.json.',
|
|
61
|
+
'',
|
|
62
|
+
].join('\n');
|
|
63
|
+
/** Write or replace one journey's section of <seedDir>/REQUIRED-DATA.md. Returns the file path. */
|
|
64
|
+
export function writeRequiredData(seedDir, journeyId, journeyName, markdown) {
|
|
65
|
+
const path = join(seedDir, 'REQUIRED-DATA.md');
|
|
66
|
+
const open = `<!-- journey: ${journeyId} -->`, close = `<!-- /journey: ${journeyId} -->`;
|
|
67
|
+
const body = testSetupSection(markdown) ?? '_The journey doc has no "Test setup" section: nothing is listed._';
|
|
68
|
+
const block = `${open}\n## ${journeyName}\n\n${body}\n${close}`;
|
|
69
|
+
let text = existsSync(path) ? readFileSync(path, 'utf-8') : HEAD;
|
|
70
|
+
const at = text.indexOf(open), end = text.indexOf(close);
|
|
71
|
+
text = at !== -1 && end > at ? text.slice(0, at) + block + text.slice(end + close.length) : `${text.trimEnd()}\n\n${block}\n`;
|
|
72
|
+
mkdirSync(seedDir, { recursive: true });
|
|
73
|
+
writeFileSync(path, text, 'utf-8');
|
|
74
|
+
return path;
|
|
75
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `testforge review` — judge a journey's failures again, without the verify
|
|
3
|
+
* and repair pass that produced them.
|
|
4
|
+
*
|
|
5
|
+
* Verify keeps its last results per journey. Review reads them (or runs the
|
|
6
|
+
* journey once, without repairing anything, when there are none or --rerun
|
|
7
|
+
* asks for it), reviews every failure — a claimed defect against its claim,
|
|
8
|
+
* an unclaimed one against the test's own oracle — and writes the verdicts
|
|
9
|
+
* where the reporters pick them up for the dashboard.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { basename, extname, join } from 'node:path';
|
|
13
|
+
import { STATE_DIR } from '../paths.js';
|
|
14
|
+
import { findMarkdownFiles } from '../find-files.js';
|
|
15
|
+
import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
|
|
16
|
+
import { loadConfig, testDirForType } from '../project/config.js';
|
|
17
|
+
import { matchesJourney } from './plan-cache.js';
|
|
18
|
+
import { claimedDefects, verifyRun } from './verify.js';
|
|
19
|
+
import { groupClaims, oracleClaim, reviewDefects } from './defect-review.js';
|
|
20
|
+
const RESULTS = join(STATE_DIR, 'verify-results');
|
|
21
|
+
const REVIEWS = join(STATE_DIR, 'defect-review');
|
|
22
|
+
export function saveVerifyResults(journeyId, failures, passes) {
|
|
23
|
+
mkdirSync(RESULTS, { recursive: true });
|
|
24
|
+
const saved = { at: new Date().toISOString(), failures, passes };
|
|
25
|
+
writeFileSync(join(RESULTS, `${journeyId}.json`), JSON.stringify(saved, null, 2), 'utf-8');
|
|
26
|
+
}
|
|
27
|
+
export function loadVerifyResults(journeyId) {
|
|
28
|
+
try {
|
|
29
|
+
return JSON.parse(readFileSync(join(RESULTS, `${journeyId}.json`), 'utf-8'));
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** The claim a failure is reviewed under: the generator's own, else the test's oracle. */
|
|
36
|
+
export function claimFor(t) {
|
|
37
|
+
let content = '';
|
|
38
|
+
try {
|
|
39
|
+
content = readFileSync(t.file, 'utf-8');
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return oracleClaim(t, '');
|
|
43
|
+
}
|
|
44
|
+
return claimedDefects(content).get(t.line) ?? oracleClaim(t, content);
|
|
45
|
+
}
|
|
46
|
+
const testFiles = (dir) => {
|
|
47
|
+
const out = [];
|
|
48
|
+
const walk = (d) => {
|
|
49
|
+
for (const e of readdirSync(d, { withFileTypes: true })) {
|
|
50
|
+
const p = join(d, e.name);
|
|
51
|
+
if (e.isDirectory())
|
|
52
|
+
walk(p);
|
|
53
|
+
else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
|
|
54
|
+
out.push(p);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
walk(dir);
|
|
58
|
+
return out;
|
|
59
|
+
};
|
|
60
|
+
export async function runReview(opts = {}) {
|
|
61
|
+
const config = loadConfig();
|
|
62
|
+
const docsDir = opts.docsDir ?? 'docs';
|
|
63
|
+
const out = [];
|
|
64
|
+
for (const file of findMarkdownFiles(docsDir)) {
|
|
65
|
+
const markdown = readFileSync(join(docsDir, file), 'utf-8');
|
|
66
|
+
if (notAJourneyDoc(file, markdown))
|
|
67
|
+
continue;
|
|
68
|
+
const area = /[\\/]/.test(file) ? file.split(/[\\/]/)[0] : undefined;
|
|
69
|
+
const journey = parseJourney(markdown, area, basename(file, extname(file)));
|
|
70
|
+
if (opts.journey && !matchesJourney(file, journey.id, opts.journey))
|
|
71
|
+
continue;
|
|
72
|
+
const testDir = join(testDirForType(config, journey.area), journey.id);
|
|
73
|
+
if (!existsSync(testDir))
|
|
74
|
+
continue;
|
|
75
|
+
let saved = opts.rerun ? undefined : loadVerifyResults(journey.id);
|
|
76
|
+
if (!saved) {
|
|
77
|
+
console.log(` ${journey.id}: running the journey once (no repair) to get its failures…`);
|
|
78
|
+
// A whole journey runs far longer than one file: give it the time a full run takes.
|
|
79
|
+
const files = testFiles(testDir);
|
|
80
|
+
const run = await verifyRun(config, journey.area, files, Math.max(config.verifyTimeoutMs ?? 600_000, files.length * 180_000));
|
|
81
|
+
if (!run.ran || run.infra) {
|
|
82
|
+
out.push({ journey: journey.id, failures: 0, verdicts: {}, skipped: run.infra ?? run.reason ?? 'the run produced no results' });
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
saveVerifyResults(journey.id, run.failures, run.passes);
|
|
86
|
+
saved = loadVerifyResults(journey.id);
|
|
87
|
+
}
|
|
88
|
+
if (!saved.failures.length) {
|
|
89
|
+
out.push({ journey: journey.id, failures: 0, verdicts: {} });
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const groups = groupClaims(saved.failures, claimFor);
|
|
93
|
+
console.log(` ${journey.id}: reviewing ${saved.failures.length} failure(s) in ${groups.length} group(s)…`);
|
|
94
|
+
const reviews = await reviewDefects(journey, groups, saved.passes, undefined, join(REVIEWS, `${journey.id}.partial.json`));
|
|
95
|
+
mkdirSync(REVIEWS, { recursive: true });
|
|
96
|
+
writeFileSync(join(REVIEWS, `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
|
|
97
|
+
rmSync(join(REVIEWS, `${journey.id}.partial.json`), { force: true });
|
|
98
|
+
const verdicts = {};
|
|
99
|
+
for (const r of reviews)
|
|
100
|
+
verdicts[r.verdict] = (verdicts[r.verdict] ?? 0) + r.tests.length;
|
|
101
|
+
out.push({ journey: journey.id, failures: saved.failures.length, verdicts });
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
@@ -33,7 +33,9 @@ import { normalisePending } from './pending.js';
|
|
|
33
33
|
import { packageRoot } from '../run/run-tests.js';
|
|
34
34
|
import { groundJourney } from '../docs/grounding.js';
|
|
35
35
|
import { generateSeed } from './seed-generator.js';
|
|
36
|
-
import {
|
|
36
|
+
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
37
|
+
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
38
|
+
import { isSyntaxValid, checkTestGates, duplicatedHelpers } from '../gates/validate.js';
|
|
37
39
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
38
40
|
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
39
41
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
@@ -409,7 +411,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
409
411
|
// A freshly written file also may not hold placeholders; a repair is held to
|
|
410
412
|
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
411
413
|
const genGate = (content, filePath) => testGate(content) ??
|
|
412
|
-
checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
|
|
414
|
+
checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath, maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 4 });
|
|
413
415
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
414
416
|
// to retry (a resume pass fills those in without touching real files).
|
|
415
417
|
if (!force &&
|
|
@@ -427,6 +429,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
427
429
|
continue;
|
|
428
430
|
}
|
|
429
431
|
const warnings = [];
|
|
432
|
+
const usageStart = aiUsage();
|
|
430
433
|
const collision = collisionWarning(areaDir, journey.id, ext);
|
|
431
434
|
if (collision)
|
|
432
435
|
warnings.push(collision);
|
|
@@ -568,12 +571,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
568
571
|
.join(', ')}${aiFailureNote()}`);
|
|
569
572
|
}
|
|
570
573
|
// 2. Seed file (only when using AI — basic mode has no seed concept).
|
|
574
|
+
// A repository without its own data store gets no seed code it could
|
|
575
|
+
// never run: its journey's "Test setup" goes into REQUIRED-DATA.md.
|
|
571
576
|
let seedStatus = 'none';
|
|
572
|
-
|
|
577
|
+
// Per area: in a monorepo the api package owns the store, the web package does not.
|
|
578
|
+
if (useAI && dataAccess(config, packageRoot(areaDir)) === 'api') {
|
|
579
|
+
writeRequiredData(seedDir, journey.id, journey.name, markdown);
|
|
580
|
+
}
|
|
581
|
+
else if (useAI && !force && existsSync(seedPath)) {
|
|
573
582
|
seedStatus = 'ok'; // kept, like an existing test file
|
|
574
583
|
}
|
|
575
584
|
else if (useAI) {
|
|
576
|
-
|
|
585
|
+
let seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
|
|
586
|
+
// A usage limit is not a failed seed: wait for the AI like every other step, then try again.
|
|
587
|
+
while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
|
|
588
|
+
seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
|
|
589
|
+
}
|
|
577
590
|
if (seed !== null) {
|
|
578
591
|
mkdirSync(dirname(seedPath), { recursive: true });
|
|
579
592
|
writeFileSync(seedPath, seed, 'utf-8');
|
|
@@ -605,6 +618,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
605
618
|
// Only files testforge generated: a project's own config is never edited.
|
|
606
619
|
...[harnessPath, join(dirname(testDir), 'playwright.config.js')].filter((f) => !!f && existsSync(f) && readFileSync(f, 'utf-8').includes(GEN_MARKER)),
|
|
607
620
|
];
|
|
621
|
+
// One helper copied into several files diverges and fails them all at once: name each copy.
|
|
622
|
+
const dupes = duplicatedHelpers(testFiles.map((t) => (typeof t === 'string' ? t : t.path)).filter((p) => existsSync(p)).map((p) => ({ path: relative(process.cwd(), p), content: readFileSync(p, 'utf-8') })));
|
|
623
|
+
for (const d of dupes) {
|
|
624
|
+
warnings.push(`helper "${d.name}" is copied into ${d.files.length} files (${d.files.join(', ')}) — it belongs in the testkit`);
|
|
625
|
+
}
|
|
608
626
|
const lint = lintGenerated(lintTargets);
|
|
609
627
|
if (lint.fixed > 0) {
|
|
610
628
|
console.log(` tidied ${lint.fixed} lint problem(s) in ${lint.checked} generated file(s)`);
|
|
@@ -668,6 +686,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
668
686
|
stubbed.add(file);
|
|
669
687
|
else
|
|
670
688
|
stubbed.delete(file);
|
|
689
|
+
const ai = usageSince(usageStart);
|
|
690
|
+
if (ai.calls)
|
|
691
|
+
console.log(` ${journey.id}: ${describeUsage(ai)}`);
|
|
671
692
|
results.push({
|
|
672
693
|
journeyName: journey.name,
|
|
673
694
|
testDir,
|
|
@@ -678,6 +699,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
678
699
|
verified,
|
|
679
700
|
warnings: warnings.length ? warnings : undefined,
|
|
680
701
|
reviewed: reviewed.length ? reviewed : undefined,
|
|
702
|
+
ai,
|
|
681
703
|
});
|
|
682
704
|
}
|
|
683
705
|
for (const [area, b] of blockedAreas) {
|