@activate-core/test-forge 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +25 -5
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +17 -1
- package/dist/core/generate/ai-generator.js +208 -283
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +122 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +63 -68
- package/dist/core/generate/suite-planner.js +52 -3
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — an independent review of every claimed product defect.
|
|
3
|
+
*
|
|
4
|
+
* The repair step decides, for each failing test, whether the test or the
|
|
5
|
+
* application is at fault, and marks the second kind in the file. That is
|
|
6
|
+
* the same model that wrote the test, judging its own work while trying to
|
|
7
|
+
* make it pass — and it leans towards "the application is wrong". A real
|
|
8
|
+
* run showed what that costs: tests that raced the app's own redirect,
|
|
9
|
+
* stubs that returned what the server never sends, and open product
|
|
10
|
+
* questions, all reported as defects.
|
|
11
|
+
*
|
|
12
|
+
* So no claim reaches the report on the repair's word. Each one goes to a
|
|
13
|
+
* separate reviewer that starts from "not proven", reads the test, the
|
|
14
|
+
* failure, the journey document, the application source and the project's
|
|
15
|
+
* list of open decisions, and must show the application code at fault. Only
|
|
16
|
+
* what survives is reported as a defect; everything else is said for what
|
|
17
|
+
* it is.
|
|
18
|
+
*/
|
|
19
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
20
|
+
import { relative } from 'node:path';
|
|
21
|
+
import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
|
|
22
|
+
import { runWithStdin } from './ai-generator.js';
|
|
23
|
+
const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
|
|
24
|
+
/**
|
|
25
|
+
* Group failing tests by the claim marked above them. A parametrised test
|
|
26
|
+
* shares one declaration — and so one claim — across all its rows, and one
|
|
27
|
+
* missing error message is one problem however many inputs reach it.
|
|
28
|
+
*/
|
|
29
|
+
export function groupClaims(failures, claimOf) {
|
|
30
|
+
const groups = new Map();
|
|
31
|
+
for (const t of failures) {
|
|
32
|
+
const claim = claimOf(t);
|
|
33
|
+
if (!claim)
|
|
34
|
+
continue;
|
|
35
|
+
const g = groups.get(claim) ?? { claim, tests: [] };
|
|
36
|
+
g.tests.push(t);
|
|
37
|
+
groups.set(claim, g);
|
|
38
|
+
}
|
|
39
|
+
return [...groups.values()];
|
|
40
|
+
}
|
|
41
|
+
/** The project's open decisions: inline when it is a file, otherwise a link the reviewer reads. */
|
|
42
|
+
export function decisionsSource(spec) {
|
|
43
|
+
if (!spec)
|
|
44
|
+
return {};
|
|
45
|
+
if (existsSync(spec))
|
|
46
|
+
return { text: readFileSync(spec, 'utf-8') };
|
|
47
|
+
return { link: spec };
|
|
48
|
+
}
|
|
49
|
+
export function buildDefectReviewPrompt(journey, group, passes, decisions) {
|
|
50
|
+
const files = [...new Set(group.tests.map((t) => relative(process.cwd(), t.file)))];
|
|
51
|
+
const sample = group.tests.slice(0, 3);
|
|
52
|
+
return [
|
|
53
|
+
`You review ONE claimed product defect before it is reported to a team that`,
|
|
54
|
+
`will use any false claim to dismiss every real one. Start from "NOT PROVEN"`,
|
|
55
|
+
`and change your mind only on evidence you can point to.`,
|
|
56
|
+
``,
|
|
57
|
+
`THE CLAIM (written by the tool that generated and repaired the test):`,
|
|
58
|
+
` ${group.claim}`,
|
|
59
|
+
``,
|
|
60
|
+
`FAILING TESTS that carry it (${group.tests.length}), numbered:`,
|
|
61
|
+
...group.tests.slice(0, MAX_LISTED).map((t, i) => ` ${i + 1}. ${relative(process.cwd(), t.file)}:${t.line} ${t.title}`),
|
|
62
|
+
...(group.tests.length > MAX_LISTED ? [` … and ${group.tests.length - MAX_LISTED} more`] : []),
|
|
63
|
+
``,
|
|
64
|
+
`WHAT THE RUNNER REPORTED (first ${sample.length}):`,
|
|
65
|
+
...sample.map((t) => `--- ${t.title}\n${t.detail}`),
|
|
66
|
+
``,
|
|
67
|
+
...(passes.length
|
|
68
|
+
? [
|
|
69
|
+
`TESTS IN THE SAME FILE(S) THAT PASSED in this run (a sibling that asserts the`,
|
|
70
|
+
`same behaviour and passes means the failing one is at fault, not the app):`,
|
|
71
|
+
...passes.slice(0, 40).map((p) => ` - ${p.line}: ${p.title}`),
|
|
72
|
+
``,
|
|
73
|
+
]
|
|
74
|
+
: []),
|
|
75
|
+
`Read the test file(s) (${files.join(', ')}), the application source the test`,
|
|
76
|
+
`exercises, and the server code behind it. You have read-only tools.`,
|
|
77
|
+
``,
|
|
78
|
+
`Decide exactly one verdict:`,
|
|
79
|
+
`- "defect": ALL of these hold — (a) the criterion is stated in the journey`,
|
|
80
|
+
` document below and tagged [spec], and the claim's detail is in the quoted`,
|
|
81
|
+
` source, not added to it; (b) the test really reached the behaviour: the`,
|
|
82
|
+
` element, request or response it checks is the right one, nothing the test`,
|
|
83
|
+
` did itself (a navigation of its own, a stub, injected storage, a`,
|
|
84
|
+
` precondition, timing) produced the failure, and any stub answers only with`,
|
|
85
|
+
` what the real server can send; (c) you found the application code that`,
|
|
86
|
+
` causes it — give its file:line.`,
|
|
87
|
+
`- "finding": (b) and (c) hold, but the criterion is [baseline] or [observed].`,
|
|
88
|
+
`- "pending-decision": the behaviour is covered by an open decision or a known`,
|
|
89
|
+
` issue in the list below — whatever the tests say.`,
|
|
90
|
+
`- "test-fault": the test, its stub, its precondition or its claim is wrong.`,
|
|
91
|
+
`- "unproven": none of the above can be shown.`,
|
|
92
|
+
``,
|
|
93
|
+
...(decisions.text
|
|
94
|
+
? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet):`, decisions.text, ``]
|
|
95
|
+
: decisions.link
|
|
96
|
+
? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet) — read it: ${decisions.link}`, ``]
|
|
97
|
+
: [`No list of open decisions was provided.`, ``]),
|
|
98
|
+
`--- JOURNEY DOCUMENT ---`,
|
|
99
|
+
journey.rawMarkdown,
|
|
100
|
+
`--- END ---`,
|
|
101
|
+
``,
|
|
102
|
+
`One claim often covers many rows of the same test (one input per row). Judge`,
|
|
103
|
+
`every row: when some rows fall under a different verdict than the rest (one`,
|
|
104
|
+
`input breaks a [spec] rule, another only an open decision), give the verdict`,
|
|
105
|
+
`that fits most rows and list the others as exceptions, by number.`,
|
|
106
|
+
``,
|
|
107
|
+
`Reply with ONLY this JSON object, nothing before or after it:`,
|
|
108
|
+
`{"verdict": "defect|finding|pending-decision|test-fault|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
|
|
109
|
+
` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "..."}]}`,
|
|
110
|
+
].join('\n');
|
|
111
|
+
}
|
|
112
|
+
/** How many rows of one claim the reviewer is shown and may single out. */
|
|
113
|
+
const MAX_LISTED = 40;
|
|
114
|
+
const asVerdict = (v) => (VERDICTS.includes(v) ? v : 'unproven');
|
|
115
|
+
const text = (v) => String(v ?? '').trim() || undefined;
|
|
116
|
+
/** Read the reviewer's verdict; anything unreadable is "unproven", never a defect. */
|
|
117
|
+
export function parseReview(raw) {
|
|
118
|
+
const start = raw.indexOf('{');
|
|
119
|
+
const end = raw.lastIndexOf('}');
|
|
120
|
+
try {
|
|
121
|
+
const o = JSON.parse(raw.slice(start, end + 1));
|
|
122
|
+
const exceptions = (Array.isArray(o.exceptions) ? o.exceptions : [])
|
|
123
|
+
.map((e) => ({
|
|
124
|
+
tests: (Array.isArray(e?.tests) ? e.tests : []).map(Number).filter((n) => Number.isInteger(n) && n > 0),
|
|
125
|
+
verdict: asVerdict(e?.verdict),
|
|
126
|
+
reason: text(e?.reason) ?? '',
|
|
127
|
+
evidence: text(e?.evidence),
|
|
128
|
+
}))
|
|
129
|
+
.filter((e) => e.tests.length);
|
|
130
|
+
return { verdict: asVerdict(o.verdict), reason: text(o.reason) ?? '', evidence: text(o.evidence), exceptions };
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return { verdict: 'unproven', reason: 'the review could not be read', exceptions: [] };
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/** One review per verdict: the group's own, and one for each set of exceptions. */
|
|
137
|
+
export function splitReview(group, parsed) {
|
|
138
|
+
const taken = new Set();
|
|
139
|
+
const out = [];
|
|
140
|
+
for (const e of parsed.exceptions) {
|
|
141
|
+
const tests = e.tests.filter((n) => n <= group.tests.length && !taken.has(n));
|
|
142
|
+
tests.forEach((n) => taken.add(n));
|
|
143
|
+
if (tests.length) {
|
|
144
|
+
out.push({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
const rest = group.tests.filter((_, i) => !taken.has(i + 1));
|
|
148
|
+
if (rest.length)
|
|
149
|
+
out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
|
|
150
|
+
return out;
|
|
151
|
+
}
|
|
152
|
+
/** Review every claim group, a few at a time. Never throws. */
|
|
153
|
+
export async function reviewDefects(journey, groups, passes) {
|
|
154
|
+
const config = loadConfig();
|
|
155
|
+
const decisions = decisionsSource(config.decisions);
|
|
156
|
+
// A linked list (an internal Notion page) needs the reference tools to be read.
|
|
157
|
+
const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
|
|
158
|
+
const { program, args } = tools(splitCommand(config.aiCommand));
|
|
159
|
+
const out = [];
|
|
160
|
+
const queue = [...groups];
|
|
161
|
+
const worker = async () => {
|
|
162
|
+
for (let g = queue.shift(); g; g = queue.shift()) {
|
|
163
|
+
const files = new Set(g.tests.map((t) => t.file));
|
|
164
|
+
const siblings = passes.filter((p) => files.has(p.file));
|
|
165
|
+
let raw = '';
|
|
166
|
+
try {
|
|
167
|
+
raw = await runWithStdin(program, args, buildDefectReviewPrompt(journey, g, siblings, decisions), config.aiTimeoutMs);
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
// an unanswered review proves nothing
|
|
171
|
+
}
|
|
172
|
+
out.push(...splitReview(g, parseReview(raw)));
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(config.aiConcurrency ?? 4, groups.length)) }, worker));
|
|
176
|
+
return out;
|
|
177
|
+
}
|