@activate-core/test-forge 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -17
- package/dist/cli/index.js +12 -2
- package/dist/core/docs/discover.js +62 -40
- package/dist/core/docs/doc-check.js +257 -0
- package/dist/core/docs/doc-guidance.js +3 -176
- package/dist/core/docs/grounding.js +37 -58
- package/dist/core/docs/reference-text.js +24 -6
- package/dist/core/docs/reference.js +23 -41
- package/dist/core/docs/review.js +17 -40
- package/dist/core/gates/validate.js +78 -0
- package/dist/core/generate/ai-generator.js +138 -87
- package/dist/core/generate/ai-usage.js +9 -1
- package/dist/core/generate/defect-review.js +69 -66
- package/dist/core/generate/doc-diff.js +25 -1
- package/dist/core/generate/harness.js +176 -210
- package/dist/core/generate/journey-memory.js +39 -3
- package/dist/core/generate/run-generate.js +9 -8
- package/dist/core/generate/seed-generator.js +56 -65
- package/dist/core/generate/update-journey.js +3 -1
- package/dist/core/project/init.js +98 -0
- package/dist/core/project/setup.js +15 -20
- package/dist/core/prompts/blocks.js +340 -0
- package/dist/core/run/name-scan.js +14 -6
- package/dist/core/run/run-tests.js +21 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +25 -1
- package/reporters/jest-reporter.cjs +7 -6
- package/reporters/playwright-reporter.cjs +5 -5
|
@@ -19,179 +19,6 @@
|
|
|
19
19
|
* Nothing becomes a "# TODO": what only product can decide is a question,
|
|
20
20
|
* what the test environment needs is test setup.
|
|
21
21
|
*/
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
`describes a journey in a few lines ("the user enters a username and a`,
|
|
26
|
-
`password"); YOUR job is to expand it into every scenario a senior QA`,
|
|
27
|
-
`engineer would test: the happy path, every validation of every field,`,
|
|
28
|
-
`every state of every control, every failure and edge case. For the`,
|
|
29
|
-
`journey, document, where they apply:`,
|
|
30
|
-
`- every distinct sub-flow and lifecycle STATE it can be in, and the`,
|
|
31
|
-
` transitions between them (not just the happy path);`,
|
|
32
|
-
`- each endpoint/screen involved, with real inputs, outputs, status`,
|
|
33
|
-
` codes, and side effects;`,
|
|
34
|
-
`- edge cases, race conditions, idempotency, retries, webhooks and`,
|
|
35
|
-
` signature checks, refunds/reversals, permissions/auth, and any`,
|
|
36
|
-
` money/amount CALCULATIONS — with the actual rules and formulas, not`,
|
|
37
|
-
` vague mentions;`,
|
|
38
|
-
`- concrete acceptance criteria per case (exactly what must hold).`,
|
|
39
|
-
`Aim for the depth of a spec a new engineer could implement from.`,
|
|
40
|
-
``,
|
|
41
|
-
`TIERS. Every acceptance criterion becomes an ORACLE: a test is generated`,
|
|
42
|
-
`from it, and a product that disagrees is reported. So start EVERY`,
|
|
43
|
-
`criterion with its tier, which says where it comes from and how a`,
|
|
44
|
-
`failure is reported:`,
|
|
45
|
-
`- "[spec]" — the source states this behaviour. A failure is a defect`,
|
|
46
|
-
` against the spec.`,
|
|
47
|
-
`- "[baseline]" — universal robustness that holds for ANY well-built app.`,
|
|
48
|
-
` The client describes what to build; it rarely says how it must NOT`,
|
|
49
|
-
` break. That is your job: think like a tester who wants to break it and`,
|
|
50
|
-
` like a user who is technical enough to find its loopholes. Add these`,
|
|
51
|
-
` YOURSELF for every screen, form, list, action and endpoint the journey`,
|
|
52
|
-
` has, even when the source says nothing about them. Write each one's`,
|
|
53
|
-
` LEVEL and CATEGORY right after the tag, e.g. "[baseline] (must,`,
|
|
54
|
-
` security)". Every "must" that fits the journey has to be there.`,
|
|
55
|
-
` MUST — security:`,
|
|
56
|
-
` * One user never reaches another user's data or actions: changing an id`,
|
|
57
|
-
` in a URL, a request or a hidden field is refused (not "not found by`,
|
|
58
|
-
` accident" — refused). A page or endpoint that needs a login is refused`,
|
|
59
|
-
` without one; a role cannot use another role's actions.`,
|
|
60
|
-
` * Whatever the client sends is not trusted: a price, an amount, a`,
|
|
61
|
-
` discount, a role, an owner id or a status changed in the request is`,
|
|
62
|
-
` refused or recomputed by the server.`,
|
|
63
|
-
` * An HTML, script or injection payload in any input is stored and shown`,
|
|
64
|
-
` as text, never executed, never a server error (5xx).`,
|
|
65
|
-
` * Passwords are masked; passwords, tokens and codes never appear in a`,
|
|
66
|
-
` URL, a query string or a log the user can see.`,
|
|
67
|
-
` * Repeated wrong attempts (password, code, coupon, PIN) are limited and`,
|
|
68
|
-
` the limit is told to the user.`,
|
|
69
|
-
` MUST — money and data:`,
|
|
70
|
-
` * Double submission: an action with a side effect (pay, book, order,`,
|
|
71
|
-
` send, create) done`,
|
|
72
|
-
` twice — a double click, a repeated Enter, two tabs, a retry after a`,
|
|
73
|
-
` timeout — leaves exactly one record, one charge, one message.`,
|
|
74
|
-
` * A flow that fails half-way leaves no half-made record and no money`,
|
|
75
|
-
` taken without the thing bought; the user sees what happened.`,
|
|
76
|
-
` * Amounts, totals, discounts and limits are the same on every screen and`,
|
|
77
|
-
` in the server's answer.`,
|
|
78
|
-
` MUST — concurrency:`,
|
|
79
|
-
` * Two users or two tabs acting on the same thing at once (the same slot,`,
|
|
80
|
-
` the last item, a code with limited uses, the same record edited) —`,
|
|
81
|
-
` exactly one wins and the other gets a clear reason.`,
|
|
82
|
-
` * Nothing hangs: every action ends within its timeout with a result or a`,
|
|
83
|
-
` visible error; a request that never answers never leaves a spinner`,
|
|
84
|
-
` forever or a control disabled for good.`,
|
|
85
|
-
` MUST — error boundaries:`,
|
|
86
|
-
` * Every load and every action, when the server answers 500, the network`,
|
|
87
|
-
` drops, it times out, or the answer is empty or malformed: a visible`,
|
|
88
|
-
` message, the page stays usable, no blank page, no crash, the user's`,
|
|
89
|
-
` input is kept.`,
|
|
90
|
-
` * One part of a page failing never takes the whole page down.`,
|
|
91
|
-
` SHOULD — validation:`,
|
|
92
|
-
` * Inputs: empty and whitespace-only values; leading/trailing spaces;`,
|
|
93
|
-
` a very long value (5,000 characters); unicode and emoji. Each is`,
|
|
94
|
-
` accepted or rejected with a visible reason, never a crash.`,
|
|
95
|
-
` * No silent failure: every rejected submit says why, next to the field`,
|
|
96
|
-
` or the form (a wrong password with no message is a defect).`,
|
|
97
|
-
` * The screen and the server accept exactly the same values.`,
|
|
98
|
-
` SHOULD — state and navigation:`,
|
|
99
|
-
` * Refresh, Back, Forward and opening a mid-flow URL directly never crash`,
|
|
100
|
-
` or leave a blank page; an expired session sends the user to log in and`,
|
|
101
|
-
` back; after logging out nothing of the account is shown.`,
|
|
102
|
-
` * Choice controls (dropdown, radio, checkbox, list): every option shown`,
|
|
103
|
-
` can be selected and submitted; long lists page or scroll.`,
|
|
104
|
-
` SHOULD — regression:`,
|
|
105
|
-
` * Every defect the source or the doc already names keeps a test, so it`,
|
|
106
|
-
` cannot quietly come back.`,
|
|
107
|
-
` COULD — accessibility:`,
|
|
108
|
-
` * The form can be filled and submitted with the keyboard alone; Enter`,
|
|
109
|
-
` submits; pop-ups close with Escape; controls have readable names.`,
|
|
110
|
-
` [baseline] is ROBUSTNESS only. Never present product POLICY you`,
|
|
111
|
-
` invented (a maximum length, a password rule, an error wording, a`,
|
|
112
|
-
` limit) as [baseline].`,
|
|
113
|
-
`- "[observed]" — the source is silent, but the code fixes a concrete`,
|
|
114
|
-
` rule: a validation pattern, a length limit, allowed values, the`,
|
|
115
|
-
` options a list offers, a rate limit, a timeout. State the rule with`,
|
|
116
|
-
` its real value, use it to build valid data and boundary cases (just`,
|
|
117
|
-
` inside and just outside), and require every layer to enforce it the`,
|
|
118
|
-
` same way (the screen accepts exactly what the server accepts). A`,
|
|
119
|
-
` failure is an inconsistency, not a spec defect. Never use [observed]`,
|
|
120
|
-
` to override or soften a [spec] criterion.`,
|
|
121
|
-
` CHECK AN [observed] RULE BEFORE YOU STATE IT. Run each pattern in your`,
|
|
122
|
-
` head against the concrete values you list — what it really accepts and`,
|
|
123
|
-
` refuses, not what its name suggests. For the server's verdict, follow`,
|
|
124
|
-
` the request through everything that runs BEFORE validation: global`,
|
|
125
|
-
` middleware that trims, normalises or converts empty strings, and the`,
|
|
126
|
-
` input mutators the handler applies. A row whose verdict is wrong`,
|
|
127
|
-
` becomes a test that fails on a correct product.`,
|
|
128
|
-
` An [observed] criterion is about what the customer sees or what the`,
|
|
129
|
-
` server accepts or refuses — never an implementation detail (a storage`,
|
|
130
|
-
` key name, a CSS class, an internal flag, a request field name, a log`,
|
|
131
|
-
` line, a lock, a cache) unless the source itself is about it.`,
|
|
132
|
-
` Only what a test in THIS area can drive is a criterion. Behaviour no`,
|
|
133
|
-
` screen or endpoint of this area can reach (a backend job, another app,`,
|
|
134
|
-
` an operator's action) is not tagged — list it under "## Not requirements"`,
|
|
135
|
-
` with where it is checked, or mention it in Test setup if the generator`,
|
|
136
|
-
` needs to know.`,
|
|
137
|
-
``,
|
|
138
|
-
`METHOD. Work through these steps for every journey, in order:`,
|
|
139
|
-
`1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
|
|
140
|
-
` interactive element and where it appears, each row's first cell its id`,
|
|
141
|
-
` C1, C2, C3 …: each input, button, link,`,
|
|
142
|
-
` checkbox, code box, dropdown, pop-up, menu item, and the page itself.`,
|
|
143
|
-
` Secondary controls count as much as the main form: Resend, Back,`,
|
|
144
|
-
` show/hide, close, apply, sign-out, provider buttons. Apply every`,
|
|
145
|
-
` [baseline] item that fits to EACH row, not just to the main form.`,
|
|
146
|
-
`2. COMBINATIONS. When the source states a rule across alternatives and`,
|
|
147
|
-
` conditions ("for login and sign-up, also on a slow connection and on`,
|
|
148
|
-
` a retry"), check in the code which of them take a different path. Write`,
|
|
149
|
-
` one criterion per combination that takes its own path; alternatives`,
|
|
150
|
-
` handled by the same code share ONE criterion that names them all. A`,
|
|
151
|
-
` criterion per opener, per page or per input that all reach the same`,
|
|
152
|
-
` code becomes one test per item — volume, not coverage.`,
|
|
153
|
-
`3. CODE BRANCHES. Read the code the journey runs — screens, components,`,
|
|
154
|
-
` client API calls, and the server handlers they reach (related repos`,
|
|
155
|
-
` included) — and walk EVERY validation rule, error response and`,
|
|
156
|
-
` conditional branch: account states (unverified, banned, deleted,`,
|
|
157
|
-
` other roles), limits and expiry, empty and failure states, refresh`,
|
|
158
|
-
` and deep links, what is stored in the browser and when it is`,
|
|
159
|
-
` cleared, input handling (paste, clearing after an error). Each branch`,
|
|
160
|
-
` becomes an [observed] criterion, or a Question when only product can`,
|
|
161
|
-
` say what it should do. A skipped branch is the commonest gap.`,
|
|
162
|
-
`4. QUOTE [spec]. End every [spec] criterion with the source sentence it`,
|
|
163
|
-
` rests on, quoted: (source: "..."). A criterion you cannot quote is`,
|
|
164
|
-
` NOT [spec] — make it [baseline] or [observed], or a Question.`,
|
|
165
|
-
`5. SHARED RULES AND SHARED CONTROLS. A rule or control shared by several`,
|
|
166
|
-
` journeys (a password rule, a code-entry step, the login form, sign-out)`,
|
|
167
|
-
` is specified in full ONCE, in the journey whose subject it is. Other`,
|
|
168
|
-
` journeys that pass through it state only their own part: how the`,
|
|
169
|
-
` customer reaches it from there and the outcome specific to that journey,`,
|
|
170
|
-
` and name the journey that holds the full rule.`,
|
|
171
|
-
`6. COVERAGE. After the numbered sections, add "## Coverage": one row per`,
|
|
172
|
-
` control from step 1, naming the section that covers each [baseline]`,
|
|
173
|
-
` item that fits it. A blank cell is a gap — fill it before finishing.`,
|
|
174
|
-
``,
|
|
175
|
-
`TEST DATA. Tests fill data themselves. Never ask for an exact value:`,
|
|
176
|
-
`valid data comes from the stated or [observed] rules; invalid data`,
|
|
177
|
-
`breaks one rule at a time; a choice control is filled with an option`,
|
|
178
|
-
`the screen actually offers at run time, never a hard-coded one.`,
|
|
179
|
-
``,
|
|
180
|
-
`NO TODOs. Never write a "# TODO" line or leave a placeholder. Instead:`,
|
|
181
|
-
`- What the test environment must provide — accounts and records to`,
|
|
182
|
-
` seed, settings or flags that must be on, external services to stub`,
|
|
183
|
-
` (SMS/OTP, payments, OAuth, email) and the endpoint or SDK to stub,`,
|
|
184
|
-
` browser dialogs to handle — goes in a "## Test setup" section, written`,
|
|
185
|
-
` as instructions for the test generator.`,
|
|
186
|
-
`- What only product can decide, and neither the source nor the code`,
|
|
187
|
-
` fixes, goes in a "## Questions for product (not asserted)" section.`,
|
|
188
|
-
` Keep it short: a question is a last resort, after [spec], [baseline]`,
|
|
189
|
-
` and [observed] have been used.`,
|
|
190
|
-
``,
|
|
191
|
-
`Organise the body with numbered "## N. <Section>" headings, ONE`,
|
|
192
|
-
`concern per section, because each section later becomes its own`,
|
|
193
|
-
`focused test file — so the richer and better-separated the sections,`,
|
|
194
|
-
`the better the test suite. "## Controls" comes first; "## Coverage",`,
|
|
195
|
-
`"## Test setup" and "## Questions for product (not asserted)" come`,
|
|
196
|
-
`last. None of these four is numbered.`,
|
|
197
|
-
].join('\n');
|
|
22
|
+
import { BASELINE, DOC_GUIDANCE, TIERS } from '../prompts/blocks.js';
|
|
23
|
+
/** The tiers, the baseline checklist and the doc format, for every prompt that writes a doc. */
|
|
24
|
+
export const DEEP_DOC_GUIDANCE = [TIERS, BASELINE, DOC_GUIDANCE].join('\n\n');
|
|
@@ -45,78 +45,57 @@ function groundingPath(journey) {
|
|
|
45
45
|
function buildGroundingPrompt(journey, map, srcDir, excerpts = '') {
|
|
46
46
|
const isUi = journey.type !== 'api';
|
|
47
47
|
return [
|
|
48
|
-
`You are a senior engineer
|
|
49
|
-
`
|
|
50
|
-
`
|
|
48
|
+
`You are a senior engineer writing the reference sheet for one user journey,`,
|
|
49
|
+
`so each test writer gets the mechanics without re-reading the source. A fact`,
|
|
50
|
+
`invented here is copied into every test file of the journey. Report only what`,
|
|
51
|
+
`you actually read, and write "unknown" for anything you could not find.`,
|
|
51
52
|
``,
|
|
52
53
|
...(excerpts
|
|
53
54
|
? [
|
|
54
|
-
`
|
|
55
|
-
`
|
|
56
|
-
`SOURCE EXCERPTS below, with line numbers. You have no tools: work`,
|
|
57
|
-
`from those excerpts only.`,
|
|
55
|
+
`You have no tools. Work only from <source_excerpts>: the files the journey`,
|
|
56
|
+
`cites, with line numbers.`,
|
|
58
57
|
]
|
|
59
58
|
: [
|
|
60
|
-
`You are inside the
|
|
61
|
-
`
|
|
62
|
-
`
|
|
63
|
-
`controller/handler, the service it calls, and the models it touches.`,
|
|
59
|
+
`You are inside the repository with read-only tools (Read, Grep, Glob, LS).`,
|
|
60
|
+
`Under "${srcDir}", start from the route the journey names, then its handler,`,
|
|
61
|
+
`the service it calls and the models it touches.`,
|
|
64
62
|
]),
|
|
65
63
|
``,
|
|
66
|
-
`
|
|
67
|
-
`
|
|
68
|
-
`
|
|
69
|
-
`
|
|
70
|
-
`it
|
|
71
|
-
|
|
72
|
-
`
|
|
73
|
-
`
|
|
74
|
-
`
|
|
75
|
-
`
|
|
76
|
-
`
|
|
77
|
-
`
|
|
78
|
-
`
|
|
79
|
-
`
|
|
80
|
-
`
|
|
81
|
-
`
|
|
82
|
-
` matter to this journey (including enum values where they exist).`,
|
|
83
|
-
` For each table a test must SEED, record two things exactly as the`,
|
|
84
|
-
` schema declares them, never as the ORM or another table suggests:`,
|
|
85
|
-
` * every column that is NOT NULL and has NO default — a seed that`,
|
|
86
|
-
` omits one is rejected outright, and the row never exists;`,
|
|
87
|
-
` * the TYPE of the primary key and of every foreign key a test`,
|
|
88
|
-
` will supply or query by. Do not assume one id type covers the`,
|
|
89
|
-
` schema: a users table keyed by uuid sits happily beside tables`,
|
|
90
|
-
` keyed by integer or bigint, and a uuid handed to one of those`,
|
|
91
|
-
` is refused by the database, not merely unmatched.`,
|
|
92
|
-
` Read this from the migrations, the schema dump, or the models'`,
|
|
93
|
-
` declarations — whichever the project actually keeps.`,
|
|
94
|
-
` EXTERNALS — any third-party client the code calls, the module name`,
|
|
95
|
-
` a test would have to stand in for, and the env var its base URL or`,
|
|
96
|
-
` key comes from.`,
|
|
64
|
+
`Write compact plain text under these headings:`,
|
|
65
|
+
`ROUTES: every endpoint the journey uses, with its HTTP method and full`,
|
|
66
|
+
` mounted path, including any version or area prefix and the env var the`,
|
|
67
|
+
` prefix comes from.`,
|
|
68
|
+
`AUTH: for each route, whether it is declared authenticated; whether a bearer`,
|
|
69
|
+
` token still populates the request user through a pre-auth hook; and the`,
|
|
70
|
+
` field that ownership is scoped on.`,
|
|
71
|
+
`SHAPES: for each route, the request payload, the response body as its`,
|
|
72
|
+
` serializer builds it, and every status code the handler can return.`,
|
|
73
|
+
`DATA: the tables or collections read or written, the fields that matter, and`,
|
|
74
|
+
` their enum values. For each table a test must seed, copy from the`,
|
|
75
|
+
` migrations, schema dump or model declarations: every NOT NULL column without`,
|
|
76
|
+
` a default, and the type of the primary key and of every foreign key a test`,
|
|
77
|
+
` will use. Never assume one id type for the whole schema.`,
|
|
78
|
+
`EXTERNALS: each third-party client called, the exact module a test must mock,`,
|
|
79
|
+
` and the env var for its base URL or key.`,
|
|
97
80
|
...(isUi
|
|
98
81
|
? [
|
|
99
|
-
`
|
|
100
|
-
`
|
|
101
|
-
`
|
|
102
|
-
`
|
|
103
|
-
`
|
|
104
|
-
`
|
|
105
|
-
`
|
|
106
|
-
` 'FAQs' (role link)". Each test file is given only the lines for its`,
|
|
107
|
-
` own controls, so a line without its ids reaches every file and a`,
|
|
108
|
-
` line with the wrong ids reaches none.`,
|
|
82
|
+
`SCREENS: the page and component files rendered, and the locators they really`,
|
|
83
|
+
` expose: role and accessible name, label or placeholder text, visible text,`,
|
|
84
|
+
` input name or id, and a data-testid only if it is in the source. Start each`,
|
|
85
|
+
` line with the Controls ids it serves:`,
|
|
86
|
+
` [C3,C4] header/index.tsx: link 'FAQs' (role link)`,
|
|
87
|
+
` Each test file receives only the lines for its own controls, so a line with`,
|
|
88
|
+
` missing or wrong ids is hidden from the files that need it.`,
|
|
109
89
|
]
|
|
110
90
|
: []),
|
|
111
91
|
``,
|
|
112
|
-
`
|
|
113
|
-
`restating the journey, no advice about testing. Aim for well under 200`,
|
|
114
|
-
`lines. Output the headings and their contents only.`,
|
|
92
|
+
`This is a terse reference sheet: no preamble, no advice, under 200 lines.`,
|
|
115
93
|
``,
|
|
116
|
-
...(excerpts ? [] : [
|
|
117
|
-
|
|
94
|
+
...(excerpts ? [] : [`<project_map>`, map, `</project_map>`, ``]),
|
|
95
|
+
`<journey>`,
|
|
118
96
|
...(excerpts ? [compactJourney(journey.rawMarkdown || journey.name)] : [journey.rawMarkdown || journey.name]),
|
|
119
|
-
|
|
97
|
+
`</journey>`,
|
|
98
|
+
...(excerpts ? [`<source_excerpts>`, excerpts, `</source_excerpts>`] : []),
|
|
120
99
|
].join('\n');
|
|
121
100
|
}
|
|
122
101
|
/** The journey as its controls and one line per rule: what the facts must serve, without the prose. */
|
|
@@ -22,18 +22,36 @@ export function readReference(reference, refresh = false) {
|
|
|
22
22
|
if (!refresh && existsSync(path))
|
|
23
23
|
return readFileSync(path, 'utf-8');
|
|
24
24
|
const { program, args } = withReferenceTools(splitCommand(loadConfig().aiCommand));
|
|
25
|
-
const
|
|
25
|
+
const ask = (after) => runAiSync(program, args, {
|
|
26
26
|
input: [
|
|
27
|
-
|
|
27
|
+
after
|
|
28
|
+
? `Continue printing this reference as Markdown, word for word, from the section that comes after the heading "${after}":`
|
|
29
|
+
: `Fetch this reference and print its full content as Markdown, word for word:`,
|
|
28
30
|
` ${reference}`,
|
|
29
|
-
`It may be a Notion page, a URL or a document. Keep every heading, list
|
|
30
|
-
`
|
|
31
|
-
`
|
|
32
|
-
|
|
31
|
+
`It may be a Notion page, a URL or a document. Keep every heading, list, table`,
|
|
32
|
+
`and sentence; summarise nothing and add nothing. Include sub-pages only when`,
|
|
33
|
+
`the page's own text is just a list of links to them. Print only the content.`,
|
|
34
|
+
``,
|
|
35
|
+
`If you cannot read it, print one line starting "ERROR:".`,
|
|
36
|
+
`If it is too long to print in one answer, stop at the end of a heading's`,
|
|
37
|
+
`section and print a last line "CONTINUE AFTER: <that heading>". You will be`,
|
|
38
|
+
`asked for the rest.`,
|
|
33
39
|
].join('\n'),
|
|
34
40
|
encoding: 'utf-8',
|
|
35
41
|
maxBuffer: 10 * 1024 * 1024,
|
|
36
42
|
}).trim();
|
|
43
|
+
// A long page comes in parts; each part says where the next one starts.
|
|
44
|
+
const parts = [];
|
|
45
|
+
let after;
|
|
46
|
+
for (let k = 0; k < 12; k++) {
|
|
47
|
+
const part = ask(after);
|
|
48
|
+
const cont = /\n?CONTINUE AFTER:\s*(.+)\s*$/.exec(part);
|
|
49
|
+
parts.push(cont ? part.slice(0, cont.index).trimEnd() : part);
|
|
50
|
+
if (!cont || cont[1].trim() === after)
|
|
51
|
+
break;
|
|
52
|
+
after = cont[1].trim();
|
|
53
|
+
}
|
|
54
|
+
const text = parts.join('\n\n').trim();
|
|
37
55
|
if (!text || /^ERROR:/m.test(text.slice(0, 300)))
|
|
38
56
|
throw new Error(`Could not read the reference "${reference}":\n${text.slice(0, 500)}`);
|
|
39
57
|
mkdirSync(REFERENCE_DIR, { recursive: true });
|
|
@@ -161,18 +161,11 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
|
|
|
161
161
|
`Each file starts with this header (required):`,
|
|
162
162
|
``,
|
|
163
163
|
`# Journey: <Name>`,
|
|
164
|
-
``,
|
|
165
164
|
`Type: web (use "web" for a page/UI, "api" for an endpoint)`,
|
|
166
165
|
`Route: <path> (the URL path or endpoint, e.g. /login)`,
|
|
166
|
+
`Coverage: functional, security (add others that apply: payment, session, …)`,
|
|
167
167
|
``,
|
|
168
|
-
`
|
|
169
|
-
`- functional`,
|
|
170
|
-
`- security (add more if relevant: performance, accessibility, ...)`,
|
|
171
|
-
``,
|
|
172
|
-
`Then whichever of these sections apply: Fields, Actions, Expected,`,
|
|
173
|
-
`Negative Cases, Race Conditions, Webhooks, Security Notes, Data, and`,
|
|
174
|
-
`last "Test setup" and "Questions for product (not asserted)". For the`,
|
|
175
|
-
`Data section use the map's data layer.`,
|
|
168
|
+
`Then the sections and criterion lines exactly as FORMAT above describes.`,
|
|
176
169
|
``,
|
|
177
170
|
`Rules:`,
|
|
178
171
|
`- Tag every criterion [spec], [baseline] or [observed] as described`,
|
|
@@ -187,8 +180,8 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
|
|
|
187
180
|
` the code does>" under it. Never turn it into a question.`,
|
|
188
181
|
`- Base technical detail on real code; do not invent it with false`,
|
|
189
182
|
` confidence.`,
|
|
190
|
-
`- No "# TODO" lines and no "Suggested" section: use "
|
|
191
|
-
` "Questions for product (not asserted)" as described above.`,
|
|
183
|
+
`- No "# TODO" lines and no "Suggested" section: use "Not requirements",`,
|
|
184
|
+
` "Test setup" and "Questions for product (not asserted)" as described above.`,
|
|
192
185
|
`- Never include secrets — refer to env var names only.`,
|
|
193
186
|
`- Create the area subfolders as needed. Write NOTHING at the root of`,
|
|
194
187
|
` "${docsDir}" and no file that is not a journey doc — a shared note`,
|
|
@@ -224,46 +217,35 @@ export function draftDocsFromReference(srcDir, docsDir, reference, journey, exis
|
|
|
224
217
|
*/
|
|
225
218
|
export function buildJourneyDraftPrompt(journey, section, excerpts, areas, otherDocs) {
|
|
226
219
|
return [
|
|
227
|
-
`You are a senior QA engineer writing
|
|
228
|
-
`
|
|
229
|
-
`for how the product is EXPECTED to behave; where the code does something`,
|
|
230
|
-
`else, document the reference (the code may be buggy).`,
|
|
220
|
+
`You are a senior QA engineer writing the journey doc for "${journey}".`,
|
|
221
|
+
`Tests are generated from this doc, so they can only be as good as it is.`,
|
|
231
222
|
``,
|
|
232
|
-
|
|
223
|
+
`<reference> is the source of truth for expected behaviour. <source_excerpts>`,
|
|
224
|
+
`is this project's code. Use it only for technical detail (routes, field names,`,
|
|
225
|
+
`labels the screen shows) and for [observed] rules. Where the code differs from`,
|
|
226
|
+
`the reference, document the reference. You have no tools: anything the`,
|
|
227
|
+
`excerpts do not show, you do not state.`,
|
|
233
228
|
``,
|
|
234
|
-
|
|
235
|
-
`for this journey (with the page's shared rules), and SOURCE EXCERPTS, the`,
|
|
236
|
-
`lines of this project's code the journey's texts and files point at. Use the`,
|
|
237
|
-
`excerpts only for TECHNICAL detail (routes, field names, labels the screen`,
|
|
238
|
-
`really shows) and for [observed] rules; [spec] comes only from the reference.`,
|
|
239
|
-
`Where the excerpts do not show something, do not invent it.`,
|
|
229
|
+
DEEP_DOC_GUIDANCE,
|
|
240
230
|
``,
|
|
241
231
|
...(otherDocs.length
|
|
242
|
-
? [`
|
|
232
|
+
? [`Rules owned by other journeys are named here, not repeated:`, ...otherDocs.map((d) => ` - ${d}`), ``]
|
|
243
233
|
: []),
|
|
244
|
-
`
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
`Type: web
|
|
234
|
+
`Output: first one line "AREA: <one of ${areas.join(', ')}>", then the doc, starting`,
|
|
235
|
+
`with this header:`,
|
|
236
|
+
`# Journey: <name>`,
|
|
237
|
+
`Type: <web for a page, api for an endpoint>`,
|
|
248
238
|
`Route: <path>`,
|
|
239
|
+
`Coverage: functional, security`,
|
|
240
|
+
`Write nothing before or after.`,
|
|
249
241
|
``,
|
|
250
|
-
|
|
251
|
-
`- functional`,
|
|
252
|
-
`- security`,
|
|
253
|
-
``,
|
|
254
|
-
`Rules: tag every criterion as described above; [spec] only where the`,
|
|
255
|
-
`reference states it, with its quote. When the code seems to behave`,
|
|
256
|
-
`differently from a [spec] criterion, keep the criterion and add a line`,
|
|
257
|
-
`"Likely defect: <what the code does>" under it. No "# TODO", no secrets.`,
|
|
258
|
-
``,
|
|
259
|
-
`OUTPUT: first one line "AREA: <one of ${areas.join(', ')}>", then the doc`,
|
|
260
|
-
`itself, starting with "# Journey:". Nothing before or after.`,
|
|
261
|
-
``,
|
|
262
|
-
`--- REFERENCE ---`,
|
|
242
|
+
`<reference>`,
|
|
263
243
|
section,
|
|
244
|
+
`</reference>`,
|
|
264
245
|
``,
|
|
265
|
-
|
|
246
|
+
`<source_excerpts>`,
|
|
266
247
|
excerpts || '(no source the reference points at was found: the feature may not be built yet)',
|
|
248
|
+
`</source_excerpts>`,
|
|
267
249
|
].join('\n');
|
|
268
250
|
}
|
|
269
251
|
/** One no-tools call that answers with a doc; returns its area and markdown. */
|
package/dist/core/docs/review.js
CHANGED
|
@@ -17,6 +17,7 @@ import { runWithStdin } from '../generate/ai-generator.js';
|
|
|
17
17
|
import { HAS_SOURCE } from '../generate/defect-review.js';
|
|
18
18
|
import { pool } from '../generate/run-generate.js';
|
|
19
19
|
import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
|
|
20
|
+
import { withCoverage } from './doc-check.js';
|
|
20
21
|
/** Build the prompt that reviews and completes ONE journey doc. */
|
|
21
22
|
export function buildReviewPrompt(docPath, srcDir, reference) {
|
|
22
23
|
const source = reference
|
|
@@ -52,53 +53,16 @@ export function buildReviewPrompt(docPath, srcDir, reference) {
|
|
|
52
53
|
` it into a "Likely defect" note or a Question.`,
|
|
53
54
|
`- Shared rules: read the other docs in the same folder and make shared`,
|
|
54
55
|
` cases (password, redirects, payloads, formats) identical.`,
|
|
55
|
-
`- Coverage:
|
|
56
|
+
`- Coverage: every control is named by at least one criterion ([C<ids>]);`,
|
|
57
|
+
` TestForge builds the Coverage table from those, so do not write one.`,
|
|
56
58
|
``,
|
|
57
59
|
`Rules: edit "${docPath}" in place and touch no other file. Keep every`,
|
|
58
60
|
`sound criterion. Keep the header, the section numbering style and the`,
|
|
59
|
-
`last sections (
|
|
61
|
+
`last sections (Not requirements, Test setup, Questions for product). Never`,
|
|
60
62
|
`write "# TODO". When done, print ONE line: what you added or fixed,`,
|
|
61
63
|
`with counts.`,
|
|
62
64
|
].join('\n');
|
|
63
65
|
}
|
|
64
|
-
/**
|
|
65
|
-
* The same review in one call with no tools: the reference's part, the source
|
|
66
|
-
* excerpts and the draft are in the prompt, and the answer is the whole
|
|
67
|
-
* revised doc.
|
|
68
|
-
*/
|
|
69
|
-
export function buildLeanReviewPrompt(doc, section, excerpts, otherDocs) {
|
|
70
|
-
return [
|
|
71
|
-
`You are a senior QA engineer reviewing ONE journey doc that a first pass`,
|
|
72
|
-
`drafted. Make it COMPLETE, so no scenario a careful QA engineer, or a user`,
|
|
73
|
-
`trying to break it, would test is missing. You have no tools: the`,
|
|
74
|
-
`reference's part for this journey, the source excerpts it points at and the`,
|
|
75
|
-
`draft are below. [spec] comes only from the reference; the excerpts give`,
|
|
76
|
-
`technical detail and [observed] rules.`,
|
|
77
|
-
``,
|
|
78
|
-
`The doc must follow this guidance:`,
|
|
79
|
-
DEEP_DOC_GUIDANCE,
|
|
80
|
-
``,
|
|
81
|
-
`Check it against the METHOD above and fix what you find: every control in`,
|
|
82
|
-
`the table; every [baseline] item that fits each control, with its level`,
|
|
83
|
-
`and category, and every MUST that fits; every combination the reference`,
|
|
84
|
-
`states; every branch the excerpts show, as [observed] or a Question; every`,
|
|
85
|
-
`[spec] quoting a real reference sentence (re-tag any that does not);`,
|
|
86
|
-
`the Coverage table with no blank cell. Keep every sound criterion, the`,
|
|
87
|
-
`header and the section numbering. Never write "# TODO".`,
|
|
88
|
-
...(otherDocs.length ? [`Other journeys' docs (shared rules stay with them):`, ...otherDocs.map((d) => ` - ${d}`)] : []),
|
|
89
|
-
``,
|
|
90
|
-
`OUTPUT: the complete revised doc, starting with "# Journey:". Nothing else.`,
|
|
91
|
-
``,
|
|
92
|
-
`--- REFERENCE ---`,
|
|
93
|
-
section,
|
|
94
|
-
``,
|
|
95
|
-
`--- SOURCE EXCERPTS ---`,
|
|
96
|
-
excerpts || '(none found)',
|
|
97
|
-
``,
|
|
98
|
-
`--- THE DRAFT ---`,
|
|
99
|
-
doc,
|
|
100
|
-
].join('\n');
|
|
101
|
-
}
|
|
102
66
|
/**
|
|
103
67
|
* A [spec] line is a requirement the client wrote, so it must carry the
|
|
104
68
|
* client's words: `(source: "…")`, or `(source as above)` pointing at the quote before it. A [spec] line without that quote is the
|
|
@@ -174,4 +138,17 @@ export async function reviewDocs(docsDir, docs, srcDir, reference) {
|
|
|
174
138
|
}
|
|
175
139
|
});
|
|
176
140
|
enforceSpecSourcesOnDisk(docsDir, docs);
|
|
141
|
+
// The Coverage table is built from the doc, never trusted from the model.
|
|
142
|
+
for (const doc of docs) {
|
|
143
|
+
const path = join(docsDir, doc);
|
|
144
|
+
try {
|
|
145
|
+
const text = readFileSync(path, 'utf-8');
|
|
146
|
+
const next = withCoverage(text);
|
|
147
|
+
if (next !== text)
|
|
148
|
+
writeFileSync(path, next, 'utf-8');
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
// a doc that cannot be read was reported above
|
|
152
|
+
}
|
|
153
|
+
}
|
|
177
154
|
}
|
|
@@ -184,6 +184,15 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
184
184
|
if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
|
|
185
185
|
return 'the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out';
|
|
186
186
|
}
|
|
187
|
+
// F24 — one comment policy: the header, an oracle line above each test,
|
|
188
|
+
// cannot-test lines, a timeout reason, and a short note on a parametrised
|
|
189
|
+
// row. Anything else is the writer narrating, and it goes back once.
|
|
190
|
+
if (ctx.forbidPlaceholders) {
|
|
191
|
+
const stray = strayComment(content);
|
|
192
|
+
if (stray) {
|
|
193
|
+
return `comment not allowed (line ${stray.line}): "${stray.text.slice(0, 60)}" — the only comments are the // @generated header, one // oracle: line above each test, // cannot-test: lines, a // timeout: reason, and a one-line note on a parametrised row`;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
187
196
|
// F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
|
|
188
197
|
// variable, seeded data). Skipping because an element is not on the page
|
|
189
198
|
// hides exactly what a test written from the doc exists to report: the
|
|
@@ -681,3 +690,72 @@ export function testsByTier(content) {
|
|
|
681
690
|
out[c.tier] += 1;
|
|
682
691
|
return out;
|
|
683
692
|
}
|
|
693
|
+
/** Comments TestForge itself, the runner or the language writes, and the ones the writer is allowed. */
|
|
694
|
+
const ALLOWED_COMMENT = /^\/\/\s*(?:@generated\b|oracle:|cannot-test:|timeout:|testforge-|eslint-|@ts-|prettier-ignore|=====\s*FILE:)|^\/\/\/\s*<reference|^\/\*\s*eslint/i;
|
|
695
|
+
/**
|
|
696
|
+
* The first full-line comment that is not allowed, with its line number. A
|
|
697
|
+
* comment just above a row of a parametrised table (the next line opens a
|
|
698
|
+
* row: {, [, a string or a number) is that row's note and is allowed.
|
|
699
|
+
* Comments after code on the same line are not looked at.
|
|
700
|
+
*/
|
|
701
|
+
export function strayComment(content) {
|
|
702
|
+
const lines = content.split('\n');
|
|
703
|
+
let inBlock = false;
|
|
704
|
+
let inTemplate = false;
|
|
705
|
+
for (let i = 0; i < lines.length; i++) {
|
|
706
|
+
const t = lines[i].trim();
|
|
707
|
+
// A multi-line template literal holds text, not comments.
|
|
708
|
+
const ticks = (lines[i].replace(/\\`/g, '').match(/`/g) ?? []).length;
|
|
709
|
+
if (inTemplate) {
|
|
710
|
+
if (ticks % 2 === 1)
|
|
711
|
+
inTemplate = false;
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
if (inBlock) {
|
|
715
|
+
if (t.includes('*/'))
|
|
716
|
+
inBlock = false;
|
|
717
|
+
continue;
|
|
718
|
+
}
|
|
719
|
+
if (t.startsWith('/*')) {
|
|
720
|
+
if (!ALLOWED_COMMENT.test(t))
|
|
721
|
+
return { line: i + 1, text: t };
|
|
722
|
+
if (!t.includes('*/'))
|
|
723
|
+
inBlock = true;
|
|
724
|
+
continue;
|
|
725
|
+
}
|
|
726
|
+
if (t.startsWith('//')) {
|
|
727
|
+
if (ALLOWED_COMMENT.test(t))
|
|
728
|
+
continue;
|
|
729
|
+
const next = lines.slice(i + 1).find((l) => l.trim() && !l.trim().startsWith('//'))?.trim() ?? '';
|
|
730
|
+
if (/^[{[\'"`\d-]/.test(next) && !/^[{[]\s*$/.test(next) && t.length <= 120)
|
|
731
|
+
continue;
|
|
732
|
+
return { line: i + 1, text: t };
|
|
733
|
+
}
|
|
734
|
+
if (ticks % 2 === 1)
|
|
735
|
+
inTemplate = true;
|
|
736
|
+
}
|
|
737
|
+
return null;
|
|
738
|
+
}
|
|
739
|
+
/**
|
|
740
|
+
* The writer's answer without the comments the policy does not allow, so a
|
|
741
|
+
* narrating header costs nothing instead of a second call. Only whole comment
|
|
742
|
+
* lines go; code is never touched.
|
|
743
|
+
*/
|
|
744
|
+
export function stripStrayComments(content) {
|
|
745
|
+
let out = content;
|
|
746
|
+
for (let guard = 0; guard < 500; guard++) {
|
|
747
|
+
const stray = strayComment(out);
|
|
748
|
+
if (!stray)
|
|
749
|
+
break;
|
|
750
|
+
const lines = out.split('\n');
|
|
751
|
+
const i = stray.line - 1;
|
|
752
|
+
let end = i + 1;
|
|
753
|
+
if (lines[i].trim().startsWith('/*') && !lines[i].includes('*/')) {
|
|
754
|
+
while (end < lines.length && !lines[end - 1].includes('*/'))
|
|
755
|
+
end += 1;
|
|
756
|
+
}
|
|
757
|
+
lines.splice(i, end - i);
|
|
758
|
+
out = lines.join('\n');
|
|
759
|
+
}
|
|
760
|
+
return out.replace(/\n{3,}/g, '\n\n');
|
|
761
|
+
}
|