@activate-core/test-forge 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -17
- package/dist/cli/index.js +12 -2
- package/dist/core/docs/discover.js +62 -40
- package/dist/core/docs/doc-check.js +257 -0
- package/dist/core/docs/doc-guidance.js +3 -176
- package/dist/core/docs/grounding.js +37 -58
- package/dist/core/docs/reference-text.js +24 -6
- package/dist/core/docs/reference.js +23 -41
- package/dist/core/docs/review.js +17 -40
- package/dist/core/gates/validate.js +78 -0
- package/dist/core/generate/ai-generator.js +138 -87
- package/dist/core/generate/ai-usage.js +9 -1
- package/dist/core/generate/defect-review.js +69 -66
- package/dist/core/generate/doc-diff.js +25 -1
- package/dist/core/generate/harness.js +176 -210
- package/dist/core/generate/journey-memory.js +39 -3
- package/dist/core/generate/run-generate.js +9 -8
- package/dist/core/generate/seed-generator.js +56 -65
- package/dist/core/generate/update-journey.js +3 -1
- package/dist/core/project/init.js +98 -0
- package/dist/core/project/setup.js +15 -20
- package/dist/core/prompts/blocks.js +340 -0
- package/dist/core/run/name-scan.js +14 -6
- package/dist/core/run/run-tests.js +21 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +25 -1
- package/reporters/jest-reporter.cjs +7 -6
- package/reporters/playwright-reporter.cjs +5 -5
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — the guidance every prompt shares, written once.
|
|
3
|
+
*
|
|
4
|
+
* The journey draft and the doc check both teach the same tiers, baseline
|
|
5
|
+
* checklist and doc format; the web and API test writers share the same
|
|
6
|
+
* rules of evidence. Each block lives here and is pulled into the prompts
|
|
7
|
+
* that need it, so a rule is changed in one place and no prompt carries two
|
|
8
|
+
* copies of it.
|
|
9
|
+
*/
|
|
10
|
+
/** Where a criterion comes from, and how a failure against it is reported. */
|
|
11
|
+
export const TIERS = `TIERS. Every acceptance criterion is an oracle: a test is generated from it,
|
|
12
|
+
and a product that disagrees is reported. Each criterion starts with its tier.
|
|
13
|
+
|
|
14
|
+
[spec] The reference states this behaviour. End the line with the sentence it
|
|
15
|
+
rests on: (source: "…"). If you cannot quote it, it is not [spec].
|
|
16
|
+
A failure is a defect.
|
|
17
|
+
|
|
18
|
+
[baseline] Robustness any well-built app has, added by you even when the
|
|
19
|
+
reference is silent (see the checklist). Put level and category after the tag:
|
|
20
|
+
[baseline] (must, security). Robustness only: never present a policy you made
|
|
21
|
+
up (a length limit, a password rule, an error wording) as [baseline].
|
|
22
|
+
A failure is a finding.
|
|
23
|
+
|
|
24
|
+
[observed] The reference is silent, but the code fixes a concrete rule: a
|
|
25
|
+
pattern, a length limit, allowed values, list options, a rate limit, a timeout.
|
|
26
|
+
State the real value, build boundary cases just inside and just outside, and
|
|
27
|
+
require the screen and the server to agree. Before you state it, run the
|
|
28
|
+
pattern against the values you list, and follow the request through anything
|
|
29
|
+
that runs before validation (trimming middleware, empty-to-null conversion,
|
|
30
|
+
input mutators). Describe what the customer sees or what the server accepts,
|
|
31
|
+
never a storage key, CSS class, internal flag or log line. Never use [observed]
|
|
32
|
+
to soften a [spec] rule. A failure is a finding.
|
|
33
|
+
|
|
34
|
+
Only behaviour a test in this area can drive is a criterion. Anything else (a
|
|
35
|
+
background job, another app, an operator action) goes under
|
|
36
|
+
"## Not requirements", with where it is checked.`;
|
|
37
|
+
/** The robustness every app owes, applied only to the controls it fits. */
|
|
38
|
+
export const BASELINE = `BASELINE CHECKLIST. Apply each item only to the controls it fits: a link gets
|
|
39
|
+
no injection tests, and a read-only page gets no double-submit tests.
|
|
40
|
+
|
|
41
|
+
MUST, security
|
|
42
|
+
- Another user's data or actions are refused when an id in a URL, request or
|
|
43
|
+
hidden field is changed. Pages and endpoints that need a login refuse without
|
|
44
|
+
one; a role cannot use another role's actions.
|
|
45
|
+
- A price, amount, discount, role, owner id or status sent by the client is
|
|
46
|
+
refused or recomputed by the server.
|
|
47
|
+
- HTML, script or injection payloads in inputs are stored and shown as text:
|
|
48
|
+
never executed, never a 5xx.
|
|
49
|
+
- Passwords are masked; passwords, tokens and codes never appear in a URL, a
|
|
50
|
+
query string or a log the user can see.
|
|
51
|
+
- Repeated wrong attempts (password, code, coupon, PIN) are limited, and the
|
|
52
|
+
user is told.
|
|
53
|
+
MUST, money and data (actions with side effects only)
|
|
54
|
+
- A double submit (double click, repeated Enter, two tabs, retry after a
|
|
55
|
+
timeout) leaves exactly one record, charge or message.
|
|
56
|
+
- A flow that fails halfway leaves no half-made record and takes no money; the
|
|
57
|
+
user sees what happened.
|
|
58
|
+
- Amounts, totals, discounts and limits match on every screen and in the
|
|
59
|
+
server response.
|
|
60
|
+
MUST, concurrency (shared or limited resources only)
|
|
61
|
+
- Two users or tabs acting on the same thing: exactly one wins, and the other
|
|
62
|
+
gets a clear reason.
|
|
63
|
+
- Nothing hangs: every action ends with a result or a visible error, with no
|
|
64
|
+
endless spinner and no control left disabled.
|
|
65
|
+
MUST, error handling
|
|
66
|
+
- On a 500, a dropped network, a timeout, or an empty or malformed response:
|
|
67
|
+
a visible message, the page stays usable, and the user's input is kept.
|
|
68
|
+
- One failing part of a page never takes the whole page down.
|
|
69
|
+
SHOULD, validation (inputs only)
|
|
70
|
+
- Empty, whitespace-only, leading or trailing spaces, 5,000 characters, unicode
|
|
71
|
+
and emoji: accepted, or rejected with a visible reason.
|
|
72
|
+
- Every rejected submit says why, next to the field or the form.
|
|
73
|
+
- The screen and the server accept exactly the same values.
|
|
74
|
+
SHOULD, state and navigation
|
|
75
|
+
- Refresh, Back, Forward and opening a mid-flow URL directly never crash or
|
|
76
|
+
blank the page. An expired session goes to login and back. After logout,
|
|
77
|
+
nothing of the account shows.
|
|
78
|
+
- Every option a choice control shows can be selected and submitted.
|
|
79
|
+
SHOULD, regression
|
|
80
|
+
- Every defect the reference or the doc names keeps a test.
|
|
81
|
+
COULD, accessibility
|
|
82
|
+
- The form can be filled and submitted with the keyboard alone; Enter submits;
|
|
83
|
+
Escape closes pop-ups; controls have readable names.`;
|
|
84
|
+
/** How deep a journey doc goes, its exact format, and the method that fills it. */
|
|
85
|
+
export const DOC_GUIDANCE = `DEPTH FOLLOWS RISK. A journey that takes money, changes accounts or handles
|
|
86
|
+
auth gets full detail: states and transitions, every endpoint with its inputs,
|
|
87
|
+
outputs and status codes, idempotency, retries, webhooks, refunds, and
|
|
88
|
+
calculations with their real formulas. A simple page (a link, a static view)
|
|
89
|
+
gets its controls, the baseline items that fit them and its [observed] rules,
|
|
90
|
+
and nothing more. Never invent a formula, limit or state that the reference and
|
|
91
|
+
the code do not show.
|
|
92
|
+
|
|
93
|
+
FORMAT
|
|
94
|
+
- "## Controls" comes first: a table with one row per interactive element.
|
|
95
|
+
That means inputs, buttons, links, checkboxes, dropdowns, pop-ups, menu items
|
|
96
|
+
and the page itself. Secondary controls count too: Resend, Back, show/hide,
|
|
97
|
+
close, sign-out, provider buttons. The first column is the id: C1, C2, …
|
|
98
|
+
An api journey has no screen: its rows are its endpoints (method and path).
|
|
99
|
+
- Then numbered "## N. <Concern>" sections, one concern each. Each section
|
|
100
|
+
becomes one test file.
|
|
101
|
+
- Every criterion is one line in exactly this form:
|
|
102
|
+
- R<n> [tier] (level, category) [C<ids>] <what must hold> (source: "…")
|
|
103
|
+
Level and category appear only for [baseline]; the source appears only for
|
|
104
|
+
[spec]. [C<ids>] lists the controls the criterion covers; an api criterion
|
|
105
|
+
may leave it out. R numbers are unique across the doc.
|
|
106
|
+
- Under a [spec] criterion that the code seems to break, add the line:
|
|
107
|
+
Likely defect: <what the code does>
|
|
108
|
+
- Last come three unnumbered sections, in this order:
|
|
109
|
+
"## Not requirements";
|
|
110
|
+
"## Test setup", which lists what the environment must provide (accounts and
|
|
111
|
+
records to seed, flags to turn on, external services to stub with their
|
|
112
|
+
endpoint or SDK, browser dialogs), written as instructions to the test
|
|
113
|
+
generator;
|
|
114
|
+
"## Questions for product (not asserted)", which holds only what neither the
|
|
115
|
+
reference nor the code settles. Keep it short.
|
|
116
|
+
- Do not write a Coverage table. It is generated from the [C<ids>] on each
|
|
117
|
+
criterion.
|
|
118
|
+
- No TODOs, placeholders or secrets.
|
|
119
|
+
|
|
120
|
+
METHOD
|
|
121
|
+
1. Controls. List every control, then apply each baseline item that fits it.
|
|
122
|
+
2. Combinations. When the reference states a rule across alternatives ("for
|
|
123
|
+
login and sign-up, also on retry"), check the code. Alternatives handled by
|
|
124
|
+
the same code share one criterion that names them all. Only an alternative
|
|
125
|
+
that takes its own path gets its own criterion.
|
|
126
|
+
3. Code branches. Walk every validation rule, error response and branch the
|
|
127
|
+
excerpts show: account states (unverified, banned, deleted, other roles),
|
|
128
|
+
limits and expiry, empty and failure states, deep links, what the browser
|
|
129
|
+
stores and when it is cleared, paste, and clearing after an error. Each
|
|
130
|
+
branch becomes an [observed] criterion or a Question. A skipped branch is the
|
|
131
|
+
most common gap.
|
|
132
|
+
4. Shared rules. A rule or control used by several journeys (a password rule,
|
|
133
|
+
code entry, the login form, sign-out) is specified in full once, in the
|
|
134
|
+
journey it belongs to. The other journeys state only how the customer
|
|
135
|
+
reaches it and their own outcome, and name the owning journey.
|
|
136
|
+
5. Test data. Never ask for an exact value. Valid data follows the stated or
|
|
137
|
+
[observed] rules. Invalid data breaks one rule at a time. A choice control
|
|
138
|
+
uses an option the screen offers at run time.`;
|
|
139
|
+
/** What every line of a generated test must rest on, for web and API alike. */
|
|
140
|
+
export const EVIDENCE = `PRIME DIRECTIVE. Teams hold releases on these tests. A test that fails when
|
|
141
|
+
the product is correct is worse than no test, because it teaches the team to
|
|
142
|
+
ignore failures. A test may fail only when the product contradicts a rule in
|
|
143
|
+
<journey_rules>.
|
|
144
|
+
|
|
145
|
+
SCOPE. Write tests for exactly the rules listed in <task>, and no others.
|
|
146
|
+
Security, edge and abuse cases come from the [baseline] rules in the doc, not
|
|
147
|
+
from you. A rule the code does not support yet still gets a test, which fails
|
|
148
|
+
until the feature ships.
|
|
149
|
+
|
|
150
|
+
EVIDENCE. Every line of a test needs a source in this message.
|
|
151
|
+
|
|
152
|
+
E1. What you assert comes from a rule in <journey_rules>.
|
|
153
|
+
- Put one comment above each test: // oracle: R<n> [tier] <short paraphrase>
|
|
154
|
+
- Assert what the rule states, in the loosest form that still proves it. Add
|
|
155
|
+
no exact wording, order, id, timestamp or count that it does not fix.
|
|
156
|
+
- "No 5xx" or "the page loads" proves only a rule that says exactly that.
|
|
157
|
+
"The user is told why" is proved by a non-empty visible message, unless the
|
|
158
|
+
rule fixes the words.
|
|
159
|
+
- Assert the real end state: the response status and body, the stored record,
|
|
160
|
+
or the screen. Never assert only that nothing threw, and never a value your
|
|
161
|
+
own stub returned.
|
|
162
|
+
- Assert one outcome. Do not use a regex or condition that accepts both the
|
|
163
|
+
good and the bad result, an expect() that only runs inside if, else or
|
|
164
|
+
catch, or a comparison of a page's whole text with itself.
|
|
165
|
+
- A security probe asserts the secure outcome, unless the doc records the
|
|
166
|
+
weakness as accepted.
|
|
167
|
+
- The doc is the oracle, not the code. If the code contradicts the doc, assert
|
|
168
|
+
the doc and let the test fail. Never reshape an assertion to match what the
|
|
169
|
+
code does.
|
|
170
|
+
|
|
171
|
+
E2. What you touch comes from <code_facts> and <testkit>.
|
|
172
|
+
- Routes, fields, labels, storage keys and ids come from <code_facts>, or
|
|
173
|
+
from TEST_IDS and setup(). Use each route as its full mounted path, including
|
|
174
|
+
any version prefix, and use the same prefix throughout the file.
|
|
175
|
+
- A response field comes from the serializer for that endpoint in
|
|
176
|
+
<code_facts>, not from the types of another endpoint.
|
|
177
|
+
- A name that <code_facts> does not show comes from need() (see NAMES).
|
|
178
|
+
Never guess one.
|
|
179
|
+
- Call testkit helpers exactly as <testkit> lists them, and never write your
|
|
180
|
+
own copy.
|
|
181
|
+
|
|
182
|
+
E3. What you need first comes from the testkit, or from the product's own way
|
|
183
|
+
of making it.
|
|
184
|
+
- Create accounts, records and states through the testkit or the product's
|
|
185
|
+
API. Satisfy the schema in <code_facts>: every NOT NULL column, and the real
|
|
186
|
+
type of each id.
|
|
187
|
+
- Each role uses its own credential. Treat DUMMY, replace_me, an empty value
|
|
188
|
+
or a <bracketed> value as absent.
|
|
189
|
+
- Owner-scoped data ("my orders") follows the token owner, not a query
|
|
190
|
+
parameter. A non-owner check needs a second token; without one, skip and
|
|
191
|
+
name the missing identity.
|
|
192
|
+
- If a precondition really cannot be met, skip at run time and state what was
|
|
193
|
+
observed ("TEST_X is not set", "GET /x answered 404"), never a guessed
|
|
194
|
+
cause.
|
|
195
|
+
|
|
196
|
+
E4. What you wait for is the event that actually happens. Wait on a
|
|
197
|
+
condition, never for a fixed time.
|
|
198
|
+
|
|
199
|
+
E5. Isolation.
|
|
200
|
+
- Do not depend on test order, or on data another test created. Use a serial
|
|
201
|
+
block only when a test consumes state an earlier test in the same block
|
|
202
|
+
created.
|
|
203
|
+
- For shared state (a shared account, a seeded record), read it first and
|
|
204
|
+
restore it in finally or after-each.
|
|
205
|
+
- A "bad" input differs from the valid one at the level the product checks:
|
|
206
|
+
change the decoded bytes, not an encoded character. Where you can, assert
|
|
207
|
+
the valid case as well.
|
|
208
|
+
|
|
209
|
+
DATA A TEST CREATES
|
|
210
|
+
- Other runs share the same accounts. A test that creates a record does it as
|
|
211
|
+
the 'run' role, makes its identifying values with kit.unique(label), and
|
|
212
|
+
records it with kit.created(...), unless a testkit helper already did.
|
|
213
|
+
- Assert only on what this test created, by id or unique value. Never assert
|
|
214
|
+
on list counts, "the first item" or an empty list.
|
|
215
|
+
- Read-only tests use the fixture role and the seeded ids.
|
|
216
|
+
- For two different people at once, use one fixture role and one 'run' role,
|
|
217
|
+
and skip unless kit.distinctRunAccount() is true.
|
|
218
|
+
|
|
219
|
+
HOW MANY TESTS
|
|
220
|
+
- One test per distinct outcome: each rule, plus each branch or boundary the
|
|
221
|
+
code takes for it. Inputs that reach the same branch with the same outcome
|
|
222
|
+
are one case.
|
|
223
|
+
- At most <MAX_PER_RULE> tests per rule. More inputs go into one parametrised test, with
|
|
224
|
+
each row commented with its branch.
|
|
225
|
+
- Do not re-test behaviour another journey owns; test only how this journey
|
|
226
|
+
reaches it.
|
|
227
|
+
- Do not assert implementation details (storage keys, CSS classes, internal
|
|
228
|
+
flags) unless the rule is about them.
|
|
229
|
+
- Every rule gets a test that can fail. The only exception is a rule that
|
|
230
|
+
needs an outside service the test cannot drive: the inside of a payment
|
|
231
|
+
provider, an inbox, a push notification, another app, or the server clock.
|
|
232
|
+
That rule gets one line instead:
|
|
233
|
+
// cannot-test: R<n> — <service>
|
|
234
|
+
A missing route, label or table is never that reason; use need().
|
|
235
|
+
|
|
236
|
+
NAMES THE PRODUCT GIVES. When <code_facts> show a name, use it. Otherwise:
|
|
237
|
+
const { need } = require('@activate-core/test-forge/reporters/interface.cjs');
|
|
238
|
+
const ORDER_SUMMARY_URL = need('order summary URL');
|
|
239
|
+
Ask only for a name, and end it with what it is: … URL, … button label,
|
|
240
|
+
… table, … field, … status value, … query parameter, … command. Never ask for
|
|
241
|
+
data; the test makes its own. Reuse names from <names_already_asked> exactly.
|
|
242
|
+
|
|
243
|
+
SELF-CHECK BEFORE OUTPUT. For each test, name the rule that is its oracle,
|
|
244
|
+
the line of <code_facts> or <testkit> behind each locator and route, and the
|
|
245
|
+
product behaviour that would make it fail. Fix any test that cannot fail. Then
|
|
246
|
+
confirm every rule id has either a test or a cannot-test line.`;
|
|
247
|
+
/** Playwright only. */
|
|
248
|
+
export const WEB = `WEB (Playwright)
|
|
249
|
+
- import { test, expect } from '@playwright/test', and group tests with
|
|
250
|
+
test.describe. Do not use test.todo, it.*, describe.skip or test.fixme.
|
|
251
|
+
- Find elements with: const pay = await kit.find(page, kind, name).
|
|
252
|
+
kind is what the doc calls the element (button, link, input, dropdown,
|
|
253
|
+
checkbox, radio, tab, dialog, heading, alert, toast …). name is the text the
|
|
254
|
+
screen shows, taken from the SCREENS lines of <code_facts>. Use the doc's
|
|
255
|
+
wording only when <code_facts> lack the element. Always await kit.find
|
|
256
|
+
before using its result. Use page.getBy* only for what kit.find cannot
|
|
257
|
+
express. Never use a data-testid that <code_facts> do not show.
|
|
258
|
+
- When a label also appears in an always-mounted part of the page (a header,
|
|
259
|
+
a login pop-up), scope to the journey's own container. The SCREENS lines say
|
|
260
|
+
which container renders each element.
|
|
261
|
+
- page.goto() takes a concrete URL built with a template, such as
|
|
262
|
+
\`\${base}/s/\${code}\`. Never pass a placeholder like /user/:id.
|
|
263
|
+
- Open a later step of a multi-step flow with
|
|
264
|
+
kit.openStep(page, step, input), using a step name from kit.STEPS. If it
|
|
265
|
+
does not resolve true, skip with the reason it returns. Only a test of the
|
|
266
|
+
guard itself opens the bare URL.
|
|
267
|
+
- kit.login(page) prepares only that page. Log out through the app's own Log
|
|
268
|
+
Out control. A second tab needs its own kit.login. A rule about a login in
|
|
269
|
+
one tab reaching another tab gets a cannot-test line.
|
|
270
|
+
- For a code the test cannot read (SMS, email, magic link, third-party
|
|
271
|
+
sign-in), drive the real UI to the code step and use kit.bypassVerification
|
|
272
|
+
for that one step only, and put "(verification bypassed)" in the title.
|
|
273
|
+
Wrong-code and expired-code tests never bypass.
|
|
274
|
+
- Use kit.settledClick for buttons in pop-ups or in forms that grow,
|
|
275
|
+
kit.fillFrameInputs for inputs inside an iframe, and kit.watchDialogs for
|
|
276
|
+
alerts. kit.login and kit.suppressPopups stop the app's own pop-ups.
|
|
277
|
+
- A state the test cannot create (a cancelled booking, an expired code, a
|
|
278
|
+
banned account, an empty list) is stubbed with page.route(). Shape the stub
|
|
279
|
+
like the response in <code_facts>, and put "(stubbed state)" in the test
|
|
280
|
+
title. Match the stub by exact host, port, path prefix or a request
|
|
281
|
+
predicate, never by a substring that also matches the app's own pages.
|
|
282
|
+
- Waiting: prefer retrying assertions. Register waitForResponse or
|
|
283
|
+
waitForRequest before the action that triggers it. After a login, sign-out
|
|
284
|
+
or navigation, wait for the control you will use, then assert the URL the
|
|
285
|
+
app chooses. A page that never renders is reported as that, not as the rule
|
|
286
|
+
failing.
|
|
287
|
+
- A missing element is a failure, never a skip.
|
|
288
|
+
- For API calls inside a UI test, call the request fixture directly
|
|
289
|
+
(request.get, request.post). For a separate context with its own headers,
|
|
290
|
+
use kit.apiContext(request, role). Never call request.newContext() or
|
|
291
|
+
playwright.apiRequest.
|
|
292
|
+
- Check outcomes, not structure: the URL the app lands on, the request it
|
|
293
|
+
sends, the text and state the user sees, and what survives a refresh.`;
|
|
294
|
+
/** Jest only. */
|
|
295
|
+
export const API = `API (Jest)
|
|
296
|
+
- Send every request with kit.call(method, path, { body, headers, role }). It
|
|
297
|
+
reaches the server the way this project allows (in-process for a Node
|
|
298
|
+
server, over HTTP to the running server otherwise) and sends the role's
|
|
299
|
+
token. Never start Playwright or a server of your own.
|
|
300
|
+
- Assert the status code, the response body and its shape, and auth: no
|
|
301
|
+
token, the wrong role, and a different owner.
|
|
302
|
+
- Check stored state only through testkit resolvers or helpers. Never open
|
|
303
|
+
your own database connection.
|
|
304
|
+
- Keep tests hermetic. Every external service listed under EXTERNALS in
|
|
305
|
+
<code_facts> is stood in for:
|
|
306
|
+
- when the server runs inside the test process, with jest.mock on the exact
|
|
307
|
+
module named there, and any base URL that comes from an env var set in
|
|
308
|
+
beforeAll to a fixed test host;
|
|
309
|
+
- when the server runs in its own process, jest.mock cannot reach it: the
|
|
310
|
+
stand-in is the server's own, the env var that points it at a stub or a
|
|
311
|
+
fake that <code_facts> or the test setup names. If none is named, a test
|
|
312
|
+
that would reach the real service gets a cannot-test line.
|
|
313
|
+
Do not mock clients the code does not use.
|
|
314
|
+
- Do not use describe.skip, test.todo or test.fixme.`;
|
|
315
|
+
/** What the writer's answer looks like, and what the gate refuses. */
|
|
316
|
+
export function codeOutput(language, maxPerRule) {
|
|
317
|
+
const lang = language === 'JavaScript'
|
|
318
|
+
? 'Plain JavaScript (CommonJS: require and module.exports, no import, no type annotations)'
|
|
319
|
+
: 'TypeScript';
|
|
320
|
+
return `OUTPUT
|
|
321
|
+
- ${lang} only, ready to save and run. The first line is:
|
|
322
|
+
// @generated by testforge
|
|
323
|
+
- Write nothing before or after the code, and no markdown fences.
|
|
324
|
+
- Allowed comments: that header, one // oracle: line above each test,
|
|
325
|
+
// cannot-test: lines, a one-line branch note per parametrised row, and a
|
|
326
|
+
// timeout: <why> line on a test that needs a longer timeout.
|
|
327
|
+
No other comments, and no logging.
|
|
328
|
+
- Test titles are plain words for a reader; never a rule id in a title.
|
|
329
|
+
- No TODO or FIXME. Leave out what you cannot establish.
|
|
330
|
+
- No helper functions of your own; use the testkit.
|
|
331
|
+
- Files are checked automatically. A file is refused and rewritten if it:
|
|
332
|
+
sleeps for a fixed time; skips because an element is missing; has a test
|
|
333
|
+
without expect(); has more than ${maxPerRule} tests for one rule; uses kit.find without
|
|
334
|
+
await; uses an openStep step the testkit does not list; or names a control
|
|
335
|
+
that is in neither <code_facts> nor the rules.`;
|
|
336
|
+
}
|
|
337
|
+
/** The evidence block with the configured cap on tests per rule. */
|
|
338
|
+
export function evidence(maxPerRule) {
|
|
339
|
+
return EVIDENCE.replace('<MAX_PER_RULE>', String(maxPerRule));
|
|
340
|
+
}
|
|
@@ -193,14 +193,22 @@ export function scanNames(blank, root = '.') {
|
|
|
193
193
|
if (ask.length) {
|
|
194
194
|
const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
|
|
195
195
|
const prompt = [
|
|
196
|
-
'You match names a test needs to values in code. For each
|
|
197
|
-
|
|
198
|
-
'
|
|
199
|
-
'
|
|
196
|
+
'You match names a test needs to values in code. For each name in <names>,',
|
|
197
|
+
"pick its value from that name's own lines only, or null.",
|
|
198
|
+
'',
|
|
199
|
+
'A value is copied from the lines, never invented.',
|
|
200
|
+
'- A URL is the path a client calls: it starts with "/" and includes every',
|
|
201
|
+
' route prefix the lines show (a route inside a prefix group gets the prefix).',
|
|
202
|
+
'- A command is the command name a person types.',
|
|
203
|
+
'- A field, key or column is the exact key.',
|
|
200
204
|
'If no line clearly is that thing, the value is null.',
|
|
201
|
-
...(known.length ? ['', '
|
|
205
|
+
...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
|
|
206
|
+
'',
|
|
207
|
+
'<names>',
|
|
208
|
+
...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`)]),
|
|
209
|
+
'</names>',
|
|
202
210
|
'',
|
|
203
|
-
|
|
211
|
+
'Answer with one JSON object, {"<name>": "<value>" | null}, and nothing else.',
|
|
204
212
|
].join('\n');
|
|
205
213
|
try {
|
|
206
214
|
const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand), 'Answer only with the JSON object asked for.');
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
import { testFilesIn } from './readiness.js';
|
|
16
16
|
import { scanNames } from './name-scan.js';
|
|
17
17
|
import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
|
|
18
|
+
import { harnessInfo } from '../generate/harness.js';
|
|
19
|
+
import { missingPackages } from '../generate/verify.js';
|
|
18
20
|
import { spawnSync } from 'node:child_process';
|
|
19
21
|
import { existsSync, readFileSync } from 'node:fs';
|
|
20
22
|
import { join, dirname, resolve, relative } from 'node:path';
|
|
@@ -117,6 +119,11 @@ export function runTests(opts = {}) {
|
|
|
117
119
|
console.log(`No env loaded from "${envFile}" — copy .env.testing.example to it and fill values.`);
|
|
118
120
|
}
|
|
119
121
|
const env = { ...process.env, ...envVars };
|
|
122
|
+
// A key with nowhere to send the run: the reporter would skip publishing, and the
|
|
123
|
+
// only sign of it is one line at the very end of the runner's output.
|
|
124
|
+
if (env.TESTFORGE_KEY && !env.TESTFORGE_DASHBOARD_URL && !config.dashboardUrl) {
|
|
125
|
+
console.log('Note: TESTFORGE_KEY is set but "dashboardUrl" in testforge.config.json is empty, so this run will not be published to the dashboard.');
|
|
126
|
+
}
|
|
120
127
|
const areas = Object.keys(config.testOutput ?? {}).filter((a) => !opts.area || a === opts.area);
|
|
121
128
|
if (areas.length === 0) {
|
|
122
129
|
console.log('No areas configured (testOutput is empty). Run `testforge init`.');
|
|
@@ -124,6 +131,7 @@ export function runTests(opts = {}) {
|
|
|
124
131
|
}
|
|
125
132
|
let anyFail = false;
|
|
126
133
|
let ranAnything = false;
|
|
134
|
+
let stopped = false;
|
|
127
135
|
for (const area of areas) {
|
|
128
136
|
const testDir = testDirForType(config, area);
|
|
129
137
|
if (!existsSync(testDir)) {
|
|
@@ -133,6 +141,17 @@ export function runTests(opts = {}) {
|
|
|
133
141
|
const framework = resolveFramework(config, area, area);
|
|
134
142
|
const root = packageRoot(testDir);
|
|
135
143
|
const rel = relative(root, testDir) || '.';
|
|
144
|
+
// A package the testkit needs and the project lacks fails every test that
|
|
145
|
+
// touches it, each one looking like a product failure. Say so once, before.
|
|
146
|
+
const kit = harnessInfo(testDir, framework === 'jest' ? 'jest' : 'playwright', config.testExtension ?? '.test.ts').path;
|
|
147
|
+
const lacking = existsSync(kit) ? missingPackages(kit, root) : [];
|
|
148
|
+
if (lacking.length) {
|
|
149
|
+
console.log(`\n[stop] ${area}: the testkit needs ${lacking.join(', ')}, which this project does not have.`);
|
|
150
|
+
console.log(` Install it, then run again: npm i -D ${lacking.join(' ')} [in ${root}]`);
|
|
151
|
+
anyFail = true;
|
|
152
|
+
stopped = true;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
136
155
|
// Every test runs: a feature that is not there yet fails, and its report
|
|
137
156
|
// says so. Before the run, the names those tests wait for are looked for
|
|
138
157
|
// in the code — only where that kind of name lives, only when those lines
|
|
@@ -162,7 +181,8 @@ export function runTests(opts = {}) {
|
|
|
162
181
|
anyFail = true;
|
|
163
182
|
}
|
|
164
183
|
if (!ranAnything) {
|
|
165
|
-
|
|
184
|
+
if (!stopped)
|
|
185
|
+
console.log('\nNothing to run — generate tests first (`testforge generate`).');
|
|
166
186
|
return 1;
|
|
167
187
|
}
|
|
168
188
|
// The runner's own summary above says how many ran and how many skipped; a run where
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -297,6 +297,15 @@ function gitInfo(root) {
|
|
|
297
297
|
// NOT the git branch — a reporter can't sniff the DB, so it's set explicitly via TESTFORGE_ENV
|
|
298
298
|
// in the same env-file that points the tests at that DB (stage DB config → TESTFORGE_ENV=stage).
|
|
299
299
|
// Bare local run with nothing set ⇒ "local". Branch/sha are recorded separately as identity only.
|
|
300
|
+
// A test's title as a reader sees it: rule ids ("R92:", "R33 ", "R1, R2 —") are
|
|
301
|
+
// for the tool, so they are left out of what the dashboard shows.
|
|
302
|
+
function plainTitle(title) {
|
|
303
|
+
return String(title || '')
|
|
304
|
+
.replace(/(^|[\s(])R\d+(?:\s*[,/&]\s*R\d+)*\s*[:—–-]?\s+/g, '$1')
|
|
305
|
+
.replace(/\s*\(R\d+(?:\s*[,/&]\s*R\d+)*\)/g, '')
|
|
306
|
+
.trim();
|
|
307
|
+
}
|
|
308
|
+
|
|
300
309
|
function resolveEnv() {
|
|
301
310
|
return process.env.TESTFORGE_ENV || 'local';
|
|
302
311
|
}
|
|
@@ -400,4 +409,19 @@ function waitingFor(raw) {
|
|
|
400
409
|
return m ? m[1] : undefined;
|
|
401
410
|
}
|
|
402
411
|
|
|
403
|
-
|
|
412
|
+
// A failure the test environment caused before the product was reached (a package
|
|
413
|
+
// the testkit needs is not installed): reported as skipped with what to do, never
|
|
414
|
+
// as a product defect.
|
|
415
|
+
function environmentFor(raw) {
|
|
416
|
+
const m = /testkit: missing package ([^\s]+) — run: (npm i -D [^\n]+)/.exec(strip(String(raw || '')));
|
|
417
|
+
return m ? `Environment: package ${m[1]} is not installed — run: ${m[2].trim()}` : undefined;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// Why a failed test is not the product's: it waits for a name, or the environment lacks something.
|
|
421
|
+
function notTheProduct(raw) {
|
|
422
|
+
const waiting = waitingFor(raw);
|
|
423
|
+
if (waiting) return `Waiting for the product: "${waiting}" is not in it yet`;
|
|
424
|
+
return environmentFor(raw);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts,
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
@@ -16,16 +16,17 @@ module.exports = class TestForgeJestReporter {
|
|
|
16
16
|
const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
|
|
17
17
|
for (const a of (f.testResults || f.assertionResults || [])) {
|
|
18
18
|
const raw = (a.failureMessages || [])[0];
|
|
19
|
-
// A test waiting for a name the product does not have yet
|
|
20
|
-
|
|
21
|
-
const
|
|
19
|
+
// A test waiting for a name the product does not have yet, or stopped by the
|
|
20
|
+
// environment before it reached the product, is not a defect.
|
|
21
|
+
const notProduct = a.status === 'failed' ? notTheProduct(raw) : undefined;
|
|
22
|
+
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !notProduct ? 'failed' : 'skipped';
|
|
22
23
|
su.tests.push({
|
|
23
|
-
name: a.fullName || a.title,
|
|
24
|
+
name: plainTitle(a.fullName || a.title),
|
|
24
25
|
status,
|
|
25
26
|
error: status === 'failed' ? cleanError(raw) : undefined,
|
|
26
27
|
sig: status === 'failed' ? sig(raw) : undefined,
|
|
27
28
|
note: status === 'skipped'
|
|
28
|
-
? (
|
|
29
|
+
? (notProduct || (a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution'))
|
|
29
30
|
: undefined,
|
|
30
31
|
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
31
32
|
...(status === 'failed' ? expectation(raw) : {}),
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts,
|
|
4
|
+
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
@@ -37,16 +37,16 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
37
37
|
const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
|
|
38
38
|
const raw = result.error && result.error.message;
|
|
39
39
|
// A test waiting for a name the product does not have yet is not a defect.
|
|
40
|
-
const
|
|
41
|
-
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ||
|
|
40
|
+
const notProduct = result.status !== 'passed' && result.status !== 'skipped' ? notTheProduct(raw) : undefined;
|
|
41
|
+
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' || notProduct ? 'skipped' : 'failed';
|
|
42
42
|
const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
|
|
43
43
|
const skipAnno = pend.find((x) => x.description) || pend[0];
|
|
44
44
|
su.tests.push({
|
|
45
|
-
name: test.title,
|
|
45
|
+
name: plainTitle(test.title),
|
|
46
46
|
status,
|
|
47
47
|
error: status === 'failed' ? cleanError(raw) : undefined,
|
|
48
48
|
sig: status === 'failed' ? sig(raw) : undefined,
|
|
49
|
-
note: status === 'skipped' ? (
|
|
49
|
+
note: status === 'skipped' ? (notProduct || (skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
|
|
50
50
|
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
51
51
|
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
52
52
|
...(status === 'failed' ? this.failed(test, result) : {}),
|