@activate-core/test-forge 1.0.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -335,11 +335,41 @@ The test runner produces its own pass/fail report.
335
335
  Every AI phase may read them, so the map captures the real data layer and
336
336
  tests can check what the API stores. A headless run cannot read outside
337
337
  the project otherwise.
338
+ - **decisions** — a file or link listing what is NOT a requirement yet:
339
+ open product decisions and known issues left out on purpose. Docs do not
340
+ turn them into criteria, and a failing test that falls under one is
341
+ reported as a pending decision, never as a defect. Keeps the reference
342
+ you share with a client clean while the tests still know what is undecided.
343
+
344
+ ## How `generate` checks its own work
345
+
346
+ Every test is run against the real app before it is handed over, so a
347
+ mistake in a generated test does not reach your team as a product defect:
348
+
349
+ 1. **Each file on its own.** A test that fails because of its own mistake
350
+ (a wrong locator, a field the server never sends, a bad precondition) is
351
+ repaired from the runner's output. A guard refuses any repair that
352
+ deletes a test, adds a skip or removes an assertion.
353
+ 2. **The whole journey together**, the way CI runs it. Files that pass alone
354
+ but break each other (one signs in again as a shared account and ends
355
+ another's session) are repaired with that context.
356
+ 3. **An independent review of every claimed defect.** A separate reviewer
357
+ starts from "not proven" and must find the application code at fault.
358
+ `generate` then lists, one line per root cause:
359
+ - **defects**: a [spec] requirement the app breaks;
360
+ - **findings**: real behaviour against a [baseline]/[observed] criterion;
361
+ - **pending decisions**: covered by `decisions`.
362
+ A claim it rejects is repaired once more. Anything still unexplained is
363
+ reported as unverified, with the advice not to report it.
364
+
365
+ Each run is time-limited (`verifyTimeoutMs`, per file). On expiry the
366
+ runner stops with everything it started.
338
367
 
339
368
  ## Notes
340
369
 
341
- - Generated tests and docs are a strong draft, not final. Review them —
342
- the AI leaves `TODO` comments where it could not be certain.
370
+ - Generated tests and docs are a strong draft, not final. Review them.
371
+ Where something could not be established, a test is pending, with the
372
+ reason in its annotation (and in the dashboard).
343
373
  - Secrets are never written to docs, configs, or tests. Connections use
344
374
  environment variable names only; real values come from your
345
375
  environment at run time.
package/dist/cli/index.js CHANGED
@@ -29,7 +29,7 @@ import { runUpdate } from '../core/project/update.js';
29
29
  import { runEnvTemplate } from '../core/project/env-template.js';
30
30
  import { loadConfig } from '../core/project/config.js';
31
31
  import { publishReport, loadReportConfig, loadDotenv, scanInventory, publishInventory } from '../core/run/report.js';
32
- import { dirname } from 'node:path';
32
+ import { basename, dirname } from 'node:path';
33
33
  const command = process.argv[2];
34
34
  const useBasic = process.argv.includes('--basic');
35
35
  const force = process.argv.includes('--force');
@@ -190,6 +190,9 @@ async function main() {
190
190
  for (const w of r.warnings ?? []) {
191
191
  console.log(` ! ${w}`);
192
192
  }
193
+ if (r.verified === true)
194
+ console.log(' verified: every file ran; every failure left was reviewed and is listed below');
195
+ printReviewed(r.reviewed ?? []);
193
196
  }
194
197
  const generated = results.filter((r) => r.status !== 'skipped').length;
195
198
  // Rows with no test dir are notices (a non-journey file, the AI going
@@ -239,7 +242,7 @@ async function main() {
239
242
  return;
240
243
  }
241
244
  if (!resultsPath || !project || !dashboardUrl || !apiKey) {
242
- console.error('Usage: testforge report --results <jest-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
245
+ console.error('Usage: testforge report --results <jest-or-playwright-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
243
246
  console.error(' project / dashboard may also come from testforge.config.json.');
244
247
  console.error(' TESTFORGE_KEY must be set in the environment (the key the dashboard gave you).');
245
248
  process.exit(1);
@@ -257,6 +260,28 @@ async function main() {
257
260
  printUsage();
258
261
  }
259
262
  }
263
+ /** Reviewed failures, one line per root cause, strongest first. */
264
+ function printReviewed(reviewed) {
265
+ const sections = [
266
+ ['defect', 'DEFECTS — a [spec] requirement the app breaks'],
267
+ ['finding', 'FINDINGS — real app behaviour against a [baseline]/[observed] criterion (lower priority)'],
268
+ ['pending-decision', 'PENDING DECISIONS — not requirements yet; take them to product, do not report as defects'],
269
+ ];
270
+ for (const [verdict, heading] of sections) {
271
+ const rows = reviewed.filter((r) => r.verdict === verdict);
272
+ if (!rows.length)
273
+ continue;
274
+ console.log(` ${heading}:`);
275
+ for (const r of rows) {
276
+ const where = r.tests.slice(0, 3).map((t) => `${basename(t.file)}:${t.line}`).join(', ');
277
+ const more = r.tests.length > 3 ? ` +${r.tests.length - 3} more` : '';
278
+ console.log(` - ${r.claim}`);
279
+ console.log(` ${r.tests.length} test(s): ${where}${more}${r.evidence ? ` · app: ${r.evidence}` : ''}`);
280
+ if (r.reason)
281
+ console.log(` ${r.reason}`);
282
+ }
283
+ }
284
+ }
260
285
  main().catch((err) => {
261
286
  console.error(err instanceof Error ? err.message : String(err));
262
287
  process.exit(1);
@@ -80,6 +80,20 @@ export const DEEP_DOC_GUIDANCE = [
80
80
  ` same way (the screen accepts exactly what the server accepts). A`,
81
81
  ` failure is an inconsistency, not a spec defect. Never use [observed]`,
82
82
  ` to override or soften a [spec] criterion.`,
83
+ ` CHECK AN [observed] RULE BEFORE YOU STATE IT. Run each pattern in your`,
84
+ ` head against the concrete values you list — what it really accepts and`,
85
+ ` refuses, not what its name suggests. For the server's verdict, follow`,
86
+ ` the request through everything that runs BEFORE validation: global`,
87
+ ` middleware that trims, normalises or converts empty strings, and the`,
88
+ ` input mutators the handler applies. A row whose verdict is wrong`,
89
+ ` becomes a test that fails on a correct product.`,
90
+ ` An [observed] criterion is about what the customer sees or what the`,
91
+ ` server accepts or refuses — never an implementation detail (a storage`,
92
+ ` key name, a CSS class, an internal flag, a request field name) unless the`,
93
+ ` source itself is about it.`,
94
+ ` Only what a test can drive is a criterion. Behaviour no screen or`,
95
+ ` endpoint can reach ("not tested through the screen") is not tagged —`,
96
+ ` leave it out, or mention it in Test setup if the generator needs to know.`,
83
97
  ``,
84
98
  `METHOD. Work through these steps for every journey, in order:`,
85
99
  `1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
@@ -90,8 +104,11 @@ export const DEEP_DOC_GUIDANCE = [
90
104
  ` [baseline] item that fits to EACH row, not just to the main form.`,
91
105
  `2. COMBINATIONS. When the source states a rule across alternatives and`,
92
106
  ` conditions ("for login and sign-up, also on a slow connection and on`,
93
- ` a retry"), write one criterion per combination (login × slow, login ×`,
94
- ` retry, sign-up × slow, sign-up × retry), never just one of them.`,
107
+ ` a retry"), check in the code which of them take a different path. Write`,
108
+ ` one criterion per combination that takes its own path; alternatives`,
109
+ ` handled by the same code share ONE criterion that names them all. A`,
110
+ ` criterion per opener, per page or per input that all reach the same`,
111
+ ` code becomes one test per item — volume, not coverage.`,
95
112
  `3. CODE BRANCHES. Read the code the journey runs — screens, components,`,
96
113
  ` client API calls, and the server handlers they reach (related repos`,
97
114
  ` included) — and walk EVERY validation rule, error response and`,
@@ -104,9 +121,12 @@ export const DEEP_DOC_GUIDANCE = [
104
121
  `4. QUOTE [spec]. End every [spec] criterion with the source sentence it`,
105
122
  ` rests on, quoted: (source: "..."). A criterion you cannot quote is`,
106
123
  ` NOT [spec] — make it [baseline] or [observed], or a Question.`,
107
- `5. SHARED RULES. A rule shared by several journeys (password, redirects,`,
108
- ` attack payloads, accepted formats) uses the same cases and values in`,
109
- ` every doc.`,
124
+ `5. SHARED RULES AND SHARED CONTROLS. A rule or control shared by several`,
125
+ ` journeys (a password rule, a code-entry step, the login form, sign-out)`,
126
+ ` is specified in full ONCE, in the journey whose subject it is. Other`,
127
+ ` journeys that pass through it state only their own part: how the`,
128
+ ` customer reaches it from there and the outcome specific to that journey,`,
129
+ ` and name the journey that holds the full rule.`,
110
130
  `6. COVERAGE. After the numbered sections, add "## Coverage": one row per`,
111
131
  ` control from step 1, naming the section that covers each [baseline]`,
112
132
  ` item that fits it. A blank cell is a gap — fill it before finishing.`,
@@ -109,6 +109,15 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
109
109
  DEEP_DOC_GUIDANCE,
110
110
  ``,
111
111
  `Reference: ${reference}`,
112
+ ...(loadConfig().decisions
113
+ ? [
114
+ `NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
115
+ `known issues left out on purpose at: ${loadConfig().decisions}`,
116
+ `Read it. Nothing listed there becomes a criterion — no [spec], no`,
117
+ `[observed], no "Likely defect"; name it under "Questions for product`,
118
+ `(not asserted)" if the journey touches it.`,
119
+ ]
120
+ : []),
112
121
  `Read it FULLY with your available tools (read the file, fetch the`,
113
122
  `URL/page — follow sub-pages/links the reference points to). Do not`,
114
123
  `summarise shallowly; pull out every flow, rule, state and edge case`,
@@ -20,6 +20,7 @@ import { execFileSync } from 'node:child_process';
20
20
  import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
21
21
  import { join } from 'node:path';
22
22
  import { tmpdir } from 'node:os';
23
+ import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
23
24
  /**
24
25
  * Third-party network SDKs common enough to be worth gating everywhere:
25
26
  * a backend test that imports one without mocking it is talking to a real
@@ -163,6 +164,21 @@ export function checkTestGates(content, _ext, ctx) {
163
164
  'too; scope the stub to one of those, or name the ones you ruled out');
164
165
  }
165
166
  }
167
+ // F16 — no placeholders in a freshly written file: a placeholder never runs,
168
+ // and a suite full of them reads as coverage it does not have.
169
+ if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
170
+ return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
171
+ }
172
+ // F15 — a pending test says why. A reasonless fixme/skip reaches the report
173
+ // as a bare "fixme", indistinguishable from the tool giving up. Runs on
174
+ // content normalisePending() has already given every reason it could find.
175
+ if (ctx.framework === 'playwright') {
176
+ const bare = reasonlessPending(content);
177
+ if (bare.length) {
178
+ return (`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
179
+ `test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
180
+ }
181
+ }
166
182
  // F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
167
183
  // on a guessed selector. Unchecked, the generator invents plausible
168
184
  // data-testids that time out at runtime — the journey is never really
@@ -221,7 +237,7 @@ function stripPendingBlocks(content) {
221
237
  * skipping over quoted strings, template literals, and comments. Returns -1
222
238
  * if no match is found.
223
239
  */
224
- function matchParen(s, open) {
240
+ export function matchParen(s, open) {
225
241
  let depth = 0;
226
242
  for (let i = open; i < s.length; i++) {
227
243
  const c = s[i];