@activate-core/test-forge 1.0.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -335,11 +335,41 @@ The test runner produces its own pass/fail report.
335
335
  Every AI phase may read them, so the map captures the real data layer and
336
336
  tests can check what the API stores. A headless run cannot read outside
337
337
  the project otherwise.
338
+ - **decisions** — a file or link listing what is NOT a requirement yet:
339
+ open product decisions and known issues left out on purpose. Docs do not
340
+ turn them into criteria, and a failing test that falls under one is
341
+ reported as a pending decision, never as a defect. Keeps the reference
342
+ you share with a client clean while the tests still know what is undecided.
343
+
344
+ ## How `generate` checks its own work
345
+
346
+ Every test is run against the real app before it is handed over, so a
347
+ mistake in a generated test does not reach your team as a product defect:
348
+
349
+ 1. **Each file on its own.** A test that fails because of its own mistake
350
+ (a wrong locator, a field the server never sends, a bad precondition) is
351
+ repaired from the runner's output. A guard refuses any repair that
352
+ deletes a test, adds a skip or removes an assertion.
353
+ 2. **The whole journey together**, the way CI runs it. Files that pass alone
354
+ but break each other (one signs in again as a shared account and ends
355
+ another's session) are repaired with that context.
356
+ 3. **An independent review of every claimed defect.** A separate reviewer
357
+ starts from "not proven" and must find the application code at fault.
358
+ `generate` then lists, one line per root cause:
359
+ - **defects**: a [spec] requirement the app breaks;
360
+ - **findings**: real behaviour against a [baseline]/[observed] criterion;
361
+ - **pending decisions**: covered by `decisions`.
362
+ A claim it rejects is repaired once more. Anything still unexplained is
363
+ reported as unverified, with the advice not to report it.
364
+
365
+ Each run is time-limited (`verifyTimeoutMs`, per file). On expiry the
366
+ runner stops with everything it started.
338
367
 
339
368
  ## Notes
340
369
 
341
- - Generated tests and docs are a strong draft, not final. Review them —
342
- the AI leaves `TODO` comments where it could not be certain.
370
+ - Generated tests and docs are a strong draft, not final. Review them.
371
+ Where something could not be established, a test is pending, with the
372
+ reason in its annotation (and in the dashboard).
343
373
  - Secrets are never written to docs, configs, or tests. Connections use
344
374
  environment variable names only; real values come from your
345
375
  environment at run time.
package/dist/cli/index.js CHANGED
@@ -19,7 +19,7 @@
19
19
  * (an auth-gated page can move unseen).
20
20
  */
21
21
  import { VERSION } from '../core/index.js';
22
- import { runGenerate, runHarness } from '../core/generate/run-generate.js';
22
+ import { runGenerate } from '../core/generate/run-generate.js';
23
23
  import { runTests } from '../core/run/run-tests.js';
24
24
  import { runDiscover } from '../core/docs/discover.js';
25
25
  import { runMap } from '../core/project/map.js';
@@ -29,7 +29,7 @@ import { runUpdate } from '../core/project/update.js';
29
29
  import { runEnvTemplate } from '../core/project/env-template.js';
30
30
  import { loadConfig } from '../core/project/config.js';
31
31
  import { publishReport, loadReportConfig, loadDotenv, scanInventory, publishInventory } from '../core/run/report.js';
32
- import { dirname } from 'node:path';
32
+ import { basename, dirname } from 'node:path';
33
33
  const command = process.argv[2];
34
34
  const useBasic = process.argv.includes('--basic');
35
35
  const force = process.argv.includes('--force');
@@ -79,7 +79,7 @@ function printUsage() {
79
79
  console.log(' testforge generate --force Regenerate everything');
80
80
  console.log(' testforge generate --no-wait Stop when the AI hits its usage limit (default: wait for the reset and resume)');
81
81
  console.log(' testforge generate --no-verify Skip running each journey once after generating (default: run and repair)');
82
- console.log(' testforge generate --journey <name> Generate just ONE journey', ' testforge generate --harness Regenerate only the shared harness (testkit), no test files');
82
+ console.log(' testforge generate --journey <name> Generate just ONE journey');
83
83
  console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
84
84
  console.log(' testforge discover Draft journey docs (asks: map or reference)');
85
85
  console.log(' testforge discover --journey <name> Draft just ONE journey doc');
@@ -144,13 +144,6 @@ async function main() {
144
144
  });
145
145
  process.exitCode = code;
146
146
  }
147
- else if (command === 'generate' && process.argv.includes('--harness')) {
148
- // Regenerate only the shared harness (testkit), leaving every test file
149
- // as it is — e.g. after a TestForge upgrade changes what a harness does.
150
- console.log('Regenerating the shared test harness...\n');
151
- const written = await runHarness(flagValue('--area'));
152
- console.log(written.length ? `Written:\n ${written.join('\n ')}` : 'No harness written.');
153
- }
154
147
  else if (command === 'generate') {
155
148
  const useAI = !useBasic;
156
149
  console.log(useAI
@@ -197,6 +190,9 @@ async function main() {
197
190
  for (const w of r.warnings ?? []) {
198
191
  console.log(` ! ${w}`);
199
192
  }
193
+ if (r.verified === true)
194
+ console.log(' verified: every file ran; every failure left was reviewed and is listed below');
195
+ printReviewed(r.reviewed ?? []);
200
196
  }
201
197
  const generated = results.filter((r) => r.status !== 'skipped').length;
202
198
  // Rows with no test dir are notices (a non-journey file, the AI going
@@ -246,7 +242,7 @@ async function main() {
246
242
  return;
247
243
  }
248
244
  if (!resultsPath || !project || !dashboardUrl || !apiKey) {
249
- console.error('Usage: testforge report --results <jest-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
245
+ console.error('Usage: testforge report --results <jest-or-playwright-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
250
246
  console.error(' project / dashboard may also come from testforge.config.json.');
251
247
  console.error(' TESTFORGE_KEY must be set in the environment (the key the dashboard gave you).');
252
248
  process.exit(1);
@@ -264,6 +260,28 @@ async function main() {
264
260
  printUsage();
265
261
  }
266
262
  }
263
+ /** Reviewed failures, one line per root cause, strongest first. */
264
+ function printReviewed(reviewed) {
265
+ const sections = [
266
+ ['defect', 'DEFECTS — a [spec] requirement the app breaks'],
267
+ ['finding', 'FINDINGS — real app behaviour against a [baseline]/[observed] criterion (lower priority)'],
268
+ ['pending-decision', 'PENDING DECISIONS — not requirements yet; take them to product, do not report as defects'],
269
+ ];
270
+ for (const [verdict, heading] of sections) {
271
+ const rows = reviewed.filter((r) => r.verdict === verdict);
272
+ if (!rows.length)
273
+ continue;
274
+ console.log(` ${heading}:`);
275
+ for (const r of rows) {
276
+ const where = r.tests.slice(0, 3).map((t) => `${basename(t.file)}:${t.line}`).join(', ');
277
+ const more = r.tests.length > 3 ? ` +${r.tests.length - 3} more` : '';
278
+ console.log(` - ${r.claim}`);
279
+ console.log(` ${r.tests.length} test(s): ${where}${more}${r.evidence ? ` · app: ${r.evidence}` : ''}`);
280
+ if (r.reason)
281
+ console.log(` ${r.reason}`);
282
+ }
283
+ }
284
+ }
267
285
  main().catch((err) => {
268
286
  console.error(err instanceof Error ? err.message : String(err));
269
287
  process.exit(1);
@@ -80,6 +80,16 @@ export const DEEP_DOC_GUIDANCE = [
80
80
  ` same way (the screen accepts exactly what the server accepts). A`,
81
81
  ` failure is an inconsistency, not a spec defect. Never use [observed]`,
82
82
  ` to override or soften a [spec] criterion.`,
83
+ ` CHECK AN [observed] RULE BEFORE YOU STATE IT. Run each pattern in your`,
84
+ ` head against the concrete values you list — what it really accepts and`,
85
+ ` refuses, not what its name suggests. For the server's verdict, follow`,
86
+ ` the request through everything that runs BEFORE validation: global`,
87
+ ` middleware that trims, normalises or converts empty strings, and the`,
88
+ ` input mutators the handler applies. A row whose verdict is wrong`,
89
+ ` becomes a test that fails on a correct product.`,
90
+ ` Only what a test can drive is a criterion. Behaviour no screen or`,
91
+ ` endpoint can reach ("not tested through the screen") is not tagged —`,
92
+ ` leave it out, or mention it in Test setup if the generator needs to know.`,
83
93
  ``,
84
94
  `METHOD. Work through these steps for every journey, in order:`,
85
95
  `1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
@@ -109,6 +109,15 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
109
109
  DEEP_DOC_GUIDANCE,
110
110
  ``,
111
111
  `Reference: ${reference}`,
112
+ ...(loadConfig().decisions
113
+ ? [
114
+ `NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
115
+ `known issues left out on purpose at: ${loadConfig().decisions}`,
116
+ `Read it. Nothing listed there becomes a criterion — no [spec], no`,
117
+ `[observed], no "Likely defect"; name it under "Questions for product`,
118
+ `(not asserted)" if the journey touches it.`,
119
+ ]
120
+ : []),
112
121
  `Read it FULLY with your available tools (read the file, fetch the`,
113
122
  `URL/page — follow sub-pages/links the reference points to). Do not`,
114
123
  `summarise shallowly; pull out every flow, rule, state and edge case`,
@@ -20,6 +20,7 @@ import { execFileSync } from 'node:child_process';
20
20
  import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
21
21
  import { join } from 'node:path';
22
22
  import { tmpdir } from 'node:os';
23
+ import { reasonlessPending } from '../generate/pending.js';
23
24
  /**
24
25
  * Third-party network SDKs common enough to be worth gating everywhere:
25
26
  * a backend test that imports one without mocking it is talking to a real
@@ -163,6 +164,16 @@ export function checkTestGates(content, _ext, ctx) {
163
164
  'too; scope the stub to one of those, or name the ones you ruled out');
164
165
  }
165
166
  }
167
+ // F15 — a pending test says why. A reasonless fixme/skip reaches the report
168
+ // as a bare "fixme", indistinguishable from the tool giving up. Runs on
169
+ // content normalisePending() has already given every reason it could find.
170
+ if (ctx.framework === 'playwright') {
171
+ const bare = reasonlessPending(content);
172
+ if (bare.length) {
173
+ return (`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
174
+ `test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
175
+ }
176
+ }
166
177
  // F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
167
178
  // on a guessed selector. Unchecked, the generator invents plausible
168
179
  // data-testids that time out at runtime — the journey is never really
@@ -221,7 +232,7 @@ function stripPendingBlocks(content) {
221
232
  * skipping over quoted strings, template literals, and comments. Returns -1
222
233
  * if no match is found.
223
234
  */
224
- function matchParen(s, open) {
235
+ export function matchParen(s, open) {
225
236
  let depth = 0;
226
237
  for (let i = open; i < s.length; i++) {
227
238
  const c = s[i];