@activate-core/test-forge 1.3.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +70 -29
  6. package/dist/core/docs/excerpts.js +294 -0
  7. package/dist/core/docs/grounding.js +55 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/test-quality.js +295 -0
  15. package/dist/core/gates/validate.js +171 -7
  16. package/dist/core/generate/ai-generator.js +267 -117
  17. package/dist/core/generate/ai-usage.js +116 -5
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/complete-file.js +77 -0
  20. package/dist/core/generate/defect-review.js +59 -9
  21. package/dist/core/generate/doc-diff.js +140 -0
  22. package/dist/core/generate/harness.js +18 -34
  23. package/dist/core/generate/journey-memory.js +394 -0
  24. package/dist/core/generate/plan-cache.js +114 -7
  25. package/dist/core/generate/review-journey.js +3 -3
  26. package/dist/core/generate/run-generate.js +240 -82
  27. package/dist/core/generate/suite-planner.js +128 -23
  28. package/dist/core/generate/test-blocks.js +84 -0
  29. package/dist/core/generate/update-journey.js +118 -0
  30. package/dist/core/paths.js +90 -13
  31. package/dist/core/project/config.js +25 -2
  32. package/dist/core/project/env-template.js +2 -1
  33. package/dist/core/run/ci.js +1 -1
  34. package/dist/core/run/clean.js +1 -1
  35. package/dist/core/run/readiness.js +146 -0
  36. package/dist/core/run/run-tests.js +26 -2
  37. package/package.json +1 -1
  38. package/reporters/_shared.cjs +2 -2
  39. package/reporters/run-data.cjs +1 -1
  40. package/reporters/ui-helpers.cjs +78 -1
  41. package/dist/core/generate/verify-journey.js +0 -215
@@ -12,8 +12,8 @@
12
12
  */
13
13
  import { spawn } from 'node:child_process';
14
14
  import { readFileSync, existsSync } from 'node:fs';
15
- import { loadConfig, splitCommand, withReadOnlyTools, resolveFramework, } from '../project/config.js';
16
- import { runAiSync, noteAiCall } from './ai-usage.js';
15
+ import { loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
16
+ import { runAiSync, noteAiCall, traceArgs, traceCall } from './ai-usage.js';
17
17
  const MAP_PATH = 'testforge.md';
18
18
  /**
19
19
  * Spawn a command, feed `input` on stdin, resolve with its stdout.
@@ -23,7 +23,7 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
23
23
  const started = Date.now();
24
24
  let produced = '';
25
25
  return new Promise((resolve, reject) => {
26
- const child = spawn(program, args, {
26
+ const child = spawn(program, traceArgs(program, args), {
27
27
  stdio: ['pipe', 'pipe', 'pipe'],
28
28
  });
29
29
  let out = '';
@@ -40,6 +40,8 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
40
40
  // otherwise never close — kill it and surface a clear, resumable error.
41
41
  const timer = setTimeout(() => {
42
42
  child.kill('SIGKILL');
43
+ if (!done)
44
+ traceCall(input, out, started, `timed out after ${timeoutMs}ms`);
43
45
  finish(() => reject(new Error(`${program} timed out after ${timeoutMs}ms — the AI CLI did not respond. ` +
44
46
  `Check it runs non-interactively (auth set up, no TTY prompt), ` +
45
47
  `or raise aiTimeoutMs in testforge.config.json.`)));
@@ -52,8 +54,9 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
52
54
  finish(() => {
53
55
  produced = out;
54
56
  if (code === 0)
55
- resolve(out);
57
+ resolve(traceCall(input, out, started));
56
58
  else {
59
+ traceCall(input, out, started, `exited ${code}`);
57
60
  // The limit message is not always on stderr — some CLI versions
58
61
  // print it to stdout, some print nothing at all. Say what we have,
59
62
  // and say plainly when that is nothing, so a silent exit is never
@@ -67,7 +70,7 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
67
70
  }).finally(() => noteAiCall(input.length, produced.length, Date.now() - started));
68
71
  }
69
72
  /** Build the prompt we send to the AI for one journey. */
70
- export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding) {
73
+ export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding, memory) {
71
74
  const config = loadConfig();
72
75
  // Opinionated: testforge only generates Jest (api/backend) or Playwright
73
76
  // (web/UI) tests — any other framework (e.g. a project's Cypress setup)
@@ -86,7 +89,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
86
89
  const language = isJs
87
90
  ? 'plain JavaScript (CommonJS: use require/module.exports, NOT import; NO TypeScript type annotations)'
88
91
  : 'TypeScript';
89
- return [
92
+ const prompt = [
90
93
  `You are a senior QA engineer. Generate a complete ${framework}`,
91
94
  `for the user journey described below. (The specific file to write is`,
92
95
  `stated at the very end, after the journey — read all the context first.)`,
@@ -124,9 +127,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
124
127
  ` how to establish preconditions — follow it.`,
125
128
  ` - Assert what the line states, at the loosest form that still proves it.`,
126
129
  ` Add no condition it does not state, and no exact copy, ordering, id,`,
127
- ` timestamp or count it does not fix. A [baseline] robustness line ("opens`,
128
- ` without a crash") is proved by the page loading with no error and no`,
129
- ` 5xx, not by a URL, heading or element it never names. "The customer is`,
130
+ ` timestamp or count it does not fix. Only a line that says exactly "opens`,
131
+ ` without a crash" is proved by the page loading with no error and no 5xx;`,
132
+ ` every other line is proved by its own outcome, never by "no 5xx". "The customer is`,
130
133
  ` told why" is proved by a non-empty message VISIBLE where the app shows`,
131
134
  ` it, unless the line fixes the words. Check where the page actually`,
132
135
  ` renders a value before asserting it there.`,
@@ -249,10 +252,25 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
249
252
  ``,
250
253
  `A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
251
254
  `later step of a multi-step flow by its URL alone: call`,
252
- `kit.openStep(page, step, input) and skip with its reason when it does not`,
253
- `resolve true. Only a test that checks the guard itself (what a visitor who`,
255
+ `kit.openStep(page, step, input) with a step name the TESTKIT lists for it`,
256
+ `(no other name exists), and skip with its reason when it does not resolve`,
257
+ `true. A screen after the last listed step is reached from that step through`,
258
+ `the UI. Only a test that checks the guard itself (what a visitor who`,
254
259
  `skipped the earlier steps sees) opens the bare URL.`,
255
260
  ``,
261
+ `kit.find RETURNS A PROMISE. Always \`const pay = await kit.find(page, 'button',`,
262
+ `'Pay')\` first, then use pay. Never chain on kit.find(...) (.first(), .click()),`,
263
+ `never keep it in a variable without await, and never pass it to expect()`,
264
+ `without await.`,
265
+ ``,
266
+ `kit.login PREPARES ONE PAGE. It sets the session for the page it is given and`,
267
+ `restores it when that page loads again. So:`,
268
+ ` - log out through the app's own Log Out control, never by clearing storage or`,
269
+ ` cookies (the restored session comes straight back);`,
270
+ ` - a second tab or page is not logged in by it: call kit.login on that page`,
271
+ ` too. "Logged in in another tab" — one tab's login reaching another — is not`,
272
+ ` something the testkit can set up: such a rule gets a cannot-test line.`,
273
+ ``,
256
274
  `DATA A TEST CREATES. Other runs use the same accounts, so what they left`,
257
275
  `behind is already there when this test starts.`,
258
276
  ` - A test that creates a record (a booking, an order, a vehicle, an`,
@@ -265,6 +283,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
265
283
  ` or a list being empty. "Exactly one booking for this request" is checked`,
266
284
  ` among the records that carry this test's unique value.`,
267
285
  ` - A test that only reads data uses the fixture role and the seeded ids.`,
286
+ ` - Two DIFFERENT people at once (two customers paying for one slot): log one`,
287
+ ` in as the fixture role and one as "run", and skip at the top unless`,
288
+ ` kit.distinctRunAccount() is true (the environment has no second account).`,
289
+ ` A TESTKIT without distinctRunAccount gives that rule a cannot-test line.`,
268
290
  ``,
269
291
  `AT MOST ${maxPerRule()} TESTS FOR ONE CRITERION. Two tests whose oracle quotes the same`,
270
292
  `criterion must reach different branches; more inputs for that criterion go`,
@@ -291,38 +313,74 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
291
313
  ` declarations. A precondition the environment lacks is a run-time skip`,
292
314
  ` inside a real test, stating what was observed.`,
293
315
  ``,
316
+ `EVERY RULE GETS A TEST THAT CAN FAIL. Each rule id this file is given gets`,
317
+ `at least one test, and that test fails when the product breaks the rule.`,
318
+ ` - A rule whose STATE this area cannot set up — data that lives in another`,
319
+ ` system, a backend-only effect the screen never shows, an external service`,
320
+ ` it cannot drive — gets ONE line instead of a test:`,
321
+ ` // cannot-test: R7 — <the state this area cannot set up>`,
322
+ ` An element, locator or route you were not shown is NEVER that reason:`,
323
+ ` find the element with kit.find(page, kind, <the name the doc gives it>),`,
324
+ ` which needs nothing from the CODE FACTS. Two different people at once`,
325
+ ` is not that reason either: see kit.distinctRunAccount() below.`,
326
+ ` Never leave a rule out silently, and never write a test that passes`,
327
+ ` whatever the product does in its place.`,
328
+ ` - Assert the ONE outcome the rule expects. Never a regex or a condition`,
329
+ ` that accepts the good and the bad outcome alike (/confirm|not available/),`,
330
+ ` never the page's whole text (body or main innerText) compared with`,
331
+ ` itself, and never an expect() only inside an if, else or catch.`,
332
+ ` - A rule about a specific thing (the operator, the total, the time, the`,
333
+ ` link's address) reads THAT element and compares its value: "the same`,
334
+ ` operator" means the operator's name read before and after.`,
335
+ ` - "No 5xx", "the page loads", "body is not empty", "a £ sign is shown",`,
336
+ ` "the count of a text that never appears is 0" prove only a rule that says`,
337
+ ` exactly that.`,
338
+ ` - A stub's own words are never the assertion: assert what the app does`,
339
+ ` with the stubbed state.`,
340
+ ``,
294
341
  `SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
295
- `each line, its evidence under E1–E5, and the distinct outcome each test`,
296
- `adds. A line with no evidence is fixed, or its test is removed; a test that`,
297
- `adds no new outcome is removed. When in doubt, report LESS: a missing test`,
298
- `costs one line of coverage; a false failure costs the team a release.`,
342
+ `each line, its evidence under E1–E5, and the outcome that makes it fail. A`,
343
+ `line with no evidence is fixed; a test that cannot fail is rewritten, or`,
344
+ `its rule gets a cannot-test line. Then check every rule id you were given`,
345
+ `has a test or a cannot-test line.`,
299
346
  ``,
300
- ...(grounding
347
+ ...(memory
301
348
  ? [
302
- `THE MECHANICS ARE ALREADY ESTABLISHED. The CODE FACTS section`,
303
- `below was read from this project's real source for this journey —`,
304
- `routes with their full mounted paths, auth behaviour, request and`,
305
- `response shapes, models, externals, and the selectors the screens`,
306
- `genuinely expose. Use it. Do NOT re-derive it: opening the same`,
307
- `controllers and components again for every file in a journey is`,
308
- `what makes generation take an hour instead of minutes, and it`,
309
- `arrives at the same answer. Read source only for a specific detail`,
310
- `the CODE FACTS section marks "unknown" or genuinely does not`,
311
- `cover. Those facts are the MECHANICS: HOW to reach and`,
312
- `drive the behaviour. But the EXPECTED OUTCOME you`,
349
+ `EVERYTHING YOU NEED IS IN THIS MESSAGE. You have no tools and cannot`,
350
+ `open files. The CODE FACTS below were read from this project's source`,
351
+ `once for the whole journey, the TESTKIT list is the harness's real`,
352
+ `exports, and the JOURNEY block holds exactly the rules this file tests`,
353
+ `with their context. Those are the MECHANICS: HOW to reach and drive`,
354
+ `the behaviour. Never invent a test id, route, request field or helper`,
355
+ `that appears nowhere below. An element is found by the name the doc`,
356
+ `gives it, with kit.find. But the EXPECTED OUTCOME you`,
313
357
  ]
314
- : [
315
- `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
316
- `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
317
- `BEFORE writing tests, look up the actual code for this journey — use`,
318
- `the route/endpoint named below to grep for the router, then read the`,
319
- `route file, its controller/handler, the service it calls, and the`,
320
- `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
321
- `request/response shapes, status codes, auth requirements, and function`,
322
- `signatures from the code — that is the MECHANICS: HOW to reach and`,
323
- `drive the behaviour (paths, methods, payload/response shape,`,
324
- `selectors, model/collection names). But the EXPECTED OUTCOME you`,
325
- ]),
358
+ : grounding
359
+ ? [
360
+ `THE MECHANICS ARE ALREADY ESTABLISHED. The CODE FACTS section`,
361
+ `below was read from this project's real source for this journey —`,
362
+ `routes with their full mounted paths, auth behaviour, request and`,
363
+ `response shapes, models, externals, and the selectors the screens`,
364
+ `genuinely expose. Use it. Do NOT re-derive it: opening the same`,
365
+ `controllers and components again for every file in a journey is`,
366
+ `what makes generation take an hour instead of minutes, and it`,
367
+ `arrives at the same answer. Read source only for a specific detail`,
368
+ `the CODE FACTS section marks "unknown" or genuinely does not`,
369
+ `cover. Those facts are the MECHANICS: HOW to reach and`,
370
+ `drive the behaviour. But the EXPECTED OUTCOME you`,
371
+ ]
372
+ : [
373
+ `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
374
+ `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
375
+ `BEFORE writing tests, look up the actual code for this journey — use`,
376
+ `the route/endpoint named below to grep for the router, then read the`,
377
+ `route file, its controller/handler, the service it calls, and the`,
378
+ `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
379
+ `request/response shapes, status codes, auth requirements, and function`,
380
+ `signatures from the code — that is the MECHANICS: HOW to reach and`,
381
+ `drive the behaviour (paths, methods, payload/response shape,`,
382
+ `selectors, model/collection names). But the EXPECTED OUTCOME you`,
383
+ ]),
326
384
  `assert on is the ORACLE, and its source of truth is the JOURNEY`,
327
385
  `DOCUMENT below — NOT what the code currently does. The code tells you`,
328
386
  `HOW; the document tells you WHAT is correct. If the code's real`,
@@ -394,13 +452,34 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
394
452
  ` 3. getByText(...) — real on-screen text`,
395
453
  ` 4. locator('[name="…"]') / by id — a real input name/id attribute`,
396
454
  ` 5. getByTestId(...) / [data-cy=…] — ONLY if that hook truly exists`,
455
+ `FIND EVERY ELEMENT WITH kit.find(page, kind, name): kind is what the doc`,
456
+ `says it is (button, link, input, dropdown, checkbox, radio, toggle, tab,`,
457
+ `menuitem, dialog, heading, alert, toast, image, list, table …), name is`,
458
+ `the text the SCREEN shows for it — read it from the CODE FACTS (the`,
459
+ `component's text, label or aria-label); the doc's word only when the facts`,
460
+ `do not show that element ("help link" in the doc may read "FAQs" on the`,
461
+ `screen). Written out, never a value the test made up (kit.unique, a stub's`,
462
+ `id). It tries the proper role first and falls back`,
463
+ `to the label, placeholder, aria-label, alt text and visible text, adding`,
464
+ `a finding when the element is not built as what it is — so a test written`,
465
+ `before the feature still finds it once it ships. Use page.getBy* only for`,
466
+ `what kit.find cannot express. CHECK THE OUTCOME, NOT THE BUILD: the URL`,
467
+ `the app lands on, the request it sends and its data, the text and state`,
468
+ `the user sees after, what survives a refresh — never a tag, a class or`,
469
+ `the page's structure. A missing element is a failure, never a skip:`,
470
+ `skip only when the ENVIRONMENT lacks something (an account, a variable).`,
471
+ `A STATE THE TEST CANNOT CREATE (a disputed or cancelled booking, an`,
472
+ `expired code, a banned account, an empty list) is STUBBED, not skipped:`,
473
+ `answer the API call that returns it with page.route(), shaped exactly as`,
474
+ `the response type in the CODE FACTS, with only the fields that state needs`,
475
+ `changed, and put "(stubbed state)" in the test title. Then test the screen.`,
397
476
  `The first four are present in almost any real DOM without special`,
398
477
  `hooks, so a page with zero data-testids is still fully groundable —`,
399
478
  `ground on role/label/text/name instead of inventing a test-id.`,
400
479
  `NEVER invent a data-testid/data-cy: an unverified test-id is`,
401
480
  `REJECTED by the selector-grounding gate and wastes the whole file.`,
402
- `Only when an element cannot be identified by ANY real handle, leave`,
403
- `that single case out and name the control in a comment — never guess.`,
481
+ `An element the doc names is found with kit.find(page, kind, <that name>)`,
482
+ `even when the source shows no handle for it; never guess a test id.`,
404
483
  `NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
405
484
  `never a description/comment, and never a route-file placeholder`,
406
485
  `like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
@@ -469,9 +548,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
469
548
  `- Never write "TODO" or "FIXME" in the file. Something you cannot`,
470
549
  ` establish is left out, never a note to finish later.`,
471
550
  ``,
472
- `--- PROJECT MAP (data layer) ---`,
473
- map,
474
- ``,
551
+ // A web file's slice already names its screens; the data-layer map is for API files.
552
+ ...(memory && !isApi ? [] : [`--- PROJECT MAP (data layer) ---`, map, ``]),
553
+ // The same for every file of the journey, so it sits in the cached shared part.
554
+ ...(memory ? [`--- TESTKIT (the shared harness's exports) ---`, memory.helpers, ``] : []),
475
555
  // Read from the project's real source once per journey, before any file is
476
556
  // written, and handed unchanged to every file in it. Sits with the map and
477
557
  // the journey so the whole block stays a stable prefix across the journey.
@@ -489,12 +569,19 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
489
569
  ``,
490
570
  ]
491
571
  : []),
492
- `--- JOURNEY ---`,
493
- `It may include sections beyond fields — actions, expected results,`,
494
- `negative cases, race conditions, webhooks, security notes, and a`,
495
- `Data section. Use ALL of it.`,
496
- ``,
497
- journey.rawMarkdown,
572
+ ...(memory
573
+ ? [
574
+ `--- JOURNEY (only the part this file tests; other files test the rest) ---`,
575
+ memory.journey,
576
+ ]
577
+ : [
578
+ `--- JOURNEY ---`,
579
+ `It may include sections beyond fields — actions, expected results,`,
580
+ `negative cases, race conditions, webhooks, security notes, and a`,
581
+ `Data section. Use ALL of it.`,
582
+ ``,
583
+ journey.rawMarkdown,
584
+ ]),
498
585
  // ---- PER-FILE TASK (kept LAST so the big context above stays a stable,
499
586
  // cache-reusable prefix across every file in this journey, and so the
500
587
  // concrete task is the most recent thing the model reads) ----
@@ -507,9 +594,16 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
507
594
  ` ${focus.title}`,
508
595
  ` ${focus.covers}`,
509
596
  `Do NOT test the journey's other concerns here — separate files`,
510
- `cover those. Go DEEP on this one concern (happy path + its own`,
511
- `edge/negative/security cases) rather than broad-but-shallow.`,
597
+ `cover those. ${memory?.rules.length ? 'Test each rule below; add no case the rules do not state.' : 'Go DEEP on this one concern (happy path + its own edge/negative/security cases) rather than broad-but-shallow.'}`,
512
598
  `Use this exact describe() title: "${focus.title}".`,
599
+ ...(memory?.rules.length
600
+ ? [
601
+ `Test exactly these rules: ${memory.rules.join(', ')}. EVERY one gets a test or a`,
602
+ `"// cannot-test: <id> — <why>" line. Name the rule id AND its`,
603
+ `tag in each test's oracle comment, e.g. "// oracle: ${memory.rules[0]} [spec] …":`,
604
+ `the tag is how a failure is later graded as a defect or a finding.`,
605
+ ]
606
+ : []),
513
607
  ]
514
608
  : []),
515
609
  ...(testPath
@@ -523,9 +617,67 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
523
617
  ]
524
618
  : []),
525
619
  ``,
620
+ ...(memory
621
+ ? [
622
+ `YOUR FILE IS REFUSED, AND WRITTEN AGAIN, IF IT:`,
623
+ `- calls page.waitForTimeout() or sleeps for a fixed time;`,
624
+ `- skips a test because an element is missing (test.skip(true, …), or a`,
625
+ ` condition that counts or looks for an element) — that is a failure to assert;`,
626
+ `- has a test with no expect(), an expect() only inside an if/else/catch, or more`,
627
+ ` than ${maxPerRule()} tests for one rule;`,
628
+ `- leaves a rule with neither a test nor a "// cannot-test: <id> — <why>" line;`,
629
+ `- asserts a regex that accepts the good and the bad outcome, compares the page's`,
630
+ ` whole text, or asserts the words its own stub returns;`,
631
+ `- uses kit.find(...) without await, or a kit.openStep step the TESTKIT does not list;`,
632
+ `- names a button, link, field or label that is in neither the CODE FACTS nor the rules;`,
633
+ `- defines its own helper functions (use the TESTKIT) or comments anything but`,
634
+ ` the one "// oracle: <rule id> [tag] …" line above each test and cannot-test lines.`,
635
+ `Keep each test short: the steps, then the expect()s. No logging.`,
636
+ ``,
637
+ ]
638
+ : []),
526
639
  `Now output ONLY the raw ${isJs ? 'JavaScript' : 'TypeScript'} test code`,
527
640
  `for THIS file — the first character must be the first line of code.`,
528
641
  ].join('\n');
642
+ return memory ? withoutFileReading(prompt) : prompt;
643
+ }
644
+ /**
645
+ * A tool-less call's prompt in two: the part every file of the journey shares
646
+ * (the rules for writing a test, the harness, the language), sent as the
647
+ * system prompt so it is cached once and read cheaply by every later write and
648
+ * repair, and the file's own part (its facts, its slice, its task) on stdin.
649
+ */
650
+ export function splitShared(prompt) {
651
+ const at = ['\n--- CODE FACTS', '\n--- JOURNEY (only'].map((m) => prompt.indexOf(m)).filter((i) => i > 0);
652
+ if (!at.length)
653
+ return { system: '', message: prompt };
654
+ const cut = Math.min(...at);
655
+ return { system: prompt.slice(0, cut).trim(), message: prompt.slice(cut).trim() };
656
+ }
657
+ /** The reason the previous answer was refused, after the task, where it is read last. */
658
+ function withRefusal(prompt, refusal) {
659
+ return refusal
660
+ ? `${prompt}\n\n--- YOUR PREVIOUS ANSWER WAS REFUSED ---\nIt was not run. Reason: ${refusal}\nWrite it again without that problem.`
661
+ : prompt;
662
+ }
663
+ /** Program and args for one call: tool-less with the shared part as system prompt, or the old explorer. */
664
+ function writerCommand(prompt, memory) {
665
+ const cmd = splitCommand(loadConfig().aiCommand);
666
+ if (!memory)
667
+ return { ...withReadOnlyTools(cmd), input: prompt };
668
+ const { system, message } = splitShared(prompt);
669
+ return system ? { ...withNoTools(cmd, system), input: message } : { ...withNoTools(cmd), input: prompt };
670
+ }
671
+ /**
672
+ * The general rules tell a writer with tools to go and read a component or
673
+ * the testkit. A writer without tools is pointed at the block that already
674
+ * holds it instead.
675
+ */
676
+ function withoutFileReading(prompt) {
677
+ return prompt
678
+ .replace(/the UI\. FIRST read this page's real components: grep the webapp\nsource for the route, page name, and on-screen text, then open\nthose component files and use only locators that GENUINELY exist\./, 'the UI. Use only locators that the CODE FACTS or the JOURNEY block below\nshow GENUINELY exist.')
679
+ .replace('Read a harness helper before calling it, to see what it really returns.', 'Call a testkit helper exactly as its line under TESTKIT describes it.')
680
+ .replace(/Read that file to confirm its\nexact exports before using them\./, 'Its real exports are listed under\nTESTKIT below.');
529
681
  }
530
682
  /**
531
683
  * Call the configured AI command to generate a test file for one
@@ -558,86 +710,84 @@ export function generateWithAI(journey, testPath, focus, grounding) {
558
710
  */
559
711
  /** Marks a test whose failure the repair judged to be the application's. */
560
712
  /** The most tests one file may spend on one criterion (config maxTestsPerRule, default 4). */
561
- export const maxPerRule = () => loadConfig().maxTestsPerRule ?? 4;
713
+ export const maxPerRule = () => loadConfig().maxTestsPerRule ?? 2;
562
714
  export const DEFECT_MARK = 'testforge-defect:';
563
715
  /** A failure that is real application behaviour against a [baseline]/[observed] rule: kept failing, never a defect. */
564
716
  export const FINDING_MARK = 'testforge-finding:';
565
717
  /**
566
- * Repair a generated test from the runner's own failure output.
567
- *
568
- * The failure is evidence the generator did not have: the real selector, the
569
- * real URL shape, the real column type. So the repair prompt leads with it
570
- * and forbids the two easy ways out — deleting the assertion, or skipping the
571
- * test — because either turns a caught defect into a silent one.
718
+ * Tests for some rules of a file that already exists: new rules, or rules
719
+ * whose wording changed. The AI sees the file's top (imports, setup), the old
720
+ * tests of a changed rule, and the slice for those rules only, and answers
721
+ * with test blocks that are put into the file in place. The rest of the file
722
+ * is neither sent nor rewritten.
572
723
  */
573
- export async function repairWithAIAsync(journey, testPath, current, failure, harnessPath, transport, grounding) {
574
- const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
575
- const base = buildPrompt(journey, map, testPath, undefined, harnessPath, transport, grounding);
724
+ export async function writeTestsWithAI(journey, testPath, fileHead, title, oldTests, harnessPath, transport, grounding, memory, refusal) {
576
725
  const prompt = [
577
- base,
726
+ buildPrompt(journey, '', testPath, { title, covers: `Only the rules ${memory.rules.join(', ')}.` }, harnessPath, transport, grounding, memory),
578
727
  '',
579
- '=== REPAIR ===',
580
- `The file below was generated for ${testPath} and RUN against the real`,
581
- 'application. The tests listed under RUN RESULT did not pass. The runner',
582
- 'output is the ground truth: it tells you what the application actually',
583
- 'renders, routes and returns.',
728
+ '--- THIS FILE ALREADY EXISTS ---',
729
+ 'Its top (imports and setup), which your tests run under:',
730
+ fileHead,
584
731
  '',
585
- 'Decide, for EACH listed test, which of these it is — from the evidence,',
586
- 'not from what you hope:',
587
- ' TEST FAULT — the test never reached the behaviour it checks, or checked',
588
- ' it wrongly: a locator that matched nothing or the wrong',
589
- ' element, a container that does not hold the element, a wait',
590
- ' for the wrong event, a response field the server does not',
591
- ' send, a precondition its own setup got wrong, an assertion',
592
- ' the journey document does not state. FIX the test.',
593
- ' A "strict mode violation" means the locator matched several',
594
- ' elements: narrow it to the one the criterion means — never',
595
- ' .first() on a guess.',
596
- ' When the app sent the test back to an EARLIER step or page,',
597
- ' first suspect a missing precondition the earlier steps',
598
- ' would have set: reach the step with kit.openStep().',
599
- ' APP DEFECT — the test reached the behaviour (the element or response it',
600
- ' checks is there) and the application contradicts a criterion',
601
- ' the journey document states. Change NOTHING in that test.',
602
- ` Put one comment line directly above its declaration:`,
603
- ` // ${DEFECT_MARK} <the criterion, quoted with its tag> — <what the app did instead>`,
604
- ` Only a [spec] criterion can be a defect. When the criterion is`,
605
- ` [baseline] or [observed], write the same line with`,
606
- ` "${FINDING_MARK}" instead: it stays failing, as a finding.`,
607
- ' SKIPPED — a test listed under SKIPPED skipped itself. If its own',
608
- ' setup looked in the wrong place (wrong field, wrong route,',
609
- ' wrong id), fix that so it runs. If what it needs truly does',
610
- ' not exist in this environment, leave it exactly as it is.',
732
+ oldTests
733
+ ? `These tests were written for an earlier wording of ${memory.rules.join(', ')}. Rewrite them for the rules as they are now:\n${oldTests}`
734
+ : `The file already tests its other rules. Write tests only for ${memory.rules.join(', ')}.`,
611
735
  '',
612
- 'Rules, in addition to every rule above:',
613
- ' 1. Never delete, rename, weaken or invert an assertion, and never delete',
614
- ' or rename a test.',
615
- ' 2. Never add test.skip / test.fixme / try-catch to make a failure go away.',
616
- ' 3. Do not touch a test that is not listed below.',
617
- ' 4. When in doubt between TEST FAULT and APP DEFECT, look again at the',
618
- ' component and server source: an element that is in the DOM but hidden,',
619
- ' a message the server sends but the screen does not show, a value the',
620
- ' screen accepts but the server refuses — those are the application.',
621
- '',
622
- '--- CURRENT FILE ---',
623
- current,
624
- '--- RUN RESULT ---',
625
- failure,
626
- '--- END ---',
736
+ 'Output ONLY the test(...) blocks, each with its "// oracle:" comment above it: no imports, no',
737
+ 'describe, no setup. The first character of your answer is the first line of code.',
738
+ ].join('\n');
739
+ const config = loadConfig();
740
+ const { program, args, input } = writerCommand(withRefusal(prompt, refusal), memory);
741
+ return cleanOutput(await runWithStdin(program, args, input, config.aiTimeoutMs));
742
+ }
743
+ /** The line that opens each file in a batch answer. */
744
+ const FILE_MARK = (path) => `// ===== FILE: ${path} =====`;
745
+ /**
746
+ * Several files of one journey in one call. Their shared part (the rules for
747
+ * writing a test, the harness, the testkit) is the cached system prompt once;
748
+ * each file adds only its own facts, rules and task. The answer holds every
749
+ * file after its marker line; a file missing from it is simply not returned.
750
+ */
751
+ export async function writeFilesWithAI(files) {
752
+ const parts = files.map((f) => ({ ...f, ...splitShared(f.prompt) }));
753
+ const system = parts[0].system;
754
+ const message = [
755
+ `Write ${files.length} test files. Each one's facts, rules and task follow; the`,
756
+ `rules for writing a test are the same for all of them.`,
627
757
  '',
628
- 'Output ONLY the corrected file. No commentary.',
758
+ ...parts.flatMap((p, k) => [`=== FILE ${k + 1} of ${files.length}: ${p.path} ===`, p.message, '']),
759
+ `OUTPUT: all ${files.length} files, in this order. Before each, one line exactly:`,
760
+ ...files.map((f) => ` ${FILE_MARK(f.path)}`),
761
+ `then that file's code. Nothing before the first line and nothing after the last file.`,
629
762
  ].join('\n');
630
763
  const config = loadConfig();
631
- const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
632
- const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
633
- return cleanOutput(stdout);
764
+ const cmd = splitCommand(config.aiCommand);
765
+ const { program, args } = system ? withNoTools(cmd, system) : withNoTools(cmd);
766
+ return splitBatchAnswer(await runWithStdin(program, args, message, config.aiTimeoutMs), files.map((f) => f.path));
767
+ }
768
+ /** Each file's code from a batch answer, by its marker line. */
769
+ export function splitBatchAnswer(out, paths) {
770
+ const result = new Map();
771
+ const marks = paths
772
+ .map((path) => ({ path, at: out.indexOf(FILE_MARK(path)) }))
773
+ .filter((m) => m.at >= 0)
774
+ .sort((a, b) => a.at - b.at);
775
+ marks.forEach((m, k) => {
776
+ const end = k + 1 < marks.length ? marks[k + 1].at : out.length;
777
+ const code = cleanOutput(out.slice(m.at + FILE_MARK(m.path).length, end));
778
+ if (code.trim())
779
+ result.set(m.path, code);
780
+ });
781
+ return result;
634
782
  }
635
- export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding) {
783
+ export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding, memory,
784
+ /** Why the last answer was refused before it ran; told to the next try. */
785
+ refusal) {
636
786
  const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
637
- const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding);
787
+ const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding, memory);
638
788
  const config = loadConfig();
639
- const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
640
- const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
789
+ const { program, args, input } = writerCommand(withRefusal(prompt, refusal), memory);
790
+ const stdout = await runWithStdin(program, args, input, config.aiTimeoutMs);
641
791
  return cleanOutput(stdout);
642
792
  }
643
793
  /**