@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -10,17 +10,20 @@
10
10
  * We call it in print mode (`claude -p`) and feed the prompt via
11
11
  * stdin so long prompts and special characters are handled safely.
12
12
  */
13
- import { execFileSync, spawn } from 'node:child_process';
13
+ import { spawn } from 'node:child_process';
14
14
  import { readFileSync, existsSync } from 'node:fs';
15
- import { loadConfig, splitCommand, withReadOnlyTools, resolveFramework, } from '../project/config.js';
15
+ import { loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
16
+ import { runAiSync, noteAiCall, traceArgs, traceCall } from './ai-usage.js';
16
17
  const MAP_PATH = 'testforge.md';
17
18
  /**
18
19
  * Spawn a command, feed `input` on stdin, resolve with its stdout.
19
20
  * Async (unlike execFileSync) so many calls can run concurrently.
20
21
  */
21
22
  export function runWithStdin(program, args, input, timeoutMs = 300_000) {
23
+ const started = Date.now();
24
+ let produced = '';
22
25
  return new Promise((resolve, reject) => {
23
- const child = spawn(program, args, {
26
+ const child = spawn(program, traceArgs(program, args), {
24
27
  stdio: ['pipe', 'pipe', 'pipe'],
25
28
  });
26
29
  let out = '';
@@ -37,6 +40,8 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
37
40
  // otherwise never close — kill it and surface a clear, resumable error.
38
41
  const timer = setTimeout(() => {
39
42
  child.kill('SIGKILL');
43
+ if (!done)
44
+ traceCall(input, out, started, `timed out after ${timeoutMs}ms`);
40
45
  finish(() => reject(new Error(`${program} timed out after ${timeoutMs}ms — the AI CLI did not respond. ` +
41
46
  `Check it runs non-interactively (auth set up, no TTY prompt), ` +
42
47
  `or raise aiTimeoutMs in testforge.config.json.`)));
@@ -47,9 +52,11 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
47
52
  child.on('error', (e) => finish(() => reject(e)));
48
53
  child.on('close', (code) => {
49
54
  finish(() => {
55
+ produced = out;
50
56
  if (code === 0)
51
- resolve(out);
57
+ resolve(traceCall(input, out, started));
52
58
  else {
59
+ traceCall(input, out, started, `exited ${code}`);
53
60
  // The limit message is not always on stderr — some CLI versions
54
61
  // print it to stdout, some print nothing at all. Say what we have,
55
62
  // and say plainly when that is nothing, so a silent exit is never
@@ -60,10 +67,10 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
60
67
  });
61
68
  });
62
69
  child.stdin.end(input);
63
- });
70
+ }).finally(() => noteAiCall(input.length, produced.length, Date.now() - started));
64
71
  }
65
72
  /** Build the prompt we send to the AI for one journey. */
66
- export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding) {
73
+ export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding, memory) {
67
74
  const config = loadConfig();
68
75
  // Opinionated: testforge only generates Jest (api/backend) or Playwright
69
76
  // (web/UI) tests — any other framework (e.g. a project's Cypress setup)
@@ -82,7 +89,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
82
89
  const language = isJs
83
90
  ? 'plain JavaScript (CommonJS: use require/module.exports, NOT import; NO TypeScript type annotations)'
84
91
  : 'TypeScript';
85
- return [
92
+ const prompt = [
86
93
  `You are a senior QA engineer. Generate a complete ${framework}`,
87
94
  `for the user journey described below. (The specific file to write is`,
88
95
  `stated at the very end, after the journey — read all the context first.)`,
@@ -234,6 +241,38 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
234
241
  ` (change decoded bytes, not an encoded character), and the valid control`,
235
242
  ` is asserted too where you can.`,
236
243
  ``,
244
+ `USE THE TESTKIT'S HELPERS, NEVER YOUR OWN COPY. A helper written into several`,
245
+ `files diverges, and one mistake in it fails every file at once.`,
246
+ ` - kit.settledClick(locator) for a button inside a pop-up or a form that`,
247
+ ` grows while you fill it (payment, address, any modal with a scrollbar);`,
248
+ ` kit.fillFrameInputs(page, framePrefix, { name: value }) for inputs inside`,
249
+ ` an embedded iframe; kit.watchDialogs(page) to accept and record alerts.`,
250
+ ` - Anything else two files of this journey would both need belongs in the`,
251
+ ` testkit, not copied into each file. Small one-line locators are fine.`,
252
+ ``,
253
+ `A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
254
+ `later step of a multi-step flow by its URL alone: call`,
255
+ `kit.openStep(page, step, input) and skip with its reason when it does not`,
256
+ `resolve true. Only a test that checks the guard itself (what a visitor who`,
257
+ `skipped the earlier steps sees) opens the bare URL.`,
258
+ ``,
259
+ `DATA A TEST CREATES. Other runs use the same accounts, so what they left`,
260
+ `behind is already there when this test starts.`,
261
+ ` - A test that creates a record (a booking, an order, a vehicle, an`,
262
+ ` address, a payment) does it as the testkit's "run" role, makes every`,
263
+ ` identifying value with kit.unique(label) (a registration, a name, a`,
264
+ ` note), and records what it made with kit.created(...) unless a testkit`,
265
+ ` helper already did.`,
266
+ ` - It asserts on what IT created — found by its own id or its unique value —`,
267
+ ` never on a count of an account-wide list, "the first" or "the only" item,`,
268
+ ` or a list being empty. "Exactly one booking for this request" is checked`,
269
+ ` among the records that carry this test's unique value.`,
270
+ ` - A test that only reads data uses the fixture role and the seeded ids.`,
271
+ ``,
272
+ `AT MOST ${maxPerRule()} TESTS FOR ONE CRITERION. Two tests whose oracle quotes the same`,
273
+ `criterion must reach different branches; more inputs for that criterion go`,
274
+ `into ONE parametrised test (a loop over rows), not more tests.`,
275
+ ``,
237
276
  `COVER EACH OUTCOME ONCE. A suite is judged by the defects it can catch, not`,
238
277
  `by its size; every redundant test costs run time and buries the ones that`,
239
278
  `matter.`,
@@ -261,32 +300,43 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
261
300
  `adds no new outcome is removed. When in doubt, report LESS: a missing test`,
262
301
  `costs one line of coverage; a false failure costs the team a release.`,
263
302
  ``,
264
- ...(grounding
303
+ ...(memory
265
304
  ? [
266
- `THE MECHANICS ARE ALREADY ESTABLISHED. The CODE FACTS section`,
267
- `below was read from this project's real source for this journey —`,
268
- `routes with their full mounted paths, auth behaviour, request and`,
269
- `response shapes, models, externals, and the selectors the screens`,
270
- `genuinely expose. Use it. Do NOT re-derive it: opening the same`,
271
- `controllers and components again for every file in a journey is`,
272
- `what makes generation take an hour instead of minutes, and it`,
273
- `arrives at the same answer. Read source only for a specific detail`,
274
- `the CODE FACTS section marks "unknown" or genuinely does not`,
275
- `cover. Those facts are the MECHANICS: HOW to reach and`,
276
- `drive the behaviour. But the EXPECTED OUTCOME you`,
305
+ `EVERYTHING YOU NEED IS IN THIS MESSAGE. You have no tools and cannot`,
306
+ `open files. The CODE FACTS below were read from this project's source`,
307
+ `once for the whole journey, the TESTKIT list is the harness's real`,
308
+ `exports, and the JOURNEY block holds exactly the rules this file tests`,
309
+ `with their context. Those are the MECHANICS: HOW to reach and drive`,
310
+ `the behaviour. Never invent a selector, route, field or helper that`,
311
+ `appears nowhere below; a case you cannot reach with what is here is`,
312
+ `left out. But the EXPECTED OUTCOME you`,
277
313
  ]
278
- : [
279
- `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
280
- `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
281
- `BEFORE writing tests, look up the actual code for this journey — use`,
282
- `the route/endpoint named below to grep for the router, then read the`,
283
- `route file, its controller/handler, the service it calls, and the`,
284
- `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
285
- `request/response shapes, status codes, auth requirements, and function`,
286
- `signatures from the code — that is the MECHANICS: HOW to reach and`,
287
- `drive the behaviour (paths, methods, payload/response shape,`,
288
- `selectors, model/collection names). But the EXPECTED OUTCOME you`,
289
- ]),
314
+ : grounding
315
+ ? [
316
+ `THE MECHANICS ARE ALREADY ESTABLISHED. The CODE FACTS section`,
317
+ `below was read from this project's real source for this journey —`,
318
+ `routes with their full mounted paths, auth behaviour, request and`,
319
+ `response shapes, models, externals, and the selectors the screens`,
320
+ `genuinely expose. Use it. Do NOT re-derive it: opening the same`,
321
+ `controllers and components again for every file in a journey is`,
322
+ `what makes generation take an hour instead of minutes, and it`,
323
+ `arrives at the same answer. Read source only for a specific detail`,
324
+ `the CODE FACTS section marks "unknown" or genuinely does not`,
325
+ `cover. Those facts are the MECHANICS: HOW to reach and`,
326
+ `drive the behaviour. But the EXPECTED OUTCOME you`,
327
+ ]
328
+ : [
329
+ `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
330
+ `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
331
+ `BEFORE writing tests, look up the actual code for this journey — use`,
332
+ `the route/endpoint named below to grep for the router, then read the`,
333
+ `route file, its controller/handler, the service it calls, and the`,
334
+ `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
335
+ `request/response shapes, status codes, auth requirements, and function`,
336
+ `signatures from the code — that is the MECHANICS: HOW to reach and`,
337
+ `drive the behaviour (paths, methods, payload/response shape,`,
338
+ `selectors, model/collection names). But the EXPECTED OUTCOME you`,
339
+ ]),
290
340
  `assert on is the ORACLE, and its source of truth is the JOURNEY`,
291
341
  `DOCUMENT below — NOT what the code currently does. The code tells you`,
292
342
  `HOW; the document tells you WHAT is correct. If the code's real`,
@@ -358,6 +408,23 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
358
408
  ` 3. getByText(...) — real on-screen text`,
359
409
  ` 4. locator('[name="…"]') / by id — a real input name/id attribute`,
360
410
  ` 5. getByTestId(...) / [data-cy=…] — ONLY if that hook truly exists`,
411
+ `FIND EVERY ELEMENT WITH kit.find(page, kind, name): kind is what the doc`,
412
+ `says it is (button, link, input, dropdown, checkbox, radio, toggle, tab,`,
413
+ `menuitem, dialog, heading, alert, toast, image, list, table …), name is`,
414
+ `the text the doc gives it. It tries the proper role first and falls back`,
415
+ `to the label, placeholder, aria-label, alt text and visible text, adding`,
416
+ `a finding when the element is not built as what it is — so a test written`,
417
+ `before the feature still finds it once it ships. Use page.getBy* only for`,
418
+ `what kit.find cannot express. CHECK THE OUTCOME, NOT THE BUILD: the URL`,
419
+ `the app lands on, the request it sends and its data, the text and state`,
420
+ `the user sees after, what survives a refresh — never a tag, a class or`,
421
+ `the page's structure. A missing element is a failure, never a skip:`,
422
+ `skip only when the ENVIRONMENT lacks something (an account, a variable).`,
423
+ `A STATE THE TEST CANNOT CREATE (a disputed or cancelled booking, an`,
424
+ `expired code, a banned account, an empty list) is STUBBED, not skipped:`,
425
+ `answer the API call that returns it with page.route(), shaped exactly as`,
426
+ `the response type in the CODE FACTS, with only the fields that state needs`,
427
+ `changed, and put "(stubbed state)" in the test title. Then test the screen.`,
361
428
  `The first four are present in almost any real DOM without special`,
362
429
  `hooks, so a page with zero data-testids is still fully groundable —`,
363
430
  `ground on role/label/text/name instead of inventing a test-id.`,
@@ -433,9 +500,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
433
500
  `- Never write "TODO" or "FIXME" in the file. Something you cannot`,
434
501
  ` establish is left out, never a note to finish later.`,
435
502
  ``,
436
- `--- PROJECT MAP (data layer) ---`,
437
- map,
438
- ``,
503
+ // A web file's slice already names its screens; the data-layer map is for API files.
504
+ ...(memory && !isApi ? [] : [`--- PROJECT MAP (data layer) ---`, map, ``]),
505
+ // The same for every file of the journey, so it sits in the cached shared part.
506
+ ...(memory ? [`--- TESTKIT (the shared harness's exports) ---`, memory.helpers, ``] : []),
439
507
  // Read from the project's real source once per journey, before any file is
440
508
  // written, and handed unchanged to every file in it. Sits with the map and
441
509
  // the journey so the whole block stays a stable prefix across the journey.
@@ -453,12 +521,19 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
453
521
  ``,
454
522
  ]
455
523
  : []),
456
- `--- JOURNEY ---`,
457
- `It may include sections beyond fields — actions, expected results,`,
458
- `negative cases, race conditions, webhooks, security notes, and a`,
459
- `Data section. Use ALL of it.`,
460
- ``,
461
- journey.rawMarkdown,
524
+ ...(memory
525
+ ? [
526
+ `--- JOURNEY (only the part this file tests; other files test the rest) ---`,
527
+ memory.journey,
528
+ ]
529
+ : [
530
+ `--- JOURNEY ---`,
531
+ `It may include sections beyond fields — actions, expected results,`,
532
+ `negative cases, race conditions, webhooks, security notes, and a`,
533
+ `Data section. Use ALL of it.`,
534
+ ``,
535
+ journey.rawMarkdown,
536
+ ]),
462
537
  // ---- PER-FILE TASK (kept LAST so the big context above stays a stable,
463
538
  // cache-reusable prefix across every file in this journey, and so the
464
539
  // concrete task is the most recent thing the model reads) ----
@@ -474,6 +549,13 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
474
549
  `cover those. Go DEEP on this one concern (happy path + its own`,
475
550
  `edge/negative/security cases) rather than broad-but-shallow.`,
476
551
  `Use this exact describe() title: "${focus.title}".`,
552
+ ...(memory?.rules.length
553
+ ? [
554
+ `Test exactly these rules: ${memory.rules.join(', ')}. Name the rule id AND its`,
555
+ `tag in each test's oracle comment, e.g. "// oracle: ${memory.rules[0]} [spec] …":`,
556
+ `the tag is how a failure is later graded as a defect or a finding.`,
557
+ ]
558
+ : []),
477
559
  ]
478
560
  : []),
479
561
  ...(testPath
@@ -487,9 +569,63 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
487
569
  ]
488
570
  : []),
489
571
  ``,
572
+ ...(memory
573
+ ? [
574
+ `YOUR FILE IS REFUSED, AND WRITTEN AGAIN, IF IT:`,
575
+ `- calls page.waitForTimeout() or sleeps for a fixed time;`,
576
+ `- skips a test because an element is missing (test.skip(true, …), or a`,
577
+ ` condition that counts or looks for an element) — that is a failure to assert;`,
578
+ `- has a test with no expect(), or more than ${maxPerRule()} tests for one rule;`,
579
+ `- has no test at all: every rule gets one; stub a state you cannot create;`,
580
+ `- names a button, link, field or label that is in neither the CODE FACTS nor the rules;`,
581
+ `- defines its own helper functions (use the TESTKIT) or comments anything but`,
582
+ ` the one "// oracle: <rule id> [tag] …" line above each test.`,
583
+ `Keep each test short: the steps, then the expect()s. No logging.`,
584
+ ``,
585
+ ]
586
+ : []),
490
587
  `Now output ONLY the raw ${isJs ? 'JavaScript' : 'TypeScript'} test code`,
491
588
  `for THIS file — the first character must be the first line of code.`,
492
589
  ].join('\n');
590
+ return memory ? withoutFileReading(prompt) : prompt;
591
+ }
592
+ /**
593
+ * A tool-less call's prompt in two: the part every file of the journey shares
594
+ * (the rules for writing a test, the harness, the language), sent as the
595
+ * system prompt so it is cached once and read cheaply by every later write and
596
+ * repair, and the file's own part (its facts, its slice, its task) on stdin.
597
+ */
598
+ export function splitShared(prompt) {
599
+ const at = ['\n--- CODE FACTS', '\n--- JOURNEY (only'].map((m) => prompt.indexOf(m)).filter((i) => i > 0);
600
+ if (!at.length)
601
+ return { system: '', message: prompt };
602
+ const cut = Math.min(...at);
603
+ return { system: prompt.slice(0, cut).trim(), message: prompt.slice(cut).trim() };
604
+ }
605
+ /** The reason the previous answer was refused, after the task, where it is read last. */
606
+ function withRefusal(prompt, refusal) {
607
+ return refusal
608
+ ? `${prompt}\n\n--- YOUR PREVIOUS ANSWER WAS REFUSED ---\nIt was not run. Reason: ${refusal}\nWrite it again without that problem.`
609
+ : prompt;
610
+ }
611
+ /** Program and args for one call: tool-less with the shared part as system prompt, or the old explorer. */
612
+ function writerCommand(prompt, memory) {
613
+ const cmd = splitCommand(loadConfig().aiCommand);
614
+ if (!memory)
615
+ return { ...withReadOnlyTools(cmd), input: prompt };
616
+ const { system, message } = splitShared(prompt);
617
+ return system ? { ...withNoTools(cmd, system), input: message } : { ...withNoTools(cmd), input: prompt };
618
+ }
619
+ /**
620
+ * The general rules tell a writer with tools to go and read a component or
621
+ * the testkit. A writer without tools is pointed at the block that already
622
+ * holds it instead.
623
+ */
624
+ function withoutFileReading(prompt) {
625
+ return prompt
626
+ .replace(/the UI\. FIRST read this page's real components: grep the webapp\nsource for the route, page name, and on-screen text, then open\nthose component files and use only locators that GENUINELY exist\./, 'the UI. Use only locators that the CODE FACTS or the JOURNEY block below\nshow GENUINELY exist.')
627
+ .replace('Read a harness helper before calling it, to see what it really returns.', 'Call a testkit helper exactly as its line under TESTKIT describes it.')
628
+ .replace(/Read that file to confirm its\nexact exports before using them\./, 'Its real exports are listed under\nTESTKIT below.');
493
629
  }
494
630
  /**
495
631
  * Call the configured AI command to generate a test file for one
@@ -504,7 +640,7 @@ export function generateWithAI(journey, testPath, focus, grounding) {
504
640
  // still can't create files — it reads, then prints the code.
505
641
  const config = loadConfig();
506
642
  const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
507
- const output = execFileSync(program, args, {
643
+ const output = runAiSync(program, args, {
508
644
  input: prompt,
509
645
  encoding: 'utf-8',
510
646
  maxBuffer: 10 * 1024 * 1024,
@@ -521,74 +657,85 @@ export function generateWithAI(journey, testPath, focus, grounding) {
521
657
  * `claude` process). Same prompt, same read-only exploration.
522
658
  */
523
659
  /** Marks a test whose failure the repair judged to be the application's. */
660
+ /** The most tests one file may spend on one criterion (config maxTestsPerRule, default 4). */
661
+ export const maxPerRule = () => loadConfig().maxTestsPerRule ?? 2;
524
662
  export const DEFECT_MARK = 'testforge-defect:';
663
+ /** A failure that is real application behaviour against a [baseline]/[observed] rule: kept failing, never a defect. */
664
+ export const FINDING_MARK = 'testforge-finding:';
525
665
  /**
526
- * Repair a generated test from the runner's own failure output.
527
- *
528
- * The failure is evidence the generator did not have: the real selector, the
529
- * real URL shape, the real column type. So the repair prompt leads with it
530
- * and forbids the two easy ways out — deleting the assertion, or skipping the
531
- * test — because either turns a caught defect into a silent one.
666
+ * Tests for some rules of a file that already exists: new rules, or rules
667
+ * whose wording changed. The AI sees the file's top (imports, setup), the old
668
+ * tests of a changed rule, and the slice for those rules only, and answers
669
+ * with test blocks that are put into the file in place. The rest of the file
670
+ * is neither sent nor rewritten.
532
671
  */
533
- export async function repairWithAIAsync(journey, testPath, current, failure, harnessPath, transport, grounding) {
534
- const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
535
- const base = buildPrompt(journey, map, testPath, undefined, harnessPath, transport, grounding);
672
+ export async function writeTestsWithAI(journey, testPath, fileHead, title, oldTests, harnessPath, transport, grounding, memory, refusal) {
536
673
  const prompt = [
537
- base,
538
- '',
539
- '=== REPAIR ===',
540
- `The file below was generated for ${testPath} and RUN against the real`,
541
- 'application. The tests listed under RUN RESULT did not pass. The runner',
542
- 'output is the ground truth: it tells you what the application actually',
543
- 'renders, routes and returns.',
674
+ buildPrompt(journey, '', testPath, { title, covers: `Only the rules ${memory.rules.join(', ')}.` }, harnessPath, transport, grounding, memory),
544
675
  '',
545
- 'Decide, for EACH listed test, which of these it is — from the evidence,',
546
- 'not from what you hope:',
547
- ' TEST FAULT — the test never reached the behaviour it checks, or checked',
548
- ' it wrongly: a locator that matched nothing or the wrong',
549
- ' element, a container that does not hold the element, a wait',
550
- ' for the wrong event, a response field the server does not',
551
- ' send, a precondition its own setup got wrong, an assertion',
552
- ' the journey document does not state. FIX the test.',
553
- ' APP DEFECT — the test reached the behaviour (the element or response it',
554
- ' checks is there) and the application contradicts a criterion',
555
- ' the journey document states. Change NOTHING in that test.',
556
- ` Put one comment line directly above its declaration:`,
557
- ` // ${DEFECT_MARK} <the criterion, quoted with its tag> — <what the app did instead>`,
558
- ' SKIPPED — a test listed under SKIPPED skipped itself. If its own',
559
- ' setup looked in the wrong place (wrong field, wrong route,',
560
- ' wrong id), fix that so it runs. If what it needs truly does',
561
- ' not exist in this environment, leave it exactly as it is.',
676
+ '--- THIS FILE ALREADY EXISTS ---',
677
+ 'Its top (imports and setup), which your tests run under:',
678
+ fileHead,
562
679
  '',
563
- 'Rules, in addition to every rule above:',
564
- ' 1. Never delete, rename, weaken or invert an assertion, and never delete',
565
- ' or rename a test.',
566
- ' 2. Never add test.skip / test.fixme / try-catch to make a failure go away.',
567
- ' 3. Do not touch a test that is not listed below.',
568
- ' 4. When in doubt between TEST FAULT and APP DEFECT, look again at the',
569
- ' component and server source: an element that is in the DOM but hidden,',
570
- ' a message the server sends but the screen does not show, a value the',
571
- ' screen accepts but the server refuses — those are the application.',
680
+ oldTests
681
+ ? `These tests were written for an earlier wording of ${memory.rules.join(', ')}. Rewrite them for the rules as they are now:\n${oldTests}`
682
+ : `The file already tests its other rules. Write tests only for ${memory.rules.join(', ')}.`,
572
683
  '',
573
- '--- CURRENT FILE ---',
574
- current,
575
- '--- RUN RESULT ---',
576
- failure,
577
- '--- END ---',
684
+ 'Output ONLY the test(...) blocks, each with its "// oracle:" comment above it: no imports, no',
685
+ 'describe, no setup. The first character of your answer is the first line of code.',
686
+ ].join('\n');
687
+ const config = loadConfig();
688
+ const { program, args, input } = writerCommand(withRefusal(prompt, refusal), memory);
689
+ return cleanOutput(await runWithStdin(program, args, input, config.aiTimeoutMs));
690
+ }
691
+ /** The line that opens each file in a batch answer. */
692
+ const FILE_MARK = (path) => `// ===== FILE: ${path} =====`;
693
+ /**
694
+ * Several files of one journey in one call. Their shared part (the rules for
695
+ * writing a test, the harness, the testkit) is the cached system prompt once;
696
+ * each file adds only its own facts, rules and task. The answer holds every
697
+ * file after its marker line; a file missing from it is simply not returned.
698
+ */
699
+ export async function writeFilesWithAI(files) {
700
+ const parts = files.map((f) => ({ ...f, ...splitShared(f.prompt) }));
701
+ const system = parts[0].system;
702
+ const message = [
703
+ `Write ${files.length} test files. Each one's facts, rules and task follow; the`,
704
+ `rules for writing a test are the same for all of them.`,
578
705
  '',
579
- 'Output ONLY the corrected file. No commentary.',
706
+ ...parts.flatMap((p, k) => [`=== FILE ${k + 1} of ${files.length}: ${p.path} ===`, p.message, '']),
707
+ `OUTPUT: all ${files.length} files, in this order. Before each, one line exactly:`,
708
+ ...files.map((f) => ` ${FILE_MARK(f.path)}`),
709
+ `then that file's code. Nothing before the first line and nothing after the last file.`,
580
710
  ].join('\n');
581
711
  const config = loadConfig();
582
- const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
583
- const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
584
- return cleanOutput(stdout);
712
+ const cmd = splitCommand(config.aiCommand);
713
+ const { program, args } = system ? withNoTools(cmd, system) : withNoTools(cmd);
714
+ return splitBatchAnswer(await runWithStdin(program, args, message, config.aiTimeoutMs), files.map((f) => f.path));
585
715
  }
586
- export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding) {
716
+ /** Each file's code from a batch answer, by its marker line. */
717
+ export function splitBatchAnswer(out, paths) {
718
+ const result = new Map();
719
+ const marks = paths
720
+ .map((path) => ({ path, at: out.indexOf(FILE_MARK(path)) }))
721
+ .filter((m) => m.at >= 0)
722
+ .sort((a, b) => a.at - b.at);
723
+ marks.forEach((m, k) => {
724
+ const end = k + 1 < marks.length ? marks[k + 1].at : out.length;
725
+ const code = cleanOutput(out.slice(m.at + FILE_MARK(m.path).length, end));
726
+ if (code.trim())
727
+ result.set(m.path, code);
728
+ });
729
+ return result;
730
+ }
731
+ export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding, memory,
732
+ /** Why the last answer was refused before it ran; told to the next try. */
733
+ refusal) {
587
734
  const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
588
- const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding);
735
+ const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding, memory);
589
736
  const config = loadConfig();
590
- const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
591
- const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
737
+ const { program, args, input } = writerCommand(withRefusal(prompt, refusal), memory);
738
+ const stdout = await runWithStdin(program, args, input, config.aiTimeoutMs);
592
739
  return cleanOutput(stdout);
593
740
  }
594
741
  /**
@@ -20,6 +20,8 @@ import { pingAi } from './ai-generator.js';
20
20
  // every seed retried against a dead CLI. Record the reason, put it in the
21
21
  // warning, and once the failure is clearly "AI unavailable" stop starting new
22
22
  // journeys: the deferred stubs are re-attempted automatically on the next run.
23
+ /** A limit that resets with the month, not in hours: waiting for it is pointless. */
24
+ const MONTHLY = /monthly (?:spend )?limit|switch to another model/i;
23
25
  let lastAiFailure = null;
24
26
  let aiUnavailable = null;
25
27
  // Process failures in a row with no success between them. The CLI does not
@@ -85,6 +87,12 @@ export async function waitForAi(config) {
85
87
  console.log(`\nAI unavailable: ${aiUnavailable}\n` +
86
88
  `Waiting for it to come back — checking every ${mins(poll)}, for up to ${mins(max)}. ` +
87
89
  `Ctrl-C is safe: re-running later resumes from exactly here.`);
90
+ // A monthly cap does not reset in hours: waiting only burns the day.
91
+ if (aiUnavailable && MONTHLY.test(aiUnavailable)) {
92
+ console.log('This is a MONTHLY limit — it will not come back by waiting. Use another model for TestForge ' +
93
+ '(testforge.config.json "aiCommand": "claude -p --model sonnet"), or ask your admin to raise the limit.');
94
+ return false;
95
+ }
88
96
  const started = Date.now();
89
97
  while (Date.now() - started < max) {
90
98
  await new Promise((r) => setTimeout(r, poll));
@@ -0,0 +1,150 @@
1
+ /**
2
+ * What the AI cost: calls, wall time and an estimate of tokens, counted in
3
+ * one place every AI call goes through. Tokens are estimated as characters
4
+ * sent and received divided by four — close enough to compare journeys and
5
+ * runs, not a bill.
6
+ *
7
+ * Every Claude call is run with its event stream, so the counts above use the
8
+ * real token use and cost it reports. TESTFORGE_TRACE=1 also records every call in order under
9
+ * testforge/runs/trace/<run>/: the prompt, the CLI's full event stream, and
10
+ * one index.jsonl line with the stage, the real token usage and cost the
11
+ * Claude CLI reports, and every tool the AI used (which files it read, what
12
+ * it searched). The characters ÷ 4 estimate misses those reads; this does not.
13
+ */
14
+ import { execFileSync } from 'node:child_process';
15
+ import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
16
+ import { basename, join } from 'node:path';
17
+ import { isClaude } from '../project/config.js';
18
+ import { RUNS_DIR } from '../paths.js';
19
+ const total = { calls: 0, ms: 0, inChars: 0, outChars: 0, tokens: 0, costUsd: 0 };
20
+ export function noteAiCall(inChars, outChars, ms) {
21
+ total.calls += 1;
22
+ total.ms += ms;
23
+ total.inChars += inChars;
24
+ total.outChars += outChars;
25
+ }
26
+ /** A snapshot of the running totals. */
27
+ export function aiUsage() {
28
+ return { ...total };
29
+ }
30
+ /** What was spent between two snapshots. */
31
+ export function usageSince(start, now = aiUsage()) {
32
+ return {
33
+ calls: now.calls - start.calls, ms: now.ms - start.ms, inChars: now.inChars - start.inChars, outChars: now.outChars - start.outChars,
34
+ tokens: now.tokens - start.tokens, costUsd: now.costUsd - start.costUsd,
35
+ };
36
+ }
37
+ export const approxTokens = (u) => Math.round((u.inChars + u.outChars) / 4);
38
+ export function describeUsage(u) {
39
+ const min = u.ms / 60_000;
40
+ const time = min >= 1 ? `${min.toFixed(1)} min` : `${Math.round(u.ms / 1000)} s`;
41
+ // The CLI's own count includes what the AI read with its tools and every
42
+ // turn's re-sent context; characters ÷ 4 sees only our prompt and its answer.
43
+ const tokens = u.tokens
44
+ ? `${u.tokens.toLocaleString('en-GB')} tokens, $${u.costUsd.toFixed(2)}`
45
+ : `~${approxTokens(u).toLocaleString('en-GB')} tokens (estimate)`;
46
+ return `${u.calls} AI call(s), ${time}, ${tokens}`;
47
+ }
48
+ /** execFileSync for an AI CLI, counted. Same signature as the call it replaces. */
49
+ export function runAiSync(program, args, options) {
50
+ const started = Date.now();
51
+ let out = '';
52
+ try {
53
+ // The event stream is larger than the answer alone: give it room.
54
+ out = execFileSync(program, traceArgs(program, args), { ...options, maxBuffer: Math.max(options.maxBuffer ?? 0, 64 * 1024 * 1024) });
55
+ return traceCall(String(options.input ?? ''), out, started);
56
+ }
57
+ catch (e) {
58
+ traceCall(String(options.input ?? ''), String(e.stdout ?? ''), started, String(e));
59
+ throw e;
60
+ }
61
+ finally {
62
+ noteAiCall(String(options.input ?? '').length, out.length, Date.now() - started);
63
+ }
64
+ }
65
+ // ---- trace (TESTFORGE_TRACE=1) ----
66
+ const TRACE_DIR = process.env.TESTFORGE_TRACE
67
+ ? join(RUNS_DIR, 'trace', new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19))
68
+ : '';
69
+ let traceSeq = 0;
70
+ /** Which step a prompt belongs to, from its opening words. Repair first: it embeds the write prompt. */
71
+ const STAGES = [
72
+ [/^=== REPAIR ===$/m, 'repair'],
73
+ [/^You are a senior QA engineer\. Generate a complete/, 'write'],
74
+ [/^You are a senior test architect/, 'plan'],
75
+ [/^You are a senior engineer establishing the MECHANICS/, 'grounding'],
76
+ [/^You review ONE claimed product defect/, 'defect-review'],
77
+ [/^You are a senior SDET writing the SHARED test harness/, 'harness'],
78
+ [/^You are a senior SDET writing a TEST DATA SEED/, 'seed'],
79
+ [/^You are a senior QA engineer creating per-journey documentation/, 'discover'],
80
+ [/^You are a senior QA engineer writing ONE journey doc/, 'discover'],
81
+ [/^Fetch this reference and print its FULL content/, 'reference-fetch'],
82
+ [/^You are a senior QA engineer reviewing ONE journey doc/, 'doc-review'],
83
+ [/^You are a senior QA engineer\. The developer has given you an/, 'reference'],
84
+ [/^You are a senior engineer mapping a codebase/, 'map'],
85
+ [/^You are setting up the QA tool/, 'init'],
86
+ [/^--- (CODE FACTS|TESTKIT)/, 'write'],
87
+ [/^Reply with exactly: PONG/, 'ping'],
88
+ ];
89
+ /**
90
+ * The CLI args, plus the event-stream flags for a Claude call: its result
91
+ * event carries the real token use and cost, which every run counts, and the
92
+ * stream is what a trace saves.
93
+ */
94
+ export function traceArgs(program, args) {
95
+ return isClaude(program) ? [...args, '--output-format', 'stream-json', '--verbose'] : args;
96
+ }
97
+ /**
98
+ * Record one finished call and return its answer text. Without tracing the
99
+ * raw output IS the answer; with it, the answer is the stream's result event.
100
+ */
101
+ export function traceCall(prompt, raw, started, error) {
102
+ const events = raw.split('\n').flatMap((l) => {
103
+ if (!l.startsWith('{'))
104
+ return [];
105
+ try {
106
+ return [JSON.parse(l)];
107
+ }
108
+ catch {
109
+ return [];
110
+ }
111
+ });
112
+ const result = events.find((e) => e.type === 'result');
113
+ const text = events.length ? String(result?.result ?? '') : raw;
114
+ const ru = result?.usage ?? {};
115
+ total.tokens += ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
116
+ total.costUsd += Number(result?.total_cost_usd) || 0;
117
+ if (!TRACE_DIR)
118
+ return text;
119
+ const tools = events
120
+ .filter((e) => e.type === 'assistant')
121
+ .flatMap((e) => e.message?.content ?? [])
122
+ .filter((c) => c.type === 'tool_use')
123
+ .map((c) => {
124
+ const i = c.input ?? {};
125
+ return `${c.name} ${String(i.file_path ?? i.pattern ?? i.path ?? i.command ?? JSON.stringify(i)).slice(0, 200)}`;
126
+ });
127
+ const seq = String(++traceSeq).padStart(3, '0');
128
+ const stage = STAGES.find(([re]) => re.test(prompt))?.[1] ?? 'ai';
129
+ const file = /Save this file at EXACTLY: (\S+)/.exec(prompt)?.[1];
130
+ const name = `${seq}-${stage}${file ? `-${basename(file).replace(/\.[^.]+$/, '').replace(/\.test$/, '')}` : ''}`;
131
+ try {
132
+ mkdirSync(TRACE_DIR, { recursive: true });
133
+ writeFileSync(join(TRACE_DIR, `${name}.prompt.md`), prompt);
134
+ writeFileSync(join(TRACE_DIR, `${name}.${events.length ? 'stream.jsonl' : 'out.txt'}`), raw);
135
+ const u = result?.usage ?? {};
136
+ appendFileSync(join(TRACE_DIR, 'index.jsonl'), JSON.stringify({
137
+ seq, stage, file: file ? basename(file) : undefined,
138
+ startedAt: new Date(started).toISOString(), ms: Date.now() - started,
139
+ promptChars: prompt.length, answerChars: text.length,
140
+ turns: result?.num_turns, costUsd: result?.total_cost_usd,
141
+ inputTokens: u.input_tokens, outputTokens: u.output_tokens,
142
+ cacheReadTokens: u.cache_read_input_tokens, cacheWriteTokens: u.cache_creation_input_tokens,
143
+ tools, error,
144
+ }) + '\n');
145
+ }
146
+ catch (e) {
147
+ console.warn(` trace: could not record call ${seq}: ${e.message}`);
148
+ }
149
+ return text;
150
+ }