@activate-core/test-forge 0.16.0 → 0.17.0

package/dist/core/ai-generator.js

@@ -57,7 +57,7 @@ function runWithStdin(program, args, input, timeoutMs = 300_000) {
     });
 }
 /** Build the prompt we send to the AI for one journey. */
-export function buildPrompt(journey, map, testPath, focus, harnessPath, transport) {
+export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding) {
     const config = loadConfig();
     // Opinionated: testforge only generates Jest (api/backend) or Playwright
     // (web/UI) tests — any other framework (e.g. a project's Cypress setup)
@@ -118,6 +118,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    shipping"). If the expectation is merely implied, inferred, or`,
         `    "good practice" — including a rule that sounds obviously right —`,
         `    it is NOT an oracle: write it ${pending} with a "// TODO:" note.`,
+        `    But look properly before you conclude it is absent: the requirement`,
+        `    is often stated in another section of the same document, in a §`,
+        `    the text cross-references, or in an acceptance-criteria list rather`,
+        `    than the prose. Only after checking those is it genuinely unstated.`,
         `    An assertion you cannot trace to a stated requirement is a false`,
         `    positive waiting to happen. Anything under a "Suggested (not a`,
         `    requirement…)" heading is explicitly NOT an oracle — never assert`,
@@ -129,6 +133,14 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    test with a one-line reason naming exactly what was missing. Never`,
         `    let a missing precondition reach an assertion and report as a bug.`,
         `    Guard such tests at the top: check the precondition, skip if unmet.`,
+        `    ESTABLISH IT BEFORE YOU SKIP, THOUGH. A precondition is missing far`,
+        `    less often than it looks: the credential may be under a different`,
+        `    env name, the second identity may already exist in the harness, the`,
+        `    fixture may be one seed call away, the service may be on another`,
+        `    port. Look for it the way you would look for a bug, and only skip`,
+        `    when it genuinely is not there — then name exactly what you looked`,
+        `    for. A suite that skips what it could have set up reports nothing`,
+        `    about the product, and nobody can tell that from real coverage.`,
         `    A CREDENTIAL THAT EXISTS IS NOT A CREDENTIAL THAT WORKS. A supplied`,
         `    token expires; treat "present" and "usable" as different checks.`,
         `    Where the harness accepts a pre-minted token, verify it is still in`,
@@ -162,13 +174,30 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    input is accepted and the invalid one is not. An input that only`,
         `    looks tampered turns correct behaviour into a reported breach, which`,
         `    is the most damaging false alarm there is.`,
-        `(6) A STUB MUST NEVER INTERCEPT THE APPLICATION ITSELF. When you fake a`,
-        `    third party, match it by HOST (or an exact URL), never by a substring`,
-        `    of the URL: a pattern like "**evil.example**" also matches your own`,
-        `    page's "?next=https%3A%2F%2Fevil.example", so the app is served the`,
-        `    stub, nothing renders, and the test fails on a screen that never`,
-        `    loaded. Compare hostnames case-insensitively. After installing`,
-        `    stubs, satisfy yourself the page under test still loads.`,
+        `(6) A STUB MUST NEVER INTERCEPT THE APPLICATION ITSELF — AND "I MIGHT`,
+        `    HIT THE APP" IS NOT A REASON TO GIVE UP. Never match a third party by`,
+        `    a substring of the URL: a pattern like "**evil.example**" also matches`,
+        `    your own page's "?next=https%3A%2F%2Fevil.example", so the app is`,
+        `    served the stub, nothing renders, and the test fails on a screen that`,
+        `    never loaded.`,
+        `    Instead, find the NARROWEST thing that tells the dependency's traffic`,
+        `    apart from the app's own, and scope the stub to exactly that. Work`,
+        `    down this list and stop at the first that separates them:`,
+        `      - a different host — match the hostname, case-insensitively;`,
+        `      - the same host on a different PORT — match host AND port: a`,
+        `        service on :4000 is a different origin from an app on :3000, and`,
+        `        scoping to the bare hostname would swallow both;`,
+        `      - the same origin under a different PATH prefix (/socket.io/,`,
+        `        /api/chat/) — match the prefix;`,
+        `      - otherwise a predicate on the request itself: its method, its`,
+        `        resource type, a header only that client sends.`,
+        `    A dependency you cannot separate from the app by ANY of those is the`,
+        `    only one you may leave pending, and then say which of these you`,
+        `    checked. Do not abandon a whole area because the dependency happens to`,
+        `    sit on the same hostname — that is usually just a different port, and`,
+        `    writing it off costs the product its coverage for no reason.`,
+        `    After installing stubs, satisfy yourself the page under test still`,
+        `    loads.`,
         `(7) BIND A FIXTURE TO THE IDENTITY THE SCREEN IS SCOPED BY. Before`,
         `    seeding for a role-scoped page, find the exact key the server scopes`,
         `    on — a user _id, a phone number, an account id — and which sign-in`,
@@ -184,11 +213,25 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    fields that transition writes. A hand-assembled record combining`,
         `    fields no code path puts together is not a test case — the product`,
         `    owes it no behaviour, so whatever it does with it proves nothing.`,
-        `    Before seeding a state, find the write that produces it; if there`,
-        `    is none, the scenario is imaginary — drop it.`,
-        `(9) A TIMEOUT IS NOT A BUG REPORT. If an action legitimately waits on`,
-        `    something external, verify availability first and skip when absent.`,
-        `    A test that times out tells the team nothing about the product.`,
+        `    Before seeding a state, find the write that produces it. Search for`,
+        `    it properly — the transition may live in a cron job, a webhook`,
+        `    handler, an admin route or a background worker rather than the`,
+        `    obvious controller, and it may set the field under a different`,
+        `    name. Only when no code path anywhere produces the state is the`,
+        `    scenario imaginary — then drop it and say what you searched.`,
+        `(9) A TIMEOUT IS NOT A BUG REPORT — AND IT IS USUALLY YOU WAITING FOR`,
+        `    THE WRONG THING. Before you accept a wait, ask what event actually`,
+        `    arrives. A page whose job is to REDIRECT never fires a load event on`,
+        `    the URL you asked for, so waiting for one burns the whole timeout on`,
+        `    a page that already left: ask for the response instead and let the`,
+        `    assertions that follow wait for the real outcome. A control the app`,
+        `    hides from the accessibility tree is not there for a role query, so`,
+        `    query it the way the app exposes it. A message that only renders`,
+        `    after a round-trip needs a retrying assertion, not a longer sleep.`,
+        `    Only once you are sure you are waiting for the right thing does`,
+        `    availability come into it: verify it first and skip when absent. A`,
+        `    test that times out tells the team nothing about the product, and`,
+        `    "it timed out" is never the finding — find what it was waiting for.`,
         `(10) THE HARNESS IS SOURCE TOO — READ IT BEFORE YOU CALL IT. Open the`,
         `    helper you are about to use and check what it actually RETURNS: a`,
         `    setup routine that publishes env and returns nothing hands you`,
@@ -214,16 +257,32 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `When rules conflict, prefer reporting LESS: a pending test costs a line`,
         `in the report; a false failure costs the team a release.`,
         ``,
-        `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
-        `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
-        `BEFORE writing tests, look up the actual code for this journey — use`,
-        `the route/endpoint named below to grep for the router, then read the`,
-        `route file, its controller/handler, the service it calls, and the`,
-        `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
-        `request/response shapes, status codes, auth requirements, and function`,
-        `signatures from the code — that is the MECHANICS: HOW to reach and`,
-        `drive the behaviour (paths, methods, payload/response shape,`,
-        `selectors, model/collection names). But the EXPECTED OUTCOME you`,
+        ...(grounding
+            ? [
+                `THE MECHANICS ARE ALREADY ESTABLISHED. The CODE FACTS section`,
+                `below was read from this project's real source for this journey —`,
+                `routes with their full mounted paths, auth behaviour, request and`,
+                `response shapes, models, externals, and the selectors the screens`,
+                `genuinely expose. Use it. Do NOT re-derive it: opening the same`,
+                `controllers and components again for every file in a journey is`,
+                `what makes generation take an hour instead of minutes, and it`,
+                `arrives at the same answer. Read source only for a specific detail`,
+                `the CODE FACTS section marks "unknown" or genuinely does not`,
+                `cover. Those facts are the MECHANICS: HOW to reach and`,
+                `drive the behaviour. But the EXPECTED OUTCOME you`,
+            ]
+            : [
+                `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
+                `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
+                `BEFORE writing tests, look up the actual code for this journey — use`,
+                `the route/endpoint named below to grep for the router, then read the`,
+                `route file, its controller/handler, the service it calls, and the`,
+                `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
+                `request/response shapes, status codes, auth requirements, and function`,
+                `signatures from the code — that is the MECHANICS: HOW to reach and`,
+                `drive the behaviour (paths, methods, payload/response shape,`,
+                `selectors, model/collection names). But the EXPECTED OUTCOME you`,
+            ]),
         `assert on is the ORACLE, and its source of truth is the JOURNEY`,
         `DOCUMENT below — NOT what the code currently does. The code tells you`,
         `HOW; the document tells you WHAT is correct. If the code's real`,
@@ -314,7 +373,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
                 `never a description/comment, and never a route-file placeholder`,
                 `like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
                 `value (a seeded id/code, or the concrete route) using a backtick`,
-                "template, e.g. page.goto(`${base}/s/${code}`). A goto with prose",
+                'template, e.g. page.goto(`${base}/s/${code}`). A goto with prose',
                 `or an unfilled [param]/:param is rejected by the gate.`,
                 `CRITICAL Playwright API: import { test, expect } from`,
                 `'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
@@ -339,7 +398,11 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
             ].join('\n'),
         ``,
         ...(transport
-            ? [transport, `Do NOT import any library that is not already a dependency of this project.`, ``]
+            ? [
+                transport,
+                `Do NOT import any library that is not already a dependency of this project.`,
+                ``,
+            ]
             : []),
         ...(harnessPath
             ? [
@@ -378,6 +441,12 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `--- PROJECT MAP (data layer) ---`,
         map,
         ``,
+        // Read from the project's real source once per journey, before any file is
+        // written, and handed unchanged to every file in it. Sits with the map and
+        // the journey so the whole block stays a stable prefix across the journey.
+        ...(grounding
+            ? [`--- CODE FACTS (read from this project's source) ---`, grounding, ``]
+            : []),
         `--- JOURNEY ---`,
         `It may include sections beyond fields — actions, expected results,`,
         `negative cases, race conditions, webhooks, security notes, and a`,
@@ -421,9 +490,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
  * journey. Reads the project map (testforge.md) for data-layer
  * context. Returns the test file content.
  */
-export function generateWithAI(journey, testPath, focus) {
+export function generateWithAI(journey, testPath, focus, grounding) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
-    const prompt = buildPrompt(journey, map, testPath, focus);
+    const prompt = buildPrompt(journey, map, testPath, focus, undefined, undefined, grounding);
     // Allow read-only exploration so the model grounds the test in the
     // real source (routes/controllers/models); Write stays denied, so it
     // still can't create files — it reads, then prints the code.
@@ -445,9 +514,9 @@ export function generateWithAI(journey, testPath, focus) {
  * journey's suite can be generated concurrently (each spawns its own
  * `claude` process). Same prompt, same read-only exploration.
  */
-export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport) {
+export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
-    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport);
+    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding);
     const config = loadConfig();
     const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
     const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
@@ -513,5 +582,8 @@ export function cleanOutput(text) {
             break;
         }
     }
-    return lines.slice(codeStart, codeEnd + 1).join('\n').trim();
+    return lines
+        .slice(codeStart, codeEnd + 1)
+        .join('\n')
+        .trim();
 }

package/dist/core/grounding.js

@@ -0,0 +1,143 @@
+/**
+ * testforge — reading the journey's real code ONCE.
+ *
+ * Every per-file generation used to carry the same instruction: grep for the
+ * router, open the route file, its controller, its service, its models, and —
+ * for a UI journey — the page's components. A journey that splits into
+ * fourteen files therefore read the same handful of files fourteen times, and
+ * each of those reads is an agentic round-trip. That, not the size of the
+ * prompt text, is what made one journey take the better part of an hour.
+ *
+ * The mechanics a test needs — real paths, methods, status codes, model names,
+ * the selectors a page actually exposes — are the same for every file in the
+ * journey. So we establish them once, cache them beside the suite plan, and
+ * hand the same block to each file.
+ *
+ * What this is NOT: an oracle. It records only HOW to reach the behaviour. The
+ * journey document remains the sole source of what is CORRECT, and the
+ * generation prompt says so in the same breath as it presents these facts.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { loadConfig, splitCommand, withReadOnlyTools } from './config.js';
+import { hashProjectSource } from './cache.js';
+import { STATE_DIR } from './paths.js';
+const MAP_PATH = 'testforge.md';
+/** Where a journey's grounded facts live, mirroring the plan cache layout. */
+function groundingPath(journey) {
+    const area = (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-');
+    const slug = (journey.name || 'journey')
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-|-$/g, '');
+    return join(STATE_DIR, 'grounding', area, `${slug}.json`);
+}
+function buildGroundingPrompt(journey, map, srcDir) {
+    const isUi = journey.type !== 'api';
+    return [
+        `You are a senior engineer establishing the MECHANICS of one user`,
+        `journey so tests can be written against it without every writer`,
+        `re-reading the same source.`,
+        ``,
+        `You are inside the project's repository with read-only tools (Read,`,
+        `Grep, Glob, LS). Read the real code for this journey under "${srcDir}":`,
+        `start from the route/endpoint the journey names, then the`,
+        `controller/handler, the service it calls, and the models it touches.`,
+        ``,
+        `Report ONLY what you actually read. Never guess a path, a status code`,
+        `or a field name: a fact invented here is repeated into every test file`,
+        `in the journey, so one wrong line becomes a whole suite that cannot`,
+        `run. If something is genuinely not discoverable, write "unknown" for`,
+        `it rather than filling the gap.`,
+        ``,
+        `Record, as compact plain text under these headings:`,
+        `  ROUTES — every endpoint this journey uses, as the FULL mounted path`,
+        `    the router registers, including any version/area prefix and the`,
+        `    env var the prefix is read from. Give the HTTP method for each.`,
+        `  AUTH — for each route: is it declared authenticated? Does a bearer`,
+        `    still populate the request user via a pre-auth extension? Which`,
+        `    field is ownership scoped on?`,
+        `  SHAPES — the request payload and response body each route actually`,
+        `    returns, and the status codes the handler can emit.`,
+        `  DATA — the models/collections written or read, and the fields that`,
+        `    matter to this journey (including enum values where they exist).`,
+        `  EXTERNALS — any third-party client the code calls, the module name`,
+        `    a test would have to stand in for, and the env var its base URL or`,
+        `    key comes from.`,
+        ...(isUi
+            ? [
+                `  SCREENS — the page/component files this journey renders, and the`,
+                `    locators they GENUINELY expose: roles with accessible names,`,
+                `    label/placeholder text, on-screen text, input name/id`,
+                `    attributes, and any data-testid that truly exists in the`,
+                `    source. Never list a selector you did not see.`,
+            ]
+            : []),
+        ``,
+        `Be terse. This is a reference sheet, not prose: no preamble, no`,
+        `restating the journey, no advice about testing. Aim for well under 200`,
+        `lines. Output the headings and their contents only.`,
+        ``,
+        `--- PROJECT MAP ---`,
+        map,
+        ``,
+        `--- JOURNEY ---`,
+        journey.rawMarkdown || journey.name,
+    ].join('\n');
+}
+/**
+ * Establish (or reuse) the journey's code facts. Cached against the doc hash
+ * AND the source hash: the mechanics only move when the code or the journey
+ * moves, so an unchanged re-run costs nothing.
+ *
+ * Never throws. If the AI call fails we return null and the caller falls back
+ * to telling each file to read the source itself — slower, but still correct.
+ */
+export function groundJourney(journey, docHash, srcDir = 'src') {
+    const path = groundingPath(journey);
+    const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
+    let sourceHash = '';
+    try {
+        sourceHash = hashProjectSource(srcDir);
+    }
+    catch {
+        // unreadable source tree — fall back to keying on the doc alone
+    }
+    const key = `${docHash}:${sourceHash}`;
+    if (existsSync(path)) {
+        try {
+            const cached = JSON.parse(readFileSync(path, 'utf-8'));
+            if (cached.key === key && typeof cached.facts === 'string' && cached.facts.trim()) {
+                return cached.facts;
+            }
+        }
+        catch {
+            // corrupt cache — re-ground
+        }
+    }
+    const config = loadConfig();
+    const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
+    let facts;
+    try {
+        facts = execFileSync(program, args, {
+            input: buildGroundingPrompt(journey, map, srcDir),
+            encoding: 'utf-8',
+            maxBuffer: 10 * 1024 * 1024,
+            timeout: config.aiTimeoutMs ?? 300_000,
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+    if (!facts)
+        return null;
+    try {
+        mkdirSync(dirname(path), { recursive: true });
+        writeFileSync(path, JSON.stringify({ key, facts }, null, 2), 'utf-8');
+    }
+    catch {
+        // best-effort cache
+    }
+    return facts;
+}

package/dist/core/run-generate.js

@@ -24,6 +24,7 @@ import { parseJourney } from './parser.js';
 import { generateTestCases } from './generator.js';
 import { writeDeferredStub } from './playwright-writer.js';
 import { generateWithAI, generateWithAIAsync } from './ai-generator.js';
+import { groundJourney } from './grounding.js';
 import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
 import { isSyntaxValid, checkTestGates, analyzeFile } from './validate.js';
@@ -97,9 +98,9 @@ function generateValidated(generate, ext, gate) {
  * syntax gate + retry) and falls back to the deterministic basic
  * generator if AI output can't be made valid.
  */
-function buildTest(journey, useAI, ext, testPath, framework, focus) {
+function buildTest(journey, useAI, ext, testPath, framework, focus, grounding) {
     if (useAI) {
-        const ai = generateValidated(() => generateWithAI(journey, testPath, focus), ext);
+        const ai = generateValidated(() => generateWithAI(journey, testPath, focus, grounding), ext);
         if (ai !== null) {
             return { content: ai, status: 'ai' };
         }
@@ -459,6 +460,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             ? loadOrPlan(journey, hash)
             : [{ slug: journey.id, title: journey.name, covers: '' }];
         const plannedSlugs = planned.map((p) => p.slug);
+        // Read the journey's real code ONCE, before any file is written, and hand
+        // the same facts to every file. Without this each file re-opened the same
+        // routes, controllers and components — the same answer, N times over, and
+        // the single biggest cost in a multi-file journey. Best-effort: if it
+        // fails, `grounding` stays undefined and each file falls back to reading
+        // the source itself, which is slower but still correct.
+        const grounding = useAI
+            ? groundJourney(journey, hash) ?? undefined
+            : undefined;
         const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
         if (staleRemoved.length) {
             warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
@@ -495,7 +505,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             let content;
             let status;
             if (useAI) {
-                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport), ext, testGate);
+                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, grounding), ext, testGate);
                 if (ai !== null) {
                     content = ai;
                     status = 'ai';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {