@activate-core/test-forge 0.11.0 → 0.12.0

package/dist/core/ai-generator.js

@@ -18,22 +18,40 @@ const MAP_PATH = 'testforge.md';
  * Spawn a command, feed `input` on stdin, resolve with its stdout.
  * Async (unlike execFileSync) so many calls can run concurrently.
  */
-function runWithStdin(program, args, input) {
+function runWithStdin(program, args, input, timeoutMs = 300_000) {
     return new Promise((resolve, reject) => {
         const child = spawn(program, args, {
             stdio: ['pipe', 'pipe', 'pipe'],
         });
         let out = '';
         let err = '';
+        let done = false;
+        const finish = (fn) => {
+            if (done)
+                return;
+            done = true;
+            clearTimeout(timer);
+            fn();
+        };
+        // A blocked AI CLI (interactive auth, TTY wait, rate-limit, hang) would
+        // otherwise never close — kill it and surface a clear, resumable error.
+        const timer = setTimeout(() => {
+            child.kill('SIGKILL');
+            finish(() => reject(new Error(`${program} timed out after ${timeoutMs}ms — the AI CLI did not respond. ` +
+                `Check it runs non-interactively (auth set up, no TTY prompt), ` +
+                `or raise aiTimeoutMs in testforge.config.json.`)));
+        }, timeoutMs);
         child.stdout.setEncoding('utf-8');
         child.stdout.on('data', (d) => (out += d));
         child.stderr.on('data', (d) => (err += d));
-        child.on('error', reject);
+        child.on('error', (e) => finish(() => reject(e)));
         child.on('close', (code) => {
-            if (code === 0)
-                resolve(out);
-            else
-                reject(new Error(`${program} exited ${code}: ${err.slice(0, 500)}`));
+            finish(() => {
+                if (code === 0)
+                    resolve(out);
+                else
+                    reject(new Error(`${program} exited ${code}: ${err.slice(0, 500)}`));
+            });
         });
         child.stdin.end(input);
     });
@@ -106,15 +124,44 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `route file, its controller/handler, the service it calls, and the`,
         `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
         `request/response shapes, status codes, auth requirements, and function`,
-        `signatures from the code. Base every assertion on what the code`,
-        `actually does. Only fall back to a pending test (${pending}) when`,
-        `the behavior genuinely cannot be determined from the source — do not`,
-        `leave something pending that a quick read of the code would answer.`,
+        `signatures from the code — that is the MECHANICS: HOW to reach and`,
+        `drive the behaviour (paths, methods, payload/response shape,`,
+        `selectors, model/collection names). But the EXPECTED OUTCOME you`,
+        `assert on is the ORACLE, and its source of truth is the JOURNEY`,
+        `DOCUMENT below — NOT what the code currently does. The code tells you`,
+        `HOW; the document tells you WHAT is correct. If the code's real`,
+        `behaviour contradicts the document, assert the DOCUMENT's expectation`,
+        `and let the test FAIL — a real bug surfaced is the goal, never`,
+        `something to paper over. NEVER reshape an assertion to match observed`,
+        `code behaviour just to make it green; that rubber-stamps the bug. If`,
+        `the document does not state the expected outcome for a case, make it`,
+        `pending (${pending}) with a "// TODO:" note — do not invent the`,
+        `expectation from the code.`,
+        `REQUEST URLS: build every request path as the FULL mounted route the`,
+        `router actually registers — INCLUDING the version/area prefix (e.g.`,
+        `/v{VERSION}/<area>/<action>, reading the real version from its env var).`,
+        `Never post to a bare "/<action>": a missing prefix 404s and the test`,
+        `fails for a fake reason. Use the SAME prefix consistently for every`,
+        `call in the file.`,
+        `AUTH & SCOPING: a route may be declared auth:false yet STILL populate`,
+        `request.user when a Bearer token is sent (a pre-auth extension). So`,
+        `owner-scoped data ("my bookings/payouts") is derived from the TOKEN`,
+        `owner — NOT from a query param. Do NOT assume a "?ownerId=/?makerId="`,
+        `fallback overrides the token for an AUTHENTICATED call: the testkit`,
+        `seeds the owner fixtures under the authenticated identity, so expect`,
+        `the token owner's rows. A second, non-owner identity (for IDOR/scope`,
+        `tests) needs a distinct token; if unavailable, make the case pending.`,
         `Do NOT write or edit any file; output only the test code to stdout.`,
         `EVERY runnable test MUST contain at least one real assertion. If you`,
         `genuinely cannot assert an outcome, make the case pending (${pending})`,
         `— never write a test body with no assertion, which passes without`,
         `testing anything.`,
+        `ASSERTION DEPTH: assert the REAL end-state — persisted DB/state, the`,
+        `actual response body/status, or observable UI — not merely that "no`,
+        `error was thrown". CRITICAL: if you mock an external/API response, do`,
+        `NOT then "verify" a value your own mock hard-coded (asserting the mock`,
+        `returned what you told it to proves nothing). Read back the real effect`,
+        `(DB row, downstream call, UI change) the behaviour was supposed to have.`,
         ``,
         isApi
             ? [
@@ -126,26 +173,52 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
                 `duplicate exists, correct fields) using the database/ORM the`,
                 `project uses. Connect via the env var named in the map (e.g.`,
                 `process.env.DB_URL) — never hardcode a connection string.`,
-                `MOCKING: if the code calls an external service (e.g. an identity`,
-                `or payment API), mock it AT THE SEAM THE CODE ACTUALLY USES —`,
-                `read the source to see which client it imports (e.g. axios) and`,
-                `how it is called, then jest.mock that exact module so the mock`,
-                `truly intercepts. Do NOT mock a client the code doesn't use, and`,
-                `do NOT drive the server out-of-process while expecting an`,
+                `MOCKING (MANDATORY — the test must be HERMETIC): the test must`,
+                `NEVER depend on a live external service. For EVERY external`,
+                `service the exercised code reaches (identity/IM, payment/Stripe,`,
+                `SMS, email, etc.), mock it AT THE SEAM THE CODE ACTUALLY USES —`,
+                `read the source to see which client it imports (e.g. axios, the`,
+                `stripe SDK) and how it is called, then jest.mock that EXACT module`,
+                `so the mock truly intercepts. A test that leaves any such call`,
+                `unmocked (so it would hit a real host like localhost/an IM/Stripe`,
+                `URL) is WRONG — it will fail whenever that service is down and is`,
+                `not deterministic. Do NOT mock a client the code doesn't use.`,
+                `HERMETIC BASE URLS: if the client's target is built from an env`,
+                `var (e.g. process.env.IM_API_URL), do NOT rely on the ambient`,
+                `value — set it yourself in beforeAll to a fixed test host AND`,
+                `mock that same client, so every run behaves identically`,
+                `regardless of how the environment is configured. Prefer mocking`,
+                `the client module directly (host-agnostic) over matching a URL.`,
+                `Do NOT drive the server out-of-process while expecting an`,
                 `in-process mock to fire — they cannot both be true.`,
             ].join('\n')
             : [
                 `Because this is a web journey, write Playwright tests that drive`,
-                `the UI. FIRST find the REAL selectors: with your read-only tools,`,
-                `locate and READ this page's actual React/Vue components (grep the`,
-                `webapp source for the route, page name, and on-screen text, then`,
-                `open those component files). Use the selectors that genuinely`,
-                `exist there — data-testid/data-cy, input name/id, aria-role,`,
-                `label text, or button text — via Playwright locators`,
-                `(getByRole/getByLabel/getByTestId/getByText). Only leave a`,
-                `"// TODO: confirm selector" when the element truly cannot be`,
-                `found in the component source; do NOT default everything to a`,
-                `TODO. Never invent a selector as if it were real.`,
+                `the UI. FIRST read this page's real components: grep the webapp`,
+                `source for the route, page name, and on-screen text, then open`,
+                `those component files and use only locators that GENUINELY exist.`,
+                `SELECTOR PREFERENCE — most apps have NO dedicated test hooks, so`,
+                `do NOT reach for a data-testid/data-cy unless you have SEEN that`,
+                `exact attribute in the component source. Prefer, in this order:`,
+                `  1. getByRole(role, { name }) — real roles / accessible names`,
+                `  2. getByLabel(...) or getByPlaceholder(...) — real <label>/placeholder`,
+                `  3. getByText(...) — real on-screen text`,
+                `  4. locator('[name="…"]') / by id — a real input name/id attribute`,
+                `  5. getByTestId(...) / [data-cy=…] — ONLY if that hook truly exists`,
+                `The first four are present in almost any real DOM without special`,
+                `hooks, so a page with zero data-testids is still fully groundable —`,
+                `ground on role/label/text/name instead of inventing a test-id.`,
+                `NEVER invent a data-testid/data-cy: an unverified test-id is`,
+                `REJECTED by the selector-grounding gate and wastes the whole file.`,
+                `Only when an element cannot be identified by ANY real handle,`,
+                `defer that single case with test.fixme(...) and a`,
+                `"// TODO: confirm selector" note — do not default to TODO, never guess.`,
+                `NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
+                `never a description/comment, and never a route-file placeholder`,
+                `like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
+                `value (a seeded id/code, or the concrete route) using a backtick`,
+                "template, e.g. page.goto(`${base}/s/${code}`). A goto with prose",
+                `or an unfilled [param]/:param is rejected by the gate.`,
                 `CRITICAL Playwright API: import { test, expect } from`,
                 `'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
                 `in Playwright — using them throws at collection and breaks the`,
@@ -161,7 +234,11 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
                 `different name: \`import { request as apiRequest } from`,
                 `'@playwright/test'\` then \`const ctx = await`,
                 `apiRequest.newContext({ baseURL, extraHTTPHeaders })\`. Never call`,
-                `.newContext() on the fixture.`,
+                `.newContext() on the fixture, and there is NO \`.apiRequest\``,
+                `property on the \`playwright\` fixture — \`playwright.apiRequest`,
+                `.newContext(...)\` throws "Cannot read properties of undefined".`,
+                `The ONLY correct source of newContext is the imported request`,
+                `module (aliased as apiRequest above).`,
             ].join('\n'),
         ``,
         ...(testPath
@@ -234,11 +311,16 @@ export function generateWithAI(journey, testPath, focus) {
     // Allow read-only exploration so the model grounds the test in the
     // real source (routes/controllers/models); Write stays denied, so it
     // still can't create files — it reads, then prints the code.
-    const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
+    const config = loadConfig();
+    const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
     const output = execFileSync(program, args, {
         input: prompt,
         encoding: 'utf-8',
         maxBuffer: 10 * 1024 * 1024,
+        // Same anti-hang guard as the async path: a blocked CLI throws here
+        // instead of freezing the run forever.
+        timeout: config.aiTimeoutMs ?? 300_000,
+        killSignal: 'SIGKILL',
     });
     return cleanOutput(output);
 }
@@ -250,8 +332,9 @@ export function generateWithAI(journey, testPath, focus) {
 export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
     const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport);
-    const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
-    const stdout = await runWithStdin(program, args, prompt);
+    const config = loadConfig();
+    const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
+    const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
     return cleanOutput(stdout);
 }
 /** Does this line look like the start (or body) of real code? */

package/dist/core/config.js

@@ -30,6 +30,8 @@ import { join } from 'node:path';
 const CONFIG_PATH = 'testforge.config.json';
 const DEFAULT_CONFIG = {
     aiCommand: 'claude -p',
+    aiTimeoutMs: 300_000,
+    aiConcurrency: 4,
     testOutput: {},
     framework: { api: 'jest', web: 'playwright' },
     seedOutput: 'generated-tests/seed',

package/dist/core/detect.js

@@ -12,7 +12,7 @@
  */
 import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { builtinModules } from 'node:module';
-import { dirname, join } from 'node:path';
+import { dirname, extname, join } from 'node:path';
 /** Read a package.json's dependency names (all dependency fields). */
 function depsOf(pkgPath) {
     try {
@@ -94,6 +94,83 @@ export function installedModules(fromDir) {
     }
     return mods;
 }
+/**
+ * The set of UI test-hook values (`data-testid` / `data-cy` / `data-test`)
+ * that genuinely exist in a package's component source — extracted by
+ * reading the real files, so it works on ANY project. Feeds the F10
+ * selector-grounding gate: a generated UI test may only assert on a hook
+ * that is actually in the components; anything else is an invented selector.
+ *
+ * Static string attributes only (`data-testid="pay-now"`). Dynamic ones
+ * (`data-testid={expr}`) can't be resolved statically and are skipped — the
+ * gate stays lenient (only the placeholder-marker check applies to those).
+ * Bounded walk so a huge monorepo can't stall generation.
+ */
+export function projectSelectors(fromDir) {
+    const root = packageRoot(fromDir);
+    const found = new Set();
+    const SRC_EXT = new Set(['.jsx', '.tsx', '.vue', '.svelte', '.js', '.ts', '.html']);
+    const SKIP_DIR = new Set([
+        'node_modules', '.git', 'dist', 'build', '.next', 'out',
+        'coverage', '.turbo', '.cache', 'tests', 'test', '__tests__', '__mocks__',
+    ]);
+    const attrRe = /\bdata-(?:testid|cy|test)\s*=\s*(["'])([^"']+)\1/g;
+    let filesLeft = 4000;
+    const walk = (dir) => {
+        if (filesLeft <= 0)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (filesLeft <= 0)
+                return;
+            if (e.name.startsWith('.'))
+                continue;
+            const full = join(dir, e.name);
+            if (e.isDirectory()) {
+                if (!SKIP_DIR.has(e.name))
+                    walk(full);
+                continue;
+            }
+            if (!SRC_EXT.has(extname(e.name)))
+                continue;
+            filesLeft--;
+            let content;
+            try {
+                content = readFileSync(full, 'utf-8');
+            }
+            catch {
+                continue;
+            }
+            if (content.length > 400_000)
+                continue; // skip bundles/minified
+            let m;
+            attrRe.lastIndex = 0;
+            while ((m = attrRe.exec(content)) !== null)
+                found.add(m[2]);
+        }
+    };
+    walk(root);
+    return found;
+}
+/** Nearest ancestor directory (incl. self) that has a package.json. */
+function packageRoot(fromDir) {
+    let dir = fromDir;
+    for (let i = 0; i < 12; i++) {
+        if (existsSync(join(dir, 'package.json')))
+            return dir;
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return fromDir;
+}
 /**
  * A concrete instruction for how a backend (Jest) suite should reach the
  * server, based on what the project actually depends on — so the AI does

package/dist/core/harness.js

@@ -24,20 +24,57 @@ const MAP_PATH = 'testforge.md';
 export const HARNESS_CONTRACT = [
     `The testkit module MUST export exactly this stable API (CommonJS`,
     `module.exports), implemented against the project's real data layer:`,
-    `  - connect(): Promise<void>      open the DB connection (env DB var`,
-    `                                  from the map, e.g. process.env.DB_URL)`,
+    `  - connect(): Promise<void>      open the DB connection(s). Some apps`,
+    `                                  span MULTIPLE databases (e.g. a main DB,`,
+    `                                  a separate chat/conversation DB, and an`,
+    `                                  identity/IM DB) — READ the real data`,
+    `                                  config to find how many and their env-var`,
+    `                                  names, and connect to EACH. Do NOT assume`,
+    `                                  a single process.env.DB_URL.`,
     `  - disconnect(): Promise<void>   close it`,
-    `  - reset(): Promise<void>        remove all test data (only records`,
-    `                                  with the "qa-"/"QA-TEST" prefix)`,
+    `  - reset(): Promise<void>        remove ONLY test data (records with the`,
+    `                                  "qa-"/"QA-TEST" prefix or the well-known`,
+    `                                  TEST_IDS), cascading children by`,
+    `                                  projectId. MUST be gated — see SAFETY.`,
     `  - seed(spec?): Promise<object>  idempotently upsert the test records`,
     `                                  the tests need (using the real models`,
     `                                  /collections + enums), returning the`,
-    `                                  created ids/handles`,
-    `  - TEST_IDS: object              the stable well-known test ids/handles`,
+    `                                  created ids/handles. Seed EACH record`,
+    `                                  into the database it actually belongs to`,
+    `                                  (e.g. the identity/IM user in the IM DB,`,
+    `                                  a conversation in the chat DB) — not all`,
+    `                                  into one DB. Upsert with { $set: mutable,`,
+    `                                  $setOnInsert: { createdAt } } — never`,
+    `                                  replaceOne, never clobber creation`,
+    `                                  metadata or a pre-existing richer record.`,
+    `                                  Seeds are TRUSTED fixtures: do NOT let the`,
+    `                                  app's own schema quirks block them — if an`,
+    `                                  update-validator rejects a value the app`,
+    `                                  itself defaults (e.g. a required String`,
+    `                                  whose default is ''), write WITHOUT update`,
+    `                                  validators (runValidators:false) rather`,
+    `                                  than failing the whole seed.`,
+    `  - TEST_IDS: object              the stable well-known test ids/handles.`,
+    `                                  EVERY id MUST be env-overridable: resolve`,
+    `                                  each as process.env.TEST_<NAME>_ID ||`,
+    `                                  '<stable default>' (e.g. TEST_PROJECT_ID,`,
+    `                                  TEST_WEARER_ID). This lets a caller point`,
+    `                                  the suite at data created BY HAND on`,
+    `                                  staging/prod (with SEED_MODE=existing)`,
+    `                                  without editing code. Resolve non-id`,
+    `                                  handles (phone, email, auth subject) the`,
+    `                                  same env-overridable way.`,
     `  - setup(): Promise<void>        THE one-call entry point (idempotent,`,
     `                                  safe to call from every beforeAll AND`,
     `                                  from a Playwright globalSetup). It:`,
-    `                                    1. connect()s + seed()s;`,
+    `                                    1. connect()s; then seed()s — UNLESS`,
+    `                                       process.env.SEED_MODE === 'existing',`,
+    `                                       in which case it SKIPS seed() and`,
+    `                                       reset() and runs against data already`,
+    `                                       in the DB (staging/prod, where the`,
+    `                                       fixtures were created by hand and`,
+    `                                       addressed via the env-overridable`,
+    `                                       TEST_IDS);`,
     `                                    2. MINTS a fresh auth access token at`,
     `                                       runtime — read the project's REAL`,
     `                                       login/refresh auth (controllers/`,
@@ -64,11 +101,66 @@ export const HARNESS_CONTRACT = [
     `    fixture — \`const { request: apiRequest } = require('@playwright/`,
     `    test'); return apiRequest.newContext({ baseURL, extraHTTPHeaders })\`.`,
     `    The \`request\` FIXTURE has NO .newContext(); calling it throws.`,
-    `  - login(page): Promise<void>    drive/seed an authenticated UI session`,
+    `  - login(page): Promise<void>    establish an authenticated UI session`,
+    `                                  the webapp actually accepts. Inject the`,
+    `                                  SAME real minted access token (a valid,`,
+    `                                  decodable JWT — the app gate typically`,
+    `                                  jwtDecode()s it and checks exp) into the`,
+    `                                  EXACT localStorage/cookie keys the app`,
+    `                                  reads (GREP the auth reducer/guard for the`,
+    `                                  key names). A fabricated/opaque token that`,
+    `                                  fails the app's validity check leaves the`,
+    `                                  user logged-out (redirect loop). The token`,
+    `                                  owner MUST be the identity that owns the`,
+    `                                  seeded fixtures (see IDENTITY BINDING), so`,
+    `                                  the UI shows the seeded data.`,
     `Use only env var NAMES for secrets/connection — never literal values.`,
     `The ONLY credentials a developer should have to provide are the DB`,
     `connection, a Stripe test key, and ONE seed/refresh token (or seeded`,
     `login) — everything else setup() derives at runtime.`,
+    ``,
+    `SAFETY — this testkit must NEVER be able to damage a real environment:`,
+    `  - reset() and ANY delete/drop MUST refuse unless BOTH hold: the opt-in`,
+    `    flag process.env.TESTFORGE_ALLOW_RESET === 'true' is set, AND the`,
+    `    connection is clearly NOT production (e.g. host is localhost or the`,
+    `    DB name contains "test"). Otherwise throw a clear Error naming the`,
+    `    target. Default is SAFE: with the flag unset, reset() throws and`,
+    `    deletes nothing.`,
+    `  - When process.env.SEED_MODE === 'existing', reset() and all seed()`,
+    `    WRITES are skipped entirely — the suite runs against data already`,
+    `    present (hand-created on staging/prod), addressed via TEST_IDS. Do`,
+    `    not mutate that data destructively.`,
+    `  - Never widen a delete beyond the "qa-"/"QA-TEST" prefix + TEST_IDS`,
+    `    scope; always cascade children by projectId, never a bare drop.`,
+    ``,
+    `IDENTITY BINDING — the #1 cause of false failures, get this right.`,
+    `Owner-scoped endpoints ("my bookings", "my payouts") derive the owner`,
+    `from the AUTHENTICATED TOKEN, not from a query param — and a route`,
+    `declared auth:false may STILL populate request.user via a pre-auth`,
+    `extension when a Bearer is sent. Therefore:`,
+    `  - setup() MUST resolve the authenticated identity's REAL id (decode`,
+    `    the minted token's subject/_id claim, or call the profile/whoami`,
+    `    endpoint) and seed the OWNER fixtures (the maker/wearer/customer`,
+    `    whose data the tests read) under THAT id — then set the TEST_IDS`,
+    `    owner ids to it.`,
+    `  - Do NOT hardcode an owner id the token does not own: the seeded rows`,
+    `    would never be returned to the authenticated caller, and every`,
+    `    owner-scoped test would fail for a fake reason.`,
+    `  - A scope/IDOR test needing a SECOND, non-owner identity needs a`,
+    `    distinct token (env). If only one identity is available, make those`,
+    `    cases pending — never assert a scope the token cannot actually prove.`,
+    ``,
+    `CONNECTION LIFECYCLE — setup() must be safe to call repeatedly across`,
+    `suites, or one suite's teardown breaks the next (MongoNotConnectedError).`,
+    `  - connect() and setup() MUST check the live connection state before`,
+    `    use (e.g. mongoose.connection.readyState !== 1) and REOPEN if closed.`,
+    `    Do NOT gate reconnection behind a memoized "already set up" promise —`,
+    `    a cached promise that resolved once will NOT reopen a since-closed`,
+    `    connection. Guard on the ACTUAL connection state, not a boolean flag.`,
+    `  - Generated tests must NOT call disconnect() on the shared connection`,
+    `    in an afterAll/afterEach. Leave teardown to a single global teardown`,
+    `    (Playwright globalTeardown / Jest globalTeardown) that runs ONCE after`,
+    `    the whole run — never per-file.`,
 ].join('\n');
 /** Where an area's shared testkit lives, and how tests import it. */
 export function harnessInfo(areaDir, framework, ext) {
@@ -96,7 +188,10 @@ function buildHarnessPrompt(harnessPath, framework) {
         `You are inside the repository with read-only tools (Read, Grep, Glob,`,
         `LS). READ the real DB config, the models/schemas, AND the real auth`,
         `flow (login / refresh-token / OTP controllers + middleware) so setup()`,
-        `mints tokens the way the app actually does. Also GREP the sibling test`,
+        `mints tokens the way the app actually does. Determine whether the app`,
+        `uses MORE THAN ONE database (e.g. a main DB, a chat/conversation DB, an`,
+        `identity/IM DB); if so, connect to and seed EACH via its own env-driven`,
+        `connection — do not collapse them into one. Also GREP the sibling test`,
         `files next to this testkit for every process.env.<NAME> they read`,
         `(especially *TOKEN*, *_ID, base URLs) and make setup() POPULATE those`,
         `exact names — the tests must run without any hand-pasted token.`,
@@ -199,3 +294,54 @@ export function globalSetupContent() {
         ``,
     ].join('\n');
 }
+/**
+ * A fixture/environment guide written next to the testkit. Documents the
+ * two run modes and — crucially — how to run against data created BY HAND
+ * on staging/prod (env-addressable TEST_IDS + SEED_MODE=existing), plus the
+ * safety gates. Deterministic and project-agnostic; the exact per-entity
+ * shape lives in the generated seed()/TEST_IDS.
+ */
+export function fixturesDoc() {
+    return [
+        `# Test fixtures & environments`,
+        ``,
+        `Generated by testforge. The same tests run in two modes.`,
+        ``,
+        `## Modes`,
+        `- **auto** (default — local / CI): the testkit seeds its own fixtures`,
+        `  into a throwaway TEST database and resets them between runs. No manual`,
+        `  setup.`,
+        `- **existing** (staging / production): the testkit does NOT seed or`,
+        `  reset — it runs against data that already exists. Enable with:`,
+        ``,
+        `      SEED_MODE=existing`,
+        ``,
+        `## Running against data you created by hand (staging / prod)`,
+        `1. Create the fixture entities in the target database yourself (a`,
+        `   project, its price line(s), the wearer/maker users, etc.) — mirror`,
+        `   what the seed builders in \`helpers/testkit\` produce.`,
+        `2. Point the suite at them — every well-known id is env-overridable:`,
+        ``,
+        `      TEST_PROJECT_ID=<your project id>`,
+        `      TEST_WEARER_ID=<...>        # and any other TEST_*_ID the suite reads`,
+        ``,
+        `3. Provide auth + connection env (DB url/name(s), a seed/refresh token`,
+        `   or seeded login, Stripe test key).`,
+        `4. Run with \`SEED_MODE=existing\` so nothing is seeded or deleted.`,
+        ``,
+        `## Safety`,
+        `- \`reset()\` and any delete are DISABLED unless BOTH hold:`,
+        `  \`TESTFORGE_ALLOW_RESET=true\` AND the target DB is clearly`,
+        `  non-production (localhost, or a DB name containing "test"). Otherwise`,
+        `  it throws.`,
+        `- In \`SEED_MODE=existing\`, seeding and reset are skipped entirely —`,
+        `  your data is never mutated destructively.`,
+        `- NEVER point auto mode at a production database.`,
+        ``,
+        `## The exact fixture shape`,
+        `The precise fields/enums each fixture needs are defined by \`seed()\``,
+        `and \`TEST_IDS\` in \`helpers/testkit\`. Read those to reproduce a`,
+        `matching entity by hand.`,
+        ``,
+    ].join('\n');
+}

package/dist/core/run-generate.js

@@ -25,8 +25,8 @@ import { generateWithAI, generateWithAIAsync } from './ai-generator.js';
 import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
 import { isSyntaxValid, checkTestGates, analyzeFile } from './validate.js';
-import { installedModules, transportHint } from './detect.js';
-import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, } from './harness.js';
+import { installedModules, transportHint, projectSelectors } from './detect.js';
+import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, } from './harness.js';
 import { hashContent, loadCache, saveCache } from './cache.js';
 import { findMarkdownFiles } from './find-files.js';
 import { loadConfig, testDirForType, resolveFramework } from './config.js';
@@ -175,6 +175,7 @@ export async function runHarness(only) {
         }
         mkdirSync(dirname(info.path), { recursive: true });
         writeFileSync(info.path, kit, 'utf-8');
+        writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
         if (framework === 'playwright') {
             const gsPath = join(info.dir, 'global-setup.js');
             writeFileSync(gsPath, globalSetupContent(), 'utf-8');
@@ -200,10 +201,47 @@ const GEN_MARKER = '@generated by testforge';
  * and tell the user to confirm the spec first.
  */
 function isStubDoc(markdown) {
-    return (/^#.*\btodo\b/im.test(markdown) ||
-        /not covered by the reference/i.test(markdown) ||
-        /confirm (the )?expected behaviou?r/i.test(markdown) ||
-        /todo:\s*not covered/i.test(markdown));
+    // The doc's FIRST heading is its premise. A confirmed journey opens with a
+    // real title (e.g. "# Journey: ..."); a genuine stub opens by declaring
+    // itself uncovered ("# TODO: not covered by the reference ...").
+    //
+    // We deliberately look ONLY at the opening heading. Partial "reference
+    // SILENT" annotations on individual SECTIONS of an otherwise-complete doc,
+    // and trailing "developer verification checklist" sections, are healthy —
+    // the previous whole-document scan wrongly skipped confirmed journeys that
+    // merely carried such notes.
+    const firstHeading = (markdown.match(/^#{1,6}\s+.+$/m) || [''])[0];
+    return /\btodo\b|not covered by the reference|reference[-\s]silent|unconfirmed/i.test(firstHeading);
+}
+/**
+ * True if a generated file is a testforge DEFERRED stub — a placeholder
+ * written when AI generation was unavailable. It is NOT real coverage, so
+ * per-file resume must RE-ATTEMPT it (unlike a real file, which is kept).
+ */
+function isDeferredStub(content) {
+    return content.includes('DEFERRED stub');
+}
+/** True if the journey's output dir already holds any deferred stub. */
+function hasDeferredStubs(testDir, ext) {
+    let files;
+    try {
+        files = readdirSync(testDir);
+    }
+    catch {
+        return false;
+    }
+    for (const f of files) {
+        if (!f.endsWith(ext))
+            continue;
+        try {
+            if (isDeferredStub(readFileSync(join(testDir, f), 'utf-8')))
+                return true;
+        }
+        catch {
+            /* ignore unreadable */
+        }
+    }
+    return false;
 }
 /** Slugs recorded in a journey's existing plan cache (the previous run). */
 function readPlanSlugs(journey) {
@@ -295,6 +333,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
     const cache = loadCache(CACHE_PATH);
     const results = [];
+    // Real UI test-hooks per area, extracted once from the components — feeds
+    // the F10 selector-grounding gate so generated UI tests can only assert on
+    // selectors that genuinely exist (never an invented data-testid).
+    const selectorsByArea = new Map();
     // Shared harness per area — generated once, then every test in that
     // area imports it (so files stop inventing their own helper paths).
     const harnessByArea = new Map();
@@ -318,6 +360,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         }
         mkdirSync(dirname(info.path), { recursive: true });
         writeFileSync(info.path, kit, 'utf-8');
+        writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
         // Playwright areas need a config + globalSetup so the specs can run
         // AND get their tokens minted once before the suite.
         if (framework === 'playwright') {
@@ -366,9 +409,20 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         // AI can't fall back to an uninstalled convention (e.g. supertest).
         const allowedModules = installedModules(testDir);
         const transport = transportHint(testDir, framework);
-        const testGate = (content) => checkTestGates(content, ext, { framework, allowedModules });
-        // Skip if unchanged.
-        if (!force && cache[file] === hash) {
+        // F10 — selector grounding (Playwright areas only): the real test-hooks
+        // in this area's components, so a generated UI test can't invent one.
+        let knownSelectors;
+        if (framework === 'playwright') {
+            knownSelectors = selectorsByArea.get(areaDir);
+            if (!knownSelectors) {
+                knownSelectors = projectSelectors(areaDir);
+                selectorsByArea.set(areaDir, knownSelectors);
+            }
+        }
+        const testGate = (content) => checkTestGates(content, ext, { framework, allowedModules, knownSelectors });
+        // Skip if unchanged — but NOT if the journey still has deferred stubs
+        // to retry (a resume pass fills those in without touching real files).
+        if (!force && cache[file] === hash && !hasDeferredStubs(testDir, ext)) {
             results.push({
                 journeyName: journey.name,
                 testDir,
@@ -421,9 +475,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
         }
         // Generate all focused files CONCURRENTLY (bounded pool). Each AI
-        // call is independent, so a 15-file journey finishes in ~4 waves
-        // instead of 15 serial calls.
-        const generated = await pool(planned, GEN_CONCURRENCY, async (pf) => {
+        // call is independent, so a 15-file journey finishes in several waves
+        // instead of one-at-a-time. Concurrency is configurable: too many
+        // parallel agentic AI runs starve each other and time out, so a flaky
+        // environment can lower `aiConcurrency` (e.g. 2) for reliability.
+        const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
+        const generated = await pool(planned, concurrency, async (pf) => {
             // Flat (jest): prefix the file with the journey so many journeys can
             // share one dir without clashing. Folder (playwright): the journey
             // folder already namespaces it, so the slug alone is enough.
@@ -435,7 +492,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             // (--force ignores this and regenerates.)
             if (!force && existsSync(filePath)) {
                 try {
-                    if (isSyntaxValid(readFileSync(filePath, 'utf-8'), ext)) {
+                    const existing = readFileSync(filePath, 'utf-8');
+                    // Keep a real, parsing file — but a DEFERRED stub is not real
+                    // coverage, so fall through and re-attempt it on this resume pass.
+                    if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
                         return { slug: pf.slug, path: filePath, status: 'ai' };
                     }
                 }

package/dist/core/seed-generator.js

@@ -42,11 +42,22 @@ export function buildSeedPrompt(journeyDoc, seedPath) {
         `the seed code to stdout.`,
         ``,
         `Requirements for the seed you write:`,
-        `- Connect using the env var named in the map (e.g.`,
-        `  process.env.DB_URL). NEVER hardcode a connection string or any`,
-        `  secret.`,
-        `- Make it IDEMPOTENT: use upsert by a stable, well-known test id so`,
-        `  running it repeatedly (fresh DB or already-seeded) is safe.`,
+        `- Connect using the env var(s) named in the map. If the app spans`,
+        `  MULTIPLE databases (e.g. main + chat/conversation + identity/IM),`,
+        `  connect to and seed EACH into the DB it belongs to — do not assume a`,
+        `  single process.env.DB_URL. NEVER hardcode a connection string or secret.`,
+        `- Make it IDEMPOTENT with an explicit field split: upsert by a stable`,
+        `  id using { $set: <mutable fields>, $setOnInsert: { createdAt, ...other`,
+        `  creation-only fields } } (updateOne/findOneAndUpdate, upsert:true). Do`,
+        `  NOT use replaceOne, and do NOT put createdAt/creation metadata in $set`,
+        `  — that clobbers it on every re-run. Seed shared/base entities (e.g. an`,
+        `  identity user) INSERT-ONLY ($setOnInsert only) so a developer's richer`,
+        `  local record is never overwritten.`,
+        `- Every well-known id MUST be env-overridable: read`,
+        `  process.env.TEST_<NAME>_ID (falling back to a stable default), using`,
+        `  the SAME names the testkit's TEST_IDS exposes. This lets the same`,
+        `  fixtures target data created BY HAND on staging/prod without code`,
+        `  edits (used with SEED_MODE=existing, where writes are skipped).`,
         `- Prefix test data clearly (e.g. names/ids starting with "qa-" or`,
         `  "QA-TEST") so it is obvious and easy to clean up.`,
         `- Insert only what THIS feature's tests need as a precondition,`,

package/dist/core/validate.js

@@ -32,12 +32,20 @@ export function checkTestGates(content, _ext, ctx) {
         return "jest area must not import '@playwright/test'";
     }
     // F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
-    // pasted description. Literal args containing whitespace are prose.
+    // pasted description and not an unfilled route-file placeholder. Literal
+    // args with whitespace are prose; `[param]`/`:param`/`{param}`/`<param>`
+    // are route templates that navigate to a garbage literal path at runtime.
+    // (Only quote-delimited literals are inspected — a backtick template like
+    // `/s/${code}` is the correct, resolved form and is intentionally exempt.)
     const gotoRe = /\.goto\(\s*(['"])([\s\S]*?)\1/g;
     let m;
     while ((m = gotoRe.exec(content)) !== null) {
-        if (/\s/.test(m[2])) {
-            return `goto() argument looks like prose, not a URL/path: "${m[2].slice(0, 40)}…"`;
+        const arg = m[2];
+        if (/\s/.test(arg)) {
+            return `goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`;
+        }
+        if (/\[[A-Za-z_]|\/:[A-Za-z_]|\{[A-Za-z_]|<[A-Za-z_]/.test(arg)) {
+            return `goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`;
         }
     }
     // F2 — no silent false-green: a file with RUNNABLE tests must assert
@@ -61,8 +69,139 @@ export function checkTestGates(content, _ext, ctx) {
             }
         }
     }
+    // F11 — hermetic seam (jest/backend): a test that imports a known
+    // EXTERNAL network service (payment/SMS/email/cloud) must mock it, or the
+    // suite silently depends on a live third party — it flakes offline and can
+    // fire real charges/messages. DB clients are intentionally NOT listed:
+    // backend integration tests legitimately use a real (local) database.
+    if (ctx.framework === 'jest') {
+        const EXTERNAL = new Set([
+            'stripe', 'twilio', '@sendgrid/mail', '@sendgrid/client',
+            'nodemailer', 'aws-sdk', '@aws-sdk/client-ses', '@aws-sdk/client-sns',
+        ]);
+        for (const mod of importedModules(content)) {
+            const top = topLevelModule(mod);
+            if (!EXTERNAL.has(top))
+                continue;
+            const mocked = content.includes(`jest.mock('${mod}'`) ||
+                content.includes(`jest.mock("${mod}"`) ||
+                content.includes(`jest.mock('${top}'`) ||
+                content.includes(`jest.mock("${top}"`);
+            if (!mocked) {
+                return `imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`;
+            }
+        }
+    }
+    // F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
+    // on a guessed selector. Unchecked, the generator invents plausible
+    // data-testids that time out at runtime — the journey is never really
+    // exercised, and the red looks like a product failure instead of a test
+    // gap. A guessed selector must be grounded in a real component attribute,
+    // or the test must be explicitly deferred (test.fixme/skip). This is
+    // feature-agnostic — it guards every UI journey, not any one flow.
+    if (ctx.framework === 'playwright') {
+        const active = stripPendingBlocks(content);
+        // (a) The generator's own tell: a selector sitting next to a
+        // placeholder marker on the same line (e.g. `// TODO: real selector`).
+        const marker = String.raw `(?:TODO|FIXME|placeholder|real selector|guess(?:ed)?|unconfirmed|not\s+verified)`;
+        const sel = String.raw `(?:getByTestId\s*\(|\[\s*data-(?:testid|cy|test)\s*=)`;
+        const selNearMarker = new RegExp(`(?:${sel}[^\\n]*?\\b${marker}\\b)|(?:\\b${marker}\\b[^\\n]*?${sel})`, 'i');
+        if (selNearMarker.test(active)) {
+            return 'a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)';
+        }
+        // (b) When the project's real test-hooks are known, any test-id a
+        // runnable test uses that isn't one of them is an invented selector.
+        if (ctx.knownSelectors && ctx.knownSelectors.size > 0) {
+            for (const id of usedTestIds(active)) {
+                if (!ctx.knownSelectors.has(id)) {
+                    return `references selector '${id}' not found in the project's components — ground it or defer the test`;
+                }
+            }
+        }
+    }
     return null;
 }
+/**
+ * Remove pending test blocks (`it/test/describe.skip|fixme|todo(...)`) so the
+ * F9 selector check only ever inspects RUNNABLE code. A guessed selector
+ * inside a deferred test is fine — it's an honest gap, not a false red.
+ * String- and comment-aware paren matching so a `(` inside a body or string
+ * doesn't unbalance the scan.
+ */
+function stripPendingBlocks(content) {
+    const startRe = /\b(?:it|test|describe)\.(?:skip|fixme|todo)\s*\(/g;
+    let out = '';
+    let last = 0;
+    let m;
+    while ((m = startRe.exec(content)) !== null) {
+        const openIdx = m.index + m[0].length - 1; // index of the '('
+        const end = matchParen(content, openIdx);
+        if (end === -1)
+            continue; // unbalanced — leave as-is, don't drop tail
+        out += content.slice(last, m.index);
+        last = end + 1;
+        startRe.lastIndex = last;
+    }
+    out += content.slice(last);
+    return out;
+}
+/**
+ * Given the index of an opening `(`, return the index of its matching `)`,
+ * skipping over quoted strings, template literals, and comments. Returns -1
+ * if no match is found.
+ */
+function matchParen(s, open) {
+    let depth = 0;
+    for (let i = open; i < s.length; i++) {
+        const c = s[i];
+        // skip strings / templates
+        if (c === "'" || c === '"' || c === '`') {
+            const q = c;
+            i++;
+            while (i < s.length && s[i] !== q) {
+                if (s[i] === '\\')
+                    i++;
+                i++;
+            }
+            continue;
+        }
+        // skip comments
+        if (c === '/' && s[i + 1] === '/') {
+            while (i < s.length && s[i] !== '\n')
+                i++;
+            continue;
+        }
+        if (c === '/' && s[i + 1] === '*') {
+            i += 2;
+            while (i < s.length && !(s[i] === '*' && s[i + 1] === '/'))
+                i++;
+            i++;
+            continue;
+        }
+        if (c === '(')
+            depth++;
+        else if (c === ')') {
+            depth--;
+            if (depth === 0)
+                return i;
+        }
+    }
+    return -1;
+}
+/** Test-hook values a file references: getByTestId('x'), [data-testid="x"], data-cy, data-test. */
+function usedTestIds(content) {
+    const ids = new Set();
+    const patterns = [
+        /getByTestId\s*\(\s*(['"`])([^'"`]+)\1/g,
+        /\[\s*data-(?:testid|cy|test)\s*=\s*(['"])([^'"]+)\1\s*\]/g,
+    ];
+    for (const re of patterns) {
+        let m;
+        while ((m = re.exec(content)) !== null)
+            ids.add(m[2]);
+    }
+    return [...ids];
+}
 /**
  * F8 — lightweight coverage stats for one generated file, so a manifest
  * can tell REAL coverage from deferred placeholders (a green run is never

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {