@activate-core/test-forge 0.10.2 → 0.12.0

package/dist/core/ai-generator.js

@@ -18,36 +18,58 @@ const MAP_PATH = 'testforge.md';
  * Spawn a command, feed `input` on stdin, resolve with its stdout.
  * Async (unlike execFileSync) so many calls can run concurrently.
  */
-function runWithStdin(program, args, input) {
+function runWithStdin(program, args, input, timeoutMs = 300_000) {
     return new Promise((resolve, reject) => {
         const child = spawn(program, args, {
             stdio: ['pipe', 'pipe', 'pipe'],
         });
         let out = '';
         let err = '';
+        let done = false;
+        const finish = (fn) => {
+            if (done)
+                return;
+            done = true;
+            clearTimeout(timer);
+            fn();
+        };
+        // A blocked AI CLI (interactive auth, TTY wait, rate-limit, hang) would
+        // otherwise never close — kill it and surface a clear, resumable error.
+        const timer = setTimeout(() => {
+            child.kill('SIGKILL');
+            finish(() => reject(new Error(`${program} timed out after ${timeoutMs}ms — the AI CLI did not respond. ` +
+                `Check it runs non-interactively (auth set up, no TTY prompt), ` +
+                `or raise aiTimeoutMs in testforge.config.json.`)));
+        }, timeoutMs);
         child.stdout.setEncoding('utf-8');
         child.stdout.on('data', (d) => (out += d));
         child.stderr.on('data', (d) => (err += d));
-        child.on('error', reject);
+        child.on('error', (e) => finish(() => reject(e)));
         child.on('close', (code) => {
-            if (code === 0)
-                resolve(out);
-            else
-                reject(new Error(`${program} exited ${code}: ${err.slice(0, 500)}`));
+            finish(() => {
+                if (code === 0)
+                    resolve(out);
+                else
+                    reject(new Error(`${program} exited ${code}: ${err.slice(0, 500)}`));
+            });
         });
         child.stdin.end(input);
     });
 }
 /** Build the prompt we send to the AI for one journey. */
-export function buildPrompt(journey, map, testPath, focus, harnessPath) {
+export function buildPrompt(journey, map, testPath, focus, harnessPath, transport) {
     const config = loadConfig();
     // Opinionated: testforge only generates Jest (api/backend) or Playwright
     // (web/UI) tests — any other framework (e.g. a project's Cypress setup)
     // is clamped to the right one for the journey. Single source of truth in
     // resolveFramework, shared with the output-layout decision.
     const isApi = resolveFramework(config, journey.area, journey.type) === 'jest';
+    // Do NOT assume an HTTP client here — the transport is detected from the
+    // project's real dependencies and passed in (Hapi/Fastify inject,
+    // supertest only if installed, or native http). Hardcoding supertest
+    // was what produced ~half the API suites that never loaded.
     const framework = isApi
-        ? 'Jest (with supertest for HTTP) for an API/backend test'
+        ? 'Jest for an API/backend test'
         : 'Playwright for a web/UI test';
     // Pending-test syntax differs by runner: Jest has test.todo/it.skip;
     // Playwright has NEITHER (test.todo throws) — it uses test.fixme/skip.
@@ -102,11 +124,44 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
         `route file, its controller/handler, the service it calls, and the`,
         `models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
         `request/response shapes, status codes, auth requirements, and function`,
-        `signatures from the code. Base every assertion on what the code`,
-        `actually does. Only fall back to a pending test (${pending}) when`,
-        `the behavior genuinely cannot be determined from the source — do not`,
-        `leave something pending that a quick read of the code would answer.`,
+        `signatures from the code — that is the MECHANICS: HOW to reach and`,
+        `drive the behaviour (paths, methods, payload/response shape,`,
+        `selectors, model/collection names). But the EXPECTED OUTCOME you`,
+        `assert on is the ORACLE, and its source of truth is the JOURNEY`,
+        `DOCUMENT below — NOT what the code currently does. The code tells you`,
+        `HOW; the document tells you WHAT is correct. If the code's real`,
+        `behaviour contradicts the document, assert the DOCUMENT's expectation`,
+        `and let the test FAIL — a real bug surfaced is the goal, never`,
+        `something to paper over. NEVER reshape an assertion to match observed`,
+        `code behaviour just to make it green; that rubber-stamps the bug. If`,
+        `the document does not state the expected outcome for a case, make it`,
+        `pending (${pending}) with a "// TODO:" note — do not invent the`,
+        `expectation from the code.`,
+        `REQUEST URLS: build every request path as the FULL mounted route the`,
+        `router actually registers — INCLUDING the version/area prefix (e.g.`,
+        `/v{VERSION}/<area>/<action>, reading the real version from its env var).`,
+        `Never post to a bare "/<action>": a missing prefix 404s and the test`,
+        `fails for a fake reason. Use the SAME prefix consistently for every`,
+        `call in the file.`,
+        `AUTH & SCOPING: a route may be declared auth:false yet STILL populate`,
+        `request.user when a Bearer token is sent (a pre-auth extension). So`,
+        `owner-scoped data ("my bookings/payouts") is derived from the TOKEN`,
+        `owner — NOT from a query param. Do NOT assume a "?ownerId=/?makerId="`,
+        `fallback overrides the token for an AUTHENTICATED call: the testkit`,
+        `seeds the owner fixtures under the authenticated identity, so expect`,
+        `the token owner's rows. A second, non-owner identity (for IDOR/scope`,
+        `tests) needs a distinct token; if unavailable, make the case pending.`,
         `Do NOT write or edit any file; output only the test code to stdout.`,
+        `EVERY runnable test MUST contain at least one real assertion. If you`,
+        `genuinely cannot assert an outcome, make the case pending (${pending})`,
+        `— never write a test body with no assertion, which passes without`,
+        `testing anything.`,
+        `ASSERTION DEPTH: assert the REAL end-state — persisted DB/state, the`,
+        `actual response body/status, or observable UI — not merely that "no`,
+        `error was thrown". CRITICAL: if you mock an external/API response, do`,
+        `NOT then "verify" a value your own mock hard-coded (asserting the mock`,
+        `returned what you told it to proves nothing). Read back the real effect`,
+        `(DB row, downstream call, UI change) the behaviour was supposed to have.`,
         ``,
         isApi
             ? [
@@ -118,19 +173,52 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
                 `duplicate exists, correct fields) using the database/ORM the`,
                 `project uses. Connect via the env var named in the map (e.g.`,
                 `process.env.DB_URL) — never hardcode a connection string.`,
+                `MOCKING (MANDATORY — the test must be HERMETIC): the test must`,
+                `NEVER depend on a live external service. For EVERY external`,
+                `service the exercised code reaches (identity/IM, payment/Stripe,`,
+                `SMS, email, etc.), mock it AT THE SEAM THE CODE ACTUALLY USES —`,
+                `read the source to see which client it imports (e.g. axios, the`,
+                `stripe SDK) and how it is called, then jest.mock that EXACT module`,
+                `so the mock truly intercepts. A test that leaves any such call`,
+                `unmocked (so it would hit a real host like localhost/an IM/Stripe`,
+                `URL) is WRONG — it will fail whenever that service is down and is`,
+                `not deterministic. Do NOT mock a client the code doesn't use.`,
+                `HERMETIC BASE URLS: if the client's target is built from an env`,
+                `var (e.g. process.env.IM_API_URL), do NOT rely on the ambient`,
+                `value — set it yourself in beforeAll to a fixed test host AND`,
+                `mock that same client, so every run behaves identically`,
+                `regardless of how the environment is configured. Prefer mocking`,
+                `the client module directly (host-agnostic) over matching a URL.`,
+                `Do NOT drive the server out-of-process while expecting an`,
+                `in-process mock to fire — they cannot both be true.`,
             ].join('\n')
             : [
                 `Because this is a web journey, write Playwright tests that drive`,
-                `the UI. FIRST find the REAL selectors: with your read-only tools,`,
-                `locate and READ this page's actual React/Vue components (grep the`,
-                `webapp source for the route, page name, and on-screen text, then`,
-                `open those component files). Use the selectors that genuinely`,
-                `exist there — data-testid/data-cy, input name/id, aria-role,`,
-                `label text, or button text — via Playwright locators`,
-                `(getByRole/getByLabel/getByTestId/getByText). Only leave a`,
-                `"// TODO: confirm selector" when the element truly cannot be`,
-                `found in the component source; do NOT default everything to a`,
-                `TODO. Never invent a selector as if it were real.`,
+                `the UI. FIRST read this page's real components: grep the webapp`,
+                `source for the route, page name, and on-screen text, then open`,
+                `those component files and use only locators that GENUINELY exist.`,
+                `SELECTOR PREFERENCE — most apps have NO dedicated test hooks, so`,
+                `do NOT reach for a data-testid/data-cy unless you have SEEN that`,
+                `exact attribute in the component source. Prefer, in this order:`,
+                `  1. getByRole(role, { name }) — real roles / accessible names`,
+                `  2. getByLabel(...) or getByPlaceholder(...) — real <label>/placeholder`,
+                `  3. getByText(...) — real on-screen text`,
+                `  4. locator('[name="…"]') / by id — a real input name/id attribute`,
+                `  5. getByTestId(...) / [data-cy=…] — ONLY if that hook truly exists`,
+                `The first four are present in almost any real DOM without special`,
+                `hooks, so a page with zero data-testids is still fully groundable —`,
+                `ground on role/label/text/name instead of inventing a test-id.`,
+                `NEVER invent a data-testid/data-cy: an unverified test-id is`,
+                `REJECTED by the selector-grounding gate and wastes the whole file.`,
+                `Only when an element cannot be identified by ANY real handle,`,
+                `defer that single case with test.fixme(...) and a`,
+                `"// TODO: confirm selector" note — do not default to TODO, never guess.`,
+                `NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
+                `never a description/comment, and never a route-file placeholder`,
+                `like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
+                `value (a seeded id/code, or the concrete route) using a backtick`,
+                "template, e.g. page.goto(`${base}/s/${code}`). A goto with prose",
+                `or an unfilled [param]/:param is rejected by the gate.`,
                 `CRITICAL Playwright API: import { test, expect } from`,
                 `'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
                 `in Playwright — using them throws at collection and breaks the`,
@@ -146,7 +234,11 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
                 `different name: \`import { request as apiRequest } from`,
                 `'@playwright/test'\` then \`const ctx = await`,
                 `apiRequest.newContext({ baseURL, extraHTTPHeaders })\`. Never call`,
-                `.newContext() on the fixture.`,
+                `.newContext() on the fixture, and there is NO \`.apiRequest\``,
+                `property on the \`playwright\` fixture — \`playwright.apiRequest`,
+                `.newContext(...)\` throws "Cannot read properties of undefined".`,
+                `The ONLY correct source of newContext is the imported request`,
+                `module (aliased as apiRequest above).`,
             ].join('\n'),
         ``,
         ...(testPath
@@ -160,6 +252,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
                 ``,
             ]
             : []),
+        ...(transport
+            ? [transport, `Do NOT import any library that is not already a dependency of this project.`, ``]
+            : []),
         ...(harnessPath
             ? [
                 `SHARED HARNESS — a testkit already exists at:`,
@@ -216,11 +311,16 @@ export function generateWithAI(journey, testPath, focus) {
     // Allow read-only exploration so the model grounds the test in the
     // real source (routes/controllers/models); Write stays denied, so it
     // still can't create files — it reads, then prints the code.
-    const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
+    const config = loadConfig();
+    const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
     const output = execFileSync(program, args, {
         input: prompt,
         encoding: 'utf-8',
         maxBuffer: 10 * 1024 * 1024,
+        // Same anti-hang guard as the async path: a blocked CLI throws here
+        // instead of freezing the run forever.
+        timeout: config.aiTimeoutMs ?? 300_000,
+        killSignal: 'SIGKILL',
     });
     return cleanOutput(output);
 }
@@ -229,11 +329,12 @@ export function generateWithAI(journey, testPath, focus) {
  * journey's suite can be generated concurrently (each spawns its own
  * `claude` process). Same prompt, same read-only exploration.
  */
-export async function generateWithAIAsync(journey, testPath, focus, harnessPath) {
+export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
-    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath);
-    const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
-    const stdout = await runWithStdin(program, args, prompt);
+    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport);
+    const config = loadConfig();
+    const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
+    const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
     return cleanOutput(stdout);
 }
 /** Does this line look like the start (or body) of real code? */

package/dist/core/config.js

@@ -30,6 +30,8 @@ import { join } from 'node:path';
 const CONFIG_PATH = 'testforge.config.json';
 const DEFAULT_CONFIG = {
     aiCommand: 'claude -p',
+    aiTimeoutMs: 300_000,
+    aiConcurrency: 4,
     testOutput: {},
     framework: { api: 'jest', web: 'playwright' },
     seedOutput: 'generated-tests/seed',

package/dist/core/detect.js

@@ -0,0 +1,210 @@
+/**
+ * testforge — project capability detection (dependency-aware, no guessing).
+ *
+ * Generated tests must fit the project they land in: they may only import
+ * libraries the project actually has, and backend suites must drive the
+ * server the way the project's stack supports (Hapi/Fastify inject,
+ * supertest, or a live HTTP call) — never by assuming a convention.
+ *
+ * Everything here is read from the real files (package.json + node_modules)
+ * walking up from the test location, so it works on ANY project, including
+ * a greenfield one with no existing tests.
+ */
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { builtinModules } from 'node:module';
+import { dirname, extname, join } from 'node:path';
+/** Read a package.json's dependency names (all dependency fields). */
+function depsOf(pkgPath) {
+    try {
+        const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+        const fields = [
+            'dependencies',
+            'devDependencies',
+            'peerDependencies',
+            'optionalDependencies',
+        ];
+        const names = [];
+        for (const f of fields) {
+            const obj = pkg[f];
+            if (obj)
+                names.push(...Object.keys(obj));
+        }
+        return names;
+    }
+    catch {
+        return [];
+    }
+}
+/** Top-level entries of a node_modules dir, incl. scoped (@scope/pkg). */
+function nodeModulesNames(nmDir) {
+    if (!existsSync(nmDir))
+        return [];
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(nmDir);
+    }
+    catch {
+        return [];
+    }
+    for (const e of entries) {
+        if (e.startsWith('.'))
+            continue;
+        if (e.startsWith('@')) {
+            try {
+                for (const sub of readdirSync(join(nmDir, e))) {
+                    if (!sub.startsWith('.'))
+                        out.push(`${e}/${sub}`);
+                }
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        else {
+            out.push(e);
+        }
+    }
+    return out;
+}
+/**
+ * The set of module specifiers a test at `fromDir` may import: every
+ * dependency declared in any package.json from `fromDir` up to the repo
+ * root, every package physically present in node_modules along the way,
+ * and all Node builtins. Union is intentional — being permissive here
+ * only risks NOT rejecting a real dep; the goal is to catch clearly
+ * absent libraries (e.g. supertest) without false-rejecting hoisted ones.
+ */
+export function installedModules(fromDir) {
+    const mods = new Set();
+    for (const b of builtinModules) {
+        mods.add(b);
+        mods.add(`node:${b}`);
+    }
+    let dir = fromDir;
+    for (let i = 0; i < 12; i++) {
+        const pkg = join(dir, 'package.json');
+        if (existsSync(pkg))
+            depsOf(pkg).forEach((d) => mods.add(d));
+        nodeModulesNames(join(dir, 'node_modules')).forEach((d) => mods.add(d));
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return mods;
+}
+/**
+ * The set of UI test-hook values (`data-testid` / `data-cy` / `data-test`)
+ * that genuinely exist in a package's component source — extracted by
+ * reading the real files, so it works on ANY project. Feeds the F10
+ * selector-grounding gate: a generated UI test may only assert on a hook
+ * that is actually in the components; anything else is an invented selector.
+ *
+ * Static string attributes only (`data-testid="pay-now"`). Dynamic ones
+ * (`data-testid={expr}`) can't be resolved statically and are skipped — the
+ * gate stays lenient (only the placeholder-marker check applies to those).
+ * Bounded walk so a huge monorepo can't stall generation.
+ */
+export function projectSelectors(fromDir) {
+    const root = packageRoot(fromDir);
+    const found = new Set();
+    const SRC_EXT = new Set(['.jsx', '.tsx', '.vue', '.svelte', '.js', '.ts', '.html']);
+    const SKIP_DIR = new Set([
+        'node_modules', '.git', 'dist', 'build', '.next', 'out',
+        'coverage', '.turbo', '.cache', 'tests', 'test', '__tests__', '__mocks__',
+    ]);
+    const attrRe = /\bdata-(?:testid|cy|test)\s*=\s*(["'])([^"']+)\1/g;
+    let filesLeft = 4000;
+    const walk = (dir) => {
+        if (filesLeft <= 0)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (filesLeft <= 0)
+                return;
+            if (e.name.startsWith('.'))
+                continue;
+            const full = join(dir, e.name);
+            if (e.isDirectory()) {
+                if (!SKIP_DIR.has(e.name))
+                    walk(full);
+                continue;
+            }
+            if (!SRC_EXT.has(extname(e.name)))
+                continue;
+            filesLeft--;
+            let content;
+            try {
+                content = readFileSync(full, 'utf-8');
+            }
+            catch {
+                continue;
+            }
+            if (content.length > 400_000)
+                continue; // skip bundles/minified
+            let m;
+            attrRe.lastIndex = 0;
+            while ((m = attrRe.exec(content)) !== null)
+                found.add(m[2]);
+        }
+    };
+    walk(root);
+    return found;
+}
+/** Nearest ancestor directory (incl. self) that has a package.json. */
+function packageRoot(fromDir) {
+    let dir = fromDir;
+    for (let i = 0; i < 12; i++) {
+        if (existsSync(join(dir, 'package.json')))
+            return dir;
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return fromDir;
+}
+/**
+ * A concrete instruction for how a backend (Jest) suite should reach the
+ * server, based on what the project actually depends on — so the AI does
+ * not fall back to an uninstalled default (the supertest failure).
+ */
+export function transportHint(fromDir, framework) {
+    if (framework === 'playwright') {
+        return `This is a Playwright area: use the browser + the testkit's apiContext(request) for API calls. Do NOT use supertest.`;
+    }
+    const mods = installedModules(fromDir);
+    const has = (m) => mods.has(m);
+    if (has('@hapi/hapi') || has('hapi')) {
+        return [
+            `TRANSPORT: this is a Hapi backend. Drive the server IN-PROCESS with`,
+            `\`server.inject({ method, url, payload, headers })\` — the same way`,
+            `the project's own tests do. Do NOT use supertest (it is not a`,
+            `dependency and would run a separate process the in-process mocks`,
+            `can't intercept). Require the app's server/init from the real source.`,
+        ].join('\n');
+    }
+    if (has('fastify')) {
+        return [
+            `TRANSPORT: this is a Fastify backend. Drive it IN-PROCESS with`,
+            `\`app.inject({ method, url, payload, headers })\`. Do NOT use supertest.`,
+        ].join('\n');
+    }
+    if (has('supertest')) {
+        return `TRANSPORT: use supertest (it is installed) against the app's exported Express/Koa/HTTP handler.`;
+    }
+    // Express/Koa/etc. without supertest, or unknown: don't invent a dep.
+    return [
+        `TRANSPORT: do NOT import any HTTP-testing library that is not already`,
+        `a dependency of this project. Prefer the framework's own in-process`,
+        `injection if it has one; otherwise use Node's built-in http/fetch`,
+        `against process.env.BASE_URL. Never add a new dependency.`,
+    ].join('\n');
+}