@activate-core/test-forge 1.0.1 → 1.0.3

package/dist/cli/index.js

@@ -19,7 +19,7 @@
  *                                      (an auth-gated page can move unseen).
  */
 import { VERSION } from '../core/index.js';
-import { runGenerate } from '../core/generate/run-generate.js';
+import { runGenerate, runHarness } from '../core/generate/run-generate.js';
 import { runTests } from '../core/run/run-tests.js';
 import { runDiscover } from '../core/docs/discover.js';
 import { runMap } from '../core/project/map.js';
@@ -79,7 +79,7 @@ function printUsage() {
     console.log('  testforge generate --force   Regenerate everything');
     console.log('  testforge generate --no-wait  Stop when the AI hits its usage limit (default: wait for the reset and resume)');
     console.log('  testforge generate --no-verify  Skip running each journey once after generating (default: run and repair)');
-    console.log('  testforge generate --journey <name>  Generate just ONE journey');
+    console.log('  testforge generate --journey <name>  Generate just ONE journey', '  testforge generate --harness         Regenerate only the shared harness (testkit), no test files');
     console.log('  testforge generate --area <area>     Generate a whole area (e.g. api)');
     console.log('  testforge discover          Draft journey docs (asks: map or reference)');
     console.log('  testforge discover --journey <name>  Draft just ONE journey doc');
@@ -144,6 +144,13 @@ async function main() {
         });
         process.exitCode = code;
     }
+    else if (command === 'generate' && process.argv.includes('--harness')) {
+        // Regenerate only the shared harness (testkit), leaving every test file
+        // as it is — e.g. after a TestForge upgrade changes what a harness does.
+        console.log('Regenerating the shared test harness...\n');
+        const written = await runHarness(flagValue('--area'));
+        console.log(written.length ? `Written:\n  ${written.join('\n  ')}` : 'No harness written.');
+    }
     else if (command === 'generate') {
         const useAI = !useBasic;
         console.log(useAI

package/dist/core/docs/review.js

@@ -41,6 +41,9 @@ export function buildReviewPrompt(docPath, srcDir, reference) {
         `- Code branches: read the code this journey runs, client and server.`,
         `  Is every validation, error response and conditional branch an`,
         `  [observed] criterion or a Question? Add the missing ones.`,
+        `- [observed]: open the exact code each [observed] criterion rests on and`,
+        `  confirm it states what that code DOES (follow the real state shape and`,
+        `  call path, not a similarly named field). Fix any that misread it.`,
         `- Tiers: does every [spec] criterion quote a real source sentence?`,
         `  Re-tag any that does not. Does any criterion contradict the code`,
         `  with nothing to test (a rule the screen has no way to break)? Turn`,

package/dist/core/generate/ai-generator.js

@@ -297,6 +297,46 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    list, and treat a template placeholder (DUMMY, replace_me, an empty`,
         `    value, a bracketed name) as absent rather than as a credential.`,
         `    Say which actor was missing in the skip reason.`,
+        `(13) A [baseline] CHECK PROVES ROBUSTNESS, NOT A VALUE. "Opens without a`,
+        `    crash" is proved by the page loading with no error and no 5xx — not by`,
+        `    an exact URL, trailing slash, heading text or element the document`,
+        `    does not name. Never add an extra condition the criterion does not`,
+        `    state (e.g. "double-click opens exactly one pop-up" when the document`,
+        `    only says the button opens it), and never assert data in a place the`,
+        `    page does not render it — check where the app shows it first.`,
+        `(14) SCOPE EVERY LOCATOR TO THE JOURNEY'S OWN UI. Global components`,
+        `    (a login pop-up, a header, a cookie bar) share ids and labels with`,
+        `    the page under test — "#password" can match the login modal's field.`,
+        `    Locate inside the journey's form/container, never page-wide by a`,
+        `    bare id or label that another always-mounted component also uses.`,
+        `(15) NOTHING YOU ARE NOT TESTING MAY BLOCK YOU. Apps open their own`,
+        `    pop-ups for a signed-in or new user (promotions, cookie consent,`,
+        `    onboarding tips, notification prompts). Use the harness's login /`,
+        `    storage helpers that pre-suppress them; if one still opens, close it`,
+        `    before you continue. A click intercepted by an overlay the test is`,
+        `    not about is your bug, not the product's.`,
+        `(16) "INSIDE" MEANS THE MIDDLE. To click inside a panel, click its centre`,
+        `    or a non-interactive child — never a corner offset like (5,5): with a`,
+        `    border-radius that point is outside the element and lands on the`,
+        `    overlay behind it.`,
+        `(17) LET THE APP SETTLE BEFORE YOU CHANGE THE SESSION. After a login,`,
+        `    sign-out or navigation the app sends its own requests and may`,
+        `    redirect by itself. Wait for the network to go idle before rotating`,
+        `    a session, logging in elsewhere or navigating away, and assert the`,
+        `    URL the APP chooses — never force a navigation that can cut off the`,
+        `    app's own redirect.`,
+        `(18) ONE SOURCE FOR SEEDED IDS. Read every seeded id and handle from the`,
+        `    harness (TEST_IDS / the values setup() publishes). If a test needs a`,
+        `    private account, override EVERY id that belongs to it, never a`,
+        `    subset — a leftover shared id makes you compare against another`,
+        `    customer's row.`,
+        `(19) VERIFICATION CODES THE TEST CANNOT READ (SMS/email OTP, magic`,
+        `    links, OAuth). Drive the real UI up to the code step, then use the`,
+        `    harness's bypassVerification() so the final verify/register request`,
+        `    is answered with a REAL account and a REAL token; everything after it`,
+        `    (logged-in state, redirect, payment) is then tested for real. Put`,
+        `    "(verification bypassed)" in that test's title. Wrong-code and`,
+        `    expired-code tests do NOT bypass: the real server must refuse them.`,
         `When rules conflict, prefer reporting LESS: a pending test costs a line`,
         `in the report; a false failure costs the team a release.`,
         ``,

package/dist/core/generate/harness.js

@@ -105,6 +105,8 @@ export const HARNESS_CONTRACT = [
     `                                       process.env work with NO hand-`,
     `                                       pasted tokens.`,
     `  - token(role?): string          the current access token accessor`,
+    `  - REQUIRED_PACKAGES, resetRateLimits(), bypassVerification(): see`,
+    `    RUN ENVIRONMENT below.`,
     `Plus, for Playwright suites, ALSO export:`,
     `  - apiContext(request): a helper that returns an authed API request`,
     `    context (Bearer = the minted token — NEVER hardcode a secret).`,
@@ -180,6 +182,54 @@ export const HARNESS_CONTRACT = [
     `    distinct token (env). If only one identity is available, make those`,
     `    cases pending — never assert a scope the token cannot actually prove.`,
     ``,
+    `RUN ENVIRONMENT — the suite must run on a fresh machine given only the`,
+    `env file, in WHATEVER stack this project uses. Decide everything below`,
+    `from what you read in THIS project; nothing here names a stack.`,
+    `  - PACKAGES: use Node built-ins, the test runner, and what the project`,
+    `    already has installed. If reaching the project's data layer needs a`,
+    `    client the project lacks (a driver for its database), require it`,
+    `    lazily, throw "testkit: missing package <name> — run: npm i -D <name>"`,
+    `    when absent, and list it in REQUIRED_PACKAGES (an array literal) so`,
+    `    the runner can check before the suite starts. Never add a package`,
+    `    for anything else.`,
+    `  - NO OTHER RUNTIMES: never shell out to another language runtime, a`,
+    `    framework CLI or a container tool — the test machine need not have`,
+    `    them.`,
+    `  - CREDENTIALS COME FROM THE ENV FILE: every role the suite logs in as`,
+    `    (customer, staff, admin, second customer, …) is an account whose`,
+    `    login identifier and password are read from the env file`,
+    `    (TEST_<ROLE>_EMAIL / TEST_<ROLE>_PASSWORD, or the identifier the app`,
+    `    logs in with). The testkit NEVER hashes or invents a password. The`,
+    `    seed may fill in the profile data such an account needs (addresses,`,
+    `    vehicles, settings) but never its credential.`,
+    `  - EXTRA ACCOUNT STATES (unverified, banned, deleted, another role, an`,
+    `    expired credential): create them the way the app itself would —`,
+    `    through its API where one exists, otherwise through the data layer`,
+    `    by COPYING the stored credential of an env-provided account onto the`,
+    `    new record, then setting the state. Never compute a credential.`,
+    `  - SEED THE DATA TESTS NEED: codes, links, other customers' records —`,
+    `    seeded and exposed through TEST_IDS and the env setup() publishes,`,
+    `    so in auto mode no test waits on hand-made data. DB assertions (rows`,
+    `    created, state persisted) go through the same data-layer client.`,
+    `  - LOG IN ONCE: mint each role's token once per run (globalSetup) and`,
+    `    reuse it; login(page) injects it. If the backend rate-limits and`,
+    `    its limiter state lives in a store the testkit can reach in local`,
+    `    mode, export resetRateLimits() that clears ONLY the limiter entries,`,
+    `    gated exactly like reset(), and call it from setup().`,
+    `  - SUPPRESS THE APP'S OWN POP-UPS: find, in the client, any dialog that`,
+    `    opens by itself for a seeded role (promotions, consent, onboarding,`,
+    `    permission prompts) and have login(page) and seeded browser storage`,
+    `    pre-set the flags that suppress it.`,
+    `  - VERIFICATION BYPASS (Playwright): when the app verifies by a code the`,
+    `    test cannot read (a code sent by message or email, a magic link, a`,
+    `    third-party sign-in), export bypassVerification(page, { url, role }):`,
+    `    route the request at \`url\` (the final verify/register call) and`,
+    `    fulfil it with the app's REAL success response shape (read the`,
+    `    server handler and the client that consumes it) carrying a REAL token`,
+    `    for an env-provided account of that role, obtained through the real`,
+    `    login. Only the code check is skipped; everything after it runs`,
+    `    against the real backend.`,
+    ``,
     `CONNECTION LIFECYCLE — setup() must be safe to call repeatedly across`,
     `suites, or one suite's teardown breaks the next (MongoNotConnectedError).`,
     `  - connect() and setup() MUST check the live connection state before`,

package/dist/core/generate/run-generate.js

@@ -27,7 +27,7 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
 import { generateTestCases } from './generator.js';
 import { writeDeferredStub } from './playwright-writer.js';
 import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
-import { verifyFiles, browserPreflight } from './verify.js';
+import { verifyFiles, browserPreflight, missingPackages } from './verify.js';
 import { packageRoot } from '../run/run-tests.js';
 import { groundJourney } from '../docs/grounding.js';
 import { generateSeed } from './seed-generator.js';
@@ -594,7 +594,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         // is reported and left UNCACHED, so it is retried rather than trusted.
         let verified;
         const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
-        if (useAI && verifyOn && targets.length) {
+        const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
+        if (useAI && verifyOn && targets.length && missing.length) {
+            // The suite cannot even start; running it would only feed repair a
+            // failure no test edit can fix.
+            verified = false;
+            warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
+        }
+        else if (useAI && verifyOn && targets.length) {
             for (let attempt = 0; attempt <= verifyAttempts; attempt += 1) {
                 const outcome = verifyFiles(config, journey.area, targets);
                 if (outcome.infra) {

package/dist/core/generate/verify.js

@@ -13,8 +13,9 @@
  * anything we could infer.
  */
 import { spawnSync } from 'node:child_process';
-import { existsSync } from 'node:fs';
-import { relative } from 'node:path';
+import { existsSync, readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join, relative } from 'node:path';
 import { loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
 import { runnerCommand, packageRoot } from '../run/run-tests.js';
 /** How much runner output to carry into a repair prompt. */
@@ -54,6 +55,10 @@ const INFRA_FAILURES = [
         /Executable doesn't exist at|browserType\.launch:|Host system is missing dependencies|does not support (chromium|firefox|webkit) on/i,
         'the test browser cannot start — install the one this Playwright version needs (`npx playwright install chromium`), or pin a Playwright version your OS supports',
     ],
+    [
+        /missing package|package is not installed|Cannot find module '(?![./])/i,
+        'a package the tests or their harness need is not installed in the project — install it as a dev dependency (the error names it)',
+    ],
     [
         /net::ERR_CONNECTION_REFUSED|ECONNREFUSED/i,
         'the app (or a service it needs) is not reachable — start it and check BASE_URL / the DB settings in the env file',
@@ -138,3 +143,30 @@ export function verifyFiles(config, area, files) {
     }
     return { ran: true, ok: true, output };
 }
+/**
+ * Packages the generated harness declares it needs (REQUIRED_PACKAGES) that
+ * the project cannot resolve. Read from the source, never executed.
+ */
+export function missingPackages(harnessFile, root) {
+    let src = '';
+    try {
+        src = readFileSync(harnessFile, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    const list = src.match(/REQUIRED_PACKAGES\s*=\s*\[([^\]]*)\]/);
+    if (!list)
+        return [];
+    const names = [...list[1].matchAll(/['"]([^'"]+)['"]/g)].map((m) => m[1]);
+    const req = createRequire(join(root, 'package.json'));
+    return names.filter((n) => {
+        try {
+            req.resolve(n);
+            return false;
+        }
+        catch {
+            return true;
+        }
+    });
+}

package/dist/core/project/env-template.js

@@ -15,6 +15,7 @@ import { readFileSync, writeFileSync, existsSync, readdirSync, } from 'node:fs';
 import { join } from 'node:path';
 import { findMarkdownFiles } from '../find-files.js';
 import { STATE_DIR, LEGACY_STATE_DIR } from '../paths.js';
+import { loadConfig } from './config.js';
 /** Pull env var names from a chunk of text. */
 function extractEnvVars(text) {
     const vars = new Set();
@@ -25,6 +26,10 @@ function extractEnvVars(text) {
     for (const m of text.matchAll(/process\.env\[\s*['"]([A-Z0-9_]+)['"]\s*\]/gi)) {
         vars.add(m[1]);
     }
+    // A generated suite often reads env through a small helper: env('TEST_X', default).
+    for (const m of text.matchAll(/\benv\(\s*['"]([A-Z][A-Z0-9_]+)['"]/g)) {
+        vars.add(m[1]);
+    }
     // backtick-wrapped ALL_CAPS names in "env var" lines
     for (const line of text.split('\n')) {
         if (/env var|secret|credential/i.test(line)) {
@@ -189,6 +194,15 @@ export function runEnvTemplate(docsDir, outPath, srcDir = '.', testDirs = []) {
         return;
     }
     const files = findMarkdownFiles(docsDir);
+    // Every caller gets the generated suite scanned, not only the one that
+    // remembered to pass its dirs: the tests' own TEST_* variables are exactly
+    // what a developer cannot guess.
+    const config = loadConfig();
+    if (!testDirs.length) {
+        testDirs = [...Object.values(config.testOutput ?? {}), config.seedOutput ?? '']
+            .filter((d) => !!d && existsSync(d));
+    }
+    const envFile = config.envFile ?? '.env.testing';
     // area -> var -> set of journeys
     const byArea = new Map();
     for (const file of files) {
@@ -254,9 +268,9 @@ export function runEnvTemplate(docsDir, outPath, srcDir = '.', testDirs = []) {
         '# .env.testing.example',
         '#',
         '# Test credentials the generated tests/seeds need, grouped by area.',
-        '# Copy this to .env.testing and fill in real values.',
+        `# Copy this to ${envFile} and fill in real values.`,
         '# Only variables for the features you actually run are required.',
-        '# Keep .env.testing OUT of version control.',
+        `# Keep ${envFile} OUT of version control.`,
         '',
         `# ${'='.repeat(40)}`,
         '# TESTFORGE DASHBOARD (auto-publish each run)',
@@ -296,5 +310,5 @@ export function runEnvTemplate(docsDir, outPath, srcDir = '.', testDirs = []) {
     }
     writeFileSync(outPath, lines.join('\n'), 'utf-8');
     console.log(`Env template written to ${outPath}`);
-    console.log('Copy it to .env.testing and fill in real values.');
+    console.log(`Copy it to ${envFile} and fill in real values.`);
 }

package/dist/core/run/run-tests.js

@@ -37,8 +37,13 @@ export function packageRoot(dir) {
 export function runnerCommand(config, area, framework, root, rel, journey, targets) {
     const explicit = config.testCommand?.[area];
     const paths = (targets ?? []).map((t) => relative(root, t) || t);
-    if (explicit)
-        return paths.length ? `${explicit} ${paths.join(' ')}` : explicit;
+    if (explicit) {
+        if (paths.length)
+            return `${explicit} ${paths.join(' ')}`;
+        // A custom command still honours --journey: narrow it to that journey's
+        // tests, or `run --journey x` silently runs the whole suite.
+        return journey ? `${explicit} ${framework === 'playwright' ? `${rel}/${journey}` : `--testPathPattern "${journey}"`}` : explicit;
+    }
     if (framework === 'playwright') {
         const target = paths.length
             ? paths.join(' ')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {