@activate-core/test-forge 0.18.10 → 0.19.1

package/dist/cli/index.js

@@ -78,6 +78,7 @@ function printUsage() {
     console.log('  testforge generate --basic   Use the basic generator only');
     console.log('  testforge generate --force   Regenerate everything');
     console.log('  testforge generate --no-wait  Stop when the AI hits its usage limit (default: wait for the reset and resume)');
+    console.log('  testforge generate --no-verify  Skip running each journey once after generating (default: run and repair)');
     console.log('  testforge generate --journey <name>  Generate just ONE journey');
     console.log('  testforge generate --area <area>     Generate a whole area (e.g. api)');
     console.log('  testforge discover          Draft journey docs (asks: map or reference)');
@@ -154,6 +155,11 @@ async function main() {
                 : process.argv.includes('--wait-for-limit')
                     ? true
                     : undefined, // undefined -> config.waitForLimit (default on)
+            verify: process.argv.includes('--no-verify')
+                ? false
+                : process.argv.includes('--verify')
+                    ? true
+                    : undefined, // undefined -> config.verify (default on)
         });
         if (results.length === 0) {
             console.log('No journey files found in docs/. Add a .md file and try again.');

package/dist/core/docs/grounding.js

@@ -23,7 +23,19 @@ import { dirname, join } from 'node:path';
 import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
 import { hashProjectSource } from './cache.js';
 import { STATE_DIR } from '../paths.js';
+import { loadEnvFile } from '../project/config.js';
+import { originsFrom, pathsFrom, observeApp } from './observe.js';
+import { observeSchema } from './schema.js';
 const MAP_PATH = 'testforge.md';
+/**
+ * Bumped whenever the SHAPE of a facts block changes.
+ *
+ * The cache is keyed on the document and the source, neither of which moves
+ * when testforge itself gains a new kind of fact. Without this, a project
+ * that upgrades keeps serving the facts it cached under the old version —
+ * so the new facts never reach a single prompt and the upgrade does nothing.
+ */
+const FACTS_VERSION = 2;
 /** Where a journey's grounded facts live, mirroring the plan cache layout. */
 function groundingPath(journey) {
     const area = (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-');
@@ -105,7 +117,7 @@ function buildGroundingPrompt(journey, map, srcDir) {
  * Never throws. If the AI call fails we return null and the caller falls back
  * to telling each file to read the source itself — slower, but still correct.
  */
-export function groundJourney(journey, docHash, srcDir = 'src') {
+export async function groundJourney(journey, docHash, srcDir = 'src', markdown = '') {
     const path = groundingPath(journey);
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
     let sourceHash = '';
@@ -115,7 +127,7 @@ export function groundJourney(journey, docHash, srcDir = 'src') {
     catch {
         // unreadable source tree — fall back to keying on the doc alone
     }
-    const key = `${docHash}:${sourceHash}`;
+    const key = `v${FACTS_VERSION}:${docHash}:${sourceHash}`;
     if (existsSync(path)) {
         try {
             const cached = JSON.parse(readFileSync(path, 'utf-8'));
@@ -143,6 +155,28 @@ export function groundJourney(journey, docHash, srcDir = 'src') {
     }
     if (!facts)
         return null;
+    // Everything above was READ from source. Measure the rest: which origin
+    // actually serves each path, what the URL looks like once the app has had
+    // its say. A test encodes those literally, so an inferred one is a test
+    // that fails for a reason the product is not responsible for.
+    try {
+        const config2 = loadConfig();
+        const env = loadEnvFile(config2.envFile ?? '.env.testing');
+        const origins = await originsFrom(env);
+        const observed = await observeApp(origins, pathsFrom(markdown, facts));
+        if (observed)
+            facts = `${facts}\n\n${observed}`;
+        // The schema is a query, never an inference. A seed that invents a table
+        // name, omits a NOT NULL column or hands a boolean to a smallint fails at
+        // run time as a SKIPPED test — which reads as "fine" rather than
+        // "nothing was checked".
+        const schema = await observeSchema(process.cwd(), env, `${markdown}\n${facts}`);
+        if (schema)
+            facts = `${facts}\n\n${schema}`;
+    }
+    catch {
+        // the app not being up is not a fact about the app — carry on without it
+    }
     try {
         mkdirSync(dirname(path), { recursive: true });
         writeFileSync(path, JSON.stringify({ key, facts }, null, 2), 'utf-8');

package/dist/core/docs/observe.js

@@ -0,0 +1,223 @@
+/**
+ * testforge — facts MEASURED from the running application, not inferred.
+ *
+ * Everything else the generator knows about mechanics comes from reading
+ * source code: routes, selectors, shapes. Reading tells you what the code
+ * intends. It does not tell you what the deployment actually serves — which
+ * origin answers a path, whether the URL keeps its trailing slash, whether a
+ * locale segment survives. Those are the facts a test encodes literally, and
+ * guessing them produces a test that looks right and fails for no reason
+ * connected to the product.
+ *
+ * So we ask the application. Read-only: GET, against the origins the
+ * project's own test env already names, following redirects and recording
+ * where each one lands.
+ */
+const MAX_PATHS = 24;
+const PROBE_TIMEOUT = 15_000;
+/**
+ * Probes run a few at a time. A dev server answering its first request for a
+ * route can take seconds, and forty at once simply times them all out — which
+ * reads back as "the app does not serve this", the exact wrong fact.
+ */
+const PROBE_CONCURRENCY = 4;
+async function inPool(items, fn) {
+    const out = new Array(items.length);
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(PROBE_CONCURRENCY, items.length) }, async () => {
+        for (;;) {
+            const i = next++;
+            if (i >= items.length)
+                return;
+            out[i] = await fn(items[i]);
+        }
+    }));
+    return out;
+}
+/** Loopback, link-local and RFC1918 — the addresses a test environment uses. */
+function isPrivateAddress(ip) {
+    if (/^127\./.test(ip) || ip === '::1')
+        return true;
+    if (/^10\./.test(ip) || /^192\.168\./.test(ip) || /^169\.254\./.test(ip))
+        return true;
+    if (/^172\.(1[6-9]|2\d|3[01])\./.test(ip))
+        return true;
+    if (/^(fc|fd)/i.test(ip))
+        return true;
+    // Docker/compose networks frequently sit on a private /8 that is not in the
+    // list above only because the host chose it; anything non-routable is fine.
+    return /^2[0-2][0-9]\.|^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(ip);
+}
+/**
+ * Origins named by the project's own test env, narrowed to the ones that
+ * resolve to a PRIVATE address.
+ *
+ * An env file also names third-party services — a payment sandbox, an identity
+ * provider. Probing those would send this machine's traffic to somebody else's
+ * system to learn nothing about the project, so a host that resolves to a
+ * public address is never contacted. Duplicates collapse, and vars named like
+ * a base URL are preferred so an alias does not crowd out the real thing.
+ */
+export async function originsFrom(env) {
+    const { lookup } = await import('node:dns/promises');
+    const byUrl = new Map();
+    const entries = Object.entries(env).sort((a, b) => rank(a[0]) - rank(b[0]) || a[0].localeCompare(b[0]));
+    for (const [name, raw] of entries) {
+        const value = (raw ?? '').trim().replace(/\/+$/, '');
+        if (!/^https?:\/\/[^/\s]+$/i.test(value))
+            continue;
+        if (/\b(DB|DATABASE|REDIS|QUEUE|MAIL|SMTP)\b/i.test(name))
+            continue;
+        if (!byUrl.has(value))
+            byUrl.set(value, name);
+    }
+    const kept = [];
+    for (const [url, name] of byUrl) {
+        if (kept.length >= 6)
+            break;
+        let host;
+        try {
+            host = new URL(url).hostname;
+        }
+        catch {
+            continue;
+        }
+        if (host === 'localhost') {
+            kept.push({ name, url });
+            continue;
+        }
+        try {
+            const { address } = await lookup(host);
+            if (isPrivateAddress(address))
+                kept.push({ name, url });
+        }
+        catch {
+            // unresolvable — not reachable, so not worth probing either
+        }
+    }
+    return kept;
+}
+/** Vars that name the application's own roots come first. */
+function rank(name) {
+    if (/BASE_URL$/i.test(name))
+        return 0;
+    if (/^(APP|WEB|API|SPA)_/i.test(name) || /_(APP|WEB|API|SPA)_URL$/i.test(name))
+        return 1;
+    return 2;
+}
+/**
+ * Paths worth probing, taken from the journey document and whatever the
+ * source-reading step already reported. A path is any absolute, literal
+ * route; templated segments ({id}, :id) are dropped because a probe of a
+ * placeholder measures nothing.
+ */
+export function pathsFrom(...texts) {
+    const found = new Set();
+    for (const text of texts) {
+        for (const m of (text || '').matchAll(/(?<![\w:/])\/[a-z0-9][a-z0-9\-/]*/gi)) {
+            const p = m[0].replace(/[.,)\]]+$/, '');
+            if (p.length < 2 || p.length > 80)
+                continue;
+            if (/\.(js|ts|tsx|jsx|json|md|php|css|png|svg)$/i.test(p))
+                continue;
+            if (/^\/(src|app|tests?|node_modules|vendor|storage|public)\b/i.test(p))
+                continue;
+            found.add(p);
+            if (found.size >= MAX_PATHS)
+                return [...found];
+        }
+    }
+    return [...found];
+}
+async function probe(url) {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), PROBE_TIMEOUT);
+    try {
+        const res = await fetch(url, { redirect: 'follow', signal: ac.signal });
+        return { status: res.status, final: res.url || url };
+    }
+    catch {
+        return null;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/** Strip the origin so the block shows paths, not repeated hostnames. */
+function pathOf(full, origin) {
+    return full.startsWith(origin) ? full.slice(origin.length) || '/' : full;
+}
+/**
+ * Probe every path against every origin and render the result as a facts
+ * block. Returns null when nothing could be reached — the app being down is
+ * not a fact about the app.
+ */
+export async function observeApp(origins, paths) {
+    if (!origins.length || !paths.length)
+        return null;
+    const jobs = paths.flatMap((path) => origins.map((o) => ({ path, o })));
+    const answered = await inPool(jobs, async ({ path, o }) => {
+        const r = await probe(`${o.url}${path}`);
+        return {
+            path,
+            origin: o.name,
+            status: r ? r.status : 0,
+            final: r ? pathOf(r.final, o.url) : path,
+        };
+    });
+    const probes = paths.map((path) => ({
+        path,
+        results: answered.filter((a) => a.path === path && a.status !== 0),
+    }));
+    const reachable = probes.filter((p) => p.results.some((r) => r.status < 500));
+    if (!reachable.length)
+        return null;
+    const lines = [
+        'OBSERVED — measured against the running application just now. These are',
+        'facts, not inferences: prefer them over anything read from source code.',
+        '',
+        `  Origins probed: ${origins.map((o) => `${o.name}=${o.url}`).join('  ')}`,
+        '',
+        '  Path -> how each origin answered (status, and where it landed):',
+    ];
+    for (const p of reachable) {
+        const cells = p.results
+            .map((r) => `${r.origin} ${r.status}${r.final !== p.path ? ` -> ${r.final}` : ''}`)
+            .join('   ');
+        lines.push(`    ${p.path.padEnd(42)} ${cells}`);
+    }
+    // URL style, read off the probes rather than asserted.
+    const slashed = reachable.filter((p) => p.results.some((r) => r.status === 200 && r.final === `${p.path}/`)).length;
+    if (slashed) {
+        lines.push('', `  URL STYLE: the application serves ${slashed} of these paths with a TRAILING`, '  SLASH. Any URL assertion, and any href-suffix selector, must include it.');
+    }
+    // The primary origin is the one that answers most of these paths — the
+    // remaining probes are worth spending on it rather than on an alias.
+    const tally = new Map();
+    for (const p of reachable)
+        for (const r of p.results)
+            if (r.status === 200)
+                tally.set(r.origin, (tally.get(r.origin) ?? 0) + 1);
+    const primaryName = [...tally].sort((a, b) => b[1] - a[1])[0]?.[0];
+    const primary = origins.find((o) => o.name === primaryName);
+    if (primary) {
+        // A single-page app often answers 200 for EVERY path, including ones it
+        // does not implement. Without this, "200" reads as "the page exists" and
+        // a test is written against a catch-all shell that renders nothing it
+        // expects. Ask for a path nothing could implement and see what happens.
+        const miss = await probe(`${primary.url}/__testforge_no_such_path__`);
+        if (miss && miss.status === 200) {
+            lines.push('', `  CAUTION: ${primary.name} answers 200 for a path that cannot exist, so it`, '  serves a catch-all. On this origin a 200 is NOT evidence that a screen', '  exists — confirm by content the document names, not by status code.');
+        }
+        // A locale segment that disappears on the way is a segment the app does
+        // not use — a test that hard-codes it asserts against a redirect.
+        const sample = reachable.find((p) => p.results.some((r) => r.origin === primary.name && r.status === 200));
+        if (sample) {
+            const r = await probe(`${primary.url}/en${sample.path}`);
+            if (r && r.status < 400 && !r.final.includes('/en/')) {
+                lines.push('', `  LOCALE: /en${sample.path} lands on ${pathOf(r.final, primary.url)} — the application`, '  does NOT keep a locale segment in its URLs. Do not build one into a path.');
+            }
+        }
+    }
+    return lines.join('\n');
+}

package/dist/core/docs/schema.js

@@ -0,0 +1,136 @@
+/**
+ * testforge — the database's own account of itself.
+ *
+ * A seed is the part of a test most easily written from memory: a plausible
+ * table name, a boolean where the column is a smallint, a row that omits a
+ * NOT NULL column. None of that is visible in the test file, so every gate
+ * passes and the failure only appears when the suite runs — usually as a
+ * skipped test, which reads as "fine" rather than "nothing was checked".
+ *
+ * The schema is not something to infer. It is a query. We ask the project's
+ * own test database, through the project's own driver, and hand the answer
+ * to the generator as fact.
+ */
+import { createRequire } from 'node:module';
+/** Cap the block: a generator needs the tables in play, not the whole database. */
+const MAX_TABLES_DETAILED = 12;
+const MAX_TABLE_NAMES = 250;
+/** Connection settings, in the order a project's test env usually provides them. */
+function connectionFrom(env) {
+    const url = env.DATABASE_URL || env.DB_URL || env.PG_URL;
+    // Laravel writes pgsql://, node-postgres only understands postgres(ql)://.
+    // Same server either way, so normalise rather than refuse to look.
+    const scheme = url?.match(/^([a-z+]+):\/\//i)?.[1]?.toLowerCase();
+    if (url && scheme && ['postgres', 'postgresql', 'pgsql', 'pg'].includes(scheme)) {
+        return { connectionString: url.replace(/^[a-z+]+:\/\//i, 'postgresql://') };
+    }
+    const host = env.DB_HOST || env.PGHOST;
+    const database = env.DB_DATABASE || env.DB_NAME || env.PGDATABASE;
+    if (!host || !database)
+        return null;
+    return {
+        host,
+        port: Number(env.DB_PORT || env.PGPORT || 5432),
+        database,
+        user: env.DB_USERNAME || env.DB_USER || env.PGUSER,
+        password: env.DB_PASSWORD || env.PGPASSWORD,
+    };
+}
+/**
+ * The project's Postgres driver. testforge has no runtime dependencies and
+ * should not grow one for this: a project whose tests touch a database
+ * already ships a driver, and using theirs guarantees the same version the
+ * generated seed will run under.
+ */
+function loadDriver(projectRoot) {
+    try {
+        const require = createRequire(`${projectRoot.replace(/\/+$/, '')}/package.json`);
+        return require('pg');
+    }
+    catch {
+        return null;
+    }
+}
+/** Short type names keep the block readable; the distinction that matters is kept. */
+function shortType(t) {
+    return t
+        .replace('character varying', 'varchar')
+        .replace('timestamp without time zone', 'timestamp')
+        .replace('timestamp with time zone', 'timestamptz')
+        .replace('double precision', 'float8');
+}
+/**
+ * Read the schema and render the facts a seed actually needs. Returns null
+ * whenever the database cannot be reached — an unreachable database is not a
+ * fact about the schema, and generation carries on without the block.
+ */
+export async function observeSchema(projectRoot, env, mentionedIn) {
+    const config = connectionFrom(env);
+    const pg = config && loadDriver(projectRoot);
+    if (!config || !pg)
+        return null;
+    const client = new pg.Client({ ...config, connectionTimeoutMillis: 8_000 });
+    try {
+        await client.connect();
+        const names = (await client.query(`select table_name from information_schema.tables
+          where table_schema = 'public' and table_type = 'BASE TABLE'
+          order by table_name`)).rows.map((r) => r.table_name);
+        if (!names.length)
+            return null;
+        // Detail the tables this journey actually talks about. Matching against
+        // the REAL list is what makes an invented name impossible to carry
+        // forward: a name nobody has is simply not in the block.
+        const text = mentionedIn.toLowerCase();
+        const wanted = names
+            .filter((n) => text.includes(n.toLowerCase()))
+            .sort((a, b) => b.length - a.length)
+            .slice(0, MAX_TABLES_DETAILED);
+        const lines = [
+            'SCHEMA — read from this project\'s own test database just now. These are',
+            'facts: a table or column that is not listed here does not exist.',
+            '',
+            `  Tables (${names.length}):`,
+        ];
+        for (let i = 0; i < Math.min(names.length, MAX_TABLE_NAMES); i += 6) {
+            lines.push(`    ${names.slice(i, i + 6).join(', ')}`);
+        }
+        if (wanted.length) {
+            const cols = (await client.query(`select table_name, column_name, data_type, is_nullable, column_default
+             from information_schema.columns
+            where table_schema = 'public' and table_name = any($1)
+            order by table_name, ordinal_position`, [wanted])).rows;
+            for (const table of wanted) {
+                const own = cols.filter((c) => c.table_name === table);
+                if (!own.length)
+                    continue;
+                const required = own.filter((c) => c.is_nullable === 'NO' && !c.column_default);
+                // Only the columns a seed is likely to hand a boolean: a smallint
+                // that reads like a flag or a state. Listing every integer would bury
+                // that under ids, counts and measurements, and "pass 0 or 1" is not
+                // even true for a column like doors.
+                const smallish = own.filter((c) => c.data_type === 'smallint' &&
+                    !/^id$/.test(c.column_name) &&
+                    !/_id$/.test(c.column_name));
+                lines.push('', `  ${table}:`);
+                if (required.length) {
+                    lines.push('    MUST be provided by a seed (NOT NULL, no default):', `      ${required.map((c) => `${c.column_name}:${shortType(c.data_type)}`).join(', ')}`);
+                }
+                if (smallish.length) {
+                    lines.push('    smallint columns — NEVER pass a JS boolean. The driver sends the', '    literal "false" and the column rejects it (invalid input syntax', '    for type smallint). Use the numeric value the column expects:', `      ${smallish.map((c) => c.column_name).join(', ')}`);
+                }
+            }
+        }
+        return lines.join('\n');
+    }
+    catch {
+        return null;
+    }
+    finally {
+        try {
+            await client.end();
+        }
+        catch {
+            /* already closed */
+        }
+    }
+}

package/dist/core/gates/coverage.js

@@ -33,6 +33,10 @@ export function writeCoverageManifest(results) {
             journey: r.journeyName,
             real: files.filter((f) => f.real).length,
             deferred: files.filter((f) => !f.real).length,
+            // Whether these files were RUN and passed. A file can assert plenty
+            // and still target something the app does not have, so "real" alone
+            // was never enough to call the coverage honest.
+            verified: r.verified,
             files,
         };
     });
@@ -42,12 +46,27 @@ export function writeCoverageManifest(results) {
     const totalDeferred = journeys.reduce((n, j) => n + j.deferred, 0);
     try {
         mkdirSync(STATE_DIR, { recursive: true });
-        writeFileSync(COVERAGE_PATH, JSON.stringify({ real: totalReal, deferred: totalDeferred, journeys }, null, 2), 'utf-8');
+        writeFileSync(COVERAGE_PATH, JSON.stringify({
+            real: totalReal,
+            deferred: totalDeferred,
+            verified: journeys.filter((j) => j.verified === true).length,
+            failedVerify: journeys.filter((j) => j.verified === false).length,
+            notVerified: journeys.filter((j) => j.verified === undefined).length,
+            journeys,
+        }, null, 2), 'utf-8');
     }
     catch {
         /* best-effort */
     }
     console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see ${COVERAGE_PATH}`);
+    const ranGreen = journeys.filter((j) => j.verified === true).length;
+    const ranRed = journeys.filter((j) => j.verified === false).length;
+    const unrun = journeys.filter((j) => j.verified === undefined).length;
+    if (ranGreen || ranRed || unrun) {
+        console.log(`Verified: ${ranGreen} journey(s) ran green` +
+            (ranRed ? `, ${ranRed} still failing` : '') +
+            (unrun ? `, ${unrun} not run` : ''));
+    }
     if (totalDeferred) {
         console.log(`  (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
     }

package/dist/core/generate/ai-generator.js

@@ -485,7 +485,18 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         // written, and handed unchanged to every file in it. Sits with the map and
         // the journey so the whole block stays a stable prefix across the journey.
         ...(grounding
-            ? [`--- CODE FACTS (read from this project's source) ---`, grounding, ``]
+            ? [
+                `--- CODE FACTS (read from this project's source) ---`,
+                grounding,
+                ``,
+                `Where this block carries an "OBSERVED" or "SCHEMA" section, those`,
+                `lines were MEASURED — against the running application and its own`,
+                `database — not read from code. They outrank everything else here.`,
+                `Take the origin, the exact URL shape (including a trailing slash),`,
+                `any locale note, and every table and column name from them verbatim.`,
+                `Never invent a path, origin, table or column that appears nowhere above.`,
+                ``,
+            ]
             : []),
         `--- JOURNEY ---`,
         `It may include sections beyond fields — actions, expected results,`,
@@ -554,6 +565,52 @@ export function generateWithAI(journey, testPath, focus, grounding) {
  * journey's suite can be generated concurrently (each spawns its own
  * `claude` process). Same prompt, same read-only exploration.
  */
+/**
+ * Repair a generated test from the runner's own failure output.
+ *
+ * The failure is evidence the generator did not have: the real selector, the
+ * real URL shape, the real column type. So the repair prompt leads with it
+ * and forbids the two easy ways out — deleting the assertion, or skipping the
+ * test — because either turns a caught defect into a silent one.
+ */
+export async function repairWithAIAsync(journey, testPath, current, failure, harnessPath, transport, grounding) {
+    const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
+    const base = buildPrompt(journey, map, testPath, undefined, harnessPath, transport, grounding);
+    const prompt = [
+        base,
+        '',
+        '=== REPAIR ===',
+        `The file below was generated for ${testPath} and RUN against the real`,
+        'application. It FAILED. The runner output is the ground truth: it tells',
+        'you what the application actually renders, routes and stores.',
+        '',
+        'Rewrite the file so it passes for the RIGHT reason.',
+        '',
+        'Rules for this repair, in addition to every rule above:',
+        '  1. Do NOT delete, weaken or invert an assertion to make it pass.',
+        '  2. Do NOT add test.skip / test.fixme / try-catch to swallow the failure.',
+        '  3. If the output shows the app uses a different selector, URL shape,',
+        '     path prefix, origin, table name or column type than the file assumed,',
+        '     the FILE is wrong — correct it to match the application.',
+        '  4. If a value is genuinely missing because the page has not finished',
+        '     loading, wait for it properly; do not raise timeouts blindly.',
+        '  5. If the output shows the APPLICATION is genuinely wrong (it contradicts',
+        '     the journey document), keep the assertion exactly as it is and change',
+        '     nothing — a real defect must stay caught.',
+        '',
+        '--- CURRENT FILE ---',
+        current,
+        '--- RUNNER OUTPUT ---',
+        failure,
+        '--- END ---',
+        '',
+        'Output ONLY the corrected file. No commentary.',
+    ].join('\n');
+    const config = loadConfig();
+    const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
+    const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
+    return cleanOutput(stdout);
+}
 export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
     const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding);
@@ -592,6 +649,14 @@ function looksLikeCode(line) {
         t.startsWith('function ') ||
         t.startsWith('async ') ||
         t.startsWith('require(') ||
+        // A test file often opens straight on its first construct, with no
+        // import and no comment above it. Without these an unfenced but
+        // perfectly good file is thrown away as "no recognizable code".
+        t.startsWith('describe(') ||
+        t.startsWith('test(') ||
+        t.startsWith('it(') ||
+        t.startsWith('module.exports') ||
+        t.startsWith('class ') ||
         t.startsWith("'use strict'") ||
         t.startsWith('"use strict"'));
 }

package/dist/core/generate/run-generate.js

@@ -26,7 +26,8 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
 import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
 import { generateTestCases } from './generator.js';
 import { writeDeferredStub } from './playwright-writer.js';
-import { generateWithAI, generateWithAIAsync, } from './ai-generator.js';
+import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
+import { verifyFiles } from './verify.js';
 import { groundJourney } from '../docs/grounding.js';
 import { generateSeed } from './seed-generator.js';
 import { isSyntaxValid, checkTestGates } from '../gates/validate.js';
@@ -173,6 +174,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         return [];
     }
     const config = loadConfig();
+    // Verify is on unless the project or the caller turns it off. verifyFiles
+    // itself reports (and skips) the cases where the app cannot be reached, so
+    // "on" never means "fail the run because nothing is running".
+    const verifyOn = opts.verify ?? config.verify ?? true;
+    const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
+    const verifySkipNoted = new Set();
     const ext = config.testExtension ?? '.test.ts';
     const seedDir = config.seedOutput ?? 'generated-tests/seed';
     const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
@@ -434,7 +441,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         // fails, `grounding` stays undefined and each file falls back to reading
         // the source itself, which is slower but still correct.
         const grounding = useAI
-            ? groundJourney(journey, hash) ?? undefined
+            ? (await groundJourney(journey, hash, 'src', markdown)) ?? undefined
             : undefined;
         const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
         if (staleRemoved.length) {
@@ -561,7 +568,59 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
                 lint.remaining.slice(0, 5).join('; '));
         }
-        const degraded = useAI && (status === 'basic' || seedStatus === 'failed');
+        // Run what we just wrote, once, against the real application. A test
+        // that passes every static gate can still target a selector, route or
+        // column the app does not have — only running it can tell. The runner's
+        // own output is the repair prompt; a journey that still fails afterwards
+        // is reported and left UNCACHED, so it is retried rather than trusted.
+        let verified;
+        const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
+        if (useAI && verifyOn && targets.length) {
+            for (let attempt = 0; attempt <= verifyAttempts; attempt += 1) {
+                const outcome = verifyFiles(config, journey.area, targets);
+                if (!outcome.ran) {
+                    // Say WHY once. The reason is a property of the project (no env
+                    // file, no runner), not of this journey, so repeating it per
+                    // journey buries the rest of the output.
+                    if (outcome.reason && !verifySkipNoted.has(outcome.reason)) {
+                        verifySkipNoted.add(outcome.reason);
+                        console.log(`  not verified — ${outcome.reason}`);
+                    }
+                    break;
+                }
+                if (outcome.ok) {
+                    verified = true;
+                    if (attempt > 0)
+                        console.log(`  verified after ${attempt} repair(s)`);
+                    break;
+                }
+                verified = false;
+                if (attempt === verifyAttempts) {
+                    warnings.push(`verify: still failing after ${verifyAttempts} repair attempt(s) — ` +
+                        'this may be a real defect in the application, or a generation ' +
+                        'defect this run could not correct. Left uncached for a retry.');
+                    break;
+                }
+                console.log(`  failed when run — repairing (attempt ${attempt + 1}/${verifyAttempts})`);
+                for (const target of targets) {
+                    let current;
+                    try {
+                        current = readFileSync(target, 'utf-8');
+                    }
+                    catch {
+                        continue;
+                    }
+                    const repaired = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, outcome.output, harnessPath, transport, grounding), ext, testGate);
+                    if (repaired === null)
+                        continue;
+                    const stamped = repaired.includes(GEN_MARKER)
+                        ? repaired
+                        : `// ${GEN_MARKER}\n${repaired}`;
+                    writeFileSync(target, stamped, 'utf-8');
+                }
+            }
+        }
+        const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
         if (!degraded) {
             cache[file] = hash;
         }
@@ -584,6 +643,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             seedPath,
             status,
             seedStatus,
+            verified,
             warnings: warnings.length ? warnings : undefined,
         });
     }

package/dist/core/generate/verify.js

@@ -0,0 +1,108 @@
+/**
+ * testforge — run a freshly generated test once, before calling it done.
+ *
+ * The static gates read the FILE: does it assert, does it parse, is it a
+ * stub. They cannot see the running application, so a test that looks
+ * perfect and targets a selector, URL shape or column type the app does not
+ * have still passes every gate. Those tests fail later in a way that says
+ * nothing about the product — the expensive kind of noise.
+ *
+ * So after a journey's files and seed are written we execute them once. A
+ * failure here is a generation defect until proven otherwise, and its output
+ * is the repair prompt: the runner's own error is far better evidence than
+ * anything we could infer.
+ */
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { relative } from 'node:path';
+import { loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
+import { runnerCommand, packageRoot } from '../run/run-tests.js';
+/** How much runner output to carry into a repair prompt. */
+const MAX_OUTPUT = 12_000;
+/**
+ * Keep the END of the output: runners print the failure detail and the
+ * summary last, and that is the part a repair needs.
+ */
+function trimOutput(s) {
+    const clean = s.replace(/\u001b\[[0-9;]*m/g, '');
+    return clean.length <= MAX_OUTPUT
+        ? clean.trim()
+        : `…(${clean.length - MAX_OUTPUT} chars trimmed)…\n${clean.slice(-MAX_OUTPUT).trim()}`;
+}
+/**
+ * What the runner actually executed. Both Jest and Playwright print these
+ * counts; anything we cannot read comes back null and is not guessed at.
+ */
+function countsFrom(output) {
+    const n = (re) => {
+        const m = output.match(re);
+        return m ? Number(m[1]) : null;
+    };
+    const passed = n(/(\d+)\s+passed/i) ?? n(/Tests:.*?(\d+)\s+passed/i);
+    const skipped = n(/(\d+)\s+skipped/i) ?? n(/(\d+)\s+todo/i) ?? n(/(\d+)\s+pending/i);
+    if (passed === null && skipped === null)
+        return null;
+    return { passed: passed ?? 0, skipped: skipped ?? 0 };
+}
+/**
+ * Run exactly the given generated files. Never throws: a verify step that
+ * breaks generation would be worse than no verify step at all.
+ */
+export function verifyFiles(config, area, files) {
+    if (!files.length)
+        return { ran: false, ok: false, output: '', reason: 'no files to verify' };
+    const testDir = testDirForType(config, area);
+    const root = packageRoot(testDir);
+    const rel = relative(root, testDir) || '.';
+    const framework = resolveFramework(config, area, area);
+    const present = files.filter((f) => existsSync(f));
+    if (!present.length) {
+        return { ran: false, ok: false, output: '', reason: 'generated files are not on disk' };
+    }
+    const cmd = runnerCommand(config, area, framework, root, rel, undefined, present);
+    const envFile = config.envFile ?? '.env.testing';
+    const envVars = loadEnvFile(envFile);
+    if (!Object.keys(envVars).length && !existsSync(envFile)) {
+        return {
+            ran: false,
+            ok: false,
+            output: '',
+            reason: `no ${envFile} — cannot reach the app to verify`,
+        };
+    }
+    const res = spawnSync(cmd, {
+        shell: true,
+        cwd: root,
+        env: { ...process.env, ...envVars },
+        encoding: 'utf-8',
+        timeout: config.verifyTimeoutMs ?? 600_000,
+        maxBuffer: 32 * 1024 * 1024,
+    });
+    if (res.error) {
+        return { ran: false, ok: false, output: '', reason: res.error.message };
+    }
+    const output = trimOutput(`${res.stdout ?? ''}\n${res.stderr ?? ''}`);
+    // A runner that could not start at all (no binary, bad config) is not a
+    // failing test — reporting it as one would send the AI chasing a ghost.
+    if (res.status === null) {
+        return { ran: false, ok: false, output, reason: 'runner timed out' };
+    }
+    if (/command not found|Cannot find module '@playwright|is not recognized/i.test(output)) {
+        return { ran: false, ok: false, output, reason: 'test runner is not installed' };
+    }
+    if (res.status !== 0)
+        return { ran: true, ok: false, output };
+    // A suite where every test skipped ALSO exits 0. Treating that as verified
+    // is the exact dishonesty this step exists to remove: the seed could not be
+    // established, or a precondition was not met, and nothing was checked at
+    // all. Report it as a failure, with the reason the runner already printed.
+    const counts = countsFrom(output);
+    if (counts && counts.passed === 0 && counts.skipped > 0) {
+        return {
+            ran: true,
+            ok: false,
+            output: `Every test was SKIPPED (${counts.skipped} skipped, 0 passed) — nothing was actually verified.\nThe precondition or seed below could not be established; fix that so the tests run.\n\n${output}`,
+        };
+    }
+    return { ran: true, ok: true, output };
+}

package/dist/core/run/run-tests.js

@@ -17,7 +17,7 @@ import { existsSync } from 'node:fs';
 import { join, dirname, resolve, relative } from 'node:path';
 import { loadConfig, loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
 /** Nearest ancestor dir containing a package.json (the runner's cwd). */
-function packageRoot(dir) {
+export function packageRoot(dir) {
     let d = resolve(dir);
     for (;;) {
         if (existsSync(join(d, 'package.json')))
@@ -28,6 +28,65 @@ function packageRoot(dir) {
         d = parent;
     }
 }
+/**
+ * The command that runs an area's tests, and nothing else — so `generate`'s
+ * verify step and `run` can never drift into invoking the suite differently.
+ * `targets` narrows the run to specific files (used when verifying a single
+ * journey); without it the whole area runs.
+ */
+export function runnerCommand(config, area, framework, root, rel, journey, targets) {
+    const explicit = config.testCommand?.[area];
+    const paths = (targets ?? []).map((t) => relative(root, t) || t);
+    if (explicit)
+        return paths.length ? `${explicit} ${paths.join(' ')}` : explicit;
+    if (framework === 'playwright') {
+        const target = paths.length
+            ? paths.join(' ')
+            : journey
+                ? `${rel}/${journey}`
+                : rel;
+        // Name the config explicitly. It is written beside the AREA's tests
+        // (tests/Feature/playwright.config.js for tests/Feature/Web), not at
+        // the package root, so Playwright's own discovery finds nothing and
+        // silently falls back to defaults — no globalSetup, no reporters, and
+        // therefore no run published to the dashboard. Walk up from the test
+        // directory to the package root and take the first config found.
+        let cfgFlag = '';
+        for (let dir = rel;; dir = dirname(dir)) {
+            const hit = ['playwright.config.js', 'playwright.config.ts']
+                .map((f) => (dir === '.' ? f : join(dir, f)))
+                .find((f) => existsSync(join(root, f)));
+            if (hit) {
+                cfgFlag = ` --config=${hit}`;
+                break;
+            }
+            if (dir === '.' || dir === dirname(dir))
+                break;
+        }
+        return `npx playwright test${cfgFlag} ${target}`;
+    }
+    // Jest needs its config named for the same reason Playwright does: the one
+    // testforge writes sits beside the AREA's tests, not at the package root, so
+    // Jest's own discovery misses it — and with it the reporter that publishes
+    // the run. Walk up from the test directory and take the first config found.
+    let jestCfg = '';
+    for (let dir = rel;; dir = dirname(dir)) {
+        const hit = ['jest.config.js', 'jest.config.cjs', 'jest.config.ts']
+            .map((f) => (dir === '.' ? f : join(dir, f)))
+            .find((f) => existsSync(join(root, f)));
+        if (hit) {
+            jestCfg = ` --config=${hit}`;
+            break;
+        }
+        if (dir === '.' || dir === dirname(dir))
+            break;
+    }
+    if (paths.length)
+        return `npx jest${jestCfg} ${paths.join(' ')}`;
+    return journey
+        ? `npx jest${jestCfg} --testPathPattern "${journey}"`
+        : `npx jest${jestCfg} ${rel}`;
+}
 export function runTests(opts = {}) {
     const config = loadConfig();
     const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
@@ -55,36 +114,7 @@ export function runTests(opts = {}) {
         const framework = resolveFramework(config, area, area);
         const root = packageRoot(testDir);
         const rel = relative(root, testDir) || '.';
-        let cmd = config.testCommand?.[area];
-        if (!cmd) {
-            if (framework === 'playwright') {
-                const target = opts.journey ? `${rel}/${opts.journey}` : rel;
-                // Name the config explicitly. It is written beside the AREA's tests
-                // (tests/Feature/playwright.config.js for tests/Feature/Web), not at
-                // the package root, so Playwright's own discovery finds nothing and
-                // silently falls back to defaults — no globalSetup, no reporters, and
-                // therefore no run published to the dashboard. Walk up from the test
-                // directory to the package root and take the first config found.
-                let cfgFlag = '';
-                for (let dir = rel;; dir = dirname(dir)) {
-                    const hit = ['playwright.config.js', 'playwright.config.ts']
-                        .map((f) => (dir === '.' ? f : join(dir, f)))
-                        .find((f) => existsSync(join(root, f)));
-                    if (hit) {
-                        cfgFlag = ` --config=${hit}`;
-                        break;
-                    }
-                    if (dir === '.' || dir === dirname(dir))
-                        break;
-                }
-                cmd = `npx playwright test${cfgFlag} ${target}`;
-            }
-            else {
-                cmd = opts.journey
-                    ? `npx jest --testPathPattern "${opts.journey}"`
-                    : `npx jest ${rel}`;
-            }
-        }
+        let cmd = runnerCommand(config, area, framework, root, rel, opts.journey);
         if (opts.passThrough?.length)
             cmd += ` ${opts.passThrough.join(' ')}`;
         ranAnything = true;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.18.10",
+  "version": "0.19.1",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {