@activate-core/test-forge 0.10.1 → 0.11.0

package/dist/core/ai-generator.js

@@ -39,15 +39,19 @@ function runWithStdin(program, args, input) {
     });
 }
 /** Build the prompt we send to the AI for one journey. */
-export function buildPrompt(journey, map, testPath, focus, harnessPath) {
+export function buildPrompt(journey, map, testPath, focus, harnessPath, transport) {
     const config = loadConfig();
     // Opinionated: testforge only generates Jest (api/backend) or Playwright
     // (web/UI) tests — any other framework (e.g. a project's Cypress setup)
     // is clamped to the right one for the journey. Single source of truth in
     // resolveFramework, shared with the output-layout decision.
     const isApi = resolveFramework(config, journey.area, journey.type) === 'jest';
+    // Do NOT assume an HTTP client here — the transport is detected from the
+    // project's real dependencies and passed in (Hapi/Fastify inject,
+    // supertest only if installed, or native http). Hardcoding supertest
+    // was what produced ~half the API suites that never loaded.
     const framework = isApi
-        ? 'Jest (with supertest for HTTP) for an API/backend test'
+        ? 'Jest for an API/backend test'
         : 'Playwright for a web/UI test';
     // Pending-test syntax differs by runner: Jest has test.todo/it.skip;
     // Playwright has NEITHER (test.todo throws) — it uses test.fixme/skip.
@@ -107,6 +111,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
         `the behavior genuinely cannot be determined from the source — do not`,
         `leave something pending that a quick read of the code would answer.`,
         `Do NOT write or edit any file; output only the test code to stdout.`,
+        `EVERY runnable test MUST contain at least one real assertion. If you`,
+        `genuinely cannot assert an outcome, make the case pending (${pending})`,
+        `— never write a test body with no assertion, which passes without`,
+        `testing anything.`,
         ``,
         isApi
             ? [
@@ -118,6 +126,13 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
                 `duplicate exists, correct fields) using the database/ORM the`,
                 `project uses. Connect via the env var named in the map (e.g.`,
                 `process.env.DB_URL) — never hardcode a connection string.`,
+                `MOCKING: if the code calls an external service (e.g. an identity`,
+                `or payment API), mock it AT THE SEAM THE CODE ACTUALLY USES —`,
+                `read the source to see which client it imports (e.g. axios) and`,
+                `how it is called, then jest.mock that exact module so the mock`,
+                `truly intercepts. Do NOT mock a client the code doesn't use, and`,
+                `do NOT drive the server out-of-process while expecting an`,
+                `in-process mock to fire — they cannot both be true.`,
             ].join('\n')
             : [
                 `Because this is a web journey, write Playwright tests that drive`,
@@ -160,6 +175,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath) {
                 ``,
             ]
             : []),
+        ...(transport
+            ? [transport, `Do NOT import any library that is not already a dependency of this project.`, ``]
+            : []),
         ...(harnessPath
             ? [
                 `SHARED HARNESS — a testkit already exists at:`,
@@ -229,9 +247,9 @@ export function generateWithAI(journey, testPath, focus) {
  * journey's suite can be generated concurrently (each spawns its own
  * `claude` process). Same prompt, same read-only exploration.
  */
-export async function generateWithAIAsync(journey, testPath, focus, harnessPath) {
+export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
-    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath);
+    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport);
     const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
     const stdout = await runWithStdin(program, args, prompt);
     return cleanOutput(stdout);

package/dist/core/detect.js

@@ -0,0 +1,133 @@
+/**
+ * testforge — project capability detection (dependency-aware, no guessing).
+ *
+ * Generated tests must fit the project they land in: they may only import
+ * libraries the project actually has, and backend suites must drive the
+ * server the way the project's stack supports (Hapi/Fastify inject,
+ * supertest, or a live HTTP call) — never by assuming a convention.
+ *
+ * Everything here is read from the real files (package.json + node_modules)
+ * walking up from the test location, so it works on ANY project, including
+ * a greenfield one with no existing tests.
+ */
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { builtinModules } from 'node:module';
+import { dirname, join } from 'node:path';
+/** Read a package.json's dependency names (all dependency fields). */
+function depsOf(pkgPath) {
+    try {
+        const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+        const fields = [
+            'dependencies',
+            'devDependencies',
+            'peerDependencies',
+            'optionalDependencies',
+        ];
+        const names = [];
+        for (const f of fields) {
+            const obj = pkg[f];
+            if (obj)
+                names.push(...Object.keys(obj));
+        }
+        return names;
+    }
+    catch {
+        return [];
+    }
+}
+/** Top-level entries of a node_modules dir, incl. scoped (@scope/pkg). */
+function nodeModulesNames(nmDir) {
+    if (!existsSync(nmDir))
+        return [];
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(nmDir);
+    }
+    catch {
+        return [];
+    }
+    for (const e of entries) {
+        if (e.startsWith('.'))
+            continue;
+        if (e.startsWith('@')) {
+            try {
+                for (const sub of readdirSync(join(nmDir, e))) {
+                    if (!sub.startsWith('.'))
+                        out.push(`${e}/${sub}`);
+                }
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        else {
+            out.push(e);
+        }
+    }
+    return out;
+}
+/**
+ * The set of module specifiers a test at `fromDir` may import: every
+ * dependency declared in any package.json from `fromDir` up to the repo
+ * root, every package physically present in node_modules along the way,
+ * and all Node builtins. Union is intentional — being permissive here
+ * only risks NOT rejecting a real dep; the goal is to catch clearly
+ * absent libraries (e.g. supertest) without false-rejecting hoisted ones.
+ */
+export function installedModules(fromDir) {
+    const mods = new Set();
+    for (const b of builtinModules) {
+        mods.add(b);
+        mods.add(`node:${b}`);
+    }
+    let dir = fromDir;
+    for (let i = 0; i < 12; i++) {
+        const pkg = join(dir, 'package.json');
+        if (existsSync(pkg))
+            depsOf(pkg).forEach((d) => mods.add(d));
+        nodeModulesNames(join(dir, 'node_modules')).forEach((d) => mods.add(d));
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return mods;
+}
+/**
+ * A concrete instruction for how a backend (Jest) suite should reach the
+ * server, based on what the project actually depends on — so the AI does
+ * not fall back to an uninstalled default (the supertest failure).
+ */
+export function transportHint(fromDir, framework) {
+    if (framework === 'playwright') {
+        return `This is a Playwright area: use the browser + the testkit's apiContext(request) for API calls. Do NOT use supertest.`;
+    }
+    const mods = installedModules(fromDir);
+    const has = (m) => mods.has(m);
+    if (has('@hapi/hapi') || has('hapi')) {
+        return [
+            `TRANSPORT: this is a Hapi backend. Drive the server IN-PROCESS with`,
+            `\`server.inject({ method, url, payload, headers })\` — the same way`,
+            `the project's own tests do. Do NOT use supertest (it is not a`,
+            `dependency and would run a separate process the in-process mocks`,
+            `can't intercept). Require the app's server/init from the real source.`,
+        ].join('\n');
+    }
+    if (has('fastify')) {
+        return [
+            `TRANSPORT: this is a Fastify backend. Drive it IN-PROCESS with`,
+            `\`app.inject({ method, url, payload, headers })\`. Do NOT use supertest.`,
+        ].join('\n');
+    }
+    if (has('supertest')) {
+        return `TRANSPORT: use supertest (it is installed) against the app's exported Express/Koa/HTTP handler.`;
+    }
+    // Express/Koa/etc. without supertest, or unknown: don't invent a dep.
+    return [
+        `TRANSPORT: do NOT import any HTTP-testing library that is not already`,
+        `a dependency of this project. Prefer the framework's own in-process`,
+        `injection if it has one; otherwise use Node's built-in http/fetch`,
+        `against process.env.BASE_URL. Never add a new dependency.`,
+    ].join('\n');
+}

package/dist/core/harness.js

@@ -102,6 +102,18 @@ function buildHarnessPrompt(harnessPath, framework) {
         `exact names — the tests must run without any hand-pasted token.`,
         ``,
         `This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area.`,
+        ...(framework === 'jest'
+            ? [
+                `Because this is a JEST/backend area, do NOT require or import`,
+                `'@playwright/test' anywhere (it is not installed in a backend`,
+                `package — a top-level require would break every suite that loads`,
+                `this testkit). OMIT the Playwright-only apiContext/login exports.`,
+                `Backend suites drive the server in-process via the framework's own`,
+                `inject (or an already-installed HTTP client) — never Playwright,`,
+                `and never a library this project has not installed.`,
+                ``,
+            ]
+            : []),
         HARNESS_CONTRACT,
         ``,
         `The file will be saved at EXACTLY: ${harnessPath}`,

package/dist/core/playwright-writer.js

@@ -1,47 +1,62 @@
 /**
- * testforge — Playwright test file writer.
+ * testforge — deferred-stub writer (fallback only).
  *
- * Takes a Journey and its generated test cases and produces the
- * text of a runnable Playwright test file (.test.ts). It does not
- * write to disk itself; it returns the file content as a string so
- * the caller decides where to save it.
+ * When AI generation is unavailable (usage limit / repeated invalid
+ * output), testforge still needs to leave SOMETHING at the file path so
+ * the journey is visibly accounted for. It must NEVER be a test that
+ * runs and passes with no assertions — a silent false-green is the worst
+ * QA outcome. So this writer emits PENDING placeholders only:
+ *   - Jest       → test.todo(...)   (pending; no assertions; no import)
+ *   - Playwright → test.fixme(...)  (skipped; never counts as a pass)
  *
- * The generated tests use placeholder selectors and values, marked
- * with TODO comments, because testforge cannot know the real page
- * structure. A developer fills those in. Every test is tagged
- * @generated so it is easy to spot auto-created tests.
+ * It is also runner-aware: a Jest area never receives a
+ * `@playwright/test` import (which would break every suite that loads).
+ * A developer (or a later AI run) replaces these with real tests.
  */
-/** Build the import header and describe block opening. */
-function header(journey) {
+const BANNER = [
+    `// @generated by testforge — DEFERRED stub (AI generation was unavailable).`,
+    `// These are PLACEHOLDERS, not real tests: they are pending so they do`,
+    `// NOT pass silently. Regenerate with AI once available, or implement`,
+    `// them by hand.`,
+].join('\n');
+/** One pending entry, phrased for the area's runner. */
+function pendingLine(testCase, framework) {
+    const outcome = testCase.expect === 'pass' ? 'succeed' : 'be rejected';
+    const title = `${testCase.title} — TODO: implement (expected to ${outcome})`;
+    const safe = title.replace(/"/g, '\\"');
+    return framework === 'jest'
+        ? `  test.todo("${safe}");`
+        : `  test.fixme("${safe}", async () => {});`;
+}
+/**
+ * Build the full deferred-stub file for a journey, correct for the area's
+ * test runner. Every entry is pending, so the file contributes zero
+ * false-passing tests.
+ */
+export function writeDeferredStub(journey, cases, framework) {
+    const entries = (cases.length ? cases : [{ title: journey.name, expect: 'pass' }])
+        .map((c) => pendingLine(c, framework))
+        .join('\n');
+    if (framework === 'jest') {
+        // Jest globals — no import (a `@playwright/test` import here would
+        // break the whole suite in a backend package).
+        return [
+            BANNER,
+            ``,
+            `describe(${JSON.stringify(journey.name)}, () => {`,
+            entries,
+            `});`,
+            ``,
+        ].join('\n');
+    }
     return [
-        `import { test, expect } from "@playwright/test";`,
+        `import { test } from "@playwright/test";`,
+        ``,
+        BANNER,
         ``,
-        `// @generated by testforge — review before relying on these tests.`,
-        `test.describe("${journey.name}", () => {`,
+        `test.describe(${JSON.stringify(journey.name)}, () => {`,
+        entries,
+        `});`,
         ``,
     ].join('\n');
 }
-/** Build a single Playwright test block from one test case. */
-function testBlock(journey, testCase) {
-    const lines = [];
-    lines.push(`  // @generated`);
-    lines.push(`  test("${testCase.title}", async ({ page }) => {`);
-    lines.push(`    await page.goto("${journey.route}");`);
-    lines.push(``);
-    lines.push(`    // TODO: fill in the steps for this test.`);
-    if (testCase.expect === 'pass') {
-        lines.push(`    // This case is expected to succeed.`);
-        lines.push(`    // TODO: assert the success outcome (e.g. redirect or success message).`);
-    }
-    else {
-        lines.push(`    // This case is expected to fail / be rejected.`);
-        lines.push(`    // TODO: assert the error outcome (e.g. validation message shown).`);
-    }
-    lines.push(`  });`);
-    return lines.join('\n');
-}
-/** Build the full Playwright test file content for a journey. */
-export function writePlaywrightFile(journey, cases) {
-    const blocks = cases.map((c) => testBlock(journey, c)).join('\n\n');
-    return header(journey) + blocks + `\n\n});\n`;
-}

package/dist/core/run-generate.js

@@ -16,15 +16,16 @@
  * so tests/seeds land where the project expects them. Defaults to
  * generated-tests/ when not configured.
  */
-import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, } from 'node:fs';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync, } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { parseJourney } from './parser.js';
 import { generateTestCases } from './generator.js';
-import { writePlaywrightFile } from './playwright-writer.js';
+import { writeDeferredStub } from './playwright-writer.js';
 import { generateWithAI, generateWithAIAsync } from './ai-generator.js';
 import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
-import { isSyntaxValid } from './validate.js';
+import { isSyntaxValid, checkTestGates, analyzeFile } from './validate.js';
+import { installedModules, transportHint } from './detect.js';
 import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, } from './harness.js';
 import { hashContent, loadCache, saveCache } from './cache.js';
 import { findMarkdownFiles } from './find-files.js';
@@ -55,7 +56,7 @@ async function pool(items, limit, fn) {
     return results;
 }
 /** Async version of generateValidated (for concurrent per-file gen). */
-async function generateValidatedAsync(generate, ext) {
+async function generateValidatedAsync(generate, ext, gate) {
     let content = '';
     for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
         try {
@@ -64,7 +65,7 @@ async function generateValidatedAsync(generate, ext) {
         catch {
             continue;
         }
-        if (isSyntaxValid(content, ext))
+        if (isSyntaxValid(content, ext) && !(gate && gate(content)))
             return content;
     }
     return null;
@@ -74,7 +75,7 @@ async function generateValidatedAsync(generate, ext) {
  * couple of times on malformed results. Returns the valid content, or
  * null if every attempt failed to produce parseable code.
  */
-function generateValidated(generate, ext) {
+function generateValidated(generate, ext, gate) {
     let content = '';
     for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
         try {
@@ -84,7 +85,7 @@ function generateValidated(generate, ext) {
             // generation (or output cleaning) threw — retry
             continue;
         }
-        if (isSyntaxValid(content, ext)) {
+        if (isSyntaxValid(content, ext) && !(gate && gate(content))) {
             return content;
         }
     }
@@ -95,16 +96,18 @@ function generateValidated(generate, ext) {
  * syntax gate + retry) and falls back to the deterministic basic
  * generator if AI output can't be made valid.
  */
-function buildTest(journey, useAI, ext, testPath, focus) {
+function buildTest(journey, useAI, ext, testPath, framework, focus) {
     if (useAI) {
         const ai = generateValidated(() => generateWithAI(journey, testPath, focus), ext);
         if (ai !== null) {
             return { content: ai, status: 'ai' };
         }
-        // fall through to the basic generator
+        // fall through to the deferred stub
     }
+    // NOT a runnable test: pending placeholders only, correct for the
+    // runner, so nothing ever passes silently.
     const cases = generateTestCases(journey);
-    return { content: writePlaywrightFile(journey, cases), status: 'basic' };
+    return { content: writeDeferredStub(journey, cases, framework), status: 'basic' };
 }
 /**
  * Turn a journey id into the alternate naming conventions we might
@@ -188,6 +191,60 @@ export async function runHarness(only) {
     return written;
 }
 const PLAN_DIR = '.testforge/plans';
+/** Marker every testforge-written test carries, so stale-file cleanup
+ * (F6) only ever removes OUR files, never hand-authored ones. */
+const GEN_MARKER = '@generated by testforge';
+/**
+ * F9 — a journey doc that is still an unconfirmed stub. Deep-generating
+ * confident assertions from it produces UNVALIDATED tests, so we skip it
+ * and tell the user to confirm the spec first.
+ */
+function isStubDoc(markdown) {
+    return (/^#.*\btodo\b/im.test(markdown) ||
+        /not covered by the reference/i.test(markdown) ||
+        /confirm (the )?expected behaviou?r/i.test(markdown) ||
+        /todo:\s*not covered/i.test(markdown));
+}
+/** Slugs recorded in a journey's existing plan cache (the previous run). */
+function readPlanSlugs(journey) {
+    const p = join(PLAN_DIR, `${journey.area}__${journey.id}.json`);
+    try {
+        const c = JSON.parse(readFileSync(p, 'utf-8'));
+        if (Array.isArray(c.files)) {
+            return c.files.map((f) => f.slug).filter((s) => !!s);
+        }
+    }
+    catch {
+        /* no previous plan */
+    }
+    return [];
+}
+/**
+ * F6 — replace, don't duplicate. When a journey is (re)generated and its
+ * plan changed, the files for slugs that no longer exist are stale. Delete
+ * them — but ONLY files that carry our marker, so we never touch a
+ * developer's own tests. Prevents the twin-suite accumulation.
+ */
+function cleanStaleFiles(testDir, journeyId, ext, flat, keepSlugs, dropSlugs) {
+    const removed = [];
+    const keep = new Set(keepSlugs);
+    for (const slug of dropSlugs) {
+        if (keep.has(slug))
+            continue;
+        const base = flat ? `${journeyId}-${slug}` : slug;
+        const f = join(testDir, `${base}${ext}`);
+        try {
+            if (existsSync(f) && readFileSync(f, 'utf-8').includes(GEN_MARKER)) {
+                rmSync(f);
+                removed.push(`${base}${ext}`);
+            }
+        }
+        catch {
+            /* best-effort */
+        }
+    }
+    return removed;
+}
 /**
  * Get a journey's file plan, cached by doc hash. Caching the plan keeps
  * the file slugs STABLE across runs, so per-file resume (skip-existing)
@@ -304,6 +361,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const flat = framework === 'jest';
         const testDir = flat ? areaDir : join(areaDir, journey.id);
         const seedPath = join(seedDir, `${journey.id}${seedExt}`);
+        // Dependency-aware guardrails (F1): what this area may import, and how
+        // it should reach the server — both read from the real project, so the
+        // AI can't fall back to an uninstalled convention (e.g. supertest).
+        const allowedModules = installedModules(testDir);
+        const transport = transportHint(testDir, framework);
+        const testGate = (content) => checkTestGates(content, ext, { framework, allowedModules });
         // Skip if unchanged.
         if (!force && cache[file] === hash) {
             results.push({
@@ -320,6 +383,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const collision = collisionWarning(areaDir, journey.id, ext);
         if (collision)
             warnings.push(collision);
+        // F9 — don't deep-generate from an unconfirmed TODO/stub doc: the
+        // assertions would be guesses. Skip it and ask for a confirmed spec.
+        if (isStubDoc(markdown)) {
+            results.push({
+                journeyName: journey.name,
+                testDir,
+                testFiles: [],
+                seedPath,
+                status: 'skipped',
+                seedStatus: 'none',
+                warnings: [
+                    'doc is an unconfirmed TODO/stub — confirm the expected behaviour before generating (skipped to avoid unvalidated tests)',
+                ],
+            });
+            continue;
+        }
         // Ensure this area's shared harness exists (once), so every file
         // imports the same testkit instead of inventing its own.
         const harnessPath = useAI
@@ -330,9 +409,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         }
         // 1. Plan the suite (deep mode), then generate one focused file per
         //    concern. In basic mode there is no AI planner — one file only.
+        //    Capture the PREVIOUS plan's slugs first so we can clean up any
+        //    files whose slug no longer exists (F6 replace-don't-duplicate).
+        const prevSlugs = useAI ? readPlanSlugs(journey) : [];
         const planned = useAI
             ? loadOrPlan(journey, hash)
             : [{ slug: journey.id, title: journey.name, covers: '' }];
+        const plannedSlugs = planned.map((p) => p.slug);
+        const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
+        if (staleRemoved.length) {
+            warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
+        }
         // Generate all focused files CONCURRENTLY (bounded pool). Each AI
         // call is independent, so a 15-file journey finishes in ~4 waves
         // instead of 15 serial calls.
@@ -359,22 +446,27 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             let content;
             let status;
             if (useAI) {
-                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath), ext);
+                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport), ext, testGate);
                 if (ai !== null) {
                     content = ai;
                     status = 'ai';
                 }
                 else {
-                    const built = buildTest(journey, false, ext, filePath);
+                    const built = buildTest(journey, false, ext, filePath, framework);
                     content = built.content;
                     status = 'basic';
                 }
             }
             else {
-                const built = buildTest(journey, false, ext, filePath);
+                const built = buildTest(journey, false, ext, filePath, framework);
                 content = built.content;
                 status = 'basic';
             }
+            // Stamp our marker (F6) so stale-file cleanup can recognise OUR
+            // files later without ever touching a developer's own tests.
+            if (!content.includes(GEN_MARKER)) {
+                content = `// ${GEN_MARKER}\n${content}`;
+            }
             // Write IMMEDIATELY (not batched at the end of the journey), so an
             // interruption preserves every file generated so far.
             mkdirSync(dirname(filePath), { recursive: true });
@@ -389,6 +481,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const anyAI = testFiles.some((t) => t.status === 'ai');
         const anyBasic = testFiles.some((t) => t.status === 'basic');
         const status = anyAI || !anyBasic ? 'ai' : 'basic';
+        // F10 — complete-generation honesty: any file that fell back to a
+        // deferred stub is NOT real coverage. Surface it clearly (and the
+        // journey stays uncached below, so a later run retries it with AI).
+        const deferred = testFiles.filter((t) => t.status === 'basic');
+        if (useAI && deferred.length) {
+            warnings.push(`${deferred.length} file(s) could not be generated with AI and were left as DEFERRED stubs (pending, not real coverage): ${deferred
+                .map((t) => t.slug)
+                .join(', ')}`);
+        }
         // 2. Seed file (only when using AI — basic mode has no seed concept).
         let seedStatus = 'none';
         if (useAI) {
@@ -433,5 +534,49 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         });
     }
     saveCache(CACHE_PATH, cache);
+    writeCoverageManifest(results);
     return results;
 }
+/**
+ * F8 — write an honest coverage manifest. For every generated file it
+ * records real (has assertions) vs deferred (pending only), so a "green"
+ * run is never mistaken for full coverage. Written to
+ * .testforge/coverage.json and summarised to the console.
+ */
+function writeCoverageManifest(results) {
+    const journeys = results
+        .filter((r) => r.testFiles.length)
+        .map((r) => {
+        const files = r.testFiles.map((t) => {
+            let a = { activeTests: 0, pending: 0, assertions: 0, real: false };
+            try {
+                a = analyzeFile(readFileSync(t.path, 'utf-8'));
+            }
+            catch {
+                /* file unreadable — count as empty */
+            }
+            return { file: t.path, status: t.status, ...a };
+        });
+        return {
+            journey: r.journeyName,
+            real: files.filter((f) => f.real).length,
+            deferred: files.filter((f) => !f.real).length,
+            files,
+        };
+    });
+    if (!journeys.length)
+        return;
+    const totalReal = journeys.reduce((n, j) => n + j.real, 0);
+    const totalDeferred = journeys.reduce((n, j) => n + j.deferred, 0);
+    try {
+        mkdirSync('.testforge', { recursive: true });
+        writeFileSync(join('.testforge', 'coverage.json'), JSON.stringify({ real: totalReal, deferred: totalDeferred, journeys }, null, 2), 'utf-8');
+    }
+    catch {
+        /* best-effort */
+    }
+    console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see .testforge/coverage.json`);
+    if (totalDeferred) {
+        console.log(`  (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
+    }
+}

package/dist/core/validate.js

@@ -20,6 +20,86 @@ import { execFileSync } from 'node:child_process';
 import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
+/**
+ * Semantic gate for a generated test file. Returns a human-readable
+ * reason string if the content violates a guardrail, or null if it is
+ * clean. Runs AFTER syntax validation.
+ */
+export function checkTestGates(content, _ext, ctx) {
+    // F3 — runner correctness: a Jest/backend area must never pull in
+    // Playwright (not installed there; a top-level import breaks the suite).
+    if (ctx.framework === 'jest' && /@playwright\/test/.test(content)) {
+        return "jest area must not import '@playwright/test'";
+    }
+    // F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
+    // pasted description. Literal args containing whitespace are prose.
+    const gotoRe = /\.goto\(\s*(['"])([\s\S]*?)\1/g;
+    let m;
+    while ((m = gotoRe.exec(content)) !== null) {
+        if (/\s/.test(m[2])) {
+            return `goto() argument looks like prose, not a URL/path: "${m[2].slice(0, 40)}…"`;
+        }
+    }
+    // F2 — no silent false-green: a file with RUNNABLE tests must assert
+    // something. Pending placeholders (test.todo/skip/fixme) are exempt —
+    // they don't count as passing. This is what stops an empty-but-green
+    // file (the worst QA outcome) from ever being written.
+    const activeTests = (content.match(/\b(?:it|test)\s*\(/g) || []).length +
+        (content.match(/\b(?:it|test)\.(?:each|concurrent)\b/g) || []).length;
+    const assertions = (content.match(/\bexpect\s*\(/g) || []).length +
+        (content.match(/\bassert\b/g) || []).length;
+    if (activeTests > 0 && assertions === 0) {
+        return 'has runnable test(s) but no assertions — would pass without testing anything';
+    }
+    // F1 — dependency awareness: reject imports/requires of a module the
+    // project can't resolve (bare specifier not in allowedModules).
+    if (ctx.allowedModules) {
+        for (const mod of importedModules(content)) {
+            const top = topLevelModule(mod);
+            if (!ctx.allowedModules.has(top)) {
+                return `imports '${mod}' which the project does not have installed`;
+            }
+        }
+    }
+    return null;
+}
+/**
+ * F8 — lightweight coverage stats for one generated file, so a manifest
+ * can tell REAL coverage from deferred placeholders (a green run is never
+ * mistaken for full coverage).
+ */
+export function analyzeFile(content) {
+    const activeTests = (content.match(/\b(?:it|test)\s*\(/g) || []).length +
+        (content.match(/\b(?:it|test)\.(?:each|concurrent)\b/g) || []).length;
+    const pending = (content.match(/\b(?:it|test|describe)\.(?:todo|skip|fixme)\b/g) || []).length;
+    const assertions = (content.match(/\bexpect\s*\(/g) || []).length;
+    return { activeTests, pending, assertions, real: activeTests > 0 && assertions > 0 };
+}
+/** Extract bare module specifiers from import/require (skips relative). */
+function importedModules(content) {
+    const mods = new Set();
+    const patterns = [
+        /\bimport\b[^'"]*?from\s*(['"])([^'"]+)\1/g,
+        /\bimport\s*(['"])([^'"]+)\1/g,
+        /\brequire\(\s*(['"])([^'"]+)\1\s*\)/g,
+    ];
+    for (const re of patterns) {
+        let m;
+        while ((m = re.exec(content)) !== null) {
+            const spec = m[2];
+            if (!spec.startsWith('.') && !spec.startsWith('/'))
+                mods.add(spec);
+        }
+    }
+    return [...mods];
+}
+/** `@scope/pkg/sub` -> `@scope/pkg`; `pkg/sub` -> `pkg`; `node:fs` -> `node:fs`. */
+function topLevelModule(spec) {
+    if (spec.startsWith('node:'))
+        return spec;
+    const parts = spec.split('/');
+    return spec.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0];
+}
 /** True if `content` parses (or, for TS, passes a structural check). */
 export function isSyntaxValid(content, ext) {
     if (!content.trim())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.10.1",
+  "version": "0.11.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {