@activate-core/test-forge 0.10.2 → 0.12.0

package/dist/core/run-generate.js

@@ -16,16 +16,17 @@
  * so tests/seeds land where the project expects them. Defaults to
  * generated-tests/ when not configured.
  */
-import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, } from 'node:fs';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync, } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { parseJourney } from './parser.js';
 import { generateTestCases } from './generator.js';
-import { writePlaywrightFile } from './playwright-writer.js';
+import { writeDeferredStub } from './playwright-writer.js';
 import { generateWithAI, generateWithAIAsync } from './ai-generator.js';
 import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
-import { isSyntaxValid } from './validate.js';
-import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, } from './harness.js';
+import { isSyntaxValid, checkTestGates, analyzeFile } from './validate.js';
+import { installedModules, transportHint, projectSelectors } from './detect.js';
+import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, } from './harness.js';
 import { hashContent, loadCache, saveCache } from './cache.js';
 import { findMarkdownFiles } from './find-files.js';
 import { loadConfig, testDirForType, resolveFramework } from './config.js';
@@ -55,7 +56,7 @@ async function pool(items, limit, fn) {
     return results;
 }
 /** Async version of generateValidated (for concurrent per-file gen). */
-async function generateValidatedAsync(generate, ext) {
+async function generateValidatedAsync(generate, ext, gate) {
     let content = '';
     for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
         try {
@@ -64,7 +65,7 @@ async function generateValidatedAsync(generate, ext) {
         catch {
             continue;
         }
-        if (isSyntaxValid(content, ext))
+        if (isSyntaxValid(content, ext) && !(gate && gate(content)))
             return content;
     }
     return null;
@@ -74,7 +75,7 @@ async function generateValidatedAsync(generate, ext) {
  * couple of times on malformed results. Returns the valid content, or
  * null if every attempt failed to produce parseable code.
  */
-function generateValidated(generate, ext) {
+function generateValidated(generate, ext, gate) {
     let content = '';
     for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
         try {
@@ -84,7 +85,7 @@ function generateValidated(generate, ext) {
             // generation (or output cleaning) threw — retry
             continue;
         }
-        if (isSyntaxValid(content, ext)) {
+        if (isSyntaxValid(content, ext) && !(gate && gate(content))) {
             return content;
         }
     }
@@ -95,16 +96,18 @@ function generateValidated(generate, ext) {
  * syntax gate + retry) and falls back to the deterministic basic
  * generator if AI output can't be made valid.
  */
-function buildTest(journey, useAI, ext, testPath, focus) {
+function buildTest(journey, useAI, ext, testPath, framework, focus) {
     if (useAI) {
         const ai = generateValidated(() => generateWithAI(journey, testPath, focus), ext);
         if (ai !== null) {
             return { content: ai, status: 'ai' };
         }
-        // fall through to the basic generator
+        // fall through to the deferred stub
     }
+    // NOT a runnable test: pending placeholders only, correct for the
+    // runner, so nothing ever passes silently.
     const cases = generateTestCases(journey);
-    return { content: writePlaywrightFile(journey, cases), status: 'basic' };
+    return { content: writeDeferredStub(journey, cases, framework), status: 'basic' };
 }
 /**
  * Turn a journey id into the alternate naming conventions we might
@@ -172,6 +175,7 @@ export async function runHarness(only) {
         }
         mkdirSync(dirname(info.path), { recursive: true });
         writeFileSync(info.path, kit, 'utf-8');
+        writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
         if (framework === 'playwright') {
             const gsPath = join(info.dir, 'global-setup.js');
             writeFileSync(gsPath, globalSetupContent(), 'utf-8');
@@ -188,6 +192,97 @@ export async function runHarness(only) {
     return written;
 }
 const PLAN_DIR = '.testforge/plans';
+/** Marker every testforge-written test carries, so stale-file cleanup
+ * (F6) only ever removes OUR files, never hand-authored ones. */
+const GEN_MARKER = '@generated by testforge';
+/**
+ * F9 — a journey doc that is still an unconfirmed stub. Deep-generating
+ * confident assertions from it produces UNVALIDATED tests, so we skip it
+ * and tell the user to confirm the spec first.
+ */
+function isStubDoc(markdown) {
+    // The doc's FIRST heading is its premise. A confirmed journey opens with a
+    // real title (e.g. "# Journey: ..."); a genuine stub opens by declaring
+    // itself uncovered ("# TODO: not covered by the reference ...").
+    //
+    // We deliberately look ONLY at the opening heading. Partial "reference
+    // SILENT" annotations on individual SECTIONS of an otherwise-complete doc,
+    // and trailing "developer verification checklist" sections, are healthy —
+    // the previous whole-document scan wrongly skipped confirmed journeys that
+    // merely carried such notes.
+    const firstHeading = (markdown.match(/^#{1,6}\s+.+$/m) || [''])[0];
+    return /\btodo\b|not covered by the reference|reference[-\s]silent|unconfirmed/i.test(firstHeading);
+}
+/**
+ * True if a generated file is a testforge DEFERRED stub — a placeholder
+ * written when AI generation was unavailable. It is NOT real coverage, so
+ * per-file resume must RE-ATTEMPT it (unlike a real file, which is kept).
+ */
+function isDeferredStub(content) {
+    return content.includes('DEFERRED stub');
+}
+/** True if the journey's output dir already holds any deferred stub. */
+function hasDeferredStubs(testDir, ext) {
+    let files;
+    try {
+        files = readdirSync(testDir);
+    }
+    catch {
+        return false;
+    }
+    for (const f of files) {
+        if (!f.endsWith(ext))
+            continue;
+        try {
+            if (isDeferredStub(readFileSync(join(testDir, f), 'utf-8')))
+                return true;
+        }
+        catch {
+            /* ignore unreadable */
+        }
+    }
+    return false;
+}
+/** Slugs recorded in a journey's existing plan cache (the previous run). */
+function readPlanSlugs(journey) {
+    const p = join(PLAN_DIR, `${journey.area}__${journey.id}.json`);
+    try {
+        const c = JSON.parse(readFileSync(p, 'utf-8'));
+        if (Array.isArray(c.files)) {
+            return c.files.map((f) => f.slug).filter((s) => !!s);
+        }
+    }
+    catch {
+        /* no previous plan */
+    }
+    return [];
+}
+/**
+ * F6 — replace, don't duplicate. When a journey is (re)generated and its
+ * plan changed, the files for slugs that no longer exist are stale. Delete
+ * them — but ONLY files that carry our marker, so we never touch a
+ * developer's own tests. Prevents the twin-suite accumulation.
+ */
+function cleanStaleFiles(testDir, journeyId, ext, flat, keepSlugs, dropSlugs) {
+    const removed = [];
+    const keep = new Set(keepSlugs);
+    for (const slug of dropSlugs) {
+        if (keep.has(slug))
+            continue;
+        const base = flat ? `${journeyId}-${slug}` : slug;
+        const f = join(testDir, `${base}${ext}`);
+        try {
+            if (existsSync(f) && readFileSync(f, 'utf-8').includes(GEN_MARKER)) {
+                rmSync(f);
+                removed.push(`${base}${ext}`);
+            }
+        }
+        catch {
+            /* best-effort */
+        }
+    }
+    return removed;
+}
 /**
  * Get a journey's file plan, cached by doc hash. Caching the plan keeps
  * the file slugs STABLE across runs, so per-file resume (skip-existing)
@@ -238,6 +333,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
     const cache = loadCache(CACHE_PATH);
     const results = [];
+    // Real UI test-hooks per area, extracted once from the components — feeds
+    // the F10 selector-grounding gate so generated UI tests can only assert on
+    // selectors that genuinely exist (never an invented data-testid).
+    const selectorsByArea = new Map();
     // Shared harness per area — generated once, then every test in that
     // area imports it (so files stop inventing their own helper paths).
     const harnessByArea = new Map();
@@ -261,6 +360,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         }
         mkdirSync(dirname(info.path), { recursive: true });
         writeFileSync(info.path, kit, 'utf-8');
+        writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
         // Playwright areas need a config + globalSetup so the specs can run
         // AND get their tokens minted once before the suite.
         if (framework === 'playwright') {
@@ -304,8 +404,25 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const flat = framework === 'jest';
         const testDir = flat ? areaDir : join(areaDir, journey.id);
         const seedPath = join(seedDir, `${journey.id}${seedExt}`);
-        // Skip if unchanged.
-        if (!force && cache[file] === hash) {
+        // Dependency-aware guardrails (F1): what this area may import, and how
+        // it should reach the server — both read from the real project, so the
+        // AI can't fall back to an uninstalled convention (e.g. supertest).
+        const allowedModules = installedModules(testDir);
+        const transport = transportHint(testDir, framework);
+        // F10 — selector grounding (Playwright areas only): the real test-hooks
+        // in this area's components, so a generated UI test can't invent one.
+        let knownSelectors;
+        if (framework === 'playwright') {
+            knownSelectors = selectorsByArea.get(areaDir);
+            if (!knownSelectors) {
+                knownSelectors = projectSelectors(areaDir);
+                selectorsByArea.set(areaDir, knownSelectors);
+            }
+        }
+        const testGate = (content) => checkTestGates(content, ext, { framework, allowedModules, knownSelectors });
+        // Skip if unchanged — but NOT if the journey still has deferred stubs
+        // to retry (a resume pass fills those in without touching real files).
+        if (!force && cache[file] === hash && !hasDeferredStubs(testDir, ext)) {
             results.push({
                 journeyName: journey.name,
                 testDir,
@@ -320,6 +437,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const collision = collisionWarning(areaDir, journey.id, ext);
         if (collision)
             warnings.push(collision);
+        // F9 — don't deep-generate from an unconfirmed TODO/stub doc: the
+        // assertions would be guesses. Skip it and ask for a confirmed spec.
+        if (isStubDoc(markdown)) {
+            results.push({
+                journeyName: journey.name,
+                testDir,
+                testFiles: [],
+                seedPath,
+                status: 'skipped',
+                seedStatus: 'none',
+                warnings: [
+                    'doc is an unconfirmed TODO/stub — confirm the expected behaviour before generating (skipped to avoid unvalidated tests)',
+                ],
+            });
+            continue;
+        }
         // Ensure this area's shared harness exists (once), so every file
         // imports the same testkit instead of inventing its own.
         const harnessPath = useAI
@@ -330,13 +463,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         }
         // 1. Plan the suite (deep mode), then generate one focused file per
         //    concern. In basic mode there is no AI planner — one file only.
+        //    Capture the PREVIOUS plan's slugs first so we can clean up any
+        //    files whose slug no longer exists (F6 replace-don't-duplicate).
+        const prevSlugs = useAI ? readPlanSlugs(journey) : [];
         const planned = useAI
             ? loadOrPlan(journey, hash)
             : [{ slug: journey.id, title: journey.name, covers: '' }];
+        const plannedSlugs = planned.map((p) => p.slug);
+        const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
+        if (staleRemoved.length) {
+            warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
+        }
         // Generate all focused files CONCURRENTLY (bounded pool). Each AI
-        // call is independent, so a 15-file journey finishes in ~4 waves
-        // instead of 15 serial calls.
-        const generated = await pool(planned, GEN_CONCURRENCY, async (pf) => {
+        // call is independent, so a 15-file journey finishes in several waves
+        // instead of one-at-a-time. Concurrency is configurable: too many
+        // parallel agentic AI runs starve each other and time out, so a flaky
+        // environment can lower `aiConcurrency` (e.g. 2) for reliability.
+        const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
+        const generated = await pool(planned, concurrency, async (pf) => {
             // Flat (jest): prefix the file with the journey so many journeys can
             // share one dir without clashing. Folder (playwright): the journey
             // folder already namespaces it, so the slug alone is enough.
@@ -348,7 +492,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             // (--force ignores this and regenerates.)
             if (!force && existsSync(filePath)) {
                 try {
-                    if (isSyntaxValid(readFileSync(filePath, 'utf-8'), ext)) {
+                    const existing = readFileSync(filePath, 'utf-8');
+                    // Keep a real, parsing file — but a DEFERRED stub is not real
+                    // coverage, so fall through and re-attempt it on this resume pass.
+                    if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
                         return { slug: pf.slug, path: filePath, status: 'ai' };
                     }
                 }
@@ -359,22 +506,27 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             let content;
             let status;
             if (useAI) {
-                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath), ext);
+                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport), ext, testGate);
                 if (ai !== null) {
                     content = ai;
                     status = 'ai';
                 }
                 else {
-                    const built = buildTest(journey, false, ext, filePath);
+                    const built = buildTest(journey, false, ext, filePath, framework);
                     content = built.content;
                     status = 'basic';
                 }
             }
             else {
-                const built = buildTest(journey, false, ext, filePath);
+                const built = buildTest(journey, false, ext, filePath, framework);
                 content = built.content;
                 status = 'basic';
             }
+            // Stamp our marker (F6) so stale-file cleanup can recognise OUR
+            // files later without ever touching a developer's own tests.
+            if (!content.includes(GEN_MARKER)) {
+                content = `// ${GEN_MARKER}\n${content}`;
+            }
             // Write IMMEDIATELY (not batched at the end of the journey), so an
             // interruption preserves every file generated so far.
             mkdirSync(dirname(filePath), { recursive: true });
@@ -389,6 +541,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const anyAI = testFiles.some((t) => t.status === 'ai');
         const anyBasic = testFiles.some((t) => t.status === 'basic');
         const status = anyAI || !anyBasic ? 'ai' : 'basic';
+        // F10 — complete-generation honesty: any file that fell back to a
+        // deferred stub is NOT real coverage. Surface it clearly (and the
+        // journey stays uncached below, so a later run retries it with AI).
+        const deferred = testFiles.filter((t) => t.status === 'basic');
+        if (useAI && deferred.length) {
+            warnings.push(`${deferred.length} file(s) could not be generated with AI and were left as DEFERRED stubs (pending, not real coverage): ${deferred
+                .map((t) => t.slug)
+                .join(', ')}`);
+        }
         // 2. Seed file (only when using AI — basic mode has no seed concept).
         let seedStatus = 'none';
         if (useAI) {
@@ -433,5 +594,49 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         });
     }
     saveCache(CACHE_PATH, cache);
+    writeCoverageManifest(results);
     return results;
 }
+/**
+ * F8 — write an honest coverage manifest. For every generated file it
+ * records real (has assertions) vs deferred (pending only), so a "green"
+ * run is never mistaken for full coverage. Written to
+ * .testforge/coverage.json and summarised to the console.
+ */
+function writeCoverageManifest(results) {
+    const journeys = results
+        .filter((r) => r.testFiles.length)
+        .map((r) => {
+        const files = r.testFiles.map((t) => {
+            let a = { activeTests: 0, pending: 0, assertions: 0, real: false };
+            try {
+                a = analyzeFile(readFileSync(t.path, 'utf-8'));
+            }
+            catch {
+                /* file unreadable — count as empty */
+            }
+            return { file: t.path, status: t.status, ...a };
+        });
+        return {
+            journey: r.journeyName,
+            real: files.filter((f) => f.real).length,
+            deferred: files.filter((f) => !f.real).length,
+            files,
+        };
+    });
+    if (!journeys.length)
+        return;
+    const totalReal = journeys.reduce((n, j) => n + j.real, 0);
+    const totalDeferred = journeys.reduce((n, j) => n + j.deferred, 0);
+    try {
+        mkdirSync('.testforge', { recursive: true });
+        writeFileSync(join('.testforge', 'coverage.json'), JSON.stringify({ real: totalReal, deferred: totalDeferred, journeys }, null, 2), 'utf-8');
+    }
+    catch {
+        /* best-effort */
+    }
+    console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see .testforge/coverage.json`);
+    if (totalDeferred) {
+        console.log(`  (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
+    }
+}

package/dist/core/seed-generator.js

@@ -42,11 +42,22 @@ export function buildSeedPrompt(journeyDoc, seedPath) {
         `the seed code to stdout.`,
         ``,
         `Requirements for the seed you write:`,
-        `- Connect using the env var named in the map (e.g.`,
-        `  process.env.DB_URL). NEVER hardcode a connection string or any`,
-        `  secret.`,
-        `- Make it IDEMPOTENT: use upsert by a stable, well-known test id so`,
-        `  running it repeatedly (fresh DB or already-seeded) is safe.`,
+        `- Connect using the env var(s) named in the map. If the app spans`,
+        `  MULTIPLE databases (e.g. main + chat/conversation + identity/IM),`,
+        `  connect to and seed EACH into the DB it belongs to — do not assume a`,
+        `  single process.env.DB_URL. NEVER hardcode a connection string or secret.`,
+        `- Make it IDEMPOTENT with an explicit field split: upsert by a stable`,
+        `  id using { $set: <mutable fields>, $setOnInsert: { createdAt, ...other`,
+        `  creation-only fields } } (updateOne/findOneAndUpdate, upsert:true). Do`,
+        `  NOT use replaceOne, and do NOT put createdAt/creation metadata in $set`,
+        `  — that clobbers it on every re-run. Seed shared/base entities (e.g. an`,
+        `  identity user) INSERT-ONLY ($setOnInsert only) so a developer's richer`,
+        `  local record is never overwritten.`,
+        `- Every well-known id MUST be env-overridable: read`,
+        `  process.env.TEST_<NAME>_ID (falling back to a stable default), using`,
+        `  the SAME names the testkit's TEST_IDS exposes. This lets the same`,
+        `  fixtures target data created BY HAND on staging/prod without code`,
+        `  edits (used with SEED_MODE=existing, where writes are skipped).`,
         `- Prefix test data clearly (e.g. names/ids starting with "qa-" or`,
         `  "QA-TEST") so it is obvious and easy to clean up.`,
         `- Insert only what THIS feature's tests need as a precondition,`,

package/dist/core/validate.js

@@ -20,6 +20,225 @@ import { execFileSync } from 'node:child_process';
 import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
+/**
+ * Semantic gate for a generated test file. Returns a human-readable
+ * reason string if the content violates a guardrail, or null if it is
+ * clean. Runs AFTER syntax validation.
+ */
+export function checkTestGates(content, _ext, ctx) {
+    // F3 — runner correctness: a Jest/backend area must never pull in
+    // Playwright (not installed there; a top-level import breaks the suite).
+    if (ctx.framework === 'jest' && /@playwright\/test/.test(content)) {
+        return "jest area must not import '@playwright/test'";
+    }
+    // F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
+    // pasted description and not an unfilled route-file placeholder. Literal
+    // args with whitespace are prose; `[param]`/`:param`/`{param}`/`<param>`
+    // are route templates that navigate to a garbage literal path at runtime.
+    // (Only quote-delimited literals are inspected — a backtick template like
+    // `/s/${code}` is the correct, resolved form and is intentionally exempt.)
+    const gotoRe = /\.goto\(\s*(['"])([\s\S]*?)\1/g;
+    let m;
+    while ((m = gotoRe.exec(content)) !== null) {
+        const arg = m[2];
+        if (/\s/.test(arg)) {
+            return `goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`;
+        }
+        if (/\[[A-Za-z_]|\/:[A-Za-z_]|\{[A-Za-z_]|<[A-Za-z_]/.test(arg)) {
+            return `goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`;
+        }
+    }
+    // F2 — no silent false-green: a file with RUNNABLE tests must assert
+    // something. Pending placeholders (test.todo/skip/fixme) are exempt —
+    // they don't count as passing. This is what stops an empty-but-green
+    // file (the worst QA outcome) from ever being written.
+    const activeTests = (content.match(/\b(?:it|test)\s*\(/g) || []).length +
+        (content.match(/\b(?:it|test)\.(?:each|concurrent)\b/g) || []).length;
+    const assertions = (content.match(/\bexpect\s*\(/g) || []).length +
+        (content.match(/\bassert\b/g) || []).length;
+    if (activeTests > 0 && assertions === 0) {
+        return 'has runnable test(s) but no assertions — would pass without testing anything';
+    }
+    // F1 — dependency awareness: reject imports/requires of a module the
+    // project can't resolve (bare specifier not in allowedModules).
+    if (ctx.allowedModules) {
+        for (const mod of importedModules(content)) {
+            const top = topLevelModule(mod);
+            if (!ctx.allowedModules.has(top)) {
+                return `imports '${mod}' which the project does not have installed`;
+            }
+        }
+    }
+    // F11 — hermetic seam (jest/backend): a test that imports a known
+    // EXTERNAL network service (payment/SMS/email/cloud) must mock it, or the
+    // suite silently depends on a live third party — it flakes offline and can
+    // fire real charges/messages. DB clients are intentionally NOT listed:
+    // backend integration tests legitimately use a real (local) database.
+    if (ctx.framework === 'jest') {
+        const EXTERNAL = new Set([
+            'stripe', 'twilio', '@sendgrid/mail', '@sendgrid/client',
+            'nodemailer', 'aws-sdk', '@aws-sdk/client-ses', '@aws-sdk/client-sns',
+        ]);
+        for (const mod of importedModules(content)) {
+            const top = topLevelModule(mod);
+            if (!EXTERNAL.has(top))
+                continue;
+            const mocked = content.includes(`jest.mock('${mod}'`) ||
+                content.includes(`jest.mock("${mod}"`) ||
+                content.includes(`jest.mock('${top}'`) ||
+                content.includes(`jest.mock("${top}"`);
+            if (!mocked) {
+                return `imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`;
+            }
+        }
+    }
+    // F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
+    // on a guessed selector. Unchecked, the generator invents plausible
+    // data-testids that time out at runtime — the journey is never really
+    // exercised, and the red looks like a product failure instead of a test
+    // gap. A guessed selector must be grounded in a real component attribute,
+    // or the test must be explicitly deferred (test.fixme/skip). This is
+    // feature-agnostic — it guards every UI journey, not any one flow.
+    if (ctx.framework === 'playwright') {
+        const active = stripPendingBlocks(content);
+        // (a) The generator's own tell: a selector sitting next to a
+        // placeholder marker on the same line (e.g. `// TODO: real selector`).
+        const marker = String.raw `(?:TODO|FIXME|placeholder|real selector|guess(?:ed)?|unconfirmed|not\s+verified)`;
+        const sel = String.raw `(?:getByTestId\s*\(|\[\s*data-(?:testid|cy|test)\s*=)`;
+        const selNearMarker = new RegExp(`(?:${sel}[^\\n]*?\\b${marker}\\b)|(?:\\b${marker}\\b[^\\n]*?${sel})`, 'i');
+        if (selNearMarker.test(active)) {
+            return 'a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)';
+        }
+        // (b) When the project's real test-hooks are known, any test-id a
+        // runnable test uses that isn't one of them is an invented selector.
+        if (ctx.knownSelectors && ctx.knownSelectors.size > 0) {
+            for (const id of usedTestIds(active)) {
+                if (!ctx.knownSelectors.has(id)) {
+                    return `references selector '${id}' not found in the project's components — ground it or defer the test`;
+                }
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Remove pending test blocks (`it/test/describe.skip|fixme|todo(...)`) so the
+ * F9 selector check only ever inspects RUNNABLE code. A guessed selector
+ * inside a deferred test is fine — it's an honest gap, not a false red.
+ * String- and comment-aware paren matching so a `(` inside a body or string
+ * doesn't unbalance the scan.
+ */
+function stripPendingBlocks(content) {
+    const startRe = /\b(?:it|test|describe)\.(?:skip|fixme|todo)\s*\(/g;
+    let out = '';
+    let last = 0;
+    let m;
+    while ((m = startRe.exec(content)) !== null) {
+        const openIdx = m.index + m[0].length - 1; // index of the '('
+        const end = matchParen(content, openIdx);
+        if (end === -1)
+            continue; // unbalanced — leave as-is, don't drop tail
+        out += content.slice(last, m.index);
+        last = end + 1;
+        startRe.lastIndex = last;
+    }
+    out += content.slice(last);
+    return out;
+}
+/**
+ * Given the index of an opening `(`, return the index of its matching `)`,
+ * skipping over quoted strings, template literals, and comments. Returns -1
+ * if no match is found.
+ */
+function matchParen(s, open) {
+    let depth = 0;
+    for (let i = open; i < s.length; i++) {
+        const c = s[i];
+        // skip strings / templates
+        if (c === "'" || c === '"' || c === '`') {
+            const q = c;
+            i++;
+            while (i < s.length && s[i] !== q) {
+                if (s[i] === '\\')
+                    i++;
+                i++;
+            }
+            continue;
+        }
+        // skip comments
+        if (c === '/' && s[i + 1] === '/') {
+            while (i < s.length && s[i] !== '\n')
+                i++;
+            continue;
+        }
+        if (c === '/' && s[i + 1] === '*') {
+            i += 2;
+            while (i < s.length && !(s[i] === '*' && s[i + 1] === '/'))
+                i++;
+            i++;
+            continue;
+        }
+        if (c === '(')
+            depth++;
+        else if (c === ')') {
+            depth--;
+            if (depth === 0)
+                return i;
+        }
+    }
+    return -1;
+}
+/** Test-hook values a file references: getByTestId('x'), [data-testid="x"], data-cy, data-test. */
+function usedTestIds(content) {
+    const ids = new Set();
+    const patterns = [
+        /getByTestId\s*\(\s*(['"`])([^'"`]+)\1/g,
+        /\[\s*data-(?:testid|cy|test)\s*=\s*(['"])([^'"]+)\1\s*\]/g,
+    ];
+    for (const re of patterns) {
+        let m;
+        while ((m = re.exec(content)) !== null)
+            ids.add(m[2]);
+    }
+    return [...ids];
+}
+/**
+ * F8 — lightweight coverage stats for one generated file, so a manifest
+ * can tell REAL coverage from deferred placeholders (a green run is never
+ * mistaken for full coverage).
+ */
+export function analyzeFile(content) {
+    const activeTests = (content.match(/\b(?:it|test)\s*\(/g) || []).length +
+        (content.match(/\b(?:it|test)\.(?:each|concurrent)\b/g) || []).length;
+    const pending = (content.match(/\b(?:it|test|describe)\.(?:todo|skip|fixme)\b/g) || []).length;
+    const assertions = (content.match(/\bexpect\s*\(/g) || []).length;
+    return { activeTests, pending, assertions, real: activeTests > 0 && assertions > 0 };
+}
+/** Extract bare module specifiers from import/require (skips relative). */
+function importedModules(content) {
+    const mods = new Set();
+    const patterns = [
+        /\bimport\b[^'"]*?from\s*(['"])([^'"]+)\1/g,
+        /\bimport\s*(['"])([^'"]+)\1/g,
+        /\brequire\(\s*(['"])([^'"]+)\1\s*\)/g,
+    ];
+    for (const re of patterns) {
+        let m;
+        while ((m = re.exec(content)) !== null) {
+            const spec = m[2];
+            if (!spec.startsWith('.') && !spec.startsWith('/'))
+                mods.add(spec);
+        }
+    }
+    return [...mods];
+}
+/** `@scope/pkg/sub` -> `@scope/pkg`; `pkg/sub` -> `pkg`; `node:fs` -> `node:fs`. */
+function topLevelModule(spec) {
+    if (spec.startsWith('node:'))
+        return spec;
+    const parts = spec.split('/');
+    return spec.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0];
+}
 /** True if `content` parses (or, for TS, passes a structural check). */
 export function isSyntaxValid(content, ext) {
     if (!content.trim())

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.10.2",
+  "version": "0.12.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {