@activate-core/test-forge 0.18.1 → 0.18.2

package/dist/cli/index.js

@@ -77,6 +77,7 @@ function printUsage() {
     console.log('  testforge generate           Generate tests from the journey docs');
     console.log('  testforge generate --basic   Use the basic generator only');
     console.log('  testforge generate --force   Regenerate everything');
+    console.log('  testforge generate --no-wait  Stop when the AI hits its usage limit (default: wait for the reset and resume)');
     console.log('  testforge generate --journey <name>  Generate just ONE journey');
     console.log('  testforge generate --area <area>     Generate a whole area (e.g. api)');
     console.log('  testforge discover          Draft journey docs (asks: map or reference)');
@@ -136,14 +137,22 @@ async function main() {
         console.log(useAI
             ? 'Generating tests with AI for changed journeys...\n'
             : 'Generating tests with the basic generator...\n');
-        const results = await runGenerate('docs', useAI, force, flagValue('--journey'), flagValue('--area'));
+        const results = await runGenerate('docs', useAI, force, flagValue('--journey'), flagValue('--area'), {
+            waitForLimit: process.argv.includes('--no-wait')
+                ? false
+                : process.argv.includes('--wait-for-limit')
+                    ? true
+                    : undefined, // undefined -> config.waitForLimit (default on)
+        });
         if (results.length === 0) {
             console.log('No journey files found in docs/. Add a .md file and try again.');
         }
         else {
             for (const r of results) {
                 if (r.status === 'skipped') {
-                    console.log(`  [skip] ${r.journeyName} (unchanged)`);
+                    // A row with no test dir is the run-level notice (AI unavailable), not an
+                    // unchanged journey — do not label it as one.
+                    console.log(r.testDir ? `  [skip] ${r.journeyName} (unchanged)` : `  [skip] ${r.journeyName}`);
                 }
                 else {
                     const tag = r.status === 'ai' ? '[AI]' : '[basic]';

package/dist/core/ai-generator.js

@@ -254,6 +254,28 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    the fixture and every later suite reads it as real — one defect`,
         `    becomes a hundred failures in journeys that were never broken, and`,
         `    the run cannot be trusted until someone repairs the data by hand.`,
+        `    CLEAN UP BY A HANDLE THAT CANNOT GO MISSING. If cleanup matches a`,
+        `    name prefix, check that every record the file creates can actually`,
+        `    carry that prefix — validation minimums, enum values and generated`,
+        `    identifiers routinely forbid it. One record the filter cannot see`,
+        `    survives, and the next run collides with its own leftovers and`,
+        `    reports the collision as a product error. Prefer an owning field`,
+        `    that is always set (a parent, an owner id, a namespace).`,
+        `(12) A ROLE'S CREDENTIAL MUST BE THAT ROLE'S — NEVER FALL THROUGH TO`,
+        `    ANOTHER ONE. When a test needs a specific actor (an admin, a staff`,
+        `    user, a second customer) and that actor's credential is missing or`,
+        `    expired, the answer is to SKIP, not to reach for whichever token`,
+        `    the harness happens to hold. A live token for the wrong actor`,
+        `    passes every "is it present / is it in date" check and then gets`,
+        `    401/403 from every route the real actor is allowed to use — which`,
+        `    reads exactly like the product refusing legitimate access. This is`,
+        `    the single most expensive mistake available to you: one fallback in`,
+        `    a shared helper turns a whole area's suite into false defects, and`,
+        `    nothing in the output says the wrong identity was used. So: resolve`,
+        `    each role from its OWN source, never chain roles in a fallback`,
+        `    list, and treat a template placeholder (DUMMY, replace_me, an empty`,
+        `    value, a bracketed name) as absent rather than as a credential.`,
+        `    Say which actor was missing in the skip reason.`,
         `When rules conflict, prefer reporting LESS: a pending test costs a line`,
         `in the report; a false failure costs the team a release.`,
         ``,
@@ -522,6 +544,22 @@ export async function generateWithAIAsync(journey, testPath, focus, harnessPath,
     const stdout = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
     return cleanOutput(stdout);
 }
+/**
+ * True if the AI CLI answers at all. Used to wait out a usage limit: the CLI
+ * exits non-zero with a limit message until the window resets, then answers
+ * again — this is the cheapest question that tells the two apart.
+ */
+export async function pingAi(timeoutMs = 60_000) {
+    try {
+        const config = loadConfig();
+        const { program, args } = splitCommand(config.aiCommand);
+        const out = await runWithStdin(program, args, 'Reply with exactly: PONG', timeoutMs);
+        return /PONG/.test(out);
+    }
+    catch {
+        return false;
+    }
+}
 /** Does this line look like the start (or body) of real code? */
 function looksLikeCode(line) {
     const t = line.trim();

package/dist/core/config.js

@@ -39,6 +39,9 @@ const DEFAULT_CONFIG = {
     testCommand: {},
     envFile: '.env.testing',
     externalModules: [],
+    waitForLimit: true,
+    aiWaitPollMs: 20 * 60_000,
+    aiWaitMaxMs: 8 * 3_600_000,
 };
 /** Load config from the project, falling back to defaults. */
 export function loadConfig() {

package/dist/core/harness.js

@@ -138,11 +138,30 @@ export const HARNESS_CONTRACT = [
     `from the AUTHENTICATED TOKEN, not from a query param — and a route`,
     `declared auth:false may STILL populate request.user via a pre-auth`,
     `extension when a Bearer is sent. Therefore:`,
-    `  - setup() MUST resolve the authenticated identity's REAL id (decode`,
-    `    the minted token's subject/_id claim, or call the profile/whoami`,
-    `    endpoint) and seed the OWNER fixtures (whichever role in this`,
-    `    product owns the data the tests read) under THAT id — then set`,
-    `    the TEST_IDS owner ids to it.`,
+    `  - setup() MUST resolve the authenticated identity's REAL id by ASKING`,
+    `    THE PRODUCT — the profile/whoami/detail endpoint — and seed the`,
+    `    OWNER fixtures (whichever role in this product owns the data the`,
+    `    tests read) under THAT id, then set the TEST_IDS owner ids to it.`,
+    `    The token's subject is the IDENTITY PROVIDER's handle, not the`,
+    `    app's record id: "sms|<hex>" and "auth0|<hex>" look like an id with`,
+    `    the prefix stripped, and the value underneath is routinely a`,
+    `    DIFFERENT record from the one the app stores. Decode the subject`,
+    `    only as a fallback when the endpoint cannot be reached, and say so`,
+    `    where you do it. Seeding from a provider subject writes an id the`,
+    `    app can never cast: owner reads come back empty and writes answer`,
+    `    500, both of which read as the product being broken.`,
+    `  - A seeded BROWSER session carries the same id. Whatever the app`,
+    `    stores about the signed-in user (its web storage keys) must hold`,
+    `    the resolved app id, because the client copies it into the`,
+    `    payloads it sends. And resolve it FOR THE ROLE being seeded — a`,
+    `    maker session built from the wearer's identity tests nothing.`,
+    `  - Each role resolves from its OWN credential. Never chain roles in a`,
+    `    fallback list: a live token for the wrong actor passes every`,
+    `    presence and expiry check and then collects 401/403 from every`,
+    `    route the real actor is allowed to use. Treat a template`,
+    `    placeholder (DUMMY, replace_me, empty, <bracketed>) as absent, and`,
+    `    when a role's credential is missing, SKIP with its name in the`,
+    `    reason.`,
     `  - Do NOT hardcode an owner id the token does not own: the seeded rows`,
     `    would never be returned to the authenticated caller, and every`,
     `    owner-scoped test would fail for a fake reason.`,

package/dist/core/parser.js

@@ -72,7 +72,7 @@ function parseField(line) {
  * output location and framework. When omitted (a journey read outside an
  * area folder), it falls back to the parsed `Type:`.
  */
-export function parseJourney(markdown, area) {
+export function parseJourney(markdown, area, fallbackName) {
     const lines = markdown.split('\n').map((l) => l.trim());
     let name = '';
     let type = 'web';
@@ -115,6 +115,16 @@ export function parseJourney(markdown, area) {
             }
         }
     }
+    // Not every doc opens with "# Journey: …". A reference dropped straight
+    // into docs/ usually carries a plain title; take that, and failing any
+    // heading at all, the file's own name. An empty name became an empty id and
+    // a test folder literally called "<file>.md", listed with no journey name.
+    if (!name) {
+        const heading = lines.find((l) => /^#{1,6}\s+\S/.test(l) && !/^#{1,6}\s+TODO\b/i.test(l));
+        name = heading
+            ? heading.replace(/^#{1,6}\s+/, '').replace(/^Journey:\s*/i, '').trim()
+            : (fallbackName ?? '').trim();
+    }
     return {
         id: toId(name),
         name,

package/dist/core/run-generate.js

@@ -17,13 +17,13 @@
  * generated-tests/ when not configured.
  */
 import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync, } from 'node:fs';
-import { join, dirname } from 'node:path';
+import { join, dirname, basename, extname } from 'node:path';
 import { CACHE_PATH, COVERAGE_PATH, PLANS_DIR, STATE_DIR } from './paths.js';
 import { lintGenerated } from './lint.js';
 import { parseJourney } from './parser.js';
 import { generateTestCases } from './generator.js';
 import { writeDeferredStub } from './playwright-writer.js';
-import { generateWithAI, generateWithAIAsync } from './ai-generator.js';
+import { generateWithAI, generateWithAIAsync, pingAi, } from './ai-generator.js';
 import { groundJourney } from './grounding.js';
 import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
@@ -57,14 +57,66 @@ async function pool(items, limit, fn) {
     await Promise.all(Array.from({ length: n }, () => worker()));
     return results;
 }
+// WHY the last AI attempt failed, and whether the AI is gone for the whole run.
+//
+// Both validators used to swallow the error and retry, so when the AI CLI was
+// unavailable — usage limit reached mid-run, not logged in, not installed —
+// the operator saw "could not be generated with AI" forty-three times and no
+// reason, while every remaining journey was burned into a deferred stub and
+// every seed retried against a dead CLI. Record the reason, put it in the
+// warning, and once the failure is clearly "AI unavailable" stop starting new
+// journeys: the deferred stubs are re-attempted automatically on the next run.
+let lastAiFailure = null;
+let aiUnavailable = null;
+const AI_UNAVAILABLE = /usage limit|spend limit|limit will reset|rate limit|\b429\b|not logged in|please log ?in|unauthori[sz]ed|invalid api key|credit balance|quota|spawn \S+ ENOENT/i;
+function noteAiFailure(e) {
+    const msg = (e instanceof Error ? e.message : String(e)).replace(/\s+/g, ' ').trim();
+    lastAiFailure = msg.slice(0, 300);
+    if (AI_UNAVAILABLE.test(msg))
+        aiUnavailable = lastAiFailure;
+}
+function aiFailureNote() {
+    return lastAiFailure ? ` — last AI error: ${lastAiFailure}` : '';
+}
+/**
+ * Wait for the AI CLI to answer again after a usage limit. Polls with a
+ * one-line question every `aiWaitPollMs` for up to `aiWaitMaxMs`; true when
+ * it is back (and the failure state is cleared), false when the cap is hit.
+ * Progress goes to the console because a run that sits silent for hours
+ * looks hung.
+ */
+async function waitForAi(config) {
+    const poll = config.aiWaitPollMs ?? 20 * 60_000;
+    const max = config.aiWaitMaxMs ?? 8 * 3_600_000;
+    const mins = (ms) => `${Math.max(1, Math.round(ms / 60_000))} min`;
+    console.log(`\nAI unavailable: ${aiUnavailable}\n` +
+        `Waiting for it to come back — checking every ${mins(poll)}, for up to ${mins(max)}. ` +
+        `Ctrl-C is safe: re-running later resumes from exactly here.`);
+    const started = Date.now();
+    while (Date.now() - started < max) {
+        await new Promise((r) => setTimeout(r, poll));
+        if (await pingAi(60_000)) {
+            console.log(`AI is back (${new Date().toLocaleTimeString()}) — resuming.`);
+            aiUnavailable = null;
+            lastAiFailure = null;
+            return true;
+        }
+        console.log(`${new Date().toLocaleTimeString()} still unavailable — next check in ${mins(poll)}`);
+    }
+    console.log(`Gave up waiting after ${mins(max)}.`);
+    return false;
+}
 /** Async version of generateValidated (for concurrent per-file gen). */
 async function generateValidatedAsync(generate, ext, gate) {
     let content = '';
     for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
+        if (aiUnavailable)
+            return null; // retrying a dead CLI only burns time
         try {
             content = await generate();
         }
-        catch {
+        catch (e) {
+            noteAiFailure(e);
             continue;
         }
         if (isSyntaxValid(content, ext) && !(gate && gate(content)))
@@ -80,11 +132,14 @@ async function generateValidatedAsync(generate, ext, gate) {
 function generateValidated(generate, ext, gate) {
     let content = '';
     for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
+        if (aiUnavailable)
+            return null; // retrying a dead CLI only burns time
         try {
             content = generate();
         }
-        catch {
-            // generation (or output cleaning) threw — retry
+        catch (e) {
+            // generation (or output cleaning) threw — remember why, then retry
+            noteAiFailure(e);
             continue;
         }
         if (isSyntaxValid(content, ext) && !(gate && gate(content))) {
@@ -318,7 +373,7 @@ function matchesJourney(file, journeyId, sel) {
         norm(stem).endsWith(`-${n}`) ||
         norm(stem).split('/').pop() === n);
 }
-export async function runGenerate(docsDir, useAI = true, force = false, only, areaOnly) {
+export async function runGenerate(docsDir, useAI = true, force = false, only, areaOnly, opts = {}) {
     const files = findMarkdownFiles(docsDir);
     if (files.length === 0) {
         return [];
@@ -329,6 +384,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
     const cache = loadCache(CACHE_PATH);
     const results = [];
+    lastAiFailure = null;
+    aiUnavailable = null;
     // Real UI test-hooks per area, extracted once from the components — feeds
     // the F10 selector-grounding gate so generated UI tests can only assert on
     // selectors that genuinely exist (never an invented data-testid).
@@ -363,15 +420,58 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         harnessByArea.set(area, info.path);
         return info.path;
     }
-    for (const file of files) {
+    const waitForLimit = opts.waitForLimit ?? config.waitForLimit ?? true;
+    let notAttempted = 0;
+    const redone = new Set();
+    // Where each journey's result row sits, so a redo after the AI comes back
+    // replaces the degraded row instead of reporting the journey twice (once as
+    // stubs, once real) and counting the stale stubs in the coverage header.
+    const resultRow = new Map();
+    for (let i = 0; i < files.length; i++) {
+        const file = files[i];
+        // The AI is gone (limit reached, logged out, not installed). Every further
+        // journey would only become deferred stubs. Either wait for it to come back
+        // (--wait-for-limit) or stop here; either way the stubs already written are
+        // re-attempted automatically on the next run.
+        if (aiUnavailable && useAI) {
+            if (waitForLimit && (await waitForAi(config))) {
+                // A shared harness that failed while the AI was down was memoised as
+                // "none" for its area; forget that so the redo below rebuilds it.
+                for (const [k, v] of harnessByArea)
+                    if (v === null)
+                        harnessByArea.delete(k);
+                // The journey BEFORE this one degraded into stubs while the AI was
+                // going down. Its real files are kept by per-file resume, so redoing
+                // it now costs only the stubs — once, then carry on.
+                if (i > 0 && !redone.has(i - 1)) {
+                    redone.add(i - 1);
+                    const stale = resultRow.get(files[i - 1]);
+                    if (stale !== undefined) {
+                        results.splice(stale, 1);
+                        resultRow.delete(files[i - 1]);
+                    }
+                    i -= 2;
+                    continue;
+                }
+            }
+            else {
+                notAttempted += 1;
+                continue;
+            }
+        }
         const markdown = readFileSync(join(docsDir, file), 'utf-8');
         const hash = hashContent(markdown);
         // The area is the doc's top-level folder (api/web/ops/lambda/...). It
         // decides where the test goes and which framework is used — NOT the
         // "Type:" line, which only says UI-vs-endpoint. This keeps ops/lambda
         // journeys in their own package instead of leaking into web/api.
-        const area = file.split(/[\\/]/)[0] || undefined;
-        const journey = parseJourney(markdown, area);
+        //
+        // A doc dropped straight into docs/ has NO folder: splitting on "/" then
+        // returns the file name itself, which became the "area" and produced a
+        // test folder literally called generated-tests/<file>.md/. No folder means
+        // no area — the parser falls back to the doc's Type.
+        const area = /[\\/]/.test(file) ? file.split(/[\\/]/)[0] : undefined;
+        const journey = parseJourney(markdown, area, basename(file, extname(file)));
         // --area <a>: only this area's journeys (e.g. do all of api first).
         if (areaOnly && journey.area !== areaOnly)
             continue;
@@ -553,7 +653,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         if (useAI && deferred.length) {
             warnings.push(`${deferred.length} file(s) could not be generated with AI and were left as DEFERRED stubs (pending, not real coverage): ${deferred
                 .map((t) => t.slug)
-                .join(', ')}`);
+                .join(', ')}${aiFailureNote()}`);
         }
         // 2. Seed file (only when using AI — basic mode has no seed concept).
         let seedStatus = 'none';
@@ -568,7 +668,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
                 // Do NOT write a broken seed. Report it so the user sees which
                 // seeds were not produced.
                 seedStatus = 'failed';
-                warnings.push(`seed generation failed after ${MAX_GEN_ATTEMPTS} attempts — no seed written`);
+                warnings.push(`seed generation failed after ${MAX_GEN_ATTEMPTS} attempts — no seed written${aiFailureNote()}`);
             }
         }
         // Only mark a journey "done" in the cache when it actually SUCCEEDED
@@ -605,6 +705,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             delete cache[file];
         }
         saveCache(CACHE_PATH, cache);
+        resultRow.set(file, results.length);
         results.push({
             journeyName: journey.name,
             testDir,
@@ -615,6 +716,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             warnings: warnings.length ? warnings : undefined,
         });
     }
+    if (notAttempted > 0 && aiUnavailable) {
+        // One line, not one per journey: the operator needs the reason and the
+        // count, then to re-run once the CLI is back.
+        results.push({
+            journeyName: `${notAttempted} journey(s) not attempted — AI unavailable`,
+            testDir: '',
+            testFiles: [],
+            seedPath: '',
+            status: 'skipped',
+            seedStatus: 'none',
+            warnings: [
+                `${aiUnavailable}. Nothing was written for these; re-run \`testforge generate\` when the AI CLI is back — deferred stubs from this run are re-attempted automatically.`,
+            ],
+        });
+    }
     saveCache(CACHE_PATH, cache);
     writeCoverageManifest(results);
     return results;

package/dist/core/validate.js

@@ -70,6 +70,32 @@ export function checkTestGates(content, _ext, ctx) {
             return `goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`;
         }
     }
+    // F13 — a swallowed action must be bounded. `.click({ force: true })`
+    // followed by `.catch(() => {})` is the shape used for a tap that is
+    // EXPECTED to do nothing (the handler has already disabled the control).
+    // `force` skips the actionability checks but still waits for the element,
+    // and with no timeout that wait is unbounded: the swallowed promise sits
+    // until the test deadline and the failure then surfaces on whatever line
+    // ran next, naming an innocent call. Any `.catch()`-swallowed action must
+    // carry its own timeout.
+    const swallowed = /\.(?:click|fill|press|hover|check|uncheck|selectOption|tap)\(\s*\{[^}]*\}\s*\)\s*\.catch\s*\(/g;
+    let sw;
+    while ((sw = swallowed.exec(content)) !== null) {
+        if (!/timeout\s*:/.test(sw[0])) {
+            return ('an action whose failure is swallowed with .catch() has no timeout — ' +
+                'it will wait until the whole test times out and blame a later line');
+        }
+    }
+    // F14 — arm the wait BEFORE the thing it waits for. A waitForResponse /
+    // waitForRequest / waitForEvent written AFTER the navigation or click that
+    // triggers it is a race the test usually loses: the response has already
+    // arrived, so the wait sits out its full timeout waiting for a second one
+    // that never comes. Hold the promise first, then act, then await it.
+    const lateWait = /\b(?:await\s+)?page\.goto\([^\n]*\);?\s*\n(?:\s*\/\/[^\n]*\n)*\s*await\s+page\.waitFor(?:Response|Request|Event)\s*\(/;
+    if (lateWait.test(content)) {
+        return ('page.waitForResponse/Request is armed AFTER the navigation that triggers ' +
+            'it — hold the promise first, then navigate, then await it');
+    }
     // F2 — no silent false-green: a file with RUNNABLE tests must assert
     // something. Pending placeholders (test.todo/skip/fixme) are exempt —
     // they don't count as passing. This is what stops an empty-but-green

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.18.1",
+  "version": "0.18.2",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {