@activate-core/test-forge 0.18.2 → 0.18.4

package/dist/cli/index.js

@@ -175,9 +175,13 @@ async function main() {
                 }
             }
             const generated = results.filter((r) => r.status !== 'skipped').length;
-            const skipped = results.filter((r) => r.status === 'skipped').length;
+            // Rows with no test dir are notices (a non-journey file, the AI going
+            // away), not journeys that were unchanged — count them apart.
+            const notices = results.filter((r) => r.status === 'skipped' && !r.testDir).length;
+            const skipped = results.filter((r) => r.status === 'skipped' && r.testDir).length;
             const seedFailures = results.filter((r) => r.seedStatus === 'failed').length;
-            console.log(`\nDone. ${generated} generated, ${skipped} skipped (unchanged).`);
+            console.log(`\nDone. ${generated} generated, ${skipped} skipped (unchanged)` +
+                (notices ? `, ${notices} notice(s) above.` : '.'));
             if (seedFailures > 0) {
                 console.log(`Warning: ${seedFailures} seed file(s) could not be generated — see notes above.`);
             }

package/dist/core/ai-generator.js

@@ -49,8 +49,14 @@ function runWithStdin(program, args, input, timeoutMs = 300_000) {
             finish(() => {
                 if (code === 0)
                     resolve(out);
-                else
-                    reject(new Error(`${program} exited ${code}: ${err.slice(0, 500)}`));
+                else {
+                    // The limit message is not always on stderr — some CLI versions
+                    // print it to stdout, some print nothing at all. Say what we have,
+                    // and say plainly when that is nothing, so a silent exit is never
+                    // mistaken for a message-less mystery downstream.
+                    const detail = (err.trim() || out.trim()).slice(0, 500) || '(no output)';
+                    reject(new Error(`${program} exited ${code}: ${detail}`));
+                }
             });
         });
         child.stdin.end(input);

package/dist/core/discover.js

@@ -15,8 +15,9 @@
  * keeps this simple and works for projects of any size.
  */
 import { execFileSync } from 'node:child_process';
-import { existsSync, readdirSync, readFileSync, writeFileSync, } from 'node:fs';
-import { join, relative, sep } from 'node:path';
+import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, renameSync, } from 'node:fs';
+import { join, relative, sep, basename } from 'node:path';
+import { notAJourneyDoc } from './parser.js';
 import { loadConfig, splitCommand, withEditTools } from './config.js';
 import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
 import { promptConfirm } from './prompt.js';
@@ -58,7 +59,10 @@ export function buildDiscoverPrompt(srcDir, docsDir, journey) {
         `  "${docsDir}/web/sign-in.md"`,
         `  "${docsDir}/ops/login.md"`,
         `Do NOT put area in the file name (no "api-payments.md") — the area`,
-        `is the folder. Create the area subfolders as needed.`,
+        `is the folder. Create the area subfolders as needed. Write NOTHING at`,
+        `the root of "${docsDir}" and no file that is not a journey doc — a`,
+        `shared note that every journey should know about goes under`,
+        `"${docsDir}/_reference/" and is never a journey.`,
         ``,
         `Each file should be a RICH description of the journey. Start with`,
         `this header (required):`,
@@ -208,6 +212,27 @@ export async function runDiscover(srcDir, docsDir, opts = {}, journey) {
             else if (prior !== text)
                 rewritten.push(path);
         }
+        // The prompt says "<area>/<journey>.md". The model still sometimes leaves a
+        // shared side-note at the docs root ("data traps every journey must know
+        // about"). Useful text, wrong place: generate would read it as a journey
+        // and build a suite named after the file. Move such files under
+        // _reference/ — kept, never a journey — and drop them from the record.
+        const quarantined = [];
+        for (const path of [...added]) {
+            const why = notAJourneyDoc(path, after.get(path));
+            if (!why)
+                continue;
+            const dest = join('_reference', basename(path));
+            mkdirSync(join(docsDir, '_reference'), { recursive: true });
+            renameSync(join(docsDir, path), join(docsDir, dest));
+            added.splice(added.indexOf(path), 1);
+            quarantined.push(`${path} -> ${dest} (${why})`);
+        }
+        if (quarantined.length) {
+            console.log(`\n${quarantined.length} file(s) were not journey docs and were moved under ` +
+                `${docsDir}/_reference/ (kept as notes, never generated from):\n  ` +
+                quarantined.join('\n  '));
+        }
         // Put back anything it reached for that this reference does not own. The
         // prompt asks it not to; this makes it true whether or not it complied.
         const restored = rewritten.filter((path) => wholeProject ? ownedElsewhere.has(path) : true);

package/dist/core/parser.js

@@ -65,6 +65,24 @@ function parseField(line) {
     const required = parts.includes('required');
     return { name, type, required };
 }
+/**
+ * Is this file a journey doc at all? The layout discover promises — and every
+ * downstream step assumes — is "<area>/<journey>.md" with a "# Journey:"
+ * heading. Anything else under docs/ is reference material: a project's own
+ * notes checked in before TestForge arrived, or a shared side-note the AI wrote
+ * next to the journeys. Treating those as journeys produced test folders named
+ * after the file and suites with no name. Returns the reason when it is not.
+ */
+export function notAJourneyDoc(relPath, markdown) {
+    const parts = relPath.split(/[\\/]/).filter(Boolean);
+    if (parts.length < 2)
+        return 'sits at the docs root, not in an area folder';
+    if (parts[0].startsWith('_'))
+        return `lives under ${parts[0]}/, which holds reference material`;
+    if (!/^#\s*Journey:/m.test(markdown))
+        return 'has no "# Journey:" heading';
+    return null;
+}
 /**
  * Parse a full markdown string into a Journey object.
  *

package/dist/core/reference.js

@@ -175,7 +175,10 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
         `- Where the reference is silent on a needed detail, add a "# TODO:"`,
         `  line describing what the developer must verify.`,
         `- Never include secrets — refer to env var names only.`,
-        `- Create the area subfolders as needed.`,
+        `- Create the area subfolders as needed. Write NOTHING at the root of`,
+        `  "${docsDir}" and no file that is not a journey doc — a shared note`,
+        `  every journey should know about goes under "${docsDir}/_reference/"`,
+        `  and is never a journey.`,
         `- After creating the files, print the list of journeys you documented`,
         `  and remind the developer to review them.`,
     ].join('\n');

package/dist/core/run-generate.js

@@ -20,7 +20,7 @@ import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync
 import { join, dirname, basename, extname } from 'node:path';
 import { CACHE_PATH, COVERAGE_PATH, PLANS_DIR, STATE_DIR } from './paths.js';
 import { lintGenerated } from './lint.js';
-import { parseJourney } from './parser.js';
+import { parseJourney, notAJourneyDoc } from './parser.js';
 import { generateTestCases } from './generator.js';
 import { writeDeferredStub } from './playwright-writer.js';
 import { generateWithAI, generateWithAIAsync, pingAi, } from './ai-generator.js';
@@ -68,13 +68,52 @@ async function pool(items, limit, fn) {
 // journeys: the deferred stubs are re-attempted automatically on the next run.
 let lastAiFailure = null;
 let aiUnavailable = null;
+// Process failures in a row with no success between them. The CLI does not
+// always SAY it is out — on wagonex it exited 1 with empty stderr for every
+// call and the message-matching below saw nothing to match, so thirteen
+// journeys were burned before anyone noticed. Two silent failures in a row
+// earn a ping; if the ping fails too, the AI is unavailable, message or not.
+let consecutiveAiFailures = 0;
 const AI_UNAVAILABLE = /usage limit|spend limit|limit will reset|rate limit|\b429\b|not logged in|please log ?in|unauthori[sz]ed|invalid api key|credit balance|quota|spawn \S+ ENOENT/i;
 function noteAiFailure(e) {
-    const msg = (e instanceof Error ? e.message : String(e)).replace(/\s+/g, ' ').trim();
+    // execFileSync attaches what the process said; node's own message is just
+    // "Command failed: <cmd>", which explains nothing.
+    const x = e;
+    const said = [x.stderr, x.stdout].map((v) => String(v ?? '').trim()).find(Boolean);
+    let msg = (x.message ?? String(e)).replace(/\s+/g, ' ').trim();
+    if (said && !msg.includes(said.slice(0, 40)))
+        msg += `: ${said}`;
+    if (x.signal === 'SIGKILL' && !/timed out/.test(msg))
+        msg += ' (killed — hit aiTimeoutMs)';
     lastAiFailure = msg.slice(0, 300);
+    consecutiveAiFailures += 1;
     if (AI_UNAVAILABLE.test(msg))
         aiUnavailable = lastAiFailure;
 }
+function noteAiRejected(why) {
+    // The AI answered, but with something we will not write. Not a sign it is
+    // gone — do not count it towards unavailability — but the operator still
+    // deserves to know why the file is a stub.
+    lastAiFailure = `AI output rejected: ${why}`.slice(0, 300);
+}
+function noteAiSuccess() {
+    consecutiveAiFailures = 0;
+}
+/**
+ * Two silent failures in a row: ask the CLI directly whether it is alive.
+ * Returns true when it was failing but answers now — the caller then redoes
+ * the journey those failures degraded, exactly as after a wait.
+ */
+async function confirmAiAlive() {
+    if (aiUnavailable || consecutiveAiFailures < 2)
+        return false;
+    if (!(await pingAi(60_000))) {
+        aiUnavailable = lastAiFailure ?? 'the AI CLI is not answering';
+        return false;
+    }
+    consecutiveAiFailures = 0;
+    return true;
+}
 function aiFailureNote() {
     return lastAiFailure ? ` — last AI error: ${lastAiFailure}` : '';
 }
@@ -119,8 +158,11 @@ async function generateValidatedAsync(generate, ext, gate) {
             noteAiFailure(e);
             continue;
         }
-        if (isSyntaxValid(content, ext) && !(gate && gate(content)))
+        noteAiSuccess();
+        const gateMsg = gate ? gate(content) : null;
+        if (isSyntaxValid(content, ext) && !gateMsg)
             return content;
+        noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
     }
     return null;
 }
@@ -142,7 +184,12 @@ function generateValidated(generate, ext, gate) {
             noteAiFailure(e);
             continue;
         }
-        if (isSyntaxValid(content, ext) && !(gate && gate(content))) {
+        noteAiSuccess();
+        const gateMsg = gate ? gate(content) : null;
+        if (!(isSyntaxValid(content, ext) && !gateMsg)) {
+            noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
+        }
+        if (isSyntaxValid(content, ext) && !gateMsg) {
             return content;
         }
     }
@@ -374,8 +421,8 @@ function matchesJourney(file, journeyId, sel) {
         norm(stem).split('/').pop() === n);
 }
 export async function runGenerate(docsDir, useAI = true, force = false, only, areaOnly, opts = {}) {
-    const files = findMarkdownFiles(docsDir);
-    if (files.length === 0) {
+    const allFiles = findMarkdownFiles(docsDir);
+    if (allFiles.length === 0) {
         return [];
     }
     const config = loadConfig();
@@ -386,6 +433,35 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     const results = [];
     lastAiFailure = null;
     aiUnavailable = null;
+    consecutiveAiFailures = 0;
+    // Only journey docs are generated from. A file at the docs root, under a
+    // "_reference/" folder, or without a "# Journey:" heading is reference
+    // material — report it once and leave it alone rather than inventing a
+    // journey (and a test folder) out of it.
+    for (const file of allFiles) {
+        let why = null;
+        try {
+            why = notAJourneyDoc(file, readFileSync(join(docsDir, file), 'utf-8'));
+        }
+        catch {
+            why = 'could not be read';
+        }
+        if (!why)
+            continue;
+        results.push({
+            journeyName: file,
+            testDir: '',
+            testFiles: [],
+            seedPath: '',
+            status: 'skipped',
+            seedStatus: 'none',
+            warnings: [
+                `not a journey doc — it ${why}. Kept as reference material and ignored by generate. ` +
+                    `Journeys live at ${docsDir}/<area>/<name>.md and open with "# Journey: …".`,
+            ],
+        });
+    }
+    const files = allFiles.filter((f) => !results.some((r) => r.journeyName === f));
     // Real UI test-hooks per area, extracted once from the components — feeds
     // the F10 selector-grounding gate so generated UI tests can only assert on
     // selectors that genuinely exist (never an invented data-testid).
@@ -427,32 +503,54 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     // replaces the degraded row instead of reporting the journey twice (once as
     // stubs, once real) and counting the stale stubs in the coverage header.
     const resultRow = new Map();
+    // Journeys that ended this run with at least one deferred stub while the AI
+    // was failing. An outage that ends MID-journey (say, during its seed) leaves
+    // the counter reset and the stubs behind; redo such a journey once, as soon
+    // as the AI is answering — the stubs are all it costs.
+    const stubbed = new Set();
     for (let i = 0; i < files.length; i++) {
         const file = files[i];
+        // Redo the journey BEFORE this one: it degraded into stubs while the AI
+        // was going down. Its real files are kept by per-file resume, so the redo
+        // costs only the stubs — once, then carry on. A shared harness that failed
+        // during the outage was memoised as "none" for its area; forget that so
+        // the redo rebuilds it. Returns true when a redo was scheduled.
+        const redoPrevious = () => {
+            for (const [k, v] of harnessByArea)
+                if (v === null)
+                    harnessByArea.delete(k);
+            if (i === 0 || redone.has(i - 1))
+                return false;
+            redone.add(i - 1);
+            const stale = resultRow.get(files[i - 1]);
+            if (stale !== undefined) {
+                results.splice(stale, 1);
+                resultRow.delete(files[i - 1]);
+            }
+            i -= 2;
+            return true;
+        };
+        // Silent failures say nothing to match on — ask the CLI directly. If it
+        // was failing a moment ago and answers now, the outage has already passed:
+        // redo what it broke and carry on.
+        if (useAI && (await confirmAiAlive()) && redoPrevious())
+            continue;
+        if (useAI &&
+            !aiUnavailable &&
+            consecutiveAiFailures === 0 &&
+            i > 0 &&
+            stubbed.has(files[i - 1]) &&
+            redoPrevious()) {
+            continue;
+        }
         // The AI is gone (limit reached, logged out, not installed). Every further
         // journey would only become deferred stubs. Either wait for it to come back
         // (--wait-for-limit) or stop here; either way the stubs already written are
         // re-attempted automatically on the next run.
         if (aiUnavailable && useAI) {
             if (waitForLimit && (await waitForAi(config))) {
-                // A shared harness that failed while the AI was down was memoised as
-                // "none" for its area; forget that so the redo below rebuilds it.
-                for (const [k, v] of harnessByArea)
-                    if (v === null)
-                        harnessByArea.delete(k);
-                // The journey BEFORE this one degraded into stubs while the AI was
-                // going down. Its real files are kept by per-file resume, so redoing
-                // it now costs only the stubs — once, then carry on.
-                if (i > 0 && !redone.has(i - 1)) {
-                    redone.add(i - 1);
-                    const stale = resultRow.get(files[i - 1]);
-                    if (stale !== undefined) {
-                        results.splice(stale, 1);
-                        resultRow.delete(files[i - 1]);
-                    }
-                    i -= 2;
+                if (redoPrevious())
                     continue;
-                }
             }
             else {
                 notAttempted += 1;
@@ -706,6 +804,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         }
         saveCache(CACHE_PATH, cache);
         resultRow.set(file, results.length);
+        if (useAI && testFiles.some((t) => t.status === 'basic'))
+            stubbed.add(file);
+        else
+            stubbed.delete(file);
         results.push({
             journeyName: journey.name,
             testDir,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.18.2",
+  "version": "0.18.4",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {