@activate-core/test-forge 0.18.3 → 0.18.5

package/dist/core/ai-generator.js

@@ -49,8 +49,14 @@ function runWithStdin(program, args, input, timeoutMs = 300_000) {
             finish(() => {
                 if (code === 0)
                     resolve(out);
-                else
-                    reject(new Error(`${program} exited ${code}: ${err.slice(0, 500)}`));
+                else {
+                    // The limit message is not always on stderr — some CLI versions
+                    // print it to stdout, some print nothing at all. Say what we have,
+                    // and say plainly when that is nothing, so a silent exit is never
+                    // mistaken for a message-less mystery downstream.
+                    const detail = (err.trim() || out.trim()).slice(0, 500) || '(no output)';
+                    reject(new Error(`${program} exited ${code}: ${detail}`));
+                }
             });
         });
         child.stdin.end(input);

package/dist/core/run-generate.js

@@ -68,13 +68,52 @@ async function pool(items, limit, fn) {
 // journeys: the deferred stubs are re-attempted automatically on the next run.
 let lastAiFailure = null;
 let aiUnavailable = null;
+// Process failures in a row with no success between them. The CLI does not
+// always SAY it is out — on wagonex it exited 1 with empty stderr for every
+// call and the message-matching below saw nothing to match, so thirteen
+// journeys were burned before anyone noticed. Two silent failures in a row
+// earn a ping; if the ping fails too, the AI is unavailable, message or not.
+let consecutiveAiFailures = 0;
 const AI_UNAVAILABLE = /usage limit|spend limit|limit will reset|rate limit|\b429\b|not logged in|please log ?in|unauthori[sz]ed|invalid api key|credit balance|quota|spawn \S+ ENOENT/i;
 function noteAiFailure(e) {
-    const msg = (e instanceof Error ? e.message : String(e)).replace(/\s+/g, ' ').trim();
+    // execFileSync attaches what the process said; node's own message is just
+    // "Command failed: <cmd>", which explains nothing.
+    const x = e;
+    const said = [x.stderr, x.stdout].map((v) => String(v ?? '').trim()).find(Boolean);
+    let msg = (x.message ?? String(e)).replace(/\s+/g, ' ').trim();
+    if (said && !msg.includes(said.slice(0, 40)))
+        msg += `: ${said}`;
+    if (x.signal === 'SIGKILL' && !/timed out/.test(msg))
+        msg += ' (killed — hit aiTimeoutMs)';
     lastAiFailure = msg.slice(0, 300);
+    consecutiveAiFailures += 1;
     if (AI_UNAVAILABLE.test(msg))
         aiUnavailable = lastAiFailure;
 }
+function noteAiRejected(why) {
+    // The AI answered, but with something we will not write. Not a sign it is
+    // gone — do not count it towards unavailability — but the operator still
+    // deserves to know why the file is a stub.
+    lastAiFailure = `AI output rejected: ${why}`.slice(0, 300);
+}
+function noteAiSuccess() {
+    consecutiveAiFailures = 0;
+}
+/**
+ * Two silent failures in a row: ask the CLI directly whether it is alive.
+ * Returns true when it was failing but answers now — the caller then redoes
+ * the journey those failures degraded, exactly as after a wait.
+ */
+async function confirmAiAlive() {
+    if (aiUnavailable || consecutiveAiFailures < 2)
+        return false;
+    if (!(await pingAi(60_000))) {
+        aiUnavailable = lastAiFailure ?? 'the AI CLI is not answering';
+        return false;
+    }
+    consecutiveAiFailures = 0;
+    return true;
+}
 function aiFailureNote() {
     return lastAiFailure ? ` — last AI error: ${lastAiFailure}` : '';
 }
@@ -119,8 +158,11 @@ async function generateValidatedAsync(generate, ext, gate) {
             noteAiFailure(e);
             continue;
         }
-        if (isSyntaxValid(content, ext) && !(gate && gate(content)))
+        noteAiSuccess();
+        const gateMsg = gate ? gate(content) : null;
+        if (isSyntaxValid(content, ext) && !gateMsg)
             return content;
+        noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
     }
     return null;
 }
@@ -142,7 +184,12 @@ function generateValidated(generate, ext, gate) {
             noteAiFailure(e);
             continue;
         }
-        if (isSyntaxValid(content, ext) && !(gate && gate(content))) {
+        noteAiSuccess();
+        const gateMsg = gate ? gate(content) : null;
+        if (!(isSyntaxValid(content, ext) && !gateMsg)) {
+            noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
+        }
+        if (isSyntaxValid(content, ext) && !gateMsg) {
             return content;
         }
     }
@@ -386,6 +433,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     const results = [];
     lastAiFailure = null;
     aiUnavailable = null;
+    consecutiveAiFailures = 0;
     // Only journey docs are generated from. A file at the docs root, under a
     // "_reference/" folder, or without a "# Journey:" heading is reference
     // material — report it once and leave it alone rather than inventing a
@@ -455,32 +503,60 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     // replaces the degraded row instead of reporting the journey twice (once as
     // stubs, once real) and counting the stale stubs in the coverage header.
     const resultRow = new Map();
+    // Journeys that ended this run with at least one deferred stub while the AI
+    // was failing. An outage that ends MID-journey (say, during its seed) leaves
+    // the counter reset and the stubs behind; redo such a journey once, as soon
+    // as the AI is answering — the stubs are all it costs.
+    const stubbed = new Set();
+    // Areas whose test framework is not installed. A Playwright spec MUST import
+    // "@playwright/test"; without it every generated file is refused by the
+    // dependency gate and becomes a deferred stub. That is an environment
+    // problem, not something to discover thirteen times over a night, so the
+    // area is skipped with the install command and no AI is called at all.
+    const blockedAreas = new Map();
     for (let i = 0; i < files.length; i++) {
         const file = files[i];
+        // Redo the journey BEFORE this one: it degraded into stubs while the AI
+        // was going down. Its real files are kept by per-file resume, so the redo
+        // costs only the stubs — once, then carry on. A shared harness that failed
+        // during the outage was memoised as "none" for its area; forget that so
+        // the redo rebuilds it. Returns true when a redo was scheduled.
+        const redoPrevious = () => {
+            for (const [k, v] of harnessByArea)
+                if (v === null)
+                    harnessByArea.delete(k);
+            if (i === 0 || redone.has(i - 1))
+                return false;
+            redone.add(i - 1);
+            const stale = resultRow.get(files[i - 1]);
+            if (stale !== undefined) {
+                results.splice(stale, 1);
+                resultRow.delete(files[i - 1]);
+            }
+            i -= 2;
+            return true;
+        };
+        // Silent failures say nothing to match on — ask the CLI directly. If it
+        // was failing a moment ago and answers now, the outage has already passed:
+        // redo what it broke and carry on.
+        if (useAI && (await confirmAiAlive()) && redoPrevious())
+            continue;
+        if (useAI &&
+            !aiUnavailable &&
+            consecutiveAiFailures === 0 &&
+            i > 0 &&
+            stubbed.has(files[i - 1]) &&
+            redoPrevious()) {
+            continue;
+        }
         // The AI is gone (limit reached, logged out, not installed). Every further
         // journey would only become deferred stubs. Either wait for it to come back
         // (--wait-for-limit) or stop here; either way the stubs already written are
         // re-attempted automatically on the next run.
         if (aiUnavailable && useAI) {
             if (waitForLimit && (await waitForAi(config))) {
-                // A shared harness that failed while the AI was down was memoised as
-                // "none" for its area; forget that so the redo below rebuilds it.
-                for (const [k, v] of harnessByArea)
-                    if (v === null)
-                        harnessByArea.delete(k);
-                // The journey BEFORE this one degraded into stubs while the AI was
-                // going down. Its real files are kept by per-file resume, so redoing
-                // it now costs only the stubs — once, then carry on.
-                if (i > 0 && !redone.has(i - 1)) {
-                    redone.add(i - 1);
-                    const stale = resultRow.get(files[i - 1]);
-                    if (stale !== undefined) {
-                        results.splice(stale, 1);
-                        resultRow.delete(files[i - 1]);
-                    }
-                    i -= 2;
+                if (redoPrevious())
                     continue;
-                }
             }
             else {
                 notAttempted += 1;
@@ -522,6 +598,19 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         // AI can't fall back to an uninstalled convention (e.g. supertest).
         const allowedModules = installedModules(testDir);
         const transport = transportHint(testDir, framework);
+        // Playwright is the one framework whose import is mandatory in every spec
+        // (jest's describe/it/expect are ambient, so a jest area generates fine
+        // either way). If it is missing, stop before spending a single AI call.
+        if (framework === 'playwright' && !allowedModules.has('@playwright/test')) {
+            const blocked = blockedAreas.get(journey.area) ?? {
+                pkg: '@playwright/test',
+                dir: areaDir,
+                count: 0,
+            };
+            blocked.count += 1;
+            blockedAreas.set(journey.area, blocked);
+            continue;
+        }
         // F10 — selector grounding (Playwright areas only): the real test-hooks
         // in this area's components, so a generated UI test can't invent one.
         let knownSelectors;
@@ -734,6 +823,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         }
         saveCache(CACHE_PATH, cache);
         resultRow.set(file, results.length);
+        if (useAI && testFiles.some((t) => t.status === 'basic'))
+            stubbed.add(file);
+        else
+            stubbed.delete(file);
         results.push({
             journeyName: journey.name,
             testDir,
@@ -744,6 +837,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
             warnings: warnings.length ? warnings : undefined,
         });
     }
+    for (const [area, b] of blockedAreas) {
+        results.push({
+            journeyName: `${b.count} ${area} journey(s) not attempted — ${b.pkg} is not installed`,
+            testDir: '',
+            testFiles: [],
+            seedPath: '',
+            status: 'skipped',
+            seedStatus: 'none',
+            warnings: [
+                `every ${area} spec must import ${b.pkg}, so without it the dependency gate ` +
+                    `refuses each generated file and leaves a deferred stub — hours of AI calls for nothing. ` +
+                    `Install it, then re-run: npm i -D ${b.pkg} && npx playwright install`,
+            ],
+        });
+    }
     if (notAttempted > 0 && aiUnavailable) {
         // One line, not one per journey: the operator needs the reason and the
         // count, then to re-run once the CLI is back.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.18.3",
+  "version": "0.18.5",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {