@activate-core/test-forge 0.19.3 → 1.0.0

package/dist/core/docs/discover.js

@@ -23,6 +23,7 @@ import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
 import { promptConfirm } from '../prompt.js';
 import { loadSourceChoice, resolveDocsSource, saveJourneyReferences, saveSourceChoice, } from './source-of-truth.js';
 import { analyzeReference, draftDocsFromReference, DIFF_PATH, } from './reference.js';
+import { reviewDocs } from './review.js';
 /** Scope line: one journey, or all of them. */
 function scopeLines(journey) {
     return journey
@@ -180,7 +181,7 @@ export async function runDiscover(srcDir, docsDir, opts = {}, journey) {
             if (!ok) {
                 console.log('Basing docs on testforge.md instead.\n');
                 saveSourceChoice({ mode: 'map' });
-                return runCodeDiscover(srcDir, docsDir, journey);
+                return discoverFromCode(srcDir, docsDir, journey);
             }
         }
         // Is this the product's specification, or one feature's? A ticket's PDF
@@ -246,6 +247,10 @@ export async function runDiscover(srcDir, docsDir, opts = {}, journey) {
         for (const path of restored) {
             writeFileSync(join(docsDir, path), before.get(path), 'utf-8');
         }
+        // Second, focused pass per written doc: the drafting pass covers every
+        // journey at once and drops the small scenarios.
+        const written = [...added, ...rewritten.filter((p) => !restored.includes(p))];
+        await reviewDocs(docsDir, written, srcDir, source.reference);
         if (wholeProject) {
             saveSourceChoice({ mode: 'reference', reference: source.reference });
             if (added.length)
@@ -270,5 +275,15 @@ export async function runDiscover(srcDir, docsDir, opts = {}, journey) {
         return out;
     }
     saveSourceChoice({ mode: 'map' });
-    return runCodeDiscover(srcDir, docsDir, journey);
+    return discoverFromCode(srcDir, docsDir, journey);
+}
+/** Draft from the code, then review every doc the draft wrote or changed. */
+async function discoverFromCode(srcDir, docsDir, journey) {
+    const before = snapshotDocs(docsDir);
+    const out = runCodeDiscover(srcDir, docsDir, journey);
+    const changed = [...snapshotDocs(docsDir)]
+        .filter(([path, text]) => before.get(path) !== text && !notAJourneyDoc(path, text))
+        .map(([path]) => path);
+    await reviewDocs(docsDir, changed, srcDir);
+    return out;
 }

package/dist/core/docs/doc-guidance.js

@@ -81,6 +81,36 @@ export const DEEP_DOC_GUIDANCE = [
     `  failure is an inconsistency, not a spec defect. Never use [observed]`,
     `  to override or soften a [spec] criterion.`,
     ``,
+    `METHOD. Work through these steps for every journey, in order:`,
+    `1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
+    `   interactive element and where it appears: each input, button, link,`,
+    `   checkbox, code box, dropdown, pop-up, menu item, and the page itself.`,
+    `   Secondary controls count as much as the main form: Resend, Back,`,
+    `   show/hide, close, apply, sign-out, provider buttons. Apply every`,
+    `   [baseline] item that fits to EACH row, not just to the main form.`,
+    `2. COMBINATIONS. When the source states a rule across alternatives and`,
+    `   conditions ("for login and sign-up, also on a slow connection and on`,
+    `   a retry"), write one criterion per combination (login × slow, login ×`,
+    `   retry, sign-up × slow, sign-up × retry), never just one of them.`,
+    `3. CODE BRANCHES. Read the code the journey runs — screens, components,`,
+    `   client API calls, and the server handlers they reach (related repos`,
+    `   included) — and walk EVERY validation rule, error response and`,
+    `   conditional branch: account states (unverified, banned, deleted,`,
+    `   other roles), limits and expiry, empty and failure states, refresh`,
+    `   and deep links, what is stored in the browser and when it is`,
+    `   cleared, input handling (paste, clearing after an error). Each branch`,
+    `   becomes an [observed] criterion, or a Question when only product can`,
+    `   say what it should do. A skipped branch is the commonest gap.`,
+    `4. QUOTE [spec]. End every [spec] criterion with the source sentence it`,
+    `   rests on, quoted: (source: "..."). A criterion you cannot quote is`,
+    `   NOT [spec] — make it [baseline] or [observed], or a Question.`,
+    `5. SHARED RULES. A rule shared by several journeys (password, redirects,`,
+    `   attack payloads, accepted formats) uses the same cases and values in`,
+    `   every doc.`,
+    `6. COVERAGE. After the numbered sections, add "## Coverage": one row per`,
+    `   control from step 1, naming the section that covers each [baseline]`,
+    `   item that fits it. A blank cell is a gap — fill it before finishing.`,
+    ``,
     `TEST DATA. Tests fill data themselves. Never ask for an exact value:`,
     `valid data comes from the stated or [observed] rules; invalid data`,
     `breaks one rule at a time; a choice control is filled with an option`,
@@ -100,6 +130,7 @@ export const DEEP_DOC_GUIDANCE = [
     `Organise the body with numbered "## N. <Section>" headings, ONE`,
     `concern per section, because each section later becomes its own`,
     `focused test file — so the richer and better-separated the sections,`,
-    `the better the test suite. Put "## Test setup" and "## Questions for`,
-    `product (not asserted)" last, without numbers.`,
+    `the better the test suite. "## Controls" comes first; "## Coverage",`,
+    `"## Test setup" and "## Questions for product (not asserted)" come`,
+    `last. None of these four is numbered.`,
 ].join('\n');

package/dist/core/docs/review.js

@@ -0,0 +1,81 @@
+/**
+ * testforge — journey doc gap review.
+ *
+ * One drafting pass writes every journey at once, and it reliably runs out
+ * of attention before the small things: a Resend button with no failure
+ * cases, a code box nobody pasted into, one combination of the four the
+ * source asked for, a server branch never read. So each freshly written doc
+ * gets a second, focused pass of its own: re-run the method from
+ * DEEP_DOC_GUIDANCE against that one journey, fill what is missing, and fix
+ * tiers that claim more than their source says. The reviewer edits the file
+ * in place and never deletes a sound criterion.
+ */
+import { join } from 'node:path';
+import { loadConfig, splitCommand, withEditTools, withReferenceTools } from '../project/config.js';
+import { runWithStdin } from '../generate/ai-generator.js';
+import { pool } from '../generate/run-generate.js';
+import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
+/** Build the prompt that reviews and completes ONE journey doc. */
+export function buildReviewPrompt(docPath, srcDir, reference) {
+    const source = reference
+        ? `The source of truth is the reference: ${reference}. Read it fully; [spec]` +
+            ` criteria come only from it, and the code under "${srcDir}" (and any` +
+            ` related repos) supplies technical detail and [observed] rules.`
+        : `The source of truth is the code under "${srcDir}" (and any related repos).`;
+    return [
+        `You are a senior QA engineer reviewing ONE journey doc that a first`,
+        `pass drafted: "${docPath}". Your job is to make it COMPLETE, so that`,
+        `no scenario a careful QA engineer would test is missing.`,
+        ``,
+        source,
+        ``,
+        `The doc must follow this guidance:`,
+        DEEP_DOC_GUIDANCE,
+        ``,
+        `Review it step by step against the METHOD above and fix what you find:`,
+        `- Controls: is every interactive element of this journey in the table,`,
+        `  secondary controls included? Add the missing ones.`,
+        `- Baseline: does every control have every [baseline] item that fits`,
+        `  it? Add the missing criteria to the right section.`,
+        `- Combinations: is every combination the source states written out?`,
+        `- Code branches: read the code this journey runs, client and server.`,
+        `  Is every validation, error response and conditional branch an`,
+        `  [observed] criterion or a Question? Add the missing ones.`,
+        `- Tiers: does every [spec] criterion quote a real source sentence?`,
+        `  Re-tag any that does not. Does any criterion contradict the code`,
+        `  with nothing to test (a rule the screen has no way to break)? Turn`,
+        `  it into a "Likely defect" note or a Question.`,
+        `- Shared rules: read the other docs in the same folder and make shared`,
+        `  cases (password, redirects, payloads, formats) identical.`,
+        `- Coverage: rebuild the "## Coverage" table; no blank cell may remain.`,
+        ``,
+        `Rules: edit "${docPath}" in place and touch no other file. Keep every`,
+        `sound criterion. Keep the header, the section numbering style and the`,
+        `last sections (Coverage, Test setup, Questions for product). Never`,
+        `write "# TODO". When done, print ONE line: what you added or fixed,`,
+        `with counts.`,
+    ].join('\n');
+}
+/**
+ * Review each doc (paths relative to docsDir) in its own AI pass, a few at
+ * a time. A failed review leaves that doc as drafted and is reported, never
+ * fatal: the draft is still a valid doc.
+ */
+export async function reviewDocs(docsDir, docs, srcDir, reference) {
+    if (!docs.length)
+        return;
+    const config = loadConfig();
+    const base = splitCommand(config.aiCommand);
+    const { program, args } = reference ? withReferenceTools(base) : withEditTools(base);
+    console.log(`\nReviewing ${docs.length} journey doc(s) for missing scenarios...`);
+    await pool(docs, Math.max(1, config.aiConcurrency ?? 4), async (doc) => {
+        const path = join(docsDir, doc);
+        try {
+            const out = await runWithStdin(program, args, buildReviewPrompt(path, srcDir, reference), config.aiTimeoutMs);
+            console.log(`  ${doc}: ${out.trim().split('\n').pop() ?? 'reviewed'}`);
+        }
+        catch (err) {
+            console.log(`  ${doc}: review failed, kept as drafted (${err.message.split('\n')[0]})`);
+        }
+    });
+}

package/dist/core/generate/ai-generator.js

@@ -18,7 +18,7 @@ const MAP_PATH = 'testforge.md';
  * Spawn a command, feed `input` on stdin, resolve with its stdout.
  * Async (unlike execFileSync) so many calls can run concurrently.
  */
-function runWithStdin(program, args, input, timeoutMs = 300_000) {
+export function runWithStdin(program, args, input, timeoutMs = 300_000) {
     return new Promise((resolve, reject) => {
         const child = spawn(program, args, {
             stdio: ['pipe', 'pipe', 'pipe'],

package/dist/core/generate/run-generate.js

@@ -45,7 +45,7 @@ const GEN_CONCURRENCY = 4;
  * the results. Keeps a heavy fan-out (15 AI calls) from spawning 15
  * processes at once while still finishing in a few waves, not serially.
  */
-async function pool(items, limit, fn) {
+export async function pool(items, limit, fn) {
     const results = new Array(items.length);
     let next = 0;
     async function worker() {

package/dist/core/generate/suite-planner.js

@@ -40,9 +40,9 @@ function buildPlanPrompt(journey, map) {
         `  refunds) may need many. Do not pad, do not over-split, do not force`,
         `  a fixed number.`,
         `- Each file covers ONE concern and must be independently meaningful.`,
-        `- "## Test setup" is what every file needs to run, not a behaviour, and`,
-        `  "## Questions for product (not asserted)" is not tested: plan no file`,
-        `  for either.`,
+        `- "## Controls" and "## Coverage" are indexes, "## Test setup" is what`,
+        `  every file needs to run, and "## Questions for product (not`,
+        `  asserted)" is not tested: plan no file for any of them.`,
         ``,
         `Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
         `  [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.19.3",
+  "version": "1.0.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {