@activate-core/test-forge 0.19.1 → 0.19.3

package/README.md

@@ -330,6 +330,11 @@ The test runner produces its own pass/fail report.
   added to them, never instead of them — so name only what is particular
   to your product. Every domain has something: a shipping carrier, a CMS,
   a ticketing or streaming backend, an ad network.
+- **relatedDirs** — other repos this project depends on, read-only, e.g.
+  `["../backend"]` for a frontend whose API and database live next door.
+  Every AI phase may read them, so the map captures the real data layer and
+  tests can check what the API stores. A headless run cannot read outside
+  the project otherwise.
 
 ## Notes
 

package/dist/core/docs/discover.js

@@ -92,11 +92,18 @@ export function buildDiscoverPrompt(srcDir, docsDir, journey) {
         `                  and the key fields/enum values a test would need to`,
         `                  seed or verify (use the data layer from the map)`,
         ``,
+        `Also add "Test setup" and "Questions for product (not asserted)" last,`,
+        `as described above.`,
+        ``,
         `Rules:`,
         `- Base everything on real evidence in the code and the map. Do not`,
         `  invent routes, fields, or behavior with false confidence.`,
-        `- Where you are unsure, add a line starting with "# TODO:"`,
-        `  describing what the developer must verify.`,
+        `- With no external reference, the code is the source: behaviour the`,
+        `  code clearly and deliberately implements is [spec]. Still add the`,
+        `  full [baseline] scenarios for every field, control and action.`,
+        `- Never write a "# TODO" line: what the tests need goes in "Test`,
+        `  setup", what only product can decide goes in "Questions for product`,
+        `  (not asserted)".`,
         `- For the Data section, never include secrets — refer to env var`,
         `  names only (as in the map).`,
         `- Capture real risks you notice (e.g. a charge that could happen`,

package/dist/core/docs/doc-guidance.js

@@ -5,11 +5,28 @@
  * depth is the ceiling on test quality. This wording is shared by both
  * discover prompts (code-based and reference-based) so a doc is deep and
  * well-sectioned regardless of where it is drafted from.
+ *
+ * A source (a client spec, a ticket) describes a journey in a few lines;
+ * the doc must turn that into EVERY scenario a QA engineer would test. So
+ * each criterion carries a tier saying where it came from, and a failing
+ * test is reported under that tier:
+ *   [spec]     — the source states it. A failure is a defect against the spec.
+ *   [baseline] — universal robustness every well-built app owes, added for
+ *                every field, control and action even when the source is
+ *                silent. Robustness only, never product policy.
+ *   [observed] — the source is silent but the code fixes a concrete rule;
+ *                tested for consistency across layers, not as the spec.
+ * Nothing becomes a "# TODO": what only product can decide is a question,
+ * what the test environment needs is test setup.
  */
 export const DEEP_DOC_GUIDANCE = [
     `Make each doc DEEP and exhaustive — the generated tests can only be as`,
-    `good as this doc, so do NOT stop at a thin outline. For the journey,`,
-    `document, where they apply:`,
+    `good as this doc, so do NOT stop at a thin outline. A source usually`,
+    `describes a journey in a few lines ("the user enters a username and a`,
+    `password"); YOUR job is to expand it into every scenario a senior QA`,
+    `engineer would test: the happy path, every validation of every field,`,
+    `every state of every control, every failure and edge case. For the`,
+    `journey, document, where they apply:`,
     `- every distinct sub-flow and lifecycle STATE it can be in, and the`,
     `  transitions between them (not just the happy path);`,
     `- each endpoint/screen involved, with real inputs, outputs, status`,
@@ -20,14 +37,69 @@ export const DEEP_DOC_GUIDANCE = [
     `  vague mentions;`,
     `- concrete acceptance criteria per case (exactly what must hold).`,
     `Aim for the depth of a spec a new engineer could implement from.`,
-    `Every acceptance criterion you write becomes an ORACLE — tests are`,
-    `generated from it, and a product that disagrees is reported to the team`,
-    `as a defect. So state a criterion ONLY where the source actually fixes`,
-    `the behaviour. Anything you merely believe SHOULD hold goes under a`,
-    `"Suggested (not a requirement — do not assert)" heading, never into the`,
-    `acceptance criteria: an invented requirement becomes a false bug report.`,
+    ``,
+    `TIERS. Every acceptance criterion becomes an ORACLE: a test is generated`,
+    `from it, and a product that disagrees is reported. So start EVERY`,
+    `criterion with its tier, which says where it comes from and how a`,
+    `failure is reported:`,
+    `- "[spec]" — the source states this behaviour. A failure is a defect`,
+    `  against the spec.`,
+    `- "[baseline]" — universal robustness that holds for ANY well-built app.`,
+    `  Add these YOURSELF for every field, control and action the journey`,
+    `  has, even when the source says nothing about them:`,
+    `  * Inputs: empty and whitespace-only values; leading/trailing spaces;`,
+    `    a very long value (5,000 characters); unicode and emoji; an HTML or`,
+    `    script payload. The app never crashes, never answers with a server`,
+    `    error (5xx), never executes the payload (it is shown as text), and`,
+    `    either accepts the value or rejects it with a visible reason.`,
+    `  * No silent failure: every rejected submit shows a visible reason; a`,
+    `    server error, network failure or timeout shows a visible message`,
+    `    and leaves the form usable.`,
+    `  * Double submission: a double click or a repeated Enter sends at most`,
+    `    one request with a side effect (one record, one charge, one`,
+    `    message); the control is disabled or ignored while a request is`,
+    `    pending.`,
+    `  * Keyboard: the form can be filled and submitted with the keyboard`,
+    `    alone; Enter submits.`,
+    `  * Navigation: refresh, Back, and opening a mid-flow URL directly never`,
+    `    crash or leave a blank, broken page.`,
+    `  * Sensitive data: passwords are masked; passwords and tokens never`,
+    `    appear in a URL or query string.`,
+    `  * Access: a page or action that needs a login is not usable logged`,
+    `    out; one user never sees another user's data.`,
+    `  * Choice controls (dropdown, radio, checkbox, list): every option`,
+    `    shown can be selected and submitted.`,
+    `  [baseline] is ROBUSTNESS only. Never present product POLICY you`,
+    `  invented (a maximum length, a password rule, an error wording, a`,
+    `  limit) as [baseline].`,
+    `- "[observed]" — the source is silent, but the code fixes a concrete`,
+    `  rule: a validation pattern, a length limit, allowed values, the`,
+    `  options a list offers, a rate limit, a timeout. State the rule with`,
+    `  its real value, use it to build valid data and boundary cases (just`,
+    `  inside and just outside), and require every layer to enforce it the`,
+    `  same way (the screen accepts exactly what the server accepts). A`,
+    `  failure is an inconsistency, not a spec defect. Never use [observed]`,
+    `  to override or soften a [spec] criterion.`,
+    ``,
+    `TEST DATA. Tests fill data themselves. Never ask for an exact value:`,
+    `valid data comes from the stated or [observed] rules; invalid data`,
+    `breaks one rule at a time; a choice control is filled with an option`,
+    `the screen actually offers at run time, never a hard-coded one.`,
+    ``,
+    `NO TODOs. Never write a "# TODO" line or leave a placeholder. Instead:`,
+    `- What the test environment must provide — accounts and records to`,
+    `  seed, settings or flags that must be on, external services to stub`,
+    `  (SMS/OTP, payments, OAuth, email) and the endpoint or SDK to stub,`,
+    `  browser dialogs to handle — goes in a "## Test setup" section, written`,
+    `  as instructions for the test generator.`,
+    `- What only product can decide, and neither the source nor the code`,
+    `  fixes, goes in a "## Questions for product (not asserted)" section.`,
+    `  Keep it short: a question is a last resort, after [spec], [baseline]`,
+    `  and [observed] have been used.`,
+    ``,
     `Organise the body with numbered "## N. <Section>" headings, ONE`,
     `concern per section, because each section later becomes its own`,
     `focused test file — so the richer and better-separated the sections,`,
-    `the better the test suite.`,
+    `the better the test suite. Put "## Test setup" and "## Questions for`,
+    `product (not asserted)" last, without numbers.`,
 ].join('\n');

package/dist/core/docs/reference.js

@@ -115,10 +115,11 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
         `it describes. The comparison you need is already in "${DIFF_PATH}".`,
         ``,
         `Use the project map at "${MAP_PATH}" and the source under "${srcDir}"`,
-        `ONLY for TECHNICAL detail the reference does not spell out: routes,`,
-        `field names, models/collections, enums, and env var names. NEVER take`,
-        `expected behavior, acceptance criteria, or negative cases from the`,
-        `code — those come from the reference.`,
+        `for TECHNICAL detail the reference does not spell out (routes, field`,
+        `names, models/collections, enums, env var names), for the concrete`,
+        `rules behind [observed] criteria, and for what the Test setup needs.`,
+        `NEVER take a [spec] criterion from the code — [spec] comes only from`,
+        `the reference.`,
         ``,
         `For EACH journey the reference describes, create one markdown file at`,
         `"${docsDir}/<area>/<journey>.md" (area = folder such as api/web/ops,`,
@@ -137,10 +138,9 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
                 ``,
             ]
             : [
-                `Also create a doc for any journey that exists in the map but NOT in`,
-                `the reference, drafted from the map, and add a line`,
-                `"# TODO: not covered by the reference — confirm expected behavior"`,
-                `at its top so it is clearly flagged.`,
+                `Create docs ONLY for journeys the reference describes. A journey`,
+                `that is in the map but not in the reference gets no file; name it`,
+                `in your final summary as "not covered by this reference".`,
                 ``,
             ]),
         `Each file starts with this header (required):`,
@@ -155,25 +155,25 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
         `- security   (add more if relevant: performance, accessibility, ...)`,
         ``,
         `Then whichever of these sections apply: Fields, Actions, Expected,`,
-        `Negative Cases, Race Conditions, Webhooks, Security Notes, Data. The`,
-        `Expected / Negative Cases / acceptance criteria MUST reflect the`,
-        `reference. For the Data section use the map's data layer.`,
+        `Negative Cases, Race Conditions, Webhooks, Security Notes, Data, and`,
+        `last "Test setup" and "Questions for product (not asserted)". For the`,
+        `Data section use the map's data layer.`,
         ``,
         `Rules:`,
-        `- Base behavior on the reference and technical detail on real code;`,
-        `  do not invent either with false confidence.`,
-        `- EVERY requirement you write becomes an ORACLE: tests will be`,
-        `  generated from it, and a product that disagrees will be reported to`,
-        `  the team as a defect. So write a requirement ONLY when the reference`,
-        `  actually states it. Do NOT add hardening, validation, limits, or`,
-        `  guards that the reference does not ask for, however sensible they`,
-        `  seem — an invented requirement becomes a false bug report and costs`,
-        `  the team a release.`,
-        `- When you believe something SHOULD hold but the reference does not say`,
-        `  so, put it under a "Suggested (not a requirement — do not assert)"`,
-        `  heading, never in Expected / Negative Cases / acceptance criteria.`,
-        `- Where the reference is silent on a needed detail, add a "# TODO:"`,
-        `  line describing what the developer must verify.`,
+        `- Tag every criterion [spec], [baseline] or [observed] as described`,
+        `  above. [spec] only where the reference states the behaviour; never`,
+        `  present your own idea of product policy as [spec].`,
+        `- Expand the reference: a field, control or action it mentions gets`,
+        `  its full [baseline] scenarios and, where the code fixes a rule, its`,
+        `  [observed] scenarios, even when the reference names it in passing.`,
+        `- When the code seems to behave differently from a [spec] criterion,`,
+        `  keep the criterion exactly as the reference states it — the test`,
+        `  will report the difference — and add a line "Likely defect: <what`,
+        `  the code does>" under it. Never turn it into a question.`,
+        `- Base technical detail on real code; do not invent it with false`,
+        `  confidence.`,
+        `- No "# TODO" lines and no "Suggested" section: use "Test setup" and`,
+        `  "Questions for product (not asserted)" as described above.`,
         `- Never include secrets — refer to env var names only.`,
         `- Create the area subfolders as needed. Write NOTHING at the root of`,
         `  "${docsDir}" and no file that is not a journey doc — a shared note`,

package/dist/core/generate/ai-generator.js

@@ -130,8 +130,11 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `    than the prose. Only after checking those is it genuinely unstated.`,
         `    An assertion you cannot trace to a stated requirement is a false`,
         `    positive waiting to happen. Anything under a "Suggested (not a`,
-        `    requirement…)" heading is explicitly NOT an oracle — never assert`,
-        `    it; make it ${pending} if you cover it at all.`,
+        `    requirement…)" or "Questions for product (not asserted)" heading is`,
+        `    explicitly NOT an oracle — never assert it; make it ${pending} if you`,
+        `    cover it at all. Criteria tagged [spec], [baseline] or [observed]`,
+        `    ARE stated oracles. The "Test setup" section says how to establish`,
+        `    preconditions (seeds, settings, stubs, dialogs) — follow it.`,
         `(2) INFRASTRUCTURE IS NEVER A PRODUCT DEFECT. If a precondition cannot`,
         `    be established — a dependency is unreachable, a required env var or`,
         `    credential is absent, seed/fixture data is missing, a second`,

package/dist/core/generate/suite-planner.js

@@ -40,6 +40,9 @@ function buildPlanPrompt(journey, map) {
         `  refunds) may need many. Do not pad, do not over-split, do not force`,
         `  a fixed number.`,
         `- Each file covers ONE concern and must be independently meaningful.`,
+        `- "## Test setup" is what every file needs to run, not a behaviour, and`,
+        `  "## Questions for product (not asserted)" is not tested: plan no file`,
+        `  for either.`,
         ``,
         `Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
         `  [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,

package/dist/core/project/config.js

@@ -26,7 +26,7 @@
  * into the output. See withTextOnlyTools / withEditTools.
  */
 import { readFileSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 const CONFIG_PATH = 'testforge.config.json';
 const DEFAULT_CONFIG = {
     aiCommand: 'claude -p',
@@ -146,25 +146,42 @@ export function withTextOnlyTools(cmd) {
  * assumptions: without source access the model leaves "endpoint shape
  * unconfirmed" TODOs; with it, those become real assertions.
  */
-export function withReadOnlyTools(cmd) {
+export function withReadOnlyTools(cmd, relatedDirs = loadConfig().relatedDirs ?? []) {
     if (!isClaude(cmd.program))
         return cmd;
     return {
         program: cmd.program,
         // Space-separated allowlist; Write/Edit/Bash deliberately excluded.
-        args: [...cmd.args, '--allowedTools', 'Read Grep Glob LS'],
+        args: [...cmd.args, '--allowedTools', 'Read Grep Glob LS', ...addDirArgs(relatedDirs)],
     };
 }
+/**
+ * Read-only tools a headless exploring phase needs pre-approved. The model
+ * reaches for shell `grep`/`ls` as often as for the Grep tool; under
+ * `acceptEdits` alone those are DENIED (no one is there to approve), and
+ * it gives up on what it was mapping. Only commands that read are listed.
+ */
+const EXPLORE_TOOLS = ['Read', 'Grep', 'Glob', 'LS', 'Bash(grep:*)', 'Bash(rg:*)', 'Bash(ls:*)', 'Bash(wc:*)'];
+/** `--add-dir` per related repo: reads outside the project are otherwise denied. */
+function addDirArgs(relatedDirs) {
+    return relatedDirs.flatMap((dir) => ['--add-dir', resolve(dir)]);
+}
 /**
  * Args for a phase where the model must read/write files (init, map,
- * discover). Mirrors withTextOnlyTools but grants edit permission.
+ * discover). Mirrors withTextOnlyTools but grants edit permission, plus the
+ * read-only exploring tools and the related repos.
  */
-export function withEditTools(cmd) {
+export function withEditTools(cmd, relatedDirs = loadConfig().relatedDirs ?? []) {
     if (!isClaude(cmd.program))
         return cmd;
     return {
         program: cmd.program,
-        args: [...cmd.args, '--permission-mode', 'acceptEdits'],
+        args: [
+            ...cmd.args,
+            '--permission-mode', 'acceptEdits',
+            '--allowedTools', EXPLORE_TOOLS.join(' '),
+            ...addDirArgs(relatedDirs),
+        ],
     };
 }
 /**
@@ -179,13 +196,11 @@ export function withEditTools(cmd) {
  * `claude_ai_Notion`, and a CLI-added server conventionally named
  * `notion`). Non-Claude CLIs get the command unchanged.
  */
-export function withReferenceTools(cmd) {
+export function withReferenceTools(cmd, relatedDirs = loadConfig().relatedDirs ?? []) {
     if (!isClaude(cmd.program))
         return cmd;
     const allow = [
-        'Read',
-        'Glob',
-        'Grep',
+        ...EXPLORE_TOOLS,
         'WebFetch',
         'WebSearch',
         'mcp__notion__*',
@@ -193,7 +208,12 @@ export function withReferenceTools(cmd) {
     ].join(' ');
     return {
         program: cmd.program,
-        args: [...cmd.args, '--permission-mode', 'acceptEdits', '--allowedTools', allow],
+        args: [
+            ...cmd.args,
+            '--permission-mode', 'acceptEdits',
+            '--allowedTools', allow,
+            ...addDirArgs(relatedDirs),
+        ],
     };
 }
 /**

package/dist/core/project/map.js

@@ -20,12 +20,23 @@
 import { execFileSync } from 'node:child_process';
 import { loadConfig, splitCommand, withEditTools } from './config.js';
 /** Build the prompt that asks the AI to produce the project map. */
-export function buildMapPrompt(srcDir) {
+export function buildMapPrompt(srcDir, relatedDirs = []) {
+    const related = relatedDirs.length
+        ? [
+            `This project depends on related repos, which you may READ but must`,
+            `not change: ${relatedDirs.map((d) => `"${d}"`).join(', ')}. Journeys`,
+            `come only from "${srcDir}". When the data layer (database, models,`,
+            `enums) or the API this project calls lives in a related repo, map it`,
+            `from there, and name the repo it came from.`,
+            ``,
+        ]
+        : [];
     return [
         `You are a senior engineer mapping a codebase so a QA tool can`,
         `plan tests and seed test data. Read the source code under`,
         `"${srcDir}" and produce a SHORT, high-level map of the project.`,
         ``,
+        ...related,
         `The map has two parts: journeys and the data layer.`,
         ``,
         `PART 1 — Journeys.`,
@@ -79,8 +90,9 @@ export function buildMapPrompt(srcDir) {
 }
 /** Run the map builder via the configured AI command. */
 export function runMap(srcDir) {
-    const prompt = buildMapPrompt(srcDir);
-    const { program, args } = withEditTools(splitCommand(loadConfig().aiCommand));
+    const config = loadConfig();
+    const prompt = buildMapPrompt(srcDir, config.relatedDirs ?? []);
+    const { program, args } = withEditTools(splitCommand(config.aiCommand), config.relatedDirs ?? []);
     const output = execFileSync(program, args, {
         input: prompt,
         encoding: 'utf-8',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.19.1",
+  "version": "0.19.3",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {
@@ -14,7 +14,8 @@
   ],
   "scripts": {
     "build": "rm -rf dist && tsc",
-    "dev": "tsc --watch"
+    "dev": "tsc --watch",
+    "test": "tsx --test test/*.test.ts"
   },
   "license": "MIT",
   "publishConfig": {