@activate-core/test-forge 0.13.0 → 0.14.0

package/dist/core/ai-generator.js

@@ -83,21 +83,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         : 'TypeScript';
     return [
         `You are a senior QA engineer. Generate a complete ${framework}`,
-        `for the user journey described below.`,
+        `for the user journey described below. (The specific file to write is`,
+        `stated at the very end, after the journey — read all the context first.)`,
         ``,
-        ...(focus
-            ? [
-                `SCOPE — this is ONE file in a larger suite for this journey.`,
-                `Cover ONLY this concern in this file:`,
-                `  ${focus.title}`,
-                `  ${focus.covers}`,
-                `Do NOT test the journey's other concerns here — separate files`,
-                `cover those. Go DEEP on this one concern (happy path + its own`,
-                `edge/negative/security cases) rather than broad-but-shallow.`,
-                `Use this exact describe() title: "${focus.title}".`,
-                ``,
-            ]
-            : []),
         `Think like an experienced tester whose goal is to break the`,
         `application. Cover, where relevant: happy path, missing/empty`,
         `required fields, invalid formats and boundary values (very long,`,
@@ -117,6 +105,47 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `guard is missing and produce a failing test that falsely brands the`,
         `product as vulnerable.`,
         ``,
+        `PRIME DIRECTIVE — A FAILING TEST MUST MEAN A REAL PRODUCT DEFECT.`,
+        `Teams hold releases on what these tests report. A test that fails for`,
+        `any reason other than the product being wrong is worse than no test at`,
+        `all: it burns the team's trust, and the next real defect gets waved`,
+        `through as "probably the tool again". So a test may FAIL only when the`,
+        `product provably contradicts an expectation the journey document`,
+        `states EXPLICITLY. Apply these rules without exception:`,
+        `(1) EXPLICIT-ORACLE ONLY. Before writing an assertion, find the exact`,
+        `    line in the journey document that fixes the expected outcome, and`,
+        `    cite it in a short comment (e.g. "// oracle: §4 total = fee +`,
+        `    shipping"). If the expectation is merely implied, inferred, or`,
+        `    "good practice" — including a rule that sounds obviously right —`,
+        `    it is NOT an oracle: write it ${pending} with a "// TODO:" note.`,
+        `    An assertion you cannot trace to a stated requirement is a false`,
+        `    positive waiting to happen. Anything under a "Suggested (not a`,
+        `    requirement…)" heading is explicitly NOT an oracle — never assert`,
+        `    it; make it ${pending} if you cover it at all.`,
+        `(2) INFRASTRUCTURE IS NEVER A PRODUCT DEFECT. If a precondition cannot`,
+        `    be established — a dependency is unreachable, a required env var or`,
+        `    credential is absent, seed/fixture data is missing, a second`,
+        `    identity is unavailable, an external provider is down — SKIP the`,
+        `    test with a one-line reason naming exactly what was missing. Never`,
+        `    let a missing precondition reach an assertion and report as a bug.`,
+        `    Guard such tests at the top: check the precondition, skip if unmet.`,
+        `(3) NO FLAKY TESTS. Every test must pass or fail for the same reason on`,
+        `    every run, in any order. Do not depend on execution order, on data`,
+        `    another test creates or mutates, on wall-clock timing, or on`,
+        `    ambient totals that other tests change (assert on the rows YOU`,
+        `    seeded, by id — never on a global count). Set up what you assert`,
+        `    on, and assert only on that. Wait for a specific condition, never a`,
+        `    fixed sleep.`,
+        `(4) NEVER ASSERT AN UNFIXED VALUE. Do not assert exact user-facing copy,`,
+        `    incidental ordering, generated ids, timestamps, or counts the`,
+        `    document does not fix. Assert the behaviour the document specifies,`,
+        `    at the loosest form that still proves it.`,
+        `(5) A TIMEOUT IS NOT A BUG REPORT. If an action legitimately waits on`,
+        `    something external, verify availability first and skip when absent.`,
+        `    A test that times out tells the team nothing about the product.`,
+        `When rules conflict, prefer reporting LESS: a pending test costs a line`,
+        `in the report; a false failure costs the team a release.`,
+        ``,
         `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
         `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
         `BEFORE writing tests, look up the actual code for this journey — use`,
@@ -241,17 +270,6 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
                 `module (aliased as apiRequest above).`,
             ].join('\n'),
         ``,
-        ...(testPath
-            ? [
-                `This test file will be saved at EXACTLY this path:`,
-                `  ${testPath}`,
-                `Compute every require()/import path (to app source, helpers, the`,
-                `seed file, etc.) RELATIVE TO THAT EXACT location — count the`,
-                `directory depth from that path, do not assume a conventional`,
-                `depth. Getting the relative depth wrong breaks the whole file.`,
-                ``,
-            ]
-            : []),
         ...(transport
             ? [transport, `Do NOT import any library that is not already a dependency of this project.`, ``]
             : []),
@@ -298,6 +316,36 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `Data section. Use ALL of it. Respect any "# TODO:" notes.`,
         ``,
         journey.rawMarkdown,
+        // ---- PER-FILE TASK (kept LAST so the big context above stays a stable,
+        // cache-reusable prefix across every file in this journey, and so the
+        // concrete task is the most recent thing the model reads) ----
+        ``,
+        ...(focus
+            ? [
+                `--- THIS FILE (the one to write now) ---`,
+                `This is ONE file in a larger suite for this journey.`,
+                `Cover ONLY this concern here:`,
+                `  ${focus.title}`,
+                `  ${focus.covers}`,
+                `Do NOT test the journey's other concerns here — separate files`,
+                `cover those. Go DEEP on this one concern (happy path + its own`,
+                `edge/negative/security cases) rather than broad-but-shallow.`,
+                `Use this exact describe() title: "${focus.title}".`,
+            ]
+            : []),
+        ...(testPath
+            ? [
+                ``,
+                `Save this file at EXACTLY: ${testPath}`,
+                `Compute every require()/import path (app source, helpers, the seed`,
+                `file, etc.) RELATIVE TO THAT EXACT location — count the directory`,
+                `depth from that path; do not assume a conventional depth. Getting`,
+                `the relative depth wrong breaks the whole file.`,
+            ]
+            : []),
+        ``,
+        `Now output ONLY the raw ${isJs ? 'JavaScript' : 'TypeScript'} test code`,
+        `for THIS file — the first character must be the first line of code.`,
     ].join('\n');
 }
 /**

package/dist/core/doc-guidance.js

@@ -20,6 +20,12 @@ export const DEEP_DOC_GUIDANCE = [
     `  vague mentions;`,
     `- concrete acceptance criteria per case (exactly what must hold).`,
     `Aim for the depth of a spec a new engineer could implement from.`,
+    `Every acceptance criterion you write becomes an ORACLE — tests are`,
+    `generated from it, and a product that disagrees is reported to the team`,
+    `as a defect. So state a criterion ONLY where the source actually fixes`,
+    `the behaviour. Anything you merely believe SHOULD hold goes under a`,
+    `"Suggested (not a requirement — do not assert)" heading, never into the`,
+    `acceptance criteria: an invented requirement becomes a false bug report.`,
     `Organise the body with numbered "## N. <Section>" headings, ONE`,
     `concern per section, because each section later becomes its own`,
     `focused test file — so the richer and better-separated the sections,`,

package/dist/core/harness.js

@@ -285,10 +285,10 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
         `  testMatch: /.*\\.(test|spec)\\.[jt]s$/,`,
         `  globalSetup: require.resolve('${globalSetupRelToPackage}'),`,
         `  timeout: 60000,`,
-        `  // Console line output + Allure results (see ALLURE.md to view the report).`,
-        `  // Last reporter auto-publishes the run to the TestForge dashboard (needs`,
-        `  // TESTFORGE_KEY in env + project/dashboardUrl in testforge.config.json).`,
-        `  reporter: [['line'], ['allure-playwright'], [require.resolve('@activate-core/test-forge/reporters/playwright-reporter.cjs'), { category: ${JSON.stringify(category)} }]],`,
+        `  // Console line output. The last reporter auto-publishes the run to the`,
+        `  // TestForge dashboard (needs TESTFORGE_KEY in env + project/dashboardUrl`,
+        `  // in testforge.config.json).`,
+        `  reporter: [['line'], [require.resolve('@activate-core/test-forge/reporters/playwright-reporter.cjs'), { category: ${JSON.stringify(category)} }]],`,
         `  use: {`,
         `    baseURL: process.env.BASE_URL || 'http://localhost:3000',`,
         `    trace: 'on-first-retry',`,
@@ -300,8 +300,8 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
 }
 /**
  * Jest config for a backend (jest) area — self-contained so `npm test` / a bare
- * `npx jest` runs with no shell setup, and emits Allure results.
- *   - testEnvironment 'allure-jest/node': captures each test into allure-results/
+ * `npx jest` runs with no shell setup.
+ *   - testEnvironment 'node': the default Node test environment
  *   - setupFiles: loads .env.testing + NODE_ENV + the reset opt-in before any
  *     test file imports the app
  *   - serial (maxWorkers 1): suites share one test DB and reset between tests
@@ -309,9 +309,9 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
 export function jestConfig(testDirRelToPackage, setupRelToPackage, category = 'api') {
     return [
         `// @generated by testforge`,
-        `// Install once:  npm i -D jest allure-jest allure-commandline  (see ALLURE.md)`,
+        `// Install once:  npm i -D jest`,
         `module.exports = {`,
-        `  testEnvironment: 'allure-jest/node',`,
+        `  testEnvironment: 'node',`,
         `  setupFiles: ['<rootDir>/${setupRelToPackage}'],`,
         `  testMatch: ['<rootDir>/${testDirRelToPackage}/**/*.test.js'],`,
         `  // Serial: the suites share ONE test DB and reset it between tests.`,
@@ -350,49 +350,6 @@ export function jestSetupContent() {
         ``,
     ].join('\n');
 }
-/** Allure failure categories — buckets that show up in the report's "Categories"
- *  tab. Copied into allure-results/ so the report groups failures meaningfully. */
-export function allureCategories() {
-    return JSON.stringify([
-        { name: 'Product defect', matchedStatuses: ['failed'], messageRegex: '.*expect.*|.*assert.*' },
-        { name: 'Test / harness issue', matchedStatuses: ['broken'] },
-        { name: 'Flaky', matchedStatuses: ['failed', 'broken'], flaky: true },
-    ], null, 2);
-}
-/** ALLURE.md — how to install the reporter deps and view the report. */
-export function allureDoc() {
-    return [
-        `# Allure reporting (generated by testforge)`,
-        ``,
-        `testforge wires the suites to emit **Allure** results. To view the report:`,
-        ``,
-        `## 1. Install once (per package)`,
-        '```bash',
-        `# web (Playwright):  npm i -D allure-playwright allure-commandline`,
-        `# api  (Jest):       npm i -D allure-jest allure-commandline`,
-        '```',
-        ``,
-        `## 2. Run the tests`,
-        `Running the suite writes raw results into \`allure-results/\`.`,
-        ``,
-        `## 3. Generate + open the report`,
-        '```bash',
-        `npx allure generate allure-results --clean -o allure-report`,
-        `npx allure open allure-report`,
-        '```',
-        `Or add scripts to package.json:`,
-        '```json',
-        `"report": "allure generate allure-results --clean -o allure-report && allure open allure-report"`,
-        '```',
-        ``,
-        `## Categories`,
-        `\`allure-categories.json\` groups failures (Product defect / Test-harness / Flaky).`,
-        `Copy it to \`allure-results/categories.json\` before generating to activate it.`,
-        ``,
-        `> \`allure-results/\`, \`allure-report/\` are build output — add them to .gitignore.`,
-        ``,
-    ].join('\n');
-}
 /**
  * Playwright globalSetup file: runs the testkit's one-call setup() once
  * before the suite (connect + seed + mint/populate tokens). Sits next to

package/dist/core/reference.js

@@ -147,6 +147,16 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
         `Rules:`,
         `- Base behavior on the reference and technical detail on real code;`,
         `  do not invent either with false confidence.`,
+        `- EVERY requirement you write becomes an ORACLE: tests will be`,
+        `  generated from it, and a product that disagrees will be reported to`,
+        `  the team as a defect. So write a requirement ONLY when the reference`,
+        `  actually states it. Do NOT add hardening, validation, limits, or`,
+        `  guards that the reference does not ask for, however sensible they`,
+        `  seem — an invented requirement becomes a false bug report and costs`,
+        `  the team a release.`,
+        `- When you believe something SHOULD hold but the reference does not say`,
+        `  so, put it under a "Suggested (not a requirement — do not assert)"`,
+        `  heading, never in Expected / Negative Cases / acceptance criteria.`,
         `- Where the reference is silent on a needed detail, add a "# TODO:"`,
         `  line describing what the developer must verify.`,
         `- Never include secrets — refer to env var names only.`,

package/dist/core/report.js

@@ -145,7 +145,7 @@ for (const f of ['jest.config.js','jest.config.cjs','jest.config.json']) {
 if (!Object.keys(base).length && fs.existsSync(path.join(__dirname,'package.json'))) base = require('./package.json').jest || {};
 module.exports = {
   ...base,
-  testEnvironment: 'node', // avoid allure-jest side effects during a count-only run
+  testEnvironment: 'node', // plain node env for a count-only run
   setupFilesAfterEnv: [...(base.setupFilesAfterEnv || []), path.resolve(__dirname, '.tf-collect-patch.cjs')],
 };
 `;

package/dist/core/run-generate.js

@@ -27,7 +27,7 @@ import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
 import { isSyntaxValid, checkTestGates, analyzeFile } from './validate.js';
 import { installedModules, transportHint, projectSelectors } from './detect.js';
-import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, allureCategories, allureDoc, } from './harness.js';
+import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
 import { hashContent, loadCache, saveCache } from './cache.js';
 import { findMarkdownFiles } from './find-files.js';
 import { loadConfig, testDirForType, resolveFramework } from './config.js';
@@ -356,7 +356,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         writeFileSync(info.path, kit, 'utf-8');
         writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
         // Config + globalSetup (playwright) / config + setup (jest) so the specs run
-        // AND get their tokens minted once, plus Allure reporting scaffolding.
+        // AND get their tokens minted once, plus the runner config scaffolding.
         writeHarnessConfigs(framework, areaDir, info.dir, area);
         harnessByArea.set(area, info.path);
         return info.path;
@@ -376,16 +376,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         // --journey <x>: generate just the one matching journey.
         if (only && !matchesJourney(file, journey.id, only))
             continue;
-        // Layout depends on the framework's test runner:
-        //  - Playwright (web/ops) recurses, so each journey gets its own
-        //    FOLDER of focused specs (testDir/<journey>/<slug>).
-        //  - Jest is often pointed at a flat glob (e.g. the-seam's
-        //    `**/service/*.test.js`, which does NOT recurse), so api/lambda
-        //    files stay FLAT in the test dir, journey-prefixed to stay unique
-        //    (testDir/<journey>-<slug>). Otherwise they'd silently not run.
+        // EVERY area gets its own per-journey FOLDER of focused files
+        // (testDir/<journey>/<slug>), for both Playwright and Jest. The folder
+        // namespaces the files, so the on-disk name is just the clean <slug>
+        // (no journey prefix) — which is also what the dashboard shows as the
+        // suite name. testforge's generated jest.config uses a recursive
+        // `tests/**/*.test.js` glob, so foldered jest files are still discovered.
         const areaDir = testDirForType(config, journey.area);
         const framework = resolveFramework(config, journey.area, journey.type);
-        const flat = framework === 'jest';
+        const flat = false; // foldered layout for all runners (clean, categorised)
         const testDir = flat ? areaDir : join(areaDir, journey.id);
         const seedPath = join(seedDir, `${journey.id}${seedExt}`);
         // Dependency-aware guardrails (F1): what this area may import, and how
@@ -631,23 +630,15 @@ function writeCoverageManifest(results) {
     }
 }
 /**
- * Write the per-area runner config + Allure reporting scaffolding.
- *  - playwright: global-setup.js + playwright.config.js (allure-playwright reporter)
- *  - jest:       helpers/jest.setup.js + jest.config.js (allure-jest env)
- *  - both:       ALLURE.md + allure-categories.json at the package root
+ * Write the per-area runner config.
+ *  - playwright: global-setup.js + playwright.config.js
+ *  - jest:       helpers/jest.setup.js + jest.config.js
  * Config files are written only if absent, so a project's hand-tuned config is
  * never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
  */
 function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
     const pkgRoot = dirname(areaDir);
     const rel = areaDir.slice(pkgRoot.length + 1) || '.';
-    // Allure docs + categories (both frameworks); never overwrite.
-    const allureMd = join(pkgRoot, 'ALLURE.md');
-    if (!existsSync(allureMd))
-        writeFileSync(allureMd, allureDoc(), 'utf-8');
-    const cats = join(pkgRoot, 'allure-categories.json');
-    if (!existsSync(cats))
-        writeFileSync(cats, allureCategories(), 'utf-8');
     if (framework === 'playwright') {
         const gsPath = join(harnessDir, 'global-setup.js');
         writeFileSync(gsPath, globalSetupContent(), 'utf-8');

package/dist/core/suite-planner.js

@@ -44,7 +44,13 @@ function buildPlanPrompt(journey, map) {
         `Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
         `  [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
         `     "covers": "<one or two sentences: exactly what this file tests>" }]`,
-        `slug: lowercase, words joined by "-", no path and no file extension.`,
+        `slug: lowercase kebab-case, no path, no extension — and SHORT: name the`,
+        `  one concern in 2–4 words (4 max), the way a human names a file. Do NOT`,
+        `  restate the journey name and do NOT pile on qualifiers.`,
+        `  Good: "amount-validation", "3ds-challenge", "zero-amount", "rate-limit".`,
+        `  Bad:  "charge-amount-sanity-negative-nonnumeric-oversized",`,
+        `        "otp-session-otp-send-rate-limit".`,
+        `title: human-readable and specific, e.g. "Payment — Amount validation".`,
         `The very first character of your response must be "[".`,
         ``,
         `--- PROJECT MAP (data layer) ---`,
@@ -71,14 +77,16 @@ function extractJsonArray(text) {
         return [];
     }
 }
-/** Normalise a raw slug into a safe kebab-case file stem. */
+/** Normalise a raw slug into a safe, SHORT kebab-case file stem. */
 function safeSlug(raw) {
-    return String(raw ?? '')
+    const kebab = String(raw ?? '')
         .trim()
         .toLowerCase()
         .replace(/\.(spec|test)\.[jt]sx?$/, '') // drop any extension the AI added
         .replace(/[^a-z0-9]+/g, '-')
         .replace(/^-|-$/g, '');
+    // Hard cap: keep names short & human — at most 4 words, whatever the AI returns.
+    return kebab.split('-').filter(Boolean).slice(0, 4).join('-');
 }
 /**
  * Plan the suite for one journey: returns the list of focused files to

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {

package/reporters/_shared.cjs

@@ -42,6 +42,50 @@ function loadConfig(root) {
 
 const strip = (s) => (s || '').replace(/\x1b\[[0-9;]*m/g, '');
 
+// jest's raw matcher line (`expect(received).toBeGreaterThanOrEqual(expected)`) reads
+// backwards to a human. Translate the matcher into a plain, direction-correct sentence
+// so an engineer sees intent, not jest internals.
+const MATCHER = {
+  toBe: 'equal', toEqual: 'equal', toStrictEqual: 'equal',
+  toBeGreaterThanOrEqual: 'be at least', toBeGreaterThan: 'be greater than',
+  toBeLessThanOrEqual: 'be at most', toBeLessThan: 'be less than',
+  toContain: 'contain', toContainEqual: 'contain', toMatch: 'match',
+  toHaveLength: 'have length', toHaveProperty: 'have property', toBeCloseTo: 'be about',
+};
+function humanize(body) {
+  const m = body.match(/\.(\w+)\((?:expected|n|value|substring|sample|element)?\)/) || body.match(/expect\([^)]*\)\.(\w+)/);
+  const phrase = m && MATCHER[m[1]];
+  if (!phrase) return null;
+  const exp = (body.match(/Expected:?\s*(.+)/) || [])[1];
+  const rec = (body.match(/Received:?\s*(.+)/) || [])[1];
+  const ce = (exp || '').replace(/^[><=!~\s]+/, '').trim();
+  const cr = (rec || '').trim();
+  if (!ce || !cr) return null;
+  return `The check expected the result to ${phrase} ${ce}, but it was ${cr}.`;
+}
+
+// A readable failure detail for an engineer: a plain-language lead (what the check
+// wanted vs what it got) + the Expected/Received values + any custom message. Drops
+// jest's cryptic matcher line and the noisy stack frames / internal paths. Capped so
+// it stays a summary, not a full dump.
+function cleanError(raw) {
+  const s = strip(raw || '');
+  if (!s) return '';
+  const kept = s.split('\n').filter((l) => {
+    const t = l.trim();
+    if (!t) return true; // keep blank lines for readability (collapsed below)
+    if (/^at\s/.test(t)) return false;                       // stack frame
+    if (/node_modules|node:internal|\/dist\//.test(t)) return false; // library/internal noise
+    return true;
+  });
+  const body = kept.join('\n').replace(/\n{3,}/g, '\n\n').trim();
+  const lead = humanize(body);
+  if (!lead) return body.slice(0, 700);
+  // The plain lead replaces jest's `expect(received)…` matcher line — drop it, keep the rest.
+  const rest = body.split('\n').filter((l) => !/^(Error:\s*)?expect\(/.test(l.trim())).join('\n').replace(/\n{3,}/g, '\n\n').trim();
+  return (lead + (rest ? '\n\n' + rest : '')).slice(0, 700);
+}
+
 function sh(cmd, cwd) {
   try { return (execSync(cmd, { cwd, stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim()) || undefined; }
   catch { return undefined; }
@@ -73,11 +117,27 @@ function resolveEnv() {
 
 function sig(msg) {
   const m = strip(msg || '').replace(/\n/g, ' ');
-  if (/403|whoami|invalid_grant|refresh token/i.test(m)) return 'IM auth (403)';
+  // Order matters, and each pattern has to be precise. A failing assertion
+  // carries plenty of incidental words — Playwright prints "Timeout: 5000ms"
+  // inside every expect() failure, and a status-code assertion legitimately
+  // mentions 403 — so loose patterns used to swallow most failures into
+  // "Timeout" / "IM auth" and made the whole triage useless.
+
+  // 1) A REAL timeout: the runner itself gave up, or the socket did.
+  if (/test timeout of \d+ ?ms exceeded|exceeded timeout of \d+ ?ms|timed out after \d+|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNABORTED/i.test(m))
+    return 'Timeout';
+
+  // 2) An assertion that ran and disagreed — the common, useful case.
+  if (/expect\(.*\)|toBe\b|toEqual\b|toHaveCount\b|Received:|Expected:|expected the result to/i.test(m))
+    return 'Assertion mismatch';
+
+  // 3) Auth against the identity service — needs auth wording, not a bare 403.
+  if (/whoami|invalid_grant|refresh token|Unauthorized|Forbidden|invalid token|jwt (expired|malformed)/i.test(m))
+    return 'IM auth (403)';
+
   if (/\$gt|\$exists|injection/i.test(m)) return 'NoSQL injection accepted';
-  if (/timeout|timed out|ETIMEDOUT/i.test(m)) return 'Timeout';
   if (/\b500\b|Internal Server/i.test(m)) return 'Server 500';
-  if (/expect|toBe|toEqual|Received|Expected/i.test(m)) return 'Assertion mismatch';
+  if (/timeout|timed out/i.test(m)) return 'Timeout';   // looser fallback, last
   return 'Uncategorised';
 }
 
@@ -106,4 +166,4 @@ async function publish(category, suites, durationSec) {
   }
 }
 
-module.exports = { strip, sig, publish, gitInfo, resolveEnv };
+module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv };

package/reporters/jest-reporter.cjs

@@ -1,7 +1,7 @@
 // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
 // Add to jest.config.js:
 //   reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
-const { strip, sig, publish } = require('./_shared.cjs');
+const { cleanError, sig, publish } = require('./_shared.cjs');
 
 module.exports = class TestForgeJestReporter {
   constructor(_globalConfig, options) {
@@ -15,12 +15,14 @@ module.exports = class TestForgeJestReporter {
       for (const a of (f.testResults || f.assertionResults || [])) {
         const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
         const raw = (a.failureMessages || [])[0];
-        const err = strip(raw || '').split('\n').find((l) => l && l.trim()) || '';
         su.tests.push({
           name: a.fullName || a.title,
           status,
-          error: status === 'failed' ? err.slice(0, 200) : undefined,
+          error: status === 'failed' ? cleanError(raw) : undefined,
           sig: status === 'failed' ? sig(raw) : undefined,
+          note: status === 'skipped'
+            ? (a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
+            : undefined,
         });
       }
     }

package/reporters/playwright-reporter.cjs

@@ -1,7 +1,7 @@
 // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
 // Add to playwright.config.js reporter array:
 //   ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
-const { strip, sig, publish } = require('./_shared.cjs');
+const { cleanError, sig, publish } = require('./_shared.cjs');
 
 module.exports = class TestForgePlaywrightReporter {
   constructor(options) {
@@ -15,12 +15,13 @@ module.exports = class TestForgePlaywrightReporter {
     const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
     const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
     const raw = result.error && result.error.message;
-    const err = strip(raw || '').split('\n').find((l) => l && l.trim()) || '';
+    const skipAnno = (test.annotations || []).find((x) => x.type === 'skip' || x.type === 'fixme');
     su.tests.push({
       name: test.title,
       status,
-      error: status === 'failed' ? err.slice(0, 200) : undefined,
+      error: status === 'failed' ? cleanError(raw) : undefined,
       sig: status === 'failed' ? sig(raw) : undefined,
+      note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
     });
   }
   async onEnd(result) {