@activate-core/test-forge 0.13.1 → 0.14.0

package/dist/core/ai-generator.js

@@ -105,6 +105,47 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
         `guard is missing and produce a failing test that falsely brands the`,
         `product as vulnerable.`,
         ``,
+        `PRIME DIRECTIVE — A FAILING TEST MUST MEAN A REAL PRODUCT DEFECT.`,
+        `Teams hold releases on what these tests report. A test that fails for`,
+        `any reason other than the product being wrong is worse than no test at`,
+        `all: it burns the team's trust, and the next real defect gets waved`,
+        `through as "probably the tool again". So a test may FAIL only when the`,
+        `product provably contradicts an expectation the journey document`,
+        `states EXPLICITLY. Apply these rules without exception:`,
+        `(1) EXPLICIT-ORACLE ONLY. Before writing an assertion, find the exact`,
+        `    line in the journey document that fixes the expected outcome, and`,
+        `    cite it in a short comment (e.g. "// oracle: §4 total = fee +`,
+        `    shipping"). If the expectation is merely implied, inferred, or`,
+        `    "good practice" — including a rule that sounds obviously right —`,
+        `    it is NOT an oracle: write it ${pending} with a "// TODO:" note.`,
+        `    An assertion you cannot trace to a stated requirement is a false`,
+        `    positive waiting to happen. Anything under a "Suggested (not a`,
+        `    requirement…)" heading is explicitly NOT an oracle — never assert`,
+        `    it; make it ${pending} if you cover it at all.`,
+        `(2) INFRASTRUCTURE IS NEVER A PRODUCT DEFECT. If a precondition cannot`,
+        `    be established — a dependency is unreachable, a required env var or`,
+        `    credential is absent, seed/fixture data is missing, a second`,
+        `    identity is unavailable, an external provider is down — SKIP the`,
+        `    test with a one-line reason naming exactly what was missing. Never`,
+        `    let a missing precondition reach an assertion and report as a bug.`,
+        `    Guard such tests at the top: check the precondition, skip if unmet.`,
+        `(3) NO FLAKY TESTS. Every test must pass or fail for the same reason on`,
+        `    every run, in any order. Do not depend on execution order, on data`,
+        `    another test creates or mutates, on wall-clock timing, or on`,
+        `    ambient totals that other tests change (assert on the rows YOU`,
+        `    seeded, by id — never on a global count). Set up what you assert`,
+        `    on, and assert only on that. Wait for a specific condition, never a`,
+        `    fixed sleep.`,
+        `(4) NEVER ASSERT AN UNFIXED VALUE. Do not assert exact user-facing copy,`,
+        `    incidental ordering, generated ids, timestamps, or counts the`,
+        `    document does not fix. Assert the behaviour the document specifies,`,
+        `    at the loosest form that still proves it.`,
+        `(5) A TIMEOUT IS NOT A BUG REPORT. If an action legitimately waits on`,
+        `    something external, verify availability first and skip when absent.`,
+        `    A test that times out tells the team nothing about the product.`,
+        `When rules conflict, prefer reporting LESS: a pending test costs a line`,
+        `in the report; a false failure costs the team a release.`,
+        ``,
         `GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
         `project's repository and have read-only tools (Read, Grep, Glob, LS).`,
         `BEFORE writing tests, look up the actual code for this journey — use`,

package/dist/core/doc-guidance.js

@@ -20,6 +20,12 @@ export const DEEP_DOC_GUIDANCE = [
     `  vague mentions;`,
     `- concrete acceptance criteria per case (exactly what must hold).`,
     `Aim for the depth of a spec a new engineer could implement from.`,
+    `Every acceptance criterion you write becomes an ORACLE — tests are`,
+    `generated from it, and a product that disagrees is reported to the team`,
+    `as a defect. So state a criterion ONLY where the source actually fixes`,
+    `the behaviour. Anything you merely believe SHOULD hold goes under a`,
+    `"Suggested (not a requirement — do not assert)" heading, never into the`,
+    `acceptance criteria: an invented requirement becomes a false bug report.`,
     `Organise the body with numbered "## N. <Section>" headings, ONE`,
     `concern per section, because each section later becomes its own`,
     `focused test file — so the richer and better-separated the sections,`,

package/dist/core/reference.js

@@ -147,6 +147,16 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
         `Rules:`,
         `- Base behavior on the reference and technical detail on real code;`,
         `  do not invent either with false confidence.`,
+        `- EVERY requirement you write becomes an ORACLE: tests will be`,
+        `  generated from it, and a product that disagrees will be reported to`,
+        `  the team as a defect. So write a requirement ONLY when the reference`,
+        `  actually states it. Do NOT add hardening, validation, limits, or`,
+        `  guards that the reference does not ask for, however sensible they`,
+        `  seem — an invented requirement becomes a false bug report and costs`,
+        `  the team a release.`,
+        `- When you believe something SHOULD hold but the reference does not say`,
+        `  so, put it under a "Suggested (not a requirement — do not assert)"`,
+        `  heading, never in Expected / Negative Cases / acceptance criteria.`,
         `- Where the reference is silent on a needed detail, add a "# TODO:"`,
         `  line describing what the developer must verify.`,
         `- Never include secrets — refer to env var names only.`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.13.1",
+  "version": "0.14.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {

package/reporters/_shared.cjs

@@ -117,11 +117,27 @@ function resolveEnv() {
 
 function sig(msg) {
   const m = strip(msg || '').replace(/\n/g, ' ');
-  if (/403|whoami|invalid_grant|refresh token/i.test(m)) return 'IM auth (403)';
+  // Order matters, and each pattern has to be precise. A failing assertion
+  // carries plenty of incidental words — Playwright prints "Timeout: 5000ms"
+  // inside every expect() failure, and a status-code assertion legitimately
+  // mentions 403 — so loose patterns used to swallow most failures into
+  // "Timeout" / "IM auth" and made the whole triage useless.
+
+  // 1) A REAL timeout: the runner itself gave up, or the socket did.
+  if (/test timeout of \d+ ?ms exceeded|exceeded timeout of \d+ ?ms|timed out after \d+|ETIMEDOUT|ESOCKETTIMEDOUT|ECONNABORTED/i.test(m))
+    return 'Timeout';
+
+  // 2) An assertion that ran and disagreed — the common, useful case.
+  if (/expect\(.*\)|toBe\b|toEqual\b|toHaveCount\b|Received:|Expected:|expected the result to/i.test(m))
+    return 'Assertion mismatch';
+
+  // 3) Auth against the identity service — needs auth wording, not a bare 403.
+  if (/whoami|invalid_grant|refresh token|Unauthorized|Forbidden|invalid token|jwt (expired|malformed)/i.test(m))
+    return 'IM auth (403)';
+
   if (/\$gt|\$exists|injection/i.test(m)) return 'NoSQL injection accepted';
-  if (/timeout|timed out|ETIMEDOUT/i.test(m)) return 'Timeout';
   if (/\b500\b|Internal Server/i.test(m)) return 'Server 500';
-  if (/expect|toBe|toEqual|Received|Expected/i.test(m)) return 'Assertion mismatch';
+  if (/timeout|timed out/i.test(m)) return 'Timeout';   // looser fallback, last
   return 'Uncategorised';
 }