@activate-core/test-forge 1.0.0 → 1.0.1

package/dist/core/docs/source-of-truth.js

@@ -22,7 +22,7 @@
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { isInteractive, promptChoice, promptLine } from '../prompt.js';
-import { SOURCE_PATH } from '../paths.js';
+import { SOURCE_PATH, ensureStateDirIgnored } from '../paths.js';
 /** Read the remembered source choice, or null if none/invalid. */
 export function loadSourceChoice() {
     if (!existsSync(SOURCE_PATH))
@@ -58,6 +58,7 @@ export function saveSourceChoice(choice) {
     if (journeys && Object.keys(journeys).length)
         next.journeys = journeys;
     mkdirSync(dirname(SOURCE_PATH), { recursive: true });
+    ensureStateDirIgnored();
     writeFileSync(SOURCE_PATH, JSON.stringify(next, null, 2), 'utf-8');
 }
 /**

package/dist/core/gates/lint.js

@@ -17,7 +17,7 @@
  */
 import { spawnSync } from 'node:child_process';
 import { resolve } from 'node:path';
-import { existsSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 /** Walk up from a file to the nearest package that has ESLint installed. */
 function eslintFor(file) {
@@ -77,7 +77,9 @@ export function lintGenerated(files) {
             encoding: 'utf-8',
             timeout: 120_000,
         });
-        const after = readProblems(g.bin, cwd, g.files);
+        let after = readProblems(g.bin, cwd, g.files);
+        if (relaxTypingRules(after.typingRules))
+            after = readProblems(g.bin, cwd, g.files);
         checked += g.files.length;
         fixed += Math.max(0, before - after.count);
         remaining.push(...after.lines);
@@ -87,6 +89,32 @@ export function lintGenerated(files) {
 function countProblems(bin, cwd, files) {
     return readProblems(bin, cwd, files).count;
 }
+/**
+ * Strictness rules a house config applies to app code that say nothing about
+ * whether a generated TEST is right: `any` in a fixture, `require` in a
+ * CommonJS helper. Left in place they fail `npm run lint` in CI on files no
+ * one hand-edits, so a generated file gets a header turning off exactly the
+ * ones it trips — every other rule stays on.
+ */
+const TYPING_RULES = new Set([
+    '@typescript-eslint/no-explicit-any',
+    '@typescript-eslint/no-require-imports',
+    '@typescript-eslint/no-var-requires',
+    '@typescript-eslint/ban-ts-comment',
+]);
+/** Add the header to each file; true when any file changed. */
+function relaxTypingRules(byFile) {
+    let changed = false;
+    for (const [file, rules] of byFile) {
+        if (!rules.size)
+            continue;
+        const text = readFileSync(file, 'utf-8');
+        const header = `/* eslint-disable ${[...rules].sort().join(', ')} -- generated by testforge */\n`;
+        writeFileSync(file, header + text, 'utf-8');
+        changed = true;
+    }
+    return changed;
+}
 function readProblems(bin, cwd, files) {
     const res = spawnSync(bin, ['-f', 'json', '--no-error-on-unmatched-pattern', ...files], {
         cwd,
@@ -99,14 +127,20 @@ function readProblems(bin, cwd, files) {
         parsed = JSON.parse(res.stdout || '[]');
     }
     catch {
-        return { count: 0, lines: [] }; // linter unavailable or output unusable
+        return { count: 0, lines: [], typingRules: new Map() }; // linter unavailable or output unusable
     }
     const lines = [];
+    const typingRules = new Map();
     let count = 0;
     for (const file of parsed) {
         for (const m of file.messages) {
             if (m.severity !== 2)
                 continue; // warnings do not fail a build
+            if (m.ruleId && TYPING_RULES.has(m.ruleId)) {
+                const set = typingRules.get(file.filePath) ?? new Set();
+                set.add(m.ruleId);
+                typingRules.set(file.filePath, set);
+            }
             count += 1;
             if (lines.length < 20) {
                 const rel = file.filePath.replace(cwd + '/', '');
@@ -114,5 +148,5 @@ function readProblems(bin, cwd, files) {
             }
         }
     }
-    return { count, lines };
+    return { count, lines, typingRules };
 }

package/dist/core/generate/harness.js

@@ -303,7 +303,7 @@ export function generateHarnessAsync(harnessPath, framework) {
  * dir, treats *.test.js as specs, and runs the testkit's setup() once via
  * globalSetup (mints/populates tokens before any worker starts).
  */
-export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, category = 'web') {
+export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, category = 'web', outputDirRelToPackage = 'test-results') {
     return [
         `// @generated by testforge`,
         `// Minimal Playwright config so testforge's generated specs run.`,
@@ -315,6 +315,9 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
         `  testMatch: /.*\\.(test|spec)\\.[jt]s$/,`,
         `  globalSetup: require.resolve('${globalSetupRelToPackage}'),`,
         `  timeout: 60000,`,
+        `  // Run output (failure reports, traces) stays with testforge's local state,`,
+        `  // which is not committed.`,
+        `  outputDir: '${outputDirRelToPackage}',`,
         `  // Serial, like the Jest config: these suites share ONE test database`,
         `  // and seed/reset it, so parallel workers race on the same rows and the`,
         `  // same run gives different answers twice. It also keeps --headed to a`,

package/dist/core/generate/run-generate.js

@@ -17,8 +17,8 @@
  * generated-tests/ when not configured.
  */
 import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
-import { join, dirname, basename, extname } from 'node:path';
-import { CACHE_PATH } from '../paths.js';
+import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
+import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored } from '../paths.js';
 import { lintGenerated } from '../gates/lint.js';
 import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
 import { writeCoverageManifest } from '../gates/coverage.js';
@@ -27,7 +27,8 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
 import { generateTestCases } from './generator.js';
 import { writeDeferredStub } from './playwright-writer.js';
 import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
-import { verifyFiles } from './verify.js';
+import { verifyFiles, browserPreflight } from './verify.js';
+import { packageRoot } from '../run/run-tests.js';
 import { groundJourney } from '../docs/grounding.js';
 import { generateSeed } from './seed-generator.js';
 import { isSyntaxValid, checkTestGates } from '../gates/validate.js';
@@ -183,6 +184,20 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
     const ext = config.testExtension ?? '.test.ts';
     const seedDir = config.seedOutput ?? 'generated-tests/seed';
     const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
+    // A browser that cannot start fails every verify, and each failure would be
+    // sent for repair. Find out once, before any AI call is spent.
+    if (useAI && verifyOn) {
+        for (const area of Object.keys(config.testOutput ?? {})) {
+            if (resolveFramework(config, area, area) !== 'playwright')
+                continue;
+            const problem = browserPreflight(packageRoot(testDirForType(config, area)));
+            if (problem) {
+                throw new Error(`Cannot verify "${area}" tests: ${problem}.\n` +
+                    'Fix that and re-run, or pass --no-verify to write tests without running them.');
+            }
+        }
+    }
+    ensureStateDirIgnored();
     const cache = loadCache(CACHE_PATH);
     const results = [];
     resetAiSession();
@@ -559,6 +574,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         const lintTargets = [
             ...testFiles.map((t) => (typeof t === 'string' ? t : t.path)),
             ...(seedPath ? [seedPath] : []),
+            // The shared harness and the runner config are generated too, and CI
+            // lints them like any other file.
+            // Only files testforge generated: a project's own config is never edited.
+            ...[harnessPath, join(dirname(testDir), 'playwright.config.js')].filter((f) => !!f && existsSync(f) && readFileSync(f, 'utf-8').includes(GEN_MARKER)),
         ];
         const lint = lintGenerated(lintTargets);
         if (lint.fixed > 0) {
@@ -578,6 +597,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
         if (useAI && verifyOn && targets.length) {
             for (let attempt = 0; attempt <= verifyAttempts; attempt += 1) {
                 const outcome = verifyFiles(config, journey.area, targets);
+                if (outcome.infra) {
+                    // Not the tests' fault, so no repair; and not verified, so the
+                    // journey stays uncached and the next run checks it again.
+                    verified = false;
+                    warnings.push(`verify: not run — ${outcome.reason}`);
+                    break;
+                }
                 if (!outcome.ran) {
                     // Say WHY once. The reason is a property of the project (no env
                     // file, no runner), not of this journey, so repeating it per
@@ -689,25 +715,39 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
  * Config files are written only if absent, so a project's hand-tuned config is
  * never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
  */
-function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
-    const pkgRoot = dirname(areaDir);
-    const rel = areaDir.slice(pkgRoot.length + 1) || '.';
+export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
+    const posix = (from, to) => relative(from, resolve(to)).split(sep).join('/') || '.';
     if (framework === 'playwright') {
         const gsPath = join(harnessDir, 'global-setup.js');
         writeFileSync(gsPath, globalSetupContent(), 'utf-8');
-        const cfg = join(pkgRoot, 'playwright.config.js');
+        const home = configHome(areaDir, 'playwright.config.js');
+        const cfg = join(home, 'playwright.config.js');
         if (!existsSync(cfg)) {
-            const gsRel = gsPath.slice(pkgRoot.length + 1);
-            writeFileSync(cfg, playwrightConfig(`./${rel}`, `./${gsRel}`, area), 'utf-8');
+            writeFileSync(cfg, playwrightConfig(`./${posix(home, areaDir)}`, `./${posix(home, gsPath)}`, area, posix(home, RESULTS_DIR)), 'utf-8');
         }
     }
     else if (framework === 'jest') {
         const setupPath = join(harnessDir, 'jest.setup.js');
         writeFileSync(setupPath, jestSetupContent(), 'utf-8');
-        const cfg = join(pkgRoot, 'jest.config.js');
+        const home = configHome(areaDir, 'jest.config.js');
+        const cfg = join(home, 'jest.config.js');
         if (!existsSync(cfg)) {
-            const setupRel = setupPath.slice(pkgRoot.length + 1);
-            writeFileSync(cfg, jestConfig(rel, setupRel, area), 'utf-8');
+            writeFileSync(cfg, jestConfig(posix(home, areaDir), posix(home, setupPath), area), 'utf-8');
         }
     }
 }
+/**
+ * Where a runner config goes: the package root, where `npx playwright test`
+ * and `npx jest` look for it by themselves — so CI, a teammate or a custom
+ * testCommand finds it with no --config flag. Tests can sit any depth below
+ * (tests/e2e, not just tests/). Only when the project already has its own
+ * config there do we fall back to the test folder's parent, never touching
+ * the project's file.
+ */
+function configHome(areaDir, name) {
+    const root = packageRoot(areaDir);
+    const existing = join(root, name);
+    if (!existsSync(existing) || readFileSync(existing, 'utf-8').includes(GEN_MARKER))
+        return root;
+    return resolve(dirname(areaDir));
+}

package/dist/core/generate/verify.js

@@ -44,6 +44,35 @@ function countsFrom(output) {
         return null;
     return { passed: passed ?? 0, skipped: skipped ?? 0 };
 }
+/**
+ * Failures no test edit can fix. Sending one to the repair prompt burns AI
+ * calls rewriting correct tests against an error they cannot cause (one
+ * night: 877 failures, every one a missing browser).
+ */
+const INFRA_FAILURES = [
+    [
+        /Executable doesn't exist at|browserType\.launch:|Host system is missing dependencies|does not support (chromium|firefox|webkit) on/i,
+        'the test browser cannot start — install the one this Playwright version needs (`npx playwright install chromium`), or pin a Playwright version your OS supports',
+    ],
+    [
+        /net::ERR_CONNECTION_REFUSED|ECONNREFUSED/i,
+        'the app (or a service it needs) is not reachable — start it and check BASE_URL / the DB settings in the env file',
+    ],
+];
+export function infraFailure(output) {
+    return INFRA_FAILURES.find(([re]) => re.test(output))?.[1] ?? null;
+}
+/**
+ * Can the test browser start? Asked once before generation spends anything,
+ * so a missing browser stops the run instead of failing every verify.
+ */
+export function browserPreflight(root) {
+    const res = spawnSync(process.execPath, ['-e', "require('@playwright/test').chromium.launch().then((b) => b.close())"], { cwd: root, encoding: 'utf-8', timeout: 120_000 });
+    if (res.status === 0)
+        return null;
+    const output = `${res.stdout ?? ''}\n${res.stderr ?? ''}`;
+    return infraFailure(output) ?? (output.trim().split('\n').find((l) => l.trim()) || 'the test browser could not start');
+}
 /**
  * Run exactly the given generated files. Never throws: a verify step that
  * breaks generation would be worse than no verify step at all.
@@ -90,6 +119,9 @@ export function verifyFiles(config, area, files) {
     if (/command not found|Cannot find module '@playwright|is not recognized/i.test(output)) {
         return { ran: false, ok: false, output, reason: 'test runner is not installed' };
     }
+    const infra = infraFailure(output);
+    if (infra)
+        return { ran: false, infra: true, ok: false, output, reason: infra };
     if (res.status !== 0)
         return { ran: true, ok: false, output };
     // A suite where every test skipped ALSO exits 0. Treating that as verified

package/dist/core/paths.js

@@ -1,3 +1,4 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 /**
  * Single source of truth for TestForge's per-project state directory and the
@@ -17,3 +18,17 @@ export const SOURCE_PATH = join(STATE_DIR, 'source.json');
 export const COVERAGE_PATH = join(STATE_DIR, 'coverage.json');
 /** Per-journey generation plans, organised per area: <PLANS_DIR>/<area>/<id>.json */
 export const PLANS_DIR = join(STATE_DIR, 'test-plans');
+/** Where Playwright writes its per-run output (failure reports, traces). */
+export const RESULTS_DIR = join(STATE_DIR, 'playwright-results');
+/**
+ * Keep the state dir out of git without touching the project's own
+ * .gitignore: caches, plans and run output are local and regenerated, only
+ * the source-of-truth choice is worth sharing with the team.
+ */
+export function ensureStateDirIgnored() {
+    const file = join(STATE_DIR, '.gitignore');
+    if (existsSync(file))
+        return;
+    mkdirSync(STATE_DIR, { recursive: true });
+    writeFileSync(file, '# testforge local state: regenerated as needed, so not committed.\n*\n!.gitignore\n!source.json\n', 'utf-8');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {