@activate-core/test-forge 0.5.0 → 0.6.0

package/dist/core/ai-generator.js

@@ -12,7 +12,7 @@
  */
 import { execFileSync, spawn } from 'node:child_process';
 import { readFileSync, existsSync } from 'node:fs';
-import { loadConfig, splitCommand, withReadOnlyTools } from './config.js';
+import { loadConfig, splitCommand, withReadOnlyTools, resolveFramework, } from './config.js';
 const MAP_PATH = 'testforge.md';
 /**
  * Spawn a command, feed `input` on stdin, resolve with its stdout.
@@ -39,17 +39,13 @@ function runWithStdin(program, args, input) {
     });
 }
 /** Build the prompt we send to the AI for one journey. */
-export function buildPrompt(journey, map, testPath, focus) {
+export function buildPrompt(journey, map, testPath, focus, harnessPath) {
     const config = loadConfig();
     // Opinionated: testforge only generates Jest (api/backend) or Playwright
-    // (web/UI) tests. We look the framework up by AREA, but clamp anything
-    // that is not jest/supertest/playwright (e.g. a project's cypress setup)
-    // to the sensible default for the journey's Type — so no other framework
-    // ever leaks into generated output.
-    const configured = (config.framework?.[journey.area] ?? '').toLowerCase();
-    const isApi = configured === 'jest' ||
-        configured === 'supertest' ||
-        (configured !== 'playwright' && journey.type === 'api');
+    // (web/UI) tests — any other framework (e.g. a project's Cypress setup)
+    // is clamped to the right one for the journey. Single source of truth in
+    // resolveFramework, shared with the output-layout decision.
+    const isApi = resolveFramework(config, journey.area, journey.type) === 'jest';
     const framework = isApi
         ? 'Jest (with supertest for HTTP) for an API/backend test'
         : 'Playwright for a web/UI test';
@@ -136,13 +132,27 @@ export function buildPrompt(journey, map, testPath, focus) {
                 ``,
             ]
             : []),
-        `This test relies on seeded test data (a separate seed file provisions`,
-        `the database before tests run). Assume the seed has created the`,
-        `records described in the journey's Data section, using stable test`,
-        `ids prefixed with "qa-"/"QA-TEST". Reference those known test ids`,
-        `rather than creating fresh data inside the test where possible. After`,
-        `a test mutates data, clean up what it created so reruns stay valid.`,
-        ``,
+        ...(harnessPath
+            ? [
+                `SHARED HARNESS — a testkit already exists at:`,
+                `  ${harnessPath}`,
+                `Import it (compute the relative path from THIS file) and use it`,
+                `for ALL setup: connect/disconnect, reset(), seed(), TEST_IDS, and`,
+                `(Playwright) apiContext()/login(). Read that file to confirm its`,
+                `exact exports before using them. Do NOT create, import, or invent`,
+                `any other seed/fixture/helper/auth module — use ONLY this testkit,`,
+                `so every file in the suite shares one consistent harness.`,
+                ``,
+            ]
+            : [
+                `This test relies on seeded test data (a separate seed file`,
+                `provisions the database before tests run). Assume the seed created`,
+                `the records described in the journey's Data section, using stable`,
+                `test ids prefixed with "qa-"/"QA-TEST". Reference those known ids`,
+                `rather than creating fresh data inside the test where possible.`,
+                `After a test mutates data, clean up what it created.`,
+                ``,
+            ]),
         `Rules for your output:`,
         `- Write the test in ${language}.`,
         `- Output ONLY the raw test code. Your entire response must be valid`,
@@ -191,9 +201,9 @@ export function generateWithAI(journey, testPath, focus) {
  * journey's suite can be generated concurrently (each spawns its own
  * `claude` process). Same prompt, same read-only exploration.
  */
-export async function generateWithAIAsync(journey, testPath, focus) {
+export async function generateWithAIAsync(journey, testPath, focus, harnessPath) {
     const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
-    const prompt = buildPrompt(journey, map, testPath, focus);
+    const prompt = buildPrompt(journey, map, testPath, focus, harnessPath);
     const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
     const stdout = await runWithStdin(program, args, prompt);
     return cleanOutput(stdout);

package/dist/core/config.js

@@ -99,6 +99,19 @@ export function splitCommand(command) {
     const args = tokens.slice(1);
     return { program, args };
 }
+/**
+ * Resolve the test framework for a journey — clamped to the only two
+ * testforge supports. A web/UI journey is always Playwright (even if the
+ * project uses Cypress); an endpoint journey is Jest. This single source
+ * of truth drives BOTH the generation prompt and the output layout.
+ */
+export function resolveFramework(config, area, type) {
+    const configured = (config.framework?.[area] ?? '').toLowerCase();
+    const isApi = configured === 'jest' ||
+        configured === 'supertest' ||
+        (configured !== 'playwright' && type === 'api');
+    return isApi ? 'jest' : 'playwright';
+}
 /** Is this command Claude's CLI (so Claude-specific flags apply)? */
 function isClaude(program) {
     const base = program.split(/[\\/]/).pop() ?? program;

package/dist/core/harness.js

@@ -0,0 +1,135 @@
+/**
+ * testforge — shared test harness scaffold.
+ *
+ * Without a shared harness, every generated test file invents its OWN
+ * seed/auth/db helper path — and they never agree, so nothing runs. This
+ * module generates ONE "testkit" per area that all of that area's tests
+ * import from a single known path, mirroring the hand-authored helpers/
+ * layout (client + seed builders + reset + auth) that a real suite uses.
+ *
+ * The testkit is generated by reading the project's REAL data layer and
+ * auth (read-only tools), so it connects and seeds against the actual
+ * schema — not a guess.
+ */
+import { execFileSync, spawn } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { loadConfig, splitCommand, withReadOnlyTools } from './config.js';
+import { cleanOutput } from './ai-generator.js';
+const MAP_PATH = 'testforge.md';
+/**
+ * The fixed contract every testkit exposes and every generated test
+ * relies on. Both the harness generator and the per-file test generator
+ * are given this exact shape, so imports and usage line up.
+ */
+export const HARNESS_CONTRACT = [
+    `The testkit module MUST export exactly this stable API (CommonJS`,
+    `module.exports), implemented against the project's real data layer:`,
+    `  - connect(): Promise<void>      open the DB connection (env DB var`,
+    `                                  from the map, e.g. process.env.DB_URL)`,
+    `  - disconnect(): Promise<void>   close it`,
+    `  - reset(): Promise<void>        remove all test data (only records`,
+    `                                  with the "qa-"/"QA-TEST" prefix)`,
+    `  - seed(spec?): Promise<object>  idempotently upsert the test records`,
+    `                                  the tests need (using the real models`,
+    `                                  /collections + enums), returning the`,
+    `                                  created ids/handles`,
+    `  - TEST_IDS: object              the stable well-known test ids/handles`,
+    `Plus, for Playwright suites, ALSO export:`,
+    `  - apiContext(request): a helper that returns an authed API request`,
+    `    context (Bearer token from an env var — NEVER hardcode a secret)`,
+    `  - login(page): Promise<void>    drive/seed an authenticated UI session`,
+    `Use only env var NAMES for secrets/connection — never literal values.`,
+].join('\n');
+/** Where an area's shared testkit lives, and how tests import it. */
+export function harnessInfo(areaDir, framework, ext) {
+    // Jest tests are flat inside <areaDir> (e.g. .../test/service), so the
+    // helpers sit one level up (.../test/helpers). Playwright tests live in
+    // per-journey folders under <areaDir> (e.g. .../tests), so helpers are
+    // <areaDir>/helpers.
+    const base = framework === 'jest' ? dirnameOf(areaDir) : areaDir;
+    const kitExt = ext.replace(/\.test|\.spec/, '') || '.js';
+    return { dir: `${base}/helpers`, path: `${base}/helpers/testkit${kitExt}` };
+}
+/** Tiny dirname without importing path just for one call. */
+function dirnameOf(p) {
+    const i = p.replace(/\/+$/, '').lastIndexOf('/');
+    return i === -1 ? '.' : p.slice(0, i);
+}
+/** Build the prompt that generates one area's testkit. */
+function buildHarnessPrompt(harnessPath, framework) {
+    const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
+    return [
+        `You are a senior SDET writing the SHARED test harness ("testkit")`,
+        `that every test in this area will import. It must be correct against`,
+        `the project's REAL data layer and auth.`,
+        ``,
+        `You are inside the repository with read-only tools (Read, Grep, Glob,`,
+        `LS). READ the real DB config, the models/schemas, and the auth`,
+        `middleware, then implement the testkit against them.`,
+        ``,
+        `This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area.`,
+        HARNESS_CONTRACT,
+        ``,
+        `The file will be saved at EXACTLY: ${harnessPath}`,
+        `Compute any require() paths to app source RELATIVE to that location.`,
+        ``,
+        `Output ONLY the testkit code (CommonJS). No prose, no markdown fences.`,
+        `The very first characters must be code. Do NOT write any file.`,
+        ``,
+        `--- PROJECT MAP (data layer) ---`,
+        map,
+    ].join('\n');
+}
+/** Generate one area's testkit content (sync). */
+export function generateHarness(harnessPath, framework) {
+    const prompt = buildHarnessPrompt(harnessPath, framework);
+    const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
+    const output = execFileSync(program, args, {
+        input: prompt,
+        encoding: 'utf-8',
+        maxBuffer: 10 * 1024 * 1024,
+    });
+    return cleanOutput(output);
+}
+/** Async twin, so a harness can generate alongside other work. */
+export function generateHarnessAsync(harnessPath, framework) {
+    const prompt = buildHarnessPrompt(harnessPath, framework);
+    const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
+    return new Promise((resolve, reject) => {
+        const child = spawn(program, args, { stdio: ['pipe', 'pipe', 'pipe'] });
+        let out = '';
+        let err = '';
+        child.stdout.setEncoding('utf-8');
+        child.stdout.on('data', (d) => (out += d));
+        child.stderr.on('data', (d) => (err += d));
+        child.on('error', reject);
+        child.on('close', (code) => code === 0
+            ? resolve(cleanOutput(out))
+            : reject(new Error(`harness gen exited ${code}: ${err.slice(0, 300)}`)));
+        child.stdin.end(prompt);
+    });
+}
+/**
+ * A minimal Playwright config so a webapp/ops area that currently uses
+ * Cypress can actually run the generated specs. Points at the area's test
+ * dir and treats *.test.js as specs.
+ */
+export function playwrightConfig(testDirRelToPackage) {
+    return [
+        `// @generated by testforge`,
+        `// Minimal Playwright config so testforge's generated specs run.`,
+        `// Install once:  npm i -D @playwright/test && npx playwright install`,
+        `const { defineConfig } = require('@playwright/test');`,
+        ``,
+        `module.exports = defineConfig({`,
+        `  testDir: '${testDirRelToPackage}',`,
+        `  testMatch: /.*\\.(test|spec)\\.[jt]s$/,`,
+        `  timeout: 60000,`,
+        `  use: {`,
+        `    baseURL: process.env.BASE_URL || 'http://localhost:3000',`,
+        `    trace: 'on-first-retry',`,
+        `  },`,
+        `});`,
+        ``,
+    ].join('\n');
+}

package/dist/core/run-generate.js

@@ -25,9 +25,10 @@ import { generateWithAI, generateWithAIAsync } from './ai-generator.js';
 import { generateSeed } from './seed-generator.js';
 import { planSuite } from './suite-planner.js';
 import { isSyntaxValid } from './validate.js';
+import { harnessInfo, generateHarnessAsync, playwrightConfig, } from './harness.js';
 import { hashContent, loadCache, saveCache } from './cache.js';
 import { findMarkdownFiles } from './find-files.js';
-import { loadConfig, testDirForType } from './config.js';
+import { loadConfig, testDirForType, resolveFramework } from './config.js';
 const CACHE_PATH = '.testforge/cache.json';
 /** How many times to re-ask the AI when output won't parse. */
 const MAX_GEN_ATTEMPTS = 2;
@@ -160,6 +161,32 @@ export async function runGenerate(docsDir, useAI = true, force = false, only) {
     const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
     const cache = loadCache(CACHE_PATH);
     const results = [];
+    // Shared harness per area — generated once, then every test in that
+    // area imports it (so files stop inventing their own helper paths).
+    const harnessByArea = new Map();
+    async function ensureHarness(areaDir, area, framework) {
+        if (harnessByArea.has(area))
+            return harnessByArea.get(area) ?? undefined;
+        const info = harnessInfo(areaDir, framework, ext);
+        const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
+        if (kit === null) {
+            harnessByArea.set(area, null);
+            return undefined;
+        }
+        mkdirSync(dirname(info.path), { recursive: true });
+        writeFileSync(info.path, kit, 'utf-8');
+        // Playwright areas need a config so the specs can actually run.
+        if (framework === 'playwright') {
+            const pkgRoot = dirname(areaDir);
+            const cfg = join(pkgRoot, 'playwright.config.js');
+            if (!existsSync(cfg)) {
+                const rel = areaDir.slice(pkgRoot.length + 1) || '.';
+                writeFileSync(cfg, playwrightConfig(`./${rel}`), 'utf-8');
+            }
+        }
+        harnessByArea.set(area, info.path);
+        return info.path;
+    }
     for (const file of files) {
         const markdown = readFileSync(join(docsDir, file), 'utf-8');
         const hash = hashContent(markdown);
@@ -172,9 +199,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only) {
         // --journey <x>: generate just the one matching journey.
         if (only && !matchesJourney(file, journey.id, only))
             continue;
-        // Deep mode: each journey becomes a FOLDER of focused test files.
+        // Layout depends on the framework's test runner:
+        //  - Playwright (web/ops) recurses, so each journey gets its own
+        //    FOLDER of focused specs (testDir/<journey>/<slug>).
+        //  - Jest is often pointed at a flat glob (e.g. the-seam's
+        //    `**/service/*.test.js`, which does NOT recurse), so api/lambda
+        //    files stay FLAT in the test dir, journey-prefixed to stay unique
+        //    (testDir/<journey>-<slug>). Otherwise they'd silently not run.
         const areaDir = testDirForType(config, journey.area);
-        const testDir = join(areaDir, journey.id);
+        const framework = resolveFramework(config, journey.area, journey.type);
+        const flat = framework === 'jest';
+        const testDir = flat ? areaDir : join(areaDir, journey.id);
         const seedPath = join(seedDir, `${journey.id}${seedExt}`);
         // Skip if unchanged.
         if (!force && cache[file] === hash) {
@@ -192,6 +227,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only) {
         const collision = collisionWarning(areaDir, journey.id, ext);
         if (collision)
             warnings.push(collision);
+        // Ensure this area's shared harness exists (once), so every file
+        // imports the same testkit instead of inventing its own.
+        const harnessPath = useAI
+            ? await ensureHarness(areaDir, journey.area, framework)
+            : undefined;
+        if (useAI && !harnessPath) {
+            warnings.push('shared harness generation failed — tests may import missing helpers');
+        }
         // 1. Plan the suite (deep mode), then generate one focused file per
         //    concern. In basic mode there is no AI planner — one file only.
         const planned = useAI
@@ -201,14 +244,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only) {
         // call is independent, so a 15-file journey finishes in ~4 waves
         // instead of 15 serial calls.
         const generated = await pool(planned, GEN_CONCURRENCY, async (pf) => {
-            const filePath = join(testDir, `${pf.slug}${ext}`);
+            // Flat (jest): prefix the file with the journey so many journeys can
+            // share one dir without clashing. Folder (playwright): the journey
+            // folder already namespaces it, so the slug alone is enough.
+            const base = flat ? `${journey.id}-${pf.slug}` : pf.slug;
+            const filePath = join(testDir, `${base}${ext}`);
             let content;
             let status;
             if (useAI) {
-                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, {
-                    title: pf.title,
-                    covers: pf.covers,
-                }), ext);
+                const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath), ext);
                 if (ai !== null) {
                     content = ai;
                     status = 'ai';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@activate-core/test-forge",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
   "type": "module",
   "bin": {
@@ -24,4 +24,4 @@
     "tsx": "^4.22.4",
     "typescript": "^5.4.0"
   }
-}
+}