@activate-core/test-forge 2.0.0 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,19 +26,22 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
26
26
  import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
- import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, } from './ai-generator.js';
29
+ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
+ import { completeFile, fileIsComplete } from './complete-file.js';
31
+ import { harnessSteps } from '../gates/test-quality.js';
30
32
  import { normalisePending } from './pending.js';
31
33
  import { packageRoot } from '../run/run-tests.js';
34
+ import { pendingReason, withPendingStamp } from '../run/readiness.js';
32
35
  import { groundJourney } from '../docs/grounding.js';
33
36
  import { generateSeed } from './seed-generator.js';
34
37
  import { dataAccess, writeRequiredData } from './required-data.js';
35
38
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
36
39
  import { testedTags } from './suite-planner.js';
37
- import { factsSlice, helperApi, journeySlice, readRules, saveMemory, testedElsewhere } from './journey-memory.js';
40
+ import { factsSlice, helperApi, journeySlice, readRules, recordWritten, saveMemory, testedElsewhere } from './journey-memory.js';
38
41
  import { loadSourceChoice } from '../docs/source-of-truth.js';
39
42
  import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
40
43
  import { updateJourney } from './update-journey.js';
41
- import { sourceText } from '../docs/excerpts.js';
44
+ import { sourceRoot, sourceText } from '../docs/excerpts.js';
42
45
  import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
43
46
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
44
47
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
@@ -424,19 +427,36 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
424
427
  // What the app can show: its source and the doc. A fresh test's names and
425
428
  // labels are checked against it before the test is run.
426
429
  let knownTexts;
430
+ const appTexts = () => framework === 'playwright'
431
+ ? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
432
+ : undefined;
427
433
  const genGate = (content, filePath) => testGate(content) ??
428
434
  checkTestGates(content, ext, {
429
435
  framework, forbidPlaceholders: true, filePath,
430
436
  maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
431
- knownTexts: framework === 'playwright'
432
- ? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
433
- : undefined,
437
+ knownTexts: appTexts(),
434
438
  });
435
439
  // Skip if unchanged — but NOT if the journey still has deferred stubs
436
440
  // to retry (a resume pass fills those in without touching real files).
441
+ // A suite whose tests leave rules out or cannot fail is not done either:
442
+ // it goes on, and only those tests are written again.
443
+ const suiteComplete = () => {
444
+ const plan = readPlan(journey);
445
+ if (!plan?.some((f) => f.rules?.length))
446
+ return true;
447
+ const kit = harnessInfo(areaDir, framework, ext).path;
448
+ const steps = existsSync(kit) ? harnessSteps(readFileSync(kit, 'utf-8')) : [];
449
+ return plan.every((f) => {
450
+ const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
451
+ if (!f.rules?.length || !existsSync(p))
452
+ return true;
453
+ return fileIsComplete(readFileSync(p, 'utf-8'), f.rules, steps, appTexts());
454
+ });
455
+ };
437
456
  if (!force &&
438
457
  cache[file] === hash &&
439
- !hasDeferredStubs(testDir, ext)) {
458
+ !hasDeferredStubs(testDir, ext) &&
459
+ suiteComplete()) {
440
460
  results.push({
441
461
  journeyName: journey.name,
442
462
  testDir,
@@ -452,8 +472,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
452
472
  // A feature written up before it is built: its tests come from the doc
453
473
  // alone. There is no code to read for facts and no app to run them on.
454
474
  const notBuilt = /^status:\s*not built/im.test(markdown);
455
- if (notBuilt)
456
- warnings.push('written before the feature exists (Status: not built yet) — run these tests once it ships');
475
+ // Not live yet (not built, or behind a flag): every file is stamped, and
476
+ // `testforge run` holds it back until the doc's Status line goes.
477
+ const pending = pendingReason(markdown);
478
+ if (pending)
479
+ warnings.push(`${pending}: its files are held back by \`testforge run\` until the doc's Status line is removed`);
457
480
  const collision = collisionWarning(areaDir, journey.id, ext);
458
481
  if (collision)
459
482
  warnings.push(collision);
@@ -493,19 +516,44 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
493
516
  const tested = testedTags();
494
517
  let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
495
518
  const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
519
+ const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
520
+ // What each file really tests once written: the memory and the report come from this, not the plan.
521
+ const written = [];
522
+ const finish = async (content, pf, filePath) => {
523
+ if (!memory || !pf.rules?.length)
524
+ return content;
525
+ const { mem, facts } = await fileMemory(pf);
526
+ const done = await completeFile(content, {
527
+ journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
528
+ harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
529
+ check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
530
+ });
531
+ written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
532
+ for (const n of done.notes)
533
+ warnings.push(`${pf.slug}: ${n}`);
534
+ return done.content;
535
+ };
496
536
  // A doc that changed since its suite was written: only what changed is
497
537
  // touched. Renames and removals are edits without AI; reworded and added
498
538
  // rules get one small call each file; new sections become new files below.
499
539
  const prevRecord = memory ? loadMemory(journey) : undefined;
500
- const prevMemory = !force && !opts.planOnly ? prevRecord : undefined;
501
- const prevPlan = prevMemory && prevMemory.hash !== hash ? readPlan(journey) : undefined;
540
+ // Rule ids always follow the last written version, even for --force and
541
+ // --plan-only: an id that moved would point the tests' oracle comments at
542
+ // another rule. Only the in-place update of the tests is skipped then.
502
543
  let changes;
503
- if (memory && prevMemory && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
504
- const { memory: aligned, diff } = alignRules(prevMemory.rules, memory);
544
+ let docDiff;
545
+ if (memory && prevRecord && prevRecord.hash !== hash) {
546
+ const { memory: aligned, diff } = alignRules(prevRecord.rules, memory, (prevRecord.removed ?? []).map((r) => r.id));
505
547
  memory = aligned;
548
+ docDiff = diff;
506
549
  changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
550
+ }
551
+ const prevPlan = docDiff && !force && !opts.planOnly ? readPlan(journey) : undefined;
552
+ if (memory && docDiff && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
553
+ const aligned = memory;
554
+ const diff = docDiff;
507
555
  console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
508
- const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
556
+ const facts = notBuilt ? undefined : (await groundJourney(journey, hash, sourceRoot(), markdown, true)) ?? undefined;
509
557
  const up = await updateJourney({
510
558
  journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
511
559
  pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
@@ -519,7 +567,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
519
567
  const planned = useAI
520
568
  ? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
521
569
  : [{ slug: journey.id, title: journey.name, covers: '' }];
522
- if (memory) {
570
+ // A plan-only run leaves the memory as the tests were last written, so the
571
+ // next generate still sees what changed in the doc and updates those tests.
572
+ if (memory && !opts.planOnly) {
523
573
  const choice = loadSourceChoice();
524
574
  saveMemory(journey, hash, memory, helpers, {
525
575
  doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
@@ -561,7 +611,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
561
611
  // on first use: re-checking an existing suite costs no grounding at all.
562
612
  let groundingP;
563
613
  const grounding = () => (groundingP ??= useAI && !notBuilt
564
- ? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
614
+ ? groundJourney(journey, hash, sourceRoot(), markdown).then((g) => g ?? undefined)
565
615
  : Promise.resolve(undefined));
566
616
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
567
617
  if (staleRemoved.length) {
@@ -593,8 +643,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
593
643
  if (useAI && memory && perCall > 1) {
594
644
  const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
595
645
  const batches = [];
596
- for (let k = 0; k < toWrite.length; k += perCall)
597
- batches.push(toWrite.slice(k, k + perCall));
646
+ // A few files to one call, but never so many rules that the writer drops
647
+ // some: 21 rules in one answer left 14 of them without a test.
648
+ const maxRules = 12;
649
+ for (const pf of toWrite) {
650
+ const last = batches[batches.length - 1];
651
+ const rules = (b) => b.reduce((n, f) => n + (f.rules?.length ?? 0), 0);
652
+ if (last && last.length < perCall && rules(last) + (pf.rules?.length ?? 0) <= maxRules)
653
+ last.push(pf);
654
+ else
655
+ batches.push([pf]);
656
+ }
598
657
  const writeBatch = async (batch) => {
599
658
  const files = await Promise.all(batch.map(async (pf) => {
600
659
  const { mem, facts } = await fileMemory(pf);
@@ -637,8 +696,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
637
696
  // A suite written before pending tests carried their reason gets
638
697
  // it now, deterministically — no regeneration, no AI.
639
698
  const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
640
- if (normalised !== existing)
641
- writeFileSync(filePath, normalised, 'utf-8');
699
+ // A kept file that leaves rules out or holds tests that cannot fail:
700
+ // only those tests are written again.
701
+ const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
702
+ if (finished !== existing)
703
+ writeFileSync(filePath, finished, 'utf-8');
642
704
  return { slug: pf.slug, path: filePath, status: 'ai' };
643
705
  }
644
706
  }
@@ -654,7 +716,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
654
716
  const { mem, facts } = await fileMemory(pf);
655
717
  const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
656
718
  if (ai !== null) {
657
- content = framework === 'playwright' ? normalisePending(ai) : ai;
719
+ content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
658
720
  status = 'ai';
659
721
  }
660
722
  else {
@@ -673,6 +735,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
673
735
  if (!content.includes(GEN_MARKER)) {
674
736
  content = `// ${GEN_MARKER}\n${content}`;
675
737
  }
738
+ content = withPendingStamp(content, pending);
676
739
  // Write IMMEDIATELY (not batched at the end of the journey), so an
677
740
  // interruption preserves every file generated so far.
678
741
  // keepDeferredStubs:false (config) — do NOT write a DEFERRED stub file;
@@ -685,6 +748,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
685
748
  }
686
749
  return { slug: pf.slug, path: filePath, status };
687
750
  });
751
+ if (memory && written.length) {
752
+ const sum = recordWritten(journey, written);
753
+ console.log(` ${journey.id}: ${sum}`);
754
+ }
688
755
  const testFiles = generated.map((g) => ({
689
756
  slug: g.slug,
690
757
  path: g.path,
@@ -74,6 +74,8 @@ export async function updateJourney(ctx) {
74
74
  const base = old.length ? spliceBlocks(content, old.map((block) => ({ block, text: '' }))) : content;
75
75
  let refusal;
76
76
  let next = null;
77
+ // Only the new tests are judged: a problem the file already had is not theirs.
78
+ const already = ctx.check(base, path);
77
79
  for (let attempt = 0; attempt < 2 && next === null; attempt++) {
78
80
  let tests = '';
79
81
  try {
@@ -85,7 +87,8 @@ export async function updateJourney(ctx) {
85
87
  break;
86
88
  }
87
89
  const candidate = appendTests(base, tests);
88
- refusal = findTestBlocks(tests).length ? ctx.check(candidate, path) ?? undefined : 'it contained no test(...) block';
90
+ const why = findTestBlocks(tests).length ? ctx.check(candidate, path) : 'it contained no test(...) block';
91
+ refusal = why && why !== already ? why : undefined;
89
92
  if (!refusal)
90
93
  next = candidate;
91
94
  }
@@ -77,7 +77,12 @@ function hasPage(path, haystack, appRoots) {
77
77
  export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
78
78
  let texts;
79
79
  return files.map((file) => {
80
- const { names, pages } = namesInTest(readFileSync(file, 'utf-8'));
80
+ const content = readFileSync(file, 'utf-8');
81
+ // The doc says the feature is not live: held back whatever the code has.
82
+ const pending = PENDING_STAMP.exec(content)?.[1];
83
+ if (pending)
84
+ return { file, state: 'not-built', missing: [pending.trim()], renamed: [] };
85
+ const { names, pages } = namesInTest(content);
81
86
  const missing = [];
82
87
  const renamed = [];
83
88
  for (const n of names) {
@@ -100,6 +105,28 @@ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
100
105
  return { file, state, missing, renamed };
101
106
  });
102
107
  }
108
+ /** The line that holds back a file whose feature is not live yet. */
109
+ export const PENDING_STAMP = /^\/\/\s*testforge-pending:\s*(.+)$/m;
110
+ /**
111
+ * The status a journey doc gives its feature: "Status: not built", or
112
+ * "Status: behind flag <name>" for code that ships switched off. Undefined
113
+ * when the feature is live.
114
+ */
115
+ export function pendingReason(markdown) {
116
+ const m = /^status:\s*(not built(?: yet)?|behind (?:a )?flag\b[^\n]*)$/im.exec(markdown);
117
+ if (!m)
118
+ return undefined;
119
+ return /^not built/i.test(m[1]) ? 'the feature is not built yet' : `the feature is ${m[1].trim().replace(/^behind (?:a )?flag\s*/i, 'behind the flag ').trim()} — run it once that is switched on`;
120
+ }
121
+ /** The file with the pending stamp set to `reason`, or removed when there is none. */
122
+ export function withPendingStamp(content, reason) {
123
+ const without = content.replace(/^\/\/\s*testforge-pending:.*\n/m, '');
124
+ if (!reason)
125
+ return without;
126
+ const marker = /^\/\/ @generated by testforge.*\n/m.exec(without);
127
+ const at = marker ? marker.index + marker[0].length : 0;
128
+ return `${without.slice(0, at)}// testforge-pending: ${reason}\n${without.slice(at)}`;
129
+ }
103
130
  /** The test files under a directory. */
104
131
  export function testFilesIn(dir) {
105
132
  const out = [];
@@ -13,7 +13,7 @@
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
15
  import { checkReadiness, testFilesIn } from './readiness.js';
16
- import { sourceText } from '../docs/excerpts.js';
16
+ import { sourceRoot, sourceText } from '../docs/excerpts.js';
17
17
  import { spawnSync } from 'node:child_process';
18
18
  import { existsSync } from 'node:fs';
19
19
  import { join, dirname, resolve, relative } from 'node:path';
@@ -136,11 +136,11 @@ export function runTests(opts = {}) {
136
136
  if (framework === 'playwright' && !opts.all) {
137
137
  const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
138
138
  const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
139
- const roots = ['src', ...(config.relatedDirs ?? [])];
139
+ const roots = [sourceRoot(), ...(config.relatedDirs ?? [])];
140
140
  const states = checkReadiness(files, sourceText(roots), () => [], [root]);
141
141
  const held = states.filter((s) => s.state === 'not-built');
142
142
  if (held.length) {
143
- console.log(`\n${held.length} file(s) pending — the code has none of the elements or pages they use yet (\`testforge bind\` shows which; --all runs them anyway):`);
143
+ console.log(`\n${held.length} file(s) pending — their feature is not live yet (\`testforge bind\` shows why; --all runs them anyway):`);
144
144
  for (const h of held.slice(0, 15))
145
145
  console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
146
146
  targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
@@ -164,6 +164,8 @@ export function runTests(opts = {}) {
164
164
  console.log('\nNothing to run — generate tests first (`testforge generate`).');
165
165
  return 1;
166
166
  }
167
- console.log(anyFail ? '\nSome tests failed.' : '\nAll test runs passed.');
167
+ // The runner's own summary above says how many ran and how many skipped; a run where
168
+ // everything skipped has no failure, but it is not a pass.
169
+ console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
168
170
  return anyFail ? 1 : 0;
169
171
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.0.0",
3
+ "version": "2.1.1",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {