@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -9,7 +9,7 @@
9
9
  * 2. An external reference — a spec/doc/PDF in the repo, a website, or
10
10
  * a Notion page capturing the human/client EXPECTED behavior.
11
11
  *
12
- * The choice is remembered in test-artifacts/source.json so later refreshes
12
+ * The choice is remembered in testforge/source.json so later refreshes
13
13
  * (`update`) keep using the same source without asking again — otherwise
14
14
  * an update would silently regenerate reference-based docs from the code.
15
15
  *
@@ -0,0 +1,56 @@
1
+ /**
2
+ * `--tickets <folder>`: a folder of exported tickets (one file per ticket, any
3
+ * text format) used as the reference. The files are bundled, in ticket order,
4
+ * into one reference document under the state folder, each under a heading
5
+ * that carries its ticket id, so every quoted source can name its ticket.
6
+ * Nothing is fetched from a tracker: the person exports what is in scope.
7
+ */
8
+ import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
9
+ import { basename, extname, join } from 'node:path';
10
+ import { REFERENCE_DIR } from '../paths.js';
11
+ export const TICKETS_REFERENCE = join(REFERENCE_DIR, 'tickets.md');
12
+ const TEXT = /\.(md|markdown|txt|text|json|csv|html?|xml|ya?ml)$/i;
13
+ const ID = /\b([A-Z][A-Z0-9]+-\d+)\b/;
14
+ const MAX_CHARS = 20_000;
15
+ /** The ticket id a file stands for: in its name, else in its first lines, else its name. */
16
+ export function ticketId(file, content) {
17
+ return ID.exec(basename(file))?.[1] ?? ID.exec(content.split('\n').slice(0, 5).join('\n'))?.[1] ?? basename(file, extname(file));
18
+ }
19
+ const order = (a, b) => {
20
+ const pa = /^(.*)-(\d+)$/.exec(a), pb = /^(.*)-(\d+)$/.exec(b);
21
+ return pa && pb && pa[1] === pb[1] ? Number(pa[2]) - Number(pb[2]) : a.localeCompare(b);
22
+ };
23
+ /** Bundle every text file under the folder into the tickets reference. Returns its path and the ids. */
24
+ export function bundleTickets(folder) {
25
+ const files = [];
26
+ const walk = (d) => {
27
+ for (const e of readdirSync(d, { withFileTypes: true })) {
28
+ const p = join(d, e.name);
29
+ if (e.isDirectory())
30
+ walk(p);
31
+ else if (TEXT.test(e.name))
32
+ files.push(p);
33
+ }
34
+ };
35
+ if (!statSync(folder).isDirectory())
36
+ throw new Error(`--tickets needs a folder of exported tickets: ${folder} is not one`);
37
+ walk(folder);
38
+ if (!files.length)
39
+ throw new Error(`--tickets: no text files in ${folder} (export each ticket as .md, .txt, .json, .html, …)`);
40
+ const tickets = files
41
+ .map((f) => { const content = readFileSync(f, 'utf-8'); return { id: ticketId(f, content), file: f, content }; })
42
+ .sort((a, b) => order(a.id, b.id));
43
+ const body = [
44
+ '# Tickets',
45
+ '',
46
+ `Exported tickets used as the reference (${tickets.length}). Each section below is one ticket,`,
47
+ 'headed by its id. Every [spec] criterion drawn from them names its ticket in its',
48
+ 'quoted source. Where tickets disagree, the journey lists it as a question for',
49
+ 'product instead of picking one.',
50
+ '',
51
+ ...tickets.flatMap((t) => [`## ${t.id} — ${basename(t.file)}`, '', t.content.trim().slice(0, MAX_CHARS), '']),
52
+ ].join('\n');
53
+ mkdirSync(REFERENCE_DIR, { recursive: true });
54
+ writeFileSync(TICKETS_REFERENCE, body, 'utf-8');
55
+ return { path: TICKETS_REFERENCE, ids: tickets.map((t) => t.id) };
56
+ }
@@ -7,13 +7,13 @@
7
7
  * as full coverage.
8
8
  */
9
9
  import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
10
- import { analyzeFile } from './validate.js';
10
+ import { analyzeFile, testsByTier } from './validate.js';
11
11
  import { COVERAGE_PATH, STATE_DIR } from '../paths.js';
12
12
  /**
13
13
  * F8 — write an honest coverage manifest. For every generated file it
14
14
  * records real (has assertions) vs deferred (pending only), so a "green"
15
15
  * run is never mistaken for full coverage. Written to
16
- * test-artifacts/coverage.json and summarised to the console.
16
+ * testforge/coverage.json and summarised to the console.
17
17
  */
18
18
  export function writeCoverageManifest(results) {
19
19
  const journeys = results
@@ -21,22 +21,24 @@ export function writeCoverageManifest(results) {
21
21
  .map((r) => {
22
22
  const files = r.testFiles.map((t) => {
23
23
  let a = { activeTests: 0, pending: 0, assertions: 0, real: false };
24
+ let tiers = { spec: 0, baseline: 0, observed: 0, untagged: 0 };
24
25
  try {
25
- a = analyzeFile(readFileSync(t.path, 'utf-8'));
26
+ const content = readFileSync(t.path, 'utf-8');
27
+ a = analyzeFile(content);
28
+ tiers = testsByTier(content);
26
29
  }
27
30
  catch {
28
31
  /* file unreadable — count as empty */
29
32
  }
30
- return { file: t.path, status: t.status, ...a };
33
+ return { file: t.path, status: t.status, ...a, tiers };
31
34
  });
35
+ // How many tests rest on a client requirement, and how many on robustness or what the app does.
36
+ const tiers = files.reduce((acc, f) => ({ spec: acc.spec + f.tiers.spec, baseline: acc.baseline + f.tiers.baseline, observed: acc.observed + f.tiers.observed, untagged: acc.untagged + f.tiers.untagged }), { spec: 0, baseline: 0, observed: 0, untagged: 0 });
32
37
  return {
33
38
  journey: r.journeyName,
34
39
  real: files.filter((f) => f.real).length,
35
40
  deferred: files.filter((f) => !f.real).length,
36
- // Whether these files were RUN and passed. A file can assert plenty
37
- // and still target something the app does not have, so "real" alone
38
- // was never enough to call the coverage honest.
39
- verified: r.verified,
41
+ tiers,
40
42
  files,
41
43
  };
42
44
  });
@@ -49,9 +51,6 @@ export function writeCoverageManifest(results) {
49
51
  writeFileSync(COVERAGE_PATH, JSON.stringify({
50
52
  real: totalReal,
51
53
  deferred: totalDeferred,
52
- verified: journeys.filter((j) => j.verified === true).length,
53
- failedVerify: journeys.filter((j) => j.verified === false).length,
54
- notVerified: journeys.filter((j) => j.verified === undefined).length,
55
54
  journeys,
56
55
  }, null, 2), 'utf-8');
57
56
  }
@@ -59,14 +58,8 @@ export function writeCoverageManifest(results) {
59
58
  /* best-effort */
60
59
  }
61
60
  console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see ${COVERAGE_PATH}`);
62
- const ranGreen = journeys.filter((j) => j.verified === true).length;
63
- const ranRed = journeys.filter((j) => j.verified === false).length;
64
- const unrun = journeys.filter((j) => j.verified === undefined).length;
65
- if (ranGreen || ranRed || unrun) {
66
- console.log(`Verified: ${ranGreen} journey(s) ran green` +
67
- (ranRed ? `, ${ranRed} still failing` : '') +
68
- (unrun ? `, ${unrun} not run` : ''));
69
- }
61
+ const t = journeys.reduce((a, j) => ({ spec: a.spec + j.tiers.spec, baseline: a.baseline + j.tiers.baseline, observed: a.observed + j.tiers.observed, untagged: a.untagged + j.tiers.untagged }), { spec: 0, baseline: 0, observed: 0, untagged: 0 });
62
+ console.log(`Tests by the rule they check: ${t.spec} client requirement [spec], ${t.baseline} robustness [baseline], ${t.observed} current behaviour [observed]${t.untagged ? `, ${t.untagged} with no tagged rule` : ''}.`);
70
63
  if (totalDeferred) {
71
64
  console.log(` (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
72
65
  }
@@ -169,6 +169,21 @@ export function checkTestGates(content, _ext, ctx) {
169
169
  if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
170
170
  return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
171
171
  }
172
+ // F22 — a fresh file exists to test its rules: one with no test at all says
173
+ // nothing, and passes. It goes back once, told to stub what it cannot create.
174
+ if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
175
+ return 'the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out';
176
+ }
177
+ // F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
178
+ // variable, seeded data). Skipping because an element is not on the page
179
+ // hides exactly what a test written from the doc exists to report: the
180
+ // element the doc names is missing, and that is a failure.
181
+ if (ctx.forbidPlaceholders) {
182
+ const why = elementSkips(content);
183
+ if (why.length) {
184
+ return `a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`;
185
+ }
186
+ }
172
187
  // F17 — relative imports resolve from where the file is written.
173
188
  if (ctx.filePath) {
174
189
  const missing = unresolvedImports(content, ctx.filePath);
@@ -176,6 +191,28 @@ export function checkTestGates(content, _ext, ctx) {
176
191
  return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
177
192
  }
178
193
  }
194
+ // F18 — no fixed sleeps. A wait for a set time either wastes it or is too
195
+ // short on a slower machine; wait for the event or the state instead.
196
+ if (ctx.framework === 'playwright' && ctx.forbidPlaceholders && /\bwaitForTimeout\s*\(/.test(content)) {
197
+ return 'page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())';
198
+ }
199
+ // F19 — a long test timeout needs a reason. A 240 s budget hides a test that
200
+ // is stuck; the cap makes it fail early with its real error.
201
+ if (ctx.maxTestTimeoutMs) {
202
+ const long = longTimeoutsWithoutReason(content, ctx.maxTestTimeoutMs);
203
+ if (long.length) {
204
+ return `test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`;
205
+ }
206
+ }
207
+ // F20 — one criterion, a few tests. Thirty inputs against a rule that only
208
+ // checks "not empty" prove nothing the first did not; they cost run time and
209
+ // turn one defect into thirty red rows.
210
+ if (ctx.maxTestsPerRule && ctx.forbidPlaceholders) {
211
+ const over = testsPerCriterion(content).find((c) => c.tests > ctx.maxTestsPerRule);
212
+ if (over) {
213
+ return `${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`;
214
+ }
215
+ }
179
216
  // F15 — a pending test says why. A reasonless fixme/skip reaches the report
180
217
  // as a bare "fixme", indistinguishable from the tool giving up. Runs on
181
218
  // content normalisePending() has already given every reason it could find.
@@ -212,9 +249,59 @@ export function checkTestGates(content, _ext, ctx) {
212
249
  }
213
250
  }
214
251
  }
252
+ // (c) A name, label or placeholder the app never renders is a guess too.
253
+ if (ctx.knownTexts) {
254
+ const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
255
+ if (missing.length) {
256
+ return `uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`;
257
+ }
258
+ }
215
259
  }
216
260
  return null;
217
261
  }
262
+ /**
263
+ * The run-time skips whose condition looks at the page: test.skip(true, …),
264
+ * or a condition that counts, checks the visibility of, or locates an element.
265
+ * Returns each condition, shortened.
266
+ */
267
+ export function elementSkips(content) {
268
+ const out = [];
269
+ for (const m of content.matchAll(/\btest\.skip\s*\(/g)) {
270
+ const open = m.index + m[0].length - 1;
271
+ const close = matchParen(content, open);
272
+ if (close === -1)
273
+ continue;
274
+ const args = content.slice(open + 1, close);
275
+ // A declaration skip (test.skip('title', fn)) is the placeholder gate's job.
276
+ if (/^\s*['"`]/.test(args))
277
+ continue;
278
+ const cond = args.split(/,\s*['"`]/)[0];
279
+ const looksAtPage = /^\s*true\s*$/.test(cond) || /\.count\(|\.isVisible\(|\.isHidden\(|getBy\w+\(|\bfind\(|\.locator\(|\$\(/.test(cond);
280
+ // …or a flag set from the page a few lines above (const burger = await …isVisible()).
281
+ const flag = /^\s*!?\s*([A-Za-z_$][\w$]*)\s*$/.exec(cond)?.[1];
282
+ const flagFromPage = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:\\.count\\(|\\.isVisible\\(|getBy\\w+\\(|\\bfind\\(|\\.locator\\()`).test(content);
283
+ if (looksAtPage || flagFromPage)
284
+ out.push(cond.replace(/\s+/g, ' ').trim().slice(0, 80));
285
+ }
286
+ return out;
287
+ }
288
+ /** Literal names a test locates by: getByRole(…, { name }), getByLabel, getByPlaceholder. */
289
+ export function usedLabels(content) {
290
+ const out = new Set();
291
+ const res = [
292
+ /getByRole\(\s*['"`][\w-]+['"`]\s*,\s*\{[^}]*?\bname\s*:\s*(['"`])((?:(?!\1).)+)\1/g,
293
+ /getBy(?:Label|Placeholder)\(\s*(['"`])((?:(?!\1).)+)\1/g,
294
+ /\bfind\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*(['"`])((?:(?!\1).)+)\1/g,
295
+ ];
296
+ for (const re of res) {
297
+ for (const m of content.matchAll(re)) {
298
+ const name = m[2].replace(/\s+/g, ' ').trim();
299
+ if (name.length >= 3 && !name.includes('${') && !name.includes('\\'))
300
+ out.add(name);
301
+ }
302
+ }
303
+ return [...out];
304
+ }
218
305
  /**
219
306
  * Remove pending test blocks (`it/test/describe.skip|fixme|todo(...)`) so the
220
307
  * F9 selector check only ever inspects RUNNABLE code. A guessed selector
@@ -388,10 +475,104 @@ function looksStructurallyClean(content) {
388
475
  t.startsWith('"use strict"');
389
476
  if (!startsWithCode)
390
477
  return false;
391
- const count = (s, ch) => s.split(ch).length - 1;
392
- const balanced = count(content, '{') === count(content, '}') &&
393
- count(content, '(') === count(content, ')');
394
- return balanced;
478
+ return unbalancedAt(content) === null;
479
+ }
480
+ /**
481
+ * Where the brackets of a TS file stop matching, or null when they all do.
482
+ * Strings, template literals (with their \${…}), comments and regex literals
483
+ * are skipped: a "(" inside a test's text is not code. Counting them was what
484
+ * refused correct files as "not valid syntax".
485
+ */
486
+ export function unbalancedAt(content) {
487
+ const stack = [];
488
+ const pairs = { ')': '(', '}': '{', ']': '[' };
489
+ let line = 1;
490
+ let prev = ''; // last significant character, to tell a regex from a division
491
+ const templates = []; // brace depth at which each open template's \${ began
492
+ for (let i = 0; i < content.length; i++) {
493
+ const c = content[i];
494
+ if (c === '\n') {
495
+ line++;
496
+ continue;
497
+ }
498
+ if (/\s/.test(c))
499
+ continue;
500
+ if (c === '/' && content[i + 1] === '/') {
501
+ while (i < content.length && content[i] !== '\n')
502
+ i++;
503
+ line++;
504
+ continue;
505
+ }
506
+ if (c === '/' && content[i + 1] === '*') {
507
+ i += 2;
508
+ while (i < content.length && !(content[i] === '*' && content[i + 1] === '/')) {
509
+ if (content[i] === '\n')
510
+ line++;
511
+ i++;
512
+ }
513
+ i++;
514
+ continue;
515
+ }
516
+ if (c === "'" || c === '"') {
517
+ for (i++; i < content.length && content[i] !== c && content[i] !== '\n'; i++)
518
+ if (content[i] === '\\')
519
+ i++;
520
+ prev = c;
521
+ continue;
522
+ }
523
+ if (c === '`' || (c === '}' && templates.length && templates[templates.length - 1] === stack.length)) {
524
+ if (c === '}')
525
+ templates.pop();
526
+ // Inside a template: up to its end, or to a \${ that opens code.
527
+ for (i++; i < content.length; i++) {
528
+ if (content[i] === '\\') {
529
+ i++;
530
+ continue;
531
+ }
532
+ if (content[i] === '\n')
533
+ line++;
534
+ if (content[i] === '`')
535
+ break;
536
+ if (content[i] === '$' && content[i + 1] === '{') {
537
+ i++;
538
+ templates.push(stack.length);
539
+ break;
540
+ }
541
+ }
542
+ prev = '`';
543
+ continue;
544
+ }
545
+ if (c === '/' && (!prev || /[(,=:[!&|?{};+\-*%<>~^]/.test(prev))) {
546
+ // A regex literal: skip to its closing slash, minding classes and escapes.
547
+ let inClass = false;
548
+ for (i++; i < content.length && content[i] !== '\n'; i++) {
549
+ if (content[i] === '\\') {
550
+ i++;
551
+ continue;
552
+ }
553
+ if (content[i] === '[')
554
+ inClass = true;
555
+ else if (content[i] === ']')
556
+ inClass = false;
557
+ else if (content[i] === '/' && !inClass)
558
+ break;
559
+ }
560
+ while (/[a-z]/i.test(content[i + 1] ?? ''))
561
+ i++;
562
+ prev = 'r';
563
+ continue;
564
+ }
565
+ if (c === '(' || c === '{' || c === '[')
566
+ stack.push({ ch: c, line });
567
+ else if (c in pairs) {
568
+ const open = stack.pop();
569
+ if (!open || open.ch !== pairs[c])
570
+ return `line ${line}: "${c}" does not close ${open ? `the "${open.ch}" from line ${open.line}` : 'anything'}`;
571
+ }
572
+ prev = c;
573
+ }
574
+ const open = stack.pop();
575
+ return open ? `the "${open.ch}" from line ${open.line} is never closed` : null;
395
576
  }
396
577
  const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
397
578
  /** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
@@ -409,3 +590,95 @@ export function unresolvedImports(content, filePath) {
409
590
  && !RESOLVE_EXTS.slice(1).some((e) => existsSync(join(target, 'index' + e)));
410
591
  });
411
592
  }
593
+ /**
594
+ * Helpers written into more than one file of a journey: the same top-level
595
+ * function (same body, whitespace aside) defined in two or more files. Such a
596
+ * copy diverges, and one mistake in it fails every file that carries it — the
597
+ * Stripe card-fill helper copied into nine files was exactly that. One-liners
598
+ * (a locator, a URL builder) are left alone.
599
+ */
600
+ export function duplicatedHelpers(files, minChars = 120) {
601
+ const seen = new Map();
602
+ const START = /^(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(|^(?:export\s+)?const\s+(\w+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|\w+)\s*(?::[^=]+)?=>/;
603
+ for (const f of files) {
604
+ const lines = f.content.split('\n');
605
+ for (let i = 0; i < lines.length; i++) {
606
+ const m = START.exec(lines[i]);
607
+ if (!m)
608
+ continue;
609
+ // The definition runs to the line where its braces close again.
610
+ let depth = 0, j = i, body = '';
611
+ for (; j < lines.length; j++) {
612
+ body += lines[j] + '\n';
613
+ for (const ch of lines[j])
614
+ depth += ch === '{' ? 1 : ch === '}' ? -1 : 0;
615
+ if (depth <= 0 && j > i - 1 && (lines[j].includes('}') || lines[j].trimEnd().endsWith(';')))
616
+ break;
617
+ }
618
+ const norm = body.replace(/\s+/g, '');
619
+ if (norm.length < minChars)
620
+ continue;
621
+ const key = norm;
622
+ const entry = seen.get(key) ?? { name: m[1] ?? m[2], files: new Set() };
623
+ entry.files.add(f.path);
624
+ seen.set(key, entry);
625
+ i = j;
626
+ }
627
+ }
628
+ return [...seen.values()].filter((e) => e.files.size > 1).map((e) => ({ name: e.name, files: [...e.files].sort() }));
629
+ }
630
+ /** test.setTimeout(N) values above the cap whose line, or the line above, carries no "// timeout:" reason. */
631
+ export function longTimeoutsWithoutReason(content, capMs) {
632
+ const lines = content.split('\n');
633
+ const out = [];
634
+ lines.forEach((line, i) => {
635
+ const m = /\btest\.setTimeout\(\s*([\d_]+)\s*\)/.exec(line);
636
+ if (!m)
637
+ return;
638
+ const ms = Number(m[1].replace(/_/g, ''));
639
+ const reason = /\/\/\s*timeout:/i.test(line) || /\/\/\s*timeout:/i.test(lines[i - 1] ?? '');
640
+ if (ms > capMs && !reason)
641
+ out.push(ms);
642
+ });
643
+ return out;
644
+ }
645
+ /** Each test's criterion: the first "// oracle:" line inside it, with its tier. */
646
+ function testCriteria(content) {
647
+ const lines = content.split('\n');
648
+ const starts = [];
649
+ lines.forEach((l, i) => { if (/^\s*(?:test|it)\s*\(/.test(l))
650
+ starts.push(i); });
651
+ return starts.map((start, n) => {
652
+ const end = starts[n + 1] ?? lines.length;
653
+ // The oracle comment sits right above the test or inside it.
654
+ const above = [];
655
+ for (let i = start - 1; i > (starts[n - 1] ?? -1) && /^\s*\/\//.test(lines[i] ?? ''); i--)
656
+ above.push(i);
657
+ for (const i of [...above, ...Array.from({ length: end - start }, (_, k) => start + k)]) {
658
+ const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
659
+ if (m) {
660
+ const tier = (/\[(spec|baseline|observed)\]/i.exec(m[1])?.[1]?.toLowerCase() ?? 'untagged');
661
+ // The criterion is the rule id when the oracle names one, else the quoted rule, else the oracle text.
662
+ const id = /^\s*(R\d+)\b/.exec(m[1])?.[1];
663
+ const quoted = /["“]([^"”]{12,})["”]/.exec(m[1]);
664
+ return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
665
+ }
666
+ }
667
+ return { criterion: '', tier: 'untagged' };
668
+ });
669
+ }
670
+ /** How many tests of a file check each criterion (tests with no oracle are not counted). */
671
+ export function testsPerCriterion(content) {
672
+ const counts = new Map();
673
+ for (const c of testCriteria(content))
674
+ if (c.criterion)
675
+ counts.set(c.criterion, (counts.get(c.criterion) ?? 0) + 1);
676
+ return [...counts].map(([criterion, tests]) => ({ criterion, tests })).sort((a, b) => b.tests - a.tests);
677
+ }
678
+ /** Tests of a file by the tier of the criterion they check. */
679
+ export function testsByTier(content) {
680
+ const out = { spec: 0, baseline: 0, observed: 0, untagged: 0 };
681
+ for (const c of testCriteria(content))
682
+ out[c.tier] += 1;
683
+ return out;
684
+ }