@polymorphism-tech/morph-spec 8.35.1 → 8.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +10 -8
- package/README.md +2 -2
- package/bin/morph-spec.js +4 -3
- package/claude-plugin.json +1 -1
- package/docs/CHEATSHEET.md +1 -1
- package/docs/QUICKSTART.md +1 -1
- package/framework/MORPH.md +24 -23
- package/framework/docs/CLI.md +212 -15
- package/framework/evals/proof-rubrics/eval-de-modelo.md +16 -0
- package/framework/evals/rubrics/condensed.md +3 -1
- package/framework/evals/rubrics/test-coverage.md +8 -0
- package/framework/hooks/shared/phase-utils.js +1 -0
- package/framework/phases.json +1 -1
- package/framework/rules/morph-workflow.md +1 -1
- package/framework/schemas/config.schema.json +7 -0
- package/framework/skills/core/create-umbraco-lp/SKILL.md +1 -1
- package/framework/skills/core/migrate-lp-to-umbraco/SKILL.md +1 -1
- package/framework/skills/core/morph-apply/SKILL.md +21 -10
- package/framework/skills/core/morph-brainstorming/SKILL.md +1 -1
- package/framework/skills/core/morph-eval/SKILL.md +1 -1
- package/framework/skills/core/morph-hotfix/SKILL.md +2 -2
- package/framework/skills/core/morph-implement/SKILL.md +20 -3
- package/framework/skills/core/morph-media/SKILL.md +1 -1
- package/framework/skills/core/morph-plan/SKILL.md +41 -3
- package/framework/skills/core/morph-proposal/SKILL.md +1 -1
- package/framework/skills/core/morph-review/SKILL.md +11 -2
- package/framework/skills/core/morph-stack-scan/SKILL.md +1 -1
- package/framework/skills/core/morph-standards/SKILL.md +1 -1
- package/framework/skills/core/morph-uiux/SKILL.md +1 -1
- package/framework/standards/STANDARDS.json +2 -2
- package/framework/standards/ai-agents/prompt-sources.md +4 -3
- package/framework/standards/ai-agents/testing-ai.md +14 -13
- package/framework/templates/dotnet/ai-kit/README.md +22 -11
- package/framework/templates/dotnet/ai-kit/tests/Morph.AiKit.Tests/Morph.AiKit.Tests.csproj +23 -10
- package/package.json +1 -1
- package/src/commands/project/doctor.js +51 -2
- package/src/commands/tasks/dag.js +13 -1
- package/src/commands/validation/verify.js +59 -17
- package/src/core/paths/output-schema.js +7 -0
- package/src/lib/detectors/index.js +225 -217
- package/src/lib/risk-detector.js +38 -18
- package/src/lib/specs/done-criteria-check.js +200 -0
- package/src/lib/specs/usecase-check.js +129 -5
- package/src/lib/tasks/dotnet-test-introspection.js +632 -0
- package/src/lib/tasks/eval-node.js +3 -3
- package/src/lib/tasks/test-nodes.js +190 -20
- package/src/lib/tasks/test-runner.js +210 -17
- package/src/lib/tasks/test-scope.js +597 -0
- package/src/lib/validators/traceability-check.js +9 -2
- package/src/lib/validators/verify-runner.js +190 -27
- package/src/lib/worktree/docker-audit.js +5 -1
|
@@ -13,9 +13,14 @@
|
|
|
13
13
|
* So the unit of verification becomes a LIST of nodes. `project.tests[]` in
|
|
14
14
|
* `.morph/config/config.json` declares them; when it is absent, the four-step
|
|
15
15
|
* fallback still runs and is synthesized into exactly one node. Declared beats
|
|
16
|
-
* guessed (D3):
|
|
17
|
-
*
|
|
18
|
-
*
|
|
16
|
+
* guessed (D3): a declaration — `tests[]` OR `testCommand` — still wins
|
|
17
|
+
* completely, and deeper auto-detection would only trade one silent false
|
|
18
|
+
* negative for another there. But with NOTHING declared, the guess stops
|
|
19
|
+
* being allowed to be a single answer: the old first-match fallback still
|
|
20
|
+
* picks the primary node exactly as before, and any vitest/jest suite found
|
|
21
|
+
* under a `package.json` (issue #87) is added BESIDE it, because that single
|
|
22
|
+
* guess is exactly what hid 3.082 vitest tests behind a green gate (F30) —
|
|
23
|
+
* the fallback found `.sln`, answered "dotnet test", and never looked twice.
|
|
19
24
|
*
|
|
20
25
|
* WHY VALIDATION IS FAIL-CLOSED HERE (D4)
|
|
21
26
|
*
|
|
@@ -32,11 +37,12 @@
|
|
|
32
37
|
*/
|
|
33
38
|
|
|
34
39
|
import { existsSync, readFileSync } from 'fs';
|
|
35
|
-
import { join, resolve } from 'path';
|
|
40
|
+
import { join, resolve, relative, sep } from 'path';
|
|
36
41
|
|
|
37
42
|
import { resolveTestPlan, readMorphConfig, isInside } from './test-runner.js';
|
|
38
43
|
import { EVAL_RUNNER, evalBlockProblem } from './eval-node.js';
|
|
39
44
|
import { atomicWriteJson, CONFIG_FILE_NAME } from '../../core/state/state-manager.js';
|
|
45
|
+
import { PACKAGE_LOCATIONS } from '../detectors/index.js';
|
|
40
46
|
|
|
41
47
|
/**
|
|
42
48
|
* The count extractors that exist (one per runner, see parseTestCount).
|
|
@@ -45,6 +51,41 @@ import { atomicWriteJson, CONFIG_FILE_NAME } from '../../core/state/state-manage
|
|
|
45
51
|
*/
|
|
46
52
|
export const TEST_RUNNERS = ['dotnet', 'vitest', 'jest', 'node-test', 'playwright', 'none', EVAL_RUNNER];
|
|
47
53
|
|
|
54
|
+
/**
|
|
55
|
+
* The task-group vocabulary a node can declare via `groups`, and that
|
|
56
|
+
* `tasks.json` entries declare via `group` (framework/schemas/tasks.schema.json
|
|
57
|
+
* `~:28`) — ONE enum, checked to agree by canary, never two that drift.
|
|
58
|
+
* Single home for B3's task-scope selection (`test-scope.js`) to import from.
|
|
59
|
+
*/
|
|
60
|
+
export const TASK_GROUPS = ['backend', 'frontend', 'tests', 'infra', 'docs'];
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Which group(s) a node serves when it does NOT declare `groups` itself,
|
|
64
|
+
* keyed by runner. `none` and `evals` are deliberately ABSENT — a command
|
|
65
|
+
* nobody could read (`none`) and a declared eval suite (`evals`) have no
|
|
66
|
+
* honest default group, so `effectiveGroups()` answers null for them rather
|
|
67
|
+
* than guessing one.
|
|
68
|
+
*/
|
|
69
|
+
export const RUNNER_DEFAULT_GROUPS = {
|
|
70
|
+
dotnet: ['backend'],
|
|
71
|
+
'node-test': ['backend'],
|
|
72
|
+
vitest: ['frontend'],
|
|
73
|
+
jest: ['frontend'],
|
|
74
|
+
playwright: ['frontend'],
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Which task group(s) this node answers to: what it DECLARED, or else its
|
|
79
|
+
* runner's default. **null is "no mapping"**, never an invented guess — B3's
|
|
80
|
+
* task-scope selection reads null as "this node is unscoped, always runs".
|
|
81
|
+
*
|
|
82
|
+
* @param {import('./test-nodes.js').TestNode} node
|
|
83
|
+
* @returns {string[]|null}
|
|
84
|
+
*/
|
|
85
|
+
export function effectiveGroups(node) {
|
|
86
|
+
return node?.groups ?? RUNNER_DEFAULT_GROUPS[node?.runner] ?? null;
|
|
87
|
+
}
|
|
88
|
+
|
|
48
89
|
/**
|
|
49
90
|
* A node id is what `--filter` matches and what keys `runs[]`. Lowercase and
|
|
50
91
|
* dash-only so it survives a shell argument and a JSON key without quoting.
|
|
@@ -168,6 +209,11 @@ function makeNode(partial) {
|
|
|
168
209
|
// eval node"** — distinct from an eval node with an empty block, which
|
|
169
210
|
// `validateTestNodes` already refused.
|
|
170
211
|
evals: partial.evals ?? null,
|
|
212
|
+
// Declared task-group(s) this node serves (§B3 task-scope selection).
|
|
213
|
+
// **null is "not declared"**, distinct from an empty array (which
|
|
214
|
+
// `firstProblem` refuses outright) — effectiveGroups() falls back to the
|
|
215
|
+
// runner's default only when this is null.
|
|
216
|
+
groups: partial.groups ?? null,
|
|
171
217
|
status: partial.status ?? null,
|
|
172
218
|
reason: partial.reason ?? null,
|
|
173
219
|
};
|
|
@@ -250,6 +296,28 @@ function firstProblem(raw, projectPath) {
|
|
|
250
296
|
return `evals applies only to runner "${EVAL_RUNNER}"`;
|
|
251
297
|
}
|
|
252
298
|
|
|
299
|
+
if (raw.groups !== undefined) {
|
|
300
|
+
if (!Array.isArray(raw.groups)) {
|
|
301
|
+
return 'groups must be a non-empty array';
|
|
302
|
+
}
|
|
303
|
+
if (raw.groups.length === 0) {
|
|
304
|
+
return 'groups must be a non-empty array';
|
|
305
|
+
}
|
|
306
|
+
const invalid = raw.groups.filter((g) => !TASK_GROUPS.includes(g));
|
|
307
|
+
if (invalid.length > 0) {
|
|
308
|
+
return `groups must contain only ${TASK_GROUPS.join(' | ')} (invalid: ${invalid.join(', ')})`;
|
|
309
|
+
}
|
|
310
|
+
const seenGroups = new Set();
|
|
311
|
+
const duplicates = new Set();
|
|
312
|
+
for (const g of raw.groups) {
|
|
313
|
+
if (seenGroups.has(g)) duplicates.add(g);
|
|
314
|
+
seenGroups.add(g);
|
|
315
|
+
}
|
|
316
|
+
if (duplicates.size > 0) {
|
|
317
|
+
return `groups must not contain duplicates (repeated: ${[...duplicates].join(', ')})`;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
253
321
|
// The cwd checks come last because they are the expensive ones, and because
|
|
254
322
|
// a node that is malformed in shape has a better error to report first.
|
|
255
323
|
const cwd = typeof raw.cwd === 'string' ? raw.cwd.trim() : '.';
|
|
@@ -322,11 +390,97 @@ export function validateTestNodes(config, projectPath = null) {
|
|
|
322
390
|
e2e: entry.e2e ?? false,
|
|
323
391
|
linuxImage: entry.linuxImage ?? null,
|
|
324
392
|
evals: entry.evals ?? null,
|
|
393
|
+
groups: entry.groups ?? null,
|
|
325
394
|
projectCount: 1,
|
|
326
395
|
});
|
|
327
396
|
});
|
|
328
397
|
}
|
|
329
398
|
|
|
399
|
+
/** Turn a relative path into an id-safe slug matching TEST_NODE_ID_PATTERN. */
|
|
400
|
+
function slugifyForId(text) {
|
|
401
|
+
return text
|
|
402
|
+
.toLowerCase()
|
|
403
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
404
|
+
.replace(/^-+|-+$/g, '');
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Guess which frontend test suites this project has, from `package.json`
|
|
409
|
+
* alone — a GUESS (issue #87), never a declaration. Only called by the
|
|
410
|
+
* fallback of `resolveTestNodes()`, and only when nothing was declared: a
|
|
411
|
+
* `package.json` that names `vitest`/`jest` in `dependencies` or
|
|
412
|
+
* `devDependencies` becomes one candidate suite, exactly the shape F30
|
|
413
|
+
* exploited when a repo had .NET AND a frontend and the first-match fallback
|
|
414
|
+
* only ever answered "dotnet test".
|
|
415
|
+
*
|
|
416
|
+
* Fail-open by construction (it IS a guess): an unreadable or malformed
|
|
417
|
+
* `package.json` is skipped silently, never a `fail` node — that fail-closed
|
|
418
|
+
* treatment is reserved for a DECLARATION (`tests[]`), which this is not.
|
|
419
|
+
*
|
|
420
|
+
* @param {string} projectPath - Project root
|
|
421
|
+
* @param {object|null} config - Parsed `.morph/config/config.json`, or null
|
|
422
|
+
* @returns {Array<{ id: string, cwd: string, runner: 'vitest'|'jest', command: string }>}
|
|
423
|
+
*/
|
|
424
|
+
export function discoverFrontendPackages(projectPath, config) {
|
|
425
|
+
const root = resolve(projectPath);
|
|
426
|
+
const candidates = [...PACKAGE_LOCATIONS];
|
|
427
|
+
|
|
428
|
+
const frontendPath = config?.project?.frontendPath;
|
|
429
|
+
if (typeof frontendPath === 'string' && frontendPath.trim()) {
|
|
430
|
+
const abs = resolve(root, frontendPath.trim());
|
|
431
|
+
// Outside the root is IGNORED, not refused — this is a guess reusing
|
|
432
|
+
// another guess (detectProject's own frontendPath), and neither one gets
|
|
433
|
+
// to address another tree.
|
|
434
|
+
if (isInside(abs, root)) candidates.push(frontendPath.trim());
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
const seenDirs = new Set();
|
|
438
|
+
const seenIds = new Set();
|
|
439
|
+
const found = [];
|
|
440
|
+
|
|
441
|
+
for (const loc of candidates) {
|
|
442
|
+
const dirAbs = resolve(root, loc);
|
|
443
|
+
if (seenDirs.has(dirAbs)) continue;
|
|
444
|
+
seenDirs.add(dirAbs);
|
|
445
|
+
|
|
446
|
+
const pkgPath = join(dirAbs, 'package.json');
|
|
447
|
+
if (!existsSync(pkgPath)) continue;
|
|
448
|
+
|
|
449
|
+
let pkg;
|
|
450
|
+
try {
|
|
451
|
+
pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
|
|
452
|
+
} catch {
|
|
453
|
+
continue; // unreadable/invalid — skipped silently, fail-open (it's a guess)
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const deps = { ...(pkg?.dependencies ?? {}), ...(pkg?.devDependencies ?? {}) };
|
|
457
|
+
let runner = null;
|
|
458
|
+
let command = null;
|
|
459
|
+
if (deps.vitest) {
|
|
460
|
+
runner = 'vitest';
|
|
461
|
+
command = 'npx vitest run';
|
|
462
|
+
} else if (deps.jest) {
|
|
463
|
+
runner = 'jest';
|
|
464
|
+
command = 'npx jest --ci';
|
|
465
|
+
}
|
|
466
|
+
if (!runner) continue;
|
|
467
|
+
|
|
468
|
+
const cwd = dirAbs === root ? '.' : relative(root, dirAbs).split(sep).join('/');
|
|
469
|
+
const baseId = cwd === '.' ? runner : `${runner}-${slugifyForId(cwd)}`;
|
|
470
|
+
let id = baseId;
|
|
471
|
+
let n = 2;
|
|
472
|
+
while (seenIds.has(id)) {
|
|
473
|
+
id = `${baseId}-${n}`;
|
|
474
|
+
n += 1;
|
|
475
|
+
}
|
|
476
|
+
seenIds.add(id);
|
|
477
|
+
|
|
478
|
+
found.push({ id, cwd, runner, command });
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
return found;
|
|
482
|
+
}
|
|
483
|
+
|
|
330
484
|
/**
|
|
331
485
|
* Resolve every test suite of a project, in the order the verifier must run them.
|
|
332
486
|
*
|
|
@@ -372,22 +526,28 @@ export function resolveTestNodes(projectPath = '.') {
|
|
|
372
526
|
|
|
373
527
|
const plan = resolveTestPlan(projectPath);
|
|
374
528
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
529
|
+
const nodes = [
|
|
530
|
+
plan.command
|
|
531
|
+
? makeNode({ id: FALLBACK_NODE_ID, command: plan.command, cwd: '.', projectCount: plan.projectCount })
|
|
532
|
+
: makeNode({ id: FALLBACK_NODE_ID, status: 'skip', reason: NO_SUITE_REASON }),
|
|
533
|
+
];
|
|
534
|
+
|
|
535
|
+
// The union (#87 PR B / F30): add discovered frontend suites BESIDE the
|
|
536
|
+
// fallback node — but only when nothing was declared. `plan.source ===
|
|
537
|
+
// 'config'` means `project.testCommand` IS a declaration (it lives inside
|
|
538
|
+
// resolveTestPlan's own step 1), and a declaration wins completely: nothing
|
|
539
|
+
// is added beside it, same as the `project.tests[]` branch above.
|
|
540
|
+
if (plan.source !== 'config') {
|
|
541
|
+
const discovered = discoverFrontendPackages(projectPath, config)
|
|
542
|
+
// The root package is already run by `npm test` when that is the
|
|
543
|
+
// fallback's own answer — adding it again would run the same suite
|
|
544
|
+
// twice under two different commands.
|
|
545
|
+
.filter((pkg) => !(pkg.cwd === '.' && plan.source === 'npm'))
|
|
546
|
+
.map((pkg) => makeNode({ id: pkg.id, command: pkg.command, cwd: pkg.cwd, runner: pkg.runner }));
|
|
547
|
+
nodes.push(...discovered);
|
|
380
548
|
}
|
|
381
549
|
|
|
382
|
-
return {
|
|
383
|
-
nodes: [makeNode({
|
|
384
|
-
id: FALLBACK_NODE_ID,
|
|
385
|
-
command: plan.command,
|
|
386
|
-
cwd: '.',
|
|
387
|
-
projectCount: plan.projectCount,
|
|
388
|
-
})],
|
|
389
|
-
source: plan.source,
|
|
390
|
-
};
|
|
550
|
+
return { nodes, source: plan.source };
|
|
391
551
|
}
|
|
392
552
|
|
|
393
553
|
/**
|
|
@@ -425,10 +585,15 @@ export function morphConfigPath(projectPath = '.') {
|
|
|
425
585
|
* - a measured id that is not declared (appending a node would be inventing
|
|
426
586
|
* the declaration the command came to update);
|
|
427
587
|
* - a declared node with no measurement (a baseline recorded for part of a
|
|
428
|
-
* fleet says nothing true about the rest of it)
|
|
588
|
+
* fleet says nothing true about the rest of it);
|
|
589
|
+
* - a measurement carrying a task-scope `selection` (#87): narrowed, scoped
|
|
590
|
+
* out or run whole INSIDE a task-scoped verify, it measured a recorte, and
|
|
591
|
+
* `expectedCount` is the whole suite's number. `verify` already refuses
|
|
592
|
+
* `--record-baseline` with a task id before the build; this is the second
|
|
593
|
+
* barrier, at the writer, so no caller can route around it.
|
|
429
594
|
*
|
|
430
595
|
* @param {string} projectPath - Project root
|
|
431
|
-
* @param {Array<{id: string, count: number|null}>} measurements - One per node that ran
|
|
596
|
+
* @param {Array<{id: string, count: number|null, selection?: object|null}>} measurements - One per node that ran
|
|
432
597
|
* @returns {{ recorded: boolean, reason: string|null, updates: Array<{id: string, from: number|null, to: number}>, path: string }}
|
|
433
598
|
*/
|
|
434
599
|
export function recordTestBaselines(projectPath, measurements) {
|
|
@@ -449,6 +614,11 @@ export function recordTestBaselines(projectPath, measurements) {
|
|
|
449
614
|
return refuse('config.json não declara project.tests[] — não há entrada onde gravar a baseline');
|
|
450
615
|
}
|
|
451
616
|
|
|
617
|
+
const recortes = measurements.filter((m) => m.selection != null).map((m) => m.id);
|
|
618
|
+
if (recortes.length > 0) {
|
|
619
|
+
return refuse(`medição de recorte de task em: ${recortes.join(', ')} — baseline só de execução inteira, sem {task}`);
|
|
620
|
+
}
|
|
621
|
+
|
|
452
622
|
const unreadable = measurements.filter((m) => !Number.isInteger(m.count)).map((m) => m.id);
|
|
453
623
|
if (unreadable.length > 0) {
|
|
454
624
|
return refuse(`contagem desconhecida em: ${unreadable.join(', ')} — desconhecido nunca vira 0`);
|
|
@@ -170,8 +170,13 @@ function collectDotnetTestProjects(projectPath) {
|
|
|
170
170
|
* says "not inside" for the same directory and a linked worktree leaks into
|
|
171
171
|
* the test command. Only `realpathSync.native` asks the OS. Fail-open to the
|
|
172
172
|
* plain resolve for a path that does not exist (yet).
|
|
173
|
+
*
|
|
174
|
+
* Exported because test-scope.js spells every task output with it (#87 F4): a
|
|
175
|
+
* mis-cased output resolves on a case-insensitive disk and was forwarded to the
|
|
176
|
+
* runner verbatim — and the `--linux` container is case-sensitive. The OS also
|
|
177
|
+
* returns the real CASE of each component, so this one call is both answers.
|
|
173
178
|
*/
|
|
174
|
-
function canonicalPath(p) {
|
|
179
|
+
export function canonicalPath(p) {
|
|
175
180
|
try {
|
|
176
181
|
return realpathSync.native(p);
|
|
177
182
|
} catch {
|
|
@@ -406,34 +411,106 @@ function runTestPrebuild({ node, cwd, exec, timeout, projectPath }) {
|
|
|
406
411
|
// How many tests actually ran (T4 / UC-08)
|
|
407
412
|
// ============================================================================
|
|
408
413
|
|
|
414
|
+
/** VSTest: one summary line per project, STARTING with the verdict. */
|
|
415
|
+
const VSTEST_SUMMARY = /^\s*(?:Aprovado|Passed|Com falha|Failed|Reprovado|Erro)!/i;
|
|
416
|
+
/**
|
|
417
|
+
* Microsoft.Testing.Platform: the header line, in the two locales MEASURED (pt-BR and
|
|
418
|
+
* en-US). `\S+` spans `ção` — and a mis-decoded version of it — without naming a byte.
|
|
419
|
+
*/
|
|
420
|
+
const MTP_SUMMARY_HEADER = /^(?:Test run summary|Resumo da execu\S+ de teste):/;
|
|
421
|
+
const MTP_TOTAL = /^\s+total:\s*(\d+)\s*$/;
|
|
422
|
+
const MTP_FAILED = /^\s+(?:failed|falhou):\s*(\d+)\s*$/;
|
|
423
|
+
const MTP_PASSED = /^\s+(?:succeeded|conclu\S*\s+com\s+\S*xito):\s*(\d+)\s*$/;
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* The indented block under an MTP summary header. It ends at the first line that
|
|
427
|
+
* starts at column 0; a block with no `total:` line yields `total: null`.
|
|
428
|
+
*/
|
|
429
|
+
function mtpSummaryBlock(lines, from) {
|
|
430
|
+
const block = { total: null, passed: 0, failed: 0 };
|
|
431
|
+
for (let i = from; i < lines.length && /^(?:\s|$)/.test(lines[i]); i++) {
|
|
432
|
+
const line = lines[i];
|
|
433
|
+
let m;
|
|
434
|
+
if ((m = MTP_TOTAL.exec(line))) block.total = Number(m[1]);
|
|
435
|
+
else if ((m = MTP_FAILED.exec(line))) block.failed = Number(m[1]);
|
|
436
|
+
else if ((m = MTP_PASSED.exec(line))) block.passed = Number(m[1]);
|
|
437
|
+
}
|
|
438
|
+
return block;
|
|
439
|
+
}
|
|
440
|
+
|
|
409
441
|
/**
|
|
410
|
-
* Sum every
|
|
442
|
+
* Sum every summary `dotnet test` printed, in either of its two formats.
|
|
443
|
+
*
|
|
444
|
+
* - **VSTest**: a line per project that starts with the verdict —
|
|
445
|
+
* `Aprovado! - Com falha: 0, Aprovado: 57, Ignorado: 1, Total: 58, …`.
|
|
446
|
+
* - **Microsoft.Testing.Platform** (xunit.v3 4.x under `global.json`
|
|
447
|
+
* `test.runner`, #115): the verdict does NOT start the line —
|
|
448
|
+
* `Test run summary: Passed!` / `Resumo da execução de teste: Aprovado!` —
|
|
449
|
+
* and the count comes in an indented block below it, with a lowercase
|
|
450
|
+
* `total:`, `failed:`/`falhou:` and `succeeded:`/`concluído com êxito:`.
|
|
451
|
+
* Measured in both locales (test/fixtures/test-output/mtp-*.txt). ONE block
|
|
452
|
+
* per `dotnet test` invocation: a solution of two projects prints a single
|
|
453
|
+
* aggregated block (mtp-solution-fail-skip.txt: `total: 6` = 4 + 2), so
|
|
454
|
+
* blocks add up only across invocations. `total` counts skipped cases, as
|
|
455
|
+
* VSTest's `Total:` does. Another locale is not read: UNKNOWN, never 0.
|
|
411
456
|
*
|
|
412
457
|
* SUM, not "the last one": a chained `a && b` command prints one summary per
|
|
413
|
-
*
|
|
414
|
-
* it — which, against a recorded baseline, is an invented count drop.
|
|
415
|
-
* `countCompletedRuns()` above counts the
|
|
458
|
+
* invocation, and reading a single one would report half a suite as the whole
|
|
459
|
+
* of it — which, against a recorded baseline, is an invented count drop.
|
|
460
|
+
* `countCompletedRuns()` above counts the VSTest lines for a different question
|
|
416
461
|
* (WHICH link was running when the clock died); this one reads their numbers.
|
|
417
462
|
*/
|
|
418
463
|
function parseDotnetCount(output) {
|
|
419
|
-
const lines = output.split('\n')
|
|
464
|
+
const lines = output.split('\n');
|
|
420
465
|
let seen = false;
|
|
421
466
|
const sum = { total: 0, passed: 0, failed: 0 };
|
|
467
|
+
const add = (total, passed, failed) => {
|
|
468
|
+
seen = true;
|
|
469
|
+
sum.total += total;
|
|
470
|
+
sum.passed += passed;
|
|
471
|
+
sum.failed += failed;
|
|
472
|
+
};
|
|
422
473
|
|
|
423
|
-
for (const line of lines) {
|
|
474
|
+
for (const line of lines.filter((l) => VSTEST_SUMMARY.test(l))) {
|
|
424
475
|
const total = /\b(?:Total de testes|Total)\s*:\s*(\d+)/i.exec(line);
|
|
425
476
|
if (!total) continue;
|
|
426
|
-
seen = true;
|
|
427
|
-
sum.total += Number(total[1]);
|
|
428
477
|
const passed = /\b(?:Aprovado|Passed)\s*:\s*(\d+)/i.exec(line);
|
|
429
478
|
const failed = /\b(?:Com falha|Failed|Reprovado)\s*:\s*(\d+)/i.exec(line);
|
|
430
|
-
|
|
431
|
-
sum.failed += failed ? Number(failed[1]) : 0;
|
|
479
|
+
add(Number(total[1]), passed ? Number(passed[1]) : 0, failed ? Number(failed[1]) : 0);
|
|
432
480
|
}
|
|
433
481
|
|
|
482
|
+
lines.forEach((line, i) => {
|
|
483
|
+
if (!MTP_SUMMARY_HEADER.test(line)) return;
|
|
484
|
+
const block = mtpSummaryBlock(lines, i + 1);
|
|
485
|
+
if (block.total !== null) add(block.total, block.passed, block.failed);
|
|
486
|
+
});
|
|
487
|
+
|
|
434
488
|
return seen ? sum : null;
|
|
435
489
|
}
|
|
436
490
|
|
|
491
|
+
/**
|
|
492
|
+
* The `total` of every MTP summary block in `output`, ANSI stripped — one per
|
|
493
|
+
* `dotnet test` invocation (measured: a solution prints ONE aggregated block).
|
|
494
|
+
* `[]` when none is readable. Read by `judgeNarrowedRun` (verify-runner.js): in a
|
|
495
|
+
* narrowed `&&` chain the SUM can be > 0 while one link ran nothing — its exit 8
|
|
496
|
+
* ignored by a path the scanner cannot read — and that link's own block says
|
|
497
|
+
* `total: 0`.
|
|
498
|
+
*
|
|
499
|
+
* @param {string|null|undefined} output
|
|
500
|
+
* @returns {number[]}
|
|
501
|
+
*/
|
|
502
|
+
export function mtpSummaryTotals(output) {
|
|
503
|
+
if (typeof output !== 'string' || !output) return [];
|
|
504
|
+
const lines = stripAnsi(output).split('\n');
|
|
505
|
+
const totals = [];
|
|
506
|
+
lines.forEach((line, i) => {
|
|
507
|
+
if (!MTP_SUMMARY_HEADER.test(line)) return;
|
|
508
|
+
const { total } = mtpSummaryBlock(lines, i + 1);
|
|
509
|
+
if (total !== null) totals.push(total);
|
|
510
|
+
});
|
|
511
|
+
return totals;
|
|
512
|
+
}
|
|
513
|
+
|
|
437
514
|
/** Last value of a `# key N` (TAP) or `ℹ key N` (spec reporter) summary line. */
|
|
438
515
|
function lastSummaryValue(output, key) {
|
|
439
516
|
const re = new RegExp(`^\\s*[#ℹ]\\s*${key}\\s+(\\d+)\\s*$`, 'gim');
|
|
@@ -488,6 +565,32 @@ function parseJestCount(output) {
|
|
|
488
565
|
};
|
|
489
566
|
}
|
|
490
567
|
|
|
568
|
+
/**
|
|
569
|
+
* CSI (`ESC [ ... <final byte @-~>`, e.g. SGR color codes) and OSC (`ESC ] ...
|
|
570
|
+
* BEL|ESC\`, e.g. hyperlinks/terminal titles) escape sequences — the two
|
|
571
|
+
* shapes a captured terminal transcript actually carries (issue #110: a
|
|
572
|
+
* vitest summary piped through `execSync` kept its SGR bytes, and every count
|
|
573
|
+
* regex in this file is anchored on `^` / exact literals, so a color code
|
|
574
|
+
* sitting between "Tests" and the number silently defeated the whole line).
|
|
575
|
+
*/
|
|
576
|
+
const ANSI_PATTERN = /\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\))/g;
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Strip ANSI escape sequences (CSI/SGR and OSC) from captured process output.
|
|
580
|
+
*
|
|
581
|
+
* Non-string input is returned UNCHANGED, never coerced to `''` — the callers
|
|
582
|
+
* that matter (`parseTestCount`, `isZeroTestRun`) already treat a non-string
|
|
583
|
+
* output as "nothing to read" on their own; making this function opinionated
|
|
584
|
+
* about that too would be a second place for the same rule to drift.
|
|
585
|
+
*
|
|
586
|
+
* @param {string} text
|
|
587
|
+
* @returns {string} `text` with escape sequences removed; unchanged if not a string
|
|
588
|
+
*/
|
|
589
|
+
export function stripAnsi(text) {
|
|
590
|
+
if (typeof text !== 'string') return text;
|
|
591
|
+
return text.replace(ANSI_PATTERN, '');
|
|
592
|
+
}
|
|
593
|
+
|
|
491
594
|
/** Playwright: `8 passed (12.4s)` and/or `2 failed`, on their own lines. */
|
|
492
595
|
function parsePlaywrightCount(output) {
|
|
493
596
|
const passed = /^\s*(\d+)\s+passed\b/im.exec(output);
|
|
@@ -501,11 +604,12 @@ function parsePlaywrightCount(output) {
|
|
|
501
604
|
/** One extractor per runner. An absent entry is an honest "I cannot read this". */
|
|
502
605
|
const COUNT_EXTRACTORS = {
|
|
503
606
|
dotnet: parseDotnetCount,
|
|
504
|
-
// An eval IS `dotnet test`, so it reads the same summary
|
|
505
|
-
//
|
|
506
|
-
// field of the fixture's summary FILE, which this
|
|
507
|
-
//
|
|
508
|
-
//
|
|
607
|
+
// An eval IS `dotnet test`, so it reads the same summary — but that number
|
|
608
|
+
// counts xunit cases, not scenarios. What an eval node is judged and recorded
|
|
609
|
+
// by is the `scenarios` field of the fixture's summary FILE, which this
|
|
610
|
+
// function cannot see: `judgeEvalRun` replaces this count with it, or with
|
|
611
|
+
// `null` (UNKNOWN, never zero) when the file is unreadable, and `judgeRun`
|
|
612
|
+
// never compares an eval node against `expectedCount` (#110).
|
|
509
613
|
evals: parseDotnetCount,
|
|
510
614
|
'node-test': parseNodeTestCount,
|
|
511
615
|
vitest: parseVitestCount,
|
|
@@ -529,7 +633,96 @@ export function parseTestCount(output, runner) {
|
|
|
529
633
|
if (typeof output !== 'string' || !output) return null;
|
|
530
634
|
const extractor = COUNT_EXTRACTORS[runner];
|
|
531
635
|
if (!extractor) return null;
|
|
532
|
-
|
|
636
|
+
// SINGLE call site: every extractor reads ANSI-stripped text, never raw
|
|
637
|
+
// bytes — a captured terminal transcript may carry SGR color codes right
|
|
638
|
+
// inside a summary line (#110) and every regex above is anchored on exact
|
|
639
|
+
// literals that a stray `\x1b[32m` would defeat.
|
|
640
|
+
return extractor(stripAnsi(output));
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// ============================================================================
|
|
644
|
+
// "Zero tests ran" signatures, per runner (#87 PR B / #110 parte 1)
|
|
645
|
+
// ============================================================================
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Per-runner signal for "the run matched/executed NOTHING" — a category
|
|
649
|
+
* distinct from both a genuine `{ total: 0, ... }` summary (a suite that
|
|
650
|
+
* legitimately declares zero cases) and from a summary `parseTestCount`
|
|
651
|
+
* simply could not read (`null`, UNKNOWN). Its reader is `judgeNarrowedRun`
|
|
652
|
+
* (verify-runner.js, #87): a narrowed `verify {feature} {task}` run that FAILED
|
|
653
|
+
* with this signature, and with no count proving tests ran, is named
|
|
654
|
+
* `narrowed-zero-tests (…)` instead of an anonymous red. (A PASSED narrowed run
|
|
655
|
+
* needs its own proof there — a count > 0, or MTP's exit 0 — never this table.)
|
|
656
|
+
*
|
|
657
|
+
* Every entry was MEASURED against a real runner, not invented, and the two
|
|
658
|
+
* dotnet entries disagree with each other on purpose:
|
|
659
|
+
*
|
|
660
|
+
* - **MTP** (`--filter-class`, this repo's own `ai-kit`, xunit.v3 4.x):
|
|
661
|
+
* a non-matching filter exits **8** (`ZeroTests`) — reliable BY EXIT
|
|
662
|
+
* CODE. See `test/fixtures/test-output/mtp-zero-tests.txt`.
|
|
663
|
+
* - **VSTest** (`--filter "FullyQualifiedName~..."`, classic xunit 2.x):
|
|
664
|
+
* a non-matching filter exits **0** — exit code CANNOT tell a zero-match
|
|
665
|
+
* run from a green one here. Only the OUTPUT STRING can:
|
|
666
|
+
* `test/fixtures/test-output/vstest-zero-tests.txt` (pt-BR, measured).
|
|
667
|
+
* The en-US wording is the brief's documented string, not measured on
|
|
668
|
+
* this pt-BR machine — kept alongside the pt-BR one because neither is
|
|
669
|
+
* derivable from the other and the machine that runs `verify` may be in
|
|
670
|
+
* either locale.
|
|
671
|
+
*
|
|
672
|
+
* Exit code 8 is therefore listed ONLY as a dotnet exit code, never paired
|
|
673
|
+
* with a string requirement — and exit code 9
|
|
674
|
+
* (`MinimumExpectedTestsPolicyViolation`, this repo's own
|
|
675
|
+
* `--minimum-expected-tests 275`) is deliberately ABSENT: it fires on a run
|
|
676
|
+
* that executed fewer cases than the configured floor, including a run that
|
|
677
|
+
* was entirely green (measured:
|
|
678
|
+
* `test/fixtures/test-output/mtp-filter-class-policy-violation.txt`, 10/10
|
|
679
|
+
* passed, exit 9). Folding 9 into this table would misreport a real pass as
|
|
680
|
+
* "zero tests ran".
|
|
681
|
+
*/
|
|
682
|
+
export const ZERO_RUN_SIGNATURES = {
|
|
683
|
+
dotnet: {
|
|
684
|
+
exitCodes: [8],
|
|
685
|
+
strings: [
|
|
686
|
+
// en-US, VSTest — documented by the brief, not measured here (pt-BR machine).
|
|
687
|
+
'No test matches the given testcase filter',
|
|
688
|
+
// pt-BR, VSTest — MEASURED, test/fixtures/test-output/vstest-zero-tests.txt.
|
|
689
|
+
'Nenhum teste corresponde ao filtro de testcase',
|
|
690
|
+
],
|
|
691
|
+
},
|
|
692
|
+
vitest: {
|
|
693
|
+
exitCodes: [],
|
|
694
|
+
strings: ['No test files found'],
|
|
695
|
+
},
|
|
696
|
+
jest: {
|
|
697
|
+
exitCodes: [],
|
|
698
|
+
strings: ['No tests found'],
|
|
699
|
+
},
|
|
700
|
+
};
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* True when `output`/`exitCode` carries this runner's "zero tests ran"
|
|
704
|
+
* signature. Every other runner (`node-test`, `playwright`, `evals`, `none`,
|
|
705
|
+
* or anything not in `ZERO_RUN_SIGNATURES`) answers `false` — an absent entry
|
|
706
|
+
* is an honest "this primitive does not know that runner's zero-tests shape
|
|
707
|
+
* yet", the same doctrine `COUNT_EXTRACTORS` uses for an unknown runner.
|
|
708
|
+
*
|
|
709
|
+
* Called by `judgeNarrowedRun` (verify-runner.js) on a FAILED narrowed run, to
|
|
710
|
+
* name it `narrowed-zero-tests (…)`.
|
|
711
|
+
*
|
|
712
|
+
* @param {string|null|undefined} output - Combined stdout+stderr of the run
|
|
713
|
+
* @param {string} runner - One of TEST_RUNNERS
|
|
714
|
+
* @param {number|null|undefined} exitCode - Must be a NUMBER to match: the
|
|
715
|
+
* exit-code check is a strict `includes`, so the string `'8'` never matches 8.
|
|
716
|
+
* @returns {boolean}
|
|
717
|
+
*/
|
|
718
|
+
export function isZeroTestRun(output, runner, exitCode) {
|
|
719
|
+
const signature = ZERO_RUN_SIGNATURES[runner];
|
|
720
|
+
if (!signature) return false;
|
|
721
|
+
|
|
722
|
+
if (signature.exitCodes.includes(exitCode)) return true;
|
|
723
|
+
|
|
724
|
+
const text = stripAnsi(typeof output === 'string' ? output : '');
|
|
725
|
+
return signature.strings.some((needle) => text.includes(needle));
|
|
533
726
|
}
|
|
534
727
|
|
|
535
728
|
/**
|