@cleocode/core 2026.10.3 → 2026.10.5-rc.ci.672

package/dist/tasks/evidence.js

@@ -25,12 +25,14 @@ import { createHash } from 'node:crypto';
 import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs';
 import { readFile, stat } from 'node:fs/promises';
 import { dirname, isAbsolute, join, resolve as resolvePath } from 'node:path';
-import { EvidenceParseError, ExitCode, GATE_EVIDENCE_REQUIREMENTS, parseEvidenceString, validateEvidenceForGate, } from '@cleocode/contracts';
+import { EvidenceParseError, GATE_EVIDENCE_REQUIREMENTS, parseEvidenceString, validateEvidenceForGate, } from '@cleocode/contracts/evidence-atom-schema.js';
+import { ExitCode } from '@cleocode/contracts/exit-codes.js';
 import { CleoError } from '../errors.js';
 import { describeUnusableEvidenceGitRoot, E_EVIDENCE_GIT_ROOT, findNestedGitWorkTrees, findWorkTreeContainingCommit, gitToplevel, isGitWorkTree, resolveDeclaredEvidenceGitRoot, } from '../git/work-tree.js';
 import { pushWarning } from '../output.js';
 import { getEffectiveHead } from '../worktree/effective-head.js';
 import { loadRecordedProjectRoots, rebaseLegacyEvidencePath } from './evidence-paths.js';
+import { HEAVY_HEAP_ENV, HEAVY_WORKERS_ENV, isHeavyTool } from './heavy-tool-env.js';
 import { DISABLE_ENV, describeMemoryLimit } from './heavy-tool-limit.js';
 import { computeCommitRevalidationKey, readCommitRevalidationEntry, writeCommitRevalidationEntry, } from './revalidation-cache.js';
 import { bindTestRunReport, coveredTestFiles, TEST_RUN_MAX_RECORDED_FILES, } from './test-run-binding.js';
@@ -1202,6 +1204,121 @@ async function validateFiles(paths, roots, taskId, commitShaFor, requireCommitAr
     }
     return { ok: true, atom: { kind: 'files', files } };
 }
+/**
+ * The counter sets a `test-run:` report may carry (gh#1804), tried in order;
+ * the first whose total key is present is read. `failed` and `notRun` sum
+ * their keys, and an absent count is 0.
+ *
+ * | Shape | Total | Passed | Failed | Skipped / todo |
+ * |---|---|---|---|---|
+ * | vitest `--reporter=json`, jest `--json` | `numTotalTests` | `numPassedTests` | `numFailedTests` | `numPendingTests`, `numTodoTests` |
+ * | a runner's summary output (bun test, tsx --test) | `total` | `passed` | `failed` | `skipped`, `todo` |
+ * | node --test summary (`# tests`, `# pass`, ...) | `tests` | `pass` | `fail`, `cancelled` | `skipped`, `todo` |
+ *
+ * Integrity (T13136 review): every count is a non-negative integer;
+ * passed + failed + skipped/todo equals the total; a failure under ANY key
+ * ({@link TEST_RUN_FAILURE_KEYS}) refuses, whichever set supplied the total;
+ * a report's `exit` / `exitCode` must be a number, and a non-zero one is
+ * refused. A `test-run:` atom proves only what its file says; `tool:test` and
+ * `ci:<pr>` are what prove the run. `cleo verify --help` documents the same
+ * sets.
+ *
+ * @task T13136
+ */
+const TEST_RUN_COUNTER_SHAPES = [
+    {
+        name: 'vitest/jest',
+        total: 'numTotalTests',
+        passed: 'numPassedTests',
+        failed: ['numFailedTests'],
+        notRun: ['numPendingTests', 'numTodoTests'],
+    },
+    {
+        name: 'summary',
+        total: 'total',
+        passed: 'passed',
+        failed: ['failed'],
+        notRun: ['skipped', 'todo'],
+    },
+    {
+        name: 'node --test',
+        total: 'tests',
+        passed: 'pass',
+        failed: ['fail', 'cancelled'],
+        notRun: ['skipped', 'todo'],
+    },
+];
+/** The keys of each counter set, as the refusals name them. */
+const TEST_RUN_SHAPE_HINT = TEST_RUN_COUNTER_SHAPES.map((s) => `${[s.total, s.passed, ...s.failed, ...s.notRun].join('/')} (${s.name})`).join(', ');
+/**
+ * Keys that name failures in any counter set, plus jest/vitest suite failures
+ * and the mocha / JUnit spellings: a positive value under any of them refuses
+ * the report, whichever set supplied the total.
+ */
+const TEST_RUN_FAILURE_KEYS = [
+    'numFailedTests',
+    'numFailedTestSuites',
+    'numRuntimeErrorTestSuites',
+    'failed',
+    'fail',
+    'cancelled',
+    'failures',
+    'errors',
+];
+/** A present, non-null JSON value. */
+const present = (value) => value !== undefined && value !== null;
+/** A non-negative integer, or `undefined` when the value is anything else. */
+const countOf = (value) => typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined;
+/**
+ * Read a report's counts from the first counter set it carries, enforcing
+ * their integrity; `null` when it carries none.
+ */
+function testRunCounts(report) {
+    const shape = TEST_RUN_COUNTER_SHAPES.find((s) => present(report[s.total]));
+    if (shape === undefined)
+        return null;
+    const invalid = (key) => ({
+        reason: `test-run report's "${key}" is ${JSON.stringify(report[key])}; every count must be a non-negative integer`,
+        codeName: 'E_EVIDENCE_INVALID',
+    });
+    const read = (key) => {
+        if (!present(report[key]))
+            return 0;
+        return countOf(report[key]) ?? invalid(key);
+    };
+    const total = read(shape.total);
+    if (typeof total !== 'number')
+        return total;
+    const keys = [shape.passed, ...shape.failed, ...shape.notRun];
+    const values = [];
+    for (const key of keys) {
+        const v = read(key);
+        if (typeof v !== 'number')
+            return v;
+        values.push(v);
+    }
+    for (const key of TEST_RUN_FAILURE_KEYS) {
+        if (!present(report[key]))
+            continue;
+        const v = countOf(report[key]);
+        if (v === undefined)
+            return invalid(key);
+        if (v > 0) {
+            return { reason: `test-run reports ${key} = ${v}`, codeName: 'E_EVIDENCE_TESTS_FAILED' };
+        }
+    }
+    const [passed = 0, ...rest] = values;
+    const failed = rest.slice(0, shape.failed.length).reduce((a, b) => a + b, 0);
+    const notRun = rest.slice(shape.failed.length).reduce((a, b) => a + b, 0);
+    if (passed + failed + notRun !== total) {
+        return {
+            reason: `test-run report's counts do not add up: ${shape.total} is ${total}, but ` +
+                `${shape.passed} + ${[...shape.failed, ...shape.notRun].join(' + ')} = ${passed + failed + notRun}`,
+            codeName: 'E_EVIDENCE_INVALID',
+        };
+    }
+    return { shape, total, passed, failed, notRun };
+}
 /**
  * `tool:test-affected` (D11150, T12635): run `testing.affectedCommand` over the
  * packages the branch diff touches plus their dependents, through the same
@@ -1209,26 +1326,31 @@ async function validateFiles(paths, roots, taskId, commitShaFor, requireCommitAr
  * memory and timeout limits apply). The atom records `scope: 'affected'` and
  * the packages, so a receipt never passes for a full run.
  */
-async function validateAffectedTests(roots) {
+async function validateAffectedTests(roots, taskId) {
     const { planAffectedTestRun } = await import('./affected-packages.js');
     const run = await planAffectedTestRun(roots.storeRoot, roots.executionRoot, { wait: true });
     if (!run.ok)
         return { ok: false, codeName: run.codeName, reason: run.reason };
-    return runAffectedTests('test-affected', run, roots);
+    return runAffectedTests('test-affected', run, roots, taskId);
 }
 /**
  * Run a planned affected-scope test run and record it under `tool` —
  * `test-affected`, or `test` when a scope-aware `tool:test` chose the
  * affected scope (T12959).
  */
-async function runAffectedTests(tool, run, roots) {
+async function runAffectedTests(tool, run, roots, taskId) {
     const { storeRoot, executionRoot } = roots;
-    const result = await runToolCached(run.command, storeRoot, { executionRoot });
+    // T13132: status shows what holds the budget — the scope and the task.
+    const result = await runToolCached(run.command, storeRoot, {
+        executionRoot,
+        semaphoreOptions: { scope: 'affected', ...(taskId !== undefined ? { task: taskId } : {}) },
+    });
     if (result.exitCode !== 0) {
         return {
             ok: false,
             codeName: result.timedOut ? 'E_EVIDENCE_TOOL_TIMEOUT' : 'E_EVIDENCE_TOOL_FAILED',
-            reason: `tool:${tool} (affected: ${[run.command.cmd, ...run.command.args].join(' ')}) exited ${result.exitCode}: ${(result.stderrTail || result.stdoutTail).trim().slice(-300)}`,
+            reason: `tool:${tool} (affected: ${[run.command.cmd, ...run.command.args].join(' ')}) exited ${result.exitCode}: ${(result.stderrTail || result.stdoutTail).trim().slice(-300)}` +
+                (result.signal !== null || result.resourceKill !== null ? resourcePlanNote(result) : ''),
         };
     }
     return {
@@ -1239,7 +1361,10 @@ async function runAffectedTests(tool, run, roots) {
             exitCode: 0,
             stdoutTail: result.stdoutTail,
             scope: 'affected',
-            scopeReason: AFFECTED_SCOPE_BASIS,
+            scopeReason: run.template.source === 'derived'
+                ? `${AFFECTED_SCOPE_BASIS}; the affected command was derived from the workspace test ` +
+                    `command "${run.template.basis}" (declare testing.affectedCommand to pin it, T13125)`
+                : AFFECTED_SCOPE_BASIS,
             affectedPackages: run.packages,
             affectedProjects: run.projects,
             ...(run.untested.length > 0 ? { untestedPackages: run.untested } : {}),
@@ -1417,14 +1542,64 @@ async function validateTestRun(path, roots, taskId) {
             codeName: 'E_EVIDENCE_INVALID',
         };
     }
-    const total = parsed.numTotalTests ?? 0;
-    const failed = parsed.numFailedTests ?? 0;
-    const passed = parsed.numPassedTests ?? 0;
-    const pending = (parsed.numPendingTests ?? 0) + (parsed.numTodoTests ?? 0);
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        return {
+            ok: false,
+            reason: `test-run file is not a JSON object. Expected test counts: ${TEST_RUN_SHAPE_HINT}`,
+            codeName: 'E_EVIDENCE_INVALID',
+        };
+    }
+    if (present(parsed.testResults) &&
+        (!Array.isArray(parsed.testResults) ||
+            parsed.testResults.some((tr) => tr === null || typeof tr !== 'object' || Array.isArray(tr)))) {
+        return {
+            ok: false,
+            reason: 'test-run report\'s "testResults" must be an array of objects',
+            codeName: 'E_EVIDENCE_INVALID',
+        };
+    }
+    const counts = testRunCounts(parsed);
+    if (counts !== null && !('shape' in counts)) {
+        return { ok: false, reason: counts.reason, codeName: counts.codeName };
+    }
+    if (counts === null) {
+        return {
+            ok: false,
+            reason: `test-run report has no test counts. Expected one of: ${TEST_RUN_SHAPE_HINT}; ` +
+                "e.g. vitest's --reporter=json --outputFile, or " +
+                '{"total":43,"passed":42,"failed":0,"skipped":1}',
+            codeName: 'E_EVIDENCE_INVALID',
+        };
+    }
+    const { total, failed, passed, notRun: pending } = counts;
+    // Both spellings are checked: a report carrying exit 0 and exitCode 1 failed
+    // (T13136 review).
+    let exit;
+    for (const exitKey of ['exit', 'exitCode']) {
+        const exitValue = parsed[exitKey];
+        if (!present(exitValue))
+            continue;
+        if (!(typeof exitValue === 'number' && Number.isInteger(exitValue))) {
+            return {
+                ok: false,
+                reason: `test-run report's "${exitKey}" is ${JSON.stringify(exitValue)}; an exit code must be a number`,
+                codeName: 'E_EVIDENCE_INVALID',
+            };
+        }
+        if (exit === undefined || exitValue !== 0)
+            exit = exitValue;
+    }
     if (total === 0) {
         return {
             ok: false,
-            reason: 'test-run reports zero total tests (no tests were executed)',
+            reason: `test-run reports zero total tests (no tests were executed): ${counts.shape.total} is 0`,
+            codeName: 'E_EVIDENCE_TESTS_FAILED',
+        };
+    }
+    if (exit !== undefined && exit !== 0) {
+        return {
+            ok: false,
+            reason: `test-run reports exit code ${exit}`,
             codeName: 'E_EVIDENCE_TESTS_FAILED',
         };
     }
@@ -1504,7 +1679,7 @@ async function validateTestRun(path, roots, taskId) {
 }
 async function validateTool(tool, roots, context) {
     if (tool === 'test-affected')
-        return validateAffectedTests(roots);
+        return validateAffectedTests(roots, context?.task.id);
     // T12959: `tool:test` runs the affected packages first when a template is
     // declared; the full suite runs only when that scope cannot be trusted, and
     // the atom says which scope ran and why.
@@ -1525,8 +1700,9 @@ async function validateTool(tool, roots, context) {
                 }
                 : {}),
         });
-        if (plan.scope === 'affected')
-            return runAffectedTests('test', plan.run, roots);
+        if (plan.scope === 'affected') {
+            return runAffectedTests('test', plan.run, roots, context?.task.id);
+        }
         if (plan.scope === 'pending') {
             return {
                 ok: false,
@@ -1581,7 +1757,16 @@ async function validateTool(tool, roots, context) {
     // gh#1220/#1226/#1230: spawn in — and fingerprint against — the tree the
     // operator actually invoked from, not the shared store root. gh#1365: the
     // root arrives as a parameter now; it is NOT re-resolved here.
-    const result = await runToolCached(resolution.command, projectRoot, { executionRoot });
+    // T13132: a heavy tool here runs the whole project (the affected scope
+    // returned above); status marks it scope=full with its task.
+    const taskId = context?.task.id;
+    const result = await runToolCached(resolution.command, projectRoot, {
+        executionRoot,
+        semaphoreOptions: {
+            ...(isHeavyTool(resolution.command.canonical) ? { scope: 'full' } : {}),
+            ...(taskId !== undefined ? { task: taskId } : {}),
+        },
+    });
     // T12025: wall-clock child-process deadline exceeded — the tool was
     // terminated and the lock released. Signal retry; do NOT record a partially
     // captured gate.
@@ -1626,7 +1811,7 @@ async function validateTool(tool, roots, context) {
             reason: `Tool "${tool}" → ${resolution.command.cmd} ${resolution.command.args.join(' ')} ` +
                 `RAN for ${Math.round(result.durationMs / 1000)}s in ${result.executionRoot} and was ` +
                 `KILLED by ${result.signal}. The binary is present and the command started — ` +
-                `this is not a resolution problem.${limit} ` +
+                `this is not a resolution problem.${limit}${resourcePlanNote(result)} ` +
                 `Nothing was cached, so a retry re-runs the tool from scratch and will be killed ` +
                 `identically unless the cause is addressed.` +
                 (result.stdoutTail || result.stderrTail
@@ -1645,9 +1830,10 @@ async function validateTool(tool, roots, context) {
             reason: `Tool "${tool}" → ${resolution.command.cmd} ${resolution.command.args.join(' ')} ` +
                 `RAN for ${Math.round(result.durationMs / 1000)}s in ${result.executionRoot} and was ` +
                 `KILLED for resources (${result.resourceKill}). This is not a verdict on the code, ` +
-                `and nothing was cached. Raise the heap (NODE_OPTIONS=--max-old-space-size=<MiB>) or lower the worker count ` +
-                `(VITEST_MAX_WORKERS=<n>) and verify again — both are part of the cache key, so the ` +
-                `retry runs fresh.` +
+                `and nothing was cached.${resourcePlanNote(result)} Raise the heap with ` +
+                `${HEAVY_HEAP_ENV}=<MiB> (an inherited NODE_OPTIONS heap above the run's budget is ` +
+                `clamped, T13122) or lower the worker count with ${HEAVY_WORKERS_ENV}=<n>, and verify ` +
+                `again — both are part of the cache key, so the retry runs fresh.` +
                 (result.stdoutTail || result.stderrTail
                     ? ` Last output: ${tailString(`${result.stdoutTail}\n${result.stderrTail}`, 512)}`
                     : ''),
@@ -1734,8 +1920,18 @@ function toolRunAtomFields(result) {
         ...(result.treeHash ? { treeHash: result.treeHash } : {}),
         cacheHit: result.cacheHit,
         ...(result.flaky ? { flaky: result.flaky } : {}),
+        // T13122: the heap and worker count the run got, and why — including an
+        // inherited NODE_OPTIONS heap or worker count clamped to the budget.
+        ...(result.resources ? { resources: result.resources } : {}),
     };
 }
+/**
+ * One sentence naming the heap and worker plan a memory-bound tool ran under
+ * (T13122), for a message about a kill; `''` for any other tool.
+ */
+function resourcePlanNote(result) {
+    return result.resources ? ` It ran with ${result.resources.summary}.` : '';
+}
 // Re-export so downstream code can keep importing the canonical-tools list
 // from evidence.ts without crossing into tool-resolver internals.
 export { CANONICAL_TOOLS };