mandrel 2.36.0 → 2.37.0

package/.agents/scripts/lib/cpu-pool.js

@@ -2,7 +2,9 @@
  * lib/cpu-pool.js — Generic worker_threads pool for CPU-bound work.
  *
  * `runOnPool(workerScript, items, opts)` spawns up to
- * `opts.concurrency ?? os.availableParallelism()` persistent workers,
+ * `resolvePoolConcurrency(opts.concurrency)` persistent workers — an
+ * explicit option, else `MANDREL_POOL_CONCURRENCY`, else a clamp of 4 under
+ * `node:test`, else `os.availableParallelism()` —
  * dispatches `items` to whichever worker is idle, and resolves with an
  * array of per-item results in input order. The queue is bounded by
  * worker count — we only have N items in flight at once, where N =
@@ -58,15 +60,93 @@ const defaultWorkerFactory = (script, options) => new Worker(script, options);
  * Pool-vs-serial cutover for `runOnPool` callers.
  *
  * Below this batch size the pool's worker spawn overhead dominates, so
- * callers fall back to in-process serial scoring. Tuned against the test
- * suite's tmpdir fixtures (n=2 stays serial; the full repo n≈200–470
- * takes the pool path). Single-sourced here so the maintainability
- * baseline scan (`maintainability-utils.js`), the CRAP scanner
- * (`crap-utils.js`), and the native review provider
- * (`review-providers/native.js`) cannot silently desynchronize on a
- * retune.
+ * callers fall back to in-process serial scoring. Single-sourced here so the
+ * maintainability baseline scan (`maintainability-utils.js`), the CRAP
+ * scanner (`crap-utils.js`), and the native review provider
+ * (`review-providers/native.js`) cannot silently desynchronize on a retune.
+ *
+ * **Retuned from 8 to 256 in Story #5109.** The original 8 was set against
+ * tmpdir fixtures and never measured against a real batch, so every
+ * pre-commit preview — the most frequent path in the framework — paid for a
+ * pool it could not amortise. Measured on this repository (18 logical cores,
+ * `.agents/scripts` + `bin` + `lib`, MI scoring):
+ *
+ * ```text
+ *   n=16   serial  125 ms   pooled  444 ms
+ *   n=64   serial  175 ms   pooled  535 ms
+ *   n=128  serial  241 ms   pooled  422 ms
+ *   n=256  serial  440 ms   pooled  707 ms   <- still serial-favourable
+ *   n=384  serial  943 ms   pooled  782 ms   <- crossover
+ *   n=619  serial 1054 ms   pooled  710 ms
+ * ```
+ *
+ * The crossover sits between 256 and 384, so 256 is the last power-of-two
+ * step the serial path provably wins. A whole-tree baseline refresh
+ * (n≈619) still takes the pool; a diff-scoped preview (n≈50) no longer
+ * spawns a worker at all. Scoring output is identical on either path — the
+ * worker runs the same scorer — so this is a cost retune, never a verdict
+ * change.
+ */
+export const POOL_SERIAL_THRESHOLD = 256;
+
+/**
+ * Hard ceiling on pool width when the process is a `node:test` child.
+ *
+ * Node's test runner already fans test *files* out across processes, so a
+ * pool inside one of them multiplies against that fan-out and can oversubscribe
+ * the host by an order of magnitude. Four is wide enough to keep the pooled
+ * code path genuinely concurrent (so the scheduling branches under test stay
+ * exercised) without letting a suite of parallel test processes each claim
+ * every core.
+ */
+const NODE_TEST_CONCURRENCY_CLAMP = 4;
+
+/**
+ * True when this process was spawned by Node's own test runner, which sets
+ * `NODE_TEST_CONTEXT` in every test child.
+ *
+ * @param {NodeJS.ProcessEnv} env
+ * @returns {boolean}
+ */
+function inNodeTestContext(env) {
+  return (
+    typeof env.NODE_TEST_CONTEXT === 'string' && env.NODE_TEST_CONTEXT !== ''
+  );
+}
+
+/**
+ * Resolve the requested pool width, in strict precedence order:
+ *
+ *   1. an explicit `opts.concurrency` — the caller knows its own budget;
+ *   2. `MANDREL_POOL_CONCURRENCY` — the operator/CI override, so a
+ *      constrained runner can bound every pool in the process tree at once;
+ *   3. `NODE_TEST_CONCURRENCY_CLAMP` when running under `node:test`;
+ *   4. `os.availableParallelism()`.
+ *
+ * Only *finite, positive* values are honoured at each step; anything else
+ * falls through to the next, so a typo in the env var degrades to the
+ * default rather than collapsing the pool to a single worker.
+ *
+ * Module-private on purpose: `runOnPool` is the only caller, and the
+ * precedence is pinned through its observable worker count rather than by
+ * an export whose sole importer would be a test.
+ *
+ * @param {number|undefined} explicit
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {number}
  */
-export const POOL_SERIAL_THRESHOLD = 8;
+function resolvePoolConcurrency(explicit, env = process.env) {
+  const positive = (value) => {
+    const n = Number(value);
+    return Number.isFinite(n) && n >= 1 ? Math.floor(n) : null;
+  };
+  return (
+    positive(explicit) ??
+    positive(env.MANDREL_POOL_CONCURRENCY) ??
+    (inNodeTestContext(env) ? NODE_TEST_CONCURRENCY_CLAMP : null) ??
+    os.availableParallelism()
+  );
+}
 
 /**
  * @template TItem, TResult
@@ -90,7 +170,7 @@ export async function runOnPool(workerScript, items, opts = {}) {
   const itemsArr = [...items];
   if (itemsArr.length === 0) return [];
 
-  const requested = opts.concurrency ?? os.availableParallelism();
+  const requested = resolvePoolConcurrency(opts.concurrency);
   const concurrency = Math.max(1, Math.min(requested, itemsArr.length));
   const workerData = opts.workerData;
   const throwOnItemError = opts.throwOnItemError === true;

package/.agents/scripts/lib/crap-utils.js

@@ -21,7 +21,10 @@ import {
 const CRAP_WORKER_URL = new URL('./workers/crap-worker.js', import.meta.url);
 
 // Pool-vs-serial cutover — single-sourced in cpu-pool.js (see the
-// POOL_SERIAL_THRESHOLD docstring for the tuning rationale).
+// POOL_SERIAL_THRESHOLD docstring for the tuning rationale). Callers may
+// override it per scan via `serialThreshold`, which is how the parity tests
+// drive the pooled path without materialising a batch bigger than the
+// cutover (Story #5109 raised it from 8 to 256).
 const SERIAL_THRESHOLD = POOL_SERIAL_THRESHOLD;
 // 1.1.0 — TypeScript support landed in 5.29.0. Bumped from 1.0.0 because
 // the scanner now emits CRAP rows for TS/TSX paths that the previous
@@ -393,6 +396,7 @@ export async function scanAndScore({
   ignoreGlobs = [],
   preScannedFiles = null,
   incremental = null,
+  serialThreshold = SERIAL_THRESHOLD,
 }) {
   if (!Array.isArray(targetDirs)) {
     throw new TypeError('scanAndScore: targetDirs must be an array');
@@ -430,7 +434,7 @@ export async function scanAndScore({
   // Serial below the pool cutover, and ALWAYS in incremental mode: the
   // per-file baseline lookup Maps the join needs do not cross the worker
   // boundary (Story #4981).
-  const runSerial = queue.length < SERIAL_THRESHOLD || Boolean(incremental);
+  const runSerial = queue.length < serialThreshold || Boolean(incremental);
   const perFile = runSerial
     ? queue.map((item) => ({ item, result: scoreFileSerial(item, coverage) }))
     : await scoreFilesViaPool(queue, coverage);

package/.agents/scripts/lib/cyclomatic-ceiling.js

@@ -30,8 +30,9 @@
  */
 
 import path from 'node:path';
+import { selectFilesToScore } from './cyclomatic-scope.js';
 import { calculateReportForFile } from './maintainability-engine.js';
-import { isIgnoredByGlobs, scanDirectory } from './maintainability-utils.js';
+import { scanDirectory } from './maintainability-utils.js';
 
 /** Default location of the committed breach baseline. */
 export const DEFAULT_CYCLOMATIC_BASELINE = 'baselines/cyclomatic.json';
@@ -96,20 +97,37 @@ function breachRowFor(file, methods, ceiling) {
  * `scoreFile` is a seam so tests can drive the reduction without the kernel;
  * production callers omit it.
  *
+ * **Scoped scoring (Story #5109).** The directory walk is cheap — a few
+ * `readdir` calls — but scoring is an escomplex parse per file, and re-parsing
+ * the whole tree on every CI run cost 0.94 s to re-derive rows that could not
+ * have moved. `scopeFiles`, when supplied, is the set of repo-relative POSIX
+ * paths that are still *scored*; everything else is walked, counted, and
+ * skipped. That is sound precisely because this gate is a ratchet: a verdict
+ * exists only for a file that either changed (so it must be re-scored) or is
+ * already recorded in the baseline (so its recorded count must be re-derived
+ * to detect an improvement or a removal). A file that is neither cannot
+ * produce `added` or `worsened`, and `removed` is computed from baseline files
+ * — all of which are in scope by construction. `scannedFiles` therefore keeps
+ * reporting the **whole walk**, so the reported scan surface does not shrink
+ * just because the scoring did. Callers pass `null` (the default) for the
+ * whole-tree scan, which `--update` and `BASELINE_SCOPE=full` always do.
+ *
  * @param {{
  *   targetDirs: string[],
  *   ignoreGlobs?: string[],
  *   ceiling: number,
  *   cwd?: string,
+ *   scopeFiles?: Set<string> | null,
  *   scoreFile?: (absPath: string) => { methods?: Array<{ cyclomatic?: number }>, parseError?: boolean },
  * }} args
- * @returns {{ rows: Array<object>, scannedFiles: number, parseErrors: number }}
+ * @returns {{ rows: Array<object>, scannedFiles: number, scoredFiles: number, parseErrors: number }}
  */
 export function scanCyclomatic({
   targetDirs,
   ignoreGlobs = [],
   ceiling,
   cwd = process.cwd(),
+  scopeFiles = null,
   scoreFile = calculateReportForFile,
 }) {
   const files = [];
@@ -118,21 +136,25 @@ export function scanCyclomatic({
     scanDirectory(abs, files, { cwd, ignoreGlobs });
   }
   files.sort();
+  const selected = selectFilesToScore(files, { cwd, ignoreGlobs, scopeFiles });
   const rows = [];
   let parseErrors = 0;
-  for (const abs of files) {
-    if (isIgnoredByGlobs(abs, ignoreGlobs, cwd)) continue;
+  for (const { abs, rel } of selected) {
     const report = scoreFile(abs);
     if (!report || report.parseError) {
       parseErrors += 1;
       continue;
     }
-    const rel = path.relative(cwd, abs).split(path.sep).join('/');
     const row = breachRowFor(rel, report.methods, ceiling);
     if (row) rows.push(row);
   }
   rows.sort((a, b) => a.file.localeCompare(b.file));
-  return { rows, scannedFiles: files.length, parseErrors };
+  return {
+    rows,
+    scannedFiles: files.length,
+    scoredFiles: selected.length,
+    parseErrors,
+  };
 }
 
 /**

package/.agents/scripts/lib/cyclomatic-scope.js

@@ -0,0 +1,144 @@
+/**
+ * cyclomatic-scope.js — which files the cyclomatic ratchet still has to score
+ * (Story #5109).
+ *
+ * `check-cyclomatic.js` re-parsed the whole tree on every run — 0.94 s of
+ * escomplex work to re-derive rows that could not have moved. The scan surface
+ * is unchanged; only the *scoring* is narrowed, and it is narrowed to exactly
+ * the two sets a ratchet verdict can come from:
+ *
+ *   - **every file already recorded in the baseline**, because `improved`,
+ *     `worsened` and `removed` are all computed by re-deriving a recorded
+ *     file's current count;
+ *   - **everything this branch changed** against `project.baseBranch`,
+ *     including the staged, unstaged and untracked working tree, because
+ *     `added` is what a change introduces — and a pre-push gate that only
+ *     looked at committed work would wave through the breach an operator is
+ *     about to commit.
+ *
+ * A file in neither set is unchanged and unrecorded: it cannot produce a
+ * verdict in any bucket.
+ *
+ * @module lib/cyclomatic-scope
+ */
+
+import path from 'node:path';
+import { createGitInterface } from './git-utils.js';
+import { isIgnoredByGlobs } from './maintainability-utils.js';
+
+/**
+ * Git probes whose union is "everything this working copy has touched".
+ *
+ * Three, not four: `git diff --name-only HEAD` already reports the staged
+ * *and* unstaged deltas, so a separate `--cached` pass would only re-list
+ * paths this one has, at the cost of another process spawn.
+ *
+ * @type {ReadonlyArray<ReadonlyArray<string>>}
+ */
+const CHANGE_PROBES = Object.freeze([
+  Object.freeze(['diff', '--name-only', '__BASE__...HEAD']),
+  Object.freeze(['diff', '--name-only', 'HEAD']),
+  Object.freeze(['ls-files', '--others', '--exclude-standard']),
+]);
+
+/**
+ * Resolve the set of repo-relative POSIX paths the ratchet still needs to
+ * score, or `null` when it must scan the whole tree.
+ *
+ * **Fails open, never closed.** Any git failure — a shallow clone, a missing
+ * base ref, a fixture directory that is not a repository at all — returns
+ * `null`, and the caller scans everything. Narrowing on an unreadable scope
+ * is how a ratchet silently stops ratcheting; paying for a whole-tree scan is
+ * not.
+ *
+ * @param {{
+ *   cwd: string,
+ *   baseRef: string,
+ *   baselineRows?: Array<{ file?: string }>,
+ *   git?: ReturnType<typeof createGitInterface>,
+ * }} args
+ * Module-private: `resolveScanScope` below is the only caller, and it is the
+ * one the CLI actually asks. Exporting both would ship a seam whose only
+ * importer is a test, which the `--production` dead-export pass discounts.
+ *
+ * @returns {Set<string> | null}
+ */
+function resolveCyclomaticScope({ cwd, baseRef, baselineRows = [], git }) {
+  const gitIface = git ?? createGitInterface({});
+  const scope = new Set();
+  for (const row of baselineRows) {
+    if (typeof row?.file === 'string' && row.file) scope.add(row.file);
+  }
+  for (const probe of CHANGE_PROBES) {
+    const argv = probe.map((arg) =>
+      arg === '__BASE__...HEAD' ? `${baseRef}...HEAD` : arg,
+    );
+    const res = gitIface.gitSpawn(cwd, ...argv);
+    if (res?.status !== 0) return null;
+    for (const line of String(res.stdout ?? '').split('\n')) {
+      const file = line.trim().replace(/\\/g, '/');
+      if (file) scope.add(file);
+    }
+  }
+  return scope;
+}
+
+/**
+ * Decide the scan scope for one `check-cyclomatic` invocation.
+ *
+ * `--update` rewrites the baseline from the current tree, and
+ * `BASELINE_SCOPE=full` is the operator's explicit "re-derive everything" —
+ * both must see the whole tree, so both return `null`.
+ *
+ * @param {{
+ *   cwd: string,
+ *   config?: { project?: { baseBranch?: string } } | null,
+ *   update?: boolean,
+ *   env?: NodeJS.ProcessEnv,
+ *   baselineRows?: Array<{ file?: string }>,
+ *   git?: ReturnType<typeof createGitInterface>,
+ * }} args
+ * @returns {Set<string> | null} `null` means "scan everything".
+ */
+export function resolveScanScope({
+  cwd,
+  config = null,
+  update = false,
+  env = process.env,
+  baselineRows = [],
+  git,
+}) {
+  if (update || env?.BASELINE_SCOPE === 'full') return null;
+  return resolveCyclomaticScope({
+    cwd,
+    baseRef: config?.project?.baseBranch ?? 'main',
+    baselineRows,
+    git,
+  });
+}
+
+/**
+ * Reduce a walked file list to the `{ abs, rel }` pairs that will actually be
+ * scored, applying the gate's `ignoreGlobs` and then the diff scope.
+ *
+ * Kept here rather than inside `scanCyclomatic` so "which files count" lives
+ * in one module: the ignore rule and the scope rule are the same decision
+ * asked twice, and splitting them across two files is how they drift.
+ *
+ * @param {string[]} files Absolute paths, as walked.
+ * @param {{ cwd: string, ignoreGlobs?: string[], scopeFiles?: Set<string> | null }} args
+ * @returns {Array<{ abs: string, rel: string }>}
+ */
+export function selectFilesToScore(
+  files,
+  { cwd, ignoreGlobs = [], scopeFiles = null },
+) {
+  const selected = [];
+  for (const abs of files) {
+    if (isIgnoredByGlobs(abs, ignoreGlobs, cwd)) continue;
+    const rel = path.relative(cwd, abs).split(path.sep).join('/');
+    if (scopeFiles && !scopeFiles.has(rel)) continue;
+    selected.push({ abs, rel });
+  }
+  return selected;
+}

package/.agents/scripts/lib/dependency-version.js

@@ -0,0 +1,86 @@
+/**
+ * dependency-version.js — read an installed package's version without
+ * evaluating the package (Story #5109).
+ *
+ * Baselines stamp the versions of the tools that produced them so a consumer
+ * can detect scorer drift. The obvious way to get one — `require(pkg).version`
+ * — pays the package's whole module-initialisation cost for a string that is
+ * sitting in a manifest a few kilobytes long. For `typescript` that was ~65 ms
+ * of CPU and ~75 MB of RSS on every `check-baselines --gate crap` and every
+ * `quality-preview` run, in a repository with no TypeScript in it at all.
+ *
+ * `require.resolve` walks the resolver without reading or executing the
+ * target, so everything here is path arithmetic plus one small `JSON.parse`.
+ *
+ * This deliberately does **not** replace `crap-utils.js#resolveEscomplexVersion`,
+ * which answers a different question: it walks `node_modules` upward from a
+ * caller-supplied `cwd` rather than resolving from this module's own location,
+ * because the escomplex stamp must describe the tree being scanned.
+ *
+ * @module lib/dependency-version
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Locate a package's manifest via the module resolver, without loading it.
+ *
+ * The `<pkg>/package.json` subpath is tried first because it is a single
+ * resolver call. A package whose `exports` map withholds `./package.json`
+ * falls back to resolving the main entry and walking up to the nearest
+ * `package.json`, which is still evaluation-free.
+ *
+ * @param {string} name Package name, e.g. `'typescript'`.
+ * @param {NodeJS.Require} requireFn A `createRequire`-produced require whose
+ *   resolution base is the calling module.
+ * @returns {string | null} Absolute manifest path, or `null` when unresolvable.
+ */
+function resolveManifestPath(name, requireFn) {
+  try {
+    return requireFn.resolve(`${name}/package.json`);
+  } catch {
+    // fall through to the main-entry walk
+  }
+  let dir;
+  try {
+    dir = path.dirname(requireFn.resolve(name));
+  } catch {
+    return null;
+  }
+  const { root } = path.parse(dir);
+  while (true) {
+    const candidate = path.join(dir, 'package.json');
+    if (fs.existsSync(candidate)) return candidate;
+    if (dir === root) return null;
+    const parent = path.dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+/**
+ * Resolve an installed package's declared version.
+ *
+ * Returns `null` — never a guess and never a throw — when the package cannot
+ * be resolved, its manifest cannot be read or parsed, or it declares no
+ * usable `version`. Callers map that onto their own sentinel (the baseline
+ * writers use `'0.0.0'`, meaning "unknown environment").
+ *
+ * @param {string} name Package name.
+ * @param {NodeJS.Require} requireFn A `createRequire`-produced require.
+ * @returns {string | null}
+ */
+export function resolveDependencyVersion(name, requireFn) {
+  const manifest = resolveManifestPath(name, requireFn);
+  if (!manifest) return null;
+  try {
+    const parsed = JSON.parse(fs.readFileSync(manifest, 'utf-8'));
+    if (parsed && typeof parsed.version === 'string' && parsed.version) {
+      return parsed.version;
+    }
+  } catch {
+    // unreadable / unparseable manifest
+  }
+  return null;
+}

package/.agents/scripts/lib/duplicate-search.js

@@ -207,7 +207,7 @@ function normalizeIssue(issue) {
  * }} args
  * @returns {Array<{ id: number, title: string, score: number, url: string }>}
  */
-export function rankOpenStoryDuplicates({
+function rankOpenStoryDuplicates({
   seed,
   openStories,
   minScore = DEFAULT_MIN_SCORE,

package/.agents/scripts/lib/errors/index.js

@@ -6,7 +6,7 @@
  * message substrings.
  */
 
-export class ConflictingTypeLabelsError extends Error {
+class ConflictingTypeLabelsError extends Error {
   constructor(message) {
     super(message);
     this.name = 'ConflictingTypeLabelsError';

package/.agents/scripts/lib/feedback-loop/graduator-core.js

@@ -60,7 +60,7 @@ import { upsertStructuredComment } from '../orchestration/ticketing.js';
  * finalize indefinitely; the walk now caps every spawn at this bound unless
  * a caller overrides it.
  */
-export const DEFAULT_RUN_CHILD_TIMEOUT_MS = 30000;
+const DEFAULT_RUN_CHILD_TIMEOUT_MS = 30000;
 
 /**
  * Default per-run filing cap. `graduate()` files at most this many follow-up
@@ -72,7 +72,7 @@ export const DEFAULT_MAX_FILINGS_PER_RUN = 20;
  * Structured-comment type used to durably persist cross-repo-deferred
  * findings on the Epic. Registered in `STRUCTURED_COMMENT_TYPES`.
  */
-export const CROSS_REPO_DEFERRED_COMMENT_TYPE = 'cross-repo-deferred';
+const CROSS_REPO_DEFERRED_COMMENT_TYPE = 'cross-repo-deferred';
 
 /**
  * Explicit, greppable opt-back-in to live issue filing from a context the

package/.agents/scripts/lib/feedback-loop/prior-feedback-fetcher.js

@@ -52,7 +52,7 @@ const FRICTION_LABEL_PREFIX = 'friction::';
  * @param {Array<{ number: number, labels?: string[] }>} issues
  * @returns {Array<{ class: string, count: number, issues: number[] }>}
  */
-export function extractRecurringDefectClasses(issues) {
+function extractRecurringDefectClasses(issues) {
   if (!Array.isArray(issues)) return [];
   /** @type {Map<string, Set<number>>} */
   const byClass = new Map();

package/.agents/scripts/lib/feedback-loop/retro-proposals-graduator.js

@@ -387,7 +387,7 @@ export function enrichRoutedProposalsWithFilings(routedProposals, filed) {
  * @param {string|null|undefined} slug
  * @returns {{ owner: string, repo: string } | null}
  */
-export function parseRepoSlug(slug) {
+function parseRepoSlug(slug) {
   if (typeof slug !== 'string') return null;
   const parts = slug.split('/');
   if (parts.length !== 2) return null;