mandrel 2.24.0 → 2.26.0

package/.agents/scripts/check-workflow-timeouts.js

@@ -0,0 +1,291 @@
+/**
+ * CLI: GitHub Actions job-timeout gate.
+ *
+ * Story #4936. Every job in this repository inherited GitHub's 360-minute
+ * default because not one of the four workflow files set `timeout-minutes`.
+ * That is not a theoretical cost: a deadlocked `Windows Smoke` job burned 44
+ * minutes of a `windows-latest` runner, and — because the run stayed
+ * `in_progress` — GitHub withheld the logs of the *required* check that had
+ * already failed on the same run, so the real merge blocker could not be
+ * diagnosed until the run was cancelled by hand. An unbounded job costs
+ * runner time and diagnosability at once.
+ *
+ * A ceiling only helps if it is bounded on both ends, so this gate fails on
+ * two shapes:
+ *
+ *   - a job with no `timeout-minutes` at all (unbounded, 360-minute default);
+ *   - a job whose `timeout-minutes` exceeds `MAX_TIMEOUT_MINUTES`, i.e. a
+ *     number generous enough to be useless as a deadlock detector.
+ *
+ * `MIN_TIMEOUT_MINUTES` is the floor a value must clear to be plausible
+ * headroom over a job that takes a couple of minutes today; below it, a
+ * transient slow runner reds the build for no defect.
+ *
+ * Contract:
+ *   - Scans the workflows directory (default `.github/workflows`, override
+ *     with `--dir <path>`), parsing each `*.yml` / `*.yaml` with js-yaml and
+ *     enumerating every `jobs.<id>` key.
+ *   - Prints `<file> job <id> — <reason>` per violation, then a one-line
+ *     summary even on a clean scan so the "ok" signal is visible in CI.
+ *   - With `--json`: writes a structured envelope to stdout and skips the
+ *     human summary.
+ *   - Exit codes: 0 = every job bounded in range; 1 = at least one violation.
+ *     A missing / empty workflows directory exits 0 (nothing to gate).
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import yaml from 'js-yaml';
+import { runAsCli } from './lib/cli-utils.js';
+
+/**
+ * Inclusive bounds every job's `timeout-minutes` must sit within.
+ *
+ * The ceiling is the load-bearing half — it is what turns a deadlock into a
+ * bounded loss instead of a six-hour one. The floor keeps a well-meaning
+ * `timeout-minutes: 2` from reddening the build on a slow runner.
+ */
+export const MIN_TIMEOUT_MINUTES = 10;
+export const MAX_TIMEOUT_MINUTES = 30;
+
+/**
+ * Enumerate workflow files (`*.yml` / `*.yaml`) directly under `dir`.
+ * Returns absolute paths sorted for deterministic output. A missing directory
+ * yields an empty list.
+ *
+ * @param {string} dir Absolute workflows directory.
+ * @param {typeof fs} [fsLike]
+ * @returns {string[]}
+ */
+export function listWorkflowFiles(dir, fsLike = fs) {
+  let entries;
+  try {
+    entries = fsLike.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((e) => e.isFile() && /\.ya?ml$/i.test(e.name))
+    .map((e) => path.join(dir, e.name))
+    .sort();
+}
+
+/**
+ * Read the `jobs` mapping out of one workflow document.
+ *
+ * The single parse point for both the violation scan and the job count, so
+ * the two can never disagree about what a workflow's job set is.
+ *
+ * @param {string} text The file contents.
+ * @param {(s: string) => unknown} parse YAML parser; injected in tests.
+ * @returns {{ jobs: Record<string, unknown> | null, error: Error | null }}
+ */
+function readJobs(text, parse) {
+  let doc;
+  try {
+    doc = parse(text);
+  } catch (err) {
+    return { jobs: null, error: err };
+  }
+  const jobs = doc && typeof doc === 'object' ? doc.jobs : null;
+  return {
+    jobs: jobs && typeof jobs === 'object' ? jobs : null,
+    error: null,
+  };
+}
+
+/**
+ * Judge one job body's `timeout-minutes`, returning the violation shape or
+ * `null` when the job is bounded in range.
+ *
+ * Reusable-workflow calls (`jobs.<id>.uses:`) are exempt: `timeout-minutes`
+ * is not a valid key on a job that delegates to another workflow — the
+ * callee's own jobs carry their timeouts, and setting one here is a schema
+ * error rather than a fix.
+ *
+ * @param {unknown} body The `jobs.<id>` value.
+ * @returns {{ timeout: number | null, reason: string } | null}
+ */
+function judgeJobTimeout(body) {
+  if (!body || typeof body !== 'object') return null;
+  if (typeof body.uses === 'string') return null;
+
+  const timeout = body['timeout-minutes'];
+  if (timeout === undefined || timeout === null) {
+    return {
+      timeout: null,
+      reason: `no timeout-minutes — the job inherits GitHub's 360-minute default`,
+    };
+  }
+  if (typeof timeout !== 'number' || !Number.isFinite(timeout)) {
+    return {
+      timeout: null,
+      reason: `timeout-minutes is not a number (${JSON.stringify(timeout)})`,
+    };
+  }
+  if (timeout > MAX_TIMEOUT_MINUTES) {
+    return {
+      timeout,
+      reason: `timeout-minutes ${timeout} exceeds the ${MAX_TIMEOUT_MINUTES}-minute ceiling`,
+    };
+  }
+  if (timeout < MIN_TIMEOUT_MINUTES) {
+    return {
+      timeout,
+      reason: `timeout-minutes ${timeout} is below the ${MIN_TIMEOUT_MINUTES}-minute floor`,
+    };
+  }
+  return null;
+}
+
+/**
+ * Pure helper: enumerate every `jobs.<id>` key in one workflow document and
+ * return the violations.
+ *
+ * An unparseable document is itself a violation: a workflow whose jobs cannot
+ * be enumerated cannot be certified as bounded, and failing open would let
+ * the very drift this gate exists to catch through.
+ *
+ * @param {string} file Relative file label used in violation rows.
+ * @param {string} text The file contents.
+ * @param {(s: string) => unknown} [parse] YAML parser; injected in tests.
+ * @returns {Array<{ file: string, job: string, timeout: number | null, reason: string }>}
+ */
+export function scanWorkflowText(file, text, parse = yaml.load) {
+  const { jobs, error } = readJobs(text, parse);
+  if (error) {
+    return [
+      {
+        file,
+        job: '(document)',
+        timeout: null,
+        reason: `workflow could not be parsed as YAML: ${error.message}`,
+      },
+    ];
+  }
+  if (!jobs) return [];
+
+  const violations = [];
+  for (const [job, body] of Object.entries(jobs)) {
+    const verdict = judgeJobTimeout(body);
+    if (verdict) violations.push({ file, job, ...verdict });
+  }
+  return violations;
+}
+
+/**
+ * Pure helper: render the human-readable report. One line per violation
+ * followed by a one-line summary carrying a `(gate fail)` / `(ok)` marker.
+ *
+ * @param {Array<{ file: string, job: string, timeout: number | null, reason: string }>} violations
+ * @param {number} jobsScanned
+ * @returns {string}
+ */
+export function renderReport(violations, jobsScanned) {
+  const lines = violations.map((v) => `${v.file} job ${v.job} — ${v.reason}`);
+  const tag = violations.length > 0 ? '(gate fail)' : '(ok)';
+  lines.push(
+    `[workflow-timeouts] jobs=${jobsScanned} violations=${violations.length} ${tag}`,
+  );
+  return lines.join('\n');
+}
+
+/**
+ * Count every `jobs.<id>` key in a parsed workflow document, including the
+ * reusable-workflow calls the violation scan exempts — the summary reports
+ * how many jobs were enumerated, not how many were eligible.
+ *
+ * @param {string} text
+ * @param {(s: string) => unknown} [parse]
+ * @returns {number}
+ */
+export function countJobs(text, parse = yaml.load) {
+  const { jobs } = readJobs(text, parse);
+  return jobs ? Object.keys(jobs).length : 0;
+}
+
+/**
+ * Top-level CLI entry. Exported so tests can drive the full pipeline against
+ * a fixture workflows directory without touching the repo's real workflows.
+ *
+ * @param {{
+ *   argv?: string[],
+ *   cwd?: string,
+ *   stdout?: { write: (s: string) => void },
+ *   stderr?: { write: (s: string) => void },
+ * }} [opts]
+ * @returns {Promise<number>} exit code: 0 = every job bounded; 1 = violation
+ */
+export async function runCli({
+  argv = process.argv.slice(2),
+  cwd = process.cwd(),
+  stdout = process.stdout,
+  stderr = process.stderr,
+} = {}) {
+  const dirIdx = argv.indexOf('--dir');
+  const dir =
+    dirIdx !== -1 && argv[dirIdx + 1] && !argv[dirIdx + 1].startsWith('--')
+      ? argv[dirIdx + 1]
+      : null;
+  const json = argv.includes('--json');
+  const resolvedDir = path.resolve(
+    cwd,
+    dir ?? path.join('.github', 'workflows'),
+  );
+
+  const files = listWorkflowFiles(resolvedDir);
+  const violations = [];
+  let jobsScanned = 0;
+  for (const file of files) {
+    let text;
+    try {
+      text = fs.readFileSync(file, 'utf-8');
+    } catch {
+      continue;
+    }
+    const label = path.relative(cwd, file);
+    jobsScanned += countJobs(text);
+    violations.push(...scanWorkflowText(label, text));
+  }
+
+  const exitCode = violations.length > 0 ? 1 : 0;
+
+  if (json) {
+    stdout.write(
+      `${JSON.stringify(
+        {
+          kind: 'workflow-timeouts-report',
+          dir: resolvedDir,
+          filesScanned: files.length,
+          jobsScanned,
+          minTimeoutMinutes: MIN_TIMEOUT_MINUTES,
+          maxTimeoutMinutes: MAX_TIMEOUT_MINUTES,
+          violations,
+          exitCode,
+        },
+        null,
+        2,
+      )}\n`,
+    );
+  } else {
+    if (files.length === 0) {
+      stderr.write(
+        `[workflow-timeouts] ⚠ no workflow files found under ${resolvedDir}\n`,
+      );
+    }
+    stdout.write(`${renderReport(violations, jobsScanned)}\n`);
+  }
+
+  return exitCode;
+}
+
+async function main() {
+  return runCli();
+}
+
+runAsCli(import.meta.url, main, {
+  source: 'workflow-timeouts',
+  propagateExitCode: true,
+  errorPrefix: '[workflow-timeouts] ❌ Fatal error',
+});

package/.agents/scripts/cleanup-repo-test-temp.js

@@ -3,9 +3,9 @@
 
 /**
  * Removes **test-reserved** per-Epic trees under `<repo>/temp/epic-<id>/` after
- * test runs. Only Epic IDs in the band defined by
- * `lib/test-reserved-epic-temp-ids.js` are deleted (999000–999999). Real Epic
- * scratch (e.g. `temp/epic-1143/`) is never touched.
+ * test runs. Only Epic IDs in the band declared by `lib/reserved-test-ids.js`
+ * are deleted (999000–999999). Real Epic scratch (e.g. `temp/epic-1143/`) is
+ * never touched.
  *
  * Keeps sibling dirs like `temp/epic-runner-logs/`, loose files, and
  * `temp/epic-<id>/` when `<id>` is outside that reserved band.
@@ -20,11 +20,13 @@
 import { existsSync, readdirSync, rmSync } from 'node:fs';
 import path from 'node:path';
 import { runAsCli } from './lib/cli-utils.js';
+import { isReservedTestId } from './lib/reserved-test-ids.js';
 
-const RESERVED_TEST_EPIC_TEMP_DIR = /^epic-999\d{3}$/;
+const EPIC_TEMP_DIR = /^epic-(\d+)$/;
 
 function isReservedTestEpicTempDirName(name) {
-  return RESERVED_TEST_EPIC_TEMP_DIR.test(name);
+  const match = EPIC_TEMP_DIR.exec(name);
+  return match !== null && isReservedTestId(Number(match[1]));
 }
 
 /**

package/.agents/scripts/deliver-light.js

@@ -337,10 +337,9 @@ async function runBackstopMode(values, deps = {}) {
     process.stderr.write(HELP);
     throw new Error('[deliver-light] --backstop requires --story <id>');
   }
-  const { result, nextCommand, exitCode, message } = await resolveFn({
-    storyId,
-  });
-  const extra = nextCommand === null ? {} : { nextCommand };
+  const { result, nextCommand, preservation, exitCode, message } =
+    await resolveFn({ storyId });
+  const extra = nextCommand === null ? {} : { nextCommand, preservation };
   emit({ mode: 'backstop', storyId, ...result, ...extra }, values.pretty);
   if (result.blocked) Logger.warn(message);
   else Logger.info(message);

package/.agents/scripts/deliver-recover.js

@@ -16,6 +16,14 @@
  * `single-story-init.js` hard-errors on an already-closed Story. Before this
  * surface, that Story had no automated way back.
  *
+ * Every command it prints **resumes** — it re-enters the in-flight worker or
+ * close, or closes the branch that is already pushed. None of them re-dispatch
+ * the Story, and that is deliberate: since Story #4876 the close-and-land tail
+ * belongs to the dispatching orchestrator, so a worker returning without a
+ * terminal envelope is the expected shape rather than evidence the Story never
+ * ran. Answering it with a fresh dispatch re-runs `single-story-init.js`
+ * underneath live work and puts a second close on one PR.
+ *
  * Usage:
  *   node .agents/scripts/deliver-recover.js --story <STORY_ID> [--cwd <main-repo>]
  *                                           [--json]
@@ -46,6 +54,11 @@ Probes a Story's live delivery state — labels, lease, branch, worktree, PR
 state and checks — and prints the single next command that resumes it, with
 the evidence it was derived from. Read-only: mutates nothing.
 
+The command always resumes the existing worker or close (or closes an already
+pushed branch); it never re-dispatches the Story, which would re-run init
+underneath live work. A worker that returned no terminal envelope is expected —
+the orchestrator owns the close-and-land tail.
+
 Mid-flight shapes (executing-*/closing-*) get a stability re-probe after a
 short settle window: matching shapes return the fresher verdict; diverging
 shapes report \`in-transition\` (a live delivery process is mutating the

package/.agents/scripts/diagnose-friction.js

@@ -16,7 +16,11 @@
  *
  * Usage:
  *   node diagnose-friction.js [--story <STORY_ID>] \
- *     [--epic <EPIC_ID>] --cmd <command with args...>
+ *     [--epic <EPIC_ID>] --cmd <cmd> <args...>
+ *
+ * `--cmd` consumes the remaining argv as separate words and spawns them with
+ * no shell. Quoting the whole command as one string is a usage error, not
+ * friction — it is refused loudly and writes no ledger row.
  *
  * Story/Epic resolution order:
  *   1. CLI flags (--story, --epic).
@@ -108,6 +112,16 @@ function classifyFrictionCategory(errorOutput) {
  */
 const INTERCEPTOR_TIMEOUT_SIGNAL = 'SIGTERM';
 
+/**
+ * A `maxBuffer` overflow presents identically to a timeout — `status: null`,
+ * `signal: 'SIGTERM'` — because Node kills the child the same way. The only
+ * discriminator is `result.error.code`, so a SIGTERM carrying this code is a
+ * buffer overflow and must never be reported as a timeout (Story #4915).
+ *
+ * @type {string}
+ */
+const OVERFLOW_ERROR_CODE = 'ENOBUFS';
+
 /** Shell convention for "the process died by signal N": exit `128 + N`. */
 const SIGNAL_EXIT_BASE = 128;
 
@@ -118,8 +132,10 @@ const SIGNAL_EXIT_BASE = 128;
  * interceptor would report success for a command it just watched get killed
  * (Story #4851).
  *
- * Which signal fired is the diagnostic value: SIGTERM points at the
- * interceptor's own bound, anything else at the host. Recording that plus the
+ * Which signal fired is the diagnostic value: SIGTERM points at one of the
+ * interceptor's own bounds, anything else at the host. A SIGTERM splits again
+ * on `error.code`: `ENOBUFS` means the output blew past `executionMaxBuffer`,
+ * anything else means `executionTimeoutMs` fired. Recording that plus the
  * bound itself is what makes the row actionable to a consumer who cannot edit
  * the materialized framework tree.
  *
@@ -128,13 +144,17 @@ const SIGNAL_EXIT_BASE = 128;
  * the file's per-file maintainability-delta headroom. The CLI contract is the
  * seam the unit tests drive.
  *
- * @param {{signal: (string|null), error?: {message?: string}}} result A
- *   `spawnSync` result whose `status` is `null`.
- * @param {number} executionTimeoutMs The resolved interceptor bound, in ms.
+ * @param {{signal: (string|null), error?: {message?: string, code?: string}}}
+ *   result A `spawnSync` result whose `status` is `null`.
+ * @param {{executionTimeoutMs: number, executionMaxBuffer: number}} bounds The
+ *   resolved interceptor bounds — the timeout in ms, the buffer in bytes.
  * @returns {{category: string, remediation: string, details: object,
  *   preview: string, exitCode: number}}
  */
-function describeAbnormalExit(result, executionTimeoutMs) {
+function describeAbnormalExit(
+  result,
+  { executionTimeoutMs, executionMaxBuffer },
+) {
   const signal = typeof result.signal === 'string' ? result.signal : null;
   if (signal === null) {
     return {
@@ -150,15 +170,42 @@ function describeAbnormalExit(result, executionTimeoutMs) {
     };
   }
 
-  const timedOut = signal === INTERCEPTOR_TIMEOUT_SIGNAL;
-  const killOrigin = timedOut ? 'interceptor-timeout' : 'external';
+  const sentByInterceptor = signal === INTERCEPTOR_TIMEOUT_SIGNAL;
+  const overflowed =
+    sentByInterceptor && result.error?.code === OVERFLOW_ERROR_CODE;
+  let killOrigin = 'external';
+  if (overflowed) killOrigin = 'buffer-overflow';
+  else if (sentByInterceptor) killOrigin = 'interceptor-timeout';
+
+  const shapes = {
+    'buffer-overflow': {
+      category: 'Tool Limitation',
+      remediation: ` - ${signal} was sent because the command's output overflowed the interceptor's executionMaxBuffer bound (${executionMaxBuffer} bytes / 10 MiB) — the executionTimeoutMs bound (${executionTimeoutMs}ms) did not fire. Do NOT split the command into smaller steps: quieten or redirect its output, or raise the buffer bound.`,
+      extraDetails: { executionMaxBuffer },
+    },
+    'interceptor-timeout': {
+      category: 'Execution Timeout',
+      remediation: ` - ${signal} matches the interceptor's own executionTimeoutMs bound (${executionTimeoutMs}ms), so the command was almost certainly cut off rather than broken. Split it into smaller steps, or raise the bound.`,
+      extraDetails: {},
+    },
+    external: {
+      category: 'Execution Killed',
+      remediation: ` - ${signal} originated outside the interceptor — the executionTimeoutMs bound (${executionTimeoutMs}ms) did not fire, so suspect an OOM kill or a hard kill from the host. Reduce the command's memory footprint or give the host more headroom.`,
+      extraDetails: {},
+    },
+  };
+
+  const shape = shapes[killOrigin];
   const signum = osConstants.signals[signal];
   return {
-    category: timedOut ? 'Execution Timeout' : 'Execution Killed',
-    remediation: timedOut
-      ? ` - ${signal} matches the interceptor's own executionTimeoutMs bound (${executionTimeoutMs}ms), so the command was almost certainly cut off rather than broken. Split it into smaller steps, or raise the bound.`
-      : ` - ${signal} originated outside the interceptor — the executionTimeoutMs bound (${executionTimeoutMs}ms) did not fire, so suspect an OOM kill or a hard kill from the host. Reduce the command's memory footprint or give the host more headroom.`,
-    details: { killedBySignal: signal, killOrigin, executionTimeoutMs },
+    category: shape.category,
+    remediation: shape.remediation,
+    details: {
+      killedBySignal: signal,
+      killOrigin,
+      executionTimeoutMs,
+      ...shape.extraDetails,
+    },
     preview: `Command terminated by signal ${signal} (${killOrigin}); executionTimeoutMs=${executionTimeoutMs}.`,
     exitCode: Number.isInteger(signum) ? SIGNAL_EXIT_BASE + signum : 1,
   };
@@ -221,7 +268,23 @@ export async function main(args = process.argv.slice(2)) {
 
   if (cmdArgs.length === 0) {
     throw new Error(
-      'Usage: node diagnose-friction.js [--story <STORY_ID>] [--epic <EPIC_ID>] --cmd <command with args...>',
+      'Usage: node diagnose-friction.js [--story <STORY_ID>] [--epic <EPIC_ID>] --cmd <cmd> <args...>',
+    );
+  }
+
+  // Story #4915 — the interceptor spawns `cmdArgs[0]` directly, with no shell.
+  // A single argument containing whitespace therefore names an executable that
+  // cannot exist, and the resulting ENOENT is a usage error in the
+  // interceptor's OWN invocation, not friction in the wrapped command. It must
+  // be reported as one and MUST NOT reach the ledger — otherwise the real
+  // friction is discarded and the roll-up eventually auto-files a framework-gap
+  // ticket about the framework's own instrumentation being misused. The
+  // discriminator is this argv shape, never the ENOENT result: a correctly
+  // split command whose binary is genuinely absent yields the identical
+  // `spawnSync` result and stays real friction.
+  if (cmdArgs.length === 1 && /\s/.test(cmdArgs[0])) {
+    throw new Error(
+      `Usage: --cmd takes the command as separate argv words, not one quoted string. Received a single quoted argument: "${cmdArgs[0]}". Drop the quotes so each word is its own argv entry — \`--cmd ${cmdArgs[0]}\`. No friction signal was recorded.`,
     );
   }
 
@@ -251,7 +314,10 @@ export async function main(args = process.argv.slice(2)) {
     // `result.signal`, not in the status.
     const abnormal =
       result.status === null
-        ? describeAbnormalExit(result, executionTimeoutMs)
+        ? describeAbnormalExit(result, {
+            executionTimeoutMs,
+            executionMaxBuffer,
+          })
         : null;
     // With both streams empty an abnormal termination names its signal; the
     // `Unknown exit code` fallback is therefore reachable only with a real
@@ -343,15 +409,15 @@ runAsCli(import.meta.url, main, {
   source: 'DiagnoseFriction',
   usage: {
     invocation:
-      'node .agents/scripts/diagnose-friction.js [--story <id>] [--epic <id>] --cmd <command with args...>',
+      'node .agents/scripts/diagnose-friction.js [--story <id>] [--epic <id>] --cmd <cmd> <args...>',
     summary:
       'Run a command through the diagnostic interceptor: stream its output, then append a local friction signal describing the failure. Never posts to the ticket.',
     flags: [
       ['--story <id>', 'Story the friction belongs to.'],
       ['--epic <id>', 'Epic the friction belongs to.'],
       [
-        '--cmd <command...>',
-        'The command to execute; everything after it is the argv (required).',
+        '--cmd <cmd> <args...>',
+        'The command to execute; everything after it is the argv, as separate words — never one quoted string (required).',
       ],
     ],
   },