forgedock 1.19.0 → 1.21.0

package/scripts/build-knowledge-index.mjs

@@ -173,12 +173,14 @@ function readForgeYaml(repoPath) {
   const yamlPath = join(repoPath, 'forge.yaml');
   if (!existsSync(yamlPath)) return null;
   try {
-    const result = spawnSync('yq', ['-r', '.project.owner + "/" + .project.repo', yamlPath], {
+    const result = spawnSync('yq', ['-r', '.project.owner, .project.repo', yamlPath], {
       encoding: 'utf8',
       timeout: 5000,
     });
-    if (result.status === 0 && result.stdout.trim()) {
-      return result.stdout.trim();
+    if (result.status === 0) {
+      const [owner, name] = result.stdout.split('\n').map(l => l.trim());
+      // Both halves must be real values: a partial forge.yaml must not yield "/" or "null/x"
+      if (owner && name && owner !== 'null' && name !== 'null') return `${owner}/${name}`;
     }
   } catch (_) {
     // ignore
@@ -1576,7 +1578,7 @@ async function main() {
     repo = readForgeYaml(repoPath);
   }
   if (!repo) {
-    process.stderr.write('[ledger] ERROR: Cannot determine GitHub repo. Pass --repo owner/repo or ensure forge.yaml is present.\n');
+    process.stderr.write('[ledger] ERROR: Cannot determine GitHub repo. Pass --repo owner/repo or ensure forge.yaml has project.owner and project.repo (yq required).\n');
     process.exit(1);
   }
 
@@ -1760,6 +1762,7 @@ async function main() {
 }
 
 main().catch(e => {
-  process.stderr.write(`[ledger] FATAL: ${e.message}\n${e.stack}\n`);
+  const showStack = process.argv.includes('--verbose') || !!process.env.FORGE_DEBUG;
+  process.stderr.write(`[ledger] FATAL: ${e && e.message ? e.message : e}\n${showStack && e && e.stack ? e.stack + '\n' : ''}`);
   process.exit(1);
 });

package/scripts/builder-resume-match.test.sh

@@ -14,7 +14,7 @@ ok()  { PASS=$((PASS+1)); echo "ok   - $1"; }
 bad() { FAILN=$((FAILN+1)); echo "FAIL - $1"; }
 eq()  { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (got '$2', want '$3')"; fi; }
 
-SELECT_JQ='[.[][] | select((.body // "") | startswith("<!-- FORGE:BUILDER -->")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | {id: .id, complete: (.body | contains("<!-- FORGE:BUILDER:COMPLETE -->")), other: (.body | test("<!-- FORGE:(INVESTIGATOR|CONTEXT|CONTRACT|ARCHITECT)"))}]'
+SELECT_JQ='[.[][] | select((.body // "") | startswith("<!-- FORGE:BUILDER -->")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | {id: .id, complete: (.body | contains("<!-- FORGE:BUILDER:COMPLETE -->")), other: (.body | test("\\n[ \\t]*<!-- FORGE:(INVESTIGATOR|CONTEXT|CONTRACT|ARCHITECT)"))}]'
 DELETE_JQ='[.[] | select(.complete | not)] | if length == 1 and (.[0].other | not) then .[0].id else empty end'
 
 # c <id> <assoc> <type> <body>
@@ -27,7 +27,7 @@ PLAN=$'<!-- FORGE:BUILDER -->\n## Implementation\n\n<!-- FORGE:BUILDER:COMPLETE
 PARTIAL=$'<!-- FORGE:BUILDER -->\n## Implementation (in progress)'
 INVEST=$'<!-- FORGE:INVESTIGATOR -->\nThe build phase later posts FORGE:BUILDER and FORGE:BUILDER:COMPLETE.'
 CTX=$'<!-- FORGE:CONTEXT -->\nmentions <!-- FORGE:BUILDER -->'
-MIXED=$'<!-- FORGE:BUILDER -->\nquoted <!-- FORGE:INVESTIGATOR -->'
+MIXED=$'<!-- FORGE:BUILDER -->\nquoted:\n<!-- FORGE:INVESTIGATOR -->'
 
 eq "INVESTIGATOR mentioning FORGE:BUILDER is not selected" "$(arch "$(c 1 NONE Bot "$INVEST")" | jq length)" 0
 eq "INVESTIGATOR mentioning FORGE:BUILDER is never deleted" "$(del_id "$(c 1 NONE Bot "$INVEST")")" ""
@@ -38,7 +38,7 @@ eq "single partial comment -> delete its id" "$(del_id "$(c 4 NONE Bot "$PARTIAL
 eq "two partial comments -> delete nothing" "$(del_id "$(c 4 NONE Bot "$PARTIAL")"$'\n'"$(c 5 NONE Bot "$PARTIAL")")" ""
 eq "untrusted author partial ignored" "$(del_id "$(c 6 NONE User "$PARTIAL")")" ""
 eq "untrusted author complete ignored" "$(n_complete "$(c 6 NONE User "$PLAN")")" 0
-eq "partial carrying another FORGE marker is not deleted" "$(del_id "$(c 7 NONE Bot "$MIXED")")" ""
+eq "partial with another FORGE marker at a line start is not deleted" "$(del_id "$(c 7 NONE Bot "$MIXED")")" ""
 eq "investigator + partial comment -> only the comment id" "$(del_id "$(c 1 NONE Bot "$INVEST")"$'\n'"$(c 8 NONE Bot "$PARTIAL")")" 8
 eq "paginated concatenated arrays" "$(printf '[%s]\n[%s]' "$(c 1 NONE Bot "$INVEST")" "$(c 9 NONE Bot "$PLAN")" | jq -s "$SELECT_JQ" | jq '[.[] | select(.complete)] | length')" 1
 
@@ -46,6 +46,15 @@ ARCHC=$'<!-- FORGE:ARCHITECT -->\nplan; the builder will post <!-- FORGE:BUILDER
 eq "ARCHITECT mentioning the marker is not selected" "$(arch "$(c 10 NONE Bot "$ARCHC")" | jq length)" 0
 eq "unparsable input fails closed" "$(printf 'not json' | jq -s "$SELECT_JQ" 2>/dev/null | jq -r "$DELETE_JQ" 2>/dev/null)" ""
 
+INLINE=$'<!-- FORGE:BUILDER -->\n## Approach\nFollowed the plan in `<!-- FORGE:ARCHITECT -->` and <!-- FORGE:INVESTIGATOR --> notes.'
+CRLF=$'<!-- FORGE:BUILDER -->\r\n<!-- FORGE:ARCHITECT -->\r\nbody'
+eq "partial quoting a marker inline is picked and deleted e1_inline_picked" "$(del_id "$(c 11 NONE Bot "$INLINE")")" 11
+eq "inline quoting partial is selected as non-other e1_inline_not_other" "$(arch "$(c 11 NONE Bot "$INLINE")" | jq -r '.[0].other')" false
+eq "marker at the start of a later line is refused e2_line_start_refused" "$(arch "$(c 12 NONE Bot "$MIXED")" | jq -r '.[0].other')" true
+eq "own first-line marker never counts as other e4_own_marker_not_other" "$(arch "$(c 13 NONE Bot "$PARTIAL")" | jq -r '.[0].other')" false
+eq "two inline-quoting partials still delete nothing e3_two_partials_none" "$(del_id "$(c 11 NONE Bot "$INLINE")"$'\n'"$(c 14 NONE Bot "$INLINE")")" ""
+eq "CRLF body with a marker line is still refused e5_crlf_refused" "$(del_id "$(c 15 NONE Bot "$CRLF")")" ""
+
 # Spec drift guard: every mutating site carries this exact predicate and no unanchored selector.
 for f in commands/work-on/build/implement.md commands/work-on/build.md commands/work-on/build/validate.md; do
   SPEC="$ROOT/$f"

package/scripts/calibration.mjs

@@ -143,10 +143,13 @@ export function readForgeYaml(repoPath) {
   const configPath = join(repoPath, 'forge.yaml');
   if (!existsSync(configPath)) return null;
   try {
-    const result = spawnSync('yq', ['-r', '.project.owner + "/" + .project.repo', configPath], {
+    const result = spawnSync('yq', ['-r', '.project.owner, .project.repo', configPath], {
       encoding: 'utf8', timeout: 5000,
     });
-    if (result.status === 0 && result.stdout.trim()) return result.stdout.trim();
+    if (result.status === 0) {
+      const [owner, name] = result.stdout.split('\n').map(l => l.trim());
+      if (owner && name && owner !== 'null' && name !== 'null') return `${owner}/${name}`;
+    }
   } catch (_) { /* ignore */ }
   return null;
 }

package/scripts/check-spec-markers.sh

@@ -177,6 +177,8 @@ MARKER_REGISTRY="
   REVIEW_TIER
   REVIEW_STARTED
   SECURITY_AUDIT
+  SELFREVIEW
+  SELFREVIEW_REPAIR
   SIGNAL_RESOLVED
   SIGNAL_UNRESOLVED
   SIZE_OVERRIDE

package/scripts/ci-local/lib.mjs

@@ -29,10 +29,18 @@ export function commandExists(name) {
   return r.code === 0;
 }
 
+// Memoized per process: stepKey runs for every step (twice: cost ordering and execution), and an unmemoized
+// `<tool> --version` is a synchronous spawn that blocks the scheduler and burns the budget (#4068 review CONC-1).
+const TOOL_VERSIONS = new Map();
 export function toolVersion(name) {
-  if (!/^[A-Za-z0-9._+-]+$/.test(name) || !commandExists(name)) return '';
-  const r = run(name, ['--version'], { timeout: 5000 });
-  return (r.out || r.err).split('\n')[0].slice(0, 120);
+  if (TOOL_VERSIONS.has(name)) return TOOL_VERSIONS.get(name);
+  let v = '';
+  if (/^[A-Za-z0-9._+-]+$/.test(name) && commandExists(name)) {
+    const r = run(name, ['--version'], { timeout: 5000 });
+    v = (r.out || r.err).split('\n')[0].slice(0, 120);
+  }
+  TOOL_VERSIONS.set(name, v);
+  return v;
 }
 
 export function hostClass() {

package/scripts/ci-local/run.mjs

@@ -29,30 +29,57 @@ function slotCount() {
 }
 function alive(pid) { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } }
 
-async function acquireSlot(owner, deadline) {
+let ticketSeq = 0;
+// Is this slot dir free to take (absent, or held by a dead pid / for over 3 h)? A stale one is reaped.
+function slotFree(slot) {
+  if (!fs.existsSync(slot)) return true;
+  let pid = 0; let ts = 0;
+  try { const [p, t] = fs.readFileSync(path.join(slot, 'owner'), 'utf8').split(' '); pid = Number(p); ts = Number(t); } catch { /* owner file not written yet */ }
+  const stale = pid ? !alive(pid) : (Date.now() - (fs.statSync(slot, { throwIfNoEntry: false })?.mtimeMs || 0) > 30000);
+  if (stale || (ts && Date.now() - ts > 3 * 3600 * 1000)) { try { fs.rmSync(slot, { recursive: true, force: true }); } catch { /* raced */ } return !fs.existsSync(slot); }
+  return false;
+}
+
+// forge#4057: FIFO-fair slot acquisition. Every waiter files a ticket (`<ms>-<seq>-<pid>-<rand>`) under ci-local.lock/queue and
+// only a ticket among the first N (N = number of free slots) may claim one, so an invocation can never be starved by polling
+// races. A ticket whose pid is dead is reaped; the ticket is removed on acquire, on a deadline miss and on every exit path.
+export async function acquireSlot(owner, deadline) {
   const dir = path.join(homeForgeDir(), 'ci-local.lock');
-  fs.mkdirSync(dir, { recursive: true });
+  const qdir = path.join(dir, 'queue');
+  fs.mkdirSync(qdir, { recursive: true });
   const n = slotCount();
-  for (;;) {
-    for (let i = 0; i < n; i++) {
-      const slot = path.join(dir, `slot-${i}`);
-      try {
-        fs.mkdirSync(slot);
-        fs.writeFileSync(path.join(slot, 'owner'), `${process.pid} ${Date.now()} ${owner}\n`);
-        const release = () => { try { fs.rmSync(slot, { recursive: true, force: true }); } catch { /* gone */ } cleanups.delete(release); };
-        cleanups.add(release);
-        return release;
-      } catch (e) {
-        if (e.code !== 'EEXIST') throw e;
-        let pid = 0; let ts = 0;
-        try { const [p, t] = fs.readFileSync(path.join(slot, 'owner'), 'utf8').split(' '); pid = Number(p); ts = Number(t); } catch { /* owner file not written yet */ }
-        const stale = pid ? !alive(pid) : (Date.now() - (fs.statSync(slot, { throwIfNoEntry: false })?.mtimeMs || 0) > 30000);
-        if (stale || (ts && Date.now() - ts > 3 * 3600 * 1000)) { try { fs.rmSync(slot, { recursive: true, force: true }); } catch { /* raced */ } i--; continue; }
+  const ticket = `${String(Date.now()).padStart(14, '0')}-${String(++ticketSeq).padStart(6, '0')}-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
+  const tpath = path.join(qdir, ticket);
+  fs.writeFileSync(tpath, `${process.pid} ${owner}\n`, { flag: 'wx' });
+  const dropTicket = () => { try { fs.rmSync(tpath, { force: true }); } catch { /* gone */ } cleanups.delete(dropTicket); };
+  cleanups.add(dropTicket);
+  try {
+    for (;;) {
+      const free = [];
+      for (let i = 0; i < n; i++) if (slotFree(path.join(dir, `slot-${i}`))) free.push(i);
+      const live = [];
+      for (const name of fs.readdirSync(qdir).sort()) {
+        if (name !== ticket && !alive(Number(name.split('-')[2]))) { try { fs.rmSync(path.join(qdir, name), { force: true }); } catch { /* raced */ } continue; }
+        live.push(name);
+      }
+      const pos = live.indexOf(ticket);
+      if (pos === -1) { try { fs.writeFileSync(tpath, `${process.pid} ${owner}\n`, { flag: 'wx' }); } catch { /* re-filed by a racing reaper */ } }
+      else if (pos < free.length) {
+        for (const i of free) {
+          const slot = path.join(dir, `slot-${i}`);
+          try {
+            fs.mkdirSync(slot);
+            fs.writeFileSync(path.join(slot, 'owner'), `${process.pid} ${Date.now()} ${owner}\n`);
+            const release = () => { try { fs.rmSync(slot, { recursive: true, force: true }); } catch { /* gone */ } cleanups.delete(release); };
+            cleanups.add(release);
+            return release;
+          } catch (e) { if (e.code !== 'EEXIST') throw e; }
+        }
       }
+      if (Date.now() > deadline) return null;
+      await sleep(250);
     }
-    if (Date.now() > deadline) return null;
-    await sleep(250);
-  }
+  } finally { dropTicket(); }
 }
 
 // ---- process execution ----
@@ -153,6 +180,16 @@ function changeHash(ctx) {
   if (!ctx.changeHash) ctx.changeHash = sha256(JSON.stringify([ctx.plan.merge_base, ctx.plan.changed_files.map((f) => `${f}:${fileHash(ctx.repo, f)}`)]));
   return ctx.changeHash;
 }
+// forge#4060: service containers are an execution-environment input; they must enter every key derived from stepKey
+// (single-flight, persisted result cache, cost probe). Order-insensitive; absent, undefined and [] fingerprint identically.
+function servicesFp(job) {
+  const svcs = Array.isArray(job.services) ? job.services : [];
+  return JSON.stringify(svcs.map((s) => {
+    const v = s || {};
+    const env = v.env || {};
+    return [v.name, v.image, Object.keys(env).sort().map((k) => `${k}=${env[k]}`), (v.ports || []).map(String), v.options || ''];
+  }).sort((a, b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : 1));
+}
 function stepKey(step, job, ctx) {
   const sel = step.scope || {};
   const first = (shellSplit(step.command) || [])[0] || '';
@@ -161,7 +198,28 @@ function stepKey(step, job, ctx) {
     ? { files: sel.files.map((f) => `${f}:${fileHash(ctx.repo, f)}`), config: configFilesHash(ctx) }
     : { change: changeHash(ctx), selected: sel.files || sel.selector || '' };
   const env = { ...job.env, ...step.env };
-  return sha256(JSON.stringify([step.command, step.cwd, step.shell, Object.keys(env).sort().map((k) => `${k}=${env[k]}`), toolVersion(path.basename(first)), process.version, inputs]));
+  return sha256(JSON.stringify([step.command, step.cwd, step.shell, Object.keys(env).sort().map((k) => `${k}=${env[k]}`), toolVersion(path.basename(first)), process.version, inputs, servicesFp(job)]));
+}
+
+function durKey(step) { return sha256(`${step.command}\0${step.cwd || ''}`).slice(0, 40); }
+
+// forge#4057: cost hint for ordering. wide = some step runs unscoped or widened; est = expected execution ms (a cached step
+// costs 0, otherwise the last recorded duration of that command, else a default). Cheap and narrow jobs go first so a budget
+// hit leaves the most coverage proven.
+function jobCost(job, ctx) {
+  let wide = 0; let est = 0;
+  for (const step of job.steps) {
+    if (step.noop || step.skip) continue;
+    if (step.builtin) continue;
+    if (!step.scope || step.scope.mode === 'full' || step.scope.widen_reason) wide = 1;
+    try {
+      if (step.kind !== 'install' && !ctx.noResultCache && fs.existsSync(path.join(cacheDir(ctx.o), stepKey(step, job, ctx) + '.json'))) continue;
+    } catch { /* unknown cost */ }
+    let d = 60000;
+    try { const v = Number(fs.readFileSync(path.join(cacheDir(ctx.o), 'dur', durKey(step)), 'utf8')); if (v >= 0) d = v; } catch { /* no history */ }
+    est += d;
+  }
+  return [wide, est];
 }
 
 // ---- one job ----
@@ -184,24 +242,30 @@ function absorbGhFiles(ghFiles, env) {
 }
 
 async function runJob(job, ctx) {
-  const t0 = Date.now(); const res = { id: job.id, name: job.name, source: job.source, status: 'pass', steps: [], reason: '', duration_ms: 0 };
+  const t0 = Date.now(); const res = { id: job.id, name: job.name, source: job.source, status: 'pass', steps: [], reason: '', duration_ms: 0, wait_ms: 0 };
   const jobTimeout = (job.timeout_s || ctx.jobTimeoutS) * 1000;
   const log = (m) => { res.notes = (res.notes || []).concat(m); };
   const runnableSteps = job.steps.filter((s) => !s.noop);
   if (!runnableSteps.length) { res.status = 'pass'; res.reason = 'nothing to run'; res.duration_ms = Date.now() - t0; return res; }
-  const needExec = runnableSteps.some((s) => !s.skip);
   let svc = null; let release = null;
-  try {
-    if (needExec) {
+  // forge#4040 (REG-5): the job timeout clock starts when the job starts executing, never while it waits for a slot.
+  // forge#4057: the slot is taken lazily, right before the first step that really executes, so a job made only of skipped,
+  // cached or deduplicated steps never queues for one. Time spent here is wait_ms (not duration_ms) and shifts tExec.
+  let tExec = Date.now();
+  const takeSlot = async () => {
+    const tw = Date.now();
+    try {
       release = await acquireSlot(job.id, ctx.deadline);
-      if (!release) { res.status = 'deferred'; res.budget = true; res.reason = 'budget exhausted waiting for a machine slot'; return res; }
+      res.wait_ms += Date.now() - tw;
+      if (!release) { res.status = 'deferred'; res.budget = true; res.reason = 'budget exhausted waiting for a machine slot'; return false; }
       if (job.services && job.services.length) {
         svc = await startServices(job, log);
-        if (svc.error) { res.status = 'deferred'; res.reason = `service containers: ${svc.error}`; return res; }
+        if (svc.error) { res.status = 'deferred'; res.reason = `service containers: ${svc.error}`; return false; }
       }
-    }
-    // forge#4040 (REG-5): the job timeout clock starts when the job starts executing, never while it waits for a slot.
-    const tExec = Date.now();
+      return true;
+    } finally { tExec += Date.now() - tw; }
+  };
+  try {
     const ghFiles = { env: path.join(ctx.tmp, `${job.id.replace(/[^A-Za-z0-9]/g, '_')}.env`), path: path.join(ctx.tmp, `${job.id.replace(/[^A-Za-z0-9]/g, '_')}.path`), output: path.join(ctx.tmp, `${job.id.replace(/[^A-Za-z0-9]/g, '_')}.out`), summary: path.join(ctx.tmp, `${job.id.replace(/[^A-Za-z0-9]/g, '_')}.sum`) };
     for (const f of Object.values(ghFiles)) fs.writeFileSync(f, '');
     const carried = {}; let failedGroup = null;
@@ -212,17 +276,23 @@ async function runJob(job, ctx) {
       if (step.skip) { sr.status = 'skipped'; sr.reason = step.skip; continue; }
       if (failedGroup && step.group !== failedGroup) break;
       if (Date.now() > ctx.deadline) { res.status = 'deferred'; res.budget = true; res.reason = 'budget exhausted'; return res; }
-      const s0 = Date.now();
+      let s0 = Date.now();
       if (step.builtin === 'portability') {
         const f = checkPortability({ repo: ctx.repo, from: ctx.plan.merge_base, changed: ctx.plan.changed_files, tracked: ctx.tracked });
         sr.duration_ms = Date.now() - s0;
         if (f.length) { sr.status = 'fail'; res.tail = f.slice(0, 40); res.status = 'fail'; res.failed_step = step.name; failedGroup = step.group || null; if (!failedGroup) break; }
         continue;
       }
-      let key = '';
+      let key = ''; let fkey = '';
       const cacheable = step.kind !== 'install' && !ctx.noResultCache;
+      if (step.kind !== 'install') {
+        const base = stepKey(step, job, ctx);
+        // forge#4057: identical checks (command, cwd, shell, env, input hash, carried env) run once per invocation. A step that
+        // reads or writes the GitHub Actions side files has effects beyond its exit code, so it is never shared.
+        if (!(job.source === 'github-actions' && /GITHUB_(ENV|PATH|OUTPUT|STEP_SUMMARY)/.test(step.command))) fkey = sha256(base + JSON.stringify(carried));
+        if (cacheable) key = base;
+      }
       if (cacheable) {
-        key = stepKey(step, job, ctx);
         const f = path.join(cacheDir(ctx.o), key + '.json');
         if (fs.existsSync(f)) { sr.cache = 'hit'; sr.duration_ms = 0; try { fs.utimesSync(f, new Date(), new Date()); } catch { /* ignore */ } continue; }
         sr.cache = 'miss';
@@ -233,14 +303,35 @@ async function runJob(job, ctx) {
       const env = jobEnv(job, step, ctx, ghFiles); Object.assign(env, carried);
       const cwd = path.resolve(ctx.repo, step.cwd || '.');
       if (!insideRepo(ctx.repo, cwd)) { res.status = 'deferred'; res.reason = 'step working-directory escapes the repository'; return res; }
-      const tail = new Tail();
-      const left = Math.max(1000, Math.min((step.timeout_s || ctx.jobTimeoutS) * 1000, jobTimeout - (Date.now() - tExec), ctx.deadline - Date.now()));
-      const r = await exec([...argv, scriptFile], { cwd, env, timeoutMs: left, onData: (b) => tail.push(b) });
-      sr.duration_ms = Date.now() - s0;
-      if (job.source === 'github-actions') { const e2 = { ...env }; absorbGhFiles(ghFiles, e2); for (const k of Object.keys(e2)) if (e2[k] !== env[k]) carried[k] = e2[k]; }
-      if (r.code === 0) { if (key) { try { fs.mkdirSync(cacheDir(ctx.o), { recursive: true }); fs.writeFileSync(path.join(cacheDir(ctx.o), key + '.json'), JSON.stringify({ cmd: step.command.slice(0, 300), ts: Date.now() })); } catch { /* cache is best effort */ } } continue; }
-      if (r.error) tail.push(Buffer.from(`${r.error}\n`));
-      const text = tail.text(60);
+      let prior = fkey ? ctx.flight.get(fkey) : null;
+      if (!prior && !release) {
+        if (!(await takeSlot())) return res;
+        s0 = Date.now();
+        prior = fkey ? ctx.flight.get(fkey) : null; // another job may have started the same check while this one queued
+      }
+      let r; let text; let left;
+      if (prior) {
+        // single-flight: wait for the first run of this exact check and reuse its result (a failure is shared, never re-run)
+        ({ r, text, left } = await prior);
+        sr.cache = 'dedup'; sr.duration_ms = 0;
+      } else {
+        const tail = new Tail();
+        left = Math.max(1000, Math.min((step.timeout_s || ctx.jobTimeoutS) * 1000, jobTimeout - (Date.now() - tExec), ctx.deadline - Date.now()));
+        const l0 = left;
+        const flight = (async () => { const r0 = await exec([...argv, scriptFile], { cwd, env, timeoutMs: l0, onData: (b) => tail.push(b) }); if (r0.error) tail.push(Buffer.from(`${r0.error}\n`)); return { r: r0, text: tail.text(60), left: l0 }; })();
+        if (fkey) ctx.flight.set(fkey, flight);
+        ({ r, text } = await flight);
+        sr.duration_ms = Date.now() - s0;
+        if (job.source === 'github-actions') { const e2 = { ...env }; absorbGhFiles(ghFiles, e2); for (const k of Object.keys(e2)) if (e2[k] !== env[k]) carried[k] = e2[k]; }
+        if (r.code === 0) {
+          try { // duration_ms in the entry and a per-command duration hint feed the scheduler's cost ordering
+            fs.mkdirSync(path.join(cacheDir(ctx.o), 'dur'), { recursive: true });
+            if (key) fs.writeFileSync(path.join(cacheDir(ctx.o), key + '.json'), JSON.stringify({ cmd: step.command.slice(0, 300), ts: Date.now(), duration_ms: sr.duration_ms }));
+            fs.writeFileSync(path.join(cacheDir(ctx.o), 'dur', durKey(step)), String(sr.duration_ms));
+          } catch { /* cache is best effort */ }
+        }
+      }
+      if (r.code === 0) continue;
       if (r.timedOut) { if (Date.now() > ctx.deadline) { res.status = 'deferred'; res.budget = true; res.reason = 'budget exhausted (job killed)'; return res; } sr.status = 'timeout'; res.status = 'degraded'; res.reason = `timed out after ${Math.round(left / 1000)}s (a timeout proves nothing about the code)`; res.tail = text; res.failed_step = step.name; return res; }
       if (step.kind === 'install') { res.status = 'deferred'; res.reason = `dependency install failed (offline?): ${step.command.slice(0, 60)}`; return res; }
       if (r.code === 127 && text.some((l) => /command not found|not found$|No such file or directory/.test(l)) && !step.continue_on_error) {
@@ -257,7 +348,7 @@ async function runJob(job, ctx) {
   } finally {
     if (svc && svc.finish) svc.finish();
     if (release) release();
-    res.duration_ms = Date.now() - t0;
+    res.duration_ms = Math.max(0, Date.now() - t0 - res.wait_ms);
   }
 }
 
@@ -277,23 +368,34 @@ export async function runPlan(plan, o = {}) {
     plan, repo, o, tmp: tmpRoot, deadline: t0 + budgetS * 1000, jobTimeoutS: Number(process.env.FORGE_CI_LOCAL_JOB_TIMEOUT) || Math.min(900, budgetS),
     tracked: runSync('git', ['-C', repo, 'ls-files', '--cached', '--others', '--exclude-standard', '-z']).out.split('\0').filter(Boolean),
     noResultCache: process.env.FORGE_CI_LOCAL_NO_CACHE === '1',
+    flight: new Map(),
   };
   const runnable = plan.jobs.filter((j) => !j.not_applicable && !j.deferred_reason);
   const ids = new Set(runnable.map((j) => j.id));
   const results = new Map(); const running = new Map();
   const pending = new Set(runnable.map((j) => j.id));
   const byId = new Map(runnable.map((j) => [j.id, j]));
+  const costs = new Map();
+  const cost = (id) => { if (!costs.has(id)) { let c = [1, 0]; try { c = jobCost(byId.get(id), ctx); } catch { /* ordering is a hint */ } costs.set(id, c); } return costs.get(id); };
+  const order = new Map([...pending].map((id, i) => [id, i]));
   while (pending.size || running.size) {
+    const ready = [];
     for (const id of [...pending]) {
       const j = byId.get(id);
       const needs = (j.needs || []).filter((n) => ids.has(n));
       const blocked = needs.find((n) => results.has(n) && results.get(n).status !== 'pass');
-      if (blocked) { pending.delete(id); const b = results.get(blocked); const r = { id, name: j.name, source: j.source, status: b.status === 'fail' ? 'skipped' : 'deferred', reason: b.status === 'fail' ? `needs ${blocked}, which failed` : `needs ${blocked}, which is not proven locally`, steps: [], duration_ms: 0 }; results.set(id, r); printJob(r); continue; }
+      if (blocked) { pending.delete(id); const b = results.get(blocked); const r = { id, name: j.name, source: j.source, status: b.status === 'fail' ? 'skipped' : 'deferred', reason: b.status === 'fail' ? `needs ${blocked}, which failed` : `needs ${blocked}, which is not proven locally`, steps: [], duration_ms: 0, wait_ms: 0 }; results.set(id, r); printJob(r); continue; }
       if (needs.some((n) => !results.has(n))) continue;
+      ready.push(id);
+    }
+    // narrow/scoped jobs first, then the cheapest; the widest or longest last (ties keep plan order). `needs` is honoured above.
+    ready.sort((x, y) => cost(x)[0] - cost(y)[0] || cost(x)[1] - cost(y)[1] || order.get(x) - order.get(y));
+    for (const id of ready) {
       if (running.size >= maxParallel) break;
-      if (Date.now() > ctx.deadline) { pending.delete(id); results.set(id, { id, name: j.name, source: j.source, status: 'deferred', budget: true, reason: 'budget exhausted before the job started', steps: [], duration_ms: 0 }); continue; }
+      const j = byId.get(id);
+      if (Date.now() > ctx.deadline) { pending.delete(id); results.set(id, { id, name: j.name, source: j.source, status: 'deferred', budget: true, reason: 'budget exhausted before the job started', steps: [], duration_ms: 0, wait_ms: 0 }); continue; }
       pending.delete(id);
-      const p = runJob(j, ctx).catch((e) => ({ id, name: j.name, source: j.source, status: 'deferred', reason: `runner error: ${e.message}`, steps: [], duration_ms: 0 })).then((r) => { results.set(id, r); running.delete(id); printJob(r); });
+      const p = runJob(j, ctx).catch((e) => ({ id, name: j.name, source: j.source, status: 'deferred', reason: `runner error: ${e.message}`, steps: [], duration_ms: 0, wait_ms: 0 })).then((r) => { results.set(id, r); running.delete(id); printJob(r); });
       running.set(id, p);
     }
     if (running.size) await Promise.race(running.values()); else if (pending.size) await sleep(10);
@@ -315,7 +417,7 @@ export async function runPlan(plan, o = {}) {
     if (s.scope.mode === 'scoped' || s.scope.mode === 'skip') narrowed++;
     if (s.scope.widen_reason) widened++;
     if (typeof s.scope.selected === 'number' && typeof s.scope.total === 'number') { selSel += s.scope.selected; selTotal += s.scope.total; }
-    if (checks++ < 40) console.log(`CI_LOCAL_CHECK: job=${r.id} step=${n + 1} selector=${s.scope.selector || 'none'} selected=${s.scope.selected ?? ''} total=${s.scope.total ?? ''} cache=${s.cache} duration_ms=${s.duration_ms} widen=${(s.scope.widen_reason || '').replace(/\s+/g, '_') || 'none'}`);
+    if (checks++ < 40) console.log(`CI_LOCAL_CHECK: job=${r.id} step=${n + 1} selector=${s.scope.selector || 'none'} selected=${s.scope.selected ?? ''} total=${s.scope.total ?? ''} cache=${s.cache} duration_ms=${s.duration_ms} widen=${(s.scope.widen_reason || '').replace(/\s+/g, '_') || 'none'} wait_ms=${r.wait_ms || 0}`);
   }
   const degraded = all.filter((r) => r.status === 'degraded');
   const budgetHit = all.some((r) => r.budget) || (all.some((r) => r.status === 'deferred' && /budget exhausted/.test(r.reason || '')));
@@ -325,18 +427,22 @@ export async function runPlan(plan, o = {}) {
   const degReason = degraded.length ? 'job_timeout' : budgetHit ? (capped ? 'budget_capped_at_invocation_limit' : 'budget_exhausted') : '';
   const state = failed.length ? 'FAIL' : degReason ? 'DEGRADED' : 'PASS';
   const dur = Date.now() - t0;
+  const waitTotal = all.reduce((a, r) => a + (r.wait_ms || 0), 0);
+  const dedups = all.reduce((a, r) => a + r.steps.filter((x) => x.cache === 'dedup').length, 0);
   if (plan.no_applicable_jobs && !all.length) console.log('CI_LOCAL_NOTE: no_applicable_jobs (CI definitions exist but no job applies to this change)');
-  console.log(`CI_LOCAL_RESULT: ${state} jobs_run=${run_.length} jobs_deferred=${defd.length} jobs_failed=${failed.length} jobs_degraded=${degraded.length} jobs_skipped=${skipped.length} checks=${checks} cache_hits=${hits} narrowed=${narrowed} widened=${widened} selected=${selSel} total=${selTotal} duration_ms=${dur}${Date.now() > ctx.deadline ? ' budget_exhausted=true' : ''}${plan.no_applicable_jobs && !all.length ? ' no_applicable_jobs=true' : ''}${degReason && state === 'DEGRADED' ? ` reason=${degReason}` : ''}`);
-  if (o.resultFile) { try { fs.writeFileSync(o.resultFile, JSON.stringify({ state, mode: plan.config && plan.config.mode, jobs: all, plan_key: plan.plan_key, duration_ms: dur }, null, 2)); } catch { /* ignore */ } }
+  console.log(`CI_LOCAL_RESULT: ${state} jobs_run=${run_.length} jobs_deferred=${defd.length} jobs_failed=${failed.length} jobs_degraded=${degraded.length} jobs_skipped=${skipped.length} checks=${checks} cache_hits=${hits} narrowed=${narrowed} widened=${widened} selected=${selSel} total=${selTotal} duration_ms=${dur} wait_ms=${waitTotal} dedup=${dedups}${Date.now() > ctx.deadline ? ' budget_exhausted=true' : ''}${plan.no_applicable_jobs && !all.length ? ' no_applicable_jobs=true' : ''}${degReason && state === 'DEGRADED' ? ` reason=${degReason}` : ''}`);
+  if (o.resultFile) { try { fs.writeFileSync(o.resultFile, JSON.stringify({ state, mode: plan.config && plan.config.mode, jobs: all, plan_key: plan.plan_key, duration_ms: dur, wait_ms: waitTotal }, null, 2)); } catch { /* ignore */ } }
   try { fs.rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ }
   return failed.length ? 1 : state === 'DEGRADED' ? 2 : 0;
 }
 
 function printJob(r) {
   const secs = (r.duration_ms / 1000).toFixed(1);
+  const waitS = ((r.wait_ms || 0) / 1000).toFixed(1);
+  const dedup = r.steps.filter((s) => s.cache === 'dedup').length;
   const stepsRun = r.steps.filter((s) => s.status !== 'skipped').length;
   const hit = r.steps.filter((s) => s.cache === 'hit').length;
-  console.log(`CI_LOCAL_JOB: ${r.status === 'pass' ? 'pass' : r.status} ${r.id} | ${r.name} | ${secs}s | steps=${stepsRun}/${r.steps.length}${hit ? ` cached=${hit}` : ''}${r.reason ? ` | ${r.reason}` : ''}`);
+  console.log(`CI_LOCAL_JOB: ${r.status === 'pass' ? 'pass' : r.status} ${r.id} | ${r.name} | ${secs}s | steps=${stepsRun}/${r.steps.length}${hit ? ` cached=${hit}` : ''}${dedup ? ` dedup=${dedup}` : ''} wait=${waitS}s${r.reason ? ` | ${r.reason}` : ''}`);
   if (r.status === 'fail' || r.status === 'degraded') {
     console.log(`${r.status === 'fail' ? 'CI_LOCAL_FAIL' : 'CI_LOCAL_DEGRADED_STEP'}: ${r.id} | step: ${r.failed_step || '?'}${r.exit !== undefined ? ` | exit ${r.exit}` : ''}`);
     if (r.failed_command) console.log(`CI_LOCAL_FAIL_COMMAND: ${r.failed_command.split('\n')[0].slice(0, 300)}`);

package/scripts/ci-local/scoping.mjs

@@ -242,8 +242,30 @@ function reachFile(sctx, f) {
   return out;
 }
 
+// Every tracked, present test file in the repo: the universe the SEC-3 "reaches at least one test" check runs against.
+function allTests(sctx) {
+  if (!sctx.cache.allTests) sctx.cache.allTests = sctx.tracked.filter((f) => isTestFile(f) && sctx.present.has(f));
+  return sctx.cache.allTests;
+}
+
+// The SEC-3 universe is limited to the step's own ecosystem: a test in another language (tests/utils.test.ts) never
+// counts as covering a changed source of this one (pkg/utils.py), or a pytest step would skip instead of widen.
+const ECO = { js: 'js', mjs: 'js', cjs: 'js', jsx: 'js', ts: 'js', tsx: 'js', mts: 'js', cts: 'js', py: 'py', pyi: 'py', sh: 'sh', bash: 'sh', bats: 'sh', rb: 'rb', php: 'php', go: 'go', java: 'jvm', kt: 'jvm', scala: 'jvm', rs: 'rs', cs: 'cs' };
+const ecoOf = (f) => { const e = extOf(f); return ECO[e] || e; };
+// #4068 review DRR-1: the per-ecosystem universe applies only to a step whose ecosystem has a changed source file. A step
+// in another ecosystem (a node step on a Python-only change) keeps the repo-wide universe it had at a0d45ae, so it skips or
+// narrows instead of widening to a full run on every single-language change.
+function ecosystemTests(sctx, tests) {
+  const ecos = new Set(tests.map(ecoOf));
+  const touched = sctx.changed.some((f) => !isTestFile(f) && ecos.has(ecoOf(f)) && (CODE_EXT.has(extOf(f)) || /^(sh|bash)$/.test(extOf(f))));
+  if (!touched) return allTests(sctx);
+  return allTests(sctx).filter((t) => ecos.has(ecoOf(t)));
+}
+
 // Which of testFiles can this change reach? { selected, total, widen } ; widen is a reason string or ''.
-export function impactedTests(sctx, testFiles) {
+// `universe` is the repo-wide test set used only for the SEC-3 reach check: a step that runs ONE test must not widen just
+// because the change does not reach that test, only because it reaches no test anywhere. Selection stays within testFiles.
+export function impactedTests(sctx, testFiles, universe = testFiles) {
   const r = reach(sctx);
   if (r.widen) return { selected: [...testFiles], total: testFiles.length, widen: r.widen };
   const selected = new Set();
@@ -254,10 +276,11 @@ export function impactedTests(sctx, testFiles) {
   // never hide it. The reason names the file.
   const srcFiles = sctx.changed.filter((f) => !isTestFile(f) && (CODE_EXT.has(extOf(f)) || /^(sh|bash)$/.test(extOf(f))));
   const unreached = [];
+  if (universe !== testFiles) universe = [...new Set([...universe, ...testFiles])];
   for (const f of srcFiles) {
     const rf = reachFile(sctx, f);
     if (rf.widen) return { selected: [...testFiles], total: testFiles.length, widen: rf.widen };
-    const hit = testFiles.some((t) => rf.files.has(t) && sctx.present.has(t)) || siblingTests(f, testFiles).length > 0;
+    const hit = universe.some((t) => rf.files.has(t) && sctx.present.has(t)) || siblingTests(f, universe).length > 0;
     if (!hit) unreached.push(f);
   }
   if (unreached.length) return { selected: [...testFiles], total: testFiles.length, widen: `empty-selection: no test reaches ${unreached.slice(0, 5).join(', ')}${unreached.length > 5 ? ` (+${unreached.length - 5} more)` : ''}` };
@@ -398,7 +421,7 @@ function scopeRunner(name, args, head, cmd, cwdRel, sctx, join) {
   if (sctx.widen) return full(cmd, sctx.widen, { selector: spec.sel, selected: candidates.length, total: candidates.length });
   const un = unmappedFile(sctx.changed.filter((f) => !isTestFile(f)), new Set([...spec.ext, ...SRC_EXT]));
   if (un) return full(cmd, `unknown-language: ${un}`, { selector: spec.sel, selected: candidates.length, total: candidates.length });
-  const imp = impactedTests(sctx, candidates);
+  const imp = impactedTests(sctx, candidates, ecosystemTests(sctx, universe));
   if (imp.widen) return full(cmd, imp.widen, { selector: spec.sel, selected: candidates.length, total: candidates.length });
   if (!imp.selected.length) return skipNo(0, candidates.length, spec.sel, 'no test reaches the changed files');
   const sel = imp.selected.map((f) => fromCwd(cwdRel, f));
@@ -414,7 +437,7 @@ function scopeSingleTest(rest, cmd, cwdRel, sctx) {
   if (!isTestFile(r) || !sctx.present.has(r)) return null;
   if (rest.slice(rest.indexOf(fileTok) + 1).length) return full(cmd, '', { selector: 'unscoped:test-args' });
   if (sctx.widen) return full(cmd, sctx.widen, { selector: 'single-test', selected: 1, total: 1 });
-  const imp = impactedTests(sctx, [r]);
+  const imp = impactedTests(sctx, [r], ecosystemTests(sctx, [r]));
   if (imp.widen) return full(cmd, imp.widen, { selector: 'single-test', selected: 1, total: 1 });
   if (!imp.selected.length) return skipNo(0, 1, 'single-test', `${r} not reached by the change`);
   return scoped(cmd, 1, 1, 'single-test');

package/scripts/ci-local.sh

@@ -76,7 +76,7 @@ field() { printf '%s\n' "$RES" | tr ' ' '\n' | sed -n "s/^$1=//p" | head -n 1; }
 STATE=$(printf '%s\n' "$RES" | awk '{print $2}')
 JR=$(field jobs_run); JD=$(field jobs_deferred); JF=$(field jobs_failed); DUR=$(field duration_ms)
 forge_emit ci_local result="${STATE:-DEGRADED}" jobs_run="${JR:-0}" jobs_deferred="${JD:-0}" jobs_failed="${JF:-0}" \
-  duration_ms="${DUR:-0}" fix_iterations="$ITER" checks="$(field checks)" cache_hits="$(field cache_hits)" \
+  duration_ms="${DUR:-0}" wait_ms="$(field wait_ms)" fix_iterations="$ITER" checks="$(field checks)" cache_hits="$(field cache_hits)" \
   narrowed="$(field narrowed)" widened="$(field widened)" selected="$(field selected)" total="$(field total)" >/dev/null 2>&1 || true
 
 # One event per scoped/widened/executed check, capped so a large plan never floods the telemetry comment.
@@ -90,7 +90,8 @@ while IFS= read -r line; do
   ch=$(printf '%s\n' "$line" | sed -n 's/.* cache=\([^ ]*\).*/\1/p')
   dm=$(printf '%s\n' "$line" | sed -n 's/.* duration_ms=\([^ ]*\).*/\1/p')
   wd=$(printf '%s\n' "$line" | sed -n 's/.* widen=\([^ ]*\).*/\1/p')
-  forge_emit ci_local_check job="$job" selector="$sel" selected="${sc:-0}" total="${tt:-0}" cache="$ch" duration_ms="${dm:-0}" widen="$wd" >/dev/null 2>&1 || true
+  wm=$(printf '%s\n' "$line" | sed -n 's/.* wait_ms=\([^ ]*\).*/\1/p')
+  forge_emit ci_local_check job="$job" selector="$sel" selected="${sc:-0}" total="${tt:-0}" cache="$ch" duration_ms="${dm:-0}" wait_ms="${wm:-0}" widen="$wd" >/dev/null 2>&1 || true
   EMITTED=$((EMITTED + 1))
 done < <(grep '^CI_LOCAL_CHECK:' "$OUT")