forgedock 1.18.0 → 1.20.0

package/bin/runner.mjs

@@ -47,8 +47,11 @@ import {
   mkdtempSync,
   rmSync,
   realpathSync,
+  lstatSync,
+  chmodSync,
 } from "fs";
-import { join, dirname, basename, relative, isAbsolute, extname, win32 as win32Path } from "path";
+import { createHash } from "crypto";
+import { join, dirname, basename, resolve, relative, isAbsolute, extname, win32 as win32Path } from "path";
 import os from "os";
 import { execSync, spawnSync } from "child_process";
 import { createRequire } from "module";
@@ -104,28 +107,65 @@ export function resolveRunTmpRoot(cwd, env = process.env) {
   return join(os.homedir(), ".forge", "tmp");
 }
 
-/** `<root>/<issue>` for a numeric issue; null for an empty root or non-numeric issue (no path-escape, never the root). */
-export function runTmpDir(root, issue) {
+/**
+ * Stable, path-safe identity of the repo checkout: `<sanitized basename>-<8 hex of sha1(realpath)>`.
+ * Local only (no network), identical across a `--retry` from the same checkout, distinct per checkout.
+ * Never empty, never `.`/`..`, never contains a path separator.
+ */
+export function repoKey(cwd = process.cwd()) {
+  let real;
+  try { real = realpathSync(cwd); } catch { real = resolve(String(cwd)); }
+  const name = basename(real).replace(/[^A-Za-z0-9._-]/g, "_").replace(/^\.+/, "") || "repo";
+  return `${name}-${createHash("sha1").update(real).digest("hex").slice(0, 8)}`;
+}
+
+/** `<root>/<repoKey>/<issue>` for a numeric issue; null for an empty root or non-numeric issue (no path-escape, never the root). */
+export function runTmpDir(root, issue, cwd = process.cwd()) {
   const id = String(issue ?? "").trim();
-  return root && /^[0-9]+$/.test(id) ? join(root, id) : null;
+  return root && /^[0-9]+$/.test(id) ? join(root, repoKey(cwd), id) : null;
+}
+
+/** True when `p` is a real directory (not a symlink) owned by the current uid (uid check skipped where unavailable). */
+function isOwnedRealDir(p) {
+  const st = lstatSync(p);
+  if (!st.isDirectory() || st.isSymbolicLink()) return false;
+  return typeof process.getuid !== "function" || st.uid === process.getuid();
 }
 
 /**
  * Point TMPDIR and CLAUDE_CODE_TMPDIR at a per-run, disk-backed dir so venvs and Claude session data
- * stay off a size-limited /tmp. No-op when the issue is unknown or the operator owns the tmp root.
- * Best-effort: a failed mkdir leaves the env unchanged.
+ * stay off a size-limited /tmp. The dir is repo-scoped and created 0700; an existing dir that is a symlink
+ * or owned by another uid is not used. No-op when the issue is unknown or the operator owns the tmp root.
+ * Best-effort: any failure leaves the env unchanged.
  */
 export function withRunTmpdir(env, { issue, cwd = process.cwd() } = {}) {
-  const dir = runTmpDir(resolveRunTmpRoot(cwd, env), issue);
+  const dir = runTmpDir(resolveRunTmpRoot(cwd, env), issue, cwd);
   if (!dir) return env;
   try {
-    mkdirSync(dir, { recursive: true });
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const parent = dirname(dir);
+    for (const d of [parent, dir]) {
+      if (!isOwnedRealDir(d)) return env;
+      chmodSync(d, 0o700);
+    }
     env.TMPDIR = dir;
     env.CLAUDE_CODE_TMPDIR = dir;
   } catch { /* scratch stays on the inherited tmp */ }
   return env;
 }
 
+const LEASE_HOLDER_ENV_RE = /^cli_[A-Za-z0-9._-]+_[0-9]+$/;
+
+/**
+ * Stamp the phase child's env with `FORGE_LEASE_HOLDER=<engine lease id>` so the engine can tell, after a
+ * SIGKILL of the engine, whether an orphaned phase child is still running (env is inherited by grandchildren).
+ * An absent or malformed holder leaves the env untouched and never throws.
+ */
+export function withLeaseHolder(env, holder) {
+  if (typeof holder === "string" && LEASE_HOLDER_ENV_RE.test(holder)) env.FORGE_LEASE_HOLDER = holder;
+  return env;
+}
+
 /** First positional arg when it is an issue number (the phase arg convention), else null. */
 function issueFromArgs(args) {
   const first = Array.isArray(args) ? args[0] : String(args ?? "").trim().split(/\s+/)[0];
@@ -1402,6 +1442,7 @@ export function runCliBackend({
   spawnFn = spawnSync,
   model = null,
   maxTurns = null,
+  leaseHolder = null,
 }) {
   // forge#3589: opt-in flags, validated again here as defence in depth. Callers
   // (runCommand) are responsible for the version gate. Unset -> argv unchanged.
@@ -1468,7 +1509,7 @@ export function runCliBackend({
     // Mirrors the existing scrub in the `run_bash` tool handler below.
     // GH_TOKEN/GITHUB_TOKEN and all other env vars are intentionally left
     // intact — only the Anthropic key is removed.
-    const childEnv = withRunTmpdir(withForgedockHome({ ...process.env }), { issue: issueFromArgs(args), cwd });
+    const childEnv = withLeaseHolder(withRunTmpdir(withForgedockHome({ ...process.env }), { issue: issueFromArgs(args), cwd }), leaseHolder);
     delete childEnv.ANTHROPIC_API_KEY;
 
     // No `shell` option (defaults to false): argv is passed as discrete,
@@ -1493,6 +1534,9 @@ export function runCliBackend({
     // intentionally omitted here since spawnSync's own `timeout` + ETIMEDOUT is
     // reliable for this single, non-looped invocation.
     const timedOut = result.status === null && result.error?.code === "ETIMEDOUT";
+    // forge#4037: only a spawn-stage failure is transient infra. A wall-clock kill leaves evidence that the child ran
+    // (a kill signal, a pid, or captured output); an ETIMEDOUT with none of those never started the child.
+    const childRan = !!result.signal || (Number.isInteger(result.pid) && result.pid > 0) || (stdout + stderr).trim().length > 0;
     if (timedOut) {
       const timeoutSecs = Math.round(timeoutMs / 1000);
       // forge#2360: spawnSync's `timeout` option kills the child outright on
@@ -1510,15 +1554,44 @@ export function runCliBackend({
       // review findings — #2277, #2292, #2293, #2355 — for that defect
       // class).
       const partial = (stdout + stderr).trim();
-      throw new Error(
+      const timeoutError = new Error(
         `claude CLI invocation timed out after ${timeoutSecs}s and was killed. ` +
           `Set FORGEDOCK_CLI_TIMEOUT_MS (ms) to adjust, or use --backend api.` +
           (partial ? ` Partial output: ${sanitizeOutputExcerptForLog(partial)}` : ""),
       );
+      if (!childRan) {
+        // forge#4021: stable transient-infra class; the engine retries it on a
+        // separate bounded counter without consuming a phase attempt.
+        timeoutError.code = "CLI_SPAWN_TRANSIENT";
+        timeoutError.spawnErrno = "ETIMEDOUT";
+        throw timeoutError;
+      }
+      // forge#4037: the child ran and was killed at the wall-clock limit: a normal, attempt-consuming failure
+      // (`cli.timeout`), with usage attached when the CLI had already emitted a parseable envelope.
+      timeoutError.code = "CLI_TIMEOUT";
+      timeoutError.usage = null;
+      try {
+        const u = JSON.parse(stdout.trim())?.usage;
+        if (u && typeof u === "object") {
+          timeoutError.usage = {
+            input_tokens: toFiniteUsageNumber(u.input_tokens),
+            output_tokens: toFiniteUsageNumber(u.output_tokens),
+            cache_creation_input_tokens: toFiniteUsageNumber(u.cache_creation_input_tokens),
+            cache_read_input_tokens: toFiniteUsageNumber(u.cache_read_input_tokens),
+          };
+        }
+      } catch {
+        // No parseable envelope from a killed child: usage stays null (never fabricated).
+      }
+      throw timeoutError;
     }
     if (result.error) {
       const error = new Error(`Failed to invoke claude CLI: ${result.error.message}`);
       error.code = result.error.code;
+      if (["EAGAIN", "ENOMEM", "EMFILE"].includes(result.error.code)) {
+        error.spawnErrno = result.error.code;
+        error.code = "CLI_SPAWN_TRANSIENT";
+      }
       throw error;
     }
 
@@ -2245,9 +2318,10 @@ export function resolveBashShell() {
  *
  * @param {string} cwd
  * @param {string|null} [runIssueNumber] - issue number scoping the per-run TMPDIR (forge#3572)
+ * @param {string|null} [leaseHolder] - engine lease id stamped as FORGE_LEASE_HOLDER in the child env
  * @returns {Record<string, (input: object) => string>}
  */
-export function getToolHandlers(cwd, runIssueNumber = null) {
+export function getToolHandlers(cwd, runIssueNumber = null, leaseHolder = null) {
   return {
     read_file: ({ path }) => {
       if (!path) throw new Error("read_file requires a 'path'");
@@ -2267,7 +2341,7 @@ export function getToolHandlers(cwd, runIssueNumber = null) {
       // into the model loop, so a prompt-injection payload could otherwise issue
       // a run_bash command that exfiltrates the key. gh/git auth tokens
       // (GH_TOKEN/GITHUB_TOKEN) are intentionally left intact.
-      const childEnv = withRunTmpdir(withForgedockHome({ ...process.env }), { issue: runIssueNumber, cwd });
+      const childEnv = withLeaseHolder(withRunTmpdir(withForgedockHome({ ...process.env }), { issue: runIssueNumber, cwd }), leaseHolder);
       delete childEnv.ANTHROPIC_API_KEY;
       // Run under bash when available so bash-style commands the model emits
       // (heredocs, scripts/*.sh, single-quote semantics) behave consistently
@@ -2525,6 +2599,7 @@ export async function runCommand(opts = {}) {
     backend = process.env.FORGEDOCK_BACKEND || "auto",
     runtimeEnv = process.env,
     logger = console,
+    leaseHolder = null,
   } = opts;
 
   assertOpenCodeNativeRuntime({
@@ -2665,6 +2740,7 @@ export async function runCommand(opts = {}) {
           logger,
           model: cliModel,
           maxTurns: cliMaxTurns,
+          leaseHolder,
         },
       });
     }
@@ -2707,7 +2783,7 @@ export async function runCommand(opts = {}) {
   // is a known Linux limitation at the OS level; closing it would require
   // a native module (e.g. prctl PR_SET_DUMPABLE) or process isolation.
   // See: issue #1370 for tracking this known limitation.
-  const handlers = getToolHandlers(cwd, issueFromArgs(args));
+  const handlers = getToolHandlers(cwd, issueFromArgs(args), leaseHolder);
   const messages = [{ role: "user", content: userMessage }];
 
   // Accumulate token usage across all messages.create() calls in this run.

package/bin/stats/aggregate.mjs

@@ -127,8 +127,10 @@ function funnelSection(events, issues) {
   for (const d of Object.values(bySeverity)) d.perReview = reviews.size ? d.total / reviews.size : null;
   const dispositions = { ISSUE: 0, NOTE: 0, INPR_FIX: 0 };
   for (const f of findings) if (f.disposition in dispositions) dispositions[f.disposition]++;
+  // Events written before the scope field existed carry none and are not counted.
+  const introduced = findings.filter((f) => f.scope === "introduced").length;
   return {
-    reviews: reviews.size, findings: findings.length, perReview: ratio(findings.length, reviews.size),
+    reviews: reviews.size, findings: findings.length, introduced, perReview: ratio(findings.length, reviews.size),
     bySeverity, dispositions,
     validated: issues.filter((i) => i.labels.includes("validated")).length,
     falsePositive: issues.filter((i) => i.labels.includes("false-positive")).length,
@@ -150,7 +152,7 @@ function gatesSection(events) {
   };
 }
 
-function issueFlowSection(issues) {
+function issueFlowSection(issues, introduced = 0) {
   const real = issues.filter((i) => !i.isPr);
   const created = {};
   for (const i of real) { const s = issueSource(i); created[s] = (created[s] ?? 0) + 1; }
@@ -162,6 +164,7 @@ function issueFlowSection(issues) {
     n: real.length, created, createdTotal: real.length, closed, closedTotal: closedIssues.length,
     net: real.length ? real.length - closedIssues.length : null,
     findingsPerMerge: ratio(created["review-finding"] ?? 0, merged),
+    introducedPerMerge: ratio(introduced, merged),
   };
 }
 
@@ -214,7 +217,8 @@ export function aggregate({ runs = [], events = [], issues = [] } = {}, opts = {
   let groups = [...keys.values()].map(({ version, repo }) => {
     const sel = (list) => list.filter((x) => x.version === version && x.repo === repo);
     const g = { version, repo, cost: costSection(sel(counted)), time: timeSection(sel(counted)), flow: flowSection(sel(counted)),
-      funnel: funnelSection(sel(evs), sel(iss)), gates: gatesSection(sel(evs)), issueFlow: issueFlowSection(sel(iss)) };
+      funnel: funnelSection(sel(evs), sel(iss)), gates: gatesSection(sel(evs)), issueFlow: null };
+    g.issueFlow = issueFlowSection(sel(iss), g.funnel.introduced);
     g.n = g.flow.n;
     g.lowN = g.n < minN;
     g.minN = minN;

package/bin/stats/cache.mjs

@@ -17,13 +17,17 @@ export function cacheDir(repo, env = process.env) {
   return join(root, repo.replace("/", "_"));
 }
 
-/** { watermark: ISO string | null, items: { "<number>": item } } */
-export const emptyCache = () => ({ v: CACHE_VERSION, watermark: null, items: {} });
+/**
+ * { watermark: ISO string | null, coveredFrom: ISO string | null, items: { "<number>": item } }
+ * The cache covers activity in [coveredFrom, watermark]; coveredFrom null means "from the beginning". A cache written
+ * before the field existed loads with coveredFrom undefined (floor unknown), never as full coverage.
+ */
+export const emptyCache = () => ({ v: CACHE_VERSION, watermark: null, coveredFrom: null, items: {} });
 
 export function loadCache(repo, env = process.env) {
   try {
     const c = JSON.parse(readFileSync(join(cacheDir(repo, env), "cache.json"), "utf8"));
-    if (c && c.v === CACHE_VERSION && c.items && typeof c.items === "object") return { ...emptyCache(), ...c };
+    if (c && c.v === CACHE_VERSION && c.items && typeof c.items === "object") return { ...emptyCache(), ...c, coveredFrom: "coveredFrom" in c ? c.coveredFrom : undefined };
   } catch { /* missing or corrupt: refetch */ }
   return emptyCache();
 }

package/bin/stats/github-source.mjs

@@ -5,17 +5,24 @@
  * Reads labels and ONLY trusted `<!-- FORGE:TELEMETRY v1 -->` comments (same predicate as
  * scripts/trusted-comments.sh: an author_association in FORGE_TRAIL_TRUSTED_ASSOCIATIONS, a Bot, or a login in
  * FORGE_TRAIL_TRUSTED_LOGINS). Every `gh` call is argv-based (no shell). The `gh` runner is injectable for tests.
- * Fetching is incremental: only items updated after the cached watermark (minus a small overlap window) are re-read.
+ * Fetching is windowed and incremental: only items updated after the crawl start (the cached watermark minus a small
+ * overlap, or the requested `--since` bound) are listed, and comments come from the repo-wide `issues/comments?since=`
+ * endpoint (a few paginated calls) instead of one call per item.
  */
-import { execFileSync } from "node:child_process";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
 import { emptyCache, loadCache, saveCache } from "./cache.mjs";
 
 export const MARKER_RE = /^<!-- FORGE:TELEMETRY v1( part=(\d+))? -->/;
 const OWNER_REPO_RE = /^[\w.-]+\/[\w.-]+$/;
 export const isValidRepo = (r) => typeof r === "string" && OWNER_REPO_RE.test(r) && !r.split("/").some((p) => p === "." || p === "..");
 
-export function defaultGh(args) {
-  return execFileSync("gh", args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 120000, maxBuffer: 256 * 1024 * 1024 });
+const execFileAsync = promisify(execFile);
+
+/** Async `gh` runner (concurrency needs it). A test stub may stay synchronous: every call site awaits the result. */
+export async function defaultGh(args) {
+  const { stdout } = await execFileAsync("gh", args, { encoding: "utf8", timeout: 120000, maxBuffer: 256 * 1024 * 1024 });
+  return stdout;
 }
 
 /** `gh api --paginate` prints one JSON array per page; join them into one flat array. */
@@ -102,7 +109,6 @@ export function runsFromRecords(records) {
 export const gateEvents = (records) => records.filter((r) => r.event !== "phase_attempt" && r.event !== "run_terminal")
   .map((r) => ({ ...r, version: r.forge_version ?? null }));
 
-const CHECKPOINT_EVERY = 25;
 const PAGE_SIZE = 100;
 // The next incremental run re-reads this much before the persisted watermark (an overlap window), so a write that races a
 // crawl or a small clock skew is re-listed. Re-reads are idempotent: items are keyed by issue number.
@@ -125,7 +131,7 @@ const sinceParam = (ts) => encodeURIComponent(ts).replace(/%3A/g, ":");
  * the cursor (more than PAGE_SIZE items share one timestamp) is drained with `page=N` for that timestamp only.
  * Yields each page's unseen items sorted updated-asc.
  */
-function* listIssuePages(gh, repo, startSince) {
+async function* listIssuePages(gh, repo, startSince) {
   const base = ["state=all", "sort=updated", "direction=asc", `per_page=${PAGE_SIZE}`];
   const seen = new Set();
   const key = (it) => `${it.number}@${it.updated_at}`;
@@ -137,8 +143,8 @@ function* listIssuePages(gh, repo, startSince) {
   const maxTs = (page) => page.reduce((m, it) => (it?.updated_at && (!m || it.updated_at > m) ? it.updated_at : m), null);
   let cursor = startSince ?? null;
   for (let n = 0; n < MAX_LIST_REQUESTS; n++) {
-    const get = (extra = []) => parsePages(gh(["api", `repos/${repo}/issues?${[...base, ...(cursor ? [`since=${sinceParam(cursor)}`] : []), ...extra].join("&")}`]));
-    const page = get();
+    const get = async (extra = []) => parsePages(await gh(["api", `repos/${repo}/issues?${[...base, ...(cursor ? [`since=${sinceParam(cursor)}`] : []), ...extra].join("&")}`]));
+    const page = await get();
     const fresh = take(page);
     if (fresh.length > 0) yield fresh;
     if (page.length < PAGE_SIZE) return;
@@ -146,7 +152,7 @@ function* listIssuePages(gh, repo, startSince) {
     if (next && next !== cursor) { cursor = next; continue; }
     // No cursor progress on a full page: drain this timestamp by page number, then step past it.
     for (let p = 2; p < MAX_LIST_REQUESTS; p++) {
-      const more = get([`page=${p}`]);
+      const more = await get([`page=${p}`]);
       const freshMore = take(more);
       if (freshMore.length > 0) yield freshMore;
       if (more.length < PAGE_SIZE || freshMore.length === 0) break;
@@ -159,35 +165,137 @@ function* listIssuePages(gh, repo, startSince) {
 const meta = (it) => ({ number: it.number, isPr: Boolean(it.pull_request), createdAt: it.created_at ?? null, closedAt: it.closed_at ?? null,
   updatedAt: it.updated_at ?? null, labels: (it.labels ?? []).map((l) => (typeof l === "string" ? l : l.name)).filter(Boolean), authorType: it.user?.type ?? null });
 
+const CHECKPOINT_EVERY = 1000;
+const RATE_CHECK_EVERY = 50;
+const RATE_LOW = 200;
+const MAX_RATE_SLEEP_MS = 60 * 60 * 1000;
+const DEFAULT_CONCURRENCY = 6;
+const isoNow = (ms) => new Date(ms).toISOString().replace(/\.\d{3}Z$/, "Z");
+
+/** Run `fn` over `list` with at most `limit` in flight; the first failure stops new work and is rethrown. */
+async function pool(list, limit, fn) {
+  let next = 0, failed = null;
+  const worker = async () => {
+    while (!failed && next < list.length) {
+      const item = list[next++];
+      try { await fn(item); } catch (err) { failed ??= err; }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), list.length) }, worker));
+  if (failed) throw failed;
+}
+
+/** Best effort: when the core rate limit is nearly spent, sleep until it resets. Never throws. */
+async function waitForRate(gh, sleep, now) {
+  try {
+    const core = JSON.parse(await gh(["api", "rate_limit"]))?.resources?.core;
+    if (core && Number.isFinite(core.remaining) && core.remaining < RATE_LOW && Number.isFinite(core.reset)) {
+      await sleep(Math.min(Math.max(core.reset * 1000 - now() + 1000, 0), MAX_RATE_SLEEP_MS));
+    }
+  } catch { /* the limit check is advisory */ }
+}
+
 /**
- * Fetch (or incrementally refresh) the repo's issues, PRs and trusted telemetry records.
- * Returns { items, records, fetched, fromCache }. `refresh` ignores the cache; `useCache:false` never touches disk.
+ * Page the repo-wide comments endpoint (comments updated since `from`, id-ascending) and return the numbers of the issues
+ * and PRs that carry at least one trusted FORGE:TELEMETRY comment in that window. Items without one cost nothing more.
  */
-export function fetchGithubRecords({ repo, refresh = false, useCache = true, gh = defaultGh, env = process.env } = {}) {
+async function scanWindowComments(call, repo, from, env, onPage) {
+  const base = ["per_page=100", ...(from ? [`since=${sinceParam(from)}`] : [])];
+  const numbers = new Set();
+  for (let p = 1; p < MAX_LIST_REQUESTS; p++) {
+    const page = parsePages(await call(["api", `repos/${repo}/issues/comments?${[...base, `page=${p}`].join("&")}`]));
+    for (const c of page) {
+      if (!MARKER_RE.test(c?.body ?? "") || !isTrusted(c, env)) continue;
+      const m = /\/issues\/(\d+)$/.exec(c.issue_url ?? "");
+      if (m) numbers.add(Number(m[1]));
+    }
+    onPage(p);
+    if (page.length < PAGE_SIZE) break;
+  }
+  return numbers;
+}
+
+/**
+ * Fetch (or incrementally refresh) the repo's issues, PRs and trusted telemetry records. Async; `gh` may be sync or async.
+ * `since` (ISO) bounds the crawl: only items and comments updated on/after it are read, plus whatever the cache already
+ * holds. The cache records the lowest window start it covers (`coveredFrom`); a request that starts earlier re-crawls from
+ * its own start and then lowers the floor, so a later wider window is never under-reported.
+ * Returns { items, records, fetched, fromCache, apiCalls, coveredFrom }. `refresh` ignores the cache; `useCache:false`
+ * never touches disk. `onProgress({phase, done, total})` is called as the crawl moves (total is null when unknown).
+ */
+export async function fetchGithubRecords({ repo, since = null, refresh = false, useCache = true, gh = defaultGh, env = process.env,
+  onProgress = () => {}, concurrency = DEFAULT_CONCURRENCY, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now,
+  rateCheck = gh === defaultGh } = {}) {
   if (!isValidRepo(repo)) throw new Error(`invalid repo: ${repo}`);
   const cache = useCache && !refresh ? loadCache(repo, env) : emptyCache();
-  const startSince = cache.watermark ? shiftIso(cache.watermark, -WATERMARK_OVERLAP_MS) : null;
+  const start = since ?? null;
+  const hasCover = cache.watermark != null;
+  const floor = cache.coveredFrom;
+  // Does the cache already hold everything from `start` on? An unknown floor (a cache from before coveredFrom) is only
+  // trusted for a full-history request, never for a window.
+  const covers = hasCover && (floor === null || (floor === undefined ? start == null : start != null && Date.parse(start) >= Date.parse(floor)));
+  const crawlFrom = covers ? shiftIso(cache.watermark, -WATERMARK_OVERLAP_MS) : start;
+  const newFloor = covers ? (floor === undefined ? null : floor) : start;
+  const scanTs = isoNow(now());
+  let apiCalls = 0, guard = Promise.resolve();
+  const call = async (args) => {
+    if (rateCheck && apiCalls > 0 && apiCalls % RATE_CHECK_EVERY === 0) guard = guard.then(() => waitForRate(gh, sleep, now));
+    apiCalls++;
+    await guard;
+    return gh(args);
+  };
   const items = { ...cache.items };
   let watermark = cache.watermark;
   let fetched = 0;
-  // Checkpoint: the watermark is the newest updated_at actually processed, and the next run starts WATERMARK_OVERLAP_MS
-  // before it, so it is a safe resume point. A failure mid-crawl (rate limit, 5xx, a hook timeout) then keeps the
-  // progress instead of restarting at zero.
-  const checkpoint = () => { if (useCache) saveCache(repo, { ...emptyCache(), watermark, items }, env); };
+  let gapDone = false;
+  // Never persist a watermark past the moment the comment scan started: a comment posted after it bumps its issue's
+  // updated_at, and the next run must still start before that.
+  const cap = (w) => (w && w > scanTs ? scanTs : w);
+  // Telemetry issues whose records are not yet in `items` (the records are re-read in memory first). While any remain the
+  // watermark must not move past where the comment scan began, or a resumed run would never re-scan their comments.
+  const pending = new Set();
+  const scanResume = () => (crawlFrom ? shiftIso(crawlFrom, WATERMARK_OVERLAP_MS) : null);
+  // Checkpoint: while the crawl extends coverage forward (fresh cache, or the covered range's tail) the watermark is the
+  // newest updated_at processed. A gap crawl (window starts before the floor) leaves watermark and floor untouched until
+  // it finishes, so a failure never claims coverage it does not have.
+  const checkpoint = () => {
+    if (!useCache) return;
+    const extend = !hasCover || covers || gapDone;
+    saveCache(repo, { ...emptyCache(), watermark: extend ? (pending.size > 0 ? scanResume() : cap(watermark)) : cache.watermark, coveredFrom: extend ? newFloor : cache.coveredFrom, items }, env);
+  };
   try {
-    for (const page of listIssuePages(gh, repo, startSince)) for (const it of page) {
+    const telemetry = [...await scanWindowComments(call, repo, crawlFrom, env, (p) => onProgress({ phase: "comments", done: p, total: null }))];
+    // A telemetry run spans several comment parts and the window holds only the touched ones, so each such issue's full
+    // comment list is re-read once (bounded pool) and replaces its records.
+    const recordsOf = new Map();
+    for (const n of telemetry) pending.add(n);
+    let done = 0;
+    await pool(telemetry, concurrency, async (n) => {
+      recordsOf.set(n, parseTelemetryComments(parsePages(await call(["api", "--paginate", `repos/${repo}/issues/${n}/comments`])), env));
+      fetched++;
+      onProgress({ phase: "telemetry", done: ++done, total: telemetry.length });
+    });
+    const listed = new Set();
+    let processed = 0;
+    for await (const page of listIssuePages(call, repo, crawlFrom)) for (const it of page) {
       const m = meta(it);
-      let records = items[it.number]?.records ?? [];
-      // A telemetry comment is edited in place (which may not bump the issue's updated_at), so every listed item with
-      // comments is re-read rather than trusting a cached copy.
-      if ((it.comments ?? 0) > 0) {
-        records = parseTelemetryComments(parsePages(gh(["api", "--paginate", `repos/${repo}/issues/${it.number}/comments`])), env);
-        fetched++;
-      }
-      items[it.number] = { ...m, records };
+      items[it.number] = { ...m, records: recordsOf.get(it.number) ?? items[it.number]?.records ?? [] };
+      listed.add(it.number);
+      pending.delete(it.number);
       if (m.updatedAt && (!watermark || m.updatedAt > watermark)) watermark = m.updatedAt;
-      if (fetched > 0 && fetched % CHECKPOINT_EVERY === 0 && (it.comments ?? 0) > 0) checkpoint();
+      if (++processed % CHECKPOINT_EVERY === 0) checkpoint();
+      if (processed % PAGE_SIZE === 0) onProgress({ phase: "issues", done: processed, total: null });
     }
+    onProgress({ phase: "issues", done: processed, total: null });
+    // A telemetry comment edited in place does not bump its issue's updated_at, so such an issue can be missing from the
+    // listing: update its cached records, and read its metadata once if the cache has never seen it.
+    await pool(telemetry.filter((n) => !listed.has(n)), concurrency, async (n) => {
+      if (items[n]) { items[n] = { ...items[n], records: recordsOf.get(n) }; pending.delete(n); return; }
+      const [it] = parsePages(await call(["api", `repos/${repo}/issues/${n}`]));
+      if (it && Number.isInteger(it.number)) items[n] = { ...meta(it), records: recordsOf.get(n) };
+      pending.delete(n);
+    });
+    gapDone = true;
   } catch (err) {
     checkpoint();
     throw err;
@@ -198,7 +306,7 @@ export function fetchGithubRecords({ repo, refresh = false, useCache = true, gh
   for (const it of Object.values(items)) for (const r of it.records ?? []) {
     if (!seen.has(r.eid)) { seen.add(r.eid); records.push(r); }
   }
-  return { items: Object.values(items), records, fetched, fromCache: cache.watermark != null };
+  return { items: Object.values(items), records, fetched, fromCache: hasCover, apiCalls, coveredFrom: newFloor };
 }
 
 /** Aggregate-model inputs for one repo from a fetch result. */

package/bin/stats/index.mjs

@@ -22,10 +22,12 @@ export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <ow
 Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
 delta against the previous version and a low-n flag for samples under 10. Missing data prints n/a.
 
-  --since <v|date>      only versions >= <v> (e.g. 1.16.0) or activity on/after <date> (YYYY-MM-DD)
+  --since <v|date>      only versions >= <v> (e.g. 1.16.0) or activity on/after <date> (YYYY-MM-DD);
+                        also bounds the GitHub crawl to that window
   --repo <owner/repo>   target repo (default: forge.yaml project, else the git origin remote)
   --json                machine-readable output
   --refresh             ignore the local cache and re-read everything from GitHub
+  (progress is printed to stderr while crawling)
   --compact             one-row-per-version table (embedded in the staging review summary)
   --no-local            ignore ~/.forge/runs (FORGE_RUNS_DIR)
   --no-cache            do not read or write ~/.forge/stats-cache
@@ -78,6 +80,38 @@ export function parseReleaseTimes(text) {
     .filter((p) => p.length === 2 && /^\d+$/.test(p[1])).map((p) => ({ v: p[0].replace(/^v/, ""), t: Number(p[1]) }));
 }
 
+// A version --since becomes the tag's release time minus this margin: unversioned records are placed by release time, so
+// the crawl must start a little before the tag.
+const VERSION_SINCE_MARGIN_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Lower-bound timestamp for the GitHub crawl from `--since`: a date as is, a version as its tag time minus a margin.
+ * Returns { since: ISO | null, note }: since null means a full crawl (no --since, or a version with no known tag time).
+ */
+export function sinceToTimestamp(since, releaseTimes = []) {
+  const p = parseSince(since);
+  const iso = (ms) => new Date(ms).toISOString().replace(/\.\d{3}Z$/, "Z");
+  if (p.sinceMs != null) return { since: iso(p.sinceMs), note: null };
+  if (p.sinceVersion) {
+    const hit = releaseTimes.find((r) => r.v === p.sinceVersion);
+    if (hit) return { since: iso(hit.t * 1000 - VERSION_SINCE_MARGIN_MS), note: null };
+    return { since: null, note: `no release time known for ${p.sinceVersion}; crawling the full history (pass --release-times or use a date to bound it)` };
+  }
+  return { since: null, note: null };
+}
+
+/** Throttled stderr progress printer (never stdout, so --json stays parseable). */
+export function makeProgressPrinter(write = (t) => process.stderr.write(t), now = Date.now) {
+  let last = 0, lastPhase = null;
+  return ({ phase, done, total }) => {
+    const t = now();
+    const final = total != null && done >= total;
+    if (phase === lastPhase && !final && t - last < 2000) return;
+    last = t; lastPhase = phase;
+    write(`forgedock stats: ${phase} ${done}${total != null ? `/${total}` : ""}\n`);
+  };
+}
+
 function gitTagTimes() {
   try {
     const out = execFileSync("git", ["-C", join(here, "..", ".."), "for-each-ref", "refs/tags", "--format=%(refname:short) %(creatordate:unix)"],
@@ -107,21 +141,24 @@ export function resolveRepo(explicit, cwd = process.cwd()) {
   return null;
 }
 
-/** Build the report object. deps: { gh, env, localRuns, now } for tests. Never touches the network when `gh` is stubbed. */
-export function buildReport(opts, deps = {}) {
+/** Build the report object. deps: { gh, env, localRuns, now, onProgress, warn, sleep } for tests. Never touches the network when `gh` is stubbed. */
+export async function buildReport(opts, deps = {}) {
   const env = deps.env ?? process.env;
   const repo = opts.repo;
-  const gh = fetchGithubRecords({ repo, refresh: opts.refresh, useCache: opts.cache, gh: deps.gh, env });
+  let releaseTimes = [];
+  if (opts.releaseTimes) releaseTimes = parseReleaseTimes(readFileSync(opts.releaseTimes, "utf8"));
+  else if (deps.releaseTimes) releaseTimes = deps.releaseTimes;
+  else releaseTimes = gitTagTimes();
+  const bound = sinceToTimestamp(opts.since, releaseTimes);
+  if (bound.note) (deps.warn ?? ((m) => process.stderr.write(`forgedock stats: ${m}\n`)))(bound.note);
+  const gh = await fetchGithubRecords({ repo, since: bound.since, refresh: opts.refresh, useCache: opts.cache, gh: deps.gh, env,
+    onProgress: deps.onProgress, sleep: deps.sleep, now: deps.now });
   const model = toModel(repo, gh);
   const ghKeys = new Set(model.runs.map((r) => `${r.repo}#${r.issue}`));
   let local = [];
   if (opts.local) {
     local = (deps.localRuns ?? readLocalRuns(runsDir(env))).filter((r) => r.repo === repo && !ghKeys.has(`${r.repo}#${r.issue}`));
   }
-  let releaseTimes = [];
-  if (opts.releaseTimes) releaseTimes = parseReleaseTimes(readFileSync(opts.releaseTimes, "utf8"));
-  else if (deps.releaseTimes) releaseTimes = deps.releaseTimes;
-  else releaseTimes = gitTagTimes();
   const report = aggregate({ runs: [...model.runs, ...local], events: model.events, issues: model.issues },
     { minN: opts.minN, releaseTimes, ...parseSince(opts.since) });
   return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, ...report };
@@ -140,7 +177,7 @@ export async function runStats(argv, deps = {}) {
     return 2;
   }
   try {
-    const report = buildReport(opts, deps);
+    const report = await buildReport(opts, { onProgress: makeProgressPrinter(), ...deps });
     process.stdout.write(`${opts.json ? renderJson(report) : opts.compact ? renderCompact(report) : renderMarkdown(report)}\n`);
     return 0;
   } catch (err) {

package/bin/stats/render.mjs

@@ -55,7 +55,7 @@ function sections(g) {
   const kv = (o) => Object.entries(o).map(([a, b]) => `${a} ${b}`).join(", ") || NA;
   out.push("### Issue flow", ...table(["metric", "value"], [
     ["created by source", kv(i.created)], ["closed by kind", kv(i.closed)],
-    ["net flow (created - closed)", isNum(i.net) ? signed(i.net, String) : NA], ["findings per merged issue", f1(i.findingsPerMerge)],
+    ["net flow (created - closed)", isNum(i.net) ? signed(i.net, String) : NA], ["findings per merged issue", f1(i.findingsPerMerge)], ["introduced findings per merged issue", f1(i.introducedPerMerge)],
   ]), "");
   if (g.delta) {
     const d = g.delta;