@humwick/server 0.0.0-stage → 0.130.0

package/src/ephemeral/sweep.ts

@@ -0,0 +1,40 @@
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { DEFAULT_EPHEMERAL_MAX_AGE_MS } from "@humwick/sdk";
+import { ephemeralBaseDir, EPHEMERAL_DIR_PREFIX, isContainedUnder } from "./profile";
+import type { EphemeralHandle } from "./instance";
+
+function pidAlive(pid: number): boolean {
+  try { process.kill(pid, 0); return true; } catch (err) { return (err as NodeJS.ErrnoException).code !== "ESRCH"; }
+}
+
+// Run on the host that owns the ephemeral base. Unknown or incomplete records are
+// retained: neither age nor the absence of a handle proves an instance is dead.
+export async function sweepExpiredInstances(teardownRoot: (eroot: string, repo: string, src: string, tmux: string) => Promise<void>, log: (m: string) => void = (m) => console.error(m)): Promise<string[]> {
+  const base = ephemeralBaseDir();
+  if (!existsSync(base)) return [];
+  const removed: string[] = [];
+  for (const entry of readdirSync(base, { withFileTypes: true })) {
+    if (!entry.isDirectory() || !entry.name.startsWith(EPHEMERAL_DIR_PREFIX)) continue;
+    const eroot = join(base, entry.name);
+    let handle: EphemeralHandle;
+    try { handle = JSON.parse(readFileSync(join(eroot, "handle.json"), "utf8")) as EphemeralHandle; } catch { continue; }
+    const created = Date.parse(handle.createdAt);
+    const ttl = handle.maxAgeMs ?? DEFAULT_EPHEMERAL_MAX_AGE_MS;
+    if (!Number.isFinite(created) || !Number.isFinite(ttl) || ttl <= 0 || Date.now() - created <= ttl) continue;
+    if (!handle.pids || (handle.ownerPid !== undefined && (!Number.isSafeInteger(handle.ownerPid) || handle.ownerPid <= 0))) continue;
+    const pids = [handle.ownerPid, handle.pids.relay, handle.pids.orchestrator];
+    if (pids.some((pid) => pid !== undefined && (!Number.isSafeInteger(pid) || pid <= 0 || pidAlive(pid)))) continue;
+    const srcDir = join(eroot, "src");
+    const tmuxTmpdir = join(eroot, "tmux");
+    if (handle.eroot !== eroot || handle.srcDir !== srcDir || handle.tmuxTmpdir !== tmuxTmpdir ||
+        typeof handle.mainRepoRoot !== "string" || !isContainedUnder(srcDir, eroot) || !isContainedUnder(tmuxTmpdir, eroot)) continue;
+    try {
+      await teardownRoot(eroot, handle.mainRepoRoot, srcDir, tmuxTmpdir);
+      removed.push(entry.name.slice(EPHEMERAL_DIR_PREFIX.length));
+    } catch (err) {
+      log(`[ephemeral] expired cleanup ${entry.name} failed: ${String(err)}`);
+    }
+  }
+  return removed;
+}

package/src/event-loop-lag.ts

@@ -0,0 +1,584 @@
+import { errMessage } from "@humwick/sdk";
+import type { HealthCheck } from "./types";
+import {
+  EVENT_LOOP_LAG_LOG_MS,
+  EVENT_LOOP_LAG_P99_TARGET_MS,
+  EVENT_LOOP_LAG_PERCENTILE_WINDOW_SAMPLES,
+  EVENT_LOOP_LAG_SAMPLE_MS,
+  EVENT_LOOP_LAG_WARN_MS,
+  EVENT_LOOP_LAG_WINDOW_SAMPLES,
+  MAINTENANCE_BREAKER_LAG_MS,
+  ON_LOOP_BLOCK_REGRESSION_MS,
+} from "./config";
+import { classifyLoopGap, describeLoopGap, mainThreadBusyMs, readMainThreadClock, type LoopGapDiagnosis, type MainThreadClock } from "./main-thread-clock";
+import { describeFreezeSegment } from "./loop-task-stamps";
+
+/**
+ * #1883 (S7, ADR 0001 §10) — the epic's EXIT METRIC. `p50Ms`/`p95Ms`/`p99Ms` are `undefined` when
+ * the percentile window is empty; they are never faked to 0, because "no samples" and "a perfectly
+ * idle loop" are the same number and only one of them means the exit criterion was met.
+ */
+export interface EventLoopLagPercentiles {
+  p50Ms?: number;
+  p95Ms?: number;
+  p99Ms?: number;
+  /** Samples currently retained in the percentile window (fills to `percentileWindowSamples`). */
+  percentileSampleCount: number;
+  percentileWindowSamples: number;
+}
+
+export interface EventLoopLagSnapshot extends EventLoopLagPercentiles {
+  currentMs: number;
+  maxMs: number;
+  maxSinceLastReadMs: number;
+  /**
+   * #1974 — the ATTRIBUTED on-loop-JS ms of the worst spike in the retained ring, and (…SinceLastRead)
+   * since the last snapshot read. These MIRROR `maxMs`/`maxSinceLastReadMs` exactly (same ring window,
+   * same reset-on-read semantics) but count only the portion of each spike that overlapping on-loop JS
+   * frames account for — capped at the wall lag, so it can never exceed the spike itself. The health
+   * warn gates on THESE, not on the raw wall maxima, so a host swap/GC/page-fault stall (large wall
+   * lag, ~0 attributed) does not fire while a genuine on-loop block (attributed ≈ wall lag) still does.
+   */
+  maxAttributedOnLoopMs: number;
+  maxAttributedOnLoopSinceLastReadMs: number;
+  /** #3037 — main-thread busy ms (on-CPU + blocked) of the worst spike, from the thread clock; same
+   *  window/reset semantics as the attributed maxima. Busy but unattributed = uninstrumented on-loop work. */
+  maxMainThreadBusyMs: number;
+  maxMainThreadBusySinceLastReadMs: number;
+  sampleCount: number;
+  windowSamples: number;
+  sampleIntervalMs: number;
+  warnThresholdMs: number;
+  /** ADR 0001 §10's p99 < 100ms exit criterion, as configured. Reported, not enforced — see config. */
+  p99TargetMs: number;
+  lastSampleAt?: number;
+}
+
+/** One maintenance run's interval, as reported by the attribution seam below. Deliberately a
+ *  narrow, dependency-free shape (not `MaintenanceRunRecord` itself) so this file never needs an
+ *  import from `../maintenance` — see `setActivityAttributor`. `timedOut` carries the #1729 shadow
+ *  through to the log line: such a run's `finishedAt` is "when we gave up," not "when it stopped." */
+export interface EventLoopLagAttributedRun {
+  runId: string;
+  jobId: string;
+  startedAt: number;
+  finishedAt?: number;
+  timedOut?: boolean;
+}
+
+/**
+ * #1728 stage 0 — named seam the maintenance scheduler registers into so a detected lag spike can
+ * be attributed to whatever ran during it. Given the [gapStartAt, gapEndAt) window of a detected
+ * stall, return every run whose interval overlaps it (empty array = nothing was running).
+ *
+ * Seam direction is load-bearing and NON-NEGOTIABLE: this file is a low-level primitive imported
+ * from all over (health checks, reapers, stats); `../maintenance` imports plenty of that surface
+ * already, so this file importing `../maintenance` back would close an import cycle. The
+ * scheduler calls `setActivityAttributor` on itself instead — same reasoning `src/db/
+ * maintenance-health.ts` documents for why ITS check lives in the db layer rather than the
+ * maintenance layer.
+ */
+export type EventLoopLagActivityAttributor = (gapStartAt: number, gapEndAt: number) => EventLoopLagAttributedRun[];
+
+let activityAttributor: EventLoopLagActivityAttributor | undefined;
+
+export function setActivityAttributor(fn: EventLoopLagActivityAttributor | undefined): void {
+  activityAttributor = fn;
+}
+
+/**
+ * #1882 (S5, ADR 0001 §5, H7) — the SECOND registered attributor: per-callsite block attribution.
+ * Same seam pattern and same NON-NEGOTIABLE import direction as `setActivityAttributor` above — this
+ * low-level primitive must never import its callers, so `src/loop-block-attribution.ts` registers its
+ * callsite view ON this file instead (closing no import cycle).
+ *
+ * Where `setActivityAttributor` names the maintenance JOB that overlapped a spike, this names the
+ * originating synchronous `file:line` — the STATEMENT half #1868/T1843 called "impossible to debug".
+ * It returns a single human-readable clause describing the synchronous frame(s) active over the gap
+ * window (empty string / undefined = nothing recorded), which `describeAttribution` appends to the
+ * spike log line. It is a diagnostic enrichment ONLY: unlike the breaker observer it has no side
+ * effects and never decides the >50ms regression alarm (that fires from the capture layer, at the
+ * exact frame boundary, with the precise callsite — see `loop-block-attribution.ts`).
+ */
+export type EventLoopLagCallsiteAttributor = (gapStartAt: number, gapEndAt: number) => string | undefined;
+
+let callsiteAttributor: EventLoopLagCallsiteAttributor | undefined;
+
+export function setCallsiteAttributor(fn: EventLoopLagCallsiteAttributor | undefined): void {
+  callsiteAttributor = fn;
+}
+
+/**
+ * #1974 — the QUANTITATIVE sibling of the callsite attributor above. Where `callsiteAttributor`
+ * returns a human clause NAMING the on-loop frames over a spike, this returns the total MILLISECONDS
+ * of on-loop JS work attributed to a spike's [gapStartAt, gapEndAt) window — the same overlap query,
+ * summed instead of formatted. It is what makes the health warn (`src/db/stats.ts`) ATTRIBUTION-AWARE:
+ * a multi-second wall stall with near-zero attributed on-loop work is a host swap/GC/page-fault stall,
+ * not an humwick regression, and must not fire the warn; a spike whose attributed on-loop work
+ * approaches the wall lag is a genuine block and must (see the warn gate for the exact threshold).
+ *
+ * Same NON-NEGOTIABLE import direction and unconditional-registration pattern as the seams above:
+ * `src/loop-block-attribution.ts` owns the frame ring, so it registers this view ON this file. When no
+ * frames overlap (a pure host stall, or an UNINSTRUMENTED block the ring never saw) it returns 0 —
+ * which correctly suppresses the health warn. The separate >50ms `ON_LOOP_BLOCK_REGRESSION_MS` alarm
+ * (#1882) still SEES such a block; since #1975 it gates its escalation on this same attributed quantity,
+ * so an unattributed multi-second gap is logged as a non-alarming host stall rather than paging "a
+ * blocking call has been re-introduced" — instrument the path to restore the page (ADR 0001 §11.2).
+ */
+export type EventLoopLagOnLoopAttributor = (gapStartAt: number, gapEndAt: number) => number;
+
+let onLoopAttributor: EventLoopLagOnLoopAttributor | undefined;
+
+export function setOnLoopAttributor(fn: EventLoopLagOnLoopAttributor | undefined): void {
+  onLoopAttributor = fn;
+}
+
+/**
+ * #1882 (S5, ADR 0001 §5) — the >50ms on-loop-block REGRESSION seam. Same seam pattern and import
+ * direction as the breaker observer below: `src/loop-block-attribution.ts` registers ON this file. It
+ * is the sampler-driven BACKSTOP that catches any >50ms block — including uninstrumented ones the
+ * precise `withLoopFrameCapture` source never saw — and turns it into an alarm (not a warn). Fired for
+ * the [gapStartAt, gapEndAt] window of any sample at/above `ON_LOOP_BLOCK_REGRESSION_MS`.
+ *
+ * DISTINCT from the breaker observer: that fires at the much higher `MAINTENANCE_BREAKER_LAG_MS`
+ * (~1s freeze) and has side effects (abort/pause a maintenance job). This fires at the 50ms
+ * regression floor and is diagnostic — its registered observer dedupes against the precise frame
+ * source and escalates a REPEATEDLY-blocking callsite, but takes no action against maintenance jobs.
+ */
+export type EventLoopLagRegressionObserver = (gapStartAt: number, gapEndAt: number, lagMs: number, diagnosis?: LoopGapDiagnosis) => void;
+
+let regressionObserver: EventLoopLagRegressionObserver | undefined;
+
+export function setLagRegressionObserver(fn: EventLoopLagRegressionObserver | undefined): void {
+  regressionObserver = fn;
+}
+
+/**
+ * #1724 stage 1 — the circuit-breaker seam. Same seam pattern (and same non-negotiable direction) as
+ * `setActivityAttributor`: this low-level primitive must never import `../maintenance`, so the
+ * scheduler registers a breaker observer ON itself instead. It is called for the [gapStartAt,
+ * gapEndAt] window of any lag sample that reached MAINTENANCE_BREAKER_LAG_MS — a genuine freeze, not
+ * mere jitter — so the scheduler can attribute that freeze to whatever maintenance run overlapped it
+ * and take a breaker strike against that job (and abort it, if it is still in flight).
+ *
+ * DISTINCT from the attributor above, which only DESCRIBES a spike in a log line. This one is
+ * allowed to have side effects (abort a run, durably pause a job), so it is fired on a higher, breaker
+ * threshold rather than the diagnostic log threshold, and is a separate registration a caller can
+ * leave unset (e.g. ephemeral mode) with no effect.
+ */
+export type EventLoopLagBreakerObserver = (gapStartAt: number, gapEndAt: number, lagMs: number) => void;
+
+let breakerObserver: EventLoopLagBreakerObserver | undefined;
+
+export function setLagBreakerObserver(fn: EventLoopLagBreakerObserver | undefined): void {
+  breakerObserver = fn;
+}
+
+const samples = new Array<number>(EVENT_LOOP_LAG_WINDOW_SAMPLES).fill(0);
+// #1974 — the ATTRIBUTED on-loop-ms of each sample, kept in lockstep with `samples` (same index, same
+// window) so the health warn can ask "was the worst spike in the retained window a genuine on-loop
+// block?" the same way `maxMs` asks "how bad was the worst spike?". A sample below the warn threshold
+// stores 0 here (it can never trip the warn, so its attribution is not computed) — see the record path.
+const attributedOnLoopSamples = new Array<number>(EVENT_LOOP_LAG_WINDOW_SAMPLES).fill(0);
+const mainThreadBusySamples = new Array<number>(EVENT_LOOP_LAG_WINDOW_SAMPLES).fill(0);
+let sampleIndex = 0;
+let sampleCount = 0;
+// #1883 — the percentile ring. SEPARATE from `samples` above by design (see the config comment on
+// EVENT_LOOP_LAG_PERCENTILE_WINDOW_SAMPLES): `samples` is the reaper's grace window, and widening it
+// to a percentile-meaningful size would silently widen `eventLoopLagGraceMs()` and the stale-reaper
+// grace it feeds. Float64Array, pre-allocated once at module load and never resized — the hot path
+// does one indexed store and two integer updates, no allocation, no branch on content.
+const percentileSamples = new Float64Array(EVENT_LOOP_LAG_PERCENTILE_WINDOW_SAMPLES);
+let percentileIndex = 0;
+let percentileCount = 0;
+// Scratch buffer for the O(n log n) sort in `getEventLoopLagPercentiles`. Allocated once and reused
+// so a health poll never allocates 900 doubles; correctness does not depend on its retained content
+// (only the first `percentileCount` entries are copied in, and only those are sorted/read).
+const percentileScratch = new Float64Array(EVENT_LOOP_LAG_PERCENTILE_WINDOW_SAMPLES);
+let currentMs = 0;
+let maxSinceLastReadMs = 0;
+// #1974 — the attributed-on-loop analogue of `maxSinceLastReadMs`; reset in lockstep with it by
+// `getEventLoopLagSnapshot` so a slow /health poller that misses the ring window still sees the
+// attributed magnitude of the spike it is reading (exactly the guarantee `maxSinceLastReadMs` gives
+// for the wall lag).
+let maxAttributedOnLoopSinceLastReadMs = 0;
+let maxMainThreadBusySinceLastReadMs = 0;
+let lastSampleAt: number | undefined;
+let timer: ReturnType<typeof setInterval> | undefined;
+let lastTickAt = 0;
+let lastTickClock: MainThreadClock | undefined;
+
+export function computeEventLoopLagMs(elapsedMs: number, expectedMs: number): number {
+  if (!Number.isFinite(elapsedMs) || !Number.isFinite(expectedMs)) return 0;
+  return Math.max(0, elapsedMs - expectedMs);
+}
+
+/**
+ * `elapsedMs` is the real wall-clock width of the gap the sample was measured over (defaults to
+ * `lagMs` for callers, e.g. tests, that only care about the lag magnitude). It anchors the
+ * [sampledAt - elapsedMs, sampledAt] window handed to the attribution seam — NOT `lagMs` alone,
+ * which excludes the sampler's own expected tick spacing and would clip the start of the window.
+ *
+ * That window mixes two clock sources: `elapsedMs` comes from a `performance.now()` delta (the
+ * sampler's own monotonic tick timer) while `sampledAt` is a `Date.now()` instant. Both normally
+ * advance at the same rate, so subtracting one from the other to build a wall-clock window is
+ * sound — but an NTP step or host suspend/resume can step `Date.now()` without moving
+ * `performance.now()` (or vice versa), which would mis-size the window this attribution's
+ * accuracy depends on. No behaviour change here, just a disclosed assumption.
+ */
+export function recordEventLoopLagSample(
+  lagMs: number,
+  sampledAt: number = Date.now(),
+  logSpike = false,
+  elapsedMs: number = lagMs,
+  diagnosis?: LoopGapDiagnosis,
+): void {
+  const normalized = Number.isFinite(lagMs) ? Math.max(0, lagMs) : 0;
+  currentMs = normalized;
+  maxSinceLastReadMs = Math.max(maxSinceLastReadMs, normalized);
+  lastSampleAt = sampledAt;
+  // #1974 — attribute the ON-LOOP portion of this spike, but ONLY when it could actually trip the warn:
+  // a sample below the warn threshold can never warn (the attributed value is capped at the wall lag,
+  // so it too is below threshold), so we skip the ring overlap query for the ~1/s common small sample
+  // and store 0. For a spike at/above the threshold we ask the registered attributor how many ms of
+  // on-loop JS frames overlap this sample's [gapStart, gapEnd) window and cap it at the wall lag —
+  // on-loop work can never exceed the gap it was measured over. 0 (no overlapping frames) is the
+  // host-stall / uninstrumented-block signature and correctly yields no warn from this arm.
+  let attributedOnLoop = 0;
+  if (onLoopAttributor && EVENT_LOOP_LAG_WARN_MS > 0 && normalized >= EVENT_LOOP_LAG_WARN_MS) {
+    try {
+      const gapEndAt = sampledAt;
+      const gapStartAt = sampledAt - Math.max(0, elapsedMs);
+      const attributed = onLoopAttributor(gapStartAt, gapEndAt);
+      if (Number.isFinite(attributed) && attributed > 0) attributedOnLoop = Math.min(normalized, attributed);
+    } catch (error) {
+      // Same guard rationale as the breaker/regression observers below: a broken attributor must never
+      // kill the sampler that watches for the freeze. Degrade to 0 (no attribution) — never crash.
+      console.warn(`[health] event-loop-lag on-loop attributor threw: ${errMessage(error)}`);
+    }
+  }
+  maxAttributedOnLoopSinceLastReadMs = Math.max(maxAttributedOnLoopSinceLastReadMs, attributedOnLoop);
+  const busy = mainThreadBusyMs(normalized, diagnosis);
+  maxMainThreadBusySinceLastReadMs = Math.max(maxMainThreadBusySinceLastReadMs, busy);
+  samples[sampleIndex] = normalized;
+  attributedOnLoopSamples[sampleIndex] = attributedOnLoop;
+  mainThreadBusySamples[sampleIndex] = busy;
+  sampleIndex = (sampleIndex + 1) % samples.length;
+  sampleCount = Math.min(sampleCount + 1, samples.length);
+  // #1883 — same O(1) shape as the ring above; the percentile work is deferred entirely to read time.
+  percentileSamples[percentileIndex] = normalized;
+  percentileIndex = (percentileIndex + 1) % percentileSamples.length;
+  percentileCount = Math.min(percentileCount + 1, percentileSamples.length);
+  if (logSpike && EVENT_LOOP_LAG_LOG_MS > 0 && normalized >= EVENT_LOOP_LAG_LOG_MS) {
+    const gap = diagnosis ? ` — ${describeLoopGap(diagnosis)}${diagnosis.segment ? `; ${diagnosis.segment}` : ""}` : "";
+    console.warn(`[health] event-loop lag ${Math.round(normalized)}ms — ${describeAttribution(sampledAt, elapsedMs)}${gap}`);
+  }
+  // #1724 stage 1 — the breaker seam fires on a genuine freeze (>= the higher breaker threshold),
+  // independent of `logSpike`: a real spike measured by the sampler should trip the breaker whether
+  // or not this particular call opted into logging. The scheduler's observer is what decides what to
+  // do with it (attribute, abort, strike, pause). Guarded exactly like the log line: same
+  // [sampledAt - elapsedMs, sampledAt] window so the breaker attributes over the true gap width.
+  if (breakerObserver && MAINTENANCE_BREAKER_LAG_MS > 0 && normalized >= MAINTENANCE_BREAKER_LAG_MS) {
+    // The observer does real work (abort a run, write a pause row). It runs INSIDE the sampler's
+    // timer callback, so a throw here would become an uncaught exception in the interval — guard it:
+    // the breaker degrading is never worth killing the loop-lag sampler that watches for the freeze.
+    try {
+      breakerObserver(sampledAt - Math.max(0, elapsedMs), sampledAt, normalized);
+    } catch (error) {
+      console.warn(`[health] event-loop-lag breaker observer threw: ${errMessage(error)}`);
+    }
+  }
+  // #1882 stage S5 — the >50ms on-loop-block regression backstop. Fires INDEPENDENTLY of `logSpike`
+  // and of the breaker (this is a much lower threshold), guarded exactly like the breaker: same
+  // [sampledAt - elapsedMs, sampledAt] window and a swallowing try/catch so a broken alarm sink never
+  // kills the sampler. The registered observer dedupes against the precise frame-capture source.
+  if (regressionObserver && ON_LOOP_BLOCK_REGRESSION_MS > 0 && normalized >= ON_LOOP_BLOCK_REGRESSION_MS) {
+    try {
+      regressionObserver(sampledAt - Math.max(0, elapsedMs), sampledAt, normalized, diagnosis);
+    } catch (error) {
+      console.warn(`[health] event-loop-lag regression observer threw: ${errMessage(error)}`);
+    }
+  }
+}
+
+// #1728 stage 0 — diagnosis only: naming a cause here changes nothing about whether the stall
+// happens, only how fast an operator can find out why it happened after the fact.
+//
+// Three distinguishable cases, and the wording must not blur them: an exoneration ("no maintenance
+// job was running") is only ever emitted for the case that actually earns it. A confident
+// exoneration in the wrong case would be worse than no attribution at all — it would point an
+// operator away from the real cause in exactly the scenario this feature exists to catch.
+function describeAttribution(sampledAt: number, elapsedMs: number): string {
+  const gapEndAt = sampledAt;
+  const gapStartAt = sampledAt - Math.max(0, elapsedMs);
+  // #1882 — the per-callsite clause is appended to WHATEVER the maintenance-job verdict is: a spike can
+  // be BOTH "during: cd-watcher run" (the job) AND caused by a specific synchronous `file:line` (the
+  // statement). The two attributors are orthogonal — one names WHAT ran, the other WHERE it blocked —
+  // so they compose rather than one replacing the other.
+  const callsite = callsiteAttributor?.(gapStartAt, gapEndAt);
+  const callsiteSuffix = callsite ? ` — ${callsite}` : "";
+  const jobVerdict = describeMaintenanceAttribution(gapStartAt, gapEndAt);
+  return `${jobVerdict}${callsiteSuffix}`;
+}
+
+function describeMaintenanceAttribution(gapStartAt: number, gapEndAt: number): string {
+  if (!activityAttributor) return "no attribution available (no attributor registered)";
+  const overlapping = activityAttributor(gapStartAt, gapEndAt);
+  if (overlapping.length === 0) {
+    return "no measured maintenance block overlapped the gap";
+  }
+  return `during: ${overlapping.map((run) => {
+    const startedLabel = `started ${new Date(run.startedAt).toISOString()}`;
+    // A timed-out run's "may still be running" status is surfaced per-entry rather than folded
+    // into a single yes/no verdict, since a gap can overlap a mix of clean and timed-out runs.
+    const suffix = run.timedOut ? ", timed out — may still be running past its declared deadline" : "";
+    return `${run.jobId} (run ${run.runId}, ${startedLabel}${suffix})`;
+  }).join(", ")}`;
+}
+
+// Max lag over the retained ring buffer. SIDE-EFFECT-FREE: unlike
+// getEventLoopLagSnapshot it does NOT reset maxSinceLastReadMs, so a caller that
+// only needs the recent-lag ceiling (the stale reaper's grace, #1048) can read it
+// without stealing the spike the /health poller owns.
+export function eventLoopLagMaxMs(): number {
+  let maxMs = 0;
+  for (let i = 0; i < sampleCount; i += 1) maxMs = Math.max(maxMs, samples[i] ?? 0);
+  return maxMs;
+}
+
+// #1974 — max ATTRIBUTED on-loop ms over the retained ring, the attribution-aware analogue of
+// eventLoopLagMaxMs. SIDE-EFFECT-FREE for the same reason (never touches the since-read maxima). This
+// is the ring arm of the health warn; `maxAttributedOnLoopSinceLastReadMs` is the slow-poller arm.
+export function eventLoopLagMaxAttributedOnLoopMs(): number {
+  let maxMs = 0;
+  for (let i = 0; i < sampleCount; i += 1) maxMs = Math.max(maxMs, attributedOnLoopSamples[i] ?? 0);
+  return maxMs;
+}
+
+function eventLoopLagMaxMainThreadBusyMs(): number {
+  let maxMs = 0;
+  for (let i = 0; i < sampleCount; i += 1) maxMs = Math.max(maxMs, mainThreadBusySamples[i] ?? 0);
+  return maxMs;
+}
+
+// Cumulative retained lag — the SUM of every sample in the ring, i.e. the total
+// event-loop processing debt across the retained window. SIDE-EFFECT-FREE (like
+// eventLoopLagMaxMs it never touches maxSinceLastReadMs, so reading it doesn't steal
+// the /health poller's spike). This is the stale/bus reaper's grace (#1055): the max
+// SINGLE sample (#1048) covers one contiguous stall (the 337s incident) but UNDER-
+// covers sustained NON-contiguous degradation — many sub-TTL stalls whose backlog
+// still sums past STALE_TTL. Summing every stall in the window extends the grace to the
+// real accumulated heartbeat backlog. Always >= eventLoopLagMaxMs(), so it can only
+// WIDEN the #1048 grace, never shrink it — no new false-reap risk introduced.
+export function eventLoopLagGraceMs(): number {
+  let total = 0;
+  for (let i = 0; i < sampleCount; i += 1) total += samples[i] ?? 0;
+  return total;
+}
+
+/**
+ * #1883 — NEAREST-RANK percentile over an ASCENDING-sorted slice: `rank = ceil(p/100 × n)`, value =
+ * `sorted[rank - 1]`, clamped into `[0, n - 1]`.
+ *
+ * The definition is stated (and pinned by tests) rather than left implicit, because the epic's exit
+ * criterion is a NUMBER COMPARED TO A TARGET — if this instrument silently used a different
+ * convention than whoever reads the metric assumes, the criterion would be judged against the wrong
+ * quantity. Nearest-rank is chosen over linear interpolation for two reasons: every reported value is
+ * a lag figure the sampler ACTUALLY MEASURED (an interpolated p99 is a number that never happened,
+ * which is a poor thing to alarm on), and it degrades honestly on a short window — with n < 100 the
+ * p99 simply IS the max, rather than a fabricated blend that looks more precise than the data.
+ *
+ * `n` is passed explicitly because the caller's array is a fixed-size ring scratch buffer whose tail
+ * is stale; only its first `n` entries are meaningful. Returns `undefined` for an empty window —
+ * never 0, which would be indistinguishable from a perfectly idle loop.
+ */
+export function nearestRankPercentile(sortedAsc: ArrayLike<number>, n: number, percentile: number): number | undefined {
+  if (!Number.isFinite(n) || n <= 0) return undefined;
+  const count = Math.min(Math.floor(n), sortedAsc.length);
+  if (count <= 0) return undefined;
+  const p = Math.min(100, Math.max(0, percentile));
+  const rank = Math.ceil((p / 100) * count);
+  const index = Math.min(count - 1, Math.max(0, rank - 1));
+  return sortedAsc[index];
+}
+
+/**
+ * #1883 — p50/p95/p99 over the retained percentile window. SIDE-EFFECT-FREE (like eventLoopLagMaxMs
+ * and unlike getEventLoopLagSnapshot, it never touches maxSinceLastReadMs), so any caller may read it
+ * without stealing the /health poller's spike.
+ *
+ * Cost is paid HERE, not on the sampling hot path: copy the live prefix of the ring into the reusable
+ * scratch buffer and sort it (O(n log n), n = 900 by default, no allocation). Ring ORDER is
+ * irrelevant to a percentile, so the wrap point needs no un-rotating — only the first
+ * `percentileCount` slots are ever populated, and a full ring populates all of them.
+ */
+export function getEventLoopLagPercentiles(): EventLoopLagPercentiles {
+  const n = percentileCount;
+  const base: EventLoopLagPercentiles = {
+    percentileSampleCount: n,
+    percentileWindowSamples: percentileSamples.length,
+  };
+  if (n === 0) return base;
+  const view = percentileScratch.subarray(0, n);
+  view.set(percentileSamples.subarray(0, n));
+  view.sort();
+  return {
+    ...base,
+    p50Ms: nearestRankPercentile(view, n, 50),
+    p95Ms: nearestRankPercentile(view, n, 95),
+    p99Ms: nearestRankPercentile(view, n, 99),
+  };
+}
+
+/**
+ * #1883 — the health-payload projection of a snapshot, kept HERE beside the fields it projects so a
+ * new metric cannot be added to the snapshot and silently miss the operator surface (and so the
+ * health check in `src/db/stats.ts` carries the semantics, not the field list). `HealthCheck.metrics`
+ * is `Record<string, number>`, so genuinely-absent values are OMITTED rather than coerced to 0.
+ */
+export function eventLoopLagHealthMetrics(snapshot: EventLoopLagSnapshot): Record<string, number> {
+  const metrics: Record<string, number> = {
+    currentMs: snapshot.currentMs,
+    maxMs: snapshot.maxMs,
+    maxSinceLastReadMs: snapshot.maxSinceLastReadMs,
+    maxAttributedOnLoopMs: snapshot.maxAttributedOnLoopMs,
+    maxAttributedOnLoopSinceLastReadMs: snapshot.maxAttributedOnLoopSinceLastReadMs,
+    maxMainThreadBusyMs: snapshot.maxMainThreadBusyMs,
+    maxMainThreadBusySinceLastReadMs: snapshot.maxMainThreadBusySinceLastReadMs,
+    sampleCount: snapshot.sampleCount,
+    windowSamples: snapshot.windowSamples,
+    sampleIntervalMs: snapshot.sampleIntervalMs,
+    warnThresholdMs: snapshot.warnThresholdMs,
+    percentileSampleCount: snapshot.percentileSampleCount,
+    percentileWindowSamples: snapshot.percentileWindowSamples,
+    p99TargetMs: snapshot.p99TargetMs,
+  };
+  if (snapshot.p50Ms !== undefined) metrics.p50Ms = snapshot.p50Ms;
+  if (snapshot.p95Ms !== undefined) metrics.p95Ms = snapshot.p95Ms;
+  if (snapshot.p99Ms !== undefined) metrics.p99Ms = snapshot.p99Ms;
+  if (snapshot.lastSampleAt !== undefined) metrics.lastSampleAt = snapshot.lastSampleAt;
+  return metrics;
+}
+
+/**
+ * #1974 — classify the event-loop-lag health check from an already-captured snapshot. Keeping this
+ * pure lets the health route report the production sampler unchanged while tests exercise the
+ * attribution contract without a process-global sampler or aggregate health-report coupling.
+ */
+export function evaluateEventLoopLagCheck(eventLoopLag: EventLoopLagSnapshot): HealthCheck {
+  // The warn is ATTRIBUTION-AWARE. The raw wall maxima are still REPORTED, but the gate fires on the
+  // ATTRIBUTED on-loop-JS magnitude of the worst spike, not raw wall lag: a host swap/page-fault
+  // stall can have multi-second wall lag with near-zero attributed on-loop work and must not warn.
+  // A genuine on-loop block attributes approximately its full wall lag and still warns, including a
+  // single severe one-off. The same threshold as before applies; only the quantity changed in #1974.
+  //
+  // An UNINSTRUMENTED on-loop block records no frame, so it attributes 0 and does not warn HERE. The
+  // separate >50ms ON_LOOP_BLOCK_REGRESSION_MS alarm still OBSERVES it (a non-alarming host-stall line
+  // since #1975, which applies this same attribution gate to that alarm's escalation) — so the block is
+  // never invisible, but naming it requires instrumenting the path (ADR 0001 §11.2).
+  const eventLoopLagHealthMaxMs = Math.max(eventLoopLag.maxMs, eventLoopLag.maxSinceLastReadMs);
+  const eventLoopLagAttributedMs = Math.max(eventLoopLag.maxAttributedOnLoopMs, eventLoopLag.maxAttributedOnLoopSinceLastReadMs);
+  const eventLoopLagWarn = eventLoopLag.warnThresholdMs > 0 && eventLoopLag.sampleCount > 0 && eventLoopLagAttributedMs >= eventLoopLag.warnThresholdMs;
+  // A large wall spike whose attributed on-loop work is below the threshold is a non-alarming host
+  // stall. Keep it visible to operators without degrading the report.
+  const eventLoopLagHostStall = !eventLoopLagWarn && eventLoopLag.sampleCount > 0
+    && eventLoopLag.warnThresholdMs > 0 && eventLoopLagHealthMaxMs >= eventLoopLag.warnThresholdMs;
+  // #3037 — a stall the main thread itself was busy for is uninstrumented on-loop work, not the host.
+  const eventLoopLagBusyMs = Math.max(eventLoopLag.maxMainThreadBusyMs, eventLoopLag.maxMainThreadBusySinceLastReadMs);
+  const stallKind = eventLoopLagBusyMs >= eventLoopLag.warnThresholdMs
+    ? `unattributed on-loop work (non-alarming): main thread busy ${Math.round(eventLoopLagBusyMs)}ms,`
+    : "host stall (non-alarming):";
+  const eventLoopLagP99Detail = eventLoopLag.p99Ms === undefined ? "" : `; p99 ${Math.round(eventLoopLag.p99Ms)}ms over ${eventLoopLag.percentileSampleCount}/${eventLoopLag.percentileWindowSamples} sample(s), target <${eventLoopLag.p99TargetMs}ms`;
+
+  return {
+    name: "event-loop-lag",
+    status: eventLoopLagWarn ? "warn" : "ok",
+    deployRole: "load",
+    detail: eventLoopLagWarn
+      ? `event loop on-loop block ${Math.round(eventLoopLagAttributedMs)}ms attributed (of ${Math.round(eventLoopLagHealthMaxMs)}ms wall lag) over ${eventLoopLag.sampleCount}/${eventLoopLag.windowSamples} sample(s)${eventLoopLagP99Detail}`
+      : eventLoopLagHostStall
+        ? `${stallKind} ${Math.round(eventLoopLagHealthMaxMs)}ms wall lag, only ${Math.round(eventLoopLagAttributedMs)}ms attributed on-loop — below the ${eventLoopLag.warnThresholdMs}ms on-loop warn threshold${eventLoopLagP99Detail}`
+        : undefined,
+    metrics: eventLoopLagHealthMetrics(eventLoopLag),
+  };
+}
+
+export function getEventLoopLagSnapshot(): EventLoopLagSnapshot {
+  const maxMs = eventLoopLagMaxMs();
+  const maxSinceLastRead = maxSinceLastReadMs;
+  maxSinceLastReadMs = 0;
+  // #1974 — read AND reset the attributed since-read max in the SAME step as the wall since-read max, so
+  // the two never drift: a read that consumes a wall spike must also consume its attributed magnitude.
+  const maxAttributedOnLoop = eventLoopLagMaxAttributedOnLoopMs();
+  const maxAttributedOnLoopSinceLastRead = maxAttributedOnLoopSinceLastReadMs;
+  maxAttributedOnLoopSinceLastReadMs = 0;
+  const maxMainThreadBusySinceLastRead = maxMainThreadBusySinceLastReadMs;
+  maxMainThreadBusySinceLastReadMs = 0;
+  return {
+    currentMs,
+    maxMs,
+    maxSinceLastReadMs: maxSinceLastRead,
+    maxAttributedOnLoopMs: maxAttributedOnLoop,
+    maxAttributedOnLoopSinceLastReadMs: maxAttributedOnLoopSinceLastRead,
+    maxMainThreadBusyMs: eventLoopLagMaxMainThreadBusyMs(),
+    maxMainThreadBusySinceLastReadMs: maxMainThreadBusySinceLastRead,
+    sampleCount,
+    windowSamples: EVENT_LOOP_LAG_WINDOW_SAMPLES,
+    sampleIntervalMs: EVENT_LOOP_LAG_SAMPLE_MS,
+    warnThresholdMs: EVENT_LOOP_LAG_WARN_MS,
+    ...getEventLoopLagPercentiles(),
+    p99TargetMs: EVENT_LOOP_LAG_P99_TARGET_MS,
+    lastSampleAt,
+  };
+}
+
+export function startEventLoopLagSampler(): void {
+  if (timer) return;
+  lastTickAt = performance.now();
+  lastTickClock = readMainThreadClock();
+  timer = setInterval(() => {
+    const now = performance.now();
+    const elapsedMs = now - lastTickAt;
+    const lagMs = computeEventLoopLagMs(elapsedMs, EVENT_LOOP_LAG_SAMPLE_MS);
+    lastTickAt = now;
+    const clock = readMainThreadClock();
+    const diagnosis = diagnoseLagGap(lagMs, now, lastTickClock, clock);
+    lastTickClock = clock;
+    recordEventLoopLagSample(lagMs, Date.now(), true, elapsedMs, diagnosis);
+  }, EVENT_LOOP_LAG_SAMPLE_MS);
+  timer.unref?.();
+}
+
+/** #3037 — main-thread verdict + the stamped segment for a lag at/above the regression floor. */
+export function diagnoseLagGap(lagMs: number, nowPerf: number, before?: MainThreadClock, after?: MainThreadClock): LoopGapDiagnosis | undefined {
+  if (ON_LOOP_BLOCK_REGRESSION_MS <= 0 || lagMs < ON_LOOP_BLOCK_REGRESSION_MS || !before || !after) return undefined;
+  const diagnosis = classifyLoopGap(lagMs, before, after);
+  const segment = describeFreezeSegment(nowPerf - lagMs, nowPerf, ON_LOOP_BLOCK_REGRESSION_MS);
+  return segment ? { ...diagnosis, segment } : diagnosis;
+}
+
+export function stopEventLoopLagSampler(): void {
+  if (!timer) return;
+  clearInterval(timer);
+  timer = undefined;
+}
+
+export function resetEventLoopLagForTests(): void {
+  stopEventLoopLagSampler();
+  samples.fill(0);
+  attributedOnLoopSamples.fill(0);
+  mainThreadBusySamples.fill(0);
+  sampleIndex = 0;
+  sampleCount = 0;
+  percentileSamples.fill(0);
+  percentileIndex = 0;
+  percentileCount = 0;
+  percentileScratch.fill(0);
+  currentMs = 0;
+  maxSinceLastReadMs = 0;
+  maxAttributedOnLoopSinceLastReadMs = 0;
+  maxMainThreadBusySinceLastReadMs = 0;
+  lastSampleAt = undefined;
+  lastTickAt = 0;
+  lastTickClock = undefined;
+}

package/src/event-loop-yield.ts

@@ -0,0 +1,27 @@
+/** One macrotask hop. `await` on synchronous work only drains microtasks, so a paced loop must hop a
+ *  macrotask to let request handlers, the event-loop-lag sampler and abort callbacks run between
+ *  its units (#1725 P0, #1835). */
+export function yieldToEventLoop(): Promise<void> {
+  return new Promise((resolve) => setImmediate(resolve));
+}
+
+/** #2712 — a maintenance sweep's paced form: hop BEFORE enumerating (so the scheduler's
+ *  synchronous-prefix lastBlockMs holds no sweep work), then one hop after every unit. Stops once
+ *  `budgetMs` is spent or `ready()` says no (e.g. another writer holds the lock) and reports
+ *  `moreWork`, so a backlog drains over runs instead of overrunning the deadline (a breaker strike). */
+export async function forEachPaced<T>(
+  items: () => Iterable<T>,
+  unit: (item: T) => void,
+  opts: { signal?: AbortSignal; budgetMs: number; ready?: () => boolean },
+): Promise<{ moreWork: boolean }> {
+  await yieldToEventLoop();
+  if (opts.signal?.aborted) return { moreWork: false };
+  const stopAt = performance.now() + opts.budgetMs;
+  for (const item of items()) {
+    if (opts.signal?.aborted) return { moreWork: false };
+    if (performance.now() >= stopAt || opts.ready?.() === false) return { moreWork: true };
+    unit(item);
+    await yieldToEventLoop();
+  }
+  return { moreWork: false };
+}