@humwick/server 0.130.1 → 0.130.3

package/src/db/writer-gateway.ts

@@ -42,17 +42,16 @@ import { WriterDrain } from "./writer-drain";
 
 import { errMessage } from "@humwick/sdk";
 import { createWorkerPool, type WorkerPool } from "@humwick/offload";
-import { getDbGeneration, getDbPath } from "./connection.ts";
-import { armedWriterFaultCount } from "./writer-fault-seam.ts";
-import { onLoopWriteTransaction } from "./on-loop-write-transaction.ts";
+import { getDbPath } from "./connection.ts";
 // On-loop-block attribution (#1882 / ADR §5). `resolveLoopCallsite` pins the effective callsite onto a
 // submission in the CALLER's context; `captureSyncFrame` times the op body where it actually runs — in
 // the in-process drain microtask, NOT the synchronous enqueue. One-way import (loop-block-attribution
 // imports only event-loop-lag + config), so no cycle.
 import { captureSyncFrame, resolveLoopCallsite } from "../loop-block-attribution.ts";
+import { markCallerResume } from "../hub-metrics.ts";
 import { reconstructDbError, serializeDbError } from "./db-error-registry.ts";
 import {
-  isIdempotentWriterOp,
+  writerOpAllowsKeyless,
   isOffLoopWriterOp,
   runWriterBatch,
   runWriterOp,
@@ -110,7 +109,7 @@ export class WriteIndeterminateError extends Error {
 // ── Options & per-call opts ───────────────────────────────────────────────────────────────────────
 
 export interface InitWriterOptions {
-  /** Overrides the path from `getDbPath()`; `:memory:`/undefined selects the in-process backend. */
+  /** File path to attach. A live file DB always uses its worker, including legacy :memory: requests. */
   path?: string;
   /** Soft high-water mark on the writer queue; past it, non-critical writes shed (§3.6). */
   queueHighWater?: number;
@@ -174,14 +173,18 @@ export interface WriterStats {
    * SQLite lock (H4 §3.6.1). A wedged writer is NOT idle — this is deliberately distinct from
    * `busy:0`, which is how the naive pool-replace path mis-reported the stall. New non-critical
    * writes shed and critical writes are held until the wedged op completes and the FIFO resumes.
+   * #3776 — names the op holding the writer and how long it has run, so health says WHICH job; null when not wedged.
    */
-  wedged: boolean;
+  wedged: { op: string; runningMs: number } | null;
   /**
    * #2276 — how many times the IN-PROCESS drain has yielded the event loop mid-convoy because its
    * time slice was spent. 0 on the worker backend (nothing to slice). A steadily climbing count is
    * the observable signal that write convoys are long enough to need the throttle.
    */
   drainYields: number;
+  /** #3798 — writes executed since boot on the main thread and on a writer thread (a batch counts its entries). */
+  onLoopOps: number;
+  writerThreadOps: number;
 }
 
 // ── Backend state ─────────────────────────────────────────────────────────────────────────────────
@@ -242,8 +245,6 @@ interface InProcessBackend {
   kind: "in-process";
   stop: AbortController;
   busy: boolean;
-  offLoop?: WorkerBackend;
-  offLoopGeneration?: number;
 }
 type Backend = WorkerBackend | InProcessBackend;
 
@@ -255,26 +256,28 @@ let writerTimeoutMs = DEFAULT_WRITER_TIMEOUT_MS;
 let drainSliceMs = DEFAULT_DRAIN_SLICE_MS;
 // Monotonic across re-inits (like any counter): a test that wants a delta reads it before and after.
 let drainYields = 0;
+let onLoopOps = 0;
+let writerThreadOps = 0;
 // Set while a writer job has blown its gateway deadline and the sole writer is stalled holding the
 // lock (§3.6.1). Cleared when that in-flight job finally settles (the op completed and released the
 // lock), which re-kicks the FIFO drain. Worker backend only — an in-process op runs synchronously on
 // the main thread, so it cannot be observed mid-stall.
-let wedged = false;
+let wedgedOp: { op: string; startedAt: number } | undefined;
 
 // Test-only fault injection for the precise H5 window: the op has COMMITTED, but its
 // acknowledgement is lost before the caller observes the result. One-shot by design so a retry
 // exercises the persisted idempotency ledger rather than repeatedly failing at the transport edge.
-let postCommitAckLossForTests: { op: string; remaining: number } | undefined;
+let postCommitAckLossForTests: { op: string } | undefined;
 
 export function rejectNextWriterAckAfterCommitForTests(op: string): void {
-  postCommitAckLossForTests = { op, remaining: 1 };
+  postCommitAckLossForTests = { op };
 }
 
-function losePostCommitAckForTests(op: string): boolean {
-  if (!postCommitAckLossForTests || postCommitAckLossForTests.op !== op) return false;
-  postCommitAckLossForTests.remaining -= 1;
-  if (postCommitAckLossForTests.remaining <= 0) postCommitAckLossForTests = undefined;
-  return true;
+function resolveWriterOp(entry: Extract<Submission, { kind: "op" }>, result: unknown): void {
+  if (postCommitAckLossForTests?.op === entry.op) {
+    postCommitAckLossForTests = undefined;
+    entry.reject(new WriteIndeterminateError(entry.op, new Error("injected post-commit lost acknowledgement"), entry.opts?.idempotencyKey));
+  } else entry.resolve(result);
 }
 
 /**
@@ -283,7 +286,8 @@ function losePostCommitAckForTests(op: string): boolean {
  * size-1 writer pool.
  */
 export function initWriter(options: InitWriterOptions = {}): void {
-  const path = options.path ?? getDbPath();
+  // A file-backed main handle never executes writer bodies, even when an old caller requests memory.
+  const path = options.path && options.path !== ":memory:" ? options.path : getDbPath() ?? options.path;
   // EVERY caller-supplied number is validated BEFORE any of them is assigned: a RangeError thrown
   // after an earlier field was already written would leave the module in a half-applied config the
   // caller never asked for.
@@ -355,14 +359,14 @@ function createFileWriter(dbPath: string): WorkerBackend {
 
 /** Server shutdown keeps already-admitted writes; test/reset teardown remains immediate. */
 export async function drainWriterOnShutdown(): Promise<void> {
-  try { await shutdownDrain.start(() => !draining && !wedged && pendingWriteDepth() === 0 && inFlightWrites() === 0); }
+  try { await shutdownDrain.start(() => !draining && !wedgedOp && pendingWriteDepth() === 0 && inFlightWrites() === 0); }
   finally { await shutdownWriter(); }
 }
 
 export async function shutdownWriter(): Promise<void> {
   const current = backend;
   backend = undefined;
-  wedged = false;
+  wedgedOp = undefined;
   // Reject anything still queued at the gateway so no caller hangs across a teardown.
   const stranded = submissionQueue.splice(0, submissionQueue.length);
   for (const entry of stranded) {
@@ -370,7 +374,6 @@ export async function shutdownWriter(): Promise<void> {
   }
   if (current?.kind === "in-process") current.stop.abort(new Error("writer gateway shut down"));
   if (current?.kind === "worker") await current.pool.destroy();
-  if (current?.kind === "in-process") await current.offLoop?.pool.destroy();
 }
 
 /** Whether the gateway is initialized (some callers guard bootstrap ordering on this). */
@@ -379,12 +382,14 @@ export function isWriterInitialized(): boolean {
 }
 
 export function writerStats(): WriterStats {
-  if (!backend) return { backend: "uninitialized", queued: 0, busy: 0, highWater: queueHighWater, wedged: false, drainYields };
+  const ops = { drainYields, onLoopOps, writerThreadOps };
+  if (!backend) return { backend: "uninitialized", queued: 0, busy: 0, highWater: queueHighWater, wedged: null, ...ops };
+  const wedged = wedgedOp ? { op: wedgedOp.op, runningMs: Math.round(performance.now() - wedgedOp.startedAt) } : null;
   if (backend.kind === "in-process") {
-    return { backend: "in-process", queued: pendingWriteDepth(), busy: inFlightWrites(), highWater: queueHighWater, wedged, drainYields };
+    return { backend: "in-process", queued: pendingWriteDepth(), busy: inFlightWrites(), highWater: queueHighWater, wedged, ...ops };
   }
   const stats = backend.pool.stats;
-  return { backend: "worker", queued: pendingWriteDepth(), busy: stats.busy, highWater: queueHighWater, wedged, drainYields };
+  return { backend: "worker", queued: pendingWriteDepth(), busy: stats.busy, highWater: queueHighWater, wedged, ...ops };
 }
 
 function requireBackend(): Backend {
@@ -448,7 +453,7 @@ function pendingWriteDepth(): number {
 
 /** Includes an in-process writer waiting asynchronously for BEGIN. */
 function inFlightWrites(): number {
-  return backend?.kind === "worker" ? backend.pool.stats.busy : Math.max(Number(backend?.busy ?? false), backend?.offLoop?.pool.stats.busy ?? 0);
+  return backend?.kind === "worker" ? backend.pool.stats.busy : Number(backend?.busy ?? false);
 }
 
 /** Reject every caller attached to a submission (used for teardown / dispatch-time backend loss). */
@@ -461,7 +466,7 @@ function rejectEntry(entry: Submission, error: unknown): void {
  *  before the write is ever submitted, so a keyless non-idempotent write can never be retried into a
  *  duplicate. `key` is the per-write key (the batch/coalesce paths pass each entry's own key). */
 function guardIdempotency(op: string, key: string | undefined): void {
-  if (key === undefined && !isIdempotentWriterOp(op)) throw new WriteRetrySafetyError(op);
+  if (key === undefined && !writerOpAllowsKeyless(op)) throw new WriteRetrySafetyError(op);
 }
 
 /** Backpressure admission (§3.6/§3.6.1). `count` is the number of writes this submission would ADD (1
@@ -480,7 +485,7 @@ function admitOrThrow(op: string, opts: WriteOpts | undefined, count: number): v
   const occupancy = pendingWriteDepth() + inFlightWrites();
   if (occupancy + count > queueHardCap) throw new WriterOverloadedError(op, occupancy);
   if (opts?.critical) return;
-  const workerWedged = wedged;
+  const workerWedged = wedgedOp !== undefined;
   if (workerWedged || occupancy + count > queueHighWater) throw new WriterOverloadedError(op, occupancy);
 }
 
@@ -511,7 +516,7 @@ async function drainLoop(): Promise<void> {
     // second concurrent drain and anything submitted meanwhile is picked up by this same loop when
     // it resumes — FIFO is preserved, the queue is only PAUSED.
     let sliceStartedAt = performance.now();
-    while (submissionQueue.length > 0 && !wedged) {
+    while (submissionQueue.length > 0 && !wedgedOp) {
       const entry = submissionQueue.shift()!;
       const onLoop = backend?.kind === "in-process";
       await dispatchEntry(entry);
@@ -521,7 +526,7 @@ async function drainLoop(): Promise<void> {
         sliceStartedAt = performance.now();
         continue;
       }
-      if (submissionQueue.length === 0 || wedged) continue;
+      if (submissionQueue.length === 0 || wedgedOp) continue;
       if (performance.now() - sliceStartedAt < drainSliceMs) continue;
       drainYields += 1;
       await yieldEventLoop();
@@ -529,7 +534,7 @@ async function drainLoop(): Promise<void> {
     }
   } finally {
     draining = false;
-    shutdownDrain.notify(!wedged && pendingWriteDepth() === 0 && inFlightWrites() === 0);
+    shutdownDrain.notify(!wedgedOp && pendingWriteDepth() === 0 && inFlightWrites() === 0);
   }
 }
 
@@ -551,19 +556,9 @@ async function dispatchEntry(entry: Submission): Promise<void> {
   if (back.kind === "in-process") {
     back.busy = true;
     try {
-      const path = getDbPath();
-      // An armed crash seam is a closure on THIS thread, so while one is armed the op runs here instead
-      // of on the file-DB worker (tests only; production never arms). Otherwise the fault would never fire.
-      if (path && path !== ":memory:" && isOffLoopWriterOp(entry.op) && armedWriterFaultCount() === 0) {
-        if (back.offLoop && back.offLoopGeneration !== getDbGeneration()) {
-          await back.offLoop.pool.destroy();
-          back.offLoop = undefined;
-        }
-        back.stop.signal.throwIfAborted();
-        back.offLoop ??= createFileWriter(path);
-        back.offLoopGeneration = getDbGeneration();
-        await dispatchWorker(entry, back.offLoop);
-      } else await dispatchInProcess(entry, back.stop.signal);
+      back.stop.signal.throwIfAborted();
+      onLoopOps += entryWrites(entry);
+      await dispatchInProcess(entry);
     } catch (error) { rejectEntry(entry, error); }
     finally { back.busy = false; }
     return;
@@ -571,6 +566,10 @@ async function dispatchEntry(entry: Submission): Promise<void> {
   await dispatchWorker(entry, back);
 }
 
+function entryWrites(entry: Submission): number {
+  return entry.kind === "op" ? 1 : entry.argsList.length;
+}
+
 /**
  * The per-entry dedup keys for a batch/coalesce job (H5 §3.7.1). A dbWriteBatch carries ONE key for
  * the whole call and derives a stable per-entry key `${key}#${i}`, so a blind retry of the same batch
@@ -598,18 +597,12 @@ function entryKeys(entry: Extract<Submission, { kind: "batch" | "coalesce" }>):
  * (`dispatchWorker`) is deliberately NOT wrapped: its op body runs off-thread, so there is no on-loop
  * block to attribute (only the sub-floor enqueue, which the record floor drops).
  */
-async function dispatchInProcess(entry: Submission, signal: AbortSignal): Promise<void> {
-  const execute = <T>(fn: () => T) => onLoopWriteTransaction(
-    fn, (attempt) => captureSyncFrame({ op: entry.op, callsite: entry.callsite }, attempt), writerTimeoutMs, signal,
-  );
+async function dispatchInProcess(entry: Submission): Promise<void> {
+  const execute = <T>(fn: () => T) => captureSyncFrame({ op: entry.op, callsite: entry.callsite }, fn);
   if (entry.kind === "op") {
     try {
-      const result = cloneIn(await execute(() => runWriterOp(entry.op, cloneIn(entry.args), { now: entry.now, presence: cloneIn(entry.presence) }, entry.opts?.idempotencyKey)));
-      if (losePostCommitAckForTests(entry.op)) {
-        entry.reject(new WriteIndeterminateError(entry.op, new Error("injected post-commit lost acknowledgement"), entry.opts?.idempotencyKey));
-        return;
-      }
-      entry.resolve(result);
+      const result = cloneIn(execute(() => runWriterOp(entry.op, cloneIn(entry.args), { now: entry.now, presence: cloneIn(entry.presence) }, entry.opts?.idempotencyKey)));
+      resolveWriterOp(entry, result);
     } catch (error) {
       entry.reject(reconstructInProcess(error));
     }
@@ -618,7 +611,7 @@ async function dispatchInProcess(entry: Submission, signal: AbortSignal): Promis
   // batch + coalesce share the SAVEPOINT-per-caller executor; only the fan-out differs.
   let coalesced: CoalescedEntry[];
   try {
-    const entries: BatchEntry[] = await execute(() => runWriterBatch(
+    const entries: BatchEntry[] = execute(() => runWriterBatch(
       entry.op,
       entry.argsList.map((a) => cloneIn(a)),
       entryKeys(entry),
@@ -650,6 +643,11 @@ async function dispatchWorker(entry: Submission, back: WorkerBackend): Promise<v
 
   // retrySafe stays false (default): a write must NEVER be auto-re-dispatched (it would repeat its
   // effect). No pool timeout is armed — the deadline is the gateway's own (§3.6.1).
+  const startedAt = performance.now();
+  // #3798 — the drain runs outside any caller, so its dispatch and result handling are charged to the
+  // caller that submitted this entry (the reader pool does the same per read).
+  markCallerResume(entry.callsite);
+  writerThreadOps += entryWrites(entry);
   const poolPromise = back.pool.run<WriterJobInput, WriterJobResult>(WRITER_TASK_NAME, input, { labels });
 
   let timer: ReturnType<typeof setTimeout> | undefined;
@@ -660,18 +658,22 @@ async function dispatchWorker(entry: Submission, back: WorkerBackend): Promise<v
     }),
   ]);
   if (timer) clearTimeout(timer);
+  markCallerResume(entry.callsite);
 
   if (raced.done === "timeout") {
     // WEDGED (§3.6.1). The op is still running and still holds the lock; do NOT replace the worker.
     // Surface the caller's INDETERMINATE outcome, hold the FIFO, and resume when the op finally
     // settles (it commits/rolls back and releases the lock on the SAME warm connection).
-    wedged = true;
+    // #3776 — name the op: "writer wedged" alone left an incident to guess which job held the lock.
+    wedgedOp = { op: entry.op, startedAt };
+    console.warn(`[writer-gateway] writer op ${entry.op} has held the writer past its ${writerTimeoutMs}ms deadline; writes are held until it settles`);
     rejectIndeterminate(entry);
     poolPromise.then(
       () => {},
       () => {},
     ).finally(() => {
-      wedged = false;
+      console.warn(`[writer-gateway] wedged writer op ${entry.op} settled after ${Math.round(performance.now() - startedAt)}ms`);
+      wedgedOp = undefined;
       kickDrain();
     }).catch((error) => console.error(`[writer-gateway] resume after wedge failed: ${errMessage(error)}`));
     return;
@@ -701,7 +703,7 @@ function fanOutWorker(entry: Submission, result: WriterJobResult): void {
       entry.reject(new Error(`writer worker returned a ${result.kind} result for a single op`));
       return;
     }
-    if (result.ok) entry.resolve(result.result);
+    if (result.ok) resolveWriterOp(entry, result.result);
     else entry.reject(reconstructDbError(result.error));
     return;
   }

package/src/dev.ts

@@ -1,4 +1,5 @@
 import { writeChildEnv } from "@humwick/sdk/env";
+import { instanceDirectoryEnv } from "@humwick/sdk/isolated-env";
 import { execFile } from "node:child_process";
 import { shellQuote } from "@humwick/sdk/shell-utils";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
@@ -209,7 +210,8 @@ export function createDevServicePlan(options: {
   }
   const envContent = `# ${DEV_MARKER}\n` + renderEnvFile(writeChildEnv({}, {
     AGENT_RELAY_TOKEN: token,
-    AGENT_RELAY_HOME: rootDir,
+    ...instanceDirectoryEnv(rootDir, prefix),
+    AGENT_RELAY_DEV_PROFILE: "1",
     DB_PATH: join(rootDir, "state", "agent-relay.db"),
     HOST: host,
     PORT: String(port),
@@ -229,6 +231,7 @@ export function createDevServicePlan(options: {
     apiPort,
     tmuxPrefix: "ar-dev",
     env: writeChildEnv({}, {
+      ...instanceDirectoryEnv(rootDir, prefix),
       AGENT_RELAY_DEV_PROFILE: "1",
     }),
   };
@@ -296,8 +299,10 @@ export async function executeDevServicePlan(plan: DevServicePlan, options: { dry
   const output: string[] = [];
   if (plan.action === "start" || plan.action === "restart") {
     // Upgrade installed environments before either service can adopt execution records (#3139).
-    const existing = readFileSync(plan.envFile, "utf8").replace(/^(?:export\s+)?AGENT_RELAY_HOME=.*$/gm, "").trimEnd();
-    await writeFile(plan.envFile, `${existing}\n${Object.entries(writeChildEnv({}, { AGENT_RELAY_HOME: plan.rootDir })).map(([key, value]) => `${key}=${shellQuote(value)}\n`).join("")}`, "utf8");
+    const paths = writeChildEnv(instanceDirectoryEnv(plan.rootDir, plan.prefix), { AGENT_RELAY_DEV_PROFILE: "1" });
+    const keys = Object.keys(paths).map((key) => key.replace("AGENT_RELAY_", "(?:AGENT_RELAY_|HUMWICK_)"));
+    const existing = readFileSync(plan.envFile, "utf8").replace(new RegExp(`^(?:export\\s+)?(?:${keys.join("|")})=.*$`, "gm"), "").trimEnd();
+    await writeFile(plan.envFile, `${existing}\n${Object.entries(paths).map(([key, value]) => `${key}=${shellQuote(value)}\n`).join("")}`, "utf8");
   }
   if (plan.action === "install") {
     for (const file of plan.files) {

package/src/dispatch-availability.ts

@@ -1,5 +1,5 @@
-import type { HostView, Orchestrator } from "@humwick/sdk";
-import type { ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
+import { errMessage, type HostView, type Orchestrator } from "@humwick/sdk";
+import { resolveProviderSelection, type ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
 import type { DispatchTarget } from "./task-dispatcher";
 import { spawnPlacementRefusal, hostSpawnAdmission, hostIdentitiesById, spawnHostCapAdmission, type SpawnCapOptions } from "./spawn-placement";
 import { liveHostDisabledProviders } from "./host-provider-store";
@@ -17,13 +17,15 @@ export interface LiveTargetAdmission {
   targetUnavailableReason: (target: DispatchTarget) => string | undefined;
 }
 
-export function liveTargetAdmission(orchestrators: readonly Orchestrator[], views: readonly HostView[], orchestratorId?: string, capOpts: SpawnCapOptions = {}): LiveTargetAdmission {
+export function liveTargetAdmission(orchestrators: readonly Orchestrator[], views: readonly HostView[], catalog: ProviderCatalogEntry[], orchestratorId?: string, capOpts: SpawnCapOptions = {}): LiveTargetAdmission {
   const identities = hostIdentitiesById(views);
   const byId = new Map(views.map((view) => [view.id, view]));
   const disabled = liveHostDisabledProviders();
   const capBlock = spawnHostCapAdmission(capOpts);
   const hosts = orchestrators.filter((host) => !orchestratorId || host.id === orchestratorId);
   const targetAdmission = ({ provider, model, effort }: DispatchTarget): TargetAdmission => {
+    try { resolveProviderSelection({ provider, model, effort }, catalog); }
+    catch { return { eligible: false, uncertainties: [] }; }
     let eligible = false;
     const uncertainties: TargetUncertainty[] = [];
     for (const host of hosts) {
@@ -34,9 +36,11 @@ export function liveTargetAdmission(orchestrators: readonly Orchestrator[], view
     }
     return { eligible, uncertainties };
   };
-  const targetUnavailableReason = (target: DispatchTarget) => spawnPlacementRefusal(
-    orchestrators, views, target.provider, target.model, target.effort, orchestratorId, capOpts,
-  );
+  const targetUnavailableReason = (target: DispatchTarget) => {
+    try { resolveProviderSelection(target, catalog); }
+    catch (error) { return errMessage(error); }
+    return spawnPlacementRefusal(orchestrators, views, target.provider, target.model, target.effort, orchestratorId, capOpts);
+  };
   return { targetAdmission, targetUnavailableReason };
 }
 

package/src/ephemeral/{landing-ingress-proof-worker.ts → landing-ingress-proof.test-fixture.ts}

@@ -1,7 +1,7 @@
 /** Precondition setup for the packaged-ephemeral landing-ingress Proof of Life.
  * Delivery and both crash boundaries run through the packaged Relay; this fixture only registers
  * the disposable workspace that its real ephemeral orchestrator will probe. */
-import { initDb } from "../db/schema.ts";
+import { closeDb, openExistingDb } from "../db/connection.ts";
 import { upsertWorkspace } from "../db/workspaces.ts";
 
 const dbPath = process.env.DB_PATH;
@@ -13,7 +13,8 @@ if (!dbPath || !worktreePath || !orchestratorId || !baseSha) {
   throw new Error("usage: landing-ingress-proof-worker <worktree-path> <orchestrator-id> <base-sha>");
 }
 
-initDb(dbPath);
+// The installed Relay already owns the schema; a fixture must not migrate its live DB (#3715).
+openExistingDb(dbPath);
 upsertWorkspace({
   id: workspaceId,
   repoRoot: worktreePath,
@@ -27,4 +28,5 @@ upsertWorkspace({
   orchestratorId,
   metadata: {},
 });
+closeDb();
 console.log(JSON.stringify({ workspaceId }));

package/src/event-loop-lag.ts

@@ -1,3 +1,4 @@
+import { trackRelayEvent } from "./relay-event-drain";
 import { errMessage } from "@humwick/sdk";
 import type { HealthCheck } from "./types";
 import {
@@ -166,7 +167,7 @@ export function setLagRegressionObserver(fn: EventLoopLagRegressionObserver | un
  * threshold rather than the diagnostic log threshold, and is a separate registration a caller can
  * leave unset (e.g. ephemeral mode) with no effect.
  */
-export type EventLoopLagBreakerObserver = (gapStartAt: number, gapEndAt: number, lagMs: number) => void;
+export type EventLoopLagBreakerObserver = (gapStartAt: number, gapEndAt: number, lagMs: number) => void | Promise<void>;
 
 let breakerObserver: EventLoopLagBreakerObserver | undefined;
 
@@ -283,7 +284,8 @@ export function recordEventLoopLagSample(
     // timer callback, so a throw here would become an uncaught exception in the interval — guard it:
     // the breaker degrading is never worth killing the loop-lag sampler that watches for the freeze.
     try {
-      breakerObserver(sampledAt - Math.max(0, elapsedMs), sampledAt, normalized);
+      const outcome = breakerObserver(sampledAt - Math.max(0, elapsedMs), sampledAt, normalized);
+      if (outcome) trackRelayEvent(outcome, "maintenance.lag-breaker");
     } catch (error) {
       console.warn(`[health] event-loop-lag breaker observer threw: ${errMessage(error)}`);
     }

package/src/event-loop-yield.ts

@@ -11,7 +11,7 @@ export function yieldToEventLoop(): Promise<void> {
  *  `moreWork`, so a backlog drains over runs instead of overrunning the deadline (a breaker strike). */
 export async function forEachPaced<T>(
   items: () => Iterable<T>,
-  unit: (item: T) => void,
+  unit: (item: T) => unknown,
   opts: { signal?: AbortSignal; budgetMs: number; ready?: () => boolean },
 ): Promise<{ moreWork: boolean }> {
   await yieldToEventLoop();
@@ -20,7 +20,7 @@ export async function forEachPaced<T>(
   for (const item of items()) {
     if (opts.signal?.aborted) return { moreWork: false };
     if (performance.now() >= stopAt || opts.ready?.() === false) return { moreWork: true };
-    unit(item);
+    await unit(item);
     await yieldToEventLoop();
   }
   return { moreWork: false };

package/src/events.ts

@@ -19,7 +19,7 @@ export interface RelayEvent {
 
 type Listener = (event: RelayEvent) => void;
 
-const listeners = new Set<Listener>();
+const listeners = new Map<Listener, string>();
 const fanoutQueue: Array<{ event: RelayEvent; lastSeq: number }> = [];
 let notifying = false;
 
@@ -33,12 +33,13 @@ export function emitRelayEvent(input: Omit<RelayEvent, "seq" | "timestamp">): Pr
       const appended = appendRelayEventInternal(input.type, input.source, input.data, input.subject, timestamp)!;
       return Promise.resolve(fanOutAppendedEvent(input, appended, timestamp));
     }
-    return emitRelayEventAsync(structuredClone(input), { idempotencyKey: crypto.randomUUID() });
+    // #3614 — keyless: nothing resubmits this append, so a ledger row would only cost writes.
+    return trackRelayEvent(appendAndFanOut(structuredClone(input), {}), input.type);
   });
 }
 
 /** #3606 — fan out to the current listeners only: no bus_outbox row, no idempotency-ledger row, no
- * replay. For state the next tick supersedes anyway (dashboard viewer presence). */
+ * replay. For state the next tick supersedes anyway (dashboard viewer presence, agent liveness ticks). */
 export function emitEphemeralRelayEvent(input: Omit<RelayEvent, "seq" | "timestamp" | "ephemeral">): RelayEvent {
   return fanOutAppendedEvent({ ...input, ephemeral: true }, { seq: 0, lastSeq: 0 }, Date.now());
 }
@@ -71,7 +72,8 @@ async function appendAndFanOut(
   } catch (error) {
     // H5: a lost acknowledgement says nothing about commit. Re-submit the exact logical append
     // with the same durable key; the writer returns the first result without a second insert.
-    if (!(error instanceof WriteIndeterminateError)) throw error;
+    // A keyless emit is at-most-once: it surfaces the error, and a committed row stays replayable.
+    if (!(error instanceof WriteIndeterminateError) || !options.idempotencyKey) throw error;
     appended = await getGateway().write("bus.append", args, { critical: true, idempotencyKey: options.idempotencyKey }) as unknown as typeof appended;
   }
   // A retry ledger result proves durable storage, not that this process owns a fresh live edge.
@@ -122,19 +124,19 @@ export function fanOutAppendedEvent(
 
 function notify(event: RelayEvent): void {
   measureEmit(event.type, true, () => {
-    for (const listener of listeners) {
+    for (const [listener, name] of listeners) {
       try {
         listener(event);
       } catch (error) {
         // Event projections are isolated so one transport cannot block another; a failed one (e.g. a
         // BUSY write in a projection, #3643) is logged, not swallowed.
-        console.warn(`[events] ${event.type} listener failed: ${errMessage(error)}`);
+        console.warn(`[events] ${event.type} listener ${name} failed: ${errMessage(error)}`, error instanceof Error ? error.stack : error);
       }
     }
   });
 }
 
-export function subscribeRelayEvents(listener: Listener): () => void {
-  listeners.add(listener);
+export function subscribeRelayEvents(listener: Listener, name = listener.name || "<anonymous>"): () => void {
+  listeners.set(listener, name);
   return () => listeners.delete(listener);
 }

package/src/host/disk-alert.ts

@@ -0,0 +1,121 @@
+// #3762 — warn before a host disk fills. Pure: the single-writer utilization op settles each beat
+// against the stored one (same seam as gate-capacity-alert.ts), so each level change alerts once.
+// Two triggers, one level ladder: used% crossing 85/92/97, or the free-space history projecting
+// full within 24 h (6 h = high). An episode alerts once per worse level and clears with a notice.
+import type { HostDiskWatch, HostMachineSample } from "@humwick/sdk";
+
+export type DiskLevel = 1 | 2 | 3;
+
+/** Used-share rungs: warn / high / critical. */
+export const DISK_USED_PCT_LEVELS: readonly [number, number, number] = [85, 92, 97];
+/** An episode only clears below this, so a disk hovering on a rung does not flap. */
+export const DISK_CLEAR_USED_PCT = 82;
+export const DISK_ETA_WARN_H = 24;
+export const DISK_ETA_HIGH_H = 6;
+/** Growth needs this much history before it is a rate rather than a blip. */
+export const DISK_MIN_TREND_SPAN_MS = 3 * 3_600_000;
+const HISTORY_SPACING_MS = 30 * 60_000;
+const HISTORY_MAX_SPAN_MS = 25 * 3_600_000;
+const TREND_WINDOW_MS = 24 * 3_600_000;
+
+export type DiskAlert =
+  | {
+    kind: "low"; level: DiskLevel; since: number; freeMb: number; totalMb: number; path?: string;
+    /** Free-space decline over the trend window, MB/h; present only while declining. */
+    rateMbPerHour?: number; etaHours?: number; spanHours?: number;
+  }
+  | { kind: "recovered"; since: number; freeMb: number; totalMb: number; path?: string };
+
+export interface DiskNotice {
+  subject: string;
+  body: string;
+  priority: "low" | "default" | "high" | "urgent";
+  idempotencyKey: string;
+}
+
+function usageLevel(usedPct: number): number {
+  return DISK_USED_PCT_LEVELS.filter((rung) => usedPct >= rung).length;
+}
+
+function growthLevel(etaHours: number | undefined): number {
+  if (etaHours === undefined) return 0;
+  return etaHours < DISK_ETA_HIGH_H ? 2 : etaHours < DISK_ETA_WARN_H ? 1 : 0;
+}
+
+function nextHistory(prev: HostDiskWatch["history"], at: number, freeMb: number): Array<[number, number]> {
+  const kept = (prev ?? []).filter(([t]) => t >= at - HISTORY_MAX_SPAN_MS && t < at);
+  const last = kept[kept.length - 1];
+  if (!last || at - last[0] >= HISTORY_SPACING_MS) kept.push([at, freeMb]);
+  return kept;
+}
+
+/** Decline rate over the oldest point inside the trend window, or undefined (too little history / not declining). */
+function trend(history: Array<[number, number]>, at: number, freeMb: number): { rateMbPerHour: number; etaHours: number; spanHours: number } | undefined {
+  const oldest = history.find(([t]) => t >= at - TREND_WINDOW_MS);
+  if (!oldest || at - oldest[0] < DISK_MIN_TREND_SPAN_MS) return undefined;
+  const declineMb = oldest[1] - freeMb;
+  if (declineMb <= 0) return undefined;
+  const spanHours = (at - oldest[0]) / 3_600_000;
+  const rateMbPerHour = declineMb / spanHours;
+  return { rateMbPerHour, etaHours: freeMb / rateMbPerHour, spanHours };
+}
+
+/**
+ * Stamps the relay-owned disk watch onto the stored sample and names the one alert owed, if any.
+ * A beat with no disk pair keeps the previous watch (missing telemetry is not a state change).
+ */
+export function settleDiskWatch(prev: HostMachineSample | undefined, next: HostMachineSample): { machine: HostMachineSample; alert?: DiskAlert } {
+  const before = prev?.diskWatch;
+  const { diskFreeMb: freeMb, diskTotalMb: totalMb } = next;
+  if (freeMb === undefined || totalMb === undefined || totalMb <= 0) {
+    return { machine: before ? { ...next, diskWatch: before } : next };
+  }
+  const history = nextHistory(before?.history, next.at, freeMb);
+  const projected = trend(history, next.at, freeMb);
+  const usedPct = ((totalMb - freeMb) / totalMb) * 100;
+  const level = Math.max(usageLevel(usedPct), growthLevel(projected?.etaHours));
+  const prevLevel = before?.level ?? 0;
+  const where = next.diskPath ? { path: next.diskPath } : {};
+
+  if (level > prevLevel) {
+    const since = before?.since ?? next.at;
+    const watch: HostDiskWatch = { level: level as DiskLevel, since, history };
+    return {
+      machine: { ...next, diskWatch: watch },
+      alert: {
+        kind: "low", level: level as DiskLevel, since, freeMb, totalMb, ...where,
+        ...(projected ? { rateMbPerHour: projected.rateMbPerHour, etaHours: projected.etaHours, spanHours: projected.spanHours } : {}),
+      },
+    };
+  }
+  if (prevLevel > 0 && level === 0 && usedPct < DISK_CLEAR_USED_PCT) {
+    return { machine: { ...next, diskWatch: { history } }, alert: { kind: "recovered", since: before!.since ?? next.at, freeMb, totalMb, ...where } };
+  }
+  return { machine: { ...next, diskWatch: { ...(before?.level ? { level: before.level, since: before.since } : {}), history } } };
+}
+
+const gb = (mb: number) => `${(mb / 1024).toFixed(mb >= 10_240 ? 0 : 1)} GB`;
+
+export function composeDiskNotice(hostId: string, label: string, alert: DiskAlert): DiskNotice {
+  const volume = alert.path ? ` (${alert.path})` : "";
+  const usedPct = Math.round(((alert.totalMb - alert.freeMb) / alert.totalMb) * 100);
+  if (alert.kind === "recovered") {
+    return {
+      subject: `Disk recovered on ${label}`,
+      body: `${label}${volume} has ${gb(alert.freeMb)} free of ${gb(alert.totalMb)} (${usedPct}% used); the low-disk episode is over. (#3762)`,
+      priority: "low",
+      idempotencyKey: `host-disk:${hostId}:${alert.since}:recovered`,
+    };
+  }
+  const word = alert.level === 3 ? "CRITICAL" : alert.level === 2 ? "very low" : "low";
+  const growth = alert.rateMbPerHour !== undefined && alert.etaHours !== undefined
+    ? ` Free space fell ~${gb(alert.rateMbPerHour * 24)}/day over the last ${Math.round(alert.spanHours ?? 0)} h; at that rate it is full in ~${Math.max(1, Math.round(alert.etaHours))} h.`
+    : "";
+  return {
+    subject: `Disk ${word} on ${label}: ${gb(alert.freeMb)} free (${usedPct}% used)`,
+    body: `${label}${volume} has ${gb(alert.freeMb)} free of ${gb(alert.totalMb)} (${usedPct}% used).${growth} `
+      + "Check workspaces, runtime logs and caches on that host before agents start failing with ENOSPC. (#3762)",
+    priority: alert.level === 3 ? "urgent" : alert.level === 2 ? "high" : "default",
+    idempotencyKey: `host-disk:${hostId}:${alert.since}:${alert.level}`,
+  };
+}