@nimbus-sh/fabric 0.8.0 → 0.10.0

package/src/budgets.ts

@@ -1,60 +1,215 @@
 /**
  * budgets.ts — per-DO accounting for the platform budgets the fabric spends:
- * the Worker Loader's two caps, the facet-ID lifetime budget, and the
- * dynamic-worker module-map ceiling.
+ * the Durable Object's Dynamic Worker concurrency limit, the facet-ID
+ * lifetime budget, and the dynamic-worker module-map ceiling.
  *
- * Measured on production workerd: a Durable Object admits ~5–6 concurrent
- * dynamic workers before the platform refuses with "Too many concurrent
- * dynamic workers", one DO method can drive at most 4 concurrent Loader
- * fetches, and loader-cache entries are never released — every DISTINCT
- * `loader.get(id)` permanently consumes one of the dynamic-worker slots for
- * the object's lifetime. Nimbus stays under the caps by construction
- * (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
- * were counted in prose. This ledger counts them at the fabric's loader call
- * sites instead — the loader pool's slots, a resident process's keyed worker,
- * a one-shot's load — so proximity is measurable and a cap failure can name
- * the ids actually holding slots.
+ * The Dynamic Worker model is Cloudflare's documented one
+ * ({@link DO_DYNAMIC_WORKER_LIMIT}): a Durable Object may have a fixed number
+ * of DISTINCT Dynamic Workers with in-flight requests at once, shared across
+ * every concurrent request to that object (one I/O context), and any number
+ * of in-flight requests to the same Dynamic Worker count as one. Only
+ * in-flight requests count: a loader id with nothing in flight holds nothing.
  *
- * Measurement only: no admission control. The caps are the platform's, they
- * are approximate ("~5–6"), and a gate on an approximate number would refuse
- * work the platform would have run.
+ * The ledger counts, per hosting actor, the distinct workers that are in
+ * flight right now, keyed by loader id (a fresh key per unkeyed `load`), plus
+ * the width fan-outs have claimed and not yet released. A fan-out spends only
+ * the {@link dynamicWorkerHeadroom} that leaves, so work a Durable Object
+ * already has in flight — a resident process, the esbuild facet, a git
+ * network op, another fan-out — keeps its slots. Work that would rather wait
+ * than be refused waits on the ledger ({@link beginLoaderFetchWhenFree}) and
+ * is let in, in the order it asked, by whichever release makes room.
  *
  * Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
- * caps are per Durable Object, and dynamic workers die with the isolate that
+ * limit is per Durable Object, and dynamic workers die with the isolate that
  * loaded them, so a ledger that goes away with its host describes nothing
  * that still exists.
  */
 
 import { classifyError } from '@nimbus-sh/platform/oom-classify.js';
+import { hostWasmIdentity } from './host-wasm.js';
+
+/**
+ * Distinct Dynamic Workers one Durable Object may have with in-flight
+ * requests at once, shared across all concurrent requests to that object;
+ * multiple in-flight requests to one Dynamic Worker count once.
+ * https://developers.cloudflare.com/changelog/post/2026-08-28-durable-objects-dynamic-workers-limit/
+ */
+export const DO_DYNAMIC_WORKER_LIMIT = 10;
+
+/**
+ * Ends one hold, idempotently. Pass the error the call failed with, if it
+ * did: a "Dynamic worker concurrency limit exceeded" refusal pauses the
+ * ledger's admissions (see {@link beginLoaderFetchWhenFree}); anything else,
+ * or nothing, just ends the hold.
+ */
+export type EndLoaderFetch = (failure?: unknown) => void;
+
+/**
+ * A width one fan-out reserved with {@link claimDynamicWorkers}. Holds taken
+ * under it (`beginLoaderFetch(ctx, key, claim)`) count inside that width, not
+ * on top of it, until `release` (idempotent).
+ */
+export interface DynamicWorkerClaim {
+  release(): void;
+}
+
+interface ClaimEntry {
+  width: number;
+  /** Loader id → open holds taken under this claim. */
+  keys: Map<string, number>;
+}
+
+interface Waiter {
+  key: string;
+  claim: ClaimEntry | undefined;
+  admit(end: EndLoaderFetch): void;
+}
 
 interface LoaderLedger {
-  /** Distinct loader ids ever gotten — each one a permanently consumed slot. */
-  ids: Set<string>;
-  /** In-flight calls into dynamic workers, right now. */
-  liveFetches: number;
-  /** The most that were ever in flight at once. */
-  peakLiveFetches: number;
+  /** Loader id → open holds on it, under a claim or not. A key is present only while held. */
+  inFlight: Map<string, number>;
+  /** Claims not yet released. */
+  claims: Set<ClaimEntry>;
+  /** The most distinct workers (holds plus claims) ever counted at once. */
+  peak: number;
+  /** Waits not yet admitted, in the order they asked. */
+  waiters: Waiter[];
+  /** Length of the pause a refusal started, while it lasts; 0 when admitting. */
+  pauseMs: number;
+  pauseTimer: ReturnType<typeof setTimeout> | undefined;
+  /** Advanced when a pause starts and when it ends; a hold keeps the one it began in. */
+  epoch: number;
+  /** Pauses in a row with no worker admitted between them. */
+  refusals: number;
 }
 
+/**
+ * The first pause after a limit refusal, doubling while refusals continue,
+ * up to {@link REFUSAL_PAUSE_MAX_MS}. A deployed Durable Object admitted a
+ * batch it had refused after a 6 s pause.
+ */
+const REFUSAL_PAUSE_MS = 50;
+const REFUSAL_PAUSE_MAX_MS = 2_000;
+
 const ledgers = new WeakMap<object, LoaderLedger>();
+const claimEntries = new WeakMap<DynamicWorkerClaim, { ledger: LoaderLedger; entry: ClaimEntry }>();
 
 function ledger(ctx: object): LoaderLedger {
   let entry = ledgers.get(ctx);
   if (!entry) {
-    entry = { ids: new Set(), liveFetches: 0, peakLiveFetches: 0 };
+    entry = {
+      inFlight: new Map(), claims: new Set(), peak: 0,
+      waiters: [], pauseMs: 0, pauseTimer: undefined, epoch: 0, refusals: 0,
+    };
     ledgers.set(ctx, entry);
   }
   return entry;
 }
 
-/** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
-export function recordLoaderId(ctx: object, id: string): void {
-  ledger(ctx).ids.add(id);
+/** Distinct workers counted: each claim's width (or more, if its holds exceed it), plus held keys no claim covers. */
+function inUse(entry: LoaderLedger): number {
+  let count = 0;
+  const covered = new Set<string>();
+  for (const claim of entry.claims) {
+    count += Math.max(claim.width, claim.keys.size);
+    for (const key of claim.keys.keys()) covered.add(key);
+  }
+  for (const key of entry.inFlight.keys()) if (!covered.has(key)) count++;
+  return count;
+}
+
+function headroom(entry: LoaderLedger): number {
+  return entry.pauseMs > 0 ? 0 : Math.max(0, DO_DYNAMIC_WORKER_LIMIT - inUse(entry));
+}
+
+function claimOf(ctx: object, claim: DynamicWorkerClaim | undefined): ClaimEntry | undefined {
+  if (claim === undefined) return undefined;
+  const owned = claimEntries.get(claim);
+  if (owned === undefined || owned.ledger !== ledger(ctx)) {
+    throw new Error('Nimbus: a Dynamic Worker claim is used only on the ledger of the actor that claimed it');
+  }
+  return owned.ledger.claims.has(owned.entry) ? owned.entry : undefined;
+}
+
+function count(map: Map<string, number>, key: string, by: 1 | -1): void {
+  const open = (map.get(key) ?? 0) + by;
+  if (open > 0) map.set(key, open);
+  else map.delete(key);
+}
+
+/** Take one hold; the caller admits waiters after. */
+function hold(entry: LoaderLedger, workerKey: string, claim: ClaimEntry | undefined): EndLoaderFetch {
+  count(entry.inFlight, workerKey, 1);
+  if (claim) count(claim.keys, workerKey, 1);
+  entry.peak = Math.max(entry.peak, inUse(entry));
+  const epoch = entry.epoch;
+  let ended = false;
+  return (failure) => {
+    if (ended) return;
+    ended = true;
+    count(entry.inFlight, workerKey, -1);
+    if (claim) count(claim.keys, workerKey, -1);
+    if (classifyError(failure) === 'dynamic_worker_cap') refused(entry, epoch);
+    else if (epoch === entry.epoch && entry.pauseMs === 0) entry.refusals = 0;
+    admitWaiters(entry);
+  };
 }
 
 /**
- * Count one call into a dynamic worker as a live Loader fetch; the returned
- * function ends it (idempotently), from the caller's own `finally`.
+ * The platform refused a worker this ledger counted room for: it still
+ * counts workers the ledger has released, which no release here can show.
+ * So nothing new is admitted until a pause has passed. A refusal of a call
+ * that began before the latest pause started or ended is the same lag and
+ * changes nothing; one of a call let in after it doubles the next pause.
+ */
+function refused(entry: LoaderLedger, epoch: number): void {
+  if (epoch !== entry.epoch) return;
+  if (entry.pauseTimer !== undefined) clearTimeout(entry.pauseTimer);
+  entry.pauseMs = Math.min(REFUSAL_PAUSE_MAX_MS, REFUSAL_PAUSE_MS * 2 ** entry.refusals);
+  entry.refusals++;
+  entry.epoch++;
+  entry.pauseTimer = setTimeout(() => {
+    entry.pauseMs = 0;
+    entry.pauseTimer = undefined;
+    entry.epoch++;
+    admitWaiters(entry);
+  }, entry.pauseMs);
+}
+
+function admissible(entry: LoaderLedger, waiter: Waiter): boolean {
+  // Requests to a worker already in flight count once, even while paused.
+  if (entry.inFlight.has(waiter.key)) return true;
+  if (entry.pauseMs > 0) return false;
+  if (waiter.claim && entry.claims.has(waiter.claim) && waiter.claim.keys.size < waiter.claim.width) return true;
+  return inUse(entry) < DO_DYNAMIC_WORKER_LIMIT;
+}
+
+/**
+ * Let in every waiter that fits, in the order they asked: each takes its
+ * hold here, so a freed slot goes to exactly one waiter and is never left
+ * between a wake and a begin. Run after every change that can make room.
+ * A waiter let in on a new key lets in the later ones on that key, and the
+ * earlier ones too: the scan starts over.
+ */
+function admitWaiters(entry: LoaderLedger): void {
+  for (let i = 0; i < entry.waiters.length;) {
+    const waiter = entry.waiters[i];
+    if (!admissible(entry, waiter)) { i++; continue; }
+    const joins = entry.inFlight.has(waiter.key);
+    entry.waiters.splice(i, 1);
+    waiter.admit(hold(entry, waiter.key, waiter.claim));
+    if (!joins) i = 0;
+  }
+}
+
+/**
+ * Hold the Dynamic Worker `workerKey` in flight on this actor's ledger; the
+ * returned function ends the hold (idempotently), from the caller's own
+ * `finally`. Holds on one key nest: the worker counts once until the last
+ * one ends, as the platform counts it. Under a `claim`, the hold counts
+ * inside the claim's width. This never waits: it is for work the actor
+ * starts regardless (a resident process); {@link beginLoaderFetchWhenFree}
+ * waits for room.
  *
  * A begin/end pair rather than a wrapper on purpose, and the shape is
  * load-bearing: wrapping the stub call in a ledger-owned async frame
@@ -67,48 +222,146 @@ export function recordLoaderId(ctx: object, id: string): void {
  * workers: an RPC stub call must stay a direct property call awaited by the
  * frame that made it, so the ledger only brackets it.
  */
-export function beginLoaderFetch(ctx: object): () => void {
+export function beginLoaderFetch(ctx: object, workerKey: string, claim?: DynamicWorkerClaim): EndLoaderFetch {
   const entry = ledger(ctx);
-  entry.liveFetches++;
-  entry.peakLiveFetches = Math.max(entry.peakLiveFetches, entry.liveFetches);
-  let ended = false;
-  return () => {
-    if (ended) return;
-    ended = true;
-    entry.liveFetches--;
+  const end = hold(entry, workerKey, claimOf(ctx, claim));
+  admitWaiters(entry);
+  return end;
+}
+
+/**
+ * {@link beginLoaderFetch} once the ledger has room: resolves, holding
+ * `workerKey`, as soon as that worker is already in flight (holds on it
+ * count once) or a distinct worker more fits — within the `claim`'s width,
+ * or the actor's headroom. Waits are let in in the order they asked, by
+ * whoever's release makes the room: a hold's end, a claim's release, a
+ * pause's end. The hold is taken as the wait is let in, so a freed slot
+ * wakes one waiter and no other caller can take it first; once resolved, it
+ * is the caller's to end.
+ *
+ * A call refused with "Dynamic worker concurrency limit exceeded" ends its
+ * hold with the refusal (`end(error)`) and waits again: the refusal pauses
+ * admission (50 ms, doubling to 2 s while refusals continue), because the
+ * platform counts a worker for a moment after its call returns and no
+ * release can show that.
+ *
+ * `signal` abandons the wait: it rejects with the signal's reason and holds
+ * nothing. A wait outlives nothing on its own: bound it with a signal when
+ * room may never come (a resident process holds its worker for as long as
+ * it runs).
+ *
+ *   const end = await beginLoaderFetchWhenFree(ctx, key, { signal });
+ *   try { return await worker.getEntrypoint().run(); }
+ *   catch (error) { end(error); throw error; }
+ *   finally { end(); }
+ */
+export function beginLoaderFetchWhenFree(
+  ctx: object,
+  workerKey: string,
+  options: { signal?: AbortSignal; claim?: DynamicWorkerClaim } = {},
+): Promise<EndLoaderFetch> {
+  const { signal } = options;
+  return new Promise<EndLoaderFetch>((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason);
+      return;
+    }
+    const entry = ledger(ctx);
+    const abandon = () => {
+      const at = entry.waiters.indexOf(waiter);
+      if (at < 0) return;
+      entry.waiters.splice(at, 1);
+      reject(signal?.reason);
+    };
+    const waiter: Waiter = {
+      key: workerKey,
+      claim: claimOf(ctx, options.claim),
+      admit(end) {
+        signal?.removeEventListener('abort', abandon);
+        resolve(end);
+      },
+    };
+    entry.waiters.push(waiter);
+    signal?.addEventListener('abort', abandon, { once: true });
+    admitWaiters(entry);
+  });
+}
+
+/**
+ * Distinct Dynamic Workers this actor may still put in flight: the limit
+ * less what is held and claimed right now, and none while a limit refusal's
+ * pause lasts. Never negative.
+ */
+export function dynamicWorkerHeadroom(ctx: object): number {
+  return headroom(ledger(ctx));
+}
+
+/**
+ * Claim `width` distinct Dynamic Workers for one fan-out, or null when the
+ * headroom cannot hold it. The claim counts until `release` (idempotent), so
+ * a second fan-out sizing itself meanwhile sees it; the claimant's own
+ * dispatches, held under the claim, count inside it.
+ */
+export function claimDynamicWorkers(ctx: object, width: number): DynamicWorkerClaim | null {
+  const entry = ledger(ctx);
+  if (width < 1 || width > headroom(entry)) return null;
+  const claim: ClaimEntry = { width, keys: new Map() };
+  entry.claims.add(claim);
+  entry.peak = Math.max(entry.peak, inUse(entry));
+  const handle: DynamicWorkerClaim = {
+    release() {
+      if (!entry.claims.delete(claim)) return;
+      admitWaiters(entry);
+    },
   };
+  claimEntries.set(handle, { ledger: entry, entry: claim });
+  return handle;
 }
 
 /** Snapshot for the diag surface. Pure read; no I/O. */
 export function loaderLedgerStats(ctx: object): {
-  idsEverGotten: string[];
-  liveFetches: number;
-  peakLiveFetches: number;
+  limit: number;
+  inFlightWorkers: string[];
+  claimed: number;
+  headroom: number;
+  peak: number;
+  /** Waits not yet admitted. */
+  waiting: number;
+  /** Length of the pause a limit refusal started, while it lasts; 0 when admitting. */
+  pauseMs: number;
 } {
   const entry = ledger(ctx);
   return {
-    idsEverGotten: [...entry.ids],
-    liveFetches: entry.liveFetches,
-    peakLiveFetches: entry.peakLiveFetches,
+    limit: DO_DYNAMIC_WORKER_LIMIT,
+    inFlightWorkers: [...entry.inFlight.keys()],
+    claimed: claimedWidth(entry),
+    headroom: headroom(entry),
+    peak: entry.peak,
+    waiting: entry.waiters.length,
+    pauseMs: entry.pauseMs,
   };
 }
 
+function claimedWidth(entry: LoaderLedger): number {
+  let width = 0;
+  for (const claim of entry.claims) width += claim.width;
+  return width;
+}
+
 /**
- * Name the per-DO accounting on a "Too many concurrent dynamic workers"
+ * Name the per-DO accounting on a "Dynamic worker concurrency limit exceeded"
  * failure; hand every other error back untouched. The platform's message
- * says only that the cap was hit — which ids hold the slots, and that a
- * keyed id can never give one back, is what the operator needs to know to
- * shrink anything.
+ * says only that the limit was hit — which workers were in flight, and what
+ * fan-outs had claimed, is what the operator needs to know to shrink anything.
  */
 export function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error {
   if (classifyError(error) !== 'dynamic_worker_cap') return error;
   const entry = ledger(ctx);
   const platform = error instanceof Error ? error.message : String(error);
   return new Error(
-    `${platform} — this Durable Object has ${entry.ids.size} loader id(s) permanently `
-      + `holding dynamic-worker slots (a loader.get id is never released): `
-      + `${[...entry.ids].join(', ') || '(none recorded)'}; live Loader fetches ${entry.liveFetches}, `
-      + `peak ${entry.peakLiveFetches}`,
+    `${platform} — this Durable Object had ${entry.inFlight.size} distinct dynamic worker(s) in flight `
+      + `(${[...entry.inFlight.keys()].join(', ') || 'none recorded'}) and ${claimedWidth(entry)} claimed by fan-outs, `
+      + `against a limit of ${DO_DYNAMIC_WORKER_LIMIT}; peak ${entry.peak}`,
     { cause: error },
   );
 }
@@ -163,18 +416,23 @@ export function assertModuleMapWithinCodeLimit(modules: Record<string, unknown>)
 
 /**
  * Bytes one module-map member carries, across the loader's content kinds
- * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`). With an
- * encoder, text is measured exactly; without one, by code-unit length.
+ * (plain string, `{ js | cjs | py | text }`, `{ wasm | data }`, a bare
+ * WebAssembly.Module). With an encoder, text is measured exactly; without
+ * one, by code-unit length. A compiled module counts the wire size its host
+ * described (host-wasm.ts); one nobody described counts nothing here and is
+ * left to the platform's own refusal, as the text undercount is.
  */
 function memberBytes(content: unknown, encoder: TextEncoder | null): number {
   const textBytes = (text: string): number =>
     encoder ? encoder.encode(text).byteLength : text.length;
   if (typeof content === 'string') return textBytes(content);
+  if (content instanceof WebAssembly.Module) return hostWasmIdentity(content)?.bytes ?? 0;
   if (content !== null && typeof content === 'object') {
     for (const value of Object.values(content)) {
       if (typeof value === 'string') return textBytes(value);
       if (value instanceof ArrayBuffer) return value.byteLength;
       if (ArrayBuffer.isView(value)) return value.byteLength;
+      if (value instanceof WebAssembly.Module) return hostWasmIdentity(value)?.bytes ?? 0;
     }
   }
   return 0;

package/src/do-calls.ts

@@ -42,6 +42,7 @@
 
 import { classifyDoCall, isRetryableDoCall, type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
 import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
+import { untraced, type SpanRecorder } from '@nimbus-sh/platform/tracing.js';
 
 /** Total attempts. Two retries is what a dropped connection or a deploy
  *  bounce needs; beyond that the object is not coming back inside this
@@ -88,8 +89,28 @@ export interface DoCallRetryPolicy {
    * its error.
    */
   onRetry?(info: DoCallRetryInfo): void;
+  /**
+   * Where the call's telemetry goes: its span's recorder. Each attempt lost
+   * to a transient or overloaded failure is recorded as an exception whose
+   * `code` is the failure's class, and once the call has settled it gets
+   * `do_call.attempts` (started), `do_call.hedges` (started by a hedge),
+   * `do_call.answered_by` (the attempt whose answer the call took, absent
+   * when none answered) and `do_call.outcome` ({@link DoCallOutcome}).
+   * Nothing recorded can change the call's answer. Records nothing when
+   * absent.
+   */
+  span?: SpanRecorder;
 }
 
+/**
+ * How an `idempotent` call ended: `answered` (an attempt succeeded),
+ * `callee_error` (the callee's own failure, which is an answer),
+ * `exhausted` (the last transient failure, no repeat left), `overloaded`
+ * (shed, nothing repeated after it), or `caller_error` (the resolver or
+ * `onRetry` threw).
+ */
+export type DoCallOutcome = 'answered' | 'callee_error' | 'exhausted' | 'overloaded' | 'caller_error';
+
 /** What one retry is answering: which call, which platform class, which
  *  attempt just failed out of how many. */
 export interface DoCallRetryInfo {
@@ -161,7 +182,7 @@ export function idempotent<S, T>(
 ): Promise<T> {
   const maxAttempts = policy.maxAttempts ?? MAX_ATTEMPTS;
   const baseDelayMs = policy.baseDelayMs ?? BASE_DELAY_MS;
-  const { hedgeAfterMs, retryWindowMs } = policy;
+  const { hedgeAfterMs, retryWindowMs, span = untraced } = policy;
   const startedAt = Date.now();
   // The executor form: fabric's library target predates Promise.withResolvers.
   return new Promise<T>((resolve, reject) => {
@@ -173,13 +194,21 @@ export function idempotent<S, T>(
     // An attempt was shed as overloaded: nothing is repeated after it.
     let refused = false;
     let settled = false;
+    // Attempts a hedge started rather than a retry or the first send.
+    let hedged = 0;
 
-    const settle = (answer: () => void): void => {
+    const settle = (outcome: DoCallOutcome, answeredBy: number | undefined, answer: () => void): void => {
       if (settled) return;
       settled = true;
       for (const timer of hedges) clearTimeout(timer);
       hedges.clear();
       answer();
+      span.set({
+        'do_call.attempts': started,
+        'do_call.hedges': hedged,
+        'do_call.answered_by': answeredBy,
+        'do_call.outcome': outcome,
+      });
     };
     /** May another attempt start at `at`? */
     const canRepeat = (at: number): boolean =>
@@ -187,13 +216,20 @@ export function idempotent<S, T>(
       && (retryWindowMs === undefined || at - startedAt <= retryWindowMs);
     /** `error` ended an attempt that will not be repeated: the call's answer, once nothing else is live. */
     const exhausted = <E>(error: E): void => {
-      if (live === 0) settle(() => reject(error));
+      if (live === 0) settle(refused ? 'overloaded' : 'exhausted', undefined, () => reject(error));
     };
 
     /** A failed attempt, numbered: repeat it after its backoff, or let it stand. */
     const failed = async <E>(number: number, error: E): Promise<void> => {
       if (settled) return;
       const classification = classifyDoCall(error);
+      if (!isRetryableDoCall(classification) && classification !== 'overloaded') {
+        // The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
+        settle('callee_error', number, () => reject(error));
+        return;
+      }
+      // A lost attempt: the call's span says which, and why.
+      span.exception(error, classification, `attempt ${number} of ${maxAttempts}: `);
       if (classification === 'overloaded') {
         // A shed call is no answer: nothing more is sent, and an attempt
         // still in flight may yet answer.
@@ -201,11 +237,6 @@ export function idempotent<S, T>(
         exhausted(error);
         return;
       }
-      if (!isRetryableDoCall(classification)) {
-        // The call ran and its answer is this error — ENOENT is a read's answer as much as bytes are.
-        settle(() => reject(error));
-        return;
-      }
       const delayMs = Math.floor(Math.random() * 2 ** number * baseDelayMs);
       if (!canRepeat(Date.now() + delayMs)) {
         exhausted(error);
@@ -236,7 +267,10 @@ export function idempotent<S, T>(
       }
       const hedge = hedgeAfterMs === undefined ? undefined : setTimeout(() => {
         if (hedge !== undefined) hedges.delete(hedge);
-        if (canRepeat(Date.now())) attempt();
+        if (canRepeat(Date.now())) {
+          hedged++;
+          attempt();
+        }
       }, hedgeAfterMs);
       if (hedge !== undefined) hedges.add(hedge);
       /** This attempt has its answer: it hedges no more, and its stub goes. */
@@ -263,12 +297,12 @@ export function idempotent<S, T>(
         disposeRpcResource(result);
         return;
       }
-      settle(() => resolve(result));
+      settle('answered', number, () => resolve(result));
     };
 
     /** Start an attempt. Whatever it throws outside the call itself fails the call. */
     const attempt = (): void => {
-      run().catch((error) => settle(() => reject(error)));
+      run().catch((error) => settle('caller_error', undefined, () => reject(error)));
     };
 
     attempt();