deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -6,6 +6,15 @@ import {
 } from './tool-result';
 import { assertNoSecretTaint } from './secret-capability';
 import type { RuntimeStepReceipt } from './ctx-types';
+import {
+  percentile,
+  runtimeJsonByteLength,
+  type RuntimeMemoryLedgerHandle,
+} from './memory-ledger';
+import type {
+  RuntimeMemoryOccupancy,
+  RuntimeMemoryOccupancyReader,
+} from './map-memory-limits';
 import type { DurableReceiptRecoverySource } from './tool-execution-outcome';
 import {
   isPlayExecutionSuspendedError,
@@ -30,6 +39,619 @@ import type {
 
 export const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
 export const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
+// A running receipt owned elsewhere is re-read with exponential backoff up to
+// this cap. Fixed 250 ms polls from every waiting group were most of the
+// gateway read traffic in a 1k map. Because the delay grows, the attempt count
+// alone no longer bounds wall time: every backoff loop must clamp itself to
+// `runtimeReceiptWaitBudgetMs`.
+export const DURABLE_RECEIPT_WAIT_MAX_DELAY_MS = 2_000;
+// The hydration lane re-examines its queue on this cadence while turns are
+// waiting, so headroom freed outside the lane reaches a waiter promptly. It is
+// a probe interval, not a width: nothing about it bounds concurrency.
+const RUNTIME_RECEIPT_HYDRATION_PROBE_MS = 25;
+
+/** Backoff delay before re-reading a running receipt on the given attempt (1-based). */
+export function runtimeReceiptWaitDelayMs(attempt: number): number {
+  const exponent = Math.max(0, Math.min(attempt, 8) - 1);
+  return Math.min(
+    DURABLE_RECEIPT_WAIT_MAX_DELAY_MS,
+    DURABLE_RECEIPT_WAIT_DELAY_MS * 2 ** exponent,
+  );
+}
+
+/**
+ * Whether a claim that came back `existing` is actually blocked by live work.
+ *
+ * A non-forced claim's status set is `('pending','failed')`, so a `running`
+ * row returns `existing` whether its owner is alive or long dead. Only an
+ * unexpired lease proves the former; anything else must be reclaimed and
+ * re-executed rather than waited on.
+ */
+export function runtimeReceiptClaimLeaseIsLive(
+  claim: Pick<RuntimeStepReceipt, 'leaseExpiresAt'>,
+  nowMs: number = Date.now(),
+): boolean {
+  const expiresAtMs = Date.parse(claim.leaseExpiresAt ?? '');
+  return !Number.isFinite(expiresAtMs) || expiresAtMs > nowMs;
+}
+
+/** Wall-clock budget that a fixed-delay attempt count used to express. */
+export function runtimeReceiptWaitBudgetMs(
+  maxAttempts: number,
+  delayMs: number = DURABLE_RECEIPT_WAIT_DELAY_MS,
+): number {
+  return Math.max(delayMs, maxAttempts * delayMs);
+}
+
+/**
+ * Settled per-key hydration samples the lane keeps to price a turn. Bounded,
+ * so past this many the reservoir below chooses which to keep.
+ */
+export const RUNTIME_RECEIPT_HYDRATION_SAMPLE_RESERVOIR = 256;
+/**
+ * Settled turns the lane needs before its own measured p90 per-key cost
+ * replaces the ledger's per-owner price. Sixteen matches the settled-row count
+ * map admission waits for before it trusts a p90, and for the same reason: it
+ * is enough for a percentile to mean something without making the first wave
+ * of a map wait for a tenth of itself.
+ */
+export const RUNTIME_RECEIPT_HYDRATION_SAMPLE_TURNS = 16;
+
+/**
+ * Bounded semantic origins for completed-receipt reads. These are runtime
+ * paths, never receipt keys, tool ids, or caller inputs, so the telemetry that
+ * explains a large read cannot become another retention surface.
+ */
+export const RUNTIME_RECEIPT_HYDRATION_SOURCES = [
+  'durable_operation',
+  'direct_tool_legacy_cache',
+  'direct_tool_result_receipt',
+  'map_cohort_read',
+  'map_legacy_claim',
+  'map_receipt_wait',
+  'receipt_completion_read_back',
+  'unknown',
+] as const;
+
+export type RuntimeReceiptHydrationSource =
+  (typeof RUNTIME_RECEIPT_HYDRATION_SOURCES)[number];
+
+export type RuntimeReceiptHydrationSourceSnapshot = {
+  turns: number;
+  receiptKeys: number;
+  bytesAdmitted: number;
+  bytesMeasured: number;
+};
+
+export type RuntimeReceiptHydrationGateSnapshot = {
+  /** Turns in flight, and turns queued behind the byte budget. */
+  active: number;
+  waiting: number;
+  /** Turns taken, and how long callers queued for them. */
+  turns: number;
+  queuedTurns: number;
+  totalWaitMs: number;
+  maxWaitMs: number;
+  /** Ledger bytes reserved by the turns in flight right now. */
+  bytesHeld: number;
+  bytesHeldPeak: number;
+  /** Ledger bytes this lane has admitted over its life. */
+  bytesAdmitted: number;
+  /** Bytes turns reported actually holding, replacing their reservation. */
+  bytesMeasured: number;
+  /** Admission attempts the byte budget refused. */
+  holds: number;
+  /** Collections taken to answer a hold before it was declared. */
+  holdCollections: number;
+  /**
+   * Turns admitted because nothing else was in flight. A turn larger than the
+   * whole budget still runs: one turn may always run, or a map whose payloads
+   * exceed the target would never finish a single row.
+   */
+  progressAdmissions: number;
+  /** Turns admitted with no readable heap, where byte admission cannot apply. */
+  unpricedAdmissions: number;
+  /** Ledger bytes the last decision priced one receipt key at. */
+  perKeyBytes: number | null;
+  /** Settled per-key samples behind that price; zero means it is the prior. */
+  perKeySamples: number;
+  /** Live bytes under the heap target at the last decision. */
+  headroomBytes: number | null;
+  /** Bounded provenance for the byte totals above. */
+  bySource: Record<
+    RuntimeReceiptHydrationSource,
+    RuntimeReceiptHydrationSourceSnapshot
+  >;
+};
+
+export type RuntimeReceiptHydrationTurn = {
+  /**
+   * Replace this turn's reservation with the bytes the read actually holds.
+   * Called once the response is in hand and before it is expanded, so the
+   * ledger stops carrying an estimate the moment a measurement exists.
+   */
+  settleBytes(bytes: number): void;
+  /** Drop the reservation. Idempotent. */
+  release(): void;
+};
+
+export type RuntimeReceiptHydrationGate = {
+  /**
+   * Take a turn on the hydration lane. `receiptKeys` is how many receipt keys
+   * the turn will read: the turn's expected bytes are that count times the
+   * price of one key. A manifest-aware reader may instead supply its exact
+   * serialized `expectedBytes` before redeeming the body.
+   */
+  acquire(input?: {
+    receiptKeys?: number;
+    expectedBytes?: number;
+    source?: RuntimeReceiptHydrationSource;
+    /**
+     * Wall-clock bound on the queue wait. A turn that cannot be admitted
+     * within it rejects with `RuntimeReceiptHydrationTurnTimeoutError` rather
+     * than parking forever behind a holder that never released.
+     */
+    timeoutMs?: number;
+    /** Abort the queue wait; the signal's reason is what rejects. */
+    signal?: AbortSignal;
+  }): Promise<RuntimeReceiptHydrationTurn>;
+  snapshot(): RuntimeReceiptHydrationGateSnapshot;
+  /**
+   * Release the admission timer and reject every still-queued turn. A closed
+   * lane will never admit again, so leaving those promises pending is the
+   * unbounded wait this contract forbids. Idempotent.
+   */
+  close(): void;
+};
+
+/**
+ * The completed-receipt hydration lane, admitted by bytes.
+ *
+ * A hydration turn holds the receipt gateway's response — the serialized
+ * outputs of every key it read — from the moment the response is parsed until
+ * the turn returns, plus the JSON expansion of those outputs while the turn
+ * hands them to their callers. Nothing else in the runtime accounts for those
+ * bytes, so this lane registers them on the byte ledger itself: an expected
+ * size on admission, the measured size once the response is read, and nothing
+ * at all once the turn is over.
+ *
+ * It used to be a counting semaphore of 8. That bound had no relation to
+ * bytes: eight turns of a megabyte-per-key shape hold two orders of magnitude
+ * more memory than eight turns of a fresh map's empty claims, and the constant
+ * could not tell them apart. Worse, the slot is taken at the dispatch group's
+ * claim and held across the gateway round trip (jacob p50 2.2 s, p90 6.3 s)
+ * before a single byte is expanded, so on a 1,000-row map 58% of turns queued
+ * 3.8 s each for a lane that was not protecting anything measurable.
+ *
+ * Policy, in order:
+ *   1. Nothing in flight admits immediately. One turn may always run, so a
+ *      turn whose payload exceeds the whole budget still makes progress
+ *      instead of deadlocking the run. This is the only floor in the lane and
+ *      it is exactly one turn.
+ *   2. A turn's expected bytes are `receiptKeys x perKeyBytes`. `perKeyBytes`
+ *      is the lane's own measured p90 of settled per-key sizes once it has
+ *      enough samples, and until then the ledger's per-owner price — the same
+ *      p90-or-prior map row admission uses.
+ *   3. Admission compares those expected live bytes — the reservation times
+ *      the `hydrationReadBack` factor, the category it registers on — against
+ *      the headroom `runtimeMemoryOccupancy` reports: heap target minus
+ *      (baseline + the ledger's per-category prediction). Because the
+ *      reservation is registered on the same ledger, an admitted turn's bytes
+ *      are already inside the next turn's occupancy; the lane does not keep a
+ *      second set of books.
+ *   4. Over budget is a question about garbage before it is a verdict about
+ *      turns: the lane collects (rate limited, through the shared reader) and
+ *      re-reads before it holds.
+ *   5. Waiters are FIFO. The head of the queue is the only candidate, so a
+ *      small turn cannot overtake a large one that has been waiting.
+ */
+export function createRuntimeReceiptHydrationGate(input: {
+  /** Byte occupancy of this process, shared with map row admission. */
+  occupancy: RuntimeMemoryOccupancyReader;
+  /** Registers this lane's reservations on the runtime byte ledger. */
+  reserve: (bytes: number) => RuntimeMemoryLedgerHandle;
+  /**
+   * Re-examines the queue while turns wait, so headroom freed by work outside
+   * this lane (a row settling, a body released) reaches a waiter without
+   * waiting for another turn to finish. Runs only while someone is waiting.
+   */
+  probeIntervalMs?: number;
+  sampleTurns?: number;
+  sampleReservoirSize?: number;
+  /** Uniform [0, 1) source for the sample reservoir; tests inject a sequence. */
+  random?: () => number;
+  now?: () => number;
+}): RuntimeReceiptHydrationGate {
+  const occupancyReader = input.occupancy;
+  const now = input.now ?? Date.now;
+  const probeIntervalMs = Math.max(
+    1,
+    Math.floor(input.probeIntervalMs ?? RUNTIME_RECEIPT_HYDRATION_PROBE_MS),
+  );
+  const sampleTurns = Math.max(
+    1,
+    Math.floor(input.sampleTurns ?? RUNTIME_RECEIPT_HYDRATION_SAMPLE_TURNS),
+  );
+  const sampleReservoirSize = Math.max(
+    1,
+    Math.floor(
+      input.sampleReservoirSize ?? RUNTIME_RECEIPT_HYDRATION_SAMPLE_RESERVOIR,
+    ),
+  );
+  const uniform = (): number => {
+    const value = input.random ? input.random() : Math.random();
+    return Number.isFinite(value)
+      ? Math.min(0.999999999, Math.max(0, value))
+      : 0;
+  };
+
+  let active = 0;
+  let turns = 0;
+  let queuedTurns = 0;
+  let totalWaitMs = 0;
+  let maxWaitMs = 0;
+  let bytesHeld = 0;
+  let bytesHeldPeak = 0;
+  let bytesAdmitted = 0;
+  let bytesMeasured = 0;
+  let holds = 0;
+  let holdCollections = 0;
+  let progressAdmissions = 0;
+  let unpricedAdmissions = 0;
+  let perKeyBytes: number | null = null;
+  let headroomBytes: number | null = null;
+  let closed = false;
+  let sampledTurns = 0;
+  const perKeySamples: number[] = [];
+  const bySource = Object.fromEntries(
+    RUNTIME_RECEIPT_HYDRATION_SOURCES.map((source) => [
+      source,
+      { turns: 0, receiptKeys: 0, bytesAdmitted: 0, bytesMeasured: 0 },
+    ]),
+  ) as Record<
+    RuntimeReceiptHydrationSource,
+    RuntimeReceiptHydrationSourceSnapshot
+  >;
+  type HydrationWaiter = {
+    receiptKeys: number;
+    expectedBytes: number | null;
+    source: RuntimeReceiptHydrationSource;
+    queuedAtMs: number;
+    /** Set when the waiter is admitted, so `acquire` can tell a queue apart
+     * from an immediate grant without scanning the queue for itself. */
+    admitted: boolean;
+    resolve: (turn: RuntimeReceiptHydrationTurn) => void;
+    /** Settles a waiter the lane will never admit. Cleared once admitted. */
+    reject: (reason: unknown) => void;
+    abandon: (() => void) | null;
+  };
+  const waiters: HydrationWaiter[] = [];
+  let timer: ReturnType<typeof setInterval> | null = null;
+
+  const recordPerKeySample = (bytesPerKey: number): void => {
+    sampledTurns += 1;
+    if (perKeySamples.length < sampleReservoirSize) {
+      perKeySamples.push(bytesPerKey);
+      return;
+    }
+    // Vitter's Algorithm R, for the same reason the ledger's settled-owner
+    // reservoir uses it: a cache-hit wave and a fresh-claim wave settle at
+    // very different sizes, and keeping the most recent samples would price
+    // the lane from whichever wave happened to be last.
+    const slot = Math.floor(uniform() * sampledTurns);
+    if (slot < sampleReservoirSize) perKeySamples[slot] = bytesPerKey;
+  };
+
+  /** Ledger bytes one receipt key is expected to cost. */
+  const priceKey = (occupancy: RuntimeMemoryOccupancy): number => {
+    const measured =
+      perKeySamples.length >= sampleTurns
+        ? percentile(perKeySamples, 0.9)
+        : null;
+    // The lane's own measurement is what a receipt read costs. Before it has
+    // one, a receipt key is priced as a row: the ledger's settled p90 per
+    // owner, or the unmeasured prior behind it. That is an overestimate for a
+    // claim that returns no output, and deliberately so — the alternative is
+    // admitting an unknown payload against a budget it may not fit.
+    return measured === null ? occupancy.ownerReserveLedgerBytes : measured;
+  };
+
+  /** Prefer a manifest's exact serialized body size over the conservative prior. */
+  const expectedHydrationBytes = (
+    waiter: HydrationWaiter,
+    occupancy: RuntimeMemoryOccupancy,
+  ): number =>
+    waiter.expectedBytes === null
+      ? Math.max(1, Math.ceil(waiter.receiptKeys * priceKey(occupancy)))
+      : Math.max(1, waiter.expectedBytes);
+
+  /**
+   * Live bytes per ledger byte for what this lane actually registers.
+   *
+   * The lane's reservations land on `hydrationReadBack`, so that is the
+   * category whose factor converts them. The process-wide mix-weighted factor
+   * is a different question's answer: a run whose ledger is mostly decoded
+   * provider bodies reports ~3x, and pricing a receipt read at 3x its wire
+   * size holds turns against headroom the read was never going to occupy.
+   */
+  const hydrationFactor = (occupancy: RuntimeMemoryOccupancy): number =>
+    occupancy.categoryFactors.hydrationReadBack;
+
+  const stopTimer = (): void => {
+    if (timer === null) return;
+    clearInterval(timer);
+    timer = null;
+  };
+
+  const startTimer = (): void => {
+    if (timer !== null || closed || waiters.length === 0) return;
+    timer = setInterval(() => {
+      admitWaiters();
+    }, probeIntervalMs);
+    if (typeof timer === 'object' && 'unref' in timer) timer.unref();
+  };
+
+  /** Drop a waiter that will never be admitted and settle it loudly. */
+  const abandonWaiter = (waiter: HydrationWaiter, reason: unknown): void => {
+    const index = waiters.indexOf(waiter);
+    if (index >= 0) waiters.splice(index, 1);
+    waiter.abandon?.();
+    waiter.abandon = null;
+    waiter.reject(reason);
+    if (waiters.length === 0) stopTimer();
+  };
+
+  const admit = (waiter: HydrationWaiter, reservedBytes: number): void => {
+    waiters.shift();
+    waiter.admitted = true;
+    waiter.abandon?.();
+    waiter.abandon = null;
+    active += 1;
+    turns += 1;
+    const waitedMs = Math.max(0, now() - waiter.queuedAtMs);
+    totalWaitMs += waitedMs;
+    maxWaitMs = Math.max(maxWaitMs, waitedMs);
+    const handle = input.reserve(reservedBytes);
+    bytesAdmitted += reservedBytes;
+    const source = bySource[waiter.source];
+    source.turns += 1;
+    source.receiptKeys += waiter.receiptKeys;
+    source.bytesAdmitted += reservedBytes;
+    bytesHeld += reservedBytes;
+    bytesHeldPeak = Math.max(bytesHeldPeak, bytesHeld);
+    let entryBytes = reservedBytes;
+    let settled = false;
+    let released = false;
+    waiter.resolve({
+      settleBytes(bytes) {
+        if (released || settled) return;
+        if (!Number.isFinite(bytes) || bytes < 0) {
+          throw new RangeError(
+            `Receipt hydration turn reported ${String(bytes)} bytes; a turn's measured size must be a finite, non-negative byte count.`,
+          );
+        }
+        settled = true;
+        const measuredBytes = Math.ceil(bytes);
+        handle.update(measuredBytes);
+        bytesHeld = Math.max(0, bytesHeld - entryBytes + measuredBytes);
+        bytesHeldPeak = Math.max(bytesHeldPeak, bytesHeld);
+        bytesMeasured += measuredBytes;
+        source.bytesMeasured += measuredBytes;
+        entryBytes = measuredBytes;
+        recordPerKeySample(
+          Math.ceil(measuredBytes / Math.max(1, waiter.receiptKeys)),
+        );
+        // A turn that turned out smaller than its reservation just freed
+        // headroom; hand it to whoever is waiting rather than to the clock.
+        admitWaiters();
+      },
+      release() {
+        if (released) return;
+        released = true;
+        handle.release();
+        bytesHeld = Math.max(0, bytesHeld - entryBytes);
+        entryBytes = 0;
+        active = Math.max(0, active - 1);
+        admitWaiters();
+      },
+    });
+  };
+
+  function admitWaiters(): void {
+    if (closed) return;
+    while (waiters.length > 0) {
+      const waiter = waiters[0];
+      if (!waiter) break;
+      // One turn may always run. Without this a group whose expected bytes
+      // exceed the whole heap target could never start, and the run would
+      // stall on a lane that is protecting an empty process.
+      if (active === 0) {
+        progressAdmissions += 1;
+        const occupancy = occupancyReader.read();
+        admit(
+          waiter,
+          occupancy === null ? 0 : expectedHydrationBytes(waiter, occupancy),
+        );
+        continue;
+      }
+      let occupancy = occupancyReader.read();
+      if (occupancy === null) {
+        // No readable heap: this runtime cannot state a budget, so the lane
+        // does not invent one. The structural receipt limits still apply.
+        unpricedAdmissions += 1;
+        admit(waiter, 0);
+        continue;
+      }
+      let reservedBytes = expectedHydrationBytes(waiter, occupancy);
+      perKeyBytes = Math.ceil(reservedBytes / Math.max(1, waiter.receiptKeys));
+      headroomBytes = occupancy.headroomBytes;
+      if (
+        reservedBytes * hydrationFactor(occupancy) <=
+        occupancy.headroomBytes
+      ) {
+        admit(waiter, reservedBytes);
+        continue;
+      }
+      // Over budget is a question about garbage first. Collect and re-read
+      // before concluding that the turns in flight are what is full.
+      const collected = occupancyReader.collectAndRead();
+      if (collected) {
+        holdCollections += 1;
+        occupancy = collected;
+        reservedBytes = expectedHydrationBytes(waiter, occupancy);
+        perKeyBytes = Math.ceil(
+          reservedBytes / Math.max(1, waiter.receiptKeys),
+        );
+        headroomBytes = occupancy.headroomBytes;
+        if (
+          reservedBytes * hydrationFactor(occupancy) <=
+          occupancy.headroomBytes
+        ) {
+          admit(waiter, reservedBytes);
+          continue;
+        }
+      }
+      holds += 1;
+      break;
+    }
+    if (waiters.length === 0) stopTimer();
+    else startTimer();
+  }
+
+  return {
+    acquire(request) {
+      const requestedKeys = request?.receiptKeys;
+      const receiptKeys =
+        typeof requestedKeys === 'number' && Number.isFinite(requestedKeys)
+          ? Math.max(1, Math.floor(requestedKeys))
+          : 1;
+      const source =
+        request?.source &&
+        RUNTIME_RECEIPT_HYDRATION_SOURCES.includes(request.source)
+          ? request.source
+          : 'unknown';
+      const expectedBytes =
+        typeof request?.expectedBytes === 'number' &&
+        Number.isFinite(request.expectedBytes) &&
+        request.expectedBytes >= 0
+          ? Math.ceil(request.expectedBytes)
+          : null;
+      const timeoutMs =
+        typeof request?.timeoutMs === 'number' &&
+        Number.isFinite(request.timeoutMs) &&
+        request.timeoutMs >= 0
+          ? Math.floor(request.timeoutMs)
+          : null;
+      const signal = request?.signal;
+      if (closed) {
+        return Promise.reject(new RuntimeReceiptHydrationGateClosedError());
+      }
+      if (signal?.aborted) {
+        return Promise.reject(hydrationAbortReason(signal));
+      }
+      return new Promise<RuntimeReceiptHydrationTurn>((resolve, reject) => {
+        const queuedAtMs = now();
+        const waiter: HydrationWaiter = {
+          receiptKeys,
+          expectedBytes,
+          source,
+          queuedAtMs,
+          admitted: false,
+          resolve,
+          reject,
+          abandon: null,
+        };
+        waiters.push(waiter);
+        let timer: ReturnType<typeof setTimeout> | null = null;
+        const onAbort = () =>
+          abandonWaiter(waiter, hydrationAbortReason(signal!));
+        if (timeoutMs !== null || signal) {
+          if (timeoutMs !== null) {
+            timer = setTimeout(() => {
+              timer = null;
+              abandonWaiter(
+                waiter,
+                new RuntimeReceiptHydrationTurnTimeoutError(
+                  Math.max(0, now() - queuedAtMs),
+                ),
+              );
+            }, timeoutMs);
+            if (typeof timer === 'object' && 'unref' in timer) timer.unref();
+          }
+          signal?.addEventListener('abort', onAbort, { once: true });
+          waiter.abandon = () => {
+            if (timer !== null) clearTimeout(timer);
+            timer = null;
+            signal?.removeEventListener('abort', onAbort);
+          };
+        }
+        admitWaiters();
+        // Counted only when the byte budget actually made this turn wait, so
+        // `queuedTurns / turns` stays the share of turns the lane delayed.
+        if (!waiter.admitted) queuedTurns += 1;
+      });
+    },
+    snapshot: () => ({
+      active,
+      waiting: waiters.length,
+      turns,
+      queuedTurns,
+      totalWaitMs,
+      maxWaitMs,
+      bytesHeld,
+      bytesHeldPeak,
+      bytesAdmitted,
+      bytesMeasured,
+      holds,
+      holdCollections,
+      progressAdmissions,
+      unpricedAdmissions,
+      perKeyBytes,
+      perKeySamples: perKeySamples.length,
+      headroomBytes,
+      bySource: Object.fromEntries(
+        RUNTIME_RECEIPT_HYDRATION_SOURCES.map((source) => [
+          source,
+          { ...bySource[source] },
+        ]),
+      ) as Record<
+        RuntimeReceiptHydrationSource,
+        RuntimeReceiptHydrationSourceSnapshot
+      >,
+    }),
+    close() {
+      closed = true;
+      stopTimer();
+      // A closed lane will never admit again. Stranding queued turns on an
+      // unsettleable promise is the wait this contract forbids.
+      const stranded = waiters.splice(0, waiters.length);
+      for (const waiter of stranded) {
+        waiter.abandon?.();
+        waiter.abandon = null;
+        waiter.reject(new RuntimeReceiptHydrationGateClosedError());
+      }
+    },
+  };
+}
+
+/**
+ * Serialized size of the receipt outputs a hydration turn is about to expand
+ * and hold. Only stored outputs are measured: a claim that returned no output
+ * costs the lane nothing but its own row, and serializing the receipt envelope
+ * would charge the lane for bytes it does not retain.
+ */
+export function runtimeStepReceiptPayloadBytes(
+  receipts: Iterable<RuntimeStepReceipt | null | undefined>,
+): number {
+  let bytes = 0;
+  for (const receipt of receipts) {
+    const output = receipt?.output;
+    if (output === undefined || output === null) continue;
+    bytes += runtimeJsonByteLength(output);
+  }
+  return bytes;
+}
+
 const TOOL_RECEIPT_DEFAULT_WAIT_MS = 300_000;
 const TOOL_RECEIPT_COMPLETION_BUFFER_MS = 30_000;
 const TOOL_RECEIPT_MAX_WAIT_MS = 30 * 60_000;
@@ -37,6 +659,32 @@ const DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS = 15_000;
 export const COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID =
   '__deepline_completed_cache_insert__';
 
+/** The error an aborted queue wait settles with. */
+function hydrationAbortReason(signal: AbortSignal): unknown {
+  if (signal.reason instanceof Error) return signal.reason;
+  return new Error(
+    typeof signal.reason === 'string' && signal.reason.trim()
+      ? signal.reason
+      : 'Receipt hydration turn aborted.',
+  );
+}
+
+export class RuntimeReceiptHydrationTurnTimeoutError extends Error {
+  constructor(readonly waitedMs: number) {
+    super(
+      `Timed out after ${waitedMs}ms waiting for a receipt hydration turn.`,
+    );
+    this.name = 'RuntimeReceiptHydrationTurnTimeoutError';
+  }
+}
+
+export class RuntimeReceiptHydrationGateClosedError extends Error {
+  constructor() {
+    super('Receipt hydration lane closed while a turn was still queued.');
+    this.name = 'RuntimeReceiptHydrationGateClosedError';
+  }
+}
+
 export class RuntimeReceiptWaitTimeoutError extends Error {
   constructor(key: string) {
     super(`Timed out waiting for durable receipt ${key}.`);
@@ -520,6 +1168,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
         );
       }
       const deadline = Date.now() + DANGEROUS_SIDE_EFFECT_LOCK_WAIT_MS;
+      let completedWaitAttempt = 0;
       while (!ownsLock) {
         const lock = await input.store.acquireExecutionLock({
           receiptKey: input.receiptKey,
@@ -538,7 +1187,13 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
         if (Date.now() >= deadline) {
           throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
         }
-        await sleepReceiptWait(DURABLE_RECEIPT_WAIT_DELAY_MS);
+        completedWaitAttempt += 1;
+        await sleepReceiptWait(
+          Math.min(
+            runtimeReceiptWaitDelayMs(completedWaitAttempt),
+            Math.max(0, deadline - Date.now()),
+          ),
+        );
       }
       if (input.force !== true) {
         const afterLock = await inspectCompletedReceipt(
@@ -616,33 +1271,30 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       if (input.force === true) {
         return result;
       }
-      // Completion normally returns a compact acknowledgement. Keep the
-      // mutation and response decode inside the hydration turn anyway because
-      // a concurrent winner may return a full persisted payload.
-      const published = await withCompletedReceiptHydration(async () => {
-        const completed = await input.store.complete(
-          input.receiptKey,
-          input.runId,
-          isToolExecuteResult(result)
-            ? serializeToolExecuteResult(result)
-            : result,
-          COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+      // The hydration window bounds how many stored payloads expand at once.
+      // A completion is a write, and on wire format 3 it answers with a
+      // compact acknowledgement carrying no payload at all — nothing expands,
+      // so nothing needs a turn. Holding a slot of the 8-wide run-global
+      // window across the whole gateway round trip only queued unrelated rows
+      // behind this row's write. Only a response that really carries a stored
+      // payload decodes under the gate.
+      const completed = await input.store.complete(
+        input.receiptKey,
+        input.runId,
+        isToolExecuteResult(result)
+          ? serializeToolExecuteResult(result)
+          : result,
+        COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
+      );
+      if (
+        completed?.status === 'completed' ||
+        completed?.status === 'skipped'
+      ) {
+        if (completed.output === undefined) return result;
+        return await withCompletedReceiptHydration(
+          async () => await recoverCompletedReceipt(completed, 'owner'),
         );
-        if (
-          completed?.status === 'completed' ||
-          completed?.status === 'skipped'
-        ) {
-          return {
-            kind: 'completed' as const,
-            output:
-              completed.output === undefined
-                ? result
-                : await recoverCompletedReceipt(completed, 'owner'),
-          };
-        }
-        return { kind: 'unresolved' as const };
-      });
-      if (published.kind === 'completed') return published.output;
+      }
       const winner = await inspectCompletedReceipt(
         () => input.store.get(input.receiptKey),
         'owner',
@@ -704,8 +1356,22 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       input.runningReceiptWaitMaxAttempts ?? DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
     const delayMs =
       input.runningReceiptWaitDelayMs ?? DURABLE_RECEIPT_WAIT_DELAY_MS;
+    // An explicit delay keeps the caller's fixed cadence; the default backs off.
+    const backoff = input.runningReceiptWaitDelayMs == null;
+    const waitDeadline =
+      Date.now() + runtimeReceiptWaitBudgetMs(maxAttempts, delayMs);
     for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
-      if (attempt > 0) await sleepReceiptWait(delayMs);
+      if (attempt > 0) {
+        if (backoff && Date.now() >= waitDeadline) break;
+        await sleepReceiptWait(
+          backoff
+            ? Math.min(
+                runtimeReceiptWaitDelayMs(attempt),
+                Math.max(0, waitDeadline - Date.now()),
+              )
+            : delayMs,
+        );
+      }
       const current = await input.store.get(input.receiptKey);
       if (current?.status === 'completed' || current?.status === 'skipped') {
         return {