deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts

@@ -0,0 +1,167 @@
+/**
+ * Sandbox egress bandwidth probe.
+ *
+ * WHY: on the jai fixture the receipt writer's completion queue reached 23 s
+ * per receipt while the gateway's heavy lane sat at 11% of one slot: the
+ * flights themselves moved ~650 KB in ~5 s, about 1 Mbit/s per flow on a
+ * 95 ms path, and the whole run achieved 0.3 MB/s of wire. The RTT probe
+ * cannot tell a per-flow window limit from a per-sandbox egress cap, and the
+ * two have different fixes (more flights in parallel vs. fewer bytes). One
+ * upload of incompressible bytes as a single flow, then as several parallel
+ * flows, tells them apart at boot: parallel throughput scaling with flow
+ * count is a per-flow window; a flat sum is the link.
+ */
+
+export const RUNTIME_GATEWAY_EGRESS_PROBE_PATH = '/egress-probe';
+/** The sink discards bytes; a caller cannot make it hold more than this. */
+export const RUNTIME_GATEWAY_EGRESS_PROBE_MAX_BYTES = 1024 * 1024;
+export const EGRESS_BANDWIDTH_PROBE_SINGLE_BYTES = 256 * 1024;
+export const EGRESS_BANDWIDTH_PROBE_PARALLEL_FLOWS = 4;
+export const EGRESS_BANDWIDTH_PROBE_PARALLEL_BYTES_EACH = 128 * 1024;
+/**
+ * At the 1 Mbit/s observed on jai the single flow alone takes ~2 s and the
+ * parallel set ~1-4 s; the budget bounds the boot cost when the link is worse.
+ */
+export const EGRESS_BANDWIDTH_PROBE_BUDGET_MS = 12_000;
+
+export type EgressBandwidthObservation = {
+  target: string;
+  singleBytes: number;
+  singleMs: number | null;
+  parallelFlows: number;
+  parallelBytesEach: number;
+  parallelMs: number | null;
+  errorClass: string | null;
+};
+
+export function egressMbitPerSecond(
+  bytes: number,
+  ms: number | null,
+): number | null {
+  if (ms === null || !(ms > 0) || !(bytes > 0)) return null;
+  return (bytes * 8) / (ms / 1000) / 1_000_000;
+}
+
+export function formatEgressBandwidthLine(
+  observation: EgressBandwidthObservation,
+): string {
+  const number = (value: number | null) =>
+    value === null ? 'none' : String(Math.round(value));
+  const rate = (value: number | null) =>
+    value === null ? 'none' : value.toFixed(2);
+  const parallelTotalBytes =
+    observation.parallelFlows * observation.parallelBytesEach;
+  return (
+    `[perf] egress_bandwidth target=${observation.target} ` +
+    `single_bytes=${observation.singleBytes} single_ms=${number(observation.singleMs)} ` +
+    `single_mbit=${rate(egressMbitPerSecond(observation.singleBytes, observation.singleMs))} ` +
+    `parallel_flows=${observation.parallelFlows} parallel_bytes_each=${observation.parallelBytesEach} ` +
+    `parallel_ms=${number(observation.parallelMs)} ` +
+    `parallel_mbit=${rate(egressMbitPerSecond(parallelTotalBytes, observation.parallelMs))}` +
+    (observation.errorClass ? ` error=${observation.errorClass}` : '')
+  );
+}
+
+/**
+ * Count and discard a request body without buffering it. Stops reading past
+ * `maxBytes` so a client cannot make the sink do more than the cap of work.
+ */
+export async function readEgressProbeBody(
+  body: ReadableStream<Uint8Array> | null,
+  maxBytes: number,
+): Promise<{ bytes: number; overLimit: boolean }> {
+  if (!body) return { bytes: 0, overLimit: false };
+  const reader = body.getReader();
+  let bytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value?.byteLength ?? 0;
+      if (bytes > maxBytes) {
+        await reader.cancel();
+        return { bytes, overLimit: true };
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return { bytes, overLimit: false };
+}
+
+/** Incompressible payload so a compressing hop cannot flatter the number. */
+export function egressProbePayload(bytes: number): Uint8Array {
+  const payload = new Uint8Array(bytes);
+  const chunk = 64 * 1024;
+  for (let offset = 0; offset < bytes; offset += chunk) {
+    globalThis.crypto.getRandomValues(
+      payload.subarray(offset, Math.min(bytes, offset + chunk)),
+    );
+  }
+  return payload;
+}
+
+export async function probeEgressBandwidth(input: {
+  target: string;
+  baseUrl: string;
+  fetchImpl: typeof fetch;
+  headers: Record<string, string>;
+  deadlineAtMs: number;
+  now?: () => number;
+  singleBytes?: number;
+  parallelFlows?: number;
+  parallelBytesEach?: number;
+}): Promise<EgressBandwidthObservation> {
+  const now = input.now ?? (() => Date.now());
+  const url = `${input.baseUrl.replace(/\/$/, '')}${RUNTIME_GATEWAY_EGRESS_PROBE_PATH}`;
+  const singleBytes = input.singleBytes ?? EGRESS_BANDWIDTH_PROBE_SINGLE_BYTES;
+  const parallelFlows =
+    input.parallelFlows ?? EGRESS_BANDWIDTH_PROBE_PARALLEL_FLOWS;
+  const parallelBytesEach =
+    input.parallelBytesEach ?? EGRESS_BANDWIDTH_PROBE_PARALLEL_BYTES_EACH;
+  const observation: EgressBandwidthObservation = {
+    target: input.target,
+    singleBytes,
+    singleMs: null,
+    parallelFlows,
+    parallelBytesEach,
+    parallelMs: null,
+    errorClass: null,
+  };
+  const upload = async (bytes: number): Promise<void> => {
+    const remainingMs = input.deadlineAtMs - now();
+    if (remainingMs <= 0) throw new Error('egress probe budget exhausted');
+    const response = await input.fetchImpl(url, {
+      method: 'POST',
+      headers: {
+        ...input.headers,
+        'content-type': 'application/octet-stream',
+      },
+      // Use the exact ArrayBuffer accepted by DOM fetch. The fresh byte view
+      // prevents Node's generic ArrayBufferLike type from widening this into
+      // a SharedArrayBuffer-capable body, while preserving the probe payload.
+      body: Uint8Array.from(egressProbePayload(bytes)).buffer,
+      signal: AbortSignal.timeout(remainingMs),
+    });
+    await response.arrayBuffer();
+    if (!response.ok) throw new Error(`http_${response.status}`);
+  };
+  try {
+    const singleStartedAt = now();
+    await upload(singleBytes);
+    observation.singleMs = now() - singleStartedAt;
+    const parallelStartedAt = now();
+    await Promise.all(
+      Array.from({ length: parallelFlows }, () => upload(parallelBytesEach)),
+    );
+    observation.parallelMs = now() - parallelStartedAt;
+  } catch (error) {
+    observation.errorClass =
+      error instanceof Error
+        ? error.message.startsWith('http_')
+          ? error.message
+          : error.constructor.name
+        : typeof error;
+  }
+  return observation;
+}

package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts

@@ -0,0 +1,103 @@
+/**
+ * Response-path stages emitted by the deployed integration execute endpoint.
+ *
+ * This is deliberately a small, additive observability contract: the runner
+ * treats a missing or malformed header as absent telemetry, never as a failed
+ * tool call. Keeping the names here lets the response writer and the runner
+ * agree without inventing a runtime protocol version.
+ */
+export const EXECUTE_SERVER_TIMING_STAGES = [
+  'auth',
+  'rate_limit',
+  'catalog',
+  'charge_open',
+  // Charge admission substages. `charge_open` keeps its original meaning — the
+  // TOTAL of all charge-admission work — so an existing reader is unchanged and
+  // these four only explain where that total went. They are disjoint slices of
+  // it, and their sum can be below it (the difference is app-side work between
+  // the round trips), never above.
+  // - `charge_eligibility`    the fast-postpaid eligibility snapshot lookup
+  //                           (Redis GET, then a Convex query on a miss)
+  // - `charge_cap_preflight`  the `preflightChargeCap` mutation, its retries
+  //                           included
+  // - `charge_open_mutation`  the `openCharge` mutation, its retries included
+  // - `charge_retry_wait`     wall slept in write-conflict/OCC/capacity-shed
+  //                           backoff across both mutations. A high value here
+  //                           is same-org admission contention, not Convex
+  //                           latency, and is the only way to tell them apart.
+  'charge_eligibility',
+  'charge_cap_preflight',
+  'charge_open_mutation',
+  'charge_retry_wait',
+  'fixture_load',
+  'execute',
+  'identity',
+  'usage_schedule',
+  'total',
+  // The execution gateway preserves these disjoint stages beside the app
+  // handler's stages. They are deliberately part of this small, additive
+  // response diagnostic contract: a long sandbox-to-headers wall is otherwise
+  // incorrectly reduced to "network" after the app handler reports quickly.
+  'auth_launch',
+  'auth_verify',
+  'body_parse',
+  'dispatch_pool',
+  'dispatch_callback',
+  // The execution relay measures its own upstream fetch. `upstream_headers`
+  // contains the app handler's `total`; subtracting those two clocks leaves
+  // only the Vercel/platform-or-transit boundary, never a made-up provider
+  // latency. `upstream_body` follows headers while the relay buffers the
+  // bounded response before it returns to the runner.
+  'upstream_headers',
+  'upstream_body',
+] as const;
+
+export type ExecuteServerTimingStage =
+  (typeof EXECUTE_SERVER_TIMING_STAGES)[number];
+
+export type ExecuteServerTiming = Record<
+  ExecuteServerTimingStage,
+  number | null
+>;
+
+/**
+ * Read the known `Server-Timing` durations from an integration execute
+ * response. Browser/proxy changes to this diagnostic header must never change
+ * execution behavior, so malformed, negative, and absent values remain null.
+ */
+export function parseExecuteServerTiming(
+  header: string | null | undefined,
+): ExecuteServerTiming | null {
+  if (typeof header !== 'string' || !header.trim()) return null;
+  const durations = new Map<string, number>();
+  for (const rawEntry of header.split(',')) {
+    const parts = rawEntry.split(';');
+    const name = parts[0]?.trim().toLowerCase();
+    if (!name) continue;
+    for (const parameter of parts.slice(1)) {
+      const separator = parameter.indexOf('=');
+      if (separator < 0) continue;
+      const key = parameter.slice(0, separator).trim().toLowerCase();
+      if (key !== 'dur') continue;
+      const value = parameter
+        .slice(separator + 1)
+        .trim()
+        .replace(/^"|"$/g, '');
+      // `Number('')` is 0; do not turn a truncated header into a confident
+      // measurement. Server-Timing durations are non-negative by definition.
+      const parsed = value === '' ? Number.NaN : Number(value);
+      if (Number.isFinite(parsed) && parsed >= 0) {
+        durations.set(name, parsed);
+      }
+    }
+  }
+  const timing = Object.fromEntries(
+    EXECUTE_SERVER_TIMING_STAGES.map((stage) => [
+      stage,
+      durations.get(stage) ?? null,
+    ]),
+  ) as ExecuteServerTiming;
+  return EXECUTE_SERVER_TIMING_STAGES.some((stage) => timing[stage] !== null)
+    ? timing
+    : null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts

@@ -0,0 +1,34 @@
+import {
+  isRuntimeReceiptValueReference,
+  type RuntimeReceiptValueReference,
+} from './runtime-receipt-value';
+
+/**
+ * A trusted execution relay sets this only after it has written the complete
+ * serialized ToolExecuteResult under the scoped result-receipt key. It is an
+ * optimization hint: callers retain the normal provider response and may
+ * always fall back to the legacy completion payload when it is absent.
+ */
+export const EXECUTION_GATEWAY_RECEIPT_OUTPUT_REFERENCE_HEADER =
+  'x-deepline-runtime-receipt-output-reference';
+
+export function encodeExecutionGatewayReceiptOutputReference(
+  reference: RuntimeReceiptValueReference,
+): string {
+  return Buffer.from(JSON.stringify(reference), 'utf8').toString('base64url');
+}
+
+export function parseExecutionGatewayReceiptOutputReference(
+  value: string | null | undefined,
+): RuntimeReceiptValueReference | null {
+  const encoded = value?.trim();
+  if (!encoded || encoded.length > 8 * 1024) return null;
+  try {
+    const parsed = JSON.parse(
+      Buffer.from(encoded, 'base64url').toString('utf8'),
+    ) as unknown;
+    return isRuntimeReceiptValueReference(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts

@@ -0,0 +1,27 @@
+/**
+ * The execution gateway buffers an upstream response before it relays it to a
+ * sandbox. `content-length` is not safe to forward: it can describe the
+ * upstream compressed representation rather than the decoded bytes the
+ * sandbox will materialize. This additive header instead describes the exact
+ * decoded ArrayBuffer the gateway has already bounded and is about to send.
+ *
+ * An older gateway simply omits the header. Callers must retain their
+ * conservative sizing fallback in that case; this is an observation, never a
+ * required protocol version.
+ */
+export const EXECUTION_GATEWAY_DECODED_BODY_BYTES_HEADER =
+  'x-deepline-execution-gateway-decoded-body-bytes';
+
+/**
+ * Parse the exact decoded size attached by a trusted execution gateway.
+ * Reject every lossy or unbounded representation before it can lower a memory
+ * reservation.
+ */
+export function executionGatewayDecodedBodyBytes(
+  headers: Headers,
+): number | null {
+  const raw = headers.get(EXECUTION_GATEWAY_DECODED_BODY_BYTES_HEADER)?.trim();
+  if (!raw || !/^(?:0|[1-9][0-9]*)$/.test(raw)) return null;
+  const bytes = Number(raw);
+  return Number.isSafeInteger(bytes) ? bytes : null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts

@@ -3,11 +3,21 @@ export type GatewayPostgresLane = 'ordinary' | 'commit';
 export const GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE = 8;
 export const GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE = 10;
 export const GATEWAY_POSTGRES_MAX_CONSECUTIVE_COMMIT_PRIORITY_GRANTS = 8;
+/**
+ * A parked waiter costs a request slot, its request body, and its abort
+ * plumbing for as long as it waits. Under a large map every Machine can be
+ * offered far more concurrent work than its pooled clients can ever drain, so
+ * an unbounded queue converts back pressure into unbounded memory and a wait
+ * no caller is still around for. Past this depth the gateway sheds instead:
+ * the provider has not been called, so the caller may retry immediately.
+ */
+export const GATEWAY_POSTGRES_MAX_WAITING_PER_LANE = 256;
 
 export type GatewayPostgresAdmissionSnapshot = {
   active: Record<GatewayPostgresLane, number> & { total: number };
   waiting: Record<GatewayPostgresLane, number>;
   cancelled: number;
+  overloaded: number;
   commitPriorityGrants: number;
 };
 
@@ -18,6 +28,32 @@ export class GatewayPostgresAdmissionCancelledError extends Error {
   }
 }
 
+/**
+ * Load shed at the admission boundary. It is only ever thrown before a waiter
+ * is enqueued, so no pooled client, statement, or provider call exists for the
+ * rejected request. Callers must map it to a retryable back-pressure response.
+ */
+export class GatewayPostgresAdmissionOverloadedError extends Error {
+  readonly code = 'GATEWAY_POSTGRES_ADMISSION_OVERLOADED';
+  readonly lane: GatewayPostgresLane;
+  readonly waiting: number;
+  readonly limit: number;
+
+  constructor(input: {
+    lane: GatewayPostgresLane;
+    waiting: number;
+    limit: number;
+  }) {
+    super(
+      `Gateway Postgres ${input.lane} admission queue is full (${input.waiting} waiting, limit ${input.limit}).`,
+    );
+    this.name = 'GatewayPostgresAdmissionOverloadedError';
+    this.lane = input.lane;
+    this.waiting = input.waiting;
+    this.limit = input.limit;
+  }
+}
+
 type Waiter = {
   lane: GatewayPostgresLane;
   signal: AbortSignal | null;
@@ -35,6 +71,7 @@ export class GatewayPostgresAdmission {
   #ordinaryActive = 0;
   #commitActive = 0;
   #cancelled = 0;
+  #overloaded = 0;
   #commitPriorityGrants = 0;
   #consecutiveCommitPriorityGrants = 0;
 
@@ -42,8 +79,19 @@ export class GatewayPostgresAdmission {
     private readonly limits: {
       maxOrdinaryActive?: number;
       maxTotalActive?: number;
+      maxWaitingPerLane?: number;
     } = {},
-  ) {}
+  ) {
+    if (
+      limits.maxWaitingPerLane !== undefined &&
+      (!Number.isSafeInteger(limits.maxWaitingPerLane) ||
+        limits.maxWaitingPerLane < 1)
+    ) {
+      throw new Error(
+        'Gateway Postgres admission maxWaitingPerLane must be a positive integer.',
+      );
+    }
+  }
 
   async acquire(
     lane: GatewayPostgresLane,
@@ -56,6 +104,16 @@ export class GatewayPostgresAdmission {
     if (this.#canGrant(lane) && this.#waiting.commit.length === 0) {
       return this.#grant(lane);
     }
+    const waiting = this.#waiting[lane].length;
+    const limit = this.#maxWaitingPerLane();
+    if (waiting >= limit) {
+      this.#overloaded += 1;
+      throw new GatewayPostgresAdmissionOverloadedError({
+        lane,
+        waiting,
+        limit,
+      });
+    }
 
     return await new Promise<() => void>((resolve, reject) => {
       const waiter: Waiter = {
@@ -93,6 +151,7 @@ export class GatewayPostgresAdmission {
         commit: this.#waiting.commit.length,
       },
       cancelled: this.#cancelled,
+      overloaded: this.#overloaded,
       commitPriorityGrants: this.#commitPriorityGrants,
     };
   }
@@ -112,6 +171,12 @@ export class GatewayPostgresAdmission {
     return this.limits.maxTotalActive ?? GATEWAY_POSTGRES_MAX_TOTAL_ACTIVE;
   }
 
+  #maxWaitingPerLane(): number {
+    return (
+      this.limits.maxWaitingPerLane ?? GATEWAY_POSTGRES_MAX_WAITING_PER_LANE
+    );
+  }
+
   #maxOrdinaryActive(): number {
     return (
       this.limits.maxOrdinaryActive ?? GATEWAY_POSTGRES_MAX_ORDINARY_ACTIVE