deepline 0.2.55 → 0.2.56

package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts

@@ -11,6 +11,20 @@ import {
 } from './runtime-pg-driver';
 
 function wrapPgClient(client: PoolClient): RuntimePoolClient {
+  let released = false;
+  const releaseClient = (destroy = false) => {
+    if (released) return;
+    released = true;
+    client.release(destroy);
+  };
+  const destroyTransport = () => {
+    // `release(true)` is node-postgres' supported pool-client destruction
+    // path. The pool removes the client and Client.end() destroys the socket
+    // when a query is active, while keeping pool accounting correct. Runtime
+    // callers still run their ordinary `finally { release() }` path after a
+    // deadline destroys the transport, so this wrapper owns idempotence.
+    releaseClient(true);
+  };
   return {
     query: <R extends Record<string, unknown> = Record<string, unknown>>(
       text: string,
@@ -24,9 +38,10 @@ function wrapPgClient(client: PoolClient): RuntimePoolClient {
       )(text, params).then((result) => ({
         rows: result.rows,
       })),
-    release: () => {
-      client.release();
+    release: (destroy = false) => {
+      releaseClient(destroy);
     },
+    destroy: destroyTransport,
   };
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts

@@ -13,7 +13,10 @@ export interface RuntimePoolClient {
     text: string,
     params?: unknown[],
   ): Promise<{ rows: R[] }>;
-  release(): void;
+  /** Destroy the physical connection when an active query exceeded its bound. */
+  release(destroy?: boolean): void;
+  /** Interrupt an active query by destroying the physical transport. */
+  destroy?(error?: Error): void;
 }
 
 export interface RuntimePool {

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts

@@ -9,7 +9,10 @@ import type {
   RuntimeStepReceipt,
   SkipRuntimeStepReceiptInput,
 } from './ctx-types';
-import { RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES } from './output-size-limits';
+import {
+  RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES,
+  RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES,
+} from './output-size-limits';
 import {
   acquireRuntimeReceiptExecutionLockViaAppRuntime,
   AppRuntimeApiResponseError,
@@ -106,6 +109,12 @@ export type RuntimeReceiptStoreHandlers = Required<
   Pick<ContextOptions, ReceiptHandlerName>
 >;
 
+// Four ambiguous 30-second receipt requests previously consumed the writer's
+// entire two-minute budget. Receipt operations are idempotent and buffered, so
+// keep retry ownership here and allow independent transports enough attempts
+// to recover from a short gateway/network impairment.
+export const RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS = 5 * 60_000;
+
 /**
  * The runner's single Work Receipt Adapter.
  *
@@ -116,6 +125,7 @@ export type RuntimeReceiptStoreHandlers = Required<
  */
 export class RuntimeReceiptStoreAdapter {
   readonly handlers: RuntimeReceiptStoreHandlers;
+  readonly claimsEstablishExecutionFence = true;
   readonly #writer: RuntimeReceiptWriter<ReceiptCommand, ReceiptOutput>;
   #serial = 0;
 
@@ -125,6 +135,7 @@ export class RuntimeReceiptStoreAdapter {
     runId: string;
     maxBatchSize?: number;
     maxBatchBytes?: number;
+    targetBatchBytes?: number;
     maxFlushMs?: number;
     maxBufferedBytes?: number;
     onRetryTelemetry?: (line: string) => void;
@@ -138,8 +149,11 @@ export class RuntimeReceiptStoreAdapter {
       maxBatchSize: options.maxBatchSize,
       maxBatchBytes:
         options.maxBatchBytes ?? RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES,
+      targetBatchBytes:
+        options.targetBatchBytes ?? RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES,
       maxFlushMs: options.maxFlushMs,
       maxBufferedBytes: options.maxBufferedBytes,
+      maxRetryElapsedMs: RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS,
       classifyRetryableError: classifyReceiptRetry,
       onRetryEvent: (event) =>
         emitReceiptWriterRetryTelemetry({
@@ -309,6 +323,7 @@ function receiptRetryErrorTelemetry(error: unknown): Record<string, unknown> {
       errorName: error.name,
       transportAttempts: error.attempts,
       transportAction: error.action,
+      transportAttemptId: error.transportAttemptId,
     };
   }
   if (error instanceof AppRuntimeApiResponseError) {
@@ -343,6 +358,7 @@ function emitReceiptWriterRetryTelemetry(input: {
     action: receiptCommandTelemetryAction(event.firstInput),
     batchId: event.batchId,
     batchSize: event.batchSize,
+    batchBytes: event.batchBytes,
     attempt: event.attempt,
     elapsedMs: event.elapsedMs,
     retryAfterMs: event.retryAfterMs,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts

@@ -27,6 +27,7 @@ export type RuntimeReceiptWriterRetryEvent<Input> = {
   attempt: number;
   batchId: number;
   batchSize: number;
+  batchBytes: number;
   firstInput: Input;
   elapsedMs: number;
   retryAfterMs: number;
@@ -103,6 +104,7 @@ export class RuntimeReceiptWriter<Input, Output> {
   readonly #estimateBytes: (input: Input) => number;
   readonly #maxBatchSize: number;
   readonly #maxBatchBytes: number;
+  readonly #targetBatchBytes: number;
   readonly #maxBufferedBytes: number;
   readonly #maxFlushMs: number;
   readonly #maxRetryElapsedMs: number;
@@ -135,6 +137,7 @@ export class RuntimeReceiptWriter<Input, Output> {
     estimateBytes?: (input: Input) => number;
     maxBatchSize?: number;
     maxBatchBytes?: number;
+    targetBatchBytes?: number;
     maxBufferedBytes?: number;
     maxFlushMs?: number;
     maxRetryElapsedMs?: number;
@@ -154,6 +157,10 @@ export class RuntimeReceiptWriter<Input, Output> {
       normalizePositiveInteger(options.maxBatchBytes, this.#maxBufferedBytes),
       this.#maxBufferedBytes,
     );
+    this.#targetBatchBytes = Math.min(
+      normalizePositiveInteger(options.targetBatchBytes, this.#maxBatchBytes),
+      this.#maxBatchBytes,
+    );
     this.#maxFlushMs = Math.max(0, Math.floor(options.maxFlushMs ?? 5));
     this.#maxRetryElapsedMs = normalizePositiveInteger(
       options.maxRetryElapsedMs,
@@ -251,7 +258,7 @@ export class RuntimeReceiptWriter<Input, Output> {
     this.#bufferedBytes += pending.bytes;
     if (
       this.#queued.length >= this.#maxBatchSize ||
-      this.#queuedBytesAtHead() >= this.#maxBatchBytes
+      this.#queuedBytesAtHead() >= this.#targetBatchBytes
     ) {
       this.#clearFlushTimer();
       this.#startPump();
@@ -334,7 +341,10 @@ export class RuntimeReceiptWriter<Input, Output> {
     while (batch.length < this.#maxBatchSize) {
       const next = this.#queued[0];
       if (!next || !Object.is(this.#batchKey(next.input), key)) break;
-      if (batch.length > 0 && batchBytes + next.bytes > this.#maxBatchBytes) {
+      if (
+        batch.length > 0 &&
+        batchBytes + next.bytes > this.#targetBatchBytes
+      ) {
         break;
       }
       this.#queued.shift();
@@ -427,6 +437,10 @@ export class RuntimeReceiptWriter<Input, Output> {
         attempt: input.attempt,
         batchId: input.batchId,
         batchSize: input.batch.length,
+        batchBytes: input.batch.reduce(
+          (total, pending) => total + pending.bytes,
+          0,
+        ),
         firstInput,
         elapsedMs: Date.now() - input.startedAt,
         retryAfterMs: input.retryAfterMs,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts

@@ -70,6 +70,14 @@ export type RuntimeSheetRowWriterSummary = {
   terminalRows: number;
 };
 
+export type RuntimeSheetRowWriterDiagnostics = {
+  activeKind: 'terminal' | 'checkpoint' | null;
+  activeRows: number;
+  activeStartedAt: number | null;
+  blockedTerminalRows: number;
+  queuedTerminalRows: number;
+};
+
 export type RuntimeSheetRowSettlement = {
   /**
    * Resolves once the bounded live buffer owns this row. Awaiting this promise
@@ -159,6 +167,7 @@ export class RuntimeSheetRowWriter<Row, Update> {
   readonly #controller = new AbortController();
 
   #active: PendingWrite<Row, Update>[] | null = null;
+  #activeStartedAt: number | null = null;
   #bufferedBytes = 0;
   #liveBytes = 0;
   #timer: ReturnType<typeof setTimeout> | null = null;
@@ -264,6 +273,20 @@ export class RuntimeSheetRowWriter<Row, Update> {
     return this.#finishPromise;
   }
 
+  diagnostics(): RuntimeSheetRowWriterDiagnostics {
+    return {
+      activeKind: this.#active?.[0]?.kind ?? null,
+      activeRows: this.#active?.length ?? 0,
+      activeStartedAt: this.#activeStartedAt,
+      blockedTerminalRows: this.#blocked.filter(
+        (entry) => entry.kind === 'terminal',
+      ).length,
+      queuedTerminalRows: [...this.#live.values()].filter(
+        (entry) => entry.kind === 'terminal',
+      ).length,
+    };
+  }
+
   #enqueueCheckpointUpdate(update: Update): Promise<void> {
     const key = normalizedKey(this.#checkpointKey(update));
     if (this.#terminalAcceptedKeys.has(key)) return Promise.resolve();
@@ -403,6 +426,7 @@ export class RuntimeSheetRowWriter<Row, Update> {
       if (batch.length === 0) return;
       const batchKind = batch[0]!.kind;
       this.#active = batch;
+      this.#activeStartedAt = Date.now();
       try {
         const result = await this.#writeIdenticalBatchWithRetry(batch);
         const resultCommitted = result ? result.committed : undefined;
@@ -429,6 +453,7 @@ export class RuntimeSheetRowWriter<Row, Update> {
         );
         this.#bufferedBytes = Math.max(0, this.#bufferedBytes - releasedBytes);
         this.#active = null;
+        this.#activeStartedAt = null;
         this.#admitBlocked();
         this.#notifyStateChanged();
       }

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -58,6 +58,9 @@ export type PlaySchedulerSubmitInput = {
   force?: boolean;
   /** Explicit cache bypass for completed ctx.tools.execute receipts. */
   forceToolRefresh?: boolean | null;
+  /** Validated per-run ceiling for provider-tool executions and ctx.fetch. */
+  maxConcurrentExternalCalls?: number | null;
+  maxConcurrentRows?: number | null;
   inputFile?: {
     name?: string;
     path?: string;

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -56,6 +56,8 @@ export type PlayExecutionSuspension =
       exitCodePath: string;
       /** Exact customer-code completion fence inside the Daytona sandbox. */
       runtimeCompletedPath?: string;
+      /** Bounded watchdog-owned cause marker for crash-only diagnostics. */
+      terminationDiagnosticPath?: string;
       startedAtMs: number;
       /** Scheduler-owned liveness deadline. The sandbox can renew it only by
        * persisting `heartbeat_at` through the receipt gateway. */

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -1,4 +1,5 @@
 import { getRuntimeEnv } from '../runtime-env';
+import type { PlayRunnerEvent } from './protocol';
 
 export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault';
 
@@ -13,7 +14,23 @@ export type RuntimeTestFaultName =
    * deterministic real-Pg-pool pressure window without changing an action's
    * SQL or outcome.
    */
-  | 'receipt_gateway_hold_ms';
+  | 'receipt_gateway_hold_ms'
+  /**
+   * Synthetic-only bulk-claim fault. The runner forwards this token on the
+   * first qualifying physical claim request only. The gateway holds the real
+   * checked-out scheduler client for the supplied milliseconds, allowing the
+   * claim-operation deadline and connection destruction path to be exercised
+   * end to end without making the fault Machine-local.
+   */
+  | 'receipt_claim_query_hold_once_ms'
+  /**
+   * Synthetic-only response-boundary fault. The runner fully receives a
+   * successful bulk receipt-claim response, then reports the configured first
+   * N physical requests as request timeouts. This proves ambiguous-delivery
+   * replay without depending on which gateway Machine accepted the request.
+   */
+  | 'receipt_claim_response_timeout'
+  | 'runtime_sheet_page_tail_hold_ms';
 
 export type RuntimeTestFaultRegistry = {
   consume(name: RuntimeTestFaultName): boolean;
@@ -75,6 +92,9 @@ const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
   'worker_receipt_complete_write_fail',
   'invocation_response_delivery_abort',
   'receipt_gateway_hold_ms',
+  'receipt_claim_query_hold_once_ms',
+  'receipt_claim_response_timeout',
+  'runtime_sheet_page_tail_hold_ms',
 ]);
 
 const RUNTIME_TEST_POLICY_MS_FIELDS = new Set([
@@ -149,6 +169,65 @@ function parseRuntimeTestFaultHeader(
   return faults;
 }
 
+/** Parse a known fault count without authorizing a request. Unknown/skewed headers return 0. */
+export function recognizedRuntimeTestFaultCount(
+  rawHeader: string | null | undefined,
+  name: RuntimeTestFaultName,
+): number {
+  const header = rawHeader?.trim();
+  if (!header) return 0;
+  const parsed = parseRuntimeTestFaultHeader(header);
+  if ('error' in parsed) return 0;
+  return parsed.get(name) ?? 0;
+}
+
+export function runtimeSheetPageTailRowsContainTarget(
+  rows: readonly unknown[],
+): boolean {
+  return rows.some((row) => {
+    if (!row || typeof row !== 'object' || Array.isArray(row)) return false;
+    const record = row as Record<string, unknown>;
+    return record.inputIndex === 999 || record.input_index === 999;
+  });
+}
+
+export function runtimeSheetPageTailWriteMarkerEvent(
+  runtimeTestFaultHeader: string | null | undefined,
+  rows: readonly Record<string, unknown>[],
+  tableNamespace: string,
+  dbSessionStrategy: string | null | undefined,
+): PlayRunnerEvent | null {
+  if (
+    dbSessionStrategy !== 'gateway_only' ||
+    recognizedRuntimeTestFaultCount(
+      runtimeTestFaultHeader,
+      'runtime_sheet_page_tail_hold_ms',
+    ) <= 0 ||
+    !runtimeSheetPageTailRowsContainTarget(rows)
+  ) {
+    return null;
+  }
+  return {
+    type: 'log',
+    at: new Date().toISOString(),
+    source: 'play',
+    line: `[runtime.sheet-page-tail-write] phase=start table=${tableNamespace} rows=${rows.length}`,
+  };
+}
+
+export function runtimeSheetPageTailHoldCompletedEvent(
+  holdMs: number | null | undefined,
+  tableNamespace: string,
+): PlayRunnerEvent | null {
+  if (!Number.isInteger(holdMs) || (holdMs ?? 0) <= 0) return null;
+  return {
+    type: 'log',
+    at: new Date().toISOString(),
+    source: 'play',
+    line: `[runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
+  };
+}
+
 function readPositiveIntegerField(input: {
   value: unknown;
   path: string;

package/dist/bundling-sources/shared_libs/security/safe-fetch.ts

@@ -19,6 +19,10 @@ function removeHeader(headers: Headers, name: string) {
   if (headers.has(name)) headers.delete(name);
 }
 
+function cancelResponseBodyBestEffort(response: Response): void {
+  void response.body?.cancel().catch(() => undefined);
+}
+
 function requestInitForRedirect(
   init: RequestInit,
   from: URL,
@@ -83,6 +87,7 @@ export async function safePublicFetch(
       return response;
     }
     if (redirectMode === 'error') {
+      cancelResponseBodyBestEffort(response);
       throw new Error(
         `Redirect blocked while fetching ${currentUrl.toString()}.`,
       );
@@ -96,11 +101,15 @@ export async function safePublicFetch(
       return response;
     }
     if (redirectCount === maxRedirects) {
+      cancelResponseBodyBestEffort(response);
       throw new Error(
         `Too many redirects while fetching ${currentUrl.toString()}.`,
       );
     }
 
+    // A redirect response can stream an unbounded body. The next request must
+    // not leave that body/socket flowing in the background.
+    cancelResponseBodyBestEffort(response);
     const nextUrl = resolveRedirectUrl(location, currentUrl);
     currentInit = requestInitForRedirect(
       currentInit,

package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts

@@ -14,6 +14,8 @@ import {
 type NodeSafeFetchOptions = {
   maxRedirects?: number;
   maxResponseBytes?: number;
+  /** Return after validated headers so the caller can enforce a body deadline. */
+  streamResponseBody?: boolean;
   truncateResponseBody?: boolean;
   sensitiveHeaders?: Iterable<string>;
   validateUrl?: (url: URL) => void;
@@ -219,62 +221,83 @@ function createRequest(
           );
           return;
         }
-
-        const chunks: Buffer[] = [];
-        let receivedBytes = 0;
-        let settled = false;
-        const resolveResponse = () => {
-          if (settled) return;
-          settled = true;
+        if (noBodyResponse) {
+          response.resume();
           resolve(
-            new Response(noBodyResponse ? null : Buffer.concat(chunks), {
+            new Response(null, {
               status,
               statusText: response.statusMessage,
               headers: response.headers as HeadersInit,
             }),
           );
-        };
-        response.on('data', (chunk) => {
-          if (settled) return;
-          const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-          const previousReceivedBytes = receivedBytes;
-          receivedBytes += buffer.byteLength;
-          if (
-            maxResponseBytes !== undefined &&
-            receivedBytes > maxResponseBytes &&
-            !truncateResponseBody
-          ) {
-            response.destroy(
-              new Error(
-                `Response body exceeds ${maxResponseBytes} byte limit.`,
-              ),
-            );
-            return;
-          }
-          if (!noBodyResponse) {
-            if (maxResponseBytes !== undefined && truncateResponseBody) {
-              const remainingBytes = Math.max(
-                0,
-                maxResponseBytes - previousReceivedBytes,
-              );
-              if (remainingBytes > 0) {
-                chunks.push(buffer.subarray(0, remainingBytes));
+          return;
+        }
+
+        let receivedBytes = 0;
+        let bodySettled = false;
+        const body = new ReadableStream<Uint8Array>({
+          start(controller) {
+            response.on('data', (chunk) => {
+              if (bodySettled) return;
+              const buffer = Buffer.isBuffer(chunk)
+                ? chunk
+                : Buffer.from(chunk);
+              const previousReceivedBytes = receivedBytes;
+              receivedBytes += buffer.byteLength;
+              if (
+                maxResponseBytes !== undefined &&
+                receivedBytes > maxResponseBytes &&
+                !truncateResponseBody
+              ) {
+                bodySettled = true;
+                const error = new Error(
+                  `Response body exceeds ${maxResponseBytes} byte limit.`,
+                );
+                controller.error(error);
+                response.destroy(error);
+                return;
               }
-              if (receivedBytes >= maxResponseBytes) {
-                resolveResponse();
-                response.destroy();
+              if (maxResponseBytes !== undefined && truncateResponseBody) {
+                const remainingBytes = Math.max(
+                  0,
+                  maxResponseBytes - previousReceivedBytes,
+                );
+                if (remainingBytes > 0) {
+                  controller.enqueue(buffer.subarray(0, remainingBytes));
+                }
+                if (receivedBytes >= maxResponseBytes) {
+                  bodySettled = true;
+                  controller.close();
+                  response.destroy();
+                }
+                return;
               }
-            } else {
-              chunks.push(buffer);
-            }
-          }
+              controller.enqueue(buffer);
+            });
+            response.on('error', (error) => {
+              if (bodySettled) return;
+              bodySettled = true;
+              controller.error(error);
+            });
+            response.on('end', () => {
+              if (bodySettled) return;
+              bodySettled = true;
+              controller.close();
+            });
+          },
+          cancel(reason) {
+            if (bodySettled) return;
+            bodySettled = true;
+            response.destroy(reason instanceof Error ? reason : undefined);
+          },
         });
-        response.on('error', (error) => {
-          if (settled) return;
-          settled = true;
-          reject(error);
-        });
-        response.on('end', resolveResponse);
+        resolve(
+          new Response(body, {
+            status,
+            statusText: response.statusMessage,
+            headers: response.headers as HeadersInit,
+          }),
+        );
       },
     );
 
@@ -359,9 +382,21 @@ export async function safeOutboundFetch(
     });
 
     if (!isRedirectStatus(response.status)) {
-      return response;
+      if (options.streamResponseBody) return response;
+      const body = response.body ? await response.arrayBuffer() : null;
+      const buffered = new Response(body, {
+        status: response.status,
+        statusText: response.statusText,
+        headers: response.headers,
+      });
+      Object.defineProperty(buffered, 'url', {
+        configurable: true,
+        value: response.url,
+      });
+      return buffered;
     }
     if (redirectMode === 'error') {
+      void response.body?.cancel().catch(() => undefined);
       throw new Error(
         `Redirect blocked while fetching ${currentUrl.toString()}.`,
       );
@@ -375,11 +410,13 @@ export async function safeOutboundFetch(
       return response;
     }
     if (redirectCount === maxRedirects) {
+      void response.body?.cancel().catch(() => undefined);
       throw new Error(
         `Too many redirects while fetching ${currentUrl.toString()}.`,
       );
     }
 
+    void response.body?.cancel().catch(() => undefined);
     const nextUrl = resolveRedirectUrl(location, currentUrl);
     currentInit = initForRedirect(
       currentInit,