deepline 0.3.127 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts

@@ -0,0 +1,215 @@
+/**
+ * Hands source pages to one running map worker pool so the pool never drains
+ * between pages. Workers that outrun the appended rows wait here; the page
+ * pump appends the next page as soon as the previous one is fully dispatched,
+ * so at most one undispatched page and one prepared page are resident.
+ */
+export type RuntimeMapRowFeedSnapshot = {
+  appendedRows: number;
+  dispatchedRows: number;
+  pagesAppended: number;
+  closed: boolean;
+  failed: boolean;
+  workersWaiting: number;
+  pumpWaitingForDrain: boolean;
+};
+
+export const RUNTIME_MAP_ROW_FEED_MAX_WAIT_MS = 30 * 60 * 1000;
+
+export class RuntimeMapRowFeedStalledError extends Error {
+  readonly snapshot: RuntimeMapRowFeedSnapshot;
+
+  constructor(input: {
+    waiter: 'worker' | 'pump';
+    waitMs: number;
+    snapshot: RuntimeMapRowFeedSnapshot;
+  }) {
+    super(
+      `Runtime map row feed stalled: ${input.waiter} waited ${input.waitMs} ms ` +
+        `(${JSON.stringify(input.snapshot)}).`,
+    );
+    this.name = 'RuntimeMapRowFeedStalledError';
+    this.snapshot = input.snapshot;
+  }
+}
+
+type Waiter<T> = {
+  resolve: (value: T) => void;
+  reject: (error: unknown) => void;
+  timer: ReturnType<typeof setTimeout>;
+};
+
+export class RuntimeMapRowFeed {
+  #appended = 0;
+  #dispatched = 0;
+  #pages = 0;
+  #closed = false;
+  #failure: unknown = null;
+  #failed = false;
+  readonly #rowWaiters = new Map<Waiter<boolean>, number>();
+  #drainWaiter: Waiter<void> | null = null;
+  #onAppended: ((fromIndex: number, toIndex: number) => void) | null = null;
+  readonly #maxWaitMs: number;
+
+  constructor(input?: { maxWaitMs?: number }) {
+    this.#maxWaitMs =
+      input?.maxWaitMs !== undefined &&
+      Number.isFinite(input.maxWaitMs) &&
+      input.maxWaitMs > 0
+        ? input.maxWaitMs
+        : RUNTIME_MAP_ROW_FEED_MAX_WAIT_MS;
+  }
+
+  onAppended(listener: (fromIndex: number, toIndex: number) => void): void {
+    this.#onAppended = listener;
+  }
+
+  append(rowCount: number): void {
+    if (this.#failed) throw this.#failure;
+    if (this.#closed) {
+      throw new Error('Runtime map row feed is closed; cannot append rows.');
+    }
+    if (!Number.isInteger(rowCount) || rowCount < 0) {
+      throw new Error(`Runtime map row feed append count must be a non-negative integer, got ${String(rowCount)}.`);
+    }
+    const fromIndex = this.#appended;
+    this.#appended += rowCount;
+    this.#pages += 1;
+    this.#onAppended?.(fromIndex, this.#appended);
+    this.#wakeRowWaiters();
+  }
+
+  close(): void {
+    if (this.#closed || this.#failed) return;
+    this.#closed = true;
+    this.#wakeRowWaiters();
+    this.#settleDrainWaiter();
+  }
+
+  fail(error: unknown): void {
+    if (this.#failed) return;
+    this.#failed = true;
+    this.#failure = error;
+    for (const [waiter] of this.#rowWaiters) {
+      clearTimeout(waiter.timer);
+      waiter.reject(error);
+    }
+    this.#rowWaiters.clear();
+    if (this.#drainWaiter) {
+      clearTimeout(this.#drainWaiter.timer);
+      this.#drainWaiter.reject(error);
+      this.#drainWaiter = null;
+    }
+  }
+
+  get failed(): boolean {
+    return this.#failed;
+  }
+
+  get closed(): boolean {
+    return this.#closed;
+  }
+
+  markDispatched(count = 1): void {
+    this.#dispatched += count;
+    if (this.#dispatched > this.#appended) {
+      throw new Error(
+        `Runtime map row feed dispatched ${this.#dispatched} rows but only ${this.#appended} were appended.`,
+      );
+    }
+    this.#settleDrainWaiter();
+  }
+
+  /**
+   * Resolves true once row `index` has been appended, false once the feed is
+   * closed with no such row. Rejects with the feed's failure.
+   */
+  waitForRows(index: number): Promise<boolean> {
+    if (this.#failed) return Promise.reject(this.#failure);
+    if (index < this.#appended) return Promise.resolve(true);
+    if (this.#closed) return Promise.resolve(false);
+    return new Promise<boolean>((resolve, reject) => {
+      const startedAt = Date.now();
+      const waiter: Waiter<boolean> = {
+        resolve,
+        reject,
+        timer: setTimeout(() => {
+          this.#rowWaiters.delete(waiter);
+          reject(
+            new RuntimeMapRowFeedStalledError({
+              waiter: 'worker',
+              waitMs: Date.now() - startedAt,
+              snapshot: this.snapshot(),
+            }),
+          );
+        }, this.#maxWaitMs),
+      };
+      waiter.timer.unref?.();
+      this.#rowWaiters.set(waiter, index);
+    });
+  }
+
+  /** Resolves once every appended row has been dispatched to a worker. */
+  waitUntilDrained(): Promise<void> {
+    if (this.#failed) return Promise.reject(this.#failure);
+    if (this.#dispatched >= this.#appended) return Promise.resolve();
+    if (this.#drainWaiter) {
+      return Promise.reject(
+        new Error('Runtime map row feed already has a pump waiting to drain.'),
+      );
+    }
+    return new Promise<void>((resolve, reject) => {
+      const startedAt = Date.now();
+      this.#drainWaiter = {
+        resolve,
+        reject,
+        timer: setTimeout(() => {
+          this.#drainWaiter = null;
+          reject(
+            new RuntimeMapRowFeedStalledError({
+              waiter: 'pump',
+              waitMs: Date.now() - startedAt,
+              snapshot: this.snapshot(),
+            }),
+          );
+        }, this.#maxWaitMs),
+      };
+      this.#drainWaiter.timer.unref?.();
+    });
+  }
+
+  snapshot(): RuntimeMapRowFeedSnapshot {
+    return {
+      appendedRows: this.#appended,
+      dispatchedRows: this.#dispatched,
+      pagesAppended: this.#pages,
+      closed: this.#closed,
+      failed: this.#failed,
+      workersWaiting: this.#rowWaiters.size,
+      pumpWaitingForDrain: this.#drainWaiter !== null,
+    };
+  }
+
+  #wakeRowWaiters(): void {
+    for (const [waiter, index] of this.#rowWaiters) {
+      if (index < this.#appended) {
+        this.#rowWaiters.delete(waiter);
+        clearTimeout(waiter.timer);
+        waiter.resolve(true);
+      } else if (this.#closed) {
+        this.#rowWaiters.delete(waiter);
+        clearTimeout(waiter.timer);
+        waiter.resolve(false);
+      }
+    }
+  }
+
+  #settleDrainWaiter(): void {
+    if (!this.#drainWaiter) return;
+    if (this.#dispatched < this.#appended && !this.#closed) return;
+    const waiter = this.#drainWaiter;
+    this.#drainWaiter = null;
+    clearTimeout(waiter.timer);
+    waiter.resolve();
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts

@@ -24,6 +24,7 @@ export const RUNTIME_OPERATION_CAPABILITIES = {
   mark_runtime_step_receipts_queued: 'receipt.write',
   complete_runtime_step_receipt: 'receipt.write',
   complete_runtime_step_receipts: 'receipt.write',
+  complete_runtime_step_receipts_by_reference: 'receipt.write',
   release_runtime_step_receipt: 'receipt.write',
   fail_runtime_step_receipt: 'receipt.write',
   fail_runtime_step_receipts: 'receipt.write',
@@ -54,6 +55,16 @@ export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
   deployed.operation({
     id: 'runtime-api-actions',
     actions: RUNTIME_OPERATION_CAPABILITIES,
+    olderReceiver: {
+      // The runner retains the complete result before attempting this
+      // compact completion. An older app receiver reports the action as
+      // unsupported, and the adapter retries the established full-payload
+      // completion without re-executing the provider call.
+      complete_runtime_step_receipts_by_reference: {
+        kind: 'equivalent-fallback',
+        action: 'complete_runtime_step_receipts',
+      },
+    },
   });
 
 export function runtimeOperationCapability(

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts

@@ -2,6 +2,7 @@ import type {
   AcquireRuntimeReceiptExecutionLockInput,
   ClaimRuntimeStepReceiptInput,
   CompleteRuntimeStepReceiptInput,
+  CompleteRuntimeStepReceiptsByReferenceInput,
   ContextOptions,
   FailRuntimeStepReceiptInput,
   ReleaseRuntimeReceiptExecutionLockInput,
@@ -18,6 +19,7 @@ import {
   AppRuntimeApiResponseError,
   AppRuntimeApiTransportError,
   claimRuntimeStepReceiptsViaAppRuntime,
+  completeRuntimeStepReceiptsByReferenceViaAppRuntime,
   completeRuntimeStepReceiptsViaAppRuntime,
   failRuntimeStepReceiptsViaAppRuntime,
   getRuntimeStepReceiptsViaAppRuntime,
@@ -31,6 +33,7 @@ import {
 import {
   RuntimeReceiptWriter,
   RuntimeReceiptWriterResultCountError,
+  writeRuntimeReceiptWithBackpressure,
 } from './runtime-receipt-writer';
 
 type ReceiptOutput =
@@ -42,7 +45,7 @@ type ReceiptOutput =
 type ReceiptCommand =
   | {
       kind: 'get';
-      input: { key: string; runId: string };
+      input: { key: string; runId: string; hydrateOutput?: boolean };
     }
   | {
       kind: 'claim';
@@ -53,6 +56,11 @@ type ReceiptCommand =
       outcome: 'complete';
       input: CompleteRuntimeStepReceiptInput;
     }
+  | {
+      kind: 'settle';
+      outcome: 'complete_reference';
+      input: CompleteRuntimeStepReceiptsByReferenceInput['receipts'][number];
+    }
   | {
       kind: 'settle';
       outcome: 'fail';
@@ -99,11 +107,13 @@ type ReceiptHandlerName =
   | 'claimRuntimeStepReceipts'
   | 'completeRuntimeStepReceipt'
   | 'completeRuntimeStepReceipts'
+  | 'completeRuntimeStepReceiptsByReference'
   | 'releaseRuntimeStepReceipt'
   | 'failRuntimeStepReceipt'
   | 'failRuntimeStepReceipts'
   | 'heartbeatRuntimeStepReceipts'
-  | 'skipRuntimeStepReceipt';
+  | 'skipRuntimeStepReceipt'
+  | 'lastRuntimeStepReceiptCompletionRequestId';
 
 export type RuntimeReceiptStoreHandlers = Required<
   Pick<ContextOptions, ReceiptHandlerName>
@@ -119,15 +129,17 @@ export const RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS = 5 * 60_000;
  * The runner's single Work Receipt Adapter.
  *
  * Receipt-domain batching and gateway transport stay behind this seam. Every
- * command shares one writer, so heterogeneous operations cannot produce
- * concurrent gateway requests. The low-level runtime client is explicitly
- * one-attempt; this Adapter is the sole retry owner.
+ * command shares one writer: operations for one receipt key stay ordered,
+ * while independent keys can use its bounded transport pool. The low-level
+ * runtime client is explicitly one-attempt; this Adapter is the sole retry
+ * owner.
  */
 export class RuntimeReceiptStoreAdapter {
   readonly handlers: RuntimeReceiptStoreHandlers;
   readonly claimsEstablishExecutionFence = true;
   readonly #writer: RuntimeReceiptWriter<ReceiptCommand, ReceiptOutput>;
   #serial = 0;
+  #lastCompletionRequestId: string | null = null;
 
   constructor(options: {
     context: WorkerRuntimeApiContext;
@@ -137,21 +149,76 @@ export class RuntimeReceiptStoreAdapter {
     maxBatchBytes?: number;
     targetBatchBytes?: number;
     maxFlushMs?: number;
+    maxBufferedItems?: number;
     maxBufferedBytes?: number;
+    maxInFlightBatches?: number;
+    /** Completion flights may use this many concurrent transports. */
+    maxCompletionInFlightBatches?: number;
+    /** Control traffic retains this many flights during a completion burst. */
+    maxControlInFlightBatches?: number;
+    maxCompletionBufferedItems?: number;
+    maxCompletionBufferedBytes?: number;
+    maxControlBufferedItems?: number;
+    maxControlBufferedBytes?: number;
     onRetryTelemetry?: (line: string) => void;
   }) {
     const context: WorkerRuntimeApiContext = {
       ...options.context,
       retryPolicy: 'none',
+      onResponseMeta: (meta) => {
+        if (
+          meta.action === 'complete_runtime_step_receipt' ||
+          meta.action === 'complete_runtime_step_receipts' ||
+          meta.action === 'complete_runtime_step_receipts_by_reference'
+        ) {
+          this.#lastCompletionRequestId = meta.requestId;
+        }
+        options.context.onResponseMeta?.(meta);
+      },
     };
+    const usesDedicatedLanes =
+      options.maxCompletionInFlightBatches !== undefined ||
+      options.maxControlInFlightBatches !== undefined;
+    const completionFlights = Math.max(
+      1,
+      Math.floor(
+        options.maxCompletionInFlightBatches ?? options.maxInFlightBatches ?? 1,
+      ),
+    );
+    const controlFlights = Math.max(
+      1,
+      Math.floor(options.maxControlInFlightBatches ?? 1),
+    );
     this.#writer = new RuntimeReceiptWriter({
       batchKey: receiptCommandBatchKey,
+      ...(usesDedicatedLanes
+        ? {
+            batchLane: receiptCommandBatchLane,
+            maxInFlightBatches: completionFlights + controlFlights,
+            maxInFlightBatchesByLane: {
+              completion: completionFlights,
+              control: controlFlights,
+            },
+            maxBufferedByLane: {
+              completion: {
+                items: options.maxCompletionBufferedItems,
+                bytes: options.maxCompletionBufferedBytes,
+              },
+              control: {
+                items: options.maxControlBufferedItems,
+                bytes: options.maxControlBufferedBytes,
+              },
+            },
+          }
+        : { maxInFlightBatches: options.maxInFlightBatches }),
+      orderingKeys: receiptCommandOrderingKeys,
       maxBatchSize: options.maxBatchSize,
       maxBatchBytes:
         options.maxBatchBytes ?? RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES,
       targetBatchBytes:
         options.targetBatchBytes ?? RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES,
       maxFlushMs: options.maxFlushMs,
+      maxBufferedItems: options.maxBufferedItems,
       maxBufferedBytes: options.maxBufferedBytes,
       maxRetryElapsedMs: RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS,
       classifyRetryableError: classifyReceiptRetry,
@@ -171,12 +238,12 @@ export class RuntimeReceiptStoreAdapter {
         }),
     });
 
-    const write = <Output extends ReceiptOutput>(
+    const write = async <Output extends ReceiptOutput>(
       command: ReceiptCommand,
     ): Promise<Output> =>
-      this.#writer.write(command, {
+      (await writeRuntimeReceiptWithBackpressure(this.#writer, command, {
         signal: options.context.signal,
-      }) as Promise<Output>;
+      })) as Output;
     const writeMany = <Output extends ReceiptOutput>(
       commands: readonly ReceiptCommand[],
     ): Promise<Output[]> =>
@@ -195,16 +262,24 @@ export class RuntimeReceiptStoreAdapter {
           input,
           serial: this.#nextSerial(),
         }),
-      getRuntimeStepReceipt: ({ key }) =>
+      getRuntimeStepReceipt: ({ key, hydrateOutput }) =>
         write({
           kind: 'get',
-          input: { key, runId: options.runId },
+          input: {
+            key,
+            runId: options.runId,
+            ...(hydrateOutput === false ? { hydrateOutput: false } : {}),
+          },
         }),
-      getRuntimeStepReceipts: ({ keys }) =>
+      getRuntimeStepReceipts: ({ keys, hydrateOutput }) =>
         writeMany(
           keys.map((key) => ({
             kind: 'get' as const,
-            input: { key, runId: options.runId },
+            input: {
+              key,
+              runId: options.runId,
+              ...(hydrateOutput === false ? { hydrateOutput: false } : {}),
+            },
           })),
         ),
       claimRuntimeStepReceipt: (input) => write({ kind: 'claim', input }),
@@ -220,6 +295,9 @@ export class RuntimeReceiptStoreAdapter {
               reclaimRunning: input.reclaimRunning,
               forceRefresh: input.forceRefresh,
               forceFailedRefresh: input.forceFailedRefresh,
+              ...(input.hydrateOutput === false
+                ? { hydrateOutput: false }
+                : {}),
               ...(input.leaseIds ? { leaseId: input.leaseIds[index] } : {}),
             },
           })),
@@ -234,6 +312,14 @@ export class RuntimeReceiptStoreAdapter {
             input,
           })),
         ),
+      completeRuntimeStepReceiptsByReference: ({ receipts }) =>
+        writeMany(
+          receipts.map((input) => ({
+            kind: 'settle' as const,
+            outcome: 'complete_reference' as const,
+            input,
+          })),
+        ),
       releaseRuntimeStepReceipt: (input) =>
         write({
           kind: 'settle',
@@ -270,6 +356,8 @@ export class RuntimeReceiptStoreAdapter {
           input,
           serial: this.#nextSerial(),
         }),
+      lastRuntimeStepReceiptCompletionRequestId: () =>
+        this.#lastCompletionRequestId,
     };
   }
 
@@ -363,7 +451,7 @@ function emitReceiptWriterRetryTelemetry(input: {
     elapsedMs: event.elapsedMs,
     retryAfterMs: event.retryAfterMs,
     queued: event.queued,
-    blocked: event.blocked,
+    bufferedItems: event.bufferedItems,
     bufferedBytes: event.bufferedBytes,
     ...(event.error === undefined
       ? { failureKind: null }
@@ -398,7 +486,9 @@ function receiptCommandBatchKey(command: ReceiptCommand): string {
         command.input.leaseId !== undefined,
       ]);
     case 'settle':
-      return command.outcome === 'complete' || command.outcome === 'fail'
+      return command.outcome === 'complete' ||
+        command.outcome === 'complete_reference' ||
+        command.outcome === 'fail'
         ? JSON.stringify([command.kind, command.outcome, command.input.runId])
         : `${command.kind}:${command.outcome}:${command.serial}`;
     case 'heartbeat':
@@ -413,6 +503,37 @@ function receiptCommandBatchKey(command: ReceiptCommand): string {
   }
 }
 
+function receiptCommandBatchLane(
+  command: ReceiptCommand,
+): 'completion' | 'control' {
+  return command.kind === 'settle' &&
+    (command.outcome === 'complete' ||
+      command.outcome === 'complete_reference' ||
+      command.outcome === 'fail')
+    ? 'completion'
+    : 'control';
+}
+
+/**
+ * The gateway conditionally owns every receipt transition, but same-receipt
+ * commands still travel in program order. Different receipt keys are
+ * independent and can safely occupy the shared bounded transport pool.
+ */
+function receiptCommandOrderingKeys(
+  command: ReceiptCommand,
+): readonly string[] {
+  switch (command.kind) {
+    case 'get':
+    case 'claim':
+    case 'heartbeat':
+    case 'acquire_execution_lock':
+    case 'release_execution_lock':
+      return [command.input.key];
+    case 'settle':
+      return [command.input.key];
+  }
+}
+
 function classifyReceiptRetry(error: unknown): { retryAfterMs: number } | null {
   if (isAppRuntimeApiCapacityError(error)) {
     return { retryAfterMs: error.retryAfterMs };
@@ -443,6 +564,12 @@ async function sendReceiptCommands(input: {
         playName: input.playName,
         runId: first.input.runId,
         keys: commands.map((command) => command.input.key),
+        // A mixed writer batch must preserve normal full reads. The manifest
+        // probe safely falls back to its byte-admitted path when it receives a
+        // body, so never omit an output a concurrent caller requested.
+        ...(commands.every((command) => command.input.hydrateOutput === false)
+          ? { hydrateOutput: false }
+          : {}),
       });
     }
     case 'claim': {
@@ -463,6 +590,12 @@ async function sendReceiptCommands(input: {
         reclaimRunning: first.input.reclaimRunning,
         forceRefresh: first.input.forceRefresh,
         forceFailedRefresh: first.input.forceFailedRefresh,
+        // A mixed writer batch must retain a full body for callers that
+        // requested one. Manifest callers detect such a body and re-enter the
+        // byte-admitted hydration path before using it.
+        ...(commands.every((command) => command.input.hydrateOutput === false)
+          ? { hydrateOutput: false }
+          : {}),
       });
     }
     case 'settle':
@@ -478,6 +611,20 @@ async function sendReceiptCommands(input: {
             receipts: commands.map((command) => command.input),
           });
         }
+        case 'complete_reference': {
+          const commands = input.commands as readonly Extract<
+            ReceiptCommand,
+            { kind: 'settle'; outcome: 'complete_reference' }
+          >[];
+          return await completeRuntimeStepReceiptsByReferenceViaAppRuntime(
+            input.context,
+            {
+              playName: input.playName,
+              runId: first.input.runId,
+              receipts: commands.map((command) => command.input),
+            },
+          );
+        }
         case 'fail': {
           const commands = input.commands as readonly Extract<
             ReceiptCommand,