deepline 0.3.128 → 0.3.130

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts

@@ -2,6 +2,7 @@ import type {
   AcquireRuntimeReceiptExecutionLockInput,
   ClaimRuntimeStepReceiptInput,
   CompleteRuntimeStepReceiptInput,
+  CompleteRuntimeStepReceiptsByReferenceInput,
   ContextOptions,
   FailRuntimeStepReceiptInput,
   ReleaseRuntimeReceiptExecutionLockInput,
@@ -18,6 +19,7 @@ import {
   AppRuntimeApiResponseError,
   AppRuntimeApiTransportError,
   claimRuntimeStepReceiptsViaAppRuntime,
+  completeRuntimeStepReceiptsByReferenceViaAppRuntime,
   completeRuntimeStepReceiptsViaAppRuntime,
   failRuntimeStepReceiptsViaAppRuntime,
   getRuntimeStepReceiptsViaAppRuntime,
@@ -31,6 +33,7 @@ import {
 import {
   RuntimeReceiptWriter,
   RuntimeReceiptWriterResultCountError,
+  writeRuntimeReceiptWithBackpressure,
 } from './runtime-receipt-writer';
 
 type ReceiptOutput =
@@ -42,7 +45,7 @@ type ReceiptOutput =
 type ReceiptCommand =
   | {
       kind: 'get';
-      input: { key: string; runId: string };
+      input: { key: string; runId: string; hydrateOutput?: boolean };
     }
   | {
       kind: 'claim';
@@ -53,6 +56,11 @@ type ReceiptCommand =
       outcome: 'complete';
       input: CompleteRuntimeStepReceiptInput;
     }
+  | {
+      kind: 'settle';
+      outcome: 'complete_reference';
+      input: CompleteRuntimeStepReceiptsByReferenceInput['receipts'][number];
+    }
   | {
       kind: 'settle';
       outcome: 'fail';
@@ -99,11 +107,13 @@ type ReceiptHandlerName =
   | 'claimRuntimeStepReceipts'
   | 'completeRuntimeStepReceipt'
   | 'completeRuntimeStepReceipts'
+  | 'completeRuntimeStepReceiptsByReference'
   | 'releaseRuntimeStepReceipt'
   | 'failRuntimeStepReceipt'
   | 'failRuntimeStepReceipts'
   | 'heartbeatRuntimeStepReceipts'
-  | 'skipRuntimeStepReceipt';
+  | 'skipRuntimeStepReceipt'
+  | 'lastRuntimeStepReceiptCompletionRequestId';
 
 export type RuntimeReceiptStoreHandlers = Required<
   Pick<ContextOptions, ReceiptHandlerName>
@@ -119,15 +129,17 @@ export const RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS = 5 * 60_000;
  * The runner's single Work Receipt Adapter.
  *
  * Receipt-domain batching and gateway transport stay behind this seam. Every
- * command shares one writer, so heterogeneous operations cannot produce
- * concurrent gateway requests. The low-level runtime client is explicitly
- * one-attempt; this Adapter is the sole retry owner.
+ * command shares one writer: operations for one receipt key stay ordered,
+ * while independent keys can use its bounded transport pool. The low-level
+ * runtime client is explicitly one-attempt; this Adapter is the sole retry
+ * owner.
  */
 export class RuntimeReceiptStoreAdapter {
   readonly handlers: RuntimeReceiptStoreHandlers;
   readonly claimsEstablishExecutionFence = true;
   readonly #writer: RuntimeReceiptWriter<ReceiptCommand, ReceiptOutput>;
   #serial = 0;
+  #lastCompletionRequestId: string | null = null;
 
   constructor(options: {
     context: WorkerRuntimeApiContext;
@@ -137,21 +149,76 @@ export class RuntimeReceiptStoreAdapter {
     maxBatchBytes?: number;
     targetBatchBytes?: number;
     maxFlushMs?: number;
+    maxBufferedItems?: number;
     maxBufferedBytes?: number;
+    maxInFlightBatches?: number;
+    /** Completion flights may use this many concurrent transports. */
+    maxCompletionInFlightBatches?: number;
+    /** Control traffic retains this many flights during a completion burst. */
+    maxControlInFlightBatches?: number;
+    maxCompletionBufferedItems?: number;
+    maxCompletionBufferedBytes?: number;
+    maxControlBufferedItems?: number;
+    maxControlBufferedBytes?: number;
     onRetryTelemetry?: (line: string) => void;
   }) {
     const context: WorkerRuntimeApiContext = {
       ...options.context,
       retryPolicy: 'none',
+      onResponseMeta: (meta) => {
+        if (
+          meta.action === 'complete_runtime_step_receipt' ||
+          meta.action === 'complete_runtime_step_receipts' ||
+          meta.action === 'complete_runtime_step_receipts_by_reference'
+        ) {
+          this.#lastCompletionRequestId = meta.requestId;
+        }
+        options.context.onResponseMeta?.(meta);
+      },
     };
+    const usesDedicatedLanes =
+      options.maxCompletionInFlightBatches !== undefined ||
+      options.maxControlInFlightBatches !== undefined;
+    const completionFlights = Math.max(
+      1,
+      Math.floor(
+        options.maxCompletionInFlightBatches ?? options.maxInFlightBatches ?? 1,
+      ),
+    );
+    const controlFlights = Math.max(
+      1,
+      Math.floor(options.maxControlInFlightBatches ?? 1),
+    );
     this.#writer = new RuntimeReceiptWriter({
       batchKey: receiptCommandBatchKey,
+      ...(usesDedicatedLanes
+        ? {
+            batchLane: receiptCommandBatchLane,
+            maxInFlightBatches: completionFlights + controlFlights,
+            maxInFlightBatchesByLane: {
+              completion: completionFlights,
+              control: controlFlights,
+            },
+            maxBufferedByLane: {
+              completion: {
+                items: options.maxCompletionBufferedItems,
+                bytes: options.maxCompletionBufferedBytes,
+              },
+              control: {
+                items: options.maxControlBufferedItems,
+                bytes: options.maxControlBufferedBytes,
+              },
+            },
+          }
+        : { maxInFlightBatches: options.maxInFlightBatches }),
+      orderingKeys: receiptCommandOrderingKeys,
       maxBatchSize: options.maxBatchSize,
       maxBatchBytes:
         options.maxBatchBytes ?? RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES,
       targetBatchBytes:
         options.targetBatchBytes ?? RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES,
       maxFlushMs: options.maxFlushMs,
+      maxBufferedItems: options.maxBufferedItems,
       maxBufferedBytes: options.maxBufferedBytes,
       maxRetryElapsedMs: RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS,
       classifyRetryableError: classifyReceiptRetry,
@@ -171,12 +238,12 @@ export class RuntimeReceiptStoreAdapter {
         }),
     });
 
-    const write = <Output extends ReceiptOutput>(
+    const write = async <Output extends ReceiptOutput>(
       command: ReceiptCommand,
     ): Promise<Output> =>
-      this.#writer.write(command, {
+      (await writeRuntimeReceiptWithBackpressure(this.#writer, command, {
         signal: options.context.signal,
-      }) as Promise<Output>;
+      })) as Output;
     const writeMany = <Output extends ReceiptOutput>(
       commands: readonly ReceiptCommand[],
     ): Promise<Output[]> =>
@@ -195,16 +262,24 @@ export class RuntimeReceiptStoreAdapter {
           input,
           serial: this.#nextSerial(),
         }),
-      getRuntimeStepReceipt: ({ key }) =>
+      getRuntimeStepReceipt: ({ key, hydrateOutput }) =>
         write({
           kind: 'get',
-          input: { key, runId: options.runId },
+          input: {
+            key,
+            runId: options.runId,
+            ...(hydrateOutput === false ? { hydrateOutput: false } : {}),
+          },
         }),
-      getRuntimeStepReceipts: ({ keys }) =>
+      getRuntimeStepReceipts: ({ keys, hydrateOutput }) =>
         writeMany(
           keys.map((key) => ({
             kind: 'get' as const,
-            input: { key, runId: options.runId },
+            input: {
+              key,
+              runId: options.runId,
+              ...(hydrateOutput === false ? { hydrateOutput: false } : {}),
+            },
           })),
         ),
       claimRuntimeStepReceipt: (input) => write({ kind: 'claim', input }),
@@ -220,6 +295,9 @@ export class RuntimeReceiptStoreAdapter {
               reclaimRunning: input.reclaimRunning,
               forceRefresh: input.forceRefresh,
               forceFailedRefresh: input.forceFailedRefresh,
+              ...(input.hydrateOutput === false
+                ? { hydrateOutput: false }
+                : {}),
               ...(input.leaseIds ? { leaseId: input.leaseIds[index] } : {}),
             },
           })),
@@ -234,6 +312,14 @@ export class RuntimeReceiptStoreAdapter {
             input,
           })),
         ),
+      completeRuntimeStepReceiptsByReference: ({ receipts }) =>
+        writeMany(
+          receipts.map((input) => ({
+            kind: 'settle' as const,
+            outcome: 'complete_reference' as const,
+            input,
+          })),
+        ),
       releaseRuntimeStepReceipt: (input) =>
         write({
           kind: 'settle',
@@ -270,6 +356,8 @@ export class RuntimeReceiptStoreAdapter {
           input,
           serial: this.#nextSerial(),
         }),
+      lastRuntimeStepReceiptCompletionRequestId: () =>
+        this.#lastCompletionRequestId,
     };
   }
 
@@ -363,7 +451,7 @@ function emitReceiptWriterRetryTelemetry(input: {
     elapsedMs: event.elapsedMs,
     retryAfterMs: event.retryAfterMs,
     queued: event.queued,
-    blocked: event.blocked,
+    bufferedItems: event.bufferedItems,
     bufferedBytes: event.bufferedBytes,
     ...(event.error === undefined
       ? { failureKind: null }
@@ -398,7 +486,9 @@ function receiptCommandBatchKey(command: ReceiptCommand): string {
         command.input.leaseId !== undefined,
       ]);
     case 'settle':
-      return command.outcome === 'complete' || command.outcome === 'fail'
+      return command.outcome === 'complete' ||
+        command.outcome === 'complete_reference' ||
+        command.outcome === 'fail'
         ? JSON.stringify([command.kind, command.outcome, command.input.runId])
         : `${command.kind}:${command.outcome}:${command.serial}`;
     case 'heartbeat':
@@ -413,6 +503,37 @@ function receiptCommandBatchKey(command: ReceiptCommand): string {
   }
 }
 
+function receiptCommandBatchLane(
+  command: ReceiptCommand,
+): 'completion' | 'control' {
+  return command.kind === 'settle' &&
+    (command.outcome === 'complete' ||
+      command.outcome === 'complete_reference' ||
+      command.outcome === 'fail')
+    ? 'completion'
+    : 'control';
+}
+
+/**
+ * The gateway conditionally owns every receipt transition, but same-receipt
+ * commands still travel in program order. Different receipt keys are
+ * independent and can safely occupy the shared bounded transport pool.
+ */
+function receiptCommandOrderingKeys(
+  command: ReceiptCommand,
+): readonly string[] {
+  switch (command.kind) {
+    case 'get':
+    case 'claim':
+    case 'heartbeat':
+    case 'acquire_execution_lock':
+    case 'release_execution_lock':
+      return [command.input.key];
+    case 'settle':
+      return [command.input.key];
+  }
+}
+
 function classifyReceiptRetry(error: unknown): { retryAfterMs: number } | null {
   if (isAppRuntimeApiCapacityError(error)) {
     return { retryAfterMs: error.retryAfterMs };
@@ -443,6 +564,12 @@ async function sendReceiptCommands(input: {
         playName: input.playName,
         runId: first.input.runId,
         keys: commands.map((command) => command.input.key),
+        // A mixed writer batch must preserve normal full reads. The manifest
+        // probe safely falls back to its byte-admitted path when it receives a
+        // body, so never omit an output a concurrent caller requested.
+        ...(commands.every((command) => command.input.hydrateOutput === false)
+          ? { hydrateOutput: false }
+          : {}),
       });
     }
     case 'claim': {
@@ -463,6 +590,12 @@ async function sendReceiptCommands(input: {
         reclaimRunning: first.input.reclaimRunning,
         forceRefresh: first.input.forceRefresh,
         forceFailedRefresh: first.input.forceFailedRefresh,
+        // A mixed writer batch must retain a full body for callers that
+        // requested one. Manifest callers detect such a body and re-enter the
+        // byte-admitted hydration path before using it.
+        ...(commands.every((command) => command.input.hydrateOutput === false)
+          ? { hydrateOutput: false }
+          : {}),
       });
     }
     case 'settle':
@@ -478,6 +611,20 @@ async function sendReceiptCommands(input: {
             receipts: commands.map((command) => command.input),
           });
         }
+        case 'complete_reference': {
+          const commands = input.commands as readonly Extract<
+            ReceiptCommand,
+            { kind: 'settle'; outcome: 'complete_reference' }
+          >[];
+          return await completeRuntimeStepReceiptsByReferenceViaAppRuntime(
+            input.context,
+            {
+              playName: input.playName,
+              runId: first.input.runId,
+              receipts: commands.map((command) => command.input),
+            },
+          );
+        }
         case 'fail': {
           const commands = input.commands as readonly Extract<
             ReceiptCommand,

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts

@@ -0,0 +1,184 @@
+/**
+ * An immutable body stored outside a runtime receipt row.
+ *
+ * Receipt rows remain the authority for whether an outcome exists. A reference
+ * carries bytes only; it is never a cache miss, execution lease, or permission
+ * to run a provider operation again.
+ */
+/**
+ * R2 owns a referenced receipt's complete result body. These are the small,
+ * selected fields that may travel with its manifest for routing, presentation,
+ * and admission without redeeming that body.
+ */
+export const RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_BYTES = 16 * 1024;
+export const RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_KEYS = 32;
+const RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_DEPTH = 16;
+
+export type RuntimeReceiptValueInlineValue =
+  | string
+  | number
+  | boolean
+  | null
+  | RuntimeReceiptValueInlineValue[]
+  | { [key: string]: RuntimeReceiptValueInlineValue };
+
+export type RuntimeReceiptValueInlineFields = Record<
+  string,
+  RuntimeReceiptValueInlineValue
+>;
+
+export type RuntimeReceiptValueReference = {
+  storage: 'r2';
+  /**
+   * New manifests use the named JSON codec. The compatibility reader accepts
+   * the already-written legacy spelling without promoting a numeric protocol.
+   */
+  codec: 'json' | 'json-v1';
+  key: string;
+  digest: string;
+  byteLength: number;
+  /** SHA-256 binding for the authenticated organization, never its raw id. */
+  orgBinding: string;
+  /** SHA-256 binding for the receipt key that owns this value. */
+  receiptBinding: string;
+  /**
+   * Bounded selected fields beside an R2 body. Never a second full result.
+   * This field is additive so readers that predate it still use the reference.
+   */
+  inlineFields?: RuntimeReceiptValueInlineFields;
+};
+
+/**
+ * Canonicalize the small manifest payload before it crosses a receipt boundary.
+ * It is intentionally stricter than JSON.stringify: undefined values, Dates,
+ * non-finite numbers, and custom objects fail rather than changing meaning.
+ */
+export function normalizeRuntimeReceiptValueInlineFields(
+  value: unknown,
+): RuntimeReceiptValueInlineFields {
+  if (!isPlainRecord(value)) {
+    throw new Error('Runtime receipt inline fields must be a plain object.');
+  }
+  const keys = Object.keys(value);
+  if (keys.length > RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_KEYS) {
+    throw new Error(
+      `Runtime receipt inline fields exceed ${RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_KEYS} keys.`,
+    );
+  }
+  const copied = copyInlineValue(value, '$', new Set<object>(), 0);
+  if (!isPlainRecord(copied)) {
+    throw new Error('Runtime receipt inline fields must normalize to an object.');
+  }
+  const serialized = JSON.stringify(copied);
+  const bytes = new TextEncoder().encode(serialized).byteLength;
+  if (bytes > RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_BYTES) {
+    throw new Error(
+      `Runtime receipt inline fields exceed ${RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_BYTES} bytes.`,
+    );
+  }
+  return copied;
+}
+
+export function isRuntimeReceiptValueReference(
+  value: unknown,
+): value is RuntimeReceiptValueReference {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const candidate = value as Record<string, unknown>;
+  return (
+    candidate.storage === 'r2' &&
+    (candidate.codec === 'json' || candidate.codec === 'json-v1') &&
+    typeof candidate.key === 'string' &&
+    candidate.key.length > 0 &&
+    /^[a-f0-9]{64}$/i.test(String(candidate.digest ?? '')) &&
+    Number.isSafeInteger(candidate.byteLength) &&
+    Number(candidate.byteLength) >= 0 &&
+    /^[a-f0-9]{64}$/i.test(String(candidate.orgBinding ?? '')) &&
+    /^[a-f0-9]{64}$/i.test(String(candidate.receiptBinding ?? '')) &&
+    (candidate.inlineFields === undefined ||
+      isRuntimeReceiptValueInlineFields(candidate.inlineFields))
+  );
+}
+
+function isRuntimeReceiptValueInlineFields(
+  value: unknown,
+): value is RuntimeReceiptValueInlineFields {
+  try {
+    normalizeRuntimeReceiptValueInlineFields(value);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function copyInlineValue(
+  value: unknown,
+  path: string,
+  ancestors: Set<object>,
+  depth: number,
+): RuntimeReceiptValueInlineValue {
+  if (depth > RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_DEPTH) {
+    throw new Error(
+      `Runtime receipt inline field ${path} exceeds ${RUNTIME_RECEIPT_VALUE_INLINE_FIELDS_MAX_DEPTH} nested levels.`,
+    );
+  }
+  if (value === null) return null;
+  if (typeof value === 'string' || typeof value === 'boolean') return value;
+  if (typeof value === 'number') {
+    if (Number.isFinite(value)) return value;
+    throw new Error(`Runtime receipt inline field ${path} must be finite.`);
+  }
+  if (Array.isArray(value)) {
+    if (ancestors.has(value)) {
+      throw new Error(`Runtime receipt inline field ${path} cannot be cyclic.`);
+    }
+    ancestors.add(value);
+    try {
+      return value.map((entry, index) =>
+        copyInlineValue(entry, `${path}[${index}]`, ancestors, depth + 1),
+      );
+    } finally {
+      ancestors.delete(value);
+    }
+  }
+  if (isPlainRecord(value)) {
+    if (ancestors.has(value)) {
+      throw new Error(`Runtime receipt inline field ${path} cannot be cyclic.`);
+    }
+    ancestors.add(value);
+    try {
+      const copied: Record<string, RuntimeReceiptValueInlineValue> = {};
+      for (const [key, entry] of Object.entries(value)) {
+        copied[key] = copyInlineValue(
+          entry,
+          `${path}.${key}`,
+          ancestors,
+          depth + 1,
+        );
+      }
+      return copied;
+    } finally {
+      ancestors.delete(value);
+    }
+  }
+  throw new Error(
+    `Runtime receipt inline field ${path} must be JSON data, not ${
+      value === undefined ? 'undefined' : typeof value
+    }.`,
+  );
+}
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+export class RuntimeReceiptValueReadError extends Error {
+  constructor(
+    readonly code: 'missing' | 'corrupt' | 'unavailable' | 'scope_mismatch',
+    message: string,
+  ) {
+    super(message);
+    this.name = 'RuntimeReceiptValueReadError';
+  }
+}