deepline 0.3.127 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -18,14 +18,22 @@ import {
   runtimeReceiptCompletionResponse,
   type LegacyRuntimeReceiptWireFormat,
 } from '../runtime-contract';
+import { BoundedBatchCapacityError } from '../bounded-batch-function';
 import type {
   RuntimeReceiptAction,
   RuntimeReceiptFailure,
   RuntimeReceiptLease,
 } from '../runtime-actions';
+import type { RuntimeStepReceipt } from '../ctx-types';
+import {
+  isRuntimeReceiptValueReference,
+  normalizeRuntimeReceiptValueInlineFields,
+  type RuntimeReceiptValueReference,
+} from '../runtime-receipt-value';
 import {
   redactLedgerString,
   sanitizeRuntimeReceiptPayload,
+  sanitizeRuntimeReceiptPayloadForStorage,
 } from '../ledger-safe-payload';
 import {
   claimPostgresSchedulerReceipt,
@@ -53,6 +61,15 @@ import {
 import { transientServiceUnavailableDetail } from '../transient-service-error';
 import { isRuntimePostgresAdmissionCapacityError } from '../runtime-postgres-admission';
 import { normalizeToolExecutionFailure } from '../../plays/tool-execution-error';
+import {
+  assertRuntimeReceiptValueReferenceScope,
+  createRuntimeReceiptValuePersister,
+  createRuntimeReceiptValueReader,
+  RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES,
+  type RuntimeReceiptValueBlobStore,
+  type RuntimeReceiptValuePersister,
+  type RuntimeReceiptValueReader,
+} from './runtime-receipt-value-storage';
 
 type RuntimeReceiptActionWithOptionalScope<
   TAction extends RuntimeReceiptAction = RuntimeReceiptAction,
@@ -82,6 +99,7 @@ export const RECEIPT_GATEWAY_ACTIONS = new Set<string>([
   'mark_runtime_step_receipts_queued',
   'complete_runtime_step_receipt',
   'complete_runtime_step_receipts',
+  'complete_runtime_step_receipts_by_reference',
   'release_runtime_step_receipt',
   'fail_runtime_step_receipt',
   'fail_runtime_step_receipts',
@@ -213,6 +231,25 @@ export function receiptGatewayFailureResponse(
       },
     );
   }
+  if (error instanceof BoundedBatchCapacityError) {
+    return receiptGatewayJsonResponse(
+      {
+        error: `${operationLabel} is waiting for bounded batch capacity.`,
+        code: 'runtime_batch_backpressure',
+        retryable: true,
+        retry_after_ms: 1_000,
+        detail: error.message,
+        ...(context.action ? { action: context.action } : {}),
+      },
+      {
+        status: 503,
+        headers: {
+          'retry-after': '1',
+          'x-deepline-error-code': 'runtime_batch_backpressure',
+        },
+      },
+    );
+  }
   if (isRuntimeSchedulerCapacityError(error)) {
     const retryAfterMs = runtimeSchedulerCapacityRetryAfterMs(error);
     return receiptGatewayJsonResponse(
@@ -377,6 +414,216 @@ function receiptKeys(value: unknown): Array<string> | Response {
   return keys;
 }
 
+/**
+ * Redact, serialize, and size-check one receipt output exactly once.
+ *
+ * `outputJson` is the canonical Postgres JSON text for `output`, so the
+ * completion statement binds it straight to `jsonb` instead of re-running
+ * `stringifyPostgresJson` over the same value. Serializing a 600 KiB result
+ * four times (gate, pre-redaction measure, post-redaction measure, parameter
+ * build) was roughly half of gateway CPU on `complete_runtime_step_receipts`.
+ */
+async function receiptCompletionOutput(input: {
+  orgId: string;
+  key: string;
+  output: unknown;
+  inlineFields?: unknown;
+  persist: RuntimeReceiptValuePersister;
+}): Promise<{
+  outputJson: string | null;
+  outputRef?: RuntimeReceiptValueReference;
+  /**
+   * Wall spent in the value persister after the value was redacted and
+   * serialized. For a referenced value this includes its bounded R2 admission
+   * wait and immutable PUT; for a small value it is the inline fast path.
+   *
+   * This is intentionally kept gateway-internal. The compact completion
+   * acknowledgement never carries storage timing or an object identity.
+   */
+  valuePersistMs: number;
+  valuePersistedByReference: boolean;
+}> {
+  const sanitized = sanitizeRuntimeReceiptPayloadForStorage(
+    input.output ?? null,
+  );
+  const inlineFields =
+    input.inlineFields === undefined
+      ? undefined
+      : normalizeRuntimeReceiptValueInlineFields(
+          sanitizeRuntimeReceiptPayload(input.inlineFields),
+        );
+  const persistStartedAt = Date.now();
+  const stored = await input.persist({
+    orgId: input.orgId,
+    receiptKey: input.key,
+    serialized: sanitized.serialized,
+    inlineFields,
+  });
+  return {
+    outputJson: stored.outputJson,
+    ...(stored.kind === 'reference' ? { outputRef: stored.outputRef } : {}),
+    valuePersistMs: Math.max(0, Date.now() - persistStartedAt),
+    valuePersistedByReference: stored.kind === 'reference',
+  };
+}
+
+export type ReceiptGatewayValuePersistenceObservation = {
+  /** Number of receipt outputs in this HTTP completion request. */
+  values: number;
+  /** Outputs actually persisted through the bounded immutable-object writer. */
+  referencedValues: number;
+  /**
+   * Slowest individual persistence wall in this HTTP request. It includes the
+   * process-shared R2 writer's admission wait, so it is the useful value to
+   * compare with the gateway callback wall. It is zero only when every output
+   * took the inline path.
+   */
+  maxPersistMs: number;
+};
+
+type ReceiptCompletionOutputWithPersistence = Awaited<
+  ReturnType<typeof receiptCompletionOutput>
+>;
+
+/**
+ * Publish a deliberately best-effort view of the value-persistence portion of
+ * a completion. The ordinary and bulk completion actions share this helper so
+ * a legacy single receipt cannot make the gateway look faster than it is.
+ */
+function observeReceiptValuePersistence(
+  outputs: readonly ReceiptCompletionOutputWithPersistence[],
+  observe?: (observation: ReceiptGatewayValuePersistenceObservation) => void,
+): void {
+  try {
+    observe?.({
+      values: outputs.length,
+      referencedValues: outputs.filter(
+        (value) => value.valuePersistedByReference,
+      ).length,
+      maxPersistMs: outputs.reduce(
+        (max, value) => Math.max(max, value.valuePersistMs),
+        0,
+      ),
+    });
+  } catch {
+    // Observability is intentionally best-effort.
+  }
+}
+
+/**
+ * Project one `complete_runtime_step_receipts` body into storage completions.
+ *
+ * Shared by the coalesced commit and the client-bound switch so both paths
+ * sanitize, serialize, and scope identically; `keys` are the already
+ * scope-checked receipt keys, in request order.
+ */
+async function buildReceiptCompletions(input: {
+  orgId: string;
+  persist: RuntimeReceiptValuePersister;
+  receipts: readonly unknown[];
+  keys: readonly string[];
+  runId: string;
+  onValuePersistence?: (
+    observation: ReceiptGatewayValuePersistenceObservation,
+  ) => void;
+}): Promise<ReceiptGatewayCompletionEntry[]> {
+  const built = await Promise.all(
+    (input.receipts as Array<Record<string, unknown>>).map(
+      async (receipt, index) => {
+        const output = await receiptCompletionOutput({
+          orgId: input.orgId,
+          key: input.keys[index]!,
+          output: receipt.output,
+          inlineFields: receipt.inlineFields,
+          persist: input.persist,
+        });
+        return {
+          key: input.keys[index]!,
+          runId: input.runId,
+          runAttempt:
+            typeof receipt.runAttempt === 'number' ? receipt.runAttempt : null,
+          leaseId: typeof receipt.leaseId === 'string' ? receipt.leaseId : null,
+          ...output,
+        };
+      },
+    ),
+  );
+  observeReceiptValuePersistence(built, input.onValuePersistence);
+  return built.map(
+    ({
+      valuePersistMs: _valuePersistMs,
+      valuePersistedByReference: _ref,
+      ...completion
+    }) => completion,
+  );
+}
+
+/**
+ * The execution relay writes a complete, serialized ToolExecuteResult to the
+ * receipt-owned immutable key before it attaches the reference to its normal
+ * provider response. This receiver validates that the reference belongs to
+ * the same authenticated receipt and commits only its compact manifest.
+ */
+async function buildReferencedReceiptCompletions(input: {
+  orgId: string;
+  receipts: readonly unknown[];
+  keys: readonly string[];
+  runId: string;
+  store?: RuntimeReceiptValueBlobStore;
+}): Promise<ReceiptGatewayCompletionEntry[]> {
+  return await Promise.all(
+    (input.receipts as Array<Record<string, unknown>>).map(
+      async (receipt, index) => {
+        const outputRef = receipt.outputRef;
+        if (!isRuntimeReceiptValueReference(outputRef)) {
+          throw new Error(
+            'Referenced receipt completion requires a valid outputRef.',
+          );
+        }
+        const key = input.keys[index]!;
+        assertRuntimeReceiptValueReferenceScope({
+          orgId: input.orgId,
+          receiptKey: key,
+          reference: outputRef,
+        });
+        // A reference must not point at an object a later receipt read cannot
+        // redeem. The execution relay has already recovered an ambiguous PUT
+        // before sending it, so this is a cheap integrity fence, not a second
+        // provider-side operation.
+        if (input.store && !(await input.store.exists(outputRef.key))) {
+          throw new Error(
+            'Referenced receipt output is not present in immutable storage.',
+          );
+        }
+        return {
+          key,
+          runId: input.runId,
+          runAttempt:
+            typeof receipt.runAttempt === 'number' ? receipt.runAttempt : null,
+          leaseId: typeof receipt.leaseId === 'string' ? receipt.leaseId : null,
+          outputJson: null,
+          outputRef,
+        };
+      },
+    ),
+  );
+}
+
+/** Serialized completion size, used only for the coalescer's byte bound. */
+function receiptCompletionBytes(
+  completions: readonly ReceiptGatewayCompletionEntry[],
+): number {
+  let bytes = 0;
+  for (const completion of completions) {
+    const storedPayload =
+      completion.outputJson ?? JSON.stringify(completion.outputRef ?? null);
+    bytes +=
+      Buffer.byteLength(completion.key, 'utf8') +
+      Buffer.byteLength(storedPayload, 'utf8');
+  }
+  return bytes;
+}
+
 export type ReceiptGatewayDispatcher = (
   scope: ReceiptGatewayScope,
   body: unknown,
@@ -397,6 +644,15 @@ export type ReceiptGatewayDispatchOptions = {
     receipts: readonly unknown[];
     forceRefresh: boolean;
   }) => void;
+  /**
+   * On-the-wire body bytes of the HTTP request being dispatched, when the
+   * transport knows them and they differ from the decoded size — that is, when
+   * the body arrived gzipped. Telemetry only: it is forwarded to the completion
+   * commit so a coalesced statement can report what the network actually moved
+   * beside the decoded bytes it was admitted on. Absent (uncompressed request,
+   * PGlite, tests) the commit's wire bytes default to its decoded estimate.
+   */
+  requestWireBytes?: number | null;
 };
 
 async function injectReceiptCompletionFailure(input: {
@@ -436,13 +692,152 @@ async function injectReceiptCompletionFailure(input: {
   }
 }
 
+/**
+ * One receipt's completion, already scope-checked and serialized once by the
+ * gateway. It carries either canonical inline `jsonb` text or a compact,
+ * already-published value reference, so a coalescing commit never re-walks a
+ * provider body.
+ */
+export type ReceiptGatewayCompletionEntry = {
+  key: string;
+  runId: string;
+  runAttempt: number | null;
+  leaseId: string | null;
+  outputJson: string | null;
+  outputRef?: RuntimeReceiptValueReference;
+};
+
 export function createReceiptGatewayDispatcher(input: {
   withClient: <T>(
     fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
   ) => Promise<T>;
   schema?: string;
+  /** Test seam and the Fly value-store adapter for large receipt bodies. */
+  runtimeReceiptValueStore?: RuntimeReceiptValueBlobStore;
+  runtimeReceiptValueInlineMaxBytes?: number;
+  /**
+   * The Fly reader-fence adapter enables this only after every durable gateway
+   * reader proves it understands referenced values. Tests and local adapters
+   * retain their explicit injected-store behavior when this is omitted.
+   */
+  runtimeReceiptValueReferencesEnabled?: boolean;
+  runtimeReceiptValuePersister?: RuntimeReceiptValuePersister;
+  runtimeReceiptValueReader?: RuntimeReceiptValueReader;
+  /**
+   * Optional gateway-local observation of large receipt-value persistence.
+   * It is diagnostic-only: a slow or throwing observer must not affect the
+   * durable completion it observes.
+   */
+  onReceiptValuePersistence?: (
+    observation: ReceiptGatewayValuePersistenceObservation,
+  ) => void;
   consumeReceiptCompleteFault?: (runId: string) => boolean | Promise<boolean>;
+  /**
+   * Commit bulk receipt completions outside this dispatcher's own client.
+   *
+   * The production gateway supplies a coalescing commit that merges concurrent
+   * requests of the same org/schema/wire format into one statement under one
+   * heavy-admission slot. It MUST return results index-aligned with the
+   * supplied receipts, and MUST throw the error the caller would have seen for
+   * an uncoalesced request so every merged request stays retry-safe. Absent
+   * (tests, PGlite), completions run through `withClient` exactly as before.
+   */
+  commitReceiptCompletions?: (commit: {
+    orgId: string;
+    schema: string | null;
+    wireFormat: LegacyRuntimeReceiptWireFormat;
+    receipts: ReceiptGatewayCompletionEntry[];
+    bytes: number;
+    /**
+     * On-the-wire body bytes of the request that carried these receipts.
+     * Equals `bytes` when the request arrived uncompressed. Telemetry only.
+     */
+    wireBytes: number;
+  }) => Promise<Array<RuntimeStepReceipt | null>>;
 }): ReceiptGatewayDispatcher {
+  const inlineMaxBytes =
+    input.runtimeReceiptValueInlineMaxBytes ??
+    RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES;
+  const persistReceiptValue: RuntimeReceiptValuePersister =
+    input.runtimeReceiptValueReferencesEnabled === false
+      ? async (value) => ({ kind: 'inline', outputJson: value.serialized })
+      : (input.runtimeReceiptValuePersister ??
+        (input.runtimeReceiptValueStore
+          ? createRuntimeReceiptValuePersister({
+              store: input.runtimeReceiptValueStore,
+              inlineMaxBytes,
+            })
+          : async (value) => {
+              // Expand readers ship independently from writers. Test/local dispatchers
+              // remain inline until their Fly host explicitly supplies the authenticated
+              // value store; a reference is never published merely because a process
+              // happens to have ambient R2 credentials.
+              return { kind: 'inline', outputJson: value.serialized };
+            }));
+
+  const readReceiptValue: RuntimeReceiptValueReader =
+    input.runtimeReceiptValueReader ??
+    createRuntimeReceiptValueReader(
+      input.runtimeReceiptValueStore
+        ? { store: input.runtimeReceiptValueStore }
+        : {},
+    );
+  const hydrateReceipt = async (
+    receipt: RuntimeStepReceipt | null,
+    orgId: string,
+  ): Promise<RuntimeStepReceipt | null> => {
+    if (!receipt?.outputRef) return receipt;
+    const output = await readReceiptValue({
+      orgId,
+      receiptKey: receipt.key,
+      reference: receipt.outputRef,
+    });
+    const { outputRef: _outputRef, ...hydrated } = receipt;
+    return {
+      ...hydrated,
+      ...(receipt.outputRef.inlineFields
+        ? { inlineFields: receipt.outputRef.inlineFields }
+        : {}),
+      output,
+    };
+  };
+  /**
+   * A completed-cache probe needs terminal state and a safe byte price, not a
+   * second copy of the result. Keep R2 references gateway-internal: only the
+   * scalar size crosses to the runner, and a later normal read still redeems
+   * and validates the body before it can become a cache hit.
+   */
+  const receiptManifest = (
+    receipt: RuntimeStepReceipt | null,
+  ): RuntimeStepReceipt | null => {
+    if (!receipt) return null;
+    const { output, outputRef, ...manifest } = receipt;
+    const outputByteLength =
+      outputRef?.byteLength ??
+      (output === undefined
+        ? null
+        : new TextEncoder().encode(JSON.stringify(output)).byteLength);
+    return {
+      ...manifest,
+      ...(outputByteLength === null ? {} : { outputByteLength }),
+    };
+  };
+  const hydrateReceipts = async (
+    receipts: readonly (RuntimeStepReceipt | null)[],
+    orgId: string,
+  ): Promise<Array<RuntimeStepReceipt | null>> =>
+    await Promise.all(
+      receipts.map(async (receipt) => await hydrateReceipt(receipt, orgId)),
+    );
+  const presentCompletionReceipt = async (input: {
+    receipt: RuntimeStepReceipt | null;
+    orgId: string;
+    wireFormat: LegacyRuntimeReceiptWireFormat;
+  }): Promise<RuntimeStepReceipt | null> =>
+    input.wireFormat < 3
+      ? await hydrateReceipt(input.receipt, input.orgId)
+      : input.receipt;
+
   return async (scope, rawBody, dispatchOptions = {}) => {
     if (!rawBody || typeof rawBody !== 'object') {
       return receiptGatewayJsonResponse(
@@ -454,6 +849,11 @@ export function createReceiptGatewayDispatcher(input: {
     const scopeError = requireReceiptScope(scope, body);
     if (scopeError) return scopeError;
     const runId = String(body.runId || scope.runId);
+    // Additive read presentation. Older callers omit this and receive the
+    // exact hydrated body they did before; only a Foundation cache probe asks
+    // for the compact manifest.
+    const hydrateOutput =
+      !('hydrateOutput' in body) || body.hydrateOutput !== false;
     const wireFormat = dispatchOptions.wireFormat ?? 3;
     // Org scope is proven by the executor token at the transport (production:
     // apps/runtime-receipt-gateway/index.ts passes auth.orgId here). The
@@ -461,6 +861,67 @@ export function createReceiptGatewayDispatcher(input: {
     // through identically for the test/PGlite dispatch to match production.
     const options = { schema: input.schema, orgId: scope.orgId };
 
+    // A coalescing commit owns admission, the pooled client, and the statement
+    // for a whole window of requests, so this path must not take a client of
+    // its own — that is the entire cost being removed. The injected
+    // `receipt_complete_write_fail` path still needs one, so it falls through
+    // to the client-bound switch with its fault decision already consumed.
+    let completionFaultInjected: boolean | null = null;
+    if (
+      body.action === 'complete_runtime_step_receipts' &&
+      input.commitReceiptCompletions
+    ) {
+      const sameRunError = requireSameRunReceipts(runId, body.receipts);
+      if (sameRunError) return sameRunError;
+      const keys = receiptKeys(
+        (body.receipts as Array<Record<string, unknown>>).map(
+          (receipt) => receipt.key,
+        ),
+      );
+      if (keys instanceof Response) return keys;
+      completionFaultInjected =
+        (await input.consumeReceiptCompleteFault?.(runId)) === true;
+      if (!completionFaultInjected) {
+        const completions = await buildReceiptCompletions({
+          orgId: scope.orgId,
+          persist: persistReceiptValue,
+          receipts: body.receipts,
+          keys,
+          runId,
+          onValuePersistence: input.onReceiptValuePersistence,
+        });
+        const bytes = receiptCompletionBytes(completions);
+        const committed = await input.commitReceiptCompletions({
+          orgId: scope.orgId,
+          schema: input.schema ?? null,
+          wireFormat,
+          receipts: completions,
+          bytes,
+          // A transport that did not report a wire size sent these bytes
+          // uncompressed, so the decoded estimate IS the wire cost.
+          wireBytes:
+            typeof dispatchOptions.requestWireBytes === 'number' &&
+            Number.isFinite(dispatchOptions.requestWireBytes)
+              ? Math.max(0, Math.floor(dispatchOptions.requestWireBytes))
+              : bytes,
+        });
+        return receiptGatewayJsonResponse(
+          await Promise.all(
+            committed.map(async (receipt) =>
+              runtimeReceiptCompletionResponse(
+                wireFormat,
+                await presentCompletionReceipt({
+                  receipt,
+                  orgId: scope.orgId,
+                  wireFormat,
+                }),
+              ),
+            ),
+          ),
+        );
+      }
+    }
+
     return await input.withClient(async (client) => {
       switch (body.action) {
         case 'get_runtime_step_receipt': {
@@ -478,7 +939,11 @@ export function createReceiptGatewayDispatcher(input: {
             keys: [key],
             receipts: [receipt],
           });
-          return receiptGatewayJsonResponse(receipt);
+          return receiptGatewayJsonResponse(
+            hydrateOutput
+              ? await hydrateReceipt(receipt, scope.orgId)
+              : receiptManifest(receipt),
+          );
         }
         case 'get_runtime_step_receipts': {
           const keys = receiptKeys(body.keys);
@@ -495,7 +960,11 @@ export function createReceiptGatewayDispatcher(input: {
             keys,
             receipts,
           });
-          return receiptGatewayJsonResponse(receipts);
+          return receiptGatewayJsonResponse(
+            hydrateOutput
+              ? await hydrateReceipts(receipts, scope.orgId)
+              : receipts.map(receiptManifest),
+          );
         }
         case 'claim_runtime_step_receipt': {
           const key = receiptKey(body.key);
@@ -524,7 +993,11 @@ export function createReceiptGatewayDispatcher(input: {
             receipts: [receipt],
             forceRefresh: body.forceRefresh === true,
           });
-          return receiptGatewayJsonResponse(receipt);
+          return receiptGatewayJsonResponse(
+            hydrateOutput
+              ? await hydrateReceipt(receipt, scope.orgId)
+              : receiptManifest(receipt),
+          );
         }
         case 'claim_runtime_step_receipts': {
           const keys = receiptKeys(body.keys);
@@ -553,7 +1026,11 @@ export function createReceiptGatewayDispatcher(input: {
             receipts,
             forceRefresh: body.forceRefresh === true,
           });
-          return receiptGatewayJsonResponse(receipts);
+          return receiptGatewayJsonResponse(
+            hydrateOutput
+              ? await hydrateReceipts(receipts, scope.orgId)
+              : receipts.map(receiptManifest),
+          );
         }
         case 'mark_runtime_step_receipts_running': {
           const sameRunError = requireSameRunReceipts(runId, body.receipts);
@@ -597,25 +1074,43 @@ export function createReceiptGatewayDispatcher(input: {
               { status: 409 },
             );
           }
+          const completionOutput = await receiptCompletionOutput({
+            orgId: scope.orgId,
+            key,
+            output: body.output,
+            inlineFields: body.inlineFields,
+            persist: persistReceiptValue,
+          });
+          observeReceiptValuePersistence(
+            [completionOutput],
+            input.onReceiptValuePersistence,
+          );
+          const {
+            valuePersistMs: _valuePersistMs,
+            valuePersistedByReference: _valuePersistedByReference,
+            ...storedCompletionOutput
+          } = completionOutput;
+          const completed = await completePostgresSchedulerReceipt(
+            client,
+            {
+              key,
+              runId,
+              runAttempt:
+                typeof body.runAttempt === 'number' ? body.runAttempt : null,
+              leaseId: typeof body.leaseId === 'string' ? body.leaseId : null,
+              ...storedCompletionOutput,
+              returnOutput: wireFormat < 3,
+            },
+            options,
+          );
           return receiptGatewayJsonResponse(
             runtimeReceiptCompletionResponse(
               wireFormat,
-              await completePostgresSchedulerReceipt(
-                client,
-                {
-                  key,
-                  runId,
-                  runAttempt:
-                    typeof body.runAttempt === 'number'
-                      ? body.runAttempt
-                      : null,
-                  leaseId:
-                    typeof body.leaseId === 'string' ? body.leaseId : null,
-                  output: sanitizeRuntimeReceiptPayload(body.output ?? null),
-                  returnOutput: wireFormat < 3,
-                },
-                options,
-              ),
+              await presentCompletionReceipt({
+                receipt: completed,
+                orgId: scope.orgId,
+                wireFormat,
+              }),
             ),
           );
         }
@@ -628,18 +1123,18 @@ export function createReceiptGatewayDispatcher(input: {
             ),
           );
           if (keys instanceof Response) return keys;
-          const completions = body.receipts.map((receipt, index) => ({
-            key: keys[index]!,
+          const completions = await buildReceiptCompletions({
+            orgId: scope.orgId,
+            persist: persistReceiptValue,
+            receipts: body.receipts,
+            keys,
             runId,
-            runAttempt:
-              typeof receipt.runAttempt === 'number'
-                ? receipt.runAttempt
-                : null,
-            leaseId:
-              typeof receipt.leaseId === 'string' ? receipt.leaseId : null,
-            output: sanitizeRuntimeReceiptPayload(receipt.output ?? null),
-          }));
-          if (await input.consumeReceiptCompleteFault?.(runId)) {
+            onValuePersistence: input.onReceiptValuePersistence,
+          });
+          if (
+            completionFaultInjected ??
+            (await input.consumeReceiptCompleteFault?.(runId))
+          ) {
             const [skippedReceipt, ...siblingReceipts] = completions;
             if (siblingReceipts.length > 0) {
               await completePostgresSchedulerReceipts(
@@ -669,18 +1164,82 @@ export function createReceiptGatewayDispatcher(input: {
               { status: 409 },
             );
           }
+          const completed = await completePostgresSchedulerReceipts(
+            client,
+            {
+              receipts: completions,
+              returnOutput: wireFormat < 3,
+            },
+            options,
+          );
+          return receiptGatewayJsonResponse(
+            await Promise.all(
+              completed.map(async (receipt) =>
+                runtimeReceiptCompletionResponse(
+                  wireFormat,
+                  await presentCompletionReceipt({
+                    receipt,
+                    orgId: scope.orgId,
+                    wireFormat,
+                  }),
+                ),
+              ),
+            ),
+          );
+        }
+        case 'complete_runtime_step_receipts_by_reference': {
+          // This action is the writer half of an expand/enable rollout. A
+          // runner never relies on it without retaining its full live result,
+          // so an unavailable receiver safely falls back to the legacy action
+          // rather than turning a receipt reference into a null output.
+          if (
+            input.runtimeReceiptValueReferencesEnabled === false ||
+            !input.runtimeReceiptValueStore
+          ) {
+            return receiptGatewayJsonResponse(
+              {
+                error:
+                  'Receipt-output reference completion is not enabled on this gateway.',
+                code: 'receipt_output_reference_unsupported',
+              },
+              { status: 409 },
+            );
+          }
+          const sameRunError = requireSameRunReceipts(runId, body.receipts);
+          if (sameRunError) return sameRunError;
+          const keys = receiptKeys(
+            (body.receipts as Array<Record<string, unknown>>).map(
+              (receipt) => receipt.key,
+            ),
+          );
+          if (keys instanceof Response) return keys;
+          const completions = await buildReferencedReceiptCompletions({
+            orgId: scope.orgId,
+            receipts: body.receipts,
+            keys,
+            runId,
+            store: input.runtimeReceiptValueStore,
+          });
+          const completed = await completePostgresSchedulerReceipts(
+            client,
+            {
+              receipts: completions,
+              returnOutput: wireFormat < 3,
+            },
+            options,
+          );
           return receiptGatewayJsonResponse(
-            (
-              await completePostgresSchedulerReceipts(
-                client,
-                {
-                  receipts: completions,
-                  returnOutput: wireFormat < 3,
-                },
-                options,
-              )
-            ).map((receipt) =>
-              runtimeReceiptCompletionResponse(wireFormat, receipt),
+            await Promise.all(
+              completed.map(async (receipt) =>
+                runtimeReceiptCompletionResponse(
+                  wireFormat,
+                  await presentCompletionReceipt({
+                    receipt,
+                    orgId: scope.orgId,
+                    wireFormat,
+                  }),
+                ),
+              ),
             ),
           );
         }