deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts

@@ -0,0 +1,888 @@
+/**
+ * Runtime receipt gateway dispatch.
+ *
+ * This is THE receipt data plane for play runners: every runtime step receipt
+ * read/write from a runner crosses HTTP to a co-located gateway process that
+ * holds the runtime Postgres (PlanetScale) pool. There is exactly one receipt
+ * backend — runners have no direct-Postgres receipt path and no app runtime
+ * API fallback (see apps/play-runner/src/entry.ts).
+ *
+ * The module is transport-agnostic on the database side so the production
+ * gateway (apps/runtime-receipt-gateway/index.ts, pooled `pg` against
+ * PlanetScale) and tests (PGlite) run the identical dispatch, scope checks,
+ * and payload sanitization.
+ */
+import {
+  LEGACY_RUNTIME_RECEIPT_WIRE_FORMAT,
+  LEGACY_RUNTIME_RECEIPT_WIRE_HEADER,
+  runtimeReceiptCompletionResponse,
+  type LegacyRuntimeReceiptWireFormat,
+} from '../runtime-contract';
+import type {
+  RuntimeReceiptAction,
+  RuntimeReceiptFailure,
+  RuntimeReceiptLease,
+} from '../runtime-actions';
+import {
+  redactLedgerString,
+  sanitizeRuntimeReceiptPayload,
+} from '../ledger-safe-payload';
+import {
+  claimPostgresSchedulerReceipt,
+  claimPostgresSchedulerReceipts,
+  completePostgresSchedulerReceipt,
+  completePostgresSchedulerReceipts,
+  failPostgresSchedulerReceipt,
+  failPostgresSchedulerReceipts,
+  getPostgresSchedulerReceipt,
+  getPostgresSchedulerReceipts,
+  heartbeatPostgresSchedulerReceipts,
+  markPostgresSchedulerReceiptsQueued,
+  markPostgresSchedulerReceiptsRunning,
+  releasePostgresSchedulerReceipt,
+  skipPostgresSchedulerReceipt,
+  upsertPostgresSchedulerObservation,
+} from '../scheduler-backends/postgres';
+import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
+import type { DocflowObservationStatus } from '../docflow-observation';
+import {
+  isRuntimeSchedulerCapacityError,
+  runtimeSchedulerCapacityRetryAfterMs,
+  type RuntimeSchedulerQueryClient,
+} from '../scheduler-backends/postgres-pool';
+import { transientServiceUnavailableDetail } from '../transient-service-error';
+import { isRuntimePostgresAdmissionCapacityError } from '../runtime-postgres-admission';
+import { normalizeToolExecutionFailure } from '../../plays/tool-execution-error';
+
+type RuntimeReceiptActionWithOptionalScope<
+  TAction extends RuntimeReceiptAction = RuntimeReceiptAction,
+> = TAction extends RuntimeReceiptAction
+  ? Omit<TAction, 'playName' | 'runId'> & {
+      playName?: string;
+      runId?: string;
+    }
+  : never;
+
+export type ReceiptGatewayActionBody = RuntimeReceiptActionWithOptionalScope;
+
+/**
+ * Canonical set of actions the receipt gateway dispatcher accepts. The
+ * production gateway transport (`apps/runtime-receipt-gateway/index.ts`) must
+ * gate inbound requests against THIS constant rather than a hand-copied list so
+ * a new dispatcher action (e.g. `observe_docflow_node`) is admitted without a
+ * separate allowlist edit. `observe_docflow_node` (ADR 0016 rule 2) rides the
+ * same durable receipt wire as the receipt actions.
+ */
+export const RECEIPT_GATEWAY_ACTIONS = new Set<string>([
+  'get_runtime_step_receipt',
+  'get_runtime_step_receipts',
+  'claim_runtime_step_receipt',
+  'claim_runtime_step_receipts',
+  'mark_runtime_step_receipts_running',
+  'mark_runtime_step_receipts_queued',
+  'complete_runtime_step_receipt',
+  'complete_runtime_step_receipts',
+  'release_runtime_step_receipt',
+  'fail_runtime_step_receipt',
+  'fail_runtime_step_receipts',
+  'heartbeat_runtime_step_receipts',
+  'skip_runtime_step_receipt',
+  'observe_docflow_node',
+]);
+
+/** Run/play scope already proven by the transport's executor-token auth. */
+export type ReceiptGatewayScope = {
+  orgId: string;
+  playName: string;
+  runId: string;
+};
+
+export type ExecutionGatewayBypassDecision = {
+  token: string;
+  source: 'launch' | 'env' | 'none';
+};
+
+/**
+ * Select the Vercel protection-bypass token the execution gateway attaches when
+ * proxying a runtime-control request to the (possibly protected) app.
+ *
+ * Precedence matches the worker's `buildPostgresSchedulerRunnerContext`: the
+ * launch-pinned, request-scoped token first, then the gateway's own env as a
+ * fallback. Returns an empty token with source `none` when neither is present.
+ * The source is logged in the gateway request trail so a protection 307 can be
+ * attributed to its credential origin without another blind CI cycle.
+ *
+ * The Daytona sandbox holds NO bypass credential (stripped by
+ * `remoteRuntimeContextForDaytona`), so the gateway is the sole hop that can
+ * attach it before the request reaches a protected Vercel preview. Without the
+ * env fallback, a launch row that never captured the token (or a re-dispatched
+ * child/retry that dropped it) reaches Vercel Deployment Protection bare and
+ * 307s to SSO.
+ */
+export function resolveExecutionGatewayBypassToken(
+  launchToken: string | null | undefined,
+  envToken: string | null | undefined,
+): ExecutionGatewayBypassDecision {
+  const fromLaunch = launchToken?.trim();
+  if (fromLaunch) return { token: fromLaunch, source: 'launch' };
+  const fromEnv = envToken?.trim();
+  if (fromEnv) return { token: fromEnv, source: 'env' };
+  return { token: '', source: 'none' };
+}
+
+/**
+ * Attach a header-only Vercel automation bypass to a gateway-to-app request.
+ * A gateway is a server-to-server hop and must never request a bypass cookie:
+ * Vercel responds with a redirect in order to establish that cookie, while the
+ * runner intentionally refuses redirects on its runtime-control plane.
+ */
+export function applyExecutionGatewayBypassHeader(
+  headers: Headers,
+  bypass: ExecutionGatewayBypassDecision,
+): void {
+  headers.delete('x-vercel-set-bypass-cookie');
+  if (bypass.token) {
+    headers.set('x-vercel-protection-bypass', bypass.token);
+  }
+}
+
+/**
+ * Detect an execution-gateway self-proxy: the launch row's runtime API base
+ * pointing back at the gateway's own public host. Forwarding would loop (every
+ * hop re-authenticates, strips the bypass header, and re-enters the proxy), so
+ * the gateway must refuse loudly instead. `ownHost` is the Host header of the
+ * inbound request — Fly presents the app's public hostname there.
+ */
+export function isExecutionGatewaySelfProxy(
+  upstreamUrl: string,
+  ownHost: string | null | undefined,
+): boolean {
+  const normalizedOwnHost = ownHost?.trim().toLowerCase();
+  if (!normalizedOwnHost) return false;
+  try {
+    return new URL(upstreamUrl).host.toLowerCase() === normalizedOwnHost;
+  } catch {
+    return false;
+  }
+}
+
+export function receiptGatewayJsonResponse(
+  body: unknown,
+  init: ResponseInit = {},
+): Response {
+  const headers = new Headers(init.headers);
+  headers.set('content-type', 'application/json');
+  headers.set(
+    LEGACY_RUNTIME_RECEIPT_WIRE_HEADER,
+    String(LEGACY_RUNTIME_RECEIPT_WIRE_FORMAT),
+  );
+  return new Response(JSON.stringify(body), { ...init, headers });
+}
+
+export type RuntimeGatewayFailureContext = {
+  action?: string | null;
+  operation?: 'receipt' | 'runtime_sheet' | 'runtime_durability';
+};
+
+export function receiptGatewayFailureResponse(
+  error: unknown,
+  context: RuntimeGatewayFailureContext = {},
+): Response {
+  const operationLabel =
+    context.operation === 'runtime_sheet'
+      ? 'Runtime Sheet persistence'
+      : context.operation === 'runtime_durability'
+        ? 'Runtime durability operation'
+        : 'Work Receipt persistence';
+  if (isRuntimePostgresAdmissionCapacityError(error)) {
+    return receiptGatewayJsonResponse(
+      {
+        error: `${operationLabel} is waiting for database capacity.`,
+        code: 'runtime_postgres_admission_backpressure',
+        retryable: true,
+        retry_after_ms: 1_000,
+        detail: error.message,
+        ...(context.action ? { action: context.action } : {}),
+      },
+      {
+        status: 503,
+        headers: {
+          'retry-after': '1',
+          'x-deepline-error-code': 'runtime_postgres_admission_backpressure',
+        },
+      },
+    );
+  }
+  if (isRuntimeSchedulerCapacityError(error)) {
+    const retryAfterMs = runtimeSchedulerCapacityRetryAfterMs(error);
+    return receiptGatewayJsonResponse(
+      {
+        error: `${operationLabel} delayed.`,
+        code: 'scheduler_capacity',
+        retry_after_ms: retryAfterMs,
+        detail: error.message,
+        ...(context.action ? { action: context.action } : {}),
+      },
+      {
+        status: 503,
+        headers: {
+          'retry-after': String(Math.max(1, Math.ceil(retryAfterMs / 1_000))),
+          'x-deepline-error-code': 'scheduler_capacity',
+        },
+      },
+    );
+  }
+
+  const detail =
+    redactLedgerString(error instanceof Error ? error.message : error) ??
+    'Unknown runtime gateway failure.';
+  const headerDetail = detail.replace(/[\r\n]+/g, ' ').slice(0, 500);
+  const unavailableDetail = transientServiceUnavailableDetail(error);
+  if (unavailableDetail) {
+    return receiptGatewayJsonResponse(
+      {
+        error: `${operationLabel} dependency is temporarily unavailable.`,
+        code: 'upstream_service_unavailable',
+        retryable: true,
+        detail: unavailableDetail,
+        ...(context.action ? { action: context.action } : {}),
+      },
+      {
+        status: 503,
+        headers: {
+          'retry-after': '1',
+          'x-deepline-error-code': 'upstream_service_unavailable',
+          'x-deepline-error-detail': unavailableDetail
+            .replace(/[\r\n]+/g, ' ')
+            .slice(0, 500),
+        },
+      },
+    );
+  }
+  const failure =
+    context.operation === 'runtime_sheet'
+      ? {
+          error: 'Runtime Sheet persistence failed.',
+          code: 'runtime_sheet_persistence_failed',
+        }
+      : context.operation === 'runtime_durability'
+        ? {
+            error: 'Runtime durability operation failed.',
+            code: 'runtime_durability_internal',
+          }
+        : {
+            error: 'Work Receipt persistence failed.',
+            code: 'work_receipt_persistence_failed',
+          };
+  return receiptGatewayJsonResponse(
+    {
+      ...failure,
+      retryable: false,
+      detail,
+      ...(context.action ? { action: context.action } : {}),
+    },
+    {
+      status: 500,
+      // Surface the underlying failure on a header so the gateway request log
+      // (which only reads response headers, never the body) can print WHY a
+      // 500 happened. Without this a connect/query failure is an opaque
+      // `status: 500, errorCode: null` line.
+      headers: {
+        'x-deepline-error-code': failure.code,
+        'x-deepline-error-detail': headerDetail,
+      },
+    },
+  );
+}
+
+function requireReceiptScope(
+  scope: ReceiptGatewayScope,
+  body: ReceiptGatewayActionBody,
+): Response | null {
+  if (!scope.orgId || !scope.playName || !scope.runId) {
+    return receiptGatewayJsonResponse(
+      { error: 'Executor token is missing org/run/play scope.' },
+      { status: 403 },
+    );
+  }
+  if (body.playName !== scope.playName) {
+    return receiptGatewayJsonResponse(
+      { error: 'Receipt request playName does not match executor token.' },
+      { status: 403 },
+    );
+  }
+  if (body.runId && body.runId !== scope.runId) {
+    console.warn('[runtime-receipt-gateway.scope_mismatch]', {
+      action: body.action ?? 'unknown',
+      tokenRunId: scope.runId,
+      requestRunId: body.runId,
+    });
+    return receiptGatewayJsonResponse(
+      { error: 'Receipt request runId does not match executor token.' },
+      { status: 403 },
+    );
+  }
+  return null;
+}
+
+function requireSameRunReceipts(
+  runId: string,
+  receipts: unknown,
+): Response | null {
+  if (!Array.isArray(receipts)) {
+    return receiptGatewayJsonResponse(
+      { error: 'receipts must be an array.' },
+      { status: 400 },
+    );
+  }
+  const invalid = receipts.find(
+    (receipt) =>
+      !receipt ||
+      typeof receipt !== 'object' ||
+      (receipt as { runId?: unknown }).runId !== runId,
+  );
+  return invalid
+    ? receiptGatewayJsonResponse(
+        { error: 'Receipt runId does not match executor token scope.' },
+        { status: 403 },
+      )
+    : null;
+}
+
+function receiptKey(value: unknown): string | Response {
+  const key = typeof value === 'string' ? value.trim() : '';
+  return (
+    key ||
+    receiptGatewayJsonResponse(
+      { error: 'Receipt key is required.' },
+      { status: 400 },
+    )
+  );
+}
+
+function receiptKeys(value: unknown): Array<string> | Response {
+  if (!Array.isArray(value)) {
+    return receiptGatewayJsonResponse(
+      { error: 'keys must be an array.' },
+      { status: 400 },
+    );
+  }
+  const keys = value.map((key) => (typeof key === 'string' ? key.trim() : ''));
+  if (keys.some((key) => !key)) {
+    return receiptGatewayJsonResponse(
+      { error: 'All receipt keys must be non-empty strings.' },
+      { status: 400 },
+    );
+  }
+  return keys;
+}
+
+export type ReceiptGatewayDispatcher = (
+  scope: ReceiptGatewayScope,
+  body: unknown,
+  options?: ReceiptGatewayDispatchOptions,
+) => Promise<Response>;
+
+export type ReceiptGatewayDispatchOptions = {
+  /** Legacy response presentation only; never an admission range. */
+  wireFormat?: LegacyRuntimeReceiptWireFormat;
+  validateKeys?: (keys: readonly string[]) => Response | null;
+  onRead?: (input: {
+    action: 'get_runtime_step_receipt' | 'get_runtime_step_receipts';
+    keys: readonly string[];
+    receipts: readonly unknown[];
+  }) => void;
+  onClaim?: (input: {
+    action: 'claim_runtime_step_receipt' | 'claim_runtime_step_receipts';
+    receipts: readonly unknown[];
+    forceRefresh: boolean;
+  }) => void;
+};
+
+async function injectReceiptCompletionFailure(input: {
+  client: RuntimeSchedulerQueryClient;
+  orgId: string;
+  key: string;
+  runId: string;
+  schema?: string;
+}): Promise<void> {
+  const options = { schema: input.schema, orgId: input.orgId };
+  const current = await getPostgresSchedulerReceipt(
+    input.client,
+    { key: input.key },
+    options,
+  );
+  if (current?.status !== 'running' || current.runId !== input.runId) {
+    throw new Error(
+      `Cannot inject receipt completion failure for ${input.key}: expected a running receipt owned by ${input.runId}.`,
+    );
+  }
+  const failed = await failPostgresSchedulerReceipt(
+    input.client,
+    {
+      key: input.key,
+      runId: input.runId,
+      runAttempt: current.leaseOwnerAttempt,
+      leaseId: current.leaseId,
+      error: 'Injected runtime test fault: receipt_complete_write_fail.',
+      failureKind: 'repairable',
+    },
+    options,
+  );
+  if (failed?.status !== 'failed' || failed.failureKind !== 'repairable') {
+    throw new Error(
+      `Failed to install repairable receipt completion fault for ${input.key}.`,
+    );
+  }
+}
+
+export function createReceiptGatewayDispatcher(input: {
+  withClient: <T>(
+    fn: (client: RuntimeSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+  schema?: string;
+  consumeReceiptCompleteFault?: (runId: string) => boolean | Promise<boolean>;
+}): ReceiptGatewayDispatcher {
+  return async (scope, rawBody, dispatchOptions = {}) => {
+    if (!rawBody || typeof rawBody !== 'object') {
+      return receiptGatewayJsonResponse(
+        { error: 'Receipt gateway action body is required.' },
+        { status: 400 },
+      );
+    }
+    const body = rawBody as ReceiptGatewayActionBody;
+    const scopeError = requireReceiptScope(scope, body);
+    if (scopeError) return scopeError;
+    const runId = String(body.runId || scope.runId);
+    const wireFormat = dispatchOptions.wireFormat ?? 3;
+    // Org scope is proven by the executor token at the transport (production:
+    // apps/runtime-receipt-gateway/index.ts passes auth.orgId here). The
+    // (org_id, k) composite-key receipt functions require it, so thread it
+    // through identically for the test/PGlite dispatch to match production.
+    const options = { schema: input.schema, orgId: scope.orgId };
+
+    return await input.withClient(async (client) => {
+      switch (body.action) {
+        case 'get_runtime_step_receipt': {
+          const key = receiptKey(body.key);
+          if (key instanceof Response) return key;
+          const keyScopeError = dispatchOptions.validateKeys?.([key]);
+          if (keyScopeError) return keyScopeError;
+          const receipt = await getPostgresSchedulerReceipt(
+            client,
+            { key },
+            options,
+          );
+          dispatchOptions.onRead?.({
+            action: body.action,
+            keys: [key],
+            receipts: [receipt],
+          });
+          return receiptGatewayJsonResponse(receipt);
+        }
+        case 'get_runtime_step_receipts': {
+          const keys = receiptKeys(body.keys);
+          if (keys instanceof Response) return keys;
+          const keyScopeError = dispatchOptions.validateKeys?.(keys);
+          if (keyScopeError) return keyScopeError;
+          const receipts = await getPostgresSchedulerReceipts(
+            client,
+            { keys },
+            options,
+          );
+          dispatchOptions.onRead?.({
+            action: body.action,
+            keys,
+            receipts,
+          });
+          return receiptGatewayJsonResponse(receipts);
+        }
+        case 'claim_runtime_step_receipt': {
+          const key = receiptKey(body.key);
+          if (key instanceof Response) return key;
+          const keyScopeError = dispatchOptions.validateKeys?.([key]);
+          if (keyScopeError) return keyScopeError;
+          const receipt = await claimPostgresSchedulerReceipt(
+            client,
+            {
+              key,
+              ...(typeof body.leaseId === 'string'
+                ? { leaseId: body.leaseId }
+                : {}),
+              runId,
+              runAttempt:
+                typeof body.runAttempt === 'number' ? body.runAttempt : null,
+              leaseAware: body.leaseAware === true,
+              reclaimRunning: body.reclaimRunning === true,
+              forceRefresh: body.forceRefresh === true,
+              forceFailedRefresh: body.forceFailedRefresh === true,
+            },
+            options,
+          );
+          dispatchOptions.onClaim?.({
+            action: body.action,
+            receipts: [receipt],
+            forceRefresh: body.forceRefresh === true,
+          });
+          return receiptGatewayJsonResponse(receipt);
+        }
+        case 'claim_runtime_step_receipts': {
+          const keys = receiptKeys(body.keys);
+          if (keys instanceof Response) return keys;
+          const keyScopeError = dispatchOptions.validateKeys?.(keys);
+          if (keyScopeError) return keyScopeError;
+          const receipts = await claimPostgresSchedulerReceipts(
+            client,
+            {
+              keys,
+              ...(Array.isArray(body.leaseIds)
+                ? { leaseIds: body.leaseIds as string[] }
+                : {}),
+              runId,
+              runAttempt:
+                typeof body.runAttempt === 'number' ? body.runAttempt : null,
+              leaseAware: body.leaseAware === true,
+              reclaimRunning: body.reclaimRunning === true,
+              forceRefresh: body.forceRefresh === true,
+              forceFailedRefresh: body.forceFailedRefresh === true,
+            },
+            options,
+          );
+          dispatchOptions.onClaim?.({
+            action: body.action,
+            receipts,
+            forceRefresh: body.forceRefresh === true,
+          });
+          return receiptGatewayJsonResponse(receipts);
+        }
+        case 'mark_runtime_step_receipts_running': {
+          const sameRunError = requireSameRunReceipts(runId, body.receipts);
+          if (sameRunError) return sameRunError;
+          return receiptGatewayJsonResponse(
+            await markPostgresSchedulerReceiptsRunning(
+              client,
+              { receipts: body.receipts as RuntimeReceiptLease[] },
+              options,
+            ),
+          );
+        }
+        case 'mark_runtime_step_receipts_queued': {
+          const sameRunError = requireSameRunReceipts(runId, body.receipts);
+          if (sameRunError) return sameRunError;
+          return receiptGatewayJsonResponse(
+            await markPostgresSchedulerReceiptsQueued(
+              client,
+              { receipts: body.receipts as RuntimeReceiptLease[] },
+              options,
+            ),
+          );
+        }
+        case 'complete_runtime_step_receipt': {
+          const key = receiptKey(body.key);
+          if (key instanceof Response) return key;
+          if (await input.consumeReceiptCompleteFault?.(runId)) {
+            await injectReceiptCompletionFailure({
+              client,
+              orgId: scope.orgId,
+              key,
+              runId,
+              schema: input.schema,
+            });
+            return receiptGatewayJsonResponse(
+              {
+                error:
+                  'Injected runtime test fault: receipt_complete_write_fail.',
+                code: 'runtime_test_fault',
+              },
+              { status: 409 },
+            );
+          }
+          return receiptGatewayJsonResponse(
+            runtimeReceiptCompletionResponse(
+              wireFormat,
+              await completePostgresSchedulerReceipt(
+                client,
+                {
+                  key,
+                  runId,
+                  runAttempt:
+                    typeof body.runAttempt === 'number'
+                      ? body.runAttempt
+                      : null,
+                  leaseId:
+                    typeof body.leaseId === 'string' ? body.leaseId : null,
+                  output: sanitizeRuntimeReceiptPayload(body.output ?? null),
+                  returnOutput: wireFormat < 3,
+                },
+                options,
+              ),
+            ),
+          );
+        }
+        case 'complete_runtime_step_receipts': {
+          const sameRunError = requireSameRunReceipts(runId, body.receipts);
+          if (sameRunError) return sameRunError;
+          const keys = receiptKeys(
+            (body.receipts as Array<Record<string, unknown>>).map(
+              (receipt) => receipt.key,
+            ),
+          );
+          if (keys instanceof Response) return keys;
+          const completions = body.receipts.map((receipt, index) => ({
+            key: keys[index]!,
+            runId,
+            runAttempt:
+              typeof receipt.runAttempt === 'number'
+                ? receipt.runAttempt
+                : null,
+            leaseId:
+              typeof receipt.leaseId === 'string' ? receipt.leaseId : null,
+            output: sanitizeRuntimeReceiptPayload(receipt.output ?? null),
+          }));
+          if (await input.consumeReceiptCompleteFault?.(runId)) {
+            const [skippedReceipt, ...siblingReceipts] = completions;
+            if (siblingReceipts.length > 0) {
+              await completePostgresSchedulerReceipts(
+                client,
+                {
+                  receipts: siblingReceipts,
+                  returnOutput: wireFormat < 3,
+                },
+                options,
+              );
+            }
+            if (skippedReceipt) {
+              await injectReceiptCompletionFailure({
+                client,
+                orgId: scope.orgId,
+                key: skippedReceipt.key,
+                runId,
+                schema: input.schema,
+              });
+            }
+            return receiptGatewayJsonResponse(
+              {
+                error:
+                  'Injected runtime test fault: receipt_complete_write_fail.',
+                code: 'runtime_test_fault',
+              },
+              { status: 409 },
+            );
+          }
+          return receiptGatewayJsonResponse(
+            (
+              await completePostgresSchedulerReceipts(
+                client,
+                {
+                  receipts: completions,
+                  returnOutput: wireFormat < 3,
+                },
+                options,
+              )
+            ).map((receipt) =>
+              runtimeReceiptCompletionResponse(wireFormat, receipt),
+            ),
+          );
+        }
+        case 'release_runtime_step_receipt': {
+          const key = receiptKey(body.key);
+          if (key instanceof Response) return key;
+          return receiptGatewayJsonResponse(
+            await releasePostgresSchedulerReceipt(
+              client,
+              {
+                key,
+                runId,
+                runAttempt:
+                  typeof body.runAttempt === 'number' ? body.runAttempt : null,
+                leaseId: typeof body.leaseId === 'string' ? body.leaseId : null,
+              },
+              options,
+            ),
+          );
+        }
+        case 'fail_runtime_step_receipt': {
+          const key = receiptKey(body.key);
+          if (key instanceof Response) return key;
+          const errorPayload = normalizeToolExecutionFailure(body.errorPayload);
+          return receiptGatewayJsonResponse(
+            await failPostgresSchedulerReceipt(
+              client,
+              {
+                key,
+                runId,
+                runAttempt:
+                  typeof body.runAttempt === 'number' ? body.runAttempt : null,
+                leaseId: typeof body.leaseId === 'string' ? body.leaseId : null,
+                error: redactLedgerString(body.error) ?? '',
+                ...(errorPayload ? { errorPayload } : {}),
+                failureKind:
+                  body.failureKind === 'repairable' ? 'repairable' : 'terminal',
+              },
+              options,
+            ),
+          );
+        }
+        case 'fail_runtime_step_receipts': {
+          const sameRunError = requireSameRunReceipts(runId, body.receipts);
+          if (sameRunError) return sameRunError;
+          const keys = receiptKeys(
+            (body.receipts as Array<Record<string, unknown>>).map(
+              (receipt) => receipt.key,
+            ),
+          );
+          if (keys instanceof Response) return keys;
+          return receiptGatewayJsonResponse(
+            await failPostgresSchedulerReceipts(
+              client,
+              {
+                receipts: (body.receipts as RuntimeReceiptFailure[]).map(
+                  (receipt, index) => {
+                    const errorPayload = normalizeToolExecutionFailure(
+                      receipt.errorPayload,
+                    );
+                    return {
+                      key: keys[index]!,
+                      runId,
+                      runAttempt:
+                        typeof receipt.runAttempt === 'number'
+                          ? receipt.runAttempt
+                          : null,
+                      leaseId:
+                        typeof receipt.leaseId === 'string'
+                          ? receipt.leaseId
+                          : null,
+                      error: redactLedgerString(receipt.error) ?? '',
+                      ...(errorPayload ? { errorPayload } : {}),
+                      failureKind:
+                        receipt.failureKind === 'repairable'
+                          ? 'repairable'
+                          : 'terminal',
+                    };
+                  },
+                ),
+              },
+              options,
+            ),
+          );
+        }
+        case 'heartbeat_runtime_step_receipts': {
+          const keys = receiptKeys(body.keys);
+          if (keys instanceof Response) return keys;
+          return receiptGatewayJsonResponse(
+            await heartbeatPostgresSchedulerReceipts(
+              client,
+              {
+                keys,
+                runId,
+                runAttempt:
+                  typeof body.runAttempt === 'number' ? body.runAttempt : null,
+                ...(Array.isArray(body.leaseIds)
+                  ? {
+                      leaseIds: body.leaseIds.map((leaseId) =>
+                        String(leaseId ?? ''),
+                      ),
+                    }
+                  : { leaseId: String(body.leaseId ?? '') }),
+              },
+              options,
+            ),
+          );
+        }
+        case 'skip_runtime_step_receipt': {
+          const key = receiptKey(body.key);
+          if (key instanceof Response) return key;
+          return receiptGatewayJsonResponse(
+            await skipPostgresSchedulerReceipt(
+              client,
+              {
+                key,
+                runId,
+                runAttempt:
+                  typeof body.runAttempt === 'number' ? body.runAttempt : null,
+                leaseId: typeof body.leaseId === 'string' ? body.leaseId : null,
+                output: sanitizeRuntimeReceiptPayload(body.output ?? null),
+              },
+              options,
+            ),
+          );
+        }
+        case 'observe_docflow_node': {
+          // ADR 0016 rule 2: durable per-node-invocation observation on the
+          // same wire receipts use. The play body fires this fire-and-forget,
+          // so the gateway persists on a best-effort basis and never gates
+          // execution. Previews are re-normalized (bounded) as defense in depth
+          // — the wrapper already caps them.
+          const nodeId =
+            typeof body.nodeId === 'string' ? body.nodeId.trim() : '';
+          const status = body.status;
+          if (
+            !nodeId ||
+            (status !== 'started' &&
+              status !== 'completed' &&
+              status !== 'failed')
+          ) {
+            return receiptGatewayJsonResponse(
+              { error: 'Observation requires a nodeId and a valid status.' },
+              { status: 400 },
+            );
+          }
+          const attempt =
+            typeof body.attempt === 'number' && Number.isFinite(body.attempt)
+              ? Math.max(0, Math.trunc(body.attempt))
+              : 0;
+          const invocationSeq =
+            typeof body.invocationSeq === 'number' &&
+            Number.isFinite(body.invocationSeq)
+              ? Math.max(0, Math.trunc(body.invocationSeq))
+              : 0;
+          const io = normalizePlayDocflowNodeIoState({
+            attempt,
+            ...(typeof body.invocationId === 'string'
+              ? { invocationId: body.invocationId }
+              : {}),
+            inputs: body.inputs,
+            outputs: body.outputs,
+            inputsTruncated: body.inputsTruncated === true,
+            outputsTruncated: body.outputsTruncated === true,
+            ...(typeof body.error === 'string' ? { error: body.error } : {}),
+          });
+          await upsertPostgresSchedulerObservation(
+            client,
+            {
+              runId,
+              attempt,
+              nodeId,
+              invocationSeq,
+              invocationId:
+                typeof body.invocationId === 'string'
+                  ? body.invocationId
+                  : null,
+              status: status as DocflowObservationStatus,
+              inputs: io?.inputs ?? null,
+              outputs: io?.outputs ?? null,
+              inputsTruncated: io?.inputsTruncated === true,
+              outputsTruncated: io?.outputsTruncated === true,
+              error: io?.error ?? null,
+              at:
+                typeof body.at === 'number' && Number.isFinite(body.at)
+                  ? body.at
+                  : Date.now(),
+            },
+            options,
+          );
+          return receiptGatewayJsonResponse({ ok: true });
+        }
+        default:
+          return receiptGatewayJsonResponse(
+            {
+              error: 'Unsupported receipt gateway action.',
+              action: body.action,
+              playName: scope.playName,
+            },
+            { status: 400 },
+          );
+      }
+    });
+  };
+}