deepline 0.1.0 → 0.1.2

package/dist/repo/shared_libs/play-runtime/runtime-api.ts

@@ -0,0 +1,1873 @@
+import { createHash } from 'node:crypto';
+import {
+  createRuntimePool,
+  type RuntimePool,
+  type RuntimePoolClient,
+} from './runtime-pg-driver';
+import { createDeferredPlayDataset, type PlayDataset } from '../plays/dataset';
+import type {
+  PlayStaticPipeline,
+  PlaySheetContract,
+} from './static-pipeline-types';
+import type {
+  CreateDbSessionResponse,
+  DbLogicalTable,
+  DbSessionLimits,
+  DbSessionOperation,
+  PreloadedRuntimeDbSession,
+  RowsWriteResponse,
+} from './db-session';
+import {
+  derivePlayRowIdentity,
+  normalizePlayNameForSheet,
+  normalizeTableNamespace,
+} from '../plays/row-identity';
+import {
+  createDbSessionResponseSchema,
+  rowsWriteResponseSchema,
+} from './db-session';
+import {
+  dbSessionPostgresUrlAad,
+  decryptDbSessionPostgresUrl,
+  decryptDbSessionPostgresUrlWithPrivateKey,
+  generateDbSessionPostgresUrlDecryptionKey,
+  type PostgresUrlDecryptionKey,
+} from './db-session-crypto';
+import { createRuntimeDatasetId } from './dataset-id';
+
+type RuntimeApiContext = {
+  baseUrl?: string | null;
+  executorToken?: string | null;
+  orgId?: string | null;
+  playName?: string | null;
+  userEmail?: string | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
+  vercelProtectionBypassToken?: string | null;
+  disablePostgresPoolCache?: boolean | null;
+  postgresSessionUnwrapKey?: string | null;
+};
+
+type DbSessionCacheEntry = {
+  session: CreateDbSessionResponse;
+};
+
+const dbSessionCache = new Map<string, DbSessionCacheEntry>();
+const postgresPools = new Map<string, RuntimePool>();
+const DIRECT_POSTGRES_BATCH_SIZE = 5_000;
+const APPEND_KEY_SUFFIX_LENGTH = 12;
+const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
+const RUNTIME_DB_SESSION_RENEWAL_WINDOW_MS = 60_000;
+const RUNTIME_API_MAX_ATTEMPTS = 5;
+const RUNTIME_API_DEFAULT_RETRY_AFTER_MS = 2_000;
+const RUNTIME_API_REQUEST_TIMEOUT_MS = 30_000;
+const RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS = 4;
+const RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
+const RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS = 4;
+const RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
+
+export type ResolvedRuntimePlay = {
+  playId: string;
+  sourceCode?: string | null;
+  artifact?: unknown;
+  codeFormat?: 'function' | 'cjs_module' | 'esm_module';
+  contractSnapshot?: Record<string, unknown> | null;
+};
+
+export type PrepareRuntimeSheetResult = {
+  inserted: number;
+  skipped: number;
+  pendingRows: Record<string, unknown>[];
+  completedRows: Record<string, unknown>[];
+  tableNamespace: string;
+};
+
+type RuntimeApiActionRequest =
+  | {
+      action: 'resolve_play';
+      playRef: string;
+    }
+  | {
+      action: 'ensure_sheet';
+      playName: string;
+      tableNamespace: string;
+      sheetContract?: PlaySheetContract | null;
+      userEmail?: string | null;
+    }
+  | {
+      action: 'start_sheet_dataset';
+      playName: string;
+      tableNamespace: string;
+      sheetContract: PlaySheetContract;
+      rows: Record<string, unknown>[];
+      runId: string;
+      userEmail?: string | null;
+    }
+  | {
+      action: 'create_db_session';
+      playName: string;
+      target: {
+        tableNamespace: string;
+        logicalTable: 'sheet_rows' | 'dataset_rows' | 'play_output';
+      };
+      operations: DbSessionOperation[];
+      limits?: {
+        maxRows?: number;
+        maxBytes?: number;
+        maxRequests?: number;
+      };
+      sheetContract?: PlaySheetContract | null;
+      ttlSeconds?: number;
+      userEmail?: string | null;
+      postgresUrlEncryption?: {
+        alg: 'RSA-OAEP-256+A256GCM';
+        publicKeyJwk: JsonWebKey;
+      } | null;
+    };
+
+type RuntimeApiRowRecord = {
+  key: string;
+  data: Record<string, unknown>;
+  cellMetaPatch?: Record<string, unknown>;
+  inputIndex?: number | null;
+};
+
+type RuntimePostgresSession = CreateDbSessionResponse & {
+  postgresUrl: string;
+  postgres: NonNullable<CreateDbSessionResponse['postgres']>;
+};
+
+function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
+  const baseUrl = context.baseUrl?.trim();
+  if (!baseUrl) {
+    throw new Error('Runner runtime API requires a baseUrl.');
+  }
+  return `${baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`;
+}
+
+function resolveRuntimeApiHeaders(
+  context: RuntimeApiContext,
+): Record<string, string> {
+  const token = context.executorToken?.trim();
+  if (!token) {
+    throw new Error('Runner runtime API requires an executorToken.');
+  }
+  return {
+    'content-type': 'application/json',
+    authorization: `Bearer ${token}`,
+    ...(context.vercelProtectionBypassToken
+      ? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
+      : {}),
+  };
+}
+
+function normalizeRuntimeUserEmail(value: string | null | undefined): string | null {
+  const email = value?.trim();
+  return email ? email : null;
+}
+
+async function postRuntimeApi<TResponse>(
+  context: RuntimeApiContext,
+  body: RuntimeApiActionRequest,
+): Promise<TResponse> {
+  const url = resolveRuntimeApiUrl(context);
+  for (let attempt = 1; attempt <= RUNTIME_API_MAX_ATTEMPTS; attempt += 1) {
+    let response: Response;
+    const abortController = new AbortController();
+    const timeout = setTimeout(
+      () => abortController.abort(),
+      RUNTIME_API_REQUEST_TIMEOUT_MS,
+    );
+    try {
+      response = await fetch(url, {
+        method: 'POST',
+        headers: resolveRuntimeApiHeaders(context),
+        body: JSON.stringify(body),
+        signal: abortController.signal,
+      });
+    } catch (error) {
+      if (attempt < RUNTIME_API_MAX_ATTEMPTS) {
+        await new Promise((resolve) =>
+          setTimeout(resolve, RUNTIME_API_DEFAULT_RETRY_AFTER_MS),
+        );
+        continue;
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(
+        `Runtime API request to ${url} failed before receiving a response: ${message}`,
+      );
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    const parsed = (await response.json().catch(() => null)) as Record<
+      string,
+      unknown
+    > | null;
+    if (response.ok) {
+      return parsed as TResponse;
+    }
+
+    const retryAfterMs =
+      typeof parsed?.retry_after_ms === 'number' &&
+      Number.isFinite(parsed.retry_after_ms)
+        ? parsed.retry_after_ms
+        : RUNTIME_API_DEFAULT_RETRY_AFTER_MS;
+    const shouldRetryRuntimeResponse =
+      (response.status === 503 &&
+        parsed?.code === 'ingestion_plane_not_ready') ||
+      (body.action === 'create_db_session' &&
+        (response.status === 502 ||
+          response.status === 503 ||
+          response.status === 504));
+    if (shouldRetryRuntimeResponse && attempt < RUNTIME_API_MAX_ATTEMPTS) {
+      await new Promise((resolve) => setTimeout(resolve, retryAfterMs));
+      continue;
+    }
+
+    const errorMessage =
+      typeof parsed?.error === 'string'
+        ? parsed.error
+        : `Runtime API request failed with status ${response.status}.`;
+    const details =
+      typeof parsed?.details === 'string'
+        ? parsed.details
+        : typeof parsed?.debug_error === 'string'
+          ? parsed.debug_error
+          : null;
+    throw new Error(
+      details
+        ? `${errorMessage} (status ${response.status}): ${details}`
+        : `${errorMessage} (status ${response.status}).`,
+    );
+  }
+
+  throw new Error('Runtime API request failed after retries.');
+}
+
+function getDbSessionCacheKey(input: {
+  baseUrl?: string | null;
+  executorToken?: string | null;
+  playName: string;
+  tableNamespace: string;
+  logicalTable: DbLogicalTable;
+  operations: DbSessionOperation[];
+  limits?: DbSessionLimits;
+  sheetContract?: PlaySheetContract | null;
+  userEmail?: string | null;
+}): string {
+  // Worker processes are long-lived. Include a hash of the executor token plus
+  // the full requested access shape so a pooled runtime cannot reuse a scoped
+  // Postgres URL across orgs, runs, logical tables, or privilege sets.
+  const tokenHash = createHash('sha256')
+    .update(input.executorToken?.trim() ?? '')
+    .digest('hex')
+    .slice(0, 24);
+  return [
+    input.baseUrl?.trim() ?? '',
+    tokenHash,
+    input.playName,
+    input.tableNamespace,
+    input.logicalTable,
+    [...input.operations].sort().join(','),
+    JSON.stringify(input.limits ?? {}),
+    JSON.stringify(input.sheetContract ?? null),
+    input.userEmail?.trim() ?? '',
+  ].join('::');
+}
+
+function sameOperations(
+  left: readonly DbSessionOperation[],
+  right: readonly DbSessionOperation[],
+): boolean {
+  return [...left].sort().join(',') === [...right].sort().join(',');
+}
+
+function limitsSatisfyRequest(
+  candidate: DbSessionLimits | undefined,
+  requested: DbSessionLimits | undefined,
+): boolean {
+  const candidateLimits = candidate ?? {};
+  const requestedLimits = requested ?? {};
+  for (const key of ['maxRows', 'maxBytes', 'maxRequests'] as const) {
+    const requestedValue = requestedLimits[key];
+    if (requestedValue === undefined) {
+      continue;
+    }
+    const candidateValue = candidateLimits[key];
+    // A preloaded session with no advisory limit means the app intentionally
+    // minted a table-scoped grant for the whole run. The actual database
+    // boundary is still the temporary role's table/operation privileges.
+    if (candidateValue === undefined) {
+      continue;
+    }
+    if (candidateValue < requestedValue) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function sessionHasRenewalWindow(session: CreateDbSessionResponse): boolean {
+  const expiresAtMs = Date.parse(session.expiresAt);
+  return (
+    Number.isFinite(expiresAtMs) &&
+    expiresAtMs - Date.now() > RUNTIME_DB_SESSION_RENEWAL_WINDOW_MS
+  );
+}
+
+function preloadedSessionMatchesRequest(
+  context: RuntimeApiContext & { playName: string },
+  input: {
+    tableNamespace: string;
+    logicalTable: DbLogicalTable;
+    operations: DbSessionOperation[];
+    limits?: DbSessionLimits;
+  },
+  preloaded: PreloadedRuntimeDbSession,
+): boolean {
+  const session = preloaded.session;
+  const requestedTableNamespace = normalizeTableNamespace(input.tableNamespace);
+  return (
+    sessionHasRenewalWindow(session) &&
+    (!context.orgId || session.target.orgId === context.orgId) &&
+    session.playName === context.playName &&
+    normalizeTableNamespace(preloaded.tableNamespace) ===
+      requestedTableNamespace &&
+    normalizeTableNamespace(session.target.tableNamespace) ===
+      requestedTableNamespace &&
+    preloaded.logicalTable === input.logicalTable &&
+    session.target.logicalTable === input.logicalTable &&
+    sameOperations(preloaded.operations, input.operations) &&
+    sameOperations(session.operations, input.operations) &&
+    limitsSatisfyRequest(preloaded.limits, input.limits) &&
+    limitsSatisfyRequest(session.limits, input.limits)
+  );
+}
+
+function findPreloadedRuntimeDbSession(
+  context: RuntimeApiContext & { playName: string },
+  input: {
+    tableNamespace: string;
+    logicalTable: DbLogicalTable;
+    operations: DbSessionOperation[];
+    limits?: DbSessionLimits;
+  },
+): CreateDbSessionResponse | null {
+  for (const preloaded of context.preloadedDbSessions ?? []) {
+    const parsed = createDbSessionResponseSchema.safeParse(preloaded.session);
+    if (!parsed.success) {
+      continue;
+    }
+    const candidate = {
+      ...preloaded,
+      session: parsed.data,
+    };
+    if (preloadedSessionMatchesRequest(context, input, candidate)) {
+      return candidate.session;
+    }
+  }
+  return null;
+}
+
+async function unwrapRuntimeDbSession(
+  context: RuntimeApiContext,
+  session: CreateDbSessionResponse,
+  decryptionKey?: PostgresUrlDecryptionKey | null,
+): Promise<CreateDbSessionResponse> {
+  if (session.postgresUrl) {
+    return session;
+  }
+  if (!session.encryptedPostgresUrl) {
+    return session;
+  }
+  if (session.encryptedPostgresUrl.alg === 'RSA-OAEP-256+A256GCM') {
+    if (!decryptionKey) {
+      throw new Error(
+        'Runtime DB session response used public-key encryption, but no private key was retained for unwrap.',
+      );
+    }
+    const { encryptedPostgresUrl: _encryptedPostgresUrl, ...sessionWithoutUrl } =
+      session;
+    void _encryptedPostgresUrl;
+    return {
+      ...sessionWithoutUrl,
+      postgresUrl: await decryptDbSessionPostgresUrlWithPrivateKey({
+        encrypted: session.encryptedPostgresUrl,
+        privateKey: decryptionKey.privateKey,
+        aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
+      }),
+    };
+  }
+  const unwrapKey = context.postgresSessionUnwrapKey?.trim();
+  if (!unwrapKey) {
+    throw new Error(
+      'Runtime DB session response is encrypted, but no harness unwrap key was provided.',
+    );
+  }
+  const { encryptedPostgresUrl: _encryptedPostgresUrl, ...sessionWithoutUrl } =
+    session;
+  void _encryptedPostgresUrl;
+  return {
+    ...sessionWithoutUrl,
+    postgresUrl: await decryptDbSessionPostgresUrl({
+      encrypted: session.encryptedPostgresUrl,
+      secret: unwrapKey,
+      aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
+    }),
+  };
+}
+
+async function getRuntimeDbSession(
+  context: RuntimeApiContext & { playName: string },
+  input: {
+    tableNamespace: string;
+    logicalTable: DbLogicalTable;
+    operations: DbSessionOperation[];
+    limits?: DbSessionLimits;
+    sheetContract?: PlaySheetContract | null;
+  },
+): Promise<CreateDbSessionResponse> {
+  const userEmail = normalizeRuntimeUserEmail(context.userEmail);
+  const cacheKey = getDbSessionCacheKey({
+    baseUrl: context.baseUrl,
+    executorToken: context.executorToken,
+    playName: context.playName,
+    tableNamespace: input.tableNamespace,
+    logicalTable: input.logicalTable,
+    operations: input.operations,
+    limits: input.limits,
+    sheetContract: input.sheetContract,
+    userEmail,
+  });
+  const cached = dbSessionCache.get(cacheKey)?.session;
+  if (cached) {
+    if (sessionHasRenewalWindow(cached)) {
+      return cached;
+    }
+    await deleteRuntimeDbSessionCacheEntry(cacheKey, cached);
+  }
+
+  const preloaded = findPreloadedRuntimeDbSession(context, input);
+  if (preloaded) {
+    const unwrappedPreloaded = await unwrapRuntimeDbSession(context, preloaded);
+    dbSessionCache.set(cacheKey, { session: unwrappedPreloaded });
+    return unwrappedPreloaded;
+  }
+
+  const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
+  const response = await unwrapRuntimeDbSession(
+    context,
+    createDbSessionResponseSchema.parse(
+      await postRuntimeApi<CreateDbSessionResponse>(context, {
+        action: 'create_db_session',
+        playName: context.playName,
+        target: {
+          tableNamespace: input.tableNamespace,
+          logicalTable: input.logicalTable,
+        },
+        operations: input.operations,
+        limits: input.limits,
+        sheetContract: input.sheetContract ?? null,
+        ttlSeconds: RUNTIME_DB_SESSION_TTL_SECONDS,
+        userEmail,
+        postgresUrlEncryption: decryptionKey.request,
+      }),
+    ),
+    decryptionKey,
+  );
+  dbSessionCache.set(cacheKey, { session: response });
+  return response;
+}
+
+async function deleteRuntimeDbSessionCacheEntry(
+  cacheKey: string,
+  session: CreateDbSessionResponse,
+): Promise<void> {
+  dbSessionCache.delete(cacheKey);
+  if (session.postgresUrl) {
+    const pool = postgresPools.get(session.postgresUrl);
+    postgresPools.delete(session.postgresUrl);
+    await pool?.end().catch(() => {});
+  }
+}
+
+function requireRuntimePostgresSession(
+  session: CreateDbSessionResponse,
+): RuntimePostgresSession {
+  if (!session.postgresUrl || !session.postgres) {
+    throw new Error(
+      'Runtime DB session did not include a scoped Postgres URL. Direct Postgres sheet IO is required.',
+    );
+  }
+  return session as RuntimePostgresSession;
+}
+
+export async function prewarmRuntimePostgresSessions(
+  context: RuntimeApiContext,
+): Promise<void> {
+  const sessions = context.preloadedDbSessions ?? [];
+  if (sessions.length === 0) {
+    return;
+  }
+  for (const preloaded of sessions) {
+    const parsed = createDbSessionResponseSchema.parse(preloaded.session);
+    const session = requireRuntimePostgresSession(parsed);
+    await prewarmRuntimePostgresSession(session);
+  }
+}
+
+async function prewarmRuntimePostgresSession(
+  session: RuntimePostgresSession,
+): Promise<void> {
+  for (
+    let attempt = 1;
+    attempt <= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS;
+    attempt += 1
+  ) {
+    try {
+      await withRuntimePostgres(session, async (client) => {
+        await client.query('SELECT 1');
+      });
+      return;
+    } catch (error) {
+      if (
+        attempt >= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS ||
+        !isTransientRuntimePostgresConnectionError(error)
+      ) {
+        const pool = postgresPools.get(session.postgresUrl);
+        postgresPools.delete(session.postgresUrl);
+        if (pool) {
+          await Promise.resolve(pool.end()).catch(() => {});
+        }
+        throw error;
+      }
+      const pool = postgresPools.get(session.postgresUrl);
+      postgresPools.delete(session.postgresUrl);
+      if (pool) {
+        await Promise.resolve(pool.end()).catch(() => {});
+      }
+      await sleep(
+        RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[attempt - 1] ??
+          RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[
+            RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS.length - 1
+          ],
+      );
+    }
+  }
+}
+
+function isTransientRuntimePostgresConnectionError(error: unknown): boolean {
+  if (!error || typeof error !== 'object') {
+    return false;
+  }
+  const code = 'code' in error ? String(error.code) : '';
+  if (
+    code === 'ECONNRESET' ||
+    code === 'ETIMEDOUT' ||
+    code === 'ECONNREFUSED' ||
+    code === '57P01'
+  ) {
+    return true;
+  }
+  const message = 'message' in error ? String(error.message) : '';
+  return /connection (terminated|timeout|timed out|closed|reset)/i.test(
+    message,
+  );
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function getPostgresPool(postgresUrl: string, cachePool = true): RuntimePool {
+  if (!cachePool) {
+    return createRuntimePool({
+      connectionString: postgresUrl,
+      maxConnections: 1,
+      idleTimeoutMs: 0,
+      connectTimeoutMs: 10_000,
+    });
+  }
+  const existing = postgresPools.get(postgresUrl);
+  if (existing) {
+    return existing;
+  }
+  const pool = createRuntimePool({
+    connectionString: postgresUrl,
+    maxConnections: 4,
+    idleTimeoutMs: 15_000,
+    connectTimeoutMs: 10_000,
+  });
+  postgresPools.set(postgresUrl, pool);
+  return pool;
+}
+
+function canReuseRuntimePostgresPoolsAcrossRequests(): boolean {
+  const navigatorLike = globalThis as typeof globalThis & {
+    navigator?: { userAgent?: string };
+  };
+  const userAgent = navigatorLike.navigator?.userAgent ?? '';
+  return !/\bCloudflare-Workers\b/i.test(userAgent);
+}
+
+async function withRuntimePostgres<T>(
+  session: RuntimePostgresSession,
+  fn: (client: RuntimePoolClient) => Promise<T>,
+  options: { cachePool?: boolean } = {},
+): Promise<T> {
+  let client: RuntimePoolClient | null = null;
+  let requestLocalPool: RuntimePool | null = null;
+  const cachePool =
+    (options.cachePool ?? true) && canReuseRuntimePostgresPoolsAcrossRequests();
+  for (
+    let attempt = 1;
+    attempt <= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS;
+    attempt += 1
+  ) {
+    try {
+      const pool = getPostgresPool(session.postgresUrl, cachePool);
+      if (!cachePool) {
+        requestLocalPool = pool;
+      }
+      client = await pool.connect();
+      break;
+    } catch (error) {
+      if (cachePool) {
+        const pool = postgresPools.get(session.postgresUrl);
+        postgresPools.delete(session.postgresUrl);
+        if (pool) {
+          await Promise.resolve(pool.end()).catch(() => {});
+        }
+      } else if (requestLocalPool) {
+        await Promise.resolve(requestLocalPool.end()).catch(() => {});
+        requestLocalPool = null;
+      }
+      if (
+        attempt >= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS ||
+        !isTransientRuntimePostgresConnectionError(error)
+      ) {
+        throw error;
+      }
+      await sleep(
+        RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS[attempt - 1] ??
+          RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS[
+            RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS.length - 1
+          ],
+      );
+    }
+  }
+  if (!client) {
+    throw new Error('Runtime Postgres connection was not acquired.');
+  }
+  try {
+    return await fn(client);
+  } finally {
+    client.release();
+    if (requestLocalPool) {
+      await Promise.resolve(requestLocalPool.end()).catch(() => {});
+    }
+  }
+}
+
+function quoteIdentifier(value: string): string {
+  return `"${value.replace(/"/g, '""')}"`;
+}
+
+function quoteLiteral(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`;
+}
+
+function fqRuntimeTable(
+  session: RuntimePostgresSession,
+  table: string,
+): string {
+  return `${quoteIdentifier(session.postgres.schema)}.${quoteIdentifier(table)}`;
+}
+
+function sheetTable(session: RuntimePostgresSession): string {
+  return fqRuntimeTable(session, session.postgres.sheetTable);
+}
+
+function eventTable(session: RuntimePostgresSession): string {
+  return fqRuntimeTable(session, session.postgres.eventTable);
+}
+
+function summaryTable(session: RuntimePostgresSession): string {
+  return fqRuntimeTable(session, session.postgres.summaryTable);
+}
+
+function columnSummaryTable(session: RuntimePostgresSession): string {
+  return fqRuntimeTable(session, session.postgres.columnSummaryTable);
+}
+
+type PhysicalSheetColumnProjection = {
+  sqlName: string;
+  fieldName: string;
+};
+
+function physicalSheetColumnProjections(
+  sheetContract: PlaySheetContract | null | undefined,
+): PhysicalSheetColumnProjection[] {
+  if (!sheetContract) {
+    return [];
+  }
+  const seen = new Set<string>();
+  const columns: PhysicalSheetColumnProjection[] = [];
+  for (const column of sheetContract.columns) {
+    const name = column.sqlName.trim();
+    if (!name || name.startsWith('_') || seen.has(name)) {
+      continue;
+    }
+    seen.add(name);
+    columns.push({
+      sqlName: name,
+      fieldName:
+        column.source === 'input' || column.source === 'mapField'
+          ? column.field?.trim() || column.id.trim() || name
+          : name,
+    });
+  }
+  return columns;
+}
+
+async function ensureRuntimePhysicalSheetColumns(
+  client: RuntimePoolClient,
+  session: RuntimePostgresSession,
+  sheetContract: PlaySheetContract | null | undefined,
+): Promise<void> {
+  const physicalColumns = physicalSheetColumnProjections(sheetContract).map(
+    (column) => column.sqlName,
+  );
+  if (physicalColumns.length === 0) {
+    return;
+  }
+  await client.query(`
+    ALTER TABLE ${sheetTable(session)}
+    ${physicalColumns
+      .map(
+        (column) => `ADD COLUMN IF NOT EXISTS ${quoteIdentifier(column)} jsonb`,
+      )
+      .join(',\n    ')}
+  `);
+}
+
+function physicalSheetColumnNames(
+  sheetContract: PlaySheetContract | null | undefined,
+): string[] {
+  return physicalSheetColumnProjections(sheetContract).map(
+    (column) => column.sqlName,
+  );
+}
+
+function isSystemSheetColumn(columnName: string): boolean {
+  switch (columnName) {
+    case '_key':
+    case '_status':
+    case '_run_id':
+    case '_error':
+    case '_stage':
+    case '_provider':
+    case '_input_index':
+    case '_created_at':
+    case '_updated_at':
+    case '_cell_meta':
+    case 'seq':
+      return true;
+    default:
+      return false;
+  }
+}
+
+function mapRuntimePostgresRow(
+  raw: Record<string, unknown>,
+): RuntimeApiRowRecord {
+  return {
+    key: String(raw._key ?? ''),
+    data: Object.fromEntries(
+      Object.entries(raw).filter(([key]) => !isSystemSheetColumn(key)),
+    ),
+    inputIndex: raw._input_index != null ? Number(raw._input_index) : undefined,
+  };
+}
+
+function mergeRuntimeCompletedRow(input: {
+  inputRow: Record<string, unknown>;
+  completedData: Record<string, unknown>;
+  sheetContract: PlaySheetContract;
+}): Record<string, unknown> {
+  const syntheticNullInputColumns = new Set(
+    input.sheetContract.columns.flatMap((column) => {
+      const field = column.field;
+      if (
+        column.source !== 'input' ||
+        typeof field !== 'string' ||
+        field in input.inputRow ||
+        input.completedData[field] != null
+      ) {
+        return [];
+      }
+      return [field];
+    }),
+  );
+  const cleanedCompletedData = Object.fromEntries(
+    Object.entries(input.completedData).filter(
+      ([key]) => !syntheticNullInputColumns.has(key),
+    ),
+  );
+  return {
+    ...input.inputRow,
+    ...cleanedCompletedData,
+  };
+}
+
+function buildAppendedRowKey(input: {
+  row: Record<string, unknown>;
+  tableNamespace: string;
+  idempotencyKey: string;
+  ordinal: number;
+}): string {
+  const baseKey = derivePlayRowIdentity(input.row, input.tableNamespace);
+  const suffix = createHash('sha1')
+    .update(`${input.idempotencyKey}:${input.ordinal}`)
+    .digest('hex')
+    .slice(0, APPEND_KEY_SUFFIX_LENGTH);
+  return `${baseKey}:append:${suffix}`;
+}
+
+function chunkValues<T>(values: readonly T[], chunkSize: number): T[][] {
+  const chunks: T[][] = [];
+  for (let index = 0; index < values.length; index += chunkSize) {
+    chunks.push(values.slice(index, index + chunkSize));
+  }
+  return chunks;
+}
+
+async function readRuntimeRows(
+  session: RuntimePostgresSession,
+  input: { limit: number; offset: number },
+): Promise<RuntimeApiRowRecord[]> {
+  return await withRuntimePostgres(session, async (client) => {
+    const { rows } = await client.query(
+      `SELECT *
+       FROM ${sheetTable(session)}
+       ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC
+       LIMIT $1 OFFSET $2`,
+      [input.limit, input.offset],
+    );
+    return rows.map(mapRuntimePostgresRow);
+  });
+}
+
+async function readRuntimeRowsPage(
+  session: RuntimePostgresSession,
+  input: {
+    afterInputIndex: number;
+    limit: number;
+    statuses?: string[];
+  },
+): Promise<RuntimeApiRowRecord[]> {
+  return await withRuntimePostgres(session, async (client) => {
+    const rawStatuses = (input.statuses ?? []).map((status) =>
+      status === 'completed'
+        ? 'enriched'
+        : status === 'queued'
+          ? 'pending'
+          : status,
+    );
+    const params: unknown[] = [input.afterInputIndex, input.limit];
+    const conditions = ['_input_index > $1'];
+    if (rawStatuses.length > 0) {
+      params.push(rawStatuses);
+      conditions.push(`_status = ANY($${params.length}::text[])`);
+    }
+    const { rows } = await client.query(
+      `SELECT *
+       FROM ${sheetTable(session)}
+       WHERE ${conditions.join(' AND ')}
+       ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC
+       LIMIT $2`,
+      params,
+    );
+    return rows.map(mapRuntimePostgresRow);
+  });
+}
+
+async function readRuntimeRowsByKeys(
+  client: RuntimePoolClient,
+  session: RuntimePostgresSession,
+  keys: string[],
+): Promise<RuntimeApiRowRecord[]> {
+  if (keys.length === 0) {
+    return [];
+  }
+  const { rows } = await client.query<Record<string, unknown>>(
+    `SELECT *
+     FROM ${sheetTable(session)}
+     WHERE _key = ANY($1::text[])
+     ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC`,
+    [keys],
+  );
+  return rows.map(mapRuntimePostgresRow);
+}
+
+async function readRuntimeSummary(
+  session: RuntimePostgresSession,
+): Promise<{ stats: { total: number } }> {
+  const normalizedPlayName = normalizePlayNameForSheet(session.playName);
+  const normalizedTableNamespace = normalizeTableNamespace(
+    session.target.tableNamespace,
+  );
+  return await withRuntimePostgres(session, async (client) => {
+    const { rows } = await client.query(
+      `SELECT total
+       FROM ${summaryTable(session)}
+       WHERE play_name = $1 AND table_namespace = $2
+       LIMIT 1`,
+      [normalizedPlayName, normalizedTableNamespace],
+    );
+    return { stats: { total: Number(rows[0]?.total ?? 0) } };
+  });
+}
+
+async function writeRuntimeRows(
+  session: RuntimePostgresSession,
+  input: {
+    tableNamespace: string;
+    rows: Record<string, unknown>[];
+    runId: string;
+    idempotencyKey: string;
+    sheetContract: PlaySheetContract;
+    mode: 'append' | 'upsert' | 'replace';
+  },
+): Promise<RowsWriteResponse> {
+  const physicalColumnProjections = physicalSheetColumnProjections(
+    input.sheetContract,
+  );
+  const physicalColumns = physicalColumnProjections.map(
+    (column) => column.sqlName,
+  );
+  const physicalInsertColumnsSql =
+    physicalColumns.length > 0
+      ? `, ${physicalColumns.map(quoteIdentifier).join(', ')}`
+      : '';
+  const physicalInsertValuesSql =
+    physicalColumns.length > 0
+      ? `, ${physicalColumnProjections
+          .map((column) => `payload -> ${quoteLiteral(column.fieldName)}`)
+          .join(', ')}`
+      : '';
+
+  const rowsToWrite = input.rows.filter(
+    (row) => row && typeof row === 'object' && !Array.isArray(row),
+  );
+  if (rowsToWrite.length === 0) {
+    return { disposition: 'completed', writtenRows: 0 };
+  }
+  const normalizedPlayName = normalizePlayNameForSheet(session.playName);
+  const normalizedTableNamespace = normalizeTableNamespace(
+    input.tableNamespace,
+  );
+
+  // Build write entries first so we know chunk count up front. Single-chunk
+  // writes skip the BEGIN/COMMIT pair entirely (the mega-CTE is atomic at
+  // statement level), and the per-chunk mega-CTE folds: starting-index lookup,
+  // sheet insert, event insert, and summary upsert into one round-trip.
+  const rowEntries =
+    input.mode === 'upsert'
+      ? Array.from(
+          rowsToWrite
+            .reduce((uniqueRows, row) => {
+              const key = derivePlayRowIdentity(row, input.tableNamespace);
+              if (key && !uniqueRows.has(key)) {
+                uniqueRows.set(key, row);
+              }
+              return uniqueRows;
+            }, new Map<string, Record<string, unknown>>())
+            .entries(),
+        ).map(([key, row], inputIndex) => ({ key, row, inputIndex }))
+      : rowsToWrite.map((row, index) => ({
+          key: buildAppendedRowKey({
+            row,
+            tableNamespace: input.tableNamespace,
+            idempotencyKey: input.idempotencyKey,
+            ordinal: index,
+          }),
+          row,
+          // For append/replace modes the starting offset is computed in SQL
+          // as `coalesce(max(_input_index), -1) + 1 + (ord - 1)`; we only
+          // pass the per-chunk ordinal here so the SQL can add it to the
+          // dynamic starting index without an extra round-trip.
+          inputIndex: index,
+        }));
+
+  const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
+  const needsTransaction = input.mode === 'replace' || chunks.length > 1;
+
+  return await withRuntimePostgres(session, async (client) => {
+    await ensureRuntimePhysicalSheetColumns(
+      client,
+      session,
+      input.sheetContract,
+    );
+    if (needsTransaction) await client.query('BEGIN');
+    try {
+      if (input.mode === 'replace') {
+        // Collapse 4 cleanup statements into one CTE-shaped query. Postgres
+        // executes data-modifying CTEs against the same snapshot, but each
+        // operates on a distinct table so ordering does not matter.
+        await client.query(
+          `WITH cleared_sheet AS (
+             DELETE FROM ${sheetTable(session)} RETURNING 1
+           ),
+           cleared_events AS (
+             DELETE FROM ${eventTable(session)}
+             WHERE play_name = $1 AND table_namespace = $2
+             RETURNING 1
+           ),
+           reset_summary AS (
+             UPDATE ${summaryTable(session)}
+                SET total = 0, queued = 0, running = 0, completed = 0, failed = 0, _updated_at = now()
+              WHERE play_name = $1 AND table_namespace = $2
+              RETURNING 1
+           ),
+           cleared_col_summary AS (
+             DELETE FROM ${columnSummaryTable(session)}
+             WHERE play_name = $1 AND table_namespace = $2
+             RETURNING 1
+           )
+           SELECT 1`,
+          [normalizedPlayName, normalizedTableNamespace],
+        );
+      }
+
+      // For append/replace: SQL computes _input_index from the live max so
+      // we never need a separate SELECT round-trip. For upsert: the JS
+      // ordinals (0..N) are passed straight through, matching prior shape.
+      const computesStartingIndexInSql =
+        input.mode === 'append' || input.mode === 'replace';
+
+      let writtenRows = 0;
+      for (const chunk of chunks) {
+        const chunkKeys = chunk.map((entry) => entry.key);
+        const chunkPayloads = chunk.map((entry) => JSON.stringify(entry.row));
+        const chunkInputIndexes = chunk.map((entry) => entry.inputIndex);
+
+        const startingIndexCte = computesStartingIndexInSql
+          ? `starting_index AS (
+              SELECT coalesce(max(_input_index), -1)::bigint AS v FROM ${sheetTable(session)}
+            ),`
+          : '';
+        const inputIndexExpr = computesStartingIndexInSql
+          ? `(SELECT v FROM starting_index) + index_values._input_index::bigint + 1`
+          : `index_values._input_index::bigint`;
+        const insertedRowsCte =
+          input.mode === 'upsert'
+            ? `inserted_rows AS (
+                INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
+                SELECT _key, 'pending', $4, _input_index${physicalInsertValuesSql}
+                FROM input_rows
+                ON CONFLICT (_key) DO UPDATE SET
+                  _status = 'pending',
+                  _run_id = EXCLUDED._run_id,
+                  _input_index = EXCLUDED._input_index
+                WHERE ${sheetTable(session)}._status = 'stale'
+                RETURNING _key
+              ),`
+            : `inserted_rows AS (
+                INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
+                SELECT _key, 'pending', $4, _input_index${physicalInsertValuesSql}
+                FROM input_rows
+                RETURNING _key
+              ),`;
+        const eventsWhere =
+          input.mode === 'upsert'
+            ? `WHERE _key = ANY(SELECT _key FROM inserted_rows)`
+            : '';
+
+        const sql = `
+          WITH ${startingIndexCte}
+          input_rows AS (
+            SELECT key_values._key, payload_values.payload,
+                   ${inputIndexExpr} AS _input_index
+            FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
+            JOIN unnest($2::jsonb[]) WITH ORDINALITY AS payload_values(payload, ord)
+              ON payload_values.ord = key_values.ord
+            JOIN unnest($3::bigint[]) WITH ORDINALITY AS index_values(_input_index, ord)
+              ON index_values.ord = key_values.ord
+          ),
+          ${insertedRowsCte}
+          inserted_events AS (
+            INSERT INTO ${eventTable(session)} (play_name, table_namespace, _key, _status, _run_id, _stage, data_patch)
+            SELECT $5, $6, _key, 'pending', $4, 'queued', payload
+            FROM input_rows
+            ${eventsWhere}
+            RETURNING 1
+          ),
+          inserted_count_cte AS (
+            SELECT count(*)::bigint AS c FROM inserted_rows
+          ),
+          summary_upsert AS (
+            INSERT INTO ${summaryTable(session)} (play_name, table_namespace, total, queued, running, completed, failed)
+            SELECT $5, $6, c, c, 0, 0, 0
+              FROM inserted_count_cte
+             WHERE c > 0
+            ON CONFLICT (play_name, table_namespace) DO UPDATE SET
+              total = ${summaryTable(session)}.total + EXCLUDED.total,
+              queued = ${summaryTable(session)}.queued + EXCLUDED.queued,
+              _updated_at = now()
+            RETURNING 1
+          )
+          SELECT c::int AS inserted_count FROM inserted_count_cte
+        `;
+
+        const { rows } = await client.query(sql, [
+          chunkKeys,
+          chunkPayloads,
+          chunkInputIndexes,
+          input.runId,
+          normalizedPlayName,
+          normalizedTableNamespace,
+        ]);
+        writtenRows += Number(rows[0]?.inserted_count ?? 0);
+      }
+
+      if (needsTransaction) await client.query('COMMIT');
+      return { disposition: 'completed', writtenRows };
+    } catch (error) {
+      if (needsTransaction) {
+        await client.query('ROLLBACK').catch(() => {});
+      }
+      throw error;
+    }
+  });
+}
+
+export async function resolveRuntimeReferencedPlay(
+  context: RuntimeApiContext,
+  playRef: string,
+): Promise<ResolvedRuntimePlay | null> {
+  const response = await postRuntimeApi<{ play: ResolvedRuntimePlay | null }>(
+    context,
+    {
+      action: 'resolve_play',
+      playRef,
+    },
+  );
+  return response.play;
+}
+
+export async function ensureRuntimeSheet(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    tableNamespace: string;
+    sheetContract: PlaySheetContract;
+  },
+): Promise<void> {
+  await postRuntimeApi<{ ok: true }>(context, {
+    action: 'ensure_sheet',
+    ...input,
+    userEmail: normalizeRuntimeUserEmail(context.userEmail),
+  });
+}
+
+export async function startRuntimeSheetDataset(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    tableNamespace: string;
+    playInput?: Record<string, unknown> | null;
+    sheetContract: PlaySheetContract;
+    rows: Record<string, unknown>[];
+    runId: string;
+  },
+): Promise<PrepareRuntimeSheetResult> {
+  const playName = context.playName?.trim() || input.playName;
+  if (!playName) {
+    throw new Error('Runtime DB sessions require a playName.');
+  }
+  const session = requireRuntimePostgresSession(
+    await getRuntimeDbSession(
+      {
+        ...context,
+        playName,
+      },
+      {
+        tableNamespace: input.tableNamespace,
+        logicalTable: 'sheet_rows',
+        operations: ['rows.read', 'rows.upsert'],
+        limits: {
+          maxRows: Math.max(input.rows.length, 1),
+        },
+        sheetContract: input.sheetContract,
+      },
+    ),
+  );
+  const uniqueRows = new Map<string, Record<string, unknown>>();
+  for (const row of input.rows) {
+    const { __deeplineRowKey: _rowKey, ...cleanedRow } = row;
+    void _rowKey;
+    const key =
+      typeof row.__deeplineRowKey === 'string'
+        ? row.__deeplineRowKey
+        : derivePlayRowIdentity(cleanedRow, input.tableNamespace);
+    if (key && !uniqueRows.has(key)) {
+      uniqueRows.set(key, cleanedRow);
+    }
+  }
+  const rowEntries = [...uniqueRows.entries()].map(
+    ([key, row], inputIndex) => ({
+      key,
+      row,
+      inputIndex,
+    }),
+  );
+  if (rowEntries.length === 0) {
+    return {
+      inserted: 0,
+      skipped: input.rows.length,
+      pendingRows: [],
+      completedRows: [],
+      tableNamespace: input.tableNamespace,
+    };
+  }
+
+  const physicalColumns = physicalSheetColumnNames(input.sheetContract);
+  const physicalInsertColumnsSql =
+    physicalColumns.length > 0
+      ? `, ${physicalColumns.map(quoteIdentifier).join(', ')}`
+      : '';
+  const physicalInsertValuesSql =
+    physicalColumns.length > 0
+      ? `, ${physicalColumns
+          .map((column) => `payload -> ${quoteLiteral(column)}`)
+          .join(', ')}`
+      : '';
+  const normalizedPlayName = normalizePlayNameForSheet(playName);
+  const normalizedTableNamespace = normalizeTableNamespace(
+    input.tableNamespace,
+  );
+
+  const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
+  const needsTransaction = chunks.length > 1;
+
+  return await withRuntimePostgres(
+    session,
+    async (client) => {
+      await ensureRuntimePhysicalSheetColumns(
+        client,
+        session,
+        input.sheetContract,
+      );
+      if (needsTransaction) await client.query('BEGIN');
+      try {
+        let inserted = 0;
+        const pendingKeys: string[] = [];
+        for (const chunk of chunks) {
+          const chunkKeys = chunk.map((entry) => entry.key);
+          const chunkPayloads = chunk.map((entry) => JSON.stringify(entry.row));
+          const chunkInputIndexes = chunk.map((entry) => entry.inputIndex);
+          // Mega-CTE: row insert + event insert + status-derived pending set +
+          // summary upsert in one round-trip. Saves 2 RTTs per chunk vs. the
+          // prior "INSERT then summary INSERT then summary UPDATE" sequence.
+          const { rows } = await client.query(
+            `
+            WITH input_rows AS (
+              SELECT key_values._key, payload_values.payload, index_values._input_index
+              FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
+              JOIN unnest($2::jsonb[]) WITH ORDINALITY AS payload_values(payload, ord)
+                ON payload_values.ord = key_values.ord
+              JOIN unnest($3::bigint[]) WITH ORDINALITY AS index_values(_input_index, ord)
+                ON index_values.ord = key_values.ord
+            ),
+            existing_rows AS (
+              UPDATE ${sheetTable(session)} AS target
+              SET _input_index = input_rows._input_index
+              FROM input_rows
+              WHERE target._key = input_rows._key
+                AND target._input_index IS DISTINCT FROM input_rows._input_index
+              RETURNING target._key
+            ),
+            inserted_rows AS (
+              INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
+              SELECT _key, 'pending', $4, _input_index${physicalInsertValuesSql}
+              FROM input_rows
+              ON CONFLICT (_key) DO UPDATE SET
+                _status = 'pending',
+                _run_id = EXCLUDED._run_id,
+                _input_index = EXCLUDED._input_index
+              WHERE ${sheetTable(session)}._status = 'stale'
+              RETURNING _key
+            ),
+            inserted_events AS (
+              INSERT INTO ${eventTable(session)} (play_name, table_namespace, _key, _status, _run_id, _stage, data_patch)
+              SELECT $5, $6, _key, 'pending', $4, 'queued', payload
+              FROM input_rows
+              WHERE _key = ANY(SELECT _key FROM inserted_rows)
+              RETURNING 1
+            ),
+            pending_rows AS (
+              SELECT _key
+              FROM inserted_rows
+              UNION
+              SELECT existing._key
+              FROM ${sheetTable(session)} AS existing
+              JOIN input_rows ON input_rows._key = existing._key
+              WHERE existing._status IN ('pending', 'running', 'failed')
+            ),
+            inserted_count_cte AS (
+              SELECT count(*)::bigint AS c FROM inserted_rows
+            ),
+            summary_upsert AS (
+              INSERT INTO ${summaryTable(session)} (play_name, table_namespace, total, queued, running, completed, failed)
+              SELECT $5, $6, c, c, 0, 0, 0
+                FROM inserted_count_cte
+               WHERE c > 0
+              ON CONFLICT (play_name, table_namespace) DO UPDATE SET
+                total = ${summaryTable(session)}.total + EXCLUDED.total,
+                queued = ${summaryTable(session)}.queued + EXCLUDED.queued,
+                _updated_at = now()
+              RETURNING 1
+            )
+            SELECT
+              (SELECT c::int FROM inserted_count_cte) AS inserted_count,
+              coalesce((SELECT array_agg(_key) FROM pending_rows), '{}'::text[]) AS pending_keys,
+              (SELECT count(*)::int FROM existing_rows) AS reordered_count,
+              (SELECT count(*)::int FROM inserted_events) AS event_count
+          `,
+            [
+              chunkKeys,
+              chunkPayloads,
+              chunkInputIndexes,
+              input.runId,
+              normalizedPlayName,
+              normalizedTableNamespace,
+            ],
+          );
+          inserted += Number(rows[0]?.inserted_count ?? 0);
+          if (Array.isArray(rows[0]?.pending_keys)) {
+            pendingKeys.push(...(rows[0]?.pending_keys as string[]));
+          }
+        }
+
+        const pendingKeySet = new Set(pendingKeys);
+        const pendingRows = rowEntries
+          .filter((entry) => pendingKeySet.has(entry.key))
+          .map((entry) => entry.row);
+        const completedKeys = rowEntries
+          .filter((entry) => !pendingKeySet.has(entry.key))
+          .map((entry) => entry.key);
+        const completedRowsByKey = new Map(
+          (await readRuntimeRowsByKeys(client, session, completedKeys)).map(
+            (row) => [row.key, row.data],
+          ),
+        );
+        const completedRows = rowEntries
+          .filter((entry) => !pendingKeySet.has(entry.key))
+          .map((entry) => ({
+            ...mergeRuntimeCompletedRow({
+              inputRow: entry.row,
+              completedData: completedRowsByKey.get(entry.key) ?? {},
+              sheetContract: input.sheetContract,
+            }),
+            __deeplineRowKey: entry.key,
+          }));
+        if (needsTransaction) await client.query('COMMIT');
+        return {
+          inserted,
+          skipped:
+            rowEntries.length -
+            inserted +
+            (input.rows.length - rowEntries.length),
+          pendingRows,
+          completedRows,
+          tableNamespace: input.tableNamespace,
+        };
+      } catch (error) {
+        if (needsTransaction) {
+          await client.query('ROLLBACK').catch(() => {});
+        }
+        throw error;
+      }
+    },
+    { cachePool: !context.disablePostgresPoolCache },
+  );
+}
+
+/**
+ * Mark map rows completed in the per-run scoped Postgres sheet table by key.
+ * Mirrors server-side `store.completeSheetRows` semantics: UPDATE-by-key with
+ * the row's enriched output values written into materialized physical
+ * columns, _status flipped to 'enriched', and a completion event row appended.
+ *
+ * Used by the workers_edge harness's `persistCompletedMapRows` so map
+ * completion writes go directly to Neon (via @neondatabase/serverless) and
+ * skip the per-chunk Vercel hop.
+ */
+export async function completeRuntimeMapRows(
+  context: RuntimeApiContext & { playName: string },
+  input: {
+    tableNamespace: string;
+    sheetContract: PlaySheetContract;
+    rows: RuntimeApiRowRecord[];
+    runId: string;
+  },
+): Promise<{ updated: number }> {
+  if (input.rows.length === 0) {
+    return { updated: 0 };
+  }
+  const session = requireRuntimePostgresSession(
+    await getRuntimeDbSession(
+      { ...context, playName: context.playName },
+      {
+        tableNamespace: input.tableNamespace,
+        logicalTable: 'sheet_rows',
+        operations: ['rows.read', 'rows.upsert'],
+        limits: {
+          maxRows: Math.max(input.rows.length, 1),
+        },
+        sheetContract: input.sheetContract,
+      },
+    ),
+  );
+
+  // Dedupe by key — duplicates within the same chunk are idempotent on the
+  // sheet, but they bloat the events table. Last write wins on payload.
+  const uniqueRows = new Map<string, Record<string, unknown>>();
+  const cellMetaPatchByKey = new Map<string, Record<string, unknown>>();
+  for (const row of input.rows) {
+    if (!row.key) continue;
+    uniqueRows.set(row.key, row.data);
+    if (row.cellMetaPatch) {
+      cellMetaPatchByKey.set(row.key, row.cellMetaPatch);
+    }
+  }
+  if (uniqueRows.size === 0) {
+    return { updated: 0 };
+  }
+
+  const projections = physicalSheetColumnProjections(input.sheetContract);
+  const physicalUpdateSetSql =
+    projections.length > 0
+      ? `,\n        ${projections
+          .map((column) => {
+            const quoted = quoteIdentifier(column.sqlName);
+            const literal = quoteLiteral(column.fieldName);
+            return `${quoted} = CASE
+          WHEN coalesce(updates.data_patch, '{}'::jsonb) ? ${literal}
+            THEN updates.data_patch -> ${literal}
+          ELSE target.${quoted}
+        END`;
+          })
+          .join(',\n        ')}`
+      : '';
+
+  const normalizedPlayName = normalizePlayNameForSheet(session.playName);
+  const normalizedTableNamespace = normalizeTableNamespace(
+    input.tableNamespace,
+  );
+  const completedRows = [...uniqueRows.entries()].map(([key, data]) => ({
+    key,
+    data_patch: data,
+    cell_meta_patch: cellMetaPatchByKey.get(key) ?? {},
+  }));
+  const chunks = chunkValues(completedRows, DIRECT_POSTGRES_BATCH_SIZE);
+  const needsTransaction = chunks.length > 1;
+
+  return await withRuntimePostgres(
+    session,
+    async (client) => {
+      await ensureRuntimePhysicalSheetColumns(
+        client,
+        session,
+        input.sheetContract,
+      );
+      if (needsTransaction) await client.query('BEGIN');
+      let updated = 0;
+      try {
+        for (const chunk of chunks) {
+          const { rows: appliedRows } = await client.query<{ _key: string }>(
+            `WITH updates AS (
+             SELECT * FROM jsonb_to_recordset($1::jsonb) AS incoming(
+               key text,
+               data_patch jsonb,
+               cell_meta_patch jsonb
+             )
+           ),
+           applied_rows AS (
+             UPDATE ${sheetTable(session)} AS target
+             SET _status = 'enriched',
+                 _run_id = $4,
+                 _error = NULL,
+                 _updated_at = now(),
+                 _cell_meta = coalesce(target._cell_meta, '{}'::jsonb)
+                   || coalesce(updates.cell_meta_patch, '{}'::jsonb)${physicalUpdateSetSql}
+             FROM updates
+             WHERE target._key = updates.key
+             RETURNING target._key
+           ),
+           inserted_events AS (
+             INSERT INTO ${eventTable(session)}
+               (play_name, table_namespace, _key, _status, _run_id, _stage, data_patch)
+             SELECT $2, $3, updates.key, 'enriched', $4, 'completed',
+                    coalesce(updates.data_patch, '{}'::jsonb)
+             FROM updates
+             WHERE updates.key = ANY(SELECT _key FROM applied_rows)
+             RETURNING 1
+           )
+           SELECT _key FROM applied_rows`,
+            [
+              JSON.stringify(chunk),
+              normalizedPlayName,
+              normalizedTableNamespace,
+              input.runId,
+            ],
+          );
+          updated += appliedRows.length;
+        }
+        if (needsTransaction) await client.query('COMMIT');
+      } catch (error) {
+        if (needsTransaction) {
+          await client.query('ROLLBACK').catch(() => undefined);
+        }
+        throw error;
+      }
+      return { updated };
+    },
+    { cachePool: !context.disablePostgresPoolCache },
+  );
+}
+
+export async function persistRuntimeCsvDataset(
+  context: RuntimeApiContext & { playName: string },
+  input: {
+    tableNamespace: string;
+    playInput?: Record<string, unknown> | null;
+    sheetContract: PlaySheetContract;
+    rows: Record<string, unknown>[];
+    runId: string;
+    sourceLabel?: string | null;
+  },
+): Promise<PlayDataset<Record<string, unknown>>> {
+  await startRuntimeSheetDataset(context, {
+    playName: context.playName,
+    tableNamespace: input.tableNamespace,
+    playInput: input.playInput,
+    sheetContract: input.sheetContract,
+    rows: input.rows,
+    runId: input.runId,
+  });
+  return createRuntimeBackedPlayDataset({
+    context,
+    tableNamespace: input.tableNamespace,
+    datasetKind: 'csv',
+    sourceLabel: input.sourceLabel,
+    initialCount: input.rows.length,
+    initialPreviewRows: input.rows.slice(0, 10),
+  });
+}
+
+export async function upsertRuntimeRows(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    tableNamespace: string;
+    rows: Record<string, unknown>[];
+    runId: string;
+    sheetContract: PlaySheetContract;
+    skipEnsureSheet?: boolean;
+  },
+): Promise<RowsWriteResponse> {
+  if (!context.playName) {
+    throw new Error('Runtime DB sessions require a playName.');
+  }
+  const session = await getRuntimeDbSession(
+    { ...context, playName: context.playName },
+    {
+      tableNamespace: input.tableNamespace,
+      logicalTable: 'sheet_rows',
+      operations: ['rows.upsert'],
+      limits: {
+        maxRows: Math.max(input.rows.length, 1),
+      },
+      sheetContract: input.sheetContract,
+    },
+  );
+  return rowsWriteResponseSchema.parse(
+    await writeRuntimeRows(requireRuntimePostgresSession(session), {
+      tableNamespace: input.tableNamespace,
+      rows: input.rows,
+      runId: input.runId,
+      sheetContract: input.sheetContract,
+      idempotencyKey: createHash('sha1')
+        .update(
+          `${input.playName}:${input.tableNamespace}:${input.runId}:${JSON.stringify(input.rows)}`,
+        )
+        .digest('hex'),
+      mode: 'upsert',
+    }),
+  );
+}
+
+export async function appendRuntimeRows(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    tableNamespace: string;
+    rows: Record<string, unknown>[];
+    runId: string;
+    idempotencyKey: string;
+    sheetContract: PlaySheetContract;
+  },
+): Promise<RowsWriteResponse> {
+  if (!context.playName) {
+    throw new Error('Runtime DB sessions require a playName.');
+  }
+  const session = await getRuntimeDbSession(
+    { ...context, playName: context.playName },
+    {
+      tableNamespace: input.tableNamespace,
+      logicalTable: 'sheet_rows',
+      operations: ['rows.append'],
+      limits: {
+        maxRows: Math.max(input.rows.length, 1),
+      },
+      sheetContract: input.sheetContract,
+    },
+  );
+  return rowsWriteResponseSchema.parse(
+    await writeRuntimeRows(requireRuntimePostgresSession(session), {
+      tableNamespace: input.tableNamespace,
+      rows: input.rows,
+      runId: input.runId,
+      sheetContract: input.sheetContract,
+      idempotencyKey: input.idempotencyKey,
+      mode: 'append',
+    }),
+  );
+}
+
+export async function replaceRuntimeRows(
+  context: RuntimeApiContext,
+  input: {
+    playName: string;
+    tableNamespace: string;
+    rows: Record<string, unknown>[];
+    runId: string;
+    idempotencyKey: string;
+    sheetContract: PlaySheetContract;
+  },
+): Promise<RowsWriteResponse> {
+  if (!context.playName) {
+    throw new Error('Runtime DB sessions require a playName.');
+  }
+  const session = await getRuntimeDbSession(
+    { ...context, playName: context.playName },
+    {
+      tableNamespace: input.tableNamespace,
+      logicalTable: 'sheet_rows',
+      operations: ['rows.replace'],
+      limits: {
+        maxRows: Math.max(input.rows.length, 1),
+      },
+      sheetContract: input.sheetContract,
+    },
+  );
+  return rowsWriteResponseSchema.parse(
+    await writeRuntimeRows(requireRuntimePostgresSession(session), {
+      tableNamespace: input.tableNamespace,
+      rows: input.rows,
+      runId: input.runId,
+      sheetContract: input.sheetContract,
+      idempotencyKey: input.idempotencyKey,
+      mode: 'replace',
+    }),
+  );
+}
+
+export function createRuntimeBackedPlayDataset(input: {
+  context: RuntimeApiContext & {
+    playName: string;
+  };
+  tableNamespace: string;
+  datasetKind: 'csv' | 'map';
+  sourceLabel?: string | null;
+  initialCount?: number;
+  initialPreviewRows?: Record<string, unknown>[];
+}): PlayDataset<Record<string, unknown>> {
+  const csvRows =
+    input.datasetKind === 'csv' &&
+    (input.initialPreviewRows?.length ?? 0) >= (input.initialCount ?? 0)
+      ? [...(input.initialPreviewRows ?? [])]
+      : null;
+  if (csvRows) {
+    return createDeferredPlayDataset({
+      datasetKind: input.datasetKind,
+      datasetId: createRuntimeDatasetId(
+        input.context.playName,
+        input.tableNamespace,
+      ),
+      count: input.initialCount ?? csvRows.length,
+      previewRows: csvRows.slice(0, 10),
+      sourceLabel: input.sourceLabel ?? null,
+      resolvers: {
+        count: async () => csvRows.length,
+        peek: async (limit) => csvRows.slice(0, Math.max(0, limit)),
+        materialize: async (limit) =>
+          limit === undefined
+            ? [...csvRows]
+            : csvRows.slice(0, Math.max(0, limit)),
+        iterate: () =>
+          ({
+            async *[Symbol.asyncIterator]() {
+              for (const row of csvRows) {
+                yield row;
+              }
+            },
+          }) as AsyncIterable<Record<string, unknown>>,
+      },
+    });
+  }
+  return createDeferredPlayDataset({
+    datasetKind: input.datasetKind,
+    datasetId: createRuntimeDatasetId(
+      input.context.playName,
+      input.tableNamespace,
+    ),
+    count: input.initialCount ?? 0,
+    backing: {
+      storage: 'neon_sheet',
+      sheet: {
+        playName: input.context.playName,
+        tableNamespace: input.tableNamespace,
+      },
+    },
+    previewRows: input.initialPreviewRows ?? [],
+    sourceLabel: input.sourceLabel ?? null,
+    tableNamespace: input.tableNamespace,
+    resolvers: {
+      count: async () => {
+        const summarySession = await getRuntimeDbSession(input.context, {
+          tableNamespace: input.tableNamespace,
+          logicalTable: 'sheet_rows',
+          operations: ['rows.read'],
+        });
+        const summary = await readRuntimeSummary(
+          requireRuntimePostgresSession(summarySession),
+        );
+        return Number(summary.stats.total ?? 0);
+      },
+      peek: async (limit) => {
+        const session = await getRuntimeDbSession(input.context, {
+          tableNamespace: input.tableNamespace,
+          logicalTable: 'sheet_rows',
+          operations: ['rows.read'],
+        });
+        const rows = await readRuntimeRows(
+          requireRuntimePostgresSession(session),
+          {
+            limit,
+            offset: 0,
+          },
+        );
+        return rows.map((row) => row.data);
+      },
+      materialize: async (limit) => {
+        const pageSize = 1000;
+        const materialized: Record<string, unknown>[] = [];
+        let offset = 0;
+        while (true) {
+          const session = await getRuntimeDbSession(input.context, {
+            tableNamespace: input.tableNamespace,
+            logicalTable: 'sheet_rows',
+            operations: ['rows.read'],
+          });
+          const rows = await readRuntimeRows(
+            requireRuntimePostgresSession(session),
+            {
+              limit:
+                limit !== undefined
+                  ? Math.min(pageSize, Math.max(0, limit - materialized.length))
+                  : pageSize,
+              offset,
+            },
+          );
+          if (rows.length === 0) {
+            break;
+          }
+          materialized.push(...rows.map((row) => row.data));
+          if (limit !== undefined && materialized.length >= limit) {
+            return materialized.slice(0, limit);
+          }
+          offset += rows.length;
+        }
+        return materialized;
+      },
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            let afterInputIndex = -1;
+            while (true) {
+              const session = await getRuntimeDbSession(input.context, {
+                tableNamespace: input.tableNamespace,
+                logicalTable: 'sheet_rows',
+                operations: ['rows.read'],
+              });
+              const page = await readRuntimeRowsPage(
+                requireRuntimePostgresSession(session),
+                {
+                  afterInputIndex,
+                  limit: 1000,
+                  statuses: ['pending', 'completed', 'failed'],
+                },
+              );
+              if (page.length === 0) {
+                return;
+              }
+              for (const row of page) {
+                afterInputIndex = Math.max(
+                  afterInputIndex,
+                  Number(row.inputIndex ?? afterInputIndex),
+                );
+                yield row.data;
+              }
+            }
+          },
+        }) as AsyncIterable<Record<string, unknown>>,
+    },
+  });
+}
+
+export function resolveRuntimeSheetContract(
+  pipeline: PlayStaticPipeline | null | undefined,
+  tableNamespace: string | null | undefined,
+): PlaySheetContract | null {
+  const requestedNamespace = tableNamespace?.trim();
+  if (!pipeline || !requestedNamespace) {
+    return null;
+  }
+  const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
+
+  const rootNamespace = pipeline.tableNamespace?.trim();
+  if (
+    rootNamespace &&
+    normalizeTableNamespace(rootNamespace) === normalizedNamespace
+  ) {
+    return pipeline.sheetContract ?? null;
+  }
+
+  for (const substep of [...(pipeline.stages ?? []), ...pipeline.substeps]) {
+    if (substep.type !== 'map') {
+      continue;
+    }
+    const substepNamespace = substep.tableNamespace?.trim();
+    if (
+      substepNamespace &&
+      normalizeTableNamespace(substepNamespace) === normalizedNamespace
+    ) {
+      return substep.sheetContract ?? null;
+    }
+  }
+
+  return null;
+}