deepline 0.3.227 → 0.3.229

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -24,6 +24,10 @@ import {
   physicalSheetColumnProjections,
   type PhysicalSheetColumnProjection,
 } from '../play-data-plane/sheet-contract';
+import {
+  RuntimeSheetColumns,
+  type RuntimeSheetColumnTypes,
+} from '../play-data-plane/sheet-column-storage';
 import { RUNTIME_SHEET_CELL_META_COLUMN } from '../play-data-plane/runtime-sheet-storage-contract';
 import {
   createRuntimeSheetOrm,
@@ -2925,7 +2929,7 @@ function missingOutputCellSql(
       const quoted = `${tableAlias}.${quoteIdentifier(column.sqlName)}`;
       const staleAt = `${tableAlias}._cell_meta -> ${quoteLiteral(column.fieldName)} -> 'staleAt'`;
       const staleAtMs = `(CASE WHEN jsonb_typeof(${staleAt}) = 'number' THEN (${staleAt})::text::double precision ELSE NULL END)`;
-      return `(${quoted} IS NULL OR ${quoted} = 'null'::jsonb OR ${quoted} = '""'::jsonb OR (${staleAtMs} IS NOT NULL AND ${staleAtMs} <= extract(epoch from now()) * 1000))`;
+      return `(${quoted} IS NULL OR to_jsonb(${quoted}) = 'null'::jsonb OR to_jsonb(${quoted}) = '""'::jsonb OR (${staleAtMs} IS NOT NULL AND ${staleAtMs} <= extract(epoch from now()) * 1000))`;
     })
     .join(' OR ');
 }
@@ -2941,7 +2945,7 @@ function changedPatchedCellSql(
   return projections
     .map((column) => {
       const quoted = `${tableAlias}.${quoteIdentifier(column.sqlName)}`;
-      return `(${patchAlias} ? ${quoteLiteral(column.fieldName)} AND ${quoted} IS DISTINCT FROM ${patchAlias} -> ${quoteLiteral(column.fieldName)})`;
+      return `(${patchAlias} ? ${quoteLiteral(column.fieldName)} AND coalesce(to_jsonb(${quoted}), 'null'::jsonb) IS DISTINCT FROM ${patchAlias} -> ${quoteLiteral(column.fieldName)})`;
     })
     .join(' OR ');
 }
@@ -3331,12 +3335,6 @@ async function writeRuntimeRows(
     physicalColumns.length > 0
       ? `, ${physicalColumns.map(quoteIdentifier).join(', ')}`
       : '';
-  const physicalInsertValuesSql =
-    physicalColumns.length > 0
-      ? `, ${physicalColumnProjections
-          .map((column) => `payload -> ${quoteLiteral(column.fieldName)}`)
-          .join(', ')}`
-      : '';
 
   const rowsToWrite = input.rows.filter(
     (row) => row && typeof row === 'object' && !Array.isArray(row),
@@ -3349,10 +3347,8 @@ async function writeRuntimeRows(
     input.tableNamespace,
   );
 
-  // Build write entries first so we know chunk count up front. Single-chunk
-  // writes skip the BEGIN/COMMIT pair entirely (the mega-CTE is atomic at
-  // statement level), and the per-chunk mega-CTE folds: starting-index lookup,
-  // sheet insert, and summary upsert into one round-trip.
+  // Keep every chunk's row insert and summary update in the same transaction
+  // as column inference, so a concurrent writer cannot invalidate its schema.
   const rowEntries =
     input.mode === 'upsert'
       ? Array.from(
@@ -3365,7 +3361,11 @@ async function writeRuntimeRows(
               return uniqueRows;
             }, new Map<string, Record<string, unknown>>())
             .entries(),
-        ).map(([key, row], inputIndex) => ({ key, row, inputIndex }))
+        ).map(([key, row], inputIndex) => ({
+          key,
+          row: sanitizePostgresJsonValue(row),
+          inputIndex,
+        }))
       : rowsToWrite.map((row, index) => ({
           key: buildAppendedRowKey({
             row,
@@ -3373,7 +3373,7 @@ async function writeRuntimeRows(
             idempotencyKey: input.idempotencyKey,
             ordinal: index,
           }),
-          row,
+          row: sanitizePostgresJsonValue(row),
           // For append/replace modes the starting offset is computed in SQL
           // as `coalesce(max(_input_index), -1) + 1 + (ord - 1)`; we only
           // pass the per-chunk ordinal here so the SQL can add it to the
@@ -3382,12 +3382,47 @@ async function writeRuntimeRows(
         }));
 
   const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
-  const needsTransaction = input.mode === 'replace' || chunks.length > 1;
+  // Type evolution and the row mutation share one transaction and schema lock.
+  const needsTransaction = true;
 
   return await withRuntimePostgres(session, async (client, transaction) => {
     const managesTransaction = needsTransaction && !transaction.active;
     if (managesTransaction) await client.query('BEGIN');
     try {
+      const columnTypes = await RuntimeSheetColumns.prepareWrite(client, {
+        schema: session.postgres.schema,
+        table: session.postgres.sheetTable,
+        columns: physicalColumnProjections,
+        rows: async () => {
+          if (input.mode !== 'upsert')
+            return rowEntries.map((entry) => entry.row);
+          const existing = await client.query<{ _key: string }>(
+            `SELECT _key FROM ${sheetTable(session)} WHERE _key = ANY($1::text[])`,
+            [rowEntries.map((entry) => entry.key)],
+          );
+          const existingKeys = new Set(existing.rows.map((row) => row._key));
+          return rowEntries
+            .filter((entry) => !existingKeys.has(entry.key))
+            .map((entry) => entry.row);
+        },
+      });
+      const physicalInsertValuesSql =
+        physicalColumns.length > 0
+          ? `, ${physicalColumnProjections
+              .map((column) => {
+                const incoming = RuntimeSheetColumns.jsonValueSql(
+                  `input_rows.payload -> ${quoteLiteral(column.fieldName)}`,
+                  sheetTable(session),
+                  column.sqlName,
+                  columnTypes,
+                );
+                return input.mode === 'upsert'
+                  ? `CASE WHEN existing_input._key IS NOT NULL THEN existing_input.${quoteIdentifier(column.sqlName)} ELSE ${incoming} END`
+                  : incoming;
+              })
+              .join(', ')}`
+          : '';
+
       if (input.mode === 'replace') {
         // Collapse 4 cleanup statements into one CTE-shaped query. Postgres
         // executes data-modifying CTEs against the same snapshot, but each
@@ -3438,8 +3473,9 @@ async function writeRuntimeRows(
           input.mode === 'upsert'
             ? `inserted_rows AS (
                 INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _created_at${physicalInsertColumnsSql})
-                SELECT _key, 'pending', $4::text, _input_index, now()${physicalInsertValuesSql}
+                SELECT input_rows._key, 'pending', $4::text, input_rows._input_index, now()${physicalInsertValuesSql}
                 FROM input_rows
+                LEFT JOIN ${sheetTable(session)} AS existing_input ON existing_input._key = input_rows._key
                 ON CONFLICT (_key) DO UPDATE SET
                   _status = 'pending',
                   _run_id = EXCLUDED._run_id,
@@ -3764,8 +3800,9 @@ async function prepareRuntimeSheetDatasetRows(
         ),
         inserted_rows AS (
           INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id, _created_at${input.physicalInsertColumnsSql})
-          SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text, now()${input.physicalInsertValuesSql}
+          SELECT input_rows._key, 'pending', $4::text, input_rows._input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text, now()${input.physicalInsertValuesSql}
           FROM input_rows
+          LEFT JOIN ${sheetTable(session)} AS existing_input ON existing_input._key = input_rows._key
           ON CONFLICT (_key) DO UPDATE SET
             _status = 'pending',
             _run_id = EXCLUDED._run_id,
@@ -4111,9 +4148,10 @@ async function prepareNetNewRuntimeSheetDatasetRows(
          INSERT INTO ${sheetTable(session)}
          (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id,
           _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id, _created_at${input.physicalInsertColumnsSql})
-       SELECT _key, 'pending', $4::text, _input_index, $5::text, $6::text,
+       SELECT input_rows._key, 'pending', $4::text, input_rows._input_index, $5::text, $6::text,
               $7::timestamptz, $8::integer, $9::bigint, $4::text, now()${input.physicalInsertValuesSql}
        FROM input_rows
+       LEFT JOIN ${sheetTable(session)} AS existing_input ON existing_input._key = input_rows._key
        ON CONFLICT (_key) DO NOTHING
        RETURNING _key
        ),
@@ -6591,7 +6629,7 @@ export async function startRuntimeSheetDataset(
       derivePlayRowIdentity(cleanedRow, input.tableNamespace);
     if (key && !uniqueRows.has(key)) {
       uniqueRows.set(key, {
-        row: cleanedRow,
+        row: sanitizePostgresJsonValue(cleanedRow),
         inputIndex: normalizeRuntimeMapInputIndex(
           row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex],
         ),
@@ -6635,12 +6673,6 @@ export async function startRuntimeSheetDataset(
     physicalColumns.length > 0
       ? `, ${physicalColumns.map(quoteIdentifier).join(', ')}`
       : '';
-  const physicalInsertValuesSql =
-    physicalColumns.length > 0
-      ? `, ${physicalColumns
-          .map((column) => `payload -> ${quoteLiteral(column)}`)
-          .join(', ')}`
-      : '';
   const outputPhysicalColumns = outputPhysicalSheetColumnProjections(
     input.sheetContract,
   );
@@ -6650,15 +6682,6 @@ export async function startRuntimeSheetDataset(
   const physicalRefreshColumns = physicalColumns.filter(
     (column) => !outputPhysicalColumnNames.has(column),
   );
-  const physicalRefreshSetSql =
-    physicalRefreshColumns.length > 0
-      ? `, ${physicalRefreshColumns
-          .map(
-            (column) =>
-              `${quoteIdentifier(column)} = input_rows.payload -> ${quoteLiteral(column)}`,
-          )
-          .join(', ')}`
-      : '';
   const physicalUpsertSetSql =
     physicalColumns.length > 0
       ? `, ${physicalColumns
@@ -6705,6 +6728,77 @@ export async function startRuntimeSheetDataset(
         session,
         input.writeVersion,
       );
+      const columnTypes = await RuntimeSheetColumns.prepareWrite(client, {
+        schema: session.postgres.schema,
+        table: session.postgres.sheetTable,
+        columns: physicalSheetColumnProjections(input.sheetContract).map(
+          (column) => ({ ...column, fieldName: column.sqlName }),
+        ),
+        rows: async () => {
+          const existing = await client.query<{
+            _key: string;
+            _status: string;
+            _run_id: string | null;
+            row_write_stamp: string | number;
+            _input_index: string | number | null;
+          }>(
+            `SELECT _key, _status, _run_id, _write_version AS row_write_stamp, _input_index
+               FROM ${sheetTable(session)} WHERE _key = ANY($1::text[]) FOR UPDATE`,
+            [rowEntries.map((entry) => entry.key)],
+          );
+          const current = new Map(existing.rows.map((row) => [row._key, row]));
+          return rowEntries.flatMap((entry) => {
+            const row = current.get(entry.key);
+            if (!row) return [entry.row];
+            if (input.mode === 'net_new') return [];
+            if (row._status === 'stale') return [entry.row];
+            if (input.force !== true) return [];
+            const version = Number(row.row_write_stamp);
+            const refreshes =
+              version < writeVersion ||
+              (version === writeVersion &&
+                (row._status !== 'enriched' ||
+                  row._run_id !== input.runId ||
+                  row._input_index === null ||
+                  Number(row._input_index) !== entry.inputIndex));
+            if (!refreshes) return [];
+            return [
+              Object.fromEntries(
+                Object.entries(entry.row).filter(
+                  ([field]) => !outputPhysicalColumnNames.has(field),
+                ),
+              ),
+            ];
+          });
+        },
+      });
+      const physicalInsertValuesSql =
+        physicalColumns.length > 0
+          ? `, ${physicalColumns
+              .map((column) => {
+                const incoming = RuntimeSheetColumns.jsonValueSql(
+                  `input_rows.payload -> ${quoteLiteral(column)}`,
+                  sheetTable(session),
+                  column,
+                  columnTypes,
+                );
+                const preserveExisting =
+                  input.mode === 'net_new'
+                    ? 'existing_input._key IS NOT NULL'
+                    : "existing_input._key IS NOT NULL AND existing_input._status <> 'stale'";
+                return `CASE WHEN ${preserveExisting} THEN existing_input.${quoteIdentifier(column)} ELSE ${incoming} END`;
+              })
+              .join(', ')}`
+          : '';
+      const physicalRefreshSetSql =
+        physicalRefreshColumns.length > 0
+          ? `, ${physicalRefreshColumns
+              .map(
+                (column) =>
+                  `${quoteIdentifier(column)} = ${RuntimeSheetColumns.jsonValueSql(`input_rows.payload -> ${quoteLiteral(column)}`, sheetTable(session), column, columnTypes)}`,
+              )
+              .join(', ')}`
+          : '';
       timings.push({
         phase: 'attempt_expiry_mint',
         ms: Date.now() - attemptExpiryStartedAt,
@@ -7105,9 +7199,28 @@ export async function releaseRuntimeWorkReceipts(
   );
 }
 
+function physicalRuntimeSheetUpdateSql(
+  session: RuntimePostgresSession,
+  projections: readonly PhysicalSheetColumnProjection[],
+  columnTypes: RuntimeSheetColumnTypes,
+): string {
+  return projections.length > 0
+    ? `,\n        ${projections
+        .map((column) => {
+          const quoted = quoteIdentifier(column.sqlName);
+          const literal = quoteLiteral(column.fieldName);
+          return `${quoted} = CASE
+          WHEN coalesce(updates.data_patch, '{}'::jsonb) ? ${literal}
+            THEN ${RuntimeSheetColumns.jsonValueSql(`updates.data_patch -> ${literal}`, sheetTable(session), column.sqlName, columnTypes)}
+          ELSE target.${quoted}
+        END`;
+        })
+        .join(',\n        ')}`
+    : '';
+}
+
 type CompleteRuntimeMapRowChunksInput = {
   chunks: RuntimePreparedCompletedRow[][];
-  physicalUpdateSetSql: string;
   physicalColumnProjections: PhysicalSheetColumnProjection[];
   runId: string;
   attemptId: string | null;
@@ -7385,12 +7498,7 @@ async function completeRuntimeMapRowChunks(
         OR coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $10::text
       )
     )`;
-    const { rows } = await client.query<{
-      updated: number;
-      matched_keys: string[];
-      fenced_rows: unknown;
-    }>(
-      `WITH updates AS (
+    const matchingUpdatesSql = `WITH mutation_context AS (SELECT $6::text AS play_name, $7::text AS table_namespace, $11::timestamptz AS attempt_expires_at, $12::integer AS attempt_seq), updates AS (
          SELECT key_values._key,
                 input_index_values.input_index,
                 data_values.data_patch,
@@ -7414,23 +7522,8 @@ async function completeRuntimeMapRowChunks(
            FROM updates
            JOIN ${sheetTable(session)} AS target
              ON target._key = updates._key
-       ),
-       applied_rows AS (
-         UPDATE ${sheetTable(session)} AS target
-         SET _status = 'enriched',
-             _run_id = $5::text,
-             _writer_run_id = $5::text,
-             _write_version = COALESCE($13::bigint, target._write_version),
-             _error = NULL,
-             _attempt_id = $9::text,
-             _attempt_owner_run_id = $10::text,
-             _attempt_expires_at = $11::timestamptz,
-             _attempt_seq = $12::integer,
-             _updated_at = now(),
-             _version = ${nextRuntimeSheetVersionExpression(session)},
-             _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
-         FROM matched_updates AS updates
-         WHERE target._key = updates.matched_key
+       )`;
+    const writableRowsSql = `target._key = updates.matched_key
 		           AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
                    AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
@@ -7471,7 +7564,63 @@ async function completeRuntimeMapRowChunks(
                  FROM unnest($8::text[]) AS field_values(field)
                 WHERE coalesce(target._cell_meta -> field_values.field ->> 'status', '') <> 'completed'
              )
-           )
+           )`;
+    const mutationParams = [
+      chunkKeys,
+      chunkInputIndexes,
+      chunkDataPatches,
+      chunkCellMetaPatches,
+      input.runId,
+      input.normalizedPlayName,
+      input.normalizedTableNamespace,
+      [...new Set(input.outputFields)],
+      input.attemptId,
+      input.attemptOwnerRunId,
+      input.attemptExpiresAt,
+      input.attemptSeq,
+      input.writeVersion,
+    ];
+    const columnTypes = await RuntimeSheetColumns.prepareWrite(client, {
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      columns: input.physicalColumnProjections,
+      rows: async () => {
+        const accepted = await client.query<{
+          data_patch: Record<string, unknown>;
+        }>(
+          `${matchingUpdatesSql}
+           SELECT updates.data_patch
+           FROM matched_updates AS updates
+           JOIN ${sheetTable(session)} AS target ON target._key = updates.matched_key
+           WHERE ${writableRowsSql}
+           FOR UPDATE OF target`,
+          mutationParams,
+        );
+        return accepted.rows.map((row) => row.data_patch);
+      },
+    });
+    const { rows } = await client.query<{
+      updated: number;
+      matched_keys: string[];
+      fenced_rows: unknown;
+    }>(
+      `${matchingUpdatesSql},
+       applied_rows AS (
+         UPDATE ${sheetTable(session)} AS target
+         SET _status = 'enriched',
+             _run_id = $5::text,
+             _writer_run_id = $5::text,
+             _write_version = COALESCE($13::bigint, target._write_version),
+             _error = NULL,
+             _attempt_id = $9::text,
+             _attempt_owner_run_id = $10::text,
+             _attempt_expires_at = $11::timestamptz,
+             _attempt_seq = $12::integer,
+             _updated_at = now(),
+             _version = ${nextRuntimeSheetVersionExpression(session)},
+             _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${physicalRuntimeSheetUpdateSql(session, input.physicalColumnProjections, columnTypes)}
+         FROM matched_updates AS updates
+         WHERE ${writableRowsSql}
          RETURNING target._key, updates.prev_status, updates.prev_cell_meta
        ),
        applied_count AS (
@@ -7595,21 +7744,7 @@ async function completeRuntimeMapRowChunks(
          coalesce((SELECT jsonb_agg(jsonb_build_object('key', _key, 'reason', reason, 'rowVersionStamp', row_write_version)) FROM fenced_rows), '[]'::jsonb) AS fenced_rows,
          (SELECT count(*)::int FROM summary_delta) AS summary_delta_count,
          (SELECT count(*)::int FROM column_delta) AS column_delta_count`,
-      [
-        chunkKeys,
-        chunkInputIndexes,
-        chunkDataPatches,
-        chunkCellMetaPatches,
-        input.runId,
-        input.normalizedPlayName,
-        input.normalizedTableNamespace,
-        [...new Set(input.outputFields)],
-        input.attemptId,
-        input.attemptOwnerRunId,
-        input.attemptExpiresAt,
-        input.attemptSeq,
-        input.writeVersion,
-      ],
+      mutationParams,
     );
     const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
     const chunkFencedRows = parseRuntimeMapRowsFencedRows(rows[0]?.fenced_rows);
@@ -7663,12 +7798,7 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
       '$11::timestamptz',
       '$12',
     );
-    const { rows } = await client.query<{
-      _key?: string;
-      fenced_rows?: unknown;
-      updated?: number;
-    }>(
-      `WITH updates AS (
+    const matchingUpdatesSql = `WITH mutation_context AS (SELECT $6::text AS play_name, $7::text AS table_namespace, $11::timestamptz AS attempt_expires_at, $12::integer AS attempt_seq), updates AS (
          SELECT key_values._key,
                 input_index_values.input_index,
                 data_values.data_patch,
@@ -7696,24 +7826,8 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
                AND target._input_index = updates.input_index
              )
           ORDER BY target._key, (target._key = updates._key) DESC
-       ),
-       applied_rows AS (
-         UPDATE ${sheetTable(session)} AS target
-         SET _status = 'enriched',
-             _run_id = $5::text,
-             _writer_run_id = $5::text,
-             _write_version = COALESCE($13::bigint, target._write_version),
-             _error = NULL,
-             _attempt_id = $9::text,
-             _attempt_owner_run_id = $10::text,
-             _attempt_expires_at = $11::timestamptz,
-             _attempt_seq = $12::integer,
-             _updated_at = now(),
-             _version = ${nextRuntimeSheetVersionExpression(session)},
-             _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
-         FROM matched_updates AS updates, ${sheetTable(session)} AS prev
-         WHERE target._key = updates.matched_key
-           AND prev._key = target._key
+       )`;
+    const writableRowsSql = `target._key = updates.matched_key
            AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
            AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
@@ -7749,7 +7863,64 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
                  FROM unnest($8::text[]) AS field_values(field)
                 WHERE coalesce(target._cell_meta -> field_values.field ->> 'status', '') <> 'completed'
              )
-           )
+           )`;
+    const mutationParams = [
+      chunkKeys,
+      chunkInputIndexes,
+      chunkDataPatches,
+      chunkCellMetaPatches,
+      input.runId,
+      input.normalizedPlayName,
+      input.normalizedTableNamespace,
+      [...new Set(input.outputFields)],
+      input.attemptId,
+      input.attemptOwnerRunId,
+      input.attemptExpiresAt,
+      input.attemptSeq,
+      input.writeVersion,
+    ];
+    const columnTypes = await RuntimeSheetColumns.prepareWrite(client, {
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      columns: input.physicalColumnProjections,
+      rows: async () => {
+        const accepted = await client.query<{
+          data_patch: Record<string, unknown>;
+        }>(
+          `${matchingUpdatesSql}
+           SELECT updates.data_patch
+           FROM matched_updates AS updates
+           JOIN ${sheetTable(session)} AS target ON target._key = updates.matched_key
+           WHERE ${writableRowsSql}
+           FOR UPDATE OF target`,
+          mutationParams,
+        );
+        return accepted.rows.map((row) => row.data_patch);
+      },
+    });
+    const { rows } = await client.query<{
+      _key?: string;
+      fenced_rows?: unknown;
+      updated?: number;
+    }>(
+      `${matchingUpdatesSql},
+       applied_rows AS (
+         UPDATE ${sheetTable(session)} AS target
+         SET _status = 'enriched',
+             _run_id = $5::text,
+             _writer_run_id = $5::text,
+             _write_version = COALESCE($13::bigint, target._write_version),
+             _error = NULL,
+             _attempt_id = $9::text,
+             _attempt_owner_run_id = $10::text,
+             _attempt_expires_at = $11::timestamptz,
+             _attempt_seq = $12::integer,
+             _updated_at = now(),
+             _version = ${nextRuntimeSheetVersionExpression(session)},
+             _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${physicalRuntimeSheetUpdateSql(session, input.physicalColumnProjections, columnTypes)}
+         FROM matched_updates AS updates, ${sheetTable(session)} AS prev
+         WHERE ${writableRowsSql}
+           AND prev._key = target._key
          RETURNING target._key, prev._status AS prev_status, prev._cell_meta AS prev_cell_meta
        ),
        applied_count AS (
@@ -7868,21 +8039,7 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
        SELECT
          (SELECT count(*)::int FROM applied_rows) AS updated,
          coalesce((SELECT jsonb_agg(jsonb_build_object('key', _key, 'reason', reason, 'rowVersionStamp', row_write_version)) FROM fenced_rows), '[]'::jsonb) AS fenced_rows`,
-      [
-        chunkKeys,
-        chunkInputIndexes,
-        chunkDataPatches,
-        chunkCellMetaPatches,
-        input.runId,
-        input.normalizedPlayName,
-        input.normalizedTableNamespace,
-        [...new Set(input.outputFields)],
-        input.attemptId,
-        input.attemptOwnerRunId,
-        input.attemptExpiresAt,
-        input.attemptSeq,
-        input.writeVersion,
-      ],
+      mutationParams,
     );
     updated += Number(rows[0]?.updated ?? 0);
     fencedRows.push(...parseRuntimeMapRowsFencedRows(rows[0]?.fenced_rows));
@@ -7902,21 +8059,56 @@ async function insertMissingCompletedMapRowChunks(
           .map((column) => quoteIdentifier(column.sqlName))
           .join(', ')}`
       : '';
-  const physicalInsertValuesSql =
-    input.physicalColumnProjections.length > 0
-      ? `, ${input.physicalColumnProjections
-          .map(
-            (column) =>
-              `missing_rows.data_patch -> ${quoteLiteral(column.fieldName)}`,
-          )
-          .join(', ')}`
-      : '';
 
   for (const chunk of input.chunks) {
     const chunkKeys = chunk.map((row) => row.key);
     const chunkInputIndexes = chunk.map((row) => row.input_index);
     const chunkDataPatches = chunk.map((row) => row.data_patch_json);
     const chunkCellMetaPatches = chunk.map((row) => row.cell_meta_patch_json);
+    const columnTypes = await RuntimeSheetColumns.prepareWrite(client, {
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      columns: input.physicalColumnProjections,
+      rows: async () => {
+        const accepted = await client.query<{
+          data_patch: Record<string, unknown>;
+        }>(
+          `SELECT incoming.data_patch FROM jsonb_to_recordset($1::jsonb)
+             AS incoming(key text, input_index bigint, data_patch jsonb)
+           WHERE NOT EXISTS (
+             SELECT 1 FROM ${sheetTable(session)} AS target
+             WHERE target._key = incoming.key
+               OR (incoming.input_index IS NOT NULL AND target._run_id = $2::text
+                   AND target._input_index = incoming.input_index)
+           )`,
+          [
+            stringifyPostgresJson(
+              chunk.map((row) => ({
+                key: row.key,
+                input_index: row.input_index,
+                data_patch: JSON.parse(row.data_patch_json),
+              })),
+            ),
+            input.runId,
+          ],
+        );
+        return accepted.rows.map((row) => row.data_patch);
+      },
+    });
+    const physicalInsertValuesSql =
+      input.physicalColumnProjections.length > 0
+        ? `, ${input.physicalColumnProjections
+            .map((column) =>
+              RuntimeSheetColumns.jsonValueSql(
+                `missing_rows.data_patch -> ${quoteLiteral(column.fieldName)}`,
+                sheetTable(session),
+                column.sqlName,
+                columnTypes,
+              ),
+            )
+            .join(', ')}`
+        : '';
+
     const { rows } = await client.query<{ inserted: number }>(
       `WITH input_rows AS (
          SELECT key_values._key,
@@ -8068,7 +8260,6 @@ async function failRuntimeMapRowChunks(
   session: RuntimePostgresSession,
   input: {
     chunks: RuntimePreparedFailedRow[][];
-    physicalUpdateSetSql: string;
     physicalColumnProjections: PhysicalSheetColumnProjection[];
     forceTerminal?: boolean;
     runId: string;
@@ -8112,11 +8303,7 @@ async function failRuntimeMapRowChunks(
     const failedCellTotals = failedCellFields.map(
       (field) => failedCellCounts.get(field) ?? 0,
     );
-    const { rows } = await client.query<{
-      updated?: number;
-      fenced_rows?: unknown;
-    }>(
-      `WITH updates AS (
+    const matchingUpdatesSql = `WITH mutation_context AS (SELECT $7::text AS play_name, $8::text AS table_namespace, $9::text[] AS failed_fields, $10::int[] AS failed_totals, $12::text AS attempt_owner_run_id, $13::timestamptz AS attempt_expires_at, $14::integer AS attempt_seq), updates AS (
          SELECT key_values._key,
                 input_index_values.input_index,
                 data_values.data_patch,
@@ -8148,25 +8335,9 @@ async function failRuntimeMapRowChunks(
                AND target._input_index = updates.input_index
              )
           ORDER BY target._key, (target._key = updates._key) DESC
-       ),
-       applied_rows AS (
-         UPDATE ${sheetTable(session)} AS target
-         SET _status = 'failed',
-             _run_id = $6::text,
-             _writer_run_id = $6::text,
-             _write_version = COALESCE($16::bigint, target._write_version),
-             _error = updates.error,
-             _attempt_id = $11::text,
-             _attempt_owner_run_id = $12::text,
-             _attempt_expires_at = $13::timestamptz,
-             _attempt_seq = $14::integer,
-             _updated_at = now(),
-             _version = ${nextRuntimeSheetVersionExpression(session)},
-             _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
-         FROM matched_updates AS updates, ${sheetTable(session)} AS prev
-         WHERE target._key = updates.matched_key
+       )`;
+    const writableRowsSql = `target._key = updates.matched_key
            AND ($16::bigint IS NULL OR target._write_version = $16::bigint)
-           AND prev._key = target._key
            AND ($15::boolean OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
            AND (
@@ -8183,7 +8354,66 @@ async function failRuntimeMapRowChunks(
              AND target._run_id = $6::text
              AND target._status IN ('failed', 'enriched')
              AND updates.submitted_key = target._key
-           )
+           )`;
+    const mutationParams = [
+      chunkKeys,
+      chunkInputIndexes,
+      chunkDataPatches,
+      chunkCellMetaPatches,
+      chunkErrors,
+      input.runId,
+      input.normalizedPlayName,
+      input.normalizedTableNamespace,
+      failedCellFields,
+      failedCellTotals,
+      input.attemptId,
+      input.attemptOwnerRunId,
+      input.attemptExpiresAt,
+      input.attemptSeq,
+      input.forceTerminal === true,
+      input.writeVersion,
+    ];
+    const columnTypes = await RuntimeSheetColumns.prepareWrite(client, {
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      columns: input.physicalColumnProjections,
+      rows: async () => {
+        const accepted = await client.query<{
+          data_patch: Record<string, unknown>;
+        }>(
+          `${matchingUpdatesSql}
+           SELECT updates.data_patch
+           FROM matched_updates AS updates
+           JOIN ${sheetTable(session)} AS target ON target._key = updates.matched_key
+           WHERE ${writableRowsSql}
+           FOR UPDATE OF target`,
+          mutationParams,
+        );
+        return accepted.rows.map((row) => row.data_patch);
+      },
+    });
+    const { rows } = await client.query<{
+      updated?: number;
+      fenced_rows?: unknown;
+    }>(
+      `${matchingUpdatesSql},
+       applied_rows AS (
+         UPDATE ${sheetTable(session)} AS target
+         SET _status = 'failed',
+             _run_id = $6::text,
+             _writer_run_id = $6::text,
+             _write_version = COALESCE($16::bigint, target._write_version),
+             _error = updates.error,
+             _attempt_id = $11::text,
+             _attempt_owner_run_id = $12::text,
+             _attempt_expires_at = $13::timestamptz,
+             _attempt_seq = $14::integer,
+             _updated_at = now(),
+             _version = ${nextRuntimeSheetVersionExpression(session)},
+             _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${physicalRuntimeSheetUpdateSql(session, input.physicalColumnProjections, columnTypes)}
+         FROM matched_updates AS updates, ${sheetTable(session)} AS prev
+         WHERE ${writableRowsSql}
+           AND prev._key = target._key
          RETURNING target._key, prev._status AS prev_status
        ),
        applied_count AS (
@@ -8280,24 +8510,7 @@ async function failRuntimeMapRowChunks(
        SELECT
          (SELECT count(*)::int FROM applied_rows) AS updated,
          coalesce((SELECT jsonb_agg(jsonb_build_object('key', _key, 'reason', reason, 'rowVersionStamp', row_write_version)) FROM fenced_rows), '[]'::jsonb) AS fenced_rows`,
-      [
-        chunkKeys,
-        chunkInputIndexes,
-        chunkDataPatches,
-        chunkCellMetaPatches,
-        chunkErrors,
-        input.runId,
-        input.normalizedPlayName,
-        input.normalizedTableNamespace,
-        failedCellFields,
-        failedCellTotals,
-        input.attemptId,
-        input.attemptOwnerRunId,
-        input.attemptExpiresAt,
-        input.attemptSeq,
-        input.forceTerminal === true,
-        input.writeVersion,
-      ],
+      mutationParams,
     );
     const appliedCount = Number(rows[0]?.updated ?? 0);
     const chunkFencedRows = parseRuntimeMapRowsFencedRows(rows[0]?.fenced_rows);
@@ -8433,20 +8646,6 @@ export async function completeRuntimeMapRows(
   }
 
   const projections = physicalSheetColumnProjections(sheetContract);
-  const physicalUpdateSetSql =
-    projections.length > 0
-      ? `,\n        ${projections
-          .map((column) => {
-            const quoted = quoteIdentifier(column.sqlName);
-            const literal = quoteLiteral(column.fieldName);
-            return `${quoted} = CASE
-          WHEN coalesce(updates.data_patch, '{}'::jsonb) ? ${literal}
-            THEN updates.data_patch -> ${literal}
-          ELSE target.${quoted}
-        END`;
-          })
-          .join(',\n        ')}`
-      : '';
 
   const { completedRows, failedRows } = prepareRuntimeSheetRowTransitions({
     rows: uniqueRows.values(),
@@ -8455,7 +8654,7 @@ export async function completeRuntimeMapRows(
   });
   const chunks = chunkValues(completedRows, DIRECT_POSTGRES_BATCH_SIZE);
   const failedChunks = chunkValues(failedRows, DIRECT_POSTGRES_BATCH_SIZE);
-  const needsTransaction = chunks.length + failedChunks.length > 1;
+  const needsTransaction = true;
   const outputFields = [...new Set(input.outputFields ?? [])];
   const attemptId = input.attemptId?.trim() || null;
   const attemptOwnerRunId =
@@ -8488,7 +8687,6 @@ export async function completeRuntimeMapRows(
           ? await (async () => {
               const chunkInput = {
                 chunks,
-                physicalUpdateSetSql,
                 physicalColumnProjections: projections,
                 runId: input.runId,
                 attemptId,
@@ -8522,7 +8720,6 @@ export async function completeRuntimeMapRows(
         failedRows.length > 0
           ? await failRuntimeMapRowChunks(client, session, {
               chunks: failedChunks,
-              physicalUpdateSetSql,
               physicalColumnProjections: projections,
               forceTerminal: input.forceFailedRows === true,
               runId: input.runId,