deepline 0.1.119 → 0.1.121

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +4 -0
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +21 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +185 -0
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-batch.ts +107 -0
  5. package/dist/{repo → bundling-sources}/sdk/src/client.ts +116 -12
  6. package/dist/bundling-sources/sdk/src/compat.ts +191 -0
  7. package/dist/bundling-sources/sdk/src/gtm.ts +146 -0
  8. package/dist/bundling-sources/sdk/src/helpers.ts +12 -0
  9. package/dist/{repo → bundling-sources}/sdk/src/index.ts +2 -1
  10. package/dist/{repo → bundling-sources}/sdk/src/play.ts +3 -1
  11. package/dist/{repo → bundling-sources}/sdk/src/plays/bundle-play-file.ts +17 -5
  12. package/dist/{repo → bundling-sources}/sdk/src/release.ts +2 -2
  13. package/dist/{repo → bundling-sources}/sdk/src/runs/observe-transport.ts +2 -3
  14. package/dist/bundling-sources/shared_libs/play-data-plane/index.ts +3 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +838 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5510 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +261 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +828 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +50 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +20 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/event-wait-tools.ts +9 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +171 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-diagnosis.ts +321 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-target.ts +158 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/internal-step-ids.ts +34 -0
  27. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +34 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +50 -0
  29. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +119 -0
  30. package/dist/{repo → bundling-sources}/shared_libs/play-runtime/map-row-identity.ts +1 -1
  31. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +636 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts +9 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/progress-emitter.ts +197 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/projection.ts +262 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +143 -0
  36. package/dist/bundling-sources/shared_libs/play-runtime/public-play-contract.ts +42 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +40 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +178 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +4015 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/runtime-constraints.ts +2 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +238 -0
  42. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +53 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +149 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +68 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/tool-batch-executor.ts +149 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +159 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/tracing.ts +33 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/waterfall-replay.ts +79 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +139 -0
  50. package/dist/bundling-sources/shared_libs/plays/artifact-transport.ts +14 -0
  51. package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +49 -0
  52. package/dist/bundling-sources/shared_libs/plays/compiler-manifest.ts +41 -0
  53. package/dist/bundling-sources/shared_libs/plays/dataset-summary.ts +163 -0
  54. package/dist/bundling-sources/shared_libs/plays/definition.ts +267 -0
  55. package/dist/bundling-sources/shared_libs/plays/file-refs.ts +11 -0
  56. package/dist/bundling-sources/shared_libs/plays/input-contract.ts +146 -0
  57. package/dist/bundling-sources/shared_libs/plays/resolve-static-pipeline.ts +190 -0
  58. package/dist/bundling-sources/shared_libs/plays/runtime-validation.ts +417 -0
  59. package/dist/bundling-sources/shared_libs/plays/tool-codegen.ts +142 -0
  60. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +274 -0
  61. package/dist/bundling-sources/shared_libs/temporal/preview-config.ts +150 -0
  62. package/dist/cli/index.js +811 -2207
  63. package/dist/cli/index.mjs +847 -2258
  64. package/dist/compiler-manifest-BjoRENv9.d.mts +227 -0
  65. package/dist/compiler-manifest-BjoRENv9.d.ts +227 -0
  66. package/dist/index.d.mts +8 -231
  67. package/dist/index.d.ts +8 -231
  68. package/dist/index.js +101 -15
  69. package/dist/index.mjs +101 -15
  70. package/dist/plays/bundle-play-file.d.mts +120 -0
  71. package/dist/plays/bundle-play-file.d.ts +120 -0
  72. package/dist/plays/bundle-play-file.mjs +1830 -0
  73. package/package.json +4 -9
  74. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/child-play-await.ts +0 -0
  75. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/child-play-submit.ts +0 -0
  76. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/coordinator-entry.ts +0 -0
  77. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/dedup-do.ts +0 -0
  78. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/entry.ts +0 -0
  79. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/csv-rows.ts +0 -0
  80. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/dataset-handles.ts +0 -0
  81. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +0 -0
  82. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/live-progress.ts +0 -0
  83. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +0 -0
  84. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/receipts.ts +0 -0
  85. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/row-isolation.ts +0 -0
  86. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/runtime/tool-http-errors.ts +0 -0
  87. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/workflow-instance-create.ts +0 -0
  88. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/workflow-retry-state.ts +0 -0
  89. /package/dist/{repo → bundling-sources}/apps/play-runner-workers/src/workflow-retry.ts +0 -0
  90. /package/dist/{repo → bundling-sources}/sdk/src/agent-runtime.ts +0 -0
  91. /package/dist/{repo → bundling-sources}/sdk/src/config.ts +0 -0
  92. /package/dist/{repo → bundling-sources}/sdk/src/errors.ts +0 -0
  93. /package/dist/{repo → bundling-sources}/sdk/src/http.ts +0 -0
  94. /package/dist/{repo → bundling-sources}/sdk/src/plays/harness-stub.ts +0 -0
  95. /package/dist/{repo → bundling-sources}/sdk/src/plays/local-file-discovery.ts +0 -0
  96. /package/dist/{repo → bundling-sources}/sdk/src/stream-reconnect.ts +0 -0
  97. /package/dist/{repo → bundling-sources}/sdk/src/tool-output.ts +0 -0
  98. /package/dist/{repo → bundling-sources}/sdk/src/types.ts +0 -0
  99. /package/dist/{repo → bundling-sources}/sdk/src/version.ts +0 -0
  100. /package/dist/{repo → bundling-sources}/sdk/src/worker-play-entry.ts +0 -0
  101. /package/dist/{repo → bundling-sources}/shared_libs/play-data-plane/cell-policy.ts +0 -0
  102. /package/dist/{repo → bundling-sources}/shared_libs/play-data-plane/column-names.ts +0 -0
  103. /package/dist/{repo → bundling-sources}/shared_libs/play-data-plane/sheet-contract.ts +0 -0
  104. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/backend.ts +0 -0
  105. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/batch-runtime.ts +0 -0
  106. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/batching-types.ts +0 -0
  107. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/cell-staleness.ts +0 -0
  108. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/coordinator-headers.ts +0 -0
  109. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/csv-rename.ts +0 -0
  110. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/db-session-crypto.ts +0 -0
  111. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/db-session-plan.ts +0 -0
  112. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/db-session.ts +0 -0
  113. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/dedup-backend.ts +0 -0
  114. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/default-batch-strategies.ts +0 -0
  115. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/email-status.ts +0 -0
  116. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/execution-plan.ts +0 -0
  117. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/extractor-targets.ts +0 -0
  118. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/fullenrich-batching.ts +0 -0
  119. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +0 -0
  120. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/governor/governor.ts +0 -0
  121. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/governor/policy.ts +0 -0
  122. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/governor/rate-state-backend.ts +0 -0
  123. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/live-events.ts +0 -0
  124. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/play-runtime-batching-registry.ts +0 -0
  125. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/profiles.ts +0 -0
  126. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/providers.ts +0 -0
  127. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/run-failure.ts +0 -0
  128. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/run-ledger.ts +0 -0
  129. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/run-snapshot-stream.ts +0 -0
  130. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/scheduler-backend.ts +0 -0
  131. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/secret-capability.ts +0 -0
  132. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/secret-redaction.ts +0 -0
  133. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/step-lifecycle-tracker.ts +0 -0
  134. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/step-program-dataset-builder.ts +0 -0
  135. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/submit-limits.ts +0 -0
  136. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/tool-result.ts +0 -0
  137. /package/dist/{repo → bundling-sources}/shared_libs/play-runtime/work-receipts.ts +0 -0
  138. /package/dist/{repo → bundling-sources}/shared_libs/plays/bootstrap-routes.ts +0 -0
  139. /package/dist/{repo → bundling-sources}/shared_libs/plays/bundling/index.ts +0 -0
  140. /package/dist/{repo → bundling-sources}/shared_libs/plays/bundling/limits.ts +0 -0
  141. /package/dist/{repo → bundling-sources}/shared_libs/plays/contracts.ts +0 -0
  142. /package/dist/{repo → bundling-sources}/shared_libs/plays/dataset.ts +0 -0
  143. /package/dist/{repo → bundling-sources}/shared_libs/plays/row-identity.ts +0 -0
  144. /package/dist/{repo → bundling-sources}/shared_libs/plays/secret-guardrails.ts +0 -0
  145. /package/dist/{repo → bundling-sources}/shared_libs/plays/static-pipeline.ts +0 -0
  146. /package/dist/{repo → bundling-sources}/shared_libs/security/outbound-url-policy.ts +0 -0
  147. /package/dist/{repo → bundling-sources}/shared_libs/security/safe-fetch.ts +0 -0
  148. /package/dist/{repo → bundling-sources}/shared_libs/temporal/constants.ts +0 -0
@@ -0,0 +1,4015 @@
1
+ import { createHash } from 'node:crypto';
2
+ import {
3
+ createRuntimePool,
4
+ createRuntimeOneShotQueryClient,
5
+ isRuntimeOneShotQueryFactoryRegistered,
6
+ type RuntimePool,
7
+ type RuntimePoolClient,
8
+ type RuntimeOneShotQueryClient,
9
+ } from './runtime-pg-driver';
10
+ import { createDeferredPlayDataset, type PlayDataset } from '../plays/dataset';
11
+ import type {
12
+ PlayStaticPipeline,
13
+ PlaySheetContract,
14
+ } from '../plays/static-pipeline';
15
+ import type { PlayBundleArtifact } from '../plays/artifact-types';
16
+ import {
17
+ augmentSheetContractWithDatasetFields,
18
+ outputFieldsFromSheetContract,
19
+ outputPhysicalSheetColumnNames,
20
+ outputPhysicalSheetColumnProjections,
21
+ physicalSheetColumnNames,
22
+ physicalSheetColumnProjections,
23
+ type PhysicalSheetColumnProjection,
24
+ } from '../play-data-plane/sheet-contract';
25
+ import type {
26
+ CreateDbSessionResponse,
27
+ DbLogicalTable,
28
+ DbSessionLimits,
29
+ DbSessionOperation,
30
+ PreloadedRuntimeDbSession,
31
+ RowsWriteResponse,
32
+ } from './db-session';
33
+ import {
34
+ derivePlayRowIdentity,
35
+ normalizePlayNameForSheet,
36
+ normalizeTableNamespace,
37
+ } from '../plays/row-identity';
38
+ import { toSerializableCsvAliasedRow } from './csv-rename';
39
+ import {
40
+ RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
41
+ RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
42
+ RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
43
+ createDbSessionResponseSchema,
44
+ rowsWriteResponseSchema,
45
+ } from './db-session';
46
+ import {
47
+ dbSessionPostgresUrlAad,
48
+ decryptDbSessionPostgresUrl,
49
+ decryptDbSessionPostgresUrlWithPrivateKey,
50
+ generateDbSessionPostgresUrlDecryptionKey,
51
+ type PostgresUrlDecryptionKey,
52
+ } from './db-session-crypto';
53
+ import { createRuntimeDatasetId } from './dataset-id';
54
+ import {
55
+ scopedWorkReceiptKeyPrefix,
56
+ isReusableWorkReceipt,
57
+ type WorkReceipt,
58
+ type WorkReceiptClaim,
59
+ } from './work-receipts';
60
+ import { stringifyPostgresJson } from './postgres-json';
61
+ import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
62
+ import type { MapRowOutcome } from './durability-store';
63
+ import {
64
+ DEEPLINE_CELL_META_FIELD,
65
+ cellPolicyFields,
66
+ shouldRecomputeCell,
67
+ type CellStalenessPolicyByField,
68
+ } from './cell-staleness';
69
+ import type { PlayArtifactKind } from './backend';
70
+
71
+ type RuntimeApiContext = {
72
+ baseUrl?: string | null;
73
+ executorToken?: string | null;
74
+ orgId?: string | null;
75
+ playName?: string | null;
76
+ runId?: string | null;
77
+ userEmail?: string | null;
78
+ dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | null;
79
+ preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
80
+ vercelProtectionBypassToken?: string | null;
81
+ disablePostgresPoolCache?: boolean | null;
82
+ postgresSessionUnwrapKey?: string | null;
83
+ };
84
+
85
+ type DbSessionCacheEntry = {
86
+ session: CreateDbSessionResponse;
87
+ };
88
+
89
+ type RuntimeQueryClient = Pick<RuntimeOneShotQueryClient, 'query'>;
90
+ type RuntimeDatasetRowEntry = {
91
+ key: string;
92
+ row: Record<string, unknown>;
93
+ inputIndex: number;
94
+ };
95
+ type RuntimePreparedCompletedRow = {
96
+ key: string;
97
+ input_index: number | null;
98
+ data_patch: Record<string, unknown>;
99
+ cell_meta_patch: Record<string, unknown>;
100
+ };
101
+
102
+ type RuntimePreparedFailedRow = RuntimePreparedCompletedRow & {
103
+ /** Row-level error persisted to `_error`; never empty. */
104
+ error: string;
105
+ };
106
+
107
+ const dbSessionCache = new Map<string, DbSessionCacheEntry>();
108
+ const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
109
+ const postgresPools = new Map<string, RuntimePool>();
110
+ const runtimeWorkReceiptEnsureCache = new Map<string, Promise<void>>();
111
+ const runtimeSheetEnsureCache = new Map<
112
+ string,
113
+ { expiresAt: number; promise: Promise<void> }
114
+ >();
115
+ const DIRECT_POSTGRES_BATCH_SIZE = 10_000;
116
+ const RUNTIME_DB_SESSION_ROW_LIMIT_FLOOR = DIRECT_POSTGRES_BATCH_SIZE;
117
+ const APPEND_KEY_SUFFIX_LENGTH = 12;
118
+ const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
119
+ const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
120
+ const RUNTIME_DB_SESSION_RENEWAL_WINDOW_MS = 60_000;
121
+ const RUNTIME_API_RETRY_DELAYS_MS = [
122
+ 250, 500, 1_000, 2_000, 4_000, 8_000, 8_000,
123
+ ] as const;
124
+ const RUNTIME_API_MAX_ATTEMPTS = RUNTIME_API_RETRY_DELAYS_MS.length + 1;
125
+ const RUNTIME_API_DEFAULT_RETRY_AFTER_MS = 2_000;
126
+ const RUNTIME_API_REQUEST_TIMEOUT_MS = 30_000;
127
+ const vercelProtectionCookieCache = new Map<string, Promise<string | null>>();
128
+ const RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS = 4;
129
+ const RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
130
+ const RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS = 4;
131
+ const RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS = [250, 750, 1_500] as const;
132
+ // Runtime DB sessions are minted against the pooled tenant endpoint. A healthy
133
+ // connect is sub-second; spending minutes on one sandbox dial only hides a
134
+ // broken route and stalls the whole play. Keep retries bounded and loud.
135
+ const RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS = 5_000;
136
+ // Daytona executes one play per runner process. This is the bounded data-plane
137
+ // budget for that one run's sheet reads/writes, receipts, and terminal/export
138
+ // reads over the PgBouncer URL; it is not a scheduler/control-plane fan-out.
139
+ const RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS = 8;
140
+ const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
141
+ const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
142
+ const RECEIPT_STATUS_COMPLETED = RECEIPT_STATUS_CODE.completed;
143
+ const RECEIPT_STATUS_FAILED = RECEIPT_STATUS_CODE.failed;
144
+ const RECEIPT_STATUS_SKIPPED = RECEIPT_STATUS_CODE.skipped;
145
+
146
+ export type ResolvedRuntimePlay = {
147
+ playId: string;
148
+ sourceCode?: string | null;
149
+ artifact?: PlayBundleArtifact | null;
150
+ codeFormat?: 'function' | 'cjs_module' | 'esm_module';
151
+ contractSnapshot?: Record<string, unknown> | null;
152
+ };
153
+
154
+ export type PrepareRuntimeSheetResult = {
155
+ inserted: number;
156
+ skipped: number;
157
+ pendingRows: Record<string, unknown>[];
158
+ completedRows: Record<string, unknown>[];
159
+ tableNamespace: string;
160
+ timings?: RuntimeSheetTiming[];
161
+ };
162
+
163
+ export type RuntimeSheetTiming = {
164
+ phase: string;
165
+ ms: number;
166
+ rows?: number;
167
+ chunks?: number;
168
+ inserted?: number;
169
+ skipped?: number;
170
+ pending?: number;
171
+ completed?: number;
172
+ ready?: boolean;
173
+ cached?: boolean;
174
+ retried?: boolean;
175
+ error?: string;
176
+ };
177
+
178
+ type RuntimeApiActionRequest =
179
+ | {
180
+ action: 'resolve_play';
181
+ playRef: string;
182
+ /**
183
+ * Artifact kind the caller can execute. Node runners (postgres
184
+ * scheduler inline executor, Daytona/local-process) request
185
+ * `cjs_node20` so a child published with an esm_workers artifact is
186
+ * re-bundled server-side into a `Module._compile`-loadable module.
187
+ * Omitted = the stored artifact is returned verbatim.
188
+ */
189
+ artifactKind?: PlayArtifactKind;
190
+ }
191
+ | {
192
+ action: 'ensure_sheet';
193
+ playName: string;
194
+ runId?: string | null;
195
+ tableNamespace: string;
196
+ sheetContract?: PlaySheetContract | null;
197
+ userEmail?: string | null;
198
+ }
199
+ | {
200
+ action: 'create_db_session';
201
+ playName: string;
202
+ runId?: string | null;
203
+ target: {
204
+ tableNamespace: string;
205
+ logicalTable: DbLogicalTable;
206
+ };
207
+ operations: DbSessionOperation[];
208
+ limits?: {
209
+ maxRows?: number;
210
+ maxBytes?: number;
211
+ maxRequests?: number;
212
+ };
213
+ sheetContract?: PlaySheetContract | null;
214
+ ttlSeconds?: number;
215
+ userEmail?: string | null;
216
+ postgresUrlEncryption?: {
217
+ alg: 'RSA-OAEP-256+A256GCM';
218
+ publicKeyJwk: JsonWebKey;
219
+ } | null;
220
+ };
221
+
222
+ type RuntimeApiRowRecord = MapRowOutcome & {
223
+ inputIndex?: number | null;
224
+ };
225
+
226
+ type RuntimePostgresSession = CreateDbSessionResponse & {
227
+ postgresUrl: string;
228
+ postgres: NonNullable<CreateDbSessionResponse['postgres']>;
229
+ };
230
+
231
+ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
232
+ const baseUrl = context.baseUrl?.trim();
233
+ if (!baseUrl) {
234
+ throw new Error('Runner runtime API requires a baseUrl.');
235
+ }
236
+ const url = new URL(
237
+ `${baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`,
238
+ );
239
+ const bypassToken = context.vercelProtectionBypassToken?.trim();
240
+ if (bypassToken) {
241
+ url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
242
+ url.searchParams.set('x-vercel-protection-bypass', bypassToken);
243
+ }
244
+ return url.toString();
245
+ }
246
+
247
+ function resolveRuntimeApiHeaders(
248
+ context: RuntimeApiContext,
249
+ vercelProtectionCookie?: string | null,
250
+ ): Record<string, string> {
251
+ const token = context.executorToken?.trim();
252
+ if (!token) {
253
+ throw new Error('Runner runtime API requires an executorToken.');
254
+ }
255
+ return {
256
+ 'content-type': 'application/json',
257
+ authorization: `Bearer ${token}`,
258
+ ...(context.vercelProtectionBypassToken
259
+ ? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
260
+ : {}),
261
+ ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
262
+ };
263
+ }
264
+
265
+ function setCookieHeaders(headers: Headers): string[] {
266
+ const getter = (headers as Headers & { getSetCookie?: () => string[] })
267
+ .getSetCookie;
268
+ if (typeof getter === 'function') {
269
+ return getter.call(headers);
270
+ }
271
+ const single = headers.get('set-cookie');
272
+ return single ? [single] : [];
273
+ }
274
+
275
+ function cookieHeaderFromSetCookie(headers: Headers): string | null {
276
+ const cookies = setCookieHeaders(headers)
277
+ .flatMap((header) => header.split(/,(?=\s*[^;,=]+=[^;,]+)/g))
278
+ .map((header) => header.split(';', 1)[0]?.trim() ?? '')
279
+ .filter(Boolean);
280
+ return cookies.length > 0 ? cookies.join('; ') : null;
281
+ }
282
+
283
+ function resolveVercelProtectionCookie(
284
+ context: RuntimeApiContext,
285
+ ): Promise<string | null> {
286
+ const bypassToken = context.vercelProtectionBypassToken?.trim();
287
+ const baseUrl = context.baseUrl?.trim().replace(/\/$/, '');
288
+ if (!bypassToken || !baseUrl) return Promise.resolve(null);
289
+
290
+ const cacheKey = `${baseUrl}\n${bypassToken}`;
291
+ const cached = vercelProtectionCookieCache.get(cacheKey);
292
+ if (cached) return cached;
293
+
294
+ const promise = (async () => {
295
+ const url = new URL(`${baseUrl}/api/v2/health`);
296
+ url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
297
+ url.searchParams.set('x-vercel-protection-bypass', bypassToken);
298
+ const response = await fetch(url.toString(), {
299
+ headers: { 'x-vercel-protection-bypass': bypassToken },
300
+ }).catch(() => null);
301
+ return response ? cookieHeaderFromSetCookie(response.headers) : null;
302
+ })();
303
+ vercelProtectionCookieCache.set(cacheKey, promise);
304
+ return promise;
305
+ }
306
+
307
+ function normalizeRuntimeUserEmail(
308
+ value: string | null | undefined,
309
+ ): string | null {
310
+ const email = value?.trim();
311
+ return email ? email : null;
312
+ }
313
+
314
+ async function postRuntimeApi<TResponse>(
315
+ context: RuntimeApiContext,
316
+ body: RuntimeApiActionRequest,
317
+ ): Promise<TResponse> {
318
+ const url = resolveRuntimeApiUrl(context);
319
+ const vercelProtectionCookie = await resolveVercelProtectionCookie(context);
320
+ for (let attempt = 1; attempt <= RUNTIME_API_MAX_ATTEMPTS; attempt += 1) {
321
+ let response: Response;
322
+ const abortController = new AbortController();
323
+ const timeout = setTimeout(
324
+ () => abortController.abort(),
325
+ RUNTIME_API_REQUEST_TIMEOUT_MS,
326
+ );
327
+ try {
328
+ response = await fetch(url, {
329
+ method: 'POST',
330
+ headers: resolveRuntimeApiHeaders(context, vercelProtectionCookie),
331
+ body: JSON.stringify(body),
332
+ signal: abortController.signal,
333
+ });
334
+ } catch (error) {
335
+ if (attempt < RUNTIME_API_MAX_ATTEMPTS) {
336
+ await sleepRuntimeApiRetry(attempt);
337
+ continue;
338
+ }
339
+ const message = error instanceof Error ? error.message : String(error);
340
+ throw new Error(
341
+ `Runtime API request to ${url} failed before receiving a response: ${message}`,
342
+ );
343
+ } finally {
344
+ clearTimeout(timeout);
345
+ }
346
+
347
+ const parsed = (await response.json().catch(() => null)) as Record<
348
+ string,
349
+ unknown
350
+ > | null;
351
+ if (response.ok) {
352
+ return parsed as TResponse;
353
+ }
354
+
355
+ const retryAfterMs =
356
+ typeof parsed?.retry_after_ms === 'number' &&
357
+ Number.isFinite(parsed.retry_after_ms)
358
+ ? parsed.retry_after_ms
359
+ : RUNTIME_API_DEFAULT_RETRY_AFTER_MS;
360
+ const details =
361
+ typeof parsed?.details === 'string'
362
+ ? parsed.details
363
+ : typeof parsed?.debug_error === 'string'
364
+ ? parsed.debug_error
365
+ : null;
366
+ const shouldRetryRuntimeResponse =
367
+ (response.status === 503 &&
368
+ parsed?.code === 'ingestion_plane_not_ready') ||
369
+ response.status === 408 ||
370
+ response.status === 429 ||
371
+ response.status === 502 ||
372
+ response.status === 503 ||
373
+ response.status === 504 ||
374
+ (response.status === 500 &&
375
+ isTransientRuntimeApiErrorMessage(
376
+ `${typeof parsed?.error === 'string' ? parsed.error : ''}\n${details ?? ''}`,
377
+ ));
378
+ if (shouldRetryRuntimeResponse && attempt < RUNTIME_API_MAX_ATTEMPTS) {
379
+ await sleepRuntimeApiRetry(attempt, retryAfterMs);
380
+ continue;
381
+ }
382
+
383
+ const errorMessage =
384
+ typeof parsed?.error === 'string'
385
+ ? parsed.error
386
+ : `Runtime API request failed with status ${response.status}.`;
387
+ throw new Error(
388
+ details
389
+ ? `${errorMessage} (status ${response.status}): ${details}`
390
+ : `${errorMessage} (status ${response.status}).`,
391
+ );
392
+ }
393
+
394
+ throw new Error('Runtime API request failed after retries.');
395
+ }
396
+
397
+ function runtimeApiRetryDelayMs(
398
+ attempt: number,
399
+ retryAfterMs?: number,
400
+ ): number {
401
+ const configuredDelay =
402
+ RUNTIME_API_RETRY_DELAYS_MS[attempt - 1] ??
403
+ RUNTIME_API_DEFAULT_RETRY_AFTER_MS;
404
+ return retryAfterMs === undefined
405
+ ? configuredDelay
406
+ : Math.max(retryAfterMs, configuredDelay);
407
+ }
408
+
409
+ async function sleepRuntimeApiRetry(
410
+ attempt: number,
411
+ retryAfterMs?: number,
412
+ ): Promise<void> {
413
+ await new Promise((resolve) =>
414
+ setTimeout(resolve, runtimeApiRetryDelayMs(attempt, retryAfterMs)),
415
+ );
416
+ }
417
+
418
+ function isTransientRuntimeApiErrorMessage(message: string): boolean {
419
+ return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access/i.test(
420
+ message,
421
+ );
422
+ }
423
+
424
+ function getDbSessionCacheKey(input: {
425
+ baseUrl?: string | null;
426
+ executorToken?: string | null;
427
+ playName: string;
428
+ runId?: string | null;
429
+ tableNamespace: string;
430
+ logicalTable: DbLogicalTable;
431
+ operations: DbSessionOperation[];
432
+ limits?: DbSessionLimits;
433
+ sheetContract?: PlaySheetContract | null;
434
+ userEmail?: string | null;
435
+ }): string {
436
+ // Worker processes are long-lived. Include a hash of the executor token plus
437
+ // the full requested access shape so a pooled runtime cannot reuse a scoped
438
+ // Postgres URL across orgs, runs, logical tables, or privilege sets.
439
+ const tokenHash = createHash('sha256')
440
+ .update(input.executorToken?.trim() ?? '')
441
+ .digest('hex')
442
+ .slice(0, 24);
443
+ return [
444
+ input.baseUrl?.trim() ?? '',
445
+ tokenHash,
446
+ input.playName,
447
+ input.runId?.trim() ?? '',
448
+ input.tableNamespace,
449
+ input.logicalTable,
450
+ [...input.operations].sort().join(','),
451
+ JSON.stringify(input.limits ?? {}),
452
+ JSON.stringify(input.sheetContract ?? null),
453
+ input.userEmail?.trim() ?? '',
454
+ ].join('::');
455
+ }
456
+
457
+ function getRuntimeSheetEnsureCacheKey(input: {
458
+ baseUrl?: string | null;
459
+ orgId?: string | null;
460
+ playName: string;
461
+ runId?: string | null;
462
+ tableNamespace: string;
463
+ sheetContract: PlaySheetContract;
464
+ userEmail?: string | null;
465
+ }): string {
466
+ const contractHash = createHash('sha256')
467
+ .update(JSON.stringify(input.sheetContract))
468
+ .digest('hex')
469
+ .slice(0, 24);
470
+ return [
471
+ input.baseUrl?.trim() ?? '',
472
+ input.orgId?.trim() ?? '',
473
+ input.playName,
474
+ input.runId?.trim() ?? '',
475
+ normalizeTableNamespace(input.tableNamespace),
476
+ contractHash,
477
+ input.userEmail?.trim() ?? '',
478
+ ].join('::');
479
+ }
480
+
481
+ async function isRuntimeSheetSchemaReady(
482
+ session: RuntimePostgresSession,
483
+ input: {
484
+ sheetContract: PlaySheetContract;
485
+ },
486
+ ): Promise<boolean> {
487
+ const physicalColumns = physicalSheetColumnNames(input.sheetContract);
488
+ const selectList =
489
+ physicalColumns.length > 0
490
+ ? physicalColumns.map(quoteIdentifier).join(', ')
491
+ : '1';
492
+ const query = async (client: RuntimeQueryClient): Promise<void> => {
493
+ await client.query(
494
+ `SELECT ${selectList}
495
+ FROM ${sheetTable(session)}
496
+ LIMIT 0`,
497
+ );
498
+ };
499
+ try {
500
+ if (isRuntimeOneShotQueryFactoryRegistered()) {
501
+ await withRuntimeOneShotPostgres(session, query);
502
+ } else {
503
+ await withRuntimePostgres(session, query);
504
+ }
505
+ return true;
506
+ } catch (error) {
507
+ if (isMissingRelationError(error)) {
508
+ return false;
509
+ }
510
+ throw error;
511
+ }
512
+ }
513
+
514
+ async function ensureRuntimeSheetForPreloadedSession(
515
+ context: RuntimeApiContext & { playName: string },
516
+ input: {
517
+ tableNamespace: string;
518
+ sheetContract: PlaySheetContract;
519
+ session: RuntimePostgresSession;
520
+ timings?: RuntimeSheetTiming[];
521
+ },
522
+ ): Promise<void> {
523
+ const cacheKey = getRuntimeSheetEnsureCacheKey({
524
+ baseUrl: context.baseUrl,
525
+ orgId: input.session.target.orgId,
526
+ playName: context.playName,
527
+ runId: context.runId,
528
+ tableNamespace: input.tableNamespace,
529
+ sheetContract: input.sheetContract,
530
+ userEmail: normalizeRuntimeUserEmail(context.userEmail),
531
+ });
532
+ const now = Date.now();
533
+ const cached = runtimeSheetEnsureCache.get(cacheKey);
534
+ if (cached && cached.expiresAt > now) {
535
+ input.timings?.push({
536
+ phase: 'ensure_sheet_for_preloaded_session_cached',
537
+ ms: 0,
538
+ cached: true,
539
+ });
540
+ await cached.promise;
541
+ return;
542
+ }
543
+ if (cached) {
544
+ runtimeSheetEnsureCache.delete(cacheKey);
545
+ }
546
+ const checkStartedAt = Date.now();
547
+ const ready = await isRuntimeSheetSchemaReady(input.session, {
548
+ sheetContract: input.sheetContract,
549
+ });
550
+ input.timings?.push({
551
+ phase: 'schema_check_for_preloaded_session',
552
+ ms: Date.now() - checkStartedAt,
553
+ ready,
554
+ });
555
+ if (ready) {
556
+ runtimeSheetEnsureCache.set(cacheKey, {
557
+ expiresAt: now + RUNTIME_SHEET_ENSURE_CACHE_TTL_MS,
558
+ promise: Promise.resolve(),
559
+ });
560
+ return;
561
+ }
562
+ const ensureStartedAt = Date.now();
563
+ const promise = ensureRuntimeSheet(context, {
564
+ playName: context.playName,
565
+ tableNamespace: input.tableNamespace,
566
+ sheetContract: input.sheetContract,
567
+ });
568
+ runtimeSheetEnsureCache.set(cacheKey, {
569
+ expiresAt: now + RUNTIME_SHEET_ENSURE_CACHE_TTL_MS,
570
+ promise,
571
+ });
572
+ try {
573
+ await promise;
574
+ input.timings?.push({
575
+ phase: 'ensure_sheet_for_preloaded_session',
576
+ ms: Date.now() - ensureStartedAt,
577
+ });
578
+ } catch (error) {
579
+ if (runtimeSheetEnsureCache.get(cacheKey)?.promise === promise) {
580
+ runtimeSheetEnsureCache.delete(cacheKey);
581
+ }
582
+ throw error;
583
+ }
584
+ }
585
+
586
+ function operationsSatisfyRequest(
587
+ candidate: readonly DbSessionOperation[],
588
+ requested: readonly DbSessionOperation[],
589
+ ): boolean {
590
+ const candidateOperations = new Set(candidate);
591
+ return requested.every((operation) => candidateOperations.has(operation));
592
+ }
593
+
594
+ function limitsSatisfyRequest(
595
+ candidate: DbSessionLimits | undefined,
596
+ requested: DbSessionLimits | undefined,
597
+ ): boolean {
598
+ const candidateLimits = candidate ?? {};
599
+ const requestedLimits = requested ?? {};
600
+ for (const key of ['maxRows', 'maxBytes', 'maxRequests'] as const) {
601
+ const requestedValue = requestedLimits[key];
602
+ if (requestedValue === undefined) {
603
+ continue;
604
+ }
605
+ const candidateValue = candidateLimits[key];
606
+ // A preloaded session with no advisory limit means the launch contract
607
+ // intentionally authorized the whole run for this target. Runtime scope is
608
+ // enforced by signed session metadata plus constructed SQL identifiers;
609
+ // this fast path does not create per-session database roles or grants.
610
+ if (candidateValue === undefined) {
611
+ continue;
612
+ }
613
+ if (candidateValue < requestedValue) {
614
+ return false;
615
+ }
616
+ }
617
+ return true;
618
+ }
619
+
620
+ function requirePreloadedRuntimeDbSessionOrgId(
621
+ context: Pick<RuntimeApiContext, 'orgId'>,
622
+ ): string {
623
+ const orgId = context.orgId?.trim();
624
+ if (!orgId) {
625
+ throw new Error(
626
+ 'Preloaded Runtime DB sessions require an orgId to validate session scope.',
627
+ );
628
+ }
629
+ return orgId;
630
+ }
631
+
632
+ function shouldEnsureRuntimeSheetBeforeSession(input: {
633
+ logicalTable: DbLogicalTable;
634
+ operations: DbSessionOperation[];
635
+ sheetContract?: PlaySheetContract | null;
636
+ }): input is typeof input & { sheetContract: PlaySheetContract } {
637
+ return (
638
+ input.logicalTable === 'sheet_rows' &&
639
+ !!input.sheetContract &&
640
+ input.operations.some((operation) => operation !== 'rows.read')
641
+ );
642
+ }
643
+
644
+ function sessionHasRenewalWindow(session: CreateDbSessionResponse): boolean {
645
+ const expiresAtMs = Date.parse(session.expiresAt);
646
+ return (
647
+ Number.isFinite(expiresAtMs) &&
648
+ expiresAtMs - Date.now() > RUNTIME_DB_SESSION_RENEWAL_WINDOW_MS
649
+ );
650
+ }
651
+
652
+ function preloadedSessionMatchesRequest(
653
+ context: RuntimeApiContext & { playName: string },
654
+ input: {
655
+ tableNamespace: string;
656
+ logicalTable: DbLogicalTable;
657
+ operations: DbSessionOperation[];
658
+ limits?: DbSessionLimits;
659
+ },
660
+ preloaded: PreloadedRuntimeDbSession,
661
+ expectedOrgId: string,
662
+ ): boolean {
663
+ const session = preloaded.session;
664
+ const requestedTableNamespace = normalizeTableNamespace(input.tableNamespace);
665
+ return (
666
+ sessionHasRenewalWindow(session) &&
667
+ session.target.orgId === expectedOrgId &&
668
+ session.playName === context.playName &&
669
+ normalizeTableNamespace(preloaded.tableNamespace) ===
670
+ requestedTableNamespace &&
671
+ normalizeTableNamespace(session.target.tableNamespace) ===
672
+ requestedTableNamespace &&
673
+ preloaded.logicalTable === input.logicalTable &&
674
+ session.target.logicalTable === input.logicalTable &&
675
+ operationsSatisfyRequest(preloaded.operations, input.operations) &&
676
+ operationsSatisfyRequest(session.operations, input.operations) &&
677
+ limitsSatisfyRequest(preloaded.limits, input.limits) &&
678
+ limitsSatisfyRequest(session.limits, input.limits)
679
+ );
680
+ }
681
+
682
+ function findPreloadedRuntimeDbSession(
683
+ context: RuntimeApiContext & { playName: string },
684
+ input: {
685
+ tableNamespace: string;
686
+ logicalTable: DbLogicalTable;
687
+ operations: DbSessionOperation[];
688
+ limits?: DbSessionLimits;
689
+ },
690
+ ): CreateDbSessionResponse | null {
691
+ if (context.dbSessionStrategy === 'sandbox_public_key') {
692
+ return null;
693
+ }
694
+ const preloadedSessions = context.preloadedDbSessions ?? [];
695
+ if (preloadedSessions.length === 0) {
696
+ return null;
697
+ }
698
+ const expectedOrgId = requirePreloadedRuntimeDbSessionOrgId(context);
699
+ for (const preloaded of preloadedSessions) {
700
+ const parsed = createDbSessionResponseSchema.safeParse(preloaded.session);
701
+ if (!parsed.success) {
702
+ continue;
703
+ }
704
+ const candidate = {
705
+ ...preloaded,
706
+ session: parsed.data,
707
+ };
708
+ if (
709
+ preloadedSessionMatchesRequest(context, input, candidate, expectedOrgId)
710
+ ) {
711
+ return candidate.session;
712
+ }
713
+ }
714
+ return null;
715
+ }
716
+
717
+ async function unwrapRuntimeDbSession(
718
+ context: RuntimeApiContext,
719
+ session: CreateDbSessionResponse,
720
+ decryptionKey?: PostgresUrlDecryptionKey | null,
721
+ ): Promise<CreateDbSessionResponse> {
722
+ if (session.postgresUrl) {
723
+ if (context.postgresSessionUnwrapKey?.trim()) {
724
+ throw new Error(
725
+ 'Harness preloaded Runtime DB sessions must carry an encryptedPostgresUrl, not a raw postgresUrl.',
726
+ );
727
+ }
728
+ return session;
729
+ }
730
+ if (!session.encryptedPostgresUrl) {
731
+ return session;
732
+ }
733
+ if (session.encryptedPostgresUrl.alg === 'RSA-OAEP-256+A256GCM') {
734
+ if (!decryptionKey) {
735
+ throw new Error(
736
+ 'Runtime DB session response used public-key encryption, but no private key was retained for unwrap.',
737
+ );
738
+ }
739
+ const {
740
+ encryptedPostgresUrl: _encryptedPostgresUrl,
741
+ ...sessionWithoutUrl
742
+ } = session;
743
+ void _encryptedPostgresUrl;
744
+ return {
745
+ ...sessionWithoutUrl,
746
+ postgresUrl: await decryptDbSessionPostgresUrlWithPrivateKey({
747
+ encrypted: session.encryptedPostgresUrl,
748
+ privateKey: decryptionKey.privateKey,
749
+ aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
750
+ }),
751
+ };
752
+ }
753
+ if (decryptionKey) {
754
+ throw new Error(
755
+ 'Runtime DB session response used the shared-secret envelope after the runner requested public-key encryption.',
756
+ );
757
+ }
758
+ const unwrapKey = context.postgresSessionUnwrapKey?.trim();
759
+ if (!unwrapKey) {
760
+ throw new Error(
761
+ 'Runtime DB session response is encrypted, but no harness unwrap key was provided.',
762
+ );
763
+ }
764
+ const { encryptedPostgresUrl: _encryptedPostgresUrl, ...sessionWithoutUrl } =
765
+ session;
766
+ void _encryptedPostgresUrl;
767
+ return {
768
+ ...sessionWithoutUrl,
769
+ postgresUrl: await decryptDbSessionPostgresUrl({
770
+ encrypted: session.encryptedPostgresUrl,
771
+ secret: unwrapKey,
772
+ aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
773
+ }),
774
+ };
775
+ }
776
+
777
+ async function getRuntimeDbSession(
778
+ context: RuntimeApiContext & { playName: string },
779
+ input: {
780
+ tableNamespace: string;
781
+ logicalTable: DbLogicalTable;
782
+ operations: DbSessionOperation[];
783
+ limits?: DbSessionLimits;
784
+ sheetContract?: PlaySheetContract | null;
785
+ timings?: RuntimeSheetTiming[];
786
+ },
787
+ ): Promise<CreateDbSessionResponse> {
788
+ const userEmail = normalizeRuntimeUserEmail(context.userEmail);
789
+ const cacheKey = getDbSessionCacheKey({
790
+ baseUrl: context.baseUrl,
791
+ executorToken: context.executorToken,
792
+ playName: context.playName,
793
+ runId: context.runId,
794
+ tableNamespace: input.tableNamespace,
795
+ logicalTable: input.logicalTable,
796
+ operations: input.operations,
797
+ limits: input.limits,
798
+ sheetContract: input.sheetContract,
799
+ userEmail,
800
+ });
801
+ const cached = dbSessionCache.get(cacheKey)?.session;
802
+ if (cached) {
803
+ if (sessionHasRenewalWindow(cached)) {
804
+ return cached;
805
+ }
806
+ await deleteRuntimeDbSessionCacheEntry(cacheKey, cached);
807
+ }
808
+
809
+ const pending = dbSessionInFlight.get(cacheKey);
810
+ if (pending) {
811
+ return await pending;
812
+ }
813
+
814
+ const sessionPromise = (async (): Promise<CreateDbSessionResponse> => {
815
+ const preloaded = findPreloadedRuntimeDbSession(context, input);
816
+ if (preloaded) {
817
+ const unwrappedPreloaded = await unwrapRuntimeDbSession(
818
+ context,
819
+ preloaded,
820
+ );
821
+ if (input.logicalTable === 'sheet_rows' && input.sheetContract) {
822
+ await ensureRuntimeSheetForPreloadedSession(context, {
823
+ tableNamespace: input.tableNamespace,
824
+ sheetContract: input.sheetContract,
825
+ session: requireRuntimePostgresSession(unwrappedPreloaded),
826
+ timings: input.timings,
827
+ });
828
+ }
829
+ dbSessionCache.set(cacheKey, { session: unwrappedPreloaded });
830
+ return unwrappedPreloaded;
831
+ }
832
+
833
+ if (shouldEnsureRuntimeSheetBeforeSession(input)) {
834
+ await ensureRuntimeSheet(context, {
835
+ playName: context.playName,
836
+ tableNamespace: input.tableNamespace,
837
+ sheetContract: input.sheetContract,
838
+ });
839
+ }
840
+
841
+ const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
842
+ const response = await unwrapRuntimeDbSession(
843
+ context,
844
+ createDbSessionResponseSchema.parse(
845
+ await postRuntimeApi<CreateDbSessionResponse>(context, {
846
+ action: 'create_db_session',
847
+ playName: context.playName,
848
+ runId: context.runId ?? null,
849
+ target: {
850
+ tableNamespace: input.tableNamespace,
851
+ logicalTable: input.logicalTable,
852
+ },
853
+ operations: input.operations,
854
+ limits: input.limits,
855
+ sheetContract: input.sheetContract ?? null,
856
+ ttlSeconds: RUNTIME_DB_SESSION_TTL_SECONDS,
857
+ userEmail,
858
+ postgresUrlEncryption: decryptionKey.request,
859
+ }),
860
+ ),
861
+ decryptionKey,
862
+ );
863
+ dbSessionCache.set(cacheKey, { session: response });
864
+ return response;
865
+ })();
866
+ dbSessionInFlight.set(cacheKey, sessionPromise);
867
+ try {
868
+ return await sessionPromise;
869
+ } finally {
870
+ if (dbSessionInFlight.get(cacheKey) === sessionPromise) {
871
+ dbSessionInFlight.delete(cacheKey);
872
+ }
873
+ }
874
+ }
875
+
876
+ async function deleteRuntimeDbSessionCacheEntry(
877
+ cacheKey: string,
878
+ session: CreateDbSessionResponse,
879
+ ): Promise<void> {
880
+ dbSessionCache.delete(cacheKey);
881
+ if (session.postgresUrl) {
882
+ const pool = postgresPools.get(session.postgresUrl);
883
+ postgresPools.delete(session.postgresUrl);
884
+ await pool?.end().catch(() => {});
885
+ }
886
+ }
887
+
888
+ function requireRuntimePostgresSession(
889
+ session: CreateDbSessionResponse,
890
+ ): RuntimePostgresSession {
891
+ if (!session.postgresUrl || !session.postgres) {
892
+ throw new Error(
893
+ 'Runtime DB session did not include a scoped Postgres URL. Direct Postgres sheet IO is required.',
894
+ );
895
+ }
896
+ return session as RuntimePostgresSession;
897
+ }
898
+
899
+ function validatePreloadedRuntimeDbSessionScope(
900
+ context: RuntimeApiContext,
901
+ session: CreateDbSessionResponse,
902
+ ): void {
903
+ const expectedPlayName = context.playName?.trim();
904
+ if (!expectedPlayName) {
905
+ throw new Error(
906
+ 'Preloaded Runtime DB sessions require a playName to validate session scope.',
907
+ );
908
+ }
909
+ if (session.playName !== expectedPlayName) {
910
+ throw new Error(
911
+ 'Preloaded Runtime DB session is outside the requested play scope.',
912
+ );
913
+ }
914
+ const expectedOrgId = requirePreloadedRuntimeDbSessionOrgId(context);
915
+ if (session.target.orgId !== expectedOrgId) {
916
+ throw new Error(
917
+ 'Preloaded Runtime DB session is outside the requested org scope.',
918
+ );
919
+ }
920
+ }
921
+
922
+ function runtimeDbSessionRowLimit(rowCount: number): number {
923
+ return Math.max(
924
+ Math.max(1, Math.floor(rowCount)),
925
+ RUNTIME_DB_SESSION_ROW_LIMIT_FLOOR,
926
+ );
927
+ }
928
+
929
+ export async function prewarmRuntimePostgresSessions(
930
+ context: RuntimeApiContext,
931
+ ): Promise<void> {
932
+ if (context.dbSessionStrategy === 'sandbox_public_key') {
933
+ return;
934
+ }
935
+ const sessions = context.preloadedDbSessions ?? [];
936
+ if (sessions.length === 0) {
937
+ return;
938
+ }
939
+ for (const preloaded of sessions) {
940
+ const parsed = createDbSessionResponseSchema.parse(preloaded.session);
941
+ validatePreloadedRuntimeDbSessionScope(context, parsed);
942
+ const session = requireRuntimePostgresSession(
943
+ await unwrapRuntimeDbSession(context, parsed),
944
+ );
945
+ if (isRuntimeOneShotQueryFactoryRegistered()) {
946
+ await prewarmRuntimeOneShotPostgresSession(session);
947
+ } else {
948
+ await prewarmRuntimePostgresSession(session);
949
+ }
950
+ }
951
+ }
952
+
953
+ async function prewarmRuntimeOneShotPostgresSession(
954
+ session: RuntimePostgresSession,
955
+ ): Promise<void> {
956
+ for (
957
+ let attempt = 1;
958
+ attempt <= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS;
959
+ attempt += 1
960
+ ) {
961
+ try {
962
+ await createRuntimeOneShotQueryClient({
963
+ connectionString: session.postgresUrl,
964
+ }).query('SELECT 1');
965
+ return;
966
+ } catch (error) {
967
+ if (
968
+ attempt >= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS ||
969
+ !isTransientRuntimePostgresConnectionError(error)
970
+ ) {
971
+ throw error;
972
+ }
973
+ await sleep(
974
+ RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[attempt - 1] ??
975
+ RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[
976
+ RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS.length - 1
977
+ ],
978
+ );
979
+ }
980
+ }
981
+ }
982
+
983
+ async function prewarmRuntimePostgresSession(
984
+ session: RuntimePostgresSession,
985
+ ): Promise<void> {
986
+ for (
987
+ let attempt = 1;
988
+ attempt <= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS;
989
+ attempt += 1
990
+ ) {
991
+ try {
992
+ await withRuntimePostgres(session, async (client) => {
993
+ await client.query('SELECT 1');
994
+ });
995
+ return;
996
+ } catch (error) {
997
+ if (
998
+ attempt >= RUNTIME_POSTGRES_PREWARM_MAX_ATTEMPTS ||
999
+ !isTransientRuntimePostgresConnectionError(error)
1000
+ ) {
1001
+ const pool = postgresPools.get(session.postgresUrl);
1002
+ postgresPools.delete(session.postgresUrl);
1003
+ if (pool) {
1004
+ await Promise.resolve(pool.end()).catch(() => {});
1005
+ }
1006
+ throw error;
1007
+ }
1008
+ const pool = postgresPools.get(session.postgresUrl);
1009
+ postgresPools.delete(session.postgresUrl);
1010
+ if (pool) {
1011
+ await Promise.resolve(pool.end()).catch(() => {});
1012
+ }
1013
+ await sleep(
1014
+ RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[attempt - 1] ??
1015
+ RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS[
1016
+ RUNTIME_POSTGRES_PREWARM_RETRY_DELAYS_MS.length - 1
1017
+ ],
1018
+ );
1019
+ }
1020
+ }
1021
+ }
1022
+
1023
+ function isTransientRuntimePostgresConnectionError(error: unknown): boolean {
1024
+ if (!error || typeof error !== 'object') {
1025
+ return false;
1026
+ }
1027
+ const nestedErrors = (error as { errors?: unknown }).errors;
1028
+ if (
1029
+ Array.isArray(nestedErrors) &&
1030
+ nestedErrors.some(isTransientRuntimePostgresConnectionError)
1031
+ ) {
1032
+ return true;
1033
+ }
1034
+ const code = 'code' in error ? String(error.code) : '';
1035
+ if (
1036
+ code === 'ECONNRESET' ||
1037
+ code === 'ETIMEDOUT' ||
1038
+ code === 'ECONNREFUSED' ||
1039
+ code === '57P01'
1040
+ ) {
1041
+ return true;
1042
+ }
1043
+ const name = 'name' in error ? String(error.name) : '';
1044
+ const message = 'message' in error ? String(error.message) : '';
1045
+ return /connection (terminated|timeout|timed out|closed|reset)|ETIMEDOUT|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i.test(
1046
+ `${name} ${message} ${String(error)}`,
1047
+ );
1048
+ }
1049
+
1050
+ function sleep(ms: number): Promise<void> {
1051
+ return new Promise((resolve) => setTimeout(resolve, ms));
1052
+ }
1053
+
1054
+ async function connectRuntimePostgresPool(
1055
+ pool: RuntimePool,
1056
+ ): Promise<RuntimePoolClient> {
1057
+ let timedOut = false;
1058
+ let timeout: ReturnType<typeof setTimeout> | null = null;
1059
+ const connectPromise = pool.connect();
1060
+ connectPromise
1061
+ .then((client) => {
1062
+ if (timedOut) {
1063
+ client.release();
1064
+ }
1065
+ })
1066
+ .catch(() => {});
1067
+ try {
1068
+ return await Promise.race([
1069
+ connectPromise,
1070
+ new Promise<never>((_, reject) => {
1071
+ timeout = setTimeout(() => {
1072
+ timedOut = true;
1073
+ reject(
1074
+ new Error(
1075
+ `Runtime Postgres connection timed out after ${RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS}ms.`,
1076
+ ),
1077
+ );
1078
+ }, RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS);
1079
+ }),
1080
+ ]);
1081
+ } finally {
1082
+ if (timeout) {
1083
+ clearTimeout(timeout);
1084
+ }
1085
+ }
1086
+ }
1087
+
1088
+ function getPostgresPool(postgresUrl: string, cachePool = true): RuntimePool {
1089
+ if (!cachePool) {
1090
+ return createRuntimePool({
1091
+ connectionString: postgresUrl,
1092
+ maxConnections: 1,
1093
+ idleTimeoutMs: 0,
1094
+ connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
1095
+ });
1096
+ }
1097
+ const existing = postgresPools.get(postgresUrl);
1098
+ if (existing) {
1099
+ return existing;
1100
+ }
1101
+ const pool = createRuntimePool({
1102
+ connectionString: postgresUrl,
1103
+ maxConnections: RUNTIME_POSTGRES_POOL_MAX_CONNECTIONS,
1104
+ idleTimeoutMs: 15_000,
1105
+ connectTimeoutMs: RUNTIME_POSTGRES_CONNECT_TIMEOUT_MS,
1106
+ });
1107
+ postgresPools.set(postgresUrl, pool);
1108
+ return pool;
1109
+ }
1110
+
1111
+ function canReuseRuntimePostgresPoolsAcrossRequests(): boolean {
1112
+ return true;
1113
+ }
1114
+
1115
+ async function withRuntimePostgres<T>(
1116
+ session: RuntimePostgresSession,
1117
+ fn: (client: RuntimePoolClient) => Promise<T>,
1118
+ options: { cachePool?: boolean } = {},
1119
+ ): Promise<T> {
1120
+ let client: RuntimePoolClient | null = null;
1121
+ let requestLocalPool: RuntimePool | null = null;
1122
+ const cachePool =
1123
+ (options.cachePool ?? true) && canReuseRuntimePostgresPoolsAcrossRequests();
1124
+ for (
1125
+ let attempt = 1;
1126
+ attempt <= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS;
1127
+ attempt += 1
1128
+ ) {
1129
+ try {
1130
+ const pool = getPostgresPool(session.postgresUrl, cachePool);
1131
+ if (!cachePool) {
1132
+ requestLocalPool = pool;
1133
+ }
1134
+ client = await connectRuntimePostgresPool(pool);
1135
+ break;
1136
+ } catch (error) {
1137
+ if (cachePool) {
1138
+ const pool = postgresPools.get(session.postgresUrl);
1139
+ postgresPools.delete(session.postgresUrl);
1140
+ if (pool) {
1141
+ await Promise.resolve(pool.end()).catch(() => {});
1142
+ }
1143
+ } else if (requestLocalPool) {
1144
+ await Promise.resolve(requestLocalPool.end()).catch(() => {});
1145
+ requestLocalPool = null;
1146
+ }
1147
+ if (
1148
+ attempt >= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS ||
1149
+ !isTransientRuntimePostgresConnectionError(error)
1150
+ ) {
1151
+ throw error;
1152
+ }
1153
+ await sleep(
1154
+ RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS[attempt - 1] ??
1155
+ RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS[
1156
+ RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS.length - 1
1157
+ ],
1158
+ );
1159
+ }
1160
+ }
1161
+ if (!client) {
1162
+ throw new Error('Runtime Postgres connection was not acquired.');
1163
+ }
1164
+ try {
1165
+ return await fn(client);
1166
+ } finally {
1167
+ client.release();
1168
+ if (requestLocalPool) {
1169
+ await Promise.resolve(requestLocalPool.end()).catch(() => {});
1170
+ }
1171
+ }
1172
+ }
1173
+
1174
+ async function withRuntimeOneShotPostgres<T>(
1175
+ session: RuntimePostgresSession,
1176
+ operation: (client: RuntimeOneShotQueryClient) => Promise<T>,
1177
+ ): Promise<T> {
1178
+ for (
1179
+ let attempt = 1;
1180
+ attempt <= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS;
1181
+ attempt += 1
1182
+ ) {
1183
+ try {
1184
+ return await operation(
1185
+ createRuntimeOneShotQueryClient({
1186
+ connectionString: session.postgresUrl,
1187
+ }),
1188
+ );
1189
+ } catch (error) {
1190
+ if (
1191
+ attempt >= RUNTIME_POSTGRES_CONNECT_MAX_ATTEMPTS ||
1192
+ !isTransientRuntimePostgresConnectionError(error)
1193
+ ) {
1194
+ throw error;
1195
+ }
1196
+ await sleep(
1197
+ RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS[attempt - 1] ??
1198
+ RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS[
1199
+ RUNTIME_POSTGRES_CONNECT_RETRY_DELAYS_MS.length - 1
1200
+ ],
1201
+ );
1202
+ }
1203
+ }
1204
+ throw new Error('Runtime Postgres one-shot connection was not acquired.');
1205
+ }
1206
+
1207
+ /**
1208
+ * True when a Postgres error means the backing table/column is not provisioned
1209
+ * in the session's *physical* database — `42P01` (undefined_table), `42703`
1210
+ * (undefined_column), or the equivalent messages.
1211
+ *
1212
+ * Both data planes (sheet rows and work receipts) treat this as "re-provision,
1213
+ * then retry once" rather than a hard failure, because a cached "already
1214
+ * ensured" entry can outlive the physical relation it stands for — a reset Neon
1215
+ * branch, or an ensure cache primed in one Worker isolate while the operation
1216
+ * runs against another. This is the single classifier that keeps both planes
1217
+ * self-healing; do not fork it.
1218
+ */
1219
+ function isMissingRelationError(error: unknown): boolean {
1220
+ if (!error || typeof error !== 'object') {
1221
+ return false;
1222
+ }
1223
+ const code = 'code' in error ? String(error.code) : '';
1224
+ if (code === '42P01' || code === '42703') {
1225
+ return true;
1226
+ }
1227
+ const message = 'message' in error ? String(error.message) : '';
1228
+ return (
1229
+ /relation .* does not exist/i.test(message) ||
1230
+ /column .* does not exist/i.test(message)
1231
+ );
1232
+ }
1233
+
1234
+ async function withRuntimeSheetProvisioningRetry<T>(
1235
+ context: RuntimeApiContext,
1236
+ input: {
1237
+ playName: string;
1238
+ tableNamespace: string;
1239
+ sheetContract: PlaySheetContract;
1240
+ timings?: RuntimeSheetTiming[];
1241
+ },
1242
+ operation: () => Promise<T>,
1243
+ ): Promise<T> {
1244
+ const firstAttemptStartedAt = Date.now();
1245
+ try {
1246
+ const result = await operation();
1247
+ input.timings?.push({
1248
+ phase: 'operation_attempt',
1249
+ ms: Date.now() - firstAttemptStartedAt,
1250
+ retried: false,
1251
+ });
1252
+ return result;
1253
+ } catch (error) {
1254
+ input.timings?.push({
1255
+ phase: 'operation_attempt',
1256
+ ms: Date.now() - firstAttemptStartedAt,
1257
+ retried: false,
1258
+ error: error instanceof Error ? error.message : String(error),
1259
+ });
1260
+ if (!isMissingRelationError(error)) {
1261
+ throw error;
1262
+ }
1263
+ const ensureStartedAt = Date.now();
1264
+ await ensureRuntimeSheet(context, input);
1265
+ input.timings?.push({
1266
+ phase: 'ensure_sheet_after_provisioning_error',
1267
+ ms: Date.now() - ensureStartedAt,
1268
+ retried: true,
1269
+ });
1270
+ const retryStartedAt = Date.now();
1271
+ const result = await operation();
1272
+ input.timings?.push({
1273
+ phase: 'operation_retry',
1274
+ ms: Date.now() - retryStartedAt,
1275
+ retried: true,
1276
+ });
1277
+ return result;
1278
+ }
1279
+ }
1280
+
1281
+ async function withRuntimeSheetQueryClient<T>(
1282
+ context: RuntimeApiContext,
1283
+ session: RuntimePostgresSession,
1284
+ input: {
1285
+ playName: string;
1286
+ tableNamespace: string;
1287
+ sheetContract: PlaySheetContract;
1288
+ transactional: boolean;
1289
+ timings?: RuntimeSheetTiming[];
1290
+ },
1291
+ operation: (client: RuntimeQueryClient) => Promise<T>,
1292
+ ): Promise<T> {
1293
+ const totalStartedAt = Date.now();
1294
+ const result = await withRuntimeSheetProvisioningRetry(
1295
+ context,
1296
+ input,
1297
+ async () => {
1298
+ if (!input.transactional && isRuntimeOneShotQueryFactoryRegistered()) {
1299
+ const operationStartedAt = Date.now();
1300
+ return await withRuntimeOneShotPostgres(session, operation).finally(
1301
+ () => {
1302
+ input.timings?.push({
1303
+ phase: 'one_shot_operation',
1304
+ ms: Date.now() - operationStartedAt,
1305
+ });
1306
+ },
1307
+ );
1308
+ }
1309
+
1310
+ return await withRuntimePostgres(
1311
+ session,
1312
+ async (client) => {
1313
+ if (input.transactional) await client.query('BEGIN');
1314
+ try {
1315
+ const result = await operation(client);
1316
+ if (input.transactional) await client.query('COMMIT');
1317
+ return result;
1318
+ } catch (error) {
1319
+ if (input.transactional) {
1320
+ await client.query('ROLLBACK').catch(() => {});
1321
+ }
1322
+ throw error;
1323
+ }
1324
+ },
1325
+ { cachePool: !context.disablePostgresPoolCache },
1326
+ );
1327
+ },
1328
+ );
1329
+ input.timings?.push({
1330
+ phase: 'query_client_total',
1331
+ ms: Date.now() - totalStartedAt,
1332
+ });
1333
+ return result;
1334
+ }
1335
+
1336
+ function quoteIdentifier(value: string): string {
1337
+ return `"${value.replace(/"/g, '""')}"`;
1338
+ }
1339
+
1340
+ function quoteLiteral(value: string): string {
1341
+ return `'${value.replace(/'/g, "''")}'`;
1342
+ }
1343
+
1344
+ function fqRuntimeTable(
1345
+ session: RuntimePostgresSession,
1346
+ table: string,
1347
+ ): string {
1348
+ return `${quoteIdentifier(session.postgres.schema)}.${quoteIdentifier(table)}`;
1349
+ }
1350
+
1351
+ function sheetTable(session: RuntimePostgresSession): string {
1352
+ return fqRuntimeTable(session, session.postgres.sheetTable);
1353
+ }
1354
+
1355
+ function summaryTable(session: RuntimePostgresSession): string {
1356
+ return fqRuntimeTable(session, session.postgres.summaryTable);
1357
+ }
1358
+
1359
+ function columnSummaryTable(session: RuntimePostgresSession): string {
1360
+ return fqRuntimeTable(session, session.postgres.columnSummaryTable);
1361
+ }
1362
+
1363
+ function workReceiptTable(session: RuntimePostgresSession): string {
1364
+ return fqRuntimeTable(
1365
+ session,
1366
+ session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
1367
+ );
1368
+ }
1369
+
1370
+ function postgresUrlCacheKey(value: string): string {
1371
+ return createHash('sha256').update(value).digest('hex').slice(0, 24);
1372
+ }
1373
+
1374
+ function workReceiptKeyHex(key: string): string {
1375
+ return Array.from(new TextEncoder().encode(key), (byte) =>
1376
+ byte.toString(16).padStart(2, '0'),
1377
+ ).join('');
1378
+ }
1379
+
1380
+ function validateRuntimeWorkReceiptKeyScope(
1381
+ session: RuntimePostgresSession,
1382
+ input: { key: string },
1383
+ ): void {
1384
+ const orgId = session.target.orgId.trim();
1385
+ const playName = session.playName.trim();
1386
+ const scopedReceiptPrefix = scopedWorkReceiptKeyPrefix({ orgId, playName });
1387
+ const durableCtxPrefix = `ctx:${orgId}:`;
1388
+ const isScopedReceiptKey = input.key.startsWith(scopedReceiptPrefix);
1389
+ const isDurableCtxKey = input.key.startsWith(durableCtxPrefix);
1390
+ if (!orgId || !playName || (!isScopedReceiptKey && !isDurableCtxKey)) {
1391
+ throw new Error(
1392
+ 'Runtime work receipt key is outside the scoped session scope.',
1393
+ );
1394
+ }
1395
+ }
1396
+
1397
+ function mapRuntimeWorkReceiptRow(raw: Record<string, unknown>): WorkReceipt {
1398
+ return {
1399
+ key: String(raw.k ?? ''),
1400
+ status: receiptStatusFromCode(raw.status),
1401
+ output: raw.output == null ? null : raw.output,
1402
+ error: raw.error == null ? null : String(raw.error),
1403
+ runId: raw.run_id == null ? null : String(raw.run_id),
1404
+ };
1405
+ }
1406
+
1407
+ function runtimeWorkReceiptEnsureCacheKey(
1408
+ session: RuntimePostgresSession,
1409
+ ): string {
1410
+ return `${postgresUrlCacheKey(session.postgresUrl)}::${session.postgres.schema}::${session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE}`;
1411
+ }
1412
+
1413
+ async function ensureRuntimeWorkReceiptTable(
1414
+ session: RuntimePostgresSession,
1415
+ client: RuntimeQueryClient,
1416
+ ): Promise<void> {
1417
+ const cacheKey = runtimeWorkReceiptEnsureCacheKey(session);
1418
+ const cached = runtimeWorkReceiptEnsureCache.get(cacheKey);
1419
+ if (cached) {
1420
+ await cached;
1421
+ return;
1422
+ }
1423
+ const promise = client
1424
+ .query(
1425
+ `
1426
+ CREATE TABLE IF NOT EXISTS ${workReceiptTable(session)} (
1427
+ k bytea PRIMARY KEY,
1428
+ status smallint NOT NULL DEFAULT 0,
1429
+ output jsonb,
1430
+ error text,
1431
+ run_id text,
1432
+ updated_at timestamptz NOT NULL DEFAULT now()
1433
+ )
1434
+ `,
1435
+ )
1436
+ .then(() => undefined);
1437
+ runtimeWorkReceiptEnsureCache.set(cacheKey, promise);
1438
+ try {
1439
+ await promise;
1440
+ } catch (error) {
1441
+ if (runtimeWorkReceiptEnsureCache.get(cacheKey) === promise) {
1442
+ runtimeWorkReceiptEnsureCache.delete(cacheKey);
1443
+ }
1444
+ throw error;
1445
+ }
1446
+ }
1447
+
1448
+ async function withRuntimeWorkReceiptClient<T>(
1449
+ context: RuntimeApiContext,
1450
+ session: RuntimePostgresSession,
1451
+ operation: (client: RuntimeQueryClient) => Promise<T>,
1452
+ ): Promise<T> {
1453
+ // Receipt tables are part of customer Postgres bootstrap, so the hot path
1454
+ // should not pay DDL on every fresh runtime isolate. Try the receipt query
1455
+ // first, then self-heal once on a missing relation/column. A genuinely
1456
+ // missing schema, permission error, or second failure still rethrows loudly.
1457
+ const runWithSelfHeal = async (client: RuntimeQueryClient): Promise<T> => {
1458
+ try {
1459
+ return await operation(client);
1460
+ } catch (error) {
1461
+ if (!isMissingRelationError(error)) {
1462
+ throw error;
1463
+ }
1464
+ runtimeWorkReceiptEnsureCache.delete(
1465
+ runtimeWorkReceiptEnsureCacheKey(session),
1466
+ );
1467
+ await ensureRuntimeWorkReceiptTable(session, client);
1468
+ return await operation(client);
1469
+ }
1470
+ };
1471
+
1472
+ if (isRuntimeOneShotQueryFactoryRegistered()) {
1473
+ return await withRuntimeOneShotPostgres(session, runWithSelfHeal);
1474
+ }
1475
+
1476
+ return await withRuntimePostgres(
1477
+ session,
1478
+ (client) => runWithSelfHeal(client),
1479
+ { cachePool: !context.disablePostgresPoolCache },
1480
+ );
1481
+ }
1482
+
1483
+ const PLAY_INTERNAL_SHEET_VERSION_SEQUENCE = '_deepline_sheet_version_seq';
1484
+
1485
+ function nextRuntimeSheetVersionExpression(
1486
+ session: RuntimePostgresSession,
1487
+ ): string {
1488
+ return `nextval(${quoteLiteral(`${session.postgres.schema}.${PLAY_INTERNAL_SHEET_VERSION_SEQUENCE}`)}::regclass)`;
1489
+ }
1490
+
1491
+ function missingOutputCellSql(
1492
+ tableAlias: string,
1493
+ outputPhysicalColumns: readonly string[],
1494
+ ): string {
1495
+ if (outputPhysicalColumns.length === 0) {
1496
+ return 'false';
1497
+ }
1498
+ return outputPhysicalColumns
1499
+ .map((column) => {
1500
+ const quoted = `${tableAlias}.${quoteIdentifier(column)}`;
1501
+ return `(${quoted} IS NULL OR ${quoted} = 'null'::jsonb OR ${quoted} = '""'::jsonb)`;
1502
+ })
1503
+ .join(' OR ');
1504
+ }
1505
+
1506
+ function changedPatchedCellSql(
1507
+ tableAlias: string,
1508
+ patchAlias: string,
1509
+ projections: readonly PhysicalSheetColumnProjection[],
1510
+ ): string {
1511
+ if (projections.length === 0) {
1512
+ return 'false';
1513
+ }
1514
+ return projections
1515
+ .map((column) => {
1516
+ const quoted = `${tableAlias}.${quoteIdentifier(column.sqlName)}`;
1517
+ return `(${patchAlias} ? ${quoteLiteral(column.fieldName)} AND ${quoted} IS DISTINCT FROM ${patchAlias} -> ${quoteLiteral(column.fieldName)})`;
1518
+ })
1519
+ .join(' OR ');
1520
+ }
1521
+
1522
+ function isSystemSheetColumn(columnName: string): boolean {
1523
+ switch (columnName) {
1524
+ case '_key':
1525
+ case '_status':
1526
+ case '_run_id':
1527
+ case '_error':
1528
+ case '_stage':
1529
+ case '_provider':
1530
+ case '_input_index':
1531
+ case '_created_at':
1532
+ case '_updated_at':
1533
+ case '_version':
1534
+ case '_cell_meta':
1535
+ case 'seq':
1536
+ case '__has_enriched':
1537
+ case '__has_failed':
1538
+ case '__deeplineCsvProjectedFields':
1539
+ case '__deeplineCsvProjectedValues':
1540
+ return true;
1541
+ default:
1542
+ return false;
1543
+ }
1544
+ }
1545
+
1546
+ function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
1547
+ if (value && typeof value === 'object' && !Array.isArray(value)) {
1548
+ return value as Record<string, unknown>;
1549
+ }
1550
+ if (typeof value !== 'string' || !value.trim()) {
1551
+ return {};
1552
+ }
1553
+ try {
1554
+ const parsed = JSON.parse(value) as unknown;
1555
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
1556
+ ? (parsed as Record<string, unknown>)
1557
+ : {};
1558
+ } catch {
1559
+ return {};
1560
+ }
1561
+ }
1562
+
1563
+ function mapRuntimePostgresRow(input: {
1564
+ raw: Record<string, unknown>;
1565
+ sheetContract?: PlaySheetContract | null;
1566
+ }): RuntimeApiRowRecord {
1567
+ const { raw, sheetContract } = input;
1568
+ const cellMeta = parseRuntimeCellMeta(raw._cell_meta);
1569
+ const projections = physicalSheetColumnProjections(sheetContract);
1570
+ const publicData =
1571
+ projections.length > 0
1572
+ ? Object.fromEntries(
1573
+ projections
1574
+ .filter((column) =>
1575
+ Object.prototype.hasOwnProperty.call(raw, column.sqlName),
1576
+ )
1577
+ .map((column) => [column.fieldName, raw[column.sqlName]]),
1578
+ )
1579
+ : Object.fromEntries(
1580
+ Object.entries(raw).filter(([key]) => !isSystemSheetColumn(key)),
1581
+ );
1582
+ const data =
1583
+ Object.keys(cellMeta).length > 0
1584
+ ? {
1585
+ ...publicData,
1586
+ [DEEPLINE_CELL_META_FIELD]: cellMeta,
1587
+ }
1588
+ : publicData;
1589
+ return {
1590
+ key: String(raw._key ?? ''),
1591
+ data,
1592
+ inputIndex: raw._input_index != null ? Number(raw._input_index) : undefined,
1593
+ };
1594
+ }
1595
+
1596
+ function mapRuntimeProjectedRows(
1597
+ rows: readonly Record<string, unknown>[],
1598
+ outputColumns: readonly PhysicalSheetColumnProjection[],
1599
+ ): RuntimeApiRowRecord[] {
1600
+ return rows.map((raw) => {
1601
+ const key = String(raw._key ?? '');
1602
+ const data = Object.fromEntries(
1603
+ outputColumns
1604
+ .filter((column) =>
1605
+ Object.prototype.hasOwnProperty.call(raw, column.fieldName),
1606
+ )
1607
+ .map((column) => [column.fieldName, raw[column.fieldName]]),
1608
+ );
1609
+ return { key, data };
1610
+ });
1611
+ }
1612
+
1613
+ function cachedRuntimeCellMetaPatch(runId: string): Record<string, unknown> {
1614
+ return {
1615
+ status: 'cached',
1616
+ runId,
1617
+ reused: true,
1618
+ };
1619
+ }
1620
+
1621
+ function completedRuntimeCellMetaPatch(input: {
1622
+ runId: string;
1623
+ outputFields: readonly string[];
1624
+ rowPatch?: Record<string, unknown>;
1625
+ }): Record<string, unknown> {
1626
+ const patch: Record<string, unknown> = {};
1627
+ const completedAt = Date.now();
1628
+ for (const field of input.outputFields) {
1629
+ const existing =
1630
+ input.rowPatch?.[field] &&
1631
+ typeof input.rowPatch[field] === 'object' &&
1632
+ !Array.isArray(input.rowPatch[field])
1633
+ ? (input.rowPatch[field] as Record<string, unknown>)
1634
+ : {};
1635
+ patch[field] = {
1636
+ status: 'completed',
1637
+ runId: input.runId,
1638
+ completedAt,
1639
+ ...existing,
1640
+ };
1641
+ }
1642
+ for (const [field, meta] of Object.entries(input.rowPatch ?? {})) {
1643
+ if (!Object.hasOwn(patch, field)) {
1644
+ patch[field] = meta;
1645
+ }
1646
+ }
1647
+ return patch;
1648
+ }
1649
+
1650
+ function cachedRuntimeCellMetaUpdateSql(
1651
+ tableAlias: string,
1652
+ outputFields: readonly string[],
1653
+ patchSql: string,
1654
+ ): string {
1655
+ const uniqueFields = [...new Set(outputFields)];
1656
+ if (uniqueFields.length === 0) {
1657
+ return `${tableAlias}._cell_meta`;
1658
+ }
1659
+ return uniqueFields.reduce((expression, field) => {
1660
+ const fieldLiteral = quoteLiteral(field);
1661
+ return `jsonb_set(
1662
+ ${expression},
1663
+ ARRAY[${fieldLiteral}]::text[],
1664
+ coalesce(${tableAlias}._cell_meta -> ${fieldLiteral}, '{}'::jsonb) || ${patchSql},
1665
+ true
1666
+ )`;
1667
+ }, `coalesce(${tableAlias}._cell_meta, '{}'::jsonb)`);
1668
+ }
1669
+
1670
+ function mergeRuntimeCellMetaPatchSql(
1671
+ targetExpression: string,
1672
+ patchExpression: string,
1673
+ ): string {
1674
+ return `coalesce(${targetExpression}, '{}'::jsonb) || (
1675
+ SELECT coalesce(
1676
+ jsonb_object_agg(patch.key, coalesce(${targetExpression} -> patch.key, '{}'::jsonb) || patch.value),
1677
+ '{}'::jsonb
1678
+ )
1679
+ FROM jsonb_each(coalesce(${patchExpression}, '{}'::jsonb)) AS patch(key, value)
1680
+ )`;
1681
+ }
1682
+
1683
+ async function insertCachedRuntimeColumnSummaryDelta(
1684
+ client: RuntimeQueryClient,
1685
+ session: RuntimePostgresSession,
1686
+ input: {
1687
+ tableNamespace: string;
1688
+ outputFields: readonly string[];
1689
+ cached: number;
1690
+ },
1691
+ ): Promise<void> {
1692
+ const uniqueFields = [...new Set(input.outputFields)];
1693
+ if (uniqueFields.length === 0 || input.cached <= 0) {
1694
+ return;
1695
+ }
1696
+ const normalizedPlayName = normalizePlayNameForSheet(session.playName);
1697
+ const normalizedTableNamespace = normalizeTableNamespace(
1698
+ input.tableNamespace,
1699
+ );
1700
+ await client.query(
1701
+ `
1702
+ INSERT INTO ${columnSummaryTable(session)} AS target (
1703
+ play_name,
1704
+ table_namespace,
1705
+ field,
1706
+ cached
1707
+ )
1708
+ SELECT $1::text, $2::text, field_values.field, $3::int
1709
+ FROM unnest($4::text[]) AS field_values(field)
1710
+ ON CONFLICT (play_name, table_namespace, field) DO UPDATE
1711
+ SET cached = GREATEST(target.cached + EXCLUDED.cached, 0),
1712
+ _updated_at = now()
1713
+ `,
1714
+ [normalizedPlayName, normalizedTableNamespace, input.cached, uniqueFields],
1715
+ );
1716
+ }
1717
+
1718
+ async function markAndReadRuntimeCompletedRowsCached(
1719
+ client: RuntimeQueryClient,
1720
+ session: RuntimePostgresSession,
1721
+ input: {
1722
+ tableNamespace: string;
1723
+ keys: string[];
1724
+ runId: string;
1725
+ outputFields: string[];
1726
+ sheetContract: PlaySheetContract;
1727
+ },
1728
+ ): Promise<RuntimeApiRowRecord[]> {
1729
+ if (input.keys.length === 0) {
1730
+ return [];
1731
+ }
1732
+ const cellMetaSql = cachedRuntimeCellMetaUpdateSql(
1733
+ 'target',
1734
+ input.outputFields,
1735
+ '$3::jsonb',
1736
+ );
1737
+
1738
+ const { rows } = await client.query<Record<string, unknown>>(
1739
+ `
1740
+ WITH updated_rows AS (
1741
+ UPDATE ${sheetTable(session)} AS target
1742
+ SET _run_id = $2::text,
1743
+ _updated_at = now(),
1744
+ _version = ${nextRuntimeSheetVersionExpression(session)},
1745
+ _cell_meta = ${cellMetaSql}
1746
+ WHERE target._key = ANY($1::text[])
1747
+ AND target._status = 'enriched'
1748
+ RETURNING target.*
1749
+ )
1750
+ SELECT *
1751
+ FROM updated_rows
1752
+ ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC
1753
+ `,
1754
+ [
1755
+ input.keys,
1756
+ input.runId,
1757
+ JSON.stringify(cachedRuntimeCellMetaPatch(input.runId)),
1758
+ ],
1759
+ );
1760
+ await insertCachedRuntimeColumnSummaryDelta(client, session, {
1761
+ tableNamespace: input.tableNamespace,
1762
+ outputFields: input.outputFields,
1763
+ cached: rows.length,
1764
+ });
1765
+ return rows.map((raw) =>
1766
+ mapRuntimePostgresRow({ raw, sheetContract: input.sheetContract }),
1767
+ );
1768
+ }
1769
+
1770
+ async function markAndReadRuntimeCompletedRowsCachedProjection(
1771
+ client: RuntimeQueryClient,
1772
+ session: RuntimePostgresSession,
1773
+ input: {
1774
+ tableNamespace: string;
1775
+ keys: string[];
1776
+ runId: string;
1777
+ outputFields: string[];
1778
+ outputColumns: PhysicalSheetColumnProjection[];
1779
+ timings?: RuntimeSheetTiming[];
1780
+ },
1781
+ ): Promise<RuntimeApiRowRecord[]> {
1782
+ if (input.keys.length === 0) {
1783
+ return [];
1784
+ }
1785
+ const cellMetaSql = cachedRuntimeCellMetaUpdateSql(
1786
+ 'target',
1787
+ input.outputFields,
1788
+ '$3::jsonb',
1789
+ );
1790
+ const outputReturnSql =
1791
+ input.outputColumns.length > 0
1792
+ ? `, ${input.outputColumns
1793
+ .map(
1794
+ (column) =>
1795
+ `target.${quoteIdentifier(column.sqlName)} AS ${quoteIdentifier(
1796
+ column.fieldName,
1797
+ )}`,
1798
+ )
1799
+ .join(', ')}`
1800
+ : '';
1801
+
1802
+ const queryStartedAt = Date.now();
1803
+ const { rows } = await client.query<Record<string, unknown>>(
1804
+ `
1805
+ WITH updated_rows AS (
1806
+ UPDATE ${sheetTable(session)} AS target
1807
+ SET _run_id = $2::text,
1808
+ _updated_at = now(),
1809
+ _version = ${nextRuntimeSheetVersionExpression(session)},
1810
+ _cell_meta = ${cellMetaSql}
1811
+ WHERE target._key = ANY($1::text[])
1812
+ AND target._status = 'enriched'
1813
+ RETURNING target._key${outputReturnSql}
1814
+ )
1815
+ SELECT *
1816
+ FROM updated_rows
1817
+ `,
1818
+ [
1819
+ input.keys,
1820
+ input.runId,
1821
+ JSON.stringify(cachedRuntimeCellMetaPatch(input.runId)),
1822
+ ],
1823
+ );
1824
+ input.timings?.push({
1825
+ phase: 'cached_fast_path.mark_read_projected_query',
1826
+ ms: Date.now() - queryStartedAt,
1827
+ rows: input.keys.length,
1828
+ completed: rows.length,
1829
+ cached: true,
1830
+ });
1831
+
1832
+ const mapStartedAt = Date.now();
1833
+ const mapped = mapRuntimeProjectedRows(rows, input.outputColumns);
1834
+ input.timings?.push({
1835
+ phase: 'cached_fast_path.map_projected_rows',
1836
+ ms: Date.now() - mapStartedAt,
1837
+ rows: mapped.length,
1838
+ cached: true,
1839
+ });
1840
+ return mapped;
1841
+ }
1842
+
1843
+ async function readRuntimeRowsByKey(
1844
+ client: RuntimeQueryClient,
1845
+ session: RuntimePostgresSession,
1846
+ keys: readonly string[],
1847
+ sheetContract?: PlaySheetContract | null,
1848
+ ): Promise<RuntimeApiRowRecord[]> {
1849
+ if (keys.length === 0) {
1850
+ return [];
1851
+ }
1852
+ const { rows } = await client.query<Record<string, unknown>>(
1853
+ `
1854
+ SELECT *
1855
+ FROM ${sheetTable(session)}
1856
+ WHERE _key = ANY($1::text[])
1857
+ ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC
1858
+ `,
1859
+ [keys],
1860
+ );
1861
+ return rows.map((raw) => mapRuntimePostgresRow({ raw, sheetContract }));
1862
+ }
1863
+
1864
+ function mergeRuntimeCompletedRow(input: {
1865
+ inputRow: Record<string, unknown>;
1866
+ completedData: Record<string, unknown>;
1867
+ sheetContract: PlaySheetContract;
1868
+ }): Record<string, unknown> {
1869
+ const syntheticNullInputColumns = new Set(
1870
+ input.sheetContract.columns.flatMap((column) => {
1871
+ const field = column.field;
1872
+ if (
1873
+ column.source !== 'input' ||
1874
+ typeof field !== 'string' ||
1875
+ field in input.inputRow ||
1876
+ input.completedData[field] != null
1877
+ ) {
1878
+ return [];
1879
+ }
1880
+ return [field];
1881
+ }),
1882
+ );
1883
+ const cleanedCompletedData = Object.fromEntries(
1884
+ Object.entries(input.completedData).filter(
1885
+ ([key]) => !syntheticNullInputColumns.has(key),
1886
+ ),
1887
+ );
1888
+ return {
1889
+ ...input.inputRow,
1890
+ ...cleanedCompletedData,
1891
+ };
1892
+ }
1893
+
1894
+ function buildAppendedRowKey(input: {
1895
+ row: Record<string, unknown>;
1896
+ tableNamespace: string;
1897
+ idempotencyKey: string;
1898
+ ordinal: number;
1899
+ }): string {
1900
+ const baseKey = derivePlayRowIdentity(input.row, input.tableNamespace);
1901
+ const suffix = createHash('sha1')
1902
+ .update(`${input.idempotencyKey}:${input.ordinal}`)
1903
+ .digest('hex')
1904
+ .slice(0, APPEND_KEY_SUFFIX_LENGTH);
1905
+ return `${baseKey}:append:${suffix}`;
1906
+ }
1907
+
1908
+ function chunkValues<T>(values: readonly T[], chunkSize: number): T[][] {
1909
+ const chunks: T[][] = [];
1910
+ for (let index = 0; index < values.length; index += chunkSize) {
1911
+ chunks.push(values.slice(index, index + chunkSize));
1912
+ }
1913
+ return chunks;
1914
+ }
1915
+
1916
+ async function readRuntimeRows(
1917
+ session: RuntimePostgresSession,
1918
+ input: {
1919
+ limit: number;
1920
+ offset: number;
1921
+ runId?: string | null;
1922
+ sheetContract?: PlaySheetContract | null;
1923
+ },
1924
+ ): Promise<RuntimeApiRowRecord[]> {
1925
+ return await withRuntimePostgres(session, async (client) => {
1926
+ if (input.runId) {
1927
+ const { rows } = await client.query(
1928
+ `WITH scoped AS (
1929
+ SELECT *,
1930
+ bool_or(_status = 'enriched') OVER () AS __has_enriched,
1931
+ bool_or(_status = 'failed') OVER () AS __has_failed
1932
+ FROM ${sheetTable(session)}
1933
+ WHERE _run_id = $1::text
1934
+ )
1935
+ SELECT *
1936
+ FROM scoped
1937
+ WHERE (__has_enriched AND _status = 'enriched')
1938
+ OR (NOT __has_enriched AND __has_failed AND _status = 'failed')
1939
+ OR (NOT __has_enriched AND NOT __has_failed)
1940
+ ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC
1941
+ LIMIT $2 OFFSET $3`,
1942
+ [input.runId, input.limit, input.offset],
1943
+ );
1944
+ return rows.map((raw) =>
1945
+ mapRuntimePostgresRow({ raw, sheetContract: input.sheetContract }),
1946
+ );
1947
+ }
1948
+ const { rows } = await client.query(
1949
+ `SELECT *
1950
+ FROM ${sheetTable(session)}
1951
+ ORDER BY _input_index ASC NULLS LAST, _created_at ASC, _key ASC
1952
+ LIMIT $1 OFFSET $2`,
1953
+ [input.limit, input.offset],
1954
+ );
1955
+ return rows.map((raw) =>
1956
+ mapRuntimePostgresRow({ raw, sheetContract: input.sheetContract }),
1957
+ );
1958
+ });
1959
+ }
1960
+
1961
+ async function readRuntimeSummary(
1962
+ session: RuntimePostgresSession,
1963
+ ): Promise<{ stats: { total: number } }> {
1964
+ const normalizedPlayName = normalizePlayNameForSheet(session.playName);
1965
+ const normalizedTableNamespace = normalizeTableNamespace(
1966
+ session.target.tableNamespace,
1967
+ );
1968
+ return await withRuntimePostgres(session, async (client) => {
1969
+ const { rows } = await client.query(
1970
+ `SELECT total
1971
+ FROM ${summaryTable(session)}
1972
+ WHERE play_name = $1 AND table_namespace = $2
1973
+ LIMIT 1`,
1974
+ [normalizedPlayName, normalizedTableNamespace],
1975
+ );
1976
+ return { stats: { total: Number(rows[0]?.total ?? 0) } };
1977
+ });
1978
+ }
1979
+
1980
+ async function writeRuntimeRows(
1981
+ session: RuntimePostgresSession,
1982
+ input: {
1983
+ tableNamespace: string;
1984
+ rows: Record<string, unknown>[];
1985
+ runId: string;
1986
+ idempotencyKey: string;
1987
+ sheetContract: PlaySheetContract;
1988
+ mode: 'append' | 'upsert' | 'replace';
1989
+ },
1990
+ ): Promise<RowsWriteResponse> {
1991
+ const physicalColumnProjections = physicalSheetColumnProjections(
1992
+ input.sheetContract,
1993
+ );
1994
+ const physicalColumns = physicalColumnProjections.map(
1995
+ (column) => column.sqlName,
1996
+ );
1997
+ const physicalInsertColumnsSql =
1998
+ physicalColumns.length > 0
1999
+ ? `, ${physicalColumns.map(quoteIdentifier).join(', ')}`
2000
+ : '';
2001
+ const physicalInsertValuesSql =
2002
+ physicalColumns.length > 0
2003
+ ? `, ${physicalColumnProjections
2004
+ .map((column) => `payload -> ${quoteLiteral(column.fieldName)}`)
2005
+ .join(', ')}`
2006
+ : '';
2007
+
2008
+ const rowsToWrite = input.rows.filter(
2009
+ (row) => row && typeof row === 'object' && !Array.isArray(row),
2010
+ );
2011
+ if (rowsToWrite.length === 0) {
2012
+ return { disposition: 'completed', writtenRows: 0 };
2013
+ }
2014
+ const normalizedPlayName = normalizePlayNameForSheet(session.playName);
2015
+ const normalizedTableNamespace = normalizeTableNamespace(
2016
+ input.tableNamespace,
2017
+ );
2018
+
2019
+ // Build write entries first so we know chunk count up front. Single-chunk
2020
+ // writes skip the BEGIN/COMMIT pair entirely (the mega-CTE is atomic at
2021
+ // statement level), and the per-chunk mega-CTE folds: starting-index lookup,
2022
+ // sheet insert, and summary upsert into one round-trip.
2023
+ const rowEntries =
2024
+ input.mode === 'upsert'
2025
+ ? Array.from(
2026
+ rowsToWrite
2027
+ .reduce((uniqueRows, row) => {
2028
+ const key = derivePlayRowIdentity(row, input.tableNamespace);
2029
+ if (key && !uniqueRows.has(key)) {
2030
+ uniqueRows.set(key, row);
2031
+ }
2032
+ return uniqueRows;
2033
+ }, new Map<string, Record<string, unknown>>())
2034
+ .entries(),
2035
+ ).map(([key, row], inputIndex) => ({ key, row, inputIndex }))
2036
+ : rowsToWrite.map((row, index) => ({
2037
+ key: buildAppendedRowKey({
2038
+ row,
2039
+ tableNamespace: input.tableNamespace,
2040
+ idempotencyKey: input.idempotencyKey,
2041
+ ordinal: index,
2042
+ }),
2043
+ row,
2044
+ // For append/replace modes the starting offset is computed in SQL
2045
+ // as `coalesce(max(_input_index), -1) + 1 + (ord - 1)`; we only
2046
+ // pass the per-chunk ordinal here so the SQL can add it to the
2047
+ // dynamic starting index without an extra round-trip.
2048
+ inputIndex: index,
2049
+ }));
2050
+
2051
+ const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
2052
+ const needsTransaction = input.mode === 'replace' || chunks.length > 1;
2053
+
2054
+ return await withRuntimePostgres(session, async (client) => {
2055
+ if (needsTransaction) await client.query('BEGIN');
2056
+ try {
2057
+ if (input.mode === 'replace') {
2058
+ // Collapse 4 cleanup statements into one CTE-shaped query. Postgres
2059
+ // executes data-modifying CTEs against the same snapshot, but each
2060
+ // operates on a distinct table so ordering does not matter.
2061
+ await client.query(
2062
+ `WITH cleared_sheet AS (
2063
+ DELETE FROM ${sheetTable(session)} RETURNING 1
2064
+ ),
2065
+ reset_summary AS (
2066
+ UPDATE ${summaryTable(session)}
2067
+ SET total = 0, queued = 0, running = 0, completed = 0, failed = 0, _updated_at = now()
2068
+ WHERE play_name = $1 AND table_namespace = $2
2069
+ RETURNING 1
2070
+ ),
2071
+ cleared_col_summary AS (
2072
+ DELETE FROM ${columnSummaryTable(session)}
2073
+ WHERE play_name = $1 AND table_namespace = $2
2074
+ RETURNING 1
2075
+ )
2076
+ SELECT 1`,
2077
+ [normalizedPlayName, normalizedTableNamespace],
2078
+ );
2079
+ }
2080
+
2081
+ // For append/replace: SQL computes _input_index from the live max so
2082
+ // we never need a separate SELECT round-trip. For upsert: the JS
2083
+ // ordinals (0..N) are passed straight through, matching prior shape.
2084
+ const computesStartingIndexInSql =
2085
+ input.mode === 'append' || input.mode === 'replace';
2086
+
2087
+ let writtenRows = 0;
2088
+ for (const chunk of chunks) {
2089
+ const chunkKeys = chunk.map((entry) => entry.key);
2090
+ const chunkPayloads = chunk.map((entry) =>
2091
+ stringifyPostgresJson(entry.row),
2092
+ );
2093
+ const chunkInputIndexes = chunk.map((entry) => entry.inputIndex);
2094
+
2095
+ const startingIndexCte = computesStartingIndexInSql
2096
+ ? `starting_index AS (
2097
+ SELECT coalesce(max(_input_index), -1)::bigint AS v FROM ${sheetTable(session)}
2098
+ ),`
2099
+ : '';
2100
+ const inputIndexExpr = computesStartingIndexInSql
2101
+ ? `(SELECT v FROM starting_index) + index_values._input_index::bigint + 1`
2102
+ : `index_values._input_index::bigint`;
2103
+ const insertedRowsCte =
2104
+ input.mode === 'upsert'
2105
+ ? `inserted_rows AS (
2106
+ INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
2107
+ SELECT _key, 'pending', $4::text, _input_index${physicalInsertValuesSql}
2108
+ FROM input_rows
2109
+ ON CONFLICT (_key) DO UPDATE SET
2110
+ _status = 'pending',
2111
+ _run_id = EXCLUDED._run_id,
2112
+ _input_index = EXCLUDED._input_index,
2113
+ _updated_at = now(),
2114
+ _version = ${nextRuntimeSheetVersionExpression(session)}
2115
+ WHERE ${sheetTable(session)}._status = 'stale'
2116
+ RETURNING _key
2117
+ ),`
2118
+ : `inserted_rows AS (
2119
+ INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
2120
+ SELECT _key, 'pending', $4::text, _input_index${physicalInsertValuesSql}
2121
+ FROM input_rows
2122
+ RETURNING _key
2123
+ ),`;
2124
+
2125
+ const sql = `
2126
+ WITH ${startingIndexCte}
2127
+ input_rows AS (
2128
+ SELECT DISTINCT ON (key_values._key)
2129
+ key_values._key, payload_values.payload,
2130
+ ${inputIndexExpr} AS _input_index
2131
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
2132
+ JOIN unnest($2::jsonb[]) WITH ORDINALITY AS payload_values(payload, ord)
2133
+ ON payload_values.ord = key_values.ord
2134
+ JOIN unnest($3::bigint[]) WITH ORDINALITY AS index_values(_input_index, ord)
2135
+ ON index_values.ord = key_values.ord
2136
+ ORDER BY key_values._key, key_values.ord
2137
+ ),
2138
+ ${insertedRowsCte}
2139
+ inserted_count_cte AS (
2140
+ SELECT count(*)::bigint AS c FROM inserted_rows
2141
+ ),
2142
+ summary_upsert AS (
2143
+ INSERT INTO ${summaryTable(session)} (play_name, table_namespace, total, queued, running, completed, failed)
2144
+ SELECT $5::text, $6::text, c, c, 0, 0, 0
2145
+ FROM inserted_count_cte
2146
+ WHERE c > 0
2147
+ ON CONFLICT (play_name, table_namespace) DO UPDATE SET
2148
+ total = ${summaryTable(session)}.total + EXCLUDED.total,
2149
+ queued = ${summaryTable(session)}.queued + EXCLUDED.queued,
2150
+ _updated_at = now()
2151
+ RETURNING 1
2152
+ )
2153
+ SELECT c::int AS inserted_count FROM inserted_count_cte
2154
+ `;
2155
+
2156
+ const { rows } = await client.query(sql, [
2157
+ chunkKeys,
2158
+ chunkPayloads,
2159
+ chunkInputIndexes,
2160
+ input.runId,
2161
+ normalizedPlayName,
2162
+ normalizedTableNamespace,
2163
+ ]);
2164
+ writtenRows += Number(rows[0]?.inserted_count ?? 0);
2165
+ }
2166
+
2167
+ if (needsTransaction) await client.query('COMMIT');
2168
+ return { disposition: 'completed', writtenRows };
2169
+ } catch (error) {
2170
+ if (needsTransaction) {
2171
+ await client.query('ROLLBACK').catch(() => {});
2172
+ }
2173
+ throw error;
2174
+ }
2175
+ });
2176
+ }
2177
+
2178
+ export async function resolveRuntimeReferencedPlay(
2179
+ context: RuntimeApiContext,
2180
+ playRef: string,
2181
+ options?: { artifactKind?: PlayArtifactKind },
2182
+ ): Promise<ResolvedRuntimePlay | null> {
2183
+ const response = await postRuntimeApi<{ play: ResolvedRuntimePlay | null }>(
2184
+ context,
2185
+ {
2186
+ action: 'resolve_play',
2187
+ playRef,
2188
+ ...(options?.artifactKind ? { artifactKind: options.artifactKind } : {}),
2189
+ },
2190
+ );
2191
+ return response.play;
2192
+ }
2193
+
2194
+ export async function ensureRuntimeSheet(
2195
+ context: RuntimeApiContext,
2196
+ input: {
2197
+ playName: string;
2198
+ tableNamespace: string;
2199
+ sheetContract: PlaySheetContract;
2200
+ },
2201
+ ): Promise<void> {
2202
+ await postRuntimeApi<{ ok: true }>(context, {
2203
+ action: 'ensure_sheet',
2204
+ ...input,
2205
+ runId: context.runId ?? null,
2206
+ userEmail: normalizeRuntimeUserEmail(context.userEmail),
2207
+ });
2208
+ }
2209
+
2210
+ async function prepareRuntimeSheetDatasetRows(
2211
+ client: RuntimeQueryClient,
2212
+ session: RuntimePostgresSession,
2213
+ input: {
2214
+ chunks: RuntimeDatasetRowEntry[][];
2215
+ runId: string;
2216
+ normalizedPlayName: string;
2217
+ normalizedTableNamespace: string;
2218
+ physicalInsertColumnsSql: string;
2219
+ physicalInsertValuesSql: string;
2220
+ outputPhysicalColumns: string[];
2221
+ },
2222
+ ): Promise<{ inserted: number; pendingKeys: string[] }> {
2223
+ let inserted = 0;
2224
+ const pendingKeys: string[] = [];
2225
+ for (const chunk of input.chunks) {
2226
+ const chunkKeys = chunk.map((entry) => entry.key);
2227
+ const chunkPayloads = chunk.map((entry) =>
2228
+ stringifyPostgresJson(entry.row),
2229
+ );
2230
+ const chunkInputIndexes = chunk.map((entry) => entry.inputIndex);
2231
+ const existingMissingOutputSql = missingOutputCellSql(
2232
+ 'existing',
2233
+ input.outputPhysicalColumns,
2234
+ );
2235
+ const targetMissingOutputSql = missingOutputCellSql(
2236
+ 'target',
2237
+ input.outputPhysicalColumns,
2238
+ );
2239
+ const { rows } = await client.query(
2240
+ `
2241
+ WITH input_rows AS (
2242
+ SELECT DISTINCT ON (key_values._key)
2243
+ key_values._key, payload_values.payload, index_values._input_index
2244
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
2245
+ JOIN unnest($2::jsonb[]) WITH ORDINALITY AS payload_values(payload, ord)
2246
+ ON payload_values.ord = key_values.ord
2247
+ JOIN unnest($3::bigint[]) WITH ORDINALITY AS index_values(_input_index, ord)
2248
+ ON index_values.ord = key_values.ord
2249
+ ORDER BY key_values._key, key_values.ord
2250
+ ),
2251
+ existing_rows AS (
2252
+ UPDATE ${sheetTable(session)} AS target
2253
+ SET _input_index = input_rows._input_index,
2254
+ _updated_at = now(),
2255
+ _version = ${nextRuntimeSheetVersionExpression(session)}
2256
+ FROM input_rows
2257
+ WHERE target._key = input_rows._key
2258
+ AND target._input_index IS DISTINCT FROM input_rows._input_index
2259
+ RETURNING target._key
2260
+ ),
2261
+ inserted_rows AS (
2262
+ INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${input.physicalInsertColumnsSql})
2263
+ SELECT _key, 'pending', $4::text, _input_index${input.physicalInsertValuesSql}
2264
+ FROM input_rows
2265
+ ON CONFLICT (_key) DO UPDATE SET
2266
+ _status = 'pending',
2267
+ _run_id = EXCLUDED._run_id,
2268
+ _input_index = EXCLUDED._input_index,
2269
+ _updated_at = now(),
2270
+ _version = ${nextRuntimeSheetVersionExpression(session)}
2271
+ WHERE ${sheetTable(session)}._status = 'stale'
2272
+ RETURNING _key
2273
+ ),
2274
+ missing_output_rows AS (
2275
+ UPDATE ${sheetTable(session)} AS target
2276
+ SET _status = 'pending',
2277
+ _run_id = $4::text,
2278
+ _input_index = input_rows._input_index,
2279
+ _updated_at = now(),
2280
+ _version = ${nextRuntimeSheetVersionExpression(session)}
2281
+ FROM input_rows
2282
+ WHERE target._key = input_rows._key
2283
+ AND target._status = 'enriched'
2284
+ AND (${targetMissingOutputSql})
2285
+ RETURNING target._key
2286
+ ),
2287
+ pending_rows AS (
2288
+ SELECT _key
2289
+ FROM inserted_rows
2290
+ UNION
2291
+ SELECT _key
2292
+ FROM missing_output_rows
2293
+ UNION
2294
+ SELECT existing._key
2295
+ FROM ${sheetTable(session)} AS existing
2296
+ JOIN input_rows ON input_rows._key = existing._key
2297
+ WHERE existing._status IN ('pending', 'running', 'failed')
2298
+ OR (
2299
+ existing._status = 'enriched'
2300
+ AND (${existingMissingOutputSql})
2301
+ )
2302
+ ),
2303
+ inserted_count_cte AS (
2304
+ SELECT count(*)::bigint AS c FROM inserted_rows
2305
+ ),
2306
+ missing_output_count_cte AS (
2307
+ SELECT count(*)::bigint AS c FROM missing_output_rows
2308
+ ),
2309
+ summary_counts AS (
2310
+ SELECT
2311
+ (SELECT c FROM inserted_count_cte) AS inserted_count,
2312
+ (SELECT c FROM missing_output_count_cte) AS missing_output_count
2313
+ ),
2314
+ summary_delta AS (
2315
+ INSERT INTO ${summaryTable(session)} AS target (play_name, table_namespace, total, queued, running, completed, failed)
2316
+ SELECT
2317
+ $5::text,
2318
+ $6::text,
2319
+ inserted_count::int,
2320
+ (inserted_count + missing_output_count)::int,
2321
+ 0,
2322
+ (-missing_output_count)::int,
2323
+ 0
2324
+ FROM summary_counts
2325
+ WHERE inserted_count > 0 OR missing_output_count > 0
2326
+ ON CONFLICT (play_name, table_namespace) DO UPDATE SET
2327
+ total = GREATEST(target.total + EXCLUDED.total, 0),
2328
+ queued = GREATEST(target.queued + EXCLUDED.queued, 0),
2329
+ completed = GREATEST(target.completed + EXCLUDED.completed, 0),
2330
+ _updated_at = now()
2331
+ RETURNING 1
2332
+ )
2333
+ SELECT
2334
+ (SELECT c::int FROM inserted_count_cte) AS inserted_count,
2335
+ coalesce((SELECT array_agg(_key) FROM pending_rows), '{}'::text[]) AS pending_keys,
2336
+ (SELECT count(*)::int FROM existing_rows) AS reordered_count,
2337
+ (SELECT c::int FROM missing_output_count_cte) AS missing_output_count
2338
+ `,
2339
+ [
2340
+ chunkKeys,
2341
+ chunkPayloads,
2342
+ chunkInputIndexes,
2343
+ input.runId,
2344
+ input.normalizedPlayName,
2345
+ input.normalizedTableNamespace,
2346
+ ],
2347
+ );
2348
+ inserted += Number(rows[0]?.inserted_count ?? 0);
2349
+ if (Array.isArray(rows[0]?.pending_keys)) {
2350
+ pendingKeys.push(...(rows[0]?.pending_keys as string[]));
2351
+ }
2352
+ }
2353
+ return { inserted, pendingKeys };
2354
+ }
2355
+
2356
+ async function tryPrepareRuntimeSheetDatasetRowsCachedOnly(
2357
+ client: RuntimeQueryClient,
2358
+ session: RuntimePostgresSession,
2359
+ input: {
2360
+ chunks: RuntimeDatasetRowEntry[][];
2361
+ outputPhysicalColumns: string[];
2362
+ timings?: RuntimeSheetTiming[];
2363
+ },
2364
+ ): Promise<{ prepared: { inserted: number; pendingKeys: string[] } | null }> {
2365
+ const pendingKeys: string[] = [];
2366
+ const startedAt = Date.now();
2367
+ for (const chunk of input.chunks) {
2368
+ const chunkKeys = chunk.map((entry) => entry.key);
2369
+ const chunkInputIndexes = chunk.map((entry) => entry.inputIndex);
2370
+ const existingMissingOutputSql = missingOutputCellSql(
2371
+ 'existing',
2372
+ input.outputPhysicalColumns,
2373
+ );
2374
+ const { rows } = await client.query(
2375
+ `
2376
+ WITH input_rows AS (
2377
+ SELECT DISTINCT ON (key_values._key)
2378
+ key_values._key, index_values._input_index
2379
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
2380
+ JOIN unnest($2::bigint[]) WITH ORDINALITY AS index_values(_input_index, ord)
2381
+ ON index_values.ord = key_values.ord
2382
+ ORDER BY key_values._key, key_values.ord
2383
+ ),
2384
+ existing_rows AS (
2385
+ UPDATE ${sheetTable(session)} AS target
2386
+ SET _input_index = input_rows._input_index,
2387
+ _updated_at = now(),
2388
+ _version = ${nextRuntimeSheetVersionExpression(session)}
2389
+ FROM input_rows
2390
+ WHERE target._key = input_rows._key
2391
+ AND target._input_index IS DISTINCT FROM input_rows._input_index
2392
+ RETURNING target._key
2393
+ ),
2394
+ pending_rows AS (
2395
+ SELECT input_rows._key
2396
+ FROM input_rows
2397
+ LEFT JOIN ${sheetTable(session)} AS existing
2398
+ ON existing._key = input_rows._key
2399
+ WHERE existing._key IS NULL
2400
+ OR existing._status IN ('pending', 'running', 'failed', 'stale')
2401
+ OR (
2402
+ existing._status = 'enriched'
2403
+ AND (${existingMissingOutputSql})
2404
+ )
2405
+ )
2406
+ SELECT
2407
+ coalesce((SELECT array_agg(_key) FROM pending_rows), '{}'::text[]) AS pending_keys,
2408
+ (SELECT count(*)::int FROM existing_rows) AS reordered_count
2409
+ `,
2410
+ [chunkKeys, chunkInputIndexes],
2411
+ );
2412
+ if (Array.isArray(rows[0]?.pending_keys)) {
2413
+ pendingKeys.push(...(rows[0]?.pending_keys as string[]));
2414
+ }
2415
+ }
2416
+ input.timings?.push({
2417
+ phase:
2418
+ pendingKeys.length === 0
2419
+ ? 'cached_fast_path.prepare_probe'
2420
+ : 'cached_fast_path.prepare_probe_fallback',
2421
+ ms: Date.now() - startedAt,
2422
+ rows: input.chunks.reduce((sum, chunk) => sum + chunk.length, 0),
2423
+ chunks: input.chunks.length,
2424
+ pending: pendingKeys.length,
2425
+ cached: pendingKeys.length === 0,
2426
+ });
2427
+ if (pendingKeys.length > 0) {
2428
+ return { prepared: null };
2429
+ }
2430
+ return { prepared: { inserted: 0, pendingKeys: [] } };
2431
+ }
2432
+
2433
+ async function buildRuntimeSheetDatasetStartResult(
2434
+ client: RuntimeQueryClient,
2435
+ session: RuntimePostgresSession,
2436
+ input: {
2437
+ tableNamespace: string;
2438
+ sourceRowsLength: number;
2439
+ rowEntries: RuntimeDatasetRowEntry[];
2440
+ sheetContract: PlaySheetContract;
2441
+ runId: string;
2442
+ inserted: number;
2443
+ pendingKeys: string[];
2444
+ cellPolicies?: CellStalenessPolicyByField;
2445
+ timings?: RuntimeSheetTiming[];
2446
+ },
2447
+ ): Promise<PrepareRuntimeSheetResult> {
2448
+ const outputFields = outputFieldsFromSheetContract(input.sheetContract);
2449
+ const hasCellPolicies = cellPolicyFields(input.cellPolicies).length > 0;
2450
+ if (input.pendingKeys.length === 0 && !hasCellPolicies) {
2451
+ const fastPathStartedAt = Date.now();
2452
+ const outputColumns = outputPhysicalSheetColumnProjections(
2453
+ input.sheetContract,
2454
+ );
2455
+ const completedRowsByKey = new Map(
2456
+ (
2457
+ await markAndReadRuntimeCompletedRowsCachedProjection(client, session, {
2458
+ tableNamespace: input.tableNamespace,
2459
+ keys: input.rowEntries.map((entry) => entry.key),
2460
+ runId: input.runId,
2461
+ outputFields,
2462
+ outputColumns,
2463
+ timings: input.timings,
2464
+ })
2465
+ ).map((row) => [row.key, row.data]),
2466
+ );
2467
+ if (completedRowsByKey.size === input.rowEntries.length) {
2468
+ await insertCachedRuntimeColumnSummaryDelta(client, session, {
2469
+ tableNamespace: input.tableNamespace,
2470
+ outputFields,
2471
+ cached: completedRowsByKey.size,
2472
+ });
2473
+ input.timings?.push({
2474
+ phase: 'cached_fast_path.total',
2475
+ ms: Date.now() - fastPathStartedAt,
2476
+ rows: input.rowEntries.length,
2477
+ completed: completedRowsByKey.size,
2478
+ cached: true,
2479
+ });
2480
+ return {
2481
+ inserted: input.inserted,
2482
+ skipped:
2483
+ input.rowEntries.length -
2484
+ input.inserted +
2485
+ (input.sourceRowsLength - input.rowEntries.length),
2486
+ pendingRows: [],
2487
+ completedRows: input.rowEntries.map((entry) => ({
2488
+ ...mergeRuntimeCompletedRow({
2489
+ inputRow: entry.row,
2490
+ completedData: stripRuntimeCellMeta(
2491
+ completedRowsByKey.get(entry.key) ?? {},
2492
+ ),
2493
+ sheetContract: input.sheetContract,
2494
+ }),
2495
+ __deeplineRowKey: entry.key,
2496
+ })),
2497
+ tableNamespace: input.tableNamespace,
2498
+ };
2499
+ }
2500
+ input.timings?.push({
2501
+ phase: 'cached_fast_path.fallback_incomplete',
2502
+ ms: Date.now() - fastPathStartedAt,
2503
+ rows: input.rowEntries.length,
2504
+ completed: completedRowsByKey.size,
2505
+ cached: true,
2506
+ });
2507
+ }
2508
+ const pendingKeySet = new Set(input.pendingKeys);
2509
+ const initiallyPendingRows = input.rowEntries.filter((entry) =>
2510
+ pendingKeySet.has(entry.key),
2511
+ );
2512
+ const initiallyCompletedEntries = input.rowEntries.filter(
2513
+ (entry) => !pendingKeySet.has(entry.key),
2514
+ );
2515
+ const initiallyCompletedRows = await readRuntimeRowsByKey(
2516
+ client,
2517
+ session,
2518
+ initiallyCompletedEntries.map((entry) => entry.key),
2519
+ input.sheetContract,
2520
+ );
2521
+ const initiallyCompletedRowsByKey = new Map(
2522
+ initiallyCompletedRows.map((row) => [row.key, row]),
2523
+ );
2524
+ const staleKeySet = new Set<string>();
2525
+ for (const entry of initiallyCompletedEntries) {
2526
+ const completedRow = initiallyCompletedRowsByKey.get(entry.key);
2527
+ if (!completedRow) {
2528
+ staleKeySet.add(entry.key);
2529
+ continue;
2530
+ }
2531
+ const cellMeta =
2532
+ completedRow.data[DEEPLINE_CELL_META_FIELD] &&
2533
+ typeof completedRow.data[DEEPLINE_CELL_META_FIELD] === 'object'
2534
+ ? (completedRow.data[DEEPLINE_CELL_META_FIELD] as Record<
2535
+ string,
2536
+ unknown
2537
+ >)
2538
+ : {};
2539
+ for (const field of outputFields) {
2540
+ const decision = shouldRecomputeCell({
2541
+ hasValue:
2542
+ Object.prototype.hasOwnProperty.call(completedRow.data, field) &&
2543
+ completedRow.data[field] !== null &&
2544
+ completedRow.data[field] !== undefined &&
2545
+ !(
2546
+ typeof completedRow.data[field] === 'string' &&
2547
+ completedRow.data[field].length === 0
2548
+ ),
2549
+ value: completedRow.data[field],
2550
+ meta:
2551
+ cellMeta[field] && typeof cellMeta[field] === 'object'
2552
+ ? (cellMeta[field] as {
2553
+ status?: string;
2554
+ completedAt?: number;
2555
+ staleAt?: number | null;
2556
+ staleAfterSeconds?: number | null;
2557
+ })
2558
+ : null,
2559
+ policy: input.cellPolicies?.[field],
2560
+ });
2561
+ if (decision.action === 'recompute') {
2562
+ staleKeySet.add(entry.key);
2563
+ break;
2564
+ }
2565
+ }
2566
+ }
2567
+ const pendingRows = [
2568
+ ...initiallyPendingRows,
2569
+ ...initiallyCompletedEntries.filter((entry) => staleKeySet.has(entry.key)),
2570
+ ];
2571
+ if (staleKeySet.size > 0) {
2572
+ await markRuntimeRowsPendingForCellStaleness(client, session, {
2573
+ keys: [...staleKeySet],
2574
+ runId: input.runId,
2575
+ });
2576
+ }
2577
+ const completedKeys = initiallyCompletedEntries
2578
+ .filter((entry) => !staleKeySet.has(entry.key))
2579
+ .map((entry) => entry.key);
2580
+ const existingPendingRowsByKey = new Map(
2581
+ (
2582
+ await readRuntimeRowsByKey(
2583
+ client,
2584
+ session,
2585
+ pendingRows.map((entry) => entry.key),
2586
+ input.sheetContract,
2587
+ )
2588
+ ).map((row) => [row.key, row.data]),
2589
+ );
2590
+ const completedRowsByKey = new Map(
2591
+ (
2592
+ await markAndReadRuntimeCompletedRowsCached(client, session, {
2593
+ tableNamespace: input.tableNamespace,
2594
+ keys: completedKeys,
2595
+ runId: input.runId,
2596
+ outputFields,
2597
+ sheetContract: input.sheetContract,
2598
+ })
2599
+ ).map((row) => [row.key, row.data]),
2600
+ );
2601
+ const completedKeySet = new Set(completedKeys);
2602
+ const completedRows = input.rowEntries
2603
+ .filter((entry) => completedKeySet.has(entry.key))
2604
+ .map((entry) => ({
2605
+ ...mergeRuntimeCompletedRow({
2606
+ inputRow: entry.row,
2607
+ completedData: stripRuntimeCellMeta(
2608
+ completedRowsByKey.get(entry.key) ?? {},
2609
+ ),
2610
+ sheetContract: input.sheetContract,
2611
+ }),
2612
+ __deeplineRowKey: entry.key,
2613
+ }));
2614
+ return {
2615
+ inserted: input.inserted,
2616
+ skipped:
2617
+ completedKeys.length + (input.sourceRowsLength - input.rowEntries.length),
2618
+ pendingRows: pendingRows.map((entry) => ({
2619
+ ...mergeRuntimeCompletedRow({
2620
+ inputRow: entry.row,
2621
+ completedData: existingPendingRowsByKey.get(entry.key) ?? {},
2622
+ sheetContract: input.sheetContract,
2623
+ }),
2624
+ __deeplineRowKey: entry.key,
2625
+ })),
2626
+ completedRows,
2627
+ tableNamespace: input.tableNamespace,
2628
+ };
2629
+ }
2630
+
2631
+ async function markRuntimeRowsPendingForCellStaleness(
2632
+ client: RuntimeQueryClient,
2633
+ session: RuntimePostgresSession,
2634
+ input: {
2635
+ keys: string[];
2636
+ runId: string;
2637
+ },
2638
+ ): Promise<void> {
2639
+ if (input.keys.length === 0) return;
2640
+ await client.query(
2641
+ `UPDATE ${sheetTable(session)} AS target
2642
+ SET _status = 'pending',
2643
+ _run_id = $2::text,
2644
+ _updated_at = now(),
2645
+ _version = ${nextRuntimeSheetVersionExpression(session)}
2646
+ WHERE target._key = ANY($1::text[])`,
2647
+ [input.keys, input.runId],
2648
+ );
2649
+ }
2650
+
2651
+ function stripRuntimeCellMeta(
2652
+ data: Record<string, unknown>,
2653
+ ): Record<string, unknown> {
2654
+ if (!Object.prototype.hasOwnProperty.call(data, DEEPLINE_CELL_META_FIELD)) {
2655
+ return data;
2656
+ }
2657
+ const { [DEEPLINE_CELL_META_FIELD]: _cellMeta, ...publicData } = data;
2658
+ void _cellMeta;
2659
+ return publicData;
2660
+ }
2661
+
2662
+ async function getRuntimeWorkReceiptSession(
2663
+ context: RuntimeApiContext,
2664
+ input: {
2665
+ playName: string;
2666
+ key: string;
2667
+ },
2668
+ ): Promise<RuntimePostgresSession> {
2669
+ const playName = context.playName?.trim() || input.playName;
2670
+ if (!playName) {
2671
+ throw new Error('Runtime work receipts require a playName.');
2672
+ }
2673
+ const runtimeContext = {
2674
+ ...context,
2675
+ playName,
2676
+ };
2677
+ const preloaded = findPreloadedRuntimeDbSession(runtimeContext, {
2678
+ tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
2679
+ logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
2680
+ operations: ['rows.read', 'rows.upsert'],
2681
+ });
2682
+ const session = requireRuntimePostgresSession(
2683
+ preloaded
2684
+ ? await unwrapRuntimeDbSession(runtimeContext, preloaded)
2685
+ : await getRuntimeDbSession(runtimeContext, {
2686
+ tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
2687
+ logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
2688
+ operations: ['rows.read', 'rows.upsert'],
2689
+ }),
2690
+ );
2691
+ validateRuntimeWorkReceiptKeyScope(session, { key: input.key });
2692
+ return session;
2693
+ }
2694
+
2695
+ async function readRuntimeWorkReceipt(
2696
+ client: RuntimeQueryClient,
2697
+ session: RuntimePostgresSession,
2698
+ key: string,
2699
+ ): Promise<WorkReceipt | null> {
2700
+ const { rows } = await client.query<Record<string, unknown>>(
2701
+ `SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2702
+ FROM ${workReceiptTable(session)}
2703
+ WHERE k = decode($1, 'hex')`,
2704
+ [workReceiptKeyHex(key)],
2705
+ );
2706
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
2707
+ }
2708
+
2709
+ export async function getRuntimeWorkReceipt(
2710
+ context: RuntimeApiContext,
2711
+ input: {
2712
+ playName: string;
2713
+ key: string;
2714
+ },
2715
+ ): Promise<WorkReceipt | null> {
2716
+ const session = await getRuntimeWorkReceiptSession(context, {
2717
+ playName: input.playName,
2718
+ key: input.key,
2719
+ });
2720
+ return await withRuntimeWorkReceiptClient(context, session, async (client) =>
2721
+ readRuntimeWorkReceipt(client, session, input.key),
2722
+ );
2723
+ }
2724
+
2725
+ export async function claimRuntimeWorkReceipt(
2726
+ context: RuntimeApiContext,
2727
+ input: {
2728
+ playName: string;
2729
+ runId: string;
2730
+ key: string;
2731
+ reclaimRunning?: boolean;
2732
+ },
2733
+ ): Promise<WorkReceiptClaim> {
2734
+ const session = await getRuntimeWorkReceiptSession(context, {
2735
+ playName: input.playName,
2736
+ key: input.key,
2737
+ });
2738
+ return await withRuntimeWorkReceiptClient(
2739
+ context,
2740
+ session,
2741
+ async (client) => {
2742
+ const { rows } = await client.query<Record<string, unknown>>(
2743
+ `
2744
+ WITH claimed AS (
2745
+ INSERT INTO ${workReceiptTable(session)} (k, status, run_id, updated_at)
2746
+ VALUES (decode($1, 'hex'), $2::smallint, $3, now())
2747
+ ON CONFLICT (k) DO UPDATE
2748
+ SET status = $4::smallint,
2749
+ output = NULL,
2750
+ run_id = $3,
2751
+ error = NULL,
2752
+ updated_at = now()
2753
+ WHERE ${workReceiptTable(session)}.status IN ($5::smallint, $6::smallint)
2754
+ RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2755
+ )
2756
+ SELECT k, status, output, error, run_id, updated_at
2757
+ FROM claimed
2758
+ `,
2759
+ [
2760
+ workReceiptKeyHex(input.key),
2761
+ RECEIPT_STATUS_RUNNING,
2762
+ input.runId,
2763
+ RECEIPT_STATUS_RUNNING,
2764
+ RECEIPT_STATUS_PENDING,
2765
+ RECEIPT_STATUS_FAILED,
2766
+ ],
2767
+ );
2768
+ const claimed = rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
2769
+ if (claimed) {
2770
+ return { disposition: 'claimed', receipt: claimed };
2771
+ }
2772
+
2773
+ const latest = await readRuntimeWorkReceipt(client, session, input.key);
2774
+ if (latest && isReusableWorkReceipt(latest)) {
2775
+ return { disposition: 'reused', receipt: latest };
2776
+ }
2777
+ if (latest?.status === 'running') {
2778
+ return { disposition: 'running', receipt: latest };
2779
+ }
2780
+ if (latest?.status === 'failed') {
2781
+ return { disposition: 'failed', receipt: latest };
2782
+ }
2783
+ throw new Error(
2784
+ `Runtime receipt ${input.key} claim did not return execution ownership.`,
2785
+ );
2786
+ },
2787
+ );
2788
+ }
2789
+
2790
+ export async function completeRuntimeWorkReceipt(
2791
+ context: RuntimeApiContext,
2792
+ input: {
2793
+ playName: string;
2794
+ runId: string;
2795
+ key: string;
2796
+ output: unknown;
2797
+ },
2798
+ ): Promise<WorkReceipt | null> {
2799
+ const session = await getRuntimeWorkReceiptSession(context, {
2800
+ playName: input.playName,
2801
+ key: input.key,
2802
+ });
2803
+ return await withRuntimeWorkReceiptClient(
2804
+ context,
2805
+ session,
2806
+ async (client) => {
2807
+ const { rows } = await client.query<Record<string, unknown>>(
2808
+ `
2809
+ WITH completed AS (
2810
+ UPDATE ${workReceiptTable(session)}
2811
+ SET status = $2::smallint,
2812
+ output = $3::jsonb,
2813
+ error = NULL,
2814
+ run_id = $4,
2815
+ updated_at = now()
2816
+ WHERE k = decode($1, 'hex')
2817
+ AND (run_id IS NULL OR run_id <= $4)
2818
+ RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2819
+ ),
2820
+ latest AS (
2821
+ SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2822
+ FROM ${workReceiptTable(session)}
2823
+ WHERE k = decode($1, 'hex')
2824
+ AND NOT EXISTS (SELECT 1 FROM completed)
2825
+ )
2826
+ SELECT k, status, output, error, run_id, updated_at FROM completed
2827
+ UNION ALL
2828
+ SELECT k, status, output, error, run_id, updated_at FROM latest
2829
+ `,
2830
+ [
2831
+ workReceiptKeyHex(input.key),
2832
+ RECEIPT_STATUS_COMPLETED,
2833
+ input.output === null ? null : stringifyPostgresJson(input.output),
2834
+ input.runId,
2835
+ ],
2836
+ );
2837
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
2838
+ },
2839
+ );
2840
+ }
2841
+
2842
+ export async function failRuntimeWorkReceipt(
2843
+ context: RuntimeApiContext,
2844
+ input: {
2845
+ playName: string;
2846
+ runId: string;
2847
+ key: string;
2848
+ error: string;
2849
+ },
2850
+ ): Promise<WorkReceipt | null> {
2851
+ const session = await getRuntimeWorkReceiptSession(context, {
2852
+ playName: input.playName,
2853
+ key: input.key,
2854
+ });
2855
+ return await withRuntimeWorkReceiptClient(
2856
+ context,
2857
+ session,
2858
+ async (client) => {
2859
+ const { rows } = await client.query<Record<string, unknown>>(
2860
+ `
2861
+ WITH failed AS (
2862
+ UPDATE ${workReceiptTable(session)}
2863
+ SET status = $2::smallint,
2864
+ output = NULL,
2865
+ error = $3,
2866
+ run_id = $4,
2867
+ updated_at = now()
2868
+ WHERE k = decode($1, 'hex')
2869
+ AND status <> $5::smallint
2870
+ AND status <> $6::smallint
2871
+ AND (run_id IS NULL OR run_id <= $4)
2872
+ RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2873
+ ),
2874
+ latest AS (
2875
+ SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2876
+ FROM ${workReceiptTable(session)}
2877
+ WHERE k = decode($1, 'hex')
2878
+ AND NOT EXISTS (SELECT 1 FROM failed)
2879
+ )
2880
+ SELECT k, status, output, error, run_id, updated_at FROM failed
2881
+ UNION ALL
2882
+ SELECT k, status, output, error, run_id, updated_at FROM latest
2883
+ `,
2884
+ [
2885
+ workReceiptKeyHex(input.key),
2886
+ RECEIPT_STATUS_FAILED,
2887
+ input.error,
2888
+ input.runId,
2889
+ RECEIPT_STATUS_COMPLETED,
2890
+ RECEIPT_STATUS_SKIPPED,
2891
+ ],
2892
+ );
2893
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
2894
+ },
2895
+ );
2896
+ }
2897
+
2898
+ export async function skipRuntimeWorkReceipt(
2899
+ context: RuntimeApiContext,
2900
+ input: {
2901
+ playName: string;
2902
+ runId: string;
2903
+ key: string;
2904
+ output: unknown;
2905
+ },
2906
+ ): Promise<WorkReceipt | null> {
2907
+ const session = await getRuntimeWorkReceiptSession(context, {
2908
+ playName: input.playName,
2909
+ key: input.key,
2910
+ });
2911
+ return await withRuntimeWorkReceiptClient(
2912
+ context,
2913
+ session,
2914
+ async (client) => {
2915
+ const { rows } = await client.query<Record<string, unknown>>(
2916
+ `
2917
+ WITH skipped AS (
2918
+ UPDATE ${workReceiptTable(session)}
2919
+ SET status = $2::smallint,
2920
+ output = $3::jsonb,
2921
+ error = NULL,
2922
+ run_id = $4,
2923
+ updated_at = now()
2924
+ WHERE k = decode($1, 'hex')
2925
+ AND (run_id IS NULL OR run_id <= $4)
2926
+ RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2927
+ ),
2928
+ latest AS (
2929
+ SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2930
+ FROM ${workReceiptTable(session)}
2931
+ WHERE k = decode($1, 'hex')
2932
+ AND NOT EXISTS (SELECT 1 FROM skipped)
2933
+ )
2934
+ SELECT k, status, output, error, run_id, updated_at FROM skipped
2935
+ UNION ALL
2936
+ SELECT k, status, output, error, run_id, updated_at FROM latest
2937
+ `,
2938
+ [
2939
+ workReceiptKeyHex(input.key),
2940
+ RECEIPT_STATUS_SKIPPED,
2941
+ input.output === null ? null : stringifyPostgresJson(input.output),
2942
+ input.runId,
2943
+ ],
2944
+ );
2945
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
2946
+ },
2947
+ );
2948
+ }
2949
+
2950
+ export async function startRuntimeSheetDataset(
2951
+ context: RuntimeApiContext,
2952
+ input: {
2953
+ playName: string;
2954
+ tableNamespace: string;
2955
+ playInput?: Record<string, unknown> | null;
2956
+ sheetContract: PlaySheetContract;
2957
+ rows: Record<string, unknown>[];
2958
+ runId: string;
2959
+ inputOffset?: number;
2960
+ cellPolicies?: CellStalenessPolicyByField;
2961
+ },
2962
+ ): Promise<PrepareRuntimeSheetResult> {
2963
+ const totalStartedAt = Date.now();
2964
+ const timings: RuntimeSheetTiming[] = [];
2965
+ const playName = context.playName?.trim() || input.playName;
2966
+ if (!playName) {
2967
+ throw new Error('Runtime DB sessions require a playName.');
2968
+ }
2969
+ const sessionStartedAt = Date.now();
2970
+ const session = requireRuntimePostgresSession(
2971
+ await getRuntimeDbSession(
2972
+ {
2973
+ ...context,
2974
+ playName,
2975
+ runId: context.runId ?? input.runId,
2976
+ },
2977
+ {
2978
+ tableNamespace: input.tableNamespace,
2979
+ logicalTable: 'sheet_rows',
2980
+ operations: ['rows.read', 'rows.upsert'],
2981
+ limits: {
2982
+ maxRows: runtimeDbSessionRowLimit(input.rows.length),
2983
+ },
2984
+ sheetContract: input.sheetContract,
2985
+ timings,
2986
+ },
2987
+ ),
2988
+ );
2989
+ timings.push({
2990
+ phase: 'db_session',
2991
+ ms: Date.now() - sessionStartedAt,
2992
+ rows: input.rows.length,
2993
+ });
2994
+ const normalizeStartedAt = Date.now();
2995
+ const uniqueRows = new Map<string, Record<string, unknown>>();
2996
+ for (const row of input.rows) {
2997
+ // Materializes projected CSV aliases as visible cells and drops internal
2998
+ // __deepline* keys — a plain spread would silently lose the
2999
+ // non-enumerable alias fields on the JSON payload boundary.
3000
+ const cleanedRow = toSerializableCsvAliasedRow(row);
3001
+ const key =
3002
+ typeof row.__deeplineRowKey === 'string'
3003
+ ? row.__deeplineRowKey
3004
+ : derivePlayRowIdentity(cleanedRow, input.tableNamespace);
3005
+ if (key && !uniqueRows.has(key)) {
3006
+ uniqueRows.set(key, cleanedRow);
3007
+ }
3008
+ }
3009
+ const inputOffset = Math.max(0, Math.floor(input.inputOffset ?? 0));
3010
+ const rowEntries = [...uniqueRows.entries()].map(
3011
+ ([key, row], inputIndex) => ({
3012
+ key,
3013
+ row,
3014
+ inputIndex: inputOffset + inputIndex,
3015
+ }),
3016
+ );
3017
+ timings.push({
3018
+ phase: 'normalize_rows',
3019
+ ms: Date.now() - normalizeStartedAt,
3020
+ rows: input.rows.length,
3021
+ });
3022
+ if (rowEntries.length === 0) {
3023
+ return {
3024
+ inserted: 0,
3025
+ skipped: input.rows.length,
3026
+ pendingRows: [],
3027
+ completedRows: [],
3028
+ tableNamespace: input.tableNamespace,
3029
+ timings: [
3030
+ ...timings,
3031
+ {
3032
+ phase: 'total',
3033
+ ms: Date.now() - totalStartedAt,
3034
+ rows: input.rows.length,
3035
+ },
3036
+ ],
3037
+ };
3038
+ }
3039
+
3040
+ const physicalColumns = physicalSheetColumnNames(input.sheetContract);
3041
+ const physicalInsertColumnsSql =
3042
+ physicalColumns.length > 0
3043
+ ? `, ${physicalColumns.map(quoteIdentifier).join(', ')}`
3044
+ : '';
3045
+ const physicalInsertValuesSql =
3046
+ physicalColumns.length > 0
3047
+ ? `, ${physicalColumns
3048
+ .map((column) => `payload -> ${quoteLiteral(column)}`)
3049
+ .join(', ')}`
3050
+ : '';
3051
+ const outputPhysicalColumns = outputPhysicalSheetColumnNames(
3052
+ input.sheetContract,
3053
+ );
3054
+ const normalizedPlayName = normalizePlayNameForSheet(playName);
3055
+ const normalizedTableNamespace = normalizeTableNamespace(
3056
+ input.tableNamespace,
3057
+ );
3058
+
3059
+ const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
3060
+ const needsTransaction = chunks.length > 1;
3061
+
3062
+ const result = await withRuntimeSheetQueryClient(
3063
+ context,
3064
+ session,
3065
+ {
3066
+ playName,
3067
+ tableNamespace: input.tableNamespace,
3068
+ sheetContract: input.sheetContract,
3069
+ transactional: needsTransaction,
3070
+ timings,
3071
+ },
3072
+ async (client) => {
3073
+ const prepareStartedAt = Date.now();
3074
+ const hasCellPolicies = cellPolicyFields(input.cellPolicies).length > 0;
3075
+ const cachedProbe = hasCellPolicies
3076
+ ? { prepared: null }
3077
+ : await tryPrepareRuntimeSheetDatasetRowsCachedOnly(client, session, {
3078
+ chunks,
3079
+ outputPhysicalColumns,
3080
+ timings,
3081
+ });
3082
+ const prepared =
3083
+ cachedProbe.prepared ??
3084
+ (await prepareRuntimeSheetDatasetRows(client, session, {
3085
+ chunks,
3086
+ runId: input.runId,
3087
+ normalizedPlayName,
3088
+ normalizedTableNamespace,
3089
+ physicalInsertColumnsSql,
3090
+ physicalInsertValuesSql,
3091
+ outputPhysicalColumns,
3092
+ }));
3093
+ timings.push({
3094
+ phase: 'prepare_rows_sql',
3095
+ ms: Date.now() - prepareStartedAt,
3096
+ rows: rowEntries.length,
3097
+ chunks: chunks.length,
3098
+ inserted: prepared.inserted,
3099
+ pending: prepared.pendingKeys.length,
3100
+ });
3101
+ const buildStartedAt = Date.now();
3102
+ const built = await buildRuntimeSheetDatasetStartResult(client, session, {
3103
+ tableNamespace: input.tableNamespace,
3104
+ sourceRowsLength: input.rows.length,
3105
+ rowEntries,
3106
+ sheetContract: input.sheetContract,
3107
+ runId: input.runId,
3108
+ cellPolicies: input.cellPolicies,
3109
+ timings,
3110
+ ...prepared,
3111
+ });
3112
+ timings.push({
3113
+ phase: 'build_result',
3114
+ ms: Date.now() - buildStartedAt,
3115
+ rows: rowEntries.length,
3116
+ inserted: built.inserted,
3117
+ skipped: built.skipped,
3118
+ pending: built.pendingRows.length,
3119
+ completed: built.completedRows.length,
3120
+ });
3121
+ return built;
3122
+ },
3123
+ );
3124
+ timings.push({
3125
+ phase: 'total',
3126
+ ms: Date.now() - totalStartedAt,
3127
+ rows: input.rows.length,
3128
+ chunks: chunks.length,
3129
+ inserted: result.inserted,
3130
+ skipped: result.skipped,
3131
+ pending: result.pendingRows.length,
3132
+ completed: result.completedRows.length,
3133
+ });
3134
+ return { ...result, timings };
3135
+ }
3136
+
3137
+ async function completeRuntimeMapRowChunks(
3138
+ client: RuntimeQueryClient,
3139
+ session: RuntimePostgresSession,
3140
+ input: {
3141
+ chunks: RuntimePreparedCompletedRow[][];
3142
+ physicalUpdateSetSql: string;
3143
+ physicalColumnProjections: PhysicalSheetColumnProjection[];
3144
+ runId: string;
3145
+ normalizedPlayName: string;
3146
+ normalizedTableNamespace: string;
3147
+ outputFields: string[];
3148
+ },
3149
+ ): Promise<{ updated: number }> {
3150
+ let updated = 0;
3151
+ for (const chunk of input.chunks) {
3152
+ const chunkKeys = chunk.map((row) => row.key);
3153
+ const chunkInputIndexes = chunk.map((row) => row.input_index);
3154
+ const chunkDataPatches = chunk.map((row) =>
3155
+ stringifyPostgresJson(row.data_patch),
3156
+ );
3157
+ const chunkCellMetaPatches = chunk.map((row) =>
3158
+ stringifyPostgresJson(row.cell_meta_patch),
3159
+ );
3160
+ const targetChangedPatchedCellSql = changedPatchedCellSql(
3161
+ 'target',
3162
+ 'updates.data_patch',
3163
+ input.physicalColumnProjections,
3164
+ );
3165
+ const { rows: appliedRows } = await client.query<{ _key: string }>(
3166
+ `WITH updates AS (
3167
+ SELECT key_values._key,
3168
+ input_index_values.input_index,
3169
+ data_values.data_patch,
3170
+ cell_meta_values.cell_meta_patch
3171
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
3172
+ JOIN unnest($2::bigint[]) WITH ORDINALITY AS input_index_values(input_index, ord)
3173
+ ON input_index_values.ord = key_values.ord
3174
+ JOIN unnest($3::jsonb[]) WITH ORDINALITY AS data_values(data_patch, ord)
3175
+ ON data_values.ord = key_values.ord
3176
+ JOIN unnest($4::jsonb[]) WITH ORDINALITY AS cell_meta_values(cell_meta_patch, ord)
3177
+ ON cell_meta_values.ord = key_values.ord
3178
+ ),
3179
+ matched_updates AS (
3180
+ SELECT DISTINCT ON (target._key)
3181
+ target._key AS matched_key,
3182
+ updates.data_patch,
3183
+ updates.cell_meta_patch
3184
+ FROM updates
3185
+ JOIN ${sheetTable(session)} AS target
3186
+ ON target._key = updates._key
3187
+ OR (
3188
+ updates.input_index IS NOT NULL
3189
+ AND target._run_id = $5::text
3190
+ AND target._input_index = updates.input_index
3191
+ )
3192
+ ORDER BY target._key, (target._key = updates._key) DESC
3193
+ ),
3194
+ applied_rows AS (
3195
+ UPDATE ${sheetTable(session)} AS target
3196
+ SET _status = 'enriched',
3197
+ _run_id = $5::text,
3198
+ _error = NULL,
3199
+ _updated_at = now(),
3200
+ _version = ${nextRuntimeSheetVersionExpression(session)},
3201
+ _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
3202
+ FROM matched_updates AS updates, ${sheetTable(session)} AS prev
3203
+ WHERE target._key = updates.matched_key
3204
+ AND prev._key = target._key
3205
+ AND (target._run_id IS NULL OR target._run_id <= $5::text)
3206
+ AND (
3207
+ target._status <> 'enriched'
3208
+ OR (${targetChangedPatchedCellSql})
3209
+ OR EXISTS (
3210
+ SELECT 1
3211
+ FROM unnest($8::text[]) AS field_values(field)
3212
+ WHERE coalesce(target._cell_meta -> field_values.field ->> 'status', '') <> 'completed'
3213
+ )
3214
+ )
3215
+ RETURNING target._key, prev._status AS prev_status, prev._cell_meta AS prev_cell_meta
3216
+ ),
3217
+ applied_count AS (
3218
+ SELECT count(*)::bigint AS c,
3219
+ count(*) FILTER (WHERE prev_status = 'failed')::bigint AS from_failed,
3220
+ count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
3221
+ FROM applied_rows
3222
+ ),
3223
+ summary_delta AS (
3224
+ INSERT INTO ${summaryTable(session)} AS target (
3225
+ play_name,
3226
+ table_namespace,
3227
+ total,
3228
+ queued,
3229
+ running,
3230
+ completed,
3231
+ failed
3232
+ )
3233
+ SELECT $6::text, $7::text, 0, 0, 0, newly_completed::int, (-from_failed)::int
3234
+ FROM applied_count
3235
+ WHERE newly_completed > 0 OR from_failed > 0
3236
+ ON CONFLICT (play_name, table_namespace) DO UPDATE SET
3237
+ queued = GREATEST(target.queued - (EXCLUDED.completed + EXCLUDED.failed), 0),
3238
+ completed = GREATEST(target.completed + EXCLUDED.completed, 0),
3239
+ failed = GREATEST(target.failed + EXCLUDED.failed, 0),
3240
+ _updated_at = now()
3241
+ RETURNING 1
3242
+ ),
3243
+ completed_cell_delta AS (
3244
+ SELECT field_values.field, count(*)::bigint AS c
3245
+ FROM applied_rows
3246
+ JOIN unnest($8::text[]) AS field_values(field)
3247
+ ON applied_rows.prev_status <> 'enriched'
3248
+ OR coalesce(applied_rows.prev_cell_meta -> field_values.field ->> 'status', '') <> 'completed'
3249
+ GROUP BY field_values.field
3250
+ ),
3251
+ prev_failed_cells AS (
3252
+ SELECT field_values.field, count(*)::bigint AS c
3253
+ FROM applied_rows
3254
+ JOIN unnest($8::text[]) AS field_values(field)
3255
+ ON coalesce(applied_rows.prev_cell_meta -> field_values.field ->> 'status', '') = 'failed'
3256
+ GROUP BY field_values.field
3257
+ ),
3258
+ column_delta AS (
3259
+ INSERT INTO ${columnSummaryTable(session)} AS target (
3260
+ play_name,
3261
+ table_namespace,
3262
+ field,
3263
+ completed,
3264
+ failed
3265
+ )
3266
+ SELECT $6::text,
3267
+ $7::text,
3268
+ coalesce(completed_cell_delta.field, prev_failed_cells.field),
3269
+ coalesce(completed_cell_delta.c, 0)::int,
3270
+ (-coalesce(prev_failed_cells.c, 0))::int
3271
+ FROM completed_cell_delta
3272
+ FULL JOIN prev_failed_cells
3273
+ ON prev_failed_cells.field = completed_cell_delta.field
3274
+ WHERE coalesce(completed_cell_delta.c, 0) > 0
3275
+ OR coalesce(prev_failed_cells.c, 0) > 0
3276
+ ON CONFLICT (play_name, table_namespace, field) DO UPDATE SET
3277
+ completed = GREATEST(target.completed + EXCLUDED.completed, 0),
3278
+ failed = GREATEST(target.failed + EXCLUDED.failed, 0),
3279
+ _updated_at = now()
3280
+ RETURNING 1
3281
+ )
3282
+ SELECT _key FROM applied_rows`,
3283
+ [
3284
+ chunkKeys,
3285
+ chunkInputIndexes,
3286
+ chunkDataPatches,
3287
+ chunkCellMetaPatches,
3288
+ input.runId,
3289
+ input.normalizedPlayName,
3290
+ input.normalizedTableNamespace,
3291
+ [...new Set(input.outputFields)],
3292
+ ],
3293
+ );
3294
+ updated += appliedRows.length;
3295
+ }
3296
+ return { updated };
3297
+ }
3298
+
3299
+ /**
3300
+ * Mark map rows FAILED in the per-run scoped Postgres sheet table by key.
3301
+ *
3302
+ * Row failure isolation contract: one row's tool/provider error must not abort
3303
+ * sibling rows or the run. The failed row keeps every field value that
3304
+ * completed before the error (written through the same physical-column patch
3305
+ * as completed rows), flips `_status` to 'failed', records the row error in
3306
+ * `_error`, and merges the per-cell failure into `_cell_meta`. The next run's
3307
+ * `startRuntimeSheetDataset` returns failed rows as pending, so they re-execute
3308
+ * while their already-completed cells replay free via cell receipts.
3309
+ */
3310
+ async function failRuntimeMapRowChunks(
3311
+ client: RuntimeQueryClient,
3312
+ session: RuntimePostgresSession,
3313
+ input: {
3314
+ chunks: RuntimePreparedFailedRow[][];
3315
+ physicalUpdateSetSql: string;
3316
+ runId: string;
3317
+ normalizedPlayName: string;
3318
+ normalizedTableNamespace: string;
3319
+ },
3320
+ ): Promise<{ updated: number }> {
3321
+ let updated = 0;
3322
+ for (const chunk of input.chunks) {
3323
+ const chunkKeys = chunk.map((row) => row.key);
3324
+ const chunkInputIndexes = chunk.map((row) => row.input_index);
3325
+ const chunkDataPatches = chunk.map((row) =>
3326
+ stringifyPostgresJson(row.data_patch),
3327
+ );
3328
+ const chunkCellMetaPatches = chunk.map((row) =>
3329
+ stringifyPostgresJson(row.cell_meta_patch),
3330
+ );
3331
+ const chunkErrors = chunk.map((row) => row.error);
3332
+ // Per-field failed-cell counts for the column summary, computed from the
3333
+ // cell meta patches (a failed row records exactly which cell failed).
3334
+ const failedCellCounts = new Map<string, number>();
3335
+ for (const row of chunk) {
3336
+ for (const [field, meta] of Object.entries(row.cell_meta_patch)) {
3337
+ if (
3338
+ meta &&
3339
+ typeof meta === 'object' &&
3340
+ (meta as { status?: unknown }).status === 'failed'
3341
+ ) {
3342
+ failedCellCounts.set(field, (failedCellCounts.get(field) ?? 0) + 1);
3343
+ }
3344
+ }
3345
+ }
3346
+ const failedCellFields = [...failedCellCounts.keys()];
3347
+ const failedCellTotals = failedCellFields.map(
3348
+ (field) => failedCellCounts.get(field) ?? 0,
3349
+ );
3350
+ const { rows: appliedRows } = await client.query<{ _key: string }>(
3351
+ `WITH updates AS (
3352
+ SELECT key_values._key,
3353
+ input_index_values.input_index,
3354
+ data_values.data_patch,
3355
+ cell_meta_values.cell_meta_patch,
3356
+ error_values.error
3357
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(_key, ord)
3358
+ JOIN unnest($2::bigint[]) WITH ORDINALITY AS input_index_values(input_index, ord)
3359
+ ON input_index_values.ord = key_values.ord
3360
+ JOIN unnest($3::jsonb[]) WITH ORDINALITY AS data_values(data_patch, ord)
3361
+ ON data_values.ord = key_values.ord
3362
+ JOIN unnest($4::jsonb[]) WITH ORDINALITY AS cell_meta_values(cell_meta_patch, ord)
3363
+ ON cell_meta_values.ord = key_values.ord
3364
+ JOIN unnest($5::text[]) WITH ORDINALITY AS error_values(error, ord)
3365
+ ON error_values.ord = key_values.ord
3366
+ ),
3367
+ matched_updates AS (
3368
+ SELECT DISTINCT ON (target._key)
3369
+ target._key AS matched_key,
3370
+ updates.data_patch,
3371
+ updates.cell_meta_patch,
3372
+ updates.error
3373
+ FROM updates
3374
+ JOIN ${sheetTable(session)} AS target
3375
+ ON target._key = updates._key
3376
+ OR (
3377
+ updates.input_index IS NOT NULL
3378
+ AND target._run_id = $6::text
3379
+ AND target._input_index = updates.input_index
3380
+ )
3381
+ ORDER BY target._key, (target._key = updates._key) DESC
3382
+ ),
3383
+ applied_rows AS (
3384
+ UPDATE ${sheetTable(session)} AS target
3385
+ SET _status = 'failed',
3386
+ _run_id = $6::text,
3387
+ _error = updates.error,
3388
+ _updated_at = now(),
3389
+ _version = ${nextRuntimeSheetVersionExpression(session)},
3390
+ _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
3391
+ FROM matched_updates AS updates, ${sheetTable(session)} AS prev
3392
+ WHERE target._key = updates.matched_key
3393
+ AND prev._key = target._key
3394
+ AND target._status <> 'enriched'
3395
+ AND (target._run_id IS NULL OR target._run_id <= $6::text)
3396
+ RETURNING target._key, prev._status AS prev_status
3397
+ ),
3398
+ applied_count AS (
3399
+ SELECT count(*)::bigint AS c,
3400
+ count(*) FILTER (WHERE prev_status = 'failed')::bigint AS already_failed
3401
+ FROM applied_rows
3402
+ ),
3403
+ summary_delta AS (
3404
+ INSERT INTO ${summaryTable(session)} AS target (
3405
+ play_name,
3406
+ table_namespace,
3407
+ total,
3408
+ queued,
3409
+ running,
3410
+ completed,
3411
+ failed
3412
+ )
3413
+ SELECT $7::text, $8::text, 0, 0, 0, 0, (c - already_failed)::int
3414
+ FROM applied_count
3415
+ WHERE c > 0
3416
+ ON CONFLICT (play_name, table_namespace) DO UPDATE SET
3417
+ queued = GREATEST(target.queued - EXCLUDED.failed, 0),
3418
+ failed = GREATEST(target.failed + EXCLUDED.failed, 0),
3419
+ _updated_at = now()
3420
+ RETURNING 1
3421
+ ),
3422
+ column_delta AS (
3423
+ INSERT INTO ${columnSummaryTable(session)} AS target (
3424
+ play_name,
3425
+ table_namespace,
3426
+ field,
3427
+ failed
3428
+ )
3429
+ SELECT $7::text, $8::text, field_values.field, count_values.c
3430
+ FROM unnest($9::text[]) WITH ORDINALITY AS field_values(field, ord)
3431
+ JOIN unnest($10::int[]) WITH ORDINALITY AS count_values(c, ord)
3432
+ ON count_values.ord = field_values.ord
3433
+ WHERE EXISTS (SELECT 1 FROM applied_rows)
3434
+ ON CONFLICT (play_name, table_namespace, field) DO UPDATE SET
3435
+ failed = GREATEST(target.failed + EXCLUDED.failed, 0),
3436
+ _updated_at = now()
3437
+ RETURNING 1
3438
+ )
3439
+ SELECT _key FROM applied_rows`,
3440
+ [
3441
+ chunkKeys,
3442
+ chunkInputIndexes,
3443
+ chunkDataPatches,
3444
+ chunkCellMetaPatches,
3445
+ chunkErrors,
3446
+ input.runId,
3447
+ input.normalizedPlayName,
3448
+ input.normalizedTableNamespace,
3449
+ failedCellFields,
3450
+ failedCellTotals,
3451
+ ],
3452
+ );
3453
+ updated += appliedRows.length;
3454
+ }
3455
+ return { updated };
3456
+ }
3457
+
3458
+ function normalizeRuntimeMapInputIndex(value: unknown): number | null {
3459
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
3460
+ return null;
3461
+ }
3462
+ const normalized = Math.floor(value);
3463
+ return normalized >= 0 ? normalized : null;
3464
+ }
3465
+
3466
+ /**
3467
+ * Mark map rows terminal in the per-run scoped Postgres sheet table by key.
3468
+ * Mirrors server-side `store.completeSheetRows` semantics: UPDATE-by-key with
3469
+ * the row's enriched output values written into materialized physical
3470
+ * columns, _status flipped to 'enriched', and the sheet cursor advanced.
3471
+ *
3472
+ * Rows with `status: 'failed'` instead persist a row-isolated failure
3473
+ * (`_status='failed'`, `_error`, per-cell failure meta) while keeping the
3474
+ * field values that completed before the error — see
3475
+ * `failRuntimeMapRowChunks` for the recovery contract.
3476
+ *
3477
+ * Used by the workers_edge harness's `persistCompletedMapRows` so map
3478
+ * completion writes go through the shared runtime storage plane and skip the
3479
+ * per-chunk Vercel hop.
3480
+ */
3481
+ export async function completeRuntimeMapRows(
3482
+ context: RuntimeApiContext & { playName: string },
3483
+ input: {
3484
+ tableNamespace: string;
3485
+ sheetContract: PlaySheetContract;
3486
+ rows: RuntimeApiRowRecord[];
3487
+ outputFields?: string[];
3488
+ runId: string;
3489
+ },
3490
+ ): Promise<{ updated: number }> {
3491
+ if (input.rows.length === 0) {
3492
+ return { updated: 0 };
3493
+ }
3494
+ const sheetContract = augmentSheetContractWithDatasetFields({
3495
+ contract: input.sheetContract,
3496
+ rows: input.rows.map((row) => row.data),
3497
+ outputFields: input.outputFields,
3498
+ });
3499
+ const session = requireRuntimePostgresSession(
3500
+ await getRuntimeDbSession(
3501
+ {
3502
+ ...context,
3503
+ playName: context.playName,
3504
+ runId: context.runId ?? input.runId,
3505
+ },
3506
+ {
3507
+ tableNamespace: input.tableNamespace,
3508
+ logicalTable: 'sheet_rows',
3509
+ operations: ['rows.read', 'rows.upsert'],
3510
+ limits: {
3511
+ maxRows: runtimeDbSessionRowLimit(input.rows.length),
3512
+ },
3513
+ sheetContract,
3514
+ },
3515
+ ),
3516
+ );
3517
+
3518
+ // Dedupe by key. Last write wins on payload, status, and error.
3519
+ const uniqueRows = new Map<string, RuntimeApiRowRecord>();
3520
+ for (const row of input.rows) {
3521
+ if (!row.key) continue;
3522
+ uniqueRows.set(row.key, row);
3523
+ }
3524
+ if (uniqueRows.size === 0) {
3525
+ return { updated: 0 };
3526
+ }
3527
+
3528
+ const projections = physicalSheetColumnProjections(sheetContract);
3529
+ const physicalUpdateSetSql =
3530
+ projections.length > 0
3531
+ ? `,\n ${projections
3532
+ .map((column) => {
3533
+ const quoted = quoteIdentifier(column.sqlName);
3534
+ const literal = quoteLiteral(column.fieldName);
3535
+ return `${quoted} = CASE
3536
+ WHEN coalesce(updates.data_patch, '{}'::jsonb) ? ${literal}
3537
+ THEN updates.data_patch -> ${literal}
3538
+ ELSE target.${quoted}
3539
+ END`;
3540
+ })
3541
+ .join(',\n ')}`
3542
+ : '';
3543
+
3544
+ const completedRows: RuntimePreparedCompletedRow[] = [];
3545
+ const failedRows: RuntimePreparedFailedRow[] = [];
3546
+ for (const [key, row] of uniqueRows.entries()) {
3547
+ if (row.status === 'failed') {
3548
+ // Failed rows persist only the cell meta the caller recorded (completed
3549
+ // sibling cells + the failed cell). Defaulting every output field to
3550
+ // 'completed' here would lie about the failed cell and break re-run
3551
+ // recompute.
3552
+ failedRows.push({
3553
+ key,
3554
+ input_index: normalizeRuntimeMapInputIndex(row.inputIndex),
3555
+ data_patch: row.data,
3556
+ cell_meta_patch: row.cellMetaPatch ?? {},
3557
+ error:
3558
+ typeof row.error === 'string' && row.error.trim()
3559
+ ? row.error
3560
+ : 'Row execution failed.',
3561
+ });
3562
+ continue;
3563
+ }
3564
+ completedRows.push({
3565
+ key,
3566
+ input_index: normalizeRuntimeMapInputIndex(row.inputIndex),
3567
+ data_patch: row.data,
3568
+ cell_meta_patch: completedRuntimeCellMetaPatch({
3569
+ runId: input.runId,
3570
+ outputFields: input.outputFields ?? [],
3571
+ rowPatch: row.cellMetaPatch,
3572
+ }),
3573
+ });
3574
+ }
3575
+ const chunks = chunkValues(completedRows, DIRECT_POSTGRES_BATCH_SIZE);
3576
+ const failedChunks = chunkValues(failedRows, DIRECT_POSTGRES_BATCH_SIZE);
3577
+ const needsTransaction = chunks.length + failedChunks.length > 1;
3578
+ const outputFields = [...new Set(input.outputFields ?? [])];
3579
+
3580
+ return await withRuntimeSheetQueryClient(
3581
+ context,
3582
+ session,
3583
+ {
3584
+ playName: context.playName,
3585
+ tableNamespace: input.tableNamespace,
3586
+ sheetContract,
3587
+ transactional: needsTransaction,
3588
+ },
3589
+ async (client) => {
3590
+ const completed =
3591
+ completedRows.length > 0
3592
+ ? await completeRuntimeMapRowChunks(client, session, {
3593
+ chunks,
3594
+ physicalUpdateSetSql,
3595
+ physicalColumnProjections: projections,
3596
+ runId: input.runId,
3597
+ normalizedPlayName: normalizePlayNameForSheet(session.playName),
3598
+ normalizedTableNamespace: normalizeTableNamespace(
3599
+ input.tableNamespace,
3600
+ ),
3601
+ outputFields,
3602
+ })
3603
+ : { updated: 0 };
3604
+ const failed =
3605
+ failedRows.length > 0
3606
+ ? await failRuntimeMapRowChunks(client, session, {
3607
+ chunks: failedChunks,
3608
+ physicalUpdateSetSql,
3609
+ runId: input.runId,
3610
+ normalizedPlayName: normalizePlayNameForSheet(session.playName),
3611
+ normalizedTableNamespace: normalizeTableNamespace(
3612
+ input.tableNamespace,
3613
+ ),
3614
+ })
3615
+ : { updated: 0 };
3616
+ return { updated: completed.updated + failed.updated };
3617
+ },
3618
+ );
3619
+ }
3620
+
3621
+ export async function readRuntimeSheetDatasetRows(
3622
+ context: RuntimeApiContext & { playName: string },
3623
+ input: {
3624
+ tableNamespace: string;
3625
+ runId?: string | null;
3626
+ limit: number;
3627
+ offset: number;
3628
+ },
3629
+ ): Promise<{ rows: Record<string, unknown>[]; limit: number; offset: number }> {
3630
+ const limit = Math.max(
3631
+ 1,
3632
+ Math.min(DIRECT_POSTGRES_BATCH_SIZE, Math.floor(input.limit)),
3633
+ );
3634
+ const offset = Math.max(0, Math.floor(input.offset));
3635
+ const session = requireRuntimePostgresSession(
3636
+ await getRuntimeDbSession(
3637
+ {
3638
+ ...context,
3639
+ playName: context.playName,
3640
+ runId: context.runId ?? input.runId,
3641
+ },
3642
+ {
3643
+ tableNamespace: input.tableNamespace,
3644
+ logicalTable: 'sheet_rows',
3645
+ operations: ['rows.read'],
3646
+ limits: { maxRows: runtimeDbSessionRowLimit(limit) },
3647
+ },
3648
+ ),
3649
+ );
3650
+ const rows = await readRuntimeRows(session, {
3651
+ limit,
3652
+ offset,
3653
+ runId: input.runId ?? null,
3654
+ });
3655
+ return {
3656
+ rows: rows.map((row) => row.data),
3657
+ limit,
3658
+ offset,
3659
+ };
3660
+ }
3661
+
3662
+ export async function persistRuntimeCsvDataset(
3663
+ context: RuntimeApiContext & { playName: string },
3664
+ input: {
3665
+ tableNamespace: string;
3666
+ playInput?: Record<string, unknown> | null;
3667
+ sheetContract: PlaySheetContract;
3668
+ rows: Record<string, unknown>[];
3669
+ runId: string;
3670
+ sourceLabel?: string | null;
3671
+ },
3672
+ ): Promise<PlayDataset<Record<string, unknown>>> {
3673
+ await startRuntimeSheetDataset(context, {
3674
+ playName: context.playName,
3675
+ tableNamespace: input.tableNamespace,
3676
+ playInput: input.playInput,
3677
+ sheetContract: input.sheetContract,
3678
+ rows: input.rows,
3679
+ runId: input.runId,
3680
+ });
3681
+ return createRuntimeBackedPlayDataset({
3682
+ context,
3683
+ tableNamespace: input.tableNamespace,
3684
+ datasetKind: 'csv',
3685
+ sourceLabel: input.sourceLabel,
3686
+ initialCount: input.rows.length,
3687
+ initialPreviewRows: input.rows.slice(0, 10),
3688
+ runId: input.runId,
3689
+ });
3690
+ }
3691
+
3692
+ export async function upsertRuntimeRows(
3693
+ context: RuntimeApiContext,
3694
+ input: {
3695
+ playName: string;
3696
+ tableNamespace: string;
3697
+ rows: Record<string, unknown>[];
3698
+ runId: string;
3699
+ sheetContract: PlaySheetContract;
3700
+ skipEnsureSheet?: boolean;
3701
+ },
3702
+ ): Promise<RowsWriteResponse> {
3703
+ if (!context.playName) {
3704
+ throw new Error('Runtime DB sessions require a playName.');
3705
+ }
3706
+ const session = await getRuntimeDbSession(
3707
+ {
3708
+ ...context,
3709
+ playName: context.playName,
3710
+ runId: context.runId ?? input.runId,
3711
+ },
3712
+ {
3713
+ tableNamespace: input.tableNamespace,
3714
+ logicalTable: 'sheet_rows',
3715
+ operations: ['rows.upsert'],
3716
+ limits: {
3717
+ maxRows: runtimeDbSessionRowLimit(input.rows.length),
3718
+ },
3719
+ sheetContract: input.sheetContract,
3720
+ },
3721
+ );
3722
+ return rowsWriteResponseSchema.parse(
3723
+ await writeRuntimeRows(requireRuntimePostgresSession(session), {
3724
+ tableNamespace: input.tableNamespace,
3725
+ rows: input.rows,
3726
+ runId: input.runId,
3727
+ sheetContract: input.sheetContract,
3728
+ idempotencyKey: createHash('sha1')
3729
+ .update(
3730
+ `${input.playName}:${input.tableNamespace}:${input.runId}:${JSON.stringify(input.rows)}`,
3731
+ )
3732
+ .digest('hex'),
3733
+ mode: 'upsert',
3734
+ }),
3735
+ );
3736
+ }
3737
+
3738
+ export async function appendRuntimeRows(
3739
+ context: RuntimeApiContext,
3740
+ input: {
3741
+ playName: string;
3742
+ tableNamespace: string;
3743
+ rows: Record<string, unknown>[];
3744
+ runId: string;
3745
+ idempotencyKey: string;
3746
+ sheetContract: PlaySheetContract;
3747
+ },
3748
+ ): Promise<RowsWriteResponse> {
3749
+ if (!context.playName) {
3750
+ throw new Error('Runtime DB sessions require a playName.');
3751
+ }
3752
+ const session = await getRuntimeDbSession(
3753
+ {
3754
+ ...context,
3755
+ playName: context.playName,
3756
+ runId: context.runId ?? input.runId,
3757
+ },
3758
+ {
3759
+ tableNamespace: input.tableNamespace,
3760
+ logicalTable: 'sheet_rows',
3761
+ operations: ['rows.append'],
3762
+ limits: {
3763
+ maxRows: runtimeDbSessionRowLimit(input.rows.length),
3764
+ },
3765
+ sheetContract: input.sheetContract,
3766
+ },
3767
+ );
3768
+ return rowsWriteResponseSchema.parse(
3769
+ await writeRuntimeRows(requireRuntimePostgresSession(session), {
3770
+ tableNamespace: input.tableNamespace,
3771
+ rows: input.rows,
3772
+ runId: input.runId,
3773
+ sheetContract: input.sheetContract,
3774
+ idempotencyKey: input.idempotencyKey,
3775
+ mode: 'append',
3776
+ }),
3777
+ );
3778
+ }
3779
+
3780
+ export async function replaceRuntimeRows(
3781
+ context: RuntimeApiContext,
3782
+ input: {
3783
+ playName: string;
3784
+ tableNamespace: string;
3785
+ rows: Record<string, unknown>[];
3786
+ runId: string;
3787
+ idempotencyKey: string;
3788
+ sheetContract: PlaySheetContract;
3789
+ },
3790
+ ): Promise<RowsWriteResponse> {
3791
+ if (!context.playName) {
3792
+ throw new Error('Runtime DB sessions require a playName.');
3793
+ }
3794
+ const session = await getRuntimeDbSession(
3795
+ {
3796
+ ...context,
3797
+ playName: context.playName,
3798
+ runId: context.runId ?? input.runId,
3799
+ },
3800
+ {
3801
+ tableNamespace: input.tableNamespace,
3802
+ logicalTable: 'sheet_rows',
3803
+ operations: ['rows.replace'],
3804
+ limits: {
3805
+ maxRows: runtimeDbSessionRowLimit(input.rows.length),
3806
+ },
3807
+ sheetContract: input.sheetContract,
3808
+ },
3809
+ );
3810
+ return rowsWriteResponseSchema.parse(
3811
+ await writeRuntimeRows(requireRuntimePostgresSession(session), {
3812
+ tableNamespace: input.tableNamespace,
3813
+ rows: input.rows,
3814
+ runId: input.runId,
3815
+ sheetContract: input.sheetContract,
3816
+ idempotencyKey: input.idempotencyKey,
3817
+ mode: 'replace',
3818
+ }),
3819
+ );
3820
+ }
3821
+
3822
+ export function createRuntimeBackedPlayDataset(input: {
3823
+ context: RuntimeApiContext & {
3824
+ playName: string;
3825
+ };
3826
+ tableNamespace: string;
3827
+ datasetKind: 'csv' | 'map';
3828
+ sourceLabel?: string | null;
3829
+ initialCount?: number;
3830
+ initialPreviewRows?: Record<string, unknown>[];
3831
+ runId?: string | null;
3832
+ }): PlayDataset<Record<string, unknown>> {
3833
+ const csvRows =
3834
+ input.datasetKind === 'csv' &&
3835
+ (input.initialPreviewRows?.length ?? 0) >= (input.initialCount ?? 0)
3836
+ ? [...(input.initialPreviewRows ?? [])]
3837
+ : null;
3838
+ if (csvRows) {
3839
+ return createDeferredPlayDataset({
3840
+ datasetKind: input.datasetKind,
3841
+ datasetId: createRuntimeDatasetId(
3842
+ input.context.playName,
3843
+ input.tableNamespace,
3844
+ ),
3845
+ count: input.initialCount ?? csvRows.length,
3846
+ previewRows: csvRows.slice(0, 10),
3847
+ sourceLabel: input.sourceLabel ?? null,
3848
+ resolvers: {
3849
+ count: async () => csvRows.length,
3850
+ peek: async (limit) => csvRows.slice(0, Math.max(0, limit)),
3851
+ materialize: async (limit) =>
3852
+ limit === undefined
3853
+ ? [...csvRows]
3854
+ : csvRows.slice(0, Math.max(0, limit)),
3855
+ iterate: () =>
3856
+ ({
3857
+ async *[Symbol.asyncIterator]() {
3858
+ for (const row of csvRows) {
3859
+ yield row;
3860
+ }
3861
+ },
3862
+ }) as AsyncIterable<Record<string, unknown>>,
3863
+ },
3864
+ });
3865
+ }
3866
+ const runtimeContext = {
3867
+ ...input.context,
3868
+ runId: input.context.runId ?? input.runId,
3869
+ };
3870
+ return createDeferredPlayDataset({
3871
+ datasetKind: input.datasetKind,
3872
+ datasetId: createRuntimeDatasetId(
3873
+ input.context.playName,
3874
+ input.tableNamespace,
3875
+ ),
3876
+ count: input.initialCount ?? 0,
3877
+ backing: {
3878
+ storage: 'neon_sheet',
3879
+ sheet: {
3880
+ playName: input.context.playName,
3881
+ tableNamespace: input.tableNamespace,
3882
+ },
3883
+ },
3884
+ previewRows: input.initialPreviewRows ?? [],
3885
+ sourceLabel: input.sourceLabel ?? null,
3886
+ tableNamespace: input.tableNamespace,
3887
+ resolvers: {
3888
+ count: async () => {
3889
+ if (typeof input.initialCount === 'number') {
3890
+ return input.initialCount;
3891
+ }
3892
+ const summarySession = await getRuntimeDbSession(runtimeContext, {
3893
+ tableNamespace: input.tableNamespace,
3894
+ logicalTable: 'sheet_rows',
3895
+ operations: ['rows.read'],
3896
+ });
3897
+ const summary = await readRuntimeSummary(
3898
+ requireRuntimePostgresSession(summarySession),
3899
+ );
3900
+ return Number(summary.stats.total ?? 0);
3901
+ },
3902
+ peek: async (limit) => {
3903
+ const session = await getRuntimeDbSession(runtimeContext, {
3904
+ tableNamespace: input.tableNamespace,
3905
+ logicalTable: 'sheet_rows',
3906
+ operations: ['rows.read'],
3907
+ });
3908
+ const rows = await readRuntimeRows(
3909
+ requireRuntimePostgresSession(session),
3910
+ {
3911
+ limit,
3912
+ offset: 0,
3913
+ runId: input.runId ?? null,
3914
+ },
3915
+ );
3916
+ return rows.map((row) => row.data);
3917
+ },
3918
+ materialize: async (limit) => {
3919
+ const pageSize = 1000;
3920
+ const materialized: Record<string, unknown>[] = [];
3921
+ let offset = 0;
3922
+ while (true) {
3923
+ const session = await getRuntimeDbSession(runtimeContext, {
3924
+ tableNamespace: input.tableNamespace,
3925
+ logicalTable: 'sheet_rows',
3926
+ operations: ['rows.read'],
3927
+ });
3928
+ const rows = await readRuntimeRows(
3929
+ requireRuntimePostgresSession(session),
3930
+ {
3931
+ limit:
3932
+ limit !== undefined
3933
+ ? Math.min(pageSize, Math.max(0, limit - materialized.length))
3934
+ : pageSize,
3935
+ offset,
3936
+ runId: input.runId ?? null,
3937
+ },
3938
+ );
3939
+ if (rows.length === 0) {
3940
+ break;
3941
+ }
3942
+ materialized.push(...rows.map((row) => row.data));
3943
+ if (limit !== undefined && materialized.length >= limit) {
3944
+ return materialized.slice(0, limit);
3945
+ }
3946
+ offset += rows.length;
3947
+ }
3948
+ return materialized;
3949
+ },
3950
+ iterate: () =>
3951
+ ({
3952
+ async *[Symbol.asyncIterator]() {
3953
+ const pageSize = 1000;
3954
+ let offset = 0;
3955
+ while (true) {
3956
+ const session = await getRuntimeDbSession(runtimeContext, {
3957
+ tableNamespace: input.tableNamespace,
3958
+ logicalTable: 'sheet_rows',
3959
+ operations: ['rows.read'],
3960
+ });
3961
+ const page = await readRuntimeRows(
3962
+ requireRuntimePostgresSession(session),
3963
+ {
3964
+ limit: pageSize,
3965
+ offset,
3966
+ runId: input.runId ?? null,
3967
+ },
3968
+ );
3969
+ if (page.length === 0) {
3970
+ return;
3971
+ }
3972
+ for (const row of page) {
3973
+ yield row.data;
3974
+ }
3975
+ offset += page.length;
3976
+ }
3977
+ },
3978
+ }) as AsyncIterable<Record<string, unknown>>,
3979
+ },
3980
+ });
3981
+ }
3982
+
3983
+ export function resolveRuntimeSheetContract(
3984
+ pipeline: PlayStaticPipeline | null | undefined,
3985
+ tableNamespace: string | null | undefined,
3986
+ ): PlaySheetContract | null {
3987
+ const requestedNamespace = tableNamespace?.trim();
3988
+ if (!pipeline || !requestedNamespace) {
3989
+ return null;
3990
+ }
3991
+ const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
3992
+
3993
+ const rootNamespace = pipeline.tableNamespace?.trim();
3994
+ if (
3995
+ rootNamespace &&
3996
+ normalizeTableNamespace(rootNamespace) === normalizedNamespace
3997
+ ) {
3998
+ return pipeline.sheetContract ?? null;
3999
+ }
4000
+
4001
+ for (const substep of [...(pipeline.stages ?? []), ...pipeline.substeps]) {
4002
+ if (substep.type !== 'dataset') {
4003
+ continue;
4004
+ }
4005
+ const substepNamespace = substep.tableNamespace?.trim();
4006
+ if (
4007
+ substepNamespace &&
4008
+ normalizeTableNamespace(substepNamespace) === normalizedNamespace
4009
+ ) {
4010
+ return substep.sheetContract ?? null;
4011
+ }
4012
+ }
4013
+
4014
+ return null;
4015
+ }