deepline 0.1.182 → 0.1.184

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +639 -109
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1752 -1899
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  15. package/dist/bundling-sources/sdk/src/client.ts +41 -0
  16. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  17. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  18. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  19. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +133 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  23. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1612 -287
  26. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
  27. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  29. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  32. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  40. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  43. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  44. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +21 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2038 -262
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  52. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  53. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  58. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  59. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +28 -11
  60. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +38 -6
  61. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  62. package/dist/cli/index.js +76 -17
  63. package/dist/cli/index.mjs +76 -17
  64. package/dist/index.d.mts +2 -0
  65. package/dist/index.d.ts +2 -0
  66. package/dist/index.js +63 -9
  67. package/dist/index.mjs +63 -9
  68. package/dist/plays/bundle-play-file.mjs +20 -6
  69. package/package.json +1 -1
@@ -1,4 +1,4 @@
1
- import { createHash } from 'node:crypto';
1
+ import { createHash, randomUUID } from 'node:crypto';
2
2
  import {
3
3
  createRuntimePool,
4
4
  createRuntimeOneShotQueryClient,
@@ -55,12 +55,28 @@ import {
55
55
  isReusableWorkReceipt,
56
56
  type WorkReceipt,
57
57
  type WorkReceiptClaim,
58
+ workReceiptFailureKindCode,
59
+ workReceiptFailureKindFromCode,
60
+ type WorkReceiptFailureKind,
58
61
  } from './work-receipts';
59
62
  import {
60
63
  sanitizePostgresJsonValue,
61
64
  stringifyPostgresJson,
62
65
  } from './postgres-json';
63
66
  import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
67
+ import {
68
+ workReceiptClaimableStatusCodes,
69
+ workReceiptClaimConflictPredicateSql,
70
+ workReceiptHeartbeatPredicateSql,
71
+ workReceiptReleaseOwnerPredicateSql,
72
+ workReceiptRunningOwnerPredicateSql,
73
+ } from './receipt-sql';
74
+ import {
75
+ activeRuntimeSheetAttemptFenceSql,
76
+ newerTerminalRuntimeSheetRowSql,
77
+ releasableRuntimeSheetAttemptFenceSql,
78
+ sameOwnerTerminalAttemptEpochSql,
79
+ } from './sheet-attempt-sql';
64
80
  import type { MapRowOutcome } from './durability-store';
65
81
  import {
66
82
  normalizeRuntimeMapInputIndex,
@@ -75,6 +91,20 @@ import {
75
91
  } from './map-row-outcome';
76
92
  import { DEEPLINE_CELL_META_FIELD } from './cell-staleness';
77
93
  import type { PlayArtifactKind } from './backend';
94
+ import {
95
+ PLAY_RUNTIME_CONTRACT,
96
+ PLAY_RUNTIME_CONTRACT_HEADER,
97
+ } from './runtime-contract';
98
+ import { PLAY_RUNTIME_API_COMPAT_PATH } from './runtime-api-paths';
99
+ import {
100
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
101
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
102
+ } from './lease-policy';
103
+ import {
104
+ parseRuntimeTestFaultCounts,
105
+ PLAY_RUNTIME_TEST_FAULT_HEADER,
106
+ type RuntimeTestFaultName,
107
+ } from './test-runtime-seams';
78
108
 
79
109
  type RuntimeApiContext = {
80
110
  baseUrl?: string | null;
@@ -87,10 +117,133 @@ type RuntimeApiContext = {
87
117
  dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | null;
88
118
  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
89
119
  vercelProtectionBypassToken?: string | null;
120
+ runtimeTestFaultHeader?: string | null;
90
121
  disablePostgresPoolCache?: boolean | null;
91
122
  postgresSessionUnwrapKey?: string | null;
92
123
  };
93
124
 
125
+ type DirectRuntimeTestFaultState = {
126
+ headerValue: string;
127
+ counts: Partial<Record<RuntimeTestFaultName, number>>;
128
+ };
129
+
130
+ const directRuntimeTestFaults = new Map<string, DirectRuntimeTestFaultState>();
131
+
132
+ function consumeDirectRuntimeTestFault(input: {
133
+ context: RuntimeApiContext;
134
+ runId: string;
135
+ name: RuntimeTestFaultName;
136
+ }): boolean {
137
+ const headerValue = input.context.runtimeTestFaultHeader?.trim();
138
+ if (!headerValue) return false;
139
+ const stateKey = input.runId.trim();
140
+ if (!stateKey) return false;
141
+ let state = directRuntimeTestFaults.get(stateKey);
142
+ if (!state || state.headerValue !== headerValue) {
143
+ const parsed = parseRuntimeTestFaultCounts({
144
+ headerValue,
145
+ syntheticRunHeader: '1',
146
+ });
147
+ if (parsed.ok === false) {
148
+ throw new Error(parsed.error);
149
+ }
150
+ state = {
151
+ headerValue,
152
+ counts:
153
+ (parsed.counts as Partial<Record<RuntimeTestFaultName, number>>) ?? {},
154
+ };
155
+ directRuntimeTestFaults.set(stateKey, state);
156
+ }
157
+ const count = state.counts[input.name] ?? 0;
158
+ if (count <= 0) return false;
159
+ if (count === 1) {
160
+ delete state.counts[input.name];
161
+ } else {
162
+ state.counts[input.name] = count - 1;
163
+ }
164
+ state.headerValue = headerValue;
165
+ directRuntimeTestFaults.set(stateKey, state);
166
+ return true;
167
+ }
168
+
169
+ function runtimeTestFaultError(name: RuntimeTestFaultName): Error {
170
+ const error = new Error(`Injected runtime test fault: ${name}.`);
171
+ if (
172
+ name === 'receipt_complete_write_fail' ||
173
+ name === 'receipt_fail_write_fail'
174
+ ) {
175
+ error.name = 'RuntimeReceiptPersistenceError';
176
+ }
177
+ return error;
178
+ }
179
+
180
+ async function forceFailRuntimeWorkReceiptForRuntimeTestFault(
181
+ context: RuntimeApiContext,
182
+ input: {
183
+ playName: string;
184
+ runId: string;
185
+ key: string;
186
+ error: string;
187
+ failureKind: WorkReceiptFailureKind;
188
+ runAttempt?: number | null;
189
+ },
190
+ ): Promise<WorkReceipt | null> {
191
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
192
+ const session = await getRuntimeWorkReceiptSession(context, {
193
+ playName: input.playName,
194
+ key: input.key,
195
+ });
196
+ return await withRuntimeWorkReceiptClient(
197
+ context,
198
+ session,
199
+ async (client) => {
200
+ const { rows } = await client.query<Record<string, unknown>>(
201
+ `
202
+ WITH failed AS (
203
+ UPDATE ${workReceiptTable(session)}
204
+ SET status = $2::smallint,
205
+ output = NULL,
206
+ error = $3,
207
+ failure_kind = $5::smallint,
208
+ lease_id = NULL,
209
+ lease_owner_run_id = NULL,
210
+ lease_owner_attempt = NULL,
211
+ lease_expires_at = NULL,
212
+ updated_at = now()
213
+ WHERE k = decode($1, 'hex')
214
+ AND run_id = $4
215
+ AND COALESCE(lease_owner_attempt, 0) = $7::integer
216
+ AND status <> $6::smallint
217
+ RETURNING convert_from(k, 'UTF8') AS k,
218
+ status,
219
+ output,
220
+ error,
221
+ failure_kind,
222
+ run_id,
223
+ lease_id,
224
+ lease_owner_run_id,
225
+ lease_owner_attempt,
226
+ lease_expires_at,
227
+ updated_at
228
+ )
229
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at
230
+ FROM failed
231
+ `,
232
+ [
233
+ workReceiptKeyHex(input.key),
234
+ RECEIPT_STATUS_FAILED,
235
+ input.error,
236
+ input.runId,
237
+ workReceiptFailureKindCode(input.failureKind),
238
+ RECEIPT_STATUS_COMPLETED,
239
+ runAttempt,
240
+ ],
241
+ );
242
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
243
+ },
244
+ );
245
+ }
246
+
94
247
  type DbSessionCacheEntry = {
95
248
  session: CreateDbSessionResponse;
96
249
  };
@@ -101,6 +254,11 @@ type RuntimeDatasetRowEntry = {
101
254
  row: Record<string, unknown>;
102
255
  inputIndex: number;
103
256
  };
257
+ type RuntimeSheetPrepareDisposition = 'pending' | 'completed' | 'blocked';
258
+ type RuntimeSheetPreparedRowDisposition = {
259
+ key: string;
260
+ disposition: RuntimeSheetPrepareDisposition;
261
+ };
104
262
  const dbSessionCache = new Map<string, DbSessionCacheEntry>();
105
263
  const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
106
264
  const postgresPools = new Map<string, RuntimePool>();
@@ -119,6 +277,23 @@ const APPEND_KEY_SUFFIX_LENGTH = 12;
119
277
  // 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
120
278
  // the tenant runtime role is long-lived), not run failures.
121
279
  const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
280
+ const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
281
+ 'lease_id',
282
+ 'lease_owner_run_id',
283
+ 'lease_owner_attempt',
284
+ 'lease_expires_at',
285
+ ] as const;
286
+ const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
287
+ const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
288
+ ...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
289
+ RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
290
+ ] as const;
291
+ const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
292
+ '_attempt_id',
293
+ '_attempt_owner_run_id',
294
+ '_attempt_seq',
295
+ '_attempt_expires_at',
296
+ ] as const;
122
297
  const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
123
298
  const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [
124
299
  100, 250, 500, 1_000, 2_000,
@@ -178,8 +353,13 @@ export type PrepareRuntimeSheetResult = {
178
353
  skipped: number;
179
354
  pendingRows: Record<string, unknown>[];
180
355
  completedRows: Record<string, unknown>[];
356
+ blockedRows: Record<string, unknown>[];
181
357
  tableNamespace: string;
182
358
  timings?: RuntimeSheetTiming[];
359
+ attemptId?: string;
360
+ attemptOwnerRunId?: string;
361
+ attemptExpiresAt?: string;
362
+ attemptSeq?: number;
183
363
  };
184
364
 
185
365
  export type RuntimeSheetTiming = {
@@ -197,6 +377,22 @@ export type RuntimeSheetTiming = {
197
377
  error?: string;
198
378
  };
199
379
 
380
+ export type RuntimeSheetAttemptHeartbeatResult = {
381
+ renewed: number;
382
+ renewedKeys: string[];
383
+ attemptExpiresAt: string | null;
384
+ };
385
+
386
+ export type RuntimeSheetAttemptReleaseResult = {
387
+ released: number;
388
+ releasedKeys: string[];
389
+ };
390
+
391
+ export type RuntimeWorkReceiptReleaseResult = {
392
+ released: number;
393
+ releasedKeys: string[];
394
+ };
395
+
200
396
  type RuntimeApiActionRequest =
201
397
  | {
202
398
  action: 'resolve_play';
@@ -261,7 +457,7 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
261
457
  throw new Error('Runner runtime API requires a baseUrl.');
262
458
  }
263
459
  const url = new URL(
264
- `${baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`,
460
+ `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`,
265
461
  );
266
462
  const bypassToken = context.vercelProtectionBypassToken?.trim();
267
463
  if (bypassToken) {
@@ -282,9 +478,16 @@ function resolveRuntimeApiHeaders(
282
478
  return {
283
479
  'content-type': 'application/json',
284
480
  authorization: `Bearer ${token}`,
481
+ [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
285
482
  ...(context.vercelProtectionBypassToken
286
483
  ? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
287
484
  : {}),
485
+ ...(context.runtimeTestFaultHeader?.trim()
486
+ ? {
487
+ [PLAY_RUNTIME_TEST_FAULT_HEADER]:
488
+ context.runtimeTestFaultHeader.trim(),
489
+ }
490
+ : {}),
288
491
  ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
289
492
  };
290
493
  }
@@ -516,20 +719,36 @@ async function isRuntimeSheetSchemaReady(
516
719
  physicalColumns.length > 0
517
720
  ? physicalColumns.map(quoteIdentifier).join(', ')
518
721
  : '1';
519
- const query = async (client: RuntimeQueryClient): Promise<void> => {
722
+ const query = async (client: RuntimeQueryClient): Promise<boolean> => {
520
723
  await client.query(
521
724
  `SELECT ${selectList}
522
725
  FROM ${sheetTable(session)}
523
726
  LIMIT 0`,
524
727
  );
728
+ const attemptColumns = await client.query<Record<string, unknown>>(
729
+ `
730
+ SELECT count(*)::int AS present_count
731
+ FROM information_schema.columns
732
+ WHERE table_schema = $1
733
+ AND table_name = $2
734
+ AND column_name = ANY($3::text[])
735
+ `,
736
+ [
737
+ session.postgres.schema,
738
+ session.postgres.sheetTable,
739
+ [...RUNTIME_SHEET_ATTEMPT_COLUMNS],
740
+ ],
741
+ );
742
+ return (
743
+ Number(attemptColumns.rows[0]?.present_count ?? 0) ===
744
+ RUNTIME_SHEET_ATTEMPT_COLUMNS.length
745
+ );
525
746
  };
526
747
  try {
527
748
  if (isRuntimeOneShotQueryFactoryRegistered()) {
528
- await withRuntimeOneShotPostgres(session, query);
529
- } else {
530
- await withRuntimePostgres(session, query);
749
+ return await withRuntimeOneShotPostgres(session, query);
531
750
  }
532
- return true;
751
+ return await withRuntimePostgres(session, query);
533
752
  } catch (error) {
534
753
  if (isMissingRelationError(error)) {
535
754
  return false;
@@ -1530,6 +1749,10 @@ function workReceiptKeyHex(key: string): string {
1530
1749
  ).join('');
1531
1750
  }
1532
1751
 
1752
+ function newRuntimeWorkReceiptLeaseId(): string {
1753
+ return `receipt-lease:${randomUUID()}`;
1754
+ }
1755
+
1533
1756
  function validateRuntimeWorkReceiptKeyScope(
1534
1757
  session: RuntimePostgresSession,
1535
1758
  input: { key: string },
@@ -1548,22 +1771,86 @@ function validateRuntimeWorkReceiptKeyScope(
1548
1771
  }
1549
1772
 
1550
1773
  function mapRuntimeWorkReceiptRow(raw: Record<string, unknown>): WorkReceipt {
1774
+ const leaseExpiresAt = raw.lease_expires_at;
1551
1775
  return {
1552
1776
  key: String(raw.k ?? ''),
1553
1777
  status: receiptStatusFromCode(raw.status),
1554
1778
  output: raw.output == null ? null : raw.output,
1555
1779
  error: raw.error == null ? null : String(raw.error),
1780
+ failureKind: workReceiptFailureKindFromCode(raw.failure_kind),
1556
1781
  runId: raw.run_id == null ? null : String(raw.run_id),
1782
+ leaseId: raw.lease_id == null ? null : String(raw.lease_id),
1783
+ leaseOwnerRunId:
1784
+ raw.lease_owner_run_id == null ? null : String(raw.lease_owner_run_id),
1785
+ leaseOwnerAttempt:
1786
+ raw.lease_owner_attempt == null ||
1787
+ !Number.isFinite(Number(raw.lease_owner_attempt))
1788
+ ? null
1789
+ : Number(raw.lease_owner_attempt),
1790
+ leaseExpiresAt:
1791
+ leaseExpiresAt instanceof Date
1792
+ ? leaseExpiresAt.toISOString()
1793
+ : leaseExpiresAt == null
1794
+ ? null
1795
+ : String(leaseExpiresAt),
1557
1796
  updatedAt: raw.updated_at == null ? null : String(raw.updated_at),
1558
1797
  };
1559
1798
  }
1560
1799
 
1800
+ function isMissingRuntimeWorkReceiptSelfHealColumnError(
1801
+ error: unknown,
1802
+ ): boolean {
1803
+ if (!error || typeof error !== 'object') {
1804
+ return false;
1805
+ }
1806
+ const code = 'code' in error ? String(error.code) : '';
1807
+ const message = 'message' in error ? String(error.message) : '';
1808
+ if (code !== '42703' && !/column .* does not exist/i.test(message)) {
1809
+ return false;
1810
+ }
1811
+ return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.some((column) =>
1812
+ message.includes(column),
1813
+ );
1814
+ }
1815
+
1561
1816
  function runtimeWorkReceiptEnsureCacheKey(
1562
1817
  session: RuntimePostgresSession,
1563
1818
  ): string {
1564
1819
  return `${postgresUrlCacheKey(session.postgresUrl)}::${session.postgres.schema}::${session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE}`;
1565
1820
  }
1566
1821
 
1822
+ async function missingRuntimeWorkReceiptSelfHealColumns(
1823
+ session: RuntimePostgresSession,
1824
+ client: RuntimeQueryClient,
1825
+ ): Promise<string[]> {
1826
+ const result = await client.query(
1827
+ `
1828
+ SELECT column_name
1829
+ FROM information_schema.columns
1830
+ WHERE table_schema = $1
1831
+ AND table_name = $2
1832
+ AND column_name = ANY($3::text[])
1833
+ `,
1834
+ [
1835
+ session.postgres.schema,
1836
+ session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
1837
+ [...RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS],
1838
+ ],
1839
+ );
1840
+ const present = new Set(
1841
+ result.rows.map((row) => String(row.column_name ?? '')),
1842
+ );
1843
+ return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.filter(
1844
+ (column) => !present.has(column),
1845
+ );
1846
+ }
1847
+
1848
+ // COMPAT / SELF-HEAL — owner: runtime. Removal milestone: execution-ledger
1849
+ // cutover M1 (docs/play-runtime-execution-ledger-plan.md). The customer-Neon
1850
+ // work-receipt table (lease columns + failure_kind) is the dying surface. The
1851
+ // F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
1852
+ // columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
1853
+ // deleted together with the Neon receipt path at M1.
1567
1854
  async function ensureRuntimeWorkReceiptTable(
1568
1855
  session: RuntimePostgresSession,
1569
1856
  client: RuntimeQueryClient,
@@ -1582,11 +1869,39 @@ async function ensureRuntimeWorkReceiptTable(
1582
1869
  status smallint NOT NULL DEFAULT 0,
1583
1870
  output jsonb,
1584
1871
  error text,
1872
+ failure_kind smallint NOT NULL DEFAULT 0,
1585
1873
  run_id text,
1874
+ lease_id text,
1875
+ lease_owner_run_id text,
1876
+ lease_owner_attempt integer,
1877
+ lease_expires_at timestamptz,
1586
1878
  updated_at timestamptz NOT NULL DEFAULT now()
1587
1879
  )
1588
1880
  `,
1589
1881
  )
1882
+ .then(async () => {
1883
+ const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
1884
+ session,
1885
+ client,
1886
+ );
1887
+ if (missingColumns.length === 0) return;
1888
+ await client.query(`
1889
+ ALTER TABLE ${workReceiptTable(session)}
1890
+ ${missingColumns
1891
+ .map((column) => {
1892
+ const type =
1893
+ column === 'lease_expires_at'
1894
+ ? 'timestamptz'
1895
+ : column === 'lease_owner_attempt'
1896
+ ? 'integer'
1897
+ : column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
1898
+ ? 'smallint NOT NULL DEFAULT 0'
1899
+ : 'text';
1900
+ return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
1901
+ })
1902
+ .join(',\n ')}
1903
+ `);
1904
+ })
1590
1905
  .then(() => undefined);
1591
1906
  runtimeWorkReceiptEnsureCache.set(cacheKey, promise);
1592
1907
  try {
@@ -1612,11 +1927,26 @@ async function withRuntimeWorkReceiptClient<T>(
1612
1927
  try {
1613
1928
  return await operation(client);
1614
1929
  } catch (error) {
1615
- if (isMissingRelationError(error)) {
1930
+ if (
1931
+ isMissingRelationError(error) ||
1932
+ isMissingRuntimeWorkReceiptSelfHealColumnError(error)
1933
+ ) {
1616
1934
  runtimeWorkReceiptEnsureCache.delete(
1617
1935
  runtimeWorkReceiptEnsureCacheKey(session),
1618
1936
  );
1619
- await ensureRuntimeWorkReceiptTable(session, client);
1937
+ try {
1938
+ await ensureRuntimeWorkReceiptTable(session, client);
1939
+ } catch (ensureError) {
1940
+ if (!isPostgresPermissionDeniedError(ensureError)) {
1941
+ throw ensureError;
1942
+ }
1943
+ runtimeWorkReceiptEnsureCache.delete(
1944
+ runtimeWorkReceiptEnsureCacheKey(session),
1945
+ );
1946
+ await repairRuntimeStorageGrants(context, {
1947
+ playName: context.playName?.trim() || session.playName,
1948
+ });
1949
+ }
1620
1950
  } else if (isPostgresPermissionDeniedError(error)) {
1621
1951
  await repairRuntimeStorageGrants(context, {
1622
1952
  playName: context.playName?.trim() || session.playName,
@@ -1715,6 +2045,10 @@ function isSystemSheetColumn(columnName: string): boolean {
1715
2045
  case '_stage':
1716
2046
  case '_provider':
1717
2047
  case '_input_index':
2048
+ case '_attempt_id':
2049
+ case '_attempt_owner_run_id':
2050
+ case '_attempt_expires_at':
2051
+ case '_attempt_seq':
1718
2052
  case '_created_at':
1719
2053
  case '_updated_at':
1720
2054
  case '_version':
@@ -1732,6 +2066,64 @@ function isSystemSheetColumn(columnName: string): boolean {
1732
2066
  }
1733
2067
  }
1734
2068
 
2069
+ function normalizeRuntimeSheetAttemptId(
2070
+ value: string | null | undefined,
2071
+ ): string {
2072
+ const trimmed = value?.trim();
2073
+ return trimmed || `sheet-attempt:${randomUUID()}`;
2074
+ }
2075
+
2076
+ function normalizeRuntimeSheetAttemptOwnerRunId(input: {
2077
+ attemptOwnerRunId?: string | null;
2078
+ runId: string;
2079
+ }): string {
2080
+ return input.attemptOwnerRunId?.trim() || input.runId;
2081
+ }
2082
+
2083
+ function normalizeRuntimeRunAttempt(value: number | null | undefined): number {
2084
+ if (typeof value !== 'number' || !Number.isFinite(value)) return 0;
2085
+ return Math.max(0, Math.floor(value));
2086
+ }
2087
+
2088
+ function normalizeRuntimeLeaseTtlMs(
2089
+ value: number | null | undefined,
2090
+ fallback: number,
2091
+ ): number {
2092
+ if (
2093
+ typeof value !== 'number' ||
2094
+ !Number.isFinite(value) ||
2095
+ !Number.isInteger(value) ||
2096
+ value <= 0
2097
+ ) {
2098
+ return fallback;
2099
+ }
2100
+ return value;
2101
+ }
2102
+
2103
+ async function mintRuntimeSheetAttemptExpiresAt(
2104
+ client: RuntimeQueryClient,
2105
+ leaseTtlMs?: number | null,
2106
+ ): Promise<string> {
2107
+ const ttlMs = normalizeRuntimeLeaseTtlMs(
2108
+ leaseTtlMs,
2109
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
2110
+ );
2111
+ const { rows } = await client.query<{ attempt_expires_at: Date | string }>(
2112
+ `
2113
+ SELECT now() + ($1::double precision * interval '1 millisecond') AS attempt_expires_at
2114
+ `,
2115
+ [ttlMs],
2116
+ );
2117
+ const value = rows[0]?.attempt_expires_at;
2118
+ if (value instanceof Date) {
2119
+ return value.toISOString();
2120
+ }
2121
+ if (typeof value === 'string' && value.trim()) {
2122
+ return new Date(value).toISOString();
2123
+ }
2124
+ throw new Error('Runtime sheet attempt expiry mint did not return a value.');
2125
+ }
2126
+
1735
2127
  function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
1736
2128
  if (value && typeof value === 'object' && !Array.isArray(value)) {
1737
2129
  return value as Record<string, unknown>;
@@ -2228,6 +2620,10 @@ async function prepareRuntimeSheetDatasetRows(
2228
2620
  runId: string;
2229
2621
  normalizedPlayName: string;
2230
2622
  normalizedTableNamespace: string;
2623
+ attemptId: string;
2624
+ attemptOwnerRunId: string;
2625
+ attemptExpiresAt: string;
2626
+ attemptSeq: number;
2231
2627
  physicalInsertColumnsSql: string;
2232
2628
  physicalInsertValuesSql: string;
2233
2629
  physicalRefreshSetSql: string;
@@ -2235,9 +2631,12 @@ async function prepareRuntimeSheetDatasetRows(
2235
2631
  outputPhysicalColumns: PhysicalSheetColumnProjection[];
2236
2632
  force?: boolean;
2237
2633
  },
2238
- ): Promise<{ inserted: number; pendingKeys: string[] }> {
2634
+ ): Promise<{
2635
+ inserted: number;
2636
+ rowDispositions: RuntimeSheetPreparedRowDisposition[];
2637
+ }> {
2239
2638
  let inserted = 0;
2240
- const pendingKeys: string[] = [];
2639
+ const rowDispositions: RuntimeSheetPreparedRowDisposition[] = [];
2241
2640
  for (const chunk of input.chunks) {
2242
2641
  const chunkKeys = chunk.map((entry) => entry.key);
2243
2642
  const chunkPayloads = chunk.map((entry) =>
@@ -2252,6 +2651,41 @@ async function prepareRuntimeSheetDatasetRows(
2252
2651
  'target',
2253
2652
  input.outputPhysicalColumns,
2254
2653
  );
2654
+ const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
2655
+ 'target',
2656
+ '$7::text',
2657
+ '$8::text',
2658
+ '$9::timestamptz',
2659
+ '$10::integer',
2660
+ );
2661
+ const existingAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
2662
+ 'existing',
2663
+ '$7::text',
2664
+ '$8::text',
2665
+ '$9::timestamptz',
2666
+ '$10::integer',
2667
+ );
2668
+ const targetNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
2669
+ 'target',
2670
+ '$9::timestamptz',
2671
+ '$10::integer',
2672
+ '$7::text',
2673
+ '$8::text',
2674
+ );
2675
+ const existingNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
2676
+ 'existing',
2677
+ '$9::timestamptz',
2678
+ '$10::integer',
2679
+ '$7::text',
2680
+ '$8::text',
2681
+ );
2682
+ const existingWritableEnrichedAttemptSql =
2683
+ writableEnrichedRuntimeSheetAttemptSql(
2684
+ 'existing',
2685
+ '$8::text',
2686
+ '$9::timestamptz',
2687
+ '$10',
2688
+ );
2255
2689
  const { rows } = await client.query(
2256
2690
  `
2257
2691
  WITH input_rows AS (
@@ -2273,19 +2707,38 @@ async function prepareRuntimeSheetDatasetRows(
2273
2707
  WHERE target._key = input_rows._key
2274
2708
  AND target._status <> 'stale'
2275
2709
  AND (
2276
- $7::boolean
2710
+ NOT (${targetNewerTerminalRowSql})
2711
+ OR (
2712
+ $11::boolean
2713
+ AND target._status = 'enriched'
2714
+ AND COALESCE(target._attempt_seq, 0) <= $10::integer
2715
+ )
2716
+ )
2717
+ AND (
2718
+ ${targetAttemptFenceSql}
2719
+ OR (
2720
+ $11::boolean
2721
+ AND target._status = 'enriched'
2722
+ )
2723
+ )
2724
+ AND (
2725
+ $11::boolean
2277
2726
  OR target._input_index IS DISTINCT FROM input_rows._input_index
2278
2727
  )
2279
2728
  RETURNING target._key
2280
2729
  ),
2281
2730
  inserted_rows AS (
2282
- INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${input.physicalInsertColumnsSql})
2283
- SELECT _key, 'pending', $4::text, _input_index${input.physicalInsertValuesSql}
2731
+ INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
2732
+ SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
2284
2733
  FROM input_rows
2285
2734
  ON CONFLICT (_key) DO UPDATE SET
2286
2735
  _status = 'pending',
2287
2736
  _run_id = EXCLUDED._run_id,
2288
2737
  _input_index = EXCLUDED._input_index,
2738
+ _attempt_id = EXCLUDED._attempt_id,
2739
+ _attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
2740
+ _attempt_expires_at = EXCLUDED._attempt_expires_at,
2741
+ _attempt_seq = EXCLUDED._attempt_seq,
2289
2742
  _updated_at = now(),
2290
2743
  _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
2291
2744
  WHERE ${sheetTable(session)}._status = 'stale'
@@ -2296,26 +2749,42 @@ async function prepareRuntimeSheetDatasetRows(
2296
2749
  SET _status = 'pending',
2297
2750
  _run_id = $4::text,
2298
2751
  _input_index = input_rows._input_index,
2752
+ _attempt_id = $7::text,
2753
+ _attempt_owner_run_id = $8::text,
2754
+ _attempt_expires_at = $9::timestamptz,
2755
+ _attempt_seq = $10::integer,
2299
2756
  _updated_at = now(),
2300
2757
  _version = ${nextRuntimeSheetVersionExpression(session)}
2301
2758
  FROM input_rows
2302
2759
  WHERE target._key = input_rows._key
2303
2760
  AND target._status = 'enriched'
2304
2761
  AND (${targetMissingOutputSql})
2762
+ AND COALESCE(target._attempt_seq, 0) <= $10::integer
2305
2763
  RETURNING target._key
2306
2764
  ),
2307
2765
  claimed_existing_rows AS (
2308
2766
  UPDATE ${sheetTable(session)} AS target
2309
2767
  SET _run_id = $4::text,
2310
2768
  _input_index = input_rows._input_index,
2769
+ _attempt_id = $7::text,
2770
+ _attempt_owner_run_id = $8::text,
2771
+ _attempt_expires_at = $9::timestamptz,
2772
+ _attempt_seq = $10::integer,
2311
2773
  _updated_at = now(),
2312
2774
  _version = ${nextRuntimeSheetVersionExpression(session)}
2313
2775
  FROM input_rows
2314
2776
  WHERE target._key = input_rows._key
2315
2777
  AND target._status IN ('pending', 'running', 'failed')
2778
+ AND NOT (${targetNewerTerminalRowSql})
2779
+ AND (
2780
+ target._status = 'failed'
2781
+ OR ${targetAttemptFenceSql}
2782
+ )
2316
2783
  AND (
2317
2784
  target._run_id IS DISTINCT FROM $4::text
2318
2785
  OR target._input_index IS DISTINCT FROM input_rows._input_index
2786
+ OR target._attempt_id IS DISTINCT FROM $7::text
2787
+ OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $10::integer
2319
2788
  )
2320
2789
  RETURNING target._key
2321
2790
  ),
@@ -2329,15 +2798,61 @@ async function prepareRuntimeSheetDatasetRows(
2329
2798
  SELECT _key
2330
2799
  FROM claimed_existing_rows
2331
2800
  UNION
2801
+ SELECT _key
2802
+ FROM existing_rows
2803
+ WHERE $11::boolean
2804
+ UNION
2332
2805
  SELECT existing._key
2333
2806
  FROM ${sheetTable(session)} AS existing
2334
2807
  JOIN input_rows ON input_rows._key = existing._key
2335
- WHERE existing._status IN ('pending', 'running', 'failed')
2808
+ WHERE (
2809
+ existing._status IN ('pending', 'running', 'failed')
2810
+ AND NOT (${existingNewerTerminalRowSql})
2811
+ AND (
2812
+ existing._status = 'failed'
2813
+ OR ${existingAttemptFenceSql}
2814
+ )
2815
+ )
2336
2816
  OR (
2337
2817
  existing._status = 'enriched'
2338
2818
  AND (${existingMissingOutputSql})
2819
+ AND (
2820
+ ${existingAttemptFenceSql}
2821
+ OR ${existingWritableEnrichedAttemptSql}
2822
+ )
2339
2823
  )
2340
2824
  ),
2825
+ completed_rows AS (
2826
+ SELECT existing._key
2827
+ FROM ${sheetTable(session)} AS existing
2828
+ JOIN input_rows ON input_rows._key = existing._key
2829
+ WHERE existing._status = 'enriched'
2830
+ AND NOT ($11::boolean)
2831
+ AND NOT (${existingMissingOutputSql})
2832
+ AND NOT EXISTS (
2833
+ SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
2834
+ )
2835
+ ),
2836
+ blocked_rows AS (
2837
+ SELECT input_rows._key
2838
+ FROM input_rows
2839
+ WHERE NOT EXISTS (
2840
+ SELECT 1 FROM pending_rows WHERE pending_rows._key = input_rows._key
2841
+ )
2842
+ AND NOT EXISTS (
2843
+ SELECT 1 FROM completed_rows WHERE completed_rows._key = input_rows._key
2844
+ )
2845
+ ),
2846
+ row_dispositions AS (
2847
+ SELECT _key, 'pending'::text AS disposition
2848
+ FROM pending_rows
2849
+ UNION ALL
2850
+ SELECT _key, 'completed'::text AS disposition
2851
+ FROM completed_rows
2852
+ UNION ALL
2853
+ SELECT _key, 'blocked'::text AS disposition
2854
+ FROM blocked_rows
2855
+ ),
2341
2856
  inserted_count_cte AS (
2342
2857
  SELECT count(*)::bigint AS c FROM inserted_rows
2343
2858
  ),
@@ -2377,7 +2892,13 @@ async function prepareRuntimeSheetDatasetRows(
2377
2892
  )
2378
2893
  SELECT
2379
2894
  (SELECT c::int FROM inserted_count_cte) AS inserted_count,
2380
- coalesce((SELECT array_agg(_key) FROM pending_rows), '{}'::text[]) AS pending_keys,
2895
+ coalesce(
2896
+ (
2897
+ SELECT jsonb_agg(jsonb_build_object('key', _key, 'disposition', disposition))
2898
+ FROM row_dispositions
2899
+ ),
2900
+ '[]'::jsonb
2901
+ ) AS row_dispositions,
2381
2902
  (SELECT count(*)::int FROM existing_rows) AS reordered_count,
2382
2903
  (SELECT c::int FROM missing_output_count_cte) AS missing_output_count
2383
2904
  `,
@@ -2388,15 +2909,30 @@ async function prepareRuntimeSheetDatasetRows(
2388
2909
  input.runId,
2389
2910
  input.normalizedPlayName,
2390
2911
  input.normalizedTableNamespace,
2912
+ input.attemptId,
2913
+ input.attemptOwnerRunId,
2914
+ input.attemptExpiresAt,
2915
+ input.attemptSeq,
2391
2916
  input.force === true,
2392
2917
  ],
2393
2918
  );
2394
2919
  inserted += Number(rows[0]?.inserted_count ?? 0);
2395
- if (Array.isArray(rows[0]?.pending_keys)) {
2396
- pendingKeys.push(...(rows[0]?.pending_keys as string[]));
2920
+ if (Array.isArray(rows[0]?.row_dispositions)) {
2921
+ for (const row of rows[0]
2922
+ ?.row_dispositions as RuntimeSheetPreparedRowDisposition[]) {
2923
+ if (
2924
+ row &&
2925
+ typeof row.key === 'string' &&
2926
+ (row.disposition === 'pending' ||
2927
+ row.disposition === 'completed' ||
2928
+ row.disposition === 'blocked')
2929
+ ) {
2930
+ rowDispositions.push(row);
2931
+ }
2932
+ }
2397
2933
  }
2398
2934
  }
2399
- return { inserted, pendingKeys };
2935
+ return { inserted, rowDispositions };
2400
2936
  }
2401
2937
 
2402
2938
  async function buildRuntimeSheetDatasetStartResult(
@@ -2410,8 +2946,12 @@ async function buildRuntimeSheetDatasetStartResult(
2410
2946
  normalizedPlayName: string;
2411
2947
  normalizedTableNamespace: string;
2412
2948
  runId: string;
2949
+ attemptId: string;
2950
+ attemptOwnerRunId: string;
2951
+ attemptExpiresAt: string;
2952
+ attemptSeq: number;
2413
2953
  inserted: number;
2414
- pendingKeys: string[];
2954
+ rowDispositions: RuntimeSheetPreparedRowDisposition[];
2415
2955
  force?: boolean;
2416
2956
  timings?: RuntimeSheetTiming[];
2417
2957
  },
@@ -2421,19 +2961,20 @@ async function buildRuntimeSheetDatasetStartResult(
2421
2961
  ? [column.field]
2422
2962
  : [],
2423
2963
  );
2424
- const buildFreshPendingRows = () => {
2425
- return input.rowEntries.map((entry) => {
2426
- const row: Record<string, unknown> = {
2427
- ...sanitizePostgresJsonValue(entry.row),
2428
- __deeplineRowKey: entry.key,
2429
- };
2430
- for (const field of datasetFields) {
2431
- if (!Object.prototype.hasOwnProperty.call(row, field)) {
2432
- row[field] = null;
2433
- }
2964
+ const buildFreshRow = (entry: RuntimeDatasetRowEntry) => {
2965
+ const row: Record<string, unknown> = {
2966
+ ...sanitizePostgresJsonValue(entry.row),
2967
+ __deeplineRowKey: entry.key,
2968
+ };
2969
+ for (const field of datasetFields) {
2970
+ if (!Object.prototype.hasOwnProperty.call(row, field)) {
2971
+ row[field] = null;
2434
2972
  }
2435
- return row;
2436
- });
2973
+ }
2974
+ return row;
2975
+ };
2976
+ const buildFreshPendingRows = (entries = input.rowEntries) => {
2977
+ return entries.map(buildFreshRow);
2437
2978
  };
2438
2979
 
2439
2980
  if (input.inserted === input.rowEntries.length) {
@@ -2442,18 +2983,31 @@ async function buildRuntimeSheetDatasetStartResult(
2442
2983
  skipped: input.sourceRowsLength - input.rowEntries.length,
2443
2984
  pendingRows: buildFreshPendingRows(),
2444
2985
  completedRows: [],
2986
+ blockedRows: [],
2445
2987
  tableNamespace: input.tableNamespace,
2446
2988
  };
2447
2989
  }
2448
2990
 
2449
- const pendingKeys = new Set(input.pendingKeys);
2991
+ const dispositionsByKey = new Map(
2992
+ input.rowDispositions.map((row) => [row.key, row.disposition]),
2993
+ );
2994
+ const pendingKeys = new Set(
2995
+ input.rowDispositions.flatMap((row) =>
2996
+ row.disposition === 'pending' ? [row.key] : [],
2997
+ ),
2998
+ );
2450
2999
  const startedAt = Date.now();
2451
3000
  await markRuntimeRowsPendingForRecompute(client, session, {
2452
3001
  keys: [...pendingKeys],
2453
3002
  runId: input.runId,
3003
+ attemptId: input.attemptId,
3004
+ attemptOwnerRunId: input.attemptOwnerRunId,
3005
+ attemptExpiresAt: input.attemptExpiresAt,
3006
+ attemptSeq: input.attemptSeq,
2454
3007
  normalizedPlayName: input.normalizedPlayName,
2455
3008
  normalizedTableNamespace: input.normalizedTableNamespace,
2456
3009
  outputFields: input.force === true ? datasetFields : [],
3010
+ force: input.force === true,
2457
3011
  });
2458
3012
  if (pendingKeys.size > 0) {
2459
3013
  input.timings?.push({
@@ -2463,11 +3017,22 @@ async function buildRuntimeSheetDatasetStartResult(
2463
3017
  });
2464
3018
  }
2465
3019
  if (input.force === true) {
3020
+ const pendingRows: Record<string, unknown>[] = [];
3021
+ const blockedRows: Record<string, unknown>[] = [];
3022
+ for (const entry of input.rowEntries) {
3023
+ const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
3024
+ if (disposition === 'pending') {
3025
+ pendingRows.push(buildFreshRow(entry));
3026
+ } else if (disposition === 'blocked') {
3027
+ blockedRows.push(buildFreshRow(entry));
3028
+ }
3029
+ }
2466
3030
  return {
2467
3031
  inserted: input.inserted,
2468
3032
  skipped: input.sourceRowsLength - input.rowEntries.length,
2469
- pendingRows: buildFreshPendingRows(),
3033
+ pendingRows,
2470
3034
  completedRows: [],
3035
+ blockedRows,
2471
3036
  tableNamespace: input.tableNamespace,
2472
3037
  };
2473
3038
  }
@@ -2498,14 +3063,25 @@ async function buildRuntimeSheetDatasetStartResult(
2498
3063
  };
2499
3064
  };
2500
3065
  const pendingRows: Record<string, unknown>[] = [];
3066
+ const completedRows: Record<string, unknown>[] = [];
3067
+ const blockedRows: Record<string, unknown>[] = [];
2501
3068
  for (const entry of input.rowEntries) {
2502
- pendingRows.push(buildMergedRow(entry));
3069
+ const merged = buildMergedRow(entry);
3070
+ const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
3071
+ if (disposition === 'pending') {
3072
+ pendingRows.push(merged);
3073
+ } else if (disposition === 'completed') {
3074
+ completedRows.push(merged);
3075
+ } else {
3076
+ blockedRows.push(merged);
3077
+ }
2503
3078
  }
2504
3079
  return {
2505
3080
  inserted: input.inserted,
2506
3081
  skipped: input.sourceRowsLength - input.rowEntries.length,
2507
3082
  pendingRows,
2508
- completedRows: [],
3083
+ completedRows,
3084
+ blockedRows,
2509
3085
  tableNamespace: input.tableNamespace,
2510
3086
  };
2511
3087
  }
@@ -2516,12 +3092,31 @@ async function markRuntimeRowsPendingForRecompute(
2516
3092
  input: {
2517
3093
  keys: string[];
2518
3094
  runId: string;
3095
+ attemptId: string;
3096
+ attemptOwnerRunId: string;
3097
+ attemptExpiresAt: string;
3098
+ attemptSeq: number;
2519
3099
  normalizedPlayName: string;
2520
3100
  normalizedTableNamespace: string;
2521
3101
  outputFields?: string[];
3102
+ force?: boolean;
2522
3103
  },
2523
3104
  ): Promise<void> {
2524
3105
  if (input.keys.length === 0) return;
3106
+ const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
3107
+ 'target',
3108
+ '$5::text',
3109
+ '$6::text',
3110
+ '$7::timestamptz',
3111
+ '$8::integer',
3112
+ );
3113
+ const targetWritableEnrichedAttemptSql =
3114
+ writableEnrichedRuntimeSheetAttemptSql(
3115
+ 'target',
3116
+ '$6::text',
3117
+ '$7::timestamptz',
3118
+ '$8',
3119
+ );
2525
3120
  const outputFields = [...new Set(input.outputFields ?? [])];
2526
3121
  await client.query(
2527
3122
  `WITH target_rows AS (
@@ -2534,13 +3129,31 @@ async function markRuntimeRowsPendingForRecompute(
2534
3129
  UPDATE ${sheetTable(session)} AS target
2535
3130
  SET _status = 'pending',
2536
3131
  _run_id = $2::text,
3132
+ _attempt_id = $5::text,
3133
+ _attempt_owner_run_id = $6::text,
3134
+ _attempt_expires_at = $7::timestamptz,
3135
+ _attempt_seq = $8::integer,
2537
3136
  _updated_at = now(),
2538
3137
  _version = ${nextRuntimeSheetVersionExpression(session)}
2539
3138
  FROM target_rows
2540
3139
  WHERE target._key = target_rows._key
3140
+ AND (
3141
+ ${targetAttemptFenceSql}
3142
+ OR (
3143
+ $10::boolean
3144
+ AND target._status = 'enriched'
3145
+ AND COALESCE(target._attempt_seq, 0) <= $8::integer
3146
+ )
3147
+ OR (
3148
+ target._status = 'enriched'
3149
+ AND ${targetWritableEnrichedAttemptSql}
3150
+ )
3151
+ )
2541
3152
  AND (
2542
3153
  target._status <> 'pending'
2543
3154
  OR target._run_id IS DISTINCT FROM $2::text
3155
+ OR target._attempt_id IS DISTINCT FROM $5::text
3156
+ OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $8::integer
2544
3157
  )
2545
3158
  RETURNING target_rows._status AS previous_status,
2546
3159
  target_rows._cell_meta AS previous_cell_meta
@@ -2561,7 +3174,7 @@ async function markRuntimeRowsPendingForRecompute(
2561
3174
  WHERE previous_cell_meta -> field_values.field ->> 'status' = 'failed'
2562
3175
  )::int AS failed_to_pending
2563
3176
  FROM updated
2564
- JOIN unnest($5::text[]) AS field_values(field) ON true
3177
+ JOIN unnest($9::text[]) AS field_values(field) ON true
2565
3178
  GROUP BY field_values.field
2566
3179
  ),
2567
3180
  summary_delta AS (
@@ -2619,7 +3232,12 @@ async function markRuntimeRowsPendingForRecompute(
2619
3232
  input.runId,
2620
3233
  input.normalizedPlayName,
2621
3234
  input.normalizedTableNamespace,
3235
+ input.attemptId,
3236
+ input.attemptOwnerRunId,
3237
+ input.attemptExpiresAt,
3238
+ input.attemptSeq,
2622
3239
  outputFields,
3240
+ input.force === true,
2623
3241
  ],
2624
3242
  );
2625
3243
  }
@@ -2657,6 +3275,41 @@ async function getRuntimeWorkReceiptSession(
2657
3275
  return session;
2658
3276
  }
2659
3277
 
3278
+ /**
3279
+ * Work receipts for a play share a single table, so run-scoped operations (the
3280
+ * run-fatal lease release) need a session without a specific key to scope by.
3281
+ * This mirrors `getRuntimeWorkReceiptSession` minus the per-key scope check.
3282
+ */
3283
+ async function getRuntimeWorkReceiptSessionForRun(
3284
+ context: RuntimeApiContext,
3285
+ input: {
3286
+ playName: string;
3287
+ },
3288
+ ): Promise<RuntimePostgresSession> {
3289
+ const playName = context.playName?.trim() || input.playName;
3290
+ if (!playName) {
3291
+ throw new Error('Runtime work receipts require a playName.');
3292
+ }
3293
+ const runtimeContext = {
3294
+ ...context,
3295
+ playName,
3296
+ };
3297
+ const preloaded = findPreloadedRuntimeDbSession(runtimeContext, {
3298
+ tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
3299
+ logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
3300
+ operations: ['rows.read', 'rows.upsert'],
3301
+ });
3302
+ return requireRuntimePostgresSession(
3303
+ preloaded
3304
+ ? await unwrapRuntimeDbSession(runtimeContext, preloaded)
3305
+ : await getRuntimeDbSession(runtimeContext, {
3306
+ tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
3307
+ logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
3308
+ operations: ['rows.read', 'rows.upsert'],
3309
+ }),
3310
+ );
3311
+ }
3312
+
2660
3313
  async function getRuntimeWorkReceiptSessionForKeys(
2661
3314
  context: RuntimeApiContext,
2662
3315
  input: {
@@ -2684,7 +3337,17 @@ async function readRuntimeWorkReceipt(
2684
3337
  key: string,
2685
3338
  ): Promise<WorkReceipt | null> {
2686
3339
  const { rows } = await client.query<Record<string, unknown>>(
2687
- `SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3340
+ `SELECT convert_from(k, 'UTF8') AS k,
3341
+ status,
3342
+ output,
3343
+ error,
3344
+ failure_kind,
3345
+ run_id,
3346
+ lease_id,
3347
+ lease_owner_run_id,
3348
+ lease_owner_attempt,
3349
+ lease_expires_at,
3350
+ updated_at
2688
3351
  FROM ${workReceiptTable(session)}
2689
3352
  WHERE k = decode($1, 'hex')`,
2690
3353
  [workReceiptKeyHex(key)],
@@ -2714,7 +3377,7 @@ export async function getRuntimeWorkReceipts(
2714
3377
  playName: string;
2715
3378
  keys: string[];
2716
3379
  },
2717
- ): Promise<WorkReceipt[]> {
3380
+ ): Promise<Array<WorkReceipt | null>> {
2718
3381
  const keys = [
2719
3382
  ...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
2720
3383
  ];
@@ -2737,16 +3400,23 @@ export async function getRuntimeWorkReceipts(
2737
3400
  receipts.status,
2738
3401
  receipts.output,
2739
3402
  receipts.error,
3403
+ receipts.failure_kind,
2740
3404
  receipts.run_id,
3405
+ receipts.lease_id,
3406
+ receipts.lease_owner_run_id,
3407
+ receipts.lease_owner_attempt,
3408
+ receipts.lease_expires_at,
2741
3409
  receipts.updated_at
2742
3410
  FROM input_keys
2743
- JOIN ${workReceiptTable(session)} AS receipts
3411
+ LEFT JOIN ${workReceiptTable(session)} AS receipts
2744
3412
  ON receipts.k = decode(input_keys.key_hex, 'hex')
2745
3413
  ORDER BY input_keys.ord
2746
3414
  `,
2747
3415
  [keys.map(workReceiptKeyHex)],
2748
3416
  );
2749
- return rows.map(mapRuntimeWorkReceiptRow);
3417
+ return rows.map((row) =>
3418
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
3419
+ );
2750
3420
  },
2751
3421
  );
2752
3422
  }
@@ -2757,8 +3427,12 @@ export async function claimRuntimeWorkReceipt(
2757
3427
  playName: string;
2758
3428
  runId: string;
2759
3429
  key: string;
3430
+ runAttempt?: number | null;
3431
+ leaseAware?: boolean;
2760
3432
  reclaimRunning?: boolean;
2761
3433
  forceRefresh?: boolean;
3434
+ forceFailedRefresh?: boolean;
3435
+ leaseTtlMs?: number | null;
2762
3436
  },
2763
3437
  ): Promise<WorkReceiptClaim> {
2764
3438
  const session = await getRuntimeWorkReceiptSession(context, {
@@ -2769,36 +3443,89 @@ export async function claimRuntimeWorkReceipt(
2769
3443
  context,
2770
3444
  session,
2771
3445
  async (client) => {
2772
- const claimableStatuses = input.forceRefresh
2773
- ? [
2774
- RECEIPT_STATUS_PENDING,
2775
- RECEIPT_STATUS_FAILED,
2776
- RECEIPT_STATUS_RUNNING,
2777
- RECEIPT_STATUS_COMPLETED,
2778
- RECEIPT_STATUS_SKIPPED,
2779
- ]
2780
- : input.reclaimRunning
2781
- ? [
2782
- RECEIPT_STATUS_PENDING,
2783
- RECEIPT_STATUS_FAILED,
2784
- RECEIPT_STATUS_RUNNING,
2785
- ]
2786
- : [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
3446
+ const leaseId =
3447
+ input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null;
3448
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3449
+ const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
3450
+ input.leaseTtlMs,
3451
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
3452
+ );
3453
+ const claimableStatuses = workReceiptClaimableStatusCodes({
3454
+ forceRefresh: input.forceRefresh,
3455
+ forceFailedRefresh: input.forceFailedRefresh,
3456
+ reclaimRunning: input.reclaimRunning,
3457
+ });
2787
3458
  const { rows } = await client.query<Record<string, unknown>>(
2788
3459
  `
2789
3460
  WITH claimed AS (
2790
- INSERT INTO ${workReceiptTable(session)} (k, status, run_id, updated_at)
2791
- VALUES (decode($1, 'hex'), $2::smallint, $3, now())
3461
+ INSERT INTO ${workReceiptTable(session)} (
3462
+ k,
3463
+ status,
3464
+ run_id,
3465
+ lease_id,
3466
+ lease_owner_run_id,
3467
+ lease_owner_attempt,
3468
+ lease_expires_at,
3469
+ updated_at
3470
+ )
3471
+ VALUES (
3472
+ decode($1, 'hex'),
3473
+ $2::smallint,
3474
+ $3,
3475
+ $6,
3476
+ CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
3477
+ CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
3478
+ CASE
3479
+ WHEN $6::text IS NULL THEN NULL
3480
+ ELSE now() + ($7::double precision * interval '1 millisecond')
3481
+ END,
3482
+ now()
3483
+ )
2792
3484
  ON CONFLICT (k) DO UPDATE
2793
3485
  SET status = $4::smallint,
2794
3486
  output = NULL,
3487
+ failure_kind = 0,
2795
3488
  run_id = $3,
3489
+ lease_id = $6,
3490
+ lease_owner_run_id = CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
3491
+ lease_owner_attempt = CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
3492
+ lease_expires_at = CASE
3493
+ WHEN $6::text IS NULL THEN NULL
3494
+ ELSE now() + ($7::double precision * interval '1 millisecond')
3495
+ END,
2796
3496
  error = NULL,
2797
3497
  updated_at = now()
2798
- WHERE ${workReceiptTable(session)}.status = ANY($5::smallint[])
2799
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3498
+ WHERE ${workReceiptClaimConflictPredicateSql({
3499
+ receiptTable: workReceiptTable(session),
3500
+ claimableStatusesSql: '$5',
3501
+ claimantRunIdSql: '$3',
3502
+ claimantRunAttemptSql: '$9',
3503
+ forceRefreshSql: '$8',
3504
+ forceFailedRefreshSql: '$10',
3505
+ })}
3506
+ RETURNING convert_from(k, 'UTF8') AS k,
3507
+ status,
3508
+ output,
3509
+ error,
3510
+ failure_kind,
3511
+ run_id,
3512
+ lease_id,
3513
+ lease_owner_run_id,
3514
+ lease_owner_attempt,
3515
+ lease_expires_at,
3516
+ updated_at
2800
3517
  )
2801
- SELECT k, status, output, error, run_id, updated_at
3518
+ SELECT k,
3519
+ status,
3520
+ output,
3521
+ error,
3522
+ failure_kind,
3523
+ run_id,
3524
+ lease_id,
3525
+ lease_owner_run_id,
3526
+ lease_owner_attempt,
3527
+ lease_expires_at,
3528
+ updated_at
2802
3529
  FROM claimed
2803
3530
  `,
2804
3531
  [
@@ -2807,6 +3534,11 @@ export async function claimRuntimeWorkReceipt(
2807
3534
  input.runId,
2808
3535
  RECEIPT_STATUS_RUNNING,
2809
3536
  claimableStatuses,
3537
+ leaseId,
3538
+ leaseTtlMs,
3539
+ input.forceRefresh === true,
3540
+ runAttempt,
3541
+ input.forceFailedRefresh === true,
2810
3542
  ],
2811
3543
  );
2812
3544
  const claimed = rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -2815,14 +3547,19 @@ export async function claimRuntimeWorkReceipt(
2815
3547
  }
2816
3548
 
2817
3549
  const latest = await readRuntimeWorkReceipt(client, session, input.key);
2818
- if (latest && isReusableWorkReceipt(latest)) {
3550
+ if (
3551
+ latest &&
3552
+ input.forceRefresh !== true &&
3553
+ isReusableWorkReceipt(latest)
3554
+ ) {
2819
3555
  return { disposition: 'reused', receipt: latest };
2820
3556
  }
2821
- if (latest?.status === 'running') {
2822
- return { disposition: 'running', receipt: latest };
2823
- }
2824
- if (latest?.status === 'failed') {
2825
- return { disposition: 'failed', receipt: latest };
3557
+ if (latest) {
3558
+ return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
3559
+ receipt: latest,
3560
+ claimantRunId: input.runId,
3561
+ claimantRunAttempt: runAttempt,
3562
+ });
2826
3563
  }
2827
3564
  throw new Error(
2828
3565
  `Runtime receipt ${input.key} claim did not return execution ownership.`,
@@ -2831,14 +3568,48 @@ export async function claimRuntimeWorkReceipt(
2831
3568
  );
2832
3569
  }
2833
3570
 
3571
+ function runtimeWorkReceiptClaimDispositionForBlockedReceipt(input: {
3572
+ receipt: WorkReceipt;
3573
+ claimantRunId: string;
3574
+ claimantRunAttempt?: number | null;
3575
+ }): Exclude<WorkReceiptClaim, { disposition: 'claimed' | 'reused' }> {
3576
+ const receipt = input.receipt;
3577
+ if (receipt.status === 'failed') {
3578
+ return { disposition: 'failed', receipt };
3579
+ }
3580
+ if (receipt.status === 'pending' || receipt.status === 'running') {
3581
+ const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
3582
+ const leaseExpiresAtMs =
3583
+ typeof receipt.leaseExpiresAt === 'string'
3584
+ ? Date.parse(receipt.leaseExpiresAt)
3585
+ : NaN;
3586
+ const hasActiveForeignLease =
3587
+ ownerRunId !== null &&
3588
+ ownerRunId !== input.claimantRunId &&
3589
+ typeof receipt.leaseId === 'string' &&
3590
+ receipt.leaseId.length > 0 &&
3591
+ Number.isFinite(leaseExpiresAtMs) &&
3592
+ leaseExpiresAtMs > Date.now();
3593
+ return {
3594
+ disposition: hasActiveForeignLease ? 'blocked_active_lease' : 'running',
3595
+ receipt,
3596
+ };
3597
+ }
3598
+ return { disposition: 'running', receipt };
3599
+ }
3600
+
2834
3601
  export async function claimRuntimeWorkReceipts(
2835
3602
  context: RuntimeApiContext,
2836
3603
  input: {
2837
3604
  playName: string;
2838
3605
  runId: string;
2839
3606
  keys: string[];
3607
+ runAttempt?: number | null;
3608
+ leaseAware?: boolean;
2840
3609
  reclaimRunning?: boolean;
2841
3610
  forceRefresh?: boolean;
3611
+ forceFailedRefresh?: boolean;
3612
+ leaseTtlMs?: number | null;
2842
3613
  },
2843
3614
  ): Promise<WorkReceiptClaim[]> {
2844
3615
  const keys = [
@@ -2854,46 +3625,104 @@ export async function claimRuntimeWorkReceipts(
2854
3625
  session,
2855
3626
  async (client) => {
2856
3627
  const keyHexes = keys.map(workReceiptKeyHex);
2857
- const reclaimStatuses = input.forceRefresh
2858
- ? [
2859
- RECEIPT_STATUS_PENDING,
2860
- RECEIPT_STATUS_FAILED,
2861
- RECEIPT_STATUS_RUNNING,
2862
- RECEIPT_STATUS_COMPLETED,
2863
- RECEIPT_STATUS_SKIPPED,
2864
- ]
2865
- : input.reclaimRunning
2866
- ? [
2867
- RECEIPT_STATUS_PENDING,
2868
- RECEIPT_STATUS_FAILED,
2869
- RECEIPT_STATUS_RUNNING,
2870
- ]
2871
- : [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
3628
+ const leaseIds = keys.map(() =>
3629
+ input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
3630
+ );
3631
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3632
+ const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
3633
+ input.leaseTtlMs,
3634
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
3635
+ );
3636
+ const reclaimStatuses = workReceiptClaimableStatusCodes({
3637
+ forceRefresh: input.forceRefresh,
3638
+ forceFailedRefresh: input.forceFailedRefresh,
3639
+ reclaimRunning: input.reclaimRunning,
3640
+ });
2872
3641
  const { rows } = await client.query<Record<string, unknown>>(
2873
3642
  `
2874
3643
  WITH input_keys AS (
2875
- SELECT key_values.key_hex, key_values.ord
3644
+ SELECT key_values.key_hex, lease_values.lease_id, key_values.ord
2876
3645
  FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
3646
+ JOIN unnest($2::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
3647
+ ON lease_values.ord = key_values.ord
2877
3648
  ),
2878
3649
  claimed AS (
2879
- INSERT INTO ${workReceiptTable(session)} (k, status, run_id, updated_at)
2880
- SELECT decode(input_keys.key_hex, 'hex'), $2::smallint, $3::text, now()
3650
+ INSERT INTO ${workReceiptTable(session)} (
3651
+ k,
3652
+ status,
3653
+ run_id,
3654
+ lease_id,
3655
+ lease_owner_run_id,
3656
+ lease_owner_attempt,
3657
+ lease_expires_at,
3658
+ updated_at
3659
+ )
3660
+ SELECT decode(input_keys.key_hex, 'hex'),
3661
+ $3::smallint,
3662
+ $4::text,
3663
+ input_keys.lease_id,
3664
+ CASE
3665
+ WHEN input_keys.lease_id IS NULL THEN NULL
3666
+ ELSE $4::text
3667
+ END,
3668
+ CASE
3669
+ WHEN input_keys.lease_id IS NULL THEN NULL
3670
+ ELSE $9::integer
3671
+ END,
3672
+ CASE
3673
+ WHEN input_keys.lease_id IS NULL THEN NULL
3674
+ ELSE now() + ($7::double precision * interval '1 millisecond')
3675
+ END,
3676
+ now()
2881
3677
  FROM input_keys
2882
3678
  ON CONFLICT (k) DO UPDATE
2883
- SET status = $4::smallint,
3679
+ SET status = $5::smallint,
2884
3680
  output = NULL,
2885
- run_id = $3::text,
3681
+ failure_kind = 0,
3682
+ run_id = $4::text,
3683
+ lease_id = EXCLUDED.lease_id,
3684
+ lease_owner_run_id = CASE
3685
+ WHEN EXCLUDED.lease_id IS NULL THEN NULL
3686
+ ELSE $4::text
3687
+ END,
3688
+ lease_owner_attempt = CASE
3689
+ WHEN EXCLUDED.lease_id IS NULL THEN NULL
3690
+ ELSE $9::integer
3691
+ END,
3692
+ lease_expires_at = EXCLUDED.lease_expires_at,
2886
3693
  error = NULL,
2887
3694
  updated_at = now()
2888
- WHERE ${workReceiptTable(session)}.status = ANY($5::smallint[])
2889
- RETURNING k, status, output, error, run_id, updated_at
3695
+ WHERE ${workReceiptClaimConflictPredicateSql({
3696
+ receiptTable: workReceiptTable(session),
3697
+ claimableStatusesSql: '$6',
3698
+ claimantRunIdSql: '$4::text',
3699
+ claimantRunAttemptSql: '$9',
3700
+ forceRefreshSql: '$8',
3701
+ forceFailedRefreshSql: '$10',
3702
+ })}
3703
+ RETURNING k,
3704
+ status,
3705
+ output,
3706
+ error,
3707
+ failure_kind,
3708
+ run_id,
3709
+ lease_id,
3710
+ lease_owner_run_id,
3711
+ lease_owner_attempt,
3712
+ lease_expires_at,
3713
+ updated_at
2890
3714
  ),
2891
3715
  latest AS (
2892
3716
  SELECT receipts.k,
2893
3717
  receipts.status,
2894
3718
  receipts.output,
2895
3719
  receipts.error,
3720
+ receipts.failure_kind,
2896
3721
  receipts.run_id,
3722
+ receipts.lease_id,
3723
+ receipts.lease_owner_run_id,
3724
+ receipts.lease_owner_attempt,
3725
+ receipts.lease_expires_at,
2897
3726
  receipts.updated_at,
2898
3727
  input_keys.ord,
2899
3728
  false AS claimed
@@ -2909,21 +3738,43 @@ export async function claimRuntimeWorkReceipts(
2909
3738
  claimed.status,
2910
3739
  claimed.output,
2911
3740
  claimed.error,
3741
+ claimed.failure_kind,
2912
3742
  claimed.run_id,
3743
+ claimed.lease_id,
3744
+ claimed.lease_owner_run_id,
3745
+ claimed.lease_owner_attempt,
3746
+ claimed.lease_expires_at,
2913
3747
  claimed.updated_at,
2914
3748
  input_keys.ord,
2915
3749
  true AS claimed
2916
3750
  FROM claimed
2917
3751
  JOIN input_keys ON claimed.k = decode(input_keys.key_hex, 'hex')
2918
3752
  UNION ALL
2919
- SELECT k, status, output, error, run_id, updated_at, ord, claimed
3753
+ SELECT k,
3754
+ status,
3755
+ output,
3756
+ error,
3757
+ failure_kind,
3758
+ run_id,
3759
+ lease_id,
3760
+ lease_owner_run_id,
3761
+ lease_owner_attempt,
3762
+ lease_expires_at,
3763
+ updated_at,
3764
+ ord,
3765
+ claimed
2920
3766
  FROM latest
2921
3767
  )
2922
3768
  SELECT convert_from(k, 'UTF8') AS k,
2923
3769
  status,
2924
3770
  output,
2925
3771
  error,
3772
+ failure_kind,
2926
3773
  run_id,
3774
+ lease_id,
3775
+ lease_owner_run_id,
3776
+ lease_owner_attempt,
3777
+ lease_expires_at,
2927
3778
  updated_at,
2928
3779
  claimed
2929
3780
  FROM returned
@@ -2931,10 +3782,15 @@ export async function claimRuntimeWorkReceipts(
2931
3782
  `,
2932
3783
  [
2933
3784
  keyHexes,
3785
+ leaseIds,
2934
3786
  RECEIPT_STATUS_RUNNING,
2935
3787
  input.runId,
2936
3788
  RECEIPT_STATUS_RUNNING,
2937
3789
  reclaimStatuses,
3790
+ leaseTtlMs,
3791
+ input.forceRefresh === true,
3792
+ runAttempt,
3793
+ input.forceFailedRefresh === true,
2938
3794
  ],
2939
3795
  );
2940
3796
  return rows.map((row) => {
@@ -2942,16 +3798,14 @@ export async function claimRuntimeWorkReceipts(
2942
3798
  if (row.claimed === true) {
2943
3799
  return { disposition: 'claimed', receipt } satisfies WorkReceiptClaim;
2944
3800
  }
2945
- if (isReusableWorkReceipt(receipt)) {
3801
+ if (input.forceRefresh !== true && isReusableWorkReceipt(receipt)) {
2946
3802
  return { disposition: 'reused', receipt } satisfies WorkReceiptClaim;
2947
3803
  }
2948
- if (receipt.status === 'running') {
2949
- return { disposition: 'running', receipt } satisfies WorkReceiptClaim;
2950
- }
2951
- if (receipt.status === 'failed') {
2952
- return { disposition: 'failed', receipt } satisfies WorkReceiptClaim;
2953
- }
2954
- return { disposition: 'running', receipt } satisfies WorkReceiptClaim;
3804
+ return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
3805
+ receipt,
3806
+ claimantRunId: input.runId,
3807
+ claimantRunAttempt: runAttempt,
3808
+ });
2955
3809
  });
2956
3810
  },
2957
3811
  );
@@ -2963,17 +3817,50 @@ export async function completeRuntimeWorkReceipt(
2963
3817
  playName: string;
2964
3818
  runId: string;
2965
3819
  key: string;
3820
+ runAttempt?: number | null;
3821
+ leaseId?: string | null;
2966
3822
  output: unknown;
2967
3823
  },
2968
3824
  ): Promise<WorkReceipt | null> {
2969
- const session = await getRuntimeWorkReceiptSession(context, {
2970
- playName: input.playName,
2971
- key: input.key,
2972
- });
2973
- return await withRuntimeWorkReceiptClient(
2974
- context,
2975
- session,
2976
- async (client) => {
3825
+ if (
3826
+ consumeDirectRuntimeTestFault({
3827
+ context,
3828
+ runId: input.runId,
3829
+ name: 'receipt_complete_write_fail',
3830
+ })
3831
+ ) {
3832
+ const error = runtimeTestFaultError('receipt_complete_write_fail');
3833
+ const failed = await failRuntimeWorkReceipt(context, {
3834
+ playName: input.playName,
3835
+ runId: input.runId,
3836
+ key: input.key,
3837
+ leaseId: input.leaseId,
3838
+ error: error.message,
3839
+ failureKind: 'repairable',
3840
+ runAttempt: input.runAttempt,
3841
+ });
3842
+ if (!failed) {
3843
+ await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
3844
+ playName: input.playName,
3845
+ runId: input.runId,
3846
+ key: input.key,
3847
+ error: error.message,
3848
+ failureKind: 'repairable',
3849
+ runAttempt: input.runAttempt,
3850
+ });
3851
+ }
3852
+ throw error;
3853
+ }
3854
+ const session = await getRuntimeWorkReceiptSession(context, {
3855
+ playName: input.playName,
3856
+ key: input.key,
3857
+ });
3858
+ return await withRuntimeWorkReceiptClient(
3859
+ context,
3860
+ session,
3861
+ async (client) => {
3862
+ const leaseId = input.leaseId?.trim() || null;
3863
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
2977
3864
  const { rows } = await client.query<Record<string, unknown>>(
2978
3865
  `
2979
3866
  WITH completed AS (
@@ -2981,29 +3868,41 @@ export async function completeRuntimeWorkReceipt(
2981
3868
  SET status = $2::smallint,
2982
3869
  output = $3::jsonb,
2983
3870
  error = NULL,
3871
+ failure_kind = 0,
2984
3872
  run_id = $4,
3873
+ lease_id = NULL,
3874
+ lease_owner_run_id = NULL,
3875
+ lease_owner_attempt = NULL,
3876
+ lease_expires_at = NULL,
2985
3877
  updated_at = now()
2986
3878
  WHERE k = decode($1, 'hex')
2987
- AND status = $5::smallint
2988
- AND run_id = $4
2989
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2990
- ),
2991
- latest AS (
2992
- SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2993
- FROM ${workReceiptTable(session)}
2994
- WHERE k = decode($1, 'hex')
2995
- AND NOT EXISTS (SELECT 1 FROM completed)
2996
- )
2997
- SELECT k, status, output, error, run_id, updated_at FROM completed
2998
- UNION ALL
2999
- SELECT k, status, output, error, run_id, updated_at FROM latest
3879
+ AND ${workReceiptRunningOwnerPredicateSql({
3880
+ receiptTable: workReceiptTable(session),
3881
+ ownerRunIdSql: '$4',
3882
+ ownerRunAttemptSql: '$6',
3883
+ leaseIdSql: '$5::text',
3884
+ })}
3885
+ RETURNING convert_from(k, 'UTF8') AS k,
3886
+ status,
3887
+ output,
3888
+ error,
3889
+ failure_kind,
3890
+ run_id,
3891
+ lease_id,
3892
+ lease_owner_run_id,
3893
+ lease_owner_attempt,
3894
+ lease_expires_at,
3895
+ updated_at
3896
+ )
3897
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
3000
3898
  `,
3001
3899
  [
3002
3900
  workReceiptKeyHex(input.key),
3003
3901
  RECEIPT_STATUS_COMPLETED,
3004
3902
  input.output === null ? null : stringifyPostgresJson(input.output),
3005
3903
  input.runId,
3006
- RECEIPT_STATUS_RUNNING,
3904
+ leaseId,
3905
+ runAttempt,
3007
3906
  ],
3008
3907
  );
3009
3908
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -3015,11 +3914,63 @@ export async function completeRuntimeWorkReceipts(
3015
3914
  context: RuntimeApiContext,
3016
3915
  input: {
3017
3916
  playName: string;
3018
- receipts: Array<{ runId: string; key: string; output: unknown }>;
3917
+ receipts: Array<{
3918
+ runId: string;
3919
+ key: string;
3920
+ output: unknown;
3921
+ leaseId?: string | null;
3922
+ runAttempt?: number | null;
3923
+ }>;
3019
3924
  },
3020
- ): Promise<WorkReceipt[]> {
3925
+ ): Promise<Array<WorkReceipt | null>> {
3021
3926
  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
3022
3927
  if (receipts.length === 0) return [];
3928
+ if (
3929
+ consumeDirectRuntimeTestFault({
3930
+ context,
3931
+ runId: receipts[0]?.runId ?? '',
3932
+ name: 'receipt_complete_write_fail',
3933
+ })
3934
+ ) {
3935
+ const [skippedReceipt, ...siblingReceipts] = receipts;
3936
+ let siblingError: unknown = null;
3937
+ if (siblingReceipts.length > 0) {
3938
+ try {
3939
+ await completeRuntimeWorkReceipts(context, {
3940
+ playName: input.playName,
3941
+ receipts: siblingReceipts,
3942
+ });
3943
+ } catch (error) {
3944
+ siblingError = error;
3945
+ }
3946
+ }
3947
+ const error = runtimeTestFaultError('receipt_complete_write_fail');
3948
+ if (skippedReceipt) {
3949
+ const failed = await failRuntimeWorkReceipt(context, {
3950
+ playName: input.playName,
3951
+ runId: skippedReceipt.runId,
3952
+ key: skippedReceipt.key,
3953
+ leaseId: skippedReceipt.leaseId,
3954
+ error: error.message,
3955
+ failureKind: 'repairable',
3956
+ runAttempt: skippedReceipt.runAttempt,
3957
+ });
3958
+ if (!failed) {
3959
+ await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
3960
+ playName: input.playName,
3961
+ runId: skippedReceipt.runId,
3962
+ key: skippedReceipt.key,
3963
+ error: error.message,
3964
+ failureKind: 'repairable',
3965
+ runAttempt: skippedReceipt.runAttempt,
3966
+ });
3967
+ }
3968
+ }
3969
+ if (siblingError) {
3970
+ throw siblingError;
3971
+ }
3972
+ throw error;
3973
+ }
3023
3974
  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
3024
3975
  playName: input.playName,
3025
3976
  keys: receipts.map((receipt) => receipt.key),
@@ -3033,52 +3984,79 @@ export async function completeRuntimeWorkReceipts(
3033
3984
  WITH inputs AS (
3034
3985
  SELECT key_values.key_hex,
3035
3986
  run_values.run_id,
3987
+ lease_values.lease_id,
3988
+ attempt_values.run_attempt,
3036
3989
  output_values.output,
3037
3990
  key_values.ord
3038
3991
  FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
3039
3992
  JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
3040
3993
  ON run_values.ord = key_values.ord
3041
- JOIN jsonb_array_elements($3::jsonb) WITH ORDINALITY AS output_values(output, ord)
3994
+ JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
3995
+ ON lease_values.ord = key_values.ord
3996
+ JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
3997
+ ON attempt_values.ord = key_values.ord
3998
+ JOIN jsonb_array_elements($5::jsonb) WITH ORDINALITY AS output_values(output, ord)
3042
3999
  ON output_values.ord = key_values.ord
3043
4000
  ),
3044
4001
  completed AS (
3045
4002
  UPDATE ${workReceiptTable(session)} AS target
3046
- SET status = $4::smallint,
3047
- output = inputs.output,
3048
- error = NULL,
3049
- run_id = inputs.run_id,
4003
+ SET status = $6::smallint,
4004
+ output = inputs.output,
4005
+ error = NULL,
4006
+ failure_kind = 0,
4007
+ run_id = inputs.run_id,
4008
+ lease_id = NULL,
4009
+ lease_owner_run_id = NULL,
4010
+ lease_owner_attempt = NULL,
4011
+ lease_expires_at = NULL,
3050
4012
  updated_at = now()
3051
4013
  FROM inputs
3052
4014
  WHERE target.k = decode(inputs.key_hex, 'hex')
3053
- AND target.status = $5::smallint
3054
- AND target.run_id = inputs.run_id
3055
- RETURNING target.k, target.status, target.output, target.error, target.run_id, target.updated_at, inputs.ord
4015
+ AND ${workReceiptRunningOwnerPredicateSql({
4016
+ receiptTable: 'target',
4017
+ ownerRunIdSql: 'inputs.run_id',
4018
+ ownerRunAttemptSql: 'inputs.run_attempt',
4019
+ leaseIdSql: 'inputs.lease_id',
4020
+ })}
4021
+ RETURNING target.k,
4022
+ target.status,
4023
+ target.output,
4024
+ target.error,
4025
+ target.failure_kind,
4026
+ target.run_id,
4027
+ target.lease_id,
4028
+ target.lease_owner_run_id,
4029
+ target.lease_owner_attempt,
4030
+ target.lease_expires_at,
4031
+ target.updated_at,
4032
+ inputs.ord
3056
4033
  ),
3057
- latest AS (
3058
- SELECT receipts.k,
3059
- receipts.status,
3060
- receipts.output,
3061
- receipts.error,
3062
- receipts.run_id,
3063
- receipts.updated_at,
4034
+ returned AS (
4035
+ SELECT completed.k,
4036
+ completed.status,
4037
+ completed.output,
4038
+ completed.error,
4039
+ completed.failure_kind,
4040
+ completed.run_id,
4041
+ completed.lease_id,
4042
+ completed.lease_owner_run_id,
4043
+ completed.lease_owner_attempt,
4044
+ completed.lease_expires_at,
4045
+ completed.updated_at,
3064
4046
  inputs.ord
3065
4047
  FROM inputs
3066
- JOIN ${workReceiptTable(session)} AS receipts
3067
- ON receipts.k = decode(inputs.key_hex, 'hex')
3068
- WHERE NOT EXISTS (
3069
- SELECT 1 FROM completed WHERE completed.k = receipts.k
3070
- )
3071
- ),
3072
- returned AS (
3073
- SELECT k, status, output, error, run_id, updated_at, ord FROM completed
3074
- UNION ALL
3075
- SELECT k, status, output, error, run_id, updated_at, ord FROM latest
4048
+ LEFT JOIN completed ON completed.ord = inputs.ord
3076
4049
  )
3077
4050
  SELECT convert_from(returned.k, 'UTF8') AS k,
3078
4051
  returned.status,
3079
- returned.output,
3080
- returned.error,
3081
- returned.run_id,
4052
+ returned.output,
4053
+ returned.error,
4054
+ returned.failure_kind,
4055
+ returned.run_id,
4056
+ returned.lease_id,
4057
+ returned.lease_owner_run_id,
4058
+ returned.lease_owner_attempt,
4059
+ returned.lease_expires_at,
3082
4060
  returned.updated_at
3083
4061
  FROM returned
3084
4062
  ORDER BY returned.ord
@@ -3086,16 +4064,21 @@ export async function completeRuntimeWorkReceipts(
3086
4064
  [
3087
4065
  receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
3088
4066
  receipts.map((receipt) => receipt.runId),
4067
+ receipts.map((receipt) => receipt.leaseId?.trim() || null),
4068
+ receipts.map((receipt) =>
4069
+ normalizeRuntimeRunAttempt(receipt.runAttempt),
4070
+ ),
3089
4071
  stringifyPostgresJson(
3090
4072
  receipts.map((receipt) =>
3091
4073
  receipt.output === null ? null : receipt.output,
3092
4074
  ),
3093
4075
  ),
3094
4076
  RECEIPT_STATUS_COMPLETED,
3095
- RECEIPT_STATUS_RUNNING,
3096
4077
  ],
3097
4078
  );
3098
- return rows.map(mapRuntimeWorkReceiptRow);
4079
+ return rows.map((row) =>
4080
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
4081
+ );
3099
4082
  },
3100
4083
  );
3101
4084
  }
@@ -3106,7 +4089,10 @@ export async function failRuntimeWorkReceipt(
3106
4089
  playName: string;
3107
4090
  runId: string;
3108
4091
  key: string;
4092
+ runAttempt?: number | null;
4093
+ leaseId?: string | null;
3109
4094
  error: string;
4095
+ failureKind?: WorkReceiptFailureKind | null;
3110
4096
  },
3111
4097
  ): Promise<WorkReceipt | null> {
3112
4098
  const session = await getRuntimeWorkReceiptSession(context, {
@@ -3117,36 +4103,119 @@ export async function failRuntimeWorkReceipt(
3117
4103
  context,
3118
4104
  session,
3119
4105
  async (client) => {
4106
+ const leaseId = input.leaseId?.trim() || null;
4107
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3120
4108
  const { rows } = await client.query<Record<string, unknown>>(
3121
4109
  `
3122
4110
  WITH failed AS (
3123
4111
  UPDATE ${workReceiptTable(session)}
3124
- SET status = $2::smallint,
3125
- output = NULL,
3126
- error = $3,
4112
+ SET status = $2::smallint,
4113
+ output = NULL,
4114
+ error = $3,
4115
+ failure_kind = $6::smallint,
3127
4116
  run_id = $4,
4117
+ lease_id = NULL,
4118
+ lease_owner_run_id = NULL,
4119
+ lease_owner_attempt = NULL,
4120
+ lease_expires_at = NULL,
3128
4121
  updated_at = now()
3129
4122
  WHERE k = decode($1, 'hex')
3130
- AND status = $5::smallint
3131
- AND run_id = $4
3132
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3133
- ),
3134
- latest AS (
3135
- SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3136
- FROM ${workReceiptTable(session)}
3137
- WHERE k = decode($1, 'hex')
3138
- AND NOT EXISTS (SELECT 1 FROM failed)
3139
- )
3140
- SELECT k, status, output, error, run_id, updated_at FROM failed
3141
- UNION ALL
3142
- SELECT k, status, output, error, run_id, updated_at FROM latest
4123
+ AND ${workReceiptRunningOwnerPredicateSql({
4124
+ receiptTable: workReceiptTable(session),
4125
+ ownerRunIdSql: '$4',
4126
+ ownerRunAttemptSql: '$7',
4127
+ leaseIdSql: '$5::text',
4128
+ })}
4129
+ RETURNING convert_from(k, 'UTF8') AS k,
4130
+ status,
4131
+ output,
4132
+ error,
4133
+ failure_kind,
4134
+ run_id,
4135
+ lease_id,
4136
+ lease_owner_run_id,
4137
+ lease_owner_attempt,
4138
+ lease_expires_at,
4139
+ updated_at
4140
+ )
4141
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
3143
4142
  `,
3144
4143
  [
3145
4144
  workReceiptKeyHex(input.key),
3146
4145
  RECEIPT_STATUS_FAILED,
3147
4146
  input.error,
3148
4147
  input.runId,
3149
- RECEIPT_STATUS_RUNNING,
4148
+ leaseId,
4149
+ workReceiptFailureKindCode(input.failureKind),
4150
+ runAttempt,
4151
+ ],
4152
+ );
4153
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
4154
+ },
4155
+ );
4156
+ }
4157
+
4158
+ export async function releaseRuntimeWorkReceipt(
4159
+ context: RuntimeApiContext,
4160
+ input: {
4161
+ playName: string;
4162
+ runId: string;
4163
+ key: string;
4164
+ runAttempt?: number | null;
4165
+ leaseId?: string | null;
4166
+ },
4167
+ ): Promise<WorkReceipt | null> {
4168
+ const session = await getRuntimeWorkReceiptSession(context, {
4169
+ playName: input.playName,
4170
+ key: input.key,
4171
+ });
4172
+ return await withRuntimeWorkReceiptClient(
4173
+ context,
4174
+ session,
4175
+ async (client) => {
4176
+ const leaseId = input.leaseId?.trim() || null;
4177
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
4178
+ const { rows } = await client.query<Record<string, unknown>>(
4179
+ `
4180
+ WITH released AS (
4181
+ UPDATE ${workReceiptTable(session)}
4182
+ SET status = $2::smallint,
4183
+ output = NULL,
4184
+ error = NULL,
4185
+ failure_kind = 0,
4186
+ run_id = $3,
4187
+ lease_id = NULL,
4188
+ lease_owner_run_id = NULL,
4189
+ lease_owner_attempt = NULL,
4190
+ lease_expires_at = NULL,
4191
+ updated_at = now()
4192
+ WHERE k = decode($1, 'hex')
4193
+ AND ${workReceiptRunningOwnerPredicateSql({
4194
+ receiptTable: workReceiptTable(session),
4195
+ ownerRunIdSql: '$3',
4196
+ ownerRunAttemptSql: '$5',
4197
+ leaseIdSql: '$4::text',
4198
+ })}
4199
+ RETURNING convert_from(k, 'UTF8') AS k,
4200
+ status,
4201
+ output,
4202
+ error,
4203
+ failure_kind,
4204
+ run_id,
4205
+ lease_id,
4206
+ lease_owner_run_id,
4207
+ lease_owner_attempt,
4208
+ lease_expires_at,
4209
+ updated_at
4210
+ )
4211
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM released
4212
+ `,
4213
+ [
4214
+ workReceiptKeyHex(input.key),
4215
+ RECEIPT_STATUS_PENDING,
4216
+ input.runId,
4217
+ leaseId,
4218
+ runAttempt,
3150
4219
  ],
3151
4220
  );
3152
4221
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -3158,9 +4227,16 @@ export async function failRuntimeWorkReceipts(
3158
4227
  context: RuntimeApiContext,
3159
4228
  input: {
3160
4229
  playName: string;
3161
- receipts: Array<{ runId: string; key: string; error: string }>;
4230
+ receipts: Array<{
4231
+ runId: string;
4232
+ key: string;
4233
+ error: string;
4234
+ failureKind?: WorkReceiptFailureKind | null;
4235
+ leaseId?: string | null;
4236
+ runAttempt?: number | null;
4237
+ }>;
3162
4238
  },
3163
- ): Promise<WorkReceipt[]> {
4239
+ ): Promise<Array<WorkReceipt | null>> {
3164
4240
  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
3165
4241
  if (receipts.length === 0) return [];
3166
4242
  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
@@ -3176,65 +4252,207 @@ export async function failRuntimeWorkReceipts(
3176
4252
  WITH inputs AS (
3177
4253
  SELECT key_values.key_hex,
3178
4254
  run_values.run_id,
3179
- error_values.error,
3180
- key_values.ord
4255
+ lease_values.lease_id,
4256
+ attempt_values.run_attempt,
4257
+ error_values.error,
4258
+ failure_kind_values.failure_kind,
4259
+ key_values.ord
3181
4260
  FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
3182
4261
  JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
3183
4262
  ON run_values.ord = key_values.ord
3184
- JOIN unnest($3::text[]) WITH ORDINALITY AS error_values(error, ord)
3185
- ON error_values.ord = key_values.ord
4263
+ JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
4264
+ ON lease_values.ord = key_values.ord
4265
+ JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
4266
+ ON attempt_values.ord = key_values.ord
4267
+ JOIN unnest($5::text[]) WITH ORDINALITY AS error_values(error, ord)
4268
+ ON error_values.ord = key_values.ord
4269
+ JOIN unnest($6::smallint[]) WITH ORDINALITY AS failure_kind_values(failure_kind, ord)
4270
+ ON failure_kind_values.ord = key_values.ord
3186
4271
  ),
3187
4272
  failed AS (
3188
4273
  UPDATE ${workReceiptTable(session)} AS target
3189
- SET status = $4::smallint,
3190
- output = NULL,
3191
- error = inputs.error,
3192
- run_id = inputs.run_id,
4274
+ SET status = $7::smallint,
4275
+ output = NULL,
4276
+ error = inputs.error,
4277
+ failure_kind = inputs.failure_kind,
4278
+ run_id = inputs.run_id,
4279
+ lease_id = NULL,
4280
+ lease_owner_run_id = NULL,
4281
+ lease_owner_attempt = NULL,
4282
+ lease_expires_at = NULL,
3193
4283
  updated_at = now()
3194
4284
  FROM inputs
3195
4285
  WHERE target.k = decode(inputs.key_hex, 'hex')
3196
- AND target.status = $5::smallint
3197
- AND target.run_id = inputs.run_id
3198
- RETURNING target.k, target.status, target.output, target.error, target.run_id, target.updated_at, inputs.ord
4286
+ AND ${workReceiptRunningOwnerPredicateSql({
4287
+ receiptTable: 'target',
4288
+ ownerRunIdSql: 'inputs.run_id',
4289
+ ownerRunAttemptSql: 'inputs.run_attempt',
4290
+ leaseIdSql: 'inputs.lease_id',
4291
+ })}
4292
+ RETURNING target.k,
4293
+ target.status,
4294
+ target.output,
4295
+ target.error,
4296
+ target.failure_kind,
4297
+ target.run_id,
4298
+ target.lease_id,
4299
+ target.lease_owner_run_id,
4300
+ target.lease_owner_attempt,
4301
+ target.lease_expires_at,
4302
+ target.updated_at,
4303
+ inputs.ord
3199
4304
  ),
3200
- latest AS (
3201
- SELECT receipts.k,
3202
- receipts.status,
3203
- receipts.output,
3204
- receipts.error,
3205
- receipts.run_id,
3206
- receipts.updated_at,
4305
+ returned AS (
4306
+ SELECT failed.k,
4307
+ failed.status,
4308
+ failed.output,
4309
+ failed.error,
4310
+ failed.failure_kind,
4311
+ failed.run_id,
4312
+ failed.lease_id,
4313
+ failed.lease_owner_run_id,
4314
+ failed.lease_owner_attempt,
4315
+ failed.lease_expires_at,
4316
+ failed.updated_at,
3207
4317
  inputs.ord
3208
4318
  FROM inputs
3209
- JOIN ${workReceiptTable(session)} AS receipts
3210
- ON receipts.k = decode(inputs.key_hex, 'hex')
3211
- WHERE NOT EXISTS (
3212
- SELECT 1 FROM failed WHERE failed.k = receipts.k
3213
- )
4319
+ LEFT JOIN failed ON failed.ord = inputs.ord
4320
+ )
4321
+ SELECT convert_from(returned.k, 'UTF8') AS k,
4322
+ returned.status,
4323
+ returned.output,
4324
+ returned.error,
4325
+ returned.failure_kind,
4326
+ returned.run_id,
4327
+ returned.lease_id,
4328
+ returned.lease_owner_run_id,
4329
+ returned.lease_owner_attempt,
4330
+ returned.lease_expires_at,
4331
+ returned.updated_at
4332
+ FROM returned
4333
+ ORDER BY returned.ord
4334
+ `,
4335
+ [
4336
+ receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
4337
+ receipts.map((receipt) => receipt.runId),
4338
+ receipts.map((receipt) => receipt.leaseId?.trim() || null),
4339
+ receipts.map((receipt) =>
4340
+ normalizeRuntimeRunAttempt(receipt.runAttempt),
4341
+ ),
4342
+ receipts.map((receipt) => receipt.error),
4343
+ receipts.map((receipt) =>
4344
+ workReceiptFailureKindCode(receipt.failureKind),
4345
+ ),
4346
+ RECEIPT_STATUS_FAILED,
4347
+ ],
4348
+ );
4349
+ return rows.map((row) =>
4350
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
4351
+ );
4352
+ },
4353
+ );
4354
+ }
4355
+
4356
+ export async function heartbeatRuntimeWorkReceipts(
4357
+ context: RuntimeApiContext,
4358
+ input: {
4359
+ playName: string;
4360
+ runId: string;
4361
+ runAttempt?: number | null;
4362
+ leaseId: string;
4363
+ keys: string[];
4364
+ leaseTtlMs?: number | null;
4365
+ },
4366
+ ): Promise<Array<WorkReceipt | null>> {
4367
+ const keys = input.keys.map((key) => key.trim()).filter(Boolean);
4368
+ if (keys.length === 0) return [];
4369
+ const leaseId = input.leaseId.trim();
4370
+ if (!leaseId) return keys.map(() => null);
4371
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
4372
+ const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
4373
+ input.leaseTtlMs,
4374
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
4375
+ );
4376
+ const session = await getRuntimeWorkReceiptSessionForKeys(context, {
4377
+ playName: input.playName,
4378
+ keys,
4379
+ });
4380
+ return await withRuntimeWorkReceiptClient(
4381
+ context,
4382
+ session,
4383
+ async (client) => {
4384
+ const { rows } = await client.query<Record<string, unknown>>(
4385
+ `
4386
+ WITH input_keys AS (
4387
+ SELECT key_values.key_hex, key_values.ord
4388
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
4389
+ ),
4390
+ renewed AS (
4391
+ UPDATE ${workReceiptTable(session)} AS target
4392
+ SET lease_expires_at = now() + ($4::double precision * interval '1 millisecond'),
4393
+ updated_at = now()
4394
+ FROM input_keys
4395
+ WHERE target.k = decode(input_keys.key_hex, 'hex')
4396
+ AND ${workReceiptHeartbeatPredicateSql({
4397
+ receiptTable: 'target',
4398
+ ownerRunIdSql: '$2::text',
4399
+ ownerRunAttemptSql: '$5',
4400
+ leaseIdSql: '$3::text',
4401
+ })}
4402
+ RETURNING target.k,
4403
+ target.status,
4404
+ target.output,
4405
+ target.error,
4406
+ target.failure_kind,
4407
+ target.run_id,
4408
+ target.lease_id,
4409
+ target.lease_owner_run_id,
4410
+ target.lease_owner_attempt,
4411
+ target.lease_expires_at,
4412
+ target.updated_at,
4413
+ input_keys.ord
3214
4414
  ),
3215
4415
  returned AS (
3216
- SELECT k, status, output, error, run_id, updated_at, ord FROM failed
3217
- UNION ALL
3218
- SELECT k, status, output, error, run_id, updated_at, ord FROM latest
4416
+ SELECT renewed.k,
4417
+ renewed.status,
4418
+ renewed.output,
4419
+ renewed.error,
4420
+ renewed.failure_kind,
4421
+ renewed.run_id,
4422
+ renewed.lease_id,
4423
+ renewed.lease_owner_run_id,
4424
+ renewed.lease_owner_attempt,
4425
+ renewed.lease_expires_at,
4426
+ renewed.updated_at,
4427
+ input_keys.ord
4428
+ FROM input_keys
4429
+ LEFT JOIN renewed ON renewed.ord = input_keys.ord
3219
4430
  )
3220
4431
  SELECT convert_from(returned.k, 'UTF8') AS k,
3221
4432
  returned.status,
3222
4433
  returned.output,
3223
4434
  returned.error,
4435
+ returned.failure_kind,
3224
4436
  returned.run_id,
4437
+ returned.lease_id,
4438
+ returned.lease_owner_run_id,
4439
+ returned.lease_owner_attempt,
4440
+ returned.lease_expires_at,
3225
4441
  returned.updated_at
3226
4442
  FROM returned
3227
4443
  ORDER BY returned.ord
3228
4444
  `,
3229
4445
  [
3230
- receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
3231
- receipts.map((receipt) => receipt.runId),
3232
- receipts.map((receipt) => receipt.error),
3233
- RECEIPT_STATUS_FAILED,
3234
- RECEIPT_STATUS_RUNNING,
4446
+ keys.map(workReceiptKeyHex),
4447
+ input.runId,
4448
+ leaseId,
4449
+ leaseTtlMs,
4450
+ runAttempt,
3235
4451
  ],
3236
4452
  );
3237
- return rows.map(mapRuntimeWorkReceiptRow);
4453
+ return rows.map((row) =>
4454
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
4455
+ );
3238
4456
  },
3239
4457
  );
3240
4458
  }
@@ -3245,6 +4463,8 @@ export async function skipRuntimeWorkReceipt(
3245
4463
  playName: string;
3246
4464
  runId: string;
3247
4465
  key: string;
4466
+ runAttempt?: number | null;
4467
+ leaseId?: string | null;
3248
4468
  output: unknown;
3249
4469
  },
3250
4470
  ): Promise<WorkReceipt | null> {
@@ -3256,36 +4476,50 @@ export async function skipRuntimeWorkReceipt(
3256
4476
  context,
3257
4477
  session,
3258
4478
  async (client) => {
4479
+ const leaseId = input.leaseId?.trim() || null;
4480
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3259
4481
  const { rows } = await client.query<Record<string, unknown>>(
3260
4482
  `
3261
4483
  WITH skipped AS (
3262
4484
  UPDATE ${workReceiptTable(session)}
3263
- SET status = $2::smallint,
3264
- output = $3::jsonb,
3265
- error = NULL,
4485
+ SET status = $2::smallint,
4486
+ output = $3::jsonb,
4487
+ error = NULL,
4488
+ failure_kind = 0,
3266
4489
  run_id = $4,
4490
+ lease_id = NULL,
4491
+ lease_owner_run_id = NULL,
4492
+ lease_owner_attempt = NULL,
4493
+ lease_expires_at = NULL,
3267
4494
  updated_at = now()
3268
4495
  WHERE k = decode($1, 'hex')
3269
- AND status = $5::smallint
3270
- AND run_id = $4
3271
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3272
- ),
3273
- latest AS (
3274
- SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3275
- FROM ${workReceiptTable(session)}
3276
- WHERE k = decode($1, 'hex')
3277
- AND NOT EXISTS (SELECT 1 FROM skipped)
3278
- )
3279
- SELECT k, status, output, error, run_id, updated_at FROM skipped
3280
- UNION ALL
3281
- SELECT k, status, output, error, run_id, updated_at FROM latest
4496
+ AND ${workReceiptRunningOwnerPredicateSql({
4497
+ receiptTable: workReceiptTable(session),
4498
+ ownerRunIdSql: '$4',
4499
+ ownerRunAttemptSql: '$6',
4500
+ leaseIdSql: '$5::text',
4501
+ })}
4502
+ RETURNING convert_from(k, 'UTF8') AS k,
4503
+ status,
4504
+ output,
4505
+ error,
4506
+ failure_kind,
4507
+ run_id,
4508
+ lease_id,
4509
+ lease_owner_run_id,
4510
+ lease_owner_attempt,
4511
+ lease_expires_at,
4512
+ updated_at
4513
+ )
4514
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM skipped
3282
4515
  `,
3283
4516
  [
3284
4517
  workReceiptKeyHex(input.key),
3285
4518
  RECEIPT_STATUS_SKIPPED,
3286
4519
  input.output === null ? null : stringifyPostgresJson(input.output),
3287
4520
  input.runId,
3288
- RECEIPT_STATUS_RUNNING,
4521
+ leaseId,
4522
+ runAttempt,
3289
4523
  ],
3290
4524
  );
3291
4525
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -3302,6 +4536,11 @@ export async function startRuntimeSheetDataset(
3302
4536
  sheetContract: PlaySheetContract;
3303
4537
  rows: Record<string, unknown>[];
3304
4538
  runId: string;
4539
+ attemptId?: string | null;
4540
+ attemptOwnerRunId?: string | null;
4541
+ attemptExpiresAt?: string | null;
4542
+ attemptSeq?: number | null;
4543
+ attemptLeaseTtlMs?: number | null;
3305
4544
  inputOffset?: number;
3306
4545
  force?: boolean;
3307
4546
  },
@@ -3378,6 +4617,7 @@ export async function startRuntimeSheetDataset(
3378
4617
  skipped: input.rows.length,
3379
4618
  pendingRows: [],
3380
4619
  completedRows: [],
4620
+ blockedRows: [],
3381
4621
  tableNamespace: input.tableNamespace,
3382
4622
  timings: [
3383
4623
  ...timings,
@@ -3426,7 +4666,12 @@ export async function startRuntimeSheetDataset(
3426
4666
  const normalizedTableNamespace = normalizeTableNamespace(
3427
4667
  input.tableNamespace,
3428
4668
  );
3429
-
4669
+ const attemptId = normalizeRuntimeSheetAttemptId(input.attemptId);
4670
+ const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
4671
+ attemptOwnerRunId: input.attemptOwnerRunId,
4672
+ runId: input.runId,
4673
+ });
4674
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
3430
4675
  const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
3431
4676
  const needsTransaction = chunks.length > 1;
3432
4677
 
@@ -3441,12 +4686,26 @@ export async function startRuntimeSheetDataset(
3441
4686
  timings,
3442
4687
  },
3443
4688
  async (client) => {
4689
+ const attemptExpiryStartedAt = Date.now();
4690
+ const attemptExpiresAt = await mintRuntimeSheetAttemptExpiresAt(
4691
+ client,
4692
+ input.attemptLeaseTtlMs,
4693
+ );
4694
+ timings.push({
4695
+ phase: 'attempt_expiry_mint',
4696
+ ms: Date.now() - attemptExpiryStartedAt,
4697
+ rows: rowEntries.length,
4698
+ });
3444
4699
  const prepareStartedAt = Date.now();
3445
4700
  const prepared = await prepareRuntimeSheetDatasetRows(client, session, {
3446
4701
  chunks,
3447
4702
  runId: input.runId,
3448
4703
  normalizedPlayName,
3449
4704
  normalizedTableNamespace,
4705
+ attemptId,
4706
+ attemptOwnerRunId,
4707
+ attemptExpiresAt,
4708
+ attemptSeq,
3450
4709
  physicalInsertColumnsSql,
3451
4710
  physicalInsertValuesSql,
3452
4711
  physicalRefreshSetSql:
@@ -3455,19 +4714,15 @@ export async function startRuntimeSheetDataset(
3455
4714
  outputPhysicalColumns,
3456
4715
  force: input.force === true,
3457
4716
  });
3458
- const effectivePrepared = input.force
3459
- ? {
3460
- inserted: prepared.inserted,
3461
- pendingKeys: rowEntries.map((entry) => entry.key),
3462
- }
3463
- : prepared;
3464
4717
  timings.push({
3465
4718
  phase: 'prepare_rows_sql',
3466
4719
  ms: Date.now() - prepareStartedAt,
3467
4720
  rows: rowEntries.length,
3468
4721
  chunks: chunks.length,
3469
- inserted: effectivePrepared.inserted,
3470
- pending: effectivePrepared.pendingKeys.length,
4722
+ inserted: prepared.inserted,
4723
+ pending: prepared.rowDispositions.filter(
4724
+ (row) => row.disposition === 'pending',
4725
+ ).length,
3471
4726
  ...(input.force ? { force: true } : {}),
3472
4727
  });
3473
4728
  const buildStartedAt = Date.now();
@@ -3479,9 +4734,13 @@ export async function startRuntimeSheetDataset(
3479
4734
  normalizedPlayName,
3480
4735
  normalizedTableNamespace,
3481
4736
  runId: input.runId,
4737
+ attemptId,
4738
+ attemptOwnerRunId,
4739
+ attemptExpiresAt,
4740
+ attemptSeq,
3482
4741
  timings,
3483
4742
  force: input.force === true,
3484
- ...effectivePrepared,
4743
+ ...prepared,
3485
4744
  });
3486
4745
  timings.push({
3487
4746
  phase: 'build_result',
@@ -3492,7 +4751,11 @@ export async function startRuntimeSheetDataset(
3492
4751
  pending: built.pendingRows.length,
3493
4752
  completed: built.completedRows.length,
3494
4753
  });
3495
- return built;
4754
+ return {
4755
+ ...built,
4756
+ attemptExpiresAt,
4757
+ attemptSeq,
4758
+ };
3496
4759
  },
3497
4760
  );
3498
4761
  timings.push({
@@ -3505,7 +4768,298 @@ export async function startRuntimeSheetDataset(
3505
4768
  pending: result.pendingRows.length,
3506
4769
  completed: result.completedRows.length,
3507
4770
  });
3508
- return { ...result, timings };
4771
+ return {
4772
+ ...result,
4773
+ attemptId,
4774
+ attemptOwnerRunId,
4775
+ attemptExpiresAt: result.attemptExpiresAt,
4776
+ attemptSeq,
4777
+ timings,
4778
+ };
4779
+ }
4780
+
4781
+ export async function heartbeatRuntimeSheetAttempt(
4782
+ context: RuntimeApiContext,
4783
+ input: {
4784
+ playName: string;
4785
+ tableNamespace: string;
4786
+ sheetContract: PlaySheetContract;
4787
+ runId: string;
4788
+ attemptId: string;
4789
+ attemptOwnerRunId?: string | null;
4790
+ attemptSeq?: number | null;
4791
+ attemptLeaseTtlMs?: number | null;
4792
+ keys: string[];
4793
+ },
4794
+ ): Promise<RuntimeSheetAttemptHeartbeatResult> {
4795
+ const playName = context.playName?.trim() || input.playName;
4796
+ if (!playName) {
4797
+ throw new Error('Runtime sheet attempt heartbeat requires a playName.');
4798
+ }
4799
+ const attemptId = input.attemptId.trim();
4800
+ const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
4801
+ attemptOwnerRunId: input.attemptOwnerRunId,
4802
+ runId: input.runId,
4803
+ });
4804
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
4805
+ const attemptLeaseTtlMs = normalizeRuntimeLeaseTtlMs(
4806
+ input.attemptLeaseTtlMs,
4807
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
4808
+ );
4809
+ const keys = [
4810
+ ...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
4811
+ ];
4812
+ if (!attemptId || keys.length === 0) {
4813
+ return { renewed: 0, renewedKeys: [], attemptExpiresAt: null };
4814
+ }
4815
+ const timings: RuntimeSheetTiming[] = [];
4816
+ const session = requireRuntimePostgresSession(
4817
+ await getRuntimeDbSession(
4818
+ {
4819
+ ...context,
4820
+ playName,
4821
+ runId: context.runId ?? input.runId,
4822
+ },
4823
+ {
4824
+ tableNamespace: input.tableNamespace,
4825
+ logicalTable: 'sheet_rows',
4826
+ operations: ['rows.upsert'],
4827
+ limits: {
4828
+ maxRows: runtimeDbSessionRowLimit(keys.length),
4829
+ },
4830
+ sheetContract: input.sheetContract,
4831
+ timings,
4832
+ },
4833
+ ),
4834
+ );
4835
+ return await withRuntimeSheetQueryClient(
4836
+ context,
4837
+ session,
4838
+ {
4839
+ playName,
4840
+ tableNamespace: input.tableNamespace,
4841
+ sheetContract: input.sheetContract,
4842
+ transactional: false,
4843
+ timings,
4844
+ },
4845
+ async (client) => {
4846
+ const { rows } = await client.query<{
4847
+ renewed_keys: string[];
4848
+ renewed: number;
4849
+ attempt_expires_at: Date | string | null;
4850
+ }>(
4851
+ `
4852
+ WITH input_keys AS (
4853
+ SELECT key_values._key
4854
+ FROM unnest($1::text[]) AS key_values(_key)
4855
+ ),
4856
+ owned_terminal AS (
4857
+ SELECT target._key
4858
+ FROM ${sheetTable(session)} AS target
4859
+ JOIN input_keys ON target._key = input_keys._key
4860
+ WHERE target._attempt_id = $2::text
4861
+ AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
4862
+ AND target._run_id = $4::text
4863
+ AND COALESCE(target._attempt_seq, 0) = $6::integer
4864
+ AND target._status = 'enriched'
4865
+ ),
4866
+ renewed AS (
4867
+ UPDATE ${sheetTable(session)} AS target
4868
+ SET _attempt_expires_at = now() + ($5::double precision * interval '1 millisecond'),
4869
+ _updated_at = now(),
4870
+ _version = ${nextRuntimeSheetVersionExpression(session)}
4871
+ FROM input_keys
4872
+ WHERE target._key = input_keys._key
4873
+ AND target._attempt_id = $2::text
4874
+ AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
4875
+ AND target._run_id = $4::text
4876
+ AND COALESCE(target._attempt_seq, 0) = $6::integer
4877
+ AND target._status IN ('pending', 'running', 'failed')
4878
+ RETURNING target._key, target._attempt_expires_at
4879
+ ),
4880
+ owned_keys AS (
4881
+ SELECT _key FROM renewed
4882
+ UNION
4883
+ SELECT _key FROM owned_terminal
4884
+ )
4885
+ SELECT coalesce((SELECT array_agg(_key ORDER BY _key) FROM owned_keys), '{}'::text[]) AS renewed_keys,
4886
+ count(*)::int AS renewed,
4887
+ max(_attempt_expires_at) AS attempt_expires_at
4888
+ FROM renewed
4889
+ `,
4890
+ [
4891
+ keys,
4892
+ attemptId,
4893
+ attemptOwnerRunId,
4894
+ input.runId,
4895
+ attemptLeaseTtlMs,
4896
+ attemptSeq,
4897
+ ],
4898
+ );
4899
+ const row = rows[0];
4900
+ const attemptExpiresAt = row?.attempt_expires_at;
4901
+ return {
4902
+ renewed: Number(row?.renewed ?? 0),
4903
+ renewedKeys: row?.renewed_keys ?? [],
4904
+ attemptExpiresAt:
4905
+ attemptExpiresAt instanceof Date
4906
+ ? attemptExpiresAt.toISOString()
4907
+ : typeof attemptExpiresAt === 'string'
4908
+ ? new Date(attemptExpiresAt).toISOString()
4909
+ : null,
4910
+ };
4911
+ },
4912
+ );
4913
+ }
4914
+
4915
+ /**
4916
+ * Controlled run-fatal teardown for runtime sheet rows: clear the attempt lease
4917
+ * (`_attempt_*`) on every row the aborting attempt still owns, so an immediate
4918
+ * rerun can claim pending rows and overwrite the aborted attempt's rows without
4919
+ * waiting out the 10-minute lease TTL. The predicate is attempt-fenced (see
4920
+ * `releasableRuntimeSheetAttemptFenceSql` / `decideRuntimeSheetRelease`): it only
4921
+ * touches rows owned by this run's attempt generation, so it can never weaken a
4922
+ * different run's live lease. TTL expiry remains the recovery path for true
4923
+ * crashes where no teardown code runs. When `attemptId` is null every attempt id
4924
+ * the run holds at `attemptSeq` is released (the workers runner mints a distinct
4925
+ * attempt id per map chunk under one owner run id + seq).
4926
+ */
4927
+ export async function releaseRuntimeSheetAttempt(
4928
+ context: RuntimeApiContext,
4929
+ input: {
4930
+ playName: string;
4931
+ tableNamespace: string;
4932
+ sheetContract: PlaySheetContract;
4933
+ runId: string;
4934
+ attemptId?: string | null;
4935
+ attemptOwnerRunId?: string | null;
4936
+ attemptSeq?: number | null;
4937
+ },
4938
+ ): Promise<RuntimeSheetAttemptReleaseResult> {
4939
+ const playName = context.playName?.trim() || input.playName;
4940
+ if (!playName) {
4941
+ throw new Error('Runtime sheet attempt release requires a playName.');
4942
+ }
4943
+ const attemptId = input.attemptId?.trim() || null;
4944
+ const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
4945
+ attemptOwnerRunId: input.attemptOwnerRunId,
4946
+ runId: input.runId,
4947
+ });
4948
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
4949
+ if (!attemptOwnerRunId) {
4950
+ return { released: 0, releasedKeys: [] };
4951
+ }
4952
+ const timings: RuntimeSheetTiming[] = [];
4953
+ const session = requireRuntimePostgresSession(
4954
+ await getRuntimeDbSession(
4955
+ {
4956
+ ...context,
4957
+ playName,
4958
+ runId: context.runId ?? input.runId,
4959
+ },
4960
+ {
4961
+ tableNamespace: input.tableNamespace,
4962
+ logicalTable: 'sheet_rows',
4963
+ operations: ['rows.upsert'],
4964
+ limits: {},
4965
+ sheetContract: input.sheetContract,
4966
+ timings,
4967
+ },
4968
+ ),
4969
+ );
4970
+ return await withRuntimeSheetQueryClient(
4971
+ context,
4972
+ session,
4973
+ {
4974
+ playName,
4975
+ tableNamespace: input.tableNamespace,
4976
+ sheetContract: input.sheetContract,
4977
+ transactional: false,
4978
+ timings,
4979
+ },
4980
+ async (client) => {
4981
+ const { rows } = await client.query<{ _key: string }>(
4982
+ `
4983
+ UPDATE ${sheetTable(session)} AS target
4984
+ SET _attempt_id = NULL,
4985
+ _attempt_owner_run_id = NULL,
4986
+ _attempt_expires_at = NULL,
4987
+ _attempt_seq = NULL,
4988
+ _updated_at = now(),
4989
+ _version = ${nextRuntimeSheetVersionExpression(session)}
4990
+ WHERE ${releasableRuntimeSheetAttemptFenceSql(
4991
+ 'target',
4992
+ '$1::text',
4993
+ '$2::text',
4994
+ '$3',
4995
+ )}
4996
+ RETURNING target._key
4997
+ `,
4998
+ [attemptId, attemptOwnerRunId, attemptSeq],
4999
+ );
5000
+ const releasedKeys = rows
5001
+ .map((row) => row._key)
5002
+ .filter((key): key is string => typeof key === 'string');
5003
+ return { released: releasedKeys.length, releasedKeys };
5004
+ },
5005
+ );
5006
+ }
5007
+
5008
+ /**
5009
+ * Controlled run-fatal teardown for work receipts: return every receipt still
5010
+ * leased `running` by the aborting run+attempt back to `pending` (clearing the
5011
+ * lease) so an immediate rerun can reclaim and recompute the interrupted work
5012
+ * without waiting out the lease TTL. Fenced by owner run id + attempt (see
5013
+ * `workReceiptReleaseOwnerPredicateSql`), so it never touches a receipt a live
5014
+ * competitor legitimately holds. Completed/skipped/failed receipts are terminal
5015
+ * and untouched, preserving cross-run idempotency (already-persisted work is
5016
+ * still reused on rerun). Work receipts for a play share one table, so a single
5017
+ * run-scoped statement covers every stranded receipt.
5018
+ */
5019
+ export async function releaseRuntimeWorkReceipts(
5020
+ context: RuntimeApiContext,
5021
+ input: {
5022
+ playName: string;
5023
+ runId: string;
5024
+ runAttempt?: number | null;
5025
+ },
5026
+ ): Promise<RuntimeWorkReceiptReleaseResult> {
5027
+ const runId = input.runId.trim();
5028
+ if (!runId) {
5029
+ return { released: 0, releasedKeys: [] };
5030
+ }
5031
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
5032
+ const session = await getRuntimeWorkReceiptSessionForRun(context, {
5033
+ playName: input.playName,
5034
+ });
5035
+ return await withRuntimeWorkReceiptClient(
5036
+ context,
5037
+ session,
5038
+ async (client) => {
5039
+ const { rows } = await client.query<{ k: string }>(
5040
+ `
5041
+ UPDATE ${workReceiptTable(session)} AS target
5042
+ SET status = $3::smallint,
5043
+ lease_id = NULL,
5044
+ lease_owner_run_id = NULL,
5045
+ lease_owner_attempt = NULL,
5046
+ lease_expires_at = NULL,
5047
+ updated_at = now()
5048
+ WHERE ${workReceiptReleaseOwnerPredicateSql({
5049
+ receiptTable: 'target',
5050
+ ownerRunIdSql: '$1',
5051
+ ownerRunAttemptSql: '$2',
5052
+ })}
5053
+ RETURNING convert_from(target.k, 'UTF8') AS k
5054
+ `,
5055
+ [runId, runAttempt, RECEIPT_STATUS_PENDING],
5056
+ );
5057
+ const releasedKeys = rows
5058
+ .map((row) => row.k)
5059
+ .filter((key): key is string => typeof key === 'string');
5060
+ return { released: releasedKeys.length, releasedKeys };
5061
+ },
5062
+ );
3509
5063
  }
3510
5064
 
3511
5065
  type CompleteRuntimeMapRowChunksInput = {
@@ -3513,17 +5067,41 @@ type CompleteRuntimeMapRowChunksInput = {
3513
5067
  physicalUpdateSetSql: string;
3514
5068
  physicalColumnProjections: PhysicalSheetColumnProjection[];
3515
5069
  runId: string;
5070
+ attemptId: string | null;
5071
+ attemptOwnerRunId: string | null;
5072
+ attemptExpiresAt: string | null;
5073
+ attemptSeq: number;
3516
5074
  normalizedPlayName: string;
3517
5075
  normalizedTableNamespace: string;
3518
5076
  outputFields: string[];
3519
5077
  };
3520
5078
 
5079
+ type RuntimeMapRowsWriteResult = {
5080
+ updated: number;
5081
+ fencedKeys: string[];
5082
+ };
5083
+
5084
+ function writableEnrichedRuntimeSheetAttemptSql(
5085
+ tableAlias: string,
5086
+ attemptOwnerRunIdExpression: string,
5087
+ attemptExpiresAtExpression: string,
5088
+ attemptSeqExpression: string,
5089
+ ): string {
5090
+ return `(
5091
+ ${tableAlias}._status = 'enriched'
5092
+ AND (
5093
+ COALESCE(${tableAlias}._attempt_seq, 0) < ${attemptSeqExpression}::integer
5094
+ )
5095
+ )`;
5096
+ }
5097
+
3521
5098
  async function completeRuntimeMapRowChunks(
3522
5099
  client: RuntimeQueryClient,
3523
5100
  session: RuntimePostgresSession,
3524
5101
  input: CompleteRuntimeMapRowChunksInput,
3525
- ): Promise<{ updated: number }> {
5102
+ ): Promise<RuntimeMapRowsWriteResult> {
3526
5103
  let updated = 0;
5104
+ const fencedKeys: string[] = [];
3527
5105
  for (const chunk of input.chunks) {
3528
5106
  const chunkKeys = chunk.map((row) => row.key);
3529
5107
  const chunkInputIndexes = chunk.map((row) => row.input_index);
@@ -3538,9 +5116,21 @@ async function completeRuntimeMapRowChunks(
3538
5116
  'updates.data_patch',
3539
5117
  input.physicalColumnProjections,
3540
5118
  );
5119
+ const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
5120
+ 'target',
5121
+ '$10::text',
5122
+ '$11::timestamptz',
5123
+ '$12',
5124
+ );
5125
+ const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
5126
+ 'target',
5127
+ '$10::text',
5128
+ '$12',
5129
+ );
3541
5130
  const { rows } = await client.query<{
3542
5131
  updated: number;
3543
5132
  matched_keys: string[];
5133
+ fenced_keys: string[];
3544
5134
  }>(
3545
5135
  `WITH updates AS (
3546
5136
  SELECT key_values._key,
@@ -3571,12 +5161,22 @@ async function completeRuntimeMapRowChunks(
3571
5161
  SET _status = 'enriched',
3572
5162
  _run_id = $5::text,
3573
5163
  _error = NULL,
5164
+ _attempt_id = $9::text,
5165
+ _attempt_owner_run_id = $10::text,
5166
+ _attempt_expires_at = $11::timestamptz,
5167
+ _attempt_seq = $12::integer,
3574
5168
  _updated_at = now(),
3575
5169
  _version = ${nextRuntimeSheetVersionExpression(session)},
3576
5170
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
3577
5171
  FROM matched_updates AS updates
3578
5172
  WHERE target._key = updates.matched_key
3579
- AND (target._run_id IS NULL OR target._run_id <= $5::text)
5173
+ AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5174
+ AND NOT (${sameOwnerTerminalEpochSql})
5175
+ AND (
5176
+ ($9::text IS NULL AND target._run_id = $5::text)
5177
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5178
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5179
+ )
3580
5180
  AND (
3581
5181
  target._status <> 'enriched'
3582
5182
  OR target._run_id IS DISTINCT FROM $5::text
@@ -3596,6 +5196,23 @@ async function completeRuntimeMapRowChunks(
3596
5196
  count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
3597
5197
  FROM applied_rows
3598
5198
  ),
5199
+ fenced_rows AS (
5200
+ SELECT updates.matched_key AS _key
5201
+ FROM matched_updates AS updates
5202
+ JOIN ${sheetTable(session)} AS target
5203
+ ON target._key = updates.matched_key
5204
+ WHERE NOT EXISTS (
5205
+ SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5206
+ )
5207
+ AND (
5208
+ (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5209
+ OR NOT (
5210
+ ($9::text IS NULL AND target._run_id = $5::text)
5211
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5212
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5213
+ )
5214
+ )
5215
+ ),
3599
5216
  summary_counts AS (
3600
5217
  SELECT newly_completed,
3601
5218
  from_failed,
@@ -3680,6 +5297,7 @@ async function completeRuntimeMapRowChunks(
3680
5297
  SELECT
3681
5298
  (SELECT count(*)::int FROM applied_rows) AS updated,
3682
5299
  coalesce((SELECT array_agg(matched_key) FROM matched_updates), '{}'::text[]) AS matched_keys,
5300
+ coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
3683
5301
  (SELECT count(*)::int FROM summary_delta) AS summary_delta_count,
3684
5302
  (SELECT count(*)::int FROM column_delta) AS column_delta_count`,
3685
5303
  [
@@ -3691,9 +5309,14 @@ async function completeRuntimeMapRowChunks(
3691
5309
  input.normalizedPlayName,
3692
5310
  input.normalizedTableNamespace,
3693
5311
  [...new Set(input.outputFields)],
5312
+ input.attemptId,
5313
+ input.attemptOwnerRunId,
5314
+ input.attemptExpiresAt,
5315
+ input.attemptSeq,
3694
5316
  ],
3695
5317
  );
3696
5318
  updated += Number(rows[0]?.updated ?? 0);
5319
+ fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
3697
5320
 
3698
5321
  const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
3699
5322
  if (matchedKeys.size === chunk.length) {
@@ -3712,16 +5335,18 @@ async function completeRuntimeMapRowChunks(
3712
5335
  { ...input, chunks: [repairChunk] },
3713
5336
  );
3714
5337
  updated += repaired.updated;
5338
+ fencedKeys.push(...repaired.fencedKeys);
3715
5339
  }
3716
- return { updated };
5340
+ return { updated, fencedKeys };
3717
5341
  }
3718
5342
 
3719
5343
  async function completeRuntimeMapRowChunksWithInputIndexRepair(
3720
5344
  client: RuntimeQueryClient,
3721
5345
  session: RuntimePostgresSession,
3722
5346
  input: CompleteRuntimeMapRowChunksInput,
3723
- ): Promise<{ updated: number }> {
5347
+ ): Promise<RuntimeMapRowsWriteResult> {
3724
5348
  let updated = 0;
5349
+ const fencedKeys: string[] = [];
3725
5350
  for (const chunk of input.chunks) {
3726
5351
  const chunkKeys = chunk.map((row) => row.key);
3727
5352
  const chunkInputIndexes = chunk.map((row) => row.input_index);
@@ -3736,7 +5361,22 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3736
5361
  'updates.data_patch',
3737
5362
  input.physicalColumnProjections,
3738
5363
  );
3739
- const { rows: appliedRows } = await client.query<{ _key: string }>(
5364
+ const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
5365
+ 'target',
5366
+ '$10::text',
5367
+ '$11::timestamptz',
5368
+ '$12',
5369
+ );
5370
+ const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
5371
+ 'target',
5372
+ '$10::text',
5373
+ '$12',
5374
+ );
5375
+ const { rows } = await client.query<{
5376
+ _key?: string;
5377
+ fenced_keys?: string[];
5378
+ updated?: number;
5379
+ }>(
3740
5380
  `WITH updates AS (
3741
5381
  SELECT key_values._key,
3742
5382
  input_index_values.input_index,
@@ -3770,13 +5410,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3770
5410
  SET _status = 'enriched',
3771
5411
  _run_id = $5::text,
3772
5412
  _error = NULL,
5413
+ _attempt_id = $9::text,
5414
+ _attempt_owner_run_id = $10::text,
5415
+ _attempt_expires_at = $11::timestamptz,
5416
+ _attempt_seq = $12::integer,
3773
5417
  _updated_at = now(),
3774
5418
  _version = ${nextRuntimeSheetVersionExpression(session)},
3775
5419
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
3776
5420
  FROM matched_updates AS updates, ${sheetTable(session)} AS prev
3777
5421
  WHERE target._key = updates.matched_key
3778
5422
  AND prev._key = target._key
3779
- AND (target._run_id IS NULL OR target._run_id <= $5::text)
5423
+ AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5424
+ AND NOT (${sameOwnerTerminalEpochSql})
5425
+ AND (
5426
+ ($9::text IS NULL AND target._run_id = $5::text)
5427
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5428
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5429
+ )
3780
5430
  AND (
3781
5431
  target._status <> 'enriched'
3782
5432
  OR target._run_id IS DISTINCT FROM $5::text
@@ -3796,6 +5446,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3796
5446
  count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
3797
5447
  FROM applied_rows
3798
5448
  ),
5449
+ fenced_rows AS (
5450
+ SELECT updates.matched_key AS _key
5451
+ FROM matched_updates AS updates
5452
+ JOIN ${sheetTable(session)} AS target
5453
+ ON target._key = updates.matched_key
5454
+ WHERE NOT EXISTS (
5455
+ SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5456
+ )
5457
+ AND (
5458
+ (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5459
+ OR NOT (
5460
+ ($9::text IS NULL AND target._run_id = $5::text)
5461
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5462
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5463
+ )
5464
+ )
5465
+ ),
3799
5466
  summary_counts AS (
3800
5467
  SELECT newly_completed,
3801
5468
  from_failed,
@@ -3877,7 +5544,9 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3877
5544
  _updated_at = now()
3878
5545
  RETURNING 1
3879
5546
  )
3880
- SELECT _key FROM applied_rows`,
5547
+ SELECT
5548
+ (SELECT count(*)::int FROM applied_rows) AS updated,
5549
+ coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
3881
5550
  [
3882
5551
  chunkKeys,
3883
5552
  chunkInputIndexes,
@@ -3887,11 +5556,16 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3887
5556
  input.normalizedPlayName,
3888
5557
  input.normalizedTableNamespace,
3889
5558
  [...new Set(input.outputFields)],
5559
+ input.attemptId,
5560
+ input.attemptOwnerRunId,
5561
+ input.attemptExpiresAt,
5562
+ input.attemptSeq,
3890
5563
  ],
3891
5564
  );
3892
- updated += appliedRows.length;
5565
+ updated += Number(rows[0]?.updated ?? 0);
5566
+ fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
3893
5567
  }
3894
- return { updated };
5568
+ return { updated, fencedKeys };
3895
5569
  }
3896
5570
 
3897
5571
  async function insertMissingCompletedMapRowChunks(
@@ -3954,8 +5628,26 @@ async function insertMissingCompletedMapRowChunks(
3954
5628
  )
3955
5629
  ),
3956
5630
  inserted_rows AS (
3957
- INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _cell_meta${physicalInsertColumnsSql})
3958
- SELECT _key, 'enriched', $5::text, input_index, cell_meta_patch${physicalInsertValuesSql}
5631
+ INSERT INTO ${sheetTable(session)} (
5632
+ _key,
5633
+ _status,
5634
+ _run_id,
5635
+ _input_index,
5636
+ _attempt_id,
5637
+ _attempt_owner_run_id,
5638
+ _attempt_expires_at,
5639
+ _attempt_seq,
5640
+ _cell_meta${physicalInsertColumnsSql}
5641
+ )
5642
+ SELECT _key,
5643
+ 'enriched',
5644
+ $5::text,
5645
+ input_index,
5646
+ $8::text,
5647
+ $9::text,
5648
+ $10::timestamptz,
5649
+ $11::integer,
5650
+ cell_meta_patch${physicalInsertValuesSql}
3959
5651
  FROM missing_rows
3960
5652
  ON CONFLICT (_key) DO NOTHING
3961
5653
  RETURNING _key
@@ -3997,7 +5689,7 @@ async function insertMissingCompletedMapRowChunks(
3997
5689
  ),
3998
5690
  completed_cell_delta AS (
3999
5691
  SELECT field_values.field, (SELECT c FROM inserted_count) AS c
4000
- FROM unnest($8::text[]) AS field_values(field)
5692
+ FROM unnest($12::text[]) AS field_values(field)
4001
5693
  WHERE (SELECT c FROM inserted_count) > 0
4002
5694
  ),
4003
5695
  column_delta AS (
@@ -4028,6 +5720,10 @@ async function insertMissingCompletedMapRowChunks(
4028
5720
  input.runId,
4029
5721
  input.normalizedPlayName,
4030
5722
  input.normalizedTableNamespace,
5723
+ input.attemptId,
5724
+ input.attemptOwnerRunId,
5725
+ input.attemptExpiresAt,
5726
+ input.attemptSeq,
4031
5727
  [...new Set(input.outputFields)],
4032
5728
  ],
4033
5729
  );
@@ -4053,12 +5749,18 @@ async function failRuntimeMapRowChunks(
4053
5749
  input: {
4054
5750
  chunks: RuntimePreparedFailedRow[][];
4055
5751
  physicalUpdateSetSql: string;
5752
+ forceTerminal?: boolean;
4056
5753
  runId: string;
5754
+ attemptId: string | null;
5755
+ attemptOwnerRunId: string | null;
5756
+ attemptExpiresAt: string | null;
5757
+ attemptSeq: number;
4057
5758
  normalizedPlayName: string;
4058
5759
  normalizedTableNamespace: string;
4059
5760
  },
4060
- ): Promise<{ updated: number }> {
5761
+ ): Promise<RuntimeMapRowsWriteResult> {
4061
5762
  let updated = 0;
5763
+ const fencedKeys: string[] = [];
4062
5764
  for (const chunk of input.chunks) {
4063
5765
  const chunkKeys = chunk.map((row) => row.key);
4064
5766
  const chunkInputIndexes = chunk.map((row) => row.input_index);
@@ -4087,7 +5789,10 @@ async function failRuntimeMapRowChunks(
4087
5789
  const failedCellTotals = failedCellFields.map(
4088
5790
  (field) => failedCellCounts.get(field) ?? 0,
4089
5791
  );
4090
- const { rows: appliedRows } = await client.query<{ _key: string }>(
5792
+ const { rows } = await client.query<{
5793
+ updated?: number;
5794
+ fenced_keys?: string[];
5795
+ }>(
4091
5796
  `WITH updates AS (
4092
5797
  SELECT key_values._key,
4093
5798
  input_index_values.input_index,
@@ -4125,26 +5830,53 @@ async function failRuntimeMapRowChunks(
4125
5830
  SET _status = 'failed',
4126
5831
  _run_id = $6::text,
4127
5832
  _error = updates.error,
5833
+ _attempt_id = $11::text,
5834
+ _attempt_owner_run_id = $12::text,
5835
+ _attempt_expires_at = $13::timestamptz,
5836
+ _attempt_seq = $14::integer,
4128
5837
  _updated_at = now(),
4129
5838
  _version = ${nextRuntimeSheetVersionExpression(session)},
4130
5839
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
4131
5840
  FROM matched_updates AS updates, ${sheetTable(session)} AS prev
4132
5841
  WHERE target._key = updates.matched_key
4133
5842
  AND prev._key = target._key
4134
- AND target._status <> 'enriched'
4135
- AND (target._run_id IS NULL OR target._run_id <= $6::text)
5843
+ AND ($15::boolean OR target._status <> 'enriched')
5844
+ AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
5845
+ AND (
5846
+ ($11::text IS NULL AND target._run_id = $6::text)
5847
+ OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
5848
+ )
4136
5849
  RETURNING target._key, prev._status AS prev_status
4137
5850
  ),
4138
5851
  applied_count AS (
4139
5852
  SELECT count(*)::bigint AS c,
4140
5853
  count(*) FILTER (WHERE prev_status = 'failed')::bigint AS already_failed,
5854
+ count(*) FILTER (WHERE prev_status = 'enriched')::bigint AS from_completed,
4141
5855
  count(*) FILTER (WHERE prev_status = 'running')::bigint AS from_running
4142
5856
  FROM applied_rows
4143
5857
  ),
5858
+ fenced_rows AS (
5859
+ SELECT updates.matched_key AS _key
5860
+ FROM matched_updates AS updates
5861
+ JOIN ${sheetTable(session)} AS target
5862
+ ON target._key = updates.matched_key
5863
+ WHERE NOT EXISTS (
5864
+ SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5865
+ )
5866
+ AND (
5867
+ (NOT $15::boolean AND target._status = 'enriched')
5868
+ OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
5869
+ OR NOT (
5870
+ ($11::text IS NULL AND target._run_id = $6::text)
5871
+ OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
5872
+ )
5873
+ )
5874
+ ),
4144
5875
  summary_counts AS (
4145
5876
  SELECT (c - already_failed)::bigint AS newly_failed,
5877
+ from_completed,
4146
5878
  from_running,
4147
- GREATEST(c - already_failed - from_running, 0)::bigint AS from_queued
5879
+ GREATEST(c - already_failed - from_completed - from_running, 0)::bigint AS from_queued
4148
5880
  FROM applied_count
4149
5881
  ),
4150
5882
  summary_delta AS (
@@ -4162,20 +5894,21 @@ async function failRuntimeMapRowChunks(
4162
5894
  0,
4163
5895
  (-from_queued)::int,
4164
5896
  (-from_running)::int,
4165
- 0,
5897
+ (-from_completed)::int,
4166
5898
  newly_failed::int
4167
5899
  FROM summary_counts
4168
5900
  WHERE newly_failed > 0
4169
5901
  ON CONFLICT (play_name, table_namespace) DO UPDATE SET
4170
5902
  queued = GREATEST(target.queued + EXCLUDED.queued, 0),
4171
5903
  running = GREATEST(target.running + EXCLUDED.running, 0),
5904
+ completed = GREATEST(target.completed + EXCLUDED.completed, 0),
4172
5905
  failed = GREATEST(target.failed + EXCLUDED.failed, 0),
4173
5906
  total = ${runtimeSummaryTotalSql({
4174
5907
  currentTotal: 'target.total',
4175
5908
  totalDelta: 'EXCLUDED.total',
4176
5909
  queued: 'GREATEST(target.queued + EXCLUDED.queued, 0)',
4177
5910
  running: 'GREATEST(target.running + EXCLUDED.running, 0)',
4178
- completed: 'target.completed',
5911
+ completed: 'GREATEST(target.completed + EXCLUDED.completed, 0)',
4179
5912
  failed: 'GREATEST(target.failed + EXCLUDED.failed, 0)',
4180
5913
  })},
4181
5914
  _updated_at = now()
@@ -4198,7 +5931,9 @@ async function failRuntimeMapRowChunks(
4198
5931
  _updated_at = now()
4199
5932
  RETURNING 1
4200
5933
  )
4201
- SELECT _key FROM applied_rows`,
5934
+ SELECT
5935
+ (SELECT count(*)::int FROM applied_rows) AS updated,
5936
+ coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
4202
5937
  [
4203
5938
  chunkKeys,
4204
5939
  chunkInputIndexes,
@@ -4210,11 +5945,17 @@ async function failRuntimeMapRowChunks(
4210
5945
  input.normalizedTableNamespace,
4211
5946
  failedCellFields,
4212
5947
  failedCellTotals,
5948
+ input.attemptId,
5949
+ input.attemptOwnerRunId,
5950
+ input.attemptExpiresAt,
5951
+ input.attemptSeq,
5952
+ input.forceTerminal === true,
4213
5953
  ],
4214
5954
  );
4215
- updated += appliedRows.length;
5955
+ updated += Number(rows[0]?.updated ?? 0);
5956
+ fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
4216
5957
  }
4217
- return { updated };
5958
+ return { updated, fencedKeys };
4218
5959
  }
4219
5960
 
4220
5961
  /**
@@ -4240,10 +5981,15 @@ export async function completeRuntimeMapRows(
4240
5981
  rows: RuntimeApiRowRecord[];
4241
5982
  outputFields?: string[];
4242
5983
  runId: string;
5984
+ attemptId?: string | null;
5985
+ attemptOwnerRunId?: string | null;
5986
+ attemptExpiresAt?: string | null;
5987
+ attemptSeq?: number | null;
5988
+ forceFailedRows?: boolean;
4243
5989
  },
4244
- ): Promise<{ updated: number }> {
5990
+ ): Promise<RuntimeMapRowsWriteResult> {
4245
5991
  if (input.rows.length === 0) {
4246
- return { updated: 0 };
5992
+ return { updated: 0, fencedKeys: [] };
4247
5993
  }
4248
5994
  const sheetContract = augmentSheetContractWithDatasetFields({
4249
5995
  contract: input.sheetContract,
@@ -4276,7 +6022,7 @@ export async function completeRuntimeMapRows(
4276
6022
  uniqueRows.set(row.key, row);
4277
6023
  }
4278
6024
  if (uniqueRows.size === 0) {
4279
- return { updated: 0 };
6025
+ return { updated: 0, fencedKeys: [] };
4280
6026
  }
4281
6027
 
4282
6028
  const projections = physicalSheetColumnProjections(sheetContract);
@@ -4304,6 +6050,11 @@ export async function completeRuntimeMapRows(
4304
6050
  const failedChunks = chunkValues(failedRows, DIRECT_POSTGRES_BATCH_SIZE);
4305
6051
  const needsTransaction = chunks.length + failedChunks.length > 1;
4306
6052
  const outputFields = [...new Set(input.outputFields ?? [])];
6053
+ const attemptId = input.attemptId?.trim() || null;
6054
+ const attemptOwnerRunId =
6055
+ input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
6056
+ const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
6057
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
4307
6058
 
4308
6059
  return await withRuntimeSheetQueryClient(
4309
6060
  context,
@@ -4323,6 +6074,10 @@ export async function completeRuntimeMapRows(
4323
6074
  physicalUpdateSetSql,
4324
6075
  physicalColumnProjections: projections,
4325
6076
  runId: input.runId,
6077
+ attemptId,
6078
+ attemptOwnerRunId,
6079
+ attemptExpiresAt,
6080
+ attemptSeq,
4326
6081
  normalizedPlayName: normalizePlayNameForSheet(session.playName),
4327
6082
  normalizedTableNamespace: normalizeTableNamespace(
4328
6083
  input.tableNamespace,
@@ -4339,22 +6094,33 @@ export async function completeRuntimeMapRows(
4339
6094
  session,
4340
6095
  chunkInput,
4341
6096
  );
4342
- return { updated: updated.updated + inserted.inserted };
6097
+ return {
6098
+ updated: updated.updated + inserted.inserted,
6099
+ fencedKeys: updated.fencedKeys,
6100
+ };
4343
6101
  })()
4344
- : { updated: 0 };
6102
+ : { updated: 0, fencedKeys: [] };
4345
6103
  const failed =
4346
6104
  failedRows.length > 0
4347
6105
  ? await failRuntimeMapRowChunks(client, session, {
4348
6106
  chunks: failedChunks,
4349
6107
  physicalUpdateSetSql,
6108
+ forceTerminal: input.forceFailedRows === true,
4350
6109
  runId: input.runId,
6110
+ attemptId,
6111
+ attemptOwnerRunId,
6112
+ attemptExpiresAt,
6113
+ attemptSeq,
4351
6114
  normalizedPlayName: normalizePlayNameForSheet(session.playName),
4352
6115
  normalizedTableNamespace: normalizeTableNamespace(
4353
6116
  input.tableNamespace,
4354
6117
  ),
4355
6118
  })
4356
- : { updated: 0 };
4357
- return { updated: completed.updated + failed.updated };
6119
+ : { updated: 0, fencedKeys: [] };
6120
+ return {
6121
+ updated: completed.updated + failed.updated,
6122
+ fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
6123
+ };
4358
6124
  },
4359
6125
  );
4360
6126
  }
@@ -4408,7 +6174,7 @@ export async function readRuntimeSheetDatasetRowKeys(
4408
6174
  tableNamespace: string;
4409
6175
  runId: string;
4410
6176
  keys: string[];
4411
- rowMode?: 'output' | 'all';
6177
+ rowMode?: 'output' | 'all' | 'terminalAnyRun';
4412
6178
  },
4413
6179
  ): Promise<{ keys: string[] }> {
4414
6180
  const keys = [
@@ -4435,6 +6201,16 @@ export async function readRuntimeSheetDatasetRowKeys(
4435
6201
  ),
4436
6202
  );
4437
6203
  const rows = await withRuntimePostgres(session, async (client) => {
6204
+ if (input.rowMode === 'terminalAnyRun') {
6205
+ const { rows: matchedRows } = await client.query<{ _key: string }>(
6206
+ `SELECT _key
6207
+ FROM ${sheetTable(session)}
6208
+ WHERE _key = ANY($1::text[])
6209
+ AND _status IN ('enriched', 'failed')`,
6210
+ [keys],
6211
+ );
6212
+ return matchedRows;
6213
+ }
4438
6214
  if (input.rowMode === 'all') {
4439
6215
  const { rows: matchedRows } = await client.query<{ _key: string }>(
4440
6216
  `SELECT _key