deepline 0.1.182 → 0.1.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1747 -1890
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  15. package/dist/bundling-sources/sdk/src/client.ts +41 -0
  16. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  17. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  18. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  19. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  23. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1605 -281
  26. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
  27. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  29. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  32. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  40. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  43. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  44. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
  47. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  51. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  52. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  53. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  54. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  58. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  59. package/dist/cli/index.js +76 -17
  60. package/dist/cli/index.mjs +76 -17
  61. package/dist/index.d.mts +2 -0
  62. package/dist/index.d.ts +2 -0
  63. package/dist/index.js +63 -9
  64. package/dist/index.mjs +63 -9
  65. package/dist/plays/bundle-play-file.mjs +2 -2
  66. package/package.json +1 -1
@@ -1,4 +1,4 @@
1
- import { createHash } from 'node:crypto';
1
+ import { createHash, randomUUID } from 'node:crypto';
2
2
  import {
3
3
  createRuntimePool,
4
4
  createRuntimeOneShotQueryClient,
@@ -55,12 +55,28 @@ import {
55
55
  isReusableWorkReceipt,
56
56
  type WorkReceipt,
57
57
  type WorkReceiptClaim,
58
+ workReceiptFailureKindCode,
59
+ workReceiptFailureKindFromCode,
60
+ type WorkReceiptFailureKind,
58
61
  } from './work-receipts';
59
62
  import {
60
63
  sanitizePostgresJsonValue,
61
64
  stringifyPostgresJson,
62
65
  } from './postgres-json';
63
66
  import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
67
+ import {
68
+ workReceiptClaimableStatusCodes,
69
+ workReceiptClaimConflictPredicateSql,
70
+ workReceiptHeartbeatPredicateSql,
71
+ workReceiptReleaseOwnerPredicateSql,
72
+ workReceiptRunningOwnerPredicateSql,
73
+ } from './receipt-sql';
74
+ import {
75
+ activeRuntimeSheetAttemptFenceSql,
76
+ newerTerminalRuntimeSheetRowSql,
77
+ releasableRuntimeSheetAttemptFenceSql,
78
+ sameOwnerTerminalAttemptEpochSql,
79
+ } from './sheet-attempt-sql';
64
80
  import type { MapRowOutcome } from './durability-store';
65
81
  import {
66
82
  normalizeRuntimeMapInputIndex,
@@ -75,6 +91,19 @@ import {
75
91
  } from './map-row-outcome';
76
92
  import { DEEPLINE_CELL_META_FIELD } from './cell-staleness';
77
93
  import type { PlayArtifactKind } from './backend';
94
+ import {
95
+ PLAY_RUNTIME_CONTRACT,
96
+ PLAY_RUNTIME_CONTRACT_HEADER,
97
+ } from './runtime-contract';
98
+ import {
99
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
100
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
101
+ } from './lease-policy';
102
+ import {
103
+ parseRuntimeTestFaultCounts,
104
+ PLAY_RUNTIME_TEST_FAULT_HEADER,
105
+ type RuntimeTestFaultName,
106
+ } from './test-runtime-seams';
78
107
 
79
108
  type RuntimeApiContext = {
80
109
  baseUrl?: string | null;
@@ -87,10 +116,133 @@ type RuntimeApiContext = {
87
116
  dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | null;
88
117
  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
89
118
  vercelProtectionBypassToken?: string | null;
119
+ runtimeTestFaultHeader?: string | null;
90
120
  disablePostgresPoolCache?: boolean | null;
91
121
  postgresSessionUnwrapKey?: string | null;
92
122
  };
93
123
 
124
+ type DirectRuntimeTestFaultState = {
125
+ headerValue: string;
126
+ counts: Partial<Record<RuntimeTestFaultName, number>>;
127
+ };
128
+
129
+ const directRuntimeTestFaults = new Map<string, DirectRuntimeTestFaultState>();
130
+
131
+ function consumeDirectRuntimeTestFault(input: {
132
+ context: RuntimeApiContext;
133
+ runId: string;
134
+ name: RuntimeTestFaultName;
135
+ }): boolean {
136
+ const headerValue = input.context.runtimeTestFaultHeader?.trim();
137
+ if (!headerValue) return false;
138
+ const stateKey = input.runId.trim();
139
+ if (!stateKey) return false;
140
+ let state = directRuntimeTestFaults.get(stateKey);
141
+ if (!state || state.headerValue !== headerValue) {
142
+ const parsed = parseRuntimeTestFaultCounts({
143
+ headerValue,
144
+ syntheticRunHeader: '1',
145
+ });
146
+ if (parsed.ok === false) {
147
+ throw new Error(parsed.error);
148
+ }
149
+ state = {
150
+ headerValue,
151
+ counts:
152
+ (parsed.counts as Partial<Record<RuntimeTestFaultName, number>>) ?? {},
153
+ };
154
+ directRuntimeTestFaults.set(stateKey, state);
155
+ }
156
+ const count = state.counts[input.name] ?? 0;
157
+ if (count <= 0) return false;
158
+ if (count === 1) {
159
+ delete state.counts[input.name];
160
+ } else {
161
+ state.counts[input.name] = count - 1;
162
+ }
163
+ state.headerValue = headerValue;
164
+ directRuntimeTestFaults.set(stateKey, state);
165
+ return true;
166
+ }
167
+
168
+ function runtimeTestFaultError(name: RuntimeTestFaultName): Error {
169
+ const error = new Error(`Injected runtime test fault: ${name}.`);
170
+ if (
171
+ name === 'receipt_complete_write_fail' ||
172
+ name === 'receipt_fail_write_fail'
173
+ ) {
174
+ error.name = 'RuntimeReceiptPersistenceError';
175
+ }
176
+ return error;
177
+ }
178
+
179
+ async function forceFailRuntimeWorkReceiptForRuntimeTestFault(
180
+ context: RuntimeApiContext,
181
+ input: {
182
+ playName: string;
183
+ runId: string;
184
+ key: string;
185
+ error: string;
186
+ failureKind: WorkReceiptFailureKind;
187
+ runAttempt?: number | null;
188
+ },
189
+ ): Promise<WorkReceipt | null> {
190
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
191
+ const session = await getRuntimeWorkReceiptSession(context, {
192
+ playName: input.playName,
193
+ key: input.key,
194
+ });
195
+ return await withRuntimeWorkReceiptClient(
196
+ context,
197
+ session,
198
+ async (client) => {
199
+ const { rows } = await client.query<Record<string, unknown>>(
200
+ `
201
+ WITH failed AS (
202
+ UPDATE ${workReceiptTable(session)}
203
+ SET status = $2::smallint,
204
+ output = NULL,
205
+ error = $3,
206
+ failure_kind = $5::smallint,
207
+ lease_id = NULL,
208
+ lease_owner_run_id = NULL,
209
+ lease_owner_attempt = NULL,
210
+ lease_expires_at = NULL,
211
+ updated_at = now()
212
+ WHERE k = decode($1, 'hex')
213
+ AND run_id = $4
214
+ AND COALESCE(lease_owner_attempt, 0) = $7::integer
215
+ AND status <> $6::smallint
216
+ RETURNING convert_from(k, 'UTF8') AS k,
217
+ status,
218
+ output,
219
+ error,
220
+ failure_kind,
221
+ run_id,
222
+ lease_id,
223
+ lease_owner_run_id,
224
+ lease_owner_attempt,
225
+ lease_expires_at,
226
+ updated_at
227
+ )
228
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at
229
+ FROM failed
230
+ `,
231
+ [
232
+ workReceiptKeyHex(input.key),
233
+ RECEIPT_STATUS_FAILED,
234
+ input.error,
235
+ input.runId,
236
+ workReceiptFailureKindCode(input.failureKind),
237
+ RECEIPT_STATUS_COMPLETED,
238
+ runAttempt,
239
+ ],
240
+ );
241
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
242
+ },
243
+ );
244
+ }
245
+
94
246
  type DbSessionCacheEntry = {
95
247
  session: CreateDbSessionResponse;
96
248
  };
@@ -101,6 +253,11 @@ type RuntimeDatasetRowEntry = {
101
253
  row: Record<string, unknown>;
102
254
  inputIndex: number;
103
255
  };
256
+ type RuntimeSheetPrepareDisposition = 'pending' | 'completed' | 'blocked';
257
+ type RuntimeSheetPreparedRowDisposition = {
258
+ key: string;
259
+ disposition: RuntimeSheetPrepareDisposition;
260
+ };
104
261
  const dbSessionCache = new Map<string, DbSessionCacheEntry>();
105
262
  const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
106
263
  const postgresPools = new Map<string, RuntimePool>();
@@ -119,6 +276,23 @@ const APPEND_KEY_SUFFIX_LENGTH = 12;
119
276
  // 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
120
277
  // the tenant runtime role is long-lived), not run failures.
121
278
  const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
279
+ const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
280
+ 'lease_id',
281
+ 'lease_owner_run_id',
282
+ 'lease_owner_attempt',
283
+ 'lease_expires_at',
284
+ ] as const;
285
+ const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
286
+ const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
287
+ ...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
288
+ RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
289
+ ] as const;
290
+ const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
291
+ '_attempt_id',
292
+ '_attempt_owner_run_id',
293
+ '_attempt_seq',
294
+ '_attempt_expires_at',
295
+ ] as const;
122
296
  const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
123
297
  const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [
124
298
  100, 250, 500, 1_000, 2_000,
@@ -178,8 +352,13 @@ export type PrepareRuntimeSheetResult = {
178
352
  skipped: number;
179
353
  pendingRows: Record<string, unknown>[];
180
354
  completedRows: Record<string, unknown>[];
355
+ blockedRows: Record<string, unknown>[];
181
356
  tableNamespace: string;
182
357
  timings?: RuntimeSheetTiming[];
358
+ attemptId?: string;
359
+ attemptOwnerRunId?: string;
360
+ attemptExpiresAt?: string;
361
+ attemptSeq?: number;
183
362
  };
184
363
 
185
364
  export type RuntimeSheetTiming = {
@@ -197,6 +376,22 @@ export type RuntimeSheetTiming = {
197
376
  error?: string;
198
377
  };
199
378
 
379
+ export type RuntimeSheetAttemptHeartbeatResult = {
380
+ renewed: number;
381
+ renewedKeys: string[];
382
+ attemptExpiresAt: string | null;
383
+ };
384
+
385
+ export type RuntimeSheetAttemptReleaseResult = {
386
+ released: number;
387
+ releasedKeys: string[];
388
+ };
389
+
390
+ export type RuntimeWorkReceiptReleaseResult = {
391
+ released: number;
392
+ releasedKeys: string[];
393
+ };
394
+
200
395
  type RuntimeApiActionRequest =
201
396
  | {
202
397
  action: 'resolve_play';
@@ -261,7 +456,7 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
261
456
  throw new Error('Runner runtime API requires a baseUrl.');
262
457
  }
263
458
  const url = new URL(
264
- `${baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`,
459
+ `${baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime`,
265
460
  );
266
461
  const bypassToken = context.vercelProtectionBypassToken?.trim();
267
462
  if (bypassToken) {
@@ -282,9 +477,16 @@ function resolveRuntimeApiHeaders(
282
477
  return {
283
478
  'content-type': 'application/json',
284
479
  authorization: `Bearer ${token}`,
480
+ [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
285
481
  ...(context.vercelProtectionBypassToken
286
482
  ? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
287
483
  : {}),
484
+ ...(context.runtimeTestFaultHeader?.trim()
485
+ ? {
486
+ [PLAY_RUNTIME_TEST_FAULT_HEADER]:
487
+ context.runtimeTestFaultHeader.trim(),
488
+ }
489
+ : {}),
288
490
  ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
289
491
  };
290
492
  }
@@ -516,20 +718,36 @@ async function isRuntimeSheetSchemaReady(
516
718
  physicalColumns.length > 0
517
719
  ? physicalColumns.map(quoteIdentifier).join(', ')
518
720
  : '1';
519
- const query = async (client: RuntimeQueryClient): Promise<void> => {
721
+ const query = async (client: RuntimeQueryClient): Promise<boolean> => {
520
722
  await client.query(
521
723
  `SELECT ${selectList}
522
724
  FROM ${sheetTable(session)}
523
725
  LIMIT 0`,
524
726
  );
727
+ const attemptColumns = await client.query<Record<string, unknown>>(
728
+ `
729
+ SELECT count(*)::int AS present_count
730
+ FROM information_schema.columns
731
+ WHERE table_schema = $1
732
+ AND table_name = $2
733
+ AND column_name = ANY($3::text[])
734
+ `,
735
+ [
736
+ session.postgres.schema,
737
+ session.postgres.sheetTable,
738
+ [...RUNTIME_SHEET_ATTEMPT_COLUMNS],
739
+ ],
740
+ );
741
+ return (
742
+ Number(attemptColumns.rows[0]?.present_count ?? 0) ===
743
+ RUNTIME_SHEET_ATTEMPT_COLUMNS.length
744
+ );
525
745
  };
526
746
  try {
527
747
  if (isRuntimeOneShotQueryFactoryRegistered()) {
528
- await withRuntimeOneShotPostgres(session, query);
529
- } else {
530
- await withRuntimePostgres(session, query);
748
+ return await withRuntimeOneShotPostgres(session, query);
531
749
  }
532
- return true;
750
+ return await withRuntimePostgres(session, query);
533
751
  } catch (error) {
534
752
  if (isMissingRelationError(error)) {
535
753
  return false;
@@ -1530,6 +1748,10 @@ function workReceiptKeyHex(key: string): string {
1530
1748
  ).join('');
1531
1749
  }
1532
1750
 
1751
+ function newRuntimeWorkReceiptLeaseId(): string {
1752
+ return `receipt-lease:${randomUUID()}`;
1753
+ }
1754
+
1533
1755
  function validateRuntimeWorkReceiptKeyScope(
1534
1756
  session: RuntimePostgresSession,
1535
1757
  input: { key: string },
@@ -1548,22 +1770,86 @@ function validateRuntimeWorkReceiptKeyScope(
1548
1770
  }
1549
1771
 
1550
1772
  function mapRuntimeWorkReceiptRow(raw: Record<string, unknown>): WorkReceipt {
1773
+ const leaseExpiresAt = raw.lease_expires_at;
1551
1774
  return {
1552
1775
  key: String(raw.k ?? ''),
1553
1776
  status: receiptStatusFromCode(raw.status),
1554
1777
  output: raw.output == null ? null : raw.output,
1555
1778
  error: raw.error == null ? null : String(raw.error),
1779
+ failureKind: workReceiptFailureKindFromCode(raw.failure_kind),
1556
1780
  runId: raw.run_id == null ? null : String(raw.run_id),
1781
+ leaseId: raw.lease_id == null ? null : String(raw.lease_id),
1782
+ leaseOwnerRunId:
1783
+ raw.lease_owner_run_id == null ? null : String(raw.lease_owner_run_id),
1784
+ leaseOwnerAttempt:
1785
+ raw.lease_owner_attempt == null ||
1786
+ !Number.isFinite(Number(raw.lease_owner_attempt))
1787
+ ? null
1788
+ : Number(raw.lease_owner_attempt),
1789
+ leaseExpiresAt:
1790
+ leaseExpiresAt instanceof Date
1791
+ ? leaseExpiresAt.toISOString()
1792
+ : leaseExpiresAt == null
1793
+ ? null
1794
+ : String(leaseExpiresAt),
1557
1795
  updatedAt: raw.updated_at == null ? null : String(raw.updated_at),
1558
1796
  };
1559
1797
  }
1560
1798
 
1799
+ function isMissingRuntimeWorkReceiptSelfHealColumnError(
1800
+ error: unknown,
1801
+ ): boolean {
1802
+ if (!error || typeof error !== 'object') {
1803
+ return false;
1804
+ }
1805
+ const code = 'code' in error ? String(error.code) : '';
1806
+ const message = 'message' in error ? String(error.message) : '';
1807
+ if (code !== '42703' && !/column .* does not exist/i.test(message)) {
1808
+ return false;
1809
+ }
1810
+ return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.some((column) =>
1811
+ message.includes(column),
1812
+ );
1813
+ }
1814
+
1561
1815
  function runtimeWorkReceiptEnsureCacheKey(
1562
1816
  session: RuntimePostgresSession,
1563
1817
  ): string {
1564
1818
  return `${postgresUrlCacheKey(session.postgresUrl)}::${session.postgres.schema}::${session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE}`;
1565
1819
  }
1566
1820
 
1821
+ async function missingRuntimeWorkReceiptSelfHealColumns(
1822
+ session: RuntimePostgresSession,
1823
+ client: RuntimeQueryClient,
1824
+ ): Promise<string[]> {
1825
+ const result = await client.query(
1826
+ `
1827
+ SELECT column_name
1828
+ FROM information_schema.columns
1829
+ WHERE table_schema = $1
1830
+ AND table_name = $2
1831
+ AND column_name = ANY($3::text[])
1832
+ `,
1833
+ [
1834
+ session.postgres.schema,
1835
+ session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
1836
+ [...RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS],
1837
+ ],
1838
+ );
1839
+ const present = new Set(
1840
+ result.rows.map((row) => String(row.column_name ?? '')),
1841
+ );
1842
+ return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.filter(
1843
+ (column) => !present.has(column),
1844
+ );
1845
+ }
1846
+
1847
+ // COMPAT / SELF-HEAL — owner: runtime. Removal milestone: execution-ledger
1848
+ // cutover M1 (docs/play-runtime-execution-ledger-plan.md). The customer-Neon
1849
+ // work-receipt table (lease columns + failure_kind) is the dying surface. The
1850
+ // F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
1851
+ // columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
1852
+ // deleted together with the Neon receipt path at M1.
1567
1853
  async function ensureRuntimeWorkReceiptTable(
1568
1854
  session: RuntimePostgresSession,
1569
1855
  client: RuntimeQueryClient,
@@ -1582,11 +1868,39 @@ async function ensureRuntimeWorkReceiptTable(
1582
1868
  status smallint NOT NULL DEFAULT 0,
1583
1869
  output jsonb,
1584
1870
  error text,
1871
+ failure_kind smallint NOT NULL DEFAULT 0,
1585
1872
  run_id text,
1873
+ lease_id text,
1874
+ lease_owner_run_id text,
1875
+ lease_owner_attempt integer,
1876
+ lease_expires_at timestamptz,
1586
1877
  updated_at timestamptz NOT NULL DEFAULT now()
1587
1878
  )
1588
1879
  `,
1589
1880
  )
1881
+ .then(async () => {
1882
+ const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
1883
+ session,
1884
+ client,
1885
+ );
1886
+ if (missingColumns.length === 0) return;
1887
+ await client.query(`
1888
+ ALTER TABLE ${workReceiptTable(session)}
1889
+ ${missingColumns
1890
+ .map((column) => {
1891
+ const type =
1892
+ column === 'lease_expires_at'
1893
+ ? 'timestamptz'
1894
+ : column === 'lease_owner_attempt'
1895
+ ? 'integer'
1896
+ : column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
1897
+ ? 'smallint NOT NULL DEFAULT 0'
1898
+ : 'text';
1899
+ return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
1900
+ })
1901
+ .join(',\n ')}
1902
+ `);
1903
+ })
1590
1904
  .then(() => undefined);
1591
1905
  runtimeWorkReceiptEnsureCache.set(cacheKey, promise);
1592
1906
  try {
@@ -1612,11 +1926,26 @@ async function withRuntimeWorkReceiptClient<T>(
1612
1926
  try {
1613
1927
  return await operation(client);
1614
1928
  } catch (error) {
1615
- if (isMissingRelationError(error)) {
1929
+ if (
1930
+ isMissingRelationError(error) ||
1931
+ isMissingRuntimeWorkReceiptSelfHealColumnError(error)
1932
+ ) {
1616
1933
  runtimeWorkReceiptEnsureCache.delete(
1617
1934
  runtimeWorkReceiptEnsureCacheKey(session),
1618
1935
  );
1619
- await ensureRuntimeWorkReceiptTable(session, client);
1936
+ try {
1937
+ await ensureRuntimeWorkReceiptTable(session, client);
1938
+ } catch (ensureError) {
1939
+ if (!isPostgresPermissionDeniedError(ensureError)) {
1940
+ throw ensureError;
1941
+ }
1942
+ runtimeWorkReceiptEnsureCache.delete(
1943
+ runtimeWorkReceiptEnsureCacheKey(session),
1944
+ );
1945
+ await repairRuntimeStorageGrants(context, {
1946
+ playName: context.playName?.trim() || session.playName,
1947
+ });
1948
+ }
1620
1949
  } else if (isPostgresPermissionDeniedError(error)) {
1621
1950
  await repairRuntimeStorageGrants(context, {
1622
1951
  playName: context.playName?.trim() || session.playName,
@@ -1715,6 +2044,10 @@ function isSystemSheetColumn(columnName: string): boolean {
1715
2044
  case '_stage':
1716
2045
  case '_provider':
1717
2046
  case '_input_index':
2047
+ case '_attempt_id':
2048
+ case '_attempt_owner_run_id':
2049
+ case '_attempt_expires_at':
2050
+ case '_attempt_seq':
1718
2051
  case '_created_at':
1719
2052
  case '_updated_at':
1720
2053
  case '_version':
@@ -1732,6 +2065,64 @@ function isSystemSheetColumn(columnName: string): boolean {
1732
2065
  }
1733
2066
  }
1734
2067
 
2068
+ function normalizeRuntimeSheetAttemptId(
2069
+ value: string | null | undefined,
2070
+ ): string {
2071
+ const trimmed = value?.trim();
2072
+ return trimmed || `sheet-attempt:${randomUUID()}`;
2073
+ }
2074
+
2075
+ function normalizeRuntimeSheetAttemptOwnerRunId(input: {
2076
+ attemptOwnerRunId?: string | null;
2077
+ runId: string;
2078
+ }): string {
2079
+ return input.attemptOwnerRunId?.trim() || input.runId;
2080
+ }
2081
+
2082
+ function normalizeRuntimeRunAttempt(value: number | null | undefined): number {
2083
+ if (typeof value !== 'number' || !Number.isFinite(value)) return 0;
2084
+ return Math.max(0, Math.floor(value));
2085
+ }
2086
+
2087
+ function normalizeRuntimeLeaseTtlMs(
2088
+ value: number | null | undefined,
2089
+ fallback: number,
2090
+ ): number {
2091
+ if (
2092
+ typeof value !== 'number' ||
2093
+ !Number.isFinite(value) ||
2094
+ !Number.isInteger(value) ||
2095
+ value <= 0
2096
+ ) {
2097
+ return fallback;
2098
+ }
2099
+ return value;
2100
+ }
2101
+
2102
+ async function mintRuntimeSheetAttemptExpiresAt(
2103
+ client: RuntimeQueryClient,
2104
+ leaseTtlMs?: number | null,
2105
+ ): Promise<string> {
2106
+ const ttlMs = normalizeRuntimeLeaseTtlMs(
2107
+ leaseTtlMs,
2108
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
2109
+ );
2110
+ const { rows } = await client.query<{ attempt_expires_at: Date | string }>(
2111
+ `
2112
+ SELECT now() + ($1::double precision * interval '1 millisecond') AS attempt_expires_at
2113
+ `,
2114
+ [ttlMs],
2115
+ );
2116
+ const value = rows[0]?.attempt_expires_at;
2117
+ if (value instanceof Date) {
2118
+ return value.toISOString();
2119
+ }
2120
+ if (typeof value === 'string' && value.trim()) {
2121
+ return new Date(value).toISOString();
2122
+ }
2123
+ throw new Error('Runtime sheet attempt expiry mint did not return a value.');
2124
+ }
2125
+
1735
2126
  function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
1736
2127
  if (value && typeof value === 'object' && !Array.isArray(value)) {
1737
2128
  return value as Record<string, unknown>;
@@ -2228,6 +2619,10 @@ async function prepareRuntimeSheetDatasetRows(
2228
2619
  runId: string;
2229
2620
  normalizedPlayName: string;
2230
2621
  normalizedTableNamespace: string;
2622
+ attemptId: string;
2623
+ attemptOwnerRunId: string;
2624
+ attemptExpiresAt: string;
2625
+ attemptSeq: number;
2231
2626
  physicalInsertColumnsSql: string;
2232
2627
  physicalInsertValuesSql: string;
2233
2628
  physicalRefreshSetSql: string;
@@ -2235,9 +2630,12 @@ async function prepareRuntimeSheetDatasetRows(
2235
2630
  outputPhysicalColumns: PhysicalSheetColumnProjection[];
2236
2631
  force?: boolean;
2237
2632
  },
2238
- ): Promise<{ inserted: number; pendingKeys: string[] }> {
2633
+ ): Promise<{
2634
+ inserted: number;
2635
+ rowDispositions: RuntimeSheetPreparedRowDisposition[];
2636
+ }> {
2239
2637
  let inserted = 0;
2240
- const pendingKeys: string[] = [];
2638
+ const rowDispositions: RuntimeSheetPreparedRowDisposition[] = [];
2241
2639
  for (const chunk of input.chunks) {
2242
2640
  const chunkKeys = chunk.map((entry) => entry.key);
2243
2641
  const chunkPayloads = chunk.map((entry) =>
@@ -2252,6 +2650,41 @@ async function prepareRuntimeSheetDatasetRows(
2252
2650
  'target',
2253
2651
  input.outputPhysicalColumns,
2254
2652
  );
2653
+ const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
2654
+ 'target',
2655
+ '$7::text',
2656
+ '$8::text',
2657
+ '$9::timestamptz',
2658
+ '$10::integer',
2659
+ );
2660
+ const existingAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
2661
+ 'existing',
2662
+ '$7::text',
2663
+ '$8::text',
2664
+ '$9::timestamptz',
2665
+ '$10::integer',
2666
+ );
2667
+ const targetNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
2668
+ 'target',
2669
+ '$9::timestamptz',
2670
+ '$10::integer',
2671
+ '$7::text',
2672
+ '$8::text',
2673
+ );
2674
+ const existingNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
2675
+ 'existing',
2676
+ '$9::timestamptz',
2677
+ '$10::integer',
2678
+ '$7::text',
2679
+ '$8::text',
2680
+ );
2681
+ const existingWritableEnrichedAttemptSql =
2682
+ writableEnrichedRuntimeSheetAttemptSql(
2683
+ 'existing',
2684
+ '$8::text',
2685
+ '$9::timestamptz',
2686
+ '$10',
2687
+ );
2255
2688
  const { rows } = await client.query(
2256
2689
  `
2257
2690
  WITH input_rows AS (
@@ -2273,19 +2706,38 @@ async function prepareRuntimeSheetDatasetRows(
2273
2706
  WHERE target._key = input_rows._key
2274
2707
  AND target._status <> 'stale'
2275
2708
  AND (
2276
- $7::boolean
2709
+ NOT (${targetNewerTerminalRowSql})
2710
+ OR (
2711
+ $11::boolean
2712
+ AND target._status = 'enriched'
2713
+ AND COALESCE(target._attempt_seq, 0) <= $10::integer
2714
+ )
2715
+ )
2716
+ AND (
2717
+ ${targetAttemptFenceSql}
2718
+ OR (
2719
+ $11::boolean
2720
+ AND target._status = 'enriched'
2721
+ )
2722
+ )
2723
+ AND (
2724
+ $11::boolean
2277
2725
  OR target._input_index IS DISTINCT FROM input_rows._input_index
2278
2726
  )
2279
2727
  RETURNING target._key
2280
2728
  ),
2281
2729
  inserted_rows AS (
2282
- INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${input.physicalInsertColumnsSql})
2283
- SELECT _key, 'pending', $4::text, _input_index${input.physicalInsertValuesSql}
2730
+ INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
2731
+ SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
2284
2732
  FROM input_rows
2285
2733
  ON CONFLICT (_key) DO UPDATE SET
2286
2734
  _status = 'pending',
2287
2735
  _run_id = EXCLUDED._run_id,
2288
2736
  _input_index = EXCLUDED._input_index,
2737
+ _attempt_id = EXCLUDED._attempt_id,
2738
+ _attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
2739
+ _attempt_expires_at = EXCLUDED._attempt_expires_at,
2740
+ _attempt_seq = EXCLUDED._attempt_seq,
2289
2741
  _updated_at = now(),
2290
2742
  _version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
2291
2743
  WHERE ${sheetTable(session)}._status = 'stale'
@@ -2296,26 +2748,42 @@ async function prepareRuntimeSheetDatasetRows(
2296
2748
  SET _status = 'pending',
2297
2749
  _run_id = $4::text,
2298
2750
  _input_index = input_rows._input_index,
2751
+ _attempt_id = $7::text,
2752
+ _attempt_owner_run_id = $8::text,
2753
+ _attempt_expires_at = $9::timestamptz,
2754
+ _attempt_seq = $10::integer,
2299
2755
  _updated_at = now(),
2300
2756
  _version = ${nextRuntimeSheetVersionExpression(session)}
2301
2757
  FROM input_rows
2302
2758
  WHERE target._key = input_rows._key
2303
2759
  AND target._status = 'enriched'
2304
2760
  AND (${targetMissingOutputSql})
2761
+ AND COALESCE(target._attempt_seq, 0) <= $10::integer
2305
2762
  RETURNING target._key
2306
2763
  ),
2307
2764
  claimed_existing_rows AS (
2308
2765
  UPDATE ${sheetTable(session)} AS target
2309
2766
  SET _run_id = $4::text,
2310
2767
  _input_index = input_rows._input_index,
2768
+ _attempt_id = $7::text,
2769
+ _attempt_owner_run_id = $8::text,
2770
+ _attempt_expires_at = $9::timestamptz,
2771
+ _attempt_seq = $10::integer,
2311
2772
  _updated_at = now(),
2312
2773
  _version = ${nextRuntimeSheetVersionExpression(session)}
2313
2774
  FROM input_rows
2314
2775
  WHERE target._key = input_rows._key
2315
2776
  AND target._status IN ('pending', 'running', 'failed')
2777
+ AND NOT (${targetNewerTerminalRowSql})
2778
+ AND (
2779
+ target._status = 'failed'
2780
+ OR ${targetAttemptFenceSql}
2781
+ )
2316
2782
  AND (
2317
2783
  target._run_id IS DISTINCT FROM $4::text
2318
2784
  OR target._input_index IS DISTINCT FROM input_rows._input_index
2785
+ OR target._attempt_id IS DISTINCT FROM $7::text
2786
+ OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $10::integer
2319
2787
  )
2320
2788
  RETURNING target._key
2321
2789
  ),
@@ -2329,15 +2797,61 @@ async function prepareRuntimeSheetDatasetRows(
2329
2797
  SELECT _key
2330
2798
  FROM claimed_existing_rows
2331
2799
  UNION
2800
+ SELECT _key
2801
+ FROM existing_rows
2802
+ WHERE $11::boolean
2803
+ UNION
2332
2804
  SELECT existing._key
2333
2805
  FROM ${sheetTable(session)} AS existing
2334
2806
  JOIN input_rows ON input_rows._key = existing._key
2335
- WHERE existing._status IN ('pending', 'running', 'failed')
2807
+ WHERE (
2808
+ existing._status IN ('pending', 'running', 'failed')
2809
+ AND NOT (${existingNewerTerminalRowSql})
2810
+ AND (
2811
+ existing._status = 'failed'
2812
+ OR ${existingAttemptFenceSql}
2813
+ )
2814
+ )
2336
2815
  OR (
2337
2816
  existing._status = 'enriched'
2338
2817
  AND (${existingMissingOutputSql})
2818
+ AND (
2819
+ ${existingAttemptFenceSql}
2820
+ OR ${existingWritableEnrichedAttemptSql}
2821
+ )
2339
2822
  )
2340
2823
  ),
2824
+ completed_rows AS (
2825
+ SELECT existing._key
2826
+ FROM ${sheetTable(session)} AS existing
2827
+ JOIN input_rows ON input_rows._key = existing._key
2828
+ WHERE existing._status = 'enriched'
2829
+ AND NOT ($11::boolean)
2830
+ AND NOT (${existingMissingOutputSql})
2831
+ AND NOT EXISTS (
2832
+ SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
2833
+ )
2834
+ ),
2835
+ blocked_rows AS (
2836
+ SELECT input_rows._key
2837
+ FROM input_rows
2838
+ WHERE NOT EXISTS (
2839
+ SELECT 1 FROM pending_rows WHERE pending_rows._key = input_rows._key
2840
+ )
2841
+ AND NOT EXISTS (
2842
+ SELECT 1 FROM completed_rows WHERE completed_rows._key = input_rows._key
2843
+ )
2844
+ ),
2845
+ row_dispositions AS (
2846
+ SELECT _key, 'pending'::text AS disposition
2847
+ FROM pending_rows
2848
+ UNION ALL
2849
+ SELECT _key, 'completed'::text AS disposition
2850
+ FROM completed_rows
2851
+ UNION ALL
2852
+ SELECT _key, 'blocked'::text AS disposition
2853
+ FROM blocked_rows
2854
+ ),
2341
2855
  inserted_count_cte AS (
2342
2856
  SELECT count(*)::bigint AS c FROM inserted_rows
2343
2857
  ),
@@ -2377,7 +2891,13 @@ async function prepareRuntimeSheetDatasetRows(
2377
2891
  )
2378
2892
  SELECT
2379
2893
  (SELECT c::int FROM inserted_count_cte) AS inserted_count,
2380
- coalesce((SELECT array_agg(_key) FROM pending_rows), '{}'::text[]) AS pending_keys,
2894
+ coalesce(
2895
+ (
2896
+ SELECT jsonb_agg(jsonb_build_object('key', _key, 'disposition', disposition))
2897
+ FROM row_dispositions
2898
+ ),
2899
+ '[]'::jsonb
2900
+ ) AS row_dispositions,
2381
2901
  (SELECT count(*)::int FROM existing_rows) AS reordered_count,
2382
2902
  (SELECT c::int FROM missing_output_count_cte) AS missing_output_count
2383
2903
  `,
@@ -2388,15 +2908,30 @@ async function prepareRuntimeSheetDatasetRows(
2388
2908
  input.runId,
2389
2909
  input.normalizedPlayName,
2390
2910
  input.normalizedTableNamespace,
2911
+ input.attemptId,
2912
+ input.attemptOwnerRunId,
2913
+ input.attemptExpiresAt,
2914
+ input.attemptSeq,
2391
2915
  input.force === true,
2392
2916
  ],
2393
2917
  );
2394
2918
  inserted += Number(rows[0]?.inserted_count ?? 0);
2395
- if (Array.isArray(rows[0]?.pending_keys)) {
2396
- pendingKeys.push(...(rows[0]?.pending_keys as string[]));
2919
+ if (Array.isArray(rows[0]?.row_dispositions)) {
2920
+ for (const row of rows[0]
2921
+ ?.row_dispositions as RuntimeSheetPreparedRowDisposition[]) {
2922
+ if (
2923
+ row &&
2924
+ typeof row.key === 'string' &&
2925
+ (row.disposition === 'pending' ||
2926
+ row.disposition === 'completed' ||
2927
+ row.disposition === 'blocked')
2928
+ ) {
2929
+ rowDispositions.push(row);
2930
+ }
2931
+ }
2397
2932
  }
2398
2933
  }
2399
- return { inserted, pendingKeys };
2934
+ return { inserted, rowDispositions };
2400
2935
  }
2401
2936
 
2402
2937
  async function buildRuntimeSheetDatasetStartResult(
@@ -2410,8 +2945,12 @@ async function buildRuntimeSheetDatasetStartResult(
2410
2945
  normalizedPlayName: string;
2411
2946
  normalizedTableNamespace: string;
2412
2947
  runId: string;
2948
+ attemptId: string;
2949
+ attemptOwnerRunId: string;
2950
+ attemptExpiresAt: string;
2951
+ attemptSeq: number;
2413
2952
  inserted: number;
2414
- pendingKeys: string[];
2953
+ rowDispositions: RuntimeSheetPreparedRowDisposition[];
2415
2954
  force?: boolean;
2416
2955
  timings?: RuntimeSheetTiming[];
2417
2956
  },
@@ -2421,19 +2960,20 @@ async function buildRuntimeSheetDatasetStartResult(
2421
2960
  ? [column.field]
2422
2961
  : [],
2423
2962
  );
2424
- const buildFreshPendingRows = () => {
2425
- return input.rowEntries.map((entry) => {
2426
- const row: Record<string, unknown> = {
2427
- ...sanitizePostgresJsonValue(entry.row),
2428
- __deeplineRowKey: entry.key,
2429
- };
2430
- for (const field of datasetFields) {
2431
- if (!Object.prototype.hasOwnProperty.call(row, field)) {
2432
- row[field] = null;
2433
- }
2963
+ const buildFreshRow = (entry: RuntimeDatasetRowEntry) => {
2964
+ const row: Record<string, unknown> = {
2965
+ ...sanitizePostgresJsonValue(entry.row),
2966
+ __deeplineRowKey: entry.key,
2967
+ };
2968
+ for (const field of datasetFields) {
2969
+ if (!Object.prototype.hasOwnProperty.call(row, field)) {
2970
+ row[field] = null;
2434
2971
  }
2435
- return row;
2436
- });
2972
+ }
2973
+ return row;
2974
+ };
2975
+ const buildFreshPendingRows = (entries = input.rowEntries) => {
2976
+ return entries.map(buildFreshRow);
2437
2977
  };
2438
2978
 
2439
2979
  if (input.inserted === input.rowEntries.length) {
@@ -2442,18 +2982,31 @@ async function buildRuntimeSheetDatasetStartResult(
2442
2982
  skipped: input.sourceRowsLength - input.rowEntries.length,
2443
2983
  pendingRows: buildFreshPendingRows(),
2444
2984
  completedRows: [],
2985
+ blockedRows: [],
2445
2986
  tableNamespace: input.tableNamespace,
2446
2987
  };
2447
2988
  }
2448
2989
 
2449
- const pendingKeys = new Set(input.pendingKeys);
2990
+ const dispositionsByKey = new Map(
2991
+ input.rowDispositions.map((row) => [row.key, row.disposition]),
2992
+ );
2993
+ const pendingKeys = new Set(
2994
+ input.rowDispositions.flatMap((row) =>
2995
+ row.disposition === 'pending' ? [row.key] : [],
2996
+ ),
2997
+ );
2450
2998
  const startedAt = Date.now();
2451
2999
  await markRuntimeRowsPendingForRecompute(client, session, {
2452
3000
  keys: [...pendingKeys],
2453
3001
  runId: input.runId,
3002
+ attemptId: input.attemptId,
3003
+ attemptOwnerRunId: input.attemptOwnerRunId,
3004
+ attemptExpiresAt: input.attemptExpiresAt,
3005
+ attemptSeq: input.attemptSeq,
2454
3006
  normalizedPlayName: input.normalizedPlayName,
2455
3007
  normalizedTableNamespace: input.normalizedTableNamespace,
2456
3008
  outputFields: input.force === true ? datasetFields : [],
3009
+ force: input.force === true,
2457
3010
  });
2458
3011
  if (pendingKeys.size > 0) {
2459
3012
  input.timings?.push({
@@ -2463,11 +3016,22 @@ async function buildRuntimeSheetDatasetStartResult(
2463
3016
  });
2464
3017
  }
2465
3018
  if (input.force === true) {
3019
+ const pendingRows: Record<string, unknown>[] = [];
3020
+ const blockedRows: Record<string, unknown>[] = [];
3021
+ for (const entry of input.rowEntries) {
3022
+ const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
3023
+ if (disposition === 'pending') {
3024
+ pendingRows.push(buildFreshRow(entry));
3025
+ } else if (disposition === 'blocked') {
3026
+ blockedRows.push(buildFreshRow(entry));
3027
+ }
3028
+ }
2466
3029
  return {
2467
3030
  inserted: input.inserted,
2468
3031
  skipped: input.sourceRowsLength - input.rowEntries.length,
2469
- pendingRows: buildFreshPendingRows(),
3032
+ pendingRows,
2470
3033
  completedRows: [],
3034
+ blockedRows,
2471
3035
  tableNamespace: input.tableNamespace,
2472
3036
  };
2473
3037
  }
@@ -2498,14 +3062,25 @@ async function buildRuntimeSheetDatasetStartResult(
2498
3062
  };
2499
3063
  };
2500
3064
  const pendingRows: Record<string, unknown>[] = [];
3065
+ const completedRows: Record<string, unknown>[] = [];
3066
+ const blockedRows: Record<string, unknown>[] = [];
2501
3067
  for (const entry of input.rowEntries) {
2502
- pendingRows.push(buildMergedRow(entry));
3068
+ const merged = buildMergedRow(entry);
3069
+ const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
3070
+ if (disposition === 'pending') {
3071
+ pendingRows.push(merged);
3072
+ } else if (disposition === 'completed') {
3073
+ completedRows.push(merged);
3074
+ } else {
3075
+ blockedRows.push(merged);
3076
+ }
2503
3077
  }
2504
3078
  return {
2505
3079
  inserted: input.inserted,
2506
3080
  skipped: input.sourceRowsLength - input.rowEntries.length,
2507
3081
  pendingRows,
2508
- completedRows: [],
3082
+ completedRows,
3083
+ blockedRows,
2509
3084
  tableNamespace: input.tableNamespace,
2510
3085
  };
2511
3086
  }
@@ -2516,12 +3091,31 @@ async function markRuntimeRowsPendingForRecompute(
2516
3091
  input: {
2517
3092
  keys: string[];
2518
3093
  runId: string;
3094
+ attemptId: string;
3095
+ attemptOwnerRunId: string;
3096
+ attemptExpiresAt: string;
3097
+ attemptSeq: number;
2519
3098
  normalizedPlayName: string;
2520
3099
  normalizedTableNamespace: string;
2521
3100
  outputFields?: string[];
3101
+ force?: boolean;
2522
3102
  },
2523
3103
  ): Promise<void> {
2524
3104
  if (input.keys.length === 0) return;
3105
+ const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
3106
+ 'target',
3107
+ '$5::text',
3108
+ '$6::text',
3109
+ '$7::timestamptz',
3110
+ '$8::integer',
3111
+ );
3112
+ const targetWritableEnrichedAttemptSql =
3113
+ writableEnrichedRuntimeSheetAttemptSql(
3114
+ 'target',
3115
+ '$6::text',
3116
+ '$7::timestamptz',
3117
+ '$8',
3118
+ );
2525
3119
  const outputFields = [...new Set(input.outputFields ?? [])];
2526
3120
  await client.query(
2527
3121
  `WITH target_rows AS (
@@ -2534,13 +3128,31 @@ async function markRuntimeRowsPendingForRecompute(
2534
3128
  UPDATE ${sheetTable(session)} AS target
2535
3129
  SET _status = 'pending',
2536
3130
  _run_id = $2::text,
3131
+ _attempt_id = $5::text,
3132
+ _attempt_owner_run_id = $6::text,
3133
+ _attempt_expires_at = $7::timestamptz,
3134
+ _attempt_seq = $8::integer,
2537
3135
  _updated_at = now(),
2538
3136
  _version = ${nextRuntimeSheetVersionExpression(session)}
2539
3137
  FROM target_rows
2540
3138
  WHERE target._key = target_rows._key
3139
+ AND (
3140
+ ${targetAttemptFenceSql}
3141
+ OR (
3142
+ $10::boolean
3143
+ AND target._status = 'enriched'
3144
+ AND COALESCE(target._attempt_seq, 0) <= $8::integer
3145
+ )
3146
+ OR (
3147
+ target._status = 'enriched'
3148
+ AND ${targetWritableEnrichedAttemptSql}
3149
+ )
3150
+ )
2541
3151
  AND (
2542
3152
  target._status <> 'pending'
2543
3153
  OR target._run_id IS DISTINCT FROM $2::text
3154
+ OR target._attempt_id IS DISTINCT FROM $5::text
3155
+ OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $8::integer
2544
3156
  )
2545
3157
  RETURNING target_rows._status AS previous_status,
2546
3158
  target_rows._cell_meta AS previous_cell_meta
@@ -2561,7 +3173,7 @@ async function markRuntimeRowsPendingForRecompute(
2561
3173
  WHERE previous_cell_meta -> field_values.field ->> 'status' = 'failed'
2562
3174
  )::int AS failed_to_pending
2563
3175
  FROM updated
2564
- JOIN unnest($5::text[]) AS field_values(field) ON true
3176
+ JOIN unnest($9::text[]) AS field_values(field) ON true
2565
3177
  GROUP BY field_values.field
2566
3178
  ),
2567
3179
  summary_delta AS (
@@ -2619,7 +3231,12 @@ async function markRuntimeRowsPendingForRecompute(
2619
3231
  input.runId,
2620
3232
  input.normalizedPlayName,
2621
3233
  input.normalizedTableNamespace,
3234
+ input.attemptId,
3235
+ input.attemptOwnerRunId,
3236
+ input.attemptExpiresAt,
3237
+ input.attemptSeq,
2622
3238
  outputFields,
3239
+ input.force === true,
2623
3240
  ],
2624
3241
  );
2625
3242
  }
@@ -2657,6 +3274,41 @@ async function getRuntimeWorkReceiptSession(
2657
3274
  return session;
2658
3275
  }
2659
3276
 
3277
+ /**
3278
+ * Work receipts for a play share a single table, so run-scoped operations (the
3279
+ * run-fatal lease release) need a session without a specific key to scope by.
3280
+ * This mirrors `getRuntimeWorkReceiptSession` minus the per-key scope check.
3281
+ */
3282
+ async function getRuntimeWorkReceiptSessionForRun(
3283
+ context: RuntimeApiContext,
3284
+ input: {
3285
+ playName: string;
3286
+ },
3287
+ ): Promise<RuntimePostgresSession> {
3288
+ const playName = context.playName?.trim() || input.playName;
3289
+ if (!playName) {
3290
+ throw new Error('Runtime work receipts require a playName.');
3291
+ }
3292
+ const runtimeContext = {
3293
+ ...context,
3294
+ playName,
3295
+ };
3296
+ const preloaded = findPreloadedRuntimeDbSession(runtimeContext, {
3297
+ tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
3298
+ logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
3299
+ operations: ['rows.read', 'rows.upsert'],
3300
+ });
3301
+ return requireRuntimePostgresSession(
3302
+ preloaded
3303
+ ? await unwrapRuntimeDbSession(runtimeContext, preloaded)
3304
+ : await getRuntimeDbSession(runtimeContext, {
3305
+ tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
3306
+ logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
3307
+ operations: ['rows.read', 'rows.upsert'],
3308
+ }),
3309
+ );
3310
+ }
3311
+
2660
3312
  async function getRuntimeWorkReceiptSessionForKeys(
2661
3313
  context: RuntimeApiContext,
2662
3314
  input: {
@@ -2684,7 +3336,17 @@ async function readRuntimeWorkReceipt(
2684
3336
  key: string,
2685
3337
  ): Promise<WorkReceipt | null> {
2686
3338
  const { rows } = await client.query<Record<string, unknown>>(
2687
- `SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3339
+ `SELECT convert_from(k, 'UTF8') AS k,
3340
+ status,
3341
+ output,
3342
+ error,
3343
+ failure_kind,
3344
+ run_id,
3345
+ lease_id,
3346
+ lease_owner_run_id,
3347
+ lease_owner_attempt,
3348
+ lease_expires_at,
3349
+ updated_at
2688
3350
  FROM ${workReceiptTable(session)}
2689
3351
  WHERE k = decode($1, 'hex')`,
2690
3352
  [workReceiptKeyHex(key)],
@@ -2714,7 +3376,7 @@ export async function getRuntimeWorkReceipts(
2714
3376
  playName: string;
2715
3377
  keys: string[];
2716
3378
  },
2717
- ): Promise<WorkReceipt[]> {
3379
+ ): Promise<Array<WorkReceipt | null>> {
2718
3380
  const keys = [
2719
3381
  ...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
2720
3382
  ];
@@ -2737,16 +3399,23 @@ export async function getRuntimeWorkReceipts(
2737
3399
  receipts.status,
2738
3400
  receipts.output,
2739
3401
  receipts.error,
3402
+ receipts.failure_kind,
2740
3403
  receipts.run_id,
3404
+ receipts.lease_id,
3405
+ receipts.lease_owner_run_id,
3406
+ receipts.lease_owner_attempt,
3407
+ receipts.lease_expires_at,
2741
3408
  receipts.updated_at
2742
3409
  FROM input_keys
2743
- JOIN ${workReceiptTable(session)} AS receipts
3410
+ LEFT JOIN ${workReceiptTable(session)} AS receipts
2744
3411
  ON receipts.k = decode(input_keys.key_hex, 'hex')
2745
3412
  ORDER BY input_keys.ord
2746
3413
  `,
2747
3414
  [keys.map(workReceiptKeyHex)],
2748
3415
  );
2749
- return rows.map(mapRuntimeWorkReceiptRow);
3416
+ return rows.map((row) =>
3417
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
3418
+ );
2750
3419
  },
2751
3420
  );
2752
3421
  }
@@ -2757,8 +3426,12 @@ export async function claimRuntimeWorkReceipt(
2757
3426
  playName: string;
2758
3427
  runId: string;
2759
3428
  key: string;
3429
+ runAttempt?: number | null;
3430
+ leaseAware?: boolean;
2760
3431
  reclaimRunning?: boolean;
2761
3432
  forceRefresh?: boolean;
3433
+ forceFailedRefresh?: boolean;
3434
+ leaseTtlMs?: number | null;
2762
3435
  },
2763
3436
  ): Promise<WorkReceiptClaim> {
2764
3437
  const session = await getRuntimeWorkReceiptSession(context, {
@@ -2769,36 +3442,89 @@ export async function claimRuntimeWorkReceipt(
2769
3442
  context,
2770
3443
  session,
2771
3444
  async (client) => {
2772
- const claimableStatuses = input.forceRefresh
2773
- ? [
2774
- RECEIPT_STATUS_PENDING,
2775
- RECEIPT_STATUS_FAILED,
2776
- RECEIPT_STATUS_RUNNING,
2777
- RECEIPT_STATUS_COMPLETED,
2778
- RECEIPT_STATUS_SKIPPED,
2779
- ]
2780
- : input.reclaimRunning
2781
- ? [
2782
- RECEIPT_STATUS_PENDING,
2783
- RECEIPT_STATUS_FAILED,
2784
- RECEIPT_STATUS_RUNNING,
2785
- ]
2786
- : [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
3445
+ const leaseId =
3446
+ input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null;
3447
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3448
+ const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
3449
+ input.leaseTtlMs,
3450
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
3451
+ );
3452
+ const claimableStatuses = workReceiptClaimableStatusCodes({
3453
+ forceRefresh: input.forceRefresh,
3454
+ forceFailedRefresh: input.forceFailedRefresh,
3455
+ reclaimRunning: input.reclaimRunning,
3456
+ });
2787
3457
  const { rows } = await client.query<Record<string, unknown>>(
2788
3458
  `
2789
3459
  WITH claimed AS (
2790
- INSERT INTO ${workReceiptTable(session)} (k, status, run_id, updated_at)
2791
- VALUES (decode($1, 'hex'), $2::smallint, $3, now())
3460
+ INSERT INTO ${workReceiptTable(session)} (
3461
+ k,
3462
+ status,
3463
+ run_id,
3464
+ lease_id,
3465
+ lease_owner_run_id,
3466
+ lease_owner_attempt,
3467
+ lease_expires_at,
3468
+ updated_at
3469
+ )
3470
+ VALUES (
3471
+ decode($1, 'hex'),
3472
+ $2::smallint,
3473
+ $3,
3474
+ $6,
3475
+ CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
3476
+ CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
3477
+ CASE
3478
+ WHEN $6::text IS NULL THEN NULL
3479
+ ELSE now() + ($7::double precision * interval '1 millisecond')
3480
+ END,
3481
+ now()
3482
+ )
2792
3483
  ON CONFLICT (k) DO UPDATE
2793
3484
  SET status = $4::smallint,
2794
3485
  output = NULL,
3486
+ failure_kind = 0,
2795
3487
  run_id = $3,
3488
+ lease_id = $6,
3489
+ lease_owner_run_id = CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
3490
+ lease_owner_attempt = CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
3491
+ lease_expires_at = CASE
3492
+ WHEN $6::text IS NULL THEN NULL
3493
+ ELSE now() + ($7::double precision * interval '1 millisecond')
3494
+ END,
2796
3495
  error = NULL,
2797
3496
  updated_at = now()
2798
- WHERE ${workReceiptTable(session)}.status = ANY($5::smallint[])
2799
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3497
+ WHERE ${workReceiptClaimConflictPredicateSql({
3498
+ receiptTable: workReceiptTable(session),
3499
+ claimableStatusesSql: '$5',
3500
+ claimantRunIdSql: '$3',
3501
+ claimantRunAttemptSql: '$9',
3502
+ forceRefreshSql: '$8',
3503
+ forceFailedRefreshSql: '$10',
3504
+ })}
3505
+ RETURNING convert_from(k, 'UTF8') AS k,
3506
+ status,
3507
+ output,
3508
+ error,
3509
+ failure_kind,
3510
+ run_id,
3511
+ lease_id,
3512
+ lease_owner_run_id,
3513
+ lease_owner_attempt,
3514
+ lease_expires_at,
3515
+ updated_at
2800
3516
  )
2801
- SELECT k, status, output, error, run_id, updated_at
3517
+ SELECT k,
3518
+ status,
3519
+ output,
3520
+ error,
3521
+ failure_kind,
3522
+ run_id,
3523
+ lease_id,
3524
+ lease_owner_run_id,
3525
+ lease_owner_attempt,
3526
+ lease_expires_at,
3527
+ updated_at
2802
3528
  FROM claimed
2803
3529
  `,
2804
3530
  [
@@ -2807,6 +3533,11 @@ export async function claimRuntimeWorkReceipt(
2807
3533
  input.runId,
2808
3534
  RECEIPT_STATUS_RUNNING,
2809
3535
  claimableStatuses,
3536
+ leaseId,
3537
+ leaseTtlMs,
3538
+ input.forceRefresh === true,
3539
+ runAttempt,
3540
+ input.forceFailedRefresh === true,
2810
3541
  ],
2811
3542
  );
2812
3543
  const claimed = rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -2815,14 +3546,19 @@ export async function claimRuntimeWorkReceipt(
2815
3546
  }
2816
3547
 
2817
3548
  const latest = await readRuntimeWorkReceipt(client, session, input.key);
2818
- if (latest && isReusableWorkReceipt(latest)) {
3549
+ if (
3550
+ latest &&
3551
+ input.forceRefresh !== true &&
3552
+ isReusableWorkReceipt(latest)
3553
+ ) {
2819
3554
  return { disposition: 'reused', receipt: latest };
2820
3555
  }
2821
- if (latest?.status === 'running') {
2822
- return { disposition: 'running', receipt: latest };
2823
- }
2824
- if (latest?.status === 'failed') {
2825
- return { disposition: 'failed', receipt: latest };
3556
+ if (latest) {
3557
+ return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
3558
+ receipt: latest,
3559
+ claimantRunId: input.runId,
3560
+ claimantRunAttempt: runAttempt,
3561
+ });
2826
3562
  }
2827
3563
  throw new Error(
2828
3564
  `Runtime receipt ${input.key} claim did not return execution ownership.`,
@@ -2831,14 +3567,48 @@ export async function claimRuntimeWorkReceipt(
2831
3567
  );
2832
3568
  }
2833
3569
 
3570
+ function runtimeWorkReceiptClaimDispositionForBlockedReceipt(input: {
3571
+ receipt: WorkReceipt;
3572
+ claimantRunId: string;
3573
+ claimantRunAttempt?: number | null;
3574
+ }): Exclude<WorkReceiptClaim, { disposition: 'claimed' | 'reused' }> {
3575
+ const receipt = input.receipt;
3576
+ if (receipt.status === 'failed') {
3577
+ return { disposition: 'failed', receipt };
3578
+ }
3579
+ if (receipt.status === 'pending' || receipt.status === 'running') {
3580
+ const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
3581
+ const leaseExpiresAtMs =
3582
+ typeof receipt.leaseExpiresAt === 'string'
3583
+ ? Date.parse(receipt.leaseExpiresAt)
3584
+ : NaN;
3585
+ const hasActiveForeignLease =
3586
+ ownerRunId !== null &&
3587
+ ownerRunId !== input.claimantRunId &&
3588
+ typeof receipt.leaseId === 'string' &&
3589
+ receipt.leaseId.length > 0 &&
3590
+ Number.isFinite(leaseExpiresAtMs) &&
3591
+ leaseExpiresAtMs > Date.now();
3592
+ return {
3593
+ disposition: hasActiveForeignLease ? 'blocked_active_lease' : 'running',
3594
+ receipt,
3595
+ };
3596
+ }
3597
+ return { disposition: 'running', receipt };
3598
+ }
3599
+
2834
3600
  export async function claimRuntimeWorkReceipts(
2835
3601
  context: RuntimeApiContext,
2836
3602
  input: {
2837
3603
  playName: string;
2838
3604
  runId: string;
2839
3605
  keys: string[];
3606
+ runAttempt?: number | null;
3607
+ leaseAware?: boolean;
2840
3608
  reclaimRunning?: boolean;
2841
3609
  forceRefresh?: boolean;
3610
+ forceFailedRefresh?: boolean;
3611
+ leaseTtlMs?: number | null;
2842
3612
  },
2843
3613
  ): Promise<WorkReceiptClaim[]> {
2844
3614
  const keys = [
@@ -2854,46 +3624,104 @@ export async function claimRuntimeWorkReceipts(
2854
3624
  session,
2855
3625
  async (client) => {
2856
3626
  const keyHexes = keys.map(workReceiptKeyHex);
2857
- const reclaimStatuses = input.forceRefresh
2858
- ? [
2859
- RECEIPT_STATUS_PENDING,
2860
- RECEIPT_STATUS_FAILED,
2861
- RECEIPT_STATUS_RUNNING,
2862
- RECEIPT_STATUS_COMPLETED,
2863
- RECEIPT_STATUS_SKIPPED,
2864
- ]
2865
- : input.reclaimRunning
2866
- ? [
2867
- RECEIPT_STATUS_PENDING,
2868
- RECEIPT_STATUS_FAILED,
2869
- RECEIPT_STATUS_RUNNING,
2870
- ]
2871
- : [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
3627
+ const leaseIds = keys.map(() =>
3628
+ input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
3629
+ );
3630
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3631
+ const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
3632
+ input.leaseTtlMs,
3633
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
3634
+ );
3635
+ const reclaimStatuses = workReceiptClaimableStatusCodes({
3636
+ forceRefresh: input.forceRefresh,
3637
+ forceFailedRefresh: input.forceFailedRefresh,
3638
+ reclaimRunning: input.reclaimRunning,
3639
+ });
2872
3640
  const { rows } = await client.query<Record<string, unknown>>(
2873
3641
  `
2874
3642
  WITH input_keys AS (
2875
- SELECT key_values.key_hex, key_values.ord
3643
+ SELECT key_values.key_hex, lease_values.lease_id, key_values.ord
2876
3644
  FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
3645
+ JOIN unnest($2::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
3646
+ ON lease_values.ord = key_values.ord
2877
3647
  ),
2878
3648
  claimed AS (
2879
- INSERT INTO ${workReceiptTable(session)} (k, status, run_id, updated_at)
2880
- SELECT decode(input_keys.key_hex, 'hex'), $2::smallint, $3::text, now()
3649
+ INSERT INTO ${workReceiptTable(session)} (
3650
+ k,
3651
+ status,
3652
+ run_id,
3653
+ lease_id,
3654
+ lease_owner_run_id,
3655
+ lease_owner_attempt,
3656
+ lease_expires_at,
3657
+ updated_at
3658
+ )
3659
+ SELECT decode(input_keys.key_hex, 'hex'),
3660
+ $3::smallint,
3661
+ $4::text,
3662
+ input_keys.lease_id,
3663
+ CASE
3664
+ WHEN input_keys.lease_id IS NULL THEN NULL
3665
+ ELSE $4::text
3666
+ END,
3667
+ CASE
3668
+ WHEN input_keys.lease_id IS NULL THEN NULL
3669
+ ELSE $9::integer
3670
+ END,
3671
+ CASE
3672
+ WHEN input_keys.lease_id IS NULL THEN NULL
3673
+ ELSE now() + ($7::double precision * interval '1 millisecond')
3674
+ END,
3675
+ now()
2881
3676
  FROM input_keys
2882
3677
  ON CONFLICT (k) DO UPDATE
2883
- SET status = $4::smallint,
3678
+ SET status = $5::smallint,
2884
3679
  output = NULL,
2885
- run_id = $3::text,
3680
+ failure_kind = 0,
3681
+ run_id = $4::text,
3682
+ lease_id = EXCLUDED.lease_id,
3683
+ lease_owner_run_id = CASE
3684
+ WHEN EXCLUDED.lease_id IS NULL THEN NULL
3685
+ ELSE $4::text
3686
+ END,
3687
+ lease_owner_attempt = CASE
3688
+ WHEN EXCLUDED.lease_id IS NULL THEN NULL
3689
+ ELSE $9::integer
3690
+ END,
3691
+ lease_expires_at = EXCLUDED.lease_expires_at,
2886
3692
  error = NULL,
2887
3693
  updated_at = now()
2888
- WHERE ${workReceiptTable(session)}.status = ANY($5::smallint[])
2889
- RETURNING k, status, output, error, run_id, updated_at
3694
+ WHERE ${workReceiptClaimConflictPredicateSql({
3695
+ receiptTable: workReceiptTable(session),
3696
+ claimableStatusesSql: '$6',
3697
+ claimantRunIdSql: '$4::text',
3698
+ claimantRunAttemptSql: '$9',
3699
+ forceRefreshSql: '$8',
3700
+ forceFailedRefreshSql: '$10',
3701
+ })}
3702
+ RETURNING k,
3703
+ status,
3704
+ output,
3705
+ error,
3706
+ failure_kind,
3707
+ run_id,
3708
+ lease_id,
3709
+ lease_owner_run_id,
3710
+ lease_owner_attempt,
3711
+ lease_expires_at,
3712
+ updated_at
2890
3713
  ),
2891
3714
  latest AS (
2892
3715
  SELECT receipts.k,
2893
3716
  receipts.status,
2894
3717
  receipts.output,
2895
3718
  receipts.error,
3719
+ receipts.failure_kind,
2896
3720
  receipts.run_id,
3721
+ receipts.lease_id,
3722
+ receipts.lease_owner_run_id,
3723
+ receipts.lease_owner_attempt,
3724
+ receipts.lease_expires_at,
2897
3725
  receipts.updated_at,
2898
3726
  input_keys.ord,
2899
3727
  false AS claimed
@@ -2909,21 +3737,43 @@ export async function claimRuntimeWorkReceipts(
2909
3737
  claimed.status,
2910
3738
  claimed.output,
2911
3739
  claimed.error,
3740
+ claimed.failure_kind,
2912
3741
  claimed.run_id,
3742
+ claimed.lease_id,
3743
+ claimed.lease_owner_run_id,
3744
+ claimed.lease_owner_attempt,
3745
+ claimed.lease_expires_at,
2913
3746
  claimed.updated_at,
2914
3747
  input_keys.ord,
2915
3748
  true AS claimed
2916
3749
  FROM claimed
2917
3750
  JOIN input_keys ON claimed.k = decode(input_keys.key_hex, 'hex')
2918
3751
  UNION ALL
2919
- SELECT k, status, output, error, run_id, updated_at, ord, claimed
3752
+ SELECT k,
3753
+ status,
3754
+ output,
3755
+ error,
3756
+ failure_kind,
3757
+ run_id,
3758
+ lease_id,
3759
+ lease_owner_run_id,
3760
+ lease_owner_attempt,
3761
+ lease_expires_at,
3762
+ updated_at,
3763
+ ord,
3764
+ claimed
2920
3765
  FROM latest
2921
3766
  )
2922
3767
  SELECT convert_from(k, 'UTF8') AS k,
2923
3768
  status,
2924
3769
  output,
2925
3770
  error,
3771
+ failure_kind,
2926
3772
  run_id,
3773
+ lease_id,
3774
+ lease_owner_run_id,
3775
+ lease_owner_attempt,
3776
+ lease_expires_at,
2927
3777
  updated_at,
2928
3778
  claimed
2929
3779
  FROM returned
@@ -2931,10 +3781,15 @@ export async function claimRuntimeWorkReceipts(
2931
3781
  `,
2932
3782
  [
2933
3783
  keyHexes,
3784
+ leaseIds,
2934
3785
  RECEIPT_STATUS_RUNNING,
2935
3786
  input.runId,
2936
3787
  RECEIPT_STATUS_RUNNING,
2937
3788
  reclaimStatuses,
3789
+ leaseTtlMs,
3790
+ input.forceRefresh === true,
3791
+ runAttempt,
3792
+ input.forceFailedRefresh === true,
2938
3793
  ],
2939
3794
  );
2940
3795
  return rows.map((row) => {
@@ -2942,16 +3797,14 @@ export async function claimRuntimeWorkReceipts(
2942
3797
  if (row.claimed === true) {
2943
3798
  return { disposition: 'claimed', receipt } satisfies WorkReceiptClaim;
2944
3799
  }
2945
- if (isReusableWorkReceipt(receipt)) {
3800
+ if (input.forceRefresh !== true && isReusableWorkReceipt(receipt)) {
2946
3801
  return { disposition: 'reused', receipt } satisfies WorkReceiptClaim;
2947
3802
  }
2948
- if (receipt.status === 'running') {
2949
- return { disposition: 'running', receipt } satisfies WorkReceiptClaim;
2950
- }
2951
- if (receipt.status === 'failed') {
2952
- return { disposition: 'failed', receipt } satisfies WorkReceiptClaim;
2953
- }
2954
- return { disposition: 'running', receipt } satisfies WorkReceiptClaim;
3803
+ return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
3804
+ receipt,
3805
+ claimantRunId: input.runId,
3806
+ claimantRunAttempt: runAttempt,
3807
+ });
2955
3808
  });
2956
3809
  },
2957
3810
  );
@@ -2963,17 +3816,50 @@ export async function completeRuntimeWorkReceipt(
2963
3816
  playName: string;
2964
3817
  runId: string;
2965
3818
  key: string;
3819
+ runAttempt?: number | null;
3820
+ leaseId?: string | null;
2966
3821
  output: unknown;
2967
3822
  },
2968
3823
  ): Promise<WorkReceipt | null> {
2969
- const session = await getRuntimeWorkReceiptSession(context, {
2970
- playName: input.playName,
2971
- key: input.key,
2972
- });
2973
- return await withRuntimeWorkReceiptClient(
2974
- context,
2975
- session,
2976
- async (client) => {
3824
+ if (
3825
+ consumeDirectRuntimeTestFault({
3826
+ context,
3827
+ runId: input.runId,
3828
+ name: 'receipt_complete_write_fail',
3829
+ })
3830
+ ) {
3831
+ const error = runtimeTestFaultError('receipt_complete_write_fail');
3832
+ const failed = await failRuntimeWorkReceipt(context, {
3833
+ playName: input.playName,
3834
+ runId: input.runId,
3835
+ key: input.key,
3836
+ leaseId: input.leaseId,
3837
+ error: error.message,
3838
+ failureKind: 'repairable',
3839
+ runAttempt: input.runAttempt,
3840
+ });
3841
+ if (!failed) {
3842
+ await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
3843
+ playName: input.playName,
3844
+ runId: input.runId,
3845
+ key: input.key,
3846
+ error: error.message,
3847
+ failureKind: 'repairable',
3848
+ runAttempt: input.runAttempt,
3849
+ });
3850
+ }
3851
+ throw error;
3852
+ }
3853
+ const session = await getRuntimeWorkReceiptSession(context, {
3854
+ playName: input.playName,
3855
+ key: input.key,
3856
+ });
3857
+ return await withRuntimeWorkReceiptClient(
3858
+ context,
3859
+ session,
3860
+ async (client) => {
3861
+ const leaseId = input.leaseId?.trim() || null;
3862
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
2977
3863
  const { rows } = await client.query<Record<string, unknown>>(
2978
3864
  `
2979
3865
  WITH completed AS (
@@ -2981,29 +3867,41 @@ export async function completeRuntimeWorkReceipt(
2981
3867
  SET status = $2::smallint,
2982
3868
  output = $3::jsonb,
2983
3869
  error = NULL,
3870
+ failure_kind = 0,
2984
3871
  run_id = $4,
3872
+ lease_id = NULL,
3873
+ lease_owner_run_id = NULL,
3874
+ lease_owner_attempt = NULL,
3875
+ lease_expires_at = NULL,
2985
3876
  updated_at = now()
2986
3877
  WHERE k = decode($1, 'hex')
2987
- AND status = $5::smallint
2988
- AND run_id = $4
2989
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2990
- ),
2991
- latest AS (
2992
- SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
2993
- FROM ${workReceiptTable(session)}
2994
- WHERE k = decode($1, 'hex')
2995
- AND NOT EXISTS (SELECT 1 FROM completed)
2996
- )
2997
- SELECT k, status, output, error, run_id, updated_at FROM completed
2998
- UNION ALL
2999
- SELECT k, status, output, error, run_id, updated_at FROM latest
3878
+ AND ${workReceiptRunningOwnerPredicateSql({
3879
+ receiptTable: workReceiptTable(session),
3880
+ ownerRunIdSql: '$4',
3881
+ ownerRunAttemptSql: '$6',
3882
+ leaseIdSql: '$5::text',
3883
+ })}
3884
+ RETURNING convert_from(k, 'UTF8') AS k,
3885
+ status,
3886
+ output,
3887
+ error,
3888
+ failure_kind,
3889
+ run_id,
3890
+ lease_id,
3891
+ lease_owner_run_id,
3892
+ lease_owner_attempt,
3893
+ lease_expires_at,
3894
+ updated_at
3895
+ )
3896
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
3000
3897
  `,
3001
3898
  [
3002
3899
  workReceiptKeyHex(input.key),
3003
3900
  RECEIPT_STATUS_COMPLETED,
3004
3901
  input.output === null ? null : stringifyPostgresJson(input.output),
3005
3902
  input.runId,
3006
- RECEIPT_STATUS_RUNNING,
3903
+ leaseId,
3904
+ runAttempt,
3007
3905
  ],
3008
3906
  );
3009
3907
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -3015,11 +3913,63 @@ export async function completeRuntimeWorkReceipts(
3015
3913
  context: RuntimeApiContext,
3016
3914
  input: {
3017
3915
  playName: string;
3018
- receipts: Array<{ runId: string; key: string; output: unknown }>;
3916
+ receipts: Array<{
3917
+ runId: string;
3918
+ key: string;
3919
+ output: unknown;
3920
+ leaseId?: string | null;
3921
+ runAttempt?: number | null;
3922
+ }>;
3019
3923
  },
3020
- ): Promise<WorkReceipt[]> {
3924
+ ): Promise<Array<WorkReceipt | null>> {
3021
3925
  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
3022
3926
  if (receipts.length === 0) return [];
3927
+ if (
3928
+ consumeDirectRuntimeTestFault({
3929
+ context,
3930
+ runId: receipts[0]?.runId ?? '',
3931
+ name: 'receipt_complete_write_fail',
3932
+ })
3933
+ ) {
3934
+ const [skippedReceipt, ...siblingReceipts] = receipts;
3935
+ let siblingError: unknown = null;
3936
+ if (siblingReceipts.length > 0) {
3937
+ try {
3938
+ await completeRuntimeWorkReceipts(context, {
3939
+ playName: input.playName,
3940
+ receipts: siblingReceipts,
3941
+ });
3942
+ } catch (error) {
3943
+ siblingError = error;
3944
+ }
3945
+ }
3946
+ const error = runtimeTestFaultError('receipt_complete_write_fail');
3947
+ if (skippedReceipt) {
3948
+ const failed = await failRuntimeWorkReceipt(context, {
3949
+ playName: input.playName,
3950
+ runId: skippedReceipt.runId,
3951
+ key: skippedReceipt.key,
3952
+ leaseId: skippedReceipt.leaseId,
3953
+ error: error.message,
3954
+ failureKind: 'repairable',
3955
+ runAttempt: skippedReceipt.runAttempt,
3956
+ });
3957
+ if (!failed) {
3958
+ await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
3959
+ playName: input.playName,
3960
+ runId: skippedReceipt.runId,
3961
+ key: skippedReceipt.key,
3962
+ error: error.message,
3963
+ failureKind: 'repairable',
3964
+ runAttempt: skippedReceipt.runAttempt,
3965
+ });
3966
+ }
3967
+ }
3968
+ if (siblingError) {
3969
+ throw siblingError;
3970
+ }
3971
+ throw error;
3972
+ }
3023
3973
  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
3024
3974
  playName: input.playName,
3025
3975
  keys: receipts.map((receipt) => receipt.key),
@@ -3033,52 +3983,79 @@ export async function completeRuntimeWorkReceipts(
3033
3983
  WITH inputs AS (
3034
3984
  SELECT key_values.key_hex,
3035
3985
  run_values.run_id,
3986
+ lease_values.lease_id,
3987
+ attempt_values.run_attempt,
3036
3988
  output_values.output,
3037
3989
  key_values.ord
3038
3990
  FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
3039
3991
  JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
3040
3992
  ON run_values.ord = key_values.ord
3041
- JOIN jsonb_array_elements($3::jsonb) WITH ORDINALITY AS output_values(output, ord)
3993
+ JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
3994
+ ON lease_values.ord = key_values.ord
3995
+ JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
3996
+ ON attempt_values.ord = key_values.ord
3997
+ JOIN jsonb_array_elements($5::jsonb) WITH ORDINALITY AS output_values(output, ord)
3042
3998
  ON output_values.ord = key_values.ord
3043
3999
  ),
3044
4000
  completed AS (
3045
4001
  UPDATE ${workReceiptTable(session)} AS target
3046
- SET status = $4::smallint,
3047
- output = inputs.output,
3048
- error = NULL,
3049
- run_id = inputs.run_id,
4002
+ SET status = $6::smallint,
4003
+ output = inputs.output,
4004
+ error = NULL,
4005
+ failure_kind = 0,
4006
+ run_id = inputs.run_id,
4007
+ lease_id = NULL,
4008
+ lease_owner_run_id = NULL,
4009
+ lease_owner_attempt = NULL,
4010
+ lease_expires_at = NULL,
3050
4011
  updated_at = now()
3051
4012
  FROM inputs
3052
4013
  WHERE target.k = decode(inputs.key_hex, 'hex')
3053
- AND target.status = $5::smallint
3054
- AND target.run_id = inputs.run_id
3055
- RETURNING target.k, target.status, target.output, target.error, target.run_id, target.updated_at, inputs.ord
4014
+ AND ${workReceiptRunningOwnerPredicateSql({
4015
+ receiptTable: 'target',
4016
+ ownerRunIdSql: 'inputs.run_id',
4017
+ ownerRunAttemptSql: 'inputs.run_attempt',
4018
+ leaseIdSql: 'inputs.lease_id',
4019
+ })}
4020
+ RETURNING target.k,
4021
+ target.status,
4022
+ target.output,
4023
+ target.error,
4024
+ target.failure_kind,
4025
+ target.run_id,
4026
+ target.lease_id,
4027
+ target.lease_owner_run_id,
4028
+ target.lease_owner_attempt,
4029
+ target.lease_expires_at,
4030
+ target.updated_at,
4031
+ inputs.ord
3056
4032
  ),
3057
- latest AS (
3058
- SELECT receipts.k,
3059
- receipts.status,
3060
- receipts.output,
3061
- receipts.error,
3062
- receipts.run_id,
3063
- receipts.updated_at,
4033
+ returned AS (
4034
+ SELECT completed.k,
4035
+ completed.status,
4036
+ completed.output,
4037
+ completed.error,
4038
+ completed.failure_kind,
4039
+ completed.run_id,
4040
+ completed.lease_id,
4041
+ completed.lease_owner_run_id,
4042
+ completed.lease_owner_attempt,
4043
+ completed.lease_expires_at,
4044
+ completed.updated_at,
3064
4045
  inputs.ord
3065
4046
  FROM inputs
3066
- JOIN ${workReceiptTable(session)} AS receipts
3067
- ON receipts.k = decode(inputs.key_hex, 'hex')
3068
- WHERE NOT EXISTS (
3069
- SELECT 1 FROM completed WHERE completed.k = receipts.k
3070
- )
3071
- ),
3072
- returned AS (
3073
- SELECT k, status, output, error, run_id, updated_at, ord FROM completed
3074
- UNION ALL
3075
- SELECT k, status, output, error, run_id, updated_at, ord FROM latest
4047
+ LEFT JOIN completed ON completed.ord = inputs.ord
3076
4048
  )
3077
4049
  SELECT convert_from(returned.k, 'UTF8') AS k,
3078
4050
  returned.status,
3079
- returned.output,
3080
- returned.error,
3081
- returned.run_id,
4051
+ returned.output,
4052
+ returned.error,
4053
+ returned.failure_kind,
4054
+ returned.run_id,
4055
+ returned.lease_id,
4056
+ returned.lease_owner_run_id,
4057
+ returned.lease_owner_attempt,
4058
+ returned.lease_expires_at,
3082
4059
  returned.updated_at
3083
4060
  FROM returned
3084
4061
  ORDER BY returned.ord
@@ -3086,16 +4063,21 @@ export async function completeRuntimeWorkReceipts(
3086
4063
  [
3087
4064
  receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
3088
4065
  receipts.map((receipt) => receipt.runId),
4066
+ receipts.map((receipt) => receipt.leaseId?.trim() || null),
4067
+ receipts.map((receipt) =>
4068
+ normalizeRuntimeRunAttempt(receipt.runAttempt),
4069
+ ),
3089
4070
  stringifyPostgresJson(
3090
4071
  receipts.map((receipt) =>
3091
4072
  receipt.output === null ? null : receipt.output,
3092
4073
  ),
3093
4074
  ),
3094
4075
  RECEIPT_STATUS_COMPLETED,
3095
- RECEIPT_STATUS_RUNNING,
3096
4076
  ],
3097
4077
  );
3098
- return rows.map(mapRuntimeWorkReceiptRow);
4078
+ return rows.map((row) =>
4079
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
4080
+ );
3099
4081
  },
3100
4082
  );
3101
4083
  }
@@ -3106,7 +4088,10 @@ export async function failRuntimeWorkReceipt(
3106
4088
  playName: string;
3107
4089
  runId: string;
3108
4090
  key: string;
4091
+ runAttempt?: number | null;
4092
+ leaseId?: string | null;
3109
4093
  error: string;
4094
+ failureKind?: WorkReceiptFailureKind | null;
3110
4095
  },
3111
4096
  ): Promise<WorkReceipt | null> {
3112
4097
  const session = await getRuntimeWorkReceiptSession(context, {
@@ -3117,36 +4102,119 @@ export async function failRuntimeWorkReceipt(
3117
4102
  context,
3118
4103
  session,
3119
4104
  async (client) => {
4105
+ const leaseId = input.leaseId?.trim() || null;
4106
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3120
4107
  const { rows } = await client.query<Record<string, unknown>>(
3121
4108
  `
3122
4109
  WITH failed AS (
3123
4110
  UPDATE ${workReceiptTable(session)}
3124
- SET status = $2::smallint,
3125
- output = NULL,
3126
- error = $3,
4111
+ SET status = $2::smallint,
4112
+ output = NULL,
4113
+ error = $3,
4114
+ failure_kind = $6::smallint,
3127
4115
  run_id = $4,
4116
+ lease_id = NULL,
4117
+ lease_owner_run_id = NULL,
4118
+ lease_owner_attempt = NULL,
4119
+ lease_expires_at = NULL,
3128
4120
  updated_at = now()
3129
4121
  WHERE k = decode($1, 'hex')
3130
- AND status = $5::smallint
3131
- AND run_id = $4
3132
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3133
- ),
3134
- latest AS (
3135
- SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3136
- FROM ${workReceiptTable(session)}
3137
- WHERE k = decode($1, 'hex')
3138
- AND NOT EXISTS (SELECT 1 FROM failed)
3139
- )
3140
- SELECT k, status, output, error, run_id, updated_at FROM failed
3141
- UNION ALL
3142
- SELECT k, status, output, error, run_id, updated_at FROM latest
4122
+ AND ${workReceiptRunningOwnerPredicateSql({
4123
+ receiptTable: workReceiptTable(session),
4124
+ ownerRunIdSql: '$4',
4125
+ ownerRunAttemptSql: '$7',
4126
+ leaseIdSql: '$5::text',
4127
+ })}
4128
+ RETURNING convert_from(k, 'UTF8') AS k,
4129
+ status,
4130
+ output,
4131
+ error,
4132
+ failure_kind,
4133
+ run_id,
4134
+ lease_id,
4135
+ lease_owner_run_id,
4136
+ lease_owner_attempt,
4137
+ lease_expires_at,
4138
+ updated_at
4139
+ )
4140
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
3143
4141
  `,
3144
4142
  [
3145
4143
  workReceiptKeyHex(input.key),
3146
4144
  RECEIPT_STATUS_FAILED,
3147
4145
  input.error,
3148
4146
  input.runId,
3149
- RECEIPT_STATUS_RUNNING,
4147
+ leaseId,
4148
+ workReceiptFailureKindCode(input.failureKind),
4149
+ runAttempt,
4150
+ ],
4151
+ );
4152
+ return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
4153
+ },
4154
+ );
4155
+ }
4156
+
4157
+ export async function releaseRuntimeWorkReceipt(
4158
+ context: RuntimeApiContext,
4159
+ input: {
4160
+ playName: string;
4161
+ runId: string;
4162
+ key: string;
4163
+ runAttempt?: number | null;
4164
+ leaseId?: string | null;
4165
+ },
4166
+ ): Promise<WorkReceipt | null> {
4167
+ const session = await getRuntimeWorkReceiptSession(context, {
4168
+ playName: input.playName,
4169
+ key: input.key,
4170
+ });
4171
+ return await withRuntimeWorkReceiptClient(
4172
+ context,
4173
+ session,
4174
+ async (client) => {
4175
+ const leaseId = input.leaseId?.trim() || null;
4176
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
4177
+ const { rows } = await client.query<Record<string, unknown>>(
4178
+ `
4179
+ WITH released AS (
4180
+ UPDATE ${workReceiptTable(session)}
4181
+ SET status = $2::smallint,
4182
+ output = NULL,
4183
+ error = NULL,
4184
+ failure_kind = 0,
4185
+ run_id = $3,
4186
+ lease_id = NULL,
4187
+ lease_owner_run_id = NULL,
4188
+ lease_owner_attempt = NULL,
4189
+ lease_expires_at = NULL,
4190
+ updated_at = now()
4191
+ WHERE k = decode($1, 'hex')
4192
+ AND ${workReceiptRunningOwnerPredicateSql({
4193
+ receiptTable: workReceiptTable(session),
4194
+ ownerRunIdSql: '$3',
4195
+ ownerRunAttemptSql: '$5',
4196
+ leaseIdSql: '$4::text',
4197
+ })}
4198
+ RETURNING convert_from(k, 'UTF8') AS k,
4199
+ status,
4200
+ output,
4201
+ error,
4202
+ failure_kind,
4203
+ run_id,
4204
+ lease_id,
4205
+ lease_owner_run_id,
4206
+ lease_owner_attempt,
4207
+ lease_expires_at,
4208
+ updated_at
4209
+ )
4210
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM released
4211
+ `,
4212
+ [
4213
+ workReceiptKeyHex(input.key),
4214
+ RECEIPT_STATUS_PENDING,
4215
+ input.runId,
4216
+ leaseId,
4217
+ runAttempt,
3150
4218
  ],
3151
4219
  );
3152
4220
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -3158,9 +4226,16 @@ export async function failRuntimeWorkReceipts(
3158
4226
  context: RuntimeApiContext,
3159
4227
  input: {
3160
4228
  playName: string;
3161
- receipts: Array<{ runId: string; key: string; error: string }>;
4229
+ receipts: Array<{
4230
+ runId: string;
4231
+ key: string;
4232
+ error: string;
4233
+ failureKind?: WorkReceiptFailureKind | null;
4234
+ leaseId?: string | null;
4235
+ runAttempt?: number | null;
4236
+ }>;
3162
4237
  },
3163
- ): Promise<WorkReceipt[]> {
4238
+ ): Promise<Array<WorkReceipt | null>> {
3164
4239
  const receipts = input.receipts.filter((receipt) => receipt.key.trim());
3165
4240
  if (receipts.length === 0) return [];
3166
4241
  const session = await getRuntimeWorkReceiptSessionForKeys(context, {
@@ -3176,65 +4251,207 @@ export async function failRuntimeWorkReceipts(
3176
4251
  WITH inputs AS (
3177
4252
  SELECT key_values.key_hex,
3178
4253
  run_values.run_id,
3179
- error_values.error,
3180
- key_values.ord
4254
+ lease_values.lease_id,
4255
+ attempt_values.run_attempt,
4256
+ error_values.error,
4257
+ failure_kind_values.failure_kind,
4258
+ key_values.ord
3181
4259
  FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
3182
4260
  JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
3183
4261
  ON run_values.ord = key_values.ord
3184
- JOIN unnest($3::text[]) WITH ORDINALITY AS error_values(error, ord)
3185
- ON error_values.ord = key_values.ord
4262
+ JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
4263
+ ON lease_values.ord = key_values.ord
4264
+ JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
4265
+ ON attempt_values.ord = key_values.ord
4266
+ JOIN unnest($5::text[]) WITH ORDINALITY AS error_values(error, ord)
4267
+ ON error_values.ord = key_values.ord
4268
+ JOIN unnest($6::smallint[]) WITH ORDINALITY AS failure_kind_values(failure_kind, ord)
4269
+ ON failure_kind_values.ord = key_values.ord
3186
4270
  ),
3187
4271
  failed AS (
3188
4272
  UPDATE ${workReceiptTable(session)} AS target
3189
- SET status = $4::smallint,
3190
- output = NULL,
3191
- error = inputs.error,
3192
- run_id = inputs.run_id,
4273
+ SET status = $7::smallint,
4274
+ output = NULL,
4275
+ error = inputs.error,
4276
+ failure_kind = inputs.failure_kind,
4277
+ run_id = inputs.run_id,
4278
+ lease_id = NULL,
4279
+ lease_owner_run_id = NULL,
4280
+ lease_owner_attempt = NULL,
4281
+ lease_expires_at = NULL,
3193
4282
  updated_at = now()
3194
4283
  FROM inputs
3195
4284
  WHERE target.k = decode(inputs.key_hex, 'hex')
3196
- AND target.status = $5::smallint
3197
- AND target.run_id = inputs.run_id
3198
- RETURNING target.k, target.status, target.output, target.error, target.run_id, target.updated_at, inputs.ord
4285
+ AND ${workReceiptRunningOwnerPredicateSql({
4286
+ receiptTable: 'target',
4287
+ ownerRunIdSql: 'inputs.run_id',
4288
+ ownerRunAttemptSql: 'inputs.run_attempt',
4289
+ leaseIdSql: 'inputs.lease_id',
4290
+ })}
4291
+ RETURNING target.k,
4292
+ target.status,
4293
+ target.output,
4294
+ target.error,
4295
+ target.failure_kind,
4296
+ target.run_id,
4297
+ target.lease_id,
4298
+ target.lease_owner_run_id,
4299
+ target.lease_owner_attempt,
4300
+ target.lease_expires_at,
4301
+ target.updated_at,
4302
+ inputs.ord
3199
4303
  ),
3200
- latest AS (
3201
- SELECT receipts.k,
3202
- receipts.status,
3203
- receipts.output,
3204
- receipts.error,
3205
- receipts.run_id,
3206
- receipts.updated_at,
4304
+ returned AS (
4305
+ SELECT failed.k,
4306
+ failed.status,
4307
+ failed.output,
4308
+ failed.error,
4309
+ failed.failure_kind,
4310
+ failed.run_id,
4311
+ failed.lease_id,
4312
+ failed.lease_owner_run_id,
4313
+ failed.lease_owner_attempt,
4314
+ failed.lease_expires_at,
4315
+ failed.updated_at,
3207
4316
  inputs.ord
3208
4317
  FROM inputs
3209
- JOIN ${workReceiptTable(session)} AS receipts
3210
- ON receipts.k = decode(inputs.key_hex, 'hex')
3211
- WHERE NOT EXISTS (
3212
- SELECT 1 FROM failed WHERE failed.k = receipts.k
3213
- )
4318
+ LEFT JOIN failed ON failed.ord = inputs.ord
4319
+ )
4320
+ SELECT convert_from(returned.k, 'UTF8') AS k,
4321
+ returned.status,
4322
+ returned.output,
4323
+ returned.error,
4324
+ returned.failure_kind,
4325
+ returned.run_id,
4326
+ returned.lease_id,
4327
+ returned.lease_owner_run_id,
4328
+ returned.lease_owner_attempt,
4329
+ returned.lease_expires_at,
4330
+ returned.updated_at
4331
+ FROM returned
4332
+ ORDER BY returned.ord
4333
+ `,
4334
+ [
4335
+ receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
4336
+ receipts.map((receipt) => receipt.runId),
4337
+ receipts.map((receipt) => receipt.leaseId?.trim() || null),
4338
+ receipts.map((receipt) =>
4339
+ normalizeRuntimeRunAttempt(receipt.runAttempt),
4340
+ ),
4341
+ receipts.map((receipt) => receipt.error),
4342
+ receipts.map((receipt) =>
4343
+ workReceiptFailureKindCode(receipt.failureKind),
4344
+ ),
4345
+ RECEIPT_STATUS_FAILED,
4346
+ ],
4347
+ );
4348
+ return rows.map((row) =>
4349
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
4350
+ );
4351
+ },
4352
+ );
4353
+ }
4354
+
4355
+ export async function heartbeatRuntimeWorkReceipts(
4356
+ context: RuntimeApiContext,
4357
+ input: {
4358
+ playName: string;
4359
+ runId: string;
4360
+ runAttempt?: number | null;
4361
+ leaseId: string;
4362
+ keys: string[];
4363
+ leaseTtlMs?: number | null;
4364
+ },
4365
+ ): Promise<Array<WorkReceipt | null>> {
4366
+ const keys = input.keys.map((key) => key.trim()).filter(Boolean);
4367
+ if (keys.length === 0) return [];
4368
+ const leaseId = input.leaseId.trim();
4369
+ if (!leaseId) return keys.map(() => null);
4370
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
4371
+ const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
4372
+ input.leaseTtlMs,
4373
+ PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
4374
+ );
4375
+ const session = await getRuntimeWorkReceiptSessionForKeys(context, {
4376
+ playName: input.playName,
4377
+ keys,
4378
+ });
4379
+ return await withRuntimeWorkReceiptClient(
4380
+ context,
4381
+ session,
4382
+ async (client) => {
4383
+ const { rows } = await client.query<Record<string, unknown>>(
4384
+ `
4385
+ WITH input_keys AS (
4386
+ SELECT key_values.key_hex, key_values.ord
4387
+ FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
4388
+ ),
4389
+ renewed AS (
4390
+ UPDATE ${workReceiptTable(session)} AS target
4391
+ SET lease_expires_at = now() + ($4::double precision * interval '1 millisecond'),
4392
+ updated_at = now()
4393
+ FROM input_keys
4394
+ WHERE target.k = decode(input_keys.key_hex, 'hex')
4395
+ AND ${workReceiptHeartbeatPredicateSql({
4396
+ receiptTable: 'target',
4397
+ ownerRunIdSql: '$2::text',
4398
+ ownerRunAttemptSql: '$5',
4399
+ leaseIdSql: '$3::text',
4400
+ })}
4401
+ RETURNING target.k,
4402
+ target.status,
4403
+ target.output,
4404
+ target.error,
4405
+ target.failure_kind,
4406
+ target.run_id,
4407
+ target.lease_id,
4408
+ target.lease_owner_run_id,
4409
+ target.lease_owner_attempt,
4410
+ target.lease_expires_at,
4411
+ target.updated_at,
4412
+ input_keys.ord
3214
4413
  ),
3215
4414
  returned AS (
3216
- SELECT k, status, output, error, run_id, updated_at, ord FROM failed
3217
- UNION ALL
3218
- SELECT k, status, output, error, run_id, updated_at, ord FROM latest
4415
+ SELECT renewed.k,
4416
+ renewed.status,
4417
+ renewed.output,
4418
+ renewed.error,
4419
+ renewed.failure_kind,
4420
+ renewed.run_id,
4421
+ renewed.lease_id,
4422
+ renewed.lease_owner_run_id,
4423
+ renewed.lease_owner_attempt,
4424
+ renewed.lease_expires_at,
4425
+ renewed.updated_at,
4426
+ input_keys.ord
4427
+ FROM input_keys
4428
+ LEFT JOIN renewed ON renewed.ord = input_keys.ord
3219
4429
  )
3220
4430
  SELECT convert_from(returned.k, 'UTF8') AS k,
3221
4431
  returned.status,
3222
4432
  returned.output,
3223
4433
  returned.error,
4434
+ returned.failure_kind,
3224
4435
  returned.run_id,
4436
+ returned.lease_id,
4437
+ returned.lease_owner_run_id,
4438
+ returned.lease_owner_attempt,
4439
+ returned.lease_expires_at,
3225
4440
  returned.updated_at
3226
4441
  FROM returned
3227
4442
  ORDER BY returned.ord
3228
4443
  `,
3229
4444
  [
3230
- receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
3231
- receipts.map((receipt) => receipt.runId),
3232
- receipts.map((receipt) => receipt.error),
3233
- RECEIPT_STATUS_FAILED,
3234
- RECEIPT_STATUS_RUNNING,
4445
+ keys.map(workReceiptKeyHex),
4446
+ input.runId,
4447
+ leaseId,
4448
+ leaseTtlMs,
4449
+ runAttempt,
3235
4450
  ],
3236
4451
  );
3237
- return rows.map(mapRuntimeWorkReceiptRow);
4452
+ return rows.map((row) =>
4453
+ row.k == null ? null : mapRuntimeWorkReceiptRow(row),
4454
+ );
3238
4455
  },
3239
4456
  );
3240
4457
  }
@@ -3245,6 +4462,8 @@ export async function skipRuntimeWorkReceipt(
3245
4462
  playName: string;
3246
4463
  runId: string;
3247
4464
  key: string;
4465
+ runAttempt?: number | null;
4466
+ leaseId?: string | null;
3248
4467
  output: unknown;
3249
4468
  },
3250
4469
  ): Promise<WorkReceipt | null> {
@@ -3256,36 +4475,50 @@ export async function skipRuntimeWorkReceipt(
3256
4475
  context,
3257
4476
  session,
3258
4477
  async (client) => {
4478
+ const leaseId = input.leaseId?.trim() || null;
4479
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
3259
4480
  const { rows } = await client.query<Record<string, unknown>>(
3260
4481
  `
3261
4482
  WITH skipped AS (
3262
4483
  UPDATE ${workReceiptTable(session)}
3263
- SET status = $2::smallint,
3264
- output = $3::jsonb,
3265
- error = NULL,
4484
+ SET status = $2::smallint,
4485
+ output = $3::jsonb,
4486
+ error = NULL,
4487
+ failure_kind = 0,
3266
4488
  run_id = $4,
4489
+ lease_id = NULL,
4490
+ lease_owner_run_id = NULL,
4491
+ lease_owner_attempt = NULL,
4492
+ lease_expires_at = NULL,
3267
4493
  updated_at = now()
3268
4494
  WHERE k = decode($1, 'hex')
3269
- AND status = $5::smallint
3270
- AND run_id = $4
3271
- RETURNING convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3272
- ),
3273
- latest AS (
3274
- SELECT convert_from(k, 'UTF8') AS k, status, output, error, run_id, updated_at
3275
- FROM ${workReceiptTable(session)}
3276
- WHERE k = decode($1, 'hex')
3277
- AND NOT EXISTS (SELECT 1 FROM skipped)
3278
- )
3279
- SELECT k, status, output, error, run_id, updated_at FROM skipped
3280
- UNION ALL
3281
- SELECT k, status, output, error, run_id, updated_at FROM latest
4495
+ AND ${workReceiptRunningOwnerPredicateSql({
4496
+ receiptTable: workReceiptTable(session),
4497
+ ownerRunIdSql: '$4',
4498
+ ownerRunAttemptSql: '$6',
4499
+ leaseIdSql: '$5::text',
4500
+ })}
4501
+ RETURNING convert_from(k, 'UTF8') AS k,
4502
+ status,
4503
+ output,
4504
+ error,
4505
+ failure_kind,
4506
+ run_id,
4507
+ lease_id,
4508
+ lease_owner_run_id,
4509
+ lease_owner_attempt,
4510
+ lease_expires_at,
4511
+ updated_at
4512
+ )
4513
+ SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM skipped
3282
4514
  `,
3283
4515
  [
3284
4516
  workReceiptKeyHex(input.key),
3285
4517
  RECEIPT_STATUS_SKIPPED,
3286
4518
  input.output === null ? null : stringifyPostgresJson(input.output),
3287
4519
  input.runId,
3288
- RECEIPT_STATUS_RUNNING,
4520
+ leaseId,
4521
+ runAttempt,
3289
4522
  ],
3290
4523
  );
3291
4524
  return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
@@ -3302,6 +4535,11 @@ export async function startRuntimeSheetDataset(
3302
4535
  sheetContract: PlaySheetContract;
3303
4536
  rows: Record<string, unknown>[];
3304
4537
  runId: string;
4538
+ attemptId?: string | null;
4539
+ attemptOwnerRunId?: string | null;
4540
+ attemptExpiresAt?: string | null;
4541
+ attemptSeq?: number | null;
4542
+ attemptLeaseTtlMs?: number | null;
3305
4543
  inputOffset?: number;
3306
4544
  force?: boolean;
3307
4545
  },
@@ -3378,6 +4616,7 @@ export async function startRuntimeSheetDataset(
3378
4616
  skipped: input.rows.length,
3379
4617
  pendingRows: [],
3380
4618
  completedRows: [],
4619
+ blockedRows: [],
3381
4620
  tableNamespace: input.tableNamespace,
3382
4621
  timings: [
3383
4622
  ...timings,
@@ -3426,7 +4665,12 @@ export async function startRuntimeSheetDataset(
3426
4665
  const normalizedTableNamespace = normalizeTableNamespace(
3427
4666
  input.tableNamespace,
3428
4667
  );
3429
-
4668
+ const attemptId = normalizeRuntimeSheetAttemptId(input.attemptId);
4669
+ const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
4670
+ attemptOwnerRunId: input.attemptOwnerRunId,
4671
+ runId: input.runId,
4672
+ });
4673
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
3430
4674
  const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
3431
4675
  const needsTransaction = chunks.length > 1;
3432
4676
 
@@ -3441,12 +4685,26 @@ export async function startRuntimeSheetDataset(
3441
4685
  timings,
3442
4686
  },
3443
4687
  async (client) => {
4688
+ const attemptExpiryStartedAt = Date.now();
4689
+ const attemptExpiresAt = await mintRuntimeSheetAttemptExpiresAt(
4690
+ client,
4691
+ input.attemptLeaseTtlMs,
4692
+ );
4693
+ timings.push({
4694
+ phase: 'attempt_expiry_mint',
4695
+ ms: Date.now() - attemptExpiryStartedAt,
4696
+ rows: rowEntries.length,
4697
+ });
3444
4698
  const prepareStartedAt = Date.now();
3445
4699
  const prepared = await prepareRuntimeSheetDatasetRows(client, session, {
3446
4700
  chunks,
3447
4701
  runId: input.runId,
3448
4702
  normalizedPlayName,
3449
4703
  normalizedTableNamespace,
4704
+ attemptId,
4705
+ attemptOwnerRunId,
4706
+ attemptExpiresAt,
4707
+ attemptSeq,
3450
4708
  physicalInsertColumnsSql,
3451
4709
  physicalInsertValuesSql,
3452
4710
  physicalRefreshSetSql:
@@ -3455,19 +4713,15 @@ export async function startRuntimeSheetDataset(
3455
4713
  outputPhysicalColumns,
3456
4714
  force: input.force === true,
3457
4715
  });
3458
- const effectivePrepared = input.force
3459
- ? {
3460
- inserted: prepared.inserted,
3461
- pendingKeys: rowEntries.map((entry) => entry.key),
3462
- }
3463
- : prepared;
3464
4716
  timings.push({
3465
4717
  phase: 'prepare_rows_sql',
3466
4718
  ms: Date.now() - prepareStartedAt,
3467
4719
  rows: rowEntries.length,
3468
4720
  chunks: chunks.length,
3469
- inserted: effectivePrepared.inserted,
3470
- pending: effectivePrepared.pendingKeys.length,
4721
+ inserted: prepared.inserted,
4722
+ pending: prepared.rowDispositions.filter(
4723
+ (row) => row.disposition === 'pending',
4724
+ ).length,
3471
4725
  ...(input.force ? { force: true } : {}),
3472
4726
  });
3473
4727
  const buildStartedAt = Date.now();
@@ -3479,9 +4733,13 @@ export async function startRuntimeSheetDataset(
3479
4733
  normalizedPlayName,
3480
4734
  normalizedTableNamespace,
3481
4735
  runId: input.runId,
4736
+ attemptId,
4737
+ attemptOwnerRunId,
4738
+ attemptExpiresAt,
4739
+ attemptSeq,
3482
4740
  timings,
3483
4741
  force: input.force === true,
3484
- ...effectivePrepared,
4742
+ ...prepared,
3485
4743
  });
3486
4744
  timings.push({
3487
4745
  phase: 'build_result',
@@ -3492,7 +4750,11 @@ export async function startRuntimeSheetDataset(
3492
4750
  pending: built.pendingRows.length,
3493
4751
  completed: built.completedRows.length,
3494
4752
  });
3495
- return built;
4753
+ return {
4754
+ ...built,
4755
+ attemptExpiresAt,
4756
+ attemptSeq,
4757
+ };
3496
4758
  },
3497
4759
  );
3498
4760
  timings.push({
@@ -3505,7 +4767,298 @@ export async function startRuntimeSheetDataset(
3505
4767
  pending: result.pendingRows.length,
3506
4768
  completed: result.completedRows.length,
3507
4769
  });
3508
- return { ...result, timings };
4770
+ return {
4771
+ ...result,
4772
+ attemptId,
4773
+ attemptOwnerRunId,
4774
+ attemptExpiresAt: result.attemptExpiresAt,
4775
+ attemptSeq,
4776
+ timings,
4777
+ };
4778
+ }
4779
+
4780
+ export async function heartbeatRuntimeSheetAttempt(
4781
+ context: RuntimeApiContext,
4782
+ input: {
4783
+ playName: string;
4784
+ tableNamespace: string;
4785
+ sheetContract: PlaySheetContract;
4786
+ runId: string;
4787
+ attemptId: string;
4788
+ attemptOwnerRunId?: string | null;
4789
+ attemptSeq?: number | null;
4790
+ attemptLeaseTtlMs?: number | null;
4791
+ keys: string[];
4792
+ },
4793
+ ): Promise<RuntimeSheetAttemptHeartbeatResult> {
4794
+ const playName = context.playName?.trim() || input.playName;
4795
+ if (!playName) {
4796
+ throw new Error('Runtime sheet attempt heartbeat requires a playName.');
4797
+ }
4798
+ const attemptId = input.attemptId.trim();
4799
+ const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
4800
+ attemptOwnerRunId: input.attemptOwnerRunId,
4801
+ runId: input.runId,
4802
+ });
4803
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
4804
+ const attemptLeaseTtlMs = normalizeRuntimeLeaseTtlMs(
4805
+ input.attemptLeaseTtlMs,
4806
+ PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
4807
+ );
4808
+ const keys = [
4809
+ ...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
4810
+ ];
4811
+ if (!attemptId || keys.length === 0) {
4812
+ return { renewed: 0, renewedKeys: [], attemptExpiresAt: null };
4813
+ }
4814
+ const timings: RuntimeSheetTiming[] = [];
4815
+ const session = requireRuntimePostgresSession(
4816
+ await getRuntimeDbSession(
4817
+ {
4818
+ ...context,
4819
+ playName,
4820
+ runId: context.runId ?? input.runId,
4821
+ },
4822
+ {
4823
+ tableNamespace: input.tableNamespace,
4824
+ logicalTable: 'sheet_rows',
4825
+ operations: ['rows.upsert'],
4826
+ limits: {
4827
+ maxRows: runtimeDbSessionRowLimit(keys.length),
4828
+ },
4829
+ sheetContract: input.sheetContract,
4830
+ timings,
4831
+ },
4832
+ ),
4833
+ );
4834
+ return await withRuntimeSheetQueryClient(
4835
+ context,
4836
+ session,
4837
+ {
4838
+ playName,
4839
+ tableNamespace: input.tableNamespace,
4840
+ sheetContract: input.sheetContract,
4841
+ transactional: false,
4842
+ timings,
4843
+ },
4844
+ async (client) => {
4845
+ const { rows } = await client.query<{
4846
+ renewed_keys: string[];
4847
+ renewed: number;
4848
+ attempt_expires_at: Date | string | null;
4849
+ }>(
4850
+ `
4851
+ WITH input_keys AS (
4852
+ SELECT key_values._key
4853
+ FROM unnest($1::text[]) AS key_values(_key)
4854
+ ),
4855
+ owned_terminal AS (
4856
+ SELECT target._key
4857
+ FROM ${sheetTable(session)} AS target
4858
+ JOIN input_keys ON target._key = input_keys._key
4859
+ WHERE target._attempt_id = $2::text
4860
+ AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
4861
+ AND target._run_id = $4::text
4862
+ AND COALESCE(target._attempt_seq, 0) = $6::integer
4863
+ AND target._status = 'enriched'
4864
+ ),
4865
+ renewed AS (
4866
+ UPDATE ${sheetTable(session)} AS target
4867
+ SET _attempt_expires_at = now() + ($5::double precision * interval '1 millisecond'),
4868
+ _updated_at = now(),
4869
+ _version = ${nextRuntimeSheetVersionExpression(session)}
4870
+ FROM input_keys
4871
+ WHERE target._key = input_keys._key
4872
+ AND target._attempt_id = $2::text
4873
+ AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
4874
+ AND target._run_id = $4::text
4875
+ AND COALESCE(target._attempt_seq, 0) = $6::integer
4876
+ AND target._status IN ('pending', 'running', 'failed')
4877
+ RETURNING target._key, target._attempt_expires_at
4878
+ ),
4879
+ owned_keys AS (
4880
+ SELECT _key FROM renewed
4881
+ UNION
4882
+ SELECT _key FROM owned_terminal
4883
+ )
4884
+ SELECT coalesce((SELECT array_agg(_key ORDER BY _key) FROM owned_keys), '{}'::text[]) AS renewed_keys,
4885
+ count(*)::int AS renewed,
4886
+ max(_attempt_expires_at) AS attempt_expires_at
4887
+ FROM renewed
4888
+ `,
4889
+ [
4890
+ keys,
4891
+ attemptId,
4892
+ attemptOwnerRunId,
4893
+ input.runId,
4894
+ attemptLeaseTtlMs,
4895
+ attemptSeq,
4896
+ ],
4897
+ );
4898
+ const row = rows[0];
4899
+ const attemptExpiresAt = row?.attempt_expires_at;
4900
+ return {
4901
+ renewed: Number(row?.renewed ?? 0),
4902
+ renewedKeys: row?.renewed_keys ?? [],
4903
+ attemptExpiresAt:
4904
+ attemptExpiresAt instanceof Date
4905
+ ? attemptExpiresAt.toISOString()
4906
+ : typeof attemptExpiresAt === 'string'
4907
+ ? new Date(attemptExpiresAt).toISOString()
4908
+ : null,
4909
+ };
4910
+ },
4911
+ );
4912
+ }
4913
+
4914
+ /**
4915
+ * Controlled run-fatal teardown for runtime sheet rows: clear the attempt lease
4916
+ * (`_attempt_*`) on every row the aborting attempt still owns, so an immediate
4917
+ * rerun can claim pending rows and overwrite the aborted attempt's rows without
4918
+ * waiting out the 10-minute lease TTL. The predicate is attempt-fenced (see
4919
+ * `releasableRuntimeSheetAttemptFenceSql` / `decideRuntimeSheetRelease`): it only
4920
+ * touches rows owned by this run's attempt generation, so it can never weaken a
4921
+ * different run's live lease. TTL expiry remains the recovery path for true
4922
+ * crashes where no teardown code runs. When `attemptId` is null every attempt id
4923
+ * the run holds at `attemptSeq` is released (the workers runner mints a distinct
4924
+ * attempt id per map chunk under one owner run id + seq).
4925
+ */
4926
+ export async function releaseRuntimeSheetAttempt(
4927
+ context: RuntimeApiContext,
4928
+ input: {
4929
+ playName: string;
4930
+ tableNamespace: string;
4931
+ sheetContract: PlaySheetContract;
4932
+ runId: string;
4933
+ attemptId?: string | null;
4934
+ attemptOwnerRunId?: string | null;
4935
+ attemptSeq?: number | null;
4936
+ },
4937
+ ): Promise<RuntimeSheetAttemptReleaseResult> {
4938
+ const playName = context.playName?.trim() || input.playName;
4939
+ if (!playName) {
4940
+ throw new Error('Runtime sheet attempt release requires a playName.');
4941
+ }
4942
+ const attemptId = input.attemptId?.trim() || null;
4943
+ const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
4944
+ attemptOwnerRunId: input.attemptOwnerRunId,
4945
+ runId: input.runId,
4946
+ });
4947
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
4948
+ if (!attemptOwnerRunId) {
4949
+ return { released: 0, releasedKeys: [] };
4950
+ }
4951
+ const timings: RuntimeSheetTiming[] = [];
4952
+ const session = requireRuntimePostgresSession(
4953
+ await getRuntimeDbSession(
4954
+ {
4955
+ ...context,
4956
+ playName,
4957
+ runId: context.runId ?? input.runId,
4958
+ },
4959
+ {
4960
+ tableNamespace: input.tableNamespace,
4961
+ logicalTable: 'sheet_rows',
4962
+ operations: ['rows.upsert'],
4963
+ limits: {},
4964
+ sheetContract: input.sheetContract,
4965
+ timings,
4966
+ },
4967
+ ),
4968
+ );
4969
+ return await withRuntimeSheetQueryClient(
4970
+ context,
4971
+ session,
4972
+ {
4973
+ playName,
4974
+ tableNamespace: input.tableNamespace,
4975
+ sheetContract: input.sheetContract,
4976
+ transactional: false,
4977
+ timings,
4978
+ },
4979
+ async (client) => {
4980
+ const { rows } = await client.query<{ _key: string }>(
4981
+ `
4982
+ UPDATE ${sheetTable(session)} AS target
4983
+ SET _attempt_id = NULL,
4984
+ _attempt_owner_run_id = NULL,
4985
+ _attempt_expires_at = NULL,
4986
+ _attempt_seq = NULL,
4987
+ _updated_at = now(),
4988
+ _version = ${nextRuntimeSheetVersionExpression(session)}
4989
+ WHERE ${releasableRuntimeSheetAttemptFenceSql(
4990
+ 'target',
4991
+ '$1::text',
4992
+ '$2::text',
4993
+ '$3',
4994
+ )}
4995
+ RETURNING target._key
4996
+ `,
4997
+ [attemptId, attemptOwnerRunId, attemptSeq],
4998
+ );
4999
+ const releasedKeys = rows
5000
+ .map((row) => row._key)
5001
+ .filter((key): key is string => typeof key === 'string');
5002
+ return { released: releasedKeys.length, releasedKeys };
5003
+ },
5004
+ );
5005
+ }
5006
+
5007
+ /**
5008
+ * Controlled run-fatal teardown for work receipts: return every receipt still
5009
+ * leased `running` by the aborting run+attempt back to `pending` (clearing the
5010
+ * lease) so an immediate rerun can reclaim and recompute the interrupted work
5011
+ * without waiting out the lease TTL. Fenced by owner run id + attempt (see
5012
+ * `workReceiptReleaseOwnerPredicateSql`), so it never touches a receipt a live
5013
+ * competitor legitimately holds. Completed/skipped/failed receipts are terminal
5014
+ * and untouched, preserving cross-run idempotency (already-persisted work is
5015
+ * still reused on rerun). Work receipts for a play share one table, so a single
5016
+ * run-scoped statement covers every stranded receipt.
5017
+ */
5018
+ export async function releaseRuntimeWorkReceipts(
5019
+ context: RuntimeApiContext,
5020
+ input: {
5021
+ playName: string;
5022
+ runId: string;
5023
+ runAttempt?: number | null;
5024
+ },
5025
+ ): Promise<RuntimeWorkReceiptReleaseResult> {
5026
+ const runId = input.runId.trim();
5027
+ if (!runId) {
5028
+ return { released: 0, releasedKeys: [] };
5029
+ }
5030
+ const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
5031
+ const session = await getRuntimeWorkReceiptSessionForRun(context, {
5032
+ playName: input.playName,
5033
+ });
5034
+ return await withRuntimeWorkReceiptClient(
5035
+ context,
5036
+ session,
5037
+ async (client) => {
5038
+ const { rows } = await client.query<{ k: string }>(
5039
+ `
5040
+ UPDATE ${workReceiptTable(session)} AS target
5041
+ SET status = $3::smallint,
5042
+ lease_id = NULL,
5043
+ lease_owner_run_id = NULL,
5044
+ lease_owner_attempt = NULL,
5045
+ lease_expires_at = NULL,
5046
+ updated_at = now()
5047
+ WHERE ${workReceiptReleaseOwnerPredicateSql({
5048
+ receiptTable: 'target',
5049
+ ownerRunIdSql: '$1',
5050
+ ownerRunAttemptSql: '$2',
5051
+ })}
5052
+ RETURNING convert_from(target.k, 'UTF8') AS k
5053
+ `,
5054
+ [runId, runAttempt, RECEIPT_STATUS_PENDING],
5055
+ );
5056
+ const releasedKeys = rows
5057
+ .map((row) => row.k)
5058
+ .filter((key): key is string => typeof key === 'string');
5059
+ return { released: releasedKeys.length, releasedKeys };
5060
+ },
5061
+ );
3509
5062
  }
3510
5063
 
3511
5064
  type CompleteRuntimeMapRowChunksInput = {
@@ -3513,17 +5066,41 @@ type CompleteRuntimeMapRowChunksInput = {
3513
5066
  physicalUpdateSetSql: string;
3514
5067
  physicalColumnProjections: PhysicalSheetColumnProjection[];
3515
5068
  runId: string;
5069
+ attemptId: string | null;
5070
+ attemptOwnerRunId: string | null;
5071
+ attemptExpiresAt: string | null;
5072
+ attemptSeq: number;
3516
5073
  normalizedPlayName: string;
3517
5074
  normalizedTableNamespace: string;
3518
5075
  outputFields: string[];
3519
5076
  };
3520
5077
 
5078
+ type RuntimeMapRowsWriteResult = {
5079
+ updated: number;
5080
+ fencedKeys: string[];
5081
+ };
5082
+
5083
+ function writableEnrichedRuntimeSheetAttemptSql(
5084
+ tableAlias: string,
5085
+ attemptOwnerRunIdExpression: string,
5086
+ attemptExpiresAtExpression: string,
5087
+ attemptSeqExpression: string,
5088
+ ): string {
5089
+ return `(
5090
+ ${tableAlias}._status = 'enriched'
5091
+ AND (
5092
+ COALESCE(${tableAlias}._attempt_seq, 0) < ${attemptSeqExpression}::integer
5093
+ )
5094
+ )`;
5095
+ }
5096
+
3521
5097
  async function completeRuntimeMapRowChunks(
3522
5098
  client: RuntimeQueryClient,
3523
5099
  session: RuntimePostgresSession,
3524
5100
  input: CompleteRuntimeMapRowChunksInput,
3525
- ): Promise<{ updated: number }> {
5101
+ ): Promise<RuntimeMapRowsWriteResult> {
3526
5102
  let updated = 0;
5103
+ const fencedKeys: string[] = [];
3527
5104
  for (const chunk of input.chunks) {
3528
5105
  const chunkKeys = chunk.map((row) => row.key);
3529
5106
  const chunkInputIndexes = chunk.map((row) => row.input_index);
@@ -3538,9 +5115,21 @@ async function completeRuntimeMapRowChunks(
3538
5115
  'updates.data_patch',
3539
5116
  input.physicalColumnProjections,
3540
5117
  );
5118
+ const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
5119
+ 'target',
5120
+ '$10::text',
5121
+ '$11::timestamptz',
5122
+ '$12',
5123
+ );
5124
+ const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
5125
+ 'target',
5126
+ '$10::text',
5127
+ '$12',
5128
+ );
3541
5129
  const { rows } = await client.query<{
3542
5130
  updated: number;
3543
5131
  matched_keys: string[];
5132
+ fenced_keys: string[];
3544
5133
  }>(
3545
5134
  `WITH updates AS (
3546
5135
  SELECT key_values._key,
@@ -3571,12 +5160,22 @@ async function completeRuntimeMapRowChunks(
3571
5160
  SET _status = 'enriched',
3572
5161
  _run_id = $5::text,
3573
5162
  _error = NULL,
5163
+ _attempt_id = $9::text,
5164
+ _attempt_owner_run_id = $10::text,
5165
+ _attempt_expires_at = $11::timestamptz,
5166
+ _attempt_seq = $12::integer,
3574
5167
  _updated_at = now(),
3575
5168
  _version = ${nextRuntimeSheetVersionExpression(session)},
3576
5169
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
3577
5170
  FROM matched_updates AS updates
3578
5171
  WHERE target._key = updates.matched_key
3579
- AND (target._run_id IS NULL OR target._run_id <= $5::text)
5172
+ AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5173
+ AND NOT (${sameOwnerTerminalEpochSql})
5174
+ AND (
5175
+ ($9::text IS NULL AND target._run_id = $5::text)
5176
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5177
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5178
+ )
3580
5179
  AND (
3581
5180
  target._status <> 'enriched'
3582
5181
  OR target._run_id IS DISTINCT FROM $5::text
@@ -3596,6 +5195,23 @@ async function completeRuntimeMapRowChunks(
3596
5195
  count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
3597
5196
  FROM applied_rows
3598
5197
  ),
5198
+ fenced_rows AS (
5199
+ SELECT updates.matched_key AS _key
5200
+ FROM matched_updates AS updates
5201
+ JOIN ${sheetTable(session)} AS target
5202
+ ON target._key = updates.matched_key
5203
+ WHERE NOT EXISTS (
5204
+ SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5205
+ )
5206
+ AND (
5207
+ (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5208
+ OR NOT (
5209
+ ($9::text IS NULL AND target._run_id = $5::text)
5210
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5211
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5212
+ )
5213
+ )
5214
+ ),
3599
5215
  summary_counts AS (
3600
5216
  SELECT newly_completed,
3601
5217
  from_failed,
@@ -3680,6 +5296,7 @@ async function completeRuntimeMapRowChunks(
3680
5296
  SELECT
3681
5297
  (SELECT count(*)::int FROM applied_rows) AS updated,
3682
5298
  coalesce((SELECT array_agg(matched_key) FROM matched_updates), '{}'::text[]) AS matched_keys,
5299
+ coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
3683
5300
  (SELECT count(*)::int FROM summary_delta) AS summary_delta_count,
3684
5301
  (SELECT count(*)::int FROM column_delta) AS column_delta_count`,
3685
5302
  [
@@ -3691,9 +5308,14 @@ async function completeRuntimeMapRowChunks(
3691
5308
  input.normalizedPlayName,
3692
5309
  input.normalizedTableNamespace,
3693
5310
  [...new Set(input.outputFields)],
5311
+ input.attemptId,
5312
+ input.attemptOwnerRunId,
5313
+ input.attemptExpiresAt,
5314
+ input.attemptSeq,
3694
5315
  ],
3695
5316
  );
3696
5317
  updated += Number(rows[0]?.updated ?? 0);
5318
+ fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
3697
5319
 
3698
5320
  const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
3699
5321
  if (matchedKeys.size === chunk.length) {
@@ -3712,16 +5334,18 @@ async function completeRuntimeMapRowChunks(
3712
5334
  { ...input, chunks: [repairChunk] },
3713
5335
  );
3714
5336
  updated += repaired.updated;
5337
+ fencedKeys.push(...repaired.fencedKeys);
3715
5338
  }
3716
- return { updated };
5339
+ return { updated, fencedKeys };
3717
5340
  }
3718
5341
 
3719
5342
  async function completeRuntimeMapRowChunksWithInputIndexRepair(
3720
5343
  client: RuntimeQueryClient,
3721
5344
  session: RuntimePostgresSession,
3722
5345
  input: CompleteRuntimeMapRowChunksInput,
3723
- ): Promise<{ updated: number }> {
5346
+ ): Promise<RuntimeMapRowsWriteResult> {
3724
5347
  let updated = 0;
5348
+ const fencedKeys: string[] = [];
3725
5349
  for (const chunk of input.chunks) {
3726
5350
  const chunkKeys = chunk.map((row) => row.key);
3727
5351
  const chunkInputIndexes = chunk.map((row) => row.input_index);
@@ -3736,7 +5360,22 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3736
5360
  'updates.data_patch',
3737
5361
  input.physicalColumnProjections,
3738
5362
  );
3739
- const { rows: appliedRows } = await client.query<{ _key: string }>(
5363
+ const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
5364
+ 'target',
5365
+ '$10::text',
5366
+ '$11::timestamptz',
5367
+ '$12',
5368
+ );
5369
+ const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
5370
+ 'target',
5371
+ '$10::text',
5372
+ '$12',
5373
+ );
5374
+ const { rows } = await client.query<{
5375
+ _key?: string;
5376
+ fenced_keys?: string[];
5377
+ updated?: number;
5378
+ }>(
3740
5379
  `WITH updates AS (
3741
5380
  SELECT key_values._key,
3742
5381
  input_index_values.input_index,
@@ -3770,13 +5409,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3770
5409
  SET _status = 'enriched',
3771
5410
  _run_id = $5::text,
3772
5411
  _error = NULL,
5412
+ _attempt_id = $9::text,
5413
+ _attempt_owner_run_id = $10::text,
5414
+ _attempt_expires_at = $11::timestamptz,
5415
+ _attempt_seq = $12::integer,
3773
5416
  _updated_at = now(),
3774
5417
  _version = ${nextRuntimeSheetVersionExpression(session)},
3775
5418
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
3776
5419
  FROM matched_updates AS updates, ${sheetTable(session)} AS prev
3777
5420
  WHERE target._key = updates.matched_key
3778
5421
  AND prev._key = target._key
3779
- AND (target._run_id IS NULL OR target._run_id <= $5::text)
5422
+ AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5423
+ AND NOT (${sameOwnerTerminalEpochSql})
5424
+ AND (
5425
+ ($9::text IS NULL AND target._run_id = $5::text)
5426
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5427
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5428
+ )
3780
5429
  AND (
3781
5430
  target._status <> 'enriched'
3782
5431
  OR target._run_id IS DISTINCT FROM $5::text
@@ -3796,6 +5445,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3796
5445
  count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
3797
5446
  FROM applied_rows
3798
5447
  ),
5448
+ fenced_rows AS (
5449
+ SELECT updates.matched_key AS _key
5450
+ FROM matched_updates AS updates
5451
+ JOIN ${sheetTable(session)} AS target
5452
+ ON target._key = updates.matched_key
5453
+ WHERE NOT EXISTS (
5454
+ SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5455
+ )
5456
+ AND (
5457
+ (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
5458
+ OR NOT (
5459
+ ($9::text IS NULL AND target._run_id = $5::text)
5460
+ OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
5461
+ OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
5462
+ )
5463
+ )
5464
+ ),
3799
5465
  summary_counts AS (
3800
5466
  SELECT newly_completed,
3801
5467
  from_failed,
@@ -3877,7 +5543,9 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3877
5543
  _updated_at = now()
3878
5544
  RETURNING 1
3879
5545
  )
3880
- SELECT _key FROM applied_rows`,
5546
+ SELECT
5547
+ (SELECT count(*)::int FROM applied_rows) AS updated,
5548
+ coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
3881
5549
  [
3882
5550
  chunkKeys,
3883
5551
  chunkInputIndexes,
@@ -3887,11 +5555,16 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
3887
5555
  input.normalizedPlayName,
3888
5556
  input.normalizedTableNamespace,
3889
5557
  [...new Set(input.outputFields)],
5558
+ input.attemptId,
5559
+ input.attemptOwnerRunId,
5560
+ input.attemptExpiresAt,
5561
+ input.attemptSeq,
3890
5562
  ],
3891
5563
  );
3892
- updated += appliedRows.length;
5564
+ updated += Number(rows[0]?.updated ?? 0);
5565
+ fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
3893
5566
  }
3894
- return { updated };
5567
+ return { updated, fencedKeys };
3895
5568
  }
3896
5569
 
3897
5570
  async function insertMissingCompletedMapRowChunks(
@@ -3954,8 +5627,26 @@ async function insertMissingCompletedMapRowChunks(
3954
5627
  )
3955
5628
  ),
3956
5629
  inserted_rows AS (
3957
- INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _cell_meta${physicalInsertColumnsSql})
3958
- SELECT _key, 'enriched', $5::text, input_index, cell_meta_patch${physicalInsertValuesSql}
5630
+ INSERT INTO ${sheetTable(session)} (
5631
+ _key,
5632
+ _status,
5633
+ _run_id,
5634
+ _input_index,
5635
+ _attempt_id,
5636
+ _attempt_owner_run_id,
5637
+ _attempt_expires_at,
5638
+ _attempt_seq,
5639
+ _cell_meta${physicalInsertColumnsSql}
5640
+ )
5641
+ SELECT _key,
5642
+ 'enriched',
5643
+ $5::text,
5644
+ input_index,
5645
+ $8::text,
5646
+ $9::text,
5647
+ $10::timestamptz,
5648
+ $11::integer,
5649
+ cell_meta_patch${physicalInsertValuesSql}
3959
5650
  FROM missing_rows
3960
5651
  ON CONFLICT (_key) DO NOTHING
3961
5652
  RETURNING _key
@@ -3997,7 +5688,7 @@ async function insertMissingCompletedMapRowChunks(
3997
5688
  ),
3998
5689
  completed_cell_delta AS (
3999
5690
  SELECT field_values.field, (SELECT c FROM inserted_count) AS c
4000
- FROM unnest($8::text[]) AS field_values(field)
5691
+ FROM unnest($12::text[]) AS field_values(field)
4001
5692
  WHERE (SELECT c FROM inserted_count) > 0
4002
5693
  ),
4003
5694
  column_delta AS (
@@ -4028,6 +5719,10 @@ async function insertMissingCompletedMapRowChunks(
4028
5719
  input.runId,
4029
5720
  input.normalizedPlayName,
4030
5721
  input.normalizedTableNamespace,
5722
+ input.attemptId,
5723
+ input.attemptOwnerRunId,
5724
+ input.attemptExpiresAt,
5725
+ input.attemptSeq,
4031
5726
  [...new Set(input.outputFields)],
4032
5727
  ],
4033
5728
  );
@@ -4053,12 +5748,18 @@ async function failRuntimeMapRowChunks(
4053
5748
  input: {
4054
5749
  chunks: RuntimePreparedFailedRow[][];
4055
5750
  physicalUpdateSetSql: string;
5751
+ forceTerminal?: boolean;
4056
5752
  runId: string;
5753
+ attemptId: string | null;
5754
+ attemptOwnerRunId: string | null;
5755
+ attemptExpiresAt: string | null;
5756
+ attemptSeq: number;
4057
5757
  normalizedPlayName: string;
4058
5758
  normalizedTableNamespace: string;
4059
5759
  },
4060
- ): Promise<{ updated: number }> {
5760
+ ): Promise<RuntimeMapRowsWriteResult> {
4061
5761
  let updated = 0;
5762
+ const fencedKeys: string[] = [];
4062
5763
  for (const chunk of input.chunks) {
4063
5764
  const chunkKeys = chunk.map((row) => row.key);
4064
5765
  const chunkInputIndexes = chunk.map((row) => row.input_index);
@@ -4087,7 +5788,10 @@ async function failRuntimeMapRowChunks(
4087
5788
  const failedCellTotals = failedCellFields.map(
4088
5789
  (field) => failedCellCounts.get(field) ?? 0,
4089
5790
  );
4090
- const { rows: appliedRows } = await client.query<{ _key: string }>(
5791
+ const { rows } = await client.query<{
5792
+ updated?: number;
5793
+ fenced_keys?: string[];
5794
+ }>(
4091
5795
  `WITH updates AS (
4092
5796
  SELECT key_values._key,
4093
5797
  input_index_values.input_index,
@@ -4125,26 +5829,53 @@ async function failRuntimeMapRowChunks(
4125
5829
  SET _status = 'failed',
4126
5830
  _run_id = $6::text,
4127
5831
  _error = updates.error,
5832
+ _attempt_id = $11::text,
5833
+ _attempt_owner_run_id = $12::text,
5834
+ _attempt_expires_at = $13::timestamptz,
5835
+ _attempt_seq = $14::integer,
4128
5836
  _updated_at = now(),
4129
5837
  _version = ${nextRuntimeSheetVersionExpression(session)},
4130
5838
  _cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
4131
5839
  FROM matched_updates AS updates, ${sheetTable(session)} AS prev
4132
5840
  WHERE target._key = updates.matched_key
4133
5841
  AND prev._key = target._key
4134
- AND target._status <> 'enriched'
4135
- AND (target._run_id IS NULL OR target._run_id <= $6::text)
5842
+ AND ($15::boolean OR target._status <> 'enriched')
5843
+ AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
5844
+ AND (
5845
+ ($11::text IS NULL AND target._run_id = $6::text)
5846
+ OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
5847
+ )
4136
5848
  RETURNING target._key, prev._status AS prev_status
4137
5849
  ),
4138
5850
  applied_count AS (
4139
5851
  SELECT count(*)::bigint AS c,
4140
5852
  count(*) FILTER (WHERE prev_status = 'failed')::bigint AS already_failed,
5853
+ count(*) FILTER (WHERE prev_status = 'enriched')::bigint AS from_completed,
4141
5854
  count(*) FILTER (WHERE prev_status = 'running')::bigint AS from_running
4142
5855
  FROM applied_rows
4143
5856
  ),
5857
+ fenced_rows AS (
5858
+ SELECT updates.matched_key AS _key
5859
+ FROM matched_updates AS updates
5860
+ JOIN ${sheetTable(session)} AS target
5861
+ ON target._key = updates.matched_key
5862
+ WHERE NOT EXISTS (
5863
+ SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
5864
+ )
5865
+ AND (
5866
+ (NOT $15::boolean AND target._status = 'enriched')
5867
+ OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
5868
+ OR NOT (
5869
+ ($11::text IS NULL AND target._run_id = $6::text)
5870
+ OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
5871
+ )
5872
+ )
5873
+ ),
4144
5874
  summary_counts AS (
4145
5875
  SELECT (c - already_failed)::bigint AS newly_failed,
5876
+ from_completed,
4146
5877
  from_running,
4147
- GREATEST(c - already_failed - from_running, 0)::bigint AS from_queued
5878
+ GREATEST(c - already_failed - from_completed - from_running, 0)::bigint AS from_queued
4148
5879
  FROM applied_count
4149
5880
  ),
4150
5881
  summary_delta AS (
@@ -4162,20 +5893,21 @@ async function failRuntimeMapRowChunks(
4162
5893
  0,
4163
5894
  (-from_queued)::int,
4164
5895
  (-from_running)::int,
4165
- 0,
5896
+ (-from_completed)::int,
4166
5897
  newly_failed::int
4167
5898
  FROM summary_counts
4168
5899
  WHERE newly_failed > 0
4169
5900
  ON CONFLICT (play_name, table_namespace) DO UPDATE SET
4170
5901
  queued = GREATEST(target.queued + EXCLUDED.queued, 0),
4171
5902
  running = GREATEST(target.running + EXCLUDED.running, 0),
5903
+ completed = GREATEST(target.completed + EXCLUDED.completed, 0),
4172
5904
  failed = GREATEST(target.failed + EXCLUDED.failed, 0),
4173
5905
  total = ${runtimeSummaryTotalSql({
4174
5906
  currentTotal: 'target.total',
4175
5907
  totalDelta: 'EXCLUDED.total',
4176
5908
  queued: 'GREATEST(target.queued + EXCLUDED.queued, 0)',
4177
5909
  running: 'GREATEST(target.running + EXCLUDED.running, 0)',
4178
- completed: 'target.completed',
5910
+ completed: 'GREATEST(target.completed + EXCLUDED.completed, 0)',
4179
5911
  failed: 'GREATEST(target.failed + EXCLUDED.failed, 0)',
4180
5912
  })},
4181
5913
  _updated_at = now()
@@ -4198,7 +5930,9 @@ async function failRuntimeMapRowChunks(
4198
5930
  _updated_at = now()
4199
5931
  RETURNING 1
4200
5932
  )
4201
- SELECT _key FROM applied_rows`,
5933
+ SELECT
5934
+ (SELECT count(*)::int FROM applied_rows) AS updated,
5935
+ coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
4202
5936
  [
4203
5937
  chunkKeys,
4204
5938
  chunkInputIndexes,
@@ -4210,11 +5944,17 @@ async function failRuntimeMapRowChunks(
4210
5944
  input.normalizedTableNamespace,
4211
5945
  failedCellFields,
4212
5946
  failedCellTotals,
5947
+ input.attemptId,
5948
+ input.attemptOwnerRunId,
5949
+ input.attemptExpiresAt,
5950
+ input.attemptSeq,
5951
+ input.forceTerminal === true,
4213
5952
  ],
4214
5953
  );
4215
- updated += appliedRows.length;
5954
+ updated += Number(rows[0]?.updated ?? 0);
5955
+ fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
4216
5956
  }
4217
- return { updated };
5957
+ return { updated, fencedKeys };
4218
5958
  }
4219
5959
 
4220
5960
  /**
@@ -4240,10 +5980,15 @@ export async function completeRuntimeMapRows(
4240
5980
  rows: RuntimeApiRowRecord[];
4241
5981
  outputFields?: string[];
4242
5982
  runId: string;
5983
+ attemptId?: string | null;
5984
+ attemptOwnerRunId?: string | null;
5985
+ attemptExpiresAt?: string | null;
5986
+ attemptSeq?: number | null;
5987
+ forceFailedRows?: boolean;
4243
5988
  },
4244
- ): Promise<{ updated: number }> {
5989
+ ): Promise<RuntimeMapRowsWriteResult> {
4245
5990
  if (input.rows.length === 0) {
4246
- return { updated: 0 };
5991
+ return { updated: 0, fencedKeys: [] };
4247
5992
  }
4248
5993
  const sheetContract = augmentSheetContractWithDatasetFields({
4249
5994
  contract: input.sheetContract,
@@ -4276,7 +6021,7 @@ export async function completeRuntimeMapRows(
4276
6021
  uniqueRows.set(row.key, row);
4277
6022
  }
4278
6023
  if (uniqueRows.size === 0) {
4279
- return { updated: 0 };
6024
+ return { updated: 0, fencedKeys: [] };
4280
6025
  }
4281
6026
 
4282
6027
  const projections = physicalSheetColumnProjections(sheetContract);
@@ -4304,6 +6049,11 @@ export async function completeRuntimeMapRows(
4304
6049
  const failedChunks = chunkValues(failedRows, DIRECT_POSTGRES_BATCH_SIZE);
4305
6050
  const needsTransaction = chunks.length + failedChunks.length > 1;
4306
6051
  const outputFields = [...new Set(input.outputFields ?? [])];
6052
+ const attemptId = input.attemptId?.trim() || null;
6053
+ const attemptOwnerRunId =
6054
+ input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
6055
+ const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
6056
+ const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
4307
6057
 
4308
6058
  return await withRuntimeSheetQueryClient(
4309
6059
  context,
@@ -4323,6 +6073,10 @@ export async function completeRuntimeMapRows(
4323
6073
  physicalUpdateSetSql,
4324
6074
  physicalColumnProjections: projections,
4325
6075
  runId: input.runId,
6076
+ attemptId,
6077
+ attemptOwnerRunId,
6078
+ attemptExpiresAt,
6079
+ attemptSeq,
4326
6080
  normalizedPlayName: normalizePlayNameForSheet(session.playName),
4327
6081
  normalizedTableNamespace: normalizeTableNamespace(
4328
6082
  input.tableNamespace,
@@ -4339,22 +6093,33 @@ export async function completeRuntimeMapRows(
4339
6093
  session,
4340
6094
  chunkInput,
4341
6095
  );
4342
- return { updated: updated.updated + inserted.inserted };
6096
+ return {
6097
+ updated: updated.updated + inserted.inserted,
6098
+ fencedKeys: updated.fencedKeys,
6099
+ };
4343
6100
  })()
4344
- : { updated: 0 };
6101
+ : { updated: 0, fencedKeys: [] };
4345
6102
  const failed =
4346
6103
  failedRows.length > 0
4347
6104
  ? await failRuntimeMapRowChunks(client, session, {
4348
6105
  chunks: failedChunks,
4349
6106
  physicalUpdateSetSql,
6107
+ forceTerminal: input.forceFailedRows === true,
4350
6108
  runId: input.runId,
6109
+ attemptId,
6110
+ attemptOwnerRunId,
6111
+ attemptExpiresAt,
6112
+ attemptSeq,
4351
6113
  normalizedPlayName: normalizePlayNameForSheet(session.playName),
4352
6114
  normalizedTableNamespace: normalizeTableNamespace(
4353
6115
  input.tableNamespace,
4354
6116
  ),
4355
6117
  })
4356
- : { updated: 0 };
4357
- return { updated: completed.updated + failed.updated };
6118
+ : { updated: 0, fencedKeys: [] };
6119
+ return {
6120
+ updated: completed.updated + failed.updated,
6121
+ fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
6122
+ };
4358
6123
  },
4359
6124
  );
4360
6125
  }
@@ -4408,7 +6173,7 @@ export async function readRuntimeSheetDatasetRowKeys(
4408
6173
  tableNamespace: string;
4409
6174
  runId: string;
4410
6175
  keys: string[];
4411
- rowMode?: 'output' | 'all';
6176
+ rowMode?: 'output' | 'all' | 'terminalAnyRun';
4412
6177
  },
4413
6178
  ): Promise<{ keys: string[] }> {
4414
6179
  const keys = [
@@ -4435,6 +6200,16 @@ export async function readRuntimeSheetDatasetRowKeys(
4435
6200
  ),
4436
6201
  );
4437
6202
  const rows = await withRuntimePostgres(session, async (client) => {
6203
+ if (input.rowMode === 'terminalAnyRun') {
6204
+ const { rows: matchedRows } = await client.query<{ _key: string }>(
6205
+ `SELECT _key
6206
+ FROM ${sheetTable(session)}
6207
+ WHERE _key = ANY($1::text[])
6208
+ AND _status IN ('enriched', 'failed')`,
6209
+ [keys],
6210
+ );
6211
+ return matchedRows;
6212
+ }
4438
6213
  if (input.rowMode === 'all') {
4439
6214
  const { rows: matchedRows } = await client.query<{ _key: string }>(
4440
6215
  `SELECT _key