deepline 0.1.182 → 0.1.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1747 -1890
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +41 -0
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1605 -281
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +76 -17
- package/dist/cli/index.mjs +76 -17
- package/dist/index.d.mts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +63 -9
- package/dist/index.mjs +63 -9
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import {
|
|
3
3
|
createRuntimePool,
|
|
4
4
|
createRuntimeOneShotQueryClient,
|
|
@@ -55,12 +55,28 @@ import {
|
|
|
55
55
|
isReusableWorkReceipt,
|
|
56
56
|
type WorkReceipt,
|
|
57
57
|
type WorkReceiptClaim,
|
|
58
|
+
workReceiptFailureKindCode,
|
|
59
|
+
workReceiptFailureKindFromCode,
|
|
60
|
+
type WorkReceiptFailureKind,
|
|
58
61
|
} from './work-receipts';
|
|
59
62
|
import {
|
|
60
63
|
sanitizePostgresJsonValue,
|
|
61
64
|
stringifyPostgresJson,
|
|
62
65
|
} from './postgres-json';
|
|
63
66
|
import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
|
|
67
|
+
import {
|
|
68
|
+
workReceiptClaimableStatusCodes,
|
|
69
|
+
workReceiptClaimConflictPredicateSql,
|
|
70
|
+
workReceiptHeartbeatPredicateSql,
|
|
71
|
+
workReceiptReleaseOwnerPredicateSql,
|
|
72
|
+
workReceiptRunningOwnerPredicateSql,
|
|
73
|
+
} from './receipt-sql';
|
|
74
|
+
import {
|
|
75
|
+
activeRuntimeSheetAttemptFenceSql,
|
|
76
|
+
newerTerminalRuntimeSheetRowSql,
|
|
77
|
+
releasableRuntimeSheetAttemptFenceSql,
|
|
78
|
+
sameOwnerTerminalAttemptEpochSql,
|
|
79
|
+
} from './sheet-attempt-sql';
|
|
64
80
|
import type { MapRowOutcome } from './durability-store';
|
|
65
81
|
import {
|
|
66
82
|
normalizeRuntimeMapInputIndex,
|
|
@@ -75,6 +91,19 @@ import {
|
|
|
75
91
|
} from './map-row-outcome';
|
|
76
92
|
import { DEEPLINE_CELL_META_FIELD } from './cell-staleness';
|
|
77
93
|
import type { PlayArtifactKind } from './backend';
|
|
94
|
+
import {
|
|
95
|
+
PLAY_RUNTIME_CONTRACT,
|
|
96
|
+
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
97
|
+
} from './runtime-contract';
|
|
98
|
+
import {
|
|
99
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
100
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
101
|
+
} from './lease-policy';
|
|
102
|
+
import {
|
|
103
|
+
parseRuntimeTestFaultCounts,
|
|
104
|
+
PLAY_RUNTIME_TEST_FAULT_HEADER,
|
|
105
|
+
type RuntimeTestFaultName,
|
|
106
|
+
} from './test-runtime-seams';
|
|
78
107
|
|
|
79
108
|
type RuntimeApiContext = {
|
|
80
109
|
baseUrl?: string | null;
|
|
@@ -87,10 +116,133 @@ type RuntimeApiContext = {
|
|
|
87
116
|
dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | null;
|
|
88
117
|
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
89
118
|
vercelProtectionBypassToken?: string | null;
|
|
119
|
+
runtimeTestFaultHeader?: string | null;
|
|
90
120
|
disablePostgresPoolCache?: boolean | null;
|
|
91
121
|
postgresSessionUnwrapKey?: string | null;
|
|
92
122
|
};
|
|
93
123
|
|
|
124
|
+
type DirectRuntimeTestFaultState = {
|
|
125
|
+
headerValue: string;
|
|
126
|
+
counts: Partial<Record<RuntimeTestFaultName, number>>;
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
const directRuntimeTestFaults = new Map<string, DirectRuntimeTestFaultState>();
|
|
130
|
+
|
|
131
|
+
function consumeDirectRuntimeTestFault(input: {
|
|
132
|
+
context: RuntimeApiContext;
|
|
133
|
+
runId: string;
|
|
134
|
+
name: RuntimeTestFaultName;
|
|
135
|
+
}): boolean {
|
|
136
|
+
const headerValue = input.context.runtimeTestFaultHeader?.trim();
|
|
137
|
+
if (!headerValue) return false;
|
|
138
|
+
const stateKey = input.runId.trim();
|
|
139
|
+
if (!stateKey) return false;
|
|
140
|
+
let state = directRuntimeTestFaults.get(stateKey);
|
|
141
|
+
if (!state || state.headerValue !== headerValue) {
|
|
142
|
+
const parsed = parseRuntimeTestFaultCounts({
|
|
143
|
+
headerValue,
|
|
144
|
+
syntheticRunHeader: '1',
|
|
145
|
+
});
|
|
146
|
+
if (parsed.ok === false) {
|
|
147
|
+
throw new Error(parsed.error);
|
|
148
|
+
}
|
|
149
|
+
state = {
|
|
150
|
+
headerValue,
|
|
151
|
+
counts:
|
|
152
|
+
(parsed.counts as Partial<Record<RuntimeTestFaultName, number>>) ?? {},
|
|
153
|
+
};
|
|
154
|
+
directRuntimeTestFaults.set(stateKey, state);
|
|
155
|
+
}
|
|
156
|
+
const count = state.counts[input.name] ?? 0;
|
|
157
|
+
if (count <= 0) return false;
|
|
158
|
+
if (count === 1) {
|
|
159
|
+
delete state.counts[input.name];
|
|
160
|
+
} else {
|
|
161
|
+
state.counts[input.name] = count - 1;
|
|
162
|
+
}
|
|
163
|
+
state.headerValue = headerValue;
|
|
164
|
+
directRuntimeTestFaults.set(stateKey, state);
|
|
165
|
+
return true;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function runtimeTestFaultError(name: RuntimeTestFaultName): Error {
|
|
169
|
+
const error = new Error(`Injected runtime test fault: ${name}.`);
|
|
170
|
+
if (
|
|
171
|
+
name === 'receipt_complete_write_fail' ||
|
|
172
|
+
name === 'receipt_fail_write_fail'
|
|
173
|
+
) {
|
|
174
|
+
error.name = 'RuntimeReceiptPersistenceError';
|
|
175
|
+
}
|
|
176
|
+
return error;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function forceFailRuntimeWorkReceiptForRuntimeTestFault(
|
|
180
|
+
context: RuntimeApiContext,
|
|
181
|
+
input: {
|
|
182
|
+
playName: string;
|
|
183
|
+
runId: string;
|
|
184
|
+
key: string;
|
|
185
|
+
error: string;
|
|
186
|
+
failureKind: WorkReceiptFailureKind;
|
|
187
|
+
runAttempt?: number | null;
|
|
188
|
+
},
|
|
189
|
+
): Promise<WorkReceipt | null> {
|
|
190
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
191
|
+
const session = await getRuntimeWorkReceiptSession(context, {
|
|
192
|
+
playName: input.playName,
|
|
193
|
+
key: input.key,
|
|
194
|
+
});
|
|
195
|
+
return await withRuntimeWorkReceiptClient(
|
|
196
|
+
context,
|
|
197
|
+
session,
|
|
198
|
+
async (client) => {
|
|
199
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
200
|
+
`
|
|
201
|
+
WITH failed AS (
|
|
202
|
+
UPDATE ${workReceiptTable(session)}
|
|
203
|
+
SET status = $2::smallint,
|
|
204
|
+
output = NULL,
|
|
205
|
+
error = $3,
|
|
206
|
+
failure_kind = $5::smallint,
|
|
207
|
+
lease_id = NULL,
|
|
208
|
+
lease_owner_run_id = NULL,
|
|
209
|
+
lease_owner_attempt = NULL,
|
|
210
|
+
lease_expires_at = NULL,
|
|
211
|
+
updated_at = now()
|
|
212
|
+
WHERE k = decode($1, 'hex')
|
|
213
|
+
AND run_id = $4
|
|
214
|
+
AND COALESCE(lease_owner_attempt, 0) = $7::integer
|
|
215
|
+
AND status <> $6::smallint
|
|
216
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
217
|
+
status,
|
|
218
|
+
output,
|
|
219
|
+
error,
|
|
220
|
+
failure_kind,
|
|
221
|
+
run_id,
|
|
222
|
+
lease_id,
|
|
223
|
+
lease_owner_run_id,
|
|
224
|
+
lease_owner_attempt,
|
|
225
|
+
lease_expires_at,
|
|
226
|
+
updated_at
|
|
227
|
+
)
|
|
228
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at
|
|
229
|
+
FROM failed
|
|
230
|
+
`,
|
|
231
|
+
[
|
|
232
|
+
workReceiptKeyHex(input.key),
|
|
233
|
+
RECEIPT_STATUS_FAILED,
|
|
234
|
+
input.error,
|
|
235
|
+
input.runId,
|
|
236
|
+
workReceiptFailureKindCode(input.failureKind),
|
|
237
|
+
RECEIPT_STATUS_COMPLETED,
|
|
238
|
+
runAttempt,
|
|
239
|
+
],
|
|
240
|
+
);
|
|
241
|
+
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
242
|
+
},
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
94
246
|
type DbSessionCacheEntry = {
|
|
95
247
|
session: CreateDbSessionResponse;
|
|
96
248
|
};
|
|
@@ -101,6 +253,11 @@ type RuntimeDatasetRowEntry = {
|
|
|
101
253
|
row: Record<string, unknown>;
|
|
102
254
|
inputIndex: number;
|
|
103
255
|
};
|
|
256
|
+
type RuntimeSheetPrepareDisposition = 'pending' | 'completed' | 'blocked';
|
|
257
|
+
type RuntimeSheetPreparedRowDisposition = {
|
|
258
|
+
key: string;
|
|
259
|
+
disposition: RuntimeSheetPrepareDisposition;
|
|
260
|
+
};
|
|
104
261
|
const dbSessionCache = new Map<string, DbSessionCacheEntry>();
|
|
105
262
|
const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
|
|
106
263
|
const postgresPools = new Map<string, RuntimePool>();
|
|
@@ -119,6 +276,23 @@ const APPEND_KEY_SUFFIX_LENGTH = 12;
|
|
|
119
276
|
// 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
|
|
120
277
|
// the tenant runtime role is long-lived), not run failures.
|
|
121
278
|
const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
|
|
279
|
+
const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
|
|
280
|
+
'lease_id',
|
|
281
|
+
'lease_owner_run_id',
|
|
282
|
+
'lease_owner_attempt',
|
|
283
|
+
'lease_expires_at',
|
|
284
|
+
] as const;
|
|
285
|
+
const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
|
|
286
|
+
const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
|
|
287
|
+
...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
|
|
288
|
+
RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
|
|
289
|
+
] as const;
|
|
290
|
+
const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
|
|
291
|
+
'_attempt_id',
|
|
292
|
+
'_attempt_owner_run_id',
|
|
293
|
+
'_attempt_seq',
|
|
294
|
+
'_attempt_expires_at',
|
|
295
|
+
] as const;
|
|
122
296
|
const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
|
|
123
297
|
const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [
|
|
124
298
|
100, 250, 500, 1_000, 2_000,
|
|
@@ -178,8 +352,13 @@ export type PrepareRuntimeSheetResult = {
|
|
|
178
352
|
skipped: number;
|
|
179
353
|
pendingRows: Record<string, unknown>[];
|
|
180
354
|
completedRows: Record<string, unknown>[];
|
|
355
|
+
blockedRows: Record<string, unknown>[];
|
|
181
356
|
tableNamespace: string;
|
|
182
357
|
timings?: RuntimeSheetTiming[];
|
|
358
|
+
attemptId?: string;
|
|
359
|
+
attemptOwnerRunId?: string;
|
|
360
|
+
attemptExpiresAt?: string;
|
|
361
|
+
attemptSeq?: number;
|
|
183
362
|
};
|
|
184
363
|
|
|
185
364
|
export type RuntimeSheetTiming = {
|
|
@@ -197,6 +376,22 @@ export type RuntimeSheetTiming = {
|
|
|
197
376
|
error?: string;
|
|
198
377
|
};
|
|
199
378
|
|
|
379
|
+
export type RuntimeSheetAttemptHeartbeatResult = {
|
|
380
|
+
renewed: number;
|
|
381
|
+
renewedKeys: string[];
|
|
382
|
+
attemptExpiresAt: string | null;
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
export type RuntimeSheetAttemptReleaseResult = {
|
|
386
|
+
released: number;
|
|
387
|
+
releasedKeys: string[];
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
export type RuntimeWorkReceiptReleaseResult = {
|
|
391
|
+
released: number;
|
|
392
|
+
releasedKeys: string[];
|
|
393
|
+
};
|
|
394
|
+
|
|
200
395
|
type RuntimeApiActionRequest =
|
|
201
396
|
| {
|
|
202
397
|
action: 'resolve_play';
|
|
@@ -261,7 +456,7 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
|
|
|
261
456
|
throw new Error('Runner runtime API requires a baseUrl.');
|
|
262
457
|
}
|
|
263
458
|
const url = new URL(
|
|
264
|
-
`${baseUrl.replace(/\/$/, '')}/api/v2/
|
|
459
|
+
`${baseUrl.replace(/\/$/, '')}/api/v2/internal/play-runtime`,
|
|
265
460
|
);
|
|
266
461
|
const bypassToken = context.vercelProtectionBypassToken?.trim();
|
|
267
462
|
if (bypassToken) {
|
|
@@ -282,9 +477,16 @@ function resolveRuntimeApiHeaders(
|
|
|
282
477
|
return {
|
|
283
478
|
'content-type': 'application/json',
|
|
284
479
|
authorization: `Bearer ${token}`,
|
|
480
|
+
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
285
481
|
...(context.vercelProtectionBypassToken
|
|
286
482
|
? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
|
|
287
483
|
: {}),
|
|
484
|
+
...(context.runtimeTestFaultHeader?.trim()
|
|
485
|
+
? {
|
|
486
|
+
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
487
|
+
context.runtimeTestFaultHeader.trim(),
|
|
488
|
+
}
|
|
489
|
+
: {}),
|
|
288
490
|
...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
|
|
289
491
|
};
|
|
290
492
|
}
|
|
@@ -516,20 +718,36 @@ async function isRuntimeSheetSchemaReady(
|
|
|
516
718
|
physicalColumns.length > 0
|
|
517
719
|
? physicalColumns.map(quoteIdentifier).join(', ')
|
|
518
720
|
: '1';
|
|
519
|
-
const query = async (client: RuntimeQueryClient): Promise<
|
|
721
|
+
const query = async (client: RuntimeQueryClient): Promise<boolean> => {
|
|
520
722
|
await client.query(
|
|
521
723
|
`SELECT ${selectList}
|
|
522
724
|
FROM ${sheetTable(session)}
|
|
523
725
|
LIMIT 0`,
|
|
524
726
|
);
|
|
727
|
+
const attemptColumns = await client.query<Record<string, unknown>>(
|
|
728
|
+
`
|
|
729
|
+
SELECT count(*)::int AS present_count
|
|
730
|
+
FROM information_schema.columns
|
|
731
|
+
WHERE table_schema = $1
|
|
732
|
+
AND table_name = $2
|
|
733
|
+
AND column_name = ANY($3::text[])
|
|
734
|
+
`,
|
|
735
|
+
[
|
|
736
|
+
session.postgres.schema,
|
|
737
|
+
session.postgres.sheetTable,
|
|
738
|
+
[...RUNTIME_SHEET_ATTEMPT_COLUMNS],
|
|
739
|
+
],
|
|
740
|
+
);
|
|
741
|
+
return (
|
|
742
|
+
Number(attemptColumns.rows[0]?.present_count ?? 0) ===
|
|
743
|
+
RUNTIME_SHEET_ATTEMPT_COLUMNS.length
|
|
744
|
+
);
|
|
525
745
|
};
|
|
526
746
|
try {
|
|
527
747
|
if (isRuntimeOneShotQueryFactoryRegistered()) {
|
|
528
|
-
await withRuntimeOneShotPostgres(session, query);
|
|
529
|
-
} else {
|
|
530
|
-
await withRuntimePostgres(session, query);
|
|
748
|
+
return await withRuntimeOneShotPostgres(session, query);
|
|
531
749
|
}
|
|
532
|
-
return
|
|
750
|
+
return await withRuntimePostgres(session, query);
|
|
533
751
|
} catch (error) {
|
|
534
752
|
if (isMissingRelationError(error)) {
|
|
535
753
|
return false;
|
|
@@ -1530,6 +1748,10 @@ function workReceiptKeyHex(key: string): string {
|
|
|
1530
1748
|
).join('');
|
|
1531
1749
|
}
|
|
1532
1750
|
|
|
1751
|
+
function newRuntimeWorkReceiptLeaseId(): string {
|
|
1752
|
+
return `receipt-lease:${randomUUID()}`;
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1533
1755
|
function validateRuntimeWorkReceiptKeyScope(
|
|
1534
1756
|
session: RuntimePostgresSession,
|
|
1535
1757
|
input: { key: string },
|
|
@@ -1548,22 +1770,86 @@ function validateRuntimeWorkReceiptKeyScope(
|
|
|
1548
1770
|
}
|
|
1549
1771
|
|
|
1550
1772
|
function mapRuntimeWorkReceiptRow(raw: Record<string, unknown>): WorkReceipt {
|
|
1773
|
+
const leaseExpiresAt = raw.lease_expires_at;
|
|
1551
1774
|
return {
|
|
1552
1775
|
key: String(raw.k ?? ''),
|
|
1553
1776
|
status: receiptStatusFromCode(raw.status),
|
|
1554
1777
|
output: raw.output == null ? null : raw.output,
|
|
1555
1778
|
error: raw.error == null ? null : String(raw.error),
|
|
1779
|
+
failureKind: workReceiptFailureKindFromCode(raw.failure_kind),
|
|
1556
1780
|
runId: raw.run_id == null ? null : String(raw.run_id),
|
|
1781
|
+
leaseId: raw.lease_id == null ? null : String(raw.lease_id),
|
|
1782
|
+
leaseOwnerRunId:
|
|
1783
|
+
raw.lease_owner_run_id == null ? null : String(raw.lease_owner_run_id),
|
|
1784
|
+
leaseOwnerAttempt:
|
|
1785
|
+
raw.lease_owner_attempt == null ||
|
|
1786
|
+
!Number.isFinite(Number(raw.lease_owner_attempt))
|
|
1787
|
+
? null
|
|
1788
|
+
: Number(raw.lease_owner_attempt),
|
|
1789
|
+
leaseExpiresAt:
|
|
1790
|
+
leaseExpiresAt instanceof Date
|
|
1791
|
+
? leaseExpiresAt.toISOString()
|
|
1792
|
+
: leaseExpiresAt == null
|
|
1793
|
+
? null
|
|
1794
|
+
: String(leaseExpiresAt),
|
|
1557
1795
|
updatedAt: raw.updated_at == null ? null : String(raw.updated_at),
|
|
1558
1796
|
};
|
|
1559
1797
|
}
|
|
1560
1798
|
|
|
1799
|
+
function isMissingRuntimeWorkReceiptSelfHealColumnError(
|
|
1800
|
+
error: unknown,
|
|
1801
|
+
): boolean {
|
|
1802
|
+
if (!error || typeof error !== 'object') {
|
|
1803
|
+
return false;
|
|
1804
|
+
}
|
|
1805
|
+
const code = 'code' in error ? String(error.code) : '';
|
|
1806
|
+
const message = 'message' in error ? String(error.message) : '';
|
|
1807
|
+
if (code !== '42703' && !/column .* does not exist/i.test(message)) {
|
|
1808
|
+
return false;
|
|
1809
|
+
}
|
|
1810
|
+
return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.some((column) =>
|
|
1811
|
+
message.includes(column),
|
|
1812
|
+
);
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1561
1815
|
function runtimeWorkReceiptEnsureCacheKey(
|
|
1562
1816
|
session: RuntimePostgresSession,
|
|
1563
1817
|
): string {
|
|
1564
1818
|
return `${postgresUrlCacheKey(session.postgresUrl)}::${session.postgres.schema}::${session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE}`;
|
|
1565
1819
|
}
|
|
1566
1820
|
|
|
1821
|
+
async function missingRuntimeWorkReceiptSelfHealColumns(
|
|
1822
|
+
session: RuntimePostgresSession,
|
|
1823
|
+
client: RuntimeQueryClient,
|
|
1824
|
+
): Promise<string[]> {
|
|
1825
|
+
const result = await client.query(
|
|
1826
|
+
`
|
|
1827
|
+
SELECT column_name
|
|
1828
|
+
FROM information_schema.columns
|
|
1829
|
+
WHERE table_schema = $1
|
|
1830
|
+
AND table_name = $2
|
|
1831
|
+
AND column_name = ANY($3::text[])
|
|
1832
|
+
`,
|
|
1833
|
+
[
|
|
1834
|
+
session.postgres.schema,
|
|
1835
|
+
session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
|
|
1836
|
+
[...RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS],
|
|
1837
|
+
],
|
|
1838
|
+
);
|
|
1839
|
+
const present = new Set(
|
|
1840
|
+
result.rows.map((row) => String(row.column_name ?? '')),
|
|
1841
|
+
);
|
|
1842
|
+
return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.filter(
|
|
1843
|
+
(column) => !present.has(column),
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
// COMPAT / SELF-HEAL — owner: runtime. Removal milestone: execution-ledger
|
|
1848
|
+
// cutover M1 (docs/play-runtime-execution-ledger-plan.md). The customer-Neon
|
|
1849
|
+
// work-receipt table (lease columns + failure_kind) is the dying surface. The
|
|
1850
|
+
// F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
|
|
1851
|
+
// columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
|
|
1852
|
+
// deleted together with the Neon receipt path at M1.
|
|
1567
1853
|
async function ensureRuntimeWorkReceiptTable(
|
|
1568
1854
|
session: RuntimePostgresSession,
|
|
1569
1855
|
client: RuntimeQueryClient,
|
|
@@ -1582,11 +1868,39 @@ async function ensureRuntimeWorkReceiptTable(
|
|
|
1582
1868
|
status smallint NOT NULL DEFAULT 0,
|
|
1583
1869
|
output jsonb,
|
|
1584
1870
|
error text,
|
|
1871
|
+
failure_kind smallint NOT NULL DEFAULT 0,
|
|
1585
1872
|
run_id text,
|
|
1873
|
+
lease_id text,
|
|
1874
|
+
lease_owner_run_id text,
|
|
1875
|
+
lease_owner_attempt integer,
|
|
1876
|
+
lease_expires_at timestamptz,
|
|
1586
1877
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
1587
1878
|
)
|
|
1588
1879
|
`,
|
|
1589
1880
|
)
|
|
1881
|
+
.then(async () => {
|
|
1882
|
+
const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
|
|
1883
|
+
session,
|
|
1884
|
+
client,
|
|
1885
|
+
);
|
|
1886
|
+
if (missingColumns.length === 0) return;
|
|
1887
|
+
await client.query(`
|
|
1888
|
+
ALTER TABLE ${workReceiptTable(session)}
|
|
1889
|
+
${missingColumns
|
|
1890
|
+
.map((column) => {
|
|
1891
|
+
const type =
|
|
1892
|
+
column === 'lease_expires_at'
|
|
1893
|
+
? 'timestamptz'
|
|
1894
|
+
: column === 'lease_owner_attempt'
|
|
1895
|
+
? 'integer'
|
|
1896
|
+
: column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
|
|
1897
|
+
? 'smallint NOT NULL DEFAULT 0'
|
|
1898
|
+
: 'text';
|
|
1899
|
+
return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
|
|
1900
|
+
})
|
|
1901
|
+
.join(',\n ')}
|
|
1902
|
+
`);
|
|
1903
|
+
})
|
|
1590
1904
|
.then(() => undefined);
|
|
1591
1905
|
runtimeWorkReceiptEnsureCache.set(cacheKey, promise);
|
|
1592
1906
|
try {
|
|
@@ -1612,11 +1926,26 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
1612
1926
|
try {
|
|
1613
1927
|
return await operation(client);
|
|
1614
1928
|
} catch (error) {
|
|
1615
|
-
if (
|
|
1929
|
+
if (
|
|
1930
|
+
isMissingRelationError(error) ||
|
|
1931
|
+
isMissingRuntimeWorkReceiptSelfHealColumnError(error)
|
|
1932
|
+
) {
|
|
1616
1933
|
runtimeWorkReceiptEnsureCache.delete(
|
|
1617
1934
|
runtimeWorkReceiptEnsureCacheKey(session),
|
|
1618
1935
|
);
|
|
1619
|
-
|
|
1936
|
+
try {
|
|
1937
|
+
await ensureRuntimeWorkReceiptTable(session, client);
|
|
1938
|
+
} catch (ensureError) {
|
|
1939
|
+
if (!isPostgresPermissionDeniedError(ensureError)) {
|
|
1940
|
+
throw ensureError;
|
|
1941
|
+
}
|
|
1942
|
+
runtimeWorkReceiptEnsureCache.delete(
|
|
1943
|
+
runtimeWorkReceiptEnsureCacheKey(session),
|
|
1944
|
+
);
|
|
1945
|
+
await repairRuntimeStorageGrants(context, {
|
|
1946
|
+
playName: context.playName?.trim() || session.playName,
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1620
1949
|
} else if (isPostgresPermissionDeniedError(error)) {
|
|
1621
1950
|
await repairRuntimeStorageGrants(context, {
|
|
1622
1951
|
playName: context.playName?.trim() || session.playName,
|
|
@@ -1715,6 +2044,10 @@ function isSystemSheetColumn(columnName: string): boolean {
|
|
|
1715
2044
|
case '_stage':
|
|
1716
2045
|
case '_provider':
|
|
1717
2046
|
case '_input_index':
|
|
2047
|
+
case '_attempt_id':
|
|
2048
|
+
case '_attempt_owner_run_id':
|
|
2049
|
+
case '_attempt_expires_at':
|
|
2050
|
+
case '_attempt_seq':
|
|
1718
2051
|
case '_created_at':
|
|
1719
2052
|
case '_updated_at':
|
|
1720
2053
|
case '_version':
|
|
@@ -1732,6 +2065,64 @@ function isSystemSheetColumn(columnName: string): boolean {
|
|
|
1732
2065
|
}
|
|
1733
2066
|
}
|
|
1734
2067
|
|
|
2068
|
+
function normalizeRuntimeSheetAttemptId(
|
|
2069
|
+
value: string | null | undefined,
|
|
2070
|
+
): string {
|
|
2071
|
+
const trimmed = value?.trim();
|
|
2072
|
+
return trimmed || `sheet-attempt:${randomUUID()}`;
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
function normalizeRuntimeSheetAttemptOwnerRunId(input: {
|
|
2076
|
+
attemptOwnerRunId?: string | null;
|
|
2077
|
+
runId: string;
|
|
2078
|
+
}): string {
|
|
2079
|
+
return input.attemptOwnerRunId?.trim() || input.runId;
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
function normalizeRuntimeRunAttempt(value: number | null | undefined): number {
|
|
2083
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) return 0;
|
|
2084
|
+
return Math.max(0, Math.floor(value));
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
function normalizeRuntimeLeaseTtlMs(
|
|
2088
|
+
value: number | null | undefined,
|
|
2089
|
+
fallback: number,
|
|
2090
|
+
): number {
|
|
2091
|
+
if (
|
|
2092
|
+
typeof value !== 'number' ||
|
|
2093
|
+
!Number.isFinite(value) ||
|
|
2094
|
+
!Number.isInteger(value) ||
|
|
2095
|
+
value <= 0
|
|
2096
|
+
) {
|
|
2097
|
+
return fallback;
|
|
2098
|
+
}
|
|
2099
|
+
return value;
|
|
2100
|
+
}
|
|
2101
|
+
|
|
2102
|
+
async function mintRuntimeSheetAttemptExpiresAt(
|
|
2103
|
+
client: RuntimeQueryClient,
|
|
2104
|
+
leaseTtlMs?: number | null,
|
|
2105
|
+
): Promise<string> {
|
|
2106
|
+
const ttlMs = normalizeRuntimeLeaseTtlMs(
|
|
2107
|
+
leaseTtlMs,
|
|
2108
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
2109
|
+
);
|
|
2110
|
+
const { rows } = await client.query<{ attempt_expires_at: Date | string }>(
|
|
2111
|
+
`
|
|
2112
|
+
SELECT now() + ($1::double precision * interval '1 millisecond') AS attempt_expires_at
|
|
2113
|
+
`,
|
|
2114
|
+
[ttlMs],
|
|
2115
|
+
);
|
|
2116
|
+
const value = rows[0]?.attempt_expires_at;
|
|
2117
|
+
if (value instanceof Date) {
|
|
2118
|
+
return value.toISOString();
|
|
2119
|
+
}
|
|
2120
|
+
if (typeof value === 'string' && value.trim()) {
|
|
2121
|
+
return new Date(value).toISOString();
|
|
2122
|
+
}
|
|
2123
|
+
throw new Error('Runtime sheet attempt expiry mint did not return a value.');
|
|
2124
|
+
}
|
|
2125
|
+
|
|
1735
2126
|
function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
|
|
1736
2127
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
1737
2128
|
return value as Record<string, unknown>;
|
|
@@ -2228,6 +2619,10 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2228
2619
|
runId: string;
|
|
2229
2620
|
normalizedPlayName: string;
|
|
2230
2621
|
normalizedTableNamespace: string;
|
|
2622
|
+
attemptId: string;
|
|
2623
|
+
attemptOwnerRunId: string;
|
|
2624
|
+
attemptExpiresAt: string;
|
|
2625
|
+
attemptSeq: number;
|
|
2231
2626
|
physicalInsertColumnsSql: string;
|
|
2232
2627
|
physicalInsertValuesSql: string;
|
|
2233
2628
|
physicalRefreshSetSql: string;
|
|
@@ -2235,9 +2630,12 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2235
2630
|
outputPhysicalColumns: PhysicalSheetColumnProjection[];
|
|
2236
2631
|
force?: boolean;
|
|
2237
2632
|
},
|
|
2238
|
-
): Promise<{
|
|
2633
|
+
): Promise<{
|
|
2634
|
+
inserted: number;
|
|
2635
|
+
rowDispositions: RuntimeSheetPreparedRowDisposition[];
|
|
2636
|
+
}> {
|
|
2239
2637
|
let inserted = 0;
|
|
2240
|
-
const
|
|
2638
|
+
const rowDispositions: RuntimeSheetPreparedRowDisposition[] = [];
|
|
2241
2639
|
for (const chunk of input.chunks) {
|
|
2242
2640
|
const chunkKeys = chunk.map((entry) => entry.key);
|
|
2243
2641
|
const chunkPayloads = chunk.map((entry) =>
|
|
@@ -2252,6 +2650,41 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2252
2650
|
'target',
|
|
2253
2651
|
input.outputPhysicalColumns,
|
|
2254
2652
|
);
|
|
2653
|
+
const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
2654
|
+
'target',
|
|
2655
|
+
'$7::text',
|
|
2656
|
+
'$8::text',
|
|
2657
|
+
'$9::timestamptz',
|
|
2658
|
+
'$10::integer',
|
|
2659
|
+
);
|
|
2660
|
+
const existingAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
2661
|
+
'existing',
|
|
2662
|
+
'$7::text',
|
|
2663
|
+
'$8::text',
|
|
2664
|
+
'$9::timestamptz',
|
|
2665
|
+
'$10::integer',
|
|
2666
|
+
);
|
|
2667
|
+
const targetNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
|
|
2668
|
+
'target',
|
|
2669
|
+
'$9::timestamptz',
|
|
2670
|
+
'$10::integer',
|
|
2671
|
+
'$7::text',
|
|
2672
|
+
'$8::text',
|
|
2673
|
+
);
|
|
2674
|
+
const existingNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
|
|
2675
|
+
'existing',
|
|
2676
|
+
'$9::timestamptz',
|
|
2677
|
+
'$10::integer',
|
|
2678
|
+
'$7::text',
|
|
2679
|
+
'$8::text',
|
|
2680
|
+
);
|
|
2681
|
+
const existingWritableEnrichedAttemptSql =
|
|
2682
|
+
writableEnrichedRuntimeSheetAttemptSql(
|
|
2683
|
+
'existing',
|
|
2684
|
+
'$8::text',
|
|
2685
|
+
'$9::timestamptz',
|
|
2686
|
+
'$10',
|
|
2687
|
+
);
|
|
2255
2688
|
const { rows } = await client.query(
|
|
2256
2689
|
`
|
|
2257
2690
|
WITH input_rows AS (
|
|
@@ -2273,19 +2706,38 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2273
2706
|
WHERE target._key = input_rows._key
|
|
2274
2707
|
AND target._status <> 'stale'
|
|
2275
2708
|
AND (
|
|
2276
|
-
$
|
|
2709
|
+
NOT (${targetNewerTerminalRowSql})
|
|
2710
|
+
OR (
|
|
2711
|
+
$11::boolean
|
|
2712
|
+
AND target._status = 'enriched'
|
|
2713
|
+
AND COALESCE(target._attempt_seq, 0) <= $10::integer
|
|
2714
|
+
)
|
|
2715
|
+
)
|
|
2716
|
+
AND (
|
|
2717
|
+
${targetAttemptFenceSql}
|
|
2718
|
+
OR (
|
|
2719
|
+
$11::boolean
|
|
2720
|
+
AND target._status = 'enriched'
|
|
2721
|
+
)
|
|
2722
|
+
)
|
|
2723
|
+
AND (
|
|
2724
|
+
$11::boolean
|
|
2277
2725
|
OR target._input_index IS DISTINCT FROM input_rows._input_index
|
|
2278
2726
|
)
|
|
2279
2727
|
RETURNING target._key
|
|
2280
2728
|
),
|
|
2281
2729
|
inserted_rows AS (
|
|
2282
|
-
INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${input.physicalInsertColumnsSql})
|
|
2283
|
-
SELECT _key, 'pending', $4::text, _input_index${input.physicalInsertValuesSql}
|
|
2730
|
+
INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
|
|
2731
|
+
SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
|
|
2284
2732
|
FROM input_rows
|
|
2285
2733
|
ON CONFLICT (_key) DO UPDATE SET
|
|
2286
2734
|
_status = 'pending',
|
|
2287
2735
|
_run_id = EXCLUDED._run_id,
|
|
2288
2736
|
_input_index = EXCLUDED._input_index,
|
|
2737
|
+
_attempt_id = EXCLUDED._attempt_id,
|
|
2738
|
+
_attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
|
|
2739
|
+
_attempt_expires_at = EXCLUDED._attempt_expires_at,
|
|
2740
|
+
_attempt_seq = EXCLUDED._attempt_seq,
|
|
2289
2741
|
_updated_at = now(),
|
|
2290
2742
|
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
|
|
2291
2743
|
WHERE ${sheetTable(session)}._status = 'stale'
|
|
@@ -2296,26 +2748,42 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2296
2748
|
SET _status = 'pending',
|
|
2297
2749
|
_run_id = $4::text,
|
|
2298
2750
|
_input_index = input_rows._input_index,
|
|
2751
|
+
_attempt_id = $7::text,
|
|
2752
|
+
_attempt_owner_run_id = $8::text,
|
|
2753
|
+
_attempt_expires_at = $9::timestamptz,
|
|
2754
|
+
_attempt_seq = $10::integer,
|
|
2299
2755
|
_updated_at = now(),
|
|
2300
2756
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2301
2757
|
FROM input_rows
|
|
2302
2758
|
WHERE target._key = input_rows._key
|
|
2303
2759
|
AND target._status = 'enriched'
|
|
2304
2760
|
AND (${targetMissingOutputSql})
|
|
2761
|
+
AND COALESCE(target._attempt_seq, 0) <= $10::integer
|
|
2305
2762
|
RETURNING target._key
|
|
2306
2763
|
),
|
|
2307
2764
|
claimed_existing_rows AS (
|
|
2308
2765
|
UPDATE ${sheetTable(session)} AS target
|
|
2309
2766
|
SET _run_id = $4::text,
|
|
2310
2767
|
_input_index = input_rows._input_index,
|
|
2768
|
+
_attempt_id = $7::text,
|
|
2769
|
+
_attempt_owner_run_id = $8::text,
|
|
2770
|
+
_attempt_expires_at = $9::timestamptz,
|
|
2771
|
+
_attempt_seq = $10::integer,
|
|
2311
2772
|
_updated_at = now(),
|
|
2312
2773
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2313
2774
|
FROM input_rows
|
|
2314
2775
|
WHERE target._key = input_rows._key
|
|
2315
2776
|
AND target._status IN ('pending', 'running', 'failed')
|
|
2777
|
+
AND NOT (${targetNewerTerminalRowSql})
|
|
2778
|
+
AND (
|
|
2779
|
+
target._status = 'failed'
|
|
2780
|
+
OR ${targetAttemptFenceSql}
|
|
2781
|
+
)
|
|
2316
2782
|
AND (
|
|
2317
2783
|
target._run_id IS DISTINCT FROM $4::text
|
|
2318
2784
|
OR target._input_index IS DISTINCT FROM input_rows._input_index
|
|
2785
|
+
OR target._attempt_id IS DISTINCT FROM $7::text
|
|
2786
|
+
OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $10::integer
|
|
2319
2787
|
)
|
|
2320
2788
|
RETURNING target._key
|
|
2321
2789
|
),
|
|
@@ -2329,15 +2797,61 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2329
2797
|
SELECT _key
|
|
2330
2798
|
FROM claimed_existing_rows
|
|
2331
2799
|
UNION
|
|
2800
|
+
SELECT _key
|
|
2801
|
+
FROM existing_rows
|
|
2802
|
+
WHERE $11::boolean
|
|
2803
|
+
UNION
|
|
2332
2804
|
SELECT existing._key
|
|
2333
2805
|
FROM ${sheetTable(session)} AS existing
|
|
2334
2806
|
JOIN input_rows ON input_rows._key = existing._key
|
|
2335
|
-
WHERE
|
|
2807
|
+
WHERE (
|
|
2808
|
+
existing._status IN ('pending', 'running', 'failed')
|
|
2809
|
+
AND NOT (${existingNewerTerminalRowSql})
|
|
2810
|
+
AND (
|
|
2811
|
+
existing._status = 'failed'
|
|
2812
|
+
OR ${existingAttemptFenceSql}
|
|
2813
|
+
)
|
|
2814
|
+
)
|
|
2336
2815
|
OR (
|
|
2337
2816
|
existing._status = 'enriched'
|
|
2338
2817
|
AND (${existingMissingOutputSql})
|
|
2818
|
+
AND (
|
|
2819
|
+
${existingAttemptFenceSql}
|
|
2820
|
+
OR ${existingWritableEnrichedAttemptSql}
|
|
2821
|
+
)
|
|
2339
2822
|
)
|
|
2340
2823
|
),
|
|
2824
|
+
completed_rows AS (
|
|
2825
|
+
SELECT existing._key
|
|
2826
|
+
FROM ${sheetTable(session)} AS existing
|
|
2827
|
+
JOIN input_rows ON input_rows._key = existing._key
|
|
2828
|
+
WHERE existing._status = 'enriched'
|
|
2829
|
+
AND NOT ($11::boolean)
|
|
2830
|
+
AND NOT (${existingMissingOutputSql})
|
|
2831
|
+
AND NOT EXISTS (
|
|
2832
|
+
SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
|
|
2833
|
+
)
|
|
2834
|
+
),
|
|
2835
|
+
blocked_rows AS (
|
|
2836
|
+
SELECT input_rows._key
|
|
2837
|
+
FROM input_rows
|
|
2838
|
+
WHERE NOT EXISTS (
|
|
2839
|
+
SELECT 1 FROM pending_rows WHERE pending_rows._key = input_rows._key
|
|
2840
|
+
)
|
|
2841
|
+
AND NOT EXISTS (
|
|
2842
|
+
SELECT 1 FROM completed_rows WHERE completed_rows._key = input_rows._key
|
|
2843
|
+
)
|
|
2844
|
+
),
|
|
2845
|
+
row_dispositions AS (
|
|
2846
|
+
SELECT _key, 'pending'::text AS disposition
|
|
2847
|
+
FROM pending_rows
|
|
2848
|
+
UNION ALL
|
|
2849
|
+
SELECT _key, 'completed'::text AS disposition
|
|
2850
|
+
FROM completed_rows
|
|
2851
|
+
UNION ALL
|
|
2852
|
+
SELECT _key, 'blocked'::text AS disposition
|
|
2853
|
+
FROM blocked_rows
|
|
2854
|
+
),
|
|
2341
2855
|
inserted_count_cte AS (
|
|
2342
2856
|
SELECT count(*)::bigint AS c FROM inserted_rows
|
|
2343
2857
|
),
|
|
@@ -2377,7 +2891,13 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2377
2891
|
)
|
|
2378
2892
|
SELECT
|
|
2379
2893
|
(SELECT c::int FROM inserted_count_cte) AS inserted_count,
|
|
2380
|
-
coalesce(
|
|
2894
|
+
coalesce(
|
|
2895
|
+
(
|
|
2896
|
+
SELECT jsonb_agg(jsonb_build_object('key', _key, 'disposition', disposition))
|
|
2897
|
+
FROM row_dispositions
|
|
2898
|
+
),
|
|
2899
|
+
'[]'::jsonb
|
|
2900
|
+
) AS row_dispositions,
|
|
2381
2901
|
(SELECT count(*)::int FROM existing_rows) AS reordered_count,
|
|
2382
2902
|
(SELECT c::int FROM missing_output_count_cte) AS missing_output_count
|
|
2383
2903
|
`,
|
|
@@ -2388,15 +2908,30 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2388
2908
|
input.runId,
|
|
2389
2909
|
input.normalizedPlayName,
|
|
2390
2910
|
input.normalizedTableNamespace,
|
|
2911
|
+
input.attemptId,
|
|
2912
|
+
input.attemptOwnerRunId,
|
|
2913
|
+
input.attemptExpiresAt,
|
|
2914
|
+
input.attemptSeq,
|
|
2391
2915
|
input.force === true,
|
|
2392
2916
|
],
|
|
2393
2917
|
);
|
|
2394
2918
|
inserted += Number(rows[0]?.inserted_count ?? 0);
|
|
2395
|
-
if (Array.isArray(rows[0]?.
|
|
2396
|
-
|
|
2919
|
+
if (Array.isArray(rows[0]?.row_dispositions)) {
|
|
2920
|
+
for (const row of rows[0]
|
|
2921
|
+
?.row_dispositions as RuntimeSheetPreparedRowDisposition[]) {
|
|
2922
|
+
if (
|
|
2923
|
+
row &&
|
|
2924
|
+
typeof row.key === 'string' &&
|
|
2925
|
+
(row.disposition === 'pending' ||
|
|
2926
|
+
row.disposition === 'completed' ||
|
|
2927
|
+
row.disposition === 'blocked')
|
|
2928
|
+
) {
|
|
2929
|
+
rowDispositions.push(row);
|
|
2930
|
+
}
|
|
2931
|
+
}
|
|
2397
2932
|
}
|
|
2398
2933
|
}
|
|
2399
|
-
return { inserted,
|
|
2934
|
+
return { inserted, rowDispositions };
|
|
2400
2935
|
}
|
|
2401
2936
|
|
|
2402
2937
|
async function buildRuntimeSheetDatasetStartResult(
|
|
@@ -2410,8 +2945,12 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2410
2945
|
normalizedPlayName: string;
|
|
2411
2946
|
normalizedTableNamespace: string;
|
|
2412
2947
|
runId: string;
|
|
2948
|
+
attemptId: string;
|
|
2949
|
+
attemptOwnerRunId: string;
|
|
2950
|
+
attemptExpiresAt: string;
|
|
2951
|
+
attemptSeq: number;
|
|
2413
2952
|
inserted: number;
|
|
2414
|
-
|
|
2953
|
+
rowDispositions: RuntimeSheetPreparedRowDisposition[];
|
|
2415
2954
|
force?: boolean;
|
|
2416
2955
|
timings?: RuntimeSheetTiming[];
|
|
2417
2956
|
},
|
|
@@ -2421,19 +2960,20 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2421
2960
|
? [column.field]
|
|
2422
2961
|
: [],
|
|
2423
2962
|
);
|
|
2424
|
-
const
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
row[field] = null;
|
|
2433
|
-
}
|
|
2963
|
+
const buildFreshRow = (entry: RuntimeDatasetRowEntry) => {
|
|
2964
|
+
const row: Record<string, unknown> = {
|
|
2965
|
+
...sanitizePostgresJsonValue(entry.row),
|
|
2966
|
+
__deeplineRowKey: entry.key,
|
|
2967
|
+
};
|
|
2968
|
+
for (const field of datasetFields) {
|
|
2969
|
+
if (!Object.prototype.hasOwnProperty.call(row, field)) {
|
|
2970
|
+
row[field] = null;
|
|
2434
2971
|
}
|
|
2435
|
-
|
|
2436
|
-
|
|
2972
|
+
}
|
|
2973
|
+
return row;
|
|
2974
|
+
};
|
|
2975
|
+
const buildFreshPendingRows = (entries = input.rowEntries) => {
|
|
2976
|
+
return entries.map(buildFreshRow);
|
|
2437
2977
|
};
|
|
2438
2978
|
|
|
2439
2979
|
if (input.inserted === input.rowEntries.length) {
|
|
@@ -2442,18 +2982,31 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2442
2982
|
skipped: input.sourceRowsLength - input.rowEntries.length,
|
|
2443
2983
|
pendingRows: buildFreshPendingRows(),
|
|
2444
2984
|
completedRows: [],
|
|
2985
|
+
blockedRows: [],
|
|
2445
2986
|
tableNamespace: input.tableNamespace,
|
|
2446
2987
|
};
|
|
2447
2988
|
}
|
|
2448
2989
|
|
|
2449
|
-
const
|
|
2990
|
+
const dispositionsByKey = new Map(
|
|
2991
|
+
input.rowDispositions.map((row) => [row.key, row.disposition]),
|
|
2992
|
+
);
|
|
2993
|
+
const pendingKeys = new Set(
|
|
2994
|
+
input.rowDispositions.flatMap((row) =>
|
|
2995
|
+
row.disposition === 'pending' ? [row.key] : [],
|
|
2996
|
+
),
|
|
2997
|
+
);
|
|
2450
2998
|
const startedAt = Date.now();
|
|
2451
2999
|
await markRuntimeRowsPendingForRecompute(client, session, {
|
|
2452
3000
|
keys: [...pendingKeys],
|
|
2453
3001
|
runId: input.runId,
|
|
3002
|
+
attemptId: input.attemptId,
|
|
3003
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
3004
|
+
attemptExpiresAt: input.attemptExpiresAt,
|
|
3005
|
+
attemptSeq: input.attemptSeq,
|
|
2454
3006
|
normalizedPlayName: input.normalizedPlayName,
|
|
2455
3007
|
normalizedTableNamespace: input.normalizedTableNamespace,
|
|
2456
3008
|
outputFields: input.force === true ? datasetFields : [],
|
|
3009
|
+
force: input.force === true,
|
|
2457
3010
|
});
|
|
2458
3011
|
if (pendingKeys.size > 0) {
|
|
2459
3012
|
input.timings?.push({
|
|
@@ -2463,11 +3016,22 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2463
3016
|
});
|
|
2464
3017
|
}
|
|
2465
3018
|
if (input.force === true) {
|
|
3019
|
+
const pendingRows: Record<string, unknown>[] = [];
|
|
3020
|
+
const blockedRows: Record<string, unknown>[] = [];
|
|
3021
|
+
for (const entry of input.rowEntries) {
|
|
3022
|
+
const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
|
|
3023
|
+
if (disposition === 'pending') {
|
|
3024
|
+
pendingRows.push(buildFreshRow(entry));
|
|
3025
|
+
} else if (disposition === 'blocked') {
|
|
3026
|
+
blockedRows.push(buildFreshRow(entry));
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
2466
3029
|
return {
|
|
2467
3030
|
inserted: input.inserted,
|
|
2468
3031
|
skipped: input.sourceRowsLength - input.rowEntries.length,
|
|
2469
|
-
pendingRows
|
|
3032
|
+
pendingRows,
|
|
2470
3033
|
completedRows: [],
|
|
3034
|
+
blockedRows,
|
|
2471
3035
|
tableNamespace: input.tableNamespace,
|
|
2472
3036
|
};
|
|
2473
3037
|
}
|
|
@@ -2498,14 +3062,25 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2498
3062
|
};
|
|
2499
3063
|
};
|
|
2500
3064
|
const pendingRows: Record<string, unknown>[] = [];
|
|
3065
|
+
const completedRows: Record<string, unknown>[] = [];
|
|
3066
|
+
const blockedRows: Record<string, unknown>[] = [];
|
|
2501
3067
|
for (const entry of input.rowEntries) {
|
|
2502
|
-
|
|
3068
|
+
const merged = buildMergedRow(entry);
|
|
3069
|
+
const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
|
|
3070
|
+
if (disposition === 'pending') {
|
|
3071
|
+
pendingRows.push(merged);
|
|
3072
|
+
} else if (disposition === 'completed') {
|
|
3073
|
+
completedRows.push(merged);
|
|
3074
|
+
} else {
|
|
3075
|
+
blockedRows.push(merged);
|
|
3076
|
+
}
|
|
2503
3077
|
}
|
|
2504
3078
|
return {
|
|
2505
3079
|
inserted: input.inserted,
|
|
2506
3080
|
skipped: input.sourceRowsLength - input.rowEntries.length,
|
|
2507
3081
|
pendingRows,
|
|
2508
|
-
completedRows
|
|
3082
|
+
completedRows,
|
|
3083
|
+
blockedRows,
|
|
2509
3084
|
tableNamespace: input.tableNamespace,
|
|
2510
3085
|
};
|
|
2511
3086
|
}
|
|
@@ -2516,12 +3091,31 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2516
3091
|
input: {
|
|
2517
3092
|
keys: string[];
|
|
2518
3093
|
runId: string;
|
|
3094
|
+
attemptId: string;
|
|
3095
|
+
attemptOwnerRunId: string;
|
|
3096
|
+
attemptExpiresAt: string;
|
|
3097
|
+
attemptSeq: number;
|
|
2519
3098
|
normalizedPlayName: string;
|
|
2520
3099
|
normalizedTableNamespace: string;
|
|
2521
3100
|
outputFields?: string[];
|
|
3101
|
+
force?: boolean;
|
|
2522
3102
|
},
|
|
2523
3103
|
): Promise<void> {
|
|
2524
3104
|
if (input.keys.length === 0) return;
|
|
3105
|
+
const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
3106
|
+
'target',
|
|
3107
|
+
'$5::text',
|
|
3108
|
+
'$6::text',
|
|
3109
|
+
'$7::timestamptz',
|
|
3110
|
+
'$8::integer',
|
|
3111
|
+
);
|
|
3112
|
+
const targetWritableEnrichedAttemptSql =
|
|
3113
|
+
writableEnrichedRuntimeSheetAttemptSql(
|
|
3114
|
+
'target',
|
|
3115
|
+
'$6::text',
|
|
3116
|
+
'$7::timestamptz',
|
|
3117
|
+
'$8',
|
|
3118
|
+
);
|
|
2525
3119
|
const outputFields = [...new Set(input.outputFields ?? [])];
|
|
2526
3120
|
await client.query(
|
|
2527
3121
|
`WITH target_rows AS (
|
|
@@ -2534,13 +3128,31 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2534
3128
|
UPDATE ${sheetTable(session)} AS target
|
|
2535
3129
|
SET _status = 'pending',
|
|
2536
3130
|
_run_id = $2::text,
|
|
3131
|
+
_attempt_id = $5::text,
|
|
3132
|
+
_attempt_owner_run_id = $6::text,
|
|
3133
|
+
_attempt_expires_at = $7::timestamptz,
|
|
3134
|
+
_attempt_seq = $8::integer,
|
|
2537
3135
|
_updated_at = now(),
|
|
2538
3136
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2539
3137
|
FROM target_rows
|
|
2540
3138
|
WHERE target._key = target_rows._key
|
|
3139
|
+
AND (
|
|
3140
|
+
${targetAttemptFenceSql}
|
|
3141
|
+
OR (
|
|
3142
|
+
$10::boolean
|
|
3143
|
+
AND target._status = 'enriched'
|
|
3144
|
+
AND COALESCE(target._attempt_seq, 0) <= $8::integer
|
|
3145
|
+
)
|
|
3146
|
+
OR (
|
|
3147
|
+
target._status = 'enriched'
|
|
3148
|
+
AND ${targetWritableEnrichedAttemptSql}
|
|
3149
|
+
)
|
|
3150
|
+
)
|
|
2541
3151
|
AND (
|
|
2542
3152
|
target._status <> 'pending'
|
|
2543
3153
|
OR target._run_id IS DISTINCT FROM $2::text
|
|
3154
|
+
OR target._attempt_id IS DISTINCT FROM $5::text
|
|
3155
|
+
OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $8::integer
|
|
2544
3156
|
)
|
|
2545
3157
|
RETURNING target_rows._status AS previous_status,
|
|
2546
3158
|
target_rows._cell_meta AS previous_cell_meta
|
|
@@ -2561,7 +3173,7 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2561
3173
|
WHERE previous_cell_meta -> field_values.field ->> 'status' = 'failed'
|
|
2562
3174
|
)::int AS failed_to_pending
|
|
2563
3175
|
FROM updated
|
|
2564
|
-
JOIN unnest($
|
|
3176
|
+
JOIN unnest($9::text[]) AS field_values(field) ON true
|
|
2565
3177
|
GROUP BY field_values.field
|
|
2566
3178
|
),
|
|
2567
3179
|
summary_delta AS (
|
|
@@ -2619,7 +3231,12 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2619
3231
|
input.runId,
|
|
2620
3232
|
input.normalizedPlayName,
|
|
2621
3233
|
input.normalizedTableNamespace,
|
|
3234
|
+
input.attemptId,
|
|
3235
|
+
input.attemptOwnerRunId,
|
|
3236
|
+
input.attemptExpiresAt,
|
|
3237
|
+
input.attemptSeq,
|
|
2622
3238
|
outputFields,
|
|
3239
|
+
input.force === true,
|
|
2623
3240
|
],
|
|
2624
3241
|
);
|
|
2625
3242
|
}
|
|
@@ -2657,6 +3274,41 @@ async function getRuntimeWorkReceiptSession(
|
|
|
2657
3274
|
return session;
|
|
2658
3275
|
}
|
|
2659
3276
|
|
|
3277
|
+
/**
|
|
3278
|
+
* Work receipts for a play share a single table, so run-scoped operations (the
|
|
3279
|
+
* run-fatal lease release) need a session without a specific key to scope by.
|
|
3280
|
+
* This mirrors `getRuntimeWorkReceiptSession` minus the per-key scope check.
|
|
3281
|
+
*/
|
|
3282
|
+
async function getRuntimeWorkReceiptSessionForRun(
|
|
3283
|
+
context: RuntimeApiContext,
|
|
3284
|
+
input: {
|
|
3285
|
+
playName: string;
|
|
3286
|
+
},
|
|
3287
|
+
): Promise<RuntimePostgresSession> {
|
|
3288
|
+
const playName = context.playName?.trim() || input.playName;
|
|
3289
|
+
if (!playName) {
|
|
3290
|
+
throw new Error('Runtime work receipts require a playName.');
|
|
3291
|
+
}
|
|
3292
|
+
const runtimeContext = {
|
|
3293
|
+
...context,
|
|
3294
|
+
playName,
|
|
3295
|
+
};
|
|
3296
|
+
const preloaded = findPreloadedRuntimeDbSession(runtimeContext, {
|
|
3297
|
+
tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
|
|
3298
|
+
logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
|
|
3299
|
+
operations: ['rows.read', 'rows.upsert'],
|
|
3300
|
+
});
|
|
3301
|
+
return requireRuntimePostgresSession(
|
|
3302
|
+
preloaded
|
|
3303
|
+
? await unwrapRuntimeDbSession(runtimeContext, preloaded)
|
|
3304
|
+
: await getRuntimeDbSession(runtimeContext, {
|
|
3305
|
+
tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
|
|
3306
|
+
logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
|
|
3307
|
+
operations: ['rows.read', 'rows.upsert'],
|
|
3308
|
+
}),
|
|
3309
|
+
);
|
|
3310
|
+
}
|
|
3311
|
+
|
|
2660
3312
|
async function getRuntimeWorkReceiptSessionForKeys(
|
|
2661
3313
|
context: RuntimeApiContext,
|
|
2662
3314
|
input: {
|
|
@@ -2684,7 +3336,17 @@ async function readRuntimeWorkReceipt(
|
|
|
2684
3336
|
key: string,
|
|
2685
3337
|
): Promise<WorkReceipt | null> {
|
|
2686
3338
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2687
|
-
`SELECT convert_from(k, 'UTF8') AS k,
|
|
3339
|
+
`SELECT convert_from(k, 'UTF8') AS k,
|
|
3340
|
+
status,
|
|
3341
|
+
output,
|
|
3342
|
+
error,
|
|
3343
|
+
failure_kind,
|
|
3344
|
+
run_id,
|
|
3345
|
+
lease_id,
|
|
3346
|
+
lease_owner_run_id,
|
|
3347
|
+
lease_owner_attempt,
|
|
3348
|
+
lease_expires_at,
|
|
3349
|
+
updated_at
|
|
2688
3350
|
FROM ${workReceiptTable(session)}
|
|
2689
3351
|
WHERE k = decode($1, 'hex')`,
|
|
2690
3352
|
[workReceiptKeyHex(key)],
|
|
@@ -2714,7 +3376,7 @@ export async function getRuntimeWorkReceipts(
|
|
|
2714
3376
|
playName: string;
|
|
2715
3377
|
keys: string[];
|
|
2716
3378
|
},
|
|
2717
|
-
): Promise<WorkReceipt
|
|
3379
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
2718
3380
|
const keys = [
|
|
2719
3381
|
...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
|
|
2720
3382
|
];
|
|
@@ -2737,16 +3399,23 @@ export async function getRuntimeWorkReceipts(
|
|
|
2737
3399
|
receipts.status,
|
|
2738
3400
|
receipts.output,
|
|
2739
3401
|
receipts.error,
|
|
3402
|
+
receipts.failure_kind,
|
|
2740
3403
|
receipts.run_id,
|
|
3404
|
+
receipts.lease_id,
|
|
3405
|
+
receipts.lease_owner_run_id,
|
|
3406
|
+
receipts.lease_owner_attempt,
|
|
3407
|
+
receipts.lease_expires_at,
|
|
2741
3408
|
receipts.updated_at
|
|
2742
3409
|
FROM input_keys
|
|
2743
|
-
JOIN ${workReceiptTable(session)} AS receipts
|
|
3410
|
+
LEFT JOIN ${workReceiptTable(session)} AS receipts
|
|
2744
3411
|
ON receipts.k = decode(input_keys.key_hex, 'hex')
|
|
2745
3412
|
ORDER BY input_keys.ord
|
|
2746
3413
|
`,
|
|
2747
3414
|
[keys.map(workReceiptKeyHex)],
|
|
2748
3415
|
);
|
|
2749
|
-
return rows.map(
|
|
3416
|
+
return rows.map((row) =>
|
|
3417
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
3418
|
+
);
|
|
2750
3419
|
},
|
|
2751
3420
|
);
|
|
2752
3421
|
}
|
|
@@ -2757,8 +3426,12 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2757
3426
|
playName: string;
|
|
2758
3427
|
runId: string;
|
|
2759
3428
|
key: string;
|
|
3429
|
+
runAttempt?: number | null;
|
|
3430
|
+
leaseAware?: boolean;
|
|
2760
3431
|
reclaimRunning?: boolean;
|
|
2761
3432
|
forceRefresh?: boolean;
|
|
3433
|
+
forceFailedRefresh?: boolean;
|
|
3434
|
+
leaseTtlMs?: number | null;
|
|
2762
3435
|
},
|
|
2763
3436
|
): Promise<WorkReceiptClaim> {
|
|
2764
3437
|
const session = await getRuntimeWorkReceiptSession(context, {
|
|
@@ -2769,36 +3442,89 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2769
3442
|
context,
|
|
2770
3443
|
session,
|
|
2771
3444
|
async (client) => {
|
|
2772
|
-
const
|
|
2773
|
-
?
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
: input.
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
RECEIPT_STATUS_RUNNING,
|
|
2785
|
-
]
|
|
2786
|
-
: [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
|
|
3445
|
+
const leaseId =
|
|
3446
|
+
input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null;
|
|
3447
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3448
|
+
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
3449
|
+
input.leaseTtlMs,
|
|
3450
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
3451
|
+
);
|
|
3452
|
+
const claimableStatuses = workReceiptClaimableStatusCodes({
|
|
3453
|
+
forceRefresh: input.forceRefresh,
|
|
3454
|
+
forceFailedRefresh: input.forceFailedRefresh,
|
|
3455
|
+
reclaimRunning: input.reclaimRunning,
|
|
3456
|
+
});
|
|
2787
3457
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2788
3458
|
`
|
|
2789
3459
|
WITH claimed AS (
|
|
2790
|
-
INSERT INTO ${workReceiptTable(session)} (
|
|
2791
|
-
|
|
3460
|
+
INSERT INTO ${workReceiptTable(session)} (
|
|
3461
|
+
k,
|
|
3462
|
+
status,
|
|
3463
|
+
run_id,
|
|
3464
|
+
lease_id,
|
|
3465
|
+
lease_owner_run_id,
|
|
3466
|
+
lease_owner_attempt,
|
|
3467
|
+
lease_expires_at,
|
|
3468
|
+
updated_at
|
|
3469
|
+
)
|
|
3470
|
+
VALUES (
|
|
3471
|
+
decode($1, 'hex'),
|
|
3472
|
+
$2::smallint,
|
|
3473
|
+
$3,
|
|
3474
|
+
$6,
|
|
3475
|
+
CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
|
|
3476
|
+
CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
|
|
3477
|
+
CASE
|
|
3478
|
+
WHEN $6::text IS NULL THEN NULL
|
|
3479
|
+
ELSE now() + ($7::double precision * interval '1 millisecond')
|
|
3480
|
+
END,
|
|
3481
|
+
now()
|
|
3482
|
+
)
|
|
2792
3483
|
ON CONFLICT (k) DO UPDATE
|
|
2793
3484
|
SET status = $4::smallint,
|
|
2794
3485
|
output = NULL,
|
|
3486
|
+
failure_kind = 0,
|
|
2795
3487
|
run_id = $3,
|
|
3488
|
+
lease_id = $6,
|
|
3489
|
+
lease_owner_run_id = CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
|
|
3490
|
+
lease_owner_attempt = CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
|
|
3491
|
+
lease_expires_at = CASE
|
|
3492
|
+
WHEN $6::text IS NULL THEN NULL
|
|
3493
|
+
ELSE now() + ($7::double precision * interval '1 millisecond')
|
|
3494
|
+
END,
|
|
2796
3495
|
error = NULL,
|
|
2797
3496
|
updated_at = now()
|
|
2798
|
-
WHERE ${
|
|
2799
|
-
|
|
3497
|
+
WHERE ${workReceiptClaimConflictPredicateSql({
|
|
3498
|
+
receiptTable: workReceiptTable(session),
|
|
3499
|
+
claimableStatusesSql: '$5',
|
|
3500
|
+
claimantRunIdSql: '$3',
|
|
3501
|
+
claimantRunAttemptSql: '$9',
|
|
3502
|
+
forceRefreshSql: '$8',
|
|
3503
|
+
forceFailedRefreshSql: '$10',
|
|
3504
|
+
})}
|
|
3505
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
3506
|
+
status,
|
|
3507
|
+
output,
|
|
3508
|
+
error,
|
|
3509
|
+
failure_kind,
|
|
3510
|
+
run_id,
|
|
3511
|
+
lease_id,
|
|
3512
|
+
lease_owner_run_id,
|
|
3513
|
+
lease_owner_attempt,
|
|
3514
|
+
lease_expires_at,
|
|
3515
|
+
updated_at
|
|
2800
3516
|
)
|
|
2801
|
-
SELECT k,
|
|
3517
|
+
SELECT k,
|
|
3518
|
+
status,
|
|
3519
|
+
output,
|
|
3520
|
+
error,
|
|
3521
|
+
failure_kind,
|
|
3522
|
+
run_id,
|
|
3523
|
+
lease_id,
|
|
3524
|
+
lease_owner_run_id,
|
|
3525
|
+
lease_owner_attempt,
|
|
3526
|
+
lease_expires_at,
|
|
3527
|
+
updated_at
|
|
2802
3528
|
FROM claimed
|
|
2803
3529
|
`,
|
|
2804
3530
|
[
|
|
@@ -2807,6 +3533,11 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2807
3533
|
input.runId,
|
|
2808
3534
|
RECEIPT_STATUS_RUNNING,
|
|
2809
3535
|
claimableStatuses,
|
|
3536
|
+
leaseId,
|
|
3537
|
+
leaseTtlMs,
|
|
3538
|
+
input.forceRefresh === true,
|
|
3539
|
+
runAttempt,
|
|
3540
|
+
input.forceFailedRefresh === true,
|
|
2810
3541
|
],
|
|
2811
3542
|
);
|
|
2812
3543
|
const claimed = rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -2815,14 +3546,19 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2815
3546
|
}
|
|
2816
3547
|
|
|
2817
3548
|
const latest = await readRuntimeWorkReceipt(client, session, input.key);
|
|
2818
|
-
if (
|
|
3549
|
+
if (
|
|
3550
|
+
latest &&
|
|
3551
|
+
input.forceRefresh !== true &&
|
|
3552
|
+
isReusableWorkReceipt(latest)
|
|
3553
|
+
) {
|
|
2819
3554
|
return { disposition: 'reused', receipt: latest };
|
|
2820
3555
|
}
|
|
2821
|
-
if (latest
|
|
2822
|
-
return {
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
3556
|
+
if (latest) {
|
|
3557
|
+
return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
|
|
3558
|
+
receipt: latest,
|
|
3559
|
+
claimantRunId: input.runId,
|
|
3560
|
+
claimantRunAttempt: runAttempt,
|
|
3561
|
+
});
|
|
2826
3562
|
}
|
|
2827
3563
|
throw new Error(
|
|
2828
3564
|
`Runtime receipt ${input.key} claim did not return execution ownership.`,
|
|
@@ -2831,14 +3567,48 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2831
3567
|
);
|
|
2832
3568
|
}
|
|
2833
3569
|
|
|
3570
|
+
function runtimeWorkReceiptClaimDispositionForBlockedReceipt(input: {
|
|
3571
|
+
receipt: WorkReceipt;
|
|
3572
|
+
claimantRunId: string;
|
|
3573
|
+
claimantRunAttempt?: number | null;
|
|
3574
|
+
}): Exclude<WorkReceiptClaim, { disposition: 'claimed' | 'reused' }> {
|
|
3575
|
+
const receipt = input.receipt;
|
|
3576
|
+
if (receipt.status === 'failed') {
|
|
3577
|
+
return { disposition: 'failed', receipt };
|
|
3578
|
+
}
|
|
3579
|
+
if (receipt.status === 'pending' || receipt.status === 'running') {
|
|
3580
|
+
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
3581
|
+
const leaseExpiresAtMs =
|
|
3582
|
+
typeof receipt.leaseExpiresAt === 'string'
|
|
3583
|
+
? Date.parse(receipt.leaseExpiresAt)
|
|
3584
|
+
: NaN;
|
|
3585
|
+
const hasActiveForeignLease =
|
|
3586
|
+
ownerRunId !== null &&
|
|
3587
|
+
ownerRunId !== input.claimantRunId &&
|
|
3588
|
+
typeof receipt.leaseId === 'string' &&
|
|
3589
|
+
receipt.leaseId.length > 0 &&
|
|
3590
|
+
Number.isFinite(leaseExpiresAtMs) &&
|
|
3591
|
+
leaseExpiresAtMs > Date.now();
|
|
3592
|
+
return {
|
|
3593
|
+
disposition: hasActiveForeignLease ? 'blocked_active_lease' : 'running',
|
|
3594
|
+
receipt,
|
|
3595
|
+
};
|
|
3596
|
+
}
|
|
3597
|
+
return { disposition: 'running', receipt };
|
|
3598
|
+
}
|
|
3599
|
+
|
|
2834
3600
|
export async function claimRuntimeWorkReceipts(
|
|
2835
3601
|
context: RuntimeApiContext,
|
|
2836
3602
|
input: {
|
|
2837
3603
|
playName: string;
|
|
2838
3604
|
runId: string;
|
|
2839
3605
|
keys: string[];
|
|
3606
|
+
runAttempt?: number | null;
|
|
3607
|
+
leaseAware?: boolean;
|
|
2840
3608
|
reclaimRunning?: boolean;
|
|
2841
3609
|
forceRefresh?: boolean;
|
|
3610
|
+
forceFailedRefresh?: boolean;
|
|
3611
|
+
leaseTtlMs?: number | null;
|
|
2842
3612
|
},
|
|
2843
3613
|
): Promise<WorkReceiptClaim[]> {
|
|
2844
3614
|
const keys = [
|
|
@@ -2854,46 +3624,104 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2854
3624
|
session,
|
|
2855
3625
|
async (client) => {
|
|
2856
3626
|
const keyHexes = keys.map(workReceiptKeyHex);
|
|
2857
|
-
const
|
|
2858
|
-
?
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
]
|
|
2871
|
-
: [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
|
|
3627
|
+
const leaseIds = keys.map(() =>
|
|
3628
|
+
input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
|
|
3629
|
+
);
|
|
3630
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3631
|
+
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
3632
|
+
input.leaseTtlMs,
|
|
3633
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
3634
|
+
);
|
|
3635
|
+
const reclaimStatuses = workReceiptClaimableStatusCodes({
|
|
3636
|
+
forceRefresh: input.forceRefresh,
|
|
3637
|
+
forceFailedRefresh: input.forceFailedRefresh,
|
|
3638
|
+
reclaimRunning: input.reclaimRunning,
|
|
3639
|
+
});
|
|
2872
3640
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2873
3641
|
`
|
|
2874
3642
|
WITH input_keys AS (
|
|
2875
|
-
SELECT key_values.key_hex, key_values.ord
|
|
3643
|
+
SELECT key_values.key_hex, lease_values.lease_id, key_values.ord
|
|
2876
3644
|
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
3645
|
+
JOIN unnest($2::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
|
|
3646
|
+
ON lease_values.ord = key_values.ord
|
|
2877
3647
|
),
|
|
2878
3648
|
claimed AS (
|
|
2879
|
-
INSERT INTO ${workReceiptTable(session)} (
|
|
2880
|
-
|
|
3649
|
+
INSERT INTO ${workReceiptTable(session)} (
|
|
3650
|
+
k,
|
|
3651
|
+
status,
|
|
3652
|
+
run_id,
|
|
3653
|
+
lease_id,
|
|
3654
|
+
lease_owner_run_id,
|
|
3655
|
+
lease_owner_attempt,
|
|
3656
|
+
lease_expires_at,
|
|
3657
|
+
updated_at
|
|
3658
|
+
)
|
|
3659
|
+
SELECT decode(input_keys.key_hex, 'hex'),
|
|
3660
|
+
$3::smallint,
|
|
3661
|
+
$4::text,
|
|
3662
|
+
input_keys.lease_id,
|
|
3663
|
+
CASE
|
|
3664
|
+
WHEN input_keys.lease_id IS NULL THEN NULL
|
|
3665
|
+
ELSE $4::text
|
|
3666
|
+
END,
|
|
3667
|
+
CASE
|
|
3668
|
+
WHEN input_keys.lease_id IS NULL THEN NULL
|
|
3669
|
+
ELSE $9::integer
|
|
3670
|
+
END,
|
|
3671
|
+
CASE
|
|
3672
|
+
WHEN input_keys.lease_id IS NULL THEN NULL
|
|
3673
|
+
ELSE now() + ($7::double precision * interval '1 millisecond')
|
|
3674
|
+
END,
|
|
3675
|
+
now()
|
|
2881
3676
|
FROM input_keys
|
|
2882
3677
|
ON CONFLICT (k) DO UPDATE
|
|
2883
|
-
SET status = $
|
|
3678
|
+
SET status = $5::smallint,
|
|
2884
3679
|
output = NULL,
|
|
2885
|
-
|
|
3680
|
+
failure_kind = 0,
|
|
3681
|
+
run_id = $4::text,
|
|
3682
|
+
lease_id = EXCLUDED.lease_id,
|
|
3683
|
+
lease_owner_run_id = CASE
|
|
3684
|
+
WHEN EXCLUDED.lease_id IS NULL THEN NULL
|
|
3685
|
+
ELSE $4::text
|
|
3686
|
+
END,
|
|
3687
|
+
lease_owner_attempt = CASE
|
|
3688
|
+
WHEN EXCLUDED.lease_id IS NULL THEN NULL
|
|
3689
|
+
ELSE $9::integer
|
|
3690
|
+
END,
|
|
3691
|
+
lease_expires_at = EXCLUDED.lease_expires_at,
|
|
2886
3692
|
error = NULL,
|
|
2887
3693
|
updated_at = now()
|
|
2888
|
-
WHERE ${
|
|
2889
|
-
|
|
3694
|
+
WHERE ${workReceiptClaimConflictPredicateSql({
|
|
3695
|
+
receiptTable: workReceiptTable(session),
|
|
3696
|
+
claimableStatusesSql: '$6',
|
|
3697
|
+
claimantRunIdSql: '$4::text',
|
|
3698
|
+
claimantRunAttemptSql: '$9',
|
|
3699
|
+
forceRefreshSql: '$8',
|
|
3700
|
+
forceFailedRefreshSql: '$10',
|
|
3701
|
+
})}
|
|
3702
|
+
RETURNING k,
|
|
3703
|
+
status,
|
|
3704
|
+
output,
|
|
3705
|
+
error,
|
|
3706
|
+
failure_kind,
|
|
3707
|
+
run_id,
|
|
3708
|
+
lease_id,
|
|
3709
|
+
lease_owner_run_id,
|
|
3710
|
+
lease_owner_attempt,
|
|
3711
|
+
lease_expires_at,
|
|
3712
|
+
updated_at
|
|
2890
3713
|
),
|
|
2891
3714
|
latest AS (
|
|
2892
3715
|
SELECT receipts.k,
|
|
2893
3716
|
receipts.status,
|
|
2894
3717
|
receipts.output,
|
|
2895
3718
|
receipts.error,
|
|
3719
|
+
receipts.failure_kind,
|
|
2896
3720
|
receipts.run_id,
|
|
3721
|
+
receipts.lease_id,
|
|
3722
|
+
receipts.lease_owner_run_id,
|
|
3723
|
+
receipts.lease_owner_attempt,
|
|
3724
|
+
receipts.lease_expires_at,
|
|
2897
3725
|
receipts.updated_at,
|
|
2898
3726
|
input_keys.ord,
|
|
2899
3727
|
false AS claimed
|
|
@@ -2909,21 +3737,43 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2909
3737
|
claimed.status,
|
|
2910
3738
|
claimed.output,
|
|
2911
3739
|
claimed.error,
|
|
3740
|
+
claimed.failure_kind,
|
|
2912
3741
|
claimed.run_id,
|
|
3742
|
+
claimed.lease_id,
|
|
3743
|
+
claimed.lease_owner_run_id,
|
|
3744
|
+
claimed.lease_owner_attempt,
|
|
3745
|
+
claimed.lease_expires_at,
|
|
2913
3746
|
claimed.updated_at,
|
|
2914
3747
|
input_keys.ord,
|
|
2915
3748
|
true AS claimed
|
|
2916
3749
|
FROM claimed
|
|
2917
3750
|
JOIN input_keys ON claimed.k = decode(input_keys.key_hex, 'hex')
|
|
2918
3751
|
UNION ALL
|
|
2919
|
-
SELECT k,
|
|
3752
|
+
SELECT k,
|
|
3753
|
+
status,
|
|
3754
|
+
output,
|
|
3755
|
+
error,
|
|
3756
|
+
failure_kind,
|
|
3757
|
+
run_id,
|
|
3758
|
+
lease_id,
|
|
3759
|
+
lease_owner_run_id,
|
|
3760
|
+
lease_owner_attempt,
|
|
3761
|
+
lease_expires_at,
|
|
3762
|
+
updated_at,
|
|
3763
|
+
ord,
|
|
3764
|
+
claimed
|
|
2920
3765
|
FROM latest
|
|
2921
3766
|
)
|
|
2922
3767
|
SELECT convert_from(k, 'UTF8') AS k,
|
|
2923
3768
|
status,
|
|
2924
3769
|
output,
|
|
2925
3770
|
error,
|
|
3771
|
+
failure_kind,
|
|
2926
3772
|
run_id,
|
|
3773
|
+
lease_id,
|
|
3774
|
+
lease_owner_run_id,
|
|
3775
|
+
lease_owner_attempt,
|
|
3776
|
+
lease_expires_at,
|
|
2927
3777
|
updated_at,
|
|
2928
3778
|
claimed
|
|
2929
3779
|
FROM returned
|
|
@@ -2931,10 +3781,15 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2931
3781
|
`,
|
|
2932
3782
|
[
|
|
2933
3783
|
keyHexes,
|
|
3784
|
+
leaseIds,
|
|
2934
3785
|
RECEIPT_STATUS_RUNNING,
|
|
2935
3786
|
input.runId,
|
|
2936
3787
|
RECEIPT_STATUS_RUNNING,
|
|
2937
3788
|
reclaimStatuses,
|
|
3789
|
+
leaseTtlMs,
|
|
3790
|
+
input.forceRefresh === true,
|
|
3791
|
+
runAttempt,
|
|
3792
|
+
input.forceFailedRefresh === true,
|
|
2938
3793
|
],
|
|
2939
3794
|
);
|
|
2940
3795
|
return rows.map((row) => {
|
|
@@ -2942,16 +3797,14 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2942
3797
|
if (row.claimed === true) {
|
|
2943
3798
|
return { disposition: 'claimed', receipt } satisfies WorkReceiptClaim;
|
|
2944
3799
|
}
|
|
2945
|
-
if (isReusableWorkReceipt(receipt)) {
|
|
3800
|
+
if (input.forceRefresh !== true && isReusableWorkReceipt(receipt)) {
|
|
2946
3801
|
return { disposition: 'reused', receipt } satisfies WorkReceiptClaim;
|
|
2947
3802
|
}
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
}
|
|
2954
|
-
return { disposition: 'running', receipt } satisfies WorkReceiptClaim;
|
|
3803
|
+
return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
|
|
3804
|
+
receipt,
|
|
3805
|
+
claimantRunId: input.runId,
|
|
3806
|
+
claimantRunAttempt: runAttempt,
|
|
3807
|
+
});
|
|
2955
3808
|
});
|
|
2956
3809
|
},
|
|
2957
3810
|
);
|
|
@@ -2963,17 +3816,50 @@ export async function completeRuntimeWorkReceipt(
|
|
|
2963
3816
|
playName: string;
|
|
2964
3817
|
runId: string;
|
|
2965
3818
|
key: string;
|
|
3819
|
+
runAttempt?: number | null;
|
|
3820
|
+
leaseId?: string | null;
|
|
2966
3821
|
output: unknown;
|
|
2967
3822
|
},
|
|
2968
3823
|
): Promise<WorkReceipt | null> {
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
3824
|
+
if (
|
|
3825
|
+
consumeDirectRuntimeTestFault({
|
|
3826
|
+
context,
|
|
3827
|
+
runId: input.runId,
|
|
3828
|
+
name: 'receipt_complete_write_fail',
|
|
3829
|
+
})
|
|
3830
|
+
) {
|
|
3831
|
+
const error = runtimeTestFaultError('receipt_complete_write_fail');
|
|
3832
|
+
const failed = await failRuntimeWorkReceipt(context, {
|
|
3833
|
+
playName: input.playName,
|
|
3834
|
+
runId: input.runId,
|
|
3835
|
+
key: input.key,
|
|
3836
|
+
leaseId: input.leaseId,
|
|
3837
|
+
error: error.message,
|
|
3838
|
+
failureKind: 'repairable',
|
|
3839
|
+
runAttempt: input.runAttempt,
|
|
3840
|
+
});
|
|
3841
|
+
if (!failed) {
|
|
3842
|
+
await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
|
|
3843
|
+
playName: input.playName,
|
|
3844
|
+
runId: input.runId,
|
|
3845
|
+
key: input.key,
|
|
3846
|
+
error: error.message,
|
|
3847
|
+
failureKind: 'repairable',
|
|
3848
|
+
runAttempt: input.runAttempt,
|
|
3849
|
+
});
|
|
3850
|
+
}
|
|
3851
|
+
throw error;
|
|
3852
|
+
}
|
|
3853
|
+
const session = await getRuntimeWorkReceiptSession(context, {
|
|
3854
|
+
playName: input.playName,
|
|
3855
|
+
key: input.key,
|
|
3856
|
+
});
|
|
3857
|
+
return await withRuntimeWorkReceiptClient(
|
|
3858
|
+
context,
|
|
3859
|
+
session,
|
|
3860
|
+
async (client) => {
|
|
3861
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
3862
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
2977
3863
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2978
3864
|
`
|
|
2979
3865
|
WITH completed AS (
|
|
@@ -2981,29 +3867,41 @@ export async function completeRuntimeWorkReceipt(
|
|
|
2981
3867
|
SET status = $2::smallint,
|
|
2982
3868
|
output = $3::jsonb,
|
|
2983
3869
|
error = NULL,
|
|
3870
|
+
failure_kind = 0,
|
|
2984
3871
|
run_id = $4,
|
|
3872
|
+
lease_id = NULL,
|
|
3873
|
+
lease_owner_run_id = NULL,
|
|
3874
|
+
lease_owner_attempt = NULL,
|
|
3875
|
+
lease_expires_at = NULL,
|
|
2985
3876
|
updated_at = now()
|
|
2986
3877
|
WHERE k = decode($1, 'hex')
|
|
2987
|
-
AND
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3878
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
3879
|
+
receiptTable: workReceiptTable(session),
|
|
3880
|
+
ownerRunIdSql: '$4',
|
|
3881
|
+
ownerRunAttemptSql: '$6',
|
|
3882
|
+
leaseIdSql: '$5::text',
|
|
3883
|
+
})}
|
|
3884
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
3885
|
+
status,
|
|
3886
|
+
output,
|
|
3887
|
+
error,
|
|
3888
|
+
failure_kind,
|
|
3889
|
+
run_id,
|
|
3890
|
+
lease_id,
|
|
3891
|
+
lease_owner_run_id,
|
|
3892
|
+
lease_owner_attempt,
|
|
3893
|
+
lease_expires_at,
|
|
3894
|
+
updated_at
|
|
3895
|
+
)
|
|
3896
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
|
|
3000
3897
|
`,
|
|
3001
3898
|
[
|
|
3002
3899
|
workReceiptKeyHex(input.key),
|
|
3003
3900
|
RECEIPT_STATUS_COMPLETED,
|
|
3004
3901
|
input.output === null ? null : stringifyPostgresJson(input.output),
|
|
3005
3902
|
input.runId,
|
|
3006
|
-
|
|
3903
|
+
leaseId,
|
|
3904
|
+
runAttempt,
|
|
3007
3905
|
],
|
|
3008
3906
|
);
|
|
3009
3907
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -3015,11 +3913,63 @@ export async function completeRuntimeWorkReceipts(
|
|
|
3015
3913
|
context: RuntimeApiContext,
|
|
3016
3914
|
input: {
|
|
3017
3915
|
playName: string;
|
|
3018
|
-
receipts: Array<{
|
|
3916
|
+
receipts: Array<{
|
|
3917
|
+
runId: string;
|
|
3918
|
+
key: string;
|
|
3919
|
+
output: unknown;
|
|
3920
|
+
leaseId?: string | null;
|
|
3921
|
+
runAttempt?: number | null;
|
|
3922
|
+
}>;
|
|
3019
3923
|
},
|
|
3020
|
-
): Promise<WorkReceipt
|
|
3924
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
3021
3925
|
const receipts = input.receipts.filter((receipt) => receipt.key.trim());
|
|
3022
3926
|
if (receipts.length === 0) return [];
|
|
3927
|
+
if (
|
|
3928
|
+
consumeDirectRuntimeTestFault({
|
|
3929
|
+
context,
|
|
3930
|
+
runId: receipts[0]?.runId ?? '',
|
|
3931
|
+
name: 'receipt_complete_write_fail',
|
|
3932
|
+
})
|
|
3933
|
+
) {
|
|
3934
|
+
const [skippedReceipt, ...siblingReceipts] = receipts;
|
|
3935
|
+
let siblingError: unknown = null;
|
|
3936
|
+
if (siblingReceipts.length > 0) {
|
|
3937
|
+
try {
|
|
3938
|
+
await completeRuntimeWorkReceipts(context, {
|
|
3939
|
+
playName: input.playName,
|
|
3940
|
+
receipts: siblingReceipts,
|
|
3941
|
+
});
|
|
3942
|
+
} catch (error) {
|
|
3943
|
+
siblingError = error;
|
|
3944
|
+
}
|
|
3945
|
+
}
|
|
3946
|
+
const error = runtimeTestFaultError('receipt_complete_write_fail');
|
|
3947
|
+
if (skippedReceipt) {
|
|
3948
|
+
const failed = await failRuntimeWorkReceipt(context, {
|
|
3949
|
+
playName: input.playName,
|
|
3950
|
+
runId: skippedReceipt.runId,
|
|
3951
|
+
key: skippedReceipt.key,
|
|
3952
|
+
leaseId: skippedReceipt.leaseId,
|
|
3953
|
+
error: error.message,
|
|
3954
|
+
failureKind: 'repairable',
|
|
3955
|
+
runAttempt: skippedReceipt.runAttempt,
|
|
3956
|
+
});
|
|
3957
|
+
if (!failed) {
|
|
3958
|
+
await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
|
|
3959
|
+
playName: input.playName,
|
|
3960
|
+
runId: skippedReceipt.runId,
|
|
3961
|
+
key: skippedReceipt.key,
|
|
3962
|
+
error: error.message,
|
|
3963
|
+
failureKind: 'repairable',
|
|
3964
|
+
runAttempt: skippedReceipt.runAttempt,
|
|
3965
|
+
});
|
|
3966
|
+
}
|
|
3967
|
+
}
|
|
3968
|
+
if (siblingError) {
|
|
3969
|
+
throw siblingError;
|
|
3970
|
+
}
|
|
3971
|
+
throw error;
|
|
3972
|
+
}
|
|
3023
3973
|
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
3024
3974
|
playName: input.playName,
|
|
3025
3975
|
keys: receipts.map((receipt) => receipt.key),
|
|
@@ -3033,52 +3983,79 @@ export async function completeRuntimeWorkReceipts(
|
|
|
3033
3983
|
WITH inputs AS (
|
|
3034
3984
|
SELECT key_values.key_hex,
|
|
3035
3985
|
run_values.run_id,
|
|
3986
|
+
lease_values.lease_id,
|
|
3987
|
+
attempt_values.run_attempt,
|
|
3036
3988
|
output_values.output,
|
|
3037
3989
|
key_values.ord
|
|
3038
3990
|
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
3039
3991
|
JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
|
|
3040
3992
|
ON run_values.ord = key_values.ord
|
|
3041
|
-
JOIN
|
|
3993
|
+
JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
|
|
3994
|
+
ON lease_values.ord = key_values.ord
|
|
3995
|
+
JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
|
|
3996
|
+
ON attempt_values.ord = key_values.ord
|
|
3997
|
+
JOIN jsonb_array_elements($5::jsonb) WITH ORDINALITY AS output_values(output, ord)
|
|
3042
3998
|
ON output_values.ord = key_values.ord
|
|
3043
3999
|
),
|
|
3044
4000
|
completed AS (
|
|
3045
4001
|
UPDATE ${workReceiptTable(session)} AS target
|
|
3046
|
-
SET status = $
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
4002
|
+
SET status = $6::smallint,
|
|
4003
|
+
output = inputs.output,
|
|
4004
|
+
error = NULL,
|
|
4005
|
+
failure_kind = 0,
|
|
4006
|
+
run_id = inputs.run_id,
|
|
4007
|
+
lease_id = NULL,
|
|
4008
|
+
lease_owner_run_id = NULL,
|
|
4009
|
+
lease_owner_attempt = NULL,
|
|
4010
|
+
lease_expires_at = NULL,
|
|
3050
4011
|
updated_at = now()
|
|
3051
4012
|
FROM inputs
|
|
3052
4013
|
WHERE target.k = decode(inputs.key_hex, 'hex')
|
|
3053
|
-
AND
|
|
3054
|
-
|
|
3055
|
-
|
|
4014
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4015
|
+
receiptTable: 'target',
|
|
4016
|
+
ownerRunIdSql: 'inputs.run_id',
|
|
4017
|
+
ownerRunAttemptSql: 'inputs.run_attempt',
|
|
4018
|
+
leaseIdSql: 'inputs.lease_id',
|
|
4019
|
+
})}
|
|
4020
|
+
RETURNING target.k,
|
|
4021
|
+
target.status,
|
|
4022
|
+
target.output,
|
|
4023
|
+
target.error,
|
|
4024
|
+
target.failure_kind,
|
|
4025
|
+
target.run_id,
|
|
4026
|
+
target.lease_id,
|
|
4027
|
+
target.lease_owner_run_id,
|
|
4028
|
+
target.lease_owner_attempt,
|
|
4029
|
+
target.lease_expires_at,
|
|
4030
|
+
target.updated_at,
|
|
4031
|
+
inputs.ord
|
|
3056
4032
|
),
|
|
3057
|
-
|
|
3058
|
-
SELECT
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
4033
|
+
returned AS (
|
|
4034
|
+
SELECT completed.k,
|
|
4035
|
+
completed.status,
|
|
4036
|
+
completed.output,
|
|
4037
|
+
completed.error,
|
|
4038
|
+
completed.failure_kind,
|
|
4039
|
+
completed.run_id,
|
|
4040
|
+
completed.lease_id,
|
|
4041
|
+
completed.lease_owner_run_id,
|
|
4042
|
+
completed.lease_owner_attempt,
|
|
4043
|
+
completed.lease_expires_at,
|
|
4044
|
+
completed.updated_at,
|
|
3064
4045
|
inputs.ord
|
|
3065
4046
|
FROM inputs
|
|
3066
|
-
JOIN
|
|
3067
|
-
ON receipts.k = decode(inputs.key_hex, 'hex')
|
|
3068
|
-
WHERE NOT EXISTS (
|
|
3069
|
-
SELECT 1 FROM completed WHERE completed.k = receipts.k
|
|
3070
|
-
)
|
|
3071
|
-
),
|
|
3072
|
-
returned AS (
|
|
3073
|
-
SELECT k, status, output, error, run_id, updated_at, ord FROM completed
|
|
3074
|
-
UNION ALL
|
|
3075
|
-
SELECT k, status, output, error, run_id, updated_at, ord FROM latest
|
|
4047
|
+
LEFT JOIN completed ON completed.ord = inputs.ord
|
|
3076
4048
|
)
|
|
3077
4049
|
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
3078
4050
|
returned.status,
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
4051
|
+
returned.output,
|
|
4052
|
+
returned.error,
|
|
4053
|
+
returned.failure_kind,
|
|
4054
|
+
returned.run_id,
|
|
4055
|
+
returned.lease_id,
|
|
4056
|
+
returned.lease_owner_run_id,
|
|
4057
|
+
returned.lease_owner_attempt,
|
|
4058
|
+
returned.lease_expires_at,
|
|
3082
4059
|
returned.updated_at
|
|
3083
4060
|
FROM returned
|
|
3084
4061
|
ORDER BY returned.ord
|
|
@@ -3086,16 +4063,21 @@ export async function completeRuntimeWorkReceipts(
|
|
|
3086
4063
|
[
|
|
3087
4064
|
receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
|
|
3088
4065
|
receipts.map((receipt) => receipt.runId),
|
|
4066
|
+
receipts.map((receipt) => receipt.leaseId?.trim() || null),
|
|
4067
|
+
receipts.map((receipt) =>
|
|
4068
|
+
normalizeRuntimeRunAttempt(receipt.runAttempt),
|
|
4069
|
+
),
|
|
3089
4070
|
stringifyPostgresJson(
|
|
3090
4071
|
receipts.map((receipt) =>
|
|
3091
4072
|
receipt.output === null ? null : receipt.output,
|
|
3092
4073
|
),
|
|
3093
4074
|
),
|
|
3094
4075
|
RECEIPT_STATUS_COMPLETED,
|
|
3095
|
-
RECEIPT_STATUS_RUNNING,
|
|
3096
4076
|
],
|
|
3097
4077
|
);
|
|
3098
|
-
return rows.map(
|
|
4078
|
+
return rows.map((row) =>
|
|
4079
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
4080
|
+
);
|
|
3099
4081
|
},
|
|
3100
4082
|
);
|
|
3101
4083
|
}
|
|
@@ -3106,7 +4088,10 @@ export async function failRuntimeWorkReceipt(
|
|
|
3106
4088
|
playName: string;
|
|
3107
4089
|
runId: string;
|
|
3108
4090
|
key: string;
|
|
4091
|
+
runAttempt?: number | null;
|
|
4092
|
+
leaseId?: string | null;
|
|
3109
4093
|
error: string;
|
|
4094
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
3110
4095
|
},
|
|
3111
4096
|
): Promise<WorkReceipt | null> {
|
|
3112
4097
|
const session = await getRuntimeWorkReceiptSession(context, {
|
|
@@ -3117,36 +4102,119 @@ export async function failRuntimeWorkReceipt(
|
|
|
3117
4102
|
context,
|
|
3118
4103
|
session,
|
|
3119
4104
|
async (client) => {
|
|
4105
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
4106
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3120
4107
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
3121
4108
|
`
|
|
3122
4109
|
WITH failed AS (
|
|
3123
4110
|
UPDATE ${workReceiptTable(session)}
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
4111
|
+
SET status = $2::smallint,
|
|
4112
|
+
output = NULL,
|
|
4113
|
+
error = $3,
|
|
4114
|
+
failure_kind = $6::smallint,
|
|
3127
4115
|
run_id = $4,
|
|
4116
|
+
lease_id = NULL,
|
|
4117
|
+
lease_owner_run_id = NULL,
|
|
4118
|
+
lease_owner_attempt = NULL,
|
|
4119
|
+
lease_expires_at = NULL,
|
|
3128
4120
|
updated_at = now()
|
|
3129
4121
|
WHERE k = decode($1, 'hex')
|
|
3130
|
-
AND
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
4122
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4123
|
+
receiptTable: workReceiptTable(session),
|
|
4124
|
+
ownerRunIdSql: '$4',
|
|
4125
|
+
ownerRunAttemptSql: '$7',
|
|
4126
|
+
leaseIdSql: '$5::text',
|
|
4127
|
+
})}
|
|
4128
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4129
|
+
status,
|
|
4130
|
+
output,
|
|
4131
|
+
error,
|
|
4132
|
+
failure_kind,
|
|
4133
|
+
run_id,
|
|
4134
|
+
lease_id,
|
|
4135
|
+
lease_owner_run_id,
|
|
4136
|
+
lease_owner_attempt,
|
|
4137
|
+
lease_expires_at,
|
|
4138
|
+
updated_at
|
|
4139
|
+
)
|
|
4140
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
|
|
3143
4141
|
`,
|
|
3144
4142
|
[
|
|
3145
4143
|
workReceiptKeyHex(input.key),
|
|
3146
4144
|
RECEIPT_STATUS_FAILED,
|
|
3147
4145
|
input.error,
|
|
3148
4146
|
input.runId,
|
|
3149
|
-
|
|
4147
|
+
leaseId,
|
|
4148
|
+
workReceiptFailureKindCode(input.failureKind),
|
|
4149
|
+
runAttempt,
|
|
4150
|
+
],
|
|
4151
|
+
);
|
|
4152
|
+
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
4153
|
+
},
|
|
4154
|
+
);
|
|
4155
|
+
}
|
|
4156
|
+
|
|
4157
|
+
export async function releaseRuntimeWorkReceipt(
|
|
4158
|
+
context: RuntimeApiContext,
|
|
4159
|
+
input: {
|
|
4160
|
+
playName: string;
|
|
4161
|
+
runId: string;
|
|
4162
|
+
key: string;
|
|
4163
|
+
runAttempt?: number | null;
|
|
4164
|
+
leaseId?: string | null;
|
|
4165
|
+
},
|
|
4166
|
+
): Promise<WorkReceipt | null> {
|
|
4167
|
+
const session = await getRuntimeWorkReceiptSession(context, {
|
|
4168
|
+
playName: input.playName,
|
|
4169
|
+
key: input.key,
|
|
4170
|
+
});
|
|
4171
|
+
return await withRuntimeWorkReceiptClient(
|
|
4172
|
+
context,
|
|
4173
|
+
session,
|
|
4174
|
+
async (client) => {
|
|
4175
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
4176
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
4177
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
4178
|
+
`
|
|
4179
|
+
WITH released AS (
|
|
4180
|
+
UPDATE ${workReceiptTable(session)}
|
|
4181
|
+
SET status = $2::smallint,
|
|
4182
|
+
output = NULL,
|
|
4183
|
+
error = NULL,
|
|
4184
|
+
failure_kind = 0,
|
|
4185
|
+
run_id = $3,
|
|
4186
|
+
lease_id = NULL,
|
|
4187
|
+
lease_owner_run_id = NULL,
|
|
4188
|
+
lease_owner_attempt = NULL,
|
|
4189
|
+
lease_expires_at = NULL,
|
|
4190
|
+
updated_at = now()
|
|
4191
|
+
WHERE k = decode($1, 'hex')
|
|
4192
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4193
|
+
receiptTable: workReceiptTable(session),
|
|
4194
|
+
ownerRunIdSql: '$3',
|
|
4195
|
+
ownerRunAttemptSql: '$5',
|
|
4196
|
+
leaseIdSql: '$4::text',
|
|
4197
|
+
})}
|
|
4198
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4199
|
+
status,
|
|
4200
|
+
output,
|
|
4201
|
+
error,
|
|
4202
|
+
failure_kind,
|
|
4203
|
+
run_id,
|
|
4204
|
+
lease_id,
|
|
4205
|
+
lease_owner_run_id,
|
|
4206
|
+
lease_owner_attempt,
|
|
4207
|
+
lease_expires_at,
|
|
4208
|
+
updated_at
|
|
4209
|
+
)
|
|
4210
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM released
|
|
4211
|
+
`,
|
|
4212
|
+
[
|
|
4213
|
+
workReceiptKeyHex(input.key),
|
|
4214
|
+
RECEIPT_STATUS_PENDING,
|
|
4215
|
+
input.runId,
|
|
4216
|
+
leaseId,
|
|
4217
|
+
runAttempt,
|
|
3150
4218
|
],
|
|
3151
4219
|
);
|
|
3152
4220
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -3158,9 +4226,16 @@ export async function failRuntimeWorkReceipts(
|
|
|
3158
4226
|
context: RuntimeApiContext,
|
|
3159
4227
|
input: {
|
|
3160
4228
|
playName: string;
|
|
3161
|
-
receipts: Array<{
|
|
4229
|
+
receipts: Array<{
|
|
4230
|
+
runId: string;
|
|
4231
|
+
key: string;
|
|
4232
|
+
error: string;
|
|
4233
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
4234
|
+
leaseId?: string | null;
|
|
4235
|
+
runAttempt?: number | null;
|
|
4236
|
+
}>;
|
|
3162
4237
|
},
|
|
3163
|
-
): Promise<WorkReceipt
|
|
4238
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
3164
4239
|
const receipts = input.receipts.filter((receipt) => receipt.key.trim());
|
|
3165
4240
|
if (receipts.length === 0) return [];
|
|
3166
4241
|
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
@@ -3176,65 +4251,207 @@ export async function failRuntimeWorkReceipts(
|
|
|
3176
4251
|
WITH inputs AS (
|
|
3177
4252
|
SELECT key_values.key_hex,
|
|
3178
4253
|
run_values.run_id,
|
|
3179
|
-
|
|
3180
|
-
|
|
4254
|
+
lease_values.lease_id,
|
|
4255
|
+
attempt_values.run_attempt,
|
|
4256
|
+
error_values.error,
|
|
4257
|
+
failure_kind_values.failure_kind,
|
|
4258
|
+
key_values.ord
|
|
3181
4259
|
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
3182
4260
|
JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
|
|
3183
4261
|
ON run_values.ord = key_values.ord
|
|
3184
|
-
JOIN unnest($3::text[]) WITH ORDINALITY AS
|
|
3185
|
-
ON
|
|
4262
|
+
JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
|
|
4263
|
+
ON lease_values.ord = key_values.ord
|
|
4264
|
+
JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
|
|
4265
|
+
ON attempt_values.ord = key_values.ord
|
|
4266
|
+
JOIN unnest($5::text[]) WITH ORDINALITY AS error_values(error, ord)
|
|
4267
|
+
ON error_values.ord = key_values.ord
|
|
4268
|
+
JOIN unnest($6::smallint[]) WITH ORDINALITY AS failure_kind_values(failure_kind, ord)
|
|
4269
|
+
ON failure_kind_values.ord = key_values.ord
|
|
3186
4270
|
),
|
|
3187
4271
|
failed AS (
|
|
3188
4272
|
UPDATE ${workReceiptTable(session)} AS target
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
4273
|
+
SET status = $7::smallint,
|
|
4274
|
+
output = NULL,
|
|
4275
|
+
error = inputs.error,
|
|
4276
|
+
failure_kind = inputs.failure_kind,
|
|
4277
|
+
run_id = inputs.run_id,
|
|
4278
|
+
lease_id = NULL,
|
|
4279
|
+
lease_owner_run_id = NULL,
|
|
4280
|
+
lease_owner_attempt = NULL,
|
|
4281
|
+
lease_expires_at = NULL,
|
|
3193
4282
|
updated_at = now()
|
|
3194
4283
|
FROM inputs
|
|
3195
4284
|
WHERE target.k = decode(inputs.key_hex, 'hex')
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
4285
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4286
|
+
receiptTable: 'target',
|
|
4287
|
+
ownerRunIdSql: 'inputs.run_id',
|
|
4288
|
+
ownerRunAttemptSql: 'inputs.run_attempt',
|
|
4289
|
+
leaseIdSql: 'inputs.lease_id',
|
|
4290
|
+
})}
|
|
4291
|
+
RETURNING target.k,
|
|
4292
|
+
target.status,
|
|
4293
|
+
target.output,
|
|
4294
|
+
target.error,
|
|
4295
|
+
target.failure_kind,
|
|
4296
|
+
target.run_id,
|
|
4297
|
+
target.lease_id,
|
|
4298
|
+
target.lease_owner_run_id,
|
|
4299
|
+
target.lease_owner_attempt,
|
|
4300
|
+
target.lease_expires_at,
|
|
4301
|
+
target.updated_at,
|
|
4302
|
+
inputs.ord
|
|
3199
4303
|
),
|
|
3200
|
-
|
|
3201
|
-
SELECT
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
4304
|
+
returned AS (
|
|
4305
|
+
SELECT failed.k,
|
|
4306
|
+
failed.status,
|
|
4307
|
+
failed.output,
|
|
4308
|
+
failed.error,
|
|
4309
|
+
failed.failure_kind,
|
|
4310
|
+
failed.run_id,
|
|
4311
|
+
failed.lease_id,
|
|
4312
|
+
failed.lease_owner_run_id,
|
|
4313
|
+
failed.lease_owner_attempt,
|
|
4314
|
+
failed.lease_expires_at,
|
|
4315
|
+
failed.updated_at,
|
|
3207
4316
|
inputs.ord
|
|
3208
4317
|
FROM inputs
|
|
3209
|
-
JOIN
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
4318
|
+
LEFT JOIN failed ON failed.ord = inputs.ord
|
|
4319
|
+
)
|
|
4320
|
+
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
4321
|
+
returned.status,
|
|
4322
|
+
returned.output,
|
|
4323
|
+
returned.error,
|
|
4324
|
+
returned.failure_kind,
|
|
4325
|
+
returned.run_id,
|
|
4326
|
+
returned.lease_id,
|
|
4327
|
+
returned.lease_owner_run_id,
|
|
4328
|
+
returned.lease_owner_attempt,
|
|
4329
|
+
returned.lease_expires_at,
|
|
4330
|
+
returned.updated_at
|
|
4331
|
+
FROM returned
|
|
4332
|
+
ORDER BY returned.ord
|
|
4333
|
+
`,
|
|
4334
|
+
[
|
|
4335
|
+
receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
|
|
4336
|
+
receipts.map((receipt) => receipt.runId),
|
|
4337
|
+
receipts.map((receipt) => receipt.leaseId?.trim() || null),
|
|
4338
|
+
receipts.map((receipt) =>
|
|
4339
|
+
normalizeRuntimeRunAttempt(receipt.runAttempt),
|
|
4340
|
+
),
|
|
4341
|
+
receipts.map((receipt) => receipt.error),
|
|
4342
|
+
receipts.map((receipt) =>
|
|
4343
|
+
workReceiptFailureKindCode(receipt.failureKind),
|
|
4344
|
+
),
|
|
4345
|
+
RECEIPT_STATUS_FAILED,
|
|
4346
|
+
],
|
|
4347
|
+
);
|
|
4348
|
+
return rows.map((row) =>
|
|
4349
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
4350
|
+
);
|
|
4351
|
+
},
|
|
4352
|
+
);
|
|
4353
|
+
}
|
|
4354
|
+
|
|
4355
|
+
export async function heartbeatRuntimeWorkReceipts(
|
|
4356
|
+
context: RuntimeApiContext,
|
|
4357
|
+
input: {
|
|
4358
|
+
playName: string;
|
|
4359
|
+
runId: string;
|
|
4360
|
+
runAttempt?: number | null;
|
|
4361
|
+
leaseId: string;
|
|
4362
|
+
keys: string[];
|
|
4363
|
+
leaseTtlMs?: number | null;
|
|
4364
|
+
},
|
|
4365
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
4366
|
+
const keys = input.keys.map((key) => key.trim()).filter(Boolean);
|
|
4367
|
+
if (keys.length === 0) return [];
|
|
4368
|
+
const leaseId = input.leaseId.trim();
|
|
4369
|
+
if (!leaseId) return keys.map(() => null);
|
|
4370
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
4371
|
+
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
4372
|
+
input.leaseTtlMs,
|
|
4373
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
4374
|
+
);
|
|
4375
|
+
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
4376
|
+
playName: input.playName,
|
|
4377
|
+
keys,
|
|
4378
|
+
});
|
|
4379
|
+
return await withRuntimeWorkReceiptClient(
|
|
4380
|
+
context,
|
|
4381
|
+
session,
|
|
4382
|
+
async (client) => {
|
|
4383
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
4384
|
+
`
|
|
4385
|
+
WITH input_keys AS (
|
|
4386
|
+
SELECT key_values.key_hex, key_values.ord
|
|
4387
|
+
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
4388
|
+
),
|
|
4389
|
+
renewed AS (
|
|
4390
|
+
UPDATE ${workReceiptTable(session)} AS target
|
|
4391
|
+
SET lease_expires_at = now() + ($4::double precision * interval '1 millisecond'),
|
|
4392
|
+
updated_at = now()
|
|
4393
|
+
FROM input_keys
|
|
4394
|
+
WHERE target.k = decode(input_keys.key_hex, 'hex')
|
|
4395
|
+
AND ${workReceiptHeartbeatPredicateSql({
|
|
4396
|
+
receiptTable: 'target',
|
|
4397
|
+
ownerRunIdSql: '$2::text',
|
|
4398
|
+
ownerRunAttemptSql: '$5',
|
|
4399
|
+
leaseIdSql: '$3::text',
|
|
4400
|
+
})}
|
|
4401
|
+
RETURNING target.k,
|
|
4402
|
+
target.status,
|
|
4403
|
+
target.output,
|
|
4404
|
+
target.error,
|
|
4405
|
+
target.failure_kind,
|
|
4406
|
+
target.run_id,
|
|
4407
|
+
target.lease_id,
|
|
4408
|
+
target.lease_owner_run_id,
|
|
4409
|
+
target.lease_owner_attempt,
|
|
4410
|
+
target.lease_expires_at,
|
|
4411
|
+
target.updated_at,
|
|
4412
|
+
input_keys.ord
|
|
3214
4413
|
),
|
|
3215
4414
|
returned AS (
|
|
3216
|
-
SELECT k,
|
|
3217
|
-
|
|
3218
|
-
|
|
4415
|
+
SELECT renewed.k,
|
|
4416
|
+
renewed.status,
|
|
4417
|
+
renewed.output,
|
|
4418
|
+
renewed.error,
|
|
4419
|
+
renewed.failure_kind,
|
|
4420
|
+
renewed.run_id,
|
|
4421
|
+
renewed.lease_id,
|
|
4422
|
+
renewed.lease_owner_run_id,
|
|
4423
|
+
renewed.lease_owner_attempt,
|
|
4424
|
+
renewed.lease_expires_at,
|
|
4425
|
+
renewed.updated_at,
|
|
4426
|
+
input_keys.ord
|
|
4427
|
+
FROM input_keys
|
|
4428
|
+
LEFT JOIN renewed ON renewed.ord = input_keys.ord
|
|
3219
4429
|
)
|
|
3220
4430
|
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
3221
4431
|
returned.status,
|
|
3222
4432
|
returned.output,
|
|
3223
4433
|
returned.error,
|
|
4434
|
+
returned.failure_kind,
|
|
3224
4435
|
returned.run_id,
|
|
4436
|
+
returned.lease_id,
|
|
4437
|
+
returned.lease_owner_run_id,
|
|
4438
|
+
returned.lease_owner_attempt,
|
|
4439
|
+
returned.lease_expires_at,
|
|
3225
4440
|
returned.updated_at
|
|
3226
4441
|
FROM returned
|
|
3227
4442
|
ORDER BY returned.ord
|
|
3228
4443
|
`,
|
|
3229
4444
|
[
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
4445
|
+
keys.map(workReceiptKeyHex),
|
|
4446
|
+
input.runId,
|
|
4447
|
+
leaseId,
|
|
4448
|
+
leaseTtlMs,
|
|
4449
|
+
runAttempt,
|
|
3235
4450
|
],
|
|
3236
4451
|
);
|
|
3237
|
-
return rows.map(
|
|
4452
|
+
return rows.map((row) =>
|
|
4453
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
4454
|
+
);
|
|
3238
4455
|
},
|
|
3239
4456
|
);
|
|
3240
4457
|
}
|
|
@@ -3245,6 +4462,8 @@ export async function skipRuntimeWorkReceipt(
|
|
|
3245
4462
|
playName: string;
|
|
3246
4463
|
runId: string;
|
|
3247
4464
|
key: string;
|
|
4465
|
+
runAttempt?: number | null;
|
|
4466
|
+
leaseId?: string | null;
|
|
3248
4467
|
output: unknown;
|
|
3249
4468
|
},
|
|
3250
4469
|
): Promise<WorkReceipt | null> {
|
|
@@ -3256,36 +4475,50 @@ export async function skipRuntimeWorkReceipt(
|
|
|
3256
4475
|
context,
|
|
3257
4476
|
session,
|
|
3258
4477
|
async (client) => {
|
|
4478
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
4479
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3259
4480
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
3260
4481
|
`
|
|
3261
4482
|
WITH skipped AS (
|
|
3262
4483
|
UPDATE ${workReceiptTable(session)}
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
4484
|
+
SET status = $2::smallint,
|
|
4485
|
+
output = $3::jsonb,
|
|
4486
|
+
error = NULL,
|
|
4487
|
+
failure_kind = 0,
|
|
3266
4488
|
run_id = $4,
|
|
4489
|
+
lease_id = NULL,
|
|
4490
|
+
lease_owner_run_id = NULL,
|
|
4491
|
+
lease_owner_attempt = NULL,
|
|
4492
|
+
lease_expires_at = NULL,
|
|
3267
4493
|
updated_at = now()
|
|
3268
4494
|
WHERE k = decode($1, 'hex')
|
|
3269
|
-
AND
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
4495
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4496
|
+
receiptTable: workReceiptTable(session),
|
|
4497
|
+
ownerRunIdSql: '$4',
|
|
4498
|
+
ownerRunAttemptSql: '$6',
|
|
4499
|
+
leaseIdSql: '$5::text',
|
|
4500
|
+
})}
|
|
4501
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4502
|
+
status,
|
|
4503
|
+
output,
|
|
4504
|
+
error,
|
|
4505
|
+
failure_kind,
|
|
4506
|
+
run_id,
|
|
4507
|
+
lease_id,
|
|
4508
|
+
lease_owner_run_id,
|
|
4509
|
+
lease_owner_attempt,
|
|
4510
|
+
lease_expires_at,
|
|
4511
|
+
updated_at
|
|
4512
|
+
)
|
|
4513
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM skipped
|
|
3282
4514
|
`,
|
|
3283
4515
|
[
|
|
3284
4516
|
workReceiptKeyHex(input.key),
|
|
3285
4517
|
RECEIPT_STATUS_SKIPPED,
|
|
3286
4518
|
input.output === null ? null : stringifyPostgresJson(input.output),
|
|
3287
4519
|
input.runId,
|
|
3288
|
-
|
|
4520
|
+
leaseId,
|
|
4521
|
+
runAttempt,
|
|
3289
4522
|
],
|
|
3290
4523
|
);
|
|
3291
4524
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -3302,6 +4535,11 @@ export async function startRuntimeSheetDataset(
|
|
|
3302
4535
|
sheetContract: PlaySheetContract;
|
|
3303
4536
|
rows: Record<string, unknown>[];
|
|
3304
4537
|
runId: string;
|
|
4538
|
+
attemptId?: string | null;
|
|
4539
|
+
attemptOwnerRunId?: string | null;
|
|
4540
|
+
attemptExpiresAt?: string | null;
|
|
4541
|
+
attemptSeq?: number | null;
|
|
4542
|
+
attemptLeaseTtlMs?: number | null;
|
|
3305
4543
|
inputOffset?: number;
|
|
3306
4544
|
force?: boolean;
|
|
3307
4545
|
},
|
|
@@ -3378,6 +4616,7 @@ export async function startRuntimeSheetDataset(
|
|
|
3378
4616
|
skipped: input.rows.length,
|
|
3379
4617
|
pendingRows: [],
|
|
3380
4618
|
completedRows: [],
|
|
4619
|
+
blockedRows: [],
|
|
3381
4620
|
tableNamespace: input.tableNamespace,
|
|
3382
4621
|
timings: [
|
|
3383
4622
|
...timings,
|
|
@@ -3426,7 +4665,12 @@ export async function startRuntimeSheetDataset(
|
|
|
3426
4665
|
const normalizedTableNamespace = normalizeTableNamespace(
|
|
3427
4666
|
input.tableNamespace,
|
|
3428
4667
|
);
|
|
3429
|
-
|
|
4668
|
+
const attemptId = normalizeRuntimeSheetAttemptId(input.attemptId);
|
|
4669
|
+
const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
|
|
4670
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
4671
|
+
runId: input.runId,
|
|
4672
|
+
});
|
|
4673
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
3430
4674
|
const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
|
|
3431
4675
|
const needsTransaction = chunks.length > 1;
|
|
3432
4676
|
|
|
@@ -3441,12 +4685,26 @@ export async function startRuntimeSheetDataset(
|
|
|
3441
4685
|
timings,
|
|
3442
4686
|
},
|
|
3443
4687
|
async (client) => {
|
|
4688
|
+
const attemptExpiryStartedAt = Date.now();
|
|
4689
|
+
const attemptExpiresAt = await mintRuntimeSheetAttemptExpiresAt(
|
|
4690
|
+
client,
|
|
4691
|
+
input.attemptLeaseTtlMs,
|
|
4692
|
+
);
|
|
4693
|
+
timings.push({
|
|
4694
|
+
phase: 'attempt_expiry_mint',
|
|
4695
|
+
ms: Date.now() - attemptExpiryStartedAt,
|
|
4696
|
+
rows: rowEntries.length,
|
|
4697
|
+
});
|
|
3444
4698
|
const prepareStartedAt = Date.now();
|
|
3445
4699
|
const prepared = await prepareRuntimeSheetDatasetRows(client, session, {
|
|
3446
4700
|
chunks,
|
|
3447
4701
|
runId: input.runId,
|
|
3448
4702
|
normalizedPlayName,
|
|
3449
4703
|
normalizedTableNamespace,
|
|
4704
|
+
attemptId,
|
|
4705
|
+
attemptOwnerRunId,
|
|
4706
|
+
attemptExpiresAt,
|
|
4707
|
+
attemptSeq,
|
|
3450
4708
|
physicalInsertColumnsSql,
|
|
3451
4709
|
physicalInsertValuesSql,
|
|
3452
4710
|
physicalRefreshSetSql:
|
|
@@ -3455,19 +4713,15 @@ export async function startRuntimeSheetDataset(
|
|
|
3455
4713
|
outputPhysicalColumns,
|
|
3456
4714
|
force: input.force === true,
|
|
3457
4715
|
});
|
|
3458
|
-
const effectivePrepared = input.force
|
|
3459
|
-
? {
|
|
3460
|
-
inserted: prepared.inserted,
|
|
3461
|
-
pendingKeys: rowEntries.map((entry) => entry.key),
|
|
3462
|
-
}
|
|
3463
|
-
: prepared;
|
|
3464
4716
|
timings.push({
|
|
3465
4717
|
phase: 'prepare_rows_sql',
|
|
3466
4718
|
ms: Date.now() - prepareStartedAt,
|
|
3467
4719
|
rows: rowEntries.length,
|
|
3468
4720
|
chunks: chunks.length,
|
|
3469
|
-
inserted:
|
|
3470
|
-
pending:
|
|
4721
|
+
inserted: prepared.inserted,
|
|
4722
|
+
pending: prepared.rowDispositions.filter(
|
|
4723
|
+
(row) => row.disposition === 'pending',
|
|
4724
|
+
).length,
|
|
3471
4725
|
...(input.force ? { force: true } : {}),
|
|
3472
4726
|
});
|
|
3473
4727
|
const buildStartedAt = Date.now();
|
|
@@ -3479,9 +4733,13 @@ export async function startRuntimeSheetDataset(
|
|
|
3479
4733
|
normalizedPlayName,
|
|
3480
4734
|
normalizedTableNamespace,
|
|
3481
4735
|
runId: input.runId,
|
|
4736
|
+
attemptId,
|
|
4737
|
+
attemptOwnerRunId,
|
|
4738
|
+
attemptExpiresAt,
|
|
4739
|
+
attemptSeq,
|
|
3482
4740
|
timings,
|
|
3483
4741
|
force: input.force === true,
|
|
3484
|
-
...
|
|
4742
|
+
...prepared,
|
|
3485
4743
|
});
|
|
3486
4744
|
timings.push({
|
|
3487
4745
|
phase: 'build_result',
|
|
@@ -3492,7 +4750,11 @@ export async function startRuntimeSheetDataset(
|
|
|
3492
4750
|
pending: built.pendingRows.length,
|
|
3493
4751
|
completed: built.completedRows.length,
|
|
3494
4752
|
});
|
|
3495
|
-
return
|
|
4753
|
+
return {
|
|
4754
|
+
...built,
|
|
4755
|
+
attemptExpiresAt,
|
|
4756
|
+
attemptSeq,
|
|
4757
|
+
};
|
|
3496
4758
|
},
|
|
3497
4759
|
);
|
|
3498
4760
|
timings.push({
|
|
@@ -3505,7 +4767,298 @@ export async function startRuntimeSheetDataset(
|
|
|
3505
4767
|
pending: result.pendingRows.length,
|
|
3506
4768
|
completed: result.completedRows.length,
|
|
3507
4769
|
});
|
|
3508
|
-
return {
|
|
4770
|
+
return {
|
|
4771
|
+
...result,
|
|
4772
|
+
attemptId,
|
|
4773
|
+
attemptOwnerRunId,
|
|
4774
|
+
attemptExpiresAt: result.attemptExpiresAt,
|
|
4775
|
+
attemptSeq,
|
|
4776
|
+
timings,
|
|
4777
|
+
};
|
|
4778
|
+
}
|
|
4779
|
+
|
|
4780
|
+
export async function heartbeatRuntimeSheetAttempt(
|
|
4781
|
+
context: RuntimeApiContext,
|
|
4782
|
+
input: {
|
|
4783
|
+
playName: string;
|
|
4784
|
+
tableNamespace: string;
|
|
4785
|
+
sheetContract: PlaySheetContract;
|
|
4786
|
+
runId: string;
|
|
4787
|
+
attemptId: string;
|
|
4788
|
+
attemptOwnerRunId?: string | null;
|
|
4789
|
+
attemptSeq?: number | null;
|
|
4790
|
+
attemptLeaseTtlMs?: number | null;
|
|
4791
|
+
keys: string[];
|
|
4792
|
+
},
|
|
4793
|
+
): Promise<RuntimeSheetAttemptHeartbeatResult> {
|
|
4794
|
+
const playName = context.playName?.trim() || input.playName;
|
|
4795
|
+
if (!playName) {
|
|
4796
|
+
throw new Error('Runtime sheet attempt heartbeat requires a playName.');
|
|
4797
|
+
}
|
|
4798
|
+
const attemptId = input.attemptId.trim();
|
|
4799
|
+
const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
|
|
4800
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
4801
|
+
runId: input.runId,
|
|
4802
|
+
});
|
|
4803
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
4804
|
+
const attemptLeaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
4805
|
+
input.attemptLeaseTtlMs,
|
|
4806
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
4807
|
+
);
|
|
4808
|
+
const keys = [
|
|
4809
|
+
...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
|
|
4810
|
+
];
|
|
4811
|
+
if (!attemptId || keys.length === 0) {
|
|
4812
|
+
return { renewed: 0, renewedKeys: [], attemptExpiresAt: null };
|
|
4813
|
+
}
|
|
4814
|
+
const timings: RuntimeSheetTiming[] = [];
|
|
4815
|
+
const session = requireRuntimePostgresSession(
|
|
4816
|
+
await getRuntimeDbSession(
|
|
4817
|
+
{
|
|
4818
|
+
...context,
|
|
4819
|
+
playName,
|
|
4820
|
+
runId: context.runId ?? input.runId,
|
|
4821
|
+
},
|
|
4822
|
+
{
|
|
4823
|
+
tableNamespace: input.tableNamespace,
|
|
4824
|
+
logicalTable: 'sheet_rows',
|
|
4825
|
+
operations: ['rows.upsert'],
|
|
4826
|
+
limits: {
|
|
4827
|
+
maxRows: runtimeDbSessionRowLimit(keys.length),
|
|
4828
|
+
},
|
|
4829
|
+
sheetContract: input.sheetContract,
|
|
4830
|
+
timings,
|
|
4831
|
+
},
|
|
4832
|
+
),
|
|
4833
|
+
);
|
|
4834
|
+
return await withRuntimeSheetQueryClient(
|
|
4835
|
+
context,
|
|
4836
|
+
session,
|
|
4837
|
+
{
|
|
4838
|
+
playName,
|
|
4839
|
+
tableNamespace: input.tableNamespace,
|
|
4840
|
+
sheetContract: input.sheetContract,
|
|
4841
|
+
transactional: false,
|
|
4842
|
+
timings,
|
|
4843
|
+
},
|
|
4844
|
+
async (client) => {
|
|
4845
|
+
const { rows } = await client.query<{
|
|
4846
|
+
renewed_keys: string[];
|
|
4847
|
+
renewed: number;
|
|
4848
|
+
attempt_expires_at: Date | string | null;
|
|
4849
|
+
}>(
|
|
4850
|
+
`
|
|
4851
|
+
WITH input_keys AS (
|
|
4852
|
+
SELECT key_values._key
|
|
4853
|
+
FROM unnest($1::text[]) AS key_values(_key)
|
|
4854
|
+
),
|
|
4855
|
+
owned_terminal AS (
|
|
4856
|
+
SELECT target._key
|
|
4857
|
+
FROM ${sheetTable(session)} AS target
|
|
4858
|
+
JOIN input_keys ON target._key = input_keys._key
|
|
4859
|
+
WHERE target._attempt_id = $2::text
|
|
4860
|
+
AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
|
|
4861
|
+
AND target._run_id = $4::text
|
|
4862
|
+
AND COALESCE(target._attempt_seq, 0) = $6::integer
|
|
4863
|
+
AND target._status = 'enriched'
|
|
4864
|
+
),
|
|
4865
|
+
renewed AS (
|
|
4866
|
+
UPDATE ${sheetTable(session)} AS target
|
|
4867
|
+
SET _attempt_expires_at = now() + ($5::double precision * interval '1 millisecond'),
|
|
4868
|
+
_updated_at = now(),
|
|
4869
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
4870
|
+
FROM input_keys
|
|
4871
|
+
WHERE target._key = input_keys._key
|
|
4872
|
+
AND target._attempt_id = $2::text
|
|
4873
|
+
AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
|
|
4874
|
+
AND target._run_id = $4::text
|
|
4875
|
+
AND COALESCE(target._attempt_seq, 0) = $6::integer
|
|
4876
|
+
AND target._status IN ('pending', 'running', 'failed')
|
|
4877
|
+
RETURNING target._key, target._attempt_expires_at
|
|
4878
|
+
),
|
|
4879
|
+
owned_keys AS (
|
|
4880
|
+
SELECT _key FROM renewed
|
|
4881
|
+
UNION
|
|
4882
|
+
SELECT _key FROM owned_terminal
|
|
4883
|
+
)
|
|
4884
|
+
SELECT coalesce((SELECT array_agg(_key ORDER BY _key) FROM owned_keys), '{}'::text[]) AS renewed_keys,
|
|
4885
|
+
count(*)::int AS renewed,
|
|
4886
|
+
max(_attempt_expires_at) AS attempt_expires_at
|
|
4887
|
+
FROM renewed
|
|
4888
|
+
`,
|
|
4889
|
+
[
|
|
4890
|
+
keys,
|
|
4891
|
+
attemptId,
|
|
4892
|
+
attemptOwnerRunId,
|
|
4893
|
+
input.runId,
|
|
4894
|
+
attemptLeaseTtlMs,
|
|
4895
|
+
attemptSeq,
|
|
4896
|
+
],
|
|
4897
|
+
);
|
|
4898
|
+
const row = rows[0];
|
|
4899
|
+
const attemptExpiresAt = row?.attempt_expires_at;
|
|
4900
|
+
return {
|
|
4901
|
+
renewed: Number(row?.renewed ?? 0),
|
|
4902
|
+
renewedKeys: row?.renewed_keys ?? [],
|
|
4903
|
+
attemptExpiresAt:
|
|
4904
|
+
attemptExpiresAt instanceof Date
|
|
4905
|
+
? attemptExpiresAt.toISOString()
|
|
4906
|
+
: typeof attemptExpiresAt === 'string'
|
|
4907
|
+
? new Date(attemptExpiresAt).toISOString()
|
|
4908
|
+
: null,
|
|
4909
|
+
};
|
|
4910
|
+
},
|
|
4911
|
+
);
|
|
4912
|
+
}
|
|
4913
|
+
|
|
4914
|
+
/**
|
|
4915
|
+
* Controlled run-fatal teardown for runtime sheet rows: clear the attempt lease
|
|
4916
|
+
* (`_attempt_*`) on every row the aborting attempt still owns, so an immediate
|
|
4917
|
+
* rerun can claim pending rows and overwrite the aborted attempt's rows without
|
|
4918
|
+
* waiting out the 10-minute lease TTL. The predicate is attempt-fenced (see
|
|
4919
|
+
* `releasableRuntimeSheetAttemptFenceSql` / `decideRuntimeSheetRelease`): it only
|
|
4920
|
+
* touches rows owned by this run's attempt generation, so it can never weaken a
|
|
4921
|
+
* different run's live lease. TTL expiry remains the recovery path for true
|
|
4922
|
+
* crashes where no teardown code runs. When `attemptId` is null every attempt id
|
|
4923
|
+
* the run holds at `attemptSeq` is released (the workers runner mints a distinct
|
|
4924
|
+
* attempt id per map chunk under one owner run id + seq).
|
|
4925
|
+
*/
|
|
4926
|
+
export async function releaseRuntimeSheetAttempt(
|
|
4927
|
+
context: RuntimeApiContext,
|
|
4928
|
+
input: {
|
|
4929
|
+
playName: string;
|
|
4930
|
+
tableNamespace: string;
|
|
4931
|
+
sheetContract: PlaySheetContract;
|
|
4932
|
+
runId: string;
|
|
4933
|
+
attemptId?: string | null;
|
|
4934
|
+
attemptOwnerRunId?: string | null;
|
|
4935
|
+
attemptSeq?: number | null;
|
|
4936
|
+
},
|
|
4937
|
+
): Promise<RuntimeSheetAttemptReleaseResult> {
|
|
4938
|
+
const playName = context.playName?.trim() || input.playName;
|
|
4939
|
+
if (!playName) {
|
|
4940
|
+
throw new Error('Runtime sheet attempt release requires a playName.');
|
|
4941
|
+
}
|
|
4942
|
+
const attemptId = input.attemptId?.trim() || null;
|
|
4943
|
+
const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
|
|
4944
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
4945
|
+
runId: input.runId,
|
|
4946
|
+
});
|
|
4947
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
4948
|
+
if (!attemptOwnerRunId) {
|
|
4949
|
+
return { released: 0, releasedKeys: [] };
|
|
4950
|
+
}
|
|
4951
|
+
const timings: RuntimeSheetTiming[] = [];
|
|
4952
|
+
const session = requireRuntimePostgresSession(
|
|
4953
|
+
await getRuntimeDbSession(
|
|
4954
|
+
{
|
|
4955
|
+
...context,
|
|
4956
|
+
playName,
|
|
4957
|
+
runId: context.runId ?? input.runId,
|
|
4958
|
+
},
|
|
4959
|
+
{
|
|
4960
|
+
tableNamespace: input.tableNamespace,
|
|
4961
|
+
logicalTable: 'sheet_rows',
|
|
4962
|
+
operations: ['rows.upsert'],
|
|
4963
|
+
limits: {},
|
|
4964
|
+
sheetContract: input.sheetContract,
|
|
4965
|
+
timings,
|
|
4966
|
+
},
|
|
4967
|
+
),
|
|
4968
|
+
);
|
|
4969
|
+
return await withRuntimeSheetQueryClient(
|
|
4970
|
+
context,
|
|
4971
|
+
session,
|
|
4972
|
+
{
|
|
4973
|
+
playName,
|
|
4974
|
+
tableNamespace: input.tableNamespace,
|
|
4975
|
+
sheetContract: input.sheetContract,
|
|
4976
|
+
transactional: false,
|
|
4977
|
+
timings,
|
|
4978
|
+
},
|
|
4979
|
+
async (client) => {
|
|
4980
|
+
const { rows } = await client.query<{ _key: string }>(
|
|
4981
|
+
`
|
|
4982
|
+
UPDATE ${sheetTable(session)} AS target
|
|
4983
|
+
SET _attempt_id = NULL,
|
|
4984
|
+
_attempt_owner_run_id = NULL,
|
|
4985
|
+
_attempt_expires_at = NULL,
|
|
4986
|
+
_attempt_seq = NULL,
|
|
4987
|
+
_updated_at = now(),
|
|
4988
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
4989
|
+
WHERE ${releasableRuntimeSheetAttemptFenceSql(
|
|
4990
|
+
'target',
|
|
4991
|
+
'$1::text',
|
|
4992
|
+
'$2::text',
|
|
4993
|
+
'$3',
|
|
4994
|
+
)}
|
|
4995
|
+
RETURNING target._key
|
|
4996
|
+
`,
|
|
4997
|
+
[attemptId, attemptOwnerRunId, attemptSeq],
|
|
4998
|
+
);
|
|
4999
|
+
const releasedKeys = rows
|
|
5000
|
+
.map((row) => row._key)
|
|
5001
|
+
.filter((key): key is string => typeof key === 'string');
|
|
5002
|
+
return { released: releasedKeys.length, releasedKeys };
|
|
5003
|
+
},
|
|
5004
|
+
);
|
|
5005
|
+
}
|
|
5006
|
+
|
|
5007
|
+
/**
|
|
5008
|
+
* Controlled run-fatal teardown for work receipts: return every receipt still
|
|
5009
|
+
* leased `running` by the aborting run+attempt back to `pending` (clearing the
|
|
5010
|
+
* lease) so an immediate rerun can reclaim and recompute the interrupted work
|
|
5011
|
+
* without waiting out the lease TTL. Fenced by owner run id + attempt (see
|
|
5012
|
+
* `workReceiptReleaseOwnerPredicateSql`), so it never touches a receipt a live
|
|
5013
|
+
* competitor legitimately holds. Completed/skipped/failed receipts are terminal
|
|
5014
|
+
* and untouched, preserving cross-run idempotency (already-persisted work is
|
|
5015
|
+
* still reused on rerun). Work receipts for a play share one table, so a single
|
|
5016
|
+
* run-scoped statement covers every stranded receipt.
|
|
5017
|
+
*/
|
|
5018
|
+
export async function releaseRuntimeWorkReceipts(
|
|
5019
|
+
context: RuntimeApiContext,
|
|
5020
|
+
input: {
|
|
5021
|
+
playName: string;
|
|
5022
|
+
runId: string;
|
|
5023
|
+
runAttempt?: number | null;
|
|
5024
|
+
},
|
|
5025
|
+
): Promise<RuntimeWorkReceiptReleaseResult> {
|
|
5026
|
+
const runId = input.runId.trim();
|
|
5027
|
+
if (!runId) {
|
|
5028
|
+
return { released: 0, releasedKeys: [] };
|
|
5029
|
+
}
|
|
5030
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
5031
|
+
const session = await getRuntimeWorkReceiptSessionForRun(context, {
|
|
5032
|
+
playName: input.playName,
|
|
5033
|
+
});
|
|
5034
|
+
return await withRuntimeWorkReceiptClient(
|
|
5035
|
+
context,
|
|
5036
|
+
session,
|
|
5037
|
+
async (client) => {
|
|
5038
|
+
const { rows } = await client.query<{ k: string }>(
|
|
5039
|
+
`
|
|
5040
|
+
UPDATE ${workReceiptTable(session)} AS target
|
|
5041
|
+
SET status = $3::smallint,
|
|
5042
|
+
lease_id = NULL,
|
|
5043
|
+
lease_owner_run_id = NULL,
|
|
5044
|
+
lease_owner_attempt = NULL,
|
|
5045
|
+
lease_expires_at = NULL,
|
|
5046
|
+
updated_at = now()
|
|
5047
|
+
WHERE ${workReceiptReleaseOwnerPredicateSql({
|
|
5048
|
+
receiptTable: 'target',
|
|
5049
|
+
ownerRunIdSql: '$1',
|
|
5050
|
+
ownerRunAttemptSql: '$2',
|
|
5051
|
+
})}
|
|
5052
|
+
RETURNING convert_from(target.k, 'UTF8') AS k
|
|
5053
|
+
`,
|
|
5054
|
+
[runId, runAttempt, RECEIPT_STATUS_PENDING],
|
|
5055
|
+
);
|
|
5056
|
+
const releasedKeys = rows
|
|
5057
|
+
.map((row) => row.k)
|
|
5058
|
+
.filter((key): key is string => typeof key === 'string');
|
|
5059
|
+
return { released: releasedKeys.length, releasedKeys };
|
|
5060
|
+
},
|
|
5061
|
+
);
|
|
3509
5062
|
}
|
|
3510
5063
|
|
|
3511
5064
|
type CompleteRuntimeMapRowChunksInput = {
|
|
@@ -3513,17 +5066,41 @@ type CompleteRuntimeMapRowChunksInput = {
|
|
|
3513
5066
|
physicalUpdateSetSql: string;
|
|
3514
5067
|
physicalColumnProjections: PhysicalSheetColumnProjection[];
|
|
3515
5068
|
runId: string;
|
|
5069
|
+
attemptId: string | null;
|
|
5070
|
+
attemptOwnerRunId: string | null;
|
|
5071
|
+
attemptExpiresAt: string | null;
|
|
5072
|
+
attemptSeq: number;
|
|
3516
5073
|
normalizedPlayName: string;
|
|
3517
5074
|
normalizedTableNamespace: string;
|
|
3518
5075
|
outputFields: string[];
|
|
3519
5076
|
};
|
|
3520
5077
|
|
|
5078
|
+
type RuntimeMapRowsWriteResult = {
|
|
5079
|
+
updated: number;
|
|
5080
|
+
fencedKeys: string[];
|
|
5081
|
+
};
|
|
5082
|
+
|
|
5083
|
+
function writableEnrichedRuntimeSheetAttemptSql(
|
|
5084
|
+
tableAlias: string,
|
|
5085
|
+
attemptOwnerRunIdExpression: string,
|
|
5086
|
+
attemptExpiresAtExpression: string,
|
|
5087
|
+
attemptSeqExpression: string,
|
|
5088
|
+
): string {
|
|
5089
|
+
return `(
|
|
5090
|
+
${tableAlias}._status = 'enriched'
|
|
5091
|
+
AND (
|
|
5092
|
+
COALESCE(${tableAlias}._attempt_seq, 0) < ${attemptSeqExpression}::integer
|
|
5093
|
+
)
|
|
5094
|
+
)`;
|
|
5095
|
+
}
|
|
5096
|
+
|
|
3521
5097
|
async function completeRuntimeMapRowChunks(
|
|
3522
5098
|
client: RuntimeQueryClient,
|
|
3523
5099
|
session: RuntimePostgresSession,
|
|
3524
5100
|
input: CompleteRuntimeMapRowChunksInput,
|
|
3525
|
-
): Promise<
|
|
5101
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
3526
5102
|
let updated = 0;
|
|
5103
|
+
const fencedKeys: string[] = [];
|
|
3527
5104
|
for (const chunk of input.chunks) {
|
|
3528
5105
|
const chunkKeys = chunk.map((row) => row.key);
|
|
3529
5106
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -3538,9 +5115,21 @@ async function completeRuntimeMapRowChunks(
|
|
|
3538
5115
|
'updates.data_patch',
|
|
3539
5116
|
input.physicalColumnProjections,
|
|
3540
5117
|
);
|
|
5118
|
+
const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
|
|
5119
|
+
'target',
|
|
5120
|
+
'$10::text',
|
|
5121
|
+
'$11::timestamptz',
|
|
5122
|
+
'$12',
|
|
5123
|
+
);
|
|
5124
|
+
const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
|
|
5125
|
+
'target',
|
|
5126
|
+
'$10::text',
|
|
5127
|
+
'$12',
|
|
5128
|
+
);
|
|
3541
5129
|
const { rows } = await client.query<{
|
|
3542
5130
|
updated: number;
|
|
3543
5131
|
matched_keys: string[];
|
|
5132
|
+
fenced_keys: string[];
|
|
3544
5133
|
}>(
|
|
3545
5134
|
`WITH updates AS (
|
|
3546
5135
|
SELECT key_values._key,
|
|
@@ -3571,12 +5160,22 @@ async function completeRuntimeMapRowChunks(
|
|
|
3571
5160
|
SET _status = 'enriched',
|
|
3572
5161
|
_run_id = $5::text,
|
|
3573
5162
|
_error = NULL,
|
|
5163
|
+
_attempt_id = $9::text,
|
|
5164
|
+
_attempt_owner_run_id = $10::text,
|
|
5165
|
+
_attempt_expires_at = $11::timestamptz,
|
|
5166
|
+
_attempt_seq = $12::integer,
|
|
3574
5167
|
_updated_at = now(),
|
|
3575
5168
|
_version = ${nextRuntimeSheetVersionExpression(session)},
|
|
3576
5169
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
3577
5170
|
FROM matched_updates AS updates
|
|
3578
5171
|
WHERE target._key = updates.matched_key
|
|
3579
|
-
|
|
5172
|
+
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5173
|
+
AND NOT (${sameOwnerTerminalEpochSql})
|
|
5174
|
+
AND (
|
|
5175
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5176
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5177
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5178
|
+
)
|
|
3580
5179
|
AND (
|
|
3581
5180
|
target._status <> 'enriched'
|
|
3582
5181
|
OR target._run_id IS DISTINCT FROM $5::text
|
|
@@ -3596,6 +5195,23 @@ async function completeRuntimeMapRowChunks(
|
|
|
3596
5195
|
count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
|
|
3597
5196
|
FROM applied_rows
|
|
3598
5197
|
),
|
|
5198
|
+
fenced_rows AS (
|
|
5199
|
+
SELECT updates.matched_key AS _key
|
|
5200
|
+
FROM matched_updates AS updates
|
|
5201
|
+
JOIN ${sheetTable(session)} AS target
|
|
5202
|
+
ON target._key = updates.matched_key
|
|
5203
|
+
WHERE NOT EXISTS (
|
|
5204
|
+
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5205
|
+
)
|
|
5206
|
+
AND (
|
|
5207
|
+
(${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5208
|
+
OR NOT (
|
|
5209
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5210
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5211
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5212
|
+
)
|
|
5213
|
+
)
|
|
5214
|
+
),
|
|
3599
5215
|
summary_counts AS (
|
|
3600
5216
|
SELECT newly_completed,
|
|
3601
5217
|
from_failed,
|
|
@@ -3680,6 +5296,7 @@ async function completeRuntimeMapRowChunks(
|
|
|
3680
5296
|
SELECT
|
|
3681
5297
|
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
3682
5298
|
coalesce((SELECT array_agg(matched_key) FROM matched_updates), '{}'::text[]) AS matched_keys,
|
|
5299
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
|
|
3683
5300
|
(SELECT count(*)::int FROM summary_delta) AS summary_delta_count,
|
|
3684
5301
|
(SELECT count(*)::int FROM column_delta) AS column_delta_count`,
|
|
3685
5302
|
[
|
|
@@ -3691,9 +5308,14 @@ async function completeRuntimeMapRowChunks(
|
|
|
3691
5308
|
input.normalizedPlayName,
|
|
3692
5309
|
input.normalizedTableNamespace,
|
|
3693
5310
|
[...new Set(input.outputFields)],
|
|
5311
|
+
input.attemptId,
|
|
5312
|
+
input.attemptOwnerRunId,
|
|
5313
|
+
input.attemptExpiresAt,
|
|
5314
|
+
input.attemptSeq,
|
|
3694
5315
|
],
|
|
3695
5316
|
);
|
|
3696
5317
|
updated += Number(rows[0]?.updated ?? 0);
|
|
5318
|
+
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
3697
5319
|
|
|
3698
5320
|
const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
|
|
3699
5321
|
if (matchedKeys.size === chunk.length) {
|
|
@@ -3712,16 +5334,18 @@ async function completeRuntimeMapRowChunks(
|
|
|
3712
5334
|
{ ...input, chunks: [repairChunk] },
|
|
3713
5335
|
);
|
|
3714
5336
|
updated += repaired.updated;
|
|
5337
|
+
fencedKeys.push(...repaired.fencedKeys);
|
|
3715
5338
|
}
|
|
3716
|
-
return { updated };
|
|
5339
|
+
return { updated, fencedKeys };
|
|
3717
5340
|
}
|
|
3718
5341
|
|
|
3719
5342
|
async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
3720
5343
|
client: RuntimeQueryClient,
|
|
3721
5344
|
session: RuntimePostgresSession,
|
|
3722
5345
|
input: CompleteRuntimeMapRowChunksInput,
|
|
3723
|
-
): Promise<
|
|
5346
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
3724
5347
|
let updated = 0;
|
|
5348
|
+
const fencedKeys: string[] = [];
|
|
3725
5349
|
for (const chunk of input.chunks) {
|
|
3726
5350
|
const chunkKeys = chunk.map((row) => row.key);
|
|
3727
5351
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -3736,7 +5360,22 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3736
5360
|
'updates.data_patch',
|
|
3737
5361
|
input.physicalColumnProjections,
|
|
3738
5362
|
);
|
|
3739
|
-
const
|
|
5363
|
+
const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
|
|
5364
|
+
'target',
|
|
5365
|
+
'$10::text',
|
|
5366
|
+
'$11::timestamptz',
|
|
5367
|
+
'$12',
|
|
5368
|
+
);
|
|
5369
|
+
const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
|
|
5370
|
+
'target',
|
|
5371
|
+
'$10::text',
|
|
5372
|
+
'$12',
|
|
5373
|
+
);
|
|
5374
|
+
const { rows } = await client.query<{
|
|
5375
|
+
_key?: string;
|
|
5376
|
+
fenced_keys?: string[];
|
|
5377
|
+
updated?: number;
|
|
5378
|
+
}>(
|
|
3740
5379
|
`WITH updates AS (
|
|
3741
5380
|
SELECT key_values._key,
|
|
3742
5381
|
input_index_values.input_index,
|
|
@@ -3770,13 +5409,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3770
5409
|
SET _status = 'enriched',
|
|
3771
5410
|
_run_id = $5::text,
|
|
3772
5411
|
_error = NULL,
|
|
5412
|
+
_attempt_id = $9::text,
|
|
5413
|
+
_attempt_owner_run_id = $10::text,
|
|
5414
|
+
_attempt_expires_at = $11::timestamptz,
|
|
5415
|
+
_attempt_seq = $12::integer,
|
|
3773
5416
|
_updated_at = now(),
|
|
3774
5417
|
_version = ${nextRuntimeSheetVersionExpression(session)},
|
|
3775
5418
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
3776
5419
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
3777
5420
|
WHERE target._key = updates.matched_key
|
|
3778
5421
|
AND prev._key = target._key
|
|
3779
|
-
|
|
5422
|
+
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5423
|
+
AND NOT (${sameOwnerTerminalEpochSql})
|
|
5424
|
+
AND (
|
|
5425
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5426
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5427
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5428
|
+
)
|
|
3780
5429
|
AND (
|
|
3781
5430
|
target._status <> 'enriched'
|
|
3782
5431
|
OR target._run_id IS DISTINCT FROM $5::text
|
|
@@ -3796,6 +5445,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3796
5445
|
count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
|
|
3797
5446
|
FROM applied_rows
|
|
3798
5447
|
),
|
|
5448
|
+
fenced_rows AS (
|
|
5449
|
+
SELECT updates.matched_key AS _key
|
|
5450
|
+
FROM matched_updates AS updates
|
|
5451
|
+
JOIN ${sheetTable(session)} AS target
|
|
5452
|
+
ON target._key = updates.matched_key
|
|
5453
|
+
WHERE NOT EXISTS (
|
|
5454
|
+
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5455
|
+
)
|
|
5456
|
+
AND (
|
|
5457
|
+
(${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5458
|
+
OR NOT (
|
|
5459
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5460
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5461
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5462
|
+
)
|
|
5463
|
+
)
|
|
5464
|
+
),
|
|
3799
5465
|
summary_counts AS (
|
|
3800
5466
|
SELECT newly_completed,
|
|
3801
5467
|
from_failed,
|
|
@@ -3877,7 +5543,9 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3877
5543
|
_updated_at = now()
|
|
3878
5544
|
RETURNING 1
|
|
3879
5545
|
)
|
|
3880
|
-
SELECT
|
|
5546
|
+
SELECT
|
|
5547
|
+
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
5548
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
|
|
3881
5549
|
[
|
|
3882
5550
|
chunkKeys,
|
|
3883
5551
|
chunkInputIndexes,
|
|
@@ -3887,11 +5555,16 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3887
5555
|
input.normalizedPlayName,
|
|
3888
5556
|
input.normalizedTableNamespace,
|
|
3889
5557
|
[...new Set(input.outputFields)],
|
|
5558
|
+
input.attemptId,
|
|
5559
|
+
input.attemptOwnerRunId,
|
|
5560
|
+
input.attemptExpiresAt,
|
|
5561
|
+
input.attemptSeq,
|
|
3890
5562
|
],
|
|
3891
5563
|
);
|
|
3892
|
-
updated +=
|
|
5564
|
+
updated += Number(rows[0]?.updated ?? 0);
|
|
5565
|
+
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
3893
5566
|
}
|
|
3894
|
-
return { updated };
|
|
5567
|
+
return { updated, fencedKeys };
|
|
3895
5568
|
}
|
|
3896
5569
|
|
|
3897
5570
|
async function insertMissingCompletedMapRowChunks(
|
|
@@ -3954,8 +5627,26 @@ async function insertMissingCompletedMapRowChunks(
|
|
|
3954
5627
|
)
|
|
3955
5628
|
),
|
|
3956
5629
|
inserted_rows AS (
|
|
3957
|
-
INSERT INTO ${sheetTable(session)} (
|
|
3958
|
-
|
|
5630
|
+
INSERT INTO ${sheetTable(session)} (
|
|
5631
|
+
_key,
|
|
5632
|
+
_status,
|
|
5633
|
+
_run_id,
|
|
5634
|
+
_input_index,
|
|
5635
|
+
_attempt_id,
|
|
5636
|
+
_attempt_owner_run_id,
|
|
5637
|
+
_attempt_expires_at,
|
|
5638
|
+
_attempt_seq,
|
|
5639
|
+
_cell_meta${physicalInsertColumnsSql}
|
|
5640
|
+
)
|
|
5641
|
+
SELECT _key,
|
|
5642
|
+
'enriched',
|
|
5643
|
+
$5::text,
|
|
5644
|
+
input_index,
|
|
5645
|
+
$8::text,
|
|
5646
|
+
$9::text,
|
|
5647
|
+
$10::timestamptz,
|
|
5648
|
+
$11::integer,
|
|
5649
|
+
cell_meta_patch${physicalInsertValuesSql}
|
|
3959
5650
|
FROM missing_rows
|
|
3960
5651
|
ON CONFLICT (_key) DO NOTHING
|
|
3961
5652
|
RETURNING _key
|
|
@@ -3997,7 +5688,7 @@ async function insertMissingCompletedMapRowChunks(
|
|
|
3997
5688
|
),
|
|
3998
5689
|
completed_cell_delta AS (
|
|
3999
5690
|
SELECT field_values.field, (SELECT c FROM inserted_count) AS c
|
|
4000
|
-
FROM unnest($
|
|
5691
|
+
FROM unnest($12::text[]) AS field_values(field)
|
|
4001
5692
|
WHERE (SELECT c FROM inserted_count) > 0
|
|
4002
5693
|
),
|
|
4003
5694
|
column_delta AS (
|
|
@@ -4028,6 +5719,10 @@ async function insertMissingCompletedMapRowChunks(
|
|
|
4028
5719
|
input.runId,
|
|
4029
5720
|
input.normalizedPlayName,
|
|
4030
5721
|
input.normalizedTableNamespace,
|
|
5722
|
+
input.attemptId,
|
|
5723
|
+
input.attemptOwnerRunId,
|
|
5724
|
+
input.attemptExpiresAt,
|
|
5725
|
+
input.attemptSeq,
|
|
4031
5726
|
[...new Set(input.outputFields)],
|
|
4032
5727
|
],
|
|
4033
5728
|
);
|
|
@@ -4053,12 +5748,18 @@ async function failRuntimeMapRowChunks(
|
|
|
4053
5748
|
input: {
|
|
4054
5749
|
chunks: RuntimePreparedFailedRow[][];
|
|
4055
5750
|
physicalUpdateSetSql: string;
|
|
5751
|
+
forceTerminal?: boolean;
|
|
4056
5752
|
runId: string;
|
|
5753
|
+
attemptId: string | null;
|
|
5754
|
+
attemptOwnerRunId: string | null;
|
|
5755
|
+
attemptExpiresAt: string | null;
|
|
5756
|
+
attemptSeq: number;
|
|
4057
5757
|
normalizedPlayName: string;
|
|
4058
5758
|
normalizedTableNamespace: string;
|
|
4059
5759
|
},
|
|
4060
|
-
): Promise<
|
|
5760
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
4061
5761
|
let updated = 0;
|
|
5762
|
+
const fencedKeys: string[] = [];
|
|
4062
5763
|
for (const chunk of input.chunks) {
|
|
4063
5764
|
const chunkKeys = chunk.map((row) => row.key);
|
|
4064
5765
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -4087,7 +5788,10 @@ async function failRuntimeMapRowChunks(
|
|
|
4087
5788
|
const failedCellTotals = failedCellFields.map(
|
|
4088
5789
|
(field) => failedCellCounts.get(field) ?? 0,
|
|
4089
5790
|
);
|
|
4090
|
-
const { rows
|
|
5791
|
+
const { rows } = await client.query<{
|
|
5792
|
+
updated?: number;
|
|
5793
|
+
fenced_keys?: string[];
|
|
5794
|
+
}>(
|
|
4091
5795
|
`WITH updates AS (
|
|
4092
5796
|
SELECT key_values._key,
|
|
4093
5797
|
input_index_values.input_index,
|
|
@@ -4125,26 +5829,53 @@ async function failRuntimeMapRowChunks(
|
|
|
4125
5829
|
SET _status = 'failed',
|
|
4126
5830
|
_run_id = $6::text,
|
|
4127
5831
|
_error = updates.error,
|
|
5832
|
+
_attempt_id = $11::text,
|
|
5833
|
+
_attempt_owner_run_id = $12::text,
|
|
5834
|
+
_attempt_expires_at = $13::timestamptz,
|
|
5835
|
+
_attempt_seq = $14::integer,
|
|
4128
5836
|
_updated_at = now(),
|
|
4129
5837
|
_version = ${nextRuntimeSheetVersionExpression(session)},
|
|
4130
5838
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
4131
5839
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
4132
5840
|
WHERE target._key = updates.matched_key
|
|
4133
5841
|
AND prev._key = target._key
|
|
4134
|
-
AND target._status <> 'enriched'
|
|
4135
|
-
|
|
5842
|
+
AND ($15::boolean OR target._status <> 'enriched')
|
|
5843
|
+
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
|
|
5844
|
+
AND (
|
|
5845
|
+
($11::text IS NULL AND target._run_id = $6::text)
|
|
5846
|
+
OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
|
|
5847
|
+
)
|
|
4136
5848
|
RETURNING target._key, prev._status AS prev_status
|
|
4137
5849
|
),
|
|
4138
5850
|
applied_count AS (
|
|
4139
5851
|
SELECT count(*)::bigint AS c,
|
|
4140
5852
|
count(*) FILTER (WHERE prev_status = 'failed')::bigint AS already_failed,
|
|
5853
|
+
count(*) FILTER (WHERE prev_status = 'enriched')::bigint AS from_completed,
|
|
4141
5854
|
count(*) FILTER (WHERE prev_status = 'running')::bigint AS from_running
|
|
4142
5855
|
FROM applied_rows
|
|
4143
5856
|
),
|
|
5857
|
+
fenced_rows AS (
|
|
5858
|
+
SELECT updates.matched_key AS _key
|
|
5859
|
+
FROM matched_updates AS updates
|
|
5860
|
+
JOIN ${sheetTable(session)} AS target
|
|
5861
|
+
ON target._key = updates.matched_key
|
|
5862
|
+
WHERE NOT EXISTS (
|
|
5863
|
+
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5864
|
+
)
|
|
5865
|
+
AND (
|
|
5866
|
+
(NOT $15::boolean AND target._status = 'enriched')
|
|
5867
|
+
OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
|
|
5868
|
+
OR NOT (
|
|
5869
|
+
($11::text IS NULL AND target._run_id = $6::text)
|
|
5870
|
+
OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
|
|
5871
|
+
)
|
|
5872
|
+
)
|
|
5873
|
+
),
|
|
4144
5874
|
summary_counts AS (
|
|
4145
5875
|
SELECT (c - already_failed)::bigint AS newly_failed,
|
|
5876
|
+
from_completed,
|
|
4146
5877
|
from_running,
|
|
4147
|
-
GREATEST(c - already_failed - from_running, 0)::bigint AS from_queued
|
|
5878
|
+
GREATEST(c - already_failed - from_completed - from_running, 0)::bigint AS from_queued
|
|
4148
5879
|
FROM applied_count
|
|
4149
5880
|
),
|
|
4150
5881
|
summary_delta AS (
|
|
@@ -4162,20 +5893,21 @@ async function failRuntimeMapRowChunks(
|
|
|
4162
5893
|
0,
|
|
4163
5894
|
(-from_queued)::int,
|
|
4164
5895
|
(-from_running)::int,
|
|
4165
|
-
|
|
5896
|
+
(-from_completed)::int,
|
|
4166
5897
|
newly_failed::int
|
|
4167
5898
|
FROM summary_counts
|
|
4168
5899
|
WHERE newly_failed > 0
|
|
4169
5900
|
ON CONFLICT (play_name, table_namespace) DO UPDATE SET
|
|
4170
5901
|
queued = GREATEST(target.queued + EXCLUDED.queued, 0),
|
|
4171
5902
|
running = GREATEST(target.running + EXCLUDED.running, 0),
|
|
5903
|
+
completed = GREATEST(target.completed + EXCLUDED.completed, 0),
|
|
4172
5904
|
failed = GREATEST(target.failed + EXCLUDED.failed, 0),
|
|
4173
5905
|
total = ${runtimeSummaryTotalSql({
|
|
4174
5906
|
currentTotal: 'target.total',
|
|
4175
5907
|
totalDelta: 'EXCLUDED.total',
|
|
4176
5908
|
queued: 'GREATEST(target.queued + EXCLUDED.queued, 0)',
|
|
4177
5909
|
running: 'GREATEST(target.running + EXCLUDED.running, 0)',
|
|
4178
|
-
completed: 'target.completed',
|
|
5910
|
+
completed: 'GREATEST(target.completed + EXCLUDED.completed, 0)',
|
|
4179
5911
|
failed: 'GREATEST(target.failed + EXCLUDED.failed, 0)',
|
|
4180
5912
|
})},
|
|
4181
5913
|
_updated_at = now()
|
|
@@ -4198,7 +5930,9 @@ async function failRuntimeMapRowChunks(
|
|
|
4198
5930
|
_updated_at = now()
|
|
4199
5931
|
RETURNING 1
|
|
4200
5932
|
)
|
|
4201
|
-
SELECT
|
|
5933
|
+
SELECT
|
|
5934
|
+
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
5935
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
|
|
4202
5936
|
[
|
|
4203
5937
|
chunkKeys,
|
|
4204
5938
|
chunkInputIndexes,
|
|
@@ -4210,11 +5944,17 @@ async function failRuntimeMapRowChunks(
|
|
|
4210
5944
|
input.normalizedTableNamespace,
|
|
4211
5945
|
failedCellFields,
|
|
4212
5946
|
failedCellTotals,
|
|
5947
|
+
input.attemptId,
|
|
5948
|
+
input.attemptOwnerRunId,
|
|
5949
|
+
input.attemptExpiresAt,
|
|
5950
|
+
input.attemptSeq,
|
|
5951
|
+
input.forceTerminal === true,
|
|
4213
5952
|
],
|
|
4214
5953
|
);
|
|
4215
|
-
updated +=
|
|
5954
|
+
updated += Number(rows[0]?.updated ?? 0);
|
|
5955
|
+
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
4216
5956
|
}
|
|
4217
|
-
return { updated };
|
|
5957
|
+
return { updated, fencedKeys };
|
|
4218
5958
|
}
|
|
4219
5959
|
|
|
4220
5960
|
/**
|
|
@@ -4240,10 +5980,15 @@ export async function completeRuntimeMapRows(
|
|
|
4240
5980
|
rows: RuntimeApiRowRecord[];
|
|
4241
5981
|
outputFields?: string[];
|
|
4242
5982
|
runId: string;
|
|
5983
|
+
attemptId?: string | null;
|
|
5984
|
+
attemptOwnerRunId?: string | null;
|
|
5985
|
+
attemptExpiresAt?: string | null;
|
|
5986
|
+
attemptSeq?: number | null;
|
|
5987
|
+
forceFailedRows?: boolean;
|
|
4243
5988
|
},
|
|
4244
|
-
): Promise<
|
|
5989
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
4245
5990
|
if (input.rows.length === 0) {
|
|
4246
|
-
return { updated: 0 };
|
|
5991
|
+
return { updated: 0, fencedKeys: [] };
|
|
4247
5992
|
}
|
|
4248
5993
|
const sheetContract = augmentSheetContractWithDatasetFields({
|
|
4249
5994
|
contract: input.sheetContract,
|
|
@@ -4276,7 +6021,7 @@ export async function completeRuntimeMapRows(
|
|
|
4276
6021
|
uniqueRows.set(row.key, row);
|
|
4277
6022
|
}
|
|
4278
6023
|
if (uniqueRows.size === 0) {
|
|
4279
|
-
return { updated: 0 };
|
|
6024
|
+
return { updated: 0, fencedKeys: [] };
|
|
4280
6025
|
}
|
|
4281
6026
|
|
|
4282
6027
|
const projections = physicalSheetColumnProjections(sheetContract);
|
|
@@ -4304,6 +6049,11 @@ export async function completeRuntimeMapRows(
|
|
|
4304
6049
|
const failedChunks = chunkValues(failedRows, DIRECT_POSTGRES_BATCH_SIZE);
|
|
4305
6050
|
const needsTransaction = chunks.length + failedChunks.length > 1;
|
|
4306
6051
|
const outputFields = [...new Set(input.outputFields ?? [])];
|
|
6052
|
+
const attemptId = input.attemptId?.trim() || null;
|
|
6053
|
+
const attemptOwnerRunId =
|
|
6054
|
+
input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
|
|
6055
|
+
const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
|
|
6056
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
4307
6057
|
|
|
4308
6058
|
return await withRuntimeSheetQueryClient(
|
|
4309
6059
|
context,
|
|
@@ -4323,6 +6073,10 @@ export async function completeRuntimeMapRows(
|
|
|
4323
6073
|
physicalUpdateSetSql,
|
|
4324
6074
|
physicalColumnProjections: projections,
|
|
4325
6075
|
runId: input.runId,
|
|
6076
|
+
attemptId,
|
|
6077
|
+
attemptOwnerRunId,
|
|
6078
|
+
attemptExpiresAt,
|
|
6079
|
+
attemptSeq,
|
|
4326
6080
|
normalizedPlayName: normalizePlayNameForSheet(session.playName),
|
|
4327
6081
|
normalizedTableNamespace: normalizeTableNamespace(
|
|
4328
6082
|
input.tableNamespace,
|
|
@@ -4339,22 +6093,33 @@ export async function completeRuntimeMapRows(
|
|
|
4339
6093
|
session,
|
|
4340
6094
|
chunkInput,
|
|
4341
6095
|
);
|
|
4342
|
-
return {
|
|
6096
|
+
return {
|
|
6097
|
+
updated: updated.updated + inserted.inserted,
|
|
6098
|
+
fencedKeys: updated.fencedKeys,
|
|
6099
|
+
};
|
|
4343
6100
|
})()
|
|
4344
|
-
: { updated: 0 };
|
|
6101
|
+
: { updated: 0, fencedKeys: [] };
|
|
4345
6102
|
const failed =
|
|
4346
6103
|
failedRows.length > 0
|
|
4347
6104
|
? await failRuntimeMapRowChunks(client, session, {
|
|
4348
6105
|
chunks: failedChunks,
|
|
4349
6106
|
physicalUpdateSetSql,
|
|
6107
|
+
forceTerminal: input.forceFailedRows === true,
|
|
4350
6108
|
runId: input.runId,
|
|
6109
|
+
attemptId,
|
|
6110
|
+
attemptOwnerRunId,
|
|
6111
|
+
attemptExpiresAt,
|
|
6112
|
+
attemptSeq,
|
|
4351
6113
|
normalizedPlayName: normalizePlayNameForSheet(session.playName),
|
|
4352
6114
|
normalizedTableNamespace: normalizeTableNamespace(
|
|
4353
6115
|
input.tableNamespace,
|
|
4354
6116
|
),
|
|
4355
6117
|
})
|
|
4356
|
-
: { updated: 0 };
|
|
4357
|
-
return {
|
|
6118
|
+
: { updated: 0, fencedKeys: [] };
|
|
6119
|
+
return {
|
|
6120
|
+
updated: completed.updated + failed.updated,
|
|
6121
|
+
fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
|
|
6122
|
+
};
|
|
4358
6123
|
},
|
|
4359
6124
|
);
|
|
4360
6125
|
}
|
|
@@ -4408,7 +6173,7 @@ export async function readRuntimeSheetDatasetRowKeys(
|
|
|
4408
6173
|
tableNamespace: string;
|
|
4409
6174
|
runId: string;
|
|
4410
6175
|
keys: string[];
|
|
4411
|
-
rowMode?: 'output' | 'all';
|
|
6176
|
+
rowMode?: 'output' | 'all' | 'terminalAnyRun';
|
|
4412
6177
|
},
|
|
4413
6178
|
): Promise<{ keys: string[] }> {
|
|
4414
6179
|
const keys = [
|
|
@@ -4435,6 +6200,16 @@ export async function readRuntimeSheetDatasetRowKeys(
|
|
|
4435
6200
|
),
|
|
4436
6201
|
);
|
|
4437
6202
|
const rows = await withRuntimePostgres(session, async (client) => {
|
|
6203
|
+
if (input.rowMode === 'terminalAnyRun') {
|
|
6204
|
+
const { rows: matchedRows } = await client.query<{ _key: string }>(
|
|
6205
|
+
`SELECT _key
|
|
6206
|
+
FROM ${sheetTable(session)}
|
|
6207
|
+
WHERE _key = ANY($1::text[])
|
|
6208
|
+
AND _status IN ('enriched', 'failed')`,
|
|
6209
|
+
[keys],
|
|
6210
|
+
);
|
|
6211
|
+
return matchedRows;
|
|
6212
|
+
}
|
|
4438
6213
|
if (input.rowMode === 'all') {
|
|
4439
6214
|
const { rows: matchedRows } = await client.query<{ _key: string }>(
|
|
4440
6215
|
`SELECT _key
|