deepline 0.1.182 → 0.1.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +639 -109
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1752 -1899
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +41 -0
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +133 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1612 -287
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +105 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2038 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +28 -11
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +38 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +76 -17
- package/dist/cli/index.mjs +76 -17
- package/dist/index.d.mts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +63 -9
- package/dist/index.mjs +63 -9
- package/dist/plays/bundle-play-file.mjs +20 -6
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import {
|
|
3
3
|
createRuntimePool,
|
|
4
4
|
createRuntimeOneShotQueryClient,
|
|
@@ -55,12 +55,28 @@ import {
|
|
|
55
55
|
isReusableWorkReceipt,
|
|
56
56
|
type WorkReceipt,
|
|
57
57
|
type WorkReceiptClaim,
|
|
58
|
+
workReceiptFailureKindCode,
|
|
59
|
+
workReceiptFailureKindFromCode,
|
|
60
|
+
type WorkReceiptFailureKind,
|
|
58
61
|
} from './work-receipts';
|
|
59
62
|
import {
|
|
60
63
|
sanitizePostgresJsonValue,
|
|
61
64
|
stringifyPostgresJson,
|
|
62
65
|
} from './postgres-json';
|
|
63
66
|
import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
|
|
67
|
+
import {
|
|
68
|
+
workReceiptClaimableStatusCodes,
|
|
69
|
+
workReceiptClaimConflictPredicateSql,
|
|
70
|
+
workReceiptHeartbeatPredicateSql,
|
|
71
|
+
workReceiptReleaseOwnerPredicateSql,
|
|
72
|
+
workReceiptRunningOwnerPredicateSql,
|
|
73
|
+
} from './receipt-sql';
|
|
74
|
+
import {
|
|
75
|
+
activeRuntimeSheetAttemptFenceSql,
|
|
76
|
+
newerTerminalRuntimeSheetRowSql,
|
|
77
|
+
releasableRuntimeSheetAttemptFenceSql,
|
|
78
|
+
sameOwnerTerminalAttemptEpochSql,
|
|
79
|
+
} from './sheet-attempt-sql';
|
|
64
80
|
import type { MapRowOutcome } from './durability-store';
|
|
65
81
|
import {
|
|
66
82
|
normalizeRuntimeMapInputIndex,
|
|
@@ -75,6 +91,20 @@ import {
|
|
|
75
91
|
} from './map-row-outcome';
|
|
76
92
|
import { DEEPLINE_CELL_META_FIELD } from './cell-staleness';
|
|
77
93
|
import type { PlayArtifactKind } from './backend';
|
|
94
|
+
import {
|
|
95
|
+
PLAY_RUNTIME_CONTRACT,
|
|
96
|
+
PLAY_RUNTIME_CONTRACT_HEADER,
|
|
97
|
+
} from './runtime-contract';
|
|
98
|
+
import { PLAY_RUNTIME_API_COMPAT_PATH } from './runtime-api-paths';
|
|
99
|
+
import {
|
|
100
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
101
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
102
|
+
} from './lease-policy';
|
|
103
|
+
import {
|
|
104
|
+
parseRuntimeTestFaultCounts,
|
|
105
|
+
PLAY_RUNTIME_TEST_FAULT_HEADER,
|
|
106
|
+
type RuntimeTestFaultName,
|
|
107
|
+
} from './test-runtime-seams';
|
|
78
108
|
|
|
79
109
|
type RuntimeApiContext = {
|
|
80
110
|
baseUrl?: string | null;
|
|
@@ -87,10 +117,133 @@ type RuntimeApiContext = {
|
|
|
87
117
|
dbSessionStrategy?: 'preloaded' | 'sandbox_public_key' | null;
|
|
88
118
|
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
89
119
|
vercelProtectionBypassToken?: string | null;
|
|
120
|
+
runtimeTestFaultHeader?: string | null;
|
|
90
121
|
disablePostgresPoolCache?: boolean | null;
|
|
91
122
|
postgresSessionUnwrapKey?: string | null;
|
|
92
123
|
};
|
|
93
124
|
|
|
125
|
+
type DirectRuntimeTestFaultState = {
|
|
126
|
+
headerValue: string;
|
|
127
|
+
counts: Partial<Record<RuntimeTestFaultName, number>>;
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const directRuntimeTestFaults = new Map<string, DirectRuntimeTestFaultState>();
|
|
131
|
+
|
|
132
|
+
function consumeDirectRuntimeTestFault(input: {
|
|
133
|
+
context: RuntimeApiContext;
|
|
134
|
+
runId: string;
|
|
135
|
+
name: RuntimeTestFaultName;
|
|
136
|
+
}): boolean {
|
|
137
|
+
const headerValue = input.context.runtimeTestFaultHeader?.trim();
|
|
138
|
+
if (!headerValue) return false;
|
|
139
|
+
const stateKey = input.runId.trim();
|
|
140
|
+
if (!stateKey) return false;
|
|
141
|
+
let state = directRuntimeTestFaults.get(stateKey);
|
|
142
|
+
if (!state || state.headerValue !== headerValue) {
|
|
143
|
+
const parsed = parseRuntimeTestFaultCounts({
|
|
144
|
+
headerValue,
|
|
145
|
+
syntheticRunHeader: '1',
|
|
146
|
+
});
|
|
147
|
+
if (parsed.ok === false) {
|
|
148
|
+
throw new Error(parsed.error);
|
|
149
|
+
}
|
|
150
|
+
state = {
|
|
151
|
+
headerValue,
|
|
152
|
+
counts:
|
|
153
|
+
(parsed.counts as Partial<Record<RuntimeTestFaultName, number>>) ?? {},
|
|
154
|
+
};
|
|
155
|
+
directRuntimeTestFaults.set(stateKey, state);
|
|
156
|
+
}
|
|
157
|
+
const count = state.counts[input.name] ?? 0;
|
|
158
|
+
if (count <= 0) return false;
|
|
159
|
+
if (count === 1) {
|
|
160
|
+
delete state.counts[input.name];
|
|
161
|
+
} else {
|
|
162
|
+
state.counts[input.name] = count - 1;
|
|
163
|
+
}
|
|
164
|
+
state.headerValue = headerValue;
|
|
165
|
+
directRuntimeTestFaults.set(stateKey, state);
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function runtimeTestFaultError(name: RuntimeTestFaultName): Error {
|
|
170
|
+
const error = new Error(`Injected runtime test fault: ${name}.`);
|
|
171
|
+
if (
|
|
172
|
+
name === 'receipt_complete_write_fail' ||
|
|
173
|
+
name === 'receipt_fail_write_fail'
|
|
174
|
+
) {
|
|
175
|
+
error.name = 'RuntimeReceiptPersistenceError';
|
|
176
|
+
}
|
|
177
|
+
return error;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function forceFailRuntimeWorkReceiptForRuntimeTestFault(
|
|
181
|
+
context: RuntimeApiContext,
|
|
182
|
+
input: {
|
|
183
|
+
playName: string;
|
|
184
|
+
runId: string;
|
|
185
|
+
key: string;
|
|
186
|
+
error: string;
|
|
187
|
+
failureKind: WorkReceiptFailureKind;
|
|
188
|
+
runAttempt?: number | null;
|
|
189
|
+
},
|
|
190
|
+
): Promise<WorkReceipt | null> {
|
|
191
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
192
|
+
const session = await getRuntimeWorkReceiptSession(context, {
|
|
193
|
+
playName: input.playName,
|
|
194
|
+
key: input.key,
|
|
195
|
+
});
|
|
196
|
+
return await withRuntimeWorkReceiptClient(
|
|
197
|
+
context,
|
|
198
|
+
session,
|
|
199
|
+
async (client) => {
|
|
200
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
201
|
+
`
|
|
202
|
+
WITH failed AS (
|
|
203
|
+
UPDATE ${workReceiptTable(session)}
|
|
204
|
+
SET status = $2::smallint,
|
|
205
|
+
output = NULL,
|
|
206
|
+
error = $3,
|
|
207
|
+
failure_kind = $5::smallint,
|
|
208
|
+
lease_id = NULL,
|
|
209
|
+
lease_owner_run_id = NULL,
|
|
210
|
+
lease_owner_attempt = NULL,
|
|
211
|
+
lease_expires_at = NULL,
|
|
212
|
+
updated_at = now()
|
|
213
|
+
WHERE k = decode($1, 'hex')
|
|
214
|
+
AND run_id = $4
|
|
215
|
+
AND COALESCE(lease_owner_attempt, 0) = $7::integer
|
|
216
|
+
AND status <> $6::smallint
|
|
217
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
218
|
+
status,
|
|
219
|
+
output,
|
|
220
|
+
error,
|
|
221
|
+
failure_kind,
|
|
222
|
+
run_id,
|
|
223
|
+
lease_id,
|
|
224
|
+
lease_owner_run_id,
|
|
225
|
+
lease_owner_attempt,
|
|
226
|
+
lease_expires_at,
|
|
227
|
+
updated_at
|
|
228
|
+
)
|
|
229
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at
|
|
230
|
+
FROM failed
|
|
231
|
+
`,
|
|
232
|
+
[
|
|
233
|
+
workReceiptKeyHex(input.key),
|
|
234
|
+
RECEIPT_STATUS_FAILED,
|
|
235
|
+
input.error,
|
|
236
|
+
input.runId,
|
|
237
|
+
workReceiptFailureKindCode(input.failureKind),
|
|
238
|
+
RECEIPT_STATUS_COMPLETED,
|
|
239
|
+
runAttempt,
|
|
240
|
+
],
|
|
241
|
+
);
|
|
242
|
+
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
243
|
+
},
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
|
|
94
247
|
type DbSessionCacheEntry = {
|
|
95
248
|
session: CreateDbSessionResponse;
|
|
96
249
|
};
|
|
@@ -101,6 +254,11 @@ type RuntimeDatasetRowEntry = {
|
|
|
101
254
|
row: Record<string, unknown>;
|
|
102
255
|
inputIndex: number;
|
|
103
256
|
};
|
|
257
|
+
type RuntimeSheetPrepareDisposition = 'pending' | 'completed' | 'blocked';
|
|
258
|
+
type RuntimeSheetPreparedRowDisposition = {
|
|
259
|
+
key: string;
|
|
260
|
+
disposition: RuntimeSheetPrepareDisposition;
|
|
261
|
+
};
|
|
104
262
|
const dbSessionCache = new Map<string, DbSessionCacheEntry>();
|
|
105
263
|
const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
|
|
106
264
|
const postgresPools = new Map<string, RuntimePool>();
|
|
@@ -119,6 +277,23 @@ const APPEND_KEY_SUFFIX_LENGTH = 12;
|
|
|
119
277
|
// 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
|
|
120
278
|
// the tenant runtime role is long-lived), not run failures.
|
|
121
279
|
const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
|
|
280
|
+
const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
|
|
281
|
+
'lease_id',
|
|
282
|
+
'lease_owner_run_id',
|
|
283
|
+
'lease_owner_attempt',
|
|
284
|
+
'lease_expires_at',
|
|
285
|
+
] as const;
|
|
286
|
+
const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
|
|
287
|
+
const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
|
|
288
|
+
...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
|
|
289
|
+
RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
|
|
290
|
+
] as const;
|
|
291
|
+
const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
|
|
292
|
+
'_attempt_id',
|
|
293
|
+
'_attempt_owner_run_id',
|
|
294
|
+
'_attempt_seq',
|
|
295
|
+
'_attempt_expires_at',
|
|
296
|
+
] as const;
|
|
122
297
|
const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
|
|
123
298
|
const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [
|
|
124
299
|
100, 250, 500, 1_000, 2_000,
|
|
@@ -178,8 +353,13 @@ export type PrepareRuntimeSheetResult = {
|
|
|
178
353
|
skipped: number;
|
|
179
354
|
pendingRows: Record<string, unknown>[];
|
|
180
355
|
completedRows: Record<string, unknown>[];
|
|
356
|
+
blockedRows: Record<string, unknown>[];
|
|
181
357
|
tableNamespace: string;
|
|
182
358
|
timings?: RuntimeSheetTiming[];
|
|
359
|
+
attemptId?: string;
|
|
360
|
+
attemptOwnerRunId?: string;
|
|
361
|
+
attemptExpiresAt?: string;
|
|
362
|
+
attemptSeq?: number;
|
|
183
363
|
};
|
|
184
364
|
|
|
185
365
|
export type RuntimeSheetTiming = {
|
|
@@ -197,6 +377,22 @@ export type RuntimeSheetTiming = {
|
|
|
197
377
|
error?: string;
|
|
198
378
|
};
|
|
199
379
|
|
|
380
|
+
export type RuntimeSheetAttemptHeartbeatResult = {
|
|
381
|
+
renewed: number;
|
|
382
|
+
renewedKeys: string[];
|
|
383
|
+
attemptExpiresAt: string | null;
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
export type RuntimeSheetAttemptReleaseResult = {
|
|
387
|
+
released: number;
|
|
388
|
+
releasedKeys: string[];
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
export type RuntimeWorkReceiptReleaseResult = {
|
|
392
|
+
released: number;
|
|
393
|
+
releasedKeys: string[];
|
|
394
|
+
};
|
|
395
|
+
|
|
200
396
|
type RuntimeApiActionRequest =
|
|
201
397
|
| {
|
|
202
398
|
action: 'resolve_play';
|
|
@@ -261,7 +457,7 @@ function resolveRuntimeApiUrl(context: RuntimeApiContext): string {
|
|
|
261
457
|
throw new Error('Runner runtime API requires a baseUrl.');
|
|
262
458
|
}
|
|
263
459
|
const url = new URL(
|
|
264
|
-
`${baseUrl.replace(/\/$/, '')}
|
|
460
|
+
`${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`,
|
|
265
461
|
);
|
|
266
462
|
const bypassToken = context.vercelProtectionBypassToken?.trim();
|
|
267
463
|
if (bypassToken) {
|
|
@@ -282,9 +478,16 @@ function resolveRuntimeApiHeaders(
|
|
|
282
478
|
return {
|
|
283
479
|
'content-type': 'application/json',
|
|
284
480
|
authorization: `Bearer ${token}`,
|
|
481
|
+
[PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
|
|
285
482
|
...(context.vercelProtectionBypassToken
|
|
286
483
|
? { 'x-vercel-protection-bypass': context.vercelProtectionBypassToken }
|
|
287
484
|
: {}),
|
|
485
|
+
...(context.runtimeTestFaultHeader?.trim()
|
|
486
|
+
? {
|
|
487
|
+
[PLAY_RUNTIME_TEST_FAULT_HEADER]:
|
|
488
|
+
context.runtimeTestFaultHeader.trim(),
|
|
489
|
+
}
|
|
490
|
+
: {}),
|
|
288
491
|
...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
|
|
289
492
|
};
|
|
290
493
|
}
|
|
@@ -516,20 +719,36 @@ async function isRuntimeSheetSchemaReady(
|
|
|
516
719
|
physicalColumns.length > 0
|
|
517
720
|
? physicalColumns.map(quoteIdentifier).join(', ')
|
|
518
721
|
: '1';
|
|
519
|
-
const query = async (client: RuntimeQueryClient): Promise<
|
|
722
|
+
const query = async (client: RuntimeQueryClient): Promise<boolean> => {
|
|
520
723
|
await client.query(
|
|
521
724
|
`SELECT ${selectList}
|
|
522
725
|
FROM ${sheetTable(session)}
|
|
523
726
|
LIMIT 0`,
|
|
524
727
|
);
|
|
728
|
+
const attemptColumns = await client.query<Record<string, unknown>>(
|
|
729
|
+
`
|
|
730
|
+
SELECT count(*)::int AS present_count
|
|
731
|
+
FROM information_schema.columns
|
|
732
|
+
WHERE table_schema = $1
|
|
733
|
+
AND table_name = $2
|
|
734
|
+
AND column_name = ANY($3::text[])
|
|
735
|
+
`,
|
|
736
|
+
[
|
|
737
|
+
session.postgres.schema,
|
|
738
|
+
session.postgres.sheetTable,
|
|
739
|
+
[...RUNTIME_SHEET_ATTEMPT_COLUMNS],
|
|
740
|
+
],
|
|
741
|
+
);
|
|
742
|
+
return (
|
|
743
|
+
Number(attemptColumns.rows[0]?.present_count ?? 0) ===
|
|
744
|
+
RUNTIME_SHEET_ATTEMPT_COLUMNS.length
|
|
745
|
+
);
|
|
525
746
|
};
|
|
526
747
|
try {
|
|
527
748
|
if (isRuntimeOneShotQueryFactoryRegistered()) {
|
|
528
|
-
await withRuntimeOneShotPostgres(session, query);
|
|
529
|
-
} else {
|
|
530
|
-
await withRuntimePostgres(session, query);
|
|
749
|
+
return await withRuntimeOneShotPostgres(session, query);
|
|
531
750
|
}
|
|
532
|
-
return
|
|
751
|
+
return await withRuntimePostgres(session, query);
|
|
533
752
|
} catch (error) {
|
|
534
753
|
if (isMissingRelationError(error)) {
|
|
535
754
|
return false;
|
|
@@ -1530,6 +1749,10 @@ function workReceiptKeyHex(key: string): string {
|
|
|
1530
1749
|
).join('');
|
|
1531
1750
|
}
|
|
1532
1751
|
|
|
1752
|
+
function newRuntimeWorkReceiptLeaseId(): string {
|
|
1753
|
+
return `receipt-lease:${randomUUID()}`;
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1533
1756
|
function validateRuntimeWorkReceiptKeyScope(
|
|
1534
1757
|
session: RuntimePostgresSession,
|
|
1535
1758
|
input: { key: string },
|
|
@@ -1548,22 +1771,86 @@ function validateRuntimeWorkReceiptKeyScope(
|
|
|
1548
1771
|
}
|
|
1549
1772
|
|
|
1550
1773
|
function mapRuntimeWorkReceiptRow(raw: Record<string, unknown>): WorkReceipt {
|
|
1774
|
+
const leaseExpiresAt = raw.lease_expires_at;
|
|
1551
1775
|
return {
|
|
1552
1776
|
key: String(raw.k ?? ''),
|
|
1553
1777
|
status: receiptStatusFromCode(raw.status),
|
|
1554
1778
|
output: raw.output == null ? null : raw.output,
|
|
1555
1779
|
error: raw.error == null ? null : String(raw.error),
|
|
1780
|
+
failureKind: workReceiptFailureKindFromCode(raw.failure_kind),
|
|
1556
1781
|
runId: raw.run_id == null ? null : String(raw.run_id),
|
|
1782
|
+
leaseId: raw.lease_id == null ? null : String(raw.lease_id),
|
|
1783
|
+
leaseOwnerRunId:
|
|
1784
|
+
raw.lease_owner_run_id == null ? null : String(raw.lease_owner_run_id),
|
|
1785
|
+
leaseOwnerAttempt:
|
|
1786
|
+
raw.lease_owner_attempt == null ||
|
|
1787
|
+
!Number.isFinite(Number(raw.lease_owner_attempt))
|
|
1788
|
+
? null
|
|
1789
|
+
: Number(raw.lease_owner_attempt),
|
|
1790
|
+
leaseExpiresAt:
|
|
1791
|
+
leaseExpiresAt instanceof Date
|
|
1792
|
+
? leaseExpiresAt.toISOString()
|
|
1793
|
+
: leaseExpiresAt == null
|
|
1794
|
+
? null
|
|
1795
|
+
: String(leaseExpiresAt),
|
|
1557
1796
|
updatedAt: raw.updated_at == null ? null : String(raw.updated_at),
|
|
1558
1797
|
};
|
|
1559
1798
|
}
|
|
1560
1799
|
|
|
1800
|
+
function isMissingRuntimeWorkReceiptSelfHealColumnError(
|
|
1801
|
+
error: unknown,
|
|
1802
|
+
): boolean {
|
|
1803
|
+
if (!error || typeof error !== 'object') {
|
|
1804
|
+
return false;
|
|
1805
|
+
}
|
|
1806
|
+
const code = 'code' in error ? String(error.code) : '';
|
|
1807
|
+
const message = 'message' in error ? String(error.message) : '';
|
|
1808
|
+
if (code !== '42703' && !/column .* does not exist/i.test(message)) {
|
|
1809
|
+
return false;
|
|
1810
|
+
}
|
|
1811
|
+
return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.some((column) =>
|
|
1812
|
+
message.includes(column),
|
|
1813
|
+
);
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1561
1816
|
function runtimeWorkReceiptEnsureCacheKey(
|
|
1562
1817
|
session: RuntimePostgresSession,
|
|
1563
1818
|
): string {
|
|
1564
1819
|
return `${postgresUrlCacheKey(session.postgresUrl)}::${session.postgres.schema}::${session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE}`;
|
|
1565
1820
|
}
|
|
1566
1821
|
|
|
1822
|
+
async function missingRuntimeWorkReceiptSelfHealColumns(
|
|
1823
|
+
session: RuntimePostgresSession,
|
|
1824
|
+
client: RuntimeQueryClient,
|
|
1825
|
+
): Promise<string[]> {
|
|
1826
|
+
const result = await client.query(
|
|
1827
|
+
`
|
|
1828
|
+
SELECT column_name
|
|
1829
|
+
FROM information_schema.columns
|
|
1830
|
+
WHERE table_schema = $1
|
|
1831
|
+
AND table_name = $2
|
|
1832
|
+
AND column_name = ANY($3::text[])
|
|
1833
|
+
`,
|
|
1834
|
+
[
|
|
1835
|
+
session.postgres.schema,
|
|
1836
|
+
session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
|
|
1837
|
+
[...RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS],
|
|
1838
|
+
],
|
|
1839
|
+
);
|
|
1840
|
+
const present = new Set(
|
|
1841
|
+
result.rows.map((row) => String(row.column_name ?? '')),
|
|
1842
|
+
);
|
|
1843
|
+
return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.filter(
|
|
1844
|
+
(column) => !present.has(column),
|
|
1845
|
+
);
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
// COMPAT / SELF-HEAL — owner: runtime. Removal milestone: execution-ledger
|
|
1849
|
+
// cutover M1 (docs/play-runtime-execution-ledger-plan.md). The customer-Neon
|
|
1850
|
+
// work-receipt table (lease columns + failure_kind) is the dying surface. The
|
|
1851
|
+
// F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
|
|
1852
|
+
// columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
|
|
1853
|
+
// deleted together with the Neon receipt path at M1.
|
|
1567
1854
|
async function ensureRuntimeWorkReceiptTable(
|
|
1568
1855
|
session: RuntimePostgresSession,
|
|
1569
1856
|
client: RuntimeQueryClient,
|
|
@@ -1582,11 +1869,39 @@ async function ensureRuntimeWorkReceiptTable(
|
|
|
1582
1869
|
status smallint NOT NULL DEFAULT 0,
|
|
1583
1870
|
output jsonb,
|
|
1584
1871
|
error text,
|
|
1872
|
+
failure_kind smallint NOT NULL DEFAULT 0,
|
|
1585
1873
|
run_id text,
|
|
1874
|
+
lease_id text,
|
|
1875
|
+
lease_owner_run_id text,
|
|
1876
|
+
lease_owner_attempt integer,
|
|
1877
|
+
lease_expires_at timestamptz,
|
|
1586
1878
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
1587
1879
|
)
|
|
1588
1880
|
`,
|
|
1589
1881
|
)
|
|
1882
|
+
.then(async () => {
|
|
1883
|
+
const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
|
|
1884
|
+
session,
|
|
1885
|
+
client,
|
|
1886
|
+
);
|
|
1887
|
+
if (missingColumns.length === 0) return;
|
|
1888
|
+
await client.query(`
|
|
1889
|
+
ALTER TABLE ${workReceiptTable(session)}
|
|
1890
|
+
${missingColumns
|
|
1891
|
+
.map((column) => {
|
|
1892
|
+
const type =
|
|
1893
|
+
column === 'lease_expires_at'
|
|
1894
|
+
? 'timestamptz'
|
|
1895
|
+
: column === 'lease_owner_attempt'
|
|
1896
|
+
? 'integer'
|
|
1897
|
+
: column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
|
|
1898
|
+
? 'smallint NOT NULL DEFAULT 0'
|
|
1899
|
+
: 'text';
|
|
1900
|
+
return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
|
|
1901
|
+
})
|
|
1902
|
+
.join(',\n ')}
|
|
1903
|
+
`);
|
|
1904
|
+
})
|
|
1590
1905
|
.then(() => undefined);
|
|
1591
1906
|
runtimeWorkReceiptEnsureCache.set(cacheKey, promise);
|
|
1592
1907
|
try {
|
|
@@ -1612,11 +1927,26 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
1612
1927
|
try {
|
|
1613
1928
|
return await operation(client);
|
|
1614
1929
|
} catch (error) {
|
|
1615
|
-
if (
|
|
1930
|
+
if (
|
|
1931
|
+
isMissingRelationError(error) ||
|
|
1932
|
+
isMissingRuntimeWorkReceiptSelfHealColumnError(error)
|
|
1933
|
+
) {
|
|
1616
1934
|
runtimeWorkReceiptEnsureCache.delete(
|
|
1617
1935
|
runtimeWorkReceiptEnsureCacheKey(session),
|
|
1618
1936
|
);
|
|
1619
|
-
|
|
1937
|
+
try {
|
|
1938
|
+
await ensureRuntimeWorkReceiptTable(session, client);
|
|
1939
|
+
} catch (ensureError) {
|
|
1940
|
+
if (!isPostgresPermissionDeniedError(ensureError)) {
|
|
1941
|
+
throw ensureError;
|
|
1942
|
+
}
|
|
1943
|
+
runtimeWorkReceiptEnsureCache.delete(
|
|
1944
|
+
runtimeWorkReceiptEnsureCacheKey(session),
|
|
1945
|
+
);
|
|
1946
|
+
await repairRuntimeStorageGrants(context, {
|
|
1947
|
+
playName: context.playName?.trim() || session.playName,
|
|
1948
|
+
});
|
|
1949
|
+
}
|
|
1620
1950
|
} else if (isPostgresPermissionDeniedError(error)) {
|
|
1621
1951
|
await repairRuntimeStorageGrants(context, {
|
|
1622
1952
|
playName: context.playName?.trim() || session.playName,
|
|
@@ -1715,6 +2045,10 @@ function isSystemSheetColumn(columnName: string): boolean {
|
|
|
1715
2045
|
case '_stage':
|
|
1716
2046
|
case '_provider':
|
|
1717
2047
|
case '_input_index':
|
|
2048
|
+
case '_attempt_id':
|
|
2049
|
+
case '_attempt_owner_run_id':
|
|
2050
|
+
case '_attempt_expires_at':
|
|
2051
|
+
case '_attempt_seq':
|
|
1718
2052
|
case '_created_at':
|
|
1719
2053
|
case '_updated_at':
|
|
1720
2054
|
case '_version':
|
|
@@ -1732,6 +2066,64 @@ function isSystemSheetColumn(columnName: string): boolean {
|
|
|
1732
2066
|
}
|
|
1733
2067
|
}
|
|
1734
2068
|
|
|
2069
|
+
function normalizeRuntimeSheetAttemptId(
|
|
2070
|
+
value: string | null | undefined,
|
|
2071
|
+
): string {
|
|
2072
|
+
const trimmed = value?.trim();
|
|
2073
|
+
return trimmed || `sheet-attempt:${randomUUID()}`;
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
function normalizeRuntimeSheetAttemptOwnerRunId(input: {
|
|
2077
|
+
attemptOwnerRunId?: string | null;
|
|
2078
|
+
runId: string;
|
|
2079
|
+
}): string {
|
|
2080
|
+
return input.attemptOwnerRunId?.trim() || input.runId;
|
|
2081
|
+
}
|
|
2082
|
+
|
|
2083
|
+
function normalizeRuntimeRunAttempt(value: number | null | undefined): number {
|
|
2084
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) return 0;
|
|
2085
|
+
return Math.max(0, Math.floor(value));
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
function normalizeRuntimeLeaseTtlMs(
|
|
2089
|
+
value: number | null | undefined,
|
|
2090
|
+
fallback: number,
|
|
2091
|
+
): number {
|
|
2092
|
+
if (
|
|
2093
|
+
typeof value !== 'number' ||
|
|
2094
|
+
!Number.isFinite(value) ||
|
|
2095
|
+
!Number.isInteger(value) ||
|
|
2096
|
+
value <= 0
|
|
2097
|
+
) {
|
|
2098
|
+
return fallback;
|
|
2099
|
+
}
|
|
2100
|
+
return value;
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
async function mintRuntimeSheetAttemptExpiresAt(
|
|
2104
|
+
client: RuntimeQueryClient,
|
|
2105
|
+
leaseTtlMs?: number | null,
|
|
2106
|
+
): Promise<string> {
|
|
2107
|
+
const ttlMs = normalizeRuntimeLeaseTtlMs(
|
|
2108
|
+
leaseTtlMs,
|
|
2109
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
2110
|
+
);
|
|
2111
|
+
const { rows } = await client.query<{ attempt_expires_at: Date | string }>(
|
|
2112
|
+
`
|
|
2113
|
+
SELECT now() + ($1::double precision * interval '1 millisecond') AS attempt_expires_at
|
|
2114
|
+
`,
|
|
2115
|
+
[ttlMs],
|
|
2116
|
+
);
|
|
2117
|
+
const value = rows[0]?.attempt_expires_at;
|
|
2118
|
+
if (value instanceof Date) {
|
|
2119
|
+
return value.toISOString();
|
|
2120
|
+
}
|
|
2121
|
+
if (typeof value === 'string' && value.trim()) {
|
|
2122
|
+
return new Date(value).toISOString();
|
|
2123
|
+
}
|
|
2124
|
+
throw new Error('Runtime sheet attempt expiry mint did not return a value.');
|
|
2125
|
+
}
|
|
2126
|
+
|
|
1735
2127
|
function parseRuntimeCellMeta(value: unknown): Record<string, unknown> {
|
|
1736
2128
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
1737
2129
|
return value as Record<string, unknown>;
|
|
@@ -2228,6 +2620,10 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2228
2620
|
runId: string;
|
|
2229
2621
|
normalizedPlayName: string;
|
|
2230
2622
|
normalizedTableNamespace: string;
|
|
2623
|
+
attemptId: string;
|
|
2624
|
+
attemptOwnerRunId: string;
|
|
2625
|
+
attemptExpiresAt: string;
|
|
2626
|
+
attemptSeq: number;
|
|
2231
2627
|
physicalInsertColumnsSql: string;
|
|
2232
2628
|
physicalInsertValuesSql: string;
|
|
2233
2629
|
physicalRefreshSetSql: string;
|
|
@@ -2235,9 +2631,12 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2235
2631
|
outputPhysicalColumns: PhysicalSheetColumnProjection[];
|
|
2236
2632
|
force?: boolean;
|
|
2237
2633
|
},
|
|
2238
|
-
): Promise<{
|
|
2634
|
+
): Promise<{
|
|
2635
|
+
inserted: number;
|
|
2636
|
+
rowDispositions: RuntimeSheetPreparedRowDisposition[];
|
|
2637
|
+
}> {
|
|
2239
2638
|
let inserted = 0;
|
|
2240
|
-
const
|
|
2639
|
+
const rowDispositions: RuntimeSheetPreparedRowDisposition[] = [];
|
|
2241
2640
|
for (const chunk of input.chunks) {
|
|
2242
2641
|
const chunkKeys = chunk.map((entry) => entry.key);
|
|
2243
2642
|
const chunkPayloads = chunk.map((entry) =>
|
|
@@ -2252,6 +2651,41 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2252
2651
|
'target',
|
|
2253
2652
|
input.outputPhysicalColumns,
|
|
2254
2653
|
);
|
|
2654
|
+
const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
2655
|
+
'target',
|
|
2656
|
+
'$7::text',
|
|
2657
|
+
'$8::text',
|
|
2658
|
+
'$9::timestamptz',
|
|
2659
|
+
'$10::integer',
|
|
2660
|
+
);
|
|
2661
|
+
const existingAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
2662
|
+
'existing',
|
|
2663
|
+
'$7::text',
|
|
2664
|
+
'$8::text',
|
|
2665
|
+
'$9::timestamptz',
|
|
2666
|
+
'$10::integer',
|
|
2667
|
+
);
|
|
2668
|
+
const targetNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
|
|
2669
|
+
'target',
|
|
2670
|
+
'$9::timestamptz',
|
|
2671
|
+
'$10::integer',
|
|
2672
|
+
'$7::text',
|
|
2673
|
+
'$8::text',
|
|
2674
|
+
);
|
|
2675
|
+
const existingNewerTerminalRowSql = newerTerminalRuntimeSheetRowSql(
|
|
2676
|
+
'existing',
|
|
2677
|
+
'$9::timestamptz',
|
|
2678
|
+
'$10::integer',
|
|
2679
|
+
'$7::text',
|
|
2680
|
+
'$8::text',
|
|
2681
|
+
);
|
|
2682
|
+
const existingWritableEnrichedAttemptSql =
|
|
2683
|
+
writableEnrichedRuntimeSheetAttemptSql(
|
|
2684
|
+
'existing',
|
|
2685
|
+
'$8::text',
|
|
2686
|
+
'$9::timestamptz',
|
|
2687
|
+
'$10',
|
|
2688
|
+
);
|
|
2255
2689
|
const { rows } = await client.query(
|
|
2256
2690
|
`
|
|
2257
2691
|
WITH input_rows AS (
|
|
@@ -2273,19 +2707,38 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2273
2707
|
WHERE target._key = input_rows._key
|
|
2274
2708
|
AND target._status <> 'stale'
|
|
2275
2709
|
AND (
|
|
2276
|
-
$
|
|
2710
|
+
NOT (${targetNewerTerminalRowSql})
|
|
2711
|
+
OR (
|
|
2712
|
+
$11::boolean
|
|
2713
|
+
AND target._status = 'enriched'
|
|
2714
|
+
AND COALESCE(target._attempt_seq, 0) <= $10::integer
|
|
2715
|
+
)
|
|
2716
|
+
)
|
|
2717
|
+
AND (
|
|
2718
|
+
${targetAttemptFenceSql}
|
|
2719
|
+
OR (
|
|
2720
|
+
$11::boolean
|
|
2721
|
+
AND target._status = 'enriched'
|
|
2722
|
+
)
|
|
2723
|
+
)
|
|
2724
|
+
AND (
|
|
2725
|
+
$11::boolean
|
|
2277
2726
|
OR target._input_index IS DISTINCT FROM input_rows._input_index
|
|
2278
2727
|
)
|
|
2279
2728
|
RETURNING target._key
|
|
2280
2729
|
),
|
|
2281
2730
|
inserted_rows AS (
|
|
2282
|
-
INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${input.physicalInsertColumnsSql})
|
|
2283
|
-
SELECT _key, 'pending', $4::text, _input_index${input.physicalInsertValuesSql}
|
|
2731
|
+
INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq${input.physicalInsertColumnsSql})
|
|
2732
|
+
SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer${input.physicalInsertValuesSql}
|
|
2284
2733
|
FROM input_rows
|
|
2285
2734
|
ON CONFLICT (_key) DO UPDATE SET
|
|
2286
2735
|
_status = 'pending',
|
|
2287
2736
|
_run_id = EXCLUDED._run_id,
|
|
2288
2737
|
_input_index = EXCLUDED._input_index,
|
|
2738
|
+
_attempt_id = EXCLUDED._attempt_id,
|
|
2739
|
+
_attempt_owner_run_id = EXCLUDED._attempt_owner_run_id,
|
|
2740
|
+
_attempt_expires_at = EXCLUDED._attempt_expires_at,
|
|
2741
|
+
_attempt_seq = EXCLUDED._attempt_seq,
|
|
2289
2742
|
_updated_at = now(),
|
|
2290
2743
|
_version = ${nextRuntimeSheetVersionExpression(session)}${input.physicalUpsertSetSql}
|
|
2291
2744
|
WHERE ${sheetTable(session)}._status = 'stale'
|
|
@@ -2296,26 +2749,42 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2296
2749
|
SET _status = 'pending',
|
|
2297
2750
|
_run_id = $4::text,
|
|
2298
2751
|
_input_index = input_rows._input_index,
|
|
2752
|
+
_attempt_id = $7::text,
|
|
2753
|
+
_attempt_owner_run_id = $8::text,
|
|
2754
|
+
_attempt_expires_at = $9::timestamptz,
|
|
2755
|
+
_attempt_seq = $10::integer,
|
|
2299
2756
|
_updated_at = now(),
|
|
2300
2757
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2301
2758
|
FROM input_rows
|
|
2302
2759
|
WHERE target._key = input_rows._key
|
|
2303
2760
|
AND target._status = 'enriched'
|
|
2304
2761
|
AND (${targetMissingOutputSql})
|
|
2762
|
+
AND COALESCE(target._attempt_seq, 0) <= $10::integer
|
|
2305
2763
|
RETURNING target._key
|
|
2306
2764
|
),
|
|
2307
2765
|
claimed_existing_rows AS (
|
|
2308
2766
|
UPDATE ${sheetTable(session)} AS target
|
|
2309
2767
|
SET _run_id = $4::text,
|
|
2310
2768
|
_input_index = input_rows._input_index,
|
|
2769
|
+
_attempt_id = $7::text,
|
|
2770
|
+
_attempt_owner_run_id = $8::text,
|
|
2771
|
+
_attempt_expires_at = $9::timestamptz,
|
|
2772
|
+
_attempt_seq = $10::integer,
|
|
2311
2773
|
_updated_at = now(),
|
|
2312
2774
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2313
2775
|
FROM input_rows
|
|
2314
2776
|
WHERE target._key = input_rows._key
|
|
2315
2777
|
AND target._status IN ('pending', 'running', 'failed')
|
|
2778
|
+
AND NOT (${targetNewerTerminalRowSql})
|
|
2779
|
+
AND (
|
|
2780
|
+
target._status = 'failed'
|
|
2781
|
+
OR ${targetAttemptFenceSql}
|
|
2782
|
+
)
|
|
2316
2783
|
AND (
|
|
2317
2784
|
target._run_id IS DISTINCT FROM $4::text
|
|
2318
2785
|
OR target._input_index IS DISTINCT FROM input_rows._input_index
|
|
2786
|
+
OR target._attempt_id IS DISTINCT FROM $7::text
|
|
2787
|
+
OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $10::integer
|
|
2319
2788
|
)
|
|
2320
2789
|
RETURNING target._key
|
|
2321
2790
|
),
|
|
@@ -2329,15 +2798,61 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2329
2798
|
SELECT _key
|
|
2330
2799
|
FROM claimed_existing_rows
|
|
2331
2800
|
UNION
|
|
2801
|
+
SELECT _key
|
|
2802
|
+
FROM existing_rows
|
|
2803
|
+
WHERE $11::boolean
|
|
2804
|
+
UNION
|
|
2332
2805
|
SELECT existing._key
|
|
2333
2806
|
FROM ${sheetTable(session)} AS existing
|
|
2334
2807
|
JOIN input_rows ON input_rows._key = existing._key
|
|
2335
|
-
WHERE
|
|
2808
|
+
WHERE (
|
|
2809
|
+
existing._status IN ('pending', 'running', 'failed')
|
|
2810
|
+
AND NOT (${existingNewerTerminalRowSql})
|
|
2811
|
+
AND (
|
|
2812
|
+
existing._status = 'failed'
|
|
2813
|
+
OR ${existingAttemptFenceSql}
|
|
2814
|
+
)
|
|
2815
|
+
)
|
|
2336
2816
|
OR (
|
|
2337
2817
|
existing._status = 'enriched'
|
|
2338
2818
|
AND (${existingMissingOutputSql})
|
|
2819
|
+
AND (
|
|
2820
|
+
${existingAttemptFenceSql}
|
|
2821
|
+
OR ${existingWritableEnrichedAttemptSql}
|
|
2822
|
+
)
|
|
2339
2823
|
)
|
|
2340
2824
|
),
|
|
2825
|
+
completed_rows AS (
|
|
2826
|
+
SELECT existing._key
|
|
2827
|
+
FROM ${sheetTable(session)} AS existing
|
|
2828
|
+
JOIN input_rows ON input_rows._key = existing._key
|
|
2829
|
+
WHERE existing._status = 'enriched'
|
|
2830
|
+
AND NOT ($11::boolean)
|
|
2831
|
+
AND NOT (${existingMissingOutputSql})
|
|
2832
|
+
AND NOT EXISTS (
|
|
2833
|
+
SELECT 1 FROM pending_rows WHERE pending_rows._key = existing._key
|
|
2834
|
+
)
|
|
2835
|
+
),
|
|
2836
|
+
blocked_rows AS (
|
|
2837
|
+
SELECT input_rows._key
|
|
2838
|
+
FROM input_rows
|
|
2839
|
+
WHERE NOT EXISTS (
|
|
2840
|
+
SELECT 1 FROM pending_rows WHERE pending_rows._key = input_rows._key
|
|
2841
|
+
)
|
|
2842
|
+
AND NOT EXISTS (
|
|
2843
|
+
SELECT 1 FROM completed_rows WHERE completed_rows._key = input_rows._key
|
|
2844
|
+
)
|
|
2845
|
+
),
|
|
2846
|
+
row_dispositions AS (
|
|
2847
|
+
SELECT _key, 'pending'::text AS disposition
|
|
2848
|
+
FROM pending_rows
|
|
2849
|
+
UNION ALL
|
|
2850
|
+
SELECT _key, 'completed'::text AS disposition
|
|
2851
|
+
FROM completed_rows
|
|
2852
|
+
UNION ALL
|
|
2853
|
+
SELECT _key, 'blocked'::text AS disposition
|
|
2854
|
+
FROM blocked_rows
|
|
2855
|
+
),
|
|
2341
2856
|
inserted_count_cte AS (
|
|
2342
2857
|
SELECT count(*)::bigint AS c FROM inserted_rows
|
|
2343
2858
|
),
|
|
@@ -2377,7 +2892,13 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2377
2892
|
)
|
|
2378
2893
|
SELECT
|
|
2379
2894
|
(SELECT c::int FROM inserted_count_cte) AS inserted_count,
|
|
2380
|
-
coalesce(
|
|
2895
|
+
coalesce(
|
|
2896
|
+
(
|
|
2897
|
+
SELECT jsonb_agg(jsonb_build_object('key', _key, 'disposition', disposition))
|
|
2898
|
+
FROM row_dispositions
|
|
2899
|
+
),
|
|
2900
|
+
'[]'::jsonb
|
|
2901
|
+
) AS row_dispositions,
|
|
2381
2902
|
(SELECT count(*)::int FROM existing_rows) AS reordered_count,
|
|
2382
2903
|
(SELECT c::int FROM missing_output_count_cte) AS missing_output_count
|
|
2383
2904
|
`,
|
|
@@ -2388,15 +2909,30 @@ async function prepareRuntimeSheetDatasetRows(
|
|
|
2388
2909
|
input.runId,
|
|
2389
2910
|
input.normalizedPlayName,
|
|
2390
2911
|
input.normalizedTableNamespace,
|
|
2912
|
+
input.attemptId,
|
|
2913
|
+
input.attemptOwnerRunId,
|
|
2914
|
+
input.attemptExpiresAt,
|
|
2915
|
+
input.attemptSeq,
|
|
2391
2916
|
input.force === true,
|
|
2392
2917
|
],
|
|
2393
2918
|
);
|
|
2394
2919
|
inserted += Number(rows[0]?.inserted_count ?? 0);
|
|
2395
|
-
if (Array.isArray(rows[0]?.
|
|
2396
|
-
|
|
2920
|
+
if (Array.isArray(rows[0]?.row_dispositions)) {
|
|
2921
|
+
for (const row of rows[0]
|
|
2922
|
+
?.row_dispositions as RuntimeSheetPreparedRowDisposition[]) {
|
|
2923
|
+
if (
|
|
2924
|
+
row &&
|
|
2925
|
+
typeof row.key === 'string' &&
|
|
2926
|
+
(row.disposition === 'pending' ||
|
|
2927
|
+
row.disposition === 'completed' ||
|
|
2928
|
+
row.disposition === 'blocked')
|
|
2929
|
+
) {
|
|
2930
|
+
rowDispositions.push(row);
|
|
2931
|
+
}
|
|
2932
|
+
}
|
|
2397
2933
|
}
|
|
2398
2934
|
}
|
|
2399
|
-
return { inserted,
|
|
2935
|
+
return { inserted, rowDispositions };
|
|
2400
2936
|
}
|
|
2401
2937
|
|
|
2402
2938
|
async function buildRuntimeSheetDatasetStartResult(
|
|
@@ -2410,8 +2946,12 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2410
2946
|
normalizedPlayName: string;
|
|
2411
2947
|
normalizedTableNamespace: string;
|
|
2412
2948
|
runId: string;
|
|
2949
|
+
attemptId: string;
|
|
2950
|
+
attemptOwnerRunId: string;
|
|
2951
|
+
attemptExpiresAt: string;
|
|
2952
|
+
attemptSeq: number;
|
|
2413
2953
|
inserted: number;
|
|
2414
|
-
|
|
2954
|
+
rowDispositions: RuntimeSheetPreparedRowDisposition[];
|
|
2415
2955
|
force?: boolean;
|
|
2416
2956
|
timings?: RuntimeSheetTiming[];
|
|
2417
2957
|
},
|
|
@@ -2421,19 +2961,20 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2421
2961
|
? [column.field]
|
|
2422
2962
|
: [],
|
|
2423
2963
|
);
|
|
2424
|
-
const
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
row[field] = null;
|
|
2433
|
-
}
|
|
2964
|
+
const buildFreshRow = (entry: RuntimeDatasetRowEntry) => {
|
|
2965
|
+
const row: Record<string, unknown> = {
|
|
2966
|
+
...sanitizePostgresJsonValue(entry.row),
|
|
2967
|
+
__deeplineRowKey: entry.key,
|
|
2968
|
+
};
|
|
2969
|
+
for (const field of datasetFields) {
|
|
2970
|
+
if (!Object.prototype.hasOwnProperty.call(row, field)) {
|
|
2971
|
+
row[field] = null;
|
|
2434
2972
|
}
|
|
2435
|
-
|
|
2436
|
-
|
|
2973
|
+
}
|
|
2974
|
+
return row;
|
|
2975
|
+
};
|
|
2976
|
+
const buildFreshPendingRows = (entries = input.rowEntries) => {
|
|
2977
|
+
return entries.map(buildFreshRow);
|
|
2437
2978
|
};
|
|
2438
2979
|
|
|
2439
2980
|
if (input.inserted === input.rowEntries.length) {
|
|
@@ -2442,18 +2983,31 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2442
2983
|
skipped: input.sourceRowsLength - input.rowEntries.length,
|
|
2443
2984
|
pendingRows: buildFreshPendingRows(),
|
|
2444
2985
|
completedRows: [],
|
|
2986
|
+
blockedRows: [],
|
|
2445
2987
|
tableNamespace: input.tableNamespace,
|
|
2446
2988
|
};
|
|
2447
2989
|
}
|
|
2448
2990
|
|
|
2449
|
-
const
|
|
2991
|
+
const dispositionsByKey = new Map(
|
|
2992
|
+
input.rowDispositions.map((row) => [row.key, row.disposition]),
|
|
2993
|
+
);
|
|
2994
|
+
const pendingKeys = new Set(
|
|
2995
|
+
input.rowDispositions.flatMap((row) =>
|
|
2996
|
+
row.disposition === 'pending' ? [row.key] : [],
|
|
2997
|
+
),
|
|
2998
|
+
);
|
|
2450
2999
|
const startedAt = Date.now();
|
|
2451
3000
|
await markRuntimeRowsPendingForRecompute(client, session, {
|
|
2452
3001
|
keys: [...pendingKeys],
|
|
2453
3002
|
runId: input.runId,
|
|
3003
|
+
attemptId: input.attemptId,
|
|
3004
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
3005
|
+
attemptExpiresAt: input.attemptExpiresAt,
|
|
3006
|
+
attemptSeq: input.attemptSeq,
|
|
2454
3007
|
normalizedPlayName: input.normalizedPlayName,
|
|
2455
3008
|
normalizedTableNamespace: input.normalizedTableNamespace,
|
|
2456
3009
|
outputFields: input.force === true ? datasetFields : [],
|
|
3010
|
+
force: input.force === true,
|
|
2457
3011
|
});
|
|
2458
3012
|
if (pendingKeys.size > 0) {
|
|
2459
3013
|
input.timings?.push({
|
|
@@ -2463,11 +3017,22 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2463
3017
|
});
|
|
2464
3018
|
}
|
|
2465
3019
|
if (input.force === true) {
|
|
3020
|
+
const pendingRows: Record<string, unknown>[] = [];
|
|
3021
|
+
const blockedRows: Record<string, unknown>[] = [];
|
|
3022
|
+
for (const entry of input.rowEntries) {
|
|
3023
|
+
const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
|
|
3024
|
+
if (disposition === 'pending') {
|
|
3025
|
+
pendingRows.push(buildFreshRow(entry));
|
|
3026
|
+
} else if (disposition === 'blocked') {
|
|
3027
|
+
blockedRows.push(buildFreshRow(entry));
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
2466
3030
|
return {
|
|
2467
3031
|
inserted: input.inserted,
|
|
2468
3032
|
skipped: input.sourceRowsLength - input.rowEntries.length,
|
|
2469
|
-
pendingRows
|
|
3033
|
+
pendingRows,
|
|
2470
3034
|
completedRows: [],
|
|
3035
|
+
blockedRows,
|
|
2471
3036
|
tableNamespace: input.tableNamespace,
|
|
2472
3037
|
};
|
|
2473
3038
|
}
|
|
@@ -2498,14 +3063,25 @@ async function buildRuntimeSheetDatasetStartResult(
|
|
|
2498
3063
|
};
|
|
2499
3064
|
};
|
|
2500
3065
|
const pendingRows: Record<string, unknown>[] = [];
|
|
3066
|
+
const completedRows: Record<string, unknown>[] = [];
|
|
3067
|
+
const blockedRows: Record<string, unknown>[] = [];
|
|
2501
3068
|
for (const entry of input.rowEntries) {
|
|
2502
|
-
|
|
3069
|
+
const merged = buildMergedRow(entry);
|
|
3070
|
+
const disposition = dispositionsByKey.get(entry.key) ?? 'blocked';
|
|
3071
|
+
if (disposition === 'pending') {
|
|
3072
|
+
pendingRows.push(merged);
|
|
3073
|
+
} else if (disposition === 'completed') {
|
|
3074
|
+
completedRows.push(merged);
|
|
3075
|
+
} else {
|
|
3076
|
+
blockedRows.push(merged);
|
|
3077
|
+
}
|
|
2503
3078
|
}
|
|
2504
3079
|
return {
|
|
2505
3080
|
inserted: input.inserted,
|
|
2506
3081
|
skipped: input.sourceRowsLength - input.rowEntries.length,
|
|
2507
3082
|
pendingRows,
|
|
2508
|
-
completedRows
|
|
3083
|
+
completedRows,
|
|
3084
|
+
blockedRows,
|
|
2509
3085
|
tableNamespace: input.tableNamespace,
|
|
2510
3086
|
};
|
|
2511
3087
|
}
|
|
@@ -2516,12 +3092,31 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2516
3092
|
input: {
|
|
2517
3093
|
keys: string[];
|
|
2518
3094
|
runId: string;
|
|
3095
|
+
attemptId: string;
|
|
3096
|
+
attemptOwnerRunId: string;
|
|
3097
|
+
attemptExpiresAt: string;
|
|
3098
|
+
attemptSeq: number;
|
|
2519
3099
|
normalizedPlayName: string;
|
|
2520
3100
|
normalizedTableNamespace: string;
|
|
2521
3101
|
outputFields?: string[];
|
|
3102
|
+
force?: boolean;
|
|
2522
3103
|
},
|
|
2523
3104
|
): Promise<void> {
|
|
2524
3105
|
if (input.keys.length === 0) return;
|
|
3106
|
+
const targetAttemptFenceSql = activeRuntimeSheetAttemptFenceSql(
|
|
3107
|
+
'target',
|
|
3108
|
+
'$5::text',
|
|
3109
|
+
'$6::text',
|
|
3110
|
+
'$7::timestamptz',
|
|
3111
|
+
'$8::integer',
|
|
3112
|
+
);
|
|
3113
|
+
const targetWritableEnrichedAttemptSql =
|
|
3114
|
+
writableEnrichedRuntimeSheetAttemptSql(
|
|
3115
|
+
'target',
|
|
3116
|
+
'$6::text',
|
|
3117
|
+
'$7::timestamptz',
|
|
3118
|
+
'$8',
|
|
3119
|
+
);
|
|
2525
3120
|
const outputFields = [...new Set(input.outputFields ?? [])];
|
|
2526
3121
|
await client.query(
|
|
2527
3122
|
`WITH target_rows AS (
|
|
@@ -2534,13 +3129,31 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2534
3129
|
UPDATE ${sheetTable(session)} AS target
|
|
2535
3130
|
SET _status = 'pending',
|
|
2536
3131
|
_run_id = $2::text,
|
|
3132
|
+
_attempt_id = $5::text,
|
|
3133
|
+
_attempt_owner_run_id = $6::text,
|
|
3134
|
+
_attempt_expires_at = $7::timestamptz,
|
|
3135
|
+
_attempt_seq = $8::integer,
|
|
2537
3136
|
_updated_at = now(),
|
|
2538
3137
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2539
3138
|
FROM target_rows
|
|
2540
3139
|
WHERE target._key = target_rows._key
|
|
3140
|
+
AND (
|
|
3141
|
+
${targetAttemptFenceSql}
|
|
3142
|
+
OR (
|
|
3143
|
+
$10::boolean
|
|
3144
|
+
AND target._status = 'enriched'
|
|
3145
|
+
AND COALESCE(target._attempt_seq, 0) <= $8::integer
|
|
3146
|
+
)
|
|
3147
|
+
OR (
|
|
3148
|
+
target._status = 'enriched'
|
|
3149
|
+
AND ${targetWritableEnrichedAttemptSql}
|
|
3150
|
+
)
|
|
3151
|
+
)
|
|
2541
3152
|
AND (
|
|
2542
3153
|
target._status <> 'pending'
|
|
2543
3154
|
OR target._run_id IS DISTINCT FROM $2::text
|
|
3155
|
+
OR target._attempt_id IS DISTINCT FROM $5::text
|
|
3156
|
+
OR COALESCE(target._attempt_seq, 0) IS DISTINCT FROM $8::integer
|
|
2544
3157
|
)
|
|
2545
3158
|
RETURNING target_rows._status AS previous_status,
|
|
2546
3159
|
target_rows._cell_meta AS previous_cell_meta
|
|
@@ -2561,7 +3174,7 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2561
3174
|
WHERE previous_cell_meta -> field_values.field ->> 'status' = 'failed'
|
|
2562
3175
|
)::int AS failed_to_pending
|
|
2563
3176
|
FROM updated
|
|
2564
|
-
JOIN unnest($
|
|
3177
|
+
JOIN unnest($9::text[]) AS field_values(field) ON true
|
|
2565
3178
|
GROUP BY field_values.field
|
|
2566
3179
|
),
|
|
2567
3180
|
summary_delta AS (
|
|
@@ -2619,7 +3232,12 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2619
3232
|
input.runId,
|
|
2620
3233
|
input.normalizedPlayName,
|
|
2621
3234
|
input.normalizedTableNamespace,
|
|
3235
|
+
input.attemptId,
|
|
3236
|
+
input.attemptOwnerRunId,
|
|
3237
|
+
input.attemptExpiresAt,
|
|
3238
|
+
input.attemptSeq,
|
|
2622
3239
|
outputFields,
|
|
3240
|
+
input.force === true,
|
|
2623
3241
|
],
|
|
2624
3242
|
);
|
|
2625
3243
|
}
|
|
@@ -2657,6 +3275,41 @@ async function getRuntimeWorkReceiptSession(
|
|
|
2657
3275
|
return session;
|
|
2658
3276
|
}
|
|
2659
3277
|
|
|
3278
|
+
/**
|
|
3279
|
+
* Work receipts for a play share a single table, so run-scoped operations (the
|
|
3280
|
+
* run-fatal lease release) need a session without a specific key to scope by.
|
|
3281
|
+
* This mirrors `getRuntimeWorkReceiptSession` minus the per-key scope check.
|
|
3282
|
+
*/
|
|
3283
|
+
async function getRuntimeWorkReceiptSessionForRun(
|
|
3284
|
+
context: RuntimeApiContext,
|
|
3285
|
+
input: {
|
|
3286
|
+
playName: string;
|
|
3287
|
+
},
|
|
3288
|
+
): Promise<RuntimePostgresSession> {
|
|
3289
|
+
const playName = context.playName?.trim() || input.playName;
|
|
3290
|
+
if (!playName) {
|
|
3291
|
+
throw new Error('Runtime work receipts require a playName.');
|
|
3292
|
+
}
|
|
3293
|
+
const runtimeContext = {
|
|
3294
|
+
...context,
|
|
3295
|
+
playName,
|
|
3296
|
+
};
|
|
3297
|
+
const preloaded = findPreloadedRuntimeDbSession(runtimeContext, {
|
|
3298
|
+
tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
|
|
3299
|
+
logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
|
|
3300
|
+
operations: ['rows.read', 'rows.upsert'],
|
|
3301
|
+
});
|
|
3302
|
+
return requireRuntimePostgresSession(
|
|
3303
|
+
preloaded
|
|
3304
|
+
? await unwrapRuntimeDbSession(runtimeContext, preloaded)
|
|
3305
|
+
: await getRuntimeDbSession(runtimeContext, {
|
|
3306
|
+
tableNamespace: RUNTIME_WORK_RECEIPT_TABLE_NAMESPACE,
|
|
3307
|
+
logicalTable: RUNTIME_WORK_RECEIPT_LOGICAL_TABLE,
|
|
3308
|
+
operations: ['rows.read', 'rows.upsert'],
|
|
3309
|
+
}),
|
|
3310
|
+
);
|
|
3311
|
+
}
|
|
3312
|
+
|
|
2660
3313
|
async function getRuntimeWorkReceiptSessionForKeys(
|
|
2661
3314
|
context: RuntimeApiContext,
|
|
2662
3315
|
input: {
|
|
@@ -2684,7 +3337,17 @@ async function readRuntimeWorkReceipt(
|
|
|
2684
3337
|
key: string,
|
|
2685
3338
|
): Promise<WorkReceipt | null> {
|
|
2686
3339
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2687
|
-
`SELECT convert_from(k, 'UTF8') AS k,
|
|
3340
|
+
`SELECT convert_from(k, 'UTF8') AS k,
|
|
3341
|
+
status,
|
|
3342
|
+
output,
|
|
3343
|
+
error,
|
|
3344
|
+
failure_kind,
|
|
3345
|
+
run_id,
|
|
3346
|
+
lease_id,
|
|
3347
|
+
lease_owner_run_id,
|
|
3348
|
+
lease_owner_attempt,
|
|
3349
|
+
lease_expires_at,
|
|
3350
|
+
updated_at
|
|
2688
3351
|
FROM ${workReceiptTable(session)}
|
|
2689
3352
|
WHERE k = decode($1, 'hex')`,
|
|
2690
3353
|
[workReceiptKeyHex(key)],
|
|
@@ -2714,7 +3377,7 @@ export async function getRuntimeWorkReceipts(
|
|
|
2714
3377
|
playName: string;
|
|
2715
3378
|
keys: string[];
|
|
2716
3379
|
},
|
|
2717
|
-
): Promise<WorkReceipt
|
|
3380
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
2718
3381
|
const keys = [
|
|
2719
3382
|
...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
|
|
2720
3383
|
];
|
|
@@ -2737,16 +3400,23 @@ export async function getRuntimeWorkReceipts(
|
|
|
2737
3400
|
receipts.status,
|
|
2738
3401
|
receipts.output,
|
|
2739
3402
|
receipts.error,
|
|
3403
|
+
receipts.failure_kind,
|
|
2740
3404
|
receipts.run_id,
|
|
3405
|
+
receipts.lease_id,
|
|
3406
|
+
receipts.lease_owner_run_id,
|
|
3407
|
+
receipts.lease_owner_attempt,
|
|
3408
|
+
receipts.lease_expires_at,
|
|
2741
3409
|
receipts.updated_at
|
|
2742
3410
|
FROM input_keys
|
|
2743
|
-
JOIN ${workReceiptTable(session)} AS receipts
|
|
3411
|
+
LEFT JOIN ${workReceiptTable(session)} AS receipts
|
|
2744
3412
|
ON receipts.k = decode(input_keys.key_hex, 'hex')
|
|
2745
3413
|
ORDER BY input_keys.ord
|
|
2746
3414
|
`,
|
|
2747
3415
|
[keys.map(workReceiptKeyHex)],
|
|
2748
3416
|
);
|
|
2749
|
-
return rows.map(
|
|
3417
|
+
return rows.map((row) =>
|
|
3418
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
3419
|
+
);
|
|
2750
3420
|
},
|
|
2751
3421
|
);
|
|
2752
3422
|
}
|
|
@@ -2757,8 +3427,12 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2757
3427
|
playName: string;
|
|
2758
3428
|
runId: string;
|
|
2759
3429
|
key: string;
|
|
3430
|
+
runAttempt?: number | null;
|
|
3431
|
+
leaseAware?: boolean;
|
|
2760
3432
|
reclaimRunning?: boolean;
|
|
2761
3433
|
forceRefresh?: boolean;
|
|
3434
|
+
forceFailedRefresh?: boolean;
|
|
3435
|
+
leaseTtlMs?: number | null;
|
|
2762
3436
|
},
|
|
2763
3437
|
): Promise<WorkReceiptClaim> {
|
|
2764
3438
|
const session = await getRuntimeWorkReceiptSession(context, {
|
|
@@ -2769,36 +3443,89 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2769
3443
|
context,
|
|
2770
3444
|
session,
|
|
2771
3445
|
async (client) => {
|
|
2772
|
-
const
|
|
2773
|
-
?
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
: input.
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
RECEIPT_STATUS_RUNNING,
|
|
2785
|
-
]
|
|
2786
|
-
: [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
|
|
3446
|
+
const leaseId =
|
|
3447
|
+
input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null;
|
|
3448
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3449
|
+
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
3450
|
+
input.leaseTtlMs,
|
|
3451
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
3452
|
+
);
|
|
3453
|
+
const claimableStatuses = workReceiptClaimableStatusCodes({
|
|
3454
|
+
forceRefresh: input.forceRefresh,
|
|
3455
|
+
forceFailedRefresh: input.forceFailedRefresh,
|
|
3456
|
+
reclaimRunning: input.reclaimRunning,
|
|
3457
|
+
});
|
|
2787
3458
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2788
3459
|
`
|
|
2789
3460
|
WITH claimed AS (
|
|
2790
|
-
INSERT INTO ${workReceiptTable(session)} (
|
|
2791
|
-
|
|
3461
|
+
INSERT INTO ${workReceiptTable(session)} (
|
|
3462
|
+
k,
|
|
3463
|
+
status,
|
|
3464
|
+
run_id,
|
|
3465
|
+
lease_id,
|
|
3466
|
+
lease_owner_run_id,
|
|
3467
|
+
lease_owner_attempt,
|
|
3468
|
+
lease_expires_at,
|
|
3469
|
+
updated_at
|
|
3470
|
+
)
|
|
3471
|
+
VALUES (
|
|
3472
|
+
decode($1, 'hex'),
|
|
3473
|
+
$2::smallint,
|
|
3474
|
+
$3,
|
|
3475
|
+
$6,
|
|
3476
|
+
CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
|
|
3477
|
+
CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
|
|
3478
|
+
CASE
|
|
3479
|
+
WHEN $6::text IS NULL THEN NULL
|
|
3480
|
+
ELSE now() + ($7::double precision * interval '1 millisecond')
|
|
3481
|
+
END,
|
|
3482
|
+
now()
|
|
3483
|
+
)
|
|
2792
3484
|
ON CONFLICT (k) DO UPDATE
|
|
2793
3485
|
SET status = $4::smallint,
|
|
2794
3486
|
output = NULL,
|
|
3487
|
+
failure_kind = 0,
|
|
2795
3488
|
run_id = $3,
|
|
3489
|
+
lease_id = $6,
|
|
3490
|
+
lease_owner_run_id = CASE WHEN $6::text IS NULL THEN NULL ELSE $3 END,
|
|
3491
|
+
lease_owner_attempt = CASE WHEN $6::text IS NULL THEN NULL ELSE $9::integer END,
|
|
3492
|
+
lease_expires_at = CASE
|
|
3493
|
+
WHEN $6::text IS NULL THEN NULL
|
|
3494
|
+
ELSE now() + ($7::double precision * interval '1 millisecond')
|
|
3495
|
+
END,
|
|
2796
3496
|
error = NULL,
|
|
2797
3497
|
updated_at = now()
|
|
2798
|
-
WHERE ${
|
|
2799
|
-
|
|
3498
|
+
WHERE ${workReceiptClaimConflictPredicateSql({
|
|
3499
|
+
receiptTable: workReceiptTable(session),
|
|
3500
|
+
claimableStatusesSql: '$5',
|
|
3501
|
+
claimantRunIdSql: '$3',
|
|
3502
|
+
claimantRunAttemptSql: '$9',
|
|
3503
|
+
forceRefreshSql: '$8',
|
|
3504
|
+
forceFailedRefreshSql: '$10',
|
|
3505
|
+
})}
|
|
3506
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
3507
|
+
status,
|
|
3508
|
+
output,
|
|
3509
|
+
error,
|
|
3510
|
+
failure_kind,
|
|
3511
|
+
run_id,
|
|
3512
|
+
lease_id,
|
|
3513
|
+
lease_owner_run_id,
|
|
3514
|
+
lease_owner_attempt,
|
|
3515
|
+
lease_expires_at,
|
|
3516
|
+
updated_at
|
|
2800
3517
|
)
|
|
2801
|
-
SELECT k,
|
|
3518
|
+
SELECT k,
|
|
3519
|
+
status,
|
|
3520
|
+
output,
|
|
3521
|
+
error,
|
|
3522
|
+
failure_kind,
|
|
3523
|
+
run_id,
|
|
3524
|
+
lease_id,
|
|
3525
|
+
lease_owner_run_id,
|
|
3526
|
+
lease_owner_attempt,
|
|
3527
|
+
lease_expires_at,
|
|
3528
|
+
updated_at
|
|
2802
3529
|
FROM claimed
|
|
2803
3530
|
`,
|
|
2804
3531
|
[
|
|
@@ -2807,6 +3534,11 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2807
3534
|
input.runId,
|
|
2808
3535
|
RECEIPT_STATUS_RUNNING,
|
|
2809
3536
|
claimableStatuses,
|
|
3537
|
+
leaseId,
|
|
3538
|
+
leaseTtlMs,
|
|
3539
|
+
input.forceRefresh === true,
|
|
3540
|
+
runAttempt,
|
|
3541
|
+
input.forceFailedRefresh === true,
|
|
2810
3542
|
],
|
|
2811
3543
|
);
|
|
2812
3544
|
const claimed = rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -2815,14 +3547,19 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2815
3547
|
}
|
|
2816
3548
|
|
|
2817
3549
|
const latest = await readRuntimeWorkReceipt(client, session, input.key);
|
|
2818
|
-
if (
|
|
3550
|
+
if (
|
|
3551
|
+
latest &&
|
|
3552
|
+
input.forceRefresh !== true &&
|
|
3553
|
+
isReusableWorkReceipt(latest)
|
|
3554
|
+
) {
|
|
2819
3555
|
return { disposition: 'reused', receipt: latest };
|
|
2820
3556
|
}
|
|
2821
|
-
if (latest
|
|
2822
|
-
return {
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
3557
|
+
if (latest) {
|
|
3558
|
+
return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
|
|
3559
|
+
receipt: latest,
|
|
3560
|
+
claimantRunId: input.runId,
|
|
3561
|
+
claimantRunAttempt: runAttempt,
|
|
3562
|
+
});
|
|
2826
3563
|
}
|
|
2827
3564
|
throw new Error(
|
|
2828
3565
|
`Runtime receipt ${input.key} claim did not return execution ownership.`,
|
|
@@ -2831,14 +3568,48 @@ export async function claimRuntimeWorkReceipt(
|
|
|
2831
3568
|
);
|
|
2832
3569
|
}
|
|
2833
3570
|
|
|
3571
|
+
function runtimeWorkReceiptClaimDispositionForBlockedReceipt(input: {
|
|
3572
|
+
receipt: WorkReceipt;
|
|
3573
|
+
claimantRunId: string;
|
|
3574
|
+
claimantRunAttempt?: number | null;
|
|
3575
|
+
}): Exclude<WorkReceiptClaim, { disposition: 'claimed' | 'reused' }> {
|
|
3576
|
+
const receipt = input.receipt;
|
|
3577
|
+
if (receipt.status === 'failed') {
|
|
3578
|
+
return { disposition: 'failed', receipt };
|
|
3579
|
+
}
|
|
3580
|
+
if (receipt.status === 'pending' || receipt.status === 'running') {
|
|
3581
|
+
const ownerRunId = receipt.leaseOwnerRunId ?? receipt.runId ?? null;
|
|
3582
|
+
const leaseExpiresAtMs =
|
|
3583
|
+
typeof receipt.leaseExpiresAt === 'string'
|
|
3584
|
+
? Date.parse(receipt.leaseExpiresAt)
|
|
3585
|
+
: NaN;
|
|
3586
|
+
const hasActiveForeignLease =
|
|
3587
|
+
ownerRunId !== null &&
|
|
3588
|
+
ownerRunId !== input.claimantRunId &&
|
|
3589
|
+
typeof receipt.leaseId === 'string' &&
|
|
3590
|
+
receipt.leaseId.length > 0 &&
|
|
3591
|
+
Number.isFinite(leaseExpiresAtMs) &&
|
|
3592
|
+
leaseExpiresAtMs > Date.now();
|
|
3593
|
+
return {
|
|
3594
|
+
disposition: hasActiveForeignLease ? 'blocked_active_lease' : 'running',
|
|
3595
|
+
receipt,
|
|
3596
|
+
};
|
|
3597
|
+
}
|
|
3598
|
+
return { disposition: 'running', receipt };
|
|
3599
|
+
}
|
|
3600
|
+
|
|
2834
3601
|
export async function claimRuntimeWorkReceipts(
|
|
2835
3602
|
context: RuntimeApiContext,
|
|
2836
3603
|
input: {
|
|
2837
3604
|
playName: string;
|
|
2838
3605
|
runId: string;
|
|
2839
3606
|
keys: string[];
|
|
3607
|
+
runAttempt?: number | null;
|
|
3608
|
+
leaseAware?: boolean;
|
|
2840
3609
|
reclaimRunning?: boolean;
|
|
2841
3610
|
forceRefresh?: boolean;
|
|
3611
|
+
forceFailedRefresh?: boolean;
|
|
3612
|
+
leaseTtlMs?: number | null;
|
|
2842
3613
|
},
|
|
2843
3614
|
): Promise<WorkReceiptClaim[]> {
|
|
2844
3615
|
const keys = [
|
|
@@ -2854,46 +3625,104 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2854
3625
|
session,
|
|
2855
3626
|
async (client) => {
|
|
2856
3627
|
const keyHexes = keys.map(workReceiptKeyHex);
|
|
2857
|
-
const
|
|
2858
|
-
?
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
]
|
|
2871
|
-
: [RECEIPT_STATUS_PENDING, RECEIPT_STATUS_FAILED];
|
|
3628
|
+
const leaseIds = keys.map(() =>
|
|
3629
|
+
input.leaseAware === true ? newRuntimeWorkReceiptLeaseId() : null,
|
|
3630
|
+
);
|
|
3631
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3632
|
+
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
3633
|
+
input.leaseTtlMs,
|
|
3634
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
3635
|
+
);
|
|
3636
|
+
const reclaimStatuses = workReceiptClaimableStatusCodes({
|
|
3637
|
+
forceRefresh: input.forceRefresh,
|
|
3638
|
+
forceFailedRefresh: input.forceFailedRefresh,
|
|
3639
|
+
reclaimRunning: input.reclaimRunning,
|
|
3640
|
+
});
|
|
2872
3641
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2873
3642
|
`
|
|
2874
3643
|
WITH input_keys AS (
|
|
2875
|
-
SELECT key_values.key_hex, key_values.ord
|
|
3644
|
+
SELECT key_values.key_hex, lease_values.lease_id, key_values.ord
|
|
2876
3645
|
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
3646
|
+
JOIN unnest($2::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
|
|
3647
|
+
ON lease_values.ord = key_values.ord
|
|
2877
3648
|
),
|
|
2878
3649
|
claimed AS (
|
|
2879
|
-
INSERT INTO ${workReceiptTable(session)} (
|
|
2880
|
-
|
|
3650
|
+
INSERT INTO ${workReceiptTable(session)} (
|
|
3651
|
+
k,
|
|
3652
|
+
status,
|
|
3653
|
+
run_id,
|
|
3654
|
+
lease_id,
|
|
3655
|
+
lease_owner_run_id,
|
|
3656
|
+
lease_owner_attempt,
|
|
3657
|
+
lease_expires_at,
|
|
3658
|
+
updated_at
|
|
3659
|
+
)
|
|
3660
|
+
SELECT decode(input_keys.key_hex, 'hex'),
|
|
3661
|
+
$3::smallint,
|
|
3662
|
+
$4::text,
|
|
3663
|
+
input_keys.lease_id,
|
|
3664
|
+
CASE
|
|
3665
|
+
WHEN input_keys.lease_id IS NULL THEN NULL
|
|
3666
|
+
ELSE $4::text
|
|
3667
|
+
END,
|
|
3668
|
+
CASE
|
|
3669
|
+
WHEN input_keys.lease_id IS NULL THEN NULL
|
|
3670
|
+
ELSE $9::integer
|
|
3671
|
+
END,
|
|
3672
|
+
CASE
|
|
3673
|
+
WHEN input_keys.lease_id IS NULL THEN NULL
|
|
3674
|
+
ELSE now() + ($7::double precision * interval '1 millisecond')
|
|
3675
|
+
END,
|
|
3676
|
+
now()
|
|
2881
3677
|
FROM input_keys
|
|
2882
3678
|
ON CONFLICT (k) DO UPDATE
|
|
2883
|
-
SET status = $
|
|
3679
|
+
SET status = $5::smallint,
|
|
2884
3680
|
output = NULL,
|
|
2885
|
-
|
|
3681
|
+
failure_kind = 0,
|
|
3682
|
+
run_id = $4::text,
|
|
3683
|
+
lease_id = EXCLUDED.lease_id,
|
|
3684
|
+
lease_owner_run_id = CASE
|
|
3685
|
+
WHEN EXCLUDED.lease_id IS NULL THEN NULL
|
|
3686
|
+
ELSE $4::text
|
|
3687
|
+
END,
|
|
3688
|
+
lease_owner_attempt = CASE
|
|
3689
|
+
WHEN EXCLUDED.lease_id IS NULL THEN NULL
|
|
3690
|
+
ELSE $9::integer
|
|
3691
|
+
END,
|
|
3692
|
+
lease_expires_at = EXCLUDED.lease_expires_at,
|
|
2886
3693
|
error = NULL,
|
|
2887
3694
|
updated_at = now()
|
|
2888
|
-
WHERE ${
|
|
2889
|
-
|
|
3695
|
+
WHERE ${workReceiptClaimConflictPredicateSql({
|
|
3696
|
+
receiptTable: workReceiptTable(session),
|
|
3697
|
+
claimableStatusesSql: '$6',
|
|
3698
|
+
claimantRunIdSql: '$4::text',
|
|
3699
|
+
claimantRunAttemptSql: '$9',
|
|
3700
|
+
forceRefreshSql: '$8',
|
|
3701
|
+
forceFailedRefreshSql: '$10',
|
|
3702
|
+
})}
|
|
3703
|
+
RETURNING k,
|
|
3704
|
+
status,
|
|
3705
|
+
output,
|
|
3706
|
+
error,
|
|
3707
|
+
failure_kind,
|
|
3708
|
+
run_id,
|
|
3709
|
+
lease_id,
|
|
3710
|
+
lease_owner_run_id,
|
|
3711
|
+
lease_owner_attempt,
|
|
3712
|
+
lease_expires_at,
|
|
3713
|
+
updated_at
|
|
2890
3714
|
),
|
|
2891
3715
|
latest AS (
|
|
2892
3716
|
SELECT receipts.k,
|
|
2893
3717
|
receipts.status,
|
|
2894
3718
|
receipts.output,
|
|
2895
3719
|
receipts.error,
|
|
3720
|
+
receipts.failure_kind,
|
|
2896
3721
|
receipts.run_id,
|
|
3722
|
+
receipts.lease_id,
|
|
3723
|
+
receipts.lease_owner_run_id,
|
|
3724
|
+
receipts.lease_owner_attempt,
|
|
3725
|
+
receipts.lease_expires_at,
|
|
2897
3726
|
receipts.updated_at,
|
|
2898
3727
|
input_keys.ord,
|
|
2899
3728
|
false AS claimed
|
|
@@ -2909,21 +3738,43 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2909
3738
|
claimed.status,
|
|
2910
3739
|
claimed.output,
|
|
2911
3740
|
claimed.error,
|
|
3741
|
+
claimed.failure_kind,
|
|
2912
3742
|
claimed.run_id,
|
|
3743
|
+
claimed.lease_id,
|
|
3744
|
+
claimed.lease_owner_run_id,
|
|
3745
|
+
claimed.lease_owner_attempt,
|
|
3746
|
+
claimed.lease_expires_at,
|
|
2913
3747
|
claimed.updated_at,
|
|
2914
3748
|
input_keys.ord,
|
|
2915
3749
|
true AS claimed
|
|
2916
3750
|
FROM claimed
|
|
2917
3751
|
JOIN input_keys ON claimed.k = decode(input_keys.key_hex, 'hex')
|
|
2918
3752
|
UNION ALL
|
|
2919
|
-
SELECT k,
|
|
3753
|
+
SELECT k,
|
|
3754
|
+
status,
|
|
3755
|
+
output,
|
|
3756
|
+
error,
|
|
3757
|
+
failure_kind,
|
|
3758
|
+
run_id,
|
|
3759
|
+
lease_id,
|
|
3760
|
+
lease_owner_run_id,
|
|
3761
|
+
lease_owner_attempt,
|
|
3762
|
+
lease_expires_at,
|
|
3763
|
+
updated_at,
|
|
3764
|
+
ord,
|
|
3765
|
+
claimed
|
|
2920
3766
|
FROM latest
|
|
2921
3767
|
)
|
|
2922
3768
|
SELECT convert_from(k, 'UTF8') AS k,
|
|
2923
3769
|
status,
|
|
2924
3770
|
output,
|
|
2925
3771
|
error,
|
|
3772
|
+
failure_kind,
|
|
2926
3773
|
run_id,
|
|
3774
|
+
lease_id,
|
|
3775
|
+
lease_owner_run_id,
|
|
3776
|
+
lease_owner_attempt,
|
|
3777
|
+
lease_expires_at,
|
|
2927
3778
|
updated_at,
|
|
2928
3779
|
claimed
|
|
2929
3780
|
FROM returned
|
|
@@ -2931,10 +3782,15 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2931
3782
|
`,
|
|
2932
3783
|
[
|
|
2933
3784
|
keyHexes,
|
|
3785
|
+
leaseIds,
|
|
2934
3786
|
RECEIPT_STATUS_RUNNING,
|
|
2935
3787
|
input.runId,
|
|
2936
3788
|
RECEIPT_STATUS_RUNNING,
|
|
2937
3789
|
reclaimStatuses,
|
|
3790
|
+
leaseTtlMs,
|
|
3791
|
+
input.forceRefresh === true,
|
|
3792
|
+
runAttempt,
|
|
3793
|
+
input.forceFailedRefresh === true,
|
|
2938
3794
|
],
|
|
2939
3795
|
);
|
|
2940
3796
|
return rows.map((row) => {
|
|
@@ -2942,16 +3798,14 @@ export async function claimRuntimeWorkReceipts(
|
|
|
2942
3798
|
if (row.claimed === true) {
|
|
2943
3799
|
return { disposition: 'claimed', receipt } satisfies WorkReceiptClaim;
|
|
2944
3800
|
}
|
|
2945
|
-
if (isReusableWorkReceipt(receipt)) {
|
|
3801
|
+
if (input.forceRefresh !== true && isReusableWorkReceipt(receipt)) {
|
|
2946
3802
|
return { disposition: 'reused', receipt } satisfies WorkReceiptClaim;
|
|
2947
3803
|
}
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
}
|
|
2954
|
-
return { disposition: 'running', receipt } satisfies WorkReceiptClaim;
|
|
3804
|
+
return runtimeWorkReceiptClaimDispositionForBlockedReceipt({
|
|
3805
|
+
receipt,
|
|
3806
|
+
claimantRunId: input.runId,
|
|
3807
|
+
claimantRunAttempt: runAttempt,
|
|
3808
|
+
});
|
|
2955
3809
|
});
|
|
2956
3810
|
},
|
|
2957
3811
|
);
|
|
@@ -2963,17 +3817,50 @@ export async function completeRuntimeWorkReceipt(
|
|
|
2963
3817
|
playName: string;
|
|
2964
3818
|
runId: string;
|
|
2965
3819
|
key: string;
|
|
3820
|
+
runAttempt?: number | null;
|
|
3821
|
+
leaseId?: string | null;
|
|
2966
3822
|
output: unknown;
|
|
2967
3823
|
},
|
|
2968
3824
|
): Promise<WorkReceipt | null> {
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
3825
|
+
if (
|
|
3826
|
+
consumeDirectRuntimeTestFault({
|
|
3827
|
+
context,
|
|
3828
|
+
runId: input.runId,
|
|
3829
|
+
name: 'receipt_complete_write_fail',
|
|
3830
|
+
})
|
|
3831
|
+
) {
|
|
3832
|
+
const error = runtimeTestFaultError('receipt_complete_write_fail');
|
|
3833
|
+
const failed = await failRuntimeWorkReceipt(context, {
|
|
3834
|
+
playName: input.playName,
|
|
3835
|
+
runId: input.runId,
|
|
3836
|
+
key: input.key,
|
|
3837
|
+
leaseId: input.leaseId,
|
|
3838
|
+
error: error.message,
|
|
3839
|
+
failureKind: 'repairable',
|
|
3840
|
+
runAttempt: input.runAttempt,
|
|
3841
|
+
});
|
|
3842
|
+
if (!failed) {
|
|
3843
|
+
await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
|
|
3844
|
+
playName: input.playName,
|
|
3845
|
+
runId: input.runId,
|
|
3846
|
+
key: input.key,
|
|
3847
|
+
error: error.message,
|
|
3848
|
+
failureKind: 'repairable',
|
|
3849
|
+
runAttempt: input.runAttempt,
|
|
3850
|
+
});
|
|
3851
|
+
}
|
|
3852
|
+
throw error;
|
|
3853
|
+
}
|
|
3854
|
+
const session = await getRuntimeWorkReceiptSession(context, {
|
|
3855
|
+
playName: input.playName,
|
|
3856
|
+
key: input.key,
|
|
3857
|
+
});
|
|
3858
|
+
return await withRuntimeWorkReceiptClient(
|
|
3859
|
+
context,
|
|
3860
|
+
session,
|
|
3861
|
+
async (client) => {
|
|
3862
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
3863
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
2977
3864
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
2978
3865
|
`
|
|
2979
3866
|
WITH completed AS (
|
|
@@ -2981,29 +3868,41 @@ export async function completeRuntimeWorkReceipt(
|
|
|
2981
3868
|
SET status = $2::smallint,
|
|
2982
3869
|
output = $3::jsonb,
|
|
2983
3870
|
error = NULL,
|
|
3871
|
+
failure_kind = 0,
|
|
2984
3872
|
run_id = $4,
|
|
3873
|
+
lease_id = NULL,
|
|
3874
|
+
lease_owner_run_id = NULL,
|
|
3875
|
+
lease_owner_attempt = NULL,
|
|
3876
|
+
lease_expires_at = NULL,
|
|
2985
3877
|
updated_at = now()
|
|
2986
3878
|
WHERE k = decode($1, 'hex')
|
|
2987
|
-
AND
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3879
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
3880
|
+
receiptTable: workReceiptTable(session),
|
|
3881
|
+
ownerRunIdSql: '$4',
|
|
3882
|
+
ownerRunAttemptSql: '$6',
|
|
3883
|
+
leaseIdSql: '$5::text',
|
|
3884
|
+
})}
|
|
3885
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
3886
|
+
status,
|
|
3887
|
+
output,
|
|
3888
|
+
error,
|
|
3889
|
+
failure_kind,
|
|
3890
|
+
run_id,
|
|
3891
|
+
lease_id,
|
|
3892
|
+
lease_owner_run_id,
|
|
3893
|
+
lease_owner_attempt,
|
|
3894
|
+
lease_expires_at,
|
|
3895
|
+
updated_at
|
|
3896
|
+
)
|
|
3897
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM completed
|
|
3000
3898
|
`,
|
|
3001
3899
|
[
|
|
3002
3900
|
workReceiptKeyHex(input.key),
|
|
3003
3901
|
RECEIPT_STATUS_COMPLETED,
|
|
3004
3902
|
input.output === null ? null : stringifyPostgresJson(input.output),
|
|
3005
3903
|
input.runId,
|
|
3006
|
-
|
|
3904
|
+
leaseId,
|
|
3905
|
+
runAttempt,
|
|
3007
3906
|
],
|
|
3008
3907
|
);
|
|
3009
3908
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -3015,11 +3914,63 @@ export async function completeRuntimeWorkReceipts(
|
|
|
3015
3914
|
context: RuntimeApiContext,
|
|
3016
3915
|
input: {
|
|
3017
3916
|
playName: string;
|
|
3018
|
-
receipts: Array<{
|
|
3917
|
+
receipts: Array<{
|
|
3918
|
+
runId: string;
|
|
3919
|
+
key: string;
|
|
3920
|
+
output: unknown;
|
|
3921
|
+
leaseId?: string | null;
|
|
3922
|
+
runAttempt?: number | null;
|
|
3923
|
+
}>;
|
|
3019
3924
|
},
|
|
3020
|
-
): Promise<WorkReceipt
|
|
3925
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
3021
3926
|
const receipts = input.receipts.filter((receipt) => receipt.key.trim());
|
|
3022
3927
|
if (receipts.length === 0) return [];
|
|
3928
|
+
if (
|
|
3929
|
+
consumeDirectRuntimeTestFault({
|
|
3930
|
+
context,
|
|
3931
|
+
runId: receipts[0]?.runId ?? '',
|
|
3932
|
+
name: 'receipt_complete_write_fail',
|
|
3933
|
+
})
|
|
3934
|
+
) {
|
|
3935
|
+
const [skippedReceipt, ...siblingReceipts] = receipts;
|
|
3936
|
+
let siblingError: unknown = null;
|
|
3937
|
+
if (siblingReceipts.length > 0) {
|
|
3938
|
+
try {
|
|
3939
|
+
await completeRuntimeWorkReceipts(context, {
|
|
3940
|
+
playName: input.playName,
|
|
3941
|
+
receipts: siblingReceipts,
|
|
3942
|
+
});
|
|
3943
|
+
} catch (error) {
|
|
3944
|
+
siblingError = error;
|
|
3945
|
+
}
|
|
3946
|
+
}
|
|
3947
|
+
const error = runtimeTestFaultError('receipt_complete_write_fail');
|
|
3948
|
+
if (skippedReceipt) {
|
|
3949
|
+
const failed = await failRuntimeWorkReceipt(context, {
|
|
3950
|
+
playName: input.playName,
|
|
3951
|
+
runId: skippedReceipt.runId,
|
|
3952
|
+
key: skippedReceipt.key,
|
|
3953
|
+
leaseId: skippedReceipt.leaseId,
|
|
3954
|
+
error: error.message,
|
|
3955
|
+
failureKind: 'repairable',
|
|
3956
|
+
runAttempt: skippedReceipt.runAttempt,
|
|
3957
|
+
});
|
|
3958
|
+
if (!failed) {
|
|
3959
|
+
await forceFailRuntimeWorkReceiptForRuntimeTestFault(context, {
|
|
3960
|
+
playName: input.playName,
|
|
3961
|
+
runId: skippedReceipt.runId,
|
|
3962
|
+
key: skippedReceipt.key,
|
|
3963
|
+
error: error.message,
|
|
3964
|
+
failureKind: 'repairable',
|
|
3965
|
+
runAttempt: skippedReceipt.runAttempt,
|
|
3966
|
+
});
|
|
3967
|
+
}
|
|
3968
|
+
}
|
|
3969
|
+
if (siblingError) {
|
|
3970
|
+
throw siblingError;
|
|
3971
|
+
}
|
|
3972
|
+
throw error;
|
|
3973
|
+
}
|
|
3023
3974
|
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
3024
3975
|
playName: input.playName,
|
|
3025
3976
|
keys: receipts.map((receipt) => receipt.key),
|
|
@@ -3033,52 +3984,79 @@ export async function completeRuntimeWorkReceipts(
|
|
|
3033
3984
|
WITH inputs AS (
|
|
3034
3985
|
SELECT key_values.key_hex,
|
|
3035
3986
|
run_values.run_id,
|
|
3987
|
+
lease_values.lease_id,
|
|
3988
|
+
attempt_values.run_attempt,
|
|
3036
3989
|
output_values.output,
|
|
3037
3990
|
key_values.ord
|
|
3038
3991
|
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
3039
3992
|
JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
|
|
3040
3993
|
ON run_values.ord = key_values.ord
|
|
3041
|
-
JOIN
|
|
3994
|
+
JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
|
|
3995
|
+
ON lease_values.ord = key_values.ord
|
|
3996
|
+
JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
|
|
3997
|
+
ON attempt_values.ord = key_values.ord
|
|
3998
|
+
JOIN jsonb_array_elements($5::jsonb) WITH ORDINALITY AS output_values(output, ord)
|
|
3042
3999
|
ON output_values.ord = key_values.ord
|
|
3043
4000
|
),
|
|
3044
4001
|
completed AS (
|
|
3045
4002
|
UPDATE ${workReceiptTable(session)} AS target
|
|
3046
|
-
SET status = $
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
4003
|
+
SET status = $6::smallint,
|
|
4004
|
+
output = inputs.output,
|
|
4005
|
+
error = NULL,
|
|
4006
|
+
failure_kind = 0,
|
|
4007
|
+
run_id = inputs.run_id,
|
|
4008
|
+
lease_id = NULL,
|
|
4009
|
+
lease_owner_run_id = NULL,
|
|
4010
|
+
lease_owner_attempt = NULL,
|
|
4011
|
+
lease_expires_at = NULL,
|
|
3050
4012
|
updated_at = now()
|
|
3051
4013
|
FROM inputs
|
|
3052
4014
|
WHERE target.k = decode(inputs.key_hex, 'hex')
|
|
3053
|
-
AND
|
|
3054
|
-
|
|
3055
|
-
|
|
4015
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4016
|
+
receiptTable: 'target',
|
|
4017
|
+
ownerRunIdSql: 'inputs.run_id',
|
|
4018
|
+
ownerRunAttemptSql: 'inputs.run_attempt',
|
|
4019
|
+
leaseIdSql: 'inputs.lease_id',
|
|
4020
|
+
})}
|
|
4021
|
+
RETURNING target.k,
|
|
4022
|
+
target.status,
|
|
4023
|
+
target.output,
|
|
4024
|
+
target.error,
|
|
4025
|
+
target.failure_kind,
|
|
4026
|
+
target.run_id,
|
|
4027
|
+
target.lease_id,
|
|
4028
|
+
target.lease_owner_run_id,
|
|
4029
|
+
target.lease_owner_attempt,
|
|
4030
|
+
target.lease_expires_at,
|
|
4031
|
+
target.updated_at,
|
|
4032
|
+
inputs.ord
|
|
3056
4033
|
),
|
|
3057
|
-
|
|
3058
|
-
SELECT
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
4034
|
+
returned AS (
|
|
4035
|
+
SELECT completed.k,
|
|
4036
|
+
completed.status,
|
|
4037
|
+
completed.output,
|
|
4038
|
+
completed.error,
|
|
4039
|
+
completed.failure_kind,
|
|
4040
|
+
completed.run_id,
|
|
4041
|
+
completed.lease_id,
|
|
4042
|
+
completed.lease_owner_run_id,
|
|
4043
|
+
completed.lease_owner_attempt,
|
|
4044
|
+
completed.lease_expires_at,
|
|
4045
|
+
completed.updated_at,
|
|
3064
4046
|
inputs.ord
|
|
3065
4047
|
FROM inputs
|
|
3066
|
-
JOIN
|
|
3067
|
-
ON receipts.k = decode(inputs.key_hex, 'hex')
|
|
3068
|
-
WHERE NOT EXISTS (
|
|
3069
|
-
SELECT 1 FROM completed WHERE completed.k = receipts.k
|
|
3070
|
-
)
|
|
3071
|
-
),
|
|
3072
|
-
returned AS (
|
|
3073
|
-
SELECT k, status, output, error, run_id, updated_at, ord FROM completed
|
|
3074
|
-
UNION ALL
|
|
3075
|
-
SELECT k, status, output, error, run_id, updated_at, ord FROM latest
|
|
4048
|
+
LEFT JOIN completed ON completed.ord = inputs.ord
|
|
3076
4049
|
)
|
|
3077
4050
|
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
3078
4051
|
returned.status,
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
4052
|
+
returned.output,
|
|
4053
|
+
returned.error,
|
|
4054
|
+
returned.failure_kind,
|
|
4055
|
+
returned.run_id,
|
|
4056
|
+
returned.lease_id,
|
|
4057
|
+
returned.lease_owner_run_id,
|
|
4058
|
+
returned.lease_owner_attempt,
|
|
4059
|
+
returned.lease_expires_at,
|
|
3082
4060
|
returned.updated_at
|
|
3083
4061
|
FROM returned
|
|
3084
4062
|
ORDER BY returned.ord
|
|
@@ -3086,16 +4064,21 @@ export async function completeRuntimeWorkReceipts(
|
|
|
3086
4064
|
[
|
|
3087
4065
|
receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
|
|
3088
4066
|
receipts.map((receipt) => receipt.runId),
|
|
4067
|
+
receipts.map((receipt) => receipt.leaseId?.trim() || null),
|
|
4068
|
+
receipts.map((receipt) =>
|
|
4069
|
+
normalizeRuntimeRunAttempt(receipt.runAttempt),
|
|
4070
|
+
),
|
|
3089
4071
|
stringifyPostgresJson(
|
|
3090
4072
|
receipts.map((receipt) =>
|
|
3091
4073
|
receipt.output === null ? null : receipt.output,
|
|
3092
4074
|
),
|
|
3093
4075
|
),
|
|
3094
4076
|
RECEIPT_STATUS_COMPLETED,
|
|
3095
|
-
RECEIPT_STATUS_RUNNING,
|
|
3096
4077
|
],
|
|
3097
4078
|
);
|
|
3098
|
-
return rows.map(
|
|
4079
|
+
return rows.map((row) =>
|
|
4080
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
4081
|
+
);
|
|
3099
4082
|
},
|
|
3100
4083
|
);
|
|
3101
4084
|
}
|
|
@@ -3106,7 +4089,10 @@ export async function failRuntimeWorkReceipt(
|
|
|
3106
4089
|
playName: string;
|
|
3107
4090
|
runId: string;
|
|
3108
4091
|
key: string;
|
|
4092
|
+
runAttempt?: number | null;
|
|
4093
|
+
leaseId?: string | null;
|
|
3109
4094
|
error: string;
|
|
4095
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
3110
4096
|
},
|
|
3111
4097
|
): Promise<WorkReceipt | null> {
|
|
3112
4098
|
const session = await getRuntimeWorkReceiptSession(context, {
|
|
@@ -3117,36 +4103,119 @@ export async function failRuntimeWorkReceipt(
|
|
|
3117
4103
|
context,
|
|
3118
4104
|
session,
|
|
3119
4105
|
async (client) => {
|
|
4106
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
4107
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3120
4108
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
3121
4109
|
`
|
|
3122
4110
|
WITH failed AS (
|
|
3123
4111
|
UPDATE ${workReceiptTable(session)}
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
4112
|
+
SET status = $2::smallint,
|
|
4113
|
+
output = NULL,
|
|
4114
|
+
error = $3,
|
|
4115
|
+
failure_kind = $6::smallint,
|
|
3127
4116
|
run_id = $4,
|
|
4117
|
+
lease_id = NULL,
|
|
4118
|
+
lease_owner_run_id = NULL,
|
|
4119
|
+
lease_owner_attempt = NULL,
|
|
4120
|
+
lease_expires_at = NULL,
|
|
3128
4121
|
updated_at = now()
|
|
3129
4122
|
WHERE k = decode($1, 'hex')
|
|
3130
|
-
AND
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
4123
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4124
|
+
receiptTable: workReceiptTable(session),
|
|
4125
|
+
ownerRunIdSql: '$4',
|
|
4126
|
+
ownerRunAttemptSql: '$7',
|
|
4127
|
+
leaseIdSql: '$5::text',
|
|
4128
|
+
})}
|
|
4129
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4130
|
+
status,
|
|
4131
|
+
output,
|
|
4132
|
+
error,
|
|
4133
|
+
failure_kind,
|
|
4134
|
+
run_id,
|
|
4135
|
+
lease_id,
|
|
4136
|
+
lease_owner_run_id,
|
|
4137
|
+
lease_owner_attempt,
|
|
4138
|
+
lease_expires_at,
|
|
4139
|
+
updated_at
|
|
4140
|
+
)
|
|
4141
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM failed
|
|
3143
4142
|
`,
|
|
3144
4143
|
[
|
|
3145
4144
|
workReceiptKeyHex(input.key),
|
|
3146
4145
|
RECEIPT_STATUS_FAILED,
|
|
3147
4146
|
input.error,
|
|
3148
4147
|
input.runId,
|
|
3149
|
-
|
|
4148
|
+
leaseId,
|
|
4149
|
+
workReceiptFailureKindCode(input.failureKind),
|
|
4150
|
+
runAttempt,
|
|
4151
|
+
],
|
|
4152
|
+
);
|
|
4153
|
+
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
4154
|
+
},
|
|
4155
|
+
);
|
|
4156
|
+
}
|
|
4157
|
+
|
|
4158
|
+
export async function releaseRuntimeWorkReceipt(
|
|
4159
|
+
context: RuntimeApiContext,
|
|
4160
|
+
input: {
|
|
4161
|
+
playName: string;
|
|
4162
|
+
runId: string;
|
|
4163
|
+
key: string;
|
|
4164
|
+
runAttempt?: number | null;
|
|
4165
|
+
leaseId?: string | null;
|
|
4166
|
+
},
|
|
4167
|
+
): Promise<WorkReceipt | null> {
|
|
4168
|
+
const session = await getRuntimeWorkReceiptSession(context, {
|
|
4169
|
+
playName: input.playName,
|
|
4170
|
+
key: input.key,
|
|
4171
|
+
});
|
|
4172
|
+
return await withRuntimeWorkReceiptClient(
|
|
4173
|
+
context,
|
|
4174
|
+
session,
|
|
4175
|
+
async (client) => {
|
|
4176
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
4177
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
4178
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
4179
|
+
`
|
|
4180
|
+
WITH released AS (
|
|
4181
|
+
UPDATE ${workReceiptTable(session)}
|
|
4182
|
+
SET status = $2::smallint,
|
|
4183
|
+
output = NULL,
|
|
4184
|
+
error = NULL,
|
|
4185
|
+
failure_kind = 0,
|
|
4186
|
+
run_id = $3,
|
|
4187
|
+
lease_id = NULL,
|
|
4188
|
+
lease_owner_run_id = NULL,
|
|
4189
|
+
lease_owner_attempt = NULL,
|
|
4190
|
+
lease_expires_at = NULL,
|
|
4191
|
+
updated_at = now()
|
|
4192
|
+
WHERE k = decode($1, 'hex')
|
|
4193
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4194
|
+
receiptTable: workReceiptTable(session),
|
|
4195
|
+
ownerRunIdSql: '$3',
|
|
4196
|
+
ownerRunAttemptSql: '$5',
|
|
4197
|
+
leaseIdSql: '$4::text',
|
|
4198
|
+
})}
|
|
4199
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4200
|
+
status,
|
|
4201
|
+
output,
|
|
4202
|
+
error,
|
|
4203
|
+
failure_kind,
|
|
4204
|
+
run_id,
|
|
4205
|
+
lease_id,
|
|
4206
|
+
lease_owner_run_id,
|
|
4207
|
+
lease_owner_attempt,
|
|
4208
|
+
lease_expires_at,
|
|
4209
|
+
updated_at
|
|
4210
|
+
)
|
|
4211
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM released
|
|
4212
|
+
`,
|
|
4213
|
+
[
|
|
4214
|
+
workReceiptKeyHex(input.key),
|
|
4215
|
+
RECEIPT_STATUS_PENDING,
|
|
4216
|
+
input.runId,
|
|
4217
|
+
leaseId,
|
|
4218
|
+
runAttempt,
|
|
3150
4219
|
],
|
|
3151
4220
|
);
|
|
3152
4221
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -3158,9 +4227,16 @@ export async function failRuntimeWorkReceipts(
|
|
|
3158
4227
|
context: RuntimeApiContext,
|
|
3159
4228
|
input: {
|
|
3160
4229
|
playName: string;
|
|
3161
|
-
receipts: Array<{
|
|
4230
|
+
receipts: Array<{
|
|
4231
|
+
runId: string;
|
|
4232
|
+
key: string;
|
|
4233
|
+
error: string;
|
|
4234
|
+
failureKind?: WorkReceiptFailureKind | null;
|
|
4235
|
+
leaseId?: string | null;
|
|
4236
|
+
runAttempt?: number | null;
|
|
4237
|
+
}>;
|
|
3162
4238
|
},
|
|
3163
|
-
): Promise<WorkReceipt
|
|
4239
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
3164
4240
|
const receipts = input.receipts.filter((receipt) => receipt.key.trim());
|
|
3165
4241
|
if (receipts.length === 0) return [];
|
|
3166
4242
|
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
@@ -3176,65 +4252,207 @@ export async function failRuntimeWorkReceipts(
|
|
|
3176
4252
|
WITH inputs AS (
|
|
3177
4253
|
SELECT key_values.key_hex,
|
|
3178
4254
|
run_values.run_id,
|
|
3179
|
-
|
|
3180
|
-
|
|
4255
|
+
lease_values.lease_id,
|
|
4256
|
+
attempt_values.run_attempt,
|
|
4257
|
+
error_values.error,
|
|
4258
|
+
failure_kind_values.failure_kind,
|
|
4259
|
+
key_values.ord
|
|
3181
4260
|
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
3182
4261
|
JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
|
|
3183
4262
|
ON run_values.ord = key_values.ord
|
|
3184
|
-
JOIN unnest($3::text[]) WITH ORDINALITY AS
|
|
3185
|
-
ON
|
|
4263
|
+
JOIN unnest($3::text[]) WITH ORDINALITY AS lease_values(lease_id, ord)
|
|
4264
|
+
ON lease_values.ord = key_values.ord
|
|
4265
|
+
JOIN unnest($4::integer[]) WITH ORDINALITY AS attempt_values(run_attempt, ord)
|
|
4266
|
+
ON attempt_values.ord = key_values.ord
|
|
4267
|
+
JOIN unnest($5::text[]) WITH ORDINALITY AS error_values(error, ord)
|
|
4268
|
+
ON error_values.ord = key_values.ord
|
|
4269
|
+
JOIN unnest($6::smallint[]) WITH ORDINALITY AS failure_kind_values(failure_kind, ord)
|
|
4270
|
+
ON failure_kind_values.ord = key_values.ord
|
|
3186
4271
|
),
|
|
3187
4272
|
failed AS (
|
|
3188
4273
|
UPDATE ${workReceiptTable(session)} AS target
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
4274
|
+
SET status = $7::smallint,
|
|
4275
|
+
output = NULL,
|
|
4276
|
+
error = inputs.error,
|
|
4277
|
+
failure_kind = inputs.failure_kind,
|
|
4278
|
+
run_id = inputs.run_id,
|
|
4279
|
+
lease_id = NULL,
|
|
4280
|
+
lease_owner_run_id = NULL,
|
|
4281
|
+
lease_owner_attempt = NULL,
|
|
4282
|
+
lease_expires_at = NULL,
|
|
3193
4283
|
updated_at = now()
|
|
3194
4284
|
FROM inputs
|
|
3195
4285
|
WHERE target.k = decode(inputs.key_hex, 'hex')
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
4286
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4287
|
+
receiptTable: 'target',
|
|
4288
|
+
ownerRunIdSql: 'inputs.run_id',
|
|
4289
|
+
ownerRunAttemptSql: 'inputs.run_attempt',
|
|
4290
|
+
leaseIdSql: 'inputs.lease_id',
|
|
4291
|
+
})}
|
|
4292
|
+
RETURNING target.k,
|
|
4293
|
+
target.status,
|
|
4294
|
+
target.output,
|
|
4295
|
+
target.error,
|
|
4296
|
+
target.failure_kind,
|
|
4297
|
+
target.run_id,
|
|
4298
|
+
target.lease_id,
|
|
4299
|
+
target.lease_owner_run_id,
|
|
4300
|
+
target.lease_owner_attempt,
|
|
4301
|
+
target.lease_expires_at,
|
|
4302
|
+
target.updated_at,
|
|
4303
|
+
inputs.ord
|
|
3199
4304
|
),
|
|
3200
|
-
|
|
3201
|
-
SELECT
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
4305
|
+
returned AS (
|
|
4306
|
+
SELECT failed.k,
|
|
4307
|
+
failed.status,
|
|
4308
|
+
failed.output,
|
|
4309
|
+
failed.error,
|
|
4310
|
+
failed.failure_kind,
|
|
4311
|
+
failed.run_id,
|
|
4312
|
+
failed.lease_id,
|
|
4313
|
+
failed.lease_owner_run_id,
|
|
4314
|
+
failed.lease_owner_attempt,
|
|
4315
|
+
failed.lease_expires_at,
|
|
4316
|
+
failed.updated_at,
|
|
3207
4317
|
inputs.ord
|
|
3208
4318
|
FROM inputs
|
|
3209
|
-
JOIN
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
4319
|
+
LEFT JOIN failed ON failed.ord = inputs.ord
|
|
4320
|
+
)
|
|
4321
|
+
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
4322
|
+
returned.status,
|
|
4323
|
+
returned.output,
|
|
4324
|
+
returned.error,
|
|
4325
|
+
returned.failure_kind,
|
|
4326
|
+
returned.run_id,
|
|
4327
|
+
returned.lease_id,
|
|
4328
|
+
returned.lease_owner_run_id,
|
|
4329
|
+
returned.lease_owner_attempt,
|
|
4330
|
+
returned.lease_expires_at,
|
|
4331
|
+
returned.updated_at
|
|
4332
|
+
FROM returned
|
|
4333
|
+
ORDER BY returned.ord
|
|
4334
|
+
`,
|
|
4335
|
+
[
|
|
4336
|
+
receipts.map((receipt) => workReceiptKeyHex(receipt.key)),
|
|
4337
|
+
receipts.map((receipt) => receipt.runId),
|
|
4338
|
+
receipts.map((receipt) => receipt.leaseId?.trim() || null),
|
|
4339
|
+
receipts.map((receipt) =>
|
|
4340
|
+
normalizeRuntimeRunAttempt(receipt.runAttempt),
|
|
4341
|
+
),
|
|
4342
|
+
receipts.map((receipt) => receipt.error),
|
|
4343
|
+
receipts.map((receipt) =>
|
|
4344
|
+
workReceiptFailureKindCode(receipt.failureKind),
|
|
4345
|
+
),
|
|
4346
|
+
RECEIPT_STATUS_FAILED,
|
|
4347
|
+
],
|
|
4348
|
+
);
|
|
4349
|
+
return rows.map((row) =>
|
|
4350
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
4351
|
+
);
|
|
4352
|
+
},
|
|
4353
|
+
);
|
|
4354
|
+
}
|
|
4355
|
+
|
|
4356
|
+
export async function heartbeatRuntimeWorkReceipts(
|
|
4357
|
+
context: RuntimeApiContext,
|
|
4358
|
+
input: {
|
|
4359
|
+
playName: string;
|
|
4360
|
+
runId: string;
|
|
4361
|
+
runAttempt?: number | null;
|
|
4362
|
+
leaseId: string;
|
|
4363
|
+
keys: string[];
|
|
4364
|
+
leaseTtlMs?: number | null;
|
|
4365
|
+
},
|
|
4366
|
+
): Promise<Array<WorkReceipt | null>> {
|
|
4367
|
+
const keys = input.keys.map((key) => key.trim()).filter(Boolean);
|
|
4368
|
+
if (keys.length === 0) return [];
|
|
4369
|
+
const leaseId = input.leaseId.trim();
|
|
4370
|
+
if (!leaseId) return keys.map(() => null);
|
|
4371
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
4372
|
+
const leaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
4373
|
+
input.leaseTtlMs,
|
|
4374
|
+
PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
4375
|
+
);
|
|
4376
|
+
const session = await getRuntimeWorkReceiptSessionForKeys(context, {
|
|
4377
|
+
playName: input.playName,
|
|
4378
|
+
keys,
|
|
4379
|
+
});
|
|
4380
|
+
return await withRuntimeWorkReceiptClient(
|
|
4381
|
+
context,
|
|
4382
|
+
session,
|
|
4383
|
+
async (client) => {
|
|
4384
|
+
const { rows } = await client.query<Record<string, unknown>>(
|
|
4385
|
+
`
|
|
4386
|
+
WITH input_keys AS (
|
|
4387
|
+
SELECT key_values.key_hex, key_values.ord
|
|
4388
|
+
FROM unnest($1::text[]) WITH ORDINALITY AS key_values(key_hex, ord)
|
|
4389
|
+
),
|
|
4390
|
+
renewed AS (
|
|
4391
|
+
UPDATE ${workReceiptTable(session)} AS target
|
|
4392
|
+
SET lease_expires_at = now() + ($4::double precision * interval '1 millisecond'),
|
|
4393
|
+
updated_at = now()
|
|
4394
|
+
FROM input_keys
|
|
4395
|
+
WHERE target.k = decode(input_keys.key_hex, 'hex')
|
|
4396
|
+
AND ${workReceiptHeartbeatPredicateSql({
|
|
4397
|
+
receiptTable: 'target',
|
|
4398
|
+
ownerRunIdSql: '$2::text',
|
|
4399
|
+
ownerRunAttemptSql: '$5',
|
|
4400
|
+
leaseIdSql: '$3::text',
|
|
4401
|
+
})}
|
|
4402
|
+
RETURNING target.k,
|
|
4403
|
+
target.status,
|
|
4404
|
+
target.output,
|
|
4405
|
+
target.error,
|
|
4406
|
+
target.failure_kind,
|
|
4407
|
+
target.run_id,
|
|
4408
|
+
target.lease_id,
|
|
4409
|
+
target.lease_owner_run_id,
|
|
4410
|
+
target.lease_owner_attempt,
|
|
4411
|
+
target.lease_expires_at,
|
|
4412
|
+
target.updated_at,
|
|
4413
|
+
input_keys.ord
|
|
3214
4414
|
),
|
|
3215
4415
|
returned AS (
|
|
3216
|
-
SELECT k,
|
|
3217
|
-
|
|
3218
|
-
|
|
4416
|
+
SELECT renewed.k,
|
|
4417
|
+
renewed.status,
|
|
4418
|
+
renewed.output,
|
|
4419
|
+
renewed.error,
|
|
4420
|
+
renewed.failure_kind,
|
|
4421
|
+
renewed.run_id,
|
|
4422
|
+
renewed.lease_id,
|
|
4423
|
+
renewed.lease_owner_run_id,
|
|
4424
|
+
renewed.lease_owner_attempt,
|
|
4425
|
+
renewed.lease_expires_at,
|
|
4426
|
+
renewed.updated_at,
|
|
4427
|
+
input_keys.ord
|
|
4428
|
+
FROM input_keys
|
|
4429
|
+
LEFT JOIN renewed ON renewed.ord = input_keys.ord
|
|
3219
4430
|
)
|
|
3220
4431
|
SELECT convert_from(returned.k, 'UTF8') AS k,
|
|
3221
4432
|
returned.status,
|
|
3222
4433
|
returned.output,
|
|
3223
4434
|
returned.error,
|
|
4435
|
+
returned.failure_kind,
|
|
3224
4436
|
returned.run_id,
|
|
4437
|
+
returned.lease_id,
|
|
4438
|
+
returned.lease_owner_run_id,
|
|
4439
|
+
returned.lease_owner_attempt,
|
|
4440
|
+
returned.lease_expires_at,
|
|
3225
4441
|
returned.updated_at
|
|
3226
4442
|
FROM returned
|
|
3227
4443
|
ORDER BY returned.ord
|
|
3228
4444
|
`,
|
|
3229
4445
|
[
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
4446
|
+
keys.map(workReceiptKeyHex),
|
|
4447
|
+
input.runId,
|
|
4448
|
+
leaseId,
|
|
4449
|
+
leaseTtlMs,
|
|
4450
|
+
runAttempt,
|
|
3235
4451
|
],
|
|
3236
4452
|
);
|
|
3237
|
-
return rows.map(
|
|
4453
|
+
return rows.map((row) =>
|
|
4454
|
+
row.k == null ? null : mapRuntimeWorkReceiptRow(row),
|
|
4455
|
+
);
|
|
3238
4456
|
},
|
|
3239
4457
|
);
|
|
3240
4458
|
}
|
|
@@ -3245,6 +4463,8 @@ export async function skipRuntimeWorkReceipt(
|
|
|
3245
4463
|
playName: string;
|
|
3246
4464
|
runId: string;
|
|
3247
4465
|
key: string;
|
|
4466
|
+
runAttempt?: number | null;
|
|
4467
|
+
leaseId?: string | null;
|
|
3248
4468
|
output: unknown;
|
|
3249
4469
|
},
|
|
3250
4470
|
): Promise<WorkReceipt | null> {
|
|
@@ -3256,36 +4476,50 @@ export async function skipRuntimeWorkReceipt(
|
|
|
3256
4476
|
context,
|
|
3257
4477
|
session,
|
|
3258
4478
|
async (client) => {
|
|
4479
|
+
const leaseId = input.leaseId?.trim() || null;
|
|
4480
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
3259
4481
|
const { rows } = await client.query<Record<string, unknown>>(
|
|
3260
4482
|
`
|
|
3261
4483
|
WITH skipped AS (
|
|
3262
4484
|
UPDATE ${workReceiptTable(session)}
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
4485
|
+
SET status = $2::smallint,
|
|
4486
|
+
output = $3::jsonb,
|
|
4487
|
+
error = NULL,
|
|
4488
|
+
failure_kind = 0,
|
|
3266
4489
|
run_id = $4,
|
|
4490
|
+
lease_id = NULL,
|
|
4491
|
+
lease_owner_run_id = NULL,
|
|
4492
|
+
lease_owner_attempt = NULL,
|
|
4493
|
+
lease_expires_at = NULL,
|
|
3267
4494
|
updated_at = now()
|
|
3268
4495
|
WHERE k = decode($1, 'hex')
|
|
3269
|
-
AND
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
4496
|
+
AND ${workReceiptRunningOwnerPredicateSql({
|
|
4497
|
+
receiptTable: workReceiptTable(session),
|
|
4498
|
+
ownerRunIdSql: '$4',
|
|
4499
|
+
ownerRunAttemptSql: '$6',
|
|
4500
|
+
leaseIdSql: '$5::text',
|
|
4501
|
+
})}
|
|
4502
|
+
RETURNING convert_from(k, 'UTF8') AS k,
|
|
4503
|
+
status,
|
|
4504
|
+
output,
|
|
4505
|
+
error,
|
|
4506
|
+
failure_kind,
|
|
4507
|
+
run_id,
|
|
4508
|
+
lease_id,
|
|
4509
|
+
lease_owner_run_id,
|
|
4510
|
+
lease_owner_attempt,
|
|
4511
|
+
lease_expires_at,
|
|
4512
|
+
updated_at
|
|
4513
|
+
)
|
|
4514
|
+
SELECT k, status, output, error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at, updated_at FROM skipped
|
|
3282
4515
|
`,
|
|
3283
4516
|
[
|
|
3284
4517
|
workReceiptKeyHex(input.key),
|
|
3285
4518
|
RECEIPT_STATUS_SKIPPED,
|
|
3286
4519
|
input.output === null ? null : stringifyPostgresJson(input.output),
|
|
3287
4520
|
input.runId,
|
|
3288
|
-
|
|
4521
|
+
leaseId,
|
|
4522
|
+
runAttempt,
|
|
3289
4523
|
],
|
|
3290
4524
|
);
|
|
3291
4525
|
return rows[0] ? mapRuntimeWorkReceiptRow(rows[0]) : null;
|
|
@@ -3302,6 +4536,11 @@ export async function startRuntimeSheetDataset(
|
|
|
3302
4536
|
sheetContract: PlaySheetContract;
|
|
3303
4537
|
rows: Record<string, unknown>[];
|
|
3304
4538
|
runId: string;
|
|
4539
|
+
attemptId?: string | null;
|
|
4540
|
+
attemptOwnerRunId?: string | null;
|
|
4541
|
+
attemptExpiresAt?: string | null;
|
|
4542
|
+
attemptSeq?: number | null;
|
|
4543
|
+
attemptLeaseTtlMs?: number | null;
|
|
3305
4544
|
inputOffset?: number;
|
|
3306
4545
|
force?: boolean;
|
|
3307
4546
|
},
|
|
@@ -3378,6 +4617,7 @@ export async function startRuntimeSheetDataset(
|
|
|
3378
4617
|
skipped: input.rows.length,
|
|
3379
4618
|
pendingRows: [],
|
|
3380
4619
|
completedRows: [],
|
|
4620
|
+
blockedRows: [],
|
|
3381
4621
|
tableNamespace: input.tableNamespace,
|
|
3382
4622
|
timings: [
|
|
3383
4623
|
...timings,
|
|
@@ -3426,7 +4666,12 @@ export async function startRuntimeSheetDataset(
|
|
|
3426
4666
|
const normalizedTableNamespace = normalizeTableNamespace(
|
|
3427
4667
|
input.tableNamespace,
|
|
3428
4668
|
);
|
|
3429
|
-
|
|
4669
|
+
const attemptId = normalizeRuntimeSheetAttemptId(input.attemptId);
|
|
4670
|
+
const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
|
|
4671
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
4672
|
+
runId: input.runId,
|
|
4673
|
+
});
|
|
4674
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
3430
4675
|
const chunks = chunkValues(rowEntries, DIRECT_POSTGRES_BATCH_SIZE);
|
|
3431
4676
|
const needsTransaction = chunks.length > 1;
|
|
3432
4677
|
|
|
@@ -3441,12 +4686,26 @@ export async function startRuntimeSheetDataset(
|
|
|
3441
4686
|
timings,
|
|
3442
4687
|
},
|
|
3443
4688
|
async (client) => {
|
|
4689
|
+
const attemptExpiryStartedAt = Date.now();
|
|
4690
|
+
const attemptExpiresAt = await mintRuntimeSheetAttemptExpiresAt(
|
|
4691
|
+
client,
|
|
4692
|
+
input.attemptLeaseTtlMs,
|
|
4693
|
+
);
|
|
4694
|
+
timings.push({
|
|
4695
|
+
phase: 'attempt_expiry_mint',
|
|
4696
|
+
ms: Date.now() - attemptExpiryStartedAt,
|
|
4697
|
+
rows: rowEntries.length,
|
|
4698
|
+
});
|
|
3444
4699
|
const prepareStartedAt = Date.now();
|
|
3445
4700
|
const prepared = await prepareRuntimeSheetDatasetRows(client, session, {
|
|
3446
4701
|
chunks,
|
|
3447
4702
|
runId: input.runId,
|
|
3448
4703
|
normalizedPlayName,
|
|
3449
4704
|
normalizedTableNamespace,
|
|
4705
|
+
attemptId,
|
|
4706
|
+
attemptOwnerRunId,
|
|
4707
|
+
attemptExpiresAt,
|
|
4708
|
+
attemptSeq,
|
|
3450
4709
|
physicalInsertColumnsSql,
|
|
3451
4710
|
physicalInsertValuesSql,
|
|
3452
4711
|
physicalRefreshSetSql:
|
|
@@ -3455,19 +4714,15 @@ export async function startRuntimeSheetDataset(
|
|
|
3455
4714
|
outputPhysicalColumns,
|
|
3456
4715
|
force: input.force === true,
|
|
3457
4716
|
});
|
|
3458
|
-
const effectivePrepared = input.force
|
|
3459
|
-
? {
|
|
3460
|
-
inserted: prepared.inserted,
|
|
3461
|
-
pendingKeys: rowEntries.map((entry) => entry.key),
|
|
3462
|
-
}
|
|
3463
|
-
: prepared;
|
|
3464
4717
|
timings.push({
|
|
3465
4718
|
phase: 'prepare_rows_sql',
|
|
3466
4719
|
ms: Date.now() - prepareStartedAt,
|
|
3467
4720
|
rows: rowEntries.length,
|
|
3468
4721
|
chunks: chunks.length,
|
|
3469
|
-
inserted:
|
|
3470
|
-
pending:
|
|
4722
|
+
inserted: prepared.inserted,
|
|
4723
|
+
pending: prepared.rowDispositions.filter(
|
|
4724
|
+
(row) => row.disposition === 'pending',
|
|
4725
|
+
).length,
|
|
3471
4726
|
...(input.force ? { force: true } : {}),
|
|
3472
4727
|
});
|
|
3473
4728
|
const buildStartedAt = Date.now();
|
|
@@ -3479,9 +4734,13 @@ export async function startRuntimeSheetDataset(
|
|
|
3479
4734
|
normalizedPlayName,
|
|
3480
4735
|
normalizedTableNamespace,
|
|
3481
4736
|
runId: input.runId,
|
|
4737
|
+
attemptId,
|
|
4738
|
+
attemptOwnerRunId,
|
|
4739
|
+
attemptExpiresAt,
|
|
4740
|
+
attemptSeq,
|
|
3482
4741
|
timings,
|
|
3483
4742
|
force: input.force === true,
|
|
3484
|
-
...
|
|
4743
|
+
...prepared,
|
|
3485
4744
|
});
|
|
3486
4745
|
timings.push({
|
|
3487
4746
|
phase: 'build_result',
|
|
@@ -3492,7 +4751,11 @@ export async function startRuntimeSheetDataset(
|
|
|
3492
4751
|
pending: built.pendingRows.length,
|
|
3493
4752
|
completed: built.completedRows.length,
|
|
3494
4753
|
});
|
|
3495
|
-
return
|
|
4754
|
+
return {
|
|
4755
|
+
...built,
|
|
4756
|
+
attemptExpiresAt,
|
|
4757
|
+
attemptSeq,
|
|
4758
|
+
};
|
|
3496
4759
|
},
|
|
3497
4760
|
);
|
|
3498
4761
|
timings.push({
|
|
@@ -3505,7 +4768,298 @@ export async function startRuntimeSheetDataset(
|
|
|
3505
4768
|
pending: result.pendingRows.length,
|
|
3506
4769
|
completed: result.completedRows.length,
|
|
3507
4770
|
});
|
|
3508
|
-
return {
|
|
4771
|
+
return {
|
|
4772
|
+
...result,
|
|
4773
|
+
attemptId,
|
|
4774
|
+
attemptOwnerRunId,
|
|
4775
|
+
attemptExpiresAt: result.attemptExpiresAt,
|
|
4776
|
+
attemptSeq,
|
|
4777
|
+
timings,
|
|
4778
|
+
};
|
|
4779
|
+
}
|
|
4780
|
+
|
|
4781
|
+
export async function heartbeatRuntimeSheetAttempt(
|
|
4782
|
+
context: RuntimeApiContext,
|
|
4783
|
+
input: {
|
|
4784
|
+
playName: string;
|
|
4785
|
+
tableNamespace: string;
|
|
4786
|
+
sheetContract: PlaySheetContract;
|
|
4787
|
+
runId: string;
|
|
4788
|
+
attemptId: string;
|
|
4789
|
+
attemptOwnerRunId?: string | null;
|
|
4790
|
+
attemptSeq?: number | null;
|
|
4791
|
+
attemptLeaseTtlMs?: number | null;
|
|
4792
|
+
keys: string[];
|
|
4793
|
+
},
|
|
4794
|
+
): Promise<RuntimeSheetAttemptHeartbeatResult> {
|
|
4795
|
+
const playName = context.playName?.trim() || input.playName;
|
|
4796
|
+
if (!playName) {
|
|
4797
|
+
throw new Error('Runtime sheet attempt heartbeat requires a playName.');
|
|
4798
|
+
}
|
|
4799
|
+
const attemptId = input.attemptId.trim();
|
|
4800
|
+
const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
|
|
4801
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
4802
|
+
runId: input.runId,
|
|
4803
|
+
});
|
|
4804
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
4805
|
+
const attemptLeaseTtlMs = normalizeRuntimeLeaseTtlMs(
|
|
4806
|
+
input.attemptLeaseTtlMs,
|
|
4807
|
+
PLAY_RUNTIME_SHEET_ATTEMPT_LEASE_TTL_MS,
|
|
4808
|
+
);
|
|
4809
|
+
const keys = [
|
|
4810
|
+
...new Set(input.keys.map((key) => key.trim()).filter(Boolean)),
|
|
4811
|
+
];
|
|
4812
|
+
if (!attemptId || keys.length === 0) {
|
|
4813
|
+
return { renewed: 0, renewedKeys: [], attemptExpiresAt: null };
|
|
4814
|
+
}
|
|
4815
|
+
const timings: RuntimeSheetTiming[] = [];
|
|
4816
|
+
const session = requireRuntimePostgresSession(
|
|
4817
|
+
await getRuntimeDbSession(
|
|
4818
|
+
{
|
|
4819
|
+
...context,
|
|
4820
|
+
playName,
|
|
4821
|
+
runId: context.runId ?? input.runId,
|
|
4822
|
+
},
|
|
4823
|
+
{
|
|
4824
|
+
tableNamespace: input.tableNamespace,
|
|
4825
|
+
logicalTable: 'sheet_rows',
|
|
4826
|
+
operations: ['rows.upsert'],
|
|
4827
|
+
limits: {
|
|
4828
|
+
maxRows: runtimeDbSessionRowLimit(keys.length),
|
|
4829
|
+
},
|
|
4830
|
+
sheetContract: input.sheetContract,
|
|
4831
|
+
timings,
|
|
4832
|
+
},
|
|
4833
|
+
),
|
|
4834
|
+
);
|
|
4835
|
+
return await withRuntimeSheetQueryClient(
|
|
4836
|
+
context,
|
|
4837
|
+
session,
|
|
4838
|
+
{
|
|
4839
|
+
playName,
|
|
4840
|
+
tableNamespace: input.tableNamespace,
|
|
4841
|
+
sheetContract: input.sheetContract,
|
|
4842
|
+
transactional: false,
|
|
4843
|
+
timings,
|
|
4844
|
+
},
|
|
4845
|
+
async (client) => {
|
|
4846
|
+
const { rows } = await client.query<{
|
|
4847
|
+
renewed_keys: string[];
|
|
4848
|
+
renewed: number;
|
|
4849
|
+
attempt_expires_at: Date | string | null;
|
|
4850
|
+
}>(
|
|
4851
|
+
`
|
|
4852
|
+
WITH input_keys AS (
|
|
4853
|
+
SELECT key_values._key
|
|
4854
|
+
FROM unnest($1::text[]) AS key_values(_key)
|
|
4855
|
+
),
|
|
4856
|
+
owned_terminal AS (
|
|
4857
|
+
SELECT target._key
|
|
4858
|
+
FROM ${sheetTable(session)} AS target
|
|
4859
|
+
JOIN input_keys ON target._key = input_keys._key
|
|
4860
|
+
WHERE target._attempt_id = $2::text
|
|
4861
|
+
AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
|
|
4862
|
+
AND target._run_id = $4::text
|
|
4863
|
+
AND COALESCE(target._attempt_seq, 0) = $6::integer
|
|
4864
|
+
AND target._status = 'enriched'
|
|
4865
|
+
),
|
|
4866
|
+
renewed AS (
|
|
4867
|
+
UPDATE ${sheetTable(session)} AS target
|
|
4868
|
+
SET _attempt_expires_at = now() + ($5::double precision * interval '1 millisecond'),
|
|
4869
|
+
_updated_at = now(),
|
|
4870
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
4871
|
+
FROM input_keys
|
|
4872
|
+
WHERE target._key = input_keys._key
|
|
4873
|
+
AND target._attempt_id = $2::text
|
|
4874
|
+
AND coalesce(target._attempt_owner_run_id, target._run_id) = $3::text
|
|
4875
|
+
AND target._run_id = $4::text
|
|
4876
|
+
AND COALESCE(target._attempt_seq, 0) = $6::integer
|
|
4877
|
+
AND target._status IN ('pending', 'running', 'failed')
|
|
4878
|
+
RETURNING target._key, target._attempt_expires_at
|
|
4879
|
+
),
|
|
4880
|
+
owned_keys AS (
|
|
4881
|
+
SELECT _key FROM renewed
|
|
4882
|
+
UNION
|
|
4883
|
+
SELECT _key FROM owned_terminal
|
|
4884
|
+
)
|
|
4885
|
+
SELECT coalesce((SELECT array_agg(_key ORDER BY _key) FROM owned_keys), '{}'::text[]) AS renewed_keys,
|
|
4886
|
+
count(*)::int AS renewed,
|
|
4887
|
+
max(_attempt_expires_at) AS attempt_expires_at
|
|
4888
|
+
FROM renewed
|
|
4889
|
+
`,
|
|
4890
|
+
[
|
|
4891
|
+
keys,
|
|
4892
|
+
attemptId,
|
|
4893
|
+
attemptOwnerRunId,
|
|
4894
|
+
input.runId,
|
|
4895
|
+
attemptLeaseTtlMs,
|
|
4896
|
+
attemptSeq,
|
|
4897
|
+
],
|
|
4898
|
+
);
|
|
4899
|
+
const row = rows[0];
|
|
4900
|
+
const attemptExpiresAt = row?.attempt_expires_at;
|
|
4901
|
+
return {
|
|
4902
|
+
renewed: Number(row?.renewed ?? 0),
|
|
4903
|
+
renewedKeys: row?.renewed_keys ?? [],
|
|
4904
|
+
attemptExpiresAt:
|
|
4905
|
+
attemptExpiresAt instanceof Date
|
|
4906
|
+
? attemptExpiresAt.toISOString()
|
|
4907
|
+
: typeof attemptExpiresAt === 'string'
|
|
4908
|
+
? new Date(attemptExpiresAt).toISOString()
|
|
4909
|
+
: null,
|
|
4910
|
+
};
|
|
4911
|
+
},
|
|
4912
|
+
);
|
|
4913
|
+
}
|
|
4914
|
+
|
|
4915
|
+
/**
|
|
4916
|
+
* Controlled run-fatal teardown for runtime sheet rows: clear the attempt lease
|
|
4917
|
+
* (`_attempt_*`) on every row the aborting attempt still owns, so an immediate
|
|
4918
|
+
* rerun can claim pending rows and overwrite the aborted attempt's rows without
|
|
4919
|
+
* waiting out the 10-minute lease TTL. The predicate is attempt-fenced (see
|
|
4920
|
+
* `releasableRuntimeSheetAttemptFenceSql` / `decideRuntimeSheetRelease`): it only
|
|
4921
|
+
* touches rows owned by this run's attempt generation, so it can never weaken a
|
|
4922
|
+
* different run's live lease. TTL expiry remains the recovery path for true
|
|
4923
|
+
* crashes where no teardown code runs. When `attemptId` is null every attempt id
|
|
4924
|
+
* the run holds at `attemptSeq` is released (the workers runner mints a distinct
|
|
4925
|
+
* attempt id per map chunk under one owner run id + seq).
|
|
4926
|
+
*/
|
|
4927
|
+
export async function releaseRuntimeSheetAttempt(
|
|
4928
|
+
context: RuntimeApiContext,
|
|
4929
|
+
input: {
|
|
4930
|
+
playName: string;
|
|
4931
|
+
tableNamespace: string;
|
|
4932
|
+
sheetContract: PlaySheetContract;
|
|
4933
|
+
runId: string;
|
|
4934
|
+
attemptId?: string | null;
|
|
4935
|
+
attemptOwnerRunId?: string | null;
|
|
4936
|
+
attemptSeq?: number | null;
|
|
4937
|
+
},
|
|
4938
|
+
): Promise<RuntimeSheetAttemptReleaseResult> {
|
|
4939
|
+
const playName = context.playName?.trim() || input.playName;
|
|
4940
|
+
if (!playName) {
|
|
4941
|
+
throw new Error('Runtime sheet attempt release requires a playName.');
|
|
4942
|
+
}
|
|
4943
|
+
const attemptId = input.attemptId?.trim() || null;
|
|
4944
|
+
const attemptOwnerRunId = normalizeRuntimeSheetAttemptOwnerRunId({
|
|
4945
|
+
attemptOwnerRunId: input.attemptOwnerRunId,
|
|
4946
|
+
runId: input.runId,
|
|
4947
|
+
});
|
|
4948
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
4949
|
+
if (!attemptOwnerRunId) {
|
|
4950
|
+
return { released: 0, releasedKeys: [] };
|
|
4951
|
+
}
|
|
4952
|
+
const timings: RuntimeSheetTiming[] = [];
|
|
4953
|
+
const session = requireRuntimePostgresSession(
|
|
4954
|
+
await getRuntimeDbSession(
|
|
4955
|
+
{
|
|
4956
|
+
...context,
|
|
4957
|
+
playName,
|
|
4958
|
+
runId: context.runId ?? input.runId,
|
|
4959
|
+
},
|
|
4960
|
+
{
|
|
4961
|
+
tableNamespace: input.tableNamespace,
|
|
4962
|
+
logicalTable: 'sheet_rows',
|
|
4963
|
+
operations: ['rows.upsert'],
|
|
4964
|
+
limits: {},
|
|
4965
|
+
sheetContract: input.sheetContract,
|
|
4966
|
+
timings,
|
|
4967
|
+
},
|
|
4968
|
+
),
|
|
4969
|
+
);
|
|
4970
|
+
return await withRuntimeSheetQueryClient(
|
|
4971
|
+
context,
|
|
4972
|
+
session,
|
|
4973
|
+
{
|
|
4974
|
+
playName,
|
|
4975
|
+
tableNamespace: input.tableNamespace,
|
|
4976
|
+
sheetContract: input.sheetContract,
|
|
4977
|
+
transactional: false,
|
|
4978
|
+
timings,
|
|
4979
|
+
},
|
|
4980
|
+
async (client) => {
|
|
4981
|
+
const { rows } = await client.query<{ _key: string }>(
|
|
4982
|
+
`
|
|
4983
|
+
UPDATE ${sheetTable(session)} AS target
|
|
4984
|
+
SET _attempt_id = NULL,
|
|
4985
|
+
_attempt_owner_run_id = NULL,
|
|
4986
|
+
_attempt_expires_at = NULL,
|
|
4987
|
+
_attempt_seq = NULL,
|
|
4988
|
+
_updated_at = now(),
|
|
4989
|
+
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
4990
|
+
WHERE ${releasableRuntimeSheetAttemptFenceSql(
|
|
4991
|
+
'target',
|
|
4992
|
+
'$1::text',
|
|
4993
|
+
'$2::text',
|
|
4994
|
+
'$3',
|
|
4995
|
+
)}
|
|
4996
|
+
RETURNING target._key
|
|
4997
|
+
`,
|
|
4998
|
+
[attemptId, attemptOwnerRunId, attemptSeq],
|
|
4999
|
+
);
|
|
5000
|
+
const releasedKeys = rows
|
|
5001
|
+
.map((row) => row._key)
|
|
5002
|
+
.filter((key): key is string => typeof key === 'string');
|
|
5003
|
+
return { released: releasedKeys.length, releasedKeys };
|
|
5004
|
+
},
|
|
5005
|
+
);
|
|
5006
|
+
}
|
|
5007
|
+
|
|
5008
|
+
/**
|
|
5009
|
+
* Controlled run-fatal teardown for work receipts: return every receipt still
|
|
5010
|
+
* leased `running` by the aborting run+attempt back to `pending` (clearing the
|
|
5011
|
+
* lease) so an immediate rerun can reclaim and recompute the interrupted work
|
|
5012
|
+
* without waiting out the lease TTL. Fenced by owner run id + attempt (see
|
|
5013
|
+
* `workReceiptReleaseOwnerPredicateSql`), so it never touches a receipt a live
|
|
5014
|
+
* competitor legitimately holds. Completed/skipped/failed receipts are terminal
|
|
5015
|
+
* and untouched, preserving cross-run idempotency (already-persisted work is
|
|
5016
|
+
* still reused on rerun). Work receipts for a play share one table, so a single
|
|
5017
|
+
* run-scoped statement covers every stranded receipt.
|
|
5018
|
+
*/
|
|
5019
|
+
export async function releaseRuntimeWorkReceipts(
|
|
5020
|
+
context: RuntimeApiContext,
|
|
5021
|
+
input: {
|
|
5022
|
+
playName: string;
|
|
5023
|
+
runId: string;
|
|
5024
|
+
runAttempt?: number | null;
|
|
5025
|
+
},
|
|
5026
|
+
): Promise<RuntimeWorkReceiptReleaseResult> {
|
|
5027
|
+
const runId = input.runId.trim();
|
|
5028
|
+
if (!runId) {
|
|
5029
|
+
return { released: 0, releasedKeys: [] };
|
|
5030
|
+
}
|
|
5031
|
+
const runAttempt = normalizeRuntimeRunAttempt(input.runAttempt);
|
|
5032
|
+
const session = await getRuntimeWorkReceiptSessionForRun(context, {
|
|
5033
|
+
playName: input.playName,
|
|
5034
|
+
});
|
|
5035
|
+
return await withRuntimeWorkReceiptClient(
|
|
5036
|
+
context,
|
|
5037
|
+
session,
|
|
5038
|
+
async (client) => {
|
|
5039
|
+
const { rows } = await client.query<{ k: string }>(
|
|
5040
|
+
`
|
|
5041
|
+
UPDATE ${workReceiptTable(session)} AS target
|
|
5042
|
+
SET status = $3::smallint,
|
|
5043
|
+
lease_id = NULL,
|
|
5044
|
+
lease_owner_run_id = NULL,
|
|
5045
|
+
lease_owner_attempt = NULL,
|
|
5046
|
+
lease_expires_at = NULL,
|
|
5047
|
+
updated_at = now()
|
|
5048
|
+
WHERE ${workReceiptReleaseOwnerPredicateSql({
|
|
5049
|
+
receiptTable: 'target',
|
|
5050
|
+
ownerRunIdSql: '$1',
|
|
5051
|
+
ownerRunAttemptSql: '$2',
|
|
5052
|
+
})}
|
|
5053
|
+
RETURNING convert_from(target.k, 'UTF8') AS k
|
|
5054
|
+
`,
|
|
5055
|
+
[runId, runAttempt, RECEIPT_STATUS_PENDING],
|
|
5056
|
+
);
|
|
5057
|
+
const releasedKeys = rows
|
|
5058
|
+
.map((row) => row.k)
|
|
5059
|
+
.filter((key): key is string => typeof key === 'string');
|
|
5060
|
+
return { released: releasedKeys.length, releasedKeys };
|
|
5061
|
+
},
|
|
5062
|
+
);
|
|
3509
5063
|
}
|
|
3510
5064
|
|
|
3511
5065
|
type CompleteRuntimeMapRowChunksInput = {
|
|
@@ -3513,17 +5067,41 @@ type CompleteRuntimeMapRowChunksInput = {
|
|
|
3513
5067
|
physicalUpdateSetSql: string;
|
|
3514
5068
|
physicalColumnProjections: PhysicalSheetColumnProjection[];
|
|
3515
5069
|
runId: string;
|
|
5070
|
+
attemptId: string | null;
|
|
5071
|
+
attemptOwnerRunId: string | null;
|
|
5072
|
+
attemptExpiresAt: string | null;
|
|
5073
|
+
attemptSeq: number;
|
|
3516
5074
|
normalizedPlayName: string;
|
|
3517
5075
|
normalizedTableNamespace: string;
|
|
3518
5076
|
outputFields: string[];
|
|
3519
5077
|
};
|
|
3520
5078
|
|
|
5079
|
+
type RuntimeMapRowsWriteResult = {
|
|
5080
|
+
updated: number;
|
|
5081
|
+
fencedKeys: string[];
|
|
5082
|
+
};
|
|
5083
|
+
|
|
5084
|
+
function writableEnrichedRuntimeSheetAttemptSql(
|
|
5085
|
+
tableAlias: string,
|
|
5086
|
+
attemptOwnerRunIdExpression: string,
|
|
5087
|
+
attemptExpiresAtExpression: string,
|
|
5088
|
+
attemptSeqExpression: string,
|
|
5089
|
+
): string {
|
|
5090
|
+
return `(
|
|
5091
|
+
${tableAlias}._status = 'enriched'
|
|
5092
|
+
AND (
|
|
5093
|
+
COALESCE(${tableAlias}._attempt_seq, 0) < ${attemptSeqExpression}::integer
|
|
5094
|
+
)
|
|
5095
|
+
)`;
|
|
5096
|
+
}
|
|
5097
|
+
|
|
3521
5098
|
async function completeRuntimeMapRowChunks(
|
|
3522
5099
|
client: RuntimeQueryClient,
|
|
3523
5100
|
session: RuntimePostgresSession,
|
|
3524
5101
|
input: CompleteRuntimeMapRowChunksInput,
|
|
3525
|
-
): Promise<
|
|
5102
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
3526
5103
|
let updated = 0;
|
|
5104
|
+
const fencedKeys: string[] = [];
|
|
3527
5105
|
for (const chunk of input.chunks) {
|
|
3528
5106
|
const chunkKeys = chunk.map((row) => row.key);
|
|
3529
5107
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -3538,9 +5116,21 @@ async function completeRuntimeMapRowChunks(
|
|
|
3538
5116
|
'updates.data_patch',
|
|
3539
5117
|
input.physicalColumnProjections,
|
|
3540
5118
|
);
|
|
5119
|
+
const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
|
|
5120
|
+
'target',
|
|
5121
|
+
'$10::text',
|
|
5122
|
+
'$11::timestamptz',
|
|
5123
|
+
'$12',
|
|
5124
|
+
);
|
|
5125
|
+
const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
|
|
5126
|
+
'target',
|
|
5127
|
+
'$10::text',
|
|
5128
|
+
'$12',
|
|
5129
|
+
);
|
|
3541
5130
|
const { rows } = await client.query<{
|
|
3542
5131
|
updated: number;
|
|
3543
5132
|
matched_keys: string[];
|
|
5133
|
+
fenced_keys: string[];
|
|
3544
5134
|
}>(
|
|
3545
5135
|
`WITH updates AS (
|
|
3546
5136
|
SELECT key_values._key,
|
|
@@ -3571,12 +5161,22 @@ async function completeRuntimeMapRowChunks(
|
|
|
3571
5161
|
SET _status = 'enriched',
|
|
3572
5162
|
_run_id = $5::text,
|
|
3573
5163
|
_error = NULL,
|
|
5164
|
+
_attempt_id = $9::text,
|
|
5165
|
+
_attempt_owner_run_id = $10::text,
|
|
5166
|
+
_attempt_expires_at = $11::timestamptz,
|
|
5167
|
+
_attempt_seq = $12::integer,
|
|
3574
5168
|
_updated_at = now(),
|
|
3575
5169
|
_version = ${nextRuntimeSheetVersionExpression(session)},
|
|
3576
5170
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
3577
5171
|
FROM matched_updates AS updates
|
|
3578
5172
|
WHERE target._key = updates.matched_key
|
|
3579
|
-
|
|
5173
|
+
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5174
|
+
AND NOT (${sameOwnerTerminalEpochSql})
|
|
5175
|
+
AND (
|
|
5176
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5177
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5178
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5179
|
+
)
|
|
3580
5180
|
AND (
|
|
3581
5181
|
target._status <> 'enriched'
|
|
3582
5182
|
OR target._run_id IS DISTINCT FROM $5::text
|
|
@@ -3596,6 +5196,23 @@ async function completeRuntimeMapRowChunks(
|
|
|
3596
5196
|
count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
|
|
3597
5197
|
FROM applied_rows
|
|
3598
5198
|
),
|
|
5199
|
+
fenced_rows AS (
|
|
5200
|
+
SELECT updates.matched_key AS _key
|
|
5201
|
+
FROM matched_updates AS updates
|
|
5202
|
+
JOIN ${sheetTable(session)} AS target
|
|
5203
|
+
ON target._key = updates.matched_key
|
|
5204
|
+
WHERE NOT EXISTS (
|
|
5205
|
+
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5206
|
+
)
|
|
5207
|
+
AND (
|
|
5208
|
+
(${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5209
|
+
OR NOT (
|
|
5210
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5211
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5212
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5213
|
+
)
|
|
5214
|
+
)
|
|
5215
|
+
),
|
|
3599
5216
|
summary_counts AS (
|
|
3600
5217
|
SELECT newly_completed,
|
|
3601
5218
|
from_failed,
|
|
@@ -3680,6 +5297,7 @@ async function completeRuntimeMapRowChunks(
|
|
|
3680
5297
|
SELECT
|
|
3681
5298
|
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
3682
5299
|
coalesce((SELECT array_agg(matched_key) FROM matched_updates), '{}'::text[]) AS matched_keys,
|
|
5300
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
|
|
3683
5301
|
(SELECT count(*)::int FROM summary_delta) AS summary_delta_count,
|
|
3684
5302
|
(SELECT count(*)::int FROM column_delta) AS column_delta_count`,
|
|
3685
5303
|
[
|
|
@@ -3691,9 +5309,14 @@ async function completeRuntimeMapRowChunks(
|
|
|
3691
5309
|
input.normalizedPlayName,
|
|
3692
5310
|
input.normalizedTableNamespace,
|
|
3693
5311
|
[...new Set(input.outputFields)],
|
|
5312
|
+
input.attemptId,
|
|
5313
|
+
input.attemptOwnerRunId,
|
|
5314
|
+
input.attemptExpiresAt,
|
|
5315
|
+
input.attemptSeq,
|
|
3694
5316
|
],
|
|
3695
5317
|
);
|
|
3696
5318
|
updated += Number(rows[0]?.updated ?? 0);
|
|
5319
|
+
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
3697
5320
|
|
|
3698
5321
|
const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
|
|
3699
5322
|
if (matchedKeys.size === chunk.length) {
|
|
@@ -3712,16 +5335,18 @@ async function completeRuntimeMapRowChunks(
|
|
|
3712
5335
|
{ ...input, chunks: [repairChunk] },
|
|
3713
5336
|
);
|
|
3714
5337
|
updated += repaired.updated;
|
|
5338
|
+
fencedKeys.push(...repaired.fencedKeys);
|
|
3715
5339
|
}
|
|
3716
|
-
return { updated };
|
|
5340
|
+
return { updated, fencedKeys };
|
|
3717
5341
|
}
|
|
3718
5342
|
|
|
3719
5343
|
async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
3720
5344
|
client: RuntimeQueryClient,
|
|
3721
5345
|
session: RuntimePostgresSession,
|
|
3722
5346
|
input: CompleteRuntimeMapRowChunksInput,
|
|
3723
|
-
): Promise<
|
|
5347
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
3724
5348
|
let updated = 0;
|
|
5349
|
+
const fencedKeys: string[] = [];
|
|
3725
5350
|
for (const chunk of input.chunks) {
|
|
3726
5351
|
const chunkKeys = chunk.map((row) => row.key);
|
|
3727
5352
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -3736,7 +5361,22 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3736
5361
|
'updates.data_patch',
|
|
3737
5362
|
input.physicalColumnProjections,
|
|
3738
5363
|
);
|
|
3739
|
-
const
|
|
5364
|
+
const writableEnrichedAttemptSql = writableEnrichedRuntimeSheetAttemptSql(
|
|
5365
|
+
'target',
|
|
5366
|
+
'$10::text',
|
|
5367
|
+
'$11::timestamptz',
|
|
5368
|
+
'$12',
|
|
5369
|
+
);
|
|
5370
|
+
const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
|
|
5371
|
+
'target',
|
|
5372
|
+
'$10::text',
|
|
5373
|
+
'$12',
|
|
5374
|
+
);
|
|
5375
|
+
const { rows } = await client.query<{
|
|
5376
|
+
_key?: string;
|
|
5377
|
+
fenced_keys?: string[];
|
|
5378
|
+
updated?: number;
|
|
5379
|
+
}>(
|
|
3740
5380
|
`WITH updates AS (
|
|
3741
5381
|
SELECT key_values._key,
|
|
3742
5382
|
input_index_values.input_index,
|
|
@@ -3770,13 +5410,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3770
5410
|
SET _status = 'enriched',
|
|
3771
5411
|
_run_id = $5::text,
|
|
3772
5412
|
_error = NULL,
|
|
5413
|
+
_attempt_id = $9::text,
|
|
5414
|
+
_attempt_owner_run_id = $10::text,
|
|
5415
|
+
_attempt_expires_at = $11::timestamptz,
|
|
5416
|
+
_attempt_seq = $12::integer,
|
|
3773
5417
|
_updated_at = now(),
|
|
3774
5418
|
_version = ${nextRuntimeSheetVersionExpression(session)},
|
|
3775
5419
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
3776
5420
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
3777
5421
|
WHERE target._key = updates.matched_key
|
|
3778
5422
|
AND prev._key = target._key
|
|
3779
|
-
|
|
5423
|
+
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5424
|
+
AND NOT (${sameOwnerTerminalEpochSql})
|
|
5425
|
+
AND (
|
|
5426
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5427
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5428
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5429
|
+
)
|
|
3780
5430
|
AND (
|
|
3781
5431
|
target._status <> 'enriched'
|
|
3782
5432
|
OR target._run_id IS DISTINCT FROM $5::text
|
|
@@ -3796,6 +5446,23 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3796
5446
|
count(*) FILTER (WHERE prev_status <> 'enriched')::bigint AS newly_completed
|
|
3797
5447
|
FROM applied_rows
|
|
3798
5448
|
),
|
|
5449
|
+
fenced_rows AS (
|
|
5450
|
+
SELECT updates.matched_key AS _key
|
|
5451
|
+
FROM matched_updates AS updates
|
|
5452
|
+
JOIN ${sheetTable(session)} AS target
|
|
5453
|
+
ON target._key = updates.matched_key
|
|
5454
|
+
WHERE NOT EXISTS (
|
|
5455
|
+
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5456
|
+
)
|
|
5457
|
+
AND (
|
|
5458
|
+
(${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
5459
|
+
OR NOT (
|
|
5460
|
+
($9::text IS NULL AND target._run_id = $5::text)
|
|
5461
|
+
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
5462
|
+
OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
|
|
5463
|
+
)
|
|
5464
|
+
)
|
|
5465
|
+
),
|
|
3799
5466
|
summary_counts AS (
|
|
3800
5467
|
SELECT newly_completed,
|
|
3801
5468
|
from_failed,
|
|
@@ -3877,7 +5544,9 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3877
5544
|
_updated_at = now()
|
|
3878
5545
|
RETURNING 1
|
|
3879
5546
|
)
|
|
3880
|
-
SELECT
|
|
5547
|
+
SELECT
|
|
5548
|
+
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
5549
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
|
|
3881
5550
|
[
|
|
3882
5551
|
chunkKeys,
|
|
3883
5552
|
chunkInputIndexes,
|
|
@@ -3887,11 +5556,16 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
3887
5556
|
input.normalizedPlayName,
|
|
3888
5557
|
input.normalizedTableNamespace,
|
|
3889
5558
|
[...new Set(input.outputFields)],
|
|
5559
|
+
input.attemptId,
|
|
5560
|
+
input.attemptOwnerRunId,
|
|
5561
|
+
input.attemptExpiresAt,
|
|
5562
|
+
input.attemptSeq,
|
|
3890
5563
|
],
|
|
3891
5564
|
);
|
|
3892
|
-
updated +=
|
|
5565
|
+
updated += Number(rows[0]?.updated ?? 0);
|
|
5566
|
+
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
3893
5567
|
}
|
|
3894
|
-
return { updated };
|
|
5568
|
+
return { updated, fencedKeys };
|
|
3895
5569
|
}
|
|
3896
5570
|
|
|
3897
5571
|
async function insertMissingCompletedMapRowChunks(
|
|
@@ -3954,8 +5628,26 @@ async function insertMissingCompletedMapRowChunks(
|
|
|
3954
5628
|
)
|
|
3955
5629
|
),
|
|
3956
5630
|
inserted_rows AS (
|
|
3957
|
-
INSERT INTO ${sheetTable(session)} (
|
|
3958
|
-
|
|
5631
|
+
INSERT INTO ${sheetTable(session)} (
|
|
5632
|
+
_key,
|
|
5633
|
+
_status,
|
|
5634
|
+
_run_id,
|
|
5635
|
+
_input_index,
|
|
5636
|
+
_attempt_id,
|
|
5637
|
+
_attempt_owner_run_id,
|
|
5638
|
+
_attempt_expires_at,
|
|
5639
|
+
_attempt_seq,
|
|
5640
|
+
_cell_meta${physicalInsertColumnsSql}
|
|
5641
|
+
)
|
|
5642
|
+
SELECT _key,
|
|
5643
|
+
'enriched',
|
|
5644
|
+
$5::text,
|
|
5645
|
+
input_index,
|
|
5646
|
+
$8::text,
|
|
5647
|
+
$9::text,
|
|
5648
|
+
$10::timestamptz,
|
|
5649
|
+
$11::integer,
|
|
5650
|
+
cell_meta_patch${physicalInsertValuesSql}
|
|
3959
5651
|
FROM missing_rows
|
|
3960
5652
|
ON CONFLICT (_key) DO NOTHING
|
|
3961
5653
|
RETURNING _key
|
|
@@ -3997,7 +5689,7 @@ async function insertMissingCompletedMapRowChunks(
|
|
|
3997
5689
|
),
|
|
3998
5690
|
completed_cell_delta AS (
|
|
3999
5691
|
SELECT field_values.field, (SELECT c FROM inserted_count) AS c
|
|
4000
|
-
FROM unnest($
|
|
5692
|
+
FROM unnest($12::text[]) AS field_values(field)
|
|
4001
5693
|
WHERE (SELECT c FROM inserted_count) > 0
|
|
4002
5694
|
),
|
|
4003
5695
|
column_delta AS (
|
|
@@ -4028,6 +5720,10 @@ async function insertMissingCompletedMapRowChunks(
|
|
|
4028
5720
|
input.runId,
|
|
4029
5721
|
input.normalizedPlayName,
|
|
4030
5722
|
input.normalizedTableNamespace,
|
|
5723
|
+
input.attemptId,
|
|
5724
|
+
input.attemptOwnerRunId,
|
|
5725
|
+
input.attemptExpiresAt,
|
|
5726
|
+
input.attemptSeq,
|
|
4031
5727
|
[...new Set(input.outputFields)],
|
|
4032
5728
|
],
|
|
4033
5729
|
);
|
|
@@ -4053,12 +5749,18 @@ async function failRuntimeMapRowChunks(
|
|
|
4053
5749
|
input: {
|
|
4054
5750
|
chunks: RuntimePreparedFailedRow[][];
|
|
4055
5751
|
physicalUpdateSetSql: string;
|
|
5752
|
+
forceTerminal?: boolean;
|
|
4056
5753
|
runId: string;
|
|
5754
|
+
attemptId: string | null;
|
|
5755
|
+
attemptOwnerRunId: string | null;
|
|
5756
|
+
attemptExpiresAt: string | null;
|
|
5757
|
+
attemptSeq: number;
|
|
4057
5758
|
normalizedPlayName: string;
|
|
4058
5759
|
normalizedTableNamespace: string;
|
|
4059
5760
|
},
|
|
4060
|
-
): Promise<
|
|
5761
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
4061
5762
|
let updated = 0;
|
|
5763
|
+
const fencedKeys: string[] = [];
|
|
4062
5764
|
for (const chunk of input.chunks) {
|
|
4063
5765
|
const chunkKeys = chunk.map((row) => row.key);
|
|
4064
5766
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -4087,7 +5789,10 @@ async function failRuntimeMapRowChunks(
|
|
|
4087
5789
|
const failedCellTotals = failedCellFields.map(
|
|
4088
5790
|
(field) => failedCellCounts.get(field) ?? 0,
|
|
4089
5791
|
);
|
|
4090
|
-
const { rows
|
|
5792
|
+
const { rows } = await client.query<{
|
|
5793
|
+
updated?: number;
|
|
5794
|
+
fenced_keys?: string[];
|
|
5795
|
+
}>(
|
|
4091
5796
|
`WITH updates AS (
|
|
4092
5797
|
SELECT key_values._key,
|
|
4093
5798
|
input_index_values.input_index,
|
|
@@ -4125,26 +5830,53 @@ async function failRuntimeMapRowChunks(
|
|
|
4125
5830
|
SET _status = 'failed',
|
|
4126
5831
|
_run_id = $6::text,
|
|
4127
5832
|
_error = updates.error,
|
|
5833
|
+
_attempt_id = $11::text,
|
|
5834
|
+
_attempt_owner_run_id = $12::text,
|
|
5835
|
+
_attempt_expires_at = $13::timestamptz,
|
|
5836
|
+
_attempt_seq = $14::integer,
|
|
4128
5837
|
_updated_at = now(),
|
|
4129
5838
|
_version = ${nextRuntimeSheetVersionExpression(session)},
|
|
4130
5839
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
4131
5840
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
4132
5841
|
WHERE target._key = updates.matched_key
|
|
4133
5842
|
AND prev._key = target._key
|
|
4134
|
-
AND target._status <> 'enriched'
|
|
4135
|
-
|
|
5843
|
+
AND ($15::boolean OR target._status <> 'enriched')
|
|
5844
|
+
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
|
|
5845
|
+
AND (
|
|
5846
|
+
($11::text IS NULL AND target._run_id = $6::text)
|
|
5847
|
+
OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
|
|
5848
|
+
)
|
|
4136
5849
|
RETURNING target._key, prev._status AS prev_status
|
|
4137
5850
|
),
|
|
4138
5851
|
applied_count AS (
|
|
4139
5852
|
SELECT count(*)::bigint AS c,
|
|
4140
5853
|
count(*) FILTER (WHERE prev_status = 'failed')::bigint AS already_failed,
|
|
5854
|
+
count(*) FILTER (WHERE prev_status = 'enriched')::bigint AS from_completed,
|
|
4141
5855
|
count(*) FILTER (WHERE prev_status = 'running')::bigint AS from_running
|
|
4142
5856
|
FROM applied_rows
|
|
4143
5857
|
),
|
|
5858
|
+
fenced_rows AS (
|
|
5859
|
+
SELECT updates.matched_key AS _key
|
|
5860
|
+
FROM matched_updates AS updates
|
|
5861
|
+
JOIN ${sheetTable(session)} AS target
|
|
5862
|
+
ON target._key = updates.matched_key
|
|
5863
|
+
WHERE NOT EXISTS (
|
|
5864
|
+
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
5865
|
+
)
|
|
5866
|
+
AND (
|
|
5867
|
+
(NOT $15::boolean AND target._status = 'enriched')
|
|
5868
|
+
OR (${newerTerminalRuntimeSheetRowSql('target', '$13::timestamptz', '$14::integer', '$11::text', '$12::text')})
|
|
5869
|
+
OR NOT (
|
|
5870
|
+
($11::text IS NULL AND target._run_id = $6::text)
|
|
5871
|
+
OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
|
|
5872
|
+
)
|
|
5873
|
+
)
|
|
5874
|
+
),
|
|
4144
5875
|
summary_counts AS (
|
|
4145
5876
|
SELECT (c - already_failed)::bigint AS newly_failed,
|
|
5877
|
+
from_completed,
|
|
4146
5878
|
from_running,
|
|
4147
|
-
GREATEST(c - already_failed - from_running, 0)::bigint AS from_queued
|
|
5879
|
+
GREATEST(c - already_failed - from_completed - from_running, 0)::bigint AS from_queued
|
|
4148
5880
|
FROM applied_count
|
|
4149
5881
|
),
|
|
4150
5882
|
summary_delta AS (
|
|
@@ -4162,20 +5894,21 @@ async function failRuntimeMapRowChunks(
|
|
|
4162
5894
|
0,
|
|
4163
5895
|
(-from_queued)::int,
|
|
4164
5896
|
(-from_running)::int,
|
|
4165
|
-
|
|
5897
|
+
(-from_completed)::int,
|
|
4166
5898
|
newly_failed::int
|
|
4167
5899
|
FROM summary_counts
|
|
4168
5900
|
WHERE newly_failed > 0
|
|
4169
5901
|
ON CONFLICT (play_name, table_namespace) DO UPDATE SET
|
|
4170
5902
|
queued = GREATEST(target.queued + EXCLUDED.queued, 0),
|
|
4171
5903
|
running = GREATEST(target.running + EXCLUDED.running, 0),
|
|
5904
|
+
completed = GREATEST(target.completed + EXCLUDED.completed, 0),
|
|
4172
5905
|
failed = GREATEST(target.failed + EXCLUDED.failed, 0),
|
|
4173
5906
|
total = ${runtimeSummaryTotalSql({
|
|
4174
5907
|
currentTotal: 'target.total',
|
|
4175
5908
|
totalDelta: 'EXCLUDED.total',
|
|
4176
5909
|
queued: 'GREATEST(target.queued + EXCLUDED.queued, 0)',
|
|
4177
5910
|
running: 'GREATEST(target.running + EXCLUDED.running, 0)',
|
|
4178
|
-
completed: 'target.completed',
|
|
5911
|
+
completed: 'GREATEST(target.completed + EXCLUDED.completed, 0)',
|
|
4179
5912
|
failed: 'GREATEST(target.failed + EXCLUDED.failed, 0)',
|
|
4180
5913
|
})},
|
|
4181
5914
|
_updated_at = now()
|
|
@@ -4198,7 +5931,9 @@ async function failRuntimeMapRowChunks(
|
|
|
4198
5931
|
_updated_at = now()
|
|
4199
5932
|
RETURNING 1
|
|
4200
5933
|
)
|
|
4201
|
-
SELECT
|
|
5934
|
+
SELECT
|
|
5935
|
+
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
5936
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys`,
|
|
4202
5937
|
[
|
|
4203
5938
|
chunkKeys,
|
|
4204
5939
|
chunkInputIndexes,
|
|
@@ -4210,11 +5945,17 @@ async function failRuntimeMapRowChunks(
|
|
|
4210
5945
|
input.normalizedTableNamespace,
|
|
4211
5946
|
failedCellFields,
|
|
4212
5947
|
failedCellTotals,
|
|
5948
|
+
input.attemptId,
|
|
5949
|
+
input.attemptOwnerRunId,
|
|
5950
|
+
input.attemptExpiresAt,
|
|
5951
|
+
input.attemptSeq,
|
|
5952
|
+
input.forceTerminal === true,
|
|
4213
5953
|
],
|
|
4214
5954
|
);
|
|
4215
|
-
updated +=
|
|
5955
|
+
updated += Number(rows[0]?.updated ?? 0);
|
|
5956
|
+
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
4216
5957
|
}
|
|
4217
|
-
return { updated };
|
|
5958
|
+
return { updated, fencedKeys };
|
|
4218
5959
|
}
|
|
4219
5960
|
|
|
4220
5961
|
/**
|
|
@@ -4240,10 +5981,15 @@ export async function completeRuntimeMapRows(
|
|
|
4240
5981
|
rows: RuntimeApiRowRecord[];
|
|
4241
5982
|
outputFields?: string[];
|
|
4242
5983
|
runId: string;
|
|
5984
|
+
attemptId?: string | null;
|
|
5985
|
+
attemptOwnerRunId?: string | null;
|
|
5986
|
+
attemptExpiresAt?: string | null;
|
|
5987
|
+
attemptSeq?: number | null;
|
|
5988
|
+
forceFailedRows?: boolean;
|
|
4243
5989
|
},
|
|
4244
|
-
): Promise<
|
|
5990
|
+
): Promise<RuntimeMapRowsWriteResult> {
|
|
4245
5991
|
if (input.rows.length === 0) {
|
|
4246
|
-
return { updated: 0 };
|
|
5992
|
+
return { updated: 0, fencedKeys: [] };
|
|
4247
5993
|
}
|
|
4248
5994
|
const sheetContract = augmentSheetContractWithDatasetFields({
|
|
4249
5995
|
contract: input.sheetContract,
|
|
@@ -4276,7 +6022,7 @@ export async function completeRuntimeMapRows(
|
|
|
4276
6022
|
uniqueRows.set(row.key, row);
|
|
4277
6023
|
}
|
|
4278
6024
|
if (uniqueRows.size === 0) {
|
|
4279
|
-
return { updated: 0 };
|
|
6025
|
+
return { updated: 0, fencedKeys: [] };
|
|
4280
6026
|
}
|
|
4281
6027
|
|
|
4282
6028
|
const projections = physicalSheetColumnProjections(sheetContract);
|
|
@@ -4304,6 +6050,11 @@ export async function completeRuntimeMapRows(
|
|
|
4304
6050
|
const failedChunks = chunkValues(failedRows, DIRECT_POSTGRES_BATCH_SIZE);
|
|
4305
6051
|
const needsTransaction = chunks.length + failedChunks.length > 1;
|
|
4306
6052
|
const outputFields = [...new Set(input.outputFields ?? [])];
|
|
6053
|
+
const attemptId = input.attemptId?.trim() || null;
|
|
6054
|
+
const attemptOwnerRunId =
|
|
6055
|
+
input.attemptOwnerRunId?.trim() || (attemptId ? input.runId : null);
|
|
6056
|
+
const attemptExpiresAt = input.attemptExpiresAt?.trim() || null;
|
|
6057
|
+
const attemptSeq = normalizeRuntimeRunAttempt(input.attemptSeq);
|
|
4307
6058
|
|
|
4308
6059
|
return await withRuntimeSheetQueryClient(
|
|
4309
6060
|
context,
|
|
@@ -4323,6 +6074,10 @@ export async function completeRuntimeMapRows(
|
|
|
4323
6074
|
physicalUpdateSetSql,
|
|
4324
6075
|
physicalColumnProjections: projections,
|
|
4325
6076
|
runId: input.runId,
|
|
6077
|
+
attemptId,
|
|
6078
|
+
attemptOwnerRunId,
|
|
6079
|
+
attemptExpiresAt,
|
|
6080
|
+
attemptSeq,
|
|
4326
6081
|
normalizedPlayName: normalizePlayNameForSheet(session.playName),
|
|
4327
6082
|
normalizedTableNamespace: normalizeTableNamespace(
|
|
4328
6083
|
input.tableNamespace,
|
|
@@ -4339,22 +6094,33 @@ export async function completeRuntimeMapRows(
|
|
|
4339
6094
|
session,
|
|
4340
6095
|
chunkInput,
|
|
4341
6096
|
);
|
|
4342
|
-
return {
|
|
6097
|
+
return {
|
|
6098
|
+
updated: updated.updated + inserted.inserted,
|
|
6099
|
+
fencedKeys: updated.fencedKeys,
|
|
6100
|
+
};
|
|
4343
6101
|
})()
|
|
4344
|
-
: { updated: 0 };
|
|
6102
|
+
: { updated: 0, fencedKeys: [] };
|
|
4345
6103
|
const failed =
|
|
4346
6104
|
failedRows.length > 0
|
|
4347
6105
|
? await failRuntimeMapRowChunks(client, session, {
|
|
4348
6106
|
chunks: failedChunks,
|
|
4349
6107
|
physicalUpdateSetSql,
|
|
6108
|
+
forceTerminal: input.forceFailedRows === true,
|
|
4350
6109
|
runId: input.runId,
|
|
6110
|
+
attemptId,
|
|
6111
|
+
attemptOwnerRunId,
|
|
6112
|
+
attemptExpiresAt,
|
|
6113
|
+
attemptSeq,
|
|
4351
6114
|
normalizedPlayName: normalizePlayNameForSheet(session.playName),
|
|
4352
6115
|
normalizedTableNamespace: normalizeTableNamespace(
|
|
4353
6116
|
input.tableNamespace,
|
|
4354
6117
|
),
|
|
4355
6118
|
})
|
|
4356
|
-
: { updated: 0 };
|
|
4357
|
-
return {
|
|
6119
|
+
: { updated: 0, fencedKeys: [] };
|
|
6120
|
+
return {
|
|
6121
|
+
updated: completed.updated + failed.updated,
|
|
6122
|
+
fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
|
|
6123
|
+
};
|
|
4358
6124
|
},
|
|
4359
6125
|
);
|
|
4360
6126
|
}
|
|
@@ -4408,7 +6174,7 @@ export async function readRuntimeSheetDatasetRowKeys(
|
|
|
4408
6174
|
tableNamespace: string;
|
|
4409
6175
|
runId: string;
|
|
4410
6176
|
keys: string[];
|
|
4411
|
-
rowMode?: 'output' | 'all';
|
|
6177
|
+
rowMode?: 'output' | 'all' | 'terminalAnyRun';
|
|
4412
6178
|
},
|
|
4413
6179
|
): Promise<{ keys: string[] }> {
|
|
4414
6180
|
const keys = [
|
|
@@ -4435,6 +6201,16 @@ export async function readRuntimeSheetDatasetRowKeys(
|
|
|
4435
6201
|
),
|
|
4436
6202
|
);
|
|
4437
6203
|
const rows = await withRuntimePostgres(session, async (client) => {
|
|
6204
|
+
if (input.rowMode === 'terminalAnyRun') {
|
|
6205
|
+
const { rows: matchedRows } = await client.query<{ _key: string }>(
|
|
6206
|
+
`SELECT _key
|
|
6207
|
+
FROM ${sheetTable(session)}
|
|
6208
|
+
WHERE _key = ANY($1::text[])
|
|
6209
|
+
AND _status IN ('enriched', 'failed')`,
|
|
6210
|
+
[keys],
|
|
6211
|
+
);
|
|
6212
|
+
return matchedRows;
|
|
6213
|
+
}
|
|
4438
6214
|
if (input.rowMode === 'all') {
|
|
4439
6215
|
const { rows: matchedRows } = await client.query<{ _key: string }>(
|
|
4440
6216
|
`SELECT _key
|