deepline 0.1.295 → 0.1.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +237 -167
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +3 -94
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +191 -22
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +662 -0
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
|
@@ -30,10 +30,7 @@ export type RuntimeResourceSnapshot = {
|
|
|
30
30
|
};
|
|
31
31
|
sheetFlush: {
|
|
32
32
|
acquired: number;
|
|
33
|
-
active: number;
|
|
34
|
-
waiting: number;
|
|
35
33
|
bytes: number;
|
|
36
|
-
admissionWaitMsEwma: number | null;
|
|
37
34
|
latencyMsEwma: number | null;
|
|
38
35
|
};
|
|
39
36
|
providers: Record<
|
|
@@ -62,11 +59,6 @@ export interface RuntimeResourceGovernor {
|
|
|
62
59
|
toolId: string;
|
|
63
60
|
signal?: AbortSignal;
|
|
64
61
|
}): Promise<RuntimeResourceLease>;
|
|
65
|
-
acquireSheetFlush(input?: {
|
|
66
|
-
estimatedBytes?: number | null;
|
|
67
|
-
rowCount?: number | null;
|
|
68
|
-
signal?: AbortSignal;
|
|
69
|
-
}): Promise<RuntimeResourceLease>;
|
|
70
62
|
suggestedToolParallelism(toolId: string, fallback: number): Promise<number>;
|
|
71
63
|
resolveRowConcurrency(requested?: number): number;
|
|
72
64
|
reportProviderBackpressure(input: {
|
|
@@ -98,76 +90,6 @@ function createEwma(alpha = 0.2): Ewma {
|
|
|
98
90
|
};
|
|
99
91
|
}
|
|
100
92
|
|
|
101
|
-
// Sheet persistence has its own local admission lane. Receipt lifecycle calls
|
|
102
|
-
// are admitted by the receipt sink/gateway and must not queue behind a large
|
|
103
|
-
// Runtime Sheet flush in the runner. Two flushes keep the database busy while
|
|
104
|
-
// bounding resident payloads and gateway/pool pressure (ADR 0012).
|
|
105
|
-
const MAX_CONCURRENT_SHEET_FLUSHES = 2;
|
|
106
|
-
|
|
107
|
-
type SheetFlushWaiter = {
|
|
108
|
-
resolve: (lease: RuntimeResourceLease) => void;
|
|
109
|
-
reject: (error: unknown) => void;
|
|
110
|
-
signal?: AbortSignal;
|
|
111
|
-
onAbort?: () => void;
|
|
112
|
-
};
|
|
113
|
-
|
|
114
|
-
class SheetFlushAdmission {
|
|
115
|
-
#active = 0;
|
|
116
|
-
#waiters: SheetFlushWaiter[] = [];
|
|
117
|
-
|
|
118
|
-
get active(): number {
|
|
119
|
-
return this.#active;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
get waiting(): number {
|
|
123
|
-
return this.#waiters.length;
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
acquire(signal?: AbortSignal): Promise<RuntimeResourceLease> {
|
|
127
|
-
if (signal?.aborted) {
|
|
128
|
-
return Promise.reject(signal.reason ?? new Error('Sheet flush aborted'));
|
|
129
|
-
}
|
|
130
|
-
if (this.#active < MAX_CONCURRENT_SHEET_FLUSHES) {
|
|
131
|
-
this.#active += 1;
|
|
132
|
-
return Promise.resolve(this.#lease());
|
|
133
|
-
}
|
|
134
|
-
return new Promise((resolve, reject) => {
|
|
135
|
-
const waiter: SheetFlushWaiter = {
|
|
136
|
-
resolve,
|
|
137
|
-
reject,
|
|
138
|
-
...(signal ? { signal } : {}),
|
|
139
|
-
};
|
|
140
|
-
if (signal) {
|
|
141
|
-
waiter.onAbort = () => {
|
|
142
|
-
const index = this.#waiters.indexOf(waiter);
|
|
143
|
-
if (index >= 0) this.#waiters.splice(index, 1);
|
|
144
|
-
reject(signal.reason ?? new Error('Sheet flush aborted'));
|
|
145
|
-
};
|
|
146
|
-
signal.addEventListener('abort', waiter.onAbort, { once: true });
|
|
147
|
-
}
|
|
148
|
-
this.#waiters.push(waiter);
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
#lease(): RuntimeResourceLease {
|
|
153
|
-
let released = false;
|
|
154
|
-
return {
|
|
155
|
-
release: () => {
|
|
156
|
-
if (released) return;
|
|
157
|
-
released = true;
|
|
158
|
-
while (this.#waiters.length > 0) {
|
|
159
|
-
const waiter = this.#waiters.shift()!;
|
|
160
|
-
waiter.signal?.removeEventListener('abort', waiter.onAbort!);
|
|
161
|
-
if (waiter.signal?.aborted) continue;
|
|
162
|
-
waiter.resolve(this.#lease());
|
|
163
|
-
return;
|
|
164
|
-
}
|
|
165
|
-
this.#active = Math.max(0, this.#active - 1);
|
|
166
|
-
},
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
93
|
function elapsedSince(startedAt: number): number {
|
|
172
94
|
return Math.max(0, Date.now() - startedAt);
|
|
173
95
|
}
|
|
@@ -183,8 +105,6 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
183
105
|
const rowAdmissionWait = createEwma();
|
|
184
106
|
const toolAdmissionWait = createEwma();
|
|
185
107
|
const sheetFlushLatency = createEwma();
|
|
186
|
-
const sheetFlushAdmissionWait = createEwma();
|
|
187
|
-
const sheetFlushAdmission = new SheetFlushAdmission();
|
|
188
108
|
const providers = new Map<
|
|
189
109
|
string,
|
|
190
110
|
{
|
|
@@ -223,6 +143,9 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
223
143
|
rowAdmissionWait.observe(observation.rowAdmissionWaitMs);
|
|
224
144
|
toolAdmissionWait.observe(observation.toolAdmissionWaitMs);
|
|
225
145
|
sheetFlushLatency.observe(observation.sheetFlushLatencyMs);
|
|
146
|
+
if (observation.sheetFlushLatencyMs != null) {
|
|
147
|
+
sheetFlushCounters.acquired += 1;
|
|
148
|
+
}
|
|
226
149
|
if (observation.rowEstimatedBytes != null) {
|
|
227
150
|
rowCounters.estimatedBytes += Math.max(
|
|
228
151
|
0,
|
|
@@ -301,17 +224,6 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
301
224
|
);
|
|
302
225
|
},
|
|
303
226
|
|
|
304
|
-
async acquireSheetFlush(sheetInput) {
|
|
305
|
-
const startedAt = Date.now();
|
|
306
|
-
const lease = await sheetFlushAdmission.acquire(sheetInput?.signal);
|
|
307
|
-
sheetFlushCounters.acquired += 1;
|
|
308
|
-
sheetFlushAdmissionWait.observe(elapsedSince(startedAt));
|
|
309
|
-
observe({
|
|
310
|
-
sheetFlushBytes: sheetInput?.estimatedBytes ?? null,
|
|
311
|
-
});
|
|
312
|
-
return lease;
|
|
313
|
-
},
|
|
314
|
-
|
|
315
227
|
suggestedToolParallelism(toolId, fallback) {
|
|
316
228
|
return input.executionGovernor.suggestedParallelism(toolId, fallback);
|
|
317
229
|
},
|
|
@@ -344,10 +256,7 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
344
256
|
},
|
|
345
257
|
sheetFlush: {
|
|
346
258
|
acquired: sheetFlushCounters.acquired,
|
|
347
|
-
active: sheetFlushAdmission.active,
|
|
348
|
-
waiting: sheetFlushAdmission.waiting,
|
|
349
259
|
bytes: sheetFlushCounters.bytes,
|
|
350
|
-
admissionWaitMsEwma: sheetFlushAdmissionWait.value,
|
|
351
260
|
latencyMsEwma: sheetFlushLatency.value,
|
|
352
261
|
},
|
|
353
262
|
providers: Object.fromEntries(
|
|
@@ -6456,12 +6456,44 @@ type CompleteRuntimeMapRowChunksInput = {
|
|
|
6456
6456
|
};
|
|
6457
6457
|
|
|
6458
6458
|
export type RuntimeMapRowsWriteResult = {
|
|
6459
|
+
/** Submitted row identities durably accepted, including idempotent replays. */
|
|
6460
|
+
committedKeys: string[];
|
|
6461
|
+
/** Submitted row identities rejected because a newer write owns the row. */
|
|
6462
|
+
staleKeys: string[];
|
|
6463
|
+
/** Physical row mutations. Idempotent commits do not increment this count. */
|
|
6459
6464
|
updated: number;
|
|
6465
|
+
/** Compatibility alias retained for callers that still call stale rows fenced. */
|
|
6460
6466
|
fencedKeys: string[];
|
|
6461
6467
|
staleDropped?: number;
|
|
6462
6468
|
staleDroppedKeys?: string[];
|
|
6463
6469
|
};
|
|
6464
6470
|
|
|
6471
|
+
type RuntimeMapRowsMutationResult = {
|
|
6472
|
+
updated: number;
|
|
6473
|
+
fencedKeys: string[];
|
|
6474
|
+
conflictKeys: string[];
|
|
6475
|
+
};
|
|
6476
|
+
|
|
6477
|
+
function classifyRuntimeMapRowsWrite(input: {
|
|
6478
|
+
submittedKeys: Iterable<string>;
|
|
6479
|
+
updated: number;
|
|
6480
|
+
fencedKeys: Iterable<string>;
|
|
6481
|
+
}): RuntimeMapRowsWriteResult {
|
|
6482
|
+
const staleKeys = [...new Set(input.fencedKeys)];
|
|
6483
|
+
const staleSet = new Set(staleKeys);
|
|
6484
|
+
const committedKeys = [...new Set(input.submittedKeys)].filter(
|
|
6485
|
+
(key) => !staleSet.has(key),
|
|
6486
|
+
);
|
|
6487
|
+
return {
|
|
6488
|
+
committedKeys,
|
|
6489
|
+
staleKeys,
|
|
6490
|
+
updated: input.updated,
|
|
6491
|
+
fencedKeys: staleKeys,
|
|
6492
|
+
staleDropped: staleKeys.length,
|
|
6493
|
+
staleDroppedKeys: staleKeys,
|
|
6494
|
+
};
|
|
6495
|
+
}
|
|
6496
|
+
|
|
6465
6497
|
function writableEnrichedRuntimeSheetAttemptSql(
|
|
6466
6498
|
tableAlias: string,
|
|
6467
6499
|
attemptOwnerRunIdExpression: string,
|
|
@@ -6480,9 +6512,10 @@ async function completeRuntimeMapRowChunks(
|
|
|
6480
6512
|
client: RuntimeQueryClient,
|
|
6481
6513
|
session: RuntimePostgresSession,
|
|
6482
6514
|
input: CompleteRuntimeMapRowChunksInput,
|
|
6483
|
-
): Promise<
|
|
6515
|
+
): Promise<RuntimeMapRowsMutationResult> {
|
|
6484
6516
|
let updated = 0;
|
|
6485
6517
|
const fencedKeys: string[] = [];
|
|
6518
|
+
const conflictKeys: string[] = [];
|
|
6486
6519
|
for (const chunk of input.chunks) {
|
|
6487
6520
|
const chunkKeys = chunk.map((row) => row.key);
|
|
6488
6521
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -6515,6 +6548,7 @@ async function completeRuntimeMapRowChunks(
|
|
|
6515
6548
|
updated: number;
|
|
6516
6549
|
matched_keys: string[];
|
|
6517
6550
|
fenced_keys: string[];
|
|
6551
|
+
conflict_keys: string[];
|
|
6518
6552
|
}>(
|
|
6519
6553
|
`WITH updates AS (
|
|
6520
6554
|
SELECT key_values._key,
|
|
@@ -6540,6 +6574,25 @@ async function completeRuntimeMapRowChunks(
|
|
|
6540
6574
|
JOIN ${sheetTable(session)} AS target
|
|
6541
6575
|
ON target._key = updates._key
|
|
6542
6576
|
),
|
|
6577
|
+
same_version_conflicts AS (
|
|
6578
|
+
SELECT updates.matched_key AS _key
|
|
6579
|
+
FROM matched_updates AS updates
|
|
6580
|
+
JOIN ${sheetTable(session)} AS target
|
|
6581
|
+
ON target._key = updates.matched_key
|
|
6582
|
+
WHERE $13::bigint IS NOT NULL
|
|
6583
|
+
AND target._write_version = $13::bigint
|
|
6584
|
+
AND target._run_id = $5::text
|
|
6585
|
+
AND target._status IN ('enriched', 'failed')
|
|
6586
|
+
AND EXISTS (
|
|
6587
|
+
SELECT 1
|
|
6588
|
+
FROM unnest($8::text[]) AS field_values(field)
|
|
6589
|
+
WHERE target._cell_meta -> field_values.field ->> 'runId' = $5::text
|
|
6590
|
+
)
|
|
6591
|
+
AND (
|
|
6592
|
+
target._status <> 'enriched'
|
|
6593
|
+
OR (${targetChangedPatchedCellSql})
|
|
6594
|
+
)
|
|
6595
|
+
),
|
|
6543
6596
|
applied_rows AS (
|
|
6544
6597
|
UPDATE ${sheetTable(session)} AS target
|
|
6545
6598
|
SET _status = 'enriched',
|
|
@@ -6556,6 +6609,7 @@ async function completeRuntimeMapRowChunks(
|
|
|
6556
6609
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
6557
6610
|
FROM matched_updates AS updates
|
|
6558
6611
|
WHERE target._key = updates.matched_key
|
|
6612
|
+
AND NOT EXISTS (SELECT 1 FROM same_version_conflicts)
|
|
6559
6613
|
AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
|
|
6560
6614
|
AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
|
|
6561
6615
|
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
@@ -6691,6 +6745,7 @@ async function completeRuntimeMapRowChunks(
|
|
|
6691
6745
|
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
6692
6746
|
coalesce((SELECT array_agg(matched_key) FROM matched_updates), '{}'::text[]) AS matched_keys,
|
|
6693
6747
|
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
|
|
6748
|
+
coalesce((SELECT array_agg(_key) FROM same_version_conflicts), '{}'::text[]) AS conflict_keys,
|
|
6694
6749
|
(SELECT count(*)::int FROM summary_delta) AS summary_delta_count,
|
|
6695
6750
|
(SELECT count(*)::int FROM column_delta) AS column_delta_count`,
|
|
6696
6751
|
[
|
|
@@ -6711,12 +6766,18 @@ async function completeRuntimeMapRowChunks(
|
|
|
6711
6766
|
);
|
|
6712
6767
|
const matchedKeys = new Set(rows[0]?.matched_keys ?? []);
|
|
6713
6768
|
const chunkFencedKeys = rows[0]?.fenced_keys ?? [];
|
|
6769
|
+
const chunkConflictKeys = rows[0]?.conflict_keys ?? [];
|
|
6714
6770
|
// An exact-version replay that already produced the same terminal value is
|
|
6715
6771
|
// accepted idempotently even when SQL has no physical mutation to apply.
|
|
6716
6772
|
// Stale versions remain excluded through fenced_keys.
|
|
6717
6773
|
const appliedCount = Number(rows[0]?.updated ?? 0);
|
|
6718
6774
|
updated += appliedCount;
|
|
6719
6775
|
fencedKeys.push(...chunkFencedKeys);
|
|
6776
|
+
conflictKeys.push(...chunkConflictKeys);
|
|
6777
|
+
|
|
6778
|
+
if (chunkConflictKeys.length > 0) {
|
|
6779
|
+
continue;
|
|
6780
|
+
}
|
|
6720
6781
|
|
|
6721
6782
|
if (matchedKeys.size === chunk.length) {
|
|
6722
6783
|
continue;
|
|
@@ -6735,17 +6796,19 @@ async function completeRuntimeMapRowChunks(
|
|
|
6735
6796
|
);
|
|
6736
6797
|
updated += repaired.updated;
|
|
6737
6798
|
fencedKeys.push(...repaired.fencedKeys);
|
|
6799
|
+
conflictKeys.push(...repaired.conflictKeys);
|
|
6738
6800
|
}
|
|
6739
|
-
return { updated, fencedKeys };
|
|
6801
|
+
return { updated, fencedKeys, conflictKeys };
|
|
6740
6802
|
}
|
|
6741
6803
|
|
|
6742
6804
|
async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
6743
6805
|
client: RuntimeQueryClient,
|
|
6744
6806
|
session: RuntimePostgresSession,
|
|
6745
6807
|
input: CompleteRuntimeMapRowChunksInput,
|
|
6746
|
-
): Promise<
|
|
6808
|
+
): Promise<RuntimeMapRowsMutationResult> {
|
|
6747
6809
|
let updated = 0;
|
|
6748
6810
|
const fencedKeys: string[] = [];
|
|
6811
|
+
const conflictKeys: string[] = [];
|
|
6749
6812
|
for (const chunk of input.chunks) {
|
|
6750
6813
|
const chunkKeys = chunk.map((row) => row.key);
|
|
6751
6814
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
@@ -6770,6 +6833,7 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6770
6833
|
const { rows } = await client.query<{
|
|
6771
6834
|
_key?: string;
|
|
6772
6835
|
fenced_keys?: string[];
|
|
6836
|
+
conflict_keys?: string[];
|
|
6773
6837
|
updated?: number;
|
|
6774
6838
|
}>(
|
|
6775
6839
|
`WITH updates AS (
|
|
@@ -6788,6 +6852,7 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6788
6852
|
matched_updates AS (
|
|
6789
6853
|
SELECT DISTINCT ON (target._key)
|
|
6790
6854
|
target._key AS matched_key,
|
|
6855
|
+
updates._key AS submitted_key,
|
|
6791
6856
|
updates.data_patch,
|
|
6792
6857
|
updates.cell_meta_patch
|
|
6793
6858
|
FROM updates
|
|
@@ -6800,10 +6865,31 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6800
6865
|
)
|
|
6801
6866
|
ORDER BY target._key, (target._key = updates._key) DESC
|
|
6802
6867
|
),
|
|
6868
|
+
same_version_conflicts AS (
|
|
6869
|
+
SELECT updates.submitted_key AS _key
|
|
6870
|
+
FROM matched_updates AS updates
|
|
6871
|
+
JOIN ${sheetTable(session)} AS target
|
|
6872
|
+
ON target._key = updates.matched_key
|
|
6873
|
+
WHERE $13::bigint IS NOT NULL
|
|
6874
|
+
AND target._write_version = $13::bigint
|
|
6875
|
+
AND target._run_id = $5::text
|
|
6876
|
+
AND target._status IN ('enriched', 'failed')
|
|
6877
|
+
AND EXISTS (
|
|
6878
|
+
SELECT 1
|
|
6879
|
+
FROM unnest($8::text[]) AS field_values(field)
|
|
6880
|
+
WHERE target._cell_meta -> field_values.field ->> 'runId' = $5::text
|
|
6881
|
+
)
|
|
6882
|
+
AND (
|
|
6883
|
+
target._status <> 'enriched'
|
|
6884
|
+
OR (${targetChangedPatchedCellSql})
|
|
6885
|
+
)
|
|
6886
|
+
),
|
|
6803
6887
|
applied_rows AS (
|
|
6804
6888
|
UPDATE ${sheetTable(session)} AS target
|
|
6805
6889
|
SET _status = 'enriched',
|
|
6806
6890
|
_run_id = $5::text,
|
|
6891
|
+
_writer_run_id = $5::text,
|
|
6892
|
+
_write_version = COALESCE($13::bigint, target._write_version),
|
|
6807
6893
|
_error = NULL,
|
|
6808
6894
|
_attempt_id = $9::text,
|
|
6809
6895
|
_attempt_owner_run_id = $10::text,
|
|
@@ -6815,6 +6901,9 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6815
6901
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
6816
6902
|
WHERE target._key = updates.matched_key
|
|
6817
6903
|
AND prev._key = target._key
|
|
6904
|
+
AND NOT EXISTS (SELECT 1 FROM same_version_conflicts)
|
|
6905
|
+
AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
|
|
6906
|
+
AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
|
|
6818
6907
|
AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
6819
6908
|
AND NOT (${sameOwnerTerminalEpochSql})
|
|
6820
6909
|
AND (
|
|
@@ -6842,15 +6931,17 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6842
6931
|
FROM applied_rows
|
|
6843
6932
|
),
|
|
6844
6933
|
fenced_rows AS (
|
|
6845
|
-
SELECT updates.
|
|
6934
|
+
SELECT updates.submitted_key AS _key
|
|
6846
6935
|
FROM matched_updates AS updates
|
|
6847
6936
|
JOIN ${sheetTable(session)} AS target
|
|
6848
6937
|
ON target._key = updates.matched_key
|
|
6849
6938
|
WHERE NOT EXISTS (
|
|
6850
6939
|
SELECT 1 FROM applied_rows WHERE applied_rows._key = updates.matched_key
|
|
6851
|
-
|
|
6940
|
+
)
|
|
6852
6941
|
AND (
|
|
6853
|
-
($
|
|
6942
|
+
($13::bigint IS NOT NULL AND target._write_version <> $13::bigint)
|
|
6943
|
+
OR ($13::bigint IS NULL AND target._status = 'enriched')
|
|
6944
|
+
OR (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
|
|
6854
6945
|
OR NOT (
|
|
6855
6946
|
($9::text IS NULL AND target._run_id = $5::text)
|
|
6856
6947
|
OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
|
|
@@ -6941,7 +7032,8 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6941
7032
|
)
|
|
6942
7033
|
SELECT
|
|
6943
7034
|
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
6944
|
-
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys
|
|
7035
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
|
|
7036
|
+
coalesce((SELECT array_agg(_key) FROM same_version_conflicts), '{}'::text[]) AS conflict_keys`,
|
|
6945
7037
|
[
|
|
6946
7038
|
chunkKeys,
|
|
6947
7039
|
chunkInputIndexes,
|
|
@@ -6955,12 +7047,14 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
|
|
|
6955
7047
|
input.attemptOwnerRunId,
|
|
6956
7048
|
input.attemptExpiresAt,
|
|
6957
7049
|
input.attemptSeq,
|
|
7050
|
+
input.writeVersion,
|
|
6958
7051
|
],
|
|
6959
7052
|
);
|
|
6960
7053
|
updated += Number(rows[0]?.updated ?? 0);
|
|
6961
7054
|
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
7055
|
+
conflictKeys.push(...(rows[0]?.conflict_keys ?? []));
|
|
6962
7056
|
}
|
|
6963
|
-
return { updated, fencedKeys };
|
|
7057
|
+
return { updated, fencedKeys, conflictKeys };
|
|
6964
7058
|
}
|
|
6965
7059
|
|
|
6966
7060
|
async function insertMissingCompletedMapRowChunks(
|
|
@@ -7140,6 +7234,7 @@ async function failRuntimeMapRowChunks(
|
|
|
7140
7234
|
input: {
|
|
7141
7235
|
chunks: RuntimePreparedFailedRow[][];
|
|
7142
7236
|
physicalUpdateSetSql: string;
|
|
7237
|
+
physicalColumnProjections: PhysicalSheetColumnProjection[];
|
|
7143
7238
|
forceTerminal?: boolean;
|
|
7144
7239
|
runId: string;
|
|
7145
7240
|
attemptId: string | null;
|
|
@@ -7150,15 +7245,21 @@ async function failRuntimeMapRowChunks(
|
|
|
7150
7245
|
normalizedPlayName: string;
|
|
7151
7246
|
normalizedTableNamespace: string;
|
|
7152
7247
|
},
|
|
7153
|
-
): Promise<
|
|
7248
|
+
): Promise<RuntimeMapRowsMutationResult> {
|
|
7154
7249
|
let updated = 0;
|
|
7155
7250
|
const fencedKeys: string[] = [];
|
|
7251
|
+
const conflictKeys: string[] = [];
|
|
7156
7252
|
for (const chunk of input.chunks) {
|
|
7157
7253
|
const chunkKeys = chunk.map((row) => row.key);
|
|
7158
7254
|
const chunkInputIndexes = chunk.map((row) => row.input_index);
|
|
7159
7255
|
const chunkDataPatches = chunk.map((row) => row.data_patch_json);
|
|
7160
7256
|
const chunkCellMetaPatches = chunk.map((row) => row.cell_meta_patch_json);
|
|
7161
7257
|
const chunkErrors = chunk.map((row) => row.error);
|
|
7258
|
+
const targetChangedPatchedCellSql = changedPatchedCellSql(
|
|
7259
|
+
'target',
|
|
7260
|
+
'updates.data_patch',
|
|
7261
|
+
input.physicalColumnProjections,
|
|
7262
|
+
);
|
|
7162
7263
|
// Per-field failed-cell counts for the column summary, computed from the
|
|
7163
7264
|
// cell meta patches (a failed row records exactly which cell failed).
|
|
7164
7265
|
const failedCellCounts = new Map<string, number>();
|
|
@@ -7180,6 +7281,7 @@ async function failRuntimeMapRowChunks(
|
|
|
7180
7281
|
const { rows } = await client.query<{
|
|
7181
7282
|
updated?: number;
|
|
7182
7283
|
fenced_keys?: string[];
|
|
7284
|
+
conflict_keys?: string[];
|
|
7183
7285
|
}>(
|
|
7184
7286
|
`WITH updates AS (
|
|
7185
7287
|
SELECT key_values._key,
|
|
@@ -7213,6 +7315,26 @@ async function failRuntimeMapRowChunks(
|
|
|
7213
7315
|
)
|
|
7214
7316
|
ORDER BY target._key, (target._key = updates._key) DESC
|
|
7215
7317
|
),
|
|
7318
|
+
same_version_conflicts AS (
|
|
7319
|
+
SELECT updates.matched_key AS _key
|
|
7320
|
+
FROM matched_updates AS updates
|
|
7321
|
+
JOIN ${sheetTable(session)} AS target
|
|
7322
|
+
ON target._key = updates.matched_key
|
|
7323
|
+
WHERE $16::bigint IS NOT NULL
|
|
7324
|
+
AND target._write_version = $16::bigint
|
|
7325
|
+
AND target._run_id = $6::text
|
|
7326
|
+
AND target._status IN ('enriched', 'failed')
|
|
7327
|
+
AND EXISTS (
|
|
7328
|
+
SELECT 1
|
|
7329
|
+
FROM unnest($9::text[]) AS field_values(field)
|
|
7330
|
+
WHERE target._cell_meta -> field_values.field ->> 'runId' = $6::text
|
|
7331
|
+
)
|
|
7332
|
+
AND (
|
|
7333
|
+
target._status <> 'failed'
|
|
7334
|
+
OR target._error IS DISTINCT FROM updates.error
|
|
7335
|
+
OR (${targetChangedPatchedCellSql})
|
|
7336
|
+
)
|
|
7337
|
+
),
|
|
7216
7338
|
applied_rows AS (
|
|
7217
7339
|
UPDATE ${sheetTable(session)} AS target
|
|
7218
7340
|
SET _status = 'failed',
|
|
@@ -7229,6 +7351,7 @@ async function failRuntimeMapRowChunks(
|
|
|
7229
7351
|
_cell_meta = ${mergeRuntimeCellMetaPatchSql('target._cell_meta', 'updates.cell_meta_patch')}${input.physicalUpdateSetSql}
|
|
7230
7352
|
FROM matched_updates AS updates, ${sheetTable(session)} AS prev
|
|
7231
7353
|
WHERE target._key = updates.matched_key
|
|
7354
|
+
AND NOT EXISTS (SELECT 1 FROM same_version_conflicts)
|
|
7232
7355
|
AND ($16::bigint IS NULL OR target._write_version = $16::bigint)
|
|
7233
7356
|
AND prev._key = target._key
|
|
7234
7357
|
AND ($15::boolean OR target._status <> 'enriched')
|
|
@@ -7237,6 +7360,14 @@ async function failRuntimeMapRowChunks(
|
|
|
7237
7360
|
($11::text IS NULL AND target._run_id = $6::text)
|
|
7238
7361
|
OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
|
|
7239
7362
|
)
|
|
7363
|
+
AND NOT (
|
|
7364
|
+
$16::bigint IS NOT NULL
|
|
7365
|
+
AND target._write_version = $16::bigint
|
|
7366
|
+
AND target._run_id = $6::text
|
|
7367
|
+
AND target._status = 'failed'
|
|
7368
|
+
AND target._error IS NOT DISTINCT FROM updates.error
|
|
7369
|
+
AND NOT (${targetChangedPatchedCellSql})
|
|
7370
|
+
)
|
|
7240
7371
|
RETURNING target._key, prev._status AS prev_status
|
|
7241
7372
|
),
|
|
7242
7373
|
applied_count AS (
|
|
@@ -7326,7 +7457,8 @@ async function failRuntimeMapRowChunks(
|
|
|
7326
7457
|
)
|
|
7327
7458
|
SELECT
|
|
7328
7459
|
(SELECT count(*)::int FROM applied_rows) AS updated,
|
|
7329
|
-
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys
|
|
7460
|
+
coalesce((SELECT array_agg(_key) FROM fenced_rows), '{}'::text[]) AS fenced_keys,
|
|
7461
|
+
coalesce((SELECT array_agg(_key) FROM same_version_conflicts), '{}'::text[]) AS conflict_keys`,
|
|
7330
7462
|
[
|
|
7331
7463
|
chunkKeys,
|
|
7332
7464
|
chunkInputIndexes,
|
|
@@ -7348,8 +7480,9 @@ async function failRuntimeMapRowChunks(
|
|
|
7348
7480
|
);
|
|
7349
7481
|
updated += Number(rows[0]?.updated ?? 0);
|
|
7350
7482
|
fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
|
|
7483
|
+
conflictKeys.push(...(rows[0]?.conflict_keys ?? []));
|
|
7351
7484
|
}
|
|
7352
|
-
return { updated, fencedKeys };
|
|
7485
|
+
return { updated, fencedKeys, conflictKeys };
|
|
7353
7486
|
}
|
|
7354
7487
|
|
|
7355
7488
|
/**
|
|
@@ -7382,6 +7515,13 @@ export async function completeRuntimeMapRows(
|
|
|
7382
7515
|
forceFailedRows?: boolean;
|
|
7383
7516
|
},
|
|
7384
7517
|
): Promise<RuntimeMapRowsWriteResult> {
|
|
7518
|
+
if (input.rows.length === 0) {
|
|
7519
|
+
return classifyRuntimeMapRowsWrite({
|
|
7520
|
+
submittedKeys: [],
|
|
7521
|
+
updated: 0,
|
|
7522
|
+
fencedKeys: [],
|
|
7523
|
+
});
|
|
7524
|
+
}
|
|
7385
7525
|
if (context.dbSessionStrategy === 'gateway_only') {
|
|
7386
7526
|
const keyedRows = input.rows.map((row) => {
|
|
7387
7527
|
if (row.key) return row;
|
|
@@ -7395,13 +7535,27 @@ export async function completeRuntimeMapRows(
|
|
|
7395
7535
|
runId: input.runId,
|
|
7396
7536
|
outputFields: input.outputFields ?? [],
|
|
7397
7537
|
});
|
|
7398
|
-
|
|
7538
|
+
const result = await postRuntimeApi<
|
|
7539
|
+
Partial<RuntimeMapRowsWriteResult> &
|
|
7540
|
+
Pick<RuntimeMapRowsWriteResult, 'updated' | 'fencedKeys'>
|
|
7541
|
+
>(context, {
|
|
7399
7542
|
action: 'runtime_sheet_complete_map_rows',
|
|
7400
7543
|
input: { ...input, rows },
|
|
7401
7544
|
});
|
|
7402
|
-
|
|
7403
|
-
|
|
7404
|
-
|
|
7545
|
+
const submittedKeys = keyedRows
|
|
7546
|
+
.map((row) => row.key)
|
|
7547
|
+
.filter((key): key is string => Boolean(key));
|
|
7548
|
+
const staleKeys =
|
|
7549
|
+
result.staleKeys ?? result.staleDroppedKeys ?? result.fencedKeys;
|
|
7550
|
+
return {
|
|
7551
|
+
...classifyRuntimeMapRowsWrite({
|
|
7552
|
+
submittedKeys,
|
|
7553
|
+
updated: result.updated,
|
|
7554
|
+
fencedKeys: staleKeys,
|
|
7555
|
+
}),
|
|
7556
|
+
// Preserve the gateway's compatibility classification during rollout.
|
|
7557
|
+
fencedKeys: result.fencedKeys,
|
|
7558
|
+
};
|
|
7405
7559
|
}
|
|
7406
7560
|
const sheetContract = augmentSheetContractWithDatasetFields({
|
|
7407
7561
|
contract: input.sheetContract,
|
|
@@ -7441,7 +7595,11 @@ export async function completeRuntimeMapRows(
|
|
|
7441
7595
|
uniqueRows.set(resolvedKey, { ...row, key: resolvedKey });
|
|
7442
7596
|
}
|
|
7443
7597
|
if (uniqueRows.size === 0) {
|
|
7444
|
-
return {
|
|
7598
|
+
return classifyRuntimeMapRowsWrite({
|
|
7599
|
+
submittedKeys: [],
|
|
7600
|
+
updated: 0,
|
|
7601
|
+
fencedKeys: [],
|
|
7602
|
+
});
|
|
7445
7603
|
}
|
|
7446
7604
|
|
|
7447
7605
|
const projections = physicalSheetColumnProjections(sheetContract);
|
|
@@ -7526,14 +7684,16 @@ export async function completeRuntimeMapRows(
|
|
|
7526
7684
|
return {
|
|
7527
7685
|
updated: updated.updated + inserted.inserted,
|
|
7528
7686
|
fencedKeys: updated.fencedKeys,
|
|
7687
|
+
conflictKeys: updated.conflictKeys,
|
|
7529
7688
|
};
|
|
7530
7689
|
})()
|
|
7531
|
-
: { updated: 0, fencedKeys: [] };
|
|
7690
|
+
: { updated: 0, fencedKeys: [], conflictKeys: [] };
|
|
7532
7691
|
const failed =
|
|
7533
7692
|
failedRows.length > 0
|
|
7534
7693
|
? await failRuntimeMapRowChunks(client, session, {
|
|
7535
7694
|
chunks: failedChunks,
|
|
7536
7695
|
physicalUpdateSetSql,
|
|
7696
|
+
physicalColumnProjections: projections,
|
|
7537
7697
|
forceTerminal: input.forceFailedRows === true,
|
|
7538
7698
|
runId: input.runId,
|
|
7539
7699
|
attemptId,
|
|
@@ -7546,13 +7706,22 @@ export async function completeRuntimeMapRows(
|
|
|
7546
7706
|
input.tableNamespace,
|
|
7547
7707
|
),
|
|
7548
7708
|
})
|
|
7549
|
-
: { updated: 0, fencedKeys: [] };
|
|
7550
|
-
|
|
7709
|
+
: { updated: 0, fencedKeys: [], conflictKeys: [] };
|
|
7710
|
+
const conflictKeys = [...completed.conflictKeys, ...failed.conflictKeys];
|
|
7711
|
+
if (conflictKeys.length > 0) {
|
|
7712
|
+
throw new Error(
|
|
7713
|
+
`Runtime Sheet received the same write version with a different terminal payload for row(s): ${[
|
|
7714
|
+
...new Set(conflictKeys),
|
|
7715
|
+
]
|
|
7716
|
+
.slice(0, 10)
|
|
7717
|
+
.join(', ')}.`,
|
|
7718
|
+
);
|
|
7719
|
+
}
|
|
7720
|
+
return classifyRuntimeMapRowsWrite({
|
|
7721
|
+
submittedKeys: uniqueRows.keys(),
|
|
7551
7722
|
updated: completed.updated + failed.updated,
|
|
7552
7723
|
fencedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
|
|
7553
|
-
|
|
7554
|
-
staleDroppedKeys: [...completed.fencedKeys, ...failed.fencedKeys],
|
|
7555
|
-
};
|
|
7724
|
+
});
|
|
7556
7725
|
},
|
|
7557
7726
|
);
|
|
7558
7727
|
}
|