deepline 0.3.314 → 0.3.316
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +12 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/runs/overview.ts +17 -10
- package/dist/bundling-sources/sdk/src/types.ts +16 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +217 -488
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +15 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/internal-bootstrap-token.ts +13 -4
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +31 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +444 -51
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +22 -43
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +4 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +136 -87
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-authority.ts +24 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts +20 -11
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +18 -12
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +140 -9
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/workflow-executor-auth.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +3 -2
- package/dist/bundling-sources/shared_libs/plays/docflow-binding-owner.ts +61 -11
- package/dist/cli/{auth-GWF55S3N.mjs → auth-JP7ZJ4OI.mjs} +4 -4
- package/dist/cli/auth-SPUJ4EXC.js +31 -0
- package/dist/cli/{billing-WWOJN7GV.mjs → billing-KG6MC2BC.mjs} +4 -4
- package/dist/cli/{billing-2YU6E2IF.js → billing-WZGS32ME.js} +70 -70
- package/dist/cli/{chunk-OJ243ZA5.js → chunk-2HKPAZVJ.js} +1 -1
- package/dist/cli/{chunk-YVMJTXEG.js → chunk-35YXDTOB.js} +26 -26
- package/dist/cli/{chunk-QB5MAFCU.mjs → chunk-46ZOD4PQ.mjs} +1 -1
- package/dist/cli/{chunk-CPN2N6EZ.mjs → chunk-6R5ACEWN.mjs} +43 -17
- package/dist/cli/{chunk-US4S3E4Y.mjs → chunk-7PZFGB7L.mjs} +1 -1
- package/dist/cli/{chunk-3CKNUJVZ.mjs → chunk-AXAPMVUC.mjs} +4 -4
- package/dist/cli/{chunk-ENQZV7NR.js → chunk-B6OOVO3A.js} +562 -536
- package/dist/cli/{chunk-WDPRE6DU.js → chunk-BOKHMPCZ.js} +48 -48
- package/dist/cli/{chunk-VFRZG4OW.js → chunk-C6QGM72M.js} +1 -1
- package/dist/cli/{chunk-IZTIBG7A.mjs → chunk-D3NMIACS.mjs} +29 -15
- package/dist/cli/{chunk-DR3S76BG.mjs → chunk-DAFGPNVW.mjs} +1 -1
- package/dist/cli/{chunk-3K6QD7Z3.mjs → chunk-F2T5UENG.mjs} +1 -1
- package/dist/cli/{chunk-VWW6VHQX.js → chunk-G2G67D3L.js} +60 -60
- package/dist/cli/{chunk-ODEWOXWW.js → chunk-GHLWDXUB.js} +6 -6
- package/dist/cli/{chunk-25SPKKF2.mjs → chunk-IDSDRGKF.mjs} +1 -1
- package/dist/cli/{chunk-OUHCJL5B.mjs → chunk-J3ZPMADG.mjs} +1 -1
- package/dist/cli/{chunk-MLEJ2FYH.js → chunk-K3BIRBT3.js} +248 -234
- package/dist/cli/{chunk-N5V2SBB2.mjs → chunk-KP7W5NO6.mjs} +1 -1
- package/dist/cli/{chunk-B5LUKLKH.js → chunk-MCPHK7AF.js} +4 -4
- package/dist/cli/{chunk-2Q2IKUL5.mjs → chunk-OB3SYJGU.mjs} +3 -3
- package/dist/cli/{chunk-UZTTPEG2.js → chunk-PJERS5VW.js} +6 -6
- package/dist/cli/{chunk-VBRO5QCI.js → chunk-RBBSRB56.js} +15 -15
- package/dist/cli/{chunk-NG4WDOGN.js → chunk-SRUWVQ66.js} +6 -6
- package/dist/cli/{chunk-WTO3ITBC.mjs → chunk-TYSAKSX7.mjs} +2 -2
- package/dist/cli/{chunk-HTXZLCSP.mjs → chunk-VC25WFXV.mjs} +2 -2
- package/dist/cli/{chunk-3A4LJUX6.mjs → chunk-VDPBQCRQ.mjs} +2 -2
- package/dist/cli/{chunk-IALPYID6.js → chunk-XSEMQ4II.js} +3 -3
- package/dist/cli/{chunk-JNUHGWB2.js → chunk-YEMOOWZI.js} +50 -50
- package/dist/cli/{chunk-JZZT4GAY.mjs → chunk-YNSAVFWL.mjs} +5 -5
- package/dist/cli/{chunk-GSRDZD7R.js → chunk-ZNHGAWYI.js} +2 -2
- package/dist/cli/client-GMMPQT7B.js +20 -0
- package/dist/cli/{client-EYGNDORT.mjs → client-MNPM4ME3.mjs} +3 -3
- package/dist/cli/{csv-YOW5JMOH.js → csv-G72FIQWI.js} +12 -12
- package/dist/cli/{csv-NZCN7SVV.mjs → csv-ZME7HAPR.mjs} +3 -3
- package/dist/cli/{db-HALJV3KY.mjs → db-K2DS4PEY.mjs} +4 -4
- package/dist/cli/{db-6BIYLZ6F.js → db-KRBWHSMA.js} +20 -20
- package/dist/cli/{describe-KY2ZYGBR.mjs → describe-LXBJP6OP.mjs} +10 -10
- package/dist/cli/{describe-WA6R2JSH.js → describe-UKFV376D.js} +17 -17
- package/dist/cli/{executions-CBOAN6KI.mjs → executions-2V3AROY6.mjs} +4 -4
- package/dist/cli/{executions-NMCTN7XM.js → executions-E66AM4JU.js} +6 -6
- package/dist/cli/{feedback-ZZFDSCWS.js → feedback-7C2N7M37.js} +6 -6
- package/dist/cli/{feedback-FOMPE3DJ.mjs → feedback-IEPZ2XKZ.mjs} +3 -3
- package/dist/cli/{git-YQR2NBCO.mjs → git-AR5KGN4O.mjs} +4 -4
- package/dist/cli/git-EXYB66IH.js +23 -0
- package/dist/cli/index.js +171 -98
- package/dist/cli/index.mjs +114 -41
- package/dist/cli/{legacy-noops-XH2TFFI6.js → legacy-noops-A2SH45P4.js} +7 -7
- package/dist/cli/{legacy-noops-KMU6P3K4.mjs → legacy-noops-LC2EGAZU.mjs} +4 -4
- package/dist/cli/{monitors-2ONK5L3I.mjs → monitors-5YPW3IZQ.mjs} +4 -4
- package/dist/cli/{monitors-5M3BZ5OR.js → monitors-IZBUPKPW.js} +85 -85
- package/dist/cli/{org-LR5BMCGL.mjs → org-2OMYI5KJ.mjs} +4 -4
- package/dist/cli/{org-6MKULUZI.js → org-L42SQ65U.js} +60 -60
- package/dist/cli/play-CQI3IDDN.js +152 -0
- package/dist/cli/{play-4NOSD7DW.mjs → play-UPCBLAOH.mjs} +9 -9
- package/dist/cli/{providers-UFJD3VKF.js → providers-GGNDWUKB.js} +7 -7
- package/dist/cli/{providers-UBGIKGCA.mjs → providers-VE2RM56M.mjs} +4 -4
- package/dist/cli/{quickstart-KQZHZE6C.js → quickstart-4CZHURUX.js} +5 -5
- package/dist/cli/{quickstart-KDUKQRUH.mjs → quickstart-HSJ7FU7Z.mjs} +3 -3
- package/dist/cli/{search-4BVOOS5Q.mjs → search-MWBM3I2L.mjs} +5 -5
- package/dist/cli/search-PD5FBCUN.js +26 -0
- package/dist/cli/{secrets-TVXZ3WJ3.js → secrets-CWBPGDZV.js} +13 -13
- package/dist/cli/{secrets-YGV23N3N.mjs → secrets-W5DQYKGU.mjs} +4 -4
- package/dist/cli/{sessions-RWIQWDFD.js → sessions-EDCFU6P2.js} +5 -5
- package/dist/cli/{sessions-G7ROUW35.mjs → sessions-Q42GUYTF.mjs} +4 -4
- package/dist/cli/{settings-TG3E2NR7.js → settings-OKRENZGX.js} +74 -74
- package/dist/cli/{settings-QZEVGCXE.mjs → settings-SO56AJPQ.mjs} +7 -7
- package/dist/cli/{setup-3KDUNEEE.js → setup-7PLHLNLS.js} +41 -41
- package/dist/cli/{setup-HUXINKOH.mjs → setup-OTYMF5CZ.mjs} +6 -6
- package/dist/cli/{skills-LHU3DB7J.mjs → skills-4UFS4FXP.mjs} +4 -4
- package/dist/cli/skills-IP7IW7EQ.js +28 -0
- package/dist/cli/{tools-J5WQ33BB.mjs → tools-6FV543YV.mjs} +6 -6
- package/dist/cli/tools-A2YNR6PI.js +38 -0
- package/dist/cli/{update-T6744RM7.mjs → update-6POGDXYW.mjs} +6 -6
- package/dist/cli/update-QXSGKLJG.js +28 -0
- package/dist/cli/{workflow-CG2B65HD.mjs → workflow-HVZSFWYR.mjs} +5 -5
- package/dist/cli/{workflow-YF5SQJLG.js → workflow-RB4POZP5.js} +15 -15
- package/dist/{client-OBwghxPC.d.ts → client-BV68GMPo.d.ts} +19 -6
- package/dist/{client-CtVdLZux.d.mts → client-D9ACjiRC.d.mts} +19 -6
- package/dist/{compiler-manifest-wOKNiXpa.d.mts → compiler-manifest-BXcZ4P4W.d.mts} +2 -2
- package/dist/{compiler-manifest-wOKNiXpa.d.ts → compiler-manifest-BXcZ4P4W.d.ts} +2 -2
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +7 -7
- package/dist/index.mjs +4 -4
- package/dist/install-integrity.json +82 -82
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-QX2L4LPM.js +0 -31
- package/dist/cli/client-Z4VFNTUB.js +0 -20
- package/dist/cli/git-AC2ATBDK.js +0 -23
- package/dist/cli/play-U3CZ4F2W.js +0 -152
- package/dist/cli/search-Y7OTFRCI.js +0 -26
- package/dist/cli/skills-CO5YUWBK.js +0 -28
- package/dist/cli/tools-MWPT5X47.js +0 -38
- package/dist/cli/update-VVIJW7ZJ.js +0 -28
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import { RUN_EVENT_KIND } from './run-event-catalog';
|
|
2
|
+
import {
|
|
3
|
+
MapExecutionFrameStore,
|
|
4
|
+
MapExecutionVisibility,
|
|
5
|
+
} from './map-execution-frame';
|
|
2
6
|
/**
|
|
3
7
|
* PlayContextImpl — the cloud execution engine.
|
|
4
8
|
*
|
|
@@ -477,6 +481,7 @@ import {
|
|
|
477
481
|
} from './step-program-dataset-builder';
|
|
478
482
|
import {
|
|
479
483
|
createRuntimeSheetDatasetPageReader,
|
|
484
|
+
readRuntimeSheetDatasetRowKeys,
|
|
480
485
|
RuntimeSheetAttemptSupersededError,
|
|
481
486
|
} from './runtime-api';
|
|
482
487
|
import { executionGatewayDecodedBodyBytes } from './execution-gateway-response-size';
|
|
@@ -553,19 +558,11 @@ const authoredStepContext = new AsyncLocalStorage<{
|
|
|
553
558
|
const toolExecutionOverrides = new AsyncLocalStorage<{ force: boolean }>();
|
|
554
559
|
const PROGRESS_HEARTBEAT_INTERVAL_MS = 1_000;
|
|
555
560
|
const PURE_JS_HEARTBEAT_ROW_INTERVAL = 250;
|
|
556
|
-
/**
|
|
557
|
-
* How often map-frame checkpoint state is materialized (array spread + frame
|
|
558
|
-
* clone) and a map.progress event is emitted during row execution. Per-row
|
|
559
|
-
* materialization is O(rows²) in total copies and floods event consumers; a
|
|
560
|
-
* row/time throttle keeps progress fresh while staying O(rows).
|
|
561
|
-
*/
|
|
562
|
-
const MAP_FRAME_FLUSH_ROW_INTERVAL = 100;
|
|
563
561
|
/** Executed-row sheet persistence chunking: rows AND bytes, whichever first. */
|
|
564
562
|
const MAP_PERSIST_CHUNK_ROWS = 2_000;
|
|
565
563
|
const MAP_PERSIST_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
566
564
|
const MAP_ROW_WRITER_BUFFER_BYTES = 16 * 1024 * 1024;
|
|
567
565
|
const MAP_ROW_WRITER_MAX_FLUSH_MS = 1_000;
|
|
568
|
-
const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
|
|
569
566
|
const MAP_STALL_LOG_INTERVAL_MS = 60_000;
|
|
570
567
|
// Once the persistence latch has tripped, nothing a map is still waiting on can
|
|
571
568
|
// change its outcome: the map fails and the run ends. What the wait can still
|
|
@@ -2040,17 +2037,23 @@ type FieldMapRunResult = {
|
|
|
2040
2037
|
completedRowCount: number;
|
|
2041
2038
|
failedRowCount: number;
|
|
2042
2039
|
failureSamples: Array<{ rowLabel: string; error: string }>;
|
|
2043
|
-
previewRows:
|
|
2040
|
+
previewRows: Array<Pick<PersistableMapRow, 'key' | 'data'>>;
|
|
2044
2041
|
retainedRowsComplete: boolean;
|
|
2045
2042
|
};
|
|
2046
2043
|
|
|
2047
2044
|
const MAP_FAILURE_SAMPLE_LIMIT = 3;
|
|
2048
2045
|
|
|
2049
2046
|
type RuntimeMapRowPersistence = {
|
|
2047
|
+
/** Accepted-write visibility shared by every pool using this map's writer. */
|
|
2048
|
+
visibility: MapExecutionVisibility;
|
|
2050
2049
|
persistRows: (rows: PersistableMapRow[]) => {
|
|
2051
2050
|
admitted: Promise<void>;
|
|
2052
2051
|
committed: Promise<void>;
|
|
2053
2052
|
};
|
|
2053
|
+
/** Register the pool's single observer before admitting terminal writes. */
|
|
2054
|
+
observeCommittedRows: (
|
|
2055
|
+
observer: (rows: readonly PersistableMapRow[]) => void,
|
|
2056
|
+
) => void;
|
|
2054
2057
|
isPersisted: (row: PersistableMapRow) => boolean;
|
|
2055
2058
|
checkpoint: (updates: PlayRowUpdate[]) => Promise<void>;
|
|
2056
2059
|
flush: () => Promise<void>;
|
|
@@ -2059,6 +2062,7 @@ type RuntimeMapRowPersistence = {
|
|
|
2059
2062
|
|
|
2060
2063
|
function createRuntimeMapRowPersistence(
|
|
2061
2064
|
persistRows: (rows: PersistableMapRow[]) => Promise<void>,
|
|
2065
|
+
visibility: MapExecutionVisibility,
|
|
2062
2066
|
options?: {
|
|
2063
2067
|
prepareTerminalRow?: (row: PersistableMapRow) => PersistableMapRow;
|
|
2064
2068
|
persistCheckpoint?: (updates: PlayRowUpdate[]) => Promise<void>;
|
|
@@ -2075,6 +2079,8 @@ function createRuntimeMapRowPersistence(
|
|
|
2075
2079
|
): RuntimeMapRowPersistence {
|
|
2076
2080
|
const persisted = new Set<string>();
|
|
2077
2081
|
const queued = new Set<string>();
|
|
2082
|
+
let committedObserver: ((rows: readonly PersistableMapRow[]) => void) | null =
|
|
2083
|
+
null;
|
|
2078
2084
|
const writer = new RuntimeSheetRowWriter<PersistableMapRow, PlayRowUpdate>({
|
|
2079
2085
|
terminalKey: (row) => {
|
|
2080
2086
|
const identity = persistableMapRowIdentity(row);
|
|
@@ -2106,6 +2112,9 @@ function createRuntimeMapRowPersistence(
|
|
|
2106
2112
|
writeBatch: async (batch) => {
|
|
2107
2113
|
if (batch.kind === 'terminal') {
|
|
2108
2114
|
await persistRows([...batch.rows]);
|
|
2115
|
+
// Latest-view corrections describe this entire committed batch. Publish
|
|
2116
|
+
// its outcomes together, before separate per-row promise continuations.
|
|
2117
|
+
committedObserver?.(batch.rows);
|
|
2109
2118
|
return { committed: batch.rows.length };
|
|
2110
2119
|
}
|
|
2111
2120
|
if (batch.updates.length > 0) {
|
|
@@ -2121,6 +2130,7 @@ function createRuntimeMapRowPersistence(
|
|
|
2121
2130
|
if (latch?.tripped) writer.fail(runtimePersistenceLatchCauseError(latch));
|
|
2122
2131
|
|
|
2123
2132
|
return {
|
|
2133
|
+
visibility,
|
|
2124
2134
|
persistRows: (rows) => {
|
|
2125
2135
|
const admissions: Promise<void>[] = [];
|
|
2126
2136
|
const commits: Promise<void>[] = [];
|
|
@@ -2165,6 +2175,13 @@ function createRuntimeMapRowPersistence(
|
|
|
2165
2175
|
committed: Promise.all(commits).then(() => {}),
|
|
2166
2176
|
};
|
|
2167
2177
|
},
|
|
2178
|
+
observeCommittedRows: (observer) => {
|
|
2179
|
+
if (committedObserver)
|
|
2180
|
+
throw new Error(
|
|
2181
|
+
'Runtime map writer already has a committed-row observer.',
|
|
2182
|
+
);
|
|
2183
|
+
committedObserver = observer;
|
|
2184
|
+
},
|
|
2168
2185
|
isPersisted: (row) => {
|
|
2169
2186
|
const identity = persistableMapRowIdentity(row);
|
|
2170
2187
|
return identity ? persisted.has(identity) : false;
|
|
@@ -2585,25 +2602,6 @@ function emptyCheckpoint(): PlayCheckpoint {
|
|
|
2585
2602
|
};
|
|
2586
2603
|
}
|
|
2587
2604
|
|
|
2588
|
-
function cloneMapFrame(frame: MapExecutionFrame): MapExecutionFrame {
|
|
2589
|
-
return {
|
|
2590
|
-
...frame,
|
|
2591
|
-
completedRowKeys: [...frame.completedRowKeys],
|
|
2592
|
-
pendingRowKeys: [...frame.pendingRowKeys],
|
|
2593
|
-
...(frame.suspendedInputRows
|
|
2594
|
-
? {
|
|
2595
|
-
suspendedInputRows: frame.suspendedInputRows.map((row) => ({
|
|
2596
|
-
inputIndex: row.inputIndex,
|
|
2597
|
-
data: cloneCsvAliasedRow(row.data),
|
|
2598
|
-
...(row.authoredInputDigest
|
|
2599
|
-
? { authoredInputDigest: row.authoredInputDigest }
|
|
2600
|
-
: {}),
|
|
2601
|
-
})),
|
|
2602
|
-
}
|
|
2603
|
-
: {}),
|
|
2604
|
-
};
|
|
2605
|
-
}
|
|
2606
|
-
|
|
2607
2605
|
/**
|
|
2608
2606
|
* Adapt this runtime's per-tool queue hints into the Play Execution Governor's
|
|
2609
2607
|
* pacing resolver. The Governor owns per-(org, provider) pacing; this only maps
|
|
@@ -2745,6 +2743,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2745
2743
|
>();
|
|
2746
2744
|
private readonly ctxFetchHttpFailureDiagnosticIdentities = new Set<string>();
|
|
2747
2745
|
private checkpoint: PlayCheckpoint;
|
|
2746
|
+
private readonly mapFrames: MapExecutionFrameStore;
|
|
2748
2747
|
/**
|
|
2749
2748
|
* A durable boundary is a control-flow handoff, not a customer-catchable
|
|
2750
2749
|
* business failure. Keep it on the context as the source of truth so a
|
|
@@ -3259,6 +3258,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3259
3258
|
};
|
|
3260
3259
|
this.#options = options;
|
|
3261
3260
|
this.checkpoint = options.checkpoint ?? emptyCheckpoint();
|
|
3261
|
+
this.mapFrames = new MapExecutionFrameStore(this.checkpoint, (event) =>
|
|
3262
|
+
this.emitExecutionEvent(event),
|
|
3263
|
+
);
|
|
3262
3264
|
const checkpointCacheEpochMs = this.checkpoint.durableCallCacheEpochMs;
|
|
3263
3265
|
this.durableCallCacheEpochMs =
|
|
3264
3266
|
typeof checkpointCacheEpochMs === 'number' &&
|
|
@@ -4310,10 +4312,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4310
4312
|
}
|
|
4311
4313
|
|
|
4312
4314
|
private setMapFrame(frame: MapExecutionFrame): void {
|
|
4313
|
-
this.
|
|
4314
|
-
...(this.checkpoint.mapFrames ?? {}),
|
|
4315
|
-
[frame.mapInvocationId]: cloneMapFrame(frame),
|
|
4316
|
-
};
|
|
4315
|
+
this.mapFrames.set(frame);
|
|
4317
4316
|
}
|
|
4318
4317
|
|
|
4319
4318
|
private createMapExecutionScope(input: {
|
|
@@ -8180,52 +8179,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8180
8179
|
: derivePlayRowIdentity(stableInputRow(row), resolvedTableNamespace);
|
|
8181
8180
|
};
|
|
8182
8181
|
|
|
8183
|
-
// A terminal write can be durably accepted into this run's history while
|
|
8184
|
-
// losing the mutable latest-view race to newer intent. That outcome must
|
|
8185
|
-
// not leak into the dataset returned by this execution. A later same-run
|
|
8186
|
-
// reconciliation can explicitly recover a key; only that positive fact
|
|
8187
|
-
// makes it visible again.
|
|
8188
|
-
const staleCompletionKeys = new Set<string>();
|
|
8189
|
-
const staleCompletedKeys = new Set<string>();
|
|
8190
|
-
const staleFailedKeys = new Set<string>();
|
|
8191
|
-
const recoveredStaleCompletionKeys = new Set<string>();
|
|
8192
|
-
const observeMapWriteResult = (
|
|
8193
|
-
writeResult: unknown,
|
|
8194
|
-
writtenRows: readonly PersistableMapRow[] = [],
|
|
8195
|
-
): void => {
|
|
8196
|
-
if (!writeResult || typeof writeResult !== 'object') return;
|
|
8197
|
-
const result = writeResult as {
|
|
8198
|
-
staleDroppedKeys?: unknown;
|
|
8199
|
-
recoveredStaleKeys?: unknown;
|
|
8200
|
-
};
|
|
8201
|
-
const rowsByKey = new Map(writtenRows.map((row) => [row.key, row]));
|
|
8202
|
-
if (Array.isArray(result.staleDroppedKeys)) {
|
|
8203
|
-
for (const key of result.staleDroppedKeys) {
|
|
8204
|
-
if (typeof key !== 'string' || !key) continue;
|
|
8205
|
-
staleCompletionKeys.add(key);
|
|
8206
|
-
if (rowsByKey.get(key)?.status === 'failed') {
|
|
8207
|
-
staleFailedKeys.add(key);
|
|
8208
|
-
staleCompletedKeys.delete(key);
|
|
8209
|
-
} else {
|
|
8210
|
-
staleCompletedKeys.add(key);
|
|
8211
|
-
staleFailedKeys.delete(key);
|
|
8212
|
-
}
|
|
8213
|
-
}
|
|
8214
|
-
}
|
|
8215
|
-
if (Array.isArray(result.recoveredStaleKeys)) {
|
|
8216
|
-
for (const key of result.recoveredStaleKeys) {
|
|
8217
|
-
if (typeof key !== 'string' || !key) continue;
|
|
8218
|
-
recoveredStaleCompletionKeys.add(key);
|
|
8219
|
-
staleCompletionKeys.delete(key);
|
|
8220
|
-
staleCompletedKeys.delete(key);
|
|
8221
|
-
staleFailedKeys.delete(key);
|
|
8222
|
-
}
|
|
8223
|
-
}
|
|
8224
|
-
};
|
|
8225
|
-
const isVisibleTerminalRow = (row: PersistableMapRow): boolean =>
|
|
8226
|
-
!staleCompletionKeys.has(row.key) ||
|
|
8227
|
-
recoveredStaleCompletionKeys.has(row.key);
|
|
8228
|
-
|
|
8229
8182
|
// Identity for SEED rows (pre-execution raw inputs), matching the server's
|
|
8230
8183
|
// own derivation in startRuntimeSheetDataset: hash the FULL cleaned input
|
|
8231
8184
|
// row, output-named columns INCLUDED. Two distinct input rows that differ
|
|
@@ -8292,19 +8245,52 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8292
8245
|
// Completed siblings require either the Runtime Sheet or a complete
|
|
8293
8246
|
// authored replay. Never return only the pending subset as a full result.
|
|
8294
8247
|
const replayWithoutSheet = Boolean(
|
|
8295
|
-
suspendedInputRows
|
|
8248
|
+
suspendedInputRows !== null && !canReadPersistedMapResults,
|
|
8296
8249
|
);
|
|
8297
8250
|
const resumedPendingRowsOnly = Boolean(
|
|
8298
|
-
suspendedInputRows
|
|
8251
|
+
suspendedInputRows !== null &&
|
|
8299
8252
|
(canReadPersistedMapResults ||
|
|
8300
8253
|
(replayMapFrame?.completedRowKeys.length ?? 0) === 0),
|
|
8301
8254
|
);
|
|
8302
|
-
const previouslyCompletedRowKeys = resumedPendingRowsOnly
|
|
8303
|
-
? (replayMapFrame?.completedRowKeys ?? [])
|
|
8304
|
-
: [];
|
|
8305
8255
|
const previouslyFailedRowsCount = resumedPendingRowsOnly
|
|
8306
8256
|
? (replayMapFrame?.failedRowsCount ?? 0)
|
|
8307
8257
|
: 0;
|
|
8258
|
+
const {
|
|
8259
|
+
completedRowKeys: previouslyCompletedRowKeys,
|
|
8260
|
+
completedRowsCount: previouslyCompletedRowsCount,
|
|
8261
|
+
reusedRowsCount: previouslyReusedRowsCount,
|
|
8262
|
+
} = resumedPendingRowsOnly
|
|
8263
|
+
? await this.mapFrames.restoreCompletedRows(
|
|
8264
|
+
replayMapFrame!,
|
|
8265
|
+
canReadPersistedMapResults
|
|
8266
|
+
? async (keys) => {
|
|
8267
|
+
const result = await readRuntimeSheetDatasetRowKeys(
|
|
8268
|
+
{
|
|
8269
|
+
baseUrl: this.#options.baseUrl!,
|
|
8270
|
+
executorToken: this.#options.executorToken!,
|
|
8271
|
+
dbSessionStrategy: this.#options.dbSessionStrategy,
|
|
8272
|
+
playName: this.#options.playName!,
|
|
8273
|
+
userEmail: this.#options.userEmail,
|
|
8274
|
+
runId: this.#options.runId!,
|
|
8275
|
+
},
|
|
8276
|
+
{
|
|
8277
|
+
tableNamespace: replayMapFrame!.artifactTableNamespace,
|
|
8278
|
+
runId: this.#options.runId!,
|
|
8279
|
+
keys,
|
|
8280
|
+
rowMode: 'output',
|
|
8281
|
+
},
|
|
8282
|
+
);
|
|
8283
|
+
return result.keys;
|
|
8284
|
+
}
|
|
8285
|
+
: undefined,
|
|
8286
|
+
MAP_PERSIST_CHUNK_ROWS,
|
|
8287
|
+
)
|
|
8288
|
+
: { completedRowKeys: [], completedRowsCount: 0, reusedRowsCount: 0 };
|
|
8289
|
+
const visibility = new MapExecutionVisibility(
|
|
8290
|
+
resumedPendingRowsOnly ? (replayMapFrame?.supersededRows ?? 0) : 0,
|
|
8291
|
+
);
|
|
8292
|
+
const isVisibleTerminalRow = (row: PersistableMapRow): boolean =>
|
|
8293
|
+
visibility.isVisible(row.key);
|
|
8308
8294
|
if (shouldStreamRuntimeBackedDataset) {
|
|
8309
8295
|
// Net-new admission happens one source page at a time. Its progress and
|
|
8310
8296
|
// returned dataset describe admitted rows, not every provider/CSV row
|
|
@@ -8402,28 +8388,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8402
8388
|
),
|
|
8403
8389
|
staticPipeline: this.currentStaticPipeline ?? null,
|
|
8404
8390
|
});
|
|
8405
|
-
|
|
8406
|
-
|
|
8407
|
-
|
|
8408
|
-
|
|
8409
|
-
: [];
|
|
8410
|
-
if (supersededKeys.length > 0) {
|
|
8411
|
-
// The sheet rows belong to a different write version: this
|
|
8412
|
-
// attempt has been superseded and nothing it persists from here
|
|
8413
|
-
// on can be accepted. Fail the map (and trip the latch below)
|
|
8414
|
-
// instead of spending provider calls on rows that are dropped.
|
|
8415
|
-
throw new RuntimeSheetAttemptSupersededError({
|
|
8416
|
-
tableNamespace: resolvedTableNamespace,
|
|
8417
|
-
runId: this.currentRunId,
|
|
8418
|
-
writeVersionValue: null,
|
|
8419
|
-
submittedRows: chunk.length,
|
|
8420
|
-
supersededRows: supersededKeys.map((key) => ({
|
|
8421
|
-
key,
|
|
8422
|
-
reason: 'unreported' as const,
|
|
8423
|
-
rowVersionStamp: null,
|
|
8424
|
-
})),
|
|
8425
|
-
});
|
|
8426
|
-
}
|
|
8391
|
+
visibility.recordAcceptedWrite(writeResult, chunk, {
|
|
8392
|
+
tableNamespace: resolvedTableNamespace,
|
|
8393
|
+
runId: this.currentRunId,
|
|
8394
|
+
});
|
|
8427
8395
|
this.resourceGovernor.observe({
|
|
8428
8396
|
sheetFlushBytes: chunkBytes,
|
|
8429
8397
|
sheetFlushLatencyMs: Date.now() - flushStartedAt,
|
|
@@ -8601,6 +8569,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8601
8569
|
}): Promise<void> => {
|
|
8602
8570
|
const incrementalPersistence = createRuntimeMapRowPersistence(
|
|
8603
8571
|
persistMapRows,
|
|
8572
|
+
visibility,
|
|
8604
8573
|
{
|
|
8605
8574
|
prepareTerminalRow: (row) =>
|
|
8606
8575
|
this.withPendingMapRowData(row, resolvedTableNamespace),
|
|
@@ -8624,6 +8593,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8624
8593
|
|
|
8625
8594
|
this.beginActiveMapCellMeta();
|
|
8626
8595
|
this.beginActiveMapCheckpointUpdates();
|
|
8596
|
+
const settledBeforePool = visibility.settlement(
|
|
8597
|
+
processedCount - failedCount,
|
|
8598
|
+
failedCount,
|
|
8599
|
+
);
|
|
8627
8600
|
this.setMapFrame({
|
|
8628
8601
|
mapInvocationId: mapScope.mapInvocationId,
|
|
8629
8602
|
mapNodeId: mapScope.mapNodeId ?? null,
|
|
@@ -8633,13 +8606,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8633
8606
|
totalRows: totalInputCount,
|
|
8634
8607
|
completedRowKeys: [],
|
|
8635
8608
|
pendingRowKeys: execution.rowFeed ? [] : [...execution.rowKeys],
|
|
8636
|
-
completedRowsCount:
|
|
8637
|
-
0,
|
|
8638
|
-
processedCount - failedCount - staleCompletedKeys.size,
|
|
8639
|
-
),
|
|
8609
|
+
completedRowsCount: settledBeforePool.completedRows,
|
|
8640
8610
|
pendingRowsCount: Math.max(0, totalInputCount - processedCount),
|
|
8641
|
-
failedRowsCount:
|
|
8642
|
-
supersededRows:
|
|
8611
|
+
failedRowsCount: settledBeforePool.failedRows,
|
|
8612
|
+
supersededRows: visibility.supersededRows,
|
|
8643
8613
|
startedAt:
|
|
8644
8614
|
this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
|
|
8645
8615
|
Date.now(),
|
|
@@ -8662,15 +8632,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8662
8632
|
onRowError: options?.onRowError,
|
|
8663
8633
|
executionRowKeys: execution.rowKeys,
|
|
8664
8634
|
executionRowIndexes: execution.rowIndexes,
|
|
8665
|
-
|
|
8666
|
-
|
|
8667
|
-
|
|
8668
|
-
processedCount - failedCount - staleCompletedKeys.size,
|
|
8669
|
-
),
|
|
8670
|
-
progressFailedOffset: () =>
|
|
8671
|
-
Math.max(0, failedCount - staleFailedKeys.size),
|
|
8672
|
-
progressSupersededOffset: () =>
|
|
8673
|
-
staleCompletedKeys.size + staleFailedKeys.size,
|
|
8635
|
+
// Raw earlier-pool outcomes; the writer owns visibility facts.
|
|
8636
|
+
progressCompletedOffset: () => processedCount - failedCount,
|
|
8637
|
+
progressFailedOffset: () => failedCount,
|
|
8674
8638
|
progressSkippedOffset: () => duplicateReuseCount,
|
|
8675
8639
|
progressTotalRows: () => totalInputCount,
|
|
8676
8640
|
rowFeed: execution.rowFeed,
|
|
@@ -8698,13 +8662,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8698
8662
|
}
|
|
8699
8663
|
|
|
8700
8664
|
await incrementalPersistence.flush();
|
|
8701
|
-
const visibleCompletedRows =
|
|
8702
|
-
mapResult.completedRows.filter(isVisibleTerminalRow);
|
|
8703
|
-
const visibleFailedRows =
|
|
8704
|
-
mapResult.failedRows.filter(isVisibleTerminalRow);
|
|
8705
8665
|
const mapCellMeta = this.activeMapCellMeta;
|
|
8706
8666
|
const persistRows: PersistableMapRow[] = [];
|
|
8707
|
-
for (const row of
|
|
8667
|
+
for (const row of mapResult.completedRows) {
|
|
8708
8668
|
if (incrementalPersistence.isPersisted(row)) continue;
|
|
8709
8669
|
const meta = mapCellMeta?.get(row.key);
|
|
8710
8670
|
persistRows.push(
|
|
@@ -8720,16 +8680,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8720
8680
|
);
|
|
8721
8681
|
}
|
|
8722
8682
|
persistRows.push(
|
|
8723
|
-
...
|
|
8683
|
+
...mapResult.failedRows.filter(
|
|
8724
8684
|
(row) => !incrementalPersistence.isPersisted(row),
|
|
8725
8685
|
),
|
|
8726
8686
|
);
|
|
8727
8687
|
await persistMapRows(persistRows);
|
|
8688
|
+
const visibleCompletedRows =
|
|
8689
|
+
mapResult.completedRows.filter(isVisibleTerminalRow);
|
|
8728
8690
|
this.clearActiveMapCellMeta();
|
|
8729
8691
|
this.clearActiveMapCheckpointUpdates();
|
|
8730
8692
|
|
|
8731
|
-
for (const
|
|
8732
|
-
if (previewRows.length < 5
|
|
8693
|
+
for (const row of mapResult.previewRows) {
|
|
8694
|
+
if (previewRows.length < 5 && isVisibleTerminalRow(row))
|
|
8695
|
+
previewRows.push(this.toMaterializedOutputRow(row.data));
|
|
8733
8696
|
}
|
|
8734
8697
|
if (!mapResult.retainedRowsComplete) {
|
|
8735
8698
|
disableImmediateMaterializedCache(
|
|
@@ -8877,8 +8840,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8877
8840
|
}
|
|
8878
8841
|
}
|
|
8879
8842
|
|
|
8880
|
-
|
|
8881
|
-
|
|
8843
|
+
const terminalProgress = visibility.settlement(
|
|
8844
|
+
processedCount - failedCount,
|
|
8845
|
+
failedCount,
|
|
8846
|
+
);
|
|
8847
|
+
const visibleResults = visibility.settlement(
|
|
8848
|
+
successfulCount,
|
|
8849
|
+
failedCount,
|
|
8850
|
+
);
|
|
8851
|
+
successfulCount = visibleResults.completedRows;
|
|
8852
|
+
failedCount = visibleResults.failedRows;
|
|
8882
8853
|
this.#options.onMapRowsSettled?.({
|
|
8883
8854
|
playName: this.#options.playName,
|
|
8884
8855
|
runId: this.#options.runId,
|
|
@@ -8904,7 +8875,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8904
8875
|
completedRowsCount: completedRows,
|
|
8905
8876
|
pendingRowsCount: 0,
|
|
8906
8877
|
failedRowsCount: failedCount,
|
|
8907
|
-
supersededRows:
|
|
8878
|
+
supersededRows: visibility.supersededRows,
|
|
8908
8879
|
startedAt:
|
|
8909
8880
|
this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
|
|
8910
8881
|
Date.now(),
|
|
@@ -8919,7 +8890,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8919
8890
|
completedRows,
|
|
8920
8891
|
failedRows: failedCount,
|
|
8921
8892
|
totalRows: totalInputCount,
|
|
8922
|
-
supersededRows:
|
|
8893
|
+
supersededRows: visibility.supersededRows,
|
|
8923
8894
|
at: Date.now(),
|
|
8924
8895
|
});
|
|
8925
8896
|
throw new AllAdmittedRowsFailedError({
|
|
@@ -8939,16 +8910,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8939
8910
|
totalRows: totalInputCount,
|
|
8940
8911
|
completedRowKeys: [],
|
|
8941
8912
|
pendingRowKeys: [],
|
|
8942
|
-
completedRowsCount:
|
|
8943
|
-
0,
|
|
8944
|
-
processedCount -
|
|
8945
|
-
failedCount -
|
|
8946
|
-
staleFailedKeys.size -
|
|
8947
|
-
staleCompletedKeys.size,
|
|
8948
|
-
),
|
|
8913
|
+
completedRowsCount: terminalProgress.completedRows,
|
|
8949
8914
|
pendingRowsCount: 0,
|
|
8950
8915
|
failedRowsCount: failedCount,
|
|
8951
|
-
supersededRows:
|
|
8916
|
+
supersededRows: visibility.supersededRows,
|
|
8952
8917
|
startedAt:
|
|
8953
8918
|
this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
|
|
8954
8919
|
Date.now(),
|
|
@@ -8960,16 +8925,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8960
8925
|
mapNodeId: mapScope.mapNodeId ?? null,
|
|
8961
8926
|
logicalNamespace: mapScope.logicalNamespace,
|
|
8962
8927
|
artifactTableNamespace: resolvedTableNamespace,
|
|
8963
|
-
completedRows:
|
|
8964
|
-
0,
|
|
8965
|
-
processedCount -
|
|
8966
|
-
failedCount -
|
|
8967
|
-
staleFailedKeys.size -
|
|
8968
|
-
staleCompletedKeys.size,
|
|
8969
|
-
),
|
|
8928
|
+
completedRows: terminalProgress.completedRows,
|
|
8970
8929
|
failedRows: failedCount,
|
|
8971
8930
|
totalRows: totalInputCount,
|
|
8972
|
-
supersededRows:
|
|
8931
|
+
supersededRows: visibility.supersededRows,
|
|
8973
8932
|
at: Date.now(),
|
|
8974
8933
|
});
|
|
8975
8934
|
|
|
@@ -9373,6 +9332,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9373
9332
|
)
|
|
9374
9333
|
: row,
|
|
9375
9334
|
index,
|
|
9335
|
+
originalIndex: itemOriginalIndexes[index] ?? index,
|
|
9376
9336
|
};
|
|
9377
9337
|
});
|
|
9378
9338
|
const admittedItems =
|
|
@@ -9383,7 +9343,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9383
9343
|
: seededItems;
|
|
9384
9344
|
rawItems = admittedItems.map((item) => item.row);
|
|
9385
9345
|
itemsToProcess = admittedItems.map((item) => item.row);
|
|
9386
|
-
itemOriginalIndexes = admittedItems.map((item) => item.
|
|
9346
|
+
itemOriginalIndexes = admittedItems.map((item) => item.originalIndex);
|
|
9387
9347
|
totalInputCount = resumedPendingRowsOnly
|
|
9388
9348
|
? Math.max(replayMapFrame?.totalRows ?? 0, admittedItems.length)
|
|
9389
9349
|
: admittedItems.length;
|
|
@@ -9399,6 +9359,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9399
9359
|
rowIdentity(this.toOutputRow(item), itemOriginalIndexes[index] ?? index),
|
|
9400
9360
|
);
|
|
9401
9361
|
this.setMapFrame({
|
|
9362
|
+
...(resumedPendingRowsOnly ? replayMapFrame : undefined),
|
|
9402
9363
|
mapInvocationId: mapScope.mapInvocationId,
|
|
9403
9364
|
mapNodeId: mapScope.mapNodeId ?? null,
|
|
9404
9365
|
logicalNamespace: mapScope.logicalNamespace,
|
|
@@ -9407,7 +9368,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9407
9368
|
totalRows: totalInputCount,
|
|
9408
9369
|
completedRowKeys,
|
|
9409
9370
|
pendingRowKeys,
|
|
9371
|
+
completedRowsCount: previouslyCompletedRowsCount,
|
|
9410
9372
|
failedRowsCount: previouslyFailedRowsCount,
|
|
9373
|
+
supersededRows: visibility.supersededRows,
|
|
9374
|
+
activeBoundaryId: null,
|
|
9411
9375
|
startedAt: resumedPendingRowsOnly
|
|
9412
9376
|
? (replayMapFrame?.startedAt ?? Date.now())
|
|
9413
9377
|
: Date.now(),
|
|
@@ -9420,8 +9384,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9420
9384
|
logicalNamespace: mapScope.logicalNamespace,
|
|
9421
9385
|
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
9422
9386
|
totalRows: totalInputCount,
|
|
9423
|
-
completedRows:
|
|
9387
|
+
completedRows: previouslyCompletedRowsCount,
|
|
9388
|
+
failedRows: previouslyFailedRowsCount,
|
|
9424
9389
|
pendingRows: pendingRowKeys.length,
|
|
9390
|
+
supersededRows: visibility.supersededRows,
|
|
9425
9391
|
at: Date.now(),
|
|
9426
9392
|
});
|
|
9427
9393
|
|
|
@@ -9479,7 +9445,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9479
9445
|
),
|
|
9480
9446
|
staticPipeline: this.currentStaticPipeline ?? null,
|
|
9481
9447
|
});
|
|
9482
|
-
|
|
9448
|
+
visibility.recordAcceptedWrite(writeResult, chunk, {
|
|
9449
|
+
tableNamespace: resolvedTableNamespace,
|
|
9450
|
+
runId: this.currentRunId,
|
|
9451
|
+
});
|
|
9483
9452
|
this.resourceGovernor.observe({
|
|
9484
9453
|
sheetFlushBytes: chunkBytes,
|
|
9485
9454
|
sheetFlushLatencyMs: Date.now() - flushStartedAt,
|
|
@@ -9507,7 +9476,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9507
9476
|
await flushChunk();
|
|
9508
9477
|
};
|
|
9509
9478
|
const incrementalPersistence = this.#options.onMapRowsCompleted
|
|
9510
|
-
? createRuntimeMapRowPersistence(persistMapRows, {
|
|
9479
|
+
? createRuntimeMapRowPersistence(persistMapRows, visibility, {
|
|
9511
9480
|
prepareTerminalRow: (row) =>
|
|
9512
9481
|
this.withPendingMapRowData(row, resolvedTableNamespace),
|
|
9513
9482
|
persistCheckpoint: this.#options.onMapRowsCheckpoint
|
|
@@ -9556,6 +9525,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9556
9525
|
: {}),
|
|
9557
9526
|
retainedRowsMemoryBudgetBytes:
|
|
9558
9527
|
mapMemoryLimits.materializedBudgetBytes,
|
|
9528
|
+
progressCompletedOffset:
|
|
9529
|
+
previouslyReusedRowsCount + duplicateReuseCount,
|
|
9559
9530
|
progressFailedOffset: previouslyFailedRowsCount,
|
|
9560
9531
|
incrementalPersistence,
|
|
9561
9532
|
},
|
|
@@ -9576,6 +9547,41 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9576
9547
|
this.clearActiveMapCheckpointUpdates();
|
|
9577
9548
|
}
|
|
9578
9549
|
}
|
|
9550
|
+
// Persist executed rows to the tenant runtime sheet — the sheet is the
|
|
9551
|
+
// source of truth, not this in-memory results array. Chunked by rows AND
|
|
9552
|
+
// bytes so large cells (scraped pages) never produce oversized writes.
|
|
9553
|
+
if (mapResult.completedRows.length > 0 || mapResult.failedRows.length > 0) {
|
|
9554
|
+
await incrementalPersistence?.flush();
|
|
9555
|
+
const mapCellMeta = this.activeMapCellMeta;
|
|
9556
|
+
const persistRows: PersistableMapRow[] = [];
|
|
9557
|
+
for (const row of mapResult.completedRows) {
|
|
9558
|
+
if (incrementalPersistence?.isPersisted(row)) continue;
|
|
9559
|
+
const rowKey = row.key;
|
|
9560
|
+
const meta = mapCellMeta?.get(rowKey);
|
|
9561
|
+
persistRows.push(
|
|
9562
|
+
completedMapRowOutcome({
|
|
9563
|
+
key: rowKey,
|
|
9564
|
+
data: row.data,
|
|
9565
|
+
cellMetaPatch: {
|
|
9566
|
+
...(row.cellMetaPatch ?? {}),
|
|
9567
|
+
...(meta ?? {}),
|
|
9568
|
+
},
|
|
9569
|
+
}),
|
|
9570
|
+
);
|
|
9571
|
+
}
|
|
9572
|
+
persistRows.push(
|
|
9573
|
+
...mapResult.failedRows.filter(
|
|
9574
|
+
(row) => !incrementalPersistence?.isPersisted(row),
|
|
9575
|
+
),
|
|
9576
|
+
);
|
|
9577
|
+
const persistStartedAt = Date.now();
|
|
9578
|
+
await persistMapRows(persistRows);
|
|
9579
|
+
if (persistRows.length > 0) {
|
|
9580
|
+
this.runtimeLog(
|
|
9581
|
+
`Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
|
|
9582
|
+
);
|
|
9583
|
+
}
|
|
9584
|
+
}
|
|
9579
9585
|
const visibleCompletedRows =
|
|
9580
9586
|
mapResult.completedRows.filter(isVisibleTerminalRow);
|
|
9581
9587
|
const visibleFailedRows = mapResult.failedRows.filter(isVisibleTerminalRow);
|
|
@@ -9597,9 +9603,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9597
9603
|
const rowKey = rowIdentity(rawItem, index);
|
|
9598
9604
|
if (
|
|
9599
9605
|
rowKey &&
|
|
9600
|
-
(failedRowKeys.has(rowKey) ||
|
|
9601
|
-
(staleCompletionKeys.has(rowKey) &&
|
|
9602
|
-
!recoveredStaleCompletionKeys.has(rowKey)))
|
|
9606
|
+
(failedRowKeys.has(rowKey) || !visibility.isVisible(rowKey))
|
|
9603
9607
|
) {
|
|
9604
9608
|
return [];
|
|
9605
9609
|
}
|
|
@@ -9613,52 +9617,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9613
9617
|
previouslyFailedRowsCount;
|
|
9614
9618
|
const reusedCount = duplicateReuseCount;
|
|
9615
9619
|
|
|
9616
|
-
// Persist executed rows to the tenant runtime sheet — the sheet is the
|
|
9617
|
-
// source of truth, not this in-memory results array. Chunked by rows AND
|
|
9618
|
-
// bytes so large cells (scraped pages) never produce oversized writes.
|
|
9619
|
-
if (resultsByKey.size > 0 || visibleFailedRows.length > 0) {
|
|
9620
|
-
await incrementalPersistence?.flush();
|
|
9621
|
-
const mapCellMeta = this.activeMapCellMeta;
|
|
9622
|
-
const persistRows: PersistableMapRow[] = [];
|
|
9623
|
-
for (const row of visibleCompletedRows) {
|
|
9624
|
-
if (incrementalPersistence?.isPersisted(row)) continue;
|
|
9625
|
-
const rowKey = row.key;
|
|
9626
|
-
const meta = mapCellMeta?.get(rowKey);
|
|
9627
|
-
persistRows.push(
|
|
9628
|
-
completedMapRowOutcome({
|
|
9629
|
-
key: rowKey,
|
|
9630
|
-
data: row.data,
|
|
9631
|
-
cellMetaPatch: {
|
|
9632
|
-
...(row.cellMetaPatch ?? {}),
|
|
9633
|
-
...(meta ?? {}),
|
|
9634
|
-
},
|
|
9635
|
-
}),
|
|
9636
|
-
);
|
|
9637
|
-
}
|
|
9638
|
-
persistRows.push(
|
|
9639
|
-
...visibleFailedRows.filter(
|
|
9640
|
-
(row) => !incrementalPersistence?.isPersisted(row),
|
|
9641
|
-
),
|
|
9642
|
-
);
|
|
9643
|
-
const persistStartedAt = Date.now();
|
|
9644
|
-
await persistMapRows(persistRows);
|
|
9645
|
-
if (persistRows.length > 0) {
|
|
9646
|
-
this.runtimeLog(
|
|
9647
|
-
`Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
|
|
9648
|
-
);
|
|
9649
|
-
}
|
|
9650
|
-
}
|
|
9651
9620
|
this.clearActiveMapCellMeta();
|
|
9652
9621
|
this.clearActiveMapCheckpointUpdates();
|
|
9653
9622
|
|
|
9654
|
-
const
|
|
9655
|
-
|
|
9656
|
-
|
|
9657
|
-
reusedCount +
|
|
9658
|
-
visibleCompletedRows.length,
|
|
9623
|
+
const durableProgress = visibility.settlement(
|
|
9624
|
+
previouslyCompletedRowsCount + reusedCount + mapResult.completedRowCount,
|
|
9625
|
+
previouslyFailedRowsCount + mapResult.failedRowCount,
|
|
9659
9626
|
);
|
|
9660
|
-
const
|
|
9661
|
-
|
|
9627
|
+
const durableCompletedRows = durableProgress.completedRows;
|
|
9628
|
+
const durableFailedRows = durableProgress.failedRows;
|
|
9662
9629
|
const terminalFrame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
9663
9630
|
if (terminalFrame) {
|
|
9664
9631
|
this.setMapFrame({
|
|
@@ -9667,11 +9634,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9667
9634
|
// Incremental rows add keys after their commit. Pure maps use the
|
|
9668
9635
|
// durable aggregate count below instead of materializing up to millions
|
|
9669
9636
|
// of keys only to mark the terminal frame.
|
|
9670
|
-
completedRowKeys: terminalFrame.completedRowKeys
|
|
9637
|
+
completedRowKeys: terminalFrame.completedRowKeys.filter((key) =>
|
|
9638
|
+
visibility.isVisible(key),
|
|
9639
|
+
),
|
|
9671
9640
|
pendingRowKeys: [],
|
|
9672
9641
|
completedRowsCount: durableCompletedRows,
|
|
9673
9642
|
pendingRowsCount: 0,
|
|
9674
9643
|
failedRowsCount: durableFailedRows,
|
|
9644
|
+
supersededRows: visibility.supersededRows,
|
|
9675
9645
|
activeBoundaryId: null,
|
|
9676
9646
|
suspendedInputRows: undefined,
|
|
9677
9647
|
updatedAt: Date.now(),
|
|
@@ -9685,6 +9655,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9685
9655
|
artifactTableNamespace: resolvedTableNamespace,
|
|
9686
9656
|
completedRows: durableCompletedRows,
|
|
9687
9657
|
failedRows: durableFailedRows,
|
|
9658
|
+
supersededRows: visibility.supersededRows,
|
|
9688
9659
|
totalRows: totalInputCount,
|
|
9689
9660
|
...this.inlineChildAggregateEventFields(),
|
|
9690
9661
|
at: Date.now(),
|
|
@@ -9714,8 +9685,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9714
9685
|
expectedRows: Math.max(
|
|
9715
9686
|
0,
|
|
9716
9687
|
totalInputCount -
|
|
9717
|
-
|
|
9718
|
-
(resumedPendingRowsOnly
|
|
9688
|
+
visibility.supersededRows -
|
|
9689
|
+
(resumedPendingRowsOnly
|
|
9690
|
+
? durableFailedRows + previouslyReusedRowsCount
|
|
9691
|
+
: 0),
|
|
9719
9692
|
),
|
|
9720
9693
|
currentRows: results,
|
|
9721
9694
|
failedRowCount: resumedPendingRowsOnly ? 0 : visibleFailedRows.length,
|
|
@@ -9899,7 +9872,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9899
9872
|
retainedRowsMemoryBudgetBytes?: number;
|
|
9900
9873
|
progressCompletedOffset?: number | (() => number);
|
|
9901
9874
|
progressFailedOffset?: number | (() => number);
|
|
9902
|
-
progressSupersededOffset?: number | (() => number);
|
|
9903
9875
|
/** Input duplicates discovered after admission; excludes resolver outcomes. */
|
|
9904
9876
|
progressSkippedOffset?: number | (() => number);
|
|
9905
9877
|
progressTotalRows?: number | (() => number);
|
|
@@ -9987,7 +9959,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9987
9959
|
const failedRowsToPersist: PersistableMapRow[] = [];
|
|
9988
9960
|
const failureSamples: Array<{ rowLabel: string; error: string }> = [];
|
|
9989
9961
|
const failureSampleErrors = new Set<string>();
|
|
9990
|
-
const previewRows:
|
|
9962
|
+
const previewRows: Array<Pick<PersistableMapRow, 'key' | 'data'>> = [];
|
|
9991
9963
|
let completedRowCount = 0;
|
|
9992
9964
|
let failedRowCount = 0;
|
|
9993
9965
|
let retainedRowsComplete = true;
|
|
@@ -10084,6 +10056,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10084
10056
|
onCommitted: () => void,
|
|
10085
10057
|
): Promise<void> => {
|
|
10086
10058
|
if (!incrementalPersistence) {
|
|
10059
|
+
mapProgress.recordCommittedRows([row]);
|
|
10087
10060
|
onCommitted();
|
|
10088
10061
|
return Promise.resolve();
|
|
10089
10062
|
}
|
|
@@ -10131,199 +10104,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10131
10104
|
this.steps.push(datasetStep);
|
|
10132
10105
|
this.activeDatasetStep = datasetStep;
|
|
10133
10106
|
|
|
10134
|
-
|
|
10135
|
-
|
|
10136
|
-
|
|
10137
|
-
|
|
10138
|
-
|
|
10139
|
-
|
|
10140
|
-
|
|
10141
|
-
|
|
10142
|
-
|
|
10143
|
-
|
|
10144
|
-
|
|
10145
|
-
|
|
10146
|
-
|
|
10147
|
-
}
|
|
10148
|
-
|
|
10149
|
-
|
|
10150
|
-
// Scalars only: resolver admission is visible before the first row's
|
|
10151
|
-
// durable completion, without publishing one event per row or tool call.
|
|
10152
|
-
let startedResolverRows = 0;
|
|
10153
|
-
let activeResolverRows = 0;
|
|
10154
|
-
let tracksResolverActivity = false;
|
|
10155
|
-
// Persistence can move current resolvers between successful, failed and
|
|
10156
|
-
// superseded offsets. They already contribute to startedResolverRows:
|
|
10157
|
-
// only the settled baseline before this pool started belongs in admission.
|
|
10158
|
-
const initiallyStartedRows =
|
|
10159
|
-
liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
|
|
10160
|
-
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10161
|
-
liveNumber(runtimeOptions?.progressSupersededOffset, 0);
|
|
10162
|
-
const initiallySkippedRows = liveNumber(
|
|
10163
|
-
runtimeOptions?.progressSkippedOffset,
|
|
10164
|
-
0,
|
|
10107
|
+
const mapProgress = this.mapFrames.progress(mapScope, {
|
|
10108
|
+
totalRows: currentTotalRows,
|
|
10109
|
+
completedOffset: runtimeOptions?.progressCompletedOffset,
|
|
10110
|
+
failedOffset: runtimeOptions?.progressFailedOffset,
|
|
10111
|
+
visibility:
|
|
10112
|
+
incrementalPersistence?.visibility ??
|
|
10113
|
+
new MapExecutionVisibility(
|
|
10114
|
+
this.checkpoint.mapFrames?.[mapScope.mapInvocationId]
|
|
10115
|
+
?.supersededRows ?? 0,
|
|
10116
|
+
),
|
|
10117
|
+
skippedInputs: runtimeOptions?.progressSkippedOffset,
|
|
10118
|
+
checkpointRows: rowFeed ? 'counts' : 'keys',
|
|
10119
|
+
eventFields: () => this.inlineChildAggregateEventFields(),
|
|
10120
|
+
});
|
|
10121
|
+
incrementalPersistence?.observeCommittedRows((rows) =>
|
|
10122
|
+
mapProgress.recordCommittedRows(rows),
|
|
10165
10123
|
);
|
|
10166
|
-
const mapActivityCounts = () => {
|
|
10167
|
-
const startedRows =
|
|
10168
|
-
initiallyStartedRows +
|
|
10169
|
-
(liveNumber(
|
|
10170
|
-
runtimeOptions?.progressSkippedOffset,
|
|
10171
|
-
initiallySkippedRows,
|
|
10172
|
-
) -
|
|
10173
|
-
initiallySkippedRows) +
|
|
10174
|
-
startedResolverRows;
|
|
10175
|
-
return {
|
|
10176
|
-
startedRows,
|
|
10177
|
-
activeRows: activeResolverRows,
|
|
10178
|
-
waitingRows: currentTotalRows() - startedRows,
|
|
10179
|
-
};
|
|
10180
|
-
};
|
|
10181
|
-
const updateMapFrameProgress = (input: {
|
|
10182
|
-
status?: MapExecutionFrame['status'];
|
|
10183
|
-
completedRowKey?: string | null;
|
|
10184
|
-
pendingRowKey?: string | null;
|
|
10185
|
-
completedRowKeys?: string[];
|
|
10186
|
-
activeBoundaryId?: string | null;
|
|
10187
|
-
failedDelta?: number;
|
|
10188
|
-
failedRowKey?: string | null;
|
|
10189
|
-
pendingRowKeys?: string[];
|
|
10190
|
-
emitEventType?: PlayExecutionEvent['type'];
|
|
10191
|
-
}) => {
|
|
10192
|
-
const existing =
|
|
10193
|
-
this.checkpoint.mapFrames?.[mapScope.mapInvocationId] ?? null;
|
|
10194
|
-
if (!existing) {
|
|
10195
|
-
return;
|
|
10196
|
-
}
|
|
10197
|
-
liveFrameSets ??= {
|
|
10198
|
-
completedRowKeys: new Set(existing.completedRowKeys),
|
|
10199
|
-
pendingRowKeys: new Set(existing.pendingRowKeys),
|
|
10200
|
-
// Failure counts are explicit row outcomes, not the complement of the
|
|
10201
|
-
// live key sets. The sets may be intentionally sparse for a fed map
|
|
10202
|
-
// and may lag the durable row writer while a batch is in flight.
|
|
10203
|
-
failedRowsCount: 0,
|
|
10204
|
-
};
|
|
10205
|
-
const { completedRowKeys, pendingRowKeys } = liveFrameSets;
|
|
10206
|
-
const completedBefore = completedRowKeys.size;
|
|
10207
|
-
if (input.completedRowKey?.trim()) {
|
|
10208
|
-
const completedRowKey = input.completedRowKey.trim();
|
|
10209
|
-
completedRowKeys.add(completedRowKey);
|
|
10210
|
-
pendingRowKeys.delete(completedRowKey);
|
|
10211
|
-
}
|
|
10212
|
-
if (input.completedRowKeys) {
|
|
10213
|
-
for (const rawKey of input.completedRowKeys) {
|
|
10214
|
-
const completedRowKey = rawKey.trim();
|
|
10215
|
-
if (!completedRowKey) continue;
|
|
10216
|
-
completedRowKeys.add(completedRowKey);
|
|
10217
|
-
pendingRowKeys.delete(completedRowKey);
|
|
10218
|
-
}
|
|
10219
|
-
}
|
|
10220
|
-
if (input.pendingRowKey?.trim()) {
|
|
10221
|
-
pendingRowKeys.add(input.pendingRowKey.trim());
|
|
10222
|
-
}
|
|
10223
|
-
if (input.pendingRowKeys) {
|
|
10224
|
-
for (const rawKey of input.pendingRowKeys) {
|
|
10225
|
-
const pendingRowKey = rawKey.trim();
|
|
10226
|
-
if (pendingRowKey && !completedRowKeys.has(pendingRowKey)) {
|
|
10227
|
-
pendingRowKeys.add(pendingRowKey);
|
|
10228
|
-
}
|
|
10229
|
-
}
|
|
10230
|
-
}
|
|
10231
|
-
if (input.failedRowKey?.trim()) {
|
|
10232
|
-
pendingRowKeys.delete(input.failedRowKey.trim());
|
|
10233
|
-
}
|
|
10234
|
-
liveFrameSets.failedRowsCount += Math.max(0, input.failedDelta ?? 0);
|
|
10235
|
-
rowsSinceFrameFlush += completedRowKeys.size - completedBefore;
|
|
10236
|
-
|
|
10237
|
-
const isTransition =
|
|
10238
|
-
input.status !== undefined ||
|
|
10239
|
-
input.activeBoundaryId !== undefined ||
|
|
10240
|
-
input.emitEventType !== undefined;
|
|
10241
|
-
const now = Date.now();
|
|
10242
|
-
// Small maps keep per-row progress (live UX, negligible cost); the
|
|
10243
|
-
// row/time throttle only kicks in where the per-row materialization
|
|
10244
|
-
// cost matters.
|
|
10245
|
-
const shouldFlush =
|
|
10246
|
-
isTransition ||
|
|
10247
|
-
totalRows <= MAP_FRAME_FLUSH_ROW_INTERVAL ||
|
|
10248
|
-
rowsSinceFrameFlush >= MAP_FRAME_FLUSH_ROW_INTERVAL ||
|
|
10249
|
-
now - lastFrameFlushAt >= MAP_FRAME_FLUSH_INTERVAL_MS;
|
|
10250
|
-
if (shouldFlush) {
|
|
10251
|
-
lastFrameFlushAt = now;
|
|
10252
|
-
rowsSinceFrameFlush = 0;
|
|
10253
|
-
const nextFrame: MapExecutionFrame = {
|
|
10254
|
-
...existing,
|
|
10255
|
-
status: input.status ?? existing.status,
|
|
10256
|
-
// A fed map spans every source page, so its frame carries counts
|
|
10257
|
-
// only: the Runtime Sheet is the durable row record, and a key list
|
|
10258
|
-
// that grows with the dataset would be copied and shipped on every
|
|
10259
|
-
// progress flush.
|
|
10260
|
-
...(rowFeed
|
|
10261
|
-
? {
|
|
10262
|
-
completedRowKeys: [],
|
|
10263
|
-
pendingRowKeys: [],
|
|
10264
|
-
completedRowsCount:
|
|
10265
|
-
liveNumber(runtimeOptions?.progressCompletedOffset, 0) +
|
|
10266
|
-
completedRowKeys.size,
|
|
10267
|
-
pendingRowsCount: pendingRowKeys.size,
|
|
10268
|
-
failedRowsCount:
|
|
10269
|
-
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10270
|
-
liveFrameSets.failedRowsCount,
|
|
10271
|
-
}
|
|
10272
|
-
: {
|
|
10273
|
-
completedRowKeys: [...completedRowKeys],
|
|
10274
|
-
pendingRowKeys: [...pendingRowKeys],
|
|
10275
|
-
failedRowsCount:
|
|
10276
|
-
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10277
|
-
liveFrameSets.failedRowsCount,
|
|
10278
|
-
}),
|
|
10279
|
-
supersededRows: liveNumber(
|
|
10280
|
-
runtimeOptions?.progressSupersededOffset,
|
|
10281
|
-
existing.supersededRows ?? 0,
|
|
10282
|
-
),
|
|
10283
|
-
...(input.activeBoundaryId !== undefined
|
|
10284
|
-
? { activeBoundaryId: input.activeBoundaryId }
|
|
10285
|
-
: {}),
|
|
10286
|
-
updatedAt: now,
|
|
10287
|
-
};
|
|
10288
|
-
this.setMapFrame(nextFrame);
|
|
10289
|
-
}
|
|
10290
|
-
const progressEventType =
|
|
10291
|
-
input.emitEventType ??
|
|
10292
|
-
(shouldFlush && completedRowKeys.size > completedBefore
|
|
10293
|
-
? 'map.progress'
|
|
10294
|
-
: null);
|
|
10295
|
-
if (progressEventType) {
|
|
10296
|
-
const failedRows = Math.max(
|
|
10297
|
-
0,
|
|
10298
|
-
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10299
|
-
liveFrameSets.failedRowsCount,
|
|
10300
|
-
);
|
|
10301
|
-
this.emitExecutionEvent({
|
|
10302
|
-
type: progressEventType,
|
|
10303
|
-
mapInvocationId: mapScope.mapInvocationId,
|
|
10304
|
-
mapNodeId: mapScope.mapNodeId ?? null,
|
|
10305
|
-
logicalNamespace: mapScope.logicalNamespace,
|
|
10306
|
-
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
10307
|
-
completedRows:
|
|
10308
|
-
liveNumber(runtimeOptions?.progressCompletedOffset, 0) +
|
|
10309
|
-
completedRowKeys.size,
|
|
10310
|
-
failedRows,
|
|
10311
|
-
totalRows: currentTotalRows(),
|
|
10312
|
-
...(progressEventType === 'map.progress' && tracksResolverActivity
|
|
10313
|
-
? mapActivityCounts()
|
|
10314
|
-
: {}),
|
|
10315
|
-
supersededRows: liveNumber(
|
|
10316
|
-
runtimeOptions?.progressSupersededOffset,
|
|
10317
|
-
existing.supersededRows ?? 0,
|
|
10318
|
-
),
|
|
10319
|
-
// Inline child aggregates ride the single-writer progress event so
|
|
10320
|
-
// fan-out never contends on a per-child mutation. See ADR 0013.
|
|
10321
|
-
...this.inlineChildAggregateEventFields(),
|
|
10322
|
-
at: Date.now(),
|
|
10323
|
-
} as PlayExecutionEvent);
|
|
10324
|
-
}
|
|
10325
|
-
};
|
|
10326
|
-
|
|
10327
10124
|
if (this.canUsePureJsMapFastPath(definition)) {
|
|
10328
10125
|
const results = await this.runPureFieldMap(
|
|
10329
10126
|
items,
|
|
@@ -10353,7 +10150,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10353
10150
|
// post-hoc map.progress events and re-copying the completed-keys array
|
|
10354
10151
|
// per row — all theater, no live progress value.
|
|
10355
10152
|
if (!incrementalPersistence) {
|
|
10356
|
-
|
|
10153
|
+
mapProgress.update({
|
|
10357
10154
|
completedRowKeys: results.map((_row, index) =>
|
|
10358
10155
|
executionRowKey(
|
|
10359
10156
|
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
@@ -10363,7 +10160,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10363
10160
|
});
|
|
10364
10161
|
}
|
|
10365
10162
|
if (emitTerminalEvent) {
|
|
10366
|
-
|
|
10163
|
+
mapProgress.update({
|
|
10367
10164
|
status: 'completed',
|
|
10368
10165
|
activeBoundaryId: null,
|
|
10369
10166
|
emitEventType: 'map.completed',
|
|
@@ -10374,22 +10171,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10374
10171
|
this.runtimeLog(
|
|
10375
10172
|
`Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`,
|
|
10376
10173
|
);
|
|
10174
|
+
const pureCompletedRows = results.map((row, index) =>
|
|
10175
|
+
completedMapRowOutcome({
|
|
10176
|
+
key: executionRowKey(
|
|
10177
|
+
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
10178
|
+
index,
|
|
10179
|
+
),
|
|
10180
|
+
inputIndex: executionRowIndex(index),
|
|
10181
|
+
data: this.toPersistedOutputRow(row),
|
|
10182
|
+
}),
|
|
10183
|
+
);
|
|
10377
10184
|
return {
|
|
10378
|
-
completedRows:
|
|
10379
|
-
completedMapRowOutcome({
|
|
10380
|
-
key: executionRowKey(
|
|
10381
|
-
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
10382
|
-
index,
|
|
10383
|
-
),
|
|
10384
|
-
inputIndex: executionRowIndex(index),
|
|
10385
|
-
data: this.toPersistedOutputRow(row),
|
|
10386
|
-
}),
|
|
10387
|
-
),
|
|
10185
|
+
completedRows: pureCompletedRows,
|
|
10388
10186
|
failedRows: [],
|
|
10389
10187
|
completedRowCount: results.length,
|
|
10390
10188
|
failedRowCount: 0,
|
|
10391
10189
|
failureSamples: [],
|
|
10392
|
-
previewRows:
|
|
10190
|
+
previewRows: pureCompletedRows.slice(0, 5),
|
|
10393
10191
|
retainedRowsComplete: true,
|
|
10394
10192
|
};
|
|
10395
10193
|
}
|
|
@@ -10401,7 +10199,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10401
10199
|
for (let index = fromIndex; index < toIndex; index += 1) {
|
|
10402
10200
|
appendedKeys.push(keyForIndex(index));
|
|
10403
10201
|
}
|
|
10404
|
-
|
|
10202
|
+
mapProgress.update({ pendingRowKeys: appendedKeys });
|
|
10405
10203
|
});
|
|
10406
10204
|
|
|
10407
10205
|
// Bound on how many row resolver bodies exist at once. An explicit
|
|
@@ -10662,7 +10460,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10662
10460
|
rowKey,
|
|
10663
10461
|
normalizedTableNamespace,
|
|
10664
10462
|
);
|
|
10665
|
-
updateMapFrameProgress({ failedDelta: 1, failedRowKey: rowKey });
|
|
10666
10463
|
});
|
|
10667
10464
|
return FAILED_ROW;
|
|
10668
10465
|
} finally {
|
|
@@ -10753,19 +10550,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10753
10550
|
});
|
|
10754
10551
|
completedRowCount += 1;
|
|
10755
10552
|
if (previewRows.length < 5) {
|
|
10756
|
-
previewRows.push(
|
|
10553
|
+
previewRows.push({ key: completedRow.key, data: completedRow.data });
|
|
10757
10554
|
}
|
|
10758
10555
|
retainPersistedRow(completedRow, completedRowsToPersist);
|
|
10759
10556
|
await enqueueIncrementalPersist(completedRow, () => {
|
|
10760
10557
|
this.clearActiveMapCheckpointUpdate(rowKey, normalizedTableNamespace);
|
|
10761
|
-
updateMapFrameProgress({ completedRowKey: rowKey });
|
|
10762
10558
|
});
|
|
10763
10559
|
return COMPLETED_ROW;
|
|
10764
10560
|
} catch (error) {
|
|
10765
10561
|
if (isPlayRowExecutionSuspendedError(error)) {
|
|
10766
10562
|
waitingBaseRows.set(idx, baseRow);
|
|
10767
10563
|
this.pendingRowEventBoundaries.push(error.boundary);
|
|
10768
|
-
|
|
10564
|
+
mapProgress.update({
|
|
10769
10565
|
pendingRowKey: rowKey,
|
|
10770
10566
|
activeBoundaryId: error.boundary.boundaryId,
|
|
10771
10567
|
emitEventType: 'map.suspended',
|
|
@@ -10794,7 +10590,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10794
10590
|
retryAtMs: error.retryAtMs,
|
|
10795
10591
|
});
|
|
10796
10592
|
providerBackpressureRowCount += 1;
|
|
10797
|
-
|
|
10593
|
+
mapProgress.update({ pendingRowKey: rowKey });
|
|
10798
10594
|
const fieldName = activeFieldName;
|
|
10799
10595
|
if (!fieldName) {
|
|
10800
10596
|
throw new Error(
|
|
@@ -10849,20 +10645,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10849
10645
|
},
|
|
10850
10646
|
];
|
|
10851
10647
|
});
|
|
10852
|
-
|
|
10853
|
-
this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
10854
|
-
this.setMapFrame({
|
|
10855
|
-
...(existingFrame ?? {
|
|
10856
|
-
mapInvocationId: mapScope.mapInvocationId,
|
|
10857
|
-
mapNodeId: mapScope.mapNodeId ?? null,
|
|
10858
|
-
logicalNamespace: mapScope.logicalNamespace,
|
|
10859
|
-
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
10860
|
-
totalRows: currentTotalRows(),
|
|
10861
|
-
completedRowKeys: [],
|
|
10862
|
-
pendingRowKeys: [],
|
|
10863
|
-
startedAt: Date.now(),
|
|
10864
|
-
}),
|
|
10865
|
-
status: 'suspended',
|
|
10648
|
+
mapProgress.suspend({
|
|
10866
10649
|
pendingRowKeys: suspendedInputRows.map(({ data, inputIndex }) =>
|
|
10867
10650
|
executionRowKey(data, inputIndex),
|
|
10868
10651
|
),
|
|
@@ -10871,18 +10654,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10871
10654
|
error.suspension.kind === 'sleep'
|
|
10872
10655
|
? error.suspension.boundaryId
|
|
10873
10656
|
: null,
|
|
10874
|
-
updatedAt: Date.now(),
|
|
10875
|
-
});
|
|
10876
|
-
this.emitExecutionEvent({
|
|
10877
|
-
type: 'map.suspended',
|
|
10878
|
-
mapInvocationId: mapScope.mapInvocationId,
|
|
10879
|
-
mapNodeId: mapScope.mapNodeId ?? null,
|
|
10880
|
-
logicalNamespace: mapScope.logicalNamespace,
|
|
10881
|
-
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
10882
|
-
completedRows: completedKeys.size,
|
|
10883
|
-
failedRows: 0,
|
|
10884
|
-
totalRows: currentTotalRows(),
|
|
10885
|
-
at: Date.now(),
|
|
10886
10657
|
});
|
|
10887
10658
|
// ctx.sleep() emitted its boundary checkpoint before control reached
|
|
10888
10659
|
// this row catch. Emit once more after recording the admitted plan so
|
|
@@ -10892,7 +10663,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10892
10663
|
throw error;
|
|
10893
10664
|
}
|
|
10894
10665
|
const fieldName = rowContext.getStore()?.fieldName;
|
|
10895
|
-
|
|
10666
|
+
mapProgress.update({
|
|
10896
10667
|
failedDelta: 1,
|
|
10897
10668
|
status: 'failed',
|
|
10898
10669
|
emitEventType: 'map.failed',
|
|
@@ -10959,34 +10730,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10959
10730
|
items[idx] = null as unknown as T;
|
|
10960
10731
|
}
|
|
10961
10732
|
: null;
|
|
10962
|
-
|
|
10963
|
-
tracksResolverActivity = true;
|
|
10964
|
-
const publishMapActivity = () => {
|
|
10965
|
-
const counts = mapActivityCounts();
|
|
10966
|
-
const key = `${counts.startedRows}:${counts.activeRows}:${counts.waitingRows}`;
|
|
10967
|
-
if (key === lastMapActivityKey) return;
|
|
10968
|
-
lastMapActivityKey = key;
|
|
10969
|
-
// Admission samples do not rebuild checkpoint key sets. The map's
|
|
10970
|
-
// writer continues to own durable settlement and checkpoint flushing.
|
|
10971
|
-
this.emitExecutionEvent({
|
|
10972
|
-
type: 'map.progress',
|
|
10973
|
-
mapInvocationId: mapScope.mapInvocationId,
|
|
10974
|
-
mapNodeId: mapScope.mapNodeId ?? null,
|
|
10975
|
-
logicalNamespace: mapScope.logicalNamespace,
|
|
10976
|
-
artifactTableNamespace: mapScope.artifactTableNamespace,
|
|
10977
|
-
completedRows:
|
|
10978
|
-
liveNumber(runtimeOptions?.progressCompletedOffset, completedRows) +
|
|
10979
|
-
(liveFrameSets?.completedRowKeys.size ?? 0),
|
|
10980
|
-
failedRows:
|
|
10981
|
-
liveNumber(runtimeOptions?.progressFailedOffset, 0) +
|
|
10982
|
-
(liveFrameSets?.failedRowsCount ?? 0),
|
|
10983
|
-
totalRows: currentTotalRows(),
|
|
10984
|
-
supersededRows: liveNumber(runtimeOptions?.progressSupersededOffset, 0),
|
|
10985
|
-
...counts,
|
|
10986
|
-
...this.inlineChildAggregateEventFields(),
|
|
10987
|
-
at: Date.now(),
|
|
10988
|
-
});
|
|
10989
|
-
};
|
|
10733
|
+
const publishMapActivity = () => mapProgress.publishActivity();
|
|
10990
10734
|
const mapActivityTimer = setInterval(
|
|
10991
10735
|
publishMapActivity,
|
|
10992
10736
|
PROGRESS_HEARTBEAT_INTERVAL_MS,
|
|
@@ -11008,12 +10752,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11008
10752
|
releaseDispatchedInputRow?.(idx);
|
|
11009
10753
|
settledResults[idx] = await mapRowFlow.run({ baseRow }, async () => {
|
|
11010
10754
|
this.assertRowAdmissionOpen();
|
|
11011
|
-
|
|
11012
|
-
activeResolverRows += 1;
|
|
10755
|
+
mapProgress.resolverStarted();
|
|
11013
10756
|
try {
|
|
11014
10757
|
return await runRow(idx, baseRow);
|
|
11015
10758
|
} finally {
|
|
11016
|
-
|
|
10759
|
+
mapProgress.resolverFinished();
|
|
11017
10760
|
}
|
|
11018
10761
|
});
|
|
11019
10762
|
} catch (error) {
|
|
@@ -11251,24 +10994,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11251
10994
|
await incrementalPersistence?.checkpoint(checkpointUpdates);
|
|
11252
10995
|
await incrementalPersistence?.flush();
|
|
11253
10996
|
await this.runtimeStepReceiptCompletionWriter.flush();
|
|
11254
|
-
|
|
11255
|
-
|
|
11256
|
-
activeBoundaryId: null,
|
|
11257
|
-
emitEventType: 'map.suspended',
|
|
11258
|
-
});
|
|
11259
|
-
const frame = this.checkpoint.mapFrames?.[mapScope.mapInvocationId];
|
|
11260
|
-
if (!frame) {
|
|
11261
|
-
throw new Error('Suspended map lost its durable execution frame.');
|
|
11262
|
-
}
|
|
11263
|
-
this.setMapFrame({
|
|
11264
|
-
...frame,
|
|
11265
|
-
completedRowKeys: frame.completedRowKeys.filter(
|
|
11266
|
-
(key) => !waitingRowKeys.has(key),
|
|
11267
|
-
),
|
|
11268
|
-
pendingRowKeys: rowFeed ? [] : [...waitingRowKeys],
|
|
10997
|
+
mapProgress.suspend({
|
|
10998
|
+
pendingRowKeys: [...waitingRowKeys],
|
|
11269
10999
|
suspendedInputRows,
|
|
11270
11000
|
activeBoundaryId: null,
|
|
11271
|
-
updatedAt: Date.now(),
|
|
11272
11001
|
});
|
|
11273
11002
|
if (earliestHold && providerWakeAtMs !== null) {
|
|
11274
11003
|
this.runtimeLog(
|
|
@@ -11337,7 +11066,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11337
11066
|
// not announce a terminal state with that bounded egress still retained.
|
|
11338
11067
|
await this.runtimeStepReceiptCompletionWriter.flush();
|
|
11339
11068
|
if (emitTerminalEvent) {
|
|
11340
|
-
|
|
11069
|
+
mapProgress.update({
|
|
11341
11070
|
status: 'completed',
|
|
11342
11071
|
activeBoundaryId: null,
|
|
11343
11072
|
emitEventType: 'map.completed',
|