deepline 0.3.392 → 0.3.394
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +156 -24
- package/dist/bundling-sources/sdk/src/compat.ts +9 -0
- package/dist/bundling-sources/sdk/src/http.ts +5 -1
- package/dist/bundling-sources/sdk/src/index.ts +5 -0
- package/dist/bundling-sources/sdk/src/plays/run-request.ts +182 -6
- package/dist/bundling-sources/sdk/src/release.ts +4 -1
- package/dist/bundling-sources/sdk/src/types.ts +13 -0
- package/dist/bundling-sources/shared_libs/integration-contracts/request-warnings.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-storage-contract.ts +315 -0
- package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +41 -8
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +895 -54
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-policy.ts +28 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-invocation-fence.ts +36 -0
- package/dist/bundling-sources/shared_libs/play-runtime/freshness-receipt-execution.ts +251 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +11 -3
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +30 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +168 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +252 -10
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +92 -58
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +7 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +30 -7
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1161 -316
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +28 -15
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +24 -3
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +32 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +103 -23
- package/dist/bundling-sources/shared_libs/plays/core.ts +1 -1
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +172 -79
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +173 -1
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +18 -0
- package/dist/cli/auth-IHKISLJQ.js +39 -0
- package/dist/cli/{auth-6JSA2ZCW.mjs → auth-OSXDAVCF.mjs} +4 -4
- package/dist/cli/{billing-Q5BLQHIU.js → billing-EKLYOXUY.js} +101 -74
- package/dist/cli/{billing-MRRO643H.mjs → billing-ZNNKX3UR.mjs} +32 -5
- package/dist/cli/{chunk-4SFPVW7A.js → chunk-2MDDA7A4.js} +26 -26
- package/dist/cli/{chunk-JOFPQGXW.js → chunk-354JCXA2.js} +50 -50
- package/dist/cli/{chunk-7E66A24V.mjs → chunk-3HU47EGZ.mjs} +71 -9
- package/dist/cli/{chunk-DSWSEHRX.mjs → chunk-3HWALOL2.mjs} +1 -1
- package/dist/cli/{chunk-T7OKJ2T6.js → chunk-4P27AXEA.js} +73 -63
- package/dist/cli/{chunk-24VEN5PQ.mjs → chunk-5BB2RGAX.mjs} +190 -18
- package/dist/cli/{chunk-JSLPHFP7.js → chunk-6YAUMG27.js} +3 -2
- package/dist/cli/{chunk-MU6M3URX.mjs → chunk-BP33SIZF.mjs} +1 -1
- package/dist/cli/{chunk-WMS5U6YB.js → chunk-CS6RAIVY.js} +20 -19
- package/dist/cli/{chunk-CVLM3WMJ.js → chunk-E23BQ5SC.js} +242 -183
- package/dist/cli/{chunk-2GAHHSLB.mjs → chunk-FJGW5GXU.mjs} +3 -2
- package/dist/cli/{chunk-YN6YA7ZZ.mjs → chunk-JHIJIHFB.mjs} +79 -20
- package/dist/cli/{chunk-Q6EOHNRH.mjs → chunk-JI2B4RZP.mjs} +119 -6
- package/dist/cli/{chunk-5O4B6AUN.js → chunk-KB4SDX5S.js} +71 -9
- package/dist/cli/{chunk-RV6ALZIX.js → chunk-KGNTKMRG.js} +32 -7
- package/dist/cli/{chunk-6DBOU7UH.js → chunk-KI635JN4.js} +176 -87
- package/dist/cli/{chunk-2FQG2HU4.js → chunk-KIUMEOM5.js} +8 -8
- package/dist/cli/{chunk-Y6VOGHF2.mjs → chunk-KZZC33PY.mjs} +0 -181
- package/dist/cli/{chunk-2Q3GEFL4.mjs → chunk-M57J2SK4.mjs} +2 -2
- package/dist/cli/{chunk-R3O2CZ4H.mjs → chunk-MFB2VKBO.mjs} +2 -2
- package/dist/cli/{chunk-QGKPTBRR.js → chunk-MH73HMPV.js} +3 -3
- package/dist/cli/{chunk-BB3FNYFY.js → chunk-MLPU5SMZ.js} +2 -2
- package/dist/cli/{chunk-2G7DXFHI.mjs → chunk-NR3DFKNF.mjs} +27 -2
- package/dist/cli/{chunk-EY7RFC6A.js → chunk-QIQBYKLB.js} +15 -15
- package/dist/cli/{chunk-RIV2GS5E.js → chunk-RBJYWRCF.js} +638 -466
- package/dist/cli/{chunk-3VN4FTE3.js → chunk-RDXFGPJU.js} +1 -182
- package/dist/cli/{chunk-YAZCDBXX.mjs → chunk-TJI2EN6K.mjs} +3 -2
- package/dist/cli/chunk-UBKA33KY.mjs +310 -0
- package/dist/cli/{chunk-PAIMOGCM.js → chunk-UHNDVD3P.js} +125 -12
- package/dist/cli/{chunk-4TCT77UF.js → chunk-URWF7NGQ.js} +6 -1
- package/dist/cli/{chunk-2QLML6G4.mjs → chunk-UXFSWG3Q.mjs} +4 -4
- package/dist/cli/{chunk-UCKJQG25.js → chunk-VK6FYZLP.js} +8 -8
- package/dist/cli/{chunk-ZTACINO5.mjs → chunk-X63FDZJZ.mjs} +13 -3
- package/dist/cli/chunk-XDVT3GOM.js +310 -0
- package/dist/cli/{chunk-A3APGZFP.mjs → chunk-XRQ2AQL7.mjs} +2 -2
- package/dist/cli/{chunk-3FV4GBRK.mjs → chunk-XXUDVEXK.mjs} +97 -8
- package/dist/cli/{chunk-PQSA443V.mjs → chunk-ZI2ZNDJK.mjs} +6 -1
- package/dist/cli/{chunk-HGYBEAQP.mjs → chunk-ZYBGA4AR.mjs} +3 -3
- package/dist/cli/command-docs.js +2 -2
- package/dist/cli/command-docs.mjs +1 -1
- package/dist/cli/{csv-E4QGUIGI.mjs → csv-5AXRB35H.mjs} +3 -3
- package/dist/cli/{csv-26U5T24Z.js → csv-LU7O75WE.js} +12 -12
- package/dist/cli/{db-JTRJIAQW.mjs → db-7XP5UWEW.mjs} +6 -5
- package/dist/cli/{db-QHBFLMVL.js → db-F7YTFKVM.js} +31 -30
- package/dist/cli/{describe-FDV2V74F.js → describe-FRH32RDV.js} +24 -23
- package/dist/cli/{describe-E3DE3ZK3.mjs → describe-K4ES5LIL.mjs} +17 -16
- package/dist/cli/{executions-B3GTNPRE.mjs → executions-IEHEHJUO.mjs} +6 -5
- package/dist/cli/{executions-VZYOGVQY.js → executions-RZV5XPT2.js} +8 -7
- package/dist/cli/{feedback-CR55FJA2.js → feedback-3P65QTX3.js} +7 -7
- package/dist/cli/{feedback-4CEBR3OV.mjs → feedback-FPUVZBAN.mjs} +3 -3
- package/dist/cli/{git-NM2PHHZ4.mjs → git-GLVVP7TY.mjs} +4 -4
- package/dist/cli/git-MFFAWVVX.js +23 -0
- package/dist/cli/index.js +100 -96
- package/dist/cli/index.mjs +47 -43
- package/dist/cli/{legacy-noops-A7MSW5ND.mjs → legacy-noops-3MOI6NBM.mjs} +4 -4
- package/dist/cli/{legacy-noops-ZUA4YFIH.js → legacy-noops-XKSGZR7Q.js} +7 -7
- package/dist/cli/{monitors-NVPRQYP3.mjs → monitors-FTBGNEFG.mjs} +6 -5
- package/dist/cli/{monitors-ZE3TLZ4H.js → monitors-NZ2DYNST.js} +87 -86
- package/dist/cli/{org-U7A6DV5D.js → org-MCKDQXIV.js} +66 -65
- package/dist/cli/{org-KUBILNQV.mjs → org-V2J5RBCN.mjs} +6 -5
- package/dist/cli/{play-HRDXQ4QE.mjs → play-AJFGME3B.mjs} +11 -10
- package/dist/cli/play-WLSN2EO2.js +158 -0
- package/dist/cli/{providers-VD6YTVQZ.js → providers-2JEZ2264.js} +9 -8
- package/dist/cli/{providers-OHIWI453.mjs → providers-I6RQJZFZ.mjs} +6 -5
- package/dist/cli/{quickstart-UBV2FONF.mjs → quickstart-BIDQDMO4.mjs} +3 -3
- package/dist/cli/{quickstart-ETTCWPA4.js → quickstart-GJ2RPDU3.js} +5 -5
- package/dist/cli/run-identity-G257GQED.js +25 -0
- package/dist/cli/run-identity-MRNCEGV2.mjs +25 -0
- package/dist/cli/search-5VN4YD4I.js +27 -0
- package/dist/cli/{search-F44IZRT7.mjs → search-ELRXFWG5.mjs} +7 -6
- package/dist/cli/{secrets-CAEMRORF.js → secrets-53IT6VXD.js} +15 -14
- package/dist/cli/{secrets-R6XOJZKU.mjs → secrets-SQXGZXIC.mjs} +6 -5
- package/dist/cli/{sessions-RLN6UQS3.mjs → sessions-2X5QF7SL.mjs} +4 -4
- package/dist/cli/{sessions-6GC7OFTT.js → sessions-CAV5M77Y.js} +5 -5
- package/dist/cli/{settings-DPNVUAV2.js → settings-532WNAZO.js} +76 -75
- package/dist/cli/{settings-KIPAMRPP.mjs → settings-7P67VGCJ.mjs} +9 -8
- package/dist/cli/{setup-DTL3L5SG.js → setup-6L2VSUWJ.js} +41 -41
- package/dist/cli/{setup-TU2F7PU5.mjs → setup-QGGBV2LE.mjs} +6 -6
- package/dist/cli/{skills-UV5MYJ46.mjs → skills-3FE62WBG.mjs} +4 -4
- package/dist/cli/skills-S6NPZAN7.js +28 -0
- package/dist/cli/tools-647RLXQS.js +41 -0
- package/dist/cli/{tools-QRRNDCQ7.mjs → tools-DMR25DEF.mjs} +10 -9
- package/dist/cli/{update-4426TF35.mjs → update-6RZMHJZP.mjs} +6 -6
- package/dist/cli/update-WVTSC3EM.js +28 -0
- package/dist/cli/{workflow-N4U5VLZ3.mjs → workflow-L2DQRE6R.mjs} +7 -6
- package/dist/cli/{workflow-HNLOVISB.js → workflow-TX4F2PIX.js} +17 -16
- package/dist/{client-e6Qh5ZIg.d.mts → client-BktIPdIj.d.mts} +153 -4
- package/dist/{client-Uxuy-xQ2.d.ts → client-C3O1Ipf3.d.ts} +153 -4
- package/dist/{compiler-manifest-CY2v8ab0.d.ts → compiler-manifest-DLxUY5kI.d.mts} +124 -10
- package/dist/{compiler-manifest-CY2v8ab0.d.mts → compiler-manifest-DLxUY5kI.d.ts} +124 -10
- package/dist/{index-BZ6uzZJl.d.ts → index-B_GM9E3q.d.ts} +1 -1
- package/dist/{index-B9fBZH1g.d.mts → index-SCb0NvDb.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +120 -29
- package/dist/index.mjs +111 -20
- package/dist/install-integrity.json +94 -88
- package/dist/plays/bundle-play-file.d.mts +3 -3
- package/dist/plays/bundle-play-file.d.ts +3 -3
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/dist/plays/run-request.d.mts +2 -2
- package/dist/plays/run-request.d.ts +2 -2
- package/dist/plays/run-request.js +5 -2
- package/dist/plays/run-request.mjs +4 -1
- package/dist/release.d.mts +4 -1
- package/dist/release.d.ts +4 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-67TAELQL.js +0 -39
- package/dist/cli/git-I5JTK6YU.js +0 -23
- package/dist/cli/play-A2V7YDU7.js +0 -157
- package/dist/cli/search-WWXMBVPH.js +0 -26
- package/dist/cli/skills-SWOSEZLB.js +0 -28
- package/dist/cli/tools-IL2YCDQ6.js +0 -40
- package/dist/cli/update-BERDPEYR.js +0 -28
|
@@ -19,6 +19,7 @@ import { csvInputTextOptions } from '../../plays/csv-text-repair';
|
|
|
19
19
|
import { install as installSourceMapSupport } from 'source-map-support';
|
|
20
20
|
import {
|
|
21
21
|
createDeferredPlayDataset,
|
|
22
|
+
setPlayDatasetSourceRowIdentity,
|
|
22
23
|
isPlayDataset,
|
|
23
24
|
isSerializedPlayDataset,
|
|
24
25
|
iteratePlayDatasetInputPages,
|
|
@@ -128,6 +129,7 @@ import {
|
|
|
128
129
|
import {
|
|
129
130
|
resolveRuntimeReferencedPlay,
|
|
130
131
|
createCompleteRuntimeMapRowsOperation,
|
|
132
|
+
finishRuntimeSheetRunMembership,
|
|
131
133
|
createRuntimeSheetDatasetPageReader,
|
|
132
134
|
type RuntimeSheetDatasetPageReader,
|
|
133
135
|
readRuntimeSheetDatasetRowKeys,
|
|
@@ -140,6 +142,7 @@ import {
|
|
|
140
142
|
} from '../runtime-api';
|
|
141
143
|
import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
|
|
142
144
|
import { createRuntimeToolAuthScopeDigestResolver } from '../auth-scope-resolver';
|
|
145
|
+
import { RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER } from '../durable-invocation-fence';
|
|
143
146
|
import { createRunScopedChildPlayResolver } from './child-play-resolver';
|
|
144
147
|
import { normalizePlayContractCompatibility } from '../../plays/contracts';
|
|
145
148
|
import { ToolExecutionError } from '../../plays/tool-execution-error';
|
|
@@ -354,6 +357,7 @@ export function playRunnerReceiptStoreAdapterOptions() {
|
|
|
354
357
|
type RuntimeStepReceiptHandlers = RuntimeReceiptStoreHandlers;
|
|
355
358
|
|
|
356
359
|
type RuntimeToolMetadata = {
|
|
360
|
+
reusableResult?: { supported: boolean; unavailableReason: string | null };
|
|
357
361
|
provider?: string | null;
|
|
358
362
|
operation?: string | null;
|
|
359
363
|
extractors?: ToolResultMetadataInput['extractors'];
|
|
@@ -379,6 +383,8 @@ type RuntimeToolMetadata = {
|
|
|
379
383
|
|
|
380
384
|
type RuntimeToolAuthScope = {
|
|
381
385
|
digest?: unknown;
|
|
386
|
+
managedAsyncExecutionRecovery?: unknown;
|
|
387
|
+
managedAsyncExecutionRecoveryOrigin?: unknown;
|
|
382
388
|
rateScope?: {
|
|
383
389
|
bucketId?: unknown;
|
|
384
390
|
provider?: unknown;
|
|
@@ -534,7 +540,7 @@ export function runtimeSheetTerminalMapCounts(
|
|
|
534
540
|
}
|
|
535
541
|
|
|
536
542
|
async function vercelBypassHeaders(
|
|
537
|
-
config: PlayRunnerExecutionConfig,
|
|
543
|
+
config: Pick<PlayRunnerExecutionConfig, 'context'>,
|
|
538
544
|
): Promise<Record<string, string>> {
|
|
539
545
|
return vercelProtectionBypassHeader(
|
|
540
546
|
config.context.vercelProtectionBypassToken,
|
|
@@ -723,9 +729,14 @@ function createRuntimeToolMetadataResolver(config: PlayRunnerExecutionConfig) {
|
|
|
723
729
|
};
|
|
724
730
|
}
|
|
725
731
|
|
|
726
|
-
function createAppRuntimeToolAuthScopeDigestResolver(
|
|
727
|
-
config: PlayRunnerExecutionConfig,
|
|
732
|
+
export function createAppRuntimeToolAuthScopeDigestResolver(
|
|
733
|
+
config: Pick<PlayRunnerExecutionConfig, 'context'>,
|
|
728
734
|
) {
|
|
735
|
+
const asyncRecoveryOrigins = new Map<string, string>();
|
|
736
|
+
const executionRelayRecovery = new Map<
|
|
737
|
+
string,
|
|
738
|
+
Promise<{ origin: string } | null>
|
|
739
|
+
>();
|
|
729
740
|
const rateScopes = new Map<
|
|
730
741
|
string,
|
|
731
742
|
{ bucketId: string; provider: string; token: string }
|
|
@@ -783,6 +794,18 @@ function createAppRuntimeToolAuthScopeDigestResolver(
|
|
|
783
794
|
);
|
|
784
795
|
}
|
|
785
796
|
rateScopes.set(normalizedToolId, { bucketId, provider, token });
|
|
797
|
+
if (
|
|
798
|
+
response.headers.get(
|
|
799
|
+
RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER,
|
|
800
|
+
) === 'true' &&
|
|
801
|
+
body.managedAsyncExecutionRecovery === true &&
|
|
802
|
+
typeof body.managedAsyncExecutionRecoveryOrigin === 'string'
|
|
803
|
+
) {
|
|
804
|
+
asyncRecoveryOrigins.set(
|
|
805
|
+
normalizedToolId,
|
|
806
|
+
body.managedAsyncExecutionRecoveryOrigin,
|
|
807
|
+
);
|
|
808
|
+
}
|
|
786
809
|
return digest;
|
|
787
810
|
},
|
|
788
811
|
});
|
|
@@ -793,6 +816,10 @@ function createAppRuntimeToolAuthScopeDigestResolver(
|
|
|
793
816
|
const normalized = String(toolId || '').trim();
|
|
794
817
|
if (normalized) rateScopes.delete(normalized);
|
|
795
818
|
else rateScopes.clear();
|
|
819
|
+
if (normalized) asyncRecoveryOrigins.delete(normalized);
|
|
820
|
+
else asyncRecoveryOrigins.clear();
|
|
821
|
+
if (normalized) executionRelayRecovery.delete(normalized);
|
|
822
|
+
else executionRelayRecovery.clear();
|
|
796
823
|
},
|
|
797
824
|
rateScope: async (toolId: string, expectedProvider: string) => {
|
|
798
825
|
await digestResolver(toolId);
|
|
@@ -804,6 +831,60 @@ function createAppRuntimeToolAuthScopeDigestResolver(
|
|
|
804
831
|
}
|
|
805
832
|
return { bucketId: scope.bucketId, token: scope.token };
|
|
806
833
|
},
|
|
834
|
+
asyncExecutionRecovery: async (toolId: string) => {
|
|
835
|
+
const normalized = toolId.trim();
|
|
836
|
+
const digest = await digestResolver(normalized);
|
|
837
|
+
const receiptBase = config.context.baseUrl!.trim().replace(/\/$/, '');
|
|
838
|
+
const executionBase =
|
|
839
|
+
config.context.executionGatewayBaseUrl?.trim().replace(/\/$/, '') ||
|
|
840
|
+
receiptBase;
|
|
841
|
+
if (executionBase === receiptBase) {
|
|
842
|
+
const origin = asyncRecoveryOrigins.get(normalized);
|
|
843
|
+
return origin ? { origin } : null;
|
|
844
|
+
}
|
|
845
|
+
// Receipt and execution fleets roll independently. Keep canonical
|
|
846
|
+
// digest/rate truth on the receipt owner, but never treat its transport
|
|
847
|
+
// capability as proof that the execution relay forwards the app ACK.
|
|
848
|
+
let proof = executionRelayRecovery.get(normalized);
|
|
849
|
+
if (!proof) {
|
|
850
|
+
proof = (async () => {
|
|
851
|
+
const response = await fetchRunnerRuntimeControl({
|
|
852
|
+
url: `${executionBase}/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope`,
|
|
853
|
+
description: `Runtime async recovery capability for tool ${normalized}`,
|
|
854
|
+
retrySafe: true,
|
|
855
|
+
init: {
|
|
856
|
+
method: 'GET',
|
|
857
|
+
headers: {
|
|
858
|
+
Authorization: `Bearer ${config.context.executorToken}`,
|
|
859
|
+
...(await vercelBypassHeaders(config)),
|
|
860
|
+
},
|
|
861
|
+
},
|
|
862
|
+
});
|
|
863
|
+
if (
|
|
864
|
+
!response.ok ||
|
|
865
|
+
response.headers.get(
|
|
866
|
+
RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER,
|
|
867
|
+
) !== 'true'
|
|
868
|
+
)
|
|
869
|
+
return null;
|
|
870
|
+
const body = (await response.json()) as RuntimeToolAuthScope;
|
|
871
|
+
return body.digest === digest &&
|
|
872
|
+
body.managedAsyncExecutionRecovery === true &&
|
|
873
|
+
typeof body.managedAsyncExecutionRecoveryOrigin === 'string'
|
|
874
|
+
? { origin: body.managedAsyncExecutionRecoveryOrigin }
|
|
875
|
+
: null;
|
|
876
|
+
})().catch(() => {
|
|
877
|
+
// This optional capability cannot authorize replay on an unavailable
|
|
878
|
+
// hop. Preserve the existing unkeyed path and allow later discovery.
|
|
879
|
+
if (executionRelayRecovery.get(normalized) === proof) {
|
|
880
|
+
executionRelayRecovery.delete(normalized);
|
|
881
|
+
}
|
|
882
|
+
return null;
|
|
883
|
+
});
|
|
884
|
+
executionRelayRecovery.set(normalized, proof);
|
|
885
|
+
}
|
|
886
|
+
return await proof;
|
|
887
|
+
},
|
|
807
888
|
});
|
|
808
889
|
}
|
|
809
890
|
|
|
@@ -1122,11 +1203,19 @@ async function createStreamingCsvDataset(input: {
|
|
|
1122
1203
|
options?: CsvRenameOptions;
|
|
1123
1204
|
}): Promise<PlayDataset<Record<string, unknown>>> {
|
|
1124
1205
|
let cachedCount: number | null = null;
|
|
1206
|
+
const datasetId = createRuntimeDatasetId(
|
|
1207
|
+
input.playName,
|
|
1208
|
+
input.tableNamespace,
|
|
1209
|
+
);
|
|
1125
1210
|
const iterateProjected = () =>
|
|
1126
1211
|
({
|
|
1127
1212
|
async *[Symbol.asyncIterator]() {
|
|
1213
|
+
let rowIndex = 0;
|
|
1128
1214
|
for await (const row of input.source.iterate()) {
|
|
1129
|
-
yield
|
|
1215
|
+
yield setPlayDatasetSourceRowIdentity(
|
|
1216
|
+
applyCsvRenameProjection([row], input.options)[0]!,
|
|
1217
|
+
{ datasetId, rowIndex: rowIndex++ },
|
|
1218
|
+
);
|
|
1130
1219
|
}
|
|
1131
1220
|
},
|
|
1132
1221
|
}) as AsyncIterable<Record<string, unknown>>;
|
|
@@ -1142,7 +1231,7 @@ async function createStreamingCsvDataset(input: {
|
|
|
1142
1231
|
const previewRows = await collect(10);
|
|
1143
1232
|
return createDeferredPlayDataset({
|
|
1144
1233
|
datasetKind: 'csv',
|
|
1145
|
-
datasetId
|
|
1234
|
+
datasetId,
|
|
1146
1235
|
count: 0,
|
|
1147
1236
|
knownCount: null,
|
|
1148
1237
|
previewRows,
|
|
@@ -2204,6 +2293,7 @@ async function run(
|
|
|
2204
2293
|
writeVersion?: number;
|
|
2205
2294
|
}
|
|
2206
2295
|
>();
|
|
2296
|
+
const membershipCapturingMaps = new Set<string>();
|
|
2207
2297
|
const runtimeMapDatasetSettlements = new Map<
|
|
2208
2298
|
string,
|
|
2209
2299
|
{
|
|
@@ -2421,6 +2511,15 @@ async function run(
|
|
|
2421
2511
|
? 'invocation'
|
|
2422
2512
|
: null;
|
|
2423
2513
|
},
|
|
2514
|
+
getToolReusableResultSupport: async (toolId) => {
|
|
2515
|
+
const support = (await getRuntimeToolMetadata(toolId)).reusableResult;
|
|
2516
|
+
if (!support) {
|
|
2517
|
+
throw new Error(
|
|
2518
|
+
`Tool ${toolId} has no reusable-result capability metadata. Deploy the runtime metadata reader before admitting explicit-freshness artifacts.`,
|
|
2519
|
+
);
|
|
2520
|
+
}
|
|
2521
|
+
return support;
|
|
2522
|
+
},
|
|
2424
2523
|
getToolRetryPolicy: async (toolId) => {
|
|
2425
2524
|
const metadata = await getRuntimeToolMetadata(toolId);
|
|
2426
2525
|
return {
|
|
@@ -2449,6 +2548,8 @@ async function run(
|
|
|
2449
2548
|
];
|
|
2450
2549
|
},
|
|
2451
2550
|
getToolAuthScopeDigest: getRuntimeToolAuthScopeDigest,
|
|
2551
|
+
getToolAsyncExecutionRecovery:
|
|
2552
|
+
getRuntimeToolAuthScopeDigest.asyncExecutionRecovery,
|
|
2452
2553
|
getToolRateScope: getRuntimeToolAuthScopeDigest.rateScope,
|
|
2453
2554
|
invalidateToolAuthScopeDigest: (toolId) =>
|
|
2454
2555
|
getRuntimeToolAuthScopeDigest.invalidate?.(toolId),
|
|
@@ -2734,6 +2835,8 @@ async function run(
|
|
|
2734
2835
|
},
|
|
2735
2836
|
),
|
|
2736
2837
|
});
|
|
2838
|
+
if (mapStartResult.membershipCapture === true)
|
|
2839
|
+
membershipCapturingMaps.add(attemptKey);
|
|
2737
2840
|
const datasetId = createRuntimeDatasetId(playName, tableNamespace);
|
|
2738
2841
|
const datasetPath = `datasets.${tableNamespace}`;
|
|
2739
2842
|
const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
|
|
@@ -2889,11 +2992,66 @@ async function run(
|
|
|
2889
2992
|
),
|
|
2890
2993
|
});
|
|
2891
2994
|
},
|
|
2892
|
-
onMapRowsSettled: (input) => {
|
|
2893
|
-
const
|
|
2894
|
-
|
|
2895
|
-
input.
|
|
2896
|
-
|
|
2995
|
+
onMapRowsSettled: async (input) => {
|
|
2996
|
+
const playName = input.playName ?? config.context.playName!;
|
|
2997
|
+
const runId =
|
|
2998
|
+
input.runId ??
|
|
2999
|
+
config.context.runId ??
|
|
3000
|
+
config.context.workflowId ??
|
|
3001
|
+
'anonymous-run';
|
|
3002
|
+
const mapKey = runtimeSheetAttemptKey({
|
|
3003
|
+
playName,
|
|
3004
|
+
runId,
|
|
3005
|
+
tableNamespace: input.tableNamespace,
|
|
3006
|
+
});
|
|
3007
|
+
if (
|
|
3008
|
+
membershipCapturingMaps.has(mapKey) &&
|
|
3009
|
+
input.expectedRows !== undefined
|
|
3010
|
+
) {
|
|
3011
|
+
const sheetContract = resolveSheetContractForTableNamespace(
|
|
3012
|
+
input.staticPipeline ?? config.context.staticPipeline,
|
|
3013
|
+
input.tableNamespace,
|
|
3014
|
+
);
|
|
3015
|
+
if (!sheetContract)
|
|
3016
|
+
throw new Error(
|
|
3017
|
+
`Missing sheet contract while finishing membership for ${input.tableNamespace}.`,
|
|
3018
|
+
);
|
|
3019
|
+
const canUseRoot =
|
|
3020
|
+
runId === (config.context.runId ?? config.context.workflowId);
|
|
3021
|
+
await retryRuntimeSheetPersistence({
|
|
3022
|
+
description: `runtime membership finish table=${input.tableNamespace}`,
|
|
3023
|
+
log: (line) => emitWorkerLog(line, options),
|
|
3024
|
+
signal: persistenceSignal,
|
|
3025
|
+
run: () =>
|
|
3026
|
+
finishRuntimeSheetRunMembership(
|
|
3027
|
+
{
|
|
3028
|
+
executorToken: config.context.executorToken,
|
|
3029
|
+
baseUrl: config.context.baseUrl!,
|
|
3030
|
+
orgId: config.context.orgId!,
|
|
3031
|
+
playName,
|
|
3032
|
+
runId,
|
|
3033
|
+
userEmail: config.context.userEmail,
|
|
3034
|
+
vercelProtectionBypassToken:
|
|
3035
|
+
config.context.vercelProtectionBypassToken,
|
|
3036
|
+
dbSessionStrategy: config.context.dbSessionStrategy,
|
|
3037
|
+
preloadedDbSessions: canUseRoot
|
|
3038
|
+
? runtimePreloadedDbSessions
|
|
3039
|
+
: undefined,
|
|
3040
|
+
postgresSessionUnwrapKey: canUseRoot
|
|
3041
|
+
? runtimePostgresSessionUnwrapKey
|
|
3042
|
+
: config.context.executorToken,
|
|
3043
|
+
abortSignal: persistenceSignal,
|
|
3044
|
+
},
|
|
3045
|
+
{
|
|
3046
|
+
runId,
|
|
3047
|
+
tableNamespace: input.tableNamespace,
|
|
3048
|
+
sheetContract,
|
|
3049
|
+
expectedRows: input.expectedRows!,
|
|
3050
|
+
},
|
|
3051
|
+
),
|
|
3052
|
+
});
|
|
3053
|
+
}
|
|
3054
|
+
const datasetId = createRuntimeDatasetId(playName, input.tableNamespace);
|
|
2897
3055
|
const settlement = runtimeMapDatasetSettlements.get(datasetId);
|
|
2898
3056
|
if (settlement && (!input.runId || settlement.runId === input.runId)) {
|
|
2899
3057
|
Object.assign(
|
|
@@ -46,6 +46,11 @@ import {
|
|
|
46
46
|
} from '../play-data-plane/sheet-column-storage';
|
|
47
47
|
import {
|
|
48
48
|
RUNTIME_SHEET_CELL_META_COLUMN,
|
|
49
|
+
RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE,
|
|
50
|
+
RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED,
|
|
51
|
+
appendRuntimeSheetRunRowMembershipBatch,
|
|
52
|
+
isRuntimeSheetRunMembershipStorageAvailable,
|
|
53
|
+
type RuntimeSheetRunRowMembership,
|
|
49
54
|
runtimeSheetResultFreshnessSql,
|
|
50
55
|
runtimeSheetResultAdmissionSql,
|
|
51
56
|
} from '../play-data-plane/runtime-sheet-storage-contract';
|
|
@@ -66,6 +71,7 @@ import type {
|
|
|
66
71
|
} from './db-session';
|
|
67
72
|
import {
|
|
68
73
|
derivePlayRowIdentity,
|
|
74
|
+
stableStringify,
|
|
69
75
|
normalizePlayNameForSheet,
|
|
70
76
|
normalizeTableNamespace,
|
|
71
77
|
} from '../plays/row-identity';
|
|
@@ -503,6 +509,8 @@ export type PrepareRuntimeSheetResult = {
|
|
|
503
509
|
attemptExpiresAt?: string;
|
|
504
510
|
attemptSeq?: number;
|
|
505
511
|
writeVersion?: number;
|
|
512
|
+
/** Optional reader-first capture capability; absent retains legacy behavior. */
|
|
513
|
+
membershipCapture?: boolean;
|
|
506
514
|
};
|
|
507
515
|
|
|
508
516
|
export type RuntimeSheetTiming = {
|
|
@@ -645,6 +653,10 @@ type RuntimeApiActionRequest =
|
|
|
645
653
|
action: keyof typeof RUNTIME_SHEET_ADMISSION_CAPABILITIES;
|
|
646
654
|
input: Parameters<typeof startRuntimeSheetDataset>[1];
|
|
647
655
|
}
|
|
656
|
+
| {
|
|
657
|
+
action: 'runtime_sheet_finish_membership';
|
|
658
|
+
input: Parameters<typeof finishRuntimeSheetRunMembership>[1];
|
|
659
|
+
}
|
|
648
660
|
| {
|
|
649
661
|
action: 'runtime_sheet_complete_map_rows';
|
|
650
662
|
input: Parameters<typeof completeRuntimeMapRows>[1];
|
|
@@ -2872,6 +2884,151 @@ async function registerRuntimeDataset(
|
|
|
2872
2884
|
);
|
|
2873
2885
|
}
|
|
2874
2886
|
|
|
2887
|
+
async function runtimeSheetMembershipCaptureAvailable(
|
|
2888
|
+
client: RuntimeQueryClient,
|
|
2889
|
+
session: RuntimePostgresSession,
|
|
2890
|
+
): Promise<boolean> {
|
|
2891
|
+
if (!RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED) return false;
|
|
2892
|
+
return isRuntimeSheetRunMembershipStorageAvailable(
|
|
2893
|
+
async <Row>(sql: string, params?: readonly unknown[]) => {
|
|
2894
|
+
const result = await client.query(sql, params ? [...params] : undefined);
|
|
2895
|
+
return { rows: result.rows as Row[] };
|
|
2896
|
+
},
|
|
2897
|
+
session.postgres.schema,
|
|
2898
|
+
true,
|
|
2899
|
+
);
|
|
2900
|
+
}
|
|
2901
|
+
|
|
2902
|
+
async function persistRuntimeSheetMembershipBatch(
|
|
2903
|
+
client: RuntimeQueryClient,
|
|
2904
|
+
session: RuntimePostgresSession,
|
|
2905
|
+
rows: Array<Omit<RuntimeSheetRunRowMembership, 'recordedAt'>>,
|
|
2906
|
+
) {
|
|
2907
|
+
await appendRuntimeSheetRunRowMembershipBatch(
|
|
2908
|
+
async <Row>(sql: string, params?: readonly unknown[]) => {
|
|
2909
|
+
const result = await client.query(sql, params ? [...params] : undefined);
|
|
2910
|
+
return { rows: result.rows as Row[] };
|
|
2911
|
+
},
|
|
2912
|
+
rows,
|
|
2913
|
+
session.postgres.schema,
|
|
2914
|
+
);
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
async function captureRuntimeSheetTerminalMembership(
|
|
2918
|
+
client: RuntimeQueryClient,
|
|
2919
|
+
session: RuntimePostgresSession,
|
|
2920
|
+
input: {
|
|
2921
|
+
runId: string;
|
|
2922
|
+
tableNamespace: string;
|
|
2923
|
+
sheetContract: PlaySheetContract;
|
|
2924
|
+
keys: string[];
|
|
2925
|
+
},
|
|
2926
|
+
): Promise<void> {
|
|
2927
|
+
if (input.keys.length === 0) return;
|
|
2928
|
+
const datasetId = createRuntimeDatasetId(
|
|
2929
|
+
session.playName,
|
|
2930
|
+
input.tableNamespace,
|
|
2931
|
+
);
|
|
2932
|
+
// Read canonical values AFTER the fenced mutation, inside its transaction.
|
|
2933
|
+
// A stale submitted body must never become a fabricated historical result.
|
|
2934
|
+
const result = await client.query<Record<string, unknown>>(
|
|
2935
|
+
`SELECT target.*, admission.input_digest AS membership_input_digest, admission.source_ref AS membership_source_ref
|
|
2936
|
+
FROM ${sheetTable(session)} target
|
|
2937
|
+
JOIN ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} admission
|
|
2938
|
+
ON admission.run_id = $1 AND admission.dataset_id = $2 AND admission.row_key = target._key AND admission.phase = 'admitted'
|
|
2939
|
+
WHERE target._key = ANY($3::text[]) AND target._run_id = $1 AND target._status IN ('enriched', 'failed')`,
|
|
2940
|
+
[input.runId, datasetId, input.keys],
|
|
2941
|
+
);
|
|
2942
|
+
const snapshots = result.rows.map((raw) => {
|
|
2943
|
+
const attemptSeq =
|
|
2944
|
+
raw._attempt_seq == null ? null : Number(raw._attempt_seq);
|
|
2945
|
+
const rowVersion = Number(raw._version);
|
|
2946
|
+
const data = mapRuntimePostgresRow({
|
|
2947
|
+
raw,
|
|
2948
|
+
sheetContract: input.sheetContract,
|
|
2949
|
+
}).data;
|
|
2950
|
+
delete data[DEEPLINE_CELL_META_FIELD];
|
|
2951
|
+
return {
|
|
2952
|
+
runId: input.runId,
|
|
2953
|
+
datasetId,
|
|
2954
|
+
rowKey: String(raw._key),
|
|
2955
|
+
phase: 'terminal' as const,
|
|
2956
|
+
generation: `attempt:${attemptSeq ?? 'legacy'}:version:${rowVersion}`,
|
|
2957
|
+
attemptSeq,
|
|
2958
|
+
rowVersion,
|
|
2959
|
+
inputIndex: raw._input_index == null ? null : Number(raw._input_index),
|
|
2960
|
+
inputDigest: String(raw.membership_input_digest),
|
|
2961
|
+
status: raw._status as 'enriched' | 'failed',
|
|
2962
|
+
data,
|
|
2963
|
+
source: {
|
|
2964
|
+
...(raw.membership_source_ref as Record<string, unknown>),
|
|
2965
|
+
...(raw._error == null ? {} : { rowError: String(raw._error) }),
|
|
2966
|
+
},
|
|
2967
|
+
};
|
|
2968
|
+
});
|
|
2969
|
+
await persistRuntimeSheetMembershipBatch(client, session, snapshots);
|
|
2970
|
+
}
|
|
2971
|
+
|
|
2972
|
+
/** Named completion is called only after admission advertised capture. */
|
|
2973
|
+
export async function finishRuntimeSheetRunMembership(
|
|
2974
|
+
context: RuntimeApiContext & { playName: string },
|
|
2975
|
+
input: {
|
|
2976
|
+
runId: string;
|
|
2977
|
+
tableNamespace: string;
|
|
2978
|
+
sheetContract: PlaySheetContract;
|
|
2979
|
+
expectedRows: number;
|
|
2980
|
+
},
|
|
2981
|
+
): Promise<{ captured: boolean; complete: boolean }> {
|
|
2982
|
+
if (!Number.isSafeInteger(input.expectedRows) || input.expectedRows < 0)
|
|
2983
|
+
throw new Error('Membership expectedRows must be a non-negative integer.');
|
|
2984
|
+
if (context.dbSessionStrategy === 'gateway_only')
|
|
2985
|
+
return postRuntimeApi(context, {
|
|
2986
|
+
action: 'runtime_sheet_finish_membership',
|
|
2987
|
+
input,
|
|
2988
|
+
});
|
|
2989
|
+
const session = requireRuntimePostgresSession(
|
|
2990
|
+
await getRuntimeDbSession(context, {
|
|
2991
|
+
tableNamespace: input.tableNamespace,
|
|
2992
|
+
logicalTable: 'sheet_rows',
|
|
2993
|
+
operations: ['rows.read', 'rows.upsert'],
|
|
2994
|
+
limits: {},
|
|
2995
|
+
}),
|
|
2996
|
+
);
|
|
2997
|
+
return withRuntimeSheetQueryClient(
|
|
2998
|
+
context,
|
|
2999
|
+
session,
|
|
3000
|
+
{
|
|
3001
|
+
playName: context.playName,
|
|
3002
|
+
tableNamespace: input.tableNamespace,
|
|
3003
|
+
sheetContract: input.sheetContract,
|
|
3004
|
+
transactional: true,
|
|
3005
|
+
},
|
|
3006
|
+
async (client) => {
|
|
3007
|
+
if (!(await runtimeSheetMembershipCaptureAvailable(client, session)))
|
|
3008
|
+
return { captured: false, complete: false };
|
|
3009
|
+
const datasetId = createRuntimeDatasetId(
|
|
3010
|
+
session.playName,
|
|
3011
|
+
input.tableNamespace,
|
|
3012
|
+
);
|
|
3013
|
+
const result = await client.query<{ complete: boolean }>(
|
|
3014
|
+
`UPDATE ${runDatasetCatalogTable(session)} catalog
|
|
3015
|
+
SET membership_expected_rows = $3,
|
|
3016
|
+
membership_completed_at = CASE WHEN
|
|
3017
|
+
(SELECT count(*) FROM ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} WHERE run_id = $1 AND dataset_id = $2 AND phase = 'admitted') = $3
|
|
3018
|
+
AND (SELECT count(DISTINCT row_key) FROM ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} WHERE run_id = $1 AND dataset_id = $2 AND phase = 'terminal') = $3
|
|
3019
|
+
THEN coalesce(catalog.membership_completed_at, now()) ELSE NULL END
|
|
3020
|
+
WHERE run_id = $1 AND dataset_id = $2 AND membership_started_at IS NOT NULL
|
|
3021
|
+
RETURNING membership_completed_at IS NOT NULL AS complete`,
|
|
3022
|
+
[input.runId, datasetId, input.expectedRows],
|
|
3023
|
+
);
|
|
3024
|
+
return {
|
|
3025
|
+
captured: result.rows.length === 1,
|
|
3026
|
+
complete: result.rows[0]?.complete === true,
|
|
3027
|
+
};
|
|
3028
|
+
},
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
|
|
2875
3032
|
function workReceiptTable(session: RuntimePostgresSession): string {
|
|
2876
3033
|
return fqRuntimeTable(
|
|
2877
3034
|
session,
|
|
@@ -6877,9 +7034,16 @@ export async function startRuntimeSheetDataset(
|
|
|
6877
7034
|
const normalizeStartedAt = Date.now();
|
|
6878
7035
|
const uniqueRows = new Map<
|
|
6879
7036
|
string,
|
|
6880
|
-
{
|
|
7037
|
+
{
|
|
7038
|
+
row: Record<string, unknown>;
|
|
7039
|
+
inputIndex: number | null;
|
|
7040
|
+
sourceRowIndex: number | null;
|
|
7041
|
+
source: Record<string, unknown>;
|
|
7042
|
+
rowOffset: number;
|
|
7043
|
+
}
|
|
6881
7044
|
>();
|
|
6882
|
-
for (
|
|
7045
|
+
for (let rowOffset = 0; rowOffset < input.rows.length; rowOffset += 1) {
|
|
7046
|
+
const row = input.rows[rowOffset]!;
|
|
6883
7047
|
// Materializes projected CSV aliases as visible cells and drops internal
|
|
6884
7048
|
// __deepline* keys — a plain spread would silently lose the
|
|
6885
7049
|
// non-enumerable alias fields on the JSON payload boundary.
|
|
@@ -6887,23 +7051,37 @@ export async function startRuntimeSheetDataset(
|
|
|
6887
7051
|
const key =
|
|
6888
7052
|
resolveMapRowOutcomeKey(row) ??
|
|
6889
7053
|
derivePlayRowIdentity(cleanedRow, input.tableNamespace);
|
|
7054
|
+
const source =
|
|
7055
|
+
row.__deeplineSourceRow && typeof row.__deeplineSourceRow === 'object'
|
|
7056
|
+
? (sanitizePostgresJsonValue(row.__deeplineSourceRow) as Record<
|
|
7057
|
+
string,
|
|
7058
|
+
unknown
|
|
7059
|
+
>)
|
|
7060
|
+
: {};
|
|
7061
|
+
const sourceRowIndex = normalizeRuntimeMapInputIndex(source.rowIndex);
|
|
6890
7062
|
if (key && !uniqueRows.has(key)) {
|
|
6891
7063
|
uniqueRows.set(key, {
|
|
6892
7064
|
row: sanitizePostgresJsonValue(cleanedRow),
|
|
7065
|
+
source,
|
|
7066
|
+
sourceRowIndex,
|
|
6893
7067
|
inputIndex: normalizeRuntimeMapInputIndex(
|
|
6894
7068
|
row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex],
|
|
6895
7069
|
),
|
|
7070
|
+
rowOffset,
|
|
6896
7071
|
});
|
|
6897
7072
|
}
|
|
6898
7073
|
}
|
|
6899
7074
|
const inputOffset = Math.max(0, Math.floor(input.inputOffset ?? 0));
|
|
6900
|
-
const rowEntries = [...uniqueRows.entries()].map(
|
|
6901
|
-
|
|
6902
|
-
|
|
6903
|
-
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
|
|
7075
|
+
const rowEntries = [...uniqueRows.entries()].map(([key, entry]) => ({
|
|
7076
|
+
key,
|
|
7077
|
+
row: entry.row,
|
|
7078
|
+
source: entry.source,
|
|
7079
|
+
// A CSV transformation may filter or reorder the physical source rows.
|
|
7080
|
+
// Its source ordinal must win over the compacted input index stamped by
|
|
7081
|
+
// ctx.dataset while preparing this page.
|
|
7082
|
+
inputIndex:
|
|
7083
|
+
entry.sourceRowIndex ?? entry.inputIndex ?? inputOffset + entry.rowOffset,
|
|
7084
|
+
}));
|
|
6907
7085
|
timings.push({
|
|
6908
7086
|
phase: 'normalize_rows',
|
|
6909
7087
|
ms: Date.now() - normalizeStartedAt,
|
|
@@ -7135,6 +7313,60 @@ export async function startRuntimeSheetDataset(
|
|
|
7135
7313
|
force: input.force === true,
|
|
7136
7314
|
...prepared,
|
|
7137
7315
|
});
|
|
7316
|
+
const membershipCapture = await runtimeSheetMembershipCaptureAvailable(
|
|
7317
|
+
client,
|
|
7318
|
+
session,
|
|
7319
|
+
);
|
|
7320
|
+
if (membershipCapture) {
|
|
7321
|
+
const dispositions = new Set(
|
|
7322
|
+
prepared.rowDispositions
|
|
7323
|
+
.filter(
|
|
7324
|
+
(row) =>
|
|
7325
|
+
row.disposition === 'pending' ||
|
|
7326
|
+
row.disposition === 'completed',
|
|
7327
|
+
)
|
|
7328
|
+
.map((row) => row.key),
|
|
7329
|
+
);
|
|
7330
|
+
const admitted = rowEntries.filter((entry) =>
|
|
7331
|
+
dispositions.has(entry.key),
|
|
7332
|
+
);
|
|
7333
|
+
await client.query(
|
|
7334
|
+
`UPDATE ${runDatasetCatalogTable(session)} SET membership_started_at = coalesce(membership_started_at, now()) WHERE run_id = $1 AND dataset_id = $2`,
|
|
7335
|
+
[
|
|
7336
|
+
input.runId,
|
|
7337
|
+
createRuntimeDatasetId(session.playName, input.tableNamespace),
|
|
7338
|
+
],
|
|
7339
|
+
);
|
|
7340
|
+
await persistRuntimeSheetMembershipBatch(
|
|
7341
|
+
client,
|
|
7342
|
+
session,
|
|
7343
|
+
admitted.map((entry) => ({
|
|
7344
|
+
runId: input.runId,
|
|
7345
|
+
datasetId: createRuntimeDatasetId(
|
|
7346
|
+
session.playName,
|
|
7347
|
+
input.tableNamespace,
|
|
7348
|
+
),
|
|
7349
|
+
rowKey: entry.key,
|
|
7350
|
+
phase: 'admitted',
|
|
7351
|
+
generation: 'admitted',
|
|
7352
|
+
attemptSeq: null,
|
|
7353
|
+
rowVersion: null,
|
|
7354
|
+
inputIndex: entry.inputIndex,
|
|
7355
|
+
inputDigest: createHash('sha256')
|
|
7356
|
+
.update(stableStringify(entry.row))
|
|
7357
|
+
.digest('hex'),
|
|
7358
|
+
status: 'pending',
|
|
7359
|
+
data: entry.row,
|
|
7360
|
+
source: entry.source,
|
|
7361
|
+
})),
|
|
7362
|
+
);
|
|
7363
|
+
await captureRuntimeSheetTerminalMembership(client, session, {
|
|
7364
|
+
runId: input.runId,
|
|
7365
|
+
tableNamespace: input.tableNamespace,
|
|
7366
|
+
sheetContract: input.sheetContract,
|
|
7367
|
+
keys: admitted.map((entry) => entry.key),
|
|
7368
|
+
});
|
|
7369
|
+
}
|
|
7138
7370
|
timings.push({
|
|
7139
7371
|
phase: 'build_result',
|
|
7140
7372
|
ms: Date.now() - buildStartedAt,
|
|
@@ -7146,6 +7378,7 @@ export async function startRuntimeSheetDataset(
|
|
|
7146
7378
|
});
|
|
7147
7379
|
return {
|
|
7148
7380
|
...built,
|
|
7381
|
+
membershipCapture,
|
|
7149
7382
|
attemptExpiresAt,
|
|
7150
7383
|
attemptSeq,
|
|
7151
7384
|
writeVersion,
|
|
@@ -8982,11 +9215,20 @@ export async function completeRuntimeMapRows(
|
|
|
8982
9215
|
}),
|
|
8983
9216
|
)
|
|
8984
9217
|
: { updated: 0, fencedRows: [] };
|
|
8985
|
-
|
|
9218
|
+
const classified = classifyRuntimeMapRowsWrite({
|
|
8986
9219
|
submittedKeys: uniqueRows.keys(),
|
|
8987
9220
|
updated: completed.updated + failed.updated,
|
|
8988
9221
|
fencedRows: [...completed.fencedRows, ...failed.fencedRows],
|
|
8989
9222
|
});
|
|
9223
|
+
if (await runtimeSheetMembershipCaptureAvailable(client, session)) {
|
|
9224
|
+
await captureRuntimeSheetTerminalMembership(client, session, {
|
|
9225
|
+
runId: input.runId,
|
|
9226
|
+
tableNamespace: input.tableNamespace,
|
|
9227
|
+
sheetContract,
|
|
9228
|
+
keys: classified.committedKeys,
|
|
9229
|
+
});
|
|
9230
|
+
}
|
|
9231
|
+
return classified;
|
|
8990
9232
|
},
|
|
8991
9233
|
)
|
|
8992
9234
|
.then((result) => {
|