deepline 0.3.392 → 0.3.394

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +156 -24
  2. package/dist/bundling-sources/sdk/src/compat.ts +9 -0
  3. package/dist/bundling-sources/sdk/src/http.ts +5 -1
  4. package/dist/bundling-sources/sdk/src/index.ts +5 -0
  5. package/dist/bundling-sources/sdk/src/plays/run-request.ts +182 -6
  6. package/dist/bundling-sources/sdk/src/release.ts +4 -1
  7. package/dist/bundling-sources/sdk/src/types.ts +13 -0
  8. package/dist/bundling-sources/shared_libs/integration-contracts/request-warnings.ts +64 -0
  9. package/dist/bundling-sources/shared_libs/play-data-plane/runtime-sheet-storage-contract.ts +315 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/async-operation.ts +41 -8
  11. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +895 -54
  12. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +41 -4
  13. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-policy.ts +28 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/durable-invocation-fence.ts +36 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/freshness-receipt-execution.ts +251 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +11 -3
  17. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +1 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +30 -1
  19. package/dist/bundling-sources/shared_libs/play-runtime/run-identity.ts +26 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +168 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +252 -10
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +92 -58
  23. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +7 -1
  24. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +30 -7
  25. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +1161 -316
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +28 -15
  27. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +24 -3
  28. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +32 -0
  29. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +103 -23
  30. package/dist/bundling-sources/shared_libs/plays/core.ts +1 -1
  31. package/dist/bundling-sources/shared_libs/plays/dataset.ts +172 -79
  32. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +173 -1
  33. package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +18 -0
  34. package/dist/cli/auth-IHKISLJQ.js +39 -0
  35. package/dist/cli/{auth-6JSA2ZCW.mjs → auth-OSXDAVCF.mjs} +4 -4
  36. package/dist/cli/{billing-Q5BLQHIU.js → billing-EKLYOXUY.js} +101 -74
  37. package/dist/cli/{billing-MRRO643H.mjs → billing-ZNNKX3UR.mjs} +32 -5
  38. package/dist/cli/{chunk-4SFPVW7A.js → chunk-2MDDA7A4.js} +26 -26
  39. package/dist/cli/{chunk-JOFPQGXW.js → chunk-354JCXA2.js} +50 -50
  40. package/dist/cli/{chunk-7E66A24V.mjs → chunk-3HU47EGZ.mjs} +71 -9
  41. package/dist/cli/{chunk-DSWSEHRX.mjs → chunk-3HWALOL2.mjs} +1 -1
  42. package/dist/cli/{chunk-T7OKJ2T6.js → chunk-4P27AXEA.js} +73 -63
  43. package/dist/cli/{chunk-24VEN5PQ.mjs → chunk-5BB2RGAX.mjs} +190 -18
  44. package/dist/cli/{chunk-JSLPHFP7.js → chunk-6YAUMG27.js} +3 -2
  45. package/dist/cli/{chunk-MU6M3URX.mjs → chunk-BP33SIZF.mjs} +1 -1
  46. package/dist/cli/{chunk-WMS5U6YB.js → chunk-CS6RAIVY.js} +20 -19
  47. package/dist/cli/{chunk-CVLM3WMJ.js → chunk-E23BQ5SC.js} +242 -183
  48. package/dist/cli/{chunk-2GAHHSLB.mjs → chunk-FJGW5GXU.mjs} +3 -2
  49. package/dist/cli/{chunk-YN6YA7ZZ.mjs → chunk-JHIJIHFB.mjs} +79 -20
  50. package/dist/cli/{chunk-Q6EOHNRH.mjs → chunk-JI2B4RZP.mjs} +119 -6
  51. package/dist/cli/{chunk-5O4B6AUN.js → chunk-KB4SDX5S.js} +71 -9
  52. package/dist/cli/{chunk-RV6ALZIX.js → chunk-KGNTKMRG.js} +32 -7
  53. package/dist/cli/{chunk-6DBOU7UH.js → chunk-KI635JN4.js} +176 -87
  54. package/dist/cli/{chunk-2FQG2HU4.js → chunk-KIUMEOM5.js} +8 -8
  55. package/dist/cli/{chunk-Y6VOGHF2.mjs → chunk-KZZC33PY.mjs} +0 -181
  56. package/dist/cli/{chunk-2Q3GEFL4.mjs → chunk-M57J2SK4.mjs} +2 -2
  57. package/dist/cli/{chunk-R3O2CZ4H.mjs → chunk-MFB2VKBO.mjs} +2 -2
  58. package/dist/cli/{chunk-QGKPTBRR.js → chunk-MH73HMPV.js} +3 -3
  59. package/dist/cli/{chunk-BB3FNYFY.js → chunk-MLPU5SMZ.js} +2 -2
  60. package/dist/cli/{chunk-2G7DXFHI.mjs → chunk-NR3DFKNF.mjs} +27 -2
  61. package/dist/cli/{chunk-EY7RFC6A.js → chunk-QIQBYKLB.js} +15 -15
  62. package/dist/cli/{chunk-RIV2GS5E.js → chunk-RBJYWRCF.js} +638 -466
  63. package/dist/cli/{chunk-3VN4FTE3.js → chunk-RDXFGPJU.js} +1 -182
  64. package/dist/cli/{chunk-YAZCDBXX.mjs → chunk-TJI2EN6K.mjs} +3 -2
  65. package/dist/cli/chunk-UBKA33KY.mjs +310 -0
  66. package/dist/cli/{chunk-PAIMOGCM.js → chunk-UHNDVD3P.js} +125 -12
  67. package/dist/cli/{chunk-4TCT77UF.js → chunk-URWF7NGQ.js} +6 -1
  68. package/dist/cli/{chunk-2QLML6G4.mjs → chunk-UXFSWG3Q.mjs} +4 -4
  69. package/dist/cli/{chunk-UCKJQG25.js → chunk-VK6FYZLP.js} +8 -8
  70. package/dist/cli/{chunk-ZTACINO5.mjs → chunk-X63FDZJZ.mjs} +13 -3
  71. package/dist/cli/chunk-XDVT3GOM.js +310 -0
  72. package/dist/cli/{chunk-A3APGZFP.mjs → chunk-XRQ2AQL7.mjs} +2 -2
  73. package/dist/cli/{chunk-3FV4GBRK.mjs → chunk-XXUDVEXK.mjs} +97 -8
  74. package/dist/cli/{chunk-PQSA443V.mjs → chunk-ZI2ZNDJK.mjs} +6 -1
  75. package/dist/cli/{chunk-HGYBEAQP.mjs → chunk-ZYBGA4AR.mjs} +3 -3
  76. package/dist/cli/command-docs.js +2 -2
  77. package/dist/cli/command-docs.mjs +1 -1
  78. package/dist/cli/{csv-E4QGUIGI.mjs → csv-5AXRB35H.mjs} +3 -3
  79. package/dist/cli/{csv-26U5T24Z.js → csv-LU7O75WE.js} +12 -12
  80. package/dist/cli/{db-JTRJIAQW.mjs → db-7XP5UWEW.mjs} +6 -5
  81. package/dist/cli/{db-QHBFLMVL.js → db-F7YTFKVM.js} +31 -30
  82. package/dist/cli/{describe-FDV2V74F.js → describe-FRH32RDV.js} +24 -23
  83. package/dist/cli/{describe-E3DE3ZK3.mjs → describe-K4ES5LIL.mjs} +17 -16
  84. package/dist/cli/{executions-B3GTNPRE.mjs → executions-IEHEHJUO.mjs} +6 -5
  85. package/dist/cli/{executions-VZYOGVQY.js → executions-RZV5XPT2.js} +8 -7
  86. package/dist/cli/{feedback-CR55FJA2.js → feedback-3P65QTX3.js} +7 -7
  87. package/dist/cli/{feedback-4CEBR3OV.mjs → feedback-FPUVZBAN.mjs} +3 -3
  88. package/dist/cli/{git-NM2PHHZ4.mjs → git-GLVVP7TY.mjs} +4 -4
  89. package/dist/cli/git-MFFAWVVX.js +23 -0
  90. package/dist/cli/index.js +100 -96
  91. package/dist/cli/index.mjs +47 -43
  92. package/dist/cli/{legacy-noops-A7MSW5ND.mjs → legacy-noops-3MOI6NBM.mjs} +4 -4
  93. package/dist/cli/{legacy-noops-ZUA4YFIH.js → legacy-noops-XKSGZR7Q.js} +7 -7
  94. package/dist/cli/{monitors-NVPRQYP3.mjs → monitors-FTBGNEFG.mjs} +6 -5
  95. package/dist/cli/{monitors-ZE3TLZ4H.js → monitors-NZ2DYNST.js} +87 -86
  96. package/dist/cli/{org-U7A6DV5D.js → org-MCKDQXIV.js} +66 -65
  97. package/dist/cli/{org-KUBILNQV.mjs → org-V2J5RBCN.mjs} +6 -5
  98. package/dist/cli/{play-HRDXQ4QE.mjs → play-AJFGME3B.mjs} +11 -10
  99. package/dist/cli/play-WLSN2EO2.js +158 -0
  100. package/dist/cli/{providers-VD6YTVQZ.js → providers-2JEZ2264.js} +9 -8
  101. package/dist/cli/{providers-OHIWI453.mjs → providers-I6RQJZFZ.mjs} +6 -5
  102. package/dist/cli/{quickstart-UBV2FONF.mjs → quickstart-BIDQDMO4.mjs} +3 -3
  103. package/dist/cli/{quickstart-ETTCWPA4.js → quickstart-GJ2RPDU3.js} +5 -5
  104. package/dist/cli/run-identity-G257GQED.js +25 -0
  105. package/dist/cli/run-identity-MRNCEGV2.mjs +25 -0
  106. package/dist/cli/search-5VN4YD4I.js +27 -0
  107. package/dist/cli/{search-F44IZRT7.mjs → search-ELRXFWG5.mjs} +7 -6
  108. package/dist/cli/{secrets-CAEMRORF.js → secrets-53IT6VXD.js} +15 -14
  109. package/dist/cli/{secrets-R6XOJZKU.mjs → secrets-SQXGZXIC.mjs} +6 -5
  110. package/dist/cli/{sessions-RLN6UQS3.mjs → sessions-2X5QF7SL.mjs} +4 -4
  111. package/dist/cli/{sessions-6GC7OFTT.js → sessions-CAV5M77Y.js} +5 -5
  112. package/dist/cli/{settings-DPNVUAV2.js → settings-532WNAZO.js} +76 -75
  113. package/dist/cli/{settings-KIPAMRPP.mjs → settings-7P67VGCJ.mjs} +9 -8
  114. package/dist/cli/{setup-DTL3L5SG.js → setup-6L2VSUWJ.js} +41 -41
  115. package/dist/cli/{setup-TU2F7PU5.mjs → setup-QGGBV2LE.mjs} +6 -6
  116. package/dist/cli/{skills-UV5MYJ46.mjs → skills-3FE62WBG.mjs} +4 -4
  117. package/dist/cli/skills-S6NPZAN7.js +28 -0
  118. package/dist/cli/tools-647RLXQS.js +41 -0
  119. package/dist/cli/{tools-QRRNDCQ7.mjs → tools-DMR25DEF.mjs} +10 -9
  120. package/dist/cli/{update-4426TF35.mjs → update-6RZMHJZP.mjs} +6 -6
  121. package/dist/cli/update-WVTSC3EM.js +28 -0
  122. package/dist/cli/{workflow-N4U5VLZ3.mjs → workflow-L2DQRE6R.mjs} +7 -6
  123. package/dist/cli/{workflow-HNLOVISB.js → workflow-TX4F2PIX.js} +17 -16
  124. package/dist/{client-e6Qh5ZIg.d.mts → client-BktIPdIj.d.mts} +153 -4
  125. package/dist/{client-Uxuy-xQ2.d.ts → client-C3O1Ipf3.d.ts} +153 -4
  126. package/dist/{compiler-manifest-CY2v8ab0.d.ts → compiler-manifest-DLxUY5kI.d.mts} +124 -10
  127. package/dist/{compiler-manifest-CY2v8ab0.d.mts → compiler-manifest-DLxUY5kI.d.ts} +124 -10
  128. package/dist/{index-BZ6uzZJl.d.ts → index-B_GM9E3q.d.ts} +1 -1
  129. package/dist/{index-B9fBZH1g.d.mts → index-SCb0NvDb.d.mts} +1 -1
  130. package/dist/index.d.mts +5 -5
  131. package/dist/index.d.ts +5 -5
  132. package/dist/index.js +120 -29
  133. package/dist/index.mjs +111 -20
  134. package/dist/install-integrity.json +94 -88
  135. package/dist/plays/bundle-play-file.d.mts +3 -3
  136. package/dist/plays/bundle-play-file.d.ts +3 -3
  137. package/dist/plays/bundle-play-file.mjs +2 -2
  138. package/dist/plays/run-request.d.mts +2 -2
  139. package/dist/plays/run-request.d.ts +2 -2
  140. package/dist/plays/run-request.js +5 -2
  141. package/dist/plays/run-request.mjs +4 -1
  142. package/dist/release.d.mts +4 -1
  143. package/dist/release.d.ts +4 -1
  144. package/dist/release.js +2 -2
  145. package/dist/release.mjs +1 -1
  146. package/package.json +1 -1
  147. package/dist/cli/auth-67TAELQL.js +0 -39
  148. package/dist/cli/git-I5JTK6YU.js +0 -23
  149. package/dist/cli/play-A2V7YDU7.js +0 -157
  150. package/dist/cli/search-WWXMBVPH.js +0 -26
  151. package/dist/cli/skills-SWOSEZLB.js +0 -28
  152. package/dist/cli/tools-IL2YCDQ6.js +0 -40
  153. package/dist/cli/update-BERDPEYR.js +0 -28
@@ -19,6 +19,7 @@ import { csvInputTextOptions } from '../../plays/csv-text-repair';
19
19
  import { install as installSourceMapSupport } from 'source-map-support';
20
20
  import {
21
21
  createDeferredPlayDataset,
22
+ setPlayDatasetSourceRowIdentity,
22
23
  isPlayDataset,
23
24
  isSerializedPlayDataset,
24
25
  iteratePlayDatasetInputPages,
@@ -128,6 +129,7 @@ import {
128
129
  import {
129
130
  resolveRuntimeReferencedPlay,
130
131
  createCompleteRuntimeMapRowsOperation,
132
+ finishRuntimeSheetRunMembership,
131
133
  createRuntimeSheetDatasetPageReader,
132
134
  type RuntimeSheetDatasetPageReader,
133
135
  readRuntimeSheetDatasetRowKeys,
@@ -140,6 +142,7 @@ import {
140
142
  } from '../runtime-api';
141
143
  import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
142
144
  import { createRuntimeToolAuthScopeDigestResolver } from '../auth-scope-resolver';
145
+ import { RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER } from '../durable-invocation-fence';
143
146
  import { createRunScopedChildPlayResolver } from './child-play-resolver';
144
147
  import { normalizePlayContractCompatibility } from '../../plays/contracts';
145
148
  import { ToolExecutionError } from '../../plays/tool-execution-error';
@@ -354,6 +357,7 @@ export function playRunnerReceiptStoreAdapterOptions() {
354
357
  type RuntimeStepReceiptHandlers = RuntimeReceiptStoreHandlers;
355
358
 
356
359
  type RuntimeToolMetadata = {
360
+ reusableResult?: { supported: boolean; unavailableReason: string | null };
357
361
  provider?: string | null;
358
362
  operation?: string | null;
359
363
  extractors?: ToolResultMetadataInput['extractors'];
@@ -379,6 +383,8 @@ type RuntimeToolMetadata = {
379
383
 
380
384
  type RuntimeToolAuthScope = {
381
385
  digest?: unknown;
386
+ managedAsyncExecutionRecovery?: unknown;
387
+ managedAsyncExecutionRecoveryOrigin?: unknown;
382
388
  rateScope?: {
383
389
  bucketId?: unknown;
384
390
  provider?: unknown;
@@ -534,7 +540,7 @@ export function runtimeSheetTerminalMapCounts(
534
540
  }
535
541
 
536
542
  async function vercelBypassHeaders(
537
- config: PlayRunnerExecutionConfig,
543
+ config: Pick<PlayRunnerExecutionConfig, 'context'>,
538
544
  ): Promise<Record<string, string>> {
539
545
  return vercelProtectionBypassHeader(
540
546
  config.context.vercelProtectionBypassToken,
@@ -723,9 +729,14 @@ function createRuntimeToolMetadataResolver(config: PlayRunnerExecutionConfig) {
723
729
  };
724
730
  }
725
731
 
726
- function createAppRuntimeToolAuthScopeDigestResolver(
727
- config: PlayRunnerExecutionConfig,
732
+ export function createAppRuntimeToolAuthScopeDigestResolver(
733
+ config: Pick<PlayRunnerExecutionConfig, 'context'>,
728
734
  ) {
735
+ const asyncRecoveryOrigins = new Map<string, string>();
736
+ const executionRelayRecovery = new Map<
737
+ string,
738
+ Promise<{ origin: string } | null>
739
+ >();
729
740
  const rateScopes = new Map<
730
741
  string,
731
742
  { bucketId: string; provider: string; token: string }
@@ -783,6 +794,18 @@ function createAppRuntimeToolAuthScopeDigestResolver(
783
794
  );
784
795
  }
785
796
  rateScopes.set(normalizedToolId, { bucketId, provider, token });
797
+ if (
798
+ response.headers.get(
799
+ RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER,
800
+ ) === 'true' &&
801
+ body.managedAsyncExecutionRecovery === true &&
802
+ typeof body.managedAsyncExecutionRecoveryOrigin === 'string'
803
+ ) {
804
+ asyncRecoveryOrigins.set(
805
+ normalizedToolId,
806
+ body.managedAsyncExecutionRecoveryOrigin,
807
+ );
808
+ }
786
809
  return digest;
787
810
  },
788
811
  });
@@ -793,6 +816,10 @@ function createAppRuntimeToolAuthScopeDigestResolver(
793
816
  const normalized = String(toolId || '').trim();
794
817
  if (normalized) rateScopes.delete(normalized);
795
818
  else rateScopes.clear();
819
+ if (normalized) asyncRecoveryOrigins.delete(normalized);
820
+ else asyncRecoveryOrigins.clear();
821
+ if (normalized) executionRelayRecovery.delete(normalized);
822
+ else executionRelayRecovery.clear();
796
823
  },
797
824
  rateScope: async (toolId: string, expectedProvider: string) => {
798
825
  await digestResolver(toolId);
@@ -804,6 +831,60 @@ function createAppRuntimeToolAuthScopeDigestResolver(
804
831
  }
805
832
  return { bucketId: scope.bucketId, token: scope.token };
806
833
  },
834
+ asyncExecutionRecovery: async (toolId: string) => {
835
+ const normalized = toolId.trim();
836
+ const digest = await digestResolver(normalized);
837
+ const receiptBase = config.context.baseUrl!.trim().replace(/\/$/, '');
838
+ const executionBase =
839
+ config.context.executionGatewayBaseUrl?.trim().replace(/\/$/, '') ||
840
+ receiptBase;
841
+ if (executionBase === receiptBase) {
842
+ const origin = asyncRecoveryOrigins.get(normalized);
843
+ return origin ? { origin } : null;
844
+ }
845
+ // Receipt and execution fleets roll independently. Keep canonical
846
+ // digest/rate truth on the receipt owner, but never treat its transport
847
+ // capability as proof that the execution relay forwards the app ACK.
848
+ let proof = executionRelayRecovery.get(normalized);
849
+ if (!proof) {
850
+ proof = (async () => {
851
+ const response = await fetchRunnerRuntimeControl({
852
+ url: `${executionBase}/api/v2/plays/runtime-tools/${encodeURIComponent(normalized)}/auth-scope`,
853
+ description: `Runtime async recovery capability for tool ${normalized}`,
854
+ retrySafe: true,
855
+ init: {
856
+ method: 'GET',
857
+ headers: {
858
+ Authorization: `Bearer ${config.context.executorToken}`,
859
+ ...(await vercelBypassHeaders(config)),
860
+ },
861
+ },
862
+ });
863
+ if (
864
+ !response.ok ||
865
+ response.headers.get(
866
+ RUNTIME_ASYNC_EXECUTION_RECOVERY_TRANSPORT_HEADER,
867
+ ) !== 'true'
868
+ )
869
+ return null;
870
+ const body = (await response.json()) as RuntimeToolAuthScope;
871
+ return body.digest === digest &&
872
+ body.managedAsyncExecutionRecovery === true &&
873
+ typeof body.managedAsyncExecutionRecoveryOrigin === 'string'
874
+ ? { origin: body.managedAsyncExecutionRecoveryOrigin }
875
+ : null;
876
+ })().catch(() => {
877
+ // This optional capability cannot authorize replay on an unavailable
878
+ // hop. Preserve the existing unkeyed path and allow later discovery.
879
+ if (executionRelayRecovery.get(normalized) === proof) {
880
+ executionRelayRecovery.delete(normalized);
881
+ }
882
+ return null;
883
+ });
884
+ executionRelayRecovery.set(normalized, proof);
885
+ }
886
+ return await proof;
887
+ },
807
888
  });
808
889
  }
809
890
 
@@ -1122,11 +1203,19 @@ async function createStreamingCsvDataset(input: {
1122
1203
  options?: CsvRenameOptions;
1123
1204
  }): Promise<PlayDataset<Record<string, unknown>>> {
1124
1205
  let cachedCount: number | null = null;
1206
+ const datasetId = createRuntimeDatasetId(
1207
+ input.playName,
1208
+ input.tableNamespace,
1209
+ );
1125
1210
  const iterateProjected = () =>
1126
1211
  ({
1127
1212
  async *[Symbol.asyncIterator]() {
1213
+ let rowIndex = 0;
1128
1214
  for await (const row of input.source.iterate()) {
1129
- yield applyCsvRenameProjection([row], input.options)[0]!;
1215
+ yield setPlayDatasetSourceRowIdentity(
1216
+ applyCsvRenameProjection([row], input.options)[0]!,
1217
+ { datasetId, rowIndex: rowIndex++ },
1218
+ );
1130
1219
  }
1131
1220
  },
1132
1221
  }) as AsyncIterable<Record<string, unknown>>;
@@ -1142,7 +1231,7 @@ async function createStreamingCsvDataset(input: {
1142
1231
  const previewRows = await collect(10);
1143
1232
  return createDeferredPlayDataset({
1144
1233
  datasetKind: 'csv',
1145
- datasetId: createRuntimeDatasetId(input.playName, input.tableNamespace),
1234
+ datasetId,
1146
1235
  count: 0,
1147
1236
  knownCount: null,
1148
1237
  previewRows,
@@ -2204,6 +2293,7 @@ async function run(
2204
2293
  writeVersion?: number;
2205
2294
  }
2206
2295
  >();
2296
+ const membershipCapturingMaps = new Set<string>();
2207
2297
  const runtimeMapDatasetSettlements = new Map<
2208
2298
  string,
2209
2299
  {
@@ -2421,6 +2511,15 @@ async function run(
2421
2511
  ? 'invocation'
2422
2512
  : null;
2423
2513
  },
2514
+ getToolReusableResultSupport: async (toolId) => {
2515
+ const support = (await getRuntimeToolMetadata(toolId)).reusableResult;
2516
+ if (!support) {
2517
+ throw new Error(
2518
+ `Tool ${toolId} has no reusable-result capability metadata. Deploy the runtime metadata reader before admitting explicit-freshness artifacts.`,
2519
+ );
2520
+ }
2521
+ return support;
2522
+ },
2424
2523
  getToolRetryPolicy: async (toolId) => {
2425
2524
  const metadata = await getRuntimeToolMetadata(toolId);
2426
2525
  return {
@@ -2449,6 +2548,8 @@ async function run(
2449
2548
  ];
2450
2549
  },
2451
2550
  getToolAuthScopeDigest: getRuntimeToolAuthScopeDigest,
2551
+ getToolAsyncExecutionRecovery:
2552
+ getRuntimeToolAuthScopeDigest.asyncExecutionRecovery,
2452
2553
  getToolRateScope: getRuntimeToolAuthScopeDigest.rateScope,
2453
2554
  invalidateToolAuthScopeDigest: (toolId) =>
2454
2555
  getRuntimeToolAuthScopeDigest.invalidate?.(toolId),
@@ -2734,6 +2835,8 @@ async function run(
2734
2835
  },
2735
2836
  ),
2736
2837
  });
2838
+ if (mapStartResult.membershipCapture === true)
2839
+ membershipCapturingMaps.add(attemptKey);
2737
2840
  const datasetId = createRuntimeDatasetId(playName, tableNamespace);
2738
2841
  const datasetPath = `datasets.${tableNamespace}`;
2739
2842
  const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
@@ -2889,11 +2992,66 @@ async function run(
2889
2992
  ),
2890
2993
  });
2891
2994
  },
2892
- onMapRowsSettled: (input) => {
2893
- const datasetId = createRuntimeDatasetId(
2894
- input.playName ?? config.context.playName!,
2895
- input.tableNamespace,
2896
- );
2995
+ onMapRowsSettled: async (input) => {
2996
+ const playName = input.playName ?? config.context.playName!;
2997
+ const runId =
2998
+ input.runId ??
2999
+ config.context.runId ??
3000
+ config.context.workflowId ??
3001
+ 'anonymous-run';
3002
+ const mapKey = runtimeSheetAttemptKey({
3003
+ playName,
3004
+ runId,
3005
+ tableNamespace: input.tableNamespace,
3006
+ });
3007
+ if (
3008
+ membershipCapturingMaps.has(mapKey) &&
3009
+ input.expectedRows !== undefined
3010
+ ) {
3011
+ const sheetContract = resolveSheetContractForTableNamespace(
3012
+ input.staticPipeline ?? config.context.staticPipeline,
3013
+ input.tableNamespace,
3014
+ );
3015
+ if (!sheetContract)
3016
+ throw new Error(
3017
+ `Missing sheet contract while finishing membership for ${input.tableNamespace}.`,
3018
+ );
3019
+ const canUseRoot =
3020
+ runId === (config.context.runId ?? config.context.workflowId);
3021
+ await retryRuntimeSheetPersistence({
3022
+ description: `runtime membership finish table=${input.tableNamespace}`,
3023
+ log: (line) => emitWorkerLog(line, options),
3024
+ signal: persistenceSignal,
3025
+ run: () =>
3026
+ finishRuntimeSheetRunMembership(
3027
+ {
3028
+ executorToken: config.context.executorToken,
3029
+ baseUrl: config.context.baseUrl!,
3030
+ orgId: config.context.orgId!,
3031
+ playName,
3032
+ runId,
3033
+ userEmail: config.context.userEmail,
3034
+ vercelProtectionBypassToken:
3035
+ config.context.vercelProtectionBypassToken,
3036
+ dbSessionStrategy: config.context.dbSessionStrategy,
3037
+ preloadedDbSessions: canUseRoot
3038
+ ? runtimePreloadedDbSessions
3039
+ : undefined,
3040
+ postgresSessionUnwrapKey: canUseRoot
3041
+ ? runtimePostgresSessionUnwrapKey
3042
+ : config.context.executorToken,
3043
+ abortSignal: persistenceSignal,
3044
+ },
3045
+ {
3046
+ runId,
3047
+ tableNamespace: input.tableNamespace,
3048
+ sheetContract,
3049
+ expectedRows: input.expectedRows!,
3050
+ },
3051
+ ),
3052
+ });
3053
+ }
3054
+ const datasetId = createRuntimeDatasetId(playName, input.tableNamespace);
2897
3055
  const settlement = runtimeMapDatasetSettlements.get(datasetId);
2898
3056
  if (settlement && (!input.runId || settlement.runId === input.runId)) {
2899
3057
  Object.assign(
@@ -46,6 +46,11 @@ import {
46
46
  } from '../play-data-plane/sheet-column-storage';
47
47
  import {
48
48
  RUNTIME_SHEET_CELL_META_COLUMN,
49
+ RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE,
50
+ RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED,
51
+ appendRuntimeSheetRunRowMembershipBatch,
52
+ isRuntimeSheetRunMembershipStorageAvailable,
53
+ type RuntimeSheetRunRowMembership,
49
54
  runtimeSheetResultFreshnessSql,
50
55
  runtimeSheetResultAdmissionSql,
51
56
  } from '../play-data-plane/runtime-sheet-storage-contract';
@@ -66,6 +71,7 @@ import type {
66
71
  } from './db-session';
67
72
  import {
68
73
  derivePlayRowIdentity,
74
+ stableStringify,
69
75
  normalizePlayNameForSheet,
70
76
  normalizeTableNamespace,
71
77
  } from '../plays/row-identity';
@@ -503,6 +509,8 @@ export type PrepareRuntimeSheetResult = {
503
509
  attemptExpiresAt?: string;
504
510
  attemptSeq?: number;
505
511
  writeVersion?: number;
512
+ /** Optional reader-first capture capability; absent retains legacy behavior. */
513
+ membershipCapture?: boolean;
506
514
  };
507
515
 
508
516
  export type RuntimeSheetTiming = {
@@ -645,6 +653,10 @@ type RuntimeApiActionRequest =
645
653
  action: keyof typeof RUNTIME_SHEET_ADMISSION_CAPABILITIES;
646
654
  input: Parameters<typeof startRuntimeSheetDataset>[1];
647
655
  }
656
+ | {
657
+ action: 'runtime_sheet_finish_membership';
658
+ input: Parameters<typeof finishRuntimeSheetRunMembership>[1];
659
+ }
648
660
  | {
649
661
  action: 'runtime_sheet_complete_map_rows';
650
662
  input: Parameters<typeof completeRuntimeMapRows>[1];
@@ -2872,6 +2884,151 @@ async function registerRuntimeDataset(
2872
2884
  );
2873
2885
  }
2874
2886
 
2887
+ async function runtimeSheetMembershipCaptureAvailable(
2888
+ client: RuntimeQueryClient,
2889
+ session: RuntimePostgresSession,
2890
+ ): Promise<boolean> {
2891
+ if (!RUNTIME_SHEET_RUN_MEMBERSHIP_WRITES_ENABLED) return false;
2892
+ return isRuntimeSheetRunMembershipStorageAvailable(
2893
+ async <Row>(sql: string, params?: readonly unknown[]) => {
2894
+ const result = await client.query(sql, params ? [...params] : undefined);
2895
+ return { rows: result.rows as Row[] };
2896
+ },
2897
+ session.postgres.schema,
2898
+ true,
2899
+ );
2900
+ }
2901
+
2902
+ async function persistRuntimeSheetMembershipBatch(
2903
+ client: RuntimeQueryClient,
2904
+ session: RuntimePostgresSession,
2905
+ rows: Array<Omit<RuntimeSheetRunRowMembership, 'recordedAt'>>,
2906
+ ) {
2907
+ await appendRuntimeSheetRunRowMembershipBatch(
2908
+ async <Row>(sql: string, params?: readonly unknown[]) => {
2909
+ const result = await client.query(sql, params ? [...params] : undefined);
2910
+ return { rows: result.rows as Row[] };
2911
+ },
2912
+ rows,
2913
+ session.postgres.schema,
2914
+ );
2915
+ }
2916
+
2917
+ async function captureRuntimeSheetTerminalMembership(
2918
+ client: RuntimeQueryClient,
2919
+ session: RuntimePostgresSession,
2920
+ input: {
2921
+ runId: string;
2922
+ tableNamespace: string;
2923
+ sheetContract: PlaySheetContract;
2924
+ keys: string[];
2925
+ },
2926
+ ): Promise<void> {
2927
+ if (input.keys.length === 0) return;
2928
+ const datasetId = createRuntimeDatasetId(
2929
+ session.playName,
2930
+ input.tableNamespace,
2931
+ );
2932
+ // Read canonical values AFTER the fenced mutation, inside its transaction.
2933
+ // A stale submitted body must never become a fabricated historical result.
2934
+ const result = await client.query<Record<string, unknown>>(
2935
+ `SELECT target.*, admission.input_digest AS membership_input_digest, admission.source_ref AS membership_source_ref
2936
+ FROM ${sheetTable(session)} target
2937
+ JOIN ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} admission
2938
+ ON admission.run_id = $1 AND admission.dataset_id = $2 AND admission.row_key = target._key AND admission.phase = 'admitted'
2939
+ WHERE target._key = ANY($3::text[]) AND target._run_id = $1 AND target._status IN ('enriched', 'failed')`,
2940
+ [input.runId, datasetId, input.keys],
2941
+ );
2942
+ const snapshots = result.rows.map((raw) => {
2943
+ const attemptSeq =
2944
+ raw._attempt_seq == null ? null : Number(raw._attempt_seq);
2945
+ const rowVersion = Number(raw._version);
2946
+ const data = mapRuntimePostgresRow({
2947
+ raw,
2948
+ sheetContract: input.sheetContract,
2949
+ }).data;
2950
+ delete data[DEEPLINE_CELL_META_FIELD];
2951
+ return {
2952
+ runId: input.runId,
2953
+ datasetId,
2954
+ rowKey: String(raw._key),
2955
+ phase: 'terminal' as const,
2956
+ generation: `attempt:${attemptSeq ?? 'legacy'}:version:${rowVersion}`,
2957
+ attemptSeq,
2958
+ rowVersion,
2959
+ inputIndex: raw._input_index == null ? null : Number(raw._input_index),
2960
+ inputDigest: String(raw.membership_input_digest),
2961
+ status: raw._status as 'enriched' | 'failed',
2962
+ data,
2963
+ source: {
2964
+ ...(raw.membership_source_ref as Record<string, unknown>),
2965
+ ...(raw._error == null ? {} : { rowError: String(raw._error) }),
2966
+ },
2967
+ };
2968
+ });
2969
+ await persistRuntimeSheetMembershipBatch(client, session, snapshots);
2970
+ }
2971
+
2972
+ /** Named completion is called only after admission advertised capture. */
2973
+ export async function finishRuntimeSheetRunMembership(
2974
+ context: RuntimeApiContext & { playName: string },
2975
+ input: {
2976
+ runId: string;
2977
+ tableNamespace: string;
2978
+ sheetContract: PlaySheetContract;
2979
+ expectedRows: number;
2980
+ },
2981
+ ): Promise<{ captured: boolean; complete: boolean }> {
2982
+ if (!Number.isSafeInteger(input.expectedRows) || input.expectedRows < 0)
2983
+ throw new Error('Membership expectedRows must be a non-negative integer.');
2984
+ if (context.dbSessionStrategy === 'gateway_only')
2985
+ return postRuntimeApi(context, {
2986
+ action: 'runtime_sheet_finish_membership',
2987
+ input,
2988
+ });
2989
+ const session = requireRuntimePostgresSession(
2990
+ await getRuntimeDbSession(context, {
2991
+ tableNamespace: input.tableNamespace,
2992
+ logicalTable: 'sheet_rows',
2993
+ operations: ['rows.read', 'rows.upsert'],
2994
+ limits: {},
2995
+ }),
2996
+ );
2997
+ return withRuntimeSheetQueryClient(
2998
+ context,
2999
+ session,
3000
+ {
3001
+ playName: context.playName,
3002
+ tableNamespace: input.tableNamespace,
3003
+ sheetContract: input.sheetContract,
3004
+ transactional: true,
3005
+ },
3006
+ async (client) => {
3007
+ if (!(await runtimeSheetMembershipCaptureAvailable(client, session)))
3008
+ return { captured: false, complete: false };
3009
+ const datasetId = createRuntimeDatasetId(
3010
+ session.playName,
3011
+ input.tableNamespace,
3012
+ );
3013
+ const result = await client.query<{ complete: boolean }>(
3014
+ `UPDATE ${runDatasetCatalogTable(session)} catalog
3015
+ SET membership_expected_rows = $3,
3016
+ membership_completed_at = CASE WHEN
3017
+ (SELECT count(*) FROM ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} WHERE run_id = $1 AND dataset_id = $2 AND phase = 'admitted') = $3
3018
+ AND (SELECT count(DISTINCT row_key) FROM ${fqRuntimeTable(session, RUNTIME_SHEET_RUN_MEMBERSHIP_TABLE)} WHERE run_id = $1 AND dataset_id = $2 AND phase = 'terminal') = $3
3019
+ THEN coalesce(catalog.membership_completed_at, now()) ELSE NULL END
3020
+ WHERE run_id = $1 AND dataset_id = $2 AND membership_started_at IS NOT NULL
3021
+ RETURNING membership_completed_at IS NOT NULL AS complete`,
3022
+ [input.runId, datasetId, input.expectedRows],
3023
+ );
3024
+ return {
3025
+ captured: result.rows.length === 1,
3026
+ complete: result.rows[0]?.complete === true,
3027
+ };
3028
+ },
3029
+ );
3030
+ }
3031
+
2875
3032
  function workReceiptTable(session: RuntimePostgresSession): string {
2876
3033
  return fqRuntimeTable(
2877
3034
  session,
@@ -6877,9 +7034,16 @@ export async function startRuntimeSheetDataset(
6877
7034
  const normalizeStartedAt = Date.now();
6878
7035
  const uniqueRows = new Map<
6879
7036
  string,
6880
- { row: Record<string, unknown>; inputIndex: number | null }
7037
+ {
7038
+ row: Record<string, unknown>;
7039
+ inputIndex: number | null;
7040
+ sourceRowIndex: number | null;
7041
+ source: Record<string, unknown>;
7042
+ rowOffset: number;
7043
+ }
6881
7044
  >();
6882
- for (const row of input.rows) {
7045
+ for (let rowOffset = 0; rowOffset < input.rows.length; rowOffset += 1) {
7046
+ const row = input.rows[rowOffset]!;
6883
7047
  // Materializes projected CSV aliases as visible cells and drops internal
6884
7048
  // __deepline* keys — a plain spread would silently lose the
6885
7049
  // non-enumerable alias fields on the JSON payload boundary.
@@ -6887,23 +7051,37 @@ export async function startRuntimeSheetDataset(
6887
7051
  const key =
6888
7052
  resolveMapRowOutcomeKey(row) ??
6889
7053
  derivePlayRowIdentity(cleanedRow, input.tableNamespace);
7054
+ const source =
7055
+ row.__deeplineSourceRow && typeof row.__deeplineSourceRow === 'object'
7056
+ ? (sanitizePostgresJsonValue(row.__deeplineSourceRow) as Record<
7057
+ string,
7058
+ unknown
7059
+ >)
7060
+ : {};
7061
+ const sourceRowIndex = normalizeRuntimeMapInputIndex(source.rowIndex);
6890
7062
  if (key && !uniqueRows.has(key)) {
6891
7063
  uniqueRows.set(key, {
6892
7064
  row: sanitizePostgresJsonValue(cleanedRow),
7065
+ source,
7066
+ sourceRowIndex,
6893
7067
  inputIndex: normalizeRuntimeMapInputIndex(
6894
7068
  row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex],
6895
7069
  ),
7070
+ rowOffset,
6896
7071
  });
6897
7072
  }
6898
7073
  }
6899
7074
  const inputOffset = Math.max(0, Math.floor(input.inputOffset ?? 0));
6900
- const rowEntries = [...uniqueRows.entries()].map(
6901
- ([key, entry], inputIndex) => ({
6902
- key,
6903
- row: entry.row,
6904
- inputIndex: entry.inputIndex ?? inputOffset + inputIndex,
6905
- }),
6906
- );
7075
+ const rowEntries = [...uniqueRows.entries()].map(([key, entry]) => ({
7076
+ key,
7077
+ row: entry.row,
7078
+ source: entry.source,
7079
+ // A CSV transformation may filter or reorder the physical source rows.
7080
+ // Its source ordinal must win over the compacted input index stamped by
7081
+ // ctx.dataset while preparing this page.
7082
+ inputIndex:
7083
+ entry.sourceRowIndex ?? entry.inputIndex ?? inputOffset + entry.rowOffset,
7084
+ }));
6907
7085
  timings.push({
6908
7086
  phase: 'normalize_rows',
6909
7087
  ms: Date.now() - normalizeStartedAt,
@@ -7135,6 +7313,60 @@ export async function startRuntimeSheetDataset(
7135
7313
  force: input.force === true,
7136
7314
  ...prepared,
7137
7315
  });
7316
+ const membershipCapture = await runtimeSheetMembershipCaptureAvailable(
7317
+ client,
7318
+ session,
7319
+ );
7320
+ if (membershipCapture) {
7321
+ const dispositions = new Set(
7322
+ prepared.rowDispositions
7323
+ .filter(
7324
+ (row) =>
7325
+ row.disposition === 'pending' ||
7326
+ row.disposition === 'completed',
7327
+ )
7328
+ .map((row) => row.key),
7329
+ );
7330
+ const admitted = rowEntries.filter((entry) =>
7331
+ dispositions.has(entry.key),
7332
+ );
7333
+ await client.query(
7334
+ `UPDATE ${runDatasetCatalogTable(session)} SET membership_started_at = coalesce(membership_started_at, now()) WHERE run_id = $1 AND dataset_id = $2`,
7335
+ [
7336
+ input.runId,
7337
+ createRuntimeDatasetId(session.playName, input.tableNamespace),
7338
+ ],
7339
+ );
7340
+ await persistRuntimeSheetMembershipBatch(
7341
+ client,
7342
+ session,
7343
+ admitted.map((entry) => ({
7344
+ runId: input.runId,
7345
+ datasetId: createRuntimeDatasetId(
7346
+ session.playName,
7347
+ input.tableNamespace,
7348
+ ),
7349
+ rowKey: entry.key,
7350
+ phase: 'admitted',
7351
+ generation: 'admitted',
7352
+ attemptSeq: null,
7353
+ rowVersion: null,
7354
+ inputIndex: entry.inputIndex,
7355
+ inputDigest: createHash('sha256')
7356
+ .update(stableStringify(entry.row))
7357
+ .digest('hex'),
7358
+ status: 'pending',
7359
+ data: entry.row,
7360
+ source: entry.source,
7361
+ })),
7362
+ );
7363
+ await captureRuntimeSheetTerminalMembership(client, session, {
7364
+ runId: input.runId,
7365
+ tableNamespace: input.tableNamespace,
7366
+ sheetContract: input.sheetContract,
7367
+ keys: admitted.map((entry) => entry.key),
7368
+ });
7369
+ }
7138
7370
  timings.push({
7139
7371
  phase: 'build_result',
7140
7372
  ms: Date.now() - buildStartedAt,
@@ -7146,6 +7378,7 @@ export async function startRuntimeSheetDataset(
7146
7378
  });
7147
7379
  return {
7148
7380
  ...built,
7381
+ membershipCapture,
7149
7382
  attemptExpiresAt,
7150
7383
  attemptSeq,
7151
7384
  writeVersion,
@@ -8982,11 +9215,20 @@ export async function completeRuntimeMapRows(
8982
9215
  }),
8983
9216
  )
8984
9217
  : { updated: 0, fencedRows: [] };
8985
- return classifyRuntimeMapRowsWrite({
9218
+ const classified = classifyRuntimeMapRowsWrite({
8986
9219
  submittedKeys: uniqueRows.keys(),
8987
9220
  updated: completed.updated + failed.updated,
8988
9221
  fencedRows: [...completed.fencedRows, ...failed.fencedRows],
8989
9222
  });
9223
+ if (await runtimeSheetMembershipCaptureAvailable(client, session)) {
9224
+ await captureRuntimeSheetTerminalMembership(client, session, {
9225
+ runId: input.runId,
9226
+ tableNamespace: input.tableNamespace,
9227
+ sheetContract,
9228
+ keys: classified.committedKeys,
9229
+ });
9230
+ }
9231
+ return classified;
8990
9232
  },
8991
9233
  )
8992
9234
  .then((result) => {