deepline 0.1.181 → 0.1.183

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
  3. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
  4. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
  5. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1932 -1838
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
  7. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
  8. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +124 -12
  9. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
  10. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
  11. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
  12. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
  13. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
  14. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
  15. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
  16. package/dist/bundling-sources/sdk/src/client.ts +48 -5
  17. package/dist/bundling-sources/sdk/src/http.ts +23 -8
  18. package/dist/bundling-sources/sdk/src/play.ts +200 -26
  19. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
  20. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  21. package/dist/bundling-sources/sdk/src/types.ts +2 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
  23. package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
  27. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1857 -314
  28. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +106 -6
  29. package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
  33. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
  34. package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
  35. package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
  36. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
  38. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
  42. package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +120 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
  45. package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
  46. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
  47. package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
  50. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
  52. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
  53. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
  56. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
  57. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
  58. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
  59. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +139 -17
  60. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
  61. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
  62. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
  63. package/dist/cli/index.js +85 -23
  64. package/dist/cli/index.mjs +85 -23
  65. package/dist/index.d.mts +4 -1
  66. package/dist/index.d.ts +4 -1
  67. package/dist/index.js +274 -35
  68. package/dist/index.mjs +274 -35
  69. package/dist/plays/bundle-play-file.mjs +2 -2
  70. package/package.json +1 -1
@@ -36,10 +36,18 @@ import type {
36
36
  CompleteRuntimeReceiptsInput,
37
37
  FailRuntimeReceiptInput,
38
38
  FailRuntimeReceiptsInput,
39
+ HeartbeatRuntimeReceiptsInput,
40
+ ReleaseRuntimeReceiptInput,
39
41
  RuntimeApiCallInput,
40
42
  RuntimeApiCallResult,
41
43
  RuntimeReceiptInput,
42
44
  RuntimeReceiptsInput,
45
+ SheetAttemptHeartbeatInput,
46
+ SheetAttemptHeartbeatResult,
47
+ SheetAttemptReleaseInput,
48
+ SheetAttemptReleaseResult,
49
+ WorkReceiptReleaseInput,
50
+ WorkReceiptReleaseResult,
43
51
  SheetDatasetRowKeysInput,
44
52
  SheetDatasetRowKeysResult,
45
53
  SheetDatasetRowsInput,
@@ -94,7 +102,7 @@ function requireBinding(): HarnessBinding {
94
102
  if (!cachedBinding) {
95
103
  throw new Error(
96
104
  '[harness-stub] env.HARNESS is not wired. The coordinator must pass HARNESS in the WorkerLoader factory env. ' +
97
- 'See apps/play-runner-workers/src/coordinator-entry.ts loadDynamicPlayWorkerSync.',
105
+ 'See apps/play-runner-workers/src/coordinator-entry.ts -> loadDynamicPlayWorkerSync.',
98
106
  );
99
107
  }
100
108
  return cachedBinding;
@@ -157,7 +165,7 @@ export async function harnessCompleteRuntimeReceipt(
157
165
 
158
166
  export async function harnessCompleteRuntimeReceipts(
159
167
  input: CompleteRuntimeReceiptsInput,
160
- ): Promise<WorkReceipt[]> {
168
+ ): Promise<Array<WorkReceipt | null>> {
161
169
  return requireBinding().completeRuntimeReceipts(input);
162
170
  }
163
171
 
@@ -167,12 +175,24 @@ export async function harnessFailRuntimeReceipt(
167
175
  return requireBinding().failRuntimeReceipt(input);
168
176
  }
169
177
 
178
+ export async function harnessReleaseRuntimeReceipt(
179
+ input: ReleaseRuntimeReceiptInput,
180
+ ): Promise<WorkReceipt | null> {
181
+ return requireBinding().releaseRuntimeReceipt(input);
182
+ }
183
+
170
184
  export async function harnessFailRuntimeReceipts(
171
185
  input: FailRuntimeReceiptsInput,
172
- ): Promise<WorkReceipt[]> {
186
+ ): Promise<Array<WorkReceipt | null>> {
173
187
  return requireBinding().failRuntimeReceipts(input);
174
188
  }
175
189
 
190
+ export async function harnessHeartbeatRuntimeReceipts(
191
+ input: HeartbeatRuntimeReceiptsInput,
192
+ ): Promise<Array<WorkReceipt | null>> {
193
+ return requireBinding().heartbeatRuntimeReceipts(input);
194
+ }
195
+
176
196
  /**
177
197
  * Read a bounded staged-file byte range through typed harness RPC. This is the
178
198
  * only staged-file data path for per-play Workers; there is intentionally no
@@ -197,6 +217,24 @@ export async function harnessReadSheetDatasetRowKeys(
197
217
  return requireBinding().readSheetDatasetRowKeys(input);
198
218
  }
199
219
 
220
+ export async function harnessHeartbeatSheetAttempt(
221
+ input: SheetAttemptHeartbeatInput,
222
+ ): Promise<SheetAttemptHeartbeatResult> {
223
+ return requireBinding().heartbeatSheetAttempt(input);
224
+ }
225
+
226
+ export async function harnessReleaseSheetAttempt(
227
+ input: SheetAttemptReleaseInput,
228
+ ): Promise<SheetAttemptReleaseResult> {
229
+ return requireBinding().releaseSheetAttempt(input);
230
+ }
231
+
232
+ export async function harnessReleaseRuntimeReceipts(
233
+ input: WorkReceiptReleaseInput,
234
+ ): Promise<WorkReceiptReleaseResult> {
235
+ return requireBinding().releaseRuntimeReceipts(input);
236
+ }
237
+
200
238
  /**
201
239
  * Warm Postgres sessions in the long-lived harness Worker. This preserves the
202
240
  * map fast path without bundling the Neon client into every dynamic play.
@@ -224,6 +262,11 @@ export async function harnessStartSheetDataset(input: {
224
262
  sheetContract: unknown;
225
263
  rows: Array<Record<string, unknown>>;
226
264
  runId: string;
265
+ attemptId?: string | null;
266
+ attemptOwnerRunId?: string | null;
267
+ attemptExpiresAt?: string | null;
268
+ attemptSeq?: number | null;
269
+ attemptLeaseTtlMs?: number | null;
227
270
  inputOffset?: number;
228
271
  force?: boolean;
229
272
  userEmail?: string | null;
@@ -232,7 +275,12 @@ export async function harnessStartSheetDataset(input: {
232
275
  skipped: number;
233
276
  pendingRows: Array<Record<string, unknown>>;
234
277
  completedRows: Array<Record<string, unknown>>;
278
+ blockedRows: Array<Record<string, unknown>>;
235
279
  tableNamespace: string;
280
+ attemptId?: string;
281
+ attemptOwnerRunId?: string;
282
+ attemptExpiresAt?: string;
283
+ attemptSeq?: number;
236
284
  timings?: Array<Record<string, unknown>>;
237
285
  }> {
238
286
  return requireBinding().startSheetDataset(input);
@@ -253,6 +301,11 @@ export async function harnessPersistCompletedSheetRows(input: {
253
301
  rows: Array<Record<string, unknown>>;
254
302
  outputFields: string[];
255
303
  runId: string;
304
+ attemptId?: string | null;
305
+ attemptOwnerRunId?: string | null;
306
+ attemptExpiresAt?: string | null;
307
+ attemptSeq?: number | null;
308
+ forceFailedRows?: boolean;
256
309
  userEmail?: string | null;
257
310
  }): Promise<{ ok: true; rowsWritten: number; tableNamespace: string }> {
258
311
  return requireBinding().persistCompletedMapRows(input);
@@ -105,10 +105,10 @@ export const SDK_RELEASE = {
105
105
  // 0.1.154 removes the short-lived generated enrich StepOptions recompute
106
106
  // fields shipped in 0.1.153.
107
107
  // 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
108
- version: '0.1.181',
108
+ version: '0.1.183',
109
109
  apiContract: '2026-06-dataset-handle-results-hard-cutover',
110
110
  supportPolicy: {
111
- latest: '0.1.181',
111
+ latest: '0.1.183',
112
112
  minimumSupported: '0.1.53',
113
113
  deprecatedBelow: '0.1.53',
114
114
  commandMinimumSupported: [
@@ -1010,6 +1010,8 @@ export interface StartPlayRunRequest {
1010
1010
  profile?: string;
1011
1011
  /** Optional per-run provider execution mode for eval/smoke runs. */
1012
1012
  integrationMode?: 'live' | 'eval_stub' | 'fixture';
1013
+ /** Internal/dev-only runtime policy overrides for black-box durability tests. */
1014
+ testPolicyOverrides?: Record<string, unknown>;
1013
1015
  }
1014
1016
 
1015
1017
  /**
@@ -20,11 +20,23 @@ import type {
20
20
  PlayVisualNodeProgressMap,
21
21
  } from '@shared_libs/play-runtime/worker-api-types';
22
22
  import type { RuntimeStepReceipt } from '@shared_libs/play-runtime/ctx-types';
23
+ import type { WorkReceiptFailureKind } from '@shared_libs/play-runtime/work-receipts';
23
24
  import type {
24
25
  CreateDbSessionRequest,
25
26
  CreateDbSessionResponse,
26
27
  } from '@shared_libs/play-runtime/db-session';
27
28
  import type { PlayBundleArtifact } from '@shared_libs/plays/artifact-types';
29
+ import type {
30
+ PlayRunnerRateStateAcquireInput,
31
+ PlayRunnerRateStateAcquireResult,
32
+ PlayRunnerRateStatePenalizeInput,
33
+ PlayRunnerRateStateReleaseInput,
34
+ } from '@shared_libs/play-runtime/protocol';
35
+ import {
36
+ PLAY_RUNTIME_CONTRACT,
37
+ PLAY_RUNTIME_CONTRACT_HEADER,
38
+ } from '@shared_libs/play-runtime/runtime-contract';
39
+ import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
28
40
 
29
41
  export type StoredPlayArtifactPayload = {
30
42
  sourceCode: string;
@@ -157,6 +169,15 @@ type RuntimeApiRequest =
157
169
  maxCreditsPerRun?: number | null;
158
170
  finalItem: ComputeBillingItem;
159
171
  }
172
+ | ({
173
+ action: 'rate_state_acquire';
174
+ } & PlayRunnerRateStateAcquireInput)
175
+ | ({
176
+ action: 'rate_state_release';
177
+ } & PlayRunnerRateStateReleaseInput)
178
+ | ({
179
+ action: 'rate_state_penalize';
180
+ } & PlayRunnerRateStatePenalizeInput)
160
181
  | {
161
182
  action: 'get_runtime_step_receipt';
162
183
  playName: string;
@@ -168,28 +189,46 @@ type RuntimeApiRequest =
168
189
  playName: string;
169
190
  runId: string;
170
191
  key: string;
192
+ runAttempt?: number | null;
193
+ leaseAware?: boolean;
171
194
  reclaimRunning?: boolean;
172
195
  forceRefresh?: boolean;
196
+ forceFailedRefresh?: boolean;
173
197
  }
174
198
  | {
175
199
  action: 'complete_runtime_step_receipt';
176
200
  playName: string;
177
201
  runId: string;
178
202
  key: string;
203
+ runAttempt?: number | null;
204
+ leaseId?: string | null;
179
205
  output: unknown;
180
206
  }
207
+ | {
208
+ action: 'release_runtime_step_receipt';
209
+ playName: string;
210
+ runId: string;
211
+ key: string;
212
+ runAttempt?: number | null;
213
+ leaseId?: string | null;
214
+ }
181
215
  | {
182
216
  action: 'fail_runtime_step_receipt';
183
217
  playName: string;
184
218
  runId: string;
185
219
  key: string;
220
+ runAttempt?: number | null;
221
+ leaseId?: string | null;
186
222
  error: string;
223
+ failureKind?: WorkReceiptFailureKind | null;
187
224
  }
188
225
  | {
189
226
  action: 'skip_runtime_step_receipt';
190
227
  playName: string;
191
228
  runId: string;
192
229
  key: string;
230
+ runAttempt?: number | null;
231
+ leaseId?: string | null;
193
232
  output?: unknown;
194
233
  }
195
234
  | {
@@ -203,8 +242,11 @@ type RuntimeApiRequest =
203
242
  playName: string;
204
243
  runId: string;
205
244
  keys: string[];
245
+ runAttempt?: number | null;
246
+ leaseAware?: boolean;
206
247
  reclaimRunning?: boolean;
207
248
  forceRefresh?: boolean;
249
+ forceFailedRefresh?: boolean;
208
250
  }
209
251
  | {
210
252
  action: 'complete_runtime_step_receipts';
@@ -213,6 +255,8 @@ type RuntimeApiRequest =
213
255
  receipts: Array<{
214
256
  key: string;
215
257
  runId: string;
258
+ runAttempt?: number | null;
259
+ leaseId?: string | null;
216
260
  output: unknown;
217
261
  }>;
218
262
  }
@@ -223,8 +267,19 @@ type RuntimeApiRequest =
223
267
  receipts: Array<{
224
268
  key: string;
225
269
  runId: string;
270
+ runAttempt?: number | null;
271
+ leaseId?: string | null;
226
272
  error: string;
273
+ failureKind?: WorkReceiptFailureKind | null;
227
274
  }>;
275
+ }
276
+ | {
277
+ action: 'heartbeat_runtime_step_receipts';
278
+ playName: string;
279
+ runId: string;
280
+ runAttempt?: number | null;
281
+ leaseId: string;
282
+ keys: string[];
228
283
  };
229
284
 
230
285
  export type WorkerRuntimeApiContext = {
@@ -232,6 +287,7 @@ export type WorkerRuntimeApiContext = {
232
287
  executorToken: string;
233
288
  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
234
289
  vercelProtectionBypassToken?: string | null;
290
+ runtimeTestFaultHeader?: string | null;
235
291
  fetch?: typeof fetch;
236
292
  };
237
293
 
@@ -337,7 +393,10 @@ function isRetryableAppRuntimeAction(
337
393
  action === 'complete_runtime_step_receipt' ||
338
394
  action === 'complete_runtime_step_receipts' ||
339
395
  action === 'fail_runtime_step_receipt' ||
340
- action === 'fail_runtime_step_receipts'
396
+ action === 'fail_runtime_step_receipts' ||
397
+ action === 'rate_state_acquire' ||
398
+ action === 'rate_state_release' ||
399
+ action === 'rate_state_penalize'
341
400
  );
342
401
  }
343
402
 
@@ -366,7 +425,7 @@ function sleep(ms: number): Promise<void> {
366
425
 
367
426
  function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
368
427
  const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
369
- const url = new URL(`${baseUrl}/api/v2/plays/internal/runtime`);
428
+ const url = new URL(`${baseUrl}/api/v2/internal/play-runtime`);
370
429
  const bypassToken = context.vercelProtectionBypassToken?.trim();
371
430
  if (bypassToken) {
372
431
  url.searchParams.set('x-vercel-set-bypass-cookie', 'true');
@@ -443,12 +502,19 @@ async function postAppRuntimeApi<TResponse>(
443
502
  headers: {
444
503
  'content-type': 'application/json',
445
504
  authorization: `Bearer ${token}`,
505
+ [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
446
506
  ...(context.vercelProtectionBypassToken?.trim()
447
507
  ? {
448
508
  'x-vercel-protection-bypass':
449
509
  context.vercelProtectionBypassToken.trim(),
450
510
  }
451
511
  : {}),
512
+ ...(context.runtimeTestFaultHeader?.trim()
513
+ ? {
514
+ [PLAY_RUNTIME_TEST_FAULT_HEADER]:
515
+ context.runtimeTestFaultHeader.trim(),
516
+ }
517
+ : {}),
452
518
  ...(vercelProtectionCookie ? { cookie: vercelProtectionCookie } : {}),
453
519
  },
454
520
  body: JSON.stringify(body),
@@ -596,6 +662,36 @@ export async function createSignedStagedFileReadUrl(
596
662
  });
597
663
  }
598
664
 
665
+ export async function acquireRateStateViaAppRuntime(
666
+ context: WorkerRuntimeApiContext,
667
+ input: PlayRunnerRateStateAcquireInput,
668
+ ): Promise<PlayRunnerRateStateAcquireResult> {
669
+ return await postAppRuntimeApi<PlayRunnerRateStateAcquireResult>(context, {
670
+ action: 'rate_state_acquire',
671
+ ...input,
672
+ });
673
+ }
674
+
675
+ export async function releaseRateStateViaAppRuntime(
676
+ context: WorkerRuntimeApiContext,
677
+ input: PlayRunnerRateStateReleaseInput,
678
+ ): Promise<void> {
679
+ await postAppRuntimeApi<{ ok: true }>(context, {
680
+ action: 'rate_state_release',
681
+ ...input,
682
+ });
683
+ }
684
+
685
+ export async function penalizeRateStateViaAppRuntime(
686
+ context: WorkerRuntimeApiContext,
687
+ input: PlayRunnerRateStatePenalizeInput,
688
+ ): Promise<void> {
689
+ await postAppRuntimeApi<{ ok: true }>(context, {
690
+ action: 'rate_state_penalize',
691
+ ...input,
692
+ });
693
+ }
694
+
599
695
  export async function writeStagedFileFromAppRuntime(
600
696
  context: WorkerRuntimeApiContext,
601
697
  file: Pick<PlayStagedFileRef, 'storageKey'>,
@@ -839,8 +935,8 @@ export async function getRuntimeStepReceiptsViaAppRuntime(
839
935
  Extract<RuntimeApiRequest, { action: 'get_runtime_step_receipts' }>,
840
936
  'action'
841
937
  >,
842
- ): Promise<RuntimeStepReceipt[]> {
843
- return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
938
+ ): Promise<Array<RuntimeStepReceipt | null>> {
939
+ return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
844
940
  action: 'get_runtime_step_receipts',
845
941
  ...input,
846
942
  });
@@ -865,8 +961,8 @@ export async function claimRuntimeStepReceiptsViaAppRuntime(
865
961
  Extract<RuntimeApiRequest, { action: 'claim_runtime_step_receipts' }>,
866
962
  'action'
867
963
  >,
868
- ): Promise<RuntimeStepReceipt[]> {
869
- return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
964
+ ): Promise<Array<RuntimeStepReceipt | null>> {
965
+ return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
870
966
  action: 'claim_runtime_step_receipts',
871
967
  ...input,
872
968
  });
@@ -885,14 +981,27 @@ export async function completeRuntimeStepReceiptViaAppRuntime(
885
981
  });
886
982
  }
887
983
 
984
+ export async function releaseRuntimeStepReceiptViaAppRuntime(
985
+ context: WorkerRuntimeApiContext,
986
+ input: Omit<
987
+ Extract<RuntimeApiRequest, { action: 'release_runtime_step_receipt' }>,
988
+ 'action'
989
+ >,
990
+ ): Promise<RuntimeStepReceipt | null> {
991
+ return await postAppRuntimeApi<RuntimeStepReceipt | null>(context, {
992
+ action: 'release_runtime_step_receipt',
993
+ ...input,
994
+ });
995
+ }
996
+
888
997
  export async function completeRuntimeStepReceiptsViaAppRuntime(
889
998
  context: WorkerRuntimeApiContext,
890
999
  input: Omit<
891
1000
  Extract<RuntimeApiRequest, { action: 'complete_runtime_step_receipts' }>,
892
1001
  'action'
893
1002
  >,
894
- ): Promise<RuntimeStepReceipt[]> {
895
- return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
1003
+ ): Promise<Array<RuntimeStepReceipt | null>> {
1004
+ return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
896
1005
  action: 'complete_runtime_step_receipts',
897
1006
  ...input,
898
1007
  });
@@ -917,13 +1026,26 @@ export async function failRuntimeStepReceiptsViaAppRuntime(
917
1026
  Extract<RuntimeApiRequest, { action: 'fail_runtime_step_receipts' }>,
918
1027
  'action'
919
1028
  >,
920
- ): Promise<RuntimeStepReceipt[]> {
921
- return await postAppRuntimeApi<RuntimeStepReceipt[]>(context, {
1029
+ ): Promise<Array<RuntimeStepReceipt | null>> {
1030
+ return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
922
1031
  action: 'fail_runtime_step_receipts',
923
1032
  ...input,
924
1033
  });
925
1034
  }
926
1035
 
1036
+ export async function heartbeatRuntimeStepReceiptsViaAppRuntime(
1037
+ context: WorkerRuntimeApiContext,
1038
+ input: Omit<
1039
+ Extract<RuntimeApiRequest, { action: 'heartbeat_runtime_step_receipts' }>,
1040
+ 'action'
1041
+ >,
1042
+ ): Promise<Array<RuntimeStepReceipt | null>> {
1043
+ return await postAppRuntimeApi<Array<RuntimeStepReceipt | null>>(context, {
1044
+ action: 'heartbeat_runtime_step_receipts',
1045
+ ...input,
1046
+ });
1047
+ }
1048
+
927
1049
  export async function skipRuntimeStepReceiptViaAppRuntime(
928
1050
  context: WorkerRuntimeApiContext,
929
1051
  input: Omit<
@@ -0,0 +1,92 @@
1
+ export type RuntimeToolAuthScopeDigestResolver = ((
2
+ toolId: string,
3
+ ) => Promise<string>) & {
4
+ invalidate: (toolId?: string | null) => void;
5
+ };
6
+
7
+ export type RuntimeToolAuthScopeDigestFetcher = (
8
+ normalizedToolId: string,
9
+ ) => Promise<string>;
10
+
11
+ const DEFAULT_AUTH_SCOPE_RETRY_DELAYS_MS = [100, 250] as const;
12
+
13
+ function sleep(ms: number): Promise<void> {
14
+ return new Promise((resolve) => setTimeout(resolve, ms));
15
+ }
16
+
17
+ export function createRuntimeToolAuthScopeDigestResolver(input: {
18
+ fetchDigest: RuntimeToolAuthScopeDigestFetcher;
19
+ missingToolIdMessage?: string;
20
+ retryDelaysMs?: readonly number[];
21
+ sleepMs?: (ms: number) => Promise<void>;
22
+ }): RuntimeToolAuthScopeDigestResolver {
23
+ const resolved = new Map<string, string>();
24
+ const inFlight = new Map<string, Promise<string>>();
25
+ const retryDelaysMs =
26
+ input.retryDelaysMs ?? DEFAULT_AUTH_SCOPE_RETRY_DELAYS_MS;
27
+ const sleepMs = input.sleepMs ?? sleep;
28
+
29
+ const resolver = async (toolId: string): Promise<string> => {
30
+ const normalizedToolId = String(toolId || '').trim();
31
+ if (!normalizedToolId) {
32
+ throw new Error(
33
+ input.missingToolIdMessage ??
34
+ 'Runtime auth scope lookup needs tool id.',
35
+ );
36
+ }
37
+
38
+ const cachedDigest = resolved.get(normalizedToolId);
39
+ if (cachedDigest !== undefined) {
40
+ return cachedDigest;
41
+ }
42
+
43
+ const cachedPromise = inFlight.get(normalizedToolId);
44
+ if (cachedPromise) {
45
+ return await cachedPromise;
46
+ }
47
+
48
+ const promise = (async () => {
49
+ let lastError: unknown = null;
50
+ const maxAttempts = retryDelaysMs.length + 1;
51
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
52
+ try {
53
+ const digest = (await input.fetchDigest(normalizedToolId)).trim();
54
+ if (!digest) {
55
+ throw new Error(
56
+ `Runtime auth scope lookup for tool ${normalizedToolId} returned no digest.`,
57
+ );
58
+ }
59
+ resolved.set(normalizedToolId, digest);
60
+ return digest;
61
+ } catch (error) {
62
+ lastError = error;
63
+ if (attempt >= maxAttempts) {
64
+ throw error;
65
+ }
66
+ await sleepMs(retryDelaysMs[attempt - 1] ?? 250);
67
+ }
68
+ }
69
+ throw lastError instanceof Error
70
+ ? lastError
71
+ : new Error(String(lastError));
72
+ })();
73
+
74
+ inFlight.set(normalizedToolId, promise);
75
+ try {
76
+ return await promise;
77
+ } finally {
78
+ inFlight.delete(normalizedToolId);
79
+ }
80
+ };
81
+ resolver.invalidate = (toolId?: string | null): void => {
82
+ const normalizedToolId = String(toolId || '').trim();
83
+ if (!normalizedToolId) {
84
+ resolved.clear();
85
+ inFlight.clear();
86
+ return;
87
+ }
88
+ resolved.delete(normalizedToolId);
89
+ inFlight.delete(normalizedToolId);
90
+ };
91
+ return resolver;
92
+ }
@@ -9,7 +9,7 @@ export interface ChunkExecutionResult<TRequest, TResult> {
9
9
  * stay a per-request failure, not a run-level abort that discards billed
10
10
  * work (rows already persisted by completed calls stay recoverable).
11
11
  */
12
- error?: string;
12
+ error?: unknown;
13
13
  }
14
14
 
15
15
  export function formatChunkExecutionError(error: unknown): string {
@@ -83,8 +83,8 @@ export async function executeChunkedRequests<TRequest, TResult>(input: {
83
83
  execute: (request: TRequest) => Promise<TResult>;
84
84
  /**
85
85
  * Loud per-request failure hook. A rejected request is recorded as a
86
- * `result: null` entry with `error` set so the row-level state can carry
87
- * the provider error; it must never abort the chunk, the sibling requests,
86
+ * `result: null` entry with the raw `error` set so typed callers can keep
87
+ * domain errors intact; it must never abort the chunk, the sibling requests,
88
88
  * or the run. Callers use this to log and persist the failure.
89
89
  */
90
90
  onRequestError?: (request: TRequest, error: unknown) => void;
@@ -125,7 +125,7 @@ export async function executeChunkedRequests<TRequest, TResult>(input: {
125
125
  entry = {
126
126
  request,
127
127
  result: null,
128
- error: formatChunkExecutionError(error),
128
+ error,
129
129
  };
130
130
  }
131
131
  await notify(entry);
@@ -45,6 +45,13 @@ export type BatchOperationStrategy<
45
45
  batchOperation: string;
46
46
  kind: BatchStrategyKind;
47
47
  maxBatchSize: number;
48
+ /**
49
+ * Payload field names that can change the batch bucket. Static map planning
50
+ * may treat a tool call as one bucket only when these fields are absent from
51
+ * the static params/source text; otherwise it must assume worst-case bucket
52
+ * fanout because the field may vary per row.
53
+ */
54
+ bucketKeyPayloadFields?: string[];
48
55
  canBatchWith(left: TSingle, right: TSingle): boolean;
49
56
  toBucketKey(payload: TSingle): string;
50
57
  toItemKey(payload: TSingle): string;
@@ -61,6 +68,7 @@ export type AnyBatchOperationStrategy = {
61
68
  batchOperation: string;
62
69
  kind: BatchStrategyKind;
63
70
  maxBatchSize: number;
71
+ bucketKeyPayloadFields?: string[];
64
72
  canBatchWith(
65
73
  left: Record<string, unknown>,
66
74
  right: Record<string, unknown>,
@@ -0,0 +1,101 @@
1
+ import { sha256Hex, stableStringify } from '../plays/row-identity';
2
+
3
+ /**
4
+ * Canonical derivation of a `ctx.runPlay(...)` child run id.
5
+ *
6
+ * Both substrates that launch child runs derive the id here so a child's
7
+ * durable run id is stable across replays and shared code:
8
+ *
9
+ * - workers_edge: the coordinator (`apps/play-runner-workers`) derives the id
10
+ * from the durable child idempotency key it receives from the parent's
11
+ * runtime (`explicitIdempotencyKey`). Submit retries carry the same key, so
12
+ * the retry attaches to the same child run row.
13
+ * - in-process shared runtime (`context.ts`): there is no coordinator hop and
14
+ * no per-child idempotency key, so the id is derived from the semantic
15
+ * fallback (parent lineage + call key + input).
16
+ *
17
+ * The FORMAT (`play/<slug>/run/child-<digest>`) and the derivation CODE are
18
+ * identical on both sides. The DIGEST INPUTS differ by substrate on purpose:
19
+ *
20
+ * - `explicitIdempotencyKey` is only present on workers_edge (the coordinator
21
+ * path). When present it fully determines the digest; the other fields are
22
+ * ignored (they already fed the idempotency key upstream).
23
+ * - `graphHash` is only known on workers_edge (the compiled child manifest is
24
+ * threaded through the coordinator). In-process has no per-child manifest,
25
+ * so it passes `null`.
26
+ *
27
+ * Do NOT reorder or drop digest fields without accounting for coordinator
28
+ * child-submit idempotency: two calls that must attach to the same child run
29
+ * MUST produce the same id, byte-for-byte.
30
+ */
31
+ export interface ChildRunIdInputs {
32
+ /** Name of the child play being launched (`ctx.runPlay` target). */
33
+ childPlayName: string;
34
+ /** Run id of the parent making the `ctx.runPlay` call. */
35
+ parentRunId: string;
36
+ /**
37
+ * Durable child idempotency key when the launching substrate has one
38
+ * (workers_edge coordinator path). When set, it alone determines the digest.
39
+ */
40
+ explicitIdempotencyKey?: string | null;
41
+ /** Play name of the parent making the call (semantic-fallback input). */
42
+ parentPlayName?: string | null;
43
+ /** `ctx.runPlay(key, ...)` call key (semantic-fallback input). */
44
+ key?: string | null;
45
+ /**
46
+ * Canonical durable runPlay semantic key when the launching substrate has one.
47
+ * In-process runtimes pass the same key used by durable receipts so row-scoped
48
+ * receipt identity and child run identity cannot drift apart.
49
+ */
50
+ runPlaySemanticKey?: string | null;
51
+ /** Child input payload (semantic-fallback input). */
52
+ input?: Record<string, unknown> | null;
53
+ /**
54
+ * Compiled child graph hash. Only present on workers_edge; `null` in-process.
55
+ */
56
+ graphHash?: string | null;
57
+ }
58
+
59
+ function isRecord(value: unknown): value is Record<string, unknown> {
60
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
61
+ }
62
+
63
+ /** Slugify a play name into the `play/<slug>/...` path segment. */
64
+ export function slugifyPlayNameForRunId(playName: string): string {
65
+ return (
66
+ playName
67
+ .toLowerCase()
68
+ .replace(/[^a-z0-9-]/g, '-')
69
+ .replace(/-+/g, '-')
70
+ .replace(/^-+|-+$/g, '')
71
+ .slice(0, 64) || 'play'
72
+ );
73
+ }
74
+
75
+ export function buildChildRunId(inputs: ChildRunIdInputs): string {
76
+ const slug = slugifyPlayNameForRunId(inputs.childPlayName);
77
+ const explicitKey =
78
+ typeof inputs.explicitIdempotencyKey === 'string' &&
79
+ inputs.explicitIdempotencyKey.trim()
80
+ ? inputs.explicitIdempotencyKey.trim()
81
+ : null;
82
+ const runPlaySemanticKey =
83
+ typeof inputs.runPlaySemanticKey === 'string' &&
84
+ inputs.runPlaySemanticKey.trim()
85
+ ? inputs.runPlaySemanticKey.trim()
86
+ : null;
87
+ const semanticKey =
88
+ explicitKey ??
89
+ stableStringify({
90
+ parentRunId: inputs.parentRunId,
91
+ parentPlayName:
92
+ typeof inputs.parentPlayName === 'string' ? inputs.parentPlayName : null,
93
+ childPlayName: inputs.childPlayName,
94
+ key: typeof inputs.key === 'string' ? inputs.key : null,
95
+ ...(runPlaySemanticKey ? { runPlaySemanticKey } : {}),
96
+ input: isRecord(inputs.input) ? inputs.input : {},
97
+ graphHash: typeof inputs.graphHash === 'string' ? inputs.graphHash : null,
98
+ });
99
+ const digest = sha256Hex(semanticKey).slice(0, 24);
100
+ return `play/${slug}/run/child-${digest}`;
101
+ }