deepline 0.1.152 → 0.1.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +46 -6
  2. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1180 -825
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/batching.ts +34 -18
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +41 -0
  5. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +143 -8
  6. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +104 -0
  7. package/dist/bundling-sources/sdk/src/index.ts +0 -1
  8. package/dist/bundling-sources/sdk/src/play.ts +3 -48
  9. package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +27 -2
  10. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  11. package/dist/bundling-sources/sdk/src/worker-play-entry.ts +0 -10
  12. package/dist/bundling-sources/shared_libs/play-data-plane/index.ts +0 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +87 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +0 -59
  15. package/dist/bundling-sources/shared_libs/play-runtime/cell-staleness.ts +0 -253
  16. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +805 -1570
  17. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +47 -74
  18. package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +36 -14
  19. package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +145 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +284 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts +12 -5
  22. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +78 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +10 -45
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +1 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +923 -535
  26. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-neon-serverless.ts +58 -78
  27. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +12 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/step-program-dataset-builder.ts +1 -14
  29. package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts +159 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +4 -1
  31. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +32 -0
  32. package/dist/bundling-sources/shared_libs/plays/definition.ts +4 -2
  33. package/dist/bundling-sources/shared_libs/plays/runtime-validation.ts +3 -14
  34. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +1 -43
  35. package/dist/cli/index.js +1301 -399
  36. package/dist/cli/index.mjs +1269 -361
  37. package/dist/{compiler-manifest-BjoRENv9.d.ts → compiler-manifest-DW1flrHk.d.mts} +0 -9
  38. package/dist/{compiler-manifest-BjoRENv9.d.mts → compiler-manifest-DW1flrHk.d.ts} +0 -9
  39. package/dist/index.d.mts +9 -38
  40. package/dist/index.d.ts +9 -38
  41. package/dist/index.js +22 -11
  42. package/dist/index.mjs +22 -11
  43. package/dist/plays/bundle-play-file.d.mts +2 -2
  44. package/dist/plays/bundle-play-file.d.ts +2 -2
  45. package/package.json +1 -1
  46. package/dist/bundling-sources/shared_libs/play-data-plane/cell-policy.ts +0 -76
  47. package/dist/bundling-sources/shared_libs/play-runtime/progress-emitter.ts +0 -197
  48. package/dist/bundling-sources/shared_libs/play-runtime/waterfall-replay.ts +0 -79
@@ -12,41 +12,18 @@ import type { PlayQueueHint } from './governor/rate-state-backend';
12
12
  import type { GovernanceSnapshot } from './governor/governor';
13
13
  import type { AnyBatchOperationStrategy } from './batching-types';
14
14
  import type { ToolResultMetadataInput } from './tool-result';
15
- import type {
16
- AuthoredStaleAfterSeconds,
17
- CellStalenessPolicy,
18
- PreviousCell,
19
- } from './cell-staleness';
15
+ import type { PreviousCell } from './cell-staleness';
20
16
  import type { MapRowOutcome } from './durability-store';
21
17
 
22
18
  export interface RowState {
23
- waterfalls: Map<string, WaterfallState>;
24
19
  results: Map<string, unknown>;
25
20
  }
26
21
 
27
- export interface WaterfallState {
28
- status: 'pending' | 'complete' | 'failed';
29
- providerIndex: number;
30
- result?: unknown;
31
- error?: string;
32
- }
33
-
34
- export interface WaterfallRequest {
35
- rowId: number;
36
- fieldName?: string;
37
- tableNamespace?: string;
38
- rowKey?: string | null;
39
- key: string;
40
- toolName: string;
41
- input: Record<string, unknown>;
42
- providerIndex: number;
43
- opts?: WaterfallOptions;
44
- spec?: InlineWaterfallSpec;
45
- description?: string;
46
- }
47
-
48
22
  export interface ToolCallRequest {
49
23
  callId: string;
24
+ cacheKey: string;
25
+ receiptKey?: string | null;
26
+ force?: boolean;
50
27
  rowId: number;
51
28
  fieldName?: string;
52
29
  toolId: string;
@@ -78,6 +55,7 @@ export interface ClaimRuntimeStepReceiptInput {
78
55
  key: string;
79
56
  runId: string;
80
57
  reclaimRunning?: boolean;
58
+ forceRefresh?: boolean;
81
59
  }
82
60
 
83
61
  export interface CompleteRuntimeStepReceiptInput {
@@ -98,10 +76,31 @@ export interface SkipRuntimeStepReceiptInput {
98
76
  output?: unknown | null;
99
77
  }
100
78
 
101
- export interface WaterfallOptions {
102
- providers?: string[];
103
- timeout?: number;
104
- description?: string;
79
+ export interface GetRuntimeStepReceiptsInput {
80
+ keys: string[];
81
+ }
82
+
83
+ export interface ClaimRuntimeStepReceiptsInput {
84
+ keys: string[];
85
+ runId: string;
86
+ reclaimRunning?: boolean;
87
+ forceRefresh?: boolean;
88
+ }
89
+
90
+ export interface CompleteRuntimeStepReceiptsInput {
91
+ receipts: Array<{
92
+ key: string;
93
+ runId: string;
94
+ output: unknown | null;
95
+ }>;
96
+ }
97
+
98
+ export interface FailRuntimeStepReceiptsInput {
99
+ receipts: Array<{
100
+ key: string;
101
+ runId: string;
102
+ error: string;
103
+ }>;
105
104
  }
106
105
 
107
106
  export interface CsvOptions {
@@ -203,8 +202,6 @@ export interface PlayCallOptions {
203
202
  }
204
203
 
205
204
  export interface StepOptions {
206
- recompute?: boolean;
207
- recomputeOnError?: boolean;
208
205
  semanticKey?: string;
209
206
  staleAfterSeconds?: number;
210
207
  }
@@ -213,39 +210,6 @@ export interface FetchOptions {
213
210
  staleAfterSeconds?: number;
214
211
  }
215
212
 
216
- export interface InlineWaterfallToolStep {
217
- id: string;
218
- kind?: 'tool';
219
- toolId: string;
220
- mapInput: (input: Record<string, unknown>) => Record<string, unknown>;
221
- }
222
-
223
- export interface InlineWaterfallCodeStepContext {
224
- tools: {
225
- execute(request: ToolExecutionRequest): Promise<unknown>;
226
- };
227
- }
228
-
229
- export interface InlineWaterfallCodeStep {
230
- id: string;
231
- kind: 'code';
232
- run: (
233
- input: Record<string, unknown>,
234
- ctx: InlineWaterfallCodeStepContext,
235
- ) => unknown | Promise<unknown>;
236
- }
237
-
238
- export type InlineWaterfallStep =
239
- | InlineWaterfallToolStep
240
- | InlineWaterfallCodeStep;
241
-
242
- export interface InlineWaterfallSpec {
243
- id: string;
244
- output: string;
245
- minResults: number;
246
- steps: InlineWaterfallStep[];
247
- }
248
-
249
213
  export interface ResolvedPlayExecution {
250
214
  playId: string;
251
215
  code?: string | null;
@@ -298,10 +262,8 @@ export interface BatchRequest {
298
262
  }
299
263
 
300
264
  export interface MapStartResult {
301
- /** Rows that need processing (not yet completed in a prior run). */
265
+ /** Rows that need processing for this run. Previous values may be included for context. */
302
266
  pendingRows: Record<string, unknown>[];
303
- /** Rows already completed in a prior run (cached results). */
304
- completedRows: Record<string, unknown>[];
305
267
  /** Resolved table namespace. */
306
268
  tableNamespace: string;
307
269
  }
@@ -465,7 +427,6 @@ export interface ContextOptions {
465
427
  playId?: string;
466
428
  runId?: string;
467
429
  staticPipeline?: PlayStaticPipeline | null;
468
- cellPolicies?: Record<string, CellStalenessPolicy>;
469
430
  },
470
431
  ) => Promise<MapStartResult>;
471
432
  /**
@@ -490,6 +451,9 @@ export interface ContextOptions {
490
451
  getToolRetryPolicy?: (toolId: string) => Promise<{
491
452
  retrySafeTransientHttp?: boolean;
492
453
  } | null>;
454
+ getToolActionCacheVersion?: (
455
+ toolId: string,
456
+ ) => Promise<string> | string;
493
457
  getToolTargetGetters?: (
494
458
  toolId: string,
495
459
  output: string,
@@ -524,15 +488,27 @@ export interface ContextOptions {
524
488
  getRuntimeStepReceipt?: (
525
489
  input: GetRuntimeStepReceiptInput,
526
490
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
491
+ getRuntimeStepReceipts?: (
492
+ input: GetRuntimeStepReceiptsInput,
493
+ ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
527
494
  claimRuntimeStepReceipt?: (
528
495
  input: ClaimRuntimeStepReceiptInput,
529
496
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
497
+ claimRuntimeStepReceipts?: (
498
+ input: ClaimRuntimeStepReceiptsInput,
499
+ ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
530
500
  completeRuntimeStepReceipt?: (
531
501
  input: CompleteRuntimeStepReceiptInput,
532
502
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
503
+ completeRuntimeStepReceipts?: (
504
+ input: CompleteRuntimeStepReceiptsInput,
505
+ ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
533
506
  failRuntimeStepReceipt?: (
534
507
  input: FailRuntimeStepReceiptInput,
535
508
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
509
+ failRuntimeStepReceipts?: (
510
+ input: FailRuntimeStepReceiptsInput,
511
+ ) => Promise<RuntimeStepReceipt[]> | RuntimeStepReceipt[];
536
512
  skipRuntimeStepReceipt?: (
537
513
  input: SkipRuntimeStepReceiptInput,
538
514
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
@@ -616,7 +592,7 @@ export interface BatchResult {
616
592
  export interface ToolDefinition {
617
593
  toolId: string;
618
594
  provider: string;
619
- providers?: string[]; // For waterfalls
595
+ providers?: string[];
620
596
  supportsBatch?: boolean;
621
597
  }
622
598
 
@@ -678,9 +654,6 @@ export type RuntimeConditionalStepResolver<
678
654
 
679
655
  export type RuntimeStepProgramStep = {
680
656
  name: string;
681
- recompute?: boolean;
682
- recomputeOnError?: boolean;
683
- staleAfterSeconds?: AuthoredStaleAfterSeconds;
684
657
  resolver:
685
658
  | RuntimeStepResolver
686
659
  | RuntimeConditionalStepResolver
@@ -40,6 +40,38 @@ type TestRateLimitBatchResult = {
40
40
  }>;
41
41
  };
42
42
 
43
+ function testRateLimitBatchItems(
44
+ value: unknown,
45
+ ): Array<{ itemKey?: string; result?: unknown }> {
46
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
47
+ return [];
48
+ }
49
+ const record = value as Record<string, unknown>;
50
+ if (Array.isArray(record.items)) {
51
+ return record.items as Array<{ itemKey?: string; result?: unknown }>;
52
+ }
53
+ const candidates = [
54
+ record.data,
55
+ record.result,
56
+ record.output,
57
+ record.toolResponse &&
58
+ typeof record.toolResponse === 'object' &&
59
+ !Array.isArray(record.toolResponse)
60
+ ? (record.toolResponse as Record<string, unknown>).raw
61
+ : undefined,
62
+ record.toolOutput &&
63
+ typeof record.toolOutput === 'object' &&
64
+ !Array.isArray(record.toolOutput)
65
+ ? (record.toolOutput as Record<string, unknown>).raw
66
+ : undefined,
67
+ ];
68
+ for (const candidate of candidates) {
69
+ const items = testRateLimitBatchItems(candidate);
70
+ if (items.length > 0) return items;
71
+ }
72
+ return [];
73
+ }
74
+
43
75
  const testRateLimitBatchStrategy: BatchOperationStrategy<
44
76
  TestRateLimitPayload,
45
77
  TestRateLimitBatchPayload,
@@ -65,7 +97,9 @@ const testRateLimitBatchStrategy: BatchOperationStrategy<
65
97
  ? payloads[0].simulated_delay_ms
66
98
  : undefined;
67
99
  const items = payloads.map((payload, index) => ({
68
- itemKey: String(payload.lead_id || payload.row_number || `row_${index}`),
100
+ itemKey: String(
101
+ payload.lead_id || payload.row_number || payload.key || `row_${index}`,
102
+ ),
69
103
  payload,
70
104
  }));
71
105
  return {
@@ -82,19 +116,7 @@ const testRateLimitBatchStrategy: BatchOperationStrategy<
82
116
  };
83
117
  },
84
118
  splitResult(fullResult, compiled) {
85
- const container =
86
- fullResult && typeof fullResult === 'object'
87
- ? (fullResult as { data?: { items?: unknown[] }; items?: unknown[] })
88
- : {};
89
- const nestedItems =
90
- container.data && typeof container.data === 'object'
91
- ? container.data.items
92
- : undefined;
93
- const resultItems = Array.isArray(container.items)
94
- ? (container.items as Array<{ itemKey?: string; result?: unknown }>)
95
- : Array.isArray(nestedItems)
96
- ? (nestedItems as Array<{ itemKey?: string; result?: unknown }>)
97
- : [];
119
+ const resultItems = testRateLimitBatchItems(fullResult);
98
120
  return compiled.items.map((item, index) => ({
99
121
  itemKey: item.itemKey,
100
122
  result:
@@ -0,0 +1,145 @@
1
+ import {
2
+ normalizeTableNamespace,
3
+ sha256Hex,
4
+ stableStringify,
5
+ // Relative (not '@shared_libs/...') because this file ships inside the
6
+ // packed SDK's dist/bundling-sources graph, where only relative imports
7
+ // resolve.
8
+ } from '../plays/row-identity';
9
+
10
+ export const DURABLE_CALL_CACHE_POLICY_VERSION = 'call-cache-v1';
11
+
12
+ export type DurableCallKind = 'tool' | 'step' | 'fetch' | 'runPlay';
13
+
14
+ function validateStaleAfterSeconds(staleAfterSeconds?: number | null): void {
15
+ if (staleAfterSeconds === undefined || staleAfterSeconds === null) {
16
+ return;
17
+ }
18
+ if (
19
+ !Number.isFinite(staleAfterSeconds) ||
20
+ !Number.isInteger(staleAfterSeconds) ||
21
+ staleAfterSeconds <= 0
22
+ ) {
23
+ throw new Error(
24
+ 'staleAfterSeconds must be a positive whole number of seconds.',
25
+ );
26
+ }
27
+ }
28
+
29
+ export function durableCacheStaleBucket(input: {
30
+ staleAfterSeconds?: number | null;
31
+ nowMs?: number;
32
+ }): string | null {
33
+ validateStaleAfterSeconds(input.staleAfterSeconds);
34
+ if (
35
+ input.staleAfterSeconds === undefined ||
36
+ input.staleAfterSeconds === null
37
+ ) {
38
+ return null;
39
+ }
40
+ const nowMs = input.nowMs ?? Date.now();
41
+ return `${input.staleAfterSeconds}:${Math.floor(
42
+ nowMs / (input.staleAfterSeconds * 1000),
43
+ )}`;
44
+ }
45
+
46
+ export const durableCallStaleBucket = durableCacheStaleBucket;
47
+
48
+ export function buildDurableToolCallCacheKey(input: {
49
+ orgId?: string | null;
50
+ playId: string;
51
+ toolId: string;
52
+ requestInput: Record<string, unknown>;
53
+ authScopeDigest?: string | null;
54
+ providerActionVersion?: string | null;
55
+ cachePolicyVersion?: string | null;
56
+ staleAfterSeconds?: number | null;
57
+ }): string {
58
+ const orgId = input.orgId?.trim() || 'org';
59
+ const playId = input.playId.trim();
60
+ if (!playId) {
61
+ throw new Error(
62
+ 'Durable tool call cache key requires a non-empty play id.',
63
+ );
64
+ }
65
+ const toolId = input.toolId.trim();
66
+ if (!toolId) {
67
+ throw new Error(
68
+ 'Durable tool call cache key requires a non-empty tool id.',
69
+ );
70
+ }
71
+ const providerActionVersion = input.providerActionVersion?.trim();
72
+ if (!providerActionVersion) {
73
+ throw new Error(
74
+ 'Durable tool call cache key requires a provider action version.',
75
+ );
76
+ }
77
+ const digest = sha256Hex(
78
+ stableStringify({
79
+ kind: 'tool' satisfies DurableCallKind,
80
+ orgId,
81
+ playId,
82
+ toolId,
83
+ normalizedToolId: normalizeTableNamespace(toolId),
84
+ requestInput: input.requestInput,
85
+ authScopeDigest: input.authScopeDigest ?? null,
86
+ providerActionVersion,
87
+ cachePolicyVersion:
88
+ input.cachePolicyVersion ?? DURABLE_CALL_CACHE_POLICY_VERSION,
89
+ staleBucket: durableCacheStaleBucket({
90
+ staleAfterSeconds: input.staleAfterSeconds,
91
+ }),
92
+ }),
93
+ );
94
+ return `ctx:${orgId}:call:tool:${normalizeTableNamespace(toolId)}:${digest}`;
95
+ }
96
+
97
+ export function buildDurableCtxCallCacheKey(input: {
98
+ orgId?: string | null;
99
+ playId?: string | null;
100
+ kind: Exclude<DurableCallKind, 'tool'>;
101
+ id: string;
102
+ semanticKey?: string | null;
103
+ cachePolicyVersion?: string | null;
104
+ staleAfterSeconds?: number | null;
105
+ }): string {
106
+ const orgId = input.orgId?.trim() || 'org';
107
+ const playId = input.playId?.trim() || 'play';
108
+ const id = input.id.trim();
109
+ if (!id) {
110
+ throw new Error(
111
+ `Durable ${input.kind} call cache key requires a non-empty id.`,
112
+ );
113
+ }
114
+ const normalizedId = normalizeTableNamespace(id);
115
+ const digest = sha256Hex(
116
+ stableStringify({
117
+ kind: input.kind,
118
+ orgId,
119
+ playId,
120
+ id,
121
+ normalizedId,
122
+ semanticKey: input.semanticKey ?? null,
123
+ cachePolicyVersion:
124
+ input.cachePolicyVersion ?? DURABLE_CALL_CACHE_POLICY_VERSION,
125
+ staleBucket: durableCacheStaleBucket({
126
+ staleAfterSeconds: input.staleAfterSeconds,
127
+ }),
128
+ }),
129
+ );
130
+ return `ctx:${orgId}:call:${input.kind}:${normalizedId}:${digest}`;
131
+ }
132
+
133
+ export function buildDurableToolCallAuthScopeDigest(input: {
134
+ orgId?: string | null;
135
+ userEmail?: string | null;
136
+ toolId: string;
137
+ }): string {
138
+ return sha256Hex(
139
+ stableStringify({
140
+ orgId: input.orgId?.trim() || 'org',
141
+ actor: input.userEmail?.trim().toLowerCase() || 'workspace',
142
+ toolId: input.toolId.trim(),
143
+ }),
144
+ );
145
+ }
@@ -0,0 +1,284 @@
1
+ import {
2
+ deserializeToolExecuteResult,
3
+ isSerializedToolExecuteResult,
4
+ isToolExecuteResult,
5
+ serializeToolExecuteResult,
6
+ } from './tool-result';
7
+ import { assertNoSecretTaint } from './secret-capability';
8
+ import type { RuntimeStepReceipt } from './ctx-types';
9
+ import type { DurableReceiptRecoverySource } from './tool-execution-outcome';
10
+
11
+ const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
12
+ const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
13
+
14
+ export class RuntimeReceiptWaitTimeoutError extends Error {
15
+ constructor(key: string) {
16
+ super(`Timed out waiting for durable receipt ${key}.`);
17
+ this.name = 'RuntimeReceiptWaitTimeoutError';
18
+ }
19
+ }
20
+
21
+ export type DurableReceiptOperation = 'step' | 'tool' | 'fetch' | 'runPlay';
22
+
23
+ export type DurableReceiptExecutionStore = {
24
+ enabled: boolean;
25
+ get(receiptKey: string): Promise<RuntimeStepReceipt | null>;
26
+ getMany(receiptKeys: string[]): Promise<Map<string, RuntimeStepReceipt>>;
27
+ claim(
28
+ receiptKey: string,
29
+ runId: string,
30
+ reclaimRunning?: boolean,
31
+ forceRefresh?: boolean,
32
+ ): Promise<RuntimeStepReceipt | null>;
33
+ complete(
34
+ receiptKey: string,
35
+ runId: string,
36
+ output: unknown | null,
37
+ ): Promise<RuntimeStepReceipt | null>;
38
+ fail(
39
+ receiptKey: string,
40
+ runId: string,
41
+ error: string,
42
+ ): Promise<RuntimeStepReceipt | null>;
43
+ canPersistFailure: boolean;
44
+ canPersistCompletion: boolean;
45
+ };
46
+
47
+ export function runtimeReceiptOutput<T>(receipt: RuntimeStepReceipt): T {
48
+ return isSerializedToolExecuteResult(receipt.output)
49
+ ? (deserializeToolExecuteResult(receipt.output) as T)
50
+ : (receipt.output as T);
51
+ }
52
+
53
+ export async function waitForCompletedRuntimeReceipt(input: {
54
+ receiptKey: string;
55
+ store: Pick<DurableReceiptExecutionStore, 'getMany'>;
56
+ maxAttempts?: number;
57
+ delayMs?: number;
58
+ }): Promise<RuntimeStepReceipt> {
59
+ const maxAttempts = input.maxAttempts ?? DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
60
+ const delayMs = input.delayMs ?? DURABLE_RECEIPT_WAIT_DELAY_MS;
61
+ for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
62
+ if (attempt > 0) {
63
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
64
+ }
65
+ const receipt = (await input.store.getMany([input.receiptKey])).get(
66
+ input.receiptKey,
67
+ );
68
+ if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
69
+ return receipt;
70
+ }
71
+ if (receipt?.status === 'failed') {
72
+ throw new Error(
73
+ `Durable tool call ${input.receiptKey} failed: ${receipt.error ?? 'unknown error'}`,
74
+ );
75
+ }
76
+ }
77
+ throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
78
+ }
79
+
80
+ export async function executeWithDurableRuntimeReceipt<T>(input: {
81
+ operation: DurableReceiptOperation;
82
+ id: string;
83
+ runId: string;
84
+ receiptKey: string;
85
+ store: DurableReceiptExecutionStore;
86
+ force?: boolean;
87
+ repairRunningReceiptForSameRun?: boolean;
88
+ repairRunningReceiptForSameRunAfterWaitTimeout?: boolean;
89
+ runningReceiptWaitMaxAttempts?: number;
90
+ runningReceiptWaitDelayMs?: number;
91
+ reclaimRunning?: boolean;
92
+ markSkipped?: (output: T) => Promise<void> | void;
93
+ onRecovered?: (
94
+ output: T,
95
+ receipt: RuntimeStepReceipt,
96
+ source: DurableReceiptRecoverySource,
97
+ ) => T;
98
+ onClaimedResult?: (output: T, receiptKey: string) => T;
99
+ formatError: (error: unknown) => string;
100
+ log: (message: string) => void;
101
+ execute: () => Promise<T>;
102
+ }): Promise<T> {
103
+ if (!input.store.enabled) {
104
+ return await input.execute();
105
+ }
106
+
107
+ const recoverCompletedReceipt = async (
108
+ receipt: RuntimeStepReceipt,
109
+ source: DurableReceiptRecoverySource = 'cache',
110
+ ): Promise<T> => {
111
+ input.log(
112
+ `ctx.${input.operation}(${input.id}): recovered result from receipt`,
113
+ );
114
+ if (receipt.output === undefined) {
115
+ return receipt.output as T;
116
+ }
117
+ const output = runtimeReceiptOutput<T>(receipt);
118
+ const recovered = input.onRecovered
119
+ ? input.onRecovered(output, receipt, source)
120
+ : output;
121
+ if (input.markSkipped) {
122
+ await input.markSkipped(recovered);
123
+ }
124
+ return recovered;
125
+ };
126
+
127
+ const reclaimRunningReceipt = async (): Promise<
128
+ { kind: 'recovered'; output: T } | { kind: 'claimed' }
129
+ > => {
130
+ const reclaimed = await input.store.claim(
131
+ input.receiptKey,
132
+ input.runId,
133
+ true,
134
+ );
135
+ if (reclaimed?.status === 'completed' || reclaimed?.status === 'skipped') {
136
+ return {
137
+ kind: 'recovered',
138
+ output: await recoverCompletedReceipt(reclaimed, 'cache'),
139
+ };
140
+ }
141
+ if (reclaimed?.status === 'failed') {
142
+ throw new Error(
143
+ `ctx.${input.operation}(${input.id}): receipt is failed and could not be reclaimed: ${reclaimed.error ?? 'unknown error'}.`,
144
+ );
145
+ }
146
+ if (
147
+ reclaimed?.status === 'running' &&
148
+ reclaimed.claimState === 'existing'
149
+ ) {
150
+ throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
151
+ }
152
+ return { kind: 'claimed' };
153
+ };
154
+ const shouldRepairSameRunRunningReceipt = (
155
+ receipt: RuntimeStepReceipt,
156
+ ): boolean =>
157
+ input.repairRunningReceiptForSameRun === true &&
158
+ receipt.status === 'running' &&
159
+ typeof receipt.runId === 'string' &&
160
+ receipt.runId.trim() === input.runId;
161
+
162
+ const waitForRunningReceipt = async (): Promise<{
163
+ kind: 'recovered';
164
+ output: T;
165
+ }> => ({
166
+ kind: 'recovered',
167
+ output: await recoverCompletedReceipt(
168
+ await waitForCompletedRuntimeReceipt({
169
+ receiptKey: input.receiptKey,
170
+ store: input.store,
171
+ maxAttempts: input.runningReceiptWaitMaxAttempts,
172
+ delayMs: input.runningReceiptWaitDelayMs,
173
+ }),
174
+ 'in_flight',
175
+ ),
176
+ });
177
+ const waitForRunningReceiptOrTimeout = async (): Promise<{
178
+ kind: 'recovered';
179
+ output: T;
180
+ }> => waitForRunningReceipt();
181
+ const repairOrWaitForRunningReceipt = async (
182
+ receipt: RuntimeStepReceipt,
183
+ ): Promise<{ kind: 'recovered'; output: T } | { kind: 'claimed' }> => {
184
+ if (shouldRepairSameRunRunningReceipt(receipt)) {
185
+ const recovered = await reclaimRunningReceipt();
186
+ if (recovered.kind === 'recovered') return recovered;
187
+ return { kind: 'claimed' };
188
+ }
189
+ try {
190
+ return await waitForRunningReceiptOrTimeout();
191
+ } catch (error) {
192
+ if (
193
+ input.repairRunningReceiptForSameRunAfterWaitTimeout === true &&
194
+ error instanceof RuntimeReceiptWaitTimeoutError &&
195
+ receipt.status === 'running' &&
196
+ typeof receipt.runId === 'string' &&
197
+ receipt.runId.trim() === input.runId
198
+ ) {
199
+ const recovered = await reclaimRunningReceipt();
200
+ if (recovered.kind === 'recovered') return recovered;
201
+ return { kind: 'claimed' };
202
+ }
203
+ throw error;
204
+ }
205
+ };
206
+
207
+ const claimed = await input.store.claim(
208
+ input.receiptKey,
209
+ input.runId,
210
+ input.reclaimRunning === true || input.force === true,
211
+ input.force === true,
212
+ );
213
+ if (
214
+ input.force !== true &&
215
+ (claimed?.status === 'completed' || claimed?.status === 'skipped')
216
+ ) {
217
+ return await recoverCompletedReceipt(claimed);
218
+ }
219
+ if (!claimed) {
220
+ const latest = await input.store.get(input.receiptKey);
221
+ if (latest?.status === 'completed' || latest?.status === 'skipped') {
222
+ return await recoverCompletedReceipt(latest);
223
+ }
224
+ if (latest?.status === 'running') {
225
+ const recovered = await repairOrWaitForRunningReceipt(latest);
226
+ if (recovered.kind === 'recovered') return recovered.output;
227
+ } else {
228
+ if (latest?.status === 'failed') {
229
+ throw new Error(
230
+ `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${latest.error ?? 'unknown error'}.`,
231
+ );
232
+ }
233
+ throw new Error(
234
+ `ctx.${input.operation}(${input.id}): receipt claim did not return execution ownership.`,
235
+ );
236
+ }
237
+ } else if (claimed.status === 'running') {
238
+ if (claimed.claimState === 'existing') {
239
+ const recovered = await repairOrWaitForRunningReceipt(claimed);
240
+ if (recovered.kind === 'recovered') return recovered.output;
241
+ }
242
+ } else if (claimed.status === 'failed') {
243
+ throw new Error(
244
+ `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${claimed.error ?? 'unknown error'}.`,
245
+ );
246
+ }
247
+
248
+ let result: T;
249
+ try {
250
+ const executed = await input.execute();
251
+ result = input.onClaimedResult
252
+ ? input.onClaimedResult(executed, input.receiptKey)
253
+ : executed;
254
+ assertNoSecretTaint(result, `ctx.${input.operation} result`);
255
+ } catch (error) {
256
+ const failed = await input.store.fail(
257
+ input.receiptKey,
258
+ input.runId,
259
+ input.formatError(error),
260
+ );
261
+ if (!failed && input.store.canPersistFailure) {
262
+ throw new Error(
263
+ `ctx.${input.operation}(${input.id}): execution failed and failed receipt could not be persisted: ${input.formatError(error)}`,
264
+ );
265
+ }
266
+ throw error;
267
+ }
268
+
269
+ const completed = await input.store.complete(
270
+ input.receiptKey,
271
+ input.runId,
272
+ isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
273
+ );
274
+ if (!completed && input.store.canPersistCompletion) {
275
+ return result;
276
+ }
277
+ if (
278
+ completed &&
279
+ (completed.status === 'completed' || completed.status === 'skipped')
280
+ ) {
281
+ return await recoverCompletedReceipt(completed, 'owner');
282
+ }
283
+ return result;
284
+ }