deepline 0.1.301 → 0.1.302
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +8 -0
- package/dist/bundling-sources/sdk/src/errors.ts +57 -15
- package/dist/bundling-sources/sdk/src/http.ts +82 -5
- package/dist/bundling-sources/sdk/src/index.ts +4 -0
- package/dist/bundling-sources/sdk/src/play.ts +29 -1
- package/dist/bundling-sources/sdk/src/release.ts +5 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +120 -14
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +59 -11
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/execution-plan.ts +0 -15
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +136 -36
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +17 -12
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +210 -5
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +4 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-contract-version.ts +6 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -0
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +159 -12
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +69 -1
- package/dist/bundling-sources/shared_libs/tool-execution-error.ts +428 -0
- package/dist/cli/index.js +501 -179
- package/dist/cli/index.mjs +501 -179
- package/dist/index.d.mts +39 -18
- package/dist/index.d.ts +39 -18
- package/dist/index.js +394 -59
- package/dist/index.mjs +391 -59
- package/dist/plays/bundle-play-file.d.mts +4 -2
- package/dist/plays/bundle-play-file.d.ts +4 -2
- package/dist/plays/bundle-play-file.mjs +116 -19
- package/dist/{compiler-manifest-BjopNIqx.d.mts → tool-execution-error-9mH4i-mJ.d.mts} +79 -1
- package/dist/{compiler-manifest-BjopNIqx.d.ts → tool-execution-error-9mH4i-mJ.d.ts} +79 -1
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +0 -166
|
@@ -81,6 +81,17 @@ import {
|
|
|
81
81
|
} from './resource-governor';
|
|
82
82
|
import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
|
|
83
83
|
import { ProviderExhaustedError } from './run-failure';
|
|
84
|
+
import {
|
|
85
|
+
buildPlayContractCompatibility,
|
|
86
|
+
normalizePlayContractCompatibility,
|
|
87
|
+
} from '@shared_libs/plays/contracts';
|
|
88
|
+
import {
|
|
89
|
+
LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION,
|
|
90
|
+
serializeToolExecutionFailure,
|
|
91
|
+
TOOL_EXECUTION_ERROR_SCHEMA_HEADER,
|
|
92
|
+
type ToolExecutionErrorSchemaVersion,
|
|
93
|
+
type ToolExecutionFailureV1,
|
|
94
|
+
} from '../tool-execution-error';
|
|
84
95
|
import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
|
|
85
96
|
import { pacingPolicyForTool } from './pacing';
|
|
86
97
|
import {
|
|
@@ -112,7 +123,7 @@ import {
|
|
|
112
123
|
parseToolExecuteAuthScopeChangedError,
|
|
113
124
|
ToolExecuteAuthScopeChangedError,
|
|
114
125
|
} from './tool-execute-retry-policy';
|
|
115
|
-
import {
|
|
126
|
+
import { createToolHttpError } from './tool-http-errors';
|
|
116
127
|
import {
|
|
117
128
|
describeTransportError,
|
|
118
129
|
transportGatewayOriginForDiagnostic,
|
|
@@ -132,6 +143,7 @@ import {
|
|
|
132
143
|
RuntimeReceiptWaitTimeoutError,
|
|
133
144
|
executeWithDurableRuntimeReceipt,
|
|
134
145
|
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
146
|
+
runtimeReceiptFailureError,
|
|
135
147
|
runtimeReceiptFailureKindForError,
|
|
136
148
|
runtimeReceiptOutput as durableRuntimeReceiptOutput,
|
|
137
149
|
waitForCompletedRuntimeReceipt,
|
|
@@ -285,6 +297,8 @@ type InlineCompositionStore = {
|
|
|
285
297
|
governor: PlayExecutionGovernor;
|
|
286
298
|
playName: string;
|
|
287
299
|
staticPipeline: ContextOptions['staticPipeline'];
|
|
300
|
+
/** Immutable error contract pinned by the child play artifact. */
|
|
301
|
+
toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion;
|
|
288
302
|
};
|
|
289
303
|
const inlineCompositionContext =
|
|
290
304
|
new AsyncLocalStorage<InlineCompositionStore>();
|
|
@@ -1586,6 +1600,19 @@ export class PlayContextImpl {
|
|
|
1586
1600
|
return active ? active.staticPipeline : this.#options.staticPipeline;
|
|
1587
1601
|
}
|
|
1588
1602
|
|
|
1603
|
+
/**
|
|
1604
|
+
* Inline children share this context's queues, but not its artifact contract.
|
|
1605
|
+
* Each child tool call must preserve the error shape pinned when that child
|
|
1606
|
+
* was published, even when the parent was published under another schema.
|
|
1607
|
+
*/
|
|
1608
|
+
private get currentToolErrorSchemaVersion(): ToolExecutionErrorSchemaVersion {
|
|
1609
|
+
return (
|
|
1610
|
+
this.activeInlineComposition?.toolErrorSchemaVersion ??
|
|
1611
|
+
this.#options.toolErrorSchemaVersion ??
|
|
1612
|
+
LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION
|
|
1613
|
+
);
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1589
1616
|
private recordPlayCallStep(input: {
|
|
1590
1617
|
playId: string;
|
|
1591
1618
|
execution?: 'inline';
|
|
@@ -2197,6 +2224,7 @@ export class PlayContextImpl {
|
|
|
2197
2224
|
error: string,
|
|
2198
2225
|
leaseId?: string | null,
|
|
2199
2226
|
failureKind?: WorkReceiptFailureKind | null,
|
|
2227
|
+
errorPayload?: ToolExecutionFailureV1 | null,
|
|
2200
2228
|
): Promise<RuntimeStepReceipt | null> {
|
|
2201
2229
|
if (!this.#options.failRuntimeStepReceipt) {
|
|
2202
2230
|
return null;
|
|
@@ -2207,6 +2235,7 @@ export class PlayContextImpl {
|
|
|
2207
2235
|
runAttempt: this.currentRunAttempt,
|
|
2208
2236
|
...(leaseId ? { leaseId } : {}),
|
|
2209
2237
|
...(failureKind ? { failureKind } : {}),
|
|
2238
|
+
...(errorPayload ? { errorPayload } : {}),
|
|
2210
2239
|
error,
|
|
2211
2240
|
});
|
|
2212
2241
|
if (!failed || typeof failed.key !== 'string' || !failed.key.trim()) {
|
|
@@ -2642,6 +2671,7 @@ export class PlayContextImpl {
|
|
|
2642
2671
|
error: string;
|
|
2643
2672
|
leaseId?: string | null;
|
|
2644
2673
|
failureKind?: WorkReceiptFailureKind | null;
|
|
2674
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
2645
2675
|
}>,
|
|
2646
2676
|
): Promise<Map<string, RuntimeStepReceipt>> {
|
|
2647
2677
|
const normalizedInputs = receipts.filter((receipt) => receipt.key.trim());
|
|
@@ -2664,6 +2694,7 @@ export class PlayContextImpl {
|
|
|
2664
2694
|
receipt.error,
|
|
2665
2695
|
receipt.leaseId,
|
|
2666
2696
|
receipt.failureKind,
|
|
2697
|
+
receipt.errorPayload,
|
|
2667
2698
|
),
|
|
2668
2699
|
),
|
|
2669
2700
|
);
|
|
@@ -2786,13 +2817,14 @@ export class PlayContextImpl {
|
|
|
2786
2817
|
this.heartbeatRuntimeStepReceipt(receiptKey, runId, leaseId),
|
|
2787
2818
|
}
|
|
2788
2819
|
: {}),
|
|
2789
|
-
fail: (receiptKey, runId, error, leaseId, failureKind) =>
|
|
2820
|
+
fail: (receiptKey, runId, error, leaseId, failureKind, errorPayload) =>
|
|
2790
2821
|
this.failRuntimeStepReceipt(
|
|
2791
2822
|
receiptKey,
|
|
2792
2823
|
runId,
|
|
2793
2824
|
error,
|
|
2794
2825
|
leaseId,
|
|
2795
2826
|
failureKind,
|
|
2827
|
+
errorPayload,
|
|
2796
2828
|
),
|
|
2797
2829
|
canPersistFailure: Boolean(this.#options.failRuntimeStepReceipt),
|
|
2798
2830
|
canPersistCompletion: Boolean(
|
|
@@ -2832,6 +2864,7 @@ export class PlayContextImpl {
|
|
|
2832
2864
|
receiptKey: key,
|
|
2833
2865
|
store: this.durableReceiptExecutionStore(),
|
|
2834
2866
|
maxAttempts,
|
|
2867
|
+
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
2835
2868
|
});
|
|
2836
2869
|
}
|
|
2837
2870
|
|
|
@@ -2848,6 +2881,7 @@ export class PlayContextImpl {
|
|
|
2848
2881
|
store: this.durableReceiptExecutionStore(),
|
|
2849
2882
|
maxAttempts,
|
|
2850
2883
|
log: (message) => this.log(message),
|
|
2884
|
+
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
2851
2885
|
});
|
|
2852
2886
|
}
|
|
2853
2887
|
|
|
@@ -2910,6 +2944,7 @@ export class PlayContextImpl {
|
|
|
2910
2944
|
completedCacheOnly: operation === 'tool',
|
|
2911
2945
|
requiresExecutionLock: opts.requiresExecutionLock,
|
|
2912
2946
|
executionLockTtlMs: opts.executionLockTtlMs,
|
|
2947
|
+
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
2913
2948
|
formatError: (error) => this.formatRuntimeError(error),
|
|
2914
2949
|
log: (message) => this.log(message),
|
|
2915
2950
|
execute: opts.execute,
|
|
@@ -3635,6 +3670,7 @@ export class PlayContextImpl {
|
|
|
3635
3670
|
runId: this.currentReceiptOwnerRunId,
|
|
3636
3671
|
leaseId: request.receiptLeaseId ?? null,
|
|
3637
3672
|
error: message,
|
|
3673
|
+
errorPayload: serializeToolExecutionFailure(error),
|
|
3638
3674
|
failureKind: runtimeReceiptFailureKindForError(error),
|
|
3639
3675
|
},
|
|
3640
3676
|
]);
|
|
@@ -7008,6 +7044,11 @@ export class PlayContextImpl {
|
|
|
7008
7044
|
`Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
|
|
7009
7045
|
);
|
|
7010
7046
|
}
|
|
7047
|
+
const childToolErrorSchemaVersion = normalizePlayContractCompatibility(
|
|
7048
|
+
resolvedPlay.contractSnapshot?.compatibility ??
|
|
7049
|
+
resolvedPlay.artifact?.compatibility ??
|
|
7050
|
+
buildPlayContractCompatibility(),
|
|
7051
|
+
).toolErrorSchemaVersion;
|
|
7011
7052
|
const childExecutionDecision = resolveChildExecutionStrategy({
|
|
7012
7053
|
pipeline: resolvedPlay.staticPipeline,
|
|
7013
7054
|
timeoutMs: options?.timeoutMs,
|
|
@@ -7078,6 +7119,7 @@ export class PlayContextImpl {
|
|
|
7078
7119
|
governor: inlineChildGovernor,
|
|
7079
7120
|
playName: resolvedName,
|
|
7080
7121
|
staticPipeline: resolvedPlay.staticPipeline ?? null,
|
|
7122
|
+
toolErrorSchemaVersion: childToolErrorSchemaVersion,
|
|
7081
7123
|
},
|
|
7082
7124
|
() => this.executeResolvedPlay(resolvedPlay, this, input),
|
|
7083
7125
|
);
|
|
@@ -8023,7 +8065,11 @@ export class PlayContextImpl {
|
|
|
8023
8065
|
await this.rejectToolCall(
|
|
8024
8066
|
toolId,
|
|
8025
8067
|
request,
|
|
8026
|
-
|
|
8068
|
+
runtimeReceiptFailureError(
|
|
8069
|
+
reclaimed,
|
|
8070
|
+
'Durable tool call failed',
|
|
8071
|
+
this.currentToolErrorSchemaVersion,
|
|
8072
|
+
),
|
|
8027
8073
|
);
|
|
8028
8074
|
}
|
|
8029
8075
|
return;
|
|
@@ -8194,7 +8240,11 @@ export class PlayContextImpl {
|
|
|
8194
8240
|
await this.rejectToolCall(
|
|
8195
8241
|
toolId,
|
|
8196
8242
|
request,
|
|
8197
|
-
|
|
8243
|
+
runtimeReceiptFailureError(
|
|
8244
|
+
claim,
|
|
8245
|
+
'Durable tool call failed',
|
|
8246
|
+
this.currentToolErrorSchemaVersion,
|
|
8247
|
+
),
|
|
8198
8248
|
);
|
|
8199
8249
|
}
|
|
8200
8250
|
continue;
|
|
@@ -8668,6 +8718,7 @@ export class PlayContextImpl {
|
|
|
8668
8718
|
? 'execute-fenced-v1'
|
|
8669
8719
|
: 'execute';
|
|
8670
8720
|
const url = `${this.#options.baseUrl}/api/v2/integrations/${encodeURIComponent(toolId)}/${executeSuffix}`;
|
|
8721
|
+
const toolErrorSchemaVersion = this.currentToolErrorSchemaVersion;
|
|
8671
8722
|
const timeoutMs = resolveToolRuntimeTimeoutMs(toolId, options?.timeoutMs);
|
|
8672
8723
|
const provider = toolId.split(/[._]/)[0]?.trim() || 'provider';
|
|
8673
8724
|
const activityId = `provider:${toolId}`;
|
|
@@ -8678,11 +8729,39 @@ export class PlayContextImpl {
|
|
|
8678
8729
|
// execution. Per-provider rate admission is intentionally deferred until
|
|
8679
8730
|
// each physical fetch is ready to leave this process.
|
|
8680
8731
|
const admissionStartedAt = Date.now();
|
|
8681
|
-
const toolSlot = await this.resourceGovernor
|
|
8682
|
-
|
|
8683
|
-
|
|
8684
|
-
|
|
8685
|
-
|
|
8732
|
+
const toolSlot = await this.resourceGovernor
|
|
8733
|
+
.acquireTool({
|
|
8734
|
+
orgId: this.#options.orgId ?? null,
|
|
8735
|
+
providerResourceKey: `tool:${toolId}`,
|
|
8736
|
+
toolId,
|
|
8737
|
+
})
|
|
8738
|
+
.catch((error: unknown) => {
|
|
8739
|
+
if (!(error instanceof ProviderExhaustedError)) throw error;
|
|
8740
|
+
const retryAtMs = Date.parse(error.retryAt);
|
|
8741
|
+
throw createToolHttpError(
|
|
8742
|
+
toolErrorSchemaVersion,
|
|
8743
|
+
error.message,
|
|
8744
|
+
null,
|
|
8745
|
+
429,
|
|
8746
|
+
'repairable',
|
|
8747
|
+
{
|
|
8748
|
+
toolId,
|
|
8749
|
+
provider: error.provider,
|
|
8750
|
+
operation: toolId,
|
|
8751
|
+
code: error.code,
|
|
8752
|
+
origin: 'provider',
|
|
8753
|
+
category: 'rate_limit',
|
|
8754
|
+
retryable: true,
|
|
8755
|
+
statusCode: 429,
|
|
8756
|
+
requestId: null,
|
|
8757
|
+
retryAfterMs: Number.isFinite(retryAtMs)
|
|
8758
|
+
? Math.max(0, retryAtMs - Date.now())
|
|
8759
|
+
: null,
|
|
8760
|
+
networkKind: null,
|
|
8761
|
+
networkScope: null,
|
|
8762
|
+
},
|
|
8763
|
+
);
|
|
8764
|
+
});
|
|
8686
8765
|
if (runtimeReceiptReadTraceEnabled) {
|
|
8687
8766
|
this.log(
|
|
8688
8767
|
`[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
|
|
@@ -8822,11 +8901,30 @@ export class PlayContextImpl {
|
|
|
8822
8901
|
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
8823
8902
|
return;
|
|
8824
8903
|
}
|
|
8825
|
-
throw
|
|
8904
|
+
throw createToolHttpError(
|
|
8905
|
+
toolErrorSchemaVersion,
|
|
8826
8906
|
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${diagnostic.message ?? 'unknown transport error'}`,
|
|
8827
8907
|
null,
|
|
8828
8908
|
0,
|
|
8829
8909
|
'repairable',
|
|
8910
|
+
{
|
|
8911
|
+
toolId,
|
|
8912
|
+
provider,
|
|
8913
|
+
operation: toolId,
|
|
8914
|
+
code: 'NETWORK_ERROR',
|
|
8915
|
+
origin: 'deepline',
|
|
8916
|
+
category: 'network',
|
|
8917
|
+
retryable: false,
|
|
8918
|
+
statusCode: null,
|
|
8919
|
+
requestId: input.requestId,
|
|
8920
|
+
retryAfterMs: null,
|
|
8921
|
+
networkKind: diagnostic.message
|
|
8922
|
+
?.toLowerCase()
|
|
8923
|
+
.includes('timed out')
|
|
8924
|
+
? 'timeout'
|
|
8925
|
+
: 'unavailable',
|
|
8926
|
+
networkScope: 'runtime_to_deepline',
|
|
8927
|
+
},
|
|
8830
8928
|
);
|
|
8831
8929
|
};
|
|
8832
8930
|
|
|
@@ -8916,6 +9014,9 @@ export class PlayContextImpl {
|
|
|
8916
9014
|
V2_EXECUTE_RESPONSE_CONTRACT,
|
|
8917
9015
|
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
8918
9016
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
9017
|
+
[TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
|
|
9018
|
+
toolErrorSchemaVersion,
|
|
9019
|
+
),
|
|
8919
9020
|
'x-deepline-request-id': deeplineRequestId,
|
|
8920
9021
|
...(providerIdempotencyKey
|
|
8921
9022
|
? {
|
|
@@ -8982,7 +9083,8 @@ export class PlayContextImpl {
|
|
|
8982
9083
|
throw heartbeatFailure;
|
|
8983
9084
|
}
|
|
8984
9085
|
if (error instanceof ToolExecuteInvalidJsonError) {
|
|
8985
|
-
throw
|
|
9086
|
+
throw createToolHttpError(
|
|
9087
|
+
toolErrorSchemaVersion,
|
|
8986
9088
|
`Tool ${toolId} returned an invalid JSON response body.`,
|
|
8987
9089
|
null,
|
|
8988
9090
|
response?.status ?? 0,
|
|
@@ -9003,7 +9105,8 @@ export class PlayContextImpl {
|
|
|
9003
9105
|
response?.status === 402
|
|
9004
9106
|
) {
|
|
9005
9107
|
const diagnostic = describeTransportError(transportError);
|
|
9006
|
-
throw
|
|
9108
|
+
throw createToolHttpError(
|
|
9109
|
+
toolErrorSchemaVersion,
|
|
9007
9110
|
`Tool ${toolId} returned HTTP 402 but its response body could not be read; the run was halted because the payment or capacity denial could not be classified safely: ${diagnostic.message ?? 'unknown transport error'}`,
|
|
9008
9111
|
{
|
|
9009
9112
|
kind: 'billing_cap_exceeded',
|
|
@@ -9069,7 +9172,8 @@ export class PlayContextImpl {
|
|
|
9069
9172
|
error instanceof ToolExecuteResponseBodyTransportError
|
|
9070
9173
|
? 'response body transport failed after response headers'
|
|
9071
9174
|
: 'request transport failed after dispatch before response headers';
|
|
9072
|
-
throw
|
|
9175
|
+
throw createToolHttpError(
|
|
9176
|
+
toolErrorSchemaVersion,
|
|
9073
9177
|
`Tool ${toolId} ${failureBoundary}; the ambiguous call was not retried because this runtime has no durable invocation fence: ${diagnostic.message ?? 'unknown transport error'}`,
|
|
9074
9178
|
null,
|
|
9075
9179
|
0,
|
|
@@ -9147,6 +9251,7 @@ export class PlayContextImpl {
|
|
|
9147
9251
|
status: response.status,
|
|
9148
9252
|
attempt: httpFailureAttempt,
|
|
9149
9253
|
bodyText: text,
|
|
9254
|
+
schemaVersion: toolErrorSchemaVersion,
|
|
9150
9255
|
retryAfterHeader: response.headers.get('retry-after'),
|
|
9151
9256
|
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
9152
9257
|
...(providerCallElapsedMs !== null
|
|
@@ -9230,7 +9335,8 @@ export class PlayContextImpl {
|
|
|
9230
9335
|
}
|
|
9231
9336
|
|
|
9232
9337
|
if (!responseData) {
|
|
9233
|
-
throw
|
|
9338
|
+
throw createToolHttpError(
|
|
9339
|
+
toolErrorSchemaVersion,
|
|
9234
9340
|
`Tool ${toolId} returned an empty successful response body.`,
|
|
9235
9341
|
null,
|
|
9236
9342
|
response.status,
|
|
@@ -26,6 +26,10 @@ import type { WorkReceiptFailureKind } from './work-receipts';
|
|
|
26
26
|
import type { RunExecutionScope } from './run-execution-scope';
|
|
27
27
|
import type { PlayCallExecution } from './play-call-execution';
|
|
28
28
|
import type { PlayActivityEvent } from './activity-observation';
|
|
29
|
+
import type {
|
|
30
|
+
ToolExecutionErrorSchemaVersion,
|
|
31
|
+
ToolExecutionFailureV1,
|
|
32
|
+
} from '../tool-execution-error';
|
|
29
33
|
|
|
30
34
|
export interface RowState {
|
|
31
35
|
results: Map<string, unknown>;
|
|
@@ -41,6 +45,7 @@ export interface ToolCallRequest {
|
|
|
41
45
|
receiptLeaseExpiresAt?: string | null;
|
|
42
46
|
force?: boolean;
|
|
43
47
|
forceFailedRefresh?: boolean;
|
|
48
|
+
requiresExecutionFence?: boolean;
|
|
44
49
|
rowId: number;
|
|
45
50
|
fieldName?: string;
|
|
46
51
|
toolId: string;
|
|
@@ -62,6 +67,7 @@ export interface RuntimeStepReceipt {
|
|
|
62
67
|
status: 'queued' | 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
|
|
63
68
|
output?: unknown;
|
|
64
69
|
error?: string;
|
|
70
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
65
71
|
failureKind?: WorkReceiptFailureKind | null;
|
|
66
72
|
runId?: string | null;
|
|
67
73
|
leaseId?: string | null;
|
|
@@ -127,6 +133,7 @@ export interface FailRuntimeStepReceiptInput {
|
|
|
127
133
|
runAttempt?: number | null;
|
|
128
134
|
leaseId?: string | null;
|
|
129
135
|
error: string;
|
|
136
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
130
137
|
failureKind?: WorkReceiptFailureKind | null;
|
|
131
138
|
}
|
|
132
139
|
|
|
@@ -188,6 +195,7 @@ export interface FailRuntimeStepReceiptsInput {
|
|
|
188
195
|
runAttempt?: number | null;
|
|
189
196
|
leaseId?: string | null;
|
|
190
197
|
error: string;
|
|
198
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
191
199
|
failureKind?: WorkReceiptFailureKind | null;
|
|
192
200
|
}>;
|
|
193
201
|
}
|
|
@@ -509,6 +517,8 @@ export type IntegrationEventWaitHandler = {
|
|
|
509
517
|
export interface ContextOptions {
|
|
510
518
|
/** Immutable logical and physical execution identity for this context. */
|
|
511
519
|
executionScope?: RunExecutionScope;
|
|
520
|
+
/** Error shape pinned by the immutable play artifact; missing preserves legacy schema 0. */
|
|
521
|
+
toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
512
522
|
/** Short-lived HMAC-signed internal token for tool callbacks. Required for cloud execution. */
|
|
513
523
|
executorToken?: string;
|
|
514
524
|
baseUrl?: string;
|
|
@@ -20,6 +20,14 @@ import {
|
|
|
20
20
|
runtimeLeaseHeartbeatIntervalMs,
|
|
21
21
|
} from './lease-policy';
|
|
22
22
|
import { createRuntimeReceiptHeartbeatSupervisor } from './receipt-heartbeat-supervisor';
|
|
23
|
+
import {
|
|
24
|
+
deserializeToolExecutionFailure,
|
|
25
|
+
serializeToolExecutionFailure,
|
|
26
|
+
} from '../tool-execution-error';
|
|
27
|
+
import type {
|
|
28
|
+
ToolExecutionErrorSchemaVersion,
|
|
29
|
+
ToolExecutionFailureV1,
|
|
30
|
+
} from '../tool-execution-error';
|
|
23
31
|
|
|
24
32
|
const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
|
|
25
33
|
const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
|
|
@@ -196,6 +204,7 @@ export type DurableReceiptExecutionStore = {
|
|
|
196
204
|
error: string,
|
|
197
205
|
leaseId?: string | null,
|
|
198
206
|
failureKind?: WorkReceiptFailureKind | null,
|
|
207
|
+
errorPayload?: ToolExecutionFailureV1 | null,
|
|
199
208
|
): Promise<RuntimeStepReceipt | null>;
|
|
200
209
|
canPersistFailure: boolean;
|
|
201
210
|
canPersistCompletion: boolean;
|
|
@@ -290,12 +299,28 @@ export function runtimeReceiptOutput<T>(receipt: RuntimeStepReceipt): T {
|
|
|
290
299
|
: (receipt.output as T);
|
|
291
300
|
}
|
|
292
301
|
|
|
302
|
+
export function runtimeReceiptFailureError(
|
|
303
|
+
receipt: RuntimeStepReceipt,
|
|
304
|
+
legacyPrefix: string,
|
|
305
|
+
toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion,
|
|
306
|
+
): Error {
|
|
307
|
+
const message = receipt.error ?? 'unknown error';
|
|
308
|
+
return (
|
|
309
|
+
deserializeToolExecutionFailure(
|
|
310
|
+
message,
|
|
311
|
+
receipt.errorPayload,
|
|
312
|
+
toolErrorSchemaVersion,
|
|
313
|
+
) ?? new Error(`${legacyPrefix}: ${message}`)
|
|
314
|
+
);
|
|
315
|
+
}
|
|
316
|
+
|
|
293
317
|
export async function waitForCompletedRuntimeReceipt(input: {
|
|
294
318
|
receiptKey: string;
|
|
295
319
|
store: Pick<DurableReceiptExecutionStore, 'getMany'>;
|
|
296
320
|
maxAttempts?: number;
|
|
297
321
|
delayMs?: number;
|
|
298
322
|
abortSignal?: AbortSignal;
|
|
323
|
+
toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion;
|
|
299
324
|
}): Promise<RuntimeStepReceipt> {
|
|
300
325
|
const maxAttempts = input.maxAttempts ?? DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
301
326
|
const delayMs = input.delayMs ?? DURABLE_RECEIPT_WAIT_DELAY_MS;
|
|
@@ -311,8 +336,10 @@ export async function waitForCompletedRuntimeReceipt(input: {
|
|
|
311
336
|
return receipt;
|
|
312
337
|
}
|
|
313
338
|
if (receipt?.status === 'failed') {
|
|
314
|
-
throw
|
|
315
|
-
|
|
339
|
+
throw runtimeReceiptFailureError(
|
|
340
|
+
receipt,
|
|
341
|
+
`Durable tool call ${input.receiptKey} failed`,
|
|
342
|
+
input.toolErrorSchemaVersion,
|
|
316
343
|
);
|
|
317
344
|
}
|
|
318
345
|
}
|
|
@@ -326,6 +353,7 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
326
353
|
delayMs?: number;
|
|
327
354
|
abortSignal?: AbortSignal;
|
|
328
355
|
log?: (message: string) => void;
|
|
356
|
+
toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion;
|
|
329
357
|
}): Promise<{
|
|
330
358
|
completed: Map<string, RuntimeStepReceipt>;
|
|
331
359
|
failed: Map<string, Error>;
|
|
@@ -382,8 +410,10 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
382
410
|
if (receipt?.status === 'failed') {
|
|
383
411
|
failed.set(
|
|
384
412
|
key,
|
|
385
|
-
|
|
386
|
-
|
|
413
|
+
runtimeReceiptFailureError(
|
|
414
|
+
receipt,
|
|
415
|
+
`Durable tool call ${key} failed`,
|
|
416
|
+
input.toolErrorSchemaVersion,
|
|
387
417
|
),
|
|
388
418
|
);
|
|
389
419
|
pending.delete(key);
|
|
@@ -424,9 +454,13 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
424
454
|
completedCacheOnly?: boolean;
|
|
425
455
|
requiresExecutionLock?: boolean;
|
|
426
456
|
executionLockTtlMs?: number;
|
|
457
|
+
toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion;
|
|
427
458
|
formatError: (error: unknown) => string;
|
|
428
459
|
log: (message: string) => void;
|
|
429
|
-
execute: (context: {
|
|
460
|
+
execute: (context: {
|
|
461
|
+
leaseId: string | null;
|
|
462
|
+
signal?: AbortSignal;
|
|
463
|
+
}) => Promise<T>;
|
|
430
464
|
}): Promise<T> {
|
|
431
465
|
const lifecycleStartedAt = Date.now();
|
|
432
466
|
const logPhase = (phase: string, startedAt: number): void => {
|
|
@@ -477,6 +511,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
477
511
|
let lockSupervisor: ReturnType<
|
|
478
512
|
typeof createRuntimeReceiptHeartbeatSupervisor
|
|
479
513
|
> | null = null;
|
|
514
|
+
const executionAbortController = needsLock ? new AbortController() : null;
|
|
480
515
|
if (needsLock) {
|
|
481
516
|
if (
|
|
482
517
|
!input.store.acquireExecutionLock ||
|
|
@@ -521,7 +556,9 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
521
556
|
const lockTtlMs =
|
|
522
557
|
input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS;
|
|
523
558
|
lockSupervisor = createRuntimeReceiptHeartbeatSupervisor({
|
|
524
|
-
intervalMs:
|
|
559
|
+
intervalMs:
|
|
560
|
+
input.heartbeatIntervalMs ??
|
|
561
|
+
Math.max(1_000, Math.floor(lockTtlMs / 3)),
|
|
525
562
|
heartbeat: async () => {
|
|
526
563
|
const renewed = await input.store.acquireExecutionLock!({
|
|
527
564
|
receiptKey: input.receiptKey,
|
|
@@ -542,6 +579,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
542
579
|
input.log(
|
|
543
580
|
`ctx.${input.operation}(${input.id}): dangerous side-effect execution fence lost: ${error instanceof Error ? error.message : String(error)}`,
|
|
544
581
|
);
|
|
582
|
+
executionAbortController?.abort(error);
|
|
545
583
|
},
|
|
546
584
|
});
|
|
547
585
|
lockSupervisor.start();
|
|
@@ -559,7 +597,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
559
597
|
};
|
|
560
598
|
let executed: T;
|
|
561
599
|
try {
|
|
562
|
-
executed = await input.execute({
|
|
600
|
+
executed = await input.execute({
|
|
601
|
+
leaseId: null,
|
|
602
|
+
signal: executionAbortController?.signal,
|
|
603
|
+
});
|
|
604
|
+
executionAbortController?.signal.throwIfAborted();
|
|
563
605
|
} catch (error) {
|
|
564
606
|
await releaseOwnLock();
|
|
565
607
|
throw error;
|
|
@@ -622,8 +664,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
622
664
|
};
|
|
623
665
|
}
|
|
624
666
|
if (reclaimed?.status === 'failed') {
|
|
625
|
-
throw
|
|
626
|
-
|
|
667
|
+
throw runtimeReceiptFailureError(
|
|
668
|
+
reclaimed,
|
|
669
|
+
`ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused`,
|
|
670
|
+
input.toolErrorSchemaVersion,
|
|
627
671
|
);
|
|
628
672
|
}
|
|
629
673
|
if (
|
|
@@ -654,6 +698,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
654
698
|
store: input.store,
|
|
655
699
|
maxAttempts: input.runningReceiptWaitMaxAttempts,
|
|
656
700
|
delayMs: input.runningReceiptWaitDelayMs,
|
|
701
|
+
toolErrorSchemaVersion: input.toolErrorSchemaVersion,
|
|
657
702
|
}),
|
|
658
703
|
'in_flight',
|
|
659
704
|
),
|
|
@@ -684,8 +729,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
684
729
|
const repairFailedReceipt = async (
|
|
685
730
|
receipt: RuntimeStepReceipt,
|
|
686
731
|
): Promise<never> => {
|
|
687
|
-
throw
|
|
688
|
-
|
|
732
|
+
throw runtimeReceiptFailureError(
|
|
733
|
+
receipt,
|
|
734
|
+
`ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused`,
|
|
735
|
+
input.toolErrorSchemaVersion,
|
|
689
736
|
);
|
|
690
737
|
};
|
|
691
738
|
|
|
@@ -810,6 +857,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
810
857
|
input.formatError(error),
|
|
811
858
|
ownedLeaseId,
|
|
812
859
|
runtimeReceiptFailureKindForError(error),
|
|
860
|
+
serializeToolExecutionFailure(error),
|
|
813
861
|
);
|
|
814
862
|
if (!failed && input.store.canPersistFailure) {
|
|
815
863
|
throw new Error(
|
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
WorkReceiptFailureKind,
|
|
10
10
|
WorkReceiptGetCommand,
|
|
11
11
|
} from './work-receipts';
|
|
12
|
+
import type { ToolExecutionFailureV1 } from '../tool-execution-error';
|
|
12
13
|
|
|
13
14
|
export type WorkReceiptBatchGetCommand = {
|
|
14
15
|
playName: string;
|
|
@@ -21,6 +22,7 @@ export type WorkReceiptCompleteCommand = WorkReceiptCommand & {
|
|
|
21
22
|
|
|
22
23
|
export type WorkReceiptFailCommand = WorkReceiptCommand & {
|
|
23
24
|
error: string;
|
|
25
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
24
26
|
failureKind?: WorkReceiptFailureKind | null;
|
|
25
27
|
};
|
|
26
28
|
|
|
@@ -114,9 +116,7 @@ export interface ExecutionLedgerStore {
|
|
|
114
116
|
failReceipts(
|
|
115
117
|
input: WorkReceiptBatchFailCommand,
|
|
116
118
|
): Promise<Array<WorkReceipt | null>>;
|
|
117
|
-
releaseReceipt(
|
|
118
|
-
input: WorkReceiptReleaseCommand,
|
|
119
|
-
): Promise<WorkReceipt | null>;
|
|
119
|
+
releaseReceipt(input: WorkReceiptReleaseCommand): Promise<WorkReceipt | null>;
|
|
120
120
|
skipReceipt(input: WorkReceiptSkipCommand): Promise<WorkReceipt | null>;
|
|
121
121
|
heartbeatReceipts(
|
|
122
122
|
input: WorkReceiptBatchHeartbeatCommand,
|
|
@@ -59,11 +59,6 @@ export type ExecutionPlan = {
|
|
|
59
59
|
datasetBacking: 'neon_sheet';
|
|
60
60
|
compactWorkflowState: true;
|
|
61
61
|
};
|
|
62
|
-
sdkContract: {
|
|
63
|
-
apiVersion: number;
|
|
64
|
-
artifactVersion: number;
|
|
65
|
-
minRunnerVersion: number;
|
|
66
|
-
};
|
|
67
62
|
};
|
|
68
63
|
|
|
69
64
|
export type BuildExecutionPlanInput = {
|
|
@@ -73,11 +68,6 @@ export type BuildExecutionPlanInput = {
|
|
|
73
68
|
assets?: Array<{ playPath?: string | null; storageKey: string }>;
|
|
74
69
|
staticPipeline?: PlayStaticPipeline | null;
|
|
75
70
|
totalRows?: number | null;
|
|
76
|
-
sdkContract?: {
|
|
77
|
-
apiVersion?: number | null;
|
|
78
|
-
artifactVersion?: number | null;
|
|
79
|
-
minRunnerVersion?: number | null;
|
|
80
|
-
} | null;
|
|
81
71
|
};
|
|
82
72
|
|
|
83
73
|
export function buildExecutionPlan(
|
|
@@ -120,11 +110,6 @@ export function buildExecutionPlan(
|
|
|
120
110
|
datasetBacking: 'neon_sheet',
|
|
121
111
|
compactWorkflowState: true,
|
|
122
112
|
},
|
|
123
|
-
sdkContract: {
|
|
124
|
-
apiVersion: input.sdkContract?.apiVersion ?? 1,
|
|
125
|
-
artifactVersion: input.sdkContract?.artifactVersion ?? 1,
|
|
126
|
-
minRunnerVersion: input.sdkContract?.minRunnerVersion ?? 1,
|
|
127
|
-
},
|
|
128
113
|
};
|
|
129
114
|
}
|
|
130
115
|
|
|
@@ -13,6 +13,7 @@ import type { PacingRule } from './governor/rate-state-backend';
|
|
|
13
13
|
import type { GovernanceSnapshot } from './governor/governor';
|
|
14
14
|
import type { PlayRunInputPayload } from './play-input';
|
|
15
15
|
import type { PlayRunFailureDetails } from './run-failure';
|
|
16
|
+
import type { ToolExecutionErrorSchemaVersion } from '../tool-execution-error';
|
|
16
17
|
|
|
17
18
|
export type PlayRunnerRateStateBackendConfig =
|
|
18
19
|
| {
|
|
@@ -103,6 +104,8 @@ export interface PlayRunnerContextConfig {
|
|
|
103
104
|
runtimeTestFaultHeader?: string | null;
|
|
104
105
|
vercelProtectionBypassToken?: string | null;
|
|
105
106
|
integrationMode?: 'live' | 'eval_stub' | 'fixture';
|
|
107
|
+
/** Immutable tool-error payload schema copied from the run contract. */
|
|
108
|
+
toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
106
109
|
orgId?: string;
|
|
107
110
|
workflowId?: string;
|
|
108
111
|
playId?: string;
|
|
@@ -4,6 +4,7 @@ import type { PlayRowUpdate } from './ctx-types';
|
|
|
4
4
|
import type { PlaySheetContract } from '../plays/static-pipeline';
|
|
5
5
|
import type { PlayRunLedgerEvent } from './run-ledger';
|
|
6
6
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
7
|
+
import type { ToolExecutionFailureV1 } from '../tool-execution-error';
|
|
7
8
|
|
|
8
9
|
export type RuntimeAppendRunEventsAction = {
|
|
9
10
|
action: 'append_run_events';
|
|
@@ -165,6 +166,7 @@ export type RuntimeReceiptAction =
|
|
|
165
166
|
runAttempt?: number | null;
|
|
166
167
|
leaseId?: string | null;
|
|
167
168
|
error: string;
|
|
169
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
168
170
|
failureKind?: WorkReceiptFailureKind | null;
|
|
169
171
|
}
|
|
170
172
|
| {
|
|
@@ -230,6 +232,7 @@ export type RuntimeReceiptCompletion = RuntimeReceiptLease & {
|
|
|
230
232
|
|
|
231
233
|
export type RuntimeReceiptFailure = RuntimeReceiptLease & {
|
|
232
234
|
error: string;
|
|
235
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
233
236
|
failureKind?: WorkReceiptFailureKind | null;
|
|
234
237
|
};
|
|
235
238
|
|