deepline 0.3.71 → 0.3.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +712 -632
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +40 -10
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +116 -4
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +2 -14
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/native-batch.ts +9 -92
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +159 -621
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +19 -13
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +10 -5
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +1 -4
- package/dist/bundling-sources/shared_libs/plays/tool-response-contract.ts +2 -4
- package/dist/cli/index.js +324 -123
- package/dist/cli/index.mjs +324 -123
- package/dist/{compiler-manifest-IkyvsUO-.d.mts → compiler-manifest-CMNgA_JQ.d.mts} +7 -1
- package/dist/{compiler-manifest-IkyvsUO-.d.ts → compiler-manifest-CMNgA_JQ.d.ts} +7 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +32 -3
- package/dist/index.mjs +32 -3
- package/dist/install-integrity.json +2 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/package.json +1 -1
|
@@ -175,6 +175,7 @@ import {
|
|
|
175
175
|
buildDurableToolAggregateReceiptKey,
|
|
176
176
|
buildDurableToolCallAuthScopeDigest,
|
|
177
177
|
buildDurableToolCallCacheKey,
|
|
178
|
+
buildLegacyRevisionedDurableToolCallCacheKey,
|
|
178
179
|
buildDurableToolPayloadProviderIdempotencyKey,
|
|
179
180
|
buildDurableToolProviderIdempotencyKey,
|
|
180
181
|
buildDurableToolReceiptPrefix,
|
|
@@ -195,8 +196,11 @@ import {
|
|
|
195
196
|
type PlayAuthoringRuntimeContext,
|
|
196
197
|
} from '../plays/authoring-contract';
|
|
197
198
|
import {
|
|
199
|
+
LOG_LEVELS,
|
|
198
200
|
formatCtxFetchHttpFailureDiagnostic,
|
|
199
|
-
|
|
201
|
+
tagLogEntry,
|
|
202
|
+
tagLogLevel,
|
|
203
|
+
type LogLevel,
|
|
200
204
|
} from './log-provenance';
|
|
201
205
|
import {
|
|
202
206
|
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
@@ -1392,6 +1396,8 @@ function playNodeScopeForToolCallRequest(
|
|
|
1392
1396
|
type ToolExecutionApiOptions = {
|
|
1393
1397
|
timeoutMs?: number;
|
|
1394
1398
|
durableCallReceiptKey?: string | null;
|
|
1399
|
+
/** The receipt key is a completed-result cache for this read-only call. */
|
|
1400
|
+
completedResultCache?: boolean;
|
|
1395
1401
|
executionAuthScopeDigest?: string | null;
|
|
1396
1402
|
/**
|
|
1397
1403
|
* Historical receipt identity used for the API's provider/billing
|
|
@@ -1432,6 +1438,22 @@ const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
|
|
|
1432
1438
|
*/
|
|
1433
1439
|
const MAX_CELL_PRODUCER_ATTEMPTS = 12;
|
|
1434
1440
|
const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
|
|
1441
|
+
|
|
1442
|
+
/**
|
|
1443
|
+
* A 100k-row Play must not turn `ctx.log` into a second data plane. Keep an
|
|
1444
|
+
* initial useful window, then deterministic samples and explicit durable
|
|
1445
|
+
* warnings. Warn/error are never sampled: losing the actual failure is worse
|
|
1446
|
+
* than spending additional log capacity.
|
|
1447
|
+
*/
|
|
1448
|
+
const LOG_RATE_LIMIT_POLICY: Readonly<
|
|
1449
|
+
Record<Exclude<LogLevel, 'warn' | 'error'>, { first: number; every: number }>
|
|
1450
|
+
> = {
|
|
1451
|
+
debug: { first: 200, every: 100 },
|
|
1452
|
+
info: { first: 1_000, every: 100 },
|
|
1453
|
+
};
|
|
1454
|
+
const LOG_RATE_LIMIT_MARKER_EVERY = 1_000;
|
|
1455
|
+
|
|
1456
|
+
type LogRateLimitState = { seen: number; sampled: number; suppressed: number };
|
|
1435
1457
|
const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
|
|
1436
1458
|
|
|
1437
1459
|
function shouldPersistMapCellField(fieldName: string): boolean {
|
|
@@ -2095,6 +2117,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2095
2117
|
#options: ContextOptions;
|
|
2096
2118
|
private readonly executionScope: RunExecutionScope;
|
|
2097
2119
|
private logBuffer: string[] = [];
|
|
2120
|
+
private readonly logRateLimitState = new Map<
|
|
2121
|
+
Exclude<LogLevel, 'warn' | 'error'>,
|
|
2122
|
+
LogRateLimitState
|
|
2123
|
+
>();
|
|
2098
2124
|
private readonly ctxFetchHttpFailureDiagnosticIdentities = new Set<string>();
|
|
2099
2125
|
private checkpoint: PlayCheckpoint;
|
|
2100
2126
|
private readonly durableCallCacheEpochMs: number;
|
|
@@ -2740,9 +2766,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2740
2766
|
}
|
|
2741
2767
|
|
|
2742
2768
|
/**
|
|
2743
|
-
*
|
|
2744
|
-
*
|
|
2745
|
-
*
|
|
2769
|
+
* A real published artifact pins its response presentation in its launch
|
|
2770
|
+
* contract. In-process callers that omit the option are runtime consumers,
|
|
2771
|
+
* not historical Play artifacts, and retain the canonical result they pass
|
|
2772
|
+
* in. This keeps the context API backward-compatible while letting a pinned
|
|
2773
|
+
* legacy artifact request its old projection after a cache read.
|
|
2774
|
+
*/
|
|
2775
|
+
private get hasPinnedToolResponseContract(): boolean {
|
|
2776
|
+
return (
|
|
2777
|
+
this.activeInlineComposition !== null ||
|
|
2778
|
+
this.#options.toolResponseContract !== undefined
|
|
2779
|
+
);
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
/**
|
|
2783
|
+
* Compatibility metadata retained for already-published artifacts. It never
|
|
2784
|
+
* changes the completed-result cache identity.
|
|
2746
2785
|
*/
|
|
2747
2786
|
private get currentToolResponseReceiptRevision(): string | undefined {
|
|
2748
2787
|
if (this.activeInlineComposition) {
|
|
@@ -3022,7 +3061,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3022
3061
|
attempt,
|
|
3023
3062
|
retryAfter: null,
|
|
3024
3063
|
});
|
|
3025
|
-
this.
|
|
3064
|
+
this.runtimeLog(
|
|
3026
3065
|
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
3027
3066
|
stage: 'transport',
|
|
3028
3067
|
secret_name: secret.name,
|
|
@@ -3077,7 +3116,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3077
3116
|
retry_after_exceeds_cap: retry.retryAfterExceedsCap,
|
|
3078
3117
|
};
|
|
3079
3118
|
if (!response.ok) {
|
|
3080
|
-
this.
|
|
3119
|
+
this.runtimeLog(
|
|
3081
3120
|
`[runtime.secret_resolution_failure] ${JSON.stringify(responseDiagnostic)}`,
|
|
3082
3121
|
);
|
|
3083
3122
|
cancelRuntimeResponseBody(response);
|
|
@@ -3094,7 +3133,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3094
3133
|
try {
|
|
3095
3134
|
payload = await response.json();
|
|
3096
3135
|
} catch (error) {
|
|
3097
|
-
this.
|
|
3136
|
+
this.runtimeLog(
|
|
3098
3137
|
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
3099
3138
|
...responseDiagnostic,
|
|
3100
3139
|
response_kind: 'invalid_json',
|
|
@@ -3115,7 +3154,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3115
3154
|
? (payload as { value: string }).value
|
|
3116
3155
|
: null;
|
|
3117
3156
|
if (!value) {
|
|
3118
|
-
this.
|
|
3157
|
+
this.runtimeLog(
|
|
3119
3158
|
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
3120
3159
|
...responseDiagnostic,
|
|
3121
3160
|
response_kind:
|
|
@@ -3131,7 +3170,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3131
3170
|
);
|
|
3132
3171
|
}
|
|
3133
3172
|
if (attempt > 1) {
|
|
3134
|
-
this.
|
|
3173
|
+
this.runtimeLog(
|
|
3135
3174
|
`[runtime.secret_resolution_recovered] ${JSON.stringify({
|
|
3136
3175
|
secret_name: secret.name,
|
|
3137
3176
|
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
@@ -3739,7 +3778,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3739
3778
|
if (normalized) byKey.set(normalized.key, normalized);
|
|
3740
3779
|
}
|
|
3741
3780
|
if (runtimeReceiptReadTraceEnabled) {
|
|
3742
|
-
this.
|
|
3781
|
+
this.runtimeLog(
|
|
3743
3782
|
`[runtime-receipt-normalize] ${runtimeReceiptReadSummary({
|
|
3744
3783
|
requested: uniqueKeys,
|
|
3745
3784
|
receipts,
|
|
@@ -3751,7 +3790,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3751
3790
|
receipts,
|
|
3752
3791
|
byKey,
|
|
3753
3792
|
});
|
|
3754
|
-
if (trace) this.
|
|
3793
|
+
if (trace) this.runtimeLog(trace);
|
|
3755
3794
|
}
|
|
3756
3795
|
return byKey;
|
|
3757
3796
|
}
|
|
@@ -3806,7 +3845,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3806
3845
|
if (normalized) byKey.set(normalized.key, normalized);
|
|
3807
3846
|
}
|
|
3808
3847
|
if (runtimeReceiptReadTraceEnabled) {
|
|
3809
|
-
this.
|
|
3848
|
+
this.runtimeLog(
|
|
3810
3849
|
`[runtime-receipt-claim-normalize] reclaim_running=${reclaimRunning} force_refresh=${forceRefresh} force_failed_refresh=${forceFailedRefresh} ` +
|
|
3811
3850
|
runtimeReceiptReadSummary({
|
|
3812
3851
|
requested: uniqueKeys,
|
|
@@ -3997,14 +4036,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3997
4036
|
);
|
|
3998
4037
|
if (normalized) byKey.set(normalized.key, normalized);
|
|
3999
4038
|
}
|
|
4000
|
-
await this.
|
|
4039
|
+
await this.reconcileIncompleteRuntimeStepReceiptCompletions(
|
|
4001
4040
|
normalizedInputs,
|
|
4002
4041
|
byKey,
|
|
4003
4042
|
);
|
|
4004
4043
|
return byKey;
|
|
4005
4044
|
}
|
|
4006
4045
|
|
|
4007
|
-
private async
|
|
4046
|
+
private async reconcileIncompleteRuntimeStepReceiptCompletions(
|
|
4008
4047
|
receipts: Array<{
|
|
4009
4048
|
key: string;
|
|
4010
4049
|
runId: string;
|
|
@@ -4015,32 +4054,45 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4015
4054
|
completedByKey: Map<string, RuntimeStepReceipt>,
|
|
4016
4055
|
): Promise<void> {
|
|
4017
4056
|
if (
|
|
4018
|
-
|
|
4019
|
-
|
|
4020
|
-
!this.#options.getRuntimeStepReceipts)
|
|
4057
|
+
!this.#options.getRuntimeStepReceipt &&
|
|
4058
|
+
!this.#options.getRuntimeStepReceipts
|
|
4021
4059
|
) {
|
|
4022
4060
|
return;
|
|
4023
4061
|
}
|
|
4024
|
-
|
|
4062
|
+
// The receipt gateway is allowed to return a compact terminal
|
|
4063
|
+
// acknowledgement without `output`. That is sufficient for old callers
|
|
4064
|
+
// which keep the live result, but a completed-result cache must hydrate
|
|
4065
|
+
// the stored output before handing it to a cache cohort.
|
|
4066
|
+
const incompleteKeys = receipts
|
|
4025
4067
|
.map((receipt) => receipt.key.trim())
|
|
4026
|
-
.filter((key) =>
|
|
4027
|
-
|
|
4028
|
-
|
|
4029
|
-
|
|
4068
|
+
.filter((key) => {
|
|
4069
|
+
if (!key) return false;
|
|
4070
|
+
const completed = completedByKey.get(key);
|
|
4071
|
+
return (
|
|
4072
|
+
!completed ||
|
|
4073
|
+
((completed.status === 'completed' ||
|
|
4074
|
+
completed.status === 'skipped') &&
|
|
4075
|
+
completed.output === undefined)
|
|
4076
|
+
);
|
|
4077
|
+
});
|
|
4078
|
+
if (incompleteKeys.length === 0) return;
|
|
4079
|
+
const latest = await this.getRuntimeStepReceipts(incompleteKeys);
|
|
4080
|
+
for (const key of incompleteKeys) {
|
|
4030
4081
|
const recovered = latest.get(key);
|
|
4031
4082
|
if (
|
|
4032
|
-
recovered?.status === 'completed' ||
|
|
4033
|
-
|
|
4083
|
+
(recovered?.status === 'completed' ||
|
|
4084
|
+
recovered?.status === 'skipped') &&
|
|
4085
|
+
recovered.output !== undefined
|
|
4034
4086
|
) {
|
|
4035
4087
|
completedByKey.set(key, recovered);
|
|
4036
4088
|
}
|
|
4037
4089
|
}
|
|
4038
|
-
const recoveredCount =
|
|
4039
|
-
completedByKey.
|
|
4090
|
+
const recoveredCount = incompleteKeys.filter(
|
|
4091
|
+
(key) => completedByKey.get(key)?.output !== undefined,
|
|
4040
4092
|
).length;
|
|
4041
4093
|
if (recoveredCount > 0) {
|
|
4042
|
-
this.
|
|
4043
|
-
`Runtime tool receipt completion response
|
|
4094
|
+
this.runtimeLog(
|
|
4095
|
+
`Runtime tool receipt completion response hydrated ${recoveredCount}/${incompleteKeys.length} incomplete receipt(s) from durable store read-back.`,
|
|
4044
4096
|
);
|
|
4045
4097
|
}
|
|
4046
4098
|
}
|
|
@@ -4155,17 +4207,39 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4155
4207
|
);
|
|
4156
4208
|
}
|
|
4157
4209
|
|
|
4158
|
-
private async durableToolCallCacheKeyForScope(
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4210
|
+
private async durableToolCallCacheKeyForScope(input: {
|
|
4211
|
+
toolId: string;
|
|
4212
|
+
requestInput: Record<string, unknown>;
|
|
4213
|
+
executionAuthScopeDigest?: string | null;
|
|
4214
|
+
staleAfterSeconds?: number | null;
|
|
4215
|
+
playLocalScope?: string | null;
|
|
4216
|
+
}): Promise<string> {
|
|
4217
|
+
const identity = await this.durableToolCallCacheIdentity(input);
|
|
4218
|
+
return buildDurableToolCallCacheKey(identity);
|
|
4219
|
+
}
|
|
4220
|
+
|
|
4221
|
+
private async legacyDurableToolCallCacheKeyForScope(input: {
|
|
4222
|
+
toolId: string;
|
|
4223
|
+
requestInput: Record<string, unknown>;
|
|
4224
|
+
executionAuthScopeDigest?: string | null;
|
|
4225
|
+
staleAfterSeconds?: number | null;
|
|
4226
|
+
playLocalScope?: string | null;
|
|
4227
|
+
}): Promise<string | null> {
|
|
4228
|
+
const revision = this.currentToolResponseReceiptRevision;
|
|
4229
|
+
if (!revision) return null;
|
|
4230
|
+
return buildLegacyRevisionedDurableToolCallCacheKey({
|
|
4231
|
+
...(await this.durableToolCallCacheIdentity(input)),
|
|
4232
|
+
toolResponseReceiptRevision: revision,
|
|
4233
|
+
});
|
|
4234
|
+
}
|
|
4235
|
+
|
|
4236
|
+
private async durableToolCallCacheIdentity(input: {
|
|
4237
|
+
toolId: string;
|
|
4238
|
+
requestInput: Record<string, unknown>;
|
|
4239
|
+
executionAuthScopeDigest?: string | null;
|
|
4240
|
+
staleAfterSeconds?: number | null;
|
|
4241
|
+
playLocalScope?: string | null;
|
|
4242
|
+
}): Promise<Parameters<typeof buildDurableToolCallCacheKey>[0]> {
|
|
4169
4243
|
const providerActionVersion =
|
|
4170
4244
|
(await this.#options.getToolActionCacheVersion?.(input.toolId))?.trim() ??
|
|
4171
4245
|
'';
|
|
@@ -4173,7 +4247,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4173
4247
|
input.executionAuthScopeDigest?.trim() ||
|
|
4174
4248
|
(await this.resolveToolAuthScopeDigest(input.toolId))?.trim() ||
|
|
4175
4249
|
null;
|
|
4176
|
-
return
|
|
4250
|
+
return {
|
|
4177
4251
|
orgId: this.#options.orgId,
|
|
4178
4252
|
playLocalScope: input.playLocalScope,
|
|
4179
4253
|
toolId: input.toolId,
|
|
@@ -4184,29 +4258,33 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4184
4258
|
executionAuthScopeDigest,
|
|
4185
4259
|
}),
|
|
4186
4260
|
providerActionVersion,
|
|
4187
|
-
toolResponseReceiptRevision,
|
|
4188
4261
|
staleAfterSeconds: input.staleAfterSeconds,
|
|
4189
4262
|
cacheEpochMs: this.durableCallCacheEpochMs,
|
|
4263
|
+
};
|
|
4264
|
+
}
|
|
4265
|
+
|
|
4266
|
+
private async durableToolCallCacheKey(input: {
|
|
4267
|
+
toolId: string;
|
|
4268
|
+
requestInput: Record<string, unknown>;
|
|
4269
|
+
executionAuthScopeDigest?: string | null;
|
|
4270
|
+
staleAfterSeconds?: number | null;
|
|
4271
|
+
}): Promise<string> {
|
|
4272
|
+
return await this.durableToolCallCacheKeyForScope({
|
|
4273
|
+
...input,
|
|
4274
|
+
playLocalScope: this.currentGovernance.currentPlayId,
|
|
4190
4275
|
});
|
|
4191
4276
|
}
|
|
4192
4277
|
|
|
4193
|
-
private async
|
|
4194
|
-
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
.
|
|
4202
|
-
|
|
4203
|
-
return await this.durableToolCallCacheKeyForScope(
|
|
4204
|
-
{
|
|
4205
|
-
...input,
|
|
4206
|
-
playLocalScope: this.currentGovernance.currentPlayId,
|
|
4207
|
-
},
|
|
4208
|
-
toolResponseReceiptRevision,
|
|
4209
|
-
);
|
|
4278
|
+
private async legacyDurableToolCallCacheKey(input: {
|
|
4279
|
+
toolId: string;
|
|
4280
|
+
requestInput: Record<string, unknown>;
|
|
4281
|
+
executionAuthScopeDigest?: string | null;
|
|
4282
|
+
staleAfterSeconds?: number | null;
|
|
4283
|
+
}): Promise<string | null> {
|
|
4284
|
+
return await this.legacyDurableToolCallCacheKeyForScope({
|
|
4285
|
+
...input,
|
|
4286
|
+
playLocalScope: this.currentGovernance.currentPlayId,
|
|
4287
|
+
});
|
|
4210
4288
|
}
|
|
4211
4289
|
|
|
4212
4290
|
private async resolveToolAuthScopeDigest(
|
|
@@ -4326,7 +4404,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4326
4404
|
receiptKeys: keys,
|
|
4327
4405
|
store: this.durableReceiptExecutionStore(),
|
|
4328
4406
|
maxAttempts,
|
|
4329
|
-
log: (message) => this.
|
|
4407
|
+
log: (message) => this.runtimeLog(message),
|
|
4330
4408
|
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
4331
4409
|
});
|
|
4332
4410
|
}
|
|
@@ -4430,7 +4508,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4430
4508
|
executionLockTtlMs: opts.executionLockTtlMs,
|
|
4431
4509
|
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
4432
4510
|
formatError: (error) => this.formatRuntimeError(error),
|
|
4433
|
-
log: (message) => this.
|
|
4511
|
+
log: (message) => this.runtimeLog(message),
|
|
4434
4512
|
execute: ({ leaseId }) => execute(leaseId),
|
|
4435
4513
|
});
|
|
4436
4514
|
} finally {
|
|
@@ -5356,6 +5434,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5356
5434
|
return null;
|
|
5357
5435
|
}
|
|
5358
5436
|
|
|
5437
|
+
private canReuseToolResultForCurrentResponseContract(
|
|
5438
|
+
result: unknown,
|
|
5439
|
+
): boolean {
|
|
5440
|
+
if (this.currentToolResponseContract !== RAW_V2_TOOL_RESPONSE_CONTRACT) {
|
|
5441
|
+
return true;
|
|
5442
|
+
}
|
|
5443
|
+
const toolResult = isToolExecuteResult(result)
|
|
5444
|
+
? result
|
|
5445
|
+
: isSerializedToolExecuteResult(result)
|
|
5446
|
+
? deserializeToolExecuteResult(result)
|
|
5447
|
+
: null;
|
|
5448
|
+
return (
|
|
5449
|
+
toolResult !== null &&
|
|
5450
|
+
Object.prototype.hasOwnProperty.call(toolResult.toolResponse, 'rawV2')
|
|
5451
|
+
);
|
|
5452
|
+
}
|
|
5453
|
+
|
|
5359
5454
|
private cacheToolResult(
|
|
5360
5455
|
toolId: string,
|
|
5361
5456
|
rowCacheKey: string,
|
|
@@ -5468,6 +5563,38 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5468
5563
|
);
|
|
5469
5564
|
}
|
|
5470
5565
|
|
|
5566
|
+
/**
|
|
5567
|
+
* Receipts retain the canonical raw-v2 provider outcome. Older artifacts
|
|
5568
|
+
* receive their legacy projection only at the Play boundary, after cache
|
|
5569
|
+
* lookup and persistence have finished.
|
|
5570
|
+
*/
|
|
5571
|
+
private materializeToolExecutionResult(
|
|
5572
|
+
result: ToolExecuteResult,
|
|
5573
|
+
): ToolExecuteResult {
|
|
5574
|
+
if (
|
|
5575
|
+
this.currentToolResponseContract === RAW_V2_TOOL_RESPONSE_CONTRACT ||
|
|
5576
|
+
!this.hasPinnedToolResponseContract ||
|
|
5577
|
+
!Object.prototype.hasOwnProperty.call(result.toolResponse, 'rawV2')
|
|
5578
|
+
) {
|
|
5579
|
+
return result;
|
|
5580
|
+
}
|
|
5581
|
+
const serialized = serializeToolExecuteResult(result);
|
|
5582
|
+
if (serialized.__kind !== 'deepline.tool_execute_result.v2') {
|
|
5583
|
+
return result;
|
|
5584
|
+
}
|
|
5585
|
+
return deserializeToolExecuteResult({
|
|
5586
|
+
...serialized,
|
|
5587
|
+
__kind: 'deepline.tool_execute_result.v1',
|
|
5588
|
+
toolResponse: {
|
|
5589
|
+
raw: legacyRawFromToolResponseRawV2(
|
|
5590
|
+
serialized.toolResponse.rawV2,
|
|
5591
|
+
serialized.toolResponse.view,
|
|
5592
|
+
serialized.toolResponse.responseMeta,
|
|
5593
|
+
),
|
|
5594
|
+
},
|
|
5595
|
+
});
|
|
5596
|
+
}
|
|
5597
|
+
|
|
5471
5598
|
private attachCustomerDbDatasetResult(
|
|
5472
5599
|
toolId: string,
|
|
5473
5600
|
requestInput: Record<string, unknown> | undefined,
|
|
@@ -5781,7 +5908,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5781
5908
|
completed?.status !== 'completed' &&
|
|
5782
5909
|
completed?.status !== 'skipped'
|
|
5783
5910
|
) {
|
|
5784
|
-
this.
|
|
5911
|
+
this.runtimeLog(
|
|
5785
5912
|
`Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`,
|
|
5786
5913
|
);
|
|
5787
5914
|
}
|
|
@@ -6018,7 +6145,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6018
6145
|
'ctx.runSteps.options.description',
|
|
6019
6146
|
options.description,
|
|
6020
6147
|
);
|
|
6021
|
-
this.
|
|
6148
|
+
this.runtimeLog(options.description);
|
|
6022
6149
|
}
|
|
6023
6150
|
return (await this.executeStepProgram(program, input, 0, [], {
|
|
6024
6151
|
checkpointSteps: true,
|
|
@@ -6249,7 +6376,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6249
6376
|
immediateMaterializedRows.length = 0;
|
|
6250
6377
|
immediateMaterializedResidentBytes = 0;
|
|
6251
6378
|
immediateMaterializedCacheEnabled = false;
|
|
6252
|
-
this.
|
|
6379
|
+
this.runtimeLog(
|
|
6253
6380
|
`Dataset ${normalizedMapNamespace} result cache disabled (${reason}); using the durable runtime sheet.`,
|
|
6254
6381
|
);
|
|
6255
6382
|
};
|
|
@@ -6862,7 +6989,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6862
6989
|
: { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
|
|
6863
6990
|
if (dedupedMaterialized.droppedCount > 0) {
|
|
6864
6991
|
const keySample = dedupedMaterialized.duplicateKeys.join(', ');
|
|
6865
|
-
this.
|
|
6992
|
+
this.runtimeLog(
|
|
6866
6993
|
`deduped ${dedupedMaterialized.droppedCount} duplicate dataset key(s) for ctx.dataset("${normalizedMapNamespace}"); keeping first occurrence` +
|
|
6867
6994
|
(keySample ? ` (e.g. ${keySample})` : ''),
|
|
6868
6995
|
);
|
|
@@ -7159,7 +7286,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7159
7286
|
const persistStartedAt = Date.now();
|
|
7160
7287
|
await persistMapRows(unpersistedRows);
|
|
7161
7288
|
if (unpersistedRows.length > 0) {
|
|
7162
|
-
this.
|
|
7289
|
+
this.runtimeLog(
|
|
7163
7290
|
`Persisted ${unpersistedRows.length} fail-fast rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
|
|
7164
7291
|
);
|
|
7165
7292
|
}
|
|
@@ -7227,7 +7354,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7227
7354
|
const persistStartedAt = Date.now();
|
|
7228
7355
|
await persistMapRows(persistRows);
|
|
7229
7356
|
if (persistRows.length > 0) {
|
|
7230
|
-
this.
|
|
7357
|
+
this.runtimeLog(
|
|
7231
7358
|
`Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
|
|
7232
7359
|
);
|
|
7233
7360
|
}
|
|
@@ -7285,7 +7412,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7285
7412
|
expectedRows: totalInputCount,
|
|
7286
7413
|
currentRows: results,
|
|
7287
7414
|
failedRowCount: mapResult.failedRows.length,
|
|
7288
|
-
log: (line) => this.
|
|
7415
|
+
log: (line) => this.runtimeLog(line),
|
|
7289
7416
|
readPersistedRows: async (readInput) => {
|
|
7290
7417
|
if (
|
|
7291
7418
|
!this.#options.baseUrl ||
|
|
@@ -7548,7 +7675,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7548
7675
|
completedRowsToPersist.length = 0;
|
|
7549
7676
|
failedRowsToPersist.length = 0;
|
|
7550
7677
|
retainedRowsComplete = false;
|
|
7551
|
-
this.
|
|
7678
|
+
this.runtimeLog(
|
|
7552
7679
|
`Dataset ${normalizedTableNamespace} released its in-memory result cache after ${completedRowCount + failedRowCount} row(s); using the durable Runtime Sheet.`,
|
|
7553
7680
|
);
|
|
7554
7681
|
}
|
|
@@ -7583,11 +7710,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7583
7710
|
};
|
|
7584
7711
|
|
|
7585
7712
|
if (completedRows > 0 || pendingRows !== totalRows) {
|
|
7586
|
-
this.
|
|
7713
|
+
this.runtimeLog(
|
|
7587
7714
|
`Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} duplicate keys skipped; ${pendingRows} pending)`,
|
|
7588
7715
|
);
|
|
7589
7716
|
} else {
|
|
7590
|
-
this.
|
|
7717
|
+
this.runtimeLog(
|
|
7591
7718
|
`Starting map over ${items.length} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace})`,
|
|
7592
7719
|
);
|
|
7593
7720
|
}
|
|
@@ -7764,7 +7891,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7764
7891
|
}
|
|
7765
7892
|
this.lastDatasetStep = datasetStep;
|
|
7766
7893
|
this.activeDatasetStep = null;
|
|
7767
|
-
this.
|
|
7894
|
+
this.runtimeLog(
|
|
7768
7895
|
`Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`,
|
|
7769
7896
|
);
|
|
7770
7897
|
return {
|
|
@@ -7829,7 +7956,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7829
7956
|
});
|
|
7830
7957
|
mapStallObserver = new RuntimeMapStallObserver({
|
|
7831
7958
|
getWriterDiagnostics: () => incrementalPersistence?.diagnostics() ?? null,
|
|
7832
|
-
log: (line) => this.
|
|
7959
|
+
log: (line) => this.runtimeLog(line),
|
|
7833
7960
|
mapName: normalizedTableNamespace,
|
|
7834
7961
|
pageOffset:
|
|
7835
7962
|
runtimeOptions?.executionRowIndexes?.[0] ??
|
|
@@ -8281,17 +8408,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8281
8408
|
});
|
|
8282
8409
|
}
|
|
8283
8410
|
for (const failedRow of failedRowsToPersist.slice(0, 3)) {
|
|
8284
|
-
this.
|
|
8411
|
+
this.runtimeLog(
|
|
8285
8412
|
`row ${failedMapRowLogLabel(failedRow)} failed: ${failedRow.error ?? 'unknown error'}`,
|
|
8413
|
+
{ level: 'error' },
|
|
8286
8414
|
);
|
|
8287
8415
|
}
|
|
8288
8416
|
if (failedRowsToPersist.length > 3) {
|
|
8289
|
-
this.
|
|
8417
|
+
this.runtimeLog(
|
|
8290
8418
|
`${failedRowsToPersist.length - 3} additional row failure(s) omitted from map log`,
|
|
8419
|
+
{ level: 'warn' },
|
|
8291
8420
|
);
|
|
8292
8421
|
}
|
|
8293
|
-
this.
|
|
8422
|
+
this.runtimeLog(
|
|
8294
8423
|
`Map completed: ${succeededRows + completedRows} results (${succeededRows} succeeded, ${failedRowCount} failed, ${completedRows} duplicate keys skipped)`,
|
|
8424
|
+
failedRowCount > 0 ? { level: 'warn' } : undefined,
|
|
8295
8425
|
);
|
|
8296
8426
|
return {
|
|
8297
8427
|
completedRows: [...completedRowsToPersist].sort(
|
|
@@ -9055,8 +9185,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9055
9185
|
);
|
|
9056
9186
|
if (dispatchable.requests.length > 0) {
|
|
9057
9187
|
pass += 1;
|
|
9058
|
-
this.
|
|
9059
|
-
this.
|
|
9188
|
+
this.runtimeLog(` Batch pass ${pass}`);
|
|
9189
|
+
this.runtimeLog(
|
|
9060
9190
|
` Dispatcher launch: ready=${dispatchable.requests.length} ` +
|
|
9061
9191
|
`queued=${this.toolCallQueue.length} ` +
|
|
9062
9192
|
`in_flight_groups=${inFlightToolExecutions.size}`,
|
|
@@ -9246,16 +9376,26 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9246
9376
|
executionAuthScopeDigest,
|
|
9247
9377
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9248
9378
|
});
|
|
9249
|
-
let
|
|
9250
|
-
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
|
|
9254
|
-
|
|
9255
|
-
|
|
9256
|
-
|
|
9257
|
-
|
|
9258
|
-
|
|
9379
|
+
let legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
|
|
9380
|
+
toolId,
|
|
9381
|
+
requestInput: input,
|
|
9382
|
+
executionAuthScopeDigest,
|
|
9383
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9384
|
+
});
|
|
9385
|
+
// Only explicitly fenced operations use this key as a provider identity.
|
|
9386
|
+
// Ordinary calls use it solely as their completed-result cache key.
|
|
9387
|
+
let providerIdempotencyKeyBase = durableCacheKey;
|
|
9388
|
+
let directResultReceiptKey = durableCacheKey;
|
|
9389
|
+
// A raw-v2 artifact must never read an older receipt that cannot render
|
|
9390
|
+
// raw-v2. If both the primary and its revisioned recovery receipt are
|
|
9391
|
+
// unrenderable, make a live call rather than returning either cache entry.
|
|
9392
|
+
// We cannot safely replace an immutable completed receipt, so that rare
|
|
9393
|
+
// incompatible state deliberately forgoes caching until it is repaired.
|
|
9394
|
+
let bypassDirectResultCache = false;
|
|
9395
|
+
const checkpointCacheKeys = [
|
|
9396
|
+
durableCacheKey,
|
|
9397
|
+
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
9398
|
+
];
|
|
9259
9399
|
|
|
9260
9400
|
const executeTool = async (context?: {
|
|
9261
9401
|
leaseId?: string | null;
|
|
@@ -9284,7 +9424,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9284
9424
|
if (!store) {
|
|
9285
9425
|
const directCacheKey = durableCacheKey;
|
|
9286
9426
|
let initialDirectReceipt: RuntimeStepReceipt | null | undefined;
|
|
9287
|
-
const
|
|
9427
|
+
const cachedCandidate = !cacheableToolResult
|
|
9288
9428
|
? null
|
|
9289
9429
|
: toolCachePolicy.force
|
|
9290
9430
|
? null
|
|
@@ -9293,8 +9433,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9293
9433
|
checkpointCacheKeys,
|
|
9294
9434
|
'direct',
|
|
9295
9435
|
);
|
|
9436
|
+
const cached =
|
|
9437
|
+
cachedCandidate &&
|
|
9438
|
+
this.canReuseToolResultForCurrentResponseContract(
|
|
9439
|
+
cachedCandidate.result.result,
|
|
9440
|
+
)
|
|
9441
|
+
? cachedCandidate
|
|
9442
|
+
: null;
|
|
9296
9443
|
if (cached) {
|
|
9297
|
-
this.
|
|
9444
|
+
this.runtimeLog(`Calling tool: ${toolId} recovered from checkpoint`);
|
|
9298
9445
|
// Started and settled are paired in a finally: an uncovered throw
|
|
9299
9446
|
// between them strands the tool node `running` forever in the
|
|
9300
9447
|
// snapshot, which is exactly the lossy state ADR 0018 removed.
|
|
@@ -9316,7 +9463,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9316
9463
|
outcome: 'cached',
|
|
9317
9464
|
durationMs: 0,
|
|
9318
9465
|
});
|
|
9319
|
-
return recovered;
|
|
9466
|
+
return this.materializeToolExecutionResult(recovered);
|
|
9320
9467
|
} finally {
|
|
9321
9468
|
if (!settled) {
|
|
9322
9469
|
this.emitToolCallLifecycle('settled', toolId, normalizedKey, {
|
|
@@ -9335,17 +9482,36 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9335
9482
|
const historicalReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
9336
9483
|
async () => await this.getRuntimeStepReceipt(directCacheKey),
|
|
9337
9484
|
);
|
|
9338
|
-
|
|
9485
|
+
const historicalOutput = historicalReceipt
|
|
9486
|
+
? this.runtimeReceiptOutput<unknown>(historicalReceipt)
|
|
9487
|
+
: undefined;
|
|
9488
|
+
const historicalIsTerminal =
|
|
9489
|
+
historicalReceipt?.status === 'completed' ||
|
|
9490
|
+
historicalReceipt?.status === 'skipped';
|
|
9491
|
+
const historicalCanServeCurrentContract =
|
|
9492
|
+
!historicalIsTerminal ||
|
|
9493
|
+
this.canReuseToolResultForCurrentResponseContract(historicalOutput);
|
|
9494
|
+
initialDirectReceipt = historicalCanServeCurrentContract
|
|
9495
|
+
? historicalReceipt
|
|
9496
|
+
: null;
|
|
9497
|
+
if (!historicalCanServeCurrentContract) {
|
|
9498
|
+
if (!legacyDurableCacheKey) {
|
|
9499
|
+
throw new Error(
|
|
9500
|
+
`Tool ${toolId} has an unrenderable legacy result but no response-contract recovery cache key.`,
|
|
9501
|
+
);
|
|
9502
|
+
}
|
|
9503
|
+
directResultReceiptKey = legacyDurableCacheKey;
|
|
9504
|
+
}
|
|
9339
9505
|
if (
|
|
9340
9506
|
historicalReceipt &&
|
|
9341
|
-
|
|
9342
|
-
|
|
9507
|
+
historicalIsTerminal &&
|
|
9508
|
+
historicalCanServeCurrentContract &&
|
|
9343
9509
|
!isSerializedToolExecuteResult(historicalReceipt.output) &&
|
|
9344
9510
|
!isToolExecuteResult(historicalReceipt.output)
|
|
9345
9511
|
) {
|
|
9346
9512
|
const historicalOutput =
|
|
9347
9513
|
this.runtimeReceiptOutput<unknown>(historicalReceipt);
|
|
9348
|
-
|
|
9514
|
+
const recoveredHistorical = markToolExecuteResultExecutionOutcome(
|
|
9349
9515
|
historicalOutput,
|
|
9350
9516
|
toolExecutionOutcomeForDurableReceipt({
|
|
9351
9517
|
source: 'cache',
|
|
@@ -9353,9 +9519,46 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9353
9519
|
cacheKey: directCacheKey,
|
|
9354
9520
|
}),
|
|
9355
9521
|
);
|
|
9522
|
+
return isToolExecuteResult(recoveredHistorical)
|
|
9523
|
+
? this.materializeToolExecutionResult(recoveredHistorical)
|
|
9524
|
+
: recoveredHistorical;
|
|
9525
|
+
}
|
|
9526
|
+
if (
|
|
9527
|
+
(!historicalReceipt || !historicalCanServeCurrentContract) &&
|
|
9528
|
+
legacyDurableCacheKey
|
|
9529
|
+
) {
|
|
9530
|
+
const legacyReceipt = await this.withRuntimeReceiptHydrationTurn(
|
|
9531
|
+
async () =>
|
|
9532
|
+
await this.getRuntimeStepReceipt(legacyDurableCacheKey!),
|
|
9533
|
+
);
|
|
9534
|
+
if (
|
|
9535
|
+
legacyReceipt &&
|
|
9536
|
+
(legacyReceipt.status === 'completed' ||
|
|
9537
|
+
legacyReceipt.status === 'skipped') &&
|
|
9538
|
+
legacyReceipt.output !== undefined
|
|
9539
|
+
) {
|
|
9540
|
+
const recoveredLegacy = markToolExecuteResultExecutionOutcome(
|
|
9541
|
+
this.runtimeReceiptOutput(legacyReceipt),
|
|
9542
|
+
toolExecutionOutcomeForDurableReceipt({
|
|
9543
|
+
source: 'cache',
|
|
9544
|
+
receiptKey: legacyDurableCacheKey,
|
|
9545
|
+
cacheKey: directCacheKey,
|
|
9546
|
+
}),
|
|
9547
|
+
);
|
|
9548
|
+
if (
|
|
9549
|
+
this.canReuseToolResultForCurrentResponseContract(
|
|
9550
|
+
recoveredLegacy,
|
|
9551
|
+
)
|
|
9552
|
+
) {
|
|
9553
|
+
return isToolExecuteResult(recoveredLegacy)
|
|
9554
|
+
? this.materializeToolExecutionResult(recoveredLegacy)
|
|
9555
|
+
: recoveredLegacy;
|
|
9556
|
+
}
|
|
9557
|
+
bypassDirectResultCache = true;
|
|
9558
|
+
}
|
|
9356
9559
|
}
|
|
9357
9560
|
}
|
|
9358
|
-
this.
|
|
9561
|
+
this.runtimeLog(
|
|
9359
9562
|
toolCachePolicy.force
|
|
9360
9563
|
? `Calling tool: ${toolId} (force)`
|
|
9361
9564
|
: `Calling tool: ${toolId}`,
|
|
@@ -9390,23 +9593,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9390
9593
|
'Direct tool execution requires a stable logical call identity.',
|
|
9391
9594
|
);
|
|
9392
9595
|
}
|
|
9393
|
-
const
|
|
9596
|
+
const useDirectResultReceipts =
|
|
9597
|
+
cacheableToolResult && !bypassDirectResultCache;
|
|
9598
|
+
const physicalDirectKey = useDirectResultReceipts
|
|
9394
9599
|
? directCacheKey
|
|
9395
9600
|
: `${buildDurableToolReceiptPrefix({
|
|
9396
9601
|
orgId: this.#options.orgId ?? 'unknown-org',
|
|
9397
9602
|
toolId,
|
|
9398
9603
|
})}${stableDigest(`${this.currentRunId}:${logicalCallId}:always-fresh`)}`;
|
|
9604
|
+
const resultReceiptKey = directResultReceiptKey;
|
|
9399
9605
|
const providerOperationKey = providerIdempotencyKeyBase;
|
|
9400
|
-
const resultReceiptKey = directCacheKey;
|
|
9401
9606
|
const toolRetryPolicy = await this.#options
|
|
9402
9607
|
.getToolRetryPolicy?.(toolId, input)
|
|
9403
9608
|
.catch(() => null);
|
|
9404
|
-
|
|
9405
|
-
|
|
9406
|
-
|
|
9407
|
-
|
|
9408
|
-
|
|
9409
|
-
|
|
9609
|
+
if (
|
|
9610
|
+
bypassDirectResultCache &&
|
|
9611
|
+
toolRetryPolicy?.requiresExecutionFence === true
|
|
9612
|
+
) {
|
|
9613
|
+
throw new Error(
|
|
9614
|
+
`Tool ${toolId} has an unrenderable completed result under an execution fence and cannot safely refresh it.`,
|
|
9615
|
+
);
|
|
9616
|
+
}
|
|
9410
9617
|
const dispatcher = {
|
|
9411
9618
|
dispatch: async ({
|
|
9412
9619
|
providerOperationKey: operationKey,
|
|
@@ -9414,7 +9621,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9414
9621
|
providerOperationKey: string | null;
|
|
9415
9622
|
}) => {
|
|
9416
9623
|
if (
|
|
9417
|
-
|
|
9624
|
+
useDirectResultReceipts &&
|
|
9418
9625
|
operationKey !== providerOperationKey
|
|
9419
9626
|
) {
|
|
9420
9627
|
throw new Error(
|
|
@@ -9427,18 +9634,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9427
9634
|
startedAt: directCallStartedAt,
|
|
9428
9635
|
input,
|
|
9429
9636
|
options: {
|
|
9430
|
-
...(
|
|
9637
|
+
...(useDirectResultReceipts
|
|
9431
9638
|
? {
|
|
9432
9639
|
durableCallReceiptKey: physicalDirectKey,
|
|
9640
|
+
completedResultCache: true,
|
|
9433
9641
|
executionAuthScopeDigest,
|
|
9434
|
-
providerIdempotencyReceiptKey: operationKey,
|
|
9435
|
-
providerIdempotencyKey:
|
|
9436
|
-
this.providerIdempotencyKeyForToolCall({
|
|
9437
|
-
cacheKey: providerIdempotencyKeyBase,
|
|
9438
|
-
force: toolCachePolicy.force,
|
|
9439
|
-
leaseId: directReceiptLeaseId,
|
|
9440
|
-
logicalCallId,
|
|
9441
|
-
}),
|
|
9442
9642
|
...(directReceiptLeaseId &&
|
|
9443
9643
|
(this.#options.heartbeatRuntimeStepReceipts ||
|
|
9444
9644
|
this.#options.getRuntimeStepReceipt ||
|
|
@@ -9461,6 +9661,24 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9461
9661
|
: {}),
|
|
9462
9662
|
}
|
|
9463
9663
|
: {}),
|
|
9664
|
+
...(toolRetryPolicy?.requiresExecutionFence === true
|
|
9665
|
+
? {
|
|
9666
|
+
providerIdempotencyReceiptKey:
|
|
9667
|
+
cacheableToolResult && operationKey
|
|
9668
|
+
? operationKey
|
|
9669
|
+
: physicalDirectKey,
|
|
9670
|
+
providerIdempotencyKey:
|
|
9671
|
+
this.providerIdempotencyKeyForToolCall({
|
|
9672
|
+
cacheKey:
|
|
9673
|
+
useDirectResultReceipts && operationKey
|
|
9674
|
+
? providerIdempotencyKeyBase
|
|
9675
|
+
: physicalDirectKey,
|
|
9676
|
+
force: cacheableToolResult && toolCachePolicy.force,
|
|
9677
|
+
leaseId: directReceiptLeaseId,
|
|
9678
|
+
logicalCallId,
|
|
9679
|
+
}),
|
|
9680
|
+
}
|
|
9681
|
+
: {}),
|
|
9464
9682
|
playNodeScope: buildPlayNodeScope({
|
|
9465
9683
|
toolId,
|
|
9466
9684
|
callKey: normalizedKey,
|
|
@@ -9485,14 +9703,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9485
9703
|
execution: toolExecutionMetadataForOutcome({
|
|
9486
9704
|
kind: 'live',
|
|
9487
9705
|
cacheKey: directCacheKey,
|
|
9488
|
-
...(
|
|
9706
|
+
...(useDirectResultReceipts
|
|
9489
9707
|
? { receiptKey: resultReceiptKey }
|
|
9490
9708
|
: {}),
|
|
9491
9709
|
}),
|
|
9492
9710
|
});
|
|
9493
9711
|
},
|
|
9494
9712
|
};
|
|
9495
|
-
const toolCall =
|
|
9713
|
+
const toolCall = useDirectResultReceipts
|
|
9496
9714
|
? createToolCallJob({
|
|
9497
9715
|
resultReceipts: createRuntimeStepToolResultReceipts({
|
|
9498
9716
|
store: this.durableReceiptExecutionStore(),
|
|
@@ -9500,7 +9718,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9500
9718
|
...(initialDirectReceipt !== undefined
|
|
9501
9719
|
? {
|
|
9502
9720
|
initialReceipt: {
|
|
9503
|
-
key:
|
|
9721
|
+
key: directResultReceiptKey,
|
|
9504
9722
|
receipt: initialDirectReceipt,
|
|
9505
9723
|
},
|
|
9506
9724
|
}
|
|
@@ -9511,6 +9729,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9511
9729
|
? {
|
|
9512
9730
|
executionLock: {
|
|
9513
9731
|
required: true,
|
|
9732
|
+
key: providerOperationKey,
|
|
9514
9733
|
ttlMs: Math.min(
|
|
9515
9734
|
600_000,
|
|
9516
9735
|
resolveToolRuntimeTimeoutMs(
|
|
@@ -9573,11 +9792,16 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9573
9792
|
})
|
|
9574
9793
|
: await createAlwaysFreshToolCallAdapter({ dispatcher }).call(call);
|
|
9575
9794
|
settleDirectCall(toolAttemptOutcomeForResult(wrapped));
|
|
9576
|
-
if (
|
|
9577
|
-
this.cacheToolResult(
|
|
9795
|
+
if (useDirectResultReceipts) {
|
|
9796
|
+
this.cacheToolResult(
|
|
9797
|
+
toolId,
|
|
9798
|
+
directResultReceiptKey,
|
|
9799
|
+
wrapped,
|
|
9800
|
+
'direct',
|
|
9801
|
+
);
|
|
9578
9802
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
9579
9803
|
}
|
|
9580
|
-
return wrapped;
|
|
9804
|
+
return this.materializeToolExecutionResult(wrapped);
|
|
9581
9805
|
} catch (error) {
|
|
9582
9806
|
settleDirectCall('failed', error);
|
|
9583
9807
|
throw error;
|
|
@@ -9610,18 +9834,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9610
9834
|
? null
|
|
9611
9835
|
: this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
|
|
9612
9836
|
if (cached) {
|
|
9613
|
-
this.
|
|
9614
|
-
return
|
|
9615
|
-
|
|
9616
|
-
|
|
9617
|
-
|
|
9618
|
-
|
|
9619
|
-
|
|
9620
|
-
|
|
9621
|
-
|
|
9837
|
+
this.runtimeLog(` Row ${rowId} ${toolId}: recovered from checkpoint`);
|
|
9838
|
+
return this.materializeToolExecutionResult(
|
|
9839
|
+
await this.wrapToolExecutionResult({
|
|
9840
|
+
toolId,
|
|
9841
|
+
status: cached.result.result == null ? 'no_result' : 'completed',
|
|
9842
|
+
result: cached.result.result,
|
|
9843
|
+
requestInput: input,
|
|
9844
|
+
execution: toolExecutionMetadataForOutcome({
|
|
9845
|
+
kind: 'checkpoint',
|
|
9846
|
+
cacheKey: cached.cacheKey,
|
|
9847
|
+
}),
|
|
9622
9848
|
}),
|
|
9623
|
-
|
|
9849
|
+
);
|
|
9624
9850
|
}
|
|
9851
|
+
const rowToolRetryPolicy = await this.#options
|
|
9852
|
+
.getToolRetryPolicy?.(toolId, input)
|
|
9853
|
+
.catch(() => null);
|
|
9625
9854
|
|
|
9626
9855
|
// ADR 0018: record this leg before it runs. `producer` still names the
|
|
9627
9856
|
// last leg (the usual cascade winner); `producers` keeps every leg, which
|
|
@@ -9676,7 +9905,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9676
9905
|
providerIdempotencyKeyBase,
|
|
9677
9906
|
cacheable: cacheableToolResult,
|
|
9678
9907
|
receiptKey: cacheableToolResult ? durableCacheKey : null,
|
|
9908
|
+
legacyReceiptKeys: legacyDurableCacheKey
|
|
9909
|
+
? [legacyDurableCacheKey]
|
|
9910
|
+
: [],
|
|
9679
9911
|
executionAuthScopeDigest,
|
|
9912
|
+
requiresExecutionFence:
|
|
9913
|
+
rowToolRetryPolicy?.requiresExecutionFence === true,
|
|
9680
9914
|
force: toolCachePolicy.force,
|
|
9681
9915
|
forceFailedRefresh: toolCachePolicy.forceFailedRefresh,
|
|
9682
9916
|
rowId,
|
|
@@ -9696,7 +9930,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9696
9930
|
return await pendingToolCall.then(
|
|
9697
9931
|
(value) => {
|
|
9698
9932
|
settleProducerAttempt(toolAttemptOutcomeForResult(value));
|
|
9699
|
-
return
|
|
9933
|
+
return this.materializeToolExecutionResult(
|
|
9934
|
+
value as ToolExecuteResult,
|
|
9935
|
+
);
|
|
9700
9936
|
},
|
|
9701
9937
|
(error: unknown) => {
|
|
9702
9938
|
settleProducerAttempt('failed');
|
|
@@ -9730,7 +9966,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9730
9966
|
// of a silent continuation: the credential identity changed after the
|
|
9731
9967
|
// receipt key was prepared, so we evict the cached digest, re-resolve,
|
|
9732
9968
|
// and re-claim a fresh receipt under the new scope before retrying once.
|
|
9733
|
-
this.
|
|
9969
|
+
this.runtimeLog(
|
|
9734
9970
|
`ctx.tools.execute(${toolId}): auth_scope_changed_reclaim ` +
|
|
9735
9971
|
`(label: ${normalizedKey}); credential identity changed mid-run, ` +
|
|
9736
9972
|
`re-resolving auth scope and re-claiming a fresh receipt under the ` +
|
|
@@ -9745,16 +9981,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9745
9981
|
executionAuthScopeDigest,
|
|
9746
9982
|
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9747
9983
|
});
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9984
|
+
legacyDurableCacheKey = await this.legacyDurableToolCallCacheKey({
|
|
9985
|
+
toolId,
|
|
9986
|
+
requestInput: input,
|
|
9987
|
+
executionAuthScopeDigest,
|
|
9988
|
+
staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
|
|
9989
|
+
});
|
|
9990
|
+
providerIdempotencyKeyBase = durableCacheKey;
|
|
9991
|
+
directResultReceiptKey = durableCacheKey;
|
|
9992
|
+
bypassDirectResultCache = false;
|
|
9993
|
+
checkpointCacheKeys.splice(
|
|
9994
|
+
0,
|
|
9995
|
+
checkpointCacheKeys.length,
|
|
9996
|
+
durableCacheKey,
|
|
9997
|
+
...(legacyDurableCacheKey ? [legacyDurableCacheKey] : []),
|
|
9756
9998
|
);
|
|
9757
|
-
checkpointCacheKeys[0] = durableCacheKey;
|
|
9758
9999
|
}
|
|
9759
10000
|
}
|
|
9760
10001
|
|
|
@@ -9801,7 +10042,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9801
10042
|
const existing =
|
|
9802
10043
|
this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId];
|
|
9803
10044
|
if (existing?.kind === 'integration_event' && 'output' in existing) {
|
|
9804
|
-
this.
|
|
10045
|
+
this.runtimeLog(
|
|
9805
10046
|
`Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`,
|
|
9806
10047
|
);
|
|
9807
10048
|
// Wrapped like every other tool result (toolOutput.raw, getters) — the
|
|
@@ -9824,7 +10065,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9824
10065
|
boundary: preparedBoundary,
|
|
9825
10066
|
});
|
|
9826
10067
|
|
|
9827
|
-
this.
|
|
10068
|
+
this.runtimeLog(
|
|
9828
10069
|
`Armed ${handler.provider} integration event wait: ${boundary.eventKey}`,
|
|
9829
10070
|
);
|
|
9830
10071
|
this.checkpoint.resolvedBoundaries = {
|
|
@@ -9936,7 +10177,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9936
10177
|
execution: options?.execution,
|
|
9937
10178
|
childPlayName: resolvedName,
|
|
9938
10179
|
});
|
|
9939
|
-
this.
|
|
10180
|
+
this.runtimeLog(
|
|
9940
10181
|
`ctx.runPlay(${normalizedKey}): ${childExecutionDecision.strategy} (${childExecutionDecision.reason})`,
|
|
9941
10182
|
);
|
|
9942
10183
|
|
|
@@ -10182,22 +10423,89 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10182
10423
|
});
|
|
10183
10424
|
}
|
|
10184
10425
|
|
|
10185
|
-
log(msg: string): void {
|
|
10426
|
+
log(msg: string, options?: { level?: LogLevel }): void {
|
|
10186
10427
|
assertNoSecretTaint(msg, 'ctx.log');
|
|
10187
|
-
const
|
|
10428
|
+
const requestedLevel = options?.level;
|
|
10429
|
+
if (requestedLevel !== undefined && !LOG_LEVELS.includes(requestedLevel)) {
|
|
10430
|
+
throw new Error(
|
|
10431
|
+
`ctx.log level must be one of: ${LOG_LEVELS.join(', ')}.`,
|
|
10432
|
+
);
|
|
10433
|
+
}
|
|
10434
|
+
const level = requestedLevel ?? 'info';
|
|
10435
|
+
const timestamped = `[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(msg)}`;
|
|
10436
|
+
// Stamp the default too: a customer message that happens to begin with
|
|
10437
|
+
// "[error]" is still info unless the author selected a higher severity.
|
|
10438
|
+
const line = tagLogEntry('user', level, timestamped);
|
|
10439
|
+
if (!this.shouldKeepRateLimitedLog(level)) return;
|
|
10188
10440
|
this.logBuffer.push(line);
|
|
10189
10441
|
this.#options.onLog?.(line);
|
|
10190
10442
|
if (this.#options.verbose) console.log(line);
|
|
10191
10443
|
}
|
|
10192
10444
|
|
|
10445
|
+
/**
|
|
10446
|
+
* Runtime-owned progress and failure narration must stay durable even when
|
|
10447
|
+
* authored per-row logs are being sampled. Preserve the untagged historical
|
|
10448
|
+
* wire shape by default: several runtime diagnostics contain trailing JSON
|
|
10449
|
+
* consumed by older readers. Callers may opt into an explicit severity.
|
|
10450
|
+
*/
|
|
10451
|
+
private runtimeLog(msg: string, options?: { level?: LogLevel }): void {
|
|
10452
|
+
assertNoSecretTaint(msg, 'runtime log');
|
|
10453
|
+
const timestamped = `[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(msg)}`;
|
|
10454
|
+
const line = options?.level
|
|
10455
|
+
? tagLogLevel(timestamped, options.level)
|
|
10456
|
+
: timestamped;
|
|
10457
|
+
this.logBuffer.push(line);
|
|
10458
|
+
this.#options.onLog?.(line);
|
|
10459
|
+
if (this.#options.verbose) console.log(line);
|
|
10460
|
+
}
|
|
10461
|
+
|
|
10462
|
+
/** Rate-limit chatty authored logs before they consume the live stream or durable retention. */
|
|
10463
|
+
private shouldKeepRateLimitedLog(level: LogLevel): boolean {
|
|
10464
|
+
if (level === 'warn' || level === 'error') return true;
|
|
10465
|
+
const policy = LOG_RATE_LIMIT_POLICY[level];
|
|
10466
|
+
const state = this.logRateLimitState.get(level) ?? {
|
|
10467
|
+
seen: 0,
|
|
10468
|
+
sampled: 0,
|
|
10469
|
+
suppressed: 0,
|
|
10470
|
+
};
|
|
10471
|
+
state.seen += 1;
|
|
10472
|
+
const keep = state.seen <= policy.first || state.seen % policy.every === 0;
|
|
10473
|
+
if (keep) {
|
|
10474
|
+
if (state.seen > policy.first) state.sampled += 1;
|
|
10475
|
+
this.logRateLimitState.set(level, state);
|
|
10476
|
+
return true;
|
|
10477
|
+
}
|
|
10478
|
+
state.suppressed += 1;
|
|
10479
|
+
this.logRateLimitState.set(level, state);
|
|
10480
|
+
if (
|
|
10481
|
+
state.suppressed === 1 ||
|
|
10482
|
+
state.suppressed % LOG_RATE_LIMIT_MARKER_EVERY === 0
|
|
10483
|
+
) {
|
|
10484
|
+
const marker = `[log rate limited: level=${level} retained=${policy.first + state.sampled} sampled=${state.sampled} suppressed=${state.suppressed}; showing first ${policy.first}, then 1 every ${policy.every}]`;
|
|
10485
|
+
const line = tagLogEntry(
|
|
10486
|
+
'diagnostic',
|
|
10487
|
+
'warn',
|
|
10488
|
+
`[${new Date().toISOString()}] ${marker}`,
|
|
10489
|
+
);
|
|
10490
|
+
this.logBuffer.push(line);
|
|
10491
|
+
this.#options.onLog?.(line);
|
|
10492
|
+
if (this.#options.verbose) console.log(line);
|
|
10493
|
+
}
|
|
10494
|
+
return false;
|
|
10495
|
+
}
|
|
10496
|
+
|
|
10193
10497
|
/**
|
|
10194
10498
|
* Emit a runtime-authored diagnostic through the same durable log path as
|
|
10195
10499
|
* ctx.log without changing the customer-authored log wire format.
|
|
10196
10500
|
*/
|
|
10197
|
-
private runtimeDiagnosticLog(
|
|
10501
|
+
private runtimeDiagnosticLog(
|
|
10502
|
+
message: string,
|
|
10503
|
+
level: LogLevel = 'warn',
|
|
10504
|
+
): void {
|
|
10198
10505
|
assertNoSecretTaint(message, 'runtime diagnostic log');
|
|
10199
|
-
const line =
|
|
10506
|
+
const line = tagLogEntry(
|
|
10200
10507
|
'diagnostic',
|
|
10508
|
+
level,
|
|
10201
10509
|
`[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(message)}`,
|
|
10202
10510
|
);
|
|
10203
10511
|
this.logBuffer.push(line);
|
|
@@ -10438,7 +10746,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10438
10746
|
existing?.kind === 'fetch' &&
|
|
10439
10747
|
'output' in existing
|
|
10440
10748
|
) {
|
|
10441
|
-
this.
|
|
10749
|
+
this.runtimeLog(
|
|
10750
|
+
`ctx.fetch(${url}): recovered response from checkpoint`,
|
|
10751
|
+
);
|
|
10442
10752
|
const checkpointOutput = existing.output as PlayFetchResponse;
|
|
10443
10753
|
if (!checkpointOutput.ok) {
|
|
10444
10754
|
this.logCtxFetchHttpFailure({
|
|
@@ -10556,7 +10866,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10556
10866
|
canRetryTransport &&
|
|
10557
10867
|
attempt < FETCH_TRANSPORT_MAX_ATTEMPTS
|
|
10558
10868
|
) {
|
|
10559
|
-
this.
|
|
10869
|
+
this.runtimeLog(
|
|
10560
10870
|
`ctx.fetch(${method} ${url}) transport failed on attempt ${attempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`,
|
|
10561
10871
|
);
|
|
10562
10872
|
await sleepWithinCtxFetchDeadline({
|
|
@@ -10665,7 +10975,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10665
10975
|
const executeStep = async (): Promise<T> => {
|
|
10666
10976
|
const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
|
|
10667
10977
|
if (existing?.kind === 'step' && 'output' in existing) {
|
|
10668
|
-
this.
|
|
10978
|
+
this.runtimeLog(
|
|
10669
10979
|
`ctx.step(${normalizedKey}): recovered result from checkpoint`,
|
|
10670
10980
|
);
|
|
10671
10981
|
return existing.output as T;
|
|
@@ -10869,7 +11179,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10869
11179
|
leaseLost ??= error;
|
|
10870
11180
|
},
|
|
10871
11181
|
onTransientFailure: (error) => {
|
|
10872
|
-
this.
|
|
11182
|
+
this.runtimeLog(
|
|
10873
11183
|
`Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
|
|
10874
11184
|
);
|
|
10875
11185
|
},
|
|
@@ -10914,75 +11224,59 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10914
11224
|
/**
|
|
10915
11225
|
* Concrete RuntimeStepReceipt adapter for the map receipt-cohort Module.
|
|
10916
11226
|
*
|
|
10917
|
-
*
|
|
10918
|
-
*
|
|
10919
|
-
* protocol: both need an atomic claim/reclaim, a single cohort heartbeat,
|
|
10920
|
-
* and lease-fenced terminal fanout. Keeping that at one seam prevents a
|
|
10921
|
-
* native batch from becoming a second, subtly different receipt runtime.
|
|
11227
|
+
* Scalar and provider-native map calls share completed-result cache reads
|
|
11228
|
+
* and atomic successful inserts. A miss has no durable execution owner.
|
|
10922
11229
|
*/
|
|
10923
11230
|
private createRuntimeStepToolCallReceiptCohortStore(input: {
|
|
10924
11231
|
toolId: string;
|
|
10925
11232
|
requestForReceipt: ReadonlyMap<string, ToolCallRequest>;
|
|
10926
11233
|
}): ToolCallReceiptCohortStore {
|
|
10927
11234
|
const asRecord = async (
|
|
10928
|
-
|
|
11235
|
+
resultReceiptKey: string,
|
|
11236
|
+
storedReceiptKey: string,
|
|
10929
11237
|
receipt: RuntimeStepReceipt | null | undefined,
|
|
10930
11238
|
): Promise<ToolCallReceiptRecord | null> => {
|
|
10931
11239
|
if (!receipt) return null;
|
|
10932
|
-
const normalized = this.normalizeRuntimeStepReceipt(
|
|
11240
|
+
const normalized = this.normalizeRuntimeStepReceipt(
|
|
11241
|
+
storedReceiptKey,
|
|
11242
|
+
receipt,
|
|
11243
|
+
);
|
|
10933
11244
|
if (!normalized) return null;
|
|
10934
|
-
|
|
11245
|
+
if (
|
|
11246
|
+
normalized.status !== 'completed' &&
|
|
11247
|
+
normalized.status !== 'skipped'
|
|
11248
|
+
) {
|
|
11249
|
+
return null;
|
|
11250
|
+
}
|
|
11251
|
+
// A terminal acknowledgement without output is not a cache hit. The
|
|
11252
|
+
// completion path hydrates it first; if that read cannot provide an
|
|
11253
|
+
// output, retain the live result rather than resolving `undefined`.
|
|
11254
|
+
if (normalized.output === undefined) return null;
|
|
11255
|
+
const request = input.requestForReceipt.get(resultReceiptKey);
|
|
10935
11256
|
if (!request) {
|
|
10936
11257
|
throw new Error(
|
|
10937
|
-
`Unknown Tool Call Receipt Cohort receipt ${
|
|
11258
|
+
`Unknown Tool Call Receipt Cohort receipt ${resultReceiptKey}.`,
|
|
10938
11259
|
);
|
|
10939
11260
|
}
|
|
10940
|
-
const
|
|
10941
|
-
|
|
10942
|
-
|
|
10943
|
-
|
|
10944
|
-
|
|
10945
|
-
|
|
10946
|
-
|
|
10947
|
-
|
|
10948
|
-
|
|
10949
|
-
|
|
10950
|
-
|
|
10951
|
-
|
|
10952
|
-
|
|
10953
|
-
|
|
10954
|
-
|
|
10955
|
-
attachedToReceiptKey: receiptKey,
|
|
10956
|
-
}),
|
|
10957
|
-
})
|
|
10958
|
-
: undefined;
|
|
11261
|
+
const result = await this.wrapToolExecutionResult({
|
|
11262
|
+
toolId: input.toolId,
|
|
11263
|
+
status:
|
|
11264
|
+
normalized.output === null || normalized.output === undefined
|
|
11265
|
+
? 'no_result'
|
|
11266
|
+
: 'completed',
|
|
11267
|
+
result: this.runtimeReceiptOutput(normalized),
|
|
11268
|
+
requestInput: request.input,
|
|
11269
|
+
execution: toolExecutionMetadataForOutcome({
|
|
11270
|
+
kind: 'cache',
|
|
11271
|
+
cacheKey: request.cacheKey,
|
|
11272
|
+
receiptKey: resultReceiptKey,
|
|
11273
|
+
attachedToReceiptKey: storedReceiptKey,
|
|
11274
|
+
}),
|
|
11275
|
+
});
|
|
10959
11276
|
return {
|
|
10960
|
-
resultReceiptKey
|
|
11277
|
+
resultReceiptKey,
|
|
10961
11278
|
status: normalized.status,
|
|
10962
|
-
|
|
10963
|
-
...(normalized.status === 'failed'
|
|
10964
|
-
? {
|
|
10965
|
-
error: runtimeReceiptFailureError(
|
|
10966
|
-
normalized,
|
|
10967
|
-
'Durable tool call failed',
|
|
10968
|
-
this.currentToolErrorSchemaVersion,
|
|
10969
|
-
),
|
|
10970
|
-
}
|
|
10971
|
-
: {}),
|
|
10972
|
-
...(normalized.leaseId &&
|
|
10973
|
-
(normalized.leaseOwnerRunId ?? normalized.runId)?.trim()
|
|
10974
|
-
? {
|
|
10975
|
-
lease: {
|
|
10976
|
-
leaseId: normalized.leaseId,
|
|
10977
|
-
ownerRunId: (normalized.leaseOwnerRunId ??
|
|
10978
|
-
normalized.runId)!.trim(),
|
|
10979
|
-
},
|
|
10980
|
-
}
|
|
10981
|
-
: {}),
|
|
10982
|
-
...(normalized.leaseExpiresAt
|
|
10983
|
-
? { leaseExpiresAt: normalized.leaseExpiresAt }
|
|
10984
|
-
: {}),
|
|
10985
|
-
...(normalized.claimState ? { claimState: normalized.claimState } : {}),
|
|
11279
|
+
result,
|
|
10986
11280
|
};
|
|
10987
11281
|
};
|
|
10988
11282
|
const recordsFor = async (
|
|
@@ -10991,10 +11285,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10991
11285
|
): Promise<ToolCallReceiptRecord[]> =>
|
|
10992
11286
|
(
|
|
10993
11287
|
await Promise.all(
|
|
10994
|
-
receiptKeys.map(
|
|
10995
|
-
|
|
10996
|
-
|
|
10997
|
-
|
|
11288
|
+
receiptKeys.map(async (receiptKey) => {
|
|
11289
|
+
const request = input.requestForReceipt.get(receiptKey);
|
|
11290
|
+
const candidates = [
|
|
11291
|
+
receiptKey,
|
|
11292
|
+
...(request?.legacyReceiptKeys ?? []),
|
|
11293
|
+
];
|
|
11294
|
+
for (const candidate of candidates) {
|
|
11295
|
+
const record = await asRecord(
|
|
11296
|
+
receiptKey,
|
|
11297
|
+
candidate,
|
|
11298
|
+
receipts.get(candidate),
|
|
11299
|
+
);
|
|
11300
|
+
if (record) return record;
|
|
11301
|
+
}
|
|
11302
|
+
return null;
|
|
11303
|
+
}),
|
|
10998
11304
|
)
|
|
10999
11305
|
).filter((record): record is ToolCallReceiptRecord => record !== null);
|
|
11000
11306
|
|
|
@@ -11004,83 +11310,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11004
11310
|
async () =>
|
|
11005
11311
|
await recordsFor(
|
|
11006
11312
|
resultReceiptKeys,
|
|
11007
|
-
await this.getRuntimeStepReceipts(
|
|
11313
|
+
await this.getRuntimeStepReceipts(
|
|
11314
|
+
[...new Set(
|
|
11315
|
+
resultReceiptKeys.flatMap((receiptKey) => [
|
|
11316
|
+
receiptKey,
|
|
11317
|
+
...(input.requestForReceipt.get(receiptKey)
|
|
11318
|
+
?.legacyReceiptKeys ?? []),
|
|
11319
|
+
]),
|
|
11320
|
+
)],
|
|
11321
|
+
),
|
|
11008
11322
|
),
|
|
11009
11323
|
),
|
|
11010
|
-
claim: async ({ receipts }) => {
|
|
11011
|
-
const claims = new Map<string, RuntimeStepReceipt>();
|
|
11012
|
-
const groups = new Map<string, Array<(typeof receipts)[number]>>();
|
|
11013
|
-
for (const receipt of receipts) {
|
|
11014
|
-
const groupKey = [
|
|
11015
|
-
receipt.reclaimRunning ? 'reclaim' : 'ordinary',
|
|
11016
|
-
receipt.force ? 'force' : 'cached',
|
|
11017
|
-
receipt.forceFailedRefresh ? 'failed-force' : 'failed-cached',
|
|
11018
|
-
].join(':');
|
|
11019
|
-
const group = groups.get(groupKey) ?? [];
|
|
11020
|
-
group.push(receipt);
|
|
11021
|
-
groups.set(groupKey, group);
|
|
11022
|
-
}
|
|
11023
|
-
for (const group of groups.values()) {
|
|
11024
|
-
const first = group[0]!;
|
|
11025
|
-
const claimed = await this.claimRuntimeStepReceipts(
|
|
11026
|
-
group.map((receipt) => receipt.resultReceiptKey),
|
|
11027
|
-
this.currentReceiptOwnerRunId,
|
|
11028
|
-
first.reclaimRunning,
|
|
11029
|
-
first.force,
|
|
11030
|
-
first.forceFailedRefresh,
|
|
11031
|
-
);
|
|
11032
|
-
for (const [key, receipt] of claimed) claims.set(key, receipt);
|
|
11033
|
-
}
|
|
11034
|
-
const safeFailedKeys = receipts.flatMap((receipt) => {
|
|
11035
|
-
const claimed = claims.get(receipt.resultReceiptKey);
|
|
11036
|
-
return claimed && this.canRetryWithoutProviderDispatch(claimed)
|
|
11037
|
-
? [receipt.resultReceiptKey]
|
|
11038
|
-
: [];
|
|
11039
|
-
});
|
|
11040
|
-
if (safeFailedKeys.length > 0) {
|
|
11041
|
-
const refreshed = await this.claimRuntimeStepReceipts(
|
|
11042
|
-
safeFailedKeys,
|
|
11043
|
-
this.currentReceiptOwnerRunId,
|
|
11044
|
-
false,
|
|
11045
|
-
false,
|
|
11046
|
-
true,
|
|
11047
|
-
);
|
|
11048
|
-
for (const [key, receipt] of refreshed) claims.set(key, receipt);
|
|
11049
|
-
}
|
|
11050
|
-
return await recordsFor(
|
|
11051
|
-
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11052
|
-
claims,
|
|
11053
|
-
);
|
|
11054
|
-
},
|
|
11055
|
-
heartbeat: async ({ receipts }) => {
|
|
11056
|
-
if (!this.#options.heartbeatRuntimeStepReceipts) {
|
|
11057
|
-
return await recordsFor(
|
|
11058
|
-
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11059
|
-
await this.getRuntimeStepReceipts(
|
|
11060
|
-
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11061
|
-
),
|
|
11062
|
-
);
|
|
11063
|
-
}
|
|
11064
|
-
const response = await this.#options.heartbeatRuntimeStepReceipts({
|
|
11065
|
-
runId: this.currentReceiptOwnerRunId,
|
|
11066
|
-
runAttempt: this.currentRunAttempt,
|
|
11067
|
-
keys: receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11068
|
-
leaseIds: receipts.map((receipt) => receipt.lease.leaseId),
|
|
11069
|
-
});
|
|
11070
|
-
const byKey = new Map<string, RuntimeStepReceipt>();
|
|
11071
|
-
for (let index = 0; index < receipts.length; index += 1) {
|
|
11072
|
-
const request = receipts[index]!;
|
|
11073
|
-
const normalized = this.normalizeRuntimeStepReceipt(
|
|
11074
|
-
request.resultReceiptKey,
|
|
11075
|
-
response[index],
|
|
11076
|
-
);
|
|
11077
|
-
if (normalized) byKey.set(request.resultReceiptKey, normalized);
|
|
11078
|
-
}
|
|
11079
|
-
return await recordsFor(
|
|
11080
|
-
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11081
|
-
byKey,
|
|
11082
|
-
);
|
|
11083
|
-
},
|
|
11084
11324
|
complete: async ({ receipts }) => {
|
|
11085
11325
|
const completed = await this.completeRuntimeStepReceipts(
|
|
11086
11326
|
receipts.map((receipt) => ({
|
|
@@ -11095,33 +11335,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11095
11335
|
completed,
|
|
11096
11336
|
);
|
|
11097
11337
|
},
|
|
11098
|
-
fail: async ({ receipts }) => {
|
|
11099
|
-
const failed = await this.failRuntimeStepReceipts(
|
|
11100
|
-
receipts.map((receipt) => ({
|
|
11101
|
-
key: receipt.resultReceiptKey,
|
|
11102
|
-
runId: this.currentReceiptOwnerRunId,
|
|
11103
|
-
leaseId: receipt.lease.leaseId,
|
|
11104
|
-
error: this.formatRuntimeError(receipt.error),
|
|
11105
|
-
errorPayload: serializeToolExecutionFailure(receipt.error),
|
|
11106
|
-
failureKind: runtimeReceiptFailureKindForError(receipt.error),
|
|
11107
|
-
})),
|
|
11108
|
-
);
|
|
11109
|
-
return await recordsFor(
|
|
11110
|
-
receipts.map((receipt) => receipt.resultReceiptKey),
|
|
11111
|
-
failed,
|
|
11112
|
-
);
|
|
11113
|
-
},
|
|
11114
11338
|
};
|
|
11115
11339
|
}
|
|
11116
11340
|
|
|
11117
11341
|
/**
|
|
11118
|
-
*
|
|
11119
|
-
*
|
|
11120
|
-
*
|
|
11121
|
-
* Tool Call boundary to be only cosmetic. This is the narrow replacement
|
|
11122
|
-
* slice: cacheable members of a *non-native* map hand their whole receipt
|
|
11123
|
-
* lifecycle to the cohort Module, while this Context remains only the
|
|
11124
|
-
* transport/presentation Adapter.
|
|
11342
|
+
* Cacheable scalar map members share completed-result reads and successful
|
|
11343
|
+
* writes through the cohort Module. This Context only transports and
|
|
11344
|
+
* presents results.
|
|
11125
11345
|
*
|
|
11126
11346
|
* Do not widen this to physical native batches or always-fresh calls. A
|
|
11127
11347
|
* native batch owns a different aggregate operation receipt, and
|
|
@@ -11211,7 +11431,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11211
11431
|
cohortLeaseLost ??= error;
|
|
11212
11432
|
},
|
|
11213
11433
|
onTransientFailure: (error) => {
|
|
11214
|
-
this.
|
|
11434
|
+
this.runtimeLog(
|
|
11215
11435
|
`Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
11216
11436
|
error instanceof Error ? error.message : String(error)
|
|
11217
11437
|
}`,
|
|
@@ -11223,7 +11443,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11223
11443
|
const resolve = async (
|
|
11224
11444
|
request: ToolCallRequest,
|
|
11225
11445
|
result: ToolExecuteResult,
|
|
11226
|
-
source: 'cache' | '
|
|
11446
|
+
source: 'cache' | 'owner',
|
|
11227
11447
|
): Promise<void> => {
|
|
11228
11448
|
const receiptKey = request.receiptKey!;
|
|
11229
11449
|
const finalResult = cloneToolExecuteResultWithExecution(
|
|
@@ -11258,180 +11478,111 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11258
11478
|
};
|
|
11259
11479
|
const reject = async (request: ToolCallRequest, error: unknown) => {
|
|
11260
11480
|
await this.rejectToolCall(toolId, request, error, {
|
|
11261
|
-
//
|
|
11262
|
-
// legacy writer here would race its member terminal transition.
|
|
11481
|
+
// Failures are never cache entries, including failed local misses.
|
|
11263
11482
|
persistReceiptFailure: false,
|
|
11264
11483
|
});
|
|
11265
11484
|
};
|
|
11266
11485
|
|
|
11267
|
-
|
|
11268
|
-
|
|
11269
|
-
|
|
11270
|
-
|
|
11486
|
+
const handled = new Set<string>();
|
|
11487
|
+
const followerResults: Promise<void>[] = [];
|
|
11488
|
+
const collectReadyMembers = (): Array<{
|
|
11489
|
+
request: ToolCallRequest;
|
|
11490
|
+
receipts: ToolResultReceipts;
|
|
11491
|
+
}> => {
|
|
11492
|
+
const owned: Array<{
|
|
11271
11493
|
request: ToolCallRequest;
|
|
11272
11494
|
receipts: ToolResultReceipts;
|
|
11273
|
-
}>
|
|
11274
|
-
|
|
11275
|
-
|
|
11276
|
-
|
|
11277
|
-
|
|
11278
|
-
|
|
11279
|
-
|
|
11280
|
-
|
|
11281
|
-
|
|
11282
|
-
|
|
11283
|
-
|
|
11284
|
-
|
|
11285
|
-
|
|
11286
|
-
|
|
11287
|
-
|
|
11288
|
-
|
|
11289
|
-
} else if (member.kind === 'following') {
|
|
11290
|
-
followOrRecover.push(
|
|
11291
|
-
member.result.then(
|
|
11292
|
-
async (result) => await resolve(request, result, 'in_flight'),
|
|
11293
|
-
async (error: unknown) => await reject(request, error),
|
|
11294
|
-
),
|
|
11295
|
-
);
|
|
11296
|
-
} else {
|
|
11297
|
-
request.receiptLeaseId = member.lease.leaseId;
|
|
11298
|
-
request.receiptLeaseExpiresAt = member.leaseExpiresAt ?? null;
|
|
11299
|
-
owned.push({ request, receipts: member.receipts });
|
|
11300
|
-
}
|
|
11495
|
+
}> = [];
|
|
11496
|
+
for (const request of requests) {
|
|
11497
|
+
if (handled.has(request.callId)) continue;
|
|
11498
|
+
const member = cohort.member(request.callId);
|
|
11499
|
+
handled.add(request.callId);
|
|
11500
|
+
if (member.kind === 'completed') {
|
|
11501
|
+
followerResults.push(resolve(request, member.result, member.source));
|
|
11502
|
+
} else if (member.kind === 'following') {
|
|
11503
|
+
followerResults.push(
|
|
11504
|
+
member.result.then(
|
|
11505
|
+
async (result) => await resolve(request, result, 'owner'),
|
|
11506
|
+
async (error: unknown) => await reject(request, error),
|
|
11507
|
+
),
|
|
11508
|
+
);
|
|
11509
|
+
} else {
|
|
11510
|
+
owned.push({ request, receipts: member.receipts });
|
|
11301
11511
|
}
|
|
11302
|
-
|
|
11303
|
-
|
|
11512
|
+
}
|
|
11513
|
+
return owned;
|
|
11514
|
+
};
|
|
11304
11515
|
|
|
11305
|
-
|
|
11306
|
-
|
|
11307
|
-
|
|
11308
|
-
|
|
11309
|
-
|
|
11310
|
-
|
|
11311
|
-
|
|
11312
|
-
),
|
|
11516
|
+
const width = Math.min(
|
|
11517
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11518
|
+
Math.max(
|
|
11519
|
+
1,
|
|
11520
|
+
await this.resourceGovernor.suggestedToolParallelism(
|
|
11521
|
+
toolId,
|
|
11522
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11313
11523
|
),
|
|
11314
|
-
)
|
|
11315
|
-
|
|
11316
|
-
|
|
11317
|
-
|
|
11318
|
-
|
|
11319
|
-
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11326
|
-
|
|
11327
|
-
|
|
11328
|
-
|
|
11329
|
-
|
|
11330
|
-
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11339
|
-
|
|
11340
|
-
|
|
11341
|
-
|
|
11342
|
-
|
|
11343
|
-
|
|
11344
|
-
|
|
11345
|
-
|
|
11346
|
-
|
|
11347
|
-
|
|
11348
|
-
|
|
11349
|
-
|
|
11350
|
-
|
|
11351
|
-
|
|
11352
|
-
|
|
11353
|
-
);
|
|
11354
|
-
}
|
|
11355
|
-
} catch (error) {
|
|
11356
|
-
if (error instanceof ToolCallReceiptLeaseLostError) {
|
|
11357
|
-
throw new RuntimeReceiptLeaseLostError({
|
|
11358
|
-
receiptKey: request.receiptKey!,
|
|
11359
|
-
runId: this.currentReceiptOwnerRunId,
|
|
11360
|
-
leaseId:
|
|
11361
|
-
request.receiptLeaseId ?? 'missing-cohort-lease',
|
|
11362
|
-
});
|
|
11363
|
-
}
|
|
11364
|
-
throw error;
|
|
11365
|
-
}
|
|
11366
|
-
},
|
|
11367
|
-
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
11368
|
-
durableCallReceiptKey: request.receiptKey,
|
|
11369
|
-
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
11370
|
-
providerIdempotencyReceiptKey:
|
|
11371
|
-
request.providerIdempotencyKeyBase ?? request.receiptKey,
|
|
11372
|
-
providerIdempotencyKey:
|
|
11373
|
-
this.providerIdempotencyKeyForToolCall({
|
|
11374
|
-
cacheKey:
|
|
11375
|
-
request.providerIdempotencyKeyBase ??
|
|
11376
|
-
request.receiptKey!,
|
|
11377
|
-
force: request.force === true,
|
|
11378
|
-
leaseId: request.receiptLeaseId,
|
|
11379
|
-
logicalCallId: request.logicalCallId,
|
|
11380
|
-
}),
|
|
11381
|
-
receiptLeaseExpiresAt: request.receiptLeaseExpiresAt,
|
|
11382
|
-
timeoutMs:
|
|
11383
|
-
typeof request.timeoutMs === 'number' &&
|
|
11384
|
-
request.timeoutMs > 0
|
|
11385
|
-
? request.timeoutMs
|
|
11386
|
-
: undefined,
|
|
11387
|
-
retainToolSlot: (release) => {
|
|
11388
|
-
releaseToolSlot = release;
|
|
11389
|
-
},
|
|
11524
|
+
),
|
|
11525
|
+
);
|
|
11526
|
+
const dispatchOwned = async (
|
|
11527
|
+
owned: Array<{
|
|
11528
|
+
request: ToolCallRequest;
|
|
11529
|
+
receipts: ToolResultReceipts;
|
|
11530
|
+
}>,
|
|
11531
|
+
) =>
|
|
11532
|
+
await dispatchBoundedSettled(
|
|
11533
|
+
owned,
|
|
11534
|
+
width,
|
|
11535
|
+
async ({ request, receipts }) => {
|
|
11536
|
+
if (this.persistenceLatch.tripped) {
|
|
11537
|
+
const error = new RuntimePersistenceCircuitOpenError(
|
|
11538
|
+
this.persistenceLatch,
|
|
11539
|
+
);
|
|
11540
|
+
throw error;
|
|
11541
|
+
}
|
|
11542
|
+
let releaseToolSlot: () => void = () => undefined;
|
|
11543
|
+
try {
|
|
11544
|
+
const result = await this.executeMappedToolCallThroughModule({
|
|
11545
|
+
request,
|
|
11546
|
+
dispatchToolId: toolId,
|
|
11547
|
+
dispatchPayload: request.input,
|
|
11548
|
+
resultReceiptKey: request.receiptKey!,
|
|
11549
|
+
providerOperationKey:
|
|
11550
|
+
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
11551
|
+
resultReceipts: receipts,
|
|
11552
|
+
apiOptions: {
|
|
11553
|
+
playNodeScope: playNodeScopeForToolCallRequest(request),
|
|
11554
|
+
durableCallReceiptKey: request.receiptKey,
|
|
11555
|
+
completedResultCache: true,
|
|
11556
|
+
executionAuthScopeDigest: request.executionAuthScopeDigest,
|
|
11557
|
+
timeoutMs:
|
|
11558
|
+
typeof request.timeoutMs === 'number' && request.timeoutMs > 0
|
|
11559
|
+
? request.timeoutMs
|
|
11560
|
+
: undefined,
|
|
11561
|
+
retainToolSlot: (release) => {
|
|
11562
|
+
releaseToolSlot = release;
|
|
11390
11563
|
},
|
|
11391
|
-
}
|
|
11392
|
-
|
|
11393
|
-
|
|
11394
|
-
|
|
11395
|
-
|
|
11396
|
-
|
|
11397
|
-
|
|
11398
|
-
}
|
|
11399
|
-
|
|
11564
|
+
},
|
|
11565
|
+
});
|
|
11566
|
+
await resolve(request, result, 'owner');
|
|
11567
|
+
} catch (error) {
|
|
11568
|
+
await reject(request, error);
|
|
11569
|
+
} finally {
|
|
11570
|
+
releaseToolSlot();
|
|
11571
|
+
}
|
|
11572
|
+
},
|
|
11573
|
+
);
|
|
11400
11574
|
|
|
11401
|
-
|
|
11402
|
-
|
|
11403
|
-
// overlap: a broken foreign read must not strand an already-owned
|
|
11404
|
-
// provider operation behind it.
|
|
11405
|
-
const initialDispatch = dispatchOwned(collectReadyMembers());
|
|
11406
|
-
const settlement = cohort.settle();
|
|
11407
|
-
const [initialResult, settlementResult] = await Promise.allSettled([
|
|
11408
|
-
initialDispatch,
|
|
11409
|
-
settlement,
|
|
11410
|
-
]);
|
|
11411
|
-
// A recovery can reclaim an expired receipt and turn it into another
|
|
11412
|
-
// owned member. Collect its final state after the initial work settles.
|
|
11413
|
-
const recoveredDispatch = await dispatchOwned(collectReadyMembers());
|
|
11414
|
-
await Promise.allSettled(followOrRecover);
|
|
11415
|
-
// dispatchBoundedSettled intentionally captures provider failures; all
|
|
11416
|
-
// affected row resolvers have already been rejected above.
|
|
11417
|
-
void initialResult;
|
|
11418
|
-
void recoveredDispatch;
|
|
11419
|
-
if (settlementResult.status === 'rejected') {
|
|
11420
|
-
throw settlementResult.reason;
|
|
11421
|
-
}
|
|
11422
|
-
} finally {
|
|
11423
|
-
heartbeat.stop();
|
|
11424
|
-
}
|
|
11575
|
+
await dispatchOwned(collectReadyMembers());
|
|
11576
|
+
await Promise.allSettled(followerResults);
|
|
11425
11577
|
return successful;
|
|
11426
11578
|
}
|
|
11427
11579
|
|
|
11428
11580
|
/**
|
|
11429
11581
|
* Executes provider-native batches through the Native Batch Tool Call
|
|
11430
|
-
* Module.
|
|
11431
|
-
*
|
|
11432
|
-
*
|
|
11433
|
-
*
|
|
11434
|
-
* to map rows.
|
|
11582
|
+
* Module. The Module reads exact completed-result cache entries, batches
|
|
11583
|
+
* misses, then publishes each successful split result. This Context only
|
|
11584
|
+
* compiles provider payloads, invokes transport, and presents terminal
|
|
11585
|
+
* member values to map rows.
|
|
11435
11586
|
*/
|
|
11436
11587
|
private async executeCacheableNativeMapToolCallCohort(input: {
|
|
11437
11588
|
toolId: string;
|
|
@@ -11486,9 +11637,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11486
11637
|
batchParallelismCeiling,
|
|
11487
11638
|
)
|
|
11488
11639
|
: batchParallelismCeiling;
|
|
11489
|
-
const batchProviderActionVersion =
|
|
11490
|
-
(await this.#options.getToolActionCacheVersion?.(toolId))?.trim() ??
|
|
11491
|
-
'unversioned';
|
|
11492
11640
|
const successful: ToolCallRequest[] = [];
|
|
11493
11641
|
|
|
11494
11642
|
const deliver = (request: ToolCallRequest, result: ToolExecuteResult) => {
|
|
@@ -11518,7 +11666,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11518
11666
|
};
|
|
11519
11667
|
const reject = async (request: ToolCallRequest, error: unknown) => {
|
|
11520
11668
|
await this.rejectToolCall(toolId, request, error, {
|
|
11521
|
-
//
|
|
11669
|
+
// Failed provider calls are never cache entries.
|
|
11522
11670
|
persistReceiptFailure: false,
|
|
11523
11671
|
});
|
|
11524
11672
|
};
|
|
@@ -11580,15 +11728,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11580
11728
|
call.payload,
|
|
11581
11729
|
{
|
|
11582
11730
|
durableCallReceiptKey: compatibility.resultReceiptKey,
|
|
11731
|
+
completedResultCache: true,
|
|
11583
11732
|
playNodeScope: playNodeScopeForToolCallRequest(
|
|
11584
11733
|
detail.members[0]!,
|
|
11585
11734
|
call.tool.id,
|
|
11586
11735
|
),
|
|
11587
11736
|
executionAuthScopeDigest:
|
|
11588
11737
|
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
11589
|
-
providerIdempotencyReceiptKey:
|
|
11590
|
-
detail.providerOperationKey,
|
|
11591
|
-
providerIdempotencyKey: detail.providerOperationKey,
|
|
11592
11738
|
receiptLeaseExpiresAt: detail.receiptLeaseExpiresAt,
|
|
11593
11739
|
timeoutMs: detail.timeoutMs,
|
|
11594
11740
|
},
|
|
@@ -11656,8 +11802,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11656
11802
|
},
|
|
11657
11803
|
compatibility: {
|
|
11658
11804
|
resultReceiptKey: request.receiptKey!,
|
|
11659
|
-
providerOperationKey:
|
|
11660
|
-
request.providerIdempotencyKeyBase ?? request.receiptKey!,
|
|
11805
|
+
providerOperationKey: request.receiptKey!,
|
|
11661
11806
|
},
|
|
11662
11807
|
},
|
|
11663
11808
|
})),
|
|
@@ -11680,27 +11825,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11680
11825
|
'Native batch owned members changed provider operation during compilation.',
|
|
11681
11826
|
);
|
|
11682
11827
|
}
|
|
11683
|
-
const providerOperationKey =
|
|
11684
|
-
buildDurableToolPayloadProviderIdempotencyKey({
|
|
11685
|
-
orgId: this.#options.orgId,
|
|
11686
|
-
toolId: batch.batchOperation,
|
|
11687
|
-
executionAuthScopeDigest:
|
|
11688
|
-
owned[0]?.executionAuthScopeDigest ?? null,
|
|
11689
|
-
providerActionVersion: batchProviderActionVersion,
|
|
11690
|
-
payload: compiled.batchPayload,
|
|
11691
|
-
forceRefreshIdentities: owned
|
|
11692
|
-
.filter((request) => request.force === true)
|
|
11693
|
-
.map((request) =>
|
|
11694
|
-
this.providerIdempotencyKeyForToolCall({
|
|
11695
|
-
cacheKey:
|
|
11696
|
-
request.providerIdempotencyKeyBase ??
|
|
11697
|
-
request.receiptKey!,
|
|
11698
|
-
force: true,
|
|
11699
|
-
leaseId: request.receiptLeaseId,
|
|
11700
|
-
logicalCallId: request.logicalCallId,
|
|
11701
|
-
}),
|
|
11702
|
-
),
|
|
11703
|
-
});
|
|
11828
|
+
const providerOperationKey = aggregateReceiptKey;
|
|
11704
11829
|
dispatchDetails.set(aggregateReceiptKey, {
|
|
11705
11830
|
members: owned,
|
|
11706
11831
|
compiled,
|
|
@@ -11790,8 +11915,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11790
11915
|
toolResponse: publicToolResponseForBatchedItem(
|
|
11791
11916
|
aggregate,
|
|
11792
11917
|
item,
|
|
11793
|
-
|
|
11794
|
-
RAW_V2_TOOL_RESPONSE_CONTRACT,
|
|
11918
|
+
true,
|
|
11795
11919
|
),
|
|
11796
11920
|
requestInput: request.input,
|
|
11797
11921
|
execution: toolExecutionMetadataForOutcome({
|
|
@@ -11804,60 +11928,22 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11804
11928
|
}),
|
|
11805
11929
|
);
|
|
11806
11930
|
},
|
|
11807
|
-
heartbeat: {
|
|
11808
|
-
// Claim replies carry the real expiry. A fixed fallback interval
|
|
11809
|
-
// can be too slow for a short-lived gateway lease, so derive the
|
|
11810
|
-
// one cohort supervisor from the earliest owned member instead.
|
|
11811
|
-
intervalMs: ({ members }) =>
|
|
11812
|
-
Math.max(
|
|
11813
|
-
1,
|
|
11814
|
-
Math.floor(
|
|
11815
|
-
runtimeLeaseHeartbeatIntervalFromExpiry({
|
|
11816
|
-
leaseExpiresAt: members.reduce<string | null>(
|
|
11817
|
-
(earliest, member) => {
|
|
11818
|
-
const expiresAt = member.leaseExpiresAt ?? null;
|
|
11819
|
-
if (!expiresAt) return earliest;
|
|
11820
|
-
if (!earliest) return expiresAt;
|
|
11821
|
-
return Date.parse(expiresAt) < Date.parse(earliest)
|
|
11822
|
-
? expiresAt
|
|
11823
|
-
: earliest;
|
|
11824
|
-
},
|
|
11825
|
-
null,
|
|
11826
|
-
),
|
|
11827
|
-
fallbackTtlMs: PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
|
|
11828
|
-
}) / 2,
|
|
11829
|
-
),
|
|
11830
|
-
),
|
|
11831
|
-
onTransientFailure: (error) => {
|
|
11832
|
-
this.log(
|
|
11833
|
-
`Native Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
11834
|
-
error instanceof Error ? error.message : String(error)
|
|
11835
|
-
}`,
|
|
11836
|
-
);
|
|
11837
|
-
},
|
|
11838
|
-
},
|
|
11839
11931
|
});
|
|
11840
11932
|
return { batch, result };
|
|
11841
11933
|
},
|
|
11842
11934
|
onChunkComplete: async (entries) => {
|
|
11843
11935
|
for (const entry of entries) {
|
|
11844
11936
|
if (entry.error !== undefined) {
|
|
11845
|
-
//
|
|
11846
|
-
//
|
|
11847
|
-
// duplicate provider work. It must not convert a failed bulk
|
|
11848
|
-
// persistence boundary into a successful map result: callers need
|
|
11849
|
-
// the loud failure so this run can be retried from the durable
|
|
11850
|
-
// receipts instead of continuing after an ambiguous bulk write.
|
|
11937
|
+
// A failed batch is not a cache entry. Surface it to every row so
|
|
11938
|
+
// a later run can make a fresh provider attempt.
|
|
11851
11939
|
for (const request of entry.request.memberRequests) {
|
|
11852
11940
|
await reject(request, entry.error);
|
|
11853
11941
|
}
|
|
11854
11942
|
throw entry.error;
|
|
11855
11943
|
}
|
|
11856
11944
|
if (!entry.result) {
|
|
11857
|
-
//
|
|
11858
|
-
//
|
|
11859
|
-
// the same cohort still reach their rows and a stale failure never
|
|
11860
|
-
// causes this Context to write a second terminal transition.
|
|
11945
|
+
// Successful split results must be present in the completed cache
|
|
11946
|
+
// before they are delivered to rows.
|
|
11861
11947
|
const durable = await store.read({
|
|
11862
11948
|
resultReceiptKeys: entry.request.memberRequests.map(
|
|
11863
11949
|
(request) => request.receiptKey!,
|
|
@@ -11878,8 +11964,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11878
11964
|
} else {
|
|
11879
11965
|
await reject(
|
|
11880
11966
|
request,
|
|
11881
|
-
|
|
11882
|
-
entry.error ??
|
|
11967
|
+
entry.error ??
|
|
11883
11968
|
new Error(
|
|
11884
11969
|
'Native Tool Call did not reach a terminal receipt.',
|
|
11885
11970
|
),
|
|
@@ -11918,7 +12003,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11918
12003
|
|
|
11919
12004
|
const toolSettlements = await Promise.allSettled(
|
|
11920
12005
|
[...byTool.entries()].map(async ([toolId, requests]) => {
|
|
11921
|
-
this.
|
|
12006
|
+
this.runtimeLog(
|
|
12007
|
+
`Executing tool batch ${toolId}: ${requests.length} calls`,
|
|
12008
|
+
);
|
|
11922
12009
|
const successfulLiveStepCallIds = new Set<string>();
|
|
11923
12010
|
|
|
11924
12011
|
const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
|
|
@@ -11973,7 +12060,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11973
12060
|
? undefined
|
|
11974
12061
|
: this.getCachedToolResult(toolId, req.cacheKey);
|
|
11975
12062
|
if (cached?.done) {
|
|
11976
|
-
this.
|
|
12063
|
+
this.runtimeLog(
|
|
12064
|
+
` Row ${req.rowId} ${toolId}: recovered from checkpoint`,
|
|
12065
|
+
);
|
|
11977
12066
|
const resolver = this.toolCallResolvers.get(req.callId);
|
|
11978
12067
|
if (resolver) {
|
|
11979
12068
|
resolver.resolve(cached.result);
|
|
@@ -12099,6 +12188,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12099
12188
|
const cohortRequests = pendingRequests.filter(
|
|
12100
12189
|
(request) =>
|
|
12101
12190
|
request.cacheable !== false &&
|
|
12191
|
+
request.requiresExecutionFence !== true &&
|
|
12102
12192
|
Boolean(request.receiptKey?.trim()),
|
|
12103
12193
|
);
|
|
12104
12194
|
if (cohortRequests.length > 0) {
|
|
@@ -12131,6 +12221,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12131
12221
|
const cohortRequests = pendingRequests.filter(
|
|
12132
12222
|
(request) =>
|
|
12133
12223
|
request.cacheable !== false &&
|
|
12224
|
+
request.requiresExecutionFence !== true &&
|
|
12134
12225
|
Boolean(request.receiptKey?.trim()),
|
|
12135
12226
|
);
|
|
12136
12227
|
if (cohortRequests.length > 0) {
|
|
@@ -12553,7 +12644,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12553
12644
|
),
|
|
12554
12645
|
) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
12555
12646
|
if (runtimeReceiptReadTraceEnabled) {
|
|
12556
|
-
this.
|
|
12647
|
+
this.runtimeLog(
|
|
12557
12648
|
`[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` +
|
|
12558
12649
|
`request_digests=${[...pendingWaits.keys()]
|
|
12559
12650
|
.map(runtimeReceiptKeyDigest)
|
|
@@ -12650,7 +12741,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12650
12741
|
: ['unknown'];
|
|
12651
12742
|
},
|
|
12652
12743
|
);
|
|
12653
|
-
this.
|
|
12744
|
+
this.runtimeLog(
|
|
12654
12745
|
`[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
|
|
12655
12746
|
`rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
|
|
12656
12747
|
);
|
|
@@ -12957,8 +13048,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12957
13048
|
: publicToolResponseForBatchedItem(
|
|
12958
13049
|
batchExecution.execution,
|
|
12959
13050
|
splitResults[index] ?? null,
|
|
12960
|
-
|
|
12961
|
-
RAW_V2_TOOL_RESPONSE_CONTRACT,
|
|
13051
|
+
true,
|
|
12962
13052
|
),
|
|
12963
13053
|
}),
|
|
12964
13054
|
),
|
|
@@ -13114,6 +13204,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13114
13204
|
...(request.receiptKey
|
|
13115
13205
|
? {
|
|
13116
13206
|
durableCallReceiptKey: request.receiptKey,
|
|
13207
|
+
completedResultCache: true,
|
|
13117
13208
|
executionAuthScopeDigest:
|
|
13118
13209
|
request.executionAuthScopeDigest,
|
|
13119
13210
|
providerIdempotencyReceiptKey:
|
|
@@ -13540,7 +13631,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13540
13631
|
);
|
|
13541
13632
|
}
|
|
13542
13633
|
if (runtimeReceiptReadTraceEnabled) {
|
|
13543
|
-
this.
|
|
13634
|
+
this.runtimeLog(
|
|
13544
13635
|
`[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
|
|
13545
13636
|
);
|
|
13546
13637
|
}
|
|
@@ -13576,6 +13667,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13576
13667
|
.catch(() => null);
|
|
13577
13668
|
const retrySafeTransientHttp =
|
|
13578
13669
|
retryPolicy?.retrySafeTransientHttp === true;
|
|
13670
|
+
const requiresExecutionFence =
|
|
13671
|
+
retryPolicy?.requiresExecutionFence === true;
|
|
13672
|
+
const fixtureReplaySafe = this.#options.integrationMode === 'fixture';
|
|
13673
|
+
const readOnlyQuery = isQueryResultDatasetReadRequest(
|
|
13674
|
+
toolId,
|
|
13675
|
+
toolInputSnapshot,
|
|
13676
|
+
);
|
|
13677
|
+
// A completed result cache eliminates repeated successful work; it
|
|
13678
|
+
// does not prove an ambiguous provider write can be replayed. Only
|
|
13679
|
+
// a read-only query or an action which explicitly declares transient
|
|
13680
|
+
// retries safe may issue another physical request after dispatch.
|
|
13681
|
+
const ambiguousTransportReplayAllowed =
|
|
13682
|
+
readOnlyQuery || retrySafeTransientHttp;
|
|
13579
13683
|
const invocationOrgId = this.#options.orgId?.trim() ?? '';
|
|
13580
13684
|
const durableCallReceiptKey =
|
|
13581
13685
|
options?.durableCallReceiptKey?.trim() ||
|
|
@@ -13590,26 +13694,25 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13590
13694
|
const executionAuthScopeDigest = options?.executionAuthScopeDigest
|
|
13591
13695
|
? options.executionAuthScopeDigest.trim() || null
|
|
13592
13696
|
: ((await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null);
|
|
13593
|
-
const
|
|
13594
|
-
|
|
13595
|
-
|
|
13596
|
-
|
|
13597
|
-
|
|
13598
|
-
|
|
13599
|
-
|
|
13600
|
-
|
|
13601
|
-
|
|
13602
|
-
};
|
|
13603
|
-
const providerIdempotencyKey =
|
|
13604
|
-
options?.providerIdempotencyKey?.trim() || durableCallReceiptKey;
|
|
13697
|
+
const executionRequestNonce = crypto.randomUUID();
|
|
13698
|
+
// This identifies one logical tool call to billing. Physical retry
|
|
13699
|
+
// count remains in request metadata, so a transport retry cannot
|
|
13700
|
+
// reserve or charge the same call twice.
|
|
13701
|
+
const deeplineRequestId = `ctx-tool-${stableDigest(executionRequestNonce).slice(0, 32)}`;
|
|
13702
|
+
// The provider operation has a semantic identity independent of
|
|
13703
|
+
// the response receipt. Its key is supplied by the caller that owns
|
|
13704
|
+
// that compatibility contract; never substitute request/billing
|
|
13705
|
+
// attribution for it.
|
|
13605
13706
|
const providerIdempotencyReceiptKey =
|
|
13606
|
-
options?.providerIdempotencyReceiptKey?.trim() ||
|
|
13607
|
-
|
|
13608
|
-
|
|
13609
|
-
|
|
13610
|
-
|
|
13611
|
-
|
|
13612
|
-
|
|
13707
|
+
options?.providerIdempotencyReceiptKey?.trim() || null;
|
|
13708
|
+
const providerExecutionFenceKey = requiresExecutionFence
|
|
13709
|
+
? options?.providerIdempotencyKey?.trim() || null
|
|
13710
|
+
: null;
|
|
13711
|
+
if (requiresExecutionFence && !providerExecutionFenceKey) {
|
|
13712
|
+
throw new Error(
|
|
13713
|
+
`Tool ${toolId} requires a provider execution fence but no provider operation identity was supplied.`,
|
|
13714
|
+
);
|
|
13715
|
+
}
|
|
13613
13716
|
// Every runtime provider call carries at least a tool-scoped
|
|
13614
13717
|
// attribution. Callers that know the execution location (row-scoped
|
|
13615
13718
|
// map cells, batched owners) supply the fuller scope.
|
|
@@ -13624,16 +13727,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13624
13727
|
...(requestsDurableInvocationFence
|
|
13625
13728
|
? { invocation_fence_version: 1 }
|
|
13626
13729
|
: {}),
|
|
13627
|
-
// Canonical provider-operation receipts validate this semantic
|
|
13628
|
-
// plan before replay. It is intentionally removed from the
|
|
13629
|
-
// legacy byte-receipt fingerprint so mixed runner artifacts
|
|
13630
|
-
// retain their existing provider-once behavior.
|
|
13631
|
-
...(requestsDurableInvocationFence
|
|
13632
|
-
? { canonical_operation: canonicalOperation }
|
|
13633
|
-
: {}),
|
|
13634
|
-
// The worker-owned receipt coordinates this physical durable
|
|
13635
|
-
// call. Provider idempotency is carried separately below so
|
|
13636
|
-
// exact-payload reuse never replaces batch receipt ownership.
|
|
13637
13730
|
...(durableCallReceiptKey
|
|
13638
13731
|
? {
|
|
13639
13732
|
durable_call_receipt_key: durableCallReceiptKey,
|
|
@@ -13645,10 +13738,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13645
13738
|
: {}),
|
|
13646
13739
|
}
|
|
13647
13740
|
: {}),
|
|
13648
|
-
...(
|
|
13741
|
+
...(providerExecutionFenceKey
|
|
13649
13742
|
? {
|
|
13650
|
-
|
|
13651
|
-
|
|
13743
|
+
provider_idempotency_key: providerExecutionFenceKey,
|
|
13744
|
+
...(providerIdempotencyReceiptKey
|
|
13745
|
+
? {
|
|
13746
|
+
provider_idempotency_receipt_key:
|
|
13747
|
+
providerIdempotencyReceiptKey,
|
|
13748
|
+
}
|
|
13749
|
+
: {}),
|
|
13652
13750
|
}
|
|
13653
13751
|
: {}),
|
|
13654
13752
|
...(options?.customerDbDataset
|
|
@@ -13671,9 +13769,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13671
13769
|
: {}),
|
|
13672
13770
|
}
|
|
13673
13771
|
: {}),
|
|
13674
|
-
...(providerIdempotencyKey
|
|
13675
|
-
? { provider_idempotency_key: providerIdempotencyKey }
|
|
13676
|
-
: {}),
|
|
13677
13772
|
...(playNodeScope
|
|
13678
13773
|
? { play_node_scope: playNodeScopeToWire(playNodeScope) }
|
|
13679
13774
|
: {}),
|
|
@@ -13690,10 +13785,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13690
13785
|
elapsedMs: number;
|
|
13691
13786
|
requestId: string | null;
|
|
13692
13787
|
aborted: boolean;
|
|
13788
|
+
/** The ingress rejected the request before any provider dispatch. */
|
|
13789
|
+
knownNotDispatched?: boolean;
|
|
13693
13790
|
}): Promise<void> => {
|
|
13694
13791
|
transportAttempt += 1;
|
|
13695
13792
|
const diagnostic = describeTransportError(input.error);
|
|
13696
|
-
this.
|
|
13793
|
+
this.runtimeLog(
|
|
13697
13794
|
`[runtime.transport_failure] ${JSON.stringify({
|
|
13698
13795
|
tool_id: toolId,
|
|
13699
13796
|
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
@@ -13705,7 +13802,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13705
13802
|
error: diagnostic,
|
|
13706
13803
|
})}`,
|
|
13707
13804
|
);
|
|
13708
|
-
if (
|
|
13805
|
+
if (
|
|
13806
|
+
transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS &&
|
|
13807
|
+
(input.knownNotDispatched === true ||
|
|
13808
|
+
ambiguousTransportReplayAllowed ||
|
|
13809
|
+
fixtureReplaySafe)
|
|
13810
|
+
) {
|
|
13811
|
+
invocationAttempt += 1;
|
|
13709
13812
|
await this.governor.chargeBudget('retry');
|
|
13710
13813
|
const retryAfterMs =
|
|
13711
13814
|
TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
|
|
@@ -13713,7 +13816,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13713
13816
|
'plays.transport_retry_attempt',
|
|
13714
13817
|
transportAttempt,
|
|
13715
13818
|
);
|
|
13716
|
-
this.
|
|
13819
|
+
this.runtimeLog(
|
|
13717
13820
|
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${diagnostic.message ?? 'unknown transport error'}`,
|
|
13718
13821
|
);
|
|
13719
13822
|
await options?.parkProviderCall?.();
|
|
@@ -13770,7 +13873,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13770
13873
|
const ownershipStartedAt = Date.now();
|
|
13771
13874
|
await options?.beforeProviderCall?.();
|
|
13772
13875
|
if (runtimeReceiptReadTraceEnabled) {
|
|
13773
|
-
this.
|
|
13876
|
+
this.runtimeLog(
|
|
13774
13877
|
`[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
|
|
13775
13878
|
);
|
|
13776
13879
|
}
|
|
@@ -13795,7 +13898,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13795
13898
|
abortController?.abort(error);
|
|
13796
13899
|
},
|
|
13797
13900
|
onTransientFailure: (error) => {
|
|
13798
|
-
this.
|
|
13901
|
+
this.runtimeLog(
|
|
13799
13902
|
`Tool ${toolId} receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
|
|
13800
13903
|
);
|
|
13801
13904
|
},
|
|
@@ -13874,7 +13977,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13874
13977
|
signal: abortController?.signal,
|
|
13875
13978
|
onSelected: (value) => {
|
|
13876
13979
|
selected = value;
|
|
13877
|
-
this.
|
|
13980
|
+
this.runtimeLog(
|
|
13878
13981
|
`[fixture.response_delay] ${JSON.stringify({
|
|
13879
13982
|
outcome: 'scheduled',
|
|
13880
13983
|
provider: canonicalProvider,
|
|
@@ -13889,7 +13992,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13889
13992
|
},
|
|
13890
13993
|
});
|
|
13891
13994
|
} catch (error) {
|
|
13892
|
-
this.
|
|
13995
|
+
this.runtimeLog(
|
|
13893
13996
|
`[fixture.response_delay] ${JSON.stringify({
|
|
13894
13997
|
outcome: 'aborted',
|
|
13895
13998
|
provider: canonicalProvider,
|
|
@@ -13904,7 +14007,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13904
14007
|
);
|
|
13905
14008
|
throw error;
|
|
13906
14009
|
}
|
|
13907
|
-
this.
|
|
14010
|
+
this.runtimeLog(
|
|
13908
14011
|
`[fixture.response_delay] ${JSON.stringify({
|
|
13909
14012
|
outcome: 'completed',
|
|
13910
14013
|
provider: canonicalProvider,
|
|
@@ -13934,17 +14037,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13934
14037
|
'Content-Type': 'application/json',
|
|
13935
14038
|
Authorization: `Bearer ${this.#options.executorToken}`,
|
|
13936
14039
|
[EXECUTE_RESPONSE_CONTRACT_HEADER]:
|
|
13937
|
-
|
|
14040
|
+
RAW_V2_TOOL_RESPONSE_CONTRACT,
|
|
13938
14041
|
[EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
|
|
13939
14042
|
[EXECUTE_TOOL_METADATA_HEADER]: 'true',
|
|
13940
14043
|
[TOOL_EXECUTION_ERROR_SCHEMA_HEADER]: String(
|
|
13941
14044
|
toolErrorSchemaVersion,
|
|
13942
14045
|
),
|
|
13943
14046
|
'x-deepline-request-id': deeplineRequestId,
|
|
13944
|
-
...(
|
|
14047
|
+
...(providerExecutionFenceKey
|
|
13945
14048
|
? {
|
|
13946
14049
|
'x-deepline-idempotency-key':
|
|
13947
|
-
|
|
14050
|
+
providerExecutionFenceKey,
|
|
13948
14051
|
}
|
|
13949
14052
|
: {}),
|
|
13950
14053
|
...protectionHeaders,
|
|
@@ -13994,7 +14097,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13994
14097
|
runtimeReceiptReadTraceEnabled &&
|
|
13995
14098
|
integrationFetchStartedAt !== null
|
|
13996
14099
|
) {
|
|
13997
|
-
this.
|
|
14100
|
+
this.runtimeLog(
|
|
13998
14101
|
`[perf] tool call id=${toolId} phase=integration_fetch_body elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
|
|
13999
14102
|
);
|
|
14000
14103
|
}
|
|
@@ -14046,44 +14149,19 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14046
14149
|
(response === null ||
|
|
14047
14150
|
error instanceof ToolExecuteResponseBodyTransportError ||
|
|
14048
14151
|
error instanceof ToolExecuteInvalidJsonError);
|
|
14049
|
-
const hasDurableInvocationIdentity = Boolean(
|
|
14050
|
-
durableCallReceiptKey &&
|
|
14051
|
-
providerIdempotencyKey &&
|
|
14052
|
-
executionAuthScopeDigest,
|
|
14053
|
-
);
|
|
14054
14152
|
// A post-header body deadline aborts the fetch controller solely
|
|
14055
14153
|
// to stop consuming that response. It must not also cancel the
|
|
14056
14154
|
// independent gateway health check that establishes whether the
|
|
14057
14155
|
// committed response can be replayed safely. Preserve genuine
|
|
14058
14156
|
// caller/runtime cancellation by dropping the signal only when
|
|
14059
14157
|
// this exact body deadline caused the abort.
|
|
14060
|
-
const durableFenceVerificationSignal =
|
|
14061
|
-
abortController?.signal.aborted === true &&
|
|
14062
|
-
abortController.signal.reason instanceof
|
|
14063
|
-
ToolExecuteResponseBodyTimeoutError
|
|
14064
|
-
? undefined
|
|
14065
|
-
: abortController?.signal;
|
|
14066
|
-
const durableInvocationFenceVerified =
|
|
14067
|
-
hasDurableInvocationIdentity &&
|
|
14068
|
-
requestsDurableInvocationFence &&
|
|
14069
|
-
(this.#options.durableInvocationFence === true ||
|
|
14070
|
-
(await this.#options
|
|
14071
|
-
.verifyDurableInvocationFence?.(
|
|
14072
|
-
durableFenceVerificationSignal,
|
|
14073
|
-
)
|
|
14074
|
-
.catch(() => false)) === true);
|
|
14075
|
-
// Fixture execution has no provider-side effect. Replaying the
|
|
14076
|
-
// same stable request after our own transport fails is always
|
|
14077
|
-
// safe, even when this runner lacks a live-provider fence.
|
|
14078
|
-
const fixtureReplaySafe =
|
|
14079
|
-
this.#options.integrationMode === 'fixture';
|
|
14080
14158
|
const transportReplaySafe =
|
|
14081
14159
|
!ambiguousDispatchedFailure ||
|
|
14082
|
-
|
|
14160
|
+
ambiguousTransportReplayAllowed ||
|
|
14083
14161
|
fixtureReplaySafe;
|
|
14084
14162
|
if (!transportReplaySafe) {
|
|
14085
14163
|
const diagnostic = describeTransportError(transportError);
|
|
14086
|
-
this.
|
|
14164
|
+
this.runtimeLog(
|
|
14087
14165
|
`[runtime.transport_failure] ${JSON.stringify({
|
|
14088
14166
|
tool_id: toolId,
|
|
14089
14167
|
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
@@ -14172,6 +14250,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14172
14250
|
elapsedMs: providerCallElapsedMs ?? 0,
|
|
14173
14251
|
requestId: deeplineRequestId,
|
|
14174
14252
|
aborted: false,
|
|
14253
|
+
knownNotDispatched: true,
|
|
14175
14254
|
});
|
|
14176
14255
|
continue;
|
|
14177
14256
|
}
|
|
@@ -14209,13 +14288,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14209
14288
|
failure.backpressureDelayMs,
|
|
14210
14289
|
);
|
|
14211
14290
|
}
|
|
14212
|
-
if (
|
|
14213
|
-
|
|
14214
|
-
|
|
14215
|
-
|
|
14216
|
-
|
|
14217
|
-
|
|
14218
|
-
|
|
14291
|
+
if (
|
|
14292
|
+
failure.shouldRetry &&
|
|
14293
|
+
(ambiguousTransportReplayAllowed ||
|
|
14294
|
+
retrySafeTransientHttp ||
|
|
14295
|
+
fixtureReplaySafe)
|
|
14296
|
+
) {
|
|
14297
|
+
invocationAttempt += 1;
|
|
14219
14298
|
if (failure.chargeRetryBudget) {
|
|
14220
14299
|
await this.governor.chargeBudget('retry');
|
|
14221
14300
|
}
|
|
@@ -14230,8 +14309,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14230
14309
|
`plays.${retryAttributePrefix}_attempt`,
|
|
14231
14310
|
httpFailureAttempt,
|
|
14232
14311
|
);
|
|
14233
|
-
this.
|
|
14312
|
+
this.runtimeLog(
|
|
14234
14313
|
`Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`,
|
|
14314
|
+
{ level: 'warn' },
|
|
14235
14315
|
);
|
|
14236
14316
|
retryActivityEmitted = true;
|
|
14237
14317
|
this.emitExecutionEvent({
|
|
@@ -14275,7 +14355,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14275
14355
|
});
|
|
14276
14356
|
continue;
|
|
14277
14357
|
}
|
|
14278
|
-
this.
|
|
14358
|
+
this.runtimeLog(failure.error.message, { level: 'error' });
|
|
14279
14359
|
throw failure.error;
|
|
14280
14360
|
}
|
|
14281
14361
|
|