effect-agent 0.1.0-beta.97 → 0.1.0-beta.98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{AgentRuntime-B64362Id.d.mts → AgentRuntime-BMzrJos6.d.mts} +2 -2
- package/dist/{AgentRuntime-Cdvqw7l_.mjs → AgentRuntime-BWOnNc0d.mjs} +78 -42
- package/dist/AgentRuntime-BWOnNc0d.mjs.map +1 -0
- package/dist/{DurableAgentRuntime-DbUvZHDZ.mjs → DurableAgentRuntime-gF4YiEj2.mjs} +471 -252
- package/dist/DurableAgentRuntime-gF4YiEj2.mjs.map +1 -0
- package/dist/{DurableFailpoint-DQJet828.d.mts → DurableFailpoint-mx3-nsxU.d.mts} +3 -3
- package/dist/{EventSource-D6PAAEyK.d.mts → EventSource-BZ73J0s5.d.mts} +3 -3
- package/dist/{PreparedInputAdmission-omMbwd--.d.mts → PreparedInputAdmission-BkeI-Xvr.d.mts} +4 -4
- package/dist/{Records-CV49DgQK.d.mts → Records-BIiRLqlz.d.mts} +92 -128
- package/dist/{Records-BAutSSnJ.mjs → Records-DR1xtbXY.mjs} +35 -8
- package/dist/Records-DR1xtbXY.mjs.map +1 -0
- package/dist/{RunJournal-DRRRZmG7.mjs → RunJournal-DNs3PM4K.mjs} +80 -23
- package/dist/RunJournal-DNs3PM4K.mjs.map +1 -0
- package/dist/{RunOptions-Clj8cLsp.d.mts → RunOptions-Bruk54SE.d.mts} +13 -4
- package/dist/{Schedule-DePbGfgd.d.mts → Schedule-BNFlluAl.d.mts} +2 -2
- package/dist/{Scheduling-C3QpYn0W.mjs → Scheduling-HFH0pST1.mjs} +2 -2
- package/dist/{Scheduling-C3QpYn0W.mjs.map → Scheduling-HFH0pST1.mjs.map} +1 -1
- package/dist/{Subagent-Cbfzylz1.mjs → Subagent-C2s8PFrd.mjs} +2 -2
- package/dist/{Subagent-Cbfzylz1.mjs.map → Subagent-C2s8PFrd.mjs.map} +1 -1
- package/dist/{Subagent-Bm1dRecN.d.mts → Subagent-m7ywTCe4.d.mts} +3 -3
- package/dist/{Subscription-BR_8Cxh3.d.mts → Subscription-BCQ3L9Zp.d.mts} +3 -3
- package/dist/{SubscriptionInput-B95sQbuk.d.mts → SubscriptionInput-Cqu4_RW3.d.mts} +3 -3
- package/dist/{ToolReconciler-DQVaJfpE.d.mts → ToolReconciler-BmVRAHKs.d.mts} +8 -4
- package/dist/agent-registration-cXpB8cBU.mjs +199 -0
- package/dist/agent-registration-cXpB8cBU.mjs.map +1 -0
- package/dist/capabilities/Commands.d.mts +1 -1
- package/dist/capabilities/RunHooks.d.mts +1 -1
- package/dist/capabilities/Subagent.d.mts +1 -1
- package/dist/capabilities/Subagent.mjs +1 -1
- package/dist/durable/ActivityStore.d.mts +1 -1
- package/dist/durable/ActivityStore.mjs +1 -1
- package/dist/durable/Admin.d.mts +1 -1
- package/dist/durable/Admin.mjs +12 -10
- package/dist/durable/Admin.mjs.map +1 -1
- package/dist/durable/AgentRegistration.d.mts +2 -2
- package/dist/durable/AgentRegistration.mjs +4 -8
- package/dist/durable/AgentRegistration.mjs.map +1 -1
- package/dist/durable/Certification.d.mts +2 -2
- package/dist/durable/CommittedActivity.d.mts +1 -1
- package/dist/durable/CommittedActivity.mjs +1 -1
- package/dist/durable/Digest.d.mts +1 -1
- package/dist/durable/Digest.mjs +1 -1
- package/dist/durable/DurableAgentRuntime.d.mts +1 -1
- package/dist/durable/DurableAgentRuntime.mjs +1 -1
- package/dist/durable/DurableFailpoint.d.mts +1 -1
- package/dist/durable/DurableFailpoint.mjs +2 -0
- package/dist/durable/DurableFailpoint.mjs.map +1 -1
- package/dist/durable/DurableFailpointTestControl.d.mts +1 -1
- package/dist/durable/EventSource.d.mts +1 -1
- package/dist/durable/EventSource.mjs +1 -1
- package/dist/durable/GitHubWorkflowSource.d.mts +3 -3
- package/dist/durable/GitHubWorkflowSource.mjs +1 -1
- package/dist/durable/MessageDelivery.d.mts +3 -3
- package/dist/durable/MessageDelivery.mjs +1 -1
- package/dist/durable/MessageDeliveryStoreConformance.d.mts +39 -39
- package/dist/durable/MessageDeliveryStoreConformance.mjs +1 -1
- package/dist/durable/PersistentHistory.d.mts +1 -1
- package/dist/durable/PersistentHistory.mjs +2 -2
- package/dist/durable/PreparedInputAdmission.d.mts +1 -1
- package/dist/durable/Records.d.mts +2 -2
- package/dist/durable/Records.mjs +2 -2
- package/dist/durable/Recovery.d.mts +1 -1
- package/dist/durable/Recovery.mjs +9 -9
- package/dist/durable/Recovery.mjs.map +1 -1
- package/dist/durable/RunJournal.d.mts +2 -2
- package/dist/durable/RunJournal.mjs +2 -2
- package/dist/durable/Schedule.d.mts +1 -1
- package/dist/durable/Schedule.mjs +1 -1
- package/dist/durable/ScheduleStoreConformance.d.mts +1 -1
- package/dist/durable/ScheduleStoreConformance.mjs +1 -1
- package/dist/durable/ScheduleTransition.d.mts +1 -1
- package/dist/durable/ScheduleTransition.mjs +1 -1
- package/dist/durable/Scheduling.d.mts +2 -2
- package/dist/durable/Scheduling.mjs +1 -1
- package/dist/durable/SqlStorageV2Upgrade.d.mts +2 -2
- package/dist/durable/SqlStorageV2Upgrade.mjs +1 -1
- package/dist/durable/SqlSubscriptionStore.d.mts +2 -2
- package/dist/durable/SqlSubscriptionStore.mjs +1 -1
- package/dist/durable/SubmissionLedger.d.mts +1 -1
- package/dist/durable/SubmissionLedger.mjs +14 -13
- package/dist/durable/SubmissionLedger.mjs.map +1 -1
- package/dist/durable/SubmissionLedgerConformance.d.mts +1 -1
- package/dist/durable/SubmissionLedgerConformance.mjs +9 -2
- package/dist/durable/SubmissionLedgerConformance.mjs.map +1 -1
- package/dist/durable/SubmissionStatus.d.mts +1 -1
- package/dist/durable/Subscription.d.mts +1 -1
- package/dist/durable/Subscription.mjs +1 -1
- package/dist/durable/SubscriptionInput.d.mts +1 -1
- package/dist/durable/SubscriptionInput.mjs +2 -2
- package/dist/durable/SubscriptionStoreConformance.d.mts +1 -1
- package/dist/durable/SubscriptionStoreConformance.mjs +1 -1
- package/dist/durable/SubscriptionTools.d.mts +3 -3
- package/dist/durable/SubscriptionTransition.d.mts +1 -1
- package/dist/durable/SubscriptionTransition.mjs +1 -1
- package/dist/durable/Subscriptions.d.mts +5 -5
- package/dist/durable/Subscriptions.mjs +1 -1
- package/dist/durable/ThreadContextHistory.d.mts +1 -1
- package/dist/durable/ThreadContextHistory.mjs +1 -1
- package/dist/durable/ThreadContextHistoryProjection.d.mts +1 -1
- package/dist/durable/ThreadContextHistoryProjection.mjs +1 -1
- package/dist/durable/ThreadInvariants.d.mts +4 -2
- package/dist/durable/ThreadInvariants.mjs +49 -3
- package/dist/durable/ThreadInvariants.mjs.map +1 -1
- package/dist/durable/ThreadProjection.d.mts +1 -1
- package/dist/durable/ThreadProjection.mjs +1 -1
- package/dist/durable/ThreadProjectionMaintenance.d.mts +1 -1
- package/dist/durable/ThreadStore.d.mts +1 -1
- package/dist/durable/ThreadStore.mjs +1 -1
- package/dist/durable/ThreadStoreConformance.d.mts +1 -1
- package/dist/durable/ThreadStoreConformance.mjs +1 -1
- package/dist/durable/ToolReconciler.d.mts +1 -1
- package/dist/durable/ToolReconciler.mjs +9 -5
- package/dist/durable/ToolReconciler.mjs.map +1 -1
- package/dist/durable/WorkerHost.d.mts +1 -1
- package/dist/engine/AgentRuntime.d.mts +1 -1
- package/dist/engine/AgentRuntime.mjs +1 -1
- package/dist/engine/RunOptions.d.mts +1 -1
- package/dist/engine/RunOptions.mjs.map +1 -1
- package/dist/index.d.mts +11 -11
- package/dist/index.mjs +6 -6
- package/package.json +1 -1
- package/src/durable/Admin.ts +20 -11
- package/src/durable/AgentRegistration.ts +1 -5
- package/src/durable/DurableAgentRuntime.ts +936 -464
- package/src/durable/DurableFailpoint.ts +2 -0
- package/src/durable/Records.ts +30 -5
- package/src/durable/Recovery.ts +8 -8
- package/src/durable/RunJournal.ts +128 -51
- package/src/durable/SubmissionLedger.ts +13 -12
- package/src/durable/SubmissionLedgerConformance.ts +13 -2
- package/src/durable/ThreadInvariants.ts +70 -5
- package/src/durable/ToolReconciler.ts +7 -3
- package/src/durable/internal/agent-registration.ts +65 -414
- package/src/durable/internal/messaging-host.ts +0 -2
- package/src/durable/internal/worker-host.ts +34 -96
- package/src/engine/RunOptions.ts +13 -3
- package/src/engine/internal/agent-runtime.ts +160 -106
- package/dist/AgentRuntime-Cdvqw7l_.mjs.map +0 -1
- package/dist/DurableAgentRuntime-DbUvZHDZ.mjs.map +0 -1
- package/dist/Records-BAutSSnJ.mjs.map +0 -1
- package/dist/RunJournal-DRRRZmG7.mjs.map +0 -1
- package/dist/agent-registration-CXGDwvJH.mjs +0 -379
- package/dist/agent-registration-CXGDwvJH.mjs.map +0 -1
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
getToolExecutionKind,
|
|
58
58
|
SubagentParentLink,
|
|
59
59
|
} from "../core/SubagentContract.ts";
|
|
60
|
-
import { Selection,
|
|
60
|
+
import { Selection, Snapshot } from "../core/ToolExposure.ts";
|
|
61
61
|
import type { ToolParameterRejection } from "../core/ToolResult.ts";
|
|
62
62
|
import {
|
|
63
63
|
type ModelCallUsage,
|
|
@@ -84,7 +84,8 @@ import {
|
|
|
84
84
|
type RuntimeBinding,
|
|
85
85
|
} from "../engine/AgentRuntime.ts";
|
|
86
86
|
import { ContextCompactor, CompactionError } from "../engine/ContextCompactor.ts";
|
|
87
|
-
import {
|
|
87
|
+
import { ContextRolloverTool } from "../engine/ContextWindow.ts";
|
|
88
|
+
import { getToolExecutionClass } from "../engine/DurableStep.ts";
|
|
88
89
|
import { MessagingHost } from "../engine/MessagingHost.ts";
|
|
89
90
|
import {
|
|
90
91
|
CurrentToolFailureObserver,
|
|
@@ -139,7 +140,7 @@ import {
|
|
|
139
140
|
BindingUnavailable,
|
|
140
141
|
compileRegistrations,
|
|
141
142
|
definitionDigestsEqual,
|
|
142
|
-
|
|
143
|
+
toolReplayContracts,
|
|
143
144
|
type AgentRegistration,
|
|
144
145
|
type DurableBindingFailure,
|
|
145
146
|
makeLegacyWorkerBinding,
|
|
@@ -197,6 +198,8 @@ import {
|
|
|
197
198
|
ToolApprovalDecided,
|
|
198
199
|
ToolApprovalRequested,
|
|
199
200
|
ToolCallPrepared,
|
|
201
|
+
ToolOperation,
|
|
202
|
+
ToolUnavailable,
|
|
200
203
|
RunPolicyUsageReserved,
|
|
201
204
|
ToolCallResolved,
|
|
202
205
|
ToolCallSettled,
|
|
@@ -234,6 +237,7 @@ import {
|
|
|
234
237
|
projectRunJournalStream,
|
|
235
238
|
type JournalBoundary,
|
|
236
239
|
runCompletedRecordId,
|
|
240
|
+
runCompletionDigest,
|
|
237
241
|
runIdForSubmission,
|
|
238
242
|
runStartedBatchId,
|
|
239
243
|
runStartedRecordId,
|
|
@@ -557,7 +561,7 @@ export class RecoveryReport extends Schema.Class<RecoveryReport>(
|
|
|
557
561
|
decision: RecoveryDecision,
|
|
558
562
|
/**
|
|
559
563
|
* `repaired` = executed; `deferred` = a claiming worker must finish it; `none` = settled;
|
|
560
|
-
* `unknown` =
|
|
564
|
+
* `unknown` = this Submission is parked on an Unknown Outcome awaiting the authorized
|
|
561
565
|
* DUR-017 resolution path — the settlement obligation stays visible while no worker permit
|
|
562
566
|
* is consumed (durability §16).
|
|
563
567
|
*/
|
|
@@ -713,6 +717,10 @@ export type DurableWorkerRequirements<
|
|
|
713
717
|
| AgentCompletionProjectionRequirements<AgentValue>;
|
|
714
718
|
|
|
715
719
|
export interface DurableRuntimeConfigOptions {
|
|
720
|
+
/**
|
|
721
|
+
* Deployment identity and the conservative operation version for direct execution without
|
|
722
|
+
* a registered binding. Reuse asserts unchanged handler, Step, and idempotency semantics.
|
|
723
|
+
*/
|
|
716
724
|
readonly deploymentId: DeploymentId;
|
|
717
725
|
readonly producerId: ProducerId;
|
|
718
726
|
/** `awaitSettlement` ledger re-check cadence when no wake arrives (default 500ms). */
|
|
@@ -954,31 +962,6 @@ const declaredApplicationCalls = Effect.fn("DurableAgentRuntime.declaredApplicat
|
|
|
954
962
|
declaredToolCalls(messages).pipe(Effect.map(({ application }) => application)),
|
|
955
963
|
);
|
|
956
964
|
|
|
957
|
-
/** Rebuild the exact JSON text that the engine decoded from one terminal no-Tool response. */
|
|
958
|
-
const terminalAssistantText = Effect.fn("DurableAgentRuntime.terminalAssistantText")(
|
|
959
|
-
(messages: PersistedJson): Effect.Effect<string, RunJournalError> =>
|
|
960
|
-
decodePrompt(messages).pipe(
|
|
961
|
-
Effect.mapError((cause) =>
|
|
962
|
-
RunJournalError.make({
|
|
963
|
-
message: "ModelResponseRecorded messages are not Schema-encoded Prompt messages",
|
|
964
|
-
cause,
|
|
965
|
-
}),
|
|
966
|
-
),
|
|
967
|
-
Effect.map((prompt) => {
|
|
968
|
-
let text = "";
|
|
969
|
-
|
|
970
|
-
for (const message of prompt.content) {
|
|
971
|
-
if (message.role !== "assistant") continue;
|
|
972
|
-
for (const part of message.content) {
|
|
973
|
-
if (part.type === "text") text += part.text;
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
return text;
|
|
978
|
-
}),
|
|
979
|
-
),
|
|
980
|
-
);
|
|
981
|
-
|
|
982
965
|
/**
|
|
983
966
|
* The declared-but-unsettled Tool batch of one Run's last committed Turn (§2.4 batch resume):
|
|
984
967
|
* every declared call in canonical encoded form plus the recorded results of the calls that
|
|
@@ -986,6 +969,7 @@ const terminalAssistantText = Effect.fn("DurableAgentRuntime.terminalAssistantTe
|
|
|
986
969
|
* path). `undefined` when the Run's journal ends at a complete Turn boundary.
|
|
987
970
|
*/
|
|
988
971
|
interface PendingToolBatch {
|
|
972
|
+
readonly toolOperations?: ReadonlyArray<ToolOperation> | undefined;
|
|
989
973
|
readonly toolParameterRejections?: ReadonlyArray<ToolParameterRejection> | undefined;
|
|
990
974
|
readonly toolExposure?: Snapshot | undefined;
|
|
991
975
|
readonly turn: number;
|
|
@@ -1025,7 +1009,7 @@ interface AttemptLineage {
|
|
|
1025
1009
|
interface OpenCallReview {
|
|
1026
1010
|
/** No proof either way: these calls must become Unknown Outcomes (never auto-replayed). */
|
|
1027
1011
|
readonly uncertain: Array<OpenToolCallEvidence>;
|
|
1028
|
-
/**
|
|
1012
|
+
/** Reconciliation or current execution support is unproven: defer without marking Unknown. */
|
|
1029
1013
|
readonly unproven: Array<OpenToolCallEvidence>;
|
|
1030
1014
|
/** Proven safe to re-execute (never-started / safe-retry / declared idempotent contract). */
|
|
1031
1015
|
readonly retryable: Array<OpenToolCallEvidence>;
|
|
@@ -1077,6 +1061,78 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1077
1061
|
const withCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>): Effect.Effect<A, E> =>
|
|
1078
1062
|
Effect.provideService(effect, Crypto.Crypto, crypto);
|
|
1079
1063
|
|
|
1064
|
+
const contractsFor = (definition: Agent.AnyDefinition) => {
|
|
1065
|
+
const registered = registeredBindings.filter((binding) => binding.agentId === definition.id);
|
|
1066
|
+
const binding = registered.length === 1 ? registered[0] : undefined;
|
|
1067
|
+
const current = binding?.definition === definition ? binding : undefined;
|
|
1068
|
+
const contracts = current?.digests.replay?.tools;
|
|
1069
|
+
|
|
1070
|
+
return contracts === undefined
|
|
1071
|
+
? withCrypto(
|
|
1072
|
+
toolReplayContracts(definition, undefined, current?.digests.tools ?? config.deploymentId),
|
|
1073
|
+
)
|
|
1074
|
+
: Effect.succeed(contracts);
|
|
1075
|
+
};
|
|
1076
|
+
|
|
1077
|
+
const currentOperationsFor = Effect.fnUntraced(function* (agentId: AgentId) {
|
|
1078
|
+
const registered = registeredBindings.filter((binding) => binding.agentId === agentId);
|
|
1079
|
+
const binding = registered.length === 1 ? registered[0] : undefined;
|
|
1080
|
+
|
|
1081
|
+
return binding === undefined
|
|
1082
|
+
? undefined
|
|
1083
|
+
: {
|
|
1084
|
+
definition: binding.definition,
|
|
1085
|
+
contracts: yield* contractsFor(binding.definition),
|
|
1086
|
+
};
|
|
1087
|
+
});
|
|
1088
|
+
|
|
1089
|
+
const operationsFor = (records: ReadonlyArray<CanonicalRecordEnvelope>, runId: RunId) => {
|
|
1090
|
+
const operations = new Map<string, ToolOperation>();
|
|
1091
|
+
|
|
1092
|
+
for (const {
|
|
1093
|
+
record: { payload },
|
|
1094
|
+
} of records) {
|
|
1095
|
+
if (payload._tag !== "ModelResponseRecorded" || payload.runId !== runId) continue;
|
|
1096
|
+
for (const operation of payload.toolOperations ?? [])
|
|
1097
|
+
operations.set(operation.toolCallId, operation);
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
return operations;
|
|
1101
|
+
};
|
|
1102
|
+
|
|
1103
|
+
const supportsOperation = (
|
|
1104
|
+
submission: SubmissionSnapshot,
|
|
1105
|
+
call: OpenToolCallEvidence,
|
|
1106
|
+
operation: ToolOperation | undefined,
|
|
1107
|
+
prepared: ToolCallPrepared | undefined,
|
|
1108
|
+
current:
|
|
1109
|
+
| {
|
|
1110
|
+
readonly definition: Agent.AnyDefinition;
|
|
1111
|
+
readonly contracts: Readonly<Record<string, Digest>>;
|
|
1112
|
+
}
|
|
1113
|
+
| undefined,
|
|
1114
|
+
): boolean => {
|
|
1115
|
+
const tool = current?.definition.toolkit.tools[call.toolName];
|
|
1116
|
+
|
|
1117
|
+
const replay =
|
|
1118
|
+
operation?.replay ?? prepared?.replay ?? submission.agentDigests.replay?.tools[call.toolName];
|
|
1119
|
+
|
|
1120
|
+
return (
|
|
1121
|
+
tool !== undefined &&
|
|
1122
|
+
replay !== undefined &&
|
|
1123
|
+
current?.contracts[call.toolName] === replay &&
|
|
1124
|
+
(operation === undefined ||
|
|
1125
|
+
(operation.toolName === call.toolName &&
|
|
1126
|
+
operation.executionClass === getToolExecutionClass(tool) &&
|
|
1127
|
+
operation.executionKind === getToolExecutionKind(tool.annotations))) &&
|
|
1128
|
+
(prepared === undefined ||
|
|
1129
|
+
(prepared.toolName === call.toolName &&
|
|
1130
|
+
(prepared.executionClass === undefined ||
|
|
1131
|
+
prepared.executionClass === getToolExecutionClass(tool)) &&
|
|
1132
|
+
(prepared.executionKind ?? "ordinary") === getToolExecutionKind(tool.annotations)))
|
|
1133
|
+
);
|
|
1134
|
+
};
|
|
1135
|
+
|
|
1080
1136
|
const hit = (
|
|
1081
1137
|
location: DurableRuntimeFailpointLocation,
|
|
1082
1138
|
): Effect.Effect<void, DurableRuntimeFailpointError> => failpoint.hit(location);
|
|
@@ -1245,6 +1301,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1245
1301
|
threadId: ThreadId,
|
|
1246
1302
|
throughSequence: CanonicalSequence,
|
|
1247
1303
|
submissionIds: ReadonlyArray<SubmissionId>,
|
|
1304
|
+
journalOwner?: RunId,
|
|
1248
1305
|
): Effect.fn.Return<
|
|
1249
1306
|
{
|
|
1250
1307
|
readonly canonical: Stream.Stream<
|
|
@@ -1283,7 +1340,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1283
1340
|
|
|
1284
1341
|
if (
|
|
1285
1342
|
!submissionIds.every((id) => state.submissionIds.includes(id)) ||
|
|
1286
|
-
state.seed.runId !== runIdForSubmission(state.submissionId)
|
|
1343
|
+
state.seed.runId !== runIdForSubmission(state.submissionId) ||
|
|
1344
|
+
(journalOwner !== undefined && state.seed.runId !== journalOwner)
|
|
1287
1345
|
)
|
|
1288
1346
|
return full;
|
|
1289
1347
|
|
|
@@ -1550,6 +1608,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1550
1608
|
const preparedTurns = new Set<number>();
|
|
1551
1609
|
const settledIds = new Set<string>();
|
|
1552
1610
|
const resolvedIds = new Set<string>();
|
|
1611
|
+
const unknownIds = new Set<string>();
|
|
1553
1612
|
const requested: Array<PendingApprovalEvidence> = [];
|
|
1554
1613
|
const decidedIds = new Set<string>();
|
|
1555
1614
|
|
|
@@ -1611,8 +1670,17 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1611
1670
|
if (payload.runId === runId) settledIds.add(payload.toolCallId);
|
|
1612
1671
|
break;
|
|
1613
1672
|
}
|
|
1673
|
+
case "ToolCallUnknown": {
|
|
1674
|
+
if (payload.runId === runId) unknownIds.add(payload.toolCallId);
|
|
1675
|
+
break;
|
|
1676
|
+
}
|
|
1614
1677
|
case "ToolCallResolved": {
|
|
1615
|
-
if (
|
|
1678
|
+
if (
|
|
1679
|
+
payload.runId === runId &&
|
|
1680
|
+
(payload.resolution === "completed-with-result" ||
|
|
1681
|
+
payload.resolution === "failed-with-error")
|
|
1682
|
+
)
|
|
1683
|
+
resolvedIds.add(payload.toolCallId);
|
|
1616
1684
|
break;
|
|
1617
1685
|
}
|
|
1618
1686
|
case "ToolApprovalRequested": {
|
|
@@ -1766,10 +1834,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1766
1834
|
preparedKinds.get(toolCallPreparedRecordId(runId, call.turn, call.toolCallId)) ===
|
|
1767
1835
|
"orchestration";
|
|
1768
1836
|
|
|
1769
|
-
const openWorkerCalls = allOpenCalls.filter(
|
|
1837
|
+
const openWorkerCalls = allOpenCalls.filter(
|
|
1838
|
+
(call) => isPreparedWorker(call) && !unknownIds.has(call.toolCallId),
|
|
1839
|
+
);
|
|
1770
1840
|
|
|
1771
1841
|
const openToolCalls = allOpenCalls.filter(
|
|
1772
|
-
(call) =>
|
|
1842
|
+
(call) =>
|
|
1843
|
+
unknownIds.has(call.toolCallId) || (!isPreparedDelegation(call) && !isPreparedWorker(call)),
|
|
1773
1844
|
);
|
|
1774
1845
|
|
|
1775
1846
|
let declaredPendingBatch: DeclaredPendingBatchEvidence | undefined;
|
|
@@ -1802,6 +1873,53 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1802
1873
|
});
|
|
1803
1874
|
});
|
|
1804
1875
|
|
|
1876
|
+
/** Verify immutable preparation against the original declaration before any recovery side effect. */
|
|
1877
|
+
const validatePreparedCall = Effect.fnUntraced(function* (
|
|
1878
|
+
prepared: ToolCallPrepared,
|
|
1879
|
+
declared: Effect.Success<ReturnType<typeof declaredToolCalls>>,
|
|
1880
|
+
operations: ReadonlyArray<ToolOperation> | undefined,
|
|
1881
|
+
) {
|
|
1882
|
+
const call = declared.application.find((call) => call.id === prepared.toolCallId);
|
|
1883
|
+
|
|
1884
|
+
if (call === undefined || prepared.toolName !== call.name)
|
|
1885
|
+
return yield* RunJournalError.make({
|
|
1886
|
+
message: `Prepared Tool ${prepared.toolCallId} differs from its original declaration`,
|
|
1887
|
+
});
|
|
1888
|
+
|
|
1889
|
+
const parameters = yield* decodePersisted(call.params).pipe(
|
|
1890
|
+
Effect.mapError((cause) =>
|
|
1891
|
+
RunJournalError.make({ message: "Invalid canonical Tool parameters", cause }),
|
|
1892
|
+
),
|
|
1893
|
+
);
|
|
1894
|
+
|
|
1895
|
+
const declarationDigest = yield* withCrypto(digestJson(parameters)).pipe(
|
|
1896
|
+
Effect.mapError((cause) =>
|
|
1897
|
+
RunJournalError.make({ message: "Cannot verify declared Tool parameters", cause }),
|
|
1898
|
+
),
|
|
1899
|
+
);
|
|
1900
|
+
|
|
1901
|
+
const preparedDigest = yield* withCrypto(digestJson(prepared.parameters)).pipe(
|
|
1902
|
+
Effect.mapError((cause) =>
|
|
1903
|
+
RunJournalError.make({ message: "Cannot verify prepared Tool parameters", cause }),
|
|
1904
|
+
),
|
|
1905
|
+
);
|
|
1906
|
+
|
|
1907
|
+
const operation = operations?.find((entry) => entry.toolCallId === prepared.toolCallId);
|
|
1908
|
+
|
|
1909
|
+
if (
|
|
1910
|
+
prepared.parametersDigest !== declarationDigest ||
|
|
1911
|
+
preparedDigest !== declarationDigest ||
|
|
1912
|
+
(operation !== undefined &&
|
|
1913
|
+
((prepared.executionClass !== undefined &&
|
|
1914
|
+
prepared.executionClass !== operation.executionClass) ||
|
|
1915
|
+
(prepared.executionKind ?? "ordinary") !== operation.executionKind ||
|
|
1916
|
+
(prepared.replay !== undefined && prepared.replay !== operation.replay)))
|
|
1917
|
+
)
|
|
1918
|
+
return yield* RunJournalError.make({
|
|
1919
|
+
message: `Prepared Tool ${prepared.toolCallId} differs from its original operation`,
|
|
1920
|
+
});
|
|
1921
|
+
});
|
|
1922
|
+
|
|
1805
1923
|
/**
|
|
1806
1924
|
* The declared-but-unsettled Tool batch of the Run's LAST committed Turn (§2.4). Keyed on
|
|
1807
1925
|
* SETTLED coverage deliberately: a call closed by a never-started/safe-retry `ToolCallResolved`
|
|
@@ -1817,6 +1935,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1817
1935
|
| {
|
|
1818
1936
|
readonly turn: number;
|
|
1819
1937
|
readonly messages: PersistedJson;
|
|
1938
|
+
readonly toolOperations?: ReadonlyArray<ToolOperation> | undefined;
|
|
1820
1939
|
readonly toolParameterRejections?: ReadonlyArray<ToolParameterRejection> | undefined;
|
|
1821
1940
|
readonly toolExposure?: Snapshot | undefined;
|
|
1822
1941
|
}
|
|
@@ -1825,6 +1944,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1825
1944
|
const settledByCallId = new Map<
|
|
1826
1945
|
string,
|
|
1827
1946
|
{
|
|
1947
|
+
readonly recordId?: RecordId;
|
|
1948
|
+
readonly toolName?: string;
|
|
1828
1949
|
readonly result: PersistedJson;
|
|
1829
1950
|
readonly isFailure: boolean;
|
|
1830
1951
|
readonly budgetRejected?: true;
|
|
@@ -1842,6 +1963,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1842
1963
|
turn: payload.turn,
|
|
1843
1964
|
messages: payload.messages,
|
|
1844
1965
|
toolParameterRejections: payload.toolParameterRejections,
|
|
1966
|
+
toolOperations: payload.toolOperations,
|
|
1845
1967
|
...(payload.toolExposure === undefined ? {} : { toolExposure: payload.toolExposure }),
|
|
1846
1968
|
};
|
|
1847
1969
|
}
|
|
@@ -1849,6 +1971,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1849
1971
|
}
|
|
1850
1972
|
if (payload._tag === "ToolCallSettled" && payload.runId === runId) {
|
|
1851
1973
|
settledByCallId.set(payload.toolCallId, {
|
|
1974
|
+
recordId: envelope.record.recordId,
|
|
1975
|
+
toolName: payload.toolName,
|
|
1852
1976
|
result: payload.result,
|
|
1853
1977
|
isFailure: payload.isFailure,
|
|
1854
1978
|
...(payload.budgetRejected === true ? { budgetRejected: true } : {}),
|
|
@@ -1862,6 +1986,39 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1862
1986
|
if (declared.application.length === 0) return undefined;
|
|
1863
1987
|
const calls = declared.all;
|
|
1864
1988
|
|
|
1989
|
+
if (lastResponse.toolOperations !== undefined) {
|
|
1990
|
+
const identities = new Set<string>();
|
|
1991
|
+
|
|
1992
|
+
for (const operation of lastResponse.toolOperations) {
|
|
1993
|
+
if (
|
|
1994
|
+
identities.has(operation.toolCallId) ||
|
|
1995
|
+
!declared.application.some(
|
|
1996
|
+
(call) => call.id === operation.toolCallId && call.name === operation.toolName,
|
|
1997
|
+
)
|
|
1998
|
+
)
|
|
1999
|
+
return yield* RunJournalError.make({
|
|
2000
|
+
message: "Operation evidence differs from the declared Tool batch",
|
|
2001
|
+
});
|
|
2002
|
+
identities.add(operation.toolCallId);
|
|
2003
|
+
}
|
|
2004
|
+
if (identities.size !== declared.application.length)
|
|
2005
|
+
return yield* RunJournalError.make({
|
|
2006
|
+
message: "Declared Tool batch has incomplete operation evidence",
|
|
2007
|
+
});
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
for (const {
|
|
2011
|
+
record: { payload },
|
|
2012
|
+
} of records) {
|
|
2013
|
+
if (
|
|
2014
|
+
payload._tag !== "ToolCallPrepared" ||
|
|
2015
|
+
payload.runId !== runId ||
|
|
2016
|
+
payload.turn !== lastResponse.turn
|
|
2017
|
+
)
|
|
2018
|
+
continue;
|
|
2019
|
+
yield* validatePreparedCall(payload, declared, lastResponse.toolOperations);
|
|
2020
|
+
}
|
|
2021
|
+
|
|
1865
2022
|
for (const part of declared.providerResults) {
|
|
1866
2023
|
const result = yield* decodePersisted(part.result).pipe(
|
|
1867
2024
|
Effect.mapError((cause) =>
|
|
@@ -1889,7 +2046,22 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1889
2046
|
});
|
|
1890
2047
|
}
|
|
1891
2048
|
if (recorded !== undefined) {
|
|
1892
|
-
|
|
2049
|
+
const { recordId, toolName, ...result } = recorded;
|
|
2050
|
+
|
|
2051
|
+
if (
|
|
2052
|
+
call.providerExecuted !== true &&
|
|
2053
|
+
(toolName !== call.name ||
|
|
2054
|
+
recordId !==
|
|
2055
|
+
toolCallSettledRecordId(
|
|
2056
|
+
runId,
|
|
2057
|
+
lastResponse.turn,
|
|
2058
|
+
Schema.decodeSync(ToolCallId)(call.id),
|
|
2059
|
+
))
|
|
2060
|
+
)
|
|
2061
|
+
return yield* RunJournalError.make({
|
|
2062
|
+
message: `Settled Tool ${call.id} differs from its original declaration`,
|
|
2063
|
+
});
|
|
2064
|
+
settled.push({ id: call.id, ...result });
|
|
1893
2065
|
}
|
|
1894
2066
|
}
|
|
1895
2067
|
|
|
@@ -1915,6 +2087,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1915
2087
|
responseRecordId: modelResponseRecordId(runId, lastResponse.turn),
|
|
1916
2088
|
messages: lastResponse.messages,
|
|
1917
2089
|
toolParameterRejections: lastResponse.toolParameterRejections,
|
|
2090
|
+
toolOperations: lastResponse.toolOperations,
|
|
1918
2091
|
...(lastResponse.toolExposure === undefined
|
|
1919
2092
|
? {}
|
|
1920
2093
|
: { toolExposure: lastResponse.toolExposure }),
|
|
@@ -2027,9 +2200,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2027
2200
|
});
|
|
2028
2201
|
|
|
2029
2202
|
/**
|
|
2030
|
-
*
|
|
2031
|
-
*
|
|
2032
|
-
*
|
|
2203
|
+
* Existing ownership-free settlement protocol for queued aborts and joined outcomes.
|
|
2204
|
+
* Its canonical settlement reservation authorizes the append; it never authorizes tool
|
|
2205
|
+
* execution, unknown-resolution appends, or repair audits beside a live writer.
|
|
2033
2206
|
*/
|
|
2034
2207
|
const attemptContextAtTail = Effect.fn("DurableAgentRuntime.attemptContextAtTail")(function* (
|
|
2035
2208
|
threadId: ThreadId,
|
|
@@ -2295,83 +2468,174 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2295
2468
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
2296
2469
|
openCalls: ReadonlyArray<OpenToolCallEvidence>,
|
|
2297
2470
|
knownIds: Set<string>,
|
|
2298
|
-
|
|
2471
|
+
current?: {
|
|
2472
|
+
readonly definition: Agent.AnyDefinition;
|
|
2473
|
+
readonly contracts: Readonly<Record<string, Digest>>;
|
|
2474
|
+
},
|
|
2299
2475
|
): Effect.fn.Return<OpenCallReview, DurableWorkerFailure> {
|
|
2300
2476
|
const submissionId = submission.submissionId;
|
|
2301
2477
|
const runId = runIdForSubmission(submissionId);
|
|
2478
|
+
|
|
2479
|
+
yield* pendingToolBatchFor(records, runId);
|
|
2480
|
+
const operations = operationsFor(records, runId);
|
|
2302
2481
|
const preparedByCallId = new Map<string, ToolCallPrepared>();
|
|
2303
2482
|
|
|
2304
|
-
for (const
|
|
2305
|
-
const payload =
|
|
2483
|
+
for (const { record } of records) {
|
|
2484
|
+
const payload = record.payload;
|
|
2306
2485
|
|
|
2307
|
-
if (
|
|
2486
|
+
if (
|
|
2487
|
+
payload._tag === "ToolCallPrepared" &&
|
|
2488
|
+
payload.runId === runId &&
|
|
2489
|
+
openCalls.some(
|
|
2490
|
+
(call) => call.toolCallId === payload.toolCallId && call.turn === payload.turn,
|
|
2491
|
+
)
|
|
2492
|
+
) {
|
|
2493
|
+
if (
|
|
2494
|
+
preparedByCallId.has(payload.toolCallId) ||
|
|
2495
|
+
record.recordId !== toolCallPreparedRecordId(runId, payload.turn, payload.toolCallId)
|
|
2496
|
+
)
|
|
2497
|
+
return yield* RunJournalError.make({
|
|
2498
|
+
message: `Prepared Tool ${payload.toolCallId} has inconsistent canonical identity`,
|
|
2499
|
+
});
|
|
2308
2500
|
preparedByCallId.set(payload.toolCallId, payload);
|
|
2309
2501
|
}
|
|
2310
2502
|
}
|
|
2311
|
-
|
|
2503
|
+
// Validate all open declarations before the first reconciler call. A missing response or
|
|
2504
|
+
// a damaged turn number must never bypass the original parameters/identity checks.
|
|
2505
|
+
for (const call of openCalls) {
|
|
2506
|
+
const prepared = preparedByCallId.get(call.toolCallId);
|
|
2312
2507
|
|
|
2313
|
-
|
|
2314
|
-
|
|
2508
|
+
const responses = records.filter(
|
|
2509
|
+
({ record }) =>
|
|
2510
|
+
record.payload._tag === "ModelResponseRecorded" &&
|
|
2511
|
+
record.payload.runId === runId &&
|
|
2512
|
+
record.payload.turn === call.turn,
|
|
2513
|
+
);
|
|
2514
|
+
|
|
2515
|
+
const response = responses[0]?.record;
|
|
2516
|
+
|
|
2517
|
+
if (
|
|
2518
|
+
prepared === undefined ||
|
|
2519
|
+
prepared.turn !== call.turn ||
|
|
2520
|
+
prepared.toolName !== call.toolName ||
|
|
2521
|
+
responses.length !== 1 ||
|
|
2522
|
+
response?.payload._tag !== "ModelResponseRecorded" ||
|
|
2523
|
+
response.recordId !== modelResponseRecordId(runId, call.turn) ||
|
|
2524
|
+
response.payload.turnId !== turnIdForRun(runId, call.turn)
|
|
2525
|
+
)
|
|
2526
|
+
return yield* RunJournalError.make({
|
|
2527
|
+
message: `Prepared Tool ${call.toolCallId} has no unique original response`,
|
|
2528
|
+
});
|
|
2529
|
+
if (
|
|
2530
|
+
(yield* withCrypto(digestJson(response.payload.messages))) !==
|
|
2531
|
+
response.payload.messagesDigest
|
|
2532
|
+
)
|
|
2533
|
+
return yield* RunJournalError.make({
|
|
2534
|
+
message: `Original response for Tool ${call.toolCallId} has an invalid digest`,
|
|
2535
|
+
});
|
|
2536
|
+
yield* validatePreparedCall(
|
|
2537
|
+
prepared,
|
|
2538
|
+
yield* declaredToolCalls(response.payload.messages),
|
|
2539
|
+
response.payload.toolOperations,
|
|
2540
|
+
);
|
|
2315
2541
|
}
|
|
2542
|
+
|
|
2543
|
+
const intents = new Map(
|
|
2544
|
+
snapshot.unknownResolutions.map((intent) => [intent.toolCallId, intent]),
|
|
2545
|
+
);
|
|
2546
|
+
|
|
2316
2547
|
const review: OpenCallReview = { uncertain: [], unproven: [], retryable: [], recovered: 0 };
|
|
2317
2548
|
let recovered = 0;
|
|
2318
2549
|
|
|
2319
2550
|
for (const call of openCalls) {
|
|
2551
|
+
const prepared = preparedByCallId.get(call.toolCallId);
|
|
2552
|
+
const operation = operations.get(call.toolCallId);
|
|
2553
|
+
|
|
2554
|
+
const supported = supportsOperation(submission, call, operation, prepared, current);
|
|
2555
|
+
|
|
2320
2556
|
const intent = intents.get(call.toolCallId);
|
|
2557
|
+
const author = intent?.author ?? RECONCILER_AUTHOR;
|
|
2558
|
+
|
|
2559
|
+
const reason =
|
|
2560
|
+
intent?.reason ?? "A registered reconciliation policy recovered the external outcome";
|
|
2561
|
+
|
|
2562
|
+
const neverStarted = Effect.gen(function* () {
|
|
2563
|
+
if (current === undefined) {
|
|
2564
|
+
// Low-level drivers supply their current Agent only when claiming. Missing metadata
|
|
2565
|
+
// cannot prove retirement or authorize execution; the claimant checks the contract.
|
|
2566
|
+
review.unproven.push(call);
|
|
2567
|
+
|
|
2568
|
+
return;
|
|
2569
|
+
}
|
|
2570
|
+
if (supported) {
|
|
2571
|
+
review.retryable.push(call);
|
|
2572
|
+
|
|
2573
|
+
return;
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2576
|
+
const unavailable = yield* Schema.encodeEffect(ToolUnavailable)(
|
|
2577
|
+
ToolUnavailable.make({
|
|
2578
|
+
toolName: call.toolName,
|
|
2579
|
+
execution: "not-executed",
|
|
2580
|
+
message:
|
|
2581
|
+
"This operation never started and its original implementation is unavailable. Continue with the current tools.",
|
|
2582
|
+
}),
|
|
2583
|
+
).pipe(Effect.flatMap(decodePersisted), Effect.orDie);
|
|
2584
|
+
|
|
2585
|
+
yield* hit("tools:before-unavailable-append");
|
|
2586
|
+
yield* appendClosedCall(ctx, submissionId, knownIds, call, {
|
|
2587
|
+
result: { value: unavailable, isFailure: true },
|
|
2588
|
+
resolution: "never-started",
|
|
2589
|
+
author,
|
|
2590
|
+
reason,
|
|
2591
|
+
});
|
|
2592
|
+
yield* hit("tools:after-unavailable-append");
|
|
2593
|
+
recovered += 1;
|
|
2594
|
+
});
|
|
2321
2595
|
|
|
2322
2596
|
if (intent !== undefined) {
|
|
2323
2597
|
switch (intent.resolution._tag) {
|
|
2324
|
-
case "AbortSubmission":
|
|
2325
|
-
// The paired abort intent settles the Submission; the SettleAborted executor
|
|
2326
|
-
// records the `ToolCallUnknown` audit — nothing resolves here (durability §13).
|
|
2598
|
+
case "AbortSubmission":
|
|
2327
2599
|
continue;
|
|
2328
|
-
}
|
|
2329
2600
|
case "CompletedWithResult": {
|
|
2330
2601
|
yield* appendClosedCall(ctx, submissionId, knownIds, call, {
|
|
2331
|
-
result: {
|
|
2332
|
-
value: intent.resolution.result,
|
|
2333
|
-
isFailure: intent.resolution.isFailure,
|
|
2334
|
-
},
|
|
2602
|
+
result: { value: intent.resolution.result, isFailure: intent.resolution.isFailure },
|
|
2335
2603
|
resolution: intent.resolution.isFailure
|
|
2336
2604
|
? "failed-with-error"
|
|
2337
2605
|
: "completed-with-result",
|
|
2338
|
-
author
|
|
2339
|
-
reason
|
|
2606
|
+
author,
|
|
2607
|
+
reason,
|
|
2340
2608
|
});
|
|
2341
2609
|
recovered += 1;
|
|
2342
2610
|
continue;
|
|
2343
2611
|
}
|
|
2344
2612
|
case "NeverHappened": {
|
|
2345
|
-
yield*
|
|
2346
|
-
resolution: "never-started",
|
|
2347
|
-
author: intent.author,
|
|
2348
|
-
reason: intent.reason,
|
|
2349
|
-
});
|
|
2350
|
-
review.retryable.push(call);
|
|
2613
|
+
yield* neverStarted;
|
|
2351
2614
|
continue;
|
|
2352
2615
|
}
|
|
2353
2616
|
case "SafeToRetry": {
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
});
|
|
2359
|
-
review.retryable.push(call);
|
|
2617
|
+
// Permission to repeat the original operation is not proof it never happened.
|
|
2618
|
+
if (current === undefined) review.unproven.push(call);
|
|
2619
|
+
else if (supported) review.retryable.push(call);
|
|
2620
|
+
else review.uncertain.push(call);
|
|
2360
2621
|
continue;
|
|
2361
2622
|
}
|
|
2362
2623
|
}
|
|
2363
2624
|
}
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2625
|
+
const tool = current?.definition.toolkit.tools[call.toolName];
|
|
2626
|
+
|
|
2627
|
+
if (
|
|
2628
|
+
current === undefined &&
|
|
2629
|
+
(operation?.executionClass ?? prepared?.executionClass) === "idempotent"
|
|
2630
|
+
) {
|
|
2631
|
+
review.unproven.push(call);
|
|
2632
|
+
continue;
|
|
2633
|
+
}
|
|
2634
|
+
if (supported && tool !== undefined && getToolExecutionClass(tool) === "idempotent") {
|
|
2367
2635
|
review.retryable.push(call);
|
|
2368
2636
|
continue;
|
|
2369
2637
|
}
|
|
2370
|
-
const prepared = preparedByCallId.get(call.toolCallId);
|
|
2371
|
-
|
|
2372
2638
|
if (prepared === undefined) {
|
|
2373
|
-
// Open evidence without its prepared record is unreadable state: stay blocked,
|
|
2374
|
-
// never guess (fail-closed).
|
|
2375
2639
|
review.unproven.push(call);
|
|
2376
2640
|
continue;
|
|
2377
2641
|
}
|
|
@@ -2387,6 +2651,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2387
2651
|
toolName: prepared.toolName,
|
|
2388
2652
|
parameters: prepared.parameters,
|
|
2389
2653
|
parametersDigest: prepared.parametersDigest,
|
|
2654
|
+
...(prepared.executionKind === undefined
|
|
2655
|
+
? {}
|
|
2656
|
+
: { executionKind: prepared.executionKind }),
|
|
2657
|
+
...(prepared.executionClass === undefined
|
|
2658
|
+
? {}
|
|
2659
|
+
: { executionClass: prepared.executionClass }),
|
|
2660
|
+
...(prepared.replay === undefined ? {} : { replay: prepared.replay }),
|
|
2390
2661
|
}),
|
|
2391
2662
|
)
|
|
2392
2663
|
.pipe(
|
|
@@ -2405,15 +2676,20 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2405
2676
|
yield* appendClosedCall(ctx, submissionId, knownIds, call, {
|
|
2406
2677
|
result: { value: decision.result, isFailure: decision.isFailure },
|
|
2407
2678
|
resolution: decision.isFailure ? "failed-with-error" : "completed-with-result",
|
|
2408
|
-
author
|
|
2409
|
-
reason
|
|
2679
|
+
author,
|
|
2680
|
+
reason,
|
|
2410
2681
|
});
|
|
2411
2682
|
recovered += 1;
|
|
2412
2683
|
continue;
|
|
2413
2684
|
}
|
|
2414
|
-
case "NeverStarted":
|
|
2685
|
+
case "NeverStarted": {
|
|
2686
|
+
yield* neverStarted;
|
|
2687
|
+
continue;
|
|
2688
|
+
}
|
|
2415
2689
|
case "SafeToRetry": {
|
|
2416
|
-
review.
|
|
2690
|
+
if (current === undefined) review.unproven.push(call);
|
|
2691
|
+
else if (supported) review.retryable.push(call);
|
|
2692
|
+
else review.uncertain.push(call);
|
|
2417
2693
|
continue;
|
|
2418
2694
|
}
|
|
2419
2695
|
case "Uncertain": {
|
|
@@ -3445,8 +3721,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3445
3721
|
* Join a settled child after parent abort or expiry without running application handlers.
|
|
3446
3722
|
* The canonical child outcome stays intact; the parent result records why it cannot continue.
|
|
3447
3723
|
*/
|
|
3448
|
-
const
|
|
3449
|
-
"DurableAgentRuntime.
|
|
3724
|
+
const joinSettledChildWithoutHandler = Effect.fn(
|
|
3725
|
+
"DurableAgentRuntime.joinSettledChildWithoutHandler",
|
|
3450
3726
|
)(function* (
|
|
3451
3727
|
ctx: AttemptAppendContext,
|
|
3452
3728
|
parent: SubmissionSnapshot,
|
|
@@ -3455,14 +3731,14 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3455
3731
|
reservation: ChildBudgetReservationSnapshot,
|
|
3456
3732
|
toolCallId: ToolCallId,
|
|
3457
3733
|
childSubmissionId: SubmissionId,
|
|
3458
|
-
reason: "aborted" | "duration",
|
|
3734
|
+
reason: "aborted" | "duration" | "unavailable",
|
|
3459
3735
|
): Effect.fn.Return<void, DurableWorkerFailure> {
|
|
3460
3736
|
const runId = runIdForSubmission(parent.submissionId);
|
|
3461
3737
|
const request = subagent.requested.get(toolCallId);
|
|
3462
3738
|
|
|
3463
3739
|
if (request === undefined) {
|
|
3464
3740
|
return yield* LedgerError.make({
|
|
3465
|
-
operation: "
|
|
3741
|
+
operation: "joinSettledChildWithoutHandler",
|
|
3466
3742
|
message: `Tool Call ${toolCallId} has an attached child but no canonical SubagentRequested record`,
|
|
3467
3743
|
});
|
|
3468
3744
|
}
|
|
@@ -3477,22 +3753,33 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3477
3753
|
|
|
3478
3754
|
if (verification._tag === "mismatch") {
|
|
3479
3755
|
return yield* LedgerError.make({
|
|
3480
|
-
operation: "
|
|
3756
|
+
operation: "joinSettledChildWithoutHandler",
|
|
3481
3757
|
message: `Child Settlement verification failed for Tool Call ${toolCallId}: ${verification.message}`,
|
|
3482
3758
|
});
|
|
3483
3759
|
}
|
|
3484
3760
|
const verified = verification.value;
|
|
3485
3761
|
|
|
3486
|
-
//
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3762
|
+
// The child's actual outcome and usage remain authoritative even when the parent can
|
|
3763
|
+
// no longer project it with the original delegation implementation.
|
|
3764
|
+
const boundedResult = yield* decodePersisted(
|
|
3765
|
+
reason === "unavailable"
|
|
3766
|
+
? {
|
|
3767
|
+
_tag: "ToolUnavailable",
|
|
3768
|
+
toolName: subagent.preparedNames.get(toolCallId) ?? request.delegationId,
|
|
3769
|
+
execution: "unavailable",
|
|
3770
|
+
message: `The original delegation implementation is unavailable. Its original child settled ${verified.outcome}; no replacement child was started.`,
|
|
3771
|
+
}
|
|
3772
|
+
: {
|
|
3773
|
+
errorTag:
|
|
3774
|
+
reason === "aborted" ? "SubagentParentAborted" : "SubagentParentDurationExceeded",
|
|
3775
|
+
message: boundedText(
|
|
3776
|
+
`The parent Submission ${reason === "aborted" ? "aborted" : "exceeded its duration"}; attached child ${childSubmissionId} settled ${verified.outcome}`,
|
|
3777
|
+
),
|
|
3778
|
+
},
|
|
3779
|
+
).pipe(Effect.orDie);
|
|
3493
3780
|
|
|
3494
3781
|
finalAccounting = yield* decodePersisted({
|
|
3495
|
-
basis: reason
|
|
3782
|
+
basis: `${reason}-conservative`,
|
|
3496
3783
|
allocation: reservation.allocation,
|
|
3497
3784
|
}).pipe(Effect.orDie);
|
|
3498
3785
|
const childRunId = runIdForSubmission(childSubmissionId);
|
|
@@ -3587,11 +3874,12 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3587
3874
|
});
|
|
3588
3875
|
|
|
3589
3876
|
/** Release unused reservations or repair and join existing children, never admit new work. */
|
|
3590
|
-
const
|
|
3877
|
+
const reconcileRetainedChildren = Effect.fn("DurableAgentRuntime.reconcileRetainedChildren")(
|
|
3591
3878
|
function* (
|
|
3592
3879
|
ctx: AttemptAppendContext,
|
|
3593
3880
|
parent: SubmissionSnapshot,
|
|
3594
3881
|
ownershipToken: OwnershipToken,
|
|
3882
|
+
unavailableCalls?: ReadonlySet<ToolCallId>,
|
|
3595
3883
|
) {
|
|
3596
3884
|
const snapshot = yield* ledger.loadRecoverySnapshot(
|
|
3597
3885
|
RecoverySnapshotRequest.make({ submissionId: parent.submissionId }),
|
|
@@ -3603,10 +3891,46 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3603
3891
|
const knownIds = knownRecordIdsOf(records);
|
|
3604
3892
|
const subagent = subagentRecordsOf(records, runId);
|
|
3605
3893
|
|
|
3894
|
+
const notExecuted = new Set<ToolCallId>();
|
|
3895
|
+
const waiting: Array<WaitingChild> = [];
|
|
3896
|
+
const indeterminate = new Set<ToolCallId>();
|
|
3897
|
+
|
|
3898
|
+
for (const toolCallId of unavailableCalls ?? []) {
|
|
3899
|
+
if (
|
|
3900
|
+
snapshot.childReservations.some(
|
|
3901
|
+
(reservation) => reservation.parentToolCallId === toolCallId,
|
|
3902
|
+
)
|
|
3903
|
+
)
|
|
3904
|
+
continue;
|
|
3905
|
+
if (subagent.requested.has(toolCallId) || subagent.started.has(toolCallId))
|
|
3906
|
+
return yield* LedgerError.make({
|
|
3907
|
+
operation: "reconcileRetainedChildren",
|
|
3908
|
+
message: "Child request has no original reservation",
|
|
3909
|
+
});
|
|
3910
|
+
notExecuted.add(toolCallId);
|
|
3911
|
+
}
|
|
3606
3912
|
for (const reservation of snapshot.childReservations) {
|
|
3607
3913
|
const toolCallId = reservation.parentToolCallId;
|
|
3608
3914
|
|
|
3609
|
-
if (
|
|
3915
|
+
if (unavailableCalls !== undefined && !unavailableCalls.has(toolCallId)) continue;
|
|
3916
|
+
const joined = subagent.joined.get(toolCallId);
|
|
3917
|
+
|
|
3918
|
+
if (joined !== undefined) {
|
|
3919
|
+
yield* applyReservationRelease(reservation.reservationId, joined.finalAccounting);
|
|
3920
|
+
continue;
|
|
3921
|
+
}
|
|
3922
|
+
if (reservation.status !== "reserved") {
|
|
3923
|
+
if (reservation.accounting !== undefined)
|
|
3924
|
+
yield* applyReservationRelease(reservation.reservationId, reservation.accounting);
|
|
3925
|
+
// An orphan release proves non-admission; it never hides an attached child.
|
|
3926
|
+
if (reservation.childSubmissionId !== undefined || subagent.started.has(toolCallId))
|
|
3927
|
+
return yield* LedgerError.make({
|
|
3928
|
+
operation: "reconcileRetainedChildren",
|
|
3929
|
+
message: "Unjoined child reservation was already released",
|
|
3930
|
+
});
|
|
3931
|
+
notExecuted.add(toolCallId);
|
|
3932
|
+
continue;
|
|
3933
|
+
}
|
|
3610
3934
|
let started = subagent.started.get(toolCallId);
|
|
3611
3935
|
const request = subagent.requested.get(toolCallId);
|
|
3612
3936
|
|
|
@@ -3616,6 +3940,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3616
3940
|
reservation.childSubmissionId === undefined
|
|
3617
3941
|
) {
|
|
3618
3942
|
yield* releaseOrphanReservation(reservation.reservationId);
|
|
3943
|
+
notExecuted.add(toolCallId);
|
|
3619
3944
|
continue;
|
|
3620
3945
|
}
|
|
3621
3946
|
const call = pending?.calls.find((candidate) => candidate.id === toolCallId);
|
|
@@ -3633,7 +3958,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3633
3958
|
request.turnId !== pending.turnId
|
|
3634
3959
|
) {
|
|
3635
3960
|
return yield* LedgerError.make({
|
|
3636
|
-
operation: "
|
|
3961
|
+
operation: "reconcileRetainedChildren",
|
|
3637
3962
|
message: `Tool Call ${toolCallId} has inconsistent canonical child request evidence`,
|
|
3638
3963
|
});
|
|
3639
3964
|
}
|
|
@@ -3646,22 +3971,37 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3646
3971
|
}),
|
|
3647
3972
|
);
|
|
3648
3973
|
|
|
3649
|
-
if (resolution._tag === "Indeterminate")
|
|
3974
|
+
if (resolution._tag === "Indeterminate") {
|
|
3975
|
+
indeterminate.add(toolCallId);
|
|
3976
|
+
continue;
|
|
3977
|
+
}
|
|
3650
3978
|
if (resolution._tag === "NotAdmitted") {
|
|
3651
3979
|
if (reservation.childSubmissionId !== undefined) {
|
|
3652
3980
|
return yield* LedgerError.make({
|
|
3653
|
-
operation: "
|
|
3981
|
+
operation: "reconcileRetainedChildren",
|
|
3654
3982
|
message: `Tool Call ${toolCallId} has an attachment but no admitted child`,
|
|
3655
3983
|
});
|
|
3656
3984
|
}
|
|
3657
|
-
|
|
3658
|
-
|
|
3985
|
+
if (unavailableCalls === undefined) {
|
|
3986
|
+
yield* releaseOrphanReservation(reservation.reservationId);
|
|
3987
|
+
notExecuted.add(toolCallId);
|
|
3988
|
+
continue;
|
|
3989
|
+
}
|
|
3659
3990
|
}
|
|
3660
|
-
// Finish only the already-admitted child's materialization and readiness. A second
|
|
3661
|
-
// authoritative lookup must still never turn a stale observation into a new admission.
|
|
3662
|
-
const admission = yield* establishChildFromRequest(parent, request, false);
|
|
3663
3991
|
|
|
3664
|
-
|
|
3992
|
+
// A canonical request already authorized this exact idempotent admission. Finish it
|
|
3993
|
+
// when its handler is unavailable: a stale admission can still arrive after the lookup,
|
|
3994
|
+
// so NotAdmitted cannot justify releasing its reservation. Expiry keeps admission closed.
|
|
3995
|
+
const admission = yield* establishChildFromRequest(
|
|
3996
|
+
parent,
|
|
3997
|
+
request,
|
|
3998
|
+
unavailableCalls !== undefined,
|
|
3999
|
+
);
|
|
4000
|
+
|
|
4001
|
+
if (admission._tag === "indeterminate") {
|
|
4002
|
+
indeterminate.add(toolCallId);
|
|
4003
|
+
continue;
|
|
4004
|
+
}
|
|
3665
4005
|
started = SubagentStarted.make({
|
|
3666
4006
|
runId,
|
|
3667
4007
|
toolCallId,
|
|
@@ -3700,7 +4040,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3700
4040
|
started.childRunId !== runIdForSubmission(started.childSubmissionId)
|
|
3701
4041
|
) {
|
|
3702
4042
|
return yield* LedgerError.make({
|
|
3703
|
-
operation: "
|
|
4043
|
+
operation: "reconcileRetainedChildren",
|
|
3704
4044
|
message: `Tool Call ${call.id} has inconsistent canonical child attachment evidence`,
|
|
3705
4045
|
});
|
|
3706
4046
|
}
|
|
@@ -3718,8 +4058,14 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3718
4058
|
conflictToLedgerError("attachChildToReservation"),
|
|
3719
4059
|
),
|
|
3720
4060
|
);
|
|
3721
|
-
if (child.value.state !== "settled")
|
|
3722
|
-
|
|
4061
|
+
if (child.value.state !== "settled") {
|
|
4062
|
+
waiting.push(
|
|
4063
|
+
WaitingChild.make({ toolCallId, childSubmissionId: started.childSubmissionId }),
|
|
4064
|
+
);
|
|
4065
|
+
yield* wake.notify(child.value.threadId);
|
|
4066
|
+
continue;
|
|
4067
|
+
}
|
|
4068
|
+
yield* joinSettledChildWithoutHandler(
|
|
3723
4069
|
ctx,
|
|
3724
4070
|
parent,
|
|
3725
4071
|
knownIds,
|
|
@@ -3727,9 +4073,11 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3727
4073
|
reservation,
|
|
3728
4074
|
started.toolCallId,
|
|
3729
4075
|
started.childSubmissionId,
|
|
3730
|
-
"duration",
|
|
4076
|
+
unavailableCalls === undefined ? "duration" : "unavailable",
|
|
3731
4077
|
);
|
|
3732
4078
|
}
|
|
4079
|
+
|
|
4080
|
+
return { notExecuted, waiting, indeterminate };
|
|
3733
4081
|
},
|
|
3734
4082
|
);
|
|
3735
4083
|
|
|
@@ -3819,7 +4167,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3819
4167
|
});
|
|
3820
4168
|
}
|
|
3821
4169
|
if (child.value.state === "settled") {
|
|
3822
|
-
yield*
|
|
4170
|
+
yield* joinSettledChildWithoutHandler(
|
|
3823
4171
|
ctx,
|
|
3824
4172
|
parent,
|
|
3825
4173
|
knownIds,
|
|
@@ -4316,22 +4664,60 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4316
4664
|
journalSeed,
|
|
4317
4665
|
);
|
|
4318
4666
|
|
|
4319
|
-
const
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4667
|
+
const currentContracts = yield* contractsFor(agent.definition);
|
|
4668
|
+
|
|
4669
|
+
const rolloverOperation =
|
|
4670
|
+
journal.pendingContextToolCallId === undefined
|
|
4671
|
+
? undefined
|
|
4672
|
+
: operationsFor(records, runId).get(journal.pendingContextToolCallId);
|
|
4673
|
+
|
|
4674
|
+
const rolloverTool =
|
|
4675
|
+
rolloverOperation === undefined
|
|
4676
|
+
? undefined
|
|
4677
|
+
: agent.definition.toolkit.tools[rolloverOperation.toolName];
|
|
4678
|
+
|
|
4679
|
+
// A current annotation cannot reinterpret a formerly ordinary settled result as control.
|
|
4680
|
+
const pendingContextToolCallId =
|
|
4681
|
+
rolloverOperation !== undefined &&
|
|
4682
|
+
rolloverTool !== undefined &&
|
|
4683
|
+
Context.get(rolloverTool.annotations, ContextRolloverTool) &&
|
|
4684
|
+
currentContracts[rolloverOperation.toolName] === rolloverOperation.replay
|
|
4685
|
+
? journal.pendingContextToolCallId
|
|
4686
|
+
: undefined;
|
|
4326
4687
|
|
|
4327
|
-
|
|
4688
|
+
const applicationCalls = pending?.calls.filter((call) => !call.providerExecuted);
|
|
4689
|
+
|
|
4690
|
+
// Only the original single-call action-completion case may project a reconciled receipt.
|
|
4691
|
+
// Other settled calls are historical outcomes, regardless of their current completion role.
|
|
4692
|
+
const completionCandidate =
|
|
4693
|
+
applicationCalls?.length === 1 &&
|
|
4694
|
+
agent.definition.completionFromTools?.some(
|
|
4695
|
+
(declaration) => declaration.tool === applicationCalls[0]?.name,
|
|
4696
|
+
) &&
|
|
4697
|
+
pending?.settled.some(
|
|
4698
|
+
(result) => result.id === applicationCalls[0]?.id && !result.isFailure,
|
|
4699
|
+
)
|
|
4700
|
+
? applicationCalls[0]
|
|
4701
|
+
: undefined;
|
|
4328
4702
|
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4703
|
+
const settledCompletion =
|
|
4704
|
+
pending !== undefined &&
|
|
4705
|
+
completionCandidate !== undefined &&
|
|
4706
|
+
supportsOperation(
|
|
4707
|
+
submission,
|
|
4708
|
+
OpenToolCallEvidence.make({
|
|
4709
|
+
toolCallId: Schema.decodeSync(ToolCallId)(completionCandidate.id),
|
|
4710
|
+
toolName: completionCandidate.name,
|
|
4711
|
+
turn: pending.turn,
|
|
4712
|
+
}),
|
|
4713
|
+
pending.toolOperations?.find(
|
|
4714
|
+
(operation) => operation.toolCallId === completionCandidate.id,
|
|
4715
|
+
),
|
|
4716
|
+
undefined,
|
|
4717
|
+
{ definition: agent.definition, contracts: currentContracts },
|
|
4718
|
+
)
|
|
4719
|
+
? Schema.decodeSync(ToolCallId)(completionCandidate.id)
|
|
4720
|
+
: undefined;
|
|
4335
4721
|
|
|
4336
4722
|
const knownIds = knownRecordIdsOf(records);
|
|
4337
4723
|
const stepOutputs = new Map<string, PersistedJson>();
|
|
@@ -4661,167 +5047,41 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4661
5047
|
const declared = yield* declaredToolCalls(response.messages);
|
|
4662
5048
|
const calls = declared.application;
|
|
4663
5049
|
|
|
4664
|
-
let expectedOutput: {
|
|
4665
|
-
readonly encoded: Schema.Json;
|
|
4666
|
-
readonly decoded: OutputSchema["Type"];
|
|
4667
|
-
};
|
|
4668
|
-
|
|
4669
5050
|
if (calls.length > 0) {
|
|
4670
|
-
const completion = agent.definition.completion;
|
|
4671
5051
|
const call = calls[0];
|
|
4672
5052
|
|
|
4673
|
-
const actionCompletion = agent.definition.completionFromTools?.find(
|
|
4674
|
-
(declaration) => declaration.tool === call?.name,
|
|
4675
|
-
);
|
|
4676
|
-
|
|
4677
5053
|
const settled =
|
|
4678
5054
|
call === undefined
|
|
4679
5055
|
? undefined
|
|
4680
5056
|
: records.find(
|
|
4681
|
-
(
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
5057
|
+
({ record }) =>
|
|
5058
|
+
record.payload._tag === "ToolCallSettled" &&
|
|
5059
|
+
record.payload.runId === runId &&
|
|
5060
|
+
record.payload.toolCallId === call.id,
|
|
4685
5061
|
)?.record.payload;
|
|
4686
5062
|
|
|
4687
5063
|
if (
|
|
4688
5064
|
calls.length !== 1 ||
|
|
4689
5065
|
call === undefined ||
|
|
4690
|
-
(call.name !== completion?.tool && actionCompletion === undefined) ||
|
|
4691
5066
|
settled?._tag !== "ToolCallSettled" ||
|
|
4692
5067
|
settled.toolName !== call.name ||
|
|
4693
5068
|
settled.isFailure
|
|
4694
5069
|
) {
|
|
4695
5070
|
return yield* RunJournalError.make({
|
|
4696
|
-
message: `Run ${runId} has a terminal completion marker without one successful declared
|
|
5071
|
+
message: `Run ${runId} has a terminal completion marker without one successful declared Tool result`,
|
|
4697
5072
|
});
|
|
4698
5073
|
}
|
|
4699
|
-
|
|
4700
|
-
const projected = yield* (
|
|
4701
|
-
actionCompletion !== undefined
|
|
4702
|
-
? AgentRuntime.projectCompletionFromToolOutput(
|
|
4703
|
-
agent,
|
|
4704
|
-
actionCompletion,
|
|
4705
|
-
call.params,
|
|
4706
|
-
settled.result,
|
|
4707
|
-
)
|
|
4708
|
-
: completion !== undefined
|
|
4709
|
-
? Effect.map(
|
|
4710
|
-
AgentRuntime.projectCompletionOutput(
|
|
4711
|
-
agent,
|
|
4712
|
-
completion,
|
|
4713
|
-
call.params,
|
|
4714
|
-
settled.result,
|
|
4715
|
-
),
|
|
4716
|
-
Option.some,
|
|
4717
|
-
)
|
|
4718
|
-
: Effect.succeed(Option.none())
|
|
4719
|
-
).pipe(
|
|
4720
|
-
Effect.mapError((cause) =>
|
|
4721
|
-
RunJournalError.make({
|
|
4722
|
-
message: `Run ${runId} completion Tool output failed recovery projection`,
|
|
4723
|
-
cause,
|
|
4724
|
-
}),
|
|
4725
|
-
),
|
|
4726
|
-
);
|
|
4727
|
-
|
|
4728
|
-
if (Option.isNone(projected)) {
|
|
4729
|
-
return yield* RunJournalError.make({
|
|
4730
|
-
message: `Run ${runId} completion Tool declined its recorded terminal output`,
|
|
4731
|
-
});
|
|
4732
|
-
}
|
|
4733
|
-
expectedOutput = projected.value;
|
|
4734
|
-
} else {
|
|
4735
|
-
const responseText = yield* terminalAssistantText(response.messages);
|
|
4736
|
-
|
|
4737
|
-
expectedOutput = yield* AgentRuntime.decodeFinalOutput(agent, responseText).pipe(
|
|
4738
|
-
Effect.mapError((cause) =>
|
|
4739
|
-
RunJournalError.make({
|
|
4740
|
-
message: `Run ${runId} final response output failed recovery decoding`,
|
|
4741
|
-
cause,
|
|
4742
|
-
}),
|
|
4743
|
-
),
|
|
4744
|
-
);
|
|
4745
|
-
}
|
|
4746
|
-
|
|
4747
|
-
const boundedExpectedOutput = yield* decodePersisted(expectedOutput.encoded).pipe(
|
|
4748
|
-
Effect.mapError((cause) =>
|
|
4749
|
-
RunJournalError.make({
|
|
4750
|
-
message: `Run ${runId} reconstructed output exceeds canonical persistence bounds`,
|
|
4751
|
-
cause,
|
|
4752
|
-
}),
|
|
4753
|
-
),
|
|
4754
|
-
);
|
|
4755
|
-
|
|
4756
|
-
const [expectedOutputDigest, recordedOutputDigest] = yield* Effect.all([
|
|
4757
|
-
withCrypto(digestJson(boundedExpectedOutput)),
|
|
4758
|
-
withCrypto(digestJson(recordedCompletion.output)),
|
|
4759
|
-
]).pipe(
|
|
4760
|
-
Effect.mapError((cause) =>
|
|
4761
|
-
RunJournalError.make({
|
|
4762
|
-
message: `Run ${runId} completion output comparison failed`,
|
|
4763
|
-
cause,
|
|
4764
|
-
}),
|
|
4765
|
-
),
|
|
4766
|
-
);
|
|
4767
|
-
|
|
4768
|
-
if (expectedOutputDigest !== recordedOutputDigest) {
|
|
4769
|
-
return yield* RunJournalError.make({
|
|
4770
|
-
message: `Run ${runId} terminal completion output disagrees with its canonical response or Tool result`,
|
|
4771
|
-
});
|
|
4772
5074
|
}
|
|
4773
|
-
|
|
4774
|
-
const expectedRunDisposition =
|
|
4775
|
-
recordedCompletion.finishReason === "budget-exhausted"
|
|
4776
|
-
? undefined
|
|
4777
|
-
: yield* AgentRuntime.encodeRunDisposition(agent, expectedOutput.decoded).pipe(
|
|
4778
|
-
Effect.mapError((cause) =>
|
|
4779
|
-
RunJournalError.make({
|
|
4780
|
-
message: `Run ${runId} run disposition failed recovery projection`,
|
|
4781
|
-
cause,
|
|
4782
|
-
}),
|
|
4783
|
-
),
|
|
4784
|
-
);
|
|
4785
|
-
|
|
4786
5075
|
if (
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
5076
|
+
recordedCompletion.resultDigest !== undefined &&
|
|
5077
|
+
recordedCompletion.resultDigest !==
|
|
5078
|
+
(yield* withCrypto(runCompletionDigest(recordedCompletion)))
|
|
5079
|
+
)
|
|
4790
5080
|
return yield* RunJournalError.make({
|
|
4791
|
-
message:
|
|
5081
|
+
message: "RunCompleted terminal values failed their integrity check",
|
|
4792
5082
|
});
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
expectedRunDisposition !== undefined &&
|
|
4796
|
-
recordedCompletion.runDisposition !== undefined
|
|
4797
|
-
) {
|
|
4798
|
-
const boundedExpectedRunDisposition = yield* decodePersisted(expectedRunDisposition).pipe(
|
|
4799
|
-
Effect.mapError((cause) =>
|
|
4800
|
-
RunJournalError.make({
|
|
4801
|
-
message: `Run ${runId} reconstructed run disposition exceeds canonical persistence bounds`,
|
|
4802
|
-
cause,
|
|
4803
|
-
}),
|
|
4804
|
-
),
|
|
4805
|
-
);
|
|
4806
|
-
|
|
4807
|
-
const [expectedRunDispositionDigest, recordedRunDispositionDigest] = yield* Effect.all([
|
|
4808
|
-
withCrypto(digestJson(boundedExpectedRunDisposition)),
|
|
4809
|
-
withCrypto(digestJson(recordedCompletion.runDisposition)),
|
|
4810
|
-
]).pipe(
|
|
4811
|
-
Effect.mapError((cause) =>
|
|
4812
|
-
RunJournalError.make({
|
|
4813
|
-
message: `Run ${runId} run disposition comparison failed`,
|
|
4814
|
-
cause,
|
|
4815
|
-
}),
|
|
4816
|
-
),
|
|
4817
|
-
);
|
|
4818
|
-
|
|
4819
|
-
if (expectedRunDispositionDigest !== recordedRunDispositionDigest) {
|
|
4820
|
-
return yield* RunJournalError.make({
|
|
4821
|
-
message: `Run ${runId} terminal run disposition disagrees with its reconstructed output`,
|
|
4822
|
-
});
|
|
4823
|
-
}
|
|
4824
|
-
}
|
|
5083
|
+
// Output and disposition were validated before the atomic RunCompleted commit.
|
|
5084
|
+
// Future application codecs and completion functions have no authority to reinterpret it.
|
|
4825
5085
|
|
|
4826
5086
|
return {
|
|
4827
5087
|
_tag: "completed" as const,
|
|
@@ -4901,7 +5161,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4901
5161
|
|
|
4902
5162
|
if (firstAssistant > 0) {
|
|
4903
5163
|
resumeLeadingMessages = Prompt.fromMessages(
|
|
4904
|
-
pendingMessages.content
|
|
5164
|
+
pendingMessages.content
|
|
5165
|
+
.slice(0, firstAssistant)
|
|
5166
|
+
.filter((message) => message.role !== "system"),
|
|
4905
5167
|
);
|
|
4906
5168
|
}
|
|
4907
5169
|
if (firstAssistant >= 0) {
|
|
@@ -4989,16 +5251,45 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4989
5251
|
: { ...state, history },
|
|
4990
5252
|
);
|
|
4991
5253
|
|
|
4992
|
-
//
|
|
4993
|
-
//
|
|
4994
|
-
//
|
|
4995
|
-
|
|
5254
|
+
// Current instructions govern the continuation; the original user intent, steering and
|
|
5255
|
+
// committed history survive. The pending Turn re-enters through the batch continuation,
|
|
5256
|
+
// without duplicating the engine's freshly rendered input.
|
|
5257
|
+
const instructionView = (
|
|
5258
|
+
messages: ReadonlyArray<Prompt.Message>,
|
|
5259
|
+
instructions: ReadonlyArray<Prompt.Message>,
|
|
5260
|
+
insertWhenAbsent: boolean,
|
|
5261
|
+
) => {
|
|
5262
|
+
const projected: Array<Prompt.Message> = [];
|
|
5263
|
+
const lengths = [0];
|
|
5264
|
+
let inserted = false;
|
|
5265
|
+
|
|
5266
|
+
for (const message of messages) {
|
|
5267
|
+
if (message.role === "system") {
|
|
5268
|
+
if (!inserted) projected.push(...instructions);
|
|
5269
|
+
inserted = true;
|
|
5270
|
+
} else projected.push(message);
|
|
5271
|
+
lengths.push(projected.length);
|
|
5272
|
+
}
|
|
5273
|
+
if (!inserted && insertWhenAbsent) projected.push(...instructions);
|
|
5274
|
+
|
|
5275
|
+
return {
|
|
5276
|
+
messages: projected,
|
|
5277
|
+
prefixLength: (length: number) => lengths[length] ?? projected.length,
|
|
5278
|
+
};
|
|
5279
|
+
};
|
|
5280
|
+
|
|
4996
5281
|
const resumeContext: RunContextHook<never, never> = {
|
|
4997
5282
|
prepare: ({ source }) =>
|
|
4998
5283
|
Ref.get(stateRef).pipe(
|
|
4999
5284
|
Effect.map((state) => ({
|
|
5000
5285
|
prompt: Prompt.fromMessages([
|
|
5001
|
-
...
|
|
5286
|
+
...instructionView(
|
|
5287
|
+
resumeProjection.prompt.content,
|
|
5288
|
+
source.content
|
|
5289
|
+
.slice(0, state.baseLen)
|
|
5290
|
+
.filter((message) => message.role === "system"),
|
|
5291
|
+
true,
|
|
5292
|
+
).messages,
|
|
5002
5293
|
...source.content.slice(state.baseLen ?? source.content.length),
|
|
5003
5294
|
]),
|
|
5004
5295
|
})),
|
|
@@ -5089,6 +5380,15 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5089
5380
|
const batch = yield* withCrypto(
|
|
5090
5381
|
turnResponseBatch({
|
|
5091
5382
|
toolExposure: commit.toolExposure,
|
|
5383
|
+
toolOperations: commit.calls.map((call) =>
|
|
5384
|
+
ToolOperation.make({
|
|
5385
|
+
toolCallId: call.toolCallId,
|
|
5386
|
+
toolName: call.toolName,
|
|
5387
|
+
executionClass: call.executionClass,
|
|
5388
|
+
executionKind: call.executionKind,
|
|
5389
|
+
replay: currentContracts[call.toolName]!,
|
|
5390
|
+
}),
|
|
5391
|
+
),
|
|
5092
5392
|
toolParameterRejections: commit.toolParameterRejections,
|
|
5093
5393
|
runId,
|
|
5094
5394
|
turn: canonicalTurn,
|
|
@@ -5155,6 +5455,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5155
5455
|
parameters,
|
|
5156
5456
|
parametersDigest,
|
|
5157
5457
|
executionKind: call.executionKind,
|
|
5458
|
+
executionClass: call.executionClass,
|
|
5459
|
+
replay: currentContracts[call.toolName]!,
|
|
5158
5460
|
}),
|
|
5159
5461
|
),
|
|
5160
5462
|
);
|
|
@@ -5300,11 +5602,21 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5300
5602
|
});
|
|
5301
5603
|
}
|
|
5302
5604
|
|
|
5605
|
+
const state = yield* Ref.get(stateRef);
|
|
5606
|
+
|
|
5607
|
+
const comparisonView = instructionView(
|
|
5608
|
+
sourceJournal.prompt.content,
|
|
5609
|
+
state.history?.content
|
|
5610
|
+
.slice(0, state.baseLen)
|
|
5611
|
+
.filter((message) => message.role === "system") ?? [],
|
|
5612
|
+
false,
|
|
5613
|
+
);
|
|
5614
|
+
|
|
5303
5615
|
// Compare each visible message once. A transformed source can authorize only its
|
|
5304
5616
|
// exact canonical prefix; no per-candidate rereads or full encoded Prompt copies.
|
|
5305
5617
|
let matchingPrefix = 0;
|
|
5306
5618
|
let requiredThrough = 0;
|
|
5307
|
-
const comparisonLimit = Math.min(
|
|
5619
|
+
const comparisonLimit = Math.min(comparisonView.messages.length, commit.through);
|
|
5308
5620
|
|
|
5309
5621
|
for (let index = 0; index < commit.through; index += 1) {
|
|
5310
5622
|
const message = commit.source.content[index];
|
|
@@ -5318,7 +5630,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5318
5630
|
requiredThrough = index + 1;
|
|
5319
5631
|
if (index >= comparisonLimit || index !== matchingPrefix || message === undefined)
|
|
5320
5632
|
continue;
|
|
5321
|
-
const canonicalMessage =
|
|
5633
|
+
const canonicalMessage = comparisonView.messages[index];
|
|
5322
5634
|
|
|
5323
5635
|
if (canonicalMessage === undefined) continue;
|
|
5324
5636
|
|
|
@@ -5348,7 +5660,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5348
5660
|
|
|
5349
5661
|
if (following !== undefined && following.tag !== "ModelResponseRecorded") continue;
|
|
5350
5662
|
|
|
5351
|
-
const length = candidate.promptLength;
|
|
5663
|
+
const length = comparisonView.prefixLength(candidate.promptLength);
|
|
5352
5664
|
|
|
5353
5665
|
if (length === 0 || length > commit.through) continue;
|
|
5354
5666
|
if (length < requiredThrough || length > matchingPrefix) continue;
|
|
@@ -6437,6 +6749,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6437
6749
|
history: pending === undefined ? journal.historyBefore : resumeProjection.historyBefore,
|
|
6438
6750
|
onHistory,
|
|
6439
6751
|
input,
|
|
6752
|
+
...(journal.committedTurns > 0 || Schema.is(FrameworkMessage)(submission.messageAdmission)
|
|
6753
|
+
? { retainedInput: submission.inputPayload }
|
|
6754
|
+
: {}),
|
|
6440
6755
|
...(Schema.is(FrameworkMessage)(submission.messageAdmission)
|
|
6441
6756
|
? { frameworkMessage: submission.messageAdmission }
|
|
6442
6757
|
: {}),
|
|
@@ -6466,14 +6781,19 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6466
6781
|
? {}
|
|
6467
6782
|
: {
|
|
6468
6783
|
resume: {
|
|
6784
|
+
...(settledCompletion === undefined ? {} : { settledCompletion }),
|
|
6469
6785
|
turn: pending.turn,
|
|
6470
6786
|
turnId: pending.turnId,
|
|
6471
6787
|
calls: pending.calls,
|
|
6472
6788
|
toolParameterRejections: pending.toolParameterRejections,
|
|
6473
6789
|
settled: pending.settled,
|
|
6474
|
-
|
|
6475
|
-
|
|
6476
|
-
|
|
6790
|
+
// Legacy eager requests have no exposure snapshot. Their validated canonical
|
|
6791
|
+
// declarations establish the minimal original exposure needed for batch resume.
|
|
6792
|
+
toolExposure:
|
|
6793
|
+
pending.toolExposure ??
|
|
6794
|
+
Snapshot.make({
|
|
6795
|
+
exposedToolNames: [...new Set(pending.calls.map((call) => call.name))],
|
|
6796
|
+
}),
|
|
6477
6797
|
...(resumeLeadingMessages === undefined
|
|
6478
6798
|
? {}
|
|
6479
6799
|
: { leadingMessages: resumeLeadingMessages }),
|
|
@@ -6525,9 +6845,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6525
6845
|
...(journal.protectedContext === undefined
|
|
6526
6846
|
? {}
|
|
6527
6847
|
: { protectedContext: journal.protectedContext }),
|
|
6528
|
-
...(
|
|
6529
|
-
? {}
|
|
6530
|
-
: { pendingContextToolCallId: journal.pendingContextToolCallId }),
|
|
6848
|
+
...(pendingContextToolCallId === undefined ? {} : { pendingContextToolCallId }),
|
|
6531
6849
|
};
|
|
6532
6850
|
|
|
6533
6851
|
const commitPendingTurn: Effect.Effect<void, DurableWorkerFailure> = Effect.gen(function* () {
|
|
@@ -6629,18 +6947,20 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6629
6947
|
? completedRun
|
|
6630
6948
|
: undefined;
|
|
6631
6949
|
|
|
6632
|
-
const batch = yield*
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
|
|
6639
|
-
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6643
|
-
|
|
6950
|
+
const batch = yield* withCrypto(
|
|
6951
|
+
turnResultsBatch({
|
|
6952
|
+
toolSelections: stagedToolSelections,
|
|
6953
|
+
budgetRejectedCalls,
|
|
6954
|
+
runId,
|
|
6955
|
+
turn: canonicalTurn,
|
|
6956
|
+
turnId: state.pendingTurn.turnId,
|
|
6957
|
+
appended: remaining,
|
|
6958
|
+
producerId: config.producerId,
|
|
6959
|
+
deploymentId: config.deploymentId,
|
|
6960
|
+
createdAt,
|
|
6961
|
+
...(runCompletion === undefined ? {} : { runCompletion }),
|
|
6962
|
+
}),
|
|
6963
|
+
);
|
|
6644
6964
|
|
|
6645
6965
|
// A recovered external receipt is already canonical. Commit only the terminal
|
|
6646
6966
|
// marker at the same results boundary; never duplicate or rewrite its result.
|
|
@@ -7241,7 +7561,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7241
7561
|
.tailSequence;
|
|
7242
7562
|
|
|
7243
7563
|
const initialThrough = controlThrough;
|
|
7244
|
-
|
|
7564
|
+
|
|
7565
|
+
const initialView = yield* recoveryView(
|
|
7566
|
+
threadId,
|
|
7567
|
+
initialThrough,
|
|
7568
|
+
[submissionId],
|
|
7569
|
+
runIdForSubmission(submissionId),
|
|
7570
|
+
);
|
|
7245
7571
|
|
|
7246
7572
|
let journalMetadata =
|
|
7247
7573
|
initialView.seed === undefined
|
|
@@ -7384,7 +7710,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7384
7710
|
let expiredChildObligation = false;
|
|
7385
7711
|
|
|
7386
7712
|
if ((yield* Clock.currentTimeMillis) >= DateTime.toEpochMillis(runTiming.deadline)) {
|
|
7387
|
-
yield*
|
|
7713
|
+
yield* reconcileRetainedChildren(ctx, submission, yield* Ref.get(tokenRef));
|
|
7388
7714
|
expiredChildObligation = yield* completeJoinedReleases(submission);
|
|
7389
7715
|
}
|
|
7390
7716
|
|
|
@@ -7405,7 +7731,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7405
7731
|
|
|
7406
7732
|
if (reconciliationEvidence.openToolCalls.length === 0) return true;
|
|
7407
7733
|
for (const envelope of reconciliationRecords) knownIds.add(envelope.record.recordId);
|
|
7408
|
-
const tools: Record<string, Tool.Any> = agent.definition.toolkit.tools;
|
|
7409
7734
|
|
|
7410
7735
|
const review = yield* reconcileOpenCalls(
|
|
7411
7736
|
ctx,
|
|
@@ -7414,11 +7739,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7414
7739
|
reconciliationRecords,
|
|
7415
7740
|
reconciliationEvidence.openToolCalls,
|
|
7416
7741
|
knownIds,
|
|
7417
|
-
(
|
|
7418
|
-
const tool = tools[toolName];
|
|
7419
|
-
|
|
7420
|
-
return tool === undefined ? undefined : getToolExecutionClass(tool);
|
|
7421
|
-
},
|
|
7742
|
+
{ definition: agent.definition, contracts: yield* contractsFor(agent.definition) },
|
|
7422
7743
|
);
|
|
7423
7744
|
|
|
7424
7745
|
if (review.uncertain.length > 0 || review.unproven.length > 0) {
|
|
@@ -7431,7 +7752,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7431
7752
|
"An ordinary Tool call may have executed without a canonical outcome",
|
|
7432
7753
|
);
|
|
7433
7754
|
}
|
|
7434
|
-
//
|
|
7755
|
+
// This Submission is parked (marked Unknown, or reconciliation itself failed):
|
|
7435
7756
|
// release the claim without settling — the accepted-work obligation stays owed.
|
|
7436
7757
|
const ownershipToken = yield* Ref.get(tokenRef);
|
|
7437
7758
|
|
|
@@ -7448,6 +7769,161 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7448
7769
|
if (!expiredChildObligation && !(yield* ordinaryCallsResolved))
|
|
7449
7770
|
return Option.none<Settlement>();
|
|
7450
7771
|
|
|
7772
|
+
const continuationRecords = yield* refreshControl();
|
|
7773
|
+
|
|
7774
|
+
const continuation = yield* pendingToolBatchFor(
|
|
7775
|
+
continuationRecords,
|
|
7776
|
+
runIdForSubmission(submissionId),
|
|
7777
|
+
);
|
|
7778
|
+
|
|
7779
|
+
if (continuation !== undefined) {
|
|
7780
|
+
const current = {
|
|
7781
|
+
definition: agent.definition,
|
|
7782
|
+
contracts: yield* contractsFor(agent.definition),
|
|
7783
|
+
};
|
|
7784
|
+
|
|
7785
|
+
const operations = operationsFor(continuationRecords, runIdForSubmission(submissionId));
|
|
7786
|
+
const settled = new Set(continuation.settled.map((call) => call.id));
|
|
7787
|
+
|
|
7788
|
+
const rejected = new Set(
|
|
7789
|
+
continuation.toolParameterRejections?.map((call) => call.toolCallId),
|
|
7790
|
+
);
|
|
7791
|
+
|
|
7792
|
+
const unsupported: Array<OpenToolCallEvidence> = [];
|
|
7793
|
+
const unavailableDelegations = new Set<ToolCallId>();
|
|
7794
|
+
|
|
7795
|
+
for (const call of continuation.calls) {
|
|
7796
|
+
const original = continuationRecords
|
|
7797
|
+
.map(({ record }) => record.payload)
|
|
7798
|
+
.find(
|
|
7799
|
+
(payload) =>
|
|
7800
|
+
payload._tag === "ToolCallPrepared" &&
|
|
7801
|
+
payload.runId === runIdForSubmission(submissionId) &&
|
|
7802
|
+
payload.toolCallId === call.id,
|
|
7803
|
+
);
|
|
7804
|
+
|
|
7805
|
+
if (
|
|
7806
|
+
original?._tag === "ToolCallPrepared" &&
|
|
7807
|
+
original.executionKind === "delegation" &&
|
|
7808
|
+
!settled.has(call.id) &&
|
|
7809
|
+
!supportsOperation(
|
|
7810
|
+
submission,
|
|
7811
|
+
OpenToolCallEvidence.make(original),
|
|
7812
|
+
operations.get(call.id),
|
|
7813
|
+
original,
|
|
7814
|
+
current,
|
|
7815
|
+
)
|
|
7816
|
+
)
|
|
7817
|
+
unavailableDelegations.add(original.toolCallId);
|
|
7818
|
+
}
|
|
7819
|
+
|
|
7820
|
+
const children =
|
|
7821
|
+
unavailableDelegations.size === 0
|
|
7822
|
+
? undefined
|
|
7823
|
+
: yield* reconcileRetainedChildren(
|
|
7824
|
+
ctx,
|
|
7825
|
+
submission,
|
|
7826
|
+
yield* Ref.get(tokenRef),
|
|
7827
|
+
unavailableDelegations,
|
|
7828
|
+
);
|
|
7829
|
+
|
|
7830
|
+
for (const envelope of yield* refreshControl()) knownIds.add(envelope.record.recordId);
|
|
7831
|
+
|
|
7832
|
+
for (const call of continuation.calls) {
|
|
7833
|
+
if (
|
|
7834
|
+
call.providerExecuted ||
|
|
7835
|
+
settled.has(call.id) ||
|
|
7836
|
+
rejected.has(Schema.decodeSync(ToolCallId)(call.id))
|
|
7837
|
+
)
|
|
7838
|
+
continue;
|
|
7839
|
+
|
|
7840
|
+
const evidence = OpenToolCallEvidence.make({
|
|
7841
|
+
toolCallId: Schema.decodeSync(ToolCallId)(call.id),
|
|
7842
|
+
toolName: call.name,
|
|
7843
|
+
turn: continuation.turn,
|
|
7844
|
+
});
|
|
7845
|
+
|
|
7846
|
+
const operation = operations.get(call.id);
|
|
7847
|
+
|
|
7848
|
+
const prepared = continuationRecords
|
|
7849
|
+
.map(({ record }) => record.payload)
|
|
7850
|
+
.find(
|
|
7851
|
+
(payload) =>
|
|
7852
|
+
payload._tag === "ToolCallPrepared" &&
|
|
7853
|
+
payload.runId === runIdForSubmission(submissionId) &&
|
|
7854
|
+
payload.turn === continuation.turn &&
|
|
7855
|
+
payload.toolCallId === call.id,
|
|
7856
|
+
);
|
|
7857
|
+
|
|
7858
|
+
const preparation = prepared?._tag === "ToolCallPrepared" ? prepared : undefined;
|
|
7859
|
+
|
|
7860
|
+
if (supportsOperation(submission, evidence, operation, preparation, current)) continue;
|
|
7861
|
+
if (preparation !== undefined && !children?.notExecuted.has(evidence.toolCallId)) {
|
|
7862
|
+
if (!unavailableDelegations.has(evidence.toolCallId)) unsupported.push(evidence);
|
|
7863
|
+
continue;
|
|
7864
|
+
}
|
|
7865
|
+
|
|
7866
|
+
// Mutating handlers cannot cross this protocol boundary without preparation.
|
|
7867
|
+
// Readonly handlers may already have run; their unavailable result makes no nonexecution claim.
|
|
7868
|
+
const execution =
|
|
7869
|
+
children?.notExecuted.has(evidence.toolCallId) ||
|
|
7870
|
+
(operation !== undefined &&
|
|
7871
|
+
(operation.executionClass !== "readonly" || operation.executionKind !== "ordinary"))
|
|
7872
|
+
? "not-executed"
|
|
7873
|
+
: "unavailable";
|
|
7874
|
+
|
|
7875
|
+
const result = yield* Schema.encodeEffect(ToolUnavailable)(
|
|
7876
|
+
ToolUnavailable.make({
|
|
7877
|
+
toolName: call.name,
|
|
7878
|
+
execution,
|
|
7879
|
+
message:
|
|
7880
|
+
execution === "not-executed"
|
|
7881
|
+
? "This operation was not executed and its original implementation is unavailable. Continue with the current tools."
|
|
7882
|
+
: "The original operation is unavailable. No mutating handler was dispatched; readonly work may have run without a recorded result.",
|
|
7883
|
+
}),
|
|
7884
|
+
).pipe(Effect.flatMap(decodePersisted), Effect.orDie);
|
|
7885
|
+
|
|
7886
|
+
yield* hit("tools:before-unavailable-append");
|
|
7887
|
+
yield* appendClosedCall(ctx, submissionId, knownIds, evidence, {
|
|
7888
|
+
result: { value: result, isFailure: true },
|
|
7889
|
+
resolution: execution === "not-executed" ? "never-started" : "failed-with-error",
|
|
7890
|
+
author: "runtime",
|
|
7891
|
+
reason: "The declared operation cannot execute under its original contract",
|
|
7892
|
+
});
|
|
7893
|
+
yield* hit("tools:after-unavailable-append");
|
|
7894
|
+
}
|
|
7895
|
+
const firstWaiting = children?.waiting[0];
|
|
7896
|
+
|
|
7897
|
+
if (children !== undefined && firstWaiting !== undefined) {
|
|
7898
|
+
const disposition = yield* ledger.suspend(
|
|
7899
|
+
SuspendRequest.make({
|
|
7900
|
+
submissionId,
|
|
7901
|
+
ownershipToken: yield* Ref.get(tokenRef),
|
|
7902
|
+
reason: WaitingForChildSuspension.make({
|
|
7903
|
+
children: [firstWaiting, ...children.waiting.slice(1)],
|
|
7904
|
+
}),
|
|
7905
|
+
}),
|
|
7906
|
+
);
|
|
7907
|
+
|
|
7908
|
+
yield* hit("subagent:after-suspend");
|
|
7909
|
+
if (disposition === "resume-immediately") yield* wake.notify(threadId);
|
|
7910
|
+
|
|
7911
|
+
return Option.none<Settlement>();
|
|
7912
|
+
}
|
|
7913
|
+
if ((children?.indeterminate.size ?? 0) > 0) return Option.none<Settlement>();
|
|
7914
|
+
if (unsupported.length > 0) {
|
|
7915
|
+
yield* markCallsUnknown(
|
|
7916
|
+
ctx,
|
|
7917
|
+
submissionId,
|
|
7918
|
+
knownIds,
|
|
7919
|
+
unsupported,
|
|
7920
|
+
"The unfinished operation has no supported recovery implementation; its original identity and evidence remain unresolved",
|
|
7921
|
+
);
|
|
7922
|
+
|
|
7923
|
+
return Option.none<Settlement>();
|
|
7924
|
+
}
|
|
7925
|
+
}
|
|
7926
|
+
|
|
7451
7927
|
const lineage: AttemptLineage = {
|
|
7452
7928
|
attemptId: claim.attemptId,
|
|
7453
7929
|
supersededEpoch,
|
|
@@ -7594,7 +8070,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7594
8070
|
);
|
|
7595
8071
|
yield* hit("run:after-duration-append");
|
|
7596
8072
|
}
|
|
7597
|
-
yield*
|
|
8073
|
+
yield* reconcileRetainedChildren(ctx, submission, yield* Ref.get(tokenRef));
|
|
7598
8074
|
}
|
|
7599
8075
|
const openObligation = yield* completeJoinedReleases(submission);
|
|
7600
8076
|
|
|
@@ -7630,7 +8106,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7630
8106
|
}
|
|
7631
8107
|
}).pipe(Effect.scoped);
|
|
7632
8108
|
|
|
7633
|
-
/** A missing
|
|
8109
|
+
/** A missing current binding leaves both roots and children owed, with their claims released. */
|
|
7634
8110
|
type CapturedWorkerBinding = Effect.Success<ReturnType<typeof makeLegacyWorkerBinding>>;
|
|
7635
8111
|
|
|
7636
8112
|
// Register cleanup before any interruptible work can observe a granted claim. The token
|
|
@@ -7742,8 +8218,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7742
8218
|
Effect.map((binding) => ({ _tag: "resolved" as const, binding })),
|
|
7743
8219
|
Effect.catchTags({
|
|
7744
8220
|
BindingUnavailable: (failure) => Effect.succeed({ _tag: "refused" as const, failure }),
|
|
7745
|
-
BindingDigestMismatch: (failure) =>
|
|
7746
|
-
Effect.succeed({ _tag: "refused" as const, failure }),
|
|
7747
8221
|
}),
|
|
7748
8222
|
);
|
|
7749
8223
|
|
|
@@ -7760,6 +8234,22 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7760
8234
|
}),
|
|
7761
8235
|
).pipe(withThreadHeadSpan);
|
|
7762
8236
|
|
|
8237
|
+
const eligibleThreadHead = (threadId: ThreadId) =>
|
|
8238
|
+
Stream.runHead(
|
|
8239
|
+
ledger.scanNonterminal.pipe(
|
|
8240
|
+
Stream.filter((entry) => entry.threadId === threadId),
|
|
8241
|
+
Stream.filterEffect((entry) =>
|
|
8242
|
+
entry.state !== "unknown"
|
|
8243
|
+
? Effect.succeed(true)
|
|
8244
|
+
: ledger
|
|
8245
|
+
.loadRecoverySnapshot(
|
|
8246
|
+
RecoverySnapshotRequest.make({ submissionId: entry.submissionId }),
|
|
8247
|
+
)
|
|
8248
|
+
.pipe(Effect.map((snapshot) => snapshot.abortIntent !== undefined)),
|
|
8249
|
+
),
|
|
8250
|
+
),
|
|
8251
|
+
);
|
|
8252
|
+
|
|
7763
8253
|
const drainThread = (
|
|
7764
8254
|
resolve: (
|
|
7765
8255
|
submission: SubmissionSnapshot,
|
|
@@ -7770,12 +8260,21 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7770
8260
|
const settlements: Array<Settlement> = [];
|
|
7771
8261
|
|
|
7772
8262
|
while (true) {
|
|
8263
|
+
const before = yield* eligibleThreadHead(threadId);
|
|
7773
8264
|
const settlement = yield* processThreadHead(resolve, threadId);
|
|
7774
8265
|
|
|
7775
8266
|
if (Option.isNone(settlement)) {
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
//
|
|
8267
|
+
const after = yield* eligibleThreadHead(threadId);
|
|
8268
|
+
|
|
8269
|
+
// Parking an unknown operation exposes later runnable work. Other suspensions,
|
|
8270
|
+
// live ownership and a transiently unproven operation retain the existing barrier.
|
|
8271
|
+
if (
|
|
8272
|
+
Option.isSome(before) &&
|
|
8273
|
+
Option.isSome(after) &&
|
|
8274
|
+
before.value.submissionId !== after.value.submissionId
|
|
8275
|
+
)
|
|
8276
|
+
continue;
|
|
8277
|
+
|
|
7779
8278
|
return settlements;
|
|
7780
8279
|
}
|
|
7781
8280
|
settlements.push(settlement.value);
|
|
@@ -7835,92 +8334,25 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7835
8334
|
);
|
|
7836
8335
|
});
|
|
7837
8336
|
|
|
7838
|
-
const
|
|
7839
|
-
submission
|
|
7840
|
-
) {
|
|
7841
|
-
const exact = registeredBindings.find(
|
|
7842
|
-
(binding) =>
|
|
7843
|
-
binding.agentId === submission.agentId &&
|
|
7844
|
-
definitionDigestsEqual(binding.digests, submission.agentDigests),
|
|
7845
|
-
);
|
|
7846
|
-
|
|
7847
|
-
if (exact !== undefined) return exact;
|
|
7848
|
-
|
|
7849
|
-
const candidates = registeredBindings.filter(
|
|
7850
|
-
(binding) => binding.agentId === submission.agentId,
|
|
7851
|
-
);
|
|
7852
|
-
|
|
7853
|
-
const candidate = candidates[0];
|
|
7854
|
-
|
|
7855
|
-
// A missing/ambiguous registration needs no history read. If every retained Tool contract
|
|
7856
|
-
// is supported, neither does an ordinary additive deployment.
|
|
7857
|
-
if (candidate === undefined || candidates.length !== 1)
|
|
7858
|
-
return yield* resolveWorkerBinding(
|
|
7859
|
-
registeredBindings,
|
|
7860
|
-
submission.agentId,
|
|
7861
|
-
submission.agentDigests,
|
|
7862
|
-
);
|
|
7863
|
-
if (bindingSupports(candidate, submission.agentDigests, undefined, submission.inputPayload))
|
|
7864
|
-
return candidate;
|
|
7865
|
-
|
|
7866
|
-
// A checkpoint can omit a settled batch that still supplies completion projection.
|
|
7867
|
-
// Compatibility reads the bounded authoritative prefix, never a reduced checkpoint.
|
|
7868
|
-
const records = yield* Effect.gen(function* () {
|
|
7869
|
-
const tail = yield* store.inspectTail(
|
|
7870
|
-
ThreadTailRequest.make({ threadId: submission.threadId }),
|
|
7871
|
-
);
|
|
7872
|
-
|
|
7873
|
-
return yield* Stream.runCollect(
|
|
7874
|
-
canonicalRange(submission.threadId, tail.tailSequence).pipe(
|
|
7875
|
-
Stream.filter(controlRecords([submission.submissionId])),
|
|
7876
|
-
),
|
|
7877
|
-
);
|
|
7878
|
-
}).pipe(Effect.catchTag("ThreadNotMaterialized", () => Effect.succeed([])));
|
|
7879
|
-
|
|
7880
|
-
const completionTools =
|
|
7881
|
-
candidate === undefined
|
|
7882
|
-
? []
|
|
7883
|
-
: [
|
|
7884
|
-
...(candidate.definition.completion === undefined
|
|
7885
|
-
? []
|
|
7886
|
-
: [candidate.definition.completion.tool]),
|
|
7887
|
-
...(candidate.definition.completionFromTools?.map((entry) => entry.tool) ?? []),
|
|
7888
|
-
];
|
|
7889
|
-
|
|
7890
|
-
const pending = yield* pendingToolBatchFor(
|
|
7891
|
-
records,
|
|
7892
|
-
runIdForSubmission(submission.submissionId),
|
|
7893
|
-
completionTools,
|
|
7894
|
-
);
|
|
7895
|
-
|
|
7896
|
-
return yield* resolveWorkerBinding(
|
|
7897
|
-
registeredBindings,
|
|
7898
|
-
submission.agentId,
|
|
7899
|
-
submission.agentDigests,
|
|
7900
|
-
pending?.calls
|
|
7901
|
-
.filter((call) => call.providerExecuted !== true || completionTools.includes(call.name))
|
|
7902
|
-
.map((call) => call.name) ?? [],
|
|
7903
|
-
submission.inputPayload,
|
|
7904
|
-
);
|
|
7905
|
-
});
|
|
8337
|
+
const resolveCurrentBinding = (submission: SubmissionSnapshot) =>
|
|
8338
|
+
resolveWorkerBinding(registeredBindings, submission.agentId);
|
|
7906
8339
|
|
|
7907
8340
|
const processThreadResolvedImpl = (
|
|
7908
8341
|
threadId: ThreadId,
|
|
7909
8342
|
): Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure> =>
|
|
7910
|
-
drainThread(
|
|
8343
|
+
drainThread(resolveCurrentBinding, threadId);
|
|
7911
8344
|
|
|
7912
8345
|
const processThreadHeadImpl = (
|
|
7913
8346
|
threadId: ThreadId,
|
|
7914
8347
|
options?: { readonly yieldAfter?: DateTime.Utc },
|
|
7915
8348
|
): Effect.Effect<Option.Option<Settlement>, DurableWorkerFailure | DurableBindingFailure> =>
|
|
7916
|
-
processThreadHead(
|
|
8349
|
+
processThreadHead(resolveCurrentBinding, threadId, options);
|
|
7917
8350
|
|
|
7918
8351
|
const claimFor = Effect.fn("DurableAgentRuntime.claimFor")(function* (
|
|
7919
8352
|
submission: SubmissionSnapshot,
|
|
8353
|
+
decision: RecoveryDecision,
|
|
7920
8354
|
): Effect.fn.Return<Option.Option<Claim>, LedgerError | OwnershipLost, Scope.Scope> {
|
|
7921
|
-
const head = yield*
|
|
7922
|
-
ledger.scanNonterminal.pipe(Stream.filter((entry) => entry.threadId === submission.threadId)),
|
|
7923
|
-
);
|
|
8355
|
+
const head = yield* eligibleThreadHead(submission.threadId);
|
|
7924
8356
|
|
|
7925
8357
|
if (Option.isNone(head) || head.value.submissionId !== submission.submissionId) {
|
|
7926
8358
|
return Option.none();
|
|
@@ -7929,7 +8361,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7929
8361
|
|
|
7930
8362
|
if (Option.isNone(claimed)) return Option.none();
|
|
7931
8363
|
if (claimed.value.claim.submissionId !== submission.submissionId) {
|
|
7932
|
-
//
|
|
8364
|
+
// Only the first eligible Submission is claimable; selection may have changed since the read.
|
|
7933
8365
|
yield* ledger
|
|
7934
8366
|
.releaseOwnership(
|
|
7935
8367
|
ReleaseOwnershipRequest.make({
|
|
@@ -7942,20 +8374,28 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7942
8374
|
return Option.none();
|
|
7943
8375
|
}
|
|
7944
8376
|
|
|
8377
|
+
yield* annotateRepair(submission.threadId, decision, claimed.value.claim.producerEpoch);
|
|
8378
|
+
|
|
7945
8379
|
return Option.some(claimed.value.claim);
|
|
7946
8380
|
});
|
|
7947
8381
|
|
|
7948
8382
|
/**
|
|
7949
|
-
* DUR-013 audit
|
|
7950
|
-
*
|
|
7951
|
-
*
|
|
8383
|
+
* Best-effort DUR-013 audit of an owned repair attempt, before it can release its claim.
|
|
8384
|
+
* The fixed granted epoch never borrows a later writer's authority. State-only repairs
|
|
8385
|
+
* omit canonical annotations; their decision and operator author/reason remain in telemetry.
|
|
7952
8386
|
*/
|
|
7953
8387
|
const annotateRepair = (
|
|
7954
8388
|
threadId: ThreadId,
|
|
7955
|
-
submissionId: SubmissionId,
|
|
7956
8389
|
decision: RecoveryDecision,
|
|
8390
|
+
producerEpoch: ProducerEpoch,
|
|
7957
8391
|
): Effect.Effect<void> =>
|
|
7958
8392
|
Effect.gen(function* () {
|
|
8393
|
+
yield* store.materialize(ThreadMaterialization.make({ threadId, producerEpoch }));
|
|
8394
|
+
const tail = yield* store.inspectTail(ThreadTailRequest.make({ threadId }));
|
|
8395
|
+
|
|
8396
|
+
// ThreadCreated must remain the first canonical record of a new Thread.
|
|
8397
|
+
if (tail.tailSequence === ZERO_SEQUENCE) return;
|
|
8398
|
+
|
|
7959
8399
|
const encodedDecision = yield* Schema.encodeEffect(RecoveryDecision)(decision).pipe(
|
|
7960
8400
|
Effect.orDie,
|
|
7961
8401
|
);
|
|
@@ -7963,26 +8403,23 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7963
8403
|
const details = yield* Schema.decodeEffect(PersistedJson)(encodedDecision).pipe(Effect.orDie);
|
|
7964
8404
|
|
|
7965
8405
|
const envelope = yield* makeEnvelope(
|
|
7966
|
-
recoveryRepairRecordId(submissionId, decision._tag),
|
|
8406
|
+
recoveryRepairRecordId(decision.submissionId, decision._tag),
|
|
7967
8407
|
RepairAnnotated.make({ reason: `recovery:${decision._tag}`, details }),
|
|
7968
8408
|
);
|
|
7969
8409
|
|
|
7970
|
-
const tail = yield* store.inspectTail(ThreadTailRequest.make({ threadId }));
|
|
7971
|
-
|
|
7972
8410
|
yield* store.append(
|
|
7973
8411
|
FencedAppendRequest.make({
|
|
7974
8412
|
threadId,
|
|
7975
8413
|
batch: CanonicalBatch.make({
|
|
7976
|
-
batchId: recoveryRepairBatchId(submissionId, decision._tag),
|
|
8414
|
+
batchId: recoveryRepairBatchId(decision.submissionId, decision._tag),
|
|
7977
8415
|
producerId: config.producerId,
|
|
7978
8416
|
records: [envelope],
|
|
7979
8417
|
}),
|
|
7980
8418
|
expectedTailSequence: tail.tailSequence,
|
|
7981
8419
|
expectedTailDigest: tail.tailDigest,
|
|
7982
|
-
producerEpoch
|
|
8420
|
+
producerEpoch,
|
|
7983
8421
|
}),
|
|
7984
8422
|
);
|
|
7985
|
-
yield* wake.notify(threadId);
|
|
7986
8423
|
}).pipe(Effect.ignore);
|
|
7987
8424
|
|
|
7988
8425
|
const settleAbortedForRecovery = Effect.fn("DurableAgentRuntime.settleAbortedForRecovery")(
|
|
@@ -7990,7 +8427,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7990
8427
|
snapshot: RecoverySnapshot,
|
|
7991
8428
|
evidence: RecoveryEvidence,
|
|
7992
8429
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
7993
|
-
|
|
8430
|
+
decision: SettleAbortedDecision,
|
|
7994
8431
|
): Effect.fn.Return<"repaired" | "deferred", DurableWorkerFailure, Scope.Scope> {
|
|
7995
8432
|
const intent = snapshot.abortIntent;
|
|
7996
8433
|
|
|
@@ -8051,7 +8488,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8051
8488
|
}
|
|
8052
8489
|
}
|
|
8053
8490
|
}
|
|
8054
|
-
const claimed = yield* claimFor(submission);
|
|
8491
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8055
8492
|
|
|
8056
8493
|
if (Option.isNone(claimed)) {
|
|
8057
8494
|
// P7 §7(c): an aborted, never-claimed, still-queued `ready` Submission settles NOW
|
|
@@ -8111,11 +8548,11 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8111
8548
|
);
|
|
8112
8549
|
|
|
8113
8550
|
/**
|
|
8114
|
-
* Execute the reconcile-then-mark flow for a `MarkUnknown` decision (plan §2.2).
|
|
8115
|
-
*
|
|
8116
|
-
*
|
|
8117
|
-
*
|
|
8118
|
-
*
|
|
8551
|
+
* Execute the reconcile-then-mark flow for a `MarkUnknown` decision (plan §2.2). Current
|
|
8552
|
+
* registration metadata checks retry support when available. Low-level drivers supply their
|
|
8553
|
+
* Agent at claim time, so retry proofs defer that check without permitting execution. Calls
|
|
8554
|
+
* without a durable intent, original idempotent declaration, or reconciler proof stay uncertain.
|
|
8555
|
+
* Closed calls report `repaired`; deferred checks report `deferred`; uncertain calls become Unknown.
|
|
8119
8556
|
*/
|
|
8120
8557
|
const markUnknownForRecovery = Effect.fn("DurableAgentRuntime.markUnknownForRecovery")(function* (
|
|
8121
8558
|
snapshot: RecoverySnapshot,
|
|
@@ -8124,7 +8561,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8124
8561
|
decision: MarkUnknownDecision,
|
|
8125
8562
|
): Effect.fn.Return<"repaired" | "deferred" | "unknown", DurableWorkerFailure, Scope.Scope> {
|
|
8126
8563
|
const submission = snapshot.submission;
|
|
8127
|
-
const claimed = yield* claimFor(submission);
|
|
8564
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8128
8565
|
|
|
8129
8566
|
if (Option.isNone(claimed)) return "deferred";
|
|
8130
8567
|
const claim = claimed.value;
|
|
@@ -8149,7 +8586,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8149
8586
|
records,
|
|
8150
8587
|
evidence.openToolCalls.filter((call) => markableIds.has(call.toolCallId)),
|
|
8151
8588
|
knownIds,
|
|
8152
|
-
()
|
|
8589
|
+
yield* currentOperationsFor(submission.agentId),
|
|
8153
8590
|
);
|
|
8154
8591
|
|
|
8155
8592
|
let disposition: "repaired" | "deferred" | "unknown";
|
|
@@ -8180,40 +8617,61 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8180
8617
|
return disposition;
|
|
8181
8618
|
});
|
|
8182
8619
|
|
|
8620
|
+
/** A retry intent permits only the saved operation under its supported current contract. */
|
|
8621
|
+
const hasUnsupportedRetry = Effect.fnUntraced(function* (
|
|
8622
|
+
snapshot: RecoverySnapshot,
|
|
8623
|
+
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
8624
|
+
) {
|
|
8625
|
+
if (!snapshot.unknownResolutions.some((intent) => intent.resolution._tag === "SafeToRetry"))
|
|
8626
|
+
return false;
|
|
8627
|
+
const submission = snapshot.submission;
|
|
8628
|
+
const current = yield* currentOperationsFor(submission.agentId);
|
|
8629
|
+
const operations = operationsFor(records, runIdForSubmission(submission.submissionId));
|
|
8630
|
+
|
|
8631
|
+
for (const intent of snapshot.unknownResolutions) {
|
|
8632
|
+
if (intent.resolution._tag === "SafeToRetry") {
|
|
8633
|
+
const prepared = records
|
|
8634
|
+
.map(({ record }) => record.payload)
|
|
8635
|
+
.find(
|
|
8636
|
+
(payload) =>
|
|
8637
|
+
payload._tag === "ToolCallPrepared" &&
|
|
8638
|
+
payload.runId === runIdForSubmission(submission.submissionId) &&
|
|
8639
|
+
payload.toolCallId === intent.toolCallId,
|
|
8640
|
+
);
|
|
8641
|
+
|
|
8642
|
+
if (
|
|
8643
|
+
prepared?._tag !== "ToolCallPrepared" ||
|
|
8644
|
+
(current !== undefined &&
|
|
8645
|
+
!supportsOperation(
|
|
8646
|
+
submission,
|
|
8647
|
+
OpenToolCallEvidence.make(prepared),
|
|
8648
|
+
operations.get(intent.toolCallId),
|
|
8649
|
+
prepared,
|
|
8650
|
+
current,
|
|
8651
|
+
))
|
|
8652
|
+
)
|
|
8653
|
+
return true;
|
|
8654
|
+
}
|
|
8655
|
+
}
|
|
8656
|
+
|
|
8657
|
+
return false;
|
|
8658
|
+
});
|
|
8659
|
+
|
|
8183
8660
|
/**
|
|
8184
|
-
*
|
|
8185
|
-
*
|
|
8186
|
-
* exist); the wake itself is the ledger's idempotent re-check on intent replay (WP2), so a
|
|
8187
|
-
* crash anywhere in this pass converges on the next one.
|
|
8661
|
+
* Replay covering resolution intents as state-only wakes. Canonical outcomes wait for an
|
|
8662
|
+
* actual claim, so resolving an older request cannot append using a later writer's epoch.
|
|
8188
8663
|
*/
|
|
8189
8664
|
const applyUnknownResolutionsForRecovery = Effect.fn(
|
|
8190
8665
|
"DurableAgentRuntime.applyUnknownResolutionsForRecovery",
|
|
8191
8666
|
)(function* (
|
|
8192
8667
|
snapshot: RecoverySnapshot,
|
|
8193
|
-
|
|
8668
|
+
_evidence: RecoveryEvidence,
|
|
8194
8669
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
8195
|
-
): Effect.fn.Return<"repaired" | "deferred", DurableWorkerFailure> {
|
|
8670
|
+
): Effect.fn.Return<"repaired" | "deferred" | "unknown", DurableWorkerFailure> {
|
|
8671
|
+
if (yield* hasUnsupportedRetry(snapshot, records)) return "unknown";
|
|
8196
8672
|
const submission = snapshot.submission;
|
|
8197
|
-
const ctx = yield* attemptContextAtTail(submission.threadId);
|
|
8198
|
-
const knownIds = knownRecordIdsOf(records);
|
|
8199
8673
|
|
|
8200
|
-
const applied = yield* reconcileOpenCalls(
|
|
8201
|
-
ctx,
|
|
8202
|
-
submission,
|
|
8203
|
-
snapshot,
|
|
8204
|
-
records,
|
|
8205
|
-
evidence.openToolCalls,
|
|
8206
|
-
knownIds,
|
|
8207
|
-
() => undefined,
|
|
8208
|
-
).pipe(
|
|
8209
|
-
Effect.as(true),
|
|
8210
|
-
// A fence advance mid-pass means another owner took the Thread: defer to it.
|
|
8211
|
-
Effect.catchTag("FenceRejected", () => Effect.succeed(false)),
|
|
8212
|
-
);
|
|
8213
|
-
|
|
8214
|
-
if (!applied) return "deferred";
|
|
8215
8674
|
for (const intent of snapshot.unknownResolutions) {
|
|
8216
|
-
// Replaying the stored intent re-checks the covering wake idempotently (WP2 contract).
|
|
8217
8675
|
yield* ledger
|
|
8218
8676
|
.recordUnknownResolution(
|
|
8219
8677
|
UnknownResolutionCommand.make({
|
|
@@ -8234,6 +8692,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8234
8692
|
),
|
|
8235
8693
|
);
|
|
8236
8694
|
}
|
|
8695
|
+
// Canonical outcomes are applied by the next claim. A wake never takes an active epoch.
|
|
8237
8696
|
yield* wake.notify(submission.threadId);
|
|
8238
8697
|
|
|
8239
8698
|
return "repaired";
|
|
@@ -8253,6 +8712,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8253
8712
|
function* (
|
|
8254
8713
|
snapshot: RecoverySnapshot,
|
|
8255
8714
|
evidence: RecoveryEvidence,
|
|
8715
|
+
decision: RecoveryDecision,
|
|
8256
8716
|
): Effect.fn.Return<"repaired" | "deferred", DurableWorkerFailure, Scope.Scope> {
|
|
8257
8717
|
const submission = snapshot.submission;
|
|
8258
8718
|
|
|
@@ -8266,7 +8726,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8266
8726
|
const first = undecided[0];
|
|
8267
8727
|
|
|
8268
8728
|
if (first === undefined) return "deferred";
|
|
8269
|
-
const claimed = yield* claimFor(submission);
|
|
8729
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8270
8730
|
|
|
8271
8731
|
if (Option.isNone(claimed)) return "deferred";
|
|
8272
8732
|
const claim = claimed.value;
|
|
@@ -8355,7 +8815,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8355
8815
|
// A closed admission window must release provably-unused reservations or restore the
|
|
8356
8816
|
// existing child's link. Do this before ordinary Unknown Outcomes make the lane
|
|
8357
8817
|
// unclaimable, and never let binding-free recovery admit new work after expiry.
|
|
8358
|
-
const claimed = yield* claimFor(submission);
|
|
8818
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8359
8819
|
|
|
8360
8820
|
if (Option.isNone(claimed)) return "deferred";
|
|
8361
8821
|
const claim = claimed.value;
|
|
@@ -8368,7 +8828,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8368
8828
|
);
|
|
8369
8829
|
const ctx = yield* attemptContextFor(submission.threadId, claim.producerEpoch);
|
|
8370
8830
|
|
|
8371
|
-
yield*
|
|
8831
|
+
yield* reconcileRetainedChildren(ctx, submission, claim.ownershipToken);
|
|
8372
8832
|
const open = yield* completeJoinedReleases(submission);
|
|
8373
8833
|
|
|
8374
8834
|
yield* ledger
|
|
@@ -8399,12 +8859,12 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8399
8859
|
return yield* applyUnknownResolutionsForRecovery(snapshot, evidence, records);
|
|
8400
8860
|
}
|
|
8401
8861
|
case "AwaitUnknownResolution": {
|
|
8402
|
-
//
|
|
8862
|
+
// This Submission stays parked awaiting the authorized DUR-017 resolution path;
|
|
8403
8863
|
// the settlement obligation stays visible, nothing replays.
|
|
8404
8864
|
return "unknown";
|
|
8405
8865
|
}
|
|
8406
8866
|
case "AwaitApprovalDecision": {
|
|
8407
|
-
return yield* awaitApprovalForRecovery(snapshot, evidence);
|
|
8867
|
+
return yield* awaitApprovalForRecovery(snapshot, evidence, decision);
|
|
8408
8868
|
}
|
|
8409
8869
|
case "ResumeSuspended": {
|
|
8410
8870
|
return yield* resumeSuspendedForRecovery(snapshot);
|
|
@@ -8458,7 +8918,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8458
8918
|
);
|
|
8459
8919
|
|
|
8460
8920
|
if (Option.isNone(host)) return "deferred";
|
|
8461
|
-
const claimed = yield* claimFor(host.value);
|
|
8921
|
+
const claimed = yield* claimFor(host.value, decision);
|
|
8462
8922
|
|
|
8463
8923
|
if (Option.isNone(claimed)) return "deferred";
|
|
8464
8924
|
const claim = claimed.value;
|
|
@@ -8531,7 +8991,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8531
8991
|
}
|
|
8532
8992
|
case "ApplyInput":
|
|
8533
8993
|
case "RepairInputMarker": {
|
|
8534
|
-
const claimed = yield* claimFor(submission);
|
|
8994
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8535
8995
|
|
|
8536
8996
|
if (Option.isNone(claimed)) return "deferred";
|
|
8537
8997
|
const claim = claimed.value;
|
|
@@ -8581,7 +9041,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8581
9041
|
const reservation = snapshot.reservation;
|
|
8582
9042
|
|
|
8583
9043
|
if (reservation === undefined) return "deferred";
|
|
8584
|
-
const claimed = yield* claimFor(submission);
|
|
9044
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8585
9045
|
|
|
8586
9046
|
if (Option.isNone(claimed)) {
|
|
8587
9047
|
// P7 §7(c) crash replay: a queued-abort settlement that committed its reservation
|
|
@@ -8665,7 +9125,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8665
9125
|
const admission = yield* establishChildFromRequest(submission, requestedPayload);
|
|
8666
9126
|
|
|
8667
9127
|
if (admission._tag === "indeterminate") return "deferred";
|
|
8668
|
-
const claimed = yield* claimFor(submission);
|
|
9128
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8669
9129
|
|
|
8670
9130
|
if (Option.isNone(claimed)) return "deferred";
|
|
8671
9131
|
const claim = claimed.value;
|
|
@@ -8738,7 +9198,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8738
9198
|
// waitingForChild checkpoint"): claim the lane and suspend it — the ledger op ends the
|
|
8739
9199
|
// ownership period itself, so the lane holds no worker permit while each child runs
|
|
8740
9200
|
// on its own lane. Never spawns a replacement invocation (SUB-018/SUB-030).
|
|
8741
|
-
const claimed = yield* claimFor(submission);
|
|
9201
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8742
9202
|
|
|
8743
9203
|
if (Option.isNone(claimed)) return "deferred";
|
|
8744
9204
|
const claim = claimed.value;
|
|
@@ -8845,7 +9305,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8845
9305
|
}
|
|
8846
9306
|
}
|
|
8847
9307
|
if (submission.state !== "suspended") {
|
|
8848
|
-
const claimed = yield* claimFor(submission);
|
|
9308
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8849
9309
|
|
|
8850
9310
|
if (Option.isNone(claimed)) return "deferred";
|
|
8851
9311
|
const claim = claimed.value;
|
|
@@ -8965,10 +9425,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8965
9425
|
executeRecoveryDecision(snapshot, evidence, decision, history.records, history),
|
|
8966
9426
|
);
|
|
8967
9427
|
|
|
8968
|
-
if (disposition === "repaired") {
|
|
8969
|
-
yield* annotateRepair(submission.threadId, submission.submissionId, decision);
|
|
8970
|
-
}
|
|
8971
|
-
|
|
8972
9428
|
return RecoveryReport.make({
|
|
8973
9429
|
submissionId: submission.submissionId,
|
|
8974
9430
|
threadId: submission.threadId,
|
|
@@ -9131,13 +9587,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9131
9587
|
!registeredBindings.some(
|
|
9132
9588
|
(binding) =>
|
|
9133
9589
|
binding.agentId === agent.definition.id &&
|
|
9134
|
-
|
|
9135
|
-
binding.retainedManifests?.some((retained) =>
|
|
9136
|
-
definitionDigestsEqual(
|
|
9137
|
-
DefinitionDigests.make({ ...retained.digests, replay: retained.replay }),
|
|
9138
|
-
options.definitions,
|
|
9139
|
-
),
|
|
9140
|
-
)),
|
|
9590
|
+
definitionDigestsEqual(binding.digests, options.definitions),
|
|
9141
9591
|
)
|
|
9142
9592
|
) {
|
|
9143
9593
|
const retained = yield* ledger.lookup(
|
|
@@ -9574,7 +10024,15 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9574
10024
|
const nowMillis = yield* Clock.currentTimeMillis;
|
|
9575
10025
|
const explainedAt = yield* nowUtc;
|
|
9576
10026
|
const runId = runIdForSubmission(submission.submissionId);
|
|
9577
|
-
|
|
10027
|
+
|
|
10028
|
+
const resolvedIds = new Set(
|
|
10029
|
+
read.records.flatMap(({ record }) =>
|
|
10030
|
+
record.payload._tag === "ToolCallSettled" && record.payload.runId === runId
|
|
10031
|
+
? [record.payload.toolCallId]
|
|
10032
|
+
: [],
|
|
10033
|
+
),
|
|
10034
|
+
);
|
|
10035
|
+
|
|
9578
10036
|
const unknownCalls: Array<ExplainedUnknownCall> = [];
|
|
9579
10037
|
|
|
9580
10038
|
for (const envelope of read.records) {
|
|
@@ -9611,6 +10069,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9611
10069
|
inputRecorded: evidence.inputRecorded,
|
|
9612
10070
|
abortRecorded: evidence.abortRecorded,
|
|
9613
10071
|
openToolCalls: evidence.openToolCalls,
|
|
10072
|
+
pendingOperations: read.records.flatMap(({ record }) =>
|
|
10073
|
+
record.payload._tag === "ToolCallPrepared" &&
|
|
10074
|
+
record.payload.runId === runId &&
|
|
10075
|
+
!resolvedIds.has(record.payload.toolCallId)
|
|
10076
|
+
? [record.payload]
|
|
10077
|
+
: [],
|
|
10078
|
+
),
|
|
9614
10079
|
openDelegationCalls: evidence.openDelegationCalls,
|
|
9615
10080
|
approvalsPending: evidence.approvalsPending,
|
|
9616
10081
|
unknownCalls,
|
|
@@ -9629,7 +10094,15 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9629
10094
|
}),
|
|
9630
10095
|
decision,
|
|
9631
10096
|
decisionMeaning: recoveryDecisionMeaning(decision._tag),
|
|
9632
|
-
disposition:
|
|
10097
|
+
disposition:
|
|
10098
|
+
decision._tag === "ApplyUnknownResolutions" &&
|
|
10099
|
+
(yield* hasUnsupportedRetry(snapshot, read.records).pipe(
|
|
10100
|
+
Effect.mapError((cause) =>
|
|
10101
|
+
RunJournalError.make({ message: "Cannot verify the pending retry contract", cause }),
|
|
10102
|
+
),
|
|
10103
|
+
))
|
|
10104
|
+
? "unknown"
|
|
10105
|
+
: predictRecoveryDisposition(decision, snapshot),
|
|
9633
10106
|
explainedAt,
|
|
9634
10107
|
});
|
|
9635
10108
|
});
|
|
@@ -9754,9 +10227,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9754
10227
|
|
|
9755
10228
|
/**
|
|
9756
10229
|
* Safe re-drive of exactly one Submission's recovery decision (plan §3): classify, execute
|
|
9757
|
-
* the ONE repair the classifier names,
|
|
9758
|
-
*
|
|
9759
|
-
* protect the paths that own their own operations: settled work (`NoAction`),
|
|
10230
|
+
* the ONE repair the classifier names, annotate owned repair attempts with their actual
|
|
10231
|
+
* claim epoch (DUR-013), and wake the lane. Typed refusals
|
|
10232
|
+
* protect the paths that own their own operations: settled work (`NoAction`), requests parked
|
|
9760
10233
|
* on Unknown Outcomes (`resolveUnknown`, DUR-017), and lanes awaiting approval decisions
|
|
9761
10234
|
* (`resolveApproval`). The mandatory `author`/`reason` audit fields (SEC-011) annotate the
|
|
9762
10235
|
* structured operator log.
|
|
@@ -9804,7 +10277,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9804
10277
|
refusal: "await-unknown-resolution",
|
|
9805
10278
|
decisionTag: decision._tag,
|
|
9806
10279
|
message:
|
|
9807
|
-
"
|
|
10280
|
+
"This Submission is parked on Unknown Outcomes; resolve them through the authorized resolveUnknown path (DUR-017) instead of retrying.",
|
|
9808
10281
|
});
|
|
9809
10282
|
}
|
|
9810
10283
|
if (decision._tag === "AwaitApprovalDecision") {
|
|
@@ -9830,9 +10303,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9830
10303
|
executeRecoveryDecision(snapshot, evidence, decision, read.records, read),
|
|
9831
10304
|
);
|
|
9832
10305
|
|
|
9833
|
-
if (disposition === "repaired") {
|
|
9834
|
-
yield* annotateRepair(submission.threadId, command.submissionId, decision);
|
|
9835
|
-
}
|
|
9836
10306
|
yield* wake.notify(submission.threadId);
|
|
9837
10307
|
|
|
9838
10308
|
return RecoveryReport.make({
|
|
@@ -9885,11 +10355,16 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9885
10355
|
const records = (yield* readAllTolerant(submission.threadId, [submission.submissionId]))
|
|
9886
10356
|
.records;
|
|
9887
10357
|
|
|
10358
|
+
const runId = runIdForSubmission(submission.submissionId);
|
|
10359
|
+
|
|
9888
10360
|
const resolvedIds = new Set(
|
|
9889
|
-
|
|
10361
|
+
records.flatMap(({ record }) =>
|
|
10362
|
+
record.payload._tag === "ToolCallSettled" && record.payload.runId === runId
|
|
10363
|
+
? [record.payload.toolCallId]
|
|
10364
|
+
: [],
|
|
10365
|
+
),
|
|
9890
10366
|
);
|
|
9891
10367
|
|
|
9892
|
-
const runId = runIdForSubmission(submission.submissionId);
|
|
9893
10368
|
let earliest: DateTime.Utc | undefined;
|
|
9894
10369
|
|
|
9895
10370
|
for (const envelope of records) {
|
|
@@ -10104,19 +10579,16 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
10104
10579
|
* the next model request, reattaches through the prompt-coverage rule after a crash, and the
|
|
10105
10580
|
* joined Submissions settle with the host outcome (DUR-002/DUR-016).
|
|
10106
10581
|
* - `runWorker(agent)` — scan-seeded, wake-driven worker loop over every lane (WP4's host driver),
|
|
10107
|
-
*
|
|
10108
|
-
*
|
|
10109
|
-
* parent-linked head refused this way settles with the framework `ChildCompatibilityFailure`.
|
|
10582
|
+
* over a singleton current Binding: a claimed head belonging to a different Agent never
|
|
10583
|
+
* runs against this binding. Missing bindings leave accepted roots and children owed.
|
|
10110
10584
|
* - `processThreadResolved(threadId)` / `runResolvedWorker` — the
|
|
10111
|
-
*
|
|
10112
|
-
*
|
|
10113
|
-
*
|
|
10114
|
-
* `ChildCompatibilityFailure` (no application code, the child never executes), and an
|
|
10115
|
-
* unresolvable root surfaces the typed refusal after releasing the claim.
|
|
10585
|
+
* equivalents over registrations owned by the runtime Layer: each stable `agentId` selects
|
|
10586
|
+
* one current Binding. Unfinished operation contracts gate handler execution independently
|
|
10587
|
+
* of immutable admission evidence; missing bindings release the claim with a typed refusal.
|
|
10116
10588
|
* - `runRecovery()` — classify every nonterminal Submission with the pure `classifyRecovery` and
|
|
10117
|
-
* execute
|
|
10589
|
+
* execute repair decisions, annotating owned repair attempts with their granted epochs
|
|
10118
10590
|
* (DUR-013). Untouched ready input and model-resuming work are reported `deferred` for a
|
|
10119
|
-
* worker claim;
|
|
10591
|
+
* worker claim; Submissions parked on
|
|
10120
10592
|
* Unknown Outcomes are reported `unknown`. The S2 binding-free Subagent executors (admission
|
|
10121
10593
|
* completion, start-link repair, waiting restoration, wake replay, canonical join accounting,
|
|
10122
10594
|
* abort propagation, orphan reservation release) run here; the settlement join itself is
|
|
@@ -10223,8 +10695,8 @@ export class DurableAgentRuntime extends Context.Service<
|
|
|
10223
10695
|
readonly verify: (threadId: ThreadId) => Effect.Effect<IntegrityReport, DurableVerifyFailure>;
|
|
10224
10696
|
/**
|
|
10225
10697
|
* Safe re-drive of one Submission's recovery decision with mandatory author/reason audit
|
|
10226
|
-
* (SEC-011): executes exactly the repair the classifier names
|
|
10227
|
-
*
|
|
10698
|
+
* (SEC-011): executes exactly the repair the classifier names; owned repair attempts
|
|
10699
|
+
* append a best-effort `RepairAnnotated` with their actual claim epoch (DUR-013). Typed `RetryRefused`
|
|
10228
10700
|
* for settled work and for lanes owned by the resolveUnknown/resolveApproval paths.
|
|
10229
10701
|
*/
|
|
10230
10702
|
readonly retry: (command: RetryCommand) => Effect.Effect<RecoveryReport, DurableRetryFailure>;
|