effect-agent 0.1.0-beta.97 → 0.1.0-beta.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{AgentRuntime-B64362Id.d.mts → AgentRuntime-BMzrJos6.d.mts} +2 -2
- package/dist/{AgentRuntime-Cdvqw7l_.mjs → AgentRuntime-BWOnNc0d.mjs} +78 -42
- package/dist/AgentRuntime-BWOnNc0d.mjs.map +1 -0
- package/dist/{DurableAgentRuntime-DbUvZHDZ.mjs → DurableAgentRuntime-BE_OjQjz.mjs} +471 -252
- package/dist/DurableAgentRuntime-BE_OjQjz.mjs.map +1 -0
- package/dist/{DurableFailpoint-DQJet828.d.mts → DurableFailpoint-mx3-nsxU.d.mts} +3 -3
- package/dist/{EventSource-D6PAAEyK.d.mts → EventSource-BZ73J0s5.d.mts} +3 -3
- package/dist/{PreparedInputAdmission-omMbwd--.d.mts → PreparedInputAdmission-BkeI-Xvr.d.mts} +4 -4
- package/dist/{Records-CV49DgQK.d.mts → Records-BIiRLqlz.d.mts} +92 -128
- package/dist/{Records-BAutSSnJ.mjs → Records-DR1xtbXY.mjs} +35 -8
- package/dist/Records-DR1xtbXY.mjs.map +1 -0
- package/dist/{RunJournal-DRRRZmG7.mjs → RunJournal-DNs3PM4K.mjs} +80 -23
- package/dist/RunJournal-DNs3PM4K.mjs.map +1 -0
- package/dist/{RunOptions-Clj8cLsp.d.mts → RunOptions-Bruk54SE.d.mts} +13 -4
- package/dist/{Schedule-DePbGfgd.d.mts → Schedule-BNFlluAl.d.mts} +2 -2
- package/dist/{Scheduling-C3QpYn0W.mjs → Scheduling-HFH0pST1.mjs} +2 -2
- package/dist/{Scheduling-C3QpYn0W.mjs.map → Scheduling-HFH0pST1.mjs.map} +1 -1
- package/dist/{Subagent-Cbfzylz1.mjs → Subagent-C2s8PFrd.mjs} +2 -2
- package/dist/{Subagent-Cbfzylz1.mjs.map → Subagent-C2s8PFrd.mjs.map} +1 -1
- package/dist/{Subagent-Bm1dRecN.d.mts → Subagent-m7ywTCe4.d.mts} +3 -3
- package/dist/{Subscription-BR_8Cxh3.d.mts → Subscription-BCQ3L9Zp.d.mts} +3 -3
- package/dist/{SubscriptionInput-B95sQbuk.d.mts → SubscriptionInput-Cqu4_RW3.d.mts} +3 -3
- package/dist/{ToolReconciler-DQVaJfpE.d.mts → ToolReconciler-BmVRAHKs.d.mts} +8 -4
- package/dist/agent-registration-cXpB8cBU.mjs +199 -0
- package/dist/agent-registration-cXpB8cBU.mjs.map +1 -0
- package/dist/capabilities/Commands.d.mts +1 -1
- package/dist/capabilities/RunHooks.d.mts +1 -1
- package/dist/capabilities/Subagent.d.mts +1 -1
- package/dist/capabilities/Subagent.mjs +1 -1
- package/dist/durable/ActivityStore.d.mts +1 -1
- package/dist/durable/ActivityStore.mjs +1 -1
- package/dist/durable/Admin.d.mts +1 -1
- package/dist/durable/Admin.mjs +12 -10
- package/dist/durable/Admin.mjs.map +1 -1
- package/dist/durable/AgentRegistration.d.mts +2 -2
- package/dist/durable/AgentRegistration.mjs +4 -8
- package/dist/durable/AgentRegistration.mjs.map +1 -1
- package/dist/durable/Certification.d.mts +2 -2
- package/dist/durable/CommittedActivity.d.mts +1 -1
- package/dist/durable/CommittedActivity.mjs +1 -1
- package/dist/durable/Digest.d.mts +1 -1
- package/dist/durable/Digest.mjs +1 -1
- package/dist/durable/DurableAgentRuntime.d.mts +1 -1
- package/dist/durable/DurableAgentRuntime.mjs +1 -1
- package/dist/durable/DurableFailpoint.d.mts +1 -1
- package/dist/durable/DurableFailpoint.mjs +2 -0
- package/dist/durable/DurableFailpoint.mjs.map +1 -1
- package/dist/durable/DurableFailpointTestControl.d.mts +1 -1
- package/dist/durable/EventSource.d.mts +1 -1
- package/dist/durable/EventSource.mjs +1 -1
- package/dist/durable/GitHubWorkflowSource.d.mts +3 -3
- package/dist/durable/GitHubWorkflowSource.mjs +1 -1
- package/dist/durable/MessageDelivery.d.mts +3 -3
- package/dist/durable/MessageDelivery.mjs +1 -1
- package/dist/durable/MessageDeliveryStoreConformance.d.mts +39 -39
- package/dist/durable/MessageDeliveryStoreConformance.mjs +1 -1
- package/dist/durable/PersistentHistory.d.mts +1 -1
- package/dist/durable/PersistentHistory.mjs +2 -2
- package/dist/durable/PreparedInputAdmission.d.mts +1 -1
- package/dist/durable/Records.d.mts +2 -2
- package/dist/durable/Records.mjs +2 -2
- package/dist/durable/Recovery.d.mts +1 -1
- package/dist/durable/Recovery.mjs +9 -9
- package/dist/durable/Recovery.mjs.map +1 -1
- package/dist/durable/RunJournal.d.mts +2 -2
- package/dist/durable/RunJournal.mjs +2 -2
- package/dist/durable/Schedule.d.mts +1 -1
- package/dist/durable/Schedule.mjs +1 -1
- package/dist/durable/ScheduleStoreConformance.d.mts +1 -1
- package/dist/durable/ScheduleStoreConformance.mjs +1 -1
- package/dist/durable/ScheduleTransition.d.mts +1 -1
- package/dist/durable/ScheduleTransition.mjs +1 -1
- package/dist/durable/Scheduling.d.mts +2 -2
- package/dist/durable/Scheduling.mjs +1 -1
- package/dist/durable/SqlStorageV2Upgrade.d.mts +2 -2
- package/dist/durable/SqlStorageV2Upgrade.mjs +1 -1
- package/dist/durable/SqlSubscriptionStore.d.mts +2 -2
- package/dist/durable/SqlSubscriptionStore.mjs +1 -1
- package/dist/durable/SubmissionLedger.d.mts +1 -1
- package/dist/durable/SubmissionLedger.mjs +14 -13
- package/dist/durable/SubmissionLedger.mjs.map +1 -1
- package/dist/durable/SubmissionLedgerConformance.d.mts +1 -1
- package/dist/durable/SubmissionLedgerConformance.mjs +9 -2
- package/dist/durable/SubmissionLedgerConformance.mjs.map +1 -1
- package/dist/durable/SubmissionStatus.d.mts +1 -1
- package/dist/durable/Subscription.d.mts +1 -1
- package/dist/durable/Subscription.mjs +1 -1
- package/dist/durable/SubscriptionInput.d.mts +1 -1
- package/dist/durable/SubscriptionInput.mjs +2 -2
- package/dist/durable/SubscriptionStoreConformance.d.mts +1 -1
- package/dist/durable/SubscriptionStoreConformance.mjs +1 -1
- package/dist/durable/SubscriptionTools.d.mts +3 -3
- package/dist/durable/SubscriptionTransition.d.mts +1 -1
- package/dist/durable/SubscriptionTransition.mjs +1 -1
- package/dist/durable/Subscriptions.d.mts +5 -5
- package/dist/durable/Subscriptions.mjs +1 -1
- package/dist/durable/ThreadContextHistory.d.mts +1 -1
- package/dist/durable/ThreadContextHistory.mjs +1 -1
- package/dist/durable/ThreadContextHistoryProjection.d.mts +1 -1
- package/dist/durable/ThreadContextHistoryProjection.mjs +1 -1
- package/dist/durable/ThreadInvariants.d.mts +4 -2
- package/dist/durable/ThreadInvariants.mjs +49 -3
- package/dist/durable/ThreadInvariants.mjs.map +1 -1
- package/dist/durable/ThreadProjection.d.mts +1 -1
- package/dist/durable/ThreadProjection.mjs +1 -1
- package/dist/durable/ThreadProjectionMaintenance.d.mts +1 -1
- package/dist/durable/ThreadStore.d.mts +1 -1
- package/dist/durable/ThreadStore.mjs +1 -1
- package/dist/durable/ThreadStoreConformance.d.mts +1 -1
- package/dist/durable/ThreadStoreConformance.mjs +1 -1
- package/dist/durable/ToolReconciler.d.mts +1 -1
- package/dist/durable/ToolReconciler.mjs +9 -5
- package/dist/durable/ToolReconciler.mjs.map +1 -1
- package/dist/durable/WorkerHost.d.mts +1 -1
- package/dist/engine/AgentRuntime.d.mts +1 -1
- package/dist/engine/AgentRuntime.mjs +1 -1
- package/dist/engine/RunOptions.d.mts +1 -1
- package/dist/engine/RunOptions.mjs.map +1 -1
- package/dist/index.d.mts +11 -11
- package/dist/index.mjs +6 -6
- package/package.json +1 -1
- package/src/durable/Admin.ts +20 -11
- package/src/durable/AgentRegistration.ts +1 -5
- package/src/durable/DurableAgentRuntime.ts +947 -464
- package/src/durable/DurableFailpoint.ts +2 -0
- package/src/durable/Records.ts +30 -5
- package/src/durable/Recovery.ts +8 -8
- package/src/durable/RunJournal.ts +128 -51
- package/src/durable/SubmissionLedger.ts +13 -12
- package/src/durable/SubmissionLedgerConformance.ts +13 -2
- package/src/durable/ThreadInvariants.ts +70 -5
- package/src/durable/ToolReconciler.ts +7 -3
- package/src/durable/internal/agent-registration.ts +65 -414
- package/src/durable/internal/messaging-host.ts +0 -2
- package/src/durable/internal/worker-host.ts +34 -96
- package/src/engine/RunOptions.ts +13 -3
- package/src/engine/internal/agent-runtime.ts +160 -106
- package/dist/AgentRuntime-Cdvqw7l_.mjs.map +0 -1
- package/dist/DurableAgentRuntime-DbUvZHDZ.mjs.map +0 -1
- package/dist/Records-BAutSSnJ.mjs.map +0 -1
- package/dist/RunJournal-DRRRZmG7.mjs.map +0 -1
- package/dist/agent-registration-CXGDwvJH.mjs +0 -379
- package/dist/agent-registration-CXGDwvJH.mjs.map +0 -1
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
getToolExecutionKind,
|
|
58
58
|
SubagentParentLink,
|
|
59
59
|
} from "../core/SubagentContract.ts";
|
|
60
|
-
import { Selection,
|
|
60
|
+
import { Selection, Snapshot } from "../core/ToolExposure.ts";
|
|
61
61
|
import type { ToolParameterRejection } from "../core/ToolResult.ts";
|
|
62
62
|
import {
|
|
63
63
|
type ModelCallUsage,
|
|
@@ -84,7 +84,8 @@ import {
|
|
|
84
84
|
type RuntimeBinding,
|
|
85
85
|
} from "../engine/AgentRuntime.ts";
|
|
86
86
|
import { ContextCompactor, CompactionError } from "../engine/ContextCompactor.ts";
|
|
87
|
-
import {
|
|
87
|
+
import { ContextRolloverTool } from "../engine/ContextWindow.ts";
|
|
88
|
+
import { getToolExecutionClass } from "../engine/DurableStep.ts";
|
|
88
89
|
import { MessagingHost } from "../engine/MessagingHost.ts";
|
|
89
90
|
import {
|
|
90
91
|
CurrentToolFailureObserver,
|
|
@@ -139,7 +140,7 @@ import {
|
|
|
139
140
|
BindingUnavailable,
|
|
140
141
|
compileRegistrations,
|
|
141
142
|
definitionDigestsEqual,
|
|
142
|
-
|
|
143
|
+
toolReplayContracts,
|
|
143
144
|
type AgentRegistration,
|
|
144
145
|
type DurableBindingFailure,
|
|
145
146
|
makeLegacyWorkerBinding,
|
|
@@ -197,6 +198,8 @@ import {
|
|
|
197
198
|
ToolApprovalDecided,
|
|
198
199
|
ToolApprovalRequested,
|
|
199
200
|
ToolCallPrepared,
|
|
201
|
+
ToolOperation,
|
|
202
|
+
ToolUnavailable,
|
|
200
203
|
RunPolicyUsageReserved,
|
|
201
204
|
ToolCallResolved,
|
|
202
205
|
ToolCallSettled,
|
|
@@ -234,6 +237,7 @@ import {
|
|
|
234
237
|
projectRunJournalStream,
|
|
235
238
|
type JournalBoundary,
|
|
236
239
|
runCompletedRecordId,
|
|
240
|
+
runCompletionDigest,
|
|
237
241
|
runIdForSubmission,
|
|
238
242
|
runStartedBatchId,
|
|
239
243
|
runStartedRecordId,
|
|
@@ -557,7 +561,7 @@ export class RecoveryReport extends Schema.Class<RecoveryReport>(
|
|
|
557
561
|
decision: RecoveryDecision,
|
|
558
562
|
/**
|
|
559
563
|
* `repaired` = executed; `deferred` = a claiming worker must finish it; `none` = settled;
|
|
560
|
-
* `unknown` =
|
|
564
|
+
* `unknown` = this Submission is parked on an Unknown Outcome awaiting the authorized
|
|
561
565
|
* DUR-017 resolution path — the settlement obligation stays visible while no worker permit
|
|
562
566
|
* is consumed (durability §16).
|
|
563
567
|
*/
|
|
@@ -713,6 +717,10 @@ export type DurableWorkerRequirements<
|
|
|
713
717
|
| AgentCompletionProjectionRequirements<AgentValue>;
|
|
714
718
|
|
|
715
719
|
export interface DurableRuntimeConfigOptions {
|
|
720
|
+
/**
|
|
721
|
+
* Deployment identity and the conservative operation version for direct execution without
|
|
722
|
+
* a registered binding. Reuse asserts unchanged handler, Step, and idempotency semantics.
|
|
723
|
+
*/
|
|
716
724
|
readonly deploymentId: DeploymentId;
|
|
717
725
|
readonly producerId: ProducerId;
|
|
718
726
|
/** `awaitSettlement` ledger re-check cadence when no wake arrives (default 500ms). */
|
|
@@ -954,31 +962,6 @@ const declaredApplicationCalls = Effect.fn("DurableAgentRuntime.declaredApplicat
|
|
|
954
962
|
declaredToolCalls(messages).pipe(Effect.map(({ application }) => application)),
|
|
955
963
|
);
|
|
956
964
|
|
|
957
|
-
/** Rebuild the exact JSON text that the engine decoded from one terminal no-Tool response. */
|
|
958
|
-
const terminalAssistantText = Effect.fn("DurableAgentRuntime.terminalAssistantText")(
|
|
959
|
-
(messages: PersistedJson): Effect.Effect<string, RunJournalError> =>
|
|
960
|
-
decodePrompt(messages).pipe(
|
|
961
|
-
Effect.mapError((cause) =>
|
|
962
|
-
RunJournalError.make({
|
|
963
|
-
message: "ModelResponseRecorded messages are not Schema-encoded Prompt messages",
|
|
964
|
-
cause,
|
|
965
|
-
}),
|
|
966
|
-
),
|
|
967
|
-
Effect.map((prompt) => {
|
|
968
|
-
let text = "";
|
|
969
|
-
|
|
970
|
-
for (const message of prompt.content) {
|
|
971
|
-
if (message.role !== "assistant") continue;
|
|
972
|
-
for (const part of message.content) {
|
|
973
|
-
if (part.type === "text") text += part.text;
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
return text;
|
|
978
|
-
}),
|
|
979
|
-
),
|
|
980
|
-
);
|
|
981
|
-
|
|
982
965
|
/**
|
|
983
966
|
* The declared-but-unsettled Tool batch of one Run's last committed Turn (§2.4 batch resume):
|
|
984
967
|
* every declared call in canonical encoded form plus the recorded results of the calls that
|
|
@@ -986,6 +969,7 @@ const terminalAssistantText = Effect.fn("DurableAgentRuntime.terminalAssistantTe
|
|
|
986
969
|
* path). `undefined` when the Run's journal ends at a complete Turn boundary.
|
|
987
970
|
*/
|
|
988
971
|
interface PendingToolBatch {
|
|
972
|
+
readonly toolOperations?: ReadonlyArray<ToolOperation> | undefined;
|
|
989
973
|
readonly toolParameterRejections?: ReadonlyArray<ToolParameterRejection> | undefined;
|
|
990
974
|
readonly toolExposure?: Snapshot | undefined;
|
|
991
975
|
readonly turn: number;
|
|
@@ -1025,7 +1009,7 @@ interface AttemptLineage {
|
|
|
1025
1009
|
interface OpenCallReview {
|
|
1026
1010
|
/** No proof either way: these calls must become Unknown Outcomes (never auto-replayed). */
|
|
1027
1011
|
readonly uncertain: Array<OpenToolCallEvidence>;
|
|
1028
|
-
/**
|
|
1012
|
+
/** Reconciliation or current execution support is unproven: defer without marking Unknown. */
|
|
1029
1013
|
readonly unproven: Array<OpenToolCallEvidence>;
|
|
1030
1014
|
/** Proven safe to re-execute (never-started / safe-retry / declared idempotent contract). */
|
|
1031
1015
|
readonly retryable: Array<OpenToolCallEvidence>;
|
|
@@ -1077,6 +1061,89 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1077
1061
|
const withCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>): Effect.Effect<A, E> =>
|
|
1078
1062
|
Effect.provideService(effect, Crypto.Crypto, crypto);
|
|
1079
1063
|
|
|
1064
|
+
const contractsFor = (definition: Agent.AnyDefinition) => {
|
|
1065
|
+
const registered = registeredBindings.filter((binding) => binding.agentId === definition.id);
|
|
1066
|
+
const binding = registered.length === 1 ? registered[0] : undefined;
|
|
1067
|
+
|
|
1068
|
+
// Attempts may restore the accepted Run policy on a Definition copy. Replay contracts
|
|
1069
|
+
// describe Tools, so retain the registration's semantic versions while all declarations
|
|
1070
|
+
// that affect those contracts are unchanged. Replaced Tools or completion declarations
|
|
1071
|
+
// must not inherit another executable's replay authority.
|
|
1072
|
+
const current =
|
|
1073
|
+
binding?.definition.toolkit === definition.toolkit &&
|
|
1074
|
+
binding.definition.completion === definition.completion &&
|
|
1075
|
+
binding.definition.completionFromTools === definition.completionFromTools
|
|
1076
|
+
? binding
|
|
1077
|
+
: undefined;
|
|
1078
|
+
|
|
1079
|
+
const contracts = current?.digests.replay?.tools;
|
|
1080
|
+
|
|
1081
|
+
return contracts === undefined
|
|
1082
|
+
? withCrypto(
|
|
1083
|
+
toolReplayContracts(definition, undefined, current?.digests.tools ?? config.deploymentId),
|
|
1084
|
+
)
|
|
1085
|
+
: Effect.succeed(contracts);
|
|
1086
|
+
};
|
|
1087
|
+
|
|
1088
|
+
const currentOperationsFor = Effect.fnUntraced(function* (agentId: AgentId) {
|
|
1089
|
+
const registered = registeredBindings.filter((binding) => binding.agentId === agentId);
|
|
1090
|
+
const binding = registered.length === 1 ? registered[0] : undefined;
|
|
1091
|
+
|
|
1092
|
+
return binding === undefined
|
|
1093
|
+
? undefined
|
|
1094
|
+
: {
|
|
1095
|
+
definition: binding.definition,
|
|
1096
|
+
contracts: yield* contractsFor(binding.definition),
|
|
1097
|
+
};
|
|
1098
|
+
});
|
|
1099
|
+
|
|
1100
|
+
const operationsFor = (records: ReadonlyArray<CanonicalRecordEnvelope>, runId: RunId) => {
|
|
1101
|
+
const operations = new Map<string, ToolOperation>();
|
|
1102
|
+
|
|
1103
|
+
for (const {
|
|
1104
|
+
record: { payload },
|
|
1105
|
+
} of records) {
|
|
1106
|
+
if (payload._tag !== "ModelResponseRecorded" || payload.runId !== runId) continue;
|
|
1107
|
+
for (const operation of payload.toolOperations ?? [])
|
|
1108
|
+
operations.set(operation.toolCallId, operation);
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
return operations;
|
|
1112
|
+
};
|
|
1113
|
+
|
|
1114
|
+
const supportsOperation = (
|
|
1115
|
+
submission: SubmissionSnapshot,
|
|
1116
|
+
call: OpenToolCallEvidence,
|
|
1117
|
+
operation: ToolOperation | undefined,
|
|
1118
|
+
prepared: ToolCallPrepared | undefined,
|
|
1119
|
+
current:
|
|
1120
|
+
| {
|
|
1121
|
+
readonly definition: Agent.AnyDefinition;
|
|
1122
|
+
readonly contracts: Readonly<Record<string, Digest>>;
|
|
1123
|
+
}
|
|
1124
|
+
| undefined,
|
|
1125
|
+
): boolean => {
|
|
1126
|
+
const tool = current?.definition.toolkit.tools[call.toolName];
|
|
1127
|
+
|
|
1128
|
+
const replay =
|
|
1129
|
+
operation?.replay ?? prepared?.replay ?? submission.agentDigests.replay?.tools[call.toolName];
|
|
1130
|
+
|
|
1131
|
+
return (
|
|
1132
|
+
tool !== undefined &&
|
|
1133
|
+
replay !== undefined &&
|
|
1134
|
+
current?.contracts[call.toolName] === replay &&
|
|
1135
|
+
(operation === undefined ||
|
|
1136
|
+
(operation.toolName === call.toolName &&
|
|
1137
|
+
operation.executionClass === getToolExecutionClass(tool) &&
|
|
1138
|
+
operation.executionKind === getToolExecutionKind(tool.annotations))) &&
|
|
1139
|
+
(prepared === undefined ||
|
|
1140
|
+
(prepared.toolName === call.toolName &&
|
|
1141
|
+
(prepared.executionClass === undefined ||
|
|
1142
|
+
prepared.executionClass === getToolExecutionClass(tool)) &&
|
|
1143
|
+
(prepared.executionKind ?? "ordinary") === getToolExecutionKind(tool.annotations)))
|
|
1144
|
+
);
|
|
1145
|
+
};
|
|
1146
|
+
|
|
1080
1147
|
const hit = (
|
|
1081
1148
|
location: DurableRuntimeFailpointLocation,
|
|
1082
1149
|
): Effect.Effect<void, DurableRuntimeFailpointError> => failpoint.hit(location);
|
|
@@ -1245,6 +1312,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1245
1312
|
threadId: ThreadId,
|
|
1246
1313
|
throughSequence: CanonicalSequence,
|
|
1247
1314
|
submissionIds: ReadonlyArray<SubmissionId>,
|
|
1315
|
+
journalOwner?: RunId,
|
|
1248
1316
|
): Effect.fn.Return<
|
|
1249
1317
|
{
|
|
1250
1318
|
readonly canonical: Stream.Stream<
|
|
@@ -1283,7 +1351,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1283
1351
|
|
|
1284
1352
|
if (
|
|
1285
1353
|
!submissionIds.every((id) => state.submissionIds.includes(id)) ||
|
|
1286
|
-
state.seed.runId !== runIdForSubmission(state.submissionId)
|
|
1354
|
+
state.seed.runId !== runIdForSubmission(state.submissionId) ||
|
|
1355
|
+
(journalOwner !== undefined && state.seed.runId !== journalOwner)
|
|
1287
1356
|
)
|
|
1288
1357
|
return full;
|
|
1289
1358
|
|
|
@@ -1550,6 +1619,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1550
1619
|
const preparedTurns = new Set<number>();
|
|
1551
1620
|
const settledIds = new Set<string>();
|
|
1552
1621
|
const resolvedIds = new Set<string>();
|
|
1622
|
+
const unknownIds = new Set<string>();
|
|
1553
1623
|
const requested: Array<PendingApprovalEvidence> = [];
|
|
1554
1624
|
const decidedIds = new Set<string>();
|
|
1555
1625
|
|
|
@@ -1611,8 +1681,17 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1611
1681
|
if (payload.runId === runId) settledIds.add(payload.toolCallId);
|
|
1612
1682
|
break;
|
|
1613
1683
|
}
|
|
1684
|
+
case "ToolCallUnknown": {
|
|
1685
|
+
if (payload.runId === runId) unknownIds.add(payload.toolCallId);
|
|
1686
|
+
break;
|
|
1687
|
+
}
|
|
1614
1688
|
case "ToolCallResolved": {
|
|
1615
|
-
if (
|
|
1689
|
+
if (
|
|
1690
|
+
payload.runId === runId &&
|
|
1691
|
+
(payload.resolution === "completed-with-result" ||
|
|
1692
|
+
payload.resolution === "failed-with-error")
|
|
1693
|
+
)
|
|
1694
|
+
resolvedIds.add(payload.toolCallId);
|
|
1616
1695
|
break;
|
|
1617
1696
|
}
|
|
1618
1697
|
case "ToolApprovalRequested": {
|
|
@@ -1766,10 +1845,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1766
1845
|
preparedKinds.get(toolCallPreparedRecordId(runId, call.turn, call.toolCallId)) ===
|
|
1767
1846
|
"orchestration";
|
|
1768
1847
|
|
|
1769
|
-
const openWorkerCalls = allOpenCalls.filter(
|
|
1848
|
+
const openWorkerCalls = allOpenCalls.filter(
|
|
1849
|
+
(call) => isPreparedWorker(call) && !unknownIds.has(call.toolCallId),
|
|
1850
|
+
);
|
|
1770
1851
|
|
|
1771
1852
|
const openToolCalls = allOpenCalls.filter(
|
|
1772
|
-
(call) =>
|
|
1853
|
+
(call) =>
|
|
1854
|
+
unknownIds.has(call.toolCallId) || (!isPreparedDelegation(call) && !isPreparedWorker(call)),
|
|
1773
1855
|
);
|
|
1774
1856
|
|
|
1775
1857
|
let declaredPendingBatch: DeclaredPendingBatchEvidence | undefined;
|
|
@@ -1802,6 +1884,53 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1802
1884
|
});
|
|
1803
1885
|
});
|
|
1804
1886
|
|
|
1887
|
+
/** Verify immutable preparation against the original declaration before any recovery side effect. */
|
|
1888
|
+
const validatePreparedCall = Effect.fnUntraced(function* (
|
|
1889
|
+
prepared: ToolCallPrepared,
|
|
1890
|
+
declared: Effect.Success<ReturnType<typeof declaredToolCalls>>,
|
|
1891
|
+
operations: ReadonlyArray<ToolOperation> | undefined,
|
|
1892
|
+
) {
|
|
1893
|
+
const call = declared.application.find((call) => call.id === prepared.toolCallId);
|
|
1894
|
+
|
|
1895
|
+
if (call === undefined || prepared.toolName !== call.name)
|
|
1896
|
+
return yield* RunJournalError.make({
|
|
1897
|
+
message: `Prepared Tool ${prepared.toolCallId} differs from its original declaration`,
|
|
1898
|
+
});
|
|
1899
|
+
|
|
1900
|
+
const parameters = yield* decodePersisted(call.params).pipe(
|
|
1901
|
+
Effect.mapError((cause) =>
|
|
1902
|
+
RunJournalError.make({ message: "Invalid canonical Tool parameters", cause }),
|
|
1903
|
+
),
|
|
1904
|
+
);
|
|
1905
|
+
|
|
1906
|
+
const declarationDigest = yield* withCrypto(digestJson(parameters)).pipe(
|
|
1907
|
+
Effect.mapError((cause) =>
|
|
1908
|
+
RunJournalError.make({ message: "Cannot verify declared Tool parameters", cause }),
|
|
1909
|
+
),
|
|
1910
|
+
);
|
|
1911
|
+
|
|
1912
|
+
const preparedDigest = yield* withCrypto(digestJson(prepared.parameters)).pipe(
|
|
1913
|
+
Effect.mapError((cause) =>
|
|
1914
|
+
RunJournalError.make({ message: "Cannot verify prepared Tool parameters", cause }),
|
|
1915
|
+
),
|
|
1916
|
+
);
|
|
1917
|
+
|
|
1918
|
+
const operation = operations?.find((entry) => entry.toolCallId === prepared.toolCallId);
|
|
1919
|
+
|
|
1920
|
+
if (
|
|
1921
|
+
prepared.parametersDigest !== declarationDigest ||
|
|
1922
|
+
preparedDigest !== declarationDigest ||
|
|
1923
|
+
(operation !== undefined &&
|
|
1924
|
+
((prepared.executionClass !== undefined &&
|
|
1925
|
+
prepared.executionClass !== operation.executionClass) ||
|
|
1926
|
+
(prepared.executionKind ?? "ordinary") !== operation.executionKind ||
|
|
1927
|
+
(prepared.replay !== undefined && prepared.replay !== operation.replay)))
|
|
1928
|
+
)
|
|
1929
|
+
return yield* RunJournalError.make({
|
|
1930
|
+
message: `Prepared Tool ${prepared.toolCallId} differs from its original operation`,
|
|
1931
|
+
});
|
|
1932
|
+
});
|
|
1933
|
+
|
|
1805
1934
|
/**
|
|
1806
1935
|
* The declared-but-unsettled Tool batch of the Run's LAST committed Turn (§2.4). Keyed on
|
|
1807
1936
|
* SETTLED coverage deliberately: a call closed by a never-started/safe-retry `ToolCallResolved`
|
|
@@ -1817,6 +1946,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1817
1946
|
| {
|
|
1818
1947
|
readonly turn: number;
|
|
1819
1948
|
readonly messages: PersistedJson;
|
|
1949
|
+
readonly toolOperations?: ReadonlyArray<ToolOperation> | undefined;
|
|
1820
1950
|
readonly toolParameterRejections?: ReadonlyArray<ToolParameterRejection> | undefined;
|
|
1821
1951
|
readonly toolExposure?: Snapshot | undefined;
|
|
1822
1952
|
}
|
|
@@ -1825,6 +1955,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1825
1955
|
const settledByCallId = new Map<
|
|
1826
1956
|
string,
|
|
1827
1957
|
{
|
|
1958
|
+
readonly recordId?: RecordId;
|
|
1959
|
+
readonly toolName?: string;
|
|
1828
1960
|
readonly result: PersistedJson;
|
|
1829
1961
|
readonly isFailure: boolean;
|
|
1830
1962
|
readonly budgetRejected?: true;
|
|
@@ -1842,6 +1974,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1842
1974
|
turn: payload.turn,
|
|
1843
1975
|
messages: payload.messages,
|
|
1844
1976
|
toolParameterRejections: payload.toolParameterRejections,
|
|
1977
|
+
toolOperations: payload.toolOperations,
|
|
1845
1978
|
...(payload.toolExposure === undefined ? {} : { toolExposure: payload.toolExposure }),
|
|
1846
1979
|
};
|
|
1847
1980
|
}
|
|
@@ -1849,6 +1982,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1849
1982
|
}
|
|
1850
1983
|
if (payload._tag === "ToolCallSettled" && payload.runId === runId) {
|
|
1851
1984
|
settledByCallId.set(payload.toolCallId, {
|
|
1985
|
+
recordId: envelope.record.recordId,
|
|
1986
|
+
toolName: payload.toolName,
|
|
1852
1987
|
result: payload.result,
|
|
1853
1988
|
isFailure: payload.isFailure,
|
|
1854
1989
|
...(payload.budgetRejected === true ? { budgetRejected: true } : {}),
|
|
@@ -1862,6 +1997,39 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1862
1997
|
if (declared.application.length === 0) return undefined;
|
|
1863
1998
|
const calls = declared.all;
|
|
1864
1999
|
|
|
2000
|
+
if (lastResponse.toolOperations !== undefined) {
|
|
2001
|
+
const identities = new Set<string>();
|
|
2002
|
+
|
|
2003
|
+
for (const operation of lastResponse.toolOperations) {
|
|
2004
|
+
if (
|
|
2005
|
+
identities.has(operation.toolCallId) ||
|
|
2006
|
+
!declared.application.some(
|
|
2007
|
+
(call) => call.id === operation.toolCallId && call.name === operation.toolName,
|
|
2008
|
+
)
|
|
2009
|
+
)
|
|
2010
|
+
return yield* RunJournalError.make({
|
|
2011
|
+
message: "Operation evidence differs from the declared Tool batch",
|
|
2012
|
+
});
|
|
2013
|
+
identities.add(operation.toolCallId);
|
|
2014
|
+
}
|
|
2015
|
+
if (identities.size !== declared.application.length)
|
|
2016
|
+
return yield* RunJournalError.make({
|
|
2017
|
+
message: "Declared Tool batch has incomplete operation evidence",
|
|
2018
|
+
});
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
for (const {
|
|
2022
|
+
record: { payload },
|
|
2023
|
+
} of records) {
|
|
2024
|
+
if (
|
|
2025
|
+
payload._tag !== "ToolCallPrepared" ||
|
|
2026
|
+
payload.runId !== runId ||
|
|
2027
|
+
payload.turn !== lastResponse.turn
|
|
2028
|
+
)
|
|
2029
|
+
continue;
|
|
2030
|
+
yield* validatePreparedCall(payload, declared, lastResponse.toolOperations);
|
|
2031
|
+
}
|
|
2032
|
+
|
|
1865
2033
|
for (const part of declared.providerResults) {
|
|
1866
2034
|
const result = yield* decodePersisted(part.result).pipe(
|
|
1867
2035
|
Effect.mapError((cause) =>
|
|
@@ -1889,7 +2057,22 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1889
2057
|
});
|
|
1890
2058
|
}
|
|
1891
2059
|
if (recorded !== undefined) {
|
|
1892
|
-
|
|
2060
|
+
const { recordId, toolName, ...result } = recorded;
|
|
2061
|
+
|
|
2062
|
+
if (
|
|
2063
|
+
call.providerExecuted !== true &&
|
|
2064
|
+
(toolName !== call.name ||
|
|
2065
|
+
recordId !==
|
|
2066
|
+
toolCallSettledRecordId(
|
|
2067
|
+
runId,
|
|
2068
|
+
lastResponse.turn,
|
|
2069
|
+
Schema.decodeSync(ToolCallId)(call.id),
|
|
2070
|
+
))
|
|
2071
|
+
)
|
|
2072
|
+
return yield* RunJournalError.make({
|
|
2073
|
+
message: `Settled Tool ${call.id} differs from its original declaration`,
|
|
2074
|
+
});
|
|
2075
|
+
settled.push({ id: call.id, ...result });
|
|
1893
2076
|
}
|
|
1894
2077
|
}
|
|
1895
2078
|
|
|
@@ -1915,6 +2098,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
1915
2098
|
responseRecordId: modelResponseRecordId(runId, lastResponse.turn),
|
|
1916
2099
|
messages: lastResponse.messages,
|
|
1917
2100
|
toolParameterRejections: lastResponse.toolParameterRejections,
|
|
2101
|
+
toolOperations: lastResponse.toolOperations,
|
|
1918
2102
|
...(lastResponse.toolExposure === undefined
|
|
1919
2103
|
? {}
|
|
1920
2104
|
: { toolExposure: lastResponse.toolExposure }),
|
|
@@ -2027,9 +2211,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2027
2211
|
});
|
|
2028
2212
|
|
|
2029
2213
|
/**
|
|
2030
|
-
*
|
|
2031
|
-
*
|
|
2032
|
-
*
|
|
2214
|
+
* Existing ownership-free settlement protocol for queued aborts and joined outcomes.
|
|
2215
|
+
* Its canonical settlement reservation authorizes the append; it never authorizes tool
|
|
2216
|
+
* execution, unknown-resolution appends, or repair audits beside a live writer.
|
|
2033
2217
|
*/
|
|
2034
2218
|
const attemptContextAtTail = Effect.fn("DurableAgentRuntime.attemptContextAtTail")(function* (
|
|
2035
2219
|
threadId: ThreadId,
|
|
@@ -2295,83 +2479,174 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2295
2479
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
2296
2480
|
openCalls: ReadonlyArray<OpenToolCallEvidence>,
|
|
2297
2481
|
knownIds: Set<string>,
|
|
2298
|
-
|
|
2482
|
+
current?: {
|
|
2483
|
+
readonly definition: Agent.AnyDefinition;
|
|
2484
|
+
readonly contracts: Readonly<Record<string, Digest>>;
|
|
2485
|
+
},
|
|
2299
2486
|
): Effect.fn.Return<OpenCallReview, DurableWorkerFailure> {
|
|
2300
2487
|
const submissionId = submission.submissionId;
|
|
2301
2488
|
const runId = runIdForSubmission(submissionId);
|
|
2489
|
+
|
|
2490
|
+
yield* pendingToolBatchFor(records, runId);
|
|
2491
|
+
const operations = operationsFor(records, runId);
|
|
2302
2492
|
const preparedByCallId = new Map<string, ToolCallPrepared>();
|
|
2303
2493
|
|
|
2304
|
-
for (const
|
|
2305
|
-
const payload =
|
|
2494
|
+
for (const { record } of records) {
|
|
2495
|
+
const payload = record.payload;
|
|
2306
2496
|
|
|
2307
|
-
if (
|
|
2497
|
+
if (
|
|
2498
|
+
payload._tag === "ToolCallPrepared" &&
|
|
2499
|
+
payload.runId === runId &&
|
|
2500
|
+
openCalls.some(
|
|
2501
|
+
(call) => call.toolCallId === payload.toolCallId && call.turn === payload.turn,
|
|
2502
|
+
)
|
|
2503
|
+
) {
|
|
2504
|
+
if (
|
|
2505
|
+
preparedByCallId.has(payload.toolCallId) ||
|
|
2506
|
+
record.recordId !== toolCallPreparedRecordId(runId, payload.turn, payload.toolCallId)
|
|
2507
|
+
)
|
|
2508
|
+
return yield* RunJournalError.make({
|
|
2509
|
+
message: `Prepared Tool ${payload.toolCallId} has inconsistent canonical identity`,
|
|
2510
|
+
});
|
|
2308
2511
|
preparedByCallId.set(payload.toolCallId, payload);
|
|
2309
2512
|
}
|
|
2310
2513
|
}
|
|
2311
|
-
|
|
2514
|
+
// Validate all open declarations before the first reconciler call. A missing response or
|
|
2515
|
+
// a damaged turn number must never bypass the original parameters/identity checks.
|
|
2516
|
+
for (const call of openCalls) {
|
|
2517
|
+
const prepared = preparedByCallId.get(call.toolCallId);
|
|
2312
2518
|
|
|
2313
|
-
|
|
2314
|
-
|
|
2519
|
+
const responses = records.filter(
|
|
2520
|
+
({ record }) =>
|
|
2521
|
+
record.payload._tag === "ModelResponseRecorded" &&
|
|
2522
|
+
record.payload.runId === runId &&
|
|
2523
|
+
record.payload.turn === call.turn,
|
|
2524
|
+
);
|
|
2525
|
+
|
|
2526
|
+
const response = responses[0]?.record;
|
|
2527
|
+
|
|
2528
|
+
if (
|
|
2529
|
+
prepared === undefined ||
|
|
2530
|
+
prepared.turn !== call.turn ||
|
|
2531
|
+
prepared.toolName !== call.toolName ||
|
|
2532
|
+
responses.length !== 1 ||
|
|
2533
|
+
response?.payload._tag !== "ModelResponseRecorded" ||
|
|
2534
|
+
response.recordId !== modelResponseRecordId(runId, call.turn) ||
|
|
2535
|
+
response.payload.turnId !== turnIdForRun(runId, call.turn)
|
|
2536
|
+
)
|
|
2537
|
+
return yield* RunJournalError.make({
|
|
2538
|
+
message: `Prepared Tool ${call.toolCallId} has no unique original response`,
|
|
2539
|
+
});
|
|
2540
|
+
if (
|
|
2541
|
+
(yield* withCrypto(digestJson(response.payload.messages))) !==
|
|
2542
|
+
response.payload.messagesDigest
|
|
2543
|
+
)
|
|
2544
|
+
return yield* RunJournalError.make({
|
|
2545
|
+
message: `Original response for Tool ${call.toolCallId} has an invalid digest`,
|
|
2546
|
+
});
|
|
2547
|
+
yield* validatePreparedCall(
|
|
2548
|
+
prepared,
|
|
2549
|
+
yield* declaredToolCalls(response.payload.messages),
|
|
2550
|
+
response.payload.toolOperations,
|
|
2551
|
+
);
|
|
2315
2552
|
}
|
|
2553
|
+
|
|
2554
|
+
const intents = new Map(
|
|
2555
|
+
snapshot.unknownResolutions.map((intent) => [intent.toolCallId, intent]),
|
|
2556
|
+
);
|
|
2557
|
+
|
|
2316
2558
|
const review: OpenCallReview = { uncertain: [], unproven: [], retryable: [], recovered: 0 };
|
|
2317
2559
|
let recovered = 0;
|
|
2318
2560
|
|
|
2319
2561
|
for (const call of openCalls) {
|
|
2562
|
+
const prepared = preparedByCallId.get(call.toolCallId);
|
|
2563
|
+
const operation = operations.get(call.toolCallId);
|
|
2564
|
+
|
|
2565
|
+
const supported = supportsOperation(submission, call, operation, prepared, current);
|
|
2566
|
+
|
|
2320
2567
|
const intent = intents.get(call.toolCallId);
|
|
2568
|
+
const author = intent?.author ?? RECONCILER_AUTHOR;
|
|
2569
|
+
|
|
2570
|
+
const reason =
|
|
2571
|
+
intent?.reason ?? "A registered reconciliation policy recovered the external outcome";
|
|
2572
|
+
|
|
2573
|
+
const neverStarted = Effect.gen(function* () {
|
|
2574
|
+
if (current === undefined) {
|
|
2575
|
+
// Low-level drivers supply their current Agent only when claiming. Missing metadata
|
|
2576
|
+
// cannot prove retirement or authorize execution; the claimant checks the contract.
|
|
2577
|
+
review.unproven.push(call);
|
|
2578
|
+
|
|
2579
|
+
return;
|
|
2580
|
+
}
|
|
2581
|
+
if (supported) {
|
|
2582
|
+
review.retryable.push(call);
|
|
2583
|
+
|
|
2584
|
+
return;
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
const unavailable = yield* Schema.encodeEffect(ToolUnavailable)(
|
|
2588
|
+
ToolUnavailable.make({
|
|
2589
|
+
toolName: call.toolName,
|
|
2590
|
+
execution: "not-executed",
|
|
2591
|
+
message:
|
|
2592
|
+
"This operation never started and its original implementation is unavailable. Continue with the current tools.",
|
|
2593
|
+
}),
|
|
2594
|
+
).pipe(Effect.flatMap(decodePersisted), Effect.orDie);
|
|
2595
|
+
|
|
2596
|
+
yield* hit("tools:before-unavailable-append");
|
|
2597
|
+
yield* appendClosedCall(ctx, submissionId, knownIds, call, {
|
|
2598
|
+
result: { value: unavailable, isFailure: true },
|
|
2599
|
+
resolution: "never-started",
|
|
2600
|
+
author,
|
|
2601
|
+
reason,
|
|
2602
|
+
});
|
|
2603
|
+
yield* hit("tools:after-unavailable-append");
|
|
2604
|
+
recovered += 1;
|
|
2605
|
+
});
|
|
2321
2606
|
|
|
2322
2607
|
if (intent !== undefined) {
|
|
2323
2608
|
switch (intent.resolution._tag) {
|
|
2324
|
-
case "AbortSubmission":
|
|
2325
|
-
// The paired abort intent settles the Submission; the SettleAborted executor
|
|
2326
|
-
// records the `ToolCallUnknown` audit — nothing resolves here (durability §13).
|
|
2609
|
+
case "AbortSubmission":
|
|
2327
2610
|
continue;
|
|
2328
|
-
}
|
|
2329
2611
|
case "CompletedWithResult": {
|
|
2330
2612
|
yield* appendClosedCall(ctx, submissionId, knownIds, call, {
|
|
2331
|
-
result: {
|
|
2332
|
-
value: intent.resolution.result,
|
|
2333
|
-
isFailure: intent.resolution.isFailure,
|
|
2334
|
-
},
|
|
2613
|
+
result: { value: intent.resolution.result, isFailure: intent.resolution.isFailure },
|
|
2335
2614
|
resolution: intent.resolution.isFailure
|
|
2336
2615
|
? "failed-with-error"
|
|
2337
2616
|
: "completed-with-result",
|
|
2338
|
-
author
|
|
2339
|
-
reason
|
|
2617
|
+
author,
|
|
2618
|
+
reason,
|
|
2340
2619
|
});
|
|
2341
2620
|
recovered += 1;
|
|
2342
2621
|
continue;
|
|
2343
2622
|
}
|
|
2344
2623
|
case "NeverHappened": {
|
|
2345
|
-
yield*
|
|
2346
|
-
resolution: "never-started",
|
|
2347
|
-
author: intent.author,
|
|
2348
|
-
reason: intent.reason,
|
|
2349
|
-
});
|
|
2350
|
-
review.retryable.push(call);
|
|
2624
|
+
yield* neverStarted;
|
|
2351
2625
|
continue;
|
|
2352
2626
|
}
|
|
2353
2627
|
case "SafeToRetry": {
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
});
|
|
2359
|
-
review.retryable.push(call);
|
|
2628
|
+
// Permission to repeat the original operation is not proof it never happened.
|
|
2629
|
+
if (current === undefined) review.unproven.push(call);
|
|
2630
|
+
else if (supported) review.retryable.push(call);
|
|
2631
|
+
else review.uncertain.push(call);
|
|
2360
2632
|
continue;
|
|
2361
2633
|
}
|
|
2362
2634
|
}
|
|
2363
2635
|
}
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2636
|
+
const tool = current?.definition.toolkit.tools[call.toolName];
|
|
2637
|
+
|
|
2638
|
+
if (
|
|
2639
|
+
current === undefined &&
|
|
2640
|
+
(operation?.executionClass ?? prepared?.executionClass) === "idempotent"
|
|
2641
|
+
) {
|
|
2642
|
+
review.unproven.push(call);
|
|
2643
|
+
continue;
|
|
2644
|
+
}
|
|
2645
|
+
if (supported && tool !== undefined && getToolExecutionClass(tool) === "idempotent") {
|
|
2367
2646
|
review.retryable.push(call);
|
|
2368
2647
|
continue;
|
|
2369
2648
|
}
|
|
2370
|
-
const prepared = preparedByCallId.get(call.toolCallId);
|
|
2371
|
-
|
|
2372
2649
|
if (prepared === undefined) {
|
|
2373
|
-
// Open evidence without its prepared record is unreadable state: stay blocked,
|
|
2374
|
-
// never guess (fail-closed).
|
|
2375
2650
|
review.unproven.push(call);
|
|
2376
2651
|
continue;
|
|
2377
2652
|
}
|
|
@@ -2387,6 +2662,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2387
2662
|
toolName: prepared.toolName,
|
|
2388
2663
|
parameters: prepared.parameters,
|
|
2389
2664
|
parametersDigest: prepared.parametersDigest,
|
|
2665
|
+
...(prepared.executionKind === undefined
|
|
2666
|
+
? {}
|
|
2667
|
+
: { executionKind: prepared.executionKind }),
|
|
2668
|
+
...(prepared.executionClass === undefined
|
|
2669
|
+
? {}
|
|
2670
|
+
: { executionClass: prepared.executionClass }),
|
|
2671
|
+
...(prepared.replay === undefined ? {} : { replay: prepared.replay }),
|
|
2390
2672
|
}),
|
|
2391
2673
|
)
|
|
2392
2674
|
.pipe(
|
|
@@ -2405,15 +2687,20 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
2405
2687
|
yield* appendClosedCall(ctx, submissionId, knownIds, call, {
|
|
2406
2688
|
result: { value: decision.result, isFailure: decision.isFailure },
|
|
2407
2689
|
resolution: decision.isFailure ? "failed-with-error" : "completed-with-result",
|
|
2408
|
-
author
|
|
2409
|
-
reason
|
|
2690
|
+
author,
|
|
2691
|
+
reason,
|
|
2410
2692
|
});
|
|
2411
2693
|
recovered += 1;
|
|
2412
2694
|
continue;
|
|
2413
2695
|
}
|
|
2414
|
-
case "NeverStarted":
|
|
2696
|
+
case "NeverStarted": {
|
|
2697
|
+
yield* neverStarted;
|
|
2698
|
+
continue;
|
|
2699
|
+
}
|
|
2415
2700
|
case "SafeToRetry": {
|
|
2416
|
-
review.
|
|
2701
|
+
if (current === undefined) review.unproven.push(call);
|
|
2702
|
+
else if (supported) review.retryable.push(call);
|
|
2703
|
+
else review.uncertain.push(call);
|
|
2417
2704
|
continue;
|
|
2418
2705
|
}
|
|
2419
2706
|
case "Uncertain": {
|
|
@@ -3445,8 +3732,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3445
3732
|
* Join a settled child after parent abort or expiry without running application handlers.
|
|
3446
3733
|
* The canonical child outcome stays intact; the parent result records why it cannot continue.
|
|
3447
3734
|
*/
|
|
3448
|
-
const
|
|
3449
|
-
"DurableAgentRuntime.
|
|
3735
|
+
const joinSettledChildWithoutHandler = Effect.fn(
|
|
3736
|
+
"DurableAgentRuntime.joinSettledChildWithoutHandler",
|
|
3450
3737
|
)(function* (
|
|
3451
3738
|
ctx: AttemptAppendContext,
|
|
3452
3739
|
parent: SubmissionSnapshot,
|
|
@@ -3455,14 +3742,14 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3455
3742
|
reservation: ChildBudgetReservationSnapshot,
|
|
3456
3743
|
toolCallId: ToolCallId,
|
|
3457
3744
|
childSubmissionId: SubmissionId,
|
|
3458
|
-
reason: "aborted" | "duration",
|
|
3745
|
+
reason: "aborted" | "duration" | "unavailable",
|
|
3459
3746
|
): Effect.fn.Return<void, DurableWorkerFailure> {
|
|
3460
3747
|
const runId = runIdForSubmission(parent.submissionId);
|
|
3461
3748
|
const request = subagent.requested.get(toolCallId);
|
|
3462
3749
|
|
|
3463
3750
|
if (request === undefined) {
|
|
3464
3751
|
return yield* LedgerError.make({
|
|
3465
|
-
operation: "
|
|
3752
|
+
operation: "joinSettledChildWithoutHandler",
|
|
3466
3753
|
message: `Tool Call ${toolCallId} has an attached child but no canonical SubagentRequested record`,
|
|
3467
3754
|
});
|
|
3468
3755
|
}
|
|
@@ -3477,22 +3764,33 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3477
3764
|
|
|
3478
3765
|
if (verification._tag === "mismatch") {
|
|
3479
3766
|
return yield* LedgerError.make({
|
|
3480
|
-
operation: "
|
|
3767
|
+
operation: "joinSettledChildWithoutHandler",
|
|
3481
3768
|
message: `Child Settlement verification failed for Tool Call ${toolCallId}: ${verification.message}`,
|
|
3482
3769
|
});
|
|
3483
3770
|
}
|
|
3484
3771
|
const verified = verification.value;
|
|
3485
3772
|
|
|
3486
|
-
//
|
|
3487
|
-
|
|
3488
|
-
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3773
|
+
// The child's actual outcome and usage remain authoritative even when the parent can
|
|
3774
|
+
// no longer project it with the original delegation implementation.
|
|
3775
|
+
const boundedResult = yield* decodePersisted(
|
|
3776
|
+
reason === "unavailable"
|
|
3777
|
+
? {
|
|
3778
|
+
_tag: "ToolUnavailable",
|
|
3779
|
+
toolName: subagent.preparedNames.get(toolCallId) ?? request.delegationId,
|
|
3780
|
+
execution: "unavailable",
|
|
3781
|
+
message: `The original delegation implementation is unavailable. Its original child settled ${verified.outcome}; no replacement child was started.`,
|
|
3782
|
+
}
|
|
3783
|
+
: {
|
|
3784
|
+
errorTag:
|
|
3785
|
+
reason === "aborted" ? "SubagentParentAborted" : "SubagentParentDurationExceeded",
|
|
3786
|
+
message: boundedText(
|
|
3787
|
+
`The parent Submission ${reason === "aborted" ? "aborted" : "exceeded its duration"}; attached child ${childSubmissionId} settled ${verified.outcome}`,
|
|
3788
|
+
),
|
|
3789
|
+
},
|
|
3790
|
+
).pipe(Effect.orDie);
|
|
3493
3791
|
|
|
3494
3792
|
finalAccounting = yield* decodePersisted({
|
|
3495
|
-
basis: reason
|
|
3793
|
+
basis: `${reason}-conservative`,
|
|
3496
3794
|
allocation: reservation.allocation,
|
|
3497
3795
|
}).pipe(Effect.orDie);
|
|
3498
3796
|
const childRunId = runIdForSubmission(childSubmissionId);
|
|
@@ -3587,11 +3885,12 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3587
3885
|
});
|
|
3588
3886
|
|
|
3589
3887
|
/** Release unused reservations or repair and join existing children, never admit new work. */
|
|
3590
|
-
const
|
|
3888
|
+
const reconcileRetainedChildren = Effect.fn("DurableAgentRuntime.reconcileRetainedChildren")(
|
|
3591
3889
|
function* (
|
|
3592
3890
|
ctx: AttemptAppendContext,
|
|
3593
3891
|
parent: SubmissionSnapshot,
|
|
3594
3892
|
ownershipToken: OwnershipToken,
|
|
3893
|
+
unavailableCalls?: ReadonlySet<ToolCallId>,
|
|
3595
3894
|
) {
|
|
3596
3895
|
const snapshot = yield* ledger.loadRecoverySnapshot(
|
|
3597
3896
|
RecoverySnapshotRequest.make({ submissionId: parent.submissionId }),
|
|
@@ -3603,10 +3902,46 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3603
3902
|
const knownIds = knownRecordIdsOf(records);
|
|
3604
3903
|
const subagent = subagentRecordsOf(records, runId);
|
|
3605
3904
|
|
|
3905
|
+
const notExecuted = new Set<ToolCallId>();
|
|
3906
|
+
const waiting: Array<WaitingChild> = [];
|
|
3907
|
+
const indeterminate = new Set<ToolCallId>();
|
|
3908
|
+
|
|
3909
|
+
for (const toolCallId of unavailableCalls ?? []) {
|
|
3910
|
+
if (
|
|
3911
|
+
snapshot.childReservations.some(
|
|
3912
|
+
(reservation) => reservation.parentToolCallId === toolCallId,
|
|
3913
|
+
)
|
|
3914
|
+
)
|
|
3915
|
+
continue;
|
|
3916
|
+
if (subagent.requested.has(toolCallId) || subagent.started.has(toolCallId))
|
|
3917
|
+
return yield* LedgerError.make({
|
|
3918
|
+
operation: "reconcileRetainedChildren",
|
|
3919
|
+
message: "Child request has no original reservation",
|
|
3920
|
+
});
|
|
3921
|
+
notExecuted.add(toolCallId);
|
|
3922
|
+
}
|
|
3606
3923
|
for (const reservation of snapshot.childReservations) {
|
|
3607
3924
|
const toolCallId = reservation.parentToolCallId;
|
|
3608
3925
|
|
|
3609
|
-
if (
|
|
3926
|
+
if (unavailableCalls !== undefined && !unavailableCalls.has(toolCallId)) continue;
|
|
3927
|
+
const joined = subagent.joined.get(toolCallId);
|
|
3928
|
+
|
|
3929
|
+
if (joined !== undefined) {
|
|
3930
|
+
yield* applyReservationRelease(reservation.reservationId, joined.finalAccounting);
|
|
3931
|
+
continue;
|
|
3932
|
+
}
|
|
3933
|
+
if (reservation.status !== "reserved") {
|
|
3934
|
+
if (reservation.accounting !== undefined)
|
|
3935
|
+
yield* applyReservationRelease(reservation.reservationId, reservation.accounting);
|
|
3936
|
+
// An orphan release proves non-admission; it never hides an attached child.
|
|
3937
|
+
if (reservation.childSubmissionId !== undefined || subagent.started.has(toolCallId))
|
|
3938
|
+
return yield* LedgerError.make({
|
|
3939
|
+
operation: "reconcileRetainedChildren",
|
|
3940
|
+
message: "Unjoined child reservation was already released",
|
|
3941
|
+
});
|
|
3942
|
+
notExecuted.add(toolCallId);
|
|
3943
|
+
continue;
|
|
3944
|
+
}
|
|
3610
3945
|
let started = subagent.started.get(toolCallId);
|
|
3611
3946
|
const request = subagent.requested.get(toolCallId);
|
|
3612
3947
|
|
|
@@ -3616,6 +3951,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3616
3951
|
reservation.childSubmissionId === undefined
|
|
3617
3952
|
) {
|
|
3618
3953
|
yield* releaseOrphanReservation(reservation.reservationId);
|
|
3954
|
+
notExecuted.add(toolCallId);
|
|
3619
3955
|
continue;
|
|
3620
3956
|
}
|
|
3621
3957
|
const call = pending?.calls.find((candidate) => candidate.id === toolCallId);
|
|
@@ -3633,7 +3969,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3633
3969
|
request.turnId !== pending.turnId
|
|
3634
3970
|
) {
|
|
3635
3971
|
return yield* LedgerError.make({
|
|
3636
|
-
operation: "
|
|
3972
|
+
operation: "reconcileRetainedChildren",
|
|
3637
3973
|
message: `Tool Call ${toolCallId} has inconsistent canonical child request evidence`,
|
|
3638
3974
|
});
|
|
3639
3975
|
}
|
|
@@ -3646,22 +3982,37 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3646
3982
|
}),
|
|
3647
3983
|
);
|
|
3648
3984
|
|
|
3649
|
-
if (resolution._tag === "Indeterminate")
|
|
3985
|
+
if (resolution._tag === "Indeterminate") {
|
|
3986
|
+
indeterminate.add(toolCallId);
|
|
3987
|
+
continue;
|
|
3988
|
+
}
|
|
3650
3989
|
if (resolution._tag === "NotAdmitted") {
|
|
3651
3990
|
if (reservation.childSubmissionId !== undefined) {
|
|
3652
3991
|
return yield* LedgerError.make({
|
|
3653
|
-
operation: "
|
|
3992
|
+
operation: "reconcileRetainedChildren",
|
|
3654
3993
|
message: `Tool Call ${toolCallId} has an attachment but no admitted child`,
|
|
3655
3994
|
});
|
|
3656
3995
|
}
|
|
3657
|
-
|
|
3658
|
-
|
|
3996
|
+
if (unavailableCalls === undefined) {
|
|
3997
|
+
yield* releaseOrphanReservation(reservation.reservationId);
|
|
3998
|
+
notExecuted.add(toolCallId);
|
|
3999
|
+
continue;
|
|
4000
|
+
}
|
|
3659
4001
|
}
|
|
3660
|
-
// Finish only the already-admitted child's materialization and readiness. A second
|
|
3661
|
-
// authoritative lookup must still never turn a stale observation into a new admission.
|
|
3662
|
-
const admission = yield* establishChildFromRequest(parent, request, false);
|
|
3663
4002
|
|
|
3664
|
-
|
|
4003
|
+
// A canonical request already authorized this exact idempotent admission. Finish it
|
|
4004
|
+
// when its handler is unavailable: a stale admission can still arrive after the lookup,
|
|
4005
|
+
// so NotAdmitted cannot justify releasing its reservation. Expiry keeps admission closed.
|
|
4006
|
+
const admission = yield* establishChildFromRequest(
|
|
4007
|
+
parent,
|
|
4008
|
+
request,
|
|
4009
|
+
unavailableCalls !== undefined,
|
|
4010
|
+
);
|
|
4011
|
+
|
|
4012
|
+
if (admission._tag === "indeterminate") {
|
|
4013
|
+
indeterminate.add(toolCallId);
|
|
4014
|
+
continue;
|
|
4015
|
+
}
|
|
3665
4016
|
started = SubagentStarted.make({
|
|
3666
4017
|
runId,
|
|
3667
4018
|
toolCallId,
|
|
@@ -3700,7 +4051,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3700
4051
|
started.childRunId !== runIdForSubmission(started.childSubmissionId)
|
|
3701
4052
|
) {
|
|
3702
4053
|
return yield* LedgerError.make({
|
|
3703
|
-
operation: "
|
|
4054
|
+
operation: "reconcileRetainedChildren",
|
|
3704
4055
|
message: `Tool Call ${call.id} has inconsistent canonical child attachment evidence`,
|
|
3705
4056
|
});
|
|
3706
4057
|
}
|
|
@@ -3718,8 +4069,14 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3718
4069
|
conflictToLedgerError("attachChildToReservation"),
|
|
3719
4070
|
),
|
|
3720
4071
|
);
|
|
3721
|
-
if (child.value.state !== "settled")
|
|
3722
|
-
|
|
4072
|
+
if (child.value.state !== "settled") {
|
|
4073
|
+
waiting.push(
|
|
4074
|
+
WaitingChild.make({ toolCallId, childSubmissionId: started.childSubmissionId }),
|
|
4075
|
+
);
|
|
4076
|
+
yield* wake.notify(child.value.threadId);
|
|
4077
|
+
continue;
|
|
4078
|
+
}
|
|
4079
|
+
yield* joinSettledChildWithoutHandler(
|
|
3723
4080
|
ctx,
|
|
3724
4081
|
parent,
|
|
3725
4082
|
knownIds,
|
|
@@ -3727,9 +4084,11 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3727
4084
|
reservation,
|
|
3728
4085
|
started.toolCallId,
|
|
3729
4086
|
started.childSubmissionId,
|
|
3730
|
-
"duration",
|
|
4087
|
+
unavailableCalls === undefined ? "duration" : "unavailable",
|
|
3731
4088
|
);
|
|
3732
4089
|
}
|
|
4090
|
+
|
|
4091
|
+
return { notExecuted, waiting, indeterminate };
|
|
3733
4092
|
},
|
|
3734
4093
|
);
|
|
3735
4094
|
|
|
@@ -3819,7 +4178,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
3819
4178
|
});
|
|
3820
4179
|
}
|
|
3821
4180
|
if (child.value.state === "settled") {
|
|
3822
|
-
yield*
|
|
4181
|
+
yield* joinSettledChildWithoutHandler(
|
|
3823
4182
|
ctx,
|
|
3824
4183
|
parent,
|
|
3825
4184
|
knownIds,
|
|
@@ -4316,22 +4675,60 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4316
4675
|
journalSeed,
|
|
4317
4676
|
);
|
|
4318
4677
|
|
|
4319
|
-
const
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4678
|
+
const currentContracts = yield* contractsFor(agent.definition);
|
|
4679
|
+
|
|
4680
|
+
const rolloverOperation =
|
|
4681
|
+
journal.pendingContextToolCallId === undefined
|
|
4682
|
+
? undefined
|
|
4683
|
+
: operationsFor(records, runId).get(journal.pendingContextToolCallId);
|
|
4684
|
+
|
|
4685
|
+
const rolloverTool =
|
|
4686
|
+
rolloverOperation === undefined
|
|
4687
|
+
? undefined
|
|
4688
|
+
: agent.definition.toolkit.tools[rolloverOperation.toolName];
|
|
4689
|
+
|
|
4690
|
+
// A current annotation cannot reinterpret a formerly ordinary settled result as control.
|
|
4691
|
+
const pendingContextToolCallId =
|
|
4692
|
+
rolloverOperation !== undefined &&
|
|
4693
|
+
rolloverTool !== undefined &&
|
|
4694
|
+
Context.get(rolloverTool.annotations, ContextRolloverTool) &&
|
|
4695
|
+
currentContracts[rolloverOperation.toolName] === rolloverOperation.replay
|
|
4696
|
+
? journal.pendingContextToolCallId
|
|
4697
|
+
: undefined;
|
|
4326
4698
|
|
|
4327
|
-
|
|
4699
|
+
const applicationCalls = pending?.calls.filter((call) => !call.providerExecuted);
|
|
4700
|
+
|
|
4701
|
+
// Only the original single-call action-completion case may project a reconciled receipt.
|
|
4702
|
+
// Other settled calls are historical outcomes, regardless of their current completion role.
|
|
4703
|
+
const completionCandidate =
|
|
4704
|
+
applicationCalls?.length === 1 &&
|
|
4705
|
+
agent.definition.completionFromTools?.some(
|
|
4706
|
+
(declaration) => declaration.tool === applicationCalls[0]?.name,
|
|
4707
|
+
) &&
|
|
4708
|
+
pending?.settled.some(
|
|
4709
|
+
(result) => result.id === applicationCalls[0]?.id && !result.isFailure,
|
|
4710
|
+
)
|
|
4711
|
+
? applicationCalls[0]
|
|
4712
|
+
: undefined;
|
|
4328
4713
|
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4714
|
+
const settledCompletion =
|
|
4715
|
+
pending !== undefined &&
|
|
4716
|
+
completionCandidate !== undefined &&
|
|
4717
|
+
supportsOperation(
|
|
4718
|
+
submission,
|
|
4719
|
+
OpenToolCallEvidence.make({
|
|
4720
|
+
toolCallId: Schema.decodeSync(ToolCallId)(completionCandidate.id),
|
|
4721
|
+
toolName: completionCandidate.name,
|
|
4722
|
+
turn: pending.turn,
|
|
4723
|
+
}),
|
|
4724
|
+
pending.toolOperations?.find(
|
|
4725
|
+
(operation) => operation.toolCallId === completionCandidate.id,
|
|
4726
|
+
),
|
|
4727
|
+
undefined,
|
|
4728
|
+
{ definition: agent.definition, contracts: currentContracts },
|
|
4729
|
+
)
|
|
4730
|
+
? Schema.decodeSync(ToolCallId)(completionCandidate.id)
|
|
4731
|
+
: undefined;
|
|
4335
4732
|
|
|
4336
4733
|
const knownIds = knownRecordIdsOf(records);
|
|
4337
4734
|
const stepOutputs = new Map<string, PersistedJson>();
|
|
@@ -4661,167 +5058,41 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4661
5058
|
const declared = yield* declaredToolCalls(response.messages);
|
|
4662
5059
|
const calls = declared.application;
|
|
4663
5060
|
|
|
4664
|
-
let expectedOutput: {
|
|
4665
|
-
readonly encoded: Schema.Json;
|
|
4666
|
-
readonly decoded: OutputSchema["Type"];
|
|
4667
|
-
};
|
|
4668
|
-
|
|
4669
5061
|
if (calls.length > 0) {
|
|
4670
|
-
const completion = agent.definition.completion;
|
|
4671
5062
|
const call = calls[0];
|
|
4672
5063
|
|
|
4673
|
-
const actionCompletion = agent.definition.completionFromTools?.find(
|
|
4674
|
-
(declaration) => declaration.tool === call?.name,
|
|
4675
|
-
);
|
|
4676
|
-
|
|
4677
5064
|
const settled =
|
|
4678
5065
|
call === undefined
|
|
4679
5066
|
? undefined
|
|
4680
5067
|
: records.find(
|
|
4681
|
-
(
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
5068
|
+
({ record }) =>
|
|
5069
|
+
record.payload._tag === "ToolCallSettled" &&
|
|
5070
|
+
record.payload.runId === runId &&
|
|
5071
|
+
record.payload.toolCallId === call.id,
|
|
4685
5072
|
)?.record.payload;
|
|
4686
5073
|
|
|
4687
5074
|
if (
|
|
4688
5075
|
calls.length !== 1 ||
|
|
4689
5076
|
call === undefined ||
|
|
4690
|
-
(call.name !== completion?.tool && actionCompletion === undefined) ||
|
|
4691
5077
|
settled?._tag !== "ToolCallSettled" ||
|
|
4692
5078
|
settled.toolName !== call.name ||
|
|
4693
5079
|
settled.isFailure
|
|
4694
5080
|
) {
|
|
4695
5081
|
return yield* RunJournalError.make({
|
|
4696
|
-
message: `Run ${runId} has a terminal completion marker without one successful declared
|
|
5082
|
+
message: `Run ${runId} has a terminal completion marker without one successful declared Tool result`,
|
|
4697
5083
|
});
|
|
4698
5084
|
}
|
|
4699
|
-
|
|
4700
|
-
const projected = yield* (
|
|
4701
|
-
actionCompletion !== undefined
|
|
4702
|
-
? AgentRuntime.projectCompletionFromToolOutput(
|
|
4703
|
-
agent,
|
|
4704
|
-
actionCompletion,
|
|
4705
|
-
call.params,
|
|
4706
|
-
settled.result,
|
|
4707
|
-
)
|
|
4708
|
-
: completion !== undefined
|
|
4709
|
-
? Effect.map(
|
|
4710
|
-
AgentRuntime.projectCompletionOutput(
|
|
4711
|
-
agent,
|
|
4712
|
-
completion,
|
|
4713
|
-
call.params,
|
|
4714
|
-
settled.result,
|
|
4715
|
-
),
|
|
4716
|
-
Option.some,
|
|
4717
|
-
)
|
|
4718
|
-
: Effect.succeed(Option.none())
|
|
4719
|
-
).pipe(
|
|
4720
|
-
Effect.mapError((cause) =>
|
|
4721
|
-
RunJournalError.make({
|
|
4722
|
-
message: `Run ${runId} completion Tool output failed recovery projection`,
|
|
4723
|
-
cause,
|
|
4724
|
-
}),
|
|
4725
|
-
),
|
|
4726
|
-
);
|
|
4727
|
-
|
|
4728
|
-
if (Option.isNone(projected)) {
|
|
4729
|
-
return yield* RunJournalError.make({
|
|
4730
|
-
message: `Run ${runId} completion Tool declined its recorded terminal output`,
|
|
4731
|
-
});
|
|
4732
|
-
}
|
|
4733
|
-
expectedOutput = projected.value;
|
|
4734
|
-
} else {
|
|
4735
|
-
const responseText = yield* terminalAssistantText(response.messages);
|
|
4736
|
-
|
|
4737
|
-
expectedOutput = yield* AgentRuntime.decodeFinalOutput(agent, responseText).pipe(
|
|
4738
|
-
Effect.mapError((cause) =>
|
|
4739
|
-
RunJournalError.make({
|
|
4740
|
-
message: `Run ${runId} final response output failed recovery decoding`,
|
|
4741
|
-
cause,
|
|
4742
|
-
}),
|
|
4743
|
-
),
|
|
4744
|
-
);
|
|
4745
|
-
}
|
|
4746
|
-
|
|
4747
|
-
const boundedExpectedOutput = yield* decodePersisted(expectedOutput.encoded).pipe(
|
|
4748
|
-
Effect.mapError((cause) =>
|
|
4749
|
-
RunJournalError.make({
|
|
4750
|
-
message: `Run ${runId} reconstructed output exceeds canonical persistence bounds`,
|
|
4751
|
-
cause,
|
|
4752
|
-
}),
|
|
4753
|
-
),
|
|
4754
|
-
);
|
|
4755
|
-
|
|
4756
|
-
const [expectedOutputDigest, recordedOutputDigest] = yield* Effect.all([
|
|
4757
|
-
withCrypto(digestJson(boundedExpectedOutput)),
|
|
4758
|
-
withCrypto(digestJson(recordedCompletion.output)),
|
|
4759
|
-
]).pipe(
|
|
4760
|
-
Effect.mapError((cause) =>
|
|
4761
|
-
RunJournalError.make({
|
|
4762
|
-
message: `Run ${runId} completion output comparison failed`,
|
|
4763
|
-
cause,
|
|
4764
|
-
}),
|
|
4765
|
-
),
|
|
4766
|
-
);
|
|
4767
|
-
|
|
4768
|
-
if (expectedOutputDigest !== recordedOutputDigest) {
|
|
4769
|
-
return yield* RunJournalError.make({
|
|
4770
|
-
message: `Run ${runId} terminal completion output disagrees with its canonical response or Tool result`,
|
|
4771
|
-
});
|
|
4772
5085
|
}
|
|
4773
|
-
|
|
4774
|
-
const expectedRunDisposition =
|
|
4775
|
-
recordedCompletion.finishReason === "budget-exhausted"
|
|
4776
|
-
? undefined
|
|
4777
|
-
: yield* AgentRuntime.encodeRunDisposition(agent, expectedOutput.decoded).pipe(
|
|
4778
|
-
Effect.mapError((cause) =>
|
|
4779
|
-
RunJournalError.make({
|
|
4780
|
-
message: `Run ${runId} run disposition failed recovery projection`,
|
|
4781
|
-
cause,
|
|
4782
|
-
}),
|
|
4783
|
-
),
|
|
4784
|
-
);
|
|
4785
|
-
|
|
4786
5086
|
if (
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
5087
|
+
recordedCompletion.resultDigest !== undefined &&
|
|
5088
|
+
recordedCompletion.resultDigest !==
|
|
5089
|
+
(yield* withCrypto(runCompletionDigest(recordedCompletion)))
|
|
5090
|
+
)
|
|
4790
5091
|
return yield* RunJournalError.make({
|
|
4791
|
-
message:
|
|
5092
|
+
message: "RunCompleted terminal values failed their integrity check",
|
|
4792
5093
|
});
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
expectedRunDisposition !== undefined &&
|
|
4796
|
-
recordedCompletion.runDisposition !== undefined
|
|
4797
|
-
) {
|
|
4798
|
-
const boundedExpectedRunDisposition = yield* decodePersisted(expectedRunDisposition).pipe(
|
|
4799
|
-
Effect.mapError((cause) =>
|
|
4800
|
-
RunJournalError.make({
|
|
4801
|
-
message: `Run ${runId} reconstructed run disposition exceeds canonical persistence bounds`,
|
|
4802
|
-
cause,
|
|
4803
|
-
}),
|
|
4804
|
-
),
|
|
4805
|
-
);
|
|
4806
|
-
|
|
4807
|
-
const [expectedRunDispositionDigest, recordedRunDispositionDigest] = yield* Effect.all([
|
|
4808
|
-
withCrypto(digestJson(boundedExpectedRunDisposition)),
|
|
4809
|
-
withCrypto(digestJson(recordedCompletion.runDisposition)),
|
|
4810
|
-
]).pipe(
|
|
4811
|
-
Effect.mapError((cause) =>
|
|
4812
|
-
RunJournalError.make({
|
|
4813
|
-
message: `Run ${runId} run disposition comparison failed`,
|
|
4814
|
-
cause,
|
|
4815
|
-
}),
|
|
4816
|
-
),
|
|
4817
|
-
);
|
|
4818
|
-
|
|
4819
|
-
if (expectedRunDispositionDigest !== recordedRunDispositionDigest) {
|
|
4820
|
-
return yield* RunJournalError.make({
|
|
4821
|
-
message: `Run ${runId} terminal run disposition disagrees with its reconstructed output`,
|
|
4822
|
-
});
|
|
4823
|
-
}
|
|
4824
|
-
}
|
|
5094
|
+
// Output and disposition were validated before the atomic RunCompleted commit.
|
|
5095
|
+
// Future application codecs and completion functions have no authority to reinterpret it.
|
|
4825
5096
|
|
|
4826
5097
|
return {
|
|
4827
5098
|
_tag: "completed" as const,
|
|
@@ -4901,7 +5172,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4901
5172
|
|
|
4902
5173
|
if (firstAssistant > 0) {
|
|
4903
5174
|
resumeLeadingMessages = Prompt.fromMessages(
|
|
4904
|
-
pendingMessages.content
|
|
5175
|
+
pendingMessages.content
|
|
5176
|
+
.slice(0, firstAssistant)
|
|
5177
|
+
.filter((message) => message.role !== "system"),
|
|
4905
5178
|
);
|
|
4906
5179
|
}
|
|
4907
5180
|
if (firstAssistant >= 0) {
|
|
@@ -4989,16 +5262,45 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
4989
5262
|
: { ...state, history },
|
|
4990
5263
|
);
|
|
4991
5264
|
|
|
4992
|
-
//
|
|
4993
|
-
//
|
|
4994
|
-
//
|
|
4995
|
-
|
|
5265
|
+
// Current instructions govern the continuation; the original user intent, steering and
|
|
5266
|
+
// committed history survive. The pending Turn re-enters through the batch continuation,
|
|
5267
|
+
// without duplicating the engine's freshly rendered input.
|
|
5268
|
+
const instructionView = (
|
|
5269
|
+
messages: ReadonlyArray<Prompt.Message>,
|
|
5270
|
+
instructions: ReadonlyArray<Prompt.Message>,
|
|
5271
|
+
insertWhenAbsent: boolean,
|
|
5272
|
+
) => {
|
|
5273
|
+
const projected: Array<Prompt.Message> = [];
|
|
5274
|
+
const lengths = [0];
|
|
5275
|
+
let inserted = false;
|
|
5276
|
+
|
|
5277
|
+
for (const message of messages) {
|
|
5278
|
+
if (message.role === "system") {
|
|
5279
|
+
if (!inserted) projected.push(...instructions);
|
|
5280
|
+
inserted = true;
|
|
5281
|
+
} else projected.push(message);
|
|
5282
|
+
lengths.push(projected.length);
|
|
5283
|
+
}
|
|
5284
|
+
if (!inserted && insertWhenAbsent) projected.push(...instructions);
|
|
5285
|
+
|
|
5286
|
+
return {
|
|
5287
|
+
messages: projected,
|
|
5288
|
+
prefixLength: (length: number) => lengths[length] ?? projected.length,
|
|
5289
|
+
};
|
|
5290
|
+
};
|
|
5291
|
+
|
|
4996
5292
|
const resumeContext: RunContextHook<never, never> = {
|
|
4997
5293
|
prepare: ({ source }) =>
|
|
4998
5294
|
Ref.get(stateRef).pipe(
|
|
4999
5295
|
Effect.map((state) => ({
|
|
5000
5296
|
prompt: Prompt.fromMessages([
|
|
5001
|
-
...
|
|
5297
|
+
...instructionView(
|
|
5298
|
+
resumeProjection.prompt.content,
|
|
5299
|
+
source.content
|
|
5300
|
+
.slice(0, state.baseLen)
|
|
5301
|
+
.filter((message) => message.role === "system"),
|
|
5302
|
+
true,
|
|
5303
|
+
).messages,
|
|
5002
5304
|
...source.content.slice(state.baseLen ?? source.content.length),
|
|
5003
5305
|
]),
|
|
5004
5306
|
})),
|
|
@@ -5089,6 +5391,15 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5089
5391
|
const batch = yield* withCrypto(
|
|
5090
5392
|
turnResponseBatch({
|
|
5091
5393
|
toolExposure: commit.toolExposure,
|
|
5394
|
+
toolOperations: commit.calls.map((call) =>
|
|
5395
|
+
ToolOperation.make({
|
|
5396
|
+
toolCallId: call.toolCallId,
|
|
5397
|
+
toolName: call.toolName,
|
|
5398
|
+
executionClass: call.executionClass,
|
|
5399
|
+
executionKind: call.executionKind,
|
|
5400
|
+
replay: currentContracts[call.toolName]!,
|
|
5401
|
+
}),
|
|
5402
|
+
),
|
|
5092
5403
|
toolParameterRejections: commit.toolParameterRejections,
|
|
5093
5404
|
runId,
|
|
5094
5405
|
turn: canonicalTurn,
|
|
@@ -5155,6 +5466,8 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5155
5466
|
parameters,
|
|
5156
5467
|
parametersDigest,
|
|
5157
5468
|
executionKind: call.executionKind,
|
|
5469
|
+
executionClass: call.executionClass,
|
|
5470
|
+
replay: currentContracts[call.toolName]!,
|
|
5158
5471
|
}),
|
|
5159
5472
|
),
|
|
5160
5473
|
);
|
|
@@ -5300,11 +5613,21 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5300
5613
|
});
|
|
5301
5614
|
}
|
|
5302
5615
|
|
|
5616
|
+
const state = yield* Ref.get(stateRef);
|
|
5617
|
+
|
|
5618
|
+
const comparisonView = instructionView(
|
|
5619
|
+
sourceJournal.prompt.content,
|
|
5620
|
+
state.history?.content
|
|
5621
|
+
.slice(0, state.baseLen)
|
|
5622
|
+
.filter((message) => message.role === "system") ?? [],
|
|
5623
|
+
false,
|
|
5624
|
+
);
|
|
5625
|
+
|
|
5303
5626
|
// Compare each visible message once. A transformed source can authorize only its
|
|
5304
5627
|
// exact canonical prefix; no per-candidate rereads or full encoded Prompt copies.
|
|
5305
5628
|
let matchingPrefix = 0;
|
|
5306
5629
|
let requiredThrough = 0;
|
|
5307
|
-
const comparisonLimit = Math.min(
|
|
5630
|
+
const comparisonLimit = Math.min(comparisonView.messages.length, commit.through);
|
|
5308
5631
|
|
|
5309
5632
|
for (let index = 0; index < commit.through; index += 1) {
|
|
5310
5633
|
const message = commit.source.content[index];
|
|
@@ -5318,7 +5641,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5318
5641
|
requiredThrough = index + 1;
|
|
5319
5642
|
if (index >= comparisonLimit || index !== matchingPrefix || message === undefined)
|
|
5320
5643
|
continue;
|
|
5321
|
-
const canonicalMessage =
|
|
5644
|
+
const canonicalMessage = comparisonView.messages[index];
|
|
5322
5645
|
|
|
5323
5646
|
if (canonicalMessage === undefined) continue;
|
|
5324
5647
|
|
|
@@ -5348,7 +5671,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
5348
5671
|
|
|
5349
5672
|
if (following !== undefined && following.tag !== "ModelResponseRecorded") continue;
|
|
5350
5673
|
|
|
5351
|
-
const length = candidate.promptLength;
|
|
5674
|
+
const length = comparisonView.prefixLength(candidate.promptLength);
|
|
5352
5675
|
|
|
5353
5676
|
if (length === 0 || length > commit.through) continue;
|
|
5354
5677
|
if (length < requiredThrough || length > matchingPrefix) continue;
|
|
@@ -6437,6 +6760,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6437
6760
|
history: pending === undefined ? journal.historyBefore : resumeProjection.historyBefore,
|
|
6438
6761
|
onHistory,
|
|
6439
6762
|
input,
|
|
6763
|
+
...(journal.committedTurns > 0 || Schema.is(FrameworkMessage)(submission.messageAdmission)
|
|
6764
|
+
? { retainedInput: submission.inputPayload }
|
|
6765
|
+
: {}),
|
|
6440
6766
|
...(Schema.is(FrameworkMessage)(submission.messageAdmission)
|
|
6441
6767
|
? { frameworkMessage: submission.messageAdmission }
|
|
6442
6768
|
: {}),
|
|
@@ -6466,14 +6792,19 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6466
6792
|
? {}
|
|
6467
6793
|
: {
|
|
6468
6794
|
resume: {
|
|
6795
|
+
...(settledCompletion === undefined ? {} : { settledCompletion }),
|
|
6469
6796
|
turn: pending.turn,
|
|
6470
6797
|
turnId: pending.turnId,
|
|
6471
6798
|
calls: pending.calls,
|
|
6472
6799
|
toolParameterRejections: pending.toolParameterRejections,
|
|
6473
6800
|
settled: pending.settled,
|
|
6474
|
-
|
|
6475
|
-
|
|
6476
|
-
|
|
6801
|
+
// Legacy eager requests have no exposure snapshot. Their validated canonical
|
|
6802
|
+
// declarations establish the minimal original exposure needed for batch resume.
|
|
6803
|
+
toolExposure:
|
|
6804
|
+
pending.toolExposure ??
|
|
6805
|
+
Snapshot.make({
|
|
6806
|
+
exposedToolNames: [...new Set(pending.calls.map((call) => call.name))],
|
|
6807
|
+
}),
|
|
6477
6808
|
...(resumeLeadingMessages === undefined
|
|
6478
6809
|
? {}
|
|
6479
6810
|
: { leadingMessages: resumeLeadingMessages }),
|
|
@@ -6525,9 +6856,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6525
6856
|
...(journal.protectedContext === undefined
|
|
6526
6857
|
? {}
|
|
6527
6858
|
: { protectedContext: journal.protectedContext }),
|
|
6528
|
-
...(
|
|
6529
|
-
? {}
|
|
6530
|
-
: { pendingContextToolCallId: journal.pendingContextToolCallId }),
|
|
6859
|
+
...(pendingContextToolCallId === undefined ? {} : { pendingContextToolCallId }),
|
|
6531
6860
|
};
|
|
6532
6861
|
|
|
6533
6862
|
const commitPendingTurn: Effect.Effect<void, DurableWorkerFailure> = Effect.gen(function* () {
|
|
@@ -6629,18 +6958,20 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
6629
6958
|
? completedRun
|
|
6630
6959
|
: undefined;
|
|
6631
6960
|
|
|
6632
|
-
const batch = yield*
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
|
|
6639
|
-
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6643
|
-
|
|
6961
|
+
const batch = yield* withCrypto(
|
|
6962
|
+
turnResultsBatch({
|
|
6963
|
+
toolSelections: stagedToolSelections,
|
|
6964
|
+
budgetRejectedCalls,
|
|
6965
|
+
runId,
|
|
6966
|
+
turn: canonicalTurn,
|
|
6967
|
+
turnId: state.pendingTurn.turnId,
|
|
6968
|
+
appended: remaining,
|
|
6969
|
+
producerId: config.producerId,
|
|
6970
|
+
deploymentId: config.deploymentId,
|
|
6971
|
+
createdAt,
|
|
6972
|
+
...(runCompletion === undefined ? {} : { runCompletion }),
|
|
6973
|
+
}),
|
|
6974
|
+
);
|
|
6644
6975
|
|
|
6645
6976
|
// A recovered external receipt is already canonical. Commit only the terminal
|
|
6646
6977
|
// marker at the same results boundary; never duplicate or rewrite its result.
|
|
@@ -7241,7 +7572,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7241
7572
|
.tailSequence;
|
|
7242
7573
|
|
|
7243
7574
|
const initialThrough = controlThrough;
|
|
7244
|
-
|
|
7575
|
+
|
|
7576
|
+
const initialView = yield* recoveryView(
|
|
7577
|
+
threadId,
|
|
7578
|
+
initialThrough,
|
|
7579
|
+
[submissionId],
|
|
7580
|
+
runIdForSubmission(submissionId),
|
|
7581
|
+
);
|
|
7245
7582
|
|
|
7246
7583
|
let journalMetadata =
|
|
7247
7584
|
initialView.seed === undefined
|
|
@@ -7384,7 +7721,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7384
7721
|
let expiredChildObligation = false;
|
|
7385
7722
|
|
|
7386
7723
|
if ((yield* Clock.currentTimeMillis) >= DateTime.toEpochMillis(runTiming.deadline)) {
|
|
7387
|
-
yield*
|
|
7724
|
+
yield* reconcileRetainedChildren(ctx, submission, yield* Ref.get(tokenRef));
|
|
7388
7725
|
expiredChildObligation = yield* completeJoinedReleases(submission);
|
|
7389
7726
|
}
|
|
7390
7727
|
|
|
@@ -7405,7 +7742,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7405
7742
|
|
|
7406
7743
|
if (reconciliationEvidence.openToolCalls.length === 0) return true;
|
|
7407
7744
|
for (const envelope of reconciliationRecords) knownIds.add(envelope.record.recordId);
|
|
7408
|
-
const tools: Record<string, Tool.Any> = agent.definition.toolkit.tools;
|
|
7409
7745
|
|
|
7410
7746
|
const review = yield* reconcileOpenCalls(
|
|
7411
7747
|
ctx,
|
|
@@ -7414,11 +7750,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7414
7750
|
reconciliationRecords,
|
|
7415
7751
|
reconciliationEvidence.openToolCalls,
|
|
7416
7752
|
knownIds,
|
|
7417
|
-
(
|
|
7418
|
-
const tool = tools[toolName];
|
|
7419
|
-
|
|
7420
|
-
return tool === undefined ? undefined : getToolExecutionClass(tool);
|
|
7421
|
-
},
|
|
7753
|
+
{ definition: agent.definition, contracts: yield* contractsFor(agent.definition) },
|
|
7422
7754
|
);
|
|
7423
7755
|
|
|
7424
7756
|
if (review.uncertain.length > 0 || review.unproven.length > 0) {
|
|
@@ -7431,7 +7763,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7431
7763
|
"An ordinary Tool call may have executed without a canonical outcome",
|
|
7432
7764
|
);
|
|
7433
7765
|
}
|
|
7434
|
-
//
|
|
7766
|
+
// This Submission is parked (marked Unknown, or reconciliation itself failed):
|
|
7435
7767
|
// release the claim without settling — the accepted-work obligation stays owed.
|
|
7436
7768
|
const ownershipToken = yield* Ref.get(tokenRef);
|
|
7437
7769
|
|
|
@@ -7448,6 +7780,161 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7448
7780
|
if (!expiredChildObligation && !(yield* ordinaryCallsResolved))
|
|
7449
7781
|
return Option.none<Settlement>();
|
|
7450
7782
|
|
|
7783
|
+
const continuationRecords = yield* refreshControl();
|
|
7784
|
+
|
|
7785
|
+
const continuation = yield* pendingToolBatchFor(
|
|
7786
|
+
continuationRecords,
|
|
7787
|
+
runIdForSubmission(submissionId),
|
|
7788
|
+
);
|
|
7789
|
+
|
|
7790
|
+
if (continuation !== undefined) {
|
|
7791
|
+
const current = {
|
|
7792
|
+
definition: agent.definition,
|
|
7793
|
+
contracts: yield* contractsFor(agent.definition),
|
|
7794
|
+
};
|
|
7795
|
+
|
|
7796
|
+
const operations = operationsFor(continuationRecords, runIdForSubmission(submissionId));
|
|
7797
|
+
const settled = new Set(continuation.settled.map((call) => call.id));
|
|
7798
|
+
|
|
7799
|
+
const rejected = new Set(
|
|
7800
|
+
continuation.toolParameterRejections?.map((call) => call.toolCallId),
|
|
7801
|
+
);
|
|
7802
|
+
|
|
7803
|
+
const unsupported: Array<OpenToolCallEvidence> = [];
|
|
7804
|
+
const unavailableDelegations = new Set<ToolCallId>();
|
|
7805
|
+
|
|
7806
|
+
for (const call of continuation.calls) {
|
|
7807
|
+
const original = continuationRecords
|
|
7808
|
+
.map(({ record }) => record.payload)
|
|
7809
|
+
.find(
|
|
7810
|
+
(payload) =>
|
|
7811
|
+
payload._tag === "ToolCallPrepared" &&
|
|
7812
|
+
payload.runId === runIdForSubmission(submissionId) &&
|
|
7813
|
+
payload.toolCallId === call.id,
|
|
7814
|
+
);
|
|
7815
|
+
|
|
7816
|
+
if (
|
|
7817
|
+
original?._tag === "ToolCallPrepared" &&
|
|
7818
|
+
original.executionKind === "delegation" &&
|
|
7819
|
+
!settled.has(call.id) &&
|
|
7820
|
+
!supportsOperation(
|
|
7821
|
+
submission,
|
|
7822
|
+
OpenToolCallEvidence.make(original),
|
|
7823
|
+
operations.get(call.id),
|
|
7824
|
+
original,
|
|
7825
|
+
current,
|
|
7826
|
+
)
|
|
7827
|
+
)
|
|
7828
|
+
unavailableDelegations.add(original.toolCallId);
|
|
7829
|
+
}
|
|
7830
|
+
|
|
7831
|
+
const children =
|
|
7832
|
+
unavailableDelegations.size === 0
|
|
7833
|
+
? undefined
|
|
7834
|
+
: yield* reconcileRetainedChildren(
|
|
7835
|
+
ctx,
|
|
7836
|
+
submission,
|
|
7837
|
+
yield* Ref.get(tokenRef),
|
|
7838
|
+
unavailableDelegations,
|
|
7839
|
+
);
|
|
7840
|
+
|
|
7841
|
+
for (const envelope of yield* refreshControl()) knownIds.add(envelope.record.recordId);
|
|
7842
|
+
|
|
7843
|
+
for (const call of continuation.calls) {
|
|
7844
|
+
if (
|
|
7845
|
+
call.providerExecuted ||
|
|
7846
|
+
settled.has(call.id) ||
|
|
7847
|
+
rejected.has(Schema.decodeSync(ToolCallId)(call.id))
|
|
7848
|
+
)
|
|
7849
|
+
continue;
|
|
7850
|
+
|
|
7851
|
+
const evidence = OpenToolCallEvidence.make({
|
|
7852
|
+
toolCallId: Schema.decodeSync(ToolCallId)(call.id),
|
|
7853
|
+
toolName: call.name,
|
|
7854
|
+
turn: continuation.turn,
|
|
7855
|
+
});
|
|
7856
|
+
|
|
7857
|
+
const operation = operations.get(call.id);
|
|
7858
|
+
|
|
7859
|
+
const prepared = continuationRecords
|
|
7860
|
+
.map(({ record }) => record.payload)
|
|
7861
|
+
.find(
|
|
7862
|
+
(payload) =>
|
|
7863
|
+
payload._tag === "ToolCallPrepared" &&
|
|
7864
|
+
payload.runId === runIdForSubmission(submissionId) &&
|
|
7865
|
+
payload.turn === continuation.turn &&
|
|
7866
|
+
payload.toolCallId === call.id,
|
|
7867
|
+
);
|
|
7868
|
+
|
|
7869
|
+
const preparation = prepared?._tag === "ToolCallPrepared" ? prepared : undefined;
|
|
7870
|
+
|
|
7871
|
+
if (supportsOperation(submission, evidence, operation, preparation, current)) continue;
|
|
7872
|
+
if (preparation !== undefined && !children?.notExecuted.has(evidence.toolCallId)) {
|
|
7873
|
+
if (!unavailableDelegations.has(evidence.toolCallId)) unsupported.push(evidence);
|
|
7874
|
+
continue;
|
|
7875
|
+
}
|
|
7876
|
+
|
|
7877
|
+
// Mutating handlers cannot cross this protocol boundary without preparation.
|
|
7878
|
+
// Readonly handlers may already have run; their unavailable result makes no nonexecution claim.
|
|
7879
|
+
const execution =
|
|
7880
|
+
children?.notExecuted.has(evidence.toolCallId) ||
|
|
7881
|
+
(operation !== undefined &&
|
|
7882
|
+
(operation.executionClass !== "readonly" || operation.executionKind !== "ordinary"))
|
|
7883
|
+
? "not-executed"
|
|
7884
|
+
: "unavailable";
|
|
7885
|
+
|
|
7886
|
+
const result = yield* Schema.encodeEffect(ToolUnavailable)(
|
|
7887
|
+
ToolUnavailable.make({
|
|
7888
|
+
toolName: call.name,
|
|
7889
|
+
execution,
|
|
7890
|
+
message:
|
|
7891
|
+
execution === "not-executed"
|
|
7892
|
+
? "This operation was not executed and its original implementation is unavailable. Continue with the current tools."
|
|
7893
|
+
: "The original operation is unavailable. No mutating handler was dispatched; readonly work may have run without a recorded result.",
|
|
7894
|
+
}),
|
|
7895
|
+
).pipe(Effect.flatMap(decodePersisted), Effect.orDie);
|
|
7896
|
+
|
|
7897
|
+
yield* hit("tools:before-unavailable-append");
|
|
7898
|
+
yield* appendClosedCall(ctx, submissionId, knownIds, evidence, {
|
|
7899
|
+
result: { value: result, isFailure: true },
|
|
7900
|
+
resolution: execution === "not-executed" ? "never-started" : "failed-with-error",
|
|
7901
|
+
author: "runtime",
|
|
7902
|
+
reason: "The declared operation cannot execute under its original contract",
|
|
7903
|
+
});
|
|
7904
|
+
yield* hit("tools:after-unavailable-append");
|
|
7905
|
+
}
|
|
7906
|
+
const firstWaiting = children?.waiting[0];
|
|
7907
|
+
|
|
7908
|
+
if (children !== undefined && firstWaiting !== undefined) {
|
|
7909
|
+
const disposition = yield* ledger.suspend(
|
|
7910
|
+
SuspendRequest.make({
|
|
7911
|
+
submissionId,
|
|
7912
|
+
ownershipToken: yield* Ref.get(tokenRef),
|
|
7913
|
+
reason: WaitingForChildSuspension.make({
|
|
7914
|
+
children: [firstWaiting, ...children.waiting.slice(1)],
|
|
7915
|
+
}),
|
|
7916
|
+
}),
|
|
7917
|
+
);
|
|
7918
|
+
|
|
7919
|
+
yield* hit("subagent:after-suspend");
|
|
7920
|
+
if (disposition === "resume-immediately") yield* wake.notify(threadId);
|
|
7921
|
+
|
|
7922
|
+
return Option.none<Settlement>();
|
|
7923
|
+
}
|
|
7924
|
+
if ((children?.indeterminate.size ?? 0) > 0) return Option.none<Settlement>();
|
|
7925
|
+
if (unsupported.length > 0) {
|
|
7926
|
+
yield* markCallsUnknown(
|
|
7927
|
+
ctx,
|
|
7928
|
+
submissionId,
|
|
7929
|
+
knownIds,
|
|
7930
|
+
unsupported,
|
|
7931
|
+
"The unfinished operation has no supported recovery implementation; its original identity and evidence remain unresolved",
|
|
7932
|
+
);
|
|
7933
|
+
|
|
7934
|
+
return Option.none<Settlement>();
|
|
7935
|
+
}
|
|
7936
|
+
}
|
|
7937
|
+
|
|
7451
7938
|
const lineage: AttemptLineage = {
|
|
7452
7939
|
attemptId: claim.attemptId,
|
|
7453
7940
|
supersededEpoch,
|
|
@@ -7594,7 +8081,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7594
8081
|
);
|
|
7595
8082
|
yield* hit("run:after-duration-append");
|
|
7596
8083
|
}
|
|
7597
|
-
yield*
|
|
8084
|
+
yield* reconcileRetainedChildren(ctx, submission, yield* Ref.get(tokenRef));
|
|
7598
8085
|
}
|
|
7599
8086
|
const openObligation = yield* completeJoinedReleases(submission);
|
|
7600
8087
|
|
|
@@ -7630,7 +8117,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7630
8117
|
}
|
|
7631
8118
|
}).pipe(Effect.scoped);
|
|
7632
8119
|
|
|
7633
|
-
/** A missing
|
|
8120
|
+
/** A missing current binding leaves both roots and children owed, with their claims released. */
|
|
7634
8121
|
type CapturedWorkerBinding = Effect.Success<ReturnType<typeof makeLegacyWorkerBinding>>;
|
|
7635
8122
|
|
|
7636
8123
|
// Register cleanup before any interruptible work can observe a granted claim. The token
|
|
@@ -7742,8 +8229,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7742
8229
|
Effect.map((binding) => ({ _tag: "resolved" as const, binding })),
|
|
7743
8230
|
Effect.catchTags({
|
|
7744
8231
|
BindingUnavailable: (failure) => Effect.succeed({ _tag: "refused" as const, failure }),
|
|
7745
|
-
BindingDigestMismatch: (failure) =>
|
|
7746
|
-
Effect.succeed({ _tag: "refused" as const, failure }),
|
|
7747
8232
|
}),
|
|
7748
8233
|
);
|
|
7749
8234
|
|
|
@@ -7760,6 +8245,22 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7760
8245
|
}),
|
|
7761
8246
|
).pipe(withThreadHeadSpan);
|
|
7762
8247
|
|
|
8248
|
+
const eligibleThreadHead = (threadId: ThreadId) =>
|
|
8249
|
+
Stream.runHead(
|
|
8250
|
+
ledger.scanNonterminal.pipe(
|
|
8251
|
+
Stream.filter((entry) => entry.threadId === threadId),
|
|
8252
|
+
Stream.filterEffect((entry) =>
|
|
8253
|
+
entry.state !== "unknown"
|
|
8254
|
+
? Effect.succeed(true)
|
|
8255
|
+
: ledger
|
|
8256
|
+
.loadRecoverySnapshot(
|
|
8257
|
+
RecoverySnapshotRequest.make({ submissionId: entry.submissionId }),
|
|
8258
|
+
)
|
|
8259
|
+
.pipe(Effect.map((snapshot) => snapshot.abortIntent !== undefined)),
|
|
8260
|
+
),
|
|
8261
|
+
),
|
|
8262
|
+
);
|
|
8263
|
+
|
|
7763
8264
|
const drainThread = (
|
|
7764
8265
|
resolve: (
|
|
7765
8266
|
submission: SubmissionSnapshot,
|
|
@@ -7770,12 +8271,21 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7770
8271
|
const settlements: Array<Settlement> = [];
|
|
7771
8272
|
|
|
7772
8273
|
while (true) {
|
|
8274
|
+
const before = yield* eligibleThreadHead(threadId);
|
|
7773
8275
|
const settlement = yield* processThreadHead(resolve, threadId);
|
|
7774
8276
|
|
|
7775
8277
|
if (Option.isNone(settlement)) {
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
//
|
|
8278
|
+
const after = yield* eligibleThreadHead(threadId);
|
|
8279
|
+
|
|
8280
|
+
// Parking an unknown operation exposes later runnable work. Other suspensions,
|
|
8281
|
+
// live ownership and a transiently unproven operation retain the existing barrier.
|
|
8282
|
+
if (
|
|
8283
|
+
Option.isSome(before) &&
|
|
8284
|
+
Option.isSome(after) &&
|
|
8285
|
+
before.value.submissionId !== after.value.submissionId
|
|
8286
|
+
)
|
|
8287
|
+
continue;
|
|
8288
|
+
|
|
7779
8289
|
return settlements;
|
|
7780
8290
|
}
|
|
7781
8291
|
settlements.push(settlement.value);
|
|
@@ -7835,92 +8345,25 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7835
8345
|
);
|
|
7836
8346
|
});
|
|
7837
8347
|
|
|
7838
|
-
const
|
|
7839
|
-
submission
|
|
7840
|
-
) {
|
|
7841
|
-
const exact = registeredBindings.find(
|
|
7842
|
-
(binding) =>
|
|
7843
|
-
binding.agentId === submission.agentId &&
|
|
7844
|
-
definitionDigestsEqual(binding.digests, submission.agentDigests),
|
|
7845
|
-
);
|
|
7846
|
-
|
|
7847
|
-
if (exact !== undefined) return exact;
|
|
7848
|
-
|
|
7849
|
-
const candidates = registeredBindings.filter(
|
|
7850
|
-
(binding) => binding.agentId === submission.agentId,
|
|
7851
|
-
);
|
|
7852
|
-
|
|
7853
|
-
const candidate = candidates[0];
|
|
7854
|
-
|
|
7855
|
-
// A missing/ambiguous registration needs no history read. If every retained Tool contract
|
|
7856
|
-
// is supported, neither does an ordinary additive deployment.
|
|
7857
|
-
if (candidate === undefined || candidates.length !== 1)
|
|
7858
|
-
return yield* resolveWorkerBinding(
|
|
7859
|
-
registeredBindings,
|
|
7860
|
-
submission.agentId,
|
|
7861
|
-
submission.agentDigests,
|
|
7862
|
-
);
|
|
7863
|
-
if (bindingSupports(candidate, submission.agentDigests, undefined, submission.inputPayload))
|
|
7864
|
-
return candidate;
|
|
7865
|
-
|
|
7866
|
-
// A checkpoint can omit a settled batch that still supplies completion projection.
|
|
7867
|
-
// Compatibility reads the bounded authoritative prefix, never a reduced checkpoint.
|
|
7868
|
-
const records = yield* Effect.gen(function* () {
|
|
7869
|
-
const tail = yield* store.inspectTail(
|
|
7870
|
-
ThreadTailRequest.make({ threadId: submission.threadId }),
|
|
7871
|
-
);
|
|
7872
|
-
|
|
7873
|
-
return yield* Stream.runCollect(
|
|
7874
|
-
canonicalRange(submission.threadId, tail.tailSequence).pipe(
|
|
7875
|
-
Stream.filter(controlRecords([submission.submissionId])),
|
|
7876
|
-
),
|
|
7877
|
-
);
|
|
7878
|
-
}).pipe(Effect.catchTag("ThreadNotMaterialized", () => Effect.succeed([])));
|
|
7879
|
-
|
|
7880
|
-
const completionTools =
|
|
7881
|
-
candidate === undefined
|
|
7882
|
-
? []
|
|
7883
|
-
: [
|
|
7884
|
-
...(candidate.definition.completion === undefined
|
|
7885
|
-
? []
|
|
7886
|
-
: [candidate.definition.completion.tool]),
|
|
7887
|
-
...(candidate.definition.completionFromTools?.map((entry) => entry.tool) ?? []),
|
|
7888
|
-
];
|
|
7889
|
-
|
|
7890
|
-
const pending = yield* pendingToolBatchFor(
|
|
7891
|
-
records,
|
|
7892
|
-
runIdForSubmission(submission.submissionId),
|
|
7893
|
-
completionTools,
|
|
7894
|
-
);
|
|
7895
|
-
|
|
7896
|
-
return yield* resolveWorkerBinding(
|
|
7897
|
-
registeredBindings,
|
|
7898
|
-
submission.agentId,
|
|
7899
|
-
submission.agentDigests,
|
|
7900
|
-
pending?.calls
|
|
7901
|
-
.filter((call) => call.providerExecuted !== true || completionTools.includes(call.name))
|
|
7902
|
-
.map((call) => call.name) ?? [],
|
|
7903
|
-
submission.inputPayload,
|
|
7904
|
-
);
|
|
7905
|
-
});
|
|
8348
|
+
const resolveCurrentBinding = (submission: SubmissionSnapshot) =>
|
|
8349
|
+
resolveWorkerBinding(registeredBindings, submission.agentId);
|
|
7906
8350
|
|
|
7907
8351
|
const processThreadResolvedImpl = (
|
|
7908
8352
|
threadId: ThreadId,
|
|
7909
8353
|
): Effect.Effect<ReadonlyArray<Settlement>, DurableWorkerFailure | DurableBindingFailure> =>
|
|
7910
|
-
drainThread(
|
|
8354
|
+
drainThread(resolveCurrentBinding, threadId);
|
|
7911
8355
|
|
|
7912
8356
|
const processThreadHeadImpl = (
|
|
7913
8357
|
threadId: ThreadId,
|
|
7914
8358
|
options?: { readonly yieldAfter?: DateTime.Utc },
|
|
7915
8359
|
): Effect.Effect<Option.Option<Settlement>, DurableWorkerFailure | DurableBindingFailure> =>
|
|
7916
|
-
processThreadHead(
|
|
8360
|
+
processThreadHead(resolveCurrentBinding, threadId, options);
|
|
7917
8361
|
|
|
7918
8362
|
const claimFor = Effect.fn("DurableAgentRuntime.claimFor")(function* (
|
|
7919
8363
|
submission: SubmissionSnapshot,
|
|
8364
|
+
decision: RecoveryDecision,
|
|
7920
8365
|
): Effect.fn.Return<Option.Option<Claim>, LedgerError | OwnershipLost, Scope.Scope> {
|
|
7921
|
-
const head = yield*
|
|
7922
|
-
ledger.scanNonterminal.pipe(Stream.filter((entry) => entry.threadId === submission.threadId)),
|
|
7923
|
-
);
|
|
8366
|
+
const head = yield* eligibleThreadHead(submission.threadId);
|
|
7924
8367
|
|
|
7925
8368
|
if (Option.isNone(head) || head.value.submissionId !== submission.submissionId) {
|
|
7926
8369
|
return Option.none();
|
|
@@ -7929,7 +8372,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7929
8372
|
|
|
7930
8373
|
if (Option.isNone(claimed)) return Option.none();
|
|
7931
8374
|
if (claimed.value.claim.submissionId !== submission.submissionId) {
|
|
7932
|
-
//
|
|
8375
|
+
// Only the first eligible Submission is claimable; selection may have changed since the read.
|
|
7933
8376
|
yield* ledger
|
|
7934
8377
|
.releaseOwnership(
|
|
7935
8378
|
ReleaseOwnershipRequest.make({
|
|
@@ -7942,20 +8385,28 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7942
8385
|
return Option.none();
|
|
7943
8386
|
}
|
|
7944
8387
|
|
|
8388
|
+
yield* annotateRepair(submission.threadId, decision, claimed.value.claim.producerEpoch);
|
|
8389
|
+
|
|
7945
8390
|
return Option.some(claimed.value.claim);
|
|
7946
8391
|
});
|
|
7947
8392
|
|
|
7948
8393
|
/**
|
|
7949
|
-
* DUR-013 audit
|
|
7950
|
-
*
|
|
7951
|
-
*
|
|
8394
|
+
* Best-effort DUR-013 audit of an owned repair attempt, before it can release its claim.
|
|
8395
|
+
* The fixed granted epoch never borrows a later writer's authority. State-only repairs
|
|
8396
|
+
* omit canonical annotations; their decision and operator author/reason remain in telemetry.
|
|
7952
8397
|
*/
|
|
7953
8398
|
const annotateRepair = (
|
|
7954
8399
|
threadId: ThreadId,
|
|
7955
|
-
submissionId: SubmissionId,
|
|
7956
8400
|
decision: RecoveryDecision,
|
|
8401
|
+
producerEpoch: ProducerEpoch,
|
|
7957
8402
|
): Effect.Effect<void> =>
|
|
7958
8403
|
Effect.gen(function* () {
|
|
8404
|
+
yield* store.materialize(ThreadMaterialization.make({ threadId, producerEpoch }));
|
|
8405
|
+
const tail = yield* store.inspectTail(ThreadTailRequest.make({ threadId }));
|
|
8406
|
+
|
|
8407
|
+
// ThreadCreated must remain the first canonical record of a new Thread.
|
|
8408
|
+
if (tail.tailSequence === ZERO_SEQUENCE) return;
|
|
8409
|
+
|
|
7959
8410
|
const encodedDecision = yield* Schema.encodeEffect(RecoveryDecision)(decision).pipe(
|
|
7960
8411
|
Effect.orDie,
|
|
7961
8412
|
);
|
|
@@ -7963,26 +8414,23 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7963
8414
|
const details = yield* Schema.decodeEffect(PersistedJson)(encodedDecision).pipe(Effect.orDie);
|
|
7964
8415
|
|
|
7965
8416
|
const envelope = yield* makeEnvelope(
|
|
7966
|
-
recoveryRepairRecordId(submissionId, decision._tag),
|
|
8417
|
+
recoveryRepairRecordId(decision.submissionId, decision._tag),
|
|
7967
8418
|
RepairAnnotated.make({ reason: `recovery:${decision._tag}`, details }),
|
|
7968
8419
|
);
|
|
7969
8420
|
|
|
7970
|
-
const tail = yield* store.inspectTail(ThreadTailRequest.make({ threadId }));
|
|
7971
|
-
|
|
7972
8421
|
yield* store.append(
|
|
7973
8422
|
FencedAppendRequest.make({
|
|
7974
8423
|
threadId,
|
|
7975
8424
|
batch: CanonicalBatch.make({
|
|
7976
|
-
batchId: recoveryRepairBatchId(submissionId, decision._tag),
|
|
8425
|
+
batchId: recoveryRepairBatchId(decision.submissionId, decision._tag),
|
|
7977
8426
|
producerId: config.producerId,
|
|
7978
8427
|
records: [envelope],
|
|
7979
8428
|
}),
|
|
7980
8429
|
expectedTailSequence: tail.tailSequence,
|
|
7981
8430
|
expectedTailDigest: tail.tailDigest,
|
|
7982
|
-
producerEpoch
|
|
8431
|
+
producerEpoch,
|
|
7983
8432
|
}),
|
|
7984
8433
|
);
|
|
7985
|
-
yield* wake.notify(threadId);
|
|
7986
8434
|
}).pipe(Effect.ignore);
|
|
7987
8435
|
|
|
7988
8436
|
const settleAbortedForRecovery = Effect.fn("DurableAgentRuntime.settleAbortedForRecovery")(
|
|
@@ -7990,7 +8438,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
7990
8438
|
snapshot: RecoverySnapshot,
|
|
7991
8439
|
evidence: RecoveryEvidence,
|
|
7992
8440
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
7993
|
-
|
|
8441
|
+
decision: SettleAbortedDecision,
|
|
7994
8442
|
): Effect.fn.Return<"repaired" | "deferred", DurableWorkerFailure, Scope.Scope> {
|
|
7995
8443
|
const intent = snapshot.abortIntent;
|
|
7996
8444
|
|
|
@@ -8051,7 +8499,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8051
8499
|
}
|
|
8052
8500
|
}
|
|
8053
8501
|
}
|
|
8054
|
-
const claimed = yield* claimFor(submission);
|
|
8502
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8055
8503
|
|
|
8056
8504
|
if (Option.isNone(claimed)) {
|
|
8057
8505
|
// P7 §7(c): an aborted, never-claimed, still-queued `ready` Submission settles NOW
|
|
@@ -8111,11 +8559,11 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8111
8559
|
);
|
|
8112
8560
|
|
|
8113
8561
|
/**
|
|
8114
|
-
* Execute the reconcile-then-mark flow for a `MarkUnknown` decision (plan §2.2).
|
|
8115
|
-
*
|
|
8116
|
-
*
|
|
8117
|
-
*
|
|
8118
|
-
*
|
|
8562
|
+
* Execute the reconcile-then-mark flow for a `MarkUnknown` decision (plan §2.2). Current
|
|
8563
|
+
* registration metadata checks retry support when available. Low-level drivers supply their
|
|
8564
|
+
* Agent at claim time, so retry proofs defer that check without permitting execution. Calls
|
|
8565
|
+
* without a durable intent, original idempotent declaration, or reconciler proof stay uncertain.
|
|
8566
|
+
* Closed calls report `repaired`; deferred checks report `deferred`; uncertain calls become Unknown.
|
|
8119
8567
|
*/
|
|
8120
8568
|
const markUnknownForRecovery = Effect.fn("DurableAgentRuntime.markUnknownForRecovery")(function* (
|
|
8121
8569
|
snapshot: RecoverySnapshot,
|
|
@@ -8124,7 +8572,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8124
8572
|
decision: MarkUnknownDecision,
|
|
8125
8573
|
): Effect.fn.Return<"repaired" | "deferred" | "unknown", DurableWorkerFailure, Scope.Scope> {
|
|
8126
8574
|
const submission = snapshot.submission;
|
|
8127
|
-
const claimed = yield* claimFor(submission);
|
|
8575
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8128
8576
|
|
|
8129
8577
|
if (Option.isNone(claimed)) return "deferred";
|
|
8130
8578
|
const claim = claimed.value;
|
|
@@ -8149,7 +8597,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8149
8597
|
records,
|
|
8150
8598
|
evidence.openToolCalls.filter((call) => markableIds.has(call.toolCallId)),
|
|
8151
8599
|
knownIds,
|
|
8152
|
-
()
|
|
8600
|
+
yield* currentOperationsFor(submission.agentId),
|
|
8153
8601
|
);
|
|
8154
8602
|
|
|
8155
8603
|
let disposition: "repaired" | "deferred" | "unknown";
|
|
@@ -8180,40 +8628,61 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8180
8628
|
return disposition;
|
|
8181
8629
|
});
|
|
8182
8630
|
|
|
8631
|
+
/** A retry intent permits only the saved operation under its supported current contract. */
|
|
8632
|
+
const hasUnsupportedRetry = Effect.fnUntraced(function* (
|
|
8633
|
+
snapshot: RecoverySnapshot,
|
|
8634
|
+
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
8635
|
+
) {
|
|
8636
|
+
if (!snapshot.unknownResolutions.some((intent) => intent.resolution._tag === "SafeToRetry"))
|
|
8637
|
+
return false;
|
|
8638
|
+
const submission = snapshot.submission;
|
|
8639
|
+
const current = yield* currentOperationsFor(submission.agentId);
|
|
8640
|
+
const operations = operationsFor(records, runIdForSubmission(submission.submissionId));
|
|
8641
|
+
|
|
8642
|
+
for (const intent of snapshot.unknownResolutions) {
|
|
8643
|
+
if (intent.resolution._tag === "SafeToRetry") {
|
|
8644
|
+
const prepared = records
|
|
8645
|
+
.map(({ record }) => record.payload)
|
|
8646
|
+
.find(
|
|
8647
|
+
(payload) =>
|
|
8648
|
+
payload._tag === "ToolCallPrepared" &&
|
|
8649
|
+
payload.runId === runIdForSubmission(submission.submissionId) &&
|
|
8650
|
+
payload.toolCallId === intent.toolCallId,
|
|
8651
|
+
);
|
|
8652
|
+
|
|
8653
|
+
if (
|
|
8654
|
+
prepared?._tag !== "ToolCallPrepared" ||
|
|
8655
|
+
(current !== undefined &&
|
|
8656
|
+
!supportsOperation(
|
|
8657
|
+
submission,
|
|
8658
|
+
OpenToolCallEvidence.make(prepared),
|
|
8659
|
+
operations.get(intent.toolCallId),
|
|
8660
|
+
prepared,
|
|
8661
|
+
current,
|
|
8662
|
+
))
|
|
8663
|
+
)
|
|
8664
|
+
return true;
|
|
8665
|
+
}
|
|
8666
|
+
}
|
|
8667
|
+
|
|
8668
|
+
return false;
|
|
8669
|
+
});
|
|
8670
|
+
|
|
8183
8671
|
/**
|
|
8184
|
-
*
|
|
8185
|
-
*
|
|
8186
|
-
* exist); the wake itself is the ledger's idempotent re-check on intent replay (WP2), so a
|
|
8187
|
-
* crash anywhere in this pass converges on the next one.
|
|
8672
|
+
* Replay covering resolution intents as state-only wakes. Canonical outcomes wait for an
|
|
8673
|
+
* actual claim, so resolving an older request cannot append using a later writer's epoch.
|
|
8188
8674
|
*/
|
|
8189
8675
|
const applyUnknownResolutionsForRecovery = Effect.fn(
|
|
8190
8676
|
"DurableAgentRuntime.applyUnknownResolutionsForRecovery",
|
|
8191
8677
|
)(function* (
|
|
8192
8678
|
snapshot: RecoverySnapshot,
|
|
8193
|
-
|
|
8679
|
+
_evidence: RecoveryEvidence,
|
|
8194
8680
|
records: ReadonlyArray<CanonicalRecordEnvelope>,
|
|
8195
|
-
): Effect.fn.Return<"repaired" | "deferred", DurableWorkerFailure> {
|
|
8681
|
+
): Effect.fn.Return<"repaired" | "deferred" | "unknown", DurableWorkerFailure> {
|
|
8682
|
+
if (yield* hasUnsupportedRetry(snapshot, records)) return "unknown";
|
|
8196
8683
|
const submission = snapshot.submission;
|
|
8197
|
-
const ctx = yield* attemptContextAtTail(submission.threadId);
|
|
8198
|
-
const knownIds = knownRecordIdsOf(records);
|
|
8199
8684
|
|
|
8200
|
-
const applied = yield* reconcileOpenCalls(
|
|
8201
|
-
ctx,
|
|
8202
|
-
submission,
|
|
8203
|
-
snapshot,
|
|
8204
|
-
records,
|
|
8205
|
-
evidence.openToolCalls,
|
|
8206
|
-
knownIds,
|
|
8207
|
-
() => undefined,
|
|
8208
|
-
).pipe(
|
|
8209
|
-
Effect.as(true),
|
|
8210
|
-
// A fence advance mid-pass means another owner took the Thread: defer to it.
|
|
8211
|
-
Effect.catchTag("FenceRejected", () => Effect.succeed(false)),
|
|
8212
|
-
);
|
|
8213
|
-
|
|
8214
|
-
if (!applied) return "deferred";
|
|
8215
8685
|
for (const intent of snapshot.unknownResolutions) {
|
|
8216
|
-
// Replaying the stored intent re-checks the covering wake idempotently (WP2 contract).
|
|
8217
8686
|
yield* ledger
|
|
8218
8687
|
.recordUnknownResolution(
|
|
8219
8688
|
UnknownResolutionCommand.make({
|
|
@@ -8234,6 +8703,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8234
8703
|
),
|
|
8235
8704
|
);
|
|
8236
8705
|
}
|
|
8706
|
+
// Canonical outcomes are applied by the next claim. A wake never takes an active epoch.
|
|
8237
8707
|
yield* wake.notify(submission.threadId);
|
|
8238
8708
|
|
|
8239
8709
|
return "repaired";
|
|
@@ -8253,6 +8723,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8253
8723
|
function* (
|
|
8254
8724
|
snapshot: RecoverySnapshot,
|
|
8255
8725
|
evidence: RecoveryEvidence,
|
|
8726
|
+
decision: RecoveryDecision,
|
|
8256
8727
|
): Effect.fn.Return<"repaired" | "deferred", DurableWorkerFailure, Scope.Scope> {
|
|
8257
8728
|
const submission = snapshot.submission;
|
|
8258
8729
|
|
|
@@ -8266,7 +8737,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8266
8737
|
const first = undecided[0];
|
|
8267
8738
|
|
|
8268
8739
|
if (first === undefined) return "deferred";
|
|
8269
|
-
const claimed = yield* claimFor(submission);
|
|
8740
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8270
8741
|
|
|
8271
8742
|
if (Option.isNone(claimed)) return "deferred";
|
|
8272
8743
|
const claim = claimed.value;
|
|
@@ -8355,7 +8826,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8355
8826
|
// A closed admission window must release provably-unused reservations or restore the
|
|
8356
8827
|
// existing child's link. Do this before ordinary Unknown Outcomes make the lane
|
|
8357
8828
|
// unclaimable, and never let binding-free recovery admit new work after expiry.
|
|
8358
|
-
const claimed = yield* claimFor(submission);
|
|
8829
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8359
8830
|
|
|
8360
8831
|
if (Option.isNone(claimed)) return "deferred";
|
|
8361
8832
|
const claim = claimed.value;
|
|
@@ -8368,7 +8839,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8368
8839
|
);
|
|
8369
8840
|
const ctx = yield* attemptContextFor(submission.threadId, claim.producerEpoch);
|
|
8370
8841
|
|
|
8371
|
-
yield*
|
|
8842
|
+
yield* reconcileRetainedChildren(ctx, submission, claim.ownershipToken);
|
|
8372
8843
|
const open = yield* completeJoinedReleases(submission);
|
|
8373
8844
|
|
|
8374
8845
|
yield* ledger
|
|
@@ -8399,12 +8870,12 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8399
8870
|
return yield* applyUnknownResolutionsForRecovery(snapshot, evidence, records);
|
|
8400
8871
|
}
|
|
8401
8872
|
case "AwaitUnknownResolution": {
|
|
8402
|
-
//
|
|
8873
|
+
// This Submission stays parked awaiting the authorized DUR-017 resolution path;
|
|
8403
8874
|
// the settlement obligation stays visible, nothing replays.
|
|
8404
8875
|
return "unknown";
|
|
8405
8876
|
}
|
|
8406
8877
|
case "AwaitApprovalDecision": {
|
|
8407
|
-
return yield* awaitApprovalForRecovery(snapshot, evidence);
|
|
8878
|
+
return yield* awaitApprovalForRecovery(snapshot, evidence, decision);
|
|
8408
8879
|
}
|
|
8409
8880
|
case "ResumeSuspended": {
|
|
8410
8881
|
return yield* resumeSuspendedForRecovery(snapshot);
|
|
@@ -8458,7 +8929,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8458
8929
|
);
|
|
8459
8930
|
|
|
8460
8931
|
if (Option.isNone(host)) return "deferred";
|
|
8461
|
-
const claimed = yield* claimFor(host.value);
|
|
8932
|
+
const claimed = yield* claimFor(host.value, decision);
|
|
8462
8933
|
|
|
8463
8934
|
if (Option.isNone(claimed)) return "deferred";
|
|
8464
8935
|
const claim = claimed.value;
|
|
@@ -8531,7 +9002,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8531
9002
|
}
|
|
8532
9003
|
case "ApplyInput":
|
|
8533
9004
|
case "RepairInputMarker": {
|
|
8534
|
-
const claimed = yield* claimFor(submission);
|
|
9005
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8535
9006
|
|
|
8536
9007
|
if (Option.isNone(claimed)) return "deferred";
|
|
8537
9008
|
const claim = claimed.value;
|
|
@@ -8581,7 +9052,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8581
9052
|
const reservation = snapshot.reservation;
|
|
8582
9053
|
|
|
8583
9054
|
if (reservation === undefined) return "deferred";
|
|
8584
|
-
const claimed = yield* claimFor(submission);
|
|
9055
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8585
9056
|
|
|
8586
9057
|
if (Option.isNone(claimed)) {
|
|
8587
9058
|
// P7 §7(c) crash replay: a queued-abort settlement that committed its reservation
|
|
@@ -8665,7 +9136,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8665
9136
|
const admission = yield* establishChildFromRequest(submission, requestedPayload);
|
|
8666
9137
|
|
|
8667
9138
|
if (admission._tag === "indeterminate") return "deferred";
|
|
8668
|
-
const claimed = yield* claimFor(submission);
|
|
9139
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8669
9140
|
|
|
8670
9141
|
if (Option.isNone(claimed)) return "deferred";
|
|
8671
9142
|
const claim = claimed.value;
|
|
@@ -8738,7 +9209,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8738
9209
|
// waitingForChild checkpoint"): claim the lane and suspend it — the ledger op ends the
|
|
8739
9210
|
// ownership period itself, so the lane holds no worker permit while each child runs
|
|
8740
9211
|
// on its own lane. Never spawns a replacement invocation (SUB-018/SUB-030).
|
|
8741
|
-
const claimed = yield* claimFor(submission);
|
|
9212
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8742
9213
|
|
|
8743
9214
|
if (Option.isNone(claimed)) return "deferred";
|
|
8744
9215
|
const claim = claimed.value;
|
|
@@ -8845,7 +9316,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8845
9316
|
}
|
|
8846
9317
|
}
|
|
8847
9318
|
if (submission.state !== "suspended") {
|
|
8848
|
-
const claimed = yield* claimFor(submission);
|
|
9319
|
+
const claimed = yield* claimFor(submission, decision);
|
|
8849
9320
|
|
|
8850
9321
|
if (Option.isNone(claimed)) return "deferred";
|
|
8851
9322
|
const claim = claimed.value;
|
|
@@ -8965,10 +9436,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
8965
9436
|
executeRecoveryDecision(snapshot, evidence, decision, history.records, history),
|
|
8966
9437
|
);
|
|
8967
9438
|
|
|
8968
|
-
if (disposition === "repaired") {
|
|
8969
|
-
yield* annotateRepair(submission.threadId, submission.submissionId, decision);
|
|
8970
|
-
}
|
|
8971
|
-
|
|
8972
9439
|
return RecoveryReport.make({
|
|
8973
9440
|
submissionId: submission.submissionId,
|
|
8974
9441
|
threadId: submission.threadId,
|
|
@@ -9131,13 +9598,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9131
9598
|
!registeredBindings.some(
|
|
9132
9599
|
(binding) =>
|
|
9133
9600
|
binding.agentId === agent.definition.id &&
|
|
9134
|
-
|
|
9135
|
-
binding.retainedManifests?.some((retained) =>
|
|
9136
|
-
definitionDigestsEqual(
|
|
9137
|
-
DefinitionDigests.make({ ...retained.digests, replay: retained.replay }),
|
|
9138
|
-
options.definitions,
|
|
9139
|
-
),
|
|
9140
|
-
)),
|
|
9601
|
+
definitionDigestsEqual(binding.digests, options.definitions),
|
|
9141
9602
|
)
|
|
9142
9603
|
) {
|
|
9143
9604
|
const retained = yield* ledger.lookup(
|
|
@@ -9574,7 +10035,15 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9574
10035
|
const nowMillis = yield* Clock.currentTimeMillis;
|
|
9575
10036
|
const explainedAt = yield* nowUtc;
|
|
9576
10037
|
const runId = runIdForSubmission(submission.submissionId);
|
|
9577
|
-
|
|
10038
|
+
|
|
10039
|
+
const resolvedIds = new Set(
|
|
10040
|
+
read.records.flatMap(({ record }) =>
|
|
10041
|
+
record.payload._tag === "ToolCallSettled" && record.payload.runId === runId
|
|
10042
|
+
? [record.payload.toolCallId]
|
|
10043
|
+
: [],
|
|
10044
|
+
),
|
|
10045
|
+
);
|
|
10046
|
+
|
|
9578
10047
|
const unknownCalls: Array<ExplainedUnknownCall> = [];
|
|
9579
10048
|
|
|
9580
10049
|
for (const envelope of read.records) {
|
|
@@ -9611,6 +10080,13 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9611
10080
|
inputRecorded: evidence.inputRecorded,
|
|
9612
10081
|
abortRecorded: evidence.abortRecorded,
|
|
9613
10082
|
openToolCalls: evidence.openToolCalls,
|
|
10083
|
+
pendingOperations: read.records.flatMap(({ record }) =>
|
|
10084
|
+
record.payload._tag === "ToolCallPrepared" &&
|
|
10085
|
+
record.payload.runId === runId &&
|
|
10086
|
+
!resolvedIds.has(record.payload.toolCallId)
|
|
10087
|
+
? [record.payload]
|
|
10088
|
+
: [],
|
|
10089
|
+
),
|
|
9614
10090
|
openDelegationCalls: evidence.openDelegationCalls,
|
|
9615
10091
|
approvalsPending: evidence.approvalsPending,
|
|
9616
10092
|
unknownCalls,
|
|
@@ -9629,7 +10105,15 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9629
10105
|
}),
|
|
9630
10106
|
decision,
|
|
9631
10107
|
decisionMeaning: recoveryDecisionMeaning(decision._tag),
|
|
9632
|
-
disposition:
|
|
10108
|
+
disposition:
|
|
10109
|
+
decision._tag === "ApplyUnknownResolutions" &&
|
|
10110
|
+
(yield* hasUnsupportedRetry(snapshot, read.records).pipe(
|
|
10111
|
+
Effect.mapError((cause) =>
|
|
10112
|
+
RunJournalError.make({ message: "Cannot verify the pending retry contract", cause }),
|
|
10113
|
+
),
|
|
10114
|
+
))
|
|
10115
|
+
? "unknown"
|
|
10116
|
+
: predictRecoveryDisposition(decision, snapshot),
|
|
9633
10117
|
explainedAt,
|
|
9634
10118
|
});
|
|
9635
10119
|
});
|
|
@@ -9754,9 +10238,9 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9754
10238
|
|
|
9755
10239
|
/**
|
|
9756
10240
|
* Safe re-drive of exactly one Submission's recovery decision (plan §3): classify, execute
|
|
9757
|
-
* the ONE repair the classifier names,
|
|
9758
|
-
*
|
|
9759
|
-
* protect the paths that own their own operations: settled work (`NoAction`),
|
|
10241
|
+
* the ONE repair the classifier names, annotate owned repair attempts with their actual
|
|
10242
|
+
* claim epoch (DUR-013), and wake the lane. Typed refusals
|
|
10243
|
+
* protect the paths that own their own operations: settled work (`NoAction`), requests parked
|
|
9760
10244
|
* on Unknown Outcomes (`resolveUnknown`, DUR-017), and lanes awaiting approval decisions
|
|
9761
10245
|
* (`resolveApproval`). The mandatory `author`/`reason` audit fields (SEC-011) annotate the
|
|
9762
10246
|
* structured operator log.
|
|
@@ -9804,7 +10288,7 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9804
10288
|
refusal: "await-unknown-resolution",
|
|
9805
10289
|
decisionTag: decision._tag,
|
|
9806
10290
|
message:
|
|
9807
|
-
"
|
|
10291
|
+
"This Submission is parked on Unknown Outcomes; resolve them through the authorized resolveUnknown path (DUR-017) instead of retrying.",
|
|
9808
10292
|
});
|
|
9809
10293
|
}
|
|
9810
10294
|
if (decision._tag === "AwaitApprovalDecision") {
|
|
@@ -9830,9 +10314,6 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9830
10314
|
executeRecoveryDecision(snapshot, evidence, decision, read.records, read),
|
|
9831
10315
|
);
|
|
9832
10316
|
|
|
9833
|
-
if (disposition === "repaired") {
|
|
9834
|
-
yield* annotateRepair(submission.threadId, command.submissionId, decision);
|
|
9835
|
-
}
|
|
9836
10317
|
yield* wake.notify(submission.threadId);
|
|
9837
10318
|
|
|
9838
10319
|
return RecoveryReport.make({
|
|
@@ -9885,11 +10366,16 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
9885
10366
|
const records = (yield* readAllTolerant(submission.threadId, [submission.submissionId]))
|
|
9886
10367
|
.records;
|
|
9887
10368
|
|
|
10369
|
+
const runId = runIdForSubmission(submission.submissionId);
|
|
10370
|
+
|
|
9888
10371
|
const resolvedIds = new Set(
|
|
9889
|
-
|
|
10372
|
+
records.flatMap(({ record }) =>
|
|
10373
|
+
record.payload._tag === "ToolCallSettled" && record.payload.runId === runId
|
|
10374
|
+
? [record.payload.toolCallId]
|
|
10375
|
+
: [],
|
|
10376
|
+
),
|
|
9890
10377
|
);
|
|
9891
10378
|
|
|
9892
|
-
const runId = runIdForSubmission(submission.submissionId);
|
|
9893
10379
|
let earliest: DateTime.Utc | undefined;
|
|
9894
10380
|
|
|
9895
10381
|
for (const envelope of records) {
|
|
@@ -10104,19 +10590,16 @@ const make = Effect.fn("DurableAgentRuntime.make")(function* (
|
|
|
10104
10590
|
* the next model request, reattaches through the prompt-coverage rule after a crash, and the
|
|
10105
10591
|
* joined Submissions settle with the host outcome (DUR-002/DUR-016).
|
|
10106
10592
|
* - `runWorker(agent)` — scan-seeded, wake-driven worker loop over every lane (WP4's host driver),
|
|
10107
|
-
*
|
|
10108
|
-
*
|
|
10109
|
-
* parent-linked head refused this way settles with the framework `ChildCompatibilityFailure`.
|
|
10593
|
+
* over a singleton current Binding: a claimed head belonging to a different Agent never
|
|
10594
|
+
* runs against this binding. Missing bindings leave accepted roots and children owed.
|
|
10110
10595
|
* - `processThreadResolved(threadId)` / `runResolvedWorker` — the
|
|
10111
|
-
*
|
|
10112
|
-
*
|
|
10113
|
-
*
|
|
10114
|
-
* `ChildCompatibilityFailure` (no application code, the child never executes), and an
|
|
10115
|
-
* unresolvable root surfaces the typed refusal after releasing the claim.
|
|
10596
|
+
* equivalents over registrations owned by the runtime Layer: each stable `agentId` selects
|
|
10597
|
+
* one current Binding. Unfinished operation contracts gate handler execution independently
|
|
10598
|
+
* of immutable admission evidence; missing bindings release the claim with a typed refusal.
|
|
10116
10599
|
* - `runRecovery()` — classify every nonterminal Submission with the pure `classifyRecovery` and
|
|
10117
|
-
* execute
|
|
10600
|
+
* execute repair decisions, annotating owned repair attempts with their granted epochs
|
|
10118
10601
|
* (DUR-013). Untouched ready input and model-resuming work are reported `deferred` for a
|
|
10119
|
-
* worker claim;
|
|
10602
|
+
* worker claim; Submissions parked on
|
|
10120
10603
|
* Unknown Outcomes are reported `unknown`. The S2 binding-free Subagent executors (admission
|
|
10121
10604
|
* completion, start-link repair, waiting restoration, wake replay, canonical join accounting,
|
|
10122
10605
|
* abort propagation, orphan reservation release) run here; the settlement join itself is
|
|
@@ -10223,8 +10706,8 @@ export class DurableAgentRuntime extends Context.Service<
|
|
|
10223
10706
|
readonly verify: (threadId: ThreadId) => Effect.Effect<IntegrityReport, DurableVerifyFailure>;
|
|
10224
10707
|
/**
|
|
10225
10708
|
* Safe re-drive of one Submission's recovery decision with mandatory author/reason audit
|
|
10226
|
-
* (SEC-011): executes exactly the repair the classifier names
|
|
10227
|
-
*
|
|
10709
|
+
* (SEC-011): executes exactly the repair the classifier names; owned repair attempts
|
|
10710
|
+
* append a best-effort `RepairAnnotated` with their actual claim epoch (DUR-013). Typed `RetryRefused`
|
|
10228
10711
|
* for settled work and for lanes owned by the resolveUnknown/resolveApproval paths.
|
|
10229
10712
|
*/
|
|
10230
10713
|
readonly retry: (command: RetryCommand) => Effect.Effect<RecoveryReport, DurableRetryFailure>;
|