@springbrand/agent-runtime 0.2.0-alpha.61 → 0.2.0-alpha.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/db/milestone.repo.ts +8 -0
- package/src/db/submission.repo.ts +1 -0
- package/src/kernel/approval-lifecycle.ts +23 -5
- package/src/kernel/interaction-lifecycle.ts +12 -3
- package/src/kernel/state.ts +4 -1
- package/src/layers/context/budget/gate.ts +19 -4
- package/src/pi/runtime-adapter/execution.ts +12 -9
- package/src/pi/runtime-adapter/index.ts +14 -3
- package/src/pi/runtime-adapter/models.ts +1 -0
- package/src/pi/runtime-adapter/recovery.ts +20 -8
- package/src/pi/tool/compiler.ts +32 -8
- package/src/pi/tool/core.ts +15 -6
- package/src/pi/tool/workspace-sandbox.ts +4 -3
- package/src/runtime.ts +57 -29
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@springbrand/agent-runtime",
|
|
3
|
-
"version": "0.2.0-alpha.
|
|
3
|
+
"version": "0.2.0-alpha.63",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"files": [
|
|
6
6
|
"src",
|
|
@@ -35,6 +35,6 @@
|
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"agents": "0.22.0",
|
|
38
|
-
"ai": "7.0.
|
|
38
|
+
"ai": "7.0.56"
|
|
39
39
|
}
|
|
40
40
|
}
|
package/src/db/milestone.repo.ts
CHANGED
|
@@ -33,6 +33,14 @@ export class RecoveryMilestoneRepository {
|
|
|
33
33
|
// 恢复事实必须与 Submission 位于同一 Agent 存储中,不能用另一个 sql 入口拼接。
|
|
34
34
|
constructor(private readonly sql: SqlTaggedTemplate) {}
|
|
35
35
|
|
|
36
|
+
/** The existing unique index makes active Tool writes independent of Turn history size. */
|
|
37
|
+
findBody(submissionId: string, milestoneKey: string): string | null {
|
|
38
|
+
return this.sql<{ body: string }>`
|
|
39
|
+
SELECT body FROM pi_recovery_milestones
|
|
40
|
+
WHERE submission_id = ${submissionId} AND milestone_key = ${milestoneKey}
|
|
41
|
+
`[0]?.body ?? null;
|
|
42
|
+
}
|
|
43
|
+
|
|
36
44
|
// 按写入顺序返回某次 Submission 的全部恢复内容。
|
|
37
45
|
// Runtime 和 ApprovalLifecycle 在计算下一个恢复决策前调用,调用方直接把结果交给 Pi 恢复器。
|
|
38
46
|
// ORDER BY seq 是重放契约,不能改成 milestoneKey 或无排序查询,否则决策顺序会变。
|
|
@@ -72,6 +72,7 @@ interface ApprovalLifecycleOptions<
|
|
|
72
72
|
readonly db: RuntimeDatabase;
|
|
73
73
|
readonly pi: PiRuntimeAdapter;
|
|
74
74
|
readonly findSubmission: (submissionId: string) => TSubmission | null;
|
|
75
|
+
readonly isSubmissionActive: (submissionId: string) => boolean;
|
|
75
76
|
readonly applyRecoveryMutations: (
|
|
76
77
|
submission: TSubmission,
|
|
77
78
|
mutations: readonly PiDurableMutation[],
|
|
@@ -227,11 +228,23 @@ export class ApprovalLifecycle<
|
|
|
227
228
|
}
|
|
228
229
|
return value;
|
|
229
230
|
});
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
231
|
+
// Install the waiter before publishing pending state. Otherwise an early
|
|
232
|
+
// decision can schedule a second executor while this one is still alive.
|
|
233
|
+
const waiting = this.wait(pending.approval, signal);
|
|
234
|
+
try {
|
|
235
|
+
await Promise.all([
|
|
236
|
+
waiting,
|
|
237
|
+
(async () => {
|
|
238
|
+
if (pending.created) {
|
|
239
|
+
await this.options.onApprovalsChanged();
|
|
240
|
+
await onApproval?.(pending.approval);
|
|
241
|
+
}
|
|
242
|
+
})(),
|
|
243
|
+
]);
|
|
244
|
+
} catch (error) {
|
|
245
|
+
this.waiters.get(pending.approval.executionId)?.release();
|
|
246
|
+
throw error;
|
|
233
247
|
}
|
|
234
|
-
return this.wait(pending.approval, signal);
|
|
235
248
|
}
|
|
236
249
|
|
|
237
250
|
/**
|
|
@@ -362,7 +375,12 @@ export class ApprovalLifecycle<
|
|
|
362
375
|
return { ok: false };
|
|
363
376
|
}
|
|
364
377
|
if (stored.status !== "pending") {
|
|
365
|
-
|
|
378
|
+
// A released waiter does not mean its Tool has finished. Duplicate
|
|
379
|
+
// decisions must not start recovery alongside the live Submission.
|
|
380
|
+
if (
|
|
381
|
+
!this.waiters.has(stored.executionId) &&
|
|
382
|
+
!this.options.isSubmissionActive(submission.submissionId)
|
|
383
|
+
) {
|
|
366
384
|
await this.dispatchContinuation(
|
|
367
385
|
submission,
|
|
368
386
|
stored.executionId,
|
|
@@ -144,9 +144,18 @@ export class InteractionLifecycle<
|
|
|
144
144
|
}
|
|
145
145
|
return value.interaction;
|
|
146
146
|
});
|
|
147
|
-
//
|
|
148
|
-
|
|
149
|
-
|
|
147
|
+
// Register synchronously, then publish while waiting for the person. An
|
|
148
|
+
// early response must release this execution instead of starting recovery.
|
|
149
|
+
const waiting = this.wait(pending, signal);
|
|
150
|
+
try {
|
|
151
|
+
await Promise.all([
|
|
152
|
+
waiting,
|
|
153
|
+
(async () => { await this.options.onInteractionsChanged?.(); })(),
|
|
154
|
+
]);
|
|
155
|
+
} catch (error) {
|
|
156
|
+
this.waiters.get(pending.interactionId)?.();
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
150
159
|
}
|
|
151
160
|
|
|
152
161
|
/**
|
package/src/kernel/state.ts
CHANGED
|
@@ -62,7 +62,10 @@ export interface RuntimeTurnState {
|
|
|
62
62
|
activeMessageId?: string;
|
|
63
63
|
recoveryAttempt?: number;
|
|
64
64
|
recoveryMax?: number;
|
|
65
|
-
recoveryReason?:
|
|
65
|
+
recoveryReason?:
|
|
66
|
+
| "no_meaningful_model_progress"
|
|
67
|
+
| "server_tool_timeout"
|
|
68
|
+
| "transient_model_error";
|
|
66
69
|
steerable: boolean;
|
|
67
70
|
hasPendingSteer: boolean;
|
|
68
71
|
queued: RuntimeQueuedSubmission[];
|
|
@@ -186,10 +186,21 @@ export async function spillDurableToolOutput(
|
|
|
186
186
|
}
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
+
// Native media is budgeted by the model/provider, not by a text JSON limit.
|
|
190
|
+
// Counting base64 here would discard earlier images even though their bytes cannot be truncated.
|
|
191
|
+
function modelTextSize(value: unknown): number {
|
|
192
|
+
if (typeof value === "string") return value.length;
|
|
193
|
+
return JSON.stringify(value, (_key, entry: unknown) =>
|
|
194
|
+
entry !== null && typeof entry === "object" && isBinaryContentLike(entry)
|
|
195
|
+
? "[media]"
|
|
196
|
+
: entry
|
|
197
|
+
)?.length ?? 0;
|
|
198
|
+
}
|
|
199
|
+
|
|
189
200
|
// Bounds leaves first, then removes oldest array entries until the view fits.
|
|
190
201
|
function structureView(output: unknown): unknown {
|
|
191
202
|
const capped = truncateStringLeaves(output, MODEL_LEAF_MAX_CHARS);
|
|
192
|
-
if (
|
|
203
|
+
if (modelTextSize(capped) <= MODEL_REPLAY_THRESHOLD) {
|
|
193
204
|
return capped;
|
|
194
205
|
}
|
|
195
206
|
if (typeof capped !== "object" || capped === null) return capped;
|
|
@@ -204,10 +215,14 @@ function structureView(output: unknown): unknown {
|
|
|
204
215
|
let dropped = 0;
|
|
205
216
|
while (
|
|
206
217
|
items.length > 1 &&
|
|
207
|
-
|
|
218
|
+
modelTextSize({ ...record, [arrayKey]: items }) >
|
|
208
219
|
MODEL_REPLAY_THRESHOLD
|
|
209
220
|
) {
|
|
210
|
-
items.
|
|
221
|
+
const removable = items.findIndex((item) =>
|
|
222
|
+
item === null || typeof item !== "object" || !isBinaryContentLike(item)
|
|
223
|
+
);
|
|
224
|
+
if (removable < 0) break;
|
|
225
|
+
items.splice(removable, 1);
|
|
211
226
|
dropped += 1;
|
|
212
227
|
}
|
|
213
228
|
if (dropped === 0) return capped;
|
|
@@ -224,7 +239,7 @@ function structureView(output: unknown): unknown {
|
|
|
224
239
|
/** Creates a non-destructive, smaller Tool-result projection for the model. */
|
|
225
240
|
export function projectToolOutputForModel(output: unknown): unknown {
|
|
226
241
|
try {
|
|
227
|
-
return
|
|
242
|
+
return modelTextSize(output) <= MODEL_REPLAY_THRESHOLD
|
|
228
243
|
? output
|
|
229
244
|
: structureView(output);
|
|
230
245
|
} catch {
|
|
@@ -83,6 +83,8 @@ const MAX_WRAP_UP_TURNS = 2;
|
|
|
83
83
|
|
|
84
84
|
export class RetryableModelError extends Error {}
|
|
85
85
|
|
|
86
|
+
export class RetryableModelSessionError extends RetryableModelError {}
|
|
87
|
+
|
|
86
88
|
export class ContextOverflowError extends RetryableModelError {}
|
|
87
89
|
|
|
88
90
|
function normalizePlanToolCalls(message: AgentMessage): void {
|
|
@@ -610,7 +612,6 @@ export type PiCanonicalMessageCommit =
|
|
|
610
612
|
export interface PiTurnTerminalIntent {
|
|
611
613
|
readonly outcome: "succeeded" | "failed" | "aborted";
|
|
612
614
|
readonly message?: string;
|
|
613
|
-
readonly resetModelSession?: boolean;
|
|
614
615
|
}
|
|
615
616
|
|
|
616
617
|
export interface PiPreparedTurnRunOptions {
|
|
@@ -866,8 +867,8 @@ export class PreparedPiTurnAdapter {
|
|
|
866
867
|
execute: (toolCallId, input, signal, onUpdate) => {
|
|
867
868
|
const runtimeCandidate = codeExecutionPiToolCandidate(
|
|
868
869
|
factory.create(state.codeExecutionCandidates.map((inner) =>
|
|
869
|
-
// Code Mode
|
|
870
|
-
//
|
|
870
|
+
// Code Mode records its own connector calls. Pi persists the
|
|
871
|
+
// non-idempotent parent execute attempt/result; recording an inner
|
|
871
872
|
// input without a Pi settlement creates an orphan recovery
|
|
872
873
|
// action that cannot be found on the direct Tool surface.
|
|
873
874
|
bindCandidate(inner, toolCallId, false)
|
|
@@ -1058,7 +1059,8 @@ export class PreparedPiTurnAdapter {
|
|
|
1058
1059
|
) || isModelStreamStallMessage(
|
|
1059
1060
|
event.assistantMessageEvent.error.errorMessage,
|
|
1060
1061
|
) ||
|
|
1061
|
-
isRecoverableAssistantError(event.assistantMessageEvent.error)
|
|
1062
|
+
isRecoverableAssistantError(event.assistantMessageEvent.error) ||
|
|
1063
|
+
isOpenRouterServerToolTimeout(event.assistantMessageEvent.error))
|
|
1062
1064
|
) {
|
|
1063
1065
|
return;
|
|
1064
1066
|
}
|
|
@@ -1073,10 +1075,11 @@ export class PreparedPiTurnAdapter {
|
|
|
1073
1075
|
}
|
|
1074
1076
|
const stalled = message.stopReason === "error" &&
|
|
1075
1077
|
isModelStreamStallMessage(message.errorMessage);
|
|
1078
|
+
const resetModelSession = isOpenRouterServerToolTimeout(message);
|
|
1076
1079
|
const retryable = isRecoverableAssistantError(message);
|
|
1077
|
-
if (stalled || retryable) {
|
|
1080
|
+
if (stalled || retryable || resetModelSession) {
|
|
1078
1081
|
const content = visibleAssistantContent(message);
|
|
1079
|
-
if (content.length > 0) {
|
|
1082
|
+
if (!resetModelSession && content.length > 0) {
|
|
1080
1083
|
this.assistantOrdinal += 1;
|
|
1081
1084
|
await this.options.onCanonicalMessage({
|
|
1082
1085
|
kind: "commit-turn",
|
|
@@ -1090,6 +1093,9 @@ export class PreparedPiTurnAdapter {
|
|
|
1090
1093
|
if (stalled) {
|
|
1091
1094
|
throw new ChatStreamStalledError(message.errorMessage!);
|
|
1092
1095
|
}
|
|
1096
|
+
if (resetModelSession) {
|
|
1097
|
+
throw new RetryableModelSessionError(message.errorMessage);
|
|
1098
|
+
}
|
|
1093
1099
|
throw new RetryableModelError(message.errorMessage);
|
|
1094
1100
|
}
|
|
1095
1101
|
}
|
|
@@ -1136,9 +1142,6 @@ export class PreparedPiTurnAdapter {
|
|
|
1136
1142
|
);
|
|
1137
1143
|
this.terminalIntent = {
|
|
1138
1144
|
outcome: terminal.outcome,
|
|
1139
|
-
...(isOpenRouterServerToolTimeout(authoritativeMessage)
|
|
1140
|
-
? { resetModelSession: true }
|
|
1141
|
-
: {}),
|
|
1142
1145
|
...(authoritativeMessage.errorMessage
|
|
1143
1146
|
? { message: authoritativeMessage.errorMessage }
|
|
1144
1147
|
: terminal.message
|
|
@@ -2,7 +2,11 @@ import {
|
|
|
2
2
|
PreparedPiTurnAdapter,
|
|
3
3
|
type CreatePreparedPiTurnOptions,
|
|
4
4
|
} from "./execution";
|
|
5
|
-
export {
|
|
5
|
+
export {
|
|
6
|
+
ContextOverflowError,
|
|
7
|
+
RetryableModelError,
|
|
8
|
+
RetryableModelSessionError,
|
|
9
|
+
} from "./execution";
|
|
6
10
|
import {
|
|
7
11
|
uiUserMessageToPi,
|
|
8
12
|
} from "../message";
|
|
@@ -26,6 +30,8 @@ import {
|
|
|
26
30
|
} from "./assembly";
|
|
27
31
|
import {
|
|
28
32
|
decidePiRecovery,
|
|
33
|
+
recordPiToolMilestone,
|
|
34
|
+
type PiToolMilestoneInput,
|
|
29
35
|
PiRuntimeRecoveryAdapter,
|
|
30
36
|
type PiRecoveryHost,
|
|
31
37
|
type PiRecoveryInput,
|
|
@@ -176,15 +182,20 @@ export class PiRuntimeAdapter {
|
|
|
176
182
|
* 根据持久化 milestone 和一个命令算出恢复时要写什么、下一步做什么。
|
|
177
183
|
*
|
|
178
184
|
* @remarks
|
|
179
|
-
* 调用方:Runtime 和 approval
|
|
185
|
+
* 调用方:Runtime 和 approval 生命周期在检查恢复状态、处理审批或提交终态时调用。
|
|
180
186
|
*
|
|
181
187
|
* 实现理由:该方法只返回 durable mutations 和一个 effect,实际写库与调度仍由 Runtime 完成。
|
|
182
|
-
*
|
|
188
|
+
* 恢复规划保留在此入口;正常 Tool 写入复用同一命令 reducer,但不重复规划整个 Turn。
|
|
183
189
|
*/
|
|
184
190
|
decideRecovery(options: PiRecoveryInput) {
|
|
185
191
|
return decidePiRecovery(options);
|
|
186
192
|
}
|
|
187
193
|
|
|
194
|
+
/** Active execution records only the indexed Tool fact; cold recovery still replays the full log. */
|
|
195
|
+
recordToolMilestone(options: PiToolMilestoneInput) {
|
|
196
|
+
return recordPiToolMilestone(options);
|
|
197
|
+
}
|
|
198
|
+
|
|
188
199
|
/**
|
|
189
200
|
* 把 Runtime 提供的持久化和重试回调包装成通用聊天恢复端口。
|
|
190
201
|
*
|
|
@@ -215,6 +215,7 @@ export function isRecoverableAssistantError(
|
|
|
215
215
|
message.stopReason === "error" &&
|
|
216
216
|
/thought[_ ]signature/i.test(message.errorMessage ?? "")
|
|
217
217
|
) return false;
|
|
218
|
+
// Runtime resets the model Session and gives this timeout its own one-retry budget.
|
|
218
219
|
if (isOpenRouterServerToolTimeout(message)) return false;
|
|
219
220
|
return isRetryableAssistantError(message) ||
|
|
220
221
|
(message.stopReason === "error" &&
|
|
@@ -369,15 +369,11 @@ function fromDispatch(
|
|
|
369
369
|
// #region Recovery decision
|
|
370
370
|
|
|
371
371
|
/**
|
|
372
|
-
*
|
|
373
|
-
*
|
|
374
|
-
*
|
|
375
|
-
*
|
|
376
|
-
* 这里保持纯计算并只返回 mutations 和 effect;调用方必须先持久化 mutations,再执行 effect,不能把 I/O 搬进来破坏可重放性。
|
|
372
|
+
* Apply a durable command using the same validation during execution and recovery.
|
|
373
|
+
* Active Tool recording supplies only its indexed milestone; recovery supplies
|
|
374
|
+
* the full log and then plans the next effect. Neither path performs I/O.
|
|
377
375
|
*/
|
|
378
|
-
|
|
379
|
-
input: PiRecoveryInput,
|
|
380
|
-
): PiRecoveryDecision {
|
|
376
|
+
function applyPiRecoveryCommand(input: PiRecoveryInput) {
|
|
381
377
|
let bodies = [...input.milestoneBodies];
|
|
382
378
|
let state = replayPiToolRecovery(bodies);
|
|
383
379
|
const mutations: PiDurableMutation[] = [];
|
|
@@ -677,6 +673,22 @@ export function decidePiRecovery(
|
|
|
677
673
|
applied = true;
|
|
678
674
|
}
|
|
679
675
|
|
|
676
|
+
return { state, bodies, mutations, applied };
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
export interface PiToolMilestoneInput extends PiRecoveryInput {
|
|
680
|
+
readonly command: Extract<PiRecoveryCommand, {
|
|
681
|
+
kind: "record-tool-input" | "record-tool-result";
|
|
682
|
+
}>;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/** Record a single Tool fact without planning a recovery of the active Turn. */
|
|
686
|
+
export function recordPiToolMilestone(input: PiToolMilestoneInput): readonly PiDurableMutation[] {
|
|
687
|
+
return applyPiRecoveryCommand(input).mutations;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
export function decidePiRecovery(input: PiRecoveryInput): PiRecoveryDecision {
|
|
691
|
+
const { state, bodies, mutations, applied } = applyPiRecoveryCommand(input);
|
|
680
692
|
const planned = fromPlan(
|
|
681
693
|
state,
|
|
682
694
|
planPiToolRecovery(state),
|
package/src/pi/tool/compiler.ts
CHANGED
|
@@ -108,6 +108,10 @@ export interface SettledPiToolCall {
|
|
|
108
108
|
|
|
109
109
|
const MAX_TOOL_RETRIES = 3;
|
|
110
110
|
const TURN_FAILURE_CEILING = 8;
|
|
111
|
+
// DO SQLite rows also contain arguments and transcript/recovery envelopes.
|
|
112
|
+
// Keep each serialized result below half the 2 MiB row limit.
|
|
113
|
+
const DURABLE_TOOL_RESULT_MAX_BYTES = 1024 * 1024;
|
|
114
|
+
const MEDIA_CAPTION_MAX_CHARS = 4 * 1024;
|
|
111
115
|
|
|
112
116
|
interface PiToolGovernanceState {
|
|
113
117
|
readonly failures: Map<string, number>;
|
|
@@ -177,6 +181,15 @@ function artifactResult(
|
|
|
177
181
|
result: AgentToolResult<unknown>,
|
|
178
182
|
ref: ArtifactRef,
|
|
179
183
|
): AgentToolResult<unknown> {
|
|
184
|
+
let captionRemaining = MEDIA_CAPTION_MAX_CHARS;
|
|
185
|
+
const mediaContent = result.content.some((part) => part.type !== "text")
|
|
186
|
+
? result.content.flatMap((part): AgentToolResult<unknown>["content"] => {
|
|
187
|
+
if (part.type !== "text") return [part];
|
|
188
|
+
const text = part.text.slice(0, captionRemaining);
|
|
189
|
+
captionRemaining -= text.length;
|
|
190
|
+
return text ? [{ ...part, text }] : [];
|
|
191
|
+
})
|
|
192
|
+
: [];
|
|
180
193
|
return {
|
|
181
194
|
...result,
|
|
182
195
|
content: [{
|
|
@@ -187,7 +200,7 @@ function artifactResult(
|
|
|
187
200
|
`Bytes: ${ref.bytes}`,
|
|
188
201
|
`Preview:\n${ref.preview}`,
|
|
189
202
|
].join("\n"),
|
|
190
|
-
}],
|
|
203
|
+
}, ...mediaContent],
|
|
191
204
|
details: ref,
|
|
192
205
|
};
|
|
193
206
|
}
|
|
@@ -197,13 +210,24 @@ async function durableResult(
|
|
|
197
210
|
candidate: PiToolCandidate,
|
|
198
211
|
workspace: SpillWorkspace | undefined,
|
|
199
212
|
): Promise<AgentToolResult<unknown>> {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
213
|
+
const spill = (candidate.outputBudget ?? { kind: "spill" }).kind === "spill";
|
|
214
|
+
let ref = spill
|
|
215
|
+
? await spillDurableToolOutput(spillPayload(result), { workspace })
|
|
216
|
+
: null;
|
|
217
|
+
const bounded = ref
|
|
218
|
+
? artifactResult(result, ref)
|
|
219
|
+
: boundDurableToolOutput(result) as AgentToolResult<unknown>;
|
|
220
|
+
const bytes = new TextEncoder().encode(JSON.stringify(bounded)).byteLength;
|
|
221
|
+
if (bytes <= DURABLE_TOOL_RESULT_MAX_BYTES) return bounded;
|
|
222
|
+
|
|
223
|
+
// Binary blocks cannot be truncated, and this boundary has no image resizer.
|
|
224
|
+
// Preserve the original where possible and fail before any SQLite write.
|
|
225
|
+
if (!spill) ref = await spillDurableToolOutput(spillPayload(result), { workspace });
|
|
226
|
+
throw new Error(
|
|
227
|
+
`Tool output exceeds the 1 MiB durable result limit (${bytes} bytes). ` +
|
|
228
|
+
"Return a smaller result, such as lower-resolution images or fewer media items; the original cannot be included inline. " +
|
|
229
|
+
(ref ? `Original output saved to Workspace: ${ref.path}` : "The original output could not be saved to Workspace."),
|
|
230
|
+
);
|
|
207
231
|
}
|
|
208
232
|
|
|
209
233
|
async function untilAborted<T>(
|
package/src/pi/tool/core.ts
CHANGED
|
@@ -246,10 +246,17 @@ function runCodemode(
|
|
|
246
246
|
}), CODEMODE_EXECUTE_TIMEOUT_MS);
|
|
247
247
|
});
|
|
248
248
|
try {
|
|
249
|
-
|
|
249
|
+
const output = await Promise.race([
|
|
250
250
|
runtime.execute(input),
|
|
251
251
|
deadline.then(result),
|
|
252
|
-
])
|
|
252
|
+
]);
|
|
253
|
+
const details = output.details as { status?: unknown; error?: unknown } | null;
|
|
254
|
+
if (details?.status === "error") {
|
|
255
|
+
throw new Error(typeof details.error === "string"
|
|
256
|
+
? details.error
|
|
257
|
+
: `${label} failed`);
|
|
258
|
+
}
|
|
259
|
+
return project(output);
|
|
253
260
|
} finally {
|
|
254
261
|
if (timeout !== undefined) clearTimeout(timeout);
|
|
255
262
|
}
|
|
@@ -257,12 +264,12 @@ function runCodemode(
|
|
|
257
264
|
}
|
|
258
265
|
|
|
259
266
|
/**
|
|
260
|
-
* 把 Cloudflare Codemode
|
|
267
|
+
* 把 Cloudflare Codemode execution port 包装为 Pi 代码执行工具候选项。
|
|
261
268
|
*
|
|
262
269
|
* Tool Surface 收到 Host 已组装的 Code Execution Port 后调用,
|
|
263
270
|
* 模型再通过 `execute` 运行代码。
|
|
264
271
|
*
|
|
265
|
-
* Code Mode
|
|
272
|
+
* Code Mode 的外层执行声明为 safe;内部候选仍经过各自的执行档位与审批门禁。
|
|
266
273
|
*/
|
|
267
274
|
export function codeExecutionPiToolCandidate(
|
|
268
275
|
runtime: RuntimeCodeExecutionPort,
|
|
@@ -273,13 +280,15 @@ export function codeExecutionPiToolCandidate(
|
|
|
273
280
|
description: runtime.description,
|
|
274
281
|
parameters: executeParameters,
|
|
275
282
|
// Pi 工具循环在模型选择 `execute` 时调用,调用前允许 Turn 取消。
|
|
276
|
-
//
|
|
283
|
+
// 上游保留 connector 执行日志,但此调用会创建新 execution,不能把它当作旧执行的恢复。
|
|
277
284
|
execute: runCodemode(runtime, "Code Mode execute"),
|
|
278
285
|
};
|
|
279
286
|
return {
|
|
280
287
|
owner: "core:codemode",
|
|
281
288
|
requiredExecutionLevel: "safe",
|
|
282
|
-
|
|
289
|
+
// Each upstream execute starts a new execution ID. A lost parent result
|
|
290
|
+
// cannot safely replay its already-applied connector or Workspace writes.
|
|
291
|
+
retry: "non-idempotent",
|
|
283
292
|
outputBudget: { kind: "structure" },
|
|
284
293
|
source: "codemode",
|
|
285
294
|
summary: "Run JavaScript with network and configured connector access",
|
|
@@ -392,9 +392,10 @@ export function workspacePiToolCandidates(
|
|
|
392
392
|
if (typeof path !== "string") {
|
|
393
393
|
return removeTool.execute(toolCallId, params, signal, onUpdate);
|
|
394
394
|
}
|
|
395
|
-
return serializeWorkspaceMutation(workspace, path, () =>
|
|
396
|
-
|
|
397
|
-
|
|
395
|
+
return serializeWorkspaceMutation(workspace, path, () => {
|
|
396
|
+
signal?.throwIfAborted();
|
|
397
|
+
return removeTool.execute(toolCallId, params, signal, onUpdate);
|
|
398
|
+
});
|
|
398
399
|
},
|
|
399
400
|
};
|
|
400
401
|
|
package/src/runtime.ts
CHANGED
|
@@ -71,6 +71,7 @@ import {
|
|
|
71
71
|
MODEL_STREAM_STALL_TIMEOUT_MS,
|
|
72
72
|
readModelStreamStallDetails,
|
|
73
73
|
RetryableModelError,
|
|
74
|
+
RetryableModelSessionError,
|
|
74
75
|
type PiCanonicalUserInput,
|
|
75
76
|
type PiCanonicalTranscriptSnapshot,
|
|
76
77
|
type PiChatRecoveryData,
|
|
@@ -145,6 +146,7 @@ const CONTINUATION_WATCHDOG_SECONDS = 300;
|
|
|
145
146
|
// Host 侧的一次抖动会把 Session 永久卡在「队列已满」。
|
|
146
147
|
const ADMISSION_ATTEMPT_WINDOW_MS = 60_000;
|
|
147
148
|
export const CHAT_RECOVERY_MAX_ATTEMPTS = 5;
|
|
149
|
+
const SERVER_TOOL_TIMEOUT_MAX_ATTEMPTS = 2;
|
|
148
150
|
// stall 单独收窄。理由不是「stall 更不值得救」,而是它的重试**期望值和别的错不一样**:
|
|
149
151
|
// 瞬时错(5xx / 断流)重跑一次往往就好了;stall 的重跑是拿同一份 transcript 让模型
|
|
150
152
|
// 重新想同样久,如果它本来就超预算,再跑几次也一样超。把看门狗放宽到 240s 之后,
|
|
@@ -177,10 +179,7 @@ interface StoredSubmission extends SubmissionReceipt {
|
|
|
177
179
|
userMessageId?: string | null;
|
|
178
180
|
regenerateMessageId?: string | null;
|
|
179
181
|
recoveryErrorCount: number;
|
|
180
|
-
recoveryReason?:
|
|
181
|
-
| "no_meaningful_model_progress"
|
|
182
|
-
| "transient_model_error"
|
|
183
|
-
| "context_overflow";
|
|
182
|
+
recoveryReason?: SubmissionRecoveryReason;
|
|
184
183
|
runId: string | null;
|
|
185
184
|
accountId: string | null;
|
|
186
185
|
rateVersion: number | null;
|
|
@@ -249,7 +248,7 @@ type RecoveredTurnPreparation =
|
|
|
249
248
|
// 原因:终态文本会直接呈现给用户,必须说明发生了什么以及能不能重试。
|
|
250
249
|
function unresumableWaitMessage(reason: string | undefined): string {
|
|
251
250
|
return reason === "uncertain-tool"
|
|
252
|
-
? "SpringBrand stopped this turn: a non-idempotent Tool was interrupted and its result cannot be confirmed, so it was not retried.
|
|
251
|
+
? "SpringBrand stopped this turn: a non-idempotent Tool was interrupted and its result cannot be confirmed, so it was not retried. Check whether the external action already happened before sending another request."
|
|
253
252
|
: "SpringBrand could not resume this turn after an interruption.";
|
|
254
253
|
}
|
|
255
254
|
|
|
@@ -601,6 +600,7 @@ export abstract class AgentRuntimeKernel<
|
|
|
601
600
|
pi: this.pi,
|
|
602
601
|
findSubmission: (submissionId) =>
|
|
603
602
|
this.readSubmission(submissionId),
|
|
603
|
+
isSubmissionActive: (submissionId) => this.submissions.isActive(submissionId),
|
|
604
604
|
applyRecoveryMutations: (submission, mutations) =>
|
|
605
605
|
this.applyPiRecoveryMutations(submission, mutations),
|
|
606
606
|
materializeRecoveredToolResults: (submission) =>
|
|
@@ -893,7 +893,6 @@ export abstract class AgentRuntimeKernel<
|
|
|
893
893
|
terminal: {
|
|
894
894
|
outcome: "succeeded" | "failed" | "aborted";
|
|
895
895
|
message?: string;
|
|
896
|
-
resetModelSession?: boolean;
|
|
897
896
|
},
|
|
898
897
|
) => void | Promise<void>;
|
|
899
898
|
} = {},
|
|
@@ -2144,6 +2143,27 @@ export abstract class AgentRuntimeKernel<
|
|
|
2144
2143
|
});
|
|
2145
2144
|
}
|
|
2146
2145
|
|
|
2146
|
+
private recordPiToolMilestone(
|
|
2147
|
+
submission: StoredSubmission,
|
|
2148
|
+
command: Extract<PiRecoveryCommand, {
|
|
2149
|
+
kind: "record-tool-input" | "record-tool-result";
|
|
2150
|
+
}>,
|
|
2151
|
+
): readonly PiDurableMutation[] {
|
|
2152
|
+
const key = command.kind === "record-tool-input"
|
|
2153
|
+
? `tool-input:${command.record.toolCallId}`
|
|
2154
|
+
: `tool-result:${command.toolCallId}`;
|
|
2155
|
+
const body = this.db.milestones.findBody(submission.submissionId, key);
|
|
2156
|
+
return this.pi.recordToolMilestone({
|
|
2157
|
+
milestoneBodies: body === null ? [] : [body],
|
|
2158
|
+
identity: {
|
|
2159
|
+
turnId: submission.submissionId,
|
|
2160
|
+
assemblyRevision: submission.assemblyRevision,
|
|
2161
|
+
},
|
|
2162
|
+
command,
|
|
2163
|
+
now: Date.now(),
|
|
2164
|
+
});
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2147
2167
|
// 作用:把 Pi 恢复决策产生的变更写入 Runtime 数据库。
|
|
2148
2168
|
// 调用:记录 Tool、终态、审批或恢复步骤时,通常在事务内调用。
|
|
2149
2169
|
// 原因:upsert 保证重放幂等,而 terminal intent 允许后来的更强结果更新早期意图。
|
|
@@ -2207,15 +2227,12 @@ export abstract class AgentRuntimeKernel<
|
|
|
2207
2227
|
submission: StoredSubmission,
|
|
2208
2228
|
input: PiToolInputRecord,
|
|
2209
2229
|
): boolean {
|
|
2210
|
-
const decision = this.decidePiRecovery(submission, {
|
|
2211
|
-
kind: "record-tool-input",
|
|
2212
|
-
record: input,
|
|
2213
|
-
});
|
|
2214
2230
|
const appended = this.db.transaction(() => {
|
|
2215
|
-
const
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
);
|
|
2231
|
+
const mutations = this.recordPiToolMilestone(submission, {
|
|
2232
|
+
kind: "record-tool-input",
|
|
2233
|
+
record: input,
|
|
2234
|
+
});
|
|
2235
|
+
const inserted = this.applyPiRecoveryMutations(submission, mutations);
|
|
2219
2236
|
if (inserted) {
|
|
2220
2237
|
this.telemetry.capture("toolStarted", {
|
|
2221
2238
|
submissionId: submission.submissionId,
|
|
@@ -2361,15 +2378,18 @@ export abstract class AgentRuntimeKernel<
|
|
|
2361
2378
|
occurredAt: createdAt,
|
|
2362
2379
|
});
|
|
2363
2380
|
}
|
|
2364
|
-
const
|
|
2381
|
+
const mutations = this.recordPiToolMilestone(submission, {
|
|
2365
2382
|
kind: "record-tool-result",
|
|
2366
2383
|
toolCallId: call.toolCallId,
|
|
2367
2384
|
toolName: call.toolName,
|
|
2368
2385
|
result: call.result,
|
|
2369
2386
|
isError: call.isError,
|
|
2370
2387
|
timestamp: createdAt,
|
|
2388
|
+
// The current execution is already continuing. After a crash, recovery
|
|
2389
|
+
// materializes this result before resuming; no per-Tool dispatch is needed.
|
|
2390
|
+
needsContinuation: false,
|
|
2371
2391
|
});
|
|
2372
|
-
this.applyPiRecoveryMutations(submission,
|
|
2392
|
+
this.applyPiRecoveryMutations(submission, mutations);
|
|
2373
2393
|
this.approvals.commitContinuation(submission, call.toolCallId);
|
|
2374
2394
|
}
|
|
2375
2395
|
|
|
@@ -3062,7 +3082,6 @@ export abstract class AgentRuntimeKernel<
|
|
|
3062
3082
|
| {
|
|
3063
3083
|
outcome: "succeeded" | "failed" | "aborted";
|
|
3064
3084
|
message?: string;
|
|
3065
|
-
resetModelSession?: boolean;
|
|
3066
3085
|
}
|
|
3067
3086
|
| undefined;
|
|
3068
3087
|
const startedAt = ready.createdAt;
|
|
@@ -3082,13 +3101,6 @@ export abstract class AgentRuntimeKernel<
|
|
|
3082
3101
|
intent.outcome,
|
|
3083
3102
|
intent.message,
|
|
3084
3103
|
);
|
|
3085
|
-
if (intent.resetModelSession) {
|
|
3086
|
-
this.modelSessionId = crypto.randomUUID();
|
|
3087
|
-
await this.ctx.storage.put(
|
|
3088
|
-
MODEL_SESSION_ID_STORAGE_KEY,
|
|
3089
|
-
this.modelSessionId,
|
|
3090
|
-
);
|
|
3091
|
-
}
|
|
3092
3104
|
},
|
|
3093
3105
|
});
|
|
3094
3106
|
for (const pending of this.db.steers.listForSubmission(submissionId)) {
|
|
@@ -3212,13 +3224,23 @@ export abstract class AgentRuntimeKernel<
|
|
|
3212
3224
|
}
|
|
3213
3225
|
const stalled = error instanceof ChatStreamStalledError;
|
|
3214
3226
|
const contextOverflow = error instanceof ContextOverflowError;
|
|
3227
|
+
const resetModelSession = error instanceof RetryableModelSessionError;
|
|
3215
3228
|
const abortReason = this.readSubmission(submissionId)?.abortReason;
|
|
3216
3229
|
if (!abortReason && (stalled || error instanceof RetryableModelError)) {
|
|
3230
|
+
if (resetModelSession) {
|
|
3231
|
+
this.modelSessionId = crypto.randomUUID();
|
|
3232
|
+
await this.ctx.storage.put(
|
|
3233
|
+
MODEL_SESSION_ID_STORAGE_KEY,
|
|
3234
|
+
this.modelSessionId,
|
|
3235
|
+
);
|
|
3236
|
+
}
|
|
3217
3237
|
const recoveryReason: SubmissionRecoveryReason = stalled
|
|
3218
3238
|
? "no_meaningful_model_progress"
|
|
3219
|
-
:
|
|
3220
|
-
? "
|
|
3221
|
-
:
|
|
3239
|
+
: resetModelSession
|
|
3240
|
+
? "server_tool_timeout"
|
|
3241
|
+
: contextOverflow
|
|
3242
|
+
? "context_overflow"
|
|
3243
|
+
: "transient_model_error";
|
|
3222
3244
|
let contextOverflowRecoveryClaimed = false;
|
|
3223
3245
|
const recoveryErrorCount = this.db.transaction(() => {
|
|
3224
3246
|
if (contextOverflow) {
|
|
@@ -3253,7 +3275,11 @@ export abstract class AgentRuntimeKernel<
|
|
|
3253
3275
|
? contextOverflowRecoveryClaimed &&
|
|
3254
3276
|
recoveryErrorCount < CHAT_RECOVERY_MAX_ATTEMPTS
|
|
3255
3277
|
: recoveryErrorCount <
|
|
3256
|
-
(stalled
|
|
3278
|
+
(stalled
|
|
3279
|
+
? CHAT_STALL_MAX_ATTEMPTS
|
|
3280
|
+
: resetModelSession
|
|
3281
|
+
? SERVER_TOOL_TIMEOUT_MAX_ATTEMPTS
|
|
3282
|
+
: CHAT_RECOVERY_MAX_ATTEMPTS);
|
|
3257
3283
|
if (
|
|
3258
3284
|
!stoppedDuringRecovery &&
|
|
3259
3285
|
retryAllowed
|
|
@@ -4710,7 +4736,9 @@ export abstract class AgentRuntimeKernel<
|
|
|
4710
4736
|
: current.recoveryReason ===
|
|
4711
4737
|
"no_meaningful_model_progress"
|
|
4712
4738
|
? CHAT_STALL_MAX_ATTEMPTS
|
|
4713
|
-
:
|
|
4739
|
+
: current.recoveryReason === "server_tool_timeout"
|
|
4740
|
+
? SERVER_TOOL_TIMEOUT_MAX_ATTEMPTS
|
|
4741
|
+
: CHAT_RECOVERY_MAX_ATTEMPTS,
|
|
4714
4742
|
...(current.recoveryReason &&
|
|
4715
4743
|
current.recoveryReason !== "context_overflow"
|
|
4716
4744
|
? { recoveryReason: current.recoveryReason }
|