@springbrand/agent-runtime 0.2.0-alpha.61 → 0.2.0-alpha.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@springbrand/agent-runtime",
3
- "version": "0.2.0-alpha.61",
3
+ "version": "0.2.0-alpha.63",
4
4
  "type": "module",
5
5
  "files": [
6
6
  "src",
@@ -35,6 +35,6 @@
35
35
  },
36
36
  "devDependencies": {
37
37
  "agents": "0.22.0",
38
- "ai": "7.0.37"
38
+ "ai": "7.0.56"
39
39
  }
40
40
  }
@@ -33,6 +33,14 @@ export class RecoveryMilestoneRepository {
33
33
  // 恢复事实必须与 Submission 位于同一 Agent 存储中,不能用另一个 sql 入口拼接。
34
34
  constructor(private readonly sql: SqlTaggedTemplate) {}
35
35
 
36
+ /** The existing unique index makes active Tool writes independent of Turn history size. */
37
+ findBody(submissionId: string, milestoneKey: string): string | null {
38
+ return this.sql<{ body: string }>`
39
+ SELECT body FROM pi_recovery_milestones
40
+ WHERE submission_id = ${submissionId} AND milestone_key = ${milestoneKey}
41
+ `[0]?.body ?? null;
42
+ }
43
+
36
44
  // 按写入顺序返回某次 Submission 的全部恢复内容。
37
45
  // Runtime 和 ApprovalLifecycle 在计算下一个恢复决策前调用,调用方直接把结果交给 Pi 恢复器。
38
46
  // ORDER BY seq 是重放契约,不能改成 milestoneKey 或无排序查询,否则决策顺序会变。
@@ -11,6 +11,7 @@ export type SubmissionStatus =
11
11
  | "error";
12
12
  export type SubmissionRecoveryReason =
13
13
  | "no_meaningful_model_progress"
14
+ | "server_tool_timeout"
14
15
  | "transient_model_error"
15
16
  | "context_overflow";
16
17
 
@@ -72,6 +72,7 @@ interface ApprovalLifecycleOptions<
72
72
  readonly db: RuntimeDatabase;
73
73
  readonly pi: PiRuntimeAdapter;
74
74
  readonly findSubmission: (submissionId: string) => TSubmission | null;
75
+ readonly isSubmissionActive: (submissionId: string) => boolean;
75
76
  readonly applyRecoveryMutations: (
76
77
  submission: TSubmission,
77
78
  mutations: readonly PiDurableMutation[],
@@ -227,11 +228,23 @@ export class ApprovalLifecycle<
227
228
  }
228
229
  return value;
229
230
  });
230
- if (pending.created) {
231
- await this.options.onApprovalsChanged();
232
- await onApproval?.(pending.approval);
231
+ // Install the waiter before publishing pending state. Otherwise an early
232
+ // decision can schedule a second executor while this one is still alive.
233
+ const waiting = this.wait(pending.approval, signal);
234
+ try {
235
+ await Promise.all([
236
+ waiting,
237
+ (async () => {
238
+ if (pending.created) {
239
+ await this.options.onApprovalsChanged();
240
+ await onApproval?.(pending.approval);
241
+ }
242
+ })(),
243
+ ]);
244
+ } catch (error) {
245
+ this.waiters.get(pending.approval.executionId)?.release();
246
+ throw error;
233
247
  }
234
- return this.wait(pending.approval, signal);
235
248
  }
236
249
 
237
250
  /**
@@ -362,7 +375,12 @@ export class ApprovalLifecycle<
362
375
  return { ok: false };
363
376
  }
364
377
  if (stored.status !== "pending") {
365
- if (!this.waiters.has(stored.executionId)) {
378
+ // A released waiter does not mean its Tool has finished. Duplicate
379
+ // decisions must not start recovery alongside the live Submission.
380
+ if (
381
+ !this.waiters.has(stored.executionId) &&
382
+ !this.options.isSubmissionActive(submission.submissionId)
383
+ ) {
366
384
  await this.dispatchContinuation(
367
385
  submission,
368
386
  stored.executionId,
@@ -144,9 +144,18 @@ export class InteractionLifecycle<
144
144
  }
145
145
  return value.interaction;
146
146
  });
147
- // 必须在 wait 之前通知:wait 会一直挂到客户端投递,之后再通知就晚了一个回合。
148
- await this.options.onInteractionsChanged?.();
149
- return this.wait(pending, signal);
147
+ // Register synchronously, then publish while waiting for the person. An
148
+ // early response must release this execution instead of starting recovery.
149
+ const waiting = this.wait(pending, signal);
150
+ try {
151
+ await Promise.all([
152
+ waiting,
153
+ (async () => { await this.options.onInteractionsChanged?.(); })(),
154
+ ]);
155
+ } catch (error) {
156
+ this.waiters.get(pending.interactionId)?.();
157
+ throw error;
158
+ }
150
159
  }
151
160
 
152
161
  /**
@@ -62,7 +62,10 @@ export interface RuntimeTurnState {
62
62
  activeMessageId?: string;
63
63
  recoveryAttempt?: number;
64
64
  recoveryMax?: number;
65
- recoveryReason?: "no_meaningful_model_progress" | "transient_model_error";
65
+ recoveryReason?:
66
+ | "no_meaningful_model_progress"
67
+ | "server_tool_timeout"
68
+ | "transient_model_error";
66
69
  steerable: boolean;
67
70
  hasPendingSteer: boolean;
68
71
  queued: RuntimeQueuedSubmission[];
@@ -186,10 +186,21 @@ export async function spillDurableToolOutput(
186
186
  }
187
187
  }
188
188
 
189
+ // Native media is budgeted by the model/provider, not by a text JSON limit.
190
+ // Counting base64 here would discard earlier images even though their bytes cannot be truncated.
191
+ function modelTextSize(value: unknown): number {
192
+ if (typeof value === "string") return value.length;
193
+ return JSON.stringify(value, (_key, entry: unknown) =>
194
+ entry !== null && typeof entry === "object" && isBinaryContentLike(entry)
195
+ ? "[media]"
196
+ : entry
197
+ )?.length ?? 0;
198
+ }
199
+
189
200
  // Bounds leaves first, then removes oldest array entries until the view fits.
190
201
  function structureView(output: unknown): unknown {
191
202
  const capped = truncateStringLeaves(output, MODEL_LEAF_MAX_CHARS);
192
- if (serializeOutput(capped).text.length <= MODEL_REPLAY_THRESHOLD) {
203
+ if (modelTextSize(capped) <= MODEL_REPLAY_THRESHOLD) {
193
204
  return capped;
194
205
  }
195
206
  if (typeof capped !== "object" || capped === null) return capped;
@@ -204,10 +215,14 @@ function structureView(output: unknown): unknown {
204
215
  let dropped = 0;
205
216
  while (
206
217
  items.length > 1 &&
207
- serializeOutput({ ...record, [arrayKey]: items }).text.length >
218
+ modelTextSize({ ...record, [arrayKey]: items }) >
208
219
  MODEL_REPLAY_THRESHOLD
209
220
  ) {
210
- items.shift();
221
+ const removable = items.findIndex((item) =>
222
+ item === null || typeof item !== "object" || !isBinaryContentLike(item)
223
+ );
224
+ if (removable < 0) break;
225
+ items.splice(removable, 1);
211
226
  dropped += 1;
212
227
  }
213
228
  if (dropped === 0) return capped;
@@ -224,7 +239,7 @@ function structureView(output: unknown): unknown {
224
239
  /** Creates a non-destructive, smaller Tool-result projection for the model. */
225
240
  export function projectToolOutputForModel(output: unknown): unknown {
226
241
  try {
227
- return serializeOutput(output).text.length <= MODEL_REPLAY_THRESHOLD
242
+ return modelTextSize(output) <= MODEL_REPLAY_THRESHOLD
228
243
  ? output
229
244
  : structureView(output);
230
245
  } catch {
@@ -83,6 +83,8 @@ const MAX_WRAP_UP_TURNS = 2;
83
83
 
84
84
  export class RetryableModelError extends Error {}
85
85
 
86
+ export class RetryableModelSessionError extends RetryableModelError {}
87
+
86
88
  export class ContextOverflowError extends RetryableModelError {}
87
89
 
88
90
  function normalizePlanToolCalls(message: AgentMessage): void {
@@ -610,7 +612,6 @@ export type PiCanonicalMessageCommit =
610
612
  export interface PiTurnTerminalIntent {
611
613
  readonly outcome: "succeeded" | "failed" | "aborted";
612
614
  readonly message?: string;
613
- readonly resetModelSession?: boolean;
614
615
  }
615
616
 
616
617
  export interface PiPreparedTurnRunOptions {
@@ -866,8 +867,8 @@ export class PreparedPiTurnAdapter {
866
867
  execute: (toolCallId, input, signal, onUpdate) => {
867
868
  const runtimeCandidate = codeExecutionPiToolCandidate(
868
869
  factory.create(state.codeExecutionCandidates.map((inner) =>
869
- // Code Mode owns replay of its connector calls. Pi persists
870
- // only the parent execute attempt/result; recording an inner
870
+ // Code Mode records its own connector calls. Pi persists the
871
+ // non-idempotent parent execute attempt/result; recording an inner
871
872
  // input without a Pi settlement creates an orphan recovery
872
873
  // action that cannot be found on the direct Tool surface.
873
874
  bindCandidate(inner, toolCallId, false)
@@ -1058,7 +1059,8 @@ export class PreparedPiTurnAdapter {
1058
1059
  ) || isModelStreamStallMessage(
1059
1060
  event.assistantMessageEvent.error.errorMessage,
1060
1061
  ) ||
1061
- isRecoverableAssistantError(event.assistantMessageEvent.error))
1062
+ isRecoverableAssistantError(event.assistantMessageEvent.error) ||
1063
+ isOpenRouterServerToolTimeout(event.assistantMessageEvent.error))
1062
1064
  ) {
1063
1065
  return;
1064
1066
  }
@@ -1073,10 +1075,11 @@ export class PreparedPiTurnAdapter {
1073
1075
  }
1074
1076
  const stalled = message.stopReason === "error" &&
1075
1077
  isModelStreamStallMessage(message.errorMessage);
1078
+ const resetModelSession = isOpenRouterServerToolTimeout(message);
1076
1079
  const retryable = isRecoverableAssistantError(message);
1077
- if (stalled || retryable) {
1080
+ if (stalled || retryable || resetModelSession) {
1078
1081
  const content = visibleAssistantContent(message);
1079
- if (content.length > 0) {
1082
+ if (!resetModelSession && content.length > 0) {
1080
1083
  this.assistantOrdinal += 1;
1081
1084
  await this.options.onCanonicalMessage({
1082
1085
  kind: "commit-turn",
@@ -1090,6 +1093,9 @@ export class PreparedPiTurnAdapter {
1090
1093
  if (stalled) {
1091
1094
  throw new ChatStreamStalledError(message.errorMessage!);
1092
1095
  }
1096
+ if (resetModelSession) {
1097
+ throw new RetryableModelSessionError(message.errorMessage);
1098
+ }
1093
1099
  throw new RetryableModelError(message.errorMessage);
1094
1100
  }
1095
1101
  }
@@ -1136,9 +1142,6 @@ export class PreparedPiTurnAdapter {
1136
1142
  );
1137
1143
  this.terminalIntent = {
1138
1144
  outcome: terminal.outcome,
1139
- ...(isOpenRouterServerToolTimeout(authoritativeMessage)
1140
- ? { resetModelSession: true }
1141
- : {}),
1142
1145
  ...(authoritativeMessage.errorMessage
1143
1146
  ? { message: authoritativeMessage.errorMessage }
1144
1147
  : terminal.message
@@ -2,7 +2,11 @@ import {
2
2
  PreparedPiTurnAdapter,
3
3
  type CreatePreparedPiTurnOptions,
4
4
  } from "./execution";
5
- export { ContextOverflowError, RetryableModelError } from "./execution";
5
+ export {
6
+ ContextOverflowError,
7
+ RetryableModelError,
8
+ RetryableModelSessionError,
9
+ } from "./execution";
6
10
  import {
7
11
  uiUserMessageToPi,
8
12
  } from "../message";
@@ -26,6 +30,8 @@ import {
26
30
  } from "./assembly";
27
31
  import {
28
32
  decidePiRecovery,
33
+ recordPiToolMilestone,
34
+ type PiToolMilestoneInput,
29
35
  PiRuntimeRecoveryAdapter,
30
36
  type PiRecoveryHost,
31
37
  type PiRecoveryInput,
@@ -176,15 +182,20 @@ export class PiRuntimeAdapter {
176
182
  * 根据持久化 milestone 和一个命令算出恢复时要写什么、下一步做什么。
177
183
  *
178
184
  * @remarks
179
- * 调用方:Runtime 和 approval 生命周期在检查状态、记录 Tool 输入结果、处理审批或提交终态时调用。
185
+ * 调用方:Runtime 和 approval 生命周期在检查恢复状态、处理审批或提交终态时调用。
180
186
  *
181
187
  * 实现理由:该方法只返回 durable mutations 和一个 effect,实际写库与调度仍由 Runtime 完成。
182
- * replay、plan、stage、commit 和 encode 必须保持在同一决策入口,避免宿主与 Pi 各自推导出不同状态。
188
+ * 恢复规划保留在此入口;正常 Tool 写入复用同一命令 reducer,但不重复规划整个 Turn。
183
189
  */
184
190
  decideRecovery(options: PiRecoveryInput) {
185
191
  return decidePiRecovery(options);
186
192
  }
187
193
 
194
+ /** Active execution records only the indexed Tool fact; cold recovery still replays the full log. */
195
+ recordToolMilestone(options: PiToolMilestoneInput) {
196
+ return recordPiToolMilestone(options);
197
+ }
198
+
188
199
  /**
189
200
  * 把 Runtime 提供的持久化和重试回调包装成通用聊天恢复端口。
190
201
  *
@@ -215,6 +215,7 @@ export function isRecoverableAssistantError(
215
215
  message.stopReason === "error" &&
216
216
  /thought[_ ]signature/i.test(message.errorMessage ?? "")
217
217
  ) return false;
218
+ // Runtime resets the model Session and gives this timeout its own one-retry budget.
218
219
  if (isOpenRouterServerToolTimeout(message)) return false;
219
220
  return isRetryableAssistantError(message) ||
220
221
  (message.stopReason === "error" &&
@@ -369,15 +369,11 @@ function fromDispatch(
369
369
  // #region Recovery decision
370
370
 
371
371
  /**
372
- * 重放持久化里程碑,应用一条恢复命令,并算出下一批写入和后续动作。
373
- *
374
- * `PiRuntimeAdapter.decideRecovery` 在记录工具、审批、终态、续跑以及启动检查时调用它。
375
- *
376
- * 这里保持纯计算并只返回 mutations 和 effect;调用方必须先持久化 mutations,再执行 effect,不能把 I/O 搬进来破坏可重放性。
372
+ * Apply a durable command using the same validation during execution and recovery.
373
+ * Active Tool recording supplies only its indexed milestone; recovery supplies
374
+ * the full log and then plans the next effect. Neither path performs I/O.
377
375
  */
378
- export function decidePiRecovery(
379
- input: PiRecoveryInput,
380
- ): PiRecoveryDecision {
376
+ function applyPiRecoveryCommand(input: PiRecoveryInput) {
381
377
  let bodies = [...input.milestoneBodies];
382
378
  let state = replayPiToolRecovery(bodies);
383
379
  const mutations: PiDurableMutation[] = [];
@@ -677,6 +673,22 @@ export function decidePiRecovery(
677
673
  applied = true;
678
674
  }
679
675
 
676
+ return { state, bodies, mutations, applied };
677
+ }
678
+
679
+ export interface PiToolMilestoneInput extends PiRecoveryInput {
680
+ readonly command: Extract<PiRecoveryCommand, {
681
+ kind: "record-tool-input" | "record-tool-result";
682
+ }>;
683
+ }
684
+
685
+ /** Record a single Tool fact without planning a recovery of the active Turn. */
686
+ export function recordPiToolMilestone(input: PiToolMilestoneInput): readonly PiDurableMutation[] {
687
+ return applyPiRecoveryCommand(input).mutations;
688
+ }
689
+
690
+ export function decidePiRecovery(input: PiRecoveryInput): PiRecoveryDecision {
691
+ const { state, bodies, mutations, applied } = applyPiRecoveryCommand(input);
680
692
  const planned = fromPlan(
681
693
  state,
682
694
  planPiToolRecovery(state),
@@ -108,6 +108,10 @@ export interface SettledPiToolCall {
108
108
 
109
109
  const MAX_TOOL_RETRIES = 3;
110
110
  const TURN_FAILURE_CEILING = 8;
111
+ // DO SQLite rows also contain arguments and transcript/recovery envelopes.
112
+ // Keep each serialized result below half the 2 MiB row limit.
113
+ const DURABLE_TOOL_RESULT_MAX_BYTES = 1024 * 1024;
114
+ const MEDIA_CAPTION_MAX_CHARS = 4 * 1024;
111
115
 
112
116
  interface PiToolGovernanceState {
113
117
  readonly failures: Map<string, number>;
@@ -177,6 +181,15 @@ function artifactResult(
177
181
  result: AgentToolResult<unknown>,
178
182
  ref: ArtifactRef,
179
183
  ): AgentToolResult<unknown> {
184
+ let captionRemaining = MEDIA_CAPTION_MAX_CHARS;
185
+ const mediaContent = result.content.some((part) => part.type !== "text")
186
+ ? result.content.flatMap((part): AgentToolResult<unknown>["content"] => {
187
+ if (part.type !== "text") return [part];
188
+ const text = part.text.slice(0, captionRemaining);
189
+ captionRemaining -= text.length;
190
+ return text ? [{ ...part, text }] : [];
191
+ })
192
+ : [];
180
193
  return {
181
194
  ...result,
182
195
  content: [{
@@ -187,7 +200,7 @@ function artifactResult(
187
200
  `Bytes: ${ref.bytes}`,
188
201
  `Preview:\n${ref.preview}`,
189
202
  ].join("\n"),
190
- }],
203
+ }, ...mediaContent],
191
204
  details: ref,
192
205
  };
193
206
  }
@@ -197,13 +210,24 @@ async function durableResult(
197
210
  candidate: PiToolCandidate,
198
211
  workspace: SpillWorkspace | undefined,
199
212
  ): Promise<AgentToolResult<unknown>> {
200
- if ((candidate.outputBudget ?? { kind: "spill" }).kind === "spill") {
201
- const ref = await spillDurableToolOutput(spillPayload(result), {
202
- workspace,
203
- });
204
- if (ref) return artifactResult(result, ref);
205
- }
206
- return boundDurableToolOutput(result) as AgentToolResult<unknown>;
213
+ const spill = (candidate.outputBudget ?? { kind: "spill" }).kind === "spill";
214
+ let ref = spill
215
+ ? await spillDurableToolOutput(spillPayload(result), { workspace })
216
+ : null;
217
+ const bounded = ref
218
+ ? artifactResult(result, ref)
219
+ : boundDurableToolOutput(result) as AgentToolResult<unknown>;
220
+ const bytes = new TextEncoder().encode(JSON.stringify(bounded)).byteLength;
221
+ if (bytes <= DURABLE_TOOL_RESULT_MAX_BYTES) return bounded;
222
+
223
+ // Binary blocks cannot be truncated, and this boundary has no image resizer.
224
+ // Preserve the original where possible and fail before any SQLite write.
225
+ if (!spill) ref = await spillDurableToolOutput(spillPayload(result), { workspace });
226
+ throw new Error(
227
+ `Tool output exceeds the 1 MiB durable result limit (${bytes} bytes). ` +
228
+ "Return a smaller result, such as lower-resolution images or fewer media items; the original cannot be included inline. " +
229
+ (ref ? `Original output saved to Workspace: ${ref.path}` : "The original output could not be saved to Workspace."),
230
+ );
207
231
  }
208
232
 
209
233
  async function untilAborted<T>(
@@ -246,10 +246,17 @@ function runCodemode(
246
246
  }), CODEMODE_EXECUTE_TIMEOUT_MS);
247
247
  });
248
248
  try {
249
- return project(await Promise.race([
249
+ const output = await Promise.race([
250
250
  runtime.execute(input),
251
251
  deadline.then(result),
252
- ]));
252
+ ]);
253
+ const details = output.details as { status?: unknown; error?: unknown } | null;
254
+ if (details?.status === "error") {
255
+ throw new Error(typeof details.error === "string"
256
+ ? details.error
257
+ : `${label} failed`);
258
+ }
259
+ return project(output);
253
260
  } finally {
254
261
  if (timeout !== undefined) clearTimeout(timeout);
255
262
  }
@@ -257,12 +264,12 @@ function runCodemode(
257
264
  }
258
265
 
259
266
  /**
260
- * 把 Cloudflare Codemode Runtime handle 包装为 Pi 代码执行工具候选项。
267
+ * 把 Cloudflare Codemode execution port 包装为 Pi 代码执行工具候选项。
261
268
  *
262
269
  * Tool Surface 收到 Host 已组装的 Code Execution Port 后调用,
263
270
  * 模型再通过 `execute` 运行代码。
264
271
  *
265
- * Code Mode 作为一个完整的 safe 工具对外暴露,内部能力不再单独提权。
272
+ * Code Mode 的外层执行声明为 safe;内部候选仍经过各自的执行档位与审批门禁。
266
273
  */
267
274
  export function codeExecutionPiToolCandidate(
268
275
  runtime: RuntimeCodeExecutionPort,
@@ -273,13 +280,15 @@ export function codeExecutionPiToolCandidate(
273
280
  description: runtime.description,
274
281
  parameters: executeParameters,
275
282
  // Pi 工具循环在模型选择 `execute` 时调用,调用前允许 Turn 取消。
276
- // 必须通过 Runtime handle 而不是直接调用 executor,因为 Cloudflare Codemode 把重放、审批和执行日志放在持久化 Runtime 层。
283
+ // 上游保留 connector 执行日志,但此调用会创建新 execution,不能把它当作旧执行的恢复。
277
284
  execute: runCodemode(runtime, "Code Mode execute"),
278
285
  };
279
286
  return {
280
287
  owner: "core:codemode",
281
288
  requiredExecutionLevel: "safe",
282
- retry: "idempotent",
289
+ // Each upstream execute starts a new execution ID. A lost parent result
290
+ // cannot safely replay its already-applied connector or Workspace writes.
291
+ retry: "non-idempotent",
283
292
  outputBudget: { kind: "structure" },
284
293
  source: "codemode",
285
294
  summary: "Run JavaScript with network and configured connector access",
@@ -392,9 +392,10 @@ export function workspacePiToolCandidates(
392
392
  if (typeof path !== "string") {
393
393
  return removeTool.execute(toolCallId, params, signal, onUpdate);
394
394
  }
395
- return serializeWorkspaceMutation(workspace, path, () =>
396
- removeTool.execute(toolCallId, params, signal, onUpdate),
397
- );
395
+ return serializeWorkspaceMutation(workspace, path, () => {
396
+ signal?.throwIfAborted();
397
+ return removeTool.execute(toolCallId, params, signal, onUpdate);
398
+ });
398
399
  },
399
400
  };
400
401
 
package/src/runtime.ts CHANGED
@@ -71,6 +71,7 @@ import {
71
71
  MODEL_STREAM_STALL_TIMEOUT_MS,
72
72
  readModelStreamStallDetails,
73
73
  RetryableModelError,
74
+ RetryableModelSessionError,
74
75
  type PiCanonicalUserInput,
75
76
  type PiCanonicalTranscriptSnapshot,
76
77
  type PiChatRecoveryData,
@@ -145,6 +146,7 @@ const CONTINUATION_WATCHDOG_SECONDS = 300;
145
146
  // Host 侧的一次抖动会把 Session 永久卡在「队列已满」。
146
147
  const ADMISSION_ATTEMPT_WINDOW_MS = 60_000;
147
148
  export const CHAT_RECOVERY_MAX_ATTEMPTS = 5;
149
+ const SERVER_TOOL_TIMEOUT_MAX_ATTEMPTS = 2;
148
150
  // stall 单独收窄。理由不是「stall 更不值得救」,而是它的重试**期望值和别的错不一样**:
149
151
  // 瞬时错(5xx / 断流)重跑一次往往就好了;stall 的重跑是拿同一份 transcript 让模型
150
152
  // 重新想同样久,如果它本来就超预算,再跑几次也一样超。把看门狗放宽到 240s 之后,
@@ -177,10 +179,7 @@ interface StoredSubmission extends SubmissionReceipt {
177
179
  userMessageId?: string | null;
178
180
  regenerateMessageId?: string | null;
179
181
  recoveryErrorCount: number;
180
- recoveryReason?:
181
- | "no_meaningful_model_progress"
182
- | "transient_model_error"
183
- | "context_overflow";
182
+ recoveryReason?: SubmissionRecoveryReason;
184
183
  runId: string | null;
185
184
  accountId: string | null;
186
185
  rateVersion: number | null;
@@ -249,7 +248,7 @@ type RecoveredTurnPreparation =
249
248
  // 原因:终态文本会直接呈现给用户,必须说明发生了什么以及能不能重试。
250
249
  function unresumableWaitMessage(reason: string | undefined): string {
251
250
  return reason === "uncertain-tool"
252
- ? "SpringBrand stopped this turn: a non-idempotent Tool was interrupted and its result cannot be confirmed, so it was not retried. Re-send the request if you want to try again."
251
+ ? "SpringBrand stopped this turn: a non-idempotent Tool was interrupted and its result cannot be confirmed, so it was not retried. Check whether the external action already happened before sending another request."
253
252
  : "SpringBrand could not resume this turn after an interruption.";
254
253
  }
255
254
 
@@ -601,6 +600,7 @@ export abstract class AgentRuntimeKernel<
601
600
  pi: this.pi,
602
601
  findSubmission: (submissionId) =>
603
602
  this.readSubmission(submissionId),
603
+ isSubmissionActive: (submissionId) => this.submissions.isActive(submissionId),
604
604
  applyRecoveryMutations: (submission, mutations) =>
605
605
  this.applyPiRecoveryMutations(submission, mutations),
606
606
  materializeRecoveredToolResults: (submission) =>
@@ -893,7 +893,6 @@ export abstract class AgentRuntimeKernel<
893
893
  terminal: {
894
894
  outcome: "succeeded" | "failed" | "aborted";
895
895
  message?: string;
896
- resetModelSession?: boolean;
897
896
  },
898
897
  ) => void | Promise<void>;
899
898
  } = {},
@@ -2144,6 +2143,27 @@ export abstract class AgentRuntimeKernel<
2144
2143
  });
2145
2144
  }
2146
2145
 
2146
+ private recordPiToolMilestone(
2147
+ submission: StoredSubmission,
2148
+ command: Extract<PiRecoveryCommand, {
2149
+ kind: "record-tool-input" | "record-tool-result";
2150
+ }>,
2151
+ ): readonly PiDurableMutation[] {
2152
+ const key = command.kind === "record-tool-input"
2153
+ ? `tool-input:${command.record.toolCallId}`
2154
+ : `tool-result:${command.toolCallId}`;
2155
+ const body = this.db.milestones.findBody(submission.submissionId, key);
2156
+ return this.pi.recordToolMilestone({
2157
+ milestoneBodies: body === null ? [] : [body],
2158
+ identity: {
2159
+ turnId: submission.submissionId,
2160
+ assemblyRevision: submission.assemblyRevision,
2161
+ },
2162
+ command,
2163
+ now: Date.now(),
2164
+ });
2165
+ }
2166
+
2147
2167
  // 作用:把 Pi 恢复决策产生的变更写入 Runtime 数据库。
2148
2168
  // 调用:记录 Tool、终态、审批或恢复步骤时,通常在事务内调用。
2149
2169
  // 原因:upsert 保证重放幂等,而 terminal intent 允许后来的更强结果更新早期意图。
@@ -2207,15 +2227,12 @@ export abstract class AgentRuntimeKernel<
2207
2227
  submission: StoredSubmission,
2208
2228
  input: PiToolInputRecord,
2209
2229
  ): boolean {
2210
- const decision = this.decidePiRecovery(submission, {
2211
- kind: "record-tool-input",
2212
- record: input,
2213
- });
2214
2230
  const appended = this.db.transaction(() => {
2215
- const inserted = this.applyPiRecoveryMutations(
2216
- submission,
2217
- decision.mutations,
2218
- );
2231
+ const mutations = this.recordPiToolMilestone(submission, {
2232
+ kind: "record-tool-input",
2233
+ record: input,
2234
+ });
2235
+ const inserted = this.applyPiRecoveryMutations(submission, mutations);
2219
2236
  if (inserted) {
2220
2237
  this.telemetry.capture("toolStarted", {
2221
2238
  submissionId: submission.submissionId,
@@ -2361,15 +2378,18 @@ export abstract class AgentRuntimeKernel<
2361
2378
  occurredAt: createdAt,
2362
2379
  });
2363
2380
  }
2364
- const decision = this.decidePiRecovery(submission, {
2381
+ const mutations = this.recordPiToolMilestone(submission, {
2365
2382
  kind: "record-tool-result",
2366
2383
  toolCallId: call.toolCallId,
2367
2384
  toolName: call.toolName,
2368
2385
  result: call.result,
2369
2386
  isError: call.isError,
2370
2387
  timestamp: createdAt,
2388
+ // The current execution is already continuing. After a crash, recovery
2389
+ // materializes this result before resuming; no per-Tool dispatch is needed.
2390
+ needsContinuation: false,
2371
2391
  });
2372
- this.applyPiRecoveryMutations(submission, decision.mutations);
2392
+ this.applyPiRecoveryMutations(submission, mutations);
2373
2393
  this.approvals.commitContinuation(submission, call.toolCallId);
2374
2394
  }
2375
2395
 
@@ -3062,7 +3082,6 @@ export abstract class AgentRuntimeKernel<
3062
3082
  | {
3063
3083
  outcome: "succeeded" | "failed" | "aborted";
3064
3084
  message?: string;
3065
- resetModelSession?: boolean;
3066
3085
  }
3067
3086
  | undefined;
3068
3087
  const startedAt = ready.createdAt;
@@ -3082,13 +3101,6 @@ export abstract class AgentRuntimeKernel<
3082
3101
  intent.outcome,
3083
3102
  intent.message,
3084
3103
  );
3085
- if (intent.resetModelSession) {
3086
- this.modelSessionId = crypto.randomUUID();
3087
- await this.ctx.storage.put(
3088
- MODEL_SESSION_ID_STORAGE_KEY,
3089
- this.modelSessionId,
3090
- );
3091
- }
3092
3104
  },
3093
3105
  });
3094
3106
  for (const pending of this.db.steers.listForSubmission(submissionId)) {
@@ -3212,13 +3224,23 @@ export abstract class AgentRuntimeKernel<
3212
3224
  }
3213
3225
  const stalled = error instanceof ChatStreamStalledError;
3214
3226
  const contextOverflow = error instanceof ContextOverflowError;
3227
+ const resetModelSession = error instanceof RetryableModelSessionError;
3215
3228
  const abortReason = this.readSubmission(submissionId)?.abortReason;
3216
3229
  if (!abortReason && (stalled || error instanceof RetryableModelError)) {
3230
+ if (resetModelSession) {
3231
+ this.modelSessionId = crypto.randomUUID();
3232
+ await this.ctx.storage.put(
3233
+ MODEL_SESSION_ID_STORAGE_KEY,
3234
+ this.modelSessionId,
3235
+ );
3236
+ }
3217
3237
  const recoveryReason: SubmissionRecoveryReason = stalled
3218
3238
  ? "no_meaningful_model_progress"
3219
- : contextOverflow
3220
- ? "context_overflow"
3221
- : "transient_model_error";
3239
+ : resetModelSession
3240
+ ? "server_tool_timeout"
3241
+ : contextOverflow
3242
+ ? "context_overflow"
3243
+ : "transient_model_error";
3222
3244
  let contextOverflowRecoveryClaimed = false;
3223
3245
  const recoveryErrorCount = this.db.transaction(() => {
3224
3246
  if (contextOverflow) {
@@ -3253,7 +3275,11 @@ export abstract class AgentRuntimeKernel<
3253
3275
  ? contextOverflowRecoveryClaimed &&
3254
3276
  recoveryErrorCount < CHAT_RECOVERY_MAX_ATTEMPTS
3255
3277
  : recoveryErrorCount <
3256
- (stalled ? CHAT_STALL_MAX_ATTEMPTS : CHAT_RECOVERY_MAX_ATTEMPTS);
3278
+ (stalled
3279
+ ? CHAT_STALL_MAX_ATTEMPTS
3280
+ : resetModelSession
3281
+ ? SERVER_TOOL_TIMEOUT_MAX_ATTEMPTS
3282
+ : CHAT_RECOVERY_MAX_ATTEMPTS);
3257
3283
  if (
3258
3284
  !stoppedDuringRecovery &&
3259
3285
  retryAllowed
@@ -4710,7 +4736,9 @@ export abstract class AgentRuntimeKernel<
4710
4736
  : current.recoveryReason ===
4711
4737
  "no_meaningful_model_progress"
4712
4738
  ? CHAT_STALL_MAX_ATTEMPTS
4713
- : CHAT_RECOVERY_MAX_ATTEMPTS,
4739
+ : current.recoveryReason === "server_tool_timeout"
4740
+ ? SERVER_TOOL_TIMEOUT_MAX_ATTEMPTS
4741
+ : CHAT_RECOVERY_MAX_ATTEMPTS,
4714
4742
  ...(current.recoveryReason &&
4715
4743
  current.recoveryReason !== "context_overflow"
4716
4744
  ? { recoveryReason: current.recoveryReason }