@springbrand/agent-runtime 0.1.3-alpha.2 → 0.1.3-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/package.json +1 -1
  2. package/src/db/agent-tool.repo.ts +27 -0
  3. package/src/db/index.ts +33 -0
  4. package/src/db/interaction.repo.ts +185 -0
  5. package/src/db/schema.ts +15 -0
  6. package/src/db/submission.repo.ts +29 -0
  7. package/src/index.ts +8 -17
  8. package/src/kernel/approval-lifecycle.ts +41 -6
  9. package/src/kernel/bindings.ts +37 -0
  10. package/src/kernel/interaction-lifecycle.ts +395 -0
  11. package/src/kernel/public-contracts.ts +2 -0
  12. package/src/kernel/recoverable-chat-agent.ts +10 -2
  13. package/src/kernel/runtime-assembly-view.ts +37 -0
  14. package/src/kernel/runtime-assembly.ts +41 -0
  15. package/src/kernel/runtime-config.ts +4 -0
  16. package/src/kernel/runtime-load.ts +102 -0
  17. package/src/kernel/state.ts +8 -1
  18. package/src/kernel/submission-lifecycle.ts +30 -0
  19. package/src/lib/telemetry-dev.ts +7 -4
  20. package/src/pi/runtime-adapter/assembly.ts +13 -1
  21. package/src/pi/runtime-adapter/execution.ts +109 -9
  22. package/src/pi/runtime-adapter/index.ts +10 -3
  23. package/src/pi/runtime-adapter/models.ts +162 -16
  24. package/src/pi/runtime-adapter/recovery.ts +188 -1
  25. package/src/pi/tool/base.ts +62 -9
  26. package/src/pi/tool/compiler.ts +34 -0
  27. package/src/pi/tool/gateway.ts +54 -0
  28. package/src/pi/tool/index.ts +1 -0
  29. package/src/pi/tool/mcp.ts +93 -64
  30. package/src/pi/turn/index.ts +20 -0
  31. package/src/pi/turn/interaction.ts +181 -0
  32. package/src/pi/turn/tool-recovery.ts +244 -1
  33. package/src/runtime-agent.ts +246 -113
  34. package/src/{plugins.ts → runtime-assembler.ts} +65 -282
  35. package/src/runtime.ts +454 -162
@@ -1,9 +1,12 @@
1
1
  /**
2
- * dev-only 观测降级:把 observability 事件打到 console,让 `wrangler dev` /
3
- * `wrangler tail` 里能实时看到埋点(否则 publish 到零订阅 channel = 静默 no-op)。
2
+ * 观测降级:把 observability 事件打到 console,让 `wrangler dev` / `wrangler tail` 里能实时看到埋点,
3
+ * 并且(生产上开了 `observability.logs` 时)进 Workers Logs 供事后查询。
4
+ * 不装它,事件就 publish 到零订阅 channel = 静默 no-op。
4
5
  *
5
- * 仅在 env `TELEMETRY_CONSOLE==="1"` 时由 onStart 装载(`.dev.vars` 里开,prod 不设 自动关)。
6
- * 生产不靠它:prod 所有 channel 事件自动转发 Tail Worker
6
+ * onStart 在 env `TELEMETRY_CONSOLE==="1"` 时装载 —— `.dev.vars` `wrangler.jsonc` vars 里都开着。
7
+ * 注意:上游注释宣称"生产上所有 channel 事件自动转发 Tail Worker",那条路径需要 `tail_consumers`,
8
+ * 本仓没有配,所以生产**不能**指望它;这个 console sink 就是生产的唯一消费面(2026-08-06 实测:
9
+ * 未开启前 Observability API 查 universal-agent 24h 事件 count=0)。
7
10
  *
8
11
  * 用 `agents/observability` 的类型化 `subscribe`(按 channel key 订阅),避免引 `node:diagnostics_channel`
9
12
  * 与 `@types/node`(tsconfig 只带 workers-types)。
@@ -1,5 +1,6 @@
1
1
  import type {
2
2
  RuntimeBindings,
3
+ RuntimeGatewaySession,
3
4
  RuntimePlatformPort,
4
5
  } from "../../kernel/bindings";
5
6
  import type { RuntimeDegradation } from "../../kernel/degradation";
@@ -8,7 +9,7 @@ import type {
8
9
  } from "../../kernel/extensions";
9
10
  import type { RuntimeProfile } from "../../kernel/profile";
10
11
  import type { ExecutionLevel } from "../../lib/execution-level";
11
- import type { RuntimeSnapshot } from "../../plugins";
12
+ import type { RuntimeSnapshot } from "../../runtime-assembler";
12
13
  import type { PiRuntimeAssembly } from "../assembly";
13
14
  import {
14
15
  assemblePiSystemContext,
@@ -19,6 +20,7 @@ import {
19
20
  type PiLoadedExtension,
20
21
  } from "../assembly";
21
22
  import {
23
+ createPiGatewayToolCandidates,
22
24
  createPiMcpToolCandidates,
23
25
  listExtensionsPiToolCandidate,
24
26
  type PiMcpHost,
@@ -58,6 +60,7 @@ interface PiAssemblySnapshot {
58
60
  interface PreparePiAssemblyOptions {
59
61
  readonly snapshot: PiAssemblySnapshot;
60
62
  readonly mcpHost: PiMcpHost;
63
+ readonly gatewaySession?: RuntimeGatewaySession;
61
64
  // 为一个扩展生成带权限参数的 Host Fetcher。
62
65
  // loadPiExtension 在扩展需要 Host 能力时调用它。
63
66
  // Worker 掌握真实绑定,装配层只保存这个窄函数,不能自行扩大权限。
@@ -90,6 +93,8 @@ interface PreparedPiAssembly {
90
93
  export interface PreparePiRuntimeOptions {
91
94
  readonly snapshot: RuntimeSnapshot;
92
95
  readonly mcpHost: PiMcpHost;
96
+ readonly gatewaySession?: RuntimeGatewaySession;
97
+ readonly additionalDegradations?: readonly RuntimeDegradation[];
93
98
  /**
94
99
  * 为一个扩展创建受限的 Host 绑定。
95
100
  *
@@ -115,6 +120,7 @@ export interface PreparePiRuntimeOptions {
115
120
  export interface PreparedPiRuntime {
116
121
  readonly revisionDescriptor: string;
117
122
  readonly degradations: readonly RuntimeDegradation[];
123
+ readonly extensions: readonly { readonly name: string; readonly version: string }[];
118
124
  }
119
125
 
120
126
  /**
@@ -301,6 +307,7 @@ class PreparedRuntime implements PreparedPiRuntime {
301
307
  constructor(
302
308
  readonly revisionDescriptor: string,
303
309
  readonly degradations: readonly RuntimeDegradation[],
310
+ readonly extensions: readonly { readonly name: string; readonly version: string }[],
304
311
  private readonly owner: object,
305
312
  private readonly state: PreparedPiRuntimeState,
306
313
  ) {}
@@ -414,6 +421,9 @@ async function preparePiAssembly(
414
421
  candidates: snapshot.pi.toolSurface.finalize([
415
422
  ...extensions.candidates,
416
423
  listExtensionsPiToolCandidate(extensions.loaded),
424
+ ...(options.gatewaySession
425
+ ? createPiGatewayToolCandidates(options.gatewaySession)
426
+ : []),
417
427
  ...createPiMcpToolCandidates(
418
428
  options.mcpHost,
419
429
  snapshot.profile.mcpServers,
@@ -444,8 +454,10 @@ export async function preparePiRuntime(
444
454
  describeRuntime(options.snapshot, candidates, assembly.loaded),
445
455
  Object.freeze([
446
456
  ...options.snapshot.degradations,
457
+ ...(options.additionalDegradations ?? []),
447
458
  ...assembly.degradations,
448
459
  ]),
460
+ Object.freeze(assembly.loaded.map(({ name, version }) => ({ name, version }))),
449
461
  owner,
450
462
  Object.freeze({
451
463
  snapshot: options.snapshot,
@@ -16,7 +16,11 @@ import type {
16
16
  UserMessage,
17
17
  } from "@earendil-works/pi-ai";
18
18
  import { transformMessages } from "@earendil-works/pi-ai/api/transform-messages";
19
- import { requiresPiToolApproval, parkPiToolApproval } from "../turn";
19
+ import {
20
+ parkPiToolApproval,
21
+ parkPiToolInteraction,
22
+ requiresPiToolApproval,
23
+ } from "../turn";
20
24
  import { PiChunkEncoder } from "../message";
21
25
  import type { UIMessageChunk } from "ai";
22
26
  import { ChatStreamStalledError } from "agents/chat";
@@ -25,6 +29,7 @@ import {
25
29
  createPiToolGovernance,
26
30
  normalizeUpdatePlanArguments,
27
31
  type PiToolCandidate,
32
+ type PiToolInteractionSpec,
28
33
  type PiToolGovernance,
29
34
  type PiToolTelemetry,
30
35
  type SettledPiToolCall,
@@ -40,6 +45,7 @@ import {
40
45
  resolvePiApiKey,
41
46
  withProviderRetry,
42
47
  } from "./models";
48
+ import { EXECUTION_LEVELS } from "../../lib/execution-level";
43
49
 
44
50
  // #region Single-run Pi bridge
45
51
 
@@ -308,6 +314,18 @@ export interface PiTurnDurability {
308
314
  signal?: AbortSignal,
309
315
  onCreated?: () => void | Promise<void>,
310
316
  ): Promise<void>;
317
+ /**
318
+ * 保存一次客户端结算请求并等待响应。
319
+ *
320
+ * @remarks
321
+ * Tool candidate 声明了 `interaction` 时,PreparedPiTurnAdapter 在执行前调用它,`execute` 不会被调用。
322
+ * 返回后必须再读一次 settlement —— 响应、取消和跨休眠恢复三条路径都只把结果留在 settlement 里。
323
+ */
324
+ requestToolInteraction(
325
+ interaction: import("../turn").PiToolInteraction,
326
+ settle: import("../tool").PiToolInteractionSpec,
327
+ signal?: AbortSignal,
328
+ ): Promise<void>;
311
329
  /**
312
330
  * 记录工具输入,并返回它是否是第一次持久化尝试。
313
331
  *
@@ -475,13 +493,21 @@ export class PreparedPiTurnAdapter {
475
493
  ...candidate.tool,
476
494
  // 只在 Runtime 的持久化和策略门禁都放行后执行一个 Tool candidate。
477
495
  // 实时运行由 PiCore 调用它,恢复或审批续跑则由 retryTool 进入同一路径。
478
- // 顺序不能随便调整:已有结果优先,审批可能生成结果,不确定的非幂等工作不能重放。
496
+ // 顺序不能随便调整:门禁先于结果复用,审批可能生成结果,不确定的非幂等工作不能重放。
479
497
  execute: async (toolCallId, input, signal, onUpdate) => {
498
+ const requiredExecutionLevel = candidate.requiredExecutionLevelForInput
499
+ ? await candidate.requiredExecutionLevelForInput(input)
500
+ : candidate.requiredExecutionLevel;
501
+ if (!EXECUTION_LEVELS.includes(requiredExecutionLevel)) {
502
+ throw new Error(
503
+ `Pi Tool "${candidate.tool.name}" resolved an invalid execution level`,
504
+ );
505
+ }
480
506
  await state.snapshot.bindings.platform.gateTool?.({
481
507
  toolCallId,
482
508
  toolName: candidate.tool.name,
483
509
  input,
484
- requiredExecutionLevel: candidate.requiredExecutionLevel,
510
+ requiredExecutionLevel,
485
511
  signal: signal ?? new AbortController().signal,
486
512
  });
487
513
  const settled = options.durability.findToolSettlement(toolCallId);
@@ -494,10 +520,13 @@ export class PreparedPiTurnAdapter {
494
520
  }
495
521
  return settled.result;
496
522
  }
497
- if (requiresPiToolApproval({
498
- executionLevel: descriptor.executionLevel,
499
- requiredExecutionLevel: candidate.requiredExecutionLevel,
500
- })) {
523
+ if (
524
+ candidate.alwaysRequiresApproval ||
525
+ requiresPiToolApproval({
526
+ executionLevel: descriptor.executionLevel,
527
+ requiredExecutionLevel,
528
+ })
529
+ ) {
501
530
  const approval = parkPiToolApproval({
502
531
  executionId: `${options.submission.id}:${toolCallId}`,
503
532
  requestId: options.submission.requestId,
@@ -509,7 +538,7 @@ export class PreparedPiTurnAdapter {
509
538
  candidate.tool.label ??
510
539
  candidate.tool.name,
511
540
  executionLevel: descriptor.executionLevel,
512
- requiredExecutionLevel: candidate.requiredExecutionLevel,
541
+ requiredExecutionLevel,
513
542
  inputJson: JSON.stringify(input),
514
543
  createdAt: Date.now(),
515
544
  }).approval;
@@ -539,6 +568,43 @@ export class PreparedPiTurnAdapter {
539
568
  const afterApproval = options.durability
540
569
  .findToolSettlement(toolCallId);
541
570
  if (afterApproval) return afterApproval.result;
571
+ // 结果由客户端提供的 Tool 在这里 park,永远不走到下面的 execute。
572
+ // 放在审批之后:一个 Tool 既要审批又要客户端结算时,先过许可再问用户,
573
+ // 否则会先向用户要答案、再告诉他这事根本不许做。
574
+ if (candidate.interaction) {
575
+ // 必须先把 Tool 输入落成里程碑再 park。恢复出来的结算靠这条记录取回
576
+ // `args`;缺了它,Tool 醒来返回结果时 settleTool 会拿 undefined 的 args
577
+ // 去和真 args 比对,判成 "Conflicting durable Tool settlement" —— 整个
578
+ // Turn 就死在这。(审批那条路不受影响:批准后 Tool 真的执行,args 在
579
+ // 下面的 appendToolInput 里补上了。)
580
+ options.durability.appendToolInput({
581
+ toolCallId,
582
+ toolName: candidate.tool.name,
583
+ input,
584
+ retry: piToolRetryPolicy(candidate),
585
+ });
586
+ const parked = parkPiToolInteraction({
587
+ interactionId: `${options.submission.id}:${toolCallId}`,
588
+ requestId: options.submission.requestId,
589
+ toolCallId,
590
+ toolName: candidate.tool.name,
591
+ inputJson: JSON.stringify(input),
592
+ createdAt: Date.now(),
593
+ }).interaction;
594
+ await options.durability.requestToolInteraction(
595
+ parked,
596
+ candidate.interaction,
597
+ signal,
598
+ );
599
+ const responded = options.durability
600
+ .findToolSettlement(toolCallId);
601
+ if (!responded) {
602
+ throw new Error(
603
+ `Tool interaction produced no durable settlement: ${candidate.tool.name}`,
604
+ );
605
+ }
606
+ return responded.result;
607
+ }
542
608
  const retry = piToolRetryPolicy(candidate);
543
609
  const firstAttempt = options.durability.appendToolInput({
544
610
  toolCallId,
@@ -572,7 +638,11 @@ export class PreparedPiTurnAdapter {
572
638
  ),
573
639
  modelSessionId: options.submission.requestId,
574
640
  canonicalMessages: options.canonicalMessages,
575
- settle: options.durability.settleTool,
641
+ // 被中断的 Turn 不写结算。它退场路上还会吐出「工具被中止」,而工具其实
642
+ // 没有产生任何结果 —— 结算是单调的,落了这条假结果,重建出来的续跑会
643
+ // 把一个从没跑过的工具当成跑过并失败。取消不走这里:取消要的就是终态。
644
+ settle: (call) =>
645
+ this.interrupted ? undefined : options.durability.settleTool(call),
576
646
  onToolTelemetry: options.onToolTelemetry,
577
647
  transformContext: options.transformContext,
578
648
  });
@@ -582,6 +652,7 @@ export class PreparedPiTurnAdapter {
582
652
 
583
653
  private readonly systemPrompt: string;
584
654
  private readonly options: CreatePreparedPiTurnOptions;
655
+ private interrupted = false;
585
656
 
586
657
  /**
587
658
  * 请求取消这个 Turn。
@@ -595,6 +666,20 @@ export class PreparedPiTurnAdapter {
595
666
  this.turn.abort();
596
667
  }
597
668
 
669
+ /**
670
+ * 中断这个 Turn,把它交给持久续跑接手。
671
+ *
672
+ * @remarks
673
+ * 宿主在 park 期间换掉装配后调用。与 `abort` 的差别只有一处、但很要命:
674
+ * 中断途中冒出来的工具结算一律丢弃。这条执行器的系统提示和工具表锁在它被
675
+ * 创建的那一刻,接下来跑的是另一条按新装配重建的执行器,两边共用同一批
676
+ * `toolCallId` —— 让退场路上的假结果先落盘,新那条就再也纠正不过来了。
677
+ */
678
+ interrupt(): void {
679
+ this.interrupted = true;
680
+ this.abort();
681
+ }
682
+
598
683
  /**
599
684
  * 把一条用户消息加入当前 Turn。
600
685
  *
@@ -635,6 +720,21 @@ export class PreparedPiTurnAdapter {
635
720
  : false;
636
721
  }
637
722
 
723
+ /**
724
+ * 取回某个 Tool 的客户端结算声明,没有声明时返回 null。
725
+ *
726
+ * @remarks
727
+ * `Runtime.respondToolInteraction` 在把客户端响应映射成 ToolResult 前调用它。
728
+ *
729
+ * 声明里含函数,无法持久化;从 Pinned Runtime 的 candidate 现取,
730
+ * 可让「DO 一直醒着」和「park 期间睡过一觉」走同一条解析路径,而不是一条真解析、一条退化成缺省。
731
+ */
732
+ interactionSpec(toolName: string): PiToolInteractionSpec | null {
733
+ return this.candidates.find(
734
+ (item) => item.tool.name === toolName,
735
+ )?.interaction ?? null;
736
+ }
737
+
638
738
  /**
639
739
  * 运行这个 Turn,直到 PiCore 和所有等待中的事件回调结束。
640
740
  *
@@ -8,9 +8,10 @@ import {
8
8
  } from "../message";
9
9
  import type { UIMessage } from "ai";
10
10
  import { createModels, type MutableModels } from "@earendil-works/pi-ai";
11
- import { configurePiModels, resolvePiApiKey } from "./models";
11
+ import { configurePiModels, createPiModels, resolvePiApiKey } from "./models";
12
12
  export {
13
13
  MODEL_STREAM_STALL_TIMEOUT_MS,
14
+ modelRequestUrl,
14
15
  readModelStreamStallDetails,
15
16
  withProviderRetry,
16
17
  } from "./models";
@@ -128,6 +129,9 @@ export class PiRuntimeAdapter {
128
129
  prepare(
129
130
  options: PreparePiRuntimeOptions,
130
131
  ): Promise<PreparedPiRuntime> {
132
+ if (this.managesModels) {
133
+ createPiModels(options.snapshot.bindings.provider);
134
+ }
131
135
  return preparePiRuntime(options, this.owner);
132
136
  }
133
137
 
@@ -137,8 +141,7 @@ export class PiRuntimeAdapter {
137
141
  * @remarks
138
142
  * 调用方:Runtime 的 `initConfig()` 在 Prepared Runtime 和 revision 检查成功后、发布新快照前调用。
139
143
  *
140
- * 实现理由:激活会校验并注册所有 endpoint 中声明的模型,而不只校验当前选中的模型。
141
- * 这解释了为什么一个未被选中的无效模型也会让配置启动失败;不要把激活误当成首次推理请求。
144
+ * 实现理由:`prepare()` 已校验所有 endpoint 模型,这里只把同一份部署目录切换到长期 Models 实例。
142
145
  * 注入模型目录时跳过此步骤,以保留调用方对目录的配置权。
143
146
  */
144
147
  activate(snapshot: PreparePiRuntimeOptions["snapshot"]): void {
@@ -209,6 +212,10 @@ export type {
209
212
  UIChatRequestBody,
210
213
  } from "../message";
211
214
  export type { PiToolApproval } from "../turn";
215
+ export type {
216
+ PiToolInteraction,
217
+ PiToolInteractionCancelReason,
218
+ } from "../turn";
212
219
  export type {
213
220
  CreatePreparedPiTurnOptions,
214
221
  PiCanonicalUserInput,
@@ -43,6 +43,10 @@ import {
43
43
  import {
44
44
  ChatStreamStalledError,
45
45
  } from "agents/chat";
46
+ import {
47
+ genericObservability,
48
+ type ObservabilityEvent,
49
+ } from "agents/observability";
46
50
  import type {
47
51
  RuntimeModelEndpoint,
48
52
  RuntimeModelProtocol,
@@ -58,7 +62,19 @@ const CATALOGS = {
58
62
  } satisfies Record<RuntimeModelProtocol, readonly Model<Api>[]>;
59
63
 
60
64
  const PROVIDER_MAX_RETRIES = 2;
61
- export const MODEL_STREAM_STALL_TIMEOUT_MS = 60_000;
65
+ // 看门狗要抓的是「连接死了」,不是「模型想得慢」——这两件事在流上分不开:
66
+ // 推理模型在中转后面是「闷头想完再吐」,静默期一个字节都没有,也没有 keepalive。
67
+ //
68
+ // 2026-08-06 实测(gpt-5.6-sol @ api.sharkmelon.tech,一道需要真推理的题):
69
+ // 响应头 3.86s → 单段静默 **34.4s** → 1190 个 chunk 在 12s 内吐完,首个可见内容 38.3s。
70
+ // 而真实回合(长 transcript + 工具)比这道题重得多。原值 60s 卡在这条曲线的正中间:
71
+ // 简单回合(3~9s)不触发,一旦模型真开始想就必然超时 → abort → 从 transcript 整轮重跑
72
+ // → 同一个提示词又想同样久 → 再超时。**重试的对象正是那个「本来就要更久」的东西,
73
+ // 结构上不可能收敛**,表现为前端 think 转到恢复预算耗尽为止。
74
+ //
75
+ // 调到 240s:比实测静默期留约 7 倍余量。代价是连接真死时单次要等更久,
76
+ // 所以 runtime.ts 同时把 stall 的恢复次数单独收窄(见 CHAT_STALL_MAX_ATTEMPTS)。
77
+ export const MODEL_STREAM_STALL_TIMEOUT_MS = 240_000;
62
78
  export const MODEL_STREAM_STALL_MESSAGE =
63
79
  `Chat stream stalled: no activity for ${MODEL_STREAM_STALL_TIMEOUT_MS}ms; the turn was aborted by the stall watchdog.`;
64
80
  const MODEL_STREAM_STALL_DETAILS_PREFIX = `${MODEL_STREAM_STALL_MESSAGE}\n`;
@@ -180,12 +196,17 @@ function meaningfulModelProgress(
180
196
  async function* stopStalledModelStream(
181
197
  source: AsyncIterable<AssistantMessageEvent>,
182
198
  watchdog: AbortController,
199
+ probe: (phase: string, details?: Record<string, unknown>) => void,
183
200
  ): AsyncGenerator<AssistantMessageEvent> {
184
201
  const iterator = source[Symbol.asyncIterator]();
185
202
  let timer: ReturnType<typeof setTimeout> | undefined;
186
203
  let stalled = false;
187
204
  let stallError: ChatStreamStalledError | undefined;
188
205
  let idleWaitMs = 0;
206
+ let rawEventCount = 0;
207
+ let meaningfulEventCount = 0;
208
+ let lastRawEventAt: number | undefined;
209
+ let lastRawEventType: AssistantMessageEvent["type"] | undefined;
189
210
  const streamedContent = new Set<string>();
190
211
  let lastMeaningfulActivityAt = Date.now();
191
212
  let lastMeaningfulActivityType:
@@ -193,6 +214,17 @@ async function* stopStalledModelStream(
193
214
  "model_stream_started";
194
215
  const stop = (idleMs: number) => {
195
216
  stalled = true;
217
+ const now = Date.now();
218
+ probe("stall", {
219
+ idleMs,
220
+ rawEventCount,
221
+ meaningfulEventCount,
222
+ lastRawEventType: lastRawEventType ?? "none",
223
+ sinceLastRawEventMs: lastRawEventAt === undefined
224
+ ? null
225
+ : now - lastRawEventAt,
226
+ lastMeaningfulActivityType,
227
+ });
196
228
  stallError = new ChatStreamStalledError(
197
229
  MODEL_STREAM_STALL_DETAILS_PREFIX + JSON.stringify({
198
230
  lastMeaningfulActivityAt,
@@ -227,6 +259,11 @@ async function* stopStalledModelStream(
227
259
  ]);
228
260
  } catch (error) {
229
261
  if (stalled) throw stallError;
262
+ probe("iterator_error", {
263
+ errorName: error instanceof Error ? error.name : typeof error,
264
+ rawEventCount,
265
+ meaningfulEventCount,
266
+ });
230
267
  throw error;
231
268
  } finally {
232
269
  clearTimeout(timer);
@@ -235,13 +272,31 @@ async function* stopStalledModelStream(
235
272
  if (stalled) throw stallError;
236
273
  if (next.done) break;
237
274
  const event = next.value;
275
+ rawEventCount += 1;
276
+ lastRawEventAt = Date.now();
277
+ lastRawEventType = event.type;
278
+ if (rawEventCount === 1) {
279
+ probe("first_raw_event", { eventType: event.type });
280
+ }
238
281
  if (event.type === "done" || event.type === "error") {
282
+ probe(event.type, {
283
+ reason: event.reason,
284
+ stopReason: event.type === "done"
285
+ ? event.message.stopReason
286
+ : event.error.stopReason,
287
+ rawEventCount,
288
+ meaningfulEventCount,
289
+ });
239
290
  yield event;
240
291
  return;
241
292
  }
242
293
  idleWaitMs += Date.now() - waitStartedAt;
243
294
  const progress = meaningfulModelProgress(event, streamedContent);
244
295
  if (progress) {
296
+ meaningfulEventCount += 1;
297
+ if (meaningfulEventCount === 1) {
298
+ probe("first_meaningful_event", { eventType: progress });
299
+ }
245
300
  lastMeaningfulActivityAt = Date.now();
246
301
  lastMeaningfulActivityType = progress;
247
302
  idleWaitMs = 0;
@@ -252,16 +307,65 @@ async function* stopStalledModelStream(
252
307
  clearTimeout(timer);
253
308
  if (!stalled) await iterator.return?.().catch(() => {});
254
309
  }
310
+ probe("ended_without_terminal", {
311
+ rawEventCount,
312
+ meaningfulEventCount,
313
+ lastRawEventType: lastRawEventType ?? "none",
314
+ });
255
315
  throw new Error("Model stream ended without a terminal event");
256
316
  }
257
317
 
318
+ function trimTrailingSlash(value: string): string {
319
+ return value.endsWith("/") ? value.slice(0, -1) : value;
320
+ }
321
+
322
+ /**
323
+ * 算出一个已解析模型真正会被请求的 URL。
324
+ *
325
+ * {@link withProviderRetry} 在每次派发模型请求前调用它写观测日志;诊断"这个 Agent 到底调了哪个 LLM"时也可以直接复用。
326
+ *
327
+ * 各协议的路径由底层 SDK 决定,这里必须与之逐条对齐:OpenAI 兼容 SDK 用 `baseURL + /chat/completions`,
328
+ * Anthropic SDK 用 `baseURL + /v1/messages`(纯字符串拼接,不会去重 `/v1`),Google 的 baseUrl 已含版本段,
329
+ * Codex 走 `resolveCodexUrl`。不要在这里"顺手规范化"路径,否则日志会与真实请求脱节,反而掩盖配置错误。
330
+ */
331
+ export function modelRequestUrl(model: Model<Api>): string {
332
+ const base = trimTrailingSlash(model.baseUrl ?? "");
333
+ switch (model.api) {
334
+ case "openai-completions":
335
+ return `${base}/chat/completions`;
336
+ case "anthropic-messages":
337
+ return `${base}/v1/messages`;
338
+ case "google-generative-ai":
339
+ return `${base}/models/${model.id}:streamGenerateContent`;
340
+ case "openai-codex-responses":
341
+ return base.endsWith("/codex/responses") || base.endsWith("/responses")
342
+ ? base
343
+ : `${base}/codex/responses`;
344
+ default:
345
+ return base;
346
+ }
347
+ }
348
+
258
349
  /**
259
350
  * 给模型请求补上可中断的空闲终止边界。
260
351
  *
261
352
  * Runtime Turn 和 SubAgent 在把 `Models.streamSimple` 交给 Pi 前调用;显式传入的重试次数优先。
262
353
  *
263
354
  * 调用方可以覆盖默认的 2 次 Provider 重试;主 Turn 传 0,由 Submission
264
- * 统一持有恢复预算。连续 60 秒没有可展示进展时中止 provider。
355
+ * 统一持有恢复预算。连续 {@link MODEL_STREAM_STALL_TIMEOUT_MS} 毫秒没有可展示进展时中止 provider。
356
+ *
357
+ * 每个相位发一条 `ua:model` 观测事件(dispatch / response_headers / first_raw_event /
358
+ * first_meaningful_event / stall / done…)。**只发事件、不直接 console.log**:
359
+ * 这样它和 `ua:tool`、`chat:*` 共用同一个消费面,被 `TELEMETRY_CONSOLE` 一个开关统一管,
360
+ * 关掉即零订阅 no-op;将来若接上 `tail_consumers`,这条也自动跟着走。
361
+ *
362
+ * 每条都带 `url`(而不是只在 dispatch 带一次):模型路由只存在于 secret 里,
363
+ * 线上排查时最需要回答的就是"这次打到哪个 URL",让每行自解释比省几十字节值。
364
+ * payload 只含路由与时序,不含 key 和消息内容。
365
+ *
366
+ * 注意这里用的是模块级 `genericObservability`,不是 Agent 实例的 `_emit` ——
367
+ * 纯模块拿不到实例,代价是事件不带 `agent` / `name` 字段;turn 的身份由
368
+ * payload 里的 `requestId` / `sessionId` 承担。
265
369
  */
266
370
  export function withProviderRetry(
267
371
  streamFn: StreamFn,
@@ -270,21 +374,63 @@ export function withProviderRetry(
270
374
  ): StreamFn {
271
375
  return (model, context, options) =>
272
376
  lazyStream(model, async () => {
377
+ const requestId = crypto.randomUUID();
378
+ const startedAt = Date.now();
379
+ const sessionId = options?.sessionId ?? defaultSessionId;
380
+ const url = modelRequestUrl(model);
381
+ const probe = (phase: string, details: Record<string, unknown> = {}) =>
382
+ // `ua:*` 是本仓自有的事件命名,不在上游的 ObservabilityEvent 联合里,
383
+ // 故整体断言一次(runtime.ts 的 `ua:tool` 是同一处上游类型缺口)。
384
+ // 不能只把 type 断言成 never——那会把联合窄成 never,连 payload 一起报错。
385
+ genericObservability.emit({
386
+ type: "ua:model",
387
+ timestamp: Date.now(),
388
+ payload: {
389
+ requestId,
390
+ sessionId,
391
+ phase,
392
+ elapsedMs: Date.now() - startedAt,
393
+ url,
394
+ api: model.api,
395
+ provider: model.provider,
396
+ model: model.id,
397
+ ...details,
398
+ },
399
+ } as unknown as ObservabilityEvent);
400
+ probe("dispatch");
273
401
  const watchdog = new AbortController();
274
- const source = await streamFn(model, context, {
275
- ...options,
276
- signal: options?.signal
277
- ? AbortSignal.any([options.signal, watchdog.signal])
278
- : watchdog.signal,
279
- maxRetries: options?.maxRetries ?? defaultMaxRetries,
280
- sessionId: options?.sessionId ?? defaultSessionId,
281
- onPayload: async (payload, activeModel) =>
282
- pdfPayload(
283
- await options?.onPayload?.(payload, activeModel) ?? payload,
284
- activeModel.api,
285
- ),
286
- });
287
- return stopStalledModelStream(source, watchdog);
402
+ let responseCount = 0;
403
+ try {
404
+ const source = await streamFn(model, context, {
405
+ ...options,
406
+ signal: options?.signal
407
+ ? AbortSignal.any([options.signal, watchdog.signal])
408
+ : watchdog.signal,
409
+ maxRetries: options?.maxRetries ?? defaultMaxRetries,
410
+ sessionId,
411
+ onPayload: async (payload, activeModel) =>
412
+ pdfPayload(
413
+ await options?.onPayload?.(payload, activeModel) ?? payload,
414
+ activeModel.api,
415
+ ),
416
+ onResponse: async (response, activeModel) => {
417
+ const upstreamRequestId = response.headers["x-request-id"] ??
418
+ response.headers["request-id"] ?? response.headers["cf-ray"];
419
+ probe("response_headers", {
420
+ responseCount: ++responseCount,
421
+ status: response.status,
422
+ ...(upstreamRequestId ? { upstreamRequestId } : {}),
423
+ });
424
+ await options?.onResponse?.(response, activeModel);
425
+ },
426
+ });
427
+ return stopStalledModelStream(source, watchdog, probe);
428
+ } catch (error) {
429
+ probe("dispatch_error", {
430
+ errorName: error instanceof Error ? error.name : typeof error,
431
+ });
432
+ throw error;
433
+ }
288
434
  });
289
435
  }
290
436