@springbrand/agent-runtime 0.1.3-alpha.2 → 0.1.3-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/db/agent-tool.repo.ts +27 -0
- package/src/db/index.ts +33 -0
- package/src/db/interaction.repo.ts +185 -0
- package/src/db/schema.ts +15 -0
- package/src/db/submission.repo.ts +29 -0
- package/src/index.ts +8 -17
- package/src/kernel/approval-lifecycle.ts +41 -6
- package/src/kernel/bindings.ts +37 -0
- package/src/kernel/interaction-lifecycle.ts +395 -0
- package/src/kernel/public-contracts.ts +2 -0
- package/src/kernel/recoverable-chat-agent.ts +10 -2
- package/src/kernel/runtime-assembly-view.ts +37 -0
- package/src/kernel/runtime-assembly.ts +41 -0
- package/src/kernel/runtime-config.ts +4 -0
- package/src/kernel/runtime-load.ts +102 -0
- package/src/kernel/state.ts +8 -1
- package/src/kernel/submission-lifecycle.ts +30 -0
- package/src/lib/telemetry-dev.ts +7 -4
- package/src/pi/runtime-adapter/assembly.ts +13 -1
- package/src/pi/runtime-adapter/execution.ts +109 -9
- package/src/pi/runtime-adapter/index.ts +10 -3
- package/src/pi/runtime-adapter/models.ts +162 -16
- package/src/pi/runtime-adapter/recovery.ts +188 -1
- package/src/pi/tool/base.ts +62 -9
- package/src/pi/tool/compiler.ts +34 -0
- package/src/pi/tool/gateway.ts +54 -0
- package/src/pi/tool/index.ts +1 -0
- package/src/pi/tool/mcp.ts +93 -64
- package/src/pi/turn/index.ts +20 -0
- package/src/pi/turn/interaction.ts +181 -0
- package/src/pi/turn/tool-recovery.ts +244 -1
- package/src/runtime-agent.ts +246 -113
- package/src/{plugins.ts → runtime-assembler.ts} +65 -282
- package/src/runtime.ts +454 -162
package/src/lib/telemetry-dev.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
2
|
+
* 观测降级:把 observability 事件打到 console,让 `wrangler dev` / `wrangler tail` 里能实时看到埋点,
|
|
3
|
+
* 并且(生产上开了 `observability.logs` 时)进 Workers Logs 供事后查询。
|
|
4
|
+
* 不装它,事件就 publish 到零订阅 channel = 静默 no-op。
|
|
4
5
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
6
|
+
* 由 onStart 在 env `TELEMETRY_CONSOLE==="1"` 时装载 —— `.dev.vars` 和 `wrangler.jsonc` 的 vars 里都开着。
|
|
7
|
+
* 注意:上游注释宣称"生产上所有 channel 事件自动转发 Tail Worker",那条路径需要 `tail_consumers`,
|
|
8
|
+
* 本仓没有配,所以生产**不能**指望它;这个 console sink 就是生产的唯一消费面(2026-08-06 实测:
|
|
9
|
+
* 未开启前 Observability API 查 universal-agent 24h 事件 count=0)。
|
|
7
10
|
*
|
|
8
11
|
* 用 `agents/observability` 的类型化 `subscribe`(按 channel key 订阅),避免引 `node:diagnostics_channel`
|
|
9
12
|
* 与 `@types/node`(tsconfig 只带 workers-types)。
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
RuntimeBindings,
|
|
3
|
+
RuntimeGatewaySession,
|
|
3
4
|
RuntimePlatformPort,
|
|
4
5
|
} from "../../kernel/bindings";
|
|
5
6
|
import type { RuntimeDegradation } from "../../kernel/degradation";
|
|
@@ -8,7 +9,7 @@ import type {
|
|
|
8
9
|
} from "../../kernel/extensions";
|
|
9
10
|
import type { RuntimeProfile } from "../../kernel/profile";
|
|
10
11
|
import type { ExecutionLevel } from "../../lib/execution-level";
|
|
11
|
-
import type { RuntimeSnapshot } from "../../
|
|
12
|
+
import type { RuntimeSnapshot } from "../../runtime-assembler";
|
|
12
13
|
import type { PiRuntimeAssembly } from "../assembly";
|
|
13
14
|
import {
|
|
14
15
|
assemblePiSystemContext,
|
|
@@ -19,6 +20,7 @@ import {
|
|
|
19
20
|
type PiLoadedExtension,
|
|
20
21
|
} from "../assembly";
|
|
21
22
|
import {
|
|
23
|
+
createPiGatewayToolCandidates,
|
|
22
24
|
createPiMcpToolCandidates,
|
|
23
25
|
listExtensionsPiToolCandidate,
|
|
24
26
|
type PiMcpHost,
|
|
@@ -58,6 +60,7 @@ interface PiAssemblySnapshot {
|
|
|
58
60
|
interface PreparePiAssemblyOptions {
|
|
59
61
|
readonly snapshot: PiAssemblySnapshot;
|
|
60
62
|
readonly mcpHost: PiMcpHost;
|
|
63
|
+
readonly gatewaySession?: RuntimeGatewaySession;
|
|
61
64
|
// 为一个扩展生成带权限参数的 Host Fetcher。
|
|
62
65
|
// loadPiExtension 在扩展需要 Host 能力时调用它。
|
|
63
66
|
// Worker 掌握真实绑定,装配层只保存这个窄函数,不能自行扩大权限。
|
|
@@ -90,6 +93,8 @@ interface PreparedPiAssembly {
|
|
|
90
93
|
export interface PreparePiRuntimeOptions {
|
|
91
94
|
readonly snapshot: RuntimeSnapshot;
|
|
92
95
|
readonly mcpHost: PiMcpHost;
|
|
96
|
+
readonly gatewaySession?: RuntimeGatewaySession;
|
|
97
|
+
readonly additionalDegradations?: readonly RuntimeDegradation[];
|
|
93
98
|
/**
|
|
94
99
|
* 为一个扩展创建受限的 Host 绑定。
|
|
95
100
|
*
|
|
@@ -115,6 +120,7 @@ export interface PreparePiRuntimeOptions {
|
|
|
115
120
|
export interface PreparedPiRuntime {
|
|
116
121
|
readonly revisionDescriptor: string;
|
|
117
122
|
readonly degradations: readonly RuntimeDegradation[];
|
|
123
|
+
readonly extensions: readonly { readonly name: string; readonly version: string }[];
|
|
118
124
|
}
|
|
119
125
|
|
|
120
126
|
/**
|
|
@@ -301,6 +307,7 @@ class PreparedRuntime implements PreparedPiRuntime {
|
|
|
301
307
|
constructor(
|
|
302
308
|
readonly revisionDescriptor: string,
|
|
303
309
|
readonly degradations: readonly RuntimeDegradation[],
|
|
310
|
+
readonly extensions: readonly { readonly name: string; readonly version: string }[],
|
|
304
311
|
private readonly owner: object,
|
|
305
312
|
private readonly state: PreparedPiRuntimeState,
|
|
306
313
|
) {}
|
|
@@ -414,6 +421,9 @@ async function preparePiAssembly(
|
|
|
414
421
|
candidates: snapshot.pi.toolSurface.finalize([
|
|
415
422
|
...extensions.candidates,
|
|
416
423
|
listExtensionsPiToolCandidate(extensions.loaded),
|
|
424
|
+
...(options.gatewaySession
|
|
425
|
+
? createPiGatewayToolCandidates(options.gatewaySession)
|
|
426
|
+
: []),
|
|
417
427
|
...createPiMcpToolCandidates(
|
|
418
428
|
options.mcpHost,
|
|
419
429
|
snapshot.profile.mcpServers,
|
|
@@ -444,8 +454,10 @@ export async function preparePiRuntime(
|
|
|
444
454
|
describeRuntime(options.snapshot, candidates, assembly.loaded),
|
|
445
455
|
Object.freeze([
|
|
446
456
|
...options.snapshot.degradations,
|
|
457
|
+
...(options.additionalDegradations ?? []),
|
|
447
458
|
...assembly.degradations,
|
|
448
459
|
]),
|
|
460
|
+
Object.freeze(assembly.loaded.map(({ name, version }) => ({ name, version }))),
|
|
449
461
|
owner,
|
|
450
462
|
Object.freeze({
|
|
451
463
|
snapshot: options.snapshot,
|
|
@@ -16,7 +16,11 @@ import type {
|
|
|
16
16
|
UserMessage,
|
|
17
17
|
} from "@earendil-works/pi-ai";
|
|
18
18
|
import { transformMessages } from "@earendil-works/pi-ai/api/transform-messages";
|
|
19
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
parkPiToolApproval,
|
|
21
|
+
parkPiToolInteraction,
|
|
22
|
+
requiresPiToolApproval,
|
|
23
|
+
} from "../turn";
|
|
20
24
|
import { PiChunkEncoder } from "../message";
|
|
21
25
|
import type { UIMessageChunk } from "ai";
|
|
22
26
|
import { ChatStreamStalledError } from "agents/chat";
|
|
@@ -25,6 +29,7 @@ import {
|
|
|
25
29
|
createPiToolGovernance,
|
|
26
30
|
normalizeUpdatePlanArguments,
|
|
27
31
|
type PiToolCandidate,
|
|
32
|
+
type PiToolInteractionSpec,
|
|
28
33
|
type PiToolGovernance,
|
|
29
34
|
type PiToolTelemetry,
|
|
30
35
|
type SettledPiToolCall,
|
|
@@ -40,6 +45,7 @@ import {
|
|
|
40
45
|
resolvePiApiKey,
|
|
41
46
|
withProviderRetry,
|
|
42
47
|
} from "./models";
|
|
48
|
+
import { EXECUTION_LEVELS } from "../../lib/execution-level";
|
|
43
49
|
|
|
44
50
|
// #region Single-run Pi bridge
|
|
45
51
|
|
|
@@ -308,6 +314,18 @@ export interface PiTurnDurability {
|
|
|
308
314
|
signal?: AbortSignal,
|
|
309
315
|
onCreated?: () => void | Promise<void>,
|
|
310
316
|
): Promise<void>;
|
|
317
|
+
/**
|
|
318
|
+
* 保存一次客户端结算请求并等待响应。
|
|
319
|
+
*
|
|
320
|
+
* @remarks
|
|
321
|
+
* Tool candidate 声明了 `interaction` 时,PreparedPiTurnAdapter 在执行前调用它,`execute` 不会被调用。
|
|
322
|
+
* 返回后必须再读一次 settlement —— 响应、取消和跨休眠恢复三条路径都只把结果留在 settlement 里。
|
|
323
|
+
*/
|
|
324
|
+
requestToolInteraction(
|
|
325
|
+
interaction: import("../turn").PiToolInteraction,
|
|
326
|
+
settle: import("../tool").PiToolInteractionSpec,
|
|
327
|
+
signal?: AbortSignal,
|
|
328
|
+
): Promise<void>;
|
|
311
329
|
/**
|
|
312
330
|
* 记录工具输入,并返回它是否是第一次持久化尝试。
|
|
313
331
|
*
|
|
@@ -475,13 +493,21 @@ export class PreparedPiTurnAdapter {
|
|
|
475
493
|
...candidate.tool,
|
|
476
494
|
// 只在 Runtime 的持久化和策略门禁都放行后执行一个 Tool candidate。
|
|
477
495
|
// 实时运行由 PiCore 调用它,恢复或审批续跑则由 retryTool 进入同一路径。
|
|
478
|
-
//
|
|
496
|
+
// 顺序不能随便调整:门禁先于结果复用,审批可能生成结果,不确定的非幂等工作不能重放。
|
|
479
497
|
execute: async (toolCallId, input, signal, onUpdate) => {
|
|
498
|
+
const requiredExecutionLevel = candidate.requiredExecutionLevelForInput
|
|
499
|
+
? await candidate.requiredExecutionLevelForInput(input)
|
|
500
|
+
: candidate.requiredExecutionLevel;
|
|
501
|
+
if (!EXECUTION_LEVELS.includes(requiredExecutionLevel)) {
|
|
502
|
+
throw new Error(
|
|
503
|
+
`Pi Tool "${candidate.tool.name}" resolved an invalid execution level`,
|
|
504
|
+
);
|
|
505
|
+
}
|
|
480
506
|
await state.snapshot.bindings.platform.gateTool?.({
|
|
481
507
|
toolCallId,
|
|
482
508
|
toolName: candidate.tool.name,
|
|
483
509
|
input,
|
|
484
|
-
requiredExecutionLevel
|
|
510
|
+
requiredExecutionLevel,
|
|
485
511
|
signal: signal ?? new AbortController().signal,
|
|
486
512
|
});
|
|
487
513
|
const settled = options.durability.findToolSettlement(toolCallId);
|
|
@@ -494,10 +520,13 @@ export class PreparedPiTurnAdapter {
|
|
|
494
520
|
}
|
|
495
521
|
return settled.result;
|
|
496
522
|
}
|
|
497
|
-
if (
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
523
|
+
if (
|
|
524
|
+
candidate.alwaysRequiresApproval ||
|
|
525
|
+
requiresPiToolApproval({
|
|
526
|
+
executionLevel: descriptor.executionLevel,
|
|
527
|
+
requiredExecutionLevel,
|
|
528
|
+
})
|
|
529
|
+
) {
|
|
501
530
|
const approval = parkPiToolApproval({
|
|
502
531
|
executionId: `${options.submission.id}:${toolCallId}`,
|
|
503
532
|
requestId: options.submission.requestId,
|
|
@@ -509,7 +538,7 @@ export class PreparedPiTurnAdapter {
|
|
|
509
538
|
candidate.tool.label ??
|
|
510
539
|
candidate.tool.name,
|
|
511
540
|
executionLevel: descriptor.executionLevel,
|
|
512
|
-
requiredExecutionLevel
|
|
541
|
+
requiredExecutionLevel,
|
|
513
542
|
inputJson: JSON.stringify(input),
|
|
514
543
|
createdAt: Date.now(),
|
|
515
544
|
}).approval;
|
|
@@ -539,6 +568,43 @@ export class PreparedPiTurnAdapter {
|
|
|
539
568
|
const afterApproval = options.durability
|
|
540
569
|
.findToolSettlement(toolCallId);
|
|
541
570
|
if (afterApproval) return afterApproval.result;
|
|
571
|
+
// 结果由客户端提供的 Tool 在这里 park,永远不走到下面的 execute。
|
|
572
|
+
// 放在审批之后:一个 Tool 既要审批又要客户端结算时,先过许可再问用户,
|
|
573
|
+
// 否则会先向用户要答案、再告诉他这事根本不许做。
|
|
574
|
+
if (candidate.interaction) {
|
|
575
|
+
// 必须先把 Tool 输入落成里程碑再 park。恢复出来的结算靠这条记录取回
|
|
576
|
+
// `args`;缺了它,Tool 醒来返回结果时 settleTool 会拿 undefined 的 args
|
|
577
|
+
// 去和真 args 比对,判成 "Conflicting durable Tool settlement" —— 整个
|
|
578
|
+
// Turn 就死在这。(审批那条路不受影响:批准后 Tool 真的执行,args 在
|
|
579
|
+
// 下面的 appendToolInput 里补上了。)
|
|
580
|
+
options.durability.appendToolInput({
|
|
581
|
+
toolCallId,
|
|
582
|
+
toolName: candidate.tool.name,
|
|
583
|
+
input,
|
|
584
|
+
retry: piToolRetryPolicy(candidate),
|
|
585
|
+
});
|
|
586
|
+
const parked = parkPiToolInteraction({
|
|
587
|
+
interactionId: `${options.submission.id}:${toolCallId}`,
|
|
588
|
+
requestId: options.submission.requestId,
|
|
589
|
+
toolCallId,
|
|
590
|
+
toolName: candidate.tool.name,
|
|
591
|
+
inputJson: JSON.stringify(input),
|
|
592
|
+
createdAt: Date.now(),
|
|
593
|
+
}).interaction;
|
|
594
|
+
await options.durability.requestToolInteraction(
|
|
595
|
+
parked,
|
|
596
|
+
candidate.interaction,
|
|
597
|
+
signal,
|
|
598
|
+
);
|
|
599
|
+
const responded = options.durability
|
|
600
|
+
.findToolSettlement(toolCallId);
|
|
601
|
+
if (!responded) {
|
|
602
|
+
throw new Error(
|
|
603
|
+
`Tool interaction produced no durable settlement: ${candidate.tool.name}`,
|
|
604
|
+
);
|
|
605
|
+
}
|
|
606
|
+
return responded.result;
|
|
607
|
+
}
|
|
542
608
|
const retry = piToolRetryPolicy(candidate);
|
|
543
609
|
const firstAttempt = options.durability.appendToolInput({
|
|
544
610
|
toolCallId,
|
|
@@ -572,7 +638,11 @@ export class PreparedPiTurnAdapter {
|
|
|
572
638
|
),
|
|
573
639
|
modelSessionId: options.submission.requestId,
|
|
574
640
|
canonicalMessages: options.canonicalMessages,
|
|
575
|
-
|
|
641
|
+
// 被中断的 Turn 不写结算。它退场路上还会吐出「工具被中止」,而工具其实
|
|
642
|
+
// 没有产生任何结果 —— 结算是单调的,落了这条假结果,重建出来的续跑会
|
|
643
|
+
// 把一个从没跑过的工具当成跑过并失败。取消不走这里:取消要的就是终态。
|
|
644
|
+
settle: (call) =>
|
|
645
|
+
this.interrupted ? undefined : options.durability.settleTool(call),
|
|
576
646
|
onToolTelemetry: options.onToolTelemetry,
|
|
577
647
|
transformContext: options.transformContext,
|
|
578
648
|
});
|
|
@@ -582,6 +652,7 @@ export class PreparedPiTurnAdapter {
|
|
|
582
652
|
|
|
583
653
|
private readonly systemPrompt: string;
|
|
584
654
|
private readonly options: CreatePreparedPiTurnOptions;
|
|
655
|
+
private interrupted = false;
|
|
585
656
|
|
|
586
657
|
/**
|
|
587
658
|
* 请求取消这个 Turn。
|
|
@@ -595,6 +666,20 @@ export class PreparedPiTurnAdapter {
|
|
|
595
666
|
this.turn.abort();
|
|
596
667
|
}
|
|
597
668
|
|
|
669
|
+
/**
|
|
670
|
+
* 中断这个 Turn,把它交给持久续跑接手。
|
|
671
|
+
*
|
|
672
|
+
* @remarks
|
|
673
|
+
* 宿主在 park 期间换掉装配后调用。与 `abort` 的差别只有一处、但很要命:
|
|
674
|
+
* 中断途中冒出来的工具结算一律丢弃。这条执行器的系统提示和工具表锁在它被
|
|
675
|
+
* 创建的那一刻,接下来跑的是另一条按新装配重建的执行器,两边共用同一批
|
|
676
|
+
* `toolCallId` —— 让退场路上的假结果先落盘,新那条就再也纠正不过来了。
|
|
677
|
+
*/
|
|
678
|
+
interrupt(): void {
|
|
679
|
+
this.interrupted = true;
|
|
680
|
+
this.abort();
|
|
681
|
+
}
|
|
682
|
+
|
|
598
683
|
/**
|
|
599
684
|
* 把一条用户消息加入当前 Turn。
|
|
600
685
|
*
|
|
@@ -635,6 +720,21 @@ export class PreparedPiTurnAdapter {
|
|
|
635
720
|
: false;
|
|
636
721
|
}
|
|
637
722
|
|
|
723
|
+
/**
|
|
724
|
+
* 取回某个 Tool 的客户端结算声明,没有声明时返回 null。
|
|
725
|
+
*
|
|
726
|
+
* @remarks
|
|
727
|
+
* `Runtime.respondToolInteraction` 在把客户端响应映射成 ToolResult 前调用它。
|
|
728
|
+
*
|
|
729
|
+
* 声明里含函数,无法持久化;从 Pinned Runtime 的 candidate 现取,
|
|
730
|
+
* 可让「DO 一直醒着」和「park 期间睡过一觉」走同一条解析路径,而不是一条真解析、一条退化成缺省。
|
|
731
|
+
*/
|
|
732
|
+
interactionSpec(toolName: string): PiToolInteractionSpec | null {
|
|
733
|
+
return this.candidates.find(
|
|
734
|
+
(item) => item.tool.name === toolName,
|
|
735
|
+
)?.interaction ?? null;
|
|
736
|
+
}
|
|
737
|
+
|
|
638
738
|
/**
|
|
639
739
|
* 运行这个 Turn,直到 PiCore 和所有等待中的事件回调结束。
|
|
640
740
|
*
|
|
@@ -8,9 +8,10 @@ import {
|
|
|
8
8
|
} from "../message";
|
|
9
9
|
import type { UIMessage } from "ai";
|
|
10
10
|
import { createModels, type MutableModels } from "@earendil-works/pi-ai";
|
|
11
|
-
import { configurePiModels, resolvePiApiKey } from "./models";
|
|
11
|
+
import { configurePiModels, createPiModels, resolvePiApiKey } from "./models";
|
|
12
12
|
export {
|
|
13
13
|
MODEL_STREAM_STALL_TIMEOUT_MS,
|
|
14
|
+
modelRequestUrl,
|
|
14
15
|
readModelStreamStallDetails,
|
|
15
16
|
withProviderRetry,
|
|
16
17
|
} from "./models";
|
|
@@ -128,6 +129,9 @@ export class PiRuntimeAdapter {
|
|
|
128
129
|
prepare(
|
|
129
130
|
options: PreparePiRuntimeOptions,
|
|
130
131
|
): Promise<PreparedPiRuntime> {
|
|
132
|
+
if (this.managesModels) {
|
|
133
|
+
createPiModels(options.snapshot.bindings.provider);
|
|
134
|
+
}
|
|
131
135
|
return preparePiRuntime(options, this.owner);
|
|
132
136
|
}
|
|
133
137
|
|
|
@@ -137,8 +141,7 @@ export class PiRuntimeAdapter {
|
|
|
137
141
|
* @remarks
|
|
138
142
|
* 调用方:Runtime 的 `initConfig()` 在 Prepared Runtime 和 revision 检查成功后、发布新快照前调用。
|
|
139
143
|
*
|
|
140
|
-
*
|
|
141
|
-
* 这解释了为什么一个未被选中的无效模型也会让配置启动失败;不要把激活误当成首次推理请求。
|
|
144
|
+
* 实现理由:`prepare()` 已校验所有 endpoint 模型,这里只把同一份部署目录切换到长期 Models 实例。
|
|
142
145
|
* 注入模型目录时跳过此步骤,以保留调用方对目录的配置权。
|
|
143
146
|
*/
|
|
144
147
|
activate(snapshot: PreparePiRuntimeOptions["snapshot"]): void {
|
|
@@ -209,6 +212,10 @@ export type {
|
|
|
209
212
|
UIChatRequestBody,
|
|
210
213
|
} from "../message";
|
|
211
214
|
export type { PiToolApproval } from "../turn";
|
|
215
|
+
export type {
|
|
216
|
+
PiToolInteraction,
|
|
217
|
+
PiToolInteractionCancelReason,
|
|
218
|
+
} from "../turn";
|
|
212
219
|
export type {
|
|
213
220
|
CreatePreparedPiTurnOptions,
|
|
214
221
|
PiCanonicalUserInput,
|
|
@@ -43,6 +43,10 @@ import {
|
|
|
43
43
|
import {
|
|
44
44
|
ChatStreamStalledError,
|
|
45
45
|
} from "agents/chat";
|
|
46
|
+
import {
|
|
47
|
+
genericObservability,
|
|
48
|
+
type ObservabilityEvent,
|
|
49
|
+
} from "agents/observability";
|
|
46
50
|
import type {
|
|
47
51
|
RuntimeModelEndpoint,
|
|
48
52
|
RuntimeModelProtocol,
|
|
@@ -58,7 +62,19 @@ const CATALOGS = {
|
|
|
58
62
|
} satisfies Record<RuntimeModelProtocol, readonly Model<Api>[]>;
|
|
59
63
|
|
|
60
64
|
const PROVIDER_MAX_RETRIES = 2;
|
|
61
|
-
|
|
65
|
+
// 看门狗要抓的是「连接死了」,不是「模型想得慢」——这两件事在流上分不开:
|
|
66
|
+
// 推理模型在中转后面是「闷头想完再吐」,静默期一个字节都没有,也没有 keepalive。
|
|
67
|
+
//
|
|
68
|
+
// 2026-08-06 实测(gpt-5.6-sol @ api.sharkmelon.tech,一道需要真推理的题):
|
|
69
|
+
// 响应头 3.86s → 单段静默 **34.4s** → 1190 个 chunk 在 12s 内吐完,首个可见内容 38.3s。
|
|
70
|
+
// 而真实回合(长 transcript + 工具)比这道题重得多。原值 60s 卡在这条曲线的正中间:
|
|
71
|
+
// 简单回合(3~9s)不触发,一旦模型真开始想就必然超时 → abort → 从 transcript 整轮重跑
|
|
72
|
+
// → 同一个提示词又想同样久 → 再超时。**重试的对象正是那个「本来就要更久」的东西,
|
|
73
|
+
// 结构上不可能收敛**,表现为前端 think 转到恢复预算耗尽为止。
|
|
74
|
+
//
|
|
75
|
+
// 调到 240s:比实测静默期留约 7 倍余量。代价是连接真死时单次要等更久,
|
|
76
|
+
// 所以 runtime.ts 同时把 stall 的恢复次数单独收窄(见 CHAT_STALL_MAX_ATTEMPTS)。
|
|
77
|
+
export const MODEL_STREAM_STALL_TIMEOUT_MS = 240_000;
|
|
62
78
|
export const MODEL_STREAM_STALL_MESSAGE =
|
|
63
79
|
`Chat stream stalled: no activity for ${MODEL_STREAM_STALL_TIMEOUT_MS}ms; the turn was aborted by the stall watchdog.`;
|
|
64
80
|
const MODEL_STREAM_STALL_DETAILS_PREFIX = `${MODEL_STREAM_STALL_MESSAGE}\n`;
|
|
@@ -180,12 +196,17 @@ function meaningfulModelProgress(
|
|
|
180
196
|
async function* stopStalledModelStream(
|
|
181
197
|
source: AsyncIterable<AssistantMessageEvent>,
|
|
182
198
|
watchdog: AbortController,
|
|
199
|
+
probe: (phase: string, details?: Record<string, unknown>) => void,
|
|
183
200
|
): AsyncGenerator<AssistantMessageEvent> {
|
|
184
201
|
const iterator = source[Symbol.asyncIterator]();
|
|
185
202
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
186
203
|
let stalled = false;
|
|
187
204
|
let stallError: ChatStreamStalledError | undefined;
|
|
188
205
|
let idleWaitMs = 0;
|
|
206
|
+
let rawEventCount = 0;
|
|
207
|
+
let meaningfulEventCount = 0;
|
|
208
|
+
let lastRawEventAt: number | undefined;
|
|
209
|
+
let lastRawEventType: AssistantMessageEvent["type"] | undefined;
|
|
189
210
|
const streamedContent = new Set<string>();
|
|
190
211
|
let lastMeaningfulActivityAt = Date.now();
|
|
191
212
|
let lastMeaningfulActivityType:
|
|
@@ -193,6 +214,17 @@ async function* stopStalledModelStream(
|
|
|
193
214
|
"model_stream_started";
|
|
194
215
|
const stop = (idleMs: number) => {
|
|
195
216
|
stalled = true;
|
|
217
|
+
const now = Date.now();
|
|
218
|
+
probe("stall", {
|
|
219
|
+
idleMs,
|
|
220
|
+
rawEventCount,
|
|
221
|
+
meaningfulEventCount,
|
|
222
|
+
lastRawEventType: lastRawEventType ?? "none",
|
|
223
|
+
sinceLastRawEventMs: lastRawEventAt === undefined
|
|
224
|
+
? null
|
|
225
|
+
: now - lastRawEventAt,
|
|
226
|
+
lastMeaningfulActivityType,
|
|
227
|
+
});
|
|
196
228
|
stallError = new ChatStreamStalledError(
|
|
197
229
|
MODEL_STREAM_STALL_DETAILS_PREFIX + JSON.stringify({
|
|
198
230
|
lastMeaningfulActivityAt,
|
|
@@ -227,6 +259,11 @@ async function* stopStalledModelStream(
|
|
|
227
259
|
]);
|
|
228
260
|
} catch (error) {
|
|
229
261
|
if (stalled) throw stallError;
|
|
262
|
+
probe("iterator_error", {
|
|
263
|
+
errorName: error instanceof Error ? error.name : typeof error,
|
|
264
|
+
rawEventCount,
|
|
265
|
+
meaningfulEventCount,
|
|
266
|
+
});
|
|
230
267
|
throw error;
|
|
231
268
|
} finally {
|
|
232
269
|
clearTimeout(timer);
|
|
@@ -235,13 +272,31 @@ async function* stopStalledModelStream(
|
|
|
235
272
|
if (stalled) throw stallError;
|
|
236
273
|
if (next.done) break;
|
|
237
274
|
const event = next.value;
|
|
275
|
+
rawEventCount += 1;
|
|
276
|
+
lastRawEventAt = Date.now();
|
|
277
|
+
lastRawEventType = event.type;
|
|
278
|
+
if (rawEventCount === 1) {
|
|
279
|
+
probe("first_raw_event", { eventType: event.type });
|
|
280
|
+
}
|
|
238
281
|
if (event.type === "done" || event.type === "error") {
|
|
282
|
+
probe(event.type, {
|
|
283
|
+
reason: event.reason,
|
|
284
|
+
stopReason: event.type === "done"
|
|
285
|
+
? event.message.stopReason
|
|
286
|
+
: event.error.stopReason,
|
|
287
|
+
rawEventCount,
|
|
288
|
+
meaningfulEventCount,
|
|
289
|
+
});
|
|
239
290
|
yield event;
|
|
240
291
|
return;
|
|
241
292
|
}
|
|
242
293
|
idleWaitMs += Date.now() - waitStartedAt;
|
|
243
294
|
const progress = meaningfulModelProgress(event, streamedContent);
|
|
244
295
|
if (progress) {
|
|
296
|
+
meaningfulEventCount += 1;
|
|
297
|
+
if (meaningfulEventCount === 1) {
|
|
298
|
+
probe("first_meaningful_event", { eventType: progress });
|
|
299
|
+
}
|
|
245
300
|
lastMeaningfulActivityAt = Date.now();
|
|
246
301
|
lastMeaningfulActivityType = progress;
|
|
247
302
|
idleWaitMs = 0;
|
|
@@ -252,16 +307,65 @@ async function* stopStalledModelStream(
|
|
|
252
307
|
clearTimeout(timer);
|
|
253
308
|
if (!stalled) await iterator.return?.().catch(() => {});
|
|
254
309
|
}
|
|
310
|
+
probe("ended_without_terminal", {
|
|
311
|
+
rawEventCount,
|
|
312
|
+
meaningfulEventCount,
|
|
313
|
+
lastRawEventType: lastRawEventType ?? "none",
|
|
314
|
+
});
|
|
255
315
|
throw new Error("Model stream ended without a terminal event");
|
|
256
316
|
}
|
|
257
317
|
|
|
318
|
+
function trimTrailingSlash(value: string): string {
|
|
319
|
+
return value.endsWith("/") ? value.slice(0, -1) : value;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* 算出一个已解析模型真正会被请求的 URL。
|
|
324
|
+
*
|
|
325
|
+
* {@link withProviderRetry} 在每次派发模型请求前调用它写观测日志;诊断"这个 Agent 到底调了哪个 LLM"时也可以直接复用。
|
|
326
|
+
*
|
|
327
|
+
* 各协议的路径由底层 SDK 决定,这里必须与之逐条对齐:OpenAI 兼容 SDK 用 `baseURL + /chat/completions`,
|
|
328
|
+
* Anthropic SDK 用 `baseURL + /v1/messages`(纯字符串拼接,不会去重 `/v1`),Google 的 baseUrl 已含版本段,
|
|
329
|
+
* Codex 走 `resolveCodexUrl`。不要在这里"顺手规范化"路径,否则日志会与真实请求脱节,反而掩盖配置错误。
|
|
330
|
+
*/
|
|
331
|
+
export function modelRequestUrl(model: Model<Api>): string {
|
|
332
|
+
const base = trimTrailingSlash(model.baseUrl ?? "");
|
|
333
|
+
switch (model.api) {
|
|
334
|
+
case "openai-completions":
|
|
335
|
+
return `${base}/chat/completions`;
|
|
336
|
+
case "anthropic-messages":
|
|
337
|
+
return `${base}/v1/messages`;
|
|
338
|
+
case "google-generative-ai":
|
|
339
|
+
return `${base}/models/${model.id}:streamGenerateContent`;
|
|
340
|
+
case "openai-codex-responses":
|
|
341
|
+
return base.endsWith("/codex/responses") || base.endsWith("/responses")
|
|
342
|
+
? base
|
|
343
|
+
: `${base}/codex/responses`;
|
|
344
|
+
default:
|
|
345
|
+
return base;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
258
349
|
/**
|
|
259
350
|
* 给模型请求补上可中断的空闲终止边界。
|
|
260
351
|
*
|
|
261
352
|
* Runtime Turn 和 SubAgent 在把 `Models.streamSimple` 交给 Pi 前调用;显式传入的重试次数优先。
|
|
262
353
|
*
|
|
263
354
|
* 调用方可以覆盖默认的 2 次 Provider 重试;主 Turn 传 0,由 Submission
|
|
264
|
-
* 统一持有恢复预算。连续
|
|
355
|
+
* 统一持有恢复预算。连续 {@link MODEL_STREAM_STALL_TIMEOUT_MS} 毫秒没有可展示进展时中止 provider。
|
|
356
|
+
*
|
|
357
|
+
* 每个相位发一条 `ua:model` 观测事件(dispatch / response_headers / first_raw_event /
|
|
358
|
+
* first_meaningful_event / stall / done…)。**只发事件、不直接 console.log**:
|
|
359
|
+
* 这样它和 `ua:tool`、`chat:*` 共用同一个消费面,被 `TELEMETRY_CONSOLE` 一个开关统一管,
|
|
360
|
+
* 关掉即零订阅 no-op;将来若接上 `tail_consumers`,这条也自动跟着走。
|
|
361
|
+
*
|
|
362
|
+
* 每条都带 `url`(而不是只在 dispatch 带一次):模型路由只存在于 secret 里,
|
|
363
|
+
* 线上排查时最需要回答的就是"这次打到哪个 URL",让每行自解释比省几十字节值。
|
|
364
|
+
* payload 只含路由与时序,不含 key 和消息内容。
|
|
365
|
+
*
|
|
366
|
+
* 注意这里用的是模块级 `genericObservability`,不是 Agent 实例的 `_emit` ——
|
|
367
|
+
* 纯模块拿不到实例,代价是事件不带 `agent` / `name` 字段;turn 的身份由
|
|
368
|
+
* payload 里的 `requestId` / `sessionId` 承担。
|
|
265
369
|
*/
|
|
266
370
|
export function withProviderRetry(
|
|
267
371
|
streamFn: StreamFn,
|
|
@@ -270,21 +374,63 @@ export function withProviderRetry(
|
|
|
270
374
|
): StreamFn {
|
|
271
375
|
return (model, context, options) =>
|
|
272
376
|
lazyStream(model, async () => {
|
|
377
|
+
const requestId = crypto.randomUUID();
|
|
378
|
+
const startedAt = Date.now();
|
|
379
|
+
const sessionId = options?.sessionId ?? defaultSessionId;
|
|
380
|
+
const url = modelRequestUrl(model);
|
|
381
|
+
const probe = (phase: string, details: Record<string, unknown> = {}) =>
|
|
382
|
+
// `ua:*` 是本仓自有的事件命名,不在上游的 ObservabilityEvent 联合里,
|
|
383
|
+
// 故整体断言一次(runtime.ts 的 `ua:tool` 是同一处上游类型缺口)。
|
|
384
|
+
// 不能只把 type 断言成 never——那会把联合窄成 never,连 payload 一起报错。
|
|
385
|
+
genericObservability.emit({
|
|
386
|
+
type: "ua:model",
|
|
387
|
+
timestamp: Date.now(),
|
|
388
|
+
payload: {
|
|
389
|
+
requestId,
|
|
390
|
+
sessionId,
|
|
391
|
+
phase,
|
|
392
|
+
elapsedMs: Date.now() - startedAt,
|
|
393
|
+
url,
|
|
394
|
+
api: model.api,
|
|
395
|
+
provider: model.provider,
|
|
396
|
+
model: model.id,
|
|
397
|
+
...details,
|
|
398
|
+
},
|
|
399
|
+
} as unknown as ObservabilityEvent);
|
|
400
|
+
probe("dispatch");
|
|
273
401
|
const watchdog = new AbortController();
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
:
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
402
|
+
let responseCount = 0;
|
|
403
|
+
try {
|
|
404
|
+
const source = await streamFn(model, context, {
|
|
405
|
+
...options,
|
|
406
|
+
signal: options?.signal
|
|
407
|
+
? AbortSignal.any([options.signal, watchdog.signal])
|
|
408
|
+
: watchdog.signal,
|
|
409
|
+
maxRetries: options?.maxRetries ?? defaultMaxRetries,
|
|
410
|
+
sessionId,
|
|
411
|
+
onPayload: async (payload, activeModel) =>
|
|
412
|
+
pdfPayload(
|
|
413
|
+
await options?.onPayload?.(payload, activeModel) ?? payload,
|
|
414
|
+
activeModel.api,
|
|
415
|
+
),
|
|
416
|
+
onResponse: async (response, activeModel) => {
|
|
417
|
+
const upstreamRequestId = response.headers["x-request-id"] ??
|
|
418
|
+
response.headers["request-id"] ?? response.headers["cf-ray"];
|
|
419
|
+
probe("response_headers", {
|
|
420
|
+
responseCount: ++responseCount,
|
|
421
|
+
status: response.status,
|
|
422
|
+
...(upstreamRequestId ? { upstreamRequestId } : {}),
|
|
423
|
+
});
|
|
424
|
+
await options?.onResponse?.(response, activeModel);
|
|
425
|
+
},
|
|
426
|
+
});
|
|
427
|
+
return stopStalledModelStream(source, watchdog, probe);
|
|
428
|
+
} catch (error) {
|
|
429
|
+
probe("dispatch_error", {
|
|
430
|
+
errorName: error instanceof Error ? error.name : typeof error,
|
|
431
|
+
});
|
|
432
|
+
throw error;
|
|
433
|
+
}
|
|
288
434
|
});
|
|
289
435
|
}
|
|
290
436
|
|