@springbrand/agent-runtime 0.1.3-alpha.2 → 0.1.3-alpha.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@springbrand/agent-runtime",
-  "version": "0.1.3-alpha.2",
+  "version": "0.1.3-alpha.3",
   "type": "module",
   "files": [
     "src",

package/src/index.ts

@@ -126,6 +126,7 @@ export {
 } from "./layers/orchestration/temporary-agent/workspace";
 export {
   createPiModels,
+  modelRequestUrl,
   resolvePiApiKey,
   resolvePiModel,
 } from "./pi/runtime-adapter/models";

package/src/lib/telemetry-dev.ts

@@ -1,9 +1,12 @@
 /**
- * dev-only 观测降级:把 observability 事件打到 console,让 `wrangler dev` /
- * `wrangler tail` 里能实时看到埋点(否则 publish 到零订阅 channel = 静默 no-op)。
+ * 观测降级:把 observability 事件打到 console,让 `wrangler dev` / `wrangler tail` 里能实时看到埋点,
+ * 并且(生产上开了 `observability.logs` 时)进 Workers Logs 供事后查询。
+ * 不装它,事件就 publish 到零订阅 channel = 静默 no-op。
  *
- * 仅在 env `TELEMETRY_CONSOLE==="1"` 时由 onStart 装载(`.dev.vars` 里开,prod 不设 → 自动关)。
- * 生产不靠它:prod 所有 channel 事件自动转发 Tail Worker。
+ * 由 onStart 在 env `TELEMETRY_CONSOLE==="1"` 时装载 —— `.dev.vars` 和 `wrangler.jsonc` 的 vars 里都开着。
+ * 注意:上游注释宣称"生产上所有 channel 事件自动转发 Tail Worker",那条路径需要 `tail_consumers`,
+ * 本仓没有配,所以生产**不能**指望它;这个 console sink 就是生产的唯一消费面(2026-08-06 实测:
+ * 未开启前 Observability API 查 universal-agent 24h 事件 count=0)。
  *
  * 用 `agents/observability` 的类型化 `subscribe`(按 channel key 订阅),避免引 `node:diagnostics_channel`
  * 与 `@types/node`(tsconfig 只带 workers-types)。

package/src/pi/runtime-adapter/index.ts

@@ -11,6 +11,7 @@ import { createModels, type MutableModels } from "@earendil-works/pi-ai";
 import { configurePiModels, resolvePiApiKey } from "./models";
 export {
   MODEL_STREAM_STALL_TIMEOUT_MS,
+  modelRequestUrl,
   readModelStreamStallDetails,
   withProviderRetry,
 } from "./models";

package/src/pi/runtime-adapter/models.ts

@@ -43,6 +43,10 @@ import {
 import {
   ChatStreamStalledError,
 } from "agents/chat";
+import {
+  genericObservability,
+  type ObservabilityEvent,
+} from "agents/observability";
 import type {
   RuntimeModelEndpoint,
   RuntimeModelProtocol,
@@ -58,7 +62,19 @@ const CATALOGS = {
 } satisfies Record<RuntimeModelProtocol, readonly Model<Api>[]>;
 
 const PROVIDER_MAX_RETRIES = 2;
-export const MODEL_STREAM_STALL_TIMEOUT_MS = 60_000;
+// 看门狗要抓的是「连接死了」,不是「模型想得慢」——这两件事在流上分不开:
+// 推理模型在中转后面是「闷头想完再吐」,静默期一个字节都没有,也没有 keepalive。
+//
+// 2026-08-06 实测(gpt-5.6-sol @ api.sharkmelon.tech,一道需要真推理的题):
+// 响应头 3.86s → 单段静默 **34.4s** → 1190 个 chunk 在 12s 内吐完,首个可见内容 38.3s。
+// 而真实回合(长 transcript + 工具)比这道题重得多。原值 60s 卡在这条曲线的正中间:
+// 简单回合(3~9s)不触发,一旦模型真开始想就必然超时 → abort → 从 transcript 整轮重跑
+// → 同一个提示词又想同样久 → 再超时。**重试的对象正是那个「本来就要更久」的东西,
+// 结构上不可能收敛**,表现为前端 think 转到恢复预算耗尽为止。
+//
+// 调到 240s:比实测静默期留约 7 倍余量。代价是连接真死时单次要等更久,
+// 所以 runtime.ts 同时把 stall 的恢复次数单独收窄(见 CHAT_STALL_MAX_ATTEMPTS)。
+export const MODEL_STREAM_STALL_TIMEOUT_MS = 240_000;
 export const MODEL_STREAM_STALL_MESSAGE =
   `Chat stream stalled: no activity for ${MODEL_STREAM_STALL_TIMEOUT_MS}ms; the turn was aborted by the stall watchdog.`;
 const MODEL_STREAM_STALL_DETAILS_PREFIX = `${MODEL_STREAM_STALL_MESSAGE}\n`;
@@ -180,12 +196,17 @@ function meaningfulModelProgress(
 async function* stopStalledModelStream(
   source: AsyncIterable<AssistantMessageEvent>,
   watchdog: AbortController,
+  probe: (phase: string, details?: Record<string, unknown>) => void,
 ): AsyncGenerator<AssistantMessageEvent> {
   const iterator = source[Symbol.asyncIterator]();
   let timer: ReturnType<typeof setTimeout> | undefined;
   let stalled = false;
   let stallError: ChatStreamStalledError | undefined;
   let idleWaitMs = 0;
+  let rawEventCount = 0;
+  let meaningfulEventCount = 0;
+  let lastRawEventAt: number | undefined;
+  let lastRawEventType: AssistantMessageEvent["type"] | undefined;
   const streamedContent = new Set<string>();
   let lastMeaningfulActivityAt = Date.now();
   let lastMeaningfulActivityType:
@@ -193,6 +214,17 @@ async function* stopStalledModelStream(
       "model_stream_started";
   const stop = (idleMs: number) => {
     stalled = true;
+    const now = Date.now();
+    probe("stall", {
+      idleMs,
+      rawEventCount,
+      meaningfulEventCount,
+      lastRawEventType: lastRawEventType ?? "none",
+      sinceLastRawEventMs: lastRawEventAt === undefined
+        ? null
+        : now - lastRawEventAt,
+      lastMeaningfulActivityType,
+    });
     stallError = new ChatStreamStalledError(
       MODEL_STREAM_STALL_DETAILS_PREFIX + JSON.stringify({
         lastMeaningfulActivityAt,
@@ -227,6 +259,11 @@ async function* stopStalledModelStream(
         ]);
       } catch (error) {
         if (stalled) throw stallError;
+        probe("iterator_error", {
+          errorName: error instanceof Error ? error.name : typeof error,
+          rawEventCount,
+          meaningfulEventCount,
+        });
         throw error;
       } finally {
         clearTimeout(timer);
@@ -235,13 +272,31 @@ async function* stopStalledModelStream(
       if (stalled) throw stallError;
       if (next.done) break;
       const event = next.value;
+      rawEventCount += 1;
+      lastRawEventAt = Date.now();
+      lastRawEventType = event.type;
+      if (rawEventCount === 1) {
+        probe("first_raw_event", { eventType: event.type });
+      }
       if (event.type === "done" || event.type === "error") {
+        probe(event.type, {
+          reason: event.reason,
+          stopReason: event.type === "done"
+            ? event.message.stopReason
+            : event.error.stopReason,
+          rawEventCount,
+          meaningfulEventCount,
+        });
         yield event;
         return;
       }
       idleWaitMs += Date.now() - waitStartedAt;
       const progress = meaningfulModelProgress(event, streamedContent);
       if (progress) {
+        meaningfulEventCount += 1;
+        if (meaningfulEventCount === 1) {
+          probe("first_meaningful_event", { eventType: progress });
+        }
         lastMeaningfulActivityAt = Date.now();
         lastMeaningfulActivityType = progress;
         idleWaitMs = 0;
@@ -252,16 +307,65 @@ async function* stopStalledModelStream(
     clearTimeout(timer);
     if (!stalled) await iterator.return?.().catch(() => {});
   }
+  probe("ended_without_terminal", {
+    rawEventCount,
+    meaningfulEventCount,
+    lastRawEventType: lastRawEventType ?? "none",
+  });
   throw new Error("Model stream ended without a terminal event");
 }
 
+function trimTrailingSlash(value: string): string {
+  return value.endsWith("/") ? value.slice(0, -1) : value;
+}
+
+/**
+ * 算出一个已解析模型真正会被请求的 URL。
+ *
+ * {@link withProviderRetry} 在每次派发模型请求前调用它写观测日志；诊断"这个 Agent 到底调了哪个 LLM"时也可以直接复用。
+ *
+ * 各协议的路径由底层 SDK 决定，这里必须与之逐条对齐：OpenAI 兼容 SDK 用 `baseURL + /chat/completions`，
+ * Anthropic SDK 用 `baseURL + /v1/messages`（纯字符串拼接，不会去重 `/v1`），Google 的 baseUrl 已含版本段，
+ * Codex 走 `resolveCodexUrl`。不要在这里"顺手规范化"路径，否则日志会与真实请求脱节，反而掩盖配置错误。
+ */
+export function modelRequestUrl(model: Model<Api>): string {
+  const base = trimTrailingSlash(model.baseUrl ?? "");
+  switch (model.api) {
+    case "openai-completions":
+      return `${base}/chat/completions`;
+    case "anthropic-messages":
+      return `${base}/v1/messages`;
+    case "google-generative-ai":
+      return `${base}/models/${model.id}:streamGenerateContent`;
+    case "openai-codex-responses":
+      return base.endsWith("/codex/responses") || base.endsWith("/responses")
+        ? base
+        : `${base}/codex/responses`;
+    default:
+      return base;
+  }
+}
+
 /**
  * 给模型请求补上可中断的空闲终止边界。
  *
  * Runtime Turn 和 SubAgent 在把 `Models.streamSimple` 交给 Pi 前调用；显式传入的重试次数优先。
  *
  * 调用方可以覆盖默认的 2 次 Provider 重试；主 Turn 传 0，由 Submission
- * 统一持有恢复预算。连续 60 秒没有可展示进展时中止 provider。
+ * 统一持有恢复预算。连续 {@link MODEL_STREAM_STALL_TIMEOUT_MS} 毫秒没有可展示进展时中止 provider。
+ *
+ * 每个相位发一条 `ua:model` 观测事件(dispatch / response_headers / first_raw_event /
+ * first_meaningful_event / stall / done…)。**只发事件、不直接 console.log**:
+ * 这样它和 `ua:tool`、`chat:*` 共用同一个消费面,被 `TELEMETRY_CONSOLE` 一个开关统一管,
+ * 关掉即零订阅 no-op;将来若接上 `tail_consumers`,这条也自动跟着走。
+ *
+ * 每条都带 `url`(而不是只在 dispatch 带一次):模型路由只存在于 secret 里,
+ * 线上排查时最需要回答的就是"这次打到哪个 URL",让每行自解释比省几十字节值。
+ * payload 只含路由与时序,不含 key 和消息内容。
+ *
+ * 注意这里用的是模块级 `genericObservability`,不是 Agent 实例的 `_emit` ——
+ * 纯模块拿不到实例,代价是事件不带 `agent` / `name` 字段;turn 的身份由
+ * payload 里的 `requestId` / `sessionId` 承担。
  */
 export function withProviderRetry(
   streamFn: StreamFn,
@@ -270,21 +374,63 @@ export function withProviderRetry(
 ): StreamFn {
   return (model, context, options) =>
     lazyStream(model, async () => {
+      const requestId = crypto.randomUUID();
+      const startedAt = Date.now();
+      const sessionId = options?.sessionId ?? defaultSessionId;
+      const url = modelRequestUrl(model);
+      const probe = (phase: string, details: Record<string, unknown> = {}) =>
+        // `ua:*` 是本仓自有的事件命名,不在上游的 ObservabilityEvent 联合里,
+        // 故整体断言一次(runtime.ts 的 `ua:tool` 是同一处上游类型缺口)。
+        // 不能只把 type 断言成 never——那会把联合窄成 never,连 payload 一起报错。
+        genericObservability.emit({
+          type: "ua:model",
+          timestamp: Date.now(),
+          payload: {
+            requestId,
+            sessionId,
+            phase,
+            elapsedMs: Date.now() - startedAt,
+            url,
+            api: model.api,
+            provider: model.provider,
+            model: model.id,
+            ...details,
+          },
+        } as unknown as ObservabilityEvent);
+      probe("dispatch");
       const watchdog = new AbortController();
-      const source = await streamFn(model, context, {
-        ...options,
-        signal: options?.signal
-          ? AbortSignal.any([options.signal, watchdog.signal])
-          : watchdog.signal,
-        maxRetries: options?.maxRetries ?? defaultMaxRetries,
-        sessionId: options?.sessionId ?? defaultSessionId,
-        onPayload: async (payload, activeModel) =>
-          pdfPayload(
-            await options?.onPayload?.(payload, activeModel) ?? payload,
-            activeModel.api,
-          ),
-      });
-      return stopStalledModelStream(source, watchdog);
+      let responseCount = 0;
+      try {
+        const source = await streamFn(model, context, {
+          ...options,
+          signal: options?.signal
+            ? AbortSignal.any([options.signal, watchdog.signal])
+            : watchdog.signal,
+          maxRetries: options?.maxRetries ?? defaultMaxRetries,
+          sessionId,
+          onPayload: async (payload, activeModel) =>
+            pdfPayload(
+              await options?.onPayload?.(payload, activeModel) ?? payload,
+              activeModel.api,
+            ),
+          onResponse: async (response, activeModel) => {
+            const upstreamRequestId = response.headers["x-request-id"] ??
+              response.headers["request-id"] ?? response.headers["cf-ray"];
+            probe("response_headers", {
+              responseCount: ++responseCount,
+              status: response.status,
+              ...(upstreamRequestId ? { upstreamRequestId } : {}),
+            });
+            await options?.onResponse?.(response, activeModel);
+          },
+        });
+        return stopStalledModelStream(source, watchdog, probe);
+      } catch (error) {
+        probe("dispatch_error", {
+          errorName: error instanceof Error ? error.name : typeof error,
+        });
+        throw error;
+      }
     });
 }
 

package/src/runtime.ts

@@ -96,7 +96,13 @@ import {
 
 const SCHEDULED_STABLE_TIMEOUT_MS = 30_000;
 const TURN_EVENT_RETRY_SECONDS = 10;
-const CHAT_RECOVERY_MAX_ATTEMPTS = 5;
+export const CHAT_RECOVERY_MAX_ATTEMPTS = 5;
+// stall 单独收窄。理由不是「stall 更不值得救」,而是它的重试**期望值和别的错不一样**:
+// 瞬时错(5xx / 断流)重跑一次往往就好了;stall 的重跑是拿同一份 transcript 让模型
+// 重新想同样久,如果它本来就超预算,再跑几次也一样超。把看门狗放宽到 240s 之后,
+// 真该救的那一类已经在第一次就跑完了,剩下还在 stall 的基本是连接真死——
+// 那种情况下 5 次 × 240s ≈ 20 分钟的空转纯属折磨用户。3 次约 12 分钟封顶。
+export const CHAT_STALL_MAX_ATTEMPTS = 3;
 const CHAT_RECOVERY_TERMINAL_MESSAGE =
   "多次恢复仍未成功，本次生成已停止，当前进度已保留。请发送新消息继续。";
 
@@ -1923,7 +1929,8 @@ export abstract class AgentRuntimeKernel<
           ?.abortReason;
         if (
           !stoppedDuringRecovery &&
-          recoveryErrorCount < CHAT_RECOVERY_MAX_ATTEMPTS
+          recoveryErrorCount <
+            (stalled ? CHAT_STALL_MAX_ATTEMPTS : CHAT_RECOVERY_MAX_ATTEMPTS)
         ) {
           const recoveryOutcome = await this.scheduleChatRecoveryRetry(
             {
@@ -2967,7 +2974,11 @@ export abstract class AgentRuntimeKernel<
                 ? {}
                 : {
                     recoveryAttempt: current.recoveryErrorCount,
-                    recoveryMax: CHAT_RECOVERY_MAX_ATTEMPTS,
+                    // 上限按当前恢复原因取,否则 stall 会显示 "2/5" 却在第 3 次就终止。
+                    recoveryMax:
+                      current.recoveryReason === "no_meaningful_model_progress"
+                        ? CHAT_STALL_MAX_ATTEMPTS
+                        : CHAT_RECOVERY_MAX_ATTEMPTS,
                     ...(current.recoveryReason
                       ? { recoveryReason: current.recoveryReason }
                       : {}),