@caupulican/pi-agent-core 0.81.0 → 0.81.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,8 +5,9 @@
5
5
  * so long tasks that produce output are never killed (autonomy constraint).
6
6
  */
7
7
  export interface SilenceWatchdog {
8
- /** Report activity (output chunk / stream event); resets the countdown. */
9
- touch(): void;
8
+ /** Report activity (output chunk / stream event); resets the countdown.
9
+ * Pass silenceMs to also change the bound for this and subsequent countdowns. */
10
+ touch(silenceMs?: number): void;
10
11
  /** Stop permanently (normal completion). Idempotent. */
11
12
  disarm(): void;
12
13
  }
@@ -17,31 +18,53 @@ export interface SilenceWatchdogOptions {
17
18
  }
18
19
  export declare function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog;
19
20
  import type { StreamFn } from "../types.ts";
21
+ export type StallPhase = "connect" | "quiet" | "active";
20
22
  export interface StreamIdleOptions {
21
- /** Max ms between events once streaming has started (user-locked default 30s). */
22
- idleMs: number;
23
23
  /** Max ms to wait for the FIRST event (connection/first-token allowance). */
24
24
  connectMs: number;
25
+ /** Max ms between events while content is flowing — the latest content block is
26
+ * text or toolCall. A flowing stream that goes silent this long is presumed dead. */
27
+ activeIdleMs: number;
28
+ /** Max ms between events while the model is quietly working — no content blocks
29
+ * yet (provider queue / prompt prefill / unstreamed reasoning) or the latest block
30
+ * is thinking. Deep-thinking models and huge compaction prompts legitimately sit
31
+ * here for minutes, so this bound is deliberately generous. */
32
+ quietIdleMs: number;
25
33
  /** Fired once when a stall is detected, before the inner request is aborted. */
26
34
  onStall?: (info: {
27
- phase: "connect" | "stream";
35
+ phase: StallPhase;
28
36
  elapsedMs: number;
29
37
  }) => void;
30
38
  }
39
+ /** User-locked defaults: connect 120s / active 180s / quiet 600s. The quiet bound must stay
40
+ * below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the
41
+ * HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */
31
42
  export declare const DEFAULT_STREAM_IDLE: StreamIdleOptions;
43
+ /** Re-resolved at the start of every request, so hosts can wire live-tunable settings. */
44
+ export type StreamIdleOptionsResolver = () => Partial<StreamIdleOptions>;
32
45
  /**
33
46
  * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.
34
47
  *
35
- * `connectMs` bounds the wait for the first event (connection/first-token allowance);
36
- * once streaming starts, `idleMs` bounds the gap between subsequent events. On stall,
37
- * the inner request is aborted and the returned stream resolves immediately with a
38
- * synthetic `AssistantMessage` (`stopReason: "error"`, `errorMessage: "stream stalled:
39
- * no events for <n>ms"`) the exact phrasing `classifyFailure` maps to a retryable
40
- * `stream_stall`, so the host's retry/failover path takes it from there.
48
+ * Phase-aware: `connectMs` bounds the wait for the first event; after that the
49
+ * inter-event bound adapts to what the stream is doing `quietIdleMs` while the
50
+ * model is quietly working (no content blocks yet, or the latest block is thinking:
51
+ * prefill, provider queues, unstreamed reasoning) and `activeIdleMs` once content is
52
+ * flowing (latest block is text/toolCall). This keeps detection fast where silence is
53
+ * anomalous without killing healthy deep-thinking or compaction-sized requests.
54
+ * No bound ever limits total runtime (autonomy constraint).
55
+ *
56
+ * On stall, the inner request is aborted and the returned stream resolves immediately
57
+ * with a synthetic `AssistantMessage` (`stopReason: "error"`, `errorMessage: "stream
58
+ * stalled: no events for <n>ms (<phase> phase)"`) — the `stream stalled` phrasing is
59
+ * what `classifyFailure` maps to a retryable `stream_stall`, so the host's
60
+ * retry/failover path takes it from there.
61
+ *
62
+ * Options may be a resolver function; it is re-invoked at the start of every request,
63
+ * so settings changes apply without rewrapping.
41
64
  *
42
65
  * A caller-initiated abort (via the options `signal`) is never treated as a stall: it
43
66
  * is chained into the wrapper's own controller and the inner stream's own abort result
44
67
  * is forwarded untouched.
45
68
  */
46
- export declare function withStreamIdleWatchdog(streamFn: StreamFn, options?: Partial<StreamIdleOptions>): StreamFn;
69
+ export declare function withStreamIdleWatchdog(streamFn: StreamFn, options?: Partial<StreamIdleOptions> | StreamIdleOptionsResolver): StreamFn;
47
70
  //# sourceMappingURL=watchdogs.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"watchdogs.d.ts","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,eAAe;IAC/B,2EAA2E;IAC3E,KAAK,IAAI,IAAI,CAAC;IACd,wDAAwD;IACxD,MAAM,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,SAAS,EAAE,MAAM,IAAI,CAAC;CACtB;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,GAAG,eAAe,CA4BnF;AASD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,iBAAiB;IACjC,kFAAkF;IAClF,MAAM,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,SAAS,GAAG,QAAQ,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC7E;AAED,eAAO,MAAM,mBAAmB,EAAE,iBAA0D,CAAC;AAS7F;;;;;;;;;;;;;GAaG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,QAAQ,CAgGzG","sourcesContent":["/**\n * Silence/idle watchdogs for the reliability kernel.\n *\n * A silence watchdog bounds \"running but mute\" — it never bounds total runtime,\n * so long tasks that produce output are never killed (autonomy constraint).\n */\n\nexport interface SilenceWatchdog {\n\t/** Report activity (output chunk / stream event); resets the countdown. */\n\ttouch(): void;\n\t/** Stop permanently (normal completion). Idempotent. */\n\tdisarm(): void;\n}\n\nexport interface SilenceWatchdogOptions {\n\tsilenceMs: number;\n\t/** Fired at most once, after silenceMs with no touch(). The watchdog self-disarms. */\n\tonSilence: () => void;\n}\n\nexport function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog {\n\tlet timer: NodeJS.Timeout | undefined;\n\tlet disarmed = false;\n\n\tconst arm = () => {\n\t\ttimer = setTimeout(() => {\n\t\t\tdisarmed = true;\n\t\t\ttimer = undefined;\n\t\t\topts.onSilence();\n\t\t}, opts.silenceMs);\n\t\t// Never keep the host process alive just for a watchdog.\n\t\ttimer.unref?.();\n\t};\n\n\tarm();\n\n\treturn {\n\t\ttouch(): void {\n\t\t\tif (disarmed) return;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\tarm();\n\t\t},\n\t\tdisarm(): void {\n\t\t\tdisarmed = true;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\ttimer = undefined;\n\t\t},\n\t};\n}\n\n// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------\n\nimport {\n\ttype AssistantMessage,\n\ttype AssistantMessageEvent,\n\tcreateAssistantMessageEventStream,\n} from \"@caupulican/pi-ai\";\nimport type { StreamFn } from \"../types.ts\";\n\nexport interface StreamIdleOptions {\n\t/** Max ms between events once streaming has started (user-locked default 30s). */\n\tidleMs: number;\n\t/** Max ms to wait for the FIRST event (connection/first-token allowance). */\n\tconnectMs: number;\n\t/** Fired once when a stall is detected, before the inner request is aborted. */\n\tonStall?: (info: { phase: \"connect\" | \"stream\"; elapsedMs: number }) => void;\n}\n\nexport const DEFAULT_STREAM_IDLE: StreamIdleOptions = { idleMs: 30_000, connectMs: 120_000 };\n\n/** Extracts the current AssistantMessage snapshot carried by any stream event variant. */\nfunction partialFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\n/**\n * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.\n *\n * `connectMs` bounds the wait for the first event (connection/first-token allowance);\n * once streaming starts, `idleMs` bounds the gap between subsequent events. On stall,\n * the inner request is aborted and the returned stream resolves immediately with a\n * synthetic `AssistantMessage` (`stopReason: \"error\"`, `errorMessage: \"stream stalled:\n * no events for <n>ms\"`) — the exact phrasing `classifyFailure` maps to a retryable\n * `stream_stall`, so the host's retry/failover path takes it from there.\n *\n * A caller-initiated abort (via the options `signal`) is never treated as a stall: it\n * is chained into the wrapper's own controller and the inner stream's own abort result\n * is forwarded untouched.\n */\nexport function withStreamIdleWatchdog(streamFn: StreamFn, options?: Partial<StreamIdleOptions>): StreamFn {\n\tconst opts = { ...DEFAULT_STREAM_IDLE, ...options };\n\treturn async (model, context, streamOptions) => {\n\t\tconst controller = new AbortController();\n\t\tconst callerSignal = streamOptions?.signal;\n\t\tlet callerAborted = callerSignal?.aborted ?? false;\n\t\tconst onCallerAbort = () => {\n\t\t\tcallerAborted = true;\n\t\t\tcontroller.abort(callerSignal?.reason);\n\t\t};\n\t\tif (callerAborted) controller.abort(callerSignal?.reason);\n\t\telse callerSignal?.addEventListener(\"abort\", onCallerAbort, { once: true });\n\n\t\tconst inner = await streamFn(model, context, { ...streamOptions, signal: controller.signal });\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\t// Seeded so a connect-phase stall (no event ever arrived) still has a base message\n\t\t// to report on; overwritten with the latest real snapshot once events start flowing.\n\t\tlet latest: AssistantMessage = {\n\t\t\trole: \"assistant\",\n\t\t\tcontent: [],\n\t\t\tapi: model.api,\n\t\t\tprovider: model.provider,\n\t\t\tmodel: model.id,\n\t\t\tusage: {\n\t\t\t\tinput: 0,\n\t\t\t\toutput: 0,\n\t\t\t\tcacheRead: 0,\n\t\t\t\tcacheWrite: 0,\n\t\t\t\ttotalTokens: 0,\n\t\t\t\tcost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },\n\t\t\t},\n\t\t\tstopReason: \"stop\",\n\t\t\ttimestamp: Date.now(),\n\t\t};\n\t\tlet firstEventSeen = false;\n\t\tlet stalled = false;\n\n\t\t// Emits the stall result directly (rather than after the inner loop finishes) so a\n\t\t// connection that never resolves at all still yields a result promptly — providers\n\t\t// are contractually expected to end their stream after abort, but the watchdog does\n\t\t// not depend on that to report the stall itself.\n\t\tconst stall = (phase: \"connect\" | \"stream\", elapsedMs: number) => {\n\t\t\tif (callerAborted || stalled) return;\n\t\t\tstalled = true;\n\t\t\topts.onStall?.({ phase, elapsedMs });\n\t\t\tcontroller.abort(new Error(`stream stalled: no events for ${elapsedMs}ms`));\n\t\t\tconst message: AssistantMessage = {\n\t\t\t\t...latest,\n\t\t\t\tstopReason: \"error\",\n\t\t\t\terrorMessage: `stream stalled: no events for ${elapsedMs}ms`,\n\t\t\t};\n\t\t\touter.push({ type: \"error\", reason: \"error\", error: message });\n\t\t};\n\n\t\tlet watchdog = createSilenceWatchdog({\n\t\t\tsilenceMs: opts.connectMs,\n\t\t\tonSilence: () => stall(\"connect\", opts.connectMs),\n\t\t});\n\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tif (stalled) break;\n\t\t\t\t\tif (!firstEventSeen) {\n\t\t\t\t\t\tfirstEventSeen = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\twatchdog = createSilenceWatchdog({\n\t\t\t\t\t\t\tsilenceMs: opts.idleMs,\n\t\t\t\t\t\t\tonSilence: () => stall(\"stream\", opts.idleMs),\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\twatchdog.touch();\n\t\t\t\t\t}\n\t\t\t\t\tlatest = partialFromEvent(event);\n\t\t\t\t\t// A terminal event ends the turn: disarm synchronously, in the same tick as\n\t\t\t\t\t// the push below, so no watchdog can fire after the consumer's `result()`\n\t\t\t\t\t// promise resolves — a disarm that only happened once the loop later notices\n\t\t\t\t\t// `inner` is done would race with that resolution (it runs a tick or more\n\t\t\t\t\t// later) and could fire a spurious stall on an already-finished stream.\n\t\t\t\t\tconst terminal = event.type === \"done\" || event.type === \"error\";\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminal) return;\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\twatchdog.disarm();\n\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t}\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n"]}
1
+ {"version":3,"file":"watchdogs.d.ts","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,eAAe;IAC/B;sFACkF;IAClF,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,wDAAwD;IACxD,MAAM,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,SAAS,EAAE,MAAM,IAAI,CAAC;CACtB;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,GAAG,eAAe,CA8BnF;AASD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;AAExD,MAAM,WAAW,iBAAiB;IACjC,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;IAClB;0FACsF;IACtF,YAAY,EAAE,MAAM,CAAC;IACrB;;;oEAGgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACnE;AAED;;8FAE8F;AAC9F,eAAO,MAAM,mBAAmB,EAAE,iBAIjC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,MAAM,yBAAyB,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAAC,CAAC;AASzE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,yBAAyB,GAC9D,QAAQ,CA0HV","sourcesContent":["/**\n * Silence/idle watchdogs for the reliability kernel.\n *\n * A silence watchdog bounds \"running but mute\" — it never bounds total runtime,\n * so long tasks that produce output are never killed (autonomy constraint).\n */\n\nexport interface SilenceWatchdog {\n\t/** Report activity (output chunk / stream event); resets the countdown.\n\t * Pass silenceMs to also change the bound for this and subsequent countdowns. */\n\ttouch(silenceMs?: number): void;\n\t/** Stop permanently (normal completion). Idempotent. */\n\tdisarm(): void;\n}\n\nexport interface SilenceWatchdogOptions {\n\tsilenceMs: number;\n\t/** Fired at most once, after silenceMs with no touch(). The watchdog self-disarms. */\n\tonSilence: () => void;\n}\n\nexport function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog {\n\tlet timer: NodeJS.Timeout | undefined;\n\tlet disarmed = false;\n\tlet currentSilenceMs = opts.silenceMs;\n\n\tconst arm = () => {\n\t\ttimer = setTimeout(() => {\n\t\t\tdisarmed = true;\n\t\t\ttimer = undefined;\n\t\t\topts.onSilence();\n\t\t}, currentSilenceMs);\n\t\t// Never keep the host process alive just for a watchdog.\n\t\ttimer.unref?.();\n\t};\n\n\tarm();\n\n\treturn {\n\t\ttouch(silenceMs?: number): void {\n\t\t\tif (disarmed) return;\n\t\t\tif (silenceMs !== undefined) currentSilenceMs = silenceMs;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\tarm();\n\t\t},\n\t\tdisarm(): void {\n\t\t\tdisarmed = true;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\ttimer = undefined;\n\t\t},\n\t};\n}\n\n// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------\n\nimport {\n\ttype AssistantMessage,\n\ttype AssistantMessageEvent,\n\tcreateAssistantMessageEventStream,\n} from \"@caupulican/pi-ai\";\nimport type { StreamFn } from \"../types.ts\";\n\nexport type StallPhase = \"connect\" | \"quiet\" | \"active\";\n\nexport interface StreamIdleOptions {\n\t/** Max ms to wait for the FIRST event (connection/first-token allowance). */\n\tconnectMs: number;\n\t/** Max ms between events while content is flowing — the latest content block is\n\t * text or toolCall. A flowing stream that goes silent this long is presumed dead. */\n\tactiveIdleMs: number;\n\t/** Max ms between events while the model is quietly working — no content blocks\n\t * yet (provider queue / prompt prefill / unstreamed reasoning) or the latest block\n\t * is thinking. Deep-thinking models and huge compaction prompts legitimately sit\n\t * here for minutes, so this bound is deliberately generous. */\n\tquietIdleMs: number;\n\t/** Fired once when a stall is detected, before the inner request is aborted. */\n\tonStall?: (info: { phase: StallPhase; elapsedMs: number }) => void;\n}\n\n/** User-locked defaults: connect 120s / active 180s / quiet 600s. The quiet bound must stay\n * below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the\n * HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */\nexport const DEFAULT_STREAM_IDLE: StreamIdleOptions = {\n\tconnectMs: 120_000,\n\tactiveIdleMs: 180_000,\n\tquietIdleMs: 600_000,\n};\n\n/** Re-resolved at the start of every request, so hosts can wire live-tunable settings. */\nexport type StreamIdleOptionsResolver = () => Partial<StreamIdleOptions>;\n\n/** Extracts the current AssistantMessage snapshot carried by any stream event variant. */\nfunction partialFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\n/**\n * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.\n *\n * Phase-aware: `connectMs` bounds the wait for the first event; after that the\n * inter-event bound adapts to what the stream is doing — `quietIdleMs` while the\n * model is quietly working (no content blocks yet, or the latest block is thinking:\n * prefill, provider queues, unstreamed reasoning) and `activeIdleMs` once content is\n * flowing (latest block is text/toolCall). This keeps detection fast where silence is\n * anomalous without killing healthy deep-thinking or compaction-sized requests.\n * No bound ever limits total runtime (autonomy constraint).\n *\n * On stall, the inner request is aborted and the returned stream resolves immediately\n * with a synthetic `AssistantMessage` (`stopReason: \"error\"`, `errorMessage: \"stream\n * stalled: no events for <n>ms (<phase> phase)\"`) — the `stream stalled` phrasing is\n * what `classifyFailure` maps to a retryable `stream_stall`, so the host's\n * retry/failover path takes it from there.\n *\n * Options may be a resolver function; it is re-invoked at the start of every request,\n * so settings changes apply without rewrapping.\n *\n * A caller-initiated abort (via the options `signal`) is never treated as a stall: it\n * is chained into the wrapper's own controller and the inner stream's own abort result\n * is forwarded untouched.\n */\nexport function withStreamIdleWatchdog(\n\tstreamFn: StreamFn,\n\toptions?: Partial<StreamIdleOptions> | StreamIdleOptionsResolver,\n): StreamFn {\n\treturn async (model, context, streamOptions) => {\n\t\tconst resolved = typeof options === \"function\" ? options() : options;\n\t\tconst cleaned: Partial<StreamIdleOptions> = {};\n\t\tif (resolved) {\n\t\t\tfor (const [key, val] of Object.entries(resolved)) {\n\t\t\t\tif (val !== undefined) {\n\t\t\t\t\t(cleaned as any)[key] = val;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tconst opts = { ...DEFAULT_STREAM_IDLE, ...cleaned };\n\n\t\tconst controller = new AbortController();\n\t\tconst callerSignal = streamOptions?.signal;\n\t\tlet callerAborted = callerSignal?.aborted ?? false;\n\t\tconst onCallerAbort = () => {\n\t\t\tcallerAborted = true;\n\t\t\tcontroller.abort(callerSignal?.reason);\n\t\t};\n\t\tif (callerAborted) controller.abort(callerSignal?.reason);\n\t\telse callerSignal?.addEventListener(\"abort\", onCallerAbort, { once: true });\n\n\t\tconst inner = await streamFn(model, context, { ...streamOptions, signal: controller.signal });\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\t// Seeded so a connect-phase stall (no event ever arrived) still has a base message\n\t\t// to report on; overwritten with the latest real snapshot once events start flowing.\n\t\tlet latest: AssistantMessage = {\n\t\t\trole: \"assistant\",\n\t\t\tcontent: [],\n\t\t\tapi: model.api,\n\t\t\tprovider: model.provider,\n\t\t\tmodel: model.id,\n\t\t\tusage: {\n\t\t\t\tinput: 0,\n\t\t\t\toutput: 0,\n\t\t\t\tcacheRead: 0,\n\t\t\t\tcacheWrite: 0,\n\t\t\t\ttotalTokens: 0,\n\t\t\t\tcost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },\n\t\t\t},\n\t\t\tstopReason: \"stop\",\n\t\t\ttimestamp: Date.now(),\n\t\t};\n\t\tlet stalled = false;\n\t\tlet firstEventSeen = false;\n\n\t\t// The idle bound adapts per event: quiet while nothing/thinking, active while\n\t\t// text/toolCall content is flowing. Mutable so the onSilence closure always\n\t\t// reports the phase/bound that actually elapsed.\n\t\tlet currentPhase: StallPhase = \"connect\";\n\t\tlet currentBoundMs = opts.connectMs;\n\t\tconst idleBoundFor = (message: AssistantMessage): { phase: StallPhase; ms: number } => {\n\t\t\tconst lastBlock = message.content[message.content.length - 1];\n\t\t\treturn !lastBlock || lastBlock.type === \"thinking\"\n\t\t\t\t? { phase: \"quiet\", ms: opts.quietIdleMs }\n\t\t\t\t: { phase: \"active\", ms: opts.activeIdleMs };\n\t\t};\n\n\t\t// Emits the stall result directly (rather than after the inner loop finishes) so a\n\t\t// connection that never resolves at all still yields a result promptly — providers\n\t\t// are contractually expected to end their stream after abort, but the watchdog does\n\t\t// not depend on that to report the stall itself.\n\t\tconst stall = (phase: StallPhase, elapsedMs: number) => {\n\t\t\tif (callerAborted || stalled) return;\n\t\t\tstalled = true;\n\t\t\topts.onStall?.({ phase, elapsedMs });\n\t\t\tconst description = `stream stalled: no events for ${elapsedMs}ms (${phase} phase)`;\n\t\t\tcontroller.abort(new Error(description));\n\t\t\tconst message: AssistantMessage = {\n\t\t\t\t...latest,\n\t\t\t\tstopReason: \"error\",\n\t\t\t\terrorMessage: description,\n\t\t\t};\n\t\t\touter.push({ type: \"error\", reason: \"error\", error: message });\n\t\t};\n\n\t\tlet watchdog = createSilenceWatchdog({\n\t\t\tsilenceMs: opts.connectMs,\n\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t});\n\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tif (stalled) break;\n\t\t\t\t\tlatest = partialFromEvent(event);\n\t\t\t\t\tconst bound = idleBoundFor(latest);\n\t\t\t\t\tcurrentPhase = bound.phase;\n\t\t\t\t\tcurrentBoundMs = bound.ms;\n\t\t\t\t\tif (!firstEventSeen) {\n\t\t\t\t\t\tfirstEventSeen = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\twatchdog = createSilenceWatchdog({\n\t\t\t\t\t\t\tsilenceMs: bound.ms,\n\t\t\t\t\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\twatchdog.touch(bound.ms);\n\t\t\t\t\t}\n\t\t\t\t\t// A terminal event ends the turn: disarm synchronously, in the same tick as\n\t\t\t\t\t// the push below, so no watchdog can fire after the consumer's `result()`\n\t\t\t\t\t// promise resolves — a disarm that only happened once the loop later notices\n\t\t\t\t\t// `inner` is done would race with that resolution (it runs a tick or more\n\t\t\t\t\t// later) and could fire a spurious stall on an already-finished stream.\n\t\t\t\t\tconst terminal = event.type === \"done\" || event.type === \"error\";\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminal) return;\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\twatchdog.disarm();\n\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t}\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n"]}
@@ -7,20 +7,23 @@
7
7
  export function createSilenceWatchdog(opts) {
8
8
  let timer;
9
9
  let disarmed = false;
10
+ let currentSilenceMs = opts.silenceMs;
10
11
  const arm = () => {
11
12
  timer = setTimeout(() => {
12
13
  disarmed = true;
13
14
  timer = undefined;
14
15
  opts.onSilence();
15
- }, opts.silenceMs);
16
+ }, currentSilenceMs);
16
17
  // Never keep the host process alive just for a watchdog.
17
18
  timer.unref?.();
18
19
  };
19
20
  arm();
20
21
  return {
21
- touch() {
22
+ touch(silenceMs) {
22
23
  if (disarmed)
23
24
  return;
25
+ if (silenceMs !== undefined)
26
+ currentSilenceMs = silenceMs;
24
27
  if (timer)
25
28
  clearTimeout(timer);
26
29
  arm();
@@ -35,7 +38,14 @@ export function createSilenceWatchdog(opts) {
35
38
  }
36
39
  // --- Stream-idle watchdog (wraps a StreamFn) -------------------------------
37
40
  import { createAssistantMessageEventStream, } from "@caupulican/pi-ai";
38
- export const DEFAULT_STREAM_IDLE = { idleMs: 30_000, connectMs: 120_000 };
41
+ /** User-locked defaults: connect 120s / active 180s / quiet 600s. The quiet bound must stay
42
+ * below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the
43
+ * HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */
44
+ export const DEFAULT_STREAM_IDLE = {
45
+ connectMs: 120_000,
46
+ activeIdleMs: 180_000,
47
+ quietIdleMs: 600_000,
48
+ };
39
49
  /** Extracts the current AssistantMessage snapshot carried by any stream event variant. */
40
50
  function partialFromEvent(event) {
41
51
  if (event.type === "done")
@@ -47,20 +57,39 @@ function partialFromEvent(event) {
47
57
  /**
48
58
  * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.
49
59
  *
50
- * `connectMs` bounds the wait for the first event (connection/first-token allowance);
51
- * once streaming starts, `idleMs` bounds the gap between subsequent events. On stall,
52
- * the inner request is aborted and the returned stream resolves immediately with a
53
- * synthetic `AssistantMessage` (`stopReason: "error"`, `errorMessage: "stream stalled:
54
- * no events for <n>ms"`) the exact phrasing `classifyFailure` maps to a retryable
55
- * `stream_stall`, so the host's retry/failover path takes it from there.
60
+ * Phase-aware: `connectMs` bounds the wait for the first event; after that the
61
+ * inter-event bound adapts to what the stream is doing `quietIdleMs` while the
62
+ * model is quietly working (no content blocks yet, or the latest block is thinking:
63
+ * prefill, provider queues, unstreamed reasoning) and `activeIdleMs` once content is
64
+ * flowing (latest block is text/toolCall). This keeps detection fast where silence is
65
+ * anomalous without killing healthy deep-thinking or compaction-sized requests.
66
+ * No bound ever limits total runtime (autonomy constraint).
67
+ *
68
+ * On stall, the inner request is aborted and the returned stream resolves immediately
69
+ * with a synthetic `AssistantMessage` (`stopReason: "error"`, `errorMessage: "stream
70
+ * stalled: no events for <n>ms (<phase> phase)"`) — the `stream stalled` phrasing is
71
+ * what `classifyFailure` maps to a retryable `stream_stall`, so the host's
72
+ * retry/failover path takes it from there.
73
+ *
74
+ * Options may be a resolver function; it is re-invoked at the start of every request,
75
+ * so settings changes apply without rewrapping.
56
76
  *
57
77
  * A caller-initiated abort (via the options `signal`) is never treated as a stall: it
58
78
  * is chained into the wrapper's own controller and the inner stream's own abort result
59
79
  * is forwarded untouched.
60
80
  */
61
81
  export function withStreamIdleWatchdog(streamFn, options) {
62
- const opts = { ...DEFAULT_STREAM_IDLE, ...options };
63
82
  return async (model, context, streamOptions) => {
83
+ const resolved = typeof options === "function" ? options() : options;
84
+ const cleaned = {};
85
+ if (resolved) {
86
+ for (const [key, val] of Object.entries(resolved)) {
87
+ if (val !== undefined) {
88
+ cleaned[key] = val;
89
+ }
90
+ }
91
+ }
92
+ const opts = { ...DEFAULT_STREAM_IDLE, ...cleaned };
64
93
  const controller = new AbortController();
65
94
  const callerSignal = streamOptions?.signal;
66
95
  let callerAborted = callerSignal?.aborted ?? false;
@@ -93,8 +122,19 @@ export function withStreamIdleWatchdog(streamFn, options) {
93
122
  stopReason: "stop",
94
123
  timestamp: Date.now(),
95
124
  };
96
- let firstEventSeen = false;
97
125
  let stalled = false;
126
+ let firstEventSeen = false;
127
+ // The idle bound adapts per event: quiet while nothing/thinking, active while
128
+ // text/toolCall content is flowing. Mutable so the onSilence closure always
129
+ // reports the phase/bound that actually elapsed.
130
+ let currentPhase = "connect";
131
+ let currentBoundMs = opts.connectMs;
132
+ const idleBoundFor = (message) => {
133
+ const lastBlock = message.content[message.content.length - 1];
134
+ return !lastBlock || lastBlock.type === "thinking"
135
+ ? { phase: "quiet", ms: opts.quietIdleMs }
136
+ : { phase: "active", ms: opts.activeIdleMs };
137
+ };
98
138
  // Emits the stall result directly (rather than after the inner loop finishes) so a
99
139
  // connection that never resolves at all still yields a result promptly — providers
100
140
  // are contractually expected to end their stream after abort, but the watchdog does
@@ -104,35 +144,39 @@ export function withStreamIdleWatchdog(streamFn, options) {
104
144
  return;
105
145
  stalled = true;
106
146
  opts.onStall?.({ phase, elapsedMs });
107
- controller.abort(new Error(`stream stalled: no events for ${elapsedMs}ms`));
147
+ const description = `stream stalled: no events for ${elapsedMs}ms (${phase} phase)`;
148
+ controller.abort(new Error(description));
108
149
  const message = {
109
150
  ...latest,
110
151
  stopReason: "error",
111
- errorMessage: `stream stalled: no events for ${elapsedMs}ms`,
152
+ errorMessage: description,
112
153
  };
113
154
  outer.push({ type: "error", reason: "error", error: message });
114
155
  };
115
156
  let watchdog = createSilenceWatchdog({
116
157
  silenceMs: opts.connectMs,
117
- onSilence: () => stall("connect", opts.connectMs),
158
+ onSilence: () => stall(currentPhase, currentBoundMs),
118
159
  });
119
160
  void (async () => {
120
161
  try {
121
162
  for await (const event of inner) {
122
163
  if (stalled)
123
164
  break;
165
+ latest = partialFromEvent(event);
166
+ const bound = idleBoundFor(latest);
167
+ currentPhase = bound.phase;
168
+ currentBoundMs = bound.ms;
124
169
  if (!firstEventSeen) {
125
170
  firstEventSeen = true;
126
171
  watchdog.disarm();
127
172
  watchdog = createSilenceWatchdog({
128
- silenceMs: opts.idleMs,
129
- onSilence: () => stall("stream", opts.idleMs),
173
+ silenceMs: bound.ms,
174
+ onSilence: () => stall(currentPhase, currentBoundMs),
130
175
  });
131
176
  }
132
177
  else {
133
- watchdog.touch();
178
+ watchdog.touch(bound.ms);
134
179
  }
135
- latest = partialFromEvent(event);
136
180
  // A terminal event ends the turn: disarm synchronously, in the same tick as
137
181
  // the push below, so no watchdog can fire after the consumer's `result()`
138
182
  // promise resolves — a disarm that only happened once the loop later notices
@@ -1 +1 @@
1
- {"version":3,"file":"watchdogs.js","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAeH,MAAM,UAAU,qBAAqB,CAAC,IAA4B,EAAmB;IACpF,IAAI,KAAiC,CAAC;IACtC,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC;QACjB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC;YACxB,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,GAAG,SAAS,CAAC;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;QAAA,CACjB,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnB,yDAAyD;QACzD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAAA,CAChB,CAAC;IAEF,GAAG,EAAE,CAAC;IAEN,OAAO;QACN,KAAK,GAAS;YACb,IAAI,QAAQ;gBAAE,OAAO;YACrB,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,GAAG,EAAE,CAAC;QAAA,CACN;QACD,MAAM,GAAS;YACd,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,GAAG,SAAS,CAAC;QAAA,CAClB;KACD,CAAC;AAAA,CACF;AAED,8EAA8E;AAE9E,OAAO,EAGN,iCAAiC,GACjC,MAAM,mBAAmB,CAAC;AAY3B,MAAM,CAAC,MAAM,mBAAmB,GAAsB,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAE7F,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,KAA4B,EAAoB;IACzE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC;AAAA,CACrB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAkB,EAAE,OAAoC,EAAY;IAC1G,MAAM,IAAI,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,OAAO,EAAE,CAAC;IACpD,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,aAAa,EAAE,MAAM,CAAC;QAC3C,IAAI,aAAa,GAAG,YAAY,EAAE,OAAO,IAAI,KAAK,CAAC;QACnD,MAAM,aAAa,GAAG,GAAG,EAAE,CAAC;YAC3B,aAAa,GAAG,IAAI,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAAA,CACvC,CAAC;QACF,IAAI,aAAa;YAAE,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;YACrD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5E,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,GAAG,aAAa,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAElD,mFAAmF;QACnF,qFAAqF;QACrF,IAAI,MAAM,GAAqB;YAC9B,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,EAAE;YACX,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,EAAE;YACf,KAAK,EAAE;gBACN,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,WAAW,EAAE,CAAC;gBACd,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;aACpE;YACD,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC;QACF,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,mFAAmF;QACnF,qFAAmF;QACnF,oFAAoF;QACpF,iDAAiD;QACjD,MAAM,KAAK,GAAG,CAAC,KAA2B,EAAE,SAAiB,EAAE,EAAE,CAAC;YACjE,IAAI,aAAa,IAAI,OAAO;gBAAE,OAAO;YACrC,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YACrC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,iCAAiC,SAAS,IAAI,CAAC,CAAC,CAAC;YAC5E,MAAM,OAAO,GAAqB;gBACjC,GAAG,MAAM;gBACT,UAAU,EAAE,OAAO;gBACnB,YAAY,EAAE,iCAAiC,SAAS,IAAI;aAC5D,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAAA,CAC/D,CAAC;QAEF,IAAI,QAAQ,GAAG,qBAAqB,CAAC;YACpC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC;SACjD,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBACjC,IAAI,OAAO;wBAAE,MAAM;oBACnB,IAAI,CAAC,cAAc,EAAE,CAAC;wBACrB,cAAc,GAAG,IAAI,CAAC;wBACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,QAAQ,GAAG,qBAAqB,CAAC;4BAChC,SAAS,EAAE,IAAI,CAAC,MAAM;4BACtB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;yBAC7C,CAAC,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACP,QAAQ,CAAC,KAAK,EAAE,CAAC;oBAClB,CAAC;oBACD,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,4EAA4E;oBAC5E,0EAA0E;oBAC1E,+EAA6E;oBAC7E,0EAA0E;oBAC1E,wEAAwE;oBACxE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;oBACjE,IAAI,QAAQ,EAAE,CAAC;wBACd,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;oBAC3D,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,IAAI,QAAQ;wBAAE,OAAO;gBACtB,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC3D,CAAC;QAAA,CACD,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IAAA,CACb,CAAC;AAAA,CACF","sourcesContent":["/**\n * Silence/idle watchdogs for the reliability kernel.\n *\n * A silence watchdog bounds \"running but mute\" — it never bounds total runtime,\n * so long tasks that produce output are never killed (autonomy constraint).\n */\n\nexport interface SilenceWatchdog {\n\t/** Report activity (output chunk / stream event); resets the countdown. */\n\ttouch(): void;\n\t/** Stop permanently (normal completion). Idempotent. */\n\tdisarm(): void;\n}\n\nexport interface SilenceWatchdogOptions {\n\tsilenceMs: number;\n\t/** Fired at most once, after silenceMs with no touch(). The watchdog self-disarms. */\n\tonSilence: () => void;\n}\n\nexport function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog {\n\tlet timer: NodeJS.Timeout | undefined;\n\tlet disarmed = false;\n\n\tconst arm = () => {\n\t\ttimer = setTimeout(() => {\n\t\t\tdisarmed = true;\n\t\t\ttimer = undefined;\n\t\t\topts.onSilence();\n\t\t}, opts.silenceMs);\n\t\t// Never keep the host process alive just for a watchdog.\n\t\ttimer.unref?.();\n\t};\n\n\tarm();\n\n\treturn {\n\t\ttouch(): void {\n\t\t\tif (disarmed) return;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\tarm();\n\t\t},\n\t\tdisarm(): void {\n\t\t\tdisarmed = true;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\ttimer = undefined;\n\t\t},\n\t};\n}\n\n// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------\n\nimport {\n\ttype AssistantMessage,\n\ttype AssistantMessageEvent,\n\tcreateAssistantMessageEventStream,\n} from \"@caupulican/pi-ai\";\nimport type { StreamFn } from \"../types.ts\";\n\nexport interface StreamIdleOptions {\n\t/** Max ms between events once streaming has started (user-locked default 30s). */\n\tidleMs: number;\n\t/** Max ms to wait for the FIRST event (connection/first-token allowance). */\n\tconnectMs: number;\n\t/** Fired once when a stall is detected, before the inner request is aborted. */\n\tonStall?: (info: { phase: \"connect\" | \"stream\"; elapsedMs: number }) => void;\n}\n\nexport const DEFAULT_STREAM_IDLE: StreamIdleOptions = { idleMs: 30_000, connectMs: 120_000 };\n\n/** Extracts the current AssistantMessage snapshot carried by any stream event variant. */\nfunction partialFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\n/**\n * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.\n *\n * `connectMs` bounds the wait for the first event (connection/first-token allowance);\n * once streaming starts, `idleMs` bounds the gap between subsequent events. On stall,\n * the inner request is aborted and the returned stream resolves immediately with a\n * synthetic `AssistantMessage` (`stopReason: \"error\"`, `errorMessage: \"stream stalled:\n * no events for <n>ms\"`) — the exact phrasing `classifyFailure` maps to a retryable\n * `stream_stall`, so the host's retry/failover path takes it from there.\n *\n * A caller-initiated abort (via the options `signal`) is never treated as a stall: it\n * is chained into the wrapper's own controller and the inner stream's own abort result\n * is forwarded untouched.\n */\nexport function withStreamIdleWatchdog(streamFn: StreamFn, options?: Partial<StreamIdleOptions>): StreamFn {\n\tconst opts = { ...DEFAULT_STREAM_IDLE, ...options };\n\treturn async (model, context, streamOptions) => {\n\t\tconst controller = new AbortController();\n\t\tconst callerSignal = streamOptions?.signal;\n\t\tlet callerAborted = callerSignal?.aborted ?? false;\n\t\tconst onCallerAbort = () => {\n\t\t\tcallerAborted = true;\n\t\t\tcontroller.abort(callerSignal?.reason);\n\t\t};\n\t\tif (callerAborted) controller.abort(callerSignal?.reason);\n\t\telse callerSignal?.addEventListener(\"abort\", onCallerAbort, { once: true });\n\n\t\tconst inner = await streamFn(model, context, { ...streamOptions, signal: controller.signal });\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\t// Seeded so a connect-phase stall (no event ever arrived) still has a base message\n\t\t// to report on; overwritten with the latest real snapshot once events start flowing.\n\t\tlet latest: AssistantMessage = {\n\t\t\trole: \"assistant\",\n\t\t\tcontent: [],\n\t\t\tapi: model.api,\n\t\t\tprovider: model.provider,\n\t\t\tmodel: model.id,\n\t\t\tusage: {\n\t\t\t\tinput: 0,\n\t\t\t\toutput: 0,\n\t\t\t\tcacheRead: 0,\n\t\t\t\tcacheWrite: 0,\n\t\t\t\ttotalTokens: 0,\n\t\t\t\tcost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },\n\t\t\t},\n\t\t\tstopReason: \"stop\",\n\t\t\ttimestamp: Date.now(),\n\t\t};\n\t\tlet firstEventSeen = false;\n\t\tlet stalled = false;\n\n\t\t// Emits the stall result directly (rather than after the inner loop finishes) so a\n\t\t// connection that never resolves at all still yields a result promptly — providers\n\t\t// are contractually expected to end their stream after abort, but the watchdog does\n\t\t// not depend on that to report the stall itself.\n\t\tconst stall = (phase: \"connect\" | \"stream\", elapsedMs: number) => {\n\t\t\tif (callerAborted || stalled) return;\n\t\t\tstalled = true;\n\t\t\topts.onStall?.({ phase, elapsedMs });\n\t\t\tcontroller.abort(new Error(`stream stalled: no events for ${elapsedMs}ms`));\n\t\t\tconst message: AssistantMessage = {\n\t\t\t\t...latest,\n\t\t\t\tstopReason: \"error\",\n\t\t\t\terrorMessage: `stream stalled: no events for ${elapsedMs}ms`,\n\t\t\t};\n\t\t\touter.push({ type: \"error\", reason: \"error\", error: message });\n\t\t};\n\n\t\tlet watchdog = createSilenceWatchdog({\n\t\t\tsilenceMs: opts.connectMs,\n\t\t\tonSilence: () => stall(\"connect\", opts.connectMs),\n\t\t});\n\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tif (stalled) break;\n\t\t\t\t\tif (!firstEventSeen) {\n\t\t\t\t\t\tfirstEventSeen = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\twatchdog = createSilenceWatchdog({\n\t\t\t\t\t\t\tsilenceMs: opts.idleMs,\n\t\t\t\t\t\t\tonSilence: () => stall(\"stream\", opts.idleMs),\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\twatchdog.touch();\n\t\t\t\t\t}\n\t\t\t\t\tlatest = partialFromEvent(event);\n\t\t\t\t\t// A terminal event ends the turn: disarm synchronously, in the same tick as\n\t\t\t\t\t// the push below, so no watchdog can fire after the consumer's `result()`\n\t\t\t\t\t// promise resolves — a disarm that only happened once the loop later notices\n\t\t\t\t\t// `inner` is done would race with that resolution (it runs a tick or more\n\t\t\t\t\t// later) and could fire a spurious stall on an already-finished stream.\n\t\t\t\t\tconst terminal = event.type === \"done\" || event.type === \"error\";\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminal) return;\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\twatchdog.disarm();\n\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t}\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n"]}
1
+ {"version":3,"file":"watchdogs.js","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAgBH,MAAM,UAAU,qBAAqB,CAAC,IAA4B,EAAmB;IACpF,IAAI,KAAiC,CAAC;IACtC,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;IAEtC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC;QACjB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC;YACxB,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,GAAG,SAAS,CAAC;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;QAAA,CACjB,EAAE,gBAAgB,CAAC,CAAC;QACrB,yDAAyD;QACzD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAAA,CAChB,CAAC;IAEF,GAAG,EAAE,CAAC;IAEN,OAAO;QACN,KAAK,CAAC,SAAkB,EAAQ;YAC/B,IAAI,QAAQ;gBAAE,OAAO;YACrB,IAAI,SAAS,KAAK,SAAS;gBAAE,gBAAgB,GAAG,SAAS,CAAC;YAC1D,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,GAAG,EAAE,CAAC;QAAA,CACN;QACD,MAAM,GAAS;YACd,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,GAAG,SAAS,CAAC;QAAA,CAClB;KACD,CAAC;AAAA,CACF;AAED,8EAA8E;AAE9E,OAAO,EAGN,iCAAiC,GACjC,MAAM,mBAAmB,CAAC;AAoB3B;;8FAE8F;AAC9F,MAAM,CAAC,MAAM,mBAAmB,GAAsB;IACrD,SAAS,EAAE,OAAO;IAClB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,OAAO;CACpB,CAAC;AAKF,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,KAA4B,EAAoB;IACzE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC;AAAA,CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,sBAAsB,CACrC,QAAkB,EAClB,OAAgE,EACrD;IACX,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC;QAC/C,MAAM,QAAQ,GAAG,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QACrE,MAAM,OAAO,GAA+B,EAAE,CAAC;QAC/C,IAAI,QAAQ,EAAE,CAAC;YACd,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBACtB,OAAe,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAC7B,CAAC;YACF,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,OAAO,EAAE,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,aAAa,EAAE,MAAM,CAAC;QAC3C,IAAI,aAAa,GAAG,YAAY,EAAE,OAAO,IAAI,KAAK,CAAC;QACnD,MAAM,aAAa,GAAG,GAAG,EAAE,CAAC;YAC3B,aAAa,GAAG,IAAI,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAAA,CACvC,CAAC;QACF,IAAI,aAAa;YAAE,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;;YACrD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5E,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,GAAG,aAAa,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAElD,mFAAmF;QACnF,qFAAqF;QACrF,IAAI,MAAM,GAAqB;YAC9B,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,EAAE;YACX,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,EAAE;YACf,KAAK,EAAE;gBACN,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,WAAW,EAAE,CAAC;gBACd,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;aACpE;YACD,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC;QACF,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,8EAA8E;QAC9E,4EAA4E;QAC5E,iDAAiD;QACjD,IAAI,YAAY,GAAe,SAAS,CAAC;QACzC,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,MAAM,YAAY,GAAG,CAAC,OAAyB,EAAqC,EAAE,CAAC;YACtF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,UAAU;gBACjD,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE;gBAC1C,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;QAAA,CAC9C,CAAC;QAEF,mFAAmF;QACnF,qFAAmF;QACnF,oFAAoF;QACpF,iDAAiD;QACjD,MAAM,KAAK,GAAG,CAAC,KAAiB,EAAE,SAAiB,EAAE,EAAE,CAAC;YACvD,IAAI,aAAa,IAAI,OAAO;gBAAE,OAAO;YACrC,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,iCAAiC,SAAS,OAAO,KAAK,SAAS,CAAC;YACpF,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;YACzC,MAAM,OAAO,GAAqB;gBACjC,GAAG,MAAM;gBACT,UAAU,EAAE,OAAO;gBACnB,YAAY,EAAE,WAAW;aACzB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAAA,CAC/D,CAAC;QAEF,IAAI,QAAQ,GAAG,qBAAqB,CAAC;YACpC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;SACpD,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBACjC,IAAI,OAAO;wBAAE,MAAM;oBACnB,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;oBACnC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;oBAC3B,cAAc,GAAG,KAAK,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC,cAAc,EAAE,CAAC;wBACrB,cAAc,GAAG,IAAI,CAAC;wBACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,QAAQ,GAAG,qBAAqB,CAAC;4BAChC,SAAS,EAAE,KAAK,CAAC,EAAE;4BACnB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;yBACpD,CAAC,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACP,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1B,CAAC;oBACD,4EAA4E;oBAC5E,0EAA0E;oBAC1E,+EAA6E;oBAC7E,0EAA0E;oBAC1E,wEAAwE;oBACxE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;oBACjE,IAAI,QAAQ,EAAE,CAAC;wBACd,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;oBAC3D,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,IAAI,QAAQ;wBAAE,OAAO;gBACtB,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC3D,CAAC;QAAA,CACD,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IAAA,CACb,CAAC;AAAA,CACF","sourcesContent":["/**\n * Silence/idle watchdogs for the reliability kernel.\n *\n * A silence watchdog bounds \"running but mute\" — it never bounds total runtime,\n * so long tasks that produce output are never killed (autonomy constraint).\n */\n\nexport interface SilenceWatchdog {\n\t/** Report activity (output chunk / stream event); resets the countdown.\n\t * Pass silenceMs to also change the bound for this and subsequent countdowns. */\n\ttouch(silenceMs?: number): void;\n\t/** Stop permanently (normal completion). Idempotent. */\n\tdisarm(): void;\n}\n\nexport interface SilenceWatchdogOptions {\n\tsilenceMs: number;\n\t/** Fired at most once, after silenceMs with no touch(). The watchdog self-disarms. */\n\tonSilence: () => void;\n}\n\nexport function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog {\n\tlet timer: NodeJS.Timeout | undefined;\n\tlet disarmed = false;\n\tlet currentSilenceMs = opts.silenceMs;\n\n\tconst arm = () => {\n\t\ttimer = setTimeout(() => {\n\t\t\tdisarmed = true;\n\t\t\ttimer = undefined;\n\t\t\topts.onSilence();\n\t\t}, currentSilenceMs);\n\t\t// Never keep the host process alive just for a watchdog.\n\t\ttimer.unref?.();\n\t};\n\n\tarm();\n\n\treturn {\n\t\ttouch(silenceMs?: number): void {\n\t\t\tif (disarmed) return;\n\t\t\tif (silenceMs !== undefined) currentSilenceMs = silenceMs;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\tarm();\n\t\t},\n\t\tdisarm(): void {\n\t\t\tdisarmed = true;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\ttimer = undefined;\n\t\t},\n\t};\n}\n\n// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------\n\nimport {\n\ttype AssistantMessage,\n\ttype AssistantMessageEvent,\n\tcreateAssistantMessageEventStream,\n} from \"@caupulican/pi-ai\";\nimport type { StreamFn } from \"../types.ts\";\n\nexport type StallPhase = \"connect\" | \"quiet\" | \"active\";\n\nexport interface StreamIdleOptions {\n\t/** Max ms to wait for the FIRST event (connection/first-token allowance). */\n\tconnectMs: number;\n\t/** Max ms between events while content is flowing — the latest content block is\n\t * text or toolCall. A flowing stream that goes silent this long is presumed dead. */\n\tactiveIdleMs: number;\n\t/** Max ms between events while the model is quietly working — no content blocks\n\t * yet (provider queue / prompt prefill / unstreamed reasoning) or the latest block\n\t * is thinking. Deep-thinking models and huge compaction prompts legitimately sit\n\t * here for minutes, so this bound is deliberately generous. */\n\tquietIdleMs: number;\n\t/** Fired once when a stall is detected, before the inner request is aborted. */\n\tonStall?: (info: { phase: StallPhase; elapsedMs: number }) => void;\n}\n\n/** User-locked defaults: connect 120s / active 180s / quiet 600s. The quiet bound must stay\n * below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the\n * HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */\nexport const DEFAULT_STREAM_IDLE: StreamIdleOptions = {\n\tconnectMs: 120_000,\n\tactiveIdleMs: 180_000,\n\tquietIdleMs: 600_000,\n};\n\n/** Re-resolved at the start of every request, so hosts can wire live-tunable settings. */\nexport type StreamIdleOptionsResolver = () => Partial<StreamIdleOptions>;\n\n/** Extracts the current AssistantMessage snapshot carried by any stream event variant. */\nfunction partialFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\n/**\n * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.\n *\n * Phase-aware: `connectMs` bounds the wait for the first event; after that the\n * inter-event bound adapts to what the stream is doing — `quietIdleMs` while the\n * model is quietly working (no content blocks yet, or the latest block is thinking:\n * prefill, provider queues, unstreamed reasoning) and `activeIdleMs` once content is\n * flowing (latest block is text/toolCall). This keeps detection fast where silence is\n * anomalous without killing healthy deep-thinking or compaction-sized requests.\n * No bound ever limits total runtime (autonomy constraint).\n *\n * On stall, the inner request is aborted and the returned stream resolves immediately\n * with a synthetic `AssistantMessage` (`stopReason: \"error\"`, `errorMessage: \"stream\n * stalled: no events for <n>ms (<phase> phase)\"`) — the `stream stalled` phrasing is\n * what `classifyFailure` maps to a retryable `stream_stall`, so the host's\n * retry/failover path takes it from there.\n *\n * Options may be a resolver function; it is re-invoked at the start of every request,\n * so settings changes apply without rewrapping.\n *\n * A caller-initiated abort (via the options `signal`) is never treated as a stall: it\n * is chained into the wrapper's own controller and the inner stream's own abort result\n * is forwarded untouched.\n */\nexport function withStreamIdleWatchdog(\n\tstreamFn: StreamFn,\n\toptions?: Partial<StreamIdleOptions> | StreamIdleOptionsResolver,\n): StreamFn {\n\treturn async (model, context, streamOptions) => {\n\t\tconst resolved = typeof options === \"function\" ? options() : options;\n\t\tconst cleaned: Partial<StreamIdleOptions> = {};\n\t\tif (resolved) {\n\t\t\tfor (const [key, val] of Object.entries(resolved)) {\n\t\t\t\tif (val !== undefined) {\n\t\t\t\t\t(cleaned as any)[key] = val;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tconst opts = { ...DEFAULT_STREAM_IDLE, ...cleaned };\n\n\t\tconst controller = new AbortController();\n\t\tconst callerSignal = streamOptions?.signal;\n\t\tlet callerAborted = callerSignal?.aborted ?? false;\n\t\tconst onCallerAbort = () => {\n\t\t\tcallerAborted = true;\n\t\t\tcontroller.abort(callerSignal?.reason);\n\t\t};\n\t\tif (callerAborted) controller.abort(callerSignal?.reason);\n\t\telse callerSignal?.addEventListener(\"abort\", onCallerAbort, { once: true });\n\n\t\tconst inner = await streamFn(model, context, { ...streamOptions, signal: controller.signal });\n\t\tconst outer = createAssistantMessageEventStream();\n\n\t\t// Seeded so a connect-phase stall (no event ever arrived) still has a base message\n\t\t// to report on; overwritten with the latest real snapshot once events start flowing.\n\t\tlet latest: AssistantMessage = {\n\t\t\trole: \"assistant\",\n\t\t\tcontent: [],\n\t\t\tapi: model.api,\n\t\t\tprovider: model.provider,\n\t\t\tmodel: model.id,\n\t\t\tusage: {\n\t\t\t\tinput: 0,\n\t\t\t\toutput: 0,\n\t\t\t\tcacheRead: 0,\n\t\t\t\tcacheWrite: 0,\n\t\t\t\ttotalTokens: 0,\n\t\t\t\tcost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },\n\t\t\t},\n\t\t\tstopReason: \"stop\",\n\t\t\ttimestamp: Date.now(),\n\t\t};\n\t\tlet stalled = false;\n\t\tlet firstEventSeen = false;\n\n\t\t// The idle bound adapts per event: quiet while nothing/thinking, active while\n\t\t// text/toolCall content is flowing. Mutable so the onSilence closure always\n\t\t// reports the phase/bound that actually elapsed.\n\t\tlet currentPhase: StallPhase = \"connect\";\n\t\tlet currentBoundMs = opts.connectMs;\n\t\tconst idleBoundFor = (message: AssistantMessage): { phase: StallPhase; ms: number } => {\n\t\t\tconst lastBlock = message.content[message.content.length - 1];\n\t\t\treturn !lastBlock || lastBlock.type === \"thinking\"\n\t\t\t\t? { phase: \"quiet\", ms: opts.quietIdleMs }\n\t\t\t\t: { phase: \"active\", ms: opts.activeIdleMs };\n\t\t};\n\n\t\t// Emits the stall result directly (rather than after the inner loop finishes) so a\n\t\t// connection that never resolves at all still yields a result promptly — providers\n\t\t// are contractually expected to end their stream after abort, but the watchdog does\n\t\t// not depend on that to report the stall itself.\n\t\tconst stall = (phase: StallPhase, elapsedMs: number) => {\n\t\t\tif (callerAborted || stalled) return;\n\t\t\tstalled = true;\n\t\t\topts.onStall?.({ phase, elapsedMs });\n\t\t\tconst description = `stream stalled: no events for ${elapsedMs}ms (${phase} phase)`;\n\t\t\tcontroller.abort(new Error(description));\n\t\t\tconst message: AssistantMessage = {\n\t\t\t\t...latest,\n\t\t\t\tstopReason: \"error\",\n\t\t\t\terrorMessage: description,\n\t\t\t};\n\t\t\touter.push({ type: \"error\", reason: \"error\", error: message });\n\t\t};\n\n\t\tlet watchdog = createSilenceWatchdog({\n\t\t\tsilenceMs: opts.connectMs,\n\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t});\n\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tif (stalled) break;\n\t\t\t\t\tlatest = partialFromEvent(event);\n\t\t\t\t\tconst bound = idleBoundFor(latest);\n\t\t\t\t\tcurrentPhase = bound.phase;\n\t\t\t\t\tcurrentBoundMs = bound.ms;\n\t\t\t\t\tif (!firstEventSeen) {\n\t\t\t\t\t\tfirstEventSeen = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\twatchdog = createSilenceWatchdog({\n\t\t\t\t\t\t\tsilenceMs: bound.ms,\n\t\t\t\t\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\twatchdog.touch(bound.ms);\n\t\t\t\t\t}\n\t\t\t\t\t// A terminal event ends the turn: disarm synchronously, in the same tick as\n\t\t\t\t\t// the push below, so no watchdog can fire after the consumer's `result()`\n\t\t\t\t\t// promise resolves — a disarm that only happened once the loop later notices\n\t\t\t\t\t// `inner` is done would race with that resolution (it runs a tick or more\n\t\t\t\t\t// later) and could fire a spurious stall on an already-finished stream.\n\t\t\t\t\tconst terminal = event.type === \"done\" || event.type === \"error\";\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminal) return;\n\t\t\t\t}\n\t\t\t} finally {\n\t\t\t\twatchdog.disarm();\n\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t}\n\t\t})();\n\n\t\treturn outer;\n\t};\n}\n"]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@caupulican/pi-agent-core",
3
- "version": "0.81.0",
3
+ "version": "0.81.2",
4
4
  "description": "General-purpose agent with transport abstraction, state management, and attachment support",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -33,7 +33,7 @@
33
33
  "prepublishOnly": "npm run clean && npm run build"
34
34
  },
35
35
  "dependencies": {
36
- "@caupulican/pi-ai": "^0.81.0",
36
+ "@caupulican/pi-ai": "^0.81.2",
37
37
  "ignore": "7.0.5",
38
38
  "typebox": "1.1.38",
39
39
  "yaml": "2.9.0"