@caupulican/pi-agent-core 0.93.13 → 0.93.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -18,16 +18,18 @@ export interface SilenceWatchdogOptions {
|
|
|
18
18
|
}
|
|
19
19
|
export declare function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog;
|
|
20
20
|
import type { StreamFn } from "../types.ts";
|
|
21
|
-
export type StallPhase = "connect" | "quiet" | "active";
|
|
21
|
+
export type StallPhase = "connect" | "first-progress" | "quiet" | "active";
|
|
22
22
|
export interface StreamIdleOptions {
|
|
23
23
|
/** Max ms to wait for the FIRST event (connection/first-token allowance). */
|
|
24
24
|
connectMs: number;
|
|
25
25
|
/** Max ms between events while content is flowing — the latest content block is
|
|
26
26
|
* text or toolCall. A flowing stream that goes silent this long is presumed dead. */
|
|
27
27
|
activeIdleMs: number;
|
|
28
|
+
/** Max ms after transport confirmation until the first non-empty model delta.
|
|
29
|
+
* Headers and structural start frames prove connectivity, not generation progress. */
|
|
30
|
+
firstProgressMs: number;
|
|
28
31
|
/** Max ms between events while the model is quietly working — no content blocks
|
|
29
|
-
*
|
|
30
|
-
* is thinking. Deep-thinking models and huge compaction prompts legitimately sit
|
|
32
|
+
* after it has emitted non-empty thinking. Deep-thinking models legitimately sit
|
|
31
33
|
* here for minutes, so this bound is deliberately generous. */
|
|
32
34
|
quietIdleMs: number;
|
|
33
35
|
/** Fired once when a stall is detected, before the inner request is aborted. */
|
|
@@ -36,7 +38,7 @@ export interface StreamIdleOptions {
|
|
|
36
38
|
elapsedMs: number;
|
|
37
39
|
}) => void;
|
|
38
40
|
}
|
|
39
|
-
/**
|
|
41
|
+
/** Defaults: connect 120s / first progress 120s / active 180s / quiet 600s. The quiet bound must stay
|
|
40
42
|
* below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the
|
|
41
43
|
* HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */
|
|
42
44
|
export declare const DEFAULT_STREAM_IDLE: StreamIdleOptions;
|
|
@@ -45,11 +47,10 @@ export type StreamIdleOptionsResolver = (...args: Parameters<StreamFn>) => Parti
|
|
|
45
47
|
/**
|
|
46
48
|
* Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.
|
|
47
49
|
*
|
|
48
|
-
* Phase-aware: `connectMs` bounds the wait for
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* flowing (latest block is text/toolCall). This keeps detection fast where silence is
|
|
50
|
+
* Phase-aware: `connectMs` bounds the wait for transport confirmation,
|
|
51
|
+
* `firstProgressMs` bounds headers and structural frames that contain no model output,
|
|
52
|
+
* `quietIdleMs` covers silence after non-empty thinking, and `activeIdleMs` covers
|
|
53
|
+
* silence after text or tool-call output. This keeps detection fast where silence is
|
|
53
54
|
* anomalous without killing healthy deep-thinking or compaction-sized requests.
|
|
54
55
|
* No bound ever limits total runtime (autonomy constraint).
|
|
55
56
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"watchdogs.d.ts","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,eAAe;IAC/B;sFACkF;IAClF,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,wDAAwD;IACxD,MAAM,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,SAAS,EAAE,MAAM,IAAI,CAAC;CACtB;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,GAAG,eAAe,CA8BnF;AAOD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"watchdogs.d.ts","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,eAAe;IAC/B;sFACkF;IAClF,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,wDAAwD;IACxD,MAAM,IAAI,IAAI,CAAC;CACf;AAED,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,sFAAsF;IACtF,SAAS,EAAE,MAAM,IAAI,CAAC;CACtB;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,GAAG,eAAe,CA8BnF;AAOD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,OAAO,GAAG,QAAQ,CAAC;AAE3E,MAAM,WAAW,iBAAiB;IACjC,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;IAClB;0FACsF;IACtF,YAAY,EAAE,MAAM,CAAC;IACrB;2FACuF;IACvF,eAAe,EAAE,MAAM,CAAC;IACxB;;oEAEgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACnE;AAED;;8FAE8F;AAC9F,eAAO,MAAM,mBAAmB,EAAE,iBAKjC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,MAAM,yBAAyB,GAAG,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAStG;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,yBAAyB,GAC9D,QAAQ,CAmNV"}
|
|
@@ -39,11 +39,12 @@ export function createSilenceWatchdog(opts) {
|
|
|
39
39
|
// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------
|
|
40
40
|
import { createAssistantMessageEventStream } from "@caupulican/pi-ai/event-stream";
|
|
41
41
|
import { createEmptyUsage } from "@caupulican/pi-ai/usage";
|
|
42
|
-
/**
|
|
42
|
+
/** Defaults: connect 120s / first progress 120s / active 180s / quiet 600s. The quiet bound must stay
|
|
43
43
|
* below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the
|
|
44
44
|
* HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */
|
|
45
45
|
export const DEFAULT_STREAM_IDLE = {
|
|
46
46
|
connectMs: 120_000,
|
|
47
|
+
firstProgressMs: 120_000,
|
|
47
48
|
activeIdleMs: 180_000,
|
|
48
49
|
quietIdleMs: 600_000,
|
|
49
50
|
};
|
|
@@ -58,11 +59,10 @@ function partialFromEvent(event) {
|
|
|
58
59
|
/**
|
|
59
60
|
* Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.
|
|
60
61
|
*
|
|
61
|
-
* Phase-aware: `connectMs` bounds the wait for
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
* flowing (latest block is text/toolCall). This keeps detection fast where silence is
|
|
62
|
+
* Phase-aware: `connectMs` bounds the wait for transport confirmation,
|
|
63
|
+
* `firstProgressMs` bounds headers and structural frames that contain no model output,
|
|
64
|
+
* `quietIdleMs` covers silence after non-empty thinking, and `activeIdleMs` covers
|
|
65
|
+
* silence after text or tool-call output. This keeps detection fast where silence is
|
|
66
66
|
* anomalous without killing healthy deep-thinking or compaction-sized requests.
|
|
67
67
|
* No bound ever limits total runtime (autonomy constraint).
|
|
68
68
|
*
|
|
@@ -120,7 +120,7 @@ export function withStreamIdleWatchdog(streamFn, options) {
|
|
|
120
120
|
timestamp: Date.now(),
|
|
121
121
|
};
|
|
122
122
|
let stalled = false;
|
|
123
|
-
let
|
|
123
|
+
let meaningfulProgressSeen = false;
|
|
124
124
|
let transportConfirmed = false;
|
|
125
125
|
let streamSetupSettled = false;
|
|
126
126
|
let terminalPushed = false;
|
|
@@ -175,17 +175,17 @@ export function withStreamIdleWatchdog(streamFn, options) {
|
|
|
175
175
|
settleReady();
|
|
176
176
|
callerSignal?.removeEventListener("abort", onCallerAbort);
|
|
177
177
|
};
|
|
178
|
-
|
|
178
|
+
const watchdog = createSilenceWatchdog({
|
|
179
179
|
silenceMs: opts.connectMs,
|
|
180
180
|
onSilence: () => stall(currentPhase, currentBoundMs),
|
|
181
181
|
});
|
|
182
182
|
const markTransportConfirmed = () => {
|
|
183
|
-
if (callerAborted || stalled ||
|
|
183
|
+
if (callerAborted || stalled || meaningfulProgressSeen || transportConfirmed)
|
|
184
184
|
return;
|
|
185
185
|
transportConfirmed = true;
|
|
186
|
-
currentPhase = "
|
|
187
|
-
currentBoundMs = opts.
|
|
188
|
-
watchdog.touch(opts.
|
|
186
|
+
currentPhase = "first-progress";
|
|
187
|
+
currentBoundMs = opts.firstProgressMs;
|
|
188
|
+
watchdog.touch(opts.firstProgressMs);
|
|
189
189
|
};
|
|
190
190
|
const originalOnResponse = streamOptions?.onResponse;
|
|
191
191
|
const pushSetupAbort = () => {
|
|
@@ -247,18 +247,16 @@ export function withStreamIdleWatchdog(streamFn, options) {
|
|
|
247
247
|
if (stalled)
|
|
248
248
|
break;
|
|
249
249
|
latest = partialFromEvent(event);
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
if (!
|
|
254
|
-
|
|
255
|
-
watchdog.disarm();
|
|
256
|
-
watchdog = createSilenceWatchdog({
|
|
257
|
-
silenceMs: bound.ms,
|
|
258
|
-
onSilence: () => stall(currentPhase, currentBoundMs),
|
|
259
|
-
});
|
|
250
|
+
const meaningfulProgress = eventHasMeaningfulProgress(event);
|
|
251
|
+
if (meaningfulProgress)
|
|
252
|
+
meaningfulProgressSeen = true;
|
|
253
|
+
if (!transportConfirmed && !meaningfulProgress) {
|
|
254
|
+
markTransportConfirmed();
|
|
260
255
|
}
|
|
261
|
-
else {
|
|
256
|
+
else if (meaningfulProgressSeen) {
|
|
257
|
+
const bound = idleBoundFor(latest);
|
|
258
|
+
currentPhase = bound.phase;
|
|
259
|
+
currentBoundMs = bound.ms;
|
|
262
260
|
watchdog.touch(bound.ms);
|
|
263
261
|
}
|
|
264
262
|
// A terminal event ends the turn: disarm synchronously, in the same tick as
|
|
@@ -300,4 +298,19 @@ export function withStreamIdleWatchdog(streamFn, options) {
|
|
|
300
298
|
return ready;
|
|
301
299
|
};
|
|
302
300
|
}
|
|
301
|
+
function eventHasMeaningfulProgress(event) {
|
|
302
|
+
switch (event.type) {
|
|
303
|
+
case "text_delta":
|
|
304
|
+
case "thinking_delta":
|
|
305
|
+
case "toolcall_delta":
|
|
306
|
+
return event.delta.length > 0;
|
|
307
|
+
case "text_end":
|
|
308
|
+
case "thinking_end":
|
|
309
|
+
return event.content.length > 0;
|
|
310
|
+
case "toolcall_end":
|
|
311
|
+
return true;
|
|
312
|
+
default:
|
|
313
|
+
return false;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
303
316
|
//# sourceMappingURL=watchdogs.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"watchdogs.js","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAgBH,MAAM,UAAU,qBAAqB,CAAC,IAA4B;IACjE,IAAI,KAAiC,CAAC;IACtC,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;IAEtC,MAAM,GAAG,GAAG,GAAG,EAAE;QAChB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACvB,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,GAAG,SAAS,CAAC;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrB,yDAAyD;QACzD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACjB,CAAC,CAAC;IAEF,GAAG,EAAE,CAAC;IAEN,OAAO;QACN,KAAK,CAAC,SAAkB;YACvB,IAAI,QAAQ;gBAAE,OAAO;YACrB,IAAI,SAAS,KAAK,SAAS;gBAAE,gBAAgB,GAAG,SAAS,CAAC;YAC1D,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,GAAG,EAAE,CAAC;QACP,CAAC;QACD,MAAM;YACL,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,GAAG,SAAS,CAAC;QACnB,CAAC;KACD,CAAC;AACH,CAAC;AAED,8EAA8E;AAE9E,OAAO,EAAE,iCAAiC,EAAE,MAAM,gCAAgC,CAAC;AAEnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAoB3D;;8FAE8F;AAC9F,MAAM,CAAC,MAAM,mBAAmB,GAAsB;IACrD,SAAS,EAAE,OAAO;IAClB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,OAAO;CACpB,CAAC;AAKF,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,KAA4B;IACrD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC;AACtB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,sBAAsB,CACrC,QAAkB,EAClB,OAAgE;IAEhE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;QACxC,MAAM,QAAQ,GAAG,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAClG,MAAM,OAAO,GAA+B,EAAE,CAAC;QAC/C,IAAI,QAAQ,EAAE,CAAC;YACd,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBACtB,OAAe,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAC7B,CAAC;YACF,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,OAAO,EAAE,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,aAAa,EAAE,MAAM,CAAC;QAC3C,IAAI,aAAa,GAAG,YAAY,EAAE,OAAO,IAAI,KAAK,CAAC;QACnD,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAClD,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,YAAY,GAAmC,GAAG,EAAE,GAAE,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,OAAO,CAAe,CAAC,OAAO,EAAE,EAAE;YACnD,YAAY,GAAG,OAAO,CAAC;QACxB,CAAC,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,GAAG,EAAE;YACxB,IAAI,YAAY;gBAAE,OAAO;YACzB,YAAY,GAAG,IAAI,CAAC;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC,CAAC;QAEF,mFAAmF;QACnF,qFAAqF;QACrF,IAAI,MAAM,GAAqB;YAC9B,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,EAAE;YACX,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,EAAE;YACf,KAAK,EAAE,gBAAgB,EAAE;YACzB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC;QACF,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,MAAM,WAAW,GAAG,CAAC,KAAe,EAAE,EAAE;YACvC,IAAI,cAAc,IAAI,OAAO;gBAAE,OAAO;YACtC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YACvD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/G,cAAc,GAAG,IAAI,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE;oBACN,GAAG,MAAM;oBACT,UAAU;oBACV,YAAY,EAAE,aAAa;wBAC1B,CAAC,CAAC,uCAAuC,MAAM,EAAE;wBACjD,CAAC,CAAC,sCAAsC,MAAM,EAAE;iBACjD;aACD,CAAC,CAAC;YACH,WAAW,EAAE,CAAC;QACf,CAAC,CAAC;QAEF,8EAA8E;QAC9E,4EAA4E;QAC5E,iDAAiD;QACjD,IAAI,YAAY,GAAe,SAAS,CAAC;QACzC,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,MAAM,YAAY,GAAG,CAAC,OAAyB,EAAqC,EAAE;YACrF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,UAAU;gBACjD,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE;gBAC1C,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/C,CAAC,CAAC;QAEF,mFAAmF;QACnF,mFAAmF;QACnF,oFAAoF;QACpF,iDAAiD;QACjD,MAAM,KAAK,GAAG,CAAC,KAAiB,EAAE,SAAiB,EAAE,EAAE;YACtD,IAAI,aAAa,IAAI,OAAO,IAAI,cAAc;gBAAE,OAAO;YACvD,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,iCAAiC,SAAS,OAAO,KAAK,SAAS,CAAC;YACpF,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;YACzC,MAAM,OAAO,GAAqB;gBACjC,GAAG,MAAM;gBACT,UAAU,EAAE,OAAO;gBACnB,YAAY,EAAE,WAAW;aACzB,CAAC;YACF,cAAc,GAAG,IAAI,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC/D,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,CAAC,CAAC;QAEF,IAAI,QAAQ,GAAG,qBAAqB,CAAC;YACpC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;SACpD,CAAC,CAAC;QACH,MAAM,sBAAsB,GAAG,GAAG,EAAE;YACnC,IAAI,aAAa,IAAI,OAAO,IAAI,cAAc,IAAI,kBAAkB;gBAAE,OAAO;YAC7E,kBAAkB,GAAG,IAAI,CAAC;YAC1B,YAAY,GAAG,OAAO,CAAC;YACvB,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC;YAClC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClC,CAAC,CAAC;QACF,MAAM,kBAAkB,GAAG,aAAa,EAAE,UAAU,CAAC;QACrD,MAAM,cAAc,GAAG,GAAG,EAAE;YAC3B,IAAI,kBAAkB,IAAI,cAAc,IAAI,OAAO;gBAAE,OAAO;YAC5D,cAAc,GAAG,IAAI,CAAC;YACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE;oBACN,GAAG,MAAM;oBACT,UAAU,EAAE,SAAS;oBACrB,YAAY,EAAE,0DAA0D;iBACxE;aACD,CAAC,CAAC;YACH,WAAW,EAAE,CAAC;QACf,CAAC,CAAC;QACF,MAAM,aAAa,GAAG,GAAG,EAAE;YAC1B,aAAa,GAAG,IAAI,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACvC,cAAc,EAAE,CAAC;QAClB,CAAC,CAAC;QACF,IAAI,aAAa,EAAE,CAAC;YACnB,aAAa,EAAE,CAAC;YAChB,OAAO,KAAK,CAAC;QACd,CAAC;QACD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvE,KAAK,CAAC,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACJ,IAAI,KAAoC,CAAC;gBACzC,IAAI,CAAC;oBACJ,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;wBACtC,GAAG,aAAa;wBAChB,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,UAAU,EAAE,KAAK,EAAE,QAA0B,EAAE,aAAyB,EAAE,EAAE;4BAC3E,sBAAsB,EAAE,CAAC;4BACzB,MAAM,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;wBACrD,CAAC;qBACD,CAAC,CAAC;oBACH,kBAAkB,GAAG,IAAI,CAAC;oBAC1B,WAAW,EAAE,CAAC;gBACf,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,kBAAkB,GAAG,IAAI,CAAC;oBAC1B,IAAI,cAAc,IAAI,OAAO;wBAAE,OAAO;oBACtC,QAAQ,CAAC,MAAM,EAAE,CAAC;oBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;oBAC1D,WAAW,CAAC,KAAK,CAAC,CAAC;oBACnB,OAAO;gBACR,CAAC;gBAED,+EAA+E;gBAC/E,8EAA8E;gBAC9E,IAAI,cAAc,IAAI,OAAO,IAAI,aAAa;oBAAE,OAAO;gBACvD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBACjC,IAAI,OAAO;wBAAE,MAAM;oBACnB,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;oBACnC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;oBAC3B,cAAc,GAAG,KAAK,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC,cAAc,EAAE,CAAC;wBACrB,cAAc,GAAG,IAAI,CAAC;wBACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,QAAQ,GAAG,qBAAqB,CAAC;4BAChC,SAAS,EAAE,KAAK,CAAC,EAAE;4BACnB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;yBACpD,CAAC,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACP,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1B,CAAC;oBACD,4EAA4E;oBAC5E,0EAA0E;oBAC1E,6EAA6E;oBAC7E,0EAA0E;oBAC1E,wEAAwE;oBACxE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;oBACjE,IAAI,QAAQ,EAAE,CAAC;wBACd,cAAc,GAAG,IAAI,CAAC;wBACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;oBAC3D,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,IAAI,QAAQ;wBAAE,OAAO;gBACtB,CAAC;gBACD,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;oBACvD,MAAM,WAAW,GAAG,aAAa;wBAChC,CAAC,CAAC,sCAAsC;wBACxC,CAAC,CAAC,qCAAqC,CAAC;oBACzC,cAAc,GAAG,IAAI,CAAC;oBACtB,KAAK,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE,UAAU;wBAClB,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE;qBAC3D,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,WAAW,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;oBAAS,CAAC;gBACV,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC3D,CAAC;QACF,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IACd,CAAC,CAAC;AACH,CAAC","sourcesContent":["/**\n * Silence/idle watchdogs for the reliability kernel.\n *\n * A silence watchdog bounds \"running but mute\" — it never bounds total runtime,\n * so long tasks that produce output are never killed (autonomy constraint).\n */\n\nexport interface SilenceWatchdog {\n\t/** Report activity (output chunk / stream event); resets the countdown.\n\t * Pass silenceMs to also change the bound for this and subsequent countdowns. */\n\ttouch(silenceMs?: number): void;\n\t/** Stop permanently (normal completion). Idempotent. */\n\tdisarm(): void;\n}\n\nexport interface SilenceWatchdogOptions {\n\tsilenceMs: number;\n\t/** Fired at most once, after silenceMs with no touch(). The watchdog self-disarms. */\n\tonSilence: () => void;\n}\n\nexport function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog {\n\tlet timer: NodeJS.Timeout | undefined;\n\tlet disarmed = false;\n\tlet currentSilenceMs = opts.silenceMs;\n\n\tconst arm = () => {\n\t\ttimer = setTimeout(() => {\n\t\t\tdisarmed = true;\n\t\t\ttimer = undefined;\n\t\t\topts.onSilence();\n\t\t}, currentSilenceMs);\n\t\t// Never keep the host process alive just for a watchdog.\n\t\ttimer.unref?.();\n\t};\n\n\tarm();\n\n\treturn {\n\t\ttouch(silenceMs?: number): void {\n\t\t\tif (disarmed) return;\n\t\t\tif (silenceMs !== undefined) currentSilenceMs = silenceMs;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\tarm();\n\t\t},\n\t\tdisarm(): void {\n\t\t\tdisarmed = true;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\ttimer = undefined;\n\t\t},\n\t};\n}\n\n// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------\n\nimport { createAssistantMessageEventStream } from \"@caupulican/pi-ai/event-stream\";\nimport type { Api, AssistantMessage, AssistantMessageEvent, Model, ProviderResponse } from \"@caupulican/pi-ai/types\";\nimport { createEmptyUsage } from \"@caupulican/pi-ai/usage\";\nimport type { StreamFn } from \"../types.ts\";\n\nexport type StallPhase = \"connect\" | \"quiet\" | \"active\";\n\nexport interface StreamIdleOptions {\n\t/** Max ms to wait for the FIRST event (connection/first-token allowance). */\n\tconnectMs: number;\n\t/** Max ms between events while content is flowing — the latest content block is\n\t * text or toolCall. A flowing stream that goes silent this long is presumed dead. */\n\tactiveIdleMs: number;\n\t/** Max ms between events while the model is quietly working — no content blocks\n\t * yet (provider queue / prompt prefill / unstreamed reasoning) or the latest block\n\t * is thinking. Deep-thinking models and huge compaction prompts legitimately sit\n\t * here for minutes, so this bound is deliberately generous. */\n\tquietIdleMs: number;\n\t/** Fired once when a stall is detected, before the inner request is aborted. */\n\tonStall?: (info: { phase: StallPhase; elapsedMs: number }) => void;\n}\n\n/** User-locked defaults: connect 120s / active 180s / quiet 600s. The quiet bound must stay\n * below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the\n * HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */\nexport const DEFAULT_STREAM_IDLE: StreamIdleOptions = {\n\tconnectMs: 120_000,\n\tactiveIdleMs: 180_000,\n\tquietIdleMs: 600_000,\n};\n\n/** Re-resolved at the start of every request, so hosts can wire live-tunable settings. */\nexport type StreamIdleOptionsResolver = (...args: Parameters<StreamFn>) => Partial<StreamIdleOptions>;\n\n/** Extracts the current AssistantMessage snapshot carried by any stream event variant. */\nfunction partialFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\n/**\n * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.\n *\n * Phase-aware: `connectMs` bounds the wait for the first event; after that the\n * inter-event bound adapts to what the stream is doing — `quietIdleMs` while the\n * model is quietly working (no content blocks yet, or the latest block is thinking:\n * prefill, provider queues, unstreamed reasoning) and `activeIdleMs` once content is\n * flowing (latest block is text/toolCall). This keeps detection fast where silence is\n * anomalous without killing healthy deep-thinking or compaction-sized requests.\n * No bound ever limits total runtime (autonomy constraint).\n *\n * On stall, the inner request is aborted and the returned stream resolves immediately\n * with a synthetic `AssistantMessage` (`stopReason: \"error\"`, `errorMessage: \"stream\n * stalled: no events for <n>ms (<phase> phase)\"`) — the `stream stalled` phrasing is\n * what `classifyFailure` maps to a retryable `stream_stall`, so the host's\n * retry/failover path takes it from there.\n *\n * Options may be a resolver function; it is re-invoked at the start of every request,\n * so settings changes apply without rewrapping.\n *\n * A caller-initiated abort (via the options `signal`) is never treated as a stall: it\n * is chained into the wrapper's own controller. The inner stream's abort result is\n * forwarded untouched when it provides one; if it just ends, the wrapper synthesizes\n * an aborted terminal event so the returned stream always settles.\n */\nexport function withStreamIdleWatchdog(\n\tstreamFn: StreamFn,\n\toptions?: Partial<StreamIdleOptions> | StreamIdleOptionsResolver,\n): StreamFn {\n\treturn (model, context, streamOptions) => {\n\t\tconst resolved = typeof options === \"function\" ? options(model, context, streamOptions) : options;\n\t\tconst cleaned: Partial<StreamIdleOptions> = {};\n\t\tif (resolved) {\n\t\t\tfor (const [key, val] of Object.entries(resolved)) {\n\t\t\t\tif (val !== undefined) {\n\t\t\t\t\t(cleaned as any)[key] = val;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tconst opts = { ...DEFAULT_STREAM_IDLE, ...cleaned };\n\n\t\tconst controller = new AbortController();\n\t\tconst callerSignal = streamOptions?.signal;\n\t\tlet callerAborted = callerSignal?.aborted ?? false;\n\t\tconst outer = createAssistantMessageEventStream();\n\t\tlet readySettled = false;\n\t\tlet resolveReady: (stream: typeof outer) => void = () => {};\n\t\tconst ready = new Promise<typeof outer>((resolve) => {\n\t\t\tresolveReady = resolve;\n\t\t});\n\t\tconst settleReady = () => {\n\t\t\tif (readySettled) return;\n\t\t\treadySettled = true;\n\t\t\tresolveReady(outer);\n\t\t};\n\n\t\t// Seeded so a connect-phase stall (no event ever arrived) still has a base message\n\t\t// to report on; overwritten with the latest real snapshot once events start flowing.\n\t\tlet latest: AssistantMessage = {\n\t\t\trole: \"assistant\",\n\t\t\tcontent: [],\n\t\t\tapi: model.api,\n\t\t\tprovider: model.provider,\n\t\t\tmodel: model.id,\n\t\t\tusage: createEmptyUsage(),\n\t\t\tstopReason: \"stop\",\n\t\t\ttimestamp: Date.now(),\n\t\t};\n\t\tlet stalled = false;\n\t\tlet firstEventSeen = false;\n\t\tlet transportConfirmed = false;\n\t\tlet streamSetupSettled = false;\n\t\tlet terminalPushed = false;\n\t\tconst pushFailure = (error?: unknown) => {\n\t\t\tif (terminalPushed || stalled) return;\n\t\t\tconst stopReason = callerAborted ? \"aborted\" : \"error\";\n\t\t\tconst detail = error instanceof Error ? `: ${error.message}` : error === undefined ? \"\" : `: ${String(error)}`;\n\t\t\tterminalPushed = true;\n\t\t\touter.push({\n\t\t\t\ttype: \"error\",\n\t\t\t\treason: stopReason,\n\t\t\t\terror: {\n\t\t\t\t\t...latest,\n\t\t\t\t\tstopReason,\n\t\t\t\t\terrorMessage: callerAborted\n\t\t\t\t\t\t? `stream aborted before terminal event${detail}`\n\t\t\t\t\t\t: `stream ended without terminal event${detail}`,\n\t\t\t\t},\n\t\t\t});\n\t\t\tsettleReady();\n\t\t};\n\n\t\t// The idle bound adapts per event: quiet while nothing/thinking, active while\n\t\t// text/toolCall content is flowing. Mutable so the onSilence closure always\n\t\t// reports the phase/bound that actually elapsed.\n\t\tlet currentPhase: StallPhase = \"connect\";\n\t\tlet currentBoundMs = opts.connectMs;\n\t\tconst idleBoundFor = (message: AssistantMessage): { phase: StallPhase; ms: number } => {\n\t\t\tconst lastBlock = message.content[message.content.length - 1];\n\t\t\treturn !lastBlock || lastBlock.type === \"thinking\"\n\t\t\t\t? { phase: \"quiet\", ms: opts.quietIdleMs }\n\t\t\t\t: { phase: \"active\", ms: opts.activeIdleMs };\n\t\t};\n\n\t\t// Emits the stall result directly (rather than after the inner loop finishes) so a\n\t\t// connection that never resolves at all still yields a result promptly — providers\n\t\t// are contractually expected to end their stream after abort, but the watchdog does\n\t\t// not depend on that to report the stall itself.\n\t\tconst stall = (phase: StallPhase, elapsedMs: number) => {\n\t\t\tif (callerAborted || stalled || terminalPushed) return;\n\t\t\tstalled = true;\n\t\t\topts.onStall?.({ phase, elapsedMs });\n\t\t\tconst description = `stream stalled: no events for ${elapsedMs}ms (${phase} phase)`;\n\t\t\tcontroller.abort(new Error(description));\n\t\t\tconst message: AssistantMessage = {\n\t\t\t\t...latest,\n\t\t\t\tstopReason: \"error\",\n\t\t\t\terrorMessage: description,\n\t\t\t};\n\t\t\tterminalPushed = true;\n\t\t\touter.push({ type: \"error\", reason: \"error\", error: message });\n\t\t\tsettleReady();\n\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t};\n\n\t\tlet watchdog = createSilenceWatchdog({\n\t\t\tsilenceMs: opts.connectMs,\n\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t});\n\t\tconst markTransportConfirmed = () => {\n\t\t\tif (callerAborted || stalled || firstEventSeen || transportConfirmed) return;\n\t\t\ttransportConfirmed = true;\n\t\t\tcurrentPhase = \"quiet\";\n\t\t\tcurrentBoundMs = opts.quietIdleMs;\n\t\t\twatchdog.touch(opts.quietIdleMs);\n\t\t};\n\t\tconst originalOnResponse = streamOptions?.onResponse;\n\t\tconst pushSetupAbort = () => {\n\t\t\tif (streamSetupSettled || terminalPushed || stalled) return;\n\t\t\tterminalPushed = true;\n\t\t\twatchdog.disarm();\n\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\touter.push({\n\t\t\t\ttype: \"error\",\n\t\t\t\treason: \"aborted\",\n\t\t\t\terror: {\n\t\t\t\t\t...latest,\n\t\t\t\t\tstopReason: \"aborted\",\n\t\t\t\t\terrorMessage: \"stream aborted before terminal event during stream setup\",\n\t\t\t\t},\n\t\t\t});\n\t\t\tsettleReady();\n\t\t};\n\t\tconst onCallerAbort = () => {\n\t\t\tcallerAborted = true;\n\t\t\tcontroller.abort(callerSignal?.reason);\n\t\t\tpushSetupAbort();\n\t\t};\n\t\tif (callerAborted) {\n\t\t\tonCallerAbort();\n\t\t\treturn ready;\n\t\t}\n\t\tcallerSignal?.addEventListener(\"abort\", onCallerAbort, { once: true });\n\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tlet inner: Awaited<ReturnType<StreamFn>>;\n\t\t\t\ttry {\n\t\t\t\t\tinner = await streamFn(model, context, {\n\t\t\t\t\t\t...streamOptions,\n\t\t\t\t\t\tsignal: controller.signal,\n\t\t\t\t\t\tonResponse: async (response: ProviderResponse, responseModel: Model<Api>) => {\n\t\t\t\t\t\t\tmarkTransportConfirmed();\n\t\t\t\t\t\t\tawait originalOnResponse?.(response, responseModel);\n\t\t\t\t\t\t},\n\t\t\t\t\t});\n\t\t\t\t\tstreamSetupSettled = true;\n\t\t\t\t\tsettleReady();\n\t\t\t\t} catch (error) {\n\t\t\t\t\tstreamSetupSettled = true;\n\t\t\t\t\tif (terminalPushed || stalled) return;\n\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\tpushFailure(error);\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\t// A setup promise may resolve after a connect stall or caller abort. The outer\n\t\t\t\t// stream is already terminal in that case, so never attach a late event pump.\n\t\t\t\tif (terminalPushed || stalled || callerAborted) return;\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tif (stalled) break;\n\t\t\t\t\tlatest = partialFromEvent(event);\n\t\t\t\t\tconst bound = idleBoundFor(latest);\n\t\t\t\t\tcurrentPhase = bound.phase;\n\t\t\t\t\tcurrentBoundMs = bound.ms;\n\t\t\t\t\tif (!firstEventSeen) {\n\t\t\t\t\t\tfirstEventSeen = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\twatchdog = createSilenceWatchdog({\n\t\t\t\t\t\t\tsilenceMs: bound.ms,\n\t\t\t\t\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t\t\t\t\t});\n\t\t\t\t\t} else {\n\t\t\t\t\t\twatchdog.touch(bound.ms);\n\t\t\t\t\t}\n\t\t\t\t\t// A terminal event ends the turn: disarm synchronously, in the same tick as\n\t\t\t\t\t// the push below, so no watchdog can fire after the consumer's `result()`\n\t\t\t\t\t// promise resolves — a disarm that only happened once the loop later notices\n\t\t\t\t\t// `inner` is done would race with that resolution (it runs a tick or more\n\t\t\t\t\t// later) and could fire a spurious stall on an already-finished stream.\n\t\t\t\t\tconst terminal = event.type === \"done\" || event.type === \"error\";\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminal) return;\n\t\t\t\t}\n\t\t\t\tif (!terminalPushed && !stalled) {\n\t\t\t\t\tconst stopReason = callerAborted ? \"aborted\" : \"error\";\n\t\t\t\t\tconst description = callerAborted\n\t\t\t\t\t\t? \"stream aborted before terminal event\"\n\t\t\t\t\t\t: \"stream ended without terminal event\";\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\touter.push({\n\t\t\t\t\t\ttype: \"error\",\n\t\t\t\t\t\treason: stopReason,\n\t\t\t\t\t\terror: { ...latest, stopReason, errorMessage: description },\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t} catch (error) {\n\t\t\t\tpushFailure(error);\n\t\t\t} finally {\n\t\t\t\twatchdog.disarm();\n\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t}\n\t\t})();\n\n\t\treturn ready;\n\t};\n}\n"]}
|
|
1
|
+
{"version":3,"file":"watchdogs.js","sourceRoot":"","sources":["../../src/reliability/watchdogs.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAgBH,MAAM,UAAU,qBAAqB,CAAC,IAA4B;IACjE,IAAI,KAAiC,CAAC;IACtC,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC;IAEtC,MAAM,GAAG,GAAG,GAAG,EAAE;QAChB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACvB,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,GAAG,SAAS,CAAC;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACrB,yDAAyD;QACzD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACjB,CAAC,CAAC;IAEF,GAAG,EAAE,CAAC;IAEN,OAAO;QACN,KAAK,CAAC,SAAkB;YACvB,IAAI,QAAQ;gBAAE,OAAO;YACrB,IAAI,SAAS,KAAK,SAAS;gBAAE,gBAAgB,GAAG,SAAS,CAAC;YAC1D,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,GAAG,EAAE,CAAC;QACP,CAAC;QACD,MAAM;YACL,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,KAAK,GAAG,SAAS,CAAC;QACnB,CAAC;KACD,CAAC;AACH,CAAC;AAED,8EAA8E;AAE9E,OAAO,EAAE,iCAAiC,EAAE,MAAM,gCAAgC,CAAC;AAEnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAsB3D;;8FAE8F;AAC9F,MAAM,CAAC,MAAM,mBAAmB,GAAsB;IACrD,SAAS,EAAE,OAAO;IAClB,eAAe,EAAE,OAAO;IACxB,YAAY,EAAE,OAAO;IACrB,WAAW,EAAE,OAAO;CACpB,CAAC;AAKF,0FAA0F;AAC1F,SAAS,gBAAgB,CAAC,KAA4B;IACrD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC;AACtB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,sBAAsB,CACrC,QAAkB,EAClB,OAAgE;IAEhE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE;QACxC,MAAM,QAAQ,GAAG,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAClG,MAAM,OAAO,GAA+B,EAAE,CAAC;QAC/C,IAAI,QAAQ,EAAE,CAAC;YACd,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBACtB,OAAe,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBAC7B,CAAC;YACF,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,OAAO,EAAE,CAAC;QAEpD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,aAAa,EAAE,MAAM,CAAC;QAC3C,IAAI,aAAa,GAAG,YAAY,EAAE,OAAO,IAAI,KAAK,CAAC;QACnD,MAAM,KAAK,GAAG,iCAAiC,EAAE,CAAC;QAClD,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,YAAY,GAAmC,GAAG,EAAE,GAAE,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,OAAO,CAAe,CAAC,OAAO,EAAE,EAAE;YACnD,YAAY,GAAG,OAAO,CAAC;QACxB,CAAC,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,GAAG,EAAE;YACxB,IAAI,YAAY;gBAAE,OAAO;YACzB,YAAY,GAAG,IAAI,CAAC;YACpB,YAAY,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC,CAAC;QAEF,mFAAmF;QACnF,qFAAqF;QACrF,IAAI,MAAM,GAAqB;YAC9B,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,EAAE;YACX,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,KAAK,EAAE,KAAK,CAAC,EAAE;YACf,KAAK,EAAE,gBAAgB,EAAE;YACzB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC;QACF,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,sBAAsB,GAAG,KAAK,CAAC;QACnC,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,MAAM,WAAW,GAAG,CAAC,KAAe,EAAE,EAAE;YACvC,IAAI,cAAc,IAAI,OAAO;gBAAE,OAAO;YACtC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;YACvD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/G,cAAc,GAAG,IAAI,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE;oBACN,GAAG,MAAM;oBACT,UAAU;oBACV,YAAY,EAAE,aAAa;wBAC1B,CAAC,CAAC,uCAAuC,MAAM,EAAE;wBACjD,CAAC,CAAC,sCAAsC,MAAM,EAAE;iBACjD;aACD,CAAC,CAAC;YACH,WAAW,EAAE,CAAC;QACf,CAAC,CAAC;QAEF,8EAA8E;QAC9E,4EAA4E;QAC5E,iDAAiD;QACjD,IAAI,YAAY,GAAe,SAAS,CAAC;QACzC,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,MAAM,YAAY,GAAG,CAAC,OAAyB,EAAqC,EAAE;YACrF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,KAAK,UAAU;gBACjD,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE;gBAC1C,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/C,CAAC,CAAC;QAEF,mFAAmF;QACnF,mFAAmF;QACnF,oFAAoF;QACpF,iDAAiD;QACjD,MAAM,KAAK,GAAG,CAAC,KAAiB,EAAE,SAAiB,EAAE,EAAE;YACtD,IAAI,aAAa,IAAI,OAAO,IAAI,cAAc;gBAAE,OAAO;YACvD,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,iCAAiC,SAAS,OAAO,KAAK,SAAS,CAAC;YACpF,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;YACzC,MAAM,OAAO,GAAqB;gBACjC,GAAG,MAAM;gBACT,UAAU,EAAE,OAAO;gBACnB,YAAY,EAAE,WAAW;aACzB,CAAC;YACF,cAAc,GAAG,IAAI,CAAC;YACtB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC/D,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,CAAC,CAAC;QAEF,MAAM,QAAQ,GAAG,qBAAqB,CAAC;YACtC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC;SACpD,CAAC,CAAC;QACH,MAAM,sBAAsB,GAAG,GAAG,EAAE;YACnC,IAAI,aAAa,IAAI,OAAO,IAAI,sBAAsB,IAAI,kBAAkB;gBAAE,OAAO;YACrF,kBAAkB,GAAG,IAAI,CAAC;YAC1B,YAAY,GAAG,gBAAgB,CAAC;YAChC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;YACtC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACtC,CAAC,CAAC;QACF,MAAM,kBAAkB,GAAG,aAAa,EAAE,UAAU,CAAC;QACrD,MAAM,cAAc,GAAG,GAAG,EAAE;YAC3B,IAAI,kBAAkB,IAAI,cAAc,IAAI,OAAO;gBAAE,OAAO;YAC5D,cAAc,GAAG,IAAI,CAAC;YACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE;oBACN,GAAG,MAAM;oBACT,UAAU,EAAE,SAAS;oBACrB,YAAY,EAAE,0DAA0D;iBACxE;aACD,CAAC,CAAC;YACH,WAAW,EAAE,CAAC;QACf,CAAC,CAAC;QACF,MAAM,aAAa,GAAG,GAAG,EAAE;YAC1B,aAAa,GAAG,IAAI,CAAC;YACrB,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACvC,cAAc,EAAE,CAAC;QAClB,CAAC,CAAC;QACF,IAAI,aAAa,EAAE,CAAC;YACnB,aAAa,EAAE,CAAC;YAChB,OAAO,KAAK,CAAC;QACd,CAAC;QACD,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvE,KAAK,CAAC,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACJ,IAAI,KAAoC,CAAC;gBACzC,IAAI,CAAC;oBACJ,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;wBACtC,GAAG,aAAa;wBAChB,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,UAAU,EAAE,KAAK,EAAE,QAA0B,EAAE,aAAyB,EAAE,EAAE;4BAC3E,sBAAsB,EAAE,CAAC;4BACzB,MAAM,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;wBACrD,CAAC;qBACD,CAAC,CAAC;oBACH,kBAAkB,GAAG,IAAI,CAAC;oBAC1B,WAAW,EAAE,CAAC;gBACf,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,kBAAkB,GAAG,IAAI,CAAC;oBAC1B,IAAI,cAAc,IAAI,OAAO;wBAAE,OAAO;oBACtC,QAAQ,CAAC,MAAM,EAAE,CAAC;oBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;oBAC1D,WAAW,CAAC,KAAK,CAAC,CAAC;oBACnB,OAAO;gBACR,CAAC;gBAED,+EAA+E;gBAC/E,8EAA8E;gBAC9E,IAAI,cAAc,IAAI,OAAO,IAAI,aAAa;oBAAE,OAAO;gBACvD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBACjC,IAAI,OAAO;wBAAE,MAAM;oBACnB,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,MAAM,kBAAkB,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;oBAC7D,IAAI,kBAAkB;wBAAE,sBAAsB,GAAG,IAAI,CAAC;oBACtD,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBAChD,sBAAsB,EAAE,CAAC;oBAC1B,CAAC;yBAAM,IAAI,sBAAsB,EAAE,CAAC;wBACnC,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;wBACnC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;wBAC3B,cAAc,GAAG,KAAK,CAAC,EAAE,CAAC;wBAC1B,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1B,CAAC;oBACD,4EAA4E;oBAC5E,0EAA0E;oBAC1E,6EAA6E;oBAC7E,0EAA0E;oBAC1E,wEAAwE;oBACxE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;oBACjE,IAAI,QAAQ,EAAE,CAAC;wBACd,cAAc,GAAG,IAAI,CAAC;wBACtB,QAAQ,CAAC,MAAM,EAAE,CAAC;wBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;oBAC3D,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAClB,IAAI,QAAQ;wBAAE,OAAO;gBACtB,CAAC;gBACD,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;oBACvD,MAAM,WAAW,GAAG,aAAa;wBAChC,CAAC,CAAC,sCAAsC;wBACxC,CAAC,CAAC,qCAAqC,CAAC;oBACzC,cAAc,GAAG,IAAI,CAAC;oBACtB,KAAK,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,OAAO;wBACb,MAAM,EAAE,UAAU;wBAClB,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE;qBAC3D,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,WAAW,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;oBAAS,CAAC;gBACV,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAC3D,CAAC;QACF,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO,KAAK,CAAC;IACd,CAAC,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAAC,KAA4B;IAC/D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,YAAY,CAAC;QAClB,KAAK,gBAAgB,CAAC;QACtB,KAAK,gBAAgB;YACpB,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAC/B,KAAK,UAAU,CAAC;QAChB,KAAK,cAAc;YAClB,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACjC,KAAK,cAAc;YAClB,OAAO,IAAI,CAAC;QACb;YACC,OAAO,KAAK,CAAC;IACf,CAAC;AACF,CAAC","sourcesContent":["/**\n * Silence/idle watchdogs for the reliability kernel.\n *\n * A silence watchdog bounds \"running but mute\" — it never bounds total runtime,\n * so long tasks that produce output are never killed (autonomy constraint).\n */\n\nexport interface SilenceWatchdog {\n\t/** Report activity (output chunk / stream event); resets the countdown.\n\t * Pass silenceMs to also change the bound for this and subsequent countdowns. */\n\ttouch(silenceMs?: number): void;\n\t/** Stop permanently (normal completion). Idempotent. */\n\tdisarm(): void;\n}\n\nexport interface SilenceWatchdogOptions {\n\tsilenceMs: number;\n\t/** Fired at most once, after silenceMs with no touch(). The watchdog self-disarms. */\n\tonSilence: () => void;\n}\n\nexport function createSilenceWatchdog(opts: SilenceWatchdogOptions): SilenceWatchdog {\n\tlet timer: NodeJS.Timeout | undefined;\n\tlet disarmed = false;\n\tlet currentSilenceMs = opts.silenceMs;\n\n\tconst arm = () => {\n\t\ttimer = setTimeout(() => {\n\t\t\tdisarmed = true;\n\t\t\ttimer = undefined;\n\t\t\topts.onSilence();\n\t\t}, currentSilenceMs);\n\t\t// Never keep the host process alive just for a watchdog.\n\t\ttimer.unref?.();\n\t};\n\n\tarm();\n\n\treturn {\n\t\ttouch(silenceMs?: number): void {\n\t\t\tif (disarmed) return;\n\t\t\tif (silenceMs !== undefined) currentSilenceMs = silenceMs;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\tarm();\n\t\t},\n\t\tdisarm(): void {\n\t\t\tdisarmed = true;\n\t\t\tif (timer) clearTimeout(timer);\n\t\t\ttimer = undefined;\n\t\t},\n\t};\n}\n\n// --- Stream-idle watchdog (wraps a StreamFn) -------------------------------\n\nimport { createAssistantMessageEventStream } from \"@caupulican/pi-ai/event-stream\";\nimport type { Api, AssistantMessage, AssistantMessageEvent, Model, ProviderResponse } from \"@caupulican/pi-ai/types\";\nimport { createEmptyUsage } from \"@caupulican/pi-ai/usage\";\nimport type { StreamFn } from \"../types.ts\";\n\nexport type StallPhase = \"connect\" | \"first-progress\" | \"quiet\" | \"active\";\n\nexport interface StreamIdleOptions {\n\t/** Max ms to wait for the FIRST event (connection/first-token allowance). */\n\tconnectMs: number;\n\t/** Max ms between events while content is flowing — the latest content block is\n\t * text or toolCall. A flowing stream that goes silent this long is presumed dead. */\n\tactiveIdleMs: number;\n\t/** Max ms after transport confirmation until the first non-empty model delta.\n\t * Headers and structural start frames prove connectivity, not generation progress. */\n\tfirstProgressMs: number;\n\t/** Max ms between events while the model is quietly working — no content blocks\n\t * after it has emitted non-empty thinking. Deep-thinking models legitimately sit\n\t * here for minutes, so this bound is deliberately generous. */\n\tquietIdleMs: number;\n\t/** Fired once when a stall is detected, before the inner request is aborted. */\n\tonStall?: (info: { phase: StallPhase; elapsedMs: number }) => void;\n}\n\n/** Defaults: connect 120s / first progress 120s / active 180s / quiet 600s. The quiet bound must stay\n * below the HTTP dispatcher idle timeout (see coding-agent http-dispatcher.ts, 660s) or the\n * HTTP layer would kill quiet-but-healthy streams before this watchdog ever sees the gap. */\nexport const DEFAULT_STREAM_IDLE: StreamIdleOptions = {\n\tconnectMs: 120_000,\n\tfirstProgressMs: 120_000,\n\tactiveIdleMs: 180_000,\n\tquietIdleMs: 600_000,\n};\n\n/** Re-resolved at the start of every request, so hosts can wire live-tunable settings. */\nexport type StreamIdleOptionsResolver = (...args: Parameters<StreamFn>) => Partial<StreamIdleOptions>;\n\n/** Extracts the current AssistantMessage snapshot carried by any stream event variant. */\nfunction partialFromEvent(event: AssistantMessageEvent): AssistantMessage {\n\tif (event.type === \"done\") return event.message;\n\tif (event.type === \"error\") return event.error;\n\treturn event.partial;\n}\n\n/**\n * Wrap a StreamFn so a silently dead connection cannot wedge a turn forever.\n *\n * Phase-aware: `connectMs` bounds the wait for transport confirmation,\n * `firstProgressMs` bounds headers and structural frames that contain no model output,\n * `quietIdleMs` covers silence after non-empty thinking, and `activeIdleMs` covers\n * silence after text or tool-call output. This keeps detection fast where silence is\n * anomalous without killing healthy deep-thinking or compaction-sized requests.\n * No bound ever limits total runtime (autonomy constraint).\n *\n * On stall, the inner request is aborted and the returned stream resolves immediately\n * with a synthetic `AssistantMessage` (`stopReason: \"error\"`, `errorMessage: \"stream\n * stalled: no events for <n>ms (<phase> phase)\"`) — the `stream stalled` phrasing is\n * what `classifyFailure` maps to a retryable `stream_stall`, so the host's\n * retry/failover path takes it from there.\n *\n * Options may be a resolver function; it is re-invoked at the start of every request,\n * so settings changes apply without rewrapping.\n *\n * A caller-initiated abort (via the options `signal`) is never treated as a stall: it\n * is chained into the wrapper's own controller. The inner stream's abort result is\n * forwarded untouched when it provides one; if it just ends, the wrapper synthesizes\n * an aborted terminal event so the returned stream always settles.\n */\nexport function withStreamIdleWatchdog(\n\tstreamFn: StreamFn,\n\toptions?: Partial<StreamIdleOptions> | StreamIdleOptionsResolver,\n): StreamFn {\n\treturn (model, context, streamOptions) => {\n\t\tconst resolved = typeof options === \"function\" ? options(model, context, streamOptions) : options;\n\t\tconst cleaned: Partial<StreamIdleOptions> = {};\n\t\tif (resolved) {\n\t\t\tfor (const [key, val] of Object.entries(resolved)) {\n\t\t\t\tif (val !== undefined) {\n\t\t\t\t\t(cleaned as any)[key] = val;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tconst opts = { ...DEFAULT_STREAM_IDLE, ...cleaned };\n\n\t\tconst controller = new AbortController();\n\t\tconst callerSignal = streamOptions?.signal;\n\t\tlet callerAborted = callerSignal?.aborted ?? false;\n\t\tconst outer = createAssistantMessageEventStream();\n\t\tlet readySettled = false;\n\t\tlet resolveReady: (stream: typeof outer) => void = () => {};\n\t\tconst ready = new Promise<typeof outer>((resolve) => {\n\t\t\tresolveReady = resolve;\n\t\t});\n\t\tconst settleReady = () => {\n\t\t\tif (readySettled) return;\n\t\t\treadySettled = true;\n\t\t\tresolveReady(outer);\n\t\t};\n\n\t\t// Seeded so a connect-phase stall (no event ever arrived) still has a base message\n\t\t// to report on; overwritten with the latest real snapshot once events start flowing.\n\t\tlet latest: AssistantMessage = {\n\t\t\trole: \"assistant\",\n\t\t\tcontent: [],\n\t\t\tapi: model.api,\n\t\t\tprovider: model.provider,\n\t\t\tmodel: model.id,\n\t\t\tusage: createEmptyUsage(),\n\t\t\tstopReason: \"stop\",\n\t\t\ttimestamp: Date.now(),\n\t\t};\n\t\tlet stalled = false;\n\t\tlet meaningfulProgressSeen = false;\n\t\tlet transportConfirmed = false;\n\t\tlet streamSetupSettled = false;\n\t\tlet terminalPushed = false;\n\t\tconst pushFailure = (error?: unknown) => {\n\t\t\tif (terminalPushed || stalled) return;\n\t\t\tconst stopReason = callerAborted ? \"aborted\" : \"error\";\n\t\t\tconst detail = error instanceof Error ? `: ${error.message}` : error === undefined ? \"\" : `: ${String(error)}`;\n\t\t\tterminalPushed = true;\n\t\t\touter.push({\n\t\t\t\ttype: \"error\",\n\t\t\t\treason: stopReason,\n\t\t\t\terror: {\n\t\t\t\t\t...latest,\n\t\t\t\t\tstopReason,\n\t\t\t\t\terrorMessage: callerAborted\n\t\t\t\t\t\t? `stream aborted before terminal event${detail}`\n\t\t\t\t\t\t: `stream ended without terminal event${detail}`,\n\t\t\t\t},\n\t\t\t});\n\t\t\tsettleReady();\n\t\t};\n\n\t\t// The idle bound adapts per event: quiet while nothing/thinking, active while\n\t\t// text/toolCall content is flowing. Mutable so the onSilence closure always\n\t\t// reports the phase/bound that actually elapsed.\n\t\tlet currentPhase: StallPhase = \"connect\";\n\t\tlet currentBoundMs = opts.connectMs;\n\t\tconst idleBoundFor = (message: AssistantMessage): { phase: StallPhase; ms: number } => {\n\t\t\tconst lastBlock = message.content[message.content.length - 1];\n\t\t\treturn !lastBlock || lastBlock.type === \"thinking\"\n\t\t\t\t? { phase: \"quiet\", ms: opts.quietIdleMs }\n\t\t\t\t: { phase: \"active\", ms: opts.activeIdleMs };\n\t\t};\n\n\t\t// Emits the stall result directly (rather than after the inner loop finishes) so a\n\t\t// connection that never resolves at all still yields a result promptly — providers\n\t\t// are contractually expected to end their stream after abort, but the watchdog does\n\t\t// not depend on that to report the stall itself.\n\t\tconst stall = (phase: StallPhase, elapsedMs: number) => {\n\t\t\tif (callerAborted || stalled || terminalPushed) return;\n\t\t\tstalled = true;\n\t\t\topts.onStall?.({ phase, elapsedMs });\n\t\t\tconst description = `stream stalled: no events for ${elapsedMs}ms (${phase} phase)`;\n\t\t\tcontroller.abort(new Error(description));\n\t\t\tconst message: AssistantMessage = {\n\t\t\t\t...latest,\n\t\t\t\tstopReason: \"error\",\n\t\t\t\terrorMessage: description,\n\t\t\t};\n\t\t\tterminalPushed = true;\n\t\t\touter.push({ type: \"error\", reason: \"error\", error: message });\n\t\t\tsettleReady();\n\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t};\n\n\t\tconst watchdog = createSilenceWatchdog({\n\t\t\tsilenceMs: opts.connectMs,\n\t\t\tonSilence: () => stall(currentPhase, currentBoundMs),\n\t\t});\n\t\tconst markTransportConfirmed = () => {\n\t\t\tif (callerAborted || stalled || meaningfulProgressSeen || transportConfirmed) return;\n\t\t\ttransportConfirmed = true;\n\t\t\tcurrentPhase = \"first-progress\";\n\t\t\tcurrentBoundMs = opts.firstProgressMs;\n\t\t\twatchdog.touch(opts.firstProgressMs);\n\t\t};\n\t\tconst originalOnResponse = streamOptions?.onResponse;\n\t\tconst pushSetupAbort = () => {\n\t\t\tif (streamSetupSettled || terminalPushed || stalled) return;\n\t\t\tterminalPushed = true;\n\t\t\twatchdog.disarm();\n\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\touter.push({\n\t\t\t\ttype: \"error\",\n\t\t\t\treason: \"aborted\",\n\t\t\t\terror: {\n\t\t\t\t\t...latest,\n\t\t\t\t\tstopReason: \"aborted\",\n\t\t\t\t\terrorMessage: \"stream aborted before terminal event during stream setup\",\n\t\t\t\t},\n\t\t\t});\n\t\t\tsettleReady();\n\t\t};\n\t\tconst onCallerAbort = () => {\n\t\t\tcallerAborted = true;\n\t\t\tcontroller.abort(callerSignal?.reason);\n\t\t\tpushSetupAbort();\n\t\t};\n\t\tif (callerAborted) {\n\t\t\tonCallerAbort();\n\t\t\treturn ready;\n\t\t}\n\t\tcallerSignal?.addEventListener(\"abort\", onCallerAbort, { once: true });\n\n\t\tvoid (async () => {\n\t\t\ttry {\n\t\t\t\tlet inner: Awaited<ReturnType<StreamFn>>;\n\t\t\t\ttry {\n\t\t\t\t\tinner = await streamFn(model, context, {\n\t\t\t\t\t\t...streamOptions,\n\t\t\t\t\t\tsignal: controller.signal,\n\t\t\t\t\t\tonResponse: async (response: ProviderResponse, responseModel: Model<Api>) => {\n\t\t\t\t\t\t\tmarkTransportConfirmed();\n\t\t\t\t\t\t\tawait originalOnResponse?.(response, responseModel);\n\t\t\t\t\t\t},\n\t\t\t\t\t});\n\t\t\t\t\tstreamSetupSettled = true;\n\t\t\t\t\tsettleReady();\n\t\t\t\t} catch (error) {\n\t\t\t\t\tstreamSetupSettled = true;\n\t\t\t\t\tif (terminalPushed || stalled) return;\n\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\tpushFailure(error);\n\t\t\t\t\treturn;\n\t\t\t\t}\n\n\t\t\t\t// A setup promise may resolve after a connect stall or caller abort. The outer\n\t\t\t\t// stream is already terminal in that case, so never attach a late event pump.\n\t\t\t\tif (terminalPushed || stalled || callerAborted) return;\n\t\t\t\tfor await (const event of inner) {\n\t\t\t\t\tif (stalled) break;\n\t\t\t\t\tlatest = partialFromEvent(event);\n\t\t\t\t\tconst meaningfulProgress = eventHasMeaningfulProgress(event);\n\t\t\t\t\tif (meaningfulProgress) meaningfulProgressSeen = true;\n\t\t\t\t\tif (!transportConfirmed && !meaningfulProgress) {\n\t\t\t\t\t\tmarkTransportConfirmed();\n\t\t\t\t\t} else if (meaningfulProgressSeen) {\n\t\t\t\t\t\tconst bound = idleBoundFor(latest);\n\t\t\t\t\t\tcurrentPhase = bound.phase;\n\t\t\t\t\t\tcurrentBoundMs = bound.ms;\n\t\t\t\t\t\twatchdog.touch(bound.ms);\n\t\t\t\t\t}\n\t\t\t\t\t// A terminal event ends the turn: disarm synchronously, in the same tick as\n\t\t\t\t\t// the push below, so no watchdog can fire after the consumer's `result()`\n\t\t\t\t\t// promise resolves — a disarm that only happened once the loop later notices\n\t\t\t\t\t// `inner` is done would race with that resolution (it runs a tick or more\n\t\t\t\t\t// later) and could fire a spurious stall on an already-finished stream.\n\t\t\t\t\tconst terminal = event.type === \"done\" || event.type === \"error\";\n\t\t\t\t\tif (terminal) {\n\t\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\t\twatchdog.disarm();\n\t\t\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t\t\t}\n\t\t\t\t\touter.push(event);\n\t\t\t\t\tif (terminal) return;\n\t\t\t\t}\n\t\t\t\tif (!terminalPushed && !stalled) {\n\t\t\t\t\tconst stopReason = callerAborted ? \"aborted\" : \"error\";\n\t\t\t\t\tconst description = callerAborted\n\t\t\t\t\t\t? \"stream aborted before terminal event\"\n\t\t\t\t\t\t: \"stream ended without terminal event\";\n\t\t\t\t\tterminalPushed = true;\n\t\t\t\t\touter.push({\n\t\t\t\t\t\ttype: \"error\",\n\t\t\t\t\t\treason: stopReason,\n\t\t\t\t\t\terror: { ...latest, stopReason, errorMessage: description },\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t} catch (error) {\n\t\t\t\tpushFailure(error);\n\t\t\t} finally {\n\t\t\t\twatchdog.disarm();\n\t\t\t\tcallerSignal?.removeEventListener(\"abort\", onCallerAbort);\n\t\t\t}\n\t\t})();\n\n\t\treturn ready;\n\t};\n}\n\nfunction eventHasMeaningfulProgress(event: AssistantMessageEvent): boolean {\n\tswitch (event.type) {\n\t\tcase \"text_delta\":\n\t\tcase \"thinking_delta\":\n\t\tcase \"toolcall_delta\":\n\t\t\treturn event.delta.length > 0;\n\t\tcase \"text_end\":\n\t\tcase \"thinking_end\":\n\t\t\treturn event.content.length > 0;\n\t\tcase \"toolcall_end\":\n\t\t\treturn true;\n\t\tdefault:\n\t\t\treturn false;\n\t}\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@caupulican/pi-agent-core",
|
|
3
|
-
"version": "0.93.
|
|
3
|
+
"version": "0.93.15",
|
|
4
4
|
"description": "General-purpose agent with transport abstraction, state management, and attachment support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -146,7 +146,7 @@
|
|
|
146
146
|
"prepublishOnly": "npm run clean && npm run build"
|
|
147
147
|
},
|
|
148
148
|
"dependencies": {
|
|
149
|
-
"@caupulican/pi-ai": "^0.93.
|
|
149
|
+
"@caupulican/pi-ai": "^0.93.15",
|
|
150
150
|
"ignore": "7.0.5",
|
|
151
151
|
"typebox": "1.1.38",
|
|
152
152
|
"yaml": "2.9.0"
|