@sema-agent/core 7.32.0 → 7.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/brain/anthropic.d.ts +1 -1
- package/dist/brain/anthropic.js +4 -1
- package/dist/brain/open-responses.d.ts +2 -1
- package/dist/brain/open-responses.js +4 -1
- package/dist/brain/openai.d.ts +2 -1
- package/dist/brain/openai.js +4 -1
- package/dist/brain/stream-engine.d.ts +8 -7
- package/dist/brain/stream-engine.js +111 -44
- package/dist/brain/timeout.d.ts +112 -53
- package/dist/brain/timeout.js +39 -5
- package/dist/brain/transport.d.ts +13 -0
- package/dist/brain/transport.js +62 -0
- package/dist/core/background-shell.d.ts +24 -8
- package/dist/core/background-shell.js +4 -3
- package/dist/core/runner/prepare-boundary-parks.js +3 -3
- package/dist/core/runner/prepare-park-ask.js +2 -2
- package/dist/core/runner/prepare-task.js +2 -2
- package/dist/core/runner/run-settle-and-teardown.js +3 -5
- package/dist/core/task-event.d.ts +7 -5
- package/dist/core/task-registry-shared.d.ts +1 -1
- package/dist/core/task-registry.d.ts +3 -6
- package/dist/core/task-registry.js +0 -9
- package/dist/engine/llm/types.d.ts +5 -5
- package/dist/orchestration/workflow.d.ts +4 -4
- package/dist/orchestration/workflow.js +2 -2
- package/package.json +6 -3
- package/wire-changes/core@7.32.1.json +78 -0
- package/wire-changes/core@7.33.0.json +351 -0
package/dist/brain/timeout.d.ts
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
import type { StreamFn } from "../internal/llm.js";
|
|
2
2
|
/**
|
|
3
|
-
* Tiered timeouts (
|
|
4
|
-
* it
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* Tiered timeouts (design/17 §5B; re-cut along the segments of a call in 7.33.0). The task-level
|
|
4
|
+
* `timeoutSec` only bounds the *whole* run; it cannot tell apart a gateway that never completes the
|
|
5
|
+
* handshake, one that accepts the request and then holds the response headers while it works, one
|
|
6
|
+
* that opens the stream and never emits a token, and one that goes quiet mid-answer. Four clocks,
|
|
7
|
+
* one per segment, each named for the segment it bounds:
|
|
7
8
|
*
|
|
8
|
-
* - **connect**: `fetch`
|
|
9
|
-
*
|
|
9
|
+
* - **connect**: TCP + TLS establishment inside `fetch` — the transport's own clock (undici's 10s by
|
|
10
|
+
* default), tuned on the process dispatcher the host installs; not a knob here (`transport.ts`).
|
|
11
|
+
* - **first-byte**: the `fetch()` call → response headers. A stall here is this process cancelling
|
|
12
|
+
* its own request at a deadline it chose, and is announced as such.
|
|
13
|
+
* - **first-token**: headers → the first content delta (a thinking delta counts).
|
|
14
|
+
* - **idle**: silence between deltas once output has started.
|
|
10
15
|
*
|
|
11
|
-
*
|
|
16
|
+
* Every field resolves through the same chain: construction config (an explicit `0` is OFF) >
|
|
17
|
+
* per-call advisory (`StreamOptions.stallTimeouts`) > the engine default constant below.
|
|
12
18
|
*/
|
|
13
19
|
/**
|
|
14
20
|
* The largest delay a host timer can actually wait for. Past this `setTimeout` does not schedule far in
|
|
@@ -28,55 +34,107 @@ export declare const MAX_TIMER_DELAY_MS = 2147483647;
|
|
|
28
34
|
*/
|
|
29
35
|
export declare function resolveStallTimeoutMs(value: number | undefined, knob: string): number | undefined;
|
|
30
36
|
/**
|
|
31
|
-
* The engine's
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
37
|
+
* The engine's DEFAULT watchdog values, one per segment of a model call — the third and last level of
|
|
38
|
+
* the chain every brain reads (`BrainTimeoutConfig` field > per-call advisory `StreamOptions.stallTimeouts`
|
|
39
|
+
* > these). The Runner supplies these same numbers as the advisory on every call, so a Brain supplied by
|
|
40
|
+
* the deployment (BYOM) is told what the engine would arm; a first-party brain arms them itself when
|
|
41
|
+
* neither a config field nor an advisory names the segment. A silent call is therefore always bounded by
|
|
42
|
+
* something finer than the hour-scale {@link BRAIN_CALL_GUARDRAIL_DEFAULT_MS} (a measured production
|
|
43
|
+
* frame once sat 680s with ZERO deltas) unless a deployment switched the watchdog for that segment off
|
|
44
|
+
* by name — an explicit `0` is the only spelling of OFF. The TCP + TLS connect has no row here: it is
|
|
45
|
+
* the transport's own clock (`transport.ts`), not a segment the engine can observe. 量纲:
|
|
46
|
+
* - first-byte 300s: the `fetch()` call → response headers. A self-hosted gateway that runs its
|
|
47
|
+
* prefill BEFORE writing headers is silent here for minutes and healthy; the 30s "connect" clock
|
|
48
|
+
* this replaces cut that prefill and re-sent the same body ten times. 300s is the reference
|
|
49
|
+
* implementation's window for a non-first-party route; the window grows with the request body
|
|
50
|
+
* ({@link firstByteWindow}) and the one retry is widened ({@link FIRST_BYTE_RETRY_WINDOW_FACTOR}).
|
|
41
51
|
* - first-token 600s: bounds the headers→first-delta wait, NOT the model's thought (the first
|
|
42
|
-
* THINKING delta already counts as the first token, design/17 O5
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* one a deployment gets by supplying no `fetchImpl` — undici destroys a body that has produced no
|
|
54
|
-
* bytes for 300s (`bodyTimeout`, default `300e3`), so byte-silence past 300s ends the attempt
|
|
55
|
-
* there and arrives as a read rejection reported `transport`. That report is CORRECT (the
|
|
56
|
-
* transport really did cut it) and the seat to change it exists: `fetchImpl` on both brain configs
|
|
57
|
-
* takes any `fetch`, so a deployment needing the full 600s passes one built on an undici
|
|
58
|
-
* `Agent`/`Dispatcher` with its own `bodyTimeout`. Named here so the next reader who measures ~300s
|
|
59
|
-
* of tolerance does not conclude this number is a lie.
|
|
60
|
-
* {@link import("../orchestration/workflow.js").WORKFLOW_AGENT_STALL_MS} is DERIVED from
|
|
61
|
-
* {@link STALL_CONNECT_MS} + this value, so the coarse agent-restart cannot sit UNDER a single
|
|
62
|
-
* silent provider call. Move this number and that one follows. It is a floor, not
|
|
63
|
-
* a guarantee of ordering — see that constant's own comment for the cases that still invert.
|
|
64
|
-
* - idle 90s: mid-stream silence after output started — a healthy stream emits steadily
|
|
65
|
-
* (comfortably above any provider keep-alive cadence).
|
|
52
|
+
* THINKING delta already counts as the first token, design/17 O5), so a slow reasoner is safe. A
|
|
53
|
+
* minutes-scale bound is FALSE for a self-hosted server working through a long prefill AFTER
|
|
54
|
+
* headers: it emits nothing for 2–3 minutes while healthy, and cancelling it burns the prefill and
|
|
55
|
+
* re-sends a request that burns it again. The default transport keeps NO clock over this segment
|
|
56
|
+
* (the request carries `bodyTimeout: 0`), so this number is the bound actually granted.
|
|
57
|
+
* {@link import("../orchestration/workflow.js").WORKFLOW_AGENT_STALL_MS} is DERIVED from the
|
|
58
|
+
* first-byte lane at {@link STALL_FIRST_BYTE_MS} ({@link firstByteLaneKnobMs}) + this value, so the coarse agent-restart cannot sit UNDER a single
|
|
59
|
+
* silent provider call (a floor, not a guarantee of ordering — see that constant's comment).
|
|
60
|
+
* - idle 300s: mid-stream silence after output started — the reference implementation's idle floor
|
|
61
|
+
* (a self-hosted server can stall for minutes between deltas while a server-side tool runs; a provider
|
|
62
|
+
* keep-alive comment is not a delta and does not count).
|
|
66
63
|
* ADVISORY: a brain whose construction-time {@link BrainTimeoutConfig} sets the corresponding field
|
|
67
64
|
* keeps it (deployment sovereignty; see the `StreamOptions.stallTimeouts` priority chain).
|
|
68
65
|
*/
|
|
69
|
-
export declare const
|
|
66
|
+
export declare const STALL_FIRST_BYTE_MS = 300000;
|
|
70
67
|
export declare const STALL_FIRST_TOKEN_MS = 600000;
|
|
71
|
-
export declare const STALL_IDLE_MS =
|
|
68
|
+
export declare const STALL_IDLE_MS = 300000;
|
|
69
|
+
/**
|
|
70
|
+
* The retired names of the header-wait bound, each with the name that replaced it: the construction key and
|
|
71
|
+
* the per-call advisory key. The type face no longer has them, but a type only refuses a LITERAL — a
|
|
72
|
+
* spread object (`{ ...timeouts }`) or a JS caller still delivers the key, and a knob that arrives and is
|
|
73
|
+
* then ignored is a deployment setting silently doing nothing. So their arrival is refused, naming the
|
|
74
|
+
* replacement: a tripwire, not a second spelling (nothing reads the value).
|
|
75
|
+
*/
|
|
76
|
+
export declare const RETIRED_TIMEOUT_KNOBS: {
|
|
77
|
+
readonly construction: {
|
|
78
|
+
readonly connectTimeoutMs: "firstByteTimeoutMs";
|
|
79
|
+
};
|
|
80
|
+
readonly advisory: {
|
|
81
|
+
readonly connectMs: "firstByteMs";
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
/** Refuse a retired timeout key present on `holder` (see {@link RETIRED_TIMEOUT_KNOBS}); `where` names the
|
|
85
|
+
* object in the message (`"createOpenAIBrain config"`, `"StreamOptions.stallTimeouts"`). */
|
|
86
|
+
export declare function refuseRetiredTimeoutKnobs(holder: object | undefined, level: keyof typeof RETIRED_TIMEOUT_KNOBS, where: string): void;
|
|
87
|
+
/** The first-byte window grows with the request body: one second per this many bytes, rounded up (the
|
|
88
|
+
* reference implementation's allowance) — a large context on a slow uplink is still UPLOADING when a
|
|
89
|
+
* fixed window would call the gateway silent. */
|
|
90
|
+
export declare const FIRST_BYTE_UPLOAD_BYTES_PER_SECOND = 32768;
|
|
91
|
+
/** A first-byte stall is re-sent at most this many times, out of its OWN budget rather than
|
|
92
|
+
* `maxRetries` (the reference implementation: "normally one retry, not the session budget"); the shared
|
|
93
|
+
* ladder counts transport failures, and this is not one. */
|
|
94
|
+
export declare const FIRST_BYTE_RETRY_BUDGET = 1;
|
|
95
|
+
/** The retry's knob is the first times this — the reference implementation widens its retry to a second
|
|
96
|
+
* knob's default (599s against a 300s first window); one knob here keeps that ratio instead. */
|
|
97
|
+
export declare const FIRST_BYTE_RETRY_WINDOW_FACTOR = 2;
|
|
98
|
+
/** The ONE resolution every watchdog knob reads: construction config (an explicit `0` is OFF) > per-call
|
|
99
|
+
* advisory > the engine default. A bad number at either level is refused ({@link resolveStallTimeoutMs});
|
|
100
|
+
* only an ABSENT value falls through. */
|
|
101
|
+
export declare function resolveWatchdogMs(configured: number | undefined, advisory: number | undefined, engineDefault: number, knob: string): number;
|
|
102
|
+
/** One armed first-byte window, with the two numbers its announcement quotes. */
|
|
103
|
+
export interface FirstByteWindow {
|
|
104
|
+
/** The knob's value for THIS attempt (the configured value, widened on the retry). */
|
|
105
|
+
knobMs: number;
|
|
106
|
+
/** The upload allowance added for the request body, ms (0 for an empty body). */
|
|
107
|
+
allowanceMs: number;
|
|
108
|
+
/** What is actually armed: `knobMs + allowanceMs`, capped at {@link MAX_TIMER_DELAY_MS}. `0` = off. */
|
|
109
|
+
windowMs: number;
|
|
110
|
+
}
|
|
111
|
+
/** The first-byte window for one attempt; `retryIndex` = first-byte stalls already suffered (0 first),
|
|
112
|
+
* each widening the knob by {@link FIRST_BYTE_RETRY_WINDOW_FACTOR}. A knob of `0` (OFF) stays off. */
|
|
113
|
+
export declare function firstByteWindow(knobMs: number | undefined, bodyBytes: number, retryIndex: number): FirstByteWindow;
|
|
114
|
+
/**
|
|
115
|
+
* The most a first-byte lane armed at `knobMs` waits on the gateway in total before it gives up: the probe
|
|
116
|
+
* window and every widened retry its budget allows ({@link FIRST_BYTE_RETRY_BUDGET},
|
|
117
|
+
* {@link FIRST_BYTE_RETRY_WINDOW_FACTOR}) — read off {@link firstByteWindow}, the same function that arms
|
|
118
|
+
* each attempt, so the sum cannot drift from what is armed. The knob part only: the upload allowance
|
|
119
|
+
* depends on the request body and the backoff between attempts is not a wait on the gateway, so the lane's
|
|
120
|
+
* real wall time is this plus both — a LOWER bound.
|
|
121
|
+
*/
|
|
122
|
+
export declare function firstByteLaneKnobMs(knobMs: number): number;
|
|
72
123
|
export interface BrainTimeoutConfig {
|
|
73
|
-
/**
|
|
74
|
-
|
|
124
|
+
/**
|
|
125
|
+
* `fetch()` call → response headers, ms: the window a gateway has to START answering (until 7.33.0 a
|
|
126
|
+
* knob named `connectTimeoutMs` bounded this wait; it is gone — the connect clock is the transport's). Absent ⇒
|
|
127
|
+
* {@link STALL_FIRST_BYTE_MS}; the armed window grows with the request body ({@link firstByteWindow});
|
|
128
|
+
* a stall earns ONE retry at a widened window out of its own budget, then fails naming this knob.
|
|
129
|
+
* Fires as a `stall` (this process cancelled its own request), never as a transport fault. `0` ⇒ off.
|
|
130
|
+
*/
|
|
131
|
+
firstByteTimeoutMs?: number;
|
|
75
132
|
/**
|
|
76
133
|
* After the stream opens, cancel it if no content delta (text / thinking / tool input) arrives
|
|
77
134
|
* within this many ms → `BrainError("network", "first-token timeout")`. For reasoning models the
|
|
78
135
|
* **first thinking delta also counts** as the first token, so a long-thinking model isn't killed
|
|
79
|
-
* for being slow to its first *visible* word (design/17 O5).
|
|
136
|
+
* for being slow to its first *visible* word (design/17 O5). Absent ⇒ {@link STALL_FIRST_TOKEN_MS};
|
|
137
|
+
* `0` ⇒ off.
|
|
80
138
|
*/
|
|
81
139
|
firstTokenTimeoutMs?: number;
|
|
82
140
|
/**
|
|
@@ -84,18 +142,19 @@ export interface BrainTimeoutConfig {
|
|
|
84
142
|
* After the first delta, re-armed on every delta; if the stream then goes silent for this many ms
|
|
85
143
|
* (a gateway that hung mid-output) → `reader.cancel()` + `BrainError("network", "stream idle timeout")`.
|
|
86
144
|
* Complements `firstTokenTimeoutMs`: that one tolerates a long *first* token (reasoning), this one
|
|
87
|
-
* catches a stall *after* output has started, where a healthy stream emits steadily.
|
|
145
|
+
* catches a stall *after* output has started, where a healthy stream emits steadily. Absent ⇒
|
|
146
|
+
* {@link STALL_IDLE_MS}; `0` ⇒ off.
|
|
88
147
|
*/
|
|
89
148
|
idleTimeoutMs?: number;
|
|
90
149
|
}
|
|
91
150
|
/**
|
|
92
|
-
* A per-
|
|
151
|
+
* A per-attempt abort controller that fires at the first-byte deadline while still forwarding an
|
|
93
152
|
* outer (task) abort. The returned `signal` governs the **whole** request including the streamed body
|
|
94
|
-
* — so once headers arrive, call `clearTimer()` (NOT `dispose()`) to stop the
|
|
95
|
-
*
|
|
96
|
-
*
|
|
153
|
+
* — so once headers arrive, call `clearTimer()` (NOT `dispose()`) to stop the deadline while keeping
|
|
154
|
+
* the outer-abort link alive for the body read; call `dispose()` only when the request is fully done
|
|
155
|
+
* or being discarded.
|
|
97
156
|
*/
|
|
98
|
-
export declare function
|
|
157
|
+
export declare function createAttemptController(firstByteWindowMs: number | undefined, outerSignal: AbortSignal | undefined): {
|
|
99
158
|
signal: AbortSignal;
|
|
100
159
|
timedOut: () => boolean;
|
|
101
160
|
clearTimer: () => void;
|
|
@@ -110,7 +169,7 @@ export declare function createConnectController(connectTimeoutMs: number | undef
|
|
|
110
169
|
* Default ceiling on how long the ENGINE waits for a single brain call (ruled 2026-08-03: one hour).
|
|
111
170
|
* Deliberately far above any legitimate call: this is a wedged-run backstop, not a latency policy —
|
|
112
171
|
* a deployment that wants a tight per-call bound sets the fine tier above ({@link BrainTimeoutConfig},
|
|
113
|
-
* or the advisory {@link
|
|
172
|
+
* or the advisory {@link STALL_FIRST_BYTE_MS} family the engine supplies by default). All of these clocks
|
|
114
173
|
* run in parallel with the task's own `limits.timeoutSec`; whichever expires first owns the run.
|
|
115
174
|
*/
|
|
116
175
|
export declare const BRAIN_CALL_GUARDRAIL_DEFAULT_MS = 3600000;
|
package/dist/brain/timeout.js
CHANGED
|
@@ -14,10 +14,44 @@ export function resolveStallTimeoutMs(value, knob) {
|
|
|
14
14
|
}
|
|
15
15
|
return value;
|
|
16
16
|
}
|
|
17
|
-
export const
|
|
17
|
+
export const STALL_FIRST_BYTE_MS = 300_000;
|
|
18
18
|
export const STALL_FIRST_TOKEN_MS = 600_000;
|
|
19
|
-
export const STALL_IDLE_MS =
|
|
20
|
-
export
|
|
19
|
+
export const STALL_IDLE_MS = 300_000;
|
|
20
|
+
export const RETIRED_TIMEOUT_KNOBS = {
|
|
21
|
+
construction: { connectTimeoutMs: "firstByteTimeoutMs" },
|
|
22
|
+
advisory: { connectMs: "firstByteMs" },
|
|
23
|
+
};
|
|
24
|
+
export function refuseRetiredTimeoutKnobs(holder, level, where) {
|
|
25
|
+
if (holder === undefined || holder === null)
|
|
26
|
+
return;
|
|
27
|
+
for (const [retired, replacement] of Object.entries(RETIRED_TIMEOUT_KNOBS[level])) {
|
|
28
|
+
if (!(retired in holder))
|
|
29
|
+
continue;
|
|
30
|
+
const e = new Error(`${where}: \`${retired}\` was removed in 7.33.0 and is refused rather than ignored — the fetch()→response-headers wait it bounded is \`${replacement}\` (0 = off); the TCP + TLS connect is the transport's own clock, tuned on the process dispatcher (setGlobalDispatcher(new Agent({ connect: { timeout } })))`);
|
|
31
|
+
e.code = "config.stall_timeout_retired";
|
|
32
|
+
throw e;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export const FIRST_BYTE_UPLOAD_BYTES_PER_SECOND = 32_768;
|
|
36
|
+
export const FIRST_BYTE_RETRY_BUDGET = 1;
|
|
37
|
+
export const FIRST_BYTE_RETRY_WINDOW_FACTOR = 2;
|
|
38
|
+
export function resolveWatchdogMs(configured, advisory, engineDefault, knob) {
|
|
39
|
+
return resolveStallTimeoutMs(configured ?? advisory, knob) ?? engineDefault;
|
|
40
|
+
}
|
|
41
|
+
export function firstByteWindow(knobMs, bodyBytes, retryIndex) {
|
|
42
|
+
if (knobMs === undefined || knobMs <= 0)
|
|
43
|
+
return { knobMs: 0, allowanceMs: 0, windowMs: 0 };
|
|
44
|
+
const widened = knobMs * FIRST_BYTE_RETRY_WINDOW_FACTOR ** Math.max(0, retryIndex);
|
|
45
|
+
const allowanceMs = Math.ceil(Math.max(0, bodyBytes) / FIRST_BYTE_UPLOAD_BYTES_PER_SECOND) * 1000;
|
|
46
|
+
return { knobMs: widened, allowanceMs, windowMs: Math.min(widened + allowanceMs, MAX_TIMER_DELAY_MS) };
|
|
47
|
+
}
|
|
48
|
+
export function firstByteLaneKnobMs(knobMs) {
|
|
49
|
+
let total = 0;
|
|
50
|
+
for (let retryIndex = 0; retryIndex <= FIRST_BYTE_RETRY_BUDGET; retryIndex++)
|
|
51
|
+
total += firstByteWindow(knobMs, 0, retryIndex).knobMs;
|
|
52
|
+
return total;
|
|
53
|
+
}
|
|
54
|
+
export function createAttemptController(firstByteWindowMs, outerSignal) {
|
|
21
55
|
const ac = new AbortController();
|
|
22
56
|
let timedOut = false;
|
|
23
57
|
const onAbort = () => ac.abort(outerSignal?.reason);
|
|
@@ -28,11 +62,11 @@ export function createConnectController(connectTimeoutMs, outerSignal) {
|
|
|
28
62
|
outerSignal.addEventListener("abort", onAbort, { once: true });
|
|
29
63
|
}
|
|
30
64
|
let timer;
|
|
31
|
-
if (
|
|
65
|
+
if (firstByteWindowMs && firstByteWindowMs > 0) {
|
|
32
66
|
timer = setTimeout(() => {
|
|
33
67
|
timedOut = true;
|
|
34
68
|
ac.abort();
|
|
35
|
-
},
|
|
69
|
+
}, firstByteWindowMs);
|
|
36
70
|
}
|
|
37
71
|
const clearTimer = () => {
|
|
38
72
|
if (timer)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Dispatcher } from "undici";
|
|
2
|
+
/**
|
|
3
|
+
* The one dispatcher every first-party brain hands its fetch: the sending members are the engine's
|
|
4
|
+
* ({@link SENDING_MEMBERS}); every other member — state, lifecycle, events — is the host's, read when
|
|
5
|
+
* accessed ({@link hostMember}), so nothing a fetch starts reading has to be added to a list here.
|
|
6
|
+
* Stateless; owns no sockets.
|
|
7
|
+
*/
|
|
8
|
+
export declare const ENGINE_DISPATCHER: Dispatcher;
|
|
9
|
+
/** The `fetch` a brain sends its requests through: the caller's `fetchImpl` (else Node's `fetch`), each
|
|
10
|
+
* request carrying {@link ENGINE_DISPATCHER} in its init. */
|
|
11
|
+
export declare function resolveFetch(config: {
|
|
12
|
+
fetchImpl?: typeof fetch;
|
|
13
|
+
}): typeof fetch;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Dispatcher, getGlobalDispatcher } from "undici";
|
|
2
|
+
import UnwrapHandler from "undici/lib/handler/unwrap-handler.js";
|
|
3
|
+
import WrapHandler from "undici/lib/handler/wrap-handler.js";
|
|
4
|
+
function handlerApi(handler) {
|
|
5
|
+
return typeof handler.onRequestStart === "function" ? 2 : 1;
|
|
6
|
+
}
|
|
7
|
+
function inSlotApi(api, handler) {
|
|
8
|
+
return api === 1 ? UnwrapHandler.unwrap(handler) : WrapHandler.wrap(handler);
|
|
9
|
+
}
|
|
10
|
+
function slot(version) {
|
|
11
|
+
const held = Reflect.get(globalThis, Symbol.for(`undici.globalDispatcher.${String(version)}`));
|
|
12
|
+
return isDispatcher(held) ? held : undefined;
|
|
13
|
+
}
|
|
14
|
+
function isDispatcher(value) {
|
|
15
|
+
return typeof value === "object" && value !== null && typeof Reflect.get(value, "dispatch") === "function";
|
|
16
|
+
}
|
|
17
|
+
function hostDispatcher(callerApi) {
|
|
18
|
+
const own = callerApi === 2 ? slot(2) : undefined;
|
|
19
|
+
return own !== undefined ? { dispatcher: own, api: 2 } : { dispatcher: slot(1) ?? getGlobalDispatcher(), api: 1 };
|
|
20
|
+
}
|
|
21
|
+
function hostDispatchers() {
|
|
22
|
+
const out = [];
|
|
23
|
+
for (const d of [slot(2), slot(1), getGlobalDispatcher()])
|
|
24
|
+
if (d !== undefined && !out.includes(d))
|
|
25
|
+
out.push(d);
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
function hostMember(key) {
|
|
29
|
+
const hosts = hostDispatchers();
|
|
30
|
+
for (const host of hosts) {
|
|
31
|
+
const value = Reflect.get(host, key, host);
|
|
32
|
+
if (value)
|
|
33
|
+
return typeof value === "function" ? value.bind(host) : value;
|
|
34
|
+
}
|
|
35
|
+
const newest = hosts[0];
|
|
36
|
+
return newest === undefined ? undefined : Reflect.get(newest, key, newest);
|
|
37
|
+
}
|
|
38
|
+
const CALLER_API = Symbol("engine.callerDispatcherApi");
|
|
39
|
+
class EngineDispatcher extends Dispatcher {
|
|
40
|
+
dispatch(options, handler) {
|
|
41
|
+
const { [CALLER_API]: callerApi, ...rest } = options;
|
|
42
|
+
const host = hostDispatcher(callerApi ?? handlerApi(handler));
|
|
43
|
+
return host.dispatcher.dispatch({ ...rest, headersTimeout: 0, bodyTimeout: 0 }, inSlotApi(host.api, handler));
|
|
44
|
+
}
|
|
45
|
+
compose(...interceptors) {
|
|
46
|
+
const composed = super.compose(...interceptors);
|
|
47
|
+
const chain = composed.dispatch;
|
|
48
|
+
const enter = (options, handler) => {
|
|
49
|
+
const stamped = { ...options, [CALLER_API]: options[CALLER_API] ?? handlerApi(handler) };
|
|
50
|
+
return chain(stamped, handler);
|
|
51
|
+
};
|
|
52
|
+
return new Proxy(composed, { get: (target, key, receiver) => (key === "dispatch" ? enter : Reflect.get(target, key, receiver)) });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const SENDING_MEMBERS = new Set(Reflect.ownKeys(Dispatcher.prototype).filter((key) => key !== "constructor" && key !== "close" && key !== "destroy"));
|
|
56
|
+
export const ENGINE_DISPATCHER = new Proxy(new EngineDispatcher(), {
|
|
57
|
+
get: (engine, key, standIn) => (SENDING_MEMBERS.has(key) ? Reflect.get(engine, key, standIn) : hostMember(key)),
|
|
58
|
+
});
|
|
59
|
+
export function resolveFetch(config) {
|
|
60
|
+
const inner = config.fetchImpl ?? fetch;
|
|
61
|
+
return (input, init) => inner(input, { ...init, dispatcher: ENGINE_DISPATCHER });
|
|
62
|
+
}
|
|
@@ -153,9 +153,10 @@ export interface BackgroundRetainLanes {
|
|
|
153
153
|
}
|
|
154
154
|
/**
|
|
155
155
|
* THE retain predicate — "is this background process kept alive past the automatic reaps?" Every face that
|
|
156
|
-
* answers it calls this function (Bash description + receipts, Monitor receipt, the row stamp, teardown settle
|
|
157
|
-
*
|
|
158
|
-
* unless the owning isolated env is being destroyed.
|
|
156
|
+
* answers it calls this function (Bash description + receipts, Monitor receipt, the row stamp, teardown settle,
|
|
157
|
+
* the run-boundary dispose at teardown and park-only parks, session reap, the exit hard face, the listing).
|
|
158
|
+
* Any declared lane retains, unless the owning isolated env is being destroyed.
|
|
159
|
+
* Contract: docs/background-process-lifetime.md.
|
|
159
160
|
*/
|
|
160
161
|
export declare function retainsBackgroundProcesses(lanes: BackgroundRetainLanes): boolean;
|
|
161
162
|
/** The env lane of {@link BackgroundRetainLanes}, read off an env (false for an env without the capability). */
|
|
@@ -192,12 +193,27 @@ export declare function sweepBackgroundShells(env: ExecutionEnv & BackgroundShel
|
|
|
192
193
|
except?: readonly string[];
|
|
193
194
|
}): Promise<void>;
|
|
194
195
|
/**
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
196
|
+
* THE env-wide dispose at a run boundary: the run's teardown (every exit path) and the park-only park (a
|
|
197
|
+
* durable park that does not pause the env — its workspace, and the processes on it, persist on the target).
|
|
198
|
+
* One rule for "which background shells does this boundary take":
|
|
199
|
+
*
|
|
200
|
+
* - nothing, when THE retain predicate holds for the lanes the boundary sees (the env's declaration, the
|
|
201
|
+
* run's `TaskSpec` lane; `envDying` overrides both);
|
|
202
|
+
* - otherwise every registered shell row still running on the env is spared (`runningShellIds`, any owner):
|
|
203
|
+
* what the run's own settle kept (session-resident rows, in-window Monitors, retained rows) and every other
|
|
204
|
+
* run's or session's rows sharing the env. Unregistered env shells are disposed;
|
|
205
|
+
* - `envDying` (an isolated owned env destroyed with this run) spares nothing: its processes die with it.
|
|
206
|
+
*
|
|
207
|
+
* A park and a teardown ask the same question because a park-only park IS the run ending for now: the run's
|
|
208
|
+
* teardown follows every park, its settle collects the run's own rows that end with it, and its dispose
|
|
209
|
+
* spares the rest — a park that took more would take what the run's end keeps (a session's loop, killed
|
|
210
|
+
* while its approval card is on screen). The sweep before a snapshot pause is a different question (the VM
|
|
211
|
+
* pause must not capture detached jobs; resume would revive untracked zombies), which is why those sites
|
|
212
|
+
* call {@link sweepBackgroundShells} with no keep-list.
|
|
198
213
|
*/
|
|
199
|
-
export declare function
|
|
200
|
-
|
|
214
|
+
export declare function sweepRunBoundaryBackgroundShells(env: ExecutionEnv & BackgroundShellCapability, registry: EnvSweepAttribution & {
|
|
215
|
+
runningShellIds(env: unknown): readonly string[];
|
|
201
216
|
}, lanes: {
|
|
202
217
|
task: boolean;
|
|
218
|
+
envDying?: boolean;
|
|
203
219
|
}): Promise<void>;
|
|
@@ -33,9 +33,10 @@ export async function sweepBackgroundShells(env, attribution, opts) {
|
|
|
33
33
|
}
|
|
34
34
|
await env.disposeBackgroundShells(opts?.except !== undefined ? { except: opts.except } : undefined);
|
|
35
35
|
}
|
|
36
|
-
export async function
|
|
37
|
-
|
|
36
|
+
export async function sweepRunBoundaryBackgroundShells(env, registry, lanes) {
|
|
37
|
+
const envDying = lanes.envDying === true;
|
|
38
|
+
if (retainsBackgroundProcesses({ env: envDeclaresRetain(env), task: lanes.task, envDying }))
|
|
38
39
|
return;
|
|
39
|
-
const except = registry.
|
|
40
|
+
const except = envDying ? [] : registry.runningShellIds(env);
|
|
40
41
|
await sweepBackgroundShells(env, registry, except.length > 0 ? { except } : undefined);
|
|
41
42
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { hasBackgroundShell, sweepBackgroundShells,
|
|
1
|
+
import { hasBackgroundShell, sweepBackgroundShells, sweepRunBoundaryBackgroundShells } from "../background-shell.js";
|
|
2
2
|
import { checkpointVersionFor, debitLedger, mintCheckpointId, mintCheckpointToken, } from "../checkpoint-store.js";
|
|
3
3
|
import { defaultTaskRegistry } from "../task-registry.js";
|
|
4
4
|
import { checkpointScopeOf } from "./checkpoint-scope.js";
|
|
@@ -37,7 +37,7 @@ export function prepareBoundaryParks(input) {
|
|
|
37
37
|
}
|
|
38
38
|
else if (parkOnlyRemoteEnv !== undefined) {
|
|
39
39
|
if (hasBackgroundShell(parkOnlyRemoteEnv))
|
|
40
|
-
await
|
|
40
|
+
await sweepRunBoundaryBackgroundShells(parkOnlyRemoteEnv, defaultTaskRegistry, { task: spec.retainBackgroundProcesses === true });
|
|
41
41
|
remoteHandle = parkOnlyHandle(parkOnlyRemoteEnv);
|
|
42
42
|
}
|
|
43
43
|
const token = mintCheckpointToken();
|
|
@@ -120,7 +120,7 @@ export function prepareBoundaryParks(input) {
|
|
|
120
120
|
}
|
|
121
121
|
else if (parkOnlyRemoteEnv !== undefined) {
|
|
122
122
|
if (hasBackgroundShell(parkOnlyRemoteEnv))
|
|
123
|
-
await
|
|
123
|
+
await sweepRunBoundaryBackgroundShells(parkOnlyRemoteEnv, defaultTaskRegistry, { task: spec.retainBackgroundProcesses === true });
|
|
124
124
|
remoteHandle = parkOnlyHandle(parkOnlyRemoteEnv);
|
|
125
125
|
}
|
|
126
126
|
const token = mintCheckpointToken();
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { uuidv7 } from "../../engine/session/uuid.js";
|
|
2
2
|
import { Value } from "typebox/value";
|
|
3
3
|
import { ASK_USER_QUESTION_TOOL_NAME, classifyQuestionOutcome, validateAskQuestions } from "../ask-question.js";
|
|
4
|
-
import { hasBackgroundShell, sweepBackgroundShells,
|
|
4
|
+
import { hasBackgroundShell, sweepBackgroundShells, sweepRunBoundaryBackgroundShells } from "../background-shell.js";
|
|
5
5
|
import { boundInputHashOf } from "../canonical-json.js";
|
|
6
6
|
import { buildRiskDescriptor, checkpointVersionFor, debitLedger, encodeAtFidelity, mintCheckpointId, mintCheckpointToken, resolveDeclaredFidelity, samePlainValue, } from "../checkpoint-store.js";
|
|
7
7
|
import { pauseOf } from "../pause-registry.js";
|
|
@@ -230,7 +230,7 @@ export function prepareParkAsk(input) {
|
|
|
230
230
|
}
|
|
231
231
|
else if (parkOnlyRemoteEnv !== undefined) {
|
|
232
232
|
if (hasBackgroundShell(parkOnlyRemoteEnv))
|
|
233
|
-
await
|
|
233
|
+
await sweepRunBoundaryBackgroundShells(parkOnlyRemoteEnv, defaultTaskRegistry, { task: spec.retainBackgroundProcesses === true });
|
|
234
234
|
remoteHandle = parkOnlyHandle(parkOnlyRemoteEnv);
|
|
235
235
|
}
|
|
236
236
|
token = mintCheckpointToken();
|
|
@@ -13,7 +13,7 @@ import { isValidReminderMark, mintReminderMark } from "../reminder-mint.js";
|
|
|
13
13
|
import { refuseInvalidApproverPosture } from "../auto-mode-directive.js";
|
|
14
14
|
import { emitTrace } from "../trace.js";
|
|
15
15
|
import { reservedCollisions, reservedFor } from "../../brain/request-params.js";
|
|
16
|
-
import {
|
|
16
|
+
import { STALL_FIRST_BYTE_MS, STALL_FIRST_TOKEN_MS, STALL_IDLE_MS, withBrainCallGuardrail } from "../../brain/timeout.js";
|
|
17
17
|
import { OFFLOAD_TOOL_NAME } from "../tool-result-store.js";
|
|
18
18
|
import { OUTPUT_TOOL_NAME, createOutputTool } from "./synthetic-tools.js";
|
|
19
19
|
import { compileOutputSchema } from "./strict-output-schema.js";
|
|
@@ -649,7 +649,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
649
649
|
: {}),
|
|
650
650
|
stallTimeoutsPerCall: () => {
|
|
651
651
|
callIssuedAtRef.current = Date.now();
|
|
652
|
-
return {
|
|
652
|
+
return { firstByteMs: STALL_FIRST_BYTE_MS, firstTokenMs: STALL_FIRST_TOKEN_MS, idleMs: STALL_IDLE_MS };
|
|
653
653
|
},
|
|
654
654
|
...(spec.resilience !== undefined ? { resilience: spec.resilience } : {}),
|
|
655
655
|
loopTrace: (step) => {
|
|
@@ -2,7 +2,7 @@ import { MAX_CONSECUTIVE_COMPACTION_FAILURES } from "./compaction-knobs.js";
|
|
|
2
2
|
import { isSyntheticApiErrorMessage } from "../../internal/harness.js";
|
|
3
3
|
import { adjudicateDerivedRoute, fallbackToPrimaryNotice, sameRouteIdentity } from "../../brain/route-adjudicator.js";
|
|
4
4
|
import { isDegenerateCutMessage } from "../../brain/terminal-cause.js";
|
|
5
|
-
import { hasBackgroundShell,
|
|
5
|
+
import { hasBackgroundShell, sweepRunBoundaryBackgroundShells } from "../background-shell.js";
|
|
6
6
|
import { resolveCheckpointStore } from "../checkpoint-store.js";
|
|
7
7
|
import { hookSeatExpiredError, runHookSeat } from "../hooks.js";
|
|
8
8
|
import { modelCostToPricing } from "../pricing.js";
|
|
@@ -315,11 +315,9 @@ export async function runSettleAndTeardown(input) {
|
|
|
315
315
|
retainProcesses: spec.retainBackgroundProcesses === true,
|
|
316
316
|
}), "taskRegistry.settleKilledForOwner", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId }));
|
|
317
317
|
const bgEnv = prepared.ownedEnv ?? runner.deps.executionEnv;
|
|
318
|
-
|
|
319
|
-
if (bgEnv && hasBackgroundShell(bgEnv) && !retainBgBySpec) {
|
|
318
|
+
if (bgEnv && hasBackgroundShell(bgEnv)) {
|
|
320
319
|
try {
|
|
321
|
-
|
|
322
|
-
await settleTeardownLeg(() => sweepBackgroundShells(bgEnv, defaultTaskRegistry, keepAlive.length > 0 ? { except: keepAlive } : undefined), "sweepBackgroundShells", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId }));
|
|
320
|
+
await settleTeardownLeg(() => sweepRunBoundaryBackgroundShells(bgEnv, defaultTaskRegistry, { task: spec.retainBackgroundProcesses === true, envDying: ownedEnvDying }), "sweepBackgroundShells", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId }));
|
|
323
321
|
}
|
|
324
322
|
catch (e) {
|
|
325
323
|
runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId });
|
|
@@ -141,12 +141,14 @@ export interface BrainStatus {
|
|
|
141
141
|
* approximate across a clock adjustment — {@link retryInMs} stays the authoritative RELATIVE quantity.
|
|
142
142
|
*/
|
|
143
143
|
retryAtMs?: number;
|
|
144
|
-
/** On a `waiting_first_token` frame: how long the
|
|
145
|
-
*
|
|
146
|
-
*
|
|
144
|
+
/** On a `waiting_first_token` frame: how long the current wait has lasted, ms, at the moment the
|
|
145
|
+
* frame is emitted (the emitting process's clock; approximate across a hop). The wait is one
|
|
146
|
+
* segment's: the response-header wait before headers arrive, the first-token wait after — each
|
|
147
|
+
* starts its own count. Absent on every other phase. */
|
|
147
148
|
elapsedMs?: number;
|
|
148
|
-
/** On a `waiting_first_token` frame: the
|
|
149
|
-
*
|
|
149
|
+
/** On a `waiting_first_token` frame: the ceiling of the segment being waited on, ms — the armed
|
|
150
|
+
* first-byte window before headers, the armed first-token watchdog after — so a consumer can render
|
|
151
|
+
* "45s of 300s". Absent on every other phase. */
|
|
150
152
|
timeoutMs?: number;
|
|
151
153
|
/** 1-based index of the attempt that just failed (the wait precedes attempt `attempt + 1`);
|
|
152
154
|
* same numbering as the `brain.retry` telemetry frame. Absent on frames that are not a retry wait. */
|
|
@@ -140,7 +140,7 @@ export interface BackgroundBashTaskHandle extends SemaTaskHandle {
|
|
|
140
140
|
* field reads as not marked. The row is the delivered daemon's ONLY management surface — while the process runs it survives the
|
|
141
141
|
* per-run evict (TaskOutput/TaskStop resolve through this map; the watcher must stay live to mint
|
|
142
142
|
* the natural-exit receipt) and rides every later run's dispose except-list on the same env
|
|
143
|
-
* ({@link TaskRegistry.
|
|
143
|
+
* ({@link TaskRegistry.runningShellIds}). A terminal row evicts normally. Terminals: explicit
|
|
144
144
|
* TaskStop, the process's own exit (watcher settle), or env destruction (watcher env-death settle)
|
|
145
145
|
* — deliberately NO timed reaper (the deliverable is the living process). A retained SESSION-resident
|
|
146
146
|
* row is likewise exempt from the session-release reap ({@link TaskRegistry.reapSessionBackground}):
|
|
@@ -386,8 +386,7 @@ export declare class TaskRegistry {
|
|
|
386
386
|
/** Background output-file read exemption: true iff `canonicalPath` is EXACTLY the registered output
|
|
387
387
|
* file of a background bash row that (a) the caller can access (same owner/scope/session visibility as
|
|
388
388
|
* a TaskOutput poll — {@link canAccess}) and (b) was registered on the caller's OWN execution env —
|
|
389
|
-
* `env` identity is part of the match, same discipline as {@link
|
|
390
|
-
* {@link runningShellIds}. A canonical path only names a file within ONE env's filesystem
|
|
389
|
+
* `env` identity is part of the match, same discipline as {@link runningShellIds}. A canonical path only names a file within ONE env's filesystem
|
|
391
390
|
* namespace: on a shared registry, a row minted on env A must never exempt a same-named outside-root
|
|
392
391
|
* file in env B's namespace (same session or not), so instance equality — never a string proxy — is
|
|
393
392
|
* required. The output file is minted by the ENGINE itself (createTempFile
|
|
@@ -610,10 +609,8 @@ export declare class TaskRegistry {
|
|
|
610
609
|
* completion notification at the terminal state. `unref()`d — a watcher never keeps the process alive.
|
|
611
610
|
* Stops itself on terminal, on a vanished shell (env disposed), or when the handle is evicted. */
|
|
612
611
|
private startBashWatcher;
|
|
613
|
-
/**
|
|
614
|
-
*
|
|
615
|
-
retainedShellIds(env: unknown): BackgroundShellId[];
|
|
616
|
-
/** Every shell-lane row still RUNNING on this env, any owner — the run teardown's dispose except-list.
|
|
612
|
+
/** Every shell-lane row still RUNNING on this env, any owner — the run-boundary dispose's except-list (the
|
|
613
|
+
* run teardown and the park-only parks: `sweepRunBoundaryBackgroundShells`).
|
|
617
614
|
* A run's teardown disposes only shells it can attribute to itself: its settle sweep has already killed
|
|
618
615
|
* and settled the rows of this run that end with it, so whatever registered row is still running here is
|
|
619
616
|
* either something that settle kept (session-resident, an in-window Monitor, retained) or belongs to
|
|
@@ -835,15 +835,6 @@ export class TaskRegistry {
|
|
|
835
835
|
handle.watcher = setInterval(() => void tick(), 1_000);
|
|
836
836
|
handle.watcher.unref?.();
|
|
837
837
|
}
|
|
838
|
-
retainedShellIds(env) {
|
|
839
|
-
const out = [];
|
|
840
|
-
for (const handle of this.handles.values()) {
|
|
841
|
-
if ((handle.type === "background_bash" || handle.type === "monitor") && handle.status === "running" && shellRowRetained(handle) && handle.env === env) {
|
|
842
|
-
out.push(handle.shellId);
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
return out;
|
|
846
|
-
}
|
|
847
838
|
runningShellIds(env) {
|
|
848
839
|
const out = [];
|
|
849
840
|
for (const handle of this.handles.values()) {
|
|
@@ -39,12 +39,12 @@ export interface ResilienceOptions {
|
|
|
39
39
|
* exists to protect the provider; circumvention only where true failure shape matters (benchmarks). */
|
|
40
40
|
bypassBreaker?: boolean;
|
|
41
41
|
}
|
|
42
|
-
/** 1.296 件A — per-call advisory stall-watchdog values (see {@link StreamOptions.stallTimeouts})
|
|
42
|
+
/** 1.296 件A — per-call advisory stall-watchdog values (see {@link StreamOptions.stallTimeouts}); the
|
|
43
|
+
* connect clock is the transport's, fixed at brain construction, so it is not among them. */
|
|
43
44
|
export interface StallTimeouts {
|
|
44
|
-
/** Advisory
|
|
45
|
-
*
|
|
46
|
-
|
|
47
|
-
connectMs?: number;
|
|
45
|
+
/** Advisory first-byte watchdog (ms) — bounds the fetch→response-headers wait; fires as a retryable
|
|
46
|
+
* `stall` out of its own small budget. */
|
|
47
|
+
firstByteMs?: number;
|
|
48
48
|
/** Advisory first-token watchdog (ms) — first thinking delta counts as the first token. */
|
|
49
49
|
firstTokenMs?: number;
|
|
50
50
|
/** Advisory mid-stream idle watchdog (ms) — re-armed on every delta after the first. */
|