@sema-agent/core 7.32.1 → 7.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/brain/anthropic.d.ts +1 -1
- package/dist/brain/anthropic.js +4 -1
- package/dist/brain/open-responses.d.ts +2 -1
- package/dist/brain/open-responses.js +4 -1
- package/dist/brain/openai.d.ts +2 -1
- package/dist/brain/openai.js +4 -1
- package/dist/brain/stream-engine.d.ts +8 -7
- package/dist/brain/stream-engine.js +111 -44
- package/dist/brain/timeout.d.ts +112 -53
- package/dist/brain/timeout.js +39 -5
- package/dist/brain/transport.d.ts +13 -0
- package/dist/brain/transport.js +62 -0
- package/dist/core/runner/prepare-task.js +2 -2
- package/dist/core/task-event.d.ts +7 -5
- package/dist/core/task-registry.d.ts +1 -7
- package/dist/core/task-registry.js +0 -9
- package/dist/engine/llm/types.d.ts +5 -5
- package/dist/orchestration/workflow.d.ts +4 -4
- package/dist/orchestration/workflow.js +2 -2
- package/package.json +6 -3
- package/wire-changes/core@7.33.0.json +351 -0
package/dist/brain/timeout.d.ts
CHANGED
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
import type { StreamFn } from "../internal/llm.js";
|
|
2
2
|
/**
|
|
3
|
-
* Tiered timeouts (
|
|
4
|
-
* it
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* Tiered timeouts (design/17 §5B; re-cut along the segments of a call in 7.33.0). The task-level
|
|
4
|
+
* `timeoutSec` only bounds the *whole* run; it cannot tell apart a gateway that never completes the
|
|
5
|
+
* handshake, one that accepts the request and then holds the response headers while it works, one
|
|
6
|
+
* that opens the stream and never emits a token, and one that goes quiet mid-answer. Four clocks,
|
|
7
|
+
* one per segment, each named for the segment it bounds:
|
|
7
8
|
*
|
|
8
|
-
* - **connect**: `fetch`
|
|
9
|
-
*
|
|
9
|
+
* - **connect**: TCP + TLS establishment inside `fetch` — the transport's own clock (undici's 10s by
|
|
10
|
+
* default), tuned on the process dispatcher the host installs; not a knob here (`transport.ts`).
|
|
11
|
+
* - **first-byte**: the `fetch()` call → response headers. A stall here is this process cancelling
|
|
12
|
+
* its own request at a deadline it chose, and is announced as such.
|
|
13
|
+
* - **first-token**: headers → the first content delta (a thinking delta counts).
|
|
14
|
+
* - **idle**: silence between deltas once output has started.
|
|
10
15
|
*
|
|
11
|
-
*
|
|
16
|
+
* Every field resolves through the same chain: construction config (an explicit `0` is OFF) >
|
|
17
|
+
* per-call advisory (`StreamOptions.stallTimeouts`) > the engine default constant below.
|
|
12
18
|
*/
|
|
13
19
|
/**
|
|
14
20
|
* The largest delay a host timer can actually wait for. Past this `setTimeout` does not schedule far in
|
|
@@ -28,55 +34,107 @@ export declare const MAX_TIMER_DELAY_MS = 2147483647;
|
|
|
28
34
|
*/
|
|
29
35
|
export declare function resolveStallTimeoutMs(value: number | undefined, knob: string): number | undefined;
|
|
30
36
|
/**
|
|
31
|
-
* The engine's
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
37
|
+
* The engine's DEFAULT watchdog values, one per segment of a model call — the third and last level of
|
|
38
|
+
* the chain every brain reads (`BrainTimeoutConfig` field > per-call advisory `StreamOptions.stallTimeouts`
|
|
39
|
+
* > these). The Runner supplies these same numbers as the advisory on every call, so a Brain supplied by
|
|
40
|
+
* the deployment (BYOM) is told what the engine would arm; a first-party brain arms them itself when
|
|
41
|
+
* neither a config field nor an advisory names the segment. A silent call is therefore always bounded by
|
|
42
|
+
* something finer than the hour-scale {@link BRAIN_CALL_GUARDRAIL_DEFAULT_MS} (a measured production
|
|
43
|
+
* frame once sat 680s with ZERO deltas) unless a deployment switched the watchdog for that segment off
|
|
44
|
+
* by name — an explicit `0` is the only spelling of OFF. The TCP + TLS connect has no row here: it is
|
|
45
|
+
* the transport's own clock (`transport.ts`), not a segment the engine can observe. 量纲:
|
|
46
|
+
* - first-byte 300s: the `fetch()` call → response headers. A self-hosted gateway that runs its
|
|
47
|
+
* prefill BEFORE writing headers is silent here for minutes and healthy; the 30s "connect" clock
|
|
48
|
+
* this replaces cut that prefill and re-sent the same body ten times. 300s is the reference
|
|
49
|
+
* implementation's window for a non-first-party route; the window grows with the request body
|
|
50
|
+
* ({@link firstByteWindow}) and the one retry is widened ({@link FIRST_BYTE_RETRY_WINDOW_FACTOR}).
|
|
41
51
|
* - first-token 600s: bounds the headers→first-delta wait, NOT the model's thought (the first
|
|
42
|
-
* THINKING delta already counts as the first token, design/17 O5
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* one a deployment gets by supplying no `fetchImpl` — undici destroys a body that has produced no
|
|
54
|
-
* bytes for 300s (`bodyTimeout`, default `300e3`), so byte-silence past 300s ends the attempt
|
|
55
|
-
* there and arrives as a read rejection reported `transport`. That report is CORRECT (the
|
|
56
|
-
* transport really did cut it) and the seat to change it exists: `fetchImpl` on both brain configs
|
|
57
|
-
* takes any `fetch`, so a deployment needing the full 600s passes one built on an undici
|
|
58
|
-
* `Agent`/`Dispatcher` with its own `bodyTimeout`. Named here so the next reader who measures ~300s
|
|
59
|
-
* of tolerance does not conclude this number is a lie.
|
|
60
|
-
* {@link import("../orchestration/workflow.js").WORKFLOW_AGENT_STALL_MS} is DERIVED from
|
|
61
|
-
* {@link STALL_CONNECT_MS} + this value, so the coarse agent-restart cannot sit UNDER a single
|
|
62
|
-
* silent provider call. Move this number and that one follows. It is a floor, not
|
|
63
|
-
* a guarantee of ordering — see that constant's own comment for the cases that still invert.
|
|
64
|
-
* - idle 90s: mid-stream silence after output started — a healthy stream emits steadily
|
|
65
|
-
* (comfortably above any provider keep-alive cadence).
|
|
52
|
+
* THINKING delta already counts as the first token, design/17 O5), so a slow reasoner is safe. A
|
|
53
|
+
* minutes-scale bound is FALSE for a self-hosted server working through a long prefill AFTER
|
|
54
|
+
* headers: it emits nothing for 2–3 minutes while healthy, and cancelling it burns the prefill and
|
|
55
|
+
* re-sends a request that burns it again. The default transport keeps NO clock over this segment
|
|
56
|
+
* (the request carries `bodyTimeout: 0`), so this number is the bound actually granted.
|
|
57
|
+
* {@link import("../orchestration/workflow.js").WORKFLOW_AGENT_STALL_MS} is DERIVED from the
|
|
58
|
+
* first-byte lane at {@link STALL_FIRST_BYTE_MS} ({@link firstByteLaneKnobMs}) + this value, so the coarse agent-restart cannot sit UNDER a single
|
|
59
|
+
* silent provider call (a floor, not a guarantee of ordering — see that constant's comment).
|
|
60
|
+
* - idle 300s: mid-stream silence after output started — the reference implementation's idle floor
|
|
61
|
+
* (a self-hosted server can stall for minutes between deltas while a server-side tool runs; a provider
|
|
62
|
+
* keep-alive comment is not a delta and does not count).
|
|
66
63
|
* ADVISORY: a brain whose construction-time {@link BrainTimeoutConfig} sets the corresponding field
|
|
67
64
|
* keeps it (deployment sovereignty; see the `StreamOptions.stallTimeouts` priority chain).
|
|
68
65
|
*/
|
|
69
|
-
export declare const
|
|
66
|
+
export declare const STALL_FIRST_BYTE_MS = 300000;
|
|
70
67
|
export declare const STALL_FIRST_TOKEN_MS = 600000;
|
|
71
|
-
export declare const STALL_IDLE_MS =
|
|
68
|
+
export declare const STALL_IDLE_MS = 300000;
|
|
69
|
+
/**
|
|
70
|
+
* The retired names of the header-wait bound, each with the name that replaced it: the construction key and
|
|
71
|
+
* the per-call advisory key. The type face no longer has them, but a type only refuses a LITERAL — a
|
|
72
|
+
* spread object (`{ ...timeouts }`) or a JS caller still delivers the key, and a knob that arrives and is
|
|
73
|
+
* then ignored is a deployment setting silently doing nothing. So their arrival is refused, naming the
|
|
74
|
+
* replacement: a tripwire, not a second spelling (nothing reads the value).
|
|
75
|
+
*/
|
|
76
|
+
export declare const RETIRED_TIMEOUT_KNOBS: {
|
|
77
|
+
readonly construction: {
|
|
78
|
+
readonly connectTimeoutMs: "firstByteTimeoutMs";
|
|
79
|
+
};
|
|
80
|
+
readonly advisory: {
|
|
81
|
+
readonly connectMs: "firstByteMs";
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
/** Refuse a retired timeout key present on `holder` (see {@link RETIRED_TIMEOUT_KNOBS}); `where` names the
|
|
85
|
+
* object in the message (`"createOpenAIBrain config"`, `"StreamOptions.stallTimeouts"`). */
|
|
86
|
+
export declare function refuseRetiredTimeoutKnobs(holder: object | undefined, level: keyof typeof RETIRED_TIMEOUT_KNOBS, where: string): void;
|
|
87
|
+
/** The first-byte window grows with the request body: one second per this many bytes, rounded up (the
|
|
88
|
+
* reference implementation's allowance) — a large context on a slow uplink is still UPLOADING when a
|
|
89
|
+
* fixed window would call the gateway silent. */
|
|
90
|
+
export declare const FIRST_BYTE_UPLOAD_BYTES_PER_SECOND = 32768;
|
|
91
|
+
/** A first-byte stall is re-sent at most this many times, out of its OWN budget rather than
|
|
92
|
+
* `maxRetries` (the reference implementation: "normally one retry, not the session budget"); the shared
|
|
93
|
+
* ladder counts transport failures, and this is not one. */
|
|
94
|
+
export declare const FIRST_BYTE_RETRY_BUDGET = 1;
|
|
95
|
+
/** The retry's knob is the first times this — the reference implementation widens its retry to a second
|
|
96
|
+
* knob's default (599s against a 300s first window); one knob here keeps that ratio instead. */
|
|
97
|
+
export declare const FIRST_BYTE_RETRY_WINDOW_FACTOR = 2;
|
|
98
|
+
/** The ONE resolution every watchdog knob reads: construction config (an explicit `0` is OFF) > per-call
|
|
99
|
+
* advisory > the engine default. A bad number at either level is refused ({@link resolveStallTimeoutMs});
|
|
100
|
+
* only an ABSENT value falls through. */
|
|
101
|
+
export declare function resolveWatchdogMs(configured: number | undefined, advisory: number | undefined, engineDefault: number, knob: string): number;
|
|
102
|
+
/** One armed first-byte window, with the two numbers its announcement quotes. */
|
|
103
|
+
export interface FirstByteWindow {
|
|
104
|
+
/** The knob's value for THIS attempt (the configured value, widened on the retry). */
|
|
105
|
+
knobMs: number;
|
|
106
|
+
/** The upload allowance added for the request body, ms (0 for an empty body). */
|
|
107
|
+
allowanceMs: number;
|
|
108
|
+
/** What is actually armed: `knobMs + allowanceMs`, capped at {@link MAX_TIMER_DELAY_MS}. `0` = off. */
|
|
109
|
+
windowMs: number;
|
|
110
|
+
}
|
|
111
|
+
/** The first-byte window for one attempt; `retryIndex` = first-byte stalls already suffered (0 first),
|
|
112
|
+
* each widening the knob by {@link FIRST_BYTE_RETRY_WINDOW_FACTOR}. A knob of `0` (OFF) stays off. */
|
|
113
|
+
export declare function firstByteWindow(knobMs: number | undefined, bodyBytes: number, retryIndex: number): FirstByteWindow;
|
|
114
|
+
/**
|
|
115
|
+
* The most a first-byte lane armed at `knobMs` waits on the gateway in total before it gives up: the probe
|
|
116
|
+
* window and every widened retry its budget allows ({@link FIRST_BYTE_RETRY_BUDGET},
|
|
117
|
+
* {@link FIRST_BYTE_RETRY_WINDOW_FACTOR}) — read off {@link firstByteWindow}, the same function that arms
|
|
118
|
+
* each attempt, so the sum cannot drift from what is armed. The knob part only: the upload allowance
|
|
119
|
+
* depends on the request body and the backoff between attempts is not a wait on the gateway, so the lane's
|
|
120
|
+
* real wall time is this plus both — a LOWER bound.
|
|
121
|
+
*/
|
|
122
|
+
export declare function firstByteLaneKnobMs(knobMs: number): number;
|
|
72
123
|
export interface BrainTimeoutConfig {
|
|
73
|
-
/**
|
|
74
|
-
|
|
124
|
+
/**
|
|
125
|
+
* `fetch()` call → response headers, ms: the window a gateway has to START answering (until 7.33.0 a
|
|
126
|
+
* knob named `connectTimeoutMs` bounded this wait; it is gone — the connect clock is the transport's). Absent ⇒
|
|
127
|
+
* {@link STALL_FIRST_BYTE_MS}; the armed window grows with the request body ({@link firstByteWindow});
|
|
128
|
+
* a stall earns ONE retry at a widened window out of its own budget, then fails naming this knob.
|
|
129
|
+
* Fires as a `stall` (this process cancelled its own request), never as a transport fault. `0` ⇒ off.
|
|
130
|
+
*/
|
|
131
|
+
firstByteTimeoutMs?: number;
|
|
75
132
|
/**
|
|
76
133
|
* After the stream opens, cancel it if no content delta (text / thinking / tool input) arrives
|
|
77
134
|
* within this many ms → `BrainError("network", "first-token timeout")`. For reasoning models the
|
|
78
135
|
* **first thinking delta also counts** as the first token, so a long-thinking model isn't killed
|
|
79
|
-
* for being slow to its first *visible* word (design/17 O5).
|
|
136
|
+
* for being slow to its first *visible* word (design/17 O5). Absent ⇒ {@link STALL_FIRST_TOKEN_MS};
|
|
137
|
+
* `0` ⇒ off.
|
|
80
138
|
*/
|
|
81
139
|
firstTokenTimeoutMs?: number;
|
|
82
140
|
/**
|
|
@@ -84,18 +142,19 @@ export interface BrainTimeoutConfig {
|
|
|
84
142
|
* After the first delta, re-armed on every delta; if the stream then goes silent for this many ms
|
|
85
143
|
* (a gateway that hung mid-output) → `reader.cancel()` + `BrainError("network", "stream idle timeout")`.
|
|
86
144
|
* Complements `firstTokenTimeoutMs`: that one tolerates a long *first* token (reasoning), this one
|
|
87
|
-
* catches a stall *after* output has started, where a healthy stream emits steadily.
|
|
145
|
+
* catches a stall *after* output has started, where a healthy stream emits steadily. Absent ⇒
|
|
146
|
+
* {@link STALL_IDLE_MS}; `0` ⇒ off.
|
|
88
147
|
*/
|
|
89
148
|
idleTimeoutMs?: number;
|
|
90
149
|
}
|
|
91
150
|
/**
|
|
92
|
-
* A per-
|
|
151
|
+
* A per-attempt abort controller that fires at the first-byte deadline while still forwarding an
|
|
93
152
|
* outer (task) abort. The returned `signal` governs the **whole** request including the streamed body
|
|
94
|
-
* — so once headers arrive, call `clearTimer()` (NOT `dispose()`) to stop the
|
|
95
|
-
*
|
|
96
|
-
*
|
|
153
|
+
* — so once headers arrive, call `clearTimer()` (NOT `dispose()`) to stop the deadline while keeping
|
|
154
|
+
* the outer-abort link alive for the body read; call `dispose()` only when the request is fully done
|
|
155
|
+
* or being discarded.
|
|
97
156
|
*/
|
|
98
|
-
export declare function
|
|
157
|
+
export declare function createAttemptController(firstByteWindowMs: number | undefined, outerSignal: AbortSignal | undefined): {
|
|
99
158
|
signal: AbortSignal;
|
|
100
159
|
timedOut: () => boolean;
|
|
101
160
|
clearTimer: () => void;
|
|
@@ -110,7 +169,7 @@ export declare function createConnectController(connectTimeoutMs: number | undef
|
|
|
110
169
|
* Default ceiling on how long the ENGINE waits for a single brain call (ruled 2026-08-03: one hour).
|
|
111
170
|
* Deliberately far above any legitimate call: this is a wedged-run backstop, not a latency policy —
|
|
112
171
|
* a deployment that wants a tight per-call bound sets the fine tier above ({@link BrainTimeoutConfig},
|
|
113
|
-
* or the advisory {@link
|
|
172
|
+
* or the advisory {@link STALL_FIRST_BYTE_MS} family the engine supplies by default). All of these clocks
|
|
114
173
|
* run in parallel with the task's own `limits.timeoutSec`; whichever expires first owns the run.
|
|
115
174
|
*/
|
|
116
175
|
export declare const BRAIN_CALL_GUARDRAIL_DEFAULT_MS = 3600000;
|
package/dist/brain/timeout.js
CHANGED
|
@@ -14,10 +14,44 @@ export function resolveStallTimeoutMs(value, knob) {
|
|
|
14
14
|
}
|
|
15
15
|
return value;
|
|
16
16
|
}
|
|
17
|
-
export const
|
|
17
|
+
export const STALL_FIRST_BYTE_MS = 300_000;
|
|
18
18
|
export const STALL_FIRST_TOKEN_MS = 600_000;
|
|
19
|
-
export const STALL_IDLE_MS =
|
|
20
|
-
export
|
|
19
|
+
export const STALL_IDLE_MS = 300_000;
|
|
20
|
+
export const RETIRED_TIMEOUT_KNOBS = {
|
|
21
|
+
construction: { connectTimeoutMs: "firstByteTimeoutMs" },
|
|
22
|
+
advisory: { connectMs: "firstByteMs" },
|
|
23
|
+
};
|
|
24
|
+
export function refuseRetiredTimeoutKnobs(holder, level, where) {
|
|
25
|
+
if (holder === undefined || holder === null)
|
|
26
|
+
return;
|
|
27
|
+
for (const [retired, replacement] of Object.entries(RETIRED_TIMEOUT_KNOBS[level])) {
|
|
28
|
+
if (!(retired in holder))
|
|
29
|
+
continue;
|
|
30
|
+
const e = new Error(`${where}: \`${retired}\` was removed in 7.33.0 and is refused rather than ignored — the fetch()→response-headers wait it bounded is \`${replacement}\` (0 = off); the TCP + TLS connect is the transport's own clock, tuned on the process dispatcher (setGlobalDispatcher(new Agent({ connect: { timeout } })))`);
|
|
31
|
+
e.code = "config.stall_timeout_retired";
|
|
32
|
+
throw e;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export const FIRST_BYTE_UPLOAD_BYTES_PER_SECOND = 32_768;
|
|
36
|
+
export const FIRST_BYTE_RETRY_BUDGET = 1;
|
|
37
|
+
export const FIRST_BYTE_RETRY_WINDOW_FACTOR = 2;
|
|
38
|
+
export function resolveWatchdogMs(configured, advisory, engineDefault, knob) {
|
|
39
|
+
return resolveStallTimeoutMs(configured ?? advisory, knob) ?? engineDefault;
|
|
40
|
+
}
|
|
41
|
+
export function firstByteWindow(knobMs, bodyBytes, retryIndex) {
|
|
42
|
+
if (knobMs === undefined || knobMs <= 0)
|
|
43
|
+
return { knobMs: 0, allowanceMs: 0, windowMs: 0 };
|
|
44
|
+
const widened = knobMs * FIRST_BYTE_RETRY_WINDOW_FACTOR ** Math.max(0, retryIndex);
|
|
45
|
+
const allowanceMs = Math.ceil(Math.max(0, bodyBytes) / FIRST_BYTE_UPLOAD_BYTES_PER_SECOND) * 1000;
|
|
46
|
+
return { knobMs: widened, allowanceMs, windowMs: Math.min(widened + allowanceMs, MAX_TIMER_DELAY_MS) };
|
|
47
|
+
}
|
|
48
|
+
export function firstByteLaneKnobMs(knobMs) {
|
|
49
|
+
let total = 0;
|
|
50
|
+
for (let retryIndex = 0; retryIndex <= FIRST_BYTE_RETRY_BUDGET; retryIndex++)
|
|
51
|
+
total += firstByteWindow(knobMs, 0, retryIndex).knobMs;
|
|
52
|
+
return total;
|
|
53
|
+
}
|
|
54
|
+
export function createAttemptController(firstByteWindowMs, outerSignal) {
|
|
21
55
|
const ac = new AbortController();
|
|
22
56
|
let timedOut = false;
|
|
23
57
|
const onAbort = () => ac.abort(outerSignal?.reason);
|
|
@@ -28,11 +62,11 @@ export function createConnectController(connectTimeoutMs, outerSignal) {
|
|
|
28
62
|
outerSignal.addEventListener("abort", onAbort, { once: true });
|
|
29
63
|
}
|
|
30
64
|
let timer;
|
|
31
|
-
if (
|
|
65
|
+
if (firstByteWindowMs && firstByteWindowMs > 0) {
|
|
32
66
|
timer = setTimeout(() => {
|
|
33
67
|
timedOut = true;
|
|
34
68
|
ac.abort();
|
|
35
|
-
},
|
|
69
|
+
}, firstByteWindowMs);
|
|
36
70
|
}
|
|
37
71
|
const clearTimer = () => {
|
|
38
72
|
if (timer)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Dispatcher } from "undici";
|
|
2
|
+
/**
|
|
3
|
+
* The one dispatcher every first-party brain hands its fetch: the sending members are the engine's
|
|
4
|
+
* ({@link SENDING_MEMBERS}); every other member — state, lifecycle, events — is the host's, read when
|
|
5
|
+
* accessed ({@link hostMember}), so nothing a fetch starts reading has to be added to a list here.
|
|
6
|
+
* Stateless; owns no sockets.
|
|
7
|
+
*/
|
|
8
|
+
export declare const ENGINE_DISPATCHER: Dispatcher;
|
|
9
|
+
/** The `fetch` a brain sends its requests through: the caller's `fetchImpl` (else Node's `fetch`), each
|
|
10
|
+
* request carrying {@link ENGINE_DISPATCHER} in its init. */
|
|
11
|
+
export declare function resolveFetch(config: {
|
|
12
|
+
fetchImpl?: typeof fetch;
|
|
13
|
+
}): typeof fetch;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Dispatcher, getGlobalDispatcher } from "undici";
|
|
2
|
+
import UnwrapHandler from "undici/lib/handler/unwrap-handler.js";
|
|
3
|
+
import WrapHandler from "undici/lib/handler/wrap-handler.js";
|
|
4
|
+
function handlerApi(handler) {
|
|
5
|
+
return typeof handler.onRequestStart === "function" ? 2 : 1;
|
|
6
|
+
}
|
|
7
|
+
function inSlotApi(api, handler) {
|
|
8
|
+
return api === 1 ? UnwrapHandler.unwrap(handler) : WrapHandler.wrap(handler);
|
|
9
|
+
}
|
|
10
|
+
function slot(version) {
|
|
11
|
+
const held = Reflect.get(globalThis, Symbol.for(`undici.globalDispatcher.${String(version)}`));
|
|
12
|
+
return isDispatcher(held) ? held : undefined;
|
|
13
|
+
}
|
|
14
|
+
function isDispatcher(value) {
|
|
15
|
+
return typeof value === "object" && value !== null && typeof Reflect.get(value, "dispatch") === "function";
|
|
16
|
+
}
|
|
17
|
+
function hostDispatcher(callerApi) {
|
|
18
|
+
const own = callerApi === 2 ? slot(2) : undefined;
|
|
19
|
+
return own !== undefined ? { dispatcher: own, api: 2 } : { dispatcher: slot(1) ?? getGlobalDispatcher(), api: 1 };
|
|
20
|
+
}
|
|
21
|
+
function hostDispatchers() {
|
|
22
|
+
const out = [];
|
|
23
|
+
for (const d of [slot(2), slot(1), getGlobalDispatcher()])
|
|
24
|
+
if (d !== undefined && !out.includes(d))
|
|
25
|
+
out.push(d);
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
function hostMember(key) {
|
|
29
|
+
const hosts = hostDispatchers();
|
|
30
|
+
for (const host of hosts) {
|
|
31
|
+
const value = Reflect.get(host, key, host);
|
|
32
|
+
if (value)
|
|
33
|
+
return typeof value === "function" ? value.bind(host) : value;
|
|
34
|
+
}
|
|
35
|
+
const newest = hosts[0];
|
|
36
|
+
return newest === undefined ? undefined : Reflect.get(newest, key, newest);
|
|
37
|
+
}
|
|
38
|
+
const CALLER_API = Symbol("engine.callerDispatcherApi");
|
|
39
|
+
class EngineDispatcher extends Dispatcher {
|
|
40
|
+
dispatch(options, handler) {
|
|
41
|
+
const { [CALLER_API]: callerApi, ...rest } = options;
|
|
42
|
+
const host = hostDispatcher(callerApi ?? handlerApi(handler));
|
|
43
|
+
return host.dispatcher.dispatch({ ...rest, headersTimeout: 0, bodyTimeout: 0 }, inSlotApi(host.api, handler));
|
|
44
|
+
}
|
|
45
|
+
compose(...interceptors) {
|
|
46
|
+
const composed = super.compose(...interceptors);
|
|
47
|
+
const chain = composed.dispatch;
|
|
48
|
+
const enter = (options, handler) => {
|
|
49
|
+
const stamped = { ...options, [CALLER_API]: options[CALLER_API] ?? handlerApi(handler) };
|
|
50
|
+
return chain(stamped, handler);
|
|
51
|
+
};
|
|
52
|
+
return new Proxy(composed, { get: (target, key, receiver) => (key === "dispatch" ? enter : Reflect.get(target, key, receiver)) });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const SENDING_MEMBERS = new Set(Reflect.ownKeys(Dispatcher.prototype).filter((key) => key !== "constructor" && key !== "close" && key !== "destroy"));
|
|
56
|
+
export const ENGINE_DISPATCHER = new Proxy(new EngineDispatcher(), {
|
|
57
|
+
get: (engine, key, standIn) => (SENDING_MEMBERS.has(key) ? Reflect.get(engine, key, standIn) : hostMember(key)),
|
|
58
|
+
});
|
|
59
|
+
export function resolveFetch(config) {
|
|
60
|
+
const inner = config.fetchImpl ?? fetch;
|
|
61
|
+
return (input, init) => inner(input, { ...init, dispatcher: ENGINE_DISPATCHER });
|
|
62
|
+
}
|
|
@@ -13,7 +13,7 @@ import { isValidReminderMark, mintReminderMark } from "../reminder-mint.js";
|
|
|
13
13
|
import { refuseInvalidApproverPosture } from "../auto-mode-directive.js";
|
|
14
14
|
import { emitTrace } from "../trace.js";
|
|
15
15
|
import { reservedCollisions, reservedFor } from "../../brain/request-params.js";
|
|
16
|
-
import {
|
|
16
|
+
import { STALL_FIRST_BYTE_MS, STALL_FIRST_TOKEN_MS, STALL_IDLE_MS, withBrainCallGuardrail } from "../../brain/timeout.js";
|
|
17
17
|
import { OFFLOAD_TOOL_NAME } from "../tool-result-store.js";
|
|
18
18
|
import { OUTPUT_TOOL_NAME, createOutputTool } from "./synthetic-tools.js";
|
|
19
19
|
import { compileOutputSchema } from "./strict-output-schema.js";
|
|
@@ -649,7 +649,7 @@ export async function prepareTask(spec, deps, sessions, resume, internals, runne
|
|
|
649
649
|
: {}),
|
|
650
650
|
stallTimeoutsPerCall: () => {
|
|
651
651
|
callIssuedAtRef.current = Date.now();
|
|
652
|
-
return {
|
|
652
|
+
return { firstByteMs: STALL_FIRST_BYTE_MS, firstTokenMs: STALL_FIRST_TOKEN_MS, idleMs: STALL_IDLE_MS };
|
|
653
653
|
},
|
|
654
654
|
...(spec.resilience !== undefined ? { resilience: spec.resilience } : {}),
|
|
655
655
|
loopTrace: (step) => {
|
|
@@ -141,12 +141,14 @@ export interface BrainStatus {
|
|
|
141
141
|
* approximate across a clock adjustment — {@link retryInMs} stays the authoritative RELATIVE quantity.
|
|
142
142
|
*/
|
|
143
143
|
retryAtMs?: number;
|
|
144
|
-
/** On a `waiting_first_token` frame: how long the
|
|
145
|
-
*
|
|
146
|
-
*
|
|
144
|
+
/** On a `waiting_first_token` frame: how long the current wait has lasted, ms, at the moment the
|
|
145
|
+
* frame is emitted (the emitting process's clock; approximate across a hop). The wait is one
|
|
146
|
+
* segment's: the response-header wait before headers arrive, the first-token wait after — each
|
|
147
|
+
* starts its own count. Absent on every other phase. */
|
|
147
148
|
elapsedMs?: number;
|
|
148
|
-
/** On a `waiting_first_token` frame: the
|
|
149
|
-
*
|
|
149
|
+
/** On a `waiting_first_token` frame: the ceiling of the segment being waited on, ms — the armed
|
|
150
|
+
* first-byte window before headers, the armed first-token watchdog after — so a consumer can render
|
|
151
|
+
* "45s of 300s". Absent on every other phase. */
|
|
150
152
|
timeoutMs?: number;
|
|
151
153
|
/** 1-based index of the attempt that just failed (the wait precedes attempt `attempt + 1`);
|
|
152
154
|
* same numbering as the `brain.retry` telemetry frame. Absent on frames that are not a retry wait. */
|
|
@@ -386,8 +386,7 @@ export declare class TaskRegistry {
|
|
|
386
386
|
/** Background output-file read exemption: true iff `canonicalPath` is EXACTLY the registered output
|
|
387
387
|
* file of a background bash row that (a) the caller can access (same owner/scope/session visibility as
|
|
388
388
|
* a TaskOutput poll — {@link canAccess}) and (b) was registered on the caller's OWN execution env —
|
|
389
|
-
* `env` identity is part of the match, same discipline as {@link
|
|
390
|
-
* {@link runningShellIds}. A canonical path only names a file within ONE env's filesystem
|
|
389
|
+
* `env` identity is part of the match, same discipline as {@link runningShellIds}. A canonical path only names a file within ONE env's filesystem
|
|
391
390
|
* namespace: on a shared registry, a row minted on env A must never exempt a same-named outside-root
|
|
392
391
|
* file in env B's namespace (same session or not), so instance equality — never a string proxy — is
|
|
393
392
|
* required. The output file is minted by the ENGINE itself (createTempFile
|
|
@@ -610,11 +609,6 @@ export declare class TaskRegistry {
|
|
|
610
609
|
* completion notification at the terminal state. `unref()`d — a watcher never keeps the process alive.
|
|
611
610
|
* Stops itself on terminal, on a vanished shell (env disposed), or when the handle is evicted. */
|
|
612
611
|
private startBashWatcher;
|
|
613
|
-
/** RETAINED shells (THE retain predicate) still running on THIS env, any owner. Env identity is part of the
|
|
614
|
-
* match (a shared registry never exempts another env's ids). No caller in src since the park-only parks
|
|
615
|
-
* moved to the run-boundary dispose (which spares every running row: {@link runningShellIds}); removal
|
|
616
|
-
* scheduled for the next minor. */
|
|
617
|
-
retainedShellIds(env: unknown): BackgroundShellId[];
|
|
618
612
|
/** Every shell-lane row still RUNNING on this env, any owner — the run-boundary dispose's except-list (the
|
|
619
613
|
* run teardown and the park-only parks: `sweepRunBoundaryBackgroundShells`).
|
|
620
614
|
* A run's teardown disposes only shells it can attribute to itself: its settle sweep has already killed
|
|
@@ -835,15 +835,6 @@ export class TaskRegistry {
|
|
|
835
835
|
handle.watcher = setInterval(() => void tick(), 1_000);
|
|
836
836
|
handle.watcher.unref?.();
|
|
837
837
|
}
|
|
838
|
-
retainedShellIds(env) {
|
|
839
|
-
const out = [];
|
|
840
|
-
for (const handle of this.handles.values()) {
|
|
841
|
-
if ((handle.type === "background_bash" || handle.type === "monitor") && handle.status === "running" && shellRowRetained(handle) && handle.env === env) {
|
|
842
|
-
out.push(handle.shellId);
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
return out;
|
|
846
|
-
}
|
|
847
838
|
runningShellIds(env) {
|
|
848
839
|
const out = [];
|
|
849
840
|
for (const handle of this.handles.values()) {
|
|
@@ -39,12 +39,12 @@ export interface ResilienceOptions {
|
|
|
39
39
|
* exists to protect the provider; circumvention only where true failure shape matters (benchmarks). */
|
|
40
40
|
bypassBreaker?: boolean;
|
|
41
41
|
}
|
|
42
|
-
/** 1.296 件A — per-call advisory stall-watchdog values (see {@link StreamOptions.stallTimeouts})
|
|
42
|
+
/** 1.296 件A — per-call advisory stall-watchdog values (see {@link StreamOptions.stallTimeouts}); the
|
|
43
|
+
* connect clock is the transport's, fixed at brain construction, so it is not among them. */
|
|
43
44
|
export interface StallTimeouts {
|
|
44
|
-
/** Advisory
|
|
45
|
-
*
|
|
46
|
-
|
|
47
|
-
connectMs?: number;
|
|
45
|
+
/** Advisory first-byte watchdog (ms) — bounds the fetch→response-headers wait; fires as a retryable
|
|
46
|
+
* `stall` out of its own small budget. */
|
|
47
|
+
firstByteMs?: number;
|
|
48
48
|
/** Advisory first-token watchdog (ms) — first thinking delta counts as the first token. */
|
|
49
49
|
firstTokenMs?: number;
|
|
50
50
|
/** Advisory mid-stream idle watchdog (ms) — re-armed on every delta after the first. */
|
|
@@ -588,8 +588,8 @@ export declare const MAX_WORKFLOW_ITEMS = 4096;
|
|
|
588
588
|
* own. This watchdog does not re-send a model call — it aborts the attempt and restarts the whole agent
|
|
589
589
|
* on a FRESH session, so tool effects from before the stall can be repeated. The engine, one layer down,
|
|
590
590
|
* already answers a silent provider call the cheap way: it re-sends that ONE call. So the floor here is
|
|
591
|
-
* what a single provider call may legitimately occupy while producing NOTHING — the
|
|
592
|
-
* first-token wait (
|
|
591
|
+
* what a single provider call may legitimately occupy while producing NOTHING — the whole first-byte lane at the default knob
|
|
592
|
+
* ({@link firstByteLaneKnobMs}) plus the first-token wait; a floor (upload allowance and backoff between attempts not counted). It was a hand-written
|
|
593
593
|
* 180_000 and CC-same until the first-token bound rose to 600s past it (measured: the same silent call
|
|
594
594
|
* then produced two agent runs on two sessions instead of one run with a re-sent call), which is the
|
|
595
595
|
* systematic inversion a derived value cannot re-open: raise or lower the engine's bound and this follows.
|
|
@@ -599,8 +599,8 @@ export declare const MAX_WORKFLOW_ITEMS = 4096;
|
|
|
599
599
|
* this timer starts before the Runner prepares the attempt (an execution-env factory, a hook, a store
|
|
600
600
|
* read all run inside this window and outside the engine's), and the first-token timer only starts once
|
|
601
601
|
* headers arrive — so the margin by which the inner deadline can land AFTER this one is preparation plus
|
|
602
|
-
* the
|
|
603
|
-
* (
|
|
602
|
+
* the header wait actually spent, less the {@link STALL_FIRST_BYTE_MS} this window already budgets for it
|
|
603
|
+
* (instant headers and more than 300s of preparation put the first inner deadline past this timer). An engine retry does not re-arm this timer either, so two silent provider attempts
|
|
604
604
|
* exceed it by construction. The derivation removes the case that was guaranteed to invert; it does not
|
|
605
605
|
* make inversion impossible. A deployment that wants the old aggression passes `stallMs` explicitly, and
|
|
606
606
|
* a deployment that wants a real guarantee must set it from its own preparation budget.
|
|
@@ -22,7 +22,7 @@ import { OUTPUT_TOOL_NAME } from "../core/runner/synthetic-tools.js";
|
|
|
22
22
|
import { compileOutputSchema } from "../core/runner/strict-output-schema.js";
|
|
23
23
|
import { WorkflowBudgetExceededError, WorkflowNestingError, WorkflowAgentBlockedError, WorkflowAgentParkedError, WorkflowAgentSchemaError, WorkflowAgentStalledError, WorkflowMaxAgentsError, WORKFLOW_SPAWN_BLOCKED_ERROR_CODE } from "./workflow-types.js";
|
|
24
24
|
import { WorkflowScriptError } from "./workflow-meta.js";
|
|
25
|
-
import {
|
|
25
|
+
import { firstByteLaneKnobMs, STALL_FIRST_BYTE_MS, STALL_FIRST_TOKEN_MS } from "../brain/timeout.js";
|
|
26
26
|
import { asThrownError, isErrorLike, readMember, thrownIs, thrownName, thrownText } from "../core/thrown-value.js";
|
|
27
27
|
export * from "./workflow-types.js";
|
|
28
28
|
const MAX_TRANSCRIPT_CHARS = 4000;
|
|
@@ -310,7 +310,7 @@ function assertFanOutOptionsShape(v, position) {
|
|
|
310
310
|
`Nothing was run.`);
|
|
311
311
|
}
|
|
312
312
|
}
|
|
313
|
-
export const WORKFLOW_AGENT_STALL_MS =
|
|
313
|
+
export const WORKFLOW_AGENT_STALL_MS = firstByteLaneKnobMs(STALL_FIRST_BYTE_MS) + STALL_FIRST_TOKEN_MS;
|
|
314
314
|
export const WORKFLOW_THROTTLE_SLOW_MS = 90_000;
|
|
315
315
|
export const WORKFLOW_AGENT_MAX_RETRIES = 5;
|
|
316
316
|
export const WORKFLOW_AGENT_THROTTLE_BACKOFF_MS = 45_000;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sema-agent/core",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.33.0",
|
|
4
4
|
"description": "Stateless, task-oriented AI agent core",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "BUSL-1.1",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"wire-changes/"
|
|
45
45
|
],
|
|
46
46
|
"engines": {
|
|
47
|
-
"node": ">=20.
|
|
47
|
+
"node": ">=20.18.1"
|
|
48
48
|
},
|
|
49
49
|
"scripts": {
|
|
50
50
|
"build": "rm -rf dist && tsc -p tsconfig.build.json && tsc -p tsconfig.build.json --emitDeclarationOnly --removeComments false",
|
|
@@ -95,7 +95,8 @@
|
|
|
95
95
|
},
|
|
96
96
|
"dependencies": {
|
|
97
97
|
"@modelcontextprotocol/sdk": "1.30.0",
|
|
98
|
-
"typebox": "1.1.39"
|
|
98
|
+
"typebox": "1.1.39",
|
|
99
|
+
"undici": "7.30.0"
|
|
99
100
|
},
|
|
100
101
|
"optionalDependencies": {
|
|
101
102
|
"sharp": "^0.35.0"
|
|
@@ -107,6 +108,8 @@
|
|
|
107
108
|
"pg-mem": "^3.0.14",
|
|
108
109
|
"tsx": "4.19.2",
|
|
109
110
|
"typescript": "5.7.2",
|
|
111
|
+
"undici6": "npm:undici@^6.29.0",
|
|
112
|
+
"undici8": "npm:undici@^8.11.2",
|
|
110
113
|
"vitest": "^2.1.8"
|
|
111
114
|
}
|
|
112
115
|
}
|