@sema-agent/core 7.32.1 → 7.33.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -1
- package/dist/brain/anthropic.d.ts +1 -1
- package/dist/brain/anthropic.js +4 -1
- package/dist/brain/open-responses.d.ts +2 -1
- package/dist/brain/open-responses.js +4 -1
- package/dist/brain/openai.d.ts +2 -1
- package/dist/brain/openai.js +4 -1
- package/dist/brain/stream-engine.d.ts +8 -7
- package/dist/brain/stream-engine.js +112 -44
- package/dist/brain/timeout.d.ts +122 -53
- package/dist/brain/timeout.js +49 -5
- package/dist/brain/transport.d.ts +13 -0
- package/dist/brain/transport.js +62 -0
- package/dist/core/runner/prepare-task.js +2 -2
- package/dist/core/runner/prepare-wiring-manifest.js +2 -1
- package/dist/core/secret-env.d.ts +13 -4
- package/dist/core/secret-env.js +17 -8
- package/dist/core/task-event.d.ts +7 -5
- package/dist/core/task-registry.d.ts +1 -7
- package/dist/core/task-registry.js +6 -10
- package/dist/core/tool-policy.d.ts +12 -12
- package/dist/core/wiring-manifest.d.ts +27 -0
- package/dist/core/wiring-manifest.js +11 -0
- package/dist/engine/llm/types.d.ts +5 -5
- package/dist/orchestration/workflow.d.ts +4 -4
- package/dist/orchestration/workflow.js +2 -2
- package/package.json +6 -3
- package/wire-changes/core@7.33.0.json +351 -0
- package/wire-changes/core@7.33.1.json +237 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,71 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 7.33.1 — 2026-09-26
|
|
4
|
+
|
|
5
|
+
### The first-byte lane's retry is a construction-time seat — `BrainTimeoutConfig.firstByteRetries?: 0 | 1` (#1128; additive; +1 optional type key; +0 exports, +0 codes) (@server)
|
|
6
|
+
|
|
7
|
+
- **Before.** The lane's budget was a module constant (`FIRST_BYTE_RETRY_BUDGET` = 1) with no seat, and `maxRetries: 0` does not reach it (the lane's budget is its own, by design). A host that keeps a failover ladder of its own — a second gateway it tries when a call fails — waited the first-byte window and then a retry at twice the knob before the failure reached that ladder: 300 s + 600 s at the default, 30 s + 60 s even at an explicit 30 s knob.
|
|
8
|
+
- **Now.** `firstByteRetries: 0 | 1` on every first-party brain's config (absent ⇒ 1, the reference implementation's one retry — unchanged). `0` ⇒ the first window's expiry is the call's terminal: one request, no retry frame, the existing one-window sentence (`[network] first-byte timeout: no response headers within <w1>ms (firstByteTimeoutMs) — …`). Independent of `maxRetries` in both directions, construction only (no per-call advisory twin). A value other than `0` / `1` fails the call before anything is sent, naming the knob (the watchdog knobs' own refusal, `config.stall_timeout_invalid`, at the same resolution point — like theirs, the code stays inside the brain and the caller reads the message) — never folded to the default or to `0`. The derived floors (`firstByteLaneKnobMs`, `WORKFLOW_AGENT_STALL_MS`) read the default and do not move.
|
|
9
|
+
- **Who is hurt.** Nobody: absent is byte-for-byte 7.33.0. **Compensation** n/a.
|
|
10
|
+
- Pins: `test/backlog1128-first-byte-retries.test.ts` (real transport, a local gateway that never writes headers: `0` ⇒ one request and the one-window terminal on all three brains, also under `maxRetries: 10`; explicit `1` and absent ⇒ two requests and one retry frame with `maxRetries: 1` under `maxRetries: 0`; twelve bad values refused by the resolver and through a brain with nothing sent; the derived floors). Docs: `docs/TIMEOUTS.md` (a retry-knob column; "a host with a failover ladder sets `0`"), `docs/sdk/03-brains-and-models.md`. Wire sheet: `docs/wire-consumption/unreleased-1128.md`.
|
|
11
|
+
|
|
12
|
+
### Fix — git's environment config group survives the secret-env scrub whole: `GIT_CONFIG_KEY_<i>` is no longer taken for a credential unless its value carries one (#1129; behavior; +0 exports; `isSecretEnvKey` gains an optional `value` argument; +0 wire keys) (@server)
|
|
13
|
+
|
|
14
|
+
- **Before.** The shape rule reads a bare `KEY` word with a numbered suffix as a credential, so `GIT_CONFIG_KEY_<i>` — git's env config, where the member holds a config entry's NAME (`user.name`, `safe.directory`, `http.extraHeader`) and `GIT_CONFIG_VALUE_<i>` its value — was dropped while `GIT_CONFIG_COUNT` and the value stayed. Every git child of a scrubbed env then refused to start: `error: missing config key GIT_CONFIG_KEY_0` / `fatal: unable to parse command-line config`, exit 128 (measured through the default exec env on a host carrying the group).
|
|
15
|
+
- **Now.** The exemption side of the exact-name arm reads a second table beside the exact names, under the same three admission criteria: NAMED FAMILIES, `<STEM>_<i>` with `i` a canonical decimal — the spelling the reading tool builds itself. One member: `GIT_CONFIG_KEY`. Its row states criterion 2 ("the value is not a credential") as a screen on the VALUE rather than a standing claim, because a config key's subsection can be a URL and git authenticates with a URL's userinfo (`url.https://x-access-token:<token>@github.com/.insteadOf`): a member whose value carries `<scheme>://<userinfo>@` (any character but `/` counts as userinfo, whitespace included) is not exempt and drops by the shape rule exactly as before (finding `suffix-rule`). The screen is the row's own admission condition, read by the one classification (`classifySecretEnvKey(name, value?)`); `scrubSecretEnv` passes each entry's value, and `isSecretEnvKey(name)` without a value reads as in 7.33.0 (a family member is `true` — the name alone cannot clear); `isSecretEnvKey(name, value)` gives the scrub's answer for that entry. Unchanged: the bare `GIT_CONFIG_KEY`, a decorated (`X_GIT_CONFIG_KEY_0`), padded (`GIT_CONFIG_KEY_01`) or suffixed (`GIT_CONFIG_KEY_ID`, `GIT_CONFIG_KEY_0_TOKEN`) name, and every credential name (`GIT_API_TOKEN`, `GIT_PASSWORD`, `GH_TOKEN`) still drop and are still reported through `config.secret_env_scrubbed` / `onSecretEnvScrub`; the credential list still wins a collision.
|
|
16
|
+
- **Who needs it.** Any deployment whose host process configures git through the environment (the `GIT_CONFIG_COUNT` protocol — a `safe.directory`, an `http.extraHeader` name, a mirror rewrite set without touching a config file): the two children the engine spawns from a scrubbed copy of its own environment — the default exec env's shell (`inheritEnv: "scrub"`) and an LSP server started without an explicit `env`.
|
|
17
|
+
- **Who is hurt / compensation.** A group whose key carries URL userinfo — a token-bearing `url.<base>.insteadOf`, and also a username-only one such as `url.ssh://git@github.com/.insteadOf` — is still dropped at that key, so git in the child still exits 128 there (fail-closed: the credential does not reach the child; KL-1129b); the compensation is to hand the group explicitly (`inheritEnv` allowlist / `shellEnv`) or move the rewrite into a git config file. `GIT_CONFIG_VALUE_<i>` can carry a credential (an `http.extraHeader: Authorization: …` value) and was already inherited: the shape rule never matched its name, and this change does not touch it either way (KL-1129a).
|
|
18
|
+
- Pins: `test/backlog1129-git-config-key-family.test.ts` (① `user.name` / `safe.directory` / a userinfo-free `insteadOf` through `scrubSecretEnv` and a real git child reading all three, and the default exec env's git child reading the entry while the withheld set names a planted credential and not the member; ② a token-bearing `insteadOf` key dropped with one finding, the token nowhere in the scrubbed env, git refusing with exit 128; the screen across schemes, a username-only userinfo, a userinfo with whitespace in it (git reads the credential out of it all the same) and an `@` in a path; ③ `isSecretEnvKey` without a value unchanged, with a clearing value `false`; an undefined entry drops; ④ the stem, decorated, padded, suffixed and credential names dropped by the shape rule even with a clearing value; the value member unmatched; the credential side winning a collision). Mutation-checked: a screen that always clears reds ②, a screen consulted without a value reds ③.
|
|
19
|
+
|
|
20
|
+
### The wiring manifest says whether a leg has hands — `WiringManifest.hands?: { mounted, reason? }` (#1039; additive; +1 optional member on `WiringManifest` / `WiringFacts`; +1 wire word `no_execution_env`; +0 exports; `configFingerprint` value moves once) (@server)
|
|
21
|
+
|
|
22
|
+
- **Before.** A leg whose deployment supplied no execution environment (no factory, no static `executionEnv`) mounts no hand band at all, and the wire said so only indirectly: an empty `mountedBy: "hands"` slice of `tools[]`, or `writeProtection.targetView: "spelling-only"` — which is absent altogether when the protection table is `[]`, where "no hands" and "protection off" read the same and the two assemblies hashed to the same `configFingerprint`.
|
|
23
|
+
- **Now.** Every engine-minted effective manifest carries `hands`, beside `lsp` and in the same shape: `{ mounted: true }` when the band mounted (a factory-minted env, or a static `executionEnv` seat that is not `undefined` — a supplied `StubExecutionEnv` included, the band mounting over it with file tools and no shell, and a JS caller's `null` likewise over the no-I/O placeholder), `{ mounted: false, reason: "no_execution_env" }` otherwise. The value is the one the hand band's mount dispatches on, relayed, not re-derived. `reason` is one literal, not a registered closed set, because the hands decision has one false branch (a second branch promotes it). Unlike `lsp`, the member is IN `configFingerprint` (an assembly fact, the class of `writeProtection`), so a with-hands and a hands-less assembly now hash apart. No `EngineNotice`, no `TaskResult` change; whether a full shell is reachable stays a `tools[]` reading.
|
|
24
|
+
- **Who is hurt.** A consumer that persisted `configFingerprint` values across engine versions sees every effective fingerprint change once (a new member in the hashed body); legs of one assembly still agree. An external `deriveWiringManifest` caller that passes an inconsistent `hands` fact is refused (the `autoMode` / `fileHistory` screen posture). **Compensation** n/a — the member is additive and its absence (older engines, the static half) is not a state.
|
|
25
|
+
- Pins: `test/backlog1039-wiring-manifest-hands.test.ts` (through the Runner: a static env, no env, a supplied `StubExecutionEnv`, a JS caller's `null`, the factory lane — each read against the same leg's `tools[]`; the fingerprint telling the two assemblies apart with no protection judge on either; the value screen's six refusals, `null` included; the static half; no aliasing). Docs: `docs/INTEGRATION-CORE.md` (`wiring_manifest` row). Wire sheet: `docs/wire-consumption/unreleased-1039.md`.
|
|
26
|
+
|
|
27
|
+
### Pins and wording — an explicit ask rule's ask and a hook's ask carry their own origin word on the card and on the park row (#1117; zero behavior face; +0 exports, +0 wire keys) (@server @cli)
|
|
28
|
+
|
|
29
|
+
- **What was unpinned.** The origin ladder already mints `ask_rule` from a decision's `matchedAskRule` and `hook` from the hook fold, and gate-level pins read the word the resolver receives — but no pin read the WIRE faces a surface renders from: `AskRequest.origin` on a live approver and `PendingAction.tool_approval.origin` / `CheckpointSummary.origin` on a park, through the real Runner. A host policy that raises an operator's must-ask as an `ask` WITH `matchedAskRule` (composed through `combinePolicies`, beside a base policy's plain ask) reaches both faces as `ask_rule` today; without the member it reads `policy`.
|
|
30
|
+
- **Now pinned**, and said where the words are listed: `docs/INTEGRATION-CORE.md` §4.6b (the enumerated set brought to its eleven words) states that `ask_rule` has one input — a minter raising an ask for a rule someone wrote must set `matchedAskRule`, or a plain policy ask reads `policy` on the card and the row — and what setting it means (the classifier does not answer it, a persisted allow rule does not clear it, an answer for this call does); the `PermissionResult.matchedAskRule` doc says the same. Two internal comments on the inherited-lane mint stations and the 7.26.0 `AskRequest.mandated` entry used "standing posture" for what a mandate refuses; the contract's word is a STANDING ANSWER (a release remembered from an earlier call) — an answer given for this call still clears a mandated ask. Wording only.
|
|
31
|
+
- Pins: `test/backlog1117-ask-origin-rule-and-hook.test.ts` (live card and park row for the operator ask folded beside a base ask and alone, with the no-`matchedAskRule` negative control reading `policy`; live card — over an allowing policy and folded over a policy ask — and park row for a hook ask). Mutation-checked: removing the ladder's `ask_rule` arm reds three, its `hook` arm two, the fold's `matchedAskRule` carry two.
|
|
32
|
+
|
|
33
|
+
### Forensics — the exit hard face says which of three causes a `no_pgid` receipt had (+2 trace lines; zero behavior face; +0 exports, +0 wire keys) (@server)
|
|
34
|
+
|
|
35
|
+
- `killAllBackgroundShellsNow` folds three facts into one `no_pgid` receipt: the env has no synchronous kill seat, the seat answered `no_pgid` itself, or the seat threw. Only the third left a trace line. Two more, same form, same opt-in (`SEMA_DEBUG_SHUTDOWN=1`, stderr): `hard kill: seat absent` and `hard kill: seat answered no_pgid`, each with `{ id, shellId }`. The seat is now read once, inside the guard, so a READ that throws (a revoked membrane) is traced as `seat threw` and costs only its row — before this, the presence check sat inside the guard too. Receipts, row states and the per-row `hard kill: row` line are unchanged; an answer outside the closed set is still folded to `no_pgid` with the per-row line only. Contract text: `docs/background-process-lifetime.md` (Receipts). Pins: `test/backlog1107-hard-kill-forensics.test.ts` (the trace sink mocked: three arms, one line each, receipts `no_pgid`, rows `running`; a throwing READ of the seat costs its row only and the next row is killed; the seat called with its env as `this`; a seat that killed or found the shell gone leaves no seat line).
|
|
36
|
+
|
|
37
|
+
### Test hygiene — `test/mcp-pagination.test.ts` no longer reads the host's `MCP_TOOL_TIMEOUT` (zero behavior face; +0 exports)
|
|
38
|
+
|
|
39
|
+
- The page-walk pins measure the DEFAULT budget; with the variable set in the host env two of them went red on any branch. The file now clears it for its own run and restores it after (red-before / green-after verified at the fix).
|
|
40
|
+
- `test/backlog994-s1-gaps-real-shells.test.ts`: the `echo x |& cat` pin compared the two pipeline stages' xtrace lines in a fixed order; the stages are concurrent processes, so the order is a race (one red in 23 546 on the stage run). The pin now compares membership; the other assertions of that pin are unchanged.
|
|
41
|
+
|
|
42
|
+
## 7.33.0 — 2026-09-26
|
|
43
|
+
|
|
44
|
+
### Model-call timeouts re-cut by segment: the header wait gets its own `firstByteTimeoutMs`, and every request carries the engine's dispatcher with the transport's header/body clocks off (#1121 #1122; behaviour: a pre-headers stall is a `stall` with its own one-retry lane, not a `transport` failure re-sent ten times; every watchdog defaults ON; +1 optional key `BrainTimeoutConfig.firstByteTimeoutMs`; **`BrainTimeoutConfig.connectTimeoutMs` REMOVED**; `StallTimeouts.connectMs` → `firstByteMs` (type-face breaking, see below); `WORKFLOW_AGENT_STALL_MS` 630 s → 1 500 s (derived); +1 dependency `undici@7.30.0`; `engines.node` `>=20.18.1`) (@server @cli @client-core)
|
|
45
|
+
|
|
46
|
+
- **Before.** Two shapes, both reported against a self-hosted gateway. (1) The knob named `connectTimeoutMs` (advisory 30 s) actually bounded the `fetch()` call → response headers. A gateway that runs its prefill BEFORE writing headers was cut at 30 s, the cut was announced as `errClass: "transport"` / "connection lost, reconnecting" (nothing about the link was observed — this process cancelled its own request), and the identical body was re-sent through the transient ladder up to `maxRetries` (10) times: ~8.5 minutes of burned prefill ending in "connect timeout after 30000ms". (2) `config.fetchImpl ?? fetch` handed the request to Node's global `fetch`, whose undici transport keeps a 300 s headers clock and a 300 s body-silence clock that nothing here set or named; they sat UNDER `firstTokenTimeoutMs` (600 s) and `idleTimeoutMs`, so a value above 300 s was never granted, the cut arrived as a read rejection reported `transport`, and raising the knob changed nothing, silently.
|
|
47
|
+
- **Three engine clocks, one per segment the engine can see, one definition (`src/brain/timeout.ts`).** The TCP + TLS connect is the transport's own clock (undici's 10 s by default, tuned on the process dispatcher the host installs); it is not a knob here.
|
|
48
|
+
|
|
49
|
+
| segment | key (`BrainTimeoutConfig`, all optional) | default | enforced by | on expiry |
|
|
50
|
+
|---|---|---|---|---|
|
|
51
|
+
| `fetch()` call → response headers | `firstByteTimeoutMs` — **new** | `STALL_FIRST_BYTE_MS` 300 s, plus 1 s per 32 KiB of request body (rounded up); the one retry runs at 2× the knob | the engine (per-attempt abort controller) | `stall`: retry frame `first-byte watchdog fired after <window>ms (firstByteTimeoutMs[ <knob> + <n> upload allowance]), retrying`, `maxRetries: 1`; terminal `[network] first-byte timeout: no response headers within <w1>ms, then <w2>ms on the retry (firstByteTimeoutMs) — if a proxy or gateway holds responses until they complete, raise firstByteTimeoutMs to wait longer` |
|
|
52
|
+
| headers → first content delta | `firstTokenTimeoutMs` | `STALL_FIRST_TOKEN_MS` 600 s | the engine | unchanged (`first-token watchdog fired …`) |
|
|
53
|
+
| delta → delta | `idleTimeoutMs` | `STALL_IDLE_MS` **300 s** (was 90 s; the reference implementation's idle floor — ruled to follow) | the engine | unchanged (`stream idle watchdog fired …`) |
|
|
54
|
+
|
|
55
|
+
- **One resolution chain for every knob** (`resolveWatchdogMs`): construction config (an explicit `0` is OFF) > per-call advisory `StreamOptions.stallTimeouts` (the Runner sends the same defaults on every call; a BYOM brain reads them as its hint) > the engine default constant. The old third level "both absent ⇒ off" is retired: a brain built with no timeout config and used outside the Runner used to have NO engine watchdog at all and rode on the transport's accidental 300 s. **Who is hurt.** A deployment that constructed a first-party brain with no timeout fields and relied on it having none: it now arms 300 / 600 / 300 s (first byte / first token / idle). The idle default is the reference implementation's 300 s floor (ruled to follow), so a call that carried no advisory keeps the same 300 s of tolerated mid-stream silence the transport's body clock used to give it — now as the engine's own watchdog with a `stall` announcement instead of a transport failure; an Anthropic `ping` is still not a content delta, so a server-side tool that keeps the stream silent past 300 s is cut and, with text already streamed, committed by tier A (unchanged bound, new announcer). The number is not changed here; it is in #1125's candidate material. **Compensation.** `0` on the field it wants off (deployment sovereignty, unchanged), or a larger `idleTimeoutMs`.
|
|
56
|
+
- **The first-byte lane** has its OWN budget (`FIRST_BYTE_RETRY_BUDGET` = 1, independent of `maxRetries`, like the thinking-only tier's), widens the knob ×2 on the retry (`FIRST_BYTE_RETRY_WINDOW_FACTOR`), re-sends the same body exactly once, and never claims the link dropped. The retry frame and the operator telemetry read one mapping (`retryAnnouncement`), the same one the post-headers watchdogs already used — a phase has no dialect of its own. The **wait disclosure** (`waiting_first_token`, first frame at 30 s then every 30 s) now runs during the header wait too, reporting the first-byte window as `timeoutMs`; before, a 5-minute header hold showed nothing until the wrong "reconnecting".
|
|
57
|
+
- **The transport seat** (`src/brain/transport.ts`, one `resolveFetch` for all three brains): every request goes to the caller's `fetchImpl` (else Node's global `fetch`) with ONE addition to its init — `ENGINE_DISPATCHER`, a transparent stand-in for the PROCESS's dispatcher that differs from it in one place: each request it dispatches carries `headersTimeout: 0, bodyTimeout: 0`. The sending members undici's Dispatcher base builds on `dispatch` (`request`, `compose`, …, read off the base class, not listed) stay the engine's, so a `fetchImpl` that composes interceptors over `init.dispatcher` still sends with the clocks off, keeps every interceptor and reaches its own fetch's slot; every other member (`isMockActive`, `close`, events, …) is read off the host's dispatcher when accessed — nothing is listed, so a host `MockAgent` sees the request body and intercepts as it does for the host's own requests. Which host dispatcher: the one the CALLING fetch would resolve on its own — undici keeps one process slot per Dispatcher-API version (`undici.globalDispatcher.1` for every undici ≤ 7 including Node's built-in fetch, `.2` for undici 8), and the handler the fetch passes says which (undici's own test: `onRequestStart` ⇒ v2); resolved per request. Proxies, CA trust, mTLS, mocks and the connect clock stay whatever the host installed with `setGlobalDispatcher`. Consequence: `firstTokenTimeoutMs` / `idleTimeoutMs` above 300 s are now actually granted, on Node's fetch and on any `fetchImpl` that forwards `init` to a fetch. On an undici 8 host a v1 caller (Node's built-in fetch) goes through undici 8's slot-1 translation, which forces HTTP/1.1 and leaves a `MockAgent`'s interceptors — for the host's own plain requests too; a `fetchImpl` that is undici 8's `fetch` reaches the real dispatcher (KL-1121h). **The slot is the API**: slot 1 (`…globalDispatcher.1`) is always handed a v1 handler and slot 2 a v2 one, through undici's own converters (`lib/handler/unwrap-handler.js` / `wrap-handler.js`), so a composed chain whose interceptor rewraps the handler to v2 (undici 7's `interceptors.retry()`) still reaches an undici 6 host in slot 1 (before this rule: `UND_ERR_INVALID_ARG invalid onError method` and a retry loop).
|
|
58
|
+
- **`fetchImpl` contract.** It receives `init.dispatcher = ENGINE_DISPATCHER` beside the method / headers / body / signal it always received. A wrapper that forwards `init` to Node's `fetch` gets the seat for free; a fetch that ignores `init.dispatcher` (another HTTP stack, a fetch that rebuilds its init) keeps its own transport's clocks under the engine's — the engine cannot see or lift them (KL-1121a): with undici's defaults its 300 s header clock fires before the engine's first-byte window (≥ 301 s: 300 s + at least 1 s of upload allowance), so the one-retry first-byte lane is never reached — the cut is reported `transport`, re-sent through the shared ladder, and a dead gateway reaches the terminal after about 11 × 300 s. The engine's three watchdogs arm regardless; a first-byte cut still requires the fetch to honour `init.signal`.
|
|
59
|
+
- **`connectTimeoutMs` removed — who is hurt / compensation.** The knob bounded the header wait, which is now `firstByteTimeoutMs`; the TCP + TLS connect it was named for is the transport's own clock, tuned on the process dispatcher (`setGlobalDispatcher(new Agent({ connect: { timeout } }))`) — the engine has no honest home for a connect number, so the key is gone rather than kept with a changed meaning or a silent no-op. A host passing it fails loudly at upgrade — a literal does not compile, and a spread (`{ ...timeouts }`) or a JS caller, which the compiler cannot see, is refused by the brain factory (`config.stall_timeout_retired`, naming `firstByteTimeoutMs`) rather than ignored — and moves the number to `firstByteTimeoutMs` (a host that wants the old 30 s header cut spells it `firstByteTimeoutMs: 30_000`). **@server** `MODEL_CONNECT_TIMEOUT_MS` fed this key through a spread (`src/brain.ts` `timeouts`) — that shape compiles, and `createBrain` throws at construction until the key is dropped; it maps to `firstByteTimeoutMs` now (the env's own doc, "连接超时 … 0=关", described the header wait), and `0` there still means off. **@cli / @client-core** the retry frame for a pre-headers cut now carries `errClass: "stall"` with a detail that names `firstByteTimeoutMs`; a renderer that special-cased "connection lost" for that case sees a stall.
|
|
60
|
+
- **Hard breaking (type face).** ① `BrainTimeoutConfig.connectTimeoutMs` is removed: a literal passing it fails to compile; a spread or a JS caller passing it is refused at run time by all three brain factories (`config.stall_timeout_retired`, the message names `firstByteTimeoutMs`) — the key's presence is refused even with an `undefined` value; the header wait it bounded is `firstByteTimeoutMs`. ② `StallTimeouts.connectMs` is removed and `firstByteMs` added: a literal with `connectMs` fails to compile; a spread or a JS caller that still sends it gets the CALL refused (stop reason `error`, the message names `firstByteMs`, nothing sent); a reader sees `firstByteMs`. ③ `WORKFLOW_AGENT_STALL_MS` is now `firstByteLaneKnobMs(STALL_FIRST_BYTE_MS) + STALL_FIRST_TOKEN_MS` = 1 500 000 (was 630 000): the floor is what one call may legitimately spend producing nothing — the first-byte probe and every widened retry the lane's budget allows (`FIRST_BYTE_RETRY_BUDGET`, `FIRST_BYTE_RETRY_WINDOW_FACTOR`), then the first-token wait — so a workflow's per-agent progress watchdog waits longer before restarting a silent agent (the derivation's whole point: an outer, coarser clock must not fire before the engine's own; the body-dependent upload allowance and the backoff between attempts are not counted, so it is a lower bound). ④ `engines.node` rises from `>=20.3.0` to `>=20.18.1`, the floor `undici@7.30.0` states; Node 20.18.1 is on the same LTS line.
|
|
61
|
+
- **Dependency decision — `undici@7.30.0` (exact) enters `dependencies`.** Node exposes neither its dispatcher classes nor `getGlobalDispatcher`; the npm package does, and it shares the process-wide dispatcher slot with Node's own `fetch` (the documented `setGlobalDispatcher` contract), which is what lets the engine forward to whatever the host installed. undici 6 would have kept `>=18.17` but is the previous major. Stated side effect of the import (KL-1121c): if slot 1 is still empty, undici writes its default `Agent` (the same defaults as Node's own) into both process slots — measured on Node 20.18.1 at import of the package entry (the host's plain `fetch` then uses it too); on Node 24 it depends on load order. Anchored on the reference implementation's shape: an independent first-byte window with its own knob and a one-retry cap, a request-body upload allowance, a widened retry, and the gateway hint — its default for a non-first-party route (300 s) is the default here.
|
|
62
|
+
- **Unchanged here, stated.** A model call whose watchdog is `0` has no bound on that segment at all now (the transport's clocks are 0 too); under the Runner the hour-scale guardrail remains, and the calls outside it (compaction, side-query, memory consolidation, the classifier, prompt suggestions) are listed with their own bounds in `docs/TIMEOUTS.md`. The memory distiller's model call is not on the seat yet (#1127).
|
|
63
|
+
- Pins: `test/backlog1121-model-timeouts.test.ts` (window arithmetic; the chain; the engine dispatcher forwarding to the process dispatcher with the clocks off — pure and on the real transport with a spied host `Agent`; the slot chosen by the handler's Dispatcher-API version — Node's built-in fetch and npm undici 7's fetch reach slot 1 past a v2-only dispatcher in slot 2, a v2 handler reaches slot 2; every other member read off the host while `compose` / `request` keep the clocks off; a host `MockAgent` with a string body matcher in either slot, net connect disabled and enabled, and end to end through a brain — matched, 0 real requests; `isMockActive` true against a real dispatcher delivers the body intact; the retired keys refused by all three factories and per call; a deadline between the transport's headers and `doFetch` returning taken as a stall; the `fetchImpl` init carrying the seat; three fake-gateway arms against a local HTTP server — headers held, headers never, first data held — plus a contrast arm with an undici body clock; the first-byte and idle defaults pinned at the call site; the header-wait disclosure and the truthful closing frame under fake timers). Re-pinned: `test/walltime-stall-watchdog.test.ts`, `test/resilience.test.ts`, `test/backlog682-stall-retry-announcement.test.ts`, `test/stream-options-forwarding.test.ts`, `test/brain-resilience-coverage.test.ts`. Wire sheet: `docs/wire-consumption/unreleased-1121.md`. The timeout table: `docs/TIMEOUTS.md`.
|
|
64
|
+
|
|
65
|
+
### Type surface — `TaskRegistry.retainedShellIds` removed (#1120 follow-up; +0 wire keys)
|
|
66
|
+
|
|
67
|
+
- **Removed (type surface).** `TaskRegistry.retainedShellIds(env)` is gone. Since 7.32.1 nothing in `src` calls it (the park-only parks share the run-boundary dispose, whose keep-list is `runningShellIds(env)`). No caller was found in the server / cli / client-core trees. Read `runningShellIds(env)` with `list(access)` rows' `retained` instead.
|
|
68
|
+
|
|
3
69
|
## 7.32.1 — 2026-09-25
|
|
4
70
|
|
|
5
71
|
### Fix — a park that does not pause the env no longer stops the session's background shells (#1120; behavior; +0 exports, +0 wire keys) (@cli @server)
|
|
@@ -1062,7 +1128,7 @@ with the next package release.
|
|
|
1062
1128
|
- **Who is hurt.** A caller that relied on the file backend folding an unnamed kind into a delete — none in-tree, and the behaviour was unreachable through a typed call. Exports +2 (`permissionRuleWriteContract`, `PermissionRuleWriteContractHooks`), removals 0.
|
|
1063
1129
|
- Reach: **@server** — the store twin's acceptance floor gains one suite to run (the sync kit is unchanged); **@cli** / **@client-core** — zero. Headless: a malformed delta from a foreign backend caller now throws where the file backend previously wrote a tombstone. `test/backlog996-write-delta-kinds.test.ts`.
|
|
1064
1130
|
### A mandate is a fact about the ask, not about a lane — `AskRequest.mandated` (#997; P2; approval face; additive, no export change)
|
|
1065
|
-
- **Before.** The only way an approval surface could learn that a card was MANDATED — that no persisted allow rule and no standing
|
|
1131
|
+
- **Before.** The only way an approval surface could learn that a card was MANDATED — that no persisted allow rule and no standing answer (a release remembered from an earlier call) could retire it, while an answer given for that call still could — was `AskRequest.ruleOffersAbsence === "mandated"`, a seat that reports what the RULE-OFFER LANE has to give. That lane speaks for one tool, only where a rule store is wired, and only on a task that can hold a rule; its own contract says absence is not a claim. Measured on the base of this entry, real Runner, `cat <path outside the session roots>`, both `shellGate` doctrines: `Bash` with a store and a principal carried `"mandated"`; the SAME command on `Monitor`, on `Bash` with no store, and on an anonymous task carried nothing. One approval card in every row; the mandate — the read station's own structural demotion — present in every row.
|
|
1066
1132
|
- **After.** `AskRequest.mandated?: true` (and the `PermissionResult` ask-arm member it is copied from) states it on the ask. Minted ONCE, at the gate's single origin stamp, from the one derivation the card's rule-offer word is also read from (`ruleClearanceOf`): the door SET decides the bit, the incumbent display ladder decides the word. Which door a card shows is a display choice; whether the ask is mandated is not — so a card worded `shadowed` over a probe mandate is mandated all the same, which the two-enumeration shape got wrong.
|
|
1067
1133
|
- **The doors,** in one place: the tool seat's `egress` / `irreversibility` marks, an operator `shellGate: "always"`, the shell read station's structural demotion or a probe that timed out or threw, a write the protection judge matched or could not resolve, `requiresRealApproval`, a hook-raised ask, and an ask decided at an ANCESTOR's frozen approver. **Not** a door: a person's own explicit `ask` permission rule — the lane words that `shadowed`, because a rule they wrote is already speaking.
|
|
1068
1134
|
- **What the bit says, and what it does not.** It says a persisted allow RULE may not clear the card. It does NOT say an approver may not answer it: a person, a hook and a deployment's blanket `onAsk: "allow"` seat all resolve a mandated ask exactly as before, and `requiresRealApproval` remains the separate, stronger "a person, not a configuration" demand. A shell whose POSTURE seat declines to auto-answer mandated cards reads this member to know which they are — @cli L-459, @client-core CC-110 (frame mirror), @server in transit. Absent on an older engine is not the opposite claim; tell the two apart by engine version, not by this seat.
|
|
@@ -9,7 +9,7 @@ export interface AnthropicBrainConfig extends StreamEngineConfig {
|
|
|
9
9
|
version?: string;
|
|
10
10
|
/** Extra headers merged into every request (e.g. `anthropic-beta`). */
|
|
11
11
|
headers?: Record<string, string>;
|
|
12
|
-
/** Custom fetch (tests / proxies)
|
|
12
|
+
/** Custom fetch (tests / proxies); absent ⇒ the engine's default transport — see `resolveFetch`. */
|
|
13
13
|
fetchImpl?: typeof fetch;
|
|
14
14
|
/** `max_tokens` when the call/model doesn't specify one (Anthropic requires it). Default 4096. */
|
|
15
15
|
defaultMaxTokens?: number;
|
package/dist/brain/anthropic.js
CHANGED
|
@@ -10,6 +10,8 @@ import { ANTHROPIC_RESERVED, OUTPUT_CAP_KEYS, applyExtraBody, effectiveOutputCap
|
|
|
10
10
|
import { adjudicateModelRoute, applyRouteCredentialHeaders, assertServableRequestRoot, createBrainRouteJudge, joinRequestUrl, resolveRequestRoot, resolveRouteCredential, routeRefusalText } from "./route-adjudicator.js";
|
|
11
11
|
import { MIN_THINKING_TOKENS, budgetCapSkipsThinking, declaredEffortLevels, reasoningBudgetShare, reasoningRequestCarried, resolveEffort } from "./reasoning.js";
|
|
12
12
|
import { runStreamingBrain } from "./stream-engine.js";
|
|
13
|
+
import { refuseRetiredTimeoutKnobs } from "./timeout.js";
|
|
14
|
+
import { resolveFetch } from "./transport.js";
|
|
13
15
|
function thinkingBudget(maxTokens, share, fixed, hardCap = false) {
|
|
14
16
|
const max = hardCap ? maxTokens : Math.max(maxTokens, MIN_THINKING_TOKENS * 2);
|
|
15
17
|
const want = fixed ?? Math.floor(max * share);
|
|
@@ -225,7 +227,8 @@ function closeToolUseBlock(acc) {
|
|
|
225
227
|
}
|
|
226
228
|
export function createAnthropicBrain(config = {}) {
|
|
227
229
|
assertServableRequestRoot(config.baseUrl, "config.baseUrl");
|
|
228
|
-
|
|
230
|
+
refuseRetiredTimeoutKnobs(config, "construction", "createAnthropicBrain config");
|
|
231
|
+
const doFetch = resolveFetch(config);
|
|
229
232
|
const cacheBreakpoints = config.cacheBreakpoints !== false;
|
|
230
233
|
const stream = (model, context, options) => {
|
|
231
234
|
let thinkingRequested = false;
|
|
@@ -7,7 +7,8 @@ export interface OpenResponsesBrainConfig extends StreamEngineConfig {
|
|
|
7
7
|
apiKey?: string;
|
|
8
8
|
/** Extra headers merged into every request (auth + content-type are hard-locked after the merge). */
|
|
9
9
|
headers?: Record<string, string>;
|
|
10
|
-
/** Inject a custom fetch (tests / proxies)
|
|
10
|
+
/** Inject a custom fetch (tests / proxies); the default transport and the `fetchImpl` seat are
|
|
11
|
+
* `resolveFetch` (transport.ts). */
|
|
11
12
|
fetchImpl?: typeof fetch;
|
|
12
13
|
/**
|
|
13
14
|
* Replay prior assistant reasoning items back to the endpoint on the turns that carry tool calls
|
|
@@ -9,6 +9,8 @@ import { OUTPUT_CAP_KEYS, RESPONSES_RESERVED, applyExtraBody, effectiveOutputCap
|
|
|
9
9
|
import { adjudicateModelRoute, applyRouteCredentialHeaders, assertServableRequestRoot, createBrainRouteJudge, joinRequestUrl, resolveRequestRoot, resolveRouteCredential, routeRefusalText } from "./route-adjudicator.js";
|
|
10
10
|
import { declaredOffSpelling, mintEffortWireValue, reasoningRequestCarried } from "./reasoning.js";
|
|
11
11
|
import { runStreamingBrain } from "./stream-engine.js";
|
|
12
|
+
import { refuseRetiredTimeoutKnobs } from "./timeout.js";
|
|
13
|
+
import { resolveFetch } from "./transport.js";
|
|
12
14
|
const DEGENERATE_POLL_CHARS = 64;
|
|
13
15
|
const MALFORMED_SAMPLE_CHARS = 160;
|
|
14
16
|
const MALFORMED_ARGS_CHARS = 200;
|
|
@@ -218,7 +220,8 @@ function mapTerminalStopReason(incompleteReason, hadToolCalls) {
|
|
|
218
220
|
}
|
|
219
221
|
export function createOpenResponsesBrain(config = {}) {
|
|
220
222
|
assertServableRequestRoot(config.baseUrl, "config.baseUrl");
|
|
221
|
-
|
|
223
|
+
refuseRetiredTimeoutKnobs(config, "construction", "createOpenResponsesBrain config");
|
|
224
|
+
const doFetch = resolveFetch(config);
|
|
222
225
|
const stream = (model, context, options) => {
|
|
223
226
|
let sentMaxTokens;
|
|
224
227
|
let sentMaxTokensLane = "unset";
|
package/dist/brain/openai.d.ts
CHANGED
|
@@ -7,7 +7,8 @@ export interface OpenAIBrainConfig extends StreamEngineConfig {
|
|
|
7
7
|
apiKey?: string;
|
|
8
8
|
/** Extra headers merged into every request. */
|
|
9
9
|
headers?: Record<string, string>;
|
|
10
|
-
/** Inject a custom fetch (tests / proxies).
|
|
10
|
+
/** Inject a custom fetch (tests / proxies). Absent ⇒ the engine's default transport; the contract for
|
|
11
|
+
* both cases is `resolveFetch` (transport.ts). */
|
|
11
12
|
fetchImpl?: typeof fetch;
|
|
12
13
|
/**
|
|
13
14
|
* For weaker models that emit tool calls as TEXT (`<tool_call>{...}</tool_call>` / fenced json)
|
package/dist/brain/openai.js
CHANGED
|
@@ -10,6 +10,8 @@ import { BrainError } from "./errors.js";
|
|
|
10
10
|
import { adjudicateModelRoute, applyRouteCredentialHeaders, assertServableRequestRoot, createBrainRouteJudge, joinRequestUrl, resolveRequestRoot, resolveRouteCredential, routeRefusalText } from "./route-adjudicator.js";
|
|
11
11
|
import { declaredOffSpelling, mintEffortWireValue, reasoningRequestCarried } from "./reasoning.js";
|
|
12
12
|
import { runStreamingBrain } from "./stream-engine.js";
|
|
13
|
+
import { refuseRetiredTimeoutKnobs } from "./timeout.js";
|
|
14
|
+
import { resolveFetch } from "./transport.js";
|
|
13
15
|
function closeToolCallAccum(acc) {
|
|
14
16
|
if (acc.closedTc)
|
|
15
17
|
return acc.closedTc;
|
|
@@ -282,7 +284,8 @@ export function inferMaxTokensField(modelId) {
|
|
|
282
284
|
}
|
|
283
285
|
export function createOpenAIBrain(config = {}) {
|
|
284
286
|
assertServableRequestRoot(config.baseUrl, "config.baseUrl");
|
|
285
|
-
|
|
287
|
+
refuseRetiredTimeoutKnobs(config, "construction", "createOpenAIBrain config");
|
|
288
|
+
const doFetch = resolveFetch(config);
|
|
286
289
|
const stream = (model, context, options) => {
|
|
287
290
|
let sentMaxTokens;
|
|
288
291
|
let sentMaxTokensLane = "unset";
|
|
@@ -4,7 +4,7 @@ import { type BrainTimeoutConfig } from "./timeout.js";
|
|
|
4
4
|
* Shared streaming engine for the OpenAI- and Anthropic-shaped brains (design/32, consolidation).
|
|
5
5
|
*
|
|
6
6
|
* It owns the **bug-prone machinery that was duplicated verbatim** across `openai.ts` and
|
|
7
|
-
* `anthropic.ts` — the
|
|
7
|
+
* `anthropic.ts` — the first-byte window + retry loop, the three-tier timer logic (first-token + idle
|
|
8
8
|
* watchdog) with their flag resets, the SSE read loop + line framing, the one-shot cleanup, and the
|
|
9
9
|
* `.catch/.finally` error wrapping. Centralizing the timer logic is the point: the same first-token /
|
|
10
10
|
* idle reset bug was fixed twice (1.38.2 / 1.40.2) precisely because this lived in two copies.
|
|
@@ -231,14 +231,15 @@ export declare function runStreamingBrain(args: {
|
|
|
231
231
|
makeParser: (ctrl: StreamControls) => StreamParser;
|
|
232
232
|
/**
|
|
233
233
|
* 1.296 件A — per-call ADVISORY stall watchdogs (`StreamOptions.stallTimeouts`). Priority chain:
|
|
234
|
-
* brain construction-time config (`BrainTimeoutConfig.
|
|
235
|
-
* `idleTimeoutMs`) wins whenever PRESENT — including an explicit 0
|
|
236
|
-
*
|
|
237
|
-
*
|
|
238
|
-
* timers can't see a gateway that never
|
|
234
|
+
* brain construction-time config (`BrainTimeoutConfig.firstByteTimeoutMs`/`firstTokenTimeoutMs`/
|
|
235
|
+
* `idleTimeoutMs`) wins whenever PRESENT — including an explicit 0, which means "deliberately
|
|
236
|
+
* disabled" — and only an ABSENT (undefined) field adopts the advisory value; both absent ⇒ the
|
|
237
|
+
* engine default constant for that segment (`timeout.ts`). `firstByteMs` bounds the
|
|
238
|
+
* fetch→response-headers wait: the engine's post-headers timers can't see a gateway that never
|
|
239
|
+
* returns headers. The connect clock is not per-call: it is the process dispatcher's (`transport.ts`).
|
|
239
240
|
*/
|
|
240
241
|
stallTimeouts?: {
|
|
241
|
-
|
|
242
|
+
firstByteMs?: number;
|
|
242
243
|
firstTokenMs?: number;
|
|
243
244
|
idleMs?: number;
|
|
244
245
|
};
|
|
@@ -6,7 +6,7 @@ import { stripUrlCredentials } from "./route-adjudicator.js";
|
|
|
6
6
|
import { classifyInputTooLong } from "./input-too-long.js";
|
|
7
7
|
import { FAST_MAX_BACKOFF_MS, providerWaitHint, retryBackoffMs } from "./retry.js";
|
|
8
8
|
import { bindBrainStatusEmitter, emitBrainStatus, emitBrainTelemetry } from "./status-sink.js";
|
|
9
|
-
import {
|
|
9
|
+
import { createAttemptController, firstByteWindow, refuseRetiredTimeoutKnobs, resolveFirstByteRetries, resolveWatchdogMs, STALL_FIRST_BYTE_MS, STALL_FIRST_TOKEN_MS, STALL_IDLE_MS, } from "./timeout.js";
|
|
10
10
|
import { readMember, thrownText } from "../core/thrown-value.js";
|
|
11
11
|
import { redactSecrets } from "../core/untrusted-egress.js";
|
|
12
12
|
const DEFAULT_MAX_RETRIES = 10;
|
|
@@ -250,10 +250,18 @@ async function sleepAnnouncingRetry(totalMs, signal, frame) {
|
|
|
250
250
|
return;
|
|
251
251
|
}
|
|
252
252
|
}
|
|
253
|
-
function
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
253
|
+
function retryAnnouncement(failure) {
|
|
254
|
+
if (failure.kind !== "stalled")
|
|
255
|
+
return { errClass: "transport", detail: "connection lost, reconnecting" };
|
|
256
|
+
const knob = failure.allowanceMs !== undefined && failure.allowanceMs > 0
|
|
257
|
+
? `${failure.knob} ${failure.timeoutMs - failure.allowanceMs} + ${failure.allowanceMs} upload allowance`
|
|
258
|
+
: failure.knob;
|
|
259
|
+
return { errClass: "stall", detail: `${failure.watchdog} watchdog fired after ${failure.timeoutMs}ms (${knob}), retrying` };
|
|
260
|
+
}
|
|
261
|
+
function firstByteTimeoutMessage(windowsMs) {
|
|
262
|
+
const [first, ...retries] = windowsMs;
|
|
263
|
+
const waited = retries.length === 0 ? `within ${first ?? 0}ms` : `within ${first ?? 0}ms, then ${retries.map((w) => `${w}ms`).join(", then ")} on the retry`;
|
|
264
|
+
return `first-byte timeout: no response headers ${waited} (firstByteTimeoutMs) — if a proxy or gateway holds responses until they complete, raise firstByteTimeoutMs to wait longer`;
|
|
257
265
|
}
|
|
258
266
|
export function runStreamingBrain(args) {
|
|
259
267
|
const { model, doFetch, signal, config, httpLabel, buildRequest, makeParser, stallTimeouts } = args;
|
|
@@ -262,6 +270,7 @@ export function runStreamingBrain(args) {
|
|
|
262
270
|
let announcedRetry = false;
|
|
263
271
|
let announcedWait = false;
|
|
264
272
|
let terminalWaitDetail = "first token arrived";
|
|
273
|
+
let firstTokenArrived = false;
|
|
265
274
|
let terminalRetryPhase = "recovered";
|
|
266
275
|
let terminalRetryDetail = "recovered after retrying";
|
|
267
276
|
let requestIdSeen;
|
|
@@ -271,7 +280,13 @@ export function runStreamingBrain(args) {
|
|
|
271
280
|
const aborted = signal?.aborted === true || isAbortError(err);
|
|
272
281
|
terminalRetryPhase = "gave_up";
|
|
273
282
|
terminalRetryDetail = aborted ? "cancelled while retrying" : "retries exhausted";
|
|
274
|
-
terminalWaitDetail =
|
|
283
|
+
terminalWaitDetail = firstTokenArrived
|
|
284
|
+
? aborted
|
|
285
|
+
? "cancelled after the first token arrived"
|
|
286
|
+
: "the call failed after the first token arrived"
|
|
287
|
+
: aborted
|
|
288
|
+
? "cancelled while waiting for the first token"
|
|
289
|
+
: "the first-token wait ended in failure";
|
|
275
290
|
const errorMsg = emptyAssistant(model);
|
|
276
291
|
errorMsg.stopReason = aborted ? "aborted" : "error";
|
|
277
292
|
errorMsg.errorMessage = thrownText(err);
|
|
@@ -315,33 +330,84 @@ export function runStreamingBrain(args) {
|
|
|
315
330
|
const baseDelay = config.retryDelayMs ?? 500;
|
|
316
331
|
const deterministicConnectBudget = Math.min(resolveDeterministicConnectRetries(config.deterministicConnectRetries), maxRetries);
|
|
317
332
|
const deterministicLaneActive = deterministicConnectBudget < maxRetries;
|
|
318
|
-
|
|
319
|
-
const
|
|
320
|
-
const
|
|
333
|
+
refuseRetiredTimeoutKnobs(stallTimeouts, "advisory", "StreamOptions.stallTimeouts");
|
|
334
|
+
const firstByteTimeoutMs = resolveWatchdogMs(config.firstByteTimeoutMs, stallTimeouts?.firstByteMs, STALL_FIRST_BYTE_MS, "firstByteTimeoutMs");
|
|
335
|
+
const firstTokenTimeoutMs = resolveWatchdogMs(config.firstTokenTimeoutMs, stallTimeouts?.firstTokenMs, STALL_FIRST_TOKEN_MS, "firstTokenTimeoutMs");
|
|
336
|
+
const idleTimeoutMs = resolveWatchdogMs(config.idleTimeoutMs, stallTimeouts?.idleMs, STALL_IDLE_MS, "idleTimeoutMs");
|
|
337
|
+
const firstByteRetryBudget = resolveFirstByteRetries(config.firstByteRetries);
|
|
338
|
+
const armWaitDisclosure = (ceilingMs) => {
|
|
339
|
+
if (!(ceilingMs > WAITING_FIRST_TOKEN_AFTER_MS))
|
|
340
|
+
return { stop: () => undefined };
|
|
341
|
+
const startedAt = Date.now();
|
|
342
|
+
const emitWaitFrame = bindBrainStatusEmitter();
|
|
343
|
+
let timer;
|
|
344
|
+
let stopped = false;
|
|
345
|
+
const schedule = (delayMs) => {
|
|
346
|
+
timer = setTimeout(() => {
|
|
347
|
+
timer = undefined;
|
|
348
|
+
if (stopped || Date.now() - startedAt >= ceilingMs)
|
|
349
|
+
return;
|
|
350
|
+
announcedWait = true;
|
|
351
|
+
emitWaitFrame({ phase: "waiting_first_token", detail: "waiting for the first token", elapsedMs: Date.now() - startedAt, timeoutMs: ceilingMs });
|
|
352
|
+
schedule(WAITING_FIRST_TOKEN_EVERY_MS);
|
|
353
|
+
}, delayMs);
|
|
354
|
+
};
|
|
355
|
+
schedule(WAITING_FIRST_TOKEN_AFTER_MS);
|
|
356
|
+
return {
|
|
357
|
+
stop: () => {
|
|
358
|
+
stopped = true;
|
|
359
|
+
if (timer)
|
|
360
|
+
clearTimeout(timer);
|
|
361
|
+
timer = undefined;
|
|
362
|
+
},
|
|
363
|
+
};
|
|
364
|
+
};
|
|
321
365
|
let attempt = 0;
|
|
322
366
|
let thinkingRetries = 0;
|
|
367
|
+
let firstByteStalls = 0;
|
|
368
|
+
const firstByteWindowsWaited = [];
|
|
323
369
|
let startEmitted = false;
|
|
324
370
|
for (;;) {
|
|
325
371
|
let res;
|
|
326
372
|
let connect;
|
|
327
|
-
for (;;
|
|
373
|
+
for (;;) {
|
|
328
374
|
if (signal?.aborted) {
|
|
329
375
|
const e = new Error("aborted");
|
|
330
376
|
e.name = "AbortError";
|
|
331
377
|
throw e;
|
|
332
378
|
}
|
|
333
|
-
const
|
|
379
|
+
const window = firstByteWindow(firstByteTimeoutMs, Buffer.byteLength(req.body), firstByteStalls);
|
|
380
|
+
const cc = createAttemptController(window.windowMs, signal);
|
|
334
381
|
let r;
|
|
335
382
|
let netErr;
|
|
383
|
+
let firstByteStall;
|
|
336
384
|
requestIdSeen = undefined;
|
|
337
385
|
attemptPartial = undefined;
|
|
386
|
+
const headerWait = armWaitDisclosure(window.windowMs);
|
|
387
|
+
const stalledAtFirstByte = () => {
|
|
388
|
+
firstByteWindowsWaited.push(window.windowMs);
|
|
389
|
+
return {
|
|
390
|
+
kind: "stalled",
|
|
391
|
+
watchdog: "first-byte",
|
|
392
|
+
knob: "firstByteTimeoutMs",
|
|
393
|
+
timeoutMs: window.windowMs,
|
|
394
|
+
allowanceMs: window.allowanceMs,
|
|
395
|
+
err: new BrainError("network", firstByteTimeoutMessage(firstByteWindowsWaited), undefined, { apiError: true }),
|
|
396
|
+
};
|
|
397
|
+
};
|
|
338
398
|
try {
|
|
339
399
|
r = await doFetch(req.url, { method: "POST", headers: req.headers, body: req.body, signal: cc.signal });
|
|
340
400
|
requestIdSeen = readProviderRequestId(r);
|
|
401
|
+
if (cc.timedOut()) {
|
|
402
|
+
discardResponseBody(r);
|
|
403
|
+
r = undefined;
|
|
404
|
+
requestIdSeen = undefined;
|
|
405
|
+
firstByteStall = stalledAtFirstByte();
|
|
406
|
+
}
|
|
341
407
|
}
|
|
342
408
|
catch (e) {
|
|
343
409
|
if (cc.timedOut()) {
|
|
344
|
-
|
|
410
|
+
firstByteStall = stalledAtFirstByte();
|
|
345
411
|
}
|
|
346
412
|
else if (isAbortError(e) || signal?.aborted) {
|
|
347
413
|
cc.dispose();
|
|
@@ -355,6 +421,9 @@ export function runStreamingBrain(args) {
|
|
|
355
421
|
netErr = e;
|
|
356
422
|
}
|
|
357
423
|
}
|
|
424
|
+
finally {
|
|
425
|
+
headerWait.stop();
|
|
426
|
+
}
|
|
358
427
|
if (r && r.ok && r.body) {
|
|
359
428
|
cc.clearTimer();
|
|
360
429
|
connect = cc;
|
|
@@ -364,6 +433,27 @@ export function runStreamingBrain(args) {
|
|
|
364
433
|
}
|
|
365
434
|
cc.clearTimer();
|
|
366
435
|
try {
|
|
436
|
+
if (firstByteStall !== undefined) {
|
|
437
|
+
if (firstByteStalls >= firstByteRetryBudget)
|
|
438
|
+
throw firstByteStall.err;
|
|
439
|
+
firstByteStalls++;
|
|
440
|
+
const announcement = retryAnnouncement(firstByteStall);
|
|
441
|
+
const delayMs = retryBackoffMs(baseDelay, firstByteStalls - 1);
|
|
442
|
+
emitBrainTelemetry({ kind: "retry", attempt: firstByteStalls, phase: "connect", errClass: announcement.errClass, nextDelayMs: delayMs });
|
|
443
|
+
announcedRetry = true;
|
|
444
|
+
cc.abort();
|
|
445
|
+
cc.dispose();
|
|
446
|
+
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
|
|
447
|
+
phase: "reconnecting",
|
|
448
|
+
detail: announcement.detail,
|
|
449
|
+
retryInSec: Math.ceil(remainingMs / 1000),
|
|
450
|
+
retryInMs: remainingMs,
|
|
451
|
+
attempt: firstByteStalls,
|
|
452
|
+
maxRetries: firstByteRetryBudget,
|
|
453
|
+
errClass: announcement.errClass,
|
|
454
|
+
}));
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
367
457
|
let overflowBody;
|
|
368
458
|
let overflowDeclineNote = "";
|
|
369
459
|
if (r !== undefined && r.status === 400) {
|
|
@@ -402,6 +492,7 @@ export function runStreamingBrain(args) {
|
|
|
402
492
|
errClass: "output_cap",
|
|
403
493
|
...(namesTheFailure(r?.status) ? { errorStatus: r.status } : {}),
|
|
404
494
|
});
|
|
495
|
+
attempt++;
|
|
405
496
|
continue;
|
|
406
497
|
}
|
|
407
498
|
overflowDeclineNote =
|
|
@@ -468,6 +559,7 @@ export function runStreamingBrain(args) {
|
|
|
468
559
|
errClass,
|
|
469
560
|
...(retryStatus !== undefined ? { errorStatus: retryStatus } : {}),
|
|
470
561
|
}));
|
|
562
|
+
attempt++;
|
|
471
563
|
continue;
|
|
472
564
|
}
|
|
473
565
|
if (netErr)
|
|
@@ -494,39 +586,14 @@ export function runStreamingBrain(args) {
|
|
|
494
586
|
let firstTokenSeen = false;
|
|
495
587
|
let firedFirstTokenDeadlineMs;
|
|
496
588
|
let ftTimer;
|
|
497
|
-
let
|
|
498
|
-
const clearWaitTimer = () => {
|
|
499
|
-
if (waitTimer) {
|
|
500
|
-
clearTimeout(waitTimer);
|
|
501
|
-
waitTimer = undefined;
|
|
502
|
-
}
|
|
503
|
-
};
|
|
589
|
+
let firstTokenWait = { stop: () => undefined };
|
|
504
590
|
if (firstTokenTimeoutMs && firstTokenTimeoutMs > 0) {
|
|
505
591
|
ftTimer = setTimeout(() => {
|
|
506
592
|
firedFirstTokenDeadlineMs = firstTokenTimeoutMs;
|
|
507
|
-
|
|
593
|
+
firstTokenWait.stop();
|
|
508
594
|
void reader.cancel().catch(() => undefined);
|
|
509
595
|
}, firstTokenTimeoutMs);
|
|
510
|
-
|
|
511
|
-
const waitStartedAt = Date.now();
|
|
512
|
-
const emitWaitFrame = bindBrainStatusEmitter();
|
|
513
|
-
const scheduleWaitFrame = (delayMs) => {
|
|
514
|
-
waitTimer = setTimeout(() => {
|
|
515
|
-
waitTimer = undefined;
|
|
516
|
-
if (firstTokenSeen || firedFirstTokenDeadlineMs !== undefined || Date.now() - waitStartedAt >= firstTokenTimeoutMs)
|
|
517
|
-
return;
|
|
518
|
-
announcedWait = true;
|
|
519
|
-
emitWaitFrame({
|
|
520
|
-
phase: "waiting_first_token",
|
|
521
|
-
detail: "waiting for the first token",
|
|
522
|
-
elapsedMs: Date.now() - waitStartedAt,
|
|
523
|
-
timeoutMs: firstTokenTimeoutMs,
|
|
524
|
-
});
|
|
525
|
-
scheduleWaitFrame(WAITING_FIRST_TOKEN_EVERY_MS);
|
|
526
|
-
}, delayMs);
|
|
527
|
-
};
|
|
528
|
-
scheduleWaitFrame(WAITING_FIRST_TOKEN_AFTER_MS);
|
|
529
|
-
}
|
|
596
|
+
firstTokenWait = armWaitDisclosure(firstTokenTimeoutMs);
|
|
530
597
|
}
|
|
531
598
|
let firedIdleDeadlineMs;
|
|
532
599
|
let idleTimer;
|
|
@@ -546,19 +613,20 @@ export function runStreamingBrain(args) {
|
|
|
546
613
|
if (firstTokenSeen)
|
|
547
614
|
return;
|
|
548
615
|
firstTokenSeen = true;
|
|
616
|
+
firstTokenArrived = true;
|
|
549
617
|
firedFirstTokenDeadlineMs = undefined;
|
|
550
618
|
if (ftTimer) {
|
|
551
619
|
clearTimeout(ftTimer);
|
|
552
620
|
ftTimer = undefined;
|
|
553
621
|
}
|
|
554
|
-
|
|
622
|
+
firstTokenWait.stop();
|
|
555
623
|
};
|
|
556
624
|
cleanup = () => {
|
|
557
625
|
if (ftTimer) {
|
|
558
626
|
clearTimeout(ftTimer);
|
|
559
627
|
ftTimer = undefined;
|
|
560
628
|
}
|
|
561
|
-
|
|
629
|
+
firstTokenWait.stop();
|
|
562
630
|
if (idleTimer) {
|
|
563
631
|
clearTimeout(idleTimer);
|
|
564
632
|
idleTimer = undefined;
|
|
@@ -631,7 +699,7 @@ export function runStreamingBrain(args) {
|
|
|
631
699
|
clearTimeout(ftTimer);
|
|
632
700
|
ftTimer = undefined;
|
|
633
701
|
}
|
|
634
|
-
|
|
702
|
+
firstTokenWait.stop();
|
|
635
703
|
if (idleTimer) {
|
|
636
704
|
clearTimeout(idleTimer);
|
|
637
705
|
idleTimer = undefined;
|
|
@@ -678,7 +746,7 @@ export function runStreamingBrain(args) {
|
|
|
678
746
|
throw failure.err;
|
|
679
747
|
cleanup?.();
|
|
680
748
|
cleanup = undefined;
|
|
681
|
-
const announcement =
|
|
749
|
+
const announcement = retryAnnouncement(failure);
|
|
682
750
|
if (snap.hasOnlyThinking) {
|
|
683
751
|
if (thinkingRetries >= THINKING_RETRY_BUDGET)
|
|
684
752
|
throw failure.err;
|