aegiscode 7.0.26 → 7.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SOURCE-OFFER.txt +9 -0
- package/package.json +4 -7
- package/scripts/check.mjs +1 -52
- package/scripts/demo.mjs +1 -94
- package/scripts/predist.mjs +1 -75
- package/scripts/runtests.mjs +1 -56
- package/scripts/seal-gate.mjs +1 -74
- package/src/agents.js +1 -139
- package/src/app.js +1 -2957
- package/src/art.js +1 -289
- package/src/chatflow.js +1 -3023
- package/src/checkpoint.js +1 -90
- package/src/clipboard.js +1 -424
- package/src/cloudsync.js +1 -528
- package/src/commands.js +1 -4089
- package/src/config.js +1 -581
- package/src/connect.js +1 -349
- package/src/credentials.js +1 -33
- package/src/deps.js +1 -232
- package/src/devrun.js +1 -187
- package/src/diff.js +1 -333
- package/src/diffview.js +1 -296
- package/src/engine.js +1 -297
- package/src/entitlement.js +1 -355
- package/src/events.js +1 -707
- package/src/export.js +1 -67
- package/src/filepreview.js +1 -153
- package/src/fingerprint.js +1 -592
- package/src/format.js +1 -259
- package/src/fuzzy.js +1 -119
- package/src/history.js +1 -423
- package/src/init.js +1 -175
- package/src/input.js +1 -139
- package/src/keys.js +1 -183
- package/src/markdown.js +1 -306
- package/src/models.js +1 -507
- package/src/naming.js +1 -57
- package/src/overlays.js +1 -462
- package/src/panels.js +1 -1418
- package/src/ping.js +1 -226
- package/src/presence.js +1 -238
- package/src/render.js +1 -516
- package/src/screen.js +1 -518
- package/src/screens.js +1 -896
- package/src/secret.js +1 -59
- package/src/securefs.js +1 -359
- package/src/settings.js +1 -155
- package/src/shared.js +1 -52
- package/src/sharedpaths.js +1 -53
- package/src/signup.js +1 -156
- package/src/summarize.js +1 -95
- package/src/sysdiag.js +1 -585
- package/src/system.js +1 -40
- package/src/theme.js +1 -570
- package/src/title.js +1 -220
- package/src/todos.js +1 -115
- package/src/tokens.js +1 -263
- package/src/topup.js +1 -210
- package/src/update.js +1 -191
- package/vendor/client/aegis.js +1 -2441
- package/vendor/client/attest.js +1 -518
- package/vendor/client/authorship-key.js +1 -410
- package/vendor/client/auto-provision.js +1 -134
- package/vendor/client/brain-catalog.js +1 -268
- package/vendor/client/byok-catalog.js +1 -199
- package/vendor/client/catalog-client.js +1 -487
- package/vendor/client/conversation-export.js +1 -177
- package/vendor/client/credentials.js +1 -494
- package/vendor/client/device-fingerprint.js +1 -141
- package/vendor/client/drive-connect.js +1 -110
- package/vendor/client/drive-upload.js +1 -234
- package/vendor/client/env-file.js +1 -1273
- package/vendor/client/fingerprint.js +1 -1064
- package/vendor/client/foreign-memory.js +1 -669
- package/vendor/client/google-signin.js +1 -133
- package/vendor/client/hooks.js +1 -430
- package/vendor/client/model-discovery.js +1 -373
- package/vendor/client/naming.js +1 -283
- package/vendor/client/platform.js +1 -937
- package/vendor/client/prefs.js +1 -351
- package/vendor/client/presence-constants.js +1 -107
- package/vendor/client/presence.js +1 -438
- package/vendor/client/record-vault.js +3 -1055
- package/vendor/client/session-store.js +1 -575
- package/vendor/client/topup.js +1 -297
- package/vendor/client/touch.js +1 -458
- package/vendor/client/update.js +1 -763
- package/vendor/client/user-models.js +1 -381
- package/vendor/desktop/lib/local/agents.js +1 -105
- package/vendor/desktop/lib/local/attachments.js +1 -276
- package/vendor/desktop/lib/local/autonomous.js +1 -1045
- package/vendor/desktop/lib/local/claude-pro.js +1 -1028
- package/vendor/desktop/lib/local/context.js +1 -125
- package/vendor/desktop/lib/local/engine.js +1 -3666
- package/vendor/desktop/lib/local/entitlement.js +1 -660
- package/vendor/desktop/lib/local/exec-plan.js +1 -269
- package/vendor/desktop/lib/local/git-scope.js +1 -228
- package/vendor/desktop/lib/local/integrity.js +1 -163
- package/vendor/desktop/lib/local/local.js +1 -2145
- package/vendor/desktop/lib/local/prompt.js +1 -280
- package/vendor/desktop/lib/local/queue.js +1 -560
- package/vendor/desktop/lib/local/session-rounds.js +1 -238
- package/vendor/desktop/lib/local/shell.js +1 -712
- package/vendor/desktop/lib/local/spill.js +1 -75
- package/vendor/desktop/lib/local/tools.js +1 -1328
- package/vendor/desktop/lib/local/turn-guard.js +1 -300
- package/vendor/desktop/lib/local/worktree-lock.js +1 -235
- package/vendor/desktop/lib/settings.js +1 -890
- package/vendor/desktop/main.js +3 -4890
- package/vendor/desktop/renderer/usage.js +1 -1239
- package/vendor/hosts/install.js +1 -638
- package/vendor/hosts/openai-shim.js +1 -515
- package/vendor/hosts/spec.js +1 -164
- package/vendor/hosts/targets.js +1 -346
- package/vendor/mcp/server.js +1 -221
- package/vendor/mcp/tools.js +1 -408
- package/vendor/one/src/child.js +1 -285
- package/vendor/one/src/permissions.js +1 -383
|
@@ -1,2147 +1,3 @@
|
|
|
1
1
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
2
|
// Copyright (C) 2026 Niklas Borneklint — AEGIS Code (aegiscode) / Aegiscloud.
|
|
3
|
-
// AEGIS Code (aegiscode) — source: https://github.com/aegisinfo/aegiscode-plugin
|
|
4
|
-
// Conveying a modified copy, or running one as a network service, owes the
|
|
5
|
-
// corresponding source under this licence (§5/§13) — see LICENSING.md.
|
|
6
|
-
'use strict';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* local.js — the local-model transport: run a model on hardware you own.
|
|
10
|
-
*
|
|
11
|
-
* This is the third class, alongside `aegis` (pooled, metered) and `byok`
|
|
12
|
-
* (relayed, handling fee). It is also the one class that bills NOTHING, which
|
|
13
|
-
* is exactly why it is fenced the way it is.
|
|
14
|
-
*
|
|
15
|
-
* The fee argument, restated because every rule below follows from it. A
|
|
16
|
-
* transport that dials a user-supplied URL has no pooled margin to take, no
|
|
17
|
-
* handling fee to charge and no account key attached, so a REMOTE call there is
|
|
18
|
-
* an unpaid turn: the relay accepts a fixed catalog of provider ids
|
|
19
|
-
* (aegis1 services/nexus_provider/catalog.py, upstream key from
|
|
20
|
-
* `X-Provider-Key`), so an arbitrary remote URL has nothing to be billed
|
|
21
|
-
* against even if a client wanted to invoice it. That is why remote direct
|
|
22
|
-
* dialing is gone from this codebase and stays gone — there is deliberately no
|
|
23
|
-
* flag or env var that re-opens it, because such a flag would be a billing
|
|
24
|
-
* bypass.
|
|
25
|
-
*
|
|
26
|
-
* A model on YOUR OWN MACHINE is the case that argument never covered. There is
|
|
27
|
-
* no vendor to pay, so there is nothing to bill and nobody being denied a
|
|
28
|
-
* payment: the compute was already bought and is sitting idle. `aegis` and
|
|
29
|
-
* `byok` are lanes to somebody else's GPU; this is the lane to your own.
|
|
30
|
-
*
|
|
31
|
-
* So the fence is the whole feature: `isLocalEndpoint()` is checked before any
|
|
32
|
-
* byte goes out, and it FAILS CLOSED — anything that does not parse, carries no
|
|
33
|
-
* host, or resolves to a public address is remote, i.e. refused. A false
|
|
34
|
-
* "local" is an unpaid turn; a false "remote" is a refusal the user fixes by
|
|
35
|
-
* pointing at a local address. In a tie, refuse.
|
|
36
|
-
*
|
|
37
|
-
* Wire-wise this speaks Ollama's OpenAI-compatible surface
|
|
38
|
-
* (`POST /v1/chat/completions`, keyless) plus its native `GET /api/tags` for
|
|
39
|
-
* the model list. Anything that serves an OpenAI-compatible endpoint on the
|
|
40
|
-
* same box — llama.cpp, LM Studio, vLLM — works by pointing `baseURL` at it;
|
|
41
|
-
* only the model LIST is Ollama-specific, and that failure is recoverable
|
|
42
|
-
* (see listTags) rather than fatal.
|
|
43
|
-
*/
|
|
44
|
-
|
|
45
|
-
// The standalone token-budget trimmer (plan P1 §5.1), used here for the one
|
|
46
|
-
// thing it was written for and never given a caller: keeping a turn's wire
|
|
47
|
-
// payload inside the window the daemon actually granted. See `fitToWindowOf`
|
|
48
|
-
// below for why the transport and not the caller owns this.
|
|
49
|
-
const { estimateTokens, trimContext } = require('./context.js');
|
|
50
|
-
const attachmentsModule = require('./attachments.js');
|
|
51
|
-
|
|
52
|
-
/** Where a local daemon listens by default (Ollama's stock port). */
|
|
53
|
-
const DEFAULT_BASE = 'http://localhost:11434';
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* How long a local turn tolerates a silent stream before failing. Local
|
|
57
|
-
* inference is SLOW to first token — a cold 7B model on a laptop CPU can think
|
|
58
|
-
* for a minute before emitting anything, and a warm one still loads weights on
|
|
59
|
-
* the first call after a restart — so this is generous where the cloud client's
|
|
60
|
-
* 60s default would abort a perfectly healthy turn.
|
|
61
|
-
*/
|
|
62
|
-
const SSE_IDLE_TIMEOUT_MS = 300000;
|
|
63
|
-
|
|
64
|
-
/**
|
|
65
|
-
* How long the daemon may take to answer with response headers at all — the
|
|
66
|
-
* window BEFORE the first byte, which `SSE_IDLE_TIMEOUT_MS` above cannot cover:
|
|
67
|
-
* that timer is armed per parsed payload, and there is no payload, and not even
|
|
68
|
-
* a body to read, until this fetch resolves. The cloud transport bounds exactly
|
|
69
|
-
* this window in `boundedFetch` (client/aegis.js: "a server that accepts the
|
|
70
|
-
* socket and never sends response headers ... nothing is watching at all"); the
|
|
71
|
-
* local lane had no bound at all, so a daemon that accepted the socket and then
|
|
72
|
-
* went quiet left the turn as a spinner with no error, no timeout and no way
|
|
73
|
-
* out but Stop.
|
|
74
|
-
*
|
|
75
|
-
* Deliberately the SAME budget as the stall timer rather than a tighter one,
|
|
76
|
-
* because this wait is a legitimate part of a local turn on the hardware this
|
|
77
|
-
* class exists for: measured on the machine this was written (Ollama 0.12,
|
|
78
|
-
* `omega-sonnet5-lite` — a 1.5B model, daemon already warm), the first byte
|
|
79
|
-
* arrived **107.8s** after the request, all of it load + prompt eval with
|
|
80
|
-
* nothing to show. A bound tight enough to catch a wedged daemon fast would
|
|
81
|
-
* abort healthy turns; five minutes catches only the genuinely dead ones, and
|
|
82
|
-
* the error says which it is.
|
|
83
|
-
*/
|
|
84
|
-
const FIRST_BYTE_TIMEOUT_MS = SSE_IDLE_TIMEOUT_MS;
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* The response-header deadline the RUNTIME imposes, independent of the one
|
|
88
|
-
* above — and the reason a second clock has to be handled here at all.
|
|
89
|
-
*
|
|
90
|
-
* Measured on this host: a live `/v1/chat/completions` turn whose prompt prefill
|
|
91
|
-
* outlasted it (a 2521-token payload on a contended CPU) failed with
|
|
92
|
-
* `TypeError: fetch failed`, `cause: HeadersTimeoutError`,
|
|
93
|
-
* `code: 'UND_ERR_HEADERS_TIMEOUT'` at ~300s — undici's default `headersTimeout`,
|
|
94
|
-
* not this transport's `FIRST_BYTE_TIMEOUT_MS`. Any caller that states a longer
|
|
95
|
-
* bound (`firstByteTimeoutMs: 500000`) therefore does not get a longer wait: the
|
|
96
|
-
* runtime cuts the turn first, and it cut it as a raw fetch failure rather than
|
|
97
|
-
* in the timeout shape every other "waited too long" on this lane uses, so the
|
|
98
|
-
* host rendered an opaque error for a plain slow prefill. A local prefill IS
|
|
99
|
-
* routinely slower than this: the daemon's own log shows 236.7s for 2048 tokens
|
|
100
|
-
* while another model was loaded.
|
|
101
|
-
*
|
|
102
|
-
* Because the engine's contract is that a timeout arrives as `status: 0` +
|
|
103
|
-
* `code: 'timeout'` (the cloud client's shape, so "waited too long" stays one
|
|
104
|
-
* branch in the renderer), the second clock is translated into that shape rather
|
|
105
|
-
* than passed through. The number is stated here as a measured constant, not read
|
|
106
|
-
* from the runtime, which does not expose it.
|
|
107
|
-
*/
|
|
108
|
-
const HEADERS_DEADLINE_MS = 300000;
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Is this rejection the runtime's own response-header deadline?
|
|
112
|
-
*
|
|
113
|
-
* Matched on the undici cause rather than on the message text, and tolerant of
|
|
114
|
-
* the several shapes it wears (`HeadersTimeoutError` name, `UND_ERR_HEADERS_TIMEOUT`
|
|
115
|
-
* code) — a socket that dies for another reason must NOT be reported as a
|
|
116
|
-
* timeout, which is why this is a predicate rather than a catch-all.
|
|
117
|
-
*/
|
|
118
|
-
function isHeadersDeadline(err) {
|
|
119
|
-
if (!err) return false;
|
|
120
|
-
const cause = err.cause || err;
|
|
121
|
-
if (!cause) return false;
|
|
122
|
-
const code = String(cause.code || '');
|
|
123
|
-
const name = String(cause.name || '');
|
|
124
|
-
return code === 'UND_ERR_HEADERS_TIMEOUT' || name === 'HeadersTimeoutError';
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/** How long the daemon probe waits. Short: this runs behind a UI spinner, and
|
|
128
|
-
* "no daemon" must read as an answer rather than as a hang. */
|
|
129
|
-
const PROBE_TIMEOUT_MS = 1500;
|
|
130
|
-
|
|
131
|
-
/**
|
|
132
|
-
* The one spelling of a base URL this module dials.
|
|
133
|
-
*
|
|
134
|
-
* A missing scheme is the COMMON case, not an edge one: `localhost:11434` and
|
|
135
|
-
* `127.0.0.1:11434` are what a user types into the Settings row, and both were
|
|
136
|
-
* previously read by `new URL()` as scheme `localhost:`/`127.0.0.1:` with an
|
|
137
|
-
* empty path — not http/https, so `isLocalEndpoint()` said "remote" and the
|
|
138
|
-
* fence REFUSED the user's own machine. The settings store refuses to write such
|
|
139
|
-
* a row, and a row already on disk from a hand edit gets a turn refused with
|
|
140
|
-
* "… is not on this machine, so it is not the local class" while Ollama is up
|
|
141
|
-
* and answering. That is the whole reported symptom: a healthy local daemon
|
|
142
|
-
* reported as a remote address.
|
|
143
|
-
*
|
|
144
|
-
* `http://` is prepended only when the string carries NO scheme, and the result
|
|
145
|
-
* is still parsed by `new URL()` — so this normalizes the dialing spelling
|
|
146
|
-
* without widening the fence by one host: `example.com:11434` becomes
|
|
147
|
-
* `http://example.com:11434`, still a dotted public name, still refused.
|
|
148
|
-
* `https://` is left exactly as given (a local daemon behind TLS is legitimate,
|
|
149
|
-
* and rewriting a stated scheme would send the turn to the wrong port).
|
|
150
|
-
*/
|
|
151
|
-
function normalizeBase(url) {
|
|
152
|
-
const raw = String(url == null ? '' : url).trim();
|
|
153
|
-
if (!raw) return DEFAULT_BASE;
|
|
154
|
-
// Any `scheme://` is taken as stated. A colon WITHOUT `//` is a host:port
|
|
155
|
-
// (`localhost:11434`), never a scheme — which is exactly the ambiguity
|
|
156
|
-
// WHATWG's parser resolves the other way, and the bug above.
|
|
157
|
-
return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(raw) ? raw : `http://${raw}`;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/** Trim trailing slashes so `${base}/v1/...` never doubles one. */
|
|
161
|
-
function baseOf(baseURL) {
|
|
162
|
-
return normalizeBase(baseURL).replace(/\/+$/, '');
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/**
|
|
166
|
-
* DeepSeek (also the chat template a locally-pulled `deepseek-*` Ollama model
|
|
167
|
-
* uses) emits its tool calls as DSML — an XML-ish dialect that the
|
|
168
|
-
* OpenAI-compatible wire format does NOT structure for us:
|
|
169
|
-
*
|
|
170
|
-
* <|DSML|tool_calls>
|
|
171
|
-
* <|DSML|invoke name="write">
|
|
172
|
-
* <|DSML|parameter name="path" string="true">src/app.js</|DSML|parameter>
|
|
173
|
-
* <|DSML|parameter name="content" string="true">…the file body…</|DSML|parameter>
|
|
174
|
-
* </|DSML|invoke>
|
|
175
|
-
* </|DSML|tool_calls>
|
|
176
|
-
*
|
|
177
|
-
* It arrives as plain `delta.content` text, and that produces two failures:
|
|
178
|
-
* the markup rendering verbatim (reported as the pasted "||DSML||" envelope),
|
|
179
|
-
* and the call itself never running because text is not a tool call. Ported
|
|
180
|
-
* from aegiscodex-dev's src/backend.js (also client/aegis.js, the other
|
|
181
|
-
* transport this engine shares), which carries the full history of this
|
|
182
|
-
* grammar's quirks (provider/template/model revision disagree about the exact
|
|
183
|
-
* identifier, angle brackets sometimes absent, tag names drift under
|
|
184
|
-
* sampling) — every part of the pattern below is deliberately optional and
|
|
185
|
-
* spacing-insensitive for that reason. `string="false"` marks a JSON
|
|
186
|
-
* parameter; `string="true"` (or an absent attribute) is the literal value,
|
|
187
|
-
* kept verbatim so a multi-line `content` is never mangled by a JSON
|
|
188
|
-
* round-trip.
|
|
189
|
-
*/
|
|
190
|
-
// `\b` after each tag-name alternative matters: without it, "call" matches as
|
|
191
|
-
// a bare PREFIX of a drifted wrapper tag like "calls" (observed verbatim —
|
|
192
|
-
// `<|| calls>`, no "tool_"/"function_" prefix at all), swallowing everything
|
|
193
|
-
// up to the next real `</invoke>` as that spurious match's unnamed body and
|
|
194
|
-
// silently discarding a whole tool call with it. "call" and "calls" are both
|
|
195
|
-
// word characters, so plain alternation order cannot tell them apart.
|
|
196
|
-
const DSML_RE_INVOKE = /<\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call)\b\s*([^>]*?)\s*>([\s\S]*?)<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call)\b\s*>/g;
|
|
197
|
-
const DSML_RE_PARAM = /<\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*([^>]*?)\s*>([\s\S]*?)<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*>/g;
|
|
198
|
-
const DSML_RE_LEADER = /<\s*\/?\s*|\s*(?:DSML\s*)?|\s*(?:(?:tool|function)[_▁ ]*calls?|reasoning)[_▁ ]*(?:begin|end)?\s*>/gi;
|
|
199
|
-
const DSML_RE_ANY_TAG = /<\s*\/?\s*|\s*(?:DSML\s*)?|\s*[A-Za-z_][A-Za-z0-9_]*[^>]{0,120}>/g;
|
|
200
|
-
const DSML_RE_SKELETON = /<\s*\/?\s*(?:tool_calls|function_calls|invoke|parameter|param|reasoning)\b[^>]{0,120}>/gi;
|
|
201
|
-
const DSML_RE_JSON_CALLS = /|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*begin[ \t]*|([\s\S]*?)|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*end[ \t]*|/gi;
|
|
202
|
-
// The leader/closer halves of DSML_RE_JSON_CALLS above, split out so the hold
|
|
203
|
-
// logic can detect "the leader arrived but its closer has not" — an arbitrary
|
|
204
|
-
// JSON blob sits between them with no '<' or recognizable structure of its
|
|
205
|
-
// own, so nothing else below would otherwise hold it back.
|
|
206
|
-
const DSML_RE_JSON_CALLS_OPEN = /|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*begin[ \t]*|/gi;
|
|
207
|
-
const DSML_RE_JSON_CALLS_CLOSE = /|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*end[ \t]*|/gi;
|
|
208
|
-
const DSML_RE_ENVELOPE_TOKEN = /|[ \t]?[A-Za-z0-9_▁]*[ \t]?|/g;
|
|
209
|
-
const DSML_RE_DRIFT_INVOKE = /<\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call)\b\s*([^>]*?)\s*>/gi;
|
|
210
|
-
const DSML_RE_DRIFT_PARAM = /<\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*([^>]*?)\s*>/gi;
|
|
211
|
-
const DSML_RE_PARAM_CLOSE = /<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*>/gi;
|
|
212
|
-
const DSML_RE_ATTR = /([A-Za-z_][A-Za-z0-9_.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
|
213
|
-
const DSML_RE_STRUCT_TAG = /<\s*\/?\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call|parameter|param|arg|reasoning|(?:tool|function)[_▁ ]*calls?)\b/gi;
|
|
214
|
-
const DSML_RE_STRUCT_CLOSER = /<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call|parameter|param|arg|reasoning|(?:tool|function)[_▁ ]*calls?)\b/gi;
|
|
215
|
-
/**
|
|
216
|
-
* A COMPLETE (its own '>' has arrived) wrapper/leader tag that carries no
|
|
217
|
-
* call of its own — observed verbatim: a bare `<||calls>`, with neither a
|
|
218
|
-
* "tool_"/"function_" prefix (so DSML_RE_LEADER above misses it) nor a JSON
|
|
219
|
-
* envelope pair. Safe to drop the moment it closes, UNLIKE invoke/parameter —
|
|
220
|
-
* excluded here via negative lookahead so a *partial* invoke/parameter head is
|
|
221
|
-
* never erased before its own close has a chance to pair it up with
|
|
222
|
-
* DSML_RE_INVOKE/DSML_RE_PARAM on a later call once more of it has arrived.
|
|
223
|
-
*/
|
|
224
|
-
const DSML_RE_WRAPPER_TAG = /<\s*\/?\s*|\s*(?:DSML\s*)?|\s*(?!(?:invoke|call|tool_call|parameter|param|arg)\b)[A-Za-z_][A-Za-z0-9_]*[^>]{0,120}>/g;
|
|
225
|
-
/** A '<' (or '</') at the tail: the head of a tag whose pipe is still in flight. */
|
|
226
|
-
const DSML_RE_TRAILING_LT = /<\s*\/?\s*$/;
|
|
227
|
-
/** An in-flight identifier token: a pipe whose partner has not arrived yet. */
|
|
228
|
-
const DSML_RE_OPEN_TOKEN = /^|[ \t]?[A-Za-z0-9_▁]*[ \t]?$/;
|
|
229
|
-
/** Past this much buffered markup, a held tail is released as-is. */
|
|
230
|
-
const DSML_HOLD_MAX = 4 * 1024 * 1024;
|
|
231
|
-
|
|
232
|
-
/** The attributes of one tag head, lower-cased: name="write" → { name: 'write' }. */
|
|
233
|
-
function dsmlAttrs(text) {
|
|
234
|
-
const out = {};
|
|
235
|
-
if (!text) return out;
|
|
236
|
-
DSML_RE_ATTR.lastIndex = 0;
|
|
237
|
-
let m;
|
|
238
|
-
while ((m = DSML_RE_ATTR.exec(text))) out[m[1].toLowerCase()] = m[2] != null ? m[2] : m[3];
|
|
239
|
-
return out;
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/** A JSON parameter — or, when it is not JSON after all, the text it came as. */
|
|
243
|
-
function dsmlJson(raw) {
|
|
244
|
-
const s = String(raw == null ? '' : raw).trim();
|
|
245
|
-
try {
|
|
246
|
-
const v = JSON.parse(s);
|
|
247
|
-
return v === undefined ? s : v;
|
|
248
|
-
} catch {
|
|
249
|
-
return s;
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/** The `parameter` tags of one `invoke` body, as a tool call's args object. */
|
|
254
|
-
function dsmlArgs(body) {
|
|
255
|
-
const args = {};
|
|
256
|
-
DSML_RE_PARAM.lastIndex = 0;
|
|
257
|
-
let p;
|
|
258
|
-
while ((p = DSML_RE_PARAM.exec(body))) {
|
|
259
|
-
const a = dsmlAttrs(p[1]);
|
|
260
|
-
const key = a.name || a.key || `arg${Object.keys(args).length}`;
|
|
261
|
-
const isJson = String(a.string || '').toLowerCase() === 'false' || String(a.json || '').toLowerCase() === 'true';
|
|
262
|
-
args[key] = isJson ? dsmlJson(p[2]) : p[2];
|
|
263
|
-
}
|
|
264
|
-
return args;
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
/**
|
|
268
|
-
* Decode every DSML tool call in `text` and return the two things a caller
|
|
269
|
-
* needs: what may be shown, and what must be executed — `{ text, calls }`.
|
|
270
|
-
* The generic `|…|` tokens are deliberately left in place here: an `invoke`
|
|
271
|
-
* block still arriving is identified by its delimiters on the next pass, and
|
|
272
|
-
* eating them early would lose the call's name and arguments. They are
|
|
273
|
-
* removed from the *released* text instead (see dsmlResidue).
|
|
274
|
-
*/
|
|
275
|
-
function decodeDeepSeekDSML(input) {
|
|
276
|
-
if (typeof input !== 'string' || !input || input.indexOf('|') === -1) {
|
|
277
|
-
return { text: input || '', calls: [] };
|
|
278
|
-
}
|
|
279
|
-
const calls = [];
|
|
280
|
-
let text = input;
|
|
281
|
-
|
|
282
|
-
// 1. One `invoke` block per call, the well-formed spelling.
|
|
283
|
-
DSML_RE_INVOKE.lastIndex = 0;
|
|
284
|
-
const spans = [];
|
|
285
|
-
let m;
|
|
286
|
-
while ((m = DSML_RE_INVOKE.exec(text))) {
|
|
287
|
-
const a = dsmlAttrs(m[1]);
|
|
288
|
-
const name = a.name || a.function || a.tool || '';
|
|
289
|
-
if (name) calls.push({ name, args: dsmlArgs(m[2]) });
|
|
290
|
-
spans.push([m.index, m.index + m[0].length]);
|
|
291
|
-
}
|
|
292
|
-
for (let i = spans.length - 1; i >= 0; i--) {
|
|
293
|
-
text = text.slice(0, spans[i][0]) + text.slice(spans[i][1]);
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
// 2. The older spelling: a JSON array wrapped in envelope tokens.
|
|
297
|
-
text = text.replace(DSML_RE_JSON_CALLS, (full, body) => {
|
|
298
|
-
const parsed = dsmlJson(body);
|
|
299
|
-
const list = Array.isArray(parsed) ? parsed : (parsed && Array.isArray(parsed.tool_calls) ? parsed.tool_calls : []);
|
|
300
|
-
for (const c of list) {
|
|
301
|
-
const fn = (c && c.function) || c || {};
|
|
302
|
-
const name = fn.name || (c && c.name) || '';
|
|
303
|
-
if (!name) continue;
|
|
304
|
-
let args = fn.arguments != null ? fn.arguments : (c && c.arguments);
|
|
305
|
-
if (typeof args === 'string') args = dsmlJson(args);
|
|
306
|
-
calls.push({ name, args: args && typeof args === 'object' ? args : {} });
|
|
307
|
-
}
|
|
308
|
-
return '';
|
|
309
|
-
});
|
|
310
|
-
|
|
311
|
-
// 3. The wrapper tags carry no call of their own.
|
|
312
|
-
text = text.replace(DSML_RE_LEADER, '');
|
|
313
|
-
return { text, calls };
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
/**
|
|
317
|
-
* Last-resort decode for markup that drifted out of well-formedness: a missing
|
|
318
|
-
* closing tag, or a stream cut mid-call. The call still names a tool and its
|
|
319
|
-
* parameters are still readable, so it is decoded rather than deleted — a call
|
|
320
|
-
* that reached the wire must not vanish because its closing tag did.
|
|
321
|
-
*/
|
|
322
|
-
function decodeDeepSeekDSMLDrift(input) {
|
|
323
|
-
if (typeof input !== 'string' || !input || input.indexOf('|') === -1) return { text: input || '', calls: [] };
|
|
324
|
-
const opens = [];
|
|
325
|
-
DSML_RE_DRIFT_INVOKE.lastIndex = 0;
|
|
326
|
-
let m;
|
|
327
|
-
while ((m = DSML_RE_DRIFT_INVOKE.exec(input))) {
|
|
328
|
-
opens.push({ at: m.index, attrs: m[1], body: DSML_RE_DRIFT_INVOKE.lastIndex });
|
|
329
|
-
}
|
|
330
|
-
if (!opens.length) return { text: input, calls: [] };
|
|
331
|
-
const calls = [];
|
|
332
|
-
let text = '';
|
|
333
|
-
let cursor = 0;
|
|
334
|
-
for (let i = 0; i < opens.length; i++) {
|
|
335
|
-
// One call runs to the next `invoke` head, or to the end of the text.
|
|
336
|
-
const end = i + 1 < opens.length ? opens[i + 1].at : input.length;
|
|
337
|
-
const body = input.slice(opens[i].body, end);
|
|
338
|
-
const a = dsmlAttrs(opens[i].attrs);
|
|
339
|
-
const name = a.name || a.function || a.tool || '';
|
|
340
|
-
const args = {};
|
|
341
|
-
const params = [];
|
|
342
|
-
DSML_RE_DRIFT_PARAM.lastIndex = 0;
|
|
343
|
-
let pm;
|
|
344
|
-
while ((pm = DSML_RE_DRIFT_PARAM.exec(body))) {
|
|
345
|
-
params.push({ at: pm.index, attrs: pm[1], from: DSML_RE_DRIFT_PARAM.lastIndex });
|
|
346
|
-
}
|
|
347
|
-
for (let k = 0; k < params.length; k++) {
|
|
348
|
-
// A value runs to the next parameter tag — its closing tag may be the
|
|
349
|
-
// thing that never arrived.
|
|
350
|
-
const stop = k + 1 < params.length ? params[k + 1].at : body.length;
|
|
351
|
-
const raw = body.slice(params[k].from, stop).replace(DSML_RE_PARAM_CLOSE, '');
|
|
352
|
-
const pa = dsmlAttrs(params[k].attrs);
|
|
353
|
-
const key = pa.name || pa.key || `arg${Object.keys(args).length}`;
|
|
354
|
-
const isJson = String(pa.string || '').toLowerCase() === 'false';
|
|
355
|
-
args[key] = isJson ? dsmlJson(raw) : raw;
|
|
356
|
-
}
|
|
357
|
-
text += input.slice(cursor, opens[i].at);
|
|
358
|
-
cursor = end;
|
|
359
|
-
if (name) calls.push({ name, args });
|
|
360
|
-
}
|
|
361
|
-
text += input.slice(cursor);
|
|
362
|
-
return { text, calls };
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
/**
|
|
366
|
-
* Residue removal for text that is about to be shown. Two spellings have to go:
|
|
367
|
-
* a leftover envelope token (`|DSML|`, `|tool▁calls▁begin|`, `|Assistant|`),
|
|
368
|
-
* and the skeleton an earlier release left behind by stripping the delimiters
|
|
369
|
-
* out of a DSML block — `<tool_calls> <invoke name="write"> <parameter …>`,
|
|
370
|
-
* which is exactly what a user pasted back at us.
|
|
371
|
-
*/
|
|
372
|
-
function dsmlResidue(text) {
|
|
373
|
-
if (!text) return text;
|
|
374
|
-
if (text.indexOf('|') === -1) return text.replace(DSML_RE_SKELETON, '');
|
|
375
|
-
return text.replace(DSML_RE_ANY_TAG, '').replace(DSML_RE_SKELETON, '').replace(DSML_RE_ENVELOPE_TOKEN, '');
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
/**
|
|
379
|
-
* Is the buffer, read from the first `|` in it, a construct that is still
|
|
380
|
-
* arriving? Only two shapes may be held back: a pipe whose partner has not
|
|
381
|
-
* arrived, and a DSML opener whose matching close has not landed. Everything
|
|
382
|
-
* else is prose — a `|` in an answer is a character, not a promise of markup,
|
|
383
|
-
* and holding the rest of the answer behind it would swallow the reply.
|
|
384
|
-
*/
|
|
385
|
-
function dsmlUnterminated(s) {
|
|
386
|
-
if (DSML_RE_OPEN_TOKEN.test(s)) return true;
|
|
387
|
-
const openers = (s.match(DSML_RE_STRUCT_TAG) || []).length;
|
|
388
|
-
const closers = (s.match(DSML_RE_STRUCT_CLOSER) || []).length;
|
|
389
|
-
return openers > closers;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
/**
|
|
393
|
-
* Where, if anywhere, an in-flight DSML construct still needs the rest of the
|
|
394
|
-
* buffer held back from view. By the time this runs, decodeDeepSeekDSML() has
|
|
395
|
-
* already removed every COMPLETE invoke/parameter span and the caller has
|
|
396
|
-
* already removed every complete non-invoke wrapper tag (DSML_RE_WRAPPER_TAG),
|
|
397
|
-
* so whatever is left can only be: clean prose, or the start of exactly ONE
|
|
398
|
-
* incomplete construct running to the end of the buffer — an opener whose own
|
|
399
|
-
* '|' has arrived but not its '>' (`<\s*\/?\s*|`, found wherever it starts,
|
|
400
|
-
* not just at the tail: a Write's `content` can be arbitrarily long, so the
|
|
401
|
-
* unresolved opener is usually far behind the cursor, not at it), a '<' that
|
|
402
|
-
* arrived with no '|' yet (the chunk boundary landed first), or a bare pipe
|
|
403
|
-
* token still accumulating its identifier (the plain envelope-leak case, e.g.
|
|
404
|
-
* '|tool▁cal'). Returns -1 when none apply — safe to release in full.
|
|
405
|
-
*/
|
|
406
|
-
function dsmlHoldFrom(buf) {
|
|
407
|
-
const tagStart = buf.search(/<\s*\/?\s*|/);
|
|
408
|
-
if (tagStart !== -1) return tagStart;
|
|
409
|
-
// The older JSON-array envelope (DSML_RE_JSON_CALLS): once its leader token
|
|
410
|
-
// has arrived, hold everything until the matching closer token lands. An
|
|
411
|
-
// arbitrary JSON blob sits between them with no '<' and no recognizable
|
|
412
|
-
// per-character structure, so without this check it streams straight
|
|
413
|
-
// through as visible prose the instant the leader's own closing '|'
|
|
414
|
-
// arrives — the hold rule below only looks at ONE pipe token at a time and
|
|
415
|
-
// has no notion of "waiting for a second, different token later on".
|
|
416
|
-
const jsonOpenIdx = buf.search(DSML_RE_JSON_CALLS_OPEN);
|
|
417
|
-
if (jsonOpenIdx !== -1 && buf.slice(jsonOpenIdx).search(DSML_RE_JSON_CALLS_CLOSE) === -1) {
|
|
418
|
-
return jsonOpenIdx;
|
|
419
|
-
}
|
|
420
|
-
const lt = DSML_RE_TRAILING_LT.exec(buf);
|
|
421
|
-
if (lt) return lt.index;
|
|
422
|
-
const openIdx = buf.indexOf('|');
|
|
423
|
-
if (openIdx !== -1 && dsmlUnterminated(buf.slice(openIdx))) return openIdx;
|
|
424
|
-
return -1;
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
/**
|
|
428
|
-
* Pull COMPLETE plain `<tool_call>` blocks out of `text` (mutating `calls`).
|
|
429
|
-
*
|
|
430
|
-
* This runs inside the DSML stripper, ahead of its skeleton cleanup, for two
|
|
431
|
-
* reasons: that cleanup strips bare `<tool_calls>` tags and would otherwise
|
|
432
|
-
* leave the JSON body behind as prose, and running here means a DSML parameter
|
|
433
|
-
* body — a Write's `content`, a whole file — is never scanned, because those
|
|
434
|
-
* bodies are consumed by the DSML decoder and never reach this text path.
|
|
435
|
-
*/
|
|
436
|
-
function decodePlainToolCallTags(text, calls) {
|
|
437
|
-
if (!text || text.indexOf('<') === -1) return text;
|
|
438
|
-
let out = '';
|
|
439
|
-
let cursor = 0;
|
|
440
|
-
for (;;) {
|
|
441
|
-
const head = TEXT_TOOL_TAG_OPEN_RE.exec(text.slice(cursor));
|
|
442
|
-
if (!head) break;
|
|
443
|
-
const at = cursor + head.index;
|
|
444
|
-
const headText = head[0];
|
|
445
|
-
if (headText.includes(DSML_TAG_MARK)) { // belongs to the DSML dialect
|
|
446
|
-
out += text.slice(cursor, at + headText.length);
|
|
447
|
-
cursor = at + headText.length;
|
|
448
|
-
continue;
|
|
449
|
-
}
|
|
450
|
-
const rest = text.slice(at + headText.length);
|
|
451
|
-
const close = TEXT_TOOL_TAG_CLOSE_RE.exec(rest);
|
|
452
|
-
if (!close) break; // never closed: leave for the hold logic
|
|
453
|
-
const decoded = textCallsIn(rest.slice(0, close.index));
|
|
454
|
-
out += text.slice(cursor, at);
|
|
455
|
-
cursor = at + headText.length + close.index + close[0].length;
|
|
456
|
-
if (decoded.length) {
|
|
457
|
-
for (const c of decoded) calls.push(c);
|
|
458
|
-
} else {
|
|
459
|
-
out += text.slice(at, cursor); // a tag that is not a call: unchanged
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
return out + text.slice(cursor);
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
/** The position of a plain `<tool_call>` head whose closer has not arrived, or
|
|
466
|
-
* -1 — the second hold rule, alongside dsmlHoldFrom(). */
|
|
467
|
-
function plainToolTagHoldAt(text) {
|
|
468
|
-
if (!text || text.indexOf('<') === -1) return -1;
|
|
469
|
-
let cursor = 0;
|
|
470
|
-
for (;;) {
|
|
471
|
-
const head = TEXT_TOOL_TAG_OPEN_RE.exec(text.slice(cursor));
|
|
472
|
-
if (!head) return -1;
|
|
473
|
-
const at = cursor + head.index;
|
|
474
|
-
if (head[0].includes(DSML_TAG_MARK)) { cursor = at + head[0].length; continue; }
|
|
475
|
-
const rest = text.slice(at + head[0].length);
|
|
476
|
-
return TEXT_TOOL_TAG_CLOSE_RE.test(rest) ? -1 : at;
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
/**
|
|
481
|
-
* The per-turn stripper this transport uses. Feed it each `delta.content`
|
|
482
|
-
* chunk: it returns the text that is safe to show, holds back whatever is
|
|
483
|
-
* still arriving (a Write's `content` parameter is a whole file — it belongs
|
|
484
|
-
* to the call, not to the screen), and collects the tool calls the model made
|
|
485
|
-
* in text form. `flush()` releases the tail once the stream is drained, and
|
|
486
|
-
* `calls()` returns everything decoded so far, which the caller merges into
|
|
487
|
-
* the turn's tool calls so a Write that arrived as DSML is executed like any
|
|
488
|
-
* other.
|
|
489
|
-
*/
|
|
490
|
-
function makeDeepSeekEnvelopeStripper() {
|
|
491
|
-
let buf = '';
|
|
492
|
-
const calls = [];
|
|
493
|
-
const release = (text) => dsmlResidue(decodePlainToolCallTags(text, calls));
|
|
494
|
-
const strip = (chunk) => {
|
|
495
|
-
buf += typeof chunk === 'string' ? chunk : '';
|
|
496
|
-
const dec = decodeDeepSeekDSML(buf);
|
|
497
|
-
buf = dec.text;
|
|
498
|
-
for (const c of dec.calls) calls.push(c);
|
|
499
|
-
buf = buf.replace(DSML_RE_WRAPPER_TAG, '');
|
|
500
|
-
const hold = dsmlHoldFrom(buf);
|
|
501
|
-
let out;
|
|
502
|
-
if (hold !== -1 && buf.length - hold <= DSML_HOLD_MAX) {
|
|
503
|
-
out = buf.slice(0, hold);
|
|
504
|
-
buf = buf.slice(hold);
|
|
505
|
-
} else {
|
|
506
|
-
out = buf;
|
|
507
|
-
buf = '';
|
|
508
|
-
}
|
|
509
|
-
// A plain `<tool_call>` head whose closer has not landed yet is held the
|
|
510
|
-
// same way the DSML envelope is: the call is one whole thing, and showing
|
|
511
|
-
// half of it would mean showing a tag the decoder is about to consume.
|
|
512
|
-
const plainAt = plainToolTagHoldAt(out);
|
|
513
|
-
if (plainAt !== -1) {
|
|
514
|
-
buf = out.slice(plainAt) + buf;
|
|
515
|
-
out = out.slice(0, plainAt);
|
|
516
|
-
}
|
|
517
|
-
return release(out);
|
|
518
|
-
};
|
|
519
|
-
strip.flush = () => {
|
|
520
|
-
const dec = decodeDeepSeekDSML(buf);
|
|
521
|
-
buf = '';
|
|
522
|
-
for (const c of dec.calls) calls.push(c);
|
|
523
|
-
const drift = decodeDeepSeekDSMLDrift(dec.text);
|
|
524
|
-
for (const c of drift.calls) calls.push(c);
|
|
525
|
-
return release(drift.text);
|
|
526
|
-
};
|
|
527
|
-
/** Every tool call decoded from text so far, as `{ name, args }`. */
|
|
528
|
-
strip.calls = () => calls.slice();
|
|
529
|
-
return strip;
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
/**
|
|
533
|
-
* Tool calls a local model wrote as TEXT, rather than in the structured field.
|
|
534
|
-
*
|
|
535
|
-
* Measured against a live daemon (ollama 0.32.1, 2026-10-05), on the same
|
|
536
|
-
* prompt this file's DSML decoder exists for:
|
|
537
|
-
*
|
|
538
|
-
* qwen2.5-coder:1.5b-instruct-q4_K_M, `tools: [read]`, "read /etc/hostname
|
|
539
|
-
* with the read tool" →
|
|
540
|
-
* `content: "```json\n{\"name\": \"read\", \"arguments\": {\"path\": …}```"`,
|
|
541
|
-
* `finish_reason: "stop"`, and NO `delta.tool_calls` at all.
|
|
542
|
-
*
|
|
543
|
-
* That is not a quirk of one tag: Ollama's OpenAI shim only fills
|
|
544
|
-
* `delta.tool_calls` for templates it knows carry tool tokens (`tools` in the
|
|
545
|
-
* model's own `capabilities`, listed by `/api/tags`). Every other local model
|
|
546
|
-
* — an older qwen, a Llama, a gguf picked up from the Hub — is handed the
|
|
547
|
-
* schemas, answers in the dialect its chat template taught it, and emits it as
|
|
548
|
-
* plain content. The engine's loop then sees a model that "said" a JSON blob
|
|
549
|
-
* and ran nothing, which is what the user sees as the local class being unable
|
|
550
|
-
* to use tools at all.
|
|
551
|
-
*
|
|
552
|
-
* So the same treatment the DSML dialect gets: decode the envelope out of the
|
|
553
|
-
* text, keep the prose, and hand the calls to the tool loop. Two shapes are
|
|
554
|
-
* recognised, because they are the two the standard templates emit:
|
|
555
|
-
*
|
|
556
|
-
* <tool_call>{"name": "read", "arguments": {"path": "/etc/hostname"}}</tool_call>
|
|
557
|
-
* ```json
|
|
558
|
-
* {"name": "read", "arguments": {"path": "/etc/hostname"}}
|
|
559
|
-
* ```
|
|
560
|
-
*
|
|
561
|
-
* Deliberately NOT recognised: a bare `{…}` with no fence and no tag. Prose
|
|
562
|
-
* questions about JSON ("what does {"name": "read"} mean?") are common and a
|
|
563
|
-
* decoder that guessed there would turn a user's own example into a real file
|
|
564
|
-
* read — the conservative half of the trade this file already makes for the
|
|
565
|
-
* endpoint gate. A fenced block whose JSON does not carry a `name` + arguments
|
|
566
|
-
* pair is left exactly as it arrived: it is a code block, not a call.
|
|
567
|
-
*/
|
|
568
|
-
const TEXT_TOOL_TAG_OPEN_RE = /<\s*tool_calls?\b[^>]{0,80}>/i;
|
|
569
|
-
const TEXT_TOOL_TAG_CLOSE_RE = /<\s*\/\s*tool_calls?\s*>/i;
|
|
570
|
-
const TEXT_FENCE_RE = /```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?([\s\S]*?)```/;
|
|
571
|
-
// The OPENing half alone, for the case where the closer has not arrived yet:
|
|
572
|
-
// TEXT_FENCE_RE cannot match there (its closer is part of the match, NOT part
|
|
573
|
-
// of the captured body — the first version of this code sliced with
|
|
574
|
-
// `fence[0].length` and so held every fence forever), which is exactly why the
|
|
575
|
-
// hold test needs a separate pattern.
|
|
576
|
-
const TEXT_FENCE_OPEN_RE = /```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?/;
|
|
577
|
-
// A DSML tag always carries the fullwidth pipe; a PLAIN `<tool_call>` never
|
|
578
|
-
// does. That single character is what tells the two dialects apart, and it is
|
|
579
|
-
// why this pass can run ahead of the DSML machinery without ever claiming a
|
|
580
|
-
// tag that belongs to it.
|
|
581
|
-
const DSML_TAG_MARK = '|';
|
|
582
|
-
|
|
583
|
-
/** Coerce one decoded envelope body into `{ name, args }`, or null when it is
|
|
584
|
-
* not a call. Accepts the argument spellings the templates actually produce:
|
|
585
|
-
* `arguments` (openai-ish, object or JSON string), `parameters`, `args`, and
|
|
586
|
-
* the `{function:{name,arguments}}` wrapper a model sometimes copies from the
|
|
587
|
-
* schema it was handed. */
|
|
588
|
-
function asTextToolCall(value) {
|
|
589
|
-
if (!value || typeof value !== 'object') return null;
|
|
590
|
-
let node = value;
|
|
591
|
-
if (node.function && typeof node.function === 'object') {
|
|
592
|
-
node = { ...node.function, name: node.function.name || node.name };
|
|
593
|
-
}
|
|
594
|
-
const rawName = node.name || node.tool || node.tool_name || node.function_name;
|
|
595
|
-
const name = typeof rawName === 'string' ? rawName.trim() : '';
|
|
596
|
-
if (!name) return null;
|
|
597
|
-
let args = node.arguments != null ? node.arguments
|
|
598
|
-
: node.parameters != null ? node.parameters
|
|
599
|
-
: node.args != null ? node.args
|
|
600
|
-
: node.tool_input != null ? node.tool_input
|
|
601
|
-
: {};
|
|
602
|
-
if (typeof args === 'string') {
|
|
603
|
-
const parsed = args.trim() ? safeJson(args) : {};
|
|
604
|
-
args = parsed && typeof parsed === 'object' ? parsed : {};
|
|
605
|
-
}
|
|
606
|
-
if (args == null || typeof args !== 'object') args = {};
|
|
607
|
-
return { name, args };
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
/** JSON.parse that returns null instead of throwing. */
|
|
611
|
-
function safeJson(text) {
|
|
612
|
-
try {
|
|
613
|
-
return JSON.parse(text);
|
|
614
|
-
} catch {
|
|
615
|
-
return null;
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
|
|
619
|
-
/** Decode one envelope BODY (tag inner text or fence inner text). Returns []
|
|
620
|
-
* when it is not a call, so the caller can hand the text back unchanged. */
|
|
621
|
-
function textCallsIn(body) {
|
|
622
|
-
const text = String(body || '').trim().replace(/^json\s*/i, '').trim();
|
|
623
|
-
if (!text) return [];
|
|
624
|
-
const parsed = safeJson(text);
|
|
625
|
-
if (!parsed) return [];
|
|
626
|
-
const list = Array.isArray(parsed) ? parsed : [parsed];
|
|
627
|
-
const calls = [];
|
|
628
|
-
for (const item of list) {
|
|
629
|
-
const call = asTextToolCall(item);
|
|
630
|
-
if (call) calls.push(call);
|
|
631
|
-
}
|
|
632
|
-
return calls;
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
/** The next envelope in `buf`: a ``` fence carrying a call, or the position of
|
|
636
|
-
* a plain `<tool_call>` head the caller must hold. `end` is where it finishes
|
|
637
|
-
* in `buf`; `complete` is false while its closer is still on the wire, in
|
|
638
|
-
* which case the caller must HOLD the tail, not show it. */
|
|
639
|
-
function nextTextToolEnvelope(buf) {
|
|
640
|
-
const fence = TEXT_FENCE_RE.exec(buf);
|
|
641
|
-
if (fence) {
|
|
642
|
-
// `fence[1]` is the body; the closing fence is part of the MATCH, so the
|
|
643
|
-
// body must come from the capture group and the end from `fence[0]`.
|
|
644
|
-
return {
|
|
645
|
-
start: fence.index,
|
|
646
|
-
end: fence.index + fence[0].length,
|
|
647
|
-
complete: true,
|
|
648
|
-
raw: fence[0],
|
|
649
|
-
body: fence[1],
|
|
650
|
-
};
|
|
651
|
-
}
|
|
652
|
-
const open = TEXT_FENCE_OPEN_RE.exec(buf);
|
|
653
|
-
if (!open) return null;
|
|
654
|
-
return {
|
|
655
|
-
start: open.index,
|
|
656
|
-
end: buf.length,
|
|
657
|
-
complete: false,
|
|
658
|
-
body: buf.slice(open.index + open[0].length),
|
|
659
|
-
};
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
/** Where the hold must start in `buf`, or -1 when nothing is pending: the
|
|
663
|
-
* index of a recognized-but-unclosed head, or of the first characters of one
|
|
664
|
-
* that is still arriving (a '`' that may become a fence, a '<' that may become
|
|
665
|
-
* a tag). Everything before that index is released as prose. */
|
|
666
|
-
function trailingHeadFrom(buf, allowBare = false) {
|
|
667
|
-
if (!buf) return -1;
|
|
668
|
-
const marker = buf.search(TEXT_TOOL_HEAD_RE);
|
|
669
|
-
if (marker !== -1) return marker; // head started, closer has not landed
|
|
670
|
-
// A fence opener still arriving: one or two backticks at the very end. It
|
|
671
|
-
// cannot be anything else — a closed inline code span would carry its closer.
|
|
672
|
-
const ticks = /`{1,2}$/.exec(buf);
|
|
673
|
-
if (ticks) return ticks.index;
|
|
674
|
-
// A tag still arriving: the last '<' has only a prefix of "tool_call" after
|
|
675
|
-
// it. Prose that merely contains '<' resolves on the very next chunk.
|
|
676
|
-
const lt = buf.lastIndexOf('<');
|
|
677
|
-
if (lt !== -1 && couldBeTextToolHead(buf.slice(lt))) return lt;
|
|
678
|
-
// A bare JSON object still arriving — the same dialect rule as above, and it
|
|
679
|
-
// has to be held for the same reason: a chunk landing on the first `{` would
|
|
680
|
-
// otherwise show that brace as prose and the object could never be recognised
|
|
681
|
-
// afterwards. Only JSON-SHAPED tails are held; an ordinary brace in prose
|
|
682
|
-
// ('use a { in your code') is released and streams normally.
|
|
683
|
-
if (allowBare) {
|
|
684
|
-
const open = buf.lastIndexOf('{');
|
|
685
|
-
if (open !== -1 && /^\{\s*(?:"[^"]*"\s*:[\s\S]*|"[^"]*"?\s*|)$/.test(buf.slice(open))) return open;
|
|
686
|
-
}
|
|
687
|
-
return -1;
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
/** Still a POSSIBLE head — a fence just starting, or a `<tool_c…` prefix that
|
|
691
|
-
* has not finished arriving. A recognized (already-matching) head is held by
|
|
692
|
-
* TEXT_TOOL_HEAD_RE, not here; this is only for the partial case, so that a
|
|
693
|
-
* chunk boundary landing mid-`<tool_call` cannot show two characters of a tag
|
|
694
|
-
* and then retract them. */
|
|
695
|
-
function couldBeTextToolHead(buf) {
|
|
696
|
-
const lead = buf.replace(/^[\s\uFEFF]+/, '');
|
|
697
|
-
if (!lead) return true; // nothing buffered yet
|
|
698
|
-
if (lead.length <= 3 && '```'.startsWith(lead)) return true;
|
|
699
|
-
if (lead.startsWith('<') && !TEXT_TOOL_HEAD_RE.test(lead)) {
|
|
700
|
-
const name = lead.slice(1).replace(/^\/?\s*/, '').toLowerCase();
|
|
701
|
-
if (name.length <= 10 && /^[a-z_]*$/.test(name)) {
|
|
702
|
-
return 'tool_call'.startsWith(name) || 'tool_calls'.startsWith(name);
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
return false;
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
/**
|
|
709
|
-
* Where a JSON object starting at `start` ends, or -1 while it is still
|
|
710
|
-
* arriving. String- and escape-aware, because the arguments of a real call
|
|
711
|
-
* routinely carry quoted braces (`{"content": "function f() { … }"}`) and a
|
|
712
|
-
* naive depth count ends the object inside one of them.
|
|
713
|
-
*/
|
|
714
|
-
function balancedJsonEnd(text, start) {
|
|
715
|
-
let depth = 0;
|
|
716
|
-
let inStr = false;
|
|
717
|
-
let esc = false;
|
|
718
|
-
for (let i = start; i < text.length; i++) {
|
|
719
|
-
const ch = text[i];
|
|
720
|
-
if (inStr) {
|
|
721
|
-
if (esc) esc = false;
|
|
722
|
-
else if (ch === '\\') esc = true;
|
|
723
|
-
else if (ch === '"') inStr = false;
|
|
724
|
-
continue;
|
|
725
|
-
}
|
|
726
|
-
if (ch === '"') inStr = true;
|
|
727
|
-
else if (ch === '{' || ch === '[') depth++;
|
|
728
|
-
else if (ch === '}' || ch === ']') {
|
|
729
|
-
depth--;
|
|
730
|
-
if (depth <= 0) return i + 1;
|
|
731
|
-
}
|
|
732
|
-
}
|
|
733
|
-
return -1;
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
/**
|
|
737
|
-
* Decode tool calls out of a local model's TEXT — the fenced and `<tool_call>`
|
|
738
|
-
* dialects, and the bare JSON object a small model emits when it ignores both.
|
|
739
|
-
*
|
|
740
|
-
* `toolNames` is the set of tools THIS request offered. It is the guard that
|
|
741
|
-
* makes the bare-JSON rule honest rather than a guess: a JSON object is only
|
|
742
|
-
* ever read as a call when the name it carries is one of the tools the model
|
|
743
|
-
* was actually given (and, for an array, every one of them). A user asking
|
|
744
|
-
* "what does {"name": "read"} mean?" in a turn with no tools, or naming a tool
|
|
745
|
-
* that was never offered, gets prose back untouched. That is measured against
|
|
746
|
-
* the real failure mode: with the fence rule alone, qwen2.5-coder answered the
|
|
747
|
-
* tool prompt with a bare object about as often as with a fenced one, and the
|
|
748
|
-
* turn ran nothing either time.
|
|
749
|
-
*
|
|
750
|
-
* Incomplete envelopes are held (never shown and retracted): the caller feeds
|
|
751
|
-
* chunks, and a fence/tag/object whose closer has not arrived stays in the
|
|
752
|
-
* buffer until it has, or until flush() decides.
|
|
753
|
-
*/
|
|
754
|
-
function makeTextToolCallStripper({ toolNames = null } = {}) {
|
|
755
|
-
const names = toolNames ? new Set([...toolNames].map((n) => String(n))) : null;
|
|
756
|
-
let buf = '';
|
|
757
|
-
const calls = [];
|
|
758
|
-
/** A decoded batch counts only if every call names an offered tool. */
|
|
759
|
-
const accept = (decoded) =>
|
|
760
|
-
Boolean(decoded.length) && (!names || decoded.every((c) => names.has(c.name)));
|
|
761
|
-
/** Show a complete envelope's TEXT when it is not a call, or swallow it and
|
|
762
|
-
* record the call when it is. */
|
|
763
|
-
const digest = (raw, body) => {
|
|
764
|
-
const decoded = textCallsIn(body);
|
|
765
|
-
if (!accept(decoded)) return raw; // prose fence / tag: shown exactly as it came
|
|
766
|
-
for (const c of decoded) calls.push(c);
|
|
767
|
-
return '';
|
|
768
|
-
};
|
|
769
|
-
const strip = (chunk) => {
|
|
770
|
-
buf += typeof chunk === 'string' ? chunk : '';
|
|
771
|
-
let out = '';
|
|
772
|
-
// Set whenever the loop stops on an envelope that is still arriving: the
|
|
773
|
-
// buffer then holds a decided-pending head and the tail release below must
|
|
774
|
-
// not undo the hold it just took.
|
|
775
|
-
let held = false;
|
|
776
|
-
for (;;) {
|
|
777
|
-
const fence = nextTextToolEnvelope(buf);
|
|
778
|
-
if (fence) {
|
|
779
|
-
out += buf.slice(0, fence.start); // prose before the envelope is never held
|
|
780
|
-
buf = buf.slice(fence.start);
|
|
781
|
-
if (!fence.complete) { held = true; break; }
|
|
782
|
-
out += digest(fence.raw, fence.body);
|
|
783
|
-
buf = buf.slice(fence.raw.length);
|
|
784
|
-
continue;
|
|
785
|
-
}
|
|
786
|
-
const tag = TEXT_TOOL_TAG_OPEN_RE.exec(buf);
|
|
787
|
-
if (tag) {
|
|
788
|
-
out += buf.slice(0, tag.index);
|
|
789
|
-
buf = buf.slice(tag.index);
|
|
790
|
-
const rest = buf.slice(tag[0].length);
|
|
791
|
-
const close = TEXT_TOOL_TAG_CLOSE_RE.exec(rest);
|
|
792
|
-
if (!close) { held = true; break; } // closer still on the wire: hold it
|
|
793
|
-
const raw = buf.slice(0, tag[0].length + close.index + close[0].length);
|
|
794
|
-
out += digest(raw, rest.slice(0, close.index));
|
|
795
|
-
buf = buf.slice(raw.length);
|
|
796
|
-
continue;
|
|
797
|
-
}
|
|
798
|
-
// Bare JSON — admitted only against the tools this turn offered, so a
|
|
799
|
-
// JSON-shaped QUESTION can never become a file read. `{"` rather than a
|
|
800
|
-
// lone `{`: the templates that emit this write an object with quoted
|
|
801
|
-
// keys, and it keeps prose braces out of the scan entirely.
|
|
802
|
-
const bare = names ? /\{\s*"/.exec(buf) : null;
|
|
803
|
-
if (!bare) break;
|
|
804
|
-
const end = balancedJsonEnd(buf, bare.index);
|
|
805
|
-
const before = buf.slice(0, bare.index);
|
|
806
|
-
if (end === -1) { out += before; buf = buf.slice(bare.index); held = true; break; } // hold
|
|
807
|
-
const decoded = textCallsIn(buf.slice(bare.index, end));
|
|
808
|
-
out += before;
|
|
809
|
-
if (accept(decoded)) {
|
|
810
|
-
for (const c of decoded) calls.push(c);
|
|
811
|
-
} else {
|
|
812
|
-
out += buf.slice(bare.index, end); // prose: shown, and not rescanned
|
|
813
|
-
}
|
|
814
|
-
buf = buf.slice(end);
|
|
815
|
-
}
|
|
816
|
-
// Nothing further can turn into an envelope: release the tail rather than
|
|
817
|
-
// delay prose behind a guard that will never fire. A recognized-but-unclosed
|
|
818
|
-
// head (or the first characters of one, e.g. the '`' a fence opener starts
|
|
819
|
-
// with) stays held — its closer is still coming, and showing it now would
|
|
820
|
-
// mean retracting it a moment later. The head can be anywhere in the
|
|
821
|
-
// buffer, not only at its start: prose streams ahead of the envelope it
|
|
822
|
-
// introduces, and the first version of this code released the whole buffer
|
|
823
|
-
// whenever the buffer did not BEGIN with a head, which let every
|
|
824
|
-
// prose-then-call turn through as text.
|
|
825
|
-
const holdAt = held ? 0 : trailingHeadFrom(buf, Boolean(names));
|
|
826
|
-
if (holdAt === -1) {
|
|
827
|
-
out += buf;
|
|
828
|
-
buf = '';
|
|
829
|
-
} else if (holdAt > 0) {
|
|
830
|
-
out += buf.slice(0, holdAt);
|
|
831
|
-
buf = buf.slice(holdAt);
|
|
832
|
-
}
|
|
833
|
-
if (buf.length > TEXT_TOOL_HOLD_MAX) {
|
|
834
|
-
// A fence or tag left open forever must not swallow the rest of the
|
|
835
|
-
// answer; past this much held text it is prose, and it is shown as such.
|
|
836
|
-
out += buf;
|
|
837
|
-
buf = '';
|
|
838
|
-
}
|
|
839
|
-
return out;
|
|
840
|
-
};
|
|
841
|
-
strip.flush = () => {
|
|
842
|
-
const rest = buf;
|
|
843
|
-
buf = '';
|
|
844
|
-
if (!rest) return '';
|
|
845
|
-
const env = nextTextToolEnvelope(rest);
|
|
846
|
-
// A fence or tag whose closer never arrived: decide NOW rather than hold
|
|
847
|
-
// forever. If its body decodes it was a call cut off mid-stream (still
|
|
848
|
-
// worth running — the arguments it did send are the caller's to validate);
|
|
849
|
-
// if it does not decode it was prose all along and is released verbatim.
|
|
850
|
-
if (env) return rest.slice(0, env.start) + digest(rest.slice(env.start), env.body);
|
|
851
|
-
const tag = TEXT_TOOL_TAG_OPEN_RE.exec(rest);
|
|
852
|
-
if (!tag) {
|
|
853
|
-
// A bare object the stream stopped inside: decide it here, so a call cut
|
|
854
|
-
// off mid-argument still runs instead of being dropped as text.
|
|
855
|
-
const bare = names ? /\{\s*"/.exec(rest) : null;
|
|
856
|
-
if (!bare) return rest;
|
|
857
|
-
const end = balancedJsonEnd(rest, bare.index);
|
|
858
|
-
return end === -1
|
|
859
|
-
? rest
|
|
860
|
-
: rest.slice(0, bare.index) + digest(rest.slice(bare.index, end), rest.slice(bare.index, end)) + rest.slice(end);
|
|
861
|
-
}
|
|
862
|
-
const body = rest.slice(tag.index + tag[0].length).replace(TEXT_TOOL_TAG_CLOSE_RE, '');
|
|
863
|
-
return rest.slice(0, tag.index) + digest(rest.slice(tag.index), body);
|
|
864
|
-
};
|
|
865
|
-
strip.calls = () => calls.slice();
|
|
866
|
-
return strip;
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
/** Where a text-form envelope could still begin — the guard that keeps a
|
|
870
|
-
* partial ` ` ` or `<tool_c` from being shown and then retracted. */
|
|
871
|
-
const TEXT_TOOL_HEAD_RE = /```|<\s*tool_calls?\b/i;
|
|
872
|
-
|
|
873
|
-
/** A fence left open forever (a model that never closed it) must not hide the
|
|
874
|
-
* rest of the answer — past this much held text, release it as prose. */
|
|
875
|
-
const TEXT_TOOL_HOLD_MAX = 64 * 1024;
|
|
876
|
-
|
|
877
|
-
/**
|
|
878
|
-
* Whether a base URL addresses hardware on this machine (or on this private
|
|
879
|
-
* network) — the gate that decides whether the free lane exists at all.
|
|
880
|
-
*
|
|
881
|
-
* True for: loopback (localhost, `*.localhost`, 127.0.0.0/8, `::1`), RFC1918
|
|
882
|
-
* private ranges (10/8, 172.16/12, 192.168/16), IPv6 unique-local fc00::/7,
|
|
883
|
-
* link-local (169.254/16, fe80::/10), the reserved local suffixes `.local`,
|
|
884
|
-
* `.internal`, `.lan`, and a bare dotless hostname (`ollama`, `gpu-box` — it
|
|
885
|
-
* can only resolve through this machine's own resolver).
|
|
886
|
-
*
|
|
887
|
-
* Private ranges are included on purpose: a box on the user's own LAN is still
|
|
888
|
-
* hardware they already own, so the "no vendor to pay" argument holds there
|
|
889
|
-
* too. What is excluded is anything routable to a third party.
|
|
890
|
-
*
|
|
891
|
-
* FAIL CLOSED: an unparseable value, a value with no host, a non-http(s)
|
|
892
|
-
* scheme, or a public address is NOT local. See the file header for why the
|
|
893
|
-
* tie breaks that way.
|
|
894
|
-
*/
|
|
895
|
-
function isLocalEndpoint(url) {
|
|
896
|
-
let host = '';
|
|
897
|
-
let scheme = '';
|
|
898
|
-
// No host stated is not evidence of this machine: '' / whitespace / null stay
|
|
899
|
-
// refused, exactly as before (the sealed list in test/local-endpoint.test.mjs
|
|
900
|
-
// pins those). Only a NON-EMPTY string gets the scheme defaulted, which is the
|
|
901
|
-
// `localhost:11434` case above — a host the user named, in the spelling they
|
|
902
|
-
// named it in.
|
|
903
|
-
if (String(url == null ? '' : url).trim() === '') return false;
|
|
904
|
-
try {
|
|
905
|
-
const parsed = new URL(normalizeBase(url));
|
|
906
|
-
host = parsed.hostname.toLowerCase();
|
|
907
|
-
scheme = parsed.protocol.toLowerCase();
|
|
908
|
-
} catch {
|
|
909
|
-
return false;
|
|
910
|
-
}
|
|
911
|
-
// Only a base URL a transport can actually dial — checked BEFORE the host, so
|
|
912
|
-
// `ftp://box.local` cannot ride the local suffix to a pass.
|
|
913
|
-
if (scheme !== 'http:' && scheme !== 'https:') return false;
|
|
914
|
-
if (!host) return false;
|
|
915
|
-
// WHATWG keeps the brackets on an IPv6 hostname; strip them so one spelling
|
|
916
|
-
// covers both `[::1]` and a bare `::1`.
|
|
917
|
-
if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
|
|
918
|
-
if (!host || /[\s/\\@]/.test(host)) return false;
|
|
919
|
-
|
|
920
|
-
// Loopback by name, plus the mDNS / split-DNS names an on-box service
|
|
921
|
-
// answers to. The suffix must END the name: `example.local.evil.com` is
|
|
922
|
-
// somebody else's host.
|
|
923
|
-
if (host === 'localhost' || host.endsWith('.localhost')) return true;
|
|
924
|
-
if (/\.(local|internal|lan)$/.test(host)) return true;
|
|
925
|
-
|
|
926
|
-
// IPv6 literals.
|
|
927
|
-
if (host.includes(':')) {
|
|
928
|
-
if (host === '::1' || host === '0:0:0:0:0:0:0:1') return true; // loopback
|
|
929
|
-
if (/^f[cd][0-9a-f]{2}:/.test(host)) return true; // fc00::/7 unique-local
|
|
930
|
-
if (/^fe[89ab][0-9a-f]:/.test(host)) return true; // fe80::/10 link-local
|
|
931
|
-
return false;
|
|
932
|
-
}
|
|
933
|
-
|
|
934
|
-
// IPv4 literals.
|
|
935
|
-
const quad = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
|
|
936
|
-
if (quad) {
|
|
937
|
-
const o = quad.slice(1).map(Number);
|
|
938
|
-
if (o.some((n) => n > 255)) return false; // not an address, and not local
|
|
939
|
-
const [a, b] = o;
|
|
940
|
-
if (a === 127) return true; // 127.0.0.0/8 loopback
|
|
941
|
-
if (a === 10) return true; // 10.0.0.0/8
|
|
942
|
-
if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12
|
|
943
|
-
if (a === 192 && b === 168) return true; // 192.168.0.0/16
|
|
944
|
-
if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local
|
|
945
|
-
return false; // a real, routable address — remote, and not this lane
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
// A dotless name can only come from this machine's own resolver / hosts file.
|
|
949
|
-
if (!host.includes('.')) return true;
|
|
950
|
-
return false;
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
/**
|
|
954
|
-
* Refuse a base URL that is not local, in the hosts' own voice.
|
|
955
|
-
*
|
|
956
|
-
* Returns null when the URL is usable. The returned error carries `status 400`
|
|
957
|
-
* so every existing caller (the CLI's error painter, the desktop turn guard)
|
|
958
|
-
* treats it as a user-fixable configuration problem rather than a crash.
|
|
959
|
-
*/
|
|
960
|
-
function remoteRefusal(url) {
|
|
961
|
-
if (isLocalEndpoint(url)) return null;
|
|
962
|
-
const err = new Error(
|
|
963
|
-
`local: ${JSON.stringify(String(url || ''))} is not on this machine, so it is not the local ` +
|
|
964
|
-
'class. Running a model you own is free because there is no vendor to pay — a remote URL ' +
|
|
965
|
-
'here would be an unpaid turn instead. Use the aegis or byok class for remote models.'
|
|
966
|
-
);
|
|
967
|
-
err.status = 400;
|
|
968
|
-
return err;
|
|
969
|
-
}
|
|
970
|
-
|
|
971
|
-
/** Probe the local daemon. Never throws — a missing daemon is a state, not an
|
|
972
|
-
* error, because the UI has to say "start it" rather than "it failed". */
|
|
973
|
-
async function probe(baseURL = DEFAULT_BASE) {
|
|
974
|
-
const base = baseOf(baseURL);
|
|
975
|
-
try {
|
|
976
|
-
const res = await fetch(`${base}/api/tags`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS) });
|
|
977
|
-
return { running: res.ok, baseURL: base };
|
|
978
|
-
} catch {
|
|
979
|
-
return { running: false, baseURL: base };
|
|
980
|
-
}
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
/**
|
|
984
|
-
* List installed model tags (`GET /api/tags` → [{ id, details }]).
|
|
985
|
-
*
|
|
986
|
-
* Ollama-specific: a non-Ollama local server will 404 here. The caller renders
|
|
987
|
-
* that as "no models listed — type one", because the transport itself does not
|
|
988
|
-
* need this call at all. Losing the list must not cost the class its usability.
|
|
989
|
-
*/
|
|
990
|
-
async function listTags(baseURL = DEFAULT_BASE) {
|
|
991
|
-
const res = await fetch(`${baseOf(baseURL)}/api/tags`);
|
|
992
|
-
if (!res.ok) {
|
|
993
|
-
const err = new Error(`local: /api/tags answered ${res.status}`);
|
|
994
|
-
err.status = res.status;
|
|
995
|
-
throw err;
|
|
996
|
-
}
|
|
997
|
-
const data = await res.json();
|
|
998
|
-
const models = (data && data.models) || [];
|
|
999
|
-
return models.map((m) => ({
|
|
1000
|
-
id: m && m.name,
|
|
1001
|
-
...(m && m.details ? { details: m.details } : {}),
|
|
1002
|
-
})).filter((m) => m.id);
|
|
1003
|
-
}
|
|
1004
|
-
|
|
1005
|
-
/** How long an unload (`keep_alive: 0`) request waits before giving up.
|
|
1006
|
-
* Generous — Ollama still has to touch the scheduler for a model that may be
|
|
1007
|
-
* mid-generation — but bounded, because this runs on the app's quit path and
|
|
1008
|
-
* must never be the reason closing the window hangs. */
|
|
1009
|
-
const UNLOAD_TIMEOUT_MS = 4000;
|
|
1010
|
-
|
|
1011
|
-
/** How long the warm-up request below may take before it is abandoned.
|
|
1012
|
-
*
|
|
1013
|
-
* It is the TURN's own first-byte budget (`FIRST_BYTE_TIMEOUT_MS`), and that
|
|
1014
|
-
* is the point: the pin exists to absorb a load the turn would otherwise wait
|
|
1015
|
-
* out, so it may legitimately take as long as the turn would have — but it may
|
|
1016
|
-
* never outlive the caller's own patience with the daemon. Measured 107.8s for
|
|
1017
|
-
* a warm 1.5B model's first byte on this class of hardware, so a bound tight
|
|
1018
|
-
* enough to abort a wedged daemon fast would abort healthy cold loads; the
|
|
1019
|
-
* turn's five-minute bound is the measured right answer, applied twice at most
|
|
1020
|
-
* (pin, then turn) rather than once.
|
|
1021
|
-
*
|
|
1022
|
-
* A caller that STATES its own window (`firstByteTimeoutMs`) is never widened:
|
|
1023
|
-
* the pin gets that number, so the turn still fails on the caller's clock
|
|
1024
|
-
* instead of the pin's. That is the bug this bound replaced — against a daemon
|
|
1025
|
-
* that accepts the socket and never answers, a pin on a budget of its own held
|
|
1026
|
-
* the turn for minutes before the turn's own 1s deadline was even armed.
|
|
1027
|
-
*
|
|
1028
|
-
* The timer is `unref`'d (see `ensureResident`), the same rule
|
|
1029
|
-
* `FIRST_BYTE_TIMEOUT_MS`'s timer follows: a short-lived CLI process
|
|
1030
|
-
* (`aegiscode -p "…"`) must never be held open by a pending warm-up. */
|
|
1031
|
-
const RESIDENT_TIMEOUT_MS = FIRST_BYTE_TIMEOUT_MS;
|
|
1032
|
-
|
|
1033
|
-
/** How long a pin asks the daemon to hold the model warm, when nothing states
|
|
1034
|
-
* otherwise — the whole point of the pin. Ollama's stock window is 5 MINUTES,
|
|
1035
|
-
* and that is the number this lane was silently running on: idle past it and
|
|
1036
|
-
* the daemon evicts, so the next turn re-pays the entire cold load. A user
|
|
1037
|
-
* who steps away from the desktop for coffee should come back to a warm model
|
|
1038
|
-
* (they paid for the load once already), and the model is released anyway on
|
|
1039
|
-
* quit (`unloadAll`) and on a model switch (`swapWarmModel`), so 30 minutes
|
|
1040
|
-
* costs nothing that is not already accounted for.
|
|
1041
|
-
*
|
|
1042
|
-
* Tunable per call (`opts.warmSecs`) and by env (`AEGIS_LOCAL_WARM_SECS`), and
|
|
1043
|
-
* `0` is meaningful: it turns the pin off and returns the lane to the daemon's
|
|
1044
|
-
* own lifecycle. */
|
|
1045
|
-
const DEFAULT_WARM_SECS = 1800;
|
|
1046
|
-
|
|
1047
|
-
/**
|
|
1048
|
-
* Evict `model` from the daemon's memory right now, instead of waiting out
|
|
1049
|
-
* its keep-alive. Ollama's documented mechanism: a chat request with
|
|
1050
|
-
* `keep_alive: 0` and nothing to answer unloads the model as its only
|
|
1051
|
-
* effect (https://github.com/ollama/ollama/blob/main/docs/faq.md). Best-effort
|
|
1052
|
-
* and NEVER throws — called from the app's quit path and from the warm-model
|
|
1053
|
-
* swap below, neither of which may fail a turn or block a shutdown over a
|
|
1054
|
-
* daemon that is merely slow or already gone.
|
|
1055
|
-
*/
|
|
1056
|
-
async function unload(baseURL, model) {
|
|
1057
|
-
if (!model) return { ok: false };
|
|
1058
|
-
const base = baseOf(baseURL);
|
|
1059
|
-
if (!isLocalEndpoint(base)) return { ok: false }; // never dial out, even to "clean up"
|
|
1060
|
-
try {
|
|
1061
|
-
const res = await fetch(`${base}/api/chat`, {
|
|
1062
|
-
method: 'POST',
|
|
1063
|
-
headers: { 'content-type': 'application/json' },
|
|
1064
|
-
body: JSON.stringify({ model: String(model), messages: [], stream: false, keep_alive: 0 }),
|
|
1065
|
-
signal: AbortSignal.timeout(UNLOAD_TIMEOUT_MS),
|
|
1066
|
-
});
|
|
1067
|
-
return { ok: res.ok };
|
|
1068
|
-
} catch {
|
|
1069
|
-
return { ok: false };
|
|
1070
|
-
}
|
|
1071
|
-
}
|
|
1072
|
-
|
|
1073
|
-
/** Normalize one history row into an OpenAI-compatible message. Tool rows and
|
|
1074
|
-
* assistant rows carrying `tool_calls` must survive intact: dropping either
|
|
1075
|
-
* leaves the model seeing a tool call it never got a result for. */
|
|
1076
|
-
function normalizeRow(m) {
|
|
1077
|
-
if (!m || typeof m !== 'object') return null;
|
|
1078
|
-
const row = { role: m.role || 'user' };
|
|
1079
|
-
if (m.content != null) row.content = m.content;
|
|
1080
|
-
if (m.name) row.name = m.name;
|
|
1081
|
-
if (m.tool_call_id) row.tool_call_id = m.tool_call_id;
|
|
1082
|
-
if (Array.isArray(m.tool_calls) && m.tool_calls.length) row.tool_calls = m.tool_calls;
|
|
1083
|
-
return row;
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
/** One SSE event's fields, or null for a keep-alive/comment frame (which
|
|
1087
|
-
* carries no payload and so must not count as progress). */
|
|
1088
|
-
function parseEvent(raw) {
|
|
1089
|
-
const lines = String(raw).split('\n');
|
|
1090
|
-
let data = '';
|
|
1091
|
-
let sawField = false;
|
|
1092
|
-
for (const line of lines) {
|
|
1093
|
-
if (!line || line.startsWith(':')) continue; // '' separator, ':' keep-alive
|
|
1094
|
-
const i = line.indexOf(':');
|
|
1095
|
-
const field = i === -1 ? line : line.slice(0, i);
|
|
1096
|
-
const value = i === -1 ? '' : line.slice(i + 1).replace(/^ /, '');
|
|
1097
|
-
if (field === 'data') {
|
|
1098
|
-
data += value;
|
|
1099
|
-
sawField = true;
|
|
1100
|
-
} else if (field === 'event' || field === 'id' || field === 'retry') {
|
|
1101
|
-
sawField = true;
|
|
1102
|
-
}
|
|
1103
|
-
}
|
|
1104
|
-
if (!sawField || !data) return null;
|
|
1105
|
-
if (data === '[DONE]') return { done: true };
|
|
1106
|
-
try {
|
|
1107
|
-
return { json: JSON.parse(data) };
|
|
1108
|
-
} catch {
|
|
1109
|
-
return null;
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
/**
|
|
1114
|
-
* Read an SSE body, calling onEvent per parsed frame.
|
|
1115
|
-
*
|
|
1116
|
-
* The stall watchdog is armed PER PAYLOAD, not per socket read. That
|
|
1117
|
-
* distinction is the point: a server that keeps the connection warm with
|
|
1118
|
-
* keep-alive comments while the model produces nothing would otherwise reset
|
|
1119
|
-
* the deadline forever, and the turn would hang with no error at all. Progress
|
|
1120
|
-
* means a parsed payload — text, tool call, or usage — and nothing else.
|
|
1121
|
-
*/
|
|
1122
|
-
async function readSSE(res, onEvent, { idleTimeoutMs = SSE_IDLE_TIMEOUT_MS } = {}) {
|
|
1123
|
-
const reader = res.body.getReader();
|
|
1124
|
-
const decoder = new TextDecoder();
|
|
1125
|
-
let buf = '';
|
|
1126
|
-
let stallTimer = null;
|
|
1127
|
-
let stallReject = () => {};
|
|
1128
|
-
const stall = new Promise((_, reject) => {
|
|
1129
|
-
stallReject = reject;
|
|
1130
|
-
});
|
|
1131
|
-
const arm = () => {
|
|
1132
|
-
if (stallTimer) clearTimeout(stallTimer);
|
|
1133
|
-
stallTimer = setTimeout(() => {
|
|
1134
|
-
stallReject(
|
|
1135
|
-
new Error(`local: the model stopped responding — no output for ${Math.round(idleTimeoutMs / 1000)}s`)
|
|
1136
|
-
);
|
|
1137
|
-
}, idleTimeoutMs);
|
|
1138
|
-
};
|
|
1139
|
-
arm();
|
|
1140
|
-
try {
|
|
1141
|
-
for (;;) {
|
|
1142
|
-
const chunk = await Promise.race([reader.read(), stall]);
|
|
1143
|
-
if (chunk.done) break;
|
|
1144
|
-
buf += decoder.decode(chunk.value, { stream: true });
|
|
1145
|
-
// Normalize CRLF so one separator check covers both spellings.
|
|
1146
|
-
buf = buf.replace(/\r\n/g, '\n');
|
|
1147
|
-
let idx;
|
|
1148
|
-
while ((idx = buf.indexOf('\n\n')) >= 0) {
|
|
1149
|
-
const raw = buf.slice(0, idx);
|
|
1150
|
-
buf = buf.slice(idx + 2);
|
|
1151
|
-
const ev = parseEvent(raw);
|
|
1152
|
-
if (!ev) continue; // keep-alive — deliberately does NOT re-arm
|
|
1153
|
-
if (ev.done) return;
|
|
1154
|
-
if (ev.json) {
|
|
1155
|
-
arm();
|
|
1156
|
-
onEvent(ev.json);
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
|
-
} finally {
|
|
1161
|
-
if (stallTimer) clearTimeout(stallTimer);
|
|
1162
|
-
try {
|
|
1163
|
-
await reader.cancel();
|
|
1164
|
-
} catch {
|
|
1165
|
-
/* already closed */
|
|
1166
|
-
}
|
|
1167
|
-
}
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1170
|
-
/**
|
|
1171
|
-
* Serializes calls that would land on the SAME daemon+model pair.
|
|
1172
|
-
*
|
|
1173
|
-
* Ollama loads each model into its own `llama-server` with exactly one
|
|
1174
|
-
* request slot (`-np 1`). That slot does not queue a second request that
|
|
1175
|
-
* arrives while it is still generating — it CANCELS the in-flight task
|
|
1176
|
-
* server-side and answers the first caller with an HTTP 500, after however
|
|
1177
|
-
* long that caller had already been waiting (reproduced live: a stuck or
|
|
1178
|
-
* merely slow turn on this machine's CPU-only inference, then a second turn
|
|
1179
|
-
* — another chat window, a subagent, a retry — lands on the same model and
|
|
1180
|
-
* the first caller's request dies with a 500 it cannot act on). Client-side
|
|
1181
|
-
* serialization is the only place this can be prevented at all; the daemon
|
|
1182
|
-
* itself does not offer a queued mode. Keyed on (base, model) rather than
|
|
1183
|
-
* model alone: two different daemons (this box and a LAN box, say) each have
|
|
1184
|
-
* their own slot and may run fully in parallel.
|
|
1185
|
-
*/
|
|
1186
|
-
const localCallQueues = new Map(); // "<base>|<model>" -> tail promise (never rejects)
|
|
1187
|
-
|
|
1188
|
-
function runSerialized(key, fn) {
|
|
1189
|
-
const prev = localCallQueues.get(key) || Promise.resolve();
|
|
1190
|
-
// Run `fn` once the previous call for this key has SETTLED, success or
|
|
1191
|
-
// failure — a failed turn must not wedge every later turn behind it.
|
|
1192
|
-
const settled = prev.then(fn, fn);
|
|
1193
|
-
// The map holds a swallowed copy so the next call's `.then` always fires;
|
|
1194
|
-
// the caller still gets `settled`, which carries the real result or error.
|
|
1195
|
-
localCallQueues.set(
|
|
1196
|
-
key,
|
|
1197
|
-
settled.then(
|
|
1198
|
-
() => {},
|
|
1199
|
-
() => {}
|
|
1200
|
-
)
|
|
1201
|
-
);
|
|
1202
|
-
return settled;
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
// How many calls are CURRENTLY in flight for a given "<base>|<model>" key —
|
|
1206
|
-
// not the queue above (which only orders them), but a live count, so the
|
|
1207
|
-
// warm-model swap below can tell "idle" from "still serving someone else".
|
|
1208
|
-
const activeCountByKey = new Map();
|
|
1209
|
-
|
|
1210
|
-
// The one model THIS transport last warmed per daemon — not every model
|
|
1211
|
-
// Ollama happens to have loaded (another process could have warmed others),
|
|
1212
|
-
// only the ones this app is responsible for cleaning up after. Keyed on the
|
|
1213
|
-
// normalized base, same spelling as the call queue above.
|
|
1214
|
-
const warmModelByBase = new Map(); // base -> model
|
|
1215
|
-
|
|
1216
|
-
/**
|
|
1217
|
-
* Swap the daemon's "warm" model for this app from `prevModel` to `model`,
|
|
1218
|
-
* evicting the outgoing one — UNLESS something else is still actively using
|
|
1219
|
-
* it right now. That guard matters: a second chat window or a subagent still
|
|
1220
|
-
* mid-turn on the old model must not be cut out from under itself just
|
|
1221
|
-
* because a DIFFERENT turn moved on to a new one. Fired after a turn
|
|
1222
|
-
* SUCCEEDS, never before — a model that was never actually reached (a 400, a
|
|
1223
|
-
* refusal) must not be recorded as warm.
|
|
1224
|
-
*
|
|
1225
|
-
* Without this, every model switch in a session left the previous one loaded
|
|
1226
|
-
* too: a few turns in, Ollama was running two or three `llama-server`
|
|
1227
|
-
* processes at once (one per model, each pinned at ~100%+ CPU on this
|
|
1228
|
-
* class of hardware — reproduced live), each fighting the others for the
|
|
1229
|
-
* same handful of cores. Capping the daemon at one warm model per app is
|
|
1230
|
-
* what keeps that from compounding turn after turn.
|
|
1231
|
-
*/
|
|
1232
|
-
function swapWarmModel(base, model) {
|
|
1233
|
-
if (!model) return;
|
|
1234
|
-
const prevModel = warmModelByBase.get(base);
|
|
1235
|
-
warmModelByBase.set(base, model);
|
|
1236
|
-
if (!prevModel || prevModel === model) return;
|
|
1237
|
-
if (activeCountByKey.get(`${base}|${prevModel}`)) return; // still in use elsewhere
|
|
1238
|
-
unload(base, prevModel).catch(() => {});
|
|
1239
|
-
}
|
|
1240
|
-
|
|
1241
|
-
/**
|
|
1242
|
-
* Unload every model this transport warmed, across every daemon it talked
|
|
1243
|
-
* to. Called once, from the app's quit path: "closing the window closes the
|
|
1244
|
-
* model" is the point — a local model left loaded after the app exits is
|
|
1245
|
-
* pure waste, the one thing the free/local lane was supposed to never cost.
|
|
1246
|
-
* Best-effort and bounded, like `unload` itself, so this resolves quickly
|
|
1247
|
-
* even against a daemon that is slow or already gone.
|
|
1248
|
-
*/
|
|
1249
|
-
async function unloadAll() {
|
|
1250
|
-
const entries = [...warmModelByBase.entries()];
|
|
1251
|
-
warmModelByBase.clear();
|
|
1252
|
-
await Promise.all(entries.map(([base, model]) => unload(base, model)));
|
|
1253
|
-
}
|
|
1254
|
-
|
|
1255
|
-
/** The window a pin should ask for: the caller's number, else the operator's,
|
|
1256
|
-
* else the default above. `0` is a real answer (pin nothing) and is passed
|
|
1257
|
-
* through rather than treated as "unstated". */
|
|
1258
|
-
function warmSecsFor({ warmSecs } = {}) {
|
|
1259
|
-
const stated = Number(warmSecs);
|
|
1260
|
-
if (Number.isFinite(stated) && stated >= 0) return Math.floor(stated);
|
|
1261
|
-
const env = Number(process.env && process.env.AEGIS_LOCAL_WARM_SECS);
|
|
1262
|
-
if (Number.isFinite(env) && env >= 0) return Math.floor(env);
|
|
1263
|
-
return DEFAULT_WARM_SECS;
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
/**
|
|
1267
|
-
* Live-pin memo: which `<base>|<model>|<prefix>` is known warm until when.
|
|
1268
|
-
*
|
|
1269
|
-
* The pin's cost is a PREFILL, not a turn, so the steady-state win is not
|
|
1270
|
-
* making the pin cheaper — it is not issuing one. Residency IS still checked
|
|
1271
|
-
* against the daemon (`/api/ps`), because the daemon is the authority on what
|
|
1272
|
-
* it holds: another app, an `ollama stop`, or a reboot can all clear a window
|
|
1273
|
-
* this process set. But a window this process set, with seconds still left on
|
|
1274
|
-
* it, cannot have expired underneath us — so the memo answers without the
|
|
1275
|
-
* round trip, and is *dropped* the moment its deadline passes, so it can never
|
|
1276
|
-
* claim residency it cannot prove.
|
|
1277
|
-
*
|
|
1278
|
-
* Keyed by the PREFIX as well as the model, deliberately: a pin only buys
|
|
1279
|
-
* anything for the exact text it warmed (llama.cpp reuses the shared prefix and
|
|
1280
|
-
* nothing else), so a warm-up for one system prompt must never suppress the pin
|
|
1281
|
-
* for a turn sending a different one — that turn would pay the cold prefill the
|
|
1282
|
-
* memo claimed to have paid for it.
|
|
1283
|
-
*
|
|
1284
|
-
* `dialer` is the `fetch` that performed the pin. A different one (a proxy, a
|
|
1285
|
-
* test's stub, a re-`require`d daemon handle) is a different daemon and cannot
|
|
1286
|
-
* be vouched for by this map.
|
|
1287
|
-
*/
|
|
1288
|
-
const residentMemo = new Map(); // `${base}\0${model}\0${prefix}` -> { until, dialer }
|
|
1289
|
-
|
|
1290
|
-
function residentKey(base, model, prefix) {
|
|
1291
|
-
return `${base}\u0000${model}\u0000${prefix}`;
|
|
1292
|
-
}
|
|
1293
|
-
|
|
1294
|
-
/** Seconds of pinned window left for this exact prefix, or 0 when unproven. */
|
|
1295
|
-
function residentSecsLeft(base, model, prefix, { dialer = fetch, now = Date.now() } = {}) {
|
|
1296
|
-
const hit = residentMemo.get(residentKey(base, model, prefix));
|
|
1297
|
-
if (!hit || hit.dialer !== dialer) return 0;
|
|
1298
|
-
const left = Math.round((hit.until - now) / 1000);
|
|
1299
|
-
if (left <= 0) {
|
|
1300
|
-
residentMemo.delete(residentKey(base, model, prefix));
|
|
1301
|
-
return 0;
|
|
1302
|
-
}
|
|
1303
|
-
return left;
|
|
1304
|
-
}
|
|
1305
|
-
|
|
1306
|
-
/**
|
|
1307
|
-
* The live memo entry itself, so a free (no-daemon-contact) memo hit can still
|
|
1308
|
-
* report the window the pin observed. The window is stored with the pin rather
|
|
1309
|
-
* than re-read because a memo hit is answered with NO daemon contact by design —
|
|
1310
|
-
* that is what makes a burst of turns free.
|
|
1311
|
-
*/
|
|
1312
|
-
function residentEntry(base, model, prefix, { dialer = fetch, now = Date.now() } = {}) {
|
|
1313
|
-
const hit = residentMemo.get(residentKey(base, model, prefix));
|
|
1314
|
-
if (!hit || hit.dialer !== dialer) return null;
|
|
1315
|
-
return hit.until - now > 0 ? hit : null;
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1318
|
-
/**
|
|
1319
|
-
* Does a `/api/ps` row name the model the user picked?
|
|
1320
|
-
*
|
|
1321
|
-
* Matched by prefix because the daemon reports the RESOLVED identity while the
|
|
1322
|
-
* row carries whatever the user chose: `llama3.2:1b` for a plain pull, and
|
|
1323
|
-
* `name:tag@sha256:…` for a digest-pinned or remotely-pulled one. The digest
|
|
1324
|
-
* form is why `@` is here and not only `:` — the rule this replaced claimed to
|
|
1325
|
-
* handle it in a comment while matching only the colon.
|
|
1326
|
-
*
|
|
1327
|
-
* One predicate rather than two copies: it was already duplicated between the
|
|
1328
|
-
* residency read and the window read, and a rule stated twice drifts.
|
|
1329
|
-
*/
|
|
1330
|
-
function sameModel(reported, picked) {
|
|
1331
|
-
const name = String(reported || '');
|
|
1332
|
-
const want = String(picked || '');
|
|
1333
|
-
if (!name || !want) return false;
|
|
1334
|
-
return name === want || name.startsWith(`${want}:`) || name.startsWith(`${want}@`);
|
|
1335
|
-
}
|
|
1336
|
-
|
|
1337
|
-
/**
|
|
1338
|
-
* How long the standalone window read (`windowOf`) waits. Short on purpose: it is
|
|
1339
|
-
* a loopback metadata call (measured at 115µs on this host) that runs BEFORE the
|
|
1340
|
-
* turn, so it must never become a place a turn can park. The probe's budget, not
|
|
1341
|
-
* the generation's — see `PROBE_TIMEOUT_MS` above for the same reasoning.
|
|
1342
|
-
*/
|
|
1343
|
-
const WINDOW_TIMEOUT_MS = 3000;
|
|
1344
|
-
|
|
1345
|
-
/**
|
|
1346
|
-
* The window of the LOADED runner for `model`, straight from `/api/ps`, or null.
|
|
1347
|
-
*
|
|
1348
|
-
* Never throws and never guesses — the same rule `environmentPreamble` follows
|
|
1349
|
-
* for a missing field. `/api/ps` is the only honest source: `/api/show` reports
|
|
1350
|
-
* the model's TRAINING capacity (`llama.context_length` 131072 for a model this
|
|
1351
|
-
* daemon runs at 4096), and fitting a turn to that would not fit it at all.
|
|
1352
|
-
*
|
|
1353
|
-
* This exists as a fallback rather than as the only read because the pin can fail
|
|
1354
|
-
* for reasons that say nothing about the window — a Stop, its own clock, a daemon
|
|
1355
|
-
* that will not enumerate models — and a turn that then sent its payload blind
|
|
1356
|
-
* would overrun the window and have its SYSTEM row shifted away by llama.cpp's
|
|
1357
|
-
* `--context-shift`. The answer to "how big is this window" should not depend on
|
|
1358
|
-
* a warm-up succeeding.
|
|
1359
|
-
*/
|
|
1360
|
-
async function windowOf(base, model, { signal } = {}) {
|
|
1361
|
-
// Bounded, because this runs BEFORE the turn and unwatched it is precisely the
|
|
1362
|
-
// hang this transport already had to fix once: a daemon that accepts the socket
|
|
1363
|
-
// and never answers would otherwise park the turn here, before `chatOne`'s own
|
|
1364
|
-
// first-byte deadline is even armed, and a request nothing is timing out is a
|
|
1365
|
-
// request nothing can fail. The budget is the probe's, not the turn's — this is
|
|
1366
|
-
// a loopback metadata call (measured at 115µs on this host) and it must not eat
|
|
1367
|
-
// the generation's clock.
|
|
1368
|
-
const ctl = new AbortController();
|
|
1369
|
-
const onCallerAbort = () => ctl.abort();
|
|
1370
|
-
if (signal) {
|
|
1371
|
-
if (signal.aborted) return null;
|
|
1372
|
-
if (typeof signal.addEventListener === 'function') signal.addEventListener('abort', onCallerAbort, { once: true });
|
|
1373
|
-
}
|
|
1374
|
-
const timer = setTimeout(() => ctl.abort(), WINDOW_TIMEOUT_MS);
|
|
1375
|
-
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
1376
|
-
try {
|
|
1377
|
-
const res = await fetch(`${base}/api/ps`, { method: 'GET', signal: ctl.signal });
|
|
1378
|
-
if (!res || !res.ok || typeof res.json !== 'function') return null;
|
|
1379
|
-
const body = await res.json();
|
|
1380
|
-
const models = (body && body.models) || [];
|
|
1381
|
-
const hit = models.find((m) => sameModel(m && (m.name || m.model), model));
|
|
1382
|
-
const win = hit && Number(hit.context_length);
|
|
1383
|
-
return Number.isFinite(win) && win > 0 ? win : null;
|
|
1384
|
-
} catch {
|
|
1385
|
-
return null;
|
|
1386
|
-
} finally {
|
|
1387
|
-
clearTimeout(timer);
|
|
1388
|
-
if (signal && typeof signal.removeEventListener === 'function') {
|
|
1389
|
-
signal.removeEventListener('abort', onCallerAbort);
|
|
1390
|
-
}
|
|
1391
|
-
}
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
/**
|
|
1395
|
-
* Make `model` resident on the local daemon and PIN its window — the extend
|
|
1396
|
-
* call this lane was missing.
|
|
1397
|
-
*
|
|
1398
|
-
* Why this exists, and why the arguments are what they are. The cost of a local
|
|
1399
|
-
* turn is a LOAD cost, not a turn cost: a cold first call pays for reading the
|
|
1400
|
-
* weights off disk, and every call after it is fast because llama.cpp reuses the
|
|
1401
|
-
* KV of the shared prefix. Without a pin this lane ran on the daemon's stock
|
|
1402
|
-
* 5-minute window and had no way to extend it: `keep_alive` on
|
|
1403
|
-
* `/v1/chat/completions` is accepted and IGNORED (measured, engine-side:
|
|
1404
|
-
* 0/120/1800 all yield the same stored window), so the ONLY endpoint that can
|
|
1405
|
-
* extend residency is Ollama's native `/api/chat`. Step away for six minutes and
|
|
1406
|
-
* the next turn re-bought the entire cold load.
|
|
1407
|
-
*
|
|
1408
|
-
* `system` is the prefix this pin warms, and it MUST be the turn's own system
|
|
1409
|
-
* text — the caller passes `opts.system` straight through from the engine's
|
|
1410
|
-
* dispatch, which is the exact string the turn is about to send. Warming
|
|
1411
|
-
* anything else is worse than warming nothing: the daemon uses the REQUEST's
|
|
1412
|
-
* system message in place of the model's baked Modelfile default, so a pin
|
|
1413
|
-
* carrying different text diverges from the turn at the first token, shares no
|
|
1414
|
-
* prefix with it, and EVICTS the very cache it was meant to warm. This is not a
|
|
1415
|
-
* theory — it is the measured failure mode that made the engine-side pin
|
|
1416
|
-
* useless (a mismatched pin cost 1.30s and left the following real turn at
|
|
1417
|
-
* 2.26s, where a matching pin cost 0.53s and left it at 0.58s). So: no
|
|
1418
|
-
* fabricated "you are a local model" line, no Modelfile guess. Whatever the
|
|
1419
|
-
* caller says, or nothing.
|
|
1420
|
-
*
|
|
1421
|
-
* Three measured constraints are honoured by the order of operations below:
|
|
1422
|
-
* * a memo hit is answered with NO daemon contact and no slot (it is free);
|
|
1423
|
-
* * a daemon that already holds an adequate window needs no pin (one GET);
|
|
1424
|
-
* * a pin that is not needed is not issued, because its prefill is the cost.
|
|
1425
|
-
*
|
|
1426
|
-
* Never throws and never rejects: the caller is a live turn, and "could not
|
|
1427
|
-
* warm" is information, not a failure. A pin that fails is simply not memoized,
|
|
1428
|
-
* so the next turn retries instead of trusting a lie.
|
|
1429
|
-
*/
|
|
1430
|
-
async function ensureResident({ baseURL = DEFAULT_BASE, model, system = '', signal, warmSecs, firstByteTimeoutMs } = {}) {
|
|
1431
|
-
const base = baseOf(baseURL);
|
|
1432
|
-
// The fence covers the warm-up too. `unload` states the same rule ("never
|
|
1433
|
-
// dial out, even to clean up"); a helper that dials where the turn may not
|
|
1434
|
-
// would be the hole in it. The turn itself (`chatOne`) still does the
|
|
1435
|
-
// refusing and erroring — this just does not add a second caller.
|
|
1436
|
-
if (!isLocalEndpoint(base)) return { ok: false, skipped: 'remote' };
|
|
1437
|
-
if (!model || !String(model).trim()) return { ok: false, skipped: 'no-model' };
|
|
1438
|
-
|
|
1439
|
-
const want = warmSecsFor({ warmSecs });
|
|
1440
|
-
const prefix = typeof system === 'string' ? system : '';
|
|
1441
|
-
const prefixSource = prefix ? 'turn-system' : 'none';
|
|
1442
|
-
const shape = { warmSecs: want, prefixSource, prefixChars: prefix.length };
|
|
1443
|
-
if (want <= 0) return { ok: false, skipped: 'warm-disabled', ...shape };
|
|
1444
|
-
|
|
1445
|
-
const dialer = fetch;
|
|
1446
|
-
// Free answer first: a window this process set, still carrying seconds, for
|
|
1447
|
-
// THIS prefix. Consulted before any request, so a burst of turns costs
|
|
1448
|
-
// nothing at all rather than a loopback GET each.
|
|
1449
|
-
const memoLeft = residentSecsLeft(base, model, prefix, { dialer });
|
|
1450
|
-
if (memoLeft >= Math.min(want, 60)) {
|
|
1451
|
-
const entry = residentEntry(base, model, prefix, { dialer });
|
|
1452
|
-
return {
|
|
1453
|
-
ok: true,
|
|
1454
|
-
already: true,
|
|
1455
|
-
memo: true,
|
|
1456
|
-
remainingSecs: memoLeft,
|
|
1457
|
-
contextLength: (entry && entry.contextLength) || null,
|
|
1458
|
-
...shape,
|
|
1459
|
-
};
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
// The caller's Stop must reach the warm-up too: a user who cancels should not
|
|
1463
|
-
// leave a 300s request running behind them.
|
|
1464
|
-
const ctl = new AbortController();
|
|
1465
|
-
const onCallerAbort = () => ctl.abort();
|
|
1466
|
-
if (signal) {
|
|
1467
|
-
if (signal.aborted) ctl.abort();
|
|
1468
|
-
else if (typeof signal.addEventListener === 'function') {
|
|
1469
|
-
signal.addEventListener('abort', onCallerAbort, { once: true });
|
|
1470
|
-
}
|
|
1471
|
-
}
|
|
1472
|
-
// The pin's own clock — ONE timer for the whole pin (the residency GET and
|
|
1473
|
-
// the warm-up POST share it), so a daemon that answers one and stalls the
|
|
1474
|
-
// other cannot spend the budget twice. A caller that STATED a window is
|
|
1475
|
-
// honoured exactly, never widened: the turn must fail on the caller's clock,
|
|
1476
|
-
// not on the pin's.
|
|
1477
|
-
const stated = Number(firstByteTimeoutMs);
|
|
1478
|
-
const budgetMs = Number.isFinite(stated) && stated > 0 ? Math.min(stated, RESIDENT_TIMEOUT_MS) : RESIDENT_TIMEOUT_MS;
|
|
1479
|
-
let pinTimedOut = false;
|
|
1480
|
-
// The window of the LOADED runner, read from `/api/ps` below and handed back
|
|
1481
|
-
// to the caller so the turn it warms can be fitted to it. `/api/ps` is the
|
|
1482
|
-
// only honest source: `/api/show` reports the model's training capacity
|
|
1483
|
-
// (`llama.context_length` 131072 on a model running at 4096), and a window
|
|
1484
|
-
// guessed from that is worse than no window at all.
|
|
1485
|
-
let daemonContext = null;
|
|
1486
|
-
const timer = setTimeout(() => {
|
|
1487
|
-
pinTimedOut = true;
|
|
1488
|
-
ctl.abort();
|
|
1489
|
-
}, budgetMs);
|
|
1490
|
-
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
1491
|
-
|
|
1492
|
-
try {
|
|
1493
|
-
// Is the daemon already holding this model long enough? One GET, and a
|
|
1494
|
-
// "yes" is worth the whole pin: the prefill is the cost we are avoiding.
|
|
1495
|
-
try {
|
|
1496
|
-
const ps = await fetch(`${base}/api/ps`, { method: 'GET', signal: ctl.signal });
|
|
1497
|
-
if (ps && ps.ok && typeof ps.json === 'function') {
|
|
1498
|
-
const body = await ps.json();
|
|
1499
|
-
const models = (body && body.models) || [];
|
|
1500
|
-
// Matched by the shared predicate: /api/ps reports the RESOLVED
|
|
1501
|
-
// identity ("llama3.2:1b", or "name:tag@sha256:…" for a digest-pinned
|
|
1502
|
-
// pull) while the row carries whatever the user picked.
|
|
1503
|
-
const hit = models.find((m) => sameModel(m && (m.name || m.model), model));
|
|
1504
|
-
// The loaded runner's window, when the daemon states one.
|
|
1505
|
-
if (hit && Number.isFinite(hit.context_length) && hit.context_length > 0) {
|
|
1506
|
-
daemonContext = hit.context_length;
|
|
1507
|
-
}
|
|
1508
|
-
if (hit && hit.expires_at) {
|
|
1509
|
-
const left = Math.round((new Date(hit.expires_at).getTime() - Date.now()) / 1000);
|
|
1510
|
-
if (Number.isFinite(left) && left >= Math.min(want, 60)) {
|
|
1511
|
-
// A proven window is worth hinting to the memo — but only for the
|
|
1512
|
-
// prefix the caller is about to send, which is the one key this
|
|
1513
|
-
// hint is filed under.
|
|
1514
|
-
residentMemo.set(residentKey(base, model, prefix), { until: Date.now() + left * 1000, dialer, contextLength: daemonContext });
|
|
1515
|
-
return { ok: true, already: true, remainingSecs: left, contextLength: daemonContext, ...shape };
|
|
1516
|
-
}
|
|
1517
|
-
}
|
|
1518
|
-
}
|
|
1519
|
-
} catch {
|
|
1520
|
-
/* a daemon that will not enumerate is pinned blind, below */
|
|
1521
|
-
}
|
|
1522
|
-
if (ctl.signal.aborted) {
|
|
1523
|
-
// The caller's Stop, or the pin's own clock. Neither is a turn failure:
|
|
1524
|
-
// the turn below runs and reports its OWN outcome, on the caller's terms.
|
|
1525
|
-
return { ok: false, aborted: true, timeout: pinTimedOut, contextLength: daemonContext, ...shape };
|
|
1526
|
-
}
|
|
1527
|
-
|
|
1528
|
-
const messages = prefix
|
|
1529
|
-
? [{ role: 'system', content: prefix }, { role: 'user', content: 'PONG' }]
|
|
1530
|
-
: [{ role: 'user', content: 'PONG' }];
|
|
1531
|
-
const res = await fetch(`${base}/api/chat`, {
|
|
1532
|
-
method: 'POST',
|
|
1533
|
-
headers: { 'content-type': 'application/json' },
|
|
1534
|
-
body: JSON.stringify({
|
|
1535
|
-
model: String(model),
|
|
1536
|
-
// The native field that actually works — see this function's note.
|
|
1537
|
-
keep_alive: want,
|
|
1538
|
-
stream: false,
|
|
1539
|
-
messages,
|
|
1540
|
-
// One token: enough to force the full prefill of the prefix, cheap
|
|
1541
|
-
// enough that the generation half of the call is free.
|
|
1542
|
-
options: { num_predict: 1 },
|
|
1543
|
-
}),
|
|
1544
|
-
signal: ctl.signal,
|
|
1545
|
-
});
|
|
1546
|
-
|
|
1547
|
-
// Read the body AND keep the daemon's own accounting from it. Ollama does
|
|
1548
|
-
// not bill a pin, but it does REPORT one, and `prompt_eval_count` is the
|
|
1549
|
-
// only honest measure of what the extend cost — throwing it away makes "is
|
|
1550
|
-
// the pin expensive?" a matter of opinion rather than a number.
|
|
1551
|
-
let promptTokens = null;
|
|
1552
|
-
let outputTokens = null;
|
|
1553
|
-
if (res && typeof res.text === 'function') {
|
|
1554
|
-
const raw = await res.text().catch(() => '');
|
|
1555
|
-
try {
|
|
1556
|
-
const parsed = JSON.parse(raw);
|
|
1557
|
-
if (Number.isFinite(parsed.prompt_eval_count)) promptTokens = parsed.prompt_eval_count;
|
|
1558
|
-
if (Number.isFinite(parsed.eval_count)) outputTokens = parsed.eval_count;
|
|
1559
|
-
} catch {
|
|
1560
|
-
/* a non-JSON body is still a successful pin */
|
|
1561
|
-
}
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
if (res && res.ok) {
|
|
1565
|
-
// A COLD load is why this re-read exists. The window is read from
|
|
1566
|
-
// `/api/ps` at the top of this function, and on a cold load the daemon has
|
|
1567
|
-
// no runner to report — the pin's own request is what loads it, so the
|
|
1568
|
-
// window was unknown at the moment it was asked for and the turn that
|
|
1569
|
-
// follows would have gone out unfitted. Measured live: nothing loaded ->
|
|
1570
|
-
// the pin reported `contextLength: null` while returning `ok: true`, and
|
|
1571
|
-
// the turn sent 4516 estimated tokens at a window of 4096. One GET after
|
|
1572
|
-
// the load closes it, on the cold path only.
|
|
1573
|
-
if (daemonContext == null) {
|
|
1574
|
-
daemonContext = (await windowOf(base, model, { signal: ctl.signal })) || null;
|
|
1575
|
-
}
|
|
1576
|
-
residentMemo.set(residentKey(base, model, prefix), { until: Date.now() + want * 1000, dialer, contextLength: daemonContext });
|
|
1577
|
-
// Record it as this app's warm model the moment the daemon ACCEPTS it,
|
|
1578
|
-
// not only after a successful turn. Otherwise a turn that fails right
|
|
1579
|
-
// after its pin (a 400, a Stop, a full context window) leaves a loaded
|
|
1580
|
-
// model that `unloadAll` never learned about — exactly the leak
|
|
1581
|
-
// `unloadAll` exists to close. `swapWarmModel` is idempotent, so the
|
|
1582
|
-
// post-turn call is still the one that keeps the "only a successful turn
|
|
1583
|
-
// moves the warm model" contract for the SWITCH this makes.
|
|
1584
|
-
swapWarmModel(base, model);
|
|
1585
|
-
}
|
|
1586
|
-
return {
|
|
1587
|
-
ok: !!(res && res.ok),
|
|
1588
|
-
status: res && res.status,
|
|
1589
|
-
pinned: true,
|
|
1590
|
-
promptTokens,
|
|
1591
|
-
outputTokens,
|
|
1592
|
-
// The window the turn that follows will actually be given, so it can be
|
|
1593
|
-
// fitted to it. Null when the daemon did not state one.
|
|
1594
|
-
contextLength: daemonContext,
|
|
1595
|
-
...shape,
|
|
1596
|
-
};
|
|
1597
|
-
} catch (e) {
|
|
1598
|
-
// Never fatal. A turn is better than a pin, always — the caller runs the
|
|
1599
|
-
// turn regardless of what this returns.
|
|
1600
|
-
return { ok: false, error: String((e && e.message) || e), timeout: pinTimedOut, contextLength: daemonContext, ...shape };
|
|
1601
|
-
} finally {
|
|
1602
|
-
clearTimeout(timer);
|
|
1603
|
-
if (signal && typeof signal.removeEventListener === 'function') {
|
|
1604
|
-
signal.removeEventListener('abort', onCallerAbort);
|
|
1605
|
-
}
|
|
1606
|
-
}
|
|
1607
|
-
}
|
|
1608
|
-
|
|
1609
|
-
/**
|
|
1610
|
-
* The result a turn the user STOPPED resolves to. Every transport the engine
|
|
1611
|
-
* dispatches to resolves an aborted request with an empty, well-formed answer
|
|
1612
|
-
* (see the engine's own guard: "an aborted request resolves with exactly that —
|
|
1613
|
-
* no text"), and this makes the local lane do the same instead of rejecting
|
|
1614
|
-
* with the transport's raw AbortError.
|
|
1615
|
-
*
|
|
1616
|
-
* `content` carries whatever had already streamed, because every other
|
|
1617
|
-
* transport returns the partial it collected and the renderer has already
|
|
1618
|
-
* painted those deltas — dropping them here would make the transcript
|
|
1619
|
-
* disagree with the bubble. Tool calls are deliberately NOT replayed: a turn
|
|
1620
|
-
* that was stopped is over, and the engine drops tools for an aborted turn
|
|
1621
|
-
* anyway, so listing them would only put an unexecuted call in the transcript.
|
|
1622
|
-
*/
|
|
1623
|
-
function stoppedTurn(model, { content = '', usage = null } = {}) {
|
|
1624
|
-
const result = {
|
|
1625
|
-
id: 'local',
|
|
1626
|
-
model: String(model),
|
|
1627
|
-
choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],
|
|
1628
|
-
};
|
|
1629
|
-
if (usage) result.usage = usage;
|
|
1630
|
-
return result;
|
|
1631
|
-
}
|
|
1632
|
-
|
|
1633
|
-
/**
|
|
1634
|
-
* Keep a turn's wire payload inside the window the daemon actually granted.
|
|
1635
|
-
*
|
|
1636
|
-
* Ollama's OpenAI shim is the endpoint this transport talks to, and it can
|
|
1637
|
-
* ACCEPT `num_ctx` without honouring it. Measured on this host: `options.num_ctx`
|
|
1638
|
-
* AND a top-level `num_ctx` on `/v1/chat/completions` both returned 200 and left
|
|
1639
|
-
* the loaded window untouched at 4096, while native `/api/chat` moved it
|
|
1640
|
-
* 4096 → 8192 — by RELOADING the runner. Asking this lane for a bigger window
|
|
1641
|
-
* therefore does not work, and asking for one that the next `/v1` turn
|
|
1642
|
-
* contradicts is worse than not asking: `/api/ps` shows the runner respawn
|
|
1643
|
-
* (pid 200467 at n_ctx 8192 → pid 200484 at 4096), and a respawn discards the
|
|
1644
|
-
* very prefix the pin just warmed.
|
|
1645
|
-
*
|
|
1646
|
-
* What remains is the other half of the same problem — not OVERRUNNING the
|
|
1647
|
-
* window we are given. llama-server runs with `--context-shift --keep 4` (its own
|
|
1648
|
-
* command line, read from the daemon log), which does not fail on an oversized
|
|
1649
|
-
* prompt: it silently discards the OLDEST tokens to make room. The oldest tokens
|
|
1650
|
-
* are the system row — the persona, the work rules and the tool schemas — so an
|
|
1651
|
-
* overrun costs the model its instructions mid-turn and it answers as though it
|
|
1652
|
-
* had never been given any. That is what "ollama is broken" looked like.
|
|
1653
|
-
*
|
|
1654
|
-
* The window is READ, never guessed. `/api/show` is not a source for it: it
|
|
1655
|
-
* reports the model's TRAINING capacity (`llama.context_length` 131072 on a
|
|
1656
|
-
* model this daemon runs at 4096). The only honest source is `/api/ps`, which
|
|
1657
|
-
* reports the window of the LOADED runner — and `ensureResident` already makes
|
|
1658
|
-
* that request. No reported window means no trimming and no guessing, which is
|
|
1659
|
-
* today's behaviour.
|
|
1660
|
-
*
|
|
1661
|
-
* Returns null when there is no window to fit against, so the caller can leave
|
|
1662
|
-
* the payload exactly as it was.
|
|
1663
|
-
*/
|
|
1664
|
-
function fitToWindowOf({ rows, system, tools, contextLength, maxTokens }) {
|
|
1665
|
-
const win = Number(contextLength);
|
|
1666
|
-
if (!Number.isFinite(win) || win <= 0) return null;
|
|
1667
|
-
const history = (Array.isArray(rows) ? rows : []).filter((r) => r && r.role !== 'system');
|
|
1668
|
-
// Tool schemas are wire material the model has to see, so they are charged
|
|
1669
|
-
// before any history is kept. Otherwise the trimmer believes a 4096-token
|
|
1670
|
-
// window has 4096 tokens free when ~1971 of them are already spoken for
|
|
1671
|
-
// (measured: 10 schemas, 7882 chars ≈ 1971 tokens — almost half the window,
|
|
1672
|
-
// spent before the user's first word).
|
|
1673
|
-
const fixedTokens = Array.isArray(tools) && tools.length ? estimateTokens(JSON.stringify(tools)) : 0;
|
|
1674
|
-
// The reply needs room to exist. A stated cap is honoured exactly; with none,
|
|
1675
|
-
// a quarter of the window — a turn that fills the window exactly leaves
|
|
1676
|
-
// nowhere to emit into, and the daemon starts shifting the prompt to write.
|
|
1677
|
-
const stated = Number(maxTokens);
|
|
1678
|
-
const reserveTokens = Number.isFinite(stated) && stated > 0 ? stated : Math.round(win / 4);
|
|
1679
|
-
const trimmed = trimContext({
|
|
1680
|
-
messages: history,
|
|
1681
|
-
system: typeof system === 'string' ? system : '',
|
|
1682
|
-
budgetTokens: win,
|
|
1683
|
-
reserveTokens,
|
|
1684
|
-
fixedTokens,
|
|
1685
|
-
});
|
|
1686
|
-
return {
|
|
1687
|
-
window: win,
|
|
1688
|
-
reserveTokens,
|
|
1689
|
-
toolTokens: fixedTokens,
|
|
1690
|
-
dropped: trimmed.dropped,
|
|
1691
|
-
messages: trimmed.messages,
|
|
1692
|
-
// Still over after dropping every turn it could: the newest turn alone does
|
|
1693
|
-
// not fit. Reported rather than hidden — the daemon will shift the prompt
|
|
1694
|
-
// instead, and which turn did it is worth knowing.
|
|
1695
|
-
over: trimmed.estimatedTokens > Math.max(0, win - reserveTokens),
|
|
1696
|
-
estimatedTokens: trimmed.estimatedTokens,
|
|
1697
|
-
};
|
|
1698
|
-
}
|
|
1699
|
-
|
|
1700
|
-
/**
|
|
1701
|
-
* One streaming turn against a local model. Keyless by construction — there is
|
|
1702
|
-
* no credential to send, which is a second reason a remote URL here could never
|
|
1703
|
-
* be billed.
|
|
1704
|
-
*
|
|
1705
|
-
* Returns the OpenAI shape the engine reads: `choices[0].message.content` and
|
|
1706
|
-
* `choices[0].message.tool_calls`.
|
|
1707
|
-
*/
|
|
1708
|
-
async function chatOne({
|
|
1709
|
-
baseURL = DEFAULT_BASE,
|
|
1710
|
-
model,
|
|
1711
|
-
messages,
|
|
1712
|
-
system,
|
|
1713
|
-
prompt,
|
|
1714
|
-
maxTokens,
|
|
1715
|
-
temperature,
|
|
1716
|
-
tools,
|
|
1717
|
-
toolChoice,
|
|
1718
|
-
signal,
|
|
1719
|
-
onDelta,
|
|
1720
|
-
firstByteTimeoutMs = FIRST_BYTE_TIMEOUT_MS,
|
|
1721
|
-
// The window the daemon actually granted, read from `/api/ps` by the pin
|
|
1722
|
-
// (`ensureResident`) and threaded through here. Absent when the daemon did not
|
|
1723
|
-
// report one — in which case nothing is trimmed and nothing is guessed.
|
|
1724
|
-
contextLength,
|
|
1725
|
-
} = {}) {
|
|
1726
|
-
const base = baseOf(baseURL);
|
|
1727
|
-
const refusal = remoteRefusal(base);
|
|
1728
|
-
if (refusal) throw refusal;
|
|
1729
|
-
if (!model || !String(model).trim()) {
|
|
1730
|
-
const err = new Error('local: no model selected — pick one from the model list, or type a tag');
|
|
1731
|
-
err.status = 400;
|
|
1732
|
-
throw err;
|
|
1733
|
-
}
|
|
1734
|
-
|
|
1735
|
-
const rows = [];
|
|
1736
|
-
if (system) rows.push({ role: 'system', content: system });
|
|
1737
|
-
for (const m of Array.isArray(messages) ? messages : []) {
|
|
1738
|
-
if (m && m.role === 'system') continue; // never two system rows
|
|
1739
|
-
const row = normalizeRow(m);
|
|
1740
|
-
if (row) rows.push(row);
|
|
1741
|
-
}
|
|
1742
|
-
// The engine passes the user's own text as `prompt` alongside the history.
|
|
1743
|
-
// Append it only if the history does not already end with it, so a caller
|
|
1744
|
-
// that includes it cannot send the same turn twice.
|
|
1745
|
-
//
|
|
1746
|
-
// `prompt` may be a content-part ARRAY when the turn carries dropped
|
|
1747
|
-
// attachments (see lib/local/attachments.js buildContent). Coercing that to
|
|
1748
|
-
// '' — which a bare `typeof prompt === 'string'` test does — would drop the
|
|
1749
|
-
// user's pictures AND their question on this transport alone, so the array
|
|
1750
|
-
// form is appended as-is and the duplicate test compares flattened text.
|
|
1751
|
-
const p = typeof prompt === 'string' || Array.isArray(prompt) ? prompt : '';
|
|
1752
|
-
const pIsEmpty = p === '' || (Array.isArray(p) && p.length === 0);
|
|
1753
|
-
if (!pIsEmpty) {
|
|
1754
|
-
const last = rows[rows.length - 1];
|
|
1755
|
-
const nowText = attachmentsModule.flattenContent(p);
|
|
1756
|
-
const lastText = last ? attachmentsModule.flattenContent(last.content) : null;
|
|
1757
|
-
if (!(last && last.role === 'user' && lastText === nowText)) {
|
|
1758
|
-
rows.push({ role: 'user', content: p });
|
|
1759
|
-
}
|
|
1760
|
-
}
|
|
1761
|
-
if (!rows.length) {
|
|
1762
|
-
const err = new Error('local: nothing to send — no prompt and no messages');
|
|
1763
|
-
err.status = 400;
|
|
1764
|
-
throw err;
|
|
1765
|
-
}
|
|
1766
|
-
|
|
1767
|
-
// Fit the turn to the window the daemon actually granted. See
|
|
1768
|
-
// `fitToWindowOf` for why this is the transport's job rather than the
|
|
1769
|
-
// caller's, and why the window is read rather than requested.
|
|
1770
|
-
let fit = null;
|
|
1771
|
-
if (contextLength != null) {
|
|
1772
|
-
const sysRow = rows[0] && rows[0].role === 'system' ? rows[0] : null;
|
|
1773
|
-
fit = fitToWindowOf({ rows, system: (sysRow && sysRow.content) || system || '', tools, contextLength, maxTokens });
|
|
1774
|
-
if (fit) {
|
|
1775
|
-
const kept = fit.messages;
|
|
1776
|
-
rows.length = 0;
|
|
1777
|
-
if (sysRow) rows.push(sysRow);
|
|
1778
|
-
rows.push(...kept);
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
|
|
1782
|
-
const body = {
|
|
1783
|
-
model: String(model),
|
|
1784
|
-
messages: rows,
|
|
1785
|
-
stream: true,
|
|
1786
|
-
// Ask for usage even on a stream: Ollama's shim reports it when asked, and
|
|
1787
|
-
// without the ask a local turn's token count is invisible in the UI.
|
|
1788
|
-
stream_options: { include_usage: true },
|
|
1789
|
-
...(maxTokens ? { max_tokens: maxTokens } : {}),
|
|
1790
|
-
...(temperature != null ? { temperature } : {}),
|
|
1791
|
-
...(tools && tools.length ? { tools } : {}),
|
|
1792
|
-
...(toolChoice ? { tool_choice: toolChoice } : {}),
|
|
1793
|
-
};
|
|
1794
|
-
|
|
1795
|
-
// The caller's abort is FORWARDED to our own controller rather than handed
|
|
1796
|
-
// straight to `fetch`, so one request can carry both the caller's intent and
|
|
1797
|
-
// the first-byte deadline below. It stays attached for the whole turn, not
|
|
1798
|
-
// just the header wait: aborting after headers is what tears down the body
|
|
1799
|
-
// mid-stream, which is how Stop reaches a generation already in flight.
|
|
1800
|
-
const ctl = new AbortController();
|
|
1801
|
-
let firstByteTimedOut = false;
|
|
1802
|
-
const firstByteTimer = setTimeout(() => {
|
|
1803
|
-
firstByteTimedOut = true;
|
|
1804
|
-
ctl.abort();
|
|
1805
|
-
}, firstByteTimeoutMs);
|
|
1806
|
-
// Never the reason a short-lived CLI process (`aegiscode -p "…"`) refuses to
|
|
1807
|
-
// exit, same rule the cloud client's boundedFetch follows.
|
|
1808
|
-
if (firstByteTimer && typeof firstByteTimer.unref === 'function') firstByteTimer.unref();
|
|
1809
|
-
const onCallerAbort = () => ctl.abort();
|
|
1810
|
-
if (signal) {
|
|
1811
|
-
if (signal.aborted) ctl.abort();
|
|
1812
|
-
else if (typeof signal.addEventListener === 'function') {
|
|
1813
|
-
signal.addEventListener('abort', onCallerAbort, { once: true });
|
|
1814
|
-
}
|
|
1815
|
-
}
|
|
1816
|
-
|
|
1817
|
-
let res;
|
|
1818
|
-
try {
|
|
1819
|
-
res = await fetch(`${base}/v1/chat/completions`, {
|
|
1820
|
-
method: 'POST',
|
|
1821
|
-
headers: { 'content-type': 'application/json' },
|
|
1822
|
-
body: JSON.stringify(body),
|
|
1823
|
-
signal: ctl.signal,
|
|
1824
|
-
});
|
|
1825
|
-
} catch (err) {
|
|
1826
|
-
if (firstByteTimedOut || isHeadersDeadline(err)) {
|
|
1827
|
-
// Two different clocks can end this wait, and the user should be told
|
|
1828
|
-
// which one did — see `isHeadersDeadline` for the second, which is not
|
|
1829
|
-
// ours and was reaching the host as an opaque `TypeError: fetch failed`.
|
|
1830
|
-
const whose = firstByteTimedOut
|
|
1831
|
-
? `sent nothing in ${Math.round(firstByteTimeoutMs / 1000)}s`
|
|
1832
|
-
: `sent no response headers before this runtime's own ${Math.round(HEADERS_DEADLINE_MS / 1000)}s deadline`;
|
|
1833
|
-
const e = new Error(
|
|
1834
|
-
`local: ${model} ${whose} — the daemon is still loading the model, or its prompt ` +
|
|
1835
|
-
'prefill is longer than that. `ollama ps` shows which; a smaller prompt or a smaller ' +
|
|
1836
|
-
'model is the way through.'
|
|
1837
|
-
);
|
|
1838
|
-
// The shape the cloud client gives its own timeout (status 0 + a code),
|
|
1839
|
-
// so "waited too long" stays one branch in the renderer for every class.
|
|
1840
|
-
e.status = 0;
|
|
1841
|
-
e.code = 'timeout';
|
|
1842
|
-
throw e;
|
|
1843
|
-
}
|
|
1844
|
-
// A STOPPED turn is not a failed one, and it must not look like one. The
|
|
1845
|
-
// engine states this contract for every transport it dispatches to
|
|
1846
|
-
// (engine.js: "an aborted request resolves with exactly that — no text",
|
|
1847
|
-
// the guard that stops an interrupted turn from billing a second call).
|
|
1848
|
-
// The cloud client satisfies it; this transport rejected with the raw
|
|
1849
|
-
// AbortError instead, so pressing Stop on a local turn surfaced in the host
|
|
1850
|
-
// as `Error occurred in handler for 'model:chat': DOMException [AbortError]`
|
|
1851
|
-
// — an error dialog and a stack for the one action the user chose. Nothing
|
|
1852
|
-
// was produced to preserve here: no byte of the response was ever read.
|
|
1853
|
-
if ((signal && signal.aborted) || ctl.signal.aborted) return stoppedTurn(model);
|
|
1854
|
-
throw err;
|
|
1855
|
-
} finally {
|
|
1856
|
-
// Time-to-headers only, the same contract boundedFetch documents: once the
|
|
1857
|
-
// headers are here the per-payload stall timer takes over, so a
|
|
1858
|
-
// legitimately long generation is never cut off by this one.
|
|
1859
|
-
clearTimeout(firstByteTimer);
|
|
1860
|
-
}
|
|
1861
|
-
|
|
1862
|
-
if (!res.ok) {
|
|
1863
|
-
// Body text, not just the status: a local daemon's 400 says which field it
|
|
1864
|
-
// disliked (commonly `tools` on an older build), and the engine retries
|
|
1865
|
-
// without tool schemas on exactly that signal.
|
|
1866
|
-
let text = '';
|
|
1867
|
-
try {
|
|
1868
|
-
text = await res.text();
|
|
1869
|
-
} catch {
|
|
1870
|
-
/* body already gone */
|
|
1871
|
-
}
|
|
1872
|
-
const err = new Error(`local: chat/completions ${res.status}${text ? ` — ${text.slice(0, 400)}` : ''}`);
|
|
1873
|
-
err.status = res.status;
|
|
1874
|
-
throw err;
|
|
1875
|
-
}
|
|
1876
|
-
|
|
1877
|
-
let content = '';
|
|
1878
|
-
let usage = null;
|
|
1879
|
-
const toolCalls = [];
|
|
1880
|
-
const byIndex = new Map();
|
|
1881
|
-
const stripEnvelope = makeDeepSeekEnvelopeStripper();
|
|
1882
|
-
// Text-form calls the templates emit that Ollama's shim does not structure
|
|
1883
|
-
// for us (see makeTextToolCallStripper). Chained AFTER the DSML pass: DSML
|
|
1884
|
-
// parameter bodies are swallowed by that pass, so a file being written can
|
|
1885
|
-
// never be scanned — and misread — as a call. The offered tool names are
|
|
1886
|
-
// handed over as the guard on the bare-JSON rule: only a name this turn
|
|
1887
|
-
// actually offered is allowed to turn text into a call.
|
|
1888
|
-
const stripTextCalls = makeTextToolCallStripper({
|
|
1889
|
-
toolNames: (Array.isArray(tools) ? tools : [])
|
|
1890
|
-
.map((t) => (t && t.function && t.function.name) || (t && t.name) || '')
|
|
1891
|
-
.filter(Boolean),
|
|
1892
|
-
});
|
|
1893
|
-
|
|
1894
|
-
try {
|
|
1895
|
-
await readSSE(
|
|
1896
|
-
res,
|
|
1897
|
-
(json) => {
|
|
1898
|
-
if (json && json.usage) usage = json.usage;
|
|
1899
|
-
const choice = json && Array.isArray(json.choices) ? json.choices[0] : null;
|
|
1900
|
-
if (!choice) return;
|
|
1901
|
-
const delta = choice.delta || {};
|
|
1902
|
-
if (typeof delta.content === 'string' && delta.content) {
|
|
1903
|
-
const clean = stripTextCalls(stripEnvelope(delta.content));
|
|
1904
|
-
if (clean) {
|
|
1905
|
-
content += clean;
|
|
1906
|
-
if (typeof onDelta === 'function') onDelta({ text: clean });
|
|
1907
|
-
}
|
|
1908
|
-
}
|
|
1909
|
-
const frags = delta.tool_calls || [];
|
|
1910
|
-
for (const f of frags) {
|
|
1911
|
-
const i = Number.isInteger(f.index) ? f.index : toolCalls.length;
|
|
1912
|
-
let acc = byIndex.get(i);
|
|
1913
|
-
if (!acc) {
|
|
1914
|
-
acc = { id: '', type: 'function', function: { name: '', arguments: '' } };
|
|
1915
|
-
byIndex.set(i, acc);
|
|
1916
|
-
}
|
|
1917
|
-
if (f.id) acc.id = f.id;
|
|
1918
|
-
if (f.type) acc.type = f.type;
|
|
1919
|
-
const fn = f.function || {};
|
|
1920
|
-
if (fn.name) acc.function.name = fn.name;
|
|
1921
|
-
if (fn.arguments) acc.function.arguments += fn.arguments;
|
|
1922
|
-
}
|
|
1923
|
-
},
|
|
1924
|
-
{ idleTimeoutMs: SSE_IDLE_TIMEOUT_MS }
|
|
1925
|
-
);
|
|
1926
|
-
} catch (err) {
|
|
1927
|
-
// Stopped mid-stream. The forwarded abort tears the body down, so the read
|
|
1928
|
-
// loop rejects with AbortError — resolve with what already streamed
|
|
1929
|
-
// instead of turning the user's own Stop into a host-side error, and let
|
|
1930
|
-
// the tail below stay unreached (an aborted stream has no unfinished
|
|
1931
|
-
// envelope worth flushing into an answer the user cancelled).
|
|
1932
|
-
if (signal && signal.aborted) return stoppedTurn(model, { content, usage });
|
|
1933
|
-
throw err;
|
|
1934
|
-
} finally {
|
|
1935
|
-
if (signal && typeof signal.removeEventListener === 'function') {
|
|
1936
|
-
signal.removeEventListener('abort', onCallerAbort);
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1939
|
-
|
|
1940
|
-
// A trailing '|…' that never closed (stream ended right after it, or it
|
|
1941
|
-
// was real prose all along) is real content, not an unfinished envelope
|
|
1942
|
-
// token — release it rather than dropping it silently.
|
|
1943
|
-
const strayTail = stripTextCalls(stripEnvelope.flush());
|
|
1944
|
-
const textTail = stripTextCalls.flush();
|
|
1945
|
-
const tail = `${strayTail || ''}${textTail || ''}`;
|
|
1946
|
-
if (tail) {
|
|
1947
|
-
content += tail;
|
|
1948
|
-
if (typeof onDelta === 'function') onDelta({ text: tail });
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
for (const [i, acc] of [...byIndex.entries()].sort((a, b) => a[0] - b[0])) {
|
|
1952
|
-
if (!acc.function.name && !acc.function.arguments) continue;
|
|
1953
|
-
// Ollama does not always mint an id; the engine's tool loop keys results
|
|
1954
|
-
// by it, so an absent one is filled rather than left to collapse every
|
|
1955
|
-
// call in the turn onto a single empty key.
|
|
1956
|
-
if (!acc.id) acc.id = `call_${i}`;
|
|
1957
|
-
toolCalls.push(acc);
|
|
1958
|
-
}
|
|
1959
|
-
|
|
1960
|
-
const message = { role: 'assistant', content };
|
|
1961
|
-
if (toolCalls.length) message.tool_calls = toolCalls;
|
|
1962
|
-
|
|
1963
|
-
// DSML calls arrive as plain text, decoded by stripEnvelope rather than
|
|
1964
|
-
// accumulated by index like structured delta.tool_calls — merged into one
|
|
1965
|
-
// normalised `{ id, name, args }` list so a call that arrived as DSML (a
|
|
1966
|
-
// locally-pulled deepseek-* model with no provider-native delta.tool_calls
|
|
1967
|
-
// at all) is executed like any other. engine.js's extractToolCalls() reads
|
|
1968
|
-
// this field first, ahead of message.tool_calls.
|
|
1969
|
-
const dsmlCalls = stripEnvelope.calls();
|
|
1970
|
-
// Calls the model wrote as text rather than in the structured field — the
|
|
1971
|
-
// same merge, because from here on they ARE tool calls: the engine's
|
|
1972
|
-
// extractToolCalls() reads this field first, ahead of message.tool_calls.
|
|
1973
|
-
const textCalls = stripTextCalls.calls();
|
|
1974
|
-
const result = {
|
|
1975
|
-
id: 'local',
|
|
1976
|
-
model: String(model),
|
|
1977
|
-
choices: [{ index: 0, message, finish_reason: toolCalls.length ? 'tool_calls' : 'stop' }],
|
|
1978
|
-
...(usage ? { usage } : {}),
|
|
1979
|
-
};
|
|
1980
|
-
if (toolCalls.length || dsmlCalls.length || textCalls.length) {
|
|
1981
|
-
result.toolCalls = [
|
|
1982
|
-
...toolCalls.map((c) => ({ id: c.id, name: c.function.name, args: c.function.arguments })),
|
|
1983
|
-
...dsmlCalls.map((c) => ({ id: '', name: c.name, args: c.args || {} })),
|
|
1984
|
-
...textCalls.map((c) => ({ id: '', name: c.name, args: c.args || {} })),
|
|
1985
|
-
];
|
|
1986
|
-
if ((dsmlCalls.length || textCalls.length) && result.choices[0].finish_reason !== 'tool_calls') {
|
|
1987
|
-
result.choices[0].finish_reason = 'tool_calls';
|
|
1988
|
-
}
|
|
1989
|
-
}
|
|
1990
|
-
// What the fit cost, reported rather than hidden: the window the daemon
|
|
1991
|
-
// granted, the fixed wire cost the tool schemas took out of it, and how much
|
|
1992
|
-
// history had to be dropped to stay inside. `over` is the honest failure — the
|
|
1993
|
-
// newest turn alone does not fit, so the daemon will shift the prompt instead.
|
|
1994
|
-
if (fit) {
|
|
1995
|
-
result.window = {
|
|
1996
|
-
contextLength: fit.window,
|
|
1997
|
-
dropped: fit.dropped,
|
|
1998
|
-
toolTokens: fit.toolTokens,
|
|
1999
|
-
estimatedTokens: fit.estimatedTokens,
|
|
2000
|
-
reserveTokens: fit.reserveTokens,
|
|
2001
|
-
over: fit.over,
|
|
2002
|
-
};
|
|
2003
|
-
}
|
|
2004
|
-
return result;
|
|
2005
|
-
}
|
|
2006
|
-
|
|
2007
|
-
/**
|
|
2008
|
-
* Public entry point: `chatOne` serialized against every other call that
|
|
2009
|
-
* would land on the same daemon+model slot (see `runSerialized` above). A
|
|
2010
|
-
* caller that wants true concurrency points at a different model or a
|
|
2011
|
-
* different daemon — both get their own key and run in parallel.
|
|
2012
|
-
*
|
|
2013
|
-
* Also tracks how many calls are live for this key (`activeCountByKey`) and,
|
|
2014
|
-
* once this one succeeds, swaps the daemon's warm model over to it — see
|
|
2015
|
-
* `swapWarmModel`. The count is kept here rather than inside `runSerialized`
|
|
2016
|
-
* because it has to span the whole call (queueing included): a model is
|
|
2017
|
-
* still "in use" while queued behind another turn on the same slot, not only
|
|
2018
|
-
* while its own fetch is running.
|
|
2019
|
-
*/
|
|
2020
|
-
async function chat(opts = {}) {
|
|
2021
|
-
const base = baseOf(opts && opts.baseURL);
|
|
2022
|
-
const model = opts && opts.model;
|
|
2023
|
-
const key = `${base}|${model}`;
|
|
2024
|
-
activeCountByKey.set(key, (activeCountByKey.get(key) || 0) + 1);
|
|
2025
|
-
try {
|
|
2026
|
-
// The pin runs INSIDE the same slot as the turn it warms, deliberately.
|
|
2027
|
-
// Two reasons, both load-bearing:
|
|
2028
|
-
// * the daemon's single request slot is per model (see `runSerialized`),
|
|
2029
|
-
// so a pin issued beside the turn rather than before it would be the
|
|
2030
|
-
// second overlapping request that hangs the loser;
|
|
2031
|
-
// * a pin that warmed the prefix and then released the slot, letting
|
|
2032
|
-
// another call's model in, would warm a cache that is gone by the time
|
|
2033
|
-
// this turn sends its first byte.
|
|
2034
|
-
// `pin` is held in the closure so the turn's own error path is untouched:
|
|
2035
|
-
// a failed pin never becomes a failed turn.
|
|
2036
|
-
let pin = null;
|
|
2037
|
-
const result = await runSerialized(key, async () => {
|
|
2038
|
-
pin = await ensureResident({
|
|
2039
|
-
baseURL: opts && opts.baseURL,
|
|
2040
|
-
model,
|
|
2041
|
-
// The turn's OWN system text — the engine hands it down as `system`
|
|
2042
|
-
// and `chatOne` sends it as the first row, so warming it is warming
|
|
2043
|
-
// the exact prefix this turn will re-send.
|
|
2044
|
-
system: opts && opts.system,
|
|
2045
|
-
signal: opts && opts.signal,
|
|
2046
|
-
warmSecs: opts && opts.warmSecs,
|
|
2047
|
-
// The turn's OWN first-byte bound, so a daemon that accepts the socket
|
|
2048
|
-
// and never answers fails the turn on the CALLER's clock. Without it
|
|
2049
|
-
// the pin sat on a budget of its own and the turn's 1s deadline was not
|
|
2050
|
-
// even armed yet — the pin silently re-armed the hang it exists to
|
|
2051
|
-
// prevent.
|
|
2052
|
-
firstByteTimeoutMs: opts && opts.firstByteTimeoutMs,
|
|
2053
|
-
});
|
|
2054
|
-
// The turn is fitted to the window the pin READ, never to one requested —
|
|
2055
|
-
// see `fitToWindowOf` for the measurement that closed that door.
|
|
2056
|
-
//
|
|
2057
|
-
// Read directly ONLY when the pin did not answer the residency question:
|
|
2058
|
-
// * a pin that SUCCEEDED is the answer, and a memo hit especially so —
|
|
2059
|
-
// that path's whole contract is "no daemon contact and no slot", so
|
|
2060
|
-
// re-reading here would turn a free turn into a dialing one;
|
|
2061
|
-
// * a remote URL is never dialed from this lane at all (fail-closed, the
|
|
2062
|
-
// rule `remoteRefusal` enforces), and a helper that dialed where the
|
|
2063
|
-
// turn may not would be the hole in it. `ensureResident` states the
|
|
2064
|
-
// same rule for the same reason.
|
|
2065
|
-
// Everything else gets one cheap local GET, because the size of the window
|
|
2066
|
-
// is a fact about the daemon rather than about the warm-up having worked:
|
|
2067
|
-
// a pin that was ATTEMPTED and came back empty-handed (its own clock, an
|
|
2068
|
-
// error, a daemon that would not enumerate) still wants the turn fitted, and
|
|
2069
|
-
// it already dialed, so one more read costs nothing it has not already
|
|
2070
|
-
// spent.
|
|
2071
|
-
//
|
|
2072
|
-
// Two things deliberately do NOT qualify, and both are existing contracts
|
|
2073
|
-
// this lane must not quietly widen:
|
|
2074
|
-
// * a pin that was SKIPPED — `warm-disabled` especially, where "nothing to
|
|
2075
|
-
// check residency for, so not even the GET is issued" is asserted in
|
|
2076
|
-
// test/local-resident.test.mjs. Trading that for a fitted turn would be
|
|
2077
|
-
// a real change to a documented behaviour for the benefit of an opt-out
|
|
2078
|
-
// configuration, so the skip stands and a warm-disabled turn is sent
|
|
2079
|
-
// unfitted. The default is warm ON (DEFAULT_WARM_SECS), which is where
|
|
2080
|
-
// the fitting happens.
|
|
2081
|
-
// * a turn that will not be sent at all. The refusals `chatOne` raises
|
|
2082
|
-
// BEFORE dialing (an empty model, a remote URL) must not be preceded by
|
|
2083
|
-
// a request of ours, or "refused before dialing" stops being true.
|
|
2084
|
-
let contextLength = (pin && pin.contextLength) || null;
|
|
2085
|
-
const willSend = isLocalEndpoint(base) && String(model || '').trim() !== '';
|
|
2086
|
-
if (!contextLength && willSend && pin && pin.ok !== true && !pin.skipped) {
|
|
2087
|
-
contextLength = (await windowOf(base, model, { signal: opts && opts.signal })) || null;
|
|
2088
|
-
}
|
|
2089
|
-
return chatOne({ ...opts, contextLength });
|
|
2090
|
-
});
|
|
2091
|
-
swapWarmModel(base, model);
|
|
2092
|
-
// What the extend cost, reported rather than discarded — the daemon's own
|
|
2093
|
-
// `prompt_eval_count` for the prefill it did. Deliberately NOT folded into
|
|
2094
|
-
// `usage`: the pin is not part of the turn's token bill, and adding it
|
|
2095
|
-
// would report tokens the user's turn never spent.
|
|
2096
|
-
if (pin && typeof result === 'object' && result) result.residency = pin;
|
|
2097
|
-
return result;
|
|
2098
|
-
} finally {
|
|
2099
|
-
const remaining = (activeCountByKey.get(key) || 1) - 1;
|
|
2100
|
-
if (remaining <= 0) activeCountByKey.delete(key);
|
|
2101
|
-
else activeCountByKey.set(key, remaining);
|
|
2102
|
-
}
|
|
2103
|
-
}
|
|
2104
|
-
|
|
2105
|
-
module.exports = {
|
|
2106
|
-
DEFAULT_BASE,
|
|
2107
|
-
DEFAULT_WARM_SECS,
|
|
2108
|
-
RESIDENT_TIMEOUT_MS,
|
|
2109
|
-
SSE_IDLE_TIMEOUT_MS,
|
|
2110
|
-
FIRST_BYTE_TIMEOUT_MS,
|
|
2111
|
-
PROBE_TIMEOUT_MS,
|
|
2112
|
-
baseOf,
|
|
2113
|
-
normalizeBase,
|
|
2114
|
-
isLocalEndpoint,
|
|
2115
|
-
remoteRefusal,
|
|
2116
|
-
probe,
|
|
2117
|
-
listTags,
|
|
2118
|
-
readSSE,
|
|
2119
|
-
chat,
|
|
2120
|
-
ensureResident,
|
|
2121
|
-
residentSecsLeft,
|
|
2122
|
-
residentEntry,
|
|
2123
|
-
unload,
|
|
2124
|
-
unloadAll,
|
|
2125
|
-
// Exported for this transport's own tests. The text-form tool-call decoders
|
|
2126
|
-
// are pure, and the only honest way to test a streaming decoder is chunk by
|
|
2127
|
-
// chunk — see test/local-tool-text.test.mjs.
|
|
2128
|
-
makeTextToolCallStripper,
|
|
2129
|
-
makeDeepSeekEnvelopeStripper,
|
|
2130
|
-
textCallsIn,
|
|
2131
|
-
// The window fit, exported for the same reason: it decides what leaves this
|
|
2132
|
-
// machine, and its arithmetic (fixed tool cost charged before history, reserve
|
|
2133
|
-
// for the reply, never a guessed window) is worth asserting rather than
|
|
2134
|
-
// eyeballing.
|
|
2135
|
-
fitToWindowOf,
|
|
2136
|
-
// The runtime's own response-header deadline, translated into this lane's
|
|
2137
|
-
// timeout shape. Exported because the shape it produces is the engine's
|
|
2138
|
-
// contract, and the real error it recognises is a `fetch failed` wrapper —
|
|
2139
|
-
// worth asserting on the wrapper rather than on a string.
|
|
2140
|
-
isHeadersDeadline,
|
|
2141
|
-
HEADERS_DEADLINE_MS,
|
|
2142
|
-
windowOf,
|
|
2143
|
-
// Exported so a test can assert the STOP path's contract (a stopped turn
|
|
2144
|
-
// resolves; it never rejects) against the real helper rather than against a
|
|
2145
|
-
// copy of its shape.
|
|
2146
|
-
stoppedTurn,
|
|
2147
|
-
};
|
|
3
|
+
"use strict";const{estimateTokens:estimateTokens,trimContext:trimContext}=require("./context.js"),attachmentsModule=require("./attachments.js"),DEFAULT_BASE="http://localhost:11434",SSE_IDLE_TIMEOUT_MS=3e5,FIRST_BYTE_TIMEOUT_MS=SSE_IDLE_TIMEOUT_MS,HEADERS_DEADLINE_MS=3e5;function isHeadersDeadline(e){if(!e)return!1;const t=e.cause||e;if(!t)return!1;const n=String(t.code||""),s=String(t.name||"");return"UND_ERR_HEADERS_TIMEOUT"===n||"HeadersTimeoutError"===s}const PROBE_TIMEOUT_MS=1500;function normalizeBase(e){const t=String(null==e?"":e).trim();return t?/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(t)?t:`http://${t}`:DEFAULT_BASE}function baseOf(e){return normalizeBase(e).replace(/\/+$/,"")}const DSML_RE_INVOKE=/<\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call)\b\s*([^>]*?)\s*>([\s\S]*?)<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call)\b\s*>/g,DSML_RE_PARAM=/<\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*([^>]*?)\s*>([\s\S]*?)<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*>/g,DSML_RE_LEADER=/<\s*\/?\s*|\s*(?:DSML\s*)?|\s*(?:(?:tool|function)[_▁ ]*calls?|reasoning)[_▁ ]*(?:begin|end)?\s*>/gi,DSML_RE_ANY_TAG=/<\s*\/?\s*|\s*(?:DSML\s*)?|\s*[A-Za-z_][A-Za-z0-9_]*[^>]{0,120}>/g,DSML_RE_SKELETON=/<\s*\/?\s*(?:tool_calls|function_calls|invoke|parameter|param|reasoning)\b[^>]{0,120}>/gi,DSML_RE_JSON_CALLS=/|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*begin[ \t]*|([\s\S]*?)|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*end[ \t]*|/gi,DSML_RE_JSON_CALLS_OPEN=/|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*begin[ \t]*|/gi,DSML_RE_JSON_CALLS_CLOSE=/|[ \t]*(?:tool|function)[_▁ ]*calls?[_▁ ]*end[ \t]*|/gi,DSML_RE_ENVELOPE_TOKEN=/|[ \t]?[A-Za-z0-9_▁]*[ \t]?|/g,DSML_RE_DRIFT_INVOKE=/<\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call)\b\s*([^>]*?)\s*>/gi,DSML_RE_DRIFT_PARAM=/<\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*([^>]*?)\s*>/gi,DSML_RE_PARAM_CLOSE=/<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:parameter|param|arg)\b\s*>/gi,DSML_RE_ATTR=/([A-Za-z_][A-Za-z0-9_.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g,DSML_RE_STRUCT_TAG=/<\s*\/?\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call|parameter|param|arg|reasoning|(?:tool|function)[_▁ ]*calls?)\b/gi,DSML_RE_STRUCT_CLOSER=/<\s*\/\s*|\s*(?:DSML\s*)?|\s*(?:invoke|call|tool_call|parameter|param|arg|reasoning|(?:tool|function)[_▁ ]*calls?)\b/gi,DSML_RE_WRAPPER_TAG=/<\s*\/?\s*|\s*(?:DSML\s*)?|\s*(?!(?:invoke|call|tool_call|parameter|param|arg)\b)[A-Za-z_][A-Za-z0-9_]*[^>]{0,120}>/g,DSML_RE_TRAILING_LT=/<\s*\/?\s*$/,DSML_RE_OPEN_TOKEN=/^|[ \t]?[A-Za-z0-9_▁]*[ \t]?$/,DSML_HOLD_MAX=4194304;function dsmlAttrs(e){const t={};if(!e)return t;let n;for(DSML_RE_ATTR.lastIndex=0;n=DSML_RE_ATTR.exec(e);)t[n[1].toLowerCase()]=null!=n[2]?n[2]:n[3];return t}function dsmlJson(e){const t=String(null==e?"":e).trim();try{const e=JSON.parse(t);return void 0===e?t:e}catch{return t}}function dsmlArgs(e){const t={};let n;for(DSML_RE_PARAM.lastIndex=0;n=DSML_RE_PARAM.exec(e);){const e=dsmlAttrs(n[1]),s=e.name||e.key||`arg${Object.keys(t).length}`,o="false"===String(e.string||"").toLowerCase()||"true"===String(e.json||"").toLowerCase();t[s]=o?dsmlJson(n[2]):n[2]}return t}function decodeDeepSeekDSML(e){if("string"!=typeof e||!e||-1===e.indexOf("|"))return{text:e||"",calls:[]};const t=[];let n=e;DSML_RE_INVOKE.lastIndex=0;const s=[];let o;for(;o=DSML_RE_INVOKE.exec(n);){const e=dsmlAttrs(o[1]),n=e.name||e.function||e.tool||"";n&&t.push({name:n,args:dsmlArgs(o[2])}),s.push([o.index,o.index+o[0].length])}for(let e=s.length-1;e>=0;e--)n=n.slice(0,s[e][0])+n.slice(s[e][1]);return n=n.replace(DSML_RE_JSON_CALLS,((e,n)=>{const s=dsmlJson(n),o=Array.isArray(s)?s:s&&Array.isArray(s.tool_calls)?s.tool_calls:[];for(const e of o){const n=e&&e.function||e||{},s=n.name||e&&e.name||"";if(!s)continue;let o=null!=n.arguments?n.arguments:e&&e.arguments;"string"==typeof o&&(o=dsmlJson(o)),t.push({name:s,args:o&&"object"==typeof o?o:{}})}return""})),n=n.replace(DSML_RE_LEADER,""),{text:n,calls:t}}function decodeDeepSeekDSMLDrift(e){if("string"!=typeof e||!e||-1===e.indexOf("|"))return{text:e||"",calls:[]};const t=[];let n;for(DSML_RE_DRIFT_INVOKE.lastIndex=0;n=DSML_RE_DRIFT_INVOKE.exec(e);)t.push({at:n.index,attrs:n[1],body:DSML_RE_DRIFT_INVOKE.lastIndex});if(!t.length)return{text:e,calls:[]};const s=[];let o="",r=0;for(let n=0;n<t.length;n++){const a=n+1<t.length?t[n+1].at:e.length,l=e.slice(t[n].body,a),i=dsmlAttrs(t[n].attrs),c=i.name||i.function||i.tool||"",u={},f=[];let _;for(DSML_RE_DRIFT_PARAM.lastIndex=0;_=DSML_RE_DRIFT_PARAM.exec(l);)f.push({at:_.index,attrs:_[1],from:DSML_RE_DRIFT_PARAM.lastIndex});for(let e=0;e<f.length;e++){const t=e+1<f.length?f[e+1].at:l.length,n=l.slice(f[e].from,t).replace(DSML_RE_PARAM_CLOSE,""),s=dsmlAttrs(f[e].attrs),o=s.name||s.key||`arg${Object.keys(u).length}`,r="false"===String(s.string||"").toLowerCase();u[o]=r?dsmlJson(n):n}o+=e.slice(r,t[n].at),r=a,c&&s.push({name:c,args:u})}return o+=e.slice(r),{text:o,calls:s}}function dsmlResidue(e){return e?-1===e.indexOf("|")?e.replace(DSML_RE_SKELETON,""):e.replace(DSML_RE_ANY_TAG,"").replace(DSML_RE_SKELETON,"").replace(DSML_RE_ENVELOPE_TOKEN,""):e}function dsmlUnterminated(e){return!!DSML_RE_OPEN_TOKEN.test(e)||(e.match(DSML_RE_STRUCT_TAG)||[]).length>(e.match(DSML_RE_STRUCT_CLOSER)||[]).length}function dsmlHoldFrom(e){const t=e.search(/<\s*\/?\s*|/);if(-1!==t)return t;const n=e.search(DSML_RE_JSON_CALLS_OPEN);if(-1!==n&&-1===e.slice(n).search(DSML_RE_JSON_CALLS_CLOSE))return n;const s=DSML_RE_TRAILING_LT.exec(e);if(s)return s.index;const o=e.indexOf("|");return-1!==o&&dsmlUnterminated(e.slice(o))?o:-1}function decodePlainToolCallTags(e,t){if(!e||-1===e.indexOf("<"))return e;let n="",s=0;for(;;){const o=TEXT_TOOL_TAG_OPEN_RE.exec(e.slice(s));if(!o)break;const r=s+o.index,a=o[0];if(a.includes(DSML_TAG_MARK)){n+=e.slice(s,r+a.length),s=r+a.length;continue}const l=e.slice(r+a.length),i=TEXT_TOOL_TAG_CLOSE_RE.exec(l);if(!i)break;const c=textCallsIn(l.slice(0,i.index));if(n+=e.slice(s,r),s=r+a.length+i.index+i[0].length,c.length)for(const e of c)t.push(e);else n+=e.slice(r,s)}return n+e.slice(s)}function plainToolTagHoldAt(e){if(!e||-1===e.indexOf("<"))return-1;let t=0;for(;;){const n=TEXT_TOOL_TAG_OPEN_RE.exec(e.slice(t));if(!n)return-1;const s=t+n.index;if(n[0].includes(DSML_TAG_MARK)){t=s+n[0].length;continue}const o=e.slice(s+n[0].length);return TEXT_TOOL_TAG_CLOSE_RE.test(o)?-1:s}}function makeDeepSeekEnvelopeStripper(){let e="";const t=[],n=e=>dsmlResidue(decodePlainToolCallTags(e,t)),s=s=>{e+="string"==typeof s?s:"";const o=decodeDeepSeekDSML(e);e=o.text;for(const e of o.calls)t.push(e);e=e.replace(DSML_RE_WRAPPER_TAG,"");const r=dsmlHoldFrom(e);let a;-1!==r&&e.length-r<=DSML_HOLD_MAX?(a=e.slice(0,r),e=e.slice(r)):(a=e,e="");const l=plainToolTagHoldAt(a);return-1!==l&&(e=a.slice(l)+e,a=a.slice(0,l)),n(a)};return s.flush=()=>{const s=decodeDeepSeekDSML(e);e="";for(const e of s.calls)t.push(e);const o=decodeDeepSeekDSMLDrift(s.text);for(const e of o.calls)t.push(e);return n(o.text)},s.calls=()=>t.slice(),s}const TEXT_TOOL_TAG_OPEN_RE=/<\s*tool_calls?\b[^>]{0,80}>/i,TEXT_TOOL_TAG_CLOSE_RE=/<\s*\/\s*tool_calls?\s*>/i,TEXT_FENCE_RE=/```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?([\s\S]*?)```/,TEXT_FENCE_OPEN_RE=/```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?/,DSML_TAG_MARK="|";function asTextToolCall(e){if(!e||"object"!=typeof e)return null;let t=e;t.function&&"object"==typeof t.function&&(t={...t.function,name:t.function.name||t.name});const n=t.name||t.tool||t.tool_name||t.function_name,s="string"==typeof n?n.trim():"";if(!s)return null;let o=null!=t.arguments?t.arguments:null!=t.parameters?t.parameters:null!=t.args?t.args:null!=t.tool_input?t.tool_input:{};if("string"==typeof o){const e=o.trim()?safeJson(o):{};o=e&&"object"==typeof e?e:{}}return null!=o&&"object"==typeof o||(o={}),{name:s,args:o}}function safeJson(e){try{return JSON.parse(e)}catch{return null}}function textCallsIn(e){const t=String(e||"").trim().replace(/^json\s*/i,"").trim();if(!t)return[];const n=safeJson(t);if(!n)return[];const s=Array.isArray(n)?n:[n],o=[];for(const e of s){const t=asTextToolCall(e);t&&o.push(t)}return o}function nextTextToolEnvelope(e){const t=TEXT_FENCE_RE.exec(e);if(t)return{start:t.index,end:t.index+t[0].length,complete:!0,raw:t[0],body:t[1]};const n=TEXT_FENCE_OPEN_RE.exec(e);return n?{start:n.index,end:e.length,complete:!1,body:e.slice(n.index+n[0].length)}:null}function trailingHeadFrom(e,t=!1){if(!e)return-1;const n=e.search(TEXT_TOOL_HEAD_RE);if(-1!==n)return n;const s=/`{1,2}$/.exec(e);if(s)return s.index;const o=e.lastIndexOf("<");if(-1!==o&&couldBeTextToolHead(e.slice(o)))return o;if(t){const t=e.lastIndexOf("{");if(-1!==t&&/^\{\s*(?:"[^"]*"\s*:[\s\S]*|"[^"]*"?\s*|)$/.test(e.slice(t)))return t}return-1}function couldBeTextToolHead(e){const t=e.replace(/^[\s\uFEFF]+/,"");if(!t)return!0;if(t.length<=3&&"```".startsWith(t))return!0;if(t.startsWith("<")&&!TEXT_TOOL_HEAD_RE.test(t)){const e=t.slice(1).replace(/^\/?\s*/,"").toLowerCase();if(e.length<=10&&/^[a-z_]*$/.test(e))return"tool_call".startsWith(e)||"tool_calls".startsWith(e)}return!1}function balancedJsonEnd(e,t){let n=0,s=!1,o=!1;for(let r=t;r<e.length;r++){const t=e[r];if(s)o?o=!1:"\\"===t?o=!0:'"'===t&&(s=!1);else if('"'===t)s=!0;else if("{"===t||"["===t)n++;else if(("}"===t||"]"===t)&&(n--,n<=0))return r+1}return-1}function makeTextToolCallStripper({toolNames:e=null}={}){const t=e?new Set([...e].map((e=>String(e)))):null;let n="";const s=[],o=e=>Boolean(e.length)&&(!t||e.every((e=>t.has(e.name)))),r=(e,t)=>{const n=textCallsIn(t);if(!o(n))return e;for(const e of n)s.push(e);return""},a=e=>{n+="string"==typeof e?e:"";let a="",l=!1;for(;;){const e=nextTextToolEnvelope(n);if(e){if(a+=n.slice(0,e.start),n=n.slice(e.start),!e.complete){l=!0;break}a+=r(e.raw,e.body),n=n.slice(e.raw.length);continue}const i=TEXT_TOOL_TAG_OPEN_RE.exec(n);if(i){a+=n.slice(0,i.index),n=n.slice(i.index);const e=n.slice(i[0].length),t=TEXT_TOOL_TAG_CLOSE_RE.exec(e);if(!t){l=!0;break}const s=n.slice(0,i[0].length+t.index+t[0].length);a+=r(s,e.slice(0,t.index)),n=n.slice(s.length);continue}const c=t?/\{\s*"/.exec(n):null;if(!c)break;const u=balancedJsonEnd(n,c.index),f=n.slice(0,c.index);if(-1===u){a+=f,n=n.slice(c.index),l=!0;break}const _=textCallsIn(n.slice(c.index,u));if(a+=f,o(_))for(const e of _)s.push(e);else a+=n.slice(c.index,u);n=n.slice(u)}const i=l?0:trailingHeadFrom(n,Boolean(t));return-1===i?(a+=n,n=""):i>0&&(a+=n.slice(0,i),n=n.slice(i)),n.length>TEXT_TOOL_HOLD_MAX&&(a+=n,n=""),a};return a.flush=()=>{const e=n;if(n="",!e)return"";const s=nextTextToolEnvelope(e);if(s)return e.slice(0,s.start)+r(e.slice(s.start),s.body);const o=TEXT_TOOL_TAG_OPEN_RE.exec(e);if(!o){const n=t?/\{\s*"/.exec(e):null;if(!n)return e;const s=balancedJsonEnd(e,n.index);return-1===s?e:e.slice(0,n.index)+r(e.slice(n.index,s),e.slice(n.index,s))+e.slice(s)}const a=e.slice(o.index+o[0].length).replace(TEXT_TOOL_TAG_CLOSE_RE,"");return e.slice(0,o.index)+r(e.slice(o.index),a)},a.calls=()=>s.slice(),a}const TEXT_TOOL_HEAD_RE=/```|<\s*tool_calls?\b/i,TEXT_TOOL_HOLD_MAX=65536;function isLocalEndpoint(e){let t="",n="";if(""===String(null==e?"":e).trim())return!1;try{const s=new URL(normalizeBase(e));t=s.hostname.toLowerCase(),n=s.protocol.toLowerCase()}catch{return!1}if("http:"!==n&&"https:"!==n)return!1;if(!t)return!1;if(t.startsWith("[")&&t.endsWith("]")&&(t=t.slice(1,-1)),!t||/[\s/\\@]/.test(t))return!1;if("localhost"===t||t.endsWith(".localhost"))return!0;if(/\.(local|internal|lan)$/.test(t))return!0;if(t.includes(":"))return"::1"===t||"0:0:0:0:0:0:0:1"===t||!!/^f[cd][0-9a-f]{2}:/.test(t)||!!/^fe[89ab][0-9a-f]:/.test(t);const s=/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(t);if(s){const e=s.slice(1).map(Number);if(e.some((e=>e>255)))return!1;const[t,n]=e;return 127===t||10===t||172===t&&n>=16&&n<=31||192===t&&168===n||169===t&&254===n}return!t.includes(".")}function remoteRefusal(e){if(isLocalEndpoint(e))return null;const t=new Error(`local: ${JSON.stringify(String(e||""))} is not on this machine, so it is not the local class. Running a model you own is free because there is no vendor to pay — a remote URL here would be an unpaid turn instead. Use the aegis or byok class for remote models.`);return t.status=400,t}async function probe(e=DEFAULT_BASE){const t=baseOf(e);try{return{running:(await fetch(`${t}/api/tags`,{signal:AbortSignal.timeout(1500)})).ok,baseURL:t}}catch{return{running:!1,baseURL:t}}}async function listTags(e=DEFAULT_BASE){const t=await fetch(`${baseOf(e)}/api/tags`);if(!t.ok){const e=new Error(`local: /api/tags answered ${t.status}`);throw e.status=t.status,e}const n=await t.json();return(n&&n.models||[]).map((e=>({id:e&&e.name,...e&&e.details?{details:e.details}:{}}))).filter((e=>e.id))}const UNLOAD_TIMEOUT_MS=4e3,RESIDENT_TIMEOUT_MS=FIRST_BYTE_TIMEOUT_MS,DEFAULT_WARM_SECS=1800;async function unload(e,t){if(!t)return{ok:!1};const n=baseOf(e);if(!isLocalEndpoint(n))return{ok:!1};try{return{ok:(await fetch(`${n}/api/chat`,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({model:String(t),messages:[],stream:!1,keep_alive:0}),signal:AbortSignal.timeout(UNLOAD_TIMEOUT_MS)})).ok}}catch{return{ok:!1}}}function normalizeRow(e){if(!e||"object"!=typeof e)return null;const t={role:e.role||"user"};return null!=e.content&&(t.content=e.content),e.name&&(t.name=e.name),e.tool_call_id&&(t.tool_call_id=e.tool_call_id),Array.isArray(e.tool_calls)&&e.tool_calls.length&&(t.tool_calls=e.tool_calls),t}function parseEvent(e){const t=String(e).split("\n");let n="",s=!1;for(const e of t){if(!e||e.startsWith(":"))continue;const t=e.indexOf(":"),o=-1===t?e:e.slice(0,t),r=-1===t?"":e.slice(t+1).replace(/^ /,"");"data"===o?(n+=r,s=!0):"event"!==o&&"id"!==o&&"retry"!==o||(s=!0)}if(!s||!n)return null;if("[DONE]"===n)return{done:!0};try{return{json:JSON.parse(n)}}catch{return null}}async function readSSE(e,t,{idleTimeoutMs:n=SSE_IDLE_TIMEOUT_MS}={}){const s=e.body.getReader(),o=new TextDecoder;let r="",a=null,l=()=>{};const i=new Promise(((e,t)=>{l=t})),c=()=>{a&&clearTimeout(a),a=setTimeout((()=>{l(new Error(`local: the model stopped responding — no output for ${Math.round(n/1e3)}s`))}),n)};c();try{for(;;){const e=await Promise.race([s.read(),i]);if(e.done)break;let n;for(r+=o.decode(e.value,{stream:!0}),r=r.replace(/\r\n/g,"\n");(n=r.indexOf("\n\n"))>=0;){const e=r.slice(0,n);r=r.slice(n+2);const s=parseEvent(e);if(s){if(s.done)return;s.json&&(c(),t(s.json))}}}}finally{a&&clearTimeout(a);try{await s.cancel()}catch{}}}const localCallQueues=new Map;function runSerialized(e,t){const n=(localCallQueues.get(e)||Promise.resolve()).then(t,t);return localCallQueues.set(e,n.then((()=>{}),(()=>{}))),n}const activeCountByKey=new Map,warmModelByBase=new Map;function swapWarmModel(e,t){if(!t)return;const n=warmModelByBase.get(e);warmModelByBase.set(e,t),n&&n!==t&&(activeCountByKey.get(`${e}|${n}`)||unload(e,n).catch((()=>{})))}async function unloadAll(){const e=[...warmModelByBase.entries()];warmModelByBase.clear(),await Promise.all(e.map((([e,t])=>unload(e,t))))}function warmSecsFor({warmSecs:e}={}){const t=Number(e);if(Number.isFinite(t)&&t>=0)return Math.floor(t);const n=Number(process.env&&process.env.AEGIS_LOCAL_WARM_SECS);return Number.isFinite(n)&&n>=0?Math.floor(n):DEFAULT_WARM_SECS}const residentMemo=new Map;function residentKey(e,t,n){return`${e}\0${t}\0${n}`}function residentSecsLeft(e,t,n,{dialer:s=fetch,now:o=Date.now()}={}){const r=residentMemo.get(residentKey(e,t,n));if(!r||r.dialer!==s)return 0;const a=Math.round((r.until-o)/1e3);return a<=0?(residentMemo.delete(residentKey(e,t,n)),0):a}function residentEntry(e,t,n,{dialer:s=fetch,now:o=Date.now()}={}){const r=residentMemo.get(residentKey(e,t,n));return r&&r.dialer===s&&r.until-o>0?r:null}function sameModel(e,t){const n=String(e||""),s=String(t||"");return!(!n||!s)&&(n===s||n.startsWith(`${s}:`)||n.startsWith(`${s}@`))}const WINDOW_TIMEOUT_MS=3e3;async function windowOf(e,t,{signal:n}={}){const s=new AbortController,o=()=>s.abort();if(n){if(n.aborted)return null;"function"==typeof n.addEventListener&&n.addEventListener("abort",o,{once:!0})}const r=setTimeout((()=>s.abort()),WINDOW_TIMEOUT_MS);r&&"function"==typeof r.unref&&r.unref();try{const n=await fetch(`${e}/api/ps`,{method:"GET",signal:s.signal});if(!n||!n.ok||"function"!=typeof n.json)return null;const o=await n.json(),r=(o&&o.models||[]).find((e=>sameModel(e&&(e.name||e.model),t))),a=r&&Number(r.context_length);return Number.isFinite(a)&&a>0?a:null}catch{return null}finally{clearTimeout(r),n&&"function"==typeof n.removeEventListener&&n.removeEventListener("abort",o)}}async function ensureResident({baseURL:e=DEFAULT_BASE,model:t,system:n="",signal:s,warmSecs:o,firstByteTimeoutMs:r}={}){const a=baseOf(e);if(!isLocalEndpoint(a))return{ok:!1,skipped:"remote"};if(!t||!String(t).trim())return{ok:!1,skipped:"no-model"};const l=warmSecsFor({warmSecs:o}),i="string"==typeof n?n:"",c={warmSecs:l,prefixSource:i?"turn-system":"none",prefixChars:i.length};if(l<=0)return{ok:!1,skipped:"warm-disabled",...c};const u=fetch,f=residentSecsLeft(a,t,i,{dialer:u});if(f>=Math.min(l,60)){const e=residentEntry(a,t,i,{dialer:u});return{ok:!0,already:!0,memo:!0,remainingSecs:f,contextLength:e&&e.contextLength||null,...c}}const _=new AbortController,d=()=>_.abort();s&&(s.aborted?_.abort():"function"==typeof s.addEventListener&&s.addEventListener("abort",d,{once:!0}));const m=Number(r),E=Number.isFinite(m)&&m>0?Math.min(m,RESIDENT_TIMEOUT_MS):RESIDENT_TIMEOUT_MS;let T=!1,p=null;const S=setTimeout((()=>{T=!0,_.abort()}),E);S&&"function"==typeof S.unref&&S.unref();try{try{const e=await fetch(`${a}/api/ps`,{method:"GET",signal:_.signal});if(e&&e.ok&&"function"==typeof e.json){const n=await e.json(),s=(n&&n.models||[]).find((e=>sameModel(e&&(e.name||e.model),t)));if(s&&Number.isFinite(s.context_length)&&s.context_length>0&&(p=s.context_length),s&&s.expires_at){const e=Math.round((new Date(s.expires_at).getTime()-Date.now())/1e3);if(Number.isFinite(e)&&e>=Math.min(l,60))return residentMemo.set(residentKey(a,t,i),{until:Date.now()+1e3*e,dialer:u,contextLength:p}),{ok:!0,already:!0,remainingSecs:e,contextLength:p,...c}}}}catch{}if(_.signal.aborted)return{ok:!1,aborted:!0,timeout:T,contextLength:p,...c};const e=i?[{role:"system",content:i},{role:"user",content:"PONG"}]:[{role:"user",content:"PONG"}],n=await fetch(`${a}/api/chat`,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify({model:String(t),keep_alive:l,stream:!1,messages:e,options:{num_predict:1}}),signal:_.signal});let s=null,o=null;if(n&&"function"==typeof n.text){const e=await n.text().catch((()=>""));try{const t=JSON.parse(e);Number.isFinite(t.prompt_eval_count)&&(s=t.prompt_eval_count),Number.isFinite(t.eval_count)&&(o=t.eval_count)}catch{}}return n&&n.ok&&(null==p&&(p=await windowOf(a,t,{signal:_.signal})||null),residentMemo.set(residentKey(a,t,i),{until:Date.now()+1e3*l,dialer:u,contextLength:p}),swapWarmModel(a,t)),{ok:!(!n||!n.ok),status:n&&n.status,pinned:!0,promptTokens:s,outputTokens:o,contextLength:p,...c}}catch(e){return{ok:!1,error:String(e&&e.message||e),timeout:T,contextLength:p,...c}}finally{clearTimeout(S),s&&"function"==typeof s.removeEventListener&&s.removeEventListener("abort",d)}}function stoppedTurn(e,{content:t="",usage:n=null}={}){const s={id:"local",model:String(e),choices:[{index:0,message:{role:"assistant",content:t},finish_reason:"stop"}]};return n&&(s.usage=n),s}function fitToWindowOf({rows:e,system:t,tools:n,contextLength:s,maxTokens:o}){const r=Number(s);if(!Number.isFinite(r)||r<=0)return null;const a=(Array.isArray(e)?e:[]).filter((e=>e&&"system"!==e.role)),l=Array.isArray(n)&&n.length?estimateTokens(JSON.stringify(n)):0,i=Number(o),c=Number.isFinite(i)&&i>0?i:Math.round(r/4),u=trimContext({messages:a,system:"string"==typeof t?t:"",budgetTokens:r,reserveTokens:c,fixedTokens:l});return{window:r,reserveTokens:c,toolTokens:l,dropped:u.dropped,messages:u.messages,over:u.estimatedTokens>Math.max(0,r-c),estimatedTokens:u.estimatedTokens}}async function chatOne({baseURL:e=DEFAULT_BASE,model:t,messages:n,system:s,prompt:o,maxTokens:r,temperature:a,tools:l,toolChoice:i,signal:c,onDelta:u,firstByteTimeoutMs:f=FIRST_BYTE_TIMEOUT_MS,contextLength:_}={}){const d=baseOf(e),m=remoteRefusal(d);if(m)throw m;if(!t||!String(t).trim()){const e=new Error("local: no model selected — pick one from the model list, or type a tag");throw e.status=400,e}const E=[];s&&E.push({role:"system",content:s});for(const e of Array.isArray(n)?n:[]){if(e&&"system"===e.role)continue;const t=normalizeRow(e);t&&E.push(t)}const T="string"==typeof o||Array.isArray(o)?o:"";if(!(""===T||Array.isArray(T)&&0===T.length)){const e=E[E.length-1],t=attachmentsModule.flattenContent(T),n=e?attachmentsModule.flattenContent(e.content):null;e&&"user"===e.role&&n===t||E.push({role:"user",content:T})}if(!E.length){const e=new Error("local: nothing to send — no prompt and no messages");throw e.status=400,e}let p=null;if(null!=_){const e=E[0]&&"system"===E[0].role?E[0]:null;if(p=fitToWindowOf({rows:E,system:e&&e.content||s||"",tools:l,contextLength:_,maxTokens:r}),p){const t=p.messages;E.length=0,e&&E.push(e),E.push(...t)}}const S={model:String(t),messages:E,stream:!0,stream_options:{include_usage:!0},...r?{max_tokens:r}:{},...null!=a?{temperature:a}:{},...l&&l.length?{tools:l}:{},...i?{tool_choice:i}:{}},g=new AbortController;let h=!1;const L=setTimeout((()=>{h=!0,g.abort()}),f);L&&"function"==typeof L.unref&&L.unref();const M=()=>g.abort();let y;c&&(c.aborted?g.abort():"function"==typeof c.addEventListener&&c.addEventListener("abort",M,{once:!0}));try{y=await fetch(`${d}/v1/chat/completions`,{method:"POST",headers:{"content-type":"application/json"},body:JSON.stringify(S),signal:g.signal})}catch(e){if(h||isHeadersDeadline(e)){const e=h?`sent nothing in ${Math.round(f/1e3)}s`:`sent no response headers before this runtime's own ${Math.round(HEADERS_DEADLINE_MS/1e3)}s deadline`,n=new Error(`local: ${t} ${e} — the daemon is still loading the model, or its prompt prefill is longer than that. \`ollama ps\` shows which; a smaller prompt or a smaller model is the way through.`);throw n.status=0,n.code="timeout",n}if(c&&c.aborted||g.signal.aborted)return stoppedTurn(t);throw e}finally{clearTimeout(L)}if(!y.ok){let e="";try{e=await y.text()}catch{}const t=new Error(`local: chat/completions ${y.status}${e?` — ${e.slice(0,400)}`:""}`);throw t.status=y.status,t}let O="",A=null;const D=[],R=new Map,x=makeDeepSeekEnvelopeStripper(),b=makeTextToolCallStripper({toolNames:(Array.isArray(l)?l:[]).map((e=>e&&e.function&&e.function.name||e&&e.name||"")).filter(Boolean)});try{await readSSE(y,(e=>{e&&e.usage&&(A=e.usage);const t=e&&Array.isArray(e.choices)?e.choices[0]:null;if(!t)return;const n=t.delta||{};if("string"==typeof n.content&&n.content){const e=b(x(n.content));e&&(O+=e,"function"==typeof u&&u({text:e}))}const s=n.tool_calls||[];for(const e of s){const t=Number.isInteger(e.index)?e.index:D.length;let n=R.get(t);n||(n={id:"",type:"function",function:{name:"",arguments:""}},R.set(t,n)),e.id&&(n.id=e.id),e.type&&(n.type=e.type);const s=e.function||{};s.name&&(n.function.name=s.name),s.arguments&&(n.function.arguments+=s.arguments)}}),{idleTimeoutMs:SSE_IDLE_TIMEOUT_MS})}catch(e){if(c&&c.aborted)return stoppedTurn(t,{content:O,usage:A});throw e}finally{c&&"function"==typeof c.removeEventListener&&c.removeEventListener("abort",M)}const w=`${b(x.flush())||""}${b.flush()||""}`;w&&(O+=w,"function"==typeof u&&u({text:w}));for(const[e,t]of[...R.entries()].sort(((e,t)=>e[0]-t[0])))(t.function.name||t.function.arguments)&&(t.id||(t.id=`call_${e}`),D.push(t));const k={role:"assistant",content:O};D.length&&(k.tool_calls=D);const N=x.calls(),I=b.calls(),C={id:"local",model:String(t),choices:[{index:0,message:k,finish_reason:D.length?"tool_calls":"stop"}],...A?{usage:A}:{}};return(D.length||N.length||I.length)&&(C.toolCalls=[...D.map((e=>({id:e.id,name:e.function.name,args:e.function.arguments}))),...N.map((e=>({id:"",name:e.name,args:e.args||{}}))),...I.map((e=>({id:"",name:e.name,args:e.args||{}})))],(N.length||I.length)&&"tool_calls"!==C.choices[0].finish_reason&&(C.choices[0].finish_reason="tool_calls")),p&&(C.window={contextLength:p.window,dropped:p.dropped,toolTokens:p.toolTokens,estimatedTokens:p.estimatedTokens,reserveTokens:p.reserveTokens,over:p.over}),C}async function chat(e={}){const t=baseOf(e&&e.baseURL),n=e&&e.model,s=`${t}|${n}`;activeCountByKey.set(s,(activeCountByKey.get(s)||0)+1);try{let o=null;const r=await runSerialized(s,(async()=>{o=await ensureResident({baseURL:e&&e.baseURL,model:n,system:e&&e.system,signal:e&&e.signal,warmSecs:e&&e.warmSecs,firstByteTimeoutMs:e&&e.firstByteTimeoutMs});let s=o&&o.contextLength||null;const r=isLocalEndpoint(t)&&""!==String(n||"").trim();return!s&&r&&o&&!0!==o.ok&&!o.skipped&&(s=await windowOf(t,n,{signal:e&&e.signal})||null),chatOne({...e,contextLength:s})}));return swapWarmModel(t,n),o&&"object"==typeof r&&r&&(r.residency=o),r}finally{const e=(activeCountByKey.get(s)||1)-1;e<=0?activeCountByKey.delete(s):activeCountByKey.set(s,e)}}module.exports={DEFAULT_BASE:DEFAULT_BASE,DEFAULT_WARM_SECS:DEFAULT_WARM_SECS,RESIDENT_TIMEOUT_MS:RESIDENT_TIMEOUT_MS,SSE_IDLE_TIMEOUT_MS:SSE_IDLE_TIMEOUT_MS,FIRST_BYTE_TIMEOUT_MS:FIRST_BYTE_TIMEOUT_MS,PROBE_TIMEOUT_MS:1500,baseOf:baseOf,normalizeBase:normalizeBase,isLocalEndpoint:isLocalEndpoint,remoteRefusal:remoteRefusal,probe:probe,listTags:listTags,readSSE:readSSE,chat:chat,ensureResident:ensureResident,residentSecsLeft:residentSecsLeft,residentEntry:residentEntry,unload:unload,unloadAll:unloadAll,makeTextToolCallStripper:makeTextToolCallStripper,makeDeepSeekEnvelopeStripper:makeDeepSeekEnvelopeStripper,textCallsIn:textCallsIn,fitToWindowOf:fitToWindowOf,isHeadersDeadline:isHeadersDeadline,HEADERS_DEADLINE_MS:HEADERS_DEADLINE_MS,windowOf:windowOf,stoppedTurn:stoppedTurn};
|