aegiscode 6.8.30 → 6.8.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/aegiscode.js +31 -0
- package/package.json +1 -1
- package/scripts/predist.mjs +32 -328
- package/src/app.js +30 -0
- package/src/commands.js +95 -0
- package/src/config.js +148 -11
- package/src/permissions.js +92 -0
- package/src/shared.js +5 -1
- package/src/topup.js +207 -0
- package/vendor/client/platform.js +7 -0
- package/vendor/client/prefs.js +318 -0
- package/vendor/client/topup.js +294 -0
- package/vendor/client/update.js +141 -3
- package/vendor/desktop/lib/local/context.js +92 -0
- package/vendor/desktop/lib/local/engine.js +13 -0
- package/vendor/desktop/lib/local/local.js +1410 -46
- package/vendor/desktop/lib/settings.js +141 -14
|
@@ -37,6 +37,12 @@
|
|
|
37
37
|
* (see listTags) rather than fatal.
|
|
38
38
|
*/
|
|
39
39
|
|
|
40
|
+
// The standalone token-budget trimmer (plan P1 §5.1), used here for the one
|
|
41
|
+
// thing it was written for and never given a caller: keeping a turn's wire
|
|
42
|
+
// payload inside the window the daemon actually granted. See `fitToWindowOf`
|
|
43
|
+
// below for why the transport and not the caller owns this.
|
|
44
|
+
const { estimateTokens, trimContext } = require('./context.js');
|
|
45
|
+
|
|
40
46
|
/** Where a local daemon listens by default (Ollama's stock port). */
|
|
41
47
|
const DEFAULT_BASE = 'http://localhost:11434';
|
|
42
48
|
|
|
@@ -49,13 +55,105 @@ const DEFAULT_BASE = 'http://localhost:11434';
|
|
|
49
55
|
*/
|
|
50
56
|
const SSE_IDLE_TIMEOUT_MS = 300000;
|
|
51
57
|
|
|
58
|
+
/**
|
|
59
|
+
* How long the daemon may take to answer with response headers at all — the
|
|
60
|
+
* window BEFORE the first byte, which `SSE_IDLE_TIMEOUT_MS` above cannot cover:
|
|
61
|
+
* that timer is armed per parsed payload, and there is no payload, and not even
|
|
62
|
+
* a body to read, until this fetch resolves. The cloud transport bounds exactly
|
|
63
|
+
* this window in `boundedFetch` (client/aegis.js: "a server that accepts the
|
|
64
|
+
* socket and never sends response headers ... nothing is watching at all"); the
|
|
65
|
+
* local lane had no bound at all, so a daemon that accepted the socket and then
|
|
66
|
+
* went quiet left the turn as a spinner with no error, no timeout and no way
|
|
67
|
+
* out but Stop.
|
|
68
|
+
*
|
|
69
|
+
* Deliberately the SAME budget as the stall timer rather than a tighter one,
|
|
70
|
+
* because this wait is a legitimate part of a local turn on the hardware this
|
|
71
|
+
* class exists for: measured on the machine this was written (Ollama 0.12,
|
|
72
|
+
* `omega-sonnet5-lite` — a 1.5B model, daemon already warm), the first byte
|
|
73
|
+
* arrived **107.8s** after the request, all of it load + prompt eval with
|
|
74
|
+
* nothing to show. A bound tight enough to catch a wedged daemon fast would
|
|
75
|
+
* abort healthy turns; five minutes catches only the genuinely dead ones, and
|
|
76
|
+
* the error says which it is.
|
|
77
|
+
*/
|
|
78
|
+
const FIRST_BYTE_TIMEOUT_MS = SSE_IDLE_TIMEOUT_MS;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The response-header deadline the RUNTIME imposes, independent of the one
|
|
82
|
+
* above — and the reason a second clock has to be handled here at all.
|
|
83
|
+
*
|
|
84
|
+
* Measured on this host: a live `/v1/chat/completions` turn whose prompt prefill
|
|
85
|
+
* outlasted it (a 2521-token payload on a contended CPU) failed with
|
|
86
|
+
* `TypeError: fetch failed`, `cause: HeadersTimeoutError`,
|
|
87
|
+
* `code: 'UND_ERR_HEADERS_TIMEOUT'` at ~300s — undici's default `headersTimeout`,
|
|
88
|
+
* not this transport's `FIRST_BYTE_TIMEOUT_MS`. Any caller that states a longer
|
|
89
|
+
* bound (`firstByteTimeoutMs: 500000`) therefore does not get a longer wait: the
|
|
90
|
+
* runtime cuts the turn first, and it cut it as a raw fetch failure rather than
|
|
91
|
+
* in the timeout shape every other "waited too long" on this lane uses, so the
|
|
92
|
+
* host rendered an opaque error for a plain slow prefill. A local prefill IS
|
|
93
|
+
* routinely slower than this: the daemon's own log shows 236.7s for 2048 tokens
|
|
94
|
+
* while another model was loaded.
|
|
95
|
+
*
|
|
96
|
+
* Because the engine's contract is that a timeout arrives as `status: 0` +
|
|
97
|
+
* `code: 'timeout'` (the cloud client's shape, so "waited too long" stays one
|
|
98
|
+
* branch in the renderer), the second clock is translated into that shape rather
|
|
99
|
+
* than passed through. The number is stated here as a measured constant, not read
|
|
100
|
+
* from the runtime, which does not expose it.
|
|
101
|
+
*/
|
|
102
|
+
const HEADERS_DEADLINE_MS = 300000;
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Is this rejection the runtime's own response-header deadline?
|
|
106
|
+
*
|
|
107
|
+
* Matched on the undici cause rather than on the message text, and tolerant of
|
|
108
|
+
* the several shapes it wears (`HeadersTimeoutError` name, `UND_ERR_HEADERS_TIMEOUT`
|
|
109
|
+
* code) — a socket that dies for another reason must NOT be reported as a
|
|
110
|
+
* timeout, which is why this is a predicate rather than a catch-all.
|
|
111
|
+
*/
|
|
112
|
+
function isHeadersDeadline(err) {
|
|
113
|
+
if (!err) return false;
|
|
114
|
+
const cause = err.cause || err;
|
|
115
|
+
if (!cause) return false;
|
|
116
|
+
const code = String(cause.code || '');
|
|
117
|
+
const name = String(cause.name || '');
|
|
118
|
+
return code === 'UND_ERR_HEADERS_TIMEOUT' || name === 'HeadersTimeoutError';
|
|
119
|
+
}
|
|
120
|
+
|
|
52
121
|
/** How long the daemon probe waits. Short: this runs behind a UI spinner, and
|
|
53
122
|
* "no daemon" must read as an answer rather than as a hang. */
|
|
54
123
|
const PROBE_TIMEOUT_MS = 1500;
|
|
55
124
|
|
|
125
|
+
/**
|
|
126
|
+
* The one spelling of a base URL this module dials.
|
|
127
|
+
*
|
|
128
|
+
* A missing scheme is the COMMON case, not an edge one: `localhost:11434` and
|
|
129
|
+
* `127.0.0.1:11434` are what a user types into the Settings row, and both were
|
|
130
|
+
* previously read by `new URL()` as scheme `localhost:`/`127.0.0.1:` with an
|
|
131
|
+
* empty path — not http/https, so `isLocalEndpoint()` said "remote" and the
|
|
132
|
+
* fence REFUSED the user's own machine. The settings store refuses to write such
|
|
133
|
+
* a row, and a row already on disk from a hand edit gets a turn refused with
|
|
134
|
+
* "… is not on this machine, so it is not the local class" while Ollama is up
|
|
135
|
+
* and answering. That is the whole reported symptom: a healthy local daemon
|
|
136
|
+
* reported as a remote address.
|
|
137
|
+
*
|
|
138
|
+
* `http://` is prepended only when the string carries NO scheme, and the result
|
|
139
|
+
* is still parsed by `new URL()` — so this normalizes the dialing spelling
|
|
140
|
+
* without widening the fence by one host: `example.com:11434` becomes
|
|
141
|
+
* `http://example.com:11434`, still a dotted public name, still refused.
|
|
142
|
+
* `https://` is left exactly as given (a local daemon behind TLS is legitimate,
|
|
143
|
+
* and rewriting a stated scheme would send the turn to the wrong port).
|
|
144
|
+
*/
|
|
145
|
+
function normalizeBase(url) {
|
|
146
|
+
const raw = String(url == null ? '' : url).trim();
|
|
147
|
+
if (!raw) return DEFAULT_BASE;
|
|
148
|
+
// Any `scheme://` is taken as stated. A colon WITHOUT `//` is a host:port
|
|
149
|
+
// (`localhost:11434`), never a scheme — which is exactly the ambiguity
|
|
150
|
+
// WHATWG's parser resolves the other way, and the bug above.
|
|
151
|
+
return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(raw) ? raw : `http://${raw}`;
|
|
152
|
+
}
|
|
153
|
+
|
|
56
154
|
/** Trim trailing slashes so `${base}/v1/...` never doubles one. */
|
|
57
155
|
function baseOf(baseURL) {
|
|
58
|
-
return
|
|
156
|
+
return normalizeBase(baseURL).replace(/\/+$/, '');
|
|
59
157
|
}
|
|
60
158
|
|
|
61
159
|
/**
|
|
@@ -320,6 +418,59 @@ function dsmlHoldFrom(buf) {
|
|
|
320
418
|
return -1;
|
|
321
419
|
}
|
|
322
420
|
|
|
421
|
+
/**
|
|
422
|
+
* Pull COMPLETE plain `<tool_call>` blocks out of `text` (mutating `calls`).
|
|
423
|
+
*
|
|
424
|
+
* This runs inside the DSML stripper, ahead of its skeleton cleanup, for two
|
|
425
|
+
* reasons: that cleanup strips bare `<tool_calls>` tags and would otherwise
|
|
426
|
+
* leave the JSON body behind as prose, and running here means a DSML parameter
|
|
427
|
+
* body — a Write's `content`, a whole file — is never scanned, because those
|
|
428
|
+
* bodies are consumed by the DSML decoder and never reach this text path.
|
|
429
|
+
*/
|
|
430
|
+
function decodePlainToolCallTags(text, calls) {
|
|
431
|
+
if (!text || text.indexOf('<') === -1) return text;
|
|
432
|
+
let out = '';
|
|
433
|
+
let cursor = 0;
|
|
434
|
+
for (;;) {
|
|
435
|
+
const head = TEXT_TOOL_TAG_OPEN_RE.exec(text.slice(cursor));
|
|
436
|
+
if (!head) break;
|
|
437
|
+
const at = cursor + head.index;
|
|
438
|
+
const headText = head[0];
|
|
439
|
+
if (headText.includes(DSML_TAG_MARK)) { // belongs to the DSML dialect
|
|
440
|
+
out += text.slice(cursor, at + headText.length);
|
|
441
|
+
cursor = at + headText.length;
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
const rest = text.slice(at + headText.length);
|
|
445
|
+
const close = TEXT_TOOL_TAG_CLOSE_RE.exec(rest);
|
|
446
|
+
if (!close) break; // never closed: leave for the hold logic
|
|
447
|
+
const decoded = textCallsIn(rest.slice(0, close.index));
|
|
448
|
+
out += text.slice(cursor, at);
|
|
449
|
+
cursor = at + headText.length + close.index + close[0].length;
|
|
450
|
+
if (decoded.length) {
|
|
451
|
+
for (const c of decoded) calls.push(c);
|
|
452
|
+
} else {
|
|
453
|
+
out += text.slice(at, cursor); // a tag that is not a call: unchanged
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
return out + text.slice(cursor);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** The position of a plain `<tool_call>` head whose closer has not arrived, or
|
|
460
|
+
* -1 — the second hold rule, alongside dsmlHoldFrom(). */
|
|
461
|
+
function plainToolTagHoldAt(text) {
|
|
462
|
+
if (!text || text.indexOf('<') === -1) return -1;
|
|
463
|
+
let cursor = 0;
|
|
464
|
+
for (;;) {
|
|
465
|
+
const head = TEXT_TOOL_TAG_OPEN_RE.exec(text.slice(cursor));
|
|
466
|
+
if (!head) return -1;
|
|
467
|
+
const at = cursor + head.index;
|
|
468
|
+
if (head[0].includes(DSML_TAG_MARK)) { cursor = at + head[0].length; continue; }
|
|
469
|
+
const rest = text.slice(at + head[0].length);
|
|
470
|
+
return TEXT_TOOL_TAG_CLOSE_RE.test(rest) ? -1 : at;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
323
474
|
/**
|
|
324
475
|
* The per-turn stripper this transport uses. Feed it each `delta.content`
|
|
325
476
|
* chunk: it returns the text that is safe to show, holds back whatever is
|
|
@@ -333,6 +484,7 @@ function dsmlHoldFrom(buf) {
|
|
|
333
484
|
function makeDeepSeekEnvelopeStripper() {
|
|
334
485
|
let buf = '';
|
|
335
486
|
const calls = [];
|
|
487
|
+
const release = (text) => dsmlResidue(decodePlainToolCallTags(text, calls));
|
|
336
488
|
const strip = (chunk) => {
|
|
337
489
|
buf += typeof chunk === 'string' ? chunk : '';
|
|
338
490
|
const dec = decodeDeepSeekDSML(buf);
|
|
@@ -348,7 +500,15 @@ function makeDeepSeekEnvelopeStripper() {
|
|
|
348
500
|
out = buf;
|
|
349
501
|
buf = '';
|
|
350
502
|
}
|
|
351
|
-
|
|
503
|
+
// A plain `<tool_call>` head whose closer has not landed yet is held the
|
|
504
|
+
// same way the DSML envelope is: the call is one whole thing, and showing
|
|
505
|
+
// half of it would mean showing a tag the decoder is about to consume.
|
|
506
|
+
const plainAt = plainToolTagHoldAt(out);
|
|
507
|
+
if (plainAt !== -1) {
|
|
508
|
+
buf = out.slice(plainAt) + buf;
|
|
509
|
+
out = out.slice(0, plainAt);
|
|
510
|
+
}
|
|
511
|
+
return release(out);
|
|
352
512
|
};
|
|
353
513
|
strip.flush = () => {
|
|
354
514
|
const dec = decodeDeepSeekDSML(buf);
|
|
@@ -356,13 +516,358 @@ function makeDeepSeekEnvelopeStripper() {
|
|
|
356
516
|
for (const c of dec.calls) calls.push(c);
|
|
357
517
|
const drift = decodeDeepSeekDSMLDrift(dec.text);
|
|
358
518
|
for (const c of drift.calls) calls.push(c);
|
|
359
|
-
return
|
|
519
|
+
return release(drift.text);
|
|
360
520
|
};
|
|
361
521
|
/** Every tool call decoded from text so far, as `{ name, args }`. */
|
|
362
522
|
strip.calls = () => calls.slice();
|
|
363
523
|
return strip;
|
|
364
524
|
}
|
|
365
525
|
|
|
526
|
+
/**
|
|
527
|
+
* Tool calls a local model wrote as TEXT, rather than in the structured field.
|
|
528
|
+
*
|
|
529
|
+
* Measured against a live daemon (ollama 0.32.1, 2026-10-05), on the same
|
|
530
|
+
* prompt this file's DSML decoder exists for:
|
|
531
|
+
*
|
|
532
|
+
* qwen2.5-coder:1.5b-instruct-q4_K_M, `tools: [read]`, "read /etc/hostname
|
|
533
|
+
* with the read tool" →
|
|
534
|
+
* `content: "```json\n{\"name\": \"read\", \"arguments\": {\"path\": …}```"`,
|
|
535
|
+
* `finish_reason: "stop"`, and NO `delta.tool_calls` at all.
|
|
536
|
+
*
|
|
537
|
+
* That is not a quirk of one tag: Ollama's OpenAI shim only fills
|
|
538
|
+
* `delta.tool_calls` for templates it knows carry tool tokens (`tools` in the
|
|
539
|
+
* model's own `capabilities`, listed by `/api/tags`). Every other local model
|
|
540
|
+
* — an older qwen, a Llama, a gguf picked up from the Hub — is handed the
|
|
541
|
+
* schemas, answers in the dialect its chat template taught it, and emits it as
|
|
542
|
+
* plain content. The engine's loop then sees a model that "said" a JSON blob
|
|
543
|
+
* and ran nothing, which is what the user sees as the local class being unable
|
|
544
|
+
* to use tools at all.
|
|
545
|
+
*
|
|
546
|
+
* So the same treatment the DSML dialect gets: decode the envelope out of the
|
|
547
|
+
* text, keep the prose, and hand the calls to the tool loop. Two shapes are
|
|
548
|
+
* recognised, because they are the two the standard templates emit:
|
|
549
|
+
*
|
|
550
|
+
* <tool_call>{"name": "read", "arguments": {"path": "/etc/hostname"}}</tool_call>
|
|
551
|
+
* ```json
|
|
552
|
+
* {"name": "read", "arguments": {"path": "/etc/hostname"}}
|
|
553
|
+
* ```
|
|
554
|
+
*
|
|
555
|
+
* Deliberately NOT recognised: a bare `{…}` with no fence and no tag. Prose
|
|
556
|
+
* questions about JSON ("what does {"name": "read"} mean?") are common and a
|
|
557
|
+
* decoder that guessed there would turn a user's own example into a real file
|
|
558
|
+
* read — the conservative half of the trade this file already makes for the
|
|
559
|
+
* endpoint gate. A fenced block whose JSON does not carry a `name` + arguments
|
|
560
|
+
* pair is left exactly as it arrived: it is a code block, not a call.
|
|
561
|
+
*/
|
|
562
|
+
const TEXT_TOOL_TAG_OPEN_RE = /<\s*tool_calls?\b[^>]{0,80}>/i;
|
|
563
|
+
const TEXT_TOOL_TAG_CLOSE_RE = /<\s*\/\s*tool_calls?\s*>/i;
|
|
564
|
+
const TEXT_FENCE_RE = /```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?([\s\S]*?)```/;
|
|
565
|
+
// The OPENing half alone, for the case where the closer has not arrived yet:
|
|
566
|
+
// TEXT_FENCE_RE cannot match there (its closer is part of the match, NOT part
|
|
567
|
+
// of the captured body — the first version of this code sliced with
|
|
568
|
+
// `fence[0].length` and so held every fence forever), which is exactly why the
|
|
569
|
+
// hold test needs a separate pattern.
|
|
570
|
+
const TEXT_FENCE_OPEN_RE = /```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?/;
|
|
571
|
+
// A DSML tag always carries the fullwidth pipe; a PLAIN `<tool_call>` never
|
|
572
|
+
// does. That single character is what tells the two dialects apart, and it is
|
|
573
|
+
// why this pass can run ahead of the DSML machinery without ever claiming a
|
|
574
|
+
// tag that belongs to it.
|
|
575
|
+
const DSML_TAG_MARK = '|';
|
|
576
|
+
|
|
577
|
+
/** Coerce one decoded envelope body into `{ name, args }`, or null when it is
|
|
578
|
+
* not a call. Accepts the argument spellings the templates actually produce:
|
|
579
|
+
* `arguments` (openai-ish, object or JSON string), `parameters`, `args`, and
|
|
580
|
+
* the `{function:{name,arguments}}` wrapper a model sometimes copies from the
|
|
581
|
+
* schema it was handed. */
|
|
582
|
+
function asTextToolCall(value) {
|
|
583
|
+
if (!value || typeof value !== 'object') return null;
|
|
584
|
+
let node = value;
|
|
585
|
+
if (node.function && typeof node.function === 'object') {
|
|
586
|
+
node = { ...node.function, name: node.function.name || node.name };
|
|
587
|
+
}
|
|
588
|
+
const rawName = node.name || node.tool || node.tool_name || node.function_name;
|
|
589
|
+
const name = typeof rawName === 'string' ? rawName.trim() : '';
|
|
590
|
+
if (!name) return null;
|
|
591
|
+
let args = node.arguments != null ? node.arguments
|
|
592
|
+
: node.parameters != null ? node.parameters
|
|
593
|
+
: node.args != null ? node.args
|
|
594
|
+
: node.tool_input != null ? node.tool_input
|
|
595
|
+
: {};
|
|
596
|
+
if (typeof args === 'string') {
|
|
597
|
+
const parsed = args.trim() ? safeJson(args) : {};
|
|
598
|
+
args = parsed && typeof parsed === 'object' ? parsed : {};
|
|
599
|
+
}
|
|
600
|
+
if (args == null || typeof args !== 'object') args = {};
|
|
601
|
+
return { name, args };
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/** JSON.parse that returns null instead of throwing. */
|
|
605
|
+
function safeJson(text) {
|
|
606
|
+
try {
|
|
607
|
+
return JSON.parse(text);
|
|
608
|
+
} catch {
|
|
609
|
+
return null;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/** Decode one envelope BODY (tag inner text or fence inner text). Returns []
|
|
614
|
+
* when it is not a call, so the caller can hand the text back unchanged. */
|
|
615
|
+
function textCallsIn(body) {
|
|
616
|
+
const text = String(body || '').trim().replace(/^json\s*/i, '').trim();
|
|
617
|
+
if (!text) return [];
|
|
618
|
+
const parsed = safeJson(text);
|
|
619
|
+
if (!parsed) return [];
|
|
620
|
+
const list = Array.isArray(parsed) ? parsed : [parsed];
|
|
621
|
+
const calls = [];
|
|
622
|
+
for (const item of list) {
|
|
623
|
+
const call = asTextToolCall(item);
|
|
624
|
+
if (call) calls.push(call);
|
|
625
|
+
}
|
|
626
|
+
return calls;
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** The next envelope in `buf`: a ``` fence carrying a call, or the position of
|
|
630
|
+
* a plain `<tool_call>` head the caller must hold. `end` is where it finishes
|
|
631
|
+
* in `buf`; `complete` is false while its closer is still on the wire, in
|
|
632
|
+
* which case the caller must HOLD the tail, not show it. */
|
|
633
|
+
function nextTextToolEnvelope(buf) {
|
|
634
|
+
const fence = TEXT_FENCE_RE.exec(buf);
|
|
635
|
+
if (fence) {
|
|
636
|
+
// `fence[1]` is the body; the closing fence is part of the MATCH, so the
|
|
637
|
+
// body must come from the capture group and the end from `fence[0]`.
|
|
638
|
+
return {
|
|
639
|
+
start: fence.index,
|
|
640
|
+
end: fence.index + fence[0].length,
|
|
641
|
+
complete: true,
|
|
642
|
+
raw: fence[0],
|
|
643
|
+
body: fence[1],
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
const open = TEXT_FENCE_OPEN_RE.exec(buf);
|
|
647
|
+
if (!open) return null;
|
|
648
|
+
return {
|
|
649
|
+
start: open.index,
|
|
650
|
+
end: buf.length,
|
|
651
|
+
complete: false,
|
|
652
|
+
body: buf.slice(open.index + open[0].length),
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** Where the hold must start in `buf`, or -1 when nothing is pending: the
|
|
657
|
+
* index of a recognized-but-unclosed head, or of the first characters of one
|
|
658
|
+
* that is still arriving (a '`' that may become a fence, a '<' that may become
|
|
659
|
+
* a tag). Everything before that index is released as prose. */
|
|
660
|
+
function trailingHeadFrom(buf, allowBare = false) {
|
|
661
|
+
if (!buf) return -1;
|
|
662
|
+
const marker = buf.search(TEXT_TOOL_HEAD_RE);
|
|
663
|
+
if (marker !== -1) return marker; // head started, closer has not landed
|
|
664
|
+
// A fence opener still arriving: one or two backticks at the very end. It
|
|
665
|
+
// cannot be anything else — a closed inline code span would carry its closer.
|
|
666
|
+
const ticks = /`{1,2}$/.exec(buf);
|
|
667
|
+
if (ticks) return ticks.index;
|
|
668
|
+
// A tag still arriving: the last '<' has only a prefix of "tool_call" after
|
|
669
|
+
// it. Prose that merely contains '<' resolves on the very next chunk.
|
|
670
|
+
const lt = buf.lastIndexOf('<');
|
|
671
|
+
if (lt !== -1 && couldBeTextToolHead(buf.slice(lt))) return lt;
|
|
672
|
+
// A bare JSON object still arriving — the same dialect rule as above, and it
|
|
673
|
+
// has to be held for the same reason: a chunk landing on the first `{` would
|
|
674
|
+
// otherwise show that brace as prose and the object could never be recognised
|
|
675
|
+
// afterwards. Only JSON-SHAPED tails are held; an ordinary brace in prose
|
|
676
|
+
// ('use a { in your code') is released and streams normally.
|
|
677
|
+
if (allowBare) {
|
|
678
|
+
const open = buf.lastIndexOf('{');
|
|
679
|
+
if (open !== -1 && /^\{\s*(?:"[^"]*"\s*:[\s\S]*|"[^"]*"?\s*|)$/.test(buf.slice(open))) return open;
|
|
680
|
+
}
|
|
681
|
+
return -1;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/** Still a POSSIBLE head — a fence just starting, or a `<tool_c…` prefix that
|
|
685
|
+
* has not finished arriving. A recognized (already-matching) head is held by
|
|
686
|
+
* TEXT_TOOL_HEAD_RE, not here; this is only for the partial case, so that a
|
|
687
|
+
* chunk boundary landing mid-`<tool_call` cannot show two characters of a tag
|
|
688
|
+
* and then retract them. */
|
|
689
|
+
function couldBeTextToolHead(buf) {
|
|
690
|
+
const lead = buf.replace(/^[\s\uFEFF]+/, '');
|
|
691
|
+
if (!lead) return true; // nothing buffered yet
|
|
692
|
+
if (lead.length <= 3 && '```'.startsWith(lead)) return true;
|
|
693
|
+
if (lead.startsWith('<') && !TEXT_TOOL_HEAD_RE.test(lead)) {
|
|
694
|
+
const name = lead.slice(1).replace(/^\/?\s*/, '').toLowerCase();
|
|
695
|
+
if (name.length <= 10 && /^[a-z_]*$/.test(name)) {
|
|
696
|
+
return 'tool_call'.startsWith(name) || 'tool_calls'.startsWith(name);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return false;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* Where a JSON object starting at `start` ends, or -1 while it is still
|
|
704
|
+
* arriving. String- and escape-aware, because the arguments of a real call
|
|
705
|
+
* routinely carry quoted braces (`{"content": "function f() { … }"}`) and a
|
|
706
|
+
* naive depth count ends the object inside one of them.
|
|
707
|
+
*/
|
|
708
|
+
function balancedJsonEnd(text, start) {
|
|
709
|
+
let depth = 0;
|
|
710
|
+
let inStr = false;
|
|
711
|
+
let esc = false;
|
|
712
|
+
for (let i = start; i < text.length; i++) {
|
|
713
|
+
const ch = text[i];
|
|
714
|
+
if (inStr) {
|
|
715
|
+
if (esc) esc = false;
|
|
716
|
+
else if (ch === '\\') esc = true;
|
|
717
|
+
else if (ch === '"') inStr = false;
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
if (ch === '"') inStr = true;
|
|
721
|
+
else if (ch === '{' || ch === '[') depth++;
|
|
722
|
+
else if (ch === '}' || ch === ']') {
|
|
723
|
+
depth--;
|
|
724
|
+
if (depth <= 0) return i + 1;
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
return -1;
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Decode tool calls out of a local model's TEXT — the fenced and `<tool_call>`
|
|
732
|
+
* dialects, and the bare JSON object a small model emits when it ignores both.
|
|
733
|
+
*
|
|
734
|
+
* `toolNames` is the set of tools THIS request offered. It is the guard that
|
|
735
|
+
* makes the bare-JSON rule honest rather than a guess: a JSON object is only
|
|
736
|
+
* ever read as a call when the name it carries is one of the tools the model
|
|
737
|
+
* was actually given (and, for an array, every one of them). A user asking
|
|
738
|
+
* "what does {"name": "read"} mean?" in a turn with no tools, or naming a tool
|
|
739
|
+
* that was never offered, gets prose back untouched. That is measured against
|
|
740
|
+
* the real failure mode: with the fence rule alone, qwen2.5-coder answered the
|
|
741
|
+
* tool prompt with a bare object about as often as with a fenced one, and the
|
|
742
|
+
* turn ran nothing either time.
|
|
743
|
+
*
|
|
744
|
+
* Incomplete envelopes are held (never shown and retracted): the caller feeds
|
|
745
|
+
* chunks, and a fence/tag/object whose closer has not arrived stays in the
|
|
746
|
+
* buffer until it has, or until flush() decides.
|
|
747
|
+
*/
|
|
748
|
+
function makeTextToolCallStripper({ toolNames = null } = {}) {
|
|
749
|
+
const names = toolNames ? new Set([...toolNames].map((n) => String(n))) : null;
|
|
750
|
+
let buf = '';
|
|
751
|
+
const calls = [];
|
|
752
|
+
/** A decoded batch counts only if every call names an offered tool. */
|
|
753
|
+
const accept = (decoded) =>
|
|
754
|
+
Boolean(decoded.length) && (!names || decoded.every((c) => names.has(c.name)));
|
|
755
|
+
/** Show a complete envelope's TEXT when it is not a call, or swallow it and
|
|
756
|
+
* record the call when it is. */
|
|
757
|
+
const digest = (raw, body) => {
|
|
758
|
+
const decoded = textCallsIn(body);
|
|
759
|
+
if (!accept(decoded)) return raw; // prose fence / tag: shown exactly as it came
|
|
760
|
+
for (const c of decoded) calls.push(c);
|
|
761
|
+
return '';
|
|
762
|
+
};
|
|
763
|
+
const strip = (chunk) => {
|
|
764
|
+
buf += typeof chunk === 'string' ? chunk : '';
|
|
765
|
+
let out = '';
|
|
766
|
+
// Set whenever the loop stops on an envelope that is still arriving: the
|
|
767
|
+
// buffer then holds a decided-pending head and the tail release below must
|
|
768
|
+
// not undo the hold it just took.
|
|
769
|
+
let held = false;
|
|
770
|
+
for (;;) {
|
|
771
|
+
const fence = nextTextToolEnvelope(buf);
|
|
772
|
+
if (fence) {
|
|
773
|
+
out += buf.slice(0, fence.start); // prose before the envelope is never held
|
|
774
|
+
buf = buf.slice(fence.start);
|
|
775
|
+
if (!fence.complete) { held = true; break; }
|
|
776
|
+
out += digest(fence.raw, fence.body);
|
|
777
|
+
buf = buf.slice(fence.raw.length);
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
const tag = TEXT_TOOL_TAG_OPEN_RE.exec(buf);
|
|
781
|
+
if (tag) {
|
|
782
|
+
out += buf.slice(0, tag.index);
|
|
783
|
+
buf = buf.slice(tag.index);
|
|
784
|
+
const rest = buf.slice(tag[0].length);
|
|
785
|
+
const close = TEXT_TOOL_TAG_CLOSE_RE.exec(rest);
|
|
786
|
+
if (!close) { held = true; break; } // closer still on the wire: hold it
|
|
787
|
+
const raw = buf.slice(0, tag[0].length + close.index + close[0].length);
|
|
788
|
+
out += digest(raw, rest.slice(0, close.index));
|
|
789
|
+
buf = buf.slice(raw.length);
|
|
790
|
+
continue;
|
|
791
|
+
}
|
|
792
|
+
// Bare JSON — admitted only against the tools this turn offered, so a
|
|
793
|
+
// JSON-shaped QUESTION can never become a file read. `{"` rather than a
|
|
794
|
+
// lone `{`: the templates that emit this write an object with quoted
|
|
795
|
+
// keys, and it keeps prose braces out of the scan entirely.
|
|
796
|
+
const bare = names ? /\{\s*"/.exec(buf) : null;
|
|
797
|
+
if (!bare) break;
|
|
798
|
+
const end = balancedJsonEnd(buf, bare.index);
|
|
799
|
+
const before = buf.slice(0, bare.index);
|
|
800
|
+
if (end === -1) { out += before; buf = buf.slice(bare.index); held = true; break; } // hold
|
|
801
|
+
const decoded = textCallsIn(buf.slice(bare.index, end));
|
|
802
|
+
out += before;
|
|
803
|
+
if (accept(decoded)) {
|
|
804
|
+
for (const c of decoded) calls.push(c);
|
|
805
|
+
} else {
|
|
806
|
+
out += buf.slice(bare.index, end); // prose: shown, and not rescanned
|
|
807
|
+
}
|
|
808
|
+
buf = buf.slice(end);
|
|
809
|
+
}
|
|
810
|
+
// Nothing further can turn into an envelope: release the tail rather than
|
|
811
|
+
// delay prose behind a guard that will never fire. A recognized-but-unclosed
|
|
812
|
+
// head (or the first characters of one, e.g. the '`' a fence opener starts
|
|
813
|
+
// with) stays held — its closer is still coming, and showing it now would
|
|
814
|
+
// mean retracting it a moment later. The head can be anywhere in the
|
|
815
|
+
// buffer, not only at its start: prose streams ahead of the envelope it
|
|
816
|
+
// introduces, and the first version of this code released the whole buffer
|
|
817
|
+
// whenever the buffer did not BEGIN with a head, which let every
|
|
818
|
+
// prose-then-call turn through as text.
|
|
819
|
+
const holdAt = held ? 0 : trailingHeadFrom(buf, Boolean(names));
|
|
820
|
+
if (holdAt === -1) {
|
|
821
|
+
out += buf;
|
|
822
|
+
buf = '';
|
|
823
|
+
} else if (holdAt > 0) {
|
|
824
|
+
out += buf.slice(0, holdAt);
|
|
825
|
+
buf = buf.slice(holdAt);
|
|
826
|
+
}
|
|
827
|
+
if (buf.length > TEXT_TOOL_HOLD_MAX) {
|
|
828
|
+
// A fence or tag left open forever must not swallow the rest of the
|
|
829
|
+
// answer; past this much held text it is prose, and it is shown as such.
|
|
830
|
+
out += buf;
|
|
831
|
+
buf = '';
|
|
832
|
+
}
|
|
833
|
+
return out;
|
|
834
|
+
};
|
|
835
|
+
strip.flush = () => {
|
|
836
|
+
const rest = buf;
|
|
837
|
+
buf = '';
|
|
838
|
+
if (!rest) return '';
|
|
839
|
+
const env = nextTextToolEnvelope(rest);
|
|
840
|
+
// A fence or tag whose closer never arrived: decide NOW rather than hold
|
|
841
|
+
// forever. If its body decodes it was a call cut off mid-stream (still
|
|
842
|
+
// worth running — the arguments it did send are the caller's to validate);
|
|
843
|
+
// if it does not decode it was prose all along and is released verbatim.
|
|
844
|
+
if (env) return rest.slice(0, env.start) + digest(rest.slice(env.start), env.body);
|
|
845
|
+
const tag = TEXT_TOOL_TAG_OPEN_RE.exec(rest);
|
|
846
|
+
if (!tag) {
|
|
847
|
+
// A bare object the stream stopped inside: decide it here, so a call cut
|
|
848
|
+
// off mid-argument still runs instead of being dropped as text.
|
|
849
|
+
const bare = names ? /\{\s*"/.exec(rest) : null;
|
|
850
|
+
if (!bare) return rest;
|
|
851
|
+
const end = balancedJsonEnd(rest, bare.index);
|
|
852
|
+
return end === -1
|
|
853
|
+
? rest
|
|
854
|
+
: rest.slice(0, bare.index) + digest(rest.slice(bare.index, end), rest.slice(bare.index, end)) + rest.slice(end);
|
|
855
|
+
}
|
|
856
|
+
const body = rest.slice(tag.index + tag[0].length).replace(TEXT_TOOL_TAG_CLOSE_RE, '');
|
|
857
|
+
return rest.slice(0, tag.index) + digest(rest.slice(tag.index), body);
|
|
858
|
+
};
|
|
859
|
+
strip.calls = () => calls.slice();
|
|
860
|
+
return strip;
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/** Where a text-form envelope could still begin — the guard that keeps a
|
|
864
|
+
* partial ` ` ` or `<tool_c` from being shown and then retracted. */
|
|
865
|
+
const TEXT_TOOL_HEAD_RE = /```|<\s*tool_calls?\b/i;
|
|
866
|
+
|
|
867
|
+
/** A fence left open forever (a model that never closed it) must not hide the
|
|
868
|
+
* rest of the answer — past this much held text, release it as prose. */
|
|
869
|
+
const TEXT_TOOL_HOLD_MAX = 64 * 1024;
|
|
870
|
+
|
|
366
871
|
/**
|
|
367
872
|
* Whether a base URL addresses hardware on this machine (or on this private
|
|
368
873
|
* network) — the gate that decides whether the free lane exists at all.
|
|
@@ -384,8 +889,14 @@ function makeDeepSeekEnvelopeStripper() {
|
|
|
384
889
|
function isLocalEndpoint(url) {
|
|
385
890
|
let host = '';
|
|
386
891
|
let scheme = '';
|
|
892
|
+
// No host stated is not evidence of this machine: '' / whitespace / null stay
|
|
893
|
+
// refused, exactly as before (the sealed list in test/local-endpoint.test.mjs
|
|
894
|
+
// pins those). Only a NON-EMPTY string gets the scheme defaulted, which is the
|
|
895
|
+
// `localhost:11434` case above — a host the user named, in the spelling they
|
|
896
|
+
// named it in.
|
|
897
|
+
if (String(url == null ? '' : url).trim() === '') return false;
|
|
387
898
|
try {
|
|
388
|
-
const parsed = new URL(
|
|
899
|
+
const parsed = new URL(normalizeBase(url));
|
|
389
900
|
host = parsed.hostname.toLowerCase();
|
|
390
901
|
scheme = parsed.protocol.toLowerCase();
|
|
391
902
|
} catch {
|
|
@@ -485,6 +996,74 @@ async function listTags(baseURL = DEFAULT_BASE) {
|
|
|
485
996
|
})).filter((m) => m.id);
|
|
486
997
|
}
|
|
487
998
|
|
|
999
|
+
/** How long an unload (`keep_alive: 0`) request waits before giving up.
|
|
1000
|
+
* Generous — Ollama still has to touch the scheduler for a model that may be
|
|
1001
|
+
* mid-generation — but bounded, because this runs on the app's quit path and
|
|
1002
|
+
* must never be the reason closing the window hangs. */
|
|
1003
|
+
const UNLOAD_TIMEOUT_MS = 4000;
|
|
1004
|
+
|
|
1005
|
+
/** How long the warm-up request below may take before it is abandoned.
|
|
1006
|
+
*
|
|
1007
|
+
* It is the TURN's own first-byte budget (`FIRST_BYTE_TIMEOUT_MS`), and that
|
|
1008
|
+
* is the point: the pin exists to absorb a load the turn would otherwise wait
|
|
1009
|
+
* out, so it may legitimately take as long as the turn would have — but it may
|
|
1010
|
+
* never outlive the caller's own patience with the daemon. Measured 107.8s for
|
|
1011
|
+
* a warm 1.5B model's first byte on this class of hardware, so a bound tight
|
|
1012
|
+
* enough to abort a wedged daemon fast would abort healthy cold loads; the
|
|
1013
|
+
* turn's five-minute bound is the measured right answer, applied twice at most
|
|
1014
|
+
* (pin, then turn) rather than once.
|
|
1015
|
+
*
|
|
1016
|
+
* A caller that STATES its own window (`firstByteTimeoutMs`) is never widened:
|
|
1017
|
+
* the pin gets that number, so the turn still fails on the caller's clock
|
|
1018
|
+
* instead of the pin's. That is the bug this bound replaced — against a daemon
|
|
1019
|
+
* that accepts the socket and never answers, a pin on a budget of its own held
|
|
1020
|
+
* the turn for minutes before the turn's own 1s deadline was even armed.
|
|
1021
|
+
*
|
|
1022
|
+
* The timer is `unref`'d (see `ensureResident`), the same rule
|
|
1023
|
+
* `FIRST_BYTE_TIMEOUT_MS`'s timer follows: a short-lived CLI process
|
|
1024
|
+
* (`aegiscode -p "…"`) must never be held open by a pending warm-up. */
|
|
1025
|
+
const RESIDENT_TIMEOUT_MS = FIRST_BYTE_TIMEOUT_MS;
|
|
1026
|
+
|
|
1027
|
+
/** How long a pin asks the daemon to hold the model warm, when nothing states
|
|
1028
|
+
* otherwise — the whole point of the pin. Ollama's stock window is 5 MINUTES,
|
|
1029
|
+
* and that is the number this lane was silently running on: idle past it and
|
|
1030
|
+
* the daemon evicts, so the next turn re-pays the entire cold load. A user
|
|
1031
|
+
* who steps away from the desktop for coffee should come back to a warm model
|
|
1032
|
+
* (they paid for the load once already), and the model is released anyway on
|
|
1033
|
+
* quit (`unloadAll`) and on a model switch (`swapWarmModel`), so 30 minutes
|
|
1034
|
+
* costs nothing that is not already accounted for.
|
|
1035
|
+
*
|
|
1036
|
+
* Tunable per call (`opts.warmSecs`) and by env (`AEGIS_LOCAL_WARM_SECS`), and
|
|
1037
|
+
* `0` is meaningful: it turns the pin off and returns the lane to the daemon's
|
|
1038
|
+
* own lifecycle. */
|
|
1039
|
+
const DEFAULT_WARM_SECS = 1800;
|
|
1040
|
+
|
|
1041
|
+
/**
|
|
1042
|
+
* Evict `model` from the daemon's memory right now, instead of waiting out
|
|
1043
|
+
* its keep-alive. Ollama's documented mechanism: a chat request with
|
|
1044
|
+
* `keep_alive: 0` and nothing to answer unloads the model as its only
|
|
1045
|
+
* effect (https://github.com/ollama/ollama/blob/main/docs/faq.md). Best-effort
|
|
1046
|
+
* and NEVER throws — called from the app's quit path and from the warm-model
|
|
1047
|
+
* swap below, neither of which may fail a turn or block a shutdown over a
|
|
1048
|
+
* daemon that is merely slow or already gone.
|
|
1049
|
+
*/
|
|
1050
|
+
async function unload(baseURL, model) {
|
|
1051
|
+
if (!model) return { ok: false };
|
|
1052
|
+
const base = baseOf(baseURL);
|
|
1053
|
+
if (!isLocalEndpoint(base)) return { ok: false }; // never dial out, even to "clean up"
|
|
1054
|
+
try {
|
|
1055
|
+
const res = await fetch(`${base}/api/chat`, {
|
|
1056
|
+
method: 'POST',
|
|
1057
|
+
headers: { 'content-type': 'application/json' },
|
|
1058
|
+
body: JSON.stringify({ model: String(model), messages: [], stream: false, keep_alive: 0 }),
|
|
1059
|
+
signal: AbortSignal.timeout(UNLOAD_TIMEOUT_MS),
|
|
1060
|
+
});
|
|
1061
|
+
return { ok: res.ok };
|
|
1062
|
+
} catch {
|
|
1063
|
+
return { ok: false };
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
|
|
488
1067
|
/** Normalize one history row into an OpenAI-compatible message. Tool rows and
|
|
489
1068
|
* assistant rows carrying `tool_calls` must survive intact: dropping either
|
|
490
1069
|
* leaves the model seeing a tool call it never got a result for. */
|
|
@@ -582,6 +1161,536 @@ async function readSSE(res, onEvent, { idleTimeoutMs = SSE_IDLE_TIMEOUT_MS } = {
|
|
|
582
1161
|
}
|
|
583
1162
|
}
|
|
584
1163
|
|
|
1164
|
+
/**
|
|
1165
|
+
* Serializes calls that would land on the SAME daemon+model pair.
|
|
1166
|
+
*
|
|
1167
|
+
* Ollama loads each model into its own `llama-server` with exactly one
|
|
1168
|
+
* request slot (`-np 1`). That slot does not queue a second request that
|
|
1169
|
+
* arrives while it is still generating — it CANCELS the in-flight task
|
|
1170
|
+
* server-side and answers the first caller with an HTTP 500, after however
|
|
1171
|
+
* long that caller had already been waiting (reproduced live: a stuck or
|
|
1172
|
+
* merely slow turn on this machine's CPU-only inference, then a second turn
|
|
1173
|
+
* — another chat window, a subagent, a retry — lands on the same model and
|
|
1174
|
+
* the first caller's request dies with a 500 it cannot act on). Client-side
|
|
1175
|
+
* serialization is the only place this can be prevented at all; the daemon
|
|
1176
|
+
* itself does not offer a queued mode. Keyed on (base, model) rather than
|
|
1177
|
+
* model alone: two different daemons (this box and a LAN box, say) each have
|
|
1178
|
+
* their own slot and may run fully in parallel.
|
|
1179
|
+
*/
|
|
1180
|
+
const localCallQueues = new Map(); // "<base>|<model>" -> tail promise (never rejects)
|
|
1181
|
+
|
|
1182
|
+
function runSerialized(key, fn) {
|
|
1183
|
+
const prev = localCallQueues.get(key) || Promise.resolve();
|
|
1184
|
+
// Run `fn` once the previous call for this key has SETTLED, success or
|
|
1185
|
+
// failure — a failed turn must not wedge every later turn behind it.
|
|
1186
|
+
const settled = prev.then(fn, fn);
|
|
1187
|
+
// The map holds a swallowed copy so the next call's `.then` always fires;
|
|
1188
|
+
// the caller still gets `settled`, which carries the real result or error.
|
|
1189
|
+
localCallQueues.set(
|
|
1190
|
+
key,
|
|
1191
|
+
settled.then(
|
|
1192
|
+
() => {},
|
|
1193
|
+
() => {}
|
|
1194
|
+
)
|
|
1195
|
+
);
|
|
1196
|
+
return settled;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
// How many calls are CURRENTLY in flight for a given "<base>|<model>" key —
|
|
1200
|
+
// not the queue above (which only orders them), but a live count, so the
|
|
1201
|
+
// warm-model swap below can tell "idle" from "still serving someone else".
|
|
1202
|
+
const activeCountByKey = new Map();
|
|
1203
|
+
|
|
1204
|
+
// The one model THIS transport last warmed per daemon — not every model
|
|
1205
|
+
// Ollama happens to have loaded (another process could have warmed others),
|
|
1206
|
+
// only the ones this app is responsible for cleaning up after. Keyed on the
|
|
1207
|
+
// normalized base, same spelling as the call queue above.
|
|
1208
|
+
const warmModelByBase = new Map(); // base -> model
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* Swap the daemon's "warm" model for this app from `prevModel` to `model`,
|
|
1212
|
+
* evicting the outgoing one — UNLESS something else is still actively using
|
|
1213
|
+
* it right now. That guard matters: a second chat window or a subagent still
|
|
1214
|
+
* mid-turn on the old model must not be cut out from under itself just
|
|
1215
|
+
* because a DIFFERENT turn moved on to a new one. Fired after a turn
|
|
1216
|
+
* SUCCEEDS, never before — a model that was never actually reached (a 400, a
|
|
1217
|
+
* refusal) must not be recorded as warm.
|
|
1218
|
+
*
|
|
1219
|
+
* Without this, every model switch in a session left the previous one loaded
|
|
1220
|
+
* too: a few turns in, Ollama was running two or three `llama-server`
|
|
1221
|
+
* processes at once (one per model, each pinned at ~100%+ CPU on this
|
|
1222
|
+
* class of hardware — reproduced live), each fighting the others for the
|
|
1223
|
+
* same handful of cores. Capping the daemon at one warm model per app is
|
|
1224
|
+
* what keeps that from compounding turn after turn.
|
|
1225
|
+
*/
|
|
1226
|
+
function swapWarmModel(base, model) {
|
|
1227
|
+
if (!model) return;
|
|
1228
|
+
const prevModel = warmModelByBase.get(base);
|
|
1229
|
+
warmModelByBase.set(base, model);
|
|
1230
|
+
if (!prevModel || prevModel === model) return;
|
|
1231
|
+
if (activeCountByKey.get(`${base}|${prevModel}`)) return; // still in use elsewhere
|
|
1232
|
+
unload(base, prevModel).catch(() => {});
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
/**
|
|
1236
|
+
* Unload every model this transport warmed, across every daemon it talked
|
|
1237
|
+
* to. Called once, from the app's quit path: "closing the window closes the
|
|
1238
|
+
* model" is the point — a local model left loaded after the app exits is
|
|
1239
|
+
* pure waste, the one thing the free/local lane was supposed to never cost.
|
|
1240
|
+
* Best-effort and bounded, like `unload` itself, so this resolves quickly
|
|
1241
|
+
* even against a daemon that is slow or already gone.
|
|
1242
|
+
*/
|
|
1243
|
+
async function unloadAll() {
|
|
1244
|
+
const entries = [...warmModelByBase.entries()];
|
|
1245
|
+
warmModelByBase.clear();
|
|
1246
|
+
await Promise.all(entries.map(([base, model]) => unload(base, model)));
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
/** The window a pin should ask for: the caller's number, else the operator's,
|
|
1250
|
+
* else the default above. `0` is a real answer (pin nothing) and is passed
|
|
1251
|
+
* through rather than treated as "unstated". */
|
|
1252
|
+
function warmSecsFor({ warmSecs } = {}) {
|
|
1253
|
+
const stated = Number(warmSecs);
|
|
1254
|
+
if (Number.isFinite(stated) && stated >= 0) return Math.floor(stated);
|
|
1255
|
+
const env = Number(process.env && process.env.AEGIS_LOCAL_WARM_SECS);
|
|
1256
|
+
if (Number.isFinite(env) && env >= 0) return Math.floor(env);
|
|
1257
|
+
return DEFAULT_WARM_SECS;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
/**
|
|
1261
|
+
* Live-pin memo: which `<base>|<model>|<prefix>` is known warm until when.
|
|
1262
|
+
*
|
|
1263
|
+
* The pin's cost is a PREFILL, not a turn, so the steady-state win is not
|
|
1264
|
+
* making the pin cheaper — it is not issuing one. Residency IS still checked
|
|
1265
|
+
* against the daemon (`/api/ps`), because the daemon is the authority on what
|
|
1266
|
+
* it holds: another app, an `ollama stop`, or a reboot can all clear a window
|
|
1267
|
+
* this process set. But a window this process set, with seconds still left on
|
|
1268
|
+
* it, cannot have expired underneath us — so the memo answers without the
|
|
1269
|
+
* round trip, and is *dropped* the moment its deadline passes, so it can never
|
|
1270
|
+
* claim residency it cannot prove.
|
|
1271
|
+
*
|
|
1272
|
+
* Keyed by the PREFIX as well as the model, deliberately: a pin only buys
|
|
1273
|
+
* anything for the exact text it warmed (llama.cpp reuses the shared prefix and
|
|
1274
|
+
* nothing else), so a warm-up for one system prompt must never suppress the pin
|
|
1275
|
+
* for a turn sending a different one — that turn would pay the cold prefill the
|
|
1276
|
+
* memo claimed to have paid for it.
|
|
1277
|
+
*
|
|
1278
|
+
* `dialer` is the `fetch` that performed the pin. A different one (a proxy, a
|
|
1279
|
+
* test's stub, a re-`require`d daemon handle) is a different daemon and cannot
|
|
1280
|
+
* be vouched for by this map.
|
|
1281
|
+
*/
|
|
1282
|
+
const residentMemo = new Map(); // `${base}\0${model}\0${prefix}` -> { until, dialer }
|
|
1283
|
+
|
|
1284
|
+
function residentKey(base, model, prefix) {
|
|
1285
|
+
return `${base}\u0000${model}\u0000${prefix}`;
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
/** Seconds of pinned window left for this exact prefix, or 0 when unproven. */
|
|
1289
|
+
function residentSecsLeft(base, model, prefix, { dialer = fetch, now = Date.now() } = {}) {
|
|
1290
|
+
const hit = residentMemo.get(residentKey(base, model, prefix));
|
|
1291
|
+
if (!hit || hit.dialer !== dialer) return 0;
|
|
1292
|
+
const left = Math.round((hit.until - now) / 1000);
|
|
1293
|
+
if (left <= 0) {
|
|
1294
|
+
residentMemo.delete(residentKey(base, model, prefix));
|
|
1295
|
+
return 0;
|
|
1296
|
+
}
|
|
1297
|
+
return left;
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
/**
|
|
1301
|
+
* The live memo entry itself, so a free (no-daemon-contact) memo hit can still
|
|
1302
|
+
* report the window the pin observed. The window is stored with the pin rather
|
|
1303
|
+
* than re-read because a memo hit is answered with NO daemon contact by design —
|
|
1304
|
+
* that is what makes a burst of turns free.
|
|
1305
|
+
*/
|
|
1306
|
+
function residentEntry(base, model, prefix, { dialer = fetch, now = Date.now() } = {}) {
|
|
1307
|
+
const hit = residentMemo.get(residentKey(base, model, prefix));
|
|
1308
|
+
if (!hit || hit.dialer !== dialer) return null;
|
|
1309
|
+
return hit.until - now > 0 ? hit : null;
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
/**
|
|
1313
|
+
* Does a `/api/ps` row name the model the user picked?
|
|
1314
|
+
*
|
|
1315
|
+
* Matched by prefix because the daemon reports the RESOLVED identity while the
|
|
1316
|
+
* row carries whatever the user chose: `llama3.2:1b` for a plain pull, and
|
|
1317
|
+
* `name:tag@sha256:…` for a digest-pinned or remotely-pulled one. The digest
|
|
1318
|
+
* form is why `@` is here and not only `:` — the rule this replaced claimed to
|
|
1319
|
+
* handle it in a comment while matching only the colon.
|
|
1320
|
+
*
|
|
1321
|
+
* One predicate rather than two copies: it was already duplicated between the
|
|
1322
|
+
* residency read and the window read, and a rule stated twice drifts.
|
|
1323
|
+
*/
|
|
1324
|
+
function sameModel(reported, picked) {
|
|
1325
|
+
const name = String(reported || '');
|
|
1326
|
+
const want = String(picked || '');
|
|
1327
|
+
if (!name || !want) return false;
|
|
1328
|
+
return name === want || name.startsWith(`${want}:`) || name.startsWith(`${want}@`);
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/**
|
|
1332
|
+
* How long the standalone window read (`windowOf`) waits. Short on purpose: it is
|
|
1333
|
+
* a loopback metadata call (measured at 115µs on this host) that runs BEFORE the
|
|
1334
|
+
* turn, so it must never become a place a turn can park. The probe's budget, not
|
|
1335
|
+
* the generation's — see `PROBE_TIMEOUT_MS` above for the same reasoning.
|
|
1336
|
+
*/
|
|
1337
|
+
const WINDOW_TIMEOUT_MS = 3000;
|
|
1338
|
+
|
|
1339
|
+
/**
|
|
1340
|
+
* The window of the LOADED runner for `model`, straight from `/api/ps`, or null.
|
|
1341
|
+
*
|
|
1342
|
+
* Never throws and never guesses — the same rule `environmentPreamble` follows
|
|
1343
|
+
* for a missing field. `/api/ps` is the only honest source: `/api/show` reports
|
|
1344
|
+
* the model's TRAINING capacity (`llama.context_length` 131072 for a model this
|
|
1345
|
+
* daemon runs at 4096), and fitting a turn to that would not fit it at all.
|
|
1346
|
+
*
|
|
1347
|
+
* This exists as a fallback rather than as the only read because the pin can fail
|
|
1348
|
+
* for reasons that say nothing about the window — a Stop, its own clock, a daemon
|
|
1349
|
+
* that will not enumerate models — and a turn that then sent its payload blind
|
|
1350
|
+
* would overrun the window and have its SYSTEM row shifted away by llama.cpp's
|
|
1351
|
+
* `--context-shift`. The answer to "how big is this window" should not depend on
|
|
1352
|
+
* a warm-up succeeding.
|
|
1353
|
+
*/
|
|
1354
|
+
async function windowOf(base, model, { signal } = {}) {
|
|
1355
|
+
// Bounded, because this runs BEFORE the turn and unwatched it is precisely the
|
|
1356
|
+
// hang this transport already had to fix once: a daemon that accepts the socket
|
|
1357
|
+
// and never answers would otherwise park the turn here, before `chatOne`'s own
|
|
1358
|
+
// first-byte deadline is even armed, and a request nothing is timing out is a
|
|
1359
|
+
// request nothing can fail. The budget is the probe's, not the turn's — this is
|
|
1360
|
+
// a loopback metadata call (measured at 115µs on this host) and it must not eat
|
|
1361
|
+
// the generation's clock.
|
|
1362
|
+
const ctl = new AbortController();
|
|
1363
|
+
const onCallerAbort = () => ctl.abort();
|
|
1364
|
+
if (signal) {
|
|
1365
|
+
if (signal.aborted) return null;
|
|
1366
|
+
if (typeof signal.addEventListener === 'function') signal.addEventListener('abort', onCallerAbort, { once: true });
|
|
1367
|
+
}
|
|
1368
|
+
const timer = setTimeout(() => ctl.abort(), WINDOW_TIMEOUT_MS);
|
|
1369
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
1370
|
+
try {
|
|
1371
|
+
const res = await fetch(`${base}/api/ps`, { method: 'GET', signal: ctl.signal });
|
|
1372
|
+
if (!res || !res.ok || typeof res.json !== 'function') return null;
|
|
1373
|
+
const body = await res.json();
|
|
1374
|
+
const models = (body && body.models) || [];
|
|
1375
|
+
const hit = models.find((m) => sameModel(m && (m.name || m.model), model));
|
|
1376
|
+
const win = hit && Number(hit.context_length);
|
|
1377
|
+
return Number.isFinite(win) && win > 0 ? win : null;
|
|
1378
|
+
} catch {
|
|
1379
|
+
return null;
|
|
1380
|
+
} finally {
|
|
1381
|
+
clearTimeout(timer);
|
|
1382
|
+
if (signal && typeof signal.removeEventListener === 'function') {
|
|
1383
|
+
signal.removeEventListener('abort', onCallerAbort);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/**
|
|
1389
|
+
* Make `model` resident on the local daemon and PIN its window — the extend
|
|
1390
|
+
* call this lane was missing.
|
|
1391
|
+
*
|
|
1392
|
+
* Why this exists, and why the arguments are what they are. The cost of a local
|
|
1393
|
+
* turn is a LOAD cost, not a turn cost: a cold first call pays for reading the
|
|
1394
|
+
* weights off disk, and every call after it is fast because llama.cpp reuses the
|
|
1395
|
+
* KV of the shared prefix. Without a pin this lane ran on the daemon's stock
|
|
1396
|
+
* 5-minute window and had no way to extend it: `keep_alive` on
|
|
1397
|
+
* `/v1/chat/completions` is accepted and IGNORED (measured, engine-side:
|
|
1398
|
+
* 0/120/1800 all yield the same stored window), so the ONLY endpoint that can
|
|
1399
|
+
* extend residency is Ollama's native `/api/chat`. Step away for six minutes and
|
|
1400
|
+
* the next turn re-bought the entire cold load.
|
|
1401
|
+
*
|
|
1402
|
+
* `system` is the prefix this pin warms, and it MUST be the turn's own system
|
|
1403
|
+
* text — the caller passes `opts.system` straight through from the engine's
|
|
1404
|
+
* dispatch, which is the exact string the turn is about to send. Warming
|
|
1405
|
+
* anything else is worse than warming nothing: the daemon uses the REQUEST's
|
|
1406
|
+
* system message in place of the model's baked Modelfile default, so a pin
|
|
1407
|
+
* carrying different text diverges from the turn at the first token, shares no
|
|
1408
|
+
* prefix with it, and EVICTS the very cache it was meant to warm. This is not a
|
|
1409
|
+
* theory — it is the measured failure mode that made the engine-side pin
|
|
1410
|
+
* useless (a mismatched pin cost 1.30s and left the following real turn at
|
|
1411
|
+
* 2.26s, where a matching pin cost 0.53s and left it at 0.58s). So: no
|
|
1412
|
+
* fabricated "you are a local model" line, no Modelfile guess. Whatever the
|
|
1413
|
+
* caller says, or nothing.
|
|
1414
|
+
*
|
|
1415
|
+
* Three measured constraints are honoured by the order of operations below:
|
|
1416
|
+
* * a memo hit is answered with NO daemon contact and no slot (it is free);
|
|
1417
|
+
* * a daemon that already holds an adequate window needs no pin (one GET);
|
|
1418
|
+
* * a pin that is not needed is not issued, because its prefill is the cost.
|
|
1419
|
+
*
|
|
1420
|
+
* Never throws and never rejects: the caller is a live turn, and "could not
|
|
1421
|
+
* warm" is information, not a failure. A pin that fails is simply not memoized,
|
|
1422
|
+
* so the next turn retries instead of trusting a lie.
|
|
1423
|
+
*/
|
|
1424
|
+
async function ensureResident({ baseURL = DEFAULT_BASE, model, system = '', signal, warmSecs, firstByteTimeoutMs } = {}) {
|
|
1425
|
+
const base = baseOf(baseURL);
|
|
1426
|
+
// The fence covers the warm-up too. `unload` states the same rule ("never
|
|
1427
|
+
// dial out, even to clean up"); a helper that dials where the turn may not
|
|
1428
|
+
// would be the hole in it. The turn itself (`chatOne`) still does the
|
|
1429
|
+
// refusing and erroring — this just does not add a second caller.
|
|
1430
|
+
if (!isLocalEndpoint(base)) return { ok: false, skipped: 'remote' };
|
|
1431
|
+
if (!model || !String(model).trim()) return { ok: false, skipped: 'no-model' };
|
|
1432
|
+
|
|
1433
|
+
const want = warmSecsFor({ warmSecs });
|
|
1434
|
+
const prefix = typeof system === 'string' ? system : '';
|
|
1435
|
+
const prefixSource = prefix ? 'turn-system' : 'none';
|
|
1436
|
+
const shape = { warmSecs: want, prefixSource, prefixChars: prefix.length };
|
|
1437
|
+
if (want <= 0) return { ok: false, skipped: 'warm-disabled', ...shape };
|
|
1438
|
+
|
|
1439
|
+
const dialer = fetch;
|
|
1440
|
+
// Free answer first: a window this process set, still carrying seconds, for
|
|
1441
|
+
// THIS prefix. Consulted before any request, so a burst of turns costs
|
|
1442
|
+
// nothing at all rather than a loopback GET each.
|
|
1443
|
+
const memoLeft = residentSecsLeft(base, model, prefix, { dialer });
|
|
1444
|
+
if (memoLeft >= Math.min(want, 60)) {
|
|
1445
|
+
const entry = residentEntry(base, model, prefix, { dialer });
|
|
1446
|
+
return {
|
|
1447
|
+
ok: true,
|
|
1448
|
+
already: true,
|
|
1449
|
+
memo: true,
|
|
1450
|
+
remainingSecs: memoLeft,
|
|
1451
|
+
contextLength: (entry && entry.contextLength) || null,
|
|
1452
|
+
...shape,
|
|
1453
|
+
};
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
// The caller's Stop must reach the warm-up too: a user who cancels should not
|
|
1457
|
+
// leave a 300s request running behind them.
|
|
1458
|
+
const ctl = new AbortController();
|
|
1459
|
+
const onCallerAbort = () => ctl.abort();
|
|
1460
|
+
if (signal) {
|
|
1461
|
+
if (signal.aborted) ctl.abort();
|
|
1462
|
+
else if (typeof signal.addEventListener === 'function') {
|
|
1463
|
+
signal.addEventListener('abort', onCallerAbort, { once: true });
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
// The pin's own clock — ONE timer for the whole pin (the residency GET and
|
|
1467
|
+
// the warm-up POST share it), so a daemon that answers one and stalls the
|
|
1468
|
+
// other cannot spend the budget twice. A caller that STATED a window is
|
|
1469
|
+
// honoured exactly, never widened: the turn must fail on the caller's clock,
|
|
1470
|
+
// not on the pin's.
|
|
1471
|
+
const stated = Number(firstByteTimeoutMs);
|
|
1472
|
+
const budgetMs = Number.isFinite(stated) && stated > 0 ? Math.min(stated, RESIDENT_TIMEOUT_MS) : RESIDENT_TIMEOUT_MS;
|
|
1473
|
+
let pinTimedOut = false;
|
|
1474
|
+
// The window of the LOADED runner, read from `/api/ps` below and handed back
|
|
1475
|
+
// to the caller so the turn it warms can be fitted to it. `/api/ps` is the
|
|
1476
|
+
// only honest source: `/api/show` reports the model's training capacity
|
|
1477
|
+
// (`llama.context_length` 131072 on a model running at 4096), and a window
|
|
1478
|
+
// guessed from that is worse than no window at all.
|
|
1479
|
+
let daemonContext = null;
|
|
1480
|
+
const timer = setTimeout(() => {
|
|
1481
|
+
pinTimedOut = true;
|
|
1482
|
+
ctl.abort();
|
|
1483
|
+
}, budgetMs);
|
|
1484
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
1485
|
+
|
|
1486
|
+
try {
|
|
1487
|
+
// Is the daemon already holding this model long enough? One GET, and a
|
|
1488
|
+
// "yes" is worth the whole pin: the prefill is the cost we are avoiding.
|
|
1489
|
+
try {
|
|
1490
|
+
const ps = await fetch(`${base}/api/ps`, { method: 'GET', signal: ctl.signal });
|
|
1491
|
+
if (ps && ps.ok && typeof ps.json === 'function') {
|
|
1492
|
+
const body = await ps.json();
|
|
1493
|
+
const models = (body && body.models) || [];
|
|
1494
|
+
// Matched by the shared predicate: /api/ps reports the RESOLVED
|
|
1495
|
+
// identity ("llama3.2:1b", or "name:tag@sha256:…" for a digest-pinned
|
|
1496
|
+
// pull) while the row carries whatever the user picked.
|
|
1497
|
+
const hit = models.find((m) => sameModel(m && (m.name || m.model), model));
|
|
1498
|
+
// The loaded runner's window, when the daemon states one.
|
|
1499
|
+
if (hit && Number.isFinite(hit.context_length) && hit.context_length > 0) {
|
|
1500
|
+
daemonContext = hit.context_length;
|
|
1501
|
+
}
|
|
1502
|
+
if (hit && hit.expires_at) {
|
|
1503
|
+
const left = Math.round((new Date(hit.expires_at).getTime() - Date.now()) / 1000);
|
|
1504
|
+
if (Number.isFinite(left) && left >= Math.min(want, 60)) {
|
|
1505
|
+
// A proven window is worth hinting to the memo — but only for the
|
|
1506
|
+
// prefix the caller is about to send, which is the one key this
|
|
1507
|
+
// hint is filed under.
|
|
1508
|
+
residentMemo.set(residentKey(base, model, prefix), { until: Date.now() + left * 1000, dialer, contextLength: daemonContext });
|
|
1509
|
+
return { ok: true, already: true, remainingSecs: left, contextLength: daemonContext, ...shape };
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
} catch {
|
|
1514
|
+
/* a daemon that will not enumerate is pinned blind, below */
|
|
1515
|
+
}
|
|
1516
|
+
if (ctl.signal.aborted) {
|
|
1517
|
+
// The caller's Stop, or the pin's own clock. Neither is a turn failure:
|
|
1518
|
+
// the turn below runs and reports its OWN outcome, on the caller's terms.
|
|
1519
|
+
return { ok: false, aborted: true, timeout: pinTimedOut, contextLength: daemonContext, ...shape };
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
const messages = prefix
|
|
1523
|
+
? [{ role: 'system', content: prefix }, { role: 'user', content: 'PONG' }]
|
|
1524
|
+
: [{ role: 'user', content: 'PONG' }];
|
|
1525
|
+
const res = await fetch(`${base}/api/chat`, {
|
|
1526
|
+
method: 'POST',
|
|
1527
|
+
headers: { 'content-type': 'application/json' },
|
|
1528
|
+
body: JSON.stringify({
|
|
1529
|
+
model: String(model),
|
|
1530
|
+
// The native field that actually works — see this function's note.
|
|
1531
|
+
keep_alive: want,
|
|
1532
|
+
stream: false,
|
|
1533
|
+
messages,
|
|
1534
|
+
// One token: enough to force the full prefill of the prefix, cheap
|
|
1535
|
+
// enough that the generation half of the call is free.
|
|
1536
|
+
options: { num_predict: 1 },
|
|
1537
|
+
}),
|
|
1538
|
+
signal: ctl.signal,
|
|
1539
|
+
});
|
|
1540
|
+
|
|
1541
|
+
// Read the body AND keep the daemon's own accounting from it. Ollama does
|
|
1542
|
+
// not bill a pin, but it does REPORT one, and `prompt_eval_count` is the
|
|
1543
|
+
// only honest measure of what the extend cost — throwing it away makes "is
|
|
1544
|
+
// the pin expensive?" a matter of opinion rather than a number.
|
|
1545
|
+
let promptTokens = null;
|
|
1546
|
+
let outputTokens = null;
|
|
1547
|
+
if (res && typeof res.text === 'function') {
|
|
1548
|
+
const raw = await res.text().catch(() => '');
|
|
1549
|
+
try {
|
|
1550
|
+
const parsed = JSON.parse(raw);
|
|
1551
|
+
if (Number.isFinite(parsed.prompt_eval_count)) promptTokens = parsed.prompt_eval_count;
|
|
1552
|
+
if (Number.isFinite(parsed.eval_count)) outputTokens = parsed.eval_count;
|
|
1553
|
+
} catch {
|
|
1554
|
+
/* a non-JSON body is still a successful pin */
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
if (res && res.ok) {
|
|
1559
|
+
// A COLD load is why this re-read exists. The window is read from
|
|
1560
|
+
// `/api/ps` at the top of this function, and on a cold load the daemon has
|
|
1561
|
+
// no runner to report — the pin's own request is what loads it, so the
|
|
1562
|
+
// window was unknown at the moment it was asked for and the turn that
|
|
1563
|
+
// follows would have gone out unfitted. Measured live: nothing loaded ->
|
|
1564
|
+
// the pin reported `contextLength: null` while returning `ok: true`, and
|
|
1565
|
+
// the turn sent 4516 estimated tokens at a window of 4096. One GET after
|
|
1566
|
+
// the load closes it, on the cold path only.
|
|
1567
|
+
if (daemonContext == null) {
|
|
1568
|
+
daemonContext = (await windowOf(base, model, { signal: ctl.signal })) || null;
|
|
1569
|
+
}
|
|
1570
|
+
residentMemo.set(residentKey(base, model, prefix), { until: Date.now() + want * 1000, dialer, contextLength: daemonContext });
|
|
1571
|
+
// Record it as this app's warm model the moment the daemon ACCEPTS it,
|
|
1572
|
+
// not only after a successful turn. Otherwise a turn that fails right
|
|
1573
|
+
// after its pin (a 400, a Stop, a full context window) leaves a loaded
|
|
1574
|
+
// model that `unloadAll` never learned about — exactly the leak
|
|
1575
|
+
// `unloadAll` exists to close. `swapWarmModel` is idempotent, so the
|
|
1576
|
+
// post-turn call is still the one that keeps the "only a successful turn
|
|
1577
|
+
// moves the warm model" contract for the SWITCH this makes.
|
|
1578
|
+
swapWarmModel(base, model);
|
|
1579
|
+
}
|
|
1580
|
+
return {
|
|
1581
|
+
ok: !!(res && res.ok),
|
|
1582
|
+
status: res && res.status,
|
|
1583
|
+
pinned: true,
|
|
1584
|
+
promptTokens,
|
|
1585
|
+
outputTokens,
|
|
1586
|
+
// The window the turn that follows will actually be given, so it can be
|
|
1587
|
+
// fitted to it. Null when the daemon did not state one.
|
|
1588
|
+
contextLength: daemonContext,
|
|
1589
|
+
...shape,
|
|
1590
|
+
};
|
|
1591
|
+
} catch (e) {
|
|
1592
|
+
// Never fatal. A turn is better than a pin, always — the caller runs the
|
|
1593
|
+
// turn regardless of what this returns.
|
|
1594
|
+
return { ok: false, error: String((e && e.message) || e), timeout: pinTimedOut, contextLength: daemonContext, ...shape };
|
|
1595
|
+
} finally {
|
|
1596
|
+
clearTimeout(timer);
|
|
1597
|
+
if (signal && typeof signal.removeEventListener === 'function') {
|
|
1598
|
+
signal.removeEventListener('abort', onCallerAbort);
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
/**
|
|
1604
|
+
* The result a turn the user STOPPED resolves to. Every transport the engine
|
|
1605
|
+
* dispatches to resolves an aborted request with an empty, well-formed answer
|
|
1606
|
+
* (see the engine's own guard: "an aborted request resolves with exactly that —
|
|
1607
|
+
* no text"), and this makes the local lane do the same instead of rejecting
|
|
1608
|
+
* with the transport's raw AbortError.
|
|
1609
|
+
*
|
|
1610
|
+
* `content` carries whatever had already streamed, because every other
|
|
1611
|
+
* transport returns the partial it collected and the renderer has already
|
|
1612
|
+
* painted those deltas — dropping them here would make the transcript
|
|
1613
|
+
* disagree with the bubble. Tool calls are deliberately NOT replayed: a turn
|
|
1614
|
+
* that was stopped is over, and the engine drops tools for an aborted turn
|
|
1615
|
+
* anyway, so listing them would only put an unexecuted call in the transcript.
|
|
1616
|
+
*/
|
|
1617
|
+
function stoppedTurn(model, { content = '', usage = null } = {}) {
|
|
1618
|
+
const result = {
|
|
1619
|
+
id: 'local',
|
|
1620
|
+
model: String(model),
|
|
1621
|
+
choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],
|
|
1622
|
+
};
|
|
1623
|
+
if (usage) result.usage = usage;
|
|
1624
|
+
return result;
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
/**
|
|
1628
|
+
* Keep a turn's wire payload inside the window the daemon actually granted.
|
|
1629
|
+
*
|
|
1630
|
+
* Ollama's OpenAI shim is the endpoint this transport talks to, and it can
|
|
1631
|
+
* ACCEPT `num_ctx` without honouring it. Measured on this host: `options.num_ctx`
|
|
1632
|
+
* AND a top-level `num_ctx` on `/v1/chat/completions` both returned 200 and left
|
|
1633
|
+
* the loaded window untouched at 4096, while native `/api/chat` moved it
|
|
1634
|
+
* 4096 → 8192 — by RELOADING the runner. Asking this lane for a bigger window
|
|
1635
|
+
* therefore does not work, and asking for one that the next `/v1` turn
|
|
1636
|
+
* contradicts is worse than not asking: `/api/ps` shows the runner respawn
|
|
1637
|
+
* (pid 200467 at n_ctx 8192 → pid 200484 at 4096), and a respawn discards the
|
|
1638
|
+
* very prefix the pin just warmed.
|
|
1639
|
+
*
|
|
1640
|
+
* What remains is the other half of the same problem — not OVERRUNNING the
|
|
1641
|
+
* window we are given. llama-server runs with `--context-shift --keep 4` (its own
|
|
1642
|
+
* command line, read from the daemon log), which does not fail on an oversized
|
|
1643
|
+
* prompt: it silently discards the OLDEST tokens to make room. The oldest tokens
|
|
1644
|
+
* are the system row — the persona, the work rules and the tool schemas — so an
|
|
1645
|
+
* overrun costs the model its instructions mid-turn and it answers as though it
|
|
1646
|
+
* had never been given any. That is what "ollama is broken" looked like.
|
|
1647
|
+
*
|
|
1648
|
+
* The window is READ, never guessed. `/api/show` is not a source for it: it
|
|
1649
|
+
* reports the model's TRAINING capacity (`llama.context_length` 131072 on a
|
|
1650
|
+
* model this daemon runs at 4096). The only honest source is `/api/ps`, which
|
|
1651
|
+
* reports the window of the LOADED runner — and `ensureResident` already makes
|
|
1652
|
+
* that request. No reported window means no trimming and no guessing, which is
|
|
1653
|
+
* today's behaviour.
|
|
1654
|
+
*
|
|
1655
|
+
* Returns null when there is no window to fit against, so the caller can leave
|
|
1656
|
+
* the payload exactly as it was.
|
|
1657
|
+
*/
|
|
1658
|
+
function fitToWindowOf({ rows, system, tools, contextLength, maxTokens }) {
|
|
1659
|
+
const win = Number(contextLength);
|
|
1660
|
+
if (!Number.isFinite(win) || win <= 0) return null;
|
|
1661
|
+
const history = (Array.isArray(rows) ? rows : []).filter((r) => r && r.role !== 'system');
|
|
1662
|
+
// Tool schemas are wire material the model has to see, so they are charged
|
|
1663
|
+
// before any history is kept. Otherwise the trimmer believes a 4096-token
|
|
1664
|
+
// window has 4096 tokens free when ~1971 of them are already spoken for
|
|
1665
|
+
// (measured: 10 schemas, 7882 chars ≈ 1971 tokens — almost half the window,
|
|
1666
|
+
// spent before the user's first word).
|
|
1667
|
+
const fixedTokens = Array.isArray(tools) && tools.length ? estimateTokens(JSON.stringify(tools)) : 0;
|
|
1668
|
+
// The reply needs room to exist. A stated cap is honoured exactly; with none,
|
|
1669
|
+
// a quarter of the window — a turn that fills the window exactly leaves
|
|
1670
|
+
// nowhere to emit into, and the daemon starts shifting the prompt to write.
|
|
1671
|
+
const stated = Number(maxTokens);
|
|
1672
|
+
const reserveTokens = Number.isFinite(stated) && stated > 0 ? stated : Math.round(win / 4);
|
|
1673
|
+
const trimmed = trimContext({
|
|
1674
|
+
messages: history,
|
|
1675
|
+
system: typeof system === 'string' ? system : '',
|
|
1676
|
+
budgetTokens: win,
|
|
1677
|
+
reserveTokens,
|
|
1678
|
+
fixedTokens,
|
|
1679
|
+
});
|
|
1680
|
+
return {
|
|
1681
|
+
window: win,
|
|
1682
|
+
reserveTokens,
|
|
1683
|
+
toolTokens: fixedTokens,
|
|
1684
|
+
dropped: trimmed.dropped,
|
|
1685
|
+
messages: trimmed.messages,
|
|
1686
|
+
// Still over after dropping every turn it could: the newest turn alone does
|
|
1687
|
+
// not fit. Reported rather than hidden — the daemon will shift the prompt
|
|
1688
|
+
// instead, and which turn did it is worth knowing.
|
|
1689
|
+
over: trimmed.estimatedTokens > Math.max(0, win - reserveTokens),
|
|
1690
|
+
estimatedTokens: trimmed.estimatedTokens,
|
|
1691
|
+
};
|
|
1692
|
+
}
|
|
1693
|
+
|
|
585
1694
|
/**
|
|
586
1695
|
* One streaming turn against a local model. Keyless by construction — there is
|
|
587
1696
|
* no credential to send, which is a second reason a remote URL here could never
|
|
@@ -590,7 +1699,7 @@ async function readSSE(res, onEvent, { idleTimeoutMs = SSE_IDLE_TIMEOUT_MS } = {
|
|
|
590
1699
|
* Returns the OpenAI shape the engine reads: `choices[0].message.content` and
|
|
591
1700
|
* `choices[0].message.tool_calls`.
|
|
592
1701
|
*/
|
|
593
|
-
async function
|
|
1702
|
+
async function chatOne({
|
|
594
1703
|
baseURL = DEFAULT_BASE,
|
|
595
1704
|
model,
|
|
596
1705
|
messages,
|
|
@@ -602,6 +1711,11 @@ async function chat({
|
|
|
602
1711
|
toolChoice,
|
|
603
1712
|
signal,
|
|
604
1713
|
onDelta,
|
|
1714
|
+
firstByteTimeoutMs = FIRST_BYTE_TIMEOUT_MS,
|
|
1715
|
+
// The window the daemon actually granted, read from `/api/ps` by the pin
|
|
1716
|
+
// (`ensureResident`) and threaded through here. Absent when the daemon did not
|
|
1717
|
+
// report one — in which case nothing is trimmed and nothing is guessed.
|
|
1718
|
+
contextLength,
|
|
605
1719
|
} = {}) {
|
|
606
1720
|
const base = baseOf(baseURL);
|
|
607
1721
|
const refusal = remoteRefusal(base);
|
|
@@ -635,6 +1749,21 @@ async function chat({
|
|
|
635
1749
|
throw err;
|
|
636
1750
|
}
|
|
637
1751
|
|
|
1752
|
+
// Fit the turn to the window the daemon actually granted. See
|
|
1753
|
+
// `fitToWindowOf` for why this is the transport's job rather than the
|
|
1754
|
+
// caller's, and why the window is read rather than requested.
|
|
1755
|
+
let fit = null;
|
|
1756
|
+
if (contextLength != null) {
|
|
1757
|
+
const sysRow = rows[0] && rows[0].role === 'system' ? rows[0] : null;
|
|
1758
|
+
fit = fitToWindowOf({ rows, system: (sysRow && sysRow.content) || system || '', tools, contextLength, maxTokens });
|
|
1759
|
+
if (fit) {
|
|
1760
|
+
const kept = fit.messages;
|
|
1761
|
+
rows.length = 0;
|
|
1762
|
+
if (sysRow) rows.push(sysRow);
|
|
1763
|
+
rows.push(...kept);
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
|
|
638
1767
|
const body = {
|
|
639
1768
|
model: String(model),
|
|
640
1769
|
messages: rows,
|
|
@@ -648,12 +1777,72 @@ async function chat({
|
|
|
648
1777
|
...(toolChoice ? { tool_choice: toolChoice } : {}),
|
|
649
1778
|
};
|
|
650
1779
|
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
1780
|
+
// The caller's abort is FORWARDED to our own controller rather than handed
|
|
1781
|
+
// straight to `fetch`, so one request can carry both the caller's intent and
|
|
1782
|
+
// the first-byte deadline below. It stays attached for the whole turn, not
|
|
1783
|
+
// just the header wait: aborting after headers is what tears down the body
|
|
1784
|
+
// mid-stream, which is how Stop reaches a generation already in flight.
|
|
1785
|
+
const ctl = new AbortController();
|
|
1786
|
+
let firstByteTimedOut = false;
|
|
1787
|
+
const firstByteTimer = setTimeout(() => {
|
|
1788
|
+
firstByteTimedOut = true;
|
|
1789
|
+
ctl.abort();
|
|
1790
|
+
}, firstByteTimeoutMs);
|
|
1791
|
+
// Never the reason a short-lived CLI process (`aegiscode -p "…"`) refuses to
|
|
1792
|
+
// exit, same rule the cloud client's boundedFetch follows.
|
|
1793
|
+
if (firstByteTimer && typeof firstByteTimer.unref === 'function') firstByteTimer.unref();
|
|
1794
|
+
const onCallerAbort = () => ctl.abort();
|
|
1795
|
+
if (signal) {
|
|
1796
|
+
if (signal.aborted) ctl.abort();
|
|
1797
|
+
else if (typeof signal.addEventListener === 'function') {
|
|
1798
|
+
signal.addEventListener('abort', onCallerAbort, { once: true });
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
let res;
|
|
1803
|
+
try {
|
|
1804
|
+
res = await fetch(`${base}/v1/chat/completions`, {
|
|
1805
|
+
method: 'POST',
|
|
1806
|
+
headers: { 'content-type': 'application/json' },
|
|
1807
|
+
body: JSON.stringify(body),
|
|
1808
|
+
signal: ctl.signal,
|
|
1809
|
+
});
|
|
1810
|
+
} catch (err) {
|
|
1811
|
+
if (firstByteTimedOut || isHeadersDeadline(err)) {
|
|
1812
|
+
// Two different clocks can end this wait, and the user should be told
|
|
1813
|
+
// which one did — see `isHeadersDeadline` for the second, which is not
|
|
1814
|
+
// ours and was reaching the host as an opaque `TypeError: fetch failed`.
|
|
1815
|
+
const whose = firstByteTimedOut
|
|
1816
|
+
? `sent nothing in ${Math.round(firstByteTimeoutMs / 1000)}s`
|
|
1817
|
+
: `sent no response headers before this runtime's own ${Math.round(HEADERS_DEADLINE_MS / 1000)}s deadline`;
|
|
1818
|
+
const e = new Error(
|
|
1819
|
+
`local: ${model} ${whose} — the daemon is still loading the model, or its prompt ` +
|
|
1820
|
+
'prefill is longer than that. `ollama ps` shows which; a smaller prompt or a smaller ' +
|
|
1821
|
+
'model is the way through.'
|
|
1822
|
+
);
|
|
1823
|
+
// The shape the cloud client gives its own timeout (status 0 + a code),
|
|
1824
|
+
// so "waited too long" stays one branch in the renderer for every class.
|
|
1825
|
+
e.status = 0;
|
|
1826
|
+
e.code = 'timeout';
|
|
1827
|
+
throw e;
|
|
1828
|
+
}
|
|
1829
|
+
// A STOPPED turn is not a failed one, and it must not look like one. The
|
|
1830
|
+
// engine states this contract for every transport it dispatches to
|
|
1831
|
+
// (engine.js: "an aborted request resolves with exactly that — no text",
|
|
1832
|
+
// the guard that stops an interrupted turn from billing a second call).
|
|
1833
|
+
// The cloud client satisfies it; this transport rejected with the raw
|
|
1834
|
+
// AbortError instead, so pressing Stop on a local turn surfaced in the host
|
|
1835
|
+
// as `Error occurred in handler for 'model:chat': DOMException [AbortError]`
|
|
1836
|
+
// — an error dialog and a stack for the one action the user chose. Nothing
|
|
1837
|
+
// was produced to preserve here: no byte of the response was ever read.
|
|
1838
|
+
if ((signal && signal.aborted) || ctl.signal.aborted) return stoppedTurn(model);
|
|
1839
|
+
throw err;
|
|
1840
|
+
} finally {
|
|
1841
|
+
// Time-to-headers only, the same contract boundedFetch documents: once the
|
|
1842
|
+
// headers are here the per-payload stall timer takes over, so a
|
|
1843
|
+
// legitimately long generation is never cut off by this one.
|
|
1844
|
+
clearTimeout(firstByteTimer);
|
|
1845
|
+
}
|
|
657
1846
|
|
|
658
1847
|
if (!res.ok) {
|
|
659
1848
|
// Body text, not just the status: a local daemon's 400 says which field it
|
|
@@ -675,46 +1864,73 @@ async function chat({
|
|
|
675
1864
|
const toolCalls = [];
|
|
676
1865
|
const byIndex = new Map();
|
|
677
1866
|
const stripEnvelope = makeDeepSeekEnvelopeStripper();
|
|
1867
|
+
// Text-form calls the templates emit that Ollama's shim does not structure
|
|
1868
|
+
// for us (see makeTextToolCallStripper). Chained AFTER the DSML pass: DSML
|
|
1869
|
+
// parameter bodies are swallowed by that pass, so a file being written can
|
|
1870
|
+
// never be scanned — and misread — as a call. The offered tool names are
|
|
1871
|
+
// handed over as the guard on the bare-JSON rule: only a name this turn
|
|
1872
|
+
// actually offered is allowed to turn text into a call.
|
|
1873
|
+
const stripTextCalls = makeTextToolCallStripper({
|
|
1874
|
+
toolNames: (Array.isArray(tools) ? tools : [])
|
|
1875
|
+
.map((t) => (t && t.function && t.function.name) || (t && t.name) || '')
|
|
1876
|
+
.filter(Boolean),
|
|
1877
|
+
});
|
|
678
1878
|
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
1879
|
+
try {
|
|
1880
|
+
await readSSE(
|
|
1881
|
+
res,
|
|
1882
|
+
(json) => {
|
|
1883
|
+
if (json && json.usage) usage = json.usage;
|
|
1884
|
+
const choice = json && Array.isArray(json.choices) ? json.choices[0] : null;
|
|
1885
|
+
if (!choice) return;
|
|
1886
|
+
const delta = choice.delta || {};
|
|
1887
|
+
if (typeof delta.content === 'string' && delta.content) {
|
|
1888
|
+
const clean = stripTextCalls(stripEnvelope(delta.content));
|
|
1889
|
+
if (clean) {
|
|
1890
|
+
content += clean;
|
|
1891
|
+
if (typeof onDelta === 'function') onDelta({ text: clean });
|
|
1892
|
+
}
|
|
691
1893
|
}
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
1894
|
+
const frags = delta.tool_calls || [];
|
|
1895
|
+
for (const f of frags) {
|
|
1896
|
+
const i = Number.isInteger(f.index) ? f.index : toolCalls.length;
|
|
1897
|
+
let acc = byIndex.get(i);
|
|
1898
|
+
if (!acc) {
|
|
1899
|
+
acc = { id: '', type: 'function', function: { name: '', arguments: '' } };
|
|
1900
|
+
byIndex.set(i, acc);
|
|
1901
|
+
}
|
|
1902
|
+
if (f.id) acc.id = f.id;
|
|
1903
|
+
if (f.type) acc.type = f.type;
|
|
1904
|
+
const fn = f.function || {};
|
|
1905
|
+
if (fn.name) acc.function.name = fn.name;
|
|
1906
|
+
if (fn.arguments) acc.function.arguments += fn.arguments;
|
|
700
1907
|
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
1908
|
+
},
|
|
1909
|
+
{ idleTimeoutMs: SSE_IDLE_TIMEOUT_MS }
|
|
1910
|
+
);
|
|
1911
|
+
} catch (err) {
|
|
1912
|
+
// Stopped mid-stream. The forwarded abort tears the body down, so the read
|
|
1913
|
+
// loop rejects with AbortError — resolve with what already streamed
|
|
1914
|
+
// instead of turning the user's own Stop into a host-side error, and let
|
|
1915
|
+
// the tail below stay unreached (an aborted stream has no unfinished
|
|
1916
|
+
// envelope worth flushing into an answer the user cancelled).
|
|
1917
|
+
if (signal && signal.aborted) return stoppedTurn(model, { content, usage });
|
|
1918
|
+
throw err;
|
|
1919
|
+
} finally {
|
|
1920
|
+
if (signal && typeof signal.removeEventListener === 'function') {
|
|
1921
|
+
signal.removeEventListener('abort', onCallerAbort);
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
710
1924
|
|
|
711
1925
|
// A trailing '|…' that never closed (stream ended right after it, or it
|
|
712
1926
|
// was real prose all along) is real content, not an unfinished envelope
|
|
713
1927
|
// token — release it rather than dropping it silently.
|
|
714
|
-
const strayTail = stripEnvelope.flush();
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
1928
|
+
const strayTail = stripTextCalls(stripEnvelope.flush());
|
|
1929
|
+
const textTail = stripTextCalls.flush();
|
|
1930
|
+
const tail = `${strayTail || ''}${textTail || ''}`;
|
|
1931
|
+
if (tail) {
|
|
1932
|
+
content += tail;
|
|
1933
|
+
if (typeof onDelta === 'function') onDelta({ text: tail });
|
|
718
1934
|
}
|
|
719
1935
|
|
|
720
1936
|
for (const [i, acc] of [...byIndex.entries()].sort((a, b) => a[0] - b[0])) {
|
|
@@ -736,33 +1952,181 @@ async function chat({
|
|
|
736
1952
|
// at all) is executed like any other. engine.js's extractToolCalls() reads
|
|
737
1953
|
// this field first, ahead of message.tool_calls.
|
|
738
1954
|
const dsmlCalls = stripEnvelope.calls();
|
|
1955
|
+
// Calls the model wrote as text rather than in the structured field — the
|
|
1956
|
+
// same merge, because from here on they ARE tool calls: the engine's
|
|
1957
|
+
// extractToolCalls() reads this field first, ahead of message.tool_calls.
|
|
1958
|
+
const textCalls = stripTextCalls.calls();
|
|
739
1959
|
const result = {
|
|
740
1960
|
id: 'local',
|
|
741
1961
|
model: String(model),
|
|
742
1962
|
choices: [{ index: 0, message, finish_reason: toolCalls.length ? 'tool_calls' : 'stop' }],
|
|
743
1963
|
...(usage ? { usage } : {}),
|
|
744
1964
|
};
|
|
745
|
-
if (toolCalls.length || dsmlCalls.length) {
|
|
1965
|
+
if (toolCalls.length || dsmlCalls.length || textCalls.length) {
|
|
746
1966
|
result.toolCalls = [
|
|
747
1967
|
...toolCalls.map((c) => ({ id: c.id, name: c.function.name, args: c.function.arguments })),
|
|
748
1968
|
...dsmlCalls.map((c) => ({ id: '', name: c.name, args: c.args || {} })),
|
|
1969
|
+
...textCalls.map((c) => ({ id: '', name: c.name, args: c.args || {} })),
|
|
749
1970
|
];
|
|
750
|
-
if (dsmlCalls.length && result.choices[0].finish_reason !== 'tool_calls') {
|
|
1971
|
+
if ((dsmlCalls.length || textCalls.length) && result.choices[0].finish_reason !== 'tool_calls') {
|
|
751
1972
|
result.choices[0].finish_reason = 'tool_calls';
|
|
752
1973
|
}
|
|
753
1974
|
}
|
|
1975
|
+
// What the fit cost, reported rather than hidden: the window the daemon
|
|
1976
|
+
// granted, the fixed wire cost the tool schemas took out of it, and how much
|
|
1977
|
+
// history had to be dropped to stay inside. `over` is the honest failure — the
|
|
1978
|
+
// newest turn alone does not fit, so the daemon will shift the prompt instead.
|
|
1979
|
+
if (fit) {
|
|
1980
|
+
result.window = {
|
|
1981
|
+
contextLength: fit.window,
|
|
1982
|
+
dropped: fit.dropped,
|
|
1983
|
+
toolTokens: fit.toolTokens,
|
|
1984
|
+
estimatedTokens: fit.estimatedTokens,
|
|
1985
|
+
reserveTokens: fit.reserveTokens,
|
|
1986
|
+
over: fit.over,
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
754
1989
|
return result;
|
|
755
1990
|
}
|
|
756
1991
|
|
|
1992
|
+
/**
|
|
1993
|
+
* Public entry point: `chatOne` serialized against every other call that
|
|
1994
|
+
* would land on the same daemon+model slot (see `runSerialized` above). A
|
|
1995
|
+
* caller that wants true concurrency points at a different model or a
|
|
1996
|
+
* different daemon — both get their own key and run in parallel.
|
|
1997
|
+
*
|
|
1998
|
+
* Also tracks how many calls are live for this key (`activeCountByKey`) and,
|
|
1999
|
+
* once this one succeeds, swaps the daemon's warm model over to it — see
|
|
2000
|
+
* `swapWarmModel`. The count is kept here rather than inside `runSerialized`
|
|
2001
|
+
* because it has to span the whole call (queueing included): a model is
|
|
2002
|
+
* still "in use" while queued behind another turn on the same slot, not only
|
|
2003
|
+
* while its own fetch is running.
|
|
2004
|
+
*/
|
|
2005
|
+
async function chat(opts = {}) {
|
|
2006
|
+
const base = baseOf(opts && opts.baseURL);
|
|
2007
|
+
const model = opts && opts.model;
|
|
2008
|
+
const key = `${base}|${model}`;
|
|
2009
|
+
activeCountByKey.set(key, (activeCountByKey.get(key) || 0) + 1);
|
|
2010
|
+
try {
|
|
2011
|
+
// The pin runs INSIDE the same slot as the turn it warms, deliberately.
|
|
2012
|
+
// Two reasons, both load-bearing:
|
|
2013
|
+
// * the daemon's single request slot is per model (see `runSerialized`),
|
|
2014
|
+
// so a pin issued beside the turn rather than before it would be the
|
|
2015
|
+
// second overlapping request that hangs the loser;
|
|
2016
|
+
// * a pin that warmed the prefix and then released the slot, letting
|
|
2017
|
+
// another call's model in, would warm a cache that is gone by the time
|
|
2018
|
+
// this turn sends its first byte.
|
|
2019
|
+
// `pin` is held in the closure so the turn's own error path is untouched:
|
|
2020
|
+
// a failed pin never becomes a failed turn.
|
|
2021
|
+
let pin = null;
|
|
2022
|
+
const result = await runSerialized(key, async () => {
|
|
2023
|
+
pin = await ensureResident({
|
|
2024
|
+
baseURL: opts && opts.baseURL,
|
|
2025
|
+
model,
|
|
2026
|
+
// The turn's OWN system text — the engine hands it down as `system`
|
|
2027
|
+
// and `chatOne` sends it as the first row, so warming it is warming
|
|
2028
|
+
// the exact prefix this turn will re-send.
|
|
2029
|
+
system: opts && opts.system,
|
|
2030
|
+
signal: opts && opts.signal,
|
|
2031
|
+
warmSecs: opts && opts.warmSecs,
|
|
2032
|
+
// The turn's OWN first-byte bound, so a daemon that accepts the socket
|
|
2033
|
+
// and never answers fails the turn on the CALLER's clock. Without it
|
|
2034
|
+
// the pin sat on a budget of its own and the turn's 1s deadline was not
|
|
2035
|
+
// even armed yet — the pin silently re-armed the hang it exists to
|
|
2036
|
+
// prevent.
|
|
2037
|
+
firstByteTimeoutMs: opts && opts.firstByteTimeoutMs,
|
|
2038
|
+
});
|
|
2039
|
+
// The turn is fitted to the window the pin READ, never to one requested —
|
|
2040
|
+
// see `fitToWindowOf` for the measurement that closed that door.
|
|
2041
|
+
//
|
|
2042
|
+
// Read directly ONLY when the pin did not answer the residency question:
|
|
2043
|
+
// * a pin that SUCCEEDED is the answer, and a memo hit especially so —
|
|
2044
|
+
// that path's whole contract is "no daemon contact and no slot", so
|
|
2045
|
+
// re-reading here would turn a free turn into a dialing one;
|
|
2046
|
+
// * a remote URL is never dialed from this lane at all (fail-closed, the
|
|
2047
|
+
// rule `remoteRefusal` enforces), and a helper that dialed where the
|
|
2048
|
+
// turn may not would be the hole in it. `ensureResident` states the
|
|
2049
|
+
// same rule for the same reason.
|
|
2050
|
+
// Everything else gets one cheap local GET, because the size of the window
|
|
2051
|
+
// is a fact about the daemon rather than about the warm-up having worked:
|
|
2052
|
+
// a pin that was ATTEMPTED and came back empty-handed (its own clock, an
|
|
2053
|
+
// error, a daemon that would not enumerate) still wants the turn fitted, and
|
|
2054
|
+
// it already dialed, so one more read costs nothing it has not already
|
|
2055
|
+
// spent.
|
|
2056
|
+
//
|
|
2057
|
+
// Two things deliberately do NOT qualify, and both are existing contracts
|
|
2058
|
+
// this lane must not quietly widen:
|
|
2059
|
+
// * a pin that was SKIPPED — `warm-disabled` especially, where "nothing to
|
|
2060
|
+
// check residency for, so not even the GET is issued" is asserted in
|
|
2061
|
+
// test/local-resident.test.mjs. Trading that for a fitted turn would be
|
|
2062
|
+
// a real change to a documented behaviour for the benefit of an opt-out
|
|
2063
|
+
// configuration, so the skip stands and a warm-disabled turn is sent
|
|
2064
|
+
// unfitted. The default is warm ON (DEFAULT_WARM_SECS), which is where
|
|
2065
|
+
// the fitting happens.
|
|
2066
|
+
// * a turn that will not be sent at all. The refusals `chatOne` raises
|
|
2067
|
+
// BEFORE dialing (an empty model, a remote URL) must not be preceded by
|
|
2068
|
+
// a request of ours, or "refused before dialing" stops being true.
|
|
2069
|
+
let contextLength = (pin && pin.contextLength) || null;
|
|
2070
|
+
const willSend = isLocalEndpoint(base) && String(model || '').trim() !== '';
|
|
2071
|
+
if (!contextLength && willSend && pin && pin.ok !== true && !pin.skipped) {
|
|
2072
|
+
contextLength = (await windowOf(base, model, { signal: opts && opts.signal })) || null;
|
|
2073
|
+
}
|
|
2074
|
+
return chatOne({ ...opts, contextLength });
|
|
2075
|
+
});
|
|
2076
|
+
swapWarmModel(base, model);
|
|
2077
|
+
// What the extend cost, reported rather than discarded — the daemon's own
|
|
2078
|
+
// `prompt_eval_count` for the prefill it did. Deliberately NOT folded into
|
|
2079
|
+
// `usage`: the pin is not part of the turn's token bill, and adding it
|
|
2080
|
+
// would report tokens the user's turn never spent.
|
|
2081
|
+
if (pin && typeof result === 'object' && result) result.residency = pin;
|
|
2082
|
+
return result;
|
|
2083
|
+
} finally {
|
|
2084
|
+
const remaining = (activeCountByKey.get(key) || 1) - 1;
|
|
2085
|
+
if (remaining <= 0) activeCountByKey.delete(key);
|
|
2086
|
+
else activeCountByKey.set(key, remaining);
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
|
|
757
2090
|
module.exports = {
|
|
758
2091
|
DEFAULT_BASE,
|
|
2092
|
+
DEFAULT_WARM_SECS,
|
|
2093
|
+
RESIDENT_TIMEOUT_MS,
|
|
759
2094
|
SSE_IDLE_TIMEOUT_MS,
|
|
2095
|
+
FIRST_BYTE_TIMEOUT_MS,
|
|
760
2096
|
PROBE_TIMEOUT_MS,
|
|
761
2097
|
baseOf,
|
|
2098
|
+
normalizeBase,
|
|
762
2099
|
isLocalEndpoint,
|
|
763
2100
|
remoteRefusal,
|
|
764
2101
|
probe,
|
|
765
2102
|
listTags,
|
|
766
2103
|
readSSE,
|
|
767
2104
|
chat,
|
|
2105
|
+
ensureResident,
|
|
2106
|
+
residentSecsLeft,
|
|
2107
|
+
residentEntry,
|
|
2108
|
+
unload,
|
|
2109
|
+
unloadAll,
|
|
2110
|
+
// Exported for this transport's own tests. The text-form tool-call decoders
|
|
2111
|
+
// are pure, and the only honest way to test a streaming decoder is chunk by
|
|
2112
|
+
// chunk — see test/local-tool-text.test.mjs.
|
|
2113
|
+
makeTextToolCallStripper,
|
|
2114
|
+
makeDeepSeekEnvelopeStripper,
|
|
2115
|
+
textCallsIn,
|
|
2116
|
+
// The window fit, exported for the same reason: it decides what leaves this
|
|
2117
|
+
// machine, and its arithmetic (fixed tool cost charged before history, reserve
|
|
2118
|
+
// for the reply, never a guessed window) is worth asserting rather than
|
|
2119
|
+
// eyeballing.
|
|
2120
|
+
fitToWindowOf,
|
|
2121
|
+
// The runtime's own response-header deadline, translated into this lane's
|
|
2122
|
+
// timeout shape. Exported because the shape it produces is the engine's
|
|
2123
|
+
// contract, and the real error it recognises is a `fetch failed` wrapper —
|
|
2124
|
+
// worth asserting on the wrapper rather than on a string.
|
|
2125
|
+
isHeadersDeadline,
|
|
2126
|
+
HEADERS_DEADLINE_MS,
|
|
2127
|
+
windowOf,
|
|
2128
|
+
// Exported so a test can assert the STOP path's contract (a stopped turn
|
|
2129
|
+
// resolves; it never rejects) against the real helper rather than against a
|
|
2130
|
+
// copy of its shape.
|
|
2131
|
+
stoppedTurn,
|
|
768
2132
|
};
|