@otto-code/brain 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/lifecycle.js +16 -2
- package/dist/models/download.js +15 -5
- package/dist/service/activity.d.ts +22 -2
- package/dist/service/activity.js +109 -23
- package/dist/service/host-api.d.ts +26 -0
- package/dist/service/host-api.js +88 -1
- package/dist/service/router.d.ts +19 -4
- package/dist/service/router.js +85 -39
- package/dist/service/scheduler.d.ts +9 -1
- package/dist/service/scheduler.js +15 -3
- package/dist/service/serve.d.ts +2 -1
- package/dist/service/serve.js +63 -24
- package/dist/service/status-events.d.ts +88 -0
- package/dist/service/status-events.js +256 -0
- package/dist/service/supervisor.d.ts +2 -2
- package/dist/service/supervisor.js +11 -5
- package/dist/sysmon.d.ts +1 -1
- package/dist/sysmon.js +43 -8
- package/package.json +1 -1
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* they are always explicit user actions, never auto-started.
|
|
6
6
|
*/
|
|
7
7
|
import { spawn } from "node:child_process";
|
|
8
|
-
import { openSync } from "node:fs";
|
|
8
|
+
import { existsSync, openSync } from "node:fs";
|
|
9
9
|
import http from "node:http";
|
|
10
10
|
import https from "node:https";
|
|
11
11
|
import { loadBrainConfig } from "../config/index.js";
|
|
@@ -30,7 +30,9 @@ export async function runServeCommand(options, _command) {
|
|
|
30
30
|
});
|
|
31
31
|
const scheme = handle.secure ? "https" : "http";
|
|
32
32
|
process.stdout.write(`router listening on ${scheme}://${handle.displayHost}:${handle.port}\n`);
|
|
33
|
-
process.stdout.write(
|
|
33
|
+
process.stdout.write(handle.model
|
|
34
|
+
? `ready: ${handle.model.displayName}, ${vram.formatGiB(handle.supervisor.vramAtReadyBytes ?? 0)} VRAM in use\n`
|
|
35
|
+
: "ready: no model loaded; use the Library tab or `otto brain pull` to download one\n");
|
|
34
36
|
process.stdout.write("press Ctrl+C to stop\n");
|
|
35
37
|
const shutdown = async () => {
|
|
36
38
|
process.stdout.write("\nstopping…\n");
|
|
@@ -83,7 +85,19 @@ export async function runStartCommand(options, command) {
|
|
|
83
85
|
});
|
|
84
86
|
}
|
|
85
87
|
const { logFile } = resolveBrainPaths();
|
|
88
|
+
// process.argv[1] is the entry script of whatever host is running us (the npm
|
|
89
|
+
// CLI's bin, the desktop bundle's dist/index.js, bin/otto-brain). If it is not
|
|
90
|
+
// a file, we are running somewhere that does not lay argv out like Node - the
|
|
91
|
+
// detached child would silently get a verb where the script belongs and parse
|
|
92
|
+
// as garbage, so say so instead.
|
|
86
93
|
const entry = process.argv[1];
|
|
94
|
+
if (!entry || !existsSync(entry)) {
|
|
95
|
+
throw new CommandError({
|
|
96
|
+
code: "NO_ENTRYPOINT",
|
|
97
|
+
message: "cannot start the brain detached: this host does not expose a CLI entry script",
|
|
98
|
+
details: "run `otto brain serve` in the foreground, or use the npm CLI (npm i -g @otto-code/cli)",
|
|
99
|
+
});
|
|
100
|
+
}
|
|
87
101
|
const args = [...invocationVerbPrefix(command), "serve"];
|
|
88
102
|
if (options.model)
|
|
89
103
|
args.push("--model", options.model);
|
package/dist/models/download.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* `<managedModelsDir>/<publisher>/<repo>/<file>` to mirror the LM Studio layout
|
|
5
5
|
* the scanner already understands.
|
|
6
6
|
*/
|
|
7
|
-
import { createWriteStream, existsSync, mkdirSync } from "node:fs";
|
|
7
|
+
import { createWriteStream, existsSync, mkdirSync, rmSync } from "node:fs";
|
|
8
8
|
import path from "node:path";
|
|
9
9
|
import { Readable } from "node:stream";
|
|
10
10
|
import { pipeline } from "node:stream/promises";
|
|
@@ -41,6 +41,10 @@ function authHeaders(token) {
|
|
|
41
41
|
* killed download never leaves a truncated file that looks complete.
|
|
42
42
|
*/
|
|
43
43
|
async function streamRepoFile(url, destPath, label, token, onProgress, received) {
|
|
44
|
+
const tmp = `${destPath}.part`;
|
|
45
|
+
// Remove leftovers from an earlier interrupted attempt before starting a
|
|
46
|
+
// fresh request, including when this request fails before opening a stream.
|
|
47
|
+
rmSync(tmp, { force: true });
|
|
44
48
|
mkdirSync(path.dirname(destPath), { recursive: true });
|
|
45
49
|
if (existsSync(destPath))
|
|
46
50
|
return false;
|
|
@@ -54,10 +58,16 @@ async function streamRepoFile(url, destPath, label, token, onProgress, received)
|
|
|
54
58
|
received.bytes += chunk.length;
|
|
55
59
|
onProgress?.({ file: label, receivedBytes: received.bytes, totalBytes });
|
|
56
60
|
});
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
+
try {
|
|
62
|
+
await pipeline(body, createWriteStream(tmp));
|
|
63
|
+
const { renameSync } = await import("node:fs");
|
|
64
|
+
renameSync(tmp, destPath);
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
// Cancellation kills the CLI child while the stream is still writing. Do
|
|
68
|
+
// not leave a truncated `.part` behind for the next quant attempt.
|
|
69
|
+
rmSync(tmp, { force: true });
|
|
70
|
+
}
|
|
61
71
|
return true;
|
|
62
72
|
}
|
|
63
73
|
/** Download the model file; returns the local path it was written to. */
|
|
@@ -58,6 +58,13 @@ export declare function withActivity<T>(kind: BrainActivityKind, options: {
|
|
|
58
58
|
*/
|
|
59
59
|
export declare function chunkHasReasoning(text: string): boolean;
|
|
60
60
|
export declare function chunkHasContent(text: string): boolean;
|
|
61
|
+
export type InferenceStage = "processing" | "thinking" | "generating";
|
|
62
|
+
export interface InferenceActivitySnapshot {
|
|
63
|
+
activeRequests: number;
|
|
64
|
+
processing: number;
|
|
65
|
+
thinking: number;
|
|
66
|
+
generating: number;
|
|
67
|
+
}
|
|
61
68
|
/**
|
|
62
69
|
* Which in-flight completions are currently mid-thought.
|
|
63
70
|
*
|
|
@@ -68,16 +75,29 @@ export declare function chunkHasContent(text: string): boolean;
|
|
|
68
75
|
* if more reasoning follows - a stream that is producing readable output should
|
|
69
76
|
* not report as though it were still silent.
|
|
70
77
|
*
|
|
71
|
-
* A
|
|
72
|
-
*
|
|
78
|
+
* A map rather than one global phase because llama-server runs several slots at
|
|
79
|
+
* once. One request can be processing a prompt while another thinks and a third
|
|
80
|
+
* generates content; the aggregate counts must preserve all three.
|
|
73
81
|
*/
|
|
74
82
|
export declare class ReasoningTracker {
|
|
75
83
|
#private;
|
|
84
|
+
/**
|
|
85
|
+
* Watch stage counts, not the per-chunk traffic behind them.
|
|
86
|
+
*
|
|
87
|
+
* The status event stream needs to publish request start, the moment a model
|
|
88
|
+
* goes silent to think and the moment it starts answering. Repeated chunks in
|
|
89
|
+
* one stage do not notify; slot sampling owns bounded token-rate updates.
|
|
90
|
+
*/
|
|
91
|
+
onChange(listener: () => void): () => void;
|
|
92
|
+
/** A completion was dispatched to llama-server and awaits its first output delta. */
|
|
93
|
+
begin(requestId: string): void;
|
|
76
94
|
/** Note a chunk of `requestId`'s stream. Cheap enough to call per chunk. */
|
|
77
95
|
observe(requestId: string, text: string): void;
|
|
78
96
|
/** Forget the request. Must be called on end *and* on error, or the flag sticks. */
|
|
79
97
|
end(requestId: string): void;
|
|
80
98
|
get active(): boolean;
|
|
81
99
|
get count(): number;
|
|
100
|
+
/** Aggregate request stages. Counts stay exact even with several parallel slots. */
|
|
101
|
+
get snapshot(): InferenceActivitySnapshot;
|
|
82
102
|
}
|
|
83
103
|
//# sourceMappingURL=activity.d.ts.map
|
package/dist/service/activity.js
CHANGED
|
@@ -3,10 +3,16 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
3
3
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
4
4
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
5
5
|
};
|
|
6
|
-
var
|
|
6
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
7
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
10
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
11
|
+
};
|
|
12
|
+
var _ReasoningTracker_instances, _ReasoningTracker_requests, _ReasoningTracker_tails, _ReasoningTracker_inlineReasoning, _ReasoningTracker_listeners, _ReasoningTracker_lastSnapshot, _ReasoningTracker_announce;
|
|
7
13
|
/**
|
|
8
|
-
* What long-running work currently owns the brain, and
|
|
9
|
-
*
|
|
14
|
+
* What long-running work currently owns the brain, and which stage each live
|
|
15
|
+
* inference request has reached.
|
|
10
16
|
*
|
|
11
17
|
* Two trackers with deliberately different lifetimes:
|
|
12
18
|
*
|
|
@@ -18,11 +24,11 @@ var _ReasoningTracker_reasoning, _ReasoningTracker_content;
|
|
|
18
24
|
* was killed with Ctrl-C never gets to clean up after itself, and a status
|
|
19
25
|
* that stays stuck on "calibrating" forever is worse than no status at all.
|
|
20
26
|
*
|
|
21
|
-
* - **
|
|
27
|
+
* - **Inference** is per-request and lives only as long as the stream does, so
|
|
22
28
|
* it is plain in-process state on the router. It never touches disk.
|
|
23
29
|
*
|
|
24
|
-
* Both
|
|
25
|
-
*
|
|
30
|
+
* Both ride on host status: ops under `activity`, inference under `inference`.
|
|
31
|
+
* The client uses those independent signals to drive the Overview and rail.
|
|
26
32
|
*/
|
|
27
33
|
import { existsSync, readFileSync, rmSync } from "node:fs";
|
|
28
34
|
import { resolveBrainPaths } from "../config/paths.js";
|
|
@@ -164,10 +170,14 @@ function clampProgress(progress) {
|
|
|
164
170
|
* would cost more than the signal is worth.
|
|
165
171
|
*/
|
|
166
172
|
export function chunkHasReasoning(text) {
|
|
167
|
-
return
|
|
173
|
+
return (/"type"\s*:\s*"(?:thinking|reasoning)_delta"/u.test(text) ||
|
|
174
|
+
/"(?:thinking|reasoning_content)"\s*:\s*"[^"]/u.test(text));
|
|
168
175
|
}
|
|
169
176
|
export function chunkHasContent(text) {
|
|
170
|
-
return
|
|
177
|
+
return (/"type"\s*:\s*"text_delta"/u.test(text) ||
|
|
178
|
+
/"content"\s*:\s*"[^"]/u.test(text) ||
|
|
179
|
+
/"tool_calls"\s*:\s*\[\s*\{/u.test(text) ||
|
|
180
|
+
/"type"\s*:\s*"(?:tool_use|input_json_delta)"/u.test(text));
|
|
171
181
|
}
|
|
172
182
|
/**
|
|
173
183
|
* Which in-flight completions are currently mid-thought.
|
|
@@ -179,38 +189,114 @@ export function chunkHasContent(text) {
|
|
|
179
189
|
* if more reasoning follows - a stream that is producing readable output should
|
|
180
190
|
* not report as though it were still silent.
|
|
181
191
|
*
|
|
182
|
-
* A
|
|
183
|
-
*
|
|
192
|
+
* A map rather than one global phase because llama-server runs several slots at
|
|
193
|
+
* once. One request can be processing a prompt while another thinks and a third
|
|
194
|
+
* generates content; the aggregate counts must preserve all three.
|
|
184
195
|
*/
|
|
185
196
|
export class ReasoningTracker {
|
|
186
197
|
constructor() {
|
|
187
|
-
|
|
188
|
-
|
|
198
|
+
_ReasoningTracker_instances.add(this);
|
|
199
|
+
_ReasoningTracker_requests.set(this, new Map());
|
|
200
|
+
/** Tail of the last transport chunk, so a field name split by TCP is still detected. */
|
|
201
|
+
_ReasoningTracker_tails.set(this, new Map());
|
|
202
|
+
/** Models/runtimes that leave reasoning inline as `<think>…</think>`. */
|
|
203
|
+
_ReasoningTracker_inlineReasoning.set(this, new Set());
|
|
204
|
+
_ReasoningTracker_listeners.set(this, new Set());
|
|
205
|
+
_ReasoningTracker_lastSnapshot.set(this, "0:0:0:0");
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Watch stage counts, not the per-chunk traffic behind them.
|
|
209
|
+
*
|
|
210
|
+
* The status event stream needs to publish request start, the moment a model
|
|
211
|
+
* goes silent to think and the moment it starts answering. Repeated chunks in
|
|
212
|
+
* one stage do not notify; slot sampling owns bounded token-rate updates.
|
|
213
|
+
*/
|
|
214
|
+
onChange(listener) {
|
|
215
|
+
__classPrivateFieldGet(this, _ReasoningTracker_listeners, "f").add(listener);
|
|
216
|
+
return () => __classPrivateFieldGet(this, _ReasoningTracker_listeners, "f").delete(listener);
|
|
217
|
+
}
|
|
218
|
+
/** A completion was dispatched to llama-server and awaits its first output delta. */
|
|
219
|
+
begin(requestId) {
|
|
220
|
+
if (__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").has(requestId))
|
|
221
|
+
return;
|
|
222
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "processing");
|
|
223
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
189
224
|
}
|
|
190
225
|
/** Note a chunk of `requestId`'s stream. Cheap enough to call per chunk. */
|
|
191
226
|
observe(requestId, text) {
|
|
192
|
-
|
|
227
|
+
const current = __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").get(requestId);
|
|
228
|
+
if (current === "generating")
|
|
193
229
|
return;
|
|
194
|
-
if (
|
|
195
|
-
__classPrivateFieldGet(this,
|
|
196
|
-
|
|
230
|
+
if (!current)
|
|
231
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "processing");
|
|
232
|
+
// Node can split an SSE JSON field name at any byte. Keeping a small tail
|
|
233
|
+
// makes stage recognition independent of transport chunk boundaries without
|
|
234
|
+
// parsing or retaining the generated content itself.
|
|
235
|
+
const combined = `${__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").get(requestId) ?? ""}${text}`;
|
|
236
|
+
__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").set(requestId, combined.slice(-128));
|
|
237
|
+
if (__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").has(requestId)) {
|
|
238
|
+
if (combined.includes("</think>")) {
|
|
239
|
+
__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").delete(requestId);
|
|
240
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "generating");
|
|
241
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
242
|
+
}
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (combined.includes("<think>")) {
|
|
246
|
+
__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").add(requestId);
|
|
247
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "thinking");
|
|
248
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (chunkHasContent(combined)) {
|
|
252
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "generating");
|
|
253
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
197
254
|
return;
|
|
198
255
|
}
|
|
199
|
-
if (chunkHasReasoning(
|
|
200
|
-
__classPrivateFieldGet(this,
|
|
256
|
+
if (chunkHasReasoning(combined) && current !== "thinking") {
|
|
257
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "thinking");
|
|
258
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
201
259
|
}
|
|
202
260
|
}
|
|
203
261
|
/** Forget the request. Must be called on end *and* on error, or the flag sticks. */
|
|
204
262
|
end(requestId) {
|
|
205
|
-
__classPrivateFieldGet(this,
|
|
206
|
-
__classPrivateFieldGet(this,
|
|
263
|
+
__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").delete(requestId);
|
|
264
|
+
__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").delete(requestId);
|
|
265
|
+
__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").delete(requestId);
|
|
266
|
+
__classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
|
|
207
267
|
}
|
|
208
268
|
get active() {
|
|
209
|
-
return
|
|
269
|
+
return this.snapshot.thinking > 0;
|
|
210
270
|
}
|
|
211
271
|
get count() {
|
|
212
|
-
return
|
|
272
|
+
return this.snapshot.thinking;
|
|
273
|
+
}
|
|
274
|
+
/** Aggregate request stages. Counts stay exact even with several parallel slots. */
|
|
275
|
+
get snapshot() {
|
|
276
|
+
const result = {
|
|
277
|
+
activeRequests: __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").size,
|
|
278
|
+
processing: 0,
|
|
279
|
+
thinking: 0,
|
|
280
|
+
generating: 0,
|
|
281
|
+
};
|
|
282
|
+
for (const stage of __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").values())
|
|
283
|
+
result[stage] += 1;
|
|
284
|
+
return result;
|
|
213
285
|
}
|
|
214
286
|
}
|
|
215
|
-
|
|
287
|
+
_ReasoningTracker_requests = new WeakMap(), _ReasoningTracker_tails = new WeakMap(), _ReasoningTracker_inlineReasoning = new WeakMap(), _ReasoningTracker_listeners = new WeakMap(), _ReasoningTracker_lastSnapshot = new WeakMap(), _ReasoningTracker_instances = new WeakSet(), _ReasoningTracker_announce = function _ReasoningTracker_announce() {
|
|
288
|
+
const snapshot = this.snapshot;
|
|
289
|
+
const key = `${snapshot.activeRequests}:${snapshot.processing}:${snapshot.thinking}:${snapshot.generating}`;
|
|
290
|
+
if (key === __classPrivateFieldGet(this, _ReasoningTracker_lastSnapshot, "f"))
|
|
291
|
+
return;
|
|
292
|
+
__classPrivateFieldSet(this, _ReasoningTracker_lastSnapshot, key, "f");
|
|
293
|
+
for (const listener of __classPrivateFieldGet(this, _ReasoningTracker_listeners, "f")) {
|
|
294
|
+
try {
|
|
295
|
+
listener();
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
// Status reporting must never break a proxied completion.
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
};
|
|
216
302
|
//# sourceMappingURL=activity.js.map
|
|
@@ -26,7 +26,16 @@ import type { RankedModel } from "../ops/results.js";
|
|
|
26
26
|
import type { GpuInfo, Model } from "../types.js";
|
|
27
27
|
import * as vram from "../vram.js";
|
|
28
28
|
import type { SystemSample } from "../sysmon.js";
|
|
29
|
+
import type { BrainStatusPublisher } from "./status-events.js";
|
|
29
30
|
import type { Supervisor } from "./supervisor.js";
|
|
31
|
+
/**
|
|
32
|
+
* The management API's own version, additive to the capability flags.
|
|
33
|
+
*
|
|
34
|
+
* Capabilities answer "can this brain do X"; this answers "which generation of
|
|
35
|
+
* the API is this" for the rare change that no single flag describes. A daemon
|
|
36
|
+
* reads both and never requires an exact package-version match.
|
|
37
|
+
*/
|
|
38
|
+
export declare const HOST_API_VERSION = 2;
|
|
30
39
|
/**
|
|
31
40
|
* What this brain can serve. The daemon folds this into `brain.host.status` and
|
|
32
41
|
* Otto gates each tab on it, because the daemon and the brain version
|
|
@@ -52,6 +61,17 @@ export interface HostCapabilities {
|
|
|
52
61
|
rename: boolean;
|
|
53
62
|
/** POST /__host/model/rename/reset */
|
|
54
63
|
reset: boolean;
|
|
64
|
+
/**
|
|
65
|
+
* GET /__host/events: a live SSE stream of complete status snapshots.
|
|
66
|
+
*
|
|
67
|
+
* The one capability a daemon reads *before* deciding how to watch this brain.
|
|
68
|
+
* False (including on every brain that predates the stream) means the daemon
|
|
69
|
+
* keeps polling `/__host/status`, which is why nothing about the older
|
|
70
|
+
* management API had to change for this to ship.
|
|
71
|
+
*/
|
|
72
|
+
events: boolean;
|
|
73
|
+
/** Bounded live inference stages, token counts and throughput on status events. */
|
|
74
|
+
liveInference: boolean;
|
|
55
75
|
/** Whether writes are currently permitted (allowRemoteConfig). */
|
|
56
76
|
writable: boolean;
|
|
57
77
|
}
|
|
@@ -72,6 +92,12 @@ export interface HostApiDeps {
|
|
|
72
92
|
/** The managed models directory, for disk accounting. Null when unresolvable. */
|
|
73
93
|
getModelsDir: () => string | null;
|
|
74
94
|
sampleResources: () => Promise<SystemSample>;
|
|
95
|
+
/**
|
|
96
|
+
* The live status source behind `GET /__host/events`. Absent (or not yet
|
|
97
|
+
* carrying a snapshot source) means this brain does not advertise events and
|
|
98
|
+
* its daemon keeps polling status.
|
|
99
|
+
*/
|
|
100
|
+
statusEvents?: BrainStatusPublisher | null;
|
|
75
101
|
}
|
|
76
102
|
/** One row of the model inventory: the scan, metadata, profile and score joined. */
|
|
77
103
|
export interface InventoryRow {
|
package/dist/service/host-api.js
CHANGED
|
@@ -7,6 +7,23 @@ import { errorMessage, readJsonBody, sendError, sendJson } from "./http-util.js"
|
|
|
7
7
|
const MAX_PATCH_BYTES = 256 * 1024;
|
|
8
8
|
const MAX_DISPLAY_NAME = 200;
|
|
9
9
|
const DEFAULT_LOG_LINES = 200;
|
|
10
|
+
/**
|
|
11
|
+
* The management API's own version, additive to the capability flags.
|
|
12
|
+
*
|
|
13
|
+
* Capabilities answer "can this brain do X"; this answers "which generation of
|
|
14
|
+
* the API is this" for the rare change that no single flag describes. A daemon
|
|
15
|
+
* reads both and never requires an exact package-version match.
|
|
16
|
+
*/
|
|
17
|
+
export const HOST_API_VERSION = 2;
|
|
18
|
+
/**
|
|
19
|
+
* How often the SSE stream writes a comment line when nothing has changed.
|
|
20
|
+
*
|
|
21
|
+
* This is transport keepalive, not a status event: a proxy or a NAT table with
|
|
22
|
+
* an idle timeout would otherwise silently drop a stream from a brain that is
|
|
23
|
+
* simply sitting still, and the daemon would report an unreachable brain that is
|
|
24
|
+
* fine. Comments are ignored by every SSE parser, so no reader sees them.
|
|
25
|
+
*/
|
|
26
|
+
const SSE_KEEPALIVE_MS = 20000;
|
|
10
27
|
function stateOf(supervisor, model) {
|
|
11
28
|
if (!supervisor.model || supervisor.model.id !== model.id)
|
|
12
29
|
return "not-loaded";
|
|
@@ -105,6 +122,11 @@ export function createHostApi(deps) {
|
|
|
105
122
|
inventory: true,
|
|
106
123
|
rename: true,
|
|
107
124
|
reset: true,
|
|
125
|
+
// Read live rather than captured: the publisher is inert until the router
|
|
126
|
+
// installs its snapshot source, and advertising a stream we cannot serve
|
|
127
|
+
// would make a daemon stop polling and see nothing.
|
|
128
|
+
events: Boolean(deps.statusEvents?.ready),
|
|
129
|
+
liveInference: Boolean(deps.statusEvents?.ready),
|
|
108
130
|
writable: deps.getAllowWrite(),
|
|
109
131
|
});
|
|
110
132
|
/** Refuse a write unless the owner opted into remote configuration. */
|
|
@@ -216,7 +238,13 @@ export function createHostApi(deps) {
|
|
|
216
238
|
return;
|
|
217
239
|
}
|
|
218
240
|
updateDisplayName(model.id, displayName);
|
|
219
|
-
|
|
241
|
+
// The catalog is kept in memory between requests. Refresh it here so
|
|
242
|
+
// the next inventory request, and all model lookups, see the persisted
|
|
243
|
+
// name immediately instead of reverting to the scan-derived name until
|
|
244
|
+
// the brain is restarted. Reset already follows this pattern below.
|
|
245
|
+
const catalog = deps.rescan();
|
|
246
|
+
const updated = resolveModel(catalog, model.id);
|
|
247
|
+
sendJson(res, { displayName: updated ? updated.displayName : displayName });
|
|
220
248
|
});
|
|
221
249
|
};
|
|
222
250
|
const handleReset = (req, res, model) => {
|
|
@@ -333,6 +361,56 @@ export function createHostApi(deps) {
|
|
|
333
361
|
command: deps.supervisor.command,
|
|
334
362
|
});
|
|
335
363
|
};
|
|
364
|
+
/**
|
|
365
|
+
* Stream complete status snapshots as SSE.
|
|
366
|
+
*
|
|
367
|
+
* Authentication is the listener's, not this route's: `withAuth` in serve.ts
|
|
368
|
+
* gates every `/__host/*` path with the same token and TLS policy, so an
|
|
369
|
+
* unauthenticated caller never reaches this function.
|
|
370
|
+
*
|
|
371
|
+
* The stream is unidirectional and outlives the request, which is exactly why
|
|
372
|
+
* SSE rather than a socket the brain would have to dial back to a daemon: a
|
|
373
|
+
* remote brain has no idea where its daemon is, and the daemon already knows
|
|
374
|
+
* how to reach the brain over an authenticated HTTP(S) endpoint.
|
|
375
|
+
*/
|
|
376
|
+
const handleEvents = (req, res, publisher) => {
|
|
377
|
+
res.writeHead(200, {
|
|
378
|
+
"content-type": "text/event-stream",
|
|
379
|
+
"cache-control": "no-cache, no-transform",
|
|
380
|
+
connection: "keep-alive",
|
|
381
|
+
// Tells nginx-shaped intermediaries not to buffer, which would defeat the
|
|
382
|
+
// whole point by holding each snapshot until the response ended.
|
|
383
|
+
"x-accel-buffering": "no",
|
|
384
|
+
});
|
|
385
|
+
res.flushHeaders?.();
|
|
386
|
+
const write = (snapshot) => {
|
|
387
|
+
if (res.writableEnded || res.destroyed)
|
|
388
|
+
return;
|
|
389
|
+
res.write(`event: status\ndata: ${JSON.stringify(snapshot)}\n\n`);
|
|
390
|
+
};
|
|
391
|
+
let unsubscribe = () => { };
|
|
392
|
+
const keepalive = setInterval(() => {
|
|
393
|
+
if (res.writableEnded || res.destroyed)
|
|
394
|
+
return;
|
|
395
|
+
res.write(": keepalive\n\n");
|
|
396
|
+
}, SSE_KEEPALIVE_MS);
|
|
397
|
+
keepalive.unref?.();
|
|
398
|
+
const teardown = () => {
|
|
399
|
+
clearInterval(keepalive);
|
|
400
|
+
unsubscribe();
|
|
401
|
+
};
|
|
402
|
+
// The publisher ends the response on host shutdown: an open SSE response is
|
|
403
|
+
// an open connection, and `server.close()` waits for those.
|
|
404
|
+
unsubscribe = publisher.subscribe(write, () => {
|
|
405
|
+
clearInterval(keepalive);
|
|
406
|
+
if (!res.writableEnded && !res.destroyed)
|
|
407
|
+
res.end();
|
|
408
|
+
});
|
|
409
|
+
// Both ends matter: `close` on the request covers a client that walked away,
|
|
410
|
+
// and `close` on the response covers the service shutting the socket down.
|
|
411
|
+
req.on("close", teardown);
|
|
412
|
+
res.on("close", teardown);
|
|
413
|
+
};
|
|
336
414
|
function handleHostApi(req, res) {
|
|
337
415
|
const raw = req.url || "";
|
|
338
416
|
if (!raw.startsWith("/__host/"))
|
|
@@ -345,6 +423,15 @@ export function createHostApi(deps) {
|
|
|
345
423
|
sendJson(res, capabilities());
|
|
346
424
|
return true;
|
|
347
425
|
}
|
|
426
|
+
if (route === "/__host/events" && method === "GET") {
|
|
427
|
+
const publisher = deps.statusEvents;
|
|
428
|
+
if (!publisher?.ready) {
|
|
429
|
+
sendError(res, 404, "this brain does not serve a status event stream");
|
|
430
|
+
return true;
|
|
431
|
+
}
|
|
432
|
+
handleEvents(req, res, publisher);
|
|
433
|
+
return true;
|
|
434
|
+
}
|
|
348
435
|
if (route === "/__host/logs" && method === "GET") {
|
|
349
436
|
handleLogs(res, params);
|
|
350
437
|
return true;
|
package/dist/service/router.d.ts
CHANGED
|
@@ -3,7 +3,8 @@ import type { Supervisor } from "./supervisor.js";
|
|
|
3
3
|
import { type RankedModel } from "../ops/results.js";
|
|
4
4
|
import type { GpuInfo, Model } from "../types.js";
|
|
5
5
|
import type { Profile } from "../config/schema.js";
|
|
6
|
-
import type
|
|
6
|
+
import { type HostApi } from "./host-api.js";
|
|
7
|
+
import type { BrainStatusPublisher } from "./status-events.js";
|
|
7
8
|
type Verdict = "ok" | "reasoning-only" | "truncated" | "failed";
|
|
8
9
|
/** A logger sink; only `warn` is used by the router. */
|
|
9
10
|
export interface Logger {
|
|
@@ -70,6 +71,8 @@ interface DescribeOptions {
|
|
|
70
71
|
/** An LM Studio-style model description, an OpenAI model object enriched. */
|
|
71
72
|
export interface ModelEntry {
|
|
72
73
|
id: string;
|
|
74
|
+
/** Brain's editable human-facing name; `id` remains the stable model key. */
|
|
75
|
+
name: string;
|
|
73
76
|
object: "model";
|
|
74
77
|
created: number;
|
|
75
78
|
owned_by: string;
|
|
@@ -80,6 +83,10 @@ export interface ModelEntry {
|
|
|
80
83
|
quantization: string | null;
|
|
81
84
|
state: ModelState;
|
|
82
85
|
max_context_length: number | null;
|
|
86
|
+
/** Whether the model exposes a chat-template reasoning channel. */
|
|
87
|
+
reasoning: boolean;
|
|
88
|
+
/** Optional per-model values accepted by the OpenAI-compatible endpoint. */
|
|
89
|
+
reasoning_efforts?: string[];
|
|
83
90
|
loaded_context_length?: number;
|
|
84
91
|
}
|
|
85
92
|
/**
|
|
@@ -156,13 +163,21 @@ export interface RouterOptions {
|
|
|
156
163
|
*/
|
|
157
164
|
hostApi?: HostApi | null;
|
|
158
165
|
/**
|
|
159
|
-
* Live system telemetry (CPU, RAM
|
|
166
|
+
* Live system telemetry (CPU, RAM and GPU), folded into `/__host/status`
|
|
160
167
|
* ONLY when the caller asks with `?resources=1`. The daemon's liveness probe
|
|
161
|
-
* polls status frequently and must not pay an `nvidia-smi` spawn for it
|
|
168
|
+
* polls status frequently and must not pay an `nvidia-smi` spawn for it. Slot
|
|
169
|
+
* activity is already part of the cheap status; the
|
|
162
170
|
* Brain page's Overview tab opts in.
|
|
163
171
|
*/
|
|
164
172
|
getResources?: (() => Promise<unknown>) | null;
|
|
173
|
+
/**
|
|
174
|
+
* The live status source served at `GET /__host/events`. The router installs
|
|
175
|
+
* its snapshot builder here and notifies it whenever something authoritative
|
|
176
|
+
* moves, so the same assembly answers both the pull and the push and the two
|
|
177
|
+
* can never disagree. Absent means this brain does not advertise events.
|
|
178
|
+
*/
|
|
179
|
+
statusEvents?: BrainStatusPublisher | null;
|
|
165
180
|
}
|
|
166
|
-
export declare function createRouter({ supervisor, telemetry, logger, getCatalog, loadModel, loadRanking, queryGpuInfo, version, getConfig, getEvals, getLockModel, getDefaultModel, applyConfigPatch, getAllowConfigWrite, hostApi, getResources, }: RouterOptions): (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
|
181
|
+
export declare function createRouter({ supervisor, telemetry, logger, getCatalog, loadModel, loadRanking, queryGpuInfo, version, getConfig, getEvals, getLockModel, getDefaultModel, applyConfigPatch, getAllowConfigWrite, hostApi, getResources, statusEvents, }: RouterOptions): (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
|
167
182
|
export {};
|
|
168
183
|
//# sourceMappingURL=router.d.ts.map
|