@otto-code/brain 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/lifecycle.js +16 -2
- package/dist/models/download.js +15 -5
- package/dist/service/activity.d.ts +22 -2
- package/dist/service/activity.js +109 -23
- package/dist/service/host-api.d.ts +26 -0
- package/dist/service/host-api.js +88 -1
- package/dist/service/router.d.ts +19 -4
- package/dist/service/router.js +85 -39
- package/dist/service/scheduler.d.ts +9 -1
- package/dist/service/scheduler.js +15 -3
- package/dist/service/serve.d.ts +2 -1
- package/dist/service/serve.js +63 -24
- package/dist/service/status-events.d.ts +88 -0
- package/dist/service/status-events.js +256 -0
- package/dist/service/supervisor.d.ts +2 -2
- package/dist/service/supervisor.js +11 -5
- package/dist/sysmon.d.ts +1 -1
- package/dist/sysmon.js +43 -8
- package/package.json +1 -1
package/dist/service/router.js
CHANGED
|
@@ -5,6 +5,7 @@ import { slots as sampleSlots } from "../sysmon.js";
|
|
|
5
5
|
import { makeVramFitPredicate, selectCodingModel } from "./model-selector.js";
|
|
6
6
|
import { query as queryGpu } from "../gpu.js";
|
|
7
7
|
import { rankModels } from "../ops/results.js";
|
|
8
|
+
import { HOST_API_VERSION } from "./host-api.js";
|
|
8
9
|
import { errorBody, errorMessage, HOP_BY_HOP, readJsonBody, sendError, sendJson, } from "./http-util.js";
|
|
9
10
|
/**
|
|
10
11
|
* Fronts the supervised llama-server on a stable port.
|
|
@@ -144,8 +145,10 @@ export function describeModel(model, options = {}) {
|
|
|
144
145
|
const { state = "not-loaded", profile = null, createdAt = null } = options;
|
|
145
146
|
const md = model.metadata || {};
|
|
146
147
|
const entry = {
|
|
147
|
-
//
|
|
148
|
-
id
|
|
148
|
+
// Keep the stable model key separate from Brain's editable display name.
|
|
149
|
+
// OpenAI-compatible clients send `id`; Otto uses `name` for presentation.
|
|
150
|
+
id: model.id,
|
|
151
|
+
name: model.displayName,
|
|
149
152
|
object: "model",
|
|
150
153
|
created: Math.floor((createdAt ? createdAt.getTime() : Date.now()) / 1000),
|
|
151
154
|
owned_by: model.publisher || "local",
|
|
@@ -157,7 +160,13 @@ export function describeModel(model, options = {}) {
|
|
|
157
160
|
quantization: model.quant || null,
|
|
158
161
|
state,
|
|
159
162
|
max_context_length: md.contextLength ?? null,
|
|
163
|
+
reasoning: Boolean(md.reasoning ?? model.thinking),
|
|
160
164
|
};
|
|
165
|
+
const reasoningEfforts = md["reasoning_efforts"];
|
|
166
|
+
if (Array.isArray(reasoningEfforts) &&
|
|
167
|
+
reasoningEfforts.every((value) => typeof value === "string")) {
|
|
168
|
+
entry.reasoning_efforts = reasoningEfforts;
|
|
169
|
+
}
|
|
161
170
|
if (state === "loaded" && profile && profile.contextSize) {
|
|
162
171
|
// llama-server splits -c across --parallel slots, so the window a single
|
|
163
172
|
// request actually gets is the total divided by the concurrency.
|
|
@@ -387,6 +396,10 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
|
|
|
387
396
|
sendError(res, 502, `Upstream llama-server error: ${error.message}`);
|
|
388
397
|
done();
|
|
389
398
|
});
|
|
399
|
+
// This is the first authoritative inference-stage signal: the request is
|
|
400
|
+
// being dispatched to llama-server and is waiting for prompt processing or
|
|
401
|
+
// its first output delta.
|
|
402
|
+
reasoning?.begin(streamId);
|
|
390
403
|
upstream.end(body);
|
|
391
404
|
});
|
|
392
405
|
}
|
|
@@ -477,10 +490,15 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
|
|
|
477
490
|
// (rare), so the router caches the ranking and re-reads it at most once per
|
|
478
491
|
// window - the cheap time-based trigger.
|
|
479
492
|
const RANKING_TTL_MS = 60000;
|
|
480
|
-
export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, }) {
|
|
493
|
+
export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, statusEvents = null, }) {
|
|
481
494
|
const agent = new http.Agent({ keepAlive: true, maxSockets: 32 });
|
|
482
495
|
const scheduler = loadModel
|
|
483
|
-
? new Scheduler({
|
|
496
|
+
? new Scheduler({
|
|
497
|
+
supervisor,
|
|
498
|
+
loadModel,
|
|
499
|
+
logger: (m) => logger?.warn?.(m),
|
|
500
|
+
onChange: statusEvents ? () => statusEvents.notify() : null,
|
|
501
|
+
})
|
|
484
502
|
: null;
|
|
485
503
|
// A (re)start means whatever produced the current warning no longer applies -
|
|
486
504
|
// either a different model is now resident, or the same one just picked up an
|
|
@@ -566,51 +584,79 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
|
|
|
566
584
|
resolved: lock ? null : resolveModel(name),
|
|
567
585
|
});
|
|
568
586
|
};
|
|
587
|
+
/**
|
|
588
|
+
* The cheap host status: everything `/__host/status` answers except the
|
|
589
|
+
* opt-in `resources` block.
|
|
590
|
+
*
|
|
591
|
+
* One assembly feeds both the pull (`/__host/status`) and the push
|
|
592
|
+
* (`/__host/events`). Keeping them as one function is the point: a field that
|
|
593
|
+
* only the polled answer carried would be a field the rail silently lost the
|
|
594
|
+
* moment a daemon stopped polling.
|
|
595
|
+
*/
|
|
596
|
+
const buildCheapStatus = async () => {
|
|
597
|
+
const schedulerStats = scheduler ? scheduler.stats() : null;
|
|
598
|
+
// Slots come from a loopback GET on the resident llama-server. That is
|
|
599
|
+
// cheap enough to pay on every sample - unlike GPU sampling, which spawns
|
|
600
|
+
// `nvidia-smi` and stays opt-in. Skipped entirely unless a model is
|
|
601
|
+
// resident, since there is nothing listening otherwise.
|
|
602
|
+
const slots = supervisor.state === "ready"
|
|
603
|
+
? await sampleSlots({ host: supervisor.host, port: supervisor.internalPort }).catch(() => null)
|
|
604
|
+
: null;
|
|
605
|
+
return {
|
|
606
|
+
version,
|
|
607
|
+
// Additive, and separate from `version`: the package version says which
|
|
608
|
+
// build this is, this says which generation of the management contract it
|
|
609
|
+
// speaks. A daemon reads this and `capabilities` rather than pinning a
|
|
610
|
+
// package version.
|
|
611
|
+
apiVersion: HOST_API_VERSION,
|
|
612
|
+
...supervisor.status(),
|
|
613
|
+
telemetry: { ...telemetry.totals, warning: telemetry.warning },
|
|
614
|
+
scheduler: schedulerStats,
|
|
615
|
+
recent: telemetry.records.slice(-10),
|
|
616
|
+
logLineCount: supervisor.logLines.length,
|
|
617
|
+
// Carried inline rather than fetched from /__host/capabilities: the
|
|
618
|
+
// daemon reads status constantly, and a separately cached copy would go
|
|
619
|
+
// stale the moment the owner toggles allowRemoteConfig.
|
|
620
|
+
capabilities: hostApi ? hostApi.capabilities() : null,
|
|
621
|
+
// The three signals the Brain rail's icon is derived from. All are cheap
|
|
622
|
+
// enough for the liveness path: `activity` is one stat of a file that is
|
|
623
|
+
// usually absent, `reasoning` is in-process state, and `queued` is
|
|
624
|
+
// already computed above.
|
|
625
|
+
activity: readActivity(),
|
|
626
|
+
reasoning: reasoningTracker.active,
|
|
627
|
+
// Exact aggregate request stages from the proxy lifecycle. Unlike slot
|
|
628
|
+
// phase sampling, this distinguishes silent prompt processing, reasoning
|
|
629
|
+
// deltas and user-visible content even when several requests overlap.
|
|
630
|
+
inference: reasoningTracker.snapshot,
|
|
631
|
+
queued: schedulerStats ? schedulerStats.queued : 0,
|
|
632
|
+
slots,
|
|
633
|
+
};
|
|
634
|
+
};
|
|
635
|
+
// Publish rather than be polled. The publisher decides what counts as a
|
|
636
|
+
// change (see status-events.ts); everything here just says "look again".
|
|
637
|
+
if (statusEvents) {
|
|
638
|
+
statusEvents.setSource(buildCheapStatus);
|
|
639
|
+
supervisor.on("state", () => statusEvents.notify());
|
|
640
|
+
supervisor.on("crashed", () => statusEvents.notify());
|
|
641
|
+
reasoningTracker.onChange(() => statusEvents.notify());
|
|
642
|
+
}
|
|
569
643
|
return function handler(req, res) {
|
|
570
644
|
// Host-management read surface (`/__host/*`): the single API both the TUI and
|
|
571
645
|
// Otto's GUI consume, so the two never drift. Status is live; config and
|
|
572
646
|
// evals are point-in-time reads the daemon proxies to its settings UI.
|
|
573
647
|
const path = (req.url || "").split("?")[0];
|
|
574
648
|
if (path === "/__host/status") {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
...supervisor.status(),
|
|
579
|
-
telemetry: { ...telemetry.totals, warning: telemetry.warning },
|
|
580
|
-
scheduler: schedulerStats,
|
|
581
|
-
recent: telemetry.records.slice(-10),
|
|
582
|
-
logLineCount: supervisor.logLines.length,
|
|
583
|
-
// Carried inline rather than fetched from /__host/capabilities: the
|
|
584
|
-
// daemon polls status constantly, and a separately cached copy would go
|
|
585
|
-
// stale the moment the owner toggles allowRemoteConfig.
|
|
586
|
-
capabilities: hostApi ? hostApi.capabilities() : null,
|
|
587
|
-
// The three signals the Brain rail's icon is derived from. All are cheap
|
|
588
|
-
// enough for the liveness poll: `activity` is one stat of a file that is
|
|
589
|
-
// usually absent, `reasoning` is in-process state, and `queued` is
|
|
590
|
-
// already computed above. Slot phases are the one that costs a round
|
|
591
|
-
// trip, and are fetched below.
|
|
592
|
-
activity: readActivity(),
|
|
593
|
-
reasoning: reasoningTracker.active,
|
|
594
|
-
queued: schedulerStats ? schedulerStats.queued : 0,
|
|
595
|
-
};
|
|
596
|
-
// Slots come from a loopback GET on the resident llama-server. That is
|
|
597
|
-
// cheap enough to pay on every poll - unlike the GPU sampling below, which
|
|
598
|
-
// spawns `nvidia-smi` and stays opt-in. Skipped entirely unless a model is
|
|
599
|
-
// resident, since there is nothing listening otherwise.
|
|
600
|
-
const slotsPromise = supervisor.state === "ready"
|
|
601
|
-
? sampleSlots({ host: supervisor.host, port: supervisor.internalPort }).catch(() => null)
|
|
602
|
-
: Promise.resolve(null);
|
|
603
|
-
// Resources cost an `nvidia-smi` spawn, so they are opt-in: the daemon's
|
|
604
|
-
// liveness probe polls this route far more often than any UI does, and
|
|
605
|
-
// must not pay for a panel it is not rendering.
|
|
649
|
+
// Resources cost an `nvidia-smi` spawn, so they are opt-in: the daemon
|
|
650
|
+
// reads this route far more often than any UI does, and must not pay for
|
|
651
|
+
// a panel it is not rendering.
|
|
606
652
|
const wantsResources = /[?&]resources=1(&|$)/.test(req.url || "");
|
|
607
653
|
if (!wantsResources || !getResources) {
|
|
608
|
-
void
|
|
654
|
+
void buildCheapStatus().then((base) => sendJson(res, base));
|
|
609
655
|
return;
|
|
610
656
|
}
|
|
611
|
-
Promise.all([
|
|
612
|
-
.then(([
|
|
613
|
-
.catch(() =>
|
|
657
|
+
Promise.all([buildCheapStatus(), getResources().catch(() => null)])
|
|
658
|
+
.then(([base, resources]) => sendJson(res, { ...base, resources }))
|
|
659
|
+
.catch((error) => sendError(res, 500, `could not build the host status: ${errorMessage(error)}`));
|
|
614
660
|
return;
|
|
615
661
|
}
|
|
616
662
|
// Config write: apply an editable patch (model/lock live, the rest persisted).
|
|
@@ -33,6 +33,13 @@ export interface SchedulerOptions {
|
|
|
33
33
|
supervisor: SchedulerSupervisor;
|
|
34
34
|
loadModel: (model: Model) => Promise<void>;
|
|
35
35
|
logger?: ((message: string) => void) | null;
|
|
36
|
+
/**
|
|
37
|
+
* Called whenever the queue depth or the turn changes - i.e. whenever
|
|
38
|
+
* `stats()` would answer differently. The status event stream publishes from
|
|
39
|
+
* this instead of sampling, so "queued behind a model switch" reaches the UI
|
|
40
|
+
* the moment it becomes true rather than up to a poll later.
|
|
41
|
+
*/
|
|
42
|
+
onChange?: (() => void) | null;
|
|
36
43
|
}
|
|
37
44
|
/** A queued completion request bound to a resolved catalog model. */
|
|
38
45
|
export interface QueuedJob {
|
|
@@ -55,7 +62,8 @@ export declare class Scheduler {
|
|
|
55
62
|
queue: QueuedJob[];
|
|
56
63
|
lastTurnId: string | null;
|
|
57
64
|
pumping: boolean;
|
|
58
|
-
|
|
65
|
+
onChange: (() => void) | null;
|
|
66
|
+
constructor({ supervisor, loadModel, logger, onChange }: SchedulerOptions);
|
|
59
67
|
/** Id of the model that is actually loaded and ready, or null. */
|
|
60
68
|
get loadedId(): string | null;
|
|
61
69
|
/** How many requests may run at once against the resident model. */
|
|
@@ -3,10 +3,10 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
3
3
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
4
4
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
5
5
|
};
|
|
6
|
-
var _Scheduler_instances, _Scheduler_take, _Scheduler_serveTurn;
|
|
6
|
+
var _Scheduler_instances, _Scheduler_announce, _Scheduler_take, _Scheduler_serveTurn;
|
|
7
7
|
const MAX_CONCURRENCY = 16;
|
|
8
8
|
export class Scheduler {
|
|
9
|
-
constructor({ supervisor, loadModel, logger = null }) {
|
|
9
|
+
constructor({ supervisor, loadModel, logger = null, onChange = null }) {
|
|
10
10
|
_Scheduler_instances.add(this);
|
|
11
11
|
this.supervisor = supervisor;
|
|
12
12
|
this.loadModel = loadModel; // async (model) => resolves once it is ready
|
|
@@ -14,6 +14,7 @@ export class Scheduler {
|
|
|
14
14
|
this.queue = []; // { modelId, model, run, resolve, reject }
|
|
15
15
|
this.lastTurnId = null;
|
|
16
16
|
this.pumping = false;
|
|
17
|
+
this.onChange = onChange;
|
|
17
18
|
}
|
|
18
19
|
/** Id of the model that is actually loaded and ready, or null. */
|
|
19
20
|
get loadedId() {
|
|
@@ -34,6 +35,7 @@ export class Scheduler {
|
|
|
34
35
|
submit(model, run) {
|
|
35
36
|
return new Promise((resolve, reject) => {
|
|
36
37
|
this.queue.push({ modelId: model.id, model, run, resolve, reject });
|
|
38
|
+
__classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_announce).call(this);
|
|
37
39
|
queueMicrotask(() => this.pump());
|
|
38
40
|
});
|
|
39
41
|
}
|
|
@@ -61,6 +63,7 @@ export class Scheduler {
|
|
|
61
63
|
}
|
|
62
64
|
}
|
|
63
65
|
this.lastTurnId = turnId;
|
|
66
|
+
__classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_announce).call(this);
|
|
64
67
|
await __classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_serveTurn).call(this, turnId);
|
|
65
68
|
}
|
|
66
69
|
}
|
|
@@ -78,12 +81,21 @@ export class Scheduler {
|
|
|
78
81
|
return { queued: this.queue.length, waiting, lastTurn: this.lastTurnId };
|
|
79
82
|
}
|
|
80
83
|
}
|
|
81
|
-
_Scheduler_instances = new WeakSet(),
|
|
84
|
+
_Scheduler_instances = new WeakSet(), _Scheduler_announce = function _Scheduler_announce() {
|
|
85
|
+
try {
|
|
86
|
+
this.onChange?.();
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
// Status reporting is not allowed to fail a queued completion.
|
|
90
|
+
}
|
|
91
|
+
}, _Scheduler_take = function _Scheduler_take(pred) {
|
|
82
92
|
const kept = [];
|
|
83
93
|
const taken = [];
|
|
84
94
|
for (const job of this.queue)
|
|
85
95
|
(pred(job) ? taken : kept).push(job);
|
|
86
96
|
this.queue = kept;
|
|
97
|
+
if (taken.length > 0)
|
|
98
|
+
__classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_announce).call(this);
|
|
87
99
|
return taken;
|
|
88
100
|
}, _Scheduler_serveTurn =
|
|
89
101
|
/** Serve one model's snapshot with bounded concurrency. */
|
package/dist/service/serve.d.ts
CHANGED
|
@@ -37,7 +37,8 @@ export interface ServiceHandle {
|
|
|
37
37
|
supervisor: Supervisor;
|
|
38
38
|
host: string;
|
|
39
39
|
port: number;
|
|
40
|
-
model
|
|
40
|
+
/** The model loaded during startup, if one was available. */
|
|
41
|
+
model: Model | null;
|
|
41
42
|
/** Whether the listener terminates TLS (config.tls.mode !== "off"). */
|
|
42
43
|
secure: boolean;
|
|
43
44
|
/** The address to show a user: the MagicDNS/cert hostname when TLS is on, else the bind host. */
|
package/dist/service/serve.js
CHANGED
|
@@ -23,6 +23,7 @@ import * as results from "../ops/results.js";
|
|
|
23
23
|
import { createCpuSampler, sample as sampleSystem } from "../sysmon.js";
|
|
24
24
|
import { createHostApi } from "./host-api.js";
|
|
25
25
|
import { createRouter, Telemetry } from "./router.js";
|
|
26
|
+
import { BrainStatusPublisher } from "./status-events.js";
|
|
26
27
|
import { Supervisor } from "./supervisor.js";
|
|
27
28
|
import * as tailscale from "./tailscale.js";
|
|
28
29
|
import { CertManager, resolveTlsOptions } from "./tls.js";
|
|
@@ -99,14 +100,10 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
99
100
|
runLog.write(line);
|
|
100
101
|
onLog(line);
|
|
101
102
|
};
|
|
103
|
+
// The management API must be useful before any setup exists: the Brain page
|
|
104
|
+
// is where the owner downloads both a runtime and their first model. Keep the
|
|
105
|
+
// listener up without either; loadModel reports the missing prerequisite.
|
|
102
106
|
const runtime = resolveRuntime(config, env);
|
|
103
|
-
if (!runtime) {
|
|
104
|
-
throw new CommandError({
|
|
105
|
-
code: "NO_RUNTIME",
|
|
106
|
-
message: "no llama.cpp runtime available",
|
|
107
|
-
details: "run `otto brain runtime install` to download one, or install LM Studio",
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
107
|
const paths = resolveBrainPaths(env);
|
|
111
108
|
const tlsOptions = await resolveTlsOptions(config, paths);
|
|
112
109
|
// `listen.host: "tailscale"` binds the tailnet interface only (invisible to the
|
|
@@ -144,10 +141,22 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
144
141
|
// this, and every reader goes through a getter so nobody holds a stale array.
|
|
145
142
|
let catalog = scanModels(config, env);
|
|
146
143
|
const needle = modelNeedle ?? config.defaultModel ?? store.lastModelId ?? undefined;
|
|
147
|
-
|
|
148
|
-
|
|
144
|
+
let model = null;
|
|
145
|
+
if (catalog.length > 0) {
|
|
146
|
+
try {
|
|
147
|
+
model = needle ? pickModel(catalog, needle) : pickAutoModel(catalog);
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
// A removed default or last-used model must not take the management
|
|
151
|
+
// service down. An explicit CLI selection remains an actionable error.
|
|
152
|
+
if (modelNeedle)
|
|
153
|
+
throw error;
|
|
154
|
+
log(`note: ${error instanceof Error ? error.message : "configured model is unavailable"}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
let profile = model ? forModel(store, model, config.defaults) : null;
|
|
149
158
|
const gpu = await queryGpu();
|
|
150
|
-
if (gpu) {
|
|
159
|
+
if (gpu && model && profile) {
|
|
151
160
|
const fit = vram.fitToBudget({
|
|
152
161
|
model,
|
|
153
162
|
profile,
|
|
@@ -155,15 +164,18 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
155
164
|
totalVramBytes: gpu.totalBytes,
|
|
156
165
|
});
|
|
157
166
|
if (!fit.adjusted && !fit.budget.fits) {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
167
|
+
// Starting the host is what exposes the Library and model profile UI.
|
|
168
|
+
// An automatic startup candidate that cannot load must therefore leave
|
|
169
|
+
// the host alive and unloaded, not make the only recovery surface vanish.
|
|
170
|
+
log(`note: not loading ${model.displayName}: ${fit.reason ?? "does not fit in available VRAM"}`);
|
|
171
|
+
model = null;
|
|
172
|
+
profile = null;
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
if (fit.adjusted && fit.reason)
|
|
176
|
+
log(`note: ${fit.reason}`);
|
|
177
|
+
profile = fit.profile;
|
|
163
178
|
}
|
|
164
|
-
if (fit.adjusted && fit.reason)
|
|
165
|
-
log(`note: ${fit.reason}`);
|
|
166
|
-
profile = fit.profile;
|
|
167
179
|
}
|
|
168
180
|
const telemetry = new Telemetry();
|
|
169
181
|
const supervisor = new Supervisor({ runtime });
|
|
@@ -179,6 +191,12 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
179
191
|
// can never overlap two supervisor.start() calls, whichever caller triggers them.
|
|
180
192
|
let modelSwitchChain = Promise.resolve();
|
|
181
193
|
const loadModelUnsafe = async (target) => {
|
|
194
|
+
// A runtime can be installed from the Library tab after this service starts.
|
|
195
|
+
// Resolve it at load time so the user does not have to restart the brain.
|
|
196
|
+
supervisor.runtime = resolveRuntime(config, env);
|
|
197
|
+
if (!supervisor.runtime) {
|
|
198
|
+
throw new Error("no llama.cpp runtime available; install one from the Library tab");
|
|
199
|
+
}
|
|
182
200
|
const gpuInfo = await queryGpu();
|
|
183
201
|
let fitProfile = forModel(store, target, config.defaults);
|
|
184
202
|
if (gpuInfo) {
|
|
@@ -240,6 +258,11 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
240
258
|
// One CPU sampler for the lifetime of the service: it reports a busy fraction
|
|
241
259
|
// between successive calls, so a fresh one per request would always return null.
|
|
242
260
|
const cpuSampler = createCpuSampler();
|
|
261
|
+
// Constructed here, ahead of both consumers, because the router installs the
|
|
262
|
+
// snapshot builder into it and the management API serves it at
|
|
263
|
+
// /__host/events. One instance, so `capabilities.events` and the stream can
|
|
264
|
+
// never disagree about whether this brain publishes.
|
|
265
|
+
const statusEvents = new BrainStatusPublisher();
|
|
243
266
|
const hostApi = createHostApi({
|
|
244
267
|
supervisor,
|
|
245
268
|
getCatalog: () => catalog,
|
|
@@ -266,6 +289,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
266
289
|
getAllowWrite: allowWrite,
|
|
267
290
|
getModelsDir: () => managedModelsDir(config, env),
|
|
268
291
|
sampleResources: () => sampleSystem(cpuSampler, { host: supervisor.host, port: supervisor.internalPort }),
|
|
292
|
+
statusEvents,
|
|
269
293
|
});
|
|
270
294
|
const handler = withAuth(createRouter({
|
|
271
295
|
supervisor,
|
|
@@ -281,7 +305,10 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
281
305
|
applyConfigPatch,
|
|
282
306
|
getAllowConfigWrite: allowWrite,
|
|
283
307
|
hostApi,
|
|
284
|
-
|
|
308
|
+
// `buildCheapStatus` already samples `/slots`. Do not hit it a second
|
|
309
|
+
// time in parallel: the shared rate tracker needs one ordered timeline.
|
|
310
|
+
getResources: () => sampleSystem(cpuSampler),
|
|
311
|
+
statusEvents,
|
|
285
312
|
}), authToken);
|
|
286
313
|
// TLS terminates in-process when configured; otherwise plain HTTP. The cert
|
|
287
314
|
// manager issues/generates the first keypair before we listen, and hot-swaps
|
|
@@ -308,9 +335,17 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
308
335
|
server.listen(port, bindHost, resolve);
|
|
309
336
|
});
|
|
310
337
|
certManager?.start();
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
338
|
+
if (model && profile && runtime) {
|
|
339
|
+
await supervisor.start(model, profile);
|
|
340
|
+
store.lastModelId = model.id;
|
|
341
|
+
saveProfilesStore(store, paths);
|
|
342
|
+
}
|
|
343
|
+
else if (!runtime) {
|
|
344
|
+
log("ready: no llama.cpp runtime installed; use the Library tab to download one");
|
|
345
|
+
}
|
|
346
|
+
else {
|
|
347
|
+
log("ready: no model installed; use the Library tab to download one");
|
|
348
|
+
}
|
|
314
349
|
writePidFile({
|
|
315
350
|
pid: process.pid,
|
|
316
351
|
host: bindHost,
|
|
@@ -319,11 +354,15 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
319
354
|
secure: Boolean(tlsOptions),
|
|
320
355
|
displayHost,
|
|
321
356
|
}, env);
|
|
322
|
-
log(`ready: ${model
|
|
357
|
+
log(`ready: ${supervisor.model?.displayName ?? "no model loaded"} on ${bindHost}:${port}; run log ${runLog.path}`);
|
|
323
358
|
const stop = async () => {
|
|
324
359
|
certManager?.stop();
|
|
325
360
|
log("Brain service stopping");
|
|
361
|
+
// Before server.close(), which waits on open connections: a subscribed
|
|
362
|
+
// daemon holds an SSE response open indefinitely by design.
|
|
363
|
+
statusEvents.close();
|
|
326
364
|
await supervisor.stop();
|
|
365
|
+
server.closeIdleConnections?.();
|
|
327
366
|
await new Promise((resolve) => server.close(() => resolve()));
|
|
328
367
|
removePidFile(env);
|
|
329
368
|
};
|
|
@@ -332,7 +371,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
332
371
|
supervisor,
|
|
333
372
|
host: bindHost,
|
|
334
373
|
port,
|
|
335
|
-
model,
|
|
374
|
+
model: supervisor.model,
|
|
336
375
|
secure: Boolean(tlsOptions),
|
|
337
376
|
displayHost,
|
|
338
377
|
stop,
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The brain's authoritative status event source.
|
|
3
|
+
*
|
|
4
|
+
* The brain owns its own state, so it publishes it rather than being polled for
|
|
5
|
+
* it. One daemon subscribes over SSE (`GET /__host/events`) and fans the result
|
|
6
|
+
* out to every connected Otto client, replacing a per-client status poll that
|
|
7
|
+
* could never show a transition sooner than its own interval.
|
|
8
|
+
*
|
|
9
|
+
* Three properties this has to keep:
|
|
10
|
+
*
|
|
11
|
+
* - **Complete snapshots, never deltas.** A reader that missed an event, or
|
|
12
|
+
* reconnected after one, must not have to reconstruct anything: the newest
|
|
13
|
+
* snapshot is the whole truth. That is what makes reconnect repair on both
|
|
14
|
+
* sides idempotent.
|
|
15
|
+
* - **Coalescing lives here, not in the reader.** A timer tick that finds
|
|
16
|
+
* nothing changed emits nothing, and a completion's traffic counters are not
|
|
17
|
+
* a state change. Otherwise "push" would just be a poll with extra steps, and
|
|
18
|
+
* a busy brain would broadcast to every client for every token.
|
|
19
|
+
* - **No work while nobody is listening.** The sampler only runs while a
|
|
20
|
+
* subscriber is attached, so a brain the daemon never subscribed to (an older
|
|
21
|
+
* daemon, or one that is not running) pays nothing.
|
|
22
|
+
*
|
|
23
|
+
* The snapshot itself is deliberately the *cheap* status: no `resources`, whose
|
|
24
|
+
* collection spawns `nvidia-smi`. That stays an opt-in pull for the Overview tab.
|
|
25
|
+
*/
|
|
26
|
+
/** A complete cheap `/__host/status` body. Opaque here; the router assembles it. */
|
|
27
|
+
export type BrainStatusSnapshot = Record<string, unknown>;
|
|
28
|
+
export type BrainStatusListener = (snapshot: BrainStatusSnapshot) => void;
|
|
29
|
+
/**
|
|
30
|
+
* The fields whose change is worth waking every connected client for.
|
|
31
|
+
*
|
|
32
|
+
* Everything omitted here still rides in the payload; it just cannot *trigger*
|
|
33
|
+
* a payload on its own. The omissions are the churning ones:
|
|
34
|
+
*
|
|
35
|
+
* - `telemetry` totals and `recent` advance on every completion.
|
|
36
|
+
* - `logLineCount` advances on every llama-server log line.
|
|
37
|
+
* - `slots.contexts` is capacity detail that changes independently of work.
|
|
38
|
+
*
|
|
39
|
+
* `slots.threads` is intentionally included. It is sampled at a bounded 4 Hz,
|
|
40
|
+
* not notified per model token, so it gives the Overview live counts and rates
|
|
41
|
+
* without turning a fast model into a token-rate broadcast.
|
|
42
|
+
*
|
|
43
|
+
* `telemetry.warning` is kept: the reasoning-only advice is a state the UI
|
|
44
|
+
* shows, not a counter.
|
|
45
|
+
*/
|
|
46
|
+
export declare function statusChangeKey(snapshot: BrainStatusSnapshot): string;
|
|
47
|
+
export interface BrainStatusPublisherOptions {
|
|
48
|
+
/** Resample cadence while subscribed. See DEFAULT_SAMPLE_INTERVAL_MS. */
|
|
49
|
+
sampleIntervalMs?: number;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Owns the current snapshot and decides when it changed.
|
|
53
|
+
*
|
|
54
|
+
* Constructed before the router (so `serve.ts` can hand the same instance to
|
|
55
|
+
* both the router and the management API) and inert until the router installs
|
|
56
|
+
* the snapshot source with {@link setSource}. `/__host/capabilities` advertises
|
|
57
|
+
* `events` only once it is {@link ready}, so a brain can never claim a stream it
|
|
58
|
+
* cannot actually serve.
|
|
59
|
+
*/
|
|
60
|
+
export declare class BrainStatusPublisher {
|
|
61
|
+
#private;
|
|
62
|
+
constructor(options?: BrainStatusPublisherOptions);
|
|
63
|
+
/** Whether a snapshot source has been installed - i.e. events can be served. */
|
|
64
|
+
get ready(): boolean;
|
|
65
|
+
get listenerCount(): number;
|
|
66
|
+
/** Install the snapshot builder. Called once, by the router. */
|
|
67
|
+
setSource(source: () => Promise<BrainStatusSnapshot>): void;
|
|
68
|
+
/**
|
|
69
|
+
* Attach a listener. It receives the current snapshot immediately when one is
|
|
70
|
+
* known, and a fresh sample is taken right away so a subscriber that arrives
|
|
71
|
+
* during a transition is not handed a stale first frame.
|
|
72
|
+
*/
|
|
73
|
+
subscribe(listener: BrainStatusListener, onClose?: () => void): () => void;
|
|
74
|
+
/**
|
|
75
|
+
* Something authoritative changed - resample now rather than at the next tick.
|
|
76
|
+
* Cheap to over-call: an unchanged snapshot emits nothing.
|
|
77
|
+
*/
|
|
78
|
+
notify(): void;
|
|
79
|
+
/**
|
|
80
|
+
* Drop every listener and end the streams behind them.
|
|
81
|
+
*
|
|
82
|
+
* Ending them is not optional at shutdown: an open SSE response is an open
|
|
83
|
+
* connection, and `server.close()` waits for those, so a brain that only
|
|
84
|
+
* unsubscribed would hang on stop until its daemon happened to disconnect.
|
|
85
|
+
*/
|
|
86
|
+
close(): void;
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=status-events.d.ts.map
|