@otto-code/brain 0.8.10 → 0.8.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/calibrate.js +9 -0
- package/dist/commands/catalog.d.ts +1 -0
- package/dist/commands/catalog.js +1 -0
- package/dist/commands/pull.d.ts +1 -0
- package/dist/commands/pull.js +12 -3
- package/dist/commands/search.d.ts +1 -0
- package/dist/commands/search.js +12 -2
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/config/profile-edit.d.ts +88 -1
- package/dist/config/profile-edit.js +280 -29
- package/dist/config/profiles.js +16 -0
- package/dist/config/schema.d.ts +608 -0
- package/dist/config/schema.js +58 -0
- package/dist/config/store.js +7 -4
- package/dist/gguf.d.ts +7 -0
- package/dist/gguf.js +15 -2
- package/dist/models/download.d.ts +1 -1
- package/dist/models/download.js +2 -2
- package/dist/models/enrich.d.ts +6 -0
- package/dist/models/enrich.js +27 -1
- package/dist/models/index.d.ts +1 -1
- package/dist/models/index.js +4 -3
- package/dist/ops/calibrate.d.ts +38 -3
- package/dist/ops/calibrate.js +68 -19
- package/dist/ops/report.js +51 -1
- package/dist/ops/results.d.ts +57 -11
- package/dist/ops/results.js +75 -10
- package/dist/ops/sweep.d.ts +38 -1
- package/dist/ops/sweep.js +61 -10
- package/dist/runtime/args.d.ts +15 -2
- package/dist/runtime/args.js +60 -5
- package/dist/runtime/managed.js +2 -2
- package/dist/service/activity.d.ts +19 -0
- package/dist/service/activity.js +47 -4
- package/dist/service/host-api.d.ts +25 -4
- package/dist/service/host-api.js +82 -16
- package/dist/service/log-format.d.ts +18 -0
- package/dist/service/log-format.js +32 -0
- package/dist/service/router.d.ts +70 -2
- package/dist/service/router.js +219 -21
- package/dist/service/run-log.d.ts +6 -1
- package/dist/service/run-log.js +46 -4
- package/dist/service/scheduler.d.ts +227 -24
- package/dist/service/scheduler.js +395 -63
- package/dist/service/serve.d.ts +4 -0
- package/dist/service/serve.js +302 -117
- package/dist/service/status-events.d.ts +14 -1
- package/dist/service/status-events.js +111 -12
- package/dist/service/supervisor.d.ts +9 -7
- package/dist/service/supervisor.js +37 -12
- package/dist/sysmon.d.ts +15 -0
- package/dist/sysmon.js +56 -9
- package/dist/tui/app.d.ts +8 -2
- package/dist/tui/app.js +65 -17
- package/dist/types.d.ts +18 -0
- package/dist/vram.d.ts +37 -0
- package/dist/vram.js +57 -18
- package/package.json +1 -1
package/dist/service/serve.js
CHANGED
|
@@ -26,16 +26,18 @@ import * as archive from "../ops/archive.js";
|
|
|
26
26
|
import { calibrate } from "../ops/calibrate.js";
|
|
27
27
|
import { sweep } from "../ops/sweep.js";
|
|
28
28
|
import * as bench from "../bench/index.js";
|
|
29
|
-
import { createCpuSampler, sample as sampleSystem } from "../sysmon.js";
|
|
29
|
+
import { createCpuSampler, sample as sampleSystem, slots as sampleSlots } from "../sysmon.js";
|
|
30
30
|
import { createHostApi } from "./host-api.js";
|
|
31
31
|
import { errorMessage } from "./http-util.js";
|
|
32
|
-
import { createRouter, Telemetry } from "./router.js";
|
|
33
|
-
import {
|
|
32
|
+
import { createRouter, createSlotEraser, Telemetry } from "./router.js";
|
|
33
|
+
import { Scheduler } from "./scheduler.js";
|
|
34
|
+
import { BrainLogPublisher, BrainStatusPublisher } from "./status-events.js";
|
|
34
35
|
import { Supervisor } from "./supervisor.js";
|
|
35
36
|
import * as tailscale from "./tailscale.js";
|
|
36
37
|
import { CertManager, resolveTlsOptions } from "./tls.js";
|
|
37
38
|
import { removePidFile, writePidFile } from "./pid-lock.js";
|
|
38
39
|
import { createBrainRunLog } from "./run-log.js";
|
|
40
|
+
import { formatBrainLog } from "./log-format.js";
|
|
39
41
|
/** The effective config with secrets masked, for the `/__host/config` read. */
|
|
40
42
|
function redactConfig(config) {
|
|
41
43
|
return {
|
|
@@ -62,16 +64,77 @@ function collectEvals() {
|
|
|
62
64
|
}
|
|
63
65
|
}
|
|
64
66
|
const REMOTE_JOB_RETENTION_MS = 5 * 60000;
|
|
67
|
+
/** Model pulls only write the model store, so independent entries can transfer together. */
|
|
68
|
+
export function canRunAlongsideModelPull(kind) {
|
|
69
|
+
return kind === "pull" || kind === "runtime-remove";
|
|
70
|
+
}
|
|
71
|
+
export function componentOnlyArgs(args, components) {
|
|
72
|
+
const separator = args.lastIndexOf("--");
|
|
73
|
+
const beforeTarget = separator === -1 ? args : args.slice(0, separator);
|
|
74
|
+
const target = separator === -1 ? [] : args.slice(separator);
|
|
75
|
+
const base = [];
|
|
76
|
+
for (let index = 0; index < beforeTarget.length; index += 1) {
|
|
77
|
+
const arg = beforeTarget[index];
|
|
78
|
+
if (arg === "--primary-only")
|
|
79
|
+
continue;
|
|
80
|
+
if (arg === "--component") {
|
|
81
|
+
index += 1;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
base.push(arg);
|
|
85
|
+
}
|
|
86
|
+
return [
|
|
87
|
+
...base,
|
|
88
|
+
"--components-only",
|
|
89
|
+
...components.flatMap((component) => ["--component", component]),
|
|
90
|
+
...target,
|
|
91
|
+
];
|
|
92
|
+
}
|
|
65
93
|
class ServiceJobRunner {
|
|
66
|
-
constructor(onPullCompleted, runResidentJob) {
|
|
94
|
+
constructor(onPullCompleted, runResidentJob, scheduler, resolveTarget, log) {
|
|
67
95
|
this.onPullCompleted = onPullCompleted;
|
|
68
96
|
this.runResidentJob = runResidentJob;
|
|
97
|
+
this.scheduler = scheduler;
|
|
98
|
+
this.resolveTarget = resolveTarget;
|
|
99
|
+
this.log = log;
|
|
69
100
|
this.jobs = new Map();
|
|
70
101
|
}
|
|
71
|
-
|
|
102
|
+
area(kind) {
|
|
103
|
+
return kind === "pull" || kind === "runtime-install" || kind === "runtime-remove"
|
|
104
|
+
? "library"
|
|
105
|
+
: "model";
|
|
106
|
+
}
|
|
107
|
+
start(kind, target, args, pull) {
|
|
108
|
+
if (kind === "pull" && pull) {
|
|
109
|
+
const existing = [...this.jobs.values()].find((job) => job.kind === "pull" && job.status === "running" && job.pull?.entryKey === pull.entryKey);
|
|
110
|
+
if (existing?.pull) {
|
|
111
|
+
const components = pull.components.filter((component) => !existing.pull.components.has(component));
|
|
112
|
+
if (components.length > 0) {
|
|
113
|
+
for (const component of components)
|
|
114
|
+
existing.pull.components.add(component);
|
|
115
|
+
existing.pull.queue.push({ args: componentOnlyArgs(args, components), components });
|
|
116
|
+
existing.message = `Queued ${components.join(", ")}…`;
|
|
117
|
+
this.log("library", `job ${existing.id} queued bundle components: ${components.join(", ")}`);
|
|
118
|
+
}
|
|
119
|
+
return this.publicJob(existing);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const isResidentOperation = kind === "calibrate" || kind === "sweep" || kind === "bench";
|
|
72
123
|
const running = [...this.jobs.values()].find((job) => job.status === "running");
|
|
73
|
-
|
|
74
|
-
|
|
124
|
+
const activeNonDownload = [...this.jobs.values()].find((job) => job.status === "running" && job.kind !== "pull");
|
|
125
|
+
// Resident operations join Scheduler instead of rejecting one another. It
|
|
126
|
+
// owns their turn order with API requests and performs every model swap.
|
|
127
|
+
const conflict = isResidentOperation
|
|
128
|
+
? undefined
|
|
129
|
+
: canRunAlongsideModelPull(kind)
|
|
130
|
+
? activeNonDownload
|
|
131
|
+
: running;
|
|
132
|
+
if (conflict)
|
|
133
|
+
throw new Error(`Another operation is already running (${conflict.label}).`);
|
|
134
|
+
const residentTarget = isResidentOperation ? this.resolveTarget(target) : null;
|
|
135
|
+
if (isResidentOperation && !residentTarget) {
|
|
136
|
+
throw new Error("No installed model is available for this operation.");
|
|
137
|
+
}
|
|
75
138
|
const job = {
|
|
76
139
|
id: `brainjob_${randomUUID()}`,
|
|
77
140
|
kind,
|
|
@@ -89,21 +152,43 @@ class ServiceJobRunner {
|
|
|
89
152
|
finishedAt: null,
|
|
90
153
|
child: null,
|
|
91
154
|
controller: null,
|
|
155
|
+
...(kind === "pull" && pull
|
|
156
|
+
? {
|
|
157
|
+
pull: {
|
|
158
|
+
entryKey: pull.entryKey,
|
|
159
|
+
components: new Set(pull.components),
|
|
160
|
+
queue: [],
|
|
161
|
+
},
|
|
162
|
+
}
|
|
163
|
+
: {}),
|
|
92
164
|
};
|
|
93
165
|
this.jobs.set(job.id, job);
|
|
166
|
+
this.log(this.area(job.kind), `job ${job.id} started: ${job.label}`);
|
|
94
167
|
if (kind === "calibrate" || kind === "sweep" || kind === "bench") {
|
|
168
|
+
const model = residentTarget;
|
|
95
169
|
const controller = new AbortController();
|
|
96
170
|
job.controller = controller;
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
job.
|
|
171
|
+
job.queuePosition = this.scheduler.stats().queued + 1;
|
|
172
|
+
job.message = `Queued for ${model.displayName}`;
|
|
173
|
+
void this.scheduler
|
|
174
|
+
.submit(model, () => controller.signal.aborted
|
|
175
|
+
? Promise.reject(new Error("Operation canceled."))
|
|
176
|
+
: this.runResidentJob(kind, model.id, {
|
|
177
|
+
message: (value) => {
|
|
178
|
+
if (job.status === "running")
|
|
179
|
+
job.message = value.slice(-1000);
|
|
180
|
+
},
|
|
181
|
+
percent: (value) => {
|
|
182
|
+
if (job.status === "running")
|
|
183
|
+
job.percent = value;
|
|
184
|
+
},
|
|
185
|
+
}, controller.signal), {
|
|
186
|
+
kind: kind === "bench" ? "benchmark" : kind,
|
|
187
|
+
onStart: () => {
|
|
188
|
+
job.queuePosition = null;
|
|
189
|
+
job.message = `${job.label} started`;
|
|
105
190
|
},
|
|
106
|
-
}
|
|
191
|
+
})
|
|
107
192
|
.then(() => this.finish(job, "succeeded", null))
|
|
108
193
|
.catch((error) => this.finish(job, controller.signal.aborted ? "canceled" : "failed", errorMessage(error)));
|
|
109
194
|
return this.publicJob(job);
|
|
@@ -113,20 +198,36 @@ class ServiceJobRunner {
|
|
|
113
198
|
const entry = process.argv[1];
|
|
114
199
|
if (!entry)
|
|
115
200
|
throw new Error("The brain service has no CLI entry point.");
|
|
201
|
+
this.startChild(job, args, entry);
|
|
202
|
+
return this.publicJob(job);
|
|
203
|
+
}
|
|
204
|
+
startChild(job, args, entry) {
|
|
116
205
|
const child = spawn(process.execPath, [entry, ...args], {
|
|
117
206
|
cwd: process.cwd(),
|
|
118
207
|
env: process.env,
|
|
119
|
-
stdio: ["ignore", "
|
|
208
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
120
209
|
windowsHide: true,
|
|
121
210
|
});
|
|
122
211
|
job.child = child;
|
|
212
|
+
this.log(this.area(job.kind), `job ${job.id} spawned pid ${child.pid ?? "unknown"}: ${args.join(" ")}`);
|
|
213
|
+
child.stdout?.setEncoding("utf8");
|
|
123
214
|
child.stderr?.setEncoding("utf8");
|
|
124
|
-
child.
|
|
215
|
+
child.stdout?.on("data", (chunk) => this.ingestOutput(job, "stdout", chunk));
|
|
216
|
+
child.stderr?.on("data", (chunk) => this.ingestOutput(job, "stderr", chunk));
|
|
125
217
|
child.once("error", (error) => this.finish(job, "failed", error.message));
|
|
126
218
|
child.once("close", (code) => {
|
|
127
219
|
if (job.status !== "running")
|
|
128
220
|
return;
|
|
129
|
-
|
|
221
|
+
job.child = null;
|
|
222
|
+
if (code === 0 && job.kind === "pull" && job.pull) {
|
|
223
|
+
const next = job.pull.queue.shift();
|
|
224
|
+
if (next) {
|
|
225
|
+
job.message = `Queued ${next.components.join(", ")}…`;
|
|
226
|
+
this.startChild(job, next.args, entry);
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (code === 0 && job.kind === "pull") {
|
|
130
231
|
try {
|
|
131
232
|
// Downloads happen in a child process, but inventory is served from
|
|
132
233
|
// this process's in-memory scan. Reconcile before reporting success
|
|
@@ -140,9 +241,8 @@ class ServiceJobRunner {
|
|
|
140
241
|
}
|
|
141
242
|
this.finish(job, code === 0 ? "succeeded" : "failed", code === 0 ? null : (job.message ?? `Exited with code ${code}.`));
|
|
142
243
|
});
|
|
143
|
-
return this.publicJob(job);
|
|
144
244
|
}
|
|
145
|
-
async query(args) {
|
|
245
|
+
async query(args, area = "library") {
|
|
146
246
|
const entry = process.argv[1];
|
|
147
247
|
if (!entry)
|
|
148
248
|
throw new Error("The brain service has no CLI entry point.");
|
|
@@ -157,10 +257,21 @@ class ServiceJobRunner {
|
|
|
157
257
|
let err = "";
|
|
158
258
|
child.stdout?.setEncoding("utf8");
|
|
159
259
|
child.stderr?.setEncoding("utf8");
|
|
160
|
-
|
|
161
|
-
child.
|
|
162
|
-
|
|
260
|
+
this.log(area, `query started: ${args.join(" ")}`);
|
|
261
|
+
child.stdout?.on("data", (chunk) => {
|
|
262
|
+
out += chunk;
|
|
263
|
+
this.logOutput(area, "query stdout", chunk);
|
|
264
|
+
});
|
|
265
|
+
child.stderr?.on("data", (chunk) => {
|
|
266
|
+
err += chunk;
|
|
267
|
+
this.logOutput(area, "query stderr", chunk);
|
|
268
|
+
});
|
|
269
|
+
child.once("error", (error) => {
|
|
270
|
+
this.log(area, `query failed to start: ${error.message}`);
|
|
271
|
+
reject(error);
|
|
272
|
+
});
|
|
163
273
|
child.once("close", (code) => {
|
|
274
|
+
this.log(area, `query exited with code ${code}`);
|
|
164
275
|
if (code !== 0)
|
|
165
276
|
return reject(new Error(err.trim() || `Exited with code ${code}.`));
|
|
166
277
|
try {
|
|
@@ -198,10 +309,11 @@ class ServiceJobRunner {
|
|
|
198
309
|
else {
|
|
199
310
|
child.kill("SIGTERM");
|
|
200
311
|
}
|
|
312
|
+
this.log(this.area(job.kind), `canceling job ${job.id}`);
|
|
201
313
|
this.finish(job, "canceled", "Canceled.");
|
|
202
314
|
return this.list();
|
|
203
315
|
}
|
|
204
|
-
ingestOutput(job, chunk) {
|
|
316
|
+
ingestOutput(job, source, chunk) {
|
|
205
317
|
for (const line of chunk
|
|
206
318
|
.split(/[\r\n]+/u)
|
|
207
319
|
.map((value) => value.trim())
|
|
@@ -209,6 +321,7 @@ class ServiceJobRunner {
|
|
|
209
321
|
const progress = /(\d{1,3})\s*%/u.exec(line);
|
|
210
322
|
if (progress)
|
|
211
323
|
job.percent = Math.max(0, Math.min(100, Number(progress[1])));
|
|
324
|
+
this.log(this.area(job.kind), `job ${job.id} ${source}: ${line}`);
|
|
212
325
|
// The final JSON result is not a useful status label. Keep progress and
|
|
213
326
|
// actionable text instead, so a failed bundle pull tells the user why.
|
|
214
327
|
if (line !== "[" && !/^[\]{}",]+$/u.test(line))
|
|
@@ -225,8 +338,17 @@ class ServiceJobRunner {
|
|
|
225
338
|
job.finishedAt = new Date().toISOString();
|
|
226
339
|
if (status === "succeeded")
|
|
227
340
|
job.percent = 100;
|
|
341
|
+
this.log(this.area(job.kind), `job ${job.id} ${status}${error ? `: ${error}` : ""}`);
|
|
342
|
+
}
|
|
343
|
+
logOutput(area, source, chunk) {
|
|
344
|
+
for (const line of chunk
|
|
345
|
+
.split(/[\r\n]+/u)
|
|
346
|
+
.map((entry) => entry.trim())
|
|
347
|
+
.filter(Boolean)) {
|
|
348
|
+
this.log(area, `${source}: ${line}`);
|
|
349
|
+
}
|
|
228
350
|
}
|
|
229
|
-
publicJob({ child: _child, controller: _controller, ...job }) {
|
|
351
|
+
publicJob({ child: _child, controller: _controller, pull: _pull, ...job }) {
|
|
230
352
|
return job;
|
|
231
353
|
}
|
|
232
354
|
}
|
|
@@ -272,9 +394,16 @@ function withAuth(inner, token) {
|
|
|
272
394
|
}
|
|
273
395
|
export async function startService({ config, modelNeedle, env = process.env, onLog = () => { }, }) {
|
|
274
396
|
const runLog = createBrainRunLog(env);
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
397
|
+
const logEvents = new BrainLogPublisher();
|
|
398
|
+
const log = (area, message) => {
|
|
399
|
+
const line = formatBrainLog(area, message);
|
|
400
|
+
for (const entry of runLog.write(line)) {
|
|
401
|
+
logEvents.publish(entry);
|
|
402
|
+
// The daemon also captures this foreground child's stderr before the
|
|
403
|
+
// management listener exists. Give it the exact durable entry, rather
|
|
404
|
+
// than a second un-timestamped rendering of the same event.
|
|
405
|
+
onLog(entry);
|
|
406
|
+
}
|
|
278
407
|
};
|
|
279
408
|
// The management API must be useful before any setup exists: the Brain page
|
|
280
409
|
// is where the owner downloads both a runtime and their first model. Keep the
|
|
@@ -331,7 +460,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
331
460
|
// service down. An explicit CLI selection remains an actionable error.
|
|
332
461
|
if (modelNeedle)
|
|
333
462
|
throw error;
|
|
334
|
-
log(`note: ${error instanceof Error ? error.message : "configured model is unavailable"}`);
|
|
463
|
+
log("server", `note: ${error instanceof Error ? error.message : "configured model is unavailable"}`);
|
|
335
464
|
}
|
|
336
465
|
}
|
|
337
466
|
let profile = model ? forModel(store, model, config.defaults) : null;
|
|
@@ -347,24 +476,25 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
347
476
|
// Starting the host is what exposes the Library and model profile UI.
|
|
348
477
|
// An automatic startup candidate that cannot load must therefore leave
|
|
349
478
|
// the host alive and unloaded, not make the only recovery surface vanish.
|
|
350
|
-
log(`note: not loading ${model.displayName}: ${fit.reason ?? "does not fit in available VRAM"}`);
|
|
479
|
+
log("model", `note: not loading ${model.displayName}: ${fit.reason ?? "does not fit in available VRAM"}`);
|
|
351
480
|
model = null;
|
|
352
481
|
profile = null;
|
|
353
482
|
}
|
|
354
483
|
else {
|
|
355
484
|
if (fit.adjusted && fit.reason)
|
|
356
|
-
log(`note: ${fit.reason}`);
|
|
485
|
+
log("model", `note: ${fit.reason}`);
|
|
357
486
|
profile = fit.profile;
|
|
358
487
|
}
|
|
359
488
|
}
|
|
360
489
|
const telemetry = new Telemetry();
|
|
361
|
-
const supervisor = new Supervisor({
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
490
|
+
const supervisor = new Supervisor({
|
|
491
|
+
runtime,
|
|
492
|
+
paths,
|
|
493
|
+
getProfilesStore: () => store,
|
|
494
|
+
logVerbosity: config.runtime.logVerbosity,
|
|
366
495
|
});
|
|
367
|
-
supervisor.on("
|
|
496
|
+
supervisor.on("log", (line) => log("server", line));
|
|
497
|
+
supervisor.on("crashed", (error) => log("model", `FATAL ${error}`));
|
|
368
498
|
// Serialize model switches: the router queues request-driven switches, but the
|
|
369
499
|
// config path (POST /__host/config) calls loadModel directly. Chaining here
|
|
370
500
|
// guarantees two switches (e.g. a config write racing a request-driven switch)
|
|
@@ -455,13 +585,6 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
455
585
|
: null;
|
|
456
586
|
if (!targetModel)
|
|
457
587
|
throw new Error("No installed model is available for this operation.");
|
|
458
|
-
const restoreModel = supervisor.model;
|
|
459
|
-
const restoreProfile = supervisor.profile;
|
|
460
|
-
const restore = async () => {
|
|
461
|
-
if (restoreModel && restoreProfile && !signal.aborted) {
|
|
462
|
-
await supervisor.start(restoreModel, restoreProfile, { preserveLogs: true });
|
|
463
|
-
}
|
|
464
|
-
};
|
|
465
588
|
if (kind === "calibrate") {
|
|
466
589
|
const runtime = supervisor.runtime ?? resolveRuntime(config, env);
|
|
467
590
|
if (!runtime)
|
|
@@ -469,34 +592,29 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
469
592
|
const profile = forModel(store, targetModel, config.defaults);
|
|
470
593
|
update.message(`Calibrating ${targetModel.displayName}`);
|
|
471
594
|
supervisor.recordLog(`operation calibrate: ${targetModel.displayName}`);
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
supervisor.recordLog(`operation calibrate: saved measurement for ${targetModel.displayName}`);
|
|
496
|
-
}
|
|
497
|
-
finally {
|
|
498
|
-
await restore();
|
|
499
|
-
}
|
|
595
|
+
const measurement = await calibrate({
|
|
596
|
+
runtime,
|
|
597
|
+
model: targetModel,
|
|
598
|
+
profile,
|
|
599
|
+
supervisor,
|
|
600
|
+
onProgress: (event) => {
|
|
601
|
+
ensureActive();
|
|
602
|
+
const message = event.phase === "loading"
|
|
603
|
+
? `Calibrating ${event.contextSize.toLocaleString()} context`
|
|
604
|
+
: event.phase === "measured"
|
|
605
|
+
? `Measured ${event.contextSize.toLocaleString()} context`
|
|
606
|
+
: (event.reason ??
|
|
607
|
+
event.error ??
|
|
608
|
+
`Skipped ${event.contextSize.toLocaleString()} context`);
|
|
609
|
+
update.message(message);
|
|
610
|
+
supervisor.recordLog(`operation calibrate: ${message}`);
|
|
611
|
+
},
|
|
612
|
+
});
|
|
613
|
+
ensureActive();
|
|
614
|
+
putCalibration(store, targetModel, profile, measurement);
|
|
615
|
+
saveProfilesStore(store, paths);
|
|
616
|
+
update.percent(100);
|
|
617
|
+
supervisor.recordLog(`operation calibrate: saved measurement for ${targetModel.displayName}`);
|
|
500
618
|
return;
|
|
501
619
|
}
|
|
502
620
|
if (kind === "sweep") {
|
|
@@ -506,42 +624,37 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
506
624
|
const profile = forModel(store, targetModel, config.defaults);
|
|
507
625
|
update.message(`Sweeping ${targetModel.displayName}`);
|
|
508
626
|
supervisor.recordLog(`operation sweep: ${targetModel.displayName}`);
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
supervisor.recordLog(`operation sweep: saved budget ${report.recommended}`);
|
|
534
|
-
}
|
|
535
|
-
update.percent(100);
|
|
536
|
-
}
|
|
537
|
-
finally {
|
|
538
|
-
await restore();
|
|
627
|
+
const report = await sweep({
|
|
628
|
+
runtime,
|
|
629
|
+
model: targetModel,
|
|
630
|
+
profile,
|
|
631
|
+
supervisor,
|
|
632
|
+
onProgress: (event) => {
|
|
633
|
+
ensureActive();
|
|
634
|
+
const message = event.phase === "loading"
|
|
635
|
+
? `Budget ${event.budget}: loading`
|
|
636
|
+
: event.phase === "generating"
|
|
637
|
+
? `Budget ${event.budget}: generating`
|
|
638
|
+
: event.phase === "done"
|
|
639
|
+
? `Budget ${event.budget}: complete`
|
|
640
|
+
: `Budget ${event.budget}: ${event.error ?? "failed"}`;
|
|
641
|
+
update.message(message);
|
|
642
|
+
supervisor.recordLog(`operation sweep: ${message}`);
|
|
643
|
+
},
|
|
644
|
+
});
|
|
645
|
+
ensureActive();
|
|
646
|
+
if (report.recommended !== null) {
|
|
647
|
+
profile.reasoningBudget = report.recommended;
|
|
648
|
+
put(store, targetModel, profile);
|
|
649
|
+
saveProfilesStore(store, paths);
|
|
650
|
+
supervisor.recordLog(`operation sweep: saved budget ${report.recommended}`);
|
|
539
651
|
}
|
|
652
|
+
update.percent(100);
|
|
540
653
|
return;
|
|
541
654
|
}
|
|
542
655
|
update.message(`Benchmarking ${targetModel.displayName}`);
|
|
543
656
|
supervisor.recordLog(`operation benchmark: ${targetModel.displayName}`);
|
|
544
|
-
|
|
657
|
+
// Scheduler loaded this exact model before admitting the exclusive turn.
|
|
545
658
|
ensureActive();
|
|
546
659
|
supervisor.recordLog(`operation benchmark: resident model ready`);
|
|
547
660
|
const profile = supervisor.profile ?? forModel(store, targetModel, config.defaults);
|
|
@@ -590,7 +703,51 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
590
703
|
update.percent(100);
|
|
591
704
|
supervisor.recordLog(`operation benchmark: saved result for ${targetModel.displayName}`);
|
|
592
705
|
};
|
|
593
|
-
const
|
|
706
|
+
const scheduler = new Scheduler({
|
|
707
|
+
supervisor,
|
|
708
|
+
loadModel,
|
|
709
|
+
logger: (message) => log("api", `WARN ${message}`),
|
|
710
|
+
onChange: () => statusEvents.notify(),
|
|
711
|
+
// Live admission: the engine's own /slots report is the source of truth
|
|
712
|
+
// for how many sequence slots are actually free right now. The profile's
|
|
713
|
+
// parallelSlots sizes the KV pool at launch; this keeps dispatch honest
|
|
714
|
+
// while llama-server is mid-eviction or saturated, and degrades to the
|
|
715
|
+
// static count when the sample is unavailable.
|
|
716
|
+
// Answers with the free slots NAMED, not just counted. The count gates
|
|
717
|
+
// admission; the ids let the scheduler pin each admitted completion to a
|
|
718
|
+
// distinct slot (`id_slot`), which is what lets the proxy attribute a
|
|
719
|
+
// request's stage - "thinking" above all, which llama-server cannot report -
|
|
720
|
+
// to the exact slot row the Overview panel draws. Handing back only a count
|
|
721
|
+
// (as this did before) leaves every request unpinned and the panel unable to
|
|
722
|
+
// say which slot is thinking and which is emitting tokens.
|
|
723
|
+
freeSlots: async () => {
|
|
724
|
+
if (supervisor.state !== "ready")
|
|
725
|
+
return null;
|
|
726
|
+
try {
|
|
727
|
+
const slots = await sampleSlots({
|
|
728
|
+
host: supervisor.host,
|
|
729
|
+
port: supervisor.internalPort,
|
|
730
|
+
});
|
|
731
|
+
return slots ? { idle: slots.idle, ids: slots.idleSlots } : null;
|
|
732
|
+
}
|
|
733
|
+
catch {
|
|
734
|
+
return null;
|
|
735
|
+
}
|
|
736
|
+
},
|
|
737
|
+
// Erase a slot's retained KV when it is handed to a different chat, so one
|
|
738
|
+
// chat's KV never bleeds into the next chat's thinking. The scheduler
|
|
739
|
+
// decides the handoff from the owner map; this is the engine-side wipe on
|
|
740
|
+
// the private port, resolved once the engine acknowledges it (see
|
|
741
|
+
// Scheduler.OWNERSHIP). The router that shares this scheduler clears the
|
|
742
|
+
// owner map on the supervisor's `starting` state.
|
|
743
|
+
eraseSlot: createSlotEraser(supervisor.host, supervisor.internalPort),
|
|
744
|
+
});
|
|
745
|
+
const jobs = new ServiceJobRunner(rescanCatalog, runResidentJob, scheduler, (target) => {
|
|
746
|
+
const modelId = target ?? supervisor.model?.id ?? store.lastModelId ?? null;
|
|
747
|
+
return modelId
|
|
748
|
+
? (catalog.find((candidate) => candidate.id === modelId || candidate.displayName === modelId) ?? null)
|
|
749
|
+
: null;
|
|
750
|
+
}, log);
|
|
594
751
|
// Assigned once `stop` exists below. This indirection lets the management API
|
|
595
752
|
// answer a remote restart request before closing its own socket.
|
|
596
753
|
let requestRestart = () => { };
|
|
@@ -611,6 +768,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
611
768
|
}
|
|
612
769
|
},
|
|
613
770
|
loadModel,
|
|
771
|
+
scheduler,
|
|
614
772
|
// The same gate as POST /__host/config. Deleting someone's model files over
|
|
615
773
|
// the network is strictly more dangerous than changing their default model,
|
|
616
774
|
// so it does not get a weaker one.
|
|
@@ -618,13 +776,19 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
618
776
|
getModelsDir: () => managedModelsDir(config, env),
|
|
619
777
|
sampleResources: () => sampleSystem(cpuSampler, { host: supervisor.host, port: supervisor.internalPort }),
|
|
620
778
|
statusEvents,
|
|
779
|
+
logEvents,
|
|
621
780
|
jobs,
|
|
781
|
+
runLog,
|
|
622
782
|
restart: () => requestRestart(),
|
|
783
|
+
log,
|
|
623
784
|
});
|
|
624
785
|
const handler = withAuth(createRouter({
|
|
625
786
|
supervisor,
|
|
626
787
|
telemetry,
|
|
627
|
-
logger: {
|
|
788
|
+
logger: {
|
|
789
|
+
info: (m) => log("api", m),
|
|
790
|
+
warn: (m) => log("api", `WARN ${m}`),
|
|
791
|
+
},
|
|
628
792
|
getCatalog: () => catalog,
|
|
629
793
|
loadModel,
|
|
630
794
|
version: resolveVersion(),
|
|
@@ -639,6 +803,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
639
803
|
// time in parallel: the shared rate tracker needs one ordered timeline.
|
|
640
804
|
getResources: () => sampleSystem(cpuSampler),
|
|
641
805
|
statusEvents,
|
|
806
|
+
scheduler,
|
|
642
807
|
}), authToken);
|
|
643
808
|
// TLS terminates in-process when configured; otherwise plain HTTP. The cert
|
|
644
809
|
// manager issues/generates the first keypair before we listen, and hot-swaps
|
|
@@ -646,12 +811,18 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
646
811
|
let certManager = null;
|
|
647
812
|
let server;
|
|
648
813
|
if (tlsOptions) {
|
|
649
|
-
certManager = new CertManager({
|
|
814
|
+
certManager = new CertManager({
|
|
815
|
+
...tlsOptions,
|
|
816
|
+
logger: {
|
|
817
|
+
info: (message) => log("server", message),
|
|
818
|
+
warn: (message) => log("server", message),
|
|
819
|
+
},
|
|
820
|
+
});
|
|
650
821
|
const secure = await certManager.load();
|
|
651
822
|
const httpsServer = https.createServer({ key: secure.key, cert: secure.cert }, handler);
|
|
652
823
|
certManager.on("renewed", (pair) => {
|
|
653
824
|
httpsServer.setSecureContext({ key: pair.key, cert: pair.cert });
|
|
654
|
-
|
|
825
|
+
log("server", "note: TLS certificate hot-swapped");
|
|
655
826
|
});
|
|
656
827
|
server = httpsServer;
|
|
657
828
|
}
|
|
@@ -660,10 +831,20 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
660
831
|
}
|
|
661
832
|
server.keepAliveTimeout = 75000;
|
|
662
833
|
server.requestTimeout = 0;
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
834
|
+
try {
|
|
835
|
+
await new Promise((resolve, reject) => {
|
|
836
|
+
server.once("error", reject);
|
|
837
|
+
server.listen(port, bindHost, resolve);
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
catch (error) {
|
|
841
|
+
// A listener can fail before the host API exists, so this cannot travel
|
|
842
|
+
// through the host's own SSE endpoint. It still belongs to this service
|
|
843
|
+
// session: the foreground daemon child relays this timestamped entry until
|
|
844
|
+
// that endpoint becomes available.
|
|
845
|
+
log("server", `FATAL Brain service startup failed: ${errorMessage(error)}`);
|
|
846
|
+
throw error;
|
|
847
|
+
}
|
|
667
848
|
certManager?.start();
|
|
668
849
|
if (model && profile && runtime) {
|
|
669
850
|
await supervisor.start(model, profile);
|
|
@@ -672,10 +853,10 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
672
853
|
saveProfilesStore(store, paths);
|
|
673
854
|
}
|
|
674
855
|
else if (!runtime) {
|
|
675
|
-
log("ready: no llama.cpp runtime installed; use the Library tab to download one");
|
|
856
|
+
log("server", "ready: no llama.cpp runtime installed; use the Library tab to download one");
|
|
676
857
|
}
|
|
677
858
|
else {
|
|
678
|
-
log("ready: no model installed; use the Library tab to download one");
|
|
859
|
+
log("server", "ready: no model installed; use the Library tab to download one");
|
|
679
860
|
}
|
|
680
861
|
writePidFile({
|
|
681
862
|
pid: process.pid,
|
|
@@ -685,14 +866,18 @@ export async function startService({ config, modelNeedle, env = process.env, onL
|
|
|
685
866
|
secure: Boolean(tlsOptions),
|
|
686
867
|
displayHost,
|
|
687
868
|
}, env);
|
|
688
|
-
log(`ready: ${supervisor.model?.displayName ?? "no model loaded"} on ${bindHost}:${port}; run log ${runLog.path}`);
|
|
869
|
+
log("server", `ready: ${supervisor.model?.displayName ?? "no model loaded"} on ${bindHost}:${port}; run log ${runLog.path}`);
|
|
689
870
|
const stop = async () => {
|
|
690
871
|
certManager?.stop();
|
|
691
|
-
log("Brain service stopping");
|
|
872
|
+
log("server", "Brain service stopping");
|
|
873
|
+
await supervisor.stop();
|
|
874
|
+
// Publish the terminal outcome before closing the SSE responses below.
|
|
875
|
+
// Once the listener is closed, the daemon can still report the child exit,
|
|
876
|
+
// but it cannot receive this service-owned, durable session-log entry.
|
|
877
|
+
log("server", "Brain service stopped");
|
|
692
878
|
// Before server.close(), which waits on open connections: a subscribed
|
|
693
879
|
// daemon holds an SSE response open indefinitely by design.
|
|
694
880
|
statusEvents.close();
|
|
695
|
-
await supervisor.stop();
|
|
696
881
|
server.closeIdleConnections?.();
|
|
697
882
|
await new Promise((resolve) => server.close(() => resolve()));
|
|
698
883
|
removePidFile(env);
|