@otto-code/brain 0.7.5

package/dist/service/scheduler.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Cooperative single-GPU scheduler with per-model concurrency.
+ *
+ * Only one model is resident at a time. Completion requests are queued rather
+ * than refused for "wrong / no model loaded".
+ *
+ *  - Requests for the resident model run concurrently, up to that model's
+ *    `parallelSlots` (the same number of sequence slots llama-server was
+ *    launched with — sending more would only queue inside llama-server). Extra
+ *    same-model requests wait for a free slot; they never trigger a load.
+ *  - Requests for a *different* model wait for a model switch. When the resident
+ *    model's current batch drains, the scheduler switches and serves the other
+ *    model's batch — so two clients wanting different models share the GPU by
+ *    taking turns.
+ *
+ * Fairness: after a model finishes a turn, the next turn prefers a *different*
+ * model when one is waiting, so a steady stream for model A cannot starve model
+ * B. A turn is a snapshot: requests that arrive for a model mid-turn wait for
+ * its next turn.
+ *
+ * The scheduler is transport-agnostic: a job is a resolved catalog model plus a
+ * `run()` that does the proxying, which keeps the turn logic unit-testable.
+ */
+import type { Model } from "../types.js";
+import type { Profile } from "../config/schema.js";
+/** The subset of the supervisor the scheduler observes to size and route turns. */
+export interface SchedulerSupervisor {
+    state: string;
+    model: Model | null;
+    profile: Profile | null;
+}
+export interface SchedulerOptions {
+    supervisor: SchedulerSupervisor;
+    loadModel: (model: Model) => Promise<void>;
+    logger?: ((message: string) => void) | null;
+}
+/** A queued completion request bound to a resolved catalog model. */
+export interface QueuedJob {
+    modelId: string;
+    model: Model;
+    run: () => Promise<unknown>;
+    resolve: (value: unknown) => void;
+    reject: (error: unknown) => void;
+}
+export interface SchedulerStats {
+    queued: number;
+    waiting: Record<string, number>;
+    lastTurn: string | null;
+}
+export declare class Scheduler {
+    #private;
+    supervisor: SchedulerSupervisor;
+    loadModel: (model: Model) => Promise<void>;
+    logger: ((message: string) => void) | null;
+    queue: QueuedJob[];
+    lastTurnId: string | null;
+    pumping: boolean;
+    constructor({ supervisor, loadModel, logger }: SchedulerOptions);
+    /** Id of the model that is actually loaded and ready, or null. */
+    get loadedId(): string | null;
+    /** How many requests may run at once against the resident model. */
+    get concurrency(): number;
+    /**
+     * Queue a job for an already-resolved catalog model. `run` is invoked once
+     * that model is the resident one; the returned promise settles when run does.
+     * Pumping is deferred a microtask so a burst of requests submitted together
+     * share one turn rather than the first one snapshotting a turn by itself.
+     */
+    submit(model: Model, run: () => Promise<unknown>): Promise<unknown>;
+    pump(): Promise<void>;
+    /** Queue snapshot for the status endpoint / UI. */
+    stats(): SchedulerStats;
+}
+//# sourceMappingURL=scheduler.d.ts.map

package/dist/service/scheduler.js

@@ -0,0 +1,110 @@
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _Scheduler_instances, _Scheduler_take, _Scheduler_serveTurn;
+const MAX_CONCURRENCY = 16;
+export class Scheduler {
+    constructor({ supervisor, loadModel, logger = null }) {
+        _Scheduler_instances.add(this);
+        this.supervisor = supervisor;
+        this.loadModel = loadModel; // async (model) => resolves once it is ready
+        this.logger = logger; // optional (message: string) => void
+        this.queue = []; // { modelId, model, run, resolve, reject }
+        this.lastTurnId = null;
+        this.pumping = false;
+    }
+    /** Id of the model that is actually loaded and ready, or null. */
+    get loadedId() {
+        return this.supervisor.state === "ready" && this.supervisor.model
+            ? this.supervisor.model.id
+            : null;
+    }
+    /** How many requests may run at once against the resident model. */
+    get concurrency() {
+        return Math.max(1, Math.min(MAX_CONCURRENCY, this.supervisor.profile?.parallelSlots || 1));
+    }
+    /**
+     * Queue a job for an already-resolved catalog model. `run` is invoked once
+     * that model is the resident one; the returned promise settles when run does.
+     * Pumping is deferred a microtask so a burst of requests submitted together
+     * share one turn rather than the first one snapshotting a turn by itself.
+     */
+    submit(model, run) {
+        return new Promise((resolve, reject) => {
+            this.queue.push({ modelId: model.id, model, run, resolve, reject });
+            queueMicrotask(() => this.pump());
+        });
+    }
+    async pump() {
+        if (this.pumping)
+            return;
+        this.pumping = true;
+        try {
+            while (this.queue.length) {
+                // Prefer a model other than the one that just took a turn, so the two
+                // sides alternate; fall back to the head of the queue when only one
+                // model is waiting (it simply keeps its turn).
+                const pick = this.queue.find((j) => j.modelId !== this.lastTurnId) || this.queue[0];
+                const turnId = pick.modelId;
+                if (this.loadedId !== turnId) {
+                    try {
+                        this.logger?.(`switching to ${pick.model.displayName}`);
+                        await this.loadModel(pick.model);
+                    }
+                    catch (error) {
+                        // The model would not load — fail exactly its queued jobs and move on.
+                        for (const job of __classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_take).call(this, (j) => j.modelId === turnId))
+                            job.reject(error);
+                        continue;
+                    }
+                }
+                this.lastTurnId = turnId;
+                await __classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_serveTurn).call(this, turnId);
+            }
+        }
+        finally {
+            this.pumping = false;
+        }
+    }
+    /** Queue snapshot for the status endpoint / UI. */
+    stats() {
+        const waiting = {};
+        for (const job of this.queue) {
+            const name = job.model.displayName;
+            waiting[name] = (waiting[name] || 0) + 1;
+        }
+        return { queued: this.queue.length, waiting, lastTurn: this.lastTurnId };
+    }
+}
+_Scheduler_instances = new WeakSet(), _Scheduler_take = function _Scheduler_take(pred) {
+    const kept = [];
+    const taken = [];
+    for (const job of this.queue)
+        (pred(job) ? taken : kept).push(job);
+    this.queue = kept;
+    return taken;
+}, _Scheduler_serveTurn = 
+/** Serve one model's snapshot with bounded concurrency. */
+async function _Scheduler_serveTurn(turnId) {
+    const jobs = __classPrivateFieldGet(this, _Scheduler_instances, "m", _Scheduler_take).call(this, (j) => j.modelId === turnId);
+    let next = 0;
+    const worker = async () => {
+        while (next < jobs.length) {
+            const job = jobs[next];
+            next += 1;
+            try {
+                job.resolve(await job.run());
+            }
+            catch (error) {
+                job.reject(error);
+            }
+        }
+    };
+    const pool = [];
+    for (let i = 0; i < Math.min(this.concurrency, jobs.length); i += 1)
+        pool.push(worker());
+    await Promise.all(pool);
+};
+//# sourceMappingURL=scheduler.js.map

package/dist/service/serve.d.ts

@@ -0,0 +1,41 @@
+/**
+ * The headless brain service: the router + supervisor bound to a port, with the
+ * VRAM fit, on-demand model switching, remote auth, built-in TLS, and pid-file
+ * lifecycle. Used both by `otto brain serve` (foreground) and by a detached
+ * `otto brain start`. It stays provider-neutral about the runtime source — it
+ * takes whatever resolveRuntime picks (managed or LM Studio).
+ *
+ * TLS is served in-process (config.tls): HTTPS with a files / self-signed /
+ * tailscale certificate, hot-swapped on renewal. This is what lets the brain be
+ * exposed securely over a network with no relay in front of it.
+ */
+import http from "node:http";
+import type { BrainConfig } from "../config/schema.js";
+import type { Model } from "../types.js";
+import { Supervisor } from "./supervisor.js";
+/**
+ * Pull the client's presented key from the request. Accepts, in order, an
+ * `Authorization: Bearer …`, an `x-api-key` (OpenAI/Anthropic convention, and
+ * what the relay accepted), or the brain's own `x-otto-brain-token`.
+ */
+export declare function extractToken(req: http.IncomingMessage): string | null;
+export interface StartServiceOptions {
+    config: BrainConfig;
+    modelNeedle?: string;
+    env?: NodeJS.ProcessEnv;
+    onLog?: (line: string) => void;
+}
+export interface ServiceHandle {
+    server: http.Server;
+    supervisor: Supervisor;
+    host: string;
+    port: number;
+    model: Model;
+    /** Whether the listener terminates TLS (config.tls.mode !== "off"). */
+    secure: boolean;
+    /** The address to show a user: the MagicDNS/cert hostname when TLS is on, else the bind host. */
+    displayHost: string;
+    stop: () => Promise<void>;
+}
+export declare function startService({ config, modelNeedle, env, onLog, }: StartServiceOptions): Promise<ServiceHandle>;
+//# sourceMappingURL=serve.d.ts.map

package/dist/service/serve.js

@@ -0,0 +1,278 @@
+/**
+ * The headless brain service: the router + supervisor bound to a port, with the
+ * VRAM fit, on-demand model switching, remote auth, built-in TLS, and pid-file
+ * lifecycle. Used both by `otto brain serve` (foreground) and by a detached
+ * `otto brain start`. It stays provider-neutral about the runtime source — it
+ * takes whatever resolveRuntime picks (managed or LM Studio).
+ *
+ * TLS is served in-process (config.tls): HTTPS with a files / self-signed /
+ * tailscale certificate, hot-swapped on renewal. This is what lets the brain be
+ * exposed securely over a network with no relay in front of it.
+ */
+import http from "node:http";
+import https from "node:https";
+import { getCalibration, forModel, loadPersistedConfig, loadProfilesStore, saveBrainConfig, saveProfilesStore, } from "../config/index.js";
+import { resolveBrainPaths } from "../config/paths.js";
+import { query as queryGpu } from "../gpu.js";
+import { pickModel, scanModels } from "../models/index.js";
+import { CommandError } from "../output/types.js";
+import { resolveRuntime } from "../runtime/index.js";
+import * as vram from "../vram.js";
+import { resolveVersion } from "../version.js";
+import * as results from "../ops/results.js";
+import { createRouter, Telemetry } from "./router.js";
+import { Supervisor } from "./supervisor.js";
+import * as tailscale from "./tailscale.js";
+import { CertManager, resolveTlsOptions } from "./tls.js";
+import { removePidFile, writePidFile } from "./pid-lock.js";
+/** The effective config with secrets masked, for the `/__host/config` read. */
+function redactConfig(config) {
+    return {
+        ...config,
+        auth: { ...config.auth, token: config.auth.token ? "********" : null },
+        // The Hugging Face token is the owner's account credential — never echo it to
+        // a caller of /__host/config (read is allowed by default on a shared brain).
+        ...(config.hfToken ? { hfToken: "********" } : {}),
+    };
+}
+/** Benchmark rankings + per-config variance + best-per-config, for `/__host/evals`. */
+function collectEvals() {
+    try {
+        const all = results.loadAll();
+        return {
+            rankings: results.rankModels(all),
+            latest: results.latestPerConfig(all),
+            variance: results.variance(all),
+            runCount: all.length,
+        };
+    }
+    catch {
+        return { rankings: [], latest: [], variance: [], runCount: 0 };
+    }
+}
+function isLoopback(host) {
+    return host === "127.0.0.1" || host === "::1" || host === "localhost";
+}
+/**
+ * Pull the client's presented key from the request. Accepts, in order, an
+ * `Authorization: Bearer …`, an `x-api-key` (OpenAI/Anthropic convention, and
+ * what the relay accepted), or the brain's own `x-otto-brain-token`.
+ */
+export function extractToken(req) {
+    const auth = req.headers.authorization;
+    if (typeof auth === "string" && auth.startsWith("Bearer "))
+        return auth.slice(7).trim();
+    const apiKey = req.headers["x-api-key"];
+    if (typeof apiKey === "string" && apiKey)
+        return apiKey;
+    const header = req.headers["x-otto-brain-token"];
+    return typeof header === "string" ? header : null;
+}
+/** Gate the router with a bearer token when configured; /health stays open. */
+function withAuth(inner, config) {
+    const token = config.auth.mode === "token" ? config.auth.token : null;
+    if (!token)
+        return inner;
+    return (req, res) => {
+        if (req.url !== "/health" && extractToken(req) !== token) {
+            res.writeHead(401, { "content-type": "application/json" });
+            res.end(JSON.stringify({ error: "unauthorized" }));
+            return;
+        }
+        inner(req, res);
+    };
+}
+export async function startService({ config, modelNeedle, env = process.env, onLog = () => { }, }) {
+    const runtime = resolveRuntime(config, env);
+    if (!runtime) {
+        throw new CommandError({
+            code: "NO_RUNTIME",
+            message: "no llama.cpp runtime available",
+            details: "run `otto brain runtime install` to download one, or install LM Studio",
+        });
+    }
+    const paths = resolveBrainPaths(env);
+    const tlsOptions = await resolveTlsOptions(config, paths);
+    // `listen.host: "tailscale"` binds the tailnet interface only (invisible to the
+    // LAN and the internet), mirroring the relay's default. Any other value binds
+    // verbatim. The cert hostname is what a client actually connects to.
+    const port = config.listen.port;
+    const bindHost = config.listen.host === "tailscale"
+        ? await tailscale.ipv4(config.tls.tailscaleExe ?? undefined)
+        : config.listen.host;
+    const displayHost = tlsOptions?.hostname ?? bindHost;
+    // Auth is orthogonal to transport: TLS encrypts the pipe, a token authorizes the
+    // caller. A non-loopback bind still needs a token even over HTTPS.
+    if (!isLoopback(bindHost) &&
+        config.auth.mode !== "token" &&
+        !config.allowInsecureBind &&
+        env.OTTO_BRAIN_ALLOW_INSECURE !== "1") {
+        throw new CommandError({
+            code: "INSECURE_BIND",
+            message: `refusing to bind ${bindHost} without auth`,
+            details: "set auth.mode=token, or allowInsecureBind=true for an open trusted-network share " +
+                "(or OTTO_BRAIN_ALLOW_INSECURE=1 to override)",
+        });
+    }
+    const store = loadProfilesStore();
+    const catalog = scanModels(config, env);
+    const needle = modelNeedle ?? config.defaultModel ?? store.lastModelId ?? undefined;
+    const model = pickModel(catalog, needle);
+    let profile = forModel(store, model, config.defaults);
+    const gpu = await queryGpu();
+    if (gpu) {
+        const fit = vram.fitToBudget({
+            model,
+            profile,
+            calibration: getCalibration(store, model, profile),
+            totalVramBytes: gpu.totalBytes,
+        });
+        if (!fit.adjusted && !fit.budget.fits) {
+            throw new CommandError({
+                code: "DOES_NOT_FIT",
+                message: `refusing to start: ${fit.reason}`,
+                details: "use a smaller quant, or run `otto brain calibrate` for a measured budget",
+            });
+        }
+        if (fit.adjusted && fit.reason)
+            onLog(`note: ${fit.reason}`);
+        profile = fit.profile;
+    }
+    const telemetry = new Telemetry();
+    const supervisor = new Supervisor({ runtime });
+    supervisor.on("log", (line) => {
+        if (/error|failed|warn/i.test(line))
+            onLog(line);
+    });
+    // Serialize model switches: the router queues request-driven switches, but the
+    // config path (POST /__host/config) calls loadModel directly. Chaining here
+    // guarantees two switches (e.g. a config write racing a request-driven switch)
+    // can never overlap two supervisor.start() calls, whichever caller triggers them.
+    let modelSwitchChain = Promise.resolve();
+    const loadModelUnsafe = async (target) => {
+        const gpuInfo = await queryGpu();
+        let fitProfile = forModel(store, target, config.defaults);
+        if (gpuInfo) {
+            const fit = vram.fitToBudget({
+                model: target,
+                profile: fitProfile,
+                calibration: getCalibration(store, target, fitProfile),
+                totalVramBytes: gpuInfo.totalBytes,
+            });
+            if (!fit.adjusted && !fit.budget.fits)
+                throw new Error(fit.reason ?? "does not fit");
+            fitProfile = fit.profile;
+        }
+        await supervisor.start(target, fitProfile);
+        store.lastModelId = target.id;
+        saveProfilesStore(store);
+    };
+    const loadModel = (target) => {
+        const run = modelSwitchChain.then(() => loadModelUnsafe(target));
+        // Keep the chain alive even if this switch fails, so a later switch still runs.
+        modelSwitchChain = run.catch(() => undefined);
+        return run;
+    };
+    // Apply an editable config patch from POST /__host/config: mutate the live
+    // config (so the lock/default getters and future starts see it), persist it to
+    // config.json without baking in env overrides, and hot-switch the model when a
+    // new default is named. Network/TLS/auth are host-owned and not accepted here.
+    const applyConfigPatch = async (patch) => {
+        if (typeof patch !== "object" || patch === null || Array.isArray(patch)) {
+            throw new Error("config patch must be an object");
+        }
+        const p = patch;
+        let switchTo = null;
+        if ("defaultModel" in p) {
+            const next = p.defaultModel;
+            if (next !== null && typeof next !== "string") {
+                throw new Error("defaultModel must be a string or null");
+            }
+            if ((next ?? null) !== config.defaultModel)
+                switchTo = next ?? null;
+            config.defaultModel = next ?? null;
+        }
+        if ("lockModel" in p) {
+            if (typeof p.lockModel !== "boolean")
+                throw new Error("lockModel must be a boolean");
+            config.lockModel = p.lockModel;
+        }
+        const persisted = loadPersistedConfig(paths);
+        persisted.defaultModel = config.defaultModel;
+        persisted.lockModel = config.lockModel;
+        saveBrainConfig(persisted, paths);
+        if (switchTo) {
+            const target = catalog.find((m) => m.displayName === switchTo || m.id === switchTo);
+            if (target)
+                await loadModel(target);
+        }
+        return redactConfig(config);
+    };
+    const handler = withAuth(createRouter({
+        supervisor,
+        telemetry,
+        logger: { warn: (m) => onLog(`WARN ${m}`) },
+        getCatalog: () => catalog,
+        loadModel,
+        version: resolveVersion(),
+        getConfig: () => redactConfig(config),
+        getEvals: collectEvals,
+        getLockModel: () => config.lockModel,
+        getDefaultModel: () => config.defaultModel,
+        applyConfigPatch,
+        getAllowConfigWrite: () => config.allowRemoteConfig,
+    }), config);
+    // TLS terminates in-process when configured; otherwise plain HTTP. The cert
+    // manager issues/generates the first keypair before we listen, and hot-swaps
+    // the secure context on renewal without dropping connections.
+    let certManager = null;
+    let server;
+    if (tlsOptions) {
+        certManager = new CertManager({ ...tlsOptions, logger: { info: onLog, warn: onLog } });
+        const secure = await certManager.load();
+        const httpsServer = https.createServer({ key: secure.key, cert: secure.cert }, handler);
+        certManager.on("renewed", (pair) => {
+            httpsServer.setSecureContext({ key: pair.key, cert: pair.cert });
+            onLog("note: TLS certificate hot-swapped");
+        });
+        server = httpsServer;
+    }
+    else {
+        server = http.createServer(handler);
+    }
+    server.keepAliveTimeout = 75000;
+    server.requestTimeout = 0;
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(port, bindHost, resolve);
+    });
+    certManager?.start();
+    await supervisor.start(model, profile);
+    store.lastModelId = model.id;
+    saveProfilesStore(store);
+    writePidFile({
+        pid: process.pid,
+        host: bindHost,
+        port,
+        startedAt: new Date().toISOString(),
+        secure: Boolean(tlsOptions),
+        displayHost,
+    }, env);
+    const stop = async () => {
+        certManager?.stop();
+        await supervisor.stop();
+        await new Promise((resolve) => server.close(() => resolve()));
+        removePidFile(env);
+    };
+    return {
+        server,
+        supervisor,
+        host: bindHost,
+        port,
+        model,
+        secure: Boolean(tlsOptions),
+        displayHost,
+        stop,
+    };
+}
+//# sourceMappingURL=serve.js.map

package/dist/service/supervisor.d.ts

@@ -0,0 +1,67 @@
+import { type ChildProcess } from "node:child_process";
+import { EventEmitter } from "node:events";
+import type { Model, Runtime } from "../types.js";
+import type { Profile } from "../config/schema.js";
+/**
+ * Default loopback port for the private llama-server child. Deliberately clear
+ * of Otto's space: 8081 (the old default) is the Expo/Metro dev port, so a brain
+ * started from the dev checkout collided with the running app - the server bound
+ * a port the app also wanted, and benchmark requests hit Metro (or a dead
+ * socket) instead of the model. This range sits above Otto's daemon ports
+ * (6788/6868) and app port (8081/19000) and below the Windows ephemeral range.
+ * Calibrate and sweep run their own supervisors at +1/+2 so they never collide
+ * with a main service already holding the base port.
+ */
+export declare const DEFAULT_INTERNAL_PORT = 20800;
+export type SupervisorState = "stopped" | "starting" | "ready" | "failed" | "stopping";
+export interface SupervisorOptions {
+    runtime: Runtime;
+    internalPort?: number;
+    host?: string;
+    readyTimeoutMs?: number;
+}
+export interface SupervisorStatus {
+    state: SupervisorState;
+    model: string | null;
+    modelId: string | null;
+    pid: number | null;
+    loadSeconds: number | null;
+    vramBytes: number | null;
+    startedAt: string | null;
+    lastError: string | null;
+    upstream: string;
+    runtime: string;
+}
+/**
+ * Owns the llama-server child process.
+ *
+ * The server always listens on a private port; `router.js` fronts it on a
+ * stable one so switching models never asks a client to reconnect elsewhere.
+ */
+export declare class Supervisor extends EventEmitter {
+    #private;
+    runtime: Runtime;
+    internalPort: number;
+    host: string;
+    readyTimeoutMs: number;
+    state: SupervisorState;
+    child: ChildProcess | null;
+    model: Model | null;
+    profile: Profile | null;
+    logLines: string[];
+    lastError: string | null;
+    startedAt: Date | null;
+    loadSeconds: number | null;
+    vramAtReadyBytes: number | null;
+    vramBaselineBytes: number | null;
+    command: string | null;
+    constructor({ runtime, internalPort, host, readyTimeoutMs, }: SupervisorOptions);
+    get upstreamBase(): string;
+    /** Start (or restart) the server for a model + profile. */
+    start(model: Model, profile: Profile): Promise<this>;
+    /** Fetch /props from the running server (modalities, template caps, defaults). */
+    props(): Promise<unknown>;
+    stop(): Promise<void>;
+    status(): SupervisorStatus;
+}
+//# sourceMappingURL=supervisor.d.ts.map