@jmcombs/pi-steward 0.0.0

package/core/llama-connection.ts

@@ -0,0 +1,215 @@
+/**
+ * How Steward finds the `llama-server` it reads.
+ *
+ * A connection is just a base URL and an API key. Where they come from depends
+ * on the host: inside Pi they come from the provider auth the operator already
+ * configured (the same resolution Pi's own llama.cpp extension performs), and
+ * outside Pi — the dev server — they come from the environment. Both paths fall
+ * back to the loopback default `llama-server` binds, so the dashboard always
+ * has somewhere to point.
+ *
+ * Pi's `LlamaClient`/`normalizeLlamaServerUrl` live at a deep path outside the
+ * package's `exports` map, so importing them would bypass Node's encapsulation
+ * and break on hosts that ship a subset shim. The eight-line normalizer is
+ * reimplemented here instead, and we talk HTTP with `fetch` ourselves.
+ *
+ * This module is only ever loaded in Node (the server half), never shipped to
+ * the browser, so `process.env` here is fine.
+ */
+
+/** A resolved way to reach one `llama-server`. */
+export interface LlamaConnection {
+  /** Normalized origin, e.g. `http://127.0.0.1:8080` — no trailing slash, no `/v1`. */
+  baseUrl: string;
+  /** Bearer key, or `""` when the server is keyless. */
+  apiKey: string;
+}
+
+/**
+ * The slice of Pi's provider auth Steward needs. Declared structurally rather
+ * than imported so a host that ships a narrower shape (or none) still type-checks.
+ */
+interface ProviderAuthResult {
+  auth: { apiKey?: string; baseUrl?: string };
+  /** Provider-scoped config resolved from credentials, e.g. `LLAMA_BASE_URL`. */
+  env?: Record<string, string>;
+}
+
+/**
+ * The one host capability this module reaches for. Optional at every level so
+ * that on a host whose extension API omits `modelRegistry` (a subset shim), the
+ * feature check simply fails and we fall back to the environment — a missing
+ * property never throws, unlike a static named import of an absent symbol.
+ */
+export interface ConnectionContext {
+  modelRegistry?: {
+    getProviderAuth?(provider: string): Promise<ProviderAuthResult | undefined>;
+  };
+}
+
+/** The provider id `llama-server` registers under, matching Pi's own extension. */
+const LLAMA_PROVIDER = "llama.cpp";
+
+/** Where `llama-server` listens unless told otherwise; already in normal form. */
+const DEFAULT_BASE_URL = "http://127.0.0.1:8080";
+
+/**
+ * Normalizes a server URL to a bare origin: http/https only, no query, no
+ * fragment, no trailing slash, and no `/v1` suffix (that is the inference path,
+ * not the server root). Throws on a non-http(s) URL rather than guessing.
+ *
+ * A faithful reimplementation of Pi's `normalizeLlamaServerUrl` — see the module
+ * comment for why it is not imported.
+ */
+export function normalizeLlamaServerUrl(value: string): string {
+  const url = new URL(value.trim());
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error("Server URL must use http or https");
+  }
+  url.hash = "";
+  url.search = "";
+  url.pathname = url.pathname.replace(/\/+$/u, "").replace(/\/v1$/u, "") || "/";
+  return url.toString().replace(/\/$/u, "");
+}
+
+/**
+ * The `host:port` a base URL points at, for the CONFIG `listen` row and the
+ * degraded overlays. Falls back to the raw string if the URL cannot be parsed,
+ * so a caller never sees `undefined`.
+ */
+export function listenAddress(baseUrl: string): string {
+  try {
+    return new URL(baseUrl).host;
+  } catch {
+    return baseUrl;
+  }
+}
+
+/** Normalizes a configured value, or falls back to the default on empty/invalid. */
+function normalizeOrDefault(value: string): string {
+  if (value.trim() === "") return DEFAULT_BASE_URL;
+  try {
+    return normalizeLlamaServerUrl(value);
+  } catch {
+    // A malformed override should degrade the dashboard, not crash it.
+    return DEFAULT_BASE_URL;
+  }
+}
+
+/**
+ * Resolves where to reach `llama-server`.
+ *
+ * Precedence mirrors Pi's llama.cpp extension: when the host exposes provider
+ * auth, `LLAMA_BASE_URL` from the resolved provider env wins, then the
+ * credential's `baseUrl`, then the loopback default; the key is the credential's
+ * `apiKey`. Without that host capability (the dev server, or a subset shim) the
+ * same two variables are read from the process environment instead.
+ *
+ * `env` is injectable so tests need not mutate the real environment.
+ */
+/** The API key from Pi's provider auth, or the environment. Never throws. */
+async function providerApiKey(
+  ctx: ConnectionContext | undefined,
+  env: Record<string, string | undefined>,
+): Promise<string> {
+  const getProviderAuth = ctx?.modelRegistry?.getProviderAuth;
+  if (typeof getProviderAuth === "function") {
+    try {
+      const result = await getProviderAuth(LLAMA_PROVIDER);
+      if (result !== undefined) return result.auth.apiKey ?? "";
+    } catch {
+      // Best-effort, same as the main path.
+    }
+  }
+  return env.LLAMA_API_KEY ?? "";
+}
+
+export async function resolveLlamaConnection(
+  ctx?: ConnectionContext,
+  env: Record<string, string | undefined> = process.env,
+  recordedBaseUrl?: string | null,
+): Promise<LlamaConnection> {
+  // `steward.json`'s baseUrl wins over everything. It is the operator telling
+  // Steward which server to watch; Pi's provider auth describes which server Pi
+  // *chats with*, and the two are allowed to differ — a testbed on one port
+  // while the daily driver answers on another is a normal thing to want.
+  //
+  // This used to be ignored, so a machine that had recorded :8091 was polled on
+  // the provider's :8080: the dashboard read "llama.cpp not reachable" and every
+  // control appeared broken while the server was perfectly healthy. The API key
+  // still comes from the provider, which is the only place it lives.
+  if (typeof recordedBaseUrl === "string" && recordedBaseUrl.trim() !== "") {
+    const apiKey = await providerApiKey(ctx, env);
+    return { baseUrl: normalizeOrDefault(recordedBaseUrl), apiKey };
+  }
+
+  const getProviderAuth = ctx?.modelRegistry?.getProviderAuth;
+  if (typeof getProviderAuth === "function") {
+    try {
+      const result = await getProviderAuth(LLAMA_PROVIDER);
+      if (result !== undefined) {
+        // An empty `LLAMA_BASE_URL` means "unset" — the same way Pi's own
+        // resolver treats it. Plain `??` would let "" shadow a configured
+        // `auth.baseUrl` and drop us onto the loopback default instead.
+        const envUrl = result.env?.LLAMA_BASE_URL;
+        const configured =
+          typeof envUrl === "string" && envUrl !== "" ? envUrl : result.auth.baseUrl;
+        return { baseUrl: normalizeOrDefault(configured ?? ""), apiKey: result.auth.apiKey ?? "" };
+      }
+    } catch {
+      // Auth resolution is best-effort: fall through to the environment.
+    }
+  }
+
+  return {
+    baseUrl: normalizeOrDefault(env.LLAMA_BASE_URL ?? ""),
+    apiKey: env.LLAMA_API_KEY ?? "",
+  };
+}
+
+/**
+ * Where Pi's provider actually points, or `null` when that cannot be
+ * established — as opposed to {@link resolveLlamaConnection}, which falls back
+ * to the loopback default so the dashboard always has somewhere to poll.
+ *
+ * That fallback is right for polling and wrong for comparing. The widget used it
+ * to decide whether Pi and Steward disagreed, so a host that could not resolve
+ * provider auth at all produced a confident `pi points at :8080` — a number
+ * nobody had configured, reported as a misconfiguration, on a machine where both
+ * of Pi's config files said :8091. An unknown must not render as a plausible
+ * value.
+ */
+export async function providerBaseUrlOrNull(
+  ctx?: ConnectionContext,
+  env: Record<string, string | undefined> = process.env,
+): Promise<string | null> {
+  const getProviderAuth = ctx?.modelRegistry?.getProviderAuth;
+  if (typeof getProviderAuth === "function") {
+    try {
+      const result = await getProviderAuth(LLAMA_PROVIDER);
+      if (result !== undefined) {
+        const envUrl = result.env?.LLAMA_BASE_URL;
+        const configured =
+          typeof envUrl === "string" && envUrl !== "" ? envUrl : result.auth.baseUrl;
+        if (typeof configured === "string" && configured.trim() !== "") {
+          try {
+            return normalizeLlamaServerUrl(configured);
+          } catch {
+            return null;
+          }
+        }
+      }
+    } catch {
+      // Unresolvable is unknown, not a default.
+    }
+  }
+  const fromEnv = env.LLAMA_BASE_URL;
+  if (typeof fromEnv === "string" && fromEnv.trim() !== "") {
+    try {
+      return normalizeLlamaServerUrl(fromEnv);
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}

package/core/llama-models.ts

@@ -0,0 +1,261 @@
+/**
+ * Turns a `llama-server` `/models` body into {@link ModelInfo} rows.
+ *
+ * The body arrives as `unknown` off the wire, so every field is narrowed before
+ * it is read — a missing or wrong-typed field degrades to a null or a default,
+ * never to `undefined` or `NaN`. The response is `{ object, data: [...] }`, but
+ * a bare array is accepted too so the parser is easy to test against a single
+ * captured model.
+ *
+ * This parser cannot decide `active`: that depends on whether any of the model's
+ * slots is processing, which lives behind a different endpoint. A loaded model
+ * is reported as `resident` here; the source upgrades it to `active` after it
+ * has cross-referenced `/slots`. Keep this module free of Node and DOM APIs.
+ */
+
+import type { ModelInfo, ModelStatus } from "./types.js";
+
+/** True for a non-null object we can read string-keyed fields off. */
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+/** A non-empty string, or `null` when absent/blank/wrong-typed. */
+function readString(value: unknown): string | null {
+  return typeof value === "string" && value.trim() !== "" ? value : null;
+}
+
+/** A finite number, or `null` when absent/wrong-typed/not finite. */
+function readNumber(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+/** The elements of `value` that are strings, or `[]` when it is not an array. */
+function readStringArray(value: unknown): string[] {
+  return Array.isArray(value)
+    ? value.filter((item): item is string => typeof item === "string")
+    : [];
+}
+
+/** The model records in a `/models` body, whether it is wrapped or bare. */
+function readModelList(raw: unknown): unknown[] {
+  if (Array.isArray(raw)) return raw;
+  if (isRecord(raw) && Array.isArray(raw.data)) return raw.data;
+  return [];
+}
+
+/**
+ * The value following the first of `flags` in a launch-args array, or `null`.
+ * `["--port", "56568"]` → `argValue(args, ["--port"])` is `"56568"`.
+ */
+function argValue(args: readonly string[], flags: readonly string[]): string | null {
+  for (let i = 0; i < args.length - 1; i += 1) {
+    if (flags.includes(args[i] ?? "")) return args[i + 1] ?? null;
+  }
+  return null;
+}
+
+/** Whether any of `flags` appears in the launch args at all. */
+function hasFlag(args: readonly string[], flags: readonly string[]): boolean {
+  return args.some((arg) => flags.includes(arg));
+}
+
+/** Common quantisation tokens, longest-first so `Q4_K_M` beats `Q4`. */
+const QUANT_PATTERN =
+  /\b(IQ\d+_[A-Z0-9]+|Q\d+_[A-Z0-9]+(?:_[A-Z0-9]+)?|Q\d+_\d+|Q\d+|F16|F32|BF16)\b/;
+
+/** Best-effort quant label from a model id, e.g. `Qwen3-0.6B-Q4_0` → `Q4_0`. */
+function quantFromId(id: string): string {
+  const match = QUANT_PATTERN.exec(id);
+  return match?.[1] ?? "";
+}
+
+/**
+ * Best-effort quant from the `--model` path an unloaded preset launches with:
+ * the token before `.gguf` in the file's basename, e.g.
+ * `…/Qwen3-0.6B-Q4_0.gguf` → `Q4_0`. `""` when the arg is absent or unparseable.
+ * This is what lets an unloaded preset card show its quant even though llama.cpp
+ * ships no `meta` (and so no `ftype`) until the model is resident.
+ */
+function quantFromArgs(args: readonly string[]): string {
+  const path = argValue(args, ["--model", "-m"]);
+  if (path === null) return "";
+  const base = path.slice(path.lastIndexOf("/") + 1).replace(/\.gguf$/i, "");
+  return quantFromId(base);
+}
+
+/** The last path/tag segment of an id: `ggml-org/Qwen3-GGUF:Q4_0` → `Qwen3-GGUF:Q4_0`. */
+function lastSegment(id: string): string {
+  const slash = id.lastIndexOf("/");
+  return slash === -1 ? id : id.slice(slash + 1);
+}
+
+/** Drops a trailing `<sep><suffix>` (case-insensitive) from `value`, if present. */
+function stripSuffix(value: string, suffix: string): string {
+  if (suffix === "") return value;
+  for (const sep of ["-", ":", "_", "."]) {
+    const tail = `${sep}${suffix}`;
+    if (value.toLowerCase().endsWith(tail.toLowerCase())) {
+      return value.slice(0, value.length - tail.length);
+    }
+  }
+  return value;
+}
+
+/** Display name: last segment with the quant tag and a `GGUF` marker trimmed. */
+function shortName(id: string, quant: string): string {
+  let short = lastSegment(id);
+  short = stripSuffix(short, quant);
+  short = stripSuffix(short, "GGUF");
+  short = stripSuffix(short, "gguf");
+  return short === "" ? id : short;
+}
+
+/**
+ * Maps llama.cpp's `status.value` to a {@link ModelStatus}. `loaded` becomes
+ * `resident` (the source may upgrade it to `active`); `sleeping` is loaded and
+ * ready, so it folds into `resident` too; anything unrecognised is treated as
+ * not loaded rather than invented.
+ */
+function mapStatus(value: string | null): ModelStatus {
+  switch (value) {
+    case "loaded":
+    case "sleeping":
+      return "resident";
+    case "loading":
+      return "loading";
+    case "downloading":
+      return "downloading";
+    default:
+      return "unloaded";
+  }
+}
+
+/** Flash-attention from launch args: an explicit value, a bare flag, or `auto`. */
+function readFlashAttn(args: readonly string[]): "on" | "off" | "auto" {
+  const value = argValue(args, ["--flash-attn", "-fa"]);
+  if (value === "on" || value === "off" || value === "auto") return value;
+  if (hasFlag(args, ["--flash-attn", "-fa"])) return "on";
+  return "auto";
+}
+
+/** KV-cache types from launch args, defaulting each side to `f16`. */
+function readKvCache(args: readonly string[]): string {
+  const k = argValue(args, ["--cache-type-k", "-ctk"]) ?? "f16";
+  const v = argValue(args, ["--cache-type-v", "-ctv"]) ?? "f16";
+  return `${k}/${v}`;
+}
+
+/** One `/models` record → a {@link ModelInfo}, or `null` when it has no id. */
+function parseModel(raw: unknown): ModelInfo | null {
+  if (!isRecord(raw)) return null;
+  const id = readString(raw.id);
+  if (id === null) return null;
+
+  const status = isRecord(raw.status) ? raw.status : {};
+  const args = readStringArray(status.args);
+  const modelStatus = mapStatus(readString(status.value));
+
+  const architecture = isRecord(raw.architecture) ? raw.architecture : {};
+  const outputs = readStringArray(architecture.output_modalities);
+  // A model that cannot emit text is an embedder. When the field is absent we
+  // assume a normal text model rather than guessing an embedder.
+  const embedding = outputs.length > 0 && !outputs.includes("text");
+
+  // `meta` is only present for loaded models, so size and the native context
+  // window are known only then; the quant falls back to the launch args (an
+  // unloaded preset names its `.gguf`) and finally to the id.
+  const meta = isRecord(raw.meta) ? raw.meta : {};
+  const ftype = readString(meta.ftype);
+  const argQuant = quantFromArgs(args);
+  const quant = ftype ?? (argQuant !== "" ? argQuant : quantFromId(id));
+  const sizeBytes = readNumber(meta.size);
+  const nativeCtx = readNumber(meta.n_ctx_train);
+
+  // `--parallel` is a plain integer when pinned; the live slot count later
+  // overrides it for a loaded model, but it is all an unloaded preset has.
+  const parallelArg = argValue(args, ["--parallel", "-np"]);
+  const parallel = parallelArg === null ? null : readNumber(Number(parallelArg));
+
+  // Per-slot context. Loaded models report it directly; an unloaded preset only
+  // states the whole `--ctx-size`, which the router splits across `--parallel`,
+  // so we divide to land on the same per-slot figure a loaded model would show.
+  const metaCtx = readNumber(meta.n_ctx);
+  const ctxSizeArg = argValue(args, ["--ctx-size", "-c"]);
+  const ctxSize = ctxSizeArg === null ? null : readNumber(Number(ctxSizeArg));
+  const ctx =
+    metaCtx ??
+    (ctxSize !== null && ctxSize > 0 && parallel !== null && parallel > 0
+      ? Math.floor(ctxSize / parallel)
+      : null);
+
+  // Only a pinned `--n-gpu-layers` counts; a missing flag stays `null` rather
+  // than becoming `Number(null) === 0`, which would invent a reading.
+  const ngl = argValue(args, ["--n-gpu-layers", "-ngl"]);
+  const gpuLayers = ngl === null ? null : readNumber(Number(ngl));
+
+  return {
+    id,
+    short: shortName(id, quant),
+    embedding,
+    quant,
+    sizeGB: sizeBytes === null ? null : sizeBytes / 1e9,
+    ctx,
+    nativeCtx,
+    gpuLayers,
+    detail: embedding ? "embedding" : null,
+    parallel,
+    flashAttn: readFlashAttn(args),
+    kvCache: readKvCache(args),
+    status: modelStatus,
+    tokensPerSecond: null,
+  };
+}
+
+/**
+ * Which model is listening on which port, from the same `/models` body — the log
+ * console's attribution map.
+ *
+ * The router prefixes every child line with `[port]` and nothing else, so a port
+ * is all the console has to go on. The log *does* state the mapping, in the
+ * `spawning … name=X on port P` line, but that line is written once per load and
+ * is typically thousands of lines behind the live tail (14,010 in one measured
+ * corpus) — so a cold-started console would attribute nothing. HTTP has the
+ * answer on demand instead, and stays right across load/unload cycles because
+ * every poll rebuilds it.
+ *
+ * Only genuinely listening ports are reported: an unloaded preset carries
+ * `--port 0`, which is not a port and would otherwise map every model in the
+ * catalogue onto one imaginary child.
+ */
+export function parseModelPorts(raw: unknown): Map<number, string> {
+  const ports = new Map<number, string>();
+  for (const entry of readModelList(raw)) {
+    if (!isRecord(entry)) continue;
+    const id = readString(entry.id);
+    if (id === null) continue;
+    const status = isRecord(entry.status) ? entry.status : {};
+    const value = argValue(readStringArray(status.args), ["--port"]);
+    if (value === null) continue;
+    const port = Number.parseInt(value, 10);
+    if (!Number.isInteger(port) || port <= 0) continue;
+    // Last wins: ports are ephemeral and a model respawned on a new port must
+    // not be shadowed by a stale entry.
+    ports.set(port, id);
+  }
+  return ports;
+}
+
+/**
+ * All models in a `/models` body, in the order the server listed them. Garbage
+ * in (`null`, `{}`, wrong types) yields `[]` or drops the offending row; it
+ * never throws.
+ */
+export function parseModels(raw: unknown): ModelInfo[] {
+  const models: ModelInfo[] = [];
+  for (const entry of readModelList(raw)) {
+    const model = parseModel(entry);
+    if (model !== null) models.push(model);
+  }
+  return models;
+}

package/core/llama-slots.ts

@@ -0,0 +1,104 @@
+/**
+ * Turns a `llama-server` `/slots` body into {@link SlotInfo} rows, and a
+ * `/metrics` body into a generation rate.
+ *
+ * Both arrive as `unknown` (or raw Prometheus text) off the wire, so each field
+ * is narrowed before it is read. The model id is not in the slot body — we know
+ * it because we asked `/slots?model=<id>` — so it is passed in and stamped onto
+ * every row. Keep this module free of Node and DOM APIs — see `./types.ts`.
+ */
+
+import type { SlotInfo } from "./types.js";
+
+/** True for a non-null object we can read string-keyed fields off. */
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+/** A finite number, or `null` when absent/wrong-typed/not finite. */
+function readNumber(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+/**
+ * Tokens generated so far, from `next_token[0].n_decoded`, or `null` when the
+ * body carries no such reading. An absent figure is not a zero: a slot the
+ * server said nothing about has generated an unknown number of tokens, and
+ * printing `0 decoded` for it would state a measurement nobody made.
+ */
+function readDecoded(value: unknown): number | null {
+  if (!Array.isArray(value)) return null;
+  const first = value[0];
+  if (!isRecord(first)) return null;
+  return readNumber(first.n_decoded);
+}
+
+/**
+ * The slots for `modelId`. `raw` that is not an array (a 400 on a bare `/slots`,
+ * a stray object) yields `[]`; a slot missing its id falls back to its position.
+ *
+ * Every reading degrades to `null` rather than to a default, and a slot whose
+ * `is_processing` is missing entirely reads `unknown` rather than `idle` — an
+ * absent flag is a body we do not understand, not a server at rest.
+ */
+export function parseSlots(raw: unknown, modelId: string): SlotInfo[] {
+  if (!Array.isArray(raw)) return [];
+  const slots: SlotInfo[] = [];
+  raw.forEach((entry, index) => {
+    if (!isRecord(entry)) return;
+    slots.push({
+      id: readNumber(entry.id) ?? index,
+      modelId,
+      promptTokens: readNumber(entry.n_prompt_tokens),
+      ctxTotal: readNumber(entry.n_ctx),
+      decoded: readDecoded(entry.next_token),
+      state:
+        entry.is_processing === true
+          ? "processing"
+          : entry.is_processing === false
+            ? "idle"
+            : "unknown",
+    });
+  });
+  return slots;
+}
+
+/** The `/metrics` values Steward reads, aggregated across a model's instance. */
+export interface LlamaMetrics {
+  /** Generation rate (`predicted_tokens_seconds`), null when absent or `nan`. */
+  tps: number | null;
+  /** Requests being processed (`requests_processing`), 0 when absent. */
+  requestsProcessing: number;
+  /** Requests deferred for a free slot (`requests_deferred`), 0 when absent. */
+  requestsDeferred: number;
+}
+
+/**
+ * Reads one `llamacpp:<name>` gauge from a Prometheus scrape, or `null` when the
+ * line is absent or its value is not finite (llama.cpp prints `nan` before the
+ * first generation).
+ */
+function readGauge(prometheusText: string, name: string): number | null {
+  const match = new RegExp(`^llamacpp:${name}\\s+([0-9eE+.-]+)`, "m").exec(prometheusText);
+  if (match?.[1] === undefined) return null;
+  const value = Number(match[1]);
+  return Number.isFinite(value) ? value : null;
+}
+
+/** The generation rate, or `null` when absent or not yet a finite number. */
+export function parseTps(prometheusText: string): number | null {
+  return readGauge(prometheusText, "predicted_tokens_seconds");
+}
+
+/**
+ * The rate and request gauges from a `/metrics` scrape. The request gauges are
+ * counts, so an absent line means zero; the rate is `null` (unknown) when
+ * absent, since 0 t/s and "no reading" are different states.
+ */
+export function parseMetrics(prometheusText: string): LlamaMetrics {
+  return {
+    tps: readGauge(prometheusText, "predicted_tokens_seconds"),
+    requestsProcessing: readGauge(prometheusText, "requests_processing") ?? 0,
+    requestsDeferred: readGauge(prometheusText, "requests_deferred") ?? 0,
+  };
+}