@schlessera/brain-ui-server 0.16.0 → 0.17.0

package/src/cron/scheduler.ts

@@ -5,7 +5,8 @@ import type { BrainClient } from "../brain/client.js";
 // Scheduling is owned by the container crontab (config/crontab) — sync at
 // 02:00, validate at 03:00, job scrape at 04:00. This module only provides
 // manual triggers (triggerJob) and run history for the /api/status endpoint.
-// Runs started by the crontab do not appear here; they log to syslog.
+// Runs started by the crontab report themselves via recordCronRun below;
+// their stdout/stderr still goes to syslog.
 
 export interface CronScheduler {
   getCronStatus(): Array<{
@@ -23,6 +24,46 @@ interface CronJob {
   handler: () => Promise<void>;
 }
 
+/** One in-flight job run being recorded into `cron_runs`. */
+export interface CronRunRecord {
+  /** Mark the run finished: success without an argument, error with one. */
+  finish(error?: string): void;
+}
+
+/**
+ * Record a job run in the `cron_runs` history that `/api/status` serves
+ * (`cronJobs`, via {@link CronScheduler.getCronStatus}).
+ *
+ * The scheduler in this module only writes the table for its own manual
+ * triggers; in the shipped deployment the jobs are executed by the container
+ * crontab, whose runs were invisible to `/api/status`. An external scheduler's
+ * wrapper calls this before the job and `finish()` after, writing exactly the
+ * rows the in-process runner writes. A run whose process died before
+ * `finish()` stays in status "running" — itself a signal.
+ */
+export function recordCronRun(db: Database, jobName: string): CronRunRecord {
+  const startedAt = Date.now();
+  db.prepare(
+    "INSERT INTO cron_runs (job_name, started_at, status) VALUES (?, ?, 'running')"
+  ).run(jobName, startedAt);
+
+  return {
+    finish(error?: string) {
+      const finishedAt = Date.now();
+      const durationMs = finishedAt - startedAt;
+      if (error === undefined) {
+        db.prepare(
+          "UPDATE cron_runs SET status = 'success', finished_at = ?, duration_ms = ? WHERE job_name = ? AND started_at = ?"
+        ).run(finishedAt, durationMs, jobName, startedAt);
+      } else {
+        db.prepare(
+          "UPDATE cron_runs SET status = 'error', finished_at = ?, duration_ms = ?, error_message = ? WHERE job_name = ? AND started_at = ?"
+        ).run(finishedAt, durationMs, error, jobName, startedAt);
+      }
+    },
+  };
+}
+
 export function createCronScheduler(deps: {
   db: Database;
   brain: BrainClient;
@@ -49,28 +90,21 @@ export function createCronScheduler(deps: {
 
   async function runJob(job: CronJob) {
     const startedAt = Date.now();
-
-    db.prepare(
-      "INSERT INTO cron_runs (job_name, started_at, status) VALUES (?, ?, 'running')"
-    ).run(job.name, startedAt);
+    // Same recorder the external-scheduler path uses, so the two can never
+    // drift in what a cron_runs row looks like.
+    const record = recordCronRun(db, job.name);
 
     try {
       await job.handler();
-      const durationMs = Date.now() - startedAt;
-      db.prepare(
-        "UPDATE cron_runs SET status = 'success', finished_at = ?, duration_ms = ? WHERE job_name = ? AND started_at = ?"
-      ).run(Date.now(), durationMs, job.name, startedAt);
+      record.finish();
       log?.emit({
         severityText: "INFO",
         body: "job completed",
-        attributes: { job: job.name, "duration.ms": durationMs },
+        attributes: { job: job.name, "duration.ms": Date.now() - startedAt },
       });
     } catch (err) {
-      const durationMs = Date.now() - startedAt;
       const message = err instanceof Error ? err.message : String(err);
-      db.prepare(
-        "UPDATE cron_runs SET status = 'error', finished_at = ?, duration_ms = ?, error_message = ? WHERE job_name = ? AND started_at = ?"
-      ).run(Date.now(), durationMs, message, job.name, startedAt);
+      record.finish(message);
       log?.emit({
         severityText: "ERROR",
         body: "job failed",

package/src/db/settings.ts

@@ -36,8 +36,8 @@ export function setSetting(db: Database, key: string, value: unknown): void {
 }
 
 /** Profile ids the user keeps out of the model picker. */
-export function getHiddenModelIds(db: Database): string[] {
-  const value = getSetting<unknown>(db, HIDDEN_MODELS_KEY, []);
+export function getHiddenModelIds(db: Database, log?: Logger): string[] {
+  const value = getSetting<unknown>(db, HIDDEN_MODELS_KEY, [], log);
   if (!Array.isArray(value)) return [];
   return value.filter((id): id is string => typeof id === "string");
 }

package/src/index.ts

@@ -34,6 +34,7 @@ export {
   assertBackendResolvable,
   createBackendRegistry,
   createStaticBackendRegistry,
+  type BackendLogFn,
   type BackendRegistry,
   type ModelDiscoverySource,
   type ModelDiscoveryState,
@@ -74,6 +75,11 @@ export {
 // their own routes on top.
 export { createBrainClient, type BrainClient } from "./brain/client.js";
 
+// Cron run history. Scheduling belongs to the deployment (container crontab);
+// an external scheduler's wrapper records each run here so /api/status's
+// `cronJobs` reflects what actually ran.
+export { recordCronRun, type CronRunRecord } from "./cron/scheduler.js";
+
 // Share staging: a deployment can sweep expired staging dirs at boot; the
 // intake route also sweeps opportunistically on every share.
 export { pruneShareStaging, shareStagingRoot } from "./share/staging.js";

package/src/middleware/auth.ts

@@ -1,3 +1,4 @@
+import type { Counter } from "@opentelemetry/api";
 import type { Logger } from "@opentelemetry/api-logs";
 import { Hono } from "hono";
 import type { Context, MiddlewareHandler } from "hono";
@@ -289,9 +290,14 @@ export function authRoutes(
      * of WebAuthn concerns.
      */
     passwordDisabled?: (c: Context) => boolean;
+    /** Where login outcomes are reported; absent means silence. */
+    log?: Logger;
+    /** Failed-login counter — the same `auth.failures` instrument passkeys use. */
+    failures?: Counter;
   } = {}
 ): Hono {
   const app = new Hono();
+  const { log, failures } = deps;
 
   app.post("/auth/login", async (c) => {
     if (mode !== "password") {
@@ -302,6 +308,12 @@ export function authRoutes(
     const perIpOk = consumeLoginToken(`ip:${key}`, LOGIN_RATE_LIMIT);
     const globalOk = consumeLoginToken("global", GLOBAL_LOGIN_RATE_LIMIT);
     if (!perIpOk || !globalOk) {
+      failures?.add(1, { reason: "rate_limited", method: "password" });
+      log?.emit({
+        severityText: "WARN",
+        body: "login rate limited",
+        attributes: { ip: key, limit: perIpOk ? "global" : "ip" },
+      });
       return c.json({ error: "Too many attempts. Try again in a minute." }, 429);
     }
 
@@ -322,15 +334,52 @@ export function authRoutes(
     }
     const password = typeof body.password === "string" ? body.password : "";
     if (!password || !hash || !secret) {
+      failures?.add(1, { reason: "invalid_password", method: "password" });
+      log?.emit({
+        severityText: "WARN",
+        body: "login failed",
+        attributes: { ip: key },
+      });
       return c.json({ error: "Invalid credentials" }, 401);
     }
 
-    const ok = await Bun.password.verify(password, hash).catch(() => false);
+    // A verify that THROWS is not a wrong password — it is a hash Bun cannot
+    // parse (a corrupt BRAIN_UI_PASSWORD_HASH locks the owner out of every
+    // login), and collapsing it into "invalid credentials" hid exactly that.
+    let ok = false;
+    let verifyError: unknown = null;
+    try {
+      ok = await Bun.password.verify(password, hash);
+    } catch (err) {
+      verifyError = err;
+    }
+    if (verifyError) {
+      failures?.add(1, { reason: "verify_error", method: "password" });
+      log?.emit({
+        severityText: "ERROR",
+        body: "password verification errored — check BRAIN_UI_PASSWORD_HASH",
+        attributes: {
+          error: verifyError instanceof Error ? verifyError.message : String(verifyError),
+        },
+      });
+      return c.json({ error: "Invalid credentials" }, 401);
+    }
     if (!ok) {
+      failures?.add(1, { reason: "invalid_password", method: "password" });
+      log?.emit({
+        severityText: "WARN",
+        body: "login failed",
+        attributes: { ip: key },
+      });
       return c.json({ error: "Invalid credentials" }, 401);
     }
 
     await issueSessionCookie(c, auth);
+    log?.emit({
+      severityText: "INFO",
+      body: "login succeeded",
+      attributes: { ip: key },
+    });
     return c.json({ ok: true });
   });
 

package/src/routes/health.ts

@@ -1,4 +1,5 @@
 import { Hono } from "hono";
+import type { Database } from "bun:sqlite";
 
 import type { MetricSnapshot } from "../observability/index.js";
 
@@ -7,13 +8,28 @@ const startTime = Date.now();
 // Public liveness probe. Deliberately minimal: no version/commit, no cron
 // detail, no session oracle — this route sits in front of the auth guard and
 // is reachable by anyone on the origin.
-export const healthRoutes = new Hono().get("/health", (c) => {
-  return c.json({
-    status: "healthy",
-    uptime: Date.now() - startTime,
-    timestamp: new Date().toISOString(),
+//
+// "Healthy" means the app's own SQLite handle answers a real read, not merely
+// that the process accepts connections: the Docker healthcheck gates on this
+// route, and a wedged database previously kept reporting healthy while every
+// stateful route failed. The probe reads sqlite_master rather than a bare
+// SELECT 1 — a constant expression touches no page of the database file, so it
+// cannot notice a locked or corrupted one. The unhealthy body carries no
+// detail — the route is public.
+export function createHealthRoutes(deps: { db: Database }): Hono {
+  return new Hono().get("/health", (c) => {
+    try {
+      deps.db.query("SELECT name FROM sqlite_master LIMIT 1").get();
+    } catch {
+      return c.json({ status: "unhealthy" }, 503);
+    }
+    return c.json({
+      status: "healthy",
+      uptime: Date.now() - startTime,
+      timestamp: new Date().toISOString(),
+    });
   });
-});
+}
 
 export interface StatusDeps {
   /** Git SHA baked at build time (SOURCE_COMMIT), "dev" when unset. */

package/src/routes/share.ts

@@ -118,7 +118,7 @@ export function createShareRoutes(deps: ShareRoutesDeps): Hono {
     lastPrune = now;
     // Deliberately not awaited: pruning is housekeeping, and the client is
     // waiting on the staging result, not on it.
-    void pruneShareStaging(brainRoot).catch((err) => {
+    void pruneShareStaging(brainRoot, Date.now(), log).catch((err) => {
       log?.emit({ severityText: "ERROR", body: "share staging prune failed", attributes: { error: err instanceof Error ? err.message : String(err) } });
     });
   }
@@ -167,12 +167,16 @@ export function createShareRoutes(deps: ShareRoutesDeps): Hono {
 
   inFlight += 1;
   try {
-    const result = await stageShare(brainRoot, {
-      title: firstString(form, "title"),
-      text: firstString(form, "text"),
-      url: firstString(form, "url"),
-      files,
-    });
+    const result = await stageShare(
+      brainRoot,
+      {
+        title: firstString(form, "title"),
+        text: firstString(form, "text"),
+        url: firstString(form, "url"),
+        files,
+      },
+      log
+    );
 
     maybePrune();
     return c.json(result, 201);

package/src/voice/keyterm-builder.ts

@@ -405,7 +405,7 @@ export function buildKeyterms(settings: KeytermSettings): KeytermsCache {
         keyterms,
         generatedAt: Date.now(),
         count: keyterms.length,
-        overrides: loadOverrides(brainPath),
+        overrides: loadOverrides(brainPath, log),
       };
     });
   } catch (err) {
@@ -429,7 +429,7 @@ export function buildKeyterms(settings: KeytermSettings): KeytermsCache {
         keyterms: [],
         generatedAt: Date.now(),
         count: 0,
-        overrides: loadOverrides(brainPath),
+        overrides: loadOverrides(brainPath, log),
         degraded: true,
       };
     }

package/src/ws/bridge.ts

@@ -38,6 +38,16 @@ export function makeBridge(
       }
       host.sendToClients(withTurnScope(msg, turn, turnId));
       if (msg.type === "result") {
+        // Only the live turn's own result may set its disposition — a late
+        // frame through a previous turn's bridge must not relabel this one.
+        if (turn.turnId === turnId) {
+          turn.lastResult =
+            msg.isError || msg.outcome === "error"
+              ? "error"
+              : msg.outcome === "cancelled"
+                ? "cancelled"
+                : "success";
+        }
         catalog.persistSession(msg, promptText, turn.providerId, backendId);
       }
     },
@@ -144,8 +154,10 @@ export function makeBridge(
 
 export function emitTurnError(host: WsHost, turn: RunningTurn, err: unknown): void {
   // startTurn resolves for runtime failures (it emits its own error frame); it
-  // only rejects for caller errors.
+  // only rejects for caller errors. Each rejection is reported server-side
+  // too — the frame alone leaves no trace once the browser tab is gone.
   if (err instanceof BackendBusyError) {
+    host.reportTurnFailed("SESSION_BUSY", turn);
     host.sendToClients(
       withTurnScope(
         { type: "error", code: "SESSION_BUSY", message: "That session already has a running turn." },
@@ -153,15 +165,15 @@ export function emitTurnError(host: WsHost, turn: RunningTurn, err: unknown): vo
       )
     );
   } else if (err instanceof BackendRequestError) {
+    host.reportTurnFailed("BACKEND_REQUEST_ERROR", turn, err.message);
     host.sendToClients(
       withTurnScope({ type: "error", code: "BACKEND_REQUEST_ERROR", message: err.message }, turn)
     );
   } else {
+    const message = err instanceof Error ? err.message : String(err);
+    host.reportTurnFailed("BACKEND_ERROR", turn, message);
     host.sendToClients(
-      withTurnScope(
-        { type: "error", code: "BACKEND_ERROR", message: err instanceof Error ? err.message : String(err) },
-        turn
-      )
+      withTurnScope({ type: "error", code: "BACKEND_ERROR", message }, turn)
     );
   }
 }

package/src/ws/clients.ts

@@ -1,8 +1,15 @@
 import type { ServerMessage } from "@schlessera/brain-ui-sdk/protocol";
 import { shrinkForReplication } from "./shrink.js";
 
-/** Minimal structural view of a live socket — all we need to write to it. */
-export type WSContext = { send: (data: string) => void };
+/**
+ * Minimal structural view of a live socket — all we need to write to it.
+ * `raw` is the underlying Bun ServerWebSocket when hono's Bun adapter built
+ * the context; its `send` RETURNS a status instead of throwing.
+ */
+export type WSContext = {
+  send: (data: string) => void;
+  raw?: { send?: (data: string) => number } | undefined;
+};
 
 /** Serialize + size-bound a frame, then send it to one specific socket. */
 export function sendTo(ws: WSContext, msg: ServerMessage): void {
@@ -38,16 +45,27 @@ export class ClientSet {
   /**
    * Broadcast a frame to every attached client. Serializes once. A failing
    * socket is skipped (its `onClose` will prune it) so one dead peer can't
-   * block delivery to the others.
+   * block delivery to the others; `onSendError` lets the owner count the skip.
    */
-  broadcast(msg: ServerMessage): void {
+  broadcast(msg: ServerMessage, onSendError?: (err: unknown) => void): void {
     if (this.clients.size === 0) return;
     const payload = JSON.stringify(shrinkForReplication(msg));
     for (const ws of this.clients) {
       try {
-        ws.send(payload);
-      } catch {
+        // Bun's ServerWebSocket reports a dropped write by RETURNING 0 (closed
+        // connection) rather than throwing, and hono's WSContext.send discards
+        // that status — so write through the raw socket where one exists. -1
+        // is backpressure: the frame is queued, not lost.
+        if (typeof ws.raw?.send === "function") {
+          if (ws.raw.send(payload) === 0) {
+            onSendError?.(new Error("send dropped: connection closed"));
+          }
+        } else {
+          ws.send(payload);
+        }
+      } catch (err) {
         // Drop; the socket's onClose handler removes it from the set.
+        onSendError?.(err);
       }
     }
   }

package/src/ws/connection.ts

@@ -141,7 +141,7 @@ export function createWsHandlers(host: WsHost) {
       // handleClientMessage is async — a rejection must not escape as an
       // unhandled rejection with no frame sent.
       void Promise.resolve()
-        .then(() => handleClientMessage(host, ws, parsed.message))
+        .then(() => handleClientMessage(host, ws, parsed.message, connection))
         .catch((err) => {
           // This used to swallow the cause entirely: the client got a generic
           // frame and the server kept no record of what threw.
@@ -161,8 +161,17 @@ export function createWsHandlers(host: WsHost) {
         });
     },
 
-    onClose(_evt: CloseEvent, ws: WSContext) {
+    onClose(evt: CloseEvent, ws: WSContext) {
       host.log.emit({ severityText: "INFO", body: "client disconnected" });
+      // No onError here on purpose: hono's Bun adapter never dispatches it
+      // (only open/message/close reach these handlers), so a transport failure
+      // is only visible as an abnormal close code. 1000/1001 are the two
+      // clean endings (normal closure, going away); anything else — before it
+      // was recorded — looked exactly like a clean disconnect.
+      const code = (evt as { code?: unknown }).code;
+      if (typeof code === "number" && code !== 1000 && code !== 1001) {
+        host.reportAbnormalClose(code);
+      }
       host.clients.remove(ws);
       // Turns keep running in the background. Only reject pending interactive
       // requests once the LAST client leaves — while another client remains it

package/src/ws/dispatch.ts

@@ -202,10 +202,12 @@ export async function handleClientMessage(
           ...(runningTurn ? { turnId: runningTurn.turnId } : {}),
         });
       } catch (err) {
+        const message = err instanceof Error ? err.message : "Failed to load session";
+        host.reportTurnFailed("SESSION_LOAD_ERROR", { sessionId: msg.sessionId }, message);
         host.sendMessage(ws, {
           type: "error",
           code: "SESSION_LOAD_ERROR",
-          message: err instanceof Error ? err.message : "Failed to load session",
+          message,
           sessionId: msg.sessionId,
         });
       }

package/src/ws/host.ts

@@ -67,6 +67,25 @@ export interface WsHostOptions {
   wsRate?: { ratePerSecond: number; burst: number };
 }
 
+/** Identity of one turn, as it appears on a log record. */
+export interface TurnLogContext {
+  sessionId?: string | null;
+  turnId?: string | null;
+  providerId?: string | null;
+}
+
+/**
+ * Log attributes for one turn. Null fields are omitted rather than stringified
+ * — a new session has no sessionId until `session_info` names it.
+ */
+export function turnLogAttributes(turn: TurnLogContext): Record<string, string> {
+  return {
+    ...(turn.sessionId ? { "session.id": turn.sessionId } : {}),
+    ...(turn.turnId ? { "turn.id": turn.turnId } : {}),
+    ...(turn.providerId ? { profile: turn.providerId } : {}),
+  };
+}
+
 /**
  * Everything one WebSocket coordinator instance owns: turn state, the session
  * catalog, the backend registry, the attached client sockets, branding copy,
@@ -88,6 +107,18 @@ export class WsHost {
   private readonly framesDropped: ReturnType<
     ReturnType<Observability["meter"]>["createCounter"]
   >;
+  private readonly turnsStarted: ReturnType<
+    ReturnType<Observability["meter"]>["createCounter"]
+  >;
+  private readonly turnsCompleted: ReturnType<
+    ReturnType<Observability["meter"]>["createCounter"]
+  >;
+  private readonly turnsFailed: ReturnType<
+    ReturnType<Observability["meter"]>["createCounter"]
+  >;
+  private readonly wsErrors: ReturnType<
+    ReturnType<Observability["meter"]>["createCounter"]
+  >;
 
   constructor(options: WsHostOptions) {
     this.registry = options.registry;
@@ -100,11 +131,22 @@ export class WsHost {
     this.wsRate =
       options.wsRate && options.wsRate.ratePerSecond > 0 ? options.wsRate : null;
     this.log = this.observability.logger("ws");
-    this.framesDropped = this.observability
-      .meter("ws")
-      .createCounter("ws.frames.dropped", {
-        description: "Inbound frames refused before reaching a handler",
-      });
+    const meter = this.observability.meter("ws");
+    this.framesDropped = meter.createCounter("ws.frames.dropped", {
+      description: "Inbound frames refused before reaching a handler",
+    });
+    this.turnsStarted = meter.createCounter("turns.started", {
+      description: "Turns handed to a backend",
+    });
+    this.turnsCompleted = meter.createCounter("turns.completed", {
+      description: "Turns whose backend call resolved without throwing",
+    });
+    this.turnsFailed = meter.createCounter("turns.failed", {
+      description: "Turn failures surfaced to the client, by error code",
+    });
+    this.wsErrors = meter.createCounter("ws.errors", {
+      description: "Transport errors reported by the socket layer",
+    });
     this.coordinator.log = this.log;
   }
 
@@ -130,9 +172,76 @@ export class WsHost {
     });
   }
 
+  /** A turn began executing: counted, and logged with its correlation ids. */
+  reportTurnStarted(turn: TurnLogContext): void {
+    this.turnsStarted.add(1);
+    this.log.emit({
+      severityText: "INFO",
+      body: "turn started",
+      attributes: turnLogAttributes(turn),
+    });
+  }
+
+  /** A turn's backend call resolved: counted, and logged with its duration. */
+  reportTurnCompleted(turn: TurnLogContext, durationMs: number): void {
+    this.turnsCompleted.add(1);
+    this.log.emit({
+      severityText: "INFO",
+      body: "turn completed",
+      attributes: { ...turnLogAttributes(turn), "duration.ms": durationMs },
+    });
+  }
+
+  /**
+   * Record a turn failure the client is being told about. Before this existed
+   * every such failure was an error FRAME only — visible on one phone screen,
+   * absent from the server's own record.
+   *
+   * `code` is the bounded error-frame code (it feeds a counter attribute);
+   * `error` is the thrown message and rides only on the log record — never a
+   * caller-supplied frame body, per the reportDroppedFrame model.
+   */
+  reportTurnFailed(code: string, turn: TurnLogContext, error?: string): void {
+    this.turnsFailed.add(1, { code });
+    this.log.emit({
+      // A busy session is the client racing itself; everything else is a
+      // failure the operator should see.
+      severityText: code === "SESSION_BUSY" ? "WARN" : "ERROR",
+      body: "turn failed",
+      attributes: {
+        code,
+        ...turnLogAttributes(turn),
+        ...(error ? { error } : {}),
+      },
+    });
+  }
+
+  /**
+   * Record a socket that closed abnormally. This is the transport-error signal
+   * available on Bun: hono's Bun adapter dispatches only open/message/close
+   * (never WSEvents.onError, and Bun's ServerWebSocket has no error callback),
+   * so a transport failure surfaces as a close with an abnormal code.
+   */
+  reportAbnormalClose(code: number): void {
+    this.wsErrors.add(1, { "close.code": code });
+    this.log.emit({
+      severityText: "WARN",
+      body: "websocket closed abnormally",
+      attributes: { "close.code": code },
+    });
+  }
+
   /** Fan a frame out to every attached client (size-bounded per frame). */
   sendToClients(msg: ServerMessage): void {
-    this.clients.broadcast(msg);
+    this.clients.broadcast(msg, () => {
+      // Outbound counterpart of reportDroppedFrame: the peer never saw this
+      // frame. Counted only — a dead socket would otherwise WARN per frame
+      // until its onClose prunes it.
+      this.framesDropped.add(1, {
+        reason: "broadcast_send_failed",
+        direction: "outbound",
+      });
+    });
   }
 
   /** Send a frame to one specific socket (size-bounded). */