@schlessera/brain-ui-server 0.17.0 → 0.19.0

package/src/activity/stream.ts

@@ -0,0 +1,366 @@
+/**
+ * Live activity streaming: view-scoped subscriptions, snapshot-then-delta.
+ *
+ * All emission is change-log driven — one code path for the server's own
+ * writes and a foreign writer's (the cron wrapper). A write on this process
+ * calls `pump()` directly (near-zero latency); foreign writes are picked up
+ * by a fast poll interval that runs ONLY while at least one subscription is
+ * live. The pump reads `changesSince(cursor)` and fans each committed change
+ * out as an `activity_delta` to matching subscribers, so ordering is the
+ * change log's and a delta can never precede its commit (persist-then-emit).
+ *
+ * The subscribe handler answers with a snapshot whose per-run high-water seq
+ * and pump baseline come from ONE store read transaction — there is no gap
+ * window between snapshot and first delta, and no duplication the client's
+ * seq-discard rule cannot handle.
+ *
+ * This is deliberately the only place broadcast is per-connection filtered;
+ * every other frame keeps the host's broadcast-to-all semantics.
+ */
+import type {
+  ActivitySpan,
+  ActivitySpanEvent,
+  ClientActivitySubscribe,
+  ClientActivityUnsubscribe,
+  ServerActivityDelta,
+  ServerActivitySnapshot,
+} from "@schlessera/brain-ui-sdk/protocol";
+import type { Logger } from "@opentelemetry/api-logs";
+
+import type { WSContext } from "../ws/clients.js";
+import { sendTo } from "../ws/clients.js";
+import type { ActivityChange, ActivityStore, SpanEventRow, SpanRow } from "./store.js";
+
+/** Events per snapshot frame — keeps each frame far below the WS size cap. */
+const SNAPSHOT_EVENT_CHUNK = 100;
+/** Spans per snapshot frame, same rationale. */
+const SNAPSHOT_SPAN_CHUNK = 100;
+/** Fast-poll cadence while subscriptions exist (foreign-writer liveness). */
+const POLL_INTERVAL_MS = 1500;
+
+const TOOL_NAME_PREFIX = "execute_tool ";
+
+interface Subscription {
+  view: "index" | "session" | "run";
+  sessionId?: string;
+  runId?: string;
+}
+
+/** Store row → wire span. Lifts the typed fields the protocol promotes
+ *  (toolName, subagent enrichment) out of the naming/attr conventions. */
+export function toWireSpan(span: SpanRow): ActivitySpan {
+  const isToolLike = span.kind === "tool" || span.kind === "subagent";
+  const toolName =
+    span.kind === "subagent" && span.name.startsWith(`${TOOL_NAME_PREFIX}Agent`)
+      ? "Agent"
+      : isToolLike && span.name.startsWith(TOOL_NAME_PREFIX)
+        ? span.name.slice(TOOL_NAME_PREFIX.length)
+        : isToolLike
+          ? span.name
+          : undefined;
+  const subagent = span.kind === "subagent" ? subagentOf(span.attrs) : undefined;
+  return {
+    spanId: span.spanId,
+    runId: span.runId,
+    parentSpanId: span.parentSpanId ?? undefined,
+    name: span.name,
+    ...(toolName !== undefined ? { toolName } : {}),
+    kind: span.kind,
+    origin: span.origin,
+    sessionId: span.sessionId ?? undefined,
+    jobName: span.jobName ?? undefined,
+    startedAt: span.startedAt,
+    waitUntil: span.waitUntil ?? undefined,
+    endedAt: span.endedAt ?? undefined,
+    outcome: span.outcome ?? undefined,
+    outcomeReason: span.outcomeReason ?? undefined,
+    usage: hasUsage(span) ? { ...span.usage } : undefined,
+    ...(subagent ? { subagent } : {}),
+    attrs: Object.keys(span.attrs).length > 0 ? span.attrs : undefined,
+  };
+}
+
+function subagentOf(attrs: Record<string, unknown>): ActivitySpan["subagent"] {
+  const type = attrs["subagent.type"];
+  const description = attrs["subagent.description"];
+  const summary = attrs["subagent.summary"];
+  const totalTokens = attrs["subagent.total_tokens"];
+  const subagent = {
+    ...(typeof type === "string" ? { type } : {}),
+    ...(typeof description === "string" ? { description } : {}),
+    ...(typeof summary === "string" ? { summary } : {}),
+    ...(typeof totalTokens === "number" ? { totalTokens } : {}),
+  };
+  return Object.keys(subagent).length > 0 ? subagent : undefined;
+}
+
+export function toWireEvent(event: SpanEventRow): ActivitySpanEvent {
+  return {
+    spanId: event.spanId,
+    eventIndex: event.eventIndex,
+    ts: event.ts,
+    eventType: event.eventType,
+    payload: event.payload,
+    ...(event.truncated ? { truncated: true } : {}),
+  };
+}
+
+export interface ActivityStream {
+  handleSubscribe(ws: WSContext, msg: ClientActivitySubscribe): void;
+  handleUnsubscribe(ws: WSContext, msg: ClientActivityUnsubscribe): void;
+  /** Forget a closed socket. */
+  dropConnection(ws: WSContext): void;
+  /** Drain new committed changes to subscribers. Reentrancy-safe. */
+  pump(): void;
+  /** Whether any connection currently watches the given session's activity. */
+  isWatched(scope: { sessionId?: string; runId?: string }): boolean;
+  /** Number of live subscriptions (all connections). */
+  subscriptionCount(): number;
+  close(): void;
+}
+
+export function createActivityStream(store: ActivityStore, log?: Logger): ActivityStream {
+  const subscriptions = new Map<WSContext, Subscription[]>();
+  let cursor = store.latestChangeCursor();
+  let pumping = false;
+  let pollHandle: ReturnType<typeof setInterval> | null = null;
+  /** runId → sessionId, learned from any span seen; a run's session never
+   *  changes, so entries are immutable. Replaces per-event store lookups
+   *  when matching event deltas against session-view subscriptions. */
+  const runSessions = new Map<string, string>();
+
+  function noteSpan(span: SpanRow): void {
+    if (span.sessionId && !runSessions.has(span.runId)) {
+      runSessions.set(span.runId, span.sessionId);
+    }
+  }
+
+  function subKey(s: Subscription): string {
+    return `${s.view}:${s.sessionId ?? ""}:${s.runId ?? ""}`;
+  }
+
+  function matches(sub: Subscription, runId: string, span?: SpanRow): boolean {
+    if (sub.view === "index") return true;
+    if (sub.view === "run") return sub.runId === runId;
+    // Session view: match by the span's session when we have it; event-only
+    // deltas resolve through the memoized run→session map.
+    if (span?.sessionId) return span.sessionId === sub.sessionId;
+    return runSessions.get(runId) === sub.sessionId;
+  }
+
+  function ensurePolling() {
+    if (pollHandle || subscriptions.size === 0) return;
+    pollHandle = setInterval(() => pump(), POLL_INTERVAL_MS);
+  }
+
+  function stopPollingIfIdle() {
+    if (pollHandle && subscriptions.size === 0) {
+      clearInterval(pollHandle);
+      pollHandle = null;
+    }
+  }
+
+  function deltaFrame(change: ActivityChange): ServerActivityDelta {
+    return change.kind === "span"
+      ? { type: "activity_delta", runId: change.runId, seq: change.seq, span: toWireSpan(change.span) }
+      : {
+          type: "activity_delta",
+          runId: change.runId,
+          seq: change.seq,
+          event: toWireEvent(change.event),
+        };
+  }
+
+  function pump(): void {
+    if (pumping) return;
+    pumping = true;
+    try {
+      // Nobody listening: skip the changes entirely, but keep the cursor at
+      // the log's head — otherwise the first subscriber's pump would replay
+      // history its snapshot already carries (harmless under the client's
+      // seq-discard rule, but pure waste). Side effects that must see every
+      // change (the notifier) keep their OWN cursor.
+      if (subscriptions.size === 0) {
+        cursor = store.latestChangeCursor();
+        return;
+      }
+      for (;;) {
+        const changes = store.changesSince(cursor, 200);
+        if (changes.length === 0) break;
+        for (const change of changes) {
+          cursor = change.changeId;
+          const span = change.kind === "span" ? change.span : undefined;
+          if (span) noteSpan(span);
+          const frame = deltaFrame(change);
+          for (const [ws, subs] of subscriptions) {
+            if (subs.some((s) => matches(s, change.runId, span))) {
+              sendTo(ws, frame);
+            }
+          }
+        }
+        if (changes.length < 200) break;
+      }
+    } catch (err) {
+      log?.emit({
+        severityText: "WARN",
+        body: "activity pump failed",
+        attributes: { error: err instanceof Error ? err.message : String(err) },
+      });
+    } finally {
+      pumping = false;
+    }
+  }
+
+  function sendSnapshot(ws: WSContext, sub: Subscription): void {
+    let spans: SpanRow[] = [];
+    let events: SpanEventRow[] = [];
+    const highWater: Record<string, number> = {};
+
+    if (sub.view === "run" && sub.runId) {
+      const snap = store.snapshotRun(sub.runId);
+      if (snap) {
+        spans = snap.spans;
+        events = snap.events;
+        highWater[snap.runId] = snap.highWaterSeq;
+        // No pump-cursor rewind: the pump cursor can only LAG the snapshot
+        // read (it never runs ahead of the DB), so unpumped changes are
+        // already inside this snapshot's high-water and the client's
+        // seq-discard rule absorbs them when the pump catches up.
+      }
+    } else if (sub.view === "session" && sub.sessionId) {
+      for (const root of store.openRootSpans()) {
+        if (root.sessionId !== sub.sessionId) continue;
+        const snap = store.snapshotRun(root.runId);
+        if (!snap) continue;
+        spans = spans.concat(snap.spans);
+        events = events.concat(snap.events);
+        highWater[snap.runId] = snap.highWaterSeq;
+      }
+    } else {
+      // Index view: open roots only — history is the REST activity API's job.
+      for (const root of store.openRootSpans()) {
+        spans.push(root);
+        highWater[root.runId] = store.runHighWaterSeq(root.runId);
+      }
+    }
+    for (const span of spans) noteSpan(span);
+
+    const base: Omit<ServerActivitySnapshot, "spans" | "events"> = {
+      type: "activity_snapshot",
+      view: sub.view,
+      ...(sub.sessionId ? { sessionId: sub.sessionId } : {}),
+      ...(sub.runId ? { runId: sub.runId } : {}),
+      highWaterSeq: highWater,
+    };
+
+    // Chunk: first span/event batch in the base frame, then append frames
+    // for the rest (the client's applySnapshot merges append frames).
+    const wireSpans = spans.map(toWireSpan);
+    sendTo(ws, {
+      ...base,
+      spans: wireSpans.slice(0, SNAPSHOT_SPAN_CHUNK),
+      events: events.slice(0, SNAPSHOT_EVENT_CHUNK).map(toWireEvent),
+    });
+    for (let i = SNAPSHOT_SPAN_CHUNK; i < wireSpans.length; i += SNAPSHOT_SPAN_CHUNK) {
+      sendTo(ws, {
+        ...base,
+        spans: wireSpans.slice(i, i + SNAPSHOT_SPAN_CHUNK),
+        events: [],
+        append: true,
+      });
+    }
+    for (let i = SNAPSHOT_EVENT_CHUNK; i < events.length; i += SNAPSHOT_EVENT_CHUNK) {
+      sendTo(ws, {
+        ...base,
+        spans: [],
+        events: events.slice(i, i + SNAPSHOT_EVENT_CHUNK).map(toWireEvent),
+        append: true,
+      });
+    }
+  }
+
+  return {
+    handleSubscribe(ws, msg) {
+      // A scoped view without its scope id can never match anything and
+      // would only leak an all-runs snapshot shape — ignore it up front.
+      if ((msg.view === "session" && !msg.sessionId) || (msg.view === "run" && !msg.runId)) {
+        return;
+      }
+      const sub: Subscription = {
+        view: msg.view,
+        ...(msg.sessionId ? { sessionId: msg.sessionId } : {}),
+        ...(msg.runId ? { runId: msg.runId } : {}),
+      };
+      const list = subscriptions.get(ws) ?? [];
+      if (!list.some((s) => subKey(s) === subKey(sub))) list.push(sub);
+      subscriptions.set(ws, list);
+      ensurePolling();
+      try {
+        sendSnapshot(ws, sub);
+      } catch (err) {
+        log?.emit({
+          severityText: "WARN",
+          body: "activity snapshot failed",
+          attributes: { error: err instanceof Error ? err.message : String(err) },
+        });
+      }
+    },
+
+    handleUnsubscribe(ws, msg) {
+      const list = subscriptions.get(ws);
+      if (!list) return;
+      const key = subKey({
+        view: msg.view,
+        ...(msg.sessionId ? { sessionId: msg.sessionId } : {}),
+        ...(msg.runId ? { runId: msg.runId } : {}),
+      });
+      const remaining = list.filter((s) => subKey(s) !== key);
+      if (remaining.length > 0) subscriptions.set(ws, remaining);
+      else subscriptions.delete(ws);
+      stopPollingIfIdle();
+    },
+
+    dropConnection(ws) {
+      subscriptions.delete(ws);
+      stopPollingIfIdle();
+    },
+
+    pump,
+
+    isWatched(scope) {
+      for (const subs of subscriptions.values()) {
+        for (const s of subs) {
+          if (s.view === "index") return true;
+          if (scope.runId && s.view === "run" && s.runId === scope.runId) return true;
+          if (scope.sessionId && s.view === "session" && s.sessionId === scope.sessionId)
+            return true;
+        }
+      }
+      return false;
+    },
+
+    subscriptionCount() {
+      let n = 0;
+      for (const subs of subscriptions.values()) n += subs.length;
+      return n;
+    },
+
+    close() {
+      if (pollHandle) clearInterval(pollHandle);
+      pollHandle = null;
+      subscriptions.clear();
+    },
+  };
+}
+
+function hasUsage(span: SpanRow): boolean {
+  const u = span.usage;
+  return (
+    u.inputTokens !== undefined ||
+    u.outputTokens !== undefined ||
+    u.cacheReadTokens !== undefined ||
+    u.cacheCreationTokens !== undefined ||
+    u.costUsd !== undefined ||
+    u.model !== undefined
+  );
+}

package/src/app.ts

@@ -8,6 +8,7 @@ import { resolveServerConfig, type ServerConfig } from "./config/env.js";
 import { createHealthRoutes, createStatusRoutes } from "./routes/health.js";
 import { createBrainRoutes } from "./routes/brain.js";
 import { createSessionRoutes } from "./routes/sessions.js";
+import { createActivityRoutes } from "./routes/activity.js";
 import { createVoiceRoutes } from "./routes/voice.js";
 import { createFilesRoutes } from "./routes/files.js";
 import { createShareRoutes, shareTargetFallbackRoutes } from "./routes/share.js";
@@ -33,6 +34,8 @@ import {
 } from "./middleware/passkeys.js";
 import { createUiDb } from "./db/client.js";
 import { getHiddenModelIds } from "./db/settings.js";
+import { createActivityRuntime } from "./activity/runtime.js";
+import { createPushRoutes } from "./routes/push.js";
 import {
   assertBackendResolvable,
   createBackendRegistry,
@@ -171,6 +174,10 @@ export function createApp(options: CreateAppOptions = {}): BrainUiApp {
   const db = createUiDb(config.dbPath, { log: dbLog });
   const brain = createBrainClient({ brainPath: config.brainPath });
   const cron = createCronScheduler({ db, brain, log: observability.logger("cron") });
+
+  // Activity record: span store + live stream + notifications + lifecycle
+  // sweeps, owned by the runtime (see activity/runtime.ts).
+  const activity = createActivityRuntime(db, { log: observability.logger("activity") });
   const registry =
     options.registry ??
     createBackendRegistry({
@@ -184,9 +191,17 @@ export function createApp(options: CreateAppOptions = {}): BrainUiApp {
     observability,
     catalog: createSessionCatalog(() => db, dbLog),
     ...(options.appName ? { appName: options.appName } : {}),
-    ...(options.turnTimeoutMs ? { turnTimeoutMs: options.turnTimeoutMs } : {}),
+    // Explicit option wins; then the env-resolved config; then the host default.
+    ...(options.turnTimeoutMs ?? config.turnTimeoutMs
+      ? { turnTimeoutMs: (options.turnTimeoutMs ?? config.turnTimeoutMs)! }
+      : {}),
     maxConcurrentSessions: () => config.maxConcurrentSessions,
     wsRate: config.wsRate,
+    activity: {
+      store: activity.store,
+      stream: activity.stream,
+      query: activity.query,
+    },
   });
   const wsUpgrade = createWsUpgrade(host);
   // One instrument for every way a login can fail — passkey ceremonies and
@@ -290,6 +305,10 @@ export function createApp(options: CreateAppOptions = {}): BrainUiApp {
     createBrainRoutes({ brain, brainPath: config.brainPath, keyterms })
   );
   app.route("/api", createSessionRoutes({ registry, db }));
+  // Behind the guard by mount position, like /api/status: the activity
+  // record leaks strictly more (session activity, errors, spend).
+  app.route("/api", createActivityRoutes({ db, store: activity.store, notifier: activity.notifier }));
+  app.route("/api", createPushRoutes({ sender: activity.pushSender }));
   app.route("/api", createVoiceRoutes({ voice: config.voice, keyterms }));
   app.route(
     "/api",
@@ -362,6 +381,7 @@ export function createApp(options: CreateAppOptions = {}): BrainUiApp {
     isTurnActive: () => host.coordinator.isTurnActive(),
     cancelActiveTurns: () => host.coordinator.cancelAll("Server shutting down"),
     close: () => {
+      activity.close();
       db.close();
     },
   };

package/src/config/env.ts

@@ -132,6 +132,13 @@ export const ENV_VARS: readonly EnvVarDescriptor[] = [
     default: "3",
     required: false,
   },
+  {
+    name: "BRAIN_UI_TURN_TIMEOUT_MS",
+    description:
+      "Hard per-turn timeout in ms; the host aborts a turn that runs past it. Raise for agent-heavy research work (e.g. 1800000 for 30 minutes).",
+    default: "600000 (10 minutes)",
+    required: false,
+  },
   // auth
   {
     name: "AUTH_MODE",
@@ -360,6 +367,11 @@ export interface ServerConfig {
   sourceCommit: string;
   allowedOrigins: string[];
   maxConcurrentSessions: number;
+  /**
+   * Per-turn timeout in ms (BRAIN_UI_TURN_TIMEOUT_MS), or null to use the
+   * WsHost default (10 minutes). An explicit createApp option still wins.
+   */
+  turnTimeoutMs: number | null;
   /** Threshold for the console log consumer (BRAIN_UI_LOG_LEVEL). */
   logLevel: Severity;
   /** Inbound WebSocket frame metering, per connection. */
@@ -448,6 +460,13 @@ export function resolveServerConfig(env: EnvRecord = process.env): ServerConfig
     sourceCommit: env.SOURCE_COMMIT ?? "dev",
     allowedOrigins: list(env.ALLOWED_ORIGINS),
     maxConcurrentSessions: Math.max(1, Number(env.MAX_CONCURRENT_SESSIONS) || 3),
+    // Positive integer or null — 0, negatives, and garbage all mean "unset",
+    // so a typo degrades to the safe default instead of an instant timeout.
+    turnTimeoutMs:
+      Number.isFinite(Number(env.BRAIN_UI_TURN_TIMEOUT_MS)) &&
+      Number(env.BRAIN_UI_TURN_TIMEOUT_MS) > 0
+        ? Math.floor(Number(env.BRAIN_UI_TURN_TIMEOUT_MS))
+        : null,
     auth: {
       mode: validMode,
       invalidMode: validMode ? null : rawAuthMode,

package/src/cron/scheduler.ts

@@ -115,15 +115,35 @@ export function createCronScheduler(deps: {
 
   return {
     getCronStatus() {
-      return jobs.map((job) => {
-        const lastRun = db
-          .query(
-            "SELECT * FROM cron_runs WHERE job_name = ? ORDER BY started_at DESC LIMIT 1"
-          )
-          .get(job.name) as any;
-
+      // Every job name that ever recorded a run, not just the two in-process
+      // triggers: module jobs and the maintain/digest crontab entries land in
+      // cron_runs through the same recorder and were previously invisible
+      // here (the "cron-status name gap"). The in-process names are unioned
+      // in so a fresh deployment still lists them before their first run.
+      // One window query yields each job's latest run.
+      const lastRuns = db
+        .query(
+          `SELECT job_name, started_at, status, duration_ms, error_message FROM (
+             SELECT *, ROW_NUMBER() OVER (
+               PARTITION BY job_name ORDER BY started_at DESC
+             ) AS rn FROM cron_runs
+           ) WHERE rn = 1 ORDER BY job_name`
+        )
+        .all() as Array<{
+        job_name: string;
+        started_at: number;
+        status: string;
+        duration_ms: number | null;
+        error_message: string | null;
+      }>;
+      const byName = new Map(lastRuns.map((r) => [r.job_name, r]));
+      const names = [
+        ...new Set([...jobs.map((j) => j.name), ...lastRuns.map((r) => r.job_name)]),
+      ];
+      return names.map((name) => {
+        const lastRun = byName.get(name);
         return {
-          name: job.name,
+          name,
           lastRunAt: lastRun?.started_at ?? null,
           lastStatus: lastRun?.status ?? null,
           lastDurationMs: lastRun?.duration_ms ?? null,

package/src/db/client.ts

@@ -26,6 +26,12 @@ export function createUiDb(dbPath: string, options: CreateUiDbOptions = {}): Dat
   const db = new Database(dbPath, { create: true });
   db.exec("PRAGMA journal_mode = WAL");
   db.exec("PRAGMA foreign_keys = ON");
+  // Two processes write this database (the server and the cron wrapper). The
+  // timeout makes a writer wait for a held lock instead of failing on it.
+  // It does NOT rescue a deferred transaction that loses the snapshot-upgrade
+  // race (that still throws SQLITE_BUSY) — which is why activity writes use
+  // immediate transactions; see src/activity/store.ts.
+  db.exec("PRAGMA busy_timeout = 5000");
   runMigrations(db, options.log);
   return db;
 }
@@ -67,11 +73,30 @@ function runMigrations(database: Database, log?: Logger) {
     log?.emit({ severityText: "INFO", body: "applying migration", attributes: { file } });
     const sql = readFileSync(join(migrationsDir, file), "utf-8");
 
-    database.transaction(() => {
-      database.exec(sql);
-      database
-        .prepare("INSERT INTO _migrations (filename, applied_at) VALUES (?, ?)")
-        .run(file, Date.now());
-    })();
+    try {
+      database.transaction(() => {
+        database.exec(sql);
+        database
+          .prepare("INSERT INTO _migrations (filename, applied_at) VALUES (?, ?)")
+          .run(file, Date.now());
+      })();
+    } catch (err) {
+      // Two processes racing a first boot: the loser's transaction fails on
+      // the UNIQUE _migrations.filename insert (or on DDL the winner already
+      // ran). If the file is now recorded as applied, treat it as such and
+      // move on; anything else is a real migration failure.
+      const nowApplied = database
+        .query("SELECT 1 FROM _migrations WHERE filename = ?")
+        .get(file);
+      const uniqueViolation =
+        err instanceof Error &&
+        err.message.includes("UNIQUE constraint failed: _migrations.filename");
+      if (!nowApplied && !uniqueViolation) throw err;
+      log?.emit({
+        severityText: "INFO",
+        body: "migration applied concurrently by another process; skipping",
+        attributes: { file },
+      });
+    }
   }
 }

package/src/index.ts

@@ -80,6 +80,28 @@ export { createBrainClient, type BrainClient } from "./brain/client.js";
 // `cronJobs` reflects what actually ran.
 export { recordCronRun, type CronRunRecord } from "./cron/scheduler.js";
 
+// The activity record. The cron wrapper is a second PROCESS writing spans
+// into the same store (root span + heartbeat + span-sink ingest); everything
+// else consumes it through the app.
+export {
+  createActivityStore,
+  SPAN_OUTCOMES,
+  type ActivityStore,
+  type SpanRow,
+  type SpanEventRow,
+  type SpanOutcome,
+  type SpanUsage,
+  type ActivityChange,
+} from "./activity/store.js";
+export { createActivityStream, type ActivityStream } from "./activity/stream.js";
+export { ingestSpanSink } from "./activity/span-sink.js";
+export {
+  generateActivityDigest,
+  latestActivityDigest,
+  DIGEST_JOB_NAME,
+  type ActivityDigest,
+} from "./activity/digest.js";
+
 // Share staging: a deployment can sweep expired staging dirs at boot; the
 // intake route also sweeps opportunistically on every share.
 export { pruneShareStaging, shareStagingRoot } from "./share/staging.js";