@schlessera/brain-ui-server 0.18.0 → 0.19.0

package/src/activity/notify.ts

@@ -0,0 +1,285 @@
+/**
+ * Notification intents: the at-least-once layer between the activity record
+ * and every delivery channel.
+ *
+ * An intent is persisted when the triggering condition is DETECTED (a
+ * terminal failure span, a stuck run) and carries its own lifecycle:
+ * pending -> sent | send_failed | suppressed. The in-app inbox is the
+ * guaranteed tier — it lists unacknowledged intents regardless of delivery
+ * status — and web push (the accelerator) marks sent/send_failed on top.
+ * Undelivered intents survive restarts by construction; the boot/tick sweep
+ * simply finds them still pending.
+ *
+ * Detection runs on the server's always-on tick with its OWN change-log
+ * cursor (the live stream's cursor only advances for subscribers), so a
+ * foreign writer's failure — the cron wrapper recording an error while
+ * nobody watches — is noticed within one tick, not at the next app open.
+ *
+ * Storm safety: a run-scoped tag deduplicates (one active intent per tag),
+ * per-job debounce collapses repeats, and a global creation rate cap stops
+ * one bad API key from paging for every job at once.
+ */
+import type { Database } from "bun:sqlite";
+import type { Logger } from "@opentelemetry/api-logs";
+import { isFailureOutcome } from "@schlessera/brain-ui-sdk/protocol";
+
+import { getSetting } from "../db/settings.js";
+import type { ActivityStore, SpanRow } from "./store.js";
+
+export type IntentKind = "failure" | "completion" | "stuck";
+
+export interface NotificationIntent {
+  id: number;
+  runId: string;
+  spanId: string | null;
+  kind: IntentKind;
+  tag: string;
+  title: string;
+  body: string;
+  status: "pending" | "sent" | "send_failed" | "suppressed";
+  acknowledged: boolean;
+  createdAt: number;
+}
+
+export interface ActivityNotifierDeps {
+  db: Database;
+  store: ActivityStore;
+  /** Was anyone watching this run/session when the event was recorded? */
+  isWatched: (scope: { runId?: string; sessionId?: string }) => boolean;
+  log?: Logger;
+}
+
+export interface ActivityNotifier {
+  /** Detect new terminal failures + stuck runs; create intents. */
+  tick(now?: number): void;
+  /** Unacknowledged intents, newest first (the inbox). */
+  inbox(limit?: number): NotificationIntent[];
+  /** Intents awaiting delivery (the push sender's queue). */
+  pending(limit?: number): NotificationIntent[];
+  markDelivered(id: number, status: "sent" | "send_failed"): void;
+  acknowledge(id: number): boolean;
+  acknowledgeAll(): number;
+  /** Retention: delete acknowledged intents untouched for longer than this. */
+  pruneAcknowledged(olderThanMs: number): number;
+}
+
+/** Global cap on intents CREATED per hour — failure-storm circuit. */
+const MAX_INTENTS_PER_HOUR = 20;
+/** Watchdog default: a live root run older than this is flagged as stuck. */
+const DEFAULT_STUCK_THRESHOLD_MS = 45 * 60 * 1000;
+
+export function createActivityNotifier(deps: ActivityNotifierDeps): ActivityNotifier {
+  const { db, store, isWatched, log } = deps;
+  let cursor = latestCursor(db);
+  /** Runs already flagged as stuck this process lifetime (tag also guards). */
+  const stuckFlagged = new Set<string>();
+
+  function latestCursor(database: Database): number {
+    const row = database
+      .query("SELECT COALESCE(MAX(change_id), 0) AS hi FROM activity_changes")
+      .get() as { hi: number };
+    return row.hi;
+  }
+
+  function createIntent(input: {
+    runId: string;
+    spanId?: string;
+    kind: IntentKind;
+    tag: string;
+    title: string;
+    body: string;
+    suppressed?: boolean;
+  }): void {
+    const now = Date.now();
+    // Tag dedupe: one live (unacknowledged) intent per tag — repeats coalesce.
+    const existing = db
+      .query(
+        "SELECT id FROM notification_intents WHERE tag = ? AND acknowledged = 0 LIMIT 1"
+      )
+      .get(input.tag);
+    if (existing) return;
+    // Global creation rate cap.
+    const recent = db
+      .query("SELECT COUNT(*) AS n FROM notification_intents WHERE created_at > ?")
+      .get(now - 60 * 60 * 1000) as { n: number };
+    const status =
+      input.suppressed || recent.n >= MAX_INTENTS_PER_HOUR ? "suppressed" : "pending";
+    db.query(
+      `INSERT INTO notification_intents
+         (run_id, span_id, kind, tag, title, body, status, acknowledged, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?)`
+    ).run(
+      input.runId,
+      input.spanId ?? null,
+      input.kind,
+      input.tag,
+      input.title,
+      input.body,
+      status,
+      now,
+      now
+    );
+    log?.emit({
+      severityText: "INFO",
+      body: "notification intent created",
+      attributes: { kind: input.kind, tag: input.tag, status },
+    });
+  }
+
+  function intentForSpan(span: SpanRow): void {
+    if (span.parentSpanId) return;
+    if (isFailureOutcome(span.outcome)) {
+      const label = span.jobName ?? span.name;
+      // "Unwatched" is evaluated when the failure is NOTICED — a live
+      // subscription to the run or its session means the user saw it happen,
+      // so the intent lands acknowledged-free but suppressed for delivery.
+      const watched = isWatched({
+        runId: span.runId,
+        ...(span.sessionId ? { sessionId: span.sessionId } : {}),
+      });
+      // Payload minimization: name + outcome word only. Lock screens render
+      // this outside the app's auth; detail is behind the tap-through.
+      createIntent({
+        runId: span.runId,
+        spanId: span.spanId,
+        kind: "failure",
+        tag: `failure:${span.jobName ?? span.sessionId ?? span.runId}`,
+        title: `${label} ${span.outcome === "interrupted" ? "was interrupted" : "failed"}`,
+        body: "Open to see the run.",
+        suppressed: watched,
+      });
+      return;
+    }
+    if (span.outcome === "success" && span.jobName) {
+      // Completion notifications are opt-in per job.
+      const optIn = getSetting<string[]>(db, "activity.notify.completions", [], log);
+      if (optIn.includes(span.jobName)) {
+        createIntent({
+          runId: span.runId,
+          spanId: span.spanId,
+          kind: "completion",
+          tag: `completion:${span.jobName}:${span.runId}`,
+          title: `${span.jobName} completed`,
+          body: "Open to see the run.",
+        });
+      }
+    }
+  }
+
+  return {
+    tick(now) {
+      try {
+        // New committed changes since our own cursor: terminal root spans
+        // among them become intents (SQL-side filter + join in the store).
+        // The head is read BEFORE the detection query so advancing past
+        // non-terminal changes can never skip a terminal write committed
+        // in between; a change seen twice is absorbed by the tag dedupe.
+        const head = latestCursor(db);
+        const rows = store.terminalRootChangesSince(cursor, 500);
+        for (const { span } of rows) intentForSpan(span);
+        const lastSeen = rows.length > 0 ? rows[rows.length - 1]!.changeId : cursor;
+        cursor = rows.length === 500 ? lastSeen : Math.max(head, lastSeen);
+
+        // Watchdog: an over-threshold LIVE root run is stuck — a signal, not
+        // an outcome. Threshold from settings, per-job override supported.
+        const defaultThreshold = getSetting<number>(
+          db,
+          "activity.watchdog.thresholdMs",
+          DEFAULT_STUCK_THRESHOLD_MS,
+          log
+        );
+        const overrides = getSetting<Record<string, number>>(
+          db,
+          "activity.watchdog.perJobMs",
+          {},
+          log
+        );
+        for (const span of store.findStuck(defaultThreshold, now)) {
+          if (stuckFlagged.has(span.runId)) continue;
+          const threshold = span.jobName ? (overrides[span.jobName] ?? defaultThreshold) : defaultThreshold;
+          if ((now ?? Date.now()) - span.startedAt < threshold) continue;
+          stuckFlagged.add(span.runId);
+          createIntent({
+            runId: span.runId,
+            spanId: span.spanId,
+            kind: "stuck",
+            tag: `stuck:${span.runId}`,
+            title: `${span.jobName ?? span.name} is taking unusually long`,
+            body: "Still running. Open to check on it.",
+          });
+        }
+      } catch (err) {
+        log?.emit({
+          severityText: "WARN",
+          body: "notifier tick failed",
+          attributes: { error: err instanceof Error ? err.message : String(err) },
+        });
+      }
+    },
+
+    inbox(limit = 50) {
+      return (
+        db
+          .query(
+            "SELECT * FROM notification_intents WHERE acknowledged = 0 ORDER BY created_at DESC LIMIT ?"
+          )
+          .all(limit) as any[]
+      ).map(rowToIntent);
+    },
+
+    pending(limit = 20) {
+      // Acknowledged means dismissed in the inbox — never push it later.
+      return (
+        db
+          .query(
+            "SELECT * FROM notification_intents WHERE status = 'pending' AND acknowledged = 0 ORDER BY created_at LIMIT ?"
+          )
+          .all(limit) as any[]
+      ).map(rowToIntent);
+    },
+
+    markDelivered(id, status) {
+      db.query("UPDATE notification_intents SET status = ?, updated_at = ? WHERE id = ?").run(
+        status,
+        Date.now(),
+        id
+      );
+    },
+
+    acknowledge(id) {
+      const res = db
+        .query("UPDATE notification_intents SET acknowledged = 1, updated_at = ? WHERE id = ?")
+        .run(Date.now(), id);
+      return res.changes > 0;
+    },
+
+    acknowledgeAll() {
+      const res = db
+        .query("UPDATE notification_intents SET acknowledged = 1, updated_at = ? WHERE acknowledged = 0")
+        .run(Date.now());
+      return res.changes;
+    },
+
+    pruneAcknowledged(olderThanMs) {
+      const res = db
+        .query("DELETE FROM notification_intents WHERE acknowledged = 1 AND updated_at < ?")
+        .run(Date.now() - olderThanMs);
+      return res.changes;
+    },
+  };
+}
+
+function rowToIntent(r: any): NotificationIntent {
+  return {
+    id: r.id,
+    runId: r.run_id,
+    spanId: r.span_id,
+    kind: r.kind,
+    tag: r.tag,
+    title: r.title,
+    body: r.body,
+    status: r.status,
+    acknowledged: r.acknowledged === 1,
+    createdAt: r.created_at,
+  };
+}

package/src/activity/push-sender.ts

@@ -0,0 +1,215 @@
+/**
+ * The accelerator tier: web push delivery for pending notification intents.
+ *
+ * VAPID keys are generated at first boot into their own table (see migration
+ * 008 for why not the settings KV) and never leave the process. Payloads are
+ * MINIMIZED — title + outcome word + a deep link — because push renders on
+ * lock screens outside the app's auth; detail lives behind the authenticated
+ * tap-through.
+ *
+ * Delivery marks the intent sent/send_failed; the inbox (guaranteed tier)
+ * is untouched by any of it. A 404/410 response prunes the subscription —
+ * the push service says that device is gone. When every send in a pass
+ * fails, that is the all-devices signal (VAPID invalidated by a KV
+ * wipe/restore, typically) and it is logged loudly; the client surfaces a
+ * re-enable prompt when its subscription check disagrees with the server.
+ */
+import type { Database } from "bun:sqlite";
+import type { Logger } from "@opentelemetry/api-logs";
+import webpush from "web-push";
+
+import type { ActivityNotifier } from "./notify.js";
+
+export interface PushSubscriptionRow {
+  endpoint: string;
+  p256dh: string;
+  auth: string;
+  label: string | null;
+  createdAt: number;
+  lastUsedAt: number | null;
+}
+
+export interface PushSender {
+  publicKey(): string;
+  subscribe(sub: { endpoint: string; keys: { p256dh: string; auth: string } }, label?: string): void;
+  unsubscribe(endpoint: string): boolean;
+  subscriptions(): PushSubscriptionRow[];
+  /** Deliver pending intents to every subscription. Returns sends attempted. */
+  deliverPending(notifier: ActivityNotifier): Promise<number>;
+}
+
+export interface CreatePushSenderOptions {
+  /** Injection seam for tests — the real webpush.sendNotification otherwise. */
+  send?: (
+    subscription: { endpoint: string; keys: { p256dh: string; auth: string } },
+    payload: string,
+    options: {
+      vapidDetails: { subject: string; publicKey: string; privateKey: string };
+      timeout?: number;
+    }
+  ) => Promise<unknown>;
+  /** `mailto:` or https contact required by the VAPID spec. */
+  subject?: string;
+  log?: Logger;
+}
+
+/** Per-send cap: a hung push service rejects instead of wedging the
+ *  delivering guard (the tick's reentrancy flag) forever. */
+const SEND_TIMEOUT_MS = 10_000;
+
+export function createPushSender(
+  db: Database,
+  options: CreatePushSenderOptions = {}
+): PushSender {
+  const log = options.log;
+  const subject = options.subject ?? "mailto:admin@localhost";
+  const send =
+    options.send ??
+    ((sub, payload, opts) =>
+      webpush.sendNotification(
+        { endpoint: sub.endpoint, keys: sub.keys },
+        payload,
+        { vapidDetails: opts.vapidDetails, timeout: opts.timeout }
+      ));
+
+  // Generate-once VAPID keys. Concurrent first boots are safe: the INSERT is
+  // OR IGNORE and the read-back wins.
+  function vapid(): { publicKey: string; privateKey: string } {
+    const row = db
+      .query("SELECT public_key AS publicKey, private_key AS privateKey FROM vapid_keys WHERE id = 1")
+      .get() as { publicKey: string; privateKey: string } | null;
+    if (row) return row;
+    const generated = webpush.generateVAPIDKeys();
+    db.query(
+      "INSERT OR IGNORE INTO vapid_keys (id, public_key, private_key, created_at) VALUES (1, ?, ?, ?)"
+    ).run(generated.publicKey, generated.privateKey, Date.now());
+    return vapid();
+  }
+
+  function rows(): PushSubscriptionRow[] {
+    return (
+      db.query("SELECT * FROM push_subscriptions ORDER BY created_at").all() as any[]
+    ).map((r) => ({
+      endpoint: r.endpoint,
+      p256dh: r.p256dh,
+      auth: r.auth,
+      label: r.label,
+      createdAt: r.created_at,
+      lastUsedAt: r.last_used_at,
+    }));
+  }
+
+  return {
+    publicKey() {
+      return vapid().publicKey;
+    },
+
+    subscribe(sub, label) {
+      db.query(
+        `INSERT INTO push_subscriptions (endpoint, p256dh, auth, label, created_at)
+         VALUES (?, ?, ?, ?, ?)
+         ON CONFLICT(endpoint) DO UPDATE SET p256dh = excluded.p256dh, auth = excluded.auth`
+      ).run(sub.endpoint, sub.keys.p256dh, sub.keys.auth, label ?? null, Date.now());
+    },
+
+    unsubscribe(endpoint) {
+      return db.query("DELETE FROM push_subscriptions WHERE endpoint = ?").run(endpoint)
+        .changes > 0;
+    },
+
+    subscriptions: rows,
+
+    async deliverPending(notifier) {
+      const pending = notifier.pending();
+      if (pending.length === 0) return 0;
+      if (rows().length === 0) {
+        // No devices: the inbox already has it; nothing to deliver. Leave
+        // the intents pending so a device subscribing later still gets them
+        // if they are fresh, and the boot sweep can retry.
+        return 0;
+      }
+      const keys = vapid();
+      let attempts = 0;
+
+      for (const intent of pending) {
+        // Re-read per intent: an endpoint pruned as dead during the previous
+        // intent's pass must not be attempted again in this one.
+        const subs = rows();
+        if (subs.length === 0) break;
+        // Minimized payload; the tag coalesces repeats per run across devices.
+        const payload = JSON.stringify({
+          title: intent.title,
+          body: intent.body,
+          tag: `brain-activity:${intent.runId}`,
+          url: `/#/activity/${encodeURIComponent(intent.runId)}`,
+        });
+        attempts += subs.length;
+        const results = await Promise.allSettled(
+          subs.map((sub) =>
+            send(
+              { endpoint: sub.endpoint, keys: { p256dh: sub.p256dh, auth: sub.auth } },
+              payload,
+              {
+                vapidDetails: { subject, publicKey: keys.publicKey, privateKey: keys.privateKey },
+                timeout: SEND_TIMEOUT_MS,
+              }
+            )
+          )
+        );
+        // DB writes only after the whole settled pass — never concurrent
+        // with in-flight sends against the same handle.
+        let delivered = 0;
+        const touched: string[] = [];
+        const dead: string[] = [];
+        results.forEach((result, i) => {
+          const sub = subs[i]!;
+          if (result.status === "fulfilled") {
+            delivered++;
+            touched.push(sub.endpoint);
+            return;
+          }
+          const err = result.reason;
+          const status = (err as { statusCode?: number }).statusCode;
+          if (status === 404 || status === 410) {
+            // The push service says this device is gone.
+            dead.push(sub.endpoint);
+            log?.emit({
+              severityText: "INFO",
+              body: "pruned dead push subscription",
+              attributes: { status: status },
+            });
+          } else {
+            log?.emit({
+              severityText: "WARN",
+              body: "push send failed",
+              attributes: {
+                error: err instanceof Error ? err.message : String(err),
+                ...(status ? { status } : {}),
+              },
+            });
+          }
+        });
+        for (const endpoint of touched) {
+          db.query("UPDATE push_subscriptions SET last_used_at = ? WHERE endpoint = ?").run(
+            Date.now(),
+            endpoint
+          );
+        }
+        for (const endpoint of dead) {
+          db.query("DELETE FROM push_subscriptions WHERE endpoint = ?").run(endpoint);
+        }
+        notifier.markDelivered(intent.id, delivered > 0 ? "sent" : "send_failed");
+        if (delivered === 0) {
+          // Every device failed — the all-devices signal (invalid VAPID after
+          // a restore, typically). Loud, because it is silent on every phone.
+          log?.emit({
+            severityText: "ERROR",
+            body: "push delivery failed on every subscription",
+            attributes: { subscriptions: subs.length },
+          });
+        }
+      }
+      return attempts;
+    },
+  };
+}

package/src/activity/query.ts

@@ -0,0 +1,160 @@
+/**
+ * The agent-facing read over the activity record — the same data the UI
+ * reads, shaped for model consumption and served through the backend
+ * bridge's `queryActivity` seam (the MCP tool calls it).
+ *
+ * Read-only by construction: nothing here writes. Free-text fields that
+ * originated outside the primary agent's own context (failure reasons /
+ * stderr tails, subagent transcript excerpts) are DATA, not instructions —
+ * the tool layer labels them as such before they reach a model.
+ */
+import type { Database } from "bun:sqlite";
+import type { ActivityQuery, ActivityQueryResult } from "@schlessera/brain-ui-sdk/server";
+import { isFailureOutcome } from "@schlessera/brain-ui-sdk/protocol";
+
+import { latestActivityDigest } from "./digest.js";
+import type { ActivityNotifier } from "./notify.js";
+import { rowToRunRollup, type ActivityStore, type SpanRow } from "./store.js";
+
+export function runActivityQuery(
+  db: Database,
+  store: ActivityStore,
+  query: ActivityQuery,
+  notifier?: ActivityNotifier
+): ActivityQueryResult {
+  const limit = Math.min(Math.max(query.limit ?? 20, 1), 100);
+  const hoursBack = Math.min(Math.max(query.hoursBack ?? 24, 1), 24 * 90);
+  const since = Date.now() - hoursBack * 60 * 60 * 1000;
+
+  switch (query.scope) {
+    case "running": {
+      return {
+        running: store.openRootSpans().map(liveSummary),
+      };
+    }
+
+    case "recent": {
+      const rows = (
+        db
+          .query(
+            "SELECT * FROM activity_run_rollups WHERE started_at >= ? ORDER BY started_at DESC LIMIT ?"
+          )
+          .all(since, limit) as any[]
+      ).map(rowToRunRollup);
+      return {
+        windowHours: hoursBack,
+        running: store.openRootSpans().map(liveSummary),
+        finished: rows.map((r) => ({
+          runId: r.runId,
+          origin: r.origin,
+          name: r.name,
+          jobName: r.jobName,
+          sessionId: r.sessionId,
+          startedAt: iso(r.startedAt),
+          outcome: r.outcome,
+          durationMs: r.durationMs,
+          costUsd: r.costUsd,
+          failureReason: r.failureReason,
+        })),
+      };
+    }
+
+    case "run": {
+      if (!query.runId) return { error: "scope 'run' needs runId" };
+      const snapshot = store.snapshotRun(query.runId);
+      if (snapshot) {
+        return {
+          runId: query.runId,
+          spans: snapshot.spans.map((s) => ({
+            spanId: s.spanId,
+            parent: s.parentSpanId,
+            name: s.name,
+            kind: s.kind,
+            startedAt: iso(s.startedAt),
+            endedAt: s.endedAt ? iso(s.endedAt) : null,
+            outcome: s.outcome,
+            reason: s.outcomeReason,
+            costUsd: s.usage.costUsd,
+            tokens:
+              s.usage.inputTokens !== undefined || s.usage.outputTokens !== undefined
+                ? { in: s.usage.inputTokens, out: s.usage.outputTokens }
+                : undefined,
+          })),
+        };
+      }
+      const row = db
+        .query("SELECT * FROM activity_run_rollups WHERE run_id = ?")
+        .get(query.runId) as any;
+      if (!row) return { error: `unknown run ${query.runId}` };
+      const rollup = rowToRunRollup(row);
+      return {
+        runId: query.runId,
+        detailPruned: true,
+        rollup: {
+          name: rollup.name,
+          jobName: rollup.jobName,
+          outcome: rollup.outcome,
+          startedAt: iso(rollup.startedAt),
+          durationMs: rollup.durationMs,
+          costUsd: rollup.costUsd,
+          failureReason: rollup.failureReason,
+        },
+      };
+    }
+
+    case "rollups": {
+      const rows = (
+        db
+          .query("SELECT * FROM activity_run_rollups WHERE started_at >= ?")
+          .all(since) as any[]
+      ).map(rowToRunRollup);
+      const digest = latestActivityDigest(db);
+      return {
+        windowHours: hoursBack,
+        runs: rows.length,
+        failures: rows.filter((r) => isFailureOutcome(r.outcome)).length,
+        costUsd: round(rows.reduce((a, r) => a + (r.costUsd ?? 0), 0)),
+        inputTokens: rows.reduce((a, r) => a + (r.inputTokens ?? 0), 0),
+        outputTokens: rows.reduce((a, r) => a + (r.outputTokens ?? 0), 0),
+        ...(digest ? { lastDigestAt: iso(digest.generatedAt) } : {}),
+      };
+    }
+
+    case "inbox": {
+      // The unacknowledged notification inbox — the same list the UI shows.
+      if (!notifier) return { error: "inbox unavailable" };
+      return {
+        intents: notifier.inbox(limit).map((i) => ({
+          kind: i.kind,
+          title: i.title,
+          status: i.status,
+          createdAt: iso(i.createdAt),
+          runId: i.runId,
+        })),
+      };
+    }
+
+    default:
+      return { error: `unknown scope` };
+  }
+}
+
+function liveSummary(span: SpanRow) {
+  return {
+    runId: span.runId,
+    origin: span.origin,
+    name: span.name,
+    jobName: span.jobName,
+    sessionId: span.sessionId,
+    startedAt: iso(span.startedAt),
+    elapsedMs: Date.now() - span.startedAt,
+  };
+}
+
+function iso(ms: number): string {
+  return new Date(ms).toISOString();
+}
+
+function round(n: number): number {
+  return Math.round(n * 10000) / 10000;
+}