@huynle/hub 0.7.1 → 0.7.3

package/src/central/http.ts

@@ -16,6 +16,10 @@ import { authPage, cookieValue, deviceName, OIDC_COOKIE, OidcClient, OidcError }
 import { isNotificationKind, Push, type PushSender } from "./push"
 import { pushEndpointProblem, resolveHost as systemResolveHost, type ResolveHost } from "./push-target"
 import { Webhooks } from "./agent/webhooks"
+import { issueAppToken } from "./agent/app-tokens"
+import { AGENT_EVENTS_AGE_MS, AGENT_EVENTS_MAX, AgentEvents, archiveEvent } from "./agent/events"
+import { EventWriters } from "./agent/event-writers"
+import { MemoryEventLog } from "./backend/memory-events"
 import { WEBHOOK_SCOPE } from "./agent/webhook-routes"
 import { hubBackends, type Backends } from "./backend"
 import { createResolver, type AgentActor, type BearerVerifier, type CredentialMode } from "./agent/actor"
@@ -33,11 +37,12 @@ import type { Launch } from "./agent/launch-store"
 import { decisionStateQuery, MessageFeed } from "./agent/message-routes"
 import { hubDirectory } from "./agent/directory-routes"
 import { agentApi, agentRoutes, agentVerifiers, isAgentPath, wellKnown, type AgentRoute } from "./agent/routes"
+import { MCP_PATH, mcpHttp } from "./mcp-http"
 import { SessionTokens } from "./agent/session-tokens"
 import { SessionTokenPush } from "./session-token-push"
 import { TokenKeys } from "./token-keys"
 import { EventLog } from "./events-log"
-import { BusyError, Registry, type RegistryMessage } from "./registry"
+import { BusyError, Registry } from "./registry"
 import { createActions, type Unsupported, unsupportedMessage } from "./actions"
 import { audioFormat, SpeechError } from "./speech/provider"
 import { SpeechService } from "./speech/service"
@@ -52,9 +57,8 @@ import { randomBytes, randomUUID } from "node:crypto"
 import { LaunchEngine } from "./agent/launch-engine"
 import { LaunchTokens } from "./agent/launch-tokens"
 import { LaunchFeed, launchInfo } from "./agent/launch-routes"
-import type { WatchRegistry } from "./agent/watch"
 import { DRAIN_RETRY_AFTER_MS, DRAIN_TIMEOUT_MS, RequestTimeoutError, RequestTooLargeError, SpokeError, SpokeHub, type DrainOptions, type Notifier, type SpokeHubOptions, type SpokeSocketData } from "./spoke-ws"
-import { ACTIVITY_INTERVAL_MAX, ACTIVITY_INTERVAL_MIN, LOCAL_USER, PUSH_PREF_KEYS, quietHoursInput, Store, VIEW_PREF_KEYS, type DeviceRecord, type LegacyPushPrefs, type PushPrefs, type UserRecord, type ViewPrefs } from "./store"
+import { ACTIVITY_INTERVAL_MAX, ACTIVITY_INTERVAL_MIN, LOCAL_USER, PUSH_PREF_KEYS, quietHoursInput, Store, VIEW_PREF_KEYS, VIEW_PREF_MAX_ARCHIVED, type DeviceRecord, type LegacyPushPrefs, type PushPrefs, type UserRecord, type ViewPrefs } from "./store"
 
 export interface Speech {
   transcribe(audio: Uint8Array, format: string): Promise<string>
@@ -137,6 +141,10 @@ export interface Central {
   readonly launchTokens: LaunchTokens
   /** Webhooks (P4-T10): registrations and the outbox sender; launch changes are published to it. */
   readonly webhooks: Webhooks
+  /** The canonical agent event journal (GET /api/agent/events, MCP events): compact events per user, 24 h / 10,000. */
+  readonly agentEvents: AgentEvents
+  /** The journal's writers (state, blockers, presence, messages, launches); settled() waits for queued appends. */
+  readonly agentEventWriters: EventWriters
   /** The JSON API's route table (method, path, access), for audits and tests. */
   readonly routes: readonly RouteInfo[]
   /**
@@ -232,8 +240,6 @@ const MAX_TITLE_CHARS = 200
 const MAX_HERDR_WORKSPACE_CHARS = 100
 /** View preferences: entries per list and characters per entry. */
 const VIEW_PREF_MAX_ENTRIES = 500
-/** hiddenSessions holds archived sessions (Archive in the console), which add up over time. */
-const VIEW_PREF_MAX_ARCHIVED = 5000
 const VIEW_PREF_MAX_CHARS = 4096
 const DECISIONS = new Set<PermissionDecision>(["once", "always", "reject"])
 /** Longest utterance accepted for conversation mode (characters, after trimming). */
@@ -535,6 +541,7 @@ export function startCentral(options: CentralOptions): Central {
       ...(alert.requestID ? { requestID: alert.requestID } : {}),
       ...(alert.subject ? { subject: alert.subject } : {}),
       ...(alert.key ? { key: alert.key } : {}),
+      ...(alert.unlessPref ? { unlessPref: alert.unlessPref } : {}),
     }
     notifier.notify(alert.kind, alert.sessionID, alert.title, alert.text, alert.owner, extra).catch((cause) => {
       console.error("[hub] push notification failed:", cause instanceof Error ? cause.message : cause)
@@ -822,20 +829,33 @@ export function startCentral(options: CentralOptions): Central {
     resolve: options.resolveHost ?? systemResolveHost,
     ...(config.webhooksAllowPrivate ? { allowPrivate: true } : {}),
     appAlive: (userID, tokenID) => store.agentData(userID).appTokens?.some((t) => t.id === tokenID && Array.isArray(t.scopes) && t.scopes.includes(WEBHOOK_SCOPE)) === true,
+    appScopes: (userID, tokenID) => store.agentData(userID).appTokens?.find((t) => t.id === tokenID)?.scopes,
     userAgent: `hub/${options.version ?? "0.0.0"}`,
   })
-  launchFeed.subscribe((userID, launch) => {
-    void webhooks
-      .onLaunch(userID, launch, launchInfo(launch))
-      // Queued here, sent by the webhooks leader (kicked when that is another pod).
-      .then((queued) => (queued && !webhookLeader.isLeader() ? webhookLeader.kick() : undefined))
-      .catch((cause) => console.error(`[hub] webhook queue failed: ${(cause as Error).message}`))
-  }, { origin: true })
   const webhookLeader = leader("webhooks", {
     onElected: () => webhooks.start(),
     onDeposed: () => webhooks.stop(),
     onKick: () => void webhooks.drain().catch((cause) => console.error(`[hub] webhook delivery failed: ${(cause as Error).message}`)),
   })
+  // The canonical agent event journal (lifecycle design, Events): each change appended once across the cluster;
+  // presence by the agent-events leader, everything else where it happens (event-writers.ts).
+  const agentEvents = new AgentEvents({
+    log: backends.agentEvents ?? new MemoryEventLog({ now, max: AGENT_EVENTS_MAX, maxAgeMs: AGENT_EVENTS_AGE_MS }),
+    kv: backends.kv,
+    now,
+    onAppend: (userID, record) => webhooks.publishCanonical(userID, record.event, record.family, record.audience).then((queued) => (queued && !webhookLeader.isLeader() ? webhookLeader.kick() : undefined)),
+  })
+  const agentEventWriters = new EventWriters({
+    events: agentEvents,
+    kv: backends.kv,
+    registry,
+    machineOwner: (machineID) => store.machines().find((m) => m.id === machineID)?.ownerID,
+    users: allUsers,
+    messages: feed,
+    launches: launchFeed,
+  })
+  agentEventWriters.start()
+  leader("agent-events", { onElected: () => agentEventWriters.elected(), onDeposed: () => agentEventWriters.deposed() })
   for (const loop of leaders) void loop.start()
 
   // One credential resolver for every /api request: device cookie, device token, session and app tokens.
@@ -889,26 +909,7 @@ export function startCentral(options: CentralOptions): Central {
     void escalations.closed(tenant, requestID).catch((cause) => console.error(`[hub] escalation close failed: ${(cause as Error).message}`))
   })
   const escalate: Escalate = (escalation) => escalations.escalate(escalation)
-  /** Cluster mode: what GET /api/agent/watch reads; status of sessions on any pod comes from the user's live events. */
-  const watchRegistry = live
-    ? (userID: string): WatchRegistry => ({
-        catalog: () => registry.catalog(),
-        sessionTenant: (sessionID) => registry.sessionTenant(sessionID),
-        subscribe: (listener) => {
-          const offCatalog = registry.subscribe((message) => {
-            if (message.type === "catalog") listener(message)
-          })
-          const subscription = live.subscribe(userID, (event) => {
-            if (event.message.type === "status") listener(event.message as unknown as RegistryMessage, userID)
-          })
-          return () => {
-            offCatalog()
-            subscription.close()
-          }
-        },
-      })
-    : undefined
-  const agentDeps = { draining: drainSignal.signal, store, sessionTokens, directory, registry, spokes, delivery, decisions, feed, heartbeatMs, now, audit, grants, grantRequests, acting, actions, escalate, launches, launchFeed, webhooks, maxPromptChars: MAX_PROMPT_CHARS, ...(watchRegistry ? { watchRegistry } : {}) }
+  const agentDeps = { draining: drainSignal.signal, store, sessionTokens, directory, registry, spokes, delivery, decisions, feed, heartbeatMs, now, audit, grants, grantRequests, acting, actions, escalate, launches, webhooks, agentEvents, eventCursorKeys: tokenKeys, maxPromptChars: MAX_PROMPT_CHARS }
   const resolveActor = createResolver({
     cookieName: COOKIE,
     findDevice: (token) => deviceOfToken(token),
@@ -927,6 +928,16 @@ export function startCentral(options: CentralOptions): Central {
     routes: [...agentRoutes(agentDeps), ...(options.agent?.routes ?? [])],
   })
 
+  // POST /mcp (MCP Streamable HTTP): `hub mcp`'s tools as the bearer's caller; their /api/agent calls are dispatched
+  // to this handler in process.
+  const mcp = mcpHttp({
+    publicURL: () => publicURL,
+    isOwnOrigin: (origin) => origins.has(origin),
+    verify: async (token) => (await resolveActor(new Request(`${url}${MCP_PATH}`, { headers: { authorization: `Bearer ${token}` } }))).ok,
+    dispatch: async (request) => (await handle(request, server)) ?? error(500, "No response"),
+    version: options.version ?? "0.0.0",
+  })
+
   async function handle(request: Request, server: Server<SpokeSocketData>): Promise<Response | undefined> {
     const url = new URL(request.url)
     const { pathname } = url
@@ -963,12 +974,20 @@ export function startCentral(options: CentralOptions): Central {
         if (poll.status === "approved") broadcast({ type: "catalog" })
         return Response.json(poll)
       }
+      // `hub login` without a token (CLI login device code): no credential; the device code is the poller's secret.
+      if (method === "POST" && pathname === "/cli/login") {
+        const input = await body(request)
+        return Response.json(await auth.startCliLogin(input.clientName, input.scopes), { headers: NO_STORE })
+      }
+      if (method === "POST" && pathname === "/cli/login/poll") return await cliLoginPoll((await body(request)).deviceCode)
+      if (method === "GET" && pathname === "/spoke/presence") return await spokePresence(request)
       if (method === "GET" && pathname === "/.well-known/hub") {
         return wellKnown({ publicURL, version: options.version ?? "0.0.0", auth: config.auth.mode === "oidc" ? { mode: "oidc", issuer: config.auth.issuer } : { mode: "pairing" } })
       }
       if (method === "GET" && pathname === "/login") return login(request, url)
       if (oidc && method === "GET" && pathname === "/auth/login") return await authLogin(url)
       if (oidc && method === "GET" && pathname === "/auth/callback") return await authCallback(request, url)
+      if (pathname === MCP_PATH) return await mcp(request)
       if (isAgentPath(pathname)) return await agent.handle(request, url)
       if (pathname.startsWith("/api/")) return await api(request, server, url)
       if (method === "GET") return await staticFile(pathname, request.headers.get("accept-encoding"))
@@ -978,6 +997,31 @@ export function startCentral(options: CentralOptions): Central {
     }
   }
 
+  /**
+   * GET /spoke/presence: which harness spokes of the caller's own machine are connected (machine token only). The
+   * one read-only query a machine credential can make about itself, for `hubctl setup verify` (spoke-connected
+   * readiness): harness, evidence level and features only, never sessions, directories, pids or other machines.
+   */
+  async function spokePresence(request: Request): Promise<Response> {
+    const refused = () => Response.json({ error: "Invalid or revoked machine token" }, { status: 401 })
+    const token = /^Bearer\s+(\S+)$/i.exec(request.headers.get("authorization") ?? "")?.[1]
+    if (!token || token.startsWith("hlt1.")) return refused()
+    const machine = await store.findMachine(token)
+    if (machine) await store.recheck({ users: [machine.ownerID] })
+    const owner = machine && store.user(machine.ownerID)
+    if (!machine || machine.revoked || !owner || owner.disabled) return refused()
+    const processes = registry.catalog().find((m) => m.id === machine.id)?.processes ?? []
+    const seen = new Set<string>()
+    const spokes = processes.flatMap((p) => {
+      const entry = { harness: p.harness, evidence: p.evidence, features: [...p.features] }
+      const key = JSON.stringify(entry)
+      if (seen.has(key)) return []
+      seen.add(key)
+      return [entry]
+    })
+    return Response.json({ machineID: machine.id, spokes })
+  }
+
   async function login(request: Request, url: URL): Promise<Response> {
     const next = url.searchParams.get("next") ?? "/"
     if (!LOGIN_NEXT.test(next)) return error(400, "Invalid next path")
@@ -1060,12 +1104,24 @@ export function startCentral(options: CentralOptions): Central {
     // The phone's session view (hidden/pinned sessions and workspaces), per user, synced to their other devices.
     route("GET", "/api/prefs/view", "user", ({ user }) => Response.json({ prefs: store.viewPrefs(user.id) }, { headers: NO_STORE })),
     route("PUT", "/api/prefs/view", "user", async ({ request, user, device }) => {
+      const before = new Set(store.viewPrefs(user.id).hiddenSessions)
       const prefs = await store.setViewPrefs(user.id, viewPrefsInput(await request.json()))
       // The user's other devices only: never another user, and not the device that made the change.
       toUser(user.id, { type: "prefs", prefs }, device.id)
+      // Archive changes made in the console go to the agent event journal like the agent API's.
+      const after = new Set(prefs.hiddenSessions)
+      const changes = [...[...after].filter((id) => !before.has(id)).map((id) => archiveEvent(id, true)), ...[...before].filter((id) => !after.has(id)).map((id) => archiveEvent(id, false))]
+      for (const change of changes) await agentEvents.append(user.id, change).catch((cause) => console.error(`[hub] archive event failed: ${(cause as Error).message}`))
       return Response.json({ prefs }, { headers: NO_STORE })
     }),
     route("GET", "/api/reads", "user", ({ user }) => Response.json({ reads: store.reads(user.id) }, { headers: NO_STORE })),
+    // A CLI login (`hub login`): the user approves it as themselves, with the scopes they tick, or denies it.
+    route("GET", "/api/cli-logins/:code", "user", async ({ params }) => {
+      const login = await auth.cliLogin(params.code!)
+      return login ? Response.json({ login }, { headers: NO_STORE }) : error(404, "No pending CLI login with that code")
+    }),
+    route("POST", "/api/cli-logins/:code/approve", "user", async ({ request, params, user }) => decideCliLogin(params.code!, "approved", user, (await body(request)).scopes)),
+    route("POST", "/api/cli-logins/:code/deny", "user", ({ params, user }) => decideCliLogin(params.code!, "denied", user)),
     route("POST", "/api/enrollments/:code/approve", "user", ({ params, user }) => decideEnrollment(params.code!, "approved", user)),
     route("POST", "/api/enrollments/:code/deny", "user", ({ params, user }) => decideEnrollment(params.code!, "denied", user)),
     route("GET", "/api/devices", "user", ({ device, user }) =>
@@ -1431,6 +1487,31 @@ export function startCentral(options: CentralOptions): Central {
     return new Response(null, { status: 204 })
   }
 
+  async function decideCliLogin(code: string, status: "approved" | "denied", user: UserRecord, scopes?: unknown): Promise<Response> {
+    const clientName = await auth.decideCliLogin(code, status, user.id, scopes)
+    if (clientName === undefined) return error(404, "No pending CLI login with that code")
+    await store.audit({ actor: user.id, action: status === "approved" ? "cli-login.approve" : "cli-login.deny", detail: clientName })
+    return new Response(null, { status: 204 })
+  }
+
+  /**
+   * The decision on a CLI login, once. Approved: a new app token of the approver (still active) with the scopes
+   * they chose, in this response only (no-store, never logged).
+   */
+  async function cliLoginPoll(deviceCode: unknown): Promise<Response> {
+    const poll = await auth.pollCliLogin(deviceCode)
+    if (poll.status !== "approved") return Response.json({ status: poll.status }, { headers: NO_STORE })
+    const owner = store.user(poll.ownerID)
+    if (!owner || owner.disabled) return Response.json({ status: "denied" }, { headers: NO_STORE })
+    try {
+      const { record, token } = await issueAppToken(store, owner.id, { label: `CLI login: ${poll.clientName}`, scopes: poll.scopes })
+      return Response.json({ status: "approved", token, tokenID: record.id, scopes: record.scopes }, { headers: NO_STORE })
+    } catch (cause) {
+      if (cause instanceof AgentError && cause.code === "rate_limited") return error(429, cause.message, "rate_limited")
+      throw cause
+    }
+  }
+
   async function revokeMachine(id: string, user: UserRecord): Promise<Response> {
     if (!await store.revokeMachine(id)) return error(404, "No such machine")
     await store.bumpMachineEpoch(id)
@@ -1930,6 +2011,8 @@ export function startCentral(options: CentralOptions): Central {
     launchFeed,
     launchTokens,
     webhooks,
+    agentEvents,
+    agentEventWriters,
     routes: [...routes.map(({ method, path, access }) => ({ method, path, access })), ...agent.routes.map(({ method, path }) => ({ method, path, access: "agent" as const }))],
     drain(drainOptions: DrainOptions = {}) {
       draining ??= runDrain(drainOptions)
@@ -1940,6 +2023,7 @@ export function startCentral(options: CentralOptions): Central {
       await Promise.allSettled([...leaders.map((loop) => loop.stop()), recommendations?.stop()])
       launches.shutdown()
       webhooks.stop()
+      agentEventWriters.stop()
       tokenPush.stop()
       delivery.stop()
       decisions.stop()

package/src/central/mcp-http.ts

@@ -0,0 +1,116 @@
+// POST /mcp: the hub's MCP tools over MCP Streamable HTTP, for MCP clients and gateways (Brain decision txznll2f).
+// Stateless: one JSON-RPC message per POST, answered as application/json (no SSE stream, no Mcp-Session-Id);
+// notifications and client responses get 202. Batches are refused (-32600, as in protocol 2025-06-18).
+//
+// Auth: `Authorization: Bearer <token>`, any credential the agent API accepts (app, device, session and act
+// tokens), checked up front with the agent API's resolver: none or invalid → 401 with `WWW-Authenticate: Bearer
+// realm="hub"` (no OAuth resource metadata, so gateways detect plain bearer auth). The tools are `hub mcp`'s
+// (packages/cli/src/mcp.ts) and call /api/agent with the caller's bearer, dispatched in process: scopes, grants,
+// tenancy, kill switches and the audit are exactly the HTTP API's. A browser Origin other than the hub's own is
+// refused (DNS rebinding); requests without one (servers, gateways) are fine. Tokens are never echoed.
+import { createMcpServer, MCP_PROTOCOL_VERSIONS, NOT_FROM_USER, RPC, rpcError, type RpcResponse } from "../../packages/cli/src/mcp"
+import { readCredentials } from "./agent/actor"
+
+export const MCP_PATH = "/mcp"
+
+/** Largest request body (bytes). */
+export const MCP_MAX_BODY_BYTES = 1 << 20
+
+export const MCP_INSTRUCTIONS =
+  "Hub: see and steer your AI agent sessions. Tools read sessions (status, sessions, show, log, who), message them (msg, ask, reply, decline, inbox), act within your grants (approve, deny, answer, prompt, stop) and launch sessions (launch, launches, launch_show, launch_stop), as this bearer token. " +
+  NOT_FROM_USER
+
+export interface McpHttpDeps {
+  /** The hub's public URL (the tools' base URL; requests never leave the process). */
+  publicURL(): string
+  /** Whether a request's Origin is the hub's own. */
+  isOwnOrigin(origin: string): boolean
+  /** Whether the agent API accepts this bearer token. */
+  verify(token: string): Promise<boolean>
+  /** The hub's own request handler (the tools' /api/agent calls). */
+  dispatch(request: Request): Promise<Response>
+  readonly version: string
+  readonly maxBodyBytes?: number
+}
+
+const NO_STORE = { "cache-control": "no-store" }
+
+const failure = (status: number, code: string, message: string, headers: Record<string, string> = {}) =>
+  Response.json({ error: { code, message } }, { status, headers: { ...NO_STORE, ...headers } })
+
+const unauthorized = (message: string) => failure(401, "unauthorized", message, { "www-authenticate": 'Bearer realm="hub"' })
+
+const answer = (response: RpcResponse) =>
+  // A message the server could not take (parse error, not a request, a batch) carries no id: HTTP 400.
+  Response.json(response, { status: response.error && response.id === null ? 400 : 200, headers: NO_STORE })
+
+/** The body as text, or undefined when it is over `max` bytes (read no further). */
+async function readCapped(request: Request, max: number): Promise<string | undefined> {
+  const length = Number(request.headers.get("content-length") ?? "")
+  if (Number.isFinite(length) && length > max) return undefined
+  if (!request.body) return ""
+  const reader = request.body.getReader()
+  const chunks: Uint8Array[] = []
+  let size = 0
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    size += value.byteLength
+    if (size > max) {
+      void reader.cancel().catch(() => {})
+      return undefined
+    }
+    chunks.push(value)
+  }
+  return new TextDecoder().decode(Buffer.concat(chunks))
+}
+
+const acceptsJSON = (accept: string | null) => {
+  if (accept === null || !accept.trim()) return true
+  return accept.split(",").some((part) => {
+    const type = part.split(";")[0]!.trim().toLowerCase()
+    return type === "application/json" || type === "*/*" || type === "application/*"
+  })
+}
+
+/** Answers a request to /mcp (any method). */
+export function mcpHttp(deps: McpHttpDeps): (request: Request) => Promise<Response> {
+  const max = deps.maxBodyBytes ?? MCP_MAX_BODY_BYTES
+  return async (request) => {
+    if (request.method !== "POST") return failure(405, "method_not_allowed", "POST JSON-RPC messages to /mcp (no SSE stream, no sessions)", { allow: "POST" })
+    const origin = request.headers.get("origin")
+    if (origin !== null && !deps.isOwnOrigin(origin)) return failure(403, "forbidden", "Cross-origin request refused")
+    // The bearer only: a cookie never authenticates /mcp.
+    const token = readCredentials(request, "").bearer
+    if (token === undefined) return unauthorized("Send Authorization: Bearer <hub token>")
+    if (!token || !(await deps.verify(token))) return unauthorized("Invalid, expired or revoked token")
+    const version = request.headers.get("mcp-protocol-version")
+    if (version !== null && !MCP_PROTOCOL_VERSIONS.includes(version)) return failure(400, "invalid", `Unsupported MCP-Protocol-Version (supported: ${MCP_PROTOCOL_VERSIONS.join(", ")})`)
+    const type = (request.headers.get("content-type") ?? "").split(";")[0]!.trim().toLowerCase()
+    if (type !== "application/json") return failure(415, "invalid", "Content-Type must be application/json")
+    if (!acceptsJSON(request.headers.get("accept"))) return failure(406, "invalid", "Accept must allow application/json")
+    const text = await readCapped(request, max)
+    if (text === undefined) return failure(413, "invalid", `Request body over ${max} bytes`)
+    let message: unknown
+    try {
+      message = JSON.parse(text)
+    } catch {
+      return answer(rpcError(null, RPC.parse, "parse error: not JSON"))
+    }
+    const server = createMcpServer({
+      credentials: () => ({ url: deps.publicURL(), token, source: "http", fetch: (input, init) => deps.dispatch(new Request(input, init)) }),
+      version: deps.version,
+      instructions: MCP_INSTRUCTIONS,
+    })
+    // The client went away: its tool calls are aborted.
+    const close = () => server.close()
+    request.signal.addEventListener("abort", close, { once: true })
+    try {
+      const response = await server.handle(message)
+      return response ? answer(response) : new Response(null, { status: 202, headers: NO_STORE })
+    } finally {
+      request.signal.removeEventListener("abort", close)
+      server.close()
+    }
+  }
+}

package/src/central/push.ts

@@ -32,6 +32,7 @@ export const NOTIFICATION_LABELS: Readonly<Record<NotificationKind, string>> = O
   activity: "Activity",
   decision: "Decision",
   message: "Message",
+  providerError: "Model provider error",
 })
 
 /** True for a known notification kind (API input). */
@@ -47,6 +48,8 @@ export interface PushExtra {
   readonly key?: string
   /** decision: the Decision, so a tap opens its card. */
   readonly decisionID?: string
+  /** Skip devices that have this kind on (they were notified through it instead). Not sent. */
+  readonly unlessPref?: NotificationKind
 }
 
 export interface PushPayload {
@@ -58,6 +61,8 @@ export interface PushPayload {
   readonly decisionID?: string
   /** A test notification shaped like `kind`. */
   readonly test?: true
+  /** Per-session kinds (providerError, subagentFailed): the session it is about, so the service worker tags each apart. */
+  readonly tagKey?: string
 }
 
 const minutes = (clock: string): number => Number(clock.slice(0, 2)) * 60 + Number(clock.slice(3, 5))
@@ -188,6 +193,8 @@ function kindTitle(kind: NotificationKind, subject: string | undefined): string
 }
 
 const OPEN = "Tap to open the session."
+/** A session ID: the only key a payload may carry (job and tool IDs stay on the hub). */
+const SESSION_ID = /^ses_[A-Za-z0-9]{1,64}$/
 
 export function pushPayload(kind: NotificationKind, sessionID: string, sessionTitle: string, text?: string, extra: PushExtra = {}): PushPayload {
   const name = shorten(sessionTitle)
@@ -213,6 +220,9 @@ export function pushPayload(kind: NotificationKind, sessionID: string, sessionTi
       return { kind, sessionID, title, body: detail || "Nothing is running or waiting. Tap to open." }
     case "activity":
       return { kind, sessionID, title, body: detail || "New activity." }
+    case "providerError":
+    case "subagentFailed":
+      return { kind, sessionID, title, body: detail || OPEN, ...(extra.key && SESSION_ID.test(extra.key) ? { tagKey: extra.key } : {}) }
     default:
       return { kind, sessionID, title, body: detail || OPEN }
   }
@@ -229,6 +239,7 @@ export function sendOptionsFor(kind: NotificationKind | "test"): { TTL: number;
     case "failed":
     case "decision":
     case "message":
+    case "providerError":
       return { TTL: URGENT_TTL_SECONDS, urgency: "high" }
     case "done":
     case "stopped":
@@ -385,7 +396,9 @@ export class Push {
     const targets = this.store.pushSubscriptions().filter((s) => {
       const owner = owners.get(s.deviceID)
       if (ownerID !== undefined && owner !== ownerID) return false
-      if (!this.store.pushPrefs(s.deviceID)[precise]) return false
+      const prefs = this.store.pushPrefs(s.deviceID)
+      if (!prefs[precise]) return false
+      if (extra?.unlessPref && prefs[extra.unlessPref]) return false
       if (precise === "handoff") return true
       const quiet = this.store.quietHours(s.deviceID)
       return !isMuted(owner) && !(quiet && inQuietHours(quiet, at))

package/src/central/registry.ts

@@ -12,7 +12,7 @@
 // holds the connection; remote state is never re-exported.
 import path from "node:path"
 import { HARNESS_OPENCODE, helloHarness, spokeFeatures, type EvidenceLevel, type ForwardedEvent, type Hello, type HerdrWorkspace, type ProcessKind, type SessionInfo, type SpokeFeature, type SpokeMode, type TerminalInfo, type Terminals } from "../protocol"
-import { StatusBoard, permissionRequest, type Blocker, type BlockerRecommendation, type PermissionRequest, type SessionStatus } from "../status"
+import { StatusBoard, errorTypeLabel, permissionRequest, providerFailureNeedsUser, providerRetryNeedsUser, type Blocker, type BlockerRecommendation, type PermissionRequest, type SessionStatus } from "../status"
 import { sanitizeLabel } from "./agent/envelope"
 import { DoneWatch } from "./done"
 import { SentenceStream } from "./say-stream"
@@ -150,6 +150,8 @@ export interface Alert {
   /** Tenant (user ID) that owns the session; push goes to this user's devices only. */
   readonly owner?: string
   readonly step?: ActivityStep
+  /** Skip devices that have this kind on (they were told through it instead). */
+  readonly unlessPref?: NotificationKind
 }
 
 /** Notifications that reach a session even while some device shows it. */
@@ -407,6 +409,8 @@ export class Registry {
   private readonly forms = new Map<string, { connIDs: string[]; sessionID: string }>()
   /** Form IDs already alerted (forms may be forwarded by several processes). */
   private readonly alertedForms = new Set<string>()
+  /** Sessions whose current run raised a providerError notification (once per run; cleared when a run starts). */
+  private readonly providerAlerted = new Set<string>()
   /** Sessions in conversation mode → the connection last told so (re-sent when another one takes over). */
   private readonly conversations = new Map<string, string | undefined>()
   /** `messageID ordinal` of texts already sent as say events (several processes forward the same text). */
@@ -1433,15 +1437,27 @@ export class Registry {
     if (!rootID) return
     const root = rootID === sessionID
     const wasRunning = before?.state === "working" || before?.state === "blocked"
-    if (type === "session.execution.started") return this.doneWatch.started(rootID, root)
+    if (type === "session.execution.started") {
+      this.providerAlerted.delete(sessionID)
+      return this.doneWatch.started(rootID, root)
+    }
     if (type === "session.text.delta" || type === "session.reasoning.delta") return
     const ended = type === "session.execution.succeeded" ? "succeeded" : type === "session.execution.failed" ? "failed" : type === "session.execution.interrupted" ? "interrupted" : undefined
     const reason = (data.error as { message?: unknown } | undefined)?.message
     const error = typeof reason === "string" && reason.trim() ? reason : undefined
+    const errorType = (data.error as { type?: unknown } | undefined)?.type
+    // A provider error the user must act on replaces the root's "failed" (raised once per run).
+    const providerFailure = ended === "failed" && typeof errorType === "string" && providerFailureNeedsUser(errorType)
+    if (providerFailure) this.providerError(rootID, sessionID, error ?? errorTypeLabel(errorType) ?? "Model provider error")
+    if (type === "session.status") {
+      const retry = this.status(rootID)?.find((s) => s.id === sessionID)?.retry
+      if (retry && providerRetryNeedsUser(retry.errorType)) this.providerError(rootID, sessionID, retry.message ?? errorTypeLabel(retry.errorType)!)
+    }
     if (ended && root) {
       this.doneWatch.rootEnded(rootID, ended, wasRunning)
       if (ended === "succeeded" && wasRunning && !this.active.has(rootID)) this.alert("turn", rootID)
-      if (ended === "failed") this.alert("failed", rootID, error ? { text: error } : {})
+      // Devices with providerError on got that instead; the rest still hear about the failure.
+      if (ended === "failed") this.alert("failed", rootID, { ...(error ? { text: error } : {}), ...(providerFailure ? { unlessPref: "providerError" as const } : {}) })
       const stopped = ended === "interrupted" ? STOP_REASONS[String(data.reason)] : undefined
       if (stopped) this.alert("stopped", rootID, { text: stopped })
       return
@@ -1456,7 +1472,10 @@ export class Registry {
     }
     if (type === "session.status" && root && !before?.retry) {
       const retry = this.status(rootID)?.find((s) => s.id === rootID)?.retry
-      if (retry) this.alert("retry", rootID, { text: [retry.attempt !== undefined ? `Attempt ${retry.attempt}` : "", retry.message ?? ""].filter(Boolean).join(": ") || "Retrying" })
+      if (retry) {
+        const why = [retry.attempt !== undefined ? `Attempt ${retry.attempt}` : "", errorTypeLabel(retry.errorType) ?? ""].filter(Boolean).join(" · ")
+        this.alert("retry", rootID, { text: [why, retry.message ?? ""].filter(Boolean).join(": ") || "Retrying" })
+      }
       return
     }
     if (type === "shell.exited" || type === "session.shell.ended" || type === "pty.exited") {
@@ -1469,6 +1488,15 @@ export class Registry {
     }
   }
 
+  /** A model provider error of `sessionID` (root or subagent) needs the user: "<session title>: <message>", once per run. */
+  private providerError(rootID: string, sessionID: string, message: string): void {
+    if (this.providerAlerted.has(sessionID)) return
+    this.providerAlerted.add(sessionID)
+    if (this.providerAlerted.size > 1000) this.providerAlerted.delete(this.providerAlerted.values().next().value!)
+    const title = this.status(rootID)?.find((s) => s.id === sessionID)?.title?.trim() || "Session"
+    this.alert("providerError", rootID, { text: `${title}: ${message}`, key: sessionID })
+  }
+
   /** Fires when a root's tree stayed settled after its run (DoneWatch). */
   private settledTree(rootID: string): boolean {
     const board = this.boards.get(rootID)
@@ -1938,7 +1966,7 @@ export class Registry {
    * Raises a notification for the root of `sessionID`'s tree. Only hand-offs, questions and permissions reach a
    * session some device shows (spoken replies check that themselves: the watching device speaks them).
    */
-  private alert(kind: NotificationKind, sessionID: string, fields: { text?: string; requestID?: string; subject?: string; key?: string } = {}): void {
+  private alert(kind: NotificationKind, sessionID: string, fields: { text?: string; requestID?: string; subject?: string; key?: string; unlessPref?: NotificationKind } = {}): void {
     const entry = [...this.boards].find(([, board]) => board.get(sessionID))
     if (!entry) return
     const [rootID, board] = entry
@@ -1951,6 +1979,7 @@ export class Registry {
       ...(fields.text !== undefined ? { text: fields.text } : {}),
       ...(fields.requestID !== undefined ? { requestID: fields.requestID } : {}),
       ...(fields.key !== undefined ? { key: fields.key } : {}),
+      ...(fields.unlessPref !== undefined ? { unlessPref: fields.unlessPref } : {}),
     }
     const owner = this.sessionTenant(rootID)
     for (const listener of this.alertListeners) listener(owner ? { ...alert, owner } : alert)

package/src/central/store.ts

@@ -58,6 +58,9 @@ export interface ViewPrefs {
 }
 
 export const VIEW_PREF_KEYS = ["hiddenSessions", "hiddenWorkspaces", "pinnedSessions", "pinnedWorkspaces"] as const
+/** Archived sessions accumulate for longer than the other view-preference lists. */
+export const VIEW_PREF_MAX_ARCHIVED = 5000
+export const VIEW_PREF_MAX_PINNED = 500
 
 export interface AuditEntry {
   at: number
@@ -118,6 +121,11 @@ export interface PushPrefs {
   decision: boolean
   /** Agent messaging needs you: a message to @me, an ask left unanswered after every re-wake, or a message loop. */
   message: boolean
+  /**
+   * A model provider error needs you: a quota, free-tier, budget, auth or no-route failure of any session in the tree
+   * (once per run), or a retry for a quota, free-tier or auth error. Replaces the root's "failed" for that failure.
+   */
+  providerError: boolean
 }
 
 export const PUSH_PREF_KEYS = [
@@ -138,6 +146,7 @@ export const PUSH_PREF_KEYS = [
   "activity",
   "decision",
   "message",
+  "providerError",
 ] as const satisfies readonly (keyof PushPrefs)[]
 
 /** Defaults: anything that needs you, all done, failed, hand-offs and spoken replies; the finer kinds are opt-in. */
@@ -160,6 +169,7 @@ export const DEFAULT_PUSH_PREFS: Readonly<PushPrefs> = Object.freeze({
   activity: false,
   decision: true,
   message: true,
+  providerError: true,
 })
 
 /** Default and allowed range of a device's activity update interval, in minutes. */
@@ -911,6 +921,56 @@ export class Store {
     })
   }
 
+  /** Changes one archived session atomically; undefined means the bounded archive list is full. */
+  setSessionArchived(id: string, sessionID: string, archived: boolean): Promise<{ changed: boolean } | undefined> {
+    return this.mutate((tx) => {
+      const user = this.user(id)
+      if (!user) return undefined
+      const next = this.viewPrefs(id)
+      const listed = next.hiddenSessions.includes(sessionID)
+      const changed = archived !== listed
+      if (!changed) return { changed }
+      if (archived && next.hiddenSessions.length >= VIEW_PREF_MAX_ARCHIVED) return undefined
+      next.hiddenSessions = archived ? [...next.hiddenSessions, sessionID] : next.hiddenSessions.filter((id) => id !== sessionID)
+      user.viewPrefs = next
+      tx.touch("users", id)
+      return { changed }
+    })
+  }
+
+  /** Changes one pinned session atomically while preserving every other preference list. */
+  setSessionPinned(id: string, sessionID: string, pinned: boolean): Promise<{ changed: boolean } | undefined> {
+    return this.mutate((tx) => {
+      const user = this.user(id)
+      if (!user) return undefined
+      const next = this.viewPrefs(id)
+      const listed = next.pinnedSessions.includes(sessionID)
+      const changed = pinned !== listed
+      if (!changed) return { changed }
+      if (pinned && next.pinnedSessions.length >= VIEW_PREF_MAX_PINNED) return undefined
+      next.pinnedSessions = pinned ? [...next.pinnedSessions, sessionID] : next.pinnedSessions.filter((id) => id !== sessionID)
+      user.viewPrefs = next
+      tx.touch("users", id)
+      return { changed }
+    })
+  }
+
+  /** Archives only while still unpinned, atomically against concurrent preference writes. */
+  archiveSessionIfUnpinned(id: string, sessionID: string): Promise<"changed" | "pinned" | "archived" | "full" | "not_found"> {
+    return this.mutate((tx) => {
+      const user = this.user(id)
+      if (!user) return "not_found"
+      const next = this.viewPrefs(id)
+      if (next.hiddenSessions.includes(sessionID)) return "archived"
+      if (next.pinnedSessions.includes(sessionID)) return "pinned"
+      if (next.hiddenSessions.length >= VIEW_PREF_MAX_ARCHIVED) return "full"
+      next.hiddenSessions = [...next.hiddenSessions, sessionID]
+      user.viewPrefs = next
+      tx.touch("users", id)
+      return "changed"
+    })
+  }
+
   // ---------- per-user, per-session model preferences ----------
 
   modelPrefs(id: string, sessionID: string): SessionModelPrefs {