@huynle/hub 0.6.1 → 0.7.2

package/src/central/push-target.ts

@@ -1,7 +1,10 @@
-// Push endpoints come from browsers, and the hub POSTs to them: without checks a subscription could make the
-// hub send requests into its own network (SSRF). An endpoint must be https on a public hostname; hosts are
-// resolved at subscribe time and rejected when any address is loopback, private, link-local, CGNAT, ULA,
-// unspecified or multicast. Real push services (FCM, Mozilla, Apple, WNS) are public names on public addresses.
+// URLs users give the hub (push endpoints from browsers, webhook URLs from apps) are URLs the hub POSTs to: without
+// checks a subscription could make the hub send requests into its own network (SSRF). publicURLProblem: https on a
+// public hostname whose every resolved address is public (not loopback, private, link-local, CGNAT, ULA,
+// unspecified or multicast); an admin opt-in (allowPrivate, webhooks only) admits http and private targets.
+// Push checks endpoints at subscribe time (real push services are public names on public addresses); webhooks
+// send through publicFetch, which re-checks (and re-resolves) right before every request, never follows a
+// redirect, times out and reads a bounded body.
 import { lookup } from "node:dns/promises"
 import { isIPv4, isIPv6 } from "node:net"
 
@@ -10,7 +13,7 @@ export type ResolveHost = (host: string) => Promise<readonly string[]>
 /** Every address the system resolver returns for a host. */
 export const resolveHost: ResolveHost = async (host) => (await lookup(host, { all: true, verbatim: true })).map((a) => a.address)
 
-/** IPv4 [network, prefix length] blocks that never reach a public push service. */
+/** IPv4 [network, prefix length] blocks that are never a public service. */
 const V4_BLOCKED: ReadonlyArray<readonly [string, number]> = [
   ["0.0.0.0", 8], // "this network", unspecified
   ["10.0.0.0", 8], // private
@@ -47,7 +50,7 @@ function v6Groups(ip: string): number[] {
   return [...left, ...fill, ...right].map((g) => parseInt(g, 16))
 }
 
-/** True for addresses a push endpoint must not resolve to. Anything that is not an IP counts as private. */
+/** True for addresses a public URL must not resolve to. Anything that is not an IP counts as private. */
 export function isPrivateAddress(address: string): boolean {
   const ip = address.trim().replace(/^\[(.*)\]$/, "$1")
   if (isIPv4(ip)) return privateV4(ip)
@@ -70,28 +73,125 @@ export function isPrivateAddress(address: string): boolean {
 
 const LOCAL_SUFFIXES = [".localhost", ".local", ".internal", ".localdomain", ".home.arpa", ".lan"]
 
-/** Why an endpoint is not acceptable for Web Push, or undefined when it is. */
-export async function pushEndpointProblem(endpoint: string, resolve: ResolveHost): Promise<string | undefined> {
+export interface PublicURLOptions {
+  readonly resolve: ResolveHost
+  /** How problems name the URL (default "URL"). */
+  readonly label?: string
+  /**
+   * The admin opt-in for private targets (hub.json agents.webhooks.allowPrivate): http, loopback, private
+   * addresses and local names pass without a lookup. Other schemes and credentials still fail.
+   */
+  readonly allowPrivate?: boolean
+}
+
+/** Why the hub must not send to a URL (SSRF), or undefined when it may. */
+export async function publicURLProblem(raw: string, options: PublicURLOptions): Promise<string | undefined> {
+  const label = options.label ?? "URL"
   let url: URL
   try {
-    url = new URL(endpoint)
+    url = new URL(raw)
   } catch {
-    return "Push endpoint is not a URL"
+    return `${label} is not a URL`
   }
-  if (url.protocol !== "https:") return "Push endpoint must be https"
-  if (url.username || url.password) return "Push endpoint must not carry credentials"
+  if (options.allowPrivate) {
+    if (url.protocol !== "https:" && url.protocol !== "http:") return `${label} must be http or https`
+  } else if (url.protocol !== "https:") return `${label} must be https`
+  if (url.username || url.password) return `${label} must not carry credentials`
+  if (options.allowPrivate) return undefined
   const host = url.hostname.toLowerCase().replace(/^\[(.*)\]$/, "$1").replace(/\.$/, "")
-  if (isIPv4(host) || isIPv6(host)) return isPrivateAddress(host) ? "Push endpoint is a private or local address" : undefined
+  if (isIPv4(host) || isIPv6(host)) return isPrivateAddress(host) ? `${label} is a private or local address` : undefined
   if (host === "localhost" || !host.includes(".") || LOCAL_SUFFIXES.some((suffix) => host.endsWith(suffix))) {
-    return "Push endpoint is a local hostname"
+    return `${label} is a local hostname`
   }
   let addresses: readonly string[]
   try {
-    addresses = await resolve(host)
+    addresses = await options.resolve(host)
   } catch {
-    return `Push endpoint host ${host} does not resolve`
+    return `${label} host ${host} does not resolve`
   }
-  if (addresses.length === 0) return `Push endpoint host ${host} does not resolve`
-  if (addresses.some(isPrivateAddress)) return "Push endpoint resolves to a private or local address"
+  if (addresses.length === 0) return `${label} host ${host} does not resolve`
+  if (addresses.some(isPrivateAddress)) return `${label} resolves to a private or local address`
   return undefined
 }
+
+/** Why an endpoint is not acceptable for Web Push, or undefined when it is. */
+export async function pushEndpointProblem(endpoint: string, resolve: ResolveHost): Promise<string | undefined> {
+  return publicURLProblem(endpoint, { resolve, label: "Push endpoint" })
+}
+
+/** The URL failed publicURLProblem at send time; nothing was sent. */
+export class PublicURLError extends Error {
+  constructor(readonly problem: string) {
+    super(problem)
+    this.name = "PublicURLError"
+  }
+}
+
+export interface PublicFetchOptions extends PublicURLOptions {
+  /** For the whole exchange, body included. */
+  readonly timeoutMs: number
+  /** Bytes of the response body read at most; the rest is discarded. */
+  readonly maxBytes: number
+  readonly fetch?: typeof globalThis.fetch
+}
+
+export interface PublicFetchResult {
+  readonly status: number
+  /** The first maxBytes of the body (UTF-8, lossy). */
+  readonly body: string
+  readonly truncated: boolean
+}
+
+/**
+ * One request to a user-supplied URL: the URL is checked (and its host re-resolved) right before sending, so a
+ * name that changed to a private address since registration is refused; redirects are never followed (a 3xx is
+ * returned as is); the timeout covers the body; at most maxBytes of the body are read.
+ * A rebinding between this lookup and fetch's own remains possible; private targets need the admin opt-in.
+ */
+export async function publicFetch(
+  url: string,
+  init: { readonly method: string; readonly headers?: Record<string, string>; readonly body: string },
+  options: PublicFetchOptions,
+): Promise<PublicFetchResult> {
+  const problem = await publicURLProblem(url, options)
+  if (problem) throw new PublicURLError(problem)
+  const controller = new AbortController()
+  let timedOut = false
+  const timer = setTimeout(() => {
+    timedOut = true
+    controller.abort()
+  }, options.timeoutMs)
+  try {
+    const response = await (options.fetch ?? globalThis.fetch)(url, { ...init, redirect: "manual", signal: controller.signal })
+    const { body, truncated } = await readCapped(response, options.maxBytes)
+    return { status: response.status, body, truncated }
+  } catch (cause) {
+    if (timedOut) throw new Error(`timed out after ${options.timeoutMs} ms`)
+    throw cause
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+async function readCapped(response: Response, maxBytes: number): Promise<{ body: string; truncated: boolean }> {
+  if (!response.body) return { body: "", truncated: false }
+  const reader = response.body.getReader()
+  const chunks: Uint8Array[] = []
+  let size = 0
+  let truncated = false
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) break
+    const room = maxBytes - size
+    if (value.byteLength > room) {
+      chunks.push(value.subarray(0, room))
+      size = maxBytes
+      truncated = true
+      await reader.cancel().catch(() => {})
+      break
+    }
+    chunks.push(value)
+    size += value.byteLength
+  }
+  return { body: Buffer.concat(chunks).toString("utf8"), truncated }
+}

package/src/central/push.ts

@@ -1,22 +1,52 @@
-// Web push (VAPID) for paired devices: hand-offs from the terminal, questions, permissions, finished and
-// failed runs, and (conversation mode, nobody watching) what the agent said. Payloads carry only {kind,
-// sessionID, title, body, requestID?}: no tokens, no transcript beyond one spoken text, shortened. Each device picks the kinds it wants, may set quiet hours, and users mute sessions.
+// Web push (VAPID) for paired devices, one toggle per notification kind (docs/notifications.md): hand-offs, questions,
+// permissions, all done, failed / stopped / retrying runs, every turn, subagents, scripts, long tools, spoken replies
+// and the activity digest. Payloads carry only {kind, sessionID, title, body, requestID?, test?}: no tokens, no
+// transcript beyond one spoken text or a short reason, shortened. Each device picks the kinds it wants, may set quiet
+// hours, and users mute sessions.
 import webpush from "web-push"
-import { type PushPrefs, type PushTarget, type QuietHours, type Store } from "./store"
+import { MemoryKv } from "./backend/memory-kv"
+import type { Kv } from "./backend/types"
+import { type PushPrefs, type PushTarget, type QuietHours, type Store, type VapidKeys } from "./store"
 
 export { DEFAULT_PUSH_PREFS, type PushPrefs } from "./store"
 
-/** What happened, as the registry and the spoke socket report it. */
-export type PushKind = "handoff" | "blocked" | "finished" | "say"
-/** What a blocked or finished notification is about. */
-export type PushDetail = "permission" | "form" | "succeeded" | "failed"
-/** The precise notification kind; one per device toggle. */
+/** The notification kind; one per device toggle. */
 export type NotificationKind = keyof PushPrefs
 
+/** Short label of each kind, shown in the title after the session name (and in Settings). */
+export const NOTIFICATION_LABELS: Readonly<Record<NotificationKind, string>> = Object.freeze({
+  handoff: "Hand-off",
+  question: "Question",
+  permission: "Needs permission",
+  done: "All done",
+  failed: "Failed",
+  turn: "Turn finished",
+  stopped: "Stopped",
+  retry: "Retrying",
+  subagentDone: "Subagent finished",
+  subagentFailed: "Subagent failed",
+  jobDone: "Script finished",
+  jobFailed: "Script failed",
+  longTool: "Long tool finished",
+  say: "Spoken reply",
+  activity: "Activity",
+  decision: "Decision",
+  message: "Message",
+})
+
+/** True for a known notification kind (API input). */
+export const isNotificationKind = (value: unknown): value is NotificationKind => typeof value === "string" && Object.hasOwn(NOTIFICATION_LABELS, value)
+
 /** Extra facts for one notification. */
 export interface PushExtra {
   /** permission: the request ID, so the notification can answer it. */
   readonly requestID?: string
+  /** What the notification is about, for the title: a subagent's title, a job's outcome ("exited 1"), a duration. */
+  readonly subject?: string
+  /** Identity of what ended (subagent session, job, tool call): the cooldown is per session, kind and key. Not sent. */
+  readonly key?: string
+  /** decision: the Decision, so a tap opens its card. */
+  readonly decisionID?: string
 }
 
 export interface PushPayload {
@@ -25,18 +55,9 @@ export interface PushPayload {
   readonly title: string
   readonly body: string
   readonly requestID?: string
-}
-
-/** The device toggle that decides an alert. */
-export function pushKindFor(kind: PushKind, detail?: PushDetail): NotificationKind {
-  switch (kind) {
-    case "blocked":
-      return detail === "form" ? "question" : "permission"
-    case "finished":
-      return detail === "failed" ? "failed" : "finished"
-    default:
-      return kind
-  }
+  readonly decisionID?: string
+  /** A test notification shaped like `kind`. */
+  readonly test?: true
 }
 
 const minutes = (clock: string): number => Number(clock.slice(0, 2)) * 60 + Number(clock.slice(3, 5))
@@ -80,6 +101,7 @@ export const URGENT_TTL_SECONDS = 3600
 /** One test notification per device in this window. */
 export const TEST_COOLDOWN_MS = 10_000
 const SEND_TIMEOUT_MS = 10_000
+const COOL_KEY = (key: string) => `push:cool:${key}`
 const MAX_NAME = 60
 
 const webPushSender: PushSender = (target, payload, options) =>
@@ -91,6 +113,51 @@ export interface PushOptions {
   readonly subject: string
   readonly sender?: PushSender
   readonly now?: () => number
+  /** Timers for delayed activity digests (tests). */
+  readonly timers?: { set(fn: () => void, ms: number): unknown; clear(handle: unknown): void }
+  /** Cooldowns (default: this process, on `now`; Valkey in a cluster, so pods share them). */
+  readonly kv?: Kv
+}
+
+/** One step of interest in a session tree, for activity updates. */
+export interface ActivityStep {
+  /** tool: a short tool call (only counted); anything else is a line of the digest and sends it. */
+  readonly kind: "tool" | "long" | "subagent" | "job" | "idle"
+  readonly text: string
+}
+
+/** An activity step of a root session, as the registry reports it. */
+export interface ActivityAlert {
+  readonly sessionID: string
+  readonly title: string
+  readonly step?: ActivityStep
+  readonly owner?: string
+}
+
+/** Lines of a digest kept (older ones become "+n more"). */
+const DIGEST_LINES = 5
+
+interface Digest {
+  title: string
+  tools: number
+  lines: string[]
+  last?: number
+  timer?: unknown
+}
+
+const realTimers = {
+  set: (fn: () => void, ms: number) => {
+    const handle = setTimeout(fn, ms)
+    ;(handle as { unref?: () => void }).unref?.()
+    return handle
+  },
+  clear: (handle: unknown) => clearTimeout(handle as ReturnType<typeof setTimeout>),
+}
+
+/** "3 tools · explore finished · bun run build exited 0". */
+export function digestBody(tools: number, lines: readonly string[]): string {
+  const shown = lines.length > DIGEST_LINES ? [`+${lines.length - DIGEST_LINES + 1} more`, ...lines.slice(-(DIGEST_LINES - 1))] : [...lines]
+  return [...(tools ? [`${tools} tool${tools === 1 ? "" : "s"}`] : []), ...shown].join(" · ")
 }
 
 function clip(text: string, max: number): string {
@@ -100,48 +167,83 @@ function clip(text: string, max: number): string {
 
 const shorten = (title: string): string => clip(title, MAX_NAME) || "Session"
 
-export function pushPayload(
-  kind: PushKind,
-  sessionID: string,
-  sessionTitle: string,
-  detail?: PushDetail,
-  text?: string,
-  extra: PushExtra = {},
-): PushPayload {
+const MAX_SUBJECT = 40
+
+/** The title after the session name: the kind's label, with its subject where it has one. */
+function kindTitle(kind: NotificationKind, subject: string | undefined): string {
+  const about = subject ? clip(subject, MAX_SUBJECT) : ""
+  switch (kind) {
+    case "subagentDone":
+      return about ? `Subagent ${about} finished` : NOTIFICATION_LABELS.subagentDone
+    case "subagentFailed":
+      return about ? `Subagent ${about} failed` : NOTIFICATION_LABELS.subagentFailed
+    case "jobDone":
+    case "jobFailed":
+      return about ? `Script ${about}` : NOTIFICATION_LABELS[kind]
+    case "longTool":
+      return about ? `${NOTIFICATION_LABELS.longTool} (${about})` : NOTIFICATION_LABELS.longTool
+    default:
+      return NOTIFICATION_LABELS[kind]
+  }
+}
+
+const OPEN = "Tap to open the session."
+
+export function pushPayload(kind: NotificationKind, sessionID: string, sessionTitle: string, text?: string, extra: PushExtra = {}): PushPayload {
   const name = shorten(sessionTitle)
-  const precise = pushKindFor(kind, detail)
-  switch (precise) {
+  const detail = clip(text ?? "", SAY_MAX_CHARS)
+  const title = `${name} · ${kindTitle(kind, extra.subject)}`
+  switch (kind) {
     case "say":
-      return { kind: precise, sessionID, title: name, body: clip(text ?? "", SAY_MAX_CHARS) || "New reply. Tap to listen." }
+      // A spoken reply reads as the session talking.
+      return { kind, sessionID, title: name, body: detail || "New reply. Tap to listen." }
     case "handoff":
-      return { kind: precise, sessionID, title: name, body: "Sent from the terminal. Tap to talk." }
+      return { kind, sessionID, title, body: "Sent from the terminal. Tap to talk." }
     case "question":
-      return { kind: precise, sessionID, title: `${name} has a question`, body: "Tap to see it." }
+      return { kind, sessionID, title, body: "Tap to see it." }
+    // An agent wrote the question or message: never in the payload (the console shows it after sign-in).
+    case "decision":
+      return { kind, sessionID, title, body: "A decision waits for you. Tap to answer.", ...(extra.decisionID ? { decisionID: extra.decisionID } : {}) }
+    case "message":
+      return { kind, sessionID, title, body: "Tap to see it." }
     case "permission":
       // Never the request's summary (it may be a command line); the service worker fetches it from the hub.
-      return {
-        kind: precise,
-        sessionID,
-        title: `${name} needs permission`,
-        body: "Tap to review and answer.",
-        ...(extra.requestID ? { requestID: extra.requestID } : {}),
-      }
-    case "failed":
-      return { kind: precise, sessionID, title: `${name} failed`, body: "Tap to open the session." }
-    case "finished":
-      return { kind: precise, sessionID, title: `${name} finished`, body: "Tap to open the session." }
+      return { kind, sessionID, title, body: "Tap to review and answer.", ...(extra.requestID ? { requestID: extra.requestID } : {}) }
+    case "done":
+      return { kind, sessionID, title, body: detail || "Nothing is running or waiting. Tap to open." }
+    case "activity":
+      return { kind, sessionID, title, body: detail || "New activity." }
+    default:
+      return { kind, sessionID, title, body: detail || OPEN }
   }
 }
 
+/** What a test notification of each kind is about. */
+const TEST_SUBJECT: Partial<Record<NotificationKind, string>> = { subagentDone: "explore", subagentFailed: "explore", jobDone: "exited 0", jobFailed: "exited 1", longTool: "45s" }
+
 /** web-push options per notification kind: urgent kinds wake a dozing phone and wait longer for it. */
 export function sendOptionsFor(kind: NotificationKind | "test"): { TTL: number; urgency: "high" | "normal" } {
   switch (kind) {
     case "question":
     case "permission":
     case "failed":
+    case "decision":
+    case "message":
       return { TTL: URGENT_TTL_SECONDS, urgency: "high" }
-    case "finished":
+    case "done":
+    case "stopped":
+    case "subagentFailed":
+    case "jobFailed":
+      // Worth seeing later, not worth waking a dozing phone.
       return { TTL: URGENT_TTL_SECONDS, urgency: "normal" }
+    case "turn":
+    case "retry":
+    case "subagentDone":
+    case "jobDone":
+    case "longTool":
+    case "activity":
+      // Progress: stale soon, replaced by the next one.
+      return { TTL: PUSH_TTL_SECONDS, urgency: "normal" }
     default:
       return { TTL: PUSH_TTL_SECONDS, urgency: "high" }
   }
@@ -170,19 +272,96 @@ export class Push {
   private readonly subject: string
   private readonly sender: PushSender
   private readonly now: () => number
-  /** `session kind` → when it was last sent and how long it cools down. */
-  private readonly lastSent = new Map<string, { at: number; cooldown: number }>()
+  /** Cooldowns: `push:cool:<session kind>` / `push:cool:test <device>` → when it was sent, expiring with the cooldown. */
+  private readonly kv: Kv
+  /**
+   * `device session` → the activity digest being collected. Kept in this process: activity is reported by the
+   * pod holding the session's connection.
+   */
+  private readonly digests = new Map<string, Digest>()
+  private readonly timers: NonNullable<PushOptions["timers"]>
 
   constructor(options: PushOptions) {
     this.store = options.store
     this.subject = options.subject
     this.sender = options.sender ?? webPushSender
     this.now = options.now ?? Date.now
+    this.timers = options.timers ?? realTimers
+    this.kv = options.kv ?? new MemoryKv({ now: this.now })
+  }
+
+  /**
+   * Activity updates: collects a session's steps per device that wants them (pref `activity`, not muted, not in
+   * quiet hours) and sends one collapsing notification: at once when the device got none for this session within
+   * its interval, else when the interval ends, with everything that happened meanwhile. Short tools only count.
+   */
+  activity(alert: ActivityAlert): void {
+    const step = alert.step
+    if (!alert.owner || !step) return
+    const at = this.now()
+    for (const deviceID of this.activityDevices(alert.owner, alert.sessionID, at)) {
+      const key = `${deviceID}\n${alert.sessionID}`
+      let digest = this.digests.get(key)
+      if (!digest) this.digests.set(key, (digest = { title: alert.title, tools: 0, lines: [] }))
+      digest.title = alert.title
+      if (step.kind === "tool") {
+        digest.tools++
+        continue
+      }
+      if (digest.lines.at(-1) !== step.text) digest.lines.push(step.text)
+      if (digest.lines.length > 50) digest.lines.splice(0, digest.lines.length - 50)
+      if (digest.timer !== undefined) continue
+      const wait = digest.last === undefined ? 0 : digest.last + this.store.activityMinutes(deviceID) * 60_000 - at
+      if (wait <= 0) void this.flushDigest(deviceID, alert.sessionID, alert.owner)
+      else {
+        const owner = alert.owner
+        digest.timer = this.timers.set(() => void this.flushDigest(deviceID, alert.sessionID, owner), wait)
+      }
+    }
+    this.pruneDigests(at)
+  }
+
+  /** Devices of `owner` with a subscription that want activity updates for the session now. */
+  private activityDevices(owner: string, sessionID: string, at: number): string[] {
+    if (this.store.mutedSessions(owner).includes(sessionID)) return []
+    const mine = new Set(this.store.devices().filter((d) => d.ownerID === owner).map((d) => d.id))
+    const devices = new Set<string>()
+    for (const s of this.store.pushSubscriptions()) {
+      if (!mine.has(s.deviceID) || devices.has(s.deviceID) || !this.store.pushPrefs(s.deviceID).activity) continue
+      const quiet = this.store.quietHours(s.deviceID)
+      if (quiet && inQuietHours(quiet, at)) continue
+      devices.add(s.deviceID)
+    }
+    return [...devices]
+  }
+
+  private async flushDigest(deviceID: string, sessionID: string, owner: string): Promise<void> {
+    const key = `${deviceID}\n${sessionID}`
+    const digest = this.digests.get(key)
+    if (!digest) return
+    if (digest.timer !== undefined) this.timers.clear(digest.timer)
+    digest.timer = undefined
+    const body = digestBody(digest.tools, digest.lines)
+    digest.tools = 0
+    digest.lines = []
+    digest.last = this.now()
+    // Settings may have changed while it waited.
+    if (!body || !this.activityDevices(owner, sessionID, digest.last).includes(deviceID)) return
+    const targets = this.store.pushSubscriptions().filter((s) => s.deviceID === deviceID)
+    await this.send(targets, pushPayload("activity", sessionID, digest.title, body))
+  }
+
+  /** Forgets digests idle for an hour (nothing pending, interval long over). */
+  private pruneDigests(at: number): void {
+    if (this.digests.size < 1000) return
+    for (const [key, digest] of this.digests) {
+      if (digest.timer === undefined && !digest.lines.length && (digest.last ?? 0) < at - 3_600_000) this.digests.delete(key)
+    }
   }
 
   /** The VAPID public key (applicationServerKey); generated and persisted on first use. */
-  publicKey(): string {
-    return this.keys().publicKey
+  async publicKey(): Promise<string> {
+    return (await this.keys()).publicKey
   }
 
   /**
@@ -191,17 +370,10 @@ export class Push {
    * in its quiet hours (hand-offs, an explicit request from the terminal, go through both). Returns the
    * number of successful deliveries.
    */
-  async notify(
-    kind: PushKind,
-    sessionID: string,
-    title: string,
-    detail?: PushDetail,
-    body?: string,
-    ownerID?: string,
-    extra?: PushExtra,
-  ): Promise<number> {
-    const precise = pushKindFor(kind, detail)
-    if (!this.cool(`${sessionID}\n${precise}`, precise === "say" ? SAY_COOLDOWN_MS : PUSH_COOLDOWN_MS)) return 0
+  async notify(kind: NotificationKind, sessionID: string, title: string, body?: string, ownerID?: string, extra?: PushExtra): Promise<number> {
+    const precise = kind
+    const key = `${sessionID}\n${precise}${extra?.key ? `\n${extra.key}` : ""}`
+    if (!(await this.cool(key, precise === "say" ? SAY_COOLDOWN_MS : PUSH_COOLDOWN_MS))) return 0
     const owners = new Map(this.store.devices().map((d) => [d.id, d.ownerID]))
     const at = this.now()
     const muted = new Map<string, boolean>()
@@ -219,21 +391,26 @@ export class Push {
       return !isMuted(owner) && !(quiet && inQuietHours(quiet, at))
     })
     if (targets.length === 0) return 0
-    const payload = pushPayload(kind, sessionID, title, detail, body, extra)
+    const payload = pushPayload(kind, sessionID, title, body, extra)
     const { delivered } = await this.send(targets, payload)
     return delivered
   }
 
-  /** Sends a test notification to `deviceID`'s own subscriptions, whatever its prefs; one per TEST_COOLDOWN_MS. */
-  async test(deviceID: string): Promise<PushTestResult> {
+  /**
+   * Sends a test notification to `deviceID`'s own subscriptions, whatever its prefs; one per TEST_COOLDOWN_MS. With
+   * a kind, it looks like that kind (title, urgency) for a "Test session".
+   */
+  async test(deviceID: string, kind?: NotificationKind): Promise<PushTestResult> {
     const key = `test\n${deviceID}`
-    const last = this.lastSent.get(key)
-    if (last && this.now() - last.at < last.cooldown) {
-      return { limited: true, retryAfter: Math.ceil((last.cooldown - (this.now() - last.at)) / 1000) }
+    if (!(await this.cool(key, TEST_COOLDOWN_MS))) {
+      const at = Number(await this.kv.get(COOL_KEY(key)))
+      const left = Number.isFinite(at) ? TEST_COOLDOWN_MS - (this.now() - at) : TEST_COOLDOWN_MS
+      return { limited: true, retryAfter: Math.max(1, Math.ceil(left / 1000)) }
     }
-    this.cool(key, TEST_COOLDOWN_MS)
     const targets = this.store.pushSubscriptions().filter((s) => s.deviceID === deviceID)
-    const payload: PushPayload = { kind: "test", sessionID: "", title: "Hub", body: "Test notification: notifications work on this device." }
+    const payload: PushPayload = kind
+      ? { ...pushPayload(kind, "", "Test session", `Test: this is how “${NOTIFICATION_LABELS[kind]}” looks on this device.`, { subject: TEST_SUBJECT[kind] }), test: true }
+      : { kind: "test", sessionID: "", title: "Hub", body: "Test notification: notifications work on this device." }
     const { delivered, failed } = await this.send(targets, payload)
     return { sent: delivered, failed }
   }
@@ -242,7 +419,7 @@ export class Push {
     targets: readonly { endpoint: string; keys: PushTarget["keys"] }[],
     payload: PushPayload,
   ): Promise<{ delivered: number; failed: Array<{ host: string; status?: number }> }> {
-    const keys = this.keys()
+    const keys = await this.keys()
     const options: PushSendOptions = {
       ...sendOptionsFor(payload.kind),
       timeout: SEND_TIMEOUT_MS,
@@ -259,27 +436,24 @@ export class Push {
       const status = statusOf(result.reason)
       const endpoint = targets[index]!.endpoint
       failed.push({ host: hostOf(endpoint), ...(status !== undefined ? { status } : {}) })
-      if (status === 404 || status === 410) this.store.dropPushSubscription(endpoint)
+      if (status === 404 || status === 410) {
+        void this.store.dropPushSubscription(endpoint).catch((cause: unknown) => console.error(`[hub] push subscription not dropped: ${(cause as Error)?.message ?? cause}`))
+      }
       else console.error(`[hub] push to ${hostOf(endpoint)} failed: ${status ?? (result.reason as Error)?.message}`)
     })
     return { delivered, failed }
   }
 
-  /** True (and records the send) when (session, kind) is outside its cooldown. */
-  private cool(key: string, cooldown: number): boolean {
-    const t = this.now()
-    for (const [k, sent] of this.lastSent) if (t - sent.at >= sent.cooldown) this.lastSent.delete(k)
-    if (this.lastSent.has(key)) return false
-    this.lastSent.set(key, { at: t, cooldown })
-    return true
+  /** True (and records the send, SET NX PX: one pod wins) when `key` is outside its cooldown. */
+  private cool(key: string, cooldown: number): Promise<boolean> {
+    return this.kv.set(COOL_KEY(key), String(this.now()), { px: cooldown, nx: true })
   }
 
-  private keys() {
+  /** The stored keys; the first call generates a pair (when pods race, every pod gets the pair stored first). */
+  private async keys(): Promise<VapidKeys> {
     const existing = this.store.vapid()
     if (existing) return existing
-    const keys = webpush.generateVAPIDKeys()
-    this.store.setVapid(keys)
-    return keys
+    return this.store.setVapid(webpush.generateVAPIDKeys())
   }
 }