@celilo/console-server 0.2.0 → 0.4.0

package/spa/assets/index-eLF_dgP-.css

@@ -0,0 +1 @@
+:root{--ground:#000;--ink:#fff;--dim:#8a8f98;--dimmer:#4a4f57;--rule:#23262b;--raised:#101216;--firing:#ff5f4d;--stale:#f5a623;--unobserved:#4a4f57;--mono:"IBM Plex Mono", ui-monospace, "SF Mono", Menlo, monospace;--sans:Theia, "IBM Plex Sans Condensed", "Helvetica Neue", Arial, sans-serif}*{box-sizing:border-box}html,body,#root{height:100%}body{background:var(--ground);color:var(--ink);font-family:var(--mono);-webkit-font-smoothing:antialiased;margin:0;font-size:13px;line-height:1.45}*,:before,:after{animation:none!important}@media (prefers-reduced-motion:reduce){*{transition:none!important}}:focus-visible{outline:1px solid var(--ink);outline-offset:2px}.eyebrow{font-family:var(--sans);text-transform:uppercase;letter-spacing:.12em;color:var(--dim);font-size:10.5px}.scroll-x{overflow-x:auto}table{border-collapse:collapse;font-variant-numeric:tabular-nums;width:100%}th{font-family:var(--sans);text-transform:uppercase;letter-spacing:.1em;color:var(--dim);text-align:left;border-bottom:1px solid var(--rule);white-space:nowrap;padding:7px 12px;font-size:10.5px;font-weight:600}td{border-bottom:1px solid var(--rule);white-space:nowrap;padding:5px 12px}button{font:inherit;color:inherit;cursor:pointer;background:0 0;border:0;padding:0}

package/spa/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>celilo</title>
+    <script type="module" crossorigin src="/assets/index-CMq3qlYr.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-eLF_dgP-.css">
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

package/src/index.ts

@@ -6,7 +6,14 @@
  * because the e2e suite asserts against their output directly rather than
  * through HTTP, which keeps a shape assertion from also being a routing test.
  */
-export { serve, createHandler, type ServeOptions, type AuthGate, type AuthSubject } from './serve';
+export {
+  serve,
+  createHandler,
+  BUNDLED_SPA_DIR,
+  type ServeOptions,
+  type AuthGate,
+  type AuthSubject,
+} from './serve';
 export { Upstream, UpstreamUnavailable, type UnavailableReason } from './upstream';
 export {
   readTopology,

package/src/main.ts

@@ -0,0 +1,127 @@
+/**
+ * The console server as a process: read the environment, build the gate, listen.
+ *
+ * `serve()` takes an `AuthGate` and has no default, on purpose — a server
+ * constructed without one would answer to anybody. This file is where the real
+ * gate is chosen for a deployed console, and it is the only place that reads
+ * `process.env`. Tests drive `serve()` with a stub instead, which is why the
+ * seam exists at all.
+ *
+ * `bun build --compile` targets THIS file. `index.ts` stays a pure export
+ * surface so importing the package never binds a port.
+ */
+
+import { createRemoteJWKSet, customFetch, jwtVerify } from 'jose';
+import { type AuthGate, type AuthSubject, serve } from './serve';
+
+/**
+ * Where `scripts/enrolment.ts` puts the key it generated on this system.
+ *
+ * Spelled out rather than defaulted to something plausible: the hook writes
+ * exactly this path, and a console pointed at any other one cannot log in.
+ */
+const DEFAULT_IDENTITY_FILE = '/etc/celilo-web-console/id_ed25519';
+
+/** Matches `wellspring`'s `DEFAULT_ENDPOINT`, which is the fleet's convention. */
+const DEFAULT_DEST = 'celilo-api@celilo-mgr';
+
+export interface OidcGateOptions {
+  /** The `iss` claim exactly — for authentik, `https://auth.<domain>/application/o/<slug>/`. */
+  issuer: string;
+  /** Also the expected audience: authentik puts the client id in `aud`. */
+  clientId: string;
+  /** Injectable so the gate's test does not need a live provider. */
+  fetchImpl?: typeof fetch;
+}
+
+/**
+ * Verify the bearer against the provider's JWKS (D8).
+ *
+ * Discovery rather than string surgery on the issuer: authentik serves its
+ * `jwks_uri` one level above the per-application issuer, so deriving it by
+ * appending a path works on some providers and silently fails on this one.
+ *
+ * Discovery is lazy and memoized rather than done at startup. Doing it eagerly
+ * would make the console refuse to boot whenever authentik is merely slow,
+ * which turns a transient dependency outage into a dead service.
+ */
+export function oidcGate(options: OidcGateOptions): AuthGate {
+  const doFetch = options.fetchImpl ?? fetch;
+  let keys: ReturnType<typeof createRemoteJWKSet> | undefined;
+
+  const jwks = async () => {
+    if (keys !== undefined) return keys;
+    const url = new URL('.well-known/openid-configuration', options.issuer);
+    const response = await doFetch(url);
+    if (!response.ok) {
+      throw new Error(`OIDC discovery at ${url} returned ${response.status}`);
+    }
+    const { jwks_uri } = (await response.json()) as { jwks_uri?: string };
+    if (typeof jwks_uri !== 'string') {
+      throw new Error(`OIDC discovery at ${url} carried no jwks_uri`);
+    }
+    // The same fetch as discovery, so one injection point covers both hops and
+    // this gate's test never reaches the network.
+    keys = createRemoteJWKSet(new URL(jwks_uri), {
+      [customFetch]: (target, init) => doFetch(target, init),
+    });
+    return keys;
+  };
+
+  return async (request: Request): Promise<AuthSubject | null> => {
+    const header = request.headers.get('authorization');
+    const token = header?.startsWith('Bearer ') ? header.slice('Bearer '.length) : undefined;
+    if (token === undefined || token.length === 0) return null;
+
+    try {
+      const { payload } = await jwtVerify(token, await jwks(), {
+        issuer: options.issuer,
+        audience: options.clientId,
+      });
+      // `sub` is the stable id; the name is what `readSession` matches a
+      // `people` row against, so prefer the username a person would recognise.
+      const name = [payload.preferred_username, payload.name, payload.email].find(
+        (value): value is string => typeof value === 'string' && value.length > 0,
+      );
+      return { subject: String(payload.sub), ...(name === undefined ? {} : { name }) };
+    } catch {
+      // A bad token is a refusal, not a 500. The reason is deliberately not
+      // reported to the caller: "expired" versus "wrong audience" tells an
+      // unauthenticated prober how the gate is configured.
+      return null;
+    }
+  };
+}
+
+/** Read one required value, or say which one is missing and who writes it. */
+function required(env: Record<string, string | undefined>, name: string): string {
+  const value = env[name];
+  if (value === undefined || value === '') {
+    throw new Error(
+      `${name} is not set. The console refuses to serve without an identity provider — see /etc/celilo-web-console-discovered.env, which the module's install hook writes once the OIDC client exists.`,
+    );
+  }
+  return value;
+}
+
+export function startFromEnv(env: Record<string, string | undefined> = process.env): {
+  port: number;
+  stop: () => void;
+} {
+  const server = serve({
+    port: Number(env.PORT ?? 8443),
+    dest: env.CELILO_API_DEST ?? DEFAULT_DEST,
+    identityFile: env.CELILO_IDENTITY_FILE ?? DEFAULT_IDENTITY_FILE,
+    ...(env.CONSOLE_SPA_DIR ? { spaDir: env.CONSOLE_SPA_DIR } : {}),
+    authenticate: oidcGate({
+      issuer: required(env, 'OIDC_ISSUER'),
+      clientId: required(env, 'OIDC_CLIENT_ID'),
+    }),
+  });
+  console.log(`[console] listening on :${server.port}`);
+  return server;
+}
+
+if (import.meta.main) {
+  startFromEnv();
+}

package/src/serve.ts

@@ -3,8 +3,8 @@
  *
  * Holds no database handle. Every fact it serves comes from `Upstream`, which
  * reaches celilo-mgr over the SSH remote API as a principal granted read ops
- * only. That is a security boundary, not a deployment preference: a console
- * sharing celilo's process would hold the whole database including the
+ * plus `alerts:ack`. That is a security boundary, not a deployment preference: a
+ * console sharing celilo's process would hold the whole database including the
  * encrypted secret store, and would be constrained only by its own code.
  *
  * It is a Bun server rather than tsrpc's own `HttpServer` for one reason worth
@@ -15,23 +15,36 @@
  * get wrong. So the RPC calls are dispatched by hand onto the same handlers.
  */
 
+import { join } from 'node:path';
 import type { ServiceType } from '@celilo/console-protocol';
 import { Upstream, type UpstreamOptions, UpstreamUnavailable } from './upstream';
 import {
+  ackAlert,
   readAlerts,
   readBackupsWithCoverage,
   readClosure,
   readModules,
+  readSession,
   readTopology,
 } from './verbs';
 
+/**
+ * Where `prepack` builds the SPA: a sibling of `src/` under the package root.
+ * The same relative path resolves in the repo and inside the npm tarball, which
+ * is why the build writes here rather than being copied from `apps/console/dist`.
+ */
+export const BUNDLED_SPA_DIR = join(import.meta.dir, '..', 'spa');
+
 export interface ServeOptions {
   port: number;
   /** `celilo-api@<server>`. */
   dest: string;
   identityFile: string;
-  /** Directory holding the built SPA. */
-  spaDir: string;
+  /**
+   * Directory holding the built SPA. Defaults to the copy `prepack` builds into
+   * the package, which is the only one that exists on a deployed system.
+   */
+  spaDir?: string;
   /** Decides who is asking. See `AuthGate`. */
   authenticate: AuthGate;
   cacheTtlMs?: number;
@@ -66,7 +79,25 @@ export interface AuthSubject {
 /** Every RPC path the protocol declares, and nothing else. */
 type ApiName = keyof ServiceType['api'];
 
-const READ_VERBS: ApiName[] = ['Topology', 'Modules', 'Closure', 'Alerts', 'Backups'];
+/**
+ * The whole surface. Five reads, one identity read, and one write.
+ *
+ * `AckAlert` is the write, and it is here rather than absent because D8 argues
+ * for exactly one: an acknowledgement records that a named person has seen an
+ * alert, and the identity it records is the reason the console requires `idp`
+ * at all. Nothing else mutates. A deploy, a redeploy, a pause or a config edit
+ * can raise an interview question that a browser has no responder for, so those
+ * are not disabled here — they do not exist.
+ */
+const SERVED: ApiName[] = [
+  'Topology',
+  'Modules',
+  'Closure',
+  'Alerts',
+  'Backups',
+  'Session',
+  'AckAlert',
+];
 
 /**
  * Map an upstream failure onto an HTTP status and a tsrpc-shaped error.
@@ -108,17 +139,21 @@ function contentType(path: string): string {
 /**
  * Dispatch one RPC call.
  *
- * The request body is the tsrpc request object. Only the five read verbs are
- * routed. `AckAlert` is declared in the protocol and deliberately NOT wired
- * here: it is a write, it needs an authenticated subject that maps to a real
- * `people` row, and the grant to perform it does not exist yet. An unrouted
- * verb 404s, which is honest. Wiring it to something that silently no-ops
- * would be worse than not having it.
+ * The request body is the tsrpc request object. `subject` is whoever the gate
+ * let in, and it arrives here rather than being looked up per handler because
+ * one verb — `AckAlert` — is only allowed to run for a subject celilo has a
+ * `people` row for.
+ *
+ * That check happens HERE, before any command is issued, and the person's name
+ * comes from `readSession` rather than from the request. A browser cannot name
+ * the acknowledger: the protocol has no actor field, and this is the code that
+ * makes that structural rather than merely intended.
  */
 async function callApi(
   name: ApiName,
   body: Record<string, unknown>,
   upstream: Upstream,
+  subject: AuthSubject,
 ): Promise<Response> {
   switch (name) {
     case 'Topology':
@@ -142,6 +177,41 @@ async function callApi(
     }
     case 'Backups':
       return Response.json({ isSucc: true, res: await readBackupsWithCoverage(upstream) });
+    case 'Session':
+      return Response.json({ isSucc: true, res: await readSession(upstream, subject) });
+    case 'AckAlert': {
+      const alertKey = body.alertKey;
+      if (typeof alertKey !== 'string' || alertKey.length === 0) {
+        return Response.json(
+          { isSucc: false, err: { message: 'alertKey is required', type: 'ApiError' } },
+          { status: 400 },
+        );
+      }
+
+      const session = await readSession(upstream, subject);
+      if (session.person === null) {
+        // 403 rather than 400: the request is well formed and the caller is
+        // authenticated. What is missing is a `people` row for them, and the
+        // message says so, because the fix is `celilo person add` and nothing
+        // about a generic denial would point an operator at it.
+        return Response.json(
+          {
+            isSucc: false,
+            err: {
+              message: `No person in this fleet matches "${session.name ?? session.subject}". An acknowledgement records WHO saw the alert, so the console will not make one it cannot attribute. Add them with \`celilo person add\`.`,
+              type: 'ApiError',
+              code: 'unmapped-subject',
+            },
+          },
+          { status: 403 },
+        );
+      }
+
+      return Response.json({
+        isSucc: true,
+        res: await ackAlert(upstream, alertKey, session.person),
+      });
+    }
     default:
       return Response.json(
         { isSucc: false, err: { message: `${name} is not served here`, type: 'ApiError' } },
@@ -155,6 +225,7 @@ async function callApi(
  * drive it with a `Request` and no listening socket.
  */
 export function createHandler(options: ServeOptions): (request: Request) => Promise<Response> {
+  const spaDir = options.spaDir ?? BUNDLED_SPA_DIR;
   const upstream = new Upstream({
     dest: options.dest,
     identityFile: options.identityFile,
@@ -178,7 +249,7 @@ export function createHandler(options: ServeOptions): (request: Request) => Prom
 
     if (url.pathname.startsWith('/api/')) {
       const name = url.pathname.slice('/api/'.length) as ApiName;
-      if (!READ_VERBS.includes(name)) {
+      if (!SERVED.includes(name)) {
         return Response.json(
           { isSucc: false, err: { message: `Unknown verb: ${name}`, type: 'ApiError' } },
           { status: 404 },
@@ -195,7 +266,7 @@ export function createHandler(options: ServeOptions): (request: Request) => Prom
       }
 
       try {
-        return await callApi(name, body, upstream);
+        return await callApi(name, body, upstream, subject);
       } catch (error) {
         if (error instanceof UpstreamUnavailable) return unavailableResponse(error);
         throw error;
@@ -205,18 +276,18 @@ export function createHandler(options: ServeOptions): (request: Request) => Prom
     // Static, then the SPA catch-all. The order matters: a request for
     // `/assets/index.js` must find the file, and a request for `/backups` must
     // find `index.html`, because the route belongs to the client-side router.
-    const asset = Bun.file(`${options.spaDir}${url.pathname}`);
+    const asset = Bun.file(`${spaDir}${url.pathname}`);
     if (url.pathname !== '/' && (await asset.exists())) {
       return new Response(asset, { headers: { 'content-type': contentType(url.pathname) } });
     }
 
-    const shell = Bun.file(`${options.spaDir}/index.html`);
+    const shell = Bun.file(`${spaDir}/index.html`);
     if (!(await shell.exists())) {
       // Said out loud rather than 404ing. A console server with no SPA built
       // into it is a packaging failure, and a bare 404 sends the reader looking
       // for a routing bug instead.
       return new Response(
-        `No SPA found at ${options.spaDir}. The console server ships the built client inside its package, and this one was started without it.`,
+        `No SPA found at ${spaDir}. The console server ships the built client inside its package, and this one was started without it.`,
         { status: 500 },
       );
     }

package/src/upstream.ts

@@ -2,11 +2,16 @@
  * How the console reaches celilo.
  *
  * The console server holds NO database handle. Every fact it serves arrives over
- * the SSH remote API as a principal whose grants are read-only, the same way
- * `@celilo/mcp` reaches celilo-mgr. That is a security boundary rather than a
- * deployment preference: a console sharing celilo's process would hold the whole
- * database including the encrypted secret store, and would be constrained only
- * by its own code.
+ * the SSH remote API, the same way `@celilo/mcp` reaches celilo-mgr. That is a
+ * security boundary rather than a deployment preference: a console sharing
+ * celilo's process would hold the whole database including the encrypted secret
+ * store, and would be constrained only by its own code.
+ *
+ * Its principal is granted the read ops and `alerts:ack`, and nothing else. The
+ * read ops are DERIVED from celilo's command registry rather than listed, so a
+ * new write verb is never granted however it is named; `alerts:ack` is the one
+ * explicit exception, argued for in D8 and useless without an identity to
+ * attribute the acknowledgement to.
  *
  * Two policies here differ deliberately from the MCP's client.
  *
@@ -92,12 +97,39 @@ export class Upstream {
     return value;
   }
 
-  /** Drop everything cached. Used when the operator asks for a hard refresh. */
+  /**
+   * Run one command that is NOT a read, and return what it said.
+   *
+   * Uncached and unparsed. The console's only write is `alerts ack`, which
+   * answers with a sentence rather than JSON, and a write must never be served
+   * from a cache — the whole point of issuing it is that it changes something.
+   *
+   * Failures classify exactly as a read's do, so a console whose principal was
+   * never granted `alerts:ack` renders a denial rather than a fault.
+   */
+  async run(argv: readonly string[]): Promise<string> {
+    return this.runText(argv);
+  }
+
+  /** Drop everything cached. Used after a write, and on a hard refresh. */
   invalidate(): void {
     this.cache.clear();
   }
 
   private async runJson<T>(argv: readonly string[]): Promise<T> {
+    const output = await this.runText(argv);
+    try {
+      return JSON.parse(output) as T;
+    } catch {
+      throw new UpstreamUnavailable(
+        'malformed',
+        argv,
+        `\`${argv.join(' ')}\` succeeded but did not return JSON.`,
+      );
+    }
+  }
+
+  private async runText(argv: readonly string[]): Promise<string> {
     const lines: string[] = [];
     const out: DisplayWriter = {
       write: (s: string) => {
@@ -145,15 +177,7 @@ export class Upstream {
       throw new UpstreamUnavailable(classifyFailure(outcome.exitCode, output), argv, output);
     }
 
-    try {
-      return JSON.parse(output) as T;
-    } catch {
-      throw new UpstreamUnavailable(
-        'malformed',
-        argv,
-        `\`${argv.join(' ')}\` succeeded but did not return JSON.`,
-      );
-    }
+    return output;
   }
 }
 

package/src/verbs.ts

@@ -14,9 +14,11 @@
  * the console draws that differently from "nothing is wrong". Returning `[]` for
  * a read that did not happen is the bug this console was specified around.
  *
- * Nothing here writes. The console's API principal is granted read ops only, so
- * a mutating verb added to this file would be refused at the API boundary rather
- * than running — but it would still be a lie in the source, so it stays absent.
+ * Exactly ONE thing here writes, and `ackAlert` is it. Everything else is a
+ * read, and the console's API principal is granted read ops only — so a second
+ * mutating verb added to this file would be refused at the API boundary rather
+ * than running. Ack is the one write D8 argues for, and it is the reason the
+ * console needs an identity at all.
  */
 
 import type {
@@ -25,13 +27,15 @@ import type {
   BackupRunProjection,
   ClosureNode,
   ModuleProjection,
+  ResAckAlert,
   ResAlerts,
   ResBackups,
   ResClosure,
   ResModules,
+  ResSession,
   ResTopology,
 } from '@celilo/console-protocol';
-import type { Upstream } from './upstream';
+import { type Upstream, UpstreamUnavailable } from './upstream';
 
 /**
  * How far back the backups read asks for.
@@ -64,15 +68,15 @@ interface CliConsoleStatus {
 /**
  * `celilo console get <id> --json`, verbatim.
  *
- * Note what is NOT here: delegation chains. D6 establishes that the firewall
- * chain is derivable (a provider's capability data names the zone it delegates
- * to) and that the derivation must be verified against a live
- * `celilo capability info firewall` before anything is built on it. That
- * verification has not happened and `computeClosure` does not produce chains,
- * so this server has none to serve.
+ * `chains` is OPTIONAL here and required in the protocol, and the gap between
+ * them is a celilo-mgr old enough not to compute them. That server answers the
+ * verb successfully and simply omits the field, so the type has to admit it
+ * (D7's older-upstream rule). `readClosure` turns the absence into an empty
+ * list, which under-draws the picture rather than misdrawing it.
  */
 interface CliClosure {
   nodes: { moduleId: string; hop: number; optional: boolean; via: string[] }[];
+  chains?: { capability: string; moduleIds: string[] }[];
   depth: number;
   truncated: boolean;
 }
@@ -98,6 +102,12 @@ interface CliAlert {
   message: string;
 }
 
+/** `celilo person list --json`, verbatim. */
+interface CliPerson {
+  name: string;
+  timezone: string;
+}
+
 /** `celilo backup list --json`, verbatim. */
 interface CliBackups {
   asOf: number;
@@ -207,12 +217,11 @@ export async function readClosure(
 
   return {
     nodes,
-    // EMPTY BECAUSE THERE IS NO PRODUCER, not because this fleet has no
-    // delegation. On the live fleet `iptables` delegates upstream to `axon`, so
-    // an empty list here is wrong rather than merely unpopulated. It is left
-    // empty rather than faked, and no console surface renders it today — the
-    // moment one does, it must say "not computed" rather than "none".
-    chains: [],
+    // Served as celilo computed them. A chain naming a module the roster does
+    // not have is kept rather than dropped, unlike a node: the chain is a claim
+    // about the packet's path, and silently shortening it would say the packet
+    // stops somewhere it does not.
+    chains: closure.chains ?? [],
     depth: closure.depth,
     truncated: closure.truncated,
   };
@@ -377,3 +386,86 @@ export async function readBackupsWithCoverage(upstream: Upstream): Promise<ResBa
 
   return { ...backups, backups: [...withStaleness, ...unhooked] };
 }
+
+/**
+ * Who is looking, and whether celilo has a `people` row for them.
+ *
+ * The mapping is by NAME. celilo's `people.name` is a kebab-case handle an
+ * operator chose (`peter`), and the claim that carries the same thing from an
+ * identity provider is the display name or preferred username. The opaque `sub`
+ * is tried second, for a provider configured to put the handle there.
+ *
+ * Compared case-insensitively, because a provider that answers `Peter` to a
+ * fleet that spells it `peter` is not a different person, and the alternative
+ * is an operator staring at a console that will not let them acknowledge
+ * anything with nothing on screen saying why.
+ *
+ * A subject that maps to nobody yields `person: null`, which is an answer. The
+ * console draws no ack control for it. What must NEVER happen is this function
+ * guessing: `celilo alerts ack` falls back to `people[0]` when no one is named,
+ * so a guess here would silently attribute an acknowledgement to a real person
+ * who did not make it.
+ */
+export async function readSession(
+  upstream: Upstream,
+  identity: { subject: string; name?: string },
+): Promise<ResSession> {
+  const people = await upstream.read<CliPerson[]>(['person', 'list', '--json']);
+  const byName = new Map(people.map((p) => [p.name.toLowerCase(), p.name]));
+
+  const candidates = [identity.name, identity.subject].filter(
+    (value): value is string => typeof value === 'string' && value.length > 0,
+  );
+  const matched = candidates.map((c) => byName.get(c.toLowerCase())).find((n) => n !== undefined);
+
+  return {
+    subject: identity.subject,
+    name: identity.name ?? null,
+    person: matched ?? null,
+  };
+}
+
+/**
+ * Acknowledge one alert AS a named person.
+ *
+ * `person` is not optional and there is no path through this function that
+ * omits `--as`. That is the whole requirement: `handleAlertsAck` falls back to
+ * `people[0]` when the flag is absent, which is reasonable for a CLI on a
+ * one-operator fleet and a lie in a browser. Making the parameter required
+ * means the fallback is unreachable from here by construction rather than by
+ * everyone remembering to pass it.
+ *
+ * The caller is responsible for having resolved `person` from an authenticated
+ * subject via `readSession`. This function will not accept a name a browser
+ * supplied, because it never sees one — the protocol has no actor field.
+ *
+ * `ackedAt` is re-read from celilo rather than stamped here. The console server
+ * and celilo-mgr are different machines, and a console that reported its own
+ * clock would be the alert-age bug again one layer down.
+ */
+export async function ackAlert(
+  upstream: Upstream,
+  alertKey: string,
+  person: string,
+): Promise<ResAckAlert> {
+  await upstream.run(['alerts', 'ack', alertKey, '--as', person]);
+
+  // The write landed, so every cached read is now wrong. Dropped before the
+  // re-read below, which would otherwise be served the pre-ack list.
+  upstream.invalidate();
+
+  const rows = await upstream.read<CliAlert[]>(['alerts', 'list', '--json']);
+  const acked = rows.find((row) => row.key === alertKey);
+  if (!acked || acked.ackedAt === null) {
+    // celilo said it acknowledged and its own list disagrees. Reported rather
+    // than papered over with `Date.now()`: a console that invents the timestamp
+    // would render a successful ack for a write that did not take.
+    throw new UpstreamUnavailable(
+      'malformed',
+      ['alerts', 'ack', alertKey],
+      `celilo acknowledged ${alertKey} but does not report it as acknowledged.`,
+    );
+  }
+
+  return { ackedAt: acked.ackedAt, ackedByName: acked.ackedBy ?? person };
+}