@yourrealm/sdk 0.0.0-alpha.1378.ce03c8dc

package/src/worker/uplink.ts

@@ -0,0 +1,523 @@
+// The worker's client loop: one session at a time, dialled by the worker and
+// never by Home. Everything the two say to each other rides it.
+//
+// The loop is deliberately dumb — dial, hello, announce, pull, keep alive,
+// dial again — with exactly three ways to stop for good, each of them a thing
+// an operator has to fix: a Home whose static key is not the pinned one, an
+// enrollment token Home refuses, and a worker whose row is gone with no token
+// to re-arm it. Everything else is a reconnect.
+//
+// A Home on another `PROTOCOL_VERSION` is not one of the three. The hello
+// succeeded, so the session is real and Home can send `update` over it; the
+// loop simply holds it without announcing, and this worker stays offline to
+// everything else until the box has been moved.
+
+import { log } from "@local/logger";
+import type { Keypair } from "@yourrealm/noise";
+import type { WorkerCapability } from "../permissions.ts";
+import { readNodeVersion, shortNodeVersion } from "../worker-protocol/index.ts";
+import {
+  type AppApplyResult,
+  BACKOFF_BASE_MS,
+  BACKOFF_CAP_MS,
+  callJson,
+  connectUplink,
+  HomeKeyMismatch,
+  MethodError,
+  type Params,
+  PING_INTERVAL_MS,
+  PONG_TIMEOUT_MS,
+  PROTOCOL_VERSION,
+  pullState,
+  REKEY_INTERVAL_MS,
+  REKEY_JITTER_MS,
+  type Result,
+  type RpcHandler,
+  type StatePull,
+  type UpdateStatus,
+  type UplinkSession,
+} from "../worker-protocol/uplink/index.ts";
+import type { Enrollment } from "./identity.ts";
+import { currentPublicIp, setPublicIpEcho } from "./public-ip.ts";
+
+/**
+ * Every uplink call rejects with this while there is no live session, rather
+ * than queueing behind a reconnect: the verify relay turns it into its 502 and
+ * the dashboard shows the rest as failures (`todo/ideas/uplink.md` § 4 RPC).
+ *
+ * `reason` is what the caller gets told instead of "the uplink to Home is
+ * down", for the case where that sentence would be a lie: a session held on a
+ * protocol mismatch is up and talking, and the 502 should say so.
+ */
+export class WorkerOffline extends Error {
+  constructor(method: string, reason = "the uplink to Home is down") {
+    super(`${reason}; ${method} cannot be answered`);
+    this.name = "WorkerOffline";
+  }
+}
+
+/**
+ * The three lines `worker.sh` greps this worker's container log for while it
+ * waits out an install: one for a session that opened, and the two failures an
+ * operator can act on. They live here, next to the `log` calls that emit them,
+ * so the installer reads the same strings the loop prints and changing one is a
+ * type error rather than an install that parks for four minutes.
+ */
+export const UPLINK_LOG = {
+  open: "Uplink open as",
+  tokenRefused: "Home refused this enrollment token",
+  keyMismatch: "Uplink key mismatch",
+} as const;
+
+/** The cadences the loop runs on. Defaulted from the protocol's constants. */
+export interface UplinkTiming {
+  pingIntervalMs: number;
+  pongTimeoutMs: number;
+  rekeyIntervalMs: number;
+  rekeyJitterMs: number;
+  backoffBaseMs: number;
+  backoffCapMs: number;
+}
+
+const DEFAULT_TIMING: UplinkTiming = {
+  pingIntervalMs: PING_INTERVAL_MS,
+  pongTimeoutMs: PONG_TIMEOUT_MS,
+  rekeyIntervalMs: REKEY_INTERVAL_MS,
+  rekeyJitterMs: REKEY_JITTER_MS,
+  backoffBaseMs: BACKOFF_BASE_MS,
+  backoffCapMs: BACKOFF_CAP_MS,
+};
+
+interface UplinkOptions {
+  /** `wss://{dashboardHost}/uplink`, or `ws://realm-home:3100/uplink`. */
+  url: string;
+  identity: Keypair;
+  /** The pin. A responder that answers with anything else is refused. */
+  homePublicKey: Uint8Array;
+  /** The OTP half of `REALM_TOKEN`, when there is one. */
+  otp: string | null;
+  /** Display name for a first enroll; the installer defaults it to the hostname. */
+  name: string | null;
+  /** What a previous boot enrolled as, or null. */
+  enrolled: Enrollment | null;
+  capabilities: () => Promise<WorkerCapability[]>;
+  /**
+   * What this box's ingress container is running, or null where it has none.
+   * Read on every announce for the same reason `capabilities` is: it is a fact
+   * about the box the worker can only observe, and a self-update moves it
+   * without the worker restarting.
+   */
+  ingressImage: () => Promise<{ imageRef: string } | null>;
+  /**
+   * What the box's updater-driven projects are doing, read on every announce
+   * for the same reason `ingressImage` is: the files move under a live
+   * session (the updater writes them), and `announce` is where Home resets its
+   * whole picture of this box.
+   */
+  updates: () => Promise<UpdateStatus[]>;
+  /** The Home → worker methods. */
+  handlers: Record<string, RpcHandler>;
+  /** The etag this worker is actually running, for `announce`. */
+  appliedEtag: () => string | null;
+  /** Runs once per connect, after the announce: connect is pull. */
+  onConnected: () => void;
+  /** Persists what `enroll` handed back. */
+  onEnrolled: (enrollment: Enrollment) => Promise<void>;
+  timing?: Partial<UplinkTiming>;
+  /** Test seam: the dial. */
+  connect?: typeof connectUplink;
+}
+
+export interface UplinkHandle {
+  /** True while a session is open and announced. */
+  readonly online: boolean;
+  /** Resolves with the reason once the loop has given up for good. */
+  readonly stopped: Promise<string>;
+  verify(params: Params<"verify">): Promise<Result<"verify">>;
+  apiAccess(params: Params<"apiAccess">): Promise<Result<"apiAccess">>;
+  pull(params: Params<"state.pull">): Promise<StatePull>;
+  applied(etag: string, apps: AppApplyResult[]): Promise<void>;
+  /** `update.status` for one project; rejects with `WorkerOffline` like the rest. */
+  updateStatus(status: UpdateStatus): Promise<void>;
+  /**
+   * Drop the live session so the loop redials. A no-op when none is open.
+   *
+   * Not `close()`, which stops the loop for good. This exists for the one class
+   * of change nothing else carries: a fact about the box that `announce`
+   * reports and that moves while the session is up. The box ingress is that
+   * fact - the updater sidecar replaces its container without restarting this
+   * process - and the announce only ever runs on connect, so without a redial
+   * Home holds the old image ref until the 24h rekey.
+   *
+   * A redial is the whole repair rather than half of it, because connect is
+   * pull: the fresh `announce` carries the moved fact AND `onConnected` re-runs
+   * the idempotent reconcilers, which is what re-attaches a replaced ingress to
+   * every routed app's network. It costs one sub-second gap (the backoff is
+   * re-armed from zero after a successful hello).
+   *
+   * A held session is dropped too: it is a session, and the redial is the only
+   * way one ever stops being held.
+   */
+  reconnect(reason: string): void;
+  /** Stops the loop and closes the session. Resolves once both are done. */
+  close(): Promise<void>;
+}
+
+/**
+ * What the announce decided: keep this session, hold it unannounced, redial,
+ * or stop for good.
+ */
+type Announced = "open" | "held" | "retry" | { stop: string };
+
+export function startUplink(opts: UplinkOptions): UplinkHandle {
+  const timing = { ...DEFAULT_TIMING, ...opts.timing };
+  const dial = opts.connect ?? connectUplink;
+
+  /** The announced session every helper rides. */
+  let session: UplinkSession | null = null;
+  /**
+   * A session whose hello Home answered on another `PROTOCOL_VERSION`. Kept
+   * open so Home can send `update` over it, and used for nothing else; `reason`
+   * is what every helper rejects with meanwhile.
+   */
+  let held: { live: UplinkSession; reason: string } | null = null;
+  /**
+   * Whatever the loop currently holds, announced or not, so `close()` can end
+   * it and `keepPinging` can tell "still mine" from "replaced".
+   */
+  let dialled: UplinkSession | null = null;
+  let enrolled = opts.enrolled;
+  let closing = false;
+  /** Wakes the backoff sleep early so `close()` never waits it out. */
+  let wake: (() => void) | null = null;
+
+  let settle: (reason: string) => void = () => {};
+  const stopped = new Promise<string>((resolve) => {
+    settle = resolve;
+  });
+
+  function requireSession(method: string): UplinkSession {
+    if (session === null) throw new WorkerOffline(method, held?.reason);
+    return session;
+  }
+
+  function sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => {
+      const id = setTimeout(() => {
+        wake = null;
+        resolve();
+      }, ms);
+      wake = () => {
+        clearTimeout(id);
+        wake = null;
+        resolve();
+      };
+    });
+  }
+
+  // Full jitter, doubling from the base to the cap. `attempt` is reset by
+  // every session that announced, so a flapping Home is retried hard and a
+  // gone one is retried slowly.
+  function backoff(attempt: number): number {
+    const ceiling = Math.min(timing.backoffCapMs, timing.backoffBaseMs * 2 ** (attempt - 1));
+    return Math.random() * ceiling;
+  }
+
+  async function readCapabilities(): Promise<WorkerCapability[]> {
+    // A bad probe must not brick boot: the worker announces nothing rather
+    // than refusing to announce, and apps needing a host kind land nowhere
+    // until the operator fixes the probe and restarts.
+    try {
+      return await opts.capabilities();
+    } catch (err) {
+      log.warn(`capabilities probe threw (${message(err)}); announcing []`);
+      return [];
+    }
+  }
+
+  async function readIngressImage(): Promise<{ imageRef: string } | null> {
+    // Same reasoning as the capabilities probe, and the same answer: a docker
+    // socket that will not answer must cost this box its version line, not its
+    // session. Null is already the shape for "no ingress here", so a failure
+    // reads as one - which is why the log line is what tells them apart.
+    try {
+      return await opts.ingressImage();
+    } catch (err) {
+      log.warn(`ingress image lookup threw (${message(err)}); announcing none`);
+      return null;
+    }
+  }
+
+  async function readUpdates(): Promise<UpdateStatus[]> {
+    // Same again. `[]` is the shape for a box with no `etc/` tree, so a read
+    // that threw reads as one, and the log line is what tells them apart.
+    try {
+      return await opts.updates();
+    } catch (err) {
+      log.warn(`update status read threw (${message(err)}); announcing []`);
+      return [];
+    }
+  }
+
+  async function enroll(
+    live: UplinkSession,
+    capabilities: WorkerCapability[],
+    publicIp: string | undefined,
+  ): Promise<Announced> {
+    const otp = opts.otp;
+    if (otp === null) {
+      return {
+        stop:
+          "this worker has not enrolled and REALM_TOKEN is unset, so there is nothing to " +
+          "enroll with. Add a worker in the dashboard and restart this box with the token " +
+          "it hands you.",
+      };
+    }
+    try {
+      const answer = await callJson(live.rpc, "enroll", {
+        otp,
+        ...(opts.name === null ? {} : { name: opts.name }),
+        capabilities,
+        version: readNodeVersion(),
+        ingress: await readIngressImage(),
+        ...(publicIp === undefined ? {} : { publicIp }),
+      });
+      enrolled = { workerId: answer.workerId, slug: answer.slug };
+      await opts.onEnrolled(enrolled);
+      setPublicIpEcho(answer.publicIpUrl);
+      log(`${UPLINK_LOG.open} ${answer.slug} (enrolled as ${answer.workerId})`);
+      return "open";
+    } catch (err) {
+      // A burnt, expired or wrong OTP is not retried: every retry would be
+      // the same refusal, and the fix is a fresh token from the dashboard.
+      if (err instanceof MethodError && (err.status === 404 || err.status === 410)) {
+        return { stop: `${UPLINK_LOG.tokenRefused} (${err.status}): ${err.body}` };
+      }
+      log.warn(`enroll failed (${message(err)})`);
+      return "retry";
+    }
+  }
+
+  async function announce(live: UplinkSession): Promise<Announced> {
+    const capabilities = await readCapabilities();
+    const publicIp = (await currentPublicIp()) ?? undefined;
+    if (enrolled === null) return await enroll(live, capabilities, publicIp);
+
+    const version = readNodeVersion();
+    let answer: Result<"hello">;
+    try {
+      answer = await callJson(live.rpc, "hello", { protocol: PROTOCOL_VERSION, version });
+    } catch (err) {
+      if (err instanceof MethodError && err.status === 404) {
+        // The row is gone. A token in the environment is how an operator
+        // re-arms a revoked box: edit the .env, restart, enroll afresh.
+        if (opts.otp === null) {
+          return {
+            stop:
+              "Home has no worker with this box's key. It was removed, or this box was " +
+              "restored from another realm's backup. Add a worker in the dashboard and " +
+              "restart this box with the token it hands you.",
+          };
+        }
+        log.warn("Home does not know this worker; enrolling with the token in REALM_TOKEN");
+        return await enroll(live, capabilities, publicIp);
+      }
+      log.warn(`hello failed (${message(err)})`);
+      return "retry";
+    }
+    setPublicIpEcho(answer.publicIpUrl);
+
+    if (answer.protocol !== PROTOCOL_VERSION) {
+      // Nothing this worker says past the door would be understood, so it says
+      // nothing. The session stays up because it is the only thing that can
+      // carry the `update` that fixes this.
+      log.warn(
+        `Home speaks uplink protocol ${answer.protocol} and this worker speaks ` +
+          `${PROTOCOL_VERSION}; holding the session and waiting for Home to update this box`,
+      );
+      held = {
+        live,
+        reason:
+          `Home speaks uplink protocol ${answer.protocol} and this worker speaks ` +
+          `${PROTOCOL_VERSION}; waiting to be updated`,
+      };
+      return "held";
+    }
+
+    try {
+      await callJson(live.rpc, "announce", {
+        capabilities,
+        ingress: await readIngressImage(),
+        appliedEtag: opts.appliedEtag(),
+        updates: await readUpdates(),
+        ...(publicIp === undefined ? {} : { publicIp }),
+      });
+    } catch (err) {
+      log.warn(`announce failed (${message(err)})`);
+      return "retry";
+    }
+    const label = shortNodeVersion(version);
+    log(`${UPLINK_LOG.open} ${answer.slug}${label === null ? "" : ` (version ${label})`}`);
+    return "open";
+  }
+
+  /**
+   * Pings on the cadence and closes the session when a pong is late. Keyed on
+   * the session the loop holds rather than on the announced one, because a held
+   * session has to stay alive to receive its `update`.
+   */
+  function keepPinging(live: UplinkSession): void {
+    void (async () => {
+      // One continuation on `closed`, not one per tick: the wait below is cut
+      // short through `wakeTick` so teardown never sits out a whole interval.
+      let ended = false;
+      let wakeTick: (() => void) | null = null;
+      void live.closed.then(() => {
+        ended = true;
+        wakeTick?.();
+      });
+      while (dialled === live && !closing && !ended) {
+        await new Promise<void>((resolve) => {
+          const id = setTimeout(resolve, timing.pingIntervalMs);
+          wakeTick = () => {
+            clearTimeout(id);
+            resolve();
+          };
+        });
+        wakeTick = null;
+        if (dialled !== live || closing || ended) return;
+        try {
+          await withDeadline(live.ping(), timing.pongTimeoutMs);
+        } catch {
+          live.close(`no pong inside ${timing.pongTimeoutMs}ms`);
+          return;
+        }
+      }
+    })();
+  }
+
+  async function run(): Promise<string> {
+    let attempt = 0;
+    while (!closing) {
+      let live: UplinkSession;
+      try {
+        live = await dial({
+          url: opts.url,
+          identity: opts.identity,
+          homePublicKey: opts.homePublicKey,
+          handlers: opts.handlers,
+        });
+      } catch (err) {
+        if (err instanceof HomeKeyMismatch) {
+          log.error(
+            `${UPLINK_LOG.keyMismatch}: ${opts.url} answered with ${err.actual}, ` +
+              `not the pinned ${err.expected}. Refusing to speak to it and not retrying; ` +
+              `this worker's apps keep running. Either something is impersonating Home, ` +
+              `or this box holds a stale pin.`,
+          );
+          return "Home's static key is not the pinned one";
+        }
+        attempt++;
+        const delay = backoff(attempt);
+        log.warn(
+          `Uplink to ${opts.url} failed (${message(err)}); retrying in ${Math.round(delay)}ms`,
+        );
+        await sleep(delay);
+        continue;
+      }
+      dialled = live;
+
+      const outcome = await announce(live);
+      if (outcome !== "open" && outcome !== "held") {
+        live.close(outcome === "retry" ? "announce failed" : "stopping");
+        await live.closed;
+        if (outcome !== "retry") {
+          log.error(outcome.stop);
+          return outcome.stop;
+        }
+        attempt++;
+        await sleep(backoff(attempt));
+        continue;
+      }
+
+      // The hello landed, so the next dial starts at full speed whether or not
+      // the announce followed it.
+      attempt = 0;
+      if (outcome === "open") session = live;
+      keepPinging(live);
+      // Rekey by reconnecting, so no session holds one key for a day.
+      const rekeyAt = timing.rekeyIntervalMs + (Math.random() * 2 - 1) * timing.rekeyJitterMs;
+      const rekey = setTimeout(() => live.close("rekey"), rekeyAt);
+      // Connect is pull: the reconcile Home dropped while this worker was
+      // down is subsumed by the one every connect runs. A held session pulls
+      // nothing — it does not know what Home would send it.
+      if (outcome === "open") opts.onConnected();
+
+      const reason = await live.closed;
+      clearTimeout(rekey);
+      session = null;
+      held = null;
+      dialled = null;
+      if (closing) break;
+      attempt++;
+      const delay = backoff(attempt);
+      log.warn(`Uplink closed (${reason}); reconnecting in ${Math.round(delay)}ms`);
+      await sleep(delay);
+    }
+    return "the worker stopped the uplink";
+  }
+
+  const loop = run().then((reason) => {
+    session = null;
+    held = null;
+    dialled = null;
+    settle(reason);
+    return reason;
+  });
+
+  return {
+    get online() {
+      return session !== null;
+    },
+    stopped,
+    verify: async (params) => await callJson(requireSession("verify").rpc, "verify", params),
+    apiAccess: async (params) =>
+      await callJson(requireSession("apiAccess").rpc, "apiAccess", params),
+    pull: async (params) => await pullState(requireSession("state.pull").rpc, params),
+    applied: async (etag, apps) => {
+      await callJson(requireSession("state.applied").rpc, "state.applied", { etag, apps });
+    },
+    updateStatus: async (status) => {
+      await callJson(requireSession("update.status").rpc, "update.status", status);
+    },
+    reconnect: (reason) => {
+      // Only a session past the door is worth dropping. Mid-dial there is
+      // nothing to replace, and the announce the loop is about to run already
+      // carries whatever moved.
+      (session ?? held?.live)?.close(reason);
+    },
+    close: async () => {
+      closing = true;
+      wake?.();
+      dialled?.close("the worker is stopping");
+      await loop;
+    },
+  };
+}
+
+async function withDeadline<T>(promise: Promise<T>, ms: number): Promise<T> {
+  promise.catch(() => {});
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const expiry = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
+  });
+  try {
+    return await Promise.race([promise, expiry]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}
+
+function message(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}

package/src/worker/validate.ts

@@ -0,0 +1,34 @@
+import type { WorkerState } from "../worker-protocol/index.ts";
+
+/**
+ * Defense-in-depth check on the deserialized wire payload before the
+ * renderer runs. Home already validates its inputs end-to-end, but a
+ * mid-flight tarball corruption or a buggy Home commit shouldn't
+ * surface as a confusing renderer crash on the worker.
+ *
+ * Today: every depends_on edge points at an app+service that actually
+ * exists in the same payload.
+ */
+export function validateWorkerState(state: WorkerState): void {
+  const services = new Map<string, Set<string>>();
+  for (const app of state.apps) {
+    services.set(app.appId, new Set(Object.keys(app.services)));
+  }
+
+  for (const app of state.apps) {
+    for (const [serviceName, edges] of Object.entries(app.dependsOn)) {
+      if (!services.get(app.appId)?.has(serviceName)) {
+        throw new Error(`dependsOn references missing service: ${app.appId}/${serviceName}`);
+      }
+      for (const edge of edges) {
+        const providerSvcs = services.get(edge.providerAppId);
+        if (!providerSvcs || !providerSvcs.has(edge.providerServiceName)) {
+          throw new Error(
+            `dependsOn edge in ${app.appId}/${serviceName} ` +
+              `points at missing producer ${edge.providerAppId}/${edge.providerServiceName}`,
+          );
+        }
+      }
+    }
+  }
+}

package/src/worker/verify-relay.ts

@@ -0,0 +1,102 @@
+// Worker-side ForwardAuth relay. The ingress gates every authenticated app
+// subdomain with a sub-request to Home's `/api/access/verify`, but the ingress
+// holds no uplink identity and must not: the worker is the one process on the
+// box that does. So the ingress dials `realm-worker:6021` by container name on
+// the management bridge and this listener opens a `verify` stream to Home.
+//
+// The neighbours are whoever the bind lets in. Told a `hostname` - the worker's
+// own address on management-net, which is what platform/worker resolves and
+// passes - that is the ingress and nothing else. Told none, the listener is on
+// every interface, which includes each UI app's `realm_<appId>` bridge, because
+// the worker joins those for the surface proxy; every such app can then dial
+// it.
+//
+// Either way only the single verify path is relayed, and that bound is
+// load-bearing: it is the whole of what a neighbour may ask Home through this
+// worker's session. A raw splice would hand the ingress - and anything else
+// that reaches this port - the worker's own credential.
+
+import { log } from "@local/logger";
+import type { Params, Result } from "../worker-protocol/uplink/index.ts";
+
+const VERIFY_PATH = "/api/access/verify";
+
+// A verify is one session lookup — anything slow is Home being down, and
+// the client is blocked on this hop, so fail fast rather than pile up.
+const UPSTREAM_TIMEOUT_MS = 10_000;
+
+interface VerifyRelayOptions {
+  port: number;
+  /**
+   * The one address to listen on: the worker's own on management-net. Omitted
+   * where the box could not resolve it, which binds every interface instead -
+   * see the warning below for why that is a fallback and not a default.
+   */
+  hostname?: string;
+  /** The `verify` method on the live session; rejects at once while it is down. */
+  verify: (params: Params<"verify">) => Promise<Result<"verify">>;
+}
+
+export function startVerifyRelay(opts: VerifyRelayOptions): Deno.HttpServer {
+  if (opts.hostname === undefined) {
+    // Not fatal: a worker off management-net has no reachable relay whichever
+    // way this binds, and refusing to start would hide that behind a second
+    // failure. But say what the wide bind costs.
+    log.warn(
+      `verify relay: no management-net address, binding every interface - every UI app whose bridge this worker joined can reach :${opts.port}`,
+    );
+  }
+  return Deno.serve({ hostname: opts.hostname ?? "0.0.0.0", port: opts.port }, async (req) => {
+    // Caddy's forward_auth rewrite sends every gate check as
+    // `GET /api/access/verify` (the original method rides
+    // X-Forwarded-Method), so anything else is not a gate check.
+    const url = new URL(req.url);
+    if (req.method !== "GET" || url.pathname !== VERIFY_PATH) {
+      return new Response("not found", { status: 404 });
+    }
+
+    let answer: Result<"verify">;
+    try {
+      answer = await withDeadline(
+        opts.verify({ search: url.search, headers: [...req.headers] }),
+        UPSTREAM_TIMEOUT_MS,
+      );
+    } catch (err) {
+      log.warn(`verify relay failed: ${(err as Error).message}`);
+      // Non-2xx is a deny, which Caddy relays to the client. Immediate while
+      // the uplink is down: waiting out a reconnect would hang every gated
+      // request on the box.
+      return new Response("verify upstream unreachable", { status: 502 });
+    }
+
+    // Status + headers verbatim: Caddy's copy_headers lifts the brokered
+    // identity headers off THIS response on 2xx, and a bounce needs its
+    // Location. Framing headers are dropped because Deno.serve re-frames the
+    // body it is handed.
+    const headers = new Headers();
+    for (const [name, value] of answer.headers) {
+      const key = name.toLowerCase();
+      if (key === "content-encoding" || key === "content-length") continue;
+      headers.append(name, value);
+    }
+    return new Response(decodeBody(answer.body), { status: answer.status, headers });
+  });
+}
+
+function decodeBody(body: string): Uint8Array<ArrayBuffer> | null {
+  if (body === "") return null;
+  return Uint8Array.fromBase64(body);
+}
+
+async function withDeadline<T>(promise: Promise<T>, ms: number): Promise<T> {
+  promise.catch(() => {});
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const expiry = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`verify timed out after ${ms}ms`)), ms);
+  });
+  try {
+    return await Promise.race([promise, expiry]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}