@diffenderfer-games/hub 2.0.0-next.1 → 2.0.0-next.10

package/src/core/offline/outbox.ts

@@ -0,0 +1,232 @@
+/**
+ * The offline operation outbox: typed operations queued in `localStorage`
+ * in order, each performed by its kind's handler, replayed in order (one tab
+ * at a time) when the hub is reachable, and reported as `queued`,
+ * `replayed` and `failed` events, to listeners and as `hub:offline-*`
+ * window events.
+ */
+import type { Json } from '@diffenderfer-games/hub-contract';
+import { HubError } from '../errors';
+import type { RestClient } from '../rest/rest-client';
+import type { KeyValueStore } from '../storage/key-value';
+import { newEntryId, type OutboxEntry } from './outbox-entry';
+import { KindRegistry, type RegisteredKind, type ReplayContext } from './outbox-kinds';
+import { planEnqueue } from './outbox-merge';
+import { OutboxReplayer, type ReplayReport } from './outbox-replayer';
+import { OutboxStore } from './outbox-store';
+import { createReplayLease } from './replay-lease';
+import { backoffMs } from './replay-policy';
+
+/** A queued operation, as `hub.offline` reports it. */
+export type OfflineOp = OutboxEntry;
+
+/** Each outbox event's detail. */
+export interface OfflineEventDetails {
+  /** An op was queued (or folded into a waiting one, which is then the `op`). */
+  queued: { readonly op: OfflineOp };
+  /** An op landed; `result` is what its handler returned. */
+  replayed: { readonly op: OfflineOp; readonly result?: Json };
+  /** An op was dropped: the hub refused it, or it kept failing. */
+  failed: { readonly op: OfflineOp; readonly error: string };
+}
+
+/** An outbox event's name. */
+export type OfflineEventName = keyof OfflineEventDetails;
+
+/** An outbox event listener. */
+export type OfflineListener<Name extends OfflineEventName> = (detail: OfflineEventDetails[Name]) => void;
+
+/** The code of the error `add` throws when the outbox is full. */
+export const OUTBOX_FULL_CODE = 'offline_queue_full';
+
+/** How long a replayer that lost the lease waits before checking again. */
+const LEASE_RECHECK_MS = 3000;
+
+/** What the outbox needs from its client. */
+export interface OutboxOptions {
+  /** The game whose outbox it is. */
+  readonly slug: string;
+  /** Where it lives (`localStorage`, or memory). */
+  readonly store: KeyValueStore;
+  /** The REST client its handlers call through. */
+  readonly rest: RestClient;
+  /** Fire a window event (the menu listens). */
+  readonly emit: (name: string, detail: OfflineEventDetails[OfflineEventName]) => void;
+  /** The hub couldn't be reached during a replay. */
+  readonly onUnreachable: () => void;
+  /** The queue length may have changed. */
+  readonly onChange: () => void;
+  /** A retry is due: replay again. */
+  readonly onRetryDue: () => void;
+  readonly now?: () => number;
+}
+
+/** One game's outbox. */
+export class Outbox {
+  /** The kinds this client can perform. */
+  readonly kinds = new KindRegistry();
+  /** The stored entries. */
+  readonly entries: OutboxStore;
+  private readonly options: OutboxOptions;
+  private readonly replayer: OutboxReplayer;
+  private readonly now: () => number;
+  private sequence = 0;
+  private retryTimer: ReturnType<typeof setTimeout> | undefined;
+  private retryAt = Number.POSITIVE_INFINITY;
+  private readonly listeners: { readonly [Name in OfflineEventName]: Set<OfflineListener<Name>> } = {
+    queued: new Set(), replayed: new Set(), failed: new Set(),
+  };
+
+  /** The outbox for `options.slug`. */
+  constructor(options: OutboxOptions) {
+    this.options = options;
+    this.now = options.now ?? Date.now;
+    this.entries = new OutboxStore(options.slug, options.store);
+    this.replayer = new OutboxReplayer({
+      store: this.entries,
+      kinds: this.kinds,
+      lease: createReplayLease(options.slug, options.store),
+      contextFor: (entry, attempt) => this.contextFor(entry, attempt, true),
+      onReplayed: (op, result) => {
+        this.fire('replayed', result === undefined ? { op } : { op, result });
+      },
+      onFailed: ({ entry, error }) => {
+        this.fire('failed', { op: entry, error });
+      },
+      onUnreachable: options.onUnreachable,
+      scheduleRetry: (delayMs) => {
+        this.scheduleRetry(delayMs);
+      },
+      now: this.now,
+    });
+  }
+
+  /** Register a kind. */
+  define(kind: string, definition: RegisteredKind): void {
+    this.kinds.register(kind, definition);
+  }
+
+  /** A new, not yet stored, op of `kind`. */
+  stamp(kind: string, payload: Json): OutboxEntry {
+    this.sequence += 1;
+    const now = this.now();
+    return { id: newEntryId(now, this.sequence), kind, payload, createdAt: now, attempts: 0, retryAt: 0 };
+  }
+
+  /**
+   * Queue an op, folding it into a waiting one where its kind says so.
+   * Throws a {@link HubError} (code {@link OUTBOX_FULL_CODE}) when the
+   * outbox is full or the storage refuses it.
+   */
+  add(op: OutboxEntry): void {
+    const kind = this.kinds.get(op.kind);
+    const plan = kind === undefined ? { put: op, remove: [], isMerged: false } : planEnqueue(this.entries.list(), op, kind);
+    const extra = plan.isMerged ? 0 : JSON.stringify(plan.put).length;
+    if (plan.remove.length === 0 && !plan.isMerged && !this.entries.fits(extra)) {
+      throw outboxFull();
+    }
+    if (!this.entries.put(plan.put)) {
+      throw outboxFull();
+    }
+    for (const id of plan.remove) {
+      this.entries.remove(id);
+    }
+    this.fire('queued', { op: plan.put });
+    this.options.onChange();
+  }
+
+  /** Perform an op now, without queueing it (the online path); throws what its handler throws. */
+  async runNow(op: OutboxEntry): Promise<Json | undefined> {
+    const kind = this.kinds.get(op.kind);
+    if (kind === undefined) {
+      throw new HubError(`No offline kind "${op.kind}" is defined.`);
+    }
+    return (await kind.run(op.payload, this.contextFor(op, 1, false))) ?? undefined;
+  }
+
+  /** Replay what is due, in order. `isEager`: the connection just came back, so ops waiting for it go at once. */
+  async replay(isEager = false): Promise<ReplayReport> {
+    this.clearRetry();
+    const report = await this.replayer.replay(isEager);
+    if ((report.stoppedBy === 'lease' && this.entries.count() > 0) || report.stoppedBy === 'unknown-kind') {
+      // Another tab or client copy is replaying, or knows a kind this one doesn't: check back later.
+      this.scheduleRetry(report.stoppedBy === 'lease' ? LEASE_RECHECK_MS : backoffMs(this.entries.list()[0]?.attempts ?? 1));
+    }
+    this.options.onChange();
+    return report;
+  }
+
+  /** The waiting ops, oldest first. */
+  pending(): OfflineOp[] {
+    return this.entries.list();
+  }
+
+  /** How many ops are waiting. */
+  count(): number {
+    return this.entries.count();
+  }
+
+  /** Listen for an event; returns the unsubscribe function. */
+  on<Name extends OfflineEventName>(name: Name, listener: OfflineListener<Name>): () => void {
+    const listeners: Set<OfflineListener<Name>> = this.listeners[name];
+    listeners.add(listener);
+    return () => {
+      listeners.delete(listener);
+    };
+  }
+
+  private fire<Name extends OfflineEventName>(name: Name, detail: OfflineEventDetails[Name]): void {
+    const listeners: Set<OfflineListener<Name>> = this.listeners[name];
+    for (const listener of listeners) {
+      try {
+        listener(detail);
+      } catch {
+        // A listener's failure must not stop the others or the replay.
+      }
+    }
+    this.options.emit(`hub:offline-${name}`, detail);
+  }
+
+  private contextFor(op: OutboxEntry, attempt: number, isReplay: boolean): ReplayContext {
+    const { rest } = this.options;
+    let step = 0;
+    return {
+      opId: op.id,
+      attempt,
+      isReplay,
+      key: (stepName) => `${op.id}.${stepName}`,
+      call: (name, ...fieldsArgument) => {
+        const opKey = `${op.id}.${String(step)}`;
+        step += 1;
+        return rest.callOnce(opKey, name, ...fieldsArgument);
+      },
+    };
+  }
+
+  private scheduleRetry(delayMs: number): void {
+    const at = this.now() + delayMs;
+    if (at >= this.retryAt) {
+      return;
+    }
+    this.clearRetry();
+    this.retryAt = at;
+    this.retryTimer = setTimeout(() => {
+      this.retryTimer = undefined;
+      this.retryAt = Number.POSITIVE_INFINITY;
+      this.options.onRetryDue();
+    }, delayMs);
+  }
+
+  private clearRetry(): void {
+    if (this.retryTimer !== undefined) {
+      clearTimeout(this.retryTimer);
+      this.retryTimer = undefined;
+    }
+    this.retryAt = Number.POSITIVE_INFINITY;
+  }
+}
+
+/** The error for a full outbox. */
+function outboxFull(): HubError {
+  return new HubError('Too many changes are waiting to sync. Reconnect to save more.', 507, { code: OUTBOX_FULL_CODE });
+}

package/src/core/offline/queue.ts

@@ -1,12 +1,8 @@
 /**
- * The offline mutation queue. Writes apply to the local cache at once and
- * queue a durable op; when the hub is reachable again the queue replays in
- * order. Ops collapse as they are queued (a save's last write wins, `incr`s
- * add up, `max`es keep the largest, a board keeps its best, profile patches
- * merge), so the backlog stays small.
- *
- * The op shapes are the v1 client's, unchanged: a page upgraded from a
- * vendored copy replays the queue that copy left in `localStorage`.
+ * The v1 client's offline queue (`hub:<slug>:queue`): its op shapes and the
+ * reader for what a vendored v1 copy left in `localStorage`. This client
+ * queues into the outbox instead (`outbox.ts`) and moves any v1 ops there on
+ * start (`hub-kinds/legacy-ops.ts`), so nothing a v1 copy queued is lost.
  */
 import type { Json } from '@diffenderfer-games/hub-contract';
 import type { SortDir } from '@diffenderfer-games/hub-contract';
@@ -128,44 +124,3 @@ export function readQueue(stored: unknown): QueueOp[] {
 export function betterScore(first: number, second: number, sort: SortDir | undefined): number {
   return sort === 'asc' ? Math.min(first, second) : Math.max(first, second);
 }
-
-/** The op that `queued` becomes when `incoming` collapses into it, or `undefined` when they don't collapse. */
-function collapsedOp(queued: QueueOp, incoming: QueueOp): QueueOp | undefined {
-  if (queued.kind === 'incrStat' && incoming.kind === 'incrStat' && queued.key === incoming.key) {
-    return { ...queued, amount: queued.amount + incoming.amount };
-  }
-  if (queued.kind === 'maxStat' && incoming.kind === 'maxStat' && queued.key === incoming.key) {
-    return { ...queued, value: Math.max(queued.value, incoming.value) };
-  }
-  if (queued.kind === 'setStat' && incoming.kind === 'setStat' && queued.key === incoming.key) {
-    return { ...queued, value: incoming.value };
-  }
-  if (queued.kind === 'submitScore' && incoming.kind === 'submitScore' && queued.key === incoming.key) {
-    return { ...queued, score: betterScore(queued.score, incoming.score, incoming.opts.sort), opts: incoming.opts };
-  }
-  if (queued.kind === 'setProfile' && incoming.kind === 'setProfile') {
-    return { ...queued, patch: { ...queued.patch, ...incoming.patch } };
-  }
-  return undefined;
-}
-
-/** Whether an op writes save slot `slot`. */
-function writesSlot(op: QueueOp, slot: string): boolean {
-  return (op.kind === 'putSave' || op.kind === 'deleteSave') && op.slot === slot;
-}
-
-/**
- * The queue with `incoming` added: a save or delete replaces any queued
- * write to the same slot (and goes last); a compatible op merges into the
- * first queued one in place; anything else is appended.
- */
-export function enqueueOp(queue: readonly QueueOp[], incoming: QueueOp): QueueOp[] {
-  if (incoming.kind === 'putSave' || incoming.kind === 'deleteSave') {
-    return [...queue.filter((op) => !writesSlot(op, incoming.slot)), incoming];
-  }
-  const mergeAt = queue.findIndex((op) => collapsedOp(op, incoming) !== undefined);
-  if (mergeAt < 0) {
-    return [...queue, incoming];
-  }
-  return queue.map((op, i) => (i === mergeAt ? (collapsedOp(op, incoming) ?? op) : op));
-}

package/src/core/offline/replay-lease.ts

@@ -0,0 +1,87 @@
+/**
+ * One replayer at a time per game, across tabs and across the client copies
+ * on one page: a Web Lock where the browser has them, else a lease in
+ * `localStorage` that its holder renews while it works and that lapses if
+ * the tab dies. A replayer that can't get it simply doesn't run; the holder
+ * replays for everyone.
+ */
+import { isRecord, optionalNumber, optionalString, parseJsonText } from '../storage/json-guards';
+import type { KeyValueStore } from '../storage/key-value';
+
+/** Runs work while holding the lease. */
+export interface ReplayLease {
+  /** Run `work` holding the lease; `false` (without running it) when another replayer holds it. */
+  hold(work: () => Promise<void>): Promise<boolean>;
+}
+
+/** How long a `localStorage` lease lasts without renewal. */
+export const STORAGE_LEASE_MS = 15_000;
+
+/** How often the holder renews it. */
+const RENEW_EVERY_MS = 5_000;
+
+/** A lease on the Web Locks API. */
+function webLockLease(locks: LockManager, name: string): ReplayLease {
+  return {
+    hold: (work) => locks.request(name, { ifAvailable: true }, async (lock) => {
+      if (lock === null) {
+        return false;
+      }
+      await work();
+      return true;
+    }),
+  };
+}
+
+/** The lease's holder and expiry, or `undefined` when there is none. */
+function readLease(text: string | null): { readonly owner: string; readonly until: number } | undefined {
+  const stored = text === null ? undefined : parseJsonText(text);
+  if (!isRecord(stored)) {
+    return undefined;
+  }
+  const owner = optionalString(stored.owner);
+  const until = optionalNumber(stored.until);
+  return owner === undefined || until === undefined ? undefined : { owner, until };
+}
+
+/** A lease kept in `store` under `key`, for pages without Web Locks. */
+export function storageLease(store: KeyValueStore, key: string, now: () => number = Date.now): ReplayLease {
+  const owner = `${String(now())}-${Math.random().toString(36).slice(2)}`;
+  const claim = (): boolean => {
+    const current = readLease(store.read(key));
+    if (current !== undefined && current.owner !== owner && current.until > now()) {
+      return false;
+    }
+    store.write(key, JSON.stringify({ owner, until: now() + STORAGE_LEASE_MS }));
+    return readLease(store.read(key))?.owner === owner;
+  };
+  return {
+    async hold(work) {
+      if (!claim()) {
+        return false;
+      }
+      const renewal = setInterval(claim, RENEW_EVERY_MS);
+      try {
+        await work();
+      } finally {
+        clearInterval(renewal);
+        if (readLease(store.read(key))?.owner === owner) {
+          store.remove(key);
+        }
+      }
+      return true;
+    },
+  };
+}
+
+/** The page's Web Locks, when it has them. */
+function pageLocks(): LockManager | undefined {
+  return typeof navigator !== 'undefined' && 'locks' in navigator ? navigator.locks : undefined;
+}
+
+/** The lease for game `slug`'s outbox: a Web Lock when available, else one in `store`. */
+export function createReplayLease(slug: string, store: KeyValueStore): ReplayLease {
+  const name = `hub:${slug}:outbox-lease`;
+  const locks = pageLocks();
+  return locks === undefined ? storageLease(store, name) : webLockLease(locks, name);
+}

package/src/core/offline/replay-policy.ts

@@ -0,0 +1,52 @@
+/**
+ * What a failed replay does next, and how long it waits:
+ *
+ * - `offline`: the hub (or the network) can't be reached. The op stays at
+ *   the head of the queue and replay waits for the connection.
+ * - `retry`: the hub answered with a passing problem (408, 429, 5xx) or a
+ *   game handler's fetch failed. The op stays at the head and is retried with
+ *   backoff, up to {@link MAX_RETRIES} times, then dropped.
+ * - `drop`: the hub refused it (any other status) or the handler threw
+ *   something else. The op is dropped and reported, and the queue moves on.
+ */
+import { HubError } from '../errors';
+
+/** What to do after a failed replay. */
+export type FailureVerdict = 'offline' | 'retry' | 'drop';
+
+/** The most tries for an op that keeps getting a passing failure. */
+export const MAX_RETRIES = 8;
+
+/** The first retry's delay. */
+const BASE_DELAY_MS = 1000;
+
+/** The longest delay between tries. */
+const MAX_DELAY_MS = 60_000;
+
+/** Statuses that mean "try again later". */
+const PASSING_STATUSES: ReadonlySet<number> = new Set([408, 425, 429]);
+
+/** The verdict for a failure. */
+// eslint-disable-next-line no-restricted-syntax -- trust boundary: whatever a replay handler threw
+export function verdictFor(failure: unknown): FailureVerdict {
+  if (failure instanceof HubError) {
+    if (failure.status === undefined) {
+      return 'offline';
+    }
+    return failure.status >= 500 || PASSING_STATUSES.has(failure.status) ? 'retry' : 'drop';
+  }
+  // A game handler's own fetch rejects with a TypeError when the network is down.
+  return failure instanceof TypeError ? 'retry' : 'drop';
+}
+
+/** The delay before try `attempts + 1`: doubling from 1 s, capped at 60 s, with jitter so tabs and players spread out. */
+export function backoffMs(attempts: number, random: () => number = Math.random): number {
+  const ceiling = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** Math.max(0, attempts - 1));
+  return Math.round(ceiling / 2 + (ceiling / 2) * random());
+}
+
+/** A failure's message, for reports. */
+// eslint-disable-next-line no-restricted-syntax -- trust boundary: whatever a replay handler threw
+export function failureMessage(failure: unknown): string {
+  return failure instanceof Error ? failure.message : String(failure);
+}

package/src/core/page-account.ts

@@ -0,0 +1,121 @@
+/**
+ * Who the page is signed in as, as the hub tells it, so `hub.userId()` and
+ * `hub.avatar` follow the account without the game calling `hub.me()`:
+ *
+ * - the menu's `hub:account` (every refresh: after it makes a new guest's
+ *   account, after sign-in) and `hub:auth` (the account changed);
+ * - the social runtime's {@link SESSION_EVENT} each time its socket says
+ *   hello (connected, or connected again after a drop);
+ * - this client's own sign-in, sign-up, claim and sign-out.
+ *
+ * One per page: every bundle's copy hears the same window events.
+ */
+import { readField } from '../realtime/cross-bundle';
+import { SESSION_EVENT } from '../realtime/runtime-api';
+import { isBrowserPage } from './page';
+
+/** The window events that say who is signed in (`{ user }`). */
+const ACCOUNT_EVENTS = ['hub:account', 'hub:auth'] as const;
+
+/** The signed-in player as the page heard it. */
+export interface PageUser {
+  readonly id: number;
+  /** Their look code when the source carried it, else `null`. */
+  readonly look: string | null;
+}
+
+/** Where the news came from: an account event, or a socket hello (a time to catch up on what was missed). */
+export type AccountSource = 'account' | 'session';
+
+/** Hears every report of who is signed in. */
+export type AccountListener = (user: PageUser | null, source: AccountSource) => void;
+
+/** The page's account, as last heard. */
+export class PageAccount {
+  /** `undefined` until something said; `null` when signed out. */
+  private heard: PageUser | null | undefined = undefined;
+  private readonly listeners = new Set<AccountListener>();
+  private isListening = false;
+
+  constructor() {
+    this.listen();
+  }
+
+  /** The signed-in player, `null` when signed out, `undefined` before anything said. */
+  get user(): PageUser | null | undefined {
+    this.listen();
+    return this.heard;
+  }
+
+  /** Record who is signed in (this client's own sign-in or sign-out) and tell the listeners. */
+  note(user: PageUser | null, source: AccountSource = 'account'): void {
+    this.heard = user;
+    for (const listener of [...this.listeners]) {
+      listener(user, source);
+    }
+  }
+
+  /** Hear every report; returns the unsubscribe function. */
+  on(listener: AccountListener): () => void {
+    this.listen();
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  /** Listen to the window events (at once in a page; made before a window existed, on first use). */
+  private listen(): void {
+    if (this.isListening || !isBrowserPage()) {
+      return;
+    }
+    this.isListening = true;
+    for (const eventName of ACCOUNT_EVENTS) {
+      window.addEventListener(eventName, (event) => { this.onEvent(event, 'account'); });
+    }
+    window.addEventListener(SESSION_EVENT, (event) => { this.onEvent(event, 'session'); });
+  }
+
+  private onEvent(event: Event, source: AccountSource): void {
+    if (!(event instanceof CustomEvent)) {
+      return;
+    }
+    const user = readUser(event.detail);
+    if (user !== undefined) {
+      this.note(user, source);
+    }
+  }
+}
+
+/** A page user from an account the hub sent (`look` when it has one). */
+export function toPageUser(user: { readonly id: number; readonly look?: string | undefined }): PageUser {
+  return { id: user.id, look: lookOrNull(user.look) };
+}
+
+/** A non-empty look code, else `null`. */
+// eslint-disable-next-line no-restricted-syntax -- trust boundary: a field of another bundle's event detail
+function lookOrNull(look: unknown): string | null {
+  return typeof look === 'string' && look !== '' ? look : null;
+}
+
+/** The user in a `{ user }` detail: the user, `null` (signed out), or `undefined` (it doesn't say). */
+// eslint-disable-next-line no-restricted-syntax -- trust boundary: another bundle's event detail, read field by field
+function readUser(detail: unknown): PageUser | null | undefined {
+  const user = readField(detail, 'user');
+  if (user === null) {
+    return null;
+  }
+  const id = readField(user, 'id');
+  if (typeof id !== 'number' || !Number.isInteger(id)) {
+    return undefined;
+  }
+  return { id, look: lookOrNull(readField(user, 'look')) };
+}
+
+let pageAccountSingleton: PageAccount | null = null;
+
+/** The page's account tracker (made on first use; the hub client makes it at load). */
+export function pageAccount(): PageAccount {
+  pageAccountSingleton ??= new PageAccount();
+  return pageAccountSingleton;
+}

package/src/core/page.ts

@@ -37,12 +37,25 @@ export function injectedConfig(): InjectedConfig {
   return { ...(slug === undefined ? {} : { slug }), ...(apiBase === undefined ? {} : { apiBase }) };
 }
 
-/** The game's slug: the injected one, else the first URL path segment, else `null`. */
+/** The game's display title: the host's injected one, else the page's `<title>`, else `fallback`. */
+export function detectGameTitle(fallback: string): string {
+  const injected = isBrowserPage() && isRecord(window.__HUB__) ? optionalString(window.__HUB__.title) : undefined;
+  if (injected !== undefined && injected !== '') {
+    return injected;
+  }
+  const pageTitle = typeof document === 'undefined' ? '' : document.title.trim();
+  return pageTitle === '' ? fallback : pageTitle;
+}
+
+/** The game's slug: the injected one; `null` on an injected site page (the catalog, `/world`); else the first path segment. */
 export function detectSlug(): string | null {
   const injectedSlug = injectedConfig().slug;
   if (injectedSlug !== undefined && injectedSlug !== '') {
     return injectedSlug;
   }
+  if (isHubInjectedPage()) {
+    return null;
+  }
   if (typeof location === 'undefined') {
     return null;
   }

package/src/core/rest/rest-client.ts

@@ -9,7 +9,7 @@ import { errorFor, HubError, readErrorFields, UNSUPPORTED_BY_HUB } from '../erro
 import { isRecord } from '../storage/json-guards';
 import { HUB_CLIENT_VERSION } from '../version/client-version.gen';
 import type { EndpointFields, EndpointName, EndpointReply } from './endpoint-types';
-import { ROUTES } from './routes.gen';
+import { OFFLINE_OP_HEADER, ROUTES } from './routes.gen';
 
 /** The header that tells the hub which client version is calling. */
 export const CLIENT_VERSION_HEADER = 'x-hub-client';
@@ -38,6 +38,12 @@ export interface RestClient {
   readonly apiBase: string;
   /** Call an endpoint; resolves with its reply body, rejects with a {@link HubError}. */
   call<K extends EndpointName>(name: K, ...fieldsArgument: CallArguments<K>): Promise<EndpointReply<K>>;
+  /**
+   * Call an endpoint as part of an offline operation: an endpoint marked for
+   * offline replay gets `opKey` in the operation header, so the hub runs it
+   * once however often it is sent; any other endpoint is called as usual.
+   */
+  callOnce<K extends EndpointName>(opKey: string, name: K, ...fieldsArgument: CallArguments<K>): Promise<EndpointReply<K>>;
 }
 
 /** One field value as URL text. */
@@ -126,14 +132,18 @@ async function failureOf(name: EndpointName, response: Response): Promise<HubErr
 /** A REST client for one API base. */
 export function createRestClient(options: RestClientOptions): RestClient {
   // eslint-disable-next-line no-restricted-syntax -- trust boundary: the caller's typed fields, walked generically
-  async function send<Reply>(name: EndpointName, fields: unknown): Promise<Reply> {
+  async function send<Reply>(name: EndpointName, fields: unknown, opKey?: string): Promise<Reply> {
     const planned = planRequest(name, fields);
     let response: Response;
     try {
       response = await options.fetch(options.apiBase + planned.pathAndQuery, {
         method: planned.method,
         credentials: 'same-origin',
-        headers: { 'content-type': 'application/json', [CLIENT_VERSION_HEADER]: HUB_CLIENT_VERSION },
+        headers: {
+          'content-type': 'application/json',
+          [CLIENT_VERSION_HEADER]: HUB_CLIENT_VERSION,
+          ...(opKey !== undefined && ROUTES[name].offline === 'replay' ? { [OFFLINE_OP_HEADER]: opKey } : {}),
+        },
         ...(planned.bodyText === undefined ? {} : { body: planned.bodyText }),
       });
     } catch (failure) {
@@ -148,5 +158,6 @@ export function createRestClient(options: RestClientOptions): RestClient {
   return {
     apiBase: options.apiBase,
     call: (name, ...fieldsArgument) => send(name, fieldsArgument[0]),
+    callOnce: (opKey, name, ...fieldsArgument) => send(name, fieldsArgument[0], opKey),
   };
 }