@diffenderfer-games/hub 2.0.0-next.0 → 2.0.0-next.10

package/src/core/offline/outbox-entry.ts

@@ -0,0 +1,59 @@
+/**
+ * One operation in the offline outbox: a `kind` (whose registered handler
+ * performs it), its JSON payload, and its replay bookkeeping. Each entry is
+ * stored under its own key, so two tabs adding and removing entries never
+ * overwrite each other's lists.
+ */
+import type { Json } from '@diffenderfer-games/hub-contract';
+import { isJson, isRecord, optionalNumber, optionalString } from '../storage/json-guards';
+
+/** A queued operation. */
+export interface OutboxEntry {
+  /** Unique and sortable by queue time: replay runs in `id` order. Also the base of its operation keys. */
+  readonly id: string;
+  /** Which handler performs it (`hub.save`, `hub.stat`, or a game's own kind). */
+  readonly kind: string;
+  /** The handler's input. */
+  readonly payload: Json;
+  /** When it was queued (epoch ms). */
+  readonly createdAt: number;
+  /** How many times replay has started it. A started entry never absorbs a later one. */
+  readonly attempts: number;
+  /** Not before this time (epoch ms); 0 means as soon as possible. */
+  readonly retryAt: number;
+  /** The last failure's message, when it has failed before. */
+  readonly lastError?: string;
+}
+
+/** How wide the time part of an id is (enough for epoch ms until the year 5138). */
+const TIME_DIGITS = 14;
+
+/** How wide the sequence part of an id is. */
+const SEQUENCE_DIGITS = 6;
+
+/**
+ * A new entry id: the queue time, a per-page sequence and a random tag, so
+ * ids sort by time and two tabs queueing in the same millisecond never clash.
+ */
+export function newEntryId(now: number, sequence: number): string {
+  const tag = Math.floor(Math.random() * 36 ** 6).toString(36).padStart(6, '0');
+  return `${String(now).padStart(TIME_DIGITS, '0')}-${String(sequence % 10 ** SEQUENCE_DIGITS).padStart(SEQUENCE_DIGITS, '0')}-${tag}`;
+}
+
+/** A stored entry, or `undefined` when it is malformed. */
+// eslint-disable-next-line no-restricted-syntax -- trust boundary: an entry read back from localStorage
+export function readEntry(stored: unknown): OutboxEntry | undefined {
+  if (!isRecord(stored) || !isJson(stored.payload)) {
+    return undefined;
+  }
+  const id = optionalString(stored.id);
+  const kind = optionalString(stored.kind);
+  const createdAt = optionalNumber(stored.createdAt);
+  const attempts = optionalNumber(stored.attempts);
+  const retryAt = optionalNumber(stored.retryAt);
+  const lastError = optionalString(stored.lastError);
+  if (id === undefined || kind === undefined || createdAt === undefined || attempts === undefined || retryAt === undefined) {
+    return undefined;
+  }
+  return { id, kind, payload: stored.payload, createdAt, attempts, retryAt, ...(lastError === undefined ? {} : { lastError }) };
+}

package/src/core/offline/outbox-kinds.ts

@@ -0,0 +1,78 @@
+/**
+ * Outbox kinds: what each sort of queued operation does when it replays,
+ * and how a new one folds into those already waiting. The hub registers its
+ * own (`hub.*`); a game adds its own with `hub.offline.define`.
+ */
+import type { Json } from '@diffenderfer-games/hub-contract';
+import type { EndpointName, EndpointReply } from '../rest/endpoint-types';
+import type { CallArguments } from '../rest/rest-client';
+
+/** What a kind's handler gets besides its payload. */
+export interface ReplayContext {
+  /** The operation's id (stable across retries). */
+  readonly opId: string;
+  /** Which try this is, from 1. */
+  readonly attempt: number;
+  /** `false` when the op runs at once (online, nothing waiting); `true` when it runs from the queue. */
+  readonly isReplay: boolean;
+  /**
+   * Call a hub endpoint. Calls to endpoints marked for offline replay carry
+   * an operation key made from the op id and the call's position in the
+   * handler, so a retried op whose earlier calls already landed gets their
+   * remembered replies instead of running them twice. Make the same calls in
+   * the same order on every try.
+   */
+  call<K extends EndpointName>(name: K, ...fieldsArgument: CallArguments<K>): Promise<EndpointReply<K>>;
+  /** A key for `step` of this op that is the same on every try: send it to a game server to deduplicate there. */
+  key(step: string): string;
+}
+
+/** Performs one operation of a kind. Throw to fail it (see the docs for which failures retry). */
+export type OfflineHandler<Payload> = (payload: Payload, context: ReplayContext) => Promise<Json | undefined> | Json | undefined;
+
+/** A kind, with optional rules for folding new operations into waiting ones. */
+export interface OfflineKindDefinition<Payload> {
+  /** Perform one operation. */
+  readonly run: OfflineHandler<Payload>;
+  /**
+   * Operations with the same merge key may fold together. Without one, every
+   * operation replays on its own.
+   */
+  readonly mergeKey?: (payload: Payload) => string;
+  /**
+   * Fold `incoming` into the newest waiting operation with the same merge key
+   * (in its place in the queue); `undefined` to queue it separately.
+   */
+  readonly merge?: (waiting: Payload, incoming: Payload) => Payload | undefined;
+  /** `true`: a new operation replaces every waiting one with the same merge key and goes last (a save slot's last write wins). */
+  readonly replaces?: boolean;
+}
+
+/** A kind as the outbox runs it: payloads are the stored JSON. */
+export interface RegisteredKind {
+  /** Perform one operation (malformed payloads are refused before this). */
+  readonly run: (payload: Json, context: ReplayContext) => Promise<Json | undefined> | Json | undefined;
+  /** Whether a stored payload is well formed for this kind. */
+  readonly accepts: (payload: Json) => boolean;
+  readonly mergeKey?: (payload: Json) => string;
+  readonly merge?: (waiting: Json, incoming: Json) => Json | undefined;
+  readonly replaces: boolean;
+}
+
+/** The prefix reserved for the hub's own kinds. */
+export const HUB_KIND_PREFIX = 'hub.';
+
+/** The registered kinds of one client. */
+export class KindRegistry {
+  private readonly kinds = new Map<string, RegisteredKind>();
+
+  /** Register (or replace) a kind. */
+  register(kind: string, definition: RegisteredKind): void {
+    this.kinds.set(kind, definition);
+  }
+
+  /** A registered kind, or `undefined`. */
+  get(kind: string): RegisteredKind | undefined {
+    return this.kinds.get(kind);
+  }
+}

package/src/core/offline/outbox-merge.ts

@@ -0,0 +1,39 @@
+/**
+ * How a new operation joins the outbox (pure, so it is easy to test): queued
+ * on its own, folded into the newest waiting one with the same merge key, or
+ * replacing every waiting one with that key. An entry replay has started is
+ * never changed or replaced: its request may already have reached the hub
+ * under its operation key, so a changed payload would be lost.
+ */
+import type { OutboxEntry } from './outbox-entry';
+import type { RegisteredKind } from './outbox-kinds';
+
+/** The store changes one enqueue makes. */
+export interface EnqueuePlan {
+  /** Entries to write (the new one, or a waiting one with a merged payload). */
+  readonly put: OutboxEntry;
+  /** Entries to remove (waiting ones the new one replaces). */
+  readonly remove: readonly string[];
+  /** `true` when `put` is a waiting entry that absorbed the new one. */
+  readonly isMerged: boolean;
+}
+
+/** How `incoming` (of kind `kind`) joins `entries` (oldest first). */
+export function planEnqueue(entries: readonly OutboxEntry[], incoming: OutboxEntry, kind: RegisteredKind): EnqueuePlan {
+  const separately: EnqueuePlan = { put: incoming, remove: [], isMerged: false };
+  const { mergeKey } = kind;
+  if (mergeKey === undefined) {
+    return separately;
+  }
+  const key = mergeKey(incoming.payload);
+  const sameKey = entries.filter((entry) => entry.kind === incoming.kind && kind.accepts(entry.payload) && mergeKey(entry.payload) === key);
+  if (kind.replaces) {
+    return { put: incoming, remove: sameKey.filter((entry) => entry.attempts === 0).map((entry) => entry.id), isMerged: false };
+  }
+  const newest = sameKey.at(-1);
+  if (kind.merge === undefined || newest === undefined || newest.attempts > 0) {
+    return separately;
+  }
+  const merged = kind.merge(newest.payload, incoming.payload);
+  return merged === undefined ? separately : { put: { ...newest, payload: merged }, remove: [], isMerged: true };
+}

package/src/core/offline/outbox-replayer.ts

@@ -0,0 +1,141 @@
+/**
+ * Replays the outbox in order, holding the game's replay lease so only one
+ * tab (or client copy) sends at a time. Each entry is marked started before
+ * its handler runs, then removed on success or on a permanent failure; a
+ * passing failure leaves it at the head with a backoff, so nothing behind it
+ * jumps the queue. An entry whose kind isn't defined here stops this
+ * replayer (and lets go of the lease) so a client that knows the kind, such
+ * as the game's own copy on the page, can carry on.
+ */
+import type { Json } from '@diffenderfer-games/hub-contract';
+import type { OutboxEntry } from './outbox-entry';
+import type { KindRegistry, ReplayContext } from './outbox-kinds';
+import type { OutboxStore } from './outbox-store';
+import type { ReplayLease } from './replay-lease';
+import { backoffMs, failureMessage, MAX_RETRIES, verdictFor } from './replay-policy';
+
+/** An op dropped during replay, with why. */
+export interface ReplayFailure {
+  /** The dropped op. */
+  readonly entry: OutboxEntry;
+  /** Why it was dropped (the hub's message, or the handler's). */
+  readonly error: string;
+}
+
+/** Why a replay stopped before the outbox was empty. */
+export type ReplayStop = 'connection' | 'backoff' | 'unknown-kind' | 'lease';
+
+/** What one replay did. */
+export interface ReplayReport {
+  /** The ops that landed, in order. */
+  readonly replayed: OutboxEntry[];
+  /** The ops dropped. */
+  readonly failed: ReplayFailure[];
+  /** Why it stopped early, or `undefined` when the outbox is empty. */
+  readonly stoppedBy?: ReplayStop;
+}
+
+/** What the replayer works with. */
+export interface ReplayerParts {
+  readonly store: OutboxStore;
+  readonly kinds: KindRegistry;
+  readonly lease: ReplayLease;
+  /** The context an entry's handler gets on try `attempt`. */
+  readonly contextFor: (entry: OutboxEntry, attempt: number) => ReplayContext;
+  /** An op landed (`result` is what its handler returned). */
+  readonly onReplayed: (entry: OutboxEntry, result: Json | undefined) => void;
+  /** An op was dropped. */
+  readonly onFailed: (failure: ReplayFailure) => void;
+  /** The hub couldn't be reached. */
+  readonly onUnreachable: () => void;
+  /** Ask for another replay in `delayMs`. */
+  readonly scheduleRetry: (delayMs: number) => void;
+  readonly now: () => number;
+}
+
+/** A report for a replay that didn't get the lease. */
+const LEASE_HELD: ReplayReport = { replayed: [], failed: [], stoppedBy: 'lease' };
+
+/** Replays one game's outbox. */
+export class OutboxReplayer {
+  private readonly parts: ReplayerParts;
+
+  /** A replayer over `parts`. */
+  constructor(parts: ReplayerParts) {
+    this.parts = parts;
+  }
+
+  /** Replay everything due, in order. `isEager` skips the backoff of ops waiting for the connection (it just came back). */
+  async replay(isEager = false): Promise<ReplayReport> {
+    let report: ReplayReport = LEASE_HELD;
+    const isHeld = await this.parts.lease.hold(async () => {
+      report = await this.drain(isEager);
+    });
+    return isHeld ? report : LEASE_HELD;
+  }
+
+  private async drain(isEager: boolean): Promise<ReplayReport> {
+    const { store, kinds, now } = this.parts;
+    const replayed: OutboxEntry[] = [];
+    const failed: ReplayFailure[] = [];
+    for (let head = store.list()[0]; head !== undefined; head = store.list()[0]) {
+      if (head.retryAt > now() && !(isEager && head.lastError === undefined)) {
+        this.parts.scheduleRetry(head.retryAt - now());
+        return { replayed, failed, stoppedBy: 'backoff' };
+      }
+      const kind = kinds.get(head.kind);
+      if (kind === undefined) {
+        return { replayed, failed, stoppedBy: 'unknown-kind' };
+      }
+      if (!kind.accepts(head.payload)) {
+        store.remove(head.id);
+        this.drop(failed, head, 'Malformed offline operation payload.');
+        continue;
+      }
+      const started: OutboxEntry = { ...head, attempts: head.attempts + 1 };
+      store.put(started);
+      try {
+        const result = await kind.run(head.payload, this.parts.contextFor(head, started.attempts));
+        store.remove(head.id);
+        replayed.push(started);
+        this.parts.onReplayed(started, result ?? undefined);
+      } catch (failure) {
+        const stop = this.settleFailure(started, failure, failed);
+        if (stop !== undefined) {
+          return { replayed, failed, stoppedBy: stop };
+        }
+      }
+    }
+    return { replayed, failed };
+  }
+
+  /** Record a failed try; answers why replay stops, or `undefined` to carry on with the next op. */
+  // eslint-disable-next-line no-restricted-syntax -- trust boundary: whatever a replay handler threw
+  private settleFailure(started: OutboxEntry, failure: unknown, failed: ReplayFailure[]): ReplayStop | undefined {
+    const { store, now } = this.parts;
+    const verdict = verdictFor(failure);
+    const message = failureMessage(failure);
+    if (verdict === 'drop' || (verdict === 'retry' && started.attempts >= MAX_RETRIES)) {
+      store.remove(started.id);
+      this.drop(failed, started, message);
+      return undefined;
+    }
+    const delay = backoffMs(started.attempts);
+    // Waiting for the connection is not an error of the op's own: an eager replay (back online) retries it at once.
+    const { id, kind, payload, createdAt, attempts } = started;
+    const retrying: OutboxEntry = { id, kind, payload, createdAt, attempts, retryAt: now() + delay };
+    const waiting: OutboxEntry = verdict === 'offline' ? retrying : { ...retrying, lastError: message };
+    store.put(waiting);
+    if (verdict === 'offline') {
+      this.parts.onUnreachable();
+    }
+    this.parts.scheduleRetry(delay);
+    return verdict === 'offline' ? 'connection' : 'backoff';
+  }
+
+  private drop(failed: ReplayFailure[], entry: OutboxEntry, error: string): void {
+    const failure = { entry, error };
+    failed.push(failure);
+    this.parts.onFailed(failure);
+  }
+}

package/src/core/offline/outbox-store.ts

@@ -0,0 +1,85 @@
+/**
+ * Where the outbox lives: one `localStorage` key per entry,
+ * `hub:<slug>:outbox:<id>`, read back in id (queue) order. A tab adding an
+ * entry and another removing one touch different keys, so neither loses the
+ * other's change. The outbox is capped by entry count and stored size.
+ */
+import { parseJsonText } from '../storage/json-guards';
+import type { KeyValueStore } from '../storage/key-value';
+import { readEntry, type OutboxEntry } from './outbox-entry';
+
+/** The most entries one game's outbox holds. */
+export const OUTBOX_MAX_ENTRIES = 500;
+
+/** The most stored characters one game's outbox holds (about 2 MB of `localStorage`). */
+export const OUTBOX_MAX_CHARS = 1_000_000;
+
+/** One game's outbox. */
+export class OutboxStore {
+  /** The key prefix of this game's entries. */
+  readonly prefix: string;
+  private readonly store: KeyValueStore;
+
+  /** The outbox for game `slug` in `store`. */
+  constructor(slug: string, store: KeyValueStore) {
+    this.prefix = `hub:${slug}:outbox:`;
+    this.store = store;
+  }
+
+  /** Every well-formed entry, oldest first. Malformed ones are removed. */
+  list(): OutboxEntry[] {
+    const entries: OutboxEntry[] = [];
+    for (const key of this.entryKeys()) {
+      const text = this.store.read(key);
+      const entry = text === null ? undefined : readEntry(parseJsonText(text));
+      if (entry === undefined) {
+        this.store.remove(key);
+      } else {
+        entries.push(entry);
+      }
+    }
+    return entries;
+  }
+
+  /** How many entries are queued. */
+  count(): number {
+    return this.entryKeys().length;
+  }
+
+  /** One entry, or `undefined` once it is gone (replayed, or removed by another tab). */
+  get(id: string): OutboxEntry | undefined {
+    const text = this.store.read(this.prefix + id);
+    return text === null ? undefined : readEntry(parseJsonText(text));
+  }
+
+  /** Whether `extra` more stored characters in one more entry still fit under the caps. */
+  fits(extraChars: number): boolean {
+    const keys = this.entryKeys();
+    if (keys.length >= OUTBOX_MAX_ENTRIES) {
+      return false;
+    }
+    const used = keys.reduce((total, key) => total + (this.store.read(key)?.length ?? 0), 0);
+    return used + extraChars <= OUTBOX_MAX_CHARS;
+  }
+
+  /** Store an entry (new or updated); `false` when the storage refused it (full or blocked). */
+  put(entry: OutboxEntry): boolean {
+    const text = JSON.stringify(entry);
+    this.store.write(this.prefix + entry.id, text);
+    return this.store.read(this.prefix + entry.id) === text;
+  }
+
+  /** Forget an entry. */
+  remove(id: string): void {
+    this.store.remove(this.prefix + id);
+  }
+
+  /** Whether a storage key belongs to this outbox (for cross-tab `storage` events). */
+  owns(key: string | null): boolean {
+    return key === null || key.startsWith(this.prefix);
+  }
+
+  private entryKeys(): string[] {
+    return this.store.keys().filter((key) => key.startsWith(this.prefix)).sort();
+  }
+}

package/src/core/offline/outbox.ts

@@ -0,0 +1,232 @@
+/**
+ * The offline operation outbox: typed operations queued in `localStorage`
+ * in order, each performed by its kind's handler, replayed in order (one tab
+ * at a time) when the hub is reachable, and reported as `queued`,
+ * `replayed` and `failed` events, to listeners and as `hub:offline-*`
+ * window events.
+ */
+import type { Json } from '@diffenderfer-games/hub-contract';
+import { HubError } from '../errors';
+import type { RestClient } from '../rest/rest-client';
+import type { KeyValueStore } from '../storage/key-value';
+import { newEntryId, type OutboxEntry } from './outbox-entry';
+import { KindRegistry, type RegisteredKind, type ReplayContext } from './outbox-kinds';
+import { planEnqueue } from './outbox-merge';
+import { OutboxReplayer, type ReplayReport } from './outbox-replayer';
+import { OutboxStore } from './outbox-store';
+import { createReplayLease } from './replay-lease';
+import { backoffMs } from './replay-policy';
+
+/** A queued operation, as `hub.offline` reports it. */
+export type OfflineOp = OutboxEntry;
+
+/** Each outbox event's detail. */
+export interface OfflineEventDetails {
+  /** An op was queued (or folded into a waiting one, which is then the `op`). */
+  queued: { readonly op: OfflineOp };
+  /** An op landed; `result` is what its handler returned. */
+  replayed: { readonly op: OfflineOp; readonly result?: Json };
+  /** An op was dropped: the hub refused it, or it kept failing. */
+  failed: { readonly op: OfflineOp; readonly error: string };
+}
+
+/** An outbox event's name. */
+export type OfflineEventName = keyof OfflineEventDetails;
+
+/** An outbox event listener. */
+export type OfflineListener<Name extends OfflineEventName> = (detail: OfflineEventDetails[Name]) => void;
+
+/** The code of the error `add` throws when the outbox is full. */
+export const OUTBOX_FULL_CODE = 'offline_queue_full';
+
+/** How long a replayer that lost the lease waits before checking again. */
+const LEASE_RECHECK_MS = 3000;
+
+/** What the outbox needs from its client. */
+export interface OutboxOptions {
+  /** The game whose outbox it is. */
+  readonly slug: string;
+  /** Where it lives (`localStorage`, or memory). */
+  readonly store: KeyValueStore;
+  /** The REST client its handlers call through. */
+  readonly rest: RestClient;
+  /** Fire a window event (the menu listens). */
+  readonly emit: (name: string, detail: OfflineEventDetails[OfflineEventName]) => void;
+  /** The hub couldn't be reached during a replay. */
+  readonly onUnreachable: () => void;
+  /** The queue length may have changed. */
+  readonly onChange: () => void;
+  /** A retry is due: replay again. */
+  readonly onRetryDue: () => void;
+  readonly now?: () => number;
+}
+
+/** One game's outbox. */
+export class Outbox {
+  /** The kinds this client can perform. */
+  readonly kinds = new KindRegistry();
+  /** The stored entries. */
+  readonly entries: OutboxStore;
+  private readonly options: OutboxOptions;
+  private readonly replayer: OutboxReplayer;
+  private readonly now: () => number;
+  private sequence = 0;
+  private retryTimer: ReturnType<typeof setTimeout> | undefined;
+  private retryAt = Number.POSITIVE_INFINITY;
+  private readonly listeners: { readonly [Name in OfflineEventName]: Set<OfflineListener<Name>> } = {
+    queued: new Set(), replayed: new Set(), failed: new Set(),
+  };
+
+  /** The outbox for `options.slug`. */
+  constructor(options: OutboxOptions) {
+    this.options = options;
+    this.now = options.now ?? Date.now;
+    this.entries = new OutboxStore(options.slug, options.store);
+    this.replayer = new OutboxReplayer({
+      store: this.entries,
+      kinds: this.kinds,
+      lease: createReplayLease(options.slug, options.store),
+      contextFor: (entry, attempt) => this.contextFor(entry, attempt, true),
+      onReplayed: (op, result) => {
+        this.fire('replayed', result === undefined ? { op } : { op, result });
+      },
+      onFailed: ({ entry, error }) => {
+        this.fire('failed', { op: entry, error });
+      },
+      onUnreachable: options.onUnreachable,
+      scheduleRetry: (delayMs) => {
+        this.scheduleRetry(delayMs);
+      },
+      now: this.now,
+    });
+  }
+
+  /** Register a kind. */
+  define(kind: string, definition: RegisteredKind): void {
+    this.kinds.register(kind, definition);
+  }
+
+  /** A new, not yet stored, op of `kind`. */
+  stamp(kind: string, payload: Json): OutboxEntry {
+    this.sequence += 1;
+    const now = this.now();
+    return { id: newEntryId(now, this.sequence), kind, payload, createdAt: now, attempts: 0, retryAt: 0 };
+  }
+
+  /**
+   * Queue an op, folding it into a waiting one where its kind says so.
+   * Throws a {@link HubError} (code {@link OUTBOX_FULL_CODE}) when the
+   * outbox is full or the storage refuses it.
+   */
+  add(op: OutboxEntry): void {
+    const kind = this.kinds.get(op.kind);
+    const plan = kind === undefined ? { put: op, remove: [], isMerged: false } : planEnqueue(this.entries.list(), op, kind);
+    const extra = plan.isMerged ? 0 : JSON.stringify(plan.put).length;
+    if (plan.remove.length === 0 && !plan.isMerged && !this.entries.fits(extra)) {
+      throw outboxFull();
+    }
+    if (!this.entries.put(plan.put)) {
+      throw outboxFull();
+    }
+    for (const id of plan.remove) {
+      this.entries.remove(id);
+    }
+    this.fire('queued', { op: plan.put });
+    this.options.onChange();
+  }
+
+  /** Perform an op now, without queueing it (the online path); throws what its handler throws. */
+  async runNow(op: OutboxEntry): Promise<Json | undefined> {
+    const kind = this.kinds.get(op.kind);
+    if (kind === undefined) {
+      throw new HubError(`No offline kind "${op.kind}" is defined.`);
+    }
+    return (await kind.run(op.payload, this.contextFor(op, 1, false))) ?? undefined;
+  }
+
+  /** Replay what is due, in order. `isEager`: the connection just came back, so ops waiting for it go at once. */
+  async replay(isEager = false): Promise<ReplayReport> {
+    this.clearRetry();
+    const report = await this.replayer.replay(isEager);
+    if ((report.stoppedBy === 'lease' && this.entries.count() > 0) || report.stoppedBy === 'unknown-kind') {
+      // Another tab or client copy is replaying, or knows a kind this one doesn't: check back later.
+      this.scheduleRetry(report.stoppedBy === 'lease' ? LEASE_RECHECK_MS : backoffMs(this.entries.list()[0]?.attempts ?? 1));
+    }
+    this.options.onChange();
+    return report;
+  }
+
+  /** The waiting ops, oldest first. */
+  pending(): OfflineOp[] {
+    return this.entries.list();
+  }
+
+  /** How many ops are waiting. */
+  count(): number {
+    return this.entries.count();
+  }
+
+  /** Listen for an event; returns the unsubscribe function. */
+  on<Name extends OfflineEventName>(name: Name, listener: OfflineListener<Name>): () => void {
+    const listeners: Set<OfflineListener<Name>> = this.listeners[name];
+    listeners.add(listener);
+    return () => {
+      listeners.delete(listener);
+    };
+  }
+
+  private fire<Name extends OfflineEventName>(name: Name, detail: OfflineEventDetails[Name]): void {
+    const listeners: Set<OfflineListener<Name>> = this.listeners[name];
+    for (const listener of listeners) {
+      try {
+        listener(detail);
+      } catch {
+        // A listener's failure must not stop the others or the replay.
+      }
+    }
+    this.options.emit(`hub:offline-${name}`, detail);
+  }
+
+  private contextFor(op: OutboxEntry, attempt: number, isReplay: boolean): ReplayContext {
+    const { rest } = this.options;
+    let step = 0;
+    return {
+      opId: op.id,
+      attempt,
+      isReplay,
+      key: (stepName) => `${op.id}.${stepName}`,
+      call: (name, ...fieldsArgument) => {
+        const opKey = `${op.id}.${String(step)}`;
+        step += 1;
+        return rest.callOnce(opKey, name, ...fieldsArgument);
+      },
+    };
+  }
+
+  private scheduleRetry(delayMs: number): void {
+    const at = this.now() + delayMs;
+    if (at >= this.retryAt) {
+      return;
+    }
+    this.clearRetry();
+    this.retryAt = at;
+    this.retryTimer = setTimeout(() => {
+      this.retryTimer = undefined;
+      this.retryAt = Number.POSITIVE_INFINITY;
+      this.options.onRetryDue();
+    }, delayMs);
+  }
+
+  private clearRetry(): void {
+    if (this.retryTimer !== undefined) {
+      clearTimeout(this.retryTimer);
+      this.retryTimer = undefined;
+    }
+    this.retryAt = Number.POSITIVE_INFINITY;
+  }
+}
+
+/** The error for a full outbox. */
+function outboxFull(): HubError {
+  return new HubError('Too many changes are waiting to sync. Reconnect to save more.', 507, { code: OUTBOX_FULL_CODE });
+}