@huynle/hub 0.6.0 → 0.7.1

package/src/central/cluster/leader.ts

@@ -0,0 +1,232 @@
+// Leader loops (design §9, decision vkxkqiuf §7): a timed job (decision deadlines, message re-wakes, the webhook
+// sender, the launch supervisor) runs on one pod at a time, the holder of a fenced lease `leader:<name>` in the
+// shared Locks. The single-process hub runs the same code on in-memory locks: it is always the leader.
+// - start(): tries to take the lease at once, then every renewMs: the leader renews it, the others try to take it.
+//   Taking it calls onElected (with the lease: its fence grows with every new leader); losing it (taken, expired, or
+//   a renewal failing too close to its expiry) calls onDeposed, so a leader stops before its lease can run out.
+// - stop(): steps down, releases the lease and tells the other pods (LEADER topic `.released`), which try at once:
+//   a graceful failover takes one bus hop; a dead leader is replaced within ttlMs (+ one renewMs tick).
+// - kick(message): any pod nudges the leader (onKick runs there only), e.g. "a launch was queued".
+// - request(message): any pod asks the leader (onRequest), wherever it runs (Bus RPC on leader.<name>.<owner>.rpc);
+//   NoLeaderError when nobody holds the lease or the holder no longer answers.
+import { realTimers } from "../backend/memory-bus"
+import { BusNoResponderError, type Bus, type Clock, type Data, type Lease, type Locks, type RequestOptions, type Timers, type Unsubscribe } from "../backend/types"
+
+/** How long a leader lease lasts without renewal (a dead leader is replaced after this). */
+export const LEADER_TTL_MS = 15_000
+
+/** The Locks key of a job's leader lease. */
+export const leaderKey = (name: string) => `leader:${name}`
+const kickTopic = (name: string) => `leader.${name}.kick`
+const releasedTopic = (name: string) => `leader.${name}.released`
+const requestTopic = (name: string, owner: string) => `leader.${name}.${owner}.rpc`
+
+/** Nobody leads the job (or its leader went away): the request cannot be answered now. */
+export class NoLeaderError extends Error {
+  constructor(name: string) {
+    super(`No leader runs ${name} right now`)
+    this.name = "NoLeaderError"
+  }
+}
+
+export interface LeaderLoopOptions {
+  readonly locks: Locks
+  readonly bus: Bus
+  /** The job (lease key leader:<name>, topics leader.<name>.*). */
+  readonly name: string
+  /** This pod's process (unique per process: a restarted pod is a new owner). */
+  readonly owner: string
+  /** Lease time (default LEADER_TTL_MS). */
+  readonly ttlMs?: number
+  /** How often the leader renews and the others try (default a third of ttlMs). */
+  readonly renewMs?: number
+  readonly timers?: Timers
+  readonly now?: Clock
+  /** This pod leads now: start the job. */
+  readonly onElected?: (lease: Lease) => void | Promise<void>
+  /** This pod no longer leads: stop the job. */
+  readonly onDeposed?: () => void | Promise<void>
+  readonly onKick?: (message: Data) => void
+  readonly onRequest?: (message: Data) => Data | Promise<Data>
+  readonly onError?: (error: unknown) => void
+}
+
+export class LeaderLoop {
+  readonly name: string
+  private readonly options: LeaderLoopOptions
+  private readonly ttlMs: number
+  private readonly renewMs: number
+  private readonly timers: Timers
+  private readonly now: Clock
+  private readonly onError: (error: unknown) => void
+  private lease: Lease | undefined
+  /** Until when (this pod's clock) the lease surely holds: the time the last successful take or renewal started + ttl. */
+  private validUntil = 0
+  private running = false
+  private timer: unknown
+  private queue: Promise<void> = Promise.resolve()
+  private readonly subscriptions: Unsubscribe[] = []
+  private serving: Unsubscribe | undefined
+
+  constructor(options: LeaderLoopOptions) {
+    this.options = options
+    this.name = options.name
+    this.ttlMs = options.ttlMs ?? LEADER_TTL_MS
+    this.renewMs = options.renewMs ?? Math.max(1, Math.floor(this.ttlMs / 3))
+    this.timers = options.timers ?? realTimers
+    this.now = options.now ?? Date.now
+    this.onError = options.onError ?? ((error) => console.error(`[hub] leader ${options.name}: ${error instanceof Error ? error.message : String(error)}`))
+  }
+
+  /** Whether this pod leads the job now. */
+  isLeader(): boolean {
+    return this.lease !== undefined
+  }
+
+  /** The current lease's fence (undefined when not leading). */
+  fence(): number | undefined {
+    return this.lease?.fence
+  }
+
+  /** Listens for kicks and releases, tries to take the lease and keeps trying (or renewing) every renewMs. */
+  async start(): Promise<void> {
+    if (this.running) return
+    this.running = true
+    const { bus, name } = this.options
+    try {
+      this.subscriptions.push(
+        await bus.subscribe(kickTopic(name), (message) => {
+          if (this.lease) this.options.onKick?.(message)
+        }),
+        await bus.subscribe(releasedTopic(name), () => {
+          if (this.running && !this.lease) void this.enqueue(() => this.attempt())
+        }),
+      )
+    } catch (error) {
+      this.onError(error)
+    }
+    await this.enqueue(() => this.attempt())
+    this.schedule()
+  }
+
+  /** Steps down (releasing the lease and telling the other pods) and stops trying. */
+  async stop(): Promise<void> {
+    if (!this.running) return
+    this.running = false
+    if (this.timer !== undefined) this.timers.clear(this.timer)
+    this.timer = undefined
+    const subscriptions = this.subscriptions.splice(0)
+    await this.enqueue(async () => {
+      if (this.lease) {
+        await this.depose()
+        await this.options.locks.release(leaderKey(this.name), this.options.owner).catch((error) => this.onError(error))
+        await this.options.bus.publish(releasedTopic(this.name), { owner: this.options.owner }).catch((error) => this.onError(error))
+      }
+      await Promise.allSettled(subscriptions.map((unsubscribe) => unsubscribe()))
+    })
+  }
+
+  /** Nudges the leader (wherever it runs). */
+  async kick(message: Data = {}): Promise<void> {
+    await this.options.bus.publish(kickTopic(this.name), message)
+  }
+
+  /** Asks the leader (here or on another pod); NoLeaderError when there is none. */
+  async request<T = Data>(message: Data, options: RequestOptions): Promise<T> {
+    const holder = await this.options.locks.holder(leaderKey(this.name))
+    if (!holder) throw new NoLeaderError(this.name)
+    if (holder.owner === this.options.owner && this.lease) {
+      if (!this.options.onRequest) throw new NoLeaderError(this.name)
+      return (await this.options.onRequest(structuredClone(message))) as T
+    }
+    try {
+      return await this.options.bus.request<T>(requestTopic(this.name, holder.owner), message, options)
+    } catch (error) {
+      if (error instanceof BusNoResponderError) throw new NoLeaderError(this.name)
+      throw error
+    }
+  }
+
+  // ---- internals ----------------------------------------------------------------------------------------
+
+  private schedule(): void {
+    if (!this.running) return
+    this.timer = this.timers.set(() => {
+      this.timer = undefined
+      void this.enqueue(() => this.attempt()).finally(() => this.schedule())
+    }, this.renewMs)
+  }
+
+  /** Renews the lease (leader) or tries to take it (the others). */
+  private async attempt(): Promise<void> {
+    if (!this.running) return
+    const { locks, owner } = this.options
+    const key = leaderKey(this.name)
+    const started = this.now()
+    if (this.lease) {
+      let renewed: Lease | undefined
+      try {
+        renewed = await locks.renew(key, owner, this.ttlMs)
+      } catch (error) {
+        this.onError(error)
+        // Not sure whether it still holds: step down while the lease surely lasts one more tick.
+        if (this.now() + this.renewMs >= this.validUntil) await this.depose()
+        return
+      }
+      if (!renewed) {
+        await this.depose()
+        return
+      }
+      this.lease = renewed
+      this.validUntil = started + this.ttlMs
+      return
+    }
+    let taken: Lease | undefined
+    try {
+      taken = await locks.acquire(key, owner, this.ttlMs)
+    } catch (error) {
+      this.onError(error)
+      return
+    }
+    if (!taken) return
+    this.lease = taken
+    this.validUntil = started + this.ttlMs
+    await this.elect(taken)
+  }
+
+  private async elect(lease: Lease): Promise<void> {
+    const { bus, name, owner } = this.options
+    try {
+      this.serving = await bus.serve(requestTopic(name, owner), async (message) => {
+        if (!this.lease || !this.options.onRequest) throw new NoLeaderError(name)
+        return this.options.onRequest(message)
+      })
+    } catch (error) {
+      this.onError(error)
+    }
+    try {
+      await this.options.onElected?.(lease)
+    } catch (error) {
+      this.onError(error)
+    }
+  }
+
+  private async depose(): Promise<void> {
+    if (!this.lease) return
+    this.lease = undefined
+    const serving = this.serving
+    this.serving = undefined
+    await serving?.().catch((error) => this.onError(error))
+    try {
+      await this.options.onDeposed?.()
+    } catch (error) {
+      this.onError(error)
+    }
+  }
+
+  private enqueue(task: () => Promise<void>): Promise<void> {
+    const run = this.queue.then(task, task)
+    this.queue = run.catch((error) => this.onError(error))
+    return this.queue
+  }
+}

package/src/central/cluster/live.ts

@@ -0,0 +1,151 @@
+// The console live stream across pods (design §9, decision vkxkqiuf §6). A user's events go to every pod over the
+// Bus topic user.<id>.live, so a console connected to any pod sees what happens on the pod holding the spoke (status,
+// activity, streamed text, messages, decisions, launches, agent audit). Every event except streamed text (delta,
+// reasoning) is appended to the shared EventLog first and carries its id, so a console that reconnects to another
+// pod resumes by Last-Event-ID; streamed text is broadcast only. Events of one user are sent in order from a pod.
+// Catalog changes are not user events here: each pod rebuilds its consoles' catalog from its own (cluster-wide)
+// registry; publishCatalog() only tells every pod that the stored machines changed.
+import type { Bus, EventLog, EventMessage, Unsubscribe } from "../backend/types"
+
+export const liveTopic = (userID: string): string => `user.${userID}.live`
+/** Stored machines changed (approved, revoked, moved): every pod sends its consoles a fresh catalog. */
+export const CATALOG_TOPIC = "live.catalog"
+/** Broadcast only, never logged (high volume; a resumed console refetches the transcript). */
+export const BROADCAST_ONLY: ReadonlySet<string> = new Set(["delta", "reasoning"])
+
+/** One user event as it travels the Bus. */
+export interface LiveEvent {
+  /** The pod that sent it. */
+  readonly pod: string
+  readonly message: EventMessage
+  /** Its EventLog id (absent for streamed text). */
+  readonly id?: string
+  /** Not for this device (the one that made the change). */
+  readonly exceptDevice?: string
+}
+
+export interface LiveSubscription {
+  /** Resolves once the pod listens on the user's topic. */
+  readonly ready: Promise<void>
+  close(): void
+}
+
+export interface LiveFanoutOptions {
+  readonly bus: Bus
+  readonly events: EventLog
+  /** This pod (HUB_POD_ID). */
+  readonly podID: string
+  readonly onError?: (error: unknown) => void
+}
+
+interface UserTopic {
+  readonly handlers: Set<(event: LiveEvent) => void>
+  readonly subscribed: Promise<Unsubscribe | undefined>
+}
+
+const isLiveEvent = (value: unknown): value is LiveEvent => {
+  const event = value as LiveEvent | null
+  return typeof event === "object" && event !== null && typeof event.message === "object" && event.message !== null && typeof event.message.type === "string"
+}
+
+export class LiveFanout {
+  readonly podID: string
+  private readonly options: LiveFanoutOptions
+  private readonly onError: (error: unknown) => void
+  /** Per user: the last send in flight (sends of one user go out in order). */
+  private readonly sending = new Map<string, Promise<void>>()
+  /** Users this pod listens to, with their local handlers. */
+  private readonly topics = new Map<string, UserTopic>()
+  private readonly catalogHandlers = new Set<() => void>()
+  private catalogSubscription?: Promise<Unsubscribe>
+
+  constructor(options: LiveFanoutOptions) {
+    this.options = options
+    this.podID = options.podID
+    this.onError = options.onError ?? ((error) => console.error(`[hub] live fan-out failed: ${error instanceof Error ? error.message : String(error)}`))
+  }
+
+  /** Listens for catalog announcements. */
+  async start(): Promise<void> {
+    this.catalogSubscription ??= this.options.bus.subscribe(CATALOG_TOPIC, () => {
+      for (const handler of [...this.catalogHandlers]) this.guard(handler)
+    })
+    await this.catalogSubscription
+  }
+
+  /** Drops this pod's subscriptions (once they are made: a Bus that never connected must not hold up a stop). */
+  async stop(): Promise<void> {
+    const catalog = this.catalogSubscription
+    this.catalogSubscription = undefined
+    const topics = [...this.topics.values()]
+    this.topics.clear()
+    for (const pending of [catalog, ...topics.map((topic) => topic.subscribed)]) void pending?.then((off) => off?.()).catch(this.onError)
+  }
+
+  /** Logs (unless streamed text) and publishes a user's event; resolves once it is on the Bus. */
+  send(userID: string, message: EventMessage, exceptDevice?: string): Promise<void> {
+    const previous = this.sending.get(userID) ?? Promise.resolve()
+    const next = previous.then(async () => {
+      const id = BROADCAST_ONLY.has(message.type) ? undefined : await this.options.events.append(userID, message, exceptDevice ? { exceptDevice } : {})
+      const event: LiveEvent = { pod: this.podID, message, ...(id ? { id } : {}), ...(exceptDevice ? { exceptDevice } : {}) }
+      await this.options.bus.publish(liveTopic(userID), event)
+    })
+    const tail = next.catch(this.onError)
+    this.sending.set(userID, tail)
+    void tail.then(() => {
+      if (this.sending.get(userID) === tail) this.sending.delete(userID)
+    })
+    return next
+  }
+
+  /** The user's events from every pod (this one included). */
+  subscribe(userID: string, handler: (event: LiveEvent) => void): LiveSubscription {
+    let topic = this.topics.get(userID)
+    if (!topic) {
+      const handlers = new Set<(event: LiveEvent) => void>()
+      const subscribed = this.options.bus
+        .subscribe(liveTopic(userID), (event) => {
+          if (!isLiveEvent(event)) return
+          for (const h of [...handlers]) this.guard(() => h(event))
+        })
+        .catch((error: unknown) => {
+          this.onError(error)
+          return undefined
+        })
+      topic = { handlers, subscribed }
+      this.topics.set(userID, topic)
+    }
+    const current = topic
+    current.handlers.add(handler)
+    let closed = false
+    return {
+      ready: current.subscribed.then(() => {}),
+      close: () => {
+        if (closed) return
+        closed = true
+        current.handlers.delete(handler)
+        if (current.handlers.size || this.topics.get(userID) !== current) return
+        this.topics.delete(userID)
+        void current.subscribed.then((off) => off?.()).catch(this.onError)
+      },
+    }
+  }
+
+  /** Tells every pod (this one included) that the stored machines changed. */
+  async publishCatalog(): Promise<void> {
+    await this.options.bus.publish(CATALOG_TOPIC, { pod: this.podID })
+  }
+
+  onCatalog(handler: () => void): () => void {
+    this.catalogHandlers.add(handler)
+    return () => void this.catalogHandlers.delete(handler)
+  }
+
+  private guard(fn: () => void): void {
+    try {
+      fn()
+    } catch (error) {
+      this.onError(error)
+    }
+  }
+}

package/src/central/cluster/recommend.ts

@@ -0,0 +1,59 @@
+// Recommendations on permissions held by another pod (design §5, §9; decision vkxkqiuf §5). An escalation shows the
+// agent's recommendation on the request's blocker (Registry.recommend), and only the pod holding the request (its
+// spoke) can change that blocker: the other pods see it through the cluster view. recommend() applies it here when
+// this pod holds the request, else publishes it on RECOMMEND_TOPIC, where every pod tries it on its own registry (only
+// the holder has it). Closing escalations needs no routing: the holder's permission-closed listener closes their
+// Decisions in the shared state.
+import type { Bus, Unsubscribe } from "../backend/types"
+import type { BlockerRecommendation } from "../../status"
+
+export const RECOMMEND_TOPIC = "cluster.recommend"
+
+interface RecommendMessage {
+  readonly from: string
+  readonly requestID: string
+  readonly recommendation: BlockerRecommendation
+}
+
+export interface ClusterRecommendationsOptions {
+  readonly bus: Bus
+  /** This pod (HUB_POD_ID). */
+  readonly podID: string
+  readonly registry: { recommend(requestID: string, recommendation: BlockerRecommendation): boolean }
+  readonly onError?: (error: unknown) => void
+}
+
+export class ClusterRecommendations {
+  private readonly options: ClusterRecommendationsOptions
+  private readonly onError: (error: unknown) => void
+  private unsubscribe: Unsubscribe | undefined
+
+  constructor(options: ClusterRecommendationsOptions) {
+    this.options = options
+    this.onError = options.onError ?? ((error) => console.error(`[hub] cluster recommendation: ${error instanceof Error ? error.message : String(error)}`))
+  }
+
+  /** Applies other pods' recommendations for requests this pod holds. */
+  async start(): Promise<void> {
+    if (this.unsubscribe) return
+    this.unsubscribe = await this.options.bus.subscribe(RECOMMEND_TOPIC, (message) => {
+      const m = message as Partial<RecommendMessage> | undefined
+      if (m?.from === this.options.podID || typeof m?.requestID !== "string" || typeof m.recommendation !== "object" || m.recommendation === null) return
+      this.options.registry.recommend(m.requestID, m.recommendation)
+    })
+  }
+
+  async stop(): Promise<void> {
+    const unsubscribe = this.unsubscribe
+    this.unsubscribe = undefined
+    await unsubscribe?.()
+  }
+
+  /** Shows the recommendation on the request's blocker: here, or (true: sent) on the pod holding it. */
+  recommend(requestID: string, recommendation: BlockerRecommendation): boolean {
+    if (this.options.registry.recommend(requestID, recommendation)) return true
+    const message: RecommendMessage = { from: this.options.podID, requestID, recommendation }
+    void this.options.bus.publish(RECOMMEND_TOPIC, message).catch((error) => this.onError(error))
+    return true
+  }
+}

package/src/central/cluster/rpc.ts

@@ -0,0 +1,150 @@
+// Pod RPC (design §9, decision vkxkqiuf §5): ops for a spoke or node connection held by another pod travel as a
+// Bus request on pod.<podID>.rpc. The serving pod runs the op on its own socket (with the same per-op timeout as a
+// local request and a tenant re-check, see SpokeHub) and replies {ok, result} or {ok:false, error:{name, message}};
+// the calling pod turns that back into SpokeError / NodeError / RequestTimeoutError / RequestTooLargeError, so
+// callers cannot tell a remote connection from a local one. The Bus waits RPC_SLACK_MS longer than the op so the
+// serving pod's own timeout answers first; a pod with no responder is gone (the connection is offline).
+// Session-token and act-token pushes (LOCAL_ONLY_OPS) never cross the Bus: refused on both ends.
+// Close notices (machine revoked, launch ended) are published to every pod, the sender included.
+import type { NodeRequest, ServerRequest } from "../../protocol"
+import { BusNoResponderError, BusTimeoutError, type Bus, type Unsubscribe } from "../backend/types"
+import { NodeError, RequestTimeoutError, RequestTooLargeError, SpokeError } from "../spoke-ws"
+
+/** How much longer the calling pod waits than the op's own timeout. */
+export const RPC_SLACK_MS = 2_000
+/** Ops that carry credentials for the spoke's shells and agents: sent only by the pod holding the connection. */
+export const LOCAL_ONLY_OPS: ReadonlySet<string> = new Set(["sessionTokens", "actTokens"])
+/** The topic a pod answers ops for its connections on. */
+export const rpcTopic = (podID: string) => `pod.${podID}.rpc`
+/** Close notices for every pod. */
+export const CLOSE_TOPIC = "cluster.close"
+
+/** An op for a connection on another pod; `tenant` is the user the calling pod acts for (checked there). */
+export type RpcCall =
+  | { readonly kind: "spoke"; readonly connID: string; readonly tenant: string; readonly req: ServerRequest }
+  | { readonly kind: "node"; readonly connID: string; readonly tenant: string; readonly req: NodeRequest }
+
+/** Disconnect a machine's sockets (revoked) or a launch's (ended) on every pod. */
+export interface CloseNotice {
+  readonly kind: "machine" | "launch"
+  readonly id: string
+  readonly reason: string
+  /** The pod that published it. */
+  readonly from: string
+}
+
+interface WireError {
+  readonly name: string
+  readonly message: string
+}
+
+type RpcReply = { readonly ok: true; readonly result?: unknown } | { readonly ok: false; readonly error: WireError }
+
+export interface PodRpcOptions {
+  readonly bus: Bus
+  /** This pod (HUB_POD_ID). */
+  readonly podID: string
+}
+
+export class PodRpc {
+  readonly podID: string
+  private readonly bus: Bus
+  private readonly unsubscribes: Unsubscribe[] = []
+
+  constructor(options: PodRpcOptions) {
+    this.bus = options.bus
+    this.podID = options.podID
+  }
+
+  /** Answers ops for this pod's connections with `handler` (its errors go back to the caller). */
+  async serve(handler: (call: RpcCall) => Promise<unknown>): Promise<void> {
+    const unsubscribe = await this.bus.serve(rpcTopic(this.podID), async (message): Promise<RpcReply> => {
+      const call = message as RpcCall
+      try {
+        if (!isCall(call)) throw new SpokeError("Invalid pod RPC call")
+        if (LOCAL_ONLY_OPS.has(call.req.op)) throw new SpokeError(`The ${call.req.op} op is not sent across pods`)
+        return { ok: true, result: await handler(call) }
+      } catch (cause) {
+        const error = cause instanceof Error ? cause : new Error(String(cause))
+        return { ok: false, error: { name: error.name, message: error.message } }
+      }
+    })
+    this.unsubscribes.push(unsubscribe)
+  }
+
+  /**
+   * Runs an op on `pod`'s connection and resolves with its result. `timeoutMs` is the op's own timeout; `onGone`
+   * is told when the pod has no responder (it is gone: forget its directory entry).
+   */
+  async call(pod: string, call: RpcCall, timeoutMs: number, options: { readonly onGone?: () => void } = {}): Promise<unknown> {
+    if (LOCAL_ONLY_OPS.has(call.req.op)) throw new SpokeError(`The ${call.req.op} op is not sent across pods`)
+    let reply: RpcReply
+    try {
+      reply = await this.bus.request<RpcReply>(rpcTopic(pod), call, { timeoutMs: timeoutMs + RPC_SLACK_MS })
+    } catch (cause) {
+      if (cause instanceof BusNoResponderError) {
+        options.onGone?.()
+        throw offline(call)
+      }
+      if (cause instanceof BusTimeoutError) throw timeout(call)
+      throw failed(call, cause instanceof Error ? cause.message : String(cause))
+    }
+    if (reply?.ok === true) return reply.result
+    if (reply?.ok === false && typeof reply.error?.message === "string") throw revive(call, reply.error)
+    throw failed(call, "Invalid pod RPC reply")
+  }
+
+  /** Tells every pod (this one included) to close a machine's or a launch's sockets. */
+  async publishClose(notice: Omit<CloseNotice, "from">): Promise<void> {
+    await this.bus.publish(CLOSE_TOPIC, { ...notice, from: this.podID } satisfies CloseNotice)
+  }
+
+  async onClose(handler: (notice: CloseNotice) => void): Promise<void> {
+    this.unsubscribes.push(
+      await this.bus.subscribe(CLOSE_TOPIC, (message) => {
+        const notice = message as CloseNotice
+        if ((notice?.kind === "machine" || notice?.kind === "launch") && typeof notice.id === "string") handler(notice)
+      }),
+    )
+  }
+
+  async stop(): Promise<void> {
+    await Promise.allSettled(this.unsubscribes.splice(0).map((unsubscribe) => unsubscribe()))
+  }
+}
+
+function isCall(call: RpcCall | undefined): call is RpcCall {
+  return (
+    (call?.kind === "spoke" || call?.kind === "node") &&
+    typeof call.connID === "string" &&
+    typeof call.tenant === "string" &&
+    typeof (call.req as { op?: unknown } | undefined)?.op === "string"
+  )
+}
+
+const offline = (call: RpcCall): Error => (call.kind === "node" ? new NodeError("The node disconnected") : new SpokeError("OpenCode process disconnected"))
+const timeout = (call: RpcCall): Error => (call.kind === "node" ? new RequestTimeoutError(call.req.op, "The node") : new RequestTimeoutError(call.req.op))
+const failed = (call: RpcCall, message: string): Error => (call.kind === "node" ? new NodeError(message) : new SpokeError(message))
+
+/** The serving pod's error as the class the caller would have got locally. */
+function revive(call: RpcCall, wire: WireError): Error {
+  let error: Error
+  switch (wire.name) {
+    case "RequestTimeoutError":
+      error = new RequestTimeoutError(call.req.op)
+      break
+    case "RequestTooLargeError":
+      error = new RequestTooLargeError(call.req.op)
+      break
+    case "SpokeError":
+      error = new SpokeError(wire.message)
+      break
+    case "NodeError":
+      error = new NodeError(wire.message)
+      break
+    default:
+      error = failed(call, wire.message)
+  }
+  error.message = wire.message
+  return error
+}