@earendil-works/chord 1.2.0 → 1.4.0

package/dist/connection.js

@@ -1,39 +1,54 @@
-import { contractFromDescription, LOCAL_VARIANT } from "./contract.js";
+import { contractFromDescription, LOCAL_VARIANT, resolveHost } from "./contract.js";
 import { encoder } from "./delta/index.js";
 import { ChordError, invalid } from "./errors.js";
-import { HandleImpl, kindOf } from "./record.js";
+import { Packed, unpack } from "./link.js";
+import { RefImpl, variantOf } from "./object.js";
 import { assertValid } from "./schema.js";
-import { fromWireError, isFrame, toWireError } from "./wire.js";
+import { fromWireError, isFrame, PROTOCOL_VERSION, toWireError } from "./wire.js";
+const DEFAULT_WINDOW = 256 * 1024;
+/** The default cap of waiting state ops per link, in windows. */
+const PENDING_WINDOWS = 4;
 const unavailable = (message) => new ChordError("unavailable", message);
 // ─── relay ───────────────────────────────────────────────────────────────────
 /**
- * A hold relayed from an origin link to a downstream link. It owns everything done through it: subscriptions and
- * calls queued while the route resolves, and the downstream twins once forwarded. `close` undoes all of it exactly
- * once (cancels calls downstream, drops subscriptions, releases the downstream hold, rejects waiters).
+ * A hold relayed from an origin link to a downstream link. It owns everything done through it: state and calls queued
+ * while the route resolves, and the downstream twins once forwarded. `close` undoes all of it exactly once (cancels
+ * calls downstream, drops what waits for the origin, releases the downstream hold, rejects waiters).
  */
-export class RelayHold {
+export class RelayRef {
     origin;
     frame;
     principal;
-    client;
+    /** The client connection the hold comes from (forwarded downstream); undefined for a host's own holds. */
+    originTag;
     /** The origin's authorization generation when the hold arrived; a revocation bumps it. */
     generation;
     down;
     downId = 0;
     closed = false;
+    /** The origin wants state (forwarded as `state: false`, `sub`, `unsub`). */
+    stateWanted;
+    /** The members of the first unit, still to forward past the origin's window. */
+    urgent;
+    #shaped = false;
+    /** Members whose ops this relay dropped: downstream ops are ignored until their reset arrives. */
+    awaiting = new Set();
     calls = new Map();
-    subs = new Map();
     #waiters = new Set();
-    constructor(origin, frame, principal, client) {
+    constructor(origin, frame, principal, originTag) {
         this.origin = origin;
         this.frame = frame;
         this.principal = principal;
-        this.client = client;
+        this.originTag = originTag;
+        this.stateWanted = frame.state !== false;
         this.generation = origin.generation;
     }
     get originId() {
         return this.frame.id;
     }
+    get client() {
+        return this.originTag !== undefined;
+    }
     /** Still authorized as it was when it arrived. */
     get current() {
         return !this.closed && this.origin.open && this.origin.generation === this.generation;
@@ -48,50 +63,77 @@ export class RelayHold {
             this.#waiters.add({ resolve: resolve, reject });
         });
     }
-    /** Forwards the hold (and what queued meanwhile) to `down`. */
+    /** Forwards the hold to `down`. */
     open(down) {
         if (this.closed)
             return;
         this.down = down;
         this.downId = down.nextId();
-        down.relayedHolds.set(this.downId, this);
+        down.relayedRefs.set(this.downId, this);
+        const { at, token, input } = this.frame;
         down.send({
-            ...this.frame,
+            t: "hold",
             id: this.downId,
+            at,
+            token,
+            input,
             principal: this.principal,
-            ...(this.client ? { client: true } : {}),
+            ...(this.originTag === undefined ? {} : { origin: this.originTag }),
+            ...(this.stateWanted ? {} : { state: false }),
+            ...(this.frame.again === true ? { again: true } : {}),
         });
-        for (const [originSubId, entry] of this.subs)
-            if (entry.downId === undefined)
-                this.#forwardSub(originSubId, entry);
         for (const waiter of this.#waiters)
             waiter.resolve(down);
         this.#waiters.clear();
     }
-    subscribe(originSubId, member) {
-        const entry = { member, downId: undefined };
-        this.subs.set(originSubId, entry);
-        this.origin.relaySubs.set(originSubId, this);
-        if (this.down !== undefined)
-            this.#forwardSub(originSubId, entry);
+    /** The origin starts or stops the hold's state. */
+    wantState(wanted) {
+        if (this.closed || wanted === this.stateWanted)
+            return;
+        this.stateWanted = wanted;
+        if (!wanted) {
+            this.origin.dropHoldState(this.originId);
+            this.awaiting.clear();
+        }
+        this.down?.send({ t: wanted ? "sub" : "unsub", hold: this.downId });
     }
-    #forwardSub(originSubId, entry) {
-        const down = this.down;
-        entry.downId = down.nextId();
-        down.relayedSubs.set(entry.downId, { relay: this, originSubId });
-        down.send({ t: "sub", id: entry.downId, hold: this.downId, member: entry.member });
+    /**
+     * The origin could not keep up and dropped what waited for member `m` (or asked for it again): ask downstream for a
+     * reset, and ignore its ops until the reset arrives.
+     */
+    resync(m) {
+        if (this.closed)
+            return;
+        this.awaiting.add(m);
+        this.down?.send({ t: "resync", hold: this.downId, m });
     }
-    unsubscribe(originSubId, forward = true) {
-        const entry = this.subs.get(originSubId);
-        if (entry === undefined)
+    /** A shape from downstream: forwarded at once; the members of the first unit then skip the origin's window. */
+    shape(frame) {
+        if (this.closed)
             return;
-        this.subs.delete(originSubId);
-        this.origin.relaySubs.delete(originSubId);
-        if (entry.downId !== undefined && this.down !== undefined) {
-            this.down.relayedSubs.delete(entry.downId);
-            if (forward)
-                this.down.send({ t: "unsub", id: entry.downId });
+        if (!this.#shaped) {
+            this.#shaped = true;
+            if (this.frame.again !== true)
+                this.urgent = new Set(frame.state);
         }
+        this.origin.send({ ...frame, id: this.originId });
+    }
+    /** A state frame from downstream. */
+    state(frame) {
+        if (this.closed)
+            return;
+        const m = frame.m;
+        if (this.awaiting.has(m)) {
+            if (frame.as === "ops")
+                return;
+            this.awaiting.delete(m);
+        }
+        const out = { ...frame, hold: this.originId };
+        if (this.urgent?.delete(m) === true) {
+            this.origin.send(out);
+            return;
+        }
+        this.origin.sendState(out, { resync: () => this.resync(m) });
     }
     /**
      * Closes the relay. `releaseDown`: the downstream hold is still there (origin released, disconnected, or was
@@ -103,12 +145,11 @@ export class RelayHold {
         this.closed = true;
         for (const call of [...this.calls.values()])
             call.cancel(error, options.releaseDown);
-        for (const originSubId of [...this.subs.keys()])
-            this.unsubscribe(originSubId, options.releaseDown);
+        this.origin.dropHoldState(this.originId);
         if (this.origin.served.get(this.originId)?.kind === "relay")
             this.origin.served.delete(this.originId);
         if (this.down !== undefined) {
-            this.down.relayedHolds.delete(this.downId);
+            this.down.relayedRefs.delete(this.downId);
             if (options.releaseDown)
                 this.down.send({ t: "release", id: this.downId });
         }
@@ -155,44 +196,97 @@ export class RelayCall {
     }
 }
 // ─── connection ──────────────────────────────────────────────────────────────
-/** One end of a link on a host: consumer of remote handles, provider for the peer's holds, and relay. */
+/**
+ * One end of a link on a host: consumer of remote references, provider for the peer's holds, and relay.
+ *
+ * State
+ * -----
+ *
+ * A hold implies the state of its variant (unless it says `state: false`). The provider answers with a unit: the shape,
+ * then a snapshot of each state member the consumer does not have yet (`shape.state` lists them), then the members'
+ * ops and resets. A reload or a variant change sends another unit with the members that are new; members that left the
+ * variant stop. The consumer applies a shape once its snapshots are here (`RemoteBinding`).
+ *
+ * Flow control
+ * ------------
+ *
+ * Problem: state is pushed as fast as it changes, but a peer may read slowly (a phone behind a relay), and a relay may
+ * hide that: Cloudflare forwards to browsers without backpressure. Everything sent then queues in front of the next
+ * call result. Example: a browser opens three long transcripts (500 KiB of snapshots each) over a 5 Mbit/s path; a
+ * result sent next waits 2.4 s, and the snapshot of a transcript the user already left is still sent in full.
+ *
+ * Solution: the peer acknowledges what it processed (`ack`, a cumulative frame count), and the sender counts the sizes
+ * of the frames not yet acknowledged. While they exceed the window, state waits here in slots, flushed in creation
+ * order as acknowledgements come:
+ * - a member's stream has one slot: ops are appended (encoded batches of one stream may be concatenated, see
+ *   `encoder`), a snapshot, reset or error replaces it;
+ * - a hold's unit has one slot, built lazily from the current values when it is sent, so a newer unit merges into it;
+ *   while it waits, the members it introduces have no stream yet.
+ *
+ * The first unit of a hold that is not a re-hold (`again`) is sent at once, past the window: a `get` waits for it, and
+ * must not wait behind other holds' state. Results, shapes, gone, reload control and pings are sent at once too: they
+ * are bounded by the requests the peer makes. Hence the one ordering exception of a link: while state waits, a result
+ * can arrive before state changes made before it. Frames of one stream stay in order.
+ *
+ * Waiting ops are capped per link (`pendingCap`): beyond, the oldest streams are replaced by their current value (a
+ * reset built at flush), or, on a relay, by a reset asked of the provider (`resync`), so memory stays bounded however
+ * long the peer stalls. The window bounds what is unacknowledged when a frame is sent, so one frame (a large snapshot)
+ * may exceed it. Both hellos must announce `acks`; with an older peer nothing is acknowledged and nothing waits. Links
+ * that do not report sizes (IPC) never fill the window.
+ */
 export class Connection {
     host;
     link;
     /** `upstream`: the link to this host's parent. `downstream`: a link this host accepted. */
     role;
     peer;
+    /** A client connection's opaque id: holds that come over it (directly or relayed) carry it as their `origin`. */
+    originId;
     principal;
     restamp;
     /** Bumped when the stamped principal is revoked or re-stamped: relays from before are refused. */
     generation = 0;
-    /** The epoch a downstream host peer runs (from its hello, then from commits). */
-    peerEpoch;
-    /** A downstream host peer is usable once it runs this host's current epoch (see `HostImpl.peerHello`). */
+    /** The code version a downstream host peer runs (from its hello, then from commits). */
+    peerCodeVersion;
+    /** A downstream host peer is usable once it runs this host's current code version (see `HostImpl.peerHello`). */
     synced;
-    /** Epochs this child announced while behind this host (fenced): this host never ran them. */
-    fencedEpochs = new Set();
-    /** Roots of fenced epochs, by the epoch of this host they were admitted in. */
+    /** Code versions this child announced while behind this host (fenced): this host never ran them. */
+    fencedCodeVersions = new Set();
+    /** Roots of fenced code versions, by the code version of this host they were admitted in. */
     #admitted = new Map();
     #parentName;
     #syncWaiters = new Set();
     #nextId = 1;
     #lastSeen = Date.now();
     #pinger;
+    #maxCalls;
+    // flow control (see the class comment)
+    #window;
+    #pendingCap;
+    /** Whether the peer acknowledges frames; until its hello is here, frames are counted in case it does. */
+    #peerAcks;
+    #sent = 0;
+    #acked = 0;
+    /** Sizes of the frames sent after the last acknowledged one. */
+    #unacked = [];
+    #unackedBytes = 0;
+    #pending = new Map();
+    /** Bytes of ops waiting in `#pending`. */
+    #pendingBytes = 0;
+    #received = 0;
+    #ackTimer;
+    #onProtocolMismatch;
+    /** The peer speaks another protocol version: nothing it sends is processed any more. */
+    #mismatched = false;
     // consumer side: what this host asked of the peer
     remotes = new Map();
     calls = new Map();
-    subs = new Map();
     // provider side: what the peer asked of this host
     served = new Map();
     incoming = new Map();
-    feeds = new Map();
-    /** Origin side of relays: the peer's sub ids that are relayed. */
-    relaySubs = new Map();
     // downstream side of relays: ids this host used on this link
-    relayedHolds = new Map();
+    relayedRefs = new Map();
     relayedCalls = new Map();
-    relayedSubs = new Map();
     /** Reload votes awaited from this (child) peer. */
     votes = new Map();
     constructor(host, link, config) {
@@ -200,14 +294,29 @@ export class Connection {
         this.link = link;
         this.role = config.role;
         this.peer = config.peer;
+        this.originId = config.originId;
         this.principal = config.principal;
         this.restamp = config.restamp;
         this.synced = !(this.role === "downstream" && this.peer.kind === "host");
+        this.#window = config.window ?? DEFAULT_WINDOW;
+        this.#pendingCap = config.pendingCap ?? this.#window * PENDING_WINDOWS;
+        this.#maxCalls = config.maxCalls ?? Number.POSITIVE_INFINITY;
+        this.#onProtocolMismatch = config.onProtocolMismatch;
         link.onClose((reason) => this.#closed(reason));
         link.receive((frame) => this.#receive(frame));
-        this.send({ t: "hello", name: host.name, kind: host.kind, epoch: host.current.id });
+        this.send({
+            t: "hello",
+            protocol: PROTOCOL_VERSION,
+            name: host.name,
+            kind: host.kind,
+            codeVersion: host.current.id,
+            acks: true,
+        });
         const interval = config.pingInterval ?? 15_000;
         const timeout = config.pingTimeout ?? 45_000;
+        // A link that was already closed closed this connection above; it needs no keepalive.
+        if (link.closed)
+            return;
         let n = 0;
         this.#pinger = setInterval(() => {
             if (Date.now() - this.#lastSeen > timeout)
@@ -225,12 +334,178 @@ export class Connection {
         return this.role === "upstream" ? this.#parentName : this.peer.name;
     }
     send(frame) {
-        this.link.send(frame);
+        const size = this.link.send(frame) ?? 0;
+        if (this.#peerAcks === false)
+            return;
+        this.#sent++;
+        this.#unacked.push(size);
+        this.#unackedBytes += size;
+    }
+    /** Sends a state frame, or keeps it waiting (see the class comment). */
+    sendState(frame, overflow) {
+        const key = stateKey(frame.hold, frame.m);
+        let slot = this.#pending.get(key);
+        if (slot === undefined && !this.#full) {
+            this.send(frame);
+            return;
+        }
+        if (slot?.kind === "unit")
+            return;
+        if (slot === undefined) {
+            slot = {
+                kind: "state",
+                hold: frame.hold,
+                m: frame.m,
+                base: undefined,
+                ops: [],
+                size: 0,
+                lazy: false,
+                overflow,
+            };
+            this.#pending.set(key, slot);
+        }
+        if (frame.as !== "ops") {
+            this.#pendingBytes -= slot.size;
+            slot.base = frame;
+            slot.ops = [];
+            slot.size = 0;
+            slot.lazy = false;
+            return;
+        }
+        // A reset built at flush already covers these ops.
+        if (slot.lazy)
+            return;
+        const size = JSON.stringify(frame.ops).length;
+        if (this.#pendingBytes + size > this.#pendingCap)
+            this.#makeRoom(size, slot);
+        if (slot.lazy || this.#pending.get(key) !== slot)
+            return;
+        for (const op of frame.ops ?? [])
+            slot.ops.push(op);
+        slot.size += size;
+        this.#pendingBytes += size;
+    }
+    /** Replaces the oldest waiting streams by resets until `size` more bytes fit; `current` last, if needed. */
+    #makeRoom(size, current) {
+        for (const slot of [...this.#pending.values()]) {
+            if (this.#pendingBytes + size <= this.#pendingCap)
+                return;
+            if (slot.kind === "state" && slot !== current && slot.size > 0)
+                this.#drop(slot);
+        }
+        if (this.#pendingBytes + size > this.#pendingCap)
+            this.#drop(current);
+    }
+    /** Drops what waits for a stream: a lazy reset replaces it (provider), or the relay asks downstream for one. */
+    #drop(slot) {
+        this.#pendingBytes -= slot.size;
+        slot.base = undefined;
+        slot.ops = [];
+        slot.size = 0;
+        if ("resend" in slot.overflow) {
+            slot.lazy = true;
+            return;
+        }
+        this.#pending.delete(stateKey(slot.hold, slot.m));
+        slot.overflow.resync();
+    }
+    /** Sends member `m` of a served hold again as a reset, through the window (`resync`). */
+    #resetLater(hold, m, resend) {
+        const key = stateKey(hold, m);
+        let slot = this.#pending.get(key);
+        if (slot?.kind === "unit")
+            return;
+        if (slot === undefined) {
+            slot = { kind: "state", hold, m, base: undefined, ops: [], size: 0, lazy: true, overflow: { resend } };
+            this.#pending.set(key, slot);
+        }
+        else {
+            this.#pendingBytes -= slot.size;
+            slot.base = undefined;
+            slot.ops = [];
+            slot.size = 0;
+            slot.lazy = true;
+        }
+        this.#flush();
+    }
+    /** Queues a hold's lazy unit (a waiting one already covers this one), and sends what the window allows. */
+    #queueUnit(hold, flush) {
+        const key = unitKey(hold);
+        if (!this.#pending.has(key))
+            this.#pending.set(key, { kind: "unit", hold, flush });
+        this.#flush();
+    }
+    /** Forgets what waits for a stream that ended. */
+    dropState(hold, m) {
+        const slot = this.#pending.get(stateKey(hold, m));
+        if (slot?.kind !== "state")
+            return;
+        this.#pendingBytes -= slot.size;
+        this.#pending.delete(stateKey(hold, m));
+    }
+    /** Forgets everything that waits for a hold (released, closed, or its state stopped). */
+    dropHoldState(hold) {
+        for (const [key, slot] of this.#pending) {
+            if (slot.hold !== hold)
+                continue;
+            if (slot.kind === "state")
+                this.#pendingBytes -= slot.size;
+            this.#pending.delete(key);
+        }
+    }
+    /** What waits for the peer: slots and bytes of ops (for tests and diagnostics). */
+    get waiting() {
+        return { slots: this.#pending.size, bytes: this.#pendingBytes };
+    }
+    get #full() {
+        return this.#peerAcks === true && this.#unackedBytes >= this.#window;
+    }
+    #onAck(n) {
+        if (this.#peerAcks !== true || n < this.#acked || n > this.#sent) {
+            this.#violation(`ack ${n} outside ${this.#acked}..${this.#sent}`);
+            return;
+        }
+        for (const size of this.#unacked.splice(0, n - this.#acked))
+            this.#unackedBytes -= size;
+        this.#acked = n;
+        this.#flush();
+    }
+    /** Sends waiting slots in creation order while the window allows. */
+    #flush() {
+        for (const [key, slot] of this.#pending) {
+            if (this.#full)
+                return;
+            this.#pending.delete(key);
+            if (slot.kind === "unit") {
+                slot.flush();
+                continue;
+            }
+            this.#pendingBytes -= slot.size;
+            if (slot.lazy) {
+                if ("resend" in slot.overflow)
+                    slot.overflow.resend();
+                continue;
+            }
+            if (slot.base !== undefined)
+                this.send(slot.base);
+            if (slot.ops.length > 0)
+                this.send({ t: "state", hold: slot.hold, m: slot.m, as: "ops", ops: slot.ops });
+        }
+    }
+    /** Acknowledges once the frames that arrived together are processed: one ack per burst, not per frame. */
+    #scheduleAck() {
+        if (this.#peerAcks !== true || this.#ackTimer !== undefined || !this.open)
+            return;
+        this.#ackTimer = setTimeout(() => {
+            this.#ackTimer = undefined;
+            this.link.send({ t: "ack", n: this.#received });
+        }, 0);
+        this.#ackTimer.unref?.();
     }
     nextId() {
         return this.#nextId++;
     }
-    /** Resolves once the peer is usable (a host peer runs this host's epoch); rejects if the link closes first. */
+    /** Resolves once the peer is usable (a host peer runs this host's code version); rejects if the link closes first. */
     whenSynced() {
         if (!this.open)
             return Promise.reject(unavailable(`link to ${this.name ?? "peer"} closed`));
@@ -238,8 +513,8 @@ export class Connection {
             return Promise.resolve();
         return new Promise((resolve, reject) => this.#syncWaiters.add({ resolve: resolve, reject }));
     }
-    markSynced(epoch) {
-        this.peerEpoch = epoch;
+    markSynced(codeVersion) {
+        this.peerCodeVersion = codeVersion;
         this.synced = true;
         for (const waiter of this.#syncWaiters)
             waiter.resolve(undefined);
@@ -258,25 +533,43 @@ export class Connection {
         else
             this.principal = next;
     }
+    /**
+     * Stops talking to a peer of another protocol version: reports it and closes the link. The close reason tells the
+     * peer this side's version (`protocol-version <n>`); a peer that noticed first told us the same way (`#closed`).
+     */
+    #protocolMismatch(peer, close = true) {
+        if (this.#mismatched)
+            return;
+        this.#mismatched = true;
+        try {
+            this.#onProtocolMismatch?.({ local: PROTOCOL_VERSION, peer });
+        }
+        catch (error) {
+            this.host.report(error);
+        }
+        if (close)
+            this.link.close(`protocol-version ${PROTOCOL_VERSION}`);
+    }
     /** A protocol violation: the peer cannot be trusted with anything on this link any more. */
     #violation(reason) {
         this.host.report(new ChordError("invalid", `link to ${this.name ?? "peer"}: ${reason}`, { reason: "protocol" }));
         this.link.close(`protocol error: ${reason}`);
     }
     // ─── consumer side (RemoteChannel) ───
-    hold(handle) {
+    hold(binding, again) {
         const id = this.nextId();
-        this.remotes.set(id, handle);
-        handle.bound(this, id);
-        const principal = handle.principal;
+        this.remotes.set(id, binding);
+        binding.bound(this, id);
+        const principal = binding.principal;
         this.send({
             t: "hold",
             id,
-            at: handle.at,
-            token: handle.contract.description,
-            input: handle.input,
+            at: binding.at,
+            token: binding.contract.description,
+            input: binding.input,
             ...(principal === undefined ? {} : { principal }),
-            ...(handle.owner === "client" ? { client: true } : {}),
+            ...(binding.stateOn ? {} : { state: false }),
+            ...(again ? { again: true } : {}),
         });
         return id;
     }
@@ -284,6 +577,10 @@ export class Connection {
         this.remotes.delete(id);
         this.send({ t: "release", id });
     }
+    wantState(hold, wanted) {
+        if (this.remotes.has(hold))
+            this.send({ t: wanted ? "sub" : "unsub", hold });
+    }
     call(hold, member, args, root, signal) {
         const id = this.nextId();
         return new Promise((resolve, reject) => {
@@ -308,36 +605,34 @@ export class Connection {
                     reject(error);
                 },
             });
-            // Clients never carry roots or epochs: the server starts roots for them.
+            // Clients never carry roots or code versions: the server starts roots for them.
             const carried = root !== undefined && this.host.kind === "host";
             if (carried)
                 this.host.rootSent(root.rootId, this);
             this.send({ t: "call", id, hold, member, args, ...(carried ? { root } : {}) });
         });
     }
-    sub(hold, member, sink) {
-        const id = this.nextId();
-        this.subs.set(id, sink);
-        this.send({ t: "sub", id, hold, member });
-        return id;
-    }
-    unsub(id) {
-        if (this.subs.delete(id))
-            this.send({ t: "unsub", id });
-    }
     // ─── receiving ───
     #receive(frame) {
+        if (this.#mismatched)
+            return;
         this.#lastSeen = Date.now();
         if (!isFrame(frame)) {
             this.#violation("malformed frame");
             return;
         }
+        if (frame.t === "ack") {
+            this.#onAck(frame.n);
+            return;
+        }
+        this.#received++;
         try {
             this.#dispatch(frame);
         }
         catch (error) {
             this.host.report(error);
         }
+        this.#scheduleAck();
     }
     #dispatch(frame) {
         switch (frame.t) {
@@ -349,6 +644,7 @@ export class Connection {
                 return;
             case "pong":
             case "bye":
+            case "ack":
                 return;
             case "hold":
                 this.#onHold(frame);
@@ -363,10 +659,11 @@ export class Connection {
                 this.incoming.get(frame.id)?.abort(new ChordError("aborted", "cancelled by caller"));
                 return;
             case "sub":
-                this.#onSub(frame);
-                return;
             case "unsub":
-                this.#onUnsub(frame.id);
+                this.#onWantState(frame.hold, frame.t === "sub");
+                return;
+            case "resync":
+                this.#onResync(frame.hold, frame.m);
                 return;
             case "shape":
             case "gone":
@@ -397,7 +694,7 @@ export class Connection {
                     this.host.invalidatePrincipal(frame.pattern).catch((error) => this.host.report(error));
                 }
                 else if (frame.t === "retire")
-                    this.host.allowRetire(frame.epoch);
+                    this.host.allowRetire(frame.codeVersion);
                 else
                     this.host.onReloadFrame(this, frame);
                 return;
@@ -417,10 +714,21 @@ export class Connection {
         }
     }
     /**
-     * The peer's hello must agree with what this side decided about it. A downstream host announces its epoch: the
-     * host either finds it in step or fences and reconciles it (never renumbers).
+     * The peer's hello must agree with what this side decided about it. A downstream host announces its code version:
+     * the host either finds it in step or fences and reconciles it (never renumbers).
      */
     #onHello(frame) {
+        if ((frame.protocol ?? 1) !== PROTOCOL_VERSION) {
+            this.#protocolMismatch(frame.protocol ?? 1);
+            return;
+        }
+        if (this.#peerAcks === undefined) {
+            this.#peerAcks = frame.acks === true;
+            if (!this.#peerAcks) {
+                this.#unacked = [];
+                this.#unackedBytes = 0;
+            }
+        }
         if (this.role === "upstream") {
             if (frame.kind !== "host") {
                 this.#violation("parent is not a host");
@@ -434,19 +742,23 @@ export class Connection {
             return;
         }
         if (this.peer.kind === "host")
-            this.host.peerHello(this, frame.epoch);
+            this.host.peerHello(this, frame.codeVersion);
     }
     /**
      * Principal stamping: a downstream link acts as the principal it was accepted with, whatever the frame says. Only
-     * the parent may forward a principal (relayed requests); its own requests act as `{ host: parentName }`.
+     * the parent may forward a principal (relayed requests); its own requests act as the principal given to `connect`.
      */
     #principalOf(frame) {
         if (this.role === "downstream")
             return this.principal;
-        return frame.principal ?? { host: this.#parentName ?? "parent" };
+        return frame.principal ?? this.principal;
     }
-    #isClient(frame) {
-        return this.role === "downstream" ? this.peer.kind === "client" : frame.client === true;
+    /**
+     * The client connection a hold comes from: this link's own id for a client link (whatever the frame says), the id
+     * the parent forwarded for a relayed hold, none for a host's own holds.
+     */
+    #originOf(frame) {
+        return this.role === "downstream" ? this.originId : frame.origin;
     }
     #onHold(frame) {
         if (this.served.has(frame.id)) {
@@ -454,55 +766,214 @@ export class Connection {
             return;
         }
         const principal = this.#principalOf(frame);
-        const client = this.#isClient(frame);
+        const origin = this.#originOf(frame);
+        const client = origin !== undefined;
         if (frame.at !== this.host.name) {
             if (this.role === "upstream") {
                 this.send({ t: "gone", id: frame.id, error: toWireError(unavailable(`${this.host.name} does not relay`)) });
                 return;
             }
-            const relay = new RelayHold(this, frame, principal, client);
+            const relay = new RelayRef(this, frame, principal, origin);
             this.served.set(frame.id, { kind: "relay", relay });
             this.host.relay(relay).catch((error) => relay.close(error, { releaseDown: true, notify: true }));
             return;
         }
-        let handle;
+        let ref;
         try {
             const token = contractFromDescription(frame.token);
             if (token.scope !== "remote")
                 throw invalid("token", `${token.name} is local; only remote types cross links`);
+            // Addressed here, so the input must place it here (also by this host's contract, checked when acquiring).
+            if (resolveHost(token.name, token.description.host, frame.input) !== this.host.name)
+                throw invalid("placement", `${token.name} with this input does not live on ${this.host.name}`);
             const acquired = this.host.acquire(token, frame.input, { principal }, client ? "client" : "connection");
-            if (!(acquired instanceof HandleImpl))
-                throw unavailable("not a local handle");
-            handle = acquired;
+            if (!(acquired instanceof RefImpl))
+                throw unavailable("not a local reference");
+            ref = acquired;
         }
         catch (error) {
             this.send({ t: "gone", id: frame.id, error: toWireError(client ? clientError(error) : error) });
             return;
         }
-        const served = { kind: "local", handle, principal, feeds: new Set() };
-        this.served.set(frame.id, served);
-        const shape = () => {
-            const current = handle.shape;
-            if (current === undefined || this.served.get(frame.id) !== served)
-                return;
-            this.send({
-                t: "shape",
-                id: frame.id,
-                kind: current.kind,
-                members: current.members,
-                contract: current.contract,
-                epoch: current.epoch,
-            });
+        const served = {
+            kind: "local",
+            id: frame.id,
+            ref,
+            principal,
+            origin,
+            wantsState: frame.state !== false,
+            again: frame.again === true,
+            unitSent: false,
+            feeds: new Map(),
         };
-        handle.onShape(shape);
-        handle.goneListeners.add((error) => {
+        this.served.set(frame.id, served);
+        ref.onShape(() => this.#unit(served));
+        ref.goneListeners.add((error) => {
             if (this.served.get(frame.id) !== served)
                 return;
             this.#dropServed(frame.id, false);
             this.send({ t: "gone", id: frame.id, error: toWireError(error) });
         });
-        if (handle.status === "live")
-            shape();
+        if (ref.status === "live")
+            this.#unit(served);
+    }
+    /** The state members of `shape` the consumer's contract knows. */
+    #stateMembers(ref, shape) {
+        return shape.members.filter((name) => ref.describe(shape.variant, name)?.kind === "state");
+    }
+    /**
+     * The shape changed (live, reload, variant) or state was asked for: members that left lose their stream, and a unit
+     * goes out with the shape and the members the consumer lacks. The first unit of a hold (not a re-hold) is urgent.
+     */
+    #unit(served) {
+        if (this.served.get(served.id) !== served)
+            return;
+        const shape = served.ref.shape;
+        if (shape === undefined)
+            return;
+        const members = new Set(served.wantsState ? this.#stateMembers(served.ref, shape) : []);
+        for (const [name, feed] of served.feeds) {
+            if (members.has(name))
+                continue;
+            feed.unsubscribe();
+            served.feeds.delete(name);
+            this.dropState(served.id, name);
+        }
+        const urgent = served.feeds.size === 0 && !(served.again && !served.unitSent);
+        if (urgent)
+            this.#sendUnit(served);
+        else
+            this.#queueUnit(served.id, () => this.#sendUnit(served));
+    }
+    /** Builds and sends a unit from the current shape and values. */
+    #sendUnit(served) {
+        this.#pending.delete(unitKey(served.id));
+        if (this.served.get(served.id) !== served)
+            return;
+        const shape = served.ref.shape;
+        if (shape === undefined)
+            return;
+        served.unitSent = true;
+        const members = served.wantsState ? this.#stateMembers(served.ref, shape) : [];
+        const listed = members.filter((name) => !served.feeds.has(name));
+        const snaps = [];
+        const attached = [];
+        for (const name of listed) {
+            const attachment = this.#attach(served, name);
+            if (attachment === undefined)
+                continue;
+            snaps.push(attachment.frame);
+            attached.push([name, attachment.feed]);
+        }
+        this.send({
+            t: "shape",
+            id: served.id,
+            ...(shape.variant === undefined ? {} : { variant: shape.variant }),
+            members: shape.members,
+            contract: shape.contract,
+            state: snaps.map((frame) => frame.m),
+        });
+        for (const frame of snaps)
+            this.send(frame);
+        for (const [name, feed] of attached)
+            this.#replay(served, name, feed);
+    }
+    /**
+     * Subscribes member `name` for the consumer: its snapshot (as a frame of type `as`), and a feed whose later events
+     * wait in its buffer until the snapshot is sent (`#replay`). Undefined if the member is not readable state.
+     */
+    #attach(served, name, as = "snap") {
+        const readable = served.ref.member(name);
+        if (readable === undefined || typeof readable.events !== "function")
+            return undefined;
+        const feed = { unsubscribe: () => { }, encoder: encoder(), buffer: [] };
+        let first;
+        feed.unsubscribe = readable.events((event) => {
+            if (first === undefined)
+                first = event;
+            else if (feed.buffer !== undefined)
+                feed.buffer.push(event);
+            else
+                this.#feedEvent(served, name, feed, event);
+        });
+        served.feeds.set(name, feed);
+        return { frame: this.#snapshot(served, name, as, first?.value), feed };
+    }
+    /** Sends the events that arrived while the snapshot was built, now that it is sent. */
+    #replay(served, name, feed) {
+        const buffered = feed.buffer ?? [];
+        feed.buffer = undefined;
+        for (const event of buffered)
+            this.#feedEvent(served, name, feed, event);
+    }
+    /** A member's state frame for a snapshot or reset: validated against the schema the consumer is promised. */
+    #snapshot(served, name, as, value) {
+        try {
+            validateSnapshot(served.ref, name, value);
+        }
+        catch (error) {
+            return { t: "state", hold: served.id, m: name, as: "error", error: toWireError(error) };
+        }
+        return { t: "state", hold: served.id, m: name, as, ...(value === undefined ? {} : { value }) };
+    }
+    /** A later event of a member's stream: ops are encoded, a reset starts a new stream. */
+    #feedEvent(served, name, feed, event) {
+        if (served.feeds.get(name) !== feed)
+            return;
+        const overflow = { resend: () => this.#resend(served, name) };
+        if (event.type === "ops") {
+            this.sendState({ t: "state", hold: served.id, m: name, as: "ops", ops: feed.encoder.encode(event.ops) }, overflow);
+            return;
+        }
+        feed.encoder = encoder();
+        this.sendState(this.#snapshot(served, name, "reset", event.value), overflow);
+    }
+    /** Sends member `name` again, now, as a reset from its current value (a lazy slot is flushed, or `resync`). */
+    #resend(served, name) {
+        const feed = served.feeds.get(name);
+        if (feed === undefined || this.served.get(served.id) !== served)
+            return;
+        feed.unsubscribe();
+        served.feeds.delete(name);
+        const attachment = this.#attach(served, name, "reset");
+        if (attachment === undefined)
+            return;
+        this.send(attachment.frame);
+        this.#replay(served, name, attachment.feed);
+    }
+    /** The consumer starts (`sub`) or stops (`unsub`) the state of a hold. */
+    #onWantState(hold, wanted) {
+        const served = this.served.get(hold);
+        if (served === undefined)
+            return;
+        if (served.kind === "relay") {
+            served.relay.wantState(wanted);
+            return;
+        }
+        if (served.wantsState === wanted)
+            return;
+        served.wantsState = wanted;
+        if (wanted) {
+            this.#unit(served);
+            return;
+        }
+        for (const feed of served.feeds.values())
+            feed.unsubscribe();
+        served.feeds.clear();
+        this.dropHoldState(hold);
+    }
+    /** The consumer (a relay that dropped ops) asks for member `m` again as a reset. */
+    #onResync(hold, m) {
+        const served = this.served.get(hold);
+        if (served === undefined)
+            return;
+        if (served.kind === "relay") {
+            served.relay.resync(m);
+            return;
+        }
+        if (!served.feeds.has(m))
+            return;
+        this.#resetLater(hold, m, () => this.#resend(served, m));
     }
     /** Forgets a served hold: unsubscribes its feeds and releases it (locally or downstream). */
     #dropServed(id, release) {
@@ -517,16 +988,23 @@ export class Connection {
             });
             return;
         }
-        for (const feed of served.feeds)
-            this.#onUnsub(feed);
+        for (const feed of served.feeds.values())
+            feed.unsubscribe();
+        served.feeds.clear();
+        this.dropHoldState(id);
         if (release)
-            served.handle.release();
+            served.ref.release();
     }
     #onCall(frame) {
         if (this.incoming.has(frame.id)) {
             this.#violation(`duplicate call id ${frame.id}`);
             return;
         }
+        if (this.incoming.size >= this.#maxCalls) {
+            const error = new ChordError("unavailable", `Too many calls in progress on this connection (${this.#maxCalls}); wait for earlier ones to finish`, { reason: "too_many_calls" });
+            this.send({ t: "result", id: frame.id, error: toWireError(error) });
+            return;
+        }
         const served = this.served.get(frame.hold);
         if (served === undefined) {
             this.send({ t: "result", id: frame.id, error: toWireError(new ChordError("gone", "unknown hold")) });
@@ -539,107 +1017,53 @@ export class Connection {
         const principal = served.kind === "local" ? served.principal : served.relay.principal;
         const options = { principal, signal: controller.signal };
         const run = served.kind === "local"
-            ? (ctx) => served.handle.record.call(served.handle, frame.member, frame.args, ctx)
-            : (ctx) => relayCall(served.relay, frame, { rootId: ctx.rootId, epoch: ctx.epoch }, controller.signal, this.host);
+            ? (ctx) => served.ref.object.call(served.ref, frame.member, unpack(frame.args), ctx)
+            : (ctx) => relayCall(served.relay, frame, { rootId: ctx.rootId, codeVersion: ctx.codeVersion }, 
+            // The root's signal: a cancel from the origin and the root's code version retiring both reach it.
+            ctx.signal, this.host);
         const result = root === undefined
             ? this.host.root(run, options)
-            : this.host.joinRoot(root.rootId, root.epoch, run, { ...options, from: this });
+            : this.host.joinRoot(root.rootId, root.codeVersion, run, { ...options, from: this });
         result
             .then((value) => this.send({ t: "result", id: frame.id, value }), (error) => this.send({ t: "result", id: frame.id, error: toWireError(error) }))
             .finally(() => this.incoming.delete(frame.id));
     }
     /**
-     * A child that connected behind this host runs roots of an epoch this host never had (it was fenced and is being,
-     * or was, reconciled). Such a root is admitted in this host's current epoch at its first call here, and keeps that
-     * epoch for its later calls (`superseded` once this host retired it). Other unknown epochs stay `superseded`.
+     * A child that connected behind this host runs roots of a code version this host never had (it was fenced and is
+     * being, or was, reconciled). Such a root is admitted in this host's current code version at its first call here,
+     * and keeps that code version for its later calls (`superseded` once this host retired it). Other unknown code
+     * versions stay `superseded`.
      */
     #admitRoot(root) {
-        if (root === undefined || this.role !== "downstream" || !this.fencedEpochs.has(root.epoch))
+        if (root === undefined || this.role !== "downstream" || !this.fencedCodeVersions.has(root.codeVersion))
             return root;
-        if (this.host.hasEpoch(root.epoch))
+        if (this.host.hasCodeVersion(root.codeVersion))
             return root;
-        let epoch = this.#admitted.get(root.rootId);
-        if (epoch === undefined) {
-            epoch = this.host.current.id;
-            this.#admitted.set(root.rootId, epoch);
+        let codeVersion = this.#admitted.get(root.rootId);
+        if (codeVersion === undefined) {
+            codeVersion = this.host.current.id;
+            this.#admitted.set(root.rootId, codeVersion);
         }
-        return { rootId: root.rootId, epoch };
-    }
-    #onSub(frame) {
-        if (this.feeds.has(frame.id) || this.relaySubs.has(frame.id)) {
-            this.#violation(`duplicate sub id ${frame.id}`);
-            return;
-        }
-        const served = this.served.get(frame.hold);
-        if (served === undefined) {
-            this.send({ t: "state", id: frame.id, error: toWireError(new ChordError("gone", "unknown hold")) });
-            return;
-        }
-        if (served.kind === "relay") {
-            served.relay.subscribe(frame.id, frame.member);
-            return;
-        }
-        const readable = served.handle.member(frame.member);
-        if (readable === undefined || typeof readable.events !== "function") {
-            this.send({
-                t: "state",
-                id: frame.id,
-                error: toWireError(new ChordError("not_in_variant", `${frame.member} is not state`)),
-            });
-            return;
-        }
-        let stream = encoder();
-        const send = (event) => {
-            if (event.type === "ops") {
-                this.send({ t: "state", id: frame.id, ops: stream.encode(event.ops) });
-                return;
-            }
-            // A loading provider handle has nothing yet: the hydrated value follows as a reset.
-            if (event.value === undefined)
-                return;
-            // Snapshots are validated against the effective schema once per (re)subscription; ops only in dev mode.
-            try {
-                validateSnapshot(served.handle, frame.member, event.value);
-            }
-            catch (error) {
-                this.send({ t: "state", id: frame.id, error: toWireError(error) });
-                return;
-            }
-            stream = encoder();
-            this.send(event.type === "snap"
-                ? { t: "state", id: frame.id, snap: event.value }
-                : { t: "state", id: frame.id, reset: event.value });
-        };
-        const unsubscribe = readable.events(send);
-        this.feeds.set(frame.id, unsubscribe);
-        served.feeds.add(frame.id);
-    }
-    #onUnsub(id) {
-        const unsubscribe = this.feeds.get(id);
-        if (unsubscribe !== undefined) {
-            this.feeds.delete(id);
-            unsubscribe();
-        }
-        this.relaySubs.get(id)?.unsubscribe(id);
+        return { rootId: root.rootId, codeVersion };
     }
     #onHoldReply(frame) {
-        const relay = this.relayedHolds.get(frame.id);
+        const relay = this.relayedRefs.get(frame.id);
         if (relay !== undefined) {
             if (frame.t === "gone")
                 relay.close(fromWireError(frame.error), { releaseDown: false, notify: true });
-            else if (!relay.closed)
-                relay.origin.send({ ...frame, id: relay.originId });
+            else
+                relay.shape(frame);
             return;
         }
-        const handle = this.remotes.get(frame.id);
-        if (handle === undefined)
+        const binding = this.remotes.get(frame.id);
+        if (binding === undefined)
             return;
         if (frame.t === "gone") {
             this.remotes.delete(frame.id);
-            handle.gone(fromWireError(frame.error));
+            binding.gone(fromWireError(frame.error));
             return;
         }
-        handle.shaped({ kind: frame.kind, members: frame.members, contract: frame.contract, epoch: frame.epoch });
+        binding.shaped({ variant: frame.variant, members: frame.members, contract: frame.contract }, frame.state);
     }
     #onResult(frame) {
         const relayed = this.relayedCalls.get(frame.id);
@@ -649,23 +1073,54 @@ export class Connection {
         }
         const pending = this.calls.get(frame.id);
         this.calls.delete(frame.id);
-        if (frame.error === undefined)
-            pending?.resolve(frame.value);
-        else
+        if (frame.error !== undefined) {
             pending?.reject(fromWireError(frame.error));
+            return;
+        }
+        let value;
+        try {
+            value = unpack(frame.value);
+        }
+        catch (error) {
+            pending?.reject(error);
+            return;
+        }
+        pending?.resolve(value);
     }
     #onState(frame) {
-        const relayed = this.relayedSubs.get(frame.id);
-        if (relayed !== undefined) {
-            relayed.relay.origin.send({ ...frame, id: relayed.originSubId });
+        const relay = this.relayedRefs.get(frame.hold);
+        if (relay !== undefined) {
+            relay.state(frame);
+            return;
+        }
+        const binding = this.remotes.get(frame.hold);
+        if (binding === undefined)
             return;
+        if (!(frame.value instanceof Packed)) {
+            binding.state(frame.m, frame);
+            return;
+        }
+        let value;
+        try {
+            value = unpack(frame.value);
         }
-        this.subs.get(frame.id)?.(frame);
+        catch (error) {
+            binding.state(frame.m, { as: "error", error: toWireError(error) });
+            return;
+        }
+        binding.state(frame.m, { ...frame, value });
     }
     // ─── lifetime ───
     #closed(reason) {
+        const told = /^protocol-version (\d+)$/.exec(reason);
+        if (told !== null && Number(told[1]) !== PROTOCOL_VERSION)
+            this.#protocolMismatch(Number(told[1]), false);
         if (this.#pinger !== undefined)
             clearInterval(this.#pinger);
+        if (this.#ackTimer !== undefined)
+            clearTimeout(this.#ackTimer);
+        this.#pending.clear();
+        this.#pendingBytes = 0;
         const error = unavailable(`link to ${this.name ?? "peer"} closed: ${reason}`);
         // As origin: everything the peer held here, relayed or not, is released (downstream calls cancelled).
         for (const id of [...this.served.keys()])
@@ -676,7 +1131,6 @@ export class Connection {
         for (const pending of this.calls.values())
             pending.reject(error);
         this.calls.clear();
-        this.subs.clear();
         for (const vote of this.votes.values())
             vote.reject(error);
         this.votes.clear();
@@ -684,22 +1138,24 @@ export class Connection {
             waiter.reject(error);
         this.#syncWaiters.clear();
         // As downstream of relays: their origins lose the hold.
-        for (const relay of [...this.relayedHolds.values()])
+        for (const relay of [...this.relayedRefs.values()])
             relay.close(error, { releaseDown: false, notify: true });
-        this.relayedHolds.clear();
+        this.relayedRefs.clear();
         this.relayedCalls.clear();
-        this.relayedSubs.clear();
-        for (const handle of this.remotes.values()) {
+        const bindings = [...this.remotes.values()];
+        this.remotes.clear();
+        for (const binding of bindings) {
             // An upstream link may be replaced (reconnect); a downstream host that went away is gone.
             if (this.role === "upstream")
-                handle.detached();
+                binding.detached();
             else
-                handle.gone(error);
+                binding.gone(error);
         }
-        this.remotes.clear();
         this.host.connectionClosed(this);
     }
 }
+const stateKey = (hold, m) => `s${hold}\u0000${m}`;
+const unitKey = (hold) => `u${hold}`;
 /**
  * A call through a relay: waits for the route, then forwards with the root it runs in. It settles exactly once; a
  * cancel (or the origin going away) is forwarded downstream and removes both mappings.
@@ -709,6 +1165,7 @@ async function relayCall(relay, frame, root, signal, host) {
     if (signal.aborted)
         throw abortError();
     let onAbort = () => { };
+    let onCancel = () => { };
     const aborted = new Promise((_, reject) => {
         onAbort = () => reject(abortError());
         signal.addEventListener("abort", onAbort, { once: true });
@@ -720,32 +1177,34 @@ async function relayCall(relay, frame, root, signal, host) {
             throw new ChordError("gone", "relayed hold closed");
         return await new Promise((resolve, reject) => {
             const call = new RelayCall(relay, frame.id, down.nextId(), { resolve, reject });
-            signal.addEventListener("abort", () => call.cancel(abortError(), true), { once: true });
+            onCancel = () => call.cancel(abortError(), true);
+            signal.addEventListener("abort", onCancel, { once: true });
             host.rootSent(root.rootId, down);
             down.send({ t: "call", id: call.downId, hold: relay.downId, member: frame.member, args: frame.args, root });
         });
     }
     finally {
         signal.removeEventListener("abort", onAbort);
+        signal.removeEventListener("abort", onCancel);
     }
 }
 /**
  * Validates a snapshot against the schema the consumer is promised: the provider's own schema for a direct binding,
  * the consumer's contract for one through an `accepts` adapter (the adapter's view is what the consumer gets).
  */
-function validateSnapshot(handle, member, value) {
+function validateSnapshot(ref, member, value) {
     if (value === undefined)
         return;
-    const installed = handle.record.currentInstalled();
+    const installed = ref.object.currentInstalled();
     if (installed === undefined)
         return;
-    const view = handle.currentView();
-    const kind = view === undefined ? kindOf(installed.behavior) : kindOf(view);
-    const description = view !== undefined && view !== installed.behavior
-        ? handle.contract.description.variants[kind ?? LOCAL_VARIANT]?.[member]
-        : installed.service.contract.description.variants[kindOf(installed.behavior) ?? LOCAL_VARIANT]?.[member];
+    const view = ref.currentView();
+    const variant = view === undefined ? variantOf(installed.implementation) : variantOf(view);
+    const description = view !== undefined && view !== installed.implementation
+        ? ref.contract.description.variants[variant ?? LOCAL_VARIANT]?.[member]
+        : installed.service.contract.description.variants[variantOf(installed.implementation) ?? LOCAL_VARIANT]?.[member];
     if (description?.kind === "state")
-        assertValid("state", `${handle.address}.${member}`, description.schema, value);
+        assertValid("state", `${ref.address}.${member}`, description.schema, value);
 }
 /** Clients (browsers) holding an outdated contract are told to reload their code. */
 function clientError(error) {