@optimystic/db-p2p 0.13.5 → 0.14.1

package/dist/src/testing/reactivity-mesh-harness.js

@@ -0,0 +1,663 @@
+/**
+ * Reactivity **mock-transport mesh harness** — the in-process, many-logical-node substrate for the
+ * reactivity *e2e* tier (`docs/reactivity.md`), layered on the cohort-topic mesh harness
+ * ({@link import("./cohort-topic-mesh-harness.js")}) rather than forking it (the same relationship the
+ * {@link import("./matchmaking-mesh-harness.js").MatchmakingMesh} has to it). It drives the **real**
+ * reactivity hot path end-to-end over a network of real-Ed25519-keyed cohort-topic hosts:
+ *
+ *  - **Origination** rides the real {@link makeCohortTopicChangeNotifier} over a real {@link StorageRepo}:
+ *    a `pend`+`commit` fires the catch-all change feed → the bridge hands the {@link CollectionChangeEvent}
+ *    + a **real** threshold {@link CommitCert} (each tail-cohort member signs the commit-vote payload with
+ *    its real Ed25519 key, assembled by {@link buildCommitCert}) to the real
+ *    {@link ReactivityOriginationManager}, which builds the {@link NotificationV1} reusing the cert's
+ *    threshold signature **unchanged** — exactly as in production (`local-change-notifier-bridge`).
+ *  - **Forwarding** runs the real {@link createReactivityForwarder} receive path (verify → dedupe → buffer)
+ *    over a real {@link PushState} (replay ring + rolling checkpoint + per-subscriber backpressure).
+ *  - **Delivery** runs the real {@link ReactivitySubscriptionManager} (register at cohort-topic tier **T3**,
+ *    verify against the cached tail-cohort {@link MembershipCertV1} with **real** collected-multisig crypto,
+ *    contiguity check, gap → backfill, `(collectionId, revision)` dedupe, surface).
+ *  - **Recovery** serves real {@link serveBackfill} / {@link serveResume} from the tail `PushState`, applied
+ *    subscriber-side by the manager's `resume()` / backfill seam.
+ *  - **Rotation** drives the real {@link BlockFillTracker} / {@link buildRotationHint} /
+ *    {@link planReRegistrationWave} / {@link buildRotationHandoffCheckpoint} lifecycle; **backpressure**
+ *    drives the real {@link PushState.enqueueForSubscribers}.
+ *
+ * **What is real vs. modeled (honest).** Every signature, registration record, replay-buffer entry, dedupe
+ * decision, checkpoint fold, and resume classification is the real db-core/db-p2p code. What the harness
+ * *models* is the **notification transport** (the application protocol that would dial each subscriber's
+ * primary and each child cohort) — it fans the originated notification to the tracked direct subscribers
+ * in-process via the real per-subscriber bounded queues — and, like the matchmaking harness, the
+ * **single-tier-0 reach**: the cohort-topic substrate serves a single tier-0 cohort (multi-tier *serving*
+ * promotion to a tier-`d ≥ 1` forwarder cohort is gated on the cohort-topic follow-ons), so a deep
+ * forwarder *tree* fan-out is tagged-unimplemented in the suites, not faked here. The tail cohort fanning
+ * to every direct subscriber — the heart of contiguous/verified delivery — is fully real.
+ *
+ * **Virtual time.** Like the sibling harnesses this is not a wall-clock simulator: `now` is an explicit
+ * virtual clock the harness advances ({@link ReactivityMesh.advanceTime}), so `T_drain` / rotation timing
+ * resolve deterministically without sleeping. The only real-time waits are the cohort harness's tiny
+ * async-settle polls.
+ */
+import { Tier, reactivityTopicId, createReactivityForwarder, createNotificationVerifier, PushState, serveBackfill, serveResume, buildRotationHint, BlockFillTracker, TailDrainGate, planReRegistrationWave, buildRotationHandoffCheckpoint, applyRotationHandoff, createRejoinJitter, mayServeAsReactivityForwarder, deltaMaxForProfile, coreProfile, edgeProfile, bytesToB64url, DEFAULT_CAP_PROMOTE_FAST, T_REJOIN_JITTER_MS, WARM_THRESHOLD_DEFAULT, } from "@optimystic/db-core";
+import { StorageRepo } from "../storage/storage-repo.js";
+import { BlockStorage } from "../storage/block-storage.js";
+import { MemoryRawStorage } from "../storage/memory-storage.js";
+import { makeCohortTopicChangeNotifier } from "../cohort-topic/change-bridge.js";
+import { buildCommitCert } from "../cluster/commit-cert.js";
+import { ReactivitySubscriptionManager } from "../reactivity/subscription-manager.js";
+import { ReactivityOriginationManager } from "../reactivity/origination-manager.js";
+import { RotationReRegistrationScheduler } from "../reactivity/rotation-rereg-scheduler.js";
+import { RotationRedirectError } from "../reactivity/recover-transport.js";
+import { addressing, buildMesh, delay, makeMembers, setupTopic, } from "./cohort-topic-mesh-harness.js";
+export { addressing, delay };
+const utf8 = new TextEncoder();
+/**
+ * The reactivity integration mesh. Build with {@link buildReactivityMesh}; register a collection with
+ * {@link registerCollection}; drive subscribers/commits/recovery with {@link subscribe} / {@link commit} /
+ * {@link resume} / {@link rotateTail}.
+ */
+export class ReactivityMesh {
+    mesh;
+    members;
+    wantK;
+    minSigs;
+    profiles;
+    collections = new Map();
+    vtime = 1_700_000_000_000;
+    corr = 0;
+    /** Pending one-shot timers armed against the virtual clock (the rotation scheduler's `setTimer` binding). */
+    virtualTimers = [];
+    constructor(mesh, members, wantK, minSigs, profiles) {
+        this.mesh = mesh;
+        this.members = members;
+        this.wantK = wantK;
+        this.minSigs = minSigs;
+        this.profiles = profiles;
+    }
+    /** Stand up an N-node reactivity mesh and start every host. */
+    static async build(opts = {}) {
+        const nodeCount = opts.nodeCount ?? 12;
+        const wantK = opts.wantK ?? Math.min(nodeCount, 8);
+        const minSigs = opts.minSigs ?? Math.max(1, wantK - 2);
+        const sizeEstimate = opts.sizeEstimate ?? 256;
+        const members = await makeMembers(nodeCount);
+        const meshOpts = {
+            wantK,
+            minSigs,
+            sizeEstimate,
+            // Single-tier-0 by default (same rationale as the matchmaking harness): test artifacts register
+            // many subscribers within a few ms, which would otherwise trip the pre-promotion slope predictor.
+            capPromote: opts.capPromote ?? 1_000_000,
+            // This mesh is virtual-time: a subscribe batch lands at one fixed virtual instant, but the cohort
+            // host stamps each register with real `Date.now()`. The growth-slope pre-promotion is a
+            // wall-clock-rate heuristic, so those real-time-separated samples make it fire *before* the
+            // `cap_promote` count is reached (redirecting the rest of the batch up a tier-1 tree the
+            // single-tier-0 milestone never instantiates). Disable slope here so promotion is driven purely by
+            // the count threshold — the same posture the cohort-topic scale specs get by stamping a constant
+            // `now` on every `handleRegister`. The slope heuristic itself is covered by the promotion unit specs.
+            promotion: { tPromoteLookaheadMs: 0 },
+            // Permissive register ceiling — the harness re-probes a cohort across many subscribes/commits.
+            // Also widen the replay-guard freshness window. The cohort host stamps each register with real
+            // `Date.now()` and evaluates the anti-DoS freshness gate against real `Date.now()` at processing
+            // time, but this mesh drives the real participant walk in-process: under full-suite CPU load the
+            // event loop can stall for far longer than the production 60 s staleness window between a register
+            // being stamped and the deciding cohort processing it. A stalled-but-legitimate register then
+            // trips the freshness gate (`no_state`), the cold-start bootstrap walk exhausts, and `register`
+            // throws `CohortBackoffError` — a pure load-induced harness flake. The wall-clock freshness
+            // defense is meaningless here (same posture as the disabled rate limiter above), so make the
+            // window effectively unbounded so test-induced stalls never read as stale/future replays.
+            // Disable the cold-start proof-of-work cost. Reactivity subscribers register at tier T3, so the
+            // participant-side bootstrap-evidence builder mints a PoW puzzle (default 20 bits ≈ 1 M hashes) on
+            // *every* register/re-probe — by far the dominant cost in this mesh (a 6-subscriber promotion test
+            // burns ~60-100 s almost entirely in `randomFillSync`/SHA-256 minting nonces). The cohort's
+            // bootstrap-evidence policy is unconfigured here, so the evidence is never even required; minting it
+            // is pure wasted work. `bits: 0` solves on the first nonce (the documented test escape hatch in
+            // `meetsDifficulty`) — same posture as the neutralized rate limiter / replay guard above, and it
+            // removes the timing-sensitivity that tips these tests into a timeout under full-suite load.
+            antiDos: { rateLimiter: { ratePerWindow: 1_000_000 }, replayGuard: { maxAgeMs: 86_400_000, maxFutureSkewMs: 86_400_000 }, powDifficultyBits: 0 },
+            ...(opts.profiles === undefined ? {} : { profiles: opts.profiles }),
+        };
+        const mesh = await buildMesh(members, meshOpts);
+        const profiles = Array.from({ length: nodeCount }, (_v, i) => opts.profiles?.[i] ?? "core");
+        return new ReactivityMesh(mesh, members, wantK, minSigs, profiles);
+    }
+    /** The current virtual time (unix ms). */
+    get now() {
+        return this.vtime;
+    }
+    /**
+     * Advance the virtual clock by `ms` (drives `T_drain` / rotation timing deterministically), then fire any
+     * virtual timers now due. A fired timer may arm another (a chained rotation), so fire one-at-a-time until
+     * the queue is quiescent — the rotation re-registration scheduler's `setTimer` binding rides this.
+     */
+    advanceTime(ms) {
+        this.vtime += ms;
+        for (;;) {
+            const idx = this.virtualTimers.findIndex((t) => !t.cancelled && t.fireAt <= this.vtime);
+            if (idx === -1) {
+                break;
+            }
+            const [timer] = this.virtualTimers.splice(idx, 1);
+            if (!timer.cancelled) {
+                timer.fn();
+            }
+        }
+    }
+    /** Arm a one-shot timer against the virtual clock; the cancel handle drops it (the scheduler's `setTimer`). */
+    armVirtualTimer(fn, delayMs) {
+        const timer = { fireAt: this.vtime + Math.max(0, delayMs), fn, cancelled: false };
+        this.virtualTimers.push(timer);
+        return () => { timer.cancelled = true; };
+    }
+    profileOf(index) {
+        return this.profiles[index] === "edge" ? edgeProfile() : coreProfile();
+    }
+    /**
+     * Register a collection on the mesh: instantiate its tail-anchored tier-0 cohort (+willingness quorum),
+     * install the real origination manager behind the real change-notifier bridge over a fresh `StorageRepo`,
+     * build the tail `PushState` (replay ring + rolling checkpoint + backpressure), and cache the tail
+     * cohort's `MembershipCertV1` into every node's verifier so end-to-end verification is real Ed25519.
+     */
+    async registerCollection(name, opts = {}) {
+        const collectionId = utf8.encode(`reactivity:${name}`);
+        const collectionIdB64 = bytesToB64url(collectionId);
+        const tailId = this.pinTailToCore(`${name}:tail-0`);
+        const topicId = reactivityTopicId(tailId);
+        const coord0 = addressing.coord0(topicId);
+        const setup = await setupTopic(this.mesh, topicId);
+        const tailCohort = this.cohortMembersAround(coord0);
+        const deltaMaxBytes = opts.deltaMaxBytes ?? deltaMaxForProfile(coreProfile());
+        const w = opts.w ?? 256;
+        const wCheckpoint = opts.wCheckpoint ?? 4096;
+        const queueMax = opts.queueMax ?? 32;
+        const rawStorage = new MemoryRawStorage();
+        const repo = new StorageRepo((blockId) => new BlockStorage(blockId, rawStorage));
+        // Origination is the tail cohort primary's responsibility — installed on the node nearest coord_0
+        // (the routed tail primary), exactly the node a commit on the tail cluster lands on in production.
+        const originationService = setup.deciding.host.service;
+        const certByAction = new Map();
+        const emitQueue = [];
+        const pushState = this.makePushState(collectionIdB64, topicId, tailId, w, wCheckpoint, queueMax, deltaMaxBytes);
+        const state = {
+            name,
+            collectionId,
+            collectionIdB64,
+            tailId,
+            topicId,
+            coord0,
+            tailCohort,
+            setup,
+            repo,
+            originationService,
+            pushState,
+            forwarder: createReactivityForwarder({ state: pushState, verifier: this.notificationVerifierFor(originationService) }),
+            emitQueue,
+            rev: 0,
+            blockSeq: 0,
+            deltaMaxBytes,
+            w,
+            wCheckpoint,
+            queueMax,
+            fillTracker: new BlockFillTracker({
+                ...(opts.blockFillSize === undefined ? {} : { blockFillSize: opts.blockFillSize }),
+                // warm_threshold must be < block_fill_size; clamp the default down for small (test-scaled) blocks.
+                warmThreshold: opts.warmThreshold ?? (opts.blockFillSize === undefined ? WARM_THRESHOLD_DEFAULT : Math.min(WARM_THRESHOLD_DEFAULT, Math.max(0, opts.blockFillSize - 1))),
+            }),
+            subscribers: [],
+            certByAction,
+        };
+        const origination = new ReactivityOriginationManager({
+            service: originationService,
+            resolveContext: () => ({
+                tailId: state.tailId,
+                deltaMaxBytes: state.deltaMaxBytes,
+                ...(state.rotationHint === undefined ? {} : { rotationHint: state.rotationHint }),
+            }),
+            emit: (n) => { emitQueue.push(n); },
+            clock: () => this.vtime,
+        });
+        origination.install();
+        state.origination = origination;
+        // The real local-change-notifier bridge: a StorageRepo commit → onLocalCommit → origination.
+        makeCohortTopicChangeNotifier({
+            source: repo,
+            service: originationService,
+            selfIsCohortMember: (e) => e.collectionId === collectionIdB64,
+            extractCommitCert: (e) => certByAction.get(e.actionId),
+        });
+        this.collections.set(name, state);
+        this.cacheTailCert(state);
+    }
+    /** The reactivity notification verifier over a node's service verifier (real collected-multisig at T3). */
+    notificationVerifierFor(service) {
+        return createNotificationVerifier({ verifier: service.verifier(), tier: Tier.T3 });
+    }
+    makePushState(collectionIdB64, topicId, tailId, w, wCheckpoint, queueMax, deltaMaxBytes) {
+        return new PushState({
+            collectionId: collectionIdB64,
+            topicId: bytesToB64url(topicId),
+            tailIdAtJoin: bytesToB64url(tailId),
+            w,
+            wCheckpoint,
+            queueMax,
+            deltaMaxBytes,
+        });
+    }
+    /** The FRET cohort (the `wantK` nearest members) around a coord. */
+    cohortMembersAround(coord) {
+        const ids = new Set(this.mesh.assembleCohort(coord, this.wantK));
+        return this.members.filter((m) => ids.has(m.idStr));
+    }
+    /** Whether the node FRET-routes nearest to `coord` (the cohort's routed primary) is an Edge profile. */
+    nearestIsEdge(coord) {
+        const nearest = this.mesh.nearest(coord);
+        const idx = this.members.findIndex((m) => m.idStr === nearest.idStr);
+        return idx >= 0 && this.profiles[idx] === "edge";
+    }
+    /**
+     * Choose a reactivity tail id derived from `baseTail` whose `coord_0` routes nearest a **Core** node.
+     * Reactivity topics serve at tier T3, and an Edge node declines a T3 cold-start by design (it serves no
+     * forwarder duty — pinned by `cohort-topic-scale-lifecycle` "Edge serves no T3"), so a topic whose
+     * `coord_0` routes nearest an Edge node can never instantiate its tail cohort: the routed primary returns
+     * `no_state`, the walk exhausts, and `register` throws `CohortBackoffError`. Because {@link makeMembers}
+     * re-randomizes the member ring layout every run, an Edge-profile mesh would otherwise *intermittently*
+     * seat the Edge node as a collection's tail primary (~1/nodeCount of runs) and flake. In production a
+     * reactivity tail cohort IS the Core cluster that served the tail, so pinning the primary to a Core node
+     * models reality. A no-op (returns `utf8(baseTail)` at the first try) when the mesh has no Edge nodes —
+     * the common case — so every all-Core suite is byte-for-byte unaffected.
+     */
+    pinTailToCore(baseTail) {
+        for (let nonce = 0;; nonce++) {
+            const tailId = utf8.encode(nonce === 0 ? baseTail : `${baseTail}#${nonce}`);
+            if (!this.nearestIsEdge(addressing.coord0(reactivityTopicId(tailId)))) {
+                return tailId;
+            }
+        }
+    }
+    collection(name) {
+        const c = this.collections.get(name);
+        if (c === undefined) {
+            throw new Error(`reactivity mesh: collection "${name}" is not registered (call registerCollection first)`);
+        }
+        return c;
+    }
+    /**
+     * Cache the tail cohort's `MembershipCertV1` (over the FRET cohort around the current tail's coord_0) into
+     * every node's participant verifier, so a subscriber's notification verify is real Ed25519 against the
+     * tail membership — mirroring `reactivity-real-crypto.spec.ts`, now over the mesh's real cohort members.
+     */
+    cacheTailCert(c) {
+        const members = c.tailCohort.map((m) => bytesToB64url(m.bytes));
+        const cert = {
+            v: 1,
+            cohortCoord: bytesToB64url(c.coord0),
+            cohortEpoch: bytesToB64url(utf8.encode(`${c.name}:${bytesToB64url(c.tailId)}`)),
+            members,
+            stabilizedAt: this.vtime,
+            thresholdSig: bytesToB64url(new Uint8Array([0])),
+            signers: members,
+        };
+        for (const node of this.mesh.nodes) {
+            node.host.service.verifier().cache(cert);
+        }
+    }
+    /** The tail cohort member ids (base64url member bytes) serving a collection's current tail. */
+    tailCohortIds(name) {
+        return this.collection(name).tailCohort.map((m) => bytesToB64url(m.bytes));
+    }
+    /** This collection's tier-0 cohort registration records carrying a reactivity subscribe payload. */
+    cohortSubscriberCount(name) {
+        const c = this.collection(name);
+        return c.setup.decidingEngine.records(c.topicId).filter((r) => r.appState !== undefined).length;
+    }
+    /** The current committed revision for a collection. */
+    currentRevision(name) {
+        return this.collection(name).rev;
+    }
+    /**
+     * Drive the deciding cohort's membership stabilization (publish its `MembershipCertV1`). Promotion is
+     * fire-and-forget off the register path; stabilizing publishes the cohort cert so a crossed-`cap_promote`
+     * decision lands deterministically (no wall-clock wait) — the same pre-step the cohort-topic scale specs use.
+     */
+    async stabilizeCohort(name) {
+        await this.collection(name).setup.decidingEngine.onStabilized(this.vtime);
+    }
+    /** Whether a collection's tier-0 cohort has promoted (its direct participants crossed `cap_promote`). */
+    isPromoted(name) {
+        const c = this.collection(name);
+        return c.setup.decidingEngine.isPromoted(c.topicId);
+    }
+    /** Whether a node profile may serve as a reactivity forwarder (T3 producer) — Edge never can. */
+    mayForward(nodeIndex) {
+        return mayServeAsReactivityForwarder(this.profileOf(nodeIndex));
+    }
+    /**
+     * Register node `nodeIndex` as a reactivity subscriber for `collection` via the **real**
+     * {@link ReactivitySubscriptionManager} → `CohortTopicService.register` walk at tier T3. The subscribe
+     * record lands in the real tier-0 cohort store; the manager owns the real verify/deliver/backfill/resume
+     * path. Returns a handle whose `delivered` array accumulates the surfaced notifications.
+     */
+    async subscribe(nodeIndex, collection, opts = {}) {
+        const c = this.collection(collection);
+        const member = this.members[nodeIndex];
+        const service = this.mesh.nodes[nodeIndex].host.service;
+        const profile = opts.profile ?? this.profileOf(nodeIndex);
+        const subId = bytesToB64url(member.bytes);
+        const handle = {
+            nodeIndex,
+            member,
+            subId,
+            manager: undefined,
+            delivered: [],
+            attachedTailB64: bytesToB64url(c.tailId),
+            collectionName: collection,
+            autoDrain: opts.autoDrain ?? true,
+            asleep: false,
+            backfills: 0,
+            chainRead: false,
+            checkpointDigests: [],
+            rotationNotices: [],
+            scheduler: undefined,
+        };
+        // The host re-registration scheduler bound to this subscriber, driven over the harness virtual clock. On
+        // fire it re-attaches the subscriber under the rotated tail — the production move the deferred Quereus
+        // `Database.watch` factory performs (`reactivity-rotation-host-wiring-e2e` §C). The single-subscriber
+        // `planReRegistration` path draws `fireAt` over `T_rejoin_jitter`, so a `advanceTime` past that window
+        // fires it deterministically.
+        const scheduler = new RotationReRegistrationScheduler({
+            reRegister: (plan) => { this.reAttachToNewTail(handle, plan); return Promise.resolve(); },
+            setTimer: (fn, delayMs) => this.armVirtualTimer(fn, delayMs),
+            now: () => this.vtime,
+        });
+        handle.scheduler = scheduler;
+        const manager = new ReactivitySubscriptionManager({
+            service,
+            collectionId: c.collectionId,
+            tailIdAtAttach: c.tailId,
+            deliver: (n) => { handle.delivered.push(n); },
+            profile,
+            lastKnownRev: opts.lastKnownRev ?? 0,
+            clock: () => this.vtime,
+            // Backfill RPC served from the tail cohort's live replay buffer (real serveBackfill).
+            signBackfill: (req) => bytesToB64url(utf8.encode(`bf:${req.fromRevision}:${req.toRevision}`)),
+            backfillTransport: (req) => { handle.backfills += 1; return Promise.resolve(serveBackfill(c.pushState.replayBuffer, req, c.collectionIdB64)); },
+            // Resume RPC served from the tail cohort's stacked windows (real serveResume).
+            signResume: () => bytesToB64url(utf8.encode("resume")),
+            subscriberCoord: subId,
+            resumeTransport: (req) => {
+                // Model the live recover serve's drain redirect (the running node's `markRotated` →
+                // `rotationRedirectFor` → `RotationRedirectError`): a resume whose `latestKnownTailId` anchors a
+                // rotated, still-draining tail is bounced to the new tree instead of served stale data.
+                const gate = c.rotationGate;
+                if (gate !== undefined && req.latestKnownTailId === c.rotatedFromTailB64 && gate.isDraining(this.vtime)) {
+                    return Promise.reject(new RotationRedirectError(gate.rotationRedirect));
+                }
+                return Promise.resolve(serveResume(req, {
+                    buffer: c.pushState.replayBuffer,
+                    checkpoint: c.pushState.checkpoint,
+                    // The cross-rotation inherited window the new tail holds after a handoff: a resume below the new
+                    // ring but within it is served `checkpoint_window` (not `out_of_window`) while it abuts the ring.
+                    inheritedCheckpoint: c.pushState.inheritedCheckpoint,
+                    currentTailId: bytesToB64url(c.tailId),
+                    currentRevision: c.rev,
+                    rotationRevision: c.rev,
+                    expectedCollectionId: c.collectionIdB64,
+                }));
+            },
+            onChainRead: () => { handle.chainRead = true; },
+            onTailRotated: (newTailId, rev) => { handle.tailRotated = [newTailId, rev]; },
+            onCheckpointDigest: (summary) => { handle.checkpointDigests.push(summary); },
+            // Surface the jittered re-registration plan from a delivered rotation pre-announce / hard rotation.
+            rejoinJitter: createRejoinJitter({ capPromote: DEFAULT_CAP_PROMOTE_FAST, random: this.deterministicRandom() }),
+            onRotation: (notice) => { handle.rotationNotices.push(notice); scheduler.schedule(notice); },
+        });
+        handle.manager = manager;
+        handle.registration = await manager.register();
+        c.subscribers.push(handle);
+        return handle;
+    }
+    /**
+     * Commit `count` transactions to `collection` through the real `StorageRepo` → change-bridge →
+     * origination path, then fan the originated notifications out to current-tail subscribers. Returns the
+     * new current revision. Each commit pre-signs a **real** threshold {@link CommitCert} with the tail
+     * cohort's Ed25519 keys (the bridge's synchronous extractor resolves it by `actionId`).
+     */
+    async commit(collection, count = 1) {
+        const c = this.collection(collection);
+        for (let i = 0; i < count; i++) {
+            const rev = c.rev + 1;
+            const actionId = `rx-${c.collectionIdB64}-${rev}-${this.corr++}`;
+            const blockId = `${c.name}-blk-${c.blockSeq++}`;
+            // Anticipatory warm-up / block-filling: the filling commit carries the rotation pre-announce.
+            const signal = c.fillTracker.onCommit();
+            if (signal.kind === "filling") {
+                const nextTail = utf8.encode(`${c.name}:tail-${rev}`);
+                c.rotationHint = buildRotationHint(bytesToB64url(nextTail), rev);
+            }
+            const cert = await this.buildTailCert(c, rev);
+            c.certByAction.set(actionId, cert);
+            await c.repo.pend({ actionId: actionId, transforms: this.insertTransforms(blockId, c.collectionIdB64), policy: "c" });
+            const result = await c.repo.commit({ actionId: actionId, blockIds: [blockId], tailId: blockId, rev });
+            if (!result.success) {
+                throw new Error(`reactivity mesh: commit failed for ${collection} rev ${rev}`);
+            }
+            c.rev = rev;
+            // The pre-announce is one-shot: it rode the filling notification; clear it for subsequent commits.
+            c.rotationHint = undefined;
+        }
+        await this.drainFanOut(c);
+        return c.rev;
+    }
+    /** Build a real threshold commit cert: every tail-cohort member signs `utf8(commitHash + ":approve")`. */
+    async buildTailCert(c, rev) {
+        const commitHash = `${c.collectionIdB64}:${rev}`;
+        const signedPayload = utf8.encode(`${commitHash}:approve`);
+        const commits = {};
+        for (const m of c.tailCohort) {
+            const sig = await m.key.sign(signedPayload);
+            commits[m.idStr] = { type: "approve", signature: bytesToB64url(sig) };
+        }
+        const record = {
+            messageHash: `mh-${commitHash}`,
+            message: { operations: [{ get: { blockIds: [] } }], expiration: this.vtime + 30_000 },
+            peers: this.clusterPeers(c.tailCohort),
+            promises: {},
+            commits,
+        };
+        return buildCommitCert(record, this.minSigs, signedPayload);
+    }
+    clusterPeers(cohort) {
+        const peers = {};
+        for (const m of cohort) {
+            peers[m.idStr] = { multiaddrs: ["/ip4/127.0.0.1/tcp/8000"], publicKey: bytesToB64url(m.peerId.publicKey.raw) };
+        }
+        return peers;
+    }
+    insertTransforms(blockId, collectionId) {
+        const block = { header: { id: blockId, type: "test", collectionId: collectionId } };
+        return { inserts: { [blockId]: block }, updates: {}, deletes: [] };
+    }
+    /**
+     * Run the originated notifications through the real forwarder receive path (populating the tail replay
+     * ring + rolling checkpoint + dedupe), then fan each out to every current-tail subscriber via its real
+     * per-subscriber bounded queue. A subscriber with `autoDrain` drains immediately; a slow subscriber's
+     * queue accumulates (drop-oldest under pressure) until it is woken.
+     */
+    async drainFanOut(c) {
+        const notifs = c.emitQueue.splice(0, c.emitQueue.length);
+        const tailB64 = bytesToB64url(c.tailId);
+        for (const n of notifs) {
+            // Forwarder receive: verify (real) → dedupe → append to the replay ring (and roll the checkpoint).
+            await c.forwarder.receive(n, this.vtime);
+            for (const s of c.subscribers) {
+                if (s.attachedTailB64 !== tailB64 || s.asleep) {
+                    continue;
+                }
+                c.pushState.perSubscriberQueue.enqueue(s.subId, n);
+                if (s.autoDrain) {
+                    await this.drainSubscriber(s);
+                }
+            }
+        }
+    }
+    /** Drain a subscriber's bounded queue (FIFO) through the real manager delivery path. */
+    async drainSubscriber(s) {
+        const c = this.collection(s.collectionName);
+        const queue = c.pushState.perSubscriberQueue.peekQueue(s.subId);
+        if (queue === undefined) {
+            return;
+        }
+        for (const n of queue.drain()) {
+            await s.manager.onNotification(n);
+        }
+    }
+    /** Wake a slow subscriber: drain its accumulated (drop-oldest) queue through the delivery path. */
+    async wakeSubscriber(s) {
+        await this.drainSubscriber(s);
+    }
+    /** Put a subscriber to sleep: fan-out skips it (its tail replay buffer still fills) until it resumes. */
+    sleepSubscriber(s) {
+        s.asleep = true;
+    }
+    /** The drop counter on a subscriber's bounded queue (drop-oldest under backpressure). */
+    droppedFor(s) {
+        const c = this.collection(s.collectionName);
+        return c.pushState.perSubscriberQueue.peekQueue(s.subId)?.dropped ?? 0;
+    }
+    /** Drive the manager's real `resume()` (one `ResumeV1` → `serveResume` → `applyResumeReply`). */
+    async resume(s) {
+        s.asleep = false;
+        return s.manager.resume();
+    }
+    /**
+     * The re-registration move the scheduler fires on a rotation notice: re-attach the subscriber under the
+     * rotated tail so subsequent fan-out targets it. The production factory builds a *fresh* manager under
+     * `plan.newTopicId`; the harness keeps the same manager (its `lastRevision` is already contiguous and its
+     * `rotationHandledFor` guard dedupes the new-tail deliveries' re-detection), so re-attaching the existing
+     * subscriber is the equivalent move with no gap.
+     */
+    reAttachToNewTail(s, plan) {
+        const c = this.collection(s.collectionName);
+        s.attachedTailB64 = bytesToB64url(plan.newTailId);
+        // Defensive: the plan's successor must be the collection's current tail (single-rotation harness model).
+        if (s.attachedTailB64 !== bytesToB64url(c.tailId)) {
+            s.attachedTailB64 = bytesToB64url(c.tailId);
+        }
+    }
+    /**
+     * Rotate a collection's tail: advance `tailId` → new `topicId`/coord/cohort, re-establish the new cohort
+     * (re-seed willingness so a post-rotation subscribe can register), re-cache the tail cert, rebuild the tail
+     * `PushState`, fold the outgoing ring into a handoff checkpoint onto the new tail, and plan the jittered
+     * re-registration wave bounded by `cap_promote_fast`. Async because re-forming the new cohort is (the real
+     * willingness convergence). With `{ autoReattach: false }` it instead models the running node's
+     * `markRotated` (drain gate) so subscribers move via the recover-redirect path. Models §Tail rotation steps 2–5.
+     */
+    async rotateTail(collection, opts = {}) {
+        const c = this.collection(collection);
+        const autoReattach = opts.autoReattach ?? true;
+        const rotationRevision = c.rev;
+        // Pin the rotated tail's routed primary to a Core node too (Edge nodes decline a T3 cold-start); a
+        // no-op for the all-Core rotation suites, matching the initial-tail pinning in `registerCollection`.
+        const newTailId = this.pinTailToCore(`${c.name}:tail-${rotationRevision}`);
+        const newTailIdB64 = bytesToB64url(newTailId);
+        const oldTailB64 = bytesToB64url(c.tailId);
+        // Buffer-to-checkpoint handoff — the ONLY state migrated across a rotation (§Tail rotation step 5).
+        const handoff = buildRotationHandoffCheckpoint(c.pushState, { rotationRevision });
+        // Active subscribers re-register under the new tree, carrying their lastRevision (continuity).
+        const live = c.subscribers.filter((s) => s.attachedTailB64 === oldTailB64);
+        const jitter = createRejoinJitter({ capPromote: DEFAULT_CAP_PROMOTE_FAST, random: this.deterministicRandom() });
+        const plans = planReRegistrationWave({
+            hint: { newTailId: newTailIdB64 },
+            subscribers: live.map((s) => ({ lastRevision: s.manager.lastRevision })),
+            now: this.vtime,
+            jitter,
+        });
+        // Roll the topic to the new tail.
+        c.tailId = newTailId;
+        c.topicId = reactivityTopicId(newTailId);
+        c.coord0 = addressing.coord0(c.topicId);
+        c.tailCohort = this.cohortMembersAround(c.coord0);
+        // Re-establish the new tail's cohort (re-seed willingness) so a subscriber can register under it after the
+        // rotation — the new tree forming, modeled deterministically (the real cohort-topic willingness convergence).
+        c.setup = await setupTopic(this.mesh, c.topicId);
+        this.cacheTailCert(c);
+        c.pushState = this.makePushState(c.collectionIdB64, c.topicId, c.tailId, c.w, c.wCheckpoint, c.queueMax, c.deltaMaxBytes);
+        c.forwarder = createReactivityForwarder({ state: c.pushState, verifier: this.notificationVerifierFor(c.originationService) });
+        if (handoff !== undefined) {
+            applyRotationHandoff(c.pushState, handoff);
+        }
+        if (autoReattach) {
+            // Re-attach the live subscribers under the new tail so subsequent fan-out targets them (the
+            // continuity/handoff model — subscribers migrate directly, no redirect involved).
+            for (const s of live) {
+                s.attachedTailB64 = newTailIdB64;
+            }
+        }
+        else {
+            // Transport-driven: model `markRotated` by arming the outgoing tail's drain gate. Live subscribers
+            // stay on the old tail until each resumes, gets the `kind:"rotated"` redirect, and re-attaches via its
+            // own onRotation → scheduler → reRegister path (§Tail rotation step 2–3).
+            c.rotationGate = new TailDrainGate({ rotatedAt: this.vtime, newTailId: newTailIdB64, effectiveAtRevision: rotationRevision + 1 });
+            c.rotatedFromTailB64 = oldTailB64;
+        }
+        return {
+            newTailId,
+            newTailIdB64,
+            rotationRevision,
+            plans,
+            peakWindowArrivals: this.peakWindowArrivals(plans),
+            ...(handoff?.checkpoint === undefined ? {} : { handoff: handoff.checkpoint }),
+        };
+    }
+    /** Peak re-registration arrivals in any `T_rejoin_jitter`-long window of a planned wave. */
+    peakWindowArrivals(plans) {
+        if (plans.length === 0) {
+            return 0;
+        }
+        const fireAts = plans.map((p) => p.fireAt).sort((a, b) => a - b);
+        const window = T_REJOIN_JITTER_MS; // the same jitter span `createRejoinJitter` spreads the wave over
+        let peak = 0;
+        for (let i = 0; i < fireAts.length; i++) {
+            let j = i;
+            while (j < fireAts.length && fireAts[j] - fireAts[i] < window) {
+                j++;
+            }
+            peak = Math.max(peak, j - i);
+        }
+        return peak;
+    }
+    /** A deterministic pseudo-random in [0,1) so the jitter wave is reproducible across runs. */
+    deterministicRandom() {
+        let seed = 0x2545f491;
+        return () => {
+            seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+            return seed / 0x7fffffff;
+        };
+    }
+    /**
+     * The tail cohort `PushState` for a collection (its live replay ring / checkpoint / backpressure). Exposed
+     * so a suite can assert the replay/dedupe/partition state directly.
+     */
+    pushStateOf(collection) {
+        return this.collection(collection).pushState;
+    }
+    /** A real forwarder receive over the tail `PushState` — used to model a forwarder-hop / partition replay. */
+    async forwarderReceive(collection, n) {
+        const c = this.collection(collection);
+        return c.forwarder.receive(n, this.vtime);
+    }
+    async stop() {
+        // Tear down every subscriber's re-registration scheduler (drops pending virtual timers + the idempotence
+        // ledger) before stopping the mesh.
+        for (const c of this.collections.values()) {
+            for (const s of c.subscribers) {
+                s.scheduler?.stop();
+            }
+        }
+        await this.mesh.stop();
+    }
+}
+/** Build and start a reactivity integration mesh. */
+export async function buildReactivityMesh(opts = {}) {
+    return ReactivityMesh.build(opts);
+}
+//# sourceMappingURL=reactivity-mesh-harness.js.map