@danceiny/gotry 0.2.0-rc.32 → 0.2.0-rc.33

package/dist/scripts/tenant-state-handoff-tests.js

@@ -0,0 +1,854 @@
+/**
+ * Graceful handoff, forget and the guarded view, as an independent review found them wanting (AaaS A1;
+ * docs/rfc/aaas-rfc.md §4.2 items 5 and 6, §4.6; threats S-6 and S-12).
+ *
+ * The contract being pinned is the one the router already states: a write is acknowledged only if the lease is held
+ * before and after it, and a graceful handoff loses nothing that was acknowledged. Each check below is a way the first
+ * implementation let an acknowledged write be lost, a flush be reported that did not land, a raw handle escape the
+ * guard, a forgotten tenant's image survive, or a throwing observer hold a lease.
+ *
+ * Offline and deterministic: a directory stands in for the object store, fake clocks move time by hand, and a slow
+ * upload is a hook that moves the clocks while the upload is "in flight". A few checks use real timers (the lease must be
+ * renewed while a handoff drains, a renewal chain, a drain deadline), each with room to spare on a busy machine. The founder's runtime state root is never opened.
+ *
+ *   H  handoff: writes after the final image are refused, the lease outlives the drain and the upload, a flush whose
+ *      lease lapsed is reported as failed, a failed flush resumes service, an abandon stays abandoned
+ *   D  the drain is bounded, a stuck tenant holds nobody else, and a tenant that is closed meanwhile settles its handoff
+ *   R  one renewal chain, however a failed handoff and a slow lease store interleave
+ *   T  run() takes a synchronous callback
+ *   V  the guarded view hands out nothing raw
+ *   F  forget: an upload under way cannot leave the tenant's image behind, and forget wins against a handoff
+ *   S  copies of a dead epoch are deleted when the tenant is next opened
+ *   P  a destructive call takes only the tenant's own paths
+ *   O  an observer that throws cannot hold a lease
+ *   L  a replica keeps no copy of a tenant it handed off with everything flushed
+ *
+ * Run: cd ts && npx tsx scripts/tenant-state-handoff-tests.ts
+ */
+import assert from 'node:assert/strict';
+import { closeSync, existsSync, fstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { DirDurableStore } from '../src/tenant-state/durable-store.js';
+import { discardLocalLedger, forgetTenant } from '../src/tenant-state/epoch-copies.js';
+import { FencedError, TenantDrainingError } from '../src/tenant-state/errors.js';
+import { deriveLedgerTenantId, leaseKeyFor, tenantRootFor } from '../src/tenant-state/ids.js';
+import { SqliteLeaseStore } from '../src/tenant-state/lease-store.js';
+import { TenantLease } from '../src/tenant-state/tenant-lease.js';
+import { TenantRouter } from '../src/tenant-state/tenant-router.js';
+import { ledgerDbPath } from '../src/state-ledger.js';
+import { FaultyDurableStore, ManualClock } from './tenant-state-test-kit.js';
+const ORG = 'acme';
+const KEY = Buffer.alloc(32, 9);
+const TENANT = deriveLedgerTenantId(KEY, 'https://idp.example/acme', 'handoff-user');
+const LEASE_KEY = leaseKeyFor(ORG, TENANT);
+const TENANT_2 = deriveLedgerTenantId(KEY, 'https://idp.example/acme', 'handoff-user-2');
+const TTL = 10_000;
+const MARGIN = 2_000;
+let passed = 0;
+const only = process.env.GOTRY_TEST_ONLY;
+async function check(label, assertion) {
+    if (only && !label.startsWith(only))
+        return;
+    await assertion();
+    passed += 1;
+    console.log(`  ok - ${label}`);
+}
+const root = mkdtempSync(join(tmpdir(), 'gotry-tenant-handoff-'));
+const cleanups = [];
+/** A call that has to come back: when it does not, the check fails with this message instead of the suite stalling. */
+async function within(promise, ms, what) {
+    let timer;
+    try {
+        return await Promise.race([promise, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(`${what} did not settle within ${ms} ms`)), ms); })]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+const settle = (promise) => promise.then((value) => ({ ok: true, value }), (error) => ({ ok: false, error }));
+async function waitFor(condition, timeoutMs, what) {
+    const until = Date.now() + timeoutMs;
+    while (!condition()) {
+        if (Date.now() > until)
+            throw new Error(`timed out waiting for ${what}`);
+        await sleep(10);
+    }
+}
+async function eventually(check, timeoutMs, what) {
+    const until = Date.now() + timeoutMs;
+    while (!await check()) {
+        if (Date.now() > until)
+            throw new Error(`timed out waiting for ${what}`);
+        await sleep(20);
+    }
+}
+const codeOf = (error) => error?.code;
+function makeFleet(name, options = {}) {
+    const dir = join(root, name);
+    const storeClock = new ManualClock(1_000_000);
+    const clocks = [];
+    const leases = options.realTime
+        ? new SqliteLeaseStore({ path: join(dir, 'leases.db') })
+        : new SqliteLeaseStore({ path: join(dir, 'leases.db'), nowMs: () => storeClock.nowMs() });
+    const durable = new FaultyDurableStore(new DirDurableStore(join(dir, 'durable')));
+    cleanups.push(() => leases.close());
+    return {
+        durable,
+        leases,
+        /** `node` names the state root; two replicas given the same one share it, as a misconfiguration would make them. */
+        replica(id, onEvent, node = id) {
+            const clock = new ManualClock(5_000);
+            clocks.push(clock);
+            const events = [];
+            const stateRoot = join(dir, 'nodes', node);
+            const router = new TenantRouter({
+                stateRoot,
+                replicaId: id,
+                leases: options.wrapLeases ? options.wrapLeases(leases) : leases,
+                durable,
+                ttlMs: options.realTime?.ttlMs ?? TTL,
+                marginMs: options.realTime?.marginMs ?? MARGIN,
+                ...(options.realTime ? {} : { clock, autoRenew: false }),
+                snapshotEveryMs: 0,
+                ...options.router,
+                onEvent: (event) => { events.push(event); onEvent?.(event); },
+            });
+            // A tenant with a call that never returns keeps close() waiting for its drain deadline: the cleanup must not wait for ever.
+            cleanups.push(() => within(router.close(), 5_000, 'router.close() in the cleanup').then(() => undefined, () => undefined));
+            return { router, clock, stateRoot, events };
+        },
+        /** Time passes everywhere: the store and every replica's own monotonic clock. */
+        advance(ms) {
+            storeClock.advance(ms);
+            for (const clock of clocks)
+                clock.advance(ms);
+        },
+    };
+}
+function insert(handle, idemKey) {
+    handle.run((ledger) => { assert.notEqual(ledger.insertEvent({ actor: 'test', kind: 'tenant-state.test', payload: { idemKey }, idemKey }), null); });
+}
+/** A write made straight through the ledger a consumer holds, outside any `run` or `runAsync` bracket. */
+function insertThroughView(handle, idemKey) {
+    assert.notEqual(handle.ledger.insertEvent({ actor: 'test', kind: 'tenant-state.test', payload: { idemKey }, idemKey }), null);
+}
+function keysOf(handle) {
+    return handle.run(ledger => ledger.db.prepare("SELECT idem_key FROM events WHERE kind = 'tenant-state.test' ORDER BY seq").all().map(row => row.idem_key));
+}
+const isDraining = (error) => error instanceof TenantDrainingError;
+try {
+    // ---- H: handoff ------------------------------------------------------------------------------------
+    console.log('-- H handoff');
+    await check('H1 a write made through the ledger after the final image is taken is refused, so nothing acknowledged is lost', async () => {
+        const fleet = makeFleet('h1');
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        const acknowledged = ['before-1', 'before-2'];
+        for (const key of acknowledged)
+            insert(handle, key);
+        let refusal;
+        // The hook runs inside the final upload, after the image was taken: the moment a late writer used to slip in.
+        fleet.durable.beforeNextList = async () => {
+            try {
+                insertThroughView(handle, 'late-1');
+                acknowledged.push('late-1');
+            }
+            catch (error) {
+                refusal = error;
+            }
+        };
+        await handle.handoff();
+        const next = await b.router.openTenant(ORG, TENANT);
+        const restored = keysOf(next);
+        for (const key of acknowledged)
+            assert.ok(restored.includes(key), `${key} was acknowledged and has to be in the successor's ledger, which holds ${JSON.stringify(restored)}`);
+        assert.ok(isDraining(refusal), `the late write is refused with the draining error, got ${String(refusal)}`);
+    });
+    await check('H2 reading through the ledger is refused during the final flush too, with the same typed error, and works again if the flush fails', async () => {
+        const fleet = makeFleet('h2');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let readDuringFlush;
+        fleet.durable.beforeNextList = async () => {
+            try {
+                handle.ledger.db.prepare('SELECT 1').get();
+            }
+            catch (error) {
+                readDuringFlush = error;
+            }
+            fleet.durable.down = true;
+        };
+        await assert.rejects(handle.handoff(), /injected outage/);
+        assert.ok(isDraining(readDuringFlush), `the view is closed while the image is being flushed, got ${String(readDuringFlush)}`);
+        fleet.durable.down = false;
+        assert.equal(handle.isOpen(), true, 'a flush that failed for want of a store leaves the tenant serving');
+        insert(handle, 'k-2');
+        assert.deepEqual(keysOf(handle), ['k-1', 'k-2']);
+        insertThroughView(handle, 'k-3');
+        await handle.handoff();
+    });
+    await check('H3 a flush whose lease lapsed during the upload is reported as failed, not as flushed', async () => {
+        const fleet = makeFleet('h3');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        // The upload takes longer than the lease: by the time it lands, the replica no longer provably holds the tenant.
+        fleet.durable.beforeNextList = async () => { fleet.advance(TTL + 1); };
+        await assert.rejects(handle.handoff(), (error) => error instanceof FencedError);
+        assert.ok(!a.events.some(event => event.type === 'released' && event.flushed), 'nothing claims the tenant was flushed');
+        assert.ok(a.events.some(event => event.type === 'snapshot_failed'), 'and the failed snapshot is told apart from a published one');
+    });
+    await check('H3b the same, with a successor that took over while the upload was under way: it keeps what it pinned, and the old owner is told its flush did not count', async () => {
+        const fleet = makeFleet('h3b');
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        await handle.snapshot();
+        insert(handle, 'k-2');
+        insert(handle, 'k-3');
+        let successor;
+        fleet.durable.beforeNextList = async () => {
+            fleet.advance(TTL + 1);
+            successor = await b.router.openTenant(ORG, TENANT);
+        };
+        await assert.rejects(handle.handoff(), (error) => error instanceof FencedError);
+        assert.ok(!a.events.some(event => event.type === 'released' && event.flushed), 'nothing claims the tenant was flushed');
+        assert.ok(successor, 'the successor took over');
+        assert.deepEqual(keysOf(successor), ['k-1'], 'the late image is above the successor\'s pinned base and is never read');
+    });
+    await check('H4 the lease is renewed while a handoff drains in-flight work, so work longer than the lease is still flushed', async () => {
+        const fleet = makeFleet('h4', { realTime: { ttlMs: 1_200, marginMs: 200 } });
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let finished = false;
+        const work = handle.runAsync(async (ledger) => {
+            await sleep(2_000);
+            assert.notEqual(ledger.insertEvent({ actor: 'test', kind: 'tenant-state.test', payload: {}, idemKey: 'k-2' }), null);
+            finished = true;
+        });
+        await sleep(100);
+        await handle.handoff();
+        await work;
+        assert.equal(finished, true);
+        assert.ok(!a.events.some(event => event.type === 'lost'), 'the lease was never lost while the drain waited');
+        const next = await b.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(next), ['k-1', 'k-2'], 'the work that outlived the lease is in the final image');
+    });
+    await check('H5 a handoff that resumes service after a failed flush does not double the renewal timers', async () => {
+        const fleet = makeFleet('h5', { realTime: { ttlMs: 900, marginMs: 100 } });
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        for (let attempt = 0; attempt < 3; attempt += 1) {
+            fleet.durable.down = true;
+            await assert.rejects(handle.handoff(), /injected outage/);
+            fleet.durable.down = false;
+        }
+        a.events.length = 0;
+        await sleep(1_300);
+        const renewals = a.events.filter(event => event.type === 'renewed').length;
+        assert.ok(renewals >= 2 && renewals <= 6, `one renewal chain runs (renewTick every 300 ms), saw ${renewals} renewals in 1.3 s`);
+        await handle.handoff();
+    });
+    await check('H6 a handoff that is overtaken by an abandon stays abandoned: it flushes nothing and does not bring the handle back', async () => {
+        const fleet = makeFleet('h6');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let release;
+        const hold = new Promise((resolve) => { release = resolve; });
+        const work = handle.runAsync(async () => { await hold; });
+        const handingOff = handle.handoff();
+        await sleep(20);
+        await handle.abandon();
+        release();
+        await work.catch(() => undefined);
+        await assert.rejects(handingOff, isDraining);
+        assert.equal(handle.isOpen(), false, 'the abandoned handle is not serving again');
+        assert.equal(a.router.handleOf(ORG, TENANT), null);
+        const stored = await fleet.durable.list(`${LEASE_KEY}/`);
+        assert.deepEqual(stored.filter(key => /snap-/.test(key)), [], `nothing was flushed after the abandon, the store holds ${JSON.stringify(stored)}`);
+    });
+    // ---- V: the guarded view ------------------------------------------------------------------------------
+    console.log('-- V the guarded view');
+    await check('V1 no method of the guarded database returns the raw one, so a lease that is gone cannot be bypassed by chaining', async () => {
+        const fleet = makeFleet('v1');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        const chained = handle.ledger.db.exec('CREATE TABLE IF NOT EXISTS probe (a INTEGER)');
+        const again = handle.ledger.db.defaultSafeIntegers(false);
+        // The replica's own clock passes the end of the lease: from here every call through the view has to fail.
+        a.clock.advance(TTL);
+        assert.throws(() => handle.ledger.db.prepare('SELECT 1'), (error) => error instanceof FencedError);
+        for (const returned of [chained, again]) {
+            assert.throws(() => returned.prepare('INSERT INTO probe(a) VALUES (1)').run(), (error) => error instanceof FencedError, 'what exec and its siblings return is still behind the gate');
+        }
+    });
+    await check('V2 closing the database through the view does not close the router\'s own connection', async () => {
+        const fleet = makeFleet('v2');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        handle.ledger.db.close();
+        handle.ledger.close();
+        insert(handle, 'k-1');
+        assert.deepEqual(keysOf(handle), ['k-1'], 'the router\'s connection is still open and the write is there');
+    });
+    await check('V3 the raw database is not reachable through a property descriptor, and the view cannot be rewritten', async () => {
+        const fleet = makeFleet('v3');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        const viaOne = Object.getOwnPropertyDescriptor(handle.ledger, 'db')?.value;
+        const viaAll = Object.getOwnPropertyDescriptors(handle.ledger).db?.value;
+        a.clock.advance(TTL);
+        for (const [name, db] of [['getOwnPropertyDescriptor', viaOne], ['getOwnPropertyDescriptors', viaAll]]) {
+            assert.throws(() => db.prepare('SELECT 1'), (error) => error instanceof FencedError, `${name} hands out the guarded database, which refuses once the lease is gone`);
+        }
+        assert.throws(() => { handle.ledger.db = null; }, TypeError, 'the view cannot be assigned to');
+        assert.throws(() => { delete handle.ledger.db; }, TypeError, 'and nothing can be deleted from it');
+        assert.throws(() => Object.setPrototypeOf(handle.ledger, null), TypeError, 'its prototype stays');
+        assert.throws(() => Object.freeze(handle.ledger), TypeError, 'and it cannot be frozen or sealed, which would reach the ledger the router holds');
+        assert.throws(() => Object.preventExtensions(handle.ledger), TypeError, 'or made non-extensible');
+        assert.equal(Object.isExtensible(handle.ledger), true, 'and the ledger the router holds is as extensible as before, whatever was tried');
+    });
+    // ---- F: forget ----------------------------------------------------------------------------------------
+    console.log('-- F forget');
+    await check('F1 a snapshot upload under way cannot outlive forget: the tenant leaves nothing in the store', async () => {
+        const fleet = makeFleet('f1');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let release;
+        const gate = new Promise((resolve) => { release = resolve; });
+        fleet.durable.beforeNextList = async () => { await gate; };
+        const uploading = handle.snapshot().catch(() => undefined);
+        await sleep(50);
+        const forgetting = handle.forget();
+        await sleep(50);
+        release();
+        await Promise.all([uploading, forgetting]);
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/`), [], 'no image, manifest or base of the forgotten tenant is left');
+        assert.equal(existsSync(tenantRootFor(a.stateRoot, ORG, TENANT)), false, 'and no local file');
+    });
+    await check('F2 a snapshot requested after forget has begun is refused and uploads nothing', async () => {
+        const fleet = makeFleet('f2');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        const forgetting = handle.forget();
+        await assert.rejects(handle.snapshot(), isDraining);
+        await forgetting;
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/`), []);
+    });
+    await check('F3 forget while the final flush is under way wins: the handoff does not release the lease, and nothing of the tenant stays in the store', async () => {
+        const fleet = makeFleet('f3');
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        insert(handle, 'k-2');
+        let release;
+        const gate = new Promise((resolve) => { release = resolve; });
+        fleet.durable.beforeNextList = async () => { await gate; };
+        const handingOff = settle(handle.handoff());
+        const forgetting = settle((async () => { await sleep(50); return handle.forget(); })());
+        await sleep(100);
+        release();
+        const [handoff, forgotten] = await Promise.all([within(handingOff, 3_000, 'the handoff'), within(forgetting, 3_000, 'forget')]);
+        assert.equal(forgotten.ok, true, `forget resolved, got ${forgotten.ok ? '' : String(forgotten.error)}`);
+        assert.equal(handoff.ok, false);
+        if (!handoff.ok)
+            assert.ok(isDraining(handoff.error), `and the handoff is rejected with the draining error, got ${String(handoff.error)}`);
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/`), [], 'no image, manifest or base of the forgotten tenant is left');
+        assert.ok(!a.events.some(event => event.type === 'released' && event.flushed), 'nothing claims the tenant was flushed');
+        const next = await b.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(next), [], 'the successor starts from an empty ledger');
+    });
+    await check('F4 forget keeps the lease renewed while it waits for a slow upload, so it is not fenced before it can delete', async () => {
+        const fleet = makeFleet('f4', { realTime: { ttlMs: 1_000, marginMs: 150 }, router: { renewEveryMs: 250 } });
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        await handle.snapshot();
+        insert(handle, 'k-2');
+        fleet.durable.slowPut = { match: /\.db$/, ms: 2_000 };
+        const uploading = settle(handle.snapshot());
+        await sleep(50);
+        const forgotten = await settle(within(handle.forget(), 10_000, 'forget over a slow upload'));
+        await uploading;
+        assert.equal(forgotten.ok, true, `forget resolved, got ${forgotten.ok ? '' : String(forgotten.error)}`);
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/`), [], 'and the image that was uploading is gone with the rest');
+    });
+    await check('F5 while forget waits for an upload, calls through the handle and the ledger are refused with the typed draining error', async () => {
+        const fleet = makeFleet('f5');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let release;
+        const gate = new Promise((resolve) => { release = resolve; });
+        fleet.durable.beforeNextList = async () => { await gate; };
+        const uploading = settle(handle.snapshot());
+        await sleep(30);
+        const forgetting = settle(handle.forget());
+        try {
+            await sleep(30);
+            assert.throws(() => insert(handle, 'late-1'), isDraining);
+            assert.throws(() => handle.run(() => 1), isDraining, 'a callback that never touches the ledger is refused too: the handle, not only the view, says no');
+            assert.throws(() => insertThroughView(handle, 'late-2'), isDraining);
+            assert.throws(() => handle.ledger.db.prepare('SELECT 1'), isDraining);
+        }
+        finally {
+            release();
+        }
+        await Promise.all([uploading, forgetting]);
+    });
+    await check('F6 while forget waits for an upload, an abandon does not take the lease from under it, and a second forget is refused', async () => {
+        const fleet = makeFleet('f6');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let release;
+        const gate = new Promise((resolve) => { release = resolve; });
+        fleet.durable.beforeNextList = async () => { await gate; };
+        const uploading = settle(handle.snapshot());
+        await sleep(30);
+        const forgetting = settle(handle.forget());
+        try {
+            await sleep(30);
+            await handle.abandon();
+            await within(assert.rejects(handle.forget(), isDraining), 1_000, 'a second forget');
+        }
+        finally {
+            release();
+        }
+        const [, forgotten] = await Promise.all([uploading, within(forgetting, 3_000, 'forget')]);
+        assert.equal(forgotten.ok, true, `forget finished what it began, got ${forgotten.ok ? '' : String(forgotten.error)}`);
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/`), [], 'and the store holds nothing of the tenant');
+    });
+    await check('F7 forget deletes only what is at or below the floor it raised: a successor that took over while it was paused keeps everything it wrote', async () => {
+        let afterFloor = null;
+        const fleet = makeFleet('f7', {
+            wrapLeases: (inner) => ({
+                read: (key) => inner.read(key),
+                acquire: (key, owner, ttlMs) => inner.acquire(key, owner, ttlMs),
+                renew: (key, owner, epoch, ttlMs) => inner.renew(key, owner, epoch, ttlMs),
+                release: (key, owner, epoch) => inner.release(key, owner, epoch),
+                markForgotten: async (key, owner, epoch) => {
+                    const record = await inner.markForgotten(key, owner, epoch);
+                    const hook = afterFloor;
+                    afterFloor = null;
+                    await hook?.();
+                    return record;
+                },
+            }),
+        });
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const c = fleet.replica('replica-c');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'a-1');
+        await handle.snapshot();
+        let successor;
+        afterFloor = async () => {
+            // The forgetting owner is paused right here, past its lease. The successor takes the tenant over, writes and publishes above the floor.
+            fleet.advance(TTL + 1);
+            successor = await b.router.openTenant(ORG, TENANT);
+            insert(successor, 'b-1');
+            await successor.snapshot();
+        };
+        await handle.forget();
+        assert.ok(successor, 'the successor took the tenant over');
+        const kept = await fleet.durable.list(`${LEASE_KEY}/e00000002/`);
+        assert.ok(kept.some(key => key.endsWith('.db')), `the successor\'s image is still in the store, the store holds ${JSON.stringify(kept)}`);
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/e00000001/`), [], 'and what the forgotten tenure left is gone');
+        await successor.abandon();
+        const next = await c.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(next), ['b-1'], 'so the successor\'s crash loses nothing it had published');
+    });
+    await check('F8 closing the router waits for a forget that is under way, though the handle has left its table', async () => {
+        const fleet = makeFleet('f8');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let release;
+        const gate = new Promise((resolve) => { release = resolve; });
+        fleet.durable.beforeNextList = async () => { await gate; };
+        const uploading = settle(handle.snapshot());
+        await sleep(30);
+        const forgetting = settle(handle.forget());
+        await sleep(30);
+        let closed = false;
+        const closing = a.router.close().then((failed) => { closed = true; return failed; });
+        try {
+            await sleep(150);
+            assert.equal(closed, false, 'close() is still waiting for the forget');
+        }
+        finally {
+            release();
+        }
+        assert.deepEqual(await within(closing, 3_000, 'router.close()'), []);
+        assert.equal((await within(forgetting, 3_000, 'forget')).ok, true);
+        await uploading;
+        assert.deepEqual(await fleet.durable.list(`${LEASE_KEY}/`), [], 'the forget finished before close() returned');
+        assert.ok(a.events.some(event => event.type === 'released' && !event.flushed), 'and it handed the lease back');
+    });
+    // ---- S: dead epochs --------------------------------------------------------------------------------------
+    console.log('-- S dead epochs');
+    await check('S1 what a superseded owner uploaded after a forget is deleted when the tenant is next opened, and nothing else is', async () => {
+        const fleet = makeFleet('s1');
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        await handle.snapshot();
+        await handle.forget();
+        const other = leaseKeyFor(ORG, TENANT_2);
+        // The late upload of an owner that did not know it was superseded lands in the dead epoch, beside a neighbour's own object.
+        await fleet.durable.putIfAbsent(`${LEASE_KEY}/e00000001/snap-000009.db`, Buffer.from('late'));
+        await fleet.durable.putIfAbsent(`${LEASE_KEY}/e00000001/snap-000009.json`, Buffer.from('{}'));
+        await fleet.durable.putIfAbsent(`${other}/e00000001/base.json`, Buffer.from('{}'));
+        assert.equal((await fleet.durable.list(`${LEASE_KEY}/`)).length, 2);
+        const next = await b.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(next), []);
+        // The sweep runs beside the open, not in it, so a slow store cannot use up the lease before the first renewal.
+        await eventually(async () => (await fleet.durable.list(`${LEASE_KEY}/`)).filter(key => /\/e0*1\//.test(key)).length === 0, 3_000, 'the dead epoch to be swept');
+        assert.equal((await fleet.durable.list(`${other}/`)).length, 1, 'and another tenant\'s objects were not touched');
+    });
+    await check('S2 a sweep that fails does not fail the open: the tenant serves, and the next open sweeps again', async () => {
+        const fleet = makeFleet('s2');
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const c = fleet.replica('replica-c');
+        const first = await a.router.openTenant(ORG, TENANT);
+        insert(first, 'k-1');
+        await first.snapshot();
+        await first.forget();
+        await fleet.durable.putIfAbsent(`${LEASE_KEY}/e00000001/snap-000009.db`, Buffer.from('late'));
+        fleet.durable.failNextDelete = true;
+        const second = await b.router.openTenant(ORG, TENANT);
+        insert(second, 'k-2');
+        assert.deepEqual(keysOf(second), ['k-2'], 'the tenant serves although the sweep beside its open failed');
+        await sleep(150);
+        assert.equal((await fleet.durable.list(`${LEASE_KEY}/e00000001/`)).length, 1, 'the failed sweep deleted nothing');
+        await second.handoff();
+        const third = await c.router.openTenant(ORG, TENANT);
+        await eventually(async () => (await fleet.durable.list(`${LEASE_KEY}/e00000001/`)).length === 0, 3_000, 'the next open to sweep the dead epoch');
+        assert.equal(third.isOpen(), true);
+    });
+    // ---- P: paths ---------------------------------------------------------------------------------------------
+    console.log('-- P paths');
+    await check('P1 forgetting never deletes a directory that is not the tenant\'s own, whatever root the caller supplies', async () => {
+        const dir = join(root, 'p1');
+        const storeClock = new ManualClock(1_000_000);
+        const leases = new SqliteLeaseStore({ path: join(dir, 'leases.db'), nowMs: () => storeClock.nowMs() });
+        cleanups.push(() => leases.close());
+        const durable = new DirDurableStore(join(dir, 'durable'));
+        const lease = await TenantLease.acquire({ store: leases, key: LEASE_KEY, replicaId: 'replica-p1', ttlMs: TTL, marginMs: MARGIN, clock: new ManualClock(5_000) });
+        const keep = (path) => { mkdirSync(path, { recursive: true }); writeFileSync(join(path, 'precious.txt'), 'keep'); };
+        const unrelated = join(dir, 'somebody-elses-files');
+        const sibling = tenantRootFor(join(dir, 'node'), ORG, TENANT_2);
+        keep(unrelated);
+        keep(sibling);
+        for (const tenantRoot of [unrelated, sibling, join(sibling, '..')]) {
+            await assert.rejects(forgetTenant({ durable, lease, organization: ORG, ledgerTenant: TENANT, tenantRoot }), RangeError, `a root of ${tenantRoot} is refused`);
+        }
+        for (const path of [unrelated, sibling])
+            assert.equal(readFileSync(join(path, 'precious.txt'), 'utf8'), 'keep', `${path} is untouched`);
+        const own = tenantRootFor(join(dir, 'node'), ORG, TENANT);
+        keep(own);
+        await forgetTenant({ durable, lease, organization: ORG, ledgerTenant: TENANT, tenantRoot: own });
+        assert.equal(existsSync(own), false, 'the tenant\'s own root is the one that goes');
+    });
+    await check('P3 two replicas that share one state root by misconfiguration: the successor\'s restore leaves the older writer holding a file that is no longer on the path, and the older writer is fenced', async () => {
+        const fleet = makeFleet('p3');
+        const a = fleet.replica('replica-a', undefined, 'shared-root');
+        const b = fleet.replica('replica-b', undefined, 'shared-root');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        await handle.snapshot();
+        const file = ledgerDbPath(tenantRootFor(a.stateRoot, ORG, TENANT));
+        // What a writer that is still running has open. Holding it also keeps its inode from being handed to the file that
+        // replaces it, which a file system may do the moment a file is deleted (Linux does; comparing inode numbers is no test).
+        const held = openSync(file, 'r');
+        try {
+            fleet.advance(TTL + 1);
+            const next = await b.router.openTenant(ORG, TENANT);
+            assert.equal(fstatSync(held).nlink, 0, 'the file the older writer has open is unlinked: the restore put another one on the path');
+            assert.notEqual(statSync(file).ino, fstatSync(held).ino, 'and the path leads to a different file');
+            assert.throws(() => insert(handle, 'zombie'), (error) => error instanceof FencedError);
+            assert.deepEqual(keysOf(next), ['k-1'], 'the successor\'s ledger is the durable copy');
+        }
+        finally {
+            closeSync(held);
+        }
+    });
+    await check('P4 the directory that was checked is the one that goes: a ".." behind a symlink cannot lead the deletion somewhere else, and discarding the ledger takes the same guard', async () => {
+        const dir = join(root, 'p4');
+        const storeClock = new ManualClock(1_000_000);
+        const leases = new SqliteLeaseStore({ path: join(dir, 'leases.db'), nowMs: () => storeClock.nowMs() });
+        cleanups.push(() => leases.close());
+        const durable = new DirDurableStore(join(dir, 'durable'));
+        const lease = await TenantLease.acquire({ store: leases, key: LEASE_KEY, replicaId: 'replica-p4', ttlMs: TTL, marginMs: MARGIN, clock: new ManualClock(5_000) });
+        const tail = join('orgs', ORG, 'tenants', TENANT);
+        const deep = join(dir, 'elsewhere', 'deep');
+        mkdirSync(deep, { recursive: true });
+        symlinkSync(deep, join(dir, 'lnk'));
+        const checked = join(dir, tail);
+        const behindTheLink = join(dir, 'elsewhere', tail);
+        for (const path of [checked, behindTheLink]) {
+            mkdirSync(path, { recursive: true });
+            writeFileSync(join(path, 'precious.txt'), 'keep');
+        }
+        // Read as text this ends in the tenant\'s own path; the operating system reads "lnk/.." as the parent of what the link points to.
+        const tenantRoot = `${join(dir, 'lnk')}/../${tail}`;
+        assert.throws(() => discardLocalLedger({ durable, lease, organization: ORG, ledgerTenant: TENANT, tenantRoot: join(dir, 'somebody-elses-files') }), RangeError);
+        await forgetTenant({ durable, lease, organization: ORG, ledgerTenant: TENANT, tenantRoot });
+        assert.equal(readFileSync(join(behindTheLink, 'precious.txt'), 'utf8'), 'keep', 'the directory the string leads to behind the link is untouched');
+        assert.equal(existsSync(checked), false, 'the one that was checked is the one that went');
+    });
+    await check('P2 deleting by prefix takes whole segments only, so a name that is the start of another one cannot take its neighbour', async () => {
+        const store = new DirDurableStore(join(root, 'p2'));
+        await store.putIfAbsent('acme/t1/e00000001/base.json', Buffer.from('1'));
+        await store.putIfAbsent('acme2/t9/e00000001/base.json', Buffer.from('2'));
+        await assert.rejects(store.deletePrefix('acme'), RangeError);
+        assert.equal((await store.list('acme2/')).length, 1, 'the neighbour is untouched');
+        assert.equal(await store.deletePrefix('acme/'), 1);
+        assert.equal((await store.list('acme2/')).length, 1);
+    });
+    // ---- D: the drain ------------------------------------------------------------------------------------
+    console.log('-- D drain');
+    await check('D1 a drain that never finishes is bounded: the handoff is rejected with the typed timeout, the operator is told, and the tenant serves again', async () => {
+        const fleet = makeFleet('d1', { realTime: { ttlMs: 2_000, marginMs: 200 }, router: { drainTimeoutMs: 300, renewEveryMs: 200 } });
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        handle.runAsync(() => new Promise(() => undefined)).catch(() => undefined);
+        const outcome = await settle(within(handle.handoff(), 3_000, 'a handoff over a call that never returns'));
+        assert.equal(outcome.ok, false);
+        if (!outcome.ok)
+            assert.equal(codeOf(outcome.error), 'GOTRY_TENANT_DRAIN_TIMEOUT', `rejected with the drain timeout, got ${String(outcome.error)}`);
+        assert.equal(handle.isOpen(), true, 'the tenant serves again');
+        insert(handle, 'k-2');
+        await handle.snapshot();
+        assert.deepEqual(keysOf(handle), ['k-1', 'k-2']);
+        assert.ok(a.events.some(event => event.type === 'drain_timeout'), 'the operator is told');
+        assert.ok(!a.events.some(event => event.type === 'released'), 'and nothing was released');
+    });
+    await check('D2 close() hands tenants off side by side, so one stuck tenant does not hold the others behind it', async () => {
+        const fleet = makeFleet('d2', { realTime: { ttlMs: 3_000, marginMs: 300 }, router: { drainTimeoutMs: 1_500, renewEveryMs: 300 } });
+        const a = fleet.replica('replica-a');
+        const first = await a.router.openTenant(ORG, TENANT);
+        const second = await a.router.openTenant(ORG, TENANT_2);
+        insert(first, 'k-1');
+        insert(second, 'k-1');
+        first.runAsync(() => new Promise(() => undefined)).catch(() => undefined);
+        const closing = a.router.close();
+        const secondKey = leaseKeyFor(ORG, TENANT_2);
+        const handedOff = () => a.events.some(event => event.type === 'released' && event.key === secondKey && event.flushed);
+        await waitFor(handedOff, 1_000, 'the second tenant to be handed off while the first is still draining');
+        assert.deepEqual(await within(closing, 8_000, 'router.close() with one tenant stuck'), [LEASE_KEY], 'the stuck tenant is reported, and only it');
+    });
+    for (const how of ['abandon', 'forget', 'lost']) {
+        await check(`D3 ${how}: a handoff that waits for a call that never returns is settled by it, not left pending`, async () => {
+            const fleet = makeFleet(`d3-${how}`, { router: { drainTimeoutMs: 10_000 } });
+            const a = fleet.replica('replica-a');
+            const b = fleet.replica('replica-b');
+            const handle = await a.router.openTenant(ORG, TENANT);
+            insert(handle, 'k-1');
+            handle.runAsync(() => new Promise(() => undefined)).catch(() => undefined);
+            const handingOff = settle(handle.handoff());
+            await sleep(30);
+            if (how === 'abandon')
+                await handle.abandon();
+            else if (how === 'forget')
+                await handle.forget();
+            else {
+                // The lease lapses, a successor takes over, and the next renewal is refused: the tenure is over.
+                fleet.advance(TTL + 1);
+                await b.router.openTenant(ORG, TENANT);
+                await handle.renew().catch(() => undefined);
+            }
+            const outcome = await within(handingOff, 1_000, `a handoff of a tenant that was ${how} meanwhile`);
+            assert.equal(outcome.ok, false);
+            if (!outcome.ok)
+                assert.ok(isDraining(outcome.error), `the handoff is rejected with the draining error, got ${String(outcome.error)}`);
+            assert.equal(handle.isOpen(), false);
+            assert.ok(!a.events.some(event => event.type === 'released' && event.flushed), 'and nothing claims a flush');
+        });
+    }
+    await check('D4 while a handoff drains, new calls through the handle are refused with the draining error, and the work that was running finishes and is flushed', async () => {
+        const fleet = makeFleet('d4');
+        const a = fleet.replica('replica-a');
+        const b = fleet.replica('replica-b');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        let finish;
+        const hold = new Promise((resolve) => { finish = resolve; });
+        const running = handle.runAsync(async (ledger) => {
+            await hold;
+            assert.notEqual(ledger.insertEvent({ actor: 'test', kind: 'tenant-state.test', payload: {}, idemKey: 'k-2' }), null);
+        });
+        const handingOff = settle(handle.handoff());
+        try {
+            await sleep(30);
+            assert.throws(() => handle.run(() => 1), isDraining, 'run() is refused, whatever the callback does');
+            await assert.rejects(handle.runAsync(async () => undefined), isDraining);
+            assert.equal(handle.isOpen(), false);
+        }
+        finally {
+            finish();
+        }
+        await running;
+        const outcome = await within(handingOff, 3_000, 'the handoff');
+        assert.equal(outcome.ok, true, `the handoff finished once the work did, got ${outcome.ok ? '' : String(outcome.error)}`);
+        const next = await b.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(next), ['k-1', 'k-2'], 'and the work that was running is in the final image');
+    });
+    await check('D5 a call on a tenant whose abandon is under way is refused with the draining error, not let through to a closed database', async () => {
+        const fleet = makeFleet('d5');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        const abandoning = handle.abandon();
+        assert.throws(() => handle.run(() => 1), isDraining);
+        await within(assert.rejects(handle.forget(), isDraining), 1_000, 'a forget on a tenant whose abandon is under way');
+        await abandoning;
+        assert.equal(a.events.filter(event => event.type === 'released').length, 1, 'and the tenure ended once');
+        assert.throws(() => handle.run(() => 1), (error) => error instanceof FencedError, 'and once the lease is handed back it is the lease that says no');
+    });
+    await check('D6 the drain options are checked, and a close concurrency of one brings the sequential close back', async () => {
+        const fleet = makeFleet('d6');
+        const build = (extra) => new TenantRouter({ stateRoot: join(root, 'd6'), replicaId: 'replica-x', leases: fleet.leases, durable: fleet.durable, ...extra });
+        for (const bad of [0, -1, 1.5, Number.NaN, 2_147_483_648])
+            assert.throws(() => build({ drainTimeoutMs: bad }), RangeError, `drainTimeoutMs ${bad} is refused`);
+        for (const bad of [0, -1, 1.5, Number.NaN])
+            assert.throws(() => build({ closeConcurrency: bad }), RangeError, `closeConcurrency ${bad} is refused`);
+        const sequential = makeFleet('d6-one', { realTime: { ttlMs: 3_000, marginMs: 300 }, router: { drainTimeoutMs: 2_000, renewEveryMs: 300, closeConcurrency: 1 } });
+        const a = sequential.replica('replica-a');
+        const first = await a.router.openTenant(ORG, TENANT);
+        const second = await a.router.openTenant(ORG, TENANT_2);
+        insert(first, 'k-1');
+        insert(second, 'k-1');
+        first.runAsync(() => new Promise(() => undefined)).catch(() => undefined);
+        const closing = a.router.close();
+        await sleep(400);
+        const secondKey = leaseKeyFor(ORG, TENANT_2);
+        assert.ok(!a.events.some(event => event.type === 'released' && event.key === secondKey), 'with one lane the second tenant waits behind the stuck first one');
+        assert.deepEqual(await within(closing, 10_000, 'router.close()'), [LEASE_KEY]);
+        assert.ok(a.events.some(event => event.type === 'released' && event.key === secondKey && event.flushed), 'and is handed off once the first one has given up');
+    });
+    // ---- R: the renewal chain ------------------------------------------------------------------------------
+    console.log('-- R renewal');
+    await check('R1 a failed handoff does not start a second renewal chain while a renewal is still under way', async () => {
+        const gate = { hold: false, started: 0, waiting: [] };
+        const fleet = makeFleet('r1', {
+            realTime: { ttlMs: 5_000, marginMs: 500 },
+            router: { renewEveryMs: 600 },
+            wrapLeases: (inner) => ({
+                read: (key) => inner.read(key),
+                acquire: (key, owner, ttlMs) => inner.acquire(key, owner, ttlMs),
+                release: (key, owner, epoch) => inner.release(key, owner, epoch),
+                markForgotten: (key, owner, epoch) => inner.markForgotten(key, owner, epoch),
+                renew: async (key, owner, epoch, ttlMs) => {
+                    gate.started += 1;
+                    if (gate.hold)
+                        await new Promise((resolve) => { gate.waiting.push(resolve); });
+                    return inner.renew(key, owner, epoch, ttlMs);
+                },
+            }),
+        });
+        const a = fleet.replica('replica-a');
+        gate.hold = true;
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        await waitFor(() => gate.started === 1, 4_000, 'the first renewal to start');
+        fleet.durable.down = true;
+        await assert.rejects(handle.handoff(), /injected outage/);
+        fleet.durable.down = false;
+        // A second chain would fire its timer in this time and queue a renewal behind the one that is still under way.
+        await sleep(900);
+        gate.hold = false;
+        for (const resume of gate.waiting.splice(0))
+            resume();
+        await sleep(120);
+        assert.equal(gate.started, 1, 'the next renewal waits for its own timer: no second chain queued one');
+        await handle.handoff();
+    });
+    // ---- T: run() takes a synchronous callback ---------------------------------------------------------------------
+    console.log('-- T run');
+    await check('T1 run() refuses a callback that returns a promise, and the promise it leaves behind cannot become an unhandled rejection', async () => {
+        const fleet = makeFleet('t1');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        const unhandled = [];
+        const onUnhandled = (reason) => { unhandled.push(reason); };
+        process.on('unhandledRejection', onUnhandled);
+        try {
+            assert.throws(() => handle.run((async () => { await sleep(10); throw new Error('floating'); })), (error) => error instanceof TypeError && /runAsync/.test(error.message));
+            await sleep(80);
+        }
+        finally {
+            process.off('unhandledRejection', onUnhandled);
+        }
+        assert.deepEqual(unhandled, [], 'the rejection of a floating async callback is not left unhandled');
+    });
+    // ---- O: observers ------------------------------------------------------------------------------------
+    console.log('-- O observers');
+    await check('O1 an observer that throws cannot hold a lease: the tenant opens, and the lease is the replica\'s to renew and release', async () => {
+        const fleet = makeFleet('o1');
+        const a = fleet.replica('replica-a', () => { throw new Error('observer bug'); });
+        const handle = await a.router.openTenant(ORG, TENANT);
+        assert.equal(handle.isOpen(), true);
+        insert(handle, 'k-1');
+        await handle.handoff();
+        const b = fleet.replica('replica-b');
+        const next = await b.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(next), ['k-1']);
+    });
+    // ---- L: local files ------------------------------------------------------------------------------------
+    console.log('-- L local files');
+    await check('L1 after a handoff that flushed everything the replica keeps no copy of the ledger; after one that did not, it keeps the only copy', async () => {
+        const fleet = makeFleet('l1');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        const file = ledgerDbPath(tenantRootFor(a.stateRoot, ORG, TENANT));
+        assert.equal(existsSync(file), true);
+        await handle.handoff();
+        for (const suffix of ['', '-wal', '-shm'])
+            assert.equal(existsSync(`${file}${suffix}`), false, `${suffix || 'the ledger'} is gone after a flushed handoff`);
+        const kept = await a.router.openTenant(ORG, TENANT);
+        insert(kept, 'k-2');
+        await kept.abandon();
+        assert.equal(existsSync(file), true, 'an abandoned tenant\'s file may hold writes that exist nowhere else, so it stays for an operator');
+    });
+    await check('L2 the local ledger of an abandoned tenure is replaced, not recovered, by the next open on the same replica: the durable copy is the authority', async () => {
+        const fleet = makeFleet('l2');
+        const a = fleet.replica('replica-a');
+        const handle = await a.router.openTenant(ORG, TENANT);
+        insert(handle, 'k-1');
+        await handle.snapshot();
+        insert(handle, 'k-2');
+        await handle.abandon();
+        const file = ledgerDbPath(tenantRootFor(a.stateRoot, ORG, TENANT));
+        assert.equal(existsSync(file), true, 'the abandoned file is there until the next open');
+        const again = await a.router.openTenant(ORG, TENANT);
+        assert.deepEqual(keysOf(again), ['k-1'], 'k-2 was written after the last snapshot: it is the declared tail of D-a and is not brought back from the old file');
+    });
+    console.log(`tenant-state-handoff: ${passed} checks passed`);
+}
+finally {
+    for (const cleanup of cleanups.reverse()) {
+        try {
+            await cleanup();
+        }
+        catch { /* best effort */ }
+    }
+    rmSync(root, { recursive: true, force: true });
+}