@danceiny/gotry 0.2.0-rc.31 → 0.2.0-rc.33

package/dist/scripts/tenant-state-drill-tests.js

@@ -0,0 +1,298 @@
+/**
+ * Fault-injection drill of the per-tenant state layer (AaaS A1.4, docs/rfc/aaas-rfc.md §4.2, §4.6 and §7; #275 checklist).
+ *
+ * Real OS processes are the replicas. Four of them contend for two tenants through one SQLite lease file and one
+ * durable directory, each on its own state root, writing numbered events and snapshotting every 120 ms. The drill
+ * kills owners with SIGKILL, freezes an owner with SIGSTOP past its lease and resumes it, and hands tenants off
+ * with SIGTERM. Afterwards a verifier takes each tenant over and checks the ledger against what the replicas logged.
+ *
+ * What it proves (all offline, on isolated roots, no network, no model):
+ *   I1  the lease epochs of a tenant are consecutive: no epoch was granted twice or skipped;
+ *   I2  each epoch has at most one writer;
+ *   I3  no write is acknowledged after the next owner took the lease (the local gate works on a real clock);
+ *   I4  the restored ledger is intact, holds only this tenant, and is exactly the lineage the durable copies describe:
+ *       every pinned snapshot holds exactly the writes acknowledged before it began, and nothing else is there;
+ *   I5  a graceful handoff loses nothing, also from a consumer that keeps writing through the ledger it holds while the
+ *       replica hands off (and is refused once the final image is being taken), over an object store that is slow to
+ *       take an image; a crash loses only the writes after the last snapshot;
+ *   I6  the two tenants never see each other's writes.
+ *
+ * A simulated trigger is not the real trigger: this is engineering evidence for A1 and counts for none of A3 or A4.
+ *
+ * Run: cd ts && npx tsx scripts/tenant-state-drill-tests.ts
+ */
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { fileURLToPath } from 'node:url';
+import { DirDurableStore } from '../src/tenant-state/durable-store.js';
+import { LeaseHeldError } from '../src/tenant-state/errors.js';
+import { deriveLedgerTenantId, leaseKeyFor } from '../src/tenant-state/ids.js';
+import { SqliteLeaseStore } from '../src/tenant-state/lease-store.js';
+import { TenantRouter } from '../src/tenant-state/tenant-router.js';
+const TS_DIR = join(dirname(fileURLToPath(import.meta.url)), '..');
+const WORKER = join(TS_DIR, 'scripts', 'tenant-state-drill-worker.ts');
+const ORG = 'acme';
+const ORG_KEY = Buffer.alloc(32, 3);
+const TENANT_A = deriveLedgerTenantId(ORG_KEY, 'https://idp.example/acme', 'user-1');
+const TENANT_B = deriveLedgerTenantId(ORG_KEY, 'https://idp.example/acme', 'user-2');
+const KEY_A = leaseKeyFor(ORG, TENANT_A);
+const KEY_B = leaseKeyFor(ORG, TENANT_B);
+const TENANTS = [{ label: 'a', tenant: TENANT_A, key: KEY_A }, { label: 'b', tenant: TENANT_B, key: KEY_B }];
+const DEADLINE_MS = 170_000;
+const root = mkdtempSync(join(tmpdir(), 'gotry-tenant-drill-'));
+const leaseDb = join(root, 'leases.db');
+const durableDir = join(root, 'durable');
+const logDir = join(root, 'logs');
+mkdirSync(logDir, { recursive: true });
+mkdirSync(join(root, 'tmp'), { recursive: true });
+const leases = new SqliteLeaseStore({ path: leaseDb });
+const durable = new DirDurableStore(durableDir);
+let passed = 0;
+function ok(condition, message) {
+    assert.ok(condition, message);
+    passed += 1;
+}
+function eq(actual, expected, message) {
+    assert.deepEqual(actual, expected, message);
+    passed += 1;
+}
+const replicas = new Map();
+let nextReplica = 1;
+function spawnReplica() {
+    const id = `w${nextReplica++}`;
+    const child = spawn(process.execPath, ['--import', 'tsx', WORKER, join(root, 'nodes', id), leaseDb, durableDir, logDir, id, ORG, TENANT_A, TENANT_B], {
+        cwd: TS_DIR,
+        // A replica killed mid-snapshot cannot clean up its scratch directory, so it gets one inside the drill root.
+        env: { ...process.env, TMPDIR: join(root, 'tmp'), TENANT_DRILL_SLOW_PUT_MS: '40' },
+        stdio: ['ignore', 'ignore', 'pipe'],
+    });
+    const replica = {
+        id,
+        child,
+        stderr: '',
+        exited: new Promise((resolve) => { child.once('close', (code, signal) => { resolve({ code, signal }); }); }),
+    };
+    child.stderr?.on('data', (chunk) => { replica.stderr += chunk.toString(); });
+    replicas.set(id, replica);
+    return replica;
+}
+const deadline = setTimeout(() => {
+    for (const replica of replicas.values())
+        replica.child.kill('SIGKILL');
+    console.error(`DRILL DEADLINE: ${DEADLINE_MS} ms exceeded, replicas killed`);
+    process.exit(1);
+}, DEADLINE_MS);
+async function waitFor(label, probe, accept, timeoutMs = 25_000) {
+    const until = Date.now() + timeoutMs;
+    for (;;) {
+        const value = await probe();
+        if (accept(value))
+            return value;
+        if (Date.now() > until)
+            throw new Error(`timed out waiting for ${label}`);
+        await sleep(25);
+    }
+}
+function readLogs() {
+    const entries = [];
+    for (const file of readdirSync(logDir)) {
+        for (const line of readFileSync(join(logDir, file), 'utf8').split('\n')) {
+            if (!line)
+                continue;
+            try {
+                entries.push(JSON.parse(line));
+            }
+            catch {
+                // A replica killed mid-write leaves one torn last line.
+            }
+        }
+    }
+    return entries.sort((a, b) => a.wall - b.wall);
+}
+const acksIn = (label, epoch) => readLogs().filter(e => e.t === 'ack' && e.tenant === label && e.epoch === epoch).length;
+const epochOf = async (key) => (await leases.read(key))?.epoch ?? 0;
+async function crash(replica) {
+    replica.child.kill('SIGKILL');
+    await replica.exited;
+    replicas.delete(replica.id);
+    spawnReplica();
+}
+async function takeover(label, key, previous) {
+    const epoch = await waitFor(`a new owner of tenant ${label}`, () => epochOf(key), value => value > previous);
+    await waitFor(`acknowledged writes by the owner of epoch ${epoch} of tenant ${label}`, () => acksIn(label, epoch), count => count >= 8);
+    return epoch;
+}
+try {
+    // ---- scenario ------------------------------------------------------------------------------------
+    console.log('-- scenario: four replicas, two tenants, kills, a pause and a graceful handoff');
+    for (let index = 0; index < 4; index += 1)
+        spawnReplica();
+    for (const { label, key } of TENANTS) {
+        await waitFor(`tenant ${label} to be served`, async () => ({ epoch: await epochOf(key), acks: acksIn(label, 1) }), value => value.epoch >= 1 && value.acks >= 15);
+    }
+    await sleep(1_200);
+    const killedEpochs = [];
+    let epochA = await epochOf(KEY_A);
+    for (let round = 1; round <= 2; round += 1) {
+        const record = await leases.read(KEY_A);
+        const owner = replicas.get(record?.owner ?? '');
+        assert.ok(owner, `the owner of tenant a (${record?.owner}) is one of ours`);
+        killedEpochs.push({ label: 'a', epoch: record?.epoch ?? 0 });
+        await crash(owner);
+        epochA = await takeover('a', KEY_A, epochA);
+    }
+    console.log(`  ok - two SIGKILLs of the owner of tenant a: epoch now ${epochA}`);
+    const recordB = await leases.read(KEY_B);
+    const frozen = replicas.get(recordB?.owner ?? '');
+    assert.ok(frozen, 'the owner of tenant b is one of ours');
+    const frozenEpoch = recordB?.epoch ?? 0;
+    frozen.child.kill('SIGSTOP');
+    const takenAt = Date.now();
+    const epochB = await takeover('b', KEY_B, frozenEpoch);
+    await sleep(700);
+    const resumedAt = Date.now();
+    frozen.child.kill('SIGCONT');
+    await sleep(1_800);
+    const afterResume = readLogs().filter(e => e.replica === frozen.id && e.wall >= resumedAt && (e.t === 'refused' || (e.t === 'event' && e.event === 'lost')));
+    assert.ok(afterResume.length > 0, 'a frozen owner that wakes up past its lease is refused or told it lost the tenant, and the logs show it');
+    console.log(`  ok - owner of tenant b frozen past its lease (epoch ${frozenEpoch} -> ${epochB}), resumed after ${resumedAt - takenAt} ms: ${afterResume.length} refusals or losses logged after the resume`);
+    const gracefulOwner = replicas.get((await leases.read(KEY_A))?.owner ?? '');
+    assert.ok(gracefulOwner, 'the owner of tenant a is one of ours');
+    const gracefulEpoch = await epochOf(KEY_A);
+    gracefulOwner.child.kill('SIGTERM');
+    await gracefulOwner.exited;
+    replicas.delete(gracefulOwner.id);
+    spawnReplica();
+    epochA = await takeover('a', KEY_A, gracefulEpoch);
+    console.log(`  ok - graceful SIGTERM handoff of tenant a: epoch ${gracefulEpoch} -> ${epochA}`);
+    await sleep(1_500);
+    await waitFor('every replica to have started', () => readLogs().filter(e => e.t === 'start').length, count => count === nextReplica - 1, 30_000);
+    for (const replica of [...replicas.values()])
+        replica.child.kill('SIGTERM');
+    for (const replica of [...replicas.values()]) {
+        const settled = await Promise.race([replica.exited, sleep(20_000).then(() => 'timeout')]);
+        if (settled === 'timeout')
+            replica.child.kill('SIGKILL');
+        const ended = await replica.exited;
+        assert.ok(settled !== 'timeout' && ended.code === 0, `replica ${replica.id} ended with ${String(ended.code)} ${String(ended.signal)}: ${replica.stderr.slice(0, 400)}`);
+    }
+    clearTimeout(deadline);
+    // ---- verification --------------------------------------------------------------------------------------
+    console.log('-- verification: a fresh replica takes each tenant over and the ledger is compared with the logs');
+    const verifier = new TenantRouter({
+        stateRoot: join(root, 'verifier'), replicaId: 'verifier', leases, durable, ttlMs: 1_500, marginMs: 400, autoRenew: true,
+    });
+    const logs = readLogs();
+    eq(logs.filter(e => e.t === 'start').length, nextReplica - 1, 'every replica logged its start');
+    const gracefulEpochs = new Map();
+    for (const entry of logs) {
+        if (entry.t === 'event' && entry.event === 'released' && entry.flushed === true && entry.key !== undefined && entry.epoch !== undefined) {
+            gracefulEpochs.set(entry.key, (gracefulEpochs.get(entry.key) ?? new Set()).add(entry.epoch));
+        }
+    }
+    let totalAcks = 0;
+    let totalLost = 0;
+    let checkedLinks = 0;
+    for (const { label, tenant, key } of TENANTS) {
+        let handle = null;
+        for (let waited = 0; handle === null; waited += 150) {
+            try {
+                handle = await verifier.openTenant(ORG, tenant);
+            }
+            catch (error) {
+                if (!(error instanceof LeaseHeldError) || waited > 15_000)
+                    throw error;
+                await sleep(150);
+            }
+        }
+        // I1: consecutive epochs, one grant each.
+        const grants = leases.history(key).filter(entry => entry.op === 'acquire');
+        eq(grants.map(entry => entry.epoch), grants.map((_, index) => index + 1), `I1 tenant ${label}: lease epochs are consecutive`);
+        ok(grants.length >= (label === 'a' ? 5 : 3), `tenant ${label} changed hands at least ${label === 'a' ? 5 : 3} times (${grants.length} grants)`);
+        const grantedAt = new Map(grants.map(entry => [entry.epoch, entry.atMs]));
+        // I2 and I3: one writer per epoch, and no acknowledgement after the next owner took the lease.
+        const acks = logs.filter(e => e.t === 'ack' && e.tenant === label);
+        const writers = new Map();
+        for (const ack of acks)
+            writers.set(ack.epoch, (writers.get(ack.epoch) ?? new Set()).add(ack.replica));
+        for (const [epoch, who] of writers)
+            ok(who.size === 1, `I2 tenant ${label} epoch ${epoch}: written by ${[...who].join(', ')}`);
+        for (const ack of acks) {
+            const next = grantedAt.get(ack.epoch + 1);
+            if (next !== undefined)
+                ok(ack.at < next, `I3 tenant ${label}: ack of epoch ${ack.epoch} #${ack.n} at ${ack.at} is before the next grant at ${next}`);
+        }
+        // I4: the lineage described by the durable copies, and the ledger equal to it.
+        const restored = handle.restoredFrom;
+        assert.ok(restored, `tenant ${label} has a durable lineage to restore`);
+        const chain = [];
+        for (let cursor = restored; cursor !== null;) {
+            chain.unshift({ epoch: cursor.epoch, seq: cursor.seq });
+            const baseBytes = await durable.get(`${key}/e${String(cursor.epoch).padStart(8, '0')}/base.json`);
+            assert.ok(baseBytes, `epoch ${cursor.epoch} pinned its base`);
+            const base = JSON.parse(Buffer.from(baseBytes).toString('utf8'));
+            cursor = base.from;
+        }
+        const startsByEpoch = new Map();
+        for (const entry of logs) {
+            if (entry.t === 'snap-start' && entry.tenant === label)
+                startsByEpoch.set(entry.epoch, [...startsByEpoch.get(entry.epoch) ?? [], entry.covered]);
+        }
+        const expected = new Set();
+        const writesInChain = new Map();
+        let previousEvents = 0;
+        for (const link of chain) {
+            const manifestBytes = await durable.get(`${key}/e${String(link.epoch).padStart(8, '0')}/snap-${String(link.seq).padStart(6, '0')}.json`);
+            assert.ok(manifestBytes, `the manifest of e${link.epoch}/snap-${link.seq} exists`);
+            const manifest = JSON.parse(Buffer.from(manifestBytes).toString('utf8'));
+            const written = manifest.events - previousEvents;
+            previousEvents = manifest.events;
+            writesInChain.set(link.epoch, written);
+            const covered = startsByEpoch.get(link.epoch)?.[link.seq - 1];
+            if (covered !== undefined) {
+                eq(written, covered, `I4 tenant ${label}: e${link.epoch}/snap-${link.seq} holds exactly the ${covered} writes acknowledged before it began`);
+                checkedLinks += 1;
+            }
+            for (let n = 1; n <= written; n += 1)
+                expected.add(`${label}:${link.epoch}:${n}`);
+        }
+        const present = handle.run(ledger => ledger.db.prepare("SELECT idem_key FROM events WHERE kind = 'tenant-state.drill'").all().map(row => row.idem_key));
+        eq(new Set(present), expected, `I4/I6 tenant ${label}: the ledger holds exactly the lineage, no write of another epoch, replica or tenant`);
+        eq(present.length, expected.size, `I4 tenant ${label}: no event twice`);
+        // I5: graceful epochs lose nothing; crashed epochs lose only the tail after their last pinned snapshot.
+        for (const epoch of gracefulEpochs.get(key) ?? []) {
+            eq(writesInChain.get(epoch) ?? 0, acksIn(label, epoch), `I5 tenant ${label}: graceful epoch ${epoch} lost no acknowledged write`);
+        }
+        let lost = 0;
+        for (const [epoch, who] of writers) {
+            void who;
+            const tail = acksIn(label, epoch) - (writesInChain.get(epoch) ?? 0);
+            ok(tail >= 0, `I5 tenant ${label} epoch ${epoch}: the lineage holds no more than was acknowledged`);
+            ok(tail <= 400, `I5 tenant ${label} epoch ${epoch}: the lost tail (${tail}) stays within a few snapshot intervals`);
+            lost += tail;
+        }
+        totalAcks += acks.length;
+        totalLost += lost;
+        console.log(`  ok - tenant ${label}: ${grants.length} epochs, ${acks.length} acknowledged writes, lineage of ${chain.length} snapshots with ${expected.size} events, ${lost} acknowledged writes in crashed tails`);
+    }
+    await verifier.close();
+    ok(checkedLinks > 0, `I4 the per-link check ran on ${checkedLinks} snapshots of the lineages, it is not skipped for want of a log`);
+    const viewAcks = logs.filter(e => e.t === 'ack' && e.via === 'view').length;
+    const viewRefusals = logs.filter(e => e.t === 'refused' && e.via === 'view');
+    ok(viewAcks > 0, `I5 a consumer that kept the ledger wrote ${viewAcks} acknowledged events through it while its replica handed off, and none of them is missing from the final images`);
+    ok(viewRefusals.length > 0 && viewRefusals.every(e => (e.code ?? '').startsWith('GOTRY_TENANT_')), `I5 every refusal of that consumer is a typed tenant error (${viewRefusals.length} refusals), and not an untyped failure`);
+    ok(viewRefusals.some(e => e.code === 'GOTRY_TENANT_DRAINING'), 'I5 at least one of them is the draining error: once the final image was being taken, the view said so (a replica starved past its lease may be fenced instead)');
+    const killed = killedEpochs.length;
+    console.log(`tenant-state-drill-tests: ${passed} assertions passed (${killed} SIGKILLs, 1 SIGSTOP past the lease, 1 graceful handoff, ${totalAcks} acknowledged writes, ${totalLost} lost in crashed tails)`);
+}
+finally {
+    clearTimeout(deadline);
+    for (const replica of replicas.values())
+        replica.child.kill('SIGKILL');
+    leases.close();
+    rmSync(root, { recursive: true, force: true });
+}

package/dist/scripts/tenant-state-drill-worker.js

@@ -0,0 +1,164 @@
+/**
+ * Child process of tenant-state-drill-tests.ts: one replica serving two tenants through a TenantRouter.
+ *
+ * argv: <state root> <lease db> <durable dir> <log dir> <replica id> <organization> <tenant a> <tenant b>
+ *
+ * The replica keeps trying to own both tenants. For each tenant it holds, it writes one numbered event at a
+ * time and publishes a snapshot every 120 ms. It logs, before and after every step, what it is about to do and
+ * what was acknowledged, in a file that survives its own death, so the drill can compare the final ledger with
+ * what the replica believed. On SIGTERM it hands every tenant off gracefully and exits; while it does, a consumer that
+ * keeps the ledger it was handed (a booking runtime does) goes on writing through it, inside a `runAsync` bracket the
+ * drain has to wait for, and then tries once more after the bracket, when the view has to refuse.
+ *
+ * TENANT_DRILL_SLOW_PUT_MS makes the object store slow to take an image, as a real one can be.
+ */
+import { closeSync, openSync, writeSync } from 'node:fs';
+import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { DirDurableStore } from '../src/tenant-state/durable-store.js';
+import { LeaseHeldError } from '../src/tenant-state/errors.js';
+import { SqliteLeaseStore } from '../src/tenant-state/lease-store.js';
+import { TenantRouter } from '../src/tenant-state/tenant-router.js';
+const [stateRoot, leaseDb, durableDir, logDir, replicaId, organization, tenantA, tenantB] = process.argv.slice(2);
+const SNAPSHOT_EVERY_MS = 120;
+const RETRY_AFTER_HELD_MS = 150;
+const DRAIN_WRITER_MS = 150;
+const SLOW_PUT_MS = Number(process.env.TENANT_DRILL_SLOW_PUT_MS ?? 0);
+/** The reference store with a delay before every image it takes, so an upload is a window and not an instant. */
+class SlowImages {
+    inner;
+    ms;
+    constructor(inner, ms) {
+        this.inner = inner;
+        this.ms = ms;
+    }
+    async putIfAbsent(key, bytes) {
+        if (key.endsWith('.db'))
+            await sleep(this.ms);
+        return this.inner.putIfAbsent(key, bytes);
+    }
+    get(key) { return this.inner.get(key); }
+    list(prefix) { return this.inner.list(prefix); }
+    deletePrefix(prefix) { return this.inner.deletePrefix(prefix); }
+}
+const log = openSync(join(logDir, `${replicaId}.jsonl`), 'a');
+function record(entry) {
+    writeSync(log, `${JSON.stringify({ ...entry, replica: replicaId, wall: Date.now() })}\n`);
+}
+const router = new TenantRouter({
+    stateRoot,
+    replicaId,
+    leases: new SqliteLeaseStore({ path: leaseDb }),
+    durable: SLOW_PUT_MS > 0 ? new SlowImages(new DirDurableStore(durableDir), SLOW_PUT_MS) : new DirDurableStore(durableDir),
+    ttlMs: 1_500,
+    marginMs: 400,
+    autoRenew: true,
+    renewEveryMs: 300,
+    onEvent: (event) => { record({ t: 'event', event: event.type, key: event.key, epoch: event.epoch, flushed: 'flushed' in event ? event.flushed : undefined }); },
+});
+const seats = [
+    { label: 'a', tenant: tenantA, handle: null, epoch: 0, next: 1, nextTryAt: 0, lastSnapshotAt: 0 },
+    { label: 'b', tenant: tenantB, handle: null, epoch: 0, next: 1, nextTryAt: 0, lastSnapshotAt: 0 },
+];
+let stopping = false;
+process.on('SIGTERM', () => { stopping = true; });
+record({ t: 'start', pid: process.pid });
+while (!stopping) {
+    for (const seat of seats) {
+        if (!seat.handle?.isOpen()) {
+            seat.handle = null;
+            if (Date.now() < seat.nextTryAt)
+                continue;
+            try {
+                const handle = await router.openTenant(organization, seat.tenant);
+                seat.handle = handle;
+                seat.epoch = handle.epoch;
+                seat.next = 1;
+                seat.lastSnapshotAt = Date.now();
+                record({ t: 'open', tenant: seat.label, epoch: handle.epoch, from: handle.restoredFrom });
+            }
+            catch (error) {
+                seat.nextTryAt = Date.now() + RETRY_AFTER_HELD_MS;
+                if (!(error instanceof LeaseHeldError))
+                    record({ t: 'open-failed', tenant: seat.label, code: error.code ?? String(error) });
+                continue;
+            }
+        }
+        const handle = seat.handle;
+        const n = seat.next;
+        try {
+            let at = 0;
+            handle.run((ledger) => {
+                at = Date.now();
+                ledger.insertEvent({ actor: 'drill', kind: 'tenant-state.drill', payload: { n }, idemKey: `${seat.label}:${seat.epoch}:${n}` });
+            });
+            record({ t: 'ack', tenant: seat.label, epoch: seat.epoch, n, at });
+            seat.next += 1;
+        }
+        catch (error) {
+            record({ t: 'refused', tenant: seat.label, epoch: seat.epoch, n, code: error.code ?? String(error) });
+            await sleep(20);
+            continue;
+        }
+        if (Date.now() - seat.lastSnapshotAt >= SNAPSHOT_EVERY_MS) {
+            // Nothing else writes in this process, so the image holds exactly the writes acknowledged so far.
+            const covered = seat.next - 1;
+            record({ t: 'snap-start', tenant: seat.label, epoch: seat.epoch, covered });
+            try {
+                const manifest = await handle.snapshot();
+                record({ t: 'snap', tenant: seat.label, epoch: seat.epoch, seq: manifest.seq, covered });
+            }
+            catch (error) {
+                record({ t: 'snap-failed', tenant: seat.label, epoch: seat.epoch, code: error.code ?? String(error) });
+            }
+            seat.lastSnapshotAt = Date.now();
+        }
+    }
+    await sleep(4);
+}
+/**
+ * The long-lived consumer. Inside a `runAsync` bracket it writes through the ledger for a while, which the drain has to
+ * wait for and the final image has to hold; once the bracket is over it tries the view again, which has to be refused
+ * as soon as the image is being taken. Every acknowledged write is logged like the loop's own, so the verification
+ * counts them the same way.
+ */
+async function drainingWriter(seat) {
+    const handle = seat.handle;
+    const note = (entry) => record({ ...entry, tenant: seat.label, epoch: seat.epoch, via: 'view' });
+    const until = Date.now() + DRAIN_WRITER_MS;
+    try {
+        await handle.runAsync(async (ledger) => {
+            while (Date.now() < until) {
+                const n = seat.next;
+                ledger.insertEvent({ actor: 'drill', kind: 'tenant-state.drill', payload: { n }, idemKey: `${seat.label}:${seat.epoch}:${n}` });
+                note({ t: 'ack', n, at: Date.now() });
+                seat.next += 1;
+                await sleep(3);
+            }
+        });
+    }
+    catch (error) {
+        note({ t: 'refused', n: seat.next, code: error.code ?? String(error) });
+        return;
+    }
+    for (let attempt = 0; attempt < 200; attempt += 1) {
+        const n = seat.next;
+        try {
+            handle.ledger.insertEvent({ actor: 'drill', kind: 'tenant-state.drill', payload: { n }, idemKey: `${seat.label}:${seat.epoch}:${n}` });
+            note({ t: 'ack', n, at: Date.now() });
+            seat.next += 1;
+        }
+        catch (error) {
+            note({ t: 'refused', n, code: error.code ?? String(error) });
+            return;
+        }
+        await sleep(2);
+    }
+}
+record({ t: 'draining' });
+const writers = seats.filter(seat => seat.handle?.isOpen()).map(seat => drainingWriter(seat));
+const failed = await router.close();
+await Promise.all(writers);
+record({ t: 'closed', failed });
+closeSync(log);
+process.exit(0);

package/dist/scripts/tenant-state-durable-conformance.js

@@ -0,0 +1,101 @@
+/**
+ * Conformance checks for any DurableStore binding (AaaS A1, docs/rfc/aaas-rfc.md §4.3).
+ *
+ * The reference directory store runs them offline. The S3-compatible binding of decision Q4 must pass
+ * the same checks before it holds a real copy. A failed check is returned, not thrown, so a suite can
+ * also prove the checks themselves can fail against deliberately broken stores.
+ */
+class Violation extends Error {
+}
+function expect(condition, detail) {
+    if (!condition)
+        throw new Violation(detail);
+}
+function text(bytes) {
+    return bytes === null ? null : Buffer.from(bytes).toString('utf8');
+}
+async function expectRangeError(run, detail) {
+    try {
+        await run();
+    }
+    catch (error) {
+        if (error instanceof RangeError)
+            return;
+        throw new Violation(`${detail}: threw ${error instanceof Error ? `${error.name}: ${error.message}` : String(error)}`);
+    }
+    throw new Violation(`${detail}: did not throw`);
+}
+export async function runDurableStoreConformance(store, namespace) {
+    const failures = [];
+    let sequence = 0;
+    async function check(name, body) {
+        sequence += 1;
+        try {
+            await body(`${namespace}/check-${sequence}`);
+        }
+        catch (error) {
+            failures.push({ check: name, detail: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    await check('put creates an absent key and never overwrites an existing one', async (prefix) => {
+        expect(await store.putIfAbsent(`${prefix}/a`, Buffer.from('first')) === true, 'the first put creates');
+        expect(await store.putIfAbsent(`${prefix}/a`, Buffer.from('second')) === false, 'the second put reports the key exists');
+        expect(text(await store.get(`${prefix}/a`)) === 'first', 'the first bytes survive');
+    });
+    await check('an unknown key reads as null', async (prefix) => {
+        expect(await store.get(`${prefix}/missing`) === null, 'get of an unknown key');
+    });
+    await check('bytes round-trip exactly, including a large object', async (prefix) => {
+        const payload = Buffer.alloc(1_048_576 + 3);
+        for (let index = 0; index < payload.length; index += 1)
+            payload[index] = (index * 31 + 7) & 0xff;
+        expect(await store.putIfAbsent(`${prefix}/big.bin`, payload) === true, 'put');
+        const back = await store.get(`${prefix}/big.bin`);
+        expect(back !== null && Buffer.compare(Buffer.from(back), payload) === 0, 'the same 1 MiB + 3 bytes come back');
+    });
+    await check('list returns sorted keys under a prefix, including a partial last segment', async (prefix) => {
+        for (const name of ['snap-2.db', 'snap-1.db', 'snap-1.json', 'base.json'])
+            await store.putIfAbsent(`${prefix}/e1/${name}`, Buffer.from(name));
+        await store.putIfAbsent(`${prefix}/e2/snap-1.db`, Buffer.from('x'));
+        const all = await store.list(`${prefix}/`);
+        expect(all.length === 5, `all five under the prefix, got ${all.length}`);
+        expect(JSON.stringify(all) === JSON.stringify([...all].sort()), 'sorted');
+        const partial = await store.list(`${prefix}/e1/snap-`);
+        expect(JSON.stringify(partial) === JSON.stringify([`${prefix}/e1/snap-1.db`, `${prefix}/e1/snap-1.json`, `${prefix}/e1/snap-2.db`]), `partial prefix: ${JSON.stringify(partial)}`);
+        expect((await store.list(`${prefix}/nothing/`)).length === 0, 'an empty prefix lists nothing');
+    });
+    await check('delete removes exactly the keys under the prefix and counts them', async (prefix) => {
+        await store.putIfAbsent(`${prefix}/keep/a`, Buffer.from('a'));
+        for (const name of ['x', 'y', 'z'])
+            await store.putIfAbsent(`${prefix}/gone/${name}`, Buffer.from(name));
+        expect(await store.deletePrefix(`${prefix}/gone/`) === 3, 'three deleted');
+        expect((await store.list(`${prefix}/gone/`)).length === 0, 'nothing left under the prefix');
+        expect(text(await store.get(`${prefix}/keep/a`)) === 'a', 'a sibling is untouched');
+        expect(await store.deletePrefix(`${prefix}/gone/`) === 0, 'deleting again deletes nothing');
+        expect(await store.putIfAbsent(`${prefix}/gone/x`, Buffer.from('again')) === true, 'a deleted key can be created again');
+    });
+    await check('sixteen concurrent puts of one key create it exactly once with intact bytes', async (prefix) => {
+        const contenders = Array.from({ length: 16 }, (_, index) => Buffer.from(`contender-${index}-${'x'.repeat(4096)}`));
+        const results = await Promise.all(contenders.map(bytes => store.putIfAbsent(`${prefix}/race`, bytes)));
+        expect(results.filter(Boolean).length === 1, `${results.filter(Boolean).length} puts reported creating the key`);
+        const stored = await store.get(`${prefix}/race`);
+        expect(stored !== null && contenders.some(bytes => Buffer.compare(bytes, Buffer.from(stored)) === 0), 'the stored bytes are exactly one contender');
+    });
+    await check('keys that could leave the store or touch its scratch area are rejected', async (prefix) => {
+        for (const bad of ['', '/', 'a//b', '../x', `${prefix}/../x`, '.tmp/x', `${prefix}/.hidden`, 'a b', 'a/\u0000b']) {
+            await expectRangeError(() => store.putIfAbsent(bad, Buffer.from('x')), `put ${JSON.stringify(bad)}`);
+            await expectRangeError(() => store.get(bad), `get ${JSON.stringify(bad)}`);
+        }
+        await expectRangeError(() => store.list('../'), 'list ../');
+        await expectRangeError(() => store.deletePrefix(''), 'delete of the empty prefix');
+    });
+    await check('delete takes whole segments only: a prefix that stops inside a name would also take its neighbours', async (prefix) => {
+        await store.putIfAbsent(`${prefix}/org/t1/base.json`, Buffer.from('1'));
+        await store.putIfAbsent(`${prefix}/org2/t9/base.json`, Buffer.from('2'));
+        await expectRangeError(() => store.deletePrefix(`${prefix}/org`), 'delete of a prefix that ends inside a segment');
+        expect((await store.list(`${prefix}/org2/`)).length === 1, 'the neighbour that shares the spelling is untouched');
+        expect(await store.deletePrefix(`${prefix}/org/`) === 1, 'the whole segment is deleted');
+        expect((await store.list(`${prefix}/org2/`)).length === 1, 'and the neighbour is still there');
+    });
+    return failures;
+}