@danceiny/gotry 0.2.0-rc.32 → 0.2.0-rc.33

package/dist/src/tenant-seam/seam.js

@@ -0,0 +1,92 @@
+/**
+ * The trusted seam of the AaaS line (docs/rfc/aaas-rfc.md §4.4, threats S-1, S-3, S-13).
+ *
+ * Exactly one place turns "who is calling" into "whose ledger": `deriveSeamTenant`. It takes the caller an
+ * authenticator vouched for, and a per-organization key, and returns the ledger tenant as an HMAC digest
+ * (`deriveLedgerTenantId`). Nothing a request body, a header, a tool argument or a model output says is ever an
+ * input to it, and every API that needs a tenant takes the `SeamTenant` it returns, never a string.
+ *
+ * What the authenticator is, on the wire, is not decided here: the A3 boundary ADR chooses per-organization
+ * credentials or signed assertions. The authenticator is an in-process interface so that decision can land
+ * without touching the seam.
+ */
+import { TenantStateError } from '../tenant-state/errors.js';
+import { assertOrganizationId, deriveLedgerTenantId } from '../tenant-state/ids.js';
+const MAX_PRINCIPAL_FIELD_CHARS = 256;
+/** A keyring over a fixed set of keys. Key storage and rotation are deployment concerns and are not designed here. */
+export class StaticOrganizationKeyring {
+    keys = new Map();
+    constructor(keys) {
+        for (const [organization, key] of Object.entries(keys)) {
+            assertOrganizationId(organization);
+            if (!(key instanceof Uint8Array) || key.byteLength < 32)
+                throw new RangeError(`the key of organization ${organization} must be at least 32 bytes`);
+            this.keys.set(organization, Uint8Array.from(key));
+        }
+    }
+    keyFor(organization) {
+        return this.keys.get(organization) ?? null;
+    }
+}
+export class SeamError extends TenantStateError {
+}
+/** The authenticated principal cannot be turned into a ledger tenant: an empty, oversized or malformed claim. */
+export class InvalidPrincipalError extends SeamError {
+    constructor(message) {
+        super('GOTRY_SEAM_PRINCIPAL_INVALID', message);
+    }
+}
+/** The caller belongs to an organization this service holds no key for. */
+export class UnknownOrganizationError extends SeamError {
+    constructor() {
+        super('GOTRY_SEAM_ORGANIZATION_UNKNOWN', 'the organization is not served here');
+    }
+}
+/** Something other than a `SeamTenant` reached an API that needs one. */
+export class SeamTenantRequiredError extends SeamError {
+    constructor() {
+        super('GOTRY_SEAM_TENANT_REQUIRED', 'a tenant derived at the seam is required');
+    }
+}
+const issued = new WeakSet();
+export function isSeamTenant(value) {
+    return typeof value === 'object' && value !== null && issued.has(value);
+}
+export function assertSeamTenant(value) {
+    if (!isSeamTenant(value))
+        throw new SeamTenantRequiredError();
+    return value;
+}
+/** `<organization>/<ledger tenant>`: the key for every in-process map and every metric. It contains no personal data. */
+export function seamTenantKey(tenant) {
+    return `${assertSeamTenant(tenant).organization}/${tenant.ledgerTenant}`;
+}
+function principalField(name, value) {
+    if (typeof value !== 'string' || value.length === 0)
+        throw new InvalidPrincipalError(`${name} must be a non-empty string`);
+    if (value.length > MAX_PRINCIPAL_FIELD_CHARS)
+        throw new InvalidPrincipalError(`${name} must be at most ${MAX_PRINCIPAL_FIELD_CHARS} characters`);
+    return value;
+}
+/**
+ * The only derivation of a ledger tenant from a caller in the service.
+ *
+ * @throws InvalidTenantIdError for an organization id that is not path-safe, or an issuer or subject that is not well-formed
+ * @throws InvalidPrincipalError for a missing or oversized subject, issuer or scope
+ * @throws UnknownOrganizationError when the keyring holds no key for the organization
+ */
+export function deriveSeamTenant(caller, keyring) {
+    if (typeof caller !== 'object' || caller === null)
+        throw new InvalidPrincipalError('the caller must be an object');
+    const organization = assertOrganizationId(caller.organization);
+    const issuer = principalField('issuer', caller.issuer);
+    const subject = principalField('subject', caller.subject);
+    principalField('scope', caller.scope);
+    const key = keyring.keyFor(organization);
+    if (key === null)
+        throw new UnknownOrganizationError();
+    const ledgerTenant = deriveLedgerTenantId(key, issuer, subject);
+    const tenant = Object.freeze({ organization, ledgerTenant });
+    issued.add(tenant);
+    return tenant;
+}

package/dist/src/tenant-state/durable-store.js

@@ -0,0 +1,158 @@
+/**
+ * Durable store of the AaaS state layer: where off-node copies of a tenant ledger live
+ * (docs/rfc/aaas-rfc.md §4.3, decision Q4).
+ *
+ * The contract is the smallest an object store gives: create-if-absent, read, list by prefix
+ * and delete by prefix. Create-if-absent is the only write, so an object is immutable once
+ * it exists and a stale writer can never replace what a newer owner relies on. Epoch-conditional
+ * appends (durability mode D-b) are a later refinement of the same primitive.
+ *
+ * Every call must settle. A binding gives each request a deadline and fails with `DurableStoreUnavailableError` when it
+ * passes: the router renews a tenant's lease while a flush or a forget waits on the store, so a call that never returns
+ * holds the tenant on its replica until the process dies.
+ *
+ * `DirDurableStore` is the reference implementation on a local directory. It stands in for the
+ * S3-compatible store of the real deployment, whose binding must pass the same conformance
+ * checks before it holds a real copy.
+ */
+import { randomBytes } from 'node:crypto';
+import { link, mkdir, open, readFile, readdir, rm, rmdir } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { DurableStoreUnavailableError } from './errors.js';
+const MAX_KEY_LENGTH = 512;
+const SEGMENT = /^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$/;
+const PARTIAL_SEGMENT = /^(?:[A-Za-z0-9_][A-Za-z0-9._-]{0,127})?$/;
+/** Keys are `/`-separated segments of `[A-Za-z0-9._-]` that do not start with a dot, so no key can leave the store or touch its scratch area. */
+export function assertDurableKey(key) {
+    if (typeof key !== 'string' || key.length === 0 || key.length > MAX_KEY_LENGTH || !key.split('/').every(segment => SEGMENT.test(segment))) {
+        throw new RangeError(`invalid durable key ${JSON.stringify(typeof key === 'string' ? key.slice(0, 80) : key)}`);
+    }
+    return key;
+}
+/** A prefix is complete segments followed by an optional partial one: `a/b/`, `a/b/snap-`, `a/b`. */
+export function assertDurablePrefix(prefix) {
+    const segments = typeof prefix === 'string' ? prefix.split('/') : [];
+    const last = segments.pop();
+    if (typeof prefix !== 'string' || prefix.length === 0 || prefix.length > MAX_KEY_LENGTH ||
+        last === undefined || !PARTIAL_SEGMENT.test(last) || !segments.every(segment => SEGMENT.test(segment))) {
+        throw new RangeError(`invalid durable prefix ${JSON.stringify(typeof prefix === 'string' ? prefix.slice(0, 80) : prefix)}`);
+    }
+    return prefix;
+}
+/** A prefix to delete by: complete segments, so `acme/` can never take `acme2/`. */
+export function assertDurableDeletePrefix(prefix) {
+    assertDurablePrefix(prefix);
+    if (!prefix.endsWith('/'))
+        throw new RangeError(`a durable delete takes whole segments and ends with "/", got ${JSON.stringify(prefix.slice(0, 80))}`);
+    return prefix;
+}
+function code(error) {
+    const value = error?.code;
+    return typeof value === 'string' ? value : undefined;
+}
+export class DirDurableStore {
+    root;
+    constructor(root) {
+        this.root = resolve(root);
+    }
+    async putIfAbsent(key, bytes) {
+        assertDurableKey(key);
+        const destination = this.pathOf(key);
+        const scratch = join(this.root, '.tmp');
+        const staged = join(scratch, randomBytes(8).toString('hex'));
+        try {
+            await mkdir(scratch, { recursive: true });
+            await mkdir(dirname(destination), { recursive: true });
+            const handle = await open(staged, 'wx');
+            try {
+                await handle.writeFile(bytes);
+                await handle.sync();
+            }
+            finally {
+                await handle.close();
+            }
+            // A hard link either creates the key with all its bytes or fails: a reader never sees a half-written object.
+            await link(staged, destination);
+            return true;
+        }
+        catch (error) {
+            if (code(error) === 'EEXIST')
+                return false;
+            throw new DurableStoreUnavailableError(`durable put failed for ${key}`, { cause: error });
+        }
+        finally {
+            await rm(staged, { force: true });
+        }
+    }
+    async get(key) {
+        assertDurableKey(key);
+        try {
+            return await readFile(this.pathOf(key));
+        }
+        catch (error) {
+            if (code(error) === 'ENOENT' || code(error) === 'EISDIR' || code(error) === 'ENOTDIR')
+                return null;
+            throw new DurableStoreUnavailableError(`durable get failed for ${key}`, { cause: error });
+        }
+    }
+    async list(prefix) {
+        assertDurablePrefix(prefix);
+        const slash = prefix.lastIndexOf('/');
+        const directory = slash === -1 ? '' : prefix.slice(0, slash);
+        const keys = [];
+        try {
+            await this.walk(directory, keys);
+        }
+        catch (error) {
+            throw new DurableStoreUnavailableError(`durable list failed for ${prefix}`, { cause: error });
+        }
+        return keys.filter(key => key.startsWith(prefix)).sort();
+    }
+    async deletePrefix(prefix) {
+        assertDurableDeletePrefix(prefix);
+        const keys = await this.list(prefix);
+        try {
+            for (const key of keys)
+                await rm(this.pathOf(key), { force: true });
+            for (const directory of new Set(keys.map(key => dirname(key))))
+                await this.pruneEmptyDirectories(directory);
+        }
+        catch (error) {
+            throw new DurableStoreUnavailableError(`durable delete failed for ${prefix}`, { cause: error });
+        }
+        return keys.length;
+    }
+    pathOf(key) {
+        return join(this.root, ...key.split('/'));
+    }
+    async walk(directory, into) {
+        let entries;
+        try {
+            entries = await readdir(directory === '' ? this.root : this.pathOf(directory), { withFileTypes: true });
+        }
+        catch (error) {
+            if (code(error) === 'ENOENT' || code(error) === 'ENOTDIR')
+                return;
+            throw error;
+        }
+        for (const entry of entries) {
+            if (directory === '' && entry.name === '.tmp')
+                continue;
+            const child = directory === '' ? entry.name : `${directory}/${entry.name}`;
+            if (entry.isDirectory())
+                await this.walk(child, into);
+            else
+                into.push(child);
+        }
+    }
+    async pruneEmptyDirectories(directory) {
+        for (let current = directory; current !== '.' && current !== ''; current = dirname(current)) {
+            try {
+                await rmdir(this.pathOf(current));
+            }
+            catch {
+                return;
+            }
+        }
+    }
+}

package/dist/src/tenant-state/epoch-copies.js

@@ -0,0 +1,422 @@
+/**
+ * Epoch-scoped durable copies of a tenant ledger (docs/rfc/aaas-rfc.md §4.2 item 3 and §4.3, mode D-a).
+ *
+ * Every owner copies only into the prefix of its own epoch:
+ *
+ *     <org>/<tenant>/e<epoch>/base.json         where this epoch's ledger started: one pinned snapshot, or empty
+ *     <org>/<tenant>/e<epoch>/snap-<n>.db       an immutable, verified image of the ledger
+ *     <org>/<tenant>/e<epoch>/snap-<n>.json     its manifest; written last, so a manifest means a complete image
+ *
+ * A new owner never trusts what it finds beside its own epoch. It restores the newest verified snapshot
+ * of the newest lineage, then pins that exact snapshot in its own `base.json` before it serves a write.
+ * Anything a superseded owner uploads later goes into a dead prefix. A newer epoch's base never points at
+ * it, so it can never become authoritative. Integrity therefore rests on the lease epoch and on these
+ * rules, not on clocks. What can be lost under D-a is the tail of acknowledged writes after the last
+ * snapshot, which is the declared RPO.
+ *
+ * Only the ledger file is copied. Files beside it (planner working files, tickets, artifacts) are not
+ * replicated yet (RFC gap 6, fact F-2).
+ */
+import Database from 'better-sqlite3';
+import { createHash } from 'node:crypto';
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, isAbsolute, join, resolve } from 'node:path';
+import { ledgerDbPath, ledgerExists, openDb } from '../state-ledger.js';
+import { CopyConflictError, RestoreVerificationError } from './errors.js';
+import { leaseKeyFor, tenantRootFor } from './ids.js';
+const MAX_SNAPSHOT_BYTES = 512 * 1024 * 1024;
+const HEX64 = /^[0-9a-f]{64}$/;
+const TENANT_TABLES = ['events', 'projection_docs', 'projection_items', 'workflow_runs', 'workflow_steps', 'pending_writes'];
+// ---- layout ------------------------------------------------------------------
+export function epochPrefixOf(leaseKey, epoch) {
+    return `${leaseKey}/e${String(epoch).padStart(8, '0')}/`;
+}
+function snapshotKeyOf(leaseKey, epoch, seq, extension) {
+    return `${epochPrefixOf(leaseKey, epoch)}snap-${String(seq).padStart(6, '0')}.${extension}`;
+}
+function baseKeyOf(leaseKey, epoch) {
+    return `${epochPrefixOf(leaseKey, epoch)}base.json`;
+}
+function epochOfKey(objectKey) {
+    const match = /\/e(\d{8,})\/[^/]+$/.exec(objectKey);
+    return match ? Number(match[1]) : null;
+}
+function snapshotSeqsOf(keys, extension) {
+    const pattern = new RegExp(`/snap-(\\d{6})\\.${extension}$`);
+    const seqs = new Set();
+    for (const key of keys) {
+        const match = pattern.exec(key);
+        if (match)
+            seqs.add(Number(match[1]));
+    }
+    return [...seqs].sort((a, b) => b - a);
+}
+// ---- manifests -----------------------------------------------------------------
+function isCount(value) {
+    return typeof value === 'number' && Number.isInteger(value) && value >= 0;
+}
+function parseJson(bytes) {
+    try {
+        const value = JSON.parse(Buffer.from(bytes).toString('utf8'));
+        return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : null;
+    }
+    catch {
+        return null;
+    }
+}
+function parseManifest(bytes, tenant, epoch, seq) {
+    const value = parseJson(bytes);
+    if (!value || value.v !== 1 || value.tenant !== tenant || value.epoch !== epoch || value.seq !== seq)
+        return null;
+    if (typeof value.sha256 !== 'string' || !HEX64.test(value.sha256))
+        return null;
+    if (![value.bytes, value.events, value.maxEventSeq, value.createdAtMs].every(isCount))
+        return null;
+    return value;
+}
+function parseBase(bytes, epoch) {
+    const value = parseJson(bytes);
+    if (!value || value.v !== 1 || value.epoch !== epoch)
+        return null;
+    if (value.from === null)
+        return { v: 1, epoch, from: null };
+    const from = value.from;
+    if (!from || typeof from !== 'object' || !isCount(from.epoch) || !isCount(from.seq) || from.epoch < 1 || from.epoch >= epoch || from.seq < 1)
+        return null;
+    if (typeof from.sha256 !== 'string' || !HEX64.test(from.sha256))
+        return null;
+    return { v: 1, epoch, from: { epoch: from.epoch, seq: from.seq, sha256: from.sha256 } };
+}
+function sha256Of(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
+}
+function refLabel(epoch, seq) {
+    return `e${epoch}/snap-${seq}`;
+}
+// ---- verification --------------------------------------------------------------
+/**
+ * A ledger image is trusted only if SQLite finds it intact, it holds rows of this tenant and no
+ * other, and replaying its events reproduces its projections (the drill D properties). The checks
+ * run on a scratch copy, so the image itself is never opened for writing.
+ */
+export function verifyLedgerImage(imagePath, ledgerTenant) {
+    const scratch = mkdtempSync(join(tmpdir(), 'gotry-verify-'));
+    try {
+        mkdirSync(dirname(ledgerDbPath(scratch)), { recursive: true });
+        copyFileSync(imagePath, ledgerDbPath(scratch));
+        const facts = inspectImage(ledgerDbPath(scratch), ledgerTenant);
+        assertFoldReproducesProjections(scratch, ledgerTenant);
+        return facts;
+    }
+    finally {
+        rmSync(scratch, { recursive: true, force: true });
+    }
+}
+function inspectImage(copyPath, ledgerTenant) {
+    const db = new Database(copyPath, { fileMustExist: true });
+    try {
+        const integrity = db.pragma('integrity_check');
+        if (integrity.length !== 1 || integrity[0].integrity_check !== 'ok')
+            throw new RestoreVerificationError('ledger image failed integrity_check');
+        for (const table of TENANT_TABLES) {
+            const foreign = db.prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE tenant_id <> ?`).get(ledgerTenant);
+            if (foreign.n !== 0)
+                throw new RestoreVerificationError(`ledger image holds rows of another tenant in ${table}`);
+        }
+        const row = db.prepare('SELECT COUNT(*) AS n, COALESCE(MAX(seq), 0) AS m FROM events').get();
+        return { events: row.n, maxEventSeq: row.m };
+    }
+    catch (error) {
+        if (error instanceof RestoreVerificationError)
+            throw error;
+        throw new RestoreVerificationError(`ledger image is not a readable ledger: ${error.message}`);
+    }
+    finally {
+        db.close();
+    }
+}
+function assertFoldReproducesProjections(scratchRoot, ledgerTenant) {
+    let ledger;
+    try {
+        ledger = openDb(scratchRoot, ledgerTenant);
+    }
+    catch (error) {
+        throw new RestoreVerificationError(`ledger image could not be opened as a ledger: ${error.message}`);
+    }
+    try {
+        const before = JSON.stringify({ wishes: ledger.readWishPool(), motivation: ledger.readMotivation() });
+        ledger.rebuildProjections();
+        const after = JSON.stringify({ wishes: ledger.readWishPool(), motivation: ledger.readMotivation() });
+        if (before !== after)
+            throw new RestoreVerificationError('ledger image: replaying its events does not reproduce its projections');
+    }
+    finally {
+        ledger.close();
+    }
+}
+function verifyCandidate(bytes, manifest, ledgerTenant) {
+    if (bytes.byteLength !== manifest.bytes)
+        throw new RestoreVerificationError('size differs from its manifest');
+    if (sha256Of(bytes) !== manifest.sha256)
+        throw new RestoreVerificationError('checksum differs from its manifest');
+    const scratch = mkdtempSync(join(tmpdir(), 'gotry-candidate-'));
+    try {
+        const imagePath = join(scratch, 'image.db');
+        writeFileSync(imagePath, bytes);
+        const facts = verifyLedgerImage(imagePath, ledgerTenant);
+        if (facts.events !== manifest.events || facts.maxEventSeq !== manifest.maxEventSeq) {
+            throw new RestoreVerificationError('event count or last sequence differs from its manifest');
+        }
+    }
+    finally {
+        rmSync(scratch, { recursive: true, force: true });
+    }
+}
+// ---- publishing ----------------------------------------------------------------
+/** The lease key of the context and the tenant root that was checked: every file operation below uses that root, not the string the caller passed. */
+function checkedContext(context) {
+    const key = leaseKeyFor(context.organization, context.ledgerTenant);
+    if (context.lease.key !== key)
+        throw new RangeError(`the lease ${context.lease.key} does not belong to tenant ${key}`);
+    return { key, root: ownTenantRoot(context) };
+}
+/**
+ * `forgetTenant` deletes the tenant root recursively, and the root comes from the caller. It has to be the directory
+ * `tenantRootFor` lays out for exactly this tenant, `<state root>/orgs/<organization>/tenants/<tenant>`, so a mistaken or
+ * hostile context can name no other directory. The resolved path is returned: the string that was checked is the one that
+ * gets deleted, so a `..` behind a symlink cannot point the deletion somewhere else.
+ */
+function ownTenantRoot(context) {
+    const expectedTail = tenantRootFor('/', context.organization, context.ledgerTenant);
+    const actual = resolve(context.tenantRoot);
+    if (!isAbsolute(context.tenantRoot) || !actual.endsWith(expectedTail)) {
+        throw new RangeError(`the tenant root ${JSON.stringify(context.tenantRoot.slice(0, 200))} is not the root of tenant ${leaseKeyFor(context.organization, context.ledgerTenant)}`);
+    }
+    return actual;
+}
+/**
+ * Takes a consistent image of the live ledger, verifies it, and publishes it into this holder's
+ * epoch prefix. The lease is checked before the work and again right before the upload.
+ */
+export async function publishSnapshot(context, options = {}) {
+    const { key, root } = checkedContext(context);
+    const { durable, lease, ledgerTenant } = context;
+    lease.assertHeld();
+    const scratch = mkdtempSync(join(tmpdir(), 'gotry-snapshot-'));
+    try {
+        const imagePath = join(scratch, 'image.db');
+        const source = new Database(ledgerDbPath(root), { fileMustExist: true });
+        try {
+            source.pragma('busy_timeout = 8000');
+            await source.backup(imagePath);
+        }
+        finally {
+            source.close();
+        }
+        // The image must not depend on a -wal or -shm file that would not travel with it.
+        const image = new Database(imagePath);
+        try {
+            image.pragma('journal_mode = DELETE');
+        }
+        finally {
+            image.close();
+        }
+        const facts = verifyLedgerImage(imagePath, ledgerTenant);
+        const bytes = readFileSync(imagePath);
+        if (bytes.byteLength > MAX_SNAPSHOT_BYTES)
+            throw new RangeError(`ledger image of ${bytes.byteLength} bytes exceeds the ${MAX_SNAPSHOT_BYTES} byte limit`);
+        const sha256 = sha256Of(bytes);
+        lease.assertHeld();
+        const existing = await durable.list(`${epochPrefixOf(key, lease.epoch)}snap-`);
+        const seq = (snapshotSeqsOf(existing, 'db')[0] ?? 0) + 1;
+        const manifest = {
+            v: 1,
+            tenant: ledgerTenant,
+            epoch: lease.epoch,
+            seq,
+            sha256,
+            bytes: bytes.byteLength,
+            events: facts.events,
+            maxEventSeq: facts.maxEventSeq,
+            createdAtMs: (options.nowMs ?? Date.now)(),
+        };
+        if (!await durable.putIfAbsent(snapshotKeyOf(key, lease.epoch, seq, 'db'), bytes)) {
+            throw new CopyConflictError(`${refLabel(lease.epoch, seq)} already exists in the prefix of epoch ${lease.epoch}`);
+        }
+        if (!await durable.putIfAbsent(snapshotKeyOf(key, lease.epoch, seq, 'json'), Buffer.from(JSON.stringify(manifest)))) {
+            throw new CopyConflictError(`the manifest of ${refLabel(lease.epoch, seq)} already exists`);
+        }
+        // The upload took time. An image that landed after the lease lapsed may sit above the base a successor already
+        // pinned, and is then never read: reporting it as published would tell a caller its writes are safe when they are not.
+        lease.assertHeld();
+        return manifest;
+    }
+    finally {
+        rmSync(scratch, { recursive: true, force: true });
+    }
+}
+/** Pins where this epoch's ledger started. Written once, before the owner serves a write or publishes a snapshot. */
+async function writeBase(context, key, from) {
+    context.lease.assertHeld();
+    const record = { v: 1, epoch: context.lease.epoch, from };
+    if (!await context.durable.putIfAbsent(baseKeyOf(key, context.lease.epoch), Buffer.from(JSON.stringify(record)))) {
+        throw new CopyConflictError(`the base of epoch ${context.lease.epoch} already exists`);
+    }
+}
+/**
+ * The newest verified snapshot of the newest lineage below `ownEpoch` and above the forgotten floor.
+ *
+ * The newest lineage is the highest epoch that left anything behind. Inside an epoch the newest snapshot
+ * that verifies wins; with none, the walk follows that epoch's `base.json` pin to the exact snapshot (and
+ * older ones of that epoch) it started from. A snapshot a superseded owner uploaded after a newer epoch
+ * pinned its base is above the pin and is never read.
+ *
+ * @throws RestoreVerificationError when copies exist but none of them verifies
+ */
+export async function resolveRestorePoint(durable, leaseKey, ledgerTenant, ownEpoch, forgottenAtEpoch, verify) {
+    const keysByEpoch = new Map();
+    for (const objectKey of await durable.list(`${leaseKey}/e`)) {
+        const epoch = epochOfKey(objectKey);
+        if (epoch === null || epoch <= forgottenAtEpoch || epoch >= ownEpoch)
+            continue;
+        keysByEpoch.set(epoch, [...keysByEpoch.get(epoch) ?? [], objectKey]);
+    }
+    const epochsWithEvidence = [...keysByEpoch.keys()].sort((a, b) => b - a);
+    const skipped = [];
+    const visited = new Set();
+    let corrupt = false;
+    let epoch = epochsWithEvidence[0] ?? null;
+    let ceiling = Number.POSITIVE_INFINITY;
+    while (epoch !== null && !visited.has(epoch)) {
+        visited.add(epoch);
+        const keys = keysByEpoch.get(epoch) ?? await durable.list(epochPrefixOf(leaseKey, epoch));
+        for (const seq of snapshotSeqsOf(keys, 'json').filter(candidate => candidate <= ceiling)) {
+            const label = refLabel(epoch, seq);
+            const manifestBytes = await durable.get(snapshotKeyOf(leaseKey, epoch, seq, 'json'));
+            const manifest = manifestBytes ? parseManifest(manifestBytes, ledgerTenant, epoch, seq) : null;
+            if (!manifest) {
+                skipped.push(`${label}: manifest is missing or malformed`);
+                corrupt = true;
+                continue;
+            }
+            const bytes = await durable.get(snapshotKeyOf(leaseKey, epoch, seq, 'db'));
+            if (!bytes) {
+                skipped.push(`${label}: image is missing`);
+                corrupt = true;
+                continue;
+            }
+            try {
+                verify(bytes, manifest);
+                return { point: { manifest, bytes }, skipped };
+            }
+            catch (error) {
+                if (!(error instanceof RestoreVerificationError))
+                    throw error;
+                skipped.push(`${label}: ${error.message}`);
+                corrupt = true;
+            }
+        }
+        const baseBytes = keys.includes(baseKeyOf(leaseKey, epoch)) ? await durable.get(baseKeyOf(leaseKey, epoch)) : null;
+        const base = baseBytes ? parseBase(baseBytes, epoch) : null;
+        if (base?.from === null)
+            break;
+        if (base?.from) {
+            if (base.from.epoch <= forgottenAtEpoch)
+                break;
+            epoch = base.from.epoch;
+            ceiling = base.from.seq;
+            continue;
+        }
+        skipped.push(`e${epoch}: base is missing or malformed`);
+        epoch = epochsWithEvidence.find(candidate => candidate < epoch) ?? null;
+        ceiling = Number.POSITIVE_INFINITY;
+    }
+    if (corrupt)
+        throw new RestoreVerificationError('durable copies exist but none of them verifies', skipped);
+    return { point: null, skipped };
+}
+function evictCachedLedger(tenantRoot, ledgerTenant) {
+    if (!ledgerExists(tenantRoot))
+        return;
+    try {
+        openDb(tenantRoot, ledgerTenant).close();
+    }
+    catch {
+        // A stale or damaged local file is about to be replaced; failing to open it changes nothing.
+    }
+}
+/**
+ * Makes the local ledger of a freshly acquired epoch equal to the newest verified durable copy, pins that
+ * copy in this epoch's `base.json`, and opens the ledger. Local files from an earlier tenure are always
+ * discarded: only the durable copy is authoritative.
+ */
+export async function restoreTenantLedger(context) {
+    const { key, root: tenantRoot } = checkedContext(context);
+    const { durable, lease, ledgerTenant } = context;
+    lease.assertHeld();
+    const resolution = await resolveRestorePoint(durable, key, ledgerTenant, lease.epoch, lease.forgottenAtEpoch, (bytes, manifest) => verifyCandidate(bytes, manifest, ledgerTenant));
+    lease.assertHeld();
+    evictCachedLedger(tenantRoot, ledgerTenant);
+    const dbPath = ledgerDbPath(tenantRoot);
+    mkdirSync(dirname(dbPath), { recursive: true });
+    for (const suffix of ['', '-wal', '-shm', '.restoring'])
+        rmSync(`${dbPath}${suffix}`, { force: true });
+    const point = resolution.point;
+    if (point) {
+        writeFileSync(`${dbPath}.restoring`, point.bytes);
+        renameSync(`${dbPath}.restoring`, dbPath);
+    }
+    const restoredFrom = point ? { epoch: point.manifest.epoch, seq: point.manifest.seq, sha256: point.manifest.sha256 } : null;
+    await writeBase(context, key, restoredFrom);
+    return {
+        ledger: openDb(tenantRoot, ledgerTenant),
+        restoredFrom,
+        skipped: resolution.skipped,
+        facts: point ? { events: point.manifest.events, maxEventSeq: point.manifest.maxEventSeq } : null,
+    };
+}
+// ---- forgetting --------------------------------------------------------------------
+/**
+ * Removes the local ledger files (the database and its -wal and -shm) of a tenure that flushed everything. Other files
+ * of the tenant's directory, such as a planner's working files, are not the ledger's and stay.
+ */
+export function discardLocalLedger(context) {
+    const file = ledgerDbPath(checkedContext(context).root);
+    for (const path of [file, `${file}-wal`, `${file}-shm`])
+        rmSync(path, { force: true });
+}
+/**
+ * Forget means delete (red line 6): the floor goes into the lease record first, so no later restore can
+ * read an older copy even if a superseded owner uploads one afterwards; then every durable copy and the
+ * local files go; then the lease is handed back. The caller closes the tenant's ledger first.
+ */
+export async function forgetTenant(context) {
+    const { key, root } = checkedContext(context);
+    context.lease.assertHeld();
+    const record = await context.lease.markForgotten();
+    // Only the epochs at or below the floor that was just raised are the forgotten tenure's. An owner that was paused between
+    // raising the floor and deleting wakes up after a successor took over: what the successor wrote is above the floor, and
+    // it is not this call's to delete.
+    const deletedObjects = await sweepMaskedCopies(context.durable, key, record.forgottenAtEpoch);
+    rmSync(root, { recursive: true, force: true });
+    await context.lease.release();
+    return { deletedObjects, floor: record.forgottenAtEpoch };
+}
+/**
+ * Deletes copies at or below the forgotten floor that a superseded owner uploaded after the forget. Every open of a
+ * tenant that was ever forgotten runs it; for one that is never opened again an operator runs it, once the previous
+ * owner's lease has certainly run out.
+ */
+export async function sweepMaskedCopies(durable, leaseKey, forgottenAtEpoch) {
+    let deleted = 0;
+    const dead = new Set();
+    for (const objectKey of await durable.list(`${leaseKey}/e`)) {
+        const epoch = epochOfKey(objectKey);
+        if (epoch !== null && epoch <= forgottenAtEpoch)
+            dead.add(epoch);
+    }
+    for (const epoch of dead)
+        deleted += await durable.deletePrefix(epochPrefixOf(leaseKey, epoch));
+    return deleted;
+}