@danceiny/gotry 0.2.0-rc.32 → 0.2.0-rc.33

package/dist/src/tenant-state/tenant-router.js

@@ -0,0 +1,650 @@
+/**
+ * The tenant router of the AaaS state layer (docs/rfc/aaas-rfc.md §4.2 items 5 and 6, §4.4).
+ *
+ * One replica asks the router for a tenant by `(organization, ledger tenant)`. The router acquires the
+ * lease, restores the ledger from the newest verified durable copy, and returns a handle. The handle is
+ * the only way to the tenant's ledger, and it is tenant-bound: no method takes a tenant id.
+ *
+ * A write is acknowledged only if the lease is provably held before the work and after it. `run` and
+ * `runAsync` bracket the work, and the ledger the handle exposes is a guarded view, so a long-lived
+ * consumer that keeps the ledger (a booking runtime, say) is stopped the same way: every call, including
+ * every prepared statement, first proves the lease. A write that was applied locally but not acknowledged
+ * is harmless, because a later owner restores from the durable copy and local files never survive.
+ *
+ * Reads go to the owner. A request for a tenant another replica owns fails with `LeaseHeldError`, whose
+ * record names the owner; forwarding to it belongs to the deployment layer (A3).
+ */
+import { discardLocalLedger, forgetTenant, publishSnapshot, restoreTenantLedger, sweepMaskedCopies, } from './epoch-copies.js';
+import { DrainTimeoutError, FencedError, LeaseLostError, ReplicaAtCapacityError, RouterClosedError, TenantDrainingError } from './errors.js';
+import { leaseKeyFor, tenantRootFor } from './ids.js';
+import { systemMonotonicClock, TenantLease } from './tenant-lease.js';
+/** An iterator that proves the lease on every step. Closing it is cleanup and always allowed. */
+function guardedIterator(iterator, assertHeld) {
+    const guarded = {
+        next: () => {
+            assertHeld();
+            return iterator.next();
+        },
+        return: (value) => iterator.return?.(value) ?? { done: true, value },
+        [Symbol.iterator]: () => guarded,
+    };
+    return guarded;
+}
+/**
+ * A prepared statement behind a gate. This is a facade, not a proxy: better-sqlite3 statements carry read-only
+ * properties, among them a raw `database` handle, that a proxy would have to return unchanged and so would leak.
+ */
+function guardedStatement(statement, assertHeld) {
+    const target = statement;
+    const facade = {};
+    for (const name of ['run', 'get', 'all', 'columns']) {
+        facade[name] = (...args) => {
+            assertHeld();
+            return Reflect.apply(target[name], statement, args);
+        };
+    }
+    facade.iterate = (...args) => {
+        assertHeld();
+        return guardedIterator(Reflect.apply(target.iterate, statement, args), assertHeld);
+    };
+    for (const name of ['bind', 'pluck', 'expand', 'raw', 'safeIntegers']) {
+        facade[name] = (...args) => {
+            assertHeld();
+            Reflect.apply(target[name], statement, args);
+            return facade;
+        };
+    }
+    for (const name of ['reader', 'readonly', 'source', 'busy']) {
+        Object.defineProperty(facade, name, { get: () => statement[name], enumerable: true });
+    }
+    Object.defineProperty(facade, 'database', { get: () => guardedDatabase(statement.database, assertHeld), enumerable: true });
+    return facade;
+}
+/** A transaction function behind a gate, with its `.deferred`, `.immediate` and `.exclusive` variants. Its raw `database` handle is not passed on. */
+function guardedTransaction(transaction, assertHeld) {
+    const gate = (fn) => function (...args) {
+        assertHeld();
+        return Reflect.apply(fn, this, args);
+    };
+    const guarded = gate(transaction);
+    for (const mode of ['default', 'deferred', 'immediate', 'exclusive']) {
+        const variant = transaction[mode];
+        if (typeof variant === 'function')
+            Object.defineProperty(guarded, mode, { value: gate(variant), enumerable: true });
+    }
+    return guarded;
+}
+/** The database of a ledger: every method proves the lease first, and statements and transactions come back guarded. */
+function guardedDatabase(db, assertHeld) {
+    const guarded = new Proxy(db, {
+        get(target, property) {
+            const value = Reflect.get(target, property, target);
+            if (typeof value !== 'function')
+                return value;
+            // The router owns the connection its own writes, its snapshots and its handoff depend on; a consumer cannot close it.
+            if (property === 'close')
+                return () => guarded;
+            return (...args) => {
+                assertHeld();
+                const result = Reflect.apply(value, target, args);
+                // `exec`, `defaultSafeIntegers`, `function`, `loadExtension` and the like return the database itself; what a
+                // caller chains onto it has to stay behind the gate.
+                if (result === target)
+                    return guarded;
+                if (property === 'prepare')
+                    return guardedStatement(result, assertHeld);
+                if (property === 'transaction')
+                    return guardedTransaction(result, assertHeld);
+                return result;
+            };
+        },
+    });
+    return guarded;
+}
+/**
+ * The ledger as a handle exposes it. The router owns its lifecycle, so `close` does nothing here, and the view is
+ * read-only: nothing a consumer does to it changes the ledger the router holds.
+ */
+function guardedLedger(ledger, assertHeld) {
+    return new Proxy(ledger, {
+        get(target, property) {
+            const value = Reflect.get(target, property, target);
+            if (property === 'db')
+                return guardedDatabase(value, assertHeld);
+            if (typeof value !== 'function')
+                return value;
+            if (property === 'close')
+                return () => undefined;
+            return (...args) => {
+                assertHeld();
+                return Reflect.apply(value, target, args);
+            };
+        },
+        // `Object.getOwnPropertyDescriptor(ledger, 'db').value` is a second way to the database that `get` does not cover.
+        getOwnPropertyDescriptor(target, property) {
+            const descriptor = Reflect.getOwnPropertyDescriptor(target, property);
+            if (property === 'db' && descriptor && 'value' in descriptor)
+                return { ...descriptor, value: guardedDatabase(descriptor.value, assertHeld) };
+            return descriptor;
+        },
+        // An assignment to a property that exists ends in defineProperty, so refusing that and delete is what makes the view read-only;
+        // the prototype and the extensibility are the rest of what a consumer could change on the ledger the router holds.
+        defineProperty: () => false,
+        deleteProperty: () => false,
+        setPrototypeOf: () => false,
+        preventExtensions: () => false,
+    });
+}
+/** A refused renewal, or a lease that was released or lost, ends the tenure for good. A merely expired local deadline does not: a renewal can resume it. */
+function endsTenure(error) {
+    return error instanceof LeaseLostError || (error instanceof FencedError && error.reason !== 'lease_expired_locally');
+}
+/** A promise in disguise: anything with a `then` function. */
+function isThenable(value) {
+    return (typeof value === 'object' || typeof value === 'function') && value !== null && typeof value.then === 'function';
+}
+class OpenTenant {
+    organization;
+    ledgerTenant;
+    epoch;
+    restoredFrom;
+    ledger;
+    raw;
+    deps;
+    // `draining`: new work is refused and what already runs may finish. `flushing` begins when the drain is over and the
+    // final image is about to be taken: from then on a write through the ledger could not be in the image, so the view is
+    // closed to every call until the tenant is closed or serving again. `forgetting`: the tenant is being deleted; nothing is
+    // served, but the lease is renewed until the deletion has handed it back.
+    state = 'serving';
+    inFlight = 0;
+    idleWaiters = new Set();
+    renewTimer = null;
+    renewTicking = false;
+    snapshotTimer = null;
+    snapshotTail = Promise.resolve();
+    snapshotQueued = 0;
+    constructor(deps) {
+        this.deps = deps;
+        this.organization = deps.organization;
+        this.ledgerTenant = deps.ledgerTenant;
+        this.epoch = deps.lease.epoch;
+        this.restoredFrom = deps.restored.restoredFrom;
+        this.raw = deps.restored.ledger;
+        // Work that began before a handoff may finish and new work is refused at `run`; the view itself stays open while
+        // the drain waits, so writes made in that time are in the final image, and closes when the image is taken.
+        this.ledger = guardedLedger(this.raw, () => this.assertViewUsable());
+    }
+    isOpen() {
+        return this.state === 'serving';
+    }
+    isDraining() {
+        return this.state === 'draining' || this.state === 'flushing';
+    }
+    /** The state as it is now, for a check after an await: another call may have changed it, which a narrowed `this.state` or getter does not show. */
+    phaseNow() {
+        return this.state;
+    }
+    /** Called once by the router, after the handle is registered. */
+    begin() {
+        this.scheduleRenewal();
+        this.scheduleSnapshot();
+    }
+    run(work) {
+        this.assertServing();
+        const result = work(this.ledger);
+        if (isThenable(result)) {
+            // The callback is already running, and what is left of it is outside the lease bracket and the drain. Nobody awaits
+            // the promise it returned, so its failure is taken here instead of becoming an unhandled rejection.
+            void Promise.resolve(result).catch(() => undefined);
+            throw new TypeError('TenantHandle.run takes a synchronous callback: use runAsync for work that awaits');
+        }
+        this.deps.lease.assertHeld();
+        return result;
+    }
+    async runAsync(work) {
+        this.assertServing();
+        this.inFlight += 1;
+        try {
+            const result = await work(this.ledger);
+            this.deps.lease.assertHeld();
+            return result;
+        }
+        finally {
+            this.inFlight -= 1;
+            if (this.inFlight === 0)
+                this.wakeIdleWaiters();
+        }
+    }
+    async renew() {
+        try {
+            await this.deps.lease.renew();
+            this.emit({ type: 'renewed', key: this.deps.key, epoch: this.epoch });
+        }
+        catch (error) {
+            if (endsTenure(error))
+                this.markLost();
+            throw error;
+        }
+    }
+    snapshot() {
+        this.snapshotQueued += 1;
+        const run = this.snapshotTail.then(() => this.publishNow(), () => this.publishNow());
+        this.snapshotTail = run.catch(() => undefined).finally(() => { this.snapshotQueued -= 1; });
+        return run;
+    }
+    async handoff() {
+        if (this.state === 'closed')
+            return;
+        if (this.state !== 'serving')
+            throw new TenantDrainingError(this.deps.key);
+        this.setState('draining');
+        // Only the periodic snapshots stop. The lease keeps being renewed through the drain and the final upload: both can
+        // take longer than a lease, and a handoff that let it lapse would flush into a prefix no successor reads.
+        this.stopSnapshotTimer();
+        try {
+            const drained = await this.drain();
+            if (drained === 'timeout') {
+                // The lease is renewed through the drain, so a call that never returns would hold the tenant here for ever.
+                this.emit({ type: 'drain_timeout', key: this.deps.key, epoch: this.epoch, inFlight: this.inFlight });
+                throw new DrainTimeoutError(this.deps.key, this.inFlight, this.deps.drainTimeoutMs);
+            }
+            // Closed meanwhile (abandoned, forgotten, lease lost): there is nothing left to flush, and a closed handle stays closed.
+            if (this.phaseNow() !== 'draining')
+                throw new TenantDrainingError(this.deps.key);
+            this.setState('flushing');
+            await this.snapshot();
+            // The same if it happened while the image was uploading. A forget deletes what this flush uploaded and needs the lease
+            // to finish; an abandon or a lost lease has ended the tenure. Releasing now, and reporting a flush, would be false.
+            if (this.phaseNow() !== 'flushing')
+                throw new TenantDrainingError(this.deps.key);
+        }
+        catch (error) {
+            // A flush that failed for want of a store leaves the tenant serving; one that failed because the lease is gone ends it.
+            if (endsTenure(error))
+                this.markLost();
+            else if (this.isDraining()) {
+                this.setState('serving');
+                this.begin();
+            }
+            throw error;
+        }
+        await this.finish(true);
+    }
+    async abandon() {
+        // A forget that is under way owns the end of the tenure: it hands the lease back itself.
+        if (this.state === 'closed' || this.state === 'forgetting')
+            return;
+        this.stopTimers();
+        await this.finish(false);
+    }
+    async forget() {
+        this.deps.lease.assertHeld();
+        // Closed with the lease still held is an abandon on its way out: it ends the tenure, and a forget on top of it would run
+        // the deletion over a tenure that is already over and report a second release.
+        if (this.state === 'forgetting' || this.state === 'closed')
+            throw new TenantDrainingError(this.deps.key);
+        // Nothing is served from here on, but the lease keeps being renewed: waiting for an upload under way can take longer than
+        // the lease, and a forget that let it lapse could no longer delete.
+        this.stopSnapshotTimer();
+        this.setState('forgetting');
+        this.closeLedger();
+        this.deps.detach(this);
+        const work = this.completeForget();
+        // The handle has left the router's table, but the deletion is not over: closing the router waits for it.
+        this.deps.retain(work.then(() => undefined, () => undefined));
+        return work;
+    }
+    async completeForget() {
+        try {
+            // An upload already under way would land after the deletion and leave the tenant's image in the store, where nothing
+            // would look for it again. Snapshots queued behind it refuse to start now that the tenant is being forgotten.
+            await this.snapshotTail.catch(() => undefined);
+            const outcome = await forgetTenant(this.deps.context);
+            this.emit({ type: 'released', key: this.deps.key, epoch: this.epoch, flushed: false });
+            return outcome;
+        }
+        finally {
+            this.setState('closed');
+            this.stopTimers();
+        }
+    }
+    assertServing() {
+        if (this.state === 'serving') {
+            this.deps.lease.assertHeld();
+            return;
+        }
+        // A lease that is gone says so first. With the lease still held, a handoff, a forget or an abandon is under way.
+        if (this.state === 'closed')
+            this.deps.lease.assertHeld();
+        throw new TenantDrainingError(this.deps.key);
+    }
+    /** The gate of the ledger a consumer holds: the lease, and a view that is open only while the tenant is served or draining. */
+    assertViewUsable() {
+        this.deps.lease.assertHeld();
+        if (this.state !== 'serving' && this.state !== 'draining')
+            throw new TenantDrainingError(this.deps.key);
+    }
+    setState(next) {
+        this.state = next;
+        // Whoever waits for the in-flight work to end is also waiting for the tenant to stay a tenant: closing it ends the wait.
+        if (next === 'closed' || next === 'forgetting')
+            this.wakeIdleWaiters();
+    }
+    emit(event) {
+        try {
+            this.deps.emit(event);
+        }
+        catch {
+            // A broken observer must never take a tenant down.
+        }
+    }
+    async publishNow() {
+        // A snapshot queued before a forget, an abandon or a lost lease must not start after it.
+        if (this.state === 'closed' || this.state === 'forgetting')
+            throw new TenantDrainingError(this.deps.key);
+        try {
+            const manifest = await publishSnapshot(this.deps.context);
+            this.emit({ type: 'snapshot', key: this.deps.key, epoch: this.epoch, seq: manifest.seq });
+            return manifest;
+        }
+        catch (error) {
+            this.emit({ type: 'snapshot_failed', key: this.deps.key, epoch: this.epoch, code: error.code ?? 'unknown' });
+            throw error;
+        }
+    }
+    /**
+     * Waits until the calls that were running are done, the tenant is closed, or `drainTimeoutMs` has passed. The timer
+     * keeps the process alive on purpose: a graceful stop that gave up the moment nothing else was pending would exit
+     * without flushing.
+     */
+    drain() {
+        if (this.inFlight === 0)
+            return Promise.resolve('idle');
+        return new Promise((resolve) => {
+            const end = (outcome) => {
+                clearTimeout(timer);
+                this.idleWaiters.delete(wake);
+                resolve(outcome);
+            };
+            const wake = () => end(this.state === 'draining' ? 'idle' : 'closed');
+            const timer = setTimeout(() => end('timeout'), this.deps.drainTimeoutMs);
+            this.idleWaiters.add(wake);
+        });
+    }
+    wakeIdleWaiters() {
+        for (const wake of [...this.idleWaiters])
+            wake();
+    }
+    closeLedger() {
+        try {
+            this.raw.close();
+        }
+        catch {
+            // Already closed.
+        }
+    }
+    async finish(flushed) {
+        this.setState('closed');
+        this.stopTimers();
+        this.closeLedger();
+        // With everything flushed the durable copy is the authority, and this replica keeps no copy of the tenant's data. An
+        // abandoned tenant keeps its file: it may hold writes that exist nowhere else, until the next open of that tenant on this
+        // replica, whose restore replaces it.
+        if (flushed) {
+            try {
+                discardLocalLedger(this.deps.context);
+            }
+            catch { /* a leftover file is discarded at the next restore */ }
+        }
+        this.deps.detach(this);
+        try {
+            await this.deps.lease.release();
+        }
+        finally {
+            this.emit({ type: 'released', key: this.deps.key, epoch: this.epoch, flushed });
+        }
+    }
+    /** The lease is gone for good: stop serving, close the ledger, leave the stale local file to be replaced by the next restore. */
+    markLost() {
+        if (this.state === 'closed')
+            return;
+        this.setState('closed');
+        this.stopTimers();
+        this.closeLedger();
+        this.deps.detach(this);
+        this.emit({ type: 'lost', key: this.deps.key, epoch: this.epoch });
+    }
+    stopTimers() {
+        if (this.renewTimer)
+            clearTimeout(this.renewTimer);
+        this.renewTimer = null;
+        this.stopSnapshotTimer();
+    }
+    stopSnapshotTimer() {
+        if (this.snapshotTimer)
+            clearTimeout(this.snapshotTimer);
+        this.snapshotTimer = null;
+    }
+    scheduleRenewal() {
+        // Idempotent, and never beside a renewal that is under way: a handoff that fails resumes service while its renewals never
+        // stopped, and a second chain would double them for the rest of the tenure.
+        if (!this.deps.autoRenew || this.state === 'closed' || this.renewTimer || this.renewTicking)
+            return;
+        this.renewTimer = setTimeout(() => { void this.renewTick(); }, this.deps.renewEveryMs);
+        this.renewTimer.unref();
+    }
+    async renewTick() {
+        this.renewTimer = null;
+        this.renewTicking = true;
+        try {
+            await this.renew();
+        }
+        catch (error) {
+            if (this.state !== 'closed')
+                this.emit({ type: 'renew_failed', key: this.deps.key, epoch: this.epoch, code: error.code ?? 'unknown' });
+        }
+        finally {
+            this.renewTicking = false;
+        }
+        this.scheduleRenewal();
+    }
+    scheduleSnapshot() {
+        if (this.deps.snapshotEveryMs <= 0 || this.state === 'closed' || this.snapshotTimer)
+            return;
+        this.snapshotTimer = setTimeout(() => { void this.snapshotTick(); }, this.deps.snapshotEveryMs);
+        this.snapshotTimer.unref();
+    }
+    async snapshotTick() {
+        this.snapshotTimer = null;
+        if (this.snapshotQueued === 0)
+            await this.snapshot().catch(() => undefined);
+        this.scheduleSnapshot();
+    }
+}
+// ---- the router -------------------------------------------------------------------------
+const DEFAULT_TTL_MS = 10_000;
+const DEFAULT_MARGIN_MS = 2_000;
+const DEFAULT_DRAIN_TIMEOUT_MS = 120_000;
+const DEFAULT_CLOSE_CONCURRENCY = 8;
+const MAX_TIMER_MS = 2_147_483_647;
+export class TenantRouter {
+    options;
+    clock;
+    handles = new Map();
+    opening = new Map();
+    forgets = new Set();
+    closed = false;
+    constructor(options) {
+        if (typeof options.replicaId !== 'string' || options.replicaId.length === 0)
+            throw new RangeError('replicaId must be a non-empty string');
+        if (options.maxOpenTenants !== undefined && (!Number.isSafeInteger(options.maxOpenTenants) || options.maxOpenTenants < 1))
+            throw new RangeError('maxOpenTenants must be a positive integer');
+        if (options.drainTimeoutMs !== undefined && (!Number.isSafeInteger(options.drainTimeoutMs) || options.drainTimeoutMs < 1 || options.drainTimeoutMs > MAX_TIMER_MS))
+            throw new RangeError(`drainTimeoutMs must be an integer within [1, ${MAX_TIMER_MS}]`);
+        if (options.closeConcurrency !== undefined && (!Number.isSafeInteger(options.closeConcurrency) || options.closeConcurrency < 1))
+            throw new RangeError('closeConcurrency must be a positive integer');
+        this.options = options;
+        this.clock = options.clock ?? systemMonotonicClock;
+    }
+    /**
+     * The handle of a tenant this replica owns, acquiring the lease and restoring the ledger first if needed.
+     * Concurrent calls for one tenant share one acquisition.
+     *
+     * @throws InvalidTenantIdError for an id that is not in the fixed format, before any store or file is touched
+     * @throws LeaseHeldError when another replica owns the tenant; the record names it
+     */
+    openTenant(organization, ledgerTenant) {
+        if (this.closed)
+            return Promise.reject(new RouterClosedError());
+        let key;
+        try {
+            key = leaseKeyFor(organization, ledgerTenant);
+        }
+        catch (error) {
+            return Promise.reject(error);
+        }
+        const existing = this.handles.get(key);
+        if (existing?.isOpen())
+            return Promise.resolve(existing);
+        // Our own lease is still held while a handoff drains, so an acquisition would only be told it is held by us.
+        if (existing?.isDraining())
+            return Promise.reject(new TenantDrainingError(key));
+        const pending = this.opening.get(key);
+        if (pending)
+            return pending;
+        const cap = this.options.maxOpenTenants;
+        // A tenant being opened already holds a slot, and so does one being handed off: both still hold their descriptors.
+        if (cap !== undefined && new Set([...this.handles.keys(), ...this.opening.keys()]).size >= cap)
+            return Promise.reject(new ReplicaAtCapacityError(cap));
+        const attempt = this.open(organization, ledgerTenant, key).finally(() => { this.opening.delete(key); });
+        this.opening.set(key, attempt);
+        return attempt;
+    }
+    /** The open handle of a tenant, if this replica currently serves it. */
+    handleOf(organization, ledgerTenant) {
+        const existing = this.handles.get(leaseKeyFor(organization, ledgerTenant));
+        return existing?.isOpen() ? existing : null;
+    }
+    openCount() {
+        return [...this.handles.values()].filter(handle => handle.isOpen()).length;
+    }
+    /**
+     * The directory of one tenant on this replica: its ledger and everything that must travel with it (planner working
+     * files, tickets). A consumer that keeps per-tenant files puts them here, never beside another tenant's.
+     */
+    tenantRootOf(organization, ledgerTenant) {
+        return tenantRootFor(this.options.stateRoot, organization, ledgerTenant);
+    }
+    /**
+     * Hands every open tenant off gracefully and refuses new ones. Resolves with the tenants whose flush failed, among them
+     * those whose drain ran past `drainTimeoutMs`. Tenants are handed off several at a time, so one whose drain cannot finish
+     * holds back only the slot it sits in, and for no longer than that deadline. A forget that is under way is waited for, for
+     * as long as a drain may take. A durable store call that never returns is the binding's to time out (`DurableStore`).
+     */
+    async close() {
+        this.closed = true;
+        const queue = [...this.handles];
+        const flushed = new Array(queue.length).fill(false);
+        let next = 0;
+        const lane = async () => {
+            for (;;) {
+                const index = next;
+                next += 1;
+                if (index >= queue.length)
+                    return;
+                try {
+                    await queue[index][1].handoff();
+                    flushed[index] = true;
+                }
+                catch {
+                    // Reported below, in the order the tenants were opened.
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(this.options.closeConcurrency ?? DEFAULT_CLOSE_CONCURRENCY, queue.length) }, lane));
+        await this.forgetsUnderWay();
+        return queue.filter((_, index) => !flushed[index]).map(([key]) => key);
+    }
+    /** Waits for the forgets that have left the table but are not over, but not longer than a drain may take. */
+    async forgetsUnderWay() {
+        if (this.forgets.size === 0)
+            return;
+        let timer;
+        const deadline = new Promise((resolve) => { timer = setTimeout(resolve, this.options.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS); });
+        try {
+            await Promise.race([Promise.allSettled([...this.forgets]), deadline]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async open(organization, ledgerTenant, key) {
+        const { options } = this;
+        // A broken observer must never leave a lease held and unrenewed: it is told what happened, and nothing it throws matters.
+        const emit = (event) => {
+            try {
+                options.onEvent?.(event);
+            }
+            catch {
+                // Ignored on purpose.
+            }
+        };
+        const lease = await TenantLease.acquire({
+            store: options.leases,
+            key,
+            replicaId: options.replicaId,
+            ttlMs: options.ttlMs ?? DEFAULT_TTL_MS,
+            marginMs: options.marginMs ?? DEFAULT_MARGIN_MS,
+            clock: this.clock,
+        });
+        emit({ type: 'acquired', key, epoch: lease.epoch });
+        const context = {
+            durable: options.durable,
+            lease,
+            organization,
+            ledgerTenant,
+            tenantRoot: tenantRootFor(options.stateRoot, organization, ledgerTenant),
+        };
+        let restored;
+        try {
+            restored = await restoreTenantLedger(context);
+        }
+        catch (error) {
+            await lease.release().catch(() => undefined);
+            throw error;
+        }
+        emit({ type: 'restored', key, epoch: lease.epoch, from: restored.restoredFrom, skipped: restored.skipped.length });
+        if (this.closed) {
+            restored.ledger.close();
+            await lease.release().catch(() => undefined);
+            throw new RouterClosedError();
+        }
+        const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
+        const handle = new OpenTenant({
+            key,
+            organization,
+            ledgerTenant,
+            lease,
+            context,
+            restored,
+            autoRenew: options.autoRenew ?? true,
+            renewEveryMs: options.renewEveryMs ?? Math.max(50, Math.floor(ttlMs / 3)),
+            snapshotEveryMs: options.snapshotEveryMs ?? 0,
+            drainTimeoutMs: options.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS,
+            emit,
+            detach: (gone) => { if (this.handles.get(key) === gone)
+                this.handles.delete(key); },
+            retain: (work) => {
+                this.forgets.add(work);
+                void work.then(() => { this.forgets.delete(work); });
+            },
+        });
+        this.handles.set(key, handle);
+        handle.begin();
+        // What a superseded owner uploaded after this tenant was forgotten is dead whoever wrote it. It is deleted here, off the
+        // open path, so a slow store cannot use up the lease before the first renewal; a failure is tried again at the next open.
+        if (lease.forgottenAtEpoch > 0)
+            void sweepMaskedCopies(options.durable, key, lease.forgottenAtEpoch).catch(() => undefined);
+        return handle;
+    }
+}

package/extension/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "Stai Travel Bridge",
-  "version": "0.2.0.32",
+  "version": "0.2.0.33",
   "description": "Stai 连接获准的旅行检索与供应商门户：检索结果经本机或授权后端桥接；门户可触发一次性登录代填，扩展不持久保存凭据。",
   "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnuzBX8rT8+RZKEUip5VRA0cY056pExy9PNdN9YCiuY3dgglRbcIBhjuFJoR6EX3/MaUv82vJVyItRVrowXLTwDk4WZof0vwFgKL0zBK304PC46pUaAKTSu1PgXeL1+j5KTAGz/9a9+RnPdbj7jND5rM/MOmBHNPVPpp1NlwxRGIgF9OxHSOfXORcH4Iyte1aJSFUJbFtNS/DaE5D+KjyupPLyIqVN/dCiKnNsTCEHQB9ngeli665PFp76hkccHQQZ4sVuMLS4zBPcazUsp1kcnW+xsQoUATkODTCY+Pzk3zUVF1QOESTjRCVnbNi8tkO43RMRs2q63gX/3inFivcEwIDAQAB",
   "permissions": [
@@ -56,5 +56,5 @@
       "run_at": "document_start"
     }
   ],
-  "version_name": "0.2.0-rc.32"
+  "version_name": "0.2.0-rc.33"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danceiny/gotry",
-  "version": "0.2.0-rc.32",
+  "version": "0.2.0-rc.33",
   "description": "GoTry — 从出发到下一次出发的 AI 旅行 Agent(dsh 插件)。npm/源码入口共用锁定的 dsh runtime + README 安装路径。",
   "type": "module",
   "main": "ts/src/index.ts",
@@ -101,6 +101,9 @@
     "schemas/booking.surface.schema.json",
     "schemas/booking.intent.schema.json",
     "ts/src/booking-surface/",
+    "ts/src/tenant-seam/",
+    "ts/src/tenant-state/",
+    "ts/src/llm-price-table.ts",
     "ts/scripts/skeleton-check.ts",
     "ts/scripts/skeleton-integration-test.ts",
     "ts/scripts/async-collect.ts",