@parcha/pi-durable-disk 0.0.0-stage → 0.1.0-beta.13

package/dist/run.js

@@ -0,0 +1,785 @@
+// The run lifecycle in two layers. `openRunLease` is the claim's lifecycle without a store or a Harness: the
+// claim, the owner lock, `run.json` with its heartbeat, the self-fence (a timer and a watchdog thread), the seal, the
+// fence and the release; an app that owns its own Harness and store connections uses it directly. `openDurableRun`
+// builds the whole run on it: the store, the seal check, pi's Harness, resume. A fence reported by the store, the claim,
+// a write of the run's own files or the lapsed lease kills the run's commands and ends the instance (exit 75). Nothing
+// is retried. The lease writes its own files through the claim directory (`ClaimDir`), never by path.
+import { mkdir } from "node:fs/promises";
+import { isAbsolute, join, relative } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
+import { Harness } from "@earendil-works/pi-durable";
+import { acquire } from "./claim.js";
+import { reapOrRefuse } from "./commands.js";
+import { archilEnv } from "./env.js";
+import { EXIT_FENCED, FencedError, HeldError, ownWriteFenced, PdaError } from "./errors.js";
+import { openArchilStore } from "./store.js";
+import { LeaseWatchdog } from "./watchdog.js";
+import { currentHolder, DEFAULT_HEARTBEAT_MS, DEFAULT_LEASE_EXPIRY_MS, DEFAULT_LEASE_MARGIN_MS, openClaimDir, persistRecord, readRunRecord, RUN_JSON, RunRecordError, writeRunRecord, } from "./status.js";
+/** The same-host owner lock under the run's root. */
+export const OWNER_LOCK = "owner.lock";
+/** pi's store under the claim's `store/` directory. */
+export const STORE_FILE = "run.sqlite";
+/** EX_DATAERR: the run's data refuses to open (a store behind its seal, an unreadable `run.json`); a restart cannot help. */
+export const EXIT_DATAERR = 65;
+/** EX_SOFTWARE: this package cannot read the head of pi's store (pi's schema moved); a restart cannot help. */
+export const EXIT_SOFTWARE = 70;
+/**
+ * Bad options, a call on a released run, or a store whose head cannot be read (STORE_HEAD_UNREADABLE: the run is marked
+ * `failed` and released, exit 70).
+ */
+export class RunError extends PdaError {
+    constructor(code, message, options = {}) {
+        super(code, message, { cause: options.cause, exitCode: code === "STORE_HEAD_UNREADABLE" ? EXIT_SOFTWARE : 1 });
+    }
+}
+/**
+ * The store's last committed sequence is below the `sealedSeq` the last clean release wrote: an acknowledged commit is
+ * missing. The run is refused before pi commits anything, and marked `failed` in `run.json`. Exit 65.
+ */
+export class StoreBehindSealError extends PdaError {
+    sealedSeq;
+    /** The store's last committed sequence. */
+    head;
+    constructor(file, sealedSeq, head) {
+        super("STORE_BEHIND_SEAL", `store ${file} is at sequence ${head}, behind its seal at ${sealedSeq}`, { exitCode: EXIT_DATAERR });
+        this.sealedSeq = sealedSeq;
+        this.head = head;
+    }
+}
+/**
+ * The self-fence rule: the last heartbeat that succeeded started longer ago than the lease expiry minus its margin. `by`
+ * names who saw it: the main thread's timer, or the watchdog thread (which had already killed the commands).
+ */
+export class LeaseLapsedError extends FencedError {
+    sinceMs;
+    by;
+    constructor(sinceMs, limitMs, by = "timer") {
+        super(`the last successful heartbeat started ${Math.round(sinceMs)} ms ago, past the ${limitMs} ms self-fence (seen by the ${by})`, { code: "LEASE_LAPSED" });
+        this.sinceMs = sinceMs;
+        this.by = by;
+    }
+}
+/**
+ * `storage`'s four reads on a frozen object that holds nothing else: no `commit` or `mintId` on it, and no reference to
+ * `storage` a cast could reach.
+ */
+function storeReads(storage) {
+    return Object.freeze({
+        scanConversations: (...args) => storage.scanConversations(...args),
+        scanTasks: (...args) => storage.scanTasks(...args),
+        scanDocuments: (...args) => storage.scanDocuments(...args),
+        document: (...args) => storage.document(...args),
+    });
+}
+/** Locks this process holds. A collected DatabaseSync closes its file, which would drop the lock without a word. */
+const heldLocks = new Set();
+const SQLITE_BUSY = 5;
+const SQLITE_LOCKED = 6;
+const isBusy = (error) => {
+    const errcode = error.errcode;
+    return typeof errcode === "number" && [SQLITE_BUSY, SQLITE_LOCKED].includes(errcode & 0xff);
+};
+/**
+ * Take the run's same-host owner lock for the life of this process. Node has no flock(2), so the lock is
+ * SQLite's exclusive POSIX lock on `<root>/owner.lock` (`locking_mode = EXCLUSIVE`, then `BEGIN EXCLUSIVE`), which the
+ * kernel drops when the process dies, however it dies. The journal stays in memory, so `owner.lock` is the only file
+ * the lock makes. Another process holding it is HeldError("owner-lock") (76); any other error is a fence, since the
+ * file is the claim's.
+ *
+ * Only this function may open `owner.lock` in the process: a POSIX lock is dropped when any descriptor the process holds
+ * on the file is closed, so a stray open and close elsewhere would silently release it.
+ */
+export function takeOwnerLock(root) {
+    const file = join(root, OWNER_LOCK);
+    let db;
+    try {
+        db = new DatabaseSync(file, { timeout: 0 });
+    }
+    catch (error) {
+        throw ownWriteFenced(file, error);
+    }
+    try {
+        db.exec("PRAGMA locking_mode = EXCLUSIVE");
+        db.exec("PRAGMA journal_mode = MEMORY");
+        db.exec("BEGIN EXCLUSIVE");
+        db.exec("COMMIT");
+    }
+    catch (error) {
+        try {
+            db.close();
+        }
+        catch {
+            // The lock was never ours; the refusal is the error to report.
+        }
+        if (isBusy(error))
+            throw new HeldError("owner-lock", `${file} is held by another process on this host`, { cause: error });
+        throw ownWriteFenced(file, error);
+    }
+    heldLocks.add(db);
+    return {
+        file,
+        release() {
+            if (!heldLocks.delete(db))
+                return;
+            try {
+                db.close();
+            }
+            catch (error) {
+                throw ownWriteFenced(file, error);
+            }
+        },
+    };
+}
+/**
+ * The last sequence the store committed: the one read of pi's internal schema in this package. pi-durable's public
+ * `Storage` has no head query (1.0.4 and 1.1.0), and the seal check must run before `Harness.open` can commit (its
+ * recovery commit would reuse a lost sequence number). pi's SqliteStorage gives each commit `durable_metadata.next_seq`
+ * and stores the successor in the same transaction, so the head is `next_seq - 1`. The query only reads. A store
+ * without that table, row or column fails closed (STORE_HEAD_UNREADABLE), so a pi whose schema moved never skips the
+ * check silently; a unit test pins the read to the sequence pi's commits return.
+ *
+ * `database` is the store's fence-aware facade (`FencedDatabase`), which turns an I/O-class error into a fence; through
+ * a raw connection such an error would read as STORE_HEAD_UNREADABLE.
+ */
+export async function storeHead(store) {
+    let row;
+    try {
+        row = await store.database.get("SELECT next_seq FROM durable_metadata WHERE singleton = 1");
+    }
+    catch (error) {
+        if (error instanceof FencedError)
+            throw error;
+        throw new RunError("STORE_HEAD_UNREADABLE", `cannot read the head of ${store.file}`, { cause: error });
+    }
+    const next = typeof row?.next_seq === "bigint" ? Number(row.next_seq) : row?.next_seq;
+    if (typeof next !== "number" || !Number.isSafeInteger(next) || next < 1) {
+        throw new RunError("STORE_HEAD_UNREADABLE", `${store.file} has no usable durable_metadata.next_seq (${JSON.stringify(row ?? null)})`);
+    }
+    return next - 1;
+}
+/**
+ * `storage` with `before` run ahead of every commit (it may throw to refuse it), every resolved commit's sequence
+ * reported to `after` and every rejected commit's error to `failed`; everything else delegates unchanged.
+ */
+function observeCommits(storage, hooks) {
+    return new Proxy(storage, {
+        get(target, key) {
+            if (key === "commit") {
+                return async (writes, context) => {
+                    hooks.before();
+                    let seq;
+                    try {
+                        seq = await target.commit(writes, context);
+                    }
+                    catch (error) {
+                        hooks.failed(error);
+                        throw error;
+                    }
+                    hooks.after(seq);
+                    return seq;
+                };
+            }
+            const value = Reflect.get(target, key, target);
+            return typeof value === "function" ? value.bind(target) : value;
+        },
+    });
+}
+// ---- the lease --------------------------------------------------------------------------------------------------------
+const asFenced = (error, what) => error instanceof FencedError ? error : new FencedError(`${what} failed, so the claim cannot be trusted`, { cause: error });
+function exitFenced(error) {
+    process.stderr.write(`pi-durable-disk: fenced (${error.code}): ${error.message}\n`);
+    process.exit(EXIT_FENCED);
+}
+function stepper(onStep) {
+    return async (name, run) => {
+        const t0 = performance.now();
+        const value = await run();
+        onStep?.(name, performance.now() - t0);
+        return value;
+    };
+}
+function leaseOf(options = {}) {
+    const lease = {
+        heartbeatMs: options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS,
+        expiryMs: options.expiryMs ?? DEFAULT_LEASE_EXPIRY_MS,
+        marginMs: options.marginMs ?? DEFAULT_LEASE_MARGIN_MS,
+        checkMs: options.checkMs ?? 1000,
+    };
+    const limit = lease.expiryMs - lease.marginMs;
+    if (!(lease.heartbeatMs > 0 && lease.checkMs > 0 && lease.marginMs >= 0 && limit > lease.heartbeatMs)) {
+        throw new RunError("INVALID_ARGUMENT", `lease ${JSON.stringify(lease)}: the self-fence (expiry minus margin) must exceed the heartbeat period`);
+    }
+    return lease;
+}
+const noCleanup = async () => undefined;
+class Lease {
+    ref;
+    claim;
+    /** The claim itself, for a layer that builds on the lease (`openDurableRun` builds the env on it). */
+    rawClaim;
+    #options;
+    #context;
+    #lease;
+    #clock;
+    #persist;
+    #cleanup;
+    #onFenced;
+    #openDir;
+    /** The claim's root held open: every write of the lease's own files goes through it. */
+    #dir;
+    #dirClosed;
+    #lock;
+    #previous;
+    #record;
+    #lastSeq = 0;
+    #fence;
+    /** Aborted by the fence, synchronously: a `run.json` write still in flight then refuses to rename. */
+    #fencing = new AbortController();
+    #releasing;
+    #released = false;
+    #writes = Promise.resolve();
+    #beating = false;
+    #lastBeatStart = 0;
+    #lastBeatNs = 0n;
+    #heartbeat;
+    #leaseCheck;
+    #watchdog;
+    constructor(ref, claim, options, context) {
+        this.ref = ref;
+        this.rawClaim = claim;
+        this.#options = options;
+        this.#context = context;
+        this.#lease = leaseOf(options.lease);
+        this.#clock = options.clock ?? (() => performance.now());
+        this.#persist = options.persist ?? persistRecord;
+        this.#cleanup = options.cleanup ?? noCleanup;
+        this.#onFenced = options.onFenced ?? exitFenced;
+        this.#openDir = options.claimDir ?? ((root) => openClaimDir(root));
+        const lease = this;
+        this.claim = {
+            ref: claim.ref,
+            disk: claim.disk,
+            root: claim.root,
+            work: claim.work,
+            store: claim.store,
+            get fenced() {
+                return lease.fenced;
+            },
+            barrier: () => lease.barrier(),
+        };
+    }
+    get record() {
+        return this.#record;
+    }
+    get generation() {
+        return this.#record.generation;
+    }
+    get sealedSeq() {
+        return this.#previous?.sealedSeq ?? null;
+    }
+    get fenced() {
+        return this.#fence !== undefined;
+    }
+    /** The first fence, once fenced. */
+    get fencedBy() {
+        return this.#fence?.error;
+    }
+    /** True once `release()` began. */
+    get releasing() {
+        return this.#releasing !== undefined || this.#released;
+    }
+    fence(error) {
+        if (this.#fence)
+            return this.#fence.error;
+        const fence = { error, done: Promise.resolve() };
+        this.#fence = fence;
+        this.#fencing.abort(error);
+        this.#stopTimers();
+        this.rawClaim.markFenced(error);
+        fence.done = this.#cleanup(this.#context)
+            .catch(() => undefined)
+            .then(() => this.#onFenced(error))
+            .catch(() => exitFenced(error));
+        void fence.done.then(() => this.#closeDir());
+        return error;
+    }
+    fenceSettled() {
+        return this.#fence?.done ?? Promise.resolve();
+    }
+    /**
+     * The order after the claim. `run.json` is written (`running`, generation + 1, this holder, a heartbeat) and the
+     * heartbeat starts right after the owner lock, before the store and the Harness open, so a supervisor never sees the
+     * previous incarnation's stale heartbeat on a live claim. That first write keeps the previous `sealedSeq`, which the
+     * seal check still needs; `live()` clears it once the store is open.
+     */
+    async open(step, holder) {
+        const claim = this.rawClaim;
+        // SQLite resolves symbolic links in a file name, `/proc/self/fd/<n>` included, so `owner.lock` is the one file opened
+        // by path: right after the mount check, and the file it names must then be the one in the claim directory.
+        this.#lock = await step("owner-lock", async () => {
+            const dir = (this.#dir = await this.#openDir(claim.root));
+            await dir.assertMounted();
+            const lock = takeOwnerLock(claim.root);
+            try {
+                await dir.assertSame(OWNER_LOCK);
+            }
+            catch (error) {
+                try {
+                    lock.release();
+                }
+                catch {
+                    // The lock is not the claim's; the failed check is the fence to report.
+                }
+                throw error;
+            }
+            return lock;
+        });
+        const previous = await this.#readPrevious();
+        this.#previous = previous;
+        this.#lastSeq = previous?.sealedSeq ?? 0;
+        await step("run.json", async () => {
+            const now = new Date().toISOString();
+            this.#noteBeat(await this.#write(() => ({
+                run: this.ref.id,
+                status: "running",
+                generation: (previous?.generation ?? 0) + 1,
+                sealedSeq: previous?.sealedSeq ?? null,
+                wakeAt: null,
+                holder,
+                heartbeatAt: now,
+                updatedAt: now,
+                detail: null,
+            })));
+        });
+        // Two self-fences: a timer on this thread, and a watchdog thread for when this thread is stuck in a FUSE request.
+        await step("heartbeat", () => {
+            this.#heartbeat = setInterval(() => this.#beat(), this.#lease.heartbeatMs).unref();
+            this.#leaseCheck = setInterval(() => this.#checkLease(), this.#lease.checkMs).unref();
+            this.#watchdog = new LeaseWatchdog({
+                limitMs: this.#lease.expiryMs - this.#lease.marginMs,
+                checkMs: this.#lease.checkMs,
+                beatNs: this.#lastBeatNs,
+                ownCgroup: this.#options.ownCgroup === true,
+            });
+        });
+        await step("layout", async () => {
+            for (const dir of [claim.store, claim.work, join(claim.root, "tmp")]) {
+                const under = relative(claim.root, dir);
+                if (under === "" || under.startsWith("..") || isAbsolute(under))
+                    throw new RunError("INVALID_ARGUMENT", `${dir} is not under the claim root ${claim.root}`);
+                try {
+                    await mkdir(join(this.#dir.at, under), { recursive: true });
+                }
+                catch (error) {
+                    throw ownWriteFenced(dir, error);
+                }
+            }
+        });
+    }
+    async checkSeal(store) {
+        const sealed = this.sealedSeq;
+        let head;
+        try {
+            head = await storeHead(store);
+        }
+        catch (error) {
+            if (error instanceof RunError)
+                await this.#markFailed({ code: error.code, sealedSeq: sealed, reason: error.message });
+            throw error;
+        }
+        this.#lastSeq = Math.max(this.#lastSeq, head);
+        if (sealed === null || head >= sealed)
+            return;
+        const refusal = new StoreBehindSealError(store.file, sealed, head);
+        await this.#markFailed({ code: refusal.code, sealedSeq: sealed, head });
+        throw refusal;
+    }
+    observe(storage) {
+        return observeCommits(storage, {
+            before: () => {
+                const lapse = this.#watchdogLapse();
+                if (lapse)
+                    throw this.fence(lapse);
+            },
+            after: (seq) => {
+                this.#lastSeq = Math.max(this.#lastSeq, seq);
+                const lapse = this.#watchdogLapse();
+                if (lapse)
+                    this.fence(lapse);
+            },
+            failed: (error) => {
+                if (error instanceof FencedError)
+                    this.fence(error);
+            },
+        });
+    }
+    async live() {
+        this.#assertUsable();
+        const now = new Date().toISOString();
+        this.#noteBeat(await this.#write((current) => ({ ...current, sealedSeq: null, heartbeatAt: now })));
+    }
+    async abandon(error, close) {
+        this.#stopTimers();
+        if (this.#fence || error instanceof FencedError) {
+            this.fence(this.#fence?.error ?? error);
+            await this.fenceSettled();
+            return;
+        }
+        try {
+            await this.#writes;
+            await close?.();
+            this.#lock?.release();
+            await this.#closeDir();
+            if (!(error instanceof HeldError))
+                await this.rawClaim.release();
+        }
+        catch (cleanupError) {
+            if (cleanupError instanceof FencedError) {
+                this.fence(cleanupError);
+                await this.fenceSettled();
+            }
+        }
+    }
+    async setStatus(status, detail = null, options = {}) {
+        this.#assertUsable();
+        if (status === "done" || status === "failed")
+            await this.barrier();
+        const now = new Date().toISOString();
+        await this.#write((current) => ({ ...current, status, detail, wakeAt: options.wakeAt ?? null, updatedAt: now }));
+    }
+    async barrier() {
+        if (this.#fence)
+            throw this.#fence.error;
+        try {
+            await this.rawClaim.barrier();
+        }
+        catch (error) {
+            if (error instanceof FencedError)
+                throw this.fence(error);
+            throw error;
+        }
+    }
+    release() {
+        if (this.#released)
+            return Promise.resolve();
+        if (this.#fence)
+            return Promise.reject(this.#fence.error);
+        this.#releasing ??= this.#release();
+        return this.#releasing;
+    }
+    async #release() {
+        const step = stepper(this.#options.onStep);
+        await step("cleanup", () => this.#cleanup(this.#context));
+        await step("barrier", () => this.barrier());
+        clearInterval(this.#heartbeat);
+        await step("seal", () => {
+            const now = new Date().toISOString();
+            return this.#write((current) => ({
+                ...current,
+                status: current.status === "running" ? "paused" : current.status,
+                sealedSeq: this.#lastSeq,
+                updatedAt: now,
+            }));
+        });
+        this.#stopTimers();
+        await step("unlock", () => {
+            try {
+                this.#lock.release();
+            }
+            catch (error) {
+                throw this.fence(asFenced(error, `releasing ${OWNER_LOCK}`));
+            }
+        });
+        await step("unmount", async () => {
+            await this.#closeDir();
+            try {
+                await this.rawClaim.release();
+            }
+            catch (error) {
+                if (error instanceof FencedError)
+                    throw this.fence(error);
+                throw error;
+            }
+        });
+        this.#released = true;
+    }
+    async #readPrevious() {
+        let previous;
+        try {
+            previous = await readRunRecord(this.#dir);
+        }
+        catch (error) {
+            if (error instanceof RunRecordError)
+                throw error;
+            throw asFenced(error, `reading ${join(this.rawClaim.root, RUN_JSON)}`);
+        }
+        if (previous !== undefined && previous.run !== this.ref.id) {
+            throw new RunRecordError(`${join(this.rawClaim.root, RUN_JSON)} names run ${JSON.stringify(previous.run)}, not ${JSON.stringify(this.ref.id)}`);
+        }
+        return previous;
+    }
+    /** `run.json` becomes `failed` with `detail`; the seal the record carries is kept as evidence. */
+    async #markFailed(detail) {
+        const now = new Date().toISOString();
+        await this.#write((current) => ({ ...current, status: "failed", updatedAt: now, detail }));
+    }
+    /** Serialize writes of `run.json`. Resolves with the clock readings when this write started; a failure fences. */
+    #write(change) {
+        const next = this.#writes.then(async () => {
+            if (this.#fence)
+                throw this.#fence.error;
+            const record = change(this.#record);
+            const started = this.#clock();
+            const startedNs = process.hrtime.bigint();
+            try {
+                const dir = this.#dir;
+                if (dir === undefined)
+                    throw new FencedError(`the claim directory of ${this.rawClaim.root} is closed`);
+                await writeRunRecord(dir, record, this.#persist, this.#fencing.signal);
+            }
+            catch (error) {
+                throw this.fence(asFenced(error, `writing ${RUN_JSON}`));
+            }
+            this.#record = record;
+            return { started, startedNs };
+        });
+        this.#writes = next.catch(() => undefined);
+        return next;
+    }
+    /**
+     * Close the claim directory once no write can still address it: a closed descriptor's number may be reused, and
+     * `/proc/self/fd/<n>` would then name another file. An open directory also keeps the mount busy, so this runs before
+     * the unmount.
+     */
+    #closeDir() {
+        this.#dirClosed ??= (async () => {
+            await this.#writes;
+            const dir = this.#dir;
+            this.#dir = undefined;
+            await dir?.close().catch(() => undefined);
+        })();
+        return this.#dirClosed;
+    }
+    #beat() {
+        if (this.#beating || this.#fence || this.#released)
+            return;
+        this.#beating = true;
+        this.#write((current) => ({ ...current, heartbeatAt: new Date().toISOString() }))
+            .then((started) => this.#noteBeat(started))
+            .catch(() => undefined)
+            .finally(() => (this.#beating = false));
+    }
+    /** A heartbeat-class write (the first, the one in `live()`, each heartbeat) succeeded. */
+    #noteBeat(write) {
+        this.#lastBeatStart = Math.max(this.#lastBeatStart, write.started);
+        if (write.startedNs > this.#lastBeatNs)
+            this.#lastBeatNs = write.startedNs;
+        this.#watchdog?.beat(write.startedNs);
+    }
+    #watchdogLapse() {
+        const since = this.#watchdog?.lapsedAfterMs;
+        return since === undefined ? undefined : new LeaseLapsedError(since, this.#lease.expiryMs - this.#lease.marginMs, "watchdog");
+    }
+    #checkLease() {
+        if (this.#fence)
+            return;
+        const lapse = this.#watchdogLapse();
+        if (lapse) {
+            this.fence(lapse);
+            return;
+        }
+        const since = this.#clock() - this.#lastBeatStart;
+        const limit = this.#lease.expiryMs - this.#lease.marginMs;
+        if (since > limit)
+            this.fence(new LeaseLapsedError(since, limit));
+    }
+    #stopTimers() {
+        clearInterval(this.#heartbeat);
+        clearInterval(this.#leaseCheck);
+        this.#heartbeat = this.#leaseCheck = undefined;
+        this.#watchdog?.stop();
+        this.#watchdog = undefined;
+    }
+    #assertUsable() {
+        if (this.#fence)
+            throw this.#fence.error;
+        if (this.#released || this.#releasing)
+            throw new RunError("RUN_RELEASED", `run ${this.ref.id} is released`);
+    }
+}
+async function openLease(ref, options, step) {
+    const context = options.context ?? BACKGROUND_CONTEXT;
+    leaseOf(options.lease);
+    let holder;
+    try {
+        holder = currentHolder(options.holder);
+    }
+    catch (error) {
+        throw new RunError("INVALID_ARGUMENT", `holder ${JSON.stringify(options.holder)}: ${error.message}`, { cause: error });
+    }
+    if (options.acquire === undefined && !options.mountToken) {
+        throw new RunError("INVALID_ARGUMENT", "mountToken is required to mount an Archil disk; a run on a local directory passes localRun(root) instead");
+    }
+    const claim = await step("acquire", () => (options.acquire ?? acquire)({
+        ref,
+        token: options.mountToken ?? "",
+        ...(options.mountRoot === undefined ? {} : { mountRoot: options.mountRoot }),
+        ...(options.force === undefined ? {} : { force: options.force }),
+        ...(options.host === undefined ? {} : { host: options.host }),
+    }));
+    const lease = new Lease(ref, claim, options, context);
+    try {
+        await lease.open(step, holder);
+    }
+    catch (error) {
+        await lease.abandon(error);
+        throw error;
+    }
+    return lease;
+}
+/**
+ * Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock, write
+ * `run.json` and start the heartbeat and the self-fence, create `store/`, `work/` and `tmp/`. Errors are typed as for
+ * `openDurableRun`: HeldError (76), FencedError (75, also passed to `onFenced`), RunRecordError (65), RunError
+ * INVALID_ARGUMENT before any claim.
+ */
+export async function openRunLease(ref, options) {
+    return openLease(ref, options, stepper(options.onStep));
+}
+// ---- the run ----------------------------------------------------------------------------------------------------------
+class Run {
+    ref;
+    env;
+    #lease;
+    #options;
+    #context;
+    #store;
+    #harness;
+    #releasing;
+    #released = false;
+    constructor(lease, env, options, context) {
+        this.ref = lease.ref;
+        this.#lease = lease;
+        this.env = env;
+        this.#options = options;
+        this.#context = context;
+    }
+    get harness() {
+        return this.#harness;
+    }
+    get store() {
+        return this.#store;
+    }
+    get claim() {
+        return this.#lease.claim;
+    }
+    get record() {
+        return this.#lease.record;
+    }
+    get generation() {
+        return this.#lease.generation;
+    }
+    get fenced() {
+        return this.#lease.fenced;
+    }
+    /** The store, the seal check, pi's Harness over the observed storage, the seal cleared, resume. */
+    async open(step) {
+        const lease = this.#lease;
+        const file = join(lease.rawClaim.store, STORE_FILE);
+        this.#store = await step("store", () => openArchilStore(file, this.#options.profile ?? "exclusive", {
+            onFenced: (error) => void lease.fence(error),
+            ...(this.#options.decorateStore === undefined ? {} : { decorate: this.#options.decorateStore }),
+        }));
+        // Both refusals here are terminal: the run is marked `failed`, keeping its seal, so the supervisor stops on it.
+        await step("seal", () => lease.checkSeal(this.#store));
+        const storage = lease.observe(this.#store.storage);
+        this.#harness = await step("harness", () => Harness.open(storage, { ...this.#options.harness, env: this.env }, this.#context));
+        await step("live", () => lease.live());
+        const before = this.#options.beforeResume;
+        if (before) {
+            // The storage the Harness was opened over (the lease observes its commits), and its head now: nothing commits
+            // between the open and the hook, since the Harness has not resumed.
+            await step("before-resume", async () => {
+                const head = (await storeHead(this.#store));
+                await before({ harness: this.#harness, context: this.#context, generation: lease.generation, record: lease.record, claim: lease.claim, storage: storeReads(storage), head });
+            });
+        }
+        await step("resume", () => this.#harness.resume());
+    }
+    /** Undo a failed open: close what this layer opened, then the lease's own undo. */
+    abandon(error) {
+        return this.#lease.abandon(error, async () => {
+            if (this.#harness)
+                await this.#harness.close(this.#context);
+            else if (this.#store)
+                await this.#store.storage.close(this.#context);
+        });
+    }
+    async setStatus(status, detail = null, options = {}) {
+        if (!this.#lease.fenced && (this.#released || this.#releasing))
+            throw new RunError("RUN_RELEASED", `run ${this.ref.id} is released`);
+        await this.#lease.setStatus(status, detail, options);
+    }
+    release() {
+        if (this.#released)
+            return Promise.resolve();
+        if (this.#lease.fenced)
+            return Promise.reject(this.#lease.fencedBy);
+        this.#releasing ??= this.#release();
+        return this.#releasing;
+    }
+    async #release() {
+        const step = stepper(this.#options.onStep);
+        // Closing first aborts running work without recording an outcome, so the next open reports it interrupted; the
+        // lease then kills the commands, barriers, seals and unmounts.
+        await step("close", async () => {
+            try {
+                await this.#harness.close(this.#context);
+            }
+            catch (error) {
+                throw this.#lease.fence(asFenced(error, "closing the Harness"));
+            }
+        });
+        await this.#lease.release();
+        this.#released = true;
+    }
+}
+/**
+ * Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock, `run.json`, heartbeat and
+ * self-fence, layout), then the store, the seal check before anything commits, pi's Harness with the app's registry,
+ * the seal cleared in `run.json`, resume.
+ *
+ * Errors are typed: HeldError (76) when another client, another process on this host or another connection holds the
+ * run; FencedError (75) when the claim cannot be trusted (also passed to `onFenced`); StoreBehindSealError (65) and
+ * RunError STORE_HEAD_UNREADABLE (70), each after marking the run `failed` and releasing the claim.
+ */
+export async function openDurableRun(ref, options) {
+    const context = options.context ?? BACKGROUND_CONTEXT;
+    const step = stepper(options.onStep);
+    let env;
+    const lease = await openLease(ref, { ...options, context, cleanup: (c) => env?.cleanup(c) ?? Promise.resolve() }, step);
+    try {
+        // A claim that records its commands (a local run's): end what a killed incarnation left running, now that the owner
+        // lock is held and before the Harness opens, so no command of it still writes in work/ once the run resumes.
+        const commands = lease.rawClaim.commands;
+        // A process of a killed incarnation that outlives its SIGKILL refuses the open (HeldError, 76): the run never resumes
+        // beside it, and its record stays for the next open.
+        if (commands !== undefined)
+            await step("commands", () => reapOrRefuse(commands));
+        env = (options.env ?? ((c) => archilEnv(c)))(lease.rawClaim);
+    }
+    catch (error) {
+        await lease.abandon(error);
+        throw error;
+    }
+    const run = new Run(lease, env, options, context);
+    try {
+        await run.open(step);
+    }
+    catch (error) {
+        await run.abandon(error);
+        throw error;
+    }
+    return run;
+}