@smartmemory/stratum 0.4.6 → 0.5.0

package/dist/engine/run_lock.js

@@ -0,0 +1,628 @@
+import { randomUUID } from "node:crypto";
+import { closeSync, fstatSync, linkSync, openSync, readSync, statSync, unlinkSync, writeFileSync, readFileSync } from "node:fs";
+import { link, mkdir, open, readdir, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { processIdentity } from "../connectors/proc_identity.js";
+import { procStartTime } from "../connectors/proc_identity.js";
+import { assertRunId } from "./state.js";
+/** The general waiter's budget. 300s, not 30s: two locked sections await LLM-scale work —
+ *  a judged ensure and the `evaluate:` runner — so a shorter budget would throw
+ *  RUN_LOCK_TIMEOUT against a perfectly healthy holder. Staleness is decided by process
+ *  identity, never by age, so a long hold is never mistaken for a dead one and this is a
+ *  backstop against a wedge rather than a liveness check. */
+export const DEFAULT_RUN_LOCK_TIMEOUT_MS = 300_000;
+/** How long `flowCancel` waits to ACQUIRE the run lock, before anything is settled. */
+export const DEFAULT_CANCEL_LOCK_WAIT_MS = 120_000;
+/** The 0.4.0 acknowledgement budget for agent teardown, whose clock starts only AFTER
+ *  settlement. A different measurement from the two above; collapsing any two of the three
+ *  reintroduces either a false timeout or a cancel that appears to hang. */
+export const DEFAULT_CANCEL_TIMEOUT_MS = 15_000;
+/** A `<runId>.lock.<token>` (or `<runId>.lock-break.<token>`) tmp older than this belongs to a
+ *  process that crashed between writing it and linking it, and is swept opportunistically. */
+const RUN_LOCK_TMP_TTL_MS = 60_000;
+/** The ONE age rule in this file, and it applies to exactly one thing: a `.lock-break` that
+ *  carries no readable identity at all.
+ *
+ *  Every other reclaim here is identity-only (R4-2) because there is an owner to judge. An
+ *  opaque break-lock has none — no pid, no start time, nothing a probe can answer about — so
+ *  identity can never clear it and the run would be wedged forever by a zero-byte file. Age is
+ *  the only remaining discriminator, and it is safe here precisely because it is conditioned on
+ *  the absence of an identity: a PARSEABLE break-lock is never aged out, however old. */
+const BREAK_LOCK_OPAQUE_TTL_MS = 60_000;
+/** A timeout that is NaN or Infinity is not a long budget, it is a missing one: `Date.now() +
+ *  NaN` is NaN, every `Date.now() >= deadline` comparison against it is false, and the loop it
+ *  bounds never ends. Reject the value where it is read, so a mistyped env var fails loudly at
+ *  the first call instead of hanging a teardown (F6). */
+export function requireDurationMs(value, name) {
+    if (!Number.isFinite(value) || value < 0)
+        throw new Error(`${name} must be a nonnegative finite number`);
+    return value;
+}
+export function runLockTimeoutMs() {
+    return requireDurationMs(Number(process.env.STRATUM_RUN_LOCK_TIMEOUT_MS ?? DEFAULT_RUN_LOCK_TIMEOUT_MS), "STRATUM_RUN_LOCK_TIMEOUT_MS");
+}
+export function cancelLockWaitMs() {
+    return requireDurationMs(Number(process.env.STRATUM_CANCEL_LOCK_WAIT_MS ?? DEFAULT_CANCEL_LOCK_WAIT_MS), "STRATUM_CANCEL_LOCK_WAIT_MS");
+}
+export function cancelTimeoutMs() {
+    return requireDurationMs(Number(process.env.STRATUM_CANCEL_TIMEOUT_MS ?? DEFAULT_CANCEL_TIMEOUT_MS), "STRATUM_CANCEL_TIMEOUT_MS");
+}
+function lockPath(root, runId) {
+    assertRunId(runId);
+    return join(root, `${runId}.lock`);
+}
+function breakLockPath(root, runId) {
+    assertRunId(runId);
+    return join(root, `${runId}.lock-break`);
+}
+function driverLeasePath(root, runId) {
+    assertRunId(runId);
+    return join(root, `${runId}.driver`);
+}
+function errorCode(error) {
+    return error?.code;
+}
+function isNotFound(error) {
+    return errorCode(error) === "ENOENT";
+}
+function isExists(error) {
+    return errorCode(error) === "EEXIST";
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function parseRecord(text) {
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof value !== "object" || value === null)
+        return undefined;
+    const record = value;
+    if (!Number.isSafeInteger(record.pid) || typeof record.startTime !== "string" || !record.startTime)
+        return undefined;
+    if (typeof record.token !== "string")
+        return undefined;
+    return record;
+}
+/** Positional read, so the same descriptor can be read more than once. `handle.readFile`
+ *  advances the file position and a second call returns "", which would silently turn the CAS
+ *  re-read below into "the record vanished". */
+async function readHandleRecord(handle) {
+    const info = await handle.stat();
+    if (info.size === 0 || info.size > 64 * 1024)
+        return undefined;
+    const buffer = Buffer.allocUnsafe(info.size);
+    await handle.read(buffer, 0, info.size, 0);
+    return parseRecord(buffer.toString("utf8"));
+}
+/** ABSENCE and FAILURE are different answers (F4). A blanket catch here reports an EACCES or an
+ *  EIO as "there is no lock", and every caller reads that as a licence to proceed — which is how
+ *  an unreadable-but-live lock file turns into two writers. Only ENOENT is absence. */
+async function readRecord(path) {
+    let handle;
+    try {
+        handle = await open(path, "r");
+    }
+    catch (error) {
+        if (isNotFound(error))
+            return undefined;
+        throw error;
+    }
+    try {
+        return await readHandleRecord(handle);
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * Publish a COMPLETE record at `path` in one atomic step (R3-2): write the whole thing to a
+ * private tmp name, then `link()` it into place. The published file therefore never exists
+ * half-written, and a crash between the two leaves an orphan tmp and NO published record.
+ *
+ * Used for the run lock AND the break-lock. The break-lock used to be `open(…, "wx")` followed
+ * by a separate write, so a crash in between left a zero-byte `.lock-break` that parses to
+ * nothing, blocks every future breaker, and — having no identity — could never be reclaimed.
+ *
+ * @returns false when the path already exists (EEXIST): someone else published first.
+ */
+async function publishRecord(path, record) {
+    const temporary = `${path}.${record.token}`;
+    writeFileSync(temporary, JSON.stringify(record), { encoding: "utf8", mode: 0o600 });
+    let published = false;
+    try {
+        await link(temporary, path);
+        published = true;
+    }
+    catch (error) {
+        if (!isExists(error)) {
+            await unlink(temporary).catch(() => undefined);
+            throw error;
+        }
+    }
+    await unlink(temporary).catch(() => undefined);
+    return published;
+}
+/**
+ * CAS-style removal: unlink `path` only while it still names the INODE we inspected AND that
+ * inode's record still carries `token`.
+ *
+ * The inspected descriptor is held OPEN across the whole comparison, and that is the load-
+ * bearing part. A closed descriptor's inode can be recycled between the read and the unlink,
+ * so a check that reads, closes, and later unlinks can pass its own comparison against a
+ * different file — which is exactly how two reclaimers both judge a dead owner, one removes it,
+ * a live successor publishes, and the delayed one deletes the successor.
+ *
+ * @returns whether this call is the one that removed the record.
+ */
+async function unlinkOwn(path, token, options = {}) {
+    let handle;
+    // `false` is reserved for the four benign verdicts: the record is gone, unparseable, carries
+    // another token, or names another inode. An EACCES/EIO is none of those — it is a lock we
+    // could not remove, and swallowing it reports a release that never happened and leaves a
+    // live-owned lock nobody can reclaim (F4).
+    try {
+        handle = await open(path, "r");
+    }
+    catch (error) {
+        if (isNotFound(error))
+            return false;
+        throw error;
+    }
+    try {
+        const record = await readHandleRecord(handle);
+        if (record === undefined || record.token !== token)
+            return false;
+        const inode = (await handle.stat()).ino;
+        if (options.beforeOwnUnlink)
+            await options.beforeOwnUnlink();
+        let current;
+        try {
+            current = await stat(path);
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return false;
+            throw error;
+        }
+        if (current.ino !== inode)
+            return false; // the path names a successor now
+        const still = await readHandleRecord(handle); // same fd, so the same inode
+        if (still === undefined || still.token !== token)
+            return false;
+        try {
+            await unlink(path);
+        }
+        catch (error) {
+            if (!isNotFound(error))
+                throw error;
+        }
+        return true;
+    }
+    finally {
+        await handle.close();
+    }
+}
+/** The opaque-break-lock reclaim (see BREAK_LOCK_OPAQUE_TTL_MS). Conditional on the inode that
+ *  was actually judged, exactly as `unlinkOwn` is, and refuses the moment the file turns out to
+ *  be parseable after all — an identity that appeared between the two reads is an owner, and
+ *  owners are judged by liveness, never by age. */
+async function reclaimOpaqueBreakLock(path, options) {
+    const ttl = options.tmpTtlMs ?? BREAK_LOCK_OPAQUE_TTL_MS;
+    const nowMs = (options.now ?? Date.now)();
+    let handle;
+    try {
+        handle = await open(path, "r");
+    }
+    catch {
+        return false;
+    }
+    try {
+        const info = await handle.stat();
+        if (await readHandleRecord(handle) !== undefined)
+            return false; // it has an identity: not ours to age out
+        if (nowMs - info.mtimeMs < ttl)
+            return false; // a breaker may still be mid-write
+        let current;
+        try {
+            current = await stat(path);
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return false;
+            throw error;
+        }
+        if (current.ino !== info.ino)
+            return false;
+        try {
+            await unlink(path);
+        }
+        catch (error) {
+            if (!isNotFound(error))
+                throw error;
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+    finally {
+        await handle.close();
+    }
+}
+/** A `<runId>.lock.<token>` that was written but never linked is a crash between publication
+ *  steps: there is no lock, only litter. Sweep the aged ones; a FRESH tmp belongs to a
+ *  process that is one `link()` away from owning the lock and must be left alone (R4-8). */
+async function sweepOrphanTemporaries(root, runId, options) {
+    const ttl = options.tmpTtlMs ?? RUN_LOCK_TMP_TTL_MS;
+    const nowMs = (options.now ?? Date.now)();
+    let names;
+    try {
+        names = await readdir(root);
+    }
+    catch {
+        return;
+    }
+    // BOTH publication tmps: the run lock's `<runId>.lock.<token>` and the break-lock's
+    // `<runId>.lock-break.<token>`. The break-lock is published by the same protocol, so it
+    // leaves the same litter and needs the same sweep.
+    const prefixes = [`${runId}.lock.`, `${runId}.lock-break.`];
+    for (const name of names) {
+        if (!prefixes.some((prefix) => name.startsWith(prefix)))
+            continue;
+        const path = join(root, name);
+        try {
+            const info = await stat(path);
+            if (nowMs - info.mtimeMs < ttl)
+                continue;
+            await unlink(path);
+        }
+        catch { /* best-effort: never block an acquisition on the sweep */ }
+    }
+}
+/**
+ * Break a lock whose owner is provably dead.
+ *
+ * Runs under its own tiny lock so two processes cannot race the sequence, and the removal is
+ * conditional on the INODE that was actually judged (R3-2): without that re-check, breaker A
+ * can read a dead owner's record, be descheduled while B breaks the lock and C acquires it,
+ * and then unlink C's live lock.
+ *
+ * There is no age-based reclaim anywhere here (R4-2). A breaker legitimately waiting out a
+ * slow identity probe would be evicted by one, and that eviction then races the breaker it
+ * evicted through the very sequence the break-lock exists to serialise.
+ *
+ * @returns true when the caller should retry the acquire immediately.
+ */
+async function breakStaleLock(root, runId, selfStartTime, identity, options) {
+    const path = lockPath(root, runId);
+    const breakPath = breakLockPath(root, runId);
+    const breakToken = randomUUID();
+    const breakRecord = { pid: process.pid, startTime: selfStartTime, token: breakToken, at: new Date().toISOString() };
+    if (!await publishRecord(breakPath, breakRecord)) {
+        // Someone else is mid-break. Reclaim only a provably dead breaker — or, if the record
+        // carries no identity at all, only an aged one (BREAK_LOCK_OPAQUE_TTL_MS).
+        const owner = await readRecord(breakPath);
+        if (owner === undefined)
+            await reclaimOpaqueBreakLock(breakPath, options);
+        else if (await identity(owner.pid, owner.startTime) === "dead")
+            await unlinkOwn(breakPath, owner.token, options);
+        return false;
+    }
+    try {
+        // The descriptor stays OPEN from the read through the comparison. Closing it first lets the
+        // inode be recycled underneath the check, which makes the inode comparison meaningless.
+        let handle;
+        try {
+            handle = await open(path, "r");
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return true; // released already
+            throw error;
+        }
+        try {
+            const inode = (await handle.stat()).ino;
+            const record = await readHandleRecord(handle);
+            // Unparseable, or missing an identity: treat as LIVE. Never guess — an unreadable
+            // identity is "unknown", and nothing reclaims on "unknown".
+            if (record === undefined)
+                return false;
+            if (await identity(record.pid, record.startTime) !== "dead")
+                return false;
+            if (options.beforeBreakUnlink)
+                await options.beforeBreakUnlink();
+            let current;
+            try {
+                current = await stat(path);
+            }
+            catch (error) {
+                if (isNotFound(error))
+                    return true;
+                throw error;
+            }
+            // Someone else already broke it and a NEW lock is published. Do NOT unlink.
+            if (current.ino !== inode)
+                return true;
+            // Inode AND token, immediately before the removal, off the descriptor we judged.
+            const still = await readHandleRecord(handle);
+            if (still === undefined || still.token !== record.token)
+                return false;
+            try {
+                await unlink(path);
+            }
+            catch (error) {
+                if (!isNotFound(error))
+                    throw error;
+            }
+            return true;
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    finally {
+        await unlinkOwn(breakPath, breakToken, options);
+    }
+}
+/**
+ * Acquire the cross-process lock for one run record.
+ *
+ * Publication is a single atomic step: the complete owner record is written to a private tmp
+ * name and then `link()`ed into place, so the lock file never exists half-written and a crash
+ * between the two leaves an orphan tmp and NO lock (R3-2).
+ */
+export async function acquireRunLock(root, runId, options = {}) {
+    assertRunId(runId);
+    const identity = options.identity ?? processIdentity;
+    const startTime = await (options.selfStartTime ?? (() => procStartTime(process.pid)))();
+    if (startTime === undefined) {
+        throw Object.assign(new Error("cannot establish process identity; run locking is unavailable"), { code: "RUN_LOCK_IDENTITY_UNAVAILABLE" });
+    }
+    const path = lockPath(root, runId);
+    const nowMs = options.now ?? Date.now;
+    // A CALLER-supplied budget gets the same validation as the env one (F4). `nowMs() >= NaN` is
+    // false forever, so a mistyped timeout did not shorten the wait, it removed the deadline: the
+    // acquire loop retried against a lock somebody else holds and never raised RUN_LOCK_TIMEOUT.
+    const deadline = nowMs() + requireDurationMs(options.timeoutMs ?? runLockTimeoutMs(), "timeoutMs");
+    await mkdir(root, { recursive: true });
+    for (;;) {
+        await sweepOrphanTemporaries(root, runId, options);
+        const token = randomUUID();
+        const record = { pid: process.pid, startTime, token, at: new Date().toISOString() };
+        if (await publishRecord(path, record))
+            return () => releaseRunLock(path, token, options);
+        if (!await breakStaleLock(root, runId, startTime, identity, options)) {
+            if (nowMs() >= deadline) {
+                const held = await readRecord(path);
+                throw Object.assign(new Error(`run ${runId} lock is held by pid ${held?.pid ?? "?"}`), { code: "RUN_LOCK_TIMEOUT", ...(held?.pid !== undefined ? { holderPid: held.pid } : {}) });
+            }
+            await delay(10 + Math.random() * 40);
+        }
+    }
+}
+/** Release unlinks only its OWN lock. A mismatched token means our lock was broken and the
+ *  run may already have been re-acquired: unlinking there would delete a live holder's lock,
+ *  which is the one way this scheme produces two concurrent writers. */
+async function releaseRunLock(path, token, options = {}) {
+    const record = await readRecord(path);
+    if (record === undefined)
+        return;
+    if (record.token !== token) {
+        process.stderr.write(`stratum: run lock ${path} is no longer ours (held by pid ${record.pid}); release is a no-op\n`);
+        return;
+    }
+    // Not a bare unlink: the same token+inode CAS the break-lock reclaim uses. The token read
+    // above is already stale by the time we act on it.
+    await unlinkOwn(path, token, options);
+}
+export async function readRunLock(root, runId) {
+    return readRecord(lockPath(root, runId));
+}
+/**
+ * The driver lease: a durable declaration that ONE live process holds this run's in-memory
+ * object (R4-4). Written on the 0 -> 1 pin and unlinked on the return to 0.
+ *
+ * Written SYNCHRONOUSLY because `retainRun` is synchronous by design — `scheduleFanout` pins
+ * with the scheduler's own run object and must not yield between the pin and the launch.
+ *
+ * The claim is EXCLUSIVE. It used to be a `rename()`, which silently overwrote whatever was
+ * there — so a second driver could publish its lease over a live one and the run would have two
+ * processes each believing they owned the in-memory copy, which is the single condition the
+ * lease exists to make impossible. `link()` fails with EEXIST instead, and the caller must
+ * resolve the incumbent by IDENTITY (dead -> reclaim, alive or unknown -> refuse) before
+ * retrying. The one case resolved here is an incumbent that is literally us: the same pid AND
+ * the same start time is our own leftover lease, and reclaiming it needs no probe.
+ *
+ * IDENTITY IS NOT OWNERSHIP (F1). The reclaim used to fire on "same pid AND same start time",
+ * which is true of every engine instance inside ONE process — so a second engine sharing a state
+ * root read a live sibling's lease as its own leftover, deleted it, and claimed the run the
+ * sibling was actively driving. The only lease this call may reclaim is one carrying a token
+ * THIS engine instance recorded for THIS run (`ownToken`); a same-identity lease with any other
+ * token is HELD, and the caller resolves it by liveness through `prepareLease`.
+ *
+ * @param ownToken the token this caller last received for `runId`, if it still holds one.
+ * @throws DRIVER_LEASE_HELD when a lease that is not ours already exists.
+ */
+export function writeDriverLeaseSync(root, runId, startTime, ownToken) {
+    const path = driverLeasePath(root, runId);
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        const token = randomUUID();
+        const lease = { pid: process.pid, startTime, token, at: new Date().toISOString() };
+        const temporary = `${path}.${process.pid}.${token}.tmp`;
+        writeFileSync(temporary, JSON.stringify(lease), { encoding: "utf8", mode: 0o600 });
+        let claimed = false;
+        try {
+            linkSync(temporary, path);
+            claimed = true;
+        }
+        catch (error) {
+            if (!isExists(error)) {
+                try {
+                    unlinkSync(temporary);
+                }
+                catch { /* best effort */ }
+                throw error;
+            }
+        }
+        try {
+            unlinkSync(temporary);
+        }
+        catch { /* best effort */ }
+        if (claimed)
+            return token;
+        const held = readDriverLeaseSync(root, runId);
+        // OUR OWN acquisition, by token — never merely our own pid.
+        if (held !== undefined && ownToken !== undefined && held.token === ownToken) {
+            releaseDriverLeaseSync(root, runId, held.token);
+            continue;
+        }
+        throw Object.assign(new Error(`run ${runId} driver lease is held${held !== undefined ? ` by pid ${held.pid}` : ""}`), { code: "DRIVER_LEASE_HELD", ...(held?.pid !== undefined ? { holderPid: held.pid } : {}) });
+    }
+    throw Object.assign(new Error(`run ${runId} driver lease could not be claimed`), { code: "DRIVER_LEASE_HELD" });
+}
+export function readDriverLeaseSync(root, runId) {
+    const path = driverLeasePath(root, runId);
+    let text;
+    // ENOENT is "no lease"; every other failure is "we could not tell", and reporting that as
+    // "no lease" is a licence to publish over a live one (F4).
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch (error) {
+        if (isNotFound(error))
+            return undefined;
+        throw error;
+    }
+    const record = parseRecord(text);
+    return record;
+}
+/** Token-checked exactly as the run lock's release is: never unlink a successor's lease.
+ *
+ *  ENOENT is the only swallowed failure. A blanket catch here turns EACCES or EIO into a lease
+ *  that is never removed, which wedges the run for every other process — and reports success
+ *  while doing it. */
+export function releaseDriverLeaseSync(root, runId, token) {
+    unlinkOwnSync(driverLeasePath(root, runId), token);
+}
+/** The synchronous twin of `unlinkOwn`: token AND inode, both re-checked off the descriptor that
+ *  was judged, immediately before the removal (F1). The read-then-unlink it replaces could delete
+ *  a SUCCESSOR's lease — our lease is reclaimed by a breaker, a new driver publishes, and our
+ *  late release unlinks theirs. `openSync` keeps the judged inode pinned across the comparison.
+ *  ENOENT is the only swallowed failure; anything else propagates (F4). */
+function unlinkOwnSync(path, token) {
+    let fd;
+    try {
+        fd = openSync(path, "r");
+    }
+    catch (error) {
+        if (isNotFound(error))
+            return;
+        throw error;
+    }
+    try {
+        const before = fstatSync(fd);
+        if (before.size === 0 || before.size > 64 * 1024)
+            return;
+        const buffer = Buffer.allocUnsafe(before.size);
+        readSync(fd, buffer, 0, before.size, 0);
+        if (parseRecord(buffer.toString("utf8"))?.token !== token)
+            return;
+        let current;
+        try {
+            current = statSync(path);
+        }
+        catch (error) {
+            if (isNotFound(error))
+                return;
+            throw error;
+        }
+        if (current.ino !== before.ino)
+            return; // the path names a successor now
+        const after = fstatSync(fd); // same fd, so the same inode
+        const recheck = Buffer.allocUnsafe(after.size);
+        readSync(fd, recheck, 0, after.size, 0);
+        if (parseRecord(recheck.toString("utf8"))?.token !== token)
+            return;
+        try {
+            unlinkSync(path);
+        }
+        catch (error) {
+            if (!isNotFound(error))
+                throw error;
+        }
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/* An UNCONDITIONAL lease removal used to live here. It is gone on purpose (F1): every reclaim
+ * now names the token it judged, so a slow reclaimer can no longer delete the lease of the
+ * successor that replaced the one it read. */
+/**
+ * Load-modify-save one run record under the real cross-process lock.
+ *
+ * `stratum learn egress` used to build its own `Map`-of-promises lock, which is local to one
+ * CLI invocation and therefore excludes nothing — not another `stratum learn`, and certainly
+ * not a running engine (R3-3c).
+ */
+export async function lockedSave(store, runId, mutate, options = {}) {
+    const release = await acquireRunLock(store.root, runId, options);
+    try {
+        // The run lock alone is NOT enough to write a run record (F3). A pinned run's authority is
+        // the DRIVER'S in-memory copy, not the file: a CLI that takes the lock, loads from disk and
+        // saves is writing a snapshot the driver has already moved past — which is how an egress
+        // receipt marked `sent` comes back as `pending`. And a cancelled run is written exactly
+        // once (invariant 4b), by the settle and nothing else.
+        await assertNoForeignDriverLease(store.root, runId, options);
+        const run = await store.load(runId);
+        assertNotCancelled(run, runId);
+        const result = await mutate(run);
+        await store.save(run);
+        return result;
+    }
+    finally {
+        await release();
+    }
+}
+/**
+ * The READ half of `lockedSave`, and a separate function on purpose (F3): a snapshot taken
+ * through the mutating path re-saves the record it only meant to read, so a pure read
+ * rewrote the file, bumped its mtime, and could revert a concurrent driver's writes.
+ *
+ * A read is refused by neither the lease nor the cancelled guard — observing a run another
+ * process drives is always safe. It takes the lock so the snapshot is not torn.
+ */
+export async function lockedRead(store, runId, read, options = {}) {
+    const release = await acquireRunLock(store.root, runId, options);
+    try {
+        return await read(await store.load(runId));
+    }
+    finally {
+        await release();
+    }
+}
+function assertNotCancelled(run, runId) {
+    if (run.status !== "cancelled" && run.cancelRequested !== true)
+        return;
+    throw Object.assign(new Error(`run ${runId} is cancelled; no further record updates are accepted`), { code: "PERSIST_ON_CANCELLED_RUN" });
+}
+/** A lease naming a LIVE process means that process owns this run's in-memory copy: refuse.
+ *  A provably dead owner's lease is reclaimed, because its memory is gone and disk is truth
+ *  again. `unknown` is never reclaimed — the same rule the engine's own pin path follows. */
+async function assertNoForeignDriverLease(root, runId, options) {
+    const lease = readDriverLeaseSync(root, runId);
+    if (lease === undefined)
+        return;
+    const identity = options.identity ?? processIdentity;
+    if (await identity(lease.pid, lease.startTime) === "dead") {
+        releaseDriverLeaseSync(root, runId, lease.token);
+        return;
+    }
+    throw Object.assign(new Error(`run ${runId} is driven by pid ${lease.pid}; it cannot be written from this process`), { code: "DRIVER_LEASE_HELD", holderPid: lease.pid });
+}
+//# sourceMappingURL=run_lock.js.map

package/dist/engine/run_lock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run_lock.js","sourceRoot":"","sources":["../../src/engine/run_lock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChI,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAmB,MAAM,kBAAkB,CAAC;AAC7F,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAsC,MAAM,YAAY,CAAC;AAE7E;;;;6DAI6D;AAC7D,MAAM,CAAC,MAAM,2BAA2B,GAAG,OAAO,CAAC;AACnD,uFAAuF;AACvF,MAAM,CAAC,MAAM,2BAA2B,GAAG,OAAO,CAAC;AACnD;;4EAE4E;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAEhD;8FAC8F;AAC9F,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAEnC;;;;;;;yFAOyF;AACzF,MAAM,wBAAwB,GAAG,MAAM,CAAC;AAExC;;;yDAGyD;AACzD,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,IAAY;IAC3D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,sCAAsC,CAAC,CAAC;IACzG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,2BAA2B,CAAC,EAAE,6BAA6B,CAAC,CAAC;AAC1I,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,2BAA2B,CAAC,EAAE,6BAA6B,CAAC,CAAC;AAC1I,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,yBAAyB,CAAC,EAAE,2BAA2B,CAAC,CAAC;AACpI,CAAC;AAuCD,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC3C,WAAW,CAAC,KAAK,CAAC,CAAC;IACnB,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa;IAChD,WAAW,CAAC,KAAK,CAAC,CAAC;IACnB,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,aAAa,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,eAAe,CAAC,IAAY,EAAE,KAAa;IAClD,WAAW,CAAC,KAAK,CAAC,CAAC;IACnB,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,SAAS,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAQ,KAA2C,EAAE,IAAI,CAAC;AAC5D,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC;AACvC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC;AACvC,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,SAAS,CAAC;IAAC,CAAC;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,MAAM,GAAG,KAA4B,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS;QAAE,OAAO,SAAS,CAAC;IACrH,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IACvD,OAAO,MAAoB,CAAC;AAC9B,CAAC;AAED;;gDAEgD;AAChD,KAAK,UAAU,gBAAgB,CAAC,MAAkB;IAChD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI;QAAE,OAAO,SAAS,CAAC;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3C,OAAO,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED;;uFAEuF;AACvF,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,MAAkB,CAAC;IACvB,IAAI,CAAC;QAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAAC,CAAC;IACvC,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IACvE,IAAI,CAAC;QAAC,OAAO,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAAC,CAAC;YAAS,CAAC;QAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAAC,CAAC;AAClF,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,MAAkB;IAC3D,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5C,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpF,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC5B,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAAC,MAAM,KAAK,CAAC;QAAC,CAAC;IACxF,CAAC;IACD,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,KAAa,EAAE,UAA0B,EAAE;IAChF,IAAI,MAAkB,CAAC;IACvB,6FAA6F;IAC7F,0FAA0F;IAC1F,yFAAyF;IACzF,2CAA2C;IAC3C,IAAI,CAAC;QAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAAC,CAAC;IACvC,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QACjE,MAAM,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC;QACxC,IAAI,OAAO,CAAC,eAAe;YAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7D,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAAC,MAAM,KAAK,CAAC;QAAC,CAAC;QACvG,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC,CAAc,iCAAiC;QACvF,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAU,6BAA6B;QACpF,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/D,IAAI,CAAC;YAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;QAAC,CAAC;QAClF,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAAC,CAAC;AACrC,CAAC;AAED;;;mDAGmD;AACnD,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,OAAuB;IACzE,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,CAAC;IACzD,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC1C,IAAI,MAAkB,CAAC;IACvB,IAAI,CAAC;QAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;IAC/D,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC,CAAG,0CAA0C;QAC5G,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,GAAG,GAAG;YAAE,OAAO,KAAK,CAAC,CAAqB,mCAAmC;QACrG,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAAC,MAAM,KAAK,CAAC;QAAC,CAAC;QACvG,IAAI,OAAO,CAAC,GAAG,KAAK,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QAC3C,IAAI,CAAC;YAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;QAAC,CAAC;QAClF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;YAAS,CAAC;QAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAAC,CAAC;AAC7D,CAAC;AAED;;4FAE4F;AAC5F,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,KAAa,EAAE,OAAuB;IACxF,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,mBAAmB,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC1C,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QAAC,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO;IAAC,CAAC;IACtD,oFAAoF;IACpF,wFAAwF;IACxF,mDAAmD;IACnD,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,QAAQ,EAAE,GAAG,KAAK,cAAc,CAAC,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAAE,SAAS;QAClE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,GAAG,GAAG;gBAAE,SAAS;YACzC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC,CAAC,0DAA0D,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAC3B,IAAY,EACZ,KAAa,EACb,aAAqB,EACrB,QAA+B,EAC/B,OAAuB;IAEvB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC7C,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC;IAChC,MAAM,WAAW,GAAe,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IAChI,IAAI,CAAC,MAAM,aAAa,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;QACjD,sFAAsF;QACtF,2EAA2E;QAC3E,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,sBAAsB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;aACrE,IAAI,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,MAAM;YAAE,MAAM,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACjH,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC;QACH,4FAA4F;QAC5F,wFAAwF;QACxF,IAAI,MAAkB,CAAC;QACvB,IAAI,CAAC;YAAC,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAAC,CAAC;QACvC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC,CAAG,mBAAmB;YACzD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC;YACxC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC9C,kFAAkF;YAClF,4DAA4D;YAC5D,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAC;YACvC,IAAI,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC1E,IAAI,OAAO,CAAC,iBAAiB;gBAAE,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACjE,IAAI,OAAO,CAAC;YACZ,IAAI,CAAC;gBAAC,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YAAC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,IAAI,UAAU,CAAC,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC;gBAAC,MAAM,KAAK,CAAC;YAAC,CAAC;YACtG,4EAA4E;YAC5E,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK;gBAAE,OAAO,IAAI,CAAC;YACvC,iFAAiF;YACjF,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;YACtE,IAAI,CAAC;gBAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YAAC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;oBAAE,MAAM,KAAK,CAAC;YAAC,CAAC;YAClF,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;IACrC,CAAC;YAAS,CAAC;QACT,MAAM,SAAS,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,KAAa,EAAE,UAA0B,EAAE;IAC5F,WAAW,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,eAAe,CAAC;IACrD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,+DAA+D,CAAC,EAC5F,EAAE,IAAI,EAAE,+BAA+B,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACtC,6FAA6F;IAC7F,8FAA8F;IAC9F,6FAA6F;IAC7F,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,SAAS,IAAI,gBAAgB,EAAE,EAAE,WAAW,CAAC,CAAC;IACnG,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,SAAS,CAAC;QACR,MAAM,sBAAsB,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAe,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAChG,IAAI,MAAM,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC;YAAE,OAAO,GAAG,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;YACrE,IAAI,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;gBACpC,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,wBAAwB,IAAI,EAAE,GAAG,IAAI,GAAG,EAAE,CAAC,EACnF,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC/F,CAAC;YACD,MAAM,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;wEAEwE;AACxE,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,KAAa,EAAE,UAA0B,EAAE;IACrF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO;IACjC,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,IAAI,mCAAmC,MAAM,CAAC,GAAG,yBAAyB,CAAC,CAAC;QACtH,OAAO;IACT,CAAC;IACD,0FAA0F;IAC1F,mDAAmD;IACnD,MAAM,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,KAAa;IAC3D,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,KAAa,EAAE,SAAiB,EAAE,QAAiB;IACpG,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAgB,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAChG,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,KAAK,MAAM,CAAC;QACxD,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACnF,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC;YAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAAC,OAAO,GAAG,IAAI,CAAC;QAAC,CAAC;QAClD,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAAC,IAAI,CAAC;oBAAC,UAAU,CAAC,SAAS,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAAC,MAAM,KAAK,CAAC;YAAC,CAAC;QACnG,CAAC;QACD,IAAI,CAAC;YAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAC1D,IAAI,OAAO;YAAE,OAAO,KAAK,CAAC;QAC1B,MAAM,IAAI,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9C,4DAA4D;QAC5D,IAAI,IAAI,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5E,sBAAsB,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAChD,SAAS;QACX,CAAC;QACD,MAAM,MAAM,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,OAAO,KAAK,wBAAwB,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAChG,EAAE,IAAI,EAAE,mBAAmB,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAC3F,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,oCAAoC,CAAC,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC,CAAC;AAClH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,KAAa;IAC7D,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,IAAI,IAAY,CAAC;IACjB,0FAA0F;IAC1F,2DAA2D;IAC3D,IAAI,CAAC;QAAC,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,CAAC;IAC1C,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IACvE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,MAAiC,CAAC;AAC3C,CAAC;AAED;;;;sBAIsB;AACtB,MAAM,UAAU,sBAAsB,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa;IAC/E,aAAa,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;AACrD,CAAC;AAED;;;;2EAI2E;AAC3E,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa;IAChD,IAAI,EAAU,CAAC;IACf,IAAI,CAAC;QAAC,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAAC,CAAC;IACjC,OAAO,KAAK,EAAE,CAAC;QAAC,IAAI,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI;YAAE,OAAO;QACzD,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxC,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,KAAK,KAAK;YAAE,OAAO;QAClE,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YAAC,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO;YAAC,MAAM,KAAK,CAAC;QAAC,CAAC;QAC/F,IAAI,OAAO,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG;YAAE,OAAO,CAAiB,iCAAiC;QACzF,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAA4B,6BAA6B;QACrF,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/C,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxC,IAAI,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,KAAK,KAAK;YAAE,OAAO;QACnE,IAAI,CAAC;YAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;QAAC,CAAC;IAClF,CAAC;YAAS,CAAC;QAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAAC,CAAC;AAC9B,CAAC;AAED;;8CAE8C;AAE9C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAiD,EACjD,KAAa,EACb,MAA6C,EAC7C,UAA0B,EAAE;IAE5B,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,2FAA2F;QAC3F,4FAA4F;QAC5F,yFAAyF;QACzF,wFAAwF;QACxF,uDAAuD;QACvD,MAAM,0BAA0B,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,OAAO,MAAM,CAAC;IAChB,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAwC,EACxC,KAAa,EACb,IAA2C,EAC3C,UAA0B,EAAE;IAE5B,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,OAAO,MAAM,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7C,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAiB,EAAE,KAAa;IAC1D,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,GAAG,CAAC,eAAe,KAAK,IAAI;QAAE,OAAO;IACvE,MAAM,MAAM,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,OAAO,KAAK,uDAAuD,CAAC,EAC9E,EAAE,IAAI,EAAE,0BAA0B,EAAE,CACrC,CAAC;AACJ,CAAC;AAED;;6FAE6F;AAC7F,KAAK,UAAU,0BAA0B,CAAC,IAAY,EAAE,KAAa,EAAE,OAAuB;IAC5F,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,eAAe,CAAC;IACrD,IAAI,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,MAAM,EAAE,CAAC;QAC1D,sBAAsB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACjD,OAAO;IACT,CAAC;IACD,MAAM,MAAM,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,OAAO,KAAK,qBAAqB,KAAK,CAAC,GAAG,0CAA0C,CAAC,EAC/F,EAAE,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,CACpD,CAAC;AACJ,CAAC"}