@agent-flow-board/bridge 0.2.0 → 0.3.0

package/dist/checks/scheduler.js

@@ -0,0 +1,383 @@
+// THE WORKER'S SCHEDULING LOGIC, PURE: EVERYTHING IT TOUCHES (THE BOARD CONNECTIONS, THE RUNNER, THE MACHINE BROKER, THE SLEEP) IS INJECTED, SO IT IS TESTED WITH FAKES.
+// What it decides: how many runs to take (never more than the slots plus the runs it may prepare ahead), to leave a run UNCLAIMED on the board while the machine is
+// overloaded, to claim as late as possible (right before the work starts), to beat the board from the moment of the claim through every waiting phase, to let the next
+// run install and build while the current one runs its check (two stages per run, `prepare` and `execute`), to let registrations take turns, to stop a run that
+// outlives its limit, and to give back only its own held runs when it stops. The board stops a claimed run that does not beat for 10 minutes (BEAT_TIMEOUT_MINUTES in
+// the board's checks.ts), so the first beat goes at claim time, not when the work starts, and the beat never depends on the stage.
+// -- CONSTANTS -- //
+const num = (name, fallback) => {
+    const v = Number(process.env[name]);
+    return Number.isFinite(v) && v > 0 ? v : fallback;
+};
+/** Idle poll, 5 s. AFB_CHECKS_POLL_MS overrides (the tests). */
+export const pollMs = () => num('AFB_CHECKS_POLL_MS', 5_000);
+/** Longest backoff after errors, 60 s. */
+export const backoffMaxMs = () => num('AFB_CHECKS_BACKOFF_MAX_MS', 60_000);
+/** A beat every 30 s. The board reports a run stuck after 10 minutes without one. */
+export const beatMs = () => num('AFB_CHECKS_BEAT_MS', 30_000);
+/** The longest one run may take, builds and waiting included: 40 minutes. */
+export const runLimitMs = () => num('AFB_CHECKS_RUN_LIMIT_MS', 40 * 60_000);
+// -- FUNCTIONS -- //
+const defaultSleep = (ms, signal) => new Promise((resolve) => {
+    const t = setTimeout(done, ms);
+    function done() {
+        clearTimeout(t);
+        signal.removeEventListener('abort', done);
+        resolve();
+    }
+    signal.addEventListener('abort', done, { once: true });
+});
+/** Waits for a promise to settle, but not longer than ms. The timer is cleared the moment it settles, so it never keeps the process alive. */
+const settleWithin = (p, ms) => new Promise((resolve) => {
+    const t = setTimeout(resolve, ms);
+    p.then(() => clearTimeout(t), () => clearTimeout(t)).finally(resolve);
+});
+const message = (e) => (e instanceof Error ? e.message : String(e));
+const toError = (e) => (e instanceof Error ? e : new Error(String(e)));
+function withTimeout(p, ms, what) {
+    return new Promise((resolve, reject) => {
+        const t = setTimeout(() => reject(new Error(`${what} did not answer within ${Math.round(ms / 1000)} seconds.`)), ms);
+        p.then((v) => {
+            clearTimeout(t);
+            resolve(v);
+        }, (e) => {
+            clearTimeout(t);
+            reject(toError(e));
+        });
+    });
+}
+/** A gate of `limit` places. When one frees, the waiter whose key holds the fewest places goes first (ties by arrival): registrations take turns. */
+export class FairGate {
+    limit;
+    holders = new Map();
+    waiters = [];
+    seq = 0;
+    inside = 0;
+    constructor(limit) {
+        this.limit = limit;
+    }
+    acquire(key, signal) {
+        if (signal?.aborted)
+            return Promise.reject(Object.assign(new Error('Stopped.'), { name: 'AbortError' }));
+        return new Promise((resolve, reject) => {
+            const w = {
+                key,
+                seq: this.seq++,
+                go: () => {
+                    signal?.removeEventListener('abort', onAbort);
+                    this.inside++;
+                    this.holders.set(key, (this.holders.get(key) ?? 0) + 1);
+                    let freed = false;
+                    resolve(() => {
+                        if (freed)
+                            return;
+                        freed = true;
+                        this.inside--;
+                        this.holders.set(key, (this.holders.get(key) ?? 1) - 1);
+                        this.next();
+                    });
+                }
+            };
+            const onAbort = () => {
+                this.waiters = this.waiters.filter((x) => x !== w);
+                reject(Object.assign(new Error('Stopped.'), { name: 'AbortError' }));
+            };
+            signal?.addEventListener('abort', onAbort, { once: true });
+            this.waiters.push(w);
+            this.next();
+        });
+    }
+    next() {
+        while (this.inside < this.limit && this.waiters.length) {
+            this.waiters.sort((a, b) => (this.holders.get(a.key) ?? 0) - (this.holders.get(b.key) ?? 0) || a.seq - b.seq);
+            this.waiters.shift()?.go();
+        }
+    }
+}
+export function createScheduler(deps, o) {
+    const slots = Math.max(1, Math.floor(o.slots));
+    const prefetch = Math.max(0, Math.floor(o.prefetch ?? 1));
+    const poll = o.pollMs ?? pollMs();
+    const backoffMax = o.backoffMaxMs ?? backoffMaxMs();
+    const beatEvery = o.beatMs ?? beatMs();
+    const limitMs = o.runLimitMs ?? runLimitMs();
+    const callMs = o.callTimeoutMs ?? 25_000;
+    const stepWait = o.stepWaitMs ?? 3_000;
+    const grace = o.graceMs ?? 5_000;
+    const sleep = deps.sleep ?? defaultSleep;
+    const now = deps.now ?? (() => Date.now());
+    const isFatal = deps.fatal ?? ((e) => e?.code === 'UNAUTHENTICATED');
+    const isGone = deps.gone ?? ((e) => /already finished/i.test(message(e)));
+    const runs = new Map();
+    const prepareGate = new FairGate(slots);
+    const backoff = new Map();
+    const dead = new Map();
+    const stepping = new Set();
+    let reserved = 0;
+    let stopping = false;
+    let wakeUp = null;
+    let overloadSaid = null;
+    let offset = 0;
+    const wake = () => {
+        const w = wakeUp;
+        wakeUp = null;
+        w?.();
+    };
+    const nap = (ms, signal) => Promise.race([
+        sleep(ms, signal),
+        new Promise((resolve) => {
+            wakeUp = resolve;
+        })
+    ]);
+    const count = (stage) => [...runs.values()].filter((r) => r.stage === stage).length;
+    const status = () => ({
+        slots,
+        claimed: runs.size,
+        preparing: count('prepare') + count('prepare-wait'),
+        waitingForSlot: count('slot-wait'),
+        executing: count('execute')
+    });
+    /** Whether one more run may be claimed right now. Checked right before the claim, so a run the machine cannot start stays queued on the board. */
+    function capacity() {
+        if (stopping)
+            return false;
+        const ready = deps.broker.ready();
+        if (!ready.ok) {
+            if (overloadSaid !== ready.code)
+                deps.log(ready.why);
+            overloadSaid = ready.code;
+            return false;
+        }
+        overloadSaid = null;
+        const pending = count('prepare-wait') + count('prepare') + count('slot-wait');
+        const load = deps.broker.totals().busy + pending + reserved;
+        // A run that waits for a slot is not in the machine total yet, so it is counted once, as pending
+        return load < slots + prefetch;
+    }
+    function startRun(conn, job) {
+        const ac = new AbortController();
+        const rec = { job, conn, ac, stage: 'prepare-wait', reason: null, reported: false };
+        runs.set(job.id, rec);
+        const ctx = {
+            runId: job.id,
+            signal: ac.signal,
+            log: (l) => deps.log(l),
+            reason: () => rec.reason
+        };
+        const stop = (why) => {
+            if (rec.reason)
+                return;
+            rec.reason = why;
+            // The reason travels on the signal, so a run knows whether it was stopped, timed out or given up on
+            ac.abort(why);
+        };
+        rec.done = (async () => {
+            // The beat starts here, at the claim, and runs until the run is over: waiting for a turn, a slot or the disk never silences it
+            let told = false;
+            const beat = () => void Promise.resolve()
+                .then(() => conn.beat(job.id))
+                .catch((e) => {
+                if (isGone(e)) {
+                    if (!told)
+                        deps.log(`The board gave up on this run: ${message(e)}`);
+                    told = true;
+                    stop('gone');
+                }
+            });
+            beat();
+            const beatTimer = setInterval(beat, beatEvery);
+            const limitTimer = setTimeout(() => stop('timeout'), limitMs);
+            const aborted = new Promise((_, reject) => {
+                ac.signal.addEventListener('abort', () => reject(Object.assign(new Error('The run was stopped.'), { name: 'AbortError' })), { once: true });
+            });
+            aborted.catch(() => undefined);
+            const key = job.registration ?? conn.key;
+            const work = (async () => {
+                const free = await prepareGate.acquire(key, ac.signal);
+                let prepared;
+                try {
+                    rec.stage = 'prepare';
+                    prepared = await deps.runner.prepare(job, ctx);
+                }
+                finally {
+                    free();
+                }
+                rec.stage = 'slot-wait';
+                wake();
+                const slot = await deps.broker.acquire({ runId: job.id, kind: 'check', registration: key }, { signal: ac.signal });
+                try {
+                    rec.stage = 'execute';
+                    await deps.runner.execute(job, prepared, ctx);
+                }
+                finally {
+                    slot.release();
+                }
+            })();
+            work.catch(() => undefined);
+            try {
+                await Promise.race([work, aborted]);
+                rec.reported = true;
+            }
+            catch (e) {
+                if (rec.reason === 'timeout') {
+                    const mins = Math.max(1, Math.round(limitMs / 60_000));
+                    const why = limitMs < 60_000 ? `${Math.round(limitMs / 1000)} seconds` : `${mins} minutes`;
+                    await deps.runner
+                        .fail(job, new Error(`The run took longer than ${why} and was stopped.`), ctx)
+                        .catch(() => undefined);
+                }
+                else if (rec.reason === null) {
+                    deps.log(`could not run ${job.id}: ${message(e)}`);
+                    await deps.runner.fail(job, toError(e), ctx).catch(() => undefined);
+                }
+                rec.reported = true;
+            }
+            finally {
+                clearInterval(beatTimer);
+                clearTimeout(limitTimer);
+                // An abandoned stage gets a moment to end by itself before the run's resources are closed
+                if (rec.reason)
+                    await settleWithin(work, grace);
+                try {
+                    await deps.runner.cleanup?.(job, ctx);
+                }
+                catch (e) {
+                    deps.log(`could not clean up ${job.id}: ${message(e)}`);
+                }
+                runs.delete(job.id);
+                wake();
+            }
+        })();
+    }
+    /** One look at one board: ask for the count, then take as many as the machine can start. Returns how many were taken. */
+    async function step(conn) {
+        let taken = 0;
+        if (!capacity())
+            return { taken, blocked: true };
+        let waiting;
+        try {
+            waiting = await withTimeout(conn.peek(), callMs, `${conn.key}: the board`);
+            backoff.delete(conn.key);
+            while (waiting-- > 0) {
+                // Late and synchronous: the capacity is read again and a place is reserved before the claim goes out
+                if (!capacity())
+                    return { taken, blocked: true };
+                reserved++;
+                try {
+                    const job = await withTimeout(conn.claim(), callMs, `${conn.key}: the board`);
+                    if (!job)
+                        break;
+                    if (stopping) {
+                        await conn.release([job.id]).catch(() => undefined);
+                        break;
+                    }
+                    startRun(conn, job);
+                    taken++;
+                }
+                finally {
+                    reserved--;
+                }
+            }
+        }
+        catch (e) {
+            if (isFatal(e)) {
+                dead.set(conn.key, conn.fingerprint ?? '');
+                deps.log(`${conn.key}: ${message(e)}`);
+            }
+            else {
+                const prev = backoff.get(conn.key)?.wait ?? poll;
+                const wait = Math.min(backoffMax, Math.max(prev * 2, poll * 2));
+                backoff.set(conn.key, { wait, until: now() + wait });
+                deps.log(`${conn.key}: ${message(e)}`);
+            }
+        }
+        return { taken, blocked: false };
+    }
+    async function pass() {
+        // The connections are read again on every pass, so a board connected or forgotten meanwhile is picked up without a restart
+        const conns = deps.connections();
+        const keys = new Set(conns.map((c) => c.key));
+        for (const k of [...dead.keys()])
+            if (!keys.has(k))
+                dead.delete(k);
+        const live = conns.filter((c) => {
+            const d = dead.get(c.key);
+            if (d !== undefined && d === (c.fingerprint ?? ''))
+                return false;
+            if (d !== undefined)
+                dead.delete(c.key);
+            return !stepping.has(c.key) && (backoff.get(c.key)?.until ?? 0) <= now();
+        });
+        // The boards take turns at being asked first
+        const ordered = live.length
+            ? [...live.slice(offset % live.length), ...live.slice(0, offset % live.length)]
+            : [];
+        offset++;
+        let taken = 0;
+        let blocked = false;
+        const steps = ordered.map(async (c) => {
+            stepping.add(c.key);
+            try {
+                const r = await step(c);
+                taken += r.taken;
+                blocked ||= r.blocked;
+            }
+            finally {
+                stepping.delete(c.key);
+            }
+        });
+        // A board that answers slowly does not hold the others back
+        await settleWithin(Promise.all(steps), stepWait);
+        return { taken, blocked };
+    }
+    async function run({ signal, once = false }) {
+        stopping = false;
+        let total = 0;
+        const onAbort = () => {
+            stopping = true;
+            wake();
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
+        try {
+            while (!signal.aborted) {
+                const { taken, blocked } = await pass();
+                total += taken;
+                if (once) {
+                    if (!taken && !(blocked && runs.size))
+                        break;
+                    if (!taken)
+                        await nap(poll, signal);
+                    continue;
+                }
+                // Something was taken: look again at once. Otherwise the idle poll, cut short when a run ends and frees room
+                await nap(taken ? 0 : poll, signal);
+            }
+            if (once && !signal.aborted)
+                await Promise.all([...runs.values()].map((r) => r.done));
+        }
+        finally {
+            signal.removeEventListener('abort', onAbort);
+            stopping = signal.aborted;
+            if (signal.aborted) {
+                // Give back exactly the runs this scheduler holds and has not reported, grouped by the connection they were claimed on
+                const byConn = new Map();
+                for (const r of runs.values()) {
+                    if (r.reported)
+                        continue;
+                    r.reason ??= 'stopped';
+                    byConn.set(r.conn, [...(byConn.get(r.conn) ?? []), r.job.id]);
+                }
+                await Promise.all([...byConn].map(([c, ids]) => c.release(ids).catch(() => undefined)));
+                for (const r of runs.values())
+                    r.ac.abort('stopped');
+                await Promise.all([...runs.values()].map((r) => r.done));
+            }
+        }
+        return total;
+    }
+    return {
+        run,
+        status,
+        heldIds: () => [...runs.values()].filter((r) => !r.reported).map((r) => r.job.id)
+    };
+}

package/dist/checks/scope.js

@@ -1,21 +1,20 @@
 // THE THINGS ONE RUN OWNS, AND WHAT HAPPENS TO THEM WHEN THE RUN ENDS, WHATEVER ENDS IT.
-// The layer that makes the resources writes a live marker file first (runs/<run id>/ACTIVE: the process id, the start time and what the run made that a
-// crash would leave behind: the preview process, the database container and the private copy). Resources are closed newest first and closing never throws.
+// The layer that makes the resources writes a live marker file first (markers/<run id>.json, in a folder the sandboxed check cannot reach: the process id, the
+// start time and what the run made that a crash would leave behind: the preview process, the database container and the private copy). Resources are closed newest first and closing never throws.
 // A later run sweeps what a dead process left: only the exact run folders and copies named by a dead marker (or by no marker and old enough), never a
 // folder that was not made by a run.
 // -- DEP-MODULES -- //
 import { execFile } from 'node:child_process';
 import { randomBytes } from 'node:crypto';
-import { existsSync, lstatSync, readFileSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { existsSync, lstatSync, readFileSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 // -- MODULES -- //
 import { pidAlive } from './lock.js';
-import { NAME, copiesDir, ensureDir, runsDir, under } from './paths.js';
-import { removeContainer } from './provision.js';
+import { NAME, copiesDir, endedPath, ensureDir, logDirFor, logsDir, markerPath, markersDir, runsDir, under } from './paths.js';
+import { dropPooledDatabase, pooledNames } from './dbpool.js';
+import { renameRetry } from './fsretry.js';
+import { CONTAINER_PREFIX, removeContainer } from './provision.js';
 // -- CONSTANTS -- //
-const MARKER = 'ACTIVE';
-/** Written when a run ends in order, so a finished run's folder is never mistaken for one whose marker was lost. */
-const ENDED = 'ENDED';
 /** A run folder with no marker is only an orphan after this long (the marker is written first, so younger ones are being made right now). */
 const NO_MARKER_GRACE_MS = 15 * 60_000;
 const DAY_MS = 86_400_000;
@@ -27,19 +26,64 @@ const noop = () => { };
 function writeWhole(file, text) {
     const tmp = `${file}.${process.pid}.tmp`;
     writeFileSync(tmp, text, { mode: 0o600 });
-    renameSync(tmp, file);
+    renameRetry(tmp, file);
 }
-function readMarker(dir) {
+/**
+ * Reads the marker of a run and checks it against what the tool itself would have written for that run id. A field that names anything else (a database,
+ * a role, a container, a process that is older than the run) is dropped and a note says so, so a marker can never make a sweep touch something that is not the
+ * run's own. A marker that does not hold a process id and its own run id is not a marker (null).
+ */
+export function readMarker(runId) {
+    let v;
     try {
-        const v = JSON.parse(readFileSync(join(dir, MARKER), 'utf8'));
-        if (!v || typeof v !== 'object')
-            return null;
-        const m = v;
-        return typeof m.pid === 'number' && typeof m.runId === 'string' ? m : null;
+        v = JSON.parse(readFileSync(markerPath(runId), 'utf8'));
     }
     catch {
         return null;
     }
+    if (!v || typeof v !== 'object')
+        return null;
+    const m = v;
+    if (typeof m.pid !== 'number' || m.runId !== runId)
+        return null;
+    const startedAt = typeof m.startedAt === 'string' ? m.startedAt : '';
+    const notes = [];
+    const out = { runId, pid: m.pid, startedAt };
+    if (m.previewPid !== undefined) {
+        const spawned = Date.parse(String(m.previewStartedAt ?? ''));
+        if (Number.isInteger(m.previewPid) &&
+            m.previewPid > 1 &&
+            m.previewPid !== m.pid &&
+            Number.isFinite(spawned) &&
+            spawned >= Date.parse(startedAt) - 1000) {
+            out.previewPid = m.previewPid;
+            out.previewStartedAt = String(m.previewStartedAt);
+        }
+        else
+            notes.push(`The marker of run ${runId} names a preview process that does not check out, so it was ignored.`);
+    }
+    if (m.containerName !== undefined) {
+        if (m.containerName === `${CONTAINER_PREFIX}${runId}`)
+            out.containerName = m.containerName;
+        else
+            notes.push(`The marker of run ${runId} names a container that is not the one this run makes, so it was ignored.`);
+    }
+    if (m.dbName !== undefined || m.roleName !== undefined || m.poolId !== undefined) {
+        const names = pooledNames(runId);
+        if (m.dbName === names.dbName &&
+            m.roleName === names.roleName &&
+            typeof m.poolId === 'string' &&
+            NAME.test(m.poolId)) {
+            out.dbName = m.dbName;
+            out.roleName = m.roleName;
+            out.poolId = m.poolId;
+        }
+        else
+            notes.push(`The marker of run ${runId} names a database or role that is not this run's, so it was ignored.`);
+    }
+    if (typeof m.copyDir === 'string')
+        out.copyDir = m.copyDir;
+    return { marker: out, notes };
 }
 export class RunScope {
     runId;
@@ -63,6 +107,8 @@ export class RunScope {
         if (this.started)
             return;
         ensureDir(this.dir);
+        ensureDir(markersDir());
+        ensureDir(logDirFor(this.runId));
         this.started = true;
         this.writeMarker();
     }
@@ -73,16 +119,29 @@ export class RunScope {
             startedAt: new Date().toISOString(),
             ...this.info
         };
-        writeWhole(join(this.dir, MARKER), JSON.stringify(marker) + '\n');
+        writeWhole(markerPath(this.runId), JSON.stringify(marker) + '\n');
     }
     /** Records what a crash would leave behind (the preview process id, the database container, the private copy) so a later sweep can remove it. */
     setInfo(info) {
+        // After the scope closed nothing may rewrite the marker: it was removed with the run
+        if (this.closed)
+            return;
         this.start();
-        this.info = { ...this.info, ...info };
+        const stamped = info.previewPid !== undefined && info.previewStartedAt === undefined
+            ? { ...info, previewStartedAt: new Date().toISOString() }
+            : info;
+        this.info = { ...this.info, ...stamped };
         this.writeMarker();
     }
     /** Registers a resource. It is closed when the scope closes, newest first. */
     add(name, dispose) {
+        // A resource made after the scope closed is closed at once: nothing will come back to do it
+        if (this.closed) {
+            void Promise.resolve()
+                .then(dispose)
+                .catch(() => this.log(`Could not close ${name}.`));
+            return;
+        }
         this.start();
         this.disposers.push({ name, dispose });
     }
@@ -104,8 +163,9 @@ export class RunScope {
         }
         try {
             if (this.started) {
-                writeWhole(join(this.dir, ENDED), new Date().toISOString() + '\n');
-                rmSync(join(this.dir, MARKER), { force: true });
+                writeWhole(endedPath(this.runId), new Date().toISOString() + '\n');
+                rmSync(markerPath(this.runId), { force: true });
+                removeTree(logDirFor(this.runId));
             }
         }
         catch {
@@ -222,6 +282,9 @@ export function installSignalHandlers(scope, onKill) {
 function removeRun(id) {
     removeTree(under(copiesDir(), id));
     removeTree(under(runsDir(), id));
+    removeTree(logDirFor(id));
+    rmSync(markerPath(id), { force: true });
+    rmSync(endedPath(id), { force: true });
 }
 const ageOf = (path) => {
     try {
@@ -236,30 +299,48 @@ const plainIds = (dir) => {
         return [];
     return readdirSync(dir).filter((n) => NAME.test(n));
 };
+/** The run ids that have a folder, a marker or an end note. */
+export function allRunIds() {
+    const ids = new Set(plainIds(runsDir()));
+    if (existsSync(markersDir()))
+        for (const f of readdirSync(markersDir())) {
+            const m = /^(.+)\.(json|ended)$/.exec(f);
+            if (m && NAME.test(m[1]))
+                ids.add(m[1]);
+        }
+    return [...ids];
+}
 /**
  * Cleans what dead runs left: a run whose marker names a dead process, or that has neither a marker nor an end note and is older than 15 minutes.
  * Stops the recorded preview process tree and database container, then removes only those run folders and copies. Returns how many runs it cleaned.
  */
-export async function sweepOrphans(log = noop) {
+export async function sweepOrphans(log = noop, deps = {}) {
     let cleaned = 0;
     const seen = new Set();
-    for (const id of plainIds(runsDir())) {
+    for (const id of allRunIds()) {
         seen.add(id);
         const dir = under(runsDir(), id);
-        const marker = readMarker(dir);
+        const checked = readMarker(id);
+        const marker = checked?.marker ?? null;
+        for (const note of checked?.notes ?? [])
+            log(note);
         if (marker) {
             if (pidAlive(marker.pid))
                 continue;
         }
-        else if (existsSync(join(dir, MARKER)) ||
-            existsSync(join(dir, ENDED)) ||
-            ageOf(dir) < NO_MARKER_GRACE_MS) {
+        else if (existsSync(markerPath(id)) ||
+            existsSync(endedPath(id)) ||
+            (existsSync(dir) ? ageOf(dir) : Infinity) < NO_MARKER_GRACE_MS) {
             continue; // Finished in order (clearRuns handles it), or too young to judge
         }
         if (marker?.previewPid && pidAlive(marker.previewPid))
             await killTree(marker.previewPid);
-        if (marker?.containerName)
-            await removeContainer(marker.containerName);
+        if (marker?.dbName && marker.roleName && marker.poolId) {
+            // A pooled run: only its own database and roles go. The pool container is never removed here, other runs live in it
+            await dropPooledDatabase({ dbName: marker.dbName, roleName: marker.roleName, poolId: marker.poolId }, { docker: deps.docker });
+        }
+        else if (marker?.containerName)
+            await removeContainer(marker.containerName, deps.docker);
         removeRun(id);
         cleaned++;
         log(`Cleaned the leftovers of run ${id}.`);
@@ -272,6 +353,14 @@ export async function sweepOrphans(log = noop) {
         cleaned++;
         log(`Removed the orphaned copy of run ${id}.`);
     }
+    // The logs of a run that has no folder, marker or copy any more: old enough that no run is still writing them
+    for (const id of plainIds(logsDir())) {
+        if (seen.has(id) || existsSync(markerPath(id)) || ageOf(logDirFor(id)) < NO_MARKER_GRACE_MS)
+            continue;
+        removeTree(logDirFor(id));
+        cleaned++;
+        log(`Removed the orphaned logs of run ${id}.`);
+    }
     return cleaned;
 }
 /**
@@ -281,11 +370,11 @@ export async function sweepOrphans(log = noop) {
 export function clearRuns(opts = {}) {
     const limit = (opts.olderThanDays ?? 7) * DAY_MS;
     let removed = 0;
-    for (const id of plainIds(runsDir())) {
+    for (const id of allRunIds()) {
         const dir = under(runsDir(), id);
-        if (existsSync(join(dir, MARKER)))
+        if (existsSync(markerPath(id)))
             continue;
-        if (!opts.all && ageOf(dir) < limit)
+        if (!opts.all && (existsSync(dir) ? ageOf(dir) : Infinity) < limit)
             continue;
         removeRun(id);
         removed++;

package/dist/checks/sealedNames.js

@@ -0,0 +1,39 @@
+// THE NAMES A SEALED VALUE MAY HAVE AND MAY BE GIVEN TO. PURE CONSTANTS AND PREDICATES (NO FILES, NO KEYS), SO THE RECIPE VALIDATOR, THE SEALED STORE AND THE
+// ENV RULES SHARE ONE SET. Ported verbatim from the old tool's lib/secrets.mjs.
+// -- CONSTANTS -- //
+/** The setting names a sealed value may be given to (and the names that may be imported): Supabase's own and the app's auth secret. NODE_OPTIONS, PATH and the like can never be one. */
+export const SUPABASE_KEY = /^((NEXT_PUBLIC_|PUBLIC_|VITE_)?SUPABASE_(URL|ANON_KEY|PUBLISHABLE_KEY)|SUPABASE_SERVICE_ROLE_KEY|SUPABASE_SECRET_KEY|ATTENDEE_AUTH_SECRET)$/;
+/** The mail settings: the app's Postmark token, From address and stream. A preview given them can send mail, so it only runs behind the mail guard. */
+export const MAIL_ENV_KEY = /^POSTMARK_(SERVER_TOKEN|FROM_EMAIL|MESSAGE_STREAM)$/;
+export const SECRET_ENV_KEY = new RegExp(`(?:${SUPABASE_KEY.source})|(?:${MAIL_ENV_KEY.source})`);
+/** Names that hold a key which bypasses row-level security, whatever the value looks like. */
+export const SERVICE_NAME = /(SERVICE_ROLE_KEY|SUPABASE_SECRET_KEY)$/;
+/** A setting a browser bundle may carry: a service key never goes into one. */
+export const CLIENT_NAME = /^(NEXT_PUBLIC_|PUBLIC_|VITE_)/;
+/** In a recipe: {secret:NAME}. */
+export const SECRET_PLACEHOLDER = /^\{secret:([A-Z0-9_]{2,60})\}$/;
+export const URL_KEY = /SUPABASE_URL$/;
+export const EMAIL = /^[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9.-]{1,200}\.[A-Za-z]{2,24}$/;
+/** The whole mail set by name: a variable of any of these providers is a mail setting. SES must be a whole word (SESSION_SECRET is not mail). */
+export const MAIL_SET_NAME = /(MAIL|SMTP|POSTMARK|SENDGRID|MAILGUN|RESEND)|(^|_)SES(_|$)/i;
+// -- FUNCTIONS -- //
+export const isMailKey = (name) => MAIL_ENV_KEY.test(String(name));
+/** Every plain sentence about how a recipe env setting `key` uses the placeholder `value`, or an empty list. Names only, never the value. */
+export function placeholderProblems(key, value) {
+    const m = SECRET_PLACEHOLDER.exec(value);
+    if (m && m[1] !== undefined && SERVICE_NAME.test(m[1]) && CLIENT_NAME.test(key))
+        return [`The recipe gives ${key}, which a browser bundle carries, the service key ${m[1]}.`];
+    if (m &&
+        m[1] !== undefined &&
+        SECRET_ENV_KEY.test(m[1]) &&
+        SECRET_ENV_KEY.test(key) &&
+        (isMailKey(key) !== isMailKey(m[1]) || (isMailKey(key) && key !== m[1])))
+        return [
+            `The recipe gives ${key} the sealed value ${m[1]}: a mail setting takes only the sealed value of its own name (another kind of value is never swapped in).`
+        ];
+    if (!m || m[1] === undefined || !SECRET_ENV_KEY.test(m[1]) || !SECRET_ENV_KEY.test(key))
+        return [
+            `The recipe gives ${key} a sealed value that is not written as {secret:NAME}, or names a key that only a Supabase setting or a mail setting (POSTMARK_SERVER_TOKEN, POSTMARK_FROM_EMAIL, POSTMARK_MESSAGE_STREAM) may take.`
+        ];
+    return [];
+}