@platless/axom-cli 0.1.0 → 0.5.9

package/dist/workstation/keepers/terminal/exec.js

@@ -0,0 +1,981 @@
+// keeper-terminal's exec MECHANISM (spec §4.2, §3.4; analysis E §3.1-3.2, §6): spawn the daemon's argv exactly, write the raw
+// stdout/stderr bytes into one durable stream log per exec, kill the process group at the daemon's absolute deadline (with or
+// without a daemon connected), enforce the output quotas, and settle execs whose keeper died. Every policy decision (argv, env,
+// deadline, cancel, result shaping, what to do on keeper loss) arrives as a parameter from the daemon.
+// Keeper-side module: node builtins and kit siblings only (the import guard enforces it).
+//
+// On disk, per exec, under `<configDir>/keepers/terminal/data/exec/<execId>/` (0700 directories, 0600 files, never /tmp):
+//   exec.json   descriptor (argv, cwd, env KEY NAMES, parameters, owner keeper, spawn identity, exit). Atomically replaced.
+//   log/        the StreamLog (`spawn`, `out`, `err`, `gap`, `exit` records), relaxed fsync (1 s / 4 MiB; spawn/exit at once).
+//   keep.jsonl  the records the quota never drops — spawn, gap, exit, and the first `retainHeadBytes` of each stream — with their
+//               original offsets, so they stay readable after the log's older segments are dropped. The head reserve and the
+//               spawn record also survive the daemon's non-final commits (the daemon re-reads the head after a restart).
+// Env values never reach the disk.
+//
+// Quota (spec §3.4): the pipe is always read. Output records beyond each stream's head reserve are "droppable"; when the
+// droppable uncommitted output exceeds `maxUncommittedBytes`, the oldest of it is dropped (down to 3/4 of the limit, so a
+// flood does not pay one commit per chunk): `gap{stream, droppedBytes, from, through}` records are written and the log's
+// watermark moves past `through` (whole 8 MiB segments are deleted). Reads below that watermark are served from keep.jsonl.
+// The keeper's total over all exec logs is bounded by 512 MiB by reclaiming the logs of the oldest ended execs first.
+// Ended execs are removed entirely 24 h after they ended (logs only: a running exec is never killed by a TTL).
+import { spawn } from 'node:child_process';
+import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeSync, chmodSync, ftruncateSync, } from 'node:fs';
+import path from 'node:path';
+import { manifestProcessLive } from '../kit/lock.js';
+import { KeeperMethodError } from '../kit/protocol.js';
+import { StreamLog } from '../kit/streamLog.js';
+import { settleOrphan, processStartTime, signalGroup } from '../kit/orphanSweep.js';
+import { absolutePath, badRequest as bad, environment } from './params.js';
+import { EXEC_CHUNK_BYTES, EXEC_FSYNC_BYTES, EXEC_FSYNC_INTERVAL_MS, EXEC_LABEL_CHARS, EXEC_RUNNING_LIMIT, EXEC_SEGMENT_BYTES, EXEC_SIGNALS, EXEC_TERMINAL_TTL_MS, EXEC_TOTAL_BYTES, ExecErrorCode, } from './protocol.js';
+const META_FILE = 'exec.json';
+const KEEP_FILE = 'keep.jsonl';
+const LOG_DIR = 'log';
+/** Accounted cost of one output record beyond its payload (its JSON envelope), so floods of tiny records are bounded too. */
+const RECORD_OVERHEAD = 64;
+/** Droppable runs are cut at about this many accounted bytes: few runs to track, and a commit inside one re-reads few records. */
+const RUN_COST = 16 * 1024;
+/** After a SIGKILL, a leader that has exited but whose pipes stay open (held by an escaped process) is closed after this long. */
+const PIPE_CLOSE_AFTER_KILL_MS = 1000;
+/** While a leader-exited exec runs (or a grace SIGKILL is due), its group is probed this often, so `groupGone` lags at most this long. */
+const GROUP_PROBE_MS = 1000;
+const MAX_ARGV = 4096;
+const MAX_READ_RECORDS = 1024;
+/** Keeps an `exec.read` answer far below the 2 MiB outbound frame. */
+const MAX_READ_BYTES = 1536 * 1024;
+const MAX_GRACE_MS = 10 * 60 * 1000;
+const EXEC_ID = /^[A-Za-z0-9_.:-]{1,128}$/;
+const START_KEYS = ['execId', 'argv', 'cwd', 'env', 'deadlineEpochMs', 'killGraceMs', 'retainHeadBytes', 'maxUncommittedBytes', 'onKeeperLoss', 'label', 'jobId'];
+/**
+ * Whether a kept record stays in keep.jsonl once the daemon committed through `committed`. The head reserve (kept `out`/`err`
+ * records: the first `retainHeadBytes` of each stream) and the spawn record survive every non-final commit: the daemon shapes the
+ * head of its result from them, also after a restart that lost everything but its checkpoint (spec §4.2; the checkpoint carries
+ * only the tails). Gap records below the commit are dropped (the daemon has seen them). Bounded: at most `retainHeadBytes` per
+ * stream. A final commit or the TTL removes the whole exec.
+ */
+function survivesCommit(record, committed) {
+    return record.o > committed || record.k === 'spawn' || record.k === 'out' || record.k === 'err';
+}
+function keys(params, allowed) {
+    for (const key of Object.keys(params))
+        if (!allowed.includes(key))
+            bad(`Unexpected parameter ${key}`);
+}
+function execIdOf(params) {
+    const id = params.execId;
+    if (typeof id !== 'string' || !EXEC_ID.test(id) || id === '.' || id === '..')
+        bad('Invalid execId');
+    return id;
+}
+function integer(value, name, min, max) {
+    if (!Number.isSafeInteger(value) || value < min || value > max)
+        bad(`Invalid ${name}`);
+    return value;
+}
+/** Atomically replaces `file` with `content` (tmp + fsync + rename + directory fsync), mode 0600. */
+function atomicWrite(file, content) {
+    const temp = `${file}.tmp`;
+    const fd = openSync(temp, 'w', 0o600);
+    try {
+        writeAllSync(fd, Buffer.from(content));
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+    renameSync(temp, file);
+    fsyncDir(path.dirname(file));
+}
+function writeAllSync(fd, data) {
+    let written = 0;
+    while (written < data.length)
+        written += writeSync(fd, data, written, data.length - written);
+}
+function fsyncDir(dir) {
+    let fd;
+    try {
+        fd = openSync(dir, 'r');
+        fsyncSync(fd);
+    }
+    catch { /* best effort */ }
+    finally {
+        if (fd !== undefined)
+            try {
+                closeSync(fd);
+            }
+            catch { /* ignore */ }
+    }
+}
+function privateDir(dir) {
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    chmodSync(dir, 0o700);
+}
+function validMeta(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const m = value;
+    const owner = m.owner;
+    return m.v === 1 && typeof m.execId === 'string' && Array.isArray(m.argv) && typeof m.cwd === 'string' && typeof m.label === 'string'
+        && Number.isSafeInteger(m.killGraceMs) && (m.onKeeperLoss === 'kill' || m.onKeeperLoss === 'leave')
+        && typeof owner === 'object' && owner !== null && Number.isSafeInteger(owner.pid) && typeof owner.bundleDir === 'string';
+}
+/** Loads keep.jsonl, truncating a torn tail (anything after the last intact line). */
+function loadKeep(file) {
+    let data;
+    try {
+        data = readFileSync(file);
+    }
+    catch {
+        return [];
+    }
+    const records = [];
+    let good = 0;
+    for (let start = 0; start < data.length;) {
+        const nl = data.indexOf(10, start);
+        if (nl < 0)
+            break;
+        try {
+            const record = JSON.parse(data.subarray(start, nl).toString('utf8'));
+            if (!Number.isSafeInteger(record?.o) || typeof record.k !== 'string')
+                break;
+            records.push(record);
+        }
+        catch {
+            break;
+        }
+        start = nl + 1;
+        good = start;
+    }
+    if (good < data.length) {
+        const fd = openSync(file, 'r+');
+        try {
+            ftruncateSync(fd, good);
+            fsyncSync(fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    return records;
+}
+export class ExecHost {
+    options;
+    entries = new Map();
+    now;
+    logLine;
+    maintenanceTimer;
+    sinceTotalCheck = 0;
+    stopping = false;
+    constructor(options) {
+        this.options = options;
+        this.now = options.now ?? Date.now;
+        this.logLine = options.log ?? (() => { });
+    }
+    // ---- startup: adopt what is on disk, settle execs whose keeper died -----------------------------------------------
+    /**
+     * Startup sweep (spec §4.2): every exec directory not owned by another live keeper is adopted. One that spawned and has no exit
+     * lost its keeper: `onKeeperLoss` is applied (`kill`: only after verifying pid + start time, see `settleOrphan`) and
+     * `exit{reason:'keeper_lost'}` is written either way. Then TTL / total quota run once and the maintenance timer starts.
+     */
+    async open() {
+        privateDir(this.options.root);
+        await this.adopt();
+        this.maintain();
+        const every = this.options.maintenanceMs ?? 60_000;
+        this.maintenanceTimer = setInterval(() => { void this.adopt().then(() => this.maintain(), (error) => this.report(error)); }, every);
+        this.maintenanceTimer.unref();
+    }
+    report(error) { this.logLine(`exec: ${error instanceof Error ? error.message : String(error)}`); }
+    async ownerAlive(owner) {
+        if (owner.pid === this.options.owner.pid)
+            return false; // ours (a previous host in this very process): adopt
+        return (this.options.ownerAlive ?? manifestProcessLive)(owner);
+    }
+    async adopt() {
+        let names;
+        try {
+            names = readdirSync(this.options.root);
+        }
+        catch {
+            return;
+        }
+        for (const name of names) {
+            if (this.entries.has(name) || !EXEC_ID.test(name))
+                continue;
+            const dir = path.join(this.options.root, name);
+            let meta;
+            try {
+                const value = JSON.parse(readFileSync(path.join(dir, META_FILE), 'utf8'));
+                if (!validMeta(value) || value.execId !== name) {
+                    this.report(new Error(`${name}: unreadable descriptor, left alone`));
+                    continue;
+                }
+                meta = value;
+            }
+            catch {
+                continue;
+            } // being created by its owner, or not an exec directory
+            try {
+                if (await this.ownerAlive(meta.owner))
+                    continue;
+                if (this.entries.has(name))
+                    continue; // adopted meanwhile
+                await this.adoptOne(meta, dir);
+            }
+            catch (error) {
+                this.report(error);
+            }
+        }
+    }
+    async adoptOne(meta, dir) {
+        const entry = this.openEntry(meta, dir);
+        if (!meta.exit && entry.log) {
+            const head = entry.log.state().head;
+            const last = head >= 0 ? entry.log.read(head, 1)[0] : undefined;
+            if (last?.k === 'exit')
+                meta.exit = { code: last.code, signal: last.signal, timedOut: last.timedOut === true, ...(last.reason ? { reason: last.reason } : {}), atMs: last.t };
+        }
+        if (!meta.exit) {
+            let outcome = 'gone';
+            if (meta.spawn)
+                outcome = await settleOrphan(meta.spawn, meta.onKeeperLoss, meta.killGraceMs, this.options.sweep);
+            this.writeExit(entry, { code: null, signal: null, timedOut: false, reason: 'keeper_lost' });
+            this.logLine(`exec ${meta.execId}: keeper lost; process group ${outcome}`);
+        }
+        meta.owner = this.options.owner;
+        this.saveMeta(entry);
+        this.entries.set(meta.execId, entry);
+    }
+    openEntry(meta, dir) {
+        const entry = {
+            meta, dir, keep: [], keepOffsets: new Set(), committed: -1, dropped: -1, droppedSinceCommit: 0,
+            runs: [], droppableCost: 0, headBytes: { out: 0, err: 0 }, streamBytes: { out: 0, err: 0 }, timedOut: false, timers: [], leaderExited: false,
+            groupGone: false, graceKillPending: false, deadlineFired: false, stoppedByKeeper: false,
+        };
+        if (meta.logReclaimed)
+            return entry;
+        entry.log = this.openLog(dir);
+        const state = entry.log.state();
+        const tail = state.committedTail;
+        entry.committed = Number.isSafeInteger(tail?.committed) ? tail.committed : state.committed;
+        entry.dropped = Number.isSafeInteger(tail?.dropped) ? tail.dropped : -1;
+        entry.keep = loadKeep(path.join(dir, KEEP_FILE)).filter((r) => survivesCommit(r, entry.committed));
+        for (const r of entry.keep) {
+            entry.keepOffsets.add(r.o);
+            if (r.k === 'gap' && Number.isSafeInteger(r.through))
+                entry.dropped = Math.max(entry.dropped, r.through);
+        }
+        return entry;
+    }
+    openLog(dir) {
+        return StreamLog.open({
+            dir: path.join(dir, LOG_DIR), segmentBytes: EXEC_SEGMENT_BYTES, fsyncIntervalMs: EXEC_FSYNC_INTERVAL_MS, fsyncBytes: EXEC_FSYNC_BYTES,
+            now: this.now, onError: (error) => this.report(error),
+        });
+    }
+    // ---- exec.start ----------------------------------------------------------------------------------------------------
+    async start(params) {
+        keys(params, START_KEYS);
+        const execId = execIdOf(params);
+        const existing = this.entries.get(execId);
+        if (existing) {
+            await existing.ready;
+            return this.describe(existing);
+        }
+        const dir = path.join(this.options.root, execId);
+        if (this.dirExists(dir))
+            throw new KeeperMethodError(ExecErrorCode.foreign, `Exec ${execId} belongs to another keeper`);
+        const argv = params.argv;
+        if (!Array.isArray(argv) || argv.length === 0 || argv.length > MAX_ARGV || argv.some((a) => typeof a !== 'string' || a.includes('\0')) || !argv[0])
+            bad('Invalid argv');
+        const cwd = absolutePath(params.cwd, 'cwd');
+        const env = environment(params.env);
+        const deadline = params.deadlineEpochMs;
+        if (deadline !== undefined && deadline !== null && (typeof deadline !== 'number' || !Number.isFinite(deadline)))
+            bad('Invalid deadlineEpochMs');
+        const killGraceMs = integer(params.killGraceMs, 'killGraceMs', 0, MAX_GRACE_MS);
+        const retainHeadBytes = integer(params.retainHeadBytes, 'retainHeadBytes', 0, 16 * 1024 * 1024);
+        const maxUncommittedBytes = integer(params.maxUncommittedBytes, 'maxUncommittedBytes', 1024, 1024 * 1024 * 1024);
+        const onKeeperLoss = params.onKeeperLoss ?? 'kill';
+        if (onKeeperLoss !== 'kill' && onKeeperLoss !== 'leave')
+            bad('Invalid onKeeperLoss');
+        if (typeof params.label !== 'string')
+            bad('Invalid label');
+        const jobId = params.jobId;
+        if (jobId !== undefined && (typeof jobId !== 'string' || !jobId || jobId.length > 256))
+            bad('Invalid jobId');
+        if (this.runningCount() >= (this.options.runningLimit ?? EXEC_RUNNING_LIMIT))
+            throw new KeeperMethodError(ExecErrorCode.limit, 'Too many running commands in keeper-terminal');
+        const meta = {
+            v: 1, execId, label: params.label.slice(0, EXEC_LABEL_CHARS), ...(jobId ? { jobId: jobId } : {}),
+            argv: argv, cwd, envKeys: Object.keys(env).sort(),
+            deadlineEpochMs: typeof deadline === 'number' ? deadline : null, killGraceMs, retainHeadBytes, maxUncommittedBytes, onKeeperLoss: onKeeperLoss,
+            createdAtMs: this.now(), owner: this.options.owner, spawn: null, exit: null, logReclaimed: null,
+        };
+        // Synchronously up to the spawn: a concurrent start of the same id finds the entry and waits for it.
+        let entry;
+        try {
+            privateDir(dir);
+            this.options.faults?.afterExecDir?.();
+            atomicWrite(path.join(dir, META_FILE), JSON.stringify(meta));
+            entry = this.openEntry(meta, dir);
+        }
+        catch (error) {
+            rmSync(dir, { recursive: true, force: true }); // never leave a directory that would read as another keeper's exec
+            throw error;
+        }
+        this.entries.set(execId, entry);
+        entry.ready = this.launch(entry, env).catch((error) => { this.report(error); });
+        await entry.ready;
+        return this.describe(entry);
+    }
+    dirExists(dir) {
+        try {
+            return statSync(dir).isDirectory();
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * `spawn(argv[0], argv.slice(1), {cwd, env, detached})` exactly: no shell, no wrapping, no fallback to this process's env.
+     * Detached = setsid, so the exec leads its own process group (pgid = pid) and every pipeline / background job it starts stays in
+     * it. stdin is a pipe that is never written (as the daemon's in-process exec always did).
+     */
+    async launch(entry, env) {
+        const { meta } = entry;
+        let child;
+        try {
+            child = spawn(meta.argv[0], meta.argv.slice(1), { cwd: meta.cwd, env, detached: true, stdio: ['pipe', 'pipe', 'pipe'] });
+        }
+        catch (error) {
+            this.writeExit(entry, { code: null, signal: null, timedOut: false, reason: 'spawn_failed', error: messageOf(error) });
+            return;
+        }
+        child.stdin.on('error', () => { });
+        if (child.pid === undefined) {
+            const error = await new Promise((resolve) => child.once('error', resolve));
+            this.writeExit(entry, { code: null, signal: null, timedOut: false, reason: 'spawn_failed', error: messageOf(error) });
+            return;
+        }
+        entry.child = child;
+        const pid = child.pid;
+        meta.spawn = { pid, pgid: pid, startTime: null, atMs: this.now() };
+        this.saveMeta(entry); // the pid is durable before anything else happens: a keeper crash from here on is settled at restart
+        child.on('error', (error) => this.report(error));
+        child.once('exit', () => { entry.leaderExited = true; this.watchGroup(entry); });
+        const closed = new Promise((resolve) => child.once('close', (code, signal) => resolve({ code, signal })));
+        // The spawn record comes first. Until it is in the log the streams are held paused (the pipe fills and the command waits
+        // briefly). Listeners are attached right away: on the leader's exit Node resumes every stdio stream, and data reaching a
+        // stream without a listener would be lost. A chunk that arrives meanwhile is held and the stream paused again.
+        let held = [];
+        for (const [name, stream] of [['out', child.stdout], ['err', child.stderr]]) {
+            stream.on('data', (chunk) => {
+                if (held) {
+                    held.push([name, chunk]);
+                    stream.pause();
+                    return;
+                }
+                this.onOutput(entry, name, chunk);
+            });
+            stream.pause();
+        }
+        // Deadline and exit handling never depend on the spawn record: if that write fails the exec still ends and still has a deadline.
+        this.armDeadline(entry);
+        const recorded = (async () => {
+            try {
+                meta.spawn.startTime = await (this.options.startTime ?? ((p) => processStartTime(p, this.options.sweep)))(pid).catch(() => null);
+                this.saveMeta(entry);
+                this.options.faults?.beforeSpawnRecord?.();
+                this.appendKept(entry, { k: 'spawn', pid, pgid: pid, startTime: meta.spawn.startTime, argv: meta.argv, cwd: meta.cwd, envKeys: meta.envKeys });
+            }
+            finally {
+                const pending = held;
+                held = undefined;
+                for (const [name, chunk] of pending)
+                    this.onOutput(entry, name, chunk);
+                child.stdout.resume();
+                child.stderr.resume();
+            }
+        })();
+        void Promise.all([closed, recorded.catch((error) => { this.report(error); })]).then(([{ code, signal }]) => {
+            for (const timer of entry.timers)
+                clearTimeout(timer);
+            entry.timers = [];
+            if (this.stopping && meta.onKeeperLoss === 'leave')
+                return; // left to the next keeper's sweep
+            if (!meta.exit)
+                this.writeExit(entry, { code, signal, timedOut: entry.timedOut, ...(entry.stoppedByKeeper ? { reason: 'keeper_stopped' } : {}) });
+            entry.child = undefined;
+            this.stopWatchingIfDone(entry);
+            entry.closing?.();
+            this.maintainSoon();
+        });
+        await recorded;
+    }
+    armDeadline(entry) {
+        const { deadlineEpochMs, killGraceMs } = entry.meta;
+        if (deadlineEpochMs === null)
+            return;
+        entry.fireDeadline = () => {
+            if (entry.deadlineFired || entry.meta.exit || !entry.meta.spawn)
+                return;
+            entry.deadlineFired = true;
+            entry.timedOut = true;
+            if (!this.signalExec(entry, 'SIGTERM') && entry.groupGone)
+                this.closePipesLater(entry, 0); // only an escaped process holds the pipes
+            // Not one of entry.timers (those end with the exec): members that ignore SIGTERM and hold no pipe outlive the exec's end.
+            entry.graceKillPending = true;
+            this.watchGroup(entry);
+            setTimeout(() => { entry.graceKillPending = false; this.killGroup(entry); this.stopWatchingIfDone(entry); }, killGraceMs).unref();
+        };
+        const delay = deadlineEpochMs - this.now();
+        if (delay <= 0)
+            entry.fireDeadline();
+        else
+            this.later(entry, delay, entry.fireDeadline);
+    }
+    /** setTimeout bounded to the 32-bit range (a far deadline re-arms), unref'd: a timer never keeps the keeper alive by itself. */
+    later(entry, ms, run) {
+        const max = 2 ** 31 - 1;
+        const timer = setTimeout(() => { entry.timers = entry.timers.filter((t) => t !== timer); if (ms > max)
+            this.later(entry, ms - max, run);
+        else
+            run(); }, Math.min(ms, max));
+        timer.unref();
+        entry.timers.push(timer);
+    }
+    /**
+     * Probes whether the group still has a member (signal 0). Only meaningful once the leader has exited: while the leader (our
+     * child) lives, its pid — and so the group id — cannot be reused. ESRCH (or EPERM: the id now names a group we do not own) makes
+     * `groupGone` stick.
+     */
+    probeGroup(entry) {
+        if (entry.groupGone)
+            return true;
+        if (!entry.leaderExited || !entry.meta.spawn)
+            return false;
+        try {
+            process.kill(-entry.meta.spawn.pgid, 0);
+            return false;
+        }
+        catch (error) {
+            if (error.code === 'ESRCH' || error.code === 'EPERM') {
+                entry.groupGone = true;
+                this.stopWatchingIfDone(entry);
+                return true;
+            }
+            return false;
+        }
+    }
+    /** After the leader exited: probe the group now and every second while the exec runs or a grace SIGKILL is still due. */
+    watchGroup(entry) {
+        if (entry.groupProbe || !entry.leaderExited || this.probeGroup(entry))
+            return;
+        entry.groupProbe = setInterval(() => { this.probeGroup(entry); }, GROUP_PROBE_MS);
+        entry.groupProbe.unref();
+    }
+    stopWatchingIfDone(entry) {
+        if (entry.groupProbe && (entry.groupGone || (!entry.child && !entry.graceKillPending))) {
+            clearInterval(entry.groupProbe);
+            entry.groupProbe = undefined;
+        }
+    }
+    /** Signals the exec's group unless it is known gone. */
+    signalExec(entry, signal) {
+        if (!entry.meta.spawn || this.probeGroup(entry))
+            return false;
+        return (this.options.signalGroup ?? signalGroup)(entry.meta.spawn.pgid, signal);
+    }
+    /**
+     * SIGKILL to the group (unless it is known gone). The group may outlive its leader (members that ignored SIGTERM). If the leader
+     * is gone but the pipes stay open (a process that left the group still holds them), they are closed shortly after so the exec can
+     * end — at once when the group is already gone.
+     */
+    killGroup(entry) {
+        if (!entry.meta.spawn)
+            return;
+        this.signalExec(entry, 'SIGKILL');
+        this.closePipesLater(entry, entry.groupGone ? 0 : PIPE_CLOSE_AFTER_KILL_MS);
+    }
+    closePipesLater(entry, ms) {
+        const timer = setTimeout(() => {
+            const child = entry.child;
+            if (child && entry.leaderExited) {
+                child.stdout.destroy();
+                child.stderr.destroy();
+                child.stdin.destroy();
+            }
+        }, ms);
+        timer.unref();
+    }
+    // ---- output and quota ----------------------------------------------------------------------------------------------
+    onOutput(entry, stream, chunk) {
+        if (!entry.log)
+            return;
+        try {
+            for (let at = 0; at < chunk.length; at += EXEC_CHUNK_BYTES) {
+                let piece = chunk.subarray(at, Math.min(chunk.length, at + EXEC_CHUNK_BYTES));
+                const headLeft = entry.meta.retainHeadBytes - entry.headBytes[stream];
+                if (headLeft > 0) {
+                    const head = piece.subarray(0, headLeft);
+                    this.appendOutput(entry, stream, head, true);
+                    piece = piece.subarray(head.length);
+                }
+                if (piece.length > 0)
+                    this.appendOutput(entry, stream, piece, false);
+            }
+        }
+        catch (error) {
+            this.report(error);
+        } // a full disk must not crash the keeper: the pipe keeps being drained
+    }
+    appendOutput(entry, stream, bytes, head) {
+        const input = { k: stream, so: entry.streamBytes[stream], b: bytes.toString('base64') };
+        entry.streamBytes[stream] += bytes.length;
+        if (head) {
+            entry.headBytes[stream] += bytes.length;
+            this.appendKept(entry, input);
+            return;
+        }
+        const record = entry.log.append(input);
+        const cost = bytes.length + RECORD_OVERHEAD;
+        const last = entry.runs[entry.runs.length - 1];
+        if (last && last.cost < RUN_COST) {
+            last.through = record.o;
+            last[stream] += bytes.length;
+            last.cost += cost;
+        }
+        else
+            entry.runs.push({ from: record.o, through: record.o, out: stream === 'out' ? bytes.length : 0, err: stream === 'err' ? bytes.length : 0, cost });
+        entry.droppableCost += cost;
+        if (entry.droppableCost > entry.meta.maxUncommittedBytes)
+            this.dropOldest(entry);
+        this.sinceTotalCheck += cost;
+        if (this.sinceTotalCheck >= 4 * 1024 * 1024) {
+            this.sinceTotalCheck = 0;
+            this.maintainSoon();
+        }
+    }
+    /**
+     * Drops the oldest droppable uncommitted output down to 3/4 of `maxUncommittedBytes`: gap records (one per stream that lost
+     * bytes, also kept), then the drop watermark moves past the dropped range; the log follows (whole segments deleted) per segment.
+     */
+    dropOldest(entry) {
+        const log = entry.log;
+        const target = Math.floor(entry.meta.maxUncommittedBytes * 3 / 4);
+        let from = -1;
+        let through = -1;
+        let out = 0;
+        let err = 0;
+        while (entry.droppableCost > target && entry.runs.length > 0) {
+            const run = entry.runs.shift();
+            if (from < 0)
+                from = run.from;
+            through = run.through;
+            out += run.out;
+            err += run.err;
+            entry.droppableCost -= run.cost;
+        }
+        if (through < 0)
+            return;
+        if (out > 0)
+            this.appendKept(entry, { k: 'gap', stream: 'out', droppedBytes: out, from, through });
+        if (err > 0)
+            this.appendKept(entry, { k: 'gap', stream: 'err', droppedBytes: err, from, through });
+        entry.dropped = Math.max(entry.dropped, through);
+        // The watermark lives in the kept gap records (written to the page cache: a keeper crash keeps them; a power cut is the
+        // keeper_lost case). The log commit (fsync + meta + segment deletion) runs once a segment's worth was dropped, after keep.jsonl
+        // is fsynced, so a flood pays a few commits instead of one per drop.
+        entry.droppedSinceCommit += Math.max(1, entry.meta.maxUncommittedBytes - target);
+        if (entry.droppedSinceCommit >= EXEC_SEGMENT_BYTES) {
+            entry.droppedSinceCommit = 0;
+            if (entry.keepFd !== undefined)
+                fsyncSync(entry.keepFd);
+            this.commitLog(entry);
+        }
+    }
+    /** Appends to the log AND to keep.jsonl (spawn, gap, exit, head output): records the quota never drops. */
+    appendKept(entry, input) {
+        const record = entry.log.append(input);
+        if (entry.keepFd === undefined)
+            entry.keepFd = openSync(path.join(entry.dir, KEEP_FILE), 'a', 0o600);
+        writeAllSync(entry.keepFd, Buffer.from(`${JSON.stringify(record)}\n`));
+        entry.keep.push(record);
+        entry.keepOffsets.add(record.o);
+        return record;
+    }
+    commitLog(entry) {
+        entry.log.commit(Math.max(entry.committed, entry.dropped), { committed: entry.committed, dropped: entry.dropped });
+    }
+    writeExit(entry, exit) {
+        const atMs = this.now();
+        if (entry.log) {
+            try {
+                this.appendKept(entry, { k: 'exit', ...exit });
+            }
+            catch (error) {
+                this.report(error);
+            }
+            if (entry.keepFd !== undefined)
+                try {
+                    fsyncSync(entry.keepFd);
+                }
+                catch { /* reported by the next open's repair */ }
+        }
+        entry.meta.exit = { ...exit, atMs };
+        entry.runs = [];
+        entry.droppableCost = 0;
+        this.saveMeta(entry);
+    }
+    saveMeta(entry) {
+        try {
+            atomicWrite(path.join(entry.dir, META_FILE), JSON.stringify(entry.meta));
+        }
+        catch (error) {
+            this.report(error);
+        }
+    }
+    // ---- read / subscribe ----------------------------------------------------------------------------------------------
+    entryOf(params) {
+        const entry = this.entries.get(execIdOf(params));
+        if (!entry)
+            throw new KeeperMethodError(ExecErrorCode.notFound, 'Exec not found');
+        return entry;
+    }
+    /**
+     * Records with `o >= from`, in order: below the drop watermark only the kept records (head, spawn, gap, exit) remain; above it
+     * the log. Below the daemon's own commit: the head reserve and the spawn record (they survive non-final commits, see
+     * {@link survivesCommit}) plus whatever else is still on disk.
+     */
+    readMerged(entry, from, limit) {
+        if (!entry.log || limit <= 0)
+            return [];
+        const dropped = entry.dropped;
+        const floor = Math.max(dropped, entry.committed);
+        const logged = entry.log.read(Math.max(from, dropped + 1), limit);
+        if (from > floor)
+            return logged;
+        const kept = entry.keep.filter((record) => record.o >= from && record.o <= floor);
+        if (kept.length === 0)
+            return logged;
+        // Both are in offset order; a kept record may also still be in the log (same offset): merge, once each.
+        const out = [];
+        let i = 0;
+        let j = 0;
+        while (out.length < limit && (i < kept.length || j < logged.length)) {
+            const a = kept[i];
+            const b = logged[j];
+            if (b === undefined || (a !== undefined && a.o < b.o)) {
+                out.push(a);
+                i += 1;
+            }
+            else {
+                if (a !== undefined && a.o === b.o)
+                    i += 1;
+                out.push(b);
+                j += 1;
+            }
+        }
+        return out;
+    }
+    read(params) {
+        keys(params, ['execId', 'from', 'limit']);
+        const entry = this.entryOf(params);
+        const from = integer(params.from, 'from', 0, Number.MAX_SAFE_INTEGER);
+        const limit = params.limit === undefined ? 256 : integer(params.limit, 'limit', 1, MAX_READ_RECORDS);
+        const records = [];
+        let bytes = 0;
+        for (const record of this.readMerged(entry, from, limit)) {
+            bytes += Buffer.byteLength(JSON.stringify(record));
+            if (records.length > 0 && bytes > MAX_READ_BYTES)
+                break;
+            records.push(record);
+        }
+        return { records, head: this.headOf(entry), next: records.length > 0 ? records[records.length - 1].o + 1 : from };
+    }
+    /** The kit stream source for `exec/<execId>` (`keeper.subscribe`, credit in records). */
+    source(execId) {
+        const entry = this.entries.get(execId);
+        if (!entry?.log)
+            return undefined;
+        const log = entry.log;
+        return {
+            read: (from, limit) => {
+                if (this.entries.get(execId) !== entry || entry.log !== log)
+                    throw new Error('Exec log is gone');
+                return this.readMerged(entry, from, limit);
+            },
+            watch: (notify) => log.watch(notify),
+        };
+    }
+    // ---- kill / commit / checkpoint ------------------------------------------------------------------------------------
+    kill(params) {
+        keys(params, ['execId', 'signal']);
+        const entry = this.entryOf(params);
+        const signal = params.signal;
+        if (typeof signal !== 'string' || !EXEC_SIGNALS.includes(signal))
+            bad('Invalid signal');
+        const state = this.stateOf(entry);
+        if (state !== 'running' || !entry.meta.spawn)
+            return { delivered: false, state };
+        const delivered = this.signalExec(entry, signal);
+        // The group is gone and only an escaped process holds the pipes: end the exec by closing them.
+        if (signal === 'SIGKILL' || entry.groupGone)
+            this.killGroup(entry);
+        return { delivered, state };
+    }
+    commit(params) {
+        keys(params, ['execId', 'through', 'final']);
+        const entry = this.entryOf(params);
+        const head = this.headOf(entry);
+        const through = integer(params.through, 'through', -1, Number.MAX_SAFE_INTEGER);
+        if (through > head)
+            bad('through is beyond the head');
+        if (params.final !== undefined && typeof params.final !== 'boolean')
+            bad('Invalid final');
+        if (params.final === true) {
+            if (!entry.meta.exit)
+                throw new KeeperMethodError(ExecErrorCode.notExited, 'Exec is still running');
+            if (through < head)
+                bad('A final commit must cover the head');
+            this.remove(entry);
+            return { committed: through, head, deleted: true };
+        }
+        if (through > entry.committed && entry.log) {
+            entry.committed = through;
+            this.commitLog(entry);
+            this.forgetCommitted(entry);
+        }
+        return { committed: entry.committed, head, deleted: false };
+    }
+    /** Drops committed records from the droppable runs (exactly: a run cut by the commit is recounted from the log) and from keep. */
+    forgetCommitted(entry) {
+        const committed = entry.committed;
+        while (entry.runs.length > 0 && entry.runs[0].through <= committed)
+            entry.droppableCost -= entry.runs.shift().cost;
+        const first = entry.runs[0];
+        if (first && first.from <= committed) {
+            entry.droppableCost -= first.cost;
+            const recount = { from: committed + 1, through: first.through, out: 0, err: 0, cost: 0 };
+            for (const record of entry.log.read(committed + 1, first.through - committed)) {
+                if (record.o > first.through)
+                    break;
+                if ((record.k !== 'out' && record.k !== 'err') || entry.keepOffsets.has(record.o))
+                    continue;
+                const size = Buffer.byteLength(record.b, 'base64');
+                recount[record.k] += size;
+                recount.cost += size + RECORD_OVERHEAD;
+            }
+            entry.runs[0] = recount;
+            entry.droppableCost += recount.cost;
+        }
+        if (entry.keep.some((r) => !survivesCommit(r, committed))) {
+            entry.keep = entry.keep.filter((r) => survivesCommit(r, committed));
+            entry.keepOffsets = new Set(entry.keep.map((r) => r.o));
+            this.rewriteKeep(entry);
+        }
+    }
+    rewriteKeep(entry) {
+        if (entry.keepFd !== undefined) {
+            try {
+                closeSync(entry.keepFd);
+            }
+            catch { /* ignore */ }
+            entry.keepFd = undefined;
+        }
+        atomicWrite(path.join(entry.dir, KEEP_FILE), entry.keep.map((r) => `${JSON.stringify(r)}\n`).join(''));
+    }
+    checkpoint(params) {
+        keys(params, ['execId', 'blob', 'offset']);
+        const entry = this.entryOf(params);
+        if (typeof params.blob !== 'string')
+            bad('Invalid blob');
+        if (!entry.log)
+            bad('Exec log was reclaimed');
+        const head = this.headOf(entry);
+        const offset = integer(params.offset, 'offset', -1, head); // required: a checkpoint stamped at the head would skip unprocessed records
+        const written = entry.log.writeCheckpoint(offset, params.blob);
+        return { seq: written.seq, offset: written.offset, atMs: written.t, blob: params.blob };
+    }
+    // ---- list / status / activity --------------------------------------------------------------------------------------
+    stateOf(entry) {
+        if (entry.meta.exit)
+            return 'exited';
+        return entry.meta.spawn ? 'running' : 'starting';
+    }
+    headOf(entry) {
+        if (entry.log)
+            return entry.log.state().head;
+        return entry.meta.logReclaimed?.head ?? -1;
+    }
+    describe(entry) {
+        const { meta } = entry;
+        return {
+            execId: meta.execId, label: meta.label, ...(meta.jobId ? { jobId: meta.jobId } : {}), state: this.stateOf(entry),
+            pid: meta.spawn?.pid ?? null, pgid: meta.spawn?.pgid ?? null, startTime: meta.spawn?.startTime ?? null,
+            createdAtMs: meta.createdAtMs, deadlineEpochMs: meta.deadlineEpochMs, onKeeperLoss: meta.onKeeperLoss, exit: meta.exit,
+            head: this.headOf(entry), committed: entry.committed, dropped: entry.dropped,
+            checkpointOffset: entry.log?.state().checkpointOffset ?? -1, logReclaimed: meta.logReclaimed !== null,
+        };
+    }
+    list() {
+        return { execs: [...this.entries.values()].sort((a, b) => a.meta.createdAtMs - b.meta.createdAtMs).map((entry) => this.describe(entry)) };
+    }
+    status(params) {
+        keys(params, ['execId']);
+        const entry = this.entryOf(params);
+        const latest = entry.log?.latestCheckpoint();
+        return { ...this.describe(entry), checkpoint: latest ? { seq: latest.seq, offset: latest.offset, atMs: latest.t, blob: latest.blob.toString('utf8') } : null };
+    }
+    runningCount() {
+        let n = 0;
+        for (const entry of this.entries.values())
+            if (!entry.meta.exit)
+                n += 1;
+        return n;
+    }
+    /** `execRunning` and one item per running exec (label and jobId as the daemon gave them; never the command line). */
+    activity = () => {
+        const items = [];
+        let running = 0;
+        for (const entry of this.entries.values()) {
+            if (entry.meta.exit)
+                continue;
+            running += 1;
+            items.push({ type: 'exec', id: entry.meta.execId, label: entry.meta.label, state: 'running', sinceEpochMs: entry.meta.spawn?.atMs ?? entry.meta.createdAtMs, ...(entry.meta.jobId ? { jobId: entry.meta.jobId } : {}) });
+        }
+        return { metrics: { execRunning: running }, items };
+    };
+    // ---- maintenance: TTL, total quota ---------------------------------------------------------------------------------
+    maintainPending = false;
+    maintainSoon() {
+        if (this.maintainPending || this.stopping)
+            return;
+        this.maintainPending = true;
+        setImmediate(() => { this.maintainPending = false; this.maintain(); });
+    }
+    /** Removes ended execs past the TTL, then reclaims the logs of the oldest ended execs while the total exceeds 512 MiB. */
+    maintain() {
+        if (this.stopping)
+            return;
+        try {
+            const now = this.now();
+            for (const entry of this.entries.values()) {
+                if (entry.meta.exit)
+                    continue;
+                // Timers run on a monotonic clock that may stand still while the machine sleeps: the deadline is wall-clock.
+                if (entry.meta.deadlineEpochMs !== null && now >= entry.meta.deadlineEpochMs)
+                    entry.fireDeadline?.();
+                if (entry.leaderExited)
+                    this.probeGroup(entry);
+            }
+            const ttl = this.options.ttlMs ?? EXEC_TERMINAL_TTL_MS;
+            for (const entry of [...this.entries.values()])
+                if (entry.meta.exit && now - entry.meta.exit.atMs >= ttl)
+                    this.remove(entry);
+            const limit = this.options.totalBytes ?? EXEC_TOTAL_BYTES;
+            let total = 0;
+            for (const entry of this.entries.values())
+                total += this.bytesOf(entry);
+            if (total <= limit)
+                return;
+            const ended = [...this.entries.values()].filter((e) => e.meta.exit && e.log).sort((a, b) => a.meta.exit.atMs - b.meta.exit.atMs);
+            for (const entry of ended) {
+                if (total <= limit)
+                    break;
+                total -= this.bytesOf(entry);
+                this.reclaimLog(entry);
+            }
+        }
+        catch (error) {
+            this.report(error);
+        }
+    }
+    bytesOf(entry) {
+        if (!entry.log)
+            return 0;
+        let keep = 0;
+        try {
+            keep = statSync(path.join(entry.dir, KEEP_FILE)).size;
+        }
+        catch { /* none */ }
+        return entry.log.state().bytes + keep;
+    }
+    closeFiles(entry) {
+        try {
+            entry.log?.close();
+        }
+        catch (error) {
+            this.report(error);
+        }
+        if (entry.keepFd !== undefined) {
+            try {
+                closeSync(entry.keepFd);
+            }
+            catch { /* ignore */ }
+            entry.keepFd = undefined;
+        }
+    }
+    reclaimLog(entry) {
+        const head = this.headOf(entry);
+        this.closeFiles(entry);
+        entry.log = undefined;
+        entry.keep = [];
+        entry.keepOffsets = new Set();
+        entry.meta.logReclaimed = { atMs: this.now(), head };
+        this.saveMeta(entry);
+        rmSync(path.join(entry.dir, LOG_DIR), { recursive: true, force: true });
+        rmSync(path.join(entry.dir, KEEP_FILE), { force: true });
+        this.logLine(`exec ${entry.meta.execId}: log reclaimed (total quota)`);
+    }
+    remove(entry) {
+        this.closeFiles(entry);
+        entry.log = undefined;
+        this.entries.delete(entry.meta.execId);
+        rmSync(entry.dir, { recursive: true, force: true });
+    }
+    // ---- shutdown ------------------------------------------------------------------------------------------------------
+    /**
+     * The keeper is stopping (graceful or force). Running execs started with `onKeeperLoss:'kill'` get SIGTERM, SIGKILL after at most
+     * `graceMs`, and their exit carries `reason:'keeper_stopped'` (also when the real exit arrived in time). `onKeeperLoss:'leave'` is
+     * honoured here too: those groups are not signalled and get no exit record; the next keeper's startup sweep settles them as it would
+     * after a crash (left running, `keeper_lost`). Logs are closed.
+     */
+    async dispose(graceMs = 2000) {
+        this.stopping = true;
+        clearInterval(this.maintenanceTimer);
+        const live = [...this.entries.values()].filter((e) => !e.meta.exit && e.child && e.meta.onKeeperLoss === 'kill');
+        const ended = (entry, ms) => new Promise((resolve) => {
+            if (entry.meta.exit) {
+                resolve();
+                return;
+            }
+            const timer = setTimeout(resolve, ms);
+            timer.unref();
+            entry.closing = () => { clearTimeout(timer); resolve(); };
+        });
+        for (const entry of live) {
+            entry.stoppedByKeeper = true;
+            this.signalExec(entry, 'SIGTERM');
+        }
+        await Promise.all(live.map((entry) => ended(entry, Math.min(graceMs, entry.meta.killGraceMs))));
+        for (const entry of live)
+            if (!entry.meta.exit)
+                this.killGroup(entry);
+        await Promise.all(live.map((entry) => ended(entry, 500)));
+        for (const entry of live)
+            if (!entry.meta.exit)
+                this.writeExit(entry, { code: null, signal: null, timedOut: entry.timedOut, reason: 'keeper_stopped' });
+        for (const entry of this.entries.values()) {
+            for (const timer of entry.timers)
+                clearTimeout(timer);
+            entry.timers = [];
+            if (entry.groupProbe) {
+                clearInterval(entry.groupProbe);
+                entry.groupProbe = undefined;
+            }
+            this.closeFiles(entry);
+        }
+    }
+}
+function messageOf(error) { return error instanceof Error ? error.message : String(error); }