excaliagent 0.2.0 → 0.4.0

package/dist/server/sources/codex.js

@@ -0,0 +1,363 @@
+// OpenAI Codex source: dormant sessions from rollouts
+// (<codexHome>/sessions/YYYY/MM/DD/rollout-*.jsonl), live ones from held
+// thread locks (codex-live.ts), names from session_index.jsonl. Codex writes no
+// status registry: busy = a turn is open in the rollout, else idle. Prompts
+// waiting for approval are not persisted, so they show as busy.
+// Formats: docs/design.md "Codex data".
+import { watch as fsWatch } from "node:fs";
+import { readdir, realpath, stat } from "node:fs/promises";
+import { homedir } from "node:os";
+import { basename, join } from "node:path";
+import { createProcessTable } from "../desktop/process-table.js";
+import { assignLive, isDaemon, looksLikeCodex, scanCodexProcesses } from "./codex-live.js";
+import { CODEX_ID, isUserThread, RolloutScanner, rolloutThreadId, SessionIndex, uuidV7Ms } from "./codex-rollout.js";
+import { GitReader } from "./git.js";
+import { onceLogger } from "./log.js";
+export const codexResumeCommand = (id) => `codex resume ${id}`;
+export function defaultCodexDir(env = process.env) {
+    return env.CODEX_HOME || join(homedir(), ".codex");
+}
+function defaultIsCodexSessionProcess(procRoot) {
+    const table = procRoot ? createProcessTable({ platform: "linux", procRoot }) : createProcessTable();
+    return async (pid) => {
+        const argv = await table.argv(pid);
+        return !!argv && looksLikeCodex(argv, await table.exe(pid)) && !isDaemon(argv);
+    };
+}
+const pad = (n) => String(n).padStart(2, "0");
+export { uuidV7Ms };
+export function codexSession(id, r, opts) {
+    const live = opts.live;
+    const cwd = r?.cwd ?? live?.cwd ?? "";
+    const s = {
+        id,
+        source: "codex",
+        cwd,
+        repo: basename(cwd),
+        status: live ? (r?.turnOpen ? "busy" : "idle") : "dormant",
+        // The newest line, else the file's mtime, else (a fresh thread has no rollout yet) the id's creation time.
+        lastActivityAt: r?.lastTimestamp || Math.floor(opts.mtimeMs ?? 0) || (uuidV7Ms(id) ?? 0),
+        cacheTtlMs: 0,
+        cacheExpiresAt: 0,
+        live: !!live,
+        prLinks: [],
+        resumeCommand: codexResumeCommand(id),
+    };
+    if (live)
+        s.pid = live.pid;
+    // Stop only where the process is known for sure, never a directory guess.
+    if (live?.exact)
+        s.canEnd = true;
+    const title = opts.name ?? r?.firstUserMessage;
+    if (title)
+        s.title = title;
+    if (r?.branch)
+        s.branch = r.branch;
+    const git = opts.git;
+    if (git) {
+        s.repo = basename(git.root);
+        s.repoRoot = git.root;
+        // A live session is on the repo's current branch; a parked one keeps the one it recorded.
+        if (live || !s.branch)
+            s.branch = git.branch;
+    }
+    return s;
+}
+export function createCodexSource(opts = {}) {
+    const codexDir = opts.codexDir ?? defaultCodexDir();
+    const sessionsDir = join(codexDir, "sessions");
+    const lockDir = join(codexDir, "thread-writer-locks");
+    const scanProcs = opts.scan ?? ((dir) => scanCodexProcesses(dir, opts.procRoot));
+    const pollMs = opts.pollMs ?? 2000;
+    const cacheMs = opts.cacheMs ?? pollMs;
+    const debounceMs = opts.debounceMs ?? 200;
+    const maxDormant = opts.maxDormant ?? 200;
+    const warn = onceLogger(opts.log ?? console.warn);
+    const scanner = new RolloutScanner({ warn });
+    const git = new GitReader({ ttlMs: opts.gitTtlMs ?? 2000 });
+    const index = new SessionIndex(join(codexDir, "session_index.jsonl"));
+    const isCodexSessionProcess = opts.isCodexSessionProcess ?? defaultIsCodexSessionProcess(opts.procRoot);
+    const kill = opts.kill ?? ((pid, signal) => process.kill(pid, signal));
+    const pathById = new Map();
+    const realCwd = new Map();
+    async function real(p) {
+        let r = realCwd.get(p);
+        if (r === undefined) {
+            r = await realpath(p).catch(() => p);
+            realCwd.set(p, r);
+        }
+        return r;
+    }
+    async function subdirs(dir) {
+        try {
+            return (await readdir(dir, { withFileTypes: true }))
+                .filter((d) => d.isDirectory())
+                .map((d) => d.name)
+                .sort()
+                .reverse();
+        }
+        catch {
+            return [];
+        }
+    }
+    /** Rollout files, newest day first, stopping once `limit` files are found. */
+    async function rolloutFiles(limit = Infinity) {
+        const out = [];
+        for (const y of await subdirs(sessionsDir))
+            for (const m of await subdirs(join(sessionsDir, y)))
+                for (const d of await subdirs(join(sessionsDir, y, m))) {
+                    if (out.length >= limit)
+                        return out;
+                    const dir = join(sessionsDir, y, m, d);
+                    const names = await readdir(dir).catch(() => []);
+                    const day = [];
+                    for (const name of names) {
+                        const id = rolloutThreadId(name);
+                        if (!id)
+                            continue;
+                        const path = join(dir, name);
+                        const st = await stat(path).catch(() => undefined);
+                        if (!st?.isFile())
+                            continue;
+                        // A thread's plain file wins over a compressed twin.
+                        const prev = day.find((f) => f.id === id);
+                        if (prev && !prev.path.endsWith(".zst"))
+                            continue;
+                        if (prev)
+                            day.splice(day.indexOf(prev), 1);
+                        day.push({ id, path, mtimeMs: st.mtimeMs });
+                    }
+                    for (const f of day)
+                        pathById.set(f.id, f.path);
+                    out.push(...day);
+                }
+        return out;
+    }
+    /** id → when a full walk last failed to find it (fresh threads have no rollout yet). */
+    const missing = new Map();
+    async function findRollout(id) {
+        const cached = pathById.get(id);
+        if (cached && (await stat(cached).catch(() => undefined)))
+            return cached;
+        pathById.delete(id);
+        // A v7 thread id carries its creation time, and the file sits in that
+        // (local) day's directory: look there first, a day either side for safety.
+        const created = uuidV7Ms(id);
+        if (created !== undefined) {
+            for (const delta of [0, -1, 1]) {
+                const d = new Date(created + delta * 86_400_000);
+                const dir = join(sessionsDir, String(d.getFullYear()), pad(d.getMonth() + 1), pad(d.getDate()));
+                const names = await readdir(dir).catch(() => []);
+                const hits = names.filter((n) => rolloutThreadId(n) === id).sort((a, b) => Number(a.endsWith(".zst")) - Number(b.endsWith(".zst")));
+                if (hits[0]) {
+                    pathById.set(id, join(dir, hits[0]));
+                    return pathById.get(id);
+                }
+            }
+        }
+        const failedAt = missing.get(id);
+        if (failedAt !== undefined && Date.now() - failedAt < 60_000)
+            return undefined;
+        await rolloutFiles();
+        if (!pathById.has(id))
+            missing.set(id, Date.now());
+        else
+            missing.delete(id);
+        return pathById.get(id);
+    }
+    async function liveThreads() {
+        const scan = await scanProcs(lockDir).catch((e) => {
+            warn(`excaliagent: reading Codex processes failed: ${e.message}`, "codex-scan");
+            return undefined;
+        });
+        if (!scan?.locks.length)
+            return [];
+        const infos = new Map();
+        for (const l of scan.locks) {
+            const path = await findRollout(l.threadId);
+            infos.set(l.threadId, path ? await scanner.scan(path) : undefined);
+        }
+        const cwds = new Map();
+        for (const [id, r] of infos)
+            cwds.set(id, r?.cwd ? await real(r.cwd) : undefined);
+        // A thread without a rollout yet is a fresh TUI's; a thread with one must be a person's.
+        return assignLive(scan.locks, scan.procs, (id) => cwds.get(id), (id) => {
+            const r = infos.get(id);
+            return !r || isUserThread(r);
+        }, (id) => infos.get(id)?.lastTimestamp);
+    }
+    /** One look at the live threads and their sessions, reused for cacheMs. */
+    let look;
+    let looking;
+    async function current(fresh = false) {
+        if (!fresh && look && Date.now() - look.at < cacheMs)
+            return look;
+        looking ??= (async () => {
+            const threads = await liveThreads();
+            const sessions = await build(threads);
+            look = { at: Date.now(), threads, sessions };
+            return look;
+        })().finally(() => (looking = undefined));
+        return looking;
+    }
+    async function list() {
+        return (await current()).sessions;
+    }
+    async function build(threads) {
+        const names = await index.read();
+        const out = [];
+        for (const live of threads) {
+            if (!CODEX_ID.test(live.threadId))
+                continue;
+            const path = pathById.get(live.threadId);
+            const r = path ? await scanner.scan(path) : undefined;
+            const st = path && !r?.lastTimestamp ? await stat(path).catch(() => undefined) : undefined;
+            const cwd = r?.cwd ?? live.cwd;
+            const name = names.get(live.threadId);
+            out.push(codexSession(live.threadId, r, {
+                live,
+                ...(st ? { mtimeMs: st.mtimeMs } : {}),
+                ...(name ? { name } : {}),
+                ...(cwd ? { git: await git.get(cwd) } : {}),
+            }));
+        }
+        return out.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+    }
+    async function dormant(f, r, names) {
+        const name = names.get(f.id);
+        return codexSession(f.id, r, {
+            mtimeMs: f.mtimeMs,
+            ...(name ? { name } : {}),
+            ...(r.cwd ? { git: await git.get(r.cwd) } : {}),
+        });
+    }
+    async function listDormant() {
+        const live = new Set((await current()).threads.map((t) => t.threadId));
+        const names = await index.read();
+        const files = (await rolloutFiles()).filter((f) => !live.has(f.id)).sort((a, b) => b.mtimeMs - a.mtimeMs);
+        const out = [];
+        for (const f of files) {
+            if (out.length >= maxDormant)
+                break;
+            if (!CODEX_ID.test(f.id))
+                continue;
+            const r = await scanner.scan(f.path);
+            if (!r || !isUserThread(r))
+                continue;
+            out.push(await dormant(f, r, names));
+        }
+        return out;
+    }
+    async function get(id) {
+        if (!CODEX_ID.test(id))
+            return undefined;
+        id = id.toLowerCase();
+        const live = (await list()).find((s) => s.id === id);
+        if (live)
+            return live;
+        const path = await findRollout(id);
+        const st = path ? await stat(path).catch(() => undefined) : undefined;
+        const r = path ? await scanner.scan(path) : undefined;
+        if (!path || !st || !r)
+            return undefined;
+        return dormant({ id, path, mtimeMs: st.mtimeMs }, r, await index.read());
+    }
+    function watch(cb) {
+        let closed = false;
+        let running = false;
+        let again = false;
+        let last;
+        let debounce;
+        let watcher;
+        const refresh = async () => {
+            if (running) {
+                again = true;
+                return;
+            }
+            running = true;
+            try {
+                do {
+                    again = false;
+                    const sessions = (await current(true)).sessions;
+                    if (closed)
+                        return;
+                    const key = JSON.stringify(sessions);
+                    if (key !== last) {
+                        last = key;
+                        try {
+                            cb(sessions);
+                        }
+                        catch (e) {
+                            warn(`excaliagent: session watcher callback threw: ${e.message}`, "codex-watch-callback");
+                        }
+                    }
+                } while (again && !closed);
+            }
+            catch (e) {
+                warn(`excaliagent: refreshing Codex sessions failed: ${e.message}`, "codex-refresh");
+            }
+            finally {
+                running = false;
+            }
+        };
+        const kick = () => {
+            if (closed)
+                return;
+            clearTimeout(debounce);
+            debounce = setTimeout(() => void refresh(), debounceMs);
+        };
+        // Lock files come and go as threads load and unload: react at once.
+        const ensureWatcher = () => {
+            if (watcher || closed)
+                return;
+            try {
+                watcher = fsWatch(lockDir, kick);
+                watcher.on("error", () => {
+                    watcher?.close();
+                    watcher = undefined;
+                });
+            }
+            catch {
+                watcher = undefined; // dir missing; the poll retries
+            }
+        };
+        ensureWatcher();
+        // Turn starts and ends are only in the rollouts: poll.
+        const poll = setInterval(() => {
+            ensureWatcher();
+            void refresh();
+        }, pollMs);
+        void refresh();
+        return () => {
+            closed = true;
+            clearInterval(poll);
+            clearTimeout(debounce);
+            watcher?.close();
+            watcher = undefined;
+        };
+    }
+    /**
+     * SIGTERM to the TUI (or `codex exec`), as when its terminal closes. The thread
+     * stays on disk and can be resumed. Only for exact matches: a thread matched
+     * by directory is "unsupported", so a guess never ends a process.
+     */
+    async function end(id) {
+        if (!CODEX_ID.test(id))
+            return "not-live";
+        const t = (await current()).threads.find((x) => x.threadId === id.toLowerCase());
+        if (!t)
+            return "not-live";
+        if (!t.exact)
+            return "unsupported";
+        if (!(await isCodexSessionProcess(t.pid).catch(() => false)))
+            return "not-live";
+        try {
+            kill(t.pid, "SIGTERM");
+        }
+        catch {
+            return "not-live";
+        }
+        return "ended";
+    }
+    return { list, listDormant, get, watch, end };
+}

package/dist/server/sources/jsonl.js

@@ -0,0 +1,142 @@
+// Incremental JSONL folding, shared by the harness adapters: keeps a byte
+// offset per file and parses only appended bytes (a partial trailing line is
+// kept for the next scan; a replaced, shrunk or rewritten file is rescanned).
+// Each adapter supplies a fold: what to keep from a line, never the content.
+// `.zst` files (Codex compresses old rollouts) are read whole and decompressed
+// when this Node has zstd (22.15+), else skipped with a warning.
+import { open, readFile, stat } from "node:fs/promises";
+import * as zlib from "node:zlib";
+import { onceLogger } from "./log.js";
+const CHUNK = 1 << 20;
+const NL = 0x0a;
+const zstdDecompress = zlib.zstdDecompress;
+/** A decompressed rollout larger than this is skipped (Codex rollouts are a few MB). */
+export const MAX_DECOMPRESSED = 256 * 1024 * 1024;
+function unzstd(data) {
+    return new Promise((resolve, reject) => zstdDecompress(data, { maxOutputLength: MAX_DECOMPRESSED }, (err, out) => (err ? reject(err) : resolve(out))));
+}
+export class JsonlScanner {
+    files = new Map();
+    inflight = new Map();
+    warn;
+    noun;
+    fold;
+    constructor(fold, opts = {}) {
+        this.fold = fold;
+        this.warn = opts.warn ?? onceLogger(opts.log ?? console.warn);
+        this.noun = opts.noun ?? "file";
+    }
+    /** Scans new bytes of `path` and returns the folded info, or undefined if the file is gone. */
+    scan(path) {
+        const running = this.inflight.get(path);
+        if (running)
+            return running;
+        const p = this.doScan(path).finally(() => this.inflight.delete(path));
+        this.inflight.set(path, p);
+        return p;
+    }
+    /** Read bookkeeping, for tests. */
+    stats(path) {
+        const s = this.files.get(path);
+        return s && { offset: s.offset, bytesRead: s.bytesRead };
+    }
+    forget(path) {
+        this.files.delete(path);
+    }
+    async doScan(path) {
+        let st;
+        try {
+            st = await stat(path);
+        }
+        catch {
+            this.files.delete(path);
+            return undefined;
+        }
+        let s = this.files.get(path);
+        const replaced = !s || s.ino !== st.ino || s.dev !== st.dev;
+        const shrunk = !!s && st.size < s.offset;
+        // Same size but touched: rewritten in place (appends always grow the file).
+        const rewritten = !!s && st.size === s.offset && st.mtimeMs !== s.mtimeMs;
+        if (!s || replaced || shrunk || rewritten) {
+            s = { ino: st.ino, dev: st.dev, mtimeMs: st.mtimeMs, offset: 0, bytesRead: s?.bytesRead ?? 0, partial: Buffer.alloc(0), acc: this.fold.empty() };
+            this.files.set(path, s);
+        }
+        if (st.size > s.offset) {
+            try {
+                if (path.endsWith(".zst"))
+                    await this.readCompressed(path, s, st.size);
+                else
+                    await this.readFrom(path, s, st.size);
+            }
+            catch (e) {
+                this.warn(`excaliagent: cannot read ${this.noun} ${path}: ${e.message}`, path);
+            }
+        }
+        s.mtimeMs = st.mtimeMs;
+        return this.fold.snapshot(s.acc);
+    }
+    async readFrom(path, s, end) {
+        const fh = await open(path, "r");
+        try {
+            const buf = Buffer.allocUnsafe(Math.min(CHUNK, end - s.offset));
+            while (s.offset < end) {
+                const { bytesRead } = await fh.read(buf, 0, Math.min(buf.length, end - s.offset), s.offset);
+                if (bytesRead === 0)
+                    break;
+                s.offset += bytesRead;
+                s.bytesRead += bytesRead;
+                this.consume(path, s, buf.subarray(0, bytesRead));
+            }
+        }
+        finally {
+            await fh.close();
+        }
+    }
+    /** A compressed file is immutable: decompress it once, whole. */
+    async readCompressed(path, s, size) {
+        s.offset = size;
+        if (!zstdDecompress) {
+            this.warn("excaliagent: this Node has no zstd (needs 22.15+); compressed .zst files are skipped", "no-zstd");
+            return;
+        }
+        const data = await readFile(path);
+        s.bytesRead += data.length;
+        this.consume(path, s, await unzstd(data));
+        if (s.partial.length) {
+            this.parseLine(path, s.acc, s.partial);
+            s.partial = Buffer.alloc(0);
+        }
+    }
+    consume(path, s, chunk) {
+        let start = 0;
+        let nl = chunk.indexOf(NL, start);
+        while (nl !== -1) {
+            const piece = chunk.subarray(start, nl);
+            const line = s.partial.length ? Buffer.concat([s.partial, piece]) : piece;
+            s.partial = Buffer.alloc(0);
+            this.parseLine(path, s.acc, line);
+            start = nl + 1;
+            nl = chunk.indexOf(NL, start);
+        }
+        if (start < chunk.length)
+            s.partial = Buffer.concat([s.partial, chunk.subarray(start)]);
+    }
+    parseLine(path, acc, line) {
+        if (line.length === 0)
+            return;
+        const text = line.toString("utf8");
+        this.fold.raw?.(acc, text);
+        if (!this.fold.interesting(text))
+            return;
+        let o;
+        try {
+            o = JSON.parse(text);
+        }
+        catch {
+            this.warn(`excaliagent: skipping malformed line(s) in ${path}`, path);
+            return;
+        }
+        if (o && typeof o === "object" && !Array.isArray(o))
+            this.fold.apply(acc, o);
+    }
+}

package/dist/server/sources/multi.js

@@ -0,0 +1,53 @@
+const byId = (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+/** Fail soft: one broken adapter must not take the others down. */
+async function settle(work, warn, what) {
+    try {
+        return await work;
+    }
+    catch (e) {
+        warn(`excaliagent: ${what} failed: ${e.message}`);
+        return [];
+    }
+}
+export function combineSources(sources, warn = console.warn) {
+    if (sources.length === 1)
+        return sources[0];
+    return {
+        async list() {
+            const lists = await Promise.all(sources.map((s) => settle(s.list(), warn, "listing sessions")));
+            return lists.flat().sort(byId);
+        },
+        watch(cb) {
+            const latest = sources.map(() => undefined);
+            const emit = () => cb(latest.flatMap((l) => l ?? []).sort(byId));
+            const offs = sources.map((s, i) => s.watch((sessions) => {
+                latest[i] = sessions;
+                emit();
+            }));
+            return () => {
+                for (const off of offs)
+                    off();
+            };
+        },
+        async listDormant() {
+            const lists = await Promise.all(sources.map((s) => settle(s.listDormant(), warn, "listing dormant sessions")));
+            return lists.flat().sort((a, b) => b.lastActivityAt - a.lastActivityAt);
+        },
+        async get(id) {
+            for (const s of sources) {
+                const found = await s.get(id).catch(() => undefined);
+                if (found)
+                    return found;
+            }
+            return undefined;
+        },
+        async end(id) {
+            for (const s of sources) {
+                const found = await s.get(id).catch(() => undefined);
+                if (found)
+                    return s.end ? s.end(id) : "unsupported";
+            }
+            return "not-live";
+        },
+    };
+}