@cruxy/cli 1.9.0 → 1.11.0

package/dist/jobs/log-store.js

@@ -0,0 +1,505 @@
+import { appendFileSync, closeSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, statSync, } from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+import { APP_VERSION } from "../constants.js";
+import { SESSION_FILE_EXT, reservedDir } from "../session/paths.js";
+/**
+ * Persisted background-job logs (#172 item 1), under the `subagents/` subtree
+ * `paths.ts` reserved for exactly this:
+ *
+ * ```
+ * ~/.cruxy/projects/<project>/
+ *   <session-id>.jsonl              ← the conversation (P2)
+ *   subagents/
+ *     <session-id>.<job-id>.jsonl   ← one background job's output (here)
+ * ```
+ *
+ * ## What this is, and what it is emphatically not
+ *
+ * It is a POST-MORTEM. It is not "your job survives a restart", and the
+ * difference is not pedantry — it is the first bug someone would otherwise
+ * file. A job is an in-process {@link ../agent/loop.js} call, `run.ts` calls
+ * `cancelAll("session exit")` on the way out, and `orphan.test.ts` pins that
+ * this kill-tree's the whole process group. So a job never outlives the session
+ * that dispatched it. What this store lets you read afterwards is what the job
+ * SAID before it stopped, which is the one thing that used to die with the
+ * process — its file mutations already survived as a checkpoint set
+ * (`rollback.ts` finds it by `runId === job.id`) and its spend already survived
+ * as a usage record keyed by the same id.
+ *
+ * ## Streamed, never flushed at exit
+ *
+ * Every line is written when it happens, through the single sink
+ * `JobManager.log` already funnels all of them through. The alternative —
+ * serialising the ring buffer once on the way out — was rejected twice over:
+ *
+ *  - it would live in the same `finally` that `process.exit` skips. The
+ *    SIGINT/SIGTERM/SIGHUP handlers in `tui/restore.ts` and
+ *    `utils/child-tree.ts` both exit the process directly, so a closed terminal
+ *    window, a dropped ssh session or a plain `kill` runs no `finally` at all.
+ *    A log that only persists on the orderly path persists exactly the runs
+ *    that did not need persisting and loses every one that did.
+ *  - the whole point of the issue is to remove the ring buffer's
+ *    drop-on-overflow. Writing the BUFFER at exit would persist the truncated
+ *    tail, i.e. ship the data loss to disk verbatim.
+ *
+ * ## Durability: the same rules as `session/log.ts`, unchanged
+ *
+ * One `appendFileSync` of one newline-terminated JSON line, `O_APPEND` so
+ * interleaving writers cannot corrupt each other, `0600` from the first line,
+ * and a `broken` latch so an unwritable disk goes inert after ONE warning
+ * rather than warning per line. A torn final line is skipped by the reader
+ * instead of prevented by the writer. Nothing here is temp-then-rename: that
+ * makes a whole-file REWRITE atomic, which is the wrong tool for a file that is
+ * only ever appended to.
+ */
+/** Extension of a persisted job log. Matches the session log's, deliberately. */
+export const JOB_LOG_EXT = ".jsonl";
+/** Schema version of the `job` header record. */
+export const JOB_LOG_VERSION = 1;
+/**
+ * How much of a file's tail is read to find its terminal record.
+ *
+ * The sweep needs one bit per file — did this job finish, and how — and reading
+ * a capped-out 20k-line log in full to learn it would put megabytes of read on
+ * the startup path that #255 spent its time removing. The terminal record is
+ * the last line by construction (nothing logs into a job after its `finally`,
+ * and both `cancelAll` and `abortSiblings` skip jobs that are already
+ * terminal), so a few KB always contains it. The tail is SCANNED for the last
+ * `end` record rather than assuming the final line is one, so a future writer
+ * that appends something after it does not silently turn every finished job
+ * into an interrupted one.
+ */
+const TAIL_BYTES = 8192;
+// ── the record shapes ───────────────────────────────────────────────────────
+//
+// `.passthrough()` throughout, matching `session/types.ts`: a newer cruxy's
+// extra fields must never make an older one discard a log it can otherwise
+// read.
+const JobHeaderSchema = z
+    .object({
+    kind: z.literal("job"),
+    version: z.number().int().positive(),
+    jobId: z.string(),
+    sessionId: z.string(),
+    label: z.string(),
+    cwd: z.string(),
+    startedAt: z.string(),
+    cliVersion: z.string().optional(),
+})
+    .passthrough();
+const JobLineSchema = z
+    .object({
+    kind: z.literal("line"),
+    atMs: z.number(),
+    stream: z.enum(["out", "err"]),
+    text: z.string(),
+})
+    .passthrough();
+const JobTruncatedSchema = z
+    .object({
+    kind: z.literal("truncated"),
+    atMs: z.number(),
+    cap: z.number().int().nonnegative(),
+})
+    .passthrough();
+const JobEndSchema = z
+    .object({
+    kind: z.literal("end"),
+    at: z.string(),
+    status: z.string(),
+    iterations: z.number().int().nonnegative().optional(),
+    error: z.string().optional(),
+})
+    .passthrough();
+/**
+ * A reader's verdict on a job that has no terminal record.
+ *
+ * NOT a {@link JobStatus}: no job is ever IN this state, and adding it to the
+ * lifecycle enum would put a case into every `switch` over a job's real states
+ * for a value the manager can never produce. It is what a FILE looks like, not
+ * what a job was.
+ */
+export const INTERRUPTED = "interrupted";
+// ── paths ───────────────────────────────────────────────────────────────────
+/** `~/.cruxy/projects/<project>/subagents` */
+export function jobLogsDir(cwd) {
+    return reservedDir(cwd, "subagents");
+}
+/**
+ * Filename-safe form of an id. Everything outside `[A-Za-z0-9_-]` collapses to
+ * `-`, so a name can never contain a path separator or the `.` that separates
+ * the two ids.
+ *
+ * Like {@link ../session/paths.js projectKey}, this is an INDEX and is not
+ * reversible. The true ids are recorded in the header record, which is what a
+ * reader reports. Real ids (a UUID session, `job-<n>-<rand>`) are already safe;
+ * this exists so a test seam's `idFactory` cannot write outside the subtree.
+ */
+export function idKey(id) {
+    const key = id.replace(/[^A-Za-z0-9_-]+/g, "-");
+    return key === "" ? "unknown" : key;
+}
+/** `~/.cruxy/projects/<project>/subagents/<session-id>.<job-id>.jsonl` */
+export function jobLogPath(cwd, sessionId, jobId) {
+    return path.join(jobLogsDir(cwd), `${idKey(sessionId)}.${idKey(jobId)}${JOB_LOG_EXT}`);
+}
+/**
+ * One background job's on-disk log.
+ *
+ * ## The per-job cap, and why one is needed at all
+ *
+ * Retention (#257) bounds how many session-shaped things the tree KEEPS. It
+ * cannot bound how big ONE of them gets, and a background job is the first
+ * writer here that can produce unbounded output on its own: it runs a full
+ * agent loop off screen, and a job stuck in a tool-call loop emits lines for as
+ * long as its budget lasts with nobody watching. Streaming it straight to disk
+ * with no ceiling is precisely the second unbounded writer under `~/.cruxy`
+ * that #257 was filed to prevent — the ring buffer's bound was doing this job
+ * implicitly, and taking the buffer off the critical path takes that bound away
+ * with it.
+ *
+ * So the file gets its own: `jobs.logFileLines`, default 20000, floored at
+ * `jobs.logBufferLines` because a file that kept LESS than memory would make
+ * persisting strictly worse than not persisting.
+ *
+ * ## The truncation marker is written at the cap, not at the end
+ *
+ * When the cap is reached a `truncated` record goes down immediately and
+ * further lines are dropped. It is not deferred to the terminal record and it
+ * does not carry a running count of what came after, for the same reason the
+ * rest of this file streams: a marker that only lands on the orderly path is
+ * absent from every log that was killed, which is where the truncation is most
+ * likely to matter. An exact count of unrecorded lines is not knowable after a
+ * SIGKILL either, so promising one would be the lie the marker exists to
+ * prevent.
+ *
+ * The terminal record is written EVEN PAST THE CAP. Suppressing it there would
+ * make every capped-out job read as interrupted, which is the one thing the
+ * reader's rule must never get wrong.
+ */
+export class JobLogWriter {
+    file;
+    cap;
+    logger;
+    /** Set once a write fails: the writer goes inert rather than warning per line. */
+    broken = false;
+    written = 0;
+    truncated = false;
+    closed = false;
+    /**
+     * The header, held until the job records something — the same lazy-meta rule
+     * `session/log.ts` adopted in #269, for the same reason: a writer that is
+     * constructed and never used should leave nothing on disk. In practice
+     * `dispatch` logs immediately, so this flushes on the job's first breath.
+     */
+    pendingHeader;
+    constructor(opts) {
+        this.file = opts.file ?? jobLogPath(opts.cwd, opts.sessionId, opts.jobId);
+        this.cap = Math.max(1, Math.floor(opts.cap));
+        this.logger = opts.logger;
+        this.pendingHeader = {
+            kind: "job",
+            version: JOB_LOG_VERSION,
+            jobId: opts.jobId,
+            sessionId: opts.sessionId,
+            label: opts.label,
+            cwd: opts.cwd,
+            startedAt: new Date().toISOString(),
+            cliVersion: APP_VERSION,
+        };
+    }
+    /** Append one log line, honouring the cap. */
+    line(line) {
+        if (this.broken || this.closed)
+            return;
+        if (this.written >= this.cap) {
+            if (this.truncated)
+                return;
+            this.truncated = true;
+            this.writeRecord({
+                kind: "truncated",
+                atMs: line.atMs,
+                cap: this.cap,
+            });
+            return;
+        }
+        if (this.writeRecord({ kind: "line", ...line }))
+            this.written++;
+    }
+    /**
+     * Write the terminal record. Called from the one `finally` that runs on every
+     * path a job can actually terminate on — so the presence of this record is
+     * exactly the fact {@link readJobLog} reads back. Idempotent.
+     */
+    end(status, iterations, error) {
+        if (this.broken || this.closed)
+            return;
+        this.closed = true;
+        this.writeRecord({
+            kind: "end",
+            at: new Date().toISOString(),
+            status,
+            iterations,
+            ...(error !== undefined ? { error } : {}),
+        });
+    }
+    writeRecord(record) {
+        if (this.broken)
+            return false;
+        const header = this.pendingHeader;
+        if (header !== null) {
+            this.pendingHeader = null;
+            if (!this.writeLine(header))
+                return false;
+        }
+        return this.writeLine(record);
+    }
+    writeLine(record) {
+        try {
+            mkdirSync(path.dirname(this.file), { recursive: true });
+            // `mode` applies only at creation — 0600 from the first line, matching
+            // the session log, credentials, usage and memory.
+            appendFileSync(this.file, `${JSON.stringify(record)}\n`, { mode: 0o600 });
+            return true;
+        }
+        catch (err) {
+            this.broken = true;
+            this.logger?.warn(`background job log not being saved: ${err.message} ` +
+                `(the job continues, and \`/logs\` still reads it from memory)`);
+            return false;
+        }
+    }
+}
+/**
+ * Persisted job logs for `cwd`'s project, most-recently-modified first.
+ *
+ * `readdir` plus one `stat` each — no file is opened, exactly as
+ * {@link ../session/list.js sessionFilesByRecency} does for sessions, and for
+ * the same reason: recency is knowable from the directory alone, so the
+ * per-file read only happens for the rows a caller actually shows.
+ *
+ * The `.jsonl` filter and the `isFile` test are load-bearing, not tidiness:
+ * everything returned here is something the sweep will consider deleting, and
+ * `statSync` succeeds on a directory. A name that does not split into exactly
+ * two ids is skipped rather than guessed at — it was not written by this
+ * writer, and the sweep must not delete files it does not own.
+ */
+export function jobLogFilesByRecency(cwd) {
+    const dir = jobLogsDir(cwd);
+    let names;
+    try {
+        names = readdirSync(dir);
+    }
+    catch {
+        return []; // no jobs have ever run here — normal, not an error
+    }
+    const refs = [];
+    for (const name of names) {
+        if (!name.endsWith(JOB_LOG_EXT))
+            continue;
+        const stem = name.slice(0, -JOB_LOG_EXT.length);
+        const dot = stem.indexOf(".");
+        if (dot <= 0 || dot === stem.length - 1)
+            continue;
+        const sessionKey = stem.slice(0, dot);
+        const jobKey = stem.slice(dot + 1);
+        if (jobKey.includes("."))
+            continue;
+        const file = path.join(dir, name);
+        try {
+            const stat = statSync(file);
+            if (!stat.isFile())
+                continue;
+            refs.push({
+                file,
+                sessionKey,
+                jobKey,
+                mtimeMs: stat.mtimeMs,
+                size: stat.size,
+            });
+        }
+        catch {
+            continue; // deleted mid-listing
+        }
+    }
+    refs.sort((a, b) => b.mtimeMs - a.mtimeMs || a.file.localeCompare(b.file));
+    return refs;
+}
+/** Refs whose job id equals `id` or begins with it (the `--resume` rule). */
+export function matchJobLogs(refs, id) {
+    const exact = refs.filter((r) => r.jobKey === id);
+    if (exact.length > 0)
+        return exact;
+    return refs.filter((r) => r.jobKey.startsWith(id));
+}
+/**
+ * Read one persisted log back.
+ *
+ * ## RECONCILIATION BY ABSENCE — the whole rule, and there is no other
+ *
+ * A job is {@link INTERRUPTED} if and only if its file carries no `end`
+ * record. `JobManager.execute`'s `finally` writes one on every path that runs
+ * at all; the paths that do not run are precisely the ones that killed the
+ * process without warning. So "no terminal record" means "the process died
+ * mid-job", every time, with nothing to maintain.
+ *
+ * WHAT THIS REPLACES, and why neither alternative works here:
+ *
+ *  - A PID + start-time liveness marker is category-wrong. A job is not a
+ *    process — it is an in-process `runAgent` call — so the pid on disk would
+ *    be the CLI's, and "that pid is alive" only ever tells you the SESSION is
+ *    alive, which every caller already knows more cheaply. It also adds
+ *    writable state that a crash can leave stale, which is the disease and not
+ *    the cure.
+ *  - A STARTUP SWEEP that rewrites non-terminal logs cannot be the load-bearing
+ *    rule, because a reader can open a log before any restart happens — `cruxy
+ *    logs` from a second terminal while the writing session is still running.
+ *    It would see a `running` job and have nothing to distinguish that from a
+ *    stale one. Deriving from absence answers the live case and the dead case
+ *    with the same sentence.
+ *
+ * Tolerant throughout, matching `replay.ts`: a torn or unknown line is skipped
+ * rather than failing the read, because a log that was killed mid-write is the
+ * exact case this function exists to serve. Returns null only when the file has
+ * no readable header — that is not a log this writer produced.
+ */
+export function readJobLog(file) {
+    let raw;
+    try {
+        raw = readFileSync(file, "utf8");
+    }
+    catch {
+        return null;
+    }
+    let header = null;
+    const lines = [];
+    let end = null;
+    let truncatedAt;
+    for (const text of raw.split("\n")) {
+        if (text.trim() === "")
+            continue;
+        let json;
+        try {
+            json = JSON.parse(text);
+        }
+        catch {
+            continue; // torn line — skip, same tolerance as replay
+        }
+        if (header === null) {
+            const h = JobHeaderSchema.safeParse(json);
+            if (h.success)
+                header = h.data;
+            continue; // nothing before the header is ours
+        }
+        const line = JobLineSchema.safeParse(json);
+        if (line.success) {
+            lines.push({
+                atMs: line.data.atMs,
+                stream: line.data.stream,
+                text: line.data.text,
+            });
+            continue;
+        }
+        const trunc = JobTruncatedSchema.safeParse(json);
+        if (trunc.success) {
+            truncatedAt = trunc.data.cap;
+            continue;
+        }
+        const e = JobEndSchema.safeParse(json);
+        if (e.success)
+            end = e.data;
+    }
+    if (header === null)
+        return null;
+    return {
+        jobId: header.jobId,
+        sessionId: header.sessionId,
+        label: header.label,
+        cwd: header.cwd,
+        startedAt: header.startedAt,
+        status: terminalStatus(end),
+        ...(end?.at !== undefined ? { endedAt: end.at } : {}),
+        ...(end?.iterations !== undefined ? { iterations: end.iterations } : {}),
+        ...(end?.error !== undefined ? { error: end.error } : {}),
+        lines,
+        ...(truncatedAt !== undefined ? { truncatedAt } : {}),
+    };
+}
+/**
+ * The terminal status of a log WITHOUT reading it in full — the tail only.
+ *
+ * This is what the sweep asks of every file it considers, so it must not cost a
+ * full read: see {@link TAIL_BYTES}. Returns {@link INTERRUPTED} when no `end`
+ * record is in the tail, which for a file this writer produced means there is
+ * none at all.
+ */
+export function readJobLogTerminal(file) {
+    let tail;
+    try {
+        tail = readTail(file, TAIL_BYTES);
+    }
+    catch {
+        return INTERRUPTED;
+    }
+    let end = null;
+    for (const text of tail.split("\n")) {
+        if (text.trim() === "")
+            continue;
+        let json;
+        try {
+            json = JSON.parse(text);
+        }
+        catch {
+            continue; // the first line of a tail read is torn by definition
+        }
+        const e = JobEndSchema.safeParse(json);
+        if (e.success)
+            end = e.data;
+    }
+    return terminalStatus(end);
+}
+/** The `end` record's status, narrowed — or {@link INTERRUPTED} when absent. */
+function terminalStatus(end) {
+    if (end === null)
+        return INTERRUPTED;
+    // The status is read back as a plain string and narrowed here rather than
+    // parsed as an enum, so a log written by a build that knows a status this one
+    // does not is reported as written instead of being discarded.
+    return end.status;
+}
+/** The last `bytes` of a file, decoded as UTF-8. Short files read whole. */
+function readTail(file, bytes) {
+    const fd = openSync(file, "r");
+    try {
+        const size = fstatSync(fd).size;
+        const length = Math.min(size, bytes);
+        const buf = Buffer.alloc(length);
+        readSync(fd, buf, 0, length, size - length);
+        return buf.toString("utf8");
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/**
+ * The {@link idKey}s of every session file currently in `cwd`'s project.
+ *
+ * KEYS AND NOT IDS, because that is the only comparison that is safe in both
+ * directions. A log's filename carries a sanitized session id, so comparing it
+ * against a raw session id would mismatch for any id `idKey` altered — and a
+ * mismatch here does not spare a file, it DELETES one, by concluding the owning
+ * session is gone. Sanitizing both sides makes the two agree by construction.
+ * (For a real session id — a UUID — `idKey` is the identity function; this
+ * matters only for a test seam or a future id format.)
+ *
+ * Derived from the same scan `prune.ts` already ran, so the sweep adds no
+ * second directory walk over the session files.
+ */
+export function sessionKeysPresent(files) {
+    const keys = new Set();
+    for (const { file } of files) {
+        keys.add(idKey(path.basename(file, SESSION_FILE_EXT)));
+    }
+    return keys;
+}