@pentoshi/clai 3.6.0 → 3.7.3

package/dist/tools/jobs.js

@@ -1,189 +1,468 @@
-import { spawn } from "node:child_process";
-import { createWriteStream } from "node:fs";
-import { readFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { randomUUID } from "node:crypto";
+import { spawn, execFileSync } from "node:child_process";
+import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { mkdir, open, stat } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { redactSecrets } from "../llm/provider.js";
 import { safeCwd } from "../os/cwd.js";
 import { getJobsDir } from "../store/paths.js";
+function displayArg(value) {
+    return /^[A-Za-z0-9_./:=@+-]+$/.test(value)
+        ? value
+        : JSON.stringify(value);
+}
+function commandDisplay(command) {
+    if (typeof command === "string")
+        return command;
+    return command.display ?? [command.command, ...command.argv].map(displayArg).join(" ");
+}
+const PER_FILE_BYTES = 1024 * 1024;
+const TOTAL_STREAM_BYTES = 16 * 1024 * 1024;
+const DEFAULT_TAIL_BYTES = 8_000;
+const REGISTRY_FILE = "registry-v1.json";
+function processAlive(pid) {
+    if (!pid)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function processIdentity(pid) {
+    if (!pid || process.platform === "win32")
+        return undefined;
+    try {
+        const value = execFileSync("ps", ["-p", String(pid), "-o", "lstart=", "-o", "command="], {
+            encoding: "utf8",
+            timeout: 2_000,
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+        return value ? createHash("sha256").update(value).digest("hex") : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+class RotatingRedactedWriter {
+    receipt;
+    stream;
+    index = 0;
+    currentBytes = 0;
+    hash = createHash("sha256");
+    pending = "";
+    static REDACTION_OVERLAP_CHARS = 4096;
+    constructor(receipt) {
+        this.receipt = receipt;
+    }
+    append(raw) {
+        this.pending += Buffer.isBuffer(raw) ? raw.toString("utf8") : raw;
+        // Complete lines are safe to redact as a unit, including secrets split
+        // across arbitrary process data events. Keep an unterminated tail as the
+        // overlap window for the next event.
+        const lastNewline = this.pending.lastIndexOf("\n");
+        if (lastNewline >= 0) {
+            this.writeRedacted(this.pending.slice(0, lastNewline + 1));
+            this.pending = this.pending.slice(lastNewline + 1);
+        }
+        if (this.pending.length <= RotatingRedactedWriter.REDACTION_OVERLAP_CHARS)
+            return;
+        const flushLength = this.pending.length - RotatingRedactedWriter.REDACTION_OVERLAP_CHARS;
+        this.writeRedacted(this.pending.slice(0, flushLength));
+        this.pending = this.pending.slice(flushLength);
+    }
+    close() {
+        if (this.pending)
+            this.writeRedacted(this.pending);
+        this.pending = "";
+        this.stream?.end();
+        this.stream = undefined;
+    }
+    writeRedacted(source) {
+        const safe = redactSecrets(source);
+        if (safe !== source)
+            this.receipt.redacted = true;
+        let data = Buffer.from(safe, "utf8");
+        if (this.receipt.bytes >= TOTAL_STREAM_BYTES) {
+            this.receipt.droppedBytes += data.length;
+            return;
+        }
+        if (this.receipt.bytes + data.length > TOTAL_STREAM_BYTES) {
+            const keep = TOTAL_STREAM_BYTES - this.receipt.bytes;
+            this.receipt.droppedBytes += data.length - keep;
+            data = data.subarray(0, keep);
+        }
+        while (data.length > 0) {
+            if (!this.stream || this.currentBytes >= PER_FILE_BYTES)
+                this.rotate();
+            const room = PER_FILE_BYTES - this.currentBytes;
+            const part = data.subarray(0, room);
+            this.stream.write(part);
+            this.hash.update(part);
+            this.currentBytes += part.length;
+            this.receipt.bytes += part.length;
+            data = data.subarray(part.length);
+        }
+        this.receipt.sha256 = this.hash.copy().digest("hex");
+    }
+    rotate() {
+        this.stream?.end();
+        const path = this.index === 0 ? this.receipt.path : `${this.receipt.path}.${this.index}`;
+        this.index += 1;
+        this.currentBytes = 0;
+        this.receipt.chunks.push(path);
+        this.stream = createWriteStream(path, { flags: "w", mode: 0o600 });
+    }
+}
 export class JobManager {
+    jobsDir;
     jobs = new Map();
     processes = new Map();
-    streams = new Map();
+    writers = new Map();
     abortControllers = new Map();
-    registerJob(id, job, ac, process) {
+    registryPath;
+    constructor(jobsDir = getJobsDir()) {
+        this.jobsDir = jobsDir;
+        this.registryPath = join(this.jobsDir, REGISTRY_FILE);
+        this.loadAndReconcile();
+    }
+    registerJob(id, job, ac, child) {
         this.jobs.set(id, job);
-        if (ac) {
+        if (ac)
             this.abortControllers.set(id, ac);
-        }
-        if (process) {
-            this.processes.set(id, process);
-        }
+        if (child)
+            this.processes.set(id, child);
+        void this.persist();
     }
     updateJobStatus(id, status, exitCode) {
         const job = this.jobs.get(id);
-        if (job) {
-            job.status = status;
-            if (exitCode !== undefined) {
-                job.exitCode = exitCode;
-            }
+        if (!job)
+            return;
+        job.status = status;
+        if (exitCode !== undefined)
+            job.exitCode = exitCode;
+        if (!["running", "starting", "stopping"].includes(status))
             job.endedAt = new Date().toISOString();
-            this.abortControllers.delete(id);
-            this.processes.delete(id);
-        }
+        this.abortControllers.delete(id);
+        this.processes.delete(id);
+        void this.persist();
     }
     async startJob(command, options) {
+        if (options?.authorization?.expiresAt) {
+            const expiry = Date.parse(options.authorization.expiresAt);
+            if (Number.isFinite(expiry) && expiry <= Date.now()) {
+                return { ok: false, output: `Engagement authorization for ${options.authorization.target} has expired.`, exitCode: 1 };
+            }
+        }
         const id = randomUUID().slice(0, 8);
         const cwd = options?.cwd ?? safeCwd();
-        const dir = getJobsDir();
-        await mkdir(dir, { recursive: true });
-        const artifactPath = join(dir, `${new Date().toISOString().replace(/[:.]/g, "-")}-${id}.txt`);
-        const stream = createWriteStream(artifactPath, { flags: "w" });
-        stream.write(`$ ${command}\n\n`);
+        await mkdir(this.jobsDir, { recursive: true });
+        const prefix = `${new Date().toISOString().replace(/[:.]/g, "-")}-${id}`;
+        const stdoutArtifact = join(this.jobsDir, `${prefix}.stdout.log`);
+        const stderrArtifact = join(this.jobsDir, `${prefix}.stderr.log`);
+        const makeReceipt = (path) => ({ path, chunks: [], bytes: 0, droppedBytes: 0, redacted: false, sha256: "" });
+        const safeDisplay = redactSecrets(commandDisplay(command));
+        const job = {
+            id,
+            command: safeDisplay,
+            commandDisplay: safeDisplay,
+            cwd,
+            status: "starting",
+            startedAt: new Date().toISOString(),
+            artifactPath: stdoutArtifact,
+            stdoutArtifact,
+            stderrArtifact,
+            artifacts: { stdout: makeReceipt(stdoutArtifact), stderr: makeReceipt(stderrArtifact) },
+            redactionProfile: "provider-secrets-v1",
+            ownerSessionId: options?.ownerSessionId ?? "unknown",
+            ...(options?.name ? { name: options.name } : {}),
+            ...(options?.authorization ? { authorization: options.authorization } : {}),
+        };
+        this.jobs.set(id, job);
+        await this.persist();
         try {
             const detached = process.platform !== "win32";
-            const child = spawn(command, {
-                cwd,
-                detached,
-                shell: true,
-                stdio: ["ignore", "pipe", "pipe"],
-            });
-            const job = {
-                id,
-                command,
-                cwd,
-                pid: child.pid,
-                status: "running",
-                startedAt: new Date().toISOString(),
-                artifactPath,
-            };
-            this.jobs.set(id, job);
+            const child = typeof command === "string"
+                ? spawn(command, {
+                    cwd,
+                    detached,
+                    shell: true,
+                    stdio: ["ignore", "pipe", "pipe"],
+                })
+                : spawn(command.command, command.argv, {
+                    cwd,
+                    detached,
+                    shell: false,
+                    stdio: [command.stdinText === undefined ? "ignore" : "pipe", "pipe", "pipe"],
+                });
+            // Sensitive input is never copied into the job record, display, logs, or
+            // artifacts. Write it once to the child and close stdin immediately.
+            if (typeof command !== "string" && command.stdinText !== undefined) {
+                child.stdin?.end(command.stdinText);
+            }
+            job.pid = child.pid;
+            job.processGroupId = detached ? child.pid : undefined;
+            job.status = "running";
+            job.heartbeatAt = new Date().toISOString();
+            job.processIdentity = processIdentity(child.pid);
+            const stdout = new RotatingRedactedWriter(job.artifacts.stdout);
+            const stderr = new RotatingRedactedWriter(job.artifacts.stderr);
+            stdout.append(`$ ${job.commandDisplay}\n\n`);
             this.processes.set(id, child);
-            this.streams.set(id, stream);
-            child.stdout?.on("data", (chunk) => {
-                stream.write(chunk);
-            });
-            child.stderr?.on("data", (chunk) => {
-                stream.write(chunk);
-            });
-            child.on("close", (code) => {
-                job.status = "exited";
+            this.writers.set(id, { stdout, stderr });
+            const expiresAt = options?.authorization?.expiresAt ? Date.parse(options.authorization.expiresAt) : Number.NaN;
+            const expiryTimer = Number.isFinite(expiresAt) && expiresAt > Date.now()
+                ? setTimeout(() => { void this.stopJob(id, { graceMs: 1_000 }); }, expiresAt - Date.now())
+                : undefined;
+            expiryTimer?.unref?.();
+            child.stdout?.on("data", (chunk) => { stdout.append(chunk); job.heartbeatAt = new Date().toISOString(); void this.persist(); });
+            child.stderr?.on("data", (chunk) => { stderr.append(chunk); job.heartbeatAt = new Date().toISOString(); void this.persist(); });
+            child.on("close", (code, signal) => {
+                if (expiryTimer)
+                    clearTimeout(expiryTimer);
                 job.exitCode = code ?? undefined;
+                job.signal = signal ?? undefined;
+                job.status = signal ? "killed" : code === 0 ? "exited" : "failed";
                 job.endedAt = new Date().toISOString();
-                stream.end();
+                stdout.close();
+                stderr.close();
                 this.processes.delete(id);
-                this.streams.delete(id);
+                this.writers.delete(id);
+                void this.persist();
             });
             child.on("error", () => {
+                if (expiryTimer)
+                    clearTimeout(expiryTimer);
                 job.status = "failed";
                 job.endedAt = new Date().toISOString();
-                stream.end();
+                stdout.close();
+                stderr.close();
                 this.processes.delete(id);
-                this.streams.delete(id);
+                this.writers.delete(id);
+                void this.persist();
             });
-            // Detach so the child outlives the parent if needed
             child.unref();
+            await this.persist();
             return {
                 ok: true,
-                output: `Background job started: id=${id} pid=${child.pid ?? "?"}\nCommand: ${command}\nOutput: ${artifactPath}`,
+                output: `Background job started: id=${id} (canonical job ID) pid=${child.pid ?? "?"}\n` +
+                    `Use shell.tail {"id":"${id}"} or shell.jobs; do not use the artifact filename as the id.\n` +
+                    `Command: ${job.commandDisplay}\nArtifact: ${stdoutArtifact}`,
+                outputPath: stdoutArtifact,
+                backgroundJob: {
+                    id,
+                    status: job.status,
+                    artifactPath: stdoutArtifact,
+                    ...(options?.profile ? { profile: options.profile } : {}),
+                    ...(options?.estimatedSeconds !== undefined ? { estimatedSeconds: options.estimatedSeconds } : {}),
+                    nextOffset: 0,
+                },
             };
         }
         catch (error) {
-            stream.end();
-            return {
-                ok: false,
-                output: `Failed to start job: ${error instanceof Error ? error.message : String(error)}`,
-                exitCode: 1,
-            };
+            job.status = "failed";
+            job.endedAt = new Date().toISOString();
+            await this.persist();
+            return { ok: false, output: `Failed to start job: ${error instanceof Error ? error.message : String(error)}`, exitCode: 1 };
         }
     }
     listJobs() {
-        if (this.jobs.size === 0) {
+        if (this.jobs.size === 0)
             return { ok: true, output: "No background jobs." };
-        }
-        const lines = [];
+        const lines = [...this.jobs.values()].map((job) => {
+            const elapsedEnd = job.endedAt ? new Date(job.endedAt).getTime() : Date.now();
+            const elapsed = Math.max(0, Math.round((elapsedEnd - new Date(job.startedAt).getTime()) / 1000));
+            const health = job.status === "running" ? (processAlive(job.pid) ? "alive" : "unresponsive") : "terminal";
+            return `[${job.id}] ${job.status} health=${health} exit=${job.exitCode ?? "?"} ${elapsed}s  ${job.commandDisplay.slice(0, 60)}`;
+        });
+        return { ok: true, output: lines.join("\n") };
+    }
+    resolveJobId(input) {
+        if (this.jobs.has(input))
+            return input;
+        const inputBase = basename(input);
         for (const job of this.jobs.values()) {
-            const status = job.status === "running" ? "▶ running" : `${job.status} (exit=${job.exitCode ?? "?"})`;
-            const elapsed = job.endedAt
-                ? `${Math.round((new Date(job.endedAt).getTime() - new Date(job.startedAt).getTime()) / 1000)}s`
-                : `${Math.round((Date.now() - new Date(job.startedAt).getTime()) / 1000)}s ago`;
-            lines.push(`[${job.id}] ${status} ${elapsed}  ${job.command.slice(0, 60)}`);
+            const artifacts = [
+                job.artifactPath,
+                job.stdoutArtifact,
+                job.stderrArtifact,
+                ...job.artifacts.stdout.chunks,
+                ...job.artifacts.stderr.chunks,
+            ];
+            if (artifacts.some((path) => input === path || inputBase === basename(path)) ||
+                inputBase.includes(`-${job.id}.`)) {
+                return job.id;
+            }
         }
-        return { ok: true, output: lines.join("\n") };
+        return undefined;
     }
     getJob(id) {
-        return this.jobs.get(id);
+        const resolved = this.resolveJobId(id);
+        return resolved ? this.jobs.get(resolved) : undefined;
     }
-    async tailJob(id, bytes) {
-        const job = this.jobs.get(id);
+    async tailJob(id, bytesOrCursor) {
+        const resolved = this.resolveJobId(id);
+        const job = resolved ? this.jobs.get(resolved) : undefined;
         if (!job) {
-            return { ok: false, output: `Job "${id}" not found.`, exitCode: 1 };
+            const known = [...this.jobs.keys()].join(", ") || "none";
+            return { ok: false, output: `Job "${id}" not found. Canonical job IDs: ${known}.`, exitCode: 1 };
         }
-        const tailBytes = bytes ?? 8_000;
+        const cursor = typeof bytesOrCursor === "number" ? { bytes: bytesOrCursor } : (bytesOrCursor ?? {});
+        const stream = cursor.stream ?? "combined";
+        const paths = stream === "stderr" ? job.artifacts.stderr.chunks : stream === "stdout" ? job.artifacts.stdout.chunks : [...job.artifacts.stdout.chunks, ...job.artifacts.stderr.chunks];
+        const readablePaths = paths.length > 0 ? paths : [stream === "stderr" ? job.stderrArtifact : job.stdoutArtifact];
+        const path = readablePaths.at(-1);
         try {
-            const content = await readFile(job.artifactPath, "utf8");
-            const tail = content.length > tailBytes
-                ? `... (showing last ${tailBytes.toLocaleString()} bytes)\n${content.slice(-tailBytes)}`
-                : content;
+            const sizes = await Promise.all(readablePaths.map(async (chunkPath) => ({ path: chunkPath, size: (await stat(chunkPath)).size })));
+            const total = sizes.reduce((sum, entry) => sum + entry.size, 0);
+            const max = Math.max(1, Math.min(cursor.bytes ?? DEFAULT_TAIL_BYTES, 1024 * 1024));
+            const start = cursor.offset === undefined ? Math.max(0, total - max) : Math.max(0, Math.min(cursor.offset, total));
+            const length = Math.min(max, total - start);
+            const parts = [];
+            let logical = 0;
+            let remaining = length;
+            for (const entry of sizes) {
+                const chunkStart = logical;
+                const chunkEnd = logical + entry.size;
+                logical = chunkEnd;
+                if (remaining <= 0 || start >= chunkEnd)
+                    continue;
+                const localStart = Math.max(0, start - chunkStart);
+                const localLength = Math.min(remaining, entry.size - localStart);
+                const handle = await open(entry.path, "r");
+                const buffer = Buffer.alloc(localLength);
+                try {
+                    await handle.read(buffer, 0, localLength, localStart);
+                }
+                finally {
+                    await handle.close();
+                }
+                parts.push(buffer);
+                remaining -= localLength;
+            }
+            const nextOffset = start + (length - remaining);
             return {
                 ok: true,
-                output: `[${job.id}] ${job.status}:\n${tail}`,
-                outputPath: job.artifactPath,
+                output: `[${job.id}] ${job.status} exit=${job.exitCode ?? "?"} signal=${job.signal ?? "?"} ` +
+                    `stream=${stream} offset=${start} nextOffset=${nextOffset} total=${total}:\n` +
+                    Buffer.concat(parts).toString("utf8"),
+                outputPath: path,
+                backgroundJob: {
+                    id: job.id,
+                    status: job.status,
+                    artifactPath: path,
+                    nextOffset,
+                    ...(job.exitCode !== undefined ? { exitCode: job.exitCode } : {}),
+                    ...(job.signal !== undefined ? { signal: job.signal } : {}),
+                },
             };
         }
         catch (error) {
-            return {
-                ok: false,
-                output: `Failed to read job output: ${error instanceof Error ? error.message : String(error)}`,
-                exitCode: 1,
-            };
+            return { ok: false, output: `Failed to read job output: ${error instanceof Error ? error.message : String(error)}`, exitCode: 1 };
         }
     }
-    stopJob(id, signal) {
-        const job = this.jobs.get(id);
-        if (!job) {
+    async stopJob(id, options) {
+        const resolved = this.resolveJobId(id);
+        const job = resolved ? this.jobs.get(resolved) : undefined;
+        if (!job || !resolved)
             return { ok: false, output: `Job "${id}" not found.`, exitCode: 1 };
+        id = resolved;
+        if (job.status !== "running" && job.status !== "starting")
+            return { ok: false, output: `Job "${id}" is already ${job.status}.`, exitCode: 1 };
+        job.status = "stopping";
+        await this.persist();
+        this.abortControllers.get(id)?.abort();
+        const pid = job.pid;
+        const send = (signal) => {
+            if (!pid)
+                return false;
+            try {
+                process.platform !== "win32" && job.processGroupId ? process.kill(-job.processGroupId, signal) : process.kill(pid, signal);
+                return true;
+            }
+            catch {
+                return !processAlive(pid);
+            }
+        };
+        const graceful = options?.signal ?? "SIGTERM";
+        if (!send(graceful) && processAlive(pid)) {
+            job.status = "running";
+            await this.persist();
+            return { ok: false, output: `Failed to signal job "${id}".`, exitCode: 1 };
         }
-        if (job.status !== "running") {
-            return {
-                ok: false,
-                output: `Job "${id}" is already ${job.status}.`,
-                exitCode: 1,
-            };
-        }
-        const ac = this.abortControllers.get(id);
-        if (ac) {
-            ac.abort();
+        const waitForExit = async (ms) => {
+            const deadline = Date.now() + ms;
+            while (Date.now() < deadline) {
+                if (!processAlive(pid))
+                    return true;
+                await new Promise((resolve) => setTimeout(resolve, 25));
+            }
+            return !processAlive(pid);
+        };
+        let stopped = await waitForExit(options?.graceMs ?? 2_000);
+        let actualSignal = graceful;
+        if (!stopped && options?.escalate !== false) {
+            actualSignal = "SIGKILL";
+            send(actualSignal);
+            stopped = await waitForExit(1_000);
         }
-        const child = this.processes.get(id);
-        if (!child?.pid) {
-            job.status = "killed";
-            job.endedAt = new Date().toISOString();
-            this.abortControllers.delete(id);
-            return { ok: true, output: `Job "${id}" killed.` };
+        if (!stopped) {
+            job.status = "running";
+            await this.persist();
+            return { ok: false, output: `Job "${id}" remains alive after ${actualSignal}.`, exitCode: 1 };
         }
+        job.status = "killed";
+        job.signal = actualSignal;
+        job.endedAt = new Date().toISOString();
+        this.abortControllers.delete(id);
+        this.processes.delete(id);
+        this.writers.get(id)?.stdout.close();
+        this.writers.get(id)?.stderr.close();
+        this.writers.delete(id);
+        await this.persist();
+        return { ok: true, output: `Job "${id}" stopped and termination verified (${actualSignal}).` };
+    }
+    getRunningJobs() { return [...this.jobs.values()].filter((job) => job.status === "running" || job.status === "starting" || job.status === "stopping"); }
+    getRecentJobs(limit = 50) { return [...this.jobs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt)).slice(0, limit); }
+    loadAndReconcile() {
         try {
-            const sig = signal ?? "SIGTERM";
-            if (process.platform !== "win32") {
-                process.kill(-child.pid, sig);
-            }
-            else {
-                child.kill(sig);
+            if (!existsSync(this.registryPath))
+                return;
+            const parsed = JSON.parse(readFileSync(this.registryPath, "utf8"));
+            if (parsed.schemaVersion !== 1 || !Array.isArray(parsed.jobs))
+                return;
+            for (const job of parsed.jobs) {
+                if (["starting", "running", "stopping"].includes(job.status)) {
+                    const identity = processIdentity(job.pid);
+                    if (!processAlive(job.pid) || !identity || !job.processIdentity || identity !== job.processIdentity) {
+                        job.status = "lost";
+                        job.endedAt = new Date().toISOString();
+                    }
+                    else {
+                        job.status = "running";
+                        job.heartbeatAt = new Date().toISOString();
+                    }
+                }
+                this.jobs.set(job.id, job);
             }
-            job.status = "killed";
-            job.endedAt = new Date().toISOString();
-            this.abortControllers.delete(id);
-            return { ok: true, output: `Job "${id}" sent ${sig}.` };
+            this.persistSync();
         }
-        catch {
-            return {
-                ok: false,
-                output: `Failed to stop job "${id}".`,
-                exitCode: 1,
-            };
+        catch { /* Corrupt registries are ignored, never trusted as running. */ }
+    }
+    registry() { return { schemaVersion: 1, jobs: [...this.jobs.values()] }; }
+    persistSync() {
+        try {
+            mkdirSync(this.jobsDir, { recursive: true });
+            const temp = `${this.registryPath}.${process.pid}.${randomUUID()}.tmp`;
+            writeFileSync(temp, `${JSON.stringify(this.registry(), null, 2)}\n`, { mode: 0o600 });
+            renameSync(temp, this.registryPath);
         }
+        catch { /* best effort during constructor reconciliation */ }
     }
-    getRunningJobs() {
-        return [...this.jobs.values()].filter((j) => j.status === "running");
+    async persist() {
+        this.persistSync();
     }
 }
 export const jobManager = new JobManager();