@sema-agent/core 1.317.0 → 1.319.0

package/dist/engine/execution-env/node-execution-env.js

@@ -1,6 +1,6 @@
 import { spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
-import { closeSync, constants, createReadStream, mkdtempSync, openSync, readSync, statSync } from "node:fs";
+import { closeSync, constants, createReadStream, mkdtempSync, openSync, readSync, statSync, truncateSync, unlinkSync } from "node:fs";
 import { access, appendFile, lstat, mkdir, mkdtemp, readdir, readFile, readlink, realpath, rm, writeFile, } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, resolve } from "node:path";
@@ -203,14 +203,19 @@ function getShellEnv(inheritEnv, baseEnv, extraEnv) {
         ...extraEnv,
     };
 }
+const EXIT_SETTLE_DRAIN_MS = 500;
+const EXEC_SPOOL_PUMP_MS = 200;
+const EXEC_SPOOL_ROTATE_BYTES = 8 * 1024 * 1024;
+const EXEC_SPOOL_FINAL_TAIL_BYTES = 8 * 1024 * 1024;
+const EXEC_SPOOL_HARD_CAP_BYTES = 16 * 1024 * 1024;
 const SIGNUM = { SIGHUP: 1, SIGINT: 2, SIGQUIT: 3, SIGABRT: 6, SIGKILL: 9, SIGSEGV: 11, SIGPIPE: 13, SIGTERM: 15 };
 export function openSpoolPair(base) {
     const outPath = `${base}.out.log`;
     const errPath = `${base}.err.log`;
-    const fdOut = openSync(outPath, "wx", 0o600);
+    const fdOut = openSync(outPath, "ax", 0o600);
     let fdErr;
     try {
-        fdErr = openSync(errPath, "wx", 0o600);
+        fdErr = openSync(errPath, "ax", 0o600);
     }
     catch (e) {
         try {
@@ -297,6 +302,10 @@ export class NodeExecutionEnv {
             let child;
             const timeoutRef = {};
             const forceSettleRef = {};
+            const spoolPumpRef = {};
+            const execSpoolRef = {};
+            const pumpSpoolRef = {};
+            const drainSpoolRef = {};
             const armForceSettle = (result) => {
                 if (forceSettleRef.current || settled)
                     return;
@@ -312,6 +321,7 @@ export class NodeExecutionEnv {
                 forceSettleRef.current.unref?.();
             };
             const partialOutput = () => {
+                drainSpoolRef.current?.();
                 const so = stdoutTail.result();
                 const se = stderrTail.result();
                 return { stdout: markTruncated(so.text, so.droppedBytes), stderr: markTruncated(se.text, se.droppedBytes) };
@@ -343,22 +353,62 @@ export class NodeExecutionEnv {
                     return;
                 }
                 settled = true;
+                if (spoolPumpRef.current)
+                    clearInterval(spoolPumpRef.current);
+                if (execSpoolRef.current && !(result.ok && result.value.detached !== undefined)) {
+                    for (const path of [execSpoolRef.current.outPath, execSpoolRef.current.errPath]) {
+                        try {
+                            unlinkSync(path);
+                        }
+                        catch {
+                        }
+                    }
+                    execSpoolRef.current = undefined;
+                }
                 resolvePromise(result);
             };
+            let execSpoolFds;
+            if (process.platform !== "win32") {
+                try {
+                    if (!this.bgSpoolDir)
+                        this.bgSpoolDir = mkdtempSync(join(tmpdir(), "sema-bg-"));
+                    const pair = openSpoolPair(join(this.bgSpoolDir, `fg_${randomUUID()}`));
+                    execSpoolRef.current = { outPath: pair.outPath, errPath: pair.errPath };
+                    execSpoolFds = pair.fds;
+                }
+                catch {
+                    execSpoolRef.current = undefined;
+                    execSpoolFds = undefined;
+                }
+            }
+            const closeSpoolFds = () => {
+                if (!execSpoolFds)
+                    return;
+                for (const fd of execSpoolFds) {
+                    try {
+                        closeSync(fd);
+                    }
+                    catch {
+                    }
+                }
+                execSpoolFds = undefined;
+            };
             try {
                 child = spawn(shellConfig.value.shell, [...shellConfig.value.args, command], {
                     cwd,
                     detached: process.platform !== "win32",
                     env: getShellEnv(this.inheritEnv, this.shellEnv, options?.env),
-                    stdio: ["ignore", "pipe", "pipe"],
+                    stdio: execSpoolFds ? ["ignore", execSpoolFds[0], execSpoolFds[1]] : ["ignore", "pipe", "pipe"],
                     windowsHide: true,
                 });
             }
             catch (error) {
+                closeSpoolFds();
                 const cause = toError(error);
                 settle(err(new ExecutionError("spawn_error", cause.message, cause)));
                 return;
             }
+            closeSpoolFds();
             if (options?.onSpawn && child.pid !== undefined) {
                 const spawnedPid = child.pid;
                 const spawnedChild = child;
@@ -392,12 +442,116 @@ export class NodeExecutionEnv {
                     options.abortSignal.addEventListener("abort", onAbort, { once: true });
                 }
             }
+            const spoolCursor = { out: 0, err: 0 };
+            const readSpoolIncrement = (path, cursor) => {
+                try {
+                    const size = statSync(path).size;
+                    if (size <= spoolCursor[cursor])
+                        return undefined;
+                    const fd = openSync(path, "r");
+                    try {
+                        const want = Math.min(size - spoolCursor[cursor], 8 * 1024 * 1024 + 4096);
+                        const buf = Buffer.alloc(want);
+                        const got = readSync(fd, buf, 0, want, spoolCursor[cursor]);
+                        if (got <= 0)
+                            return undefined;
+                        spoolCursor[cursor] += got;
+                        return buf.subarray(0, got);
+                    }
+                    finally {
+                        closeSync(fd);
+                    }
+                }
+                catch {
+                    return undefined;
+                }
+            };
+            const feed = (tail, cb, buf) => {
+                tail.push(buf);
+                if (cb === undefined || settled)
+                    return;
+                try {
+                    cb(buf.toString("utf8"));
+                }
+                catch (error) {
+                    const cause = toError(error);
+                    callbackError = new ExecutionError("callback_error", cause.message, cause);
+                    onAbort();
+                }
+            };
+            const spoolLanes = () => {
+                const sp = execSpoolRef.current;
+                if (sp === undefined)
+                    return [];
+                return [
+                    { path: sp.outPath, cursor: "out", tail: stdoutTail, cb: options?.onStdout },
+                    { path: sp.errPath, cursor: "err", tail: stderrTail, cb: options?.onStderr },
+                ];
+            };
+            const skipAhead = (lane, keep) => {
+                try {
+                    const behind = statSync(lane.path).size - spoolCursor[lane.cursor];
+                    if (behind <= keep)
+                        return;
+                    lane.tail.recordSkippedBytes(behind - keep);
+                    spoolCursor[lane.cursor] += behind - keep;
+                }
+                catch {
+                }
+            };
+            const reclaimSpool = (lane) => {
+                if (spoolCursor[lane.cursor] < EXEC_SPOOL_ROTATE_BYTES)
+                    return;
+                try {
+                    const size = statSync(lane.path).size;
+                    if (size === spoolCursor[lane.cursor]) {
+                        truncateSync(lane.path, 0);
+                        spoolCursor[lane.cursor] = 0;
+                    }
+                    else if (size >= EXEC_SPOOL_HARD_CAP_BYTES) {
+                        lane.tail.recordSkippedBytes(size - spoolCursor[lane.cursor]);
+                        truncateSync(lane.path, 0);
+                        spoolCursor[lane.cursor] = 0;
+                    }
+                }
+                catch {
+                }
+            };
+            const pumpSpool = () => {
+                for (const lane of spoolLanes()) {
+                    skipAhead(lane, EXEC_SPOOL_ROTATE_BYTES);
+                    const inc = readSpoolIncrement(lane.path, lane.cursor);
+                    if (inc)
+                        feed(lane.tail, lane.cb, inc);
+                    reclaimSpool(lane);
+                }
+            };
+            pumpSpoolRef.current = pumpSpool;
+            const drainSpoolFinal = () => {
+                for (const lane of spoolLanes()) {
+                    skipAhead(lane, EXEC_SPOOL_FINAL_TAIL_BYTES);
+                    for (let i = 0; i < 4; i++) {
+                        const inc = readSpoolIncrement(lane.path, lane.cursor);
+                        if (!inc)
+                            break;
+                        feed(lane.tail, lane.cb, inc);
+                    }
+                }
+            };
+            drainSpoolRef.current = drainSpoolFinal;
+            if (execSpoolRef.current) {
+                spoolPumpRef.current = setInterval(pumpSpool, EXEC_SPOOL_PUMP_MS);
+                spoolPumpRef.current.unref?.();
+            }
             const onDetach = () => {
                 if (settled || !child)
                     return;
-                const shellId = this.adoptRunningChild(child, stdoutTail.result().text, stderrTail.result().text);
+                pumpSpoolRef.current?.();
+                const shellId = this.adoptRunningChild(child, stdoutTail.result(), stderrTail.result(), execSpoolRef.current !== undefined ? { spool: execSpoolRef.current, cursors: { out: spoolCursor.out, err: spoolCursor.err } } : undefined);
                 if (shellId === undefined)
                     return;
+                if (spoolPumpRef.current)
+                    clearInterval(spoolPumpRef.current);
                 try {
                     options?.onDetachAdopted?.();
                 }
@@ -405,7 +559,9 @@ export class NodeExecutionEnv {
                 }
                 if (timeoutRef.current)
                     clearTimeout(timeoutRef.current);
-                settle(ok({ stdout: stdoutTail.result().text, stderr: stderrTail.result().text, exitCode: 0, detached: { shellId } }));
+                const dso = stdoutTail.result();
+                const dse = stderrTail.result();
+                settle(ok({ stdout: markTruncated(dso.text, dso.droppedBytes), stderr: markTruncated(dse.text, dse.droppedBytes), exitCode: 0, detached: { shellId } }));
             };
             if (options?.detachSignal) {
                 if (options.detachSignal.aborted)
@@ -415,30 +571,32 @@ export class NodeExecutionEnv {
                     options.detachSignal.addEventListener("abort", dl, { once: true });
                 }
             }
-            child.stdout?.setEncoding("utf8");
-            child.stderr?.setEncoding("utf8");
-            child.stdout?.on("data", (chunk) => {
-                stdoutTail.push(Buffer.from(chunk, "utf8"));
-                try {
-                    options?.onStdout?.(chunk);
-                }
-                catch (error) {
-                    const cause = toError(error);
-                    callbackError = new ExecutionError("callback_error", cause.message, cause);
-                    onAbort();
-                }
-            });
-            child.stderr?.on("data", (chunk) => {
-                stderrTail.push(Buffer.from(chunk, "utf8"));
-                try {
-                    options?.onStderr?.(chunk);
-                }
-                catch (error) {
-                    const cause = toError(error);
-                    callbackError = new ExecutionError("callback_error", cause.message, cause);
-                    onAbort();
-                }
-            });
+            if (!execSpoolRef.current) {
+                child.stdout?.setEncoding("utf8");
+                child.stderr?.setEncoding("utf8");
+                child.stdout?.on("data", (chunk) => {
+                    stdoutTail.push(Buffer.from(chunk, "utf8"));
+                    try {
+                        options?.onStdout?.(chunk);
+                    }
+                    catch (error) {
+                        const cause = toError(error);
+                        callbackError = new ExecutionError("callback_error", cause.message, cause);
+                        onAbort();
+                    }
+                });
+                child.stderr?.on("data", (chunk) => {
+                    stderrTail.push(Buffer.from(chunk, "utf8"));
+                    try {
+                        options?.onStderr?.(chunk);
+                    }
+                    catch (error) {
+                        const cause = toError(error);
+                        callbackError = new ExecutionError("callback_error", cause.message, cause);
+                        onAbort();
+                    }
+                });
+            }
             child.stdout?.on("error", () => { });
             child.stderr?.on("error", () => { });
             child.on("error", (error) => {
@@ -448,7 +606,8 @@ export class NodeExecutionEnv {
                 }
                 settle(err(new ExecutionError("spawn_error", error.message, error)));
             });
-            child.on("close", (code, signal) => {
+            const settleFromTermination = (code, signal) => {
+                drainSpoolFinal();
                 if (callbackError) {
                     const p = partialOutput();
                     callbackError.partialStdout = p.stdout;
@@ -476,6 +635,16 @@ export class NodeExecutionEnv {
                     return;
                 }
                 settle(ok({ stdout: markTruncated(so.text, so.droppedBytes), stderr: markTruncated(se.text, se.droppedBytes), exitCode: code ?? 0 }));
+            };
+            child.on("close", (code, signal) => settleFromTermination(code, signal));
+            child.on("exit", (code, signal) => {
+                if (execSpoolRef.current === undefined)
+                    return;
+                const drain = setTimeout(() => {
+                    settleFromTermination(code, signal);
+                }, EXIT_SETTLE_DRAIN_MS);
+                drain.unref?.();
+                child.once("close", () => clearTimeout(drain));
             });
         });
     }
@@ -714,7 +883,7 @@ export class NodeExecutionEnv {
             return err(toFileError(error, filePath));
         }
     }
-    adoptRunningChild(child, stdoutSoFar, stderrSoFar) {
+    adoptRunningChild(child, stdoutSoFar, stderrSoFar, handover) {
         const liveCount = [...this.bgShells.values()].filter((e) => e.status === "running").length;
         if (liveCount >= this.backgroundCapabilities.maxConcurrent)
             return undefined;
@@ -724,13 +893,18 @@ export class NodeExecutionEnv {
             stdout: { tail: new RollingTailBuffer(), totalBytes: 0, cursorBytes: 0 },
             stderr: { tail: new RollingTailBuffer(), totalBytes: 0, cursorBytes: 0 },
             status: "running",
+            ...(handover !== undefined
+                ? { spool: { outPath: handover.spool.outPath, errPath: handover.spool.errPath, fileCursor: { ...handover.cursors }, ephemeral: true } }
+                : {}),
         };
-        const seedOut = Buffer.from(stdoutSoFar, "utf8");
-        const seedErr = Buffer.from(stderrSoFar, "utf8");
+        const seedOut = Buffer.from(stdoutSoFar.text, "utf8");
+        const seedErr = Buffer.from(stderrSoFar.text, "utf8");
+        entry.stdout.tail.recordSkippedBytes(stdoutSoFar.droppedBytes);
+        entry.stderr.tail.recordSkippedBytes(stderrSoFar.droppedBytes);
         entry.stdout.tail.push(seedOut);
-        entry.stdout.totalBytes += seedOut.length;
         entry.stderr.tail.push(seedErr);
-        entry.stderr.totalBytes += seedErr.length;
+        entry.stdout.totalBytes += stdoutSoFar.droppedBytes + seedOut.length;
+        entry.stderr.totalBytes += stderrSoFar.droppedBytes + seedErr.length;
         child.stdout?.on("data", (chunk) => {
             const buf = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk;
             entry.stdout.tail.push(buf);
@@ -750,6 +924,7 @@ export class NodeExecutionEnv {
         child.on("close", (code, signal) => {
             if (entry.timer)
                 clearTimeout(entry.timer);
+            this.reclaimAdoptedSpool(entry);
             if (entry.status === "killed")
                 return;
             entry.status = "exited";
@@ -758,6 +933,7 @@ export class NodeExecutionEnv {
         if (child.exitCode !== null || child.signalCode !== null) {
             entry.status = "exited";
             entry.exitCode = child.exitCode ?? (child.signalCode ? 128 + (SIGNUM[child.signalCode] ?? 9) : 0);
+            this.reclaimAdoptedSpool(entry);
         }
         else {
             const timeoutSec = this.backgroundCapabilities.defaultBgTimeoutSec;
@@ -789,7 +965,7 @@ export class NodeExecutionEnv {
                 if (!this.bgSpoolDir)
                     this.bgSpoolDir = mkdtempSync(join(tmpdir(), "sema-bg-"));
                 const pair = openSpoolPair(join(this.bgSpoolDir, `bg_${this.bgCounter + 1}_${randomUUID()}`));
-                spool = { outPath: pair.outPath, errPath: pair.errPath };
+                spool = { outPath: pair.outPath, errPath: pair.errPath, fileCursor: { out: 0, err: 0 } };
                 spoolFds = pair.fds;
             }
             catch {
@@ -922,35 +1098,74 @@ export class NodeExecutionEnv {
         return ok(undefined);
     }
     syncSpool(entry) {
-        if (!entry.spool)
+        const spool = entry.spool;
+        if (!spool)
             return;
         const streams = [
-            [entry.stdout, entry.spool.outPath],
-            [entry.stderr, entry.spool.errPath],
+            [entry.stdout, spool.outPath, "out"],
+            [entry.stderr, spool.errPath, "err"],
         ];
-        for (const [stream, path] of streams) {
+        for (const [stream, path, lane] of streams) {
             try {
                 const size = statSync(path).size;
-                if (size <= stream.totalBytes)
-                    continue;
-                const fd = openSync(path, "r");
-                try {
-                    const want = Math.min(size - stream.totalBytes, 8 * 1024 * 1024 + 4096);
-                    const buf = Buffer.alloc(want);
-                    const got = readSync(fd, buf, 0, want, stream.totalBytes);
-                    if (got > 0) {
-                        stream.tail.push(buf.subarray(0, got));
-                        stream.totalBytes += got;
+                if (spool.ephemeral) {
+                    const behind = size - spool.fileCursor[lane];
+                    if (behind > EXEC_SPOOL_ROTATE_BYTES) {
+                        const skipped = behind - EXEC_SPOOL_ROTATE_BYTES;
+                        stream.tail.recordSkippedBytes(skipped);
+                        stream.totalBytes += skipped;
+                        spool.fileCursor[lane] += skipped;
                     }
                 }
-                finally {
-                    closeSync(fd);
+                if (size > spool.fileCursor[lane]) {
+                    const fd = openSync(path, "r");
+                    try {
+                        const want = Math.min(size - spool.fileCursor[lane], 8 * 1024 * 1024 + 4096);
+                        const buf = Buffer.alloc(want);
+                        const got = readSync(fd, buf, 0, want, spool.fileCursor[lane]);
+                        if (got > 0) {
+                            stream.tail.push(buf.subarray(0, got));
+                            stream.totalBytes += got;
+                            spool.fileCursor[lane] += got;
+                        }
+                    }
+                    finally {
+                        closeSync(fd);
+                    }
+                }
+                if (spool.ephemeral && spool.fileCursor[lane] >= EXEC_SPOOL_ROTATE_BYTES) {
+                    const size2 = statSync(path).size;
+                    if (size2 === spool.fileCursor[lane]) {
+                        truncateSync(path, 0);
+                        spool.fileCursor[lane] = 0;
+                    }
+                    else if (size2 >= EXEC_SPOOL_HARD_CAP_BYTES) {
+                        const skipped = size2 - spool.fileCursor[lane];
+                        stream.tail.recordSkippedBytes(skipped);
+                        stream.totalBytes += skipped;
+                        truncateSync(path, 0);
+                        spool.fileCursor[lane] = 0;
+                    }
                 }
             }
             catch {
             }
         }
     }
+    reclaimAdoptedSpool(entry) {
+        const spool = entry.spool;
+        if (!spool?.ephemeral)
+            return;
+        this.syncSpool(entry);
+        entry.spool = undefined;
+        for (const path of [spool.outPath, spool.errPath]) {
+            try {
+                unlinkSync(path);
+            }
+            catch {
+            }
+        }
+    }
     async disposeBackgroundShells(opts) {
         if (this.retainBackgroundProcesses)
             return;
@@ -965,6 +1180,7 @@ export class NodeExecutionEnv {
                     entry.status = "killed";
                     killProcessTree(entry.child.pid, { force: true });
                 }
+                this.reclaimAdoptedSpool(entry);
             }
             catch {
             }