@pasko70/pibo 1.12.0 → 1.13.0

package/dist/debug/index.js

@@ -989,6 +989,17 @@ function compactRunRow(run) {
         timeoutAt: run.timeoutAt,
         timeoutPhase: run.timeoutPhase,
         serviceWarning: run.serviceWarning,
+        isolation: run.resources?.isolationMode,
+        resourceUnit: run.resources?.unitName,
+        resourceLimitReason: run.resources?.limitReason,
+        minimumHostAvailableBytes: run.resources?.minimumHostAvailableBytes,
+        peakMemoryFullPsiAvg10: run.resources?.peakMemoryFullPsiAvg10,
+        peakIoFullPsiAvg10: run.resources?.peakIoFullPsiAvg10,
+        memoryPeakBytes: run.resources?.cgroup?.memoryPeakBytes,
+        memoryMaxBytes: run.resources?.cgroup?.memoryMaxBytes,
+        tasksPeak: run.resources?.cgroup?.tasksPeak,
+        ioReadBytes: run.resources?.cgroup?.ioReadBytes,
+        ioWriteBytes: run.resources?.cgroup?.ioWriteBytes,
         summary: run.summary,
     };
 }

package/dist/gateway/server.js

@@ -149,6 +149,7 @@ export class PiboGatewayServer {
             pluginRegistry: this.pluginRegistry,
             sessionStore: this.sessionStore,
             messagePreflight: createLoopMessagePreflight({ path: this.options.loopStorePath }),
+            recoverInterruptedRuntimeState: true,
         });
         this.unsubscribe = this.router.subscribe((event) => this.broadcastRouterEvent(event));
         this.server = createServer((socket) => this.handleSocket(socket));

package/dist/reliability/store.js

@@ -144,7 +144,8 @@ export class PiboReliabilityStore {
 				timeout_ms INTEGER,
 				timeout_at TEXT,
 				timeout_phase TEXT,
-				service_warning TEXT
+				service_warning TEXT,
+				resource_json TEXT
 			);
 			CREATE INDEX IF NOT EXISTS idx_pibo_runs_controller_updated
 				ON pibo_runs(controller_pibo_session_id, updated_at);
@@ -155,6 +156,7 @@ export class PiboReliabilityStore {
         ensurePiboRunColumn(this.db, "timeout_at", "TEXT");
         ensurePiboRunColumn(this.db, "timeout_phase", "TEXT");
         ensurePiboRunColumn(this.db, "service_warning", "TEXT");
+        ensurePiboRunColumn(this.db, "resource_json", "TEXT");
         this.appendEventStatement = this.db.prepare(`
 			INSERT INTO pibo_event_stream (topic, key, event_id, idempotency_key, created_at, retention_class, payload_json)
 			VALUES (?, ?, ?, ?, ?, ?, ?)
@@ -496,10 +498,10 @@ export class PiboReliabilityStore {
 				INSERT INTO pibo_runs (
 					run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
 					summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
-					completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning
-				) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?)
+					completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning, resource_json
+				) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?, ?)
 			`)
-            .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null);
+            .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null, input.resources ? JSON.stringify(input.resources) : null);
         this.claimJob(job.jobId, input.workerId ?? `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
         return this.requireRun(runId);
     }
@@ -527,10 +529,11 @@ export class PiboReliabilityStore {
 					timeout_ms = ?,
 					timeout_at = ?,
 					timeout_phase = ?,
-					service_warning = ?
+					service_warning = ?,
+					resource_json = ?
 				WHERE run_id = ?
 			`)
-            .run(next.status, next.completionPolicy, next.consumed ? 1 : 0, next.summary ?? null, next.result ? JSON.stringify(next.result) : null, next.error ?? null, next.notifiedStatus ?? null, next.acknowledgedStatus ?? null, next.updatedAt, next.completedAt ?? null, next.jobId ?? null, next.retryable ? 1 : 0, next.maxAttempts, next.timeoutMs ?? null, next.timeoutAt ?? null, next.timeoutPhase ?? null, next.serviceWarning ?? null, runId);
+            .run(next.status, next.completionPolicy, next.consumed ? 1 : 0, next.summary ?? null, next.result ? JSON.stringify(next.result) : null, next.error ?? null, next.notifiedStatus ?? null, next.acknowledgedStatus ?? null, next.updatedAt, next.completedAt ?? null, next.jobId ?? null, next.retryable ? 1 : 0, next.maxAttempts, next.timeoutMs ?? null, next.timeoutAt ?? null, next.timeoutPhase ?? null, next.serviceWarning ?? null, next.resources ? JSON.stringify(next.resources) : null, runId);
         return this.requireRun(runId);
     }
     getRun(runId) {
@@ -801,6 +804,8 @@ function runFromRow(row) {
         output.timeoutPhase = row.timeout_phase;
     if (row.service_warning)
         output.serviceWarning = row.service_warning;
+    if (row.resource_json)
+        output.resources = JSON.parse(row.resource_json);
     return output;
 }
 function retryDelayMs(attempts, input) {

package/dist/runs/registry.js

@@ -34,6 +34,8 @@ function snapshot(record) {
         output.timeoutPhase = record.timeoutPhase;
     if (record.serviceWarning)
         output.serviceWarning = record.serviceWarning;
+    if (record.resources)
+        output.resources = structuredClone(record.resources);
     if (record.completedAt)
         output.completedAt = record.completedAt;
     return output;
@@ -46,6 +48,7 @@ export class PiboRunRegistry {
     runs = new Map();
     waiters = new Map();
     listeners = new Set();
+    recoveredRuns = [];
     workerId;
     subscribe(listener) {
         this.listeners.add(listener);
@@ -55,12 +58,17 @@ export class PiboRunRegistry {
         this.options = options;
         this.workerId = options.workerId ?? `run-registry:${process.pid}:${randomUUID()}`;
         if (this.options.store) {
-            this.options.store.recoverInterruptedRuns(this.workerId);
+            for (const recovered of this.options.store.recoverInterruptedRuns(this.workerId)) {
+                this.recoveredRuns.push(snapshot(recordFromStored(recovered)));
+            }
             for (const record of this.options.store.listRuns({ includeConsumed: true, includeDetached: true })) {
                 this.runs.set(record.runId, recordFromStored(record));
             }
         }
     }
+    listRecoveredRuns() {
+        return this.recoveredRuns.map((run) => ({ ...run }));
+    }
     startToolRun(input) {
         this.prune();
         if (this.options.store) {
@@ -73,6 +81,7 @@ export class PiboRunRegistry {
                 maxAttempts: input.maxAttempts ?? 1,
                 timeoutMs: input.timeoutMs,
                 serviceWarning: input.serviceWarning,
+                resources: input.resources,
                 workerId: this.workerId,
             });
             const record = recordFromStored(stored);
@@ -98,12 +107,22 @@ export class PiboRunRegistry {
             maxAttempts: Math.max(1, input.maxAttempts ?? 1),
             ...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs, timeoutAt: runTimeoutAt(timestamp, input.timeoutMs) } : {}),
             ...(input.serviceWarning ? { serviceWarning: input.serviceWarning } : {}),
+            ...(input.resources ? { resources: structuredClone(input.resources) } : {}),
         };
         this.runs.set(runId, record);
         const output = snapshot(record);
         this.notify({ type: "run_started", run: output });
         return output;
     }
+    updateResources(runId, resources) {
+        const record = this.runs.get(runId);
+        if (!record)
+            return undefined;
+        record.resources = structuredClone(resources);
+        record.updatedAt = now();
+        this.options.store?.updateRun(runId, record);
+        return snapshot(record);
+    }
     complete(runId, result) {
         const record = this.runs.get(runId);
         if (!record || terminal(record.status))
@@ -120,6 +139,23 @@ export class PiboRunRegistry {
         this.notify({ type: "run_changed", run: output, previousStatus });
         return output;
     }
+    resourceLimit(runId, error, resources) {
+        const record = this.runs.get(runId);
+        if (!record || terminal(record.status))
+            return undefined;
+        const previousStatus = record.status;
+        record.status = "failed";
+        record.error = error;
+        record.resources = structuredClone(resources);
+        record.summary = `${record.toolName} run was stopped by yielded-run resource limits.`;
+        this.finish(record);
+        this.options.store?.updateRun(runId, record);
+        if (record.jobId)
+            this.options.store?.fail(record.jobId, this.workerId, error);
+        const output = snapshot(record);
+        this.notify({ type: "run_changed", run: output, previousStatus, reason: error });
+        return output;
+    }
     fail(runId, error) {
         const record = this.runs.get(runId);
         if (!record || terminal(record.status))
@@ -424,5 +460,6 @@ function recordFromStored(record) {
         timeoutAt: record.timeoutAt,
         timeoutPhase: record.timeoutPhase,
         serviceWarning: record.serviceWarning,
+        resources: record.resources,
     };
 }

package/dist/runs/resource-isolation.js

@@ -0,0 +1,437 @@
+import { randomUUID } from "node:crypto";
+import { execFile } from "node:child_process";
+import { existsSync, readFileSync, unlinkSync } from "node:fs";
+import { promisify } from "node:util";
+const execFileAsync = promisify(execFile);
+export class PiboRunResourceLimitError extends Error {
+    resources;
+    constructor(message, resources) {
+        super(message);
+        this.resources = resources;
+        this.name = "PiboRunResourceLimitError";
+    }
+}
+const DEFAULT_POLICY = Object.freeze({
+    mode: "systemd",
+    memoryHighBytes: 1280 * 1024 * 1024,
+    memoryMaxBytes: 1792 * 1024 * 1024,
+    tasksMax: 128,
+    cpuQuotaPercent: 200,
+    ioWeight: 100,
+    monitorIntervalMs: 500,
+    minHostAvailableBytes: 1024 * 1024 * 1024,
+    maxMemoryFullPsiAvg10: 5,
+    maxIoFullPsiAvg10: 10,
+});
+export function resolveYieldedRunResourcePolicy(env = process.env) {
+    return {
+        mode: parseIsolationMode(env.PIBO_YIELDED_RUN_ISOLATION, DEFAULT_POLICY.mode),
+        memoryHighBytes: parseNonNegativeInteger(env.PIBO_YIELDED_RUN_MEMORY_HIGH_BYTES, DEFAULT_POLICY.memoryHighBytes),
+        memoryMaxBytes: parseNonNegativeInteger(env.PIBO_YIELDED_RUN_MEMORY_MAX_BYTES, DEFAULT_POLICY.memoryMaxBytes),
+        tasksMax: parsePositiveInteger(env.PIBO_YIELDED_RUN_TASKS_MAX, DEFAULT_POLICY.tasksMax),
+        cpuQuotaPercent: parsePositiveNumber(env.PIBO_YIELDED_RUN_CPU_QUOTA_PERCENT, DEFAULT_POLICY.cpuQuotaPercent),
+        ioWeight: Math.min(10_000, parsePositiveInteger(env.PIBO_YIELDED_RUN_IO_WEIGHT, DEFAULT_POLICY.ioWeight)),
+        monitorIntervalMs: Math.max(100, parsePositiveInteger(env.PIBO_YIELDED_RUN_MONITOR_INTERVAL_MS, DEFAULT_POLICY.monitorIntervalMs)),
+        minHostAvailableBytes: parseNonNegativeInteger(env.PIBO_YIELDED_RUN_MIN_HOST_AVAILABLE_BYTES, DEFAULT_POLICY.minHostAvailableBytes),
+        maxMemoryFullPsiAvg10: parseNonNegativeNumber(env.PIBO_YIELDED_RUN_MAX_MEMORY_FULL_PSI_AVG10, DEFAULT_POLICY.maxMemoryFullPsiAvg10),
+        maxIoFullPsiAvg10: parseNonNegativeNumber(env.PIBO_YIELDED_RUN_MAX_IO_FULL_PSI_AVG10, DEFAULT_POLICY.maxIoFullPsiAvg10),
+    };
+}
+export function collectYieldedRunHostResourceSnapshot(options = {}) {
+    const memory = parseLinuxMeminfo(options.meminfo ?? readOptionalFile("/proc/meminfo"));
+    return {
+        capturedAt: (options.now ?? new Date()).toISOString(),
+        memoryFreeBytes: memory.freeBytes,
+        memoryAvailableBytes: memory.availableBytes,
+        memoryPressure: parseLinuxPressure(options.memoryPressure ?? readOptionalFile("/proc/pressure/memory")),
+        ioPressure: parseLinuxPressure(options.ioPressure ?? readOptionalFile("/proc/pressure/io")),
+    };
+}
+export function parseLinuxMeminfo(input) {
+    const values = new Map();
+    for (const line of input.split("\n")) {
+        const match = line.match(/^([A-Za-z_()]+):\s+(\d+)\s+kB$/);
+        if (match)
+            values.set(match[1], Number(match[2]) * 1024);
+    }
+    const freeBytes = values.get("MemFree") ?? 0;
+    return { freeBytes, availableBytes: values.get("MemAvailable") ?? freeBytes };
+}
+export function parseLinuxPressure(input) {
+    const output = {};
+    for (const line of input.split("\n")) {
+        const match = line.match(/^(some|full)\s+.*\bavg10=([0-9.]+)/);
+        if (!match)
+            continue;
+        const value = Number(match[2]);
+        if (!Number.isFinite(value))
+            continue;
+        if (match[1] === "some")
+            output.someAvg10 = value;
+        else
+            output.fullAvg10 = value;
+    }
+    return output;
+}
+export function prepareYieldedRunExecution(toolName, params, options = {}) {
+    const policy = resolveYieldedRunResourcePolicy(options.env);
+    const command = bashCommand(params);
+    const shouldIsolate = policy.mode === "systemd" && toolName === "bash" && command !== undefined;
+    const unitName = shouldIsolate ? options.unitName ?? yieldedRunUnitName() : undefined;
+    const metricsPath = unitName ? `/tmp/${unitName}.metrics` : undefined;
+    const resources = {
+        isolationMode: shouldIsolate ? "systemd" : "off",
+        ...(unitName ? { unitName } : {}),
+        policy,
+        admission: collectYieldedRunHostResourceSnapshot({ now: options.now }),
+    };
+    const preparedParams = shouldIsolate
+        ? { ...params, command: systemdRunCommand(command, unitName, policy, metricsPath) }
+        : params;
+    return {
+        params: preparedParams,
+        resources,
+        async execute(operation) {
+            if (!shouldIsolate || !unitName)
+                return await operation();
+            if (process.platform !== "linux" || !existsSync("/run/systemd/system")) {
+                resources.limitReason = "systemd isolation is unavailable on this host";
+                throw new PiboRunResourceLimitError("resource_limited: systemd isolation is unavailable for yielded Bash execution", resources);
+            }
+            resources.startedAt = new Date().toISOString();
+            const monitor = monitorYieldedRunResources(unitName, policy, resources, metricsPath);
+            try {
+                const value = await operation();
+                await monitor.finish();
+                if (resources.limitReason)
+                    throw new PiboRunResourceLimitError(`resource_limited: ${resources.limitReason}`, resources);
+                return value;
+            }
+            catch (error) {
+                if (!(error instanceof PiboRunResourceLimitError))
+                    await terminateSystemdUnit(unitName);
+                await monitor.finish();
+                if (error instanceof PiboRunResourceLimitError)
+                    throw error;
+                if (resources.limitReason || cgroupReachedResourceLimit(resources.cgroup)) {
+                    resources.limitReason ??= cgroupLimitReason(resources.cgroup) ?? "the yielded-run cgroup reached a configured resource limit";
+                    throw new PiboRunResourceLimitError(`resource_limited: ${resources.limitReason}`, resources);
+                }
+                throw error;
+            }
+        },
+    };
+}
+export function systemdRunCommand(command, unitName, policy, metricsPath = `/tmp/${unitName}.metrics`) {
+    const captureScript = [
+        '/bin/bash -c "$1"',
+        "status=$?",
+        "cgroup_path=$(awk -F: '$1 == \"0\" { print $3 }' /proc/self/cgroup)",
+        'cgroup_root="/sys/fs/cgroup${cgroup_path}"',
+        '{',
+        'printf "ControlGroup=%s\\n" "$cgroup_path"',
+        'printf "CgroupRoot=%s\\n" "$cgroup_root"',
+        'for file in memory.current memory.peak memory.high memory.max memory.swap.peak memory.swap.max pids.current pids.peak; do value=$(cat "$cgroup_root/$file" 2>/dev/null || true); printf "%s=%s\\n" "$file" "$value"; done',
+        'cat "$cgroup_root/memory.events" 2>/dev/null | awk \'{ print "memory.events." $1 "=" $2 }\' || true',
+        'cat "$cgroup_root/cpu.stat" 2>/dev/null | awk \'{ print "cpu.stat." $1 "=" $2 }\' || true',
+        'cat "$cgroup_root/io.stat" 2>/dev/null | sed "s/^/io.stat./" || true',
+        '} > "$2"',
+        "exit $status",
+    ].join("\n");
+    const args = [
+        "systemd-run",
+        "--quiet",
+        "--wait",
+        "--pipe",
+        "--expand-environment=no",
+        `--unit=${unitName}`,
+        "--service-type=exec",
+        "--working-directory=$PWD",
+        "--slice=pibo-yielded.slice",
+        "--property=KillMode=control-group",
+        "--property=OOMPolicy=stop",
+        `--property=MemoryHigh=${policy.memoryHighBytes}`,
+        `--property=MemoryMax=${policy.memoryMaxBytes}`,
+        "--property=MemorySwapMax=0",
+        "--property=MemoryZSwapMax=0",
+        `--property=TasksMax=${policy.tasksMax}`,
+        `--property=CPUQuota=${policy.cpuQuotaPercent}%`,
+        `--property=IOWeight=${policy.ioWeight}`,
+        "--",
+        "/bin/bash",
+        "-c",
+        captureScript,
+        "pibo-yielded",
+        command,
+        metricsPath,
+    ];
+    return args.map((value) => value === "--working-directory=$PWD" ? value : shellQuote(value)).join(" ");
+}
+function monitorYieldedRunResources(unitName, policy, resources, metricsPath) {
+    let finished = false;
+    let pollInProgress = false;
+    let killRequested = false;
+    let unitObserved = false;
+    const poll = async () => {
+        if (finished || pollInProgress)
+            return;
+        pollInProgress = true;
+        try {
+            if (!unitObserved) {
+                unitObserved = await systemdUnitExists(unitName);
+                if (!unitObserved)
+                    return;
+            }
+            const host = collectYieldedRunHostResourceSnapshot();
+            resources.minimumHostAvailableBytes = Math.min(resources.minimumHostAvailableBytes ?? host.memoryAvailableBytes, host.memoryAvailableBytes);
+            resources.peakMemoryFullPsiAvg10 = Math.max(resources.peakMemoryFullPsiAvg10 ?? 0, host.memoryPressure.fullAvg10 ?? 0);
+            resources.peakIoFullPsiAvg10 = Math.max(resources.peakIoFullPsiAvg10 ?? 0, host.ioPressure.fullAvg10 ?? 0);
+            const reason = hostLimitReason(host, policy);
+            if (reason) {
+                resources.limitReason ??= reason;
+                if (!killRequested)
+                    killRequested = await stopSystemdUnit(unitName);
+            }
+        }
+        finally {
+            pollInProgress = false;
+        }
+    };
+    const timer = setInterval(() => { void poll(); }, policy.monitorIntervalMs);
+    timer.unref?.();
+    void poll();
+    return {
+        async finish() {
+            if (finished)
+                return;
+            finished = true;
+            clearInterval(timer);
+            while (pollInProgress)
+                await new Promise((resolve) => setTimeout(resolve, 10));
+            resources.cgroup = await collectSystemdUnitResources(unitName, metricsPath);
+            resources.completedAt = new Date().toISOString();
+            if (!resources.limitReason)
+                resources.limitReason = cgroupLimitReason(resources.cgroup);
+            await resetSystemdUnit(unitName);
+        },
+    };
+}
+function hostLimitReason(snapshot, policy) {
+    if (snapshot.memoryAvailableBytes < policy.minHostAvailableBytes) {
+        return `host MemAvailable ${snapshot.memoryAvailableBytes} fell below ${policy.minHostAvailableBytes}`;
+    }
+    if ((snapshot.memoryPressure.fullAvg10 ?? 0) >= policy.maxMemoryFullPsiAvg10) {
+        return `host memory full PSI avg10 ${snapshot.memoryPressure.fullAvg10} reached ${policy.maxMemoryFullPsiAvg10}`;
+    }
+    if ((snapshot.ioPressure.fullAvg10 ?? 0) >= policy.maxIoFullPsiAvg10) {
+        return `host I/O full PSI avg10 ${snapshot.ioPressure.fullAvg10} reached ${policy.maxIoFullPsiAvg10}`;
+    }
+    return undefined;
+}
+async function collectSystemdUnitResources(unitName, metricsPath) {
+    const fileSnapshot = metricsPath ? readCgroupMetrics(metricsPath, unitName) : { unitName };
+    const properties = [
+        "ControlGroup",
+        "ActiveState",
+        "SubState",
+        "Result",
+        "ExecMainStatus",
+        "OOMKilled",
+        "MemoryCurrent",
+        "MemoryPeak",
+        "MemoryHigh",
+        "MemoryMax",
+        "MemorySwapPeak",
+        "MemorySwapMax",
+        "CPUUsageNSec",
+        "TasksCurrent",
+        "TasksPeak",
+        "IOReadBytes",
+        "IOWriteBytes",
+    ];
+    try {
+        const { stdout } = await execFileAsync("systemctl", ["show", unitName, ...properties.map((property) => `--property=${property}`)], { timeout: 5_000 });
+        const values = new Map(stdout.split("\n").flatMap((line) => {
+            const index = line.indexOf("=");
+            return index > 0 ? [[line.slice(0, index), line.slice(index + 1)]] : [];
+        }));
+        const output = { ...fileSnapshot, unitName };
+        assignString(output, "controlGroup", values.get("ControlGroup"));
+        assignString(output, "activeState", values.get("ActiveState"));
+        assignString(output, "subState", values.get("SubState"));
+        assignString(output, "result", values.get("Result"));
+        assignNumber(output, "execMainStatus", values.get("ExecMainStatus"));
+        if (values.get("OOMKilled") === "yes")
+            output.oomKilled = true;
+        assignNumber(output, "memoryCurrentBytes", values.get("MemoryCurrent"));
+        assignNumber(output, "memoryPeakBytes", values.get("MemoryPeak"));
+        assignNumber(output, "memoryHighBytes", values.get("MemoryHigh"));
+        assignNumber(output, "memoryMaxBytes", values.get("MemoryMax"));
+        assignNumber(output, "memorySwapPeakBytes", values.get("MemorySwapPeak"));
+        assignNumber(output, "memorySwapMaxBytes", values.get("MemorySwapMax"));
+        assignNumber(output, "cpuUsageNs", values.get("CPUUsageNSec"));
+        assignNumber(output, "tasksCurrent", values.get("TasksCurrent"));
+        assignNumber(output, "tasksPeak", values.get("TasksPeak"));
+        assignNumber(output, "ioReadBytes", values.get("IOReadBytes"));
+        assignNumber(output, "ioWriteBytes", values.get("IOWriteBytes"));
+        return output;
+    }
+    catch {
+        return fileSnapshot;
+    }
+}
+function readCgroupMetrics(path, unitName) {
+    try {
+        const values = new Map();
+        let ioReadBytes = 0;
+        let ioWriteBytes = 0;
+        for (const line of readFileSync(path, "utf8").split("\n")) {
+            if (line.startsWith("io.stat.")) {
+                for (const match of line.matchAll(/\b(rbytes|wbytes)=(\d+)/g)) {
+                    if (match[1] === "rbytes")
+                        ioReadBytes += Number(match[2]);
+                    else
+                        ioWriteBytes += Number(match[2]);
+                }
+                continue;
+            }
+            const equals = line.indexOf("=");
+            if (equals > 0)
+                values.set(line.slice(0, equals), line.slice(equals + 1));
+        }
+        return {
+            unitName,
+            controlGroup: nonEmpty(values.get("ControlGroup")),
+            oomKilled: (finiteNumber(values.get("memory.events.oom_kill")) ?? 0) > 0,
+            memoryCurrentBytes: finiteNumber(values.get("memory.current")),
+            memoryPeakBytes: finiteNumber(values.get("memory.peak")),
+            memoryHighBytes: finiteNumber(values.get("memory.high")),
+            memoryMaxBytes: finiteNumber(values.get("memory.max")),
+            memorySwapPeakBytes: finiteNumber(values.get("memory.swap.peak")),
+            memorySwapMaxBytes: finiteNumber(values.get("memory.swap.max")),
+            cpuUsageNs: (finiteNumber(values.get("cpu.stat.usage_usec")) ?? 0) * 1_000,
+            tasksCurrent: finiteNumber(values.get("pids.current")),
+            tasksPeak: finiteNumber(values.get("pids.peak")),
+            ioReadBytes,
+            ioWriteBytes,
+        };
+    }
+    catch {
+        return { unitName };
+    }
+    finally {
+        try {
+            unlinkSync(path);
+        }
+        catch { /* best-effort transient metrics cleanup */ }
+    }
+}
+async function systemdUnitExists(unitName) {
+    try {
+        const { stdout } = await execFileAsync("systemctl", ["show", unitName, "--property=LoadState", "--value"], { timeout: 5_000 });
+        return stdout.trim() !== "" && stdout.trim() !== "not-found";
+    }
+    catch {
+        return false;
+    }
+}
+async function stopSystemdUnit(unitName) {
+    try {
+        await execFileAsync("systemctl", ["kill", "--kill-whom=all", "--signal=SIGKILL", unitName], { timeout: 5_000 });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function terminateSystemdUnit(unitName) {
+    await stopSystemdUnit(unitName);
+    await execFileAsync("systemctl", ["stop", unitName], { timeout: 5_000 }).catch(() => undefined);
+}
+async function resetSystemdUnit(unitName) {
+    await execFileAsync("systemctl", ["reset-failed", unitName], { timeout: 5_000 }).catch(() => undefined);
+}
+function cgroupReachedResourceLimit(snapshot) {
+    if (!snapshot)
+        return false;
+    if (snapshot.oomKilled || snapshot.result === "oom-kill")
+        return true;
+    return snapshot.memoryPeakBytes !== undefined
+        && snapshot.memoryMaxBytes !== undefined
+        && snapshot.memoryMaxBytes > 0
+        && snapshot.memoryPeakBytes >= snapshot.memoryMaxBytes;
+}
+function cgroupLimitReason(snapshot) {
+    if (!snapshot)
+        return undefined;
+    if (snapshot.oomKilled || snapshot.result === "oom-kill")
+        return `yielded-run cgroup ${snapshot.unitName} was OOM-killed`;
+    if (snapshot.memoryPeakBytes !== undefined
+        && snapshot.memoryMaxBytes !== undefined
+        && snapshot.memoryMaxBytes > 0
+        && snapshot.memoryPeakBytes >= snapshot.memoryMaxBytes) {
+        return `yielded-run cgroup memory peak ${snapshot.memoryPeakBytes} reached MemoryMax ${snapshot.memoryMaxBytes}`;
+    }
+    return undefined;
+}
+function bashCommand(params) {
+    if (!params || typeof params !== "object" || Array.isArray(params))
+        return undefined;
+    const command = params.command;
+    return typeof command === "string" && command.length > 0 ? command : undefined;
+}
+function yieldedRunUnitName() {
+    return `pibo-yielded-${randomUUID().replaceAll("-", "").slice(0, 24)}.service`;
+}
+function shellQuote(value) {
+    return `'${value.replaceAll("'", `'"'"'`)}'`;
+}
+function readOptionalFile(path) {
+    try {
+        return readFileSync(path, "utf8");
+    }
+    catch {
+        return "";
+    }
+}
+function nonEmpty(value) {
+    return value && value !== "[not set]" ? value : undefined;
+}
+function assignString(output, key, value) {
+    const normalized = nonEmpty(value);
+    if (normalized !== undefined)
+        output[key] = normalized;
+}
+function assignNumber(output, key, value) {
+    const normalized = finiteNumber(value);
+    if (normalized !== undefined)
+        output[key] = normalized;
+}
+function finiteNumber(value) {
+    if (!value || value === "infinity")
+        return undefined;
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : undefined;
+}
+function parseIsolationMode(value, fallback) {
+    const normalized = value?.trim().toLowerCase();
+    if (!normalized)
+        return fallback;
+    return normalized === "off" || normalized === "0" || normalized === "false" ? "off" : "systemd";
+}
+function parsePositiveInteger(value, fallback) {
+    const parsed = Number(value);
+    return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+function parseNonNegativeInteger(value, fallback) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
+}
+function parsePositiveNumber(value, fallback) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+function parseNonNegativeNumber(value, fallback) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+}

package/dist/runs/tools.js

@@ -1,6 +1,7 @@
 import { StringEnum, Type } from "@earendil-works/pi-ai";
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { foregroundServiceWarning, hasMeaningfulTimeoutOutput, isConfiguredTimeoutError, PiboRunExecutionTimeoutError, resolveRunTimeoutMs } from "./lifecycle.js";
+import { PiboRunResourceLimitError, prepareYieldedRunExecution } from "./resource-isolation.js";
 function resultText(prefix, value) {
     return `${prefix}\n${JSON.stringify(value, null, 2)}`;
 }
@@ -46,6 +47,7 @@ export function createRunToolDefinitions(yieldableTools, controller) {
                 const tool = requireTool(yieldableTools, params.toolName);
                 const timeoutMs = resolveRunTimeoutMs(tool.name, params.arguments);
                 const serviceWarning = foregroundServiceWarning(tool.name, params.arguments, timeoutMs);
+                const prepared = prepareYieldedRunExecution(tool.name, params.arguments);
                 let observedOutput = false;
                 const run = controller.startToolRun({
                     toolName: tool.name,
@@ -53,12 +55,13 @@ export function createRunToolDefinitions(yieldableTools, controller) {
                     completionPolicy: params.completionPolicy,
                     timeoutMs,
                     serviceWarning,
+                    resources: prepared.resources,
                     async execute() {
                         try {
-                            const result = await tool.execute(toolCallId, params.arguments, signal, (update) => {
+                            const result = await prepared.execute(() => tool.execute(toolCallId, prepared.params, signal, (update) => {
                                 observedOutput ||= hasMeaningfulTimeoutOutput(update);
                                 onUpdate?.(update);
-                            }, ctx);
+                            }, ctx));
                             const resultObject = result;
                             const text = textFromToolResult(resultObject);
                             if (resultObject.isError === true) {
@@ -69,7 +72,7 @@ export function createRunToolDefinitions(yieldableTools, controller) {
                             return { text, details: resultObject.details ?? result };
                         }
                         catch (error) {
-                            if (error instanceof PiboRunExecutionTimeoutError)
+                            if (error instanceof PiboRunExecutionTimeoutError || error instanceof PiboRunResourceLimitError)
                                 throw error;
                             if (timeoutMs !== undefined && isConfiguredTimeoutError(error))
                                 throw new PiboRunExecutionTimeoutError(error instanceof Error ? error.message : String(error), observedOutput ? "lifetime" : "startup");