@ricsam/r5d-worker 0.0.109 → 0.0.111

package/README.md

@@ -71,4 +71,16 @@ worker exec macos bun test
 
 Agent shell commands receive no stdin by default: prompts read immediate EOF instead of hanging. A command started with `interactive: true` is spawned with a writable stdin pipe, and the agent's `shell_write` tool delivers input through the `exec_stdin` protocol message. The worker acknowledges each write with an `operation_result` carrying the delivered byte count, closes stdin on an explicit `eof`, on cancellation, on timeout, and at process exit, and advertises this support through the `execStdinV1` capability in its hello message. Servers refuse interactive runs and stdin writes for workers without the capability, so this is a protocol-compatible release that uses the ordinary worker update flow. Delivered stdin is audited server-side in the run's `process-runs/<runId>/stdin.log`.
 
+### Command resource limits
+
+Managed Linux workers require cgroup v2 delegation with `Delegate=memory pids`, `DelegateSubgroup=control`, and `OOMPolicy=continue`. The supervisor and control runtime stay in the service's `control` subgroup. Every command and PTY enters its own memory-limited cgroup before executing user code, and all descendants remain under `r5d-worker.service` for credential-generation cleanup. Each job has a process limit of 512 and group OOM termination; a job that exhausts memory reports a command/terminal resource failure instead of stopping the worker. Control processes receive OOM protection, while the command wrapper removes that protection before executing user code.
+
+Admission is shared across all projects in the runtime: two general commands or terminals can run concurrently. A separate two-slot utility lane, capped at 256 MiB per job, serves explicitly effect-free inline RPC commands. File operations, cancellation, and connection recovery do not wait in either queue. Inline RPC output is limited to 8 MiB per stream so a small producer cannot fill the control runtime; larger output requires streaming execution. General jobs reserve the rest of the configured aggregate budget. The managed VM sets direct jobs to 50% of effective RAM and uses a separately bounded default BuildKit builder (up to 25%, capped at 2 GiB), leaving at least 25% for the OS and control services. Effective RAM honors the tightest ancestor cgroup limit.
+
+Queued commands retain their durable operation identities across reconnects. They wait at most 60 seconds before returning a capacity error without starting a command; cancellation also prevents a command whose asynchronous preparation is still in progress from starting. This bound prevents indefinite capacity deadlocks when all running shells wait on nested `r5dctl` work. Retry later or stop an occupied command/terminal when capacity is exhausted. Transient disconnects preserve running commands and PTYs; expiry of the five-minute execution lease closes them and cancels queued work.
+
+The standalone default aggregate limit is 70% of effective RAM. Operators can set `R5D_WORKER_JOBS_MEMORY_PERCENT` (1–70), `R5D_WORKER_JOBS_MEMORY_BYTES`, `R5D_WORKER_JOB_MEMORY_BYTES`, `R5D_WORKER_UTILITY_MEMORY_BYTES`, `R5D_WORKER_JOB_CONCURRENCY`, and `R5D_WORKER_UTILITY_CONCURRENCY`. The sum of all slot reservations must fit the aggregate budget. `R5D_WORKER_JOB_QUEUE_TIMEOUT_MS` is capped at 60000; `R5D_WORKER_JOB_QUEUE_LIMIT` defaults to 128. Managed services set `R5D_WORKER_RESOURCE_ISOLATION=required` and refuse command service if delegation is unavailable. Direct workers on other platforms use bounded concurrency and explicitly report `concurrency-only`; they do not claim kernel memory isolation.
+
+Cgroups constrain direct descendants, not workloads launched by separate daemons. The managed default `BUILDX_BUILDER` selects the bounded build daemon, but explicit alternate builders and arbitrary `docker run`/Compose containers need their own memory limits. These controls prevent accidental resource exhaustion; they are not a hostile same-UID sandbox.
+
 When the connected `r5d-browser` requests a port forward, the worker opens each relayed connection only to `127.0.0.1` on the requested worker port. Browser-side and worker-side ports may differ. The worker never opens a public listener, and a disconnected worker leaves the browser's long-lived mapping unavailable until the same worker label reconnects.

package/dist/cjs/job-resources.cjs

@@ -0,0 +1,335 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var job_resources_exports = {};
+__export(job_resources_exports, {
+  WORKER_INLINE_OUTPUT_MAX_BYTES: () => WORKER_INLINE_OUTPUT_MAX_BYTES,
+  WorkerJobCancelledError: () => WorkerJobCancelledError,
+  WorkerJobCapacityError: () => WorkerJobCapacityError,
+  WorkerJobResourceController: () => WorkerJobResourceController,
+  collectWorkerInlineOutput: () => collectWorkerInlineOutput,
+  createWorkerJobResources: () => createWorkerJobResources,
+  effectiveWorkerMemory: () => effectiveWorkerMemory,
+  workerDelegatedCgroupRoot: () => workerDelegatedCgroupRoot,
+  workerJobBudget: () => workerJobBudget
+});
+module.exports = __toCommonJS(job_resources_exports);
+var import_node_fs = __toESM(require("node:fs"), 1);
+var import_node_os = __toESM(require("node:os"), 1);
+var import_node_path = __toESM(require("node:path"), 1);
+const MIB = 1024 * 1024;
+const WORKER_INLINE_OUTPUT_MAX_BYTES = 8 * MIB;
+async function collectWorkerInlineOutput(stream, maxBytes = WORKER_INLINE_OUTPUT_MAX_BYTES, signal) {
+  if (!stream) return "";
+  signal?.throwIfAborted();
+  const reader = stream.getReader();
+  const abort = () => {
+    void reader.cancel().catch(() => {
+    });
+  };
+  signal?.addEventListener("abort", abort, { once: true });
+  const decoder = new TextDecoder();
+  const parts = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      const next = await reader.read();
+      signal?.throwIfAborted();
+      if (next.done) break;
+      bytes += next.value.byteLength;
+      if (bytes > maxBytes) {
+        void reader.cancel().catch(() => {
+        });
+        throw new Error(`Inline command output exceeded ${maxBytes} bytes. Use streaming execution for large output.`);
+      }
+      parts.push(decoder.decode(next.value, { stream: true }));
+    }
+    parts.push(decoder.decode());
+    return parts.join("");
+  } finally {
+    signal?.removeEventListener("abort", abort);
+    reader.releaseLock();
+  }
+}
+function positiveInteger(value, fallback, name) {
+  const result = value === void 0 ? fallback : Number(value);
+  if (!Number.isSafeInteger(result) || result <= 0) throw new Error(`${name} must be a positive integer`);
+  return result;
+}
+function workerJobBudget(effectiveMemoryBytes, env = process.env) {
+  const generalSlots = positiveInteger(env.R5D_WORKER_JOB_CONCURRENCY, 2, "R5D_WORKER_JOB_CONCURRENCY");
+  const utilitySlots = positiveInteger(env.R5D_WORKER_UTILITY_CONCURRENCY, 2, "R5D_WORKER_UTILITY_CONCURRENCY");
+  const memoryPercent = positiveInteger(env.R5D_WORKER_JOBS_MEMORY_PERCENT, 70, "R5D_WORKER_JOBS_MEMORY_PERCENT");
+  if (memoryPercent > 70) throw new Error("Worker jobs may reserve at most 70% of effective memory");
+  const aggregateBytes = positiveInteger(
+    env.R5D_WORKER_JOBS_MEMORY_BYTES,
+    Math.floor(effectiveMemoryBytes * memoryPercent / 100),
+    "R5D_WORKER_JOBS_MEMORY_BYTES"
+  );
+  const utilityBytes = positiveInteger(
+    env.R5D_WORKER_UTILITY_MEMORY_BYTES,
+    Math.min(256 * MIB, Math.floor(aggregateBytes / 10)),
+    "R5D_WORKER_UTILITY_MEMORY_BYTES"
+  );
+  const generalBytes = positiveInteger(
+    env.R5D_WORKER_JOB_MEMORY_BYTES,
+    Math.floor((aggregateBytes - utilityBytes * utilitySlots) / generalSlots),
+    "R5D_WORKER_JOB_MEMORY_BYTES"
+  );
+  if (aggregateBytes > Math.floor(effectiveMemoryBytes * 0.7)) throw new Error("Worker jobs may reserve at most 70% of effective memory");
+  if (generalBytes * generalSlots + utilityBytes * utilitySlots > aggregateBytes)
+    throw new Error("Worker job limits exceed the aggregate memory budget");
+  const queueTimeoutMs = positiveInteger(env.R5D_WORKER_JOB_QUEUE_TIMEOUT_MS, 6e4, "R5D_WORKER_JOB_QUEUE_TIMEOUT_MS");
+  if (queueTimeoutMs > 6e4) throw new Error("Worker job queue timeout cannot exceed 60000ms");
+  return {
+    aggregateBytes,
+    generalBytes,
+    utilityBytes,
+    generalSlots,
+    utilitySlots,
+    queueTimeoutMs,
+    maxQueued: positiveInteger(env.R5D_WORKER_JOB_QUEUE_LIMIT, 128, "R5D_WORKER_JOB_QUEUE_LIMIT"),
+    pidsPerJob: positiveInteger(env.R5D_WORKER_JOB_PIDS, 512, "R5D_WORKER_JOB_PIDS")
+  };
+}
+function workerDelegatedCgroupRoot(cgroupText, mount = "/sys/fs/cgroup") {
+  const relative = cgroupText.split("\n").find((line) => line.startsWith("0::/"))?.slice(3);
+  if (!relative) return void 0;
+  const segments = relative.split("/").filter(Boolean);
+  const index = segments.indexOf("r5d-worker.service");
+  if (index < 0 || segments[index + 1] !== "control") return void 0;
+  return import_node_path.default.join(mount, ...segments.slice(0, index + 1));
+}
+function effectiveWorkerMemory(root, hostBytes = import_node_os.default.totalmem(), mount = "/sys/fs/cgroup") {
+  let result = hostBytes;
+  if (!root) return result;
+  let current = root;
+  while (current === mount || current.startsWith(`${mount}/`)) {
+    try {
+      const limit = Number(import_node_fs.default.readFileSync(import_node_path.default.join(current, "memory.max"), "utf8").trim());
+      if (Number.isSafeInteger(limit) && limit > 0) result = Math.min(result, limit);
+    } catch {
+    }
+    if (current === mount) break;
+    current = import_node_path.default.dirname(current);
+  }
+  return result;
+}
+class WorkerJobCapacityError extends Error {
+  constructor() {
+    super("Worker command capacity is busy. No command was started; retry later or stop another command or terminal.");
+    this.name = "WorkerJobCapacityError";
+  }
+}
+class WorkerJobCancelledError extends Error {
+  constructor() {
+    super("Command cancelled before it started");
+    this.name = "WorkerJobCancelledError";
+  }
+}
+class WorkerJobResourceController {
+  constructor(budget, delegatedRoot) {
+    this.budget = budget;
+    this.isolation = delegatedRoot ? "cgroup-v2" : "concurrency-only";
+    if (delegatedRoot) {
+      if (import_node_fs.default.readFileSync(import_node_path.default.join(delegatedRoot, "cgroup.procs"), "utf8").trim())
+        throw new Error("Worker cgroup delegation root must be empty; configure DelegateSubgroup=control");
+      import_node_fs.default.writeFileSync(import_node_path.default.join(delegatedRoot, "cgroup.subtree_control"), "+memory +pids");
+      this.jobsRoot = import_node_path.default.join(delegatedRoot, "jobs");
+      import_node_fs.default.mkdirSync(this.jobsRoot, { recursive: true });
+      import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "memory.max"), String(budget.aggregateBytes));
+      import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "memory.swap.max"), "0");
+      import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "memory.oom.group"), "0");
+      import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "cgroup.subtree_control"), "+memory +pids");
+      for (const name of import_node_fs.default.readdirSync(this.jobsRoot)) {
+        if (!/^job-[0-9a-f-]+$/.test(name)) continue;
+        const old = import_node_path.default.join(this.jobsRoot, name);
+        if (!import_node_fs.default.statSync(old).isDirectory()) continue;
+        import_node_fs.default.writeFileSync(import_node_path.default.join(old, "cgroup.kill"), "1");
+      }
+    }
+  }
+  budget;
+  isolation;
+  jobsRoot;
+  active = /* @__PURE__ */ new Map();
+  waiters = [];
+  timer;
+  get status() {
+    return { isolation: this.isolation, active: this.active.size, queued: this.waiters.length };
+  }
+  acquire(input) {
+    try {
+      input.assertAdmission();
+      if (this.active.has(input.id) || this.waiters.some((entry) => entry.id === input.id))
+        throw new Error(`Duplicate resource admission: ${input.id}`);
+      if (this.available(input.lane)) return Promise.resolve(this.start(input));
+      if (this.waiters.length >= this.budget.maxQueued) throw new WorkerJobCapacityError();
+      return new Promise((resolve, reject) => {
+        this.waiters.push({ ...input, resolve, reject, deadline: Date.now() + this.budget.queueTimeoutMs });
+        this.timer ??= setInterval(() => this.drain(), Math.min(100, this.budget.queueTimeoutMs));
+      });
+    } catch (error) {
+      return Promise.reject(error);
+    }
+  }
+  cancel(id) {
+    for (let index = this.waiters.length - 1; index >= 0; index--) {
+      if (this.waiters[index].id !== id) continue;
+      this.waiters.splice(index, 1)[0].reject(new WorkerJobCancelledError());
+    }
+    this.active.get(id)?.lease.kill();
+    this.drain();
+  }
+  cancelSession(sessionId) {
+    this.cancelMatching((_id, ownerSessionId) => ownerSessionId === sessionId);
+  }
+  cancelMatching(matches) {
+    const ids = /* @__PURE__ */ new Set([
+      ...this.waiters.filter((entry) => matches(entry.id, entry.sessionId)).map((entry) => entry.id),
+      ...[...this.active].filter(([id, entry]) => matches(id, entry.sessionId)).map(([id]) => id)
+    ]);
+    const errors = [];
+    for (const id of ids) {
+      try {
+        this.cancel(id);
+      } catch (error) {
+        errors.push(error);
+      }
+    }
+    if (errors.length) throw new AggregateError(errors, "Worker job cancellation was incomplete");
+  }
+  available(lane) {
+    return [...this.active.values()].filter((entry) => entry.lane === lane).length < (lane === "general" ? this.budget.generalSlots : this.budget.utilitySlots);
+  }
+  drain() {
+    for (let index = 0; index < this.waiters.length; ) {
+      const waiter = this.waiters[index];
+      try {
+        waiter.assertAdmission();
+        if (Date.now() >= waiter.deadline) throw new WorkerJobCapacityError();
+        if (!this.available(waiter.lane)) {
+          index++;
+          continue;
+        }
+        this.waiters.splice(index, 1);
+        waiter.resolve(this.start(waiter));
+      } catch (error) {
+        if (this.waiters[index] === waiter) this.waiters.splice(index, 1);
+        waiter.reject(error);
+      }
+    }
+    if (this.waiters.length === 0 && this.timer) {
+      clearInterval(this.timer);
+      this.timer = void 0;
+    }
+  }
+  start(input) {
+    input.assertAdmission();
+    const memoryBytes = input.lane === "general" ? this.budget.generalBytes : this.budget.utilityBytes;
+    const cgroupPath = this.jobsRoot ? import_node_path.default.join(this.jobsRoot, `job-${crypto.randomUUID()}`) : void 0;
+    if (cgroupPath) {
+      import_node_fs.default.mkdirSync(cgroupPath);
+      try {
+        import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "memory.max"), String(memoryBytes));
+        import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "memory.swap.max"), "0");
+        import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "memory.oom.group"), "1");
+        import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "pids.max"), String(this.budget.pidsPerJob));
+      } catch (error) {
+        import_node_fs.default.rmdirSync(cgroupPath);
+        throw error;
+      }
+    }
+    let released = false;
+    let revoked = false;
+    let releasePromise;
+    const kill = () => {
+      revoked = true;
+      if (!released && cgroupPath) import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "cgroup.kill"), "1");
+    };
+    const lease = {
+      cgroupPath,
+      memoryBytes,
+      wrap: (argv) => {
+        if (released) throw new Error("Worker job resource lease was already released");
+        if (revoked) throw new WorkerJobCancelledError();
+        if (!argv.length) throw new Error("Command argv must not be empty");
+        return cgroupPath ? [
+          "/bin/sh",
+          "-c",
+          'set -e; printf "%s" "$$" > "$1/cgroup.procs"; read -r oom < /proc/self/oom_score_adj; if [ "$oom" -lt 0 ]; then printf 0 > /proc/self/oom_score_adj; fi; shift; exec "$@"',
+          "r5d-job",
+          cgroupPath,
+          ...argv
+        ] : argv;
+      },
+      kill,
+      memoryLimitExceeded: () => {
+        if (!cgroupPath || released) return false;
+        return /^oom_kill [1-9][0-9]*$/m.test(import_node_fs.default.readFileSync(import_node_path.default.join(cgroupPath, "memory.events"), "utf8"));
+      },
+      release: () => releasePromise ??= (async () => {
+        kill();
+        if (cgroupPath) {
+          const deadline = Date.now() + 2e3;
+          while (/^populated 1$/m.test(import_node_fs.default.readFileSync(import_node_path.default.join(cgroupPath, "cgroup.events"), "utf8"))) {
+            if (Date.now() >= deadline) throw new Error(`Worker job cgroup remained populated: ${cgroupPath}`);
+            await new Promise((resolve) => setTimeout(resolve, 10));
+          }
+          import_node_fs.default.rmdirSync(cgroupPath);
+        }
+        released = true;
+        this.active.delete(input.id);
+        this.drain();
+      })()
+    };
+    this.active.set(input.id, { lane: input.lane, sessionId: input.sessionId, lease });
+    return lease;
+  }
+}
+function createWorkerJobResources(env = process.env) {
+  const mode = env.R5D_WORKER_RESOURCE_ISOLATION ?? "auto";
+  if (mode !== "auto" && mode !== "required") throw new Error("R5D_WORKER_RESOURCE_ISOLATION must be auto or required");
+  const root = process.platform === "linux" ? workerDelegatedCgroupRoot(import_node_fs.default.readFileSync("/proc/self/cgroup", "utf8")) : void 0;
+  if (mode === "required" && !root)
+    throw new Error("Worker requires delegated cgroup v2 resource isolation; configure Delegate=memory pids and DelegateSubgroup=control");
+  const budget = workerJobBudget(effectiveWorkerMemory(root), env);
+  return new WorkerJobResourceController(budget, root);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  WORKER_INLINE_OUTPUT_MAX_BYTES,
+  WorkerJobCancelledError,
+  WorkerJobCapacityError,
+  WorkerJobResourceController,
+  collectWorkerInlineOutput,
+  createWorkerJobResources,
+  effectiveWorkerMemory,
+  workerDelegatedCgroupRoot,
+  workerJobBudget
+});