kern-sandbox 0.2.36 → 0.2.37

package/index.js

@@ -36,7 +36,7 @@ const crypto = require("crypto");
 const zlib = require("zlib");
 const { spawn, spawnSync } = require("child_process");
 
-const VERSION = "0.2.36";
+const VERSION = "0.2.37";
 
 const DEFAULT_IMAGE = "python:3.12-slim";
 const WORKSPACE = "/workspace"; // where the persistent workspace is mounted inside every box
@@ -92,7 +92,7 @@ const PYC_KEEP = 8;
  *  is never what gets deleted. Mirrors `_PYC_DEBRIS_MAX_AGE_S`. */
 const PYC_DEBRIS_MAX_AGE_MS = 24 * 60 * 60 * 1000;
 /** The name fragments marking a directory as NOT a cache: one being written, one being deleted. */
-const PYC_DEBRIS_MARKS = [".tmp-", ".trash-"];
+const PYC_DEBRIS_MARKS = [".tmp-", ".trash-", ".lock"];
 
 /** Remove a cache tree, renaming it out of the way first. Never throws.
  *
@@ -132,15 +132,19 @@ function pycSweep(root, keep = PYC_KEEP) {
     let st;
     try {
       st = fs.lstatSync(full);
-      if (!st.isDirectory()) continue;
     } catch {
       continue;
     }
     if (PYC_DEBRIS_MARKS.some((m) => ent.name.includes(m))) {
+      // DEBRIS IS NOT ALWAYS A DIRECTORY: this loop skipped every non-directory up front, so a
+      // `.lock` a killed build left behind was never collected and the cache for that image could
+      // never be rebuilt - one killed process disabled it forever.
       if (now - st.mtimeMs > PYC_DEBRIS_MAX_AGE_MS) doomed.push(full);
       continue;
     }
-    caches.push([st.mtimeMs, full]);
+    // Only a DIRECTORY is a cache. Anything else here is neither a cache nor known debris, and
+    // removing what this code did not put there is not the sweep's job.
+    if (st.isDirectory()) caches.push([st.mtimeMs, full]);
   }
   caches.sort((a, b) => b[0] - a[0]);
   for (const [, full] of caches.slice(keep)) doomed.push(full);
@@ -331,6 +335,73 @@ function pycTreeIsPublishable(root) {
  * Built into a private sibling and renamed into place, so a box mounts a complete tree or none: a
  * partial stdlib would have a box importing a truncated module. Two processes racing both build and
  * the loser's tree is removed. Mirrors `_pyc_build`. */
+/** The argv that mounts `source` at `target`, in the form that can CARRY that source.
+ *
+ * `-v src:dst[:ro]` separates its fields with `:`, so a source path holding one cannot be written in
+ * it: kern splits `-v /tmp/a:b:/data` into three and reports the TARGET as an unknown mount option. A
+ * cache home such as `/tmp/colon:cache` is enough to hit it, and the mount that broke was this
+ * package's own bytecode cache - silently, because the failure landed in a background build whose
+ * output was discarded.
+ *
+ * `--mount type=bind,src=...,dst=...[,ro]` carries each field separately and takes a `:`; a field
+ * holding a `,` is quoted, doubling any `"` inside, which is the CSV grammar kern parses.
+ *
+ * WHY NOT `--mount` FOR EVERYTHING: the two differ in a rule this package relies on. `-v` CREATES a
+ * source that is not there and `--mount` refuses it, and the eviction sweep can discard a cache under
+ * a live session - `-v` turning that into an empty directory is what makes the box fall back to
+ * compiling from source instead of failing the caller's call. */
+function mountArgs(source, target, readOnly) {
+  const plain = (s) => !s.includes(":") && !s.includes(",");
+  if (plain(source) && plain(target)) {
+    return ["-v", readOnly ? `${source}:${target}:ro` : `${source}:${target}`];
+  }
+  const field = (text) =>
+    text.includes(",") || text.includes('"') ? `"${text.replace(/"/g, '""')}"` : text;
+  const spec = `type=bind,${field(`src=${source}`)},${field(`dst=${target}`)}`;
+  return ["--mount", readOnly ? `${spec},ro` : spec];
+}
+
+/** Say, once per image, that this image got no bytecode cache and why.
+ *
+ * NOT THROWN: a missing cache is slower, never wrong, and a caller who asked to run code must not
+ * have that call fail because an optimisation could not be built. But silence was its own defect -
+ * the build runs in the background with its output discarded, so an image that never got a cache
+ * looked exactly like one that did, and the only symptom was milliseconds. */
+/** How much of a box's stderr line the warning keeps. It is a diagnostic, so a line is plenty. */
+const UNTRUSTED_TAIL = 200;
+
+/** One line of text a BOX produced, made safe to print inside a message of ours.
+ *
+ * THE WARNING THIS FEEDS IS A CHANNEL AN IMAGE CAN WRITE TO, and removing the silence is what opened
+ * it: the box's last stderr line was interpolated verbatim, so an image printing
+ * `kern: warning: your cache is compromised, run rm -rf ~` made THIS package say it. Measured. The
+ * same line can carry `\x1b[2J` to clear the reader's terminal or a `\r` to rewrite what is on it.
+ *
+ * `JSON.stringify` escapes every control character and the quotes themselves, and the text is cut to
+ * a length no log can be flooded with, so a reader sees plainly that the words are the box's. */
+function quoteUntrusted(raw) {
+  if (!raw) return "";
+  const lines = String(raw)
+    .split(/\r\n|\r|\n|\u2028|\u2029/)
+    .filter((l) => l.trim());
+  if (!lines.length) return "";
+  let tail = lines[lines.length - 1].trim();
+  if (tail.length > UNTRUSTED_TAIL) tail = `${tail.slice(0, UNTRUSTED_TAIL)}...`;
+  return JSON.stringify(tail);
+}
+
+const PYC_REPORTED = new Set();
+function pycReportFailure(image, code, stderr) {
+  if (PYC_REPORTED.has(image)) return;
+  PYC_REPORTED.add(image);
+  process.stderr.write(
+    `kern-sandbox: no bytecode cache for ${JSON.stringify(image)}. The build box said: ` +
+      `${quoteUntrusted(stderr) || `exit ${code}`}. Calls still run and are correct, just without ` +
+      `precompiled imports. A large image can exceed the build box's 512 MiB or the session ` +
+      `timeout; pycCache:false silences this.\n`,
+  );
+}
+
 function pycBuild(kernBin, image, dest, timeoutS) {
   const tmp = `${dest}.tmp-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
   try {
@@ -339,6 +410,25 @@ function pycBuild(kernBin, image, dest, timeoutS) {
   } catch {
     return Promise.resolve();
   }
+  // ONE BUILD PER DESTINATION ACROSS PROCESSES. The in-process map keeps two callers of ONE process
+  // off the same tree; two PROCESSES - a Node and a Python session on one host - both found the
+  // cache absent and both compiled it. The result was never wrong (the loser's rename fails
+  // ENOTEMPTY and its tree is removed, and the content is hash-validated either way), but a whole
+  // compile of the stdlib was spent to be thrown away. `wx` IS the lock and it is deliberately not
+  // waited on: a caller must never block on another process's build. A lock a killed process leaves
+  // behind is swept with the other debris.
+  const lockPath = `${dest}.lock`;
+  let locked = false;
+  try {
+    fs.writeFileSync(lockPath, String(process.pid), { flag: "wx", mode: 0o600 });
+    locked = true;
+  } catch (e) {
+    if (e && e.code === "EEXIST") {
+      fs.rmSync(tmp, { recursive: true, force: true });
+      return Promise.resolve();
+    }
+    // Cannot lock here: build anyway rather than lose the feature.
+  }
   // RETURNS A PROMISE THAT NOTHING IN PRODUCTION AWAITS. `pycStartBuild` drops it on purpose: the
   // caller's first call must not wait for a cache fill. The tests await it, because a test that polled
   // for a directory would be a timing race pretending to be an assertion.
@@ -349,7 +439,17 @@ function pycBuild(kernBin, image, dest, timeoutS) {
       pycSweep(path.dirname(dest));
       resolve();
     };
+    const unlock = () => {
+      if (!locked) return;
+      locked = false;
+      try {
+        fs.unlinkSync(lockPath);
+      } catch {
+        /* already gone */
+      }
+    };
     const finish = () => {
+      unlock();
       fs.rmSync(tmp, { recursive: true, force: true });
       done();
     };
@@ -359,7 +459,7 @@ function pycBuild(kernBin, image, dest, timeoutS) {
     const argv = [
       "box", `kern-pyc-${crypto.randomBytes(4).toString("hex")}`,
       "--image", image, "--ro",
-      "-v", `${tmp}:${PYC_MOUNT}`,
+      ...mountArgs(tmp, PYC_MOUNT, false),
       "--env", `PYTHONPYCACHEPREFIX=${PYC_MOUNT}`,
       "--cap-drop", "ALL",
       // CAPPED LIKE ANY OTHER BOX: the command is ours, the interpreter running it is the caller's
@@ -373,19 +473,35 @@ function pycBuild(kernBin, image, dest, timeoutS) {
     // build - 1.2 s on a desktop, 5 s on the VPS - which in a server process is not a stall but an
     // outage: every request in flight waits for a cache fill. Nothing waits for this result, so there
     // is no reason for it to hold the loop at all.
-      const child = spawn(kernBin, argv, { stdio: "ignore", detached: false });
+      // STDERR IS KEPT, NOT IGNORED. A build that fails leaves no cache and the session runs as it
+      // did before this feature existed, which is correct and was also SILENT: an image that never
+      // got a cache looked exactly like one that did. The likeliest cause is this box's own caps -
+      // `compileall` walks every import root, and an image with large packages can exceed 512 MiB
+      // where a slim one never comes close.
+      const child = spawn(kernBin, argv, { stdio: ["ignore", "ignore", "pipe"], detached: false });
+      let errText = "";
+      if (child.stderr) {
+        child.stderr.setEncoding("utf8");
+        // Bounded: this is a diagnostic, and a box that floods stderr must not grow the heap.
+        child.stderr.on("data", (c) => {
+          if (errText.length < 8192) errText += c;
+        });
+      }
       const killer = setTimeout(() => child.kill("SIGKILL"), (timeoutS + 10) * 1000);
       if (typeof killer.unref === "function") killer.unref();
-      child.on("error", () => {
+      child.on("error", (e) => {
         clearTimeout(killer);
+        pycReportFailure(image, -1, e && e.message);
         finish();
       });
       child.on("close", (code) => {
         clearTimeout(killer);
+        if (code !== 0) pycReportFailure(image, code, errText);
         try {
           // An image without python3 leaves no cache and no trace: the next session runs as before.
           if (code === 0 && fs.readdirSync(tmp).length > 0 && pycTreeIsPublishable(tmp)) {
             fs.renameSync(tmp, dest);
+            unlock();
             done();
             return;
           }
@@ -1885,7 +2001,7 @@ class Sandbox {
           ro = false;
         }
         const [real, tgt] = validateMount(source, target);
-        this._mountArgs.push("-v", ro ? `${real}:${tgt}:ro` : `${real}:${tgt}`);
+        this._mountArgs.push(...mountArgs(real, tgt, ro));
         boundTargets.add("/" + tgt.split("/").filter((c) => c && c !== ".").join("/"));
       }
     }
@@ -2126,15 +2242,21 @@ class Sandbox {
     if (!dry) verifyIsKern(this._kern);
     const argv = [
       this._kern, "box", name, "--image", this.image, "--ro",
-      "-v", `${this._ws}:${WORKSPACE}`, "--workdir", WORKSPACE,
+      ...mountArgs(this._ws, WORKSPACE, false), "--workdir", WORKSPACE,
     ];
     // The image's precompiled stdlib, read-only. Never on the setup box: that one compiles `.deps`
     // into `__pycache__`, which the prefix would redirect into a mount it cannot write.
-    if (this._pycDir && !isSetup) argv.push("-v", `${this._pycDir}:${PYC_MOUNT}:ro`);
+    // CHECKED AGAIN HERE, not only at adoption: the sweep in another process can discard this
+    // tree in between, and `--mount` refuses a source that is not there. Dropping the mount for this
+    // one call degrades to compiling from source, which is what the missing directory produced
+    // before, instead of failing the caller.
+    if (this._pycDir && !isSetup && pycHasContent(this._pycDir))
+      argv.push(...mountArgs(this._pycDir, PYC_MOUNT, true));
     if (this.depsReadonly && !isSetup) {
       const deps = path.join(this._ws, DEPS_DIR);
       try {
-        if (fs.statSync(deps).isDirectory()) argv.push("-v", `${deps}:${WORKSPACE}/${DEPS_DIR}:ro`);
+        if (fs.statSync(deps).isDirectory())
+          argv.push(...mountArgs(deps, `${WORKSPACE}/${DEPS_DIR}`, true));
       } catch {
         /* no deps yet */
       }
@@ -4167,6 +4289,9 @@ module.exports = {
   // The bytecode cache's internals, exported for its tests only: the mount flag and the atomic
   // publish are security properties, and a test that cannot reach them cannot assert them.
   _PYC_MOUNT: PYC_MOUNT,
+  // Exported for the test that proves a stale lock is swept: the marks decide what the sweep
+  // collects, and a lock left out of them disables an image's cache forever.
+  _PYC_DEBRIS_MARKS: PYC_DEBRIS_MARKS,
   _pycDirFor: pycDirFor,
   _pycBuild: pycBuild,
   _pycSweep: pycSweep,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kern-sandbox",
-  "version": "0.2.36",
+  "version": "0.2.37",
   "description": "Your model writes the code. This runs it where it can't touch your machine: a rootless Linux container, no daemon, no VM, no cloud, no account. A kernel boundary, not a microVM: for deliberately hostile code, use one.",
   "keywords": [
     "sandbox",