kern-sandbox 0.2.36 → 0.2.38

package/index.js

@@ -36,7 +36,7 @@ const crypto = require("crypto");
 const zlib = require("zlib");
 const { spawn, spawnSync } = require("child_process");
 
-const VERSION = "0.2.36";
+const VERSION = "0.2.38";
 
 const DEFAULT_IMAGE = "python:3.12-slim";
 const WORKSPACE = "/workspace"; // where the persistent workspace is mounted inside every box
@@ -92,7 +92,7 @@ const PYC_KEEP = 8;
  *  is never what gets deleted. Mirrors `_PYC_DEBRIS_MAX_AGE_S`. */
 const PYC_DEBRIS_MAX_AGE_MS = 24 * 60 * 60 * 1000;
 /** The name fragments marking a directory as NOT a cache: one being written, one being deleted. */
-const PYC_DEBRIS_MARKS = [".tmp-", ".trash-"];
+const PYC_DEBRIS_MARKS = [".tmp-", ".trash-", ".lock"];
 
 /** Remove a cache tree, renaming it out of the way first. Never throws.
  *
@@ -132,15 +132,19 @@ function pycSweep(root, keep = PYC_KEEP) {
     let st;
     try {
       st = fs.lstatSync(full);
-      if (!st.isDirectory()) continue;
     } catch {
       continue;
     }
     if (PYC_DEBRIS_MARKS.some((m) => ent.name.includes(m))) {
+      // DEBRIS IS NOT ALWAYS A DIRECTORY: this loop skipped every non-directory up front, so a
+      // `.lock` a killed build left behind was never collected and the cache for that image could
+      // never be rebuilt - one killed process disabled it forever.
       if (now - st.mtimeMs > PYC_DEBRIS_MAX_AGE_MS) doomed.push(full);
       continue;
     }
-    caches.push([st.mtimeMs, full]);
+    // Only a DIRECTORY is a cache. Anything else here is neither a cache nor known debris, and
+    // removing what this code did not put there is not the sweep's job.
+    if (st.isDirectory()) caches.push([st.mtimeMs, full]);
   }
   caches.sort((a, b) => b[0] - a[0]);
   for (const [, full] of caches.slice(keep)) doomed.push(full);
@@ -156,8 +160,13 @@ let PYC_SWEPT = false;
 
 /** The host directory holding one bytecode cache per image: $XDG_CACHE_HOME, else ~/.cache. */
 function pycRoot() {
-  const base = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
-  return path.join(base, "kern-sandbox", "pyc");
+  return path.join(cacheHome(), "kern-sandbox", "pyc");
+}
+
+/** `$XDG_CACHE_HOME`, or the default. ONE spelling, because two things read it now: this package's
+ *  bytecode cache and kern's own image cache, which the identity check compares against. */
+function cacheHome() {
+  return process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
 }
 
 /** kern's own defaults, from `kern-oci/src/pull.rs`. Copied because they cross a process boundary. */
@@ -222,6 +231,87 @@ function ociCanonicalRef(image) {
  * cheaply. A moved tag therefore yields bytecode that no longer validates, which CPython handles by
  * recompiling: the cost of the imperfect key is a slow call, never a wrong one. Mirrors `_pyc_dir_for`.
  */
+/** Vectors DUMPED from `sanitize_ref` in kern-cli, not derived from reading it, and asserted in the
+ *  tests: this names files kern WROTE, so the two must agree or the identity check finds nothing. */
+const SANITIZE_VECTORS = [
+  ["python:3.12-slim", "python_3_12-slim-7d2794a436662d45"],
+  ["python", "python_latest-9e0094521a5d04a4"],
+  ["alpine:3.19", "alpine_3_19-441817d3f5f11093"],
+  ["docker.io/library/python:3.12-slim", "docker_io_library_python_3_12-slim-75b604835a32490a"],
+  ["index.docker.io/python:3.12-slim", "index_docker_io_python_3_12-slim-7eaa793ab37b8560"],
+  ["ghcr.io/owner/img:v1", "ghcr_io_owner_img_v1-774c4c7639e8355e"],
+  ["localhost:5000/x:1", "localhost_5000_x_1-03406c180a22063b"],
+  ["python@sha256:abcdef0123456789", "python_sha256_abcdef0123456789-f77941d5fa12216c"],
+  ["a.b/c_d-e:f.g", "a_b_c_d-e_f_g-479120b609c12cc0"],
+  ["UPPER/Case:Tag", "UPPER_Case_Tag-543eaf57172e6242"],
+  ["x:latest", "x_latest-e9f897124d940074"],
+  ["registry-1.docker.io/library/alpine:3.19", "registry-1_docker_io_library_alpine_3_19-49727cc13d78066f"],
+  ["my_img", "my_img_latest-68dbbb774ad018b0"],
+  ["a/b/c:d", "a_b_c_d-dcc54f31688d5fa5"],
+  ["1.2.3.4:5000/p/q:r", "1_2_3_4_5000_p_q_r-f872df9eeecd4beb"],
+];
+
+/** FNV-1a 64-bit, kern's own, used ONLY to keep a cache key collision-free. */
+function fnv1a(s) {
+  let h = 0xcbf29ce484222325n;
+  for (const b of Buffer.from(s, "utf8")) {
+    h = BigInt.asUintN(64, (h ^ BigInt(b)) * 0x100000001b3n);
+  }
+  return h.toString(16).padStart(16, "0");
+}
+
+/** The directory name kern gives an image in its own cache. A PORT, verified against the original. */
+function sanitizeRef(image) {
+  const ref = ociSplitTag(image) ? image : `${image}:latest`;
+  const out = [...ref].map((c) => (/[A-Za-z0-9_-]/.test(c) ? c : "_")).join("");
+  return `${out}-${fnv1a(ref)}`;
+}
+
+/** The file, inside a published cache, naming the image kern had when the cache was built. */
+const PYC_SOURCE_ID = ".kern-source-id";
+
+/** What kern's own image cache holds for `image`, or "" if it cannot be read.
+ *
+ * A tag is MUTABLE, so a cache keyed on its name can outlive the image it was built from. Nothing
+ * wrong is executed (CHECKED_HASH makes CPython reject the stale bytecode) but the cache stops
+ * helping and nothing rebuilds it, because a cache "exists". kern rewrites its own config sidecar and
+ * completion sentinel when it re-pulls a moved tag, so their bytes are an identity that costs a local
+ * read of a few hundred bytes - cheap enough for a check on every open().
+ *
+ * "" MEANS "CANNOT TELL" and is treated as unchanged, so a cache from before this check, or a host
+ * whose image kern has pruned, is never rebuilt in a loop. */
+function pycSourceId(image) {
+  try {
+    const root = path.join(cacheHome(), "kern", "images");
+    const safe = sanitizeRef(image);
+    const h = crypto.createHash("sha256");
+    for (const suffix of [".image", ".ok"]) {
+      h.update(fs.readFileSync(path.join(root, safe + suffix)).subarray(0, 4096));
+      h.update(Buffer.from([0]));
+    }
+    return h.digest("hex").slice(0, 32);
+  } catch {
+    return "";
+  }
+}
+
+/** Is this cache still the one this image would produce? If not, DISCARD it so a build can replace it.
+ *
+ * Discarded and not merely refused, because a rename onto a NON-EMPTY directory is ENOTEMPTY: a stale
+ * tree left in place would block its own replacement forever. */
+function pycSourceMatches(dest, image) {
+  let stored;
+  try {
+    stored = fs.readFileSync(path.join(dest, PYC_SOURCE_ID), "utf8").trim();
+  } catch {
+    return true; // built before this check, or unreadable: leave it exactly as it was
+  }
+  const current = pycSourceId(image);
+  if (!stored || !current || stored === current) return true;
+  pycDiscard(dest);
+  return false;
+}
+
 function pycDirFor(image) {
   // The full digest: the key is no longer load-bearing for correctness, and a truncation saved 48
   // characters of path against two images sharing a cache directory.
@@ -331,6 +421,73 @@ function pycTreeIsPublishable(root) {
  * Built into a private sibling and renamed into place, so a box mounts a complete tree or none: a
  * partial stdlib would have a box importing a truncated module. Two processes racing both build and
  * the loser's tree is removed. Mirrors `_pyc_build`. */
+/** The argv that mounts `source` at `target`, in the form that can CARRY that source.
+ *
+ * `-v src:dst[:ro]` separates its fields with `:`, so a source path holding one cannot be written in
+ * it: kern splits `-v /tmp/a:b:/data` into three and reports the TARGET as an unknown mount option. A
+ * cache home such as `/tmp/colon:cache` is enough to hit it, and the mount that broke was this
+ * package's own bytecode cache - silently, because the failure landed in a background build whose
+ * output was discarded.
+ *
+ * `--mount type=bind,src=...,dst=...[,ro]` carries each field separately and takes a `:`; a field
+ * holding a `,` is quoted, doubling any `"` inside, which is the CSV grammar kern parses.
+ *
+ * WHY NOT `--mount` FOR EVERYTHING: the two differ in a rule this package relies on. `-v` CREATES a
+ * source that is not there and `--mount` refuses it, and the eviction sweep can discard a cache under
+ * a live session - `-v` turning that into an empty directory is what makes the box fall back to
+ * compiling from source instead of failing the caller's call. */
+function mountArgs(source, target, readOnly) {
+  const plain = (s) => !s.includes(":") && !s.includes(",");
+  if (plain(source) && plain(target)) {
+    return ["-v", readOnly ? `${source}:${target}:ro` : `${source}:${target}`];
+  }
+  const field = (text) =>
+    text.includes(",") || text.includes('"') ? `"${text.replace(/"/g, '""')}"` : text;
+  const spec = `type=bind,${field(`src=${source}`)},${field(`dst=${target}`)}`;
+  return ["--mount", readOnly ? `${spec},ro` : spec];
+}
+
+/** Say, once per image, that this image got no bytecode cache and why.
+ *
+ * NOT THROWN: a missing cache is slower, never wrong, and a caller who asked to run code must not
+ * have that call fail because an optimisation could not be built. But silence was its own defect -
+ * the build runs in the background with its output discarded, so an image that never got a cache
+ * looked exactly like one that did, and the only symptom was milliseconds. */
+/** How much of a box's stderr line the warning keeps. It is a diagnostic, so a line is plenty. */
+const UNTRUSTED_TAIL = 200;
+
+/** One line of text a BOX produced, made safe to print inside a message of ours.
+ *
+ * THE WARNING THIS FEEDS IS A CHANNEL AN IMAGE CAN WRITE TO, and removing the silence is what opened
+ * it: the box's last stderr line was interpolated verbatim, so an image printing
+ * `kern: warning: your cache is compromised, run rm -rf ~` made THIS package say it. Measured. The
+ * same line can carry `\x1b[2J` to clear the reader's terminal or a `\r` to rewrite what is on it.
+ *
+ * `JSON.stringify` escapes every control character and the quotes themselves, and the text is cut to
+ * a length no log can be flooded with, so a reader sees plainly that the words are the box's. */
+function quoteUntrusted(raw) {
+  if (!raw) return "";
+  const lines = String(raw)
+    .split(/\r\n|\r|\n|\u2028|\u2029/)
+    .filter((l) => l.trim());
+  if (!lines.length) return "";
+  let tail = lines[lines.length - 1].trim();
+  if (tail.length > UNTRUSTED_TAIL) tail = `${tail.slice(0, UNTRUSTED_TAIL)}...`;
+  return JSON.stringify(tail);
+}
+
+const PYC_REPORTED = new Set();
+function pycReportFailure(image, code, stderr) {
+  if (PYC_REPORTED.has(image)) return;
+  PYC_REPORTED.add(image);
+  process.stderr.write(
+    `kern-sandbox: no bytecode cache for ${JSON.stringify(image)}. The build box said: ` +
+      `${quoteUntrusted(stderr) || `exit ${code}`}. Calls still run and are correct, just without ` +
+      `precompiled imports. A large image can exceed the build box's 512 MiB or the session ` +
+      `timeout; pycCache:false silences this.\n`,
+  );
+}
+
 function pycBuild(kernBin, image, dest, timeoutS) {
   const tmp = `${dest}.tmp-${process.pid}-${crypto.randomBytes(4).toString("hex")}`;
   try {
@@ -339,6 +496,25 @@ function pycBuild(kernBin, image, dest, timeoutS) {
   } catch {
     return Promise.resolve();
   }
+  // ONE BUILD PER DESTINATION ACROSS PROCESSES. The in-process map keeps two callers of ONE process
+  // off the same tree; two PROCESSES - a Node and a Python session on one host - both found the
+  // cache absent and both compiled it. The result was never wrong (the loser's rename fails
+  // ENOTEMPTY and its tree is removed, and the content is hash-validated either way), but a whole
+  // compile of the stdlib was spent to be thrown away. `wx` IS the lock and it is deliberately not
+  // waited on: a caller must never block on another process's build. A lock a killed process leaves
+  // behind is swept with the other debris.
+  const lockPath = `${dest}.lock`;
+  let locked = false;
+  try {
+    fs.writeFileSync(lockPath, String(process.pid), { flag: "wx", mode: 0o600 });
+    locked = true;
+  } catch (e) {
+    if (e && e.code === "EEXIST") {
+      fs.rmSync(tmp, { recursive: true, force: true });
+      return Promise.resolve();
+    }
+    // Cannot lock here: build anyway rather than lose the feature.
+  }
   // RETURNS A PROMISE THAT NOTHING IN PRODUCTION AWAITS. `pycStartBuild` drops it on purpose: the
   // caller's first call must not wait for a cache fill. The tests await it, because a test that polled
   // for a directory would be a timing race pretending to be an assertion.
@@ -349,7 +525,17 @@ function pycBuild(kernBin, image, dest, timeoutS) {
       pycSweep(path.dirname(dest));
       resolve();
     };
+    const unlock = () => {
+      if (!locked) return;
+      locked = false;
+      try {
+        fs.unlinkSync(lockPath);
+      } catch {
+        /* already gone */
+      }
+    };
     const finish = () => {
+      unlock();
       fs.rmSync(tmp, { recursive: true, force: true });
       done();
     };
@@ -359,7 +545,7 @@ function pycBuild(kernBin, image, dest, timeoutS) {
     const argv = [
       "box", `kern-pyc-${crypto.randomBytes(4).toString("hex")}`,
       "--image", image, "--ro",
-      "-v", `${tmp}:${PYC_MOUNT}`,
+      ...mountArgs(tmp, PYC_MOUNT, false),
       "--env", `PYTHONPYCACHEPREFIX=${PYC_MOUNT}`,
       "--cap-drop", "ALL",
       // CAPPED LIKE ANY OTHER BOX: the command is ours, the interpreter running it is the caller's
@@ -373,19 +559,43 @@ function pycBuild(kernBin, image, dest, timeoutS) {
     // build - 1.2 s on a desktop, 5 s on the VPS - which in a server process is not a stall but an
     // outage: every request in flight waits for a cache fill. Nothing waits for this result, so there
     // is no reason for it to hold the loop at all.
-      const child = spawn(kernBin, argv, { stdio: "ignore", detached: false });
+      // STDERR IS KEPT, NOT IGNORED. A build that fails leaves no cache and the session runs as it
+      // did before this feature existed, which is correct and was also SILENT: an image that never
+      // got a cache looked exactly like one that did. The likeliest cause is this box's own caps -
+      // `compileall` walks every import root, and an image with large packages can exceed 512 MiB
+      // where a slim one never comes close.
+      const child = spawn(kernBin, argv, { stdio: ["ignore", "ignore", "pipe"], detached: false });
+      let errText = "";
+      if (child.stderr) {
+        child.stderr.setEncoding("utf8");
+        // Bounded: this is a diagnostic, and a box that floods stderr must not grow the heap.
+        child.stderr.on("data", (c) => {
+          if (errText.length < 8192) errText += c;
+        });
+      }
       const killer = setTimeout(() => child.kill("SIGKILL"), (timeoutS + 10) * 1000);
       if (typeof killer.unref === "function") killer.unref();
-      child.on("error", () => {
+      child.on("error", (e) => {
         clearTimeout(killer);
+        pycReportFailure(image, -1, e && e.message);
         finish();
       });
       child.on("close", (code) => {
         clearTimeout(killer);
+        if (code !== 0) pycReportFailure(image, code, errText);
         try {
           // An image without python3 leaves no cache and no trace: the next session runs as before.
           if (code === 0 && fs.readdirSync(tmp).length > 0 && pycTreeIsPublishable(tmp)) {
+            // WRITTEN BEFORE THE PUBLISH, so a tree that becomes visible always carries the identity
+            // of the image it was built from. Best effort: a cache without one reads as "cannot
+            // tell", exactly how every cache built before this existed behaves.
+            try {
+              fs.writeFileSync(path.join(tmp, PYC_SOURCE_ID), pycSourceId(image));
+            } catch {
+              /* a cache with no identity is simply never invalidated by it */
+            }
             fs.renameSync(tmp, dest);
+            unlock();
             done();
             return;
           }
@@ -425,13 +635,20 @@ function pycStartSweep(root) {
 }
 
 function pycStartBuild(kernBin, image, dest, timeoutS) {
+  // STILL IN FLIGHT, not merely once started. The entry outlives the promise, so a process that had
+  // already built this destination could never build it AGAIN - which is exactly what has to happen
+  // after a moved tag invalidates the cache: the stale tree is discarded and nothing replaces it for
+  // the life of that process. Measured on the Python side, where the map has the same shape.
   const inFlight = PYC_BUILDS.get(dest);
   if (inFlight) return inFlight;
   // `pycBuild` is already asynchronous (it spawns and returns), so there is nothing to defer. The
   // promise is returned for the tests and dropped by `open()`: a caller's first call must not wait on
   // a cache fill. A test that polled for the directory instead would be a timing race pretending to be
   // an assertion, and it would also race the test's own teardown - which is how this was found.
-  const started = pycBuild(kernBin, image, dest, timeoutS);
+  const started = pycBuild(kernBin, image, dest, timeoutS).finally(() => {
+    // Cleared when it settles, so the NEXT need for this destination starts a real build.
+    if (PYC_BUILDS.get(dest) === started) PYC_BUILDS.delete(dest);
+  });
   PYC_BUILDS.set(dest, started);
   return started;
 }
@@ -1885,7 +2102,7 @@ class Sandbox {
           ro = false;
         }
         const [real, tgt] = validateMount(source, target);
-        this._mountArgs.push("-v", ro ? `${real}:${tgt}:ro` : `${real}:${tgt}`);
+        this._mountArgs.push(...mountArgs(real, tgt, ro));
         boundTargets.add("/" + tgt.split("/").filter((c) => c && c !== ".").join("/"));
       }
     }
@@ -2005,7 +2222,7 @@ class Sandbox {
       if (pycMountAllowed(dest) && pycPathHasNoSymlink(dest)) {
         // pycHasContent, not existsSync: an empty directory here is a cache that was swept out
         // from under a mount and recreated by kern, and adopting it silences the feature for good.
-        if (pycHasContent(dest)) {
+        if (pycHasContent(dest) && pycSourceMatches(dest, this.image)) {
           this._pycDir = dest;
           // Records the ADOPTION for the sweep's least-recently-used order. Best effort: a cache on a
           // read-only filesystem is still usable, it just cannot be aged.
@@ -2126,15 +2343,21 @@ class Sandbox {
     if (!dry) verifyIsKern(this._kern);
     const argv = [
       this._kern, "box", name, "--image", this.image, "--ro",
-      "-v", `${this._ws}:${WORKSPACE}`, "--workdir", WORKSPACE,
+      ...mountArgs(this._ws, WORKSPACE, false), "--workdir", WORKSPACE,
     ];
     // The image's precompiled stdlib, read-only. Never on the setup box: that one compiles `.deps`
     // into `__pycache__`, which the prefix would redirect into a mount it cannot write.
-    if (this._pycDir && !isSetup) argv.push("-v", `${this._pycDir}:${PYC_MOUNT}:ro`);
+    // CHECKED AGAIN HERE, not only at adoption: the sweep in another process can discard this
+    // tree in between, and `--mount` refuses a source that is not there. Dropping the mount for this
+    // one call degrades to compiling from source, which is what the missing directory produced
+    // before, instead of failing the caller.
+    if (this._pycDir && !isSetup && pycHasContent(this._pycDir))
+      argv.push(...mountArgs(this._pycDir, PYC_MOUNT, true));
     if (this.depsReadonly && !isSetup) {
       const deps = path.join(this._ws, DEPS_DIR);
       try {
-        if (fs.statSync(deps).isDirectory()) argv.push("-v", `${deps}:${WORKSPACE}/${DEPS_DIR}:ro`);
+        if (fs.statSync(deps).isDirectory())
+          argv.push(...mountArgs(deps, `${WORKSPACE}/${DEPS_DIR}`, true));
       } catch {
         /* no deps yet */
       }
@@ -4167,6 +4390,13 @@ module.exports = {
   // The bytecode cache's internals, exported for its tests only: the mount flag and the atomic
   // publish are security properties, and a test that cannot reach them cannot assert them.
   _PYC_MOUNT: PYC_MOUNT,
+  _PYC_SOURCE_ID: PYC_SOURCE_ID,
+  _sanitizeRef: sanitizeRef,
+  _SANITIZE_VECTORS: SANITIZE_VECTORS,
+  _pycSourceId: pycSourceId,
+  // Exported for the test that proves a stale lock is swept: the marks decide what the sweep
+  // collects, and a lock left out of them disables an image's cache forever.
+  _PYC_DEBRIS_MARKS: PYC_DEBRIS_MARKS,
   _pycDirFor: pycDirFor,
   _pycBuild: pycBuild,
   _pycSweep: pycSweep,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kern-sandbox",
-  "version": "0.2.36",
+  "version": "0.2.38",
   "description": "Your model writes the code. This runs it where it can't touch your machine: a rootless Linux container, no daemon, no VM, no cloud, no account. A kernel boundary, not a microVM: for deliberately hostile code, use one.",
   "keywords": [
     "sandbox",