@impz/testing 6.0.0 → 6.0.1

package/README.md

@@ -79,22 +79,10 @@ await forward.listening;
 const stream = await guest.connect(4000);
 ```
 
-The guest copies these behaviours of the real guest agent's reverse forwards:
-
-- Bytes relay both ways between the process and the forward's local end.
-- The process's end of its side reaches the local end as an EOF, and the local end's EOF ends what
-  the process reads.
-- A process that resets its connection drops the relay with no EOF.
-- A forward that asks for port 0 gets a free port, and a second forward on a port in use fails with
-  `LISTEN_FAILED`.
-- When the imp sleeps or goes away, the guest drops its connections, so a forward ends as `lost`. A
-  forward opened after the wake listens on the guest again.
-
-It does not copy:
-
-- unix socket forwards and ssh-agent forwarding, which it answers with `UNKNOWN_OP`
-- the refusal of a port below 1024 to an image whose user is not root
-- the limit of 16 clients waiting for impd, and the 10 s after which an unpaired client closes
+The guest relays bytes and EOFs as the real guest agent's reverse forwards do, and drops its
+connections when the imp sleeps.
+[Testing against impd](https://github.com/zgeoff/impz/blob/main/docs/guides/testing-impd.md) lists
+what it copies and what it does not.
 
 ## What this impd does not do
 

package/dist/index.js

@@ -21949,7 +21949,7 @@ import { join as join46 } from "path";
 // packages/daemon/package.json
 var package_default = {
   name: "@impz/daemon",
-  version: "6.0.0",
+  version: "6.0.1",
   private: true,
   description: "impd: the imp host daemon",
   type: "module",
@@ -26652,6 +26652,12 @@ import { FFIType, dlopen, read } from "bun:ffi";
 var CORE_SYMBOLS = {
   ioctl: { args: [FFIType.i32, FFIType.u64, FFIType.ptr], returns: FFIType.i32 },
   lseek: { args: [FFIType.i32, FFIType.i64, FFIType.i32], returns: FFIType.i64 },
+  mmap: {
+    args: [FFIType.u64, FFIType.u64, FFIType.i32, FFIType.i32, FFIType.i32, FFIType.i64],
+    returns: FFIType.i64
+  },
+  munmap: { args: [FFIType.u64, FFIType.u64], returns: FFIType.i32 },
+  mlock: { args: [FFIType.u64, FFIType.u64], returns: FFIType.i32 },
   __errno_location: { args: [], returns: FFIType.ptr }
 };
 var PIDFD_SYMBOLS = {
@@ -37248,8 +37254,9 @@ function createRamGovernor(deps) {
     const listed = await deps.listAwake();
     const awake = listed.filter((imp) => imp.id !== excludeId);
     const byImp = new Map;
-    let usedMib = 0;
-    let effectiveMib = 0;
+    const lockedMib = deps.readLockedMib?.() ?? 0;
+    let usedMib = lockedMib;
+    let effectiveMib = lockedMib;
     const readings = await Promise.all(awake.map((imp) => readVmMib(imp.pid, imp.apiSocket)));
     for (const [index, imp] of awake.entries()) {
       const measured = readings[index] ?? 0;
@@ -38947,6 +38954,89 @@ async function setupVm(api, config) {
   await api.putBalloon(VM_DEVICES.balloon);
 }
 
+// packages/daemon/src/vm/templates/template-locks.ts
+import { match as match6 } from "ts-pattern";
+
+// packages/daemon/src/vm/templates/can-admit-lock.ts
+var MAX_LOCKED_PERCENT = 10;
+var MIB3 = 1024 * 1024;
+function canAdmitLock(admission) {
+  const maxLockedBytes = Math.floor(admission.ramBudgetMib * MAX_LOCKED_PERCENT / 100) * MIB3;
+  return admission.heldBytes + admission.lockBytes <= maxLockedBytes;
+}
+
+// packages/daemon/src/vm/templates/pick-lock-outcome.ts
+import { P as P3, match as match5 } from "ts-pattern";
+var MIB4 = 1024 * 1024;
+function pickLockOutcome(result, isCurrent) {
+  return match5(result).with({ errno: P3.number }, (failed) => ({ kind: "failed", errno: failed.errno })).with({ declinedBytes: P3.number }, (declined) => ({
+    kind: "declined",
+    lockMib: Math.ceil(declined.declinedBytes / MIB4)
+  })).with({ lock: P3.select() }, (lock) => isCurrent ? { kind: "held", lock, lockMib: Math.ceil(lock.lockedBytes / MIB4) } : { kind: "stale", lock }).exhaustive();
+}
+
+// packages/daemon/src/vm/templates/template-locks.ts
+var MIB5 = 1024 * 1024;
+function createTemplateLocks(deps) {
+  const locks = new Map;
+  const locking = createSingleFlight();
+  const sizedBytes = new Map;
+  const state = { isStopping: false };
+  const readHeldBytes = () => [...locks.values()].reduce((total, held) => total + held.lockedBytes, 0) + [...sizedBytes.values()].reduce((total, bytes) => total + bytes, 0);
+  const runLock = async (target) => {
+    const key = target.key.slice(0, 12);
+    try {
+      const result = await deps.lockMemory(target.memFile, (lockBytes) => {
+        const isAdmitted = canAdmitLock({
+          heldBytes: readHeldBytes(),
+          lockBytes,
+          ramBudgetMib: deps.ramBudgetMib
+        });
+        if (isAdmitted) {
+          sizedBytes.set(target.buildId, lockBytes);
+        }
+        return isAdmitted;
+      });
+      match6(pickLockOutcome(result, deps.probeCurrent(target))).with({ kind: "failed" }, (failed) => {
+        deps.logger.warning("boot template {key} memory not locked (errno {errno}): a restore reads what the host dropped", { key, errno: String(failed.errno) });
+      }).with({ kind: "declined" }, (declined) => {
+        deps.logger.info("boot template {key} memory not locked: {lockMib} MiB would take the locks past a tenth of the RAM budget", { key, lockMib: String(declined.lockMib) });
+      }).with({ kind: "stale" }, (stale) => {
+        stale.lock.release();
+      }).with({ kind: "held" }, (held) => {
+        locks.set(target.key, held.lock);
+        deps.logger.info("boot template {key} memory locked: {lockMib} MiB", {
+          key,
+          lockMib: String(held.lockMib)
+        });
+      }).exhaustive();
+    } catch (error) {
+      deps.logger.warning("boot template {key} memory not locked: {error}", {
+        key,
+        error: getErrorMessage(error)
+      });
+    } finally {
+      sizedBytes.delete(target.buildId);
+    }
+  };
+  return {
+    lock: (target) => state.isStopping || locks.has(target.key) ? Promise.resolve() : locking.run(target.buildId, () => runLock(target)),
+    release: (key) => {
+      locks.get(key)?.release();
+      locks.delete(key);
+    },
+    readLockedMib: () => Math.ceil(readHeldBytes() / MIB5),
+    stop: async () => {
+      state.isStopping = true;
+      await locking.waitForAll();
+      for (const held of locks.values()) {
+        held.release();
+      }
+      locks.clear();
+    }
+  };
+}
+
 // packages/daemon/src/vm/templates/boot-templates.ts
 var TEMPLATE_BOOT_ARGS = [...BASE_BOOT_ARGS, "imp.template=1"].join(" ");
 var TEMPLATE_ADDRESS = {
@@ -38965,7 +39055,7 @@ var BUILD_JAIL_ID = "tpl-build";
 var BUILD_CPU = { limit: null, weight: 100 };
 var PLACEHOLDER_BYTES = 1024 * 1024;
 var PLACEHOLDER_NAME = "placeholder.ext4";
-var MIB3 = 1024 * 1024;
+var MIB6 = 1024 * 1024;
 var MIN_MISSES = 2;
 var MAX_TEMPLATES = 4;
 var MAX_BUILD_FAILURES = 3;
@@ -39039,6 +39129,7 @@ function createBootTemplates(deps) {
   };
   const listTemplateKeys = () => existsSync9(root) ? readdirSync3(root).filter((name) => name !== PLACEHOLDER_NAME && !name.startsWith(".")) : [];
   const removeDir = (name) => {
+    templateLocks.release(name);
     const gone = join18(root, `.gone-${Bun.randomUUIDv7()}`);
     try {
       renameSync3(join18(root, name), gone);
@@ -39048,6 +39139,12 @@ function createBootTemplates(deps) {
     rmSync15(gone, { recursive: true, force: true });
     lastUsed.delete(name);
   };
+  const templateLocks = createTemplateLocks({
+    lockMemory: deps.lockMemory,
+    probeCurrent: (target) => readMeta(target.key)?.buildId === target.buildId,
+    ramBudgetMib: deps.ramBudgetMib,
+    logger: getImpdLogger("boot")
+  });
   const readLastUsed = (key) => {
     const used = lastUsed.get(key);
     if (used !== undefined) {
@@ -39138,7 +39235,7 @@ function createBootTemplates(deps) {
       key: key.slice(0, 12),
       durationMs: String(durationMs)
     });
-    return {
+    const files = {
       key,
       buildId,
       vmstate: join18(root, key, "vmstate"),
@@ -39146,6 +39243,8 @@ function createBootTemplates(deps) {
       systemDrivePath: meta.systemDrivePath,
       placeholderPath: placeholder
     };
+    templateLocks.lock(files);
+    return files;
   };
   const runBuild = async (shape, key) => {
     const work = join18(root, `.build-${Bun.randomUUIDv7()}`);
@@ -39164,7 +39263,7 @@ function createBootTemplates(deps) {
         memoryMib: shape.memoryMib,
         maySleepImps: false
       });
-      return await (deps.diskBudget === undefined ? writeAdmitted() : deps.diskBudget.withRoom(shape.memoryMib * MIB3, writeAdmitted));
+      return await (deps.diskBudget === undefined ? writeAdmitted() : deps.diskBudget.withRoom(shape.memoryMib * MIB6, writeAdmitted));
     } catch (error) {
       if (progress.isBuilding) {
         throw error;
@@ -39279,10 +39378,98 @@ function createBootTemplates(deps) {
       const meta = readMeta(key);
       return meta === null ? [] : [meta.systemDrivePath];
     }),
-    stop: () => building.waitForAll()
+    lockKept: async () => {
+      for (const key of listTemplateKeys()) {
+        const ready = readReady(key);
+        if (ready !== null) {
+          await templateLocks.lock(ready);
+        }
+      }
+    },
+    readLockedMib: templateLocks.readLockedMib,
+    stop: async () => {
+      await Promise.all([templateLocks.stop(), building.waitForAll()]);
+    }
   };
 }
 
+// packages/daemon/src/vm/templates/create-memory-lock.ts
+import { closeSync as closeSync8, fstatSync as fstatSync3, openSync as openSync6 } from "fs";
+var PROT_READ = 1;
+var MAP_SHARED = 1;
+async function createMemoryLock(path, onSized) {
+  const libc = readLibc();
+  if (libc === null) {
+    return { errno: -1 };
+  }
+  const fd = openSync6(path, "r");
+  try {
+    const size = fstatSync3(fd).size;
+    const runs = readDataRuns(libc, fd, size);
+    if ("errno" in runs) {
+      return runs;
+    }
+    const lockedBytes = runs.reduce((total, run) => total + run.length, 0);
+    if (!onSized(lockedBytes)) {
+      return { declinedBytes: lockedBytes };
+    }
+    const address = Number(libc.mmap(0, size, PROT_READ, MAP_SHARED, fd, 0));
+    if (address === -1) {
+      return { errno: readErrno() };
+    }
+    const mapping = { isMapped: true };
+    const release = () => {
+      if (mapping.isMapped) {
+        mapping.isMapped = false;
+        libc.munmap(address, size);
+      }
+    };
+    for (const chunk of runs.flatMap((run) => splitRun(run))) {
+      if (libc.mlock(address + chunk.offset, chunk.length) !== 0) {
+        const errno = readErrno();
+        release();
+        return { errno };
+      }
+      await Bun.sleep(0);
+    }
+    return { lock: { lockedBytes, release } };
+  } finally {
+    closeSync8(fd);
+  }
+}
+var SEEK_DATA2 = 3;
+var SEEK_HOLE2 = 4;
+var ENXIO2 = 6;
+function readDataRuns(libc, fd, size) {
+  const runs = [];
+  let offset = 0;
+  while (offset < size) {
+    const data = Number(libc.lseek(fd, offset, SEEK_DATA2));
+    if (data < 0) {
+      const errno = readErrno();
+      if (errno === ENXIO2) {
+        break;
+      }
+      return { errno };
+    }
+    const hole = Number(libc.lseek(fd, data, SEEK_HOLE2));
+    if (hole < 0) {
+      return { errno: readErrno() };
+    }
+    const end = Math.min(hole, size);
+    runs.push({ offset: data, length: end - data });
+    offset = end;
+  }
+  return runs;
+}
+var CHUNK_BYTES = 4 * 1024 * 1024;
+function splitRun(run) {
+  return Array.from({ length: Math.ceil(run.length / CHUNK_BYTES) }, (_unused, index) => {
+    const offset = run.offset + index * CHUNK_BYTES;
+    return { offset, length: Math.min(CHUNK_BYTES, run.offset + run.length - offset) };
+  });
+}
+
 // packages/daemon/src/vm/vmm/vm-stats.ts
 import { readFileSync as readFileSync19, readdirSync as readdirSync4 } from "fs";
 import { readFile as readFile3 } from "fs/promises";
@@ -40855,7 +41042,9 @@ function createDefaultTemplates(deps, cgroups) {
     kernelPath: deps.config.kernelPath,
     minGuestUptimeMs: deps.config.sleepMinGuestUptimeMs,
     bootReservePercent: deps.config.bootReservePercent,
+    ramBudgetMib: deps.config.ramBudgetMib,
     buildVm: deps.vms.createTemplateVm,
+    lockMemory: createMemoryLock,
     taps: deps.taps,
     jail: deps.config.jailerBin === null ? null : { uid: TEMPLATE_BUILD_UID, gid: TEMPLATE_BUILD_UID },
     cgroups,
@@ -42464,12 +42653,12 @@ function readNumber(parsed, key) {
 // packages/daemon/src/vm/vmm/jail.ts
 import {
   chmodSync as chmodSync6,
-  closeSync as closeSync8,
+  closeSync as closeSync9,
   existsSync as existsSync12,
   lchownSync as lchownSync2,
   lstatSync as lstatSync3,
   mkdirSync as mkdirSync14,
-  openSync as openSync6,
+  openSync as openSync7,
   readdirSync as readdirSync7,
   rmSync as rmSync19,
   statSync as statSync9,
@@ -42558,7 +42747,7 @@ function createJails(deps) {
   };
   const setupReadOnlyBind = async (source, target) => {
     mkdirSync14(dirname9(target), { recursive: true });
-    closeSync8(openSync6(target, "a"));
+    closeSync9(openSync7(target, "a"));
     await runMount(["mount", "--bind", source, target]);
     await runMount(["mount", "-o", "remount,bind,ro,nosuid,nodev", target]);
   };
@@ -42705,7 +42894,7 @@ function setupOwnership(paths, user, isDiskLate) {
 }
 function setupScratchFile(source, target, user) {
   mkdirSync14(dirname9(target), { recursive: true });
-  closeSync8(openSync6(target, "wx", 384));
+  closeSync9(openSync7(target, "wx", 384));
   truncateSync3(target, statSync9(source).size);
   lchownSync2(target, user.uid, user.gid);
 }
@@ -44320,6 +44509,7 @@ function createImpdImps(config, deps, parts) {
     readRamMib: readRamMibAsync,
     readResidentMib,
     readHeadroomMib,
+    readLockedMib: () => imps.bootTemplates?.readLockedMib() ?? 0,
     isBusy,
     trySleepImp: imps.trySleepImp,
     readDiskFullError: imps.readDiskFullError,
@@ -45796,7 +45986,7 @@ function readSettled2(row, imp) {
 
 // packages/daemon/src/imp-ops/moves/move-sender.ts
 import invariant4 from "tiny-invariant";
-import { match as match5 } from "ts-pattern";
+import { match as match7 } from "ts-pattern";
 
 // packages/daemon/src/platform/db/move-send-steps.ts
 var MOVE_SEND_STEPS = [
@@ -46358,7 +46548,7 @@ function createMoveSender(deps) {
   };
   const recoverSend = async (imp, row) => {
     const log = getImpdLogger("move").with({ imp: imp.name });
-    await match5(pickSendRecovery(imp.moveState, row?.step)).with("unmark", () => removeMark(imp)).with("wait", () => {
+    await match7(pickSendRecovery(imp.moveState, row?.step)).with("unmark", () => removeMark(imp)).with("wait", () => {
       log.info("a prepared send waits for its send after a restart");
     }).with("abort", async () => {
       const isUndone = await resetSend(imp, row?.peer_url ?? null, row?.ticket ?? null);
@@ -46480,7 +46670,7 @@ function buildPublicError(name) {
 }
 function pickSendRecovery(moveState, step) {
   if (moveState === "sending") {
-    return match5(step).with(undefined, () => "unmark").with("prepared", () => "wait").with("streaming", () => "abort").with("sent", () => "commit").exhaustive();
+    return match7(step).with(undefined, () => "unmark").with("prepared", () => "wait").with("streaming", () => "abort").with("sent", () => "commit").exhaustive();
   }
   return moveState === "moved" && step === "sent" ? "commit" : "none";
 }
@@ -46532,7 +46722,7 @@ function createMoveService(deps) {
 // packages/daemon/src/surface/moves/create-move-peer-handler.ts
 import { OpenAPIHandler } from "@orpc/openapi/fetch";
 import { ORPCError as ORPCError7, implement as implement3, onError as onError3 } from "@orpc/server";
-import { match as match6 } from "ts-pattern";
+import { match as match8 } from "ts-pattern";
 function createMovePeerHandler(deps) {
   const withTicket = async (step, context, call) => {
     const startedAt = deps.now();
@@ -46583,7 +46773,7 @@ function createMovePeerHandler(deps) {
   const respondReceive = async (context) => {
     try {
       const received = await withTicket("receive", context, (checked) => deps.calls.receive(checked, context.request));
-      return match6(received).with({ kind: "part" }, (sent) => Response.json({ part: sent.part }, { status: 202 })).with({ kind: "receipt" }, (sent) => Response.json(sent.receipt)).exhaustive();
+      return match8(received).with({ kind: "part" }, (sent) => Response.json({ part: sent.part }, { status: 202 })).with({ kind: "receipt" }, (sent) => Response.json(sent.receipt)).exhaustive();
     } catch (error) {
       const failure = toPeerError(error);
       return Response.json(encodePeerError(failure), { status: failure.status });
@@ -47296,9 +47486,9 @@ function printSweep(logger, result, orphans) {
   }
   logger.info("kept {length} orphans the database does not name; `impz gc --orphans --dry-run` lists what `impz gc --orphans` would retire", { length: String(result.kept.length) });
 }
-var MIB4 = 1024 ** 2;
+var MIB7 = 1024 ** 2;
 function formatOrphan(orphan) {
-  const size = `${(orphan.bytes / MIB4).toFixed(1)} MiB`;
+  const size = `${(orphan.bytes / MIB7).toFixed(1)} MiB`;
   const created = orphan.createdAt?.toISOString() ?? "unknown";
   const snapshots = orphan.snapshots.length === 0 ? "none" : orphan.snapshots.join(", ");
   return `orphan ${orphan.kind} ${orphan.id} (${orphan.location}): ${size}, created ${created}, snapshots: ${snapshots}`;
@@ -48781,11 +48971,11 @@ class ImageLimitError extends ImpdError {
 
 // packages/daemon/src/imps/images/rootfs/rootfs-plan.ts
 var GIB3 = 1024 ** 3;
-var MIB5 = 1024 ** 2;
+var MIB8 = 1024 ** 2;
 var ROOTFS_MIN_BYTES = 4 * GIB3;
 var ROOTFS_SPARE_BYTES = 2 * GIB3;
 var BYTES_PER_INODE = 16384;
-var ROOTFS_OVERHEAD_MIN_BYTES = 128 * MIB5;
+var ROOTFS_OVERHEAD_MIN_BYTES = 128 * MIB8;
 var ROOTFS_OVERHEAD_SHARE = 64;
 function planRootfs(tree) {
   const wanted = Math.ceil((tree.bytes * 1.2 + ROOTFS_SPARE_BYTES) / GIB3) * GIB3;
@@ -49441,7 +49631,7 @@ async function readOciConfig(ref, signal, run) {
 }
 
 // packages/daemon/src/imps/images/rootfs/write-image-config.ts
-import { closeSync as closeSync9, constants as constants5, lstatSync as lstatSync5, mkdirSync as mkdirSync21, openSync as openSync7, rmSync as rmSync25, writeSync as writeSync3 } from "fs";
+import { closeSync as closeSync10, constants as constants5, lstatSync as lstatSync5, mkdirSync as mkdirSync21, openSync as openSync8, rmSync as rmSync25, writeSync as writeSync3 } from "fs";
 import { join as join29 } from "path";
 function writeImageConfig(root, contents) {
   const etc = join29(root, "etc");
@@ -49450,11 +49640,11 @@ function writeImageConfig(root, contents) {
   createDirectory(directory, "/etc/imp");
   const file = join29(directory, "image.json");
   rmSync25(file, { recursive: true, force: true });
-  const fd = openSync7(file, constants5.O_WRONLY | constants5.O_CREAT | constants5.O_EXCL | constants5.O_NOFOLLOW, 420);
+  const fd = openSync8(file, constants5.O_WRONLY | constants5.O_CREAT | constants5.O_EXCL | constants5.O_NOFOLLOW, 420);
   try {
     writeSync3(fd, contents);
   } finally {
-    closeSync9(fd);
+    closeSync10(fd);
   }
 }
 function createDirectory(path, name) {
@@ -50843,14 +51033,14 @@ function expandScopes(asked) {
 }
 
 // packages/daemon/src/platform/http/render-html.ts
-import { P as P3, match as match7 } from "ts-pattern";
+import { P as P4, match as match9 } from "ts-pattern";
 var MARKUP = Symbol("markup");
 function renderHTML(strings, ...parts) {
   const html = parts.reduce((text, part, index) => `${text}${toHTML(part)}${strings[index + 1] ?? ""}`, strings[0] ?? "");
   return { [MARKUP]: true, html };
 }
 function toHTML(part) {
-  return match7(part).with(P3.string, (text) => escapeHTML(text)).with(P3.array(), (list) => list.map((each) => each.html).join("")).with({ html: P3.string }, (markup) => markup.html).exhaustive();
+  return match9(part).with(P4.string, (text) => escapeHTML(text)).with(P4.array(), (list) => list.map((each) => each.html).join("")).with({ html: P4.string }, (markup) => markup.html).exhaustive();
 }
 function escapeHTML(text) {
   return text.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
@@ -57034,6 +57224,7 @@ async function startImpdParts(config, boundaries, plan) {
         key: key.slice(0, 12)
       });
     }
+    imps.bootTemplates?.lockKept();
   });
   await runStep("system-drives", async () => {
     const removed = await removeUnusedDrives(db, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@impz/testing",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "A real impd on stand-ins, for Bun tests of code that drives impd through @impz/client",
   "license": "MIT",
   "repository": {
@@ -22,7 +22,7 @@
     "@bufbuild/protobuf": "^2.16.0",
     "@logtape/logtape": "^2.4.2",
     "@modelcontextprotocol/server": "^2.3.1",
-    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/api": "^1.10.0",
     "@orpc/client": "^1.15.7",
     "@orpc/contract": "^1.15.7",
     "@orpc/openapi": "^1.15.7",
@@ -42,14 +42,14 @@
     "ts-pattern": "^5.9.2",
     "xstate": "^5.33.3",
     "zod": "^4.6.7",
-    "@impz/client": "6.0.0"
+    "@impz/client": "6.0.1"
   },
   "engines": {
     "bun": ">=1.4.3"
   },
   "peerDependencies": {
-    "@opentelemetry/exporter-metrics-otlp-proto": "^0.223.0",
-    "@opentelemetry/exporter-trace-otlp-proto": "^0.223.0",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.300.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.300.0",
     "@opentelemetry/resources": "^2.12.0",
     "@opentelemetry/sdk-metrics": "^2.12.0",
     "@opentelemetry/sdk-trace-base": "^2.12.0",