@volter/world-machine 2.0.0

package/src/config.ts

@@ -0,0 +1,165 @@
+// The machine's declaration: a JSON file the World service names with `--config`. Paths to the
+// guest image, Firecracker and the store are paths ON THE MACHINE HOST (`~/` is that host's home);
+// `provision` is a script beside the config on the World's host, shipped into the guest when the warm
+// base is built.
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, resolve } from 'node:path';
+
+export type MachineHost = { kind: 'linux' } | { kind: 'lima'; instance: string; limaHome: string };
+
+/** The firecracker backend: a microVM on KVM, snapshotted by Firecracker (memory diffs, a reflinked disk). The
+ *  container backend (ct.sh): the host's own processes in a container, dumped by CRIU with their tree in one reflinked
+ *  image; for a host where a VM would be nested (a Mac: first-touch memory faulted through two hypervisors ran a real
+ *  install 20–80x slower). */
+export type MachineBackend = 'firecracker' | 'container';
+
+export type MachineConfig = {
+  host: MachineHost;
+  backend: MachineBackend;
+  /** container: the OCI image the machine's tree is made from, and the tree's size (a sparse ext4 image). */
+  image: string;
+  imageSize: string;
+  /** container: host directories bound read-only into the container (binaries the machine runs that its image lacks). */
+  binds: Array<{ source: string; destination: string }>;
+  /** container: which of the machine's listening ports are published on the host's loopback at the same port, as the
+   *  machine's own localhost (publish.py): a twin delivering to `http://localhost:<port>` reaches the app, and a Lima host
+   *  forwards them on to the Mac. Only allowed ports: the machine runs code the user did not write, and a port it held
+   *  first would be taken from a host service that binds it later. None unless set. */
+  publishPorts?: { ranges: Array<[number, number]>; except: number[] };
+  /** container: a path in the machine's tree served to `client` by the host's NFS server (v3), as `uid`:`gid`. */
+  export?: { path: string; client: string; uid: number; gid: number };
+  /** The reflink-capable directory (XFS or btrfs) holding warm bases, snapshots and the slot. */
+  store: string;
+  /** A mark is refused unless the store, the file backing it and each of `paths` (machine-host paths on the
+   *  filesystems beneath it, such as a Mac's home mounted in its Lima VM) have its estimate plus `reserveBytes` free. */
+  room: { paths: string[]; reserveBytes: number };
+  /** How many of this World's snapshots the store keeps beyond every branch's head, the newest by when the host took
+   *  them; older ones are let go by gc (and named in the store's expired.list, so a restore of one says it was let go).
+   *  Another World on the store keeps its own. Every snapshot kept while a ledger names it when unset. */
+  retain?: number;
+  /** With `retain`: past the newest `retain`, the newest snapshot of each hour for a day and of each day for a month
+   *  stay too (thin.ts), and a snapshot taken with `keep` is never let go. */
+  thin?: boolean;
+  /** The one machine this service runs in place; its tap, addresses and disk path are fixed. */
+  slot: string;
+  firecracker: string;
+  snapshotEditor: string;
+  kernel: string;
+  rootfs: string;
+  /** The private key whose public half the rootfs authorizes for root. */
+  sshKey: string;
+  /** Absolute path of the provisioning script on the World's host, or undefined for none. */
+  provision?: string;
+  vcpus: number;
+  memMib: number;
+  tap: string;
+  hostIp: string;
+  guestIp: string;
+  /** A fixed guest-visible port per vendor, overriding the derived one. */
+  egressPorts: Record<string, number>;
+  /** Where the machine host reaches a twin listening on the World host's loopback. */
+  twinHost: string;
+};
+
+const FIELDS = ['host', 'backend', 'image', 'imageSize', 'binds', 'export', 'publishPorts', 'store', 'room', 'retain', 'thin', 'slot', 'firecracker', 'snapshotEditor', 'kernel', 'rootfs', 'sshKey', 'provision', 'vcpus', 'memMib', 'tap', 'hostIp', 'guestIp', 'egressPorts'];
+
+export function loadConfig(path: string): MachineConfig {
+  const file = resolve(path);
+  let raw: Record<string, unknown>;
+  try { raw = JSON.parse(readFileSync(file, 'utf8')) as Record<string, unknown>; } catch (error) { throw new Error(`world-machine: cannot read the machine config ${file}: ${(error as Error).message}`); }
+  for (const key of Object.keys(raw)) if (!FIELDS.includes(key) && !key.startsWith('//')) throw new Error(`world-machine: unknown field "${key}" in ${file} (known: ${FIELDS.join(', ')})`);
+  const str = (key: string, fallback?: string): string => {
+    const v = raw[key] ?? fallback;
+    if (typeof v !== 'string' || v === '') throw new Error(`world-machine: "${key}" must be a non-empty string in ${file}`);
+    return v;
+  };
+  const num = (key: string, fallback: number): number => {
+    const v = raw[key] ?? fallback;
+    if (typeof v !== 'number' || !Number.isInteger(v) || v <= 0) throw new Error(`world-machine: "${key}" must be a positive integer in ${file}`);
+    return v;
+  };
+  const hostRaw = (raw.host ?? (process.platform === 'darwin' ? undefined : { kind: 'linux' })) as Record<string, unknown> | undefined;
+  let host: MachineHost;
+  if (hostRaw?.kind === 'linux') host = { kind: 'linux' };
+  else if (hostRaw?.kind === 'lima' && typeof hostRaw.instance === 'string') {
+    const home = typeof hostRaw.limaHome === 'string' ? hostRaw.limaHome : process.env.LIMA_HOME ?? '~/.lima';
+    host = { kind: 'lima', instance: hostRaw.instance, limaHome: home.startsWith('~/') ? resolve(homedir(), home.slice(2)) : resolve(dirname(file), home) };
+  } else throw new Error(`world-machine: "host" must be {"kind":"linux"} or {"kind":"lima","instance":"<name>","limaHome":"<dir>"} in ${file} (on a Mac Firecracker runs inside a Lima >= 2 VM with nestedVirtualization: true)`);
+  const backend = str('backend', 'firecracker');
+  if (backend !== 'firecracker' && backend !== 'container') throw new Error(`world-machine: "backend" must be "firecracker" or "container" in ${file}`);
+  const vm = backend === 'firecracker';
+  const store = str('store', '~/.cache/volter-world-machine');
+  const egressPorts = (raw.egressPorts ?? {}) as Record<string, number>;
+  if (typeof egressPorts !== 'object' || Object.values(egressPorts).some((p) => !Number.isInteger(p))) throw new Error(`world-machine: "egressPorts" must map a vendor to a port in ${file}`);
+  const slot = str('slot', 'default');
+  if (!/^[a-z0-9-]+$/.test(slot)) throw new Error(`world-machine: "slot" must be lowercase letters, digits and dashes in ${file}`);
+  return {
+    host,
+    backend,
+    image: vm ? '' : str('image'),
+    imageSize: vm ? '' : str('imageSize', '40G'),
+    binds: vm ? [] : binds(raw.binds, file),
+    ...(raw.publishPorts === undefined ? {} : vm ? (() => { throw new Error(`world-machine: "publishPorts" is for the container backend only, in ${file}`); })() : { publishPorts: publishOf(raw.publishPorts, file) }),
+    ...(vm || raw.export === undefined ? {} : { export: exportOf(raw.export, file) }),
+    store,
+    room: roomOf(raw.room, file),
+    ...(raw.retain === undefined ? {} : { retain: num('retain', 1) }),
+    ...(raw.thin === undefined ? {} : typeof raw.thin !== 'boolean' ? (() => { throw new Error(`world-machine: "thin" must be true or false in ${file}`); })()
+      : raw.thin && raw.retain === undefined ? (() => { throw new Error(`world-machine: "thin" thins what "retain" lets go, and needs it, in ${file}`); })() : { thin: raw.thin }),
+    slot,
+    firecracker: str('firecracker', `${store}/bin/firecracker`),
+    snapshotEditor: str('snapshotEditor', `${store}/bin/snapshot-editor`),
+    kernel: vm ? str('kernel') : '',
+    rootfs: vm ? str('rootfs') : '',
+    sshKey: vm ? str('sshKey') : '',
+    ...(raw.provision === undefined ? {} : { provision: resolve(dirname(file), str('provision')) }),
+    vcpus: num('vcpus', 2),
+    memMib: num('memMib', 1024),
+    tap: str('tap', 'wm0'),
+    hostIp: str('hostIp', '172.16.0.1'),
+    guestIp: str('guestIp', '172.16.0.2'),
+    egressPorts,
+    twinHost: host.kind === 'lima' ? 'host.lima.internal' : '127.0.0.1',
+  };
+}
+
+function binds(raw: unknown, file: string): Array<{ source: string; destination: string }> {
+  if (raw === undefined) return [];
+  if (!Array.isArray(raw) || raw.some((b) => typeof b?.source !== 'string' || typeof b?.destination !== 'string' || !b.destination.startsWith('/'))) {
+    throw new Error(`world-machine: "binds" must be [{"source":"<host dir>","destination":"/<path in the machine>"}] in ${file}`);
+  }
+  return raw as Array<{ source: string; destination: string }>;
+}
+
+function exportOf(raw: unknown, file: string): { path: string; client: string; uid: number; gid: number } {
+  const e = raw as Record<string, unknown>;
+  if (typeof e?.path !== 'string' || !e.path.startsWith('/') || typeof e.client !== 'string' || !Number.isInteger(e.uid) || !Number.isInteger(e.gid)) {
+    throw new Error(`world-machine: "export" must be {"path":"/<path in the machine>","client":"<address>","uid":<n>,"gid":<n>} in ${file}`);
+  }
+  return { path: e.path, client: e.client, uid: e.uid as number, gid: e.gid as number };
+}
+
+function roomOf(raw: unknown, file: string): { paths: string[]; reserveBytes: number } {
+  const r = (raw ?? {}) as Record<string, unknown>;
+  const paths = r.paths ?? [];
+  const reserveBytes = r.reserveBytes ?? 2 * 2 ** 30;
+  if (typeof r !== 'object' || !Array.isArray(paths) || paths.some((p) => typeof p !== 'string' || p === '') || !Number.isInteger(reserveBytes) || (reserveBytes as number) < 0) {
+    throw new Error(`world-machine: "room" must be {"paths":["<machine-host path>", ...],"reserveBytes":<n>} in ${file}`);
+  }
+  return { paths: paths as string[], reserveBytes: reserveBytes as number };
+}
+
+function publishOf(raw: unknown, file: string): { ranges: Array<[number, number]>; except: number[] } {
+  const r = raw as Record<string, unknown>;
+  const port = (n: unknown): boolean => Number.isInteger(n) && (n as number) >= 1024 && (n as number) <= 65535;
+  const ranges = r?.ranges; const except = r?.except ?? [];
+  if (!Array.isArray(ranges) || ranges.some((x) => !Array.isArray(x) || x.length !== 2 || !port(x[0]) || !port(x[1]) || x[0] > x[1]) || !Array.isArray(except) || except.some((n) => !port(n))) {
+    throw new Error(`world-machine: "publishPorts" must be {"ranges":[[from,to],...],"except":[port,...]} with ports 1024-65535 in ${file}`);
+  }
+  // the kernel hands out ephemeral ports from 32768 (Linux's default range): the World's own services take theirs there
+  if ((ranges as Array<[number, number]>).some(([lo, hi]) => hi >= 32768 && lo <= 60999)) {
+    throw new Error(`world-machine: "publishPorts" ranges must stay below 32768 (the ephemeral ports the World's services use) in ${file}`);
+  }
+  return { ranges: ranges as Array<[number, number]>, except: except as number[] };
+}

package/src/container.ts

@@ -0,0 +1,411 @@
+// The container backend (config `backend: "container"`, host functions in ct.sh): the machine is the host's own
+// processes in a container run by crun, marked by CRIU with their tree in one reflinked ext4 image. It is for a host
+// where a microVM would be nested (a Mac): there first-touch memory faults through two hypervisors, and a real
+// project's install ran 20–80x slower than one level up (Volter Harness container/retake/README.md). The doors are the
+// Firecracker backend's: the same guest address on the same bridge name, the same egress forwarders, the same
+// ledger entries. What differs:
+//   the clock      the container reads the host's kernel clock, which is now; a World whose clock is frozen is not
+//                  followed (reported in the restore's facts)
+//   a mark         a full CRIU dump of the processes and a reflink of the tree (no memory diff: arm64 has no
+//                  soft-dirty tracking), recorded as a Full entry; its pages composed against the parent's
+//                  afterwards (compose.py), so a compressing store keeps only the new ones
+//   stale sockets  reset from the host in the container's network namespace (ss -K), as the guest agent resets them
+import { createHash, randomBytes } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
+import { posix } from 'node:path';
+import type { MachineConfig } from './config.ts';
+import { GUEST_PROFILE, type HostSession, hostNowMs, q } from './host.ts';
+import { appendSnapshot, headAt, headSnapshot, type SnapshotRecord, worldInstant } from './ledger.ts';
+import { Machine } from './machine.ts';
+
+/** The host functions a base is made by: the tree's fill (with the init it installs), then the provisioning run, the
+ *  init's start and the bundle every process in it runs under. The rest (marks, restores, the export, publishing) act on
+ *  a base without shaping it. Hashing all of ct.sh rebuilt the base at every change to the host functions (49 s
+ *  measured). */
+const FILL_FUNCTIONS = ['wm_ct_base_fill'];
+// the base's dump too: its CRIU options (wm_ct_mark) and the namespace its sockets live in (wm_ct_net), which a restore
+// must match
+const BASE_FUNCTIONS = ['wm_ct_provision', 'wm_ct_start', 'wm_ct_bundle', 'wm_ct_net', 'wm_ct_mark'];
+
+const sha = (inputs: unknown): string => createHash('sha256').update(JSON.stringify(inputs)).digest('hex').slice(0, 16);
+
+/** A container base's identity, and its filled tree's (`tree-<hash>`, kept in the store: a base rebuilt for anything
+ *  but its image or fill starts from a reflink of it instead of filling again, the 8-25 s of a rebuild). The tree is
+ *  the image (by its id on this host: a tag rebuilt or pulled anew is a new image), its size, the fill and the init it
+ *  installs, and the guest's address (its /etc/hosts); the base adds the provisioning, the host functions that make it,
+ *  the directories bound into it and the slot. Functions as bash prints them, so a comment is no input. Refused when the
+ *  image has no id here (podman failing, or the image removed): a hash without it names no base that exists, and a
+ *  collection working from it would delete the one that does. */
+export async function containerHashes(config: MachineConfig, session: HostSession): Promise<{ base: string; tree: string }> {
+  const json = (names: string[], extra = ''): string => `"$({ declare -f ${names.join(' ')}; ${extra} } | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')"`;
+  const r = await session.call<{ imageId: string; fill: string; functions: string }>(
+    `printf '@{"imageId":"%s","fill":%s,"functions":%s}\n' "$(sudo -n podman image inspect --format '{{.Id}}' ${q(config.image)} 2>/dev/null)" ${json(FILL_FUNCTIONS, `printf '%s' "$WM_CT_INIT";`)} ${json(BASE_FUNCTIONS)}`);
+  if (!/^[0-9a-f]{64}$/.test(r.imageId)) throw new Error(`world-machine: the image ${config.image} has no id on the machine host (podman image inspect gave ${JSON.stringify(r.imageId)}): build or pull it`);
+  const tree = sha({ image: config.image, imageId: r.imageId, imageSize: config.imageSize, fill: r.fill, hostIp: config.hostIp, guestIp: config.guestIp });
+  const provision = config.provision ? readFileSync(config.provision, 'utf8') : '';
+  const base = sha({ backend: 'container', tree, provision, functions: r.functions, binds: config.binds, slot: config.slot, tap: config.tap, hostIp: config.hostIp, guestIp: config.guestIp });
+  return { base, tree };
+}
+export const containerBaseHash = async (config: MachineConfig, session: HostSession): Promise<string> => (await containerHashes(config, session)).base;
+
+export type ContainerBaseStep = { base: string; stage: string; ready: boolean; detail?: string };
+
+/** One step of building the container base, advancing as far as it can without waiting (the Firecracker base's
+ *  contract: run it again until ready). The base is a mark like any other: the tree and the freshly started init with
+ *  its services. */
+export async function stepContainerBase(config: MachineConfig, session: HostSession): Promise<ContainerBaseStep> {
+  await session.call('wm_reflink_check');
+  // an image from a registry is pulled first (in the background): the base is named by the image's id on this host
+  if (!config.image.startsWith('localhost/') && !(await session.call<{ ok: boolean }>(`if sudo -n podman image exists ${q(config.image)}; then echo '@{"ok":true}'; else echo '@{"ok":false}'; fi`)).ok) {
+    const P = `"$WM_STORE/.pull-$WM_SLOT"`;
+    if (!(await session.call<{ ok: boolean }>(`if kill -0 "$(cat ${P}.pid 2>/dev/null)" 2>/dev/null; then echo '@{"ok":true}'; else echo '@{"ok":false}'; fi`)).ok) {
+      if ((await session.call<{ ok: boolean }>(`if [ -f ${P}.log ]; then echo '@{"ok":true}'; else echo '@{"ok":false}'; fi`)).ok) {
+        const log = await session.call<{ log: string }>(`printf '@{"log":%s}\n' "$(tail -c 400 ${P}.log | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')"; rm -f ${P}.log`);
+        throw new Error(`world-machine base: pulling ${config.image} failed:\n${log.log}`);
+      }
+      await session.call(`{ wm_closefds; sudo -n podman pull -q ${q(config.image)} && rm -f ${P}.log; } </dev/null >${P}.log 2>&1 9>&- &\necho $! > ${P}.pid; echo '@{"ok":true}'`);
+    }
+    return { base: '', stage: `pulling ${config.image}`, ready: false, detail: 'run again' };
+  }
+  const hashes = await containerHashes(config, session);
+  const base = `base-${hashes.base}`;
+  const T = `"$WM_STORE/tree-${hashes.tree}"`;
+  const B = `"$WM_STORE/${base}"`;
+  const W = `"$WM_STORE/${base}.building"`;
+  const has = async (test: string): Promise<boolean> => (await session.call<{ ok: boolean }>(`if ${test}; then echo '@{"ok":true}'; else echo '@{"ok":false}'; fi`)).ok;
+  if (await has(`[ -f ${B}/criu/inventory.img ]`)) return { base, stage: 'ready', ready: true };
+
+  // 1. the tree: a reflink of the one filled from the same image before, else made from the image in the background
+  // (an export takes 8-25 s)
+  if (!(await has(`[ -f ${W}/filled ]`)) && (await has(`[ -f ${T}/root.img ] && [ ! -f ${W}/filling ]`))) {
+    await session.call(`for d in "$WM_STORE"/base-*.building; do [ -d "$d" ] && [ "$d" != ${W} ] || continue; s=$(cat "$d/slot" 2>/dev/null || echo "$WM_SLOT"); [ "$s" = "$WM_SLOT" ] && sudo -n rm -rf "$d"; done
+# a tree another store's collection deleted meanwhile is filled again below
+mkdir -p ${W} && printf '%s' "$WM_SLOT" > ${W}/slot && printf '%s' tree-${hashes.tree} > ${W}/tree && { cp --reflink=always ${T}/root.img ${W}/root.img && : > ${W}/filled || rm -f ${W}/root.img; }; echo '@{"ok":true}'`);
+  }
+  if (!(await has(`[ -f ${W}/filled ]`))) {
+    if (await has(`[ -f ${W}/filling ] && kill -0 "$(cat ${W}/fill.pid 2>/dev/null)" 2>/dev/null`)) return { base, stage: `filling the tree from ${config.image}`, ready: false, detail: 'run again' };
+    if (await has(`[ -f ${W}/filling ]`)) {
+      const log = await session.call<{ log: string }>(`printf '@{"log":%s}\\n' "$(tail -c 400 ${W}/fill.log 2>/dev/null | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')"`);
+      throw new Error(`world-machine base: filling the tree from ${config.image} failed:\n${log.log}\n(remove ${config.store}/${base}.building to start over)`);
+    }
+    await session.call(`for d in "$WM_STORE"/base-*.building; do [ -d "$d" ] && [ "$d" != ${W} ] || continue; s=$(cat "$d/slot" 2>/dev/null || echo "$WM_SLOT"); [ "$s" = "$WM_SLOT" ] && sudo -n rm -rf "$d"; done
+mkdir -p ${W} && printf '%s' "$WM_SLOT" > ${W}/slot && printf '%s' tree-${hashes.tree} > ${W}/tree && : > ${W}/filling && { wm_closefds; wm_ct_base_fill ${q(config.image)} ${q(config.imageSize)} ${W} > ${W}/fill.log 2>&1 && : > ${W}/filled; } </dev/null >/dev/null 2>&1 9>&- &
+echo $! > ${W}/fill.pid; echo '@{"ok":true}'`);
+    return { base, stage: `filling the tree from ${config.image}`, ready: false };
+  }
+
+  // 2. the provisioning, run once in the tree placed in the slot; the tree as filled is kept first, for the next base
+  // made from it
+  if (!(await has(`[ -f ${W}/provisioned ]`))) {
+    const P = `${T}.part-"$WM_SLOT"`;
+    await session.call(`[ -f ${T}/root.img ] || { rm -rf ${P} && mkdir -p ${P} && cp --reflink=always ${W}/root.img ${P}/root.img && mv -T ${P} ${T}; } || rm -rf ${P}; echo '@{"ok":true}'`);
+    const provision = config.provision ? readFileSync(config.provision, 'utf8') : '';
+    const r = await session.call<{ code: number; log: string }>([
+      'wm_ct_stop; wm_ct_umount || return 1',
+      // the bundle names the slot's network namespace, which the host's boot does not make
+      'wm_ct_net >/dev/null || return 1',
+      `cp --reflink=always ${W}/root.img "$WM_ROOTIMG" || { wm_fail "cannot place the tree in the slot"; return 1; }`,
+      'wm_ct_mount || return 1',
+      `cat > "$WM_SLOTDIR/provision.sh" <<'__WM_PROVISION__'\n${provision || 'true'}\n__WM_PROVISION__`,
+      'wm_ct_provision "$WM_SLOTDIR/provision.sh"',
+    ].join('\n'));
+    if (r.code !== 0) throw new Error(`world-machine base: the provisioning script exited ${r.code}:\n${r.log}\n(remove ${config.store}/${base}.building to start over)`);
+    await session.call(`: > ${W}/provisioned; echo '@{"ok":true}'`);
+    return { base, stage: 'provisioned', ready: false, detail: 'run again' };
+  }
+
+  // 3. the init started with its services, marked, and the base put in place
+  const r = await session.call<{ pauseMs: number; memBytes: number }>([
+    'wm_ct_net >/dev/null || return 1',
+    'wm_ct_mount || return 1',
+    'wm_ct_start >/dev/null || return 1',
+    `wm_ct_mark ${W} || return 1`,
+    // W/tree: the tree it was made from, kept by gc while this base (or its build) is
+    `wm_ct_stop; rm -f ${W}/filling ${W}/filled ${W}/provisioned ${W}/fill.pid ${W}/fill.log; printf '%s' tree-${hashes.tree} > ${W}/tree; mv ${W} ${B}`,
+  ].join('\n'));
+  return { base, stage: 'ready', ready: true, detail: `marked in ${r.pauseMs} ms, ${Math.round((r.memBytes ?? 0) / 1e6)} MB of processes` };
+}
+
+export class ContainerMachine extends Machine {
+  /** What going back to the snapshot at `position` would change in the machine, read only (ct.sh wm_ct_compare): the
+   *  files under `path` (the config's export by default: the project a harness works in) and the processes. */
+  async compare(position: number, path?: string): Promise<Record<string, unknown>> {
+    const snap = headAt(this.root, position);
+    if (!snap) throw new Error(`no snapshot at machine position ${position}`);
+    // a relative path is the exported project's, as POST /machine/files reads it
+    const base = this.config.export?.path;
+    const where = path === undefined ? base : path.startsWith('/') ? path : base ? `${base}/${path}` : undefined;
+    if (!where) throw new Error(path === undefined ? 'compare needs a path (the machine exports none)' : `compare: ${path} is relative and the machine exports no project to resolve it against`);
+    return this.session.call<Record<string, unknown>>(`[ -d "$WM_STORE/${snap.dir}" ] || { wm_fail "the snapshot ${snap.id} is no longer kept"; return 1; }; wm_ct_compare "$WM_STORE/${snap.dir}" ${q(where)}${path === undefined ? '' : ' exact'}`);
+  }
+
+  /** A command run in the running machine (world-machine POST /machine/run; ct.sh wm_ct_run): as `user`, a name the
+   *  machine's passwd resolves, or its root; bounded by `timeoutMs` (at most 300 s). `{code, stdout, stderr}`, tails. */
+  async run(argv: string[], user?: string, timeoutMs = 60_000): Promise<Record<string, unknown>> {
+    await this.refresh();
+    if (this.state !== 'running') throw Object.assign(new Error(`refused: cannot run a command: the machine is ${this.state}${this.message ? ` (${this.message})` : ''}`), { status: 409 });
+    const secs = Math.max(1, Math.min(300, Math.ceil(timeoutMs / 1000)));
+    return this.session.call<Record<string, unknown>>(`wm_ct_run ${q(user ?? '')} ${secs} ${q(JSON.stringify(argv))}`);
+  }
+
+  /** Part of the machine's disk back at `position` (world-machine POST /machine/files), the machine running on: each
+   *  path, absolute in the machine or relative to its exported project, copied from the mark's image into the live
+   *  tree (ct.sh wm_ct_files). A partial retake's files: no restore, no branch; the next mark records the result. */
+  async restoreFiles(position: number, paths: string[], { noLinks = false, dryRun = false } = {}): Promise<Record<string, unknown>> {
+    const snap = headAt(this.root, position);
+    if (!snap) throw Object.assign(new Error(`no snapshot at machine position ${position}`), { status: 409 });
+    await this.refresh();
+    if (this.state !== 'running') throw Object.assign(new Error(`cannot restore files: the machine is ${this.state}${this.message ? ` (${this.message})` : ''}`), { status: 409 });
+    const base = this.config.export?.path;
+    const rels = paths.map((p) => {
+      if (!p.startsWith('/') && !base) throw Object.assign(new Error(`refused: files: ${p} is relative and the machine exports no project to resolve it against`), { status: 400 });
+      const abs = posix.normalize(p.startsWith('/') ? p : `${base}/${p}`);
+      // a relative path stays inside the exported project
+      if (!p.startsWith('/') && abs !== base && !abs.startsWith(`${base}/`)) throw Object.assign(new Error(`refused: files: ${p} climbs out of ${base}`), { status: 400 });
+      return abs.replace(/^\/+|\/+$/g, '');
+    });
+    return this.session.call<Record<string, unknown>>(`[ -d "$WM_STORE/${snap.dir}" ] || { wm_fail "the snapshot ${snap.id} is no longer kept"; return 1; }; wm_ct_files "$WM_STORE/${snap.dir}" ${q(JSON.stringify(rels))}${noLinks ? ' nolinks' : ' -'}${dryRun ? ' dry' : ''}`);
+  }
+
+  protected override baseDir(): string { return `$WM_STORE/base-${this.base}`; }
+
+  /** This configuration's base. A machine with a head restores that and reads no base, so a hash that cannot be made
+   *  (the image has no id on the host) is no failure there: the name stays what it was. */
+  private async baseName(): Promise<string | undefined> {
+    try { return await containerBaseHash(this.config, this.session); } catch (error) {
+      if (!headSnapshot(this.root)) throw error;
+      return this.base;
+    }
+  }
+  /** A machine with a head needs its base's name only for the records its marks write: learned after the restore
+   *  (queued behind it on the session), never before it. */
+  private basePending: Promise<void> | undefined;
+  private learnBase(): void {
+    if (this.base || this.basePending) return;
+    this.basePending = this.baseName().then((b) => { this.base = b ?? this.base; }, () => undefined).finally(() => { this.basePending = undefined; });
+  }
+
+  override async ensure(): Promise<Record<string, unknown> | null> {
+    if (this.state === 'running' || this.state === 'displaced') return null;
+    try {
+      // the store checked (and its loop put on direct I/O: a host boot mounts it without) before any restore
+      if (headSnapshot(this.root)) { await this.session.call('wm_reflink_check'); return await this.restore(); }
+      this.base = await this.baseName();
+      const { ready } = await this.session.call<{ ready: boolean }>(`wm_reflink_check >/dev/null || return 1\n[ -f "${this.baseDir()}/criu/inventory.img" ] && echo '@{"ready":true}' || echo '@{"ready":false}'`);
+      if (!headSnapshot(this.root) && !ready) {
+        this.state = 'no-base';
+        this.message = `no container base base-${this.base} in the store; build it with \`world-machine base --config <file>\` (rerun until it prints ready)`;
+        return null;
+      }
+      return await this.restore();
+    } catch (error) {
+      this.state = 'error';
+      this.message = (error as Error).message;
+      return null;
+    }
+  }
+
+  /** A retake's first half, run by the machine being left while the World branches: the mark a branch forked at
+   *  `position` of this machine's ledger will have as its head, restored into the slot now and paused at once, its way
+   *  out closed, the slot marked as handed over to that snapshot. The branch's machine then adopts it (restore) instead
+   *  of restoring it again, and this one, when the World stops it, leaves the container alone (stop). While paused the
+   *  machine runs nothing, and until adopted it reaches no twin, so it cannot write into the World being left. */
+  async prepare(position: number, live = false): Promise<Record<string, unknown>> {
+    const snap = headAt(this.root, position);
+    if (!snap) throw Object.assign(new Error(`no snapshot at machine@${position} of this ledger`), { status: 409 });
+    if (live) return this.prepareLive(snap, position);
+    const t0 = hostNowMs();
+    const restoreId = randomBytes(8).toString('hex');
+    // One locked op: the slot must still be this machine's; then its way out is closed and the mark swapped in. A
+    // failure once the swap began (wm_ct_restore stops this machine first) leaves the container stopped and its way
+    // out closed, never the mark running into this World's twins; the error says the slot was swapped. The slot's owner
+    // is then no machine's, only the handover (a status read here must not take it back, and this machine's stop leaves
+    // it alone).
+    const handover = { handover: snap.dir, restoreId, from: this.env.VOLTER_WORLD_NAME ?? null, position };
+    let r: Record<string, number>;
+    try {
+      r = await this.session.call<Record<string, number>>([
+        `[ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("token",""))' "$WM_SLOTDIR/owner.json" 2>/dev/null)" = ${q(this.token)} ] || { wm_fail "cannot prepare: the slot is not this machine's"; return 1; }`,
+        // held against a collection from here until the handover names it (gc reads both), before its presence is checked
+        `printf '%s' "$WM_STORE/${snap.dir}" > "$WM_SLOTDIR/preparing"`,
+        `[ -d "$WM_STORE/${snap.dir}" ] || { rm -f "$WM_SLOTDIR/preparing"; if grep -qx ${q(snap.dir.replace(/^snapshots\//, ''))} "$WM_STORE/expired.list" 2>/dev/null; then wm_fail "cannot prepare: the snapshot ${snap.id} was let go (past retain)"; else wm_fail "cannot prepare: the snapshot ${snap.id} is missing from the store"; fi; return 1; }`,
+        'wm_ct_net >/dev/null || { rm -f "$WM_SLOTDIR/preparing"; return 1; }',
+        'wm_egress >/dev/null',
+        `wm_ct_restore "$WM_STORE/${snap.dir}" ${q(restoreId)} || { rm -f "$WM_SLOTDIR/preparing"; wm_ct_stop; wm_fail "prepare failed after the slot was swapped (the machine is stopped): $(tail -2 "$WM_SLOTDIR/restore.out" 2>/dev/null)"; return 1; }`,
+        'sudo -n crun pause "$WM_CT" || { rm -f "$WM_SLOTDIR/preparing"; wm_ct_stop; wm_fail "prepare failed after the slot was swapped (the machine is stopped): cannot pause it"; return 1; }',
+        `wm_own ${q(JSON.stringify(handover))} || { rm -f "$WM_SLOTDIR/preparing"; wm_ct_stop; wm_fail "prepare failed after the slot was swapped (the machine is stopped)"; return 1; }`,
+        'rm -f "$WM_SLOTDIR/preparing"',
+      ].join('\n'));
+    } catch (error) {
+      const swapped = /after the slot was swapped/.test((error as Error).message);
+      if (swapped) { this.state = 'error'; this.message = (error as Error).message; }
+      // before the swap, nothing changed but the closed way out, put back while the slot is still this machine's
+      else await this.session.call(`if [ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("token",""))' "$WM_SLOTDIR/owner.json" 2>/dev/null)" = ${q(this.token)} ]; then wm_egress ${this.forwarders()} >/dev/null; fi; echo '@{"egress":true}'`).catch(() => undefined);
+      throw Object.assign(error as Error, { status: 409, swapped });
+    }
+    this.state = 'displaced';
+    this.message = `handed over: the slot holds ${snap.id} for the branch that forks at machine@${position}`;
+    return { id: snap.id, dir: snap.dir, loadMs: r.loadMs, restoreMs: r.restoreMs, totalMs: Math.round(hostNowMs() - t0) };
+  }
+
+  /** A handover with no restore: the running machine itself handed to the branch that forks at `position`, which must
+   *  be this ledger's head (a mark just taken). A partial retake uses it: the twins go back, the machine stays. It is not
+   *  paused (the harness's connection into it would go silent): its way out is closed and its connections through the
+   *  World's forwarders reset, so from here it reaches no twin of the World being left, while its processes and every
+   *  other connection run on. The branch adopts it as it adopts a prepared one, the connections into it kept; a new
+   *  restore id tells a program whose twin connection was reset that the World moved. */
+  private async prepareLive(snap: SnapshotRecord, position: number): Promise<Record<string, unknown>> {
+    const head = headSnapshot(this.root);
+    if (head?.dir !== snap.dir) throw Object.assign(new Error(`cannot hand over the live machine: machine@${position} (${snap.id}) is not this ledger's head (${head?.id ?? 'none'})`), { status: 409 });
+    await this.refresh();
+    if (this.state !== 'running') throw Object.assign(new Error(`cannot hand over the live machine: it is ${this.state}${this.message ? ` (${this.message})` : ''}`), { status: 409 });
+    const t0 = hostNowMs();
+    const handover = { handover: snap.dir, restoreId: randomBytes(8).toString('hex'), from: this.env.VOLTER_WORLD_NAME ?? null, position, live: true };
+    const spec = this.forwarders();
+    // One locked op: the slot still this machine's and running, its way out closed and those connections reset, the
+    // owner the handover. Nothing is restored or paused: a failure opens the way out again and the machine runs on.
+    try {
+      await this.session.call([
+        `[ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("token",""))' "$WM_SLOTDIR/owner.json" 2>/dev/null)" = ${q(this.token)} ] || { wm_fail "cannot hand over: the slot is not this machine's"; return 1; }`,
+        `[ "$(wm_ct_state)" = running ] || { wm_fail "cannot hand over: the container is $(wm_ct_state)"; return 1; }`,
+        'wm_egress >/dev/null',
+        `wm_ct_reset_egress ${spec} >/dev/null`,
+        `wm_own ${q(JSON.stringify(handover))} || { wm_fail "cannot hand over the slot"; return 1; }`,
+        `printf '%s' "$WM_STORE/${snap.dir}" > "$WM_SLOTDIR/ct-loaded"`,
+        `echo '@{"handedOver":true}'`,
+      ].join('\n'));
+    } catch (error) {
+      await this.session.call(`if [ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("token",""))' "$WM_SLOTDIR/owner.json" 2>/dev/null)" = ${q(this.token)} ]; then wm_egress ${spec} >/dev/null; fi; echo '@{"egress":true}'`).catch(() => undefined);
+      throw Object.assign(error as Error, { status: 409, swapped: false });
+    }
+    this.state = 'displaced';
+    this.message = `handed over live: the slot's running machine (at ${snap.id}) is the branch's that forks at machine@${position}; POST /machine/restore takes it back`;
+    return { id: snap.id, dir: snap.dir, live: true, totalMs: Math.round(hostNowMs() - t0) };
+  }
+
+  /** This branch's head, restored in place; with no head, the base. A slot a retake's prepare left holding this head,
+   *  paused, is adopted instead: its way out pointed at this World's twins, the World's files written, the connections
+   *  the restore left stale reset, and the processes resumed, with no second restore. */
+  override async restore(): Promise<Record<string, unknown>> {
+    const head = headSnapshot(this.root);
+    if (head) this.learnBase();
+    else if (!this.base) this.base = await this.baseName();
+    if (head) {
+      // an adopt that fails part way leaves the slot to the whole restore below, in this same call
+      const adopted = await this.adopt(head).catch((error: Error) => { process.stderr.write(`world-machine: adopting the prepared slot failed (${error.message}); restoring\n`); return null; });
+      if (adopted) return adopted;
+    }
+    const dir = head ? `$WM_STORE/${head.dir}` : this.baseDir();
+    const clockFile = this.env.TWIN_WORLD_CLOCK_FILE;
+    const frozenClock = Boolean(clockFile && existsSync(clockFile));
+    const t0 = hostNowMs();
+    this.restoreId = randomBytes(8).toString('hex');
+    const r = await this.session.call<Record<string, number>>([
+      // a snapshot let go past retain is named before anything is stopped
+      `[ -d "${dir}" ] || { if grep -qx ${q((head?.dir ?? '').replace(/^snapshots\//, ''))} "$WM_STORE/expired.list" 2>/dev/null; then wm_fail "the snapshot ${head?.id ?? 'base'} was let go (past retain)"; else wm_fail "the snapshot ${head?.id ?? 'base'} is missing from the store"; fi; return 1; }`,
+      // what the collector keeps while this machine runs (a base another World's configuration does not name), from
+      // before the restore reads it
+      `printf '%s' "${dir}" > "$WM_SLOTDIR/ct-loaded"`,
+      'wm_ct_net >/dev/null || return 1',
+      `wm_timed egress wm_egress ${this.forwarders()} || return 1`,
+      // the restore id is in the tree before the processes run: a restored connection is reset by its peer at once
+      `wm_ct_restore "${dir}" ${q(this.restoreId)} || return 1`,
+      `wm_own ${q(this.owner())}`, 'rm -f "$WM_SLOTDIR/preparing"',
+      // the World's env, certificates and redirect, and the login-shell profile that exports the env (the Firecracker base
+      // installs that once; a container's tree gets it with the rest)
+      `printf '%s' ${q(JSON.stringify({ ...this.guestFiles(), '/etc/profile.d/volter-world.sh': `${GUEST_PROFILE}\n` }))} | wm_ct_write >/dev/null || return 1`,
+      'wm_ct_reset',
+      this.config.publishPorts ? `wm_ct_publish ${q(JSON.stringify(this.config.publishPorts))}` : 'wm_ct_unpublish',
+    ].join('\n'));
+    this.lastUsed = this.session.call<{ used: number }>('wm_used').then((u) => u.used, () => Number.NaN);
+    this.loaded = head ? { id: head.id, label: head.label, dir: head.dir } : null;
+    this.state = 'running';
+    this.message = undefined;
+    return {
+      id: head?.id ?? null, label: head?.label ?? null, from: head ? 'head' : `base-${this.base}`, backend: 'container',
+      loadMs: r.loadMs, restoreMs: r.restoreMs, resetMs: r.resetMs, egressMs: r.egressMs,
+      ...(frozenClock ? { clock: 'the World\'s clock is frozen; the container reads the host\'s, which is now' } : {}),
+      totalMs: Math.round(hostNowMs() - t0),
+    };
+  }
+
+  private async adopt(head: SnapshotRecord): Promise<Record<string, unknown> | null> {
+    const slot = await this.session.call<{ container: string; owner: { handover?: string; restoreId?: string; live?: boolean } | null }>('wm_ct_running');
+    // a prepared slot is paused; a live handover's machine runs on (prepareLive)
+    const live = slot.owner?.live === true;
+    if (slot.container !== (live ? 'running' : 'paused') || slot.owner?.handover !== head.dir || !slot.owner.restoreId) return null;
+    const t0 = hostNowMs();
+    this.restoreId = slot.owner.restoreId;
+    const r = await this.session.call<Record<string, number>>([
+      `wm_timed egress wm_egress ${this.forwarders()} || return 1`,
+      `wm_own ${q(this.owner())}`, 'rm -f "$WM_SLOTDIR/preparing"',
+      `printf '%s' "$WM_STORE/${head.dir}" > "$WM_SLOTDIR/ct-loaded"`,
+      `printf '%s' ${q(JSON.stringify({ ...this.guestFiles(), '/etc/profile.d/volter-world.sh': `${GUEST_PROFILE}\n` }))} | wm_ct_write >/dev/null || return 1`,
+      // a live machine was never paused and its connections through the forwarders were reset at the handover: the
+      // connections into it (the harness's) are kept
+      live ? 'echo \'@{"resetMs":0}\'' : 'sudo -n crun resume "$WM_CT" || { wm_fail "cannot resume the adopted machine"; return 1; }',
+      live ? '' : 'wm_ct_reset',
+      this.config.publishPorts ? `wm_ct_publish ${q(JSON.stringify(this.config.publishPorts))}` : 'wm_ct_unpublish',
+    ].join('\n'));
+    this.lastUsed = this.session.call<{ used: number }>('wm_used').then((u) => u.used, () => Number.NaN);
+    this.loaded = { id: head.id, label: head.label, dir: head.dir };
+    this.state = 'running';
+    this.message = undefined;
+    return { id: head.id, label: head.label, from: 'head', backend: 'container', adopted: true, ...(live ? { live: true } : {}), egressMs: r.egressMs, resetMs: r.resetMs, totalMs: Math.round(hostNowMs() - t0) };
+  }
+
+  override async refresh(): Promise<void> {
+    if (this.state !== 'running' && this.state !== 'displaced') return;
+    const r = await this.session.call<{ container: string; owner: { token?: string; world?: string; handover?: string } | null }>('wm_ct_running');
+    if (r.owner?.handover) { this.state = 'displaced'; this.message = `the slot is handed over to a branch (${r.owner.handover}); POST /machine/restore takes it back`; }
+    else if (r.owner?.token !== this.token) { this.state = 'displaced'; this.message = `the slot now runs ${r.owner?.world ?? 'nothing'}'s machine; POST /machine/restore takes it back`; }
+    else if (r.container !== 'running') { this.state = 'error'; this.message = `the container is ${r.container || 'gone'}`; }
+    // this machine's publisher started again if it died (idempotent while it runs): refresh runs at every mark and status read
+    else if (this.config.publishPorts) await this.session.call(`wm_ct_publish ${q(JSON.stringify(this.config.publishPorts))} >/dev/null 2>&1; echo '@{"published":true}'`);
+  }
+
+  override async checkpoint(label: string, keep = false): Promise<Record<string, unknown>> {
+    await this.refresh();
+    if (this.state !== 'running') throw Object.assign(new Error(`cannot checkpoint: the machine is ${this.state}${this.message ? ` (${this.message})` : ''}`), { status: 409 });
+    await this.room();
+    await this.basePending;
+    const id = `snap-${Date.now().toString(36)}-${randomBytes(3).toString('hex')}`;
+    const dir = `snapshots/${id}`;
+    const r = await this.session.call<Record<string, number>>([
+      `mkdir -p "$WM_STORE/${dir}" && : > "$WM_STORE/${dir}/.pending" || return 1`,
+      // the mark it descends from: the dump's pages are composed against its
+      `wm_ct_mark "$WM_STORE/${dir}" "${this.loaded ? `$WM_STORE/${this.loaded.dir}` : this.baseDir()}" || return 1`,
+      'wm_used',
+    ].join('\n'));
+    const measured = await this.lastUsed; const before = Number.isNaN(measured) ? (r.used ?? 0) : measured;
+    const storeBytes = Math.max(0, (r.used ?? 0) - before);
+    this.lastUsed = Promise.resolve(r.used ?? before);
+    const record: SnapshotRecord = {
+      // no base named when it could not be learned (the image has no id on the host)
+      id, label, type: 'Full', parent: this.loaded?.id ?? `base-${this.base}`, base: this.base ? `base-${this.base}` : '', dir,
+      memBytes: r.memBytes ?? 0, diskBytes: Math.max(0, storeBytes - (r.memBytes ?? 0)), storeBytes,
+      pauseMs: r.pauseMs ?? 0, created: worldInstant(), ...(keep ? { keep: true } : {}),
+    };
+    await appendSnapshot(this.root, record);
+    await this.session.call(`rm -f "$WM_STORE/${dir}/.pending"; echo '@{"recorded":true}'`);
+    this.loaded = { id, label, dir };
+    return { id, pauseMs: r.pauseMs, treeMs: r.treeMs, dumpMs: r.dumpMs, memBytes: record.memBytes, diskBytes: record.diskBytes, storeBytes };
+  }
+
+  /** The container reads the host's kernel clock: there is no guest clock to be off. */
+  override async clock(): Promise<Record<string, unknown>> {
+    return { errorMs: 0, clock: 'the host kernel\'s (the container shares it)' };
+  }
+
+  override status(): Record<string, unknown> {
+    const s = super.status();
+    return { ...s, backend: 'container', guest: { ip: this.config.guestIp, container: `wm-${this.config.slot}` } };
+  }
+
+  override async stop(): Promise<void> {
+    try {
+      // a slot this machine handed over (prepare) holds no token of its own: it is the branch's to adopt, left as it is
+      await this.session.call(`if [ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("token",""))' "$WM_SLOTDIR/owner.json" 2>/dev/null)" = ${q(this.token)} ]; then wm_ct_unpublish; wm_ct_stop; wm_egress; wm_release; rm -f "$WM_SLOTDIR/ct-loaded"; fi; echo '@{"stopped":true}'`);
+    } finally { this.session.close(); }
+  }
+}