@volter/world-machine 2.0.0

package/src/ledger.ts

@@ -0,0 +1,61 @@
+// The machine's ledger: one World state service, `machine`, one entry per checkpoint, written through
+// the kernel's own write path (applyTwinWrite, as every twin records its writes). The World's `mark`
+// finds it like any twin's log and `branch --at machine@N` forks it at N; the head a branch restores is
+// the last snapshot in the branch's view, its parent's history up to the fork position and its own
+// entries after it.
+import { applyTwinWrite, wholeLog, worldNow } from '@volter/world-core';
+
+export const LEDGER = 'machine';
+
+export type SnapshotRecord = {
+  id: string;
+  label: string;
+  type: 'Full' | 'Diff';
+  /** the entry this one's memory diff was taken against, or the warm base */
+  parent: string;
+  base: string;
+  /** directory of vmstate, mem and disk.ext4, relative to the store */
+  dir: string;
+  memBytes: number;
+  diskBytes: number;
+  storeBytes: number;
+  pauseMs: number;
+  created: string;
+  /** taken to be kept (a mark the agent pinned): retention (`thin`) never lets it go */
+  keep?: boolean;
+};
+
+export async function appendSnapshot(root: string, record: SnapshotRecord): Promise<void> {
+  const { id, ...fields } = record;
+  await applyTwinWrite(LEDGER, {
+    operation: 'snapshot.create',
+    subjectType: 'snapshot',
+    subjectId: id,
+    fields,
+    occurredAt: record.created,
+    actor: { kind: 'system', id: 'world-machine' },
+  }, root);
+}
+
+/** The last snapshot in this branch's view of the ledger, or null. */
+export function headSnapshot(root: string): SnapshotRecord | null {
+  return headOf(wholeLog(LEDGER, root));
+}
+
+/** The head a branch forked from this one at `position` (entries of this branch's whole log) starts with: what
+ *  `world branch --at machine@<position>` gives the branch's machine to restore. */
+export function headAt(root: string, position: number): SnapshotRecord | null {
+  const seen = new Set<string>();
+  return headOf(wholeLog(LEDGER, root).filter((e) => (seen.has(e.id) ? false : (seen.add(e.id), true))).slice(0, position));
+}
+
+function headOf(log: ReturnType<typeof wholeLog>): SnapshotRecord | null {
+  const reverted = new Set(log.filter((e) => e.op === 'revert' && e.revertsActionId).map((e) => e.revertsActionId!));
+  for (let i = log.length - 1; i >= 0; i--) {
+    const e = log[i]!;
+    if (e.op === 'set' && e.subject.type === 'snapshot' && !reverted.has(e.id)) return { id: e.subject.id, ...(e.fields as Omit<SnapshotRecord, 'id'>) };
+  }
+  return null;
+}
+
+export const worldInstant = (): string => worldNow();

package/src/machine.ts

@@ -0,0 +1,314 @@
+// The machine: one Firecracker microVM in a slot on the machine host, run in place. A checkpoint
+// pauses it, writes its memory (a diff against what it was loaded from) and clones its disk, resumes
+// it, then materializes a full memory file for the entry and appends the entry to the ledger. A
+// restore replaces the slot's process with this branch's head (or the warm base), steps the guest's
+// wall clock to the World's, and points its one route out at this World's twins.
+import { createHash, randomBytes } from 'node:crypto';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { basename } from 'node:path';
+import type { MachineConfig } from './config.ts';
+import { GUEST_AGENT, GUEST_INJECT, GUEST_INJECT_DIR, GUEST_PROFILE, HostSession, hostNowMs, q } from './host.ts';
+import { appendSnapshot, headAt, headSnapshot, type SnapshotRecord, worldInstant } from './ledger.ts';
+
+type Loaded = { id: string; label: string; dir: string } | null;
+/** Rewritten with a new id at each restore, before stale connections are reset (see guestFiles). */
+export const RESTORE_ID_FILE = '/run/world-machine/restore-id';
+const IDLE = 'nothing is running yet: the first GET /machine, POST /machine/checkpoint or POST /machine/restore brings the machine up';
+export type MachineState = 'idle' | 'running' | 'displaced' | 'no-base' | 'error';
+
+/** A machine-host path as one shell word: `~/x` expands on that host. */
+export const hp = (path: string): string => (path.startsWith('~/') ? `"$HOME"/${q(path.slice(2))}` : q(path));
+
+/** Every `<VENDOR>_TWIN_URL` the World handed this service. */
+export function twinUrls(env: Record<string, string | undefined>): Record<string, URL> {
+  const out: Record<string, URL> = {};
+  for (const [k, v] of Object.entries(env)) {
+    const m = /^([A-Z0-9_]+)_TWIN_URL$/.exec(k);
+    if (m && v) { try { out[m[1]!] = new URL(v); } catch { /* not a URL: not a route */ } }
+  }
+  return out;
+}
+
+/** A stable guest-visible port per vendor: derived from the name (FNV-1a into 20000-29999), a
+ *  collision moving the later name in sorted order up by one; a configured port wins. */
+export function egressPorts(vendors: string[], fixed: Record<string, number>): Record<string, number> {
+  const taken = new Set(Object.values(fixed));
+  const out: Record<string, number> = {};
+  for (const v of [...vendors].sort()) {
+    if (fixed[v] !== undefined) { out[v] = fixed[v]!; continue; }
+    let h = 0x811c9dc5;
+    for (const c of v) { h ^= c.charCodeAt(0); h = Math.imul(h, 0x01000193) >>> 0; }
+    let port = 20000 + (h % 10000);
+    while (taken.has(port)) port = port === 29999 ? 20000 : port + 1;
+    taken.add(port); out[v] = port;
+  }
+  return out;
+}
+
+/** The guest kernel's command line. The guest's address is on-link with the tap's host end and it
+ *  has no gateway: no route leads anywhere but the machine host. */
+export const bootArgs = (config: MachineConfig): string => `console=ttyS0 reboot=k panic=1 ip=${config.guestIp}:::255.255.255.252::eth0:off`;
+
+/** The egress port name of the World's proxy (never a vendor's: vendors come from *_TWIN_URL names). */
+const WORLD_PROXY = 'VOLTER_WORLD_PROXY';
+
+export async function baseHash(config: MachineConfig, session: HostSession): Promise<string> {
+  const facts = await session.call<{ kernel: string; rootfs: string; fc: string }>(
+    `printf '@{"kernel":"%s","rootfs":"%s","fc":"%s"}\\n' "$(stat -c %s-%Y ${hp(config.kernel)})" "$(stat -c %s-%Y ${hp(config.rootfs)})" "$(${hp(config.firecracker)} --version 2>&1 | head -1)"`);
+  const provision = config.provision ? readFileSync(config.provision, 'utf8') : '';
+  const inputs = { ...facts, bootArgs: bootArgs(config), provision, agent: GUEST_AGENT, profile: GUEST_PROFILE, vcpus: config.vcpus, memMib: config.memMib, slot: config.slot, tap: config.tap, hostIp: config.hostIp, guestIp: config.guestIp, store: config.store };
+  return createHash('sha256').update(JSON.stringify(inputs)).digest('hex').slice(0, 16);
+}
+
+export class Machine {
+  state: MachineState = 'idle';
+  message: string | undefined = IDLE;
+  loaded: Loaded = null;
+  protected base: string | undefined;
+  protected lastUsed: Promise<number> = Promise.resolve(0);
+  protected readonly token = randomBytes(8).toString('hex');
+  /** This load's restore id (RESTORE_ID_FILE); a backend that can write it before the machine runs sets it first. */
+  protected restoreId = '';
+  readonly session: HostSession;
+  protected readonly ports: Record<string, number>;
+  protected readonly twins: Record<string, URL>;
+
+  constructor(readonly config: MachineConfig, readonly root: string, readonly env: Record<string, string | undefined>, readonly port: number) {
+    this.session = new HostSession(config);
+    this.twins = twinUrls(env);
+    // the World's egress proxy (its CONNECT door to the twins and, as its network policy allows, beyond) has a
+    // stable guest-visible port like a twin's
+    this.ports = egressPorts([...Object.keys(this.twins), WORLD_PROXY], config.egressPorts);
+  }
+
+  /** What the guest sees: vendor → its stable URL on the tap's host address. */
+  egress(): Record<string, string> {
+    return Object.fromEntries(Object.entries(this.twins).map(([v, u]) => [v, `http://${this.config.hostIp}:${this.ports[v]}${u.pathname === '/' ? '' : u.pathname}`]));
+  }
+
+  /** The World's live env (VOLTER_WORLD_ENV_FILE), `export K='v'` lines. */
+  protected worldEnv(): Record<string, string> {
+    const file = this.env.VOLTER_WORLD_ENV_FILE;
+    const out: Record<string, string> = {};
+    if (!file || !existsSync(file)) return out;
+    for (const line of readFileSync(file, 'utf8').split('\n')) {
+      const m = /^export ([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
+      if (m) out[m[1]!] = m[2]!.startsWith("'") ? m[2]!.slice(1, -1).replaceAll("'\\''", "'") : m[2]!;
+    }
+    return out;
+  }
+
+  /** The World's proxy as the machine reaches it, read from its live env at each load: its guests' listener
+   *  (`VOLTER_WORLD_GUEST_PROXY`, where every connection is kept off this host's own network, whatever NAT makes its
+   *  source look like; world-runtime redirect-proxy.ts), else, from an older World, its one listener. */
+  protected proxy(): URL | null {
+    const env = this.worldEnv();
+    try { return new URL(env.VOLTER_WORLD_GUEST_PROXY || env.VOLTER_WORLD_PROXY || ''); } catch { return null; }
+  }
+
+  /** What the guest is given at each start and restore: /etc/world.env, the World's live env as the guest sees it,
+   *  and the files it names. Twin URLs become the stable egress ones, the World's proxy its forwarder (the twins'
+   *  own addresses bypass it), NODE_OPTIONS the guest's copy of the redirect, and a value naming a certificate file on the
+   *  World's host (its CA and trust bundle, PEM) a copy of it under /etc; what only that host can use (a
+   *  directory there, a loopback URL of its other listeners) is left out and named, and an empty value is left unset. */
+  protected guestFiles(): Record<string, string> {
+    const out = Object.entries(this.egress()).map(([v, u]) => `${v}_TWIN_URL=${u}`);
+    // a new id at every load, written before the connections the load left stale are reset: a program in the machine
+    // that loses a connection can tell a restore (the id changed since it connected) from its peer going away
+    const files: Record<string, string> = { [RESTORE_ID_FILE]: `${this.restoreId || randomBytes(8).toString('hex')}\n` };
+    const env = this.worldEnv();
+    const proxy = this.proxy();
+    const guestProxy = proxy ? `http://${this.config.hostIp}:${this.ports[WORLD_PROXY]}` : null;
+    // a value naming either of the proxy's listeners (HTTPS_PROXY names the World's own) is the machine's forwarder
+    const proxyPorts = new Set([proxy?.port, ...['VOLTER_WORLD_PROXY', 'VOLTER_WORLD_GUEST_PROXY'].map((k) => { try { return new URL(env[k] ?? '').port; } catch { return ''; } })].filter(Boolean));
+    const hostPorts = new Set([...Object.values(this.twins).map((u) => u.port), String(this.port), ...proxyPorts, ...['MACHINE_URL'].map((k) => { try { return new URL(env[k] ?? '').port; } catch { return ''; } })].filter(Boolean));
+    const left: string[] = [];
+    const unset: string[] = [];
+    for (const [k, v] of Object.entries(env)) {
+      if (k.endsWith('_TWIN_URL') || v.includes('\n')) continue;
+      // an empty value (a repository's .env.example keys, detected with no value) is left unset: set, even to nothing,
+      // it would win over the app's own env files, which Next.js and dotenv never let override a set variable (measured:
+      // the app's plan ids in .env.local were hidden, and the agent had to unset them on every start)
+      if (v === '') { unset.push(k); continue; }
+      if (k === 'NODE_OPTIONS') { out.push(`NODE_OPTIONS=--require ${GUEST_INJECT_DIR}/inject.cjs`); continue; }
+      if (/^(no_proxy|NO_PROXY)$/.test(k)) { out.push(`${k}=${[v, this.config.hostIp].filter(Boolean).join(',')}`); continue; }
+      const loopback = /^[a-z]+:\/\/(?:[^@/]*@)?(?:127\.0\.0\.1|localhost):(\d+)/.exec(v);
+      if (loopback && proxy && proxyPorts.has(loopback[1]!) && guestProxy) { out.push(`${k}=${guestProxy}`); continue; }
+      if (v.startsWith('/')) {
+        let size = -1;
+        try { const s = statSync(v); if (s.isFile()) size = s.size; } catch { /* absent */ }
+        const pem = size >= 0 && size <= 1 << 20 ? readFileSync(v, 'utf8') : '';
+        if (pem.startsWith('-----BEGIN ')) {
+          const guest = `/etc/volter-world-${basename(v)}`;
+          files[guest] = pem;
+          out.push(`${k}=${guest}`);
+        } else left.push(k);
+        continue;
+      }
+      if (loopback && hostPorts.has(loopback[1]!)) { left.push(k); continue; }
+      out.push(`${k}=${v}`);
+    }
+    // Node's own fetch (corepack, npm's newer paths) uses HTTP(S)_PROXY only when asked: without it corepack could not
+    // reach the registry from the machine (measured), where the proxy is the only way out
+    if (guestProxy && !('NODE_USE_ENV_PROXY' in env)) out.push('NODE_USE_ENV_PROXY=1');
+    if (left.length) out.unshift(`# not carried (the World's host only): ${left.join(' ')}`);
+    if (unset.length) out.unshift(`# empty in the World, so left unset (the app's own env files can set them): ${unset.join(' ')}`);
+    // world-core's redirect with the files it loads: written at every load rather than kept in the warm base, whose
+    // hash would otherwise move with every change to the pack facts (and rebuild the base each time twin moves)
+    for (const [f, body] of Object.entries(GUEST_INJECT)) files[`${GUEST_INJECT_DIR}/${f}`] = body;
+    return { ...files, '/etc/world.env': `${out.join('\n')}\n` };
+  }
+
+  protected forwarders(): string {
+    const proxy = this.proxy();
+    return Object.entries(this.twins).map(([v, u]) => {
+      const host = u.hostname === '127.0.0.1' || u.hostname === 'localhost' ? this.config.twinHost : u.hostname;
+      return q(`${this.ports[v]}=${host}:${u.port || (u.protocol === 'https:' ? '443' : '80')}`);
+    }).concat(proxy ? [q(`${this.ports[WORLD_PROXY]}=${proxy.hostname === '127.0.0.1' || proxy.hostname === 'localhost' ? this.config.twinHost : proxy.hostname}:${proxy.port || '80'}`)] : []).join(' ');
+  }
+
+  protected owner(): string {
+    return JSON.stringify({ token: this.token, world: this.env.VOLTER_WORLD_NAME ?? null, root: this.root, pid: process.pid, port: this.port });
+  }
+
+  protected baseDir(): string { return `$WM_STORE/base-${this.base}`; }
+
+  /** The machine, brought up when first asked for: nothing runs until a door needs it, so a new branch's
+   *  machine is loaded once, by the restore that retakes to its mark, instead of booting its head at start and
+   *  being replaced a moment later. `GET /machine` and `checkpoint` bring up the head if the ledger has one,
+   *  else the warm base; `restore` loads the head itself. Returns the restore's facts when it brought it up. */
+  async ensure(): Promise<Record<string, unknown> | null> {
+    if (this.state === 'running' || this.state === 'displaced') return null;
+    try {
+      this.base = await baseHash(this.config, this.session);
+      const { ready } = await this.session.call<{ ready: boolean }>(`wm_reflink_check >/dev/null || return 1\n[ -f "${this.baseDir()}/vmstate" ] && echo '@{"ready":true}' || echo '@{"ready":false}'`);
+      if (!headSnapshot(this.root) && !ready) {
+        this.state = 'no-base';
+        this.message = `no warm base base-${this.base} in the store; build it with \`world-machine base --config <file>\` (rerun until it prints ready)`;
+        return null;
+      }
+      return await this.restore();
+    } catch (error) {
+      this.state = 'error';
+      this.message = (error as Error).message;
+      return null;
+    }
+  }
+
+  /** This branch's head, loaded in place; with no head, the warm base. */
+  async restore(): Promise<Record<string, unknown>> {
+    if (!this.base) { this.base = await baseHash(this.config, this.session); }
+    const head = headSnapshot(this.root);
+    const dir = head ? `$WM_STORE/${head.dir}` : this.baseDir();
+    const { offsetMs, rttMs } = await this.session.offset();
+    const clockFile = this.env.TWIN_WORLD_CLOCK_FILE;
+    const deltaS = clockFile && existsSync(clockFile) ? (Date.parse(worldInstant()) - hostNowMs()) / 1000 : 0;
+    const t0 = hostNowMs();
+    const r = await this.session.call<Record<string, number>>([
+      'wm_net >/dev/null || return 1',
+      `wm_timed egress wm_egress ${this.forwarders()} || return 1`,
+      `wm_restore "${dir}" || return 1`,
+      `wm_own ${q(this.owner())}`,
+      `printf '%s' ${q(JSON.stringify(this.guestFiles()))} | wm_timed agent wm_agent clock ${this.session.offsetS} ${deltaS} - || return 1`,
+    ].join('\n'));
+    // the store's baseline for the next checkpoint's growth, read after the answer (a syncfs is slow)
+    this.lastUsed = this.session.call<{ used: number }>('wm_used').then((u) => u.used, () => Number.NaN);
+    this.loaded = head ? { id: head.id, label: head.label, dir: head.dir } : null;
+    this.state = 'running';
+    this.message = undefined;
+    return {
+      id: head?.id ?? null, label: head?.label ?? null, from: head ? 'head' : `base-${this.base}`,
+      loadMs: r.loadMs, restoreMs: r.restoreMs, clockMs: r.clockMs, clockErrorMs: r.errorMs, clockSteps: r.steps, agentConnectMs: r.connectMs, agentFirstReplyMs: r.firstReplyMs, agentFirstHandleMs: r.firstHandleMs, agentFirstWaitMs: r.firstWaitMs, agentFirstSetupMs: r.firstSetupMs, staleReset: r.staleReset, resetMs: r.resetMs, agentMs: r.agentMs, egressMs: r.egressMs,
+      hostOffsetMs: offsetMs, hostRttMs: rttMs, totalMs: Math.round(hostNowMs() - t0),
+    };
+  }
+
+  /** Is the slot still this service's? A later restore by another World's machine takes it. */
+  async refresh(): Promise<void> {
+    if (this.state !== 'running' && this.state !== 'displaced') return;
+    const r = await this.session.call<{ fc: string; owner: { token?: string; world?: string } | null }>('wm_running');
+    if (r.owner?.token !== this.token) { this.state = 'displaced'; this.message = `the slot now runs ${r.owner?.world ?? 'nothing'}'s machine; POST /machine/restore takes it back`; }
+    else if (r.fc !== 'Running') { this.state = 'error'; this.message = `firecracker is ${r.fc}`; }
+  }
+
+  async checkpoint(label: string, keep = false): Promise<Record<string, unknown>> {
+    await this.refresh();
+    if (this.state !== 'running') throw Object.assign(new Error(`cannot checkpoint: the machine is ${this.state}${this.message ? ` (${this.message})` : ''}`), { status: 409 });
+    await this.room();
+    const id = `snap-${Date.now().toString(36)}-${randomBytes(3).toString('hex')}`;
+    const dir = `snapshots/${id}`;
+    const parentMem = this.loaded ? `$WM_STORE/${this.loaded.dir}/mem` : `${this.baseDir()}/mem`;
+    // `.pending` until the ledger names it: a collection running meanwhile (another World's serve) keeps it
+    const r = await this.session.call<Record<string, number>>([
+      `mkdir -p "$WM_STORE/${dir}" && : > "$WM_STORE/${dir}/.pending" || return 1`,
+      `wm_snapshot "$WM_STORE/${dir}" Diff || return 1`,
+      `wm_materialize "${parentMem}" "$WM_STORE/${dir}" || return 1`,
+      // the machine now runs on from this entry: its memory is the next checkpoint's parent
+      `printf '%s' "$WM_STORE/${dir}" > "$WM_SLOTDIR/loaded"`,
+      'wm_used',
+    ].join('\n'));
+    const measured = await this.lastUsed; const before = Number.isNaN(measured) ? (r.used ?? 0) : measured;
+    const storeBytes = Math.max(0, (r.used ?? 0) - before);
+    this.lastUsed = Promise.resolve(r.used ?? before);
+    const record: SnapshotRecord = {
+      id, label, type: 'Diff', parent: this.loaded?.id ?? `base-${this.base}`, base: `base-${this.base}`, dir,
+      memBytes: r.memBytes ?? 0, diskBytes: Math.max(0, storeBytes - (r.memBytes ?? 0)), storeBytes,
+      pauseMs: r.pauseMs ?? 0, created: worldInstant(), ...(keep ? { keep: true } : {}),
+    };
+    await appendSnapshot(this.root, record);
+    await this.session.call(`rm -f "$WM_STORE/${dir}/.pending"; echo '@{"recorded":true}'`);
+    this.loaded = { id, label, dir };
+    return { id, pauseMs: r.pauseMs, memBytes: record.memBytes, diskBytes: record.diskBytes, storeBytes, memWriteMs: r.memWriteMs, diskCloneMs: r.diskCloneMs, materializeMs: r.materializeMs };
+  }
+
+  /** Refused (507) unless every filesystem beneath the store has room for a mark: the last mark's store growth, at
+   *  least `firstMarkBytes` when there is none, plus the configured reserve. Checked before anything is written. */
+  protected async room(firstMarkBytes = 2 ** 30): Promise<void> {
+    const last = headSnapshot(this.root)?.storeBytes ?? 0;
+    const need = Math.max(last, firstMarkBytes) + this.config.room.reserveBytes;
+    try {
+      await this.session.call(`wm_room ${need} ${this.config.room.paths.map(hp).join(' ')}`);
+    } catch (error) {
+      // 507 only for the refusal itself; a host that did not answer is the error it was
+      throw /^no room for a mark/.test((error as Error).message) ? Object.assign(error as Error, { status: 507 }) : error;
+    }
+  }
+
+  /** For each position of this machine's ledger (a mark's `machine@N`), the snapshot a branch forked there would restore
+   *  and whether the store still holds it: gc with `retain` lets old ones go. */
+  async present(positions: number[]): Promise<Array<{ position: number; id: string | null; present: boolean }>> {
+    const at = positions.map((position) => ({ position, snap: headAt(this.root, position) }));
+    const dirs = [...new Set(at.flatMap(({ snap }) => (snap ? [snap.dir] : [])))];
+    const r = dirs.length ? await this.session.call<{ present: string[] }>(`printf '@{"present":['; s=''; for d in ${dirs.map(q).join(' ')}; do [ -d "$WM_STORE/$d" ] && [ ! -e "$WM_STORE/$d/.pending" ] && { printf '%s"%s"' "$s" "$d"; s=,; }; done; printf ']}\n'`) : { present: [] };
+    const held = new Set(r.present);
+    return at.map(({ position, snap }) => ({ position, id: snap?.id ?? null, present: snap ? held.has(snap.dir) : false }));
+  }
+
+  /** The guest's wall clock against the World's, now. */
+  async clock(): Promise<Record<string, unknown>> {
+    const { offsetMs, rttMs } = await this.session.offset();
+    const clockFile = this.env.TWIN_WORLD_CLOCK_FILE;
+    const deltaS = clockFile && existsSync(clockFile) ? (Date.parse(worldInstant()) - hostNowMs()) / 1000 : 0;
+    const r = await this.session.call<{ errorMs: number; probeRttMs: number }>(`wm_agent now ${this.session.offsetS} ${deltaS}`);
+    return { errorMs: r.errorMs, guestRttMs: r.probeRttMs, hostOffsetMs: offsetMs, hostRttMs: rttMs };
+  }
+
+  status(): Record<string, unknown> {
+    const head = headSnapshot(this.root);
+    return {
+      state: this.state,
+      head: head ? { id: head.id, label: head.label } : null,
+      loaded: this.state === 'running' ? (this.loaded ? { id: this.loaded.id, label: this.loaded.label } : { base: `base-${this.base}` }) : null,
+      guest: { ip: this.config.guestIp, sshPort: 22, sshKey: this.config.sshKey, user: 'root' },
+      egress: this.egress(),
+      ...(this.message ? { message: this.message } : {}),
+    };
+  }
+
+  /** On service stop: the slot is released only if it is still this service's. */
+  async stop(): Promise<void> {
+    try {
+      await this.session.call(`if [ "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("token",""))' "$WM_SLOTDIR/owner.json" 2>/dev/null)" = ${q(this.token)} ]; then wm_fc_stop; wm_egress; wm_release; fi; echo '@{"stopped":true}'`);
+    } finally { this.session.close(); }
+  }
+}

package/src/publish.py

@@ -0,0 +1,134 @@
+#!/usr/bin/python3
+"""publish.py <slot dir> <netns> <container> <ports JSON> <sum>: the container machine's listening TCP ports on this host's
+loopback, as the machine's own `localhost`. The machine is where the app runs, and what the app is told is its address
+(`http://localhost:3000`) is what it registers with a vendor: a twin delivering a webhook there posts from this host,
+and reached nothing (a Stripe twin's deliveries were dropped). Published here, the twin's delivery reaches the app, and
+a Lima VM forwards this host's loopback ports to the Mac, so the user's own browser opens the same URL.
+
+Only the ports the config allows (<ports JSON>: {"ranges": [[from, to], ...], "except": [port, ...]}) are published:
+the machine runs code the user did not write, and a port it held first would be taken from a service of this host that
+binds it later (the retake stack's own doors, a twin restarting). Every half second the machine's listeners are read
+from its network namespace (/proc/<init>/net/tcp and tcp6); an allowed port is published on 127.0.0.1 when
+  - the machine listens on it on a wildcard or loopback address, and not also on an address of its own (a port the
+    machine's doors bind, the job server and a harness's executor, is never published, however else it is bound), and
+  - nothing on this host listens on it.
+Each connection enters the machine's namespace and reaches its loopback, 127.0.0.1 for an IPv4 listener and ::1 for an
+IPv6 one, so it arrives at exactly the listener that was classified. A port the machine stops listening on stops being
+published. Runs as root (entering the namespace needs it), one per slot (a lock in the slot), its pid in
+<slot>/publish.pid; its socat children die with it."""
+import ctypes, fcntl, json, os, signal, subprocess, sys, time
+
+SLOT, NETNS, CONTAINER, PORTS, SUM = sys.argv[1], sys.argv[2], sys.argv[3], json.loads(sys.argv[4]), sys.argv[5]
+RANGES = [tuple(r) for r in PORTS.get('ranges', [])]
+EXCEPT = set(PORTS.get('except', []))
+children = {}  # port -> [socat processes]: one on 127.0.0.1, and one on ::1 where that is free
+PR_SET_PDEATHSIG = 1
+libc = ctypes.CDLL(None, use_errno=True)
+LOOPBACK = ('0100007F', '00000000000000000000000001000000', '0000000000000000FFFF00000100007F')
+
+
+def allowed(port):
+    return port not in EXCEPT and any(lo <= port <= hi for lo, hi in RANGES)
+
+
+def listeners(path, family):
+    """[(port, kind, family)] for the LISTEN sockets in a /proc/net/tcp{,6} file; kind is any | loopback | addr."""
+    try:
+        rows = open(path).read().splitlines()[1:]
+    except OSError:
+        return []
+    out = []
+    for row in rows:
+        f = row.split()
+        if len(f) < 4 or f[3] != '0A':
+            continue
+        addr, port = f[1].rsplit(':', 1)
+        kind = 'any' if set(addr) == {'0'} else 'loopback' if addr in LOOPBACK else 'addr'
+        out.append((int(port, 16), kind, family))
+    return out
+
+
+def machine(previous):
+    """{port: family} the machine publishes now. A failed read of the container keeps the previous set (a busy
+    `crun state` during a mark must not drop live connections); it is empty only when the container is gone."""
+    r = subprocess.run(['crun', 'state', CONTAINER], capture_output=True, timeout=5)
+    if r.returncode != 0:
+        gone = b'does not exist' in r.stderr or b'no such' in r.stderr.lower()
+        return {} if gone else previous
+    try:
+        pid = json.loads(r.stdout)['pid']
+    except (ValueError, KeyError):
+        return previous
+    kinds = {}
+    for name, family in (('tcp', 4), ('tcp6', 6)):
+        for port, kind, fam in listeners(f'/proc/{pid}/net/{name}', family):
+            kinds.setdefault(port, []).append((kind, fam))
+    out = {}
+    for port, found in kinds.items():
+        if not allowed(port) or any(kind == 'addr' for kind, _ in found):
+            continue
+        out[port] = 4 if any(fam == 4 for _, fam in found) else 6
+    return out
+
+
+def host_busy():
+    return {port for name in ('tcp', 'tcp6') for port, _, _ in listeners(f'/proc/net/{name}', 0)}
+
+
+def die_with_parent():
+    libc.prctl(PR_SET_PDEATHSIG, signal.SIGTERM)
+
+
+def stop(port):
+    """The port's listeners and every connection they forked (their own process groups: a fork does not keep the
+    parent-death signal, so an open connection would otherwise outlive the publisher)."""
+    for p in children.pop(port, []):
+        try:
+            os.killpg(p.pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+        try:
+            p.wait(2)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(p.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+
+
+def shutdown(*_):
+    for port in list(children):
+        stop(port)
+    sys.exit(0)
+
+
+lock = open(os.path.join(SLOT, 'publish.lock'), 'w')
+try:
+    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+except OSError:
+    sys.exit(0)  # this slot's publisher is running
+for name, value in (('publish.pid', str(os.getpid())), ('publish.sum', SUM)):
+    with open(os.path.join(SLOT, name), 'w') as f:
+        f.write(value)
+signal.signal(signal.SIGTERM, shutdown)
+signal.signal(signal.SIGINT, shutdown)
+wanted = {}
+while True:
+    try:
+        wanted = machine(wanted)
+        for port in list(children):
+            if port not in wanted or children[port][0].poll() is not None:
+                stop(port)
+        busy = host_busy()
+        for port, family in wanted.items():
+            if port in children or port in busy:
+                continue
+            inside = f'TCP4\\:127.0.0.1\\:{port}' if family == 4 else f'TCP6\\:[\\:\\:1]\\:{port}'
+            target = f'EXEC:ip netns exec {NETNS} socat STDIO {inside}'
+            # 127.0.0.1, and ::1 as well (a client resolving localhost to ::1 first); errors to the log
+            children[port] = [subprocess.Popen(['socat', listen, target], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+                                               stderr=sys.stdout, preexec_fn=die_with_parent, start_new_session=True)
+                              for listen in (f'TCP4-LISTEN:{port},bind=127.0.0.1,reuseaddr,fork', f'TCP6-LISTEN:{port},bind=[::1],ipv6only=1,reuseaddr,fork')]
+    except Exception as error:  # one bad pass is not the publisher's end
+        print(f'publish: {error}', flush=True)
+    time.sleep(0.5)

package/src/thin.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, test } from 'bun:test';
+import { thinned } from './thin.ts';
+
+const HOUR = 3_600_000;
+const now = Date.UTC(2026, 8, 27, 12, 0, 0);
+const snap = (dir: string, agoMs: number, keep?: boolean) => ({ dir, takenMs: now - agoMs, ...(keep ? { keep } : {}) });
+
+describe('thinned', () => {
+  test('the newest `retain` stay whatever their age', () => {
+    const s = [snap('a', 1000), snap('b', 40 * 24 * HOUR), snap('c', 50 * 24 * HOUR)];
+    expect([...thinned(s, now, { retain: 2 })].sort()).toEqual(['a', 'b']);
+  });
+
+  test('past retain, the oldest of each hour for a day', () => {
+    // three in the hour 3 h ago, one 5 h ago
+    const s = [snap('new', 0), snap('h3a', 3 * HOUR + 60_000), snap('h3b', 3 * HOUR + 120_000), snap('h3c', 3 * HOUR + 180_000), snap('h5', 5 * HOUR + 1000)];
+    expect([...thinned(s, now, { retain: 1 })].sort()).toEqual(['h3c', 'h5', 'new']);
+  });
+
+  test('past the hours, the oldest of each day for a month; older let go', () => {
+    const s = [snap('new', 0), snap('d2a', 2 * 24 * HOUR + HOUR), snap('d2b', 2 * 24 * HOUR + 2 * HOUR), snap('old', 45 * 24 * HOUR)];
+    expect([...thinned(s, now, { retain: 1 })].sort()).toEqual(['d2b', 'new']);
+  });
+
+  test('a snapshot taken to be kept is never thinned', () => {
+    const s = [snap('new', 0), snap('pin', 90 * 24 * HOUR, true), snap('old', 91 * 24 * HOUR)];
+    expect([...thinned(s, now, { retain: 1 })].sort()).toEqual(['new', 'pin']);
+  });
+
+  test('a keeper is never displaced: new marks leaving retain, and crossing into the day tier', () => {
+    // one snapshot every 10 minutes for three days, thinned at each step of the way with retain 5: a snapshot that
+    // stayed past retain goes only at the step it crosses from the hour tier into the day tier (an hour's keeper that
+    // is not its day's oldest), never because a newer one arrived; a day's keeper never goes within the month
+    const step = 10 * 60_000;
+    const all: { dir: string; takenMs: number }[] = [];
+    const start = now - 3 * 24 * HOUR;
+    let stayed: { dir: string; takenMs: number }[] = [];
+    for (let t = start, i = 0; t <= now; t += step, i++) {
+      all.push({ dir: `s${String(i).padStart(4, '0')}`, takenMs: t });
+      const kept = thinned(all, t, { retain: 5 });
+      const newest = new Set(all.slice(-5).map((x) => x.dir));
+      for (const s of stayed) {
+        if (newest.has(s.dir) || kept.has(s.dir)) continue;
+        const crossedNow = t - s.takenMs >= 24 * HOUR && t - step - s.takenMs < 24 * HOUR;
+        expect({ dir: s.dir, crossedNow }).toEqual({ dir: s.dir, crossedNow: true });
+      }
+      stayed = all.filter((x) => kept.has(x.dir) && !newest.has(x.dir));
+    }
+  });
+
+  test('pins past keepMax thin like any other', () => {
+    const s = [snap('new', 0), snap('p1', 60 * 24 * HOUR, true), snap('p2', 61 * 24 * HOUR, true)];
+    expect([...thinned(s, now, { retain: 1, keepMax: 1 })].sort()).toEqual(['new', 'p1']);
+  });
+
+  test('a bucket does not move as time passes: what stayed an hour ago stays', () => {
+    const s = [snap('new', 0), snap('x', 2 * HOUR + 10_000), snap('y', 2 * HOUR + 20_000)];
+    const before = thinned(s, now, { retain: 1 });
+    const later = thinned(s, now + 30 * 60_000, { retain: 1 });
+    expect([...later].sort()).toEqual([...before].sort());
+  });
+});

package/src/thin.ts

@@ -0,0 +1,48 @@
+// Retention thinned with age (config `thin`), as a backup's history is: every snapshot of the last `retain`, then one
+// of each calendar hour for a day, then one of each UTC day for a month; older ones are let go. The one kept in a
+// bucket is its OLDEST: nothing older can join a bucket later, so a keeper is never displaced by a newer mark leaving
+// `retain` or crossing into the day tier (the day's oldest crosses first), and what a listing showed kept stays kept
+// until its tier ends. A snapshot taken to be kept (`keep`: a mark the agent pinned) is never thinned, the newest
+// `keepMax` of them (a pin in a loop must not fill the store). Pure: gc.ts ranks the calling World's own snapshots
+// and asks which of them stay.
+
+export interface Thinnable {
+  /** The snapshot's directory name (its id). */
+  dir: string;
+  /** When the host took it (ms since the epoch). */
+  takenMs: number;
+  /** Taken to be kept: never thinned. */
+  keep?: boolean;
+}
+
+export interface ThinRule {
+  /** How many of the newest stay whatever their age. */
+  retain: number;
+  /** Hours for which one of each hour stays (after the newest `retain`). */
+  hours?: number;
+  /** Days for which one of each day stays (after the hours). */
+  days?: number;
+  /** How many `keep` snapshots are never thinned (the newest); older ones thin like any other. */
+  keepMax?: number;
+}
+
+const HOUR = 3_600_000;
+const DAY = 24 * HOUR;
+
+/** The dirs of `snapshots` that stay at `nowMs`. */
+export function thinned(snapshots: Thinnable[], nowMs: number, rule: ThinRule): Set<string> {
+  const hours = rule.hours ?? 24;
+  const days = rule.days ?? 30;
+  const newestFirst = [...snapshots].sort((a, b) => b.takenMs - a.takenMs || b.dir.localeCompare(a.dir));
+  const kept = new Set<string>();
+  const pinned = new Set(newestFirst.filter((s) => s.keep).slice(0, rule.keepMax ?? 20).map((s) => s.dir));
+  newestFirst.forEach((s, i) => { if (pinned.has(s.dir) || i < rule.retain) kept.add(s.dir); });
+  // each bucket's oldest (walked oldest first): by the calendar hour or UTC day it was taken in
+  const buckets = new Set<string>();
+  for (const s of [...newestFirst].reverse()) {
+    const age = nowMs - s.takenMs;
+    const bucket = age < hours * HOUR ? `h${Math.floor(s.takenMs / HOUR)}` : age < hours * HOUR + days * DAY ? `d${Math.floor(s.takenMs / DAY)}` : null;
+    if (bucket && !buckets.has(bucket)) { buckets.add(bucket); kept.add(s.dir); }
+  }
+  return kept;
+}