@ghalex/kvbox-server 0.0.17 → 0.1.1

package/dist/kvbox-server.js

@@ -66,6 +66,10 @@ function paths(home = kvboxHome()) {
          * The next snapshot is a diff of the pages dirtied since, merged onto this. Absent after a boot.
          */
         memBase: join(dir, "mem.base"),
+        /** `kvbox hibernate`: the box's memory image and VM state while it is hibernated; gone again after `wake` or `stop`. */
+        hibernateDir: join(dir, "hibernate"),
+        hibernateVmstate: join(dir, "hibernate", "vmstate"),
+        hibernateMem: join(dir, "hibernate", "mem"),
         sock: join(dir, "fc.sock"),
         /** Firecracker's vsock endpoint: the host connects here to reach kvbox-agent inside the box (core/agent.ts). */
         vsock: join(dir, "vsock.sock"),
@@ -295,7 +299,9 @@ var MachineMetaSchema = z2.object({
    * each one, so when this is > 0 the next diff is only standalone merged onto `mem.base`; if that file
    * is missing the lineage is broken and the next snapshot has to be a full dump. Absent: 0.
    */
-  memSnapshots: z2.number().int().optional()
+  memSnapshots: z2.number().int().optional(),
+  /** `kvbox pause`: Firecracker is alive but the VM is frozen (standby). Absent: not paused. */
+  paused: z2.boolean().optional()
 });
 var SnapshotMetaSchema = z2.object({
   name: z2.string(),
@@ -474,6 +480,7 @@ function migrate(db) {
     }
   }
 }
+var LIVE_STATES = ["running", "standby"];
 function fromRaw(r) {
   return { boxId: r.box_id, owner: r.owner, name: r.name, at: r.at, state: r.state, vcpu: r.vcpu, memMib: r.mem_mib, diskGib: r.disk_gib, by: r.by };
 }
@@ -488,13 +495,13 @@ function insertRaw(e, db) {
 }
 function insertUsageEvent(e, p = paths()) {
   const db = openDb(p);
-  if (e.state === "running") closeOpenRun(e.boxId, e.at, p);
+  if (e.state === "running" && e.by !== "resume") closeOpenRun(e.boxId, e.at, p);
   insertRaw(e, db);
 }
 function closeOpenRun(boxId, at, p = paths()) {
   const db = openDb(p);
   const last = lastEvent(boxId, db);
-  if (last === void 0 || last.state !== "running") return false;
+  if (last === void 0 || !LIVE_STATES.includes(last.state)) return false;
   insertRaw({ ...last, at, state: "stopped", by: "server" }, db);
   return true;
 }
@@ -771,6 +778,16 @@ function recordState(meta, state, by, p, at = (/* @__PURE__ */ new Date()).toISO
     console.error(`warning: usage ledger: ${err.message}`);
   }
 }
+function isHibernated(meta, p) {
+  return !pidAlive(meta.pid) && existsSync3(p.machine(meta.owner, meta.name).hibernateVmstate);
+}
+function statusOf(meta, p = paths()) {
+  if (pidAlive(meta.pid)) return meta.paused ? "standby" : "running";
+  return isHibernated(meta, p) ? "hibernated" : "stopped";
+}
+function stateMessage(ref, status) {
+  return `box ${shown(ref)} is ${status === "standby" ? "in standby" : status}`;
+}
 function toMachine(meta, p = paths()) {
   const alive = pidAlive(meta.pid);
   if (!alive && meta.pid !== null) {
@@ -792,7 +809,7 @@ function toMachine(meta, p = paths()) {
     base: meta.base ?? null,
     createdAt: meta.createdAt,
     source: meta.source,
-    status: alive ? "running" : "stopped"
+    status: statusOf(meta, p)
   };
 }
 async function readAllMeta(p) {
@@ -936,27 +953,26 @@ async function launch(spec, tap, cfg, pubkeyB64, p) {
     throw err;
   }
 }
-async function loadSnapshot(spec, snapshot, tap, cfg, p) {
+async function loadSnapshot(spec, image, tap, cfg, p) {
   const m = p.machine(spec.owner, spec.name);
-  const s = p.snapshot(snapshot.owner, snapshot.name);
   const pid = await spawnFirecracker(spec, cfg, p);
   try {
     const fc = new FirecrackerClient(m.sock);
     try {
       try {
         await fc.snapshotLoad({
-          snapshotPath: s.vmstate,
-          memFilePath: s.mem,
+          snapshotPath: image.vmstate,
+          memFilePath: image.mem,
           tap: { ifaceId: "eth0", hostDevName: tap },
           vsockPath: m.vsock,
           trackDirtyPages: true,
           resume: false
         });
       } catch (err) {
-        if (/Unknown Vsock Device/.test(err.message)) throw new KvboxError(tooOldToRestore(shown(snapshot)));
+        if (/Unknown Vsock Device/.test(err.message)) throw new KvboxError(tooOldToRestore(image.label));
         throw err;
       }
-      if (await namesOverlayAbsolutely(s.vmstate)) await fc.patchDrivePath("overlay", m.overlay);
+      if (await namesOverlayAbsolutely(image.vmstate)) await fc.patchDrivePath("overlay", m.overlay);
     } finally {
       await fc.close();
     }
@@ -1060,14 +1076,12 @@ async function restoreMachine(fromText, owner, req, cfg, p) {
     const tap = await createTap(name, cfg, { detached: true });
     tapCreated = true;
     const spec = { name, owner, ip, mac, vcpu: snap.vcpu, memMib: snap.memMib, base: snap.base };
-    pid = await loadSnapshot(spec, from, tap, cfg, p);
-    const fc = new FirecrackerClient(m.sock);
-    try {
-      await fc.resume();
-    } finally {
-      await fc.close();
-    }
-    await introduce(spec, m.vsock, cfg, shown(from));
+    pid = await loadSnapshot(spec, { vmstate: s.vmstate, mem: s.mem, label: shown(from) }, tap, cfg, p);
+    await resumeVm(m.sock);
+    await introduce(spec, m.vsock, cfg, {
+      silent: `nothing answered on the restored box's control channel for 3 s; ${tooOldToRestore(shown(from))}`,
+      noAnswer: "the restored box was discarded. kvbox-agent inside it took the request but gave no answer; try again, and if it keeps happening look at the box's journal (kvbox exec <box> -- sudo journalctl -u kvbox-agent) on a box started from the same snapshot."
+    });
     await attachTap(name, cfg);
     const meta = { ...spec, pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: `snapshot:${refKey(from)}`, memSnapshots: 1 };
     await writeMachineMeta(meta, p);
@@ -1081,7 +1095,7 @@ async function restoreMachine(fromText, owner, req, cfg, p) {
   }
 }
 var AGENT_REPLY_MS = 2e4;
-async function introduce(spec, vsock, cfg, from) {
+async function introduce(spec, vsock, cfg, failure) {
   const id = { name: spec.name, ip: spec.ip, prefixLen: prefixLenOf(cfg.subnet), gateway: cfg.gateway, mac: spec.mac, time: Math.floor(Date.now() / 1e3) };
   const deadline = Date.now() + 3e3;
   for (; ; ) {
@@ -1096,31 +1110,43 @@ async function introduce(spec, vsock, cfg, from) {
         continue;
       }
       if (transient) throw new KvboxError(`${msg}
-  nothing answered on the restored box's control channel for 3 s; ${tooOldToRestore(from)}`);
+  ${failure.silent}`);
       throw new KvboxError(`${msg}
-  the restored box was discarded. kvbox-agent inside it took the request but gave no answer; try again, and if it keeps happening look at the box's journal (kvbox exec <box> -- sudo journalctl -u kvbox-agent) on a box started from the same snapshot.`);
+  ${failure.noAnswer}`);
     }
   }
 }
+async function resumeVm(sock) {
+  const fc = new FirecrackerClient(sock);
+  try {
+    await fc.resume();
+  } finally {
+    await fc.close();
+  }
+}
 async function shutdown(meta, cfg, p, graceMs) {
   const pid = meta.pid;
   if (pidAlive(pid)) {
     const fc = new FirecrackerClient(p.machine(meta.owner, meta.name).sock);
     try {
+      if (meta.paused) await fc.resume();
       await fc.sendCtrlAltDel();
     } catch {
       process.kill(pid, "SIGTERM");
     } finally {
       await fc.close();
     }
-    if (!await waitFor(() => !pidAlive(pid), graceMs, 100)) {
-      try {
-        process.kill(pid, "SIGKILL");
-      } catch {
-      }
-      await waitFor(() => !pidAlive(pid), 1e3, 50);
-    }
+    await waitDead(pid, graceMs);
+  }
+  await cleanupProcess(meta, cfg, p);
+}
+async function waitDead(pid, graceMs) {
+  if (!await waitFor(() => !pidAlive(pid), graceMs, 100)) {
+    killQuietly(pid);
+    await waitFor(() => !pidAlive(pid), 1e3, 50);
   }
+}
+async function cleanupProcess(meta, cfg, p) {
   await deleteTap(meta.name, cfg);
   const m = p.machine(meta.owner, meta.name);
   await rm(m.sock, { force: true });
@@ -1129,10 +1155,12 @@ async function shutdown(meta, cfg, p, graceMs) {
 async function stopMachine(ref, p = paths()) {
   const meta = await readMeta(ref, p);
   const cfg = await loadConfig(p);
-  if (!pidAlive(meta.pid)) throw new KvboxError(`box ${shown(ref)} is already stopped`);
+  const m = p.machine(ref.owner, ref.name);
+  if (!pidAlive(meta.pid) && !isHibernated(meta, p)) throw new KvboxError(`box ${shown(ref)} is already stopped`);
   await shutdown(meta, cfg, p, 1e4);
-  await rm(p.machine(ref.owner, ref.name).memBase, { force: true });
-  const stopped = { ...meta, pid: null, memSnapshots: void 0 };
+  await rm(m.hibernateDir, { recursive: true, force: true });
+  await rm(m.memBase, { force: true });
+  const stopped = { ...meta, pid: null, memSnapshots: void 0, paused: void 0 };
   await writeMachineMeta(stopped, p);
   recordState(stopped, "stopped", "stop", p);
   return toMachine(stopped, p);
@@ -1140,7 +1168,10 @@ async function stopMachine(ref, p = paths()) {
 async function startMachine(ref, p = paths()) {
   const meta = await readMeta(ref, p);
   const cfg = await loadConfig(p);
-  if (pidAlive(meta.pid)) throw new KvboxError(`box ${shown(ref)} is already running`);
+  if (pidAlive(meta.pid)) throw new KvboxError(`box ${shown(ref)} is already ${meta.paused ? "up, in standby (kvbox resume " + shown(ref) + ")" : "running"}`);
+  if (isHibernated(meta, p)) {
+    throw new KvboxError(`${stateMessage(ref, "hibernated")}: kvbox wake ${shown(ref)} continues where it left off; to boot fresh instead, kvbox stop ${shown(ref)} first (discards its memory)`);
+  }
   assertAssets(cfg);
   const pubkeyB64 = await readPubkeyB64(p);
   await ensureHostNetwork(cfg);
@@ -1156,21 +1187,125 @@ async function startMachine(ref, p = paths()) {
     throw err;
   }
 }
+async function pauseMachine(ref, p = paths()) {
+  const meta = await readMeta(ref, p);
+  const status = statusOf(meta, p);
+  if (status === "standby") throw new KvboxError(`${stateMessage(ref, status)} already`);
+  if (status === "hibernated") throw new KvboxError(`${stateMessage(ref, status)}; kvbox wake ${shown(ref)} brings it back`);
+  if (status === "stopped") throw new KvboxError(`${stateMessage(ref, status)}; only a running box can be paused`);
+  const fc = new FirecrackerClient(p.machine(ref.owner, ref.name).sock);
+  try {
+    await fc.pause();
+  } finally {
+    await fc.close();
+  }
+  const paused = { ...meta, paused: true };
+  await writeMachineMeta(paused, p);
+  recordState(paused, "standby", "pause", p);
+  return toMachine(paused, p);
+}
+async function resumeMachine(ref, p = paths()) {
+  const meta = await readMeta(ref, p);
+  const status = statusOf(meta, p);
+  if (status === "running") throw new KvboxError(`${stateMessage(ref, status)} already`);
+  if (status === "hibernated") throw new KvboxError(`${stateMessage(ref, status)}, not in standby; kvbox wake ${shown(ref)} brings it back`);
+  if (status === "stopped") throw new KvboxError(`${stateMessage(ref, status)}, not in standby; kvbox start ${shown(ref)} boots it`);
+  await resumeVm(p.machine(ref.owner, ref.name).sock);
+  const running = { ...meta, paused: void 0 };
+  await writeMachineMeta(running, p);
+  recordState(running, "running", "resume", p);
+  return toMachine(running, p);
+}
+async function hibernateMachine(ref, p = paths()) {
+  const meta = await readMeta(ref, p);
+  const cfg = await loadConfig(p);
+  const status = statusOf(meta, p);
+  if (status === "hibernated") throw new KvboxError(`${stateMessage(ref, status)} already`);
+  if (status === "stopped") throw new KvboxError(`${stateMessage(ref, status)}; only a running box can be hibernated (there is no memory to keep)`);
+  if (meta.base === void 0) throw new KvboxError(`box ${shown(ref)} was made before overlays and runs on its own disk copy; it cannot be hibernated`);
+  const m = p.machine(ref.owner, ref.name);
+  await rm(m.hibernateDir, { recursive: true, force: true });
+  await mkdir3(m.hibernateDir, { recursive: true, mode: 448 });
+  const fc = new FirecrackerClient(m.sock);
+  try {
+    try {
+      await fc.pause();
+      await fc.snapshotCreate(m.hibernateVmstate, m.hibernateMem, "Full");
+    } catch (err) {
+      await rm(m.hibernateDir, { recursive: true, force: true });
+      if (!meta.paused) await fc.resume().catch(() => void 0);
+      throw err;
+    }
+  } finally {
+    await fc.close();
+  }
+  killQuietly(meta.pid);
+  await waitFor(() => !pidAlive(meta.pid), 5e3, 50);
+  await cleanupProcess(meta, cfg, p);
+  await run(["fallocate", "--dig-holes", m.hibernateMem]);
+  await rm(m.memBase, { force: true });
+  const hibernated = { ...meta, pid: null, memSnapshots: void 0, paused: void 0 };
+  await writeMachineMeta(hibernated, p);
+  recordState(hibernated, "hibernated", "hibernate", p);
+  return toMachine(hibernated, p);
+}
+async function wakeMachine(ref, p = paths()) {
+  const meta = await readMeta(ref, p);
+  const cfg = await loadConfig(p);
+  const status = statusOf(meta, p);
+  if (status === "running" || status === "standby") throw new KvboxError(`${stateMessage(ref, status)}, not hibernated`);
+  if (status === "stopped") throw new KvboxError(`${stateMessage(ref, status)}, not hibernated; kvbox start ${shown(ref)} boots it`);
+  assertAssets(cfg);
+  if (meta.base !== void 0 && !existsSync3(p.baseImage(meta.base))) {
+    throw new KvboxError(`box ${shown(ref)} stacks on base image ${meta.base}, which is gone from ${p.assets}`);
+  }
+  const m = p.machine(ref.owner, ref.name);
+  await ensureHostNetwork(cfg);
+  const tap = await createTap(ref.name, cfg, { detached: true });
+  let pid;
+  try {
+    const spec = { name: meta.name, owner: meta.owner, ip: meta.ip, mac: meta.mac, vcpu: meta.vcpu, memMib: meta.memMib, base: meta.base };
+    pid = await loadSnapshot(spec, { vmstate: m.hibernateVmstate, mem: m.hibernateMem, label: `box ${shown(ref)}'s memory image` }, tap, cfg, p);
+    await resumeVm(m.sock);
+    await introduce(spec, m.vsock, cfg, {
+      silent: `nothing answered on the box's control channel for 3 s; it stays hibernated, try kvbox wake ${shown(ref)} again`,
+      noAnswer: `the box stays hibernated. kvbox-agent inside it took the request but gave no answer; try kvbox wake ${shown(ref)} again`
+    });
+    await attachTap(ref.name, cfg);
+  } catch (err) {
+    if (pid !== void 0) killQuietly(pid);
+    await deleteTap(ref.name, cfg).catch(() => void 0);
+    throw err;
+  }
+  await rm(m.memBase, { force: true });
+  await link(m.hibernateMem, m.memBase);
+  await rm(m.hibernateDir, { recursive: true, force: true });
+  const running = { ...meta, pid, memSnapshots: 1, paused: void 0 };
+  await writeMachineMeta(running, p);
+  recordState(running, "running", "wake", p);
+  return toMachine(running, p);
+}
 async function applyAtNextBoot(meta, changes, by, cfg, p, whileStopped) {
   const wasRunning = pidAlive(meta.pid);
   if (wasRunning) {
     await shutdown(meta, cfg, p, 1e4);
     await rm(p.machine(meta.owner, meta.name).memBase, { force: true });
   }
-  const updated = { ...meta, ...changes, pid: null, memSnapshots: void 0 };
+  const updated = { ...meta, ...changes, pid: null, memSnapshots: void 0, paused: void 0 };
   await writeMachineMeta(updated, p);
   if (whileStopped) await whileStopped();
   recordState(updated, "stopped", by, p);
   return wasRunning ? startMachine(meta, p) : toMachine(updated, p);
 }
+function assertNotHibernated(meta, p) {
+  if (isHibernated(meta, p)) {
+    throw new KvboxError(`${stateMessage(meta, "hibernated")}; its memory image was made with its current shape and base: kvbox wake ${shown(meta)} first`);
+  }
+}
 async function updateMachine(ref, p = paths()) {
   const meta = await readMeta(ref, p);
   const cfg = await loadConfig(p);
+  assertNotHibernated(meta, p);
   if (meta.base === void 0) {
     throw new KvboxError(`box ${shown(ref)} was made before overlays and runs on its own disk copy; it cannot be moved to another base image`);
   }
@@ -1183,6 +1318,7 @@ async function updateMachine(ref, p = paths()) {
 async function rescaleMachine(ref, req, p = paths()) {
   const meta = await readMeta(ref, p);
   const cfg = await loadConfig(p);
+  assertNotHibernated(meta, p);
   const disk = ownDisk(meta, p);
   const currentDiskGib = Math.round(statSync(disk).size / 1024 ** 3);
   const preset = req.size !== void 0 && isSizeName(req.size) ? SIZES[req.size] : void 0;
@@ -1229,7 +1365,7 @@ function computeUsage(events, w) {
   const sorted = [...events].sort((a, b) => a.at < b.at ? -1 : a.at > b.at ? 1 : 0);
   const from = Date.parse(w.from);
   const to = Date.parse(w.to);
-  const ms = { running: 0, stopped: 0 };
+  const ms = { running: 0, standby: 0, hibernated: 0, stopped: 0 };
   let vcpuHours = 0;
   let ramGibHours = 0;
   let diskGibHours = 0;
@@ -1240,14 +1376,17 @@ function computeUsage(events, w) {
     if (end <= start) return;
     const h = (end - start) / HOUR_MS;
     ms[e.state] += end - start;
-    if (e.state === "running") {
-      vcpuHours += h * e.vcpu;
-      ramGibHours += h * e.memMib / 1024;
-    }
+    if (e.state === "running") vcpuHours += h * e.vcpu;
+    if (e.state === "running" || e.state === "standby") ramGibHours += h * e.memMib / 1024;
     diskGibHours += h * e.diskGib;
   });
   const totals = {
-    seconds: { running: Math.round(ms.running / 1e3), stopped: Math.round(ms.stopped / 1e3) },
+    seconds: {
+      running: Math.round(ms.running / 1e3),
+      standby: Math.round(ms.standby / 1e3),
+      hibernated: Math.round(ms.hibernated / 1e3),
+      stopped: Math.round(ms.stopped / 1e3)
+    },
     vcpuHours: round(vcpuHours, 3),
     ramGibHours: round(ramGibHours, 3),
     diskGibHours: round(diskGibHours, 3),
@@ -1307,7 +1446,7 @@ function versionOf(srcFileUrl, injected) {
 }
 
 // src/version.ts
-var VERSION = versionOf(import.meta.url, true ? "0.0.17" : void 0);
+var VERSION = versionOf(import.meta.url, true ? "0.1.1" : void 0);
 
 // src/caller.ts
 function resolveCaller() {
@@ -1332,6 +1471,7 @@ import { Hono as Hono2 } from "hono";
 // ../shared/src/api.ts
 import { z as z3 } from "zod";
 var SizeSchema = z3.enum(SIZE_NAMES);
+var BoxStatusSchema = z3.enum(["running", "standby", "hibernated", "stopped"]);
 var MachineSchema = z3.object({
   name: z3.string(),
   /** Who the box belongs to; names are unique per owner. A bare name is local to the host, `user@control-plane` later. */
@@ -1350,7 +1490,7 @@ var MachineSchema = z3.object({
   createdAt: z3.string(),
   source: z3.string(),
   // "base" or "snapshot:<name>"
-  status: z3.enum(["running", "stopped"])
+  status: BoxStatusSchema
 });
 var CreateMachineRequest = z3.object({
   name: z3.string().optional(),
@@ -1402,9 +1542,15 @@ var HealthSchema = z3.object({
 var MachineUsageSchema = z3.object({
   name: z3.string(),
   owner: z3.string(),
-  status: z3.enum(["running", "stopped"]),
+  status: BoxStatusSchema,
   window: z3.object({ from: z3.string(), to: z3.string() }),
-  seconds: z3.object({ running: z3.number().int(), stopped: z3.number().int() }),
+  /** standby and hibernated default to 0 so a newer CLI reads an older server's answer. */
+  seconds: z3.object({
+    running: z3.number().int(),
+    standby: z3.number().int().default(0),
+    hibernated: z3.number().int().default(0),
+    stopped: z3.number().int()
+  }),
   vcpuHours: z3.number(),
   ramGibHours: z3.number(),
   diskGibHours: z3.number(),
@@ -1427,6 +1573,10 @@ function machineRoutes() {
   });
   r.post("/:name/stop", async (c) => c.json(await stopMachine(parseRef(c.req.param("name"), c.get("owner")))));
   r.post("/:name/start", async (c) => c.json(await startMachine(parseRef(c.req.param("name"), c.get("owner")))));
+  r.post("/:name/pause", async (c) => c.json(await pauseMachine(parseRef(c.req.param("name"), c.get("owner")))));
+  r.post("/:name/resume", async (c) => c.json(await resumeMachine(parseRef(c.req.param("name"), c.get("owner")))));
+  r.post("/:name/hibernate", async (c) => c.json(await hibernateMachine(parseRef(c.req.param("name"), c.get("owner")))));
+  r.post("/:name/wake", async (c) => c.json(await wakeMachine(parseRef(c.req.param("name"), c.get("owner")))));
   r.post("/:name/update", async (c) => c.json(await updateMachine(parseRef(c.req.param("name"), c.get("owner")))));
   r.post("/:name/rescale", async (c) => {
     const body = RescaleMachineRequest.parse(await c.req.json().catch(() => ({})));
@@ -1504,6 +1654,9 @@ async function saveSnapshot(req, owner, p = paths()) {
     throw new KvboxError(`box ${req.box} was made before overlays and runs on its own disk copy; snapshots need a box made by this version of kvbox (kvbox new)`);
   }
   if (!pidAlive(box.pid)) {
+    if (existsSync4(p.machine(box.owner, box.name).hibernateVmstate)) {
+      throw new KvboxError(`box ${req.box} is hibernated; a snapshot freezes a running box (kvbox wake ${req.box} first)`);
+    }
     throw new KvboxError(`box ${req.box} is not running; a snapshot freezes a running box (kvbox start ${req.box} first)`);
   }
   const s = p.snapshot(owner, req.name);
@@ -1526,7 +1679,7 @@ async function saveSnapshot(req, owner, p = paths()) {
     } catch (err) {
       failure = err;
     } finally {
-      await fc.resume();
+      if (!box.paused) await fc.resume();
     }
     if (lineageTouched) {
       await writeMachineMeta({ ...box, memSnapshots: since + 1 }, p);