@kvbox/server 0.5.8 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -292,6 +292,8 @@ var ConfigSchema = z2.object({
292
292
  /** Who boxes made over the local socket belong to: the Unix user running the server unless set here. No `@`, no `/`. */
293
293
  owner: z2.string().regex(LOCAL_OWNER_RE, "owner: letters, digits, '.', '_', '-'; no '@' or '/'").default(localOwner),
294
294
  pricing: PricingSchema.optional(),
295
+ /** What a new box gets when `new` does not say (#47): seconds without network traffic before it is hibernated; 0: never. */
296
+ autoHibernateSec: z2.number().int().min(0).default(0),
295
297
  /** Absent until the host joins a control plane; then the server also listens on HTTPS. */
296
298
  controlPlane: ControlPlaneSchema.optional(),
297
299
  /** Absent unless the host sits behind a relay. */
@@ -370,7 +372,9 @@ var MachineSchema = z3.object({
370
372
  */
371
373
  ssh: z3.object({ host: z3.string(), port: z3.number().int(), user: z3.string() }).optional(),
372
374
  /** Whether the owner slug is a user or an org. Only the control plane knows; it fills it in on `GET /machines`. */
373
- ownerKind: z3.enum(["user", "org"]).optional()
375
+ ownerKind: z3.enum(["user", "org"]).optional(),
376
+ /** Seconds without network traffic before the host hibernates the box (#47). Absent: never. */
377
+ autoHibernateSec: z3.number().int().positive().optional()
374
378
  });
375
379
  var CreateMachineRequest = z3.object({
376
380
  name: z3.string().optional(),
@@ -384,7 +388,9 @@ var CreateMachineRequest = z3.object({
384
388
  * its SSH CA as one `cert-authority …` line (#51). Written into the box's authorized_keys at every
385
389
  * boot (#49); a host older than that ignores them.
386
390
  */
387
- sshKeys: z3.array(z3.string()).optional()
391
+ sshKeys: z3.array(z3.string()).optional(),
392
+ /** Seconds without network traffic before the box is hibernated (#47); 0: never. Absent: the host's default. */
393
+ autoHibernateSec: z3.number().int().min(0).optional()
388
394
  });
389
395
  var SetSshKeysRequest = z3.object({
390
396
  sshKeys: z3.array(z3.string())
@@ -395,6 +401,10 @@ var RescaleMachineRequest = z3.object({
395
401
  memMib: z3.number().int().min(256).optional(),
396
402
  diskGib: z3.number().int().min(1).optional()
397
403
  });
404
+ var SetMachineRequest = z3.object({
405
+ /** Seconds without network traffic before the box is hibernated (#47); 0: never. */
406
+ autoHibernateSec: z3.number().int().min(0).optional()
407
+ });
398
408
  var SnapshotSchema = z3.object({
399
409
  name: z3.string(),
400
410
  owner: z3.string(),
@@ -419,6 +429,12 @@ var CapacitySchema = z3.object({
419
429
  reservedVcpu: z3.number().int(),
420
430
  running: z3.number().int()
421
431
  });
432
+ var RecoveryReportSchema = z3.object({
433
+ at: z3.string(),
434
+ booted: z3.array(z3.string()),
435
+ leftStopped: z3.array(z3.string()),
436
+ failed: z3.array(z3.object({ box: z3.string(), error: z3.string() }))
437
+ });
422
438
  var HealthSchema = z3.object({
423
439
  ok: z3.literal(true),
424
440
  version: z3.string(),
@@ -441,8 +457,13 @@ var HealthSchema = z3.object({
441
457
  /** The public IP the relay owns for this host. */
442
458
  publicIp: z3.string(),
443
459
  /** The relay's WireGuard endpoint the tunnel dials. */
444
- endpoint: z3.string()
445
- }).nullable().optional()
460
+ endpoint: z3.string(),
461
+ /** Times the server redialed the tunnel since it started, because the relay stopped answering. Older servers omit them. */
462
+ restarts: z3.number().int().optional(),
463
+ lastRestartAt: z3.string().nullable().optional()
464
+ }).nullable().optional(),
465
+ /** Boxes brought back at the server's last start; null when there were none or it is still at it. Older servers omit it. */
466
+ recovery: RecoveryReportSchema.nullable().optional()
446
467
  });
447
468
  var MachineUsageSchema = z3.object({
448
469
  name: z3.string(),
@@ -863,250 +884,124 @@ async function requireKvm(device = KVM_DEVICE, groupFile = "/etc/group") {
863
884
  );
864
885
  }
865
886
 
866
- // src/core/network.ts
867
- import { randomInt } from "node:crypto";
868
- import { existsSync } from "node:fs";
869
- import { fileURLToPath } from "node:url";
870
- var TAP_PREFIX = "kv-";
871
- var PRIVILEGED_SCRIPT = "/usr/local/lib/kvbox/net-setup.sh";
872
- function tapName(box) {
873
- return `${TAP_PREFIX}${box}`.slice(0, 15);
874
- }
875
- function netSetupScript() {
876
- const override = process.env["KVBOX_NET_SETUP"];
877
- if (override) return override;
878
- if (existsSync(PRIVILEGED_SCRIPT)) return PRIVILEGED_SCRIPT;
879
- for (const rel of ["../scripts/net-setup.sh", "../../scripts/net-setup.sh"]) {
880
- const candidate = fileURLToPath(new URL(rel, import.meta.url));
881
- if (existsSync(candidate)) return candidate;
882
- }
883
- throw new KvboxError("net-setup.sh not found next to kvbox-server");
884
- }
885
- function bridgeExists(bridge) {
886
- return existsSync(`/sys/class/net/${bridge}`);
887
- }
888
- function netRun(cfg, args) {
889
- return run([netSetupScript(), ...args], {
890
- env: { ...process.env, KVBOX_BRIDGE: cfg.bridge, KVBOX_SUBNET: cfg.subnet, KVBOX_GATEWAY: cfg.gateway }
891
- }).catch((err) => {
892
- throw new KvboxError(
893
- `${err.message}
894
- (root is needed for host networking; install-server.sh sets up a sudoers rule for ${PRIVILEGED_SCRIPT})`
895
- );
896
- });
897
- }
898
- async function netSetup(cfg, args) {
899
- await netRun(cfg, args);
900
- }
901
- async function ensureHostNetwork(cfg) {
902
- if (bridgeExists(cfg.bridge)) return "present";
903
- await netRun(cfg, ["up"]);
904
- return "created";
905
- }
906
- async function createTap(box, cfg, opts = {}) {
907
- const tap = tapName(box);
908
- await netRun(cfg, ["tap", "add", tap, ...opts.detached ? ["detached"] : []]);
909
- return tap;
910
- }
911
- async function attachTap(box, cfg) {
912
- await netRun(cfg, ["tap", "attach", tapName(box)]);
913
- }
914
- async function deleteTap(box, cfg) {
915
- await netRun(cfg, ["tap", "del", tapName(box)]);
916
- }
917
- var SSH_PORT_MIN = 2e4;
918
- var SSH_PORT_MAX = 6e4;
919
- function pickSshPort(taken) {
920
- const span = SSH_PORT_MAX - SSH_PORT_MIN + 1;
921
- for (let i = 0; i < 32; i++) {
922
- const port = SSH_PORT_MIN + randomInt(span);
923
- if (!taken.has(port)) return port;
924
- }
925
- const start = randomInt(span);
926
- for (let i = 0; i < span; i++) {
927
- const port = SSH_PORT_MIN + (start + i) % span;
928
- if (!taken.has(port)) return port;
929
- }
930
- throw new KvboxError(`no free port left between ${SSH_PORT_MIN} and ${SSH_PORT_MAX} on this host`);
931
- }
932
- function parseListeningPorts(ssOutput) {
933
- const ports = /* @__PURE__ */ new Set();
934
- for (const line of ssOutput.split("\n")) {
935
- const cols = line.trim().split(/\s+/);
936
- const local = cols[3];
937
- if (!local) continue;
938
- const port = Number(local.slice(local.lastIndexOf(":") + 1));
939
- if (Number.isInteger(port) && port > 0) ports.add(port);
940
- }
941
- return ports;
942
- }
943
- async function listeningPorts() {
944
- try {
945
- return parseListeningPorts((await run(["ss", "-ltnH"])).stdout);
946
- } catch {
947
- return /* @__PURE__ */ new Set();
948
- }
949
- }
950
- var proxyArgs = (p) => [p.box, p.boxIp, String(p.hostPort), String(p.guestPort), p.proto ?? "tcp"];
951
- async function openProxy(proxy, cfg) {
952
- await netRun(cfg, ["proxy", "add", ...proxyArgs(proxy)]);
953
- }
954
- async function closeProxy(proxy, cfg) {
955
- await netRun(cfg, ["proxy", "rm", ...proxyArgs(proxy)]);
956
- }
957
- var ipToInt = (ip) => ip.split(".").reduce((acc, o) => (acc << 8) + Number(o), 0) >>> 0;
958
- var intToIp = (n) => [24, 16, 8, 0].map((s) => n >>> s & 255).join(".");
959
- function nextFreeIp(subnet, inUse) {
960
- const [base, lenStr] = subnet.split("/");
961
- const len = Number(lenStr);
962
- if (!base || !Number.isInteger(len) || len < 8 || len > 30) throw new KvboxError(`bad subnet: ${subnet}`);
963
- const mask = len === 0 ? 0 : ~0 << 32 - len >>> 0;
964
- const network = (ipToInt(base) & mask) >>> 0;
965
- const broadcast = (network | ~mask >>> 0) >>> 0;
966
- const used = new Set(inUse.map(ipToInt));
967
- for (let n = network + 2; n < broadcast; n++) {
968
- if (!used.has(n)) return intToIp(n);
969
- }
970
- throw new KvboxError(`no free IP left in ${subnet}`);
971
- }
972
- function macFor(ip) {
973
- const hex = ip.split(".").map((o) => Number(o).toString(16).padStart(2, "0").toUpperCase());
974
- return ["06", "00", ...hex].join(":");
975
- }
976
-
977
- // src/core/relay.ts
978
- import { chmod as chmod3, mkdir as mkdir2, rm, writeFile as writeFile2 } from "node:fs/promises";
887
+ // src/core/machines.ts
888
+ import { spawn } from "node:child_process";
889
+ import { closeSync as closeSync2, existsSync as existsSync2, openSync as openSync2, statSync } from "node:fs";
890
+ import { link, mkdir as mkdir3, readFile as readFile5, realpath, rm, truncate, writeFile as writeFile3 } from "node:fs/promises";
891
+ import { basename } from "node:path";
892
+ import { setTimeout as sleep } from "node:timers/promises";
979
893
 
980
- // ../shared/src/relay.ts
981
- import { z as z4 } from "zod";
982
- var RELAY_TUNNEL_GATEWAY = "10.77.0.1";
983
- var IPV4_RE = /^(\d{1,3})(\.\d{1,3}){3}$/;
984
- var WG_KEY_RE = /^[A-Za-z0-9+/]{43}=$/;
985
- var RelayJoinSchema = z4.object({
986
- v: z4.literal(1),
987
- /** How the relay lists this host. */
988
- name: z4.string().regex(/^[a-z0-9][a-z0-9-]{0,30}$/),
989
- /** `<relay-primary-ip>:<port>`, the WireGuard endpoint the host dials. */
990
- relay: z4.string().regex(/^(\d{1,3})(\.\d{1,3}){3}:\d{1,5}$/),
991
- /** The relay's WireGuard public key. */
992
- relayKey: z4.string().regex(WG_KEY_RE),
993
- /** This host's address inside the tunnel. */
994
- tunnelIp: z4.string().regex(IPV4_RE),
995
- /** This host's WireGuard private key, generated on the relay, kept nowhere else. */
996
- privateKey: z4.string().regex(WG_KEY_RE),
997
- /** The extra public IPv4 the relay owns for this host; what the host advertises to the control plane. */
998
- publicIp: z4.string().regex(IPV4_RE)
999
- });
1000
- function decodeRelayJoin(text) {
894
+ // src/core/agent.ts
895
+ import { connect } from "node:net";
896
+ var AGENT_PORT = 52;
897
+ async function agentCall(udsPath, req, timeoutMs = 5e3) {
898
+ const reply = await new Promise((resolve, reject) => {
899
+ const sock = connect(udsPath);
900
+ let buf = "";
901
+ let handshaken = false;
902
+ const fail = (err) => {
903
+ sock.destroy();
904
+ reject(err);
905
+ };
906
+ sock.setTimeout(timeoutMs, () => fail(new KvboxError(`kvbox-agent did not answer within ${timeoutMs} ms`)));
907
+ sock.once("error", (err) => fail(new KvboxError(`kvbox-agent unreachable: ${err.message}`)));
908
+ sock.once("connect", () => sock.write(`CONNECT ${AGENT_PORT}
909
+ `));
910
+ sock.on("data", (chunk) => {
911
+ buf += chunk.toString("utf8");
912
+ let nl;
913
+ while ((nl = buf.indexOf("\n")) !== -1) {
914
+ const line = buf.slice(0, nl);
915
+ buf = buf.slice(nl + 1);
916
+ if (!handshaken) {
917
+ if (!line.startsWith("OK ")) return fail(new KvboxError(`vsock connect refused: ${line}`));
918
+ handshaken = true;
919
+ sock.write(JSON.stringify(req) + "\n");
920
+ } else {
921
+ sock.end();
922
+ return resolve(line);
923
+ }
924
+ }
925
+ });
926
+ sock.once("close", () => {
927
+ if (!handshaken) fail(new KvboxError("vsock connection closed before the guest accepted"));
928
+ else fail(new KvboxError("kvbox-agent closed the connection without a reply"));
929
+ });
930
+ });
1001
931
  let parsed;
1002
932
  try {
1003
- parsed = JSON.parse(Buffer.from(text.trim(), "base64url").toString("utf8"));
933
+ parsed = JSON.parse(reply);
1004
934
  } catch {
1005
- return null;
935
+ throw new KvboxError(`kvbox-agent sent something that is not JSON: ${reply}`);
1006
936
  }
1007
- const res = RelayJoinSchema.safeParse(parsed);
1008
- return res.success ? res.data : null;
1009
- }
1010
-
1011
- // src/core/relay.ts
1012
- function relayConf(join2) {
1013
- return [
1014
- "# kvbox relay tunnel: made by `kvbox server relay`, installed by net-setup.sh. Do not edit.",
1015
- "[Interface]",
1016
- `PrivateKey = ${join2.privateKey}`,
1017
- `Address = ${join2.tunnelIp}/32`,
1018
- "",
1019
- "[Peer]",
1020
- `PublicKey = ${join2.relayKey}`,
1021
- `Endpoint = ${join2.relay}`,
1022
- // Only the relay's own tunnel address goes through the tunnel: the host keeps its normal internet
1023
- // route, and the relay SNATs forwarded connections so their replies come back this way.
1024
- `AllowedIPs = ${RELAY_TUNNEL_GATEWAY}/32`,
1025
- // The router forgets an idle outbound mapping; a packet every 25 s keeps the relay able to reach us.
1026
- "PersistentKeepalive = 25",
1027
- ""
1028
- ].join("\n");
937
+ if (!parsed.ok) throw new KvboxError(`kvbox-agent: ${parsed.error}`);
938
+ return parsed;
1029
939
  }
1030
- async function useRelay(joinString, p = paths()) {
1031
- const join2 = decodeRelayJoin(joinString);
1032
- if (!join2) throw new KvboxError("that is not a relay join string; copy the whole line `kvbox-relay add` printed on the relay");
1033
- const cfg = await loadConfig(p);
1034
- if (cfg.relay && cfg.relay.name !== join2.name) {
1035
- throw new KvboxError(`this host already goes through relay ${cfg.relay.name} (${cfg.relay.publicIp}); run kvbox server relay off first`);
1036
- }
1037
- if (cfg.relay && cfg.relay.publicIp !== join2.publicIp && cfg.controlPlane?.advertise === cfg.relay.publicIp) {
1038
- throw new KvboxError(
1039
- `${cfg.controlPlane.name} reaches this host at ${cfg.relay.publicIp}, not ${join2.publicIp}; run kvbox server leave first, then join again`
1040
- );
1041
- }
1042
- await mkdir2(p.relay, { recursive: true, mode: 448 });
1043
- await writeFile2(p.relayConf, relayConf(join2), { mode: 384 });
1044
- await chmod3(p.relayConf, 384);
1045
- await netSetup(cfg, ["relay", "up", p.relayConf, String(cfg.controlPlane?.port ?? DEFAULT_HTTPS_PORT)]);
1046
- const relay = { name: join2.name, publicIp: join2.publicIp, tunnelIp: join2.tunnelIp, endpoint: join2.relay };
1047
- await saveConfig({ ...cfg, relay }, p);
1048
- return relay;
940
+ function setIdentity(udsPath, id, timeoutMs) {
941
+ return agentCall(udsPath, { op: "identity", ...id }, timeoutMs);
1049
942
  }
1050
- async function dropRelay(p = paths()) {
1051
- const cfg = await loadConfig(p);
1052
- const relay = cfg.relay;
1053
- if (!relay) throw new KvboxError("this host does not go through a relay");
1054
- if (cfg.controlPlane?.advertise === relay.publicIp) {
1055
- throw new KvboxError(`${cfg.controlPlane.name} reaches this host through the relay (${relay.publicIp}); run kvbox server leave first`);
1056
- }
1057
- await netSetup(cfg, ["relay", "down"]);
1058
- await rm(p.relayConf, { force: true });
1059
- const { relay: _gone, ...rest } = cfg;
1060
- await saveConfig(rest, p);
1061
- return relay;
943
+ function setAuthorizedKeys(udsPath, lines, timeoutMs) {
944
+ return agentCall(udsPath, { op: "authorizedKeys", lines: [...lines] }, timeoutMs);
1062
945
  }
946
+ var isUnknownOp = (err) => /unknown op/.test(err.message ?? "");
1063
947
 
1064
- // src/listen.ts
1065
- import { readFile as readFile6, rm as rm4, writeFile as writeFile5 } from "node:fs/promises";
1066
- import { createServer as createHttpsServer } from "node:https";
1067
- import { createAdaptorServer } from "@hono/node-server";
948
+ // src/core/capacity.ts
949
+ import { cpus, totalmem } from "node:os";
950
+
951
+ // ../shared/src/capacity.ts
952
+ var MEM_HEADROOM_MIB = 2048;
953
+ var VCPU_OVERSUBSCRIPTION = 2;
1068
954
 
1069
955
  // src/core/state.ts
1070
956
  import { randomBytes } from "node:crypto";
1071
- import { mkdir as mkdir3, readFile as readFile4, readdir, writeFile as writeFile3 } from "node:fs/promises";
1072
- import { z as z5 } from "zod";
1073
- var MachineMetaSchema = z5.object({
957
+ import { readFileSync } from "node:fs";
958
+ import { mkdir as mkdir2, readFile as readFile4, readdir, writeFile as writeFile2 } from "node:fs/promises";
959
+ import { z as z4 } from "zod";
960
+ var MachineMetaSchema = z4.object({
1074
961
  /** Host-unique, minted at `kvbox new` (fresh or restored) and kept for the box's life; the tap is `kv-<id>`. */
1075
- id: z5.string(),
1076
- name: z5.string(),
962
+ id: z4.string(),
963
+ name: z4.string(),
1077
964
  /** Part of the box's identity: machines/<owner>/<name>. See refs.ts. */
1078
- owner: z5.string(),
1079
- ip: z5.string(),
1080
- mac: z5.string(),
1081
- pid: z5.number().int().nullable(),
965
+ owner: z4.string(),
966
+ ip: z4.string(),
967
+ mac: z4.string(),
968
+ pid: z4.number().int().nullable(),
1082
969
  // null: stopped on purpose, disk kept
1083
- vcpu: z5.number().int(),
1084
- memMib: z5.number().int(),
1085
- createdAt: z5.string(),
1086
- source: z5.string(),
970
+ /**
971
+ * The host boot `pid` belongs to (/proc/sys/kernel/random/boot_id at launch, #29). After a reboot
972
+ * the number can name some other process; a pid from another boot is dead whatever `kill -0` says.
973
+ * Absent (boxes launched before #29): the pid alone decides.
974
+ */
975
+ bootId: z4.string().optional(),
976
+ vcpu: z4.number().int(),
977
+ memMib: z4.number().int(),
978
+ createdAt: z4.string(),
979
+ source: z4.string(),
1087
980
  /** Base image version (assets/rootfs-<version>.ext4) the box's overlay stacks on. */
1088
- base: z5.string(),
981
+ base: z4.string(),
1089
982
  /**
1090
983
  * Snapshots taken since this VM started (boot or restore). Firecracker resets its dirty-page bitmap at
1091
984
  * each one, so when this is > 0 the next diff is only standalone merged onto `mem.base`; if that file
1092
985
  * is missing the lineage is broken and the next snapshot has to be a full dump. Absent: 0.
1093
986
  */
1094
- memSnapshots: z5.number().int().optional(),
987
+ memSnapshots: z4.number().int().optional(),
1095
988
  /** `kvbox pause`: Firecracker is alive but the VM is frozen (standby). Absent: not paused. */
1096
- paused: z5.boolean().optional(),
989
+ paused: z4.boolean().optional(),
1097
990
  /**
1098
991
  * authorized_keys lines the box gets on top of the host's own key, as the control plane sent them
1099
992
  * at `new` (#49). kvbox-init rewrites authorized_keys from the kernel cmdline at every boot, so
1100
993
  * `start` has to send the same lines again. Absent: host key only.
1101
994
  */
1102
- sshKeys: z5.array(z5.string()).optional(),
995
+ sshKeys: z4.array(z4.string()).optional(),
1103
996
  /**
1104
997
  * The guest's authorized_keys may not match `sshKeys`: they changed while the box was not running,
1105
998
  * or its agent could not take them (#55, #63). Cleared by a push through kvbox-agent or a boot.
1106
999
  */
1107
- sshKeysStale: z5.boolean().optional(),
1000
+ sshKeysStale: z4.boolean().optional(),
1108
1001
  /** Public port on the host DNATed to the box's SSH while it runs (#49); fixed for the box's life. Absent: none. */
1109
- sshPort: z5.number().int().optional()
1002
+ sshPort: z4.number().int().optional(),
1003
+ /** Seconds without traffic on the box's tap before the server hibernates it (#47, core/idle.ts). Absent: never. */
1004
+ autoHibernateSec: z4.number().int().positive().optional()
1110
1005
  });
1111
1006
  function newBoxId(taken) {
1112
1007
  for (; ; ) {
@@ -1114,20 +1009,20 @@ function newBoxId(taken) {
1114
1009
  if (!taken.has(id)) return id;
1115
1010
  }
1116
1011
  }
1117
- var SnapshotMetaSchema = z5.object({
1118
- name: z5.string(),
1119
- owner: z5.string(),
1012
+ var SnapshotMetaSchema = z4.object({
1013
+ name: z4.string(),
1014
+ owner: z4.string(),
1120
1015
  /** The box it was taken from, as owner/name. */
1121
- source: z5.string(),
1122
- base: z5.string(),
1123
- vcpu: z5.number().int(),
1124
- memMib: z5.number().int(),
1125
- sizeBytes: z5.number().int(),
1126
- createdAt: z5.string()
1016
+ source: z4.string(),
1017
+ base: z4.string(),
1018
+ vcpu: z4.number().int(),
1019
+ memMib: z4.number().int(),
1020
+ sizeBytes: z4.number().int(),
1021
+ createdAt: z4.string()
1127
1022
  });
1128
1023
  async function ensureLayout(p = paths()) {
1129
1024
  for (const dir of [p.home, p.ssh, p.bin, p.assets, p.machines, p.snapshots]) {
1130
- await mkdir3(dir, { recursive: true, mode: 448 });
1025
+ await mkdir2(dir, { recursive: true, mode: 448 });
1131
1026
  }
1132
1027
  return p;
1133
1028
  }
@@ -1148,8 +1043,8 @@ async function readMachineMeta(ref, p = paths()) {
1148
1043
  }
1149
1044
  async function writeMachineMeta(meta, p = paths()) {
1150
1045
  const m = p.machine(meta.owner, meta.name);
1151
- await mkdir3(m.dir, { recursive: true });
1152
- await writeFile3(m.meta, JSON.stringify(meta, null, 2) + "\n");
1046
+ await mkdir2(m.dir, { recursive: true });
1047
+ await writeFile2(m.meta, JSON.stringify(meta, null, 2) + "\n");
1153
1048
  }
1154
1049
  async function listMachineRefs(p = paths()) {
1155
1050
  return listRefs(p.machines);
@@ -1159,8 +1054,8 @@ async function readSnapshotMeta(ref, p = paths()) {
1159
1054
  }
1160
1055
  async function writeSnapshotMeta(meta, p = paths()) {
1161
1056
  const s = p.snapshot(meta.owner, meta.name);
1162
- await mkdir3(s.dir, { recursive: true });
1163
- await writeFile3(s.meta, JSON.stringify(meta, null, 2) + "\n");
1057
+ await mkdir2(s.dir, { recursive: true });
1058
+ await writeFile2(s.meta, JSON.stringify(meta, null, 2) + "\n");
1164
1059
  }
1165
1060
  async function listSnapshotRefs(p = paths()) {
1166
1061
  return listRefs(p.snapshots);
@@ -1183,131 +1078,29 @@ function pidAlive(pid) {
1183
1078
  return err.code === "EPERM";
1184
1079
  }
1185
1080
  }
1186
-
1187
- // src/app.ts
1188
- import { Hono as Hono6 } from "hono";
1189
-
1190
- // ../shared/src/version.ts
1191
- import { createRequire } from "node:module";
1192
- function versionOf(srcFileUrl, injected) {
1193
- if (typeof injected === "string") return injected;
1194
- try {
1195
- const require2 = createRequire(srcFileUrl);
1196
- return require2("../package.json").version;
1197
- } catch {
1198
- return "0.0.0-dev";
1199
- }
1200
- }
1201
-
1202
- // src/version.ts
1203
- var VERSION = versionOf(import.meta.url, true ? "0.5.8" : void 0);
1204
-
1205
- // src/caller.ts
1206
- import { createHash as createHash2, timingSafeEqual } from "node:crypto";
1207
- var OWNER_HEADER = "x-kvbox-owner";
1208
- function resolveCaller(opts = {}) {
1209
- const cp = opts.controlPlane;
1210
- if (!cp) {
1211
- return async (c, next) => {
1212
- const cfg = await loadConfig();
1213
- c.set("owner", cfg.owner);
1214
- c.set("listScope", cfg.owner);
1215
- await next();
1216
- };
1217
- }
1218
- const expected = digest(cp.secret);
1219
- return async (c, next) => {
1220
- const m = /^Bearer\s+(\S+)$/i.exec(c.req.header("authorization") ?? "");
1221
- if (!m || !timingSafeEqual(digest(m[1]), expected)) return c.json({ error: "not authorized" }, 401);
1222
- const user = c.req.header(OWNER_HEADER);
1223
- if (user === void 0) {
1224
- const path = c.req.path.replace(/\/$/, "");
1225
- const realmListing = c.req.method === "GET" && (path === "/machines" || path === "/snapshots");
1226
- if (!realmListing && /^\/(machines|snapshots)(\/|$)/.test(c.req.path)) {
1227
- return c.json({ error: "X-Kvbox-Owner header required" }, 400);
1228
- }
1229
- c.set("owner", "");
1230
- c.set("listScope", { realm: cp.name });
1231
- } else {
1232
- if (!LOCAL_OWNER_RE.test(user)) return c.json({ error: "X-Kvbox-Owner: letters, digits, '.', '_', '-'" }, 400);
1233
- const owner = `${user}@${cp.name}`;
1234
- c.set("owner", owner);
1235
- c.set("listScope", owner);
1081
+ var bootId;
1082
+ function currentBootId() {
1083
+ if (bootId === void 0) {
1084
+ try {
1085
+ bootId = { id: readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() };
1086
+ } catch {
1087
+ bootId = { id: void 0 };
1236
1088
  }
1237
- await next();
1238
- };
1089
+ }
1090
+ return bootId.id;
1239
1091
  }
1240
- function digest(secret) {
1241
- return createHash2("sha256").update(secret).digest();
1092
+ function vmAlive(meta) {
1093
+ if (meta.bootId !== void 0 && meta.bootId !== currentBootId()) return false;
1094
+ return pidAlive(meta.pid);
1242
1095
  }
1243
-
1244
- // src/core/machines.ts
1245
- import { spawn } from "node:child_process";
1246
- import { closeSync as closeSync2, existsSync as existsSync2, openSync as openSync2, statSync } from "node:fs";
1247
- import { link, mkdir as mkdir4, readFile as readFile5, realpath, rm as rm2, truncate, writeFile as writeFile4 } from "node:fs/promises";
1248
- import { basename } from "node:path";
1249
- import { setTimeout as sleep } from "node:timers/promises";
1250
-
1251
- // src/core/agent.ts
1252
- import { connect } from "node:net";
1253
- var AGENT_PORT = 52;
1254
- async function agentCall(udsPath, req, timeoutMs = 5e3) {
1255
- const reply = await new Promise((resolve, reject) => {
1256
- const sock = connect(udsPath);
1257
- let buf = "";
1258
- let handshaken = false;
1259
- const fail = (err) => {
1260
- sock.destroy();
1261
- reject(err);
1262
- };
1263
- sock.setTimeout(timeoutMs, () => fail(new KvboxError(`kvbox-agent did not answer within ${timeoutMs} ms`)));
1264
- sock.once("error", (err) => fail(new KvboxError(`kvbox-agent unreachable: ${err.message}`)));
1265
- sock.once("connect", () => sock.write(`CONNECT ${AGENT_PORT}
1266
- `));
1267
- sock.on("data", (chunk) => {
1268
- buf += chunk.toString("utf8");
1269
- let nl;
1270
- while ((nl = buf.indexOf("\n")) !== -1) {
1271
- const line = buf.slice(0, nl);
1272
- buf = buf.slice(nl + 1);
1273
- if (!handshaken) {
1274
- if (!line.startsWith("OK ")) return fail(new KvboxError(`vsock connect refused: ${line}`));
1275
- handshaken = true;
1276
- sock.write(JSON.stringify(req) + "\n");
1277
- } else {
1278
- sock.end();
1279
- return resolve(line);
1280
- }
1281
- }
1282
- });
1283
- sock.once("close", () => {
1284
- if (!handshaken) fail(new KvboxError("vsock connection closed before the guest accepted"));
1285
- else fail(new KvboxError("kvbox-agent closed the connection without a reply"));
1286
- });
1287
- });
1288
- let parsed;
1096
+ function hostBootTime() {
1289
1097
  try {
1290
- parsed = JSON.parse(reply);
1098
+ const m = /^btime (\d+)$/m.exec(readFileSync("/proc/stat", "utf8"));
1099
+ return m ? new Date(Number(m[1]) * 1e3) : null;
1291
1100
  } catch {
1292
- throw new KvboxError(`kvbox-agent sent something that is not JSON: ${reply}`);
1101
+ return null;
1293
1102
  }
1294
- if (!parsed.ok) throw new KvboxError(`kvbox-agent: ${parsed.error}`);
1295
- return parsed;
1296
- }
1297
- function setIdentity(udsPath, id, timeoutMs) {
1298
- return agentCall(udsPath, { op: "identity", ...id }, timeoutMs);
1299
- }
1300
- function setAuthorizedKeys(udsPath, lines, timeoutMs) {
1301
- return agentCall(udsPath, { op: "authorizedKeys", lines: [...lines] }, timeoutMs);
1302
1103
  }
1303
- var isUnknownOp = (err) => /unknown op/.test(err.message ?? "");
1304
-
1305
- // src/core/capacity.ts
1306
- import { cpus, totalmem } from "node:os";
1307
-
1308
- // ../shared/src/capacity.ts
1309
- var MEM_HEADROOM_MIB = 2048;
1310
- var VCPU_OVERSUBSCRIPTION = 2;
1311
1104
 
1312
1105
  // src/core/capacity.ts
1313
1106
  function hostResources() {
@@ -1318,7 +1111,7 @@ async function runningReservations(p = paths()) {
1318
1111
  for (const ref of await listMachineRefs(p)) {
1319
1112
  try {
1320
1113
  const meta = await readMachineMeta(ref, p);
1321
- if (pidAlive(meta.pid)) out.push({ name: meta.name, vcpu: meta.vcpu, memMib: meta.memMib });
1114
+ if (vmAlive(meta)) out.push({ name: meta.name, vcpu: meta.vcpu, memMib: meta.memMib });
1322
1115
  } catch {
1323
1116
  }
1324
1117
  }
@@ -1447,6 +1240,10 @@ function closeOpenRun(boxId, at, p = paths()) {
1447
1240
  insertRaw({ ...last, at, state: "stopped", by: "server" }, db);
1448
1241
  return true;
1449
1242
  }
1243
+ function closeRunCutByBoot(boxId, bootAt, now, p = paths()) {
1244
+ const last = lastEvent(boxId, openDb(p));
1245
+ return closeOpenRun(boxId, last !== void 0 && last.at < bootAt ? bootAt : now, p);
1246
+ }
1450
1247
  function usageEventsFor(boxId, since, p = paths()) {
1451
1248
  const rows = openDb(p).prepare("select * from usage_events where box_id = ? and at >= ? order by at, id").all(boxId, since);
1452
1249
  return rows.map(fromRaw);
@@ -1468,6 +1265,33 @@ function usageEventsInScope(scope, window, p = paths()) {
1468
1265
  return rows.filter((r) => inScope(r.owner, scope)).map(withId);
1469
1266
  }
1470
1267
 
1268
+ // src/core/locks.ts
1269
+ import { AsyncLocalStorage } from "node:async_hooks";
1270
+ var tails = /* @__PURE__ */ new Map();
1271
+ var held = new AsyncLocalStorage();
1272
+ async function withBoxLock(ref, fn) {
1273
+ const key = refKey(ref);
1274
+ const mine = held.getStore();
1275
+ if (mine?.has(key)) return fn();
1276
+ const previous = tails.get(key);
1277
+ let release;
1278
+ const tail = new Promise((resolve) => release = resolve);
1279
+ tails.set(key, tail);
1280
+ try {
1281
+ await previous;
1282
+ return await held.run(/* @__PURE__ */ new Set([...mine ?? [], key]), fn);
1283
+ } finally {
1284
+ release();
1285
+ if (tails.get(key) === tail) tails.delete(key);
1286
+ }
1287
+ }
1288
+ function boxBusy(ref) {
1289
+ return tails.has(refKey(ref));
1290
+ }
1291
+ function locked(fn) {
1292
+ return (...args) => withBoxLock(args[0], () => fn(...args));
1293
+ }
1294
+
1471
1295
  // src/core/firecracker.ts
1472
1296
  import { Agent, request as request2 } from "undici";
1473
1297
  var FirecrackerClient = class {
@@ -1569,13 +1393,124 @@ var FirecrackerClient = class {
1569
1393
  resume_vm: opts.resume
1570
1394
  });
1571
1395
  }
1572
- info() {
1573
- return this.call("GET", "/");
1396
+ info() {
1397
+ return this.call("GET", "/");
1398
+ }
1399
+ async close() {
1400
+ await this.agent.close();
1401
+ }
1402
+ };
1403
+
1404
+ // src/core/network.ts
1405
+ import { randomInt } from "node:crypto";
1406
+ import { existsSync } from "node:fs";
1407
+ import { fileURLToPath } from "node:url";
1408
+ var TAP_PREFIX = "kv-";
1409
+ var PRIVILEGED_SCRIPT = "/usr/local/lib/kvbox/net-setup.sh";
1410
+ function tapName(box) {
1411
+ return `${TAP_PREFIX}${box}`.slice(0, 15);
1412
+ }
1413
+ function netSetupScript() {
1414
+ const override = process.env["KVBOX_NET_SETUP"];
1415
+ if (override) return override;
1416
+ if (existsSync(PRIVILEGED_SCRIPT)) return PRIVILEGED_SCRIPT;
1417
+ for (const rel of ["../scripts/net-setup.sh", "../../scripts/net-setup.sh"]) {
1418
+ const candidate = fileURLToPath(new URL(rel, import.meta.url));
1419
+ if (existsSync(candidate)) return candidate;
1420
+ }
1421
+ throw new KvboxError("net-setup.sh not found next to kvbox-server");
1422
+ }
1423
+ function bridgeExists(bridge) {
1424
+ return existsSync(`/sys/class/net/${bridge}`);
1425
+ }
1426
+ function netRun(cfg, args) {
1427
+ return run([netSetupScript(), ...args], {
1428
+ env: { ...process.env, KVBOX_BRIDGE: cfg.bridge, KVBOX_SUBNET: cfg.subnet, KVBOX_GATEWAY: cfg.gateway }
1429
+ }).catch((err) => {
1430
+ throw new KvboxError(
1431
+ `${err.message}
1432
+ (root is needed for host networking; install-server.sh sets up a sudoers rule for ${PRIVILEGED_SCRIPT})`
1433
+ );
1434
+ });
1435
+ }
1436
+ async function netSetup(cfg, args) {
1437
+ await netRun(cfg, args);
1438
+ }
1439
+ async function ensureHostNetwork(cfg) {
1440
+ if (bridgeExists(cfg.bridge)) return "present";
1441
+ await netRun(cfg, ["up"]);
1442
+ return "created";
1443
+ }
1444
+ async function createTap(box, cfg, opts = {}) {
1445
+ const tap = tapName(box);
1446
+ await netRun(cfg, ["tap", "add", tap, ...opts.detached ? ["detached"] : []]);
1447
+ return tap;
1448
+ }
1449
+ async function attachTap(box, cfg) {
1450
+ await netRun(cfg, ["tap", "attach", tapName(box)]);
1451
+ }
1452
+ async function deleteTap(box, cfg) {
1453
+ await netRun(cfg, ["tap", "del", tapName(box)]);
1454
+ }
1455
+ var SSH_PORT_MIN = 2e4;
1456
+ var SSH_PORT_MAX = 6e4;
1457
+ function pickSshPort(taken) {
1458
+ const span = SSH_PORT_MAX - SSH_PORT_MIN + 1;
1459
+ for (let i = 0; i < 32; i++) {
1460
+ const port = SSH_PORT_MIN + randomInt(span);
1461
+ if (!taken.has(port)) return port;
1462
+ }
1463
+ const start = randomInt(span);
1464
+ for (let i = 0; i < span; i++) {
1465
+ const port = SSH_PORT_MIN + (start + i) % span;
1466
+ if (!taken.has(port)) return port;
1467
+ }
1468
+ throw new KvboxError(`no free port left between ${SSH_PORT_MIN} and ${SSH_PORT_MAX} on this host`);
1469
+ }
1470
+ function parseListeningPorts(ssOutput) {
1471
+ const ports = /* @__PURE__ */ new Set();
1472
+ for (const line of ssOutput.split("\n")) {
1473
+ const cols = line.trim().split(/\s+/);
1474
+ const local = cols[3];
1475
+ if (!local) continue;
1476
+ const port = Number(local.slice(local.lastIndexOf(":") + 1));
1477
+ if (Number.isInteger(port) && port > 0) ports.add(port);
1478
+ }
1479
+ return ports;
1480
+ }
1481
+ async function listeningPorts() {
1482
+ try {
1483
+ return parseListeningPorts((await run(["ss", "-ltnH"])).stdout);
1484
+ } catch {
1485
+ return /* @__PURE__ */ new Set();
1574
1486
  }
1575
- async close() {
1576
- await this.agent.close();
1487
+ }
1488
+ var proxyArgs = (p) => [p.box, p.boxIp, String(p.hostPort), String(p.guestPort), p.proto ?? "tcp"];
1489
+ async function openProxy(proxy, cfg) {
1490
+ await netRun(cfg, ["proxy", "add", ...proxyArgs(proxy)]);
1491
+ }
1492
+ async function closeProxy(proxy, cfg) {
1493
+ await netRun(cfg, ["proxy", "rm", ...proxyArgs(proxy)]);
1494
+ }
1495
+ var ipToInt = (ip) => ip.split(".").reduce((acc, o) => (acc << 8) + Number(o), 0) >>> 0;
1496
+ var intToIp = (n) => [24, 16, 8, 0].map((s) => n >>> s & 255).join(".");
1497
+ function nextFreeIp(subnet, inUse) {
1498
+ const [base, lenStr] = subnet.split("/");
1499
+ const len = Number(lenStr);
1500
+ if (!base || !Number.isInteger(len) || len < 8 || len > 30) throw new KvboxError(`bad subnet: ${subnet}`);
1501
+ const mask = len === 0 ? 0 : ~0 << 32 - len >>> 0;
1502
+ const network = (ipToInt(base) & mask) >>> 0;
1503
+ const broadcast = (network | ~mask >>> 0) >>> 0;
1504
+ const used = new Set(inUse.map(ipToInt));
1505
+ for (let n = network + 2; n < broadcast; n++) {
1506
+ if (!used.has(n)) return intToIp(n);
1577
1507
  }
1578
- };
1508
+ throw new KvboxError(`no free IP left in ${subnet}`);
1509
+ }
1510
+ function macFor(ip) {
1511
+ const hex = ip.split(".").map((o) => Number(o).toString(16).padStart(2, "0").toUpperCase());
1512
+ return ["06", "00", ...hex].join(":");
1513
+ }
1579
1514
 
1580
1515
  // src/core/machines.ts
1581
1516
  var NAME_RE = /^[a-z0-9][a-z0-9-]{0,11}$/;
@@ -1624,17 +1559,17 @@ function recordState(meta, state, by, p, at = (/* @__PURE__ */ new Date()).toISO
1624
1559
  }
1625
1560
  }
1626
1561
  function isHibernated(meta, p) {
1627
- return !pidAlive(meta.pid) && existsSync2(p.machine(meta.owner, meta.name).hibernateVmstate);
1562
+ return !vmAlive(meta) && existsSync2(p.machine(meta.owner, meta.name).hibernateVmstate);
1628
1563
  }
1629
1564
  function statusOf(meta, p = paths()) {
1630
- if (pidAlive(meta.pid)) return meta.paused ? "standby" : "running";
1565
+ if (vmAlive(meta)) return meta.paused ? "standby" : "running";
1631
1566
  return isHibernated(meta, p) ? "hibernated" : "stopped";
1632
1567
  }
1633
1568
  function stateMessage(ref, status) {
1634
1569
  return `box ${shown(ref)} is ${status === "standby" ? "in standby" : status}`;
1635
1570
  }
1636
1571
  function toMachine(meta, p = paths()) {
1637
- const alive = pidAlive(meta.pid);
1572
+ const alive = vmAlive(meta);
1638
1573
  if (!alive && meta.pid !== null) {
1639
1574
  try {
1640
1575
  closeOpenRun(meta.id, (/* @__PURE__ */ new Date()).toISOString(), p);
@@ -1656,9 +1591,13 @@ function toMachine(meta, p = paths()) {
1656
1591
  createdAt: meta.createdAt,
1657
1592
  source: meta.source,
1658
1593
  status: statusOf(meta, p),
1659
- ...meta.sshPort === void 0 ? {} : { sshPort: meta.sshPort }
1594
+ ...meta.sshPort === void 0 ? {} : { sshPort: meta.sshPort },
1595
+ ...meta.autoHibernateSec ? { autoHibernateSec: meta.autoHibernateSec } : {}
1660
1596
  };
1661
1597
  }
1598
+ function autoHibernateFor(req, cfg) {
1599
+ return (req.autoHibernateSec ?? cfg.autoHibernateSec) || void 0;
1600
+ }
1662
1601
  async function readAllMeta(p) {
1663
1602
  const metas = [];
1664
1603
  for (const ref of await listMachineRefs(p)) {
@@ -1739,7 +1678,7 @@ async function assertOverlayCapable(baseFile, version) {
1739
1678
  if (!ok) throw new KvboxError(`base image ${version} is not a kvbox base image (no /sbin/overlay-init); rerun install-server.sh`);
1740
1679
  }
1741
1680
  async function createOverlay(file, gib) {
1742
- await writeFile4(file, "", { mode: 384 });
1681
+ await writeFile3(file, "", { mode: 384 });
1743
1682
  await truncate(file, gib * 1024 ** 3);
1744
1683
  await run(["mkfs.ext4", "-q", "-F", "-m", "0", "-L", "kvbox-box", "-E", "lazy_itable_init=1,lazy_journal_init=1", file]);
1745
1684
  }
@@ -1760,8 +1699,8 @@ async function tailOf(file, lines = 5) {
1760
1699
  }
1761
1700
  async function spawnFirecracker(ref, cfg, p) {
1762
1701
  const m = p.machine(ref.owner, ref.name);
1763
- await rm2(m.sock, { force: true });
1764
- await rm2(m.vsock, { force: true });
1702
+ await rm(m.sock, { force: true });
1703
+ await rm(m.vsock, { force: true });
1765
1704
  const logFd = openSync2(m.log, "a");
1766
1705
  let child;
1767
1706
  try {
@@ -1871,7 +1810,7 @@ async function createMachine(req, owner, p = paths()) {
1871
1810
  const base = await currentBase(cfg, p);
1872
1811
  await assertOverlayCapable(base.file, base.version);
1873
1812
  await ensureHostNetwork(cfg);
1874
- await mkdir4(m.dir, { recursive: true, mode: 448 });
1813
+ await mkdir3(m.dir, { recursive: true, mode: 448 });
1875
1814
  let tapCreated = false;
1876
1815
  let pid;
1877
1816
  try {
@@ -1881,7 +1820,7 @@ async function createMachine(req, owner, p = paths()) {
1881
1820
  const spec = { id, name, owner, ip, mac, vcpu: size.vcpu, memMib: size.memMib, base: base.version, sshKeys, sshPort };
1882
1821
  pid = await launch(spec, tap, cfg, keysB64, p);
1883
1822
  await openSshPort(spec, cfg);
1884
- const meta = { ...spec, pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: "base" };
1823
+ const meta = { ...spec, pid, bootId: currentBootId(), createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: "base", autoHibernateSec: autoHibernateFor(req, cfg) };
1885
1824
  await writeMachineMeta(meta, p);
1886
1825
  recordState(meta, "running", "new", p, meta.createdAt);
1887
1826
  return toMachine(meta, p);
@@ -1889,7 +1828,7 @@ async function createMachine(req, owner, p = paths()) {
1889
1828
  if (pid !== void 0) killQuietly(pid);
1890
1829
  await closeSshPort({ id, ip, sshPort }, cfg);
1891
1830
  if (tapCreated) await deleteTap(id, cfg).catch(() => void 0);
1892
- await rm2(m.dir, { recursive: true, force: true });
1831
+ await rm(m.dir, { recursive: true, force: true });
1893
1832
  throw err;
1894
1833
  }
1895
1834
  }
@@ -1927,7 +1866,7 @@ async function restoreMachine(fromText, owner, req, cfg, p) {
1927
1866
  const m = p.machine(owner, name);
1928
1867
  const s = p.snapshot(from.owner, from.name);
1929
1868
  await ensureHostNetwork(cfg);
1930
- await mkdir4(m.dir, { recursive: true, mode: 448 });
1869
+ await mkdir3(m.dir, { recursive: true, mode: 448 });
1931
1870
  let tapCreated = false;
1932
1871
  let pid;
1933
1872
  try {
@@ -1945,7 +1884,7 @@ async function restoreMachine(fromText, owner, req, cfg, p) {
1945
1884
  const stale = await pushAuthorizedKeys({ owner, name, sshKeys }, m.vsock, p);
1946
1885
  await attachTap(id, cfg);
1947
1886
  await openSshPort(spec, cfg);
1948
- const meta = { ...spec, pid, createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: `snapshot:${refKey(from)}`, memSnapshots: 1, sshKeysStale: stale ? true : void 0 };
1887
+ const meta = { ...spec, pid, bootId: currentBootId(), createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: `snapshot:${refKey(from)}`, memSnapshots: 1, sshKeysStale: stale ? true : void 0, autoHibernateSec: autoHibernateFor(req, cfg) };
1949
1888
  await writeMachineMeta(meta, p);
1950
1889
  recordState(meta, "running", "new", p, meta.createdAt);
1951
1890
  return toMachine(meta, p);
@@ -1953,7 +1892,7 @@ async function restoreMachine(fromText, owner, req, cfg, p) {
1953
1892
  if (pid !== void 0) killQuietly(pid);
1954
1893
  await closeSshPort({ id, ip, sshPort }, cfg);
1955
1894
  if (tapCreated) await deleteTap(id, cfg).catch(() => void 0);
1956
- await rm2(m.dir, { recursive: true, force: true });
1895
+ await rm(m.dir, { recursive: true, force: true });
1957
1896
  throw err;
1958
1897
  }
1959
1898
  }
@@ -1999,7 +1938,7 @@ async function resumeVm(sock) {
1999
1938
  }
2000
1939
  async function shutdown(meta, cfg, p, graceMs) {
2001
1940
  const pid = meta.pid;
2002
- if (pidAlive(pid)) {
1941
+ if (vmAlive(meta)) {
2003
1942
  const fc = new FirecrackerClient(p.machine(meta.owner, meta.name).sock);
2004
1943
  try {
2005
1944
  if (meta.paused) await fc.resume();
@@ -2023,26 +1962,26 @@ async function cleanupProcess(meta, cfg, p) {
2023
1962
  await closeSshPort(meta, cfg);
2024
1963
  await deleteTap(meta.id, cfg);
2025
1964
  const m = p.machine(meta.owner, meta.name);
2026
- await rm2(m.sock, { force: true });
2027
- await rm2(m.vsock, { force: true });
1965
+ await rm(m.sock, { force: true });
1966
+ await rm(m.vsock, { force: true });
2028
1967
  }
2029
- async function stopMachine(ref, p = paths()) {
1968
+ var stopMachine = locked(async function stopMachine2(ref, p = paths()) {
2030
1969
  const meta = await readMeta(ref, p);
2031
1970
  const cfg = await loadConfig(p);
2032
1971
  const m = p.machine(ref.owner, ref.name);
2033
- if (!pidAlive(meta.pid) && !isHibernated(meta, p)) throw new KvboxError(`box ${shown(ref)} is already stopped`);
1972
+ if (!vmAlive(meta) && !isHibernated(meta, p)) throw new KvboxError(`box ${shown(ref)} is already stopped`);
2034
1973
  await shutdown(meta, cfg, p, 1e4);
2035
- await rm2(m.hibernateDir, { recursive: true, force: true });
2036
- await rm2(m.memBase, { force: true });
1974
+ await rm(m.hibernateDir, { recursive: true, force: true });
1975
+ await rm(m.memBase, { force: true });
2037
1976
  const stopped = { ...meta, pid: null, memSnapshots: void 0, paused: void 0 };
2038
1977
  await writeMachineMeta(stopped, p);
2039
1978
  recordState(stopped, "stopped", "stop", p);
2040
1979
  return toMachine(stopped, p);
2041
- }
2042
- async function startMachine(ref, p = paths()) {
1980
+ });
1981
+ var startMachine = locked(async function startMachine2(ref, p = paths()) {
2043
1982
  const meta = await readMeta(ref, p);
2044
1983
  const cfg = await loadConfig(p);
2045
- if (pidAlive(meta.pid)) throw new KvboxError(`box ${shown(ref)} is already ${meta.paused ? "up, in standby (kvbox resume " + shown(ref) + ")" : "running"}`);
1984
+ if (vmAlive(meta)) throw new KvboxError(`box ${shown(ref)} is already ${meta.paused ? "up, in standby (kvbox resume " + shown(ref) + ")" : "running"}`);
2046
1985
  if (isHibernated(meta, p)) {
2047
1986
  throw new KvboxError(`${stateMessage(ref, "hibernated")}: kvbox wake ${shown(ref)} continues where it left off; to boot fresh instead, kvbox stop ${shown(ref)} first (discards its memory)`);
2048
1987
  }
@@ -2054,7 +1993,7 @@ async function startMachine(ref, p = paths()) {
2054
1993
  try {
2055
1994
  pid = await launch(meta, tap, cfg, keysB64, p);
2056
1995
  await openSshPort(meta, cfg);
2057
- const started = { ...meta, pid, sshKeysStale: void 0 };
1996
+ const started = { ...meta, pid, bootId: currentBootId(), sshKeysStale: void 0 };
2058
1997
  await writeMachineMeta(started, p);
2059
1998
  recordState(started, "running", "start", p);
2060
1999
  return toMachine(started, p);
@@ -2064,15 +2003,15 @@ async function startMachine(ref, p = paths()) {
2064
2003
  await deleteTap(meta.id, cfg).catch(() => void 0);
2065
2004
  throw err;
2066
2005
  }
2067
- }
2068
- async function setSshKeys(ref, sshKeys, p = paths()) {
2006
+ });
2007
+ var setSshKeys = locked(async function setSshKeys2(ref, sshKeys, p = paths()) {
2069
2008
  const meta = await readMeta(ref, p);
2070
2009
  if (!isCloudBox(meta.owner)) throw new KvboxError(`box ${shown(ref)} is local: it trusts this host's key, there are no authorized_keys lines to replace`);
2071
2010
  if (sshKeys.length === 0) throw new KvboxError("sshKeys: at least one authorized_keys line (the box would keep only the host's own key)");
2072
2011
  authorizedKeysB64(await readHostPubkey(p), sshKeys);
2073
2012
  const { meta: updated, applied } = await applySshKeys(meta, sshKeys, p);
2074
2013
  return { machine: toMachine(updated, p), applied };
2075
- }
2014
+ });
2076
2015
  async function applySshKeys(meta, sshKeys, p) {
2077
2016
  const status = statusOf(meta, p);
2078
2017
  let updated = { ...meta, sshKeys: [...sshKeys], sshKeysStale: status === "stopped" ? void 0 : true };
@@ -2090,27 +2029,30 @@ async function applySshKeys(meta, sshKeys, p) {
2090
2029
  }
2091
2030
  async function trustCa(publicKeyText, p = paths()) {
2092
2031
  const publicKey = parseSshPublicKey(publicKeyText);
2093
- const report = { publicKey, fingerprint: sshKeyFingerprint(publicKey), updated: [], pending: [], failed: [], local: 0 };
2032
+ const report2 = { publicKey, fingerprint: sshKeyFingerprint(publicKey), updated: [], pending: [], failed: [], local: 0 };
2094
2033
  const hostKey = await readHostPubkey(p);
2095
- for (const meta of await readAllMeta(p)) {
2096
- if (!isCloudBox(meta.owner)) {
2097
- report.local++;
2034
+ for (const listed of await readAllMeta(p)) {
2035
+ if (!isCloudBox(listed.owner)) {
2036
+ report2.local++;
2098
2037
  continue;
2099
2038
  }
2100
- const box = refKey(meta);
2039
+ const box = refKey(listed);
2101
2040
  try {
2102
- const sshKeys = [...(meta.sshKeys ?? []).filter((l) => !isCaLine(l)), caLineFor(meta.owner, publicKey)];
2103
- authorizedKeysLines(hostKey, sshKeys);
2104
- const { applied, why } = await applySshKeys(meta, sshKeys, p);
2105
- if (applied === "now") report.updated.push(box);
2106
- else report.pending.push(why === void 0 ? { box, when: applied } : { box, when: applied, why });
2041
+ await withBoxLock(listed, async () => {
2042
+ const meta = await readMeta(listed, p);
2043
+ const sshKeys = [...(meta.sshKeys ?? []).filter((l) => !isCaLine(l)), caLineFor(meta.owner, publicKey)];
2044
+ authorizedKeysLines(hostKey, sshKeys);
2045
+ const { applied, why } = await applySshKeys(meta, sshKeys, p);
2046
+ if (applied === "now") report2.updated.push(box);
2047
+ else report2.pending.push(why === void 0 ? { box, when: applied } : { box, when: applied, why });
2048
+ });
2107
2049
  } catch (err) {
2108
- report.failed.push({ box, error: err.message });
2050
+ report2.failed.push({ box, error: err.message });
2109
2051
  }
2110
2052
  }
2111
- return report;
2053
+ return report2;
2112
2054
  }
2113
- async function pauseMachine(ref, p = paths()) {
2055
+ var pauseMachine = locked(async function pauseMachine2(ref, p = paths()) {
2114
2056
  const meta = await readMeta(ref, p);
2115
2057
  const status = statusOf(meta, p);
2116
2058
  if (status === "standby") throw new KvboxError(`${stateMessage(ref, status)} already`);
@@ -2126,8 +2068,8 @@ async function pauseMachine(ref, p = paths()) {
2126
2068
  await writeMachineMeta(paused, p);
2127
2069
  recordState(paused, "standby", "pause", p);
2128
2070
  return toMachine(paused, p);
2129
- }
2130
- async function resumeMachine(ref, p = paths()) {
2071
+ });
2072
+ var resumeMachine = locked(async function resumeMachine2(ref, p = paths()) {
2131
2073
  const meta = await readMeta(ref, p);
2132
2074
  const status = statusOf(meta, p);
2133
2075
  if (status === "running") throw new KvboxError(`${stateMessage(ref, status)} already`);
@@ -2140,23 +2082,23 @@ async function resumeMachine(ref, p = paths()) {
2140
2082
  await writeMachineMeta(running, p);
2141
2083
  recordState(running, "running", "resume", p);
2142
2084
  return toMachine(running, p);
2143
- }
2144
- async function hibernateMachine(ref, p = paths()) {
2085
+ });
2086
+ var hibernateMachine = locked(async function hibernateMachine2(ref, p = paths(), by = "hibernate") {
2145
2087
  const meta = await readMeta(ref, p);
2146
2088
  const cfg = await loadConfig(p);
2147
2089
  const status = statusOf(meta, p);
2148
2090
  if (status === "hibernated") throw new KvboxError(`${stateMessage(ref, status)} already`);
2149
2091
  if (status === "stopped") throw new KvboxError(`${stateMessage(ref, status)}; only a running box can be hibernated (there is no memory to keep)`);
2150
2092
  const m = p.machine(ref.owner, ref.name);
2151
- await rm2(m.hibernateDir, { recursive: true, force: true });
2152
- await mkdir4(m.hibernateDir, { recursive: true, mode: 448 });
2093
+ await rm(m.hibernateDir, { recursive: true, force: true });
2094
+ await mkdir3(m.hibernateDir, { recursive: true, mode: 448 });
2153
2095
  const fc = new FirecrackerClient(m.sock);
2154
2096
  try {
2155
2097
  try {
2156
2098
  await fc.pause();
2157
2099
  await fc.snapshotCreate(m.hibernateVmstate, m.hibernateMem, "Full");
2158
2100
  } catch (err) {
2159
- await rm2(m.hibernateDir, { recursive: true, force: true });
2101
+ await rm(m.hibernateDir, { recursive: true, force: true });
2160
2102
  if (!meta.paused) await fc.resume().catch(() => void 0);
2161
2103
  throw err;
2162
2104
  }
@@ -2167,13 +2109,13 @@ async function hibernateMachine(ref, p = paths()) {
2167
2109
  await waitFor(() => !pidAlive(meta.pid), 5e3, 50);
2168
2110
  await cleanupProcess(meta, cfg, p);
2169
2111
  await run(["fallocate", "--dig-holes", m.hibernateMem]);
2170
- await rm2(m.memBase, { force: true });
2112
+ await rm(m.memBase, { force: true });
2171
2113
  const hibernated = { ...meta, pid: null, memSnapshots: void 0, paused: void 0 };
2172
2114
  await writeMachineMeta(hibernated, p);
2173
- recordState(hibernated, "hibernated", "hibernate", p);
2115
+ recordState(hibernated, "hibernated", by, p);
2174
2116
  return toMachine(hibernated, p);
2175
- }
2176
- async function wakeMachine(ref, p = paths()) {
2117
+ });
2118
+ var wakeMachine = locked(async function wakeMachine2(ref, p = paths()) {
2177
2119
  const meta = await readMeta(ref, p);
2178
2120
  const cfg = await loadConfig(p);
2179
2121
  const status = statusOf(meta, p);
@@ -2205,19 +2147,19 @@ async function wakeMachine(ref, p = paths()) {
2205
2147
  await deleteTap(meta.id, cfg).catch(() => void 0);
2206
2148
  throw err;
2207
2149
  }
2208
- await rm2(m.memBase, { force: true });
2150
+ await rm(m.memBase, { force: true });
2209
2151
  await link(m.hibernateMem, m.memBase);
2210
- await rm2(m.hibernateDir, { recursive: true, force: true });
2211
- const running = { ...meta, pid, memSnapshots: 1, paused: void 0, sshKeysStale: stale ? true : void 0 };
2152
+ await rm(m.hibernateDir, { recursive: true, force: true });
2153
+ const running = { ...meta, pid, bootId: currentBootId(), memSnapshots: 1, paused: void 0, sshKeysStale: stale ? true : void 0 };
2212
2154
  await writeMachineMeta(running, p);
2213
2155
  recordState(running, "running", "wake", p);
2214
2156
  return toMachine(running, p);
2215
- }
2157
+ });
2216
2158
  async function applyAtNextBoot(meta, changes, by, cfg, p, whileStopped) {
2217
- const wasRunning = pidAlive(meta.pid);
2159
+ const wasRunning = vmAlive(meta);
2218
2160
  if (wasRunning) {
2219
2161
  await shutdown(meta, cfg, p, 1e4);
2220
- await rm2(p.machine(meta.owner, meta.name).memBase, { force: true });
2162
+ await rm(p.machine(meta.owner, meta.name).memBase, { force: true });
2221
2163
  }
2222
2164
  const updated = { ...meta, ...changes, pid: null, memSnapshots: void 0, paused: void 0 };
2223
2165
  await writeMachineMeta(updated, p);
@@ -2230,7 +2172,7 @@ function assertNotHibernated(meta, p) {
2230
2172
  throw new KvboxError(`${stateMessage(meta, "hibernated")}; its memory image was made with its current shape and base: kvbox wake ${shown(meta)} first`);
2231
2173
  }
2232
2174
  }
2233
- async function updateMachine(ref, p = paths()) {
2175
+ var updateMachine = locked(async function updateMachine2(ref, p = paths()) {
2234
2176
  const meta = await readMeta(ref, p);
2235
2177
  const cfg = await loadConfig(p);
2236
2178
  assertNotHibernated(meta, p);
@@ -2239,8 +2181,8 @@ async function updateMachine(ref, p = paths()) {
2239
2181
  if (base.version === meta.base) return toMachine(meta, p);
2240
2182
  await assertOverlayCapable(base.file, base.version);
2241
2183
  return applyAtNextBoot(meta, { base: base.version }, "update", cfg, p);
2242
- }
2243
- async function rescaleMachine(ref, req, p = paths()) {
2184
+ });
2185
+ var rescaleMachine = locked(async function rescaleMachine2(ref, req, p = paths()) {
2244
2186
  const meta = await readMeta(ref, p);
2245
2187
  const cfg = await loadConfig(p);
2246
2188
  assertNotHibernated(meta, p);
@@ -2261,7 +2203,14 @@ async function rescaleMachine(ref, req, p = paths()) {
2261
2203
  const growTo = target.diskGib > currentDiskGib ? target.diskGib : void 0;
2262
2204
  if (Object.keys(changes).length === 0 && growTo === void 0) return toMachine(meta, p);
2263
2205
  return applyAtNextBoot(meta, changes, "rescale", cfg, p, growTo === void 0 ? void 0 : () => growDisk(disk, growTo));
2264
- }
2206
+ });
2207
+ var setMachine = locked(async function setMachine2(ref, req, p = paths()) {
2208
+ const meta = await readMeta(ref, p);
2209
+ const next = { ...meta };
2210
+ if (req.autoHibernateSec !== void 0) next.autoHibernateSec = req.autoHibernateSec || void 0;
2211
+ await writeMachineMeta(next, p);
2212
+ return toMachine(next, p);
2213
+ });
2265
2214
  async function growDisk(file, gib) {
2266
2215
  await truncate(file, gib * 1024 ** 3);
2267
2216
  await run(["e2fsck", "-f", "-p", file], { okExitCodes: [1] });
@@ -2275,12 +2224,369 @@ async function readMeta(ref, p) {
2275
2224
  throw err;
2276
2225
  }
2277
2226
  }
2278
- async function destroyMachine(ref, p = paths()) {
2227
+ var destroyMachine = locked(async function destroyMachine2(ref, p = paths()) {
2279
2228
  const meta = await readMeta(ref, p);
2280
2229
  const cfg = await loadConfig(p);
2281
2230
  await shutdown(meta, cfg, p, 2e3);
2282
2231
  recordState(meta, "stopped", "rm", p);
2283
- await rm2(p.machine(ref.owner, ref.name).dir, { recursive: true, force: true });
2232
+ await rm(p.machine(ref.owner, ref.name).dir, { recursive: true, force: true });
2233
+ });
2234
+ var RECOVERY_CONCURRENCY = 2;
2235
+ async function settleDeadBoxes(p = paths()) {
2236
+ const now = (/* @__PURE__ */ new Date()).toISOString();
2237
+ const bootAt = hostBootTime()?.toISOString() ?? now;
2238
+ const boot = [];
2239
+ const leftStopped = [];
2240
+ for (const meta of await readAllMeta(p)) {
2241
+ if (!cutByReboot(meta) || vmAlive(meta)) continue;
2242
+ try {
2243
+ closeRunCutByBoot(meta.id, bootAt, now, p);
2244
+ } catch (err) {
2245
+ console.error(`warning: usage ledger: ${err.message}`);
2246
+ }
2247
+ await rm(p.machine(meta.owner, meta.name).memBase, { force: true });
2248
+ if (meta.paused) {
2249
+ await writeMachineMeta({ ...meta, pid: null, bootId: void 0, memSnapshots: void 0, paused: void 0 }, p);
2250
+ leftStopped.push(refKey(meta));
2251
+ } else {
2252
+ await writeMachineMeta({ ...meta, memSnapshots: void 0 }, p);
2253
+ boot.push({ owner: meta.owner, name: meta.name });
2254
+ }
2255
+ }
2256
+ return { boot, leftStopped };
2257
+ }
2258
+ function cutByReboot(meta) {
2259
+ return meta.pid !== null && meta.bootId !== currentBootId();
2260
+ }
2261
+ var recovery = null;
2262
+ var recoveryStopping = false;
2263
+ var recoveryRun = Promise.resolve();
2264
+ function lastRecovery() {
2265
+ return recovery;
2266
+ }
2267
+ async function stopRecovery() {
2268
+ recoveryStopping = true;
2269
+ await recoveryRun;
2270
+ }
2271
+ async function bootRecovered(settled, p = paths()) {
2272
+ const refs = settled.boot;
2273
+ if (refs.length === 0 && settled.leftStopped.length === 0) return null;
2274
+ const booted = [];
2275
+ const failed = [];
2276
+ const queue = [...refs];
2277
+ const worker = async () => {
2278
+ for (let ref = queue.shift(); ref !== void 0 && !recoveryStopping; ref = queue.shift()) {
2279
+ try {
2280
+ if (!cutByReboot(await readMeta(ref, p))) continue;
2281
+ await startMachine(ref, p);
2282
+ booted.push(refKey(ref));
2283
+ } catch (err) {
2284
+ failed.push({ box: refKey(ref), error: err.message });
2285
+ }
2286
+ }
2287
+ };
2288
+ const run2 = Promise.all(Array.from({ length: Math.min(RECOVERY_CONCURRENCY, refs.length) }, worker));
2289
+ recoveryRun = run2;
2290
+ await run2;
2291
+ recovery = { at: (/* @__PURE__ */ new Date()).toISOString(), booted, leftStopped: [...settled.leftStopped], failed };
2292
+ return recovery;
2293
+ }
2294
+
2295
+ // src/core/relay.ts
2296
+ import { chmod as chmod3, mkdir as mkdir4, rm as rm2, writeFile as writeFile4 } from "node:fs/promises";
2297
+
2298
+ // ../shared/src/relay.ts
2299
+ import { z as z5 } from "zod";
2300
+ var RELAY_TUNNEL_GATEWAY = "10.77.0.1";
2301
+ var IPV4_RE = /^(\d{1,3})(\.\d{1,3}){3}$/;
2302
+ var WG_KEY_RE = /^[A-Za-z0-9+/]{43}=$/;
2303
+ var RelayJoinSchema = z5.object({
2304
+ v: z5.literal(1),
2305
+ /** How the relay lists this host. */
2306
+ name: z5.string().regex(/^[a-z0-9][a-z0-9-]{0,30}$/),
2307
+ /** `<relay-primary-ip>:<port>`, the WireGuard endpoint the host dials. */
2308
+ relay: z5.string().regex(/^(\d{1,3})(\.\d{1,3}){3}:\d{1,5}$/),
2309
+ /** The relay's WireGuard public key. */
2310
+ relayKey: z5.string().regex(WG_KEY_RE),
2311
+ /** This host's address inside the tunnel. */
2312
+ tunnelIp: z5.string().regex(IPV4_RE),
2313
+ /** This host's WireGuard private key, generated on the relay, kept nowhere else. */
2314
+ privateKey: z5.string().regex(WG_KEY_RE),
2315
+ /** The extra public IPv4 the relay owns for this host; what the host advertises to the control plane. */
2316
+ publicIp: z5.string().regex(IPV4_RE)
2317
+ });
2318
+ function decodeRelayJoin(text) {
2319
+ let parsed;
2320
+ try {
2321
+ parsed = JSON.parse(Buffer.from(text.trim(), "base64url").toString("utf8"));
2322
+ } catch {
2323
+ return null;
2324
+ }
2325
+ const res = RelayJoinSchema.safeParse(parsed);
2326
+ return res.success ? res.data : null;
2327
+ }
2328
+
2329
+ // src/core/relay.ts
2330
+ function relayConf(join2) {
2331
+ return [
2332
+ "# kvbox relay tunnel: made by `kvbox server relay`, installed by net-setup.sh. Do not edit.",
2333
+ "[Interface]",
2334
+ `PrivateKey = ${join2.privateKey}`,
2335
+ `Address = ${join2.tunnelIp}/32`,
2336
+ "",
2337
+ "[Peer]",
2338
+ `PublicKey = ${join2.relayKey}`,
2339
+ `Endpoint = ${join2.relay}`,
2340
+ // Only the relay's own tunnel address goes through the tunnel: the host keeps its normal internet
2341
+ // route, and the relay SNATs forwarded connections so their replies come back this way.
2342
+ `AllowedIPs = ${RELAY_TUNNEL_GATEWAY}/32`,
2343
+ // The router forgets an idle outbound mapping; a packet every 25 s keeps the relay able to reach us.
2344
+ "PersistentKeepalive = 25",
2345
+ ""
2346
+ ].join("\n");
2347
+ }
2348
+ async function useRelay(joinString, p = paths()) {
2349
+ const join2 = decodeRelayJoin(joinString);
2350
+ if (!join2) throw new KvboxError("that is not a relay join string; copy the whole line `kvbox-relay add` printed on the relay");
2351
+ const cfg = await loadConfig(p);
2352
+ if (cfg.relay && cfg.relay.name !== join2.name) {
2353
+ throw new KvboxError(`this host already goes through relay ${cfg.relay.name} (${cfg.relay.publicIp}); run kvbox server relay off first`);
2354
+ }
2355
+ if (cfg.relay && cfg.relay.publicIp !== join2.publicIp && cfg.controlPlane?.advertise === cfg.relay.publicIp) {
2356
+ throw new KvboxError(
2357
+ `${cfg.controlPlane.name} reaches this host at ${cfg.relay.publicIp}, not ${join2.publicIp}; run kvbox server leave first, then join again`
2358
+ );
2359
+ }
2360
+ await mkdir4(p.relay, { recursive: true, mode: 448 });
2361
+ await writeFile4(p.relayConf, relayConf(join2), { mode: 384 });
2362
+ await chmod3(p.relayConf, 384);
2363
+ await netSetup(cfg, ["relay", "up", p.relayConf, String(cfg.controlPlane?.port ?? DEFAULT_HTTPS_PORT)]);
2364
+ const relay = { name: join2.name, publicIp: join2.publicIp, tunnelIp: join2.tunnelIp, endpoint: join2.relay };
2365
+ await saveConfig({ ...cfg, relay }, p);
2366
+ return relay;
2367
+ }
2368
+ async function dropRelay(p = paths()) {
2369
+ const cfg = await loadConfig(p);
2370
+ const relay = cfg.relay;
2371
+ if (!relay) throw new KvboxError("this host does not go through a relay");
2372
+ if (cfg.controlPlane?.advertise === relay.publicIp) {
2373
+ throw new KvboxError(`${cfg.controlPlane.name} reaches this host through the relay (${relay.publicIp}); run kvbox server leave first`);
2374
+ }
2375
+ await netSetup(cfg, ["relay", "down"]);
2376
+ await rm2(p.relayConf, { force: true });
2377
+ const { relay: _gone, ...rest } = cfg;
2378
+ await saveConfig(rest, p);
2379
+ return relay;
2380
+ }
2381
+
2382
+ // src/core/idle.ts
2383
+ import { readFile as readFile6 } from "node:fs/promises";
2384
+ var HIBERNATE_CONCURRENCY = 2;
2385
+ async function tapTraffic(id) {
2386
+ const dir = `/sys/class/net/${tapName(id)}/statistics`;
2387
+ try {
2388
+ const [rx, tx] = await Promise.all([readFile6(`${dir}/rx_bytes`, "utf8"), readFile6(`${dir}/tx_bytes`, "utf8")]);
2389
+ return Number(rx) + Number(tx);
2390
+ } catch {
2391
+ return null;
2392
+ }
2393
+ }
2394
+ async function hibernateIfRunning(box, p) {
2395
+ const ref = { owner: box.owner, name: box.name };
2396
+ return withBoxLock(ref, async () => {
2397
+ if ((await getMachine(ref, p)).status !== "running") return false;
2398
+ await hibernateMachine(ref, p, "auto-hibernate");
2399
+ return true;
2400
+ });
2401
+ }
2402
+ function startIdleWatch(opts = {}) {
2403
+ const intervalMs = opts.intervalMs ?? 6e4;
2404
+ const p = opts.p ?? paths();
2405
+ const list = opts.list ?? (() => listMachines(null, p));
2406
+ const traffic = opts.traffic ?? tapTraffic;
2407
+ const hibernate = opts.hibernate ?? ((box) => hibernateIfRunning(box, p));
2408
+ const busy = opts.busy ?? ((box) => boxBusy(box));
2409
+ const now = opts.now ?? Date.now;
2410
+ const log = opts.log ?? ((line) => console.log(line));
2411
+ const seen = /* @__PURE__ */ new Map();
2412
+ let timer;
2413
+ let stopped = false;
2414
+ const inflight = /* @__PURE__ */ new Set();
2415
+ const hibernateOne = async (box, quiet, bytes, t) => {
2416
+ inflight.add(box.id);
2417
+ try {
2418
+ if (await hibernate(box)) log(`box ${box.owner}/${box.name}: no network traffic for ${quiet} min; hibernated it (kvbox connect wakes it)`);
2419
+ seen.delete(box.id);
2420
+ } catch (err) {
2421
+ log(`warning: box ${box.owner}/${box.name}: could not hibernate it after ${quiet} min without traffic: ${err.message}`);
2422
+ seen.set(box.id, { bytes, since: t });
2423
+ } finally {
2424
+ inflight.delete(box.id);
2425
+ }
2426
+ };
2427
+ const check = async () => {
2428
+ const started = [];
2429
+ const watched = /* @__PURE__ */ new Set();
2430
+ for (const box of await list()) {
2431
+ if (!box.autoHibernateSec || box.status !== "running") continue;
2432
+ if (inflight.has(box.id)) {
2433
+ watched.add(box.id);
2434
+ continue;
2435
+ }
2436
+ const bytes = await traffic(box.id);
2437
+ if (bytes === null) continue;
2438
+ watched.add(box.id);
2439
+ const t = now();
2440
+ const last = seen.get(box.id);
2441
+ if (!last || last.bytes !== bytes || busy(box)) {
2442
+ seen.set(box.id, { bytes, since: t });
2443
+ continue;
2444
+ }
2445
+ if (t - last.since < box.autoHibernateSec * 1e3) continue;
2446
+ if (inflight.size >= HIBERNATE_CONCURRENCY) continue;
2447
+ started.push(hibernateOne(box, Math.round((t - last.since) / 6e4), bytes, t));
2448
+ }
2449
+ for (const id of seen.keys()) if (!watched.has(id)) seen.delete(id);
2450
+ return started;
2451
+ };
2452
+ const round2 = async () => {
2453
+ await Promise.all(await check());
2454
+ };
2455
+ const schedule = () => {
2456
+ if (stopped) return;
2457
+ timer = setTimeout(() => void check().catch((err) => log(`warning: idle watch: ${err.message}`)).finally(schedule), intervalMs);
2458
+ timer.unref();
2459
+ };
2460
+ schedule();
2461
+ return {
2462
+ stop: () => {
2463
+ stopped = true;
2464
+ clearTimeout(timer);
2465
+ },
2466
+ round: round2
2467
+ };
2468
+ }
2469
+
2470
+ // src/core/relayWatch.ts
2471
+ var report = { restarts: 0, lastRestartAt: null };
2472
+ function relayWatchReport() {
2473
+ return report;
2474
+ }
2475
+ async function pingRelay() {
2476
+ try {
2477
+ await run(["ping", "-c", "1", "-W", "3", RELAY_TUNNEL_GATEWAY], { timeoutMs: 5e3 });
2478
+ return true;
2479
+ } catch {
2480
+ return false;
2481
+ }
2482
+ }
2483
+ function startRelayWatch(opts = {}) {
2484
+ const intervalMs = opts.intervalMs ?? 3e4;
2485
+ const misses = opts.misses ?? 3;
2486
+ const probe = opts.probe ?? pingRelay;
2487
+ const p = opts.p ?? paths();
2488
+ const log = opts.log ?? ((line) => console.log(line));
2489
+ const restart = opts.restart ?? (async () => netSetup(await loadConfig(p), ["relay", "restart"]));
2490
+ report = { restarts: 0, lastRestartAt: null };
2491
+ let missed = 0;
2492
+ let stuck = false;
2493
+ let timer;
2494
+ let stopped = false;
2495
+ const round2 = async () => {
2496
+ const cfg = await loadConfig(p).catch(() => null);
2497
+ if (!cfg?.relay) {
2498
+ missed = 0;
2499
+ return;
2500
+ }
2501
+ if (await probe()) {
2502
+ if (stuck) log(`relay ${cfg.relay.name}: the tunnel carries traffic again`);
2503
+ missed = 0;
2504
+ stuck = false;
2505
+ return;
2506
+ }
2507
+ if (++missed < misses) return;
2508
+ missed = 0;
2509
+ if (!stuck) log(`relay ${cfg.relay.name}: no answer through the tunnel for ${misses * intervalMs / 1e3} s; restarting it`);
2510
+ try {
2511
+ await restart();
2512
+ report = { restarts: report.restarts + 1, lastRestartAt: (/* @__PURE__ */ new Date()).toISOString() };
2513
+ } catch (err) {
2514
+ if (!stuck) console.error(`warning: relay ${cfg.relay.name}: could not restart the tunnel: ${err.message}`);
2515
+ }
2516
+ stuck = true;
2517
+ };
2518
+ const schedule = () => {
2519
+ if (stopped) return;
2520
+ timer = setTimeout(() => void round2().catch((err) => console.error(`warning: relay watch: ${err.message}`)).finally(schedule), intervalMs);
2521
+ timer.unref();
2522
+ };
2523
+ schedule();
2524
+ return () => {
2525
+ stopped = true;
2526
+ clearTimeout(timer);
2527
+ };
2528
+ }
2529
+
2530
+ // src/listen.ts
2531
+ import { readFile as readFile7, rm as rm4, writeFile as writeFile5 } from "node:fs/promises";
2532
+ import { createServer as createHttpsServer } from "node:https";
2533
+ import { createAdaptorServer } from "@hono/node-server";
2534
+
2535
+ // src/app.ts
2536
+ import { Hono as Hono6 } from "hono";
2537
+
2538
+ // ../shared/src/version.ts
2539
+ import { createRequire } from "node:module";
2540
+ function versionOf(srcFileUrl, injected) {
2541
+ if (typeof injected === "string") return injected;
2542
+ try {
2543
+ const require2 = createRequire(srcFileUrl);
2544
+ return require2("../package.json").version;
2545
+ } catch {
2546
+ return "0.0.0-dev";
2547
+ }
2548
+ }
2549
+
2550
+ // src/version.ts
2551
+ var VERSION = versionOf(import.meta.url, true ? "0.5.10" : void 0);
2552
+
2553
+ // src/caller.ts
2554
+ import { createHash as createHash2, timingSafeEqual } from "node:crypto";
2555
+ var OWNER_HEADER = "x-kvbox-owner";
2556
+ function resolveCaller(opts = {}) {
2557
+ const cp = opts.controlPlane;
2558
+ if (!cp) {
2559
+ return async (c, next) => {
2560
+ const cfg = await loadConfig();
2561
+ c.set("owner", cfg.owner);
2562
+ c.set("listScope", cfg.owner);
2563
+ await next();
2564
+ };
2565
+ }
2566
+ const expected = digest(cp.secret);
2567
+ return async (c, next) => {
2568
+ const m = /^Bearer\s+(\S+)$/i.exec(c.req.header("authorization") ?? "");
2569
+ if (!m || !timingSafeEqual(digest(m[1]), expected)) return c.json({ error: "not authorized" }, 401);
2570
+ const user = c.req.header(OWNER_HEADER);
2571
+ if (user === void 0) {
2572
+ const path = c.req.path.replace(/\/$/, "");
2573
+ const realmListing = c.req.method === "GET" && (path === "/machines" || path === "/snapshots");
2574
+ if (!realmListing && /^\/(machines|snapshots)(\/|$)/.test(c.req.path)) {
2575
+ return c.json({ error: "X-Kvbox-Owner header required" }, 400);
2576
+ }
2577
+ c.set("owner", "");
2578
+ c.set("listScope", { realm: cp.name });
2579
+ } else {
2580
+ if (!LOCAL_OWNER_RE.test(user)) return c.json({ error: "X-Kvbox-Owner: letters, digits, '.', '_', '-'" }, 400);
2581
+ const owner = `${user}@${cp.name}`;
2582
+ c.set("owner", owner);
2583
+ c.set("listScope", owner);
2584
+ }
2585
+ await next();
2586
+ };
2587
+ }
2588
+ function digest(secret) {
2589
+ return createHash2("sha256").update(secret).digest();
2284
2590
  }
2285
2591
 
2286
2592
  // src/routes/capacity.ts
@@ -2329,6 +2635,10 @@ function machineRoutes() {
2329
2635
  const body = RescaleMachineRequest.parse(await c.req.json().catch(() => ({})));
2330
2636
  return c.json(await rescaleMachine(parseRef(c.req.param("name"), c.get("owner")), body));
2331
2637
  });
2638
+ r.post("/:name/set", async (c) => {
2639
+ const body = SetMachineRequest.parse(await c.req.json().catch(() => ({})));
2640
+ return c.json(await setMachine(parseRef(c.req.param("name"), c.get("owner")), body));
2641
+ });
2332
2642
  r.post("/:name/ssh-keys", async (c) => {
2333
2643
  const body = SetSshKeysRequest.safeParse(await c.req.json().catch(() => ({})));
2334
2644
  if (!body.success) throw new KvboxError("sshKeys: expected a list of authorized_keys lines");
@@ -2402,8 +2712,12 @@ async function saveSnapshot(req, owner, p = paths()) {
2402
2712
  if (!NAME_RE2.test(req.name)) {
2403
2713
  throw new KvboxError(`invalid snapshot name "${req.name}": 1-64 chars, lowercase letters, digits, '.', '_' and '-', starting with a letter or digit`);
2404
2714
  }
2405
- const box = await readBox(parseRef(req.box, owner), p);
2406
- if (!pidAlive(box.pid)) {
2715
+ const ref = parseRef(req.box, owner);
2716
+ return withBoxLock(ref, () => snapshotBox(ref, req, owner, p));
2717
+ }
2718
+ async function snapshotBox(ref, req, owner, p) {
2719
+ const box = await readBox(ref, p);
2720
+ if (!vmAlive(box)) {
2407
2721
  if (existsSync3(p.machine(box.owner, box.name).hibernateVmstate)) {
2408
2722
  throw new KvboxError(`box ${req.box} is hibernated; a snapshot freezes a running box (kvbox wake ${req.box} first)`);
2409
2723
  }
@@ -2549,7 +2863,8 @@ function createApp(opts = {}) {
2549
2863
  base: await currentBaseVersion().catch(() => null),
2550
2864
  owner: cfg.owner,
2551
2865
  controlPlane: cp ? { name: cp.name, url: cp.url, hostUrl: `https://${cp.advertise}:${cp.port}` } : null,
2552
- relay: cfg.relay ? { name: cfg.relay.name, publicIp: cfg.relay.publicIp, endpoint: cfg.relay.endpoint } : null
2866
+ relay: cfg.relay ? { name: cfg.relay.name, publicIp: cfg.relay.publicIp, endpoint: cfg.relay.endpoint, ...relayWatchReport() } : null,
2867
+ recovery: lastRecovery()
2553
2868
  });
2554
2869
  });
2555
2870
  app.route("/machines", machineRoutes());
@@ -2589,7 +2904,7 @@ async function listen(opts = {}) {
2589
2904
  let httpsServer;
2590
2905
  const cp = cfg.controlPlane;
2591
2906
  if (cp) {
2592
- const [key, cert] = await Promise.all([readFile6(p.tlsKey), readFile6(p.tlsCert)]).catch((err) => {
2907
+ const [key, cert] = await Promise.all([readFile7(p.tlsKey), readFile7(p.tlsCert)]).catch((err) => {
2593
2908
  throw new KvboxError(`joined ${cp.name} but the host's certificate is missing (${err.message}); run kvbox-server leave, then join again`);
2594
2909
  });
2595
2910
  httpsServer = createAdaptorServer({
@@ -2621,6 +2936,7 @@ async function listen(opts = {}) {
2621
2936
  const shutdown2 = async () => {
2622
2937
  server.close();
2623
2938
  httpsServer?.close();
2939
+ await stopRecovery();
2624
2940
  await rm4(socketPath, { force: true });
2625
2941
  await rm4(p.serverPid, { force: true });
2626
2942
  process.exit(0);
@@ -2634,6 +2950,7 @@ async function listen(opts = {}) {
2634
2950
  var program = new Command("kvbox-server").description("kvbox host daemon: owns Firecracker and ~/.kvbox, serves the kvbox API").version(VERSION, "-v, --version").showHelpAfterError();
2635
2951
  program.command("start", { isDefault: true }).description("serve the API on the local Unix socket (foreground)").option("--socket <path>", "socket to listen on (default: $KVBOX_HOME/server.sock)").action(async (opts) => {
2636
2952
  await requireKvm();
2953
+ const settled = await settleDeadBoxes(paths());
2637
2954
  const { socketPath, https } = await listen(opts.socket ? { socketPath: opts.socket } : {});
2638
2955
  console.log(`kvbox-server ${VERSION} listening on ${socketPath} (pid ${process.pid})`);
2639
2956
  if (https) {
@@ -2647,6 +2964,14 @@ program.command("start", { isDefault: true }).description("serve the API on the
2647
2964
  } catch (err) {
2648
2965
  console.error(`warning: host network not ready (${err.message}); run: kvbox-server net up`);
2649
2966
  }
2967
+ startRelayWatch();
2968
+ startIdleWatch();
2969
+ const recovery2 = await bootRecovered(settled, paths());
2970
+ if (recovery2) {
2971
+ for (const box of recovery2.booted) console.log(`box ${box} was running when it went down; booted it again`);
2972
+ for (const box of recovery2.leftStopped) console.log(`box ${box} was in standby when it went down; its memory is gone, left stopped`);
2973
+ for (const f of recovery2.failed) console.error(`warning: box ${f.box} was running when it went down and did not come back: ${f.error}`);
2974
+ }
2650
2975
  });
2651
2976
  program.command("join <control-plane-url>").description("register this host with a control plane (needs a one-time token minted there); it answers the control plane from its next start").requiredOption("--token <token>", "one-time join token from the control plane").option("--advertise <host-or-ip>", "address the control plane reaches this host at (default: this host's default-route address)").option("--name <name>", "how the control plane lists this host (default: hostname)").action(async (url, opts) => {
2652
2977
  const joined = await joinControlPlane({ url, ...opts }, paths());