@kvbox/server 0.5.9 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/kvbox-server.js +259 -45
- package/dist/kvbox-server.js.map +1 -1
- package/package.json +1 -1
- package/scripts/net-setup.sh +57 -6
package/dist/kvbox-server.js
CHANGED
|
@@ -292,6 +292,8 @@ var ConfigSchema = z2.object({
|
|
|
292
292
|
/** Who boxes made over the local socket belong to: the Unix user running the server unless set here. No `@`, no `/`. */
|
|
293
293
|
owner: z2.string().regex(LOCAL_OWNER_RE, "owner: letters, digits, '.', '_', '-'; no '@' or '/'").default(localOwner),
|
|
294
294
|
pricing: PricingSchema.optional(),
|
|
295
|
+
/** What a new box gets when `new` does not say (#47): seconds without network traffic before it is hibernated; 0: never. */
|
|
296
|
+
autoHibernateSec: z2.number().int().min(0).default(0),
|
|
295
297
|
/** Absent until the host joins a control plane; then the server also listens on HTTPS. */
|
|
296
298
|
controlPlane: ControlPlaneSchema.optional(),
|
|
297
299
|
/** Absent unless the host sits behind a relay. */
|
|
@@ -370,7 +372,9 @@ var MachineSchema = z3.object({
|
|
|
370
372
|
*/
|
|
371
373
|
ssh: z3.object({ host: z3.string(), port: z3.number().int(), user: z3.string() }).optional(),
|
|
372
374
|
/** Whether the owner slug is a user or an org. Only the control plane knows; it fills it in on `GET /machines`. */
|
|
373
|
-
ownerKind: z3.enum(["user", "org"]).optional()
|
|
375
|
+
ownerKind: z3.enum(["user", "org"]).optional(),
|
|
376
|
+
/** Seconds without network traffic before the host hibernates the box (#47). Absent: never. */
|
|
377
|
+
autoHibernateSec: z3.number().int().positive().optional()
|
|
374
378
|
});
|
|
375
379
|
var CreateMachineRequest = z3.object({
|
|
376
380
|
name: z3.string().optional(),
|
|
@@ -384,7 +388,9 @@ var CreateMachineRequest = z3.object({
|
|
|
384
388
|
* its SSH CA as one `cert-authority …` line (#51). Written into the box's authorized_keys at every
|
|
385
389
|
* boot (#49); a host older than that ignores them.
|
|
386
390
|
*/
|
|
387
|
-
sshKeys: z3.array(z3.string()).optional()
|
|
391
|
+
sshKeys: z3.array(z3.string()).optional(),
|
|
392
|
+
/** Seconds without network traffic before the box is hibernated (#47); 0: never. Absent: the host's default. */
|
|
393
|
+
autoHibernateSec: z3.number().int().min(0).optional()
|
|
388
394
|
});
|
|
389
395
|
var SetSshKeysRequest = z3.object({
|
|
390
396
|
sshKeys: z3.array(z3.string())
|
|
@@ -395,6 +401,10 @@ var RescaleMachineRequest = z3.object({
|
|
|
395
401
|
memMib: z3.number().int().min(256).optional(),
|
|
396
402
|
diskGib: z3.number().int().min(1).optional()
|
|
397
403
|
});
|
|
404
|
+
var SetMachineRequest = z3.object({
|
|
405
|
+
/** Seconds without network traffic before the box is hibernated (#47); 0: never. */
|
|
406
|
+
autoHibernateSec: z3.number().int().min(0).optional()
|
|
407
|
+
});
|
|
398
408
|
var SnapshotSchema = z3.object({
|
|
399
409
|
name: z3.string(),
|
|
400
410
|
owner: z3.string(),
|
|
@@ -447,7 +457,10 @@ var HealthSchema = z3.object({
|
|
|
447
457
|
/** The public IP the relay owns for this host. */
|
|
448
458
|
publicIp: z3.string(),
|
|
449
459
|
/** The relay's WireGuard endpoint the tunnel dials. */
|
|
450
|
-
endpoint: z3.string()
|
|
460
|
+
endpoint: z3.string(),
|
|
461
|
+
/** Times the server redialed the tunnel since it started, because the relay stopped answering. Older servers omit them. */
|
|
462
|
+
restarts: z3.number().int().optional(),
|
|
463
|
+
lastRestartAt: z3.string().nullable().optional()
|
|
451
464
|
}).nullable().optional(),
|
|
452
465
|
/** Boxes brought back at the server's last start; null when there were none or it is still at it. Older servers omit it. */
|
|
453
466
|
recovery: RecoveryReportSchema.nullable().optional()
|
|
@@ -986,7 +999,9 @@ var MachineMetaSchema = z4.object({
|
|
|
986
999
|
*/
|
|
987
1000
|
sshKeysStale: z4.boolean().optional(),
|
|
988
1001
|
/** Public port on the host DNATed to the box's SSH while it runs (#49); fixed for the box's life. Absent: none. */
|
|
989
|
-
sshPort: z4.number().int().optional()
|
|
1002
|
+
sshPort: z4.number().int().optional(),
|
|
1003
|
+
/** Seconds without traffic on the box's tap before the server hibernates it (#47, core/idle.ts). Absent: never. */
|
|
1004
|
+
autoHibernateSec: z4.number().int().positive().optional()
|
|
990
1005
|
});
|
|
991
1006
|
function newBoxId(taken) {
|
|
992
1007
|
for (; ; ) {
|
|
@@ -1250,6 +1265,33 @@ function usageEventsInScope(scope, window, p = paths()) {
|
|
|
1250
1265
|
return rows.filter((r) => inScope(r.owner, scope)).map(withId);
|
|
1251
1266
|
}
|
|
1252
1267
|
|
|
1268
|
+
// src/core/locks.ts
|
|
1269
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
1270
|
+
var tails = /* @__PURE__ */ new Map();
|
|
1271
|
+
var held = new AsyncLocalStorage();
|
|
1272
|
+
async function withBoxLock(ref, fn) {
|
|
1273
|
+
const key = refKey(ref);
|
|
1274
|
+
const mine = held.getStore();
|
|
1275
|
+
if (mine?.has(key)) return fn();
|
|
1276
|
+
const previous = tails.get(key);
|
|
1277
|
+
let release;
|
|
1278
|
+
const tail = new Promise((resolve) => release = resolve);
|
|
1279
|
+
tails.set(key, tail);
|
|
1280
|
+
try {
|
|
1281
|
+
await previous;
|
|
1282
|
+
return await held.run(/* @__PURE__ */ new Set([...mine ?? [], key]), fn);
|
|
1283
|
+
} finally {
|
|
1284
|
+
release();
|
|
1285
|
+
if (tails.get(key) === tail) tails.delete(key);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
function boxBusy(ref) {
|
|
1289
|
+
return tails.has(refKey(ref));
|
|
1290
|
+
}
|
|
1291
|
+
function locked(fn) {
|
|
1292
|
+
return (...args) => withBoxLock(args[0], () => fn(...args));
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1253
1295
|
// src/core/firecracker.ts
|
|
1254
1296
|
import { Agent, request as request2 } from "undici";
|
|
1255
1297
|
var FirecrackerClient = class {
|
|
@@ -1549,9 +1591,13 @@ function toMachine(meta, p = paths()) {
|
|
|
1549
1591
|
createdAt: meta.createdAt,
|
|
1550
1592
|
source: meta.source,
|
|
1551
1593
|
status: statusOf(meta, p),
|
|
1552
|
-
...meta.sshPort === void 0 ? {} : { sshPort: meta.sshPort }
|
|
1594
|
+
...meta.sshPort === void 0 ? {} : { sshPort: meta.sshPort },
|
|
1595
|
+
...meta.autoHibernateSec ? { autoHibernateSec: meta.autoHibernateSec } : {}
|
|
1553
1596
|
};
|
|
1554
1597
|
}
|
|
1598
|
+
function autoHibernateFor(req, cfg) {
|
|
1599
|
+
return (req.autoHibernateSec ?? cfg.autoHibernateSec) || void 0;
|
|
1600
|
+
}
|
|
1555
1601
|
async function readAllMeta(p) {
|
|
1556
1602
|
const metas = [];
|
|
1557
1603
|
for (const ref of await listMachineRefs(p)) {
|
|
@@ -1774,7 +1820,7 @@ async function createMachine(req, owner, p = paths()) {
|
|
|
1774
1820
|
const spec = { id, name, owner, ip, mac, vcpu: size.vcpu, memMib: size.memMib, base: base.version, sshKeys, sshPort };
|
|
1775
1821
|
pid = await launch(spec, tap, cfg, keysB64, p);
|
|
1776
1822
|
await openSshPort(spec, cfg);
|
|
1777
|
-
const meta = { ...spec, pid, bootId: currentBootId(), createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: "base" };
|
|
1823
|
+
const meta = { ...spec, pid, bootId: currentBootId(), createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: "base", autoHibernateSec: autoHibernateFor(req, cfg) };
|
|
1778
1824
|
await writeMachineMeta(meta, p);
|
|
1779
1825
|
recordState(meta, "running", "new", p, meta.createdAt);
|
|
1780
1826
|
return toMachine(meta, p);
|
|
@@ -1838,7 +1884,7 @@ async function restoreMachine(fromText, owner, req, cfg, p) {
|
|
|
1838
1884
|
const stale = await pushAuthorizedKeys({ owner, name, sshKeys }, m.vsock, p);
|
|
1839
1885
|
await attachTap(id, cfg);
|
|
1840
1886
|
await openSshPort(spec, cfg);
|
|
1841
|
-
const meta = { ...spec, pid, bootId: currentBootId(), createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: `snapshot:${refKey(from)}`, memSnapshots: 1, sshKeysStale: stale ? true : void 0 };
|
|
1887
|
+
const meta = { ...spec, pid, bootId: currentBootId(), createdAt: (/* @__PURE__ */ new Date()).toISOString(), source: `snapshot:${refKey(from)}`, memSnapshots: 1, sshKeysStale: stale ? true : void 0, autoHibernateSec: autoHibernateFor(req, cfg) };
|
|
1842
1888
|
await writeMachineMeta(meta, p);
|
|
1843
1889
|
recordState(meta, "running", "new", p, meta.createdAt);
|
|
1844
1890
|
return toMachine(meta, p);
|
|
@@ -1919,7 +1965,7 @@ async function cleanupProcess(meta, cfg, p) {
|
|
|
1919
1965
|
await rm(m.sock, { force: true });
|
|
1920
1966
|
await rm(m.vsock, { force: true });
|
|
1921
1967
|
}
|
|
1922
|
-
async function
|
|
1968
|
+
var stopMachine = locked(async function stopMachine2(ref, p = paths()) {
|
|
1923
1969
|
const meta = await readMeta(ref, p);
|
|
1924
1970
|
const cfg = await loadConfig(p);
|
|
1925
1971
|
const m = p.machine(ref.owner, ref.name);
|
|
@@ -1931,8 +1977,8 @@ async function stopMachine(ref, p = paths()) {
|
|
|
1931
1977
|
await writeMachineMeta(stopped, p);
|
|
1932
1978
|
recordState(stopped, "stopped", "stop", p);
|
|
1933
1979
|
return toMachine(stopped, p);
|
|
1934
|
-
}
|
|
1935
|
-
async function
|
|
1980
|
+
});
|
|
1981
|
+
var startMachine = locked(async function startMachine2(ref, p = paths()) {
|
|
1936
1982
|
const meta = await readMeta(ref, p);
|
|
1937
1983
|
const cfg = await loadConfig(p);
|
|
1938
1984
|
if (vmAlive(meta)) throw new KvboxError(`box ${shown(ref)} is already ${meta.paused ? "up, in standby (kvbox resume " + shown(ref) + ")" : "running"}`);
|
|
@@ -1957,15 +2003,15 @@ async function startMachine(ref, p = paths()) {
|
|
|
1957
2003
|
await deleteTap(meta.id, cfg).catch(() => void 0);
|
|
1958
2004
|
throw err;
|
|
1959
2005
|
}
|
|
1960
|
-
}
|
|
1961
|
-
async function
|
|
2006
|
+
});
|
|
2007
|
+
var setSshKeys = locked(async function setSshKeys2(ref, sshKeys, p = paths()) {
|
|
1962
2008
|
const meta = await readMeta(ref, p);
|
|
1963
2009
|
if (!isCloudBox(meta.owner)) throw new KvboxError(`box ${shown(ref)} is local: it trusts this host's key, there are no authorized_keys lines to replace`);
|
|
1964
2010
|
if (sshKeys.length === 0) throw new KvboxError("sshKeys: at least one authorized_keys line (the box would keep only the host's own key)");
|
|
1965
2011
|
authorizedKeysB64(await readHostPubkey(p), sshKeys);
|
|
1966
2012
|
const { meta: updated, applied } = await applySshKeys(meta, sshKeys, p);
|
|
1967
2013
|
return { machine: toMachine(updated, p), applied };
|
|
1968
|
-
}
|
|
2014
|
+
});
|
|
1969
2015
|
async function applySshKeys(meta, sshKeys, p) {
|
|
1970
2016
|
const status = statusOf(meta, p);
|
|
1971
2017
|
let updated = { ...meta, sshKeys: [...sshKeys], sshKeysStale: status === "stopped" ? void 0 : true };
|
|
@@ -1983,27 +2029,30 @@ async function applySshKeys(meta, sshKeys, p) {
|
|
|
1983
2029
|
}
|
|
1984
2030
|
async function trustCa(publicKeyText, p = paths()) {
|
|
1985
2031
|
const publicKey = parseSshPublicKey(publicKeyText);
|
|
1986
|
-
const
|
|
2032
|
+
const report2 = { publicKey, fingerprint: sshKeyFingerprint(publicKey), updated: [], pending: [], failed: [], local: 0 };
|
|
1987
2033
|
const hostKey = await readHostPubkey(p);
|
|
1988
|
-
for (const
|
|
1989
|
-
if (!isCloudBox(
|
|
1990
|
-
|
|
2034
|
+
for (const listed of await readAllMeta(p)) {
|
|
2035
|
+
if (!isCloudBox(listed.owner)) {
|
|
2036
|
+
report2.local++;
|
|
1991
2037
|
continue;
|
|
1992
2038
|
}
|
|
1993
|
-
const box = refKey(
|
|
2039
|
+
const box = refKey(listed);
|
|
1994
2040
|
try {
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2041
|
+
await withBoxLock(listed, async () => {
|
|
2042
|
+
const meta = await readMeta(listed, p);
|
|
2043
|
+
const sshKeys = [...(meta.sshKeys ?? []).filter((l) => !isCaLine(l)), caLineFor(meta.owner, publicKey)];
|
|
2044
|
+
authorizedKeysLines(hostKey, sshKeys);
|
|
2045
|
+
const { applied, why } = await applySshKeys(meta, sshKeys, p);
|
|
2046
|
+
if (applied === "now") report2.updated.push(box);
|
|
2047
|
+
else report2.pending.push(why === void 0 ? { box, when: applied } : { box, when: applied, why });
|
|
2048
|
+
});
|
|
2000
2049
|
} catch (err) {
|
|
2001
|
-
|
|
2050
|
+
report2.failed.push({ box, error: err.message });
|
|
2002
2051
|
}
|
|
2003
2052
|
}
|
|
2004
|
-
return
|
|
2053
|
+
return report2;
|
|
2005
2054
|
}
|
|
2006
|
-
async function
|
|
2055
|
+
var pauseMachine = locked(async function pauseMachine2(ref, p = paths()) {
|
|
2007
2056
|
const meta = await readMeta(ref, p);
|
|
2008
2057
|
const status = statusOf(meta, p);
|
|
2009
2058
|
if (status === "standby") throw new KvboxError(`${stateMessage(ref, status)} already`);
|
|
@@ -2019,8 +2068,8 @@ async function pauseMachine(ref, p = paths()) {
|
|
|
2019
2068
|
await writeMachineMeta(paused, p);
|
|
2020
2069
|
recordState(paused, "standby", "pause", p);
|
|
2021
2070
|
return toMachine(paused, p);
|
|
2022
|
-
}
|
|
2023
|
-
async function
|
|
2071
|
+
});
|
|
2072
|
+
var resumeMachine = locked(async function resumeMachine2(ref, p = paths()) {
|
|
2024
2073
|
const meta = await readMeta(ref, p);
|
|
2025
2074
|
const status = statusOf(meta, p);
|
|
2026
2075
|
if (status === "running") throw new KvboxError(`${stateMessage(ref, status)} already`);
|
|
@@ -2033,8 +2082,8 @@ async function resumeMachine(ref, p = paths()) {
|
|
|
2033
2082
|
await writeMachineMeta(running, p);
|
|
2034
2083
|
recordState(running, "running", "resume", p);
|
|
2035
2084
|
return toMachine(running, p);
|
|
2036
|
-
}
|
|
2037
|
-
async function
|
|
2085
|
+
});
|
|
2086
|
+
var hibernateMachine = locked(async function hibernateMachine2(ref, p = paths(), by = "hibernate") {
|
|
2038
2087
|
const meta = await readMeta(ref, p);
|
|
2039
2088
|
const cfg = await loadConfig(p);
|
|
2040
2089
|
const status = statusOf(meta, p);
|
|
@@ -2063,10 +2112,10 @@ async function hibernateMachine(ref, p = paths()) {
|
|
|
2063
2112
|
await rm(m.memBase, { force: true });
|
|
2064
2113
|
const hibernated = { ...meta, pid: null, memSnapshots: void 0, paused: void 0 };
|
|
2065
2114
|
await writeMachineMeta(hibernated, p);
|
|
2066
|
-
recordState(hibernated, "hibernated",
|
|
2115
|
+
recordState(hibernated, "hibernated", by, p);
|
|
2067
2116
|
return toMachine(hibernated, p);
|
|
2068
|
-
}
|
|
2069
|
-
async function
|
|
2117
|
+
});
|
|
2118
|
+
var wakeMachine = locked(async function wakeMachine2(ref, p = paths()) {
|
|
2070
2119
|
const meta = await readMeta(ref, p);
|
|
2071
2120
|
const cfg = await loadConfig(p);
|
|
2072
2121
|
const status = statusOf(meta, p);
|
|
@@ -2105,7 +2154,7 @@ async function wakeMachine(ref, p = paths()) {
|
|
|
2105
2154
|
await writeMachineMeta(running, p);
|
|
2106
2155
|
recordState(running, "running", "wake", p);
|
|
2107
2156
|
return toMachine(running, p);
|
|
2108
|
-
}
|
|
2157
|
+
});
|
|
2109
2158
|
async function applyAtNextBoot(meta, changes, by, cfg, p, whileStopped) {
|
|
2110
2159
|
const wasRunning = vmAlive(meta);
|
|
2111
2160
|
if (wasRunning) {
|
|
@@ -2123,7 +2172,7 @@ function assertNotHibernated(meta, p) {
|
|
|
2123
2172
|
throw new KvboxError(`${stateMessage(meta, "hibernated")}; its memory image was made with its current shape and base: kvbox wake ${shown(meta)} first`);
|
|
2124
2173
|
}
|
|
2125
2174
|
}
|
|
2126
|
-
async function
|
|
2175
|
+
var updateMachine = locked(async function updateMachine2(ref, p = paths()) {
|
|
2127
2176
|
const meta = await readMeta(ref, p);
|
|
2128
2177
|
const cfg = await loadConfig(p);
|
|
2129
2178
|
assertNotHibernated(meta, p);
|
|
@@ -2132,8 +2181,8 @@ async function updateMachine(ref, p = paths()) {
|
|
|
2132
2181
|
if (base.version === meta.base) return toMachine(meta, p);
|
|
2133
2182
|
await assertOverlayCapable(base.file, base.version);
|
|
2134
2183
|
return applyAtNextBoot(meta, { base: base.version }, "update", cfg, p);
|
|
2135
|
-
}
|
|
2136
|
-
async function
|
|
2184
|
+
});
|
|
2185
|
+
var rescaleMachine = locked(async function rescaleMachine2(ref, req, p = paths()) {
|
|
2137
2186
|
const meta = await readMeta(ref, p);
|
|
2138
2187
|
const cfg = await loadConfig(p);
|
|
2139
2188
|
assertNotHibernated(meta, p);
|
|
@@ -2154,7 +2203,14 @@ async function rescaleMachine(ref, req, p = paths()) {
|
|
|
2154
2203
|
const growTo = target.diskGib > currentDiskGib ? target.diskGib : void 0;
|
|
2155
2204
|
if (Object.keys(changes).length === 0 && growTo === void 0) return toMachine(meta, p);
|
|
2156
2205
|
return applyAtNextBoot(meta, changes, "rescale", cfg, p, growTo === void 0 ? void 0 : () => growDisk(disk, growTo));
|
|
2157
|
-
}
|
|
2206
|
+
});
|
|
2207
|
+
var setMachine = locked(async function setMachine2(ref, req, p = paths()) {
|
|
2208
|
+
const meta = await readMeta(ref, p);
|
|
2209
|
+
const next = { ...meta };
|
|
2210
|
+
if (req.autoHibernateSec !== void 0) next.autoHibernateSec = req.autoHibernateSec || void 0;
|
|
2211
|
+
await writeMachineMeta(next, p);
|
|
2212
|
+
return toMachine(next, p);
|
|
2213
|
+
});
|
|
2158
2214
|
async function growDisk(file, gib) {
|
|
2159
2215
|
await truncate(file, gib * 1024 ** 3);
|
|
2160
2216
|
await run(["e2fsck", "-f", "-p", file], { okExitCodes: [1] });
|
|
@@ -2168,13 +2224,13 @@ async function readMeta(ref, p) {
|
|
|
2168
2224
|
throw err;
|
|
2169
2225
|
}
|
|
2170
2226
|
}
|
|
2171
|
-
async function
|
|
2227
|
+
var destroyMachine = locked(async function destroyMachine2(ref, p = paths()) {
|
|
2172
2228
|
const meta = await readMeta(ref, p);
|
|
2173
2229
|
const cfg = await loadConfig(p);
|
|
2174
2230
|
await shutdown(meta, cfg, p, 2e3);
|
|
2175
2231
|
recordState(meta, "stopped", "rm", p);
|
|
2176
2232
|
await rm(p.machine(ref.owner, ref.name).dir, { recursive: true, force: true });
|
|
2177
|
-
}
|
|
2233
|
+
});
|
|
2178
2234
|
var RECOVERY_CONCURRENCY = 2;
|
|
2179
2235
|
async function settleDeadBoxes(p = paths()) {
|
|
2180
2236
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -2323,8 +2379,156 @@ async function dropRelay(p = paths()) {
|
|
|
2323
2379
|
return relay;
|
|
2324
2380
|
}
|
|
2325
2381
|
|
|
2382
|
+
// src/core/idle.ts
|
|
2383
|
+
import { readFile as readFile6 } from "node:fs/promises";
|
|
2384
|
+
var HIBERNATE_CONCURRENCY = 2;
|
|
2385
|
+
async function tapTraffic(id) {
|
|
2386
|
+
const dir = `/sys/class/net/${tapName(id)}/statistics`;
|
|
2387
|
+
try {
|
|
2388
|
+
const [rx, tx] = await Promise.all([readFile6(`${dir}/rx_bytes`, "utf8"), readFile6(`${dir}/tx_bytes`, "utf8")]);
|
|
2389
|
+
return Number(rx) + Number(tx);
|
|
2390
|
+
} catch {
|
|
2391
|
+
return null;
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
async function hibernateIfRunning(box, p) {
|
|
2395
|
+
const ref = { owner: box.owner, name: box.name };
|
|
2396
|
+
return withBoxLock(ref, async () => {
|
|
2397
|
+
if ((await getMachine(ref, p)).status !== "running") return false;
|
|
2398
|
+
await hibernateMachine(ref, p, "auto-hibernate");
|
|
2399
|
+
return true;
|
|
2400
|
+
});
|
|
2401
|
+
}
|
|
2402
|
+
function startIdleWatch(opts = {}) {
|
|
2403
|
+
const intervalMs = opts.intervalMs ?? 6e4;
|
|
2404
|
+
const p = opts.p ?? paths();
|
|
2405
|
+
const list = opts.list ?? (() => listMachines(null, p));
|
|
2406
|
+
const traffic = opts.traffic ?? tapTraffic;
|
|
2407
|
+
const hibernate = opts.hibernate ?? ((box) => hibernateIfRunning(box, p));
|
|
2408
|
+
const busy = opts.busy ?? ((box) => boxBusy(box));
|
|
2409
|
+
const now = opts.now ?? Date.now;
|
|
2410
|
+
const log = opts.log ?? ((line) => console.log(line));
|
|
2411
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2412
|
+
let timer;
|
|
2413
|
+
let stopped = false;
|
|
2414
|
+
const inflight = /* @__PURE__ */ new Set();
|
|
2415
|
+
const hibernateOne = async (box, quiet, bytes, t) => {
|
|
2416
|
+
inflight.add(box.id);
|
|
2417
|
+
try {
|
|
2418
|
+
if (await hibernate(box)) log(`box ${box.owner}/${box.name}: no network traffic for ${quiet} min; hibernated it (kvbox connect wakes it)`);
|
|
2419
|
+
seen.delete(box.id);
|
|
2420
|
+
} catch (err) {
|
|
2421
|
+
log(`warning: box ${box.owner}/${box.name}: could not hibernate it after ${quiet} min without traffic: ${err.message}`);
|
|
2422
|
+
seen.set(box.id, { bytes, since: t });
|
|
2423
|
+
} finally {
|
|
2424
|
+
inflight.delete(box.id);
|
|
2425
|
+
}
|
|
2426
|
+
};
|
|
2427
|
+
const check = async () => {
|
|
2428
|
+
const started = [];
|
|
2429
|
+
const watched = /* @__PURE__ */ new Set();
|
|
2430
|
+
for (const box of await list()) {
|
|
2431
|
+
if (!box.autoHibernateSec || box.status !== "running") continue;
|
|
2432
|
+
if (inflight.has(box.id)) {
|
|
2433
|
+
watched.add(box.id);
|
|
2434
|
+
continue;
|
|
2435
|
+
}
|
|
2436
|
+
const bytes = await traffic(box.id);
|
|
2437
|
+
if (bytes === null) continue;
|
|
2438
|
+
watched.add(box.id);
|
|
2439
|
+
const t = now();
|
|
2440
|
+
const last = seen.get(box.id);
|
|
2441
|
+
if (!last || last.bytes !== bytes || busy(box)) {
|
|
2442
|
+
seen.set(box.id, { bytes, since: t });
|
|
2443
|
+
continue;
|
|
2444
|
+
}
|
|
2445
|
+
if (t - last.since < box.autoHibernateSec * 1e3) continue;
|
|
2446
|
+
if (inflight.size >= HIBERNATE_CONCURRENCY) continue;
|
|
2447
|
+
started.push(hibernateOne(box, Math.round((t - last.since) / 6e4), bytes, t));
|
|
2448
|
+
}
|
|
2449
|
+
for (const id of seen.keys()) if (!watched.has(id)) seen.delete(id);
|
|
2450
|
+
return started;
|
|
2451
|
+
};
|
|
2452
|
+
const round2 = async () => {
|
|
2453
|
+
await Promise.all(await check());
|
|
2454
|
+
};
|
|
2455
|
+
const schedule = () => {
|
|
2456
|
+
if (stopped) return;
|
|
2457
|
+
timer = setTimeout(() => void check().catch((err) => log(`warning: idle watch: ${err.message}`)).finally(schedule), intervalMs);
|
|
2458
|
+
timer.unref();
|
|
2459
|
+
};
|
|
2460
|
+
schedule();
|
|
2461
|
+
return {
|
|
2462
|
+
stop: () => {
|
|
2463
|
+
stopped = true;
|
|
2464
|
+
clearTimeout(timer);
|
|
2465
|
+
},
|
|
2466
|
+
round: round2
|
|
2467
|
+
};
|
|
2468
|
+
}
|
|
2469
|
+
|
|
2470
|
+
// src/core/relayWatch.ts
|
|
2471
|
+
var report = { restarts: 0, lastRestartAt: null };
|
|
2472
|
+
function relayWatchReport() {
|
|
2473
|
+
return report;
|
|
2474
|
+
}
|
|
2475
|
+
async function pingRelay() {
|
|
2476
|
+
try {
|
|
2477
|
+
await run(["ping", "-c", "1", "-W", "3", RELAY_TUNNEL_GATEWAY], { timeoutMs: 5e3 });
|
|
2478
|
+
return true;
|
|
2479
|
+
} catch {
|
|
2480
|
+
return false;
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
function startRelayWatch(opts = {}) {
|
|
2484
|
+
const intervalMs = opts.intervalMs ?? 3e4;
|
|
2485
|
+
const misses = opts.misses ?? 3;
|
|
2486
|
+
const probe = opts.probe ?? pingRelay;
|
|
2487
|
+
const p = opts.p ?? paths();
|
|
2488
|
+
const log = opts.log ?? ((line) => console.log(line));
|
|
2489
|
+
const restart = opts.restart ?? (async () => netSetup(await loadConfig(p), ["relay", "restart"]));
|
|
2490
|
+
report = { restarts: 0, lastRestartAt: null };
|
|
2491
|
+
let missed = 0;
|
|
2492
|
+
let stuck = false;
|
|
2493
|
+
let timer;
|
|
2494
|
+
let stopped = false;
|
|
2495
|
+
const round2 = async () => {
|
|
2496
|
+
const cfg = await loadConfig(p).catch(() => null);
|
|
2497
|
+
if (!cfg?.relay) {
|
|
2498
|
+
missed = 0;
|
|
2499
|
+
return;
|
|
2500
|
+
}
|
|
2501
|
+
if (await probe()) {
|
|
2502
|
+
if (stuck) log(`relay ${cfg.relay.name}: the tunnel carries traffic again`);
|
|
2503
|
+
missed = 0;
|
|
2504
|
+
stuck = false;
|
|
2505
|
+
return;
|
|
2506
|
+
}
|
|
2507
|
+
if (++missed < misses) return;
|
|
2508
|
+
missed = 0;
|
|
2509
|
+
if (!stuck) log(`relay ${cfg.relay.name}: no answer through the tunnel for ${misses * intervalMs / 1e3} s; restarting it`);
|
|
2510
|
+
try {
|
|
2511
|
+
await restart();
|
|
2512
|
+
report = { restarts: report.restarts + 1, lastRestartAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
2513
|
+
} catch (err) {
|
|
2514
|
+
if (!stuck) console.error(`warning: relay ${cfg.relay.name}: could not restart the tunnel: ${err.message}`);
|
|
2515
|
+
}
|
|
2516
|
+
stuck = true;
|
|
2517
|
+
};
|
|
2518
|
+
const schedule = () => {
|
|
2519
|
+
if (stopped) return;
|
|
2520
|
+
timer = setTimeout(() => void round2().catch((err) => console.error(`warning: relay watch: ${err.message}`)).finally(schedule), intervalMs);
|
|
2521
|
+
timer.unref();
|
|
2522
|
+
};
|
|
2523
|
+
schedule();
|
|
2524
|
+
return () => {
|
|
2525
|
+
stopped = true;
|
|
2526
|
+
clearTimeout(timer);
|
|
2527
|
+
};
|
|
2528
|
+
}
|
|
2529
|
+
|
|
2326
2530
|
// src/listen.ts
|
|
2327
|
-
import { readFile as
|
|
2531
|
+
import { readFile as readFile7, rm as rm4, writeFile as writeFile5 } from "node:fs/promises";
|
|
2328
2532
|
import { createServer as createHttpsServer } from "node:https";
|
|
2329
2533
|
import { createAdaptorServer } from "@hono/node-server";
|
|
2330
2534
|
|
|
@@ -2344,7 +2548,7 @@ function versionOf(srcFileUrl, injected) {
|
|
|
2344
2548
|
}
|
|
2345
2549
|
|
|
2346
2550
|
// src/version.ts
|
|
2347
|
-
var VERSION = versionOf(import.meta.url, true ? "0.5.
|
|
2551
|
+
var VERSION = versionOf(import.meta.url, true ? "0.5.10" : void 0);
|
|
2348
2552
|
|
|
2349
2553
|
// src/caller.ts
|
|
2350
2554
|
import { createHash as createHash2, timingSafeEqual } from "node:crypto";
|
|
@@ -2431,6 +2635,10 @@ function machineRoutes() {
|
|
|
2431
2635
|
const body = RescaleMachineRequest.parse(await c.req.json().catch(() => ({})));
|
|
2432
2636
|
return c.json(await rescaleMachine(parseRef(c.req.param("name"), c.get("owner")), body));
|
|
2433
2637
|
});
|
|
2638
|
+
r.post("/:name/set", async (c) => {
|
|
2639
|
+
const body = SetMachineRequest.parse(await c.req.json().catch(() => ({})));
|
|
2640
|
+
return c.json(await setMachine(parseRef(c.req.param("name"), c.get("owner")), body));
|
|
2641
|
+
});
|
|
2434
2642
|
r.post("/:name/ssh-keys", async (c) => {
|
|
2435
2643
|
const body = SetSshKeysRequest.safeParse(await c.req.json().catch(() => ({})));
|
|
2436
2644
|
if (!body.success) throw new KvboxError("sshKeys: expected a list of authorized_keys lines");
|
|
@@ -2504,7 +2712,11 @@ async function saveSnapshot(req, owner, p = paths()) {
|
|
|
2504
2712
|
if (!NAME_RE2.test(req.name)) {
|
|
2505
2713
|
throw new KvboxError(`invalid snapshot name "${req.name}": 1-64 chars, lowercase letters, digits, '.', '_' and '-', starting with a letter or digit`);
|
|
2506
2714
|
}
|
|
2507
|
-
const
|
|
2715
|
+
const ref = parseRef(req.box, owner);
|
|
2716
|
+
return withBoxLock(ref, () => snapshotBox(ref, req, owner, p));
|
|
2717
|
+
}
|
|
2718
|
+
async function snapshotBox(ref, req, owner, p) {
|
|
2719
|
+
const box = await readBox(ref, p);
|
|
2508
2720
|
if (!vmAlive(box)) {
|
|
2509
2721
|
if (existsSync3(p.machine(box.owner, box.name).hibernateVmstate)) {
|
|
2510
2722
|
throw new KvboxError(`box ${req.box} is hibernated; a snapshot freezes a running box (kvbox wake ${req.box} first)`);
|
|
@@ -2651,7 +2863,7 @@ function createApp(opts = {}) {
|
|
|
2651
2863
|
base: await currentBaseVersion().catch(() => null),
|
|
2652
2864
|
owner: cfg.owner,
|
|
2653
2865
|
controlPlane: cp ? { name: cp.name, url: cp.url, hostUrl: `https://${cp.advertise}:${cp.port}` } : null,
|
|
2654
|
-
relay: cfg.relay ? { name: cfg.relay.name, publicIp: cfg.relay.publicIp, endpoint: cfg.relay.endpoint } : null,
|
|
2866
|
+
relay: cfg.relay ? { name: cfg.relay.name, publicIp: cfg.relay.publicIp, endpoint: cfg.relay.endpoint, ...relayWatchReport() } : null,
|
|
2655
2867
|
recovery: lastRecovery()
|
|
2656
2868
|
});
|
|
2657
2869
|
});
|
|
@@ -2692,7 +2904,7 @@ async function listen(opts = {}) {
|
|
|
2692
2904
|
let httpsServer;
|
|
2693
2905
|
const cp = cfg.controlPlane;
|
|
2694
2906
|
if (cp) {
|
|
2695
|
-
const [key, cert] = await Promise.all([
|
|
2907
|
+
const [key, cert] = await Promise.all([readFile7(p.tlsKey), readFile7(p.tlsCert)]).catch((err) => {
|
|
2696
2908
|
throw new KvboxError(`joined ${cp.name} but the host's certificate is missing (${err.message}); run kvbox-server leave, then join again`);
|
|
2697
2909
|
});
|
|
2698
2910
|
httpsServer = createAdaptorServer({
|
|
@@ -2752,6 +2964,8 @@ program.command("start", { isDefault: true }).description("serve the API on the
|
|
|
2752
2964
|
} catch (err) {
|
|
2753
2965
|
console.error(`warning: host network not ready (${err.message}); run: kvbox-server net up`);
|
|
2754
2966
|
}
|
|
2967
|
+
startRelayWatch();
|
|
2968
|
+
startIdleWatch();
|
|
2755
2969
|
const recovery2 = await bootRecovered(settled, paths());
|
|
2756
2970
|
if (recovery2) {
|
|
2757
2971
|
for (const box of recovery2.booted) console.log(`box ${box} was running when it went down; booted it again`);
|