@ouro.bot/cli 0.1.0-alpha.83 → 0.1.0-alpha.831

package/dist/heart/daemon/sanctuary-host-detached-supervisor.js

@@ -0,0 +1,494 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DetachedSanctuaryHostSupervisor = void 0;
+const node_crypto_1 = require("node:crypto");
+const node_child_process_1 = require("node:child_process");
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const runtime_1 = require("../../nerves/runtime");
+const SCHEMA_VERSION = 1;
+const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+const PERMIT_ID = /^permit-[A-Za-z0-9_-]{43}$/u;
+function digest(bytes) {
+    return `sha256:${(0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex")}`;
+}
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function assertAbsolute(value, label) {
+    if (!path.isAbsolute(value))
+        throw new Error(`Sanctuary host ${label} path must be absolute`);
+}
+function assertPinnedProgram(filePath, expectedDigest, expectedUid, kind) {
+    if (!DIGEST.test(expectedDigest))
+        throw new Error("Sanctuary host program digest is invalid");
+    const actual = kind === "system" ? fs.realpathSync(filePath) : filePath;
+    const stat = fs.lstatSync(actual);
+    if (!stat.isFile()
+        || stat.isSymbolicLink()
+        || stat.uid !== expectedUid
+        || (kind !== "source" && (stat.mode & 0o111) === 0)
+        || (stat.mode & 0o022) !== 0) {
+        throw new Error(`Sanctuary host program metadata is invalid: ${filePath}`);
+    }
+    const descriptor = fs.openSync(actual, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+    try {
+        if (digest(fs.readFileSync(descriptor)) !== expectedDigest) {
+            throw new Error(`Sanctuary host program digest changed: ${filePath}`);
+        }
+    }
+    finally {
+        fs.closeSync(descriptor);
+    }
+}
+function writePrivate(filePath, bytes) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
+    fs.chmodSync(path.dirname(filePath), 0o700);
+    const temporaryPath = `${filePath}.${process.pid}.tmp`;
+    const descriptor = fs.openSync(temporaryPath, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW, 0o600);
+    try {
+        fs.writeFileSync(descriptor, bytes, "utf8");
+        fs.fsyncSync(descriptor);
+    }
+    finally {
+        fs.closeSync(descriptor);
+    }
+    fs.renameSync(temporaryPath, filePath);
+    fs.chmodSync(filePath, 0o600);
+}
+function readPrivateJson(filePath, expectedUid) {
+    const stat = fs.lstatSync(filePath);
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.uid !== expectedUid || (stat.mode & 0o777) !== 0o600) {
+        throw new Error(`Sanctuary host supervisor record metadata is invalid: ${filePath}`);
+    }
+    const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+    try {
+        return JSON.parse(fs.readFileSync(descriptor, "utf8"));
+    }
+    finally {
+        fs.closeSync(descriptor);
+    }
+}
+function binding(value, permitId, specDigest) {
+    if (!isObject(value)
+        || value.schemaVersion !== SCHEMA_VERSION
+        || value.permitId !== permitId
+        || value.specDigest !== specDigest
+        || !Number.isSafeInteger(value.supervisorPid)
+        || value.supervisorPid < 1
+        || typeof value.bootId !== "string"
+        || value.bootId.length === 0
+        || typeof value.processStartTime !== "string"
+        || value.processStartTime.length === 0) {
+        throw new Error("Sanctuary host supervisor binding is invalid");
+    }
+    return value;
+}
+function defaultBootId() {
+    return fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
+}
+function defaultProcessStartTime(pid) {
+    const fields = fs.readFileSync(`/proc/${pid}/stat`, "utf8").trim().split(" ");
+    if (fields.length < 22 || !fields[21])
+        throw new Error("Sanctuary host supervisor process start time is unavailable");
+    return fields[21];
+}
+function defaultProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+function cgroupEmpty(cgroupPath) {
+    const events = fs.readFileSync(path.join(cgroupPath, "cgroup.events"), "utf8");
+    const populated = events.split("\n").find((line) => line.startsWith("populated "));
+    if (!populated)
+        throw new Error("Sanctuary host cgroup populated state is unavailable");
+    return populated === "populated 0";
+}
+class DetachedSanctuaryHostSupervisor {
+    #options;
+    #active = false;
+    constructor(options) {
+        for (const [label, value] of [
+            ["state root", options.stateRoot],
+            ["cgroup root", options.cgroupRoot],
+            ["supervisor", options.programPath],
+            ["launcher", options.launcherPath],
+            ["prlimit", options.prlimitPath],
+            ["setsid", options.setsidPath],
+            ["shell", options.shellPath],
+        ])
+            assertAbsolute(value, label);
+        this.#options = options;
+    }
+    verifyInstallation() {
+        this.#assertPins();
+    }
+    async execute(input) {
+        (0, runtime_1.emitNervesEvent)({ component: "daemon", event: "daemon.sanctuary_host_supervision_requested", message: "Independent Sanctuary host supervision requested" });
+        return this.#run(input, false);
+    }
+    async resume(input) {
+        return this.#run(input, true);
+    }
+    async reconcileOrphans(knownPermitIds) {
+        const known = new Set(knownPermitIds);
+        let stateEntries = [];
+        try {
+            stateEntries = fs.readdirSync(this.#options.stateRoot);
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        for (const permitId of stateEntries) {
+            if (!PERMIT_ID.test(permitId) || !known.has(permitId)) {
+                throw new Error("Sanctuary host supervisor state is unmatched");
+            }
+        }
+        for (const permitId of fs.readdirSync(this.#options.cgroupRoot)) {
+            const cgroupPath = path.join(this.#options.cgroupRoot, permitId);
+            if (!fs.lstatSync(cgroupPath).isDirectory())
+                continue;
+            if (!PERMIT_ID.test(permitId))
+                throw new Error("Sanctuary host cgroup entry is invalid");
+            if (!known.has(permitId) && !await this.#reconcileCgroup(cgroupPath)) {
+                throw new Error("Sanctuary orphan cgroup cleanup is unproven");
+            }
+        }
+    }
+    acknowledge(permitId) {
+        if (!PERMIT_ID.test(permitId))
+            throw new Error("Sanctuary host supervisor permit id is invalid");
+        const permitRoot = path.join(this.#options.stateRoot, permitId);
+        if (!fs.existsSync(permitRoot)) {
+            if (fs.existsSync(path.join(this.#options.cgroupRoot, permitId))) {
+                throw new Error("Sanctuary host supervisor cgroup is still present");
+            }
+            return;
+        }
+        const specPath = path.join(permitRoot, "spec.json");
+        const terminalPath = path.join(permitRoot, "terminal.json");
+        const lockPath = path.join(permitRoot, "supervisor.lock");
+        const acknowledgementPath = path.join(permitRoot, "acknowledgement.json");
+        const recoveryPath = path.join(permitRoot, "recovery.json");
+        if (fs.existsSync(lockPath) || fs.existsSync(path.join(this.#options.cgroupRoot, permitId))) {
+            throw new Error("Sanctuary host supervisor is not safely terminal");
+        }
+        if (fs.existsSync(acknowledgementPath)) {
+            const acknowledgement = readPrivateJson(acknowledgementPath, this.#options.expectedUid);
+            if (!isObject(acknowledgement) || acknowledgement.schemaVersion !== SCHEMA_VERSION || acknowledgement.permitId !== permitId) {
+                throw new Error("Sanctuary host supervisor acknowledgement changed");
+            }
+        }
+        else if (fs.existsSync(specPath) || fs.existsSync(terminalPath) || fs.existsSync(recoveryPath)) {
+            const specBytes = fs.readFileSync(specPath, "utf8");
+            if (fs.existsSync(terminalPath)) {
+                binding(readPrivateJson(terminalPath, this.#options.expectedUid), permitId, digest(specBytes));
+            }
+            else {
+                const recovery = readPrivateJson(recoveryPath, this.#options.expectedUid);
+                if (!isObject(recovery)
+                    || recovery.schemaVersion !== SCHEMA_VERSION
+                    || recovery.permitId !== permitId
+                    || recovery.specDigest !== digest(specBytes)) {
+                    throw new Error("Sanctuary host supervisor recovery acknowledgement changed");
+                }
+            }
+            writePrivate(acknowledgementPath, `${JSON.stringify({ schemaVersion: SCHEMA_VERSION, permitId })}\n`);
+        }
+        for (const name of ["start.json", "spawn.json", "ready.json", "terminal.json", "recovery.json", "spec.json", "acknowledgement.json"]) {
+            const filePath = path.join(permitRoot, name);
+            if (fs.existsSync(filePath))
+                fs.unlinkSync(filePath);
+        }
+        fs.rmdirSync(permitRoot);
+    }
+    async #run(input, resume) {
+        if (this.#active)
+            throw new Error("Sanctuary detached host supervisor is already active");
+        if (!PERMIT_ID.test(input.permit.permitId))
+            throw new Error("Sanctuary detached host permit id is invalid");
+        this.#active = true;
+        try {
+            if (!resume)
+                this.verifyInstallation();
+            const permitRoot = path.join(this.#options.stateRoot, input.permit.permitId);
+            const specPath = path.join(permitRoot, "spec.json");
+            const readyPath = path.join(permitRoot, "ready.json");
+            const terminalPath = path.join(permitRoot, "terminal.json");
+            const lockPath = path.join(permitRoot, "supervisor.lock");
+            const spawnPath = path.join(permitRoot, "spawn.json");
+            const startPath = path.join(permitRoot, "start.json");
+            const spec = {
+                schemaVersion: SCHEMA_VERSION,
+                permitId: input.permit.permitId,
+                permitArtifact: input.permitArtifact,
+                keyId: this.#options.keyId,
+                publicKeyPem: this.#options.publicKeyPem,
+                executable: input.executable,
+                arguments: input.arguments,
+                cwd: input.cwd,
+                environment: input.environment,
+                timeoutMs: input.timeoutMs,
+                cgroupRoot: this.#options.cgroupRoot,
+                launcherPath: this.#options.launcherPath,
+                launcherDigest: this.#options.launcherDigest,
+                prlimitPath: this.#options.prlimitPath,
+                prlimitDigest: this.#options.prlimitDigest,
+                setsidPath: this.#options.setsidPath,
+                setsidDigest: this.#options.setsidDigest,
+                shellPath: this.#options.shellPath,
+                shellDigest: this.#options.shellDigest,
+                supervisorProgramPath: this.#options.programPath,
+                supervisorProgramDigest: this.#options.programDigest,
+                readyPath,
+                terminalPath,
+                lockPath,
+                spawnPath,
+                startPath,
+            };
+            const specBytes = `${JSON.stringify(spec)}\n`;
+            const specDigest = digest(specBytes);
+            let ready;
+            if (resume && !fs.existsSync(specPath)) {
+                return this.#recoveryAttempt(input.permit.permitId);
+            }
+            if (fs.existsSync(specPath)) {
+                const existing = fs.readFileSync(specPath, "utf8");
+                if (existing !== specBytes)
+                    throw new Error("Sanctuary host supervisor specification changed");
+                if (fs.existsSync(terminalPath))
+                    return this.#terminal(terminalPath, input.permit.permitId, specDigest);
+                try {
+                    if (fs.existsSync(readyPath)) {
+                        ready = binding(readPrivateJson(readyPath, this.#options.expectedUid), input.permit.permitId, specDigest);
+                    }
+                    else if (fs.existsSync(spawnPath)) {
+                        ready = binding(readPrivateJson(spawnPath, this.#options.expectedUid), input.permit.permitId, specDigest);
+                        this.#assertProcessBinding(ready);
+                        if (!fs.existsSync(startPath))
+                            writePrivate(startPath, `${JSON.stringify({ schemaVersion: SCHEMA_VERSION, permitId: input.permit.permitId, specDigest })}\n`);
+                        ready = await this.#waitForReady(readyPath, input.permit.permitId, specDigest);
+                    }
+                    else {
+                        return this.#recoveryAttempt(input.permit.permitId, undefined, specDigest);
+                    }
+                    this.#assertLiveBinding(ready, lockPath);
+                }
+                catch {
+                    return this.#recoveryAttempt(input.permit.permitId, ready, specDigest);
+                }
+            }
+            else {
+                writePrivate(specPath, specBytes);
+                const child = (this.#options.spawn ?? ((value) => {
+                    const spawned = (0, node_child_process_1.spawn)(process.execPath, [value.programPath, "--spec", value.specPath], {
+                        detached: true,
+                        stdio: "ignore",
+                    });
+                    return spawned;
+                }))({ programPath: this.#options.programPath, specPath, readyPath, terminalPath, detached: true });
+                if (!child.pid)
+                    throw new Error("Sanctuary host supervisor failed to start");
+                child.unref();
+                ready = {
+                    schemaVersion: SCHEMA_VERSION,
+                    permitId: input.permit.permitId,
+                    specDigest,
+                    supervisorPid: child.pid,
+                    bootId: (this.#options.bootId ?? defaultBootId)(),
+                    processStartTime: (this.#options.processStartTime ?? defaultProcessStartTime)(child.pid),
+                };
+                writePrivate(spawnPath, `${JSON.stringify(ready)}\n`);
+                writePrivate(startPath, `${JSON.stringify({ schemaVersion: SCHEMA_VERSION, permitId: input.permit.permitId, specDigest })}\n`);
+                ready = await this.#waitForReady(readyPath, input.permit.permitId, specDigest);
+                if (ready.supervisorPid !== child.pid)
+                    throw new Error("Sanctuary host supervisor binding changed");
+                this.#assertLiveBinding(ready, lockPath);
+            }
+            try {
+                return await this.#waitForTerminal(terminalPath, lockPath, ready, input.permit.permitId, specDigest, input.timeoutMs + 20_000);
+            }
+            catch {
+                return this.#recoveryAttempt(input.permit.permitId, ready, specDigest);
+            }
+        }
+        finally {
+            this.#active = false;
+        }
+    }
+    #assertPins() {
+        for (const [filePath, expectedDigest, kind] of [
+            [this.#options.programPath, this.#options.programDigest, "source"],
+            [this.#options.launcherPath, this.#options.launcherDigest, "executable"],
+            [this.#options.prlimitPath, this.#options.prlimitDigest, "system"],
+            [this.#options.setsidPath, this.#options.setsidDigest, "system"],
+            [this.#options.shellPath, this.#options.shellDigest, "system"],
+        ])
+            assertPinnedProgram(filePath, expectedDigest, this.#options.expectedUid, kind);
+    }
+    #assertLiveBinding(value, lockPath) {
+        this.#assertProcessBinding(value);
+        const lock = readPrivateJson(lockPath, this.#options.expectedUid);
+        if (!isObject(lock)
+            || lock.schemaVersion !== SCHEMA_VERSION
+            || lock.permitId !== value.permitId
+            || lock.supervisorPid !== value.supervisorPid) {
+            throw new Error("Sanctuary host supervisor live binding changed");
+        }
+    }
+    #assertProcessBinding(value) {
+        const processAlive = this.#options.processAlive ?? defaultProcessAlive;
+        const bootId = this.#options.bootId ?? defaultBootId;
+        const processStartTime = this.#options.processStartTime ?? defaultProcessStartTime;
+        if (!processAlive(value.supervisorPid)
+            || bootId() !== value.bootId
+            || processStartTime(value.supervisorPid) !== value.processStartTime) {
+            throw new Error("Sanctuary host supervisor live binding changed");
+        }
+    }
+    async #waitForReady(filePath, permitId, specDigest) {
+        const attempts = Math.max(1, Math.ceil((this.#options.readyTimeoutMs ?? 5_000) / 25));
+        for (let index = 0; index < attempts; index += 1) {
+            if (fs.existsSync(filePath))
+                return binding(readPrivateJson(filePath, this.#options.expectedUid), permitId, specDigest);
+            await (this.#options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))))(25);
+        }
+        if (fs.existsSync(filePath))
+            return binding(readPrivateJson(filePath, this.#options.expectedUid), permitId, specDigest);
+        throw new Error("Sanctuary host supervisor readiness timed out");
+    }
+    async #waitForTerminal(filePath, lockPath, ready, permitId, specDigest, timeoutMs) {
+        const attempts = Math.max(1, Math.ceil(timeoutMs / 25));
+        for (let index = 0; index < attempts; index += 1) {
+            if (fs.existsSync(filePath) && !fs.existsSync(lockPath))
+                return this.#terminal(filePath, permitId, specDigest);
+            this.#assertProcessBinding(ready);
+            await (this.#options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))))(25);
+        }
+        if (fs.existsSync(filePath) && !fs.existsSync(lockPath))
+            return this.#terminal(filePath, permitId, specDigest);
+        throw new Error("Sanctuary host supervisor terminal record timed out");
+    }
+    #terminal(filePath, permitId, specDigest) {
+        const value = readPrivateJson(filePath, this.#options.expectedUid);
+        const record = binding(value, permitId, specDigest);
+        if (!isObject(value) || !("attempt" in value))
+            throw new Error("Sanctuary host supervisor terminal record is invalid");
+        const terminal = { ...record, attempt: value.attempt };
+        return terminal.attempt;
+    }
+    async #recoveryAttempt(permitId, supervisor, specDigest) {
+        const startedAt = this.#options.now?.() ?? new Date().toISOString();
+        if (supervisor && (this.#options.processAlive ?? defaultProcessAlive)(supervisor.supervisorPid)) {
+            try {
+                (this.#options.signalProcess ?? process.kill)(supervisor.supervisorPid, "SIGTERM");
+            }
+            catch (error) {
+                if (!isObject(error) || error.code !== "ESRCH")
+                    throw error;
+            }
+        }
+        const cgroupPath = path.join(this.#options.cgroupRoot, permitId);
+        let cleanup = !fs.existsSync(cgroupPath) || await this.#reconcileCgroup(cgroupPath)
+            ? "cgroup_empty"
+            : "cleanup_unproven";
+        if (!supervisor && fs.existsSync(path.join(this.#options.stateRoot, permitId, "supervisor.lock"))) {
+            cleanup = "cleanup_unproven";
+        }
+        if (cleanup === "cgroup_empty" && supervisor) {
+            const processAlive = this.#options.processAlive ?? defaultProcessAlive;
+            for (let attempt = 0; attempt < 400 && processAlive(supervisor.supervisorPid); attempt += 1) {
+                await (this.#options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))))(25);
+            }
+            if (processAlive(supervisor.supervisorPid)) {
+                cleanup = "cleanup_unproven";
+            }
+            else {
+                const lockPath = path.join(this.#options.stateRoot, permitId, "supervisor.lock");
+                if (fs.existsSync(lockPath)) {
+                    const lock = readPrivateJson(lockPath, this.#options.expectedUid);
+                    if (!isObject(lock) || lock.permitId !== permitId || lock.supervisorPid !== supervisor.supervisorPid) {
+                        throw new Error("Sanctuary host supervisor recovery lock changed");
+                    }
+                    fs.unlinkSync(lockPath);
+                }
+            }
+        }
+        if (cleanup === "cgroup_empty" && specDigest) {
+            writePrivate(path.join(this.#options.stateRoot, permitId, "recovery.json"), `${JSON.stringify({ schemaVersion: SCHEMA_VERSION, permitId, specDigest })}\n`);
+        }
+        return {
+            startedAt,
+            completedAt: this.#options.now?.() ?? new Date().toISOString(),
+            exitCode: null,
+            signal: null,
+            timedOut: false,
+            cancelled: false,
+            outputOverflow: false,
+            stdout: "",
+            stderr: "",
+            stdoutDigest: digest(""),
+            stderrDigest: digest(""),
+            stdoutBytes: 0,
+            stderrBytes: 0,
+            cleanup,
+            containment: "unprovable_after_approved_root_migration",
+            verificationBefore: null,
+            verificationAfter: null,
+        };
+    }
+    async #reconcileCgroup(cgroupPath) {
+        if (this.#options.killCgroup)
+            this.#options.killCgroup(cgroupPath);
+        else
+            fs.writeFileSync(path.join(cgroupPath, "cgroup.kill"), "1", "utf8");
+        for (let attempt = 0; attempt < 400; attempt += 1) {
+            if ((this.#options.cgroupEmpty ?? cgroupEmpty)(cgroupPath)) {
+                (this.#options.removeCgroup ?? fs.rmdirSync)(cgroupPath);
+                return true;
+            }
+            await (this.#options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))))(25);
+        }
+        return false;
+    }
+}
+exports.DetachedSanctuaryHostSupervisor = DetachedSanctuaryHostSupervisor;