@cjhyy/code-shell-server 0.9.47 → 0.9.49

package/dist/panels/background-services.js

@@ -0,0 +1,1159 @@
+import { randomUUID, createHash } from "node:crypto";
+import { isDeepStrictEqual } from "node:util";
+import { PanelBridgeError } from "./bridge-contract.js";
+import { mkdir } from "node:fs/promises";
+import { isAbsolute } from "node:path";
+import { panelExecutionProject, panelExecutionGate, } from "./execution-gate.js";
+import { BackgroundServiceStore, identityGroupStatus, serviceSystemIdentity, upgradeLocalServiceIdentity, } from "./background-service-store.js";
+import { spawnServiceProcess } from "./background-service-process.js";
+export const panelBackgroundServiceMethods = [
+    "services.get",
+    "services.setDesired",
+    "services.configure",
+    "services.attach",
+    "services.detach",
+    "services.read",
+    "services.control",
+];
+export const backgroundServiceLimits = {
+    maxRunning: 4,
+    maxPerApp: 2,
+    maxPerScope: 1,
+    healthyResetMs: 60_000,
+    maxRetainedScopes: 4096,
+    maxChannels: 4,
+    maxChannelBytes: 1 << 20,
+    maxConfigBytes: 64 << 10,
+    minCommitIntervalMs: 1000,
+    drainTimeoutMs: 10_000,
+    maxLifetimeMs: 21_600_000,
+    maxConsecutiveFailures: 5,
+    maxServices: 128,
+};
+const clone = (value) => structuredClone(value);
+function state(record) {
+    const { desired, desiredRevision, configRevision, phase, generation, revision, startedAt, restarts, lastError, controls, } = record;
+    return clone({
+        desired,
+        desiredRevision,
+        configRevision,
+        phase,
+        generation,
+        revision,
+        startedAt,
+        restarts,
+        lastError,
+        controls,
+    });
+}
+function requestKey(value) {
+    if (typeof value !== "string" || !value.length || value.length > 256)
+        throw new Error("SERVICE_REQUEST_KEY_REQUIRED");
+    return value;
+}
+function validateRecord(record) {
+    const counter = (value) => Number.isSafeInteger(value) && value >= 0;
+    const identifier = (value) => typeof value === "string" && /^[a-z][a-z0-9-]{0,63}$/.test(value);
+    if (!record.scope ||
+        !identifier(record.scope.appId) ||
+        !identifier(record.scope.name) ||
+        !isAbsolute(record.scope.projectPath) ||
+        ![
+            record.commit,
+            record.generation,
+            record.desiredRevision,
+            record.configRevision,
+            record.restarts,
+        ].every(counter) ||
+        [record.stopRequestedGeneration, record.failureGeneration].some((value) => value !== undefined && (!counter(value) || value > record.generation)) ||
+        !["off", "on-demand", "always"].includes(record.desired) ||
+        ![
+            "stopped",
+            "starting",
+            "running",
+            "draining",
+            "backoff",
+            "failed",
+            "paused",
+            "revoked",
+            "stop-unconfirmed",
+            "needs-consent",
+        ].includes(record.phase) ||
+        typeof record.incarnation !== "string" ||
+        typeof record.revision !== "string" ||
+        !record.channels ||
+        Object.keys(record.channels).length > 4 ||
+        !Array.isArray(record.controls) ||
+        record.controls.length > 32 ||
+        !record.definition ||
+        !record.definition.channels ||
+        (record.process !== null &&
+            (!record.process ||
+                !counter(record.process.pid) ||
+                record.process.pid <= 1 ||
+                typeof record.process.start !== "string" ||
+                typeof record.process.host !== "string")) ||
+        Object.values(record.channels).some((channel) => !counter(channel.seq) || !counter(channel.generation) || !counter(channel.committedAt)))
+        throw new Error("SERVICE_RECORD_CORRUPT");
+}
+function canonicalConfig(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalConfig).join(",")}]`;
+    if (value && typeof value === "object")
+        return `{${Object.keys(value)
+            .sort()
+            .map((key) => `${JSON.stringify(key)}:${canonicalConfig(value[key])}`)
+            .join(",")}}`;
+    return JSON.stringify(value);
+}
+/** Host-only coordinator. Package code never receives the authoritative storage path. */
+export class PanelBackgroundService {
+    options;
+    store;
+    records = new Map();
+    processes = new Map();
+    attachments = new Map();
+    pending = new Map();
+    failedCommits = new Map();
+    emergency;
+    storageRetryAt = 0;
+    publishTimers = new Map();
+    idleSince = new Map();
+    heldForUpdate = new Set();
+    tail = Promise.resolve();
+    timer;
+    initialized = false;
+    initialization;
+    initializationRetryAt = 0;
+    initializationFailures = 0;
+    initializationError;
+    recoveringStorage = false;
+    lastAvailability = false;
+    availabilityRevision = 0;
+    availabilityListeners = new Set();
+    stopping = false;
+    ticking = false;
+    unregister;
+    unregisterMutation;
+    unregisterDrain;
+    now;
+    constructor(options) {
+        this.options = options;
+        this.store = new BackgroundServiceStore(options.rootDir, options.fault, options.identities);
+        this.now = options.now ?? Date.now;
+    }
+    exclusive(work) {
+        const result = this.tail.then(work);
+        this.tail = result.catch(() => { });
+        return result;
+    }
+    isAvailable() {
+        return (this.initialized &&
+            !this.initialization &&
+            !this.recoveringStorage &&
+            !this.stopping &&
+            this.store.available());
+    }
+    /** Subscribe to transitions; callers read isAvailable() for the initial snapshot. */
+    subscribeAvailability(listener) {
+        this.availabilityListeners.add(listener);
+        return () => {
+            this.availabilityListeners.delete(listener);
+        };
+    }
+    notifyAvailability() {
+        const available = this.isAvailable();
+        if (available === this.lastAvailability)
+            return;
+        this.lastAvailability = available;
+        const revision = ++this.availabilityRevision;
+        for (const listener of [...this.availabilityListeners]) {
+            // An observer can synchronously shut down or unsubscribe another observer.
+            // A nested transition supersedes this delivery even if its value returns.
+            if (revision !== this.availabilityRevision || this.isAvailable() !== available)
+                break;
+            if (!this.availabilityListeners.has(listener))
+                continue;
+            try {
+                listener(available);
+            }
+            catch {
+                /* observers cannot abort custody or durable lifecycle work */
+            }
+        }
+    }
+    groupStatus(identity) {
+        return (this.options.groupStatus ?? identityGroupStatus)(identity);
+    }
+    legacyLocalityDiagnostic(identity) {
+        const system = this.options.identities
+            ? this.options.identities.current(process.pid)
+            : serviceSystemIdentity();
+        if (!identity.machine && identity.boot && system?.boot && identity.boot !== system.boot)
+            return "SERVICE_LEGACY_LOCALITY_UNCONFIRMED; legacy process record has no machine proof across boots; ownership must be verified before retry";
+    }
+    markStopUnconfirmed(key, record) {
+        const diagnostic = record.process && this.legacyLocalityDiagnostic(record.process);
+        if (record.phase !== "stop-unconfirmed" || (diagnostic && record.lastError !== diagnostic))
+            this.emit(this.commit(key, (next) => {
+                next.activeGeneration = null;
+                next.phase = "stop-unconfirmed";
+                if (diagnostic)
+                    next.lastError = diagnostic;
+            }));
+    }
+    startTimer() {
+        if (this.timer || this.stopping)
+            return;
+        this.timer = setInterval(() => {
+            if (this.ticking)
+                return;
+            this.ticking = true;
+            void this.tick()
+                .catch(() => { })
+                .finally(() => {
+                this.ticking = false;
+            });
+        }, 1000);
+        this.timer.unref();
+    }
+    async initialize() {
+        if (this.stopping)
+            throw new Error("SERVICE_UNAVAILABLE");
+        if (this.initialization)
+            return this.initialization;
+        if (this.initialized)
+            return;
+        // One supervisor timer also retries transient lock/inspection failures. It
+        // never releases custody belonging to an already initialized coordinator.
+        this.startTimer();
+        if (this.now() < this.initializationRetryAt)
+            throw this.initializationError;
+        const attempt = this.exclusive(async () => {
+            if (this.stopping)
+                throw new Error("SERVICE_UNAVAILABLE");
+            try {
+                this.store.acquire();
+                const records = this.store.all();
+                for (const record of records)
+                    validateRecord(record);
+                for (const record of records)
+                    this.records.set(this.store.key(record.scope), record);
+                this.unregister = (this.options.gate ?? panelExecutionGate).register(() => [...this.records.values()]
+                    .filter((record) => record.process !== null)
+                    .map((record) => ({
+                    ...record.scope,
+                    holder: {
+                        kind: "service",
+                        label: record.phase === "stop-unconfirmed"
+                            ? `${record.scope.name}: stop-unconfirmed; ${this.legacyLocalityDiagnostic(record.process) ?? `original process group ${record.process.pid} must exit before retry`}`
+                            : record.scope.name,
+                    },
+                })));
+                this.unregisterDrain = (this.options.gate ?? panelExecutionGate).registerServiceDrain((matches) => this.drain(matches));
+                this.unregisterMutation = (this.options.gate ?? panelExecutionGate).onMutationSettled(() => this.resumeAfterUpdate());
+                this.initialized = true;
+                this.initializationFailures = 0;
+                this.initializationRetryAt = 0;
+                this.initializationError = undefined;
+                for (const key of this.records.keys()) {
+                    try {
+                        await this.reconcile(key);
+                    }
+                    catch {
+                        /* one scope cannot release other scopes' custody */
+                    }
+                }
+            }
+            catch (error) {
+                if (!this.initialized && !this.processes.size) {
+                    this.unregister?.();
+                    this.unregisterMutation?.();
+                    this.unregisterDrain?.();
+                    this.store.release();
+                    this.initializationError = error;
+                    this.initializationFailures++;
+                    this.initializationRetryAt =
+                        this.now() +
+                            Math.min(60_000, 5000 * 2 ** Math.min(16, this.initializationFailures - 1));
+                }
+                throw error;
+            }
+        });
+        this.initialization = attempt;
+        try {
+            await attempt;
+        }
+        finally {
+            if (this.initialization === attempt)
+                this.initialization = undefined;
+            this.notifyAvailability();
+        }
+    }
+    commit(key, mutate) {
+        if (!this.store.available())
+            throw new Error("SERVICE_STORAGE_UNAVAILABLE");
+        const next = clone(this.records.get(key));
+        mutate(next);
+        next.commit++;
+        validateRecord(next);
+        try {
+            this.store.commit(key, next);
+        }
+        catch (error) {
+            if (!this.failedCommits.has(key))
+                this.failedCommits.set(key, next);
+            this.beginEmergencyStop();
+            throw error;
+        }
+        this.records.set(key, next);
+        return next;
+    }
+    beginEmergencyStop() {
+        this.notifyAvailability();
+        this.pending.clear();
+        for (const timer of this.publishTimers.values())
+            clearTimeout(timer);
+        this.publishTimers.clear();
+        // No disk write is a prerequisite to stopping an owned executor.
+        this.emergency ??= Promise.all([...this.processes].map(async ([key, actual]) => {
+            const confirmed = await actual.process.terminate(0).catch(() => false);
+            if (confirmed && this.processes.get(key) === actual)
+                this.processes.delete(key);
+        })).then(() => { });
+    }
+    async recoverStorage() {
+        this.beginEmergencyStop();
+        await this.emergency;
+        if (this.now() < this.storageRetryAt)
+            return;
+        this.storageRetryAt = this.now() + 5000;
+        // Do not re-enable writes while a previous owned process may still publish.
+        if (this.processes.size) {
+            this.emergency = undefined;
+            return;
+        }
+        this.recoveringStorage = true;
+        try {
+            const disk = this.store.recover(validateRecord);
+            for (const record of disk) {
+                validateRecord(record);
+                this.records.set(this.store.key(record.scope), record);
+            }
+            for (const [key, candidate] of this.failedCommits) {
+                const durable = this.records.get(key);
+                if (!durable || candidate.commit > durable.commit)
+                    this.records.set(key, candidate);
+            }
+            for (const [key, record] of this.records) {
+                const unconfirmed = record.process && this.groupStatus(record.process) !== "dead";
+                const next = this.commit(key, (value) => {
+                    value.activeGeneration = null;
+                    if (unconfirmed) {
+                        value.phase = "stop-unconfirmed";
+                        value.lastError = "Storage recovered; previous process group exit remains unconfirmed";
+                        return;
+                    }
+                    value.process = null;
+                    value.worker = null;
+                    const settled = this.exitedPhase(value);
+                    if (value.paused ||
+                        ["paused", "failed", "revoked", "needs-consent"].includes(value.phase) ||
+                        ["paused", "failed", "revoked"].includes(settled)) {
+                        if (value.paused)
+                            value.phase = "paused";
+                        else if (["paused", "failed", "revoked"].includes(settled))
+                            value.phase = settled;
+                    }
+                    else if (value.desired === "off")
+                        value.phase = value.stopPhase ?? "stopped";
+                    else {
+                        value.phase = "backoff";
+                        value.nextStartAt = Math.max(value.nextStartAt, this.now() + 5000);
+                        value.lastError =
+                            "Recovered after storage failure; previous executors confirmed exited";
+                    }
+                });
+                this.emit(next);
+            }
+            this.failedCommits.clear();
+            this.emergency = undefined;
+        }
+        catch {
+            this.store.quarantine();
+            this.beginEmergencyStop();
+        }
+        finally {
+            this.recoveringStorage = false;
+            this.notifyAvailability();
+        }
+    }
+    emit(record, channel) {
+        try {
+            this.options.onEvent?.(record.scope, {
+                name: record.scope.name,
+                state: state(record),
+                generation: record.generation,
+                ...(channel ? { channel, seq: record.channels[channel].seq } : {}),
+            });
+        }
+        catch {
+            /* observers cannot abort committed lifecycle work */
+        }
+    }
+    async ensure(scope) {
+        if (!this.initialized || this.stopping || !this.store.available())
+            throw new Error("SERVICE_UNAVAILABLE");
+        if (!/^[a-z][a-z0-9-]{0,63}$/.test(scope.name))
+            throw new Error("SERVICE_INVALID_NAME");
+        scope = { ...scope, projectPath: panelExecutionProject(scope.projectPath) };
+        const description = await this.options.describe(scope);
+        const key = this.store.key(scope);
+        if (!this.records.has(key)) {
+            if (this.records.size >= backgroundServiceLimits.maxRetainedScopes)
+                throw new Error("SERVICE_RECORD_CAPACITY");
+            this.compactInactive();
+            const record = this.emptyRecord(scope, description);
+            record.commit = 1;
+            try {
+                this.store.commit(key, record);
+            }
+            catch (error) {
+                if (!this.failedCommits.has(key))
+                    this.failedCommits.set(key, record);
+                this.beginEmergencyStop();
+                throw error;
+            }
+            this.records.set(key, record);
+        }
+        return key;
+    }
+    emptyRecord(scope, description) {
+        return {
+            format: "panel.background-service/1",
+            scope,
+            incarnation: randomUUID(),
+            commit: 0,
+            desired: "off",
+            desiredRevision: 0,
+            configRevision: 0,
+            phase: "stopped",
+            generation: 0,
+            activeGeneration: null,
+            revision: description.revision,
+            startedAt: null,
+            restarts: 0,
+            lastError: null,
+            controls: [],
+            process: null,
+            config: {},
+            channels: {},
+            definition: description.definition,
+            nextStartAt: 0,
+            paused: false,
+        };
+    }
+    compactInactive() {
+        const full = [...this.records].filter(([, record]) => !record.compacted);
+        if (full.length < backgroundServiceLimits.maxServices)
+            return;
+        const inactive = full
+            .filter(([key, record]) => record.desired === "off" &&
+            !record.process &&
+            ![...this.attachments.values()].some((value) => value.key === key))
+            .sort((a, b) => a[1].commit - b[1].commit);
+        if (!inactive.length)
+            throw new Error("SERVICE_RECORD_CAPACITY");
+        const [key] = inactive[0];
+        const compacted = this.commit(key, (record) => {
+            // Preserve incarnation, every monotonic counter, configuration and control receipts.
+            // Only obsolete snapshot payloads are compacted; null is a new publication, not ABA.
+            record.compacted = true;
+            for (const value of Object.values(record.channels)) {
+                value.value = null;
+                value.seq++;
+                value.committedAt = this.now();
+            }
+        });
+        for (const channel of Object.keys(compacted.channels))
+            this.emit(compacted, channel);
+    }
+    async inspect(scope) {
+        if (!this.initialized || this.stopping)
+            throw new Error("SERVICE_UNAVAILABLE");
+        if (!/^[a-z][a-z0-9-]{0,63}$/.test(scope.name))
+            throw new Error("SERVICE_INVALID_NAME");
+        scope = { ...scope, projectPath: panelExecutionProject(scope.projectPath) };
+        const existing = this.records.get(this.store.key(scope));
+        let description;
+        try {
+            description = await this.options.describe(scope);
+        }
+        catch (error) {
+            if (!existing ||
+                (error instanceof PanelBridgeError && ["REVOKED", "PERMISSION_DENIED"].includes(error.code)))
+                throw error;
+            return { ...existing, lastError: String(error).slice(0, 2048) };
+        }
+        const record = existing ?? this.emptyRecord(scope, description);
+        if (!this.store.available())
+            return {
+                ...record,
+                phase: "stop-unconfirmed",
+                lastError: "SERVICE_STORAGE_UNAVAILABLE; owned executors are being stopped",
+            };
+        return record;
+    }
+    receipt(record, key, action) {
+        const previous = record.controls.find((control) => control.requestKey === key);
+        if (previous && previous.action !== action)
+            throw new PanelBridgeError("SERVICE_REQUEST_KEY_CONFLICT", "SERVICE_REQUEST_KEY_CONFLICT");
+        return !!previous;
+    }
+    remember(record, key, action, result = "committed") {
+        record.controls = [
+            ...record.controls,
+            { requestKey: key, action, result, at: this.now() },
+        ].slice(-32);
+    }
+    get(scope) {
+        return this.exclusive(async () => state(await this.inspect(scope)));
+    }
+    read(scope, channel, afterSeq) {
+        return this.exclusive(async () => {
+            const record = await this.inspect(scope);
+            if (!Object.hasOwn(record.definition.channels, channel))
+                throw new Error("SERVICE_UNKNOWN_CHANNEL");
+            const snapshot = record.channels[channel];
+            if (!snapshot)
+                return { generation: record.generation, seq: 0, committedAt: null, value: null };
+            return afterSeq === snapshot.seq ? { unchanged: true } : clone(snapshot);
+        });
+    }
+    setDesired(scope, input) {
+        return this.exclusive(async () => {
+            if (!["off", "on-demand", "always"].includes(input.desired))
+                throw new Error("SERVICE_INVALID_DESIRED");
+            const key = await this.ensure(scope), receipt = requestKey(input.requestKey), current = this.records.get(key);
+            if (this.receipt(current, receipt, `desired:${input.desired}`))
+                return state(current);
+            if (input.expectedRevision !== current.desiredRevision)
+                throw new PanelBridgeError("SERVICE_REVISION_CONFLICT", "SERVICE_REVISION_CONFLICT");
+            const wantsStart = input.desired === "always" ||
+                (input.desired === "on-demand" &&
+                    [...this.attachments.values()].some((attachment) => attachment.key === key));
+            if (wantsStart && !current.process)
+                this.assertCapacity(current);
+            this.commit(key, (record) => {
+                record.desired = input.desired;
+                record.desiredRevision++;
+                record.paused = false;
+                if (input.desired !== "off") {
+                    record.restarts = 0;
+                    record.nextStartAt = 0;
+                    if (!record.process)
+                        record.phase = "stopped";
+                }
+                if (input.desired === "off") {
+                    record.stopRequestedGeneration = record.generation;
+                    record.stopPhase = "stopped";
+                    record.activeGeneration = null;
+                    record.phase = record.process ? "draining" : "stopped";
+                }
+                this.remember(record, receipt, `desired:${input.desired}`);
+            });
+            if (input.desired === "off")
+                await this.stop(key, "stopped");
+            else
+                await this.reconcile(key);
+            const result = this.records.get(key);
+            this.emit(result);
+            return state(result);
+        });
+    }
+    configure(scope, input) {
+        return this.exclusive(async () => {
+            if (!input.config ||
+                typeof input.config !== "object" ||
+                Array.isArray(input.config) ||
+                Buffer.byteLength(JSON.stringify(input.config)) > backgroundServiceLimits.maxConfigBytes)
+                throw new Error("SERVICE_INVALID_CONFIG");
+            const key = await this.ensure(scope), receipt = requestKey(input.requestKey), current = this.records.get(key);
+            const action = `configure:${createHash("sha256").update(canonicalConfig(input.config)).digest("hex")}`;
+            if (this.receipt(current, receipt, action))
+                return {
+                    configRevision: Number(current.controls.find((control) => control.requestKey === receipt).result),
+                };
+            if (isDeepStrictEqual(current.config, input.config)) {
+                this.commit(key, (record) => this.remember(record, receipt, action, String(record.configRevision)));
+                return { configRevision: current.configRevision };
+            }
+            if (input.expectedConfigRevision !== current.configRevision)
+                throw new PanelBridgeError("SERVICE_CONFIG_CONFLICT", "SERVICE_CONFIG_CONFLICT");
+            const next = this.commit(key, (record) => {
+                record.config = clone(input.config);
+                record.configRevision++;
+                this.remember(record, receipt, action, String(record.configRevision));
+            });
+            this.processes
+                .get(key)
+                ?.process.send({ t: "config", config: next.config, configRevision: next.configRevision });
+            return { configRevision: next.configRevision };
+        });
+    }
+    attach(scope, guestId) {
+        return this.exclusive(async () => {
+            const key = await this.ensure(scope);
+            const existing = [...this.attachments].find(([, value]) => value.key === key && value.guestId === guestId);
+            const attachId = existing?.[0] ?? randomUUID();
+            if (!existing) {
+                if (this.attachments.size >= 512)
+                    throw new Error("SERVICE_ATTACH_CAPACITY");
+                this.attachments.set(attachId, { key, guestId });
+            }
+            this.idleSince.delete(key);
+            await this.reconcile(key);
+            return { attachId, state: state(this.records.get(key)) };
+        });
+    }
+    detach(attachId, guestId) {
+        return this.exclusive(() => {
+            if (this.attachments.get(attachId)?.guestId === guestId)
+                this.attachments.delete(attachId);
+            return { detached: true };
+        });
+    }
+    detachGuest(guestId) {
+        for (const [id, attachment] of this.attachments)
+            if (attachment.guestId === guestId)
+                this.attachments.delete(id);
+    }
+    control(scope, input) {
+        return this.exclusive(async () => {
+            if (!["poke", "pause", "resume", "stop", "restart"].includes(input.action))
+                throw new Error("SERVICE_INVALID_ACTION");
+            const key = await this.ensure(scope), receipt = requestKey(input.requestKey), current = this.records.get(key);
+            if (this.receipt(current, receipt, input.action))
+                return state(current);
+            this.commit(key, (record) => {
+                if (input.action === "stop") {
+                    record.desired = "off";
+                    record.desiredRevision++;
+                }
+                if (input.action === "pause")
+                    record.paused = true;
+                if (input.action === "resume" || input.action === "restart") {
+                    record.paused = false;
+                    record.restarts = 0;
+                    record.nextStartAt = 0;
+                    if (!record.process)
+                        record.phase = "stopped";
+                }
+                if (["stop", "pause", "restart"].includes(input.action)) {
+                    record.stopRequestedGeneration = record.generation;
+                    record.stopPhase = input.action === "pause" ? "paused" : "stopped";
+                    record.activeGeneration = null;
+                    record.phase = record.process ? "draining" : "stopped";
+                }
+                this.remember(record, receipt, input.action);
+            });
+            if (["stop", "pause", "restart"].includes(input.action))
+                await this.stop(key, input.action === "pause" ? "paused" : "stopped");
+            await this.reconcile(key);
+            if (input.action === "poke")
+                this.processes.get(key)?.process.send({ t: "poke" });
+            const result = this.records.get(key);
+            this.emit(result);
+            return state(result);
+        });
+    }
+    async stop(key, phase) {
+        const current = this.records.get(key);
+        this.pending.delete(key);
+        const timer = this.publishTimers.get(key);
+        if (timer)
+            clearTimeout(timer);
+        this.publishTimers.delete(key);
+        let failure;
+        try {
+            this.emit(this.commit(key, (record) => {
+                record.activeGeneration = null;
+                record.stopRequestedGeneration = record.generation;
+                record.stopPhase = phase;
+                record.phase = record.process ? "draining" : phase;
+            }));
+        }
+        catch (error) {
+            failure = error;
+        }
+        const running = this.processes.get(key);
+        const confirmed = running
+            ? await running.process.terminate(failure ? 0 : backgroundServiceLimits.drainTimeoutMs)
+            : !current.process || this.groupStatus(current.process) === "dead";
+        if (confirmed)
+            this.processes.delete(key);
+        if (failure)
+            throw failure;
+        this.emit(this.commit(key, (record) => {
+            record.phase = confirmed ? phase : "stop-unconfirmed";
+            if (confirmed) {
+                record.process = null;
+                record.worker = null;
+            }
+        }));
+    }
+    assertCapacity(record) {
+        const occupied = [...this.records.values()].filter((value) => value.process);
+        if (occupied.length >= backgroundServiceLimits.maxRunning ||
+            occupied.filter((value) => value.scope.appId === record.scope.appId &&
+                value.scope.projectPath === record.scope.projectPath).length >= backgroundServiceLimits.maxPerScope ||
+            occupied.filter((value) => value.scope.appId === record.scope.appId).length >=
+                backgroundServiceLimits.maxPerApp)
+            throw new PanelBridgeError("SERVICE_CAPACITY", "SERVICE_CAPACITY");
+    }
+    exitedPhase(record) {
+        if (record.stopRequestedGeneration === record.generation) {
+            // Internal failure cleanup requests custody stop too. Its terminal state
+            // still follows the failure budget, unlike explicit stop/pause/revoke.
+            if (record.stopPhase === "backoff")
+                return record.restarts >= backgroundServiceLimits.maxConsecutiveFailures
+                    ? "failed"
+                    : "backoff";
+            return record.stopPhase ?? (record.paused ? "paused" : "stopped");
+        }
+        if (record.failureGeneration === record.generation)
+            return record.restarts >= backgroundServiceLimits.maxConsecutiveFailures
+                ? "failed"
+                : "backoff";
+        return record.paused
+            ? "paused"
+            : ["failed", "revoked", "needs-consent"].includes(record.phase)
+                ? record.phase
+                : "stopped";
+    }
+    async reconcile(key) {
+        if (this.stopping || this.heldForUpdate.has(key))
+            return;
+        if (!this.store.available()) {
+            await this.recoverStorage();
+            return;
+        }
+        let record = this.records.get(key);
+        const upgradedProcess = record.process && upgradeLocalServiceIdentity(record.process);
+        const upgradedWorker = record.worker && upgradeLocalServiceIdentity(record.worker);
+        if (upgradedProcess !== record.process || upgradedWorker !== record.worker) {
+            record = this.commit(key, (next) => {
+                next.process = upgradedProcess;
+                next.worker = upgradedWorker;
+            });
+        }
+        if (record.process && record.phase === "stop-unconfirmed" && this.processes.has(key)) {
+            if (this.groupStatus(record.process) === "dead")
+                this.processes.delete(key);
+            else {
+                this.markStopUnconfirmed(key, record);
+                return;
+            }
+        }
+        if (record.process && !this.processes.has(key)) {
+            if (this.groupStatus(record.process) !== "dead") {
+                this.markStopUnconfirmed(key, record);
+                return;
+            }
+            record = this.commit(key, (next) => {
+                next.process = null;
+                next.worker = null;
+                next.activeGeneration = null;
+                next.phase = this.exitedPhase(next);
+            });
+            this.emit(record);
+        }
+        const attached = [...this.attachments.values()].some((value) => value.key === key);
+        if (attached)
+            this.idleSince.delete(key);
+        else if (!this.idleSince.has(key))
+            this.idleSince.set(key, this.now());
+        const shouldRun = !record.paused &&
+            (record.desired === "always" ||
+                (record.desired === "on-demand" &&
+                    (attached ||
+                        (this.processes.has(key) &&
+                            this.now() - this.idleSince.get(key) < record.definition.idleGraceMs))));
+        if (!shouldRun) {
+            if (record.process)
+                await this.stop(key, record.paused ? "paused" : "stopped");
+            return;
+        }
+        if (!record.process &&
+            (record.restarts >= backgroundServiceLimits.maxConsecutiveFailures ||
+                ["failed", "revoked", "needs-consent"].includes(record.phase)))
+            return;
+        if (!record.process && record.nextStartAt > this.now())
+            return;
+        let description;
+        try {
+            description = await this.options.describe(record.scope);
+        }
+        catch (error) {
+            const revoked = error instanceof PanelBridgeError && ["REVOKED", "PERMISSION_DENIED"].includes(error.code);
+            if (revoked) {
+                if (record.phase !== "revoked") {
+                    this.commit(key, (next) => {
+                        next.desired = "off";
+                        next.desiredRevision++;
+                        next.stopRequestedGeneration = next.generation;
+                        next.stopPhase = "revoked";
+                        next.activeGeneration = null;
+                    });
+                    await this.stop(key, "revoked");
+                }
+            }
+            else {
+                if (record.process)
+                    await this.stop(key, "backoff");
+                this.failAttempt(key, error);
+            }
+            return;
+        }
+        if (this.processes.has(key)) {
+            if (record.revision !== description.revision ||
+                this.now() - record.startedAt >= record.definition.maxLifetimeMs)
+                await this.stop(key, "stopped");
+            else {
+                if (record.phase === "running" &&
+                    record.restarts &&
+                    this.now() - record.startedAt >= backgroundServiceLimits.healthyResetMs)
+                    this.emit(this.commit(key, (next) => {
+                        next.restarts = 0;
+                        next.lastError = null;
+                    }));
+                return;
+            }
+            record = this.records.get(key);
+            if (record.process)
+                return;
+        }
+        if (record.restarts >= backgroundServiceLimits.maxConsecutiveFailures ||
+            record.nextStartAt > this.now())
+            return;
+        try {
+            this.assertCapacity(record);
+        }
+        catch (error) {
+            this.emit(this.commit(key, (next) => {
+                next.phase = "backoff";
+                next.nextStartAt = this.now() + 5000;
+                next.lastError = error instanceof Error ? error.message : String(error);
+            }));
+            return;
+        }
+        const gate = this.options.gate ?? panelExecutionGate;
+        const release = gate.enter({
+            ...record.scope,
+            holder: { kind: "service", label: record.scope.name },
+        });
+        try {
+            const executable = typeof description.executable === "function"
+                ? await description.executable()
+                : description.executable;
+            if (this.stopping)
+                return;
+            await mkdir(description.cwd, { recursive: true, mode: 0o700 });
+            if (this.stopping)
+                return;
+            if (record.compacted)
+                this.compactInactive();
+            record = this.commit(key, (next) => {
+                next.generation++;
+                delete next.stopRequestedGeneration;
+                delete next.failureGeneration;
+                delete next.stopPhase;
+                next.activeGeneration = next.generation;
+                next.phase = "starting";
+                next.revision = description.revision;
+                next.definition = description.definition;
+                next.startedAt = this.now();
+                next.compacted = false;
+                next.lastError = null;
+            });
+            this.emit(record);
+            const generation = record.generation;
+            const process = await (this.options.spawn ?? spawnServiceProcess)({
+                ...description,
+                executable,
+                maxLineBytes: Math.max(...Object.values(description.definition.channels).map((channel) => channel.maxBytes)) + 4096,
+                onLine: (message) => this.onLine(key, generation, message),
+                onExit: (error) => {
+                    void this.exclusive(() => this.onExit(key, generation, error)).catch(() => { });
+                },
+            });
+            // Commit the verified custody identity BEFORE the guardian may launch package code.
+            try {
+                await this.options.afterSpawnBeforeIdentity?.(process.identity);
+                record = this.commit(key, (next) => {
+                    next.process = process.identity;
+                });
+            }
+            catch (error) {
+                await process.terminate(0);
+                throw error;
+            }
+            this.processes.set(key, { process, generation });
+            if (this.stopping) {
+                await this.stop(key, "stopped");
+                return;
+            }
+            const member = await process.prepare();
+            record = this.commit(key, (next) => {
+                next.worker = member;
+            });
+            if (this.stopping) {
+                await this.stop(key, "stopped");
+                return;
+            }
+            process.activate({
+                t: "hello",
+                protocol: 1,
+                generation,
+                config: record.config,
+                configRevision: record.configRevision,
+                last: Object.fromEntries(Object.entries(record.channels).map(([name, value]) => [name, value.value])),
+            });
+            this.emit(record);
+        }
+        catch (error) {
+            if (this.processes.has(key))
+                await this.stop(key, "backoff").catch(() => { });
+            if (this.store.available())
+                this.failAttempt(key, error);
+            throw error;
+        }
+        finally {
+            release();
+        }
+    }
+    failAttempt(key, error, generation) {
+        if (generation !== undefined && this.records.get(key).failureGeneration === generation)
+            return;
+        this.emit(this.commit(key, (next) => {
+            next.activeGeneration = null;
+            next.restarts++;
+            if (generation !== undefined)
+                next.failureGeneration = generation;
+            next.phase = next.process
+                ? "stop-unconfirmed"
+                : next.restarts >= backgroundServiceLimits.maxConsecutiveFailures
+                    ? "failed"
+                    : "backoff";
+            next.nextStartAt =
+                this.now() + Math.min(300_000, 5000 * 2 ** Math.min(16, next.restarts - 1));
+            next.lastError = String(error).slice(0, 2048);
+        }));
+    }
+    onLine(key, generation, message) {
+        const record = this.records.get(key);
+        const actual = this.processes.get(key);
+        if (!actual || actual.generation !== generation || record.activeGeneration !== generation)
+            return;
+        if (message?.t === "ready") {
+            void this.exclusive(() => {
+                const current = this.records.get(key);
+                if (this.processes.get(key) !== actual ||
+                    current.activeGeneration !== generation ||
+                    current.phase !== "starting")
+                    return;
+                this.emit(this.commit(key, (next) => {
+                    next.phase = "running";
+                }));
+            }).catch(() => { });
+            return;
+        }
+        if (message?.t !== "publish")
+            return;
+        const channel = typeof message.channel === "string" &&
+            Object.hasOwn(record.definition.channels, message.channel)
+            ? record.definition.channels[message.channel]
+            : undefined;
+        if (!channel || Buffer.byteLength(JSON.stringify(message.value)) > channel.maxBytes)
+            throw new Error("SERVICE_CHANNEL_LIMIT");
+        let pending = this.pending.get(key);
+        if (!pending) {
+            pending = { actual, values: new Map() };
+            this.pending.set(key, pending);
+        }
+        if (pending.actual !== actual) {
+            pending = { actual, values: new Map() };
+            this.pending.set(key, pending);
+        }
+        pending.values.set(message.channel, clone(message.value));
+        if (this.publishTimers.has(key))
+            return;
+        const previous = Math.max(0, ...Object.values(record.channels).map((value) => value.committedAt));
+        const timer = setTimeout(() => {
+            // External preparation may wait; stop must still reach the commit point.
+            // Keep the timer marker until completion to bound in-flight work to one.
+            void Promise.resolve()
+                .then(() => this.options.beforePublish?.())
+                .then(() => this.exclusive(() => {
+                const current = this.records.get(key);
+                if (this.processes.get(key) !== actual ||
+                    current.activeGeneration !== generation ||
+                    !["running", "starting"].includes(current.phase)) {
+                    if (this.pending.get(key)?.actual === actual)
+                        this.pending.delete(key);
+                    return;
+                }
+                const values = this.pending.get(key)?.actual === actual
+                    ? this.pending.get(key).values
+                    : undefined;
+                this.pending.delete(key);
+                if (!values?.size)
+                    return;
+                const next = this.commit(key, (record) => {
+                    for (const [name, value] of values)
+                        record.channels[name] = {
+                            generation,
+                            seq: (record.channels[name]?.seq ?? 0) + 1,
+                            committedAt: this.now(),
+                            value,
+                        };
+                });
+                for (const name of values.keys())
+                    this.emit(next, name);
+            }))
+                .catch(() => { })
+                .finally(() => {
+                if (this.publishTimers.get(key) !== timer)
+                    return;
+                this.publishTimers.delete(key);
+                const next = this.processes.get(key), pending = this.pending.get(key);
+                const value = pending && pending.actual === next
+                    ? pending.values.entries().next().value
+                    : undefined;
+                if (!this.stopping && next && value)
+                    this.onLine(key, next.generation, {
+                        t: "publish",
+                        channel: value[0],
+                        value: value[1],
+                    });
+            });
+        }, Math.max(0, previous + backgroundServiceLimits.minCommitIntervalMs - this.now()));
+        this.publishTimers.set(key, timer);
+    }
+    async onExit(key, generation, error) {
+        if (this.processes.get(key)?.generation !== generation)
+            return;
+        this.pending.delete(key);
+        if (!this.store.available()) {
+            this.beginEmergencyStop();
+            return;
+        }
+        let record = this.records.get(key);
+        // Revoking publication is not a stop request. Count a lost guardian once,
+        // even when its group remains alive until a later close event or tick.
+        const expected = record.stopRequestedGeneration === generation || this.stopping;
+        if (!expected) {
+            this.failAttempt(key, error ?? "Service exited unexpectedly", generation);
+            record = this.records.get(key);
+        }
+        if (record.process && this.groupStatus(record.process) !== "dead") {
+            this.emit(this.commit(key, (next) => {
+                next.activeGeneration = null;
+                next.phase = "stop-unconfirmed";
+            }));
+            return;
+        }
+        this.processes.delete(key);
+        const next = this.commit(key, (next) => {
+            next.process = null;
+            next.worker = null;
+            next.activeGeneration = null;
+            next.phase = this.exitedPhase(next);
+        });
+        this.emit(next);
+    }
+    tick() {
+        if (this.stopping)
+            return Promise.resolve();
+        if (!this.initialized)
+            return this.initialize();
+        return this.exclusive(async () => {
+            if (!this.store.available()) {
+                await this.recoverStorage();
+                return;
+            }
+            for (const key of this.records.keys()) {
+                try {
+                    await this.reconcile(key);
+                }
+                catch {
+                    /* other scopes still progress */
+                }
+            }
+        });
+    }
+    drain(matches) {
+        return this.exclusive(async () => {
+            const selected = [...this.records].filter(([, record]) => matches(record.scope));
+            for (const [key] of selected)
+                this.heldForUpdate.add(key);
+            await Promise.all(selected
+                .filter(([, record]) => record.process)
+                .map(([key, record]) => this.stop(key, record.paused ? "paused" : (record.stopPhase ?? "stopped"))));
+        });
+    }
+    /** Called after the catalog mutation releases its admission barrier, success or rollback. */
+    resumeAfterUpdate() {
+        this.heldForUpdate.clear();
+        void this.tick().catch(() => { });
+    }
+    revoke(appId) {
+        return this.exclusive(async () => {
+            for (const [key, record] of this.records)
+                if (record.scope.appId === appId) {
+                    this.commit(key, (next) => {
+                        next.desired = "off";
+                        next.desiredRevision++;
+                        next.stopRequestedGeneration = next.generation;
+                        next.stopPhase = "revoked";
+                        next.activeGeneration = null;
+                        next.config = {};
+                        next.configRevision++;
+                        for (const channel of Object.values(next.channels)) {
+                            channel.value = null;
+                            channel.seq++;
+                            channel.committedAt = this.now();
+                        }
+                    });
+                    await this.stop(key, "revoked");
+                }
+        });
+    }
+    async shutdown() {
+        this.stopping = true;
+        this.notifyAvailability();
+        if (this.timer)
+            clearInterval(this.timer);
+        for (const timer of this.publishTimers.values())
+            clearTimeout(timer);
+        this.publishTimers.clear();
+        await this.exclusive(async () => {
+            await Promise.all([...this.records]
+                .filter(([, record]) => record.process)
+                .map(async ([key, record]) => {
+                try {
+                    await this.stop(key, record.stopPhase ?? (record.paused ? "paused" : "stopped"));
+                }
+                catch {
+                    await this.processes.get(key)?.process.terminate(0);
+                }
+            }));
+            await this.emergency;
+        });
+        this.unregister?.();
+        this.unregisterMutation?.();
+        this.unregisterDrain?.();
+        this.store.release();
+        this.initialized = false;
+    }
+}