@femtomc/mu-server 26.2.69 → 26.2.71

package/dist/control_plane.js

@@ -1,7 +1,13 @@
-import { ApprovedCommandBroker, CommandContextResolver, MessagingOperatorRuntime, operatorExtensionPaths, PiMessagingOperatorBackend, } from "@femtomc/mu-agent";
-import { ControlPlaneCommandPipeline, ControlPlaneOutbox, ControlPlaneOutboxDispatcher, ControlPlaneRuntime, correlationFromCommandRecord, DiscordControlPlaneAdapter, getControlPlanePaths, SlackControlPlaneAdapter, TelegramControlPlaneAdapter, TelegramControlPlaneAdapterSpec, } from "@femtomc/mu-control-plane";
+import { ControlPlaneCommandPipeline, ControlPlaneOutbox, ControlPlaneRuntime, DiscordControlPlaneAdapter, getControlPlanePaths, SlackControlPlaneAdapter, TelegramControlPlaneAdapterSpec, } from "@femtomc/mu-control-plane";
 import { DEFAULT_MU_CONFIG } from "./config.js";
+import { DEFAULT_INTER_ROOT_QUEUE_POLICY, normalizeInterRootQueuePolicy, } from "./control_plane_contract.js";
 import { ControlPlaneRunSupervisor, } from "./run_supervisor.js";
+import { DurableRunQueue, queueStatesForRunStatusFilter, runSnapshotFromQueueSnapshot } from "./run_queue.js";
+import { buildMessagingOperatorRuntime, createOutboxDrainLoop } from "./control_plane_bootstrap_helpers.js";
+import { ControlPlaneRunQueueCoordinator } from "./control_plane_run_queue_coordinator.js";
+import { enqueueRunEventOutbox } from "./control_plane_run_outbox.js";
+import { buildWakeOutboundEnvelope, resolveWakeFanoutCapability, wakeDeliveryMetadataFromOutboxRecord, wakeDispatchReasonCode, wakeFanoutDedupeKey, } from "./control_plane_wake_delivery.js";
+import { TelegramAdapterGenerationManager } from "./control_plane_telegram_generation.js";
 function generationTags(generation, component) {
     return {
         generation_id: generation.generation_id,
@@ -10,6 +16,25 @@ function generationTags(generation, component) {
         component,
     };
 }
+const WAKE_OUTBOX_MAX_ATTEMPTS = 6;
+function emptyNotifyOperatorsResult() {
+    return {
+        queued: 0,
+        duplicate: 0,
+        skipped: 0,
+        decisions: [],
+    };
+}
+function normalizeIssueId(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.trim();
+    if (!/^mu-[a-z0-9][a-z0-9-]*$/i.test(trimmed)) {
+        return null;
+    }
+    return trimmed.toLowerCase();
+}
 export function detectAdapters(config) {
     const adapters = [];
     const slackSecret = config.adapters.slack.signing_secret;
@@ -31,577 +56,6 @@ export function detectAdapters(config) {
     }
     return adapters;
 }
-const TELEGRAM_GENERATION_SUPERVISOR_ID = "telegram-adapter";
-const TELEGRAM_WARMUP_TIMEOUT_MS = 2_000;
-const TELEGRAM_DRAIN_TIMEOUT_MS = 5_000;
-function cloneControlPlaneConfig(config) {
-    return JSON.parse(JSON.stringify(config));
-}
-function controlPlaneNonTelegramFingerprint(config) {
-    return JSON.stringify({
-        adapters: {
-            slack: config.adapters.slack,
-            discord: config.adapters.discord,
-            gmail: config.adapters.gmail,
-        },
-        operator: config.operator,
-    });
-}
-function telegramAdapterConfigFromControlPlane(config) {
-    const webhookSecret = config.adapters.telegram.webhook_secret;
-    if (!webhookSecret) {
-        return null;
-    }
-    return {
-        webhookSecret,
-        botToken: config.adapters.telegram.bot_token,
-        botUsername: config.adapters.telegram.bot_username,
-    };
-}
-function applyTelegramAdapterConfig(base, telegram) {
-    const next = cloneControlPlaneConfig(base);
-    next.adapters.telegram.webhook_secret = telegram?.webhookSecret ?? null;
-    next.adapters.telegram.bot_token = telegram?.botToken ?? null;
-    next.adapters.telegram.bot_username = telegram?.botUsername ?? null;
-    return next;
-}
-function cloneTelegramAdapterConfig(config) {
-    return {
-        webhookSecret: config.webhookSecret,
-        botToken: config.botToken,
-        botUsername: config.botUsername,
-    };
-}
-function describeError(err) {
-    if (err instanceof Error && err.message.trim().length > 0) {
-        return err.message;
-    }
-    return String(err);
-}
-async function runWithTimeout(opts) {
-    if (opts.timeoutMs <= 0) {
-        return await opts.run();
-    }
-    return await new Promise((resolve, reject) => {
-        let settled = false;
-        const timer = setTimeout(() => {
-            if (settled) {
-                return;
-            }
-            settled = true;
-            reject(new Error(opts.timeoutMessage));
-        }, opts.timeoutMs);
-        void opts
-            .run()
-            .then((value) => {
-            if (settled) {
-                return;
-            }
-            settled = true;
-            clearTimeout(timer);
-            resolve(value);
-        })
-            .catch((err) => {
-            if (settled) {
-                return;
-            }
-            settled = true;
-            clearTimeout(timer);
-            reject(err);
-        });
-    });
-}
-class TelegramAdapterGenerationManager {
-    #pipeline;
-    #outbox;
-    #nowMs;
-    #onOutboxEnqueued;
-    #signalObserver;
-    #hooks;
-    #generationSeq = -1;
-    #active = null;
-    #previousConfig = null;
-    #activeControlPlaneConfig;
-    constructor(opts) {
-        this.#pipeline = opts.pipeline;
-        this.#outbox = opts.outbox;
-        this.#nowMs = opts.nowMs ?? Date.now;
-        this.#onOutboxEnqueued = opts.onOutboxEnqueued ?? null;
-        this.#signalObserver = opts.signalObserver ?? null;
-        this.#hooks = opts.hooks ?? null;
-        this.#activeControlPlaneConfig = cloneControlPlaneConfig(opts.initialConfig);
-    }
-    #nextGeneration() {
-        const nextSeq = this.#generationSeq + 1;
-        return {
-            generation_id: `${TELEGRAM_GENERATION_SUPERVISOR_ID}-gen-${nextSeq}`,
-            generation_seq: nextSeq,
-        };
-    }
-    #buildAdapter(config, opts) {
-        return new TelegramControlPlaneAdapter({
-            pipeline: this.#pipeline,
-            outbox: this.#outbox,
-            webhookSecret: config.webhookSecret,
-            botUsername: config.botUsername,
-            deferredIngress: true,
-            onOutboxEnqueued: this.#onOutboxEnqueued ?? undefined,
-            signalObserver: this.#signalObserver ?? undefined,
-            acceptIngress: opts.acceptIngress,
-            ingressDrainEnabled: opts.ingressDrainEnabled,
-            nowMs: this.#nowMs,
-        });
-    }
-    async initialize() {
-        const initial = telegramAdapterConfigFromControlPlane(this.#activeControlPlaneConfig);
-        if (!initial) {
-            return;
-        }
-        const generation = this.#nextGeneration();
-        const adapter = this.#buildAdapter(initial, {
-            acceptIngress: true,
-            ingressDrainEnabled: true,
-        });
-        await adapter.warmup();
-        const health = await adapter.healthCheck();
-        if (!health.ok) {
-            await adapter.stop({ force: true, reason: "startup_health_gate_failed" });
-            throw new Error(`telegram adapter warmup health failed: ${health.reason}`);
-        }
-        this.#active = {
-            generation,
-            config: cloneTelegramAdapterConfig(initial),
-            adapter,
-        };
-        this.#generationSeq = generation.generation_seq;
-    }
-    hasActiveGeneration() {
-        return this.#active != null;
-    }
-    activeGeneration() {
-        return this.#active ? { ...this.#active.generation } : null;
-    }
-    activeBotToken() {
-        return this.#active?.config.botToken ?? null;
-    }
-    activeAdapter() {
-        return this.#active?.adapter ?? null;
-    }
-    canHandleConfig(nextConfig, reason) {
-        if (reason === "rollback") {
-            return true;
-        }
-        return (controlPlaneNonTelegramFingerprint(nextConfig) ===
-            controlPlaneNonTelegramFingerprint(this.#activeControlPlaneConfig));
-    }
-    async #rollbackToPrevious(opts) {
-        if (!opts.previous) {
-            return { ok: false, error: "rollback_unavailable" };
-        }
-        try {
-            opts.previous.adapter.activateIngress();
-            this.#active = opts.previous;
-            this.#previousConfig = cloneTelegramAdapterConfig(opts.failedRecord.config);
-            await opts.failedRecord.adapter.stop({ force: true, reason: `rollback:${opts.reason}` });
-            this.#activeControlPlaneConfig = applyTelegramAdapterConfig(this.#activeControlPlaneConfig, opts.previous.config);
-            return { ok: true };
-        }
-        catch (err) {
-            return { ok: false, error: describeError(err) };
-        }
-    }
-    async reload(opts) {
-        if (!this.canHandleConfig(opts.config, opts.reason)) {
-            return {
-                handled: false,
-                ok: false,
-                reason: opts.reason,
-                route: "/webhooks/telegram",
-                from_generation: this.#active?.generation ?? null,
-                to_generation: null,
-                active_generation: this.#active?.generation ?? null,
-                warmup: null,
-                cutover: null,
-                drain: null,
-                rollback: {
-                    requested: opts.reason === "rollback",
-                    trigger: null,
-                    attempted: false,
-                    ok: true,
-                },
-            };
-        }
-        const rollbackRequested = opts.reason === "rollback";
-        let rollbackTrigger = rollbackRequested ? "manual" : null;
-        let rollbackAttempted = false;
-        let rollbackOk = true;
-        let rollbackError;
-        const fromGeneration = this.#active?.generation ?? null;
-        const previousRecord = this.#active;
-        const warmupTimeoutMs = Math.max(0, Math.trunc(opts.warmupTimeoutMs ?? TELEGRAM_WARMUP_TIMEOUT_MS));
-        const drainTimeoutMs = Math.max(0, Math.trunc(opts.drainTimeoutMs ?? TELEGRAM_DRAIN_TIMEOUT_MS));
-        const targetConfig = rollbackRequested
-            ? this.#previousConfig
-            : telegramAdapterConfigFromControlPlane(opts.config);
-        if (rollbackRequested && !targetConfig) {
-            return {
-                handled: true,
-                ok: false,
-                reason: opts.reason,
-                route: "/webhooks/telegram",
-                from_generation: fromGeneration,
-                to_generation: null,
-                active_generation: fromGeneration,
-                warmup: null,
-                cutover: null,
-                drain: null,
-                rollback: {
-                    requested: true,
-                    trigger: "rollback_unavailable",
-                    attempted: false,
-                    ok: false,
-                    error: "rollback_unavailable",
-                },
-                error: "rollback_unavailable",
-            };
-        }
-        if (!targetConfig && !previousRecord) {
-            this.#activeControlPlaneConfig = cloneControlPlaneConfig(opts.config);
-            return {
-                handled: true,
-                ok: true,
-                reason: opts.reason,
-                route: "/webhooks/telegram",
-                from_generation: null,
-                to_generation: null,
-                active_generation: null,
-                warmup: null,
-                cutover: null,
-                drain: null,
-                rollback: {
-                    requested: rollbackRequested,
-                    trigger: rollbackTrigger,
-                    attempted: false,
-                    ok: true,
-                },
-            };
-        }
-        if (!targetConfig && previousRecord) {
-            const drainStartedAtMs = Math.trunc(this.#nowMs());
-            let forcedStop = false;
-            let drainError;
-            let drainTimedOut = false;
-            try {
-                previousRecord.adapter.beginDrain();
-                if (this.#hooks?.onDrain) {
-                    await this.#hooks.onDrain({
-                        generation: previousRecord.generation,
-                        reason: opts.reason,
-                        timeout_ms: drainTimeoutMs,
-                    });
-                }
-                const drain = await runWithTimeout({
-                    timeoutMs: drainTimeoutMs,
-                    timeoutMessage: "telegram_drain_timeout",
-                    run: async () => await previousRecord.adapter.drain({ timeoutMs: drainTimeoutMs, reason: opts.reason }),
-                });
-                drainTimedOut = drain.timed_out;
-                if (!drain.ok || drain.timed_out) {
-                    forcedStop = true;
-                    await previousRecord.adapter.stop({ force: true, reason: "disable_drain_timeout" });
-                }
-                else {
-                    await previousRecord.adapter.stop({ force: false, reason: "disable" });
-                }
-            }
-            catch (err) {
-                drainError = describeError(err);
-                forcedStop = true;
-                drainTimedOut = drainError.includes("timeout");
-                await previousRecord.adapter.stop({ force: true, reason: "disable_drain_failed" });
-            }
-            this.#previousConfig = cloneTelegramAdapterConfig(previousRecord.config);
-            this.#active = null;
-            this.#activeControlPlaneConfig = applyTelegramAdapterConfig(this.#activeControlPlaneConfig, null);
-            return {
-                handled: true,
-                ok: drainError == null,
-                reason: opts.reason,
-                route: "/webhooks/telegram",
-                from_generation: fromGeneration,
-                to_generation: null,
-                active_generation: null,
-                warmup: null,
-                cutover: {
-                    ok: true,
-                    elapsed_ms: 0,
-                },
-                drain: {
-                    ok: drainError == null && !drainTimedOut,
-                    elapsed_ms: Math.max(0, Math.trunc(this.#nowMs()) - drainStartedAtMs),
-                    timed_out: drainTimedOut,
-                    forced_stop: forcedStop,
-                    ...(drainError ? { error: drainError } : {}),
-                },
-                rollback: {
-                    requested: rollbackRequested,
-                    trigger: rollbackTrigger,
-                    attempted: false,
-                    ok: true,
-                },
-                ...(drainError ? { error: drainError } : {}),
-            };
-        }
-        const nextConfig = cloneTelegramAdapterConfig(targetConfig);
-        const toGeneration = this.#nextGeneration();
-        const nextAdapter = this.#buildAdapter(nextConfig, {
-            acceptIngress: false,
-            ingressDrainEnabled: false,
-        });
-        const nextRecord = {
-            generation: toGeneration,
-            config: nextConfig,
-            adapter: nextAdapter,
-        };
-        const warmupStartedAtMs = Math.trunc(this.#nowMs());
-        try {
-            if (this.#hooks?.onWarmup) {
-                await this.#hooks.onWarmup({ generation: toGeneration, reason: opts.reason });
-            }
-            await runWithTimeout({
-                timeoutMs: warmupTimeoutMs,
-                timeoutMessage: "telegram_warmup_timeout",
-                run: async () => {
-                    await nextAdapter.warmup();
-                    const health = await nextAdapter.healthCheck();
-                    if (!health.ok) {
-                        throw new Error(`telegram_health_gate_failed:${health.reason}`);
-                    }
-                },
-            });
-        }
-        catch (err) {
-            const error = describeError(err);
-            rollbackTrigger = error.includes("health_gate") ? "health_gate_failed" : "warmup_failed";
-            await nextAdapter.stop({ force: true, reason: "warmup_failed" });
-            return {
-                handled: true,
-                ok: false,
-                reason: opts.reason,
-                route: "/webhooks/telegram",
-                from_generation: fromGeneration,
-                to_generation: toGeneration,
-                active_generation: fromGeneration,
-                warmup: {
-                    ok: false,
-                    elapsed_ms: Math.max(0, Math.trunc(this.#nowMs()) - warmupStartedAtMs),
-                    error,
-                },
-                cutover: null,
-                drain: null,
-                rollback: {
-                    requested: rollbackRequested,
-                    trigger: rollbackTrigger,
-                    attempted: false,
-                    ok: true,
-                },
-                error,
-            };
-        }
-        const cutoverStartedAtMs = Math.trunc(this.#nowMs());
-        try {
-            if (this.#hooks?.onCutover) {
-                await this.#hooks.onCutover({
-                    from_generation: fromGeneration,
-                    to_generation: toGeneration,
-                    reason: opts.reason,
-                });
-            }
-            nextAdapter.activateIngress();
-            if (previousRecord) {
-                previousRecord.adapter.beginDrain();
-            }
-            this.#active = nextRecord;
-            this.#generationSeq = toGeneration.generation_seq;
-            const postCutoverHealth = await nextAdapter.healthCheck();
-            if (!postCutoverHealth.ok) {
-                throw new Error(`telegram_post_cutover_health_failed:${postCutoverHealth.reason}`);
-            }
-        }
-        catch (err) {
-            const error = describeError(err);
-            rollbackTrigger = error.includes("post_cutover") ? "post_cutover_health_failed" : "cutover_failed";
-            rollbackAttempted = true;
-            const rollback = await this.#rollbackToPrevious({
-                failedRecord: nextRecord,
-                previous: previousRecord,
-                reason: opts.reason,
-            });
-            rollbackOk = rollback.ok;
-            rollbackError = rollback.error;
-            if (!rollback.ok) {
-                await nextAdapter.stop({ force: true, reason: "rollback_failed" });
-                this.#active = previousRecord ?? null;
-                this.#activeControlPlaneConfig = applyTelegramAdapterConfig(this.#activeControlPlaneConfig, previousRecord?.config ?? null);
-            }
-            return {
-                handled: true,
-                ok: false,
-                reason: opts.reason,
-                route: "/webhooks/telegram",
-                from_generation: fromGeneration,
-                to_generation: toGeneration,
-                active_generation: this.#active?.generation ?? fromGeneration,
-                warmup: {
-                    ok: true,
-                    elapsed_ms: Math.max(0, cutoverStartedAtMs - warmupStartedAtMs),
-                },
-                cutover: {
-                    ok: false,
-                    elapsed_ms: Math.max(0, Math.trunc(this.#nowMs()) - cutoverStartedAtMs),
-                    error,
-                },
-                drain: null,
-                rollback: {
-                    requested: rollbackRequested,
-                    trigger: rollbackTrigger,
-                    attempted: rollbackAttempted,
-                    ok: rollbackOk,
-                    ...(rollbackError ? { error: rollbackError } : {}),
-                },
-                error,
-            };
-        }
-        let drain = null;
-        if (previousRecord) {
-            const drainStartedAtMs = Math.trunc(this.#nowMs());
-            let forcedStop = false;
-            let drainTimedOut = false;
-            let drainError;
-            try {
-                if (this.#hooks?.onDrain) {
-                    await this.#hooks.onDrain({
-                        generation: previousRecord.generation,
-                        reason: opts.reason,
-                        timeout_ms: drainTimeoutMs,
-                    });
-                }
-                const drained = await runWithTimeout({
-                    timeoutMs: drainTimeoutMs,
-                    timeoutMessage: "telegram_drain_timeout",
-                    run: async () => await previousRecord.adapter.drain({ timeoutMs: drainTimeoutMs, reason: opts.reason }),
-                });
-                drainTimedOut = drained.timed_out;
-                if (!drained.ok || drained.timed_out) {
-                    forcedStop = true;
-                    await previousRecord.adapter.stop({ force: true, reason: "generation_drain_timeout" });
-                }
-                else {
-                    await previousRecord.adapter.stop({ force: false, reason: "generation_drained" });
-                }
-            }
-            catch (err) {
-                drainError = describeError(err);
-                forcedStop = true;
-                drainTimedOut = drainError.includes("timeout");
-                await previousRecord.adapter.stop({ force: true, reason: "generation_drain_failed" });
-            }
-            drain = {
-                ok: drainError == null && !drainTimedOut,
-                elapsed_ms: Math.max(0, Math.trunc(this.#nowMs()) - drainStartedAtMs),
-                timed_out: drainTimedOut,
-                forced_stop: forcedStop,
-                ...(drainError ? { error: drainError } : {}),
-            };
-        }
-        this.#previousConfig = previousRecord ? cloneTelegramAdapterConfig(previousRecord.config) : this.#previousConfig;
-        this.#activeControlPlaneConfig = applyTelegramAdapterConfig(this.#activeControlPlaneConfig, nextConfig);
-        return {
-            handled: true,
-            ok: true,
-            reason: opts.reason,
-            route: "/webhooks/telegram",
-            from_generation: fromGeneration,
-            to_generation: toGeneration,
-            active_generation: toGeneration,
-            warmup: {
-                ok: true,
-                elapsed_ms: Math.max(0, cutoverStartedAtMs - warmupStartedAtMs),
-            },
-            cutover: {
-                ok: true,
-                elapsed_ms: Math.max(0, Math.trunc(this.#nowMs()) - cutoverStartedAtMs),
-            },
-            drain,
-            rollback: {
-                requested: rollbackRequested,
-                trigger: rollbackTrigger,
-                attempted: rollbackAttempted,
-                ok: rollbackOk,
-                ...(rollbackError ? { error: rollbackError } : {}),
-            },
-        };
-    }
-    async stop() {
-        const active = this.#active;
-        this.#active = null;
-        if (!active) {
-            return;
-        }
-        await active.adapter.stop({ force: true, reason: "shutdown" });
-    }
-}
-function sha256Hex(input) {
-    const hasher = new Bun.CryptoHasher("sha256");
-    hasher.update(input);
-    return hasher.digest("hex");
-}
-function outboxKindForRunEvent(kind) {
-    switch (kind) {
-        case "run_completed":
-            return "result";
-        case "run_failed":
-            return "error";
-        default:
-            return "lifecycle";
-    }
-}
-async function enqueueRunEventOutbox(opts) {
-    const command = opts.event.command;
-    if (!command) {
-        return null;
-    }
-    const baseCorrelation = correlationFromCommandRecord(command);
-    const correlation = {
-        ...baseCorrelation,
-        run_root_id: opts.event.run.root_issue_id ?? baseCorrelation.run_root_id,
-    };
-    const envelope = {
-        v: 1,
-        ts_ms: opts.nowMs,
-        channel: command.channel,
-        channel_tenant_id: command.channel_tenant_id,
-        channel_conversation_id: command.channel_conversation_id,
-        request_id: command.request_id,
-        response_id: `resp-${sha256Hex(`run-event:${opts.event.run.job_id}:${opts.event.seq}:${opts.nowMs}`).slice(0, 20)}`,
-        kind: outboxKindForRunEvent(opts.event.kind),
-        body: opts.event.message,
-        correlation,
-        metadata: {
-            async_run: true,
-            run_event_kind: opts.event.kind,
-            run_event_seq: opts.event.seq,
-            run: opts.event.run,
-        },
-    };
-    const decision = await opts.outbox.enqueue({
-        dedupeKey: `run-event:${opts.event.run.job_id}:${opts.event.seq}`,
-        envelope,
-        nowMs: opts.nowMs,
-        maxAttempts: 6,
-    });
-    return decision.record;
-}
 /**
  * Telegram supports a markdown dialect that uses single markers for emphasis.
  * Normalize the most common LLM/GitHub-style markers (`**bold**`, `__italic__`, headings)
@@ -664,26 +118,6 @@ async function postTelegramMessage(botToken, payload) {
         body: JSON.stringify(payload),
     });
 }
-const OUTBOX_DRAIN_INTERVAL_MS = 500;
-function buildMessagingOperatorRuntime(opts) {
-    if (!opts.config.operator.enabled) {
-        return null;
-    }
-    const backend = opts.backend ??
-        new PiMessagingOperatorBackend({
-            provider: opts.config.operator.provider ?? undefined,
-            model: opts.config.operator.model ?? undefined,
-            extensionPaths: operatorExtensionPaths,
-        });
-    return new MessagingOperatorRuntime({
-        backend,
-        broker: new ApprovedCommandBroker({
-            runTriggersEnabled: opts.config.operator.run_triggers_enabled,
-            contextResolver: new CommandContextResolver({ allowedRepoRoots: [opts.repoRoot] }),
-        }),
-        enabled: true,
-    });
-}
 export async function bootstrapControlPlane(opts) {
     const controlPlaneConfig = opts.config ?? DEFAULT_MU_CONFIG.control_plane;
     const detected = detectAdapters(controlPlaneConfig);
@@ -709,7 +143,9 @@ export async function bootstrapControlPlane(opts) {
     const runtime = new ControlPlaneRuntime({ repoRoot: opts.repoRoot });
     let pipeline = null;
     let runSupervisor = null;
-    let drainInterval = null;
+    let outboxDrainLoop = null;
+    let wakeDeliveryObserver = opts.wakeDeliveryObserver ?? null;
+    const outboundDeliveryChannels = new Set(["telegram"]);
     const adapterMap = new Map();
     try {
         await runtime.start();
@@ -725,12 +161,20 @@ export async function bootstrapControlPlane(opts) {
         });
         await outbox.load();
         let scheduleOutboxDrainRef = null;
+        const runQueue = new DurableRunQueue({ repoRoot: opts.repoRoot });
+        const interRootQueuePolicy = normalizeInterRootQueuePolicy(opts.interRootQueuePolicy ?? DEFAULT_INTER_ROOT_QUEUE_POLICY);
+        const runQueueCoordinator = new ControlPlaneRunQueueCoordinator({
+            runQueue,
+            interRootQueuePolicy,
+            getRunSupervisor: () => runSupervisor,
+        });
         runSupervisor = new ControlPlaneRunSupervisor({
             repoRoot: opts.repoRoot,
             heartbeatScheduler: opts.heartbeatScheduler,
             heartbeatIntervalMs: opts.runSupervisorHeartbeatIntervalMs,
             spawnProcess: opts.runSupervisorSpawnProcess,
             onEvent: async (event) => {
+                await runQueueCoordinator.onRunEvent(event);
                 const outboxRecord = await enqueueRunEventOutbox({
                     outbox,
                     event,
@@ -741,6 +185,7 @@ export async function bootstrapControlPlane(opts) {
                 }
             },
         });
+        await runQueueCoordinator.scheduleReconcile("bootstrap");
         pipeline = new ControlPlaneCommandPipeline({
             runtime,
             operator,
@@ -836,10 +281,7 @@ export async function bootstrapControlPlane(opts) {
                 }
                 if (record.target_type === "run start" || record.target_type === "run resume") {
                     try {
-                        const launched = await runSupervisor?.startFromCommand(record);
-                        if (!launched) {
-                            return null;
-                        }
+                        const launched = await runQueueCoordinator.launchQueuedRunFromCommand(record);
                         return {
                             terminalState: "completed",
                             result: {
@@ -863,6 +305,8 @@ export async function bootstrapControlPlane(opts) {
                                         run_mode: launched.mode,
                                         run_root_id: launched.root_issue_id,
                                         run_source: launched.source,
+                                        queue_id: launched.queue_id ?? null,
+                                        queue_state: launched.queue_state ?? null,
                                     },
                                 },
                             ],
@@ -871,7 +315,7 @@ export async function bootstrapControlPlane(opts) {
                     catch (err) {
                         return {
                             terminalState: "failed",
-                            errorCode: err instanceof Error && err.message ? err.message : "run_supervisor_start_failed",
+                            errorCode: err instanceof Error && err.message ? err.message : "run_queue_start_failed",
                             trace: {
                                 cliCommandKind: record.target_type.replaceAll(" ", "_"),
                                 runRootId: record.target_id,
@@ -880,9 +324,9 @@ export async function bootstrapControlPlane(opts) {
                     }
                 }
                 if (record.target_type === "run interrupt") {
-                    const result = runSupervisor?.interrupt({
+                    const result = await runQueueCoordinator.interruptQueuedRun({
                         rootIssueId: record.target_id,
-                    }) ?? { ok: false, reason: "not_found", run: null };
+                    });
                     if (!result.ok) {
                         return {
                             terminalState: "failed",
@@ -1001,6 +445,104 @@ export async function bootstrapControlPlane(opts) {
             },
             isActive: () => telegramManager.hasActiveGeneration(),
         });
+        const notifyOperators = async (notifyOpts) => {
+            if (!pipeline) {
+                return emptyNotifyOperatorsResult();
+            }
+            const message = notifyOpts.message.trim();
+            const dedupeKey = notifyOpts.dedupeKey.trim();
+            if (!message || !dedupeKey) {
+                return emptyNotifyOperatorsResult();
+            }
+            const wakeSource = typeof notifyOpts.wake?.wakeSource === "string" ? notifyOpts.wake.wakeSource.trim() : "";
+            const wakeProgramId = typeof notifyOpts.wake?.programId === "string" ? notifyOpts.wake.programId.trim() : "";
+            const wakeSourceTsMsRaw = notifyOpts.wake?.sourceTsMs;
+            const wakeSourceTsMs = typeof wakeSourceTsMsRaw === "number" && Number.isFinite(wakeSourceTsMsRaw)
+                ? Math.trunc(wakeSourceTsMsRaw)
+                : null;
+            const wakeId = typeof notifyOpts.wake?.wakeId === "string" && notifyOpts.wake.wakeId.trim().length > 0
+                ? notifyOpts.wake.wakeId.trim()
+                : `wake-${(() => {
+                    const hasher = new Bun.CryptoHasher("sha256");
+                    hasher.update(`${dedupeKey}:${message}`);
+                    return hasher.digest("hex").slice(0, 16);
+                })()}`;
+            const context = {
+                wakeId,
+                dedupeKey,
+                wakeSource: wakeSource || null,
+                programId: wakeProgramId || null,
+                sourceTsMs: wakeSourceTsMs,
+            };
+            const nowMs = Math.trunc(Date.now());
+            const telegramBotToken = telegramManager.activeBotToken();
+            const bindings = pipeline.identities
+                .listBindings({ includeInactive: false })
+                .filter((binding) => binding.scopes.includes("cp.ops.admin"));
+            const result = emptyNotifyOperatorsResult();
+            for (const binding of bindings) {
+                const bindingDedupeKey = wakeFanoutDedupeKey({
+                    dedupeKey,
+                    wakeId,
+                    binding,
+                });
+                const capability = resolveWakeFanoutCapability({
+                    binding,
+                    isChannelDeliverySupported: (channel) => outboundDeliveryChannels.has(channel),
+                    telegramBotToken,
+                });
+                if (!capability.ok) {
+                    result.skipped += 1;
+                    result.decisions.push({
+                        state: "skipped",
+                        reason_code: capability.reasonCode,
+                        binding_id: binding.binding_id,
+                        channel: binding.channel,
+                        dedupe_key: bindingDedupeKey,
+                        outbox_id: null,
+                    });
+                    continue;
+                }
+                const envelope = buildWakeOutboundEnvelope({
+                    repoRoot: opts.repoRoot,
+                    nowMs,
+                    message,
+                    binding,
+                    context,
+                    metadata: notifyOpts.metadata,
+                });
+                const enqueueDecision = await outbox.enqueue({
+                    dedupeKey: bindingDedupeKey,
+                    envelope,
+                    nowMs,
+                    maxAttempts: WAKE_OUTBOX_MAX_ATTEMPTS,
+                });
+                if (enqueueDecision.kind === "enqueued") {
+                    result.queued += 1;
+                    scheduleOutboxDrainRef?.();
+                    result.decisions.push({
+                        state: "queued",
+                        reason_code: "outbox_enqueued",
+                        binding_id: binding.binding_id,
+                        channel: binding.channel,
+                        dedupe_key: bindingDedupeKey,
+                        outbox_id: enqueueDecision.record.outbox_id,
+                    });
+                }
+                else {
+                    result.duplicate += 1;
+                    result.decisions.push({
+                        state: "duplicate",
+                        reason_code: "outbox_duplicate",
+                        binding_id: binding.binding_id,
+                        channel: binding.channel,
+                        dedupe_key: bindingDedupeKey,
+                        outbox_id: enqueueDecision.record.outbox_id,
+                    });
+                }
+            }
+            return result;
+        };
         const deliver = async (record) => {
             const { envelope } = record;
             if (envelope.channel === "telegram") {
@@ -1043,38 +585,38 @@ export async function bootstrapControlPlane(opts) {
             }
             return undefined;
         };
-        const dispatcher = new ControlPlaneOutboxDispatcher({ outbox, deliver });
-        let drainingOutbox = false;
-        let drainRequested = false;
-        const drainOutboxNow = async () => {
-            if (drainingOutbox) {
-                drainRequested = true;
-                return;
-            }
-            drainingOutbox = true;
-            try {
-                do {
-                    drainRequested = false;
-                    await dispatcher.drainDue();
-                } while (drainRequested);
-            }
-            catch {
-                // Swallow errors — the dispatcher handles retries internally.
-            }
-            finally {
-                drainingOutbox = false;
-            }
-        };
-        const scheduleOutboxDrain = () => {
-            queueMicrotask(() => {
-                void drainOutboxNow();
-            });
-        };
+        const outboxDrain = createOutboxDrainLoop({
+            outbox,
+            deliver,
+            onOutcome: async (outcome) => {
+                if (!wakeDeliveryObserver) {
+                    return;
+                }
+                const metadata = wakeDeliveryMetadataFromOutboxRecord(outcome.record);
+                if (!metadata) {
+                    return;
+                }
+                const state = outcome.kind === "delivered" ? "delivered" : outcome.kind === "retried" ? "retried" : "dead_letter";
+                await wakeDeliveryObserver({
+                    state,
+                    reason_code: wakeDispatchReasonCode({
+                        state,
+                        lastError: outcome.record.last_error,
+                        deadLetterReason: outcome.record.dead_letter_reason,
+                    }),
+                    wake_id: metadata.wakeId,
+                    dedupe_key: metadata.wakeDedupeKey,
+                    binding_id: metadata.bindingId,
+                    channel: metadata.channel,
+                    outbox_id: metadata.outboxId,
+                    outbox_dedupe_key: metadata.outboxDedupeKey,
+                    attempt_count: outcome.record.attempt_count,
+                });
+            },
+        });
+        const scheduleOutboxDrain = outboxDrain.scheduleOutboxDrain;
         scheduleOutboxDrainRef = scheduleOutboxDrain;
-        drainInterval = setInterval(() => {
-            scheduleOutboxDrain();
-        }, OUTBOX_DRAIN_INTERVAL_MS);
-        scheduleOutboxDrain();
+        outboxDrainLoop = outboxDrain;
         return {
             get activeAdapters() {
                 return [...adapterMap.values()].filter((entry) => entry.isActive()).map((v) => v.info);
@@ -1089,6 +631,12 @@ export async function bootstrapControlPlane(opts) {
                 }
                 return result.response;
             },
+            async notifyOperators(notifyOpts) {
+                return await notifyOperators(notifyOpts);
+            },
+            setWakeDeliveryObserver(observer) {
+                wakeDeliveryObserver = observer;
+            },
             async reloadTelegramGeneration(reloadOpts) {
                 const result = await telegramManager.reload({
                     config: reloadOpts.config,
@@ -1100,41 +648,74 @@ export async function bootstrapControlPlane(opts) {
                 return result;
             },
             async listRuns(opts = {}) {
-                return (runSupervisor?.list({
+                const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 100)));
+                const fallbackStatusFilter = queueStatesForRunStatusFilter(opts.status);
+                if (Array.isArray(fallbackStatusFilter) && fallbackStatusFilter.length === 0) {
+                    return [];
+                }
+                const queued = await runQueue.listRunSnapshots({
                     status: opts.status,
-                    limit: opts.limit,
-                }) ?? []);
+                    limit,
+                    runtimeByJobId: runQueueCoordinator.runtimeSnapshotsByJobId(),
+                });
+                const seen = new Set(queued.map((run) => run.job_id));
+                const fallbackRuns = runSupervisor?.list({ limit: 500 }) ?? [];
+                for (const run of fallbackRuns) {
+                    if (seen.has(run.job_id)) {
+                        continue;
+                    }
+                    if (fallbackStatusFilter && fallbackStatusFilter.length > 0) {
+                        const mapped = run.status === "completed"
+                            ? "done"
+                            : run.status === "failed"
+                                ? "failed"
+                                : run.status === "cancelled"
+                                    ? "cancelled"
+                                    : "active";
+                        if (!fallbackStatusFilter.includes(mapped)) {
+                            continue;
+                        }
+                    }
+                    queued.push(run);
+                    seen.add(run.job_id);
+                }
+                return queued.slice(0, limit);
             },
             async getRun(idOrRoot) {
+                const queued = await runQueue.get(idOrRoot);
+                if (queued) {
+                    const runtime = queued.job_id ? (runSupervisor?.get(queued.job_id) ?? null) : null;
+                    return runSnapshotFromQueueSnapshot(queued, runtime);
+                }
                 return runSupervisor?.get(idOrRoot) ?? null;
             },
             async startRun(startOpts) {
-                const run = await runSupervisor?.launchStart({
+                return await runQueueCoordinator.launchQueuedRun({
+                    mode: "run_start",
                     prompt: startOpts.prompt,
                     maxSteps: startOpts.maxSteps,
                     source: "api",
+                    dedupeKey: `api:run_start:${crypto.randomUUID()}`,
                 });
-                if (!run) {
-                    throw new Error("run_supervisor_unavailable");
-                }
-                return run;
             },
             async resumeRun(resumeOpts) {
-                const run = await runSupervisor?.launchResume({
-                    rootIssueId: resumeOpts.rootIssueId,
+                const rootIssueId = normalizeIssueId(resumeOpts.rootIssueId);
+                if (!rootIssueId) {
+                    throw new Error("run_resume_invalid_root_issue_id");
+                }
+                return await runQueueCoordinator.launchQueuedRun({
+                    mode: "run_resume",
+                    rootIssueId,
                     maxSteps: resumeOpts.maxSteps,
                     source: "api",
+                    dedupeKey: `api:run_resume:${rootIssueId}:${crypto.randomUUID()}`,
                 });
-                if (!run) {
-                    throw new Error("run_supervisor_unavailable");
-                }
-                return run;
             },
             async interruptRun(interruptOpts) {
-                return runSupervisor?.interrupt(interruptOpts) ?? { ok: false, reason: "not_found", run: null };
+                return await runQueueCoordinator.interruptQueuedRun(interruptOpts);
             },
             async heartbeatRun(heartbeatOpts) {
-                return runSupervisor?.heartbeat(heartbeatOpts) ?? { ok: false, reason: "not_found", run: null };
+                return await runQueueCoordinator.heartbeatQueuedRun(heartbeatOpts);
             },
             async traceRun(traceOpts) {
                 return (await runSupervisor?.trace(traceOpts.idOrRoot, { limit: traceOpts.limit })) ?? null;
@@ -1146,9 +727,11 @@ export async function bootstrapControlPlane(opts) {
                 return await pipeline.handleTerminalInbound(terminalOpts);
             },
             async stop() {
-                if (drainInterval) {
-                    clearInterval(drainInterval);
-                    drainInterval = null;
+                wakeDeliveryObserver = null;
+                runQueueCoordinator.stop();
+                if (outboxDrainLoop) {
+                    outboxDrainLoop.stop();
+                    outboxDrainLoop = null;
                 }
                 for (const { adapter } of adapterMap.values()) {
                     try {
@@ -1169,9 +752,10 @@ export async function bootstrapControlPlane(opts) {
         };
     }
     catch (err) {
-        if (drainInterval) {
-            clearInterval(drainInterval);
-            drainInterval = null;
+        wakeDeliveryObserver = null;
+        if (outboxDrainLoop) {
+            outboxDrainLoop.stop();
+            outboxDrainLoop = null;
         }
         for (const { adapter } of adapterMap.values()) {
             try {