@femtomc/mu-server 26.2.69 → 26.2.71

package/dist/control_plane_reload.js

@@ -0,0 +1,525 @@
+import { ControlPlaneGenerationSupervisor } from "./generation_supervisor.js";
+export function summarizeControlPlane(handle) {
+    if (!handle) {
+        return { active: false, adapters: [], routes: [] };
+    }
+    return {
+        active: handle.activeAdapters.length > 0,
+        adapters: handle.activeAdapters.map((adapter) => adapter.name),
+        routes: handle.activeAdapters.map((adapter) => ({ name: adapter.name, route: adapter.route })),
+    };
+}
+function describeError(err) {
+    if (err instanceof Error)
+        return err.message;
+    return String(err);
+}
+export function createReloadManager(deps) {
+    let controlPlaneCurrent = deps.initialControlPlane;
+    let reloadInFlight = null;
+    const generationTelemetry = deps.generationTelemetry;
+    const generationSupervisor = new ControlPlaneGenerationSupervisor({
+        supervisorId: "control-plane",
+        initialGeneration: controlPlaneCurrent
+            ? {
+                generation_id: "control-plane-gen-0",
+                generation_seq: 0,
+            }
+            : null,
+    });
+    const generationTagsFor = (generation, component) => ({
+        generation_id: generation.generation_id,
+        generation_seq: generation.generation_seq,
+        supervisor: "control_plane",
+        component,
+    });
+    const performControlPlaneReload = async (reason) => {
+        const startedAtMs = Date.now();
+        const planned = generationSupervisor.beginReload(reason);
+        const attempt = planned.attempt;
+        const previous = controlPlaneCurrent;
+        const previousSummary = summarizeControlPlane(previous);
+        const tags = generationTagsFor(attempt.to_generation, "server.reload");
+        const baseFields = {
+            reason,
+            attempt_id: attempt.attempt_id,
+            coalesced: planned.coalesced,
+            from_generation_id: attempt.from_generation?.generation_id ?? null,
+        };
+        const logLifecycle = (opts) => {
+            generationTelemetry.log({
+                level: opts.level,
+                message: `reload transition ${opts.stage}:${opts.state}`,
+                fields: {
+                    ...tags,
+                    ...baseFields,
+                    ...(opts.extra ?? {}),
+                },
+            });
+        };
+        let swapped = false;
+        let failedStage = "warmup";
+        let drainDurationMs = 0;
+        let drainStartedAtMs = null;
+        let nextHandle = null;
+        try {
+            logLifecycle({ level: "info", stage: "warmup", state: "start" });
+            const latestConfig = await deps.loadConfigFromDisk();
+            const telegramGeneration = (await previous?.reloadTelegramGeneration?.({
+                config: latestConfig.control_plane,
+                reason,
+            })) ?? null;
+            if (telegramGeneration?.handled) {
+                if (telegramGeneration.warmup) {
+                    logLifecycle({
+                        level: telegramGeneration.warmup.ok ? "info" : "error",
+                        stage: "warmup",
+                        state: telegramGeneration.warmup.ok ? "complete" : "failed",
+                        extra: {
+                            warmup_elapsed_ms: telegramGeneration.warmup.elapsed_ms,
+                            error: telegramGeneration.warmup.error,
+                            telegram_generation_id: telegramGeneration.to_generation?.generation_id ?? null,
+                        },
+                    });
+                }
+                else {
+                    logLifecycle({
+                        level: "info",
+                        stage: "warmup",
+                        state: "skipped",
+                        extra: {
+                            warmup_reason: "telegram_generation_no_warmup",
+                            telegram_generation_id: telegramGeneration.to_generation?.generation_id ?? null,
+                        },
+                    });
+                }
+                if (telegramGeneration.cutover) {
+                    logLifecycle({ level: "info", stage: "cutover", state: "start" });
+                    logLifecycle({
+                        level: telegramGeneration.cutover.ok ? "info" : "error",
+                        stage: "cutover",
+                        state: telegramGeneration.cutover.ok ? "complete" : "failed",
+                        extra: {
+                            cutover_elapsed_ms: telegramGeneration.cutover.elapsed_ms,
+                            error: telegramGeneration.cutover.error,
+                            active_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        },
+                    });
+                }
+                else {
+                    logLifecycle({
+                        level: "info",
+                        stage: "cutover",
+                        state: "skipped",
+                        extra: {
+                            cutover_reason: "telegram_generation_no_cutover",
+                            active_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        },
+                    });
+                }
+                if (telegramGeneration.drain) {
+                    logLifecycle({ level: "info", stage: "drain", state: "start" });
+                    drainDurationMs = Math.max(0, Math.trunc(telegramGeneration.drain.elapsed_ms));
+                    generationTelemetry.recordDrainDuration(tags, {
+                        durationMs: drainDurationMs,
+                        timedOut: telegramGeneration.drain.timed_out,
+                        metadata: {
+                            ...baseFields,
+                            telegram_forced_stop: telegramGeneration.drain.forced_stop,
+                            telegram_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        },
+                    });
+                    logLifecycle({
+                        level: telegramGeneration.drain.ok ? "info" : "warn",
+                        stage: "drain",
+                        state: telegramGeneration.drain.ok ? "complete" : "failed",
+                        extra: {
+                            drain_duration_ms: telegramGeneration.drain.elapsed_ms,
+                            drain_timed_out: telegramGeneration.drain.timed_out,
+                            forced_stop: telegramGeneration.drain.forced_stop,
+                            error: telegramGeneration.drain.error,
+                        },
+                    });
+                }
+                else {
+                    logLifecycle({
+                        level: "info",
+                        stage: "drain",
+                        state: "skipped",
+                        extra: {
+                            drain_reason: "telegram_generation_no_drain",
+                            telegram_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        },
+                    });
+                }
+                const shouldLogRollbackStart = telegramGeneration.rollback.requested ||
+                    telegramGeneration.rollback.attempted ||
+                    telegramGeneration.rollback.trigger != null ||
+                    !telegramGeneration.ok;
+                if (shouldLogRollbackStart) {
+                    logLifecycle({
+                        level: telegramGeneration.rollback.ok ? "warn" : "error",
+                        stage: "rollback",
+                        state: "start",
+                        extra: {
+                            rollback_requested: telegramGeneration.rollback.requested,
+                            rollback_trigger: telegramGeneration.rollback.trigger,
+                            rollback_attempted: telegramGeneration.rollback.attempted,
+                        },
+                    });
+                    logLifecycle({
+                        level: telegramGeneration.rollback.ok ? "info" : "error",
+                        stage: "rollback",
+                        state: telegramGeneration.rollback.ok ? "complete" : "failed",
+                        extra: {
+                            rollback_requested: telegramGeneration.rollback.requested,
+                            rollback_trigger: telegramGeneration.rollback.trigger,
+                            rollback_attempted: telegramGeneration.rollback.attempted,
+                            error: telegramGeneration.rollback.error,
+                        },
+                    });
+                }
+                else {
+                    logLifecycle({
+                        level: "debug",
+                        stage: "rollback",
+                        state: "skipped",
+                        extra: {
+                            rollback_reason: "not_requested",
+                        },
+                    });
+                }
+                if (telegramGeneration.ok) {
+                    swapped = generationSupervisor.markSwapInstalled(attempt.attempt_id);
+                    generationSupervisor.finishReload(attempt.attempt_id, "success");
+                    const elapsedMs = Math.max(0, Date.now() - startedAtMs);
+                    generationTelemetry.recordReloadSuccess(tags, {
+                        ...baseFields,
+                        elapsed_ms: elapsedMs,
+                        drain_duration_ms: drainDurationMs,
+                        telegram_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        telegram_rollback_attempted: telegramGeneration.rollback.attempted,
+                        telegram_rollback_trigger: telegramGeneration.rollback.trigger,
+                    });
+                    generationTelemetry.trace({
+                        name: "control_plane.reload",
+                        status: "ok",
+                        durationMs: elapsedMs,
+                        fields: {
+                            ...tags,
+                            ...baseFields,
+                            telegram_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        },
+                    });
+                    return {
+                        ok: true,
+                        reason,
+                        previous_control_plane: previousSummary,
+                        control_plane: summarizeControlPlane(controlPlaneCurrent),
+                        generation: {
+                            attempt_id: attempt.attempt_id,
+                            coalesced: planned.coalesced,
+                            from_generation: attempt.from_generation,
+                            to_generation: attempt.to_generation,
+                            active_generation: generationSupervisor.activeGeneration(),
+                            outcome: "success",
+                        },
+                        telegram_generation: telegramGeneration,
+                    };
+                }
+                generationSupervisor.finishReload(attempt.attempt_id, "failure");
+                const error = telegramGeneration.error ?? "telegram_generation_reload_failed";
+                const elapsedMs = Math.max(0, Date.now() - startedAtMs);
+                generationTelemetry.recordReloadFailure(tags, {
+                    ...baseFields,
+                    elapsed_ms: elapsedMs,
+                    drain_duration_ms: drainDurationMs,
+                    error,
+                    telegram_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                    telegram_rollback_trigger: telegramGeneration.rollback.trigger,
+                });
+                generationTelemetry.trace({
+                    name: "control_plane.reload",
+                    status: "error",
+                    durationMs: elapsedMs,
+                    fields: {
+                        ...tags,
+                        ...baseFields,
+                        error,
+                        telegram_generation_id: telegramGeneration.active_generation?.generation_id ?? null,
+                        telegram_rollback_trigger: telegramGeneration.rollback.trigger,
+                    },
+                });
+                return {
+                    ok: false,
+                    reason,
+                    previous_control_plane: previousSummary,
+                    control_plane: summarizeControlPlane(controlPlaneCurrent),
+                    generation: {
+                        attempt_id: attempt.attempt_id,
+                        coalesced: planned.coalesced,
+                        from_generation: attempt.from_generation,
+                        to_generation: attempt.to_generation,
+                        active_generation: generationSupervisor.activeGeneration(),
+                        outcome: "failure",
+                    },
+                    telegram_generation: telegramGeneration,
+                    error,
+                };
+            }
+            const next = await deps.controlPlaneReloader({
+                repoRoot: deps.repoRoot,
+                previous,
+                config: latestConfig.control_plane,
+                generation: attempt.to_generation,
+            });
+            nextHandle = next;
+            logLifecycle({ level: "info", stage: "warmup", state: "complete" });
+            failedStage = "cutover";
+            logLifecycle({ level: "info", stage: "cutover", state: "start" });
+            controlPlaneCurrent = next;
+            swapped = generationSupervisor.markSwapInstalled(attempt.attempt_id);
+            logLifecycle({
+                level: "info",
+                stage: "cutover",
+                state: "complete",
+                extra: {
+                    active_generation_id: generationSupervisor.activeGeneration()?.generation_id ?? null,
+                },
+            });
+            failedStage = "drain";
+            if (previous && previous !== next) {
+                logLifecycle({ level: "info", stage: "drain", state: "start" });
+                drainStartedAtMs = Date.now();
+                await previous.stop();
+                drainDurationMs = Math.max(0, Date.now() - drainStartedAtMs);
+                generationTelemetry.recordDrainDuration(tags, {
+                    durationMs: drainDurationMs,
+                    metadata: {
+                        ...baseFields,
+                    },
+                });
+                logLifecycle({
+                    level: "info",
+                    stage: "drain",
+                    state: "complete",
+                    extra: {
+                        drain_duration_ms: drainDurationMs,
+                    },
+                });
+            }
+            else {
+                logLifecycle({
+                    level: "info",
+                    stage: "drain",
+                    state: "skipped",
+                    extra: {
+                        drain_reason: "no_previous_generation",
+                    },
+                });
+            }
+            logLifecycle({
+                level: "debug",
+                stage: "rollback",
+                state: "skipped",
+                extra: {
+                    rollback_reason: "not_requested",
+                },
+            });
+            generationSupervisor.finishReload(attempt.attempt_id, "success");
+            const elapsedMs = Math.max(0, Date.now() - startedAtMs);
+            generationTelemetry.recordReloadSuccess(tags, {
+                ...baseFields,
+                elapsed_ms: elapsedMs,
+                drain_duration_ms: drainDurationMs,
+            });
+            generationTelemetry.trace({
+                name: "control_plane.reload",
+                status: "ok",
+                durationMs: elapsedMs,
+                fields: {
+                    ...tags,
+                    ...baseFields,
+                },
+            });
+            return {
+                ok: true,
+                reason,
+                previous_control_plane: previousSummary,
+                control_plane: summarizeControlPlane(next),
+                generation: {
+                    attempt_id: attempt.attempt_id,
+                    coalesced: planned.coalesced,
+                    from_generation: attempt.from_generation,
+                    to_generation: attempt.to_generation,
+                    active_generation: generationSupervisor.activeGeneration(),
+                    outcome: "success",
+                },
+            };
+        }
+        catch (err) {
+            const error = describeError(err);
+            if (failedStage === "drain" && drainStartedAtMs != null) {
+                drainDurationMs = Math.max(0, Date.now() - drainStartedAtMs);
+                generationTelemetry.recordDrainDuration(tags, {
+                    durationMs: drainDurationMs,
+                    metadata: {
+                        ...baseFields,
+                        error,
+                    },
+                });
+            }
+            logLifecycle({
+                level: "error",
+                stage: failedStage,
+                state: "failed",
+                extra: {
+                    error,
+                    drain_duration_ms: failedStage === "drain" ? drainDurationMs : undefined,
+                },
+            });
+            if (swapped) {
+                logLifecycle({
+                    level: "warn",
+                    stage: "rollback",
+                    state: "start",
+                    extra: {
+                        rollback_reason: "reload_failed_after_cutover",
+                        rollback_target_generation_id: attempt.from_generation?.generation_id ?? null,
+                        rollback_source_generation_id: attempt.to_generation.generation_id,
+                    },
+                });
+                if (!previous) {
+                    logLifecycle({
+                        level: "error",
+                        stage: "rollback",
+                        state: "failed",
+                        extra: {
+                            rollback_reason: "no_previous_generation",
+                            rollback_source_generation_id: attempt.to_generation.generation_id,
+                        },
+                    });
+                }
+                else {
+                    try {
+                        const restored = generationSupervisor.rollbackSwapInstalled(attempt.attempt_id);
+                        if (!restored) {
+                            throw new Error("generation_rollback_state_mismatch");
+                        }
+                        controlPlaneCurrent = previous;
+                        if (nextHandle && nextHandle !== previous) {
+                            await nextHandle.stop();
+                        }
+                        logLifecycle({
+                            level: "info",
+                            stage: "rollback",
+                            state: "complete",
+                            extra: {
+                                active_generation_id: generationSupervisor.activeGeneration()?.generation_id ?? null,
+                                rollback_target_generation_id: attempt.from_generation?.generation_id ?? null,
+                            },
+                        });
+                    }
+                    catch (rollbackErr) {
+                        logLifecycle({
+                            level: "error",
+                            stage: "rollback",
+                            state: "failed",
+                            extra: {
+                                error: describeError(rollbackErr),
+                                active_generation_id: generationSupervisor.activeGeneration()?.generation_id ?? null,
+                                rollback_target_generation_id: attempt.from_generation?.generation_id ?? null,
+                                rollback_source_generation_id: attempt.to_generation.generation_id,
+                            },
+                        });
+                    }
+                }
+            }
+            else {
+                logLifecycle({
+                    level: "debug",
+                    stage: "rollback",
+                    state: "skipped",
+                    extra: {
+                        rollback_reason: "cutover_not_installed",
+                    },
+                });
+            }
+            generationSupervisor.finishReload(attempt.attempt_id, "failure");
+            const elapsedMs = Math.max(0, Date.now() - startedAtMs);
+            generationTelemetry.recordReloadFailure(tags, {
+                ...baseFields,
+                elapsed_ms: elapsedMs,
+                drain_duration_ms: drainDurationMs,
+                error,
+            });
+            generationTelemetry.trace({
+                name: "control_plane.reload",
+                status: "error",
+                durationMs: elapsedMs,
+                fields: {
+                    ...tags,
+                    ...baseFields,
+                    error,
+                },
+            });
+            return {
+                ok: false,
+                reason,
+                previous_control_plane: previousSummary,
+                control_plane: summarizeControlPlane(controlPlaneCurrent),
+                generation: {
+                    attempt_id: attempt.attempt_id,
+                    coalesced: planned.coalesced,
+                    from_generation: attempt.from_generation,
+                    to_generation: attempt.to_generation,
+                    active_generation: generationSupervisor.activeGeneration(),
+                    outcome: "failure",
+                },
+                error,
+            };
+        }
+    };
+    const reloadControlPlane = async (reason) => {
+        if (reloadInFlight) {
+            const pending = generationSupervisor.pendingReload();
+            const fallbackGeneration = generationSupervisor.activeGeneration() ??
+                generationSupervisor.snapshot().last_reload?.to_generation ??
+                null;
+            const generation = pending?.to_generation ?? fallbackGeneration;
+            if (generation) {
+                generationTelemetry.recordDuplicateSignal(generationTagsFor(generation, "server.reload"), {
+                    source: "server_reload",
+                    signal: "coalesced_reload_request",
+                    dedupe_key: pending?.attempt_id ?? "reload_in_flight",
+                    record_id: pending?.attempt_id ?? "reload_in_flight",
+                    metadata: {
+                        reason,
+                        pending_reason: pending?.reason ?? null,
+                    },
+                });
+            }
+            return await reloadInFlight;
+        }
+        reloadInFlight = performControlPlaneReload(reason).finally(() => {
+            reloadInFlight = null;
+        });
+        return await reloadInFlight;
+    };
+    return {
+        reloadControlPlane,
+        getControlPlaneStatus: () => ({
+            ...summarizeControlPlane(controlPlaneCurrent),
+            generation: generationSupervisor.snapshot(),
+            observability: {
+                counters: generationTelemetry.counters(),
+            },
+        }),
+        getControlPlaneCurrent: () => controlPlaneCurrent,
+        setControlPlaneCurrent: (handle) => {
+            controlPlaneCurrent = handle;
+        },
+        generationSupervisor,
+        generationTelemetry,
+    };
+}

package/dist/control_plane_run_outbox.d.ts

@@ -0,0 +1,7 @@
+import { type ControlPlaneOutbox, type OutboxRecord } from "@femtomc/mu-control-plane";
+import type { ControlPlaneRunEvent } from "./run_supervisor.js";
+export declare function enqueueRunEventOutbox(opts: {
+    outbox: ControlPlaneOutbox;
+    event: ControlPlaneRunEvent;
+    nowMs: number;
+}): Promise<OutboxRecord | null>;

package/dist/control_plane_run_outbox.js

@@ -0,0 +1,52 @@
+import { correlationFromCommandRecord, } from "@femtomc/mu-control-plane";
+function sha256Hex(input) {
+    const hasher = new Bun.CryptoHasher("sha256");
+    hasher.update(input);
+    return hasher.digest("hex");
+}
+function outboxKindForRunEvent(kind) {
+    switch (kind) {
+        case "run_completed":
+            return "result";
+        case "run_failed":
+            return "error";
+        default:
+            return "lifecycle";
+    }
+}
+export async function enqueueRunEventOutbox(opts) {
+    const command = opts.event.command;
+    if (!command) {
+        return null;
+    }
+    const baseCorrelation = correlationFromCommandRecord(command);
+    const correlation = {
+        ...baseCorrelation,
+        run_root_id: opts.event.run.root_issue_id ?? baseCorrelation.run_root_id,
+    };
+    const envelope = {
+        v: 1,
+        ts_ms: opts.nowMs,
+        channel: command.channel,
+        channel_tenant_id: command.channel_tenant_id,
+        channel_conversation_id: command.channel_conversation_id,
+        request_id: command.request_id,
+        response_id: `resp-${sha256Hex(`run-event:${opts.event.run.job_id}:${opts.event.seq}:${opts.nowMs}`).slice(0, 20)}`,
+        kind: outboxKindForRunEvent(opts.event.kind),
+        body: opts.event.message,
+        correlation,
+        metadata: {
+            async_run: true,
+            run_event_kind: opts.event.kind,
+            run_event_seq: opts.event.seq,
+            run: opts.event.run,
+        },
+    };
+    const decision = await opts.outbox.enqueue({
+        dedupeKey: `run-event:${opts.event.run.job_id}:${opts.event.seq}`,
+        envelope,
+        nowMs: opts.nowMs,
+        maxAttempts: 6,
+    });
+    return decision.record;
+}

package/dist/control_plane_run_queue_coordinator.d.ts

@@ -0,0 +1,48 @@
+import type { CommandRecord } from "@femtomc/mu-control-plane";
+import type { InterRootQueuePolicy } from "./control_plane_contract.js";
+import { DurableRunQueue } from "./run_queue.js";
+import type { ControlPlaneRunEvent, ControlPlaneRunHeartbeatResult, ControlPlaneRunInterruptResult, ControlPlaneRunSnapshot, ControlPlaneRunSupervisor } from "./run_supervisor.js";
+export type QueueLaunchOpts = {
+    mode: "run_start" | "run_resume";
+    prompt?: string;
+    rootIssueId?: string | null;
+    maxSteps?: number;
+    source: "command" | "api";
+    command?: CommandRecord | null;
+    dedupeKey: string;
+};
+export type ControlPlaneRunQueueCoordinatorOpts = {
+    runQueue: DurableRunQueue;
+    interRootQueuePolicy: InterRootQueuePolicy;
+    getRunSupervisor: () => ControlPlaneRunSupervisor | null;
+    defaultRunMaxSteps?: number;
+};
+/**
+ * Queue/reconcile adapter for control-plane run lifecycle operations.
+ *
+ * Keeps inter-root queue execution concerns out of `control_plane.ts` while preserving
+ * queue-first semantics:
+ * - enqueue intents durably
+ * - reconcile/claim/launch deterministically
+ * - mirror runtime events back into durable queue state
+ */
+export declare class ControlPlaneRunQueueCoordinator {
+    #private;
+    constructor(opts: ControlPlaneRunQueueCoordinatorOpts);
+    runtimeSnapshotsByJobId(): Map<string, ControlPlaneRunSnapshot>;
+    scheduleReconcile(reason: string): Promise<void>;
+    launchQueuedRun(launchOpts: QueueLaunchOpts): Promise<ControlPlaneRunSnapshot>;
+    launchQueuedRunFromCommand(record: CommandRecord): Promise<ControlPlaneRunSnapshot>;
+    interruptQueuedRun(opts: {
+        jobId?: string | null;
+        rootIssueId?: string | null;
+    }): Promise<ControlPlaneRunInterruptResult>;
+    heartbeatQueuedRun(opts: {
+        jobId?: string | null;
+        rootIssueId?: string | null;
+        reason?: string | null;
+        wakeMode?: string | null;
+    }): Promise<ControlPlaneRunHeartbeatResult>;
+    onRunEvent(event: ControlPlaneRunEvent): Promise<void>;
+    stop(): void;
+}