@bridge_gpt/mcp-server 0.2.2 → 0.2.3

package/build/conductor/supervisor-runtime.js

@@ -0,0 +1,351 @@
+/**
+ * Foreground supervisor runtime loop (BAPI-396, conductor C4).
+ *
+ * `conductor supervise --run-id <id>` runs this run-scoped, visible, ephemeral
+ * loop. It polls the conductor ledger, reduces raw events into deterministic
+ * worker watchdog state, persists an operational projection, escalates stuck
+ * workers (terminal output + idempotent ledger event), and terminates when all
+ * workers are terminal or the global timeout elapses.
+ *
+ * ## STATE PRECEDENCE (source-of-truth ordering) ##
+ *   1. Raw ledger events  — the ONLY source of operational truth.
+ *   2. supervisor_projection — the resumable operational mirror (this loop owns
+ *      writes; crash/restart re-hydrates from it).
+ *   3. supervisor.assessment events — AUDIT ONLY; never override raw truth.
+ *   4. LLM judgment context — DISPOSABLE; advisory text only, never persisted as
+ *      truth and never able to execute an action.
+ *
+ * The deterministic loop owns correctness. The LLM is consulted only to classify
+ * AMBIGUOUS stalls and draft escalation text, within a per-run budget, and
+ * degrades to deterministic-only behavior whenever it is disabled, exhausted, or
+ * fails.
+ */
+import { waitForConductorEvent, getSupervisorSnapshot, upsertSupervisorProjection, sendWorkerMessage, } from "./store.js";
+import { emitSupervisorAssessmentIfNew } from "./supervisor-ledger.js";
+import { sendSupervisorEscalationWorkerMessageIfNew } from "./supervisor-message-relay.js";
+import { resolveSupervisorConfig } from "./supervisor-config.js";
+import { hydrateSupervisorRunStateFromSnapshot, applyConductorEventToSupervisorState, applySupervisorHousekeeping, isSupervisorRunTerminal, hasSupervisorGlobalTimeoutElapsed, toSupervisorProjectionInput, } from "./supervisor-state.js";
+import { findSupervisorEscalationCandidates, shouldEmitEscalation, recordEscalationResult, formatEscalationForTerminal, } from "./supervisor-escalation.js";
+import { assessSupervisorCandidate } from "./supervisor-judgment.js";
+import { makeSupervisorIdempotencyKey } from "./supervisor-ledger.js";
+import { createDefaultSupervisorJudgmentClient } from "./supervisor-judgment-python.js";
+import { resolveConductorBridgeApiAccess, } from "./bridge-api-client.js";
+import { extractMergeActionIdentityFromGateEvent } from "./merge-ledger.js";
+import { processGateMetMerge } from "./supervisor-merge.js";
+import { dispatchSupervisorNotification } from "./supervisor-notification.js";
+import { ConductorValidationError } from "./errors.js";
+/** Compact, secret-free worker snapshot for the judgment request. */
+function compactWorkerForJudgment(worker) {
+    if (!worker)
+        return null;
+    return {
+        worker_id: worker.worker_id,
+        ticket_key: worker.ticket_key,
+        state: worker.state,
+        liveness: worker.liveness,
+        last_event_time: worker.last_event_time,
+        last_progress_time: worker.last_progress_time,
+    };
+}
+/** Deterministic (degraded) assessment for an unambiguous candidate. */
+function deterministicAssessment(candidate) {
+    return {
+        classification: "stuck",
+        confidence: 1,
+        should_escalate: true,
+        reason: candidate.reason,
+        draft_escalation_text: null,
+        source: "degraded",
+    };
+}
+/** Resolve the terminal run status from worker outcomes. */
+function terminalStatus(state) {
+    const anyFailed = Object.values(state.workers).some((w) => w.state === "failed");
+    return anyFailed ? "failed" : "complete";
+}
+/**
+ * Process the escalation pipeline for the current state: find candidates, assess
+ * ambiguous ones (within budget), emit idempotent assessment events, print
+ * terminal lines for newly-emitted escalations, and record outcomes. Returns
+ * nothing; it mutates `state`.
+ */
+async function processEscalations(state, config, client, deps) {
+    const now = deps.now();
+    const candidates = findSupervisorEscalationCandidates(state, config, now);
+    for (const candidate of candidates) {
+        const decision = shouldEmitEscalation(state, candidate, config, now);
+        if (!decision.emit)
+            continue;
+        // Compute the idempotency key up front so EVERY decision in this window —
+        // including an LLM "do not escalate" — can be recorded under it.
+        const idempotency = {
+            run_id: state.run_id,
+            worker_id: candidate.worker_id,
+            reason: candidate.reason,
+            kind: candidate.kind,
+            cooldown_window: decision.cooldown_window,
+        };
+        const idempotencyKey = makeSupervisorIdempotencyKey(idempotency);
+        let assessment;
+        if (candidate.ambiguous) {
+            assessment = await assessSupervisorCandidate({
+                run_id: state.run_id,
+                candidate,
+                worker: candidate.worker_id ? compactWorkerForJudgment(state.workers[candidate.worker_id]) : null,
+            }, config, state.llm_budget, client);
+        }
+        else {
+            assessment = deterministicAssessment(candidate);
+        }
+        state.latest_assessment = assessment;
+        if (!assessment.should_escalate) {
+            // The judgment declined to escalate. Record a `suppressed` decision for
+            // this cooldown window so the candidate is NOT re-judged every wake
+            // interval — without this, a persistently-ambiguous worker would re-query
+            // the LLM each tick, exhaust the per-run budget, and then escalate anyway
+            // once the degraded fallback kicks in. The decision stands until the
+            // window rolls over.
+            recordEscalationResult(state, candidate, decision.cooldown_window, idempotencyKey, "suppressed", now);
+            continue;
+        }
+        let outcome = "skipped";
+        try {
+            const result = deps.emitAssessment({
+                run_id: state.run_id,
+                worker_id: candidate.worker_id,
+                assessment,
+                details: { elapsed_ms: candidate.elapsed_ms, ...candidate.context },
+                idempotency,
+            });
+            outcome = result.emitted ? "emitted" : "duplicate";
+        }
+        catch {
+            // Emission is best-effort; record as skipped and keep supervising.
+            outcome = "skipped";
+        }
+        recordEscalationResult(state, candidate, decision.cooldown_window, idempotencyKey, outcome, now);
+        // BAPI-397: pass the escalation to the worker through the cooperative relay.
+        // Enqueue for BOTH `emitted` and `duplicate` outcomes so a crash between
+        // assessment emission and message enqueue self-heals on restart — the store's
+        // idempotency key (`cause_seq = last_seq`) + per-type cooldown prevent a
+        // second delivery. Enqueue only when the candidate is worker-scoped (a
+        // run-level `global_timeout` has no worker to message). This is best-effort:
+        // a relay failure must never stop supervising, and message payloads/draft
+        // text are never logged on failure.
+        if ((outcome === "emitted" || outcome === "duplicate") && candidate.worker_id) {
+            try {
+                sendSupervisorEscalationWorkerMessageIfNew(candidate, assessment, state, {
+                    sendMessage: deps.sendWorkerMessage,
+                });
+            }
+            catch {
+                /* best-effort relay; keep supervising */
+            }
+        }
+        // Only print on a genuinely NEW escalation — never re-spam a duplicate after
+        // a crash/restart within the same cooldown window.
+        if (outcome === "emitted") {
+            deps.log(formatEscalationForTerminal(state.run_id, candidate, assessment.draft_escalation_text));
+            // S7: dispatch an out-of-band human notification for newly emitted escalations.
+            // Best-effort: a failure must never stop supervising.
+            if (deps.dispatchNotification) {
+                try {
+                    await deps.dispatchNotification(state.run_id, candidate, assessment, idempotencyKey);
+                }
+                catch {
+                    /* best-effort notification; keep supervising */
+                }
+            }
+        }
+    }
+}
+/**
+ * Run the foreground supervisor loop for exactly one run. Returns a
+ * {@link SupervisorRunResult} with a process exit code. Validation failures and
+ * unexpected boundary exceptions are sanitized; on an unexpected error the loop
+ * attempts to persist a `failed` projection before returning a non-zero result.
+ */
+export async function runSupervisor(options, deps = {}) {
+    const runId = typeof options.run_id === "string" ? options.run_id.trim() : "";
+    if (runId.length === 0) {
+        throw new ConductorValidationError("Supervisor requires exactly one non-empty --run-id.");
+    }
+    const config = options.config ?? resolveSupervisorConfig(options.overrides ?? {});
+    const now = deps.now ?? (() => Date.now());
+    const log = deps.log ?? ((m) => process.stdout.write(`${m}\n`));
+    const errorLog = deps.errorLog ?? ((m) => process.stderr.write(`${m}\n`));
+    const waitForEvent = deps.waitForEvent ?? waitForConductorEvent;
+    const getSnapshot = deps.getSnapshot ?? getSupervisorSnapshot;
+    const upsertProjection = deps.upsertProjection ?? upsertSupervisorProjection;
+    const emitAssessment = deps.emitAssessment ?? emitSupervisorAssessmentIfNew;
+    const sendMessage = deps.sendWorkerMessage ?? sendWorkerMessage;
+    const judgmentClient = deps.judgmentClient ?? createDefaultSupervisorJudgmentClient(config);
+    const resolveBridgeAccess = deps.resolveBridgeAccess ?? (() => resolveConductorBridgeApiAccess());
+    const processMerge = deps.processMerge ?? processGateMetMerge;
+    const dispatchNotification = deps.dispatchNotification ?? dispatchSupervisorNotification;
+    // Bridge API access for the C6 merge pipeline is resolved at most once, lazily on
+    // the first eligible gate.met event. Unavailable credentials/repo identity
+    // disable autonomous merge gracefully — no local VCS operation, no loop crash.
+    let bridgeAccess = null;
+    let bridgeAccessResolved = false;
+    const processGateMetMerges = async (events) => {
+        const eligible = events.filter((e) => extractMergeActionIdentityFromGateEvent(e) !== null);
+        if (eligible.length === 0)
+            return;
+        if (!bridgeAccessResolved) {
+            bridgeAccessResolved = true;
+            try {
+                const result = await resolveBridgeAccess();
+                bridgeAccess = result.ok ? result.access : null;
+                if (!result.ok) {
+                    errorLog(`[supervisor] run=${runId} auto-merge disabled: ${result.kind}`);
+                }
+            }
+            catch {
+                bridgeAccess = null;
+            }
+        }
+        if (!bridgeAccess)
+            return;
+        for (const event of eligible) {
+            try {
+                await processMerge(bridgeAccess, event);
+            }
+            catch {
+                // A merge-processing failure must never crash the supervisor loop.
+            }
+        }
+    };
+    const persist = (state) => {
+        try {
+            upsertProjection(toSupervisorProjectionInput(state));
+        }
+        catch {
+            // Persistence is best-effort within an iteration; the next iteration retries.
+        }
+    };
+    let state;
+    try {
+        const snapshot = getSnapshot(runId);
+        state = hydrateSupervisorRunStateFromSnapshot(snapshot, runId, config, now());
+    }
+    catch {
+        // A snapshot read failure is non-fatal — start from empty state.
+        return {
+            run_id: runId,
+            status: "failed",
+            exit_code: 2,
+            reason: "supervisor failed to load initial snapshot",
+            last_seq: 0,
+            worker_count: 0,
+        };
+    }
+    // Resume from last_seq+1 when a projection was hydrated; else from 1.
+    let sinceSeq = state.last_seq > 0 ? state.last_seq + 1 : 1;
+    // Defensive iteration cap so a frozen clock can never spin forever; the global
+    // timeout is the real bound under a live clock.
+    const iterationCap = Math.max(1000, Math.ceil(config.global_timeout_ms / Math.max(1, config.wake_interval_ms)) + 16);
+    try {
+        // Refresh liveness from the hydrated state BEFORE the first terminal check so
+        // a resumed, already-terminal run (crash/restart) can exit promptly without
+        // blocking on a full wake interval first.
+        applySupervisorHousekeeping(state, config, now());
+        for (let iteration = 0; iteration < iterationCap; iteration += 1) {
+            // Terminal / global-timeout checks run at the TOP of the loop — BEFORE the
+            // blocking wait — so a run that is already (or has just become) terminal
+            // exits immediately rather than waiting one wake interval for events that
+            // will never arrive. The blocking wait below only happens while the run is
+            // still in progress.
+            if (isSupervisorRunTerminal(state)) {
+                state.status = terminalStatus(state);
+                persist(state);
+                log(`[supervisor] run=${runId} complete status=${state.status} workers=${Object.keys(state.workers).length}`);
+                return {
+                    run_id: runId,
+                    status: state.status,
+                    exit_code: 0,
+                    reason: "all workers terminal",
+                    last_seq: state.last_seq,
+                    worker_count: Object.keys(state.workers).length,
+                };
+            }
+            if (hasSupervisorGlobalTimeoutElapsed(state, now())) {
+                state.status = "timed_out";
+                persist(state);
+                errorLog(`[supervisor] run=${runId} global timeout elapsed`);
+                return {
+                    run_id: runId,
+                    status: "timed_out",
+                    exit_code: 2,
+                    reason: "global timeout elapsed",
+                    last_seq: state.last_seq,
+                    worker_count: Object.keys(state.workers).length,
+                };
+            }
+            const waitResult = await waitForEvent({
+                since_seq: sinceSeq,
+                filter: { run_id: runId },
+                data_mode: "full",
+                timeout_ms: config.wake_interval_ms,
+                limit: config.poll_limit,
+            });
+            if (waitResult.events.length > 0) {
+                const sorted = [...waitResult.events].sort((a, b) => a.seq - b.seq);
+                for (const event of sorted) {
+                    applyConductorEventToSupervisorState(state, event, now());
+                }
+                sinceSeq = sorted[sorted.length - 1].seq + 1;
+                // C6: after reducing the batch into state, process any worker-scoped
+                // gate.met events through the API-owned merge pipeline. Eligibility,
+                // idempotency, and the dry-run/enabled decision are all enforced
+                // downstream; this loop only dispatches and never merges locally.
+                await processGateMetMerges(sorted);
+            }
+            // Housekeeping (liveness + state-specific stalls) runs every iteration so a
+            // timeout with no events still advances liveness/stall state.
+            applySupervisorHousekeeping(state, config, now());
+            await processEscalations(state, config, judgmentClient, {
+                emitAssessment,
+                sendWorkerMessage: sendMessage,
+                log,
+                now,
+                dispatchNotification,
+            });
+            // Persist after the batch AND after housekeeping/escalation, because state
+            // can change even when no events arrived. The terminal/timeout exit is
+            // re-evaluated at the top of the next iteration (no extra blocking wait).
+            persist(state);
+        }
+        // Iteration cap reached without a terminal/global-timeout exit (frozen clock):
+        // treat as a timeout rather than spinning.
+        state.status = "timed_out";
+        persist(state);
+        return {
+            run_id: runId,
+            status: "timed_out",
+            exit_code: 2,
+            reason: "supervisor iteration cap reached",
+            last_seq: state.last_seq,
+            worker_count: Object.keys(state.workers).length,
+        };
+    }
+    catch {
+        // Unexpected boundary exception — sanitize, attempt to persist failed status.
+        state.status = "failed";
+        try {
+            upsertProjection(toSupervisorProjectionInput(state));
+        }
+        catch {
+            /* best-effort */
+        }
+        errorLog(`[supervisor] run=${runId} failed unexpectedly`);
+        return {
+            run_id: runId,
+            status: "failed",
+            exit_code: 2,
+            reason: "supervisor failed unexpectedly",
+            last_seq: state.last_seq,
+            worker_count: Object.keys(state.workers).length,
+        };
+    }
+}