@lsctech/polaris 0.5.5 → 0.5.7

package/dist/qc/repair-loop.js

@@ -0,0 +1,423 @@
+"use strict";
+/**
+ * QC repair loop orchestration.
+ *
+ * The Foreman calls `runQcRepairLoop` after a completed-cluster QC run produces
+ * findings. This module:
+ *
+ *   1. Discovers a compiled repair packet manifest for the current round.
+ *   2. Converts eligible repair packets into Foreman-dispatched repair workers.
+ *   3. Awaits repair worker completion (via sealed result files / adapter).
+ *   4. Reruns QC after all repair workers in a round have completed.
+ *   5. Loops until: pass, no repairable packets, max rounds reached,
+ *      all providers failed, operator review required, or Medic referral.
+ *
+ * Dispatch boundary: the parent/orchestrator NEVER implements repair code.
+ * Each repair packet becomes a sealed WorkerPacket with worker_role: "repair".
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_MAX_REPAIR_ROUNDS = void 0;
+exports.partitionRepairPackets = partitionRepairPackets;
+exports.initRepairLoopState = initRepairLoopState;
+exports.runQcRepairLoop = runQcRepairLoop;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const repair_packets_js_1 = require("./repair-packets.js");
+const orchestration_js_1 = require("./orchestration.js");
+const store_js_1 = require("../cluster-state/store.js");
+// ── Constants ─────────────────────────────────────────────────────────────────
+/** Default maximum repair rounds when not configured. */
+exports.DEFAULT_MAX_REPAIR_ROUNDS = 2;
+// ── Helpers ────────────────────────────────────────────────────────────────────
+function appendTelemetry(telemetryFile, event) {
+    (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(telemetryFile), { recursive: true });
+    (0, node_fs_1.appendFileSync)(telemetryFile, JSON.stringify(event) + "\n", "utf-8");
+}
+/** Returns true when any finding in the results routes to repair-worker. */
+function hasRepairableFindings(results) {
+    return results.some((r) => r.findings.some((f) => (f.routingDecision === "repair-worker" || f.routingDecision === "original-worker") &&
+        f.status !== "waived" &&
+        f.status !== "autofixed" &&
+        f.status !== "repaired"));
+}
+/** Returns true when all QC providers failed in every result. */
+function allProvidersFailed(results) {
+    return (results.length > 0 &&
+        results.every((r) => r.allProvidersFailed || r.status === "failed"));
+}
+/** Returns true when any finding requires operator review. */
+function requiresOperatorReview(results) {
+    return results.some((r) => r.findings.some((f) => f.routingDecision === "operator-review" &&
+        f.status !== "waived" &&
+        f.status !== "repaired"));
+}
+/** Returns true when the QC rerun passed (no open/blocking findings). */
+function rerunPassed(results) {
+    if (results.length === 0)
+        return false;
+    return results.every((r) => r.status === "passed" || r.status === "skipped");
+}
+/** Partition packets into safe-to-parallel and must-serialize groups. */
+function partitionRepairPackets(packets) {
+    // Medic (operator-review) packets are always serialized.
+    const serialized = packets.filter((p) => p.medic || p.routingTarget === "operator-review");
+    const parallelizable = packets.filter((p) => !p.medic && p.routingTarget !== "operator-review");
+    // Group by parallelGroup assignment from the compiler.
+    const groups = new Map();
+    for (const pkt of parallelizable) {
+        const key = pkt.parallelGroup ?? `solo-${pkt.packetId}`;
+        if (!groups.has(key))
+            groups.set(key, []);
+        groups.get(key).push(pkt);
+    }
+    return {
+        parallelGroups: Array.from(groups.values()),
+        serialized,
+    };
+}
+/** Initialize a fresh QcRepairLoopState. */
+function initRepairLoopState(opts) {
+    const now = new Date().toISOString();
+    return {
+        current_round: 0,
+        max_rounds: opts.maxRounds,
+        source_qc_run_ids: opts.sourceQcRunIds,
+        manifest_path: null,
+        pending_packet_ids: [],
+        completed_packet_ids: [],
+        rerun_requested: false,
+        rerun_qc_run_ids: {},
+        terminal_outcome: null,
+        initiated_at: now,
+        updated_at: now,
+    };
+}
+// ── Main repair loop ───────────────────────────────────────────────────────────
+/**
+ * Run the bounded QC repair loop.
+ *
+ * The loop is Foreman-owned: it coordinates dispatch, never implements repairs.
+ * Returns deterministically on pass, no-repairable, max-rounds, all-providers-failed,
+ * operator-review, or medic-referral.
+ */
+async function runQcRepairLoop(options) {
+    const { clusterId, runId, branch, repoRoot, telemetryFile, config, registry, initialQcResults, dispatchRepairWorker, validationCommands = [], timeoutMs, maxRounds = exports.DEFAULT_MAX_REPAIR_ROUNDS, priorLoopState, onStateUpdate, } = options;
+    if (!config.enabled) {
+        const state = initRepairLoopState({
+            maxRounds,
+            sourceQcRunIds: initialQcResults.map((r) => r.qcRunId),
+        });
+        state.terminal_outcome = "qc-disabled";
+        state.updated_at = new Date().toISOString();
+        return {
+            outcome: "qc-disabled",
+            rounds_completed: 0,
+            final_qc_results: initialQcResults,
+            loop_state: state,
+            summary: "QC repair loop skipped — QC disabled by configuration",
+        };
+    }
+    const sourceRunIds = initialQcResults.map((r) => r.qcRunId);
+    let loopState = priorLoopState ?? initRepairLoopState({ maxRounds, sourceQcRunIds: sourceRunIds });
+    let currentResults = initialQcResults;
+    let roundsCompleted = loopState.current_round;
+    appendTelemetry(telemetryFile, {
+        event: "qc-repair-loop-started",
+        run_id: runId,
+        cluster_id: clusterId,
+        max_rounds: maxRounds,
+        source_qc_run_ids: sourceRunIds,
+        timestamp: new Date().toISOString(),
+    });
+    // ── Main loop ──────────────────────────────────────────────────────────────
+    while (roundsCompleted < maxRounds) {
+        const round = roundsCompleted + 1;
+        // Check exit conditions at the top of each round.
+        if (rerunPassed(currentResults) && round > 1) {
+            // A rerun already passed in this call frame — exit.
+            break;
+        }
+        if (allProvidersFailed(currentResults)) {
+            loopState = { ...loopState, terminal_outcome: "all-providers-failed", updated_at: new Date().toISOString() };
+            onStateUpdate?.(loopState);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-loop-terminal",
+                run_id: runId,
+                outcome: "all-providers-failed",
+                round,
+                timestamp: new Date().toISOString(),
+            });
+            return {
+                outcome: "all-providers-failed",
+                rounds_completed: roundsCompleted,
+                final_qc_results: currentResults,
+                loop_state: loopState,
+                summary: `QC repair loop halted: all QC providers failed at round ${round}`,
+            };
+        }
+        if (requiresOperatorReview(currentResults)) {
+            loopState = { ...loopState, terminal_outcome: "operator-review", updated_at: new Date().toISOString() };
+            onStateUpdate?.(loopState);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-loop-terminal",
+                run_id: runId,
+                outcome: "operator-review",
+                round,
+                timestamp: new Date().toISOString(),
+            });
+            return {
+                outcome: "operator-review",
+                rounds_completed: roundsCompleted,
+                final_qc_results: currentResults,
+                loop_state: loopState,
+                summary: `QC repair loop halted: unresolved operator-review findings at round ${round}`,
+            };
+        }
+        // ── Compile / discover repair packets ──────────────────────────────────
+        let manifest = null;
+        // Try to read an existing manifest for this round first (idempotent re-entry).
+        manifest = (0, repair_packets_js_1.readRepairPacketManifest)(clusterId, round, repoRoot);
+        if (!manifest) {
+            // Compile fresh repair packets from current QC results.
+            const compiled = (0, repair_packets_js_1.compileAndWriteRepairPackets)({
+                clusterId,
+                round,
+                qcResults: currentResults,
+                config,
+                validationCommands,
+                repoRoot,
+            });
+            manifest = compiled.manifest;
+            // Update cluster state with manifest path.
+            try {
+                const clusterState = (0, store_js_1.readClusterStateSync)(clusterId, repoRoot);
+                if (clusterState) {
+                    const repairManifests = {
+                        ...(clusterState.qc_repair_manifests ?? {}),
+                        [round]: compiled.manifestPath,
+                    };
+                    await (0, store_js_1.writeClusterState)(clusterId, { ...clusterState, state_generation: clusterState.state_generation + 1, qc_repair_manifests: repairManifests }, repoRoot);
+                }
+            }
+            catch {
+                // Non-fatal: cluster state update is best-effort.
+            }
+            loopState = {
+                ...loopState,
+                current_round: round,
+                manifest_path: compiled.manifestPath,
+                pending_packet_ids: compiled.packets.map((p) => p.packetId),
+                updated_at: new Date().toISOString(),
+            };
+            onStateUpdate?.(loopState);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-manifest-compiled",
+                run_id: runId,
+                cluster_id: clusterId,
+                round,
+                packet_count: compiled.packets.length,
+                manifest_path: compiled.manifestPath,
+                timestamp: new Date().toISOString(),
+            });
+        }
+        else {
+            loopState = {
+                ...loopState,
+                current_round: round,
+                manifest_path: loopState.manifest_path ?? (0, node_path_1.join)(repoRoot, ".polaris", "clusters", clusterId, "qc", "repair-rounds", String(round), "repair-packets.json"),
+                updated_at: new Date().toISOString(),
+            };
+            onStateUpdate?.(loopState);
+        }
+        // ── Check for repairable packets ────────────────────────────────────────
+        const repairablePackets = manifest.packets.filter((p) => p.routingTarget === "repair-worker" &&
+            p.status === "pending" &&
+            !loopState.completed_packet_ids.includes(p.packetId));
+        if (repairablePackets.length === 0 && !hasRepairableFindings(currentResults)) {
+            loopState = { ...loopState, terminal_outcome: "no-repairable", updated_at: new Date().toISOString() };
+            onStateUpdate?.(loopState);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-loop-terminal",
+                run_id: runId,
+                outcome: "no-repairable",
+                round,
+                timestamp: new Date().toISOString(),
+            });
+            return {
+                outcome: "no-repairable",
+                rounds_completed: roundsCompleted,
+                final_qc_results: currentResults,
+                loop_state: loopState,
+                summary: `QC repair loop: no repairable packets at round ${round}`,
+            };
+        }
+        // ── Dispatch repair workers (parallel groups, then serialized) ──────────
+        const { parallelGroups, serialized } = partitionRepairPackets(repairablePackets);
+        const allWorkerResults = [];
+        let hasMedicReferral = false;
+        // Dispatch parallel groups.
+        for (const group of parallelGroups) {
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-worker-group-start",
+                run_id: runId,
+                round,
+                parallel_group: group[0]?.parallelGroup ?? null,
+                packet_ids: group.map((p) => p.packetId),
+                timestamp: new Date().toISOString(),
+            });
+            // Within a group, packets are non-conflicting and can run concurrently.
+            const groupResults = await Promise.all(group.map((pkt) => dispatchRepairWorker(pkt, round, manifest)));
+            allWorkerResults.push(...groupResults);
+        }
+        // Dispatch serialized (medic/operator-review) packets sequentially.
+        for (const pkt of serialized) {
+            const result = await dispatchRepairWorker(pkt, round, manifest);
+            allWorkerResults.push(result);
+        }
+        // Record completed packet IDs.
+        const completedIds = allWorkerResults
+            .filter((r) => r.status !== "skipped")
+            .map((r) => r.packetId);
+        loopState = {
+            ...loopState,
+            completed_packet_ids: [...loopState.completed_packet_ids, ...completedIds],
+            pending_packet_ids: loopState.pending_packet_ids.filter((id) => !completedIds.includes(id)),
+            updated_at: new Date().toISOString(),
+        };
+        onStateUpdate?.(loopState);
+        // Check for failed repair workers → Medic referral.
+        const failedWorkers = allWorkerResults.filter((r) => r.status === "failure");
+        if (failedWorkers.length > 0) {
+            hasMedicReferral = true;
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-worker-failures",
+                run_id: runId,
+                round,
+                failed_packet_ids: failedWorkers.map((r) => r.packetId),
+                errors: failedWorkers.map((r) => r.errorMessage ?? "unknown"),
+                timestamp: new Date().toISOString(),
+            });
+        }
+        if (hasMedicReferral) {
+            loopState = { ...loopState, terminal_outcome: "medic-referral", updated_at: new Date().toISOString() };
+            onStateUpdate?.(loopState);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-loop-terminal",
+                run_id: runId,
+                outcome: "medic-referral",
+                round,
+                failed_count: failedWorkers.length,
+                timestamp: new Date().toISOString(),
+            });
+            return {
+                outcome: "medic-referral",
+                rounds_completed: roundsCompleted + 1,
+                final_qc_results: currentResults,
+                loop_state: loopState,
+                summary: `QC repair loop: ${failedWorkers.length} repair worker(s) failed at round ${round} — Medic referral triggered`,
+            };
+        }
+        roundsCompleted += 1;
+        // ── QC rerun after successful repairs ───────────────────────────────────
+        loopState = { ...loopState, rerun_requested: true, updated_at: new Date().toISOString() };
+        onStateUpdate?.(loopState);
+        appendTelemetry(telemetryFile, {
+            event: "qc-repair-rerun-start",
+            run_id: runId,
+            cluster_id: clusterId,
+            round,
+            timestamp: new Date().toISOString(),
+        });
+        const rerunOptions = {
+            config,
+            registry,
+            trigger: "completed-cluster",
+            repoRoot,
+            runId,
+            clusterId,
+            branch,
+            telemetryFile,
+            timeoutMs,
+        };
+        let rerunResult;
+        try {
+            rerunResult = await (0, orchestration_js_1.runQcAtTrigger)(rerunOptions);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-rerun-error",
+                run_id: runId,
+                round,
+                error: msg,
+                timestamp: new Date().toISOString(),
+            });
+            // Treat rerun error as all-providers-failed.
+            loopState = { ...loopState, terminal_outcome: "all-providers-failed", updated_at: new Date().toISOString() };
+            onStateUpdate?.(loopState);
+            return {
+                outcome: "all-providers-failed",
+                rounds_completed: roundsCompleted,
+                final_qc_results: currentResults,
+                loop_state: loopState,
+                summary: `QC repair loop: rerun at round ${round} threw an error: ${msg}`,
+            };
+        }
+        // Record rerun QC run IDs.
+        const rerunIds = rerunResult.results.map((r) => r.qcRunId).filter(Boolean);
+        loopState = {
+            ...loopState,
+            rerun_requested: false,
+            rerun_qc_run_ids: { ...loopState.rerun_qc_run_ids, [round]: rerunIds },
+            updated_at: new Date().toISOString(),
+        };
+        onStateUpdate?.(loopState);
+        appendTelemetry(telemetryFile, {
+            event: "qc-repair-rerun-complete",
+            run_id: runId,
+            cluster_id: clusterId,
+            round,
+            action: rerunResult.action,
+            summary: rerunResult.summary,
+            rerun_qc_run_ids: rerunIds,
+            timestamp: new Date().toISOString(),
+        });
+        currentResults = rerunResult.results;
+        // Pass condition: rerun returned no blocking findings.
+        if (rerunResult.action === "pass") {
+            loopState = { ...loopState, terminal_outcome: "pass", updated_at: new Date().toISOString() };
+            onStateUpdate?.(loopState);
+            appendTelemetry(telemetryFile, {
+                event: "qc-repair-loop-terminal",
+                run_id: runId,
+                outcome: "pass",
+                round,
+                timestamp: new Date().toISOString(),
+            });
+            return {
+                outcome: "pass",
+                rounds_completed: roundsCompleted,
+                final_qc_results: currentResults,
+                loop_state: loopState,
+                summary: `QC repair loop passed after ${roundsCompleted} round(s)`,
+            };
+        }
+    }
+    // ── Max rounds exhausted ──────────────────────────────────────────────────
+    loopState = { ...loopState, terminal_outcome: "max-rounds", updated_at: new Date().toISOString() };
+    onStateUpdate?.(loopState);
+    appendTelemetry(telemetryFile, {
+        event: "qc-repair-loop-terminal",
+        run_id: runId,
+        outcome: "max-rounds",
+        rounds_completed: roundsCompleted,
+        timestamp: new Date().toISOString(),
+    });
+    return {
+        outcome: "max-rounds",
+        rounds_completed: roundsCompleted,
+        final_qc_results: currentResults,
+        loop_state: loopState,
+        summary: `QC repair loop exhausted max rounds (${maxRounds}) without passing`,
+    };
+}