@lsctech/polaris 0.1.0

package/dist/loop/orphan-recovery.js

@@ -0,0 +1,276 @@
+"use strict";
+/**
+ * Orphan detection and child recovery workflow.
+ *
+ * Detects workers that have gone silent, failed to acknowledge, or left
+ * partial execution traces. Implements the 5-scenario recovery model:
+ *
+ *   A: no-worker-assignment  — packet written, no worker_id after launch_timeout
+ *   B: no-acknowledgment     — worker_id present, no ack within launch_to_first_heartbeat_ms
+ *   C: no-heartbeat          — acknowledged, heartbeat > orphan_timeout_ms ago, no result
+ *   D: missing-result-artifact — worker-result event received but expected_result_path invalid
+ *   E: stale-dispatch        — dispatched_at + stale_dispatch_timeout ago, still handoff-pending
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.emitRecoveryApproved = emitRecoveryApproved;
+exports.checkOrphans = checkOrphans;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const node_crypto_1 = require("node:crypto");
+const checkpoint_js_1 = require("./checkpoint.js");
+// ── Telemetry helpers ─────────────────────────────────────────────────────────
+function appendTelemetry(telemetryFile, event) {
+    (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(telemetryFile), { recursive: true });
+    (0, node_fs_1.appendFileSync)(telemetryFile, JSON.stringify(event) + "\n", "utf-8");
+}
+function resolveTelemetryFile(state, repoRoot) {
+    const artifactDir = state.artifact_dir ?? (0, node_path_1.join)(repoRoot, ".taskchain_artifacts", "polaris-run");
+    return (0, node_path_1.join)(artifactDir, "runs", state.run_id, "telemetry.jsonl");
+}
+/** Returns true only when path is a non-empty string and the file exists. */
+function safeResultExists(path) {
+    if (typeof path !== "string" || path.trim() === "")
+        return false;
+    try {
+        return (0, node_fs_1.existsSync)(path);
+    }
+    catch {
+        return false;
+    }
+}
+const DEFAULT_ORPHAN_TIMEOUTS = {
+    launchTimeoutMs: 30_000,
+    launchToFirstHeartbeatMs: 30_000,
+    orphanTimeoutMs: 300_000, // 5 min (was 10 min)
+    staleDispatchTimeoutMs: 1_800_000, // 30 min
+};
+// ── Telemetry scanning ────────────────────────────────────────────────────────
+function hasTelemetryEvent(telemetryFile, eventName, childId) {
+    if (!(0, node_fs_1.existsSync)(telemetryFile))
+        return false;
+    try {
+        const lines = (0, node_fs_1.readFileSync)(telemetryFile, "utf-8").trim().split("\n").filter(Boolean);
+        for (const line of lines) {
+            try {
+                const e = JSON.parse(line);
+                if (e.event === eventName && e.child_id === childId)
+                    return true;
+            }
+            catch {
+                continue;
+            }
+        }
+    }
+    catch { /* ignore */ }
+    return false;
+}
+// ── Emit recovery telemetry events ───────────────────────────────────────────
+function emitRecoveryInitiated(telemetryFile, childId, dispatchId, reason) {
+    appendTelemetry(telemetryFile, {
+        event: "child-recovery-initiated",
+        event_id: (0, node_crypto_1.randomUUID)(),
+        child_id: childId,
+        dispatch_id: dispatchId,
+        recovery_reason: reason,
+        detected_at: new Date().toISOString(),
+    });
+}
+function emitChildOrphaned(telemetryFile, childId, dispatchId, lastHeartbeatAt) {
+    appendTelemetry(telemetryFile, {
+        event: "child-orphaned",
+        event_id: (0, node_crypto_1.randomUUID)(),
+        child_id: childId,
+        dispatch_id: dispatchId,
+        last_heartbeat_at: lastHeartbeatAt,
+        orphaned_at: new Date().toISOString(),
+    });
+}
+function emitRecoveryApprovalRequested(telemetryFile, childId, dispatchId, reason) {
+    appendTelemetry(telemetryFile, {
+        event: "recovery-approval-requested",
+        event_id: (0, node_crypto_1.randomUUID)(),
+        child_id: childId,
+        dispatch_id: dispatchId,
+        recovery_reason: reason,
+        operator_notified_at: new Date().toISOString(),
+    });
+}
+function emitRecoveryApproved(telemetryFile, childId, dispatchId, approvedBy) {
+    appendTelemetry(telemetryFile, {
+        event: "recovery-approved",
+        event_id: (0, node_crypto_1.randomUUID)(),
+        child_id: childId,
+        dispatch_id: dispatchId,
+        approved_by: approvedBy,
+        approved_at: new Date().toISOString(),
+    });
+}
+function emitChildRequeued(telemetryFile, childId, newDispatchId, previousDispatchId) {
+    appendTelemetry(telemetryFile, {
+        event: "child-requeued",
+        event_id: (0, node_crypto_1.randomUUID)(),
+        child_id: childId,
+        new_dispatch_id: newDispatchId,
+        previous_dispatch_id: previousDispatchId,
+        requeued_at: new Date().toISOString(),
+    });
+}
+// ── State mutation for recovery ───────────────────────────────────────────────
+function transitionToOrphaned(stateFile, state, childId) {
+    const childMeta = state.open_children_meta?.[childId];
+    if (!childMeta?.dispatch_record)
+        return;
+    const updatedMeta = {
+        ...state.open_children_meta,
+        [childId]: {
+            ...childMeta,
+            dispatch_record: {
+                ...childMeta.dispatch_record,
+                runtime_state: "orphaned",
+            },
+        },
+    };
+    (0, checkpoint_js_1.writeStateAtomic)(stateFile, { ...state, open_children_meta: updatedMeta });
+}
+/**
+ * Reset a dispatch record for safe auto-redispatch.
+ * Preserves the old dispatch_id in the requeue event for audit.
+ */
+function resetForRedispatch(stateFile, state, childId) {
+    const childMeta = state.open_children_meta?.[childId];
+    if (!childMeta?.dispatch_record)
+        return (0, node_crypto_1.randomUUID)();
+    const newDispatchId = (0, node_crypto_1.randomUUID)();
+    // Clear dispatch record but retain audit trail via telemetry
+    const updatedMeta = {
+        ...state.open_children_meta,
+        [childId]: {
+            ...childMeta,
+            dispatch_record: undefined,
+        },
+    };
+    // Reset active_child so a new dispatch can proceed
+    (0, checkpoint_js_1.writeStateAtomic)(stateFile, {
+        ...state,
+        active_child: "",
+        open_children_meta: updatedMeta,
+    });
+    return newDispatchId;
+}
+/**
+ * Check the current state for orphaned children across all 5 scenarios.
+ * Emits telemetry events for detected cases.
+ * Auto-requeues for safe scenarios (A, B, C, E).
+ * Emits recovery-approval-requested and halts for approval scenario (D).
+ */
+function checkOrphans(options) {
+    const timeouts = {
+        ...DEFAULT_ORPHAN_TIMEOUTS,
+        ...options.timeouts,
+    };
+    let state;
+    try {
+        state = (0, checkpoint_js_1.readState)(options.stateFile);
+    }
+    catch {
+        return { detected: [], checked: 0 };
+    }
+    const telemetryFile = resolveTelemetryFile(state, options.repoRoot);
+    const now = Date.now();
+    const detected = [];
+    let checked = 0;
+    // Check the active child (if any)
+    if (state.active_child) {
+        checked++;
+        const childMeta = state.open_children_meta?.[state.active_child];
+        const dr = childMeta?.dispatch_record;
+        if (dr && dr.runtime_state !== "completed" && dr.runtime_state !== "failed") {
+            const dispatchedAt = new Date(dr.dispatched_at).getTime();
+            const elapsed = now - dispatchedAt;
+            // Scenario A: packet written, no worker_id after launch_timeout
+            if (!dr.worker_id && elapsed > timeouts.launchTimeoutMs) {
+                const detection = {
+                    childId: state.active_child,
+                    dispatchId: dr.dispatch_id,
+                    reason: "no-worker-assignment",
+                    requiresApproval: false,
+                };
+                detected.push(detection);
+                emitRecoveryInitiated(telemetryFile, state.active_child, dr.dispatch_id, "no-worker-assignment");
+                emitChildOrphaned(telemetryFile, state.active_child, dr.dispatch_id, null);
+                const newId = resetForRedispatch(options.stateFile, state, state.active_child);
+                emitChildRequeued(telemetryFile, state.active_child, newId, dr.dispatch_id);
+                return { detected, checked };
+            }
+            // Scenario B: worker_id present, no acknowledgment within launch timeout
+            if (dr.worker_id && !dr.first_heartbeat_at && elapsed > timeouts.launchToFirstHeartbeatMs) {
+                const detection = {
+                    childId: state.active_child,
+                    dispatchId: dr.dispatch_id,
+                    reason: "no-acknowledgment",
+                    requiresApproval: false,
+                };
+                detected.push(detection);
+                emitRecoveryInitiated(telemetryFile, state.active_child, dr.dispatch_id, "no-acknowledgment");
+                emitChildOrphaned(telemetryFile, state.active_child, dr.dispatch_id, null);
+                const newId = resetForRedispatch(options.stateFile, state, state.active_child);
+                emitChildRequeued(telemetryFile, state.active_child, newId, dr.dispatch_id);
+                return { detected, checked };
+            }
+            // Scenario C: acknowledged, heartbeat lost, no result — auto-replace
+            if (dr.first_heartbeat_at && !safeResultExists(dr.expected_result_path)) {
+                const lastHb = dr.last_heartbeat_at ?? dr.first_heartbeat_at;
+                const heartbeatAge = now - new Date(lastHb).getTime();
+                if (heartbeatAge > timeouts.orphanTimeoutMs) {
+                    const detection = {
+                        childId: state.active_child,
+                        dispatchId: dr.dispatch_id,
+                        reason: "no-heartbeat",
+                        requiresApproval: false,
+                    };
+                    detected.push(detection);
+                    emitRecoveryInitiated(telemetryFile, state.active_child, dr.dispatch_id, "no-heartbeat");
+                    emitChildOrphaned(telemetryFile, state.active_child, dr.dispatch_id, lastHb);
+                    const newId = resetForRedispatch(options.stateFile, state, state.active_child);
+                    emitChildRequeued(telemetryFile, state.active_child, newId, dr.dispatch_id);
+                    return { detected, checked };
+                }
+            }
+            // Scenario D: worker-result event received but result artifact missing or invalid
+            const hasWorkerResult = hasTelemetryEvent(telemetryFile, "worker-result", state.active_child);
+            if (hasWorkerResult && !safeResultExists(dr.expected_result_path)) {
+                // Check for commits in child scope (simplified: assume no commits for now)
+                const reason = "missing-result-artifact-no-commits";
+                const detection = {
+                    childId: state.active_child,
+                    dispatchId: dr.dispatch_id,
+                    reason,
+                    requiresApproval: true,
+                };
+                detected.push(detection);
+                emitRecoveryInitiated(telemetryFile, state.active_child, dr.dispatch_id, reason);
+                emitChildOrphaned(telemetryFile, state.active_child, dr.dispatch_id, dr.last_heartbeat_at ?? null);
+                emitRecoveryApprovalRequested(telemetryFile, state.active_child, dr.dispatch_id, reason);
+                transitionToOrphaned(options.stateFile, state, state.active_child);
+                return { detected, checked };
+            }
+            // Scenario E: stale dispatch — dispatched_at + stale_dispatch_timeout with no state change
+            if ((dr.runtime_state === "packet-created" || dr.runtime_state === "delegated") &&
+                elapsed > timeouts.staleDispatchTimeoutMs) {
+                const detection = {
+                    childId: state.active_child,
+                    dispatchId: dr.dispatch_id,
+                    reason: "stale-dispatch",
+                    requiresApproval: false,
+                };
+                detected.push(detection);
+                emitRecoveryInitiated(telemetryFile, state.active_child, dr.dispatch_id, "stale-dispatch");
+                emitChildOrphaned(telemetryFile, state.active_child, dr.dispatch_id, null);
+                const newId = resetForRedispatch(options.stateFile, state, state.active_child);
+                emitChildRequeued(telemetryFile, state.active_child, newId, dr.dispatch_id);
+                return { detected, checked };
+            }
+        }
+    }
+    return { detected, checked };
+}

package/dist/loop/orphan-recovery.test.js

@@ -0,0 +1,236 @@
+"use strict";
+/**
+ * Tests for orphan detection and child recovery workflow (POL-229).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const node_os_1 = require("node:os");
+const orphan_recovery_js_1 = require("./orphan-recovery.js");
+const run_bootstrap_js_1 = require("./run-bootstrap.js");
+function makeTestDir() {
+    const dir = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `polaris-orphan-test-${Date.now()}-${Math.random().toString(16).slice(2)}`);
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    return dir;
+}
+function writeState(dir, state) {
+    const stateFile = (0, node_path_1.join)(dir, "current-state.json");
+    (0, node_fs_1.writeFileSync)(stateFile, JSON.stringify(state, null, 2), "utf-8");
+    return stateFile;
+}
+function baseState(overrides = {}) {
+    return {
+        schema_version: "1.0",
+        run_id: "run-orphan-001",
+        cluster_id: "POL-100",
+        session_type: "implement",
+        branch: "test-branch",
+        active_child: "",
+        completed_children: [],
+        open_children: ["POL-101"],
+        open_children_meta: {},
+        step_cursor: "dispatch",
+        context_budget: { children_completed: 0, max_children_per_session: 3 },
+        status: "running",
+        next_open_child: "POL-101",
+        dispatch_boundary: { dispatch_epoch: 1, continue_epoch: 0, last_dispatched_child: null },
+        run_bootstrap_seal: (0, run_bootstrap_js_1.createBootstrapSeal)("run-orphan-001", "POL-100", ["POL-101"]),
+        ...overrides,
+    };
+}
+function telemetryEvents(dir) {
+    const runId = "run-orphan-001";
+    const telemetryFile = (0, node_path_1.join)(dir, ".taskchain_artifacts", "polaris-run", "runs", runId, "telemetry.jsonl");
+    if (!(0, node_fs_1.existsSync)(telemetryFile))
+        return [];
+    return (0, node_fs_1.readFileSync)(telemetryFile, "utf-8")
+        .trim()
+        .split("\n")
+        .filter(Boolean)
+        .map((l) => JSON.parse(l));
+}
+(0, vitest_1.describe)("checkOrphans", () => {
+    let testDir;
+    (0, vitest_1.beforeEach)(() => {
+        testDir = makeTestDir();
+    });
+    (0, vitest_1.afterEach)(() => {
+        (0, node_fs_1.rmSync)(testDir, { recursive: true, force: true });
+    });
+    function opts(extra = {}) {
+        return {
+            stateFile: (0, node_path_1.join)(testDir, "current-state.json"),
+            repoRoot: testDir,
+            timeouts: {
+                launchTimeoutMs: 100,
+                launchToFirstHeartbeatMs: 100,
+                orphanTimeoutMs: 100,
+                staleDispatchTimeoutMs: 100,
+            },
+            ...extra,
+        };
+    }
+    (0, vitest_1.it)("returns zero detections when no active child", () => {
+        writeState(testDir, baseState());
+        const result = (0, orphan_recovery_js_1.checkOrphans)(opts());
+        (0, vitest_1.expect)(result.detected).toHaveLength(0);
+    });
+    (0, vitest_1.it)("Scenario A: detects no-worker-assignment when worker_id missing after launch timeout", () => {
+        const pastTime = new Date(Date.now() - 500).toISOString();
+        writeState(testDir, baseState({
+            active_child: "POL-101",
+            open_children_meta: {
+                "POL-101": {
+                    dispatch_record: {
+                        dispatch_id: "d-001",
+                        child_id: "POL-101",
+                        run_id: "run-orphan-001",
+                        cluster_id: "POL-100",
+                        packet_path: "/tmp/p.json",
+                        expected_result_path: "/tmp/r.json",
+                        dispatched_at: pastTime,
+                        status: "dispatched",
+                        runtime_state: "packet-created",
+                        // No worker_id
+                    },
+                },
+            },
+        }));
+        const result = (0, orphan_recovery_js_1.checkOrphans)(opts());
+        (0, vitest_1.expect)(result.detected).toHaveLength(1);
+        (0, vitest_1.expect)(result.detected[0].reason).toBe("no-worker-assignment");
+        (0, vitest_1.expect)(result.detected[0].requiresApproval).toBe(false);
+        const events = telemetryEvents(testDir);
+        (0, vitest_1.expect)(events.some((e) => e["event"] === "child-recovery-initiated")).toBe(true);
+        (0, vitest_1.expect)(events.some((e) => e["event"] === "child-orphaned")).toBe(true);
+        (0, vitest_1.expect)(events.some((e) => e["event"] === "child-requeued")).toBe(true);
+    });
+    (0, vitest_1.it)("Scenario B: detects no-acknowledgment when worker_id present but no heartbeat", () => {
+        const pastTime = new Date(Date.now() - 500).toISOString();
+        writeState(testDir, baseState({
+            active_child: "POL-101",
+            open_children_meta: {
+                "POL-101": {
+                    dispatch_record: {
+                        dispatch_id: "d-001",
+                        child_id: "POL-101",
+                        run_id: "run-orphan-001",
+                        cluster_id: "POL-100",
+                        packet_path: "/tmp/p.json",
+                        expected_result_path: "/tmp/r.json",
+                        dispatched_at: pastTime,
+                        status: "dispatched",
+                        runtime_state: "launching",
+                        worker_id: "w-001",
+                        // No first_heartbeat_at
+                    },
+                },
+            },
+        }));
+        const result = (0, orphan_recovery_js_1.checkOrphans)(opts());
+        (0, vitest_1.expect)(result.detected).toHaveLength(1);
+        (0, vitest_1.expect)(result.detected[0].reason).toBe("no-acknowledgment");
+        const events = telemetryEvents(testDir);
+        (0, vitest_1.expect)(events.some((e) => e["event"] === "child-recovery-initiated")).toBe(true);
+        (0, vitest_1.expect)(events.some((e) => e["recovery_reason"] === "no-acknowledgment")).toBe(true);
+    });
+    (0, vitest_1.it)("Scenario C: detects no-heartbeat and requires approval", () => {
+        const pastTime = new Date(Date.now() - 500).toISOString();
+        writeState(testDir, baseState({
+            active_child: "POL-101",
+            open_children_meta: {
+                "POL-101": {
+                    dispatch_record: {
+                        dispatch_id: "d-001",
+                        child_id: "POL-101",
+                        run_id: "run-orphan-001",
+                        cluster_id: "POL-100",
+                        packet_path: "/tmp/p.json",
+                        expected_result_path: (0, node_path_1.join)(testDir, "result-missing.json"), // does not exist
+                        dispatched_at: pastTime,
+                        status: "dispatched",
+                        runtime_state: "running",
+                        worker_id: "w-001",
+                        first_heartbeat_at: pastTime,
+                        last_heartbeat_at: pastTime, // old heartbeat
+                    },
+                },
+            },
+        }));
+        const result = (0, orphan_recovery_js_1.checkOrphans)(opts());
+        (0, vitest_1.expect)(result.detected).toHaveLength(1);
+        (0, vitest_1.expect)(result.detected[0].reason).toBe("no-heartbeat");
+        (0, vitest_1.expect)(result.detected[0].requiresApproval).toBe(true);
+        const events = telemetryEvents(testDir);
+        (0, vitest_1.expect)(events.some((e) => e["event"] === "recovery-approval-requested")).toBe(true);
+    });
+    (0, vitest_1.it)("Scenario E: detects stale-dispatch and auto-requeues", () => {
+        const pastTime = new Date(Date.now() - 500).toISOString();
+        writeState(testDir, baseState({
+            active_child: "POL-101",
+            open_children_meta: {
+                "POL-101": {
+                    dispatch_record: {
+                        dispatch_id: "d-001",
+                        child_id: "POL-101",
+                        run_id: "run-orphan-001",
+                        cluster_id: "POL-100",
+                        packet_path: "/tmp/p.json",
+                        expected_result_path: "/tmp/r.json",
+                        dispatched_at: pastTime,
+                        status: "dispatched",
+                        runtime_state: "packet-created",
+                        // no worker_id, old dispatch
+                    },
+                },
+            },
+        }));
+        // Use longer launchTimeout so Scenario A doesn't trigger first
+        const result = (0, orphan_recovery_js_1.checkOrphans)({
+            ...opts(),
+            timeouts: {
+                launchTimeoutMs: 600_000, // long — won't trigger Scenario A
+                launchToFirstHeartbeatMs: 100,
+                orphanTimeoutMs: 100,
+                staleDispatchTimeoutMs: 100, // short — triggers Scenario E
+            },
+        });
+        (0, vitest_1.expect)(result.detected).toHaveLength(1);
+        (0, vitest_1.expect)(result.detected[0].reason).toBe("stale-dispatch");
+        const events = telemetryEvents(testDir);
+        (0, vitest_1.expect)(events.some((e) => e["event"] === "child-requeued")).toBe(true);
+    });
+    (0, vitest_1.it)("no detection when dispatch is fresh", () => {
+        const nowTime = new Date().toISOString();
+        writeState(testDir, baseState({
+            active_child: "POL-101",
+            open_children_meta: {
+                "POL-101": {
+                    dispatch_record: {
+                        dispatch_id: "d-001",
+                        child_id: "POL-101",
+                        run_id: "run-orphan-001",
+                        cluster_id: "POL-100",
+                        packet_path: "/tmp/p.json",
+                        expected_result_path: "/tmp/r.json",
+                        dispatched_at: nowTime,
+                        status: "dispatched",
+                        runtime_state: "launching",
+                        worker_id: "w-001",
+                    },
+                },
+            },
+        }));
+        const result = (0, orphan_recovery_js_1.checkOrphans)({
+            ...opts(),
+            timeouts: {
+                launchTimeoutMs: 60_000,
+                launchToFirstHeartbeatMs: 60_000,
+                orphanTimeoutMs: 600_000,
+                staleDispatchTimeoutMs: 1_800_000,
+            },
+        });
+        (0, vitest_1.expect)(result.detected).toHaveLength(0);
+    });
+});