@indigoai-us/hq-cli 5.345.175 → 5.345.176

package/dist/lib/lanes/loop-disposition.d.ts

@@ -0,0 +1,44 @@
+import type { LaneRecord } from "./types.js";
+export interface RecoveryIdentity {
+    isAlive(pid: number): boolean;
+    readStarttime(pid: number): number | null;
+}
+interface Entry {
+    directory: "active" | "claimed";
+    name: string;
+    sha256: string;
+    envelopeId: string | null;
+}
+export interface LoopDispositionSnapshot {
+    laneId: string;
+    version: number;
+    authority: Pick<LaneRecord, "senior" | "company" | "mesh_company" | "host">;
+    fingerprint: string;
+    leaseFingerprint: string;
+    unknownLeaseOwner: boolean;
+    envelopes: Entry[];
+    pending: Array<{
+        name: string;
+        sha256: string;
+    }>;
+    statusFingerprint: string;
+}
+export declare function loopDispositionSnapshot(hqRoot: string, lane: LaneRecord): LoopDispositionSnapshot;
+export declare function applyLoopDisposition(input: {
+    hqRoot: string;
+    laneId: string;
+    expectedVersion: number;
+    fingerprint: string;
+    disposition: "fail" | "abandon";
+    reason: string;
+    ownershipAttested?: boolean;
+    authorize(lane: LaneRecord): void;
+    identity?: RecoveryIdentity;
+    now?: () => Date;
+}): Promise<{
+    lane: LaneRecord;
+    decision: string;
+    externalEffects: "not-certified";
+}>;
+export {};
+//# sourceMappingURL=loop-disposition.d.ts.map

package/dist/lib/lanes/loop-disposition.js

@@ -0,0 +1,302 @@
+/** Explicit local effects disposition. This module never executes a phase. */
+import { createHash, randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { withAllLaneAdmissionLocks } from "./admission.js";
+import { listLoopQueue, loopControllerIsAlive, resetLoopAfterDisposition } from "./loop.js";
+import { isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime } from "./spawn.js";
+import { listQuestionIds, readLane, readQuestion, updateLane } from "./store.js";
+const digest = (value) => createHash("sha256").update(value).digest("hex");
+const defaultIdentity = {
+    isAlive(pid) {
+        try {
+            process.kill(pid, 0);
+            return true;
+        }
+        catch (error) {
+            if (error.code === "ESRCH")
+                return false;
+            if (error.code === "EPERM")
+                return true;
+            throw error;
+        }
+    },
+    readStarttime: readProcStarttime,
+};
+/** Refuse linked directory components before looking at any recovery input. */
+function assertDirectoryChain(root, directory) {
+    const relative = path.relative(root, directory);
+    if (relative.startsWith("..") || path.isAbsolute(relative))
+        throw new Error("Recovery directory is outside its root");
+    let current = root;
+    for (const component of ["", ...relative.split(path.sep).filter(Boolean)]) {
+        if (component)
+            current = path.join(current, component);
+        let stat;
+        try {
+            stat = fs.lstatSync(current);
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return;
+            throw error;
+        }
+        if (!stat.isDirectory() || stat.isSymbolicLink())
+            throw new Error("Recovery directory must be a real directory, not a symbolic link");
+    }
+}
+function recoveryRoot(runDir) {
+    return path.dirname(path.dirname(path.dirname(runDir)));
+}
+function assertRecoveryDirectories(runDir) {
+    const root = recoveryRoot(runDir);
+    for (const directory of ["", "active", "claimed", "pending", "recovery"]) {
+        assertDirectoryChain(root, path.join(runDir, "loop", directory));
+    }
+}
+function readRegularFile(file) {
+    const before = fs.lstatSync(file);
+    if (!before.isFile())
+        throw new Error("Recovery inputs must be regular files");
+    const fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0));
+    try {
+        const opened = fs.fstatSync(fd);
+        if (!opened.isFile() || opened.dev !== before.dev || opened.ino !== before.ino)
+            throw new Error("Recovery input changed while opening");
+        return fs.readFileSync(fd);
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+function fileHash(file) {
+    return digest(readRegularFile(file));
+}
+function leaseView(runDir) {
+    const lock = path.join(runDir, "loop", "runner.lock");
+    let stat;
+    try {
+        stat = fs.lstatSync(lock);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return { fingerprint: "absent" };
+        throw error;
+    }
+    // Legacy format only. A new-format/linked/unknown lock is not silently removed.
+    if (!stat.isDirectory())
+        throw new Error("Unsupported runner lease format; do not convert a live root");
+    const entries = fs.readdirSync(lock).sort().map((name) => {
+        const file = path.join(lock, name);
+        return { name, hash: fileHash(file) };
+    });
+    const fingerprint = digest(JSON.stringify({ dev: stat.dev, ino: stat.ino, entries }));
+    let owner;
+    try {
+        owner = JSON.parse(readRegularFile(path.join(lock, "owner.json")).toString("utf8"));
+    }
+    catch (error) {
+        if (error.code !== "ENOENT" && !(error instanceof SyntaxError))
+            throw error;
+    }
+    if (owner && typeof owner === "object" && Number.isSafeInteger(owner.pid) && Number(owner.pid) > 0) {
+        const value = owner;
+        return { fingerprint, owner: { pid: value.pid, ...(typeof value.starttime === "number" ? { starttime: value.starttime } : {}) } };
+    }
+    return { fingerprint };
+}
+function entries(runDir) {
+    const out = [];
+    for (const directory of ["active", "claimed"]) {
+        const dir = path.join(runDir, "loop", directory);
+        if (!fs.existsSync(dir))
+            continue;
+        for (const name of fs.readdirSync(dir).sort().filter((name) => !name.startsWith("."))) {
+            const file = path.join(dir, name);
+            const sha256 = fileHash(file);
+            let record = null;
+            try {
+                const parsed = JSON.parse(readRegularFile(file).toString("utf8"));
+                if (parsed !== null && typeof parsed === "object")
+                    record = parsed;
+            }
+            catch (error) {
+                if (!(error instanceof SyntaxError))
+                    throw error;
+            }
+            if (record?.envelope?.kind === "stop")
+                throw new Error("An intentional stop requires its ordinary stop path");
+            // Corrupt records remain opaque and bound to their exact bytes;
+            // explicit disposition archives them without interpreting/replaying them.
+            out.push({ directory, name, sha256, envelopeId: typeof record?.envelope_id === "string" ? record.envelope_id : null });
+        }
+    }
+    return out;
+}
+function assertNoLiveProcess(runDir, identity) {
+    if (loopControllerIsAlive(runDir, identity.isAlive, identity.readStarttime))
+        throw new Error("A loop controller is still alive");
+    for (const name of ["worker.pid", "watcher.pid"]) {
+        const record = readPidRecord(path.join(runDir, name));
+        if (record && isPidRecordAlive(record, identity.isAlive, identity.readStarttime))
+            throw new Error("A worker or watcher is still alive");
+    }
+    const lease = leaseView(runDir);
+    if (lease.owner && isPidRecordAlive(lease.owner, identity.isAlive, identity.readStarttime))
+        throw new Error("The lease owner is still alive");
+}
+export function loopDispositionSnapshot(hqRoot, lane) {
+    const runDir = laneRunDir(hqRoot, lane.lane_id);
+    assertRecoveryDirectories(runDir);
+    const lease = leaseView(runDir);
+    const status = path.join(runDir, "loop", "loop.json");
+    const pendingDir = path.join(runDir, "loop", "pending");
+    const value = {
+        laneId: lane.lane_id,
+        version: lane.version,
+        authority: { senior: lane.senior, company: lane.company, mesh_company: lane.mesh_company, host: lane.host },
+        leaseFingerprint: lease.fingerprint,
+        unknownLeaseOwner: lease.fingerprint !== "absent" && !lease.owner,
+        envelopes: entries(runDir),
+        pending: fs.existsSync(pendingDir) ? fs.readdirSync(pendingDir).sort().filter((name) => !name.startsWith(".")).map((name) => ({ name, sha256: fileHash(path.join(pendingDir, name)) })) : [],
+        statusFingerprint: fs.existsSync(status) ? fileHash(status) : "absent",
+    };
+    return { ...value, fingerprint: digest(JSON.stringify(value)) };
+}
+function atomicRecord(file, value, runDir) {
+    assertRecoveryDirectories(runDir);
+    assertDirectoryChain(recoveryRoot(runDir), path.dirname(file));
+    const temporary = `${file}.${randomUUID()}.tmp`;
+    const fd = fs.openSync(temporary, "wx", 0o600);
+    try {
+        fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`);
+        fs.fsyncSync(fd);
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+    fs.renameSync(temporary, file);
+    const directory = fs.openSync(path.dirname(file), "r");
+    try {
+        fs.fsyncSync(directory);
+    }
+    finally {
+        fs.closeSync(directory);
+    }
+}
+export async function applyLoopDisposition(input) {
+    if (!/^[a-f0-9]{64}$/.test(input.fingerprint))
+        throw new Error("A preview fingerprint is required");
+    if (!input.reason.trim() || input.reason.length > 2000)
+        throw new Error("A bounded disposition reason is required");
+    const identity = input.identity ?? defaultIdentity;
+    return withAllLaneAdmissionLocks(input.hqRoot, async () => {
+        const lane = readLane(input.hqRoot, input.laneId);
+        input.authorize(lane);
+        if (!lane.loop_mode)
+            throw new Error("Not a loop lane");
+        if (listQuestionIds(input.hqRoot).some((id) => {
+            const q = readQuestion(input.hqRoot, id);
+            return q.lane_id === lane.lane_id && q.status === "pending";
+        }))
+            throw new Error("Pending questions must be resolved first");
+        const runDir = laneRunDir(input.hqRoot, lane.lane_id);
+        assertRecoveryDirectories(runDir);
+        const audit = path.join(runDir, "loop", "recovery", input.fingerprint);
+        assertDirectoryChain(input.hqRoot, audit);
+        const decisionFile = path.join(audit, "decision.json");
+        let decision;
+        if (fs.existsSync(decisionFile))
+            decision = JSON.parse(readRegularFile(decisionFile).toString("utf8"));
+        if (decision && (decision.disposition !== input.disposition || decision.reason !== input.reason || decision.snapshot.version !== input.expectedVersion))
+            throw new Error("Disposition does not match the recorded decision");
+        if (lane.terminal_details?.recovery_decision === input.fingerprint) {
+            if (!decision)
+                throw new Error("The recorded recovery decision is missing");
+            const complete = path.join(audit, "complete.json");
+            if (!fs.existsSync(complete))
+                atomicRecord(complete, { laneVersion: lane.version, at: (input.now ?? (() => new Date()))().toISOString() }, runDir);
+            return { lane, decision: input.fingerprint, externalEffects: "not-certified" };
+        }
+        if (lane.version !== input.expectedVersion)
+            throw new Error("Lane version changed; inspect again");
+        if (lane.terminal_reason?.startsWith("operator_") || ["story_closed", "loop_stopped"].includes(lane.terminal_reason ?? "")) {
+            throw new Error("Intentionally protected lane must use its ordinary lifecycle path");
+        }
+        if (lane.state !== "interrupted" && lane.state !== "done")
+            throw new Error("Only held or historical done lanes may be explicitly recovered");
+        const statusFile = path.join(runDir, "loop", "loop.json");
+        const status = fs.existsSync(statusFile) ? JSON.parse(readRegularFile(statusFile).toString("utf8")) : {};
+        if (["parked", "stopped", "failed"].includes(String(status.state)) && !(decision && status.recovery_decision === input.fingerprint))
+            throw new Error("Protected loop state cannot be recovered automatically");
+        assertNoLiveProcess(runDir, identity);
+        if (!decision) {
+            const snapshot = loopDispositionSnapshot(input.hqRoot, lane);
+            if (snapshot.fingerprint !== input.fingerprint)
+                throw new Error("Envelope or lease fingerprint changed; inspect again");
+            if (snapshot.unknownLeaseOwner && !input.ownershipAttested)
+                throw new Error("Unknown lease ownership requires explicit investigated-ownership attestation");
+            decision = { snapshot, disposition: input.disposition, reason: input.reason, ownershipAttested: input.ownershipAttested === true, at: (input.now ?? (() => new Date()))().toISOString(), externalEffects: "not-certified" };
+            assertDirectoryChain(input.hqRoot, audit);
+            fs.mkdirSync(audit, { recursive: true, mode: 0o700 });
+            assertDirectoryChain(input.hqRoot, audit);
+            // Persist the human/senior decision before any file is archived or lane CAS.
+            atomicRecord(decisionFile, decision, runDir);
+        }
+        const snapshot = decision.snapshot;
+        // Retry may resume partial archival, but must not silently discard new work.
+        const currentSnapshot = loopDispositionSnapshot(input.hqRoot, lane);
+        if (JSON.stringify(currentSnapshot.pending) !== JSON.stringify(snapshot.pending))
+            throw new Error("Pending queue changed during disposition");
+        if (currentSnapshot.statusFingerprint !== snapshot.statusFingerprint) {
+            const status = JSON.parse(readRegularFile(path.join(runDir, "loop", "loop.json")).toString("utf8"));
+            if (status.recovery_decision !== input.fingerprint)
+                throw new Error("Controller status changed during disposition");
+        }
+        const expected = new Set(snapshot.envelopes.map((entry) => `${entry.directory}/${entry.name}/${entry.sha256}`));
+        if (currentSnapshot.envelopes.some((entry) => !expected.has(`${entry.directory}/${entry.name}/${entry.sha256}`)))
+            throw new Error("New recovery input appeared during disposition");
+        for (const entry of snapshot.envelopes) {
+            assertRecoveryDirectories(runDir);
+            const source = path.join(runDir, "loop", entry.directory, entry.name);
+            const destination = path.join(audit, entry.directory, entry.name);
+            if (fs.existsSync(source)) {
+                if (fileHash(source) !== entry.sha256 || fs.existsSync(destination))
+                    throw new Error("Recovery envelope changed during disposition");
+                assertDirectoryChain(input.hqRoot, path.dirname(destination));
+                fs.mkdirSync(path.dirname(destination), { recursive: true, mode: 0o700 });
+                assertDirectoryChain(input.hqRoot, path.dirname(destination));
+                assertRecoveryDirectories(runDir);
+                fs.renameSync(source, destination);
+            }
+            else if (!fs.existsSync(destination) || fileHash(destination) !== entry.sha256)
+                throw new Error("Recovery archive is incomplete");
+        }
+        assertRecoveryDirectories(runDir);
+        const lease = leaseView(runDir);
+        const quarantine = path.join(audit, "runner.lock");
+        if (snapshot.leaseFingerprint !== "absent") {
+            if (lease.fingerprint !== "absent") {
+                if (lease.fingerprint !== snapshot.leaseFingerprint || fs.existsSync(quarantine))
+                    throw new Error("Runner lease changed during disposition");
+                fs.renameSync(path.join(runDir, "loop", "runner.lock"), quarantine);
+            }
+            else if (!fs.existsSync(quarantine))
+                throw new Error("Lease quarantine is incomplete");
+        }
+        else if (lease.fingerprint !== "absent")
+            throw new Error("A runner acquired the lease during disposition");
+        const pending = input.disposition === "abandon" && lane.state !== "done" && listLoopQueue(runDir).length > 0;
+        const nextState = lane.state === "done" ? "done" : pending ? "queued" : "failed";
+        resetLoopAfterDisposition(runDir, lane.lane_id, input.fingerprint, pending);
+        const updated = updateLane(input.hqRoot, lane.lane_id, input.expectedVersion, "command", (current) => ({
+            ...current, state: nextState, queued_reason: undefined, queued_details: undefined,
+            admission_reserved_at: undefined, pending_resume: undefined,
+            terminal_reason: lane.state === "done" ? lane.terminal_reason : pending ? undefined : "loop_recovery_disposed",
+            terminal_details: { recovery_decision: input.fingerprint, external_effects: "not-certified", disposition: input.disposition, ...(pending ? { loop_recovery: true } : {}) },
+        }));
+        atomicRecord(path.join(audit, "complete.json"), { laneVersion: updated.version, at: (input.now ?? (() => new Date()))().toISOString() }, runDir);
+        return { lane: updated, decision: input.fingerprint, externalEffects: "not-certified" };
+    });
+}
+//# sourceMappingURL=loop-disposition.js.map

package/dist/lib/lanes/loop.d.ts

@@ -47,6 +47,8 @@ export interface LoopStatus {
     active_phase: string | null;
     stall_counts: Record<string, number>;
     pid?: number;
+    starttime?: number;
+    recovery_decision?: string;
     updated_at: string;
     handoff_hold?: {
         envelope_id: string;
@@ -124,6 +126,10 @@ export interface LoopRunnerOptions {
     idlePollMs?: number;
     signal?: AbortSignal;
     repairInvalidHandoff?: boolean;
+    processIdentity?: {
+        isAlive(pid: number): boolean;
+        readStarttime(pid: number): number | null;
+    };
 }
 export interface LoopRunSummary {
     state: "stopped" | "parked" | "aborted";
@@ -203,14 +209,21 @@ export declare function loopStatus(runDir: string, laneId: string): LoopStatus;
  * and changes nothing when the queue holds anything ahead of the stop.
  */
 export declare function commitIdleLoopStop(runDir: string, laneId: string, now?: () => Date): boolean;
-export declare function setLoopControllerPid(runDir: string, laneId: string, pid: number | undefined): void;
+export declare function setLoopControllerPid(runDir: string, laneId: string, pid: number | undefined, starttime?: number): void;
 export declare function clearLoopControllerPid(runDir: string, laneId: string, pid: number): void;
 /**
  * True when a loop lane's controller is alive with no phase active: the lane
  * finished its last phase and is waiting for the next envelope.
  */
-export declare function loopLaneIsIdle(runDir: string, laneId: string): boolean;
-export declare function loopControllerIsAlive(runDir: string): boolean;
+export declare function loopLaneIsIdle(runDir: string, laneId: string, isAlive?: (pid: number) => boolean, readStarttime?: (pid: number) => number | null): boolean;
+export declare function loopControllerIsAlive(runDir: string, isAlive?: (pid: number) => boolean, readStarttime?: (pid: number) => number | null): boolean;
+/** Read durable work without changing queue files or assuming effects are replayable. */
+export declare function inspectLoopRecovery(runDir: string): {
+    kind: "pending" | "hold" | "protected" | "empty";
+    reason?: string;
+};
+/** Reset only after an explicit, durable disposition; this never starts work. */
+export declare function resetLoopAfterDisposition(runDir: string, laneId: string, decision: string, hasPending: boolean): void;
 /** Run queued phases serially. Sessions are isolated by story and provider. */
 export declare function runLoopLane(options: LoopRunnerOptions): Promise<LoopRunSummary>;
 export {};