@indigoai-us/hq-cli 5.314.0 → 5.315.0

package/dist/lib/lanes/reconcile-install.js

@@ -0,0 +1,257 @@
+/**
+ * systemd user timer install for the lanes reconcile pass.
+ *
+ * Policy: hq-infra-watchdogs-systemd-not-session. The pass is a oneshot
+ * invoked by a timer, never a session-bound background job. Install writes
+ * unit files under a systemd user dir and is idempotent and reversible.
+ * Enabling the timer is optional — the CLI pass works without it.
+ */
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { LanesError } from "./errors.js";
+import { RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME } from "./reconcile.js";
+import { resolveHqBinary, resolveNodeBinary, } from "../mesh/live/daemon/install.js";
+import { SYSTEMD_PATH_ENV } from "../service-manager/systemd.js";
+export const RECONCILE_TIMER_INTERVAL = "1min";
+/**
+ * Resolve provider commands with the installing shell's PATH. The resulting
+ * directories are carried into the systemd unit, whose default PATH is too
+ * narrow for per-user provider installations.
+ *
+ * A default config enables one account per provider, so `providers` usually
+ * names every provider even on a box with one installed. A provider that does
+ * not resolve is reported in `unresolved`; the install only refuses when none
+ * of the requested providers resolves, because then the timer has nothing to
+ * run.
+ */
+export function providerPathEnv(providers, pathEnv = process.env.PATH) {
+    const searched = pathEnv ?? "";
+    const directories = searched.split(path.delimiter).filter(Boolean);
+    const resolvedDirs = [];
+    const resolved = [];
+    const unresolved = [];
+    for (const provider of providers) {
+        const found = directories.find((directory) => {
+            try {
+                fs.accessSync(path.join(directory, provider), fs.constants.X_OK);
+                return true;
+            }
+            catch {
+                return false;
+            }
+        });
+        if (!found) {
+            unresolved.push(provider);
+            continue;
+        }
+        resolved.push(provider);
+        // Both directories: the one holding the literal command name (an npm
+        // `codex` link may target `codex.js`, a versioned claude launcher has
+        // another basename; spawn.ts launches the literal name, so its directory
+        // must be on the unit PATH) and the real binary's directory (a wrapper
+        // that only resolves with the installing shell's environment is not
+        // enough for systemd). Unrelated shell PATH entries stay out.
+        const linkDir = path.resolve(found);
+        const realDir = path.dirname(fs.realpathSync(path.join(found, provider)));
+        for (const dir of [linkDir, realDir]) {
+            if (!resolvedDirs.includes(dir))
+                resolvedDirs.push(dir);
+        }
+    }
+    if (providers.length > 0 && resolved.length === 0) {
+        throw new LanesError("provider_binary_unresolved", `Cannot resolve any provider binary (${providers.join(", ")}) from PATH`, { providers: [...providers], searched_path: searched });
+    }
+    return { resolved_dirs: resolvedDirs, resolved, unresolved };
+}
+/**
+ * Directory of the hq entry point as invoked, not its realpath: the `hq`
+ * name on PATH is a symlink in a bin directory, and that bin directory is
+ * what the reconcile pass needs on PATH to run `hq` subcommands.
+ */
+function hqBinaryDir(hqBinary) {
+    return path.dirname(path.resolve(hqBinary));
+}
+/**
+ * The unit's PATH: each resolved provider binary directory, the directory of
+ * the hq binary itself, the directory of the node binary that runs it (a
+ * provider installed as a `#!/usr/bin/env node` script needs `node` on the
+ * unit PATH, and nvm/asdf installs keep it outside the baseline), then the
+ * systemd baseline, deduplicated. Nothing
+ * from the installing shell's PATH is carried over: on a box whose profile
+ * is synced from another machine that PATH holds directories from the other
+ * machine (measured 2026-09-22: `/Users/…` and `/opt/homebrew/…` entries on
+ * a Linux outpost), and none of them belong in a systemd unit.
+ */
+export function unitPathEnv(input) {
+    const baseline = SYSTEMD_PATH_ENV.split(":");
+    const nodeDir = path.dirname(path.resolve(input.nodeBinary));
+    return [
+        ...new Set([
+            ...input.providerDirs,
+            hqBinaryDir(input.hqBinary),
+            // A baseline node (/usr/bin/node) keeps its baseline position.
+            ...(baseline.includes(nodeDir) ? [] : [nodeDir]),
+            ...baseline,
+        ]),
+    ]
+        .filter(Boolean)
+        .join(":");
+}
+function shellQuote(v) {
+    return /^[A-Za-z0-9_./:-]+$/.test(v) ? v : `'${v.replace(/'/g, `'\\''`)}'`;
+}
+export function renderReconcileService(opts) {
+    const exec = [opts.nodeBinary, opts.hqBinary, "lanes", "reconcile"]
+        .map(shellQuote)
+        .join(" ");
+    const home = opts.home ?? os.homedir();
+    const pathEnv = opts.pathEnv ?? SYSTEMD_PATH_ENV;
+    return `[Unit]
+Description=HQ lanes reconcile pass
+Documentation=file://personal/projects/hq-cli-lanes/SPEC.md
+
+[Service]
+Type=oneshot
+ExecStart=${exec}
+WorkingDirectory=${shellQuote(opts.hqRoot)}
+Nice=10
+Environment=HOME=${shellQuote(home)}
+Environment=PATH=${shellQuote(pathEnv)}
+Environment=HQ_ROOT=${shellQuote(opts.hqRoot)}
+Environment=HQ_NO_UPDATE_CHECK=1
+KillMode=process
+
+[Install]
+WantedBy=default.target
+`;
+}
+export function renderReconcileTimer() {
+    return `[Unit]
+Description=Periodic HQ lanes reconcile pass
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=${RECONCILE_TIMER_INTERVAL}
+AccuracySec=10s
+Persistent=true
+Unit=${RECONCILE_UNIT_NAME}
+
+[Install]
+WantedBy=timers.target
+`;
+}
+export function defaultSystemdUserDir(home = os.homedir()) {
+    return path.join(home, ".config", "systemd", "user");
+}
+function writeIfChanged(filePath, contents) {
+    try {
+        const existing = fs.readFileSync(filePath, "utf8");
+        if (existing === contents)
+            return false;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code !== "ENOENT")
+            throw err;
+    }
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    const tmp = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+    try {
+        fs.writeFileSync(tmp, contents, { encoding: "utf8", mode: 0o644 });
+        fs.renameSync(tmp, filePath);
+    }
+    catch (err) {
+        try {
+            fs.unlinkSync(tmp);
+        }
+        catch (unlinkErr) {
+            const unlinkCode = unlinkErr.code;
+            if (unlinkCode !== "ENOENT") {
+                process.stderr.write(`hq lanes reconcile install: unlink tmp failed for ${tmp}: ${unlinkErr instanceof Error ? unlinkErr.message : String(unlinkErr)}\n`);
+            }
+        }
+        throw err;
+    }
+    return true;
+}
+function unlinkIfExists(filePath) {
+    try {
+        fs.unlinkSync(filePath);
+        return true;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return false;
+        throw err;
+    }
+}
+function runSystemctl(args, runCommand) {
+    const result = runCommand("systemctl", args);
+    if (result.status !== 0) {
+        throw new LanesError("systemd_failed", `systemctl ${args.join(" ")} failed: ${(result.stderr ?? "").trim() || `exit ${result.status}`}`, { args, status: result.status, stderr: result.stderr });
+    }
+}
+export function installReconcileTimer(opts) {
+    const nodeBinary = opts.nodeBinary ?? resolveNodeBinary();
+    const hqBinary = opts.hqBinary ?? resolveHqBinary();
+    const unitPath = path.join(opts.systemdRoot, RECONCILE_UNIT_NAME);
+    const timerPath = path.join(opts.systemdRoot, RECONCILE_TIMER_NAME);
+    const unitChanged = writeIfChanged(unitPath, renderReconcileService({
+        nodeBinary,
+        hqBinary,
+        hqRoot: opts.hqRoot,
+        home: opts.home,
+        pathEnv: unitPathEnv({ providerDirs: opts.providerDirs ?? [], hqBinary, nodeBinary }),
+    }));
+    const timerChanged = writeIfChanged(timerPath, renderReconcileTimer());
+    let enabled = false;
+    if (opts.enable) {
+        if (!opts.runCommand) {
+            throw new LanesError("systemd_enable_requires_runner", "--enable needs a systemctl runner; pass runCommand or omit --enable to write unit files only", {});
+        }
+        runSystemctl(["--user", "daemon-reload"], opts.runCommand);
+        runSystemctl(["--user", "enable", "--now", RECONCILE_TIMER_NAME], opts.runCommand);
+        enabled = true;
+    }
+    return {
+        ok: true,
+        action: "reconcile.install",
+        systemd_root: opts.systemdRoot,
+        unit: unitPath,
+        timer: timerPath,
+        written: unitChanged || timerChanged,
+        enabled,
+    };
+}
+export function uninstallReconcileTimer(opts) {
+    if (opts.disable) {
+        if (!opts.runCommand) {
+            throw new LanesError("systemd_enable_requires_runner", "--disable needs a systemctl runner", {});
+        }
+        const disable = opts.runCommand("systemctl", [
+            "--user",
+            "disable",
+            "--now",
+            RECONCILE_TIMER_NAME,
+        ]);
+        if (disable.status !== 0) {
+            process.stderr.write(`hq lanes reconcile uninstall: systemctl disable exited ${disable.status}: ${(disable.stderr ?? "").trim()}\n`);
+        }
+        opts.runCommand("systemctl", ["--user", "daemon-reload"]);
+    }
+    const unitPath = path.join(opts.systemdRoot, RECONCILE_UNIT_NAME);
+    const timerPath = path.join(opts.systemdRoot, RECONCILE_TIMER_NAME);
+    const removedUnit = unlinkIfExists(unitPath);
+    const removedTimer = unlinkIfExists(timerPath);
+    return {
+        ok: true,
+        action: "reconcile.uninstall",
+        systemd_root: opts.systemdRoot,
+        unit: unitPath,
+        timer: timerPath,
+        removed: removedUnit || removedTimer,
+    };
+}
+//# sourceMappingURL=reconcile-install.js.map

package/dist/lib/lanes/reconcile.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Periodic reconcile pass (spec §5.7, §6.8, §9).
+ *
+ * Owned by the machine, not by any lane. Reads never mutate; every write is a
+ * compare-and-set. The per-lane watcher owns progress/terminal/silence of a
+ * live process — this pass owns the transitions no single lane can own.
+ */
+import { type LivenessDeps } from "./liveness.js";
+import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
+import { type ThrottleDeps, type CpuBusyObservation } from "./throttle.js";
+import { readLanesRegistryFlag } from "./registry-flag.js";
+import { type RunawayProcessInfo } from "./runaway-tool-guard.js";
+import { type KeepAliveBlockedReason } from "./keep-alive.js";
+import type { LaneCompanyRef, LaneAccountSelection, LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
+import { type LaneMesh } from "./mesh-events.js";
+/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
+export declare const QUESTION_AGE_MS: number;
+/** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
+export declare const MAX_AGED_ADVANCES: number;
+/** Aligns with watcher silence: a host that has not reported for this long is dead. */
+export declare const DEAD_HOST_MS: number;
+export declare const RECONCILE_LOCK_STALE_MS = 30000;
+export declare const RECONCILE_LOCK_TIMEOUT_MS = 5000;
+/** A pickup probe that has not cleared its marker within this window is abandoned. */
+export declare const PICKUP_VALIDATION_MAX_AGE_MS: number;
+export declare const PICKUP_VALIDATION_TIMEOUT_REASON: "pickup_validation_timeout";
+export declare const CLOSE_INTERRUPTED_DONE_STORY_WORKERS_FLAG = "lanes.close-interrupted-done-story-workers";
+export declare const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
+export declare const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
+export interface ReconcileSkip {
+    kind: "lane" | "question" | "store" | "throttle" | "liveness";
+    id?: string;
+    why: string;
+}
+export interface ReconcileChange {
+    id: string;
+    from?: string;
+    to?: string;
+    reason?: string;
+    /** The attempted provider PATH when admission found no executable. */
+    searched_path?: string;
+}
+export interface ReconcileSummary {
+    ok: boolean;
+    action: "reconcile";
+    dry_run: boolean;
+    admitted: ReconcileChange[];
+    overdue: ReconcileChange[];
+    aged: ReconcileChange[];
+    advanced: ReconcileChange[];
+    resumed: ReconcileChange[];
+    dead_host: ReconcileChange[];
+    heartbeats: string[];
+    skipped: ReconcileSkip[];
+    keep_alive_blocked: Array<{
+        id: string;
+        reason: KeepAliveBlockedReason;
+    }>;
+    errors: Array<{
+        message: string;
+        code?: string;
+    }>;
+    heartbeat_at: string | null;
+    rollout_retention?: {
+        scanned: number;
+        compressed: string[];
+        deleted: string[];
+        bytes_freed: number;
+    };
+}
+export interface ReconcileHost {
+    hostname: string;
+    outpostId: string | null;
+}
+export interface ReconcileDeps {
+    now: () => Date;
+    host: ReconcileHost;
+    dryRun: boolean;
+    spawn?: (hqRoot: string, lane: LaneRecord) => LaneRecord;
+    spawnRepair?: (hqRoot: string, lane: LaneRecord, input: EnvelopeRepairSpawnInput) => number;
+    spawnLaneDeps?: SpawnLaneDeps;
+    throttleDeps?: ThrottleDeps;
+    livenessDeps?: LivenessDeps;
+    isAlive?: (pid: number) => boolean;
+    readSessionSenior?: (sessionId: string) => SeniorRef;
+    presenceConnected?: boolean;
+    env?: NodeJS.ProcessEnv;
+    /** CPU sample taken before acquiring provider admission locks. */
+    cpuBusyFraction?: number | null;
+    /** CPU observation and cumulative counters taken before provider locks. */
+    cpuObservation?: CpuBusyObservation;
+    /** Skip admission this pass if candidates appeared after the pre-sample snapshot. */
+    skipAdmissionWithoutCpuSample?: boolean;
+    /** Flag reader seam used to assert rollout reads happen before admission locks. */
+    readRegistryFlag?: typeof readLanesRegistryFlag;
+    /** Active-company fallback for questions raised from an unscoped lane. */
+    resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
+    /** Resume queued messages after this pass wins a local dead-worker CAS. */
+    recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
+    /** Detached mesh seam for dead-watcher recovery. */
+    mesh?: LaneMesh | null;
+    /** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
+    meshEmitTimeoutMs?: number;
+    /** Restrict keep-alive execution to the terminal-triggered lane version. */
+    terminalKeepAliveCandidate?: {
+        laneId: string;
+        version: number;
+    };
+    /** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
+    readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
+    /** Reuse the command's Claude resume selection and session-move path for keep-alive. */
+    prepareKeepAliveClaudeResume?: (lane: LaneRecord) => Promise<{
+        lane: LaneRecord;
+        selectionFailure?: string;
+        accountSelection?: LaneAccountSelection;
+    }>;
+    /** Process-table snapshot seam for the opt-in runaway-search guard. */
+    runawayToolProcessLister?: () => readonly RunawayProcessInfo[];
+    /** Test seam: prevent real process signals while exercising the guard. */
+    runawayToolProcessSignal?: (pid: number, signal: NodeJS.Signals) => void;
+    /** Test seam for interleaving a watcher signal with an append-only guard event. */
+    runawayToolEventAppend?: (target: string, line: string) => void;
+    /** Test seam for validating a candidate PID against its process-table starttime. */
+    runawayToolProcessIsCurrent?: (pid: number, expectedStarttime: number) => boolean;
+    /**
+     * Awaited after lane/question records and liveness are loaded, before any
+     * CAS write. Test seam so two in-process passes can share a snapshot.
+     */
+    afterLoad?: () => Promise<void>;
+}
+export declare function reconcileHeartbeatPath(hqRoot: string): string;
+export declare function timeoutPath(hqRoot: string, laneId: string): string;
+export declare function laneRunsHere(lane: LaneRecord, host: ReconcileHost): boolean;
+export declare function agedAdvanceCount(question: QuestionRecord): number;
+export declare function currentHopStartedAtMs(question: QuestionRecord): number;
+export declare function shouldAgeQuestion(question: QuestionRecord, audience: SeniorRef, nowMs: number, ageWindowMs?: number): boolean;
+export declare function shouldAdvanceAudienceEnded(audience: SeniorRef, liveness: LivenessState | null): boolean;
+export declare function readTimeoutDeadline(hqRoot: string, laneId: string): {
+    deadline_at: string;
+    timeout_secs?: number;
+} | null;
+export declare function isDeadlinePassed(deadlineAt: string, nowMs: number): boolean;
+export declare function resolveQuestionAudience(question: QuestionRecord, readLaneSenior: (laneId: string) => SeniorRef, readSessionSenior: (sessionId: string) => SeniorRef): SeniorRef;
+/** Unlocked pass body. `runReconcilePass` takes the process lock around this. */
+export declare function runPassBody(hqRoot: string, deps: ReconcileDeps): Promise<ReconcileSummary>;
+/**
+ * One idempotent reconcile pass. Throws only when the store directory itself
+ * cannot be listed; per-record failures are logged onto the summary and set
+ * `ok: false` rather than being swallowed.
+ */
+export declare function runReconcilePass(hqRoot: string, deps: ReconcileDeps): Promise<ReconcileSummary>;
+export interface AnsweredLaneWakeResult {
+    status: "resumed" | "queued" | "awaiting_answers" | "deferred" | "failed";
+    lane_state: LaneRecord["state"];
+    queued_reason?: QueuedReason;
+    remaining_question_ids?: string[];
+    error?: string;
+}
+/** Run the reconcile answer-resume path for one lane before questions answer returns. */
+export declare function wakeAnsweredLaneNow(hqRoot: string, laneId: string, deps: ReconcileDeps): Promise<AnsweredLaneWakeResult>;
+//# sourceMappingURL=reconcile.d.ts.map