@indigoai-us/hq-cli 5.126.0 → 5.128.0

package/dist/lib/lanes/watcher.d.ts

@@ -0,0 +1,216 @@
+/**
+ * Per-lane watcher (spec §5.7).
+ *
+ * Armed at spawn, detached from the caller. Emits on progress, terminal, and
+ * silence (20 minutes with the process still alive). Terminal classification
+ * follows spec §5: an envelope decides done/ask/blocked; a process that exits
+ * with partial output and no envelope is `interrupted`; silence is never
+ * success.
+ *
+ * Progress has two sources. Output files (worker.out, worker.err,
+ * last-message.txt) cover codex and grok, which stream as they work. They do
+ * not cover Claude: `claude -p --output-format json` writes nothing until it
+ * exits, so a healthy Claude lane used to hit the silence deadline at exactly
+ * 20 minutes every time, and a hung one looked identical. The second source
+ * is logs/tool-activity.json, which the drain hook rewrites on every
+ * PreToolUse call (dropbox.ts). A worker that is working calls tools; a
+ * worker that has not called a tool or written output for the whole window
+ * is `stalled`, and the watcher says so on the lane record.
+ *
+ * What this cannot tell apart: a worker hung inside one tool call from a
+ * worker running one legitimately long tool call (a 25-minute build); a
+ * worker looping on tool calls without making progress from a working one;
+ * a model stuck mid-generation with no tool call from one that is writing a
+ * long answer. It also depends on the drain hook being registered in the
+ * provider home (hooks-install.ts): with no hook there is no touch, and the
+ * lane reads as stalled every window, which is the pre-existing behaviour.
+ * Distinguishing a hung tool from a slow one needs the provider to report
+ * per-call state, which none of the three CLIs expose today. `stalled` is
+ * therefore a flag for the senior to look at, never a kill.
+ *
+ * Queue admission of the next lane when a slot frees is owned by the
+ * periodic reconcile pass and is out of scope here.
+ */
+import { type LaneEnvelope, type LaneRecord, type LaneState } from "./types.js";
+import { type QuotaSignal } from "./quota.js";
+import { type LaneMesh } from "./mesh-events.js";
+import { type EnvelopeRepairSpawnInput } from "./spawn.js";
+/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
+export declare const SILENCE_MS: number;
+export declare const WATCHER_POLL_MS = 5000;
+/** Synced last_host_report_at cadence. Reconcile dead-host TTL is 20 minutes. */
+export declare const WATCHER_HEARTBEAT_MS: number;
+/**
+ * Cap on logs/watcher.out. A decision line is typically under 300 bytes, so
+ * 256 KiB holds hundreds of transitions. A stuck lane cannot fill the disk:
+ * once the file would exceed this, the oldest complete lines are dropped.
+ */
+export declare const WATCHER_OUT_MAX_BYTES: number;
+export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "classification", "terminal_write", "mesh_event", "mesh_event_failed"];
+/** Terminal reason when the envelope failed to parse after the repair turn (spec §4). */
+export declare const ENVELOPE_INVALID_REASON: "envelope_invalid";
+export type WatcherDecisionEvent = (typeof WATCHER_DECISION_EVENTS)[number];
+export declare const WATCHER_SIGNAL_KINDS: readonly ["progress", "terminal", "silence"];
+export type WatcherSignalKind = (typeof WATCHER_SIGNAL_KINDS)[number];
+export interface TerminalClassification {
+    state: Extract<LaneState, "done" | "awaiting_input" | "blocked" | "interrupted" | "failed">;
+    reason?: string;
+}
+export interface WatcherSignal {
+    kind: WatcherSignalKind;
+    at: string;
+    classification?: TerminalClassification;
+}
+export interface WatcherTickInput {
+    nowMs: number;
+    processAlive: boolean;
+    progressed: boolean;
+    lastSignalMs: number;
+    envelope: LaneEnvelope | null;
+    hasPartialOutput: boolean;
+    silenceMs?: number;
+    quota?: QuotaSignal | null;
+    invalidEnvelope?: {
+        errors: string[];
+    } | null;
+}
+export interface WatcherTickResult {
+    emit?: WatcherSignal;
+    nextLastSignalMs: number;
+    done: boolean;
+}
+export declare function parseWatcherSignalKind(value: unknown): WatcherSignalKind;
+export declare function classifyLaneExit(input: {
+    envelope: LaneEnvelope | null;
+    hasPartialOutput: boolean;
+    /**
+     * Set when the worker's own result is a 429 or a usage-limit error.
+     * Distinct from a crash: the reason is account_quota_exhausted or
+     * account_model_unavailable, not exited_with_partial_output.
+     */
+    quota?: QuotaSignal | null;
+    /**
+     * Set when the run's envelope file exists but does not parse and no
+     * repair turn remains. Failed, regardless of partial output: the worker
+     * did finish, it just did not honour the contract.
+     */
+    invalidEnvelope?: {
+        errors: string[];
+    } | null;
+}): TerminalClassification;
+export declare function tickWatcher(input: WatcherTickInput): WatcherTickResult;
+export declare function watcherEventsPath(hqRoot: string, laneId: string): string;
+export declare function watcherOutLogPath(hqRoot: string, laneId: string): string;
+export declare function appendCappedLog(filePath: string, line: string, maxBytes?: number): void;
+export declare function writeWatcherDecision(filePath: string, record: Record<string, unknown>, maxBytes?: number): void;
+export declare function watcherLastSignalPath(hqRoot: string, laneId: string): string;
+export declare function appendWatcherSignal(hqRoot: string, laneId: string, signal: WatcherSignal): void;
+/**
+ * Slot-free notification. The periodic reconcile pass admits the next queued
+ * lane; this watcher must not do it. Intentionally empty.
+ */
+export declare function notifySlotFreed(_input: {
+    hqRoot: string;
+    provider: LaneRecord["provider"];
+    account?: string;
+}): void;
+export interface ApplyTerminalOpts {
+    envelope?: LaneEnvelope | null;
+    envelopeRef?: string;
+    at?: string;
+    quota?: QuotaSignal | null;
+}
+export declare function stalledReason(silenceMs: number): string;
+/**
+ * The consumer of the silence signal. A silence emit on a running lane sets
+ * `stalled` on the record so `hq lanes list` and `show` surface it; the next
+ * progress emit clears it. Nothing here changes `state`.
+ */
+export declare function markLaneStalled(hqRoot: string, laneId: string, at: string, reason: string): LaneRecord;
+export declare function clearLaneStalled(hqRoot: string, laneId: string): LaneRecord;
+export declare function applyTerminalToLane(hqRoot: string, laneId: string, classification: TerminalClassification, opts?: ApplyTerminalOpts): LaneRecord;
+/** Terminal write outcome for emitters: only the CAS winner may publish mesh effects. */
+export declare function applyTerminalToLaneWithResult(hqRoot: string, laneId: string, classification: TerminalClassification, opts?: ApplyTerminalOpts): {
+    record: LaneRecord;
+    changed: boolean;
+};
+export declare function isPidAlive(pid: number): boolean;
+export interface LatestEnvelope {
+    envelope: LaneEnvelope;
+    ref: string;
+}
+/** The newest envelope file of this run, parsed or not. */
+export interface LatestEnvelopeFile {
+    ref: string;
+    /** Absolute path of the file. */
+    path: string;
+    text: string;
+    envelope: LaneEnvelope | null;
+    /** Parser message when `envelope` is null. */
+    error: string | null;
+}
+export declare function readLatestEnvelope(hqRoot: string, laneId: string, minName?: string): LaneEnvelope | null;
+/** Valid-only view over `readLatestEnvelopeFile`; an unparseable file reads as absent. */
+export declare function readLatestEnvelopeRecord(hqRoot: string, laneId: string, minName?: string): LatestEnvelope | null;
+/**
+ * `minName` is this run's envelope slot (basename from spawn.json). Envelopes
+ * that sort before it belong to an earlier run of the same lane and must not
+ * classify this exit: a resumed run that dies before writing its own envelope
+ * would otherwise be marked done again from the previous one.
+ */
+export declare function readLatestEnvelopeFile(hqRoot: string, laneId: string, minName?: string): LatestEnvelopeFile | null;
+export interface WatcherLoopDeps {
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    isAlive?: (pid: number) => boolean;
+    readStarttime?: (pid: number) => number | null;
+    pollMs?: number;
+    silenceMs?: number;
+    watcherOutPath?: string;
+    watcherOutMaxBytes?: number;
+    /**
+     * Mesh seam (spec §8). Default: `createLaneMesh({ interactive: false })`.
+     * `null` disables emission entirely (tests that are not about mesh).
+     */
+    mesh?: LaneMesh | null;
+    /** Override for `providers.<provider>.progress_event_min_interval_ms`. */
+    progressEventMinIntervalMs?: number;
+    /** Bound a hung mesh request so watcher liveness polling continues. */
+    meshRequestTimeoutMs?: number;
+    /** Spec §4 repair spawn. Default: `spawnEnvelopeRepair`. */
+    spawnRepair?: (hqRoot: string, lane: LaneRecord, input: EnvelopeRepairSpawnInput) => number;
+}
+/**
+ * Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
+ * the event name and the error message, and never propagates: the lane's
+ * state is decided by the envelope and the process, not by the network.
+ */
+export declare function emitMeshSoft(event: string, logDecision: (record: Record<string, unknown>) => void, at: string, fn: (signal: AbortSignal) => Promise<void>, timeoutMs?: number): Promise<boolean>;
+/** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
+export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope): string[];
+export declare function runWatcherLoop(hqRoot: string, laneId: string, deps?: WatcherLoopDeps): Promise<void>;
+export type EnvelopeRepairDecision = {
+    action: "spawned";
+} | {
+    action: "give_up";
+    errors: string[];
+};
+/**
+ * Spec §4 repair decision for an exited worker whose envelope file does not
+ * parse. First failure on a resumable provider with a recorded session id:
+ * record `envelope_repair` on the lane and spawn the repair turn. A provider
+ * with no session resume gets the same single attempt as a fresh turn. Anything
+ * else (already attempted, no session id on a resumable provider, spawn
+ * failure): give up with every error collected, and the caller classifies
+ * the lane `failed` with reason `envelope_invalid`.
+ */
+export declare function decideEnvelopeRepair(hqRoot: string, laneId: string, latest: LatestEnvelopeFile, nowMs: number, spawnRepair: NonNullable<WatcherLoopDeps["spawnRepair"]>, logDecision: (record: Record<string, unknown>) => void): EnvelopeRepairDecision;
+/** Exported so tests can prove the watcher reads the live record, not spawn env. */
+export declare function laneIdentityFromRecord(hqRoot: string, laneId: string): {
+    lane_id: string;
+    company: LaneRecord["company"];
+    project_id: string;
+    story_id: string;
+    senior: LaneRecord["senior"];
+};
+//# sourceMappingURL=watcher.d.ts.map