@indigoai-us/hq-cli 5.215.0 → 5.217.0

package/dist/lib/lanes/watcher.js

@@ -35,7 +35,7 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 import { LanesError } from "./errors.js";
 import { raiseEnvelopeQuestions } from "./questions.js";
-import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, } from "./types.js";
+import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, } from "./types.js";
 import { laneStories, storyStatusForEnvelope } from "./stories.js";
 import { readLane, retryUpdateLane } from "./store.js";
 import { readWorkerRunResult } from "./delivery.js";
@@ -44,7 +44,7 @@ import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, isProviderLoginFailure, provide
 import { readToolActivity, toolActivityPath } from "./dropbox.js";
 import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
 import { resolveAdmissionHost as resolveProcessAdmissionHost } from "./admission-host.js";
-import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, progressEventDue, } from "./mesh-events.js";
+import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, laneStoryStatusSuppressionKey, progressEventDue, } from "./mesh-events.js";
 import { envelopeQuestionId } from "./questions.js";
 import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, retryLaneOnAlternateAccount, } from "./spawn.js";
 /** Spec §5.7: no signal for 20 minutes while the process is still alive. */
@@ -313,7 +313,7 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
         classification.state === "awaiting_input" &&
         opts.envelope &&
         (opts.envelope.questions?.length ?? 0) > 0) {
-        raiseEnvelopeQuestions(hqRoot, existing, opts.envelope, opts.envelopeRef ?? "envelopes/unknown.json", at);
+        raiseEnvelopeQuestions(hqRoot, existing, opts.envelope, opts.envelopeRef ?? "envelopes/unknown.json", at, opts.questionIndexes);
     }
     let changed = false;
     const updated = retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
@@ -336,6 +336,8 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
             stalled: undefined,
             stalled_at: undefined,
             stalled_reason: undefined,
+            interrupted_at: classification.state === "interrupted" ? at : undefined,
+            envelope_validation_errors: opts.envelopeValidationErrors,
         };
         changed = true;
         if (classification.reason) {
@@ -432,6 +434,81 @@ function fileNonEmpty(filePath) {
         throw err;
     }
 }
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+/** Keep only questions that the normal envelope parser accepts. */
+function recoverAskEnvelope(text, validationErrors) {
+    const raw = JSON.parse(text);
+    if (!isPlainObject(raw) || raw.decision !== "ask" || !Array.isArray(raw.questions)) {
+        return null;
+    }
+    const validQuestions = [];
+    const questionIndexes = [];
+    const questionErrors = [];
+    for (let index = 0; index < raw.questions.length; index += 1) {
+        const question = raw.questions[index];
+        try {
+            const parsed = parseLaneEnvelope({
+                schema_version: "hq-lane-envelope.v1",
+                decision: "ask",
+                summary: "Recovered operator question",
+                story_status: "in_progress",
+                artifacts: [],
+                questions: [question],
+            });
+            const item = parsed.questions?.[0];
+            if (item === undefined) {
+                throw new Error("question parser returned no question");
+            }
+            validQuestions.push(item);
+            questionIndexes.push(index);
+        }
+        catch (err) {
+            questionErrors.push(`questions[${index}] is invalid: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    if (validQuestions.length === 0) {
+        return {
+            envelope: {
+                schema_version: "hq-lane-envelope.v1",
+                decision: "ask",
+                summary: "The worker needs operator input after envelope validation failed.",
+                story_status: STORY_STATUSES.includes(raw.story_status)
+                    ? raw.story_status
+                    : "in_progress",
+                artifacts: [],
+                questions: [],
+            },
+            questionIndexes,
+            questionErrors,
+            sourceQuestionCount: raw.questions.length,
+        };
+    }
+    const contextErrors = [...new Set([...validationErrors, ...questionErrors])];
+    const storyStatus = STORY_STATUSES.includes(raw.story_status) && raw.story_status !== "done"
+        ? raw.story_status
+        : "in_progress";
+    const questions = validQuestions.map((question) => ({
+        ...question,
+        context: `${question.context}\n\nEnvelope validation errors:\n${contextErrors.map((error) => `- ${error}`).join("\n")}`,
+    }));
+    return {
+        envelope: {
+            schema_version: "hq-lane-envelope.v1",
+            decision: "ask",
+            summary: typeof raw.summary === "string" && raw.summary.length > 0
+                ? raw.summary
+                : "The worker needs operator input after envelope validation failed.",
+            story_status: storyStatus,
+            artifacts: [],
+            questions,
+        },
+        questionIndexes,
+        questionErrors,
+        sourceQuestionCount: raw.questions.length,
+    };
+}
 export function readLatestEnvelope(hqRoot, laneId, minName) {
     return readLatestEnvelopeRecord(hqRoot, laneId, minName)?.envelope ?? null;
 }
@@ -570,8 +647,8 @@ function readLaneSoft(hqRoot, laneId) {
     }
 }
 /** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
-export function envelopeQuestionIds(laneId, envelopeRef, envelope) {
-    return (envelope.questions ?? []).map((_q, i) => envelopeQuestionId(laneId, envelopeRef, i));
+export function envelopeQuestionIds(laneId, envelopeRef, envelope, questionIndexes) {
+    return (envelope.questions ?? []).map((_q, i) => envelopeQuestionId(laneId, envelopeRef, questionIndexes?.[i] ?? i));
 }
 function defaultSleep(ms) {
     return new Promise((resolve) => {
@@ -604,6 +681,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
     const progressFiles = [...logFiles, ...activityFiles];
     // Mesh seam (spec §8). Non-interactive: nobody is at this process's tty.
     const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
+    const storyStatusSuppressions = new Set();
     const meshRequestTimeoutMs = deps.meshRequestTimeoutMs ?? 10_000;
     const initialLane = readLaneSoft(hqRoot, laneId);
     if (mesh && initialLane) {
@@ -631,6 +709,10 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
             }
             catch (err) {
                 const message = err instanceof Error ? err.message : String(err);
+                if (company && initialLane.story_id &&
+                    err instanceof LanesError && err.error === "story_not_on_board") {
+                    storyStatusSuppressions.add(laneStoryStatusSuppressionKey(company, initialLane.project_id, initialLane.story_id));
+                }
                 process.stderr.write(`hq lanes watcher: mesh thread ensure failed for ${laneId}: ${message}\n`);
                 writeWatcherDecision(outPath, {
                     event: "mesh_event_failed",
@@ -701,10 +783,10 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
         const missingExpected = !processAlive && latestFile === null
             ? missingExpectedEnvelopeFile(hqRoot, laneId, manifest.envelope_path)
             : null;
-        const latest = latestFile && latestFile.envelope
+        let latest = latestFile && latestFile.envelope
             ? { envelope: latestFile.envelope, ref: latestFile.ref }
             : null;
-        const envelope = latest?.envelope ?? null;
+        let envelope = latest?.envelope ?? null;
         if (latest && latest.ref !== lastEnvelopeRef) {
             logDecision({
                 event: "envelope_observed",
@@ -751,6 +833,9 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
         // fails the same way, and would end the lane envelope_invalid without
         // ever recording the hold. The quota result applies immediately instead.
         let invalidEnvelope = null;
+        let envelopeValidationErrors;
+        let questionIndexes;
+        let recoveredAsk = false;
         const repairCandidate = latestFile ?? missingExpected;
         if (!processAlive &&
             !envelope &&
@@ -771,11 +856,70 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
                     await sleep(pollMs);
                     continue;
                 }
-                if (!repairCandidate.missing)
-                    invalidEnvelope = { errors: repair.errors };
+                if (!repairCandidate.missing) {
+                    const validationErrors = repair.errors.filter((error) => !error.startsWith("repair_spawn_failed:"));
+                    let recovery = null;
+                    try {
+                        recovery = recoverAskEnvelope(repairCandidate.text, validationErrors);
+                    }
+                    catch (err) {
+                        const message = err instanceof Error ? err.message : String(err);
+                        process.stderr.write(`hq lanes watcher: invalid ask fallback could not parse ${repairCandidate.path} for ${laneId}: ${message}\n`);
+                        logDecision({
+                            event: "envelope_repair",
+                            at: new Date(now()).toISOString(),
+                            outcome: "ask_fallback_skipped",
+                            reason: "invalid_json",
+                            error: message,
+                            validation_errors: validationErrors,
+                        });
+                    }
+                    if (recovery !== null && (recovery.envelope.questions?.length ?? 0) > 0) {
+                        const errors = [...new Set([...validationErrors, ...recovery.questionErrors])];
+                        envelopeValidationErrors = errors;
+                        questionIndexes = recovery.questionIndexes;
+                        latest = { envelope: recovery.envelope, ref: repairCandidate.ref };
+                        envelope = recovery.envelope;
+                        recoveredAsk = true;
+                        const message = `hq lanes watcher: routing invalid ask envelope for ${laneId} to awaiting_input ` +
+                            `with ${recovery.envelope.questions?.length ?? 0} validated question(s); errors: ${errors.join("; ")}`;
+                        process.stderr.write(`${message}\n`);
+                        logDecision({
+                            event: "envelope_repair",
+                            at: new Date(now()).toISOString(),
+                            outcome: "ask_fallback_routed",
+                            state: "awaiting_input",
+                            questions: recovery.envelope.questions?.length ?? 0,
+                            validation_errors: errors,
+                        });
+                    }
+                    else {
+                        const errors = recovery === null
+                            ? repair.errors
+                            : [...new Set([...repair.errors, ...recovery.questionErrors])];
+                        invalidEnvelope = { errors };
+                        if (recovery !== null) {
+                            const message = `hq lanes watcher: invalid ask envelope for ${laneId} remains failed; ` +
+                                `0 of ${recovery.sourceQuestionCount} questions passed validation`;
+                            process.stderr.write(`${message}\n`);
+                            logDecision({
+                                event: "envelope_repair",
+                                at: new Date(now()).toISOString(),
+                                outcome: "ask_fallback_skipped",
+                                reason: "no_valid_questions",
+                                questions: 0,
+                                validation_errors: errors,
+                            });
+                        }
+                    }
+                }
             }
         }
-        if (processAlive === false && envelope && latestFile) {
+        if (processAlive === false &&
+            envelope &&
+            latestFile !== null &&
+            latestFile.envelope !== null &&
+            !recoveredAsk) {
             markRepairSucceeded(hqRoot, laneId, latestFile.ref);
         }
         const tick = tickWatcher({
@@ -913,6 +1057,8 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
                 const terminalWrite = applyTerminalToLaneWithResult(hqRoot, laneId, classification, {
                     envelope,
                     envelopeRef: latest?.ref,
+                    envelopeValidationErrors,
+                    questionIndexes,
                     at: tick.emit.at,
                     quota,
                     providerRejection,
@@ -930,9 +1076,13 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
                 if (mesh && terminalWrite.changed) {
                     if (envelope && latest) {
                         const questionIds = envelope.decision === "ask"
-                            ? envelopeQuestionIds(laneId, latest.ref, envelope)
+                            ? envelopeQuestionIds(laneId, latest.ref, envelope, questionIndexes)
                             : undefined;
-                        await emitMeshSoft(`lane_${envelope.decision}`, logDecision, tick.emit.at, (signal) => emitLaneDecisionEvents(mesh, updated, envelope, { questionIds, signal }), meshRequestTimeoutMs);
+                        await emitMeshSoft(`lane_${envelope.decision}`, logDecision, tick.emit.at, (signal) => emitLaneDecisionEvents(mesh, updated, envelope, {
+                            questionIds,
+                            signal,
+                            storyStatusSuppressions,
+                        }), meshRequestTimeoutMs);
                     }
                     else if (updated.state === "failed" || updated.state === "interrupted" ||
                         updated.state === "cancelled" || updated.state === "killed") {
@@ -973,9 +1123,11 @@ export function decideEnvelopeRepair(hqRoot, laneId, latest, nowMs, spawnRepair,
     const runDir = laneRunDir(hqRoot, laneId);
     const manifest = readSpawnManifest(runDir);
     const prior = lane.envelope_repair;
-    const repairRef = missingExpectedEnvelopeFile(hqRoot, laneId, manifest.envelope_path)?.ref ??
-        prior?.envelope_ref ??
-        latest.ref;
+    // A resumed worker can write its next envelope before spawn.json points to
+    // that slot. Use the newest of the manifest's expected slot and the actual
+    // latest file so an older manifest or repair record cannot spend its budget.
+    const expectedRef = missingExpectedEnvelopeFile(hqRoot, laneId, manifest.envelope_path)?.ref;
+    const repairRef = expectedRef !== undefined && expectedRef > latest.ref ? expectedRef : latest.ref;
     const priorAppliesToSlot = prior?.envelope_ref === undefined || prior.envelope_ref === repairRef;
     if (prior && prior.attempted && priorAppliesToSlot) {
         // The repair turn ran and its output does not parse either.
@@ -1025,6 +1177,14 @@ export function decideEnvelopeRepair(hqRoot, laneId, latest, nowMs, spawnRepair,
             error,
             rawText: latest.text,
             missing: latest.missing,
+            onProfileWarning: (fields) => {
+                logDecision({
+                    event: "envelope_repair",
+                    at,
+                    outcome: "profile_unknown_meta_fields_ignored",
+                    fields: [...fields],
+                });
+            },
         });
         logDecision({ event: "envelope_repair", at, outcome: "spawned", mode, pid, session_id: sessionId, error });
         return { action: "spawned" };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.215.0",
+  "version": "5.217.0",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {