@indigoai-us/hq-cli 5.131.0 → 5.133.0

package/dist/lib/lanes/reconcile.js

@@ -8,25 +8,30 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { parseUserYaml } from "../../utils/user-yaml-error.js";
-import { accountMaxRunning, loadLanesConfig } from "./config.js";
+import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
+import { MAX_LANE_DEPTH } from "./constants.js";
 import { LanesError, isLaneVersionConflict } from "./errors.js";
 import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
 import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
 import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
-import { readQuotaSignal, recordAccountQuota } from "./quota.js";
+import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
 import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
 import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
 import { createLaneMesh, emitLaneDecisionEvents } from "./mesh-events.js";
-import { decideEnvelopeRepair, envelopeQuestionIds } from "./watcher.js";
+import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
+import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
 /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
 export const QUESTION_AGE_MS = 60 * 60 * 1000;
-/** Spec §6.6 + §9: v1 permits exactly one aged advance. */
-export const MAX_AGED_ADVANCES = 1;
+/** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
+export const MAX_AGED_ADVANCES = MAX_LANE_DEPTH - 1;
 /** Aligns with watcher silence: a host that has not reported for this long is dead. */
 export const DEAD_HOST_MS = SILENCE_MS;
 export const RECONCILE_LOCK_STALE_MS = 30_000;
 export const RECONCILE_LOCK_TIMEOUT_MS = 5_000;
+/** A pickup probe that has not cleared its marker within this window is abandoned. */
+export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
+export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
 export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
 export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
 function logSkip(skip) {
@@ -308,6 +313,89 @@ function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
     }
     throw new LanesError("version_conflict", `Exhausted CAS retries for question ${questionId}`, { question_id: questionId });
 }
+async function emitAgedQuestionAtUser(summary, deps, lane, question) {
+    if (deps.dryRun)
+        return;
+    const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
+    if (!mesh || !lane.company || !lane.thread_id)
+        return;
+    const timeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
+    let timer;
+    try {
+        await Promise.race([
+            mesh.appendThreadEvent(lane.company, lane.thread_id, "lane_question_at_user", { lane_id: lane.lane_id, question_id: question.question_id }),
+            new Promise((_resolve, reject) => {
+                timer = setTimeout(() => reject(new Error(`question-at-user mesh emit timed out after ${timeoutMs}ms`)), timeoutMs);
+            }),
+        ]);
+    }
+    catch (err) {
+        markDirty(summary, {
+            kind: "question",
+            id: question.question_id,
+            why: `question-at-user mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
+        });
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function pickupValidationPendingAtMs(lane) {
+    const raw = lane.pickup_validation_pending_since ?? lane.updated_at;
+    const at = Date.parse(raw);
+    return Number.isFinite(at) ? at : null;
+}
+function pickupValidationExpired(lane, nowMs) {
+    const pendingAtMs = pickupValidationPendingAtMs(lane);
+    return pendingAtMs === null || nowMs - pendingAtMs > PICKUP_VALIDATION_MAX_AGE_MS;
+}
+function expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now) {
+    const nowMs = now.getTime();
+    for (const lane of lanes) {
+        if (lane.state !== "queued" ||
+            lane.pickup_validation_pending !== true ||
+            !laneRunsHere(lane, deps.host) ||
+            !pickupValidationExpired(lane, nowMs)) {
+            continue;
+        }
+        try {
+            const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
+                current.pickup_validation_pending === true &&
+                pickupValidationExpired(current, nowMs), (current) => {
+                const pendingAtMs = pickupValidationPendingAtMs(current);
+                return {
+                    ...current,
+                    state: "failed",
+                    queued_reason: undefined,
+                    pickup_validation_pending: undefined,
+                    pickup_validation_pending_since: undefined,
+                    terminal_reason: PICKUP_VALIDATION_TIMEOUT_REASON,
+                    terminal_details: {
+                        pickup_validation_pending_since: current.pickup_validation_pending_since ?? current.updated_at,
+                        age_ms: pendingAtMs === null ? null : Math.max(0, nowMs - pendingAtMs),
+                        max_age_ms: PICKUP_VALIDATION_MAX_AGE_MS,
+                    },
+                };
+            }, deps.dryRun);
+            if (result.changed) {
+                summary.aged.push({
+                    id: lane.lane_id,
+                    from: "queued",
+                    to: "failed",
+                    reason: PICKUP_VALIDATION_TIMEOUT_REASON,
+                });
+            }
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `pickup validation expiry failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+}
 function fileNonEmpty(filePath) {
     try {
         return fs.statSync(filePath).size > 0;
@@ -431,13 +519,29 @@ function classifyDeadLocalLane(hqRoot, laneId) {
         path.join(runDir, "logs", "last-message.txt"),
     ].some((file) => fileNonEmpty(file));
     const lane = readLane(hqRoot, laneId);
-    const quota = envelope
-        ? null
-        : readQuotaSignal(runDir, lane.model, new Date(), readSpawnManifest(runDir).worker_out_offset ?? 0);
+    // The same routing as the watcher's exit path (watcher.ts): a provider
+    // failure on the stream is read first, a 429 or quota-worded one becomes
+    // the quota signal, any other becomes provider_rejected, and only a stream
+    // with no failure object falls back to the quota reader.
+    const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
+    const now = new Date();
+    const routed = envelope
+        ? { quota: null, rejection: null }
+        : routeProviderFailure(readProviderRejection(runDir, lane.provider, workerOutOffset), () => readQuotaSignal(runDir, lane.model, now, workerOutOffset), lane.model, now);
+    // As in the watcher: a reset-less 429 gets the provider's default hold.
+    const quota = routed.quota
+        ? applyDefaultQuotaHold(routed.quota, providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[lane.provider]), now)
+        : null;
     return {
-        classification: classifyLaneExit({ envelope, hasPartialOutput, quota }),
+        classification: classifyLaneExit({
+            envelope,
+            hasPartialOutput,
+            quota,
+            providerRejection: routed.rejection,
+        }),
         latest,
         quota,
+        providerRejection: routed.rejection,
         lane,
     };
 }
@@ -572,6 +676,10 @@ export async function runPassBody(hqRoot, deps) {
                         to: String(nextHop),
                         reason: "unanswered_timeout",
                     });
+                    if (nextAudience.kind === "user") {
+                        const originLane = laneById.get(question.lane_id) ?? readLane(hqRoot, question.lane_id);
+                        await emitAgedQuestionAtUser(summary, deps, originLane, result.record);
+                    }
                 }
             }
         }
@@ -722,12 +830,17 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                 continue;
             }
             try {
-                const { classification, latest, quota, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
+                const { classification, latest, quota, providerRejection, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
                 // Match the watcher's one-repair decision before classifying an
-                // unparseable envelope after both processes have died.
+                // unparseable envelope after both processes have died. As in the
+                // watcher, a provider rejection or a quota signal skips the repair.
                 const rawLatest = readLatestEnvelopeFile(hqRoot, lane.lane_id);
                 let invalidEnvelope = null;
-                if (!deps.dryRun && rawLatest && rawLatest.envelope === null) {
+                if (!deps.dryRun &&
+                    !providerRejection &&
+                    !quota &&
+                    rawLatest &&
+                    rawLatest.envelope === null) {
                     const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, rawLatest, nowMs, deps.spawnRepair ?? spawnEnvelopeRepair, () => undefined);
                     if (repair.action === "spawned") {
                         armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps);
@@ -757,6 +870,9 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                     state: repairedClassification.state,
                     queued_reason: undefined,
                     terminal_reason: repairedClassification.reason ?? "host_lost",
+                    ...(providerRejection
+                        ? { terminal_details: providerRejectionDetails(providerRejection, current) }
+                        : {}),
                     ...(quota?.quota_reset_at ? { quota_reset_at: quota.quota_reset_at } : {}),
                     ...(quota?.marker ? { model_quota_marker: quota.marker } : {}),
                     last_host_report_at: nowIso,
@@ -837,8 +953,14 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
     }
 }
 async function admitQueued(hqRoot, lanes, summary, deps, now) {
+    expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now);
+    // A lane whose pickup probe is still out is not admitted: its record is
+    // about to be bound or failed by the pickup, and a worker spawned now
+    // would keep running behind that write.
     const localQueued = lanes
-        .filter((lane) => lane.state === "queued" && laneRunsHere(lane, deps.host))
+        .filter((lane) => lane.state === "queued" &&
+        lane.pickup_validation_pending !== true &&
+        laneRunsHere(lane, deps.host))
         .sort(compareQueueOrder);
     if (localQueued.length === 0)
         return;

package/dist/lib/lanes/spawn.js

@@ -979,13 +979,19 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
         }
         return existing;
     }
+    // A queued lane that a move parked at the target's cap carries the turn
+    // it was resumed with: the admit spawn continues that session instead of
+    // starting fresh from the brief.
+    const pendingResume = launch.trigger === undefined && existing.state === "queued" ? existing.pending_resume : undefined;
     const trigger = launch.trigger ?? inferTrigger(existing);
     // A message into a finished lane must not turn that lane `failed` when the
     // account cannot be bound: nothing has started yet, so the record keeps its
     // state and the caller sees the error. Admission and question resume keep
     // today's behaviour of naming the terminal reason on the record.
     const markPreSpawnFailure = trigger.kind !== "message" && trigger.kind !== "resume";
-    const plan = planLaneLaunch(existing, trigger);
+    const plan = pendingResume !== undefined && providerSupportsSessionResume(existing.provider)
+        ? { mode: "resume", session_id: pendingResume.session_id }
+        : planLaneLaunch(existing, trigger);
     if (existing.host.kind === "outpost") {
         refuseOutpostSelfTarget(hqRoot, existing, deps);
     }
@@ -1065,8 +1071,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
                     ? trigger.text
                     : trigger.kind === "resume"
                         ? trigger.text
-                        : undefined,
-                resumedFrom: trigger.kind === "resume" ? trigger.from : undefined,
+                        : pendingResume?.text,
+                resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
                 answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
                     ? answersPath
                     : undefined,
@@ -1119,7 +1125,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
     // Resume-time compaction (compaction.ts). Runs before the claim so the
     // lane is never `running` without a worker while `/compact` works.
     let compaction;
-    if (resume && launch.compact !== false) {
+    const compactWanted = pendingResume?.compact ?? launch.compact;
+    if (resume && compactWanted !== false) {
         const nowFn = deps.now ?? (() => new Date());
         const decision = compactionDue(existing, nowFn().getTime(), providerCompactAfterMs(config.providers[lane.provider]));
         if (decision.due) {
@@ -1169,6 +1176,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...current,
             state: "running",
             queued_reason: undefined,
+            // An account move's reservation ends with the claim.
+            pending_resume: undefined,
             account: resolved.id,
             ...(model !== undefined ? { model } : {}),
             effort_source: effort.source,
@@ -1187,10 +1196,22 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
         }
         if (compaction)
             next.last_compaction = { ...compaction, resume_at: nowIso };
-        if (resume && trigger.kind !== "admit") {
+        if (resume && pendingResume !== undefined) {
             next.last_resume = {
                 session_id: resume.session_id,
                 at: nowIso,
+                account: resolved.id,
+                trigger: pendingResume.trigger,
+                ...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
+                ...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
+                ...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
+            };
+        }
+        else if (resume && trigger.kind !== "admit") {
+            next.last_resume = {
+                session_id: resume.session_id,
+                at: nowIso,
+                account: resolved.id,
                 trigger: trigger.kind,
                 ...(trigger.kind === "message" ? { message_id: trigger.message_id } : {}),
                 ...(trigger.kind === "resume"

package/dist/lib/lanes/store.js

@@ -325,6 +325,11 @@ export function parseLaneRecord(raw, filePath) {
             path: filePath,
         });
     }
+    if (raw.account_history !== undefined && !Array.isArray(raw.account_history)) {
+        throw new LaneSchemaError("account_history must be an array when present", {
+            path: filePath,
+        });
+    }
     if (!Array.isArray(raw.attribution_history)) {
         throw new LaneSchemaError("attribution_history must be an array", {
             path: filePath,
@@ -466,6 +471,11 @@ export function mergeLaneRecords(live, sidecars) {
         ...base,
         senior_history: unionHistory(all.map((record) => record.senior_history), entryIdentity, base.senior_history),
         attribution_history: unionHistory(all.map((record) => record.attribution_history), attributionIdentity, base.attribution_history),
+        ...(all.some((record) => record.account_history !== undefined)
+            ? {
+                account_history: unionHistory(all.map((record) => record.account_history ?? []), entryIdentity, base.account_history ?? []),
+            }
+            : {}),
     };
 }
 export function mergeQuestionRecords(live, sidecars) {

package/dist/lib/lanes/types.d.ts

@@ -250,6 +250,15 @@ export interface LaneRecord {
     senior_history: SeniorHistoryEntry[];
     state: LaneState;
     queued_reason?: QueuedReason;
+    /**
+     * Set by `hq lanes pickup` when it stores an outpost lane and cleared by
+     * the account bind once the model probe has finished. While set, the
+     * reconcile pass does not admit the lane: a spawn during the probe would
+     * leave a running worker behind a record the probe result then rewrites.
+     */
+    pickup_validation_pending?: true;
+    /** When the pickup validation marker was written. */
+    pickup_validation_pending_since?: string;
     priority: number;
     worker?: string;
     /** Registry-relative directory of the resolved worker, when --worker was set. */
@@ -258,6 +267,20 @@ export interface LaneRecord {
     worker_pack?: string;
     provider: LaneProvider;
     account?: string;
+    /**
+     * Every operator move of the lane's session to another account of the
+     * same provider (`hq lanes resume|message --account`), newest last, capped
+     * like the other histories.
+     */
+    account_history?: AccountHistoryEntry[];
+    /**
+     * Set by an account move (`--account`) until the spawn claims the lane:
+     * the turn the move was asked to run. While set it reserves a slot on the
+     * account for the capacity check of later moves. When the target was at
+     * its cap the lane is queued there and the reconcile pass resumes the
+     * recorded session with this turn instead of starting fresh from the brief.
+     */
+    pending_resume?: PendingResume;
     host: LaneHost;
     /**
      * Creating machine's identity when `host.kind` is `outpost` (spec §5.2).
@@ -308,8 +331,10 @@ export interface LaneRecord {
     terminal_reason?: string;
     /**
      * Structured detail behind a named `terminal_reason` (dispatch, pickup and
-     * spawn failures), or the missing-binary search path behind
-     * `provider_binary_missing`. Written by the command and spawn writers.
+     * spawn failures), the missing-binary search path behind
+     * `provider_binary_missing`, or the provider's own message, status, model
+     * and account behind `provider_rejected`. Written by the command, spawn
+     * and watcher writers.
      */
     terminal_details?: Record<string, unknown>;
     /** SSM command id of the `hq outposts exec` pickup that started this lane. */
@@ -368,9 +393,27 @@ export interface LaneEnvelopeRepair {
     envelope_ref?: string;
     at: string;
 }
+export interface AccountHistoryEntry {
+    from: string | null;
+    to: string;
+    at: string;
+    reason: "operator_move";
+}
+export interface PendingResume {
+    session_id: string;
+    trigger: "message" | "resume";
+    resumed_from?: "interrupted" | "done" | "blocked";
+    text?: string;
+    message_id?: string;
+    /** False when the command passed --no-compact; the deferred spawn honours it. */
+    compact?: boolean;
+    at: string;
+}
 export interface LaneResumeRecord {
     session_id: string;
     at: string;
+    /** Account the resumed run was bound to. */
+    account?: string;
     trigger: "answered_questions" | "message" | "resume";
     message_id?: string;
     /**
@@ -508,6 +551,12 @@ export interface ProviderConfig {
      * means the five-minute default.
      */
     progress_event_min_interval_ms?: number;
+    /**
+     * Hold applied to an account after a 429 whose message names no reset
+     * time: the account is skipped for this long. Absent means the
+     * fifteen-minute default.
+     */
+    quota_default_hold_ms?: number;
     accounts: Record<string, AccountConfig>;
 }
 export interface ThrottleConfig {

package/dist/lib/lanes/types.js

@@ -198,13 +198,15 @@ export const LANE_FIELD_WRITERS = {
     project_id: ["command"], story_id: ["command"], thread_id: ["creator", "command"],
     depth: ["command"], senior: ["command"], senior_history: ["command"],
     state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "reconcile"],
+    pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
+    pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
     priority: ["command"], worker: ["creator"], worker_path: ["creator"], worker_pack: ["creator"],
-    provider: ["creator"], account: ["creator", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
+    provider: ["creator"], account: ["creator", "spawn", "command"], account_history: ["command"], pending_resume: ["command", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
     version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
-    last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"],
+    last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"],
     last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
     overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
-    admitted_reason: ["reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn"],
+    admitted_reason: ["reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
     quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
     provider_session_id: ["watcher"], provider_session_id_absent: ["watcher"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], last_progress_event_at: ["watcher"],
 };

package/dist/lib/lanes/watcher.d.ts

@@ -33,6 +33,7 @@
  */
 import { type LaneEnvelope, type LaneRecord, type LaneState } from "./types.js";
 import { type QuotaSignal } from "./quota.js";
+import { type ProviderRejection } from "./provider-rejection.js";
 import { type LaneMesh } from "./mesh-events.js";
 import { type EnvelopeRepairSpawnInput } from "./spawn.js";
 /** Spec §5.7: no signal for 20 minutes while the process is still alive. */
@@ -73,6 +74,7 @@ export interface WatcherTickInput {
     invalidEnvelope?: {
         errors: string[];
     } | null;
+    providerRejection?: ProviderRejection | null;
 }
 export interface WatcherTickResult {
     emit?: WatcherSignal;
@@ -97,7 +99,30 @@ export declare function classifyLaneExit(input: {
     invalidEnvelope?: {
         errors: string[];
     } | null;
+    /**
+     * Set when the worker's output stream carries a provider-level turn
+     * failure (codex `turn.failed`, a Claude error result with an HTTP status,
+     * a grok error object). Failed, never interrupted: the provider refused
+     * the turn, so there is nothing to resume and no envelope to repair.
+     */
+    providerRejection?: ProviderRejection | null;
 }): TerminalClassification;
+/**
+ * One decision for an exited worker's output. An explicit provider status
+ * wins over the quota text heuristics: a 429 from any provider (including
+ * codex `turn.failed` and grok error objects, which the claude-shaped quota
+ * reader never sees) becomes a quota signal; any other explicit status is a
+ * rejection even when its text sounds like a quota message (a 402 "requires
+ * usage credits" is a rejection, not a hold). A failure with no status at
+ * all is matched against the quota text heuristics first: a status-less
+ * "usage limit" / "rate limit" turn.failed still creates the account hold;
+ * any other status-less message is a rejection. Only when the stream
+ * carries no failure object does the quota reader's own read apply.
+ */
+export declare function routeProviderFailure(failure: ProviderRejection | null, readQuota: () => QuotaSignal | null, model: string | undefined, now: Date): {
+    quota: QuotaSignal | null;
+    rejection: ProviderRejection | null;
+};
 export declare function tickWatcher(input: WatcherTickInput): WatcherTickResult;
 export declare function watcherEventsPath(hqRoot: string, laneId: string): string;
 export declare function watcherOutLogPath(hqRoot: string, laneId: string): string;
@@ -119,6 +144,8 @@ export interface ApplyTerminalOpts {
     envelopeRef?: string;
     at?: string;
     quota?: QuotaSignal | null;
+    /** Written to `terminal_details` with the model and account the run used. */
+    providerRejection?: ProviderRejection | null;
 }
 export declare function stalledReason(silenceMs: number): string;
 /**

package/dist/lib/lanes/watcher.js

@@ -38,9 +38,10 @@ import { raiseEnvelopeQuestions } from "./questions.js";
 import { isTerminalLaneState, parseLaneEnvelope, } from "./types.js";
 import { readLane, retryUpdateLane } from "./store.js";
 import { readWorkerRunResult } from "./delivery.js";
-import { readQuotaSignal, recordAccountQuota } from "./quota.js";
+import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
+import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
 import { readToolActivity, toolActivityPath } from "./dropbox.js";
-import { loadLanesConfig, providerProgressEventMinIntervalMs } from "./config.js";
+import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
 import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, progressEventDue, } from "./mesh-events.js";
 import { envelopeQuestionId } from "./questions.js";
 import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, } from "./spawn.js";
@@ -86,6 +87,9 @@ export function classifyLaneExit(input) {
             return { state: "blocked" };
         throw new LanesError("unknown_envelope_decision", `Unknown envelope decision ${JSON.stringify(input.envelope.decision)}. Accepted: done, ask, blocked`, { accepted: ["done", "ask", "blocked"], value: input.envelope.decision });
     }
+    if (input.providerRejection) {
+        return { state: "failed", reason: PROVIDER_REJECTED_REASON };
+    }
     if (input.invalidEnvelope) {
         return { state: "failed", reason: ENVELOPE_INVALID_REASON };
     }
@@ -100,6 +104,34 @@ export function classifyLaneExit(input) {
     }
     return { state: "failed", reason: "exited_without_envelope" };
 }
+/**
+ * One decision for an exited worker's output. An explicit provider status
+ * wins over the quota text heuristics: a 429 from any provider (including
+ * codex `turn.failed` and grok error objects, which the claude-shaped quota
+ * reader never sees) becomes a quota signal; any other explicit status is a
+ * rejection even when its text sounds like a quota message (a 402 "requires
+ * usage credits" is a rejection, not a hold). A failure with no status at
+ * all is matched against the quota text heuristics first: a status-less
+ * "usage limit" / "rate limit" turn.failed still creates the account hold;
+ * any other status-less message is a rejection. Only when the stream
+ * carries no failure object does the quota reader's own read apply.
+ */
+export function routeProviderFailure(failure, readQuota, model, now) {
+    if (failure && failure.status === QUOTA_STATUS) {
+        return {
+            quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message }, model, now),
+            rejection: null,
+        };
+    }
+    if (failure && failure.status === null) {
+        const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message }, model, now);
+        if (quota)
+            return { quota, rejection: null };
+    }
+    if (failure)
+        return { quota: null, rejection: failure };
+    return { quota: readQuota(), rejection: null };
+}
 export function tickWatcher(input) {
     const silenceMs = input.silenceMs ?? SILENCE_MS;
     const at = new Date(input.nowMs).toISOString();
@@ -109,6 +141,7 @@ export function tickWatcher(input) {
             hasPartialOutput: input.hasPartialOutput,
             quota: input.quota,
             invalidEnvelope: input.invalidEnvelope,
+            providerRejection: input.providerRejection,
         });
         return {
             emit: { kind: "terminal", at, classification },
@@ -301,6 +334,9 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
         if (opts.quota?.marker) {
             next.model_quota_marker = opts.quota.marker;
         }
+        if (opts.providerRejection) {
+            next.terminal_details = providerRejectionDetails(opts.providerRejection, current);
+        }
         if (opts.envelope && opts.envelopeRef) {
             next.last_envelope = {
                 ref: opts.envelopeRef,
@@ -589,19 +625,37 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
         const hasPartialOutput = logFiles.some((file) => fileNonEmpty(file));
         let quota = null;
         let quotaLane = null;
+        let providerRejection = null;
         if (!processAlive && !envelope) {
             try {
                 quotaLane = readLane(hqRoot, laneId);
-                quota = readQuotaSignal(runDir, quotaLane.model, new Date(now()), readSpawnManifest(runDir).worker_out_offset ?? 0);
+                const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
+                const routed = routeProviderFailure(readProviderRejection(runDir, quotaLane.provider, workerOutOffset), () => readQuotaSignal(runDir, quotaLane.model, new Date(now()), workerOutOffset), quotaLane.model, new Date(now()));
+                // A reset-less 429 still holds the account: the configured default
+                // hold becomes its reset, recorded as such.
+                quota = routed.quota
+                    ? applyDefaultQuotaHold(routed.quota, providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[quotaLane.provider]), new Date(now()))
+                    : null;
+                providerRejection = routed.rejection;
             }
             catch (err) {
                 process.stderr.write(`hq lanes watcher: quota read failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
             }
         }
         // Spec §4: the process exited and its envelope file does not parse.
-        // Exactly one repair turn, then failed with both errors.
+        // Exactly one repair turn, then failed with both errors. A provider
+        // rejection skips the repair: the provider refused the turn, so a
+        // second turn on the same model and account would be refused the same
+        // way. A quota signal skips it too: a repair turn on an exhausted account
+        // fails the same way, and would end the lane envelope_invalid without
+        // ever recording the hold. The quota result applies immediately instead.
         let invalidEnvelope = null;
-        if (!processAlive && !envelope && latestFile && latestFile.envelope === null) {
+        if (!processAlive &&
+            !envelope &&
+            !providerRejection &&
+            !quota &&
+            latestFile &&
+            latestFile.envelope === null) {
             const repair = decideEnvelopeRepair(hqRoot, laneId, latestFile, now(), spawnRepair, logDecision);
             if (repair.action === "spawned") {
                 lastSignalMs = now();
@@ -626,6 +680,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
             silenceMs,
             quota,
             invalidEnvelope,
+            providerRejection,
         });
         if (processAlive) {
             maybeHeartbeatLane(hqRoot, laneId, now());
@@ -702,6 +757,13 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
                     state: classification.state,
                     reason: classification.reason,
                     ...(invalidEnvelope ? { envelope_errors: invalidEnvelope.errors } : {}),
+                    ...(providerRejection
+                        ? {
+                            provider_status: providerRejection.status,
+                            provider_message: providerRejection.message,
+                            retryable: providerRejection.retryable,
+                        }
+                        : {}),
                 });
                 if (quota &&
                     quotaLane?.account &&
@@ -719,6 +781,7 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
                     envelopeRef: latest?.ref,
                     at: tick.emit.at,
                     quota,
+                    providerRejection,
                 });
                 const updated = terminalWrite.record;
                 logDecision({

package/dist/lib/mesh/live/backfill-held.d.ts

@@ -49,15 +49,16 @@ export interface HeldSessionRef {
     /** Task id carried on the event, if any. */
     task?: string;
     /**
-     * Per-event local evidence retained while coalescing a held session. Event
-     * timestamps are instance boundaries: a session id may be reused, so a root
-     * observed before the current state.startedAt must not attribute it.
+     * Per-event local evidence retained while coalescing a held session. A
+     * session id may be reused. The current instance is state.bindingEpisodeId,
+     * never the event timestamp: a later `at` on another episode is still stale.
      */
     eventEvidence?: Array<{
         hqRoot?: string;
         cwd?: string;
         harness?: string;
         at?: string;
+        bindingEpisodeId?: string;
     }>;
 }
 export interface BackfillHeldResult {