@indigoai-us/hq-cli 5.130.0 → 5.132.0

package/dist/lib/lanes/provider-rejection.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Provider-level turn failures read off the worker's own stdout.
+ *
+ * A worker whose turn the provider refused (a 400 for a model the account
+ * cannot use, a 402 for an exhausted balance, a 5xx) exits with partial
+ * output and no envelope. Without this module the watcher classifies that
+ * `interrupted` with `exited_with_partial_output`, and the operator has to
+ * read the raw log to learn why. Here the failure is named: the lane is
+ * `failed` with `terminal_reason: provider_rejected` and the provider's own
+ * message on `terminal_details`.
+ *
+ * A 429 is parsed here like any other status so that a codex or grok 429,
+ * which the claude-shaped quota reader cannot see, still reaches the quota
+ * path: the watcher routes a `status === QUOTA_STATUS` failure to quota.ts
+ * (reset time, account hold) and everything else to `provider_rejected`.
+ *
+ * Only the fields named below are read out of the payload. The prompt, the
+ * response body and everything else stay on disk. Nothing here logs.
+ */
+import type { LaneProvider } from "./types.js";
+export declare const PROVIDER_REJECTED_REASON: "provider_rejected";
+/** Bytes of the provider message kept on the record. The log keeps the rest. */
+export declare const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
+/** Status the watcher routes to the quota path instead of `provider_rejected`. */
+export declare const QUOTA_STATUS = 429;
+export interface ProviderRejection {
+    reason: typeof PROVIDER_REJECTED_REASON;
+    provider: LaneProvider;
+    /** The provider's error message, verbatim, capped at PROVIDER_REJECTION_MESSAGE_CAP. */
+    message: string;
+    /** HTTP status the provider reported; null when the payload named none. */
+    status: number | null;
+    /** Provider error class, e.g. `invalid_request_error`; null when absent. */
+    error_type: string | null;
+    /** Derived from the status: a transient class (408, 425, 5xx) is retryable; a 4xx is not. */
+    retryable: boolean;
+    /** Which event on the stream carried the failure. */
+    event: "codex.turn.failed" | "claude.result.is_error" | "grok.error";
+}
+export declare function capMessage(message: string): string;
+/**
+ * Whether a later run could succeed without the operator changing anything.
+ * A 400 model rejection cannot; a request timeout, a too-early, or a server
+ * error can. An unknown status is not retryable: guessing "try again" on a
+ * failure we cannot classify would loop the lane.
+ */
+export declare function retryableFromStatus(status: number | null): boolean;
+/**
+ * Codex `exec --json`: `{"type":"turn.failed","error":{"message":"<text>"}}`.
+ * Measured 2026-09-22 (lane 01M34HCCP1ZDN3H68ESCRZE7RZ): the message is
+ * itself a JSON string, `{"type":"error","status":400,"error":{"type":
+ * "invalid_request_error","message":"The 'terra' model is not supported when
+ * using Codex with a ChatGPT account."}}`. The inner message and status are
+ * read when it parses; otherwise the outer text is the message.
+ */
+export declare function parseCodexTurnFailed(text: string): ProviderRejection | null;
+/**
+ * Claude `-p --output-format json`: the last `type: "result"` object with
+ * `is_error: true` and a numeric `api_error_status`. A result that is an
+ * error with no status is not a provider rejection: it is either the quota
+ * text shape quota.ts reads, or the worker's own failure, and keeps the
+ * existing classification.
+ */
+export declare function parseClaudeResultRejection(text: string): ProviderRejection | null;
+/**
+ * Grok `--output-format json` prints one object. Measured 2026-09-22 (lane
+ * 01M33X586J18J1B6GK0E6M4Y07): `{"type":"error","message":"Internal error:
+ * {\n  \"message\": \"API error (status 402 Payment Required): Grok Build
+ * usage balance exhausted\",\n  \"http_status\": 402, ...}", ...}`. The
+ * inner object carries `message` and `http_status`; when it does not parse,
+ * the status is read from `(status NNN` in the text.
+ */
+export declare function parseGrokErrorOutput(text: string): ProviderRejection | null;
+/** The `terminal_details` record both the watcher and the reconcile pass write for a rejection. */
+export declare function providerRejectionDetails(rejection: ProviderRejection, lane: {
+    model?: string;
+    account?: string;
+}): Record<string, unknown>;
+export declare function parseProviderRejection(provider: LaneProvider, text: string): ProviderRejection | null;
+/**
+ * Read this run's bytes of `logs/worker.out` (past `workerOutOffset`, the
+ * same cut delivery.ts uses) and return the provider rejection they carry,
+ * or null. A missing file is null; any other read error propagates.
+ */
+export declare function readProviderRejection(runDir: string, provider: LaneProvider, workerOutOffset?: number): ProviderRejection | null;
+//# sourceMappingURL=provider-rejection.d.ts.map

package/dist/lib/lanes/provider-rejection.js

@@ -0,0 +1,258 @@
+/**
+ * Provider-level turn failures read off the worker's own stdout.
+ *
+ * A worker whose turn the provider refused (a 400 for a model the account
+ * cannot use, a 402 for an exhausted balance, a 5xx) exits with partial
+ * output and no envelope. Without this module the watcher classifies that
+ * `interrupted` with `exited_with_partial_output`, and the operator has to
+ * read the raw log to learn why. Here the failure is named: the lane is
+ * `failed` with `terminal_reason: provider_rejected` and the provider's own
+ * message on `terminal_details`.
+ *
+ * A 429 is parsed here like any other status so that a codex or grok 429,
+ * which the claude-shaped quota reader cannot see, still reaches the quota
+ * path: the watcher routes a `status === QUOTA_STATUS` failure to quota.ts
+ * (reset time, account hold) and everything else to `provider_rejected`.
+ *
+ * Only the fields named below are read out of the payload. The prompt, the
+ * response body and everything else stay on disk. Nothing here logs.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+export const PROVIDER_REJECTED_REASON = "provider_rejected";
+/** Bytes of the provider message kept on the record. The log keeps the rest. */
+export const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
+/** Status the watcher routes to the quota path instead of `provider_rejected`. */
+export const QUOTA_STATUS = 429;
+function isPlainObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+/**
+ * Walk the JSON lines of `text` from the last line backward, parsing one
+ * line at a time, and return the first value `pick` accepts. A long codex
+ * run writes thousands of events to worker.out; the failure, when there is
+ * one, is at the tail, so the scan stops there and never holds every parsed
+ * event at once. Lines that are not objects are skipped without parsing.
+ */
+function scanJsonLinesBackward(text, pick) {
+    let end = text.length;
+    while (end > 0) {
+        const start = text.lastIndexOf("\n", end - 1);
+        const line = text.slice(start + 1, end).trim();
+        end = start === -1 ? 0 : start;
+        if (!line.startsWith("{"))
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (!isPlainObject(parsed))
+            continue;
+        const picked = pick(parsed);
+        if (picked !== null)
+            return picked;
+    }
+    return null;
+}
+/** Parse the whole text as one JSON object (grok prints a single pretty-printed object). */
+function wholeObject(text) {
+    const trimmed = text.trim();
+    if (!trimmed.startsWith("{"))
+        return null;
+    try {
+        const parsed = JSON.parse(trimmed);
+        return isPlainObject(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+const NOT_REJECTED = Symbol("not_rejected");
+function statusOf(value) {
+    return typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599
+        ? value
+        : null;
+}
+export function capMessage(message) {
+    return message.length > PROVIDER_REJECTION_MESSAGE_CAP
+        ? `${message.slice(0, PROVIDER_REJECTION_MESSAGE_CAP)}…`
+        : message;
+}
+/**
+ * Whether a later run could succeed without the operator changing anything.
+ * A 400 model rejection cannot; a request timeout, a too-early, or a server
+ * error can. An unknown status is not retryable: guessing "try again" on a
+ * failure we cannot classify would loop the lane.
+ */
+export function retryableFromStatus(status) {
+    if (status === null)
+        return false;
+    if (status === 408 || status === 425)
+        return true;
+    return status >= 500 && status <= 599;
+}
+/**
+ * Codex `exec --json`: `{"type":"turn.failed","error":{"message":"<text>"}}`.
+ * Measured 2026-09-22 (lane 01M34HCCP1ZDN3H68ESCRZE7RZ): the message is
+ * itself a JSON string, `{"type":"error","status":400,"error":{"type":
+ * "invalid_request_error","message":"The 'terra' model is not supported when
+ * using Codex with a ChatGPT account."}}`. The inner message and status are
+ * read when it parses; otherwise the outer text is the message.
+ */
+export function parseCodexTurnFailed(text) {
+    return scanJsonLinesBackward(text, (rec) => {
+        if (rec.type !== "turn.failed")
+            return null;
+        const error = isPlainObject(rec.error) ? rec.error : {};
+        const outer = typeof error.message === "string" ? error.message : "";
+        let message = outer;
+        let status = null;
+        let errorType = null;
+        const inner = wholeObject(outer);
+        if (inner) {
+            status = statusOf(inner.status);
+            const innerError = isPlainObject(inner.error) ? inner.error : {};
+            if (typeof innerError.message === "string" && innerError.message.length > 0) {
+                message = innerError.message;
+            }
+            if (typeof innerError.type === "string")
+                errorType = innerError.type;
+        }
+        return {
+            reason: PROVIDER_REJECTED_REASON,
+            provider: "codex",
+            message: capMessage(message),
+            status,
+            error_type: errorType,
+            retryable: retryableFromStatus(status),
+            event: "codex.turn.failed",
+        };
+    });
+}
+/**
+ * Claude `-p --output-format json`: the last `type: "result"` object with
+ * `is_error: true` and a numeric `api_error_status`. A result that is an
+ * error with no status is not a provider rejection: it is either the quota
+ * text shape quota.ts reads, or the worker's own failure, and keeps the
+ * existing classification.
+ */
+export function parseClaudeResultRejection(text) {
+    // The last `result` decides: a success or a status-less error there is
+    // final (NOT_REJECTED), so the scan does not continue to earlier results.
+    const found = scanJsonLinesBackward(text, (rec) => {
+        if (rec.type !== "result")
+            return null;
+        if (rec.is_error !== true)
+            return NOT_REJECTED;
+        const status = statusOf(rec.api_error_status);
+        if (status === null)
+            return NOT_REJECTED;
+        const message = typeof rec.result === "string" ? rec.result : "";
+        return {
+            reason: PROVIDER_REJECTED_REASON,
+            provider: "claude",
+            message: capMessage(message),
+            status,
+            error_type: typeof rec.subtype === "string" ? rec.subtype : null,
+            retryable: retryableFromStatus(status),
+            event: "claude.result.is_error",
+        };
+    });
+    return found === NOT_REJECTED ? null : found;
+}
+/**
+ * Grok `--output-format json` prints one object. Measured 2026-09-22 (lane
+ * 01M33X586J18J1B6GK0E6M4Y07): `{"type":"error","message":"Internal error:
+ * {\n  \"message\": \"API error (status 402 Payment Required): Grok Build
+ * usage balance exhausted\",\n  \"http_status\": 402, ...}", ...}`. The
+ * inner object carries `message` and `http_status`; when it does not parse,
+ * the status is read from `(status NNN` in the text.
+ */
+export function parseGrokErrorOutput(text) {
+    // One pretty-printed object is the measured shape; a line stream is the
+    // fallback, read from the tail like the other providers.
+    const whole = wholeObject(text);
+    if (whole !== null)
+        return grokRejectionFrom(whole);
+    return scanJsonLinesBackward(text, grokRejectionFrom);
+}
+function grokRejectionFrom(rec) {
+    if (rec.type !== "error")
+        return null;
+    const outer = typeof rec.message === "string" ? rec.message : "";
+    let message = outer;
+    let status = null;
+    const braceAt = outer.indexOf("{");
+    const inner = braceAt >= 0 ? wholeObject(outer.slice(braceAt)) : null;
+    if (inner) {
+        if (typeof inner.message === "string" && inner.message.length > 0) {
+            message = inner.message;
+        }
+        status = statusOf(inner.http_status);
+    }
+    if (status === null) {
+        const match = /\(status\s+(\d{3})\b/.exec(outer);
+        status = match ? statusOf(Number(match[1])) : null;
+    }
+    return {
+        reason: PROVIDER_REJECTED_REASON,
+        provider: "grok",
+        message: capMessage(message),
+        status,
+        error_type: null,
+        retryable: retryableFromStatus(status),
+        event: "grok.error",
+    };
+}
+/** The `terminal_details` record both the watcher and the reconcile pass write for a rejection. */
+export function providerRejectionDetails(rejection, lane) {
+    return {
+        reason: rejection.reason,
+        provider: rejection.provider,
+        message: rejection.message,
+        status: rejection.status,
+        error_type: rejection.error_type,
+        retryable: rejection.retryable,
+        event: rejection.event,
+        model: lane.model ?? null,
+        account: lane.account ?? null,
+    };
+}
+export function parseProviderRejection(provider, text) {
+    switch (provider) {
+        case "codex":
+            return parseCodexTurnFailed(text);
+        case "claude":
+            return parseClaudeResultRejection(text);
+        case "grok":
+            return parseGrokErrorOutput(text);
+        default: {
+            const _exhaustive = provider;
+            return _exhaustive;
+        }
+    }
+}
+/**
+ * Read this run's bytes of `logs/worker.out` (past `workerOutOffset`, the
+ * same cut delivery.ts uses) and return the provider rejection they carry,
+ * or null. A missing file is null; any other read error propagates.
+ */
+export function readProviderRejection(runDir, provider, workerOutOffset = 0) {
+    const outPath = path.join(runDir, "logs", "worker.out");
+    let text;
+    try {
+        const bytes = fs.readFileSync(outPath);
+        text = bytes.subarray(Math.min(workerOutOffset, bytes.length)).toString("utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return null;
+        throw err;
+    }
+    return parseProviderRejection(provider, text);
+}
+//# sourceMappingURL=provider-rejection.js.map

package/dist/lib/lanes/quota.d.ts

@@ -26,9 +26,15 @@ export interface ModelQuota {
     /** Operator-facing marker, e.g. "needs credits for claude-fable-5-1". */
     marker?: string;
 }
+export type QuotaResetSource = "provider" | "default_hold";
 export interface AccountQuota {
     /** Set when the 429 is an account-wide usage limit with a reset time. */
     all_models_quota_reset_at?: string;
+    /**
+     * Where `all_models_quota_reset_at` came from: the provider's own message,
+     * or the configured default hold applied to a 429 that named no reset.
+     */
+    reset_source?: QuotaResetSource;
     models: Record<string, ModelQuota>;
 }
 export interface QuotaState {
@@ -43,7 +49,18 @@ export interface QuotaSignal {
     marker?: string;
     /** True when the text is a usage limit with a reset, not a per-model credits refusal. */
     account_wide?: boolean;
+    /** Set by applyDefaultQuotaHold: whether the reset is the provider's or the default hold. */
+    reset_source?: QuotaResetSource;
 }
+/**
+ * A 429 whose message names no reset time and no credits wording would
+ * otherwise record no hold at all, and the next lane on the account would
+ * hit the same 429. Give such a signal the provider's configured default
+ * hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
+ * marked `default_hold`; a signal that already carries a reset keeps it,
+ * marked `provider`. A per-model credits refusal is untouched.
+ */
+export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs: number, now: Date): QuotaSignal;
 export declare function quotaStatePath(hqRoot: string): string;
 export declare function loadQuotaState(hqRoot: string): QuotaState;
 /**

package/dist/lib/lanes/quota.js

@@ -21,6 +21,26 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
 export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
+/**
+ * A 429 whose message names no reset time and no credits wording would
+ * otherwise record no hold at all, and the next lane on the account would
+ * hit the same 429. Give such a signal the provider's configured default
+ * hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
+ * marked `default_hold`; a signal that already carries a reset keeps it,
+ * marked `provider`. A per-model credits refusal is untouched.
+ */
+export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
+    if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
+        return signal;
+    if (signal.quota_reset_at)
+        return { ...signal, reset_source: "provider" };
+    return {
+        ...signal,
+        quota_reset_at: new Date(now.getTime() + defaultHoldMs).toISOString(),
+        account_wide: true,
+        reset_source: "default_hold",
+    };
+}
 const MONTHS = {
     jan: 0,
     feb: 1,
@@ -98,6 +118,9 @@ export function loadQuotaState(hqRoot) {
             if (typeof raw.all_models_quota_reset_at === "string" &&
                 raw.all_models_quota_reset_at.length > 0) {
                 account.all_models_quota_reset_at = raw.all_models_quota_reset_at;
+                if (raw.reset_source === "provider" || raw.reset_source === "default_hold") {
+                    account.reset_source = raw.reset_source;
+                }
             }
             accounts[id] = account;
         }
@@ -244,6 +267,7 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
     const account = { ...current, models };
     if (signal.account_wide && signal.quota_reset_at) {
         account.all_models_quota_reset_at = signal.quota_reset_at;
+        account.reset_source = signal.reset_source ?? "provider";
     }
     accounts[accountId] = account;
     state.providers[provider] = accounts;

package/dist/lib/lanes/reconcile-install.d.ts

@@ -8,8 +8,12 @@
  */
 export declare const RECONCILE_TIMER_INTERVAL = "1min";
 export interface ProviderPathEnv {
-    /** PATH for the systemd unit: resolved provider dirs, systemd defaults, then the shell PATH. */
-    path: string;
+    /**
+     * For each resolved provider: the directory where the command name was
+     * found on the installing shell's PATH, then the realpath target's
+     * directory when it differs. Provider order, deduplicated.
+     */
+    resolved_dirs: string[];
     /** Providers whose binary resolved on the installing shell's PATH. */
     resolved: string[];
     /** Providers whose binary did not resolve. Reported, never fatal on their own. */
@@ -27,6 +31,22 @@ export interface ProviderPathEnv {
  * run.
  */
 export declare function providerPathEnv(providers: readonly string[], pathEnv?: string | undefined): ProviderPathEnv;
+/**
+ * The unit's PATH: each resolved provider binary directory, the directory of
+ * the hq binary itself, the directory of the node binary that runs it (a
+ * provider installed as a `#!/usr/bin/env node` script needs `node` on the
+ * unit PATH, and nvm/asdf installs keep it outside the baseline), then the
+ * systemd baseline, deduplicated. Nothing
+ * from the installing shell's PATH is carried over: on a box whose profile
+ * is synced from another machine that PATH holds directories from the other
+ * machine (measured 2026-09-22: `/Users/…` and `/opt/homebrew/…` entries on
+ * a Linux outpost), and none of them belong in a systemd unit.
+ */
+export declare function unitPathEnv(input: {
+    providerDirs: readonly string[];
+    hqBinary: string;
+    nodeBinary: string;
+}): string;
 export interface RunCommandResult {
     status: number;
     stdout?: string;
@@ -38,7 +58,8 @@ export interface ReconcileInstallOpts {
     hqRoot: string;
     nodeBinary?: string;
     hqBinary?: string;
-    pathEnv?: string;
+    /** Resolved provider binary directories (providerPathEnv().resolved_dirs). */
+    providerDirs?: readonly string[];
     home?: string;
     enable?: boolean;
     runCommand?: RunCommand;

package/dist/lib/lanes/reconcile-install.js

@@ -46,21 +46,57 @@ export function providerPathEnv(providers, pathEnv = process.env.PATH) {
             continue;
         }
         resolved.push(provider);
-        // Keep the real binary's directory, not only a wrapper directory that
-        // happened to be on the installing shell's PATH. Systemd must be able to
-        // invoke the resolved provider after it starts with its own environment.
-        resolvedDirs.push(path.dirname(fs.realpathSync(path.join(found, provider))));
+        // Both directories: the one holding the literal command name (an npm
+        // `codex` link may target `codex.js`, a versioned claude launcher has
+        // another basename; spawn.ts launches the literal name, so its directory
+        // must be on the unit PATH) and the real binary's directory (a wrapper
+        // that only resolves with the installing shell's environment is not
+        // enough for systemd). Unrelated shell PATH entries stay out.
+        const linkDir = path.resolve(found);
+        const realDir = path.dirname(fs.realpathSync(path.join(found, provider)));
+        for (const dir of [linkDir, realDir]) {
+            if (!resolvedDirs.includes(dir))
+                resolvedDirs.push(dir);
+        }
     }
     if (providers.length > 0 && resolved.length === 0) {
         throw new LanesError("provider_binary_unresolved", `Cannot resolve any provider binary (${providers.join(", ")}) from PATH`, { providers: [...providers], searched_path: searched });
     }
-    return {
-        path: [...new Set([...resolvedDirs, ...SYSTEMD_PATH_ENV.split(":"), ...directories])]
-            .filter(Boolean)
-            .join(":"),
-        resolved,
-        unresolved,
-    };
+    return { resolved_dirs: resolvedDirs, resolved, unresolved };
+}
+/**
+ * Directory of the hq entry point as invoked, not its realpath: the `hq`
+ * name on PATH is a symlink in a bin directory, and that bin directory is
+ * what the reconcile pass needs on PATH to run `hq` subcommands.
+ */
+function hqBinaryDir(hqBinary) {
+    return path.dirname(path.resolve(hqBinary));
+}
+/**
+ * The unit's PATH: each resolved provider binary directory, the directory of
+ * the hq binary itself, the directory of the node binary that runs it (a
+ * provider installed as a `#!/usr/bin/env node` script needs `node` on the
+ * unit PATH, and nvm/asdf installs keep it outside the baseline), then the
+ * systemd baseline, deduplicated. Nothing
+ * from the installing shell's PATH is carried over: on a box whose profile
+ * is synced from another machine that PATH holds directories from the other
+ * machine (measured 2026-09-22: `/Users/…` and `/opt/homebrew/…` entries on
+ * a Linux outpost), and none of them belong in a systemd unit.
+ */
+export function unitPathEnv(input) {
+    const baseline = SYSTEMD_PATH_ENV.split(":");
+    const nodeDir = path.dirname(path.resolve(input.nodeBinary));
+    return [
+        ...new Set([
+            ...input.providerDirs,
+            hqBinaryDir(input.hqBinary),
+            // A baseline node (/usr/bin/node) keeps its baseline position.
+            ...(baseline.includes(nodeDir) ? [] : [nodeDir]),
+            ...baseline,
+        ]),
+    ]
+        .filter(Boolean)
+        .join(":");
 }
 function shellQuote(v) {
     return /^[A-Za-z0-9_./:-]+$/.test(v) ? v : `'${v.replace(/'/g, `'\\''`)}'`;
@@ -167,7 +203,7 @@ export function installReconcileTimer(opts) {
         hqBinary,
         hqRoot: opts.hqRoot,
         home: opts.home,
-        pathEnv: opts.pathEnv,
+        pathEnv: unitPathEnv({ providerDirs: opts.providerDirs ?? [], hqBinary, nodeBinary }),
     }));
     const timerChanged = writeIfChanged(timerPath, renderReconcileTimer());
     let enabled = false;

package/dist/lib/lanes/reconcile.js

@@ -8,17 +8,18 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { parseUserYaml } from "../../utils/user-yaml-error.js";
-import { accountMaxRunning, loadLanesConfig } from "./config.js";
+import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
 import { LanesError, isLaneVersionConflict } from "./errors.js";
 import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
 import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
 import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
-import { readQuotaSignal, recordAccountQuota } from "./quota.js";
+import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
 import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
 import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
 import { createLaneMesh, emitLaneDecisionEvents } from "./mesh-events.js";
-import { decideEnvelopeRepair, envelopeQuestionIds } from "./watcher.js";
+import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
+import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
 /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
 export const QUESTION_AGE_MS = 60 * 60 * 1000;
 /** Spec §6.6 + §9: v1 permits exactly one aged advance. */
@@ -431,13 +432,29 @@ function classifyDeadLocalLane(hqRoot, laneId) {
         path.join(runDir, "logs", "last-message.txt"),
     ].some((file) => fileNonEmpty(file));
     const lane = readLane(hqRoot, laneId);
-    const quota = envelope
-        ? null
-        : readQuotaSignal(runDir, lane.model, new Date(), readSpawnManifest(runDir).worker_out_offset ?? 0);
+    // The same routing as the watcher's exit path (watcher.ts): a provider
+    // failure on the stream is read first, a 429 or quota-worded one becomes
+    // the quota signal, any other becomes provider_rejected, and only a stream
+    // with no failure object falls back to the quota reader.
+    const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
+    const now = new Date();
+    const routed = envelope
+        ? { quota: null, rejection: null }
+        : routeProviderFailure(readProviderRejection(runDir, lane.provider, workerOutOffset), () => readQuotaSignal(runDir, lane.model, now, workerOutOffset), lane.model, now);
+    // As in the watcher: a reset-less 429 gets the provider's default hold.
+    const quota = routed.quota
+        ? applyDefaultQuotaHold(routed.quota, providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[lane.provider]), now)
+        : null;
     return {
-        classification: classifyLaneExit({ envelope, hasPartialOutput, quota }),
+        classification: classifyLaneExit({
+            envelope,
+            hasPartialOutput,
+            quota,
+            providerRejection: routed.rejection,
+        }),
         latest,
         quota,
+        providerRejection: routed.rejection,
         lane,
     };
 }
@@ -722,12 +739,17 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                 continue;
             }
             try {
-                const { classification, latest, quota, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
+                const { classification, latest, quota, providerRejection, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
                 // Match the watcher's one-repair decision before classifying an
-                // unparseable envelope after both processes have died.
+                // unparseable envelope after both processes have died. As in the
+                // watcher, a provider rejection or a quota signal skips the repair.
                 const rawLatest = readLatestEnvelopeFile(hqRoot, lane.lane_id);
                 let invalidEnvelope = null;
-                if (!deps.dryRun && rawLatest && rawLatest.envelope === null) {
+                if (!deps.dryRun &&
+                    !providerRejection &&
+                    !quota &&
+                    rawLatest &&
+                    rawLatest.envelope === null) {
                     const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, rawLatest, nowMs, deps.spawnRepair ?? spawnEnvelopeRepair, () => undefined);
                     if (repair.action === "spawned") {
                         armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps);
@@ -757,6 +779,9 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                     state: repairedClassification.state,
                     queued_reason: undefined,
                     terminal_reason: repairedClassification.reason ?? "host_lost",
+                    ...(providerRejection
+                        ? { terminal_details: providerRejectionDetails(providerRejection, current) }
+                        : {}),
                     ...(quota?.quota_reset_at ? { quota_reset_at: quota.quota_reset_at } : {}),
                     ...(quota?.marker ? { model_quota_marker: quota.marker } : {}),
                     last_host_report_at: nowIso,
@@ -837,8 +862,13 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
     }
 }
 async function admitQueued(hqRoot, lanes, summary, deps, now) {
+    // A lane whose pickup probe is still out is not admitted: its record is
+    // about to be bound or failed by the pickup, and a worker spawned now
+    // would keep running behind that write.
     const localQueued = lanes
-        .filter((lane) => lane.state === "queued" && laneRunsHere(lane, deps.host))
+        .filter((lane) => lane.state === "queued" &&
+        lane.pickup_validation_pending !== true &&
+        laneRunsHere(lane, deps.host))
         .sort(compareQueueOrder);
     if (localQueued.length === 0)
         return;