@indigoai-us/hq-cli 5.345.127 → 5.345.128

package/dist/lib/lanes/reconcile.js

@@ -8,7 +8,7 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { parseUserYaml } from "../../utils/user-yaml-error.js";
-import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, freeClaudeAccountSlots, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
+import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, freeClaudeAccountSlots, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneOccupiesRunningSlot, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
 import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
 import { effectiveAccountHome } from "./account-home.js";
 import { enforceRegistryApprovalGate } from "./approval-gate.js";
@@ -34,7 +34,7 @@ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LE
 import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
 import { LANE_PROVIDERS, isIsoDateTime, isTerminalLaneState } from "./types.js";
 import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
-import { deliverQuotaHoldNotice, quotaHoldNotice } from "./quota-notice.js";
+import { deliverQuotaNotice, quotaLaneNotice } from "./quota-notice.js";
 import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
 import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
 import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
@@ -1724,32 +1724,16 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                     ? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
                     : classification;
                 let questionRefusalCode = null;
-                const accountRetryReason = quota
-                    ? "usage_limit"
-                    : providerRejection && classification.reason === "account_not_logged_in"
-                        ? "account_unavailable"
-                        : null;
+                // A usage limit fails only this lane; the senior decides on resume.
+                const accountRetryReason = !quota && providerRejection && classification.reason === "account_not_logged_in"
+                    ? "account_unavailable"
+                    : null;
                 if (!deps.dryRun &&
                     quota &&
                     classified.account &&
                     (classification.reason === "account_quota_exhausted" ||
                         classification.reason === "account_model_unavailable")) {
-                    const quotaState = recordAccountQuota(hqRoot, classified.provider, classified.account, quota, {
-                        now,
-                        unconfirmedHoldMs: providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[classified.provider]),
-                    });
-                    if (classification.reason === "account_quota_exhausted") {
-                        const notice = quotaHoldNotice(quotaState, classified.provider, classified.account, lane.lane_id);
-                        if (notice) {
-                            try {
-                                deliverQuotaHoldNotice(hqRoot, lane, notice, now.toISOString());
-                            }
-                            catch (err) {
-                                const message = err instanceof Error ? err.message : String(err);
-                                process.stderr.write(`hq lanes reconcile: quota hold notice failed for ${lane.lane_id}: ${message}\n`);
-                            }
-                        }
-                    }
+                    recordAccountQuota(hqRoot, classified.provider, classified.account, quota, now);
                 }
                 if (!deps.dryRun &&
                     accountRetryReason &&
@@ -1886,6 +1870,22 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                     result.record.model) {
                     clearAccountQuotaAfterSuccessfulTurnBestEffort(hqRoot, result.record.provider, result.record.account, result.record.model, new Date(nowIso));
                 }
+                if (!deps.dryRun &&
+                    result.changed &&
+                    quota &&
+                    (result.record.terminal_reason === "account_quota_exhausted" ||
+                        result.record.terminal_reason === "account_model_unavailable")) {
+                    // The dead worker's exit is classified here, not in its watcher, so
+                    // the senior notice goes out here, once, after the terminal write and
+                    // keyed to the new version. No account hold is set.
+                    try {
+                        deliverQuotaNotice(hqRoot, result.record, quotaLaneNotice(result.record, quota), nowIso);
+                    }
+                    catch (err) {
+                        const message = err instanceof Error ? err.message : String(err);
+                        process.stderr.write(`hq lanes reconcile: quota notice failed for ${lane.lane_id}: ${message}\n`);
+                    }
+                }
                 if (result.changed) {
                     summary.dead_host.push({
                         id: lane.lane_id,
@@ -3875,4 +3875,58 @@ export async function wakeAnsweredLaneNow(hqRoot, laneId, deps) {
         error: failure?.why ?? `Lane ${laneId} did not leave awaiting_input after its questions were answered`,
     };
 }
+/**
+ * Run queue admission in-process, without the rest of a reconcile pass.
+ *
+ * Commands that change capacity (create, stop, cancel, clear-quota, a lane
+ * exit) call this so queued lanes start without `hq daemon`. It takes the
+ * reconcile lock and every provider admission lock, so it is safe to run
+ * while a daemon is also reconciling: whichever runs second sees the lanes
+ * the first one already admitted.
+ */
+export async function admitQueuedNow(hqRoot, deps) {
+    const summary = emptySummary(deps.dryRun);
+    const now = deps.now();
+    // A queued lane is worth an admission attempt only while its provider has
+    // a free running slot here. Account and profile caps are left to
+    // admitQueued, which can move a lane to another free account.
+    const hasAdmittableQueued = (lanes) => {
+        const queued = lanes.filter((lane) => lane.state === "queued" && laneRunsHere(lane, deps.host));
+        if (queued.length === 0)
+            return false;
+        const config = loadLanesConfig(hqRoot);
+        const nowMs = now.getTime();
+        return queued.some((lane) => {
+            const providerRunning = lanes.filter((other) => other.lane_id !== lane.lane_id && other.provider === lane.provider &&
+                laneOccupiesRunningSlot(other, nowMs) && laneRunsHere(other, deps.host)).length;
+            return providerRunning < config.providers[lane.provider].max_running;
+        });
+    };
+    // Skip the lock and the CPU sample when nothing can start here.
+    if (!hasAdmittableQueued(loadLaneRecords(hqRoot, emptySummary(deps.dryRun)))) {
+        return { admitted: [], errors: [] };
+    }
+    const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
+    await withReconcileLock(hqRoot, async () => {
+        const previousState = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
+        const previousSnapshot = previousState === "missing" || previousState === "unreadable"
+            ? undefined
+            : previousState.cpu_snapshot;
+        const cpuObservation = deps.cpuBusyFraction !== undefined
+            ? undefined
+            : await sampleCpuBusyObservation(throttleDeps, previousSnapshot);
+        const admissionDeps = {
+            ...deps,
+            ...(cpuObservation ? { cpuBusyFraction: cpuObservation.busy, cpuObservation } : {}),
+            skipAdmissionWithoutCpuSample: false,
+        };
+        await withAllLaneAdmissionLocks(hqRoot, async () => {
+            const lanes = loadLaneRecords(hqRoot, summary);
+            if (!hasAdmittableQueued(lanes))
+                return;
+            await admitQueued(hqRoot, lanes, summary, admissionDeps, now);
+        });
+    });
+    return { admitted: summary.admitted, errors: summary.errors };
+}
 //# sourceMappingURL=reconcile.js.map

package/dist/lib/lanes/watcher.d.ts

@@ -211,6 +211,14 @@ export interface WatcherLoopDeps {
     retrySpawnDeps?: Omit<SpawnLaneDeps, "admissionHost">;
     /** Active-company fallback when an unscoped lane asks from its envelope. */
     resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
+    /**
+     * Admit queued lanes after this lane's exit frees its slot. Default: the
+     * in-process admitter (`admitQueuedNow`), so the queue drains without a
+     * daemon. Returns the admitted lane ids.
+     */
+    admitQueued?: (hqRoot: string) => Promise<string[]>;
+    /** Test seam: deliver the one-time usage-limit notice to the lane's senior. */
+    deliverQuotaNotice?: (lane: LaneRecord, text: string) => Promise<void>;
     /** Resume an atomically queued inbox after this watcher wins terminal CAS. */
     recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
     /** Deliver a lane-senior escalation through the normal lane-message router. */

package/dist/lib/lanes/watcher.js

@@ -43,7 +43,7 @@ import { appendCappedHistory, listQuestionIds, readLane, readQuestion, retryUpda
 import { enforceRegistryApprovalGate } from "./approval-gate.js";
 import { readWorkerRunResult } from "./delivery.js";
 import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
-import { deliverQuotaHoldNotice, quotaHoldNotice } from "./quota-notice.js";
+import { deliverQuotaNotice as deliverQuotaNoticeToSenior, quotaLaneNotice } from "./quota-notice.js";
 import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, isProviderLoginFailure, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
 import { appendDropboxMessageAtPath, dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
 import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
@@ -154,14 +154,15 @@ function validFailureOutputTime(value, fallback) {
         : fallback;
 }
 export function routeProviderFailure(failure, readQuota, model, now) {
+    const structured = failure && failure.resets_at !== undefined ? { resets_at: failure.resets_at } : {};
     if (failure && failure.status === QUOTA_STATUS && isAccountQuotaFailure(failure)) {
         return {
-            quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message }, model, failure.output_at ? validFailureOutputTime(failure.output_at, now) : now),
+            quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message, ...structured }, model, failure.output_at ? validFailureOutputTime(failure.output_at, now) : now),
             rejection: null,
         };
     }
     if (failure && failure.status === null && isAccountQuotaFailure(failure)) {
-        const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message }, model, failure.output_at ? validFailureOutputTime(failure.output_at, now) : now);
+        const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message, ...structured }, model, failure.output_at ? validFailureOutputTime(failure.output_at, now) : now);
         if (quota)
             return { quota, rejection: null };
     }
@@ -1167,6 +1168,28 @@ async function settleKrFreshness(hqRoot, lane, manifestTurnId, at, deps, logDeci
         };
     });
 }
+/**
+ * Tell the lane's senior once that a usage limit ended this lane. Called
+ * only on the terminal write that changed the lane; the message id (lane id
+ * and version) dedupes a retried append.
+ */
+async function deliverQuotaNotice(hqRoot, lane, quota, at, deps, logDecision) {
+    const notice = quotaLaneNotice(lane, quota);
+    try {
+        if (deps.deliverQuotaNotice) {
+            await deps.deliverQuotaNotice(lane, notice.text);
+            logDecision({ event: "quota_notice", at, delivered_to: "test_seam" });
+            return;
+        }
+        const deliveredTo = deliverQuotaNoticeToSenior(hqRoot, lane, notice, at);
+        logDecision({ event: "quota_notice", at, delivered_to: deliveredTo });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`hq lanes watcher: quota notice failed for ${lane.lane_id}: ${message}\n`);
+        logDecision({ event: "quota_notice_failed", at, error: message });
+    }
+}
 function findBlockerFreshnessSessionInbox(hqRoot, sessionId) {
     let candidates;
     try {
@@ -1457,6 +1480,18 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
     }
     const progressIntervalMs = deps.progressEventMinIntervalMs ?? readProgressIntervalMs(hqRoot, readLaneSoft(hqRoot, laneId));
     const spawnRepair = deps.spawnRepair ?? spawnEnvelopeRepair;
+    const defaultAdmitQueued = async (root) => {
+        const { admitQueuedNow } = await import("./reconcile.js");
+        const host = (deps.resolveAdmissionHost ?? resolveProcessAdmissionHost)();
+        const result = await admitQueuedNow(root, {
+            now: () => new Date(now()),
+            host,
+            dryRun: false,
+            spawnLaneDeps: { ...deps.retrySpawnDeps, admissionHost: host },
+            ...(deps.retrySpawnDeps?.env ? { env: deps.retrySpawnDeps.env } : {}),
+        });
+        return result.admitted.map((entry) => entry.id);
+    };
     const retryAccount = deps.retryAccount ?? ((root, lane, reason) => retryLaneOnAlternateAccount(root, lane, reason, "watcher", {
         ...deps.retrySpawnDeps,
         admissionHost: (deps.resolveAdmissionHost ?? resolveProcessAdmissionHost)(),
@@ -1765,28 +1800,17 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
                     (classification.reason === "account_quota_exhausted" ||
                         classification.reason === "account_model_unavailable")) {
                     try {
-                        const observedAt = new Date(now());
-                        const quotaState = recordAccountQuota(hqRoot, quotaLane.provider, quotaLane.account, quota, {
-                            now: observedAt,
-                            unconfirmedHoldMs: providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[quotaLane.provider]),
-                        });
-                        if (classification.reason === "account_quota_exhausted") {
-                            const notice = quotaHoldNotice(quotaState, quotaLane.provider, quotaLane.account, quotaLane.lane_id);
-                            if (notice) {
-                                const deliveredTo = deliverQuotaHoldNotice(hqRoot, quotaLane, notice, observedAt.toISOString());
-                                logDecision({ event: "quota_hold_notice", at: tick.emit.at, account: quotaLane.account, delivered_to: deliveredTo });
-                            }
-                        }
+                        recordAccountQuota(hqRoot, quotaLane.provider, quotaLane.account, quota, new Date(now()));
                     }
                     catch (err) {
                         process.stderr.write(`hq lanes watcher: quota record failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
                     }
                 }
-                const accountRetryReason = quota
-                    ? "usage_limit"
-                    : isProviderLoginFailure(providerRejection)
-                        ? "account_unavailable"
-                        : null;
+                // A usage limit fails only this lane: no automatic move to another
+                // account. The senior is told and decides (owner decision 2026-10-14).
+                const accountRetryReason = !quota && isProviderLoginFailure(providerRejection)
+                    ? "account_unavailable"
+                    : null;
                 if (accountRetryReason && quotaLane?.account) {
                     try {
                         const retry = await retryAccount(hqRoot, quotaLane, accountRetryReason);
@@ -1945,6 +1969,24 @@ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
                         logDecision({ event: "keep_alive_terminal_trigger_failed", at: tick.emit.at, error: message });
                     }
                 }
+                if (terminalWrite.changed &&
+                    (updated.terminal_reason === "account_quota_exhausted" || updated.terminal_reason === "account_model_unavailable")) {
+                    await deliverQuotaNotice(hqRoot, updated, quota ?? null, tick.emit.at, deps, logDecision);
+                }
+                if (terminalWrite.changed && updated.state !== "running" && updated.state !== "queued") {
+                    // The exit freed a running slot: admit queued lanes here rather
+                    // than waiting for a reconcile pass that may never run.
+                    try {
+                        const admitted = await (deps.admitQueued ?? defaultAdmitQueued)(hqRoot);
+                        if (admitted.length > 0)
+                            logDecision({ event: "queue_admitted", at: tick.emit.at, admitted });
+                    }
+                    catch (err) {
+                        const message = err instanceof Error ? err.message : String(err);
+                        process.stderr.write(`hq lanes watcher: queue admission failed after ${laneId}: ${message}\n`);
+                        logDecision({ event: "queue_admission_failed", at: tick.emit.at, error: message });
+                    }
+                }
             }
         }
         lastSignalMs = tick.nextLastSignalMs;

package/dist/utils/resident-package-preload.js

@@ -62,12 +62,12 @@ export const RESIDENT_PACKAGE_MODULES = {
     ],
     "daemon-logs-follow": [],
     "mesh-daemon": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
-    "lanes-watch": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
-    "monitor-wait": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
-    "monitor-run": ["../commands/lanes.js", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
+    "lanes-watch": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud", "../lib/lanes/reconcile.js"],
+    "monitor-wait": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud", "../lib/lanes/reconcile.js"],
+    "monitor-run": ["../commands/lanes.js", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud", "../lib/lanes/reconcile.js"],
     "agents-message-wait": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
     "integrations-oauth-wait": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
-    "outpost-wait": ["../commands/lanes.js", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud"],
+    "outpost-wait": ["../commands/lanes.js", "../lib/plan-limit-nag.js", "../utils/plan-gate-error.js", "../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "open", "../utils/cognito-session.js", "../lib/first-sync-next-step.js", "@indigoai-us/hq-cloud", "../lib/lanes/reconcile.js"],
     "secrets-sandbox": [],
     "pkg-update": ["../sentry.js", "../lib/doctor/client-health-reason-codes-flag.js", "@indigoai-us/hq-flags-client", "../utils/cognito-session.js", "@indigoai-us/hq-cloud"],
     "pack-install": [
@@ -255,6 +255,8 @@ const LOADERS = {
         () => import("../utils/cognito-session.js"),
         () => import("../lib/first-sync-next-step.js"),
         () => import("@indigoai-us/hq-cloud"),
+        // The watcher admits queued lanes in-process after a lane exits.
+        () => import("../lib/lanes/reconcile.js"),
     ],
     "monitor-wait": [
         async () => {
@@ -269,6 +271,8 @@ const LOADERS = {
         () => import("../utils/cognito-session.js"),
         () => import("../lib/first-sync-next-step.js"),
         () => import("@indigoai-us/hq-cloud"),
+        // The watcher admits queued lanes in-process after a lane exits.
+        () => import("../lib/lanes/reconcile.js"),
     ],
     "monitor-run": [
         () => import("../commands/lanes.js"),
@@ -284,6 +288,8 @@ const LOADERS = {
         () => import("../utils/cognito-session.js"),
         () => import("../lib/first-sync-next-step.js"),
         () => import("@indigoai-us/hq-cloud"),
+        // The watcher admits queued lanes in-process after a lane exits.
+        () => import("../lib/lanes/reconcile.js"),
     ],
     "agents-message-wait": [
         async () => {
@@ -327,6 +333,8 @@ const LOADERS = {
         () => import("../utils/cognito-session.js"),
         () => import("../lib/first-sync-next-step.js"),
         () => import("@indigoai-us/hq-cloud"),
+        // The watcher admits queued lanes in-process after a lane exits.
+        () => import("../lib/lanes/reconcile.js"),
     ],
     "secrets-sandbox": [],
     "pkg-update": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.345.127",
+  "version": "5.345.128",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {