@indigoai-us/hq-cli 5.149.1 → 5.150.0

package/dist/lib/lanes/admission.d.ts

@@ -0,0 +1,24 @@
+import type { LaneRecord, LanesConfig } from "./types.js";
+export interface AdmissionHost {
+    hostname: string;
+    outpostId: string | null;
+}
+export type AdmissionCapReason = "provider_cap" | "account_cap" | "profile_cap";
+export declare const ADMISSION_LOCK_STALE_MS = 30000;
+export declare const ADMISSION_LOCK_TIMEOUT_MS = 5000;
+export declare const ADMISSION_RESERVATION_GRACE_MS = 30000;
+export declare function withLaneAdmissionLock<T>(hqRoot: string, provider: LaneRecord["provider"], fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
+export declare function withAllLaneAdmissionLocks<T>(hqRoot: string, fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
+export declare function laneRunsOnHost(lane: LaneRecord, host: AdmissionHost): boolean;
+export declare function compareLaneQueueOrder(a: LaneRecord, b: LaneRecord): number;
+export declare function hasActiveAdmissionReservation(lane: LaneRecord, nowMs: number): boolean;
+export declare function hasActivePendingResume(lane: LaneRecord, nowMs: number): boolean;
+/** Return the first capacity limit that currently prevents this lane from starting. */
+export declare function laneAdmissionCapReason(lanes: LaneRecord[], candidate: LaneRecord, config: LanesConfig, host: AdmissionHost, profileMaxConcurrent?: number, nowMs?: number): AdmissionCapReason | undefined;
+/**
+ * A prior queue entry owns a free slot only when its own provider, account,
+ * and profile caps allow it to use that slot. A blocked profile therefore
+ * cannot hold an unrelated profile's queue position.
+ */
+export declare function hasEarlierEligibleQueuedLane(lanes: LaneRecord[], candidate: LaneRecord, config: LanesConfig, host: AdmissionHost, profileMaxFor: (lane: LaneRecord) => number | undefined, nowMs?: number): boolean;
+//# sourceMappingURL=admission.d.ts.map

package/dist/lib/lanes/admission.js

@@ -0,0 +1,193 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { accountMaxRunning } from "./config.js";
+import { LanesError } from "./errors.js";
+export const ADMISSION_LOCK_STALE_MS = 30_000;
+export const ADMISSION_LOCK_TIMEOUT_MS = 5_000;
+export const ADMISSION_RESERVATION_GRACE_MS = 30_000;
+function defaultAdmissionSleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function admissionLockPath(hqRoot, provider) {
+    return path.join(hqRoot, "workspace", "lanes-runs", ".admission-locks", `${provider}-provider.lock`);
+}
+function admissionLockOwnerAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code !== "ESRCH";
+    }
+}
+async function withAdmissionLock(lockPath, fn, sleep) {
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    const started = Date.now();
+    while (true) {
+        let fd;
+        try {
+            fd = fs.openSync(lockPath, "wx");
+            const ownedStat = fs.fstatSync(fd);
+            try {
+                fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n`, "utf8"));
+            }
+            catch (err) {
+                fs.closeSync(fd);
+                fd = undefined;
+                throw err;
+            }
+            try {
+                await new Promise((resolve) => setImmediate(resolve));
+                return await fn();
+            }
+            finally {
+                try {
+                    fs.closeSync(fd);
+                }
+                catch (err) {
+                    process.stderr.write(`hq lanes: admission lock close failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+                try {
+                    const currentStat = fs.statSync(lockPath);
+                    if (currentStat.dev === ownedStat.dev && currentStat.ino === ownedStat.ino) {
+                        fs.unlinkSync(lockPath);
+                    }
+                }
+                catch (err) {
+                    if (err.code !== "ENOENT") {
+                        process.stderr.write(`hq lanes: admission lock unlink failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
+                    }
+                }
+            }
+        }
+        catch (err) {
+            if (err.code !== "EEXIST")
+                throw err;
+            if (Date.now() - started > ADMISSION_LOCK_TIMEOUT_MS) {
+                throw new LanesError("store_lock_timeout", `Timed out waiting for admission lock ${lockPath}`, { path: lockPath });
+            }
+            try {
+                const raw = fs.readFileSync(lockPath, "utf8");
+                const [ownerLine, timestampLine] = raw.split("\n");
+                const ownerPid = ownerLine ? Number(ownerLine) : NaN;
+                const timestamp = timestampLine ? Number(timestampLine) : NaN;
+                if (Number.isSafeInteger(ownerPid) &&
+                    ownerPid > 0 &&
+                    Number.isFinite(timestamp) &&
+                    Date.now() - timestamp > ADMISSION_LOCK_STALE_MS &&
+                    !admissionLockOwnerAlive(ownerPid)) {
+                    if (fs.readFileSync(lockPath, "utf8") !== raw)
+                        continue;
+                    try {
+                        fs.unlinkSync(lockPath);
+                    }
+                    catch (unlinkErr) {
+                        if (unlinkErr.code !== "ENOENT")
+                            throw unlinkErr;
+                    }
+                    continue;
+                }
+            }
+            catch (probeErr) {
+                if (probeErr.code !== "ENOENT") {
+                    process.stderr.write(`hq lanes: stale admission-lock probe failed for ${lockPath}: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}\n`);
+                }
+            }
+            await sleep(10);
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+    }
+}
+export function withLaneAdmissionLock(hqRoot, provider, fn, sleep = defaultAdmissionSleep) {
+    return withAdmissionLock(admissionLockPath(hqRoot, provider), fn, sleep);
+}
+export async function withAllLaneAdmissionLocks(hqRoot, fn, sleep = defaultAdmissionSleep) {
+    const providers = ["claude", "codex", "grok"];
+    const acquire = (index) => {
+        const provider = providers[index];
+        if (provider === undefined)
+            return fn();
+        return withLaneAdmissionLock(hqRoot, provider, () => acquire(index + 1), sleep);
+    };
+    return acquire(0);
+}
+export function laneRunsOnHost(lane, host) {
+    if (host.outpostId && lane.host.kind === "outpost") {
+        return lane.host.id === host.outpostId;
+    }
+    return lane.host.kind === "local" && lane.host.id === host.hostname;
+}
+export function compareLaneQueueOrder(a, b) {
+    if (b.priority !== a.priority)
+        return b.priority - a.priority;
+    const byQueueAge = Date.parse(a.queued_at ?? a.created_at) - Date.parse(b.queued_at ?? b.created_at);
+    return byQueueAge || a.lane_id.localeCompare(b.lane_id);
+}
+export function hasActiveAdmissionReservation(lane, nowMs) {
+    if (lane.state !== "queued" || lane.queued_reason !== undefined || !lane.admission_reserved_at) {
+        return false;
+    }
+    const reservedAt = Date.parse(lane.admission_reserved_at);
+    const ageMs = nowMs - reservedAt;
+    return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
+}
+export function hasActivePendingResume(lane, nowMs) {
+    const pending = lane.pending_resume;
+    if (!pending)
+        return false;
+    const reservedAt = Date.parse(pending.at);
+    const ageMs = nowMs - reservedAt;
+    return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
+}
+/** Return the first capacity limit that currently prevents this lane from starting. */
+export function laneAdmissionCapReason(lanes, candidate, config, host, profileMaxConcurrent, nowMs = Date.now()) {
+    const running = lanes.filter((lane) => lane.lane_id !== candidate.lane_id &&
+        (lane.state === "running" ||
+            hasActiveAdmissionReservation(lane, nowMs) ||
+            hasActivePendingResume(lane, nowMs)) &&
+        laneRunsOnHost(lane, host));
+    const providerCfg = config.providers[candidate.provider];
+    const providerRunning = running.filter((lane) => lane.provider === candidate.provider);
+    if (providerRunning.length >= providerCfg.max_running)
+        return "provider_cap";
+    if (candidate.account &&
+        providerRunning.filter((lane) => lane.account === candidate.account).length >=
+            accountMaxRunning(providerCfg, candidate.account)) {
+        return "account_cap";
+    }
+    if (candidate.worker_profile &&
+        profileMaxConcurrent !== undefined &&
+        running.filter((lane) => lane.worker_profile === candidate.worker_profile).length >=
+            profileMaxConcurrent) {
+        return "profile_cap";
+    }
+    return undefined;
+}
+/**
+ * A prior queue entry owns a free slot only when its own provider, account,
+ * and profile caps allow it to use that slot. A blocked profile therefore
+ * cannot hold an unrelated profile's queue position.
+ */
+export function hasEarlierEligibleQueuedLane(lanes, candidate, config, host, profileMaxFor, nowMs = Date.now()) {
+    return lanes
+        .filter((lane) => lane.lane_id !== candidate.lane_id &&
+        lane.state === "queued" &&
+        lane.provider === candidate.provider &&
+        laneRunsOnHost(lane, host) &&
+        compareLaneQueueOrder(lane, candidate) < 0)
+        .some((lane) => {
+        const profileMaxConcurrent = profileMaxFor(lane);
+        const capReason = laneAdmissionCapReason(lanes, lane, config, host, profileMaxConcurrent, nowMs);
+        if (capReason === undefined)
+            return true;
+        if (capReason !== "account_cap" || lane.admission_accounts === undefined)
+            return false;
+        return lane.admission_accounts.some((account) => {
+            if (account === lane.account || config.providers[lane.provider].accounts[account]?.enabled !== true) {
+                return false;
+            }
+            return laneAdmissionCapReason(lanes, { ...lane, account }, config, host, profileMaxConcurrent, nowMs) === undefined;
+        });
+    });
+}
+//# sourceMappingURL=admission.js.map

package/dist/lib/lanes/delivery.js

@@ -280,9 +280,11 @@ export function resumeCommandRefusal(lane) {
             error: "lane_not_resumable",
             message: `Lane is ${lane.state}. hq lanes resume continues a lane in ${RESUME_COMMAND_STATES.join(", ")}; ${lane.state === "awaiting_input"
                 ? "use hq lanes questions answer <question-id>."
-                : lane.state === "queued" || lane.state === "running"
-                    ? "use hq lanes message for a live lane."
-                    : "a lane in this state is not resumed."}`,
+                : lane.state === "queued"
+                    ? "use hq lanes admit <lane-id>."
+                    : lane.state === "running"
+                        ? "use hq lanes message for a live lane."
+                        : "a lane in this state is not resumed."}`,
         };
     }
     if (!providerSupportsSessionResume(lane.provider)) {

package/dist/lib/lanes/index.d.ts

@@ -3,6 +3,8 @@
  */
 export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
 export { MAX_LANE_DEPTH } from "./constants.js";
+export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
+export type { AdmissionCapReason, AdmissionHost } from "./admission.js";
 export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
 export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneEnvelope, LaneHost, LaneProvider, LaneRecord, LaneStory, LaneStoryStatus, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
 export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";

package/dist/lib/lanes/index.js

@@ -3,6 +3,7 @@
  */
 export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
 export { MAX_LANE_DEPTH } from "./constants.js";
+export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
 export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
 export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
 export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerAutoCompactTokens, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";

package/dist/lib/lanes/reconcile.js

@@ -8,14 +8,16 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { parseUserYaml } from "../../utils/user-yaml-error.js";
-import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
+import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
+import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
+import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
 import { MAX_LANE_DEPTH } from "./constants.js";
-import { LanesError, isLaneVersionConflict } from "./errors.js";
+import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
 import { loadWorkerProfile } from "./profile.js";
 import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
 import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
-import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
+import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
 import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
 import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
 import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
@@ -88,10 +90,10 @@ export function timeoutPath(hqRoot, laneId) {
     return path.join(laneRunDir(hqRoot, laneId), "timeout.json");
 }
 export function laneRunsHere(lane, host) {
-    if (host.outpostId && lane.host.kind === "outpost") {
-        return lane.host.id === host.outpostId;
-    }
-    return lane.host.kind === "local" && lane.host.id === host.hostname;
+    return laneRunsOnHost(lane, host);
+}
+function hasFreshAdmissionReservation(lane, nowMs) {
+    return hasActiveAdmissionReservation(lane, nowMs);
 }
 export function agedAdvanceCount(question) {
     return question.routing_history.filter((entry) => entry.reason === "unanswered_timeout").length;
@@ -163,11 +165,6 @@ export function isDeadlinePassed(deadlineAt, nowMs) {
         return false;
     return nowMs > at;
 }
-function compareQueueOrder(a, b) {
-    if (b.priority !== a.priority)
-        return b.priority - a.priority;
-    return a.created_at.localeCompare(b.created_at);
-}
 function readDirJsonIds(dir, label) {
     let names;
     try {
@@ -308,6 +305,53 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
     }
     throw new LanesError("version_conflict", `Exhausted CAS retries for lane ${laneId}`, { lane_id: laneId });
 }
+function writeQueueReason(hqRoot, lane, reason, dryRun) {
+    if (lane.queued_reason === reason)
+        return;
+    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, admission_reserved_at: undefined }), dryRun);
+}
+function availableBalancedAccount(hqRoot, lane, occupying, config, host, profileMaxConcurrent, nowMs) {
+    if (!lane.account || !lane.admission_accounts?.includes(lane.account))
+        return undefined;
+    if (laneAdmissionCapReason(occupying, lane, config, host, profileMaxConcurrent, nowMs) !== "account_cap") {
+        return undefined;
+    }
+    const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
+        ? lane.model
+        : lane.model_source === "profile"
+            ? lane.model
+            : undefined;
+    for (const account of lane.admission_accounts) {
+        if (account === lane.account)
+            continue;
+        if (config.providers[lane.provider].accounts[account]?.enabled !== true)
+            continue;
+        const status = accountStatus(lane.provider, account, config);
+        if (!isCredentialProven(status))
+            continue;
+        try {
+            resolveAccount(lane.provider, account, config, {
+                hqRoot,
+                ...(pinnedModel !== undefined ? { model: pinnedModel } : {}),
+            });
+        }
+        catch (err) {
+            if (err instanceof AccountResolutionError)
+                continue;
+            throw err;
+        }
+        const candidate = { ...lane, account };
+        if (laneAdmissionCapReason(occupying, candidate, config, host, profileMaxConcurrent, nowMs) === undefined) {
+            return account;
+        }
+    }
+    return undefined;
+}
+function writeAdmissionAccount(hqRoot, lane, account, dryRun) {
+    return casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
+        current.account === lane.account &&
+        current.admission_accounts?.includes(account) === true, (current) => ({ ...current, account }), dryRun).record;
+}
 function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
     for (let i = 0; i < 5; i += 1) {
         const current = readQuestion(hqRoot, questionId);
@@ -707,10 +751,12 @@ export async function runPassBody(hqRoot, deps) {
             });
         }
     }
-    const freshLanes = loadLaneRecords(hqRoot, summary);
-    const freshQuestions = loadQuestionRecords(hqRoot, summary);
-    await admitQueued(hqRoot, freshLanes, summary, deps, now);
-    await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
+    await withAllLaneAdmissionLocks(hqRoot, async () => {
+        const freshLanes = loadLaneRecords(hqRoot, summary);
+        const freshQuestions = loadQuestionRecords(hqRoot, summary);
+        await admitQueued(hqRoot, freshLanes, summary, deps, now);
+        await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
+    });
     if (!deps.dryRun && summary.errors.length === 0) {
         try {
             writeHeartbeatFile(hqRoot, summary, nowIso);
@@ -1005,7 +1051,8 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         .filter((lane) => lane.state === "queued" &&
         lane.pickup_validation_pending !== true &&
         laneRunsHere(lane, deps.host))
-        .sort(compareQueueOrder);
+        .filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
+        .sort(compareLaneQueueOrder);
     if (localQueued.length === 0)
         return;
     const config = loadLanesConfig(hqRoot);
@@ -1030,31 +1077,37 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
     });
     const spawnFn = deps.spawn ??
         ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
-    const running = lanes.filter((lane) => lane.state === "running");
+    const running = lanes.filter((lane) => (lane.state === "running" ||
+        hasFreshAdmissionReservation(lane, now.getTime()) ||
+        hasActivePendingResume(lane, now.getTime())) &&
+        laneRunsHere(lane, deps.host));
     const starvation = decision.admit && decision.reason === "starvation_bound";
-    const candidates = starvation ? localQueued.slice(0, 1) : localQueued;
     const spawned = [];
     if (!decision.admit) {
+        for (const lane of localQueued) {
+            let profileMaxConcurrent;
+            if (lane.worker_profile !== undefined) {
+                try {
+                    profileMaxConcurrent = loadWorkerProfile(hqRoot, lane.worker_profile).meta.max_concurrent;
+                }
+                catch (err) {
+                    markDirty(summary, {
+                        kind: "lane",
+                        id: lane.lane_id,
+                        why: `profile admission failed: ${err instanceof Error ? err.message : String(err)}`,
+                    });
+                    continue;
+                }
+            }
+            const capReason = laneAdmissionCapReason(running, lane, config, deps.host, profileMaxConcurrent, now.getTime());
+            writeQueueReason(hqRoot, lane, capReason ?? "throttle", deps.dryRun);
+        }
         if (!deps.dryRun && throttleState !== "unreadable") {
             writeThrottleState(filePath, applyThrottleDecision(throttleState, decision, nowMs), throttleDeps);
         }
         return;
     }
-    for (const lane of candidates) {
-        const provider = lane.provider;
-        const providerCfg = config.providers[provider];
-        const providerRunning = running.filter((row) => row.provider === provider).length +
-            spawned.filter((row) => row.provider === provider).length;
-        if (providerRunning >= providerCfg.max_running && !starvation)
-            continue;
-        if (lane.account) {
-            const accountRunning = running.filter((row) => row.provider === provider && row.account === lane.account).length +
-                spawned.filter((row) => row.provider === provider && row.account === lane.account).length;
-            if (accountRunning >= accountMaxRunning(providerCfg, lane.account) &&
-                !starvation) {
-                continue;
-            }
-        }
+    for (const lane of localQueued) {
         let profileMaxConcurrent;
         if (lane.worker_profile !== undefined) {
             try {
@@ -1069,11 +1122,22 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
                 continue;
             }
         }
-        if (profileMaxConcurrent !== undefined) {
-            const profileRunning = running.filter((row) => row.worker_profile === lane.worker_profile).length +
-                spawned.filter((row) => row.worker_profile === lane.worker_profile).length;
-            if (profileRunning >= profileMaxConcurrent)
-                continue;
+        const projected = [...running, ...spawned];
+        const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
+        const admittedLane = alternateAccount !== undefined
+            ? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
+            : lane;
+        const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
+        const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
+        let blockedBy;
+        if (currentCap !== undefined) {
+            blockedBy = baseCap ?? "queue_position";
+        }
+        if (!blockedBy && !decision.admit)
+            blockedBy = "throttle";
+        if (blockedBy) {
+            writeQueueReason(hqRoot, admittedLane, blockedBy, deps.dryRun);
+            continue;
         }
         try {
             if (deps.dryRun) {
@@ -1083,12 +1147,12 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
                     to: "running",
                     reason: decision.reason,
                 });
-                spawned.push({ ...lane, state: "running" });
+                spawned.push({ ...admittedLane, state: "running" });
                 if (starvation)
                     break;
                 continue;
             }
-            const started = spawnFn(hqRoot, lane);
+            const started = spawnFn(hqRoot, admittedLane);
             if (decision.reason === "starvation_bound") {
                 casLaneIf(hqRoot, lane.lane_id, (current) => current.admitted_reason !== "starvation_bound", (current) => ({ ...current, admitted_reason: "starvation_bound" }), false);
             }
@@ -1127,25 +1191,73 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
     }
 }
 async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
-    const spawnFn = deps.spawn ??
-        ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
+    const config = loadLanesConfig(hqRoot);
     const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" && laneRunsHere(lane, deps.host));
+    if (awaiting.length === 0)
+        return;
+    const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
+    let throttleDecision;
+    if (deps.dryRun) {
+        const loaded = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
+        const state = loaded === "missing" ? emptyThrottleState() : loaded;
+        throttleDecision = evaluateThrottle({
+            busy: await resolveCpuBusyFraction(throttleDeps),
+            availableMemMb: readAvailableMemMb(throttleDeps),
+            cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
+            state,
+            throttle: config.throttle,
+            nowMs: deps.now().getTime(),
+        });
+    }
+    else {
+        const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
+        throttleDecision = throttle.decision;
+    }
+    if (!throttleDecision.admit)
+        return;
+    const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
+    const nowMs = deps.now().getTime();
+    const capacityLanes = lanes.map((lane) => {
+        try {
+            return readLane(hqRoot, lane.lane_id);
+        }
+        catch {
+            return lane;
+        }
+    });
+    const running = capacityLanes.filter((lane) => (lane.state === "running" ||
+        hasFreshAdmissionReservation(lane, nowMs) ||
+        hasActivePendingResume(lane, nowMs)) &&
+        laneRunsHere(lane, deps.host));
     for (const lane of awaiting) {
+        const current = readLane(hqRoot, lane.lane_id);
+        if (current.state !== "awaiting_input")
+            continue;
         const pending = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "pending");
         const answered = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "answered");
         if (pending.length > 0 || answered.length === 0)
             continue;
         try {
+            let profileMaxConcurrent;
+            if (current.worker_profile !== undefined) {
+                profileMaxConcurrent = loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent;
+            }
+            if (laneAdmissionCapReason(running, current, config, deps.host, profileMaxConcurrent, nowMs) !== undefined) {
+                continue;
+            }
             if (deps.dryRun) {
                 summary.resumed.push({
                     id: lane.lane_id,
                     from: "awaiting_input",
                     to: "running",
                 });
+                running.push({ ...current, state: "running" });
                 continue;
             }
             writeAnsweredQuestions(hqRoot, lane, questions);
             const started = spawnFn(hqRoot, lane);
+            if (started.state === "running")
+                running.push(started);
             summary.resumed.push({
                 id: started.lane_id,
                 from: "awaiting_input",

package/dist/lib/lanes/spawn.js

@@ -844,6 +844,8 @@ export function markSpawnFailure(hqRoot, laneId, reasonRaw, details) {
             ...current,
             state: "failed",
             queued_reason: undefined,
+            queued_at: undefined,
+            admission_reserved_at: undefined,
             terminal_reason: reason,
             terminal_details: details,
         };
@@ -1360,6 +1362,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...current,
             state: "running",
             queued_reason: undefined,
+            queued_at: undefined,
+            admission_reserved_at: undefined,
             // An account move's reservation ends with the claim.
             pending_resume: undefined,
             account: resolved.id,

package/dist/lib/lanes/store.js

@@ -13,7 +13,7 @@ import * as path from "node:path";
 import { generateUlid } from "../mesh/live/ulid.js";
 import { resolveLanesHqRoot } from "./hq-root.js";
 import { LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesError, isLaneVersionConflict, } from "./errors.js";
-import { HISTORY_CAP, HOST_KINDS, LANE_STORY_STATUSES, LANE_SCHEMA_VERSION, QUESTION_SCHEMA_VERSION, SENIOR_KINDS, isHostKind, isLaneProvider, isLaneState, isQuestionStatus, isSeniorKind, parseMajorSchema, REQUIRED_LANE_RECORD_FIELDS, LANE_FIELD_WRITERS, LANE_WRITERS, } from "./types.js";
+import { HISTORY_CAP, HOST_KINDS, LANE_STORY_STATUSES, LANE_SCHEMA_VERSION, QUESTION_SCHEMA_VERSION, SENIOR_KINDS, isHostKind, isLaneProvider, isQueuedReason, isLaneState, isQuestionStatus, isSeniorKind, parseMajorSchema, REQUIRED_LANE_RECORD_FIELDS, LANE_FIELD_WRITERS, LANE_WRITERS, } from "./types.js";
 import { laneStories } from "./stories.js";
 export const LANES_STORE_REL = path.join("workspace", "lanes");
 export const LANE_RECORDS_DIR = "lanes";
@@ -245,6 +245,25 @@ function requireString(value, field) {
     }
     return value;
 }
+function requireTimestamp(value, field, filePath) {
+    const parsedAt = typeof value === "string" ? Date.parse(value) : NaN;
+    const datePart = typeof value === "string" ? value.slice(0, 10) : "";
+    const datePartAt = /^\d{4}-\d{2}-\d{2}$/.test(datePart)
+        ? Date.parse(`${datePart}T00:00:00.000Z`)
+        : NaN;
+    if (typeof value !== "string" ||
+        !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value) ||
+        !Number.isFinite(parsedAt) ||
+        !Number.isFinite(datePartAt) ||
+        new Date(datePartAt).toISOString().slice(0, 10) !== datePart) {
+        throw new LaneSchemaError(`Expected ISO timestamp for ${field}`, {
+            path: filePath,
+            field,
+            value,
+        });
+    }
+    return value;
+}
 function requireInt(value, field) {
     if (typeof value !== "number" || !Number.isInteger(value)) {
         throw new LaneSchemaError(`Expected integer for ${field}`, { field });
@@ -371,6 +390,31 @@ export function parseLaneRecord(raw, filePath) {
     requireInt(raw.priority, "priority");
     requireString(raw.project_id, "project_id");
     requireString(raw.story_id, "story_id");
+    requireTimestamp(raw.created_at, "created_at", filePath);
+    requireTimestamp(raw.updated_at, "updated_at", filePath);
+    if (raw.queued_reason !== undefined && !isQueuedReason(raw.queued_reason)) {
+        throw new LaneSchemaError(`Unknown queued reason ${JSON.stringify(raw.queued_reason)}`, {
+            path: filePath,
+            field: "queued_reason",
+            value: raw.queued_reason,
+        });
+    }
+    if (raw.queued_at !== undefined)
+        requireTimestamp(raw.queued_at, "queued_at", filePath);
+    if (raw.admission_reserved_at !== undefined) {
+        requireTimestamp(raw.admission_reserved_at, "admission_reserved_at", filePath);
+    }
+    if (raw.admission_accounts !== undefined) {
+        if (!Array.isArray(raw.admission_accounts) ||
+            raw.admission_accounts.length === 0 ||
+            raw.admission_accounts.some((account) => typeof account !== "string" || account.trim() === "") ||
+            new Set(raw.admission_accounts).size !== raw.admission_accounts.length) {
+            throw new LaneSchemaError("admission_accounts must be a non-empty array of unique account ids", { path: filePath, field: "admission_accounts" });
+        }
+        if (typeof raw.account !== "string" || !raw.admission_accounts.includes(raw.account)) {
+            throw new LaneSchemaError("account must be present in admission_accounts", { path: filePath, field: "admission_accounts", account: raw.account });
+        }
+    }
     if (!isLaneState(raw.state)) {
         throw new LaneSchemaError(`Unknown lane state ${JSON.stringify(raw.state)}`, { path: filePath, value: raw.state });
     }
@@ -704,6 +748,9 @@ export function createLane(hqRoot, init, deps = defaultStoreDeps()) {
         version: 1,
         created_at: init.created_at ?? now,
         updated_at: init.updated_at ?? now,
+        ...(init.state === "queued"
+            ? { queued_at: init.queued_at ?? init.created_at ?? now }
+            : {}),
         senior_history: init.senior_history ?? [],
         attribution_history: init.attribution_history ?? [],
     };