@indigoai-us/hq-cli 5.149.1 → 5.151.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { Command } from "commander";
2
- import { buildReportTarball, deployOrgContext, type ChannelReport, type DeployAccessPolicy, type ReportDeployment, type ReportPatch } from "../lib/report-publish.js";
2
+ import { buildReportTarball, deployOrgContext, type ChannelReport, type DeployAccessPolicy, type ReportDeployment, type ReportPatch, ReportPublishError } from "../lib/report-publish.js";
3
3
  type ChannelRow = {
4
4
  channelId?: string;
5
5
  scope?: string;
@@ -17,7 +17,21 @@ export declare function withChannelCompany(report: ChannelReport, channels: Chan
17
17
  /** Explicit context has priority; discovery is the authenticated fallback for normal CLI use. */
18
18
  export declare function resolveReport(token: string, reportId: string, requestedChannelId?: string): Promise<ChannelReport>;
19
19
  export declare function patchReport(token: string, report: ChannelReport, patch: ReportPatchInput): Promise<ChannelReport>;
20
+ /** hq-deploy refused the subdomain: another app, not visible to this caller, holds it. */
21
+ export declare class SubdomainTakenError extends ReportPublishError {
22
+ constructor();
23
+ }
24
+ /**
25
+ * Subdomains to try, in order. The first is the report's own slug; the rest
26
+ * are used only when an app the caller cannot see already holds it (for
27
+ * example, one left by an earlier publish under a different account).
28
+ */
29
+ export declare function candidateSlugs(slug: string, attempts?: number): string[];
20
30
  export declare function resolveCompanySlug(token: string, companyUid: string): Promise<string>;
31
+ export declare function findOrCreateApp(token: string, slug: string, org: ReturnType<typeof deployOrgContext>): Promise<{
32
+ id: string;
33
+ subdomain: string;
34
+ }>;
21
35
  export declare function deployReportStatic(token: string, input: {
22
36
  slug: string;
23
37
  files: Parameters<typeof buildReportTarball>[0];
@@ -232,10 +232,28 @@ async function deployJson(token, route, method, body, extraHeaders = {}) {
232
232
  catch {
233
233
  throw new ReportPublishError("Could not reach hq-deploy.");
234
234
  }
235
- if (!res.ok)
235
+ if (!res.ok) {
236
+ const failure = await res.json().catch(() => ({}));
237
+ if (failure?.error?.code === "SUBDOMAIN_DUPLICATE")
238
+ throw new SubdomainTakenError();
236
239
  throw new ReportPublishError("hq-deploy rejected the report deployment.");
240
+ }
237
241
  return await res.json();
238
242
  }
243
+ /** hq-deploy refused the subdomain: another app, not visible to this caller, holds it. */
244
+ export class SubdomainTakenError extends ReportPublishError {
245
+ constructor() {
246
+ super("hq-deploy rejected the report deployment.");
247
+ }
248
+ }
249
+ /**
250
+ * Subdomains to try, in order. The first is the report's own slug; the rest
251
+ * are used only when an app the caller cannot see already holds it (for
252
+ * example, one left by an earlier publish under a different account).
253
+ */
254
+ export function candidateSlugs(slug, attempts = 3) {
255
+ return [slug, ...Array.from({ length: attempts }, (_, i) => `${slug}-${i + 2}`)];
256
+ }
239
257
  export async function resolveCompanySlug(token, companyUid) {
240
258
  const res = await vaultApiFetch({ token, path: `/entity/${encodeURIComponent(companyUid)}` });
241
259
  if (!res.ok)
@@ -250,12 +268,29 @@ async function orgFor(token, report) {
250
268
  const slug = report.companyUid ? await resolveCompanySlug(token, report.companyUid) : undefined;
251
269
  return deployOrgContext(report, slug);
252
270
  }
253
- async function findOrCreateApp(token, slug, org) {
271
+ export async function findOrCreateApp(token, slug, org) {
254
272
  const listed = await deployJson(token, "/api/apps", "GET", undefined, org.headers);
255
- const existing = Array.isArray(listed.apps)
256
- ? listed.apps.find((app) => !!app && typeof app === "object" && (app.subdomain === slug || app.name === slug))
257
- : undefined;
258
- const app = existing ?? await deployJson(token, "/api/apps", "POST", { name: slug, type: "static" }, org.headers);
273
+ const apps = Array.isArray(listed.apps) ? listed.apps : [];
274
+ const mine = (name) => apps.find((app) => !!app && typeof app === "object" && (app.subdomain === name || app.name === name));
275
+ let app;
276
+ for (const name of candidateSlugs(slug)) {
277
+ const existing = mine(name);
278
+ if (existing) {
279
+ app = existing;
280
+ break;
281
+ }
282
+ try {
283
+ app = await deployJson(token, "/api/apps", "POST", { name, type: "static" }, org.headers);
284
+ break;
285
+ }
286
+ catch (error) {
287
+ if (error instanceof SubdomainTakenError)
288
+ continue;
289
+ throw error;
290
+ }
291
+ }
292
+ if (!app)
293
+ throw new ReportPublishError("Every address for this report is taken on hq-deploy.");
259
294
  const id = typeof app.id === "string" ? app.id : undefined;
260
295
  const subdomain = typeof app.subdomain === "string" ? app.subdomain : undefined;
261
296
  if (!id || !subdomain)
@@ -0,0 +1,24 @@
1
+ import type { LaneRecord, LanesConfig } from "./types.js";
2
+ export interface AdmissionHost {
3
+ hostname: string;
4
+ outpostId: string | null;
5
+ }
6
+ export type AdmissionCapReason = "provider_cap" | "account_cap" | "profile_cap";
7
+ export declare const ADMISSION_LOCK_STALE_MS = 30000;
8
+ export declare const ADMISSION_LOCK_TIMEOUT_MS = 5000;
9
+ export declare const ADMISSION_RESERVATION_GRACE_MS = 30000;
10
+ export declare function withLaneAdmissionLock<T>(hqRoot: string, provider: LaneRecord["provider"], fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
11
+ export declare function withAllLaneAdmissionLocks<T>(hqRoot: string, fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
12
+ export declare function laneRunsOnHost(lane: LaneRecord, host: AdmissionHost): boolean;
13
+ export declare function compareLaneQueueOrder(a: LaneRecord, b: LaneRecord): number;
14
+ export declare function hasActiveAdmissionReservation(lane: LaneRecord, nowMs: number): boolean;
15
+ export declare function hasActivePendingResume(lane: LaneRecord, nowMs: number): boolean;
16
+ /** Return the first capacity limit that currently prevents this lane from starting. */
17
+ export declare function laneAdmissionCapReason(lanes: LaneRecord[], candidate: LaneRecord, config: LanesConfig, host: AdmissionHost, profileMaxConcurrent?: number, nowMs?: number): AdmissionCapReason | undefined;
18
+ /**
19
+ * A prior queue entry owns a free slot only when its own provider, account,
20
+ * and profile caps allow it to use that slot. A blocked profile therefore
21
+ * cannot hold an unrelated profile's queue position.
22
+ */
23
+ export declare function hasEarlierEligibleQueuedLane(lanes: LaneRecord[], candidate: LaneRecord, config: LanesConfig, host: AdmissionHost, profileMaxFor: (lane: LaneRecord) => number | undefined, nowMs?: number): boolean;
24
+ //# sourceMappingURL=admission.d.ts.map
@@ -0,0 +1,193 @@
1
+ import * as fs from "node:fs";
2
+ import * as path from "node:path";
3
+ import { accountMaxRunning } from "./config.js";
4
+ import { LanesError } from "./errors.js";
5
+ export const ADMISSION_LOCK_STALE_MS = 30_000;
6
+ export const ADMISSION_LOCK_TIMEOUT_MS = 5_000;
7
+ export const ADMISSION_RESERVATION_GRACE_MS = 30_000;
8
+ function defaultAdmissionSleep(ms) {
9
+ return new Promise((resolve) => setTimeout(resolve, ms));
10
+ }
11
+ function admissionLockPath(hqRoot, provider) {
12
+ return path.join(hqRoot, "workspace", "lanes-runs", ".admission-locks", `${provider}-provider.lock`);
13
+ }
14
+ function admissionLockOwnerAlive(pid) {
15
+ try {
16
+ process.kill(pid, 0);
17
+ return true;
18
+ }
19
+ catch (err) {
20
+ return err.code !== "ESRCH";
21
+ }
22
+ }
23
+ async function withAdmissionLock(lockPath, fn, sleep) {
24
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
25
+ const started = Date.now();
26
+ while (true) {
27
+ let fd;
28
+ try {
29
+ fd = fs.openSync(lockPath, "wx");
30
+ const ownedStat = fs.fstatSync(fd);
31
+ try {
32
+ fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n`, "utf8"));
33
+ }
34
+ catch (err) {
35
+ fs.closeSync(fd);
36
+ fd = undefined;
37
+ throw err;
38
+ }
39
+ try {
40
+ await new Promise((resolve) => setImmediate(resolve));
41
+ return await fn();
42
+ }
43
+ finally {
44
+ try {
45
+ fs.closeSync(fd);
46
+ }
47
+ catch (err) {
48
+ process.stderr.write(`hq lanes: admission lock close failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
49
+ }
50
+ try {
51
+ const currentStat = fs.statSync(lockPath);
52
+ if (currentStat.dev === ownedStat.dev && currentStat.ino === ownedStat.ino) {
53
+ fs.unlinkSync(lockPath);
54
+ }
55
+ }
56
+ catch (err) {
57
+ if (err.code !== "ENOENT") {
58
+ process.stderr.write(`hq lanes: admission lock unlink failed for ${lockPath}: ${err instanceof Error ? err.message : String(err)}\n`);
59
+ }
60
+ }
61
+ }
62
+ }
63
+ catch (err) {
64
+ if (err.code !== "EEXIST")
65
+ throw err;
66
+ if (Date.now() - started > ADMISSION_LOCK_TIMEOUT_MS) {
67
+ throw new LanesError("store_lock_timeout", `Timed out waiting for admission lock ${lockPath}`, { path: lockPath });
68
+ }
69
+ try {
70
+ const raw = fs.readFileSync(lockPath, "utf8");
71
+ const [ownerLine, timestampLine] = raw.split("\n");
72
+ const ownerPid = ownerLine ? Number(ownerLine) : NaN;
73
+ const timestamp = timestampLine ? Number(timestampLine) : NaN;
74
+ if (Number.isSafeInteger(ownerPid) &&
75
+ ownerPid > 0 &&
76
+ Number.isFinite(timestamp) &&
77
+ Date.now() - timestamp > ADMISSION_LOCK_STALE_MS &&
78
+ !admissionLockOwnerAlive(ownerPid)) {
79
+ if (fs.readFileSync(lockPath, "utf8") !== raw)
80
+ continue;
81
+ try {
82
+ fs.unlinkSync(lockPath);
83
+ }
84
+ catch (unlinkErr) {
85
+ if (unlinkErr.code !== "ENOENT")
86
+ throw unlinkErr;
87
+ }
88
+ continue;
89
+ }
90
+ }
91
+ catch (probeErr) {
92
+ if (probeErr.code !== "ENOENT") {
93
+ process.stderr.write(`hq lanes: stale admission-lock probe failed for ${lockPath}: ${probeErr instanceof Error ? probeErr.message : String(probeErr)}\n`);
94
+ }
95
+ }
96
+ await sleep(10);
97
+ await new Promise((resolve) => setImmediate(resolve));
98
+ }
99
+ }
100
+ }
101
+ export function withLaneAdmissionLock(hqRoot, provider, fn, sleep = defaultAdmissionSleep) {
102
+ return withAdmissionLock(admissionLockPath(hqRoot, provider), fn, sleep);
103
+ }
104
+ export async function withAllLaneAdmissionLocks(hqRoot, fn, sleep = defaultAdmissionSleep) {
105
+ const providers = ["claude", "codex", "grok"];
106
+ const acquire = (index) => {
107
+ const provider = providers[index];
108
+ if (provider === undefined)
109
+ return fn();
110
+ return withLaneAdmissionLock(hqRoot, provider, () => acquire(index + 1), sleep);
111
+ };
112
+ return acquire(0);
113
+ }
114
+ export function laneRunsOnHost(lane, host) {
115
+ if (host.outpostId && lane.host.kind === "outpost") {
116
+ return lane.host.id === host.outpostId;
117
+ }
118
+ return lane.host.kind === "local" && lane.host.id === host.hostname;
119
+ }
120
+ export function compareLaneQueueOrder(a, b) {
121
+ if (b.priority !== a.priority)
122
+ return b.priority - a.priority;
123
+ const byQueueAge = Date.parse(a.queued_at ?? a.created_at) - Date.parse(b.queued_at ?? b.created_at);
124
+ return byQueueAge || a.lane_id.localeCompare(b.lane_id);
125
+ }
126
+ export function hasActiveAdmissionReservation(lane, nowMs) {
127
+ if (lane.state !== "queued" || lane.queued_reason !== undefined || !lane.admission_reserved_at) {
128
+ return false;
129
+ }
130
+ const reservedAt = Date.parse(lane.admission_reserved_at);
131
+ const ageMs = nowMs - reservedAt;
132
+ return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
133
+ }
134
+ export function hasActivePendingResume(lane, nowMs) {
135
+ const pending = lane.pending_resume;
136
+ if (!pending)
137
+ return false;
138
+ const reservedAt = Date.parse(pending.at);
139
+ const ageMs = nowMs - reservedAt;
140
+ return Number.isFinite(reservedAt) && ageMs >= 0 && ageMs < ADMISSION_RESERVATION_GRACE_MS;
141
+ }
142
+ /** Return the first capacity limit that currently prevents this lane from starting. */
143
+ export function laneAdmissionCapReason(lanes, candidate, config, host, profileMaxConcurrent, nowMs = Date.now()) {
144
+ const running = lanes.filter((lane) => lane.lane_id !== candidate.lane_id &&
145
+ (lane.state === "running" ||
146
+ hasActiveAdmissionReservation(lane, nowMs) ||
147
+ hasActivePendingResume(lane, nowMs)) &&
148
+ laneRunsOnHost(lane, host));
149
+ const providerCfg = config.providers[candidate.provider];
150
+ const providerRunning = running.filter((lane) => lane.provider === candidate.provider);
151
+ if (providerRunning.length >= providerCfg.max_running)
152
+ return "provider_cap";
153
+ if (candidate.account &&
154
+ providerRunning.filter((lane) => lane.account === candidate.account).length >=
155
+ accountMaxRunning(providerCfg, candidate.account)) {
156
+ return "account_cap";
157
+ }
158
+ if (candidate.worker_profile &&
159
+ profileMaxConcurrent !== undefined &&
160
+ running.filter((lane) => lane.worker_profile === candidate.worker_profile).length >=
161
+ profileMaxConcurrent) {
162
+ return "profile_cap";
163
+ }
164
+ return undefined;
165
+ }
166
+ /**
167
+ * A prior queue entry owns a free slot only when its own provider, account,
168
+ * and profile caps allow it to use that slot. A blocked profile therefore
169
+ * cannot hold an unrelated profile's queue position.
170
+ */
171
+ export function hasEarlierEligibleQueuedLane(lanes, candidate, config, host, profileMaxFor, nowMs = Date.now()) {
172
+ return lanes
173
+ .filter((lane) => lane.lane_id !== candidate.lane_id &&
174
+ lane.state === "queued" &&
175
+ lane.provider === candidate.provider &&
176
+ laneRunsOnHost(lane, host) &&
177
+ compareLaneQueueOrder(lane, candidate) < 0)
178
+ .some((lane) => {
179
+ const profileMaxConcurrent = profileMaxFor(lane);
180
+ const capReason = laneAdmissionCapReason(lanes, lane, config, host, profileMaxConcurrent, nowMs);
181
+ if (capReason === undefined)
182
+ return true;
183
+ if (capReason !== "account_cap" || lane.admission_accounts === undefined)
184
+ return false;
185
+ return lane.admission_accounts.some((account) => {
186
+ if (account === lane.account || config.providers[lane.provider].accounts[account]?.enabled !== true) {
187
+ return false;
188
+ }
189
+ return laneAdmissionCapReason(lanes, { ...lane, account }, config, host, profileMaxConcurrent, nowMs) === undefined;
190
+ });
191
+ });
192
+ }
193
+ //# sourceMappingURL=admission.js.map
@@ -280,9 +280,11 @@ export function resumeCommandRefusal(lane) {
280
280
  error: "lane_not_resumable",
281
281
  message: `Lane is ${lane.state}. hq lanes resume continues a lane in ${RESUME_COMMAND_STATES.join(", ")}; ${lane.state === "awaiting_input"
282
282
  ? "use hq lanes questions answer <question-id>."
283
- : lane.state === "queued" || lane.state === "running"
284
- ? "use hq lanes message for a live lane."
285
- : "a lane in this state is not resumed."}`,
283
+ : lane.state === "queued"
284
+ ? "use hq lanes admit <lane-id>."
285
+ : lane.state === "running"
286
+ ? "use hq lanes message for a live lane."
287
+ : "a lane in this state is not resumed."}`,
286
288
  };
287
289
  }
288
290
  if (!providerSupportsSessionResume(lane.provider)) {
@@ -3,6 +3,8 @@
3
3
  */
4
4
  export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
5
5
  export { MAX_LANE_DEPTH } from "./constants.js";
6
+ export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
7
+ export type { AdmissionCapReason, AdmissionHost } from "./admission.js";
6
8
  export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
7
9
  export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneEnvelope, LaneHost, LaneProvider, LaneRecord, LaneStory, LaneStoryStatus, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
8
10
  export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
@@ -3,6 +3,7 @@
3
3
  */
4
4
  export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
5
5
  export { MAX_LANE_DEPTH } from "./constants.js";
6
+ export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
6
7
  export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
7
8
  export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
8
9
  export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerAutoCompactTokens, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
@@ -8,14 +8,16 @@
8
8
  import * as fs from "node:fs";
9
9
  import * as path from "node:path";
10
10
  import { parseUserYaml } from "../../utils/user-yaml-error.js";
11
- import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
11
+ import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
12
+ import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
13
+ import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
12
14
  import { MAX_LANE_DEPTH } from "./constants.js";
13
- import { LanesError, isLaneVersionConflict } from "./errors.js";
15
+ import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
14
16
  import { loadWorkerProfile } from "./profile.js";
15
17
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
16
18
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
17
19
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
18
- import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
20
+ import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
19
21
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
20
22
  import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
21
23
  import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
@@ -88,10 +90,10 @@ export function timeoutPath(hqRoot, laneId) {
88
90
  return path.join(laneRunDir(hqRoot, laneId), "timeout.json");
89
91
  }
90
92
  export function laneRunsHere(lane, host) {
91
- if (host.outpostId && lane.host.kind === "outpost") {
92
- return lane.host.id === host.outpostId;
93
- }
94
- return lane.host.kind === "local" && lane.host.id === host.hostname;
93
+ return laneRunsOnHost(lane, host);
94
+ }
95
+ function hasFreshAdmissionReservation(lane, nowMs) {
96
+ return hasActiveAdmissionReservation(lane, nowMs);
95
97
  }
96
98
  export function agedAdvanceCount(question) {
97
99
  return question.routing_history.filter((entry) => entry.reason === "unanswered_timeout").length;
@@ -163,11 +165,6 @@ export function isDeadlinePassed(deadlineAt, nowMs) {
163
165
  return false;
164
166
  return nowMs > at;
165
167
  }
166
- function compareQueueOrder(a, b) {
167
- if (b.priority !== a.priority)
168
- return b.priority - a.priority;
169
- return a.created_at.localeCompare(b.created_at);
170
- }
171
168
  function readDirJsonIds(dir, label) {
172
169
  let names;
173
170
  try {
@@ -308,6 +305,53 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
308
305
  }
309
306
  throw new LanesError("version_conflict", `Exhausted CAS retries for lane ${laneId}`, { lane_id: laneId });
310
307
  }
308
+ function writeQueueReason(hqRoot, lane, reason, dryRun) {
309
+ if (lane.queued_reason === reason)
310
+ return;
311
+ casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, admission_reserved_at: undefined }), dryRun);
312
+ }
313
+ function availableBalancedAccount(hqRoot, lane, occupying, config, host, profileMaxConcurrent, nowMs) {
314
+ if (!lane.account || !lane.admission_accounts?.includes(lane.account))
315
+ return undefined;
316
+ if (laneAdmissionCapReason(occupying, lane, config, host, profileMaxConcurrent, nowMs) !== "account_cap") {
317
+ return undefined;
318
+ }
319
+ const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
320
+ ? lane.model
321
+ : lane.model_source === "profile"
322
+ ? lane.model
323
+ : undefined;
324
+ for (const account of lane.admission_accounts) {
325
+ if (account === lane.account)
326
+ continue;
327
+ if (config.providers[lane.provider].accounts[account]?.enabled !== true)
328
+ continue;
329
+ const status = accountStatus(lane.provider, account, config);
330
+ if (!isCredentialProven(status))
331
+ continue;
332
+ try {
333
+ resolveAccount(lane.provider, account, config, {
334
+ hqRoot,
335
+ ...(pinnedModel !== undefined ? { model: pinnedModel } : {}),
336
+ });
337
+ }
338
+ catch (err) {
339
+ if (err instanceof AccountResolutionError)
340
+ continue;
341
+ throw err;
342
+ }
343
+ const candidate = { ...lane, account };
344
+ if (laneAdmissionCapReason(occupying, candidate, config, host, profileMaxConcurrent, nowMs) === undefined) {
345
+ return account;
346
+ }
347
+ }
348
+ return undefined;
349
+ }
350
+ function writeAdmissionAccount(hqRoot, lane, account, dryRun) {
351
+ return casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
352
+ current.account === lane.account &&
353
+ current.admission_accounts?.includes(account) === true, (current) => ({ ...current, account }), dryRun).record;
354
+ }
311
355
  function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
312
356
  for (let i = 0; i < 5; i += 1) {
313
357
  const current = readQuestion(hqRoot, questionId);
@@ -707,10 +751,12 @@ export async function runPassBody(hqRoot, deps) {
707
751
  });
708
752
  }
709
753
  }
710
- const freshLanes = loadLaneRecords(hqRoot, summary);
711
- const freshQuestions = loadQuestionRecords(hqRoot, summary);
712
- await admitQueued(hqRoot, freshLanes, summary, deps, now);
713
- await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
754
+ await withAllLaneAdmissionLocks(hqRoot, async () => {
755
+ const freshLanes = loadLaneRecords(hqRoot, summary);
756
+ const freshQuestions = loadQuestionRecords(hqRoot, summary);
757
+ await admitQueued(hqRoot, freshLanes, summary, deps, now);
758
+ await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
759
+ });
714
760
  if (!deps.dryRun && summary.errors.length === 0) {
715
761
  try {
716
762
  writeHeartbeatFile(hqRoot, summary, nowIso);
@@ -1005,7 +1051,8 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1005
1051
  .filter((lane) => lane.state === "queued" &&
1006
1052
  lane.pickup_validation_pending !== true &&
1007
1053
  laneRunsHere(lane, deps.host))
1008
- .sort(compareQueueOrder);
1054
+ .filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
1055
+ .sort(compareLaneQueueOrder);
1009
1056
  if (localQueued.length === 0)
1010
1057
  return;
1011
1058
  const config = loadLanesConfig(hqRoot);
@@ -1030,31 +1077,37 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1030
1077
  });
1031
1078
  const spawnFn = deps.spawn ??
1032
1079
  ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
1033
- const running = lanes.filter((lane) => lane.state === "running");
1080
+ const running = lanes.filter((lane) => (lane.state === "running" ||
1081
+ hasFreshAdmissionReservation(lane, now.getTime()) ||
1082
+ hasActivePendingResume(lane, now.getTime())) &&
1083
+ laneRunsHere(lane, deps.host));
1034
1084
  const starvation = decision.admit && decision.reason === "starvation_bound";
1035
- const candidates = starvation ? localQueued.slice(0, 1) : localQueued;
1036
1085
  const spawned = [];
1037
1086
  if (!decision.admit) {
1087
+ for (const lane of localQueued) {
1088
+ let profileMaxConcurrent;
1089
+ if (lane.worker_profile !== undefined) {
1090
+ try {
1091
+ profileMaxConcurrent = loadWorkerProfile(hqRoot, lane.worker_profile).meta.max_concurrent;
1092
+ }
1093
+ catch (err) {
1094
+ markDirty(summary, {
1095
+ kind: "lane",
1096
+ id: lane.lane_id,
1097
+ why: `profile admission failed: ${err instanceof Error ? err.message : String(err)}`,
1098
+ });
1099
+ continue;
1100
+ }
1101
+ }
1102
+ const capReason = laneAdmissionCapReason(running, lane, config, deps.host, profileMaxConcurrent, now.getTime());
1103
+ writeQueueReason(hqRoot, lane, capReason ?? "throttle", deps.dryRun);
1104
+ }
1038
1105
  if (!deps.dryRun && throttleState !== "unreadable") {
1039
1106
  writeThrottleState(filePath, applyThrottleDecision(throttleState, decision, nowMs), throttleDeps);
1040
1107
  }
1041
1108
  return;
1042
1109
  }
1043
- for (const lane of candidates) {
1044
- const provider = lane.provider;
1045
- const providerCfg = config.providers[provider];
1046
- const providerRunning = running.filter((row) => row.provider === provider).length +
1047
- spawned.filter((row) => row.provider === provider).length;
1048
- if (providerRunning >= providerCfg.max_running && !starvation)
1049
- continue;
1050
- if (lane.account) {
1051
- const accountRunning = running.filter((row) => row.provider === provider && row.account === lane.account).length +
1052
- spawned.filter((row) => row.provider === provider && row.account === lane.account).length;
1053
- if (accountRunning >= accountMaxRunning(providerCfg, lane.account) &&
1054
- !starvation) {
1055
- continue;
1056
- }
1057
- }
1110
+ for (const lane of localQueued) {
1058
1111
  let profileMaxConcurrent;
1059
1112
  if (lane.worker_profile !== undefined) {
1060
1113
  try {
@@ -1069,11 +1122,22 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1069
1122
  continue;
1070
1123
  }
1071
1124
  }
1072
- if (profileMaxConcurrent !== undefined) {
1073
- const profileRunning = running.filter((row) => row.worker_profile === lane.worker_profile).length +
1074
- spawned.filter((row) => row.worker_profile === lane.worker_profile).length;
1075
- if (profileRunning >= profileMaxConcurrent)
1076
- continue;
1125
+ const projected = [...running, ...spawned];
1126
+ const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
1127
+ const admittedLane = alternateAccount !== undefined
1128
+ ? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
1129
+ : lane;
1130
+ const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
1131
+ const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
1132
+ let blockedBy;
1133
+ if (currentCap !== undefined) {
1134
+ blockedBy = baseCap ?? "queue_position";
1135
+ }
1136
+ if (!blockedBy && !decision.admit)
1137
+ blockedBy = "throttle";
1138
+ if (blockedBy) {
1139
+ writeQueueReason(hqRoot, admittedLane, blockedBy, deps.dryRun);
1140
+ continue;
1077
1141
  }
1078
1142
  try {
1079
1143
  if (deps.dryRun) {
@@ -1083,12 +1147,12 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1083
1147
  to: "running",
1084
1148
  reason: decision.reason,
1085
1149
  });
1086
- spawned.push({ ...lane, state: "running" });
1150
+ spawned.push({ ...admittedLane, state: "running" });
1087
1151
  if (starvation)
1088
1152
  break;
1089
1153
  continue;
1090
1154
  }
1091
- const started = spawnFn(hqRoot, lane);
1155
+ const started = spawnFn(hqRoot, admittedLane);
1092
1156
  if (decision.reason === "starvation_bound") {
1093
1157
  casLaneIf(hqRoot, lane.lane_id, (current) => current.admitted_reason !== "starvation_bound", (current) => ({ ...current, admitted_reason: "starvation_bound" }), false);
1094
1158
  }
@@ -1127,25 +1191,73 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1127
1191
  }
1128
1192
  }
1129
1193
  async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
1130
- const spawnFn = deps.spawn ??
1131
- ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
1194
+ const config = loadLanesConfig(hqRoot);
1132
1195
  const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" && laneRunsHere(lane, deps.host));
1196
+ if (awaiting.length === 0)
1197
+ return;
1198
+ const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
1199
+ let throttleDecision;
1200
+ if (deps.dryRun) {
1201
+ const loaded = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
1202
+ const state = loaded === "missing" ? emptyThrottleState() : loaded;
1203
+ throttleDecision = evaluateThrottle({
1204
+ busy: await resolveCpuBusyFraction(throttleDeps),
1205
+ availableMemMb: readAvailableMemMb(throttleDeps),
1206
+ cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
1207
+ state,
1208
+ throttle: config.throttle,
1209
+ nowMs: deps.now().getTime(),
1210
+ });
1211
+ }
1212
+ else {
1213
+ const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
1214
+ throttleDecision = throttle.decision;
1215
+ }
1216
+ if (!throttleDecision.admit)
1217
+ return;
1218
+ const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
1219
+ const nowMs = deps.now().getTime();
1220
+ const capacityLanes = lanes.map((lane) => {
1221
+ try {
1222
+ return readLane(hqRoot, lane.lane_id);
1223
+ }
1224
+ catch {
1225
+ return lane;
1226
+ }
1227
+ });
1228
+ const running = capacityLanes.filter((lane) => (lane.state === "running" ||
1229
+ hasFreshAdmissionReservation(lane, nowMs) ||
1230
+ hasActivePendingResume(lane, nowMs)) &&
1231
+ laneRunsHere(lane, deps.host));
1133
1232
  for (const lane of awaiting) {
1233
+ const current = readLane(hqRoot, lane.lane_id);
1234
+ if (current.state !== "awaiting_input")
1235
+ continue;
1134
1236
  const pending = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "pending");
1135
1237
  const answered = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "answered");
1136
1238
  if (pending.length > 0 || answered.length === 0)
1137
1239
  continue;
1138
1240
  try {
1241
+ let profileMaxConcurrent;
1242
+ if (current.worker_profile !== undefined) {
1243
+ profileMaxConcurrent = loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent;
1244
+ }
1245
+ if (laneAdmissionCapReason(running, current, config, deps.host, profileMaxConcurrent, nowMs) !== undefined) {
1246
+ continue;
1247
+ }
1139
1248
  if (deps.dryRun) {
1140
1249
  summary.resumed.push({
1141
1250
  id: lane.lane_id,
1142
1251
  from: "awaiting_input",
1143
1252
  to: "running",
1144
1253
  });
1254
+ running.push({ ...current, state: "running" });
1145
1255
  continue;
1146
1256
  }
1147
1257
  writeAnsweredQuestions(hqRoot, lane, questions);
1148
1258
  const started = spawnFn(hqRoot, lane);
1259
+ if (started.state === "running")
1260
+ running.push(started);
1149
1261
  summary.resumed.push({
1150
1262
  id: started.lane_id,
1151
1263
  from: "awaiting_input",