@indigoai-us/hq-cli 5.342.14 → 5.342.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/commands/integrations-core.d.ts +5 -4
- package/dist/commands/integrations-core.js +64 -10
- package/dist/commands/lanes-accounts.js +8 -0
- package/dist/lib/lanes/admission.d.ts +7 -1
- package/dist/lib/lanes/admission.js +34 -0
- package/dist/lib/lanes/config.js +11 -0
- package/dist/lib/lanes/flag-keys.d.ts +2 -0
- package/dist/lib/lanes/flag-keys.js +2 -0
- package/dist/lib/lanes/reconcile.js +52 -6
- package/dist/lib/lanes/types.d.ts +3 -1
- package/dist/lib/lanes/types.js +1 -0
- package/dist/main.js +16 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.342.15] — 2026-10-04
|
|
6
|
+
- Preserve bounded integration subcommand and RPC error context on asynchronous gateway failures, and retain diagnostics for malformed HTTP 200 gateway errors (HQ-CLI-C8/C9).
|
|
7
|
+
- Claude admission can prioritize lanes that are seniors of live lanes and keep configurable per-host capacity reserved for them when `lanes.lead-admission-priority` is enabled.
|
|
8
|
+
|
|
5
9
|
## [5.342.14] — 2026-10-04
|
|
6
10
|
- Fixed native HQ subprocess launching and job-lock command resolution on Windows.
|
|
7
11
|
- Fixed Windows delegate/job shell parity while preserving embedded carriage returns and POSIX command/path bytes.
|
|
@@ -110,11 +110,9 @@ export interface AdminSurface {
|
|
|
110
110
|
}
|
|
111
111
|
interface GatewayMessage {
|
|
112
112
|
result?: unknown;
|
|
113
|
-
error?:
|
|
114
|
-
code?: number;
|
|
115
|
-
message?: string;
|
|
116
|
-
};
|
|
113
|
+
error?: unknown;
|
|
117
114
|
}
|
|
115
|
+
export type GatewayErrorShape = "string_error" | "object_no_code" | "object_code" | "string_code" | "nested_code" | "empty" | "redacted_empty";
|
|
118
116
|
export declare class IntegrationsCliError extends Error {
|
|
119
117
|
/**
|
|
120
118
|
* True when the error is the caller's request/state/permission (a client 4xx
|
|
@@ -152,6 +150,8 @@ export declare class IntegrationsCliError extends Error {
|
|
|
152
150
|
readonly rpcCode?: number;
|
|
153
151
|
/** Fixed vocabulary for the integrations subcommand that called the gateway. */
|
|
154
152
|
readonly gatewaySubcommand?: "tools_list" | "tools_call" | "other";
|
|
153
|
+
/** Fixed vocabulary describing the shape of an upstream JSON-RPC error. */
|
|
154
|
+
readonly gatewayErrorShape?: GatewayErrorShape;
|
|
155
155
|
/**
|
|
156
156
|
* HQ's own OAuth callback URL, carried on the errors that say a provider
|
|
157
157
|
* refused to register HQ automatically. The provider must register the exact
|
|
@@ -167,6 +167,7 @@ export declare class IntegrationsCliError extends Error {
|
|
|
167
167
|
oauthProtected?: boolean;
|
|
168
168
|
rpcCode?: number;
|
|
169
169
|
gatewaySubcommand?: "tools_list" | "tools_call" | "other";
|
|
170
|
+
gatewayErrorShape?: GatewayErrorShape;
|
|
170
171
|
redirectUri?: string;
|
|
171
172
|
});
|
|
172
173
|
}
|
|
@@ -57,6 +57,8 @@ export class IntegrationsCliError extends Error {
|
|
|
57
57
|
rpcCode;
|
|
58
58
|
/** Fixed vocabulary for the integrations subcommand that called the gateway. */
|
|
59
59
|
gatewaySubcommand;
|
|
60
|
+
/** Fixed vocabulary describing the shape of an upstream JSON-RPC error. */
|
|
61
|
+
gatewayErrorShape;
|
|
60
62
|
/**
|
|
61
63
|
* HQ's own OAuth callback URL, carried on the errors that say a provider
|
|
62
64
|
* refused to register HQ automatically. The provider must register the exact
|
|
@@ -79,6 +81,8 @@ export class IntegrationsCliError extends Error {
|
|
|
79
81
|
this.rpcCode = opts.rpcCode;
|
|
80
82
|
if (opts.gatewaySubcommand !== undefined)
|
|
81
83
|
this.gatewaySubcommand = opts.gatewaySubcommand;
|
|
84
|
+
if (opts.gatewayErrorShape !== undefined)
|
|
85
|
+
this.gatewayErrorShape = opts.gatewayErrorShape;
|
|
82
86
|
if (opts.redirectUri !== undefined) {
|
|
83
87
|
Object.defineProperty(this, "redirectUri", {
|
|
84
88
|
value: opts.redirectUri,
|
|
@@ -191,6 +195,54 @@ function expectedGatewayErrorFallback(code) {
|
|
|
191
195
|
return undefined;
|
|
192
196
|
}
|
|
193
197
|
}
|
|
198
|
+
function recordValue(value) {
|
|
199
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
200
|
+
? value
|
|
201
|
+
: undefined;
|
|
202
|
+
}
|
|
203
|
+
function numericGatewayCode(value) {
|
|
204
|
+
if (typeof value === "number" && Number.isSafeInteger(value))
|
|
205
|
+
return value;
|
|
206
|
+
if (typeof value === "string" && /^-?\d+$/.test(value.trim())) {
|
|
207
|
+
const parsed = Number(value.trim());
|
|
208
|
+
if (Number.isSafeInteger(parsed))
|
|
209
|
+
return parsed;
|
|
210
|
+
}
|
|
211
|
+
return undefined;
|
|
212
|
+
}
|
|
213
|
+
/** Read the known JSON-RPC error fields without stringifying arbitrary payload data. */
|
|
214
|
+
function gatewayErrorDetails(error) {
|
|
215
|
+
if (typeof error === "string") {
|
|
216
|
+
const message = redactErrorText(error);
|
|
217
|
+
return {
|
|
218
|
+
message,
|
|
219
|
+
shape: message ? "string_error" : "redacted_empty",
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
const record = recordValue(error);
|
|
223
|
+
if (!record)
|
|
224
|
+
return { message: "", shape: "empty" };
|
|
225
|
+
const data = recordValue(record.data);
|
|
226
|
+
const rawMessage = typeof record.message === "string" ? record.message : undefined;
|
|
227
|
+
const message = rawMessage === undefined ? "" : redactErrorText(rawMessage);
|
|
228
|
+
const directCode = record.code;
|
|
229
|
+
const nestedCode = data?.code;
|
|
230
|
+
const directRpcCode = numericGatewayCode(directCode);
|
|
231
|
+
const nestedRpcCode = numericGatewayCode(nestedCode);
|
|
232
|
+
const rpcCode = directRpcCode ?? nestedRpcCode;
|
|
233
|
+
const hasStringCode = typeof directCode === "string" || typeof nestedCode === "string";
|
|
234
|
+
const redactedEmptyMessage = rawMessage !== undefined && rawMessage.length > 0 && !message;
|
|
235
|
+
const shape = redactedEmptyMessage
|
|
236
|
+
? "redacted_empty"
|
|
237
|
+
: nestedCode !== undefined
|
|
238
|
+
? "nested_code"
|
|
239
|
+
: hasStringCode
|
|
240
|
+
? "string_code"
|
|
241
|
+
: directCode !== undefined
|
|
242
|
+
? "object_code"
|
|
243
|
+
: message ? "object_no_code" : "empty";
|
|
244
|
+
return { message, ...(rpcCode === undefined ? {} : { rpcCode }), shape };
|
|
245
|
+
}
|
|
194
246
|
// A 401 from ANY integration-gateway vault call means the caller's HQ session
|
|
195
247
|
// is expired or missing — an expected auth state fixed by `hq login`, not an
|
|
196
248
|
// hq-cli defect. Raise the same typed AuthError the vault company-resolution
|
|
@@ -446,16 +498,16 @@ export async function callGateway(token, params, idempotencyKey) {
|
|
|
446
498
|
const accepted = message;
|
|
447
499
|
const jobId = accepted && typeof accepted.jobId === "string" ? accepted.jobId : "";
|
|
448
500
|
if (accepted?.status !== "pending" || !/^imj_[A-Za-z0-9_-]+$/.test(jobId)) {
|
|
449
|
-
throw new IntegrationsCliError(`Integration gateway returned an invalid async job response (job id: ${jobId || "missing"}).`, { expected: false, status: 202 });
|
|
501
|
+
throw new IntegrationsCliError(`Integration gateway returned an invalid async job response (job id: ${jobId || "missing"}).`, { expected: false, status: 202, gatewaySubcommand });
|
|
450
502
|
}
|
|
451
503
|
const companyUid = typeof params.companyUid === "string" ? params.companyUid : "";
|
|
452
504
|
if (!companyUid) {
|
|
453
505
|
throw new IntegrationsCliError(`Cannot poll integration job ${jobId}: the company id is missing.`, { expected: true });
|
|
454
506
|
}
|
|
455
507
|
if (!idempotencyKey) {
|
|
456
|
-
throw new IntegrationsCliError(`Integration gateway accepted async job ${jobId} without an idempotency key.`, { expected: false, status: 202 });
|
|
508
|
+
throw new IntegrationsCliError(`Integration gateway accepted async job ${jobId} without an idempotency key.`, { expected: false, status: 202, gatewaySubcommand });
|
|
457
509
|
}
|
|
458
|
-
const terminal = await pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, deadline);
|
|
510
|
+
const terminal = await pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, deadline, gatewaySubcommand);
|
|
459
511
|
terminalMessage = terminal.message;
|
|
460
512
|
terminalStatus = terminal.status;
|
|
461
513
|
}
|
|
@@ -470,6 +522,7 @@ export async function callGateway(token, params, idempotencyKey) {
|
|
|
470
522
|
throw new IntegrationsCliError(`Integration gateway request failed (HTTP ${res.status}).`, { expected: isClientError(res.status), status: res.status, gatewaySubcommand });
|
|
471
523
|
}
|
|
472
524
|
if (terminalMessage?.error) {
|
|
525
|
+
const gatewayError = gatewayErrorDetails(terminalMessage.error);
|
|
473
526
|
// The gateway's error text is minted UPSTREAM (hq-pro, the integration
|
|
474
527
|
// factory, and beyond it the third-party provider), so it is untrusted:
|
|
475
528
|
// scrub credentials and bound the length here, at the throw site, because
|
|
@@ -478,21 +531,22 @@ export async function callGateway(token, params, idempotencyKey) {
|
|
|
478
531
|
// The chain is idempotent, so the unexpected path scrubbing again is a
|
|
479
532
|
// no-op. This preserves PR #298's user-visible diagnostic; it only makes
|
|
480
533
|
// it safe on the newly-expected path.
|
|
481
|
-
throw new IntegrationsCliError(
|
|
482
|
-
expectedGatewayErrorFallback(
|
|
534
|
+
throw new IntegrationsCliError(gatewayError.message ||
|
|
535
|
+
expectedGatewayErrorFallback(gatewayError.rpcCode) ||
|
|
483
536
|
"Integration gateway returned an error.", {
|
|
484
|
-
expected: isExpectedGatewayError(
|
|
537
|
+
expected: isExpectedGatewayError(gatewayError.rpcCode),
|
|
485
538
|
status: terminalStatus,
|
|
486
539
|
gatewaySubcommand,
|
|
540
|
+
gatewayErrorShape: gatewayError.shape,
|
|
487
541
|
// Carry the protocol code to the capture site as a bounded grouping
|
|
488
542
|
// discriminator (HQ-CLI collision, Sentry 7642756130). This does not
|
|
489
543
|
// alter the `expected` decision above; it only labels the error.
|
|
490
|
-
...(
|
|
544
|
+
...(gatewayError.rpcCode === undefined ? {} : { rpcCode: gatewayError.rpcCode }),
|
|
491
545
|
});
|
|
492
546
|
}
|
|
493
547
|
return terminalMessage;
|
|
494
548
|
}
|
|
495
|
-
async function pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, deadline) {
|
|
549
|
+
async function pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, deadline, gatewaySubcommand) {
|
|
496
550
|
const jobPath = `/v1/integrations/mcp/jobs/${encodeURIComponent(jobId)}`;
|
|
497
551
|
while (Date.now() < deadline) {
|
|
498
552
|
const requestRemainingMs = deadline - Date.now();
|
|
@@ -526,7 +580,7 @@ async function pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, dea
|
|
|
526
580
|
if (res.status === 404) {
|
|
527
581
|
throw new IntegrationsCliError(`Integration job ${jobId} was not found or is no longer available.`, { expected: true, status: 404 });
|
|
528
582
|
}
|
|
529
|
-
throw new IntegrationsCliError(`Integration job ${jobId} polling failed (HTTP ${res.status}).`, { expected: isClientError(res.status), status: res.status });
|
|
583
|
+
throw new IntegrationsCliError(`Integration job ${jobId} polling failed (HTTP ${res.status}).`, { expected: isClientError(res.status), status: res.status, gatewaySubcommand });
|
|
530
584
|
}
|
|
531
585
|
if (!poll || typeof poll !== "object") {
|
|
532
586
|
throw new IntegrationsCliError(`Integration job ${jobId} returned an invalid poll response.`, { expected: true });
|
|
@@ -549,7 +603,7 @@ async function pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, dea
|
|
|
549
603
|
hasResult !== hasError &&
|
|
550
604
|
(!hasError || validError);
|
|
551
605
|
if (!hasJsonRpcOutcome) {
|
|
552
|
-
throw new IntegrationsCliError(`Integration job ${jobId} returned an invalid JSON-RPC response.`, { expected: false });
|
|
606
|
+
throw new IntegrationsCliError(`Integration job ${jobId} returned an invalid JSON-RPC response.`, { expected: false, gatewaySubcommand });
|
|
553
607
|
}
|
|
554
608
|
return { message: response, status: res.status };
|
|
555
609
|
}
|
|
@@ -10,6 +10,7 @@ import { resolveLaneEffort } from "../lib/lanes/effort.js";
|
|
|
10
10
|
import { LanesError } from "../lib/lanes/errors.js";
|
|
11
11
|
import { accountQuotaView, clearAccountQuota, loadQuotaState } from "../lib/lanes/quota.js";
|
|
12
12
|
import { resolveLaneModel } from "../lib/lanes/spawn.js";
|
|
13
|
+
import { leadLaneIds } from "../lib/lanes/admission.js";
|
|
13
14
|
export function createLaneAccountsHandlers(deps) {
|
|
14
15
|
const { automaticBalanceMoveHistoryReason, ensureDir, laneWithRecoveredProviderSession, loadClaudeBalanceHostSnapshots, parseRemoteJson, prepareAccountMove, printJson, reconcileHostFromCtx, resolveCallerPath } = deps;
|
|
15
16
|
const VENDOR_LOGIN = {
|
|
@@ -269,6 +270,13 @@ export function createLaneAccountsHandlers(deps) {
|
|
|
269
270
|
balance_gate_enabled: balanceGateEnabled,
|
|
270
271
|
accounts,
|
|
271
272
|
selections,
|
|
273
|
+
lead_reserve_per_host: config.lead_reserve_per_host,
|
|
274
|
+
queued_leads: loaded.snapshots.flatMap((host) => {
|
|
275
|
+
const leads = leadLaneIds(host.lanes);
|
|
276
|
+
return host.lanes
|
|
277
|
+
.filter((lane) => lane.provider === "claude" && lane.state === "queued" && leads.has(lane.lane_id))
|
|
278
|
+
.map((lane) => ({ lane_id: lane.lane_id, host: host.capacity.host, queued_reason: lane.queued_reason ?? null }));
|
|
279
|
+
}),
|
|
272
280
|
unavailable_hosts: loaded.unavailableHosts,
|
|
273
281
|
suggestions,
|
|
274
282
|
...(opts.apply ? { moved, skipped } : {}),
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type QuotaState } from "./quota.js";
|
|
2
|
+
import { type LaneRecord, type LanesConfig } from "./types.js";
|
|
2
3
|
export interface AdmissionHost {
|
|
3
4
|
hostname: string;
|
|
4
5
|
outpostId: string | null;
|
|
@@ -11,6 +12,11 @@ export declare function withLaneAdmissionLock<T>(hqRoot: string, provider: LaneR
|
|
|
11
12
|
export declare function withAllLaneAdmissionLocks<T>(hqRoot: string, fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
|
|
12
13
|
export declare function laneRunsOnHost(lane: LaneRecord, host: AdmissionHost): boolean;
|
|
13
14
|
export declare function compareLaneQueueOrder(a: LaneRecord, b: LaneRecord): number;
|
|
15
|
+
/** A lead is identified by another live lane naming it as its lane senior. */
|
|
16
|
+
export declare function leadLaneIds(lanes: readonly LaneRecord[]): Set<string>;
|
|
17
|
+
export declare function compareLaneQueueOrderWithLeads(a: LaneRecord, b: LaneRecord, leads: ReadonlySet<string>): number;
|
|
18
|
+
/** Capacity remaining across enabled Claude accounts on one host. */
|
|
19
|
+
export declare function freeClaudeAccountSlots(lanes: readonly LaneRecord[], config: LanesConfig, host: AdmissionHost, nowMs?: number, quotaState?: QuotaState): number;
|
|
14
20
|
export declare function hasActiveAdmissionReservation(lane: LaneRecord, nowMs: number): boolean;
|
|
15
21
|
export declare function hasActivePendingResume(lane: LaneRecord, nowMs: number): boolean;
|
|
16
22
|
/** Whether a lane currently owns a running slot for its host/provider caps. */
|
|
@@ -2,6 +2,8 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { accountMaxRunning } from "./config.js";
|
|
4
4
|
import { LanesError } from "./errors.js";
|
|
5
|
+
import { quotaBlockFor } from "./quota.js";
|
|
6
|
+
import { isTerminalLaneState } from "./types.js";
|
|
5
7
|
export const ADMISSION_LOCK_STALE_MS = 30_000;
|
|
6
8
|
export const ADMISSION_LOCK_TIMEOUT_MS = 5_000;
|
|
7
9
|
export const ADMISSION_RESERVATION_GRACE_MS = 30_000;
|
|
@@ -123,6 +125,38 @@ export function compareLaneQueueOrder(a, b) {
|
|
|
123
125
|
const byQueueAge = Date.parse(a.queued_at ?? a.created_at) - Date.parse(b.queued_at ?? b.created_at);
|
|
124
126
|
return byQueueAge || a.lane_id.localeCompare(b.lane_id);
|
|
125
127
|
}
|
|
128
|
+
/** A lead is identified by another live lane naming it as its lane senior. */
|
|
129
|
+
export function leadLaneIds(lanes) {
|
|
130
|
+
const leads = new Set(lanes.filter((lane) => lane.keep_alive === true).map((lane) => lane.lane_id));
|
|
131
|
+
for (const lane of lanes) {
|
|
132
|
+
if (!isTerminalLaneState(lane.state) && lane.senior.kind === "lane" && lane.senior.id) {
|
|
133
|
+
leads.add(lane.senior.id);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return leads;
|
|
137
|
+
}
|
|
138
|
+
export function compareLaneQueueOrderWithLeads(a, b, leads) {
|
|
139
|
+
if (a.provider === "claude" && b.provider === "claude") {
|
|
140
|
+
const aLead = leads.has(a.lane_id);
|
|
141
|
+
const bLead = leads.has(b.lane_id);
|
|
142
|
+
if (aLead !== bLead)
|
|
143
|
+
return aLead ? -1 : 1;
|
|
144
|
+
}
|
|
145
|
+
return compareLaneQueueOrder(a, b);
|
|
146
|
+
}
|
|
147
|
+
/** Capacity remaining across enabled Claude accounts on one host. */
|
|
148
|
+
export function freeClaudeAccountSlots(lanes, config, host, nowMs = Date.now(), quotaState) {
|
|
149
|
+
return Object.entries(config.providers.claude.accounts).reduce((total, [account, accountConfig]) => {
|
|
150
|
+
if (accountConfig.enabled !== true)
|
|
151
|
+
return total;
|
|
152
|
+
const model = accountConfig.model ?? config.providers.claude.model;
|
|
153
|
+
if (quotaState && quotaBlockFor(quotaState, "claude", account, model, new Date(nowMs)).blocked)
|
|
154
|
+
return total;
|
|
155
|
+
const occupied = lanes.filter((lane) => lane.provider === "claude" && lane.account === account &&
|
|
156
|
+
laneOccupiesRunningSlot(lane, nowMs) && laneRunsOnHost(lane, host)).length;
|
|
157
|
+
return total + Math.max(0, accountMaxRunning(config.providers.claude, account) - occupied);
|
|
158
|
+
}, 0);
|
|
159
|
+
}
|
|
126
160
|
export function hasActiveAdmissionReservation(lane, nowMs) {
|
|
127
161
|
if (lane.state !== "queued" || lane.queued_reason !== undefined || !lane.admission_reserved_at) {
|
|
128
162
|
return false;
|
package/dist/lib/lanes/config.js
CHANGED
|
@@ -152,6 +152,14 @@ function positiveInt(value, field, fallback) {
|
|
|
152
152
|
}
|
|
153
153
|
return value;
|
|
154
154
|
}
|
|
155
|
+
function nonNegativeInt(value, field, fallback) {
|
|
156
|
+
if (value === undefined)
|
|
157
|
+
return fallback;
|
|
158
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
159
|
+
throw new LanesConfigError("invalid_config", `${field} must be a non-negative integer`, { field, value });
|
|
160
|
+
}
|
|
161
|
+
return value;
|
|
162
|
+
}
|
|
155
163
|
function positiveNumber(value, field, fallback) {
|
|
156
164
|
if (value === undefined)
|
|
157
165
|
return fallback;
|
|
@@ -459,6 +467,7 @@ export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
|
459
467
|
tracker: { checks: { ...DEFAULT_TRACKER_CHECKS } },
|
|
460
468
|
runaway_search_guard: { ...DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG },
|
|
461
469
|
low_disk_min_free_bytes: DEFAULT_LOW_DISK_MIN_FREE_BYTES,
|
|
470
|
+
lead_reserve_per_host: 1,
|
|
462
471
|
codex_rollout_compress_after_hours: DEFAULT_CODEX_ROLLOUT_COMPRESS_AFTER_HOURS,
|
|
463
472
|
codex_rollout_delete_after_days: DEFAULT_CODEX_ROLLOUT_DELETE_AFTER_DAYS,
|
|
464
473
|
session_question_timeout_secs: DEFAULT_SESSION_QUESTION_TIMEOUT_SECS,
|
|
@@ -499,6 +508,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
499
508
|
}
|
|
500
509
|
const compressAfterHours = positiveInt(raw.codex_rollout_compress_after_hours, "codex_rollout_compress_after_hours", DEFAULT_CODEX_ROLLOUT_COMPRESS_AFTER_HOURS);
|
|
501
510
|
const deleteAfterDays = positiveInt(raw.codex_rollout_delete_after_days, "codex_rollout_delete_after_days", DEFAULT_CODEX_ROLLOUT_DELETE_AFTER_DAYS);
|
|
511
|
+
const leadReservePerHost = nonNegativeInt(raw.lead_reserve_per_host, "lead_reserve_per_host", 1);
|
|
502
512
|
if (deleteAfterDays * 24 < compressAfterHours) {
|
|
503
513
|
throw new LanesConfigError("invalid_config", "codex_rollout_delete_after_days must be at least codex_rollout_compress_after_hours / 24", { codex_rollout_compress_after_hours: compressAfterHours, codex_rollout_delete_after_days: deleteAfterDays });
|
|
504
514
|
}
|
|
@@ -512,6 +522,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
512
522
|
tracker: { checks: parseTrackerChecks(raw.tracker) },
|
|
513
523
|
runaway_search_guard: parseRunawaySearchGuard(raw.runaway_search_guard),
|
|
514
524
|
low_disk_min_free_bytes: positiveInt(raw.low_disk_min_free_bytes, "low_disk_min_free_bytes", DEFAULT_LOW_DISK_MIN_FREE_BYTES),
|
|
525
|
+
lead_reserve_per_host: leadReservePerHost,
|
|
515
526
|
codex_rollout_compress_after_hours: compressAfterHours,
|
|
516
527
|
codex_rollout_delete_after_days: deleteAfterDays,
|
|
517
528
|
session_question_timeout_secs: positiveInt(raw.session_question_timeout_secs, "session_question_timeout_secs", DEFAULT_SESSION_QUESTION_TIMEOUT_SECS),
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
2
|
export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
3
|
+
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
|
+
export declare const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
3
5
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
4
6
|
export declare const LANE_REUSE_REFRESHES_PROFILE_FLAG = "lanes.reuse-refreshes-profile";
|
|
5
7
|
/** Enables one automatic continuation after a missing-envelope repair. */
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
2
|
export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
3
|
+
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
|
+
export const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
3
5
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
4
6
|
export const LANE_REUSE_REFRESHES_PROFILE_FLAG = "lanes.reuse-refreshes-profile";
|
|
5
7
|
/** Enables one automatic continuation after a missing-envelope repair. */
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
11
|
+
import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, freeClaudeAccountSlots, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
12
|
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
13
|
import { effectiveAccountHome } from "./account-home.js";
|
|
14
14
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
@@ -22,7 +22,7 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
22
22
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
23
23
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
25
|
-
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG } from "./flag-keys.js";
|
|
25
|
+
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG } from "./flag-keys.js";
|
|
26
26
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledForLane, retainCodexRollouts } from "./disk-guard.js";
|
|
27
27
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
28
28
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
@@ -30,7 +30,7 @@ import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolG
|
|
|
30
30
|
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
31
31
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
32
32
|
import { isIsoDateTime } from "./types.js";
|
|
33
|
-
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
33
|
+
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
34
34
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
35
35
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
36
36
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
@@ -1808,11 +1808,35 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1808
1808
|
(excludeLaneIds === undefined || !excludeLaneIds.has(lane.lane_id)) &&
|
|
1809
1809
|
lane.pickup_validation_pending !== true &&
|
|
1810
1810
|
laneRunsHere(lane, deps.host))
|
|
1811
|
-
.filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
|
|
1812
|
-
.sort(compareLaneQueueOrder);
|
|
1811
|
+
.filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()));
|
|
1813
1812
|
if (localQueued.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
|
|
1814
1813
|
return;
|
|
1815
1814
|
const config = loadLanesConfig(hqRoot);
|
|
1815
|
+
const admissionQuotaState = loadQuotaState(hqRoot);
|
|
1816
|
+
const candidateLeadIds = leadLaneIds(lanes);
|
|
1817
|
+
const flagReader = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
1818
|
+
const leadAdmissionEnabledById = new Map();
|
|
1819
|
+
const leadFlagByCompany = new Map();
|
|
1820
|
+
for (const leadId of candidateLeadIds) {
|
|
1821
|
+
const lead = lanes.find((lane) => lane.lane_id === leadId && lane.provider === "claude");
|
|
1822
|
+
if (!lead)
|
|
1823
|
+
continue;
|
|
1824
|
+
const resolution = resolveLaneCompany(hqRoot, lead);
|
|
1825
|
+
const company = resolution.company ?? undefined;
|
|
1826
|
+
const key = company?.uid ?? company?.slug ?? lead.company?.uid ?? lead.company?.slug ?? lead.lane_id;
|
|
1827
|
+
let enabled = leadFlagByCompany.get(key);
|
|
1828
|
+
if (!enabled) {
|
|
1829
|
+
enabled = readLanesRegistryFlagFailOff(LANE_LEAD_ADMISSION_PRIORITY_FLAG, {
|
|
1830
|
+
company,
|
|
1831
|
+
...(company ? { companyUidSource: laneCompanyUidSource(resolution.source) } : {}),
|
|
1832
|
+
env: deps.env,
|
|
1833
|
+
}, flagReader);
|
|
1834
|
+
leadFlagByCompany.set(key, enabled);
|
|
1835
|
+
}
|
|
1836
|
+
leadAdmissionEnabledById.set(leadId, await enabled);
|
|
1837
|
+
}
|
|
1838
|
+
const priorityLeads = new Set([...leadAdmissionEnabledById].filter(([, enabled]) => enabled).map(([id]) => id));
|
|
1839
|
+
localQueued.sort((a, b) => compareLaneQueueOrderWithLeads(a, b, priorityLeads));
|
|
1816
1840
|
const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
|
|
1817
1841
|
const filePath = throttleStatePath(hqRoot);
|
|
1818
1842
|
const loaded = readThrottleState(filePath, throttleDeps);
|
|
@@ -1884,6 +1908,25 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1884
1908
|
}
|
|
1885
1909
|
}
|
|
1886
1910
|
const projected = [...running, ...spawned];
|
|
1911
|
+
const laneResolution = lane.provider === "claude" ? resolveLaneCompany(hqRoot, lane) : undefined;
|
|
1912
|
+
const laneCompany = laneResolution?.company ?? undefined;
|
|
1913
|
+
const laneCompanyKey = laneCompany?.uid ?? laneCompany?.slug ?? lane.company?.uid ?? lane.company?.slug;
|
|
1914
|
+
let reserveEnabled = false;
|
|
1915
|
+
if (lane.provider === "claude" && laneCompanyKey) {
|
|
1916
|
+
let enabled = leadFlagByCompany.get(laneCompanyKey);
|
|
1917
|
+
if (!enabled) {
|
|
1918
|
+
enabled = readLanesRegistryFlagFailOff(LANE_LEAD_ADMISSION_PRIORITY_FLAG, {
|
|
1919
|
+
company: laneCompany,
|
|
1920
|
+
...(laneCompany && laneResolution ? { companyUidSource: laneCompanyUidSource(laneResolution.source) } : {}),
|
|
1921
|
+
env: deps.env,
|
|
1922
|
+
}, flagReader);
|
|
1923
|
+
leadFlagByCompany.set(laneCompanyKey, enabled);
|
|
1924
|
+
}
|
|
1925
|
+
reserveEnabled = await enabled;
|
|
1926
|
+
}
|
|
1927
|
+
const reserveBlocked = reserveEnabled && lane.provider === "claude" &&
|
|
1928
|
+
!priorityLeads.has(lane.lane_id) &&
|
|
1929
|
+
freeClaudeAccountSlots(projected, config, deps.host, now.getTime(), admissionQuotaState) <= config.lead_reserve_per_host;
|
|
1887
1930
|
const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1888
1931
|
const admittedLane = alternateAccount !== undefined
|
|
1889
1932
|
? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
|
|
@@ -1891,7 +1934,10 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1891
1934
|
const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1892
1935
|
const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1893
1936
|
let blockedBy;
|
|
1894
|
-
if (
|
|
1937
|
+
if (reserveBlocked) {
|
|
1938
|
+
blockedBy = "lead_reserve";
|
|
1939
|
+
}
|
|
1940
|
+
else if (currentCap !== undefined) {
|
|
1895
1941
|
blockedBy = baseCap ?? "queue_position";
|
|
1896
1942
|
}
|
|
1897
1943
|
if (!blockedBy && !decision.admit)
|
|
@@ -177,7 +177,7 @@ export declare const TERMINAL_LANE_STATES: readonly ["done", "failed", "interrup
|
|
|
177
177
|
export type TerminalLaneState = (typeof TERMINAL_LANE_STATES)[number];
|
|
178
178
|
export declare const LANE_PROVIDERS: readonly ["claude", "codex", "grok"];
|
|
179
179
|
export type LaneProvider = (typeof LANE_PROVIDERS)[number];
|
|
180
|
-
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "worker_profile_changed", "low_disk", "admit_failed"];
|
|
180
|
+
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "lead_reserve", "worker_profile_changed", "low_disk", "admit_failed"];
|
|
181
181
|
export type QueuedReason = (typeof QUEUED_REASONS)[number];
|
|
182
182
|
export type QueuedDetails = {
|
|
183
183
|
reason: "worker_profile_changed";
|
|
@@ -1029,6 +1029,8 @@ export interface LanesConfig {
|
|
|
1029
1029
|
runaway_search_guard?: RunawaySearchGuardConfig;
|
|
1030
1030
|
/** Optional free-space threshold override for the low-disk admission gate. */
|
|
1031
1031
|
low_disk_min_free_bytes?: number;
|
|
1032
|
+
/** Claude account slots kept available per host for senior lead lanes. */
|
|
1033
|
+
lead_reserve_per_host: number;
|
|
1032
1034
|
/** Retention ages for Codex session rollout maintenance. */
|
|
1033
1035
|
codex_rollout_compress_after_hours?: number;
|
|
1034
1036
|
codex_rollout_delete_after_days?: number;
|
package/dist/lib/lanes/types.js
CHANGED
package/dist/main.js
CHANGED
|
@@ -362,12 +362,23 @@ function qmdSpawnFailureCaptureContext(err) {
|
|
|
362
362
|
return { contexts: { qmd_spawn_failure: diagnostics } };
|
|
363
363
|
}
|
|
364
364
|
const GATEWAY_RPC_CODE_TAGS = new Map([
|
|
365
|
+
[-32700, "parse_error"],
|
|
366
|
+
[-32601, "method_not_found"],
|
|
365
367
|
[-32603, "internal_error"],
|
|
366
368
|
[-32009, "conflict"],
|
|
367
369
|
[-32003, "unauthorized"],
|
|
368
370
|
[-32602, "invalid_params"],
|
|
369
371
|
[-32050, "provider_error"],
|
|
370
372
|
]);
|
|
373
|
+
const GATEWAY_ERROR_SHAPES = new Set([
|
|
374
|
+
"string_error",
|
|
375
|
+
"object_no_code",
|
|
376
|
+
"object_code",
|
|
377
|
+
"string_code",
|
|
378
|
+
"nested_code",
|
|
379
|
+
"empty",
|
|
380
|
+
"redacted_empty",
|
|
381
|
+
]);
|
|
371
382
|
function integrationsGatewayCaptureContext(err) {
|
|
372
383
|
if (err === null || typeof err !== "object")
|
|
373
384
|
return undefined;
|
|
@@ -386,11 +397,16 @@ function integrationsGatewayCaptureContext(err) {
|
|
|
386
397
|
const codeTag = typeof rpcCode === "number" && Number.isInteger(rpcCode)
|
|
387
398
|
? GATEWAY_RPC_CODE_TAGS.get(rpcCode) ?? "other"
|
|
388
399
|
: rpcCode === undefined ? "missing" : "other";
|
|
400
|
+
const errorShape = record.gatewayErrorShape;
|
|
401
|
+
const errorShapeTag = typeof errorShape === "string" && GATEWAY_ERROR_SHAPES.has(errorShape)
|
|
402
|
+
? errorShape
|
|
403
|
+
: "unknown";
|
|
389
404
|
return {
|
|
390
405
|
tags: {
|
|
391
406
|
integration_gateway_status: statusTag,
|
|
392
407
|
integration_gateway_code: codeTag,
|
|
393
408
|
integration_subcommand: subcommandTag,
|
|
409
|
+
integration_gateway_error_shape: errorShapeTag,
|
|
394
410
|
},
|
|
395
411
|
};
|
|
396
412
|
}
|