@indigoai-us/hq-cli 5.342.14 → 5.342.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/commands/agents.d.ts +2 -0
- package/dist/commands/agents.js +19 -11
- package/dist/commands/bot.js +4 -16
- package/dist/commands/daemon.d.ts +4 -0
- package/dist/commands/daemon.js +5 -18
- package/dist/commands/integrations-connect.d.ts +12 -0
- package/dist/commands/integrations-connect.js +5 -1
- package/dist/commands/integrations-core.d.ts +5 -4
- package/dist/commands/integrations-core.js +64 -10
- package/dist/commands/lanes-accounts.js +8 -0
- package/dist/commands/monitor.d.ts +1 -0
- package/dist/commands/monitor.js +4 -3
- package/dist/lib/agents/terminal-wait.d.ts +4 -0
- package/dist/lib/agents/terminal-wait.js +6 -0
- package/dist/lib/flag-registry.js +3 -1
- package/dist/lib/flag-snapshot-file.d.ts +3 -0
- package/dist/lib/flag-snapshot-file.js +190 -30
- package/dist/lib/lanes/admission.d.ts +7 -1
- package/dist/lib/lanes/admission.js +34 -0
- package/dist/lib/lanes/config.js +11 -0
- package/dist/lib/lanes/flag-keys.d.ts +2 -0
- package/dist/lib/lanes/flag-keys.js +2 -0
- package/dist/lib/lanes/reconcile.js +52 -6
- package/dist/lib/lanes/types.d.ts +3 -1
- package/dist/lib/lanes/types.js +1 -0
- package/dist/lib/log-follow.d.ts +3 -0
- package/dist/lib/log-follow.js +20 -0
- package/dist/main.js +16 -0
- package/dist/utils/resident-package-preload.d.ts +2 -2
- package/dist/utils/resident-package-preload.js +91 -7
- package/package.json +1 -1
|
@@ -4,6 +4,104 @@ import * as os from "node:os";
|
|
|
4
4
|
import * as path from "node:path";
|
|
5
5
|
export const FLAG_SNAPSHOT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
6
6
|
const COMPANY_UID_RE = /^cmp_[A-Za-z0-9]{3,128}$/;
|
|
7
|
+
function markWriteFailure(error, context) {
|
|
8
|
+
if (!error || (typeof error !== "object" && typeof error !== "function"))
|
|
9
|
+
return;
|
|
10
|
+
const failure = error;
|
|
11
|
+
let dirOwnerMatchesUid = "unknown";
|
|
12
|
+
if (typeof process.getuid === "function") {
|
|
13
|
+
try {
|
|
14
|
+
dirOwnerMatchesUid = fs.statSync(context.directory).uid === process.getuid()
|
|
15
|
+
? "true"
|
|
16
|
+
: "false";
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// A missing or inaccessible directory has no safely knowable owner.
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
try {
|
|
23
|
+
Object.defineProperties(failure, {
|
|
24
|
+
flagSnapshotStep: { value: context.step, configurable: true },
|
|
25
|
+
cacheRootKind: { value: context.cacheRootKind, configurable: true },
|
|
26
|
+
dirOwnerMatchesUid: { value: dirOwnerMatchesUid, configurable: true },
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// Preserve the original filesystem error if it cannot carry diagnostics.
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function atWriteStep(context, operation) {
|
|
34
|
+
try {
|
|
35
|
+
return operation();
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
markWriteFailure(error, context);
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export function flagSnapshotWriteFailureTags(error) {
|
|
43
|
+
const failure = error;
|
|
44
|
+
const code = failure && typeof failure.code === "string" ? failure.code : "";
|
|
45
|
+
const allowedCodes = new Set(["EACCES", "EPERM", "EROFS", "ENOSPC"]);
|
|
46
|
+
const step = failure?.flagSnapshotStep;
|
|
47
|
+
const rootKind = failure?.cacheRootKind;
|
|
48
|
+
const owner = failure?.dirOwnerMatchesUid;
|
|
49
|
+
const tags = {
|
|
50
|
+
flag_snapshot_step: step && ["mkdir", "chmod_dir", "open_tmp", "write_tmp", "rename", "chmod_file"].includes(step)
|
|
51
|
+
? step
|
|
52
|
+
: "other",
|
|
53
|
+
errno_code: allowedCodes.has(code) ? code : "other",
|
|
54
|
+
cache_root_kind: rootKind === "xdg" || rootKind === "home" ? rootKind : "unknown",
|
|
55
|
+
dir_owner_matches_uid: owner === "true" || owner === "false" ? owner : "unknown",
|
|
56
|
+
};
|
|
57
|
+
if (fallbackCacheUsed)
|
|
58
|
+
tags.cache_root = "fallback";
|
|
59
|
+
if (failure?.fallbackFailed)
|
|
60
|
+
tags.fallback = "failed";
|
|
61
|
+
return tags;
|
|
62
|
+
}
|
|
63
|
+
let fallbackCacheUsed = false;
|
|
64
|
+
export function _resetFlagSnapshotCacheForTests() {
|
|
65
|
+
fallbackCacheUsed = false;
|
|
66
|
+
}
|
|
67
|
+
function markFallbackFailed(error) {
|
|
68
|
+
if (!error || (typeof error !== "object" && typeof error !== "function"))
|
|
69
|
+
return;
|
|
70
|
+
try {
|
|
71
|
+
Object.defineProperty(error, "fallbackFailed", { value: true, configurable: true });
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
// Keep the original filesystem error intact if it cannot carry diagnostics.
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function isPermissionFailure(error) {
|
|
78
|
+
const code = error?.code;
|
|
79
|
+
return code === "EACCES" || code === "EPERM";
|
|
80
|
+
}
|
|
81
|
+
function fallbackSnapshotPath(companyUid, create) {
|
|
82
|
+
if (typeof process.getuid !== "function")
|
|
83
|
+
return undefined;
|
|
84
|
+
const uid = process.getuid();
|
|
85
|
+
const root = path.join(os.tmpdir(), `hq-cli-flag-snapshots-${uid}`);
|
|
86
|
+
const directories = [root, path.join(root, "hq"), path.join(root, "hq", "flags")];
|
|
87
|
+
for (const directory of directories) {
|
|
88
|
+
try {
|
|
89
|
+
if (create)
|
|
90
|
+
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
91
|
+
const stat = fs.lstatSync(directory);
|
|
92
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== uid)
|
|
93
|
+
return undefined;
|
|
94
|
+
if (create)
|
|
95
|
+
fs.chmodSync(directory, 0o700);
|
|
96
|
+
else if ((stat.mode & 0o077) !== 0)
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return undefined;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return path.join(directories[2], `snapshot-${companyUid}.json`);
|
|
104
|
+
}
|
|
7
105
|
export function flagSnapshotPath(companyUid, env = process.env) {
|
|
8
106
|
if (!COMPANY_UID_RE.test(companyUid))
|
|
9
107
|
return undefined;
|
|
@@ -12,7 +110,8 @@ export function flagSnapshotPath(companyUid, env = process.env) {
|
|
|
12
110
|
}
|
|
13
111
|
/** Write just the validated flag decisions and company scope, never auth data. */
|
|
14
112
|
export function writeFlagSnapshotFile(companyUid, flags, options = {}) {
|
|
15
|
-
const
|
|
113
|
+
const env = options.env ?? process.env;
|
|
114
|
+
const file = flagSnapshotPath(companyUid, env);
|
|
16
115
|
if (!file)
|
|
17
116
|
throw new Error("Cannot persist a flag snapshot without a valid company uid");
|
|
18
117
|
const normalizedFlags = Object.fromEntries(Object.entries(flags).filter((entry) => Boolean(entry[0].trim()) && typeof entry[1] === "boolean"));
|
|
@@ -24,39 +123,78 @@ export function writeFlagSnapshotFile(companyUid, flags, options = {}) {
|
|
|
24
123
|
if (!Number.isFinite(Date.parse(snapshot.fetched_at))) {
|
|
25
124
|
throw new Error("Cannot persist a flag snapshot with an invalid fetched_at timestamp");
|
|
26
125
|
}
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
fs.
|
|
36
|
-
|
|
37
|
-
fd
|
|
38
|
-
fs.renameSync(temporary, file);
|
|
39
|
-
fs.chmodSync(file, 0o600);
|
|
40
|
-
}
|
|
41
|
-
catch (error) {
|
|
42
|
-
if (fd !== undefined)
|
|
43
|
-
fs.closeSync(fd);
|
|
126
|
+
const cacheRootKind = env.XDG_CACHE_HOME?.trim() ? "xdg" : "home";
|
|
127
|
+
const writeAt = (destination) => {
|
|
128
|
+
const directory = path.dirname(destination);
|
|
129
|
+
const context = (step) => ({
|
|
130
|
+
step,
|
|
131
|
+
cacheRootKind,
|
|
132
|
+
directory,
|
|
133
|
+
});
|
|
134
|
+
atWriteStep(context("mkdir"), () => fs.mkdirSync(directory, { recursive: true, mode: 0o700 }));
|
|
135
|
+
const temporary = `${destination}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`;
|
|
136
|
+
let fd;
|
|
44
137
|
try {
|
|
45
|
-
fs.
|
|
138
|
+
fd = atWriteStep(context("open_tmp"), () => fs.openSync(temporary, "wx", 0o600));
|
|
139
|
+
atWriteStep(context("write_tmp"), () => {
|
|
140
|
+
fs.writeFileSync(fd, `${JSON.stringify(snapshot)}\n`, "utf8");
|
|
141
|
+
fs.fsyncSync(fd);
|
|
142
|
+
fs.closeSync(fd);
|
|
143
|
+
fd = undefined;
|
|
144
|
+
});
|
|
145
|
+
atWriteStep(context("rename"), () => fs.renameSync(temporary, destination));
|
|
46
146
|
}
|
|
47
|
-
catch (
|
|
48
|
-
if (
|
|
49
|
-
|
|
147
|
+
catch (error) {
|
|
148
|
+
if (fd !== undefined)
|
|
149
|
+
fs.closeSync(fd);
|
|
150
|
+
try {
|
|
151
|
+
fs.unlinkSync(temporary);
|
|
152
|
+
}
|
|
153
|
+
catch (cleanupError) {
|
|
154
|
+
if (cleanupError.code !== "ENOENT")
|
|
155
|
+
throw cleanupError;
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
50
158
|
}
|
|
51
|
-
|
|
159
|
+
return destination;
|
|
160
|
+
};
|
|
161
|
+
const writeFallback = (originalError) => {
|
|
162
|
+
const fallbackFile = fallbackSnapshotPath(companyUid, true);
|
|
163
|
+
if (!fallbackFile) {
|
|
164
|
+
if (originalError) {
|
|
165
|
+
markFallbackFailed(originalError);
|
|
166
|
+
throw originalError;
|
|
167
|
+
}
|
|
168
|
+
const error = Object.assign(new Error("Flag snapshot fallback directory is unavailable"), { code: "EACCES" });
|
|
169
|
+
markFallbackFailed(error);
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
const written = writeAt(fallbackFile);
|
|
174
|
+
fallbackCacheUsed = true;
|
|
175
|
+
return written;
|
|
176
|
+
}
|
|
177
|
+
catch (fallbackError) {
|
|
178
|
+
if (originalError) {
|
|
179
|
+
markFallbackFailed(originalError);
|
|
180
|
+
throw originalError;
|
|
181
|
+
}
|
|
182
|
+
markFallbackFailed(fallbackError);
|
|
183
|
+
throw fallbackError;
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
if (fallbackCacheUsed)
|
|
187
|
+
return writeFallback();
|
|
188
|
+
try {
|
|
189
|
+
return writeAt(file);
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
if (!isPermissionFailure(error))
|
|
193
|
+
throw error;
|
|
194
|
+
return writeFallback(error);
|
|
52
195
|
}
|
|
53
|
-
return file;
|
|
54
196
|
}
|
|
55
|
-
|
|
56
|
-
export function readFlagSnapshotFile(companyUid, options = {}) {
|
|
57
|
-
const file = flagSnapshotPath(companyUid, options.env);
|
|
58
|
-
if (!file)
|
|
59
|
-
return null;
|
|
197
|
+
function parseSnapshotFile(companyUid, file, options) {
|
|
60
198
|
let parsed;
|
|
61
199
|
try {
|
|
62
200
|
parsed = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
@@ -71,7 +209,7 @@ export function readFlagSnapshotFile(companyUid, options = {}) {
|
|
|
71
209
|
value.company_uid !== companyUid ||
|
|
72
210
|
typeof value.fetched_at !== "string" ||
|
|
73
211
|
!value.flags || typeof value.flags !== "object" || Array.isArray(value.flags))
|
|
74
|
-
return null;
|
|
212
|
+
return null;
|
|
75
213
|
const fetchedAt = Date.parse(value.fetched_at);
|
|
76
214
|
const age = (options.now ?? Date.now()) - fetchedAt;
|
|
77
215
|
if (!Number.isFinite(fetchedAt) || age < 0 || age > (options.maxAgeMs ?? FLAG_SNAPSHOT_MAX_AGE_MS)) {
|
|
@@ -88,4 +226,26 @@ export function readFlagSnapshotFile(companyUid, options = {}) {
|
|
|
88
226
|
flags: flags,
|
|
89
227
|
};
|
|
90
228
|
}
|
|
229
|
+
/** Return null for every miss, malformed file, company mismatch, or stale file. */
|
|
230
|
+
export function readFlagSnapshotFile(companyUid, options = {}) {
|
|
231
|
+
const file = flagSnapshotPath(companyUid, options.env);
|
|
232
|
+
if (!file)
|
|
233
|
+
return null;
|
|
234
|
+
if (fallbackCacheUsed) {
|
|
235
|
+
const fallback = fallbackSnapshotPath(companyUid, false);
|
|
236
|
+
const snapshot = fallback ? parseSnapshotFile(companyUid, fallback, options) : null;
|
|
237
|
+
if (snapshot)
|
|
238
|
+
return snapshot;
|
|
239
|
+
}
|
|
240
|
+
const primary = parseSnapshotFile(companyUid, file, options);
|
|
241
|
+
if (primary)
|
|
242
|
+
return primary;
|
|
243
|
+
const fallback = fallbackSnapshotPath(companyUid, false);
|
|
244
|
+
if (!fallback)
|
|
245
|
+
return null;
|
|
246
|
+
const snapshot = parseSnapshotFile(companyUid, fallback, options);
|
|
247
|
+
if (snapshot)
|
|
248
|
+
fallbackCacheUsed = true;
|
|
249
|
+
return snapshot;
|
|
250
|
+
}
|
|
91
251
|
//# sourceMappingURL=flag-snapshot-file.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type QuotaState } from "./quota.js";
|
|
2
|
+
import { type LaneRecord, type LanesConfig } from "./types.js";
|
|
2
3
|
export interface AdmissionHost {
|
|
3
4
|
hostname: string;
|
|
4
5
|
outpostId: string | null;
|
|
@@ -11,6 +12,11 @@ export declare function withLaneAdmissionLock<T>(hqRoot: string, provider: LaneR
|
|
|
11
12
|
export declare function withAllLaneAdmissionLocks<T>(hqRoot: string, fn: () => Promise<T>, sleep?: (ms: number) => Promise<void>): Promise<T>;
|
|
12
13
|
export declare function laneRunsOnHost(lane: LaneRecord, host: AdmissionHost): boolean;
|
|
13
14
|
export declare function compareLaneQueueOrder(a: LaneRecord, b: LaneRecord): number;
|
|
15
|
+
/** A lead is identified by another live lane naming it as its lane senior. */
|
|
16
|
+
export declare function leadLaneIds(lanes: readonly LaneRecord[]): Set<string>;
|
|
17
|
+
export declare function compareLaneQueueOrderWithLeads(a: LaneRecord, b: LaneRecord, leads: ReadonlySet<string>): number;
|
|
18
|
+
/** Capacity remaining across enabled Claude accounts on one host. */
|
|
19
|
+
export declare function freeClaudeAccountSlots(lanes: readonly LaneRecord[], config: LanesConfig, host: AdmissionHost, nowMs?: number, quotaState?: QuotaState): number;
|
|
14
20
|
export declare function hasActiveAdmissionReservation(lane: LaneRecord, nowMs: number): boolean;
|
|
15
21
|
export declare function hasActivePendingResume(lane: LaneRecord, nowMs: number): boolean;
|
|
16
22
|
/** Whether a lane currently owns a running slot for its host/provider caps. */
|
|
@@ -2,6 +2,8 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { accountMaxRunning } from "./config.js";
|
|
4
4
|
import { LanesError } from "./errors.js";
|
|
5
|
+
import { quotaBlockFor } from "./quota.js";
|
|
6
|
+
import { isTerminalLaneState } from "./types.js";
|
|
5
7
|
export const ADMISSION_LOCK_STALE_MS = 30_000;
|
|
6
8
|
export const ADMISSION_LOCK_TIMEOUT_MS = 5_000;
|
|
7
9
|
export const ADMISSION_RESERVATION_GRACE_MS = 30_000;
|
|
@@ -123,6 +125,38 @@ export function compareLaneQueueOrder(a, b) {
|
|
|
123
125
|
const byQueueAge = Date.parse(a.queued_at ?? a.created_at) - Date.parse(b.queued_at ?? b.created_at);
|
|
124
126
|
return byQueueAge || a.lane_id.localeCompare(b.lane_id);
|
|
125
127
|
}
|
|
128
|
+
/** A lead is identified by another live lane naming it as its lane senior. */
|
|
129
|
+
export function leadLaneIds(lanes) {
|
|
130
|
+
const leads = new Set(lanes.filter((lane) => lane.keep_alive === true).map((lane) => lane.lane_id));
|
|
131
|
+
for (const lane of lanes) {
|
|
132
|
+
if (!isTerminalLaneState(lane.state) && lane.senior.kind === "lane" && lane.senior.id) {
|
|
133
|
+
leads.add(lane.senior.id);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return leads;
|
|
137
|
+
}
|
|
138
|
+
export function compareLaneQueueOrderWithLeads(a, b, leads) {
|
|
139
|
+
if (a.provider === "claude" && b.provider === "claude") {
|
|
140
|
+
const aLead = leads.has(a.lane_id);
|
|
141
|
+
const bLead = leads.has(b.lane_id);
|
|
142
|
+
if (aLead !== bLead)
|
|
143
|
+
return aLead ? -1 : 1;
|
|
144
|
+
}
|
|
145
|
+
return compareLaneQueueOrder(a, b);
|
|
146
|
+
}
|
|
147
|
+
/** Capacity remaining across enabled Claude accounts on one host. */
|
|
148
|
+
export function freeClaudeAccountSlots(lanes, config, host, nowMs = Date.now(), quotaState) {
|
|
149
|
+
return Object.entries(config.providers.claude.accounts).reduce((total, [account, accountConfig]) => {
|
|
150
|
+
if (accountConfig.enabled !== true)
|
|
151
|
+
return total;
|
|
152
|
+
const model = accountConfig.model ?? config.providers.claude.model;
|
|
153
|
+
if (quotaState && quotaBlockFor(quotaState, "claude", account, model, new Date(nowMs)).blocked)
|
|
154
|
+
return total;
|
|
155
|
+
const occupied = lanes.filter((lane) => lane.provider === "claude" && lane.account === account &&
|
|
156
|
+
laneOccupiesRunningSlot(lane, nowMs) && laneRunsOnHost(lane, host)).length;
|
|
157
|
+
return total + Math.max(0, accountMaxRunning(config.providers.claude, account) - occupied);
|
|
158
|
+
}, 0);
|
|
159
|
+
}
|
|
126
160
|
export function hasActiveAdmissionReservation(lane, nowMs) {
|
|
127
161
|
if (lane.state !== "queued" || lane.queued_reason !== undefined || !lane.admission_reserved_at) {
|
|
128
162
|
return false;
|
package/dist/lib/lanes/config.js
CHANGED
|
@@ -152,6 +152,14 @@ function positiveInt(value, field, fallback) {
|
|
|
152
152
|
}
|
|
153
153
|
return value;
|
|
154
154
|
}
|
|
155
|
+
function nonNegativeInt(value, field, fallback) {
|
|
156
|
+
if (value === undefined)
|
|
157
|
+
return fallback;
|
|
158
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
159
|
+
throw new LanesConfigError("invalid_config", `${field} must be a non-negative integer`, { field, value });
|
|
160
|
+
}
|
|
161
|
+
return value;
|
|
162
|
+
}
|
|
155
163
|
function positiveNumber(value, field, fallback) {
|
|
156
164
|
if (value === undefined)
|
|
157
165
|
return fallback;
|
|
@@ -459,6 +467,7 @@ export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
|
459
467
|
tracker: { checks: { ...DEFAULT_TRACKER_CHECKS } },
|
|
460
468
|
runaway_search_guard: { ...DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG },
|
|
461
469
|
low_disk_min_free_bytes: DEFAULT_LOW_DISK_MIN_FREE_BYTES,
|
|
470
|
+
lead_reserve_per_host: 1,
|
|
462
471
|
codex_rollout_compress_after_hours: DEFAULT_CODEX_ROLLOUT_COMPRESS_AFTER_HOURS,
|
|
463
472
|
codex_rollout_delete_after_days: DEFAULT_CODEX_ROLLOUT_DELETE_AFTER_DAYS,
|
|
464
473
|
session_question_timeout_secs: DEFAULT_SESSION_QUESTION_TIMEOUT_SECS,
|
|
@@ -499,6 +508,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
499
508
|
}
|
|
500
509
|
const compressAfterHours = positiveInt(raw.codex_rollout_compress_after_hours, "codex_rollout_compress_after_hours", DEFAULT_CODEX_ROLLOUT_COMPRESS_AFTER_HOURS);
|
|
501
510
|
const deleteAfterDays = positiveInt(raw.codex_rollout_delete_after_days, "codex_rollout_delete_after_days", DEFAULT_CODEX_ROLLOUT_DELETE_AFTER_DAYS);
|
|
511
|
+
const leadReservePerHost = nonNegativeInt(raw.lead_reserve_per_host, "lead_reserve_per_host", 1);
|
|
502
512
|
if (deleteAfterDays * 24 < compressAfterHours) {
|
|
503
513
|
throw new LanesConfigError("invalid_config", "codex_rollout_delete_after_days must be at least codex_rollout_compress_after_hours / 24", { codex_rollout_compress_after_hours: compressAfterHours, codex_rollout_delete_after_days: deleteAfterDays });
|
|
504
514
|
}
|
|
@@ -512,6 +522,7 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
512
522
|
tracker: { checks: parseTrackerChecks(raw.tracker) },
|
|
513
523
|
runaway_search_guard: parseRunawaySearchGuard(raw.runaway_search_guard),
|
|
514
524
|
low_disk_min_free_bytes: positiveInt(raw.low_disk_min_free_bytes, "low_disk_min_free_bytes", DEFAULT_LOW_DISK_MIN_FREE_BYTES),
|
|
525
|
+
lead_reserve_per_host: leadReservePerHost,
|
|
515
526
|
codex_rollout_compress_after_hours: compressAfterHours,
|
|
516
527
|
codex_rollout_delete_after_days: deleteAfterDays,
|
|
517
528
|
session_question_timeout_secs: positiveInt(raw.session_question_timeout_secs, "session_question_timeout_secs", DEFAULT_SESSION_QUESTION_TIMEOUT_SECS),
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
2
|
export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
3
|
+
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
|
+
export declare const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
3
5
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
4
6
|
export declare const LANE_REUSE_REFRESHES_PROFILE_FLAG = "lanes.reuse-refreshes-profile";
|
|
5
7
|
/** Enables one automatic continuation after a missing-envelope repair. */
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
2
|
export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
3
|
+
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
|
+
export const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
3
5
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
4
6
|
export const LANE_REUSE_REFRESHES_PROFILE_FLAG = "lanes.reuse-refreshes-profile";
|
|
5
7
|
/** Enables one automatic continuation after a missing-envelope repair. */
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
11
|
+
import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, freeClaudeAccountSlots, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
12
|
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
13
|
import { effectiveAccountHome } from "./account-home.js";
|
|
14
14
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
@@ -22,7 +22,7 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
22
22
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
23
23
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
25
|
-
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG } from "./flag-keys.js";
|
|
25
|
+
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG } from "./flag-keys.js";
|
|
26
26
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledForLane, retainCodexRollouts } from "./disk-guard.js";
|
|
27
27
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
28
28
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
@@ -30,7 +30,7 @@ import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolG
|
|
|
30
30
|
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
31
31
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
32
32
|
import { isIsoDateTime } from "./types.js";
|
|
33
|
-
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
33
|
+
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
34
34
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
35
35
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
36
36
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
@@ -1808,11 +1808,35 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1808
1808
|
(excludeLaneIds === undefined || !excludeLaneIds.has(lane.lane_id)) &&
|
|
1809
1809
|
lane.pickup_validation_pending !== true &&
|
|
1810
1810
|
laneRunsHere(lane, deps.host))
|
|
1811
|
-
.filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
|
|
1812
|
-
.sort(compareLaneQueueOrder);
|
|
1811
|
+
.filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()));
|
|
1813
1812
|
if (localQueued.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
|
|
1814
1813
|
return;
|
|
1815
1814
|
const config = loadLanesConfig(hqRoot);
|
|
1815
|
+
const admissionQuotaState = loadQuotaState(hqRoot);
|
|
1816
|
+
const candidateLeadIds = leadLaneIds(lanes);
|
|
1817
|
+
const flagReader = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
1818
|
+
const leadAdmissionEnabledById = new Map();
|
|
1819
|
+
const leadFlagByCompany = new Map();
|
|
1820
|
+
for (const leadId of candidateLeadIds) {
|
|
1821
|
+
const lead = lanes.find((lane) => lane.lane_id === leadId && lane.provider === "claude");
|
|
1822
|
+
if (!lead)
|
|
1823
|
+
continue;
|
|
1824
|
+
const resolution = resolveLaneCompany(hqRoot, lead);
|
|
1825
|
+
const company = resolution.company ?? undefined;
|
|
1826
|
+
const key = company?.uid ?? company?.slug ?? lead.company?.uid ?? lead.company?.slug ?? lead.lane_id;
|
|
1827
|
+
let enabled = leadFlagByCompany.get(key);
|
|
1828
|
+
if (!enabled) {
|
|
1829
|
+
enabled = readLanesRegistryFlagFailOff(LANE_LEAD_ADMISSION_PRIORITY_FLAG, {
|
|
1830
|
+
company,
|
|
1831
|
+
...(company ? { companyUidSource: laneCompanyUidSource(resolution.source) } : {}),
|
|
1832
|
+
env: deps.env,
|
|
1833
|
+
}, flagReader);
|
|
1834
|
+
leadFlagByCompany.set(key, enabled);
|
|
1835
|
+
}
|
|
1836
|
+
leadAdmissionEnabledById.set(leadId, await enabled);
|
|
1837
|
+
}
|
|
1838
|
+
const priorityLeads = new Set([...leadAdmissionEnabledById].filter(([, enabled]) => enabled).map(([id]) => id));
|
|
1839
|
+
localQueued.sort((a, b) => compareLaneQueueOrderWithLeads(a, b, priorityLeads));
|
|
1816
1840
|
const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
|
|
1817
1841
|
const filePath = throttleStatePath(hqRoot);
|
|
1818
1842
|
const loaded = readThrottleState(filePath, throttleDeps);
|
|
@@ -1884,6 +1908,25 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1884
1908
|
}
|
|
1885
1909
|
}
|
|
1886
1910
|
const projected = [...running, ...spawned];
|
|
1911
|
+
const laneResolution = lane.provider === "claude" ? resolveLaneCompany(hqRoot, lane) : undefined;
|
|
1912
|
+
const laneCompany = laneResolution?.company ?? undefined;
|
|
1913
|
+
const laneCompanyKey = laneCompany?.uid ?? laneCompany?.slug ?? lane.company?.uid ?? lane.company?.slug;
|
|
1914
|
+
let reserveEnabled = false;
|
|
1915
|
+
if (lane.provider === "claude" && laneCompanyKey) {
|
|
1916
|
+
let enabled = leadFlagByCompany.get(laneCompanyKey);
|
|
1917
|
+
if (!enabled) {
|
|
1918
|
+
enabled = readLanesRegistryFlagFailOff(LANE_LEAD_ADMISSION_PRIORITY_FLAG, {
|
|
1919
|
+
company: laneCompany,
|
|
1920
|
+
...(laneCompany && laneResolution ? { companyUidSource: laneCompanyUidSource(laneResolution.source) } : {}),
|
|
1921
|
+
env: deps.env,
|
|
1922
|
+
}, flagReader);
|
|
1923
|
+
leadFlagByCompany.set(laneCompanyKey, enabled);
|
|
1924
|
+
}
|
|
1925
|
+
reserveEnabled = await enabled;
|
|
1926
|
+
}
|
|
1927
|
+
const reserveBlocked = reserveEnabled && lane.provider === "claude" &&
|
|
1928
|
+
!priorityLeads.has(lane.lane_id) &&
|
|
1929
|
+
freeClaudeAccountSlots(projected, config, deps.host, now.getTime(), admissionQuotaState) <= config.lead_reserve_per_host;
|
|
1887
1930
|
const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1888
1931
|
const admittedLane = alternateAccount !== undefined
|
|
1889
1932
|
? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
|
|
@@ -1891,7 +1934,10 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1891
1934
|
const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1892
1935
|
const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1893
1936
|
let blockedBy;
|
|
1894
|
-
if (
|
|
1937
|
+
if (reserveBlocked) {
|
|
1938
|
+
blockedBy = "lead_reserve";
|
|
1939
|
+
}
|
|
1940
|
+
else if (currentCap !== undefined) {
|
|
1895
1941
|
blockedBy = baseCap ?? "queue_position";
|
|
1896
1942
|
}
|
|
1897
1943
|
if (!blockedBy && !decision.admit)
|
|
@@ -177,7 +177,7 @@ export declare const TERMINAL_LANE_STATES: readonly ["done", "failed", "interrup
|
|
|
177
177
|
export type TerminalLaneState = (typeof TERMINAL_LANE_STATES)[number];
|
|
178
178
|
export declare const LANE_PROVIDERS: readonly ["claude", "codex", "grok"];
|
|
179
179
|
export type LaneProvider = (typeof LANE_PROVIDERS)[number];
|
|
180
|
-
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "worker_profile_changed", "low_disk", "admit_failed"];
|
|
180
|
+
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "lead_reserve", "worker_profile_changed", "low_disk", "admit_failed"];
|
|
181
181
|
export type QueuedReason = (typeof QUEUED_REASONS)[number];
|
|
182
182
|
export type QueuedDetails = {
|
|
183
183
|
reason: "worker_profile_changed";
|
|
@@ -1029,6 +1029,8 @@ export interface LanesConfig {
|
|
|
1029
1029
|
runaway_search_guard?: RunawaySearchGuardConfig;
|
|
1030
1030
|
/** Optional free-space threshold override for the low-disk admission gate. */
|
|
1031
1031
|
low_disk_min_free_bytes?: number;
|
|
1032
|
+
/** Claude account slots kept available per host for senior lead lanes. */
|
|
1033
|
+
lead_reserve_per_host: number;
|
|
1032
1034
|
/** Retention ages for Codex session rollout maintenance. */
|
|
1033
1035
|
codex_rollout_compress_after_hours?: number;
|
|
1034
1036
|
codex_rollout_delete_after_days?: number;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
/** Follow append-only log bytes without importing package code after startup. */
|
|
3
|
+
export async function followLogFile(file, initialOffset, write) {
|
|
4
|
+
let offset = initialOffset;
|
|
5
|
+
await new Promise(() => {
|
|
6
|
+
fs.watchFile(file, { interval: 500 }, (curr) => {
|
|
7
|
+
if (curr.size < offset)
|
|
8
|
+
offset = 0;
|
|
9
|
+
if (curr.size === offset)
|
|
10
|
+
return;
|
|
11
|
+
const fd = fs.openSync(file, "r");
|
|
12
|
+
const buf = Buffer.alloc(curr.size - offset);
|
|
13
|
+
fs.readSync(fd, buf, 0, buf.length, offset);
|
|
14
|
+
fs.closeSync(fd);
|
|
15
|
+
offset = curr.size;
|
|
16
|
+
write(buf.toString("utf8"));
|
|
17
|
+
});
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=log-follow.js.map
|
package/dist/main.js
CHANGED
|
@@ -362,12 +362,23 @@ function qmdSpawnFailureCaptureContext(err) {
|
|
|
362
362
|
return { contexts: { qmd_spawn_failure: diagnostics } };
|
|
363
363
|
}
|
|
364
364
|
const GATEWAY_RPC_CODE_TAGS = new Map([
|
|
365
|
+
[-32700, "parse_error"],
|
|
366
|
+
[-32601, "method_not_found"],
|
|
365
367
|
[-32603, "internal_error"],
|
|
366
368
|
[-32009, "conflict"],
|
|
367
369
|
[-32003, "unauthorized"],
|
|
368
370
|
[-32602, "invalid_params"],
|
|
369
371
|
[-32050, "provider_error"],
|
|
370
372
|
]);
|
|
373
|
+
const GATEWAY_ERROR_SHAPES = new Set([
|
|
374
|
+
"string_error",
|
|
375
|
+
"object_no_code",
|
|
376
|
+
"object_code",
|
|
377
|
+
"string_code",
|
|
378
|
+
"nested_code",
|
|
379
|
+
"empty",
|
|
380
|
+
"redacted_empty",
|
|
381
|
+
]);
|
|
371
382
|
function integrationsGatewayCaptureContext(err) {
|
|
372
383
|
if (err === null || typeof err !== "object")
|
|
373
384
|
return undefined;
|
|
@@ -386,11 +397,16 @@ function integrationsGatewayCaptureContext(err) {
|
|
|
386
397
|
const codeTag = typeof rpcCode === "number" && Number.isInteger(rpcCode)
|
|
387
398
|
? GATEWAY_RPC_CODE_TAGS.get(rpcCode) ?? "other"
|
|
388
399
|
: rpcCode === undefined ? "missing" : "other";
|
|
400
|
+
const errorShape = record.gatewayErrorShape;
|
|
401
|
+
const errorShapeTag = typeof errorShape === "string" && GATEWAY_ERROR_SHAPES.has(errorShape)
|
|
402
|
+
? errorShape
|
|
403
|
+
: "unknown";
|
|
389
404
|
return {
|
|
390
405
|
tags: {
|
|
391
406
|
integration_gateway_status: statusTag,
|
|
392
407
|
integration_gateway_code: codeTag,
|
|
393
408
|
integration_subcommand: subcommandTag,
|
|
409
|
+
integration_gateway_error_shape: errorShapeTag,
|
|
394
410
|
},
|
|
395
411
|
};
|
|
396
412
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export type ResidentPackagePath = "agent-kit" | "agent-mcp" | "auth-login" | "bot" | "childWrapper" | "coreJobRun" | "login" | "mcp" | "daemon" | "mesh-daemon" | "lanes-watch" | "monitor-wait" | "monitor-run";
|
|
2
|
-
/**
|
|
1
|
+
export type ResidentPackagePath = "agent-kit" | "agent-mcp" | "auth-login" | "agents-terminal" | "bot" | "bot-logs-follow" | "childWrapper" | "coreJobRun" | "login" | "mcp" | "daemon" | "daemon-logs-follow" | "mesh-daemon" | "lanes-watch" | "monitor-wait" | "monitor-run" | "agents-message-wait" | "integrations-oauth-wait";
|
|
2
|
+
/** Runtime modules that resident commands may need after a wait starts. */
|
|
3
3
|
export declare const RESIDENT_PACKAGE_MODULES: Record<ResidentPackagePath, readonly string[]>;
|
|
4
4
|
export declare class ResidentPackagePreloadError extends Error {
|
|
5
5
|
readonly moduleSpecifier: string;
|