@indigoai-us/hq-cli 5.345.43 → 5.345.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/command-catalog.generated.d.ts +17 -1
- package/dist/command-catalog.generated.js +21 -1
- package/dist/commands/access.js +6 -4
- package/dist/commands/bot.js +1 -1
- package/dist/commands/db-destroy.js +5 -4
- package/dist/commands/db-remote-deps.js +4 -0
- package/dist/commands/lanes-checks.d.ts +1 -0
- package/dist/commands/lanes-checks.js +6 -1
- package/dist/commands/lanes-message.js +84 -51
- package/dist/commands/lanes-report.d.ts +2 -1
- package/dist/commands/lanes-report.js +63 -6
- package/dist/commands/lanes-resume.js +44 -0
- package/dist/commands/lanes-shared.d.ts +25 -0
- package/dist/commands/lanes.d.ts +4 -0
- package/dist/commands/lanes.js +228 -9
- package/dist/commands/meetings.js +4 -1
- package/dist/index.js +43 -7
- package/dist/lib/core-utils/sentry-report.js +8 -1
- package/dist/lib/db/control-plane.d.ts +5 -0
- package/dist/lib/db/control-plane.js +7 -0
- package/dist/lib/flag-snapshot-file.js +33 -25
- package/dist/lib/lanes/flag-keys.d.ts +4 -0
- package/dist/lib/lanes/flag-keys.js +4 -0
- package/dist/lib/lanes/focus-guard.d.ts +65 -0
- package/dist/lib/lanes/focus-guard.js +91 -0
- package/dist/lib/lanes/grants.js +1 -1
- package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
- package/dist/lib/lanes/outpost-dispatch.js +1 -0
- package/dist/lib/lanes/reconcile.d.ts +2 -0
- package/dist/lib/lanes/reconcile.js +39 -1
- package/dist/lib/lanes/spawn.d.ts +4 -0
- package/dist/lib/lanes/spawn.js +132 -6
- package/dist/lib/lanes/store.js +64 -0
- package/dist/lib/lanes/story-ledger.d.ts +14 -0
- package/dist/lib/lanes/story-ledger.js +46 -0
- package/dist/lib/lanes/tracker-report.d.ts +46 -0
- package/dist/lib/lanes/tracker-report.js +85 -1
- package/dist/lib/lanes/types.d.ts +12 -2
- package/dist/lib/lanes/types.js +5 -3
- package/dist/main.js +5 -1
- package/dist/startup-registration.d.ts +3 -0
- package/dist/startup-registration.js +11 -4
- package/dist/utils/package-use-lease.d.ts +1 -0
- package/dist/utils/package-use-lease.js +29 -0
- package/dist/utils/self-update-backoff.d.ts +17 -0
- package/dist/utils/self-update-backoff.js +73 -0
- package/dist/utils/self-update.d.ts +7 -0
- package/dist/utils/self-update.js +36 -0
- package/dist/utils/vault-api.d.ts +33 -1
- package/dist/utils/vault-api.js +55 -5
- package/dist/utils/version-gate.d.ts +13 -0
- package/dist/utils/version-gate.js +35 -5
- package/package.json +1 -1
|
@@ -78,29 +78,38 @@ function isPermissionFailure(error) {
|
|
|
78
78
|
const code = error?.code;
|
|
79
79
|
return code === "EACCES" || code === "EPERM";
|
|
80
80
|
}
|
|
81
|
-
function fallbackSnapshotPath(companyUid, create) {
|
|
82
|
-
|
|
81
|
+
function fallbackSnapshotPath(companyUid, create, env = process.env) {
|
|
82
|
+
const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
|
83
|
+
const root = uid === undefined
|
|
84
|
+
? env.LOCALAPPDATA?.trim()
|
|
85
|
+
: path.join(os.tmpdir(), `hq-cli-flag-snapshots-${uid}`);
|
|
86
|
+
if (!root)
|
|
83
87
|
return undefined;
|
|
84
|
-
const
|
|
85
|
-
const
|
|
86
|
-
|
|
88
|
+
const fallbackRoot = uid === undefined ? path.join(root, "hq-cli", "flag-snapshots") : root;
|
|
89
|
+
const directories = uid === undefined
|
|
90
|
+
? [path.join(root, "hq-cli"), fallbackRoot]
|
|
91
|
+
: [fallbackRoot, path.join(fallbackRoot, "hq"), path.join(fallbackRoot, "hq", "flags")];
|
|
87
92
|
for (const directory of directories) {
|
|
88
93
|
try {
|
|
89
94
|
if (create)
|
|
90
95
|
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
91
96
|
const stat = fs.lstatSync(directory);
|
|
92
|
-
if (!stat.isDirectory() || stat.isSymbolicLink()
|
|
93
|
-
return undefined;
|
|
94
|
-
if (create)
|
|
95
|
-
fs.chmodSync(directory, 0o700);
|
|
96
|
-
else if ((stat.mode & 0o077) !== 0)
|
|
97
|
+
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
97
98
|
return undefined;
|
|
99
|
+
if (uid !== undefined) {
|
|
100
|
+
if (stat.uid !== uid)
|
|
101
|
+
return undefined;
|
|
102
|
+
if (create)
|
|
103
|
+
fs.chmodSync(directory, 0o700);
|
|
104
|
+
else if ((stat.mode & 0o077) !== 0)
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
98
107
|
}
|
|
99
108
|
catch {
|
|
100
109
|
return undefined;
|
|
101
110
|
}
|
|
102
111
|
}
|
|
103
|
-
return path.join(directories[
|
|
112
|
+
return path.join(directories[directories.length - 1], `snapshot-${companyUid}.json`);
|
|
104
113
|
}
|
|
105
114
|
function fallbackPersonSnapshotPath(create) {
|
|
106
115
|
if (typeof process.getuid !== "function")
|
|
@@ -275,7 +284,7 @@ export function writeFlagSnapshotFile(companyUid, flags, options = {}) {
|
|
|
275
284
|
}
|
|
276
285
|
const cacheRootKind = env.XDG_CACHE_HOME?.trim() ? "xdg" : "home";
|
|
277
286
|
const writeFallback = (originalError) => {
|
|
278
|
-
const fallbackFile = fallbackSnapshotPath(companyUid, true);
|
|
287
|
+
const fallbackFile = fallbackSnapshotPath(companyUid, true, env);
|
|
279
288
|
if (!fallbackFile) {
|
|
280
289
|
if (originalError) {
|
|
281
290
|
markFallbackFailed(originalError);
|
|
@@ -347,21 +356,20 @@ export function readFlagSnapshotFile(companyUid, options = {}) {
|
|
|
347
356
|
const file = flagSnapshotPath(companyUid, options.env);
|
|
348
357
|
if (!file)
|
|
349
358
|
return null;
|
|
350
|
-
if (fallbackCacheUsed) {
|
|
351
|
-
const fallback = fallbackSnapshotPath(companyUid, false);
|
|
352
|
-
const snapshot = fallback ? parseSnapshotFile(companyUid, fallback, options) : null;
|
|
353
|
-
if (snapshot)
|
|
354
|
-
return snapshot;
|
|
355
|
-
}
|
|
356
359
|
const primary = parseSnapshotFile(companyUid, file, options);
|
|
357
|
-
|
|
360
|
+
const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
|
361
|
+
if (primary && !fallbackCacheUsed && uid !== undefined)
|
|
358
362
|
return primary;
|
|
359
|
-
const fallback = fallbackSnapshotPath(companyUid, false);
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
363
|
+
const fallback = fallbackSnapshotPath(companyUid, false, options.env);
|
|
364
|
+
const fallbackSnapshot = fallback ? parseSnapshotFile(companyUid, fallback, options) : null;
|
|
365
|
+
// A process can restart after a denied Windows rename wrote a newer snapshot
|
|
366
|
+
// to LOCALAPPDATA. Compare both valid stores so an older primary does not
|
|
367
|
+
// mask that successful fallback write. Equal timestamps keep the primary.
|
|
368
|
+
if (fallbackSnapshot
|
|
369
|
+
&& (!primary || Date.parse(fallbackSnapshot.fetched_at) > Date.parse(primary.fetched_at))) {
|
|
364
370
|
fallbackCacheUsed = true;
|
|
365
|
-
|
|
371
|
+
return fallbackSnapshot;
|
|
372
|
+
}
|
|
373
|
+
return primary;
|
|
366
374
|
}
|
|
367
375
|
//# sourceMappingURL=flag-snapshot-file.js.map
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
|
+
export declare const LANE_FOCUS_GUARD_FLAG = "lanes-focus-guard";
|
|
3
|
+
export declare const LANE_FOCUS_DASHBOARD_FLAG = "lanes-focus-dashboard";
|
|
2
4
|
export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
5
|
+
/** Enables same-provider automatic lane recovery after provider quota exhaustion. */
|
|
6
|
+
export declare const LANE_QUOTA_ACCOUNT_FAILOVER_FLAG = "lanes.quota-account-failover";
|
|
3
7
|
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
8
|
export declare const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
5
9
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
|
+
export const LANE_FOCUS_GUARD_FLAG = "lanes-focus-guard";
|
|
3
|
+
export const LANE_FOCUS_DASHBOARD_FLAG = "lanes-focus-dashboard";
|
|
2
4
|
export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
5
|
+
/** Enables same-provider automatic lane recovery after provider quota exhaustion. */
|
|
6
|
+
export const LANE_QUOTA_ACCOUNT_FAILOVER_FLAG = "lanes.quota-account-failover";
|
|
3
7
|
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
8
|
export const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
5
9
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { LanesError } from "./errors.js";
|
|
2
|
+
export declare const FOCUS_GUARD_MAX_PR_AGE_MS: number;
|
|
3
|
+
export interface FocusGuardStory {
|
|
4
|
+
id: string;
|
|
5
|
+
focus_id?: unknown;
|
|
6
|
+
kpi?: unknown;
|
|
7
|
+
state?: unknown;
|
|
8
|
+
status?: unknown;
|
|
9
|
+
lane?: unknown;
|
|
10
|
+
lanes?: unknown;
|
|
11
|
+
}
|
|
12
|
+
export interface FocusGuardPullRequest {
|
|
13
|
+
number: number;
|
|
14
|
+
created_at: string;
|
|
15
|
+
repository?: string;
|
|
16
|
+
}
|
|
17
|
+
export interface FocusGuardLane {
|
|
18
|
+
lane_id: string;
|
|
19
|
+
state: string;
|
|
20
|
+
}
|
|
21
|
+
export type FocusGuardReason = {
|
|
22
|
+
code: "story_missing_focus";
|
|
23
|
+
story_id: string;
|
|
24
|
+
} | {
|
|
25
|
+
code: "open_pr_over_48h";
|
|
26
|
+
number: number;
|
|
27
|
+
age_hours: number;
|
|
28
|
+
repository?: string;
|
|
29
|
+
} | {
|
|
30
|
+
code: "stale_story";
|
|
31
|
+
story_id: string;
|
|
32
|
+
lane_id: string | null;
|
|
33
|
+
} | {
|
|
34
|
+
code: "pr_lookup_failed";
|
|
35
|
+
};
|
|
36
|
+
export interface FocusGuardOverride {
|
|
37
|
+
reason: string;
|
|
38
|
+
by_lane: string;
|
|
39
|
+
at: string;
|
|
40
|
+
story: string;
|
|
41
|
+
}
|
|
42
|
+
export declare function appendLaneFocusOverride<T extends {
|
|
43
|
+
focus_overrides?: FocusGuardOverride[];
|
|
44
|
+
}>(lane: T, override: FocusGuardOverride): T & {
|
|
45
|
+
focus_overrides: FocusGuardOverride[];
|
|
46
|
+
};
|
|
47
|
+
export declare function evaluateFocusGuard(input: {
|
|
48
|
+
storyIds: readonly string[];
|
|
49
|
+
stories: readonly FocusGuardStory[];
|
|
50
|
+
lanes: readonly FocusGuardLane[];
|
|
51
|
+
pullRequests?: readonly FocusGuardPullRequest[];
|
|
52
|
+
pullRequestLookupFailed?: boolean;
|
|
53
|
+
now: Date;
|
|
54
|
+
checkTeamBlockers: boolean;
|
|
55
|
+
}): FocusGuardReason[];
|
|
56
|
+
export declare function evaluateFocusGuardIfEnabled(enabled: boolean, input: Parameters<typeof evaluateFocusGuard>[0]): FocusGuardReason[];
|
|
57
|
+
export declare function assertFocusGuardAllows(enabled: boolean, input: Parameters<typeof evaluateFocusGuard>[0], overrideReason?: string): FocusGuardReason[];
|
|
58
|
+
export declare function focusGuardRefusal(reasons: readonly FocusGuardReason[]): LanesError;
|
|
59
|
+
export declare function makeFocusOverride(input: {
|
|
60
|
+
reason?: string;
|
|
61
|
+
laneId: string;
|
|
62
|
+
at: string;
|
|
63
|
+
storyId: string;
|
|
64
|
+
}): FocusGuardOverride | undefined;
|
|
65
|
+
//# sourceMappingURL=focus-guard.d.ts.map
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { LanesError } from "./errors.js";
|
|
2
|
+
import { isTerminalLaneState } from "./types.js";
|
|
3
|
+
export const FOCUS_GUARD_MAX_PR_AGE_MS = 48 * 60 * 60 * 1000;
|
|
4
|
+
export function appendLaneFocusOverride(lane, override) {
|
|
5
|
+
return { ...lane, focus_overrides: [...(lane.focus_overrides ?? []), override].slice(-100) };
|
|
6
|
+
}
|
|
7
|
+
function nonempty(value) {
|
|
8
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
9
|
+
}
|
|
10
|
+
function storyLaneIds(story) {
|
|
11
|
+
const candidates = [story.lane, ...(Array.isArray(story.lanes) ? story.lanes : [])];
|
|
12
|
+
return [...new Set(candidates.flatMap((item) => {
|
|
13
|
+
if (typeof item === "string" && item.trim())
|
|
14
|
+
return [item.trim()];
|
|
15
|
+
if (item !== null && typeof item === "object" && typeof item.lane === "string") {
|
|
16
|
+
const lane = item.lane.trim();
|
|
17
|
+
return lane ? [lane] : [];
|
|
18
|
+
}
|
|
19
|
+
if (item !== null && typeof item === "object" && typeof item.lane_id === "string") {
|
|
20
|
+
const laneId = item.lane_id.trim();
|
|
21
|
+
return laneId ? [laneId] : [];
|
|
22
|
+
}
|
|
23
|
+
return [];
|
|
24
|
+
}))];
|
|
25
|
+
}
|
|
26
|
+
export function evaluateFocusGuard(input) {
|
|
27
|
+
const reasons = [];
|
|
28
|
+
const storyById = new Map(input.stories.map((story) => [story.id.trim(), story]));
|
|
29
|
+
for (const id of input.storyIds) {
|
|
30
|
+
const story = storyById.get(id);
|
|
31
|
+
if (!story || (!nonempty(story.focus_id) && !nonempty(story.kpi))) {
|
|
32
|
+
reasons.push({ code: "story_missing_focus", story_id: id });
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (!input.checkTeamBlockers)
|
|
36
|
+
return reasons;
|
|
37
|
+
if (input.pullRequestLookupFailed) {
|
|
38
|
+
reasons.push({ code: "pr_lookup_failed" });
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
for (const pr of input.pullRequests ?? []) {
|
|
42
|
+
const ageMs = input.now.getTime() - Date.parse(pr.created_at);
|
|
43
|
+
if (Number.isFinite(ageMs) && ageMs > FOCUS_GUARD_MAX_PR_AGE_MS) {
|
|
44
|
+
reasons.push({
|
|
45
|
+
code: "open_pr_over_48h",
|
|
46
|
+
number: pr.number,
|
|
47
|
+
age_hours: Math.floor(ageMs / (60 * 60 * 1000)),
|
|
48
|
+
...(pr.repository === undefined ? {} : { repository: pr.repository }),
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const liveLaneIds = new Set(input.lanes
|
|
54
|
+
.filter((lane) => !isTerminalLaneState(lane.state))
|
|
55
|
+
.map((lane) => lane.lane_id));
|
|
56
|
+
for (const story of input.stories) {
|
|
57
|
+
const storyStatus = typeof story.state === "string" ? story.state : story.status;
|
|
58
|
+
if (storyStatus !== "in_progress")
|
|
59
|
+
continue;
|
|
60
|
+
const assigned = storyLaneIds(story);
|
|
61
|
+
if (assigned.length === 0 || assigned.every((laneId) => !liveLaneIds.has(laneId))) {
|
|
62
|
+
reasons.push({ code: "stale_story", story_id: story.id, lane_id: assigned[0] ?? null });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return reasons;
|
|
66
|
+
}
|
|
67
|
+
export function evaluateFocusGuardIfEnabled(enabled, input) {
|
|
68
|
+
return enabled ? evaluateFocusGuard(input) : [];
|
|
69
|
+
}
|
|
70
|
+
export function assertFocusGuardAllows(enabled, input, overrideReason) {
|
|
71
|
+
const reasons = evaluateFocusGuardIfEnabled(enabled, input);
|
|
72
|
+
if (enabled && reasons.length > 0 && overrideReason === undefined)
|
|
73
|
+
throw focusGuardRefusal(reasons);
|
|
74
|
+
return reasons;
|
|
75
|
+
}
|
|
76
|
+
export function focusGuardRefusal(reasons) {
|
|
77
|
+
return new LanesError("focus_guard_refused", `Lane launch refused by focus guard: ${reasons.map((reason) => reason.code).join(", ")}`, { reasons });
|
|
78
|
+
}
|
|
79
|
+
export function makeFocusOverride(input) {
|
|
80
|
+
if (input.reason === undefined)
|
|
81
|
+
return undefined;
|
|
82
|
+
const reason = input.reason.trim();
|
|
83
|
+
if (!reason) {
|
|
84
|
+
throw new LanesError("focus_override_reason_required", "--focus-override requires a non-empty reason", {
|
|
85
|
+
lane_id: input.laneId,
|
|
86
|
+
story_id: input.storyId,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
return { reason, by_lane: input.laneId, at: input.at, story: input.storyId };
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=focus-guard.js.map
|
package/dist/lib/lanes/grants.js
CHANGED
|
@@ -28,7 +28,7 @@ export function terminalChildResumeHireGrantError(current, snapshot, finishedChi
|
|
|
28
28
|
}
|
|
29
29
|
if (children.length >= parentGrant)
|
|
30
30
|
return hireGrantCapacityError(parent, children);
|
|
31
|
-
const requestedGrant = current.hire_grant ?? 0;
|
|
31
|
+
const requestedGrant = current.released_hire_grant ?? current.hire_grant ?? 0;
|
|
32
32
|
if (requestedGrant > availableChildGrant) {
|
|
33
33
|
return new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${availableChildGrant} unassigned hire grant remaining, but ${terminal ? "terminal" : "finished"} child ${current.lane_id} needs ${requestedGrant} to resume. Ask its senior for more: write an ask envelope with the desired total grant and why; the senior runs hq lanes grant ${parent.lane_id} --hire <n>.`, {
|
|
34
34
|
senior_lane_id: parent.lane_id,
|
|
@@ -199,6 +199,7 @@ export function buildOutpostLaneCommand(input) {
|
|
|
199
199
|
["--text", input.text],
|
|
200
200
|
["--project", input.project],
|
|
201
201
|
["--story", input.story],
|
|
202
|
+
["--focus-override", input.focusOverride],
|
|
202
203
|
["--account", input.account],
|
|
203
204
|
["--model", input.model],
|
|
204
205
|
["--effort", input.effort],
|
|
@@ -23,7 +23,7 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
23
23
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
24
24
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
25
25
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
26
|
-
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, } from "./flag-keys.js";
|
|
26
|
+
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
|
|
27
27
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
|
|
28
28
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
29
29
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
@@ -66,6 +66,43 @@ function spawnDepsForHost(deps) {
|
|
|
66
66
|
admissionHost: deps.host,
|
|
67
67
|
};
|
|
68
68
|
}
|
|
69
|
+
async function resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now) {
|
|
70
|
+
const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
71
|
+
for (const lane of lanes) {
|
|
72
|
+
const resetAt = typeof lane.quota_reset_at === "string" ? Date.parse(lane.quota_reset_at) : NaN;
|
|
73
|
+
if (lane.state !== "interrupted" ||
|
|
74
|
+
!["quota_held", "account_quota_exhausted", "account_model_unavailable"].includes(lane.terminal_reason ?? "") ||
|
|
75
|
+
!Number.isFinite(resetAt) || resetAt > now.getTime() || !laneRunsHere(lane, deps.host))
|
|
76
|
+
continue;
|
|
77
|
+
try {
|
|
78
|
+
const company = resolveLaneCompany(hqRoot, lane);
|
|
79
|
+
const enabled = await readLanesRegistryFlagFailOff(LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, {
|
|
80
|
+
company: company.company,
|
|
81
|
+
companyUidSource: laneCompanyUidSource(company.source),
|
|
82
|
+
env: deps.env,
|
|
83
|
+
}, readFlag);
|
|
84
|
+
if (!enabled)
|
|
85
|
+
continue;
|
|
86
|
+
const recovered = await retryLaneOnAlternateAccount(hqRoot, lane, "usage_limit", "reconcile", { ...spawnDepsForHost(deps), now: () => now, quotaAccountFailoverEnabled: true, dryRun: deps.dryRun });
|
|
87
|
+
summary.resumed.push({
|
|
88
|
+
id: lane.lane_id,
|
|
89
|
+
from: "interrupted",
|
|
90
|
+
to: recovered.state,
|
|
91
|
+
account: recovered.account,
|
|
92
|
+
reason: recovered.terminal_reason === "quota_held" ? "quota_held" : "quota_reset",
|
|
93
|
+
...(recovered.quota_reset_at ? { quota_reset_at: recovered.quota_reset_at } : {}),
|
|
94
|
+
});
|
|
95
|
+
if (!deps.dryRun)
|
|
96
|
+
summary.heartbeats.push(`${lane.lane_id}:quota-account-retry`);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
100
|
+
summary.errors.push({ message: `${lane.lane_id}: quota retry failed: ${message}` });
|
|
101
|
+
summary.ok = false;
|
|
102
|
+
process.stderr.write(`hq lanes reconcile: quota retry failed for ${lane.lane_id}: ${message}\n`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
69
106
|
function logSkip(skip) {
|
|
70
107
|
process.stderr.write(`hq lanes reconcile: skipped ${skip.kind}${skip.id ? ` ${skip.id}` : ""}: ${skip.why}\n`);
|
|
71
108
|
}
|
|
@@ -940,6 +977,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
940
977
|
if (deps.terminalKeepAliveCandidate) {
|
|
941
978
|
return runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, now, deps.terminalKeepAliveCandidate);
|
|
942
979
|
}
|
|
980
|
+
await resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now);
|
|
943
981
|
let presenceConnected;
|
|
944
982
|
try {
|
|
945
983
|
presenceConnected =
|
|
@@ -66,6 +66,10 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
|
|
|
66
66
|
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
|
|
67
67
|
export interface SpawnLaneDeps {
|
|
68
68
|
env?: NodeJS.ProcessEnv;
|
|
69
|
+
/** Test seam for the default-off quota account failover registry gate. */
|
|
70
|
+
quotaAccountFailoverEnabled?: boolean;
|
|
71
|
+
/** Return the selected retry transition without writing or starting a worker. */
|
|
72
|
+
dryRun?: boolean;
|
|
69
73
|
/** Absolute CLI entry captured when the watcher is armed; tests may inject it. */
|
|
70
74
|
hqCliEntry?: string;
|
|
71
75
|
hostname?: string;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -29,6 +29,9 @@ import { registryWorkerRecord } from "./registry-persona.js";
|
|
|
29
29
|
import { resolveMachineIdentity, } from "./machine-identity.js";
|
|
30
30
|
import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
|
|
31
31
|
import { managerStatusGateHeartbeat } from "./manager-status-gate.js";
|
|
32
|
+
import { LANE_QUOTA_ACCOUNT_FAILOVER_FLAG } from "./flag-keys.js";
|
|
33
|
+
import { readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
34
|
+
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
32
35
|
import { LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT } from "./keep-alive.js";
|
|
33
36
|
import { copySessionFiles, planSessionCopy } from "./move-account.js";
|
|
34
37
|
import { ensureSessionHome } from "./session-relink.js";
|
|
@@ -516,6 +519,19 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
516
519
|
const env = deps.env ?? process.env;
|
|
517
520
|
const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
|
|
518
521
|
const hostname = deps.hostname?.trim() || admissionHost.hostname.trim() || resolveAdmissionHost().hostname;
|
|
522
|
+
const now = (deps.now ?? (() => new Date()))();
|
|
523
|
+
const at = now.toISOString();
|
|
524
|
+
const nowMs = now.getTime();
|
|
525
|
+
const retryFromState = lane.state === "interrupted" &&
|
|
526
|
+
(lane.terminal_reason === "quota_held" ||
|
|
527
|
+
lane.terminal_reason === "account_quota_exhausted" ||
|
|
528
|
+
lane.terminal_reason === "account_model_unavailable")
|
|
529
|
+
? "interrupted"
|
|
530
|
+
: "running";
|
|
531
|
+
const retryingExpiredQuota = retryFromState === "interrupted" &&
|
|
532
|
+
typeof lane.quota_reset_at === "string" &&
|
|
533
|
+
Number.isFinite(Date.parse(lane.quota_reset_at)) &&
|
|
534
|
+
Date.parse(lane.quota_reset_at) <= nowMs;
|
|
519
535
|
const modelDeps = {
|
|
520
536
|
...deps.modelDeps,
|
|
521
537
|
env: deps.modelDeps?.env ?? env,
|
|
@@ -525,13 +541,38 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
525
541
|
env,
|
|
526
542
|
};
|
|
527
543
|
const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
|
|
544
|
+
let quotaFailoverEnabled = false;
|
|
545
|
+
if (reason === "usage_limit") {
|
|
546
|
+
if (deps.quotaAccountFailoverEnabled !== undefined) {
|
|
547
|
+
quotaFailoverEnabled = deps.quotaAccountFailoverEnabled;
|
|
548
|
+
}
|
|
549
|
+
else {
|
|
550
|
+
try {
|
|
551
|
+
const laneCompany = resolveLaneCompany(hqRoot, lane);
|
|
552
|
+
quotaFailoverEnabled = await readLanesRegistryFlagFailOff(LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, { company: laneCompany.company, companyUidSource: laneCompanyUidSource(laneCompany.source), env });
|
|
553
|
+
}
|
|
554
|
+
catch (error) {
|
|
555
|
+
process.stderr.write(`${JSON.stringify({
|
|
556
|
+
event: "lanes.quota_account_failover.company_context_failed",
|
|
557
|
+
error: error instanceof Error ? error.name : "NonError",
|
|
558
|
+
fallback: false,
|
|
559
|
+
})}\n`);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
528
563
|
const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
|
|
529
564
|
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
530
565
|
? lane.model
|
|
531
566
|
: undefined;
|
|
532
567
|
const unavailableAccounts = previouslyUnavailableAccounts(lane);
|
|
533
568
|
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
534
|
-
.filter(([id, account]) => id !== lane.account
|
|
569
|
+
.filter(([id, account]) => (id !== lane.account || (quotaFailoverEnabled && reason === "usage_limit" && retryingExpiredQuota)) &&
|
|
570
|
+
!unavailableAccounts.has(id) && account.enabled)
|
|
571
|
+
.filter(([id]) => !quotaFailoverEnabled ||
|
|
572
|
+
(profile?.meta.provider !== lane.provider ||
|
|
573
|
+
(profile.meta.accounts !== undefined
|
|
574
|
+
? profile.meta.accounts.includes(id)
|
|
575
|
+
: profile.meta.account === undefined || profile.meta.account === id)))
|
|
535
576
|
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
536
577
|
existsSync,
|
|
537
578
|
hostname,
|
|
@@ -545,6 +586,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
545
586
|
existsSync,
|
|
546
587
|
accountDeps,
|
|
547
588
|
hostname,
|
|
589
|
+
now,
|
|
548
590
|
...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
|
|
549
591
|
allowedAccounts,
|
|
550
592
|
};
|
|
@@ -553,11 +595,96 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
553
595
|
target = selectBalancedAccount(lane.provider, config, modelContext);
|
|
554
596
|
}
|
|
555
597
|
catch (err) {
|
|
598
|
+
if (quotaFailoverEnabled &&
|
|
599
|
+
reason === "usage_limit" &&
|
|
600
|
+
err instanceof AccountResolutionError &&
|
|
601
|
+
err.error === "account_quota_exhausted") {
|
|
602
|
+
const quotaAccountReasons = Array.isArray(err.details.account_reasons)
|
|
603
|
+
? err.details.account_reasons
|
|
604
|
+
: [];
|
|
605
|
+
const resetAt = quotaAccountReasons
|
|
606
|
+
.map((candidate) => candidate.quota_reset_at)
|
|
607
|
+
.filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)))
|
|
608
|
+
.sort((a, b) => Date.parse(a) - Date.parse(b))[0];
|
|
609
|
+
if (resetAt) {
|
|
610
|
+
const accountReasons = Object.entries(config.providers[lane.provider].accounts).map(([id, account]) => {
|
|
611
|
+
const quotaReason = quotaAccountReasons.find((candidate) => candidate.account === id);
|
|
612
|
+
if (quotaReason)
|
|
613
|
+
return quotaReason;
|
|
614
|
+
if (id === lane.account)
|
|
615
|
+
return { account: id, reason: "source_account_quota_exhausted" };
|
|
616
|
+
if (unavailableAccounts.has(id))
|
|
617
|
+
return { account: id, reason: "previously_unavailable" };
|
|
618
|
+
if (!account.enabled)
|
|
619
|
+
return { account: id, reason: "disabled" };
|
|
620
|
+
if (quotaFailoverEnabled && profile?.meta.provider === lane.provider &&
|
|
621
|
+
profile.meta.account !== undefined && profile.meta.account !== id) {
|
|
622
|
+
return { account: id, reason: "worker_profile_account_pin" };
|
|
623
|
+
}
|
|
624
|
+
if (quotaFailoverEnabled && profile?.meta.provider === lane.provider &&
|
|
625
|
+
profile.meta.accounts !== undefined && !profile.meta.accounts.includes(id)) {
|
|
626
|
+
return { account: id, reason: "worker_profile_account_not_allowed" };
|
|
627
|
+
}
|
|
628
|
+
if (accountStatus(lane.provider, id, config, {
|
|
629
|
+
existsSync,
|
|
630
|
+
hostname,
|
|
631
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
632
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
633
|
+
}).logged_in !== true)
|
|
634
|
+
return { account: id, reason: "login_required" };
|
|
635
|
+
if (!supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps, hostname)) {
|
|
636
|
+
return { account: id, reason: "model_or_pin_incompatible" };
|
|
637
|
+
}
|
|
638
|
+
return { account: id, reason: "eligible_but_not_selected" };
|
|
639
|
+
});
|
|
640
|
+
const holdLane = (current) => {
|
|
641
|
+
if (current.state !== retryFromState ||
|
|
642
|
+
current.account !== lane.account ||
|
|
643
|
+
current.provider_session_id !== lane.provider_session_id) {
|
|
644
|
+
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while quota recovery was prepared`, { lane_id: lane.lane_id, state: current.state, account: current.account ?? null });
|
|
645
|
+
}
|
|
646
|
+
return {
|
|
647
|
+
...current,
|
|
648
|
+
state: "interrupted",
|
|
649
|
+
terminal_reason: "quota_held",
|
|
650
|
+
terminal_details: {
|
|
651
|
+
reason: err.error,
|
|
652
|
+
account_reasons: accountReasons,
|
|
653
|
+
},
|
|
654
|
+
quota_reset_at: resetAt,
|
|
655
|
+
interrupted_at: at,
|
|
656
|
+
};
|
|
657
|
+
};
|
|
658
|
+
if (deps.dryRun)
|
|
659
|
+
return holdLane(lane);
|
|
660
|
+
return retryUpdateLane(hqRoot, lane.lane_id, writer, holdLane);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
556
663
|
if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
|
|
557
664
|
throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
|
|
558
665
|
}
|
|
559
666
|
throw err;
|
|
560
667
|
}
|
|
668
|
+
if (deps.dryRun) {
|
|
669
|
+
const entry = {
|
|
670
|
+
from: lane.account ?? null,
|
|
671
|
+
to: target.id,
|
|
672
|
+
at,
|
|
673
|
+
reason,
|
|
674
|
+
};
|
|
675
|
+
return {
|
|
676
|
+
...lane,
|
|
677
|
+
state: "queued",
|
|
678
|
+
queued_reason: "account_retry",
|
|
679
|
+
queued_at: at,
|
|
680
|
+
terminal_reason: undefined,
|
|
681
|
+
terminal_details: undefined,
|
|
682
|
+
interrupted_at: undefined,
|
|
683
|
+
quota_reset_at: undefined,
|
|
684
|
+
account: target.id,
|
|
685
|
+
account_history: appendCappedHistory(lane.account_history ?? [], entry),
|
|
686
|
+
};
|
|
687
|
+
}
|
|
561
688
|
const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
|
|
562
689
|
const sessionId = providerSupportsSessionResume(lane.provider) &&
|
|
563
690
|
typeof lane.provider_session_id === "string" &&
|
|
@@ -572,12 +699,9 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
572
699
|
sessionId,
|
|
573
700
|
}));
|
|
574
701
|
}
|
|
575
|
-
const now = (deps.now ?? (() => new Date()))();
|
|
576
|
-
const at = now.toISOString();
|
|
577
|
-
const nowMs = now.getTime();
|
|
578
702
|
return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
579
703
|
const current = readLane(hqRoot, lane.lane_id);
|
|
580
|
-
if (current.state !==
|
|
704
|
+
if (current.state !== retryFromState ||
|
|
581
705
|
current.account !== lane.account ||
|
|
582
706
|
current.provider_session_id !== lane.provider_session_id) {
|
|
583
707
|
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
|
@@ -603,6 +727,8 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
603
727
|
admission_reserved_at: undefined,
|
|
604
728
|
terminal_reason: undefined,
|
|
605
729
|
terminal_details: undefined,
|
|
730
|
+
interrupted_at: undefined,
|
|
731
|
+
quota_reset_at: undefined,
|
|
606
732
|
account: target.id,
|
|
607
733
|
account_history: appendCappedHistory(row.account_history ?? [], entry),
|
|
608
734
|
pending_resume: sessionId
|
|
@@ -621,7 +747,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
621
747
|
const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
|
|
622
748
|
const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
|
|
623
749
|
const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
|
|
624
|
-
if (row.state !==
|
|
750
|
+
if (row.state !== retryFromState ||
|
|
625
751
|
row.account !== lane.account ||
|
|
626
752
|
row.provider_session_id !== lane.provider_session_id) {
|
|
627
753
|
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|