@indigoai-us/hq-cli 5.345.43 → 5.345.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/command-catalog.generated.d.ts +17 -1
  3. package/dist/command-catalog.generated.js +21 -1
  4. package/dist/commands/access.js +6 -4
  5. package/dist/commands/bot.js +1 -1
  6. package/dist/commands/db-destroy.js +5 -4
  7. package/dist/commands/db-remote-deps.js +4 -0
  8. package/dist/commands/lanes-checks.d.ts +1 -0
  9. package/dist/commands/lanes-checks.js +6 -1
  10. package/dist/commands/lanes-message.js +84 -51
  11. package/dist/commands/lanes-report.d.ts +2 -1
  12. package/dist/commands/lanes-report.js +63 -6
  13. package/dist/commands/lanes-resume.js +44 -0
  14. package/dist/commands/lanes-shared.d.ts +25 -0
  15. package/dist/commands/lanes.d.ts +4 -0
  16. package/dist/commands/lanes.js +228 -9
  17. package/dist/commands/meetings.js +4 -1
  18. package/dist/index.js +43 -7
  19. package/dist/lib/core-utils/sentry-report.js +8 -1
  20. package/dist/lib/db/control-plane.d.ts +5 -0
  21. package/dist/lib/db/control-plane.js +7 -0
  22. package/dist/lib/flag-snapshot-file.js +33 -25
  23. package/dist/lib/lanes/flag-keys.d.ts +4 -0
  24. package/dist/lib/lanes/flag-keys.js +4 -0
  25. package/dist/lib/lanes/focus-guard.d.ts +65 -0
  26. package/dist/lib/lanes/focus-guard.js +91 -0
  27. package/dist/lib/lanes/grants.js +1 -1
  28. package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
  29. package/dist/lib/lanes/outpost-dispatch.js +1 -0
  30. package/dist/lib/lanes/reconcile.d.ts +2 -0
  31. package/dist/lib/lanes/reconcile.js +39 -1
  32. package/dist/lib/lanes/spawn.d.ts +4 -0
  33. package/dist/lib/lanes/spawn.js +132 -6
  34. package/dist/lib/lanes/store.js +64 -0
  35. package/dist/lib/lanes/story-ledger.d.ts +14 -0
  36. package/dist/lib/lanes/story-ledger.js +46 -0
  37. package/dist/lib/lanes/tracker-report.d.ts +46 -0
  38. package/dist/lib/lanes/tracker-report.js +85 -1
  39. package/dist/lib/lanes/types.d.ts +12 -2
  40. package/dist/lib/lanes/types.js +5 -3
  41. package/dist/main.js +5 -1
  42. package/dist/startup-registration.d.ts +3 -0
  43. package/dist/startup-registration.js +11 -4
  44. package/dist/utils/package-use-lease.d.ts +1 -0
  45. package/dist/utils/package-use-lease.js +29 -0
  46. package/dist/utils/self-update-backoff.d.ts +17 -0
  47. package/dist/utils/self-update-backoff.js +73 -0
  48. package/dist/utils/self-update.d.ts +7 -0
  49. package/dist/utils/self-update.js +36 -0
  50. package/dist/utils/vault-api.d.ts +33 -1
  51. package/dist/utils/vault-api.js +55 -5
  52. package/dist/utils/version-gate.d.ts +13 -0
  53. package/dist/utils/version-gate.js +35 -5
  54. package/package.json +1 -1
@@ -78,29 +78,38 @@ function isPermissionFailure(error) {
78
78
  const code = error?.code;
79
79
  return code === "EACCES" || code === "EPERM";
80
80
  }
81
- function fallbackSnapshotPath(companyUid, create) {
82
- if (typeof process.getuid !== "function")
81
+ function fallbackSnapshotPath(companyUid, create, env = process.env) {
82
+ const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
83
+ const root = uid === undefined
84
+ ? env.LOCALAPPDATA?.trim()
85
+ : path.join(os.tmpdir(), `hq-cli-flag-snapshots-${uid}`);
86
+ if (!root)
83
87
  return undefined;
84
- const uid = process.getuid();
85
- const root = path.join(os.tmpdir(), `hq-cli-flag-snapshots-${uid}`);
86
- const directories = [root, path.join(root, "hq"), path.join(root, "hq", "flags")];
88
+ const fallbackRoot = uid === undefined ? path.join(root, "hq-cli", "flag-snapshots") : root;
89
+ const directories = uid === undefined
90
+ ? [path.join(root, "hq-cli"), fallbackRoot]
91
+ : [fallbackRoot, path.join(fallbackRoot, "hq"), path.join(fallbackRoot, "hq", "flags")];
87
92
  for (const directory of directories) {
88
93
  try {
89
94
  if (create)
90
95
  fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
91
96
  const stat = fs.lstatSync(directory);
92
- if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== uid)
93
- return undefined;
94
- if (create)
95
- fs.chmodSync(directory, 0o700);
96
- else if ((stat.mode & 0o077) !== 0)
97
+ if (!stat.isDirectory() || stat.isSymbolicLink())
97
98
  return undefined;
99
+ if (uid !== undefined) {
100
+ if (stat.uid !== uid)
101
+ return undefined;
102
+ if (create)
103
+ fs.chmodSync(directory, 0o700);
104
+ else if ((stat.mode & 0o077) !== 0)
105
+ return undefined;
106
+ }
98
107
  }
99
108
  catch {
100
109
  return undefined;
101
110
  }
102
111
  }
103
- return path.join(directories[2], `snapshot-${companyUid}.json`);
112
+ return path.join(directories[directories.length - 1], `snapshot-${companyUid}.json`);
104
113
  }
105
114
  function fallbackPersonSnapshotPath(create) {
106
115
  if (typeof process.getuid !== "function")
@@ -275,7 +284,7 @@ export function writeFlagSnapshotFile(companyUid, flags, options = {}) {
275
284
  }
276
285
  const cacheRootKind = env.XDG_CACHE_HOME?.trim() ? "xdg" : "home";
277
286
  const writeFallback = (originalError) => {
278
- const fallbackFile = fallbackSnapshotPath(companyUid, true);
287
+ const fallbackFile = fallbackSnapshotPath(companyUid, true, env);
279
288
  if (!fallbackFile) {
280
289
  if (originalError) {
281
290
  markFallbackFailed(originalError);
@@ -347,21 +356,20 @@ export function readFlagSnapshotFile(companyUid, options = {}) {
347
356
  const file = flagSnapshotPath(companyUid, options.env);
348
357
  if (!file)
349
358
  return null;
350
- if (fallbackCacheUsed) {
351
- const fallback = fallbackSnapshotPath(companyUid, false);
352
- const snapshot = fallback ? parseSnapshotFile(companyUid, fallback, options) : null;
353
- if (snapshot)
354
- return snapshot;
355
- }
356
359
  const primary = parseSnapshotFile(companyUid, file, options);
357
- if (primary)
360
+ const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
361
+ if (primary && !fallbackCacheUsed && uid !== undefined)
358
362
  return primary;
359
- const fallback = fallbackSnapshotPath(companyUid, false);
360
- if (!fallback)
361
- return null;
362
- const snapshot = parseSnapshotFile(companyUid, fallback, options);
363
- if (snapshot)
363
+ const fallback = fallbackSnapshotPath(companyUid, false, options.env);
364
+ const fallbackSnapshot = fallback ? parseSnapshotFile(companyUid, fallback, options) : null;
365
+ // A process can restart after a denied Windows rename wrote a newer snapshot
366
+ // to LOCALAPPDATA. Compare both valid stores so an older primary does not
367
+ // mask that successful fallback write. Equal timestamps keep the primary.
368
+ if (fallbackSnapshot
369
+ && (!primary || Date.parse(fallbackSnapshot.fetched_at) > Date.parse(primary.fetched_at))) {
364
370
  fallbackCacheUsed = true;
365
- return snapshot;
371
+ return fallbackSnapshot;
372
+ }
373
+ return primary;
366
374
  }
367
375
  //# sourceMappingURL=flag-snapshot-file.js.map
@@ -1,5 +1,9 @@
1
1
  /** hq-flags keys used by lane command behavior. */
2
+ export declare const LANE_FOCUS_GUARD_FLAG = "lanes-focus-guard";
3
+ export declare const LANE_FOCUS_DASHBOARD_FLAG = "lanes-focus-dashboard";
2
4
  export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
5
+ /** Enables same-provider automatic lane recovery after provider quota exhaustion. */
6
+ export declare const LANE_QUOTA_ACCOUNT_FAILOVER_FLAG = "lanes.quota-account-failover";
3
7
  /** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
4
8
  export declare const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
5
9
  /** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
@@ -1,5 +1,9 @@
1
1
  /** hq-flags keys used by lane command behavior. */
2
+ export const LANE_FOCUS_GUARD_FLAG = "lanes-focus-guard";
3
+ export const LANE_FOCUS_DASHBOARD_FLAG = "lanes-focus-dashboard";
2
4
  export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
5
+ /** Enables same-provider automatic lane recovery after provider quota exhaustion. */
6
+ export const LANE_QUOTA_ACCOUNT_FAILOVER_FLAG = "lanes.quota-account-failover";
3
7
  /** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
4
8
  export const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
5
9
  /** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
@@ -0,0 +1,65 @@
1
+ import { LanesError } from "./errors.js";
2
+ export declare const FOCUS_GUARD_MAX_PR_AGE_MS: number;
3
+ export interface FocusGuardStory {
4
+ id: string;
5
+ focus_id?: unknown;
6
+ kpi?: unknown;
7
+ state?: unknown;
8
+ status?: unknown;
9
+ lane?: unknown;
10
+ lanes?: unknown;
11
+ }
12
+ export interface FocusGuardPullRequest {
13
+ number: number;
14
+ created_at: string;
15
+ repository?: string;
16
+ }
17
+ export interface FocusGuardLane {
18
+ lane_id: string;
19
+ state: string;
20
+ }
21
+ export type FocusGuardReason = {
22
+ code: "story_missing_focus";
23
+ story_id: string;
24
+ } | {
25
+ code: "open_pr_over_48h";
26
+ number: number;
27
+ age_hours: number;
28
+ repository?: string;
29
+ } | {
30
+ code: "stale_story";
31
+ story_id: string;
32
+ lane_id: string | null;
33
+ } | {
34
+ code: "pr_lookup_failed";
35
+ };
36
+ export interface FocusGuardOverride {
37
+ reason: string;
38
+ by_lane: string;
39
+ at: string;
40
+ story: string;
41
+ }
42
+ export declare function appendLaneFocusOverride<T extends {
43
+ focus_overrides?: FocusGuardOverride[];
44
+ }>(lane: T, override: FocusGuardOverride): T & {
45
+ focus_overrides: FocusGuardOverride[];
46
+ };
47
+ export declare function evaluateFocusGuard(input: {
48
+ storyIds: readonly string[];
49
+ stories: readonly FocusGuardStory[];
50
+ lanes: readonly FocusGuardLane[];
51
+ pullRequests?: readonly FocusGuardPullRequest[];
52
+ pullRequestLookupFailed?: boolean;
53
+ now: Date;
54
+ checkTeamBlockers: boolean;
55
+ }): FocusGuardReason[];
56
+ export declare function evaluateFocusGuardIfEnabled(enabled: boolean, input: Parameters<typeof evaluateFocusGuard>[0]): FocusGuardReason[];
57
+ export declare function assertFocusGuardAllows(enabled: boolean, input: Parameters<typeof evaluateFocusGuard>[0], overrideReason?: string): FocusGuardReason[];
58
+ export declare function focusGuardRefusal(reasons: readonly FocusGuardReason[]): LanesError;
59
+ export declare function makeFocusOverride(input: {
60
+ reason?: string;
61
+ laneId: string;
62
+ at: string;
63
+ storyId: string;
64
+ }): FocusGuardOverride | undefined;
65
+ //# sourceMappingURL=focus-guard.d.ts.map
@@ -0,0 +1,91 @@
1
+ import { LanesError } from "./errors.js";
2
+ import { isTerminalLaneState } from "./types.js";
3
+ export const FOCUS_GUARD_MAX_PR_AGE_MS = 48 * 60 * 60 * 1000;
4
+ export function appendLaneFocusOverride(lane, override) {
5
+ return { ...lane, focus_overrides: [...(lane.focus_overrides ?? []), override].slice(-100) };
6
+ }
7
+ function nonempty(value) {
8
+ return typeof value === "string" && value.trim().length > 0;
9
+ }
10
+ function storyLaneIds(story) {
11
+ const candidates = [story.lane, ...(Array.isArray(story.lanes) ? story.lanes : [])];
12
+ return [...new Set(candidates.flatMap((item) => {
13
+ if (typeof item === "string" && item.trim())
14
+ return [item.trim()];
15
+ if (item !== null && typeof item === "object" && typeof item.lane === "string") {
16
+ const lane = item.lane.trim();
17
+ return lane ? [lane] : [];
18
+ }
19
+ if (item !== null && typeof item === "object" && typeof item.lane_id === "string") {
20
+ const laneId = item.lane_id.trim();
21
+ return laneId ? [laneId] : [];
22
+ }
23
+ return [];
24
+ }))];
25
+ }
26
+ export function evaluateFocusGuard(input) {
27
+ const reasons = [];
28
+ const storyById = new Map(input.stories.map((story) => [story.id.trim(), story]));
29
+ for (const id of input.storyIds) {
30
+ const story = storyById.get(id);
31
+ if (!story || (!nonempty(story.focus_id) && !nonempty(story.kpi))) {
32
+ reasons.push({ code: "story_missing_focus", story_id: id });
33
+ }
34
+ }
35
+ if (!input.checkTeamBlockers)
36
+ return reasons;
37
+ if (input.pullRequestLookupFailed) {
38
+ reasons.push({ code: "pr_lookup_failed" });
39
+ }
40
+ else {
41
+ for (const pr of input.pullRequests ?? []) {
42
+ const ageMs = input.now.getTime() - Date.parse(pr.created_at);
43
+ if (Number.isFinite(ageMs) && ageMs > FOCUS_GUARD_MAX_PR_AGE_MS) {
44
+ reasons.push({
45
+ code: "open_pr_over_48h",
46
+ number: pr.number,
47
+ age_hours: Math.floor(ageMs / (60 * 60 * 1000)),
48
+ ...(pr.repository === undefined ? {} : { repository: pr.repository }),
49
+ });
50
+ }
51
+ }
52
+ }
53
+ const liveLaneIds = new Set(input.lanes
54
+ .filter((lane) => !isTerminalLaneState(lane.state))
55
+ .map((lane) => lane.lane_id));
56
+ for (const story of input.stories) {
57
+ const storyStatus = typeof story.state === "string" ? story.state : story.status;
58
+ if (storyStatus !== "in_progress")
59
+ continue;
60
+ const assigned = storyLaneIds(story);
61
+ if (assigned.length === 0 || assigned.every((laneId) => !liveLaneIds.has(laneId))) {
62
+ reasons.push({ code: "stale_story", story_id: story.id, lane_id: assigned[0] ?? null });
63
+ }
64
+ }
65
+ return reasons;
66
+ }
67
+ export function evaluateFocusGuardIfEnabled(enabled, input) {
68
+ return enabled ? evaluateFocusGuard(input) : [];
69
+ }
70
+ export function assertFocusGuardAllows(enabled, input, overrideReason) {
71
+ const reasons = evaluateFocusGuardIfEnabled(enabled, input);
72
+ if (enabled && reasons.length > 0 && overrideReason === undefined)
73
+ throw focusGuardRefusal(reasons);
74
+ return reasons;
75
+ }
76
+ export function focusGuardRefusal(reasons) {
77
+ return new LanesError("focus_guard_refused", `Lane launch refused by focus guard: ${reasons.map((reason) => reason.code).join(", ")}`, { reasons });
78
+ }
79
+ export function makeFocusOverride(input) {
80
+ if (input.reason === undefined)
81
+ return undefined;
82
+ const reason = input.reason.trim();
83
+ if (!reason) {
84
+ throw new LanesError("focus_override_reason_required", "--focus-override requires a non-empty reason", {
85
+ lane_id: input.laneId,
86
+ story_id: input.storyId,
87
+ });
88
+ }
89
+ return { reason, by_lane: input.laneId, at: input.at, story: input.storyId };
90
+ }
91
+ //# sourceMappingURL=focus-guard.js.map
@@ -28,7 +28,7 @@ export function terminalChildResumeHireGrantError(current, snapshot, finishedChi
28
28
  }
29
29
  if (children.length >= parentGrant)
30
30
  return hireGrantCapacityError(parent, children);
31
- const requestedGrant = current.hire_grant ?? 0;
31
+ const requestedGrant = current.released_hire_grant ?? current.hire_grant ?? 0;
32
32
  if (requestedGrant > availableChildGrant) {
33
33
  return new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${availableChildGrant} unassigned hire grant remaining, but ${terminal ? "terminal" : "finished"} child ${current.lane_id} needs ${requestedGrant} to resume. Ask its senior for more: write an ask envelope with the desired total grant and why; the senior runs hq lanes grant ${parent.lane_id} --hire <n>.`, {
34
34
  senior_lane_id: parent.lane_id,
@@ -33,6 +33,7 @@ export interface OutpostLaneCommand {
33
33
  text?: string;
34
34
  project?: string;
35
35
  story?: string;
36
+ focusOverride?: string;
36
37
  account?: string;
37
38
  model?: string;
38
39
  effort?: string;
@@ -199,6 +199,7 @@ export function buildOutpostLaneCommand(input) {
199
199
  ["--text", input.text],
200
200
  ["--project", input.project],
201
201
  ["--story", input.story],
202
+ ["--focus-override", input.focusOverride],
202
203
  ["--account", input.account],
203
204
  ["--model", input.model],
204
205
  ["--effort", input.effort],
@@ -35,6 +35,8 @@ export interface ReconcileChange {
35
35
  from?: string;
36
36
  to?: string;
37
37
  reason?: string;
38
+ account?: string;
39
+ quota_reset_at?: string;
38
40
  /** The attempted provider PATH when admission found no executable. */
39
41
  searched_path?: string;
40
42
  }
@@ -23,7 +23,7 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
23
23
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
24
24
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
25
25
  import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
26
- import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, } from "./flag-keys.js";
26
+ import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
27
27
  import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
28
28
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
29
29
  import { appendDropboxMessage } from "./dropbox.js";
@@ -66,6 +66,43 @@ function spawnDepsForHost(deps) {
66
66
  admissionHost: deps.host,
67
67
  };
68
68
  }
69
+ async function resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now) {
70
+ const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
71
+ for (const lane of lanes) {
72
+ const resetAt = typeof lane.quota_reset_at === "string" ? Date.parse(lane.quota_reset_at) : NaN;
73
+ if (lane.state !== "interrupted" ||
74
+ !["quota_held", "account_quota_exhausted", "account_model_unavailable"].includes(lane.terminal_reason ?? "") ||
75
+ !Number.isFinite(resetAt) || resetAt > now.getTime() || !laneRunsHere(lane, deps.host))
76
+ continue;
77
+ try {
78
+ const company = resolveLaneCompany(hqRoot, lane);
79
+ const enabled = await readLanesRegistryFlagFailOff(LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, {
80
+ company: company.company,
81
+ companyUidSource: laneCompanyUidSource(company.source),
82
+ env: deps.env,
83
+ }, readFlag);
84
+ if (!enabled)
85
+ continue;
86
+ const recovered = await retryLaneOnAlternateAccount(hqRoot, lane, "usage_limit", "reconcile", { ...spawnDepsForHost(deps), now: () => now, quotaAccountFailoverEnabled: true, dryRun: deps.dryRun });
87
+ summary.resumed.push({
88
+ id: lane.lane_id,
89
+ from: "interrupted",
90
+ to: recovered.state,
91
+ account: recovered.account,
92
+ reason: recovered.terminal_reason === "quota_held" ? "quota_held" : "quota_reset",
93
+ ...(recovered.quota_reset_at ? { quota_reset_at: recovered.quota_reset_at } : {}),
94
+ });
95
+ if (!deps.dryRun)
96
+ summary.heartbeats.push(`${lane.lane_id}:quota-account-retry`);
97
+ }
98
+ catch (error) {
99
+ const message = error instanceof Error ? error.message : String(error);
100
+ summary.errors.push({ message: `${lane.lane_id}: quota retry failed: ${message}` });
101
+ summary.ok = false;
102
+ process.stderr.write(`hq lanes reconcile: quota retry failed for ${lane.lane_id}: ${message}\n`);
103
+ }
104
+ }
105
+ }
69
106
  function logSkip(skip) {
70
107
  process.stderr.write(`hq lanes reconcile: skipped ${skip.kind}${skip.id ? ` ${skip.id}` : ""}: ${skip.why}\n`);
71
108
  }
@@ -940,6 +977,7 @@ export async function runPassBody(hqRoot, deps) {
940
977
  if (deps.terminalKeepAliveCandidate) {
941
978
  return runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, now, deps.terminalKeepAliveCandidate);
942
979
  }
980
+ await resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now);
943
981
  let presenceConnected;
944
982
  try {
945
983
  presenceConnected =
@@ -66,6 +66,10 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
66
66
  export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
67
67
  export interface SpawnLaneDeps {
68
68
  env?: NodeJS.ProcessEnv;
69
+ /** Test seam for the default-off quota account failover registry gate. */
70
+ quotaAccountFailoverEnabled?: boolean;
71
+ /** Return the selected retry transition without writing or starting a worker. */
72
+ dryRun?: boolean;
69
73
  /** Absolute CLI entry captured when the watcher is armed; tests may inject it. */
70
74
  hqCliEntry?: string;
71
75
  hostname?: string;
@@ -29,6 +29,9 @@ import { registryWorkerRecord } from "./registry-persona.js";
29
29
  import { resolveMachineIdentity, } from "./machine-identity.js";
30
30
  import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
31
31
  import { managerStatusGateHeartbeat } from "./manager-status-gate.js";
32
+ import { LANE_QUOTA_ACCOUNT_FAILOVER_FLAG } from "./flag-keys.js";
33
+ import { readLanesRegistryFlagFailOff } from "./registry-flag.js";
34
+ import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
32
35
  import { LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT } from "./keep-alive.js";
33
36
  import { copySessionFiles, planSessionCopy } from "./move-account.js";
34
37
  import { ensureSessionHome } from "./session-relink.js";
@@ -516,6 +519,19 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
516
519
  const env = deps.env ?? process.env;
517
520
  const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
518
521
  const hostname = deps.hostname?.trim() || admissionHost.hostname.trim() || resolveAdmissionHost().hostname;
522
+ const now = (deps.now ?? (() => new Date()))();
523
+ const at = now.toISOString();
524
+ const nowMs = now.getTime();
525
+ const retryFromState = lane.state === "interrupted" &&
526
+ (lane.terminal_reason === "quota_held" ||
527
+ lane.terminal_reason === "account_quota_exhausted" ||
528
+ lane.terminal_reason === "account_model_unavailable")
529
+ ? "interrupted"
530
+ : "running";
531
+ const retryingExpiredQuota = retryFromState === "interrupted" &&
532
+ typeof lane.quota_reset_at === "string" &&
533
+ Number.isFinite(Date.parse(lane.quota_reset_at)) &&
534
+ Date.parse(lane.quota_reset_at) <= nowMs;
519
535
  const modelDeps = {
520
536
  ...deps.modelDeps,
521
537
  env: deps.modelDeps?.env ?? env,
@@ -525,13 +541,38 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
525
541
  env,
526
542
  };
527
543
  const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
544
+ let quotaFailoverEnabled = false;
545
+ if (reason === "usage_limit") {
546
+ if (deps.quotaAccountFailoverEnabled !== undefined) {
547
+ quotaFailoverEnabled = deps.quotaAccountFailoverEnabled;
548
+ }
549
+ else {
550
+ try {
551
+ const laneCompany = resolveLaneCompany(hqRoot, lane);
552
+ quotaFailoverEnabled = await readLanesRegistryFlagFailOff(LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, { company: laneCompany.company, companyUidSource: laneCompanyUidSource(laneCompany.source), env });
553
+ }
554
+ catch (error) {
555
+ process.stderr.write(`${JSON.stringify({
556
+ event: "lanes.quota_account_failover.company_context_failed",
557
+ error: error instanceof Error ? error.name : "NonError",
558
+ fallback: false,
559
+ })}\n`);
560
+ }
561
+ }
562
+ }
528
563
  const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
529
564
  const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
530
565
  ? lane.model
531
566
  : undefined;
532
567
  const unavailableAccounts = previouslyUnavailableAccounts(lane);
533
568
  const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
534
- .filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
569
+ .filter(([id, account]) => (id !== lane.account || (quotaFailoverEnabled && reason === "usage_limit" && retryingExpiredQuota)) &&
570
+ !unavailableAccounts.has(id) && account.enabled)
571
+ .filter(([id]) => !quotaFailoverEnabled ||
572
+ (profile?.meta.provider !== lane.provider ||
573
+ (profile.meta.accounts !== undefined
574
+ ? profile.meta.accounts.includes(id)
575
+ : profile.meta.account === undefined || profile.meta.account === id)))
535
576
  .filter(([id]) => accountStatus(lane.provider, id, config, {
536
577
  existsSync,
537
578
  hostname,
@@ -545,6 +586,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
545
586
  existsSync,
546
587
  accountDeps,
547
588
  hostname,
589
+ now,
548
590
  ...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
549
591
  allowedAccounts,
550
592
  };
@@ -553,11 +595,96 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
553
595
  target = selectBalancedAccount(lane.provider, config, modelContext);
554
596
  }
555
597
  catch (err) {
598
+ if (quotaFailoverEnabled &&
599
+ reason === "usage_limit" &&
600
+ err instanceof AccountResolutionError &&
601
+ err.error === "account_quota_exhausted") {
602
+ const quotaAccountReasons = Array.isArray(err.details.account_reasons)
603
+ ? err.details.account_reasons
604
+ : [];
605
+ const resetAt = quotaAccountReasons
606
+ .map((candidate) => candidate.quota_reset_at)
607
+ .filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)))
608
+ .sort((a, b) => Date.parse(a) - Date.parse(b))[0];
609
+ if (resetAt) {
610
+ const accountReasons = Object.entries(config.providers[lane.provider].accounts).map(([id, account]) => {
611
+ const quotaReason = quotaAccountReasons.find((candidate) => candidate.account === id);
612
+ if (quotaReason)
613
+ return quotaReason;
614
+ if (id === lane.account)
615
+ return { account: id, reason: "source_account_quota_exhausted" };
616
+ if (unavailableAccounts.has(id))
617
+ return { account: id, reason: "previously_unavailable" };
618
+ if (!account.enabled)
619
+ return { account: id, reason: "disabled" };
620
+ if (quotaFailoverEnabled && profile?.meta.provider === lane.provider &&
621
+ profile.meta.account !== undefined && profile.meta.account !== id) {
622
+ return { account: id, reason: "worker_profile_account_pin" };
623
+ }
624
+ if (quotaFailoverEnabled && profile?.meta.provider === lane.provider &&
625
+ profile.meta.accounts !== undefined && !profile.meta.accounts.includes(id)) {
626
+ return { account: id, reason: "worker_profile_account_not_allowed" };
627
+ }
628
+ if (accountStatus(lane.provider, id, config, {
629
+ existsSync,
630
+ hostname,
631
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
632
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
633
+ }).logged_in !== true)
634
+ return { account: id, reason: "login_required" };
635
+ if (!supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps, hostname)) {
636
+ return { account: id, reason: "model_or_pin_incompatible" };
637
+ }
638
+ return { account: id, reason: "eligible_but_not_selected" };
639
+ });
640
+ const holdLane = (current) => {
641
+ if (current.state !== retryFromState ||
642
+ current.account !== lane.account ||
643
+ current.provider_session_id !== lane.provider_session_id) {
644
+ throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while quota recovery was prepared`, { lane_id: lane.lane_id, state: current.state, account: current.account ?? null });
645
+ }
646
+ return {
647
+ ...current,
648
+ state: "interrupted",
649
+ terminal_reason: "quota_held",
650
+ terminal_details: {
651
+ reason: err.error,
652
+ account_reasons: accountReasons,
653
+ },
654
+ quota_reset_at: resetAt,
655
+ interrupted_at: at,
656
+ };
657
+ };
658
+ if (deps.dryRun)
659
+ return holdLane(lane);
660
+ return retryUpdateLane(hqRoot, lane.lane_id, writer, holdLane);
661
+ }
662
+ }
556
663
  if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
557
664
  throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
558
665
  }
559
666
  throw err;
560
667
  }
668
+ if (deps.dryRun) {
669
+ const entry = {
670
+ from: lane.account ?? null,
671
+ to: target.id,
672
+ at,
673
+ reason,
674
+ };
675
+ return {
676
+ ...lane,
677
+ state: "queued",
678
+ queued_reason: "account_retry",
679
+ queued_at: at,
680
+ terminal_reason: undefined,
681
+ terminal_details: undefined,
682
+ interrupted_at: undefined,
683
+ quota_reset_at: undefined,
684
+ account: target.id,
685
+ account_history: appendCappedHistory(lane.account_history ?? [], entry),
686
+ };
687
+ }
561
688
  const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
562
689
  const sessionId = providerSupportsSessionResume(lane.provider) &&
563
690
  typeof lane.provider_session_id === "string" &&
@@ -572,12 +699,9 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
572
699
  sessionId,
573
700
  }));
574
701
  }
575
- const now = (deps.now ?? (() => new Date()))();
576
- const at = now.toISOString();
577
- const nowMs = now.getTime();
578
702
  return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
579
703
  const current = readLane(hqRoot, lane.lane_id);
580
- if (current.state !== "running" ||
704
+ if (current.state !== retryFromState ||
581
705
  current.account !== lane.account ||
582
706
  current.provider_session_id !== lane.provider_session_id) {
583
707
  throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
@@ -603,6 +727,8 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
603
727
  admission_reserved_at: undefined,
604
728
  terminal_reason: undefined,
605
729
  terminal_details: undefined,
730
+ interrupted_at: undefined,
731
+ quota_reset_at: undefined,
606
732
  account: target.id,
607
733
  account_history: appendCappedHistory(row.account_history ?? [], entry),
608
734
  pending_resume: sessionId
@@ -621,7 +747,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
621
747
  const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
622
748
  const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
623
749
  const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
624
- if (row.state !== "running" ||
750
+ if (row.state !== retryFromState ||
625
751
  row.account !== lane.account ||
626
752
  row.provider_session_id !== lane.provider_session_id) {
627
753
  throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {