@indigoai-us/hq-cli 5.345.42 → 5.345.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/command-catalog.generated.d.ts +2 -2
- package/dist/command-catalog.generated.js +2 -2
- package/dist/commands/db-remote-deps.js +4 -0
- package/dist/commands/install-global.d.ts +8 -1
- package/dist/commands/install-global.js +128 -25
- package/dist/commands/lanes-checks.d.ts +1 -0
- package/dist/commands/lanes-checks.js +6 -1
- package/dist/commands/lanes-message.js +1 -0
- package/dist/commands/lanes-report.d.ts +2 -1
- package/dist/commands/lanes-report.js +63 -6
- package/dist/commands/lanes-resume.js +3 -0
- package/dist/commands/lanes.js +218 -49
- package/dist/index.js +43 -7
- package/dist/lib/anywhere-person-setting.d.ts +2 -0
- package/dist/lib/anywhere-person-setting.js +2 -1
- package/dist/lib/anywhere-runtime-fast-flag.js +8 -2
- package/dist/lib/core-utils/sentry-report.js +8 -1
- package/dist/lib/daemon/anywhere-listener.d.ts +16 -0
- package/dist/lib/daemon/anywhere-listener.js +79 -0
- package/dist/lib/daemon/run.js +30 -23
- package/dist/lib/db/control-plane.d.ts +5 -0
- package/dist/lib/db/control-plane.js +7 -0
- package/dist/lib/doctor/checks/anywhere.d.ts +10 -0
- package/dist/lib/doctor/checks/anywhere.js +45 -17
- package/dist/lib/flag-registry.d.ts +4 -0
- package/dist/lib/flag-registry.js +43 -2
- package/dist/lib/flag-snapshot-file.d.ts +17 -0
- package/dist/lib/flag-snapshot-file.js +153 -37
- package/dist/lib/lanes/flag-keys.d.ts +5 -0
- package/dist/lib/lanes/flag-keys.js +5 -0
- package/dist/lib/lanes/grants.js +1 -1
- package/dist/lib/lanes/reconcile.d.ts +2 -0
- package/dist/lib/lanes/reconcile.js +39 -1
- package/dist/lib/lanes/spawn.d.ts +4 -0
- package/dist/lib/lanes/spawn.js +132 -6
- package/dist/lib/lanes/store.js +35 -0
- package/dist/lib/lanes/tracker-report.d.ts +46 -0
- package/dist/lib/lanes/tracker-report.js +85 -1
- package/dist/lib/lanes/types.d.ts +3 -1
- package/dist/lib/lanes/types.js +1 -0
- package/dist/main.js +5 -1
- package/dist/startup-registration.d.ts +3 -0
- package/dist/startup-registration.js +11 -4
- package/dist/utils/resident-package-preload.js +5 -1
- package/package.json +1 -1
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
/** hq-flags keys used by lane command behavior. */
|
|
2
|
+
export const LANE_FOCUS_DASHBOARD_FLAG = "lanes-focus-dashboard";
|
|
2
3
|
export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
4
|
+
/** Enables same-provider automatic lane recovery after provider quota exhaustion. */
|
|
5
|
+
export const LANE_QUOTA_ACCOUNT_FAILOVER_FLAG = "lanes.quota-account-failover";
|
|
3
6
|
/** Gives live senior lanes Claude admission precedence and reserves host capacity for them. */
|
|
4
7
|
export const LANE_LEAD_ADMISSION_PRIORITY_FLAG = "lanes.lead-admission-priority";
|
|
5
8
|
/** Refresh provider/model/effort from a worker profile when a done lane takes a new story. */
|
|
@@ -24,4 +27,6 @@ export const LEAD_AUTOPILOT_HEADLINE_STALENESS_FLAG = "lanes.autopilot-headline-
|
|
|
24
27
|
export const LEAD_AUTOPILOT_OUTPOST_NATIVE_LEADS_FLAG = "lanes.autopilot-outpost-native-leads";
|
|
25
28
|
/** Allows applying an operator-reviewed repair of messages stranded in bare-ULID dropboxes. */
|
|
26
29
|
export const LANE_ORPHANED_BARE_ID_DROPBOX_REPAIR_FLAG = "lanes.orphaned-bare-id-dropbox-repair";
|
|
30
|
+
/** Allows a Claude lane to leave an exhausted profile account pin for an eligible account. */
|
|
31
|
+
export const LANE_PROFILE_PIN_EXHAUSTED_FALLBACK_FLAG = "lanes.profile-pin-exhausted-fallback";
|
|
27
32
|
//# sourceMappingURL=flag-keys.js.map
|
package/dist/lib/lanes/grants.js
CHANGED
|
@@ -28,7 +28,7 @@ export function terminalChildResumeHireGrantError(current, snapshot, finishedChi
|
|
|
28
28
|
}
|
|
29
29
|
if (children.length >= parentGrant)
|
|
30
30
|
return hireGrantCapacityError(parent, children);
|
|
31
|
-
const requestedGrant = current.hire_grant ?? 0;
|
|
31
|
+
const requestedGrant = current.released_hire_grant ?? current.hire_grant ?? 0;
|
|
32
32
|
if (requestedGrant > availableChildGrant) {
|
|
33
33
|
return new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${availableChildGrant} unassigned hire grant remaining, but ${terminal ? "terminal" : "finished"} child ${current.lane_id} needs ${requestedGrant} to resume. Ask its senior for more: write an ask envelope with the desired total grant and why; the senior runs hq lanes grant ${parent.lane_id} --hire <n>.`, {
|
|
34
34
|
senior_lane_id: parent.lane_id,
|
|
@@ -23,7 +23,7 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
|
|
|
23
23
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
24
24
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
25
25
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
26
|
-
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, } from "./flag-keys.js";
|
|
26
|
+
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
|
|
27
27
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
|
|
28
28
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
29
29
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
@@ -66,6 +66,43 @@ function spawnDepsForHost(deps) {
|
|
|
66
66
|
admissionHost: deps.host,
|
|
67
67
|
};
|
|
68
68
|
}
|
|
69
|
+
async function resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now) {
|
|
70
|
+
const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
71
|
+
for (const lane of lanes) {
|
|
72
|
+
const resetAt = typeof lane.quota_reset_at === "string" ? Date.parse(lane.quota_reset_at) : NaN;
|
|
73
|
+
if (lane.state !== "interrupted" ||
|
|
74
|
+
!["quota_held", "account_quota_exhausted", "account_model_unavailable"].includes(lane.terminal_reason ?? "") ||
|
|
75
|
+
!Number.isFinite(resetAt) || resetAt > now.getTime() || !laneRunsHere(lane, deps.host))
|
|
76
|
+
continue;
|
|
77
|
+
try {
|
|
78
|
+
const company = resolveLaneCompany(hqRoot, lane);
|
|
79
|
+
const enabled = await readLanesRegistryFlagFailOff(LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, {
|
|
80
|
+
company: company.company,
|
|
81
|
+
companyUidSource: laneCompanyUidSource(company.source),
|
|
82
|
+
env: deps.env,
|
|
83
|
+
}, readFlag);
|
|
84
|
+
if (!enabled)
|
|
85
|
+
continue;
|
|
86
|
+
const recovered = await retryLaneOnAlternateAccount(hqRoot, lane, "usage_limit", "reconcile", { ...spawnDepsForHost(deps), now: () => now, quotaAccountFailoverEnabled: true, dryRun: deps.dryRun });
|
|
87
|
+
summary.resumed.push({
|
|
88
|
+
id: lane.lane_id,
|
|
89
|
+
from: "interrupted",
|
|
90
|
+
to: recovered.state,
|
|
91
|
+
account: recovered.account,
|
|
92
|
+
reason: recovered.terminal_reason === "quota_held" ? "quota_held" : "quota_reset",
|
|
93
|
+
...(recovered.quota_reset_at ? { quota_reset_at: recovered.quota_reset_at } : {}),
|
|
94
|
+
});
|
|
95
|
+
if (!deps.dryRun)
|
|
96
|
+
summary.heartbeats.push(`${lane.lane_id}:quota-account-retry`);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
100
|
+
summary.errors.push({ message: `${lane.lane_id}: quota retry failed: ${message}` });
|
|
101
|
+
summary.ok = false;
|
|
102
|
+
process.stderr.write(`hq lanes reconcile: quota retry failed for ${lane.lane_id}: ${message}\n`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
69
106
|
function logSkip(skip) {
|
|
70
107
|
process.stderr.write(`hq lanes reconcile: skipped ${skip.kind}${skip.id ? ` ${skip.id}` : ""}: ${skip.why}\n`);
|
|
71
108
|
}
|
|
@@ -940,6 +977,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
940
977
|
if (deps.terminalKeepAliveCandidate) {
|
|
941
978
|
return runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, now, deps.terminalKeepAliveCandidate);
|
|
942
979
|
}
|
|
980
|
+
await resumeDueQuotaHeldLanes(hqRoot, lanes, summary, deps, now);
|
|
943
981
|
let presenceConnected;
|
|
944
982
|
try {
|
|
945
983
|
presenceConnected =
|
|
@@ -66,6 +66,10 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
|
|
|
66
66
|
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
|
|
67
67
|
export interface SpawnLaneDeps {
|
|
68
68
|
env?: NodeJS.ProcessEnv;
|
|
69
|
+
/** Test seam for the default-off quota account failover registry gate. */
|
|
70
|
+
quotaAccountFailoverEnabled?: boolean;
|
|
71
|
+
/** Return the selected retry transition without writing or starting a worker. */
|
|
72
|
+
dryRun?: boolean;
|
|
69
73
|
/** Absolute CLI entry captured when the watcher is armed; tests may inject it. */
|
|
70
74
|
hqCliEntry?: string;
|
|
71
75
|
hostname?: string;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -29,6 +29,9 @@ import { registryWorkerRecord } from "./registry-persona.js";
|
|
|
29
29
|
import { resolveMachineIdentity, } from "./machine-identity.js";
|
|
30
30
|
import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
|
|
31
31
|
import { managerStatusGateHeartbeat } from "./manager-status-gate.js";
|
|
32
|
+
import { LANE_QUOTA_ACCOUNT_FAILOVER_FLAG } from "./flag-keys.js";
|
|
33
|
+
import { readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
34
|
+
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
32
35
|
import { LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT } from "./keep-alive.js";
|
|
33
36
|
import { copySessionFiles, planSessionCopy } from "./move-account.js";
|
|
34
37
|
import { ensureSessionHome } from "./session-relink.js";
|
|
@@ -516,6 +519,19 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
516
519
|
const env = deps.env ?? process.env;
|
|
517
520
|
const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
|
|
518
521
|
const hostname = deps.hostname?.trim() || admissionHost.hostname.trim() || resolveAdmissionHost().hostname;
|
|
522
|
+
const now = (deps.now ?? (() => new Date()))();
|
|
523
|
+
const at = now.toISOString();
|
|
524
|
+
const nowMs = now.getTime();
|
|
525
|
+
const retryFromState = lane.state === "interrupted" &&
|
|
526
|
+
(lane.terminal_reason === "quota_held" ||
|
|
527
|
+
lane.terminal_reason === "account_quota_exhausted" ||
|
|
528
|
+
lane.terminal_reason === "account_model_unavailable")
|
|
529
|
+
? "interrupted"
|
|
530
|
+
: "running";
|
|
531
|
+
const retryingExpiredQuota = retryFromState === "interrupted" &&
|
|
532
|
+
typeof lane.quota_reset_at === "string" &&
|
|
533
|
+
Number.isFinite(Date.parse(lane.quota_reset_at)) &&
|
|
534
|
+
Date.parse(lane.quota_reset_at) <= nowMs;
|
|
519
535
|
const modelDeps = {
|
|
520
536
|
...deps.modelDeps,
|
|
521
537
|
env: deps.modelDeps?.env ?? env,
|
|
@@ -525,13 +541,38 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
525
541
|
env,
|
|
526
542
|
};
|
|
527
543
|
const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
|
|
544
|
+
let quotaFailoverEnabled = false;
|
|
545
|
+
if (reason === "usage_limit") {
|
|
546
|
+
if (deps.quotaAccountFailoverEnabled !== undefined) {
|
|
547
|
+
quotaFailoverEnabled = deps.quotaAccountFailoverEnabled;
|
|
548
|
+
}
|
|
549
|
+
else {
|
|
550
|
+
try {
|
|
551
|
+
const laneCompany = resolveLaneCompany(hqRoot, lane);
|
|
552
|
+
quotaFailoverEnabled = await readLanesRegistryFlagFailOff(LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, { company: laneCompany.company, companyUidSource: laneCompanyUidSource(laneCompany.source), env });
|
|
553
|
+
}
|
|
554
|
+
catch (error) {
|
|
555
|
+
process.stderr.write(`${JSON.stringify({
|
|
556
|
+
event: "lanes.quota_account_failover.company_context_failed",
|
|
557
|
+
error: error instanceof Error ? error.name : "NonError",
|
|
558
|
+
fallback: false,
|
|
559
|
+
})}\n`);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
528
563
|
const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
|
|
529
564
|
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
530
565
|
? lane.model
|
|
531
566
|
: undefined;
|
|
532
567
|
const unavailableAccounts = previouslyUnavailableAccounts(lane);
|
|
533
568
|
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
534
|
-
.filter(([id, account]) => id !== lane.account
|
|
569
|
+
.filter(([id, account]) => (id !== lane.account || (quotaFailoverEnabled && reason === "usage_limit" && retryingExpiredQuota)) &&
|
|
570
|
+
!unavailableAccounts.has(id) && account.enabled)
|
|
571
|
+
.filter(([id]) => !quotaFailoverEnabled ||
|
|
572
|
+
(profile?.meta.provider !== lane.provider ||
|
|
573
|
+
(profile.meta.accounts !== undefined
|
|
574
|
+
? profile.meta.accounts.includes(id)
|
|
575
|
+
: profile.meta.account === undefined || profile.meta.account === id)))
|
|
535
576
|
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
536
577
|
existsSync,
|
|
537
578
|
hostname,
|
|
@@ -545,6 +586,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
545
586
|
existsSync,
|
|
546
587
|
accountDeps,
|
|
547
588
|
hostname,
|
|
589
|
+
now,
|
|
548
590
|
...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
|
|
549
591
|
allowedAccounts,
|
|
550
592
|
};
|
|
@@ -553,11 +595,96 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
553
595
|
target = selectBalancedAccount(lane.provider, config, modelContext);
|
|
554
596
|
}
|
|
555
597
|
catch (err) {
|
|
598
|
+
if (quotaFailoverEnabled &&
|
|
599
|
+
reason === "usage_limit" &&
|
|
600
|
+
err instanceof AccountResolutionError &&
|
|
601
|
+
err.error === "account_quota_exhausted") {
|
|
602
|
+
const quotaAccountReasons = Array.isArray(err.details.account_reasons)
|
|
603
|
+
? err.details.account_reasons
|
|
604
|
+
: [];
|
|
605
|
+
const resetAt = quotaAccountReasons
|
|
606
|
+
.map((candidate) => candidate.quota_reset_at)
|
|
607
|
+
.filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)))
|
|
608
|
+
.sort((a, b) => Date.parse(a) - Date.parse(b))[0];
|
|
609
|
+
if (resetAt) {
|
|
610
|
+
const accountReasons = Object.entries(config.providers[lane.provider].accounts).map(([id, account]) => {
|
|
611
|
+
const quotaReason = quotaAccountReasons.find((candidate) => candidate.account === id);
|
|
612
|
+
if (quotaReason)
|
|
613
|
+
return quotaReason;
|
|
614
|
+
if (id === lane.account)
|
|
615
|
+
return { account: id, reason: "source_account_quota_exhausted" };
|
|
616
|
+
if (unavailableAccounts.has(id))
|
|
617
|
+
return { account: id, reason: "previously_unavailable" };
|
|
618
|
+
if (!account.enabled)
|
|
619
|
+
return { account: id, reason: "disabled" };
|
|
620
|
+
if (quotaFailoverEnabled && profile?.meta.provider === lane.provider &&
|
|
621
|
+
profile.meta.account !== undefined && profile.meta.account !== id) {
|
|
622
|
+
return { account: id, reason: "worker_profile_account_pin" };
|
|
623
|
+
}
|
|
624
|
+
if (quotaFailoverEnabled && profile?.meta.provider === lane.provider &&
|
|
625
|
+
profile.meta.accounts !== undefined && !profile.meta.accounts.includes(id)) {
|
|
626
|
+
return { account: id, reason: "worker_profile_account_not_allowed" };
|
|
627
|
+
}
|
|
628
|
+
if (accountStatus(lane.provider, id, config, {
|
|
629
|
+
existsSync,
|
|
630
|
+
hostname,
|
|
631
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
632
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
633
|
+
}).logged_in !== true)
|
|
634
|
+
return { account: id, reason: "login_required" };
|
|
635
|
+
if (!supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps, hostname)) {
|
|
636
|
+
return { account: id, reason: "model_or_pin_incompatible" };
|
|
637
|
+
}
|
|
638
|
+
return { account: id, reason: "eligible_but_not_selected" };
|
|
639
|
+
});
|
|
640
|
+
const holdLane = (current) => {
|
|
641
|
+
if (current.state !== retryFromState ||
|
|
642
|
+
current.account !== lane.account ||
|
|
643
|
+
current.provider_session_id !== lane.provider_session_id) {
|
|
644
|
+
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while quota recovery was prepared`, { lane_id: lane.lane_id, state: current.state, account: current.account ?? null });
|
|
645
|
+
}
|
|
646
|
+
return {
|
|
647
|
+
...current,
|
|
648
|
+
state: "interrupted",
|
|
649
|
+
terminal_reason: "quota_held",
|
|
650
|
+
terminal_details: {
|
|
651
|
+
reason: err.error,
|
|
652
|
+
account_reasons: accountReasons,
|
|
653
|
+
},
|
|
654
|
+
quota_reset_at: resetAt,
|
|
655
|
+
interrupted_at: at,
|
|
656
|
+
};
|
|
657
|
+
};
|
|
658
|
+
if (deps.dryRun)
|
|
659
|
+
return holdLane(lane);
|
|
660
|
+
return retryUpdateLane(hqRoot, lane.lane_id, writer, holdLane);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
556
663
|
if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
|
|
557
664
|
throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
|
|
558
665
|
}
|
|
559
666
|
throw err;
|
|
560
667
|
}
|
|
668
|
+
if (deps.dryRun) {
|
|
669
|
+
const entry = {
|
|
670
|
+
from: lane.account ?? null,
|
|
671
|
+
to: target.id,
|
|
672
|
+
at,
|
|
673
|
+
reason,
|
|
674
|
+
};
|
|
675
|
+
return {
|
|
676
|
+
...lane,
|
|
677
|
+
state: "queued",
|
|
678
|
+
queued_reason: "account_retry",
|
|
679
|
+
queued_at: at,
|
|
680
|
+
terminal_reason: undefined,
|
|
681
|
+
terminal_details: undefined,
|
|
682
|
+
interrupted_at: undefined,
|
|
683
|
+
quota_reset_at: undefined,
|
|
684
|
+
account: target.id,
|
|
685
|
+
account_history: appendCappedHistory(lane.account_history ?? [], entry),
|
|
686
|
+
};
|
|
687
|
+
}
|
|
561
688
|
const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
|
|
562
689
|
const sessionId = providerSupportsSessionResume(lane.provider) &&
|
|
563
690
|
typeof lane.provider_session_id === "string" &&
|
|
@@ -572,12 +699,9 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
572
699
|
sessionId,
|
|
573
700
|
}));
|
|
574
701
|
}
|
|
575
|
-
const now = (deps.now ?? (() => new Date()))();
|
|
576
|
-
const at = now.toISOString();
|
|
577
|
-
const nowMs = now.getTime();
|
|
578
702
|
return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
579
703
|
const current = readLane(hqRoot, lane.lane_id);
|
|
580
|
-
if (current.state !==
|
|
704
|
+
if (current.state !== retryFromState ||
|
|
581
705
|
current.account !== lane.account ||
|
|
582
706
|
current.provider_session_id !== lane.provider_session_id) {
|
|
583
707
|
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
|
@@ -603,6 +727,8 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
603
727
|
admission_reserved_at: undefined,
|
|
604
728
|
terminal_reason: undefined,
|
|
605
729
|
terminal_details: undefined,
|
|
730
|
+
interrupted_at: undefined,
|
|
731
|
+
quota_reset_at: undefined,
|
|
606
732
|
account: target.id,
|
|
607
733
|
account_history: appendCappedHistory(row.account_history ?? [], entry),
|
|
608
734
|
pending_resume: sessionId
|
|
@@ -621,7 +747,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
621
747
|
const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
|
|
622
748
|
const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
|
|
623
749
|
const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
|
|
624
|
-
if (row.state !==
|
|
750
|
+
if (row.state !== retryFromState ||
|
|
625
751
|
row.account !== lane.account ||
|
|
626
752
|
row.provider_session_id !== lane.provider_session_id) {
|
|
627
753
|
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -442,6 +442,16 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
442
442
|
value: raw.hire_grant,
|
|
443
443
|
});
|
|
444
444
|
}
|
|
445
|
+
if (raw.released_hire_grant !== undefined &&
|
|
446
|
+
(typeof raw.released_hire_grant !== "number" ||
|
|
447
|
+
!Number.isSafeInteger(raw.released_hire_grant) ||
|
|
448
|
+
raw.released_hire_grant < 0)) {
|
|
449
|
+
throw new LaneSchemaError("Expected non-negative safe integer for released_hire_grant", {
|
|
450
|
+
path: filePath,
|
|
451
|
+
field: "released_hire_grant",
|
|
452
|
+
value: raw.released_hire_grant,
|
|
453
|
+
});
|
|
454
|
+
}
|
|
445
455
|
if (raw.keep_alive !== undefined && typeof raw.keep_alive !== "boolean") {
|
|
446
456
|
throw new LaneSchemaError("keep_alive must be a boolean", {
|
|
447
457
|
path: filePath,
|
|
@@ -1603,6 +1613,31 @@ export function updateLane(hqRoot, laneId, expectedVersion, writer, updater, dep
|
|
|
1603
1613
|
throw new LanesError("lane_writer_unauthorized", `Writer ${writer} cannot write lane field ${key}`, { writer, field: key });
|
|
1604
1614
|
}
|
|
1605
1615
|
}
|
|
1616
|
+
if (!isTerminalLaneState(current.state) && isTerminalLaneState(updated.state)) {
|
|
1617
|
+
const releasedGrant = updated.hire_grant ?? current.hire_grant ?? 0;
|
|
1618
|
+
return {
|
|
1619
|
+
...updated,
|
|
1620
|
+
released_hire_grant: releasedGrant,
|
|
1621
|
+
hire_grant: 0,
|
|
1622
|
+
};
|
|
1623
|
+
}
|
|
1624
|
+
if (isTerminalLaneState(current.state) && !isTerminalLaneState(updated.state) &&
|
|
1625
|
+
current.released_hire_grant !== undefined) {
|
|
1626
|
+
return {
|
|
1627
|
+
...updated,
|
|
1628
|
+
hire_grant: current.released_hire_grant,
|
|
1629
|
+
released_hire_grant: undefined,
|
|
1630
|
+
};
|
|
1631
|
+
}
|
|
1632
|
+
if (isTerminalLaneState(updated.state) &&
|
|
1633
|
+
(updated.hire_grant !== current.hire_grant ||
|
|
1634
|
+
updated.released_hire_grant !== current.released_hire_grant)) {
|
|
1635
|
+
return {
|
|
1636
|
+
...updated,
|
|
1637
|
+
hire_grant: 0,
|
|
1638
|
+
released_hire_grant: updated.released_hire_grant ?? updated.hire_grant ?? 0,
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1606
1641
|
return updated;
|
|
1607
1642
|
}, deps, {
|
|
1608
1643
|
beforeWrite: (current) => assertLaneStoreWriteAllowed(current, laneId, writer),
|
|
@@ -12,10 +12,20 @@ interface TrackerReportStory {
|
|
|
12
12
|
id: string;
|
|
13
13
|
title: string;
|
|
14
14
|
status: ReportStoryStatus;
|
|
15
|
+
focus_id?: unknown;
|
|
16
|
+
kpi?: unknown;
|
|
17
|
+
owner?: string | null;
|
|
18
|
+
deadline?: string | null;
|
|
15
19
|
references?: string[];
|
|
16
20
|
updated_at?: string;
|
|
17
21
|
source_path: string;
|
|
18
22
|
}
|
|
23
|
+
export interface TrackerReportFocusOverride {
|
|
24
|
+
reason: string;
|
|
25
|
+
by_lane: string;
|
|
26
|
+
at: string;
|
|
27
|
+
story: string;
|
|
28
|
+
}
|
|
19
29
|
export interface TrackerReportIssue {
|
|
20
30
|
path: string;
|
|
21
31
|
field: string;
|
|
@@ -47,6 +57,7 @@ export interface TrackerReportTeamInput {
|
|
|
47
57
|
team: string;
|
|
48
58
|
project_id: string;
|
|
49
59
|
stories: readonly TrackerReportStory[];
|
|
60
|
+
focus_overrides?: readonly TrackerReportFocusOverride[];
|
|
50
61
|
pull_requests: readonly TrackerReportPullRequest[];
|
|
51
62
|
releases: readonly TrackerReportRelease[];
|
|
52
63
|
blockers: readonly TrackerReportRecord[];
|
|
@@ -61,8 +72,42 @@ export interface TrackerReportInput {
|
|
|
61
72
|
generated_at: string;
|
|
62
73
|
since: string;
|
|
63
74
|
since_source: "explicit" | "previous_report" | "first_run";
|
|
75
|
+
focus_dashboard_enabled?: boolean;
|
|
64
76
|
teams: readonly TrackerReportTeamInput[];
|
|
65
77
|
}
|
|
78
|
+
interface TrackerFocusItem {
|
|
79
|
+
team: string;
|
|
80
|
+
project_id: string;
|
|
81
|
+
story_id: string;
|
|
82
|
+
title: string;
|
|
83
|
+
owner: string | null;
|
|
84
|
+
deadline: string | null;
|
|
85
|
+
state: ReportStoryStatus;
|
|
86
|
+
focus_id: unknown;
|
|
87
|
+
kpi: unknown;
|
|
88
|
+
override_reason: string | null;
|
|
89
|
+
}
|
|
90
|
+
interface TrackerTeamFocusShare {
|
|
91
|
+
team: string;
|
|
92
|
+
project_id: string;
|
|
93
|
+
window_start: string;
|
|
94
|
+
window_end: string;
|
|
95
|
+
work_story_count: number;
|
|
96
|
+
focused_story_count: number;
|
|
97
|
+
share_percent: number | null;
|
|
98
|
+
status: "red" | "ok" | "no_work";
|
|
99
|
+
}
|
|
100
|
+
interface TrackerFocusOverride extends TrackerReportFocusOverride {
|
|
101
|
+
team: string;
|
|
102
|
+
project_id: string;
|
|
103
|
+
}
|
|
104
|
+
interface TrackerFocusDashboard {
|
|
105
|
+
window_start: string;
|
|
106
|
+
window_end: string;
|
|
107
|
+
focus_list: TrackerFocusItem[];
|
|
108
|
+
team_focus_shares: TrackerTeamFocusShare[];
|
|
109
|
+
overrides: TrackerFocusOverride[];
|
|
110
|
+
}
|
|
66
111
|
interface ReportTimedStory {
|
|
67
112
|
id: string;
|
|
68
113
|
title: string;
|
|
@@ -140,6 +185,7 @@ export interface TrackerOwnerReport {
|
|
|
140
185
|
generated_at: string;
|
|
141
186
|
since: string;
|
|
142
187
|
since_source: TrackerReportInput["since_source"];
|
|
188
|
+
focus_dashboard?: TrackerFocusDashboard;
|
|
143
189
|
teams: TrackerOwnerTeamReport[];
|
|
144
190
|
}
|
|
145
191
|
interface TrackerTeamKpi {
|
|
@@ -231,6 +231,73 @@ function validatedWindow(input) {
|
|
|
231
231
|
}
|
|
232
232
|
return { generatedAt, since, sinceTime, generatedAtTime };
|
|
233
233
|
}
|
|
234
|
+
function hasFocus(story) {
|
|
235
|
+
const present = (value) => typeof value === "string"
|
|
236
|
+
? value.trim().length > 0
|
|
237
|
+
: value !== undefined && value !== null;
|
|
238
|
+
return present(story.focus_id) || present(story.kpi);
|
|
239
|
+
}
|
|
240
|
+
function recentlyWorked(story, sinceTime, throughTime) {
|
|
241
|
+
if (story.updated_at === undefined)
|
|
242
|
+
return false;
|
|
243
|
+
const at = Date.parse(validTimestamp(story.updated_at, "updated_at", story.source_path, { story_id: story.id }));
|
|
244
|
+
return at >= sinceTime && at <= throughTime;
|
|
245
|
+
}
|
|
246
|
+
function buildFocusDashboard(teams, generatedAt, generatedAtTime) {
|
|
247
|
+
const windowStartTime = generatedAtTime - 24 * 60 * 60 * 1000;
|
|
248
|
+
const windowStart = new Date(windowStartTime).toISOString();
|
|
249
|
+
const focusList = [];
|
|
250
|
+
const teamFocusShares = [];
|
|
251
|
+
const overrides = [];
|
|
252
|
+
for (const team of teams) {
|
|
253
|
+
const teamOverrides = [...(team.focus_overrides ?? [])]
|
|
254
|
+
.filter((override) => Number.isFinite(Date.parse(override.at)) && Date.parse(override.at) <= generatedAtTime)
|
|
255
|
+
.sort((a, b) => a.at.localeCompare(b.at) || a.story.localeCompare(b.story));
|
|
256
|
+
overrides.push(...teamOverrides.map((override) => ({ ...override, team: team.team, project_id: team.project_id })));
|
|
257
|
+
const latestOverrideByStory = new Map();
|
|
258
|
+
for (const override of teamOverrides)
|
|
259
|
+
latestOverrideByStory.set(override.story, override.reason);
|
|
260
|
+
for (const story of team.stories) {
|
|
261
|
+
if (story.status === "dropped")
|
|
262
|
+
continue;
|
|
263
|
+
const activeOrRecentlyWorked = story.status === "in_progress" || recentlyWorked(story, windowStartTime, generatedAtTime);
|
|
264
|
+
const overrideReason = latestOverrideByStory.get(story.id) ?? null;
|
|
265
|
+
if (activeOrRecentlyWorked && (hasFocus(story) || overrideReason !== null)) {
|
|
266
|
+
focusList.push({
|
|
267
|
+
team: team.team,
|
|
268
|
+
project_id: team.project_id,
|
|
269
|
+
story_id: story.id,
|
|
270
|
+
title: story.title,
|
|
271
|
+
owner: story.owner ?? null,
|
|
272
|
+
deadline: story.deadline ?? null,
|
|
273
|
+
state: story.status,
|
|
274
|
+
focus_id: story.focus_id ?? null,
|
|
275
|
+
kpi: story.kpi ?? null,
|
|
276
|
+
override_reason: overrideReason,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const workStories = team.stories.filter((story) => story.status === "in_progress" || (story.status !== "dropped" && recentlyWorked(story, windowStartTime, generatedAtTime)));
|
|
281
|
+
const focusedStoryCount = workStories.filter(hasFocus).length;
|
|
282
|
+
const sharePercent = workStories.length === 0
|
|
283
|
+
? null
|
|
284
|
+
: Math.round((focusedStoryCount / workStories.length) * 1000) / 10;
|
|
285
|
+
teamFocusShares.push({
|
|
286
|
+
team: team.team,
|
|
287
|
+
project_id: team.project_id,
|
|
288
|
+
window_start: windowStart,
|
|
289
|
+
window_end: generatedAt,
|
|
290
|
+
work_story_count: workStories.length,
|
|
291
|
+
focused_story_count: focusedStoryCount,
|
|
292
|
+
share_percent: sharePercent,
|
|
293
|
+
status: sharePercent === null ? "no_work" : focusedStoryCount / workStories.length < 0.8 ? "red" : "ok",
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
focusList.sort((a, b) => (a.deadline ?? "\uffff").localeCompare(b.deadline ?? "\uffff") || a.team.localeCompare(b.team) || a.story_id.localeCompare(b.story_id));
|
|
297
|
+
teamFocusShares.sort((a, b) => a.team.localeCompare(b.team));
|
|
298
|
+
overrides.sort((a, b) => a.at.localeCompare(b.at) || a.team.localeCompare(b.team) || a.story.localeCompare(b.story));
|
|
299
|
+
return { window_start: windowStart, window_end: generatedAt, focus_list: focusList, team_focus_shares: teamFocusShares, overrides };
|
|
300
|
+
}
|
|
234
301
|
function removeInvalidEntity(team, err) {
|
|
235
302
|
if (!(err instanceof LanesError))
|
|
236
303
|
return null;
|
|
@@ -316,7 +383,7 @@ export function buildTrackerOwnerReport(input) {
|
|
|
316
383
|
}
|
|
317
384
|
seen.add(team.team);
|
|
318
385
|
}
|
|
319
|
-
|
|
386
|
+
const report = {
|
|
320
387
|
schema_version: "hq-lanes-owner-report.v1",
|
|
321
388
|
company: input.company,
|
|
322
389
|
generated_at: window.generatedAt,
|
|
@@ -326,6 +393,10 @@ export function buildTrackerOwnerReport(input) {
|
|
|
326
393
|
.map((team) => withRecoverableTeamErrors(team, (safeTeam) => teamOwnerReport(safeTeam, input.company, window.sinceTime, window.generatedAtTime)))
|
|
327
394
|
.sort((a, b) => a.team.localeCompare(b.team)),
|
|
328
395
|
};
|
|
396
|
+
if (input.focus_dashboard_enabled === true) {
|
|
397
|
+
report.focus_dashboard = buildFocusDashboard(eligibleTeams, window.generatedAt, window.generatedAtTime);
|
|
398
|
+
}
|
|
399
|
+
return report;
|
|
329
400
|
}
|
|
330
401
|
function teamKpis(team, company, throughTime) {
|
|
331
402
|
return team.kpis
|
|
@@ -460,6 +531,19 @@ export function renderTrackerOwnerReportText(report) {
|
|
|
460
531
|
`Owner report for ${report.company}`,
|
|
461
532
|
`Reporting window: ${report.since} through ${report.generated_at} (${report.since_source.replaceAll("_", " ")})`,
|
|
462
533
|
];
|
|
534
|
+
if (report.focus_dashboard) {
|
|
535
|
+
lines.push("", `Focus dashboard (last 24h: ${report.focus_dashboard.window_start} through ${report.focus_dashboard.window_end})`, "Focus list:");
|
|
536
|
+
lines.push(...listOrNone(report.focus_dashboard.focus_list, (item) => `${item.team}/${item.story_id} — ${item.title}; owner: ${item.owner ?? "unknown"}; deadline: ${item.deadline ?? "none"}; state: ${item.state}${item.override_reason ? `; override: ${item.override_reason}` : ""}`));
|
|
537
|
+
lines.push("Team focus share:");
|
|
538
|
+
lines.push(...listOrNone(report.focus_dashboard.team_focus_shares, (share) => {
|
|
539
|
+
if (share.share_percent === null)
|
|
540
|
+
return `${share.team}: no in-progress or worked stories`;
|
|
541
|
+
const color = share.status === "red" ? "RED under 80%" : "OK";
|
|
542
|
+
return `${share.team}: ${share.share_percent}% (${share.focused_story_count}/${share.work_story_count}); ${color}`;
|
|
543
|
+
}));
|
|
544
|
+
lines.push("Focus guard overrides:");
|
|
545
|
+
lines.push(...listOrNone(report.focus_dashboard.overrides, (override) => `${override.team}/${override.story} by ${override.by_lane} (${override.at}): ${override.reason}`));
|
|
546
|
+
}
|
|
463
547
|
for (const team of report.teams) {
|
|
464
548
|
lines.push("", `${team.team} (${team.project_id})`, "Progress:", " Stories moved to done:");
|
|
465
549
|
lines.push(...listOrNone(team.progress.stories_moved_to_done, (story) => `${story.id} — ${story.title} (${story.at})${story.references.length ? `; references: ${story.references.join(", ")}` : ""}`));
|
|
@@ -584,6 +584,8 @@ export interface LaneRecord {
|
|
|
584
584
|
keep_alive_exhausted_at?: string;
|
|
585
585
|
/** Maximum number of direct child lanes this lane may keep alive; old records default to 0. */
|
|
586
586
|
hire_grant?: number;
|
|
587
|
+
/** Hire grant returned to the senior while this terminal lane is inactive. */
|
|
588
|
+
released_hire_grant?: number;
|
|
587
589
|
/** Content hash of the profile last applied to this lane. */
|
|
588
590
|
worker_profile_hash?: string;
|
|
589
591
|
/** Registry persona selected by the profile, pinned at lane creation. */
|
|
@@ -785,7 +787,7 @@ export interface AccountHistoryEntry {
|
|
|
785
787
|
from: string | null;
|
|
786
788
|
to: string;
|
|
787
789
|
at: string;
|
|
788
|
-
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "keep_alive_resume" | "operator_move";
|
|
790
|
+
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "keep_alive_resume" | "operator_move" | "profile_pin_exhausted";
|
|
789
791
|
}
|
|
790
792
|
export interface FailedResumeHistoryEntry {
|
|
791
793
|
at: string;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -228,6 +228,7 @@ export const LANE_FIELD_WRITERS = {
|
|
|
228
228
|
project_id: ["command"], story_id: ["command"], no_story_reason: ["creator", "command"], ledger_sync_error: ["story-ledger"], pending_question_ids: ["story-ledger"], stories: ["creator", "command", "watcher", "reconcile", "spawn"], thread_id: ["creator", "command", "spawn"], mesh_company: ["creator", "command"],
|
|
229
229
|
depth: ["command"], senior: ["command"], senior_history: ["command"],
|
|
230
230
|
hire_grant: ["creator", "command"],
|
|
231
|
+
released_hire_grant: ["command"],
|
|
231
232
|
state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "watcher", "reconcile"], queued_details: ["command", "reconcile", "spawn", "watcher"], queued_at: ["creator", "command", "spawn", "watcher", "reconcile"], admission_reserved_at: ["creator", "command", "reconcile", "spawn", "watcher"],
|
|
232
233
|
pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
|
|
233
234
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
package/dist/main.js
CHANGED
|
@@ -43,7 +43,7 @@ import { isCompanySelectionError } from "./utils/company-selection-error.js";
|
|
|
43
43
|
import { canOfferTeamUpgrade, formatPlanGateError, offerTeamUpgrade, offerUpgradeLink, planGateErrorFromVaultError, } from "./utils/plan-gate-error.js";
|
|
44
44
|
import { upgradeToTeam } from "./utils/team-upgrade.js";
|
|
45
45
|
import { printCachedVersionNudge, printVersionAndNudge, refreshVersionCache, staleAgainstCachedLatest, } from "./utils/version-check.js";
|
|
46
|
-
import { enforceVersionGate, isLocalDependencyInstall, resolveRunningInstall, shouldSkipGate, } from "./utils/version-gate.js";
|
|
46
|
+
import { enforceVersionGate, getVersionGatePhase, isLocalDependencyInstall, resolveRunningInstall, shouldSkipGate, } from "./utils/version-gate.js";
|
|
47
47
|
import { autoUpdateAndReexec } from "./utils/self-update.js";
|
|
48
48
|
import { CLI_VERSION } from "./cli-version.js";
|
|
49
49
|
import { findLazyCommand, registerCommandCatalog } from "./lazy-commands.js";
|
|
@@ -262,6 +262,10 @@ export async function runCli() {
|
|
|
262
262
|
env: process.env,
|
|
263
263
|
entryPath: RUNNING_ENTRY_PATH,
|
|
264
264
|
stderr: process.stderr,
|
|
265
|
+
deps: {
|
|
266
|
+
resolveInstall: resolveRunningInstall,
|
|
267
|
+
versionGatePhase: getVersionGatePhase,
|
|
268
|
+
},
|
|
265
269
|
});
|
|
266
270
|
if (registration.reexecStatus !== undefined) {
|
|
267
271
|
// The re-exec'd child ran the command on the settled tree; carry its exit
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
* style of handleTopLevelError so the whole seam is unit-testable without real
|
|
17
17
|
* spawns, waits, or a real install tree.
|
|
18
18
|
*/
|
|
19
|
+
import type { VersionGatePhase } from "./utils/version-gate.js";
|
|
19
20
|
import { type InstallRewriteSignalArgs, type WaitForInstallTreeSettledArgs, type WaitForInstallTreeSettledResult } from "./utils/install-tree-torn.js";
|
|
20
21
|
/**
|
|
21
22
|
* Set on the re-exec'd child so it can NEVER wait or re-exec again — distinct
|
|
@@ -74,6 +75,8 @@ export interface RegisterRecoveryDeps {
|
|
|
74
75
|
waitForSettled?: (args: WaitForInstallTreeSettledArgs) => Promise<WaitForInstallTreeSettledResult>;
|
|
75
76
|
/** Positive writer-signal probe for the eval-throw gate (defaults to installRewriteInProgress). */
|
|
76
77
|
rewriteInProgress?: (args: InstallRewriteSignalArgs) => boolean;
|
|
78
|
+
/** Current version-gate phase for diagnostics; defaults to unknown for early entry recovery. */
|
|
79
|
+
versionGatePhase?: () => VersionGatePhase;
|
|
77
80
|
/** node flags to forward to the re-exec child (defaults to process.execArgv). */
|
|
78
81
|
execArgv?: readonly string[];
|
|
79
82
|
}
|