@indigoai-us/hq-cli 5.148.0 → 5.150.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/assets/scaffold/core/scripts/checkpoint-stop-gate.sh +8 -11
- package/dist/command-catalog.generated.d.ts +148 -0
- package/dist/command-catalog.generated.js +193 -0
- package/dist/commands/lanes.js +1000 -268
- package/dist/commands/mesh.js +82 -11
- package/dist/lib/lanes/admission.d.ts +24 -0
- package/dist/lib/lanes/admission.js +193 -0
- package/dist/lib/lanes/delivery.js +5 -3
- package/dist/lib/lanes/index.d.ts +5 -2
- package/dist/lib/lanes/index.js +3 -1
- package/dist/lib/lanes/mesh-events.d.ts +8 -0
- package/dist/lib/lanes/mesh-events.js +83 -6
- package/dist/lib/lanes/outpost-dispatch.d.ts +12 -0
- package/dist/lib/lanes/outpost-dispatch.js +17 -0
- package/dist/lib/lanes/reconcile.js +194 -55
- package/dist/lib/lanes/session-relink.d.ts +25 -0
- package/dist/lib/lanes/session-relink.js +216 -0
- package/dist/lib/lanes/spawn.d.ts +1 -0
- package/dist/lib/lanes/spawn.js +52 -5
- package/dist/lib/lanes/store.js +156 -1
- package/dist/lib/lanes/stories.d.ts +17 -0
- package/dist/lib/lanes/stories.js +73 -0
- package/dist/lib/lanes/types.d.ts +28 -1
- package/dist/lib/lanes/types.js +24 -6
- package/dist/lib/lanes/watcher.js +69 -9
- package/dist/lib/mesh/live/daemon/doctor.d.ts +27 -0
- package/dist/lib/mesh/live/daemon/doctor.js +84 -0
- package/dist/lib/mesh/live/daemon/index.d.ts +3 -3
- package/dist/lib/mesh/live/daemon/index.js +2 -2
- package/dist/lib/mesh/live/daemon/install.d.ts +40 -0
- package/dist/lib/mesh/live/daemon/install.js +88 -0
- package/dist/lib/mesh/live/daemon/run.d.ts +2 -0
- package/dist/lib/mesh/live/daemon/run.js +12 -1
- package/dist/lib/mesh/live/daemon/state.d.ts +4 -0
- package/dist/lib/mesh/project-board.d.ts +15 -0
- package/dist/lib/mesh/project-board.js +84 -0
- package/dist/lib/work-context/prd-sync.d.ts +5 -1
- package/dist/lib/work-context/prd-sync.js +22 -1
- package/package.json +5 -3
- package/scripts/mesh-daemon-if-stale.mjs +16 -0
|
@@ -8,18 +8,21 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import {
|
|
11
|
+
import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
|
+
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
|
+
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
12
14
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
13
|
-
import { LanesError, isLaneVersionConflict } from "./errors.js";
|
|
15
|
+
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
14
16
|
import { loadWorkerProfile } from "./profile.js";
|
|
15
17
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
16
18
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
17
19
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
18
|
-
import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
20
|
+
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
19
21
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
20
22
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
21
23
|
import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
|
|
22
24
|
import { createLaneMesh, emitLaneDecisionEvents, laneMeshCompany } from "./mesh-events.js";
|
|
25
|
+
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
23
26
|
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
24
27
|
import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
25
28
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
@@ -87,10 +90,10 @@ export function timeoutPath(hqRoot, laneId) {
|
|
|
87
90
|
return path.join(laneRunDir(hqRoot, laneId), "timeout.json");
|
|
88
91
|
}
|
|
89
92
|
export function laneRunsHere(lane, host) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
return lane
|
|
93
|
+
return laneRunsOnHost(lane, host);
|
|
94
|
+
}
|
|
95
|
+
function hasFreshAdmissionReservation(lane, nowMs) {
|
|
96
|
+
return hasActiveAdmissionReservation(lane, nowMs);
|
|
94
97
|
}
|
|
95
98
|
export function agedAdvanceCount(question) {
|
|
96
99
|
return question.routing_history.filter((entry) => entry.reason === "unanswered_timeout").length;
|
|
@@ -162,11 +165,6 @@ export function isDeadlinePassed(deadlineAt, nowMs) {
|
|
|
162
165
|
return false;
|
|
163
166
|
return nowMs > at;
|
|
164
167
|
}
|
|
165
|
-
function compareQueueOrder(a, b) {
|
|
166
|
-
if (b.priority !== a.priority)
|
|
167
|
-
return b.priority - a.priority;
|
|
168
|
-
return a.created_at.localeCompare(b.created_at);
|
|
169
|
-
}
|
|
170
168
|
function readDirJsonIds(dir, label) {
|
|
171
169
|
let names;
|
|
172
170
|
try {
|
|
@@ -307,6 +305,53 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
|
|
|
307
305
|
}
|
|
308
306
|
throw new LanesError("version_conflict", `Exhausted CAS retries for lane ${laneId}`, { lane_id: laneId });
|
|
309
307
|
}
|
|
308
|
+
function writeQueueReason(hqRoot, lane, reason, dryRun) {
|
|
309
|
+
if (lane.queued_reason === reason)
|
|
310
|
+
return;
|
|
311
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, admission_reserved_at: undefined }), dryRun);
|
|
312
|
+
}
|
|
313
|
+
function availableBalancedAccount(hqRoot, lane, occupying, config, host, profileMaxConcurrent, nowMs) {
|
|
314
|
+
if (!lane.account || !lane.admission_accounts?.includes(lane.account))
|
|
315
|
+
return undefined;
|
|
316
|
+
if (laneAdmissionCapReason(occupying, lane, config, host, profileMaxConcurrent, nowMs) !== "account_cap") {
|
|
317
|
+
return undefined;
|
|
318
|
+
}
|
|
319
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
|
|
320
|
+
? lane.model
|
|
321
|
+
: lane.model_source === "profile"
|
|
322
|
+
? lane.model
|
|
323
|
+
: undefined;
|
|
324
|
+
for (const account of lane.admission_accounts) {
|
|
325
|
+
if (account === lane.account)
|
|
326
|
+
continue;
|
|
327
|
+
if (config.providers[lane.provider].accounts[account]?.enabled !== true)
|
|
328
|
+
continue;
|
|
329
|
+
const status = accountStatus(lane.provider, account, config);
|
|
330
|
+
if (!isCredentialProven(status))
|
|
331
|
+
continue;
|
|
332
|
+
try {
|
|
333
|
+
resolveAccount(lane.provider, account, config, {
|
|
334
|
+
hqRoot,
|
|
335
|
+
...(pinnedModel !== undefined ? { model: pinnedModel } : {}),
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
if (err instanceof AccountResolutionError)
|
|
340
|
+
continue;
|
|
341
|
+
throw err;
|
|
342
|
+
}
|
|
343
|
+
const candidate = { ...lane, account };
|
|
344
|
+
if (laneAdmissionCapReason(occupying, candidate, config, host, profileMaxConcurrent, nowMs) === undefined) {
|
|
345
|
+
return account;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return undefined;
|
|
349
|
+
}
|
|
350
|
+
function writeAdmissionAccount(hqRoot, lane, account, dryRun) {
|
|
351
|
+
return casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
|
|
352
|
+
current.account === lane.account &&
|
|
353
|
+
current.admission_accounts?.includes(account) === true, (current) => ({ ...current, account }), dryRun).record;
|
|
354
|
+
}
|
|
310
355
|
function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
|
|
311
356
|
for (let i = 0; i < 5; i += 1) {
|
|
312
357
|
const current = readQuestion(hqRoot, questionId);
|
|
@@ -706,10 +751,12 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
706
751
|
});
|
|
707
752
|
}
|
|
708
753
|
}
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
754
|
+
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
755
|
+
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
756
|
+
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
757
|
+
await admitQueued(hqRoot, freshLanes, summary, deps, now);
|
|
758
|
+
await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
|
|
759
|
+
});
|
|
713
760
|
if (!deps.dryRun && summary.errors.length === 0) {
|
|
714
761
|
try {
|
|
715
762
|
writeHeartbeatFile(hqRoot, summary, nowIso);
|
|
@@ -882,18 +929,44 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
882
929
|
(latest.envelope.questions?.length ?? 0) > 0) {
|
|
883
930
|
raiseEnvelopeQuestions(hqRoot, readLane(hqRoot, lane.lane_id), latest.envelope, latest.ref, nowIso);
|
|
884
931
|
}
|
|
885
|
-
const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "running", (current) =>
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
932
|
+
const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "running", (current) => {
|
|
933
|
+
const next = {
|
|
934
|
+
...current,
|
|
935
|
+
state: repairedClassification.state,
|
|
936
|
+
queued_reason: undefined,
|
|
937
|
+
terminal_reason: repairedClassification.reason ?? "host_lost",
|
|
938
|
+
...(providerRejection
|
|
939
|
+
? { terminal_details: providerRejectionDetails(providerRejection, current) }
|
|
940
|
+
: {}),
|
|
941
|
+
...(quota?.quota_reset_at ? { quota_reset_at: quota.quota_reset_at } : {}),
|
|
942
|
+
...(quota?.marker ? { model_quota_marker: quota.marker } : {}),
|
|
943
|
+
last_host_report_at: nowIso,
|
|
944
|
+
};
|
|
945
|
+
if (latest?.envelope) {
|
|
946
|
+
const envelope = latest.envelope;
|
|
947
|
+
next.story_status = envelope.story_status;
|
|
948
|
+
if (envelope.story_complete !== undefined) {
|
|
949
|
+
next.story_complete = envelope.story_complete;
|
|
950
|
+
next.story_complete_reason = envelope.story_complete_reason;
|
|
951
|
+
}
|
|
952
|
+
const storyStatus = envelope.decision === "blocked"
|
|
953
|
+
? "blocked"
|
|
954
|
+
: envelope.decision === "done"
|
|
955
|
+
? "review"
|
|
956
|
+
: storyStatusForEnvelope(envelope.story_status);
|
|
957
|
+
next.stories = laneStories(current).map((story) => story.project_id === current.project_id && story.story_id === current.story_id
|
|
958
|
+
? {
|
|
959
|
+
...story,
|
|
960
|
+
status: storyStatus,
|
|
961
|
+
updated_at: nowIso,
|
|
962
|
+
...(envelope.decision === "blocked"
|
|
963
|
+
? { blocked_reason: envelope.blocked_reason ?? "" }
|
|
964
|
+
: { blocked_reason: undefined }),
|
|
965
|
+
}
|
|
966
|
+
: story);
|
|
967
|
+
}
|
|
968
|
+
return next;
|
|
969
|
+
}, deps.dryRun);
|
|
897
970
|
if (result.changed) {
|
|
898
971
|
summary.dead_host.push({
|
|
899
972
|
id: lane.lane_id,
|
|
@@ -978,7 +1051,8 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
978
1051
|
.filter((lane) => lane.state === "queued" &&
|
|
979
1052
|
lane.pickup_validation_pending !== true &&
|
|
980
1053
|
laneRunsHere(lane, deps.host))
|
|
981
|
-
.
|
|
1054
|
+
.filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
|
|
1055
|
+
.sort(compareLaneQueueOrder);
|
|
982
1056
|
if (localQueued.length === 0)
|
|
983
1057
|
return;
|
|
984
1058
|
const config = loadLanesConfig(hqRoot);
|
|
@@ -1003,31 +1077,37 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1003
1077
|
});
|
|
1004
1078
|
const spawnFn = deps.spawn ??
|
|
1005
1079
|
((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1006
|
-
const running = lanes.filter((lane) => lane.state === "running"
|
|
1080
|
+
const running = lanes.filter((lane) => (lane.state === "running" ||
|
|
1081
|
+
hasFreshAdmissionReservation(lane, now.getTime()) ||
|
|
1082
|
+
hasActivePendingResume(lane, now.getTime())) &&
|
|
1083
|
+
laneRunsHere(lane, deps.host));
|
|
1007
1084
|
const starvation = decision.admit && decision.reason === "starvation_bound";
|
|
1008
|
-
const candidates = starvation ? localQueued.slice(0, 1) : localQueued;
|
|
1009
1085
|
const spawned = [];
|
|
1010
1086
|
if (!decision.admit) {
|
|
1087
|
+
for (const lane of localQueued) {
|
|
1088
|
+
let profileMaxConcurrent;
|
|
1089
|
+
if (lane.worker_profile !== undefined) {
|
|
1090
|
+
try {
|
|
1091
|
+
profileMaxConcurrent = loadWorkerProfile(hqRoot, lane.worker_profile).meta.max_concurrent;
|
|
1092
|
+
}
|
|
1093
|
+
catch (err) {
|
|
1094
|
+
markDirty(summary, {
|
|
1095
|
+
kind: "lane",
|
|
1096
|
+
id: lane.lane_id,
|
|
1097
|
+
why: `profile admission failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1098
|
+
});
|
|
1099
|
+
continue;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
const capReason = laneAdmissionCapReason(running, lane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1103
|
+
writeQueueReason(hqRoot, lane, capReason ?? "throttle", deps.dryRun);
|
|
1104
|
+
}
|
|
1011
1105
|
if (!deps.dryRun && throttleState !== "unreadable") {
|
|
1012
1106
|
writeThrottleState(filePath, applyThrottleDecision(throttleState, decision, nowMs), throttleDeps);
|
|
1013
1107
|
}
|
|
1014
1108
|
return;
|
|
1015
1109
|
}
|
|
1016
|
-
for (const lane of
|
|
1017
|
-
const provider = lane.provider;
|
|
1018
|
-
const providerCfg = config.providers[provider];
|
|
1019
|
-
const providerRunning = running.filter((row) => row.provider === provider).length +
|
|
1020
|
-
spawned.filter((row) => row.provider === provider).length;
|
|
1021
|
-
if (providerRunning >= providerCfg.max_running && !starvation)
|
|
1022
|
-
continue;
|
|
1023
|
-
if (lane.account) {
|
|
1024
|
-
const accountRunning = running.filter((row) => row.provider === provider && row.account === lane.account).length +
|
|
1025
|
-
spawned.filter((row) => row.provider === provider && row.account === lane.account).length;
|
|
1026
|
-
if (accountRunning >= accountMaxRunning(providerCfg, lane.account) &&
|
|
1027
|
-
!starvation) {
|
|
1028
|
-
continue;
|
|
1029
|
-
}
|
|
1030
|
-
}
|
|
1110
|
+
for (const lane of localQueued) {
|
|
1031
1111
|
let profileMaxConcurrent;
|
|
1032
1112
|
if (lane.worker_profile !== undefined) {
|
|
1033
1113
|
try {
|
|
@@ -1042,11 +1122,22 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1042
1122
|
continue;
|
|
1043
1123
|
}
|
|
1044
1124
|
}
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1125
|
+
const projected = [...running, ...spawned];
|
|
1126
|
+
const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1127
|
+
const admittedLane = alternateAccount !== undefined
|
|
1128
|
+
? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
|
|
1129
|
+
: lane;
|
|
1130
|
+
const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1131
|
+
const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
1132
|
+
let blockedBy;
|
|
1133
|
+
if (currentCap !== undefined) {
|
|
1134
|
+
blockedBy = baseCap ?? "queue_position";
|
|
1135
|
+
}
|
|
1136
|
+
if (!blockedBy && !decision.admit)
|
|
1137
|
+
blockedBy = "throttle";
|
|
1138
|
+
if (blockedBy) {
|
|
1139
|
+
writeQueueReason(hqRoot, admittedLane, blockedBy, deps.dryRun);
|
|
1140
|
+
continue;
|
|
1050
1141
|
}
|
|
1051
1142
|
try {
|
|
1052
1143
|
if (deps.dryRun) {
|
|
@@ -1056,12 +1147,12 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1056
1147
|
to: "running",
|
|
1057
1148
|
reason: decision.reason,
|
|
1058
1149
|
});
|
|
1059
|
-
spawned.push({ ...
|
|
1150
|
+
spawned.push({ ...admittedLane, state: "running" });
|
|
1060
1151
|
if (starvation)
|
|
1061
1152
|
break;
|
|
1062
1153
|
continue;
|
|
1063
1154
|
}
|
|
1064
|
-
const started = spawnFn(hqRoot,
|
|
1155
|
+
const started = spawnFn(hqRoot, admittedLane);
|
|
1065
1156
|
if (decision.reason === "starvation_bound") {
|
|
1066
1157
|
casLaneIf(hqRoot, lane.lane_id, (current) => current.admitted_reason !== "starvation_bound", (current) => ({ ...current, admitted_reason: "starvation_bound" }), false);
|
|
1067
1158
|
}
|
|
@@ -1100,25 +1191,73 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
|
1100
1191
|
}
|
|
1101
1192
|
}
|
|
1102
1193
|
async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
|
|
1103
|
-
const
|
|
1104
|
-
((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1194
|
+
const config = loadLanesConfig(hqRoot);
|
|
1105
1195
|
const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" && laneRunsHere(lane, deps.host));
|
|
1196
|
+
if (awaiting.length === 0)
|
|
1197
|
+
return;
|
|
1198
|
+
const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
|
|
1199
|
+
let throttleDecision;
|
|
1200
|
+
if (deps.dryRun) {
|
|
1201
|
+
const loaded = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
|
|
1202
|
+
const state = loaded === "missing" ? emptyThrottleState() : loaded;
|
|
1203
|
+
throttleDecision = evaluateThrottle({
|
|
1204
|
+
busy: await resolveCpuBusyFraction(throttleDeps),
|
|
1205
|
+
availableMemMb: readAvailableMemMb(throttleDeps),
|
|
1206
|
+
cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
|
|
1207
|
+
state,
|
|
1208
|
+
throttle: config.throttle,
|
|
1209
|
+
nowMs: deps.now().getTime(),
|
|
1210
|
+
});
|
|
1211
|
+
}
|
|
1212
|
+
else {
|
|
1213
|
+
const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
|
|
1214
|
+
throttleDecision = throttle.decision;
|
|
1215
|
+
}
|
|
1216
|
+
if (!throttleDecision.admit)
|
|
1217
|
+
return;
|
|
1218
|
+
const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
|
|
1219
|
+
const nowMs = deps.now().getTime();
|
|
1220
|
+
const capacityLanes = lanes.map((lane) => {
|
|
1221
|
+
try {
|
|
1222
|
+
return readLane(hqRoot, lane.lane_id);
|
|
1223
|
+
}
|
|
1224
|
+
catch {
|
|
1225
|
+
return lane;
|
|
1226
|
+
}
|
|
1227
|
+
});
|
|
1228
|
+
const running = capacityLanes.filter((lane) => (lane.state === "running" ||
|
|
1229
|
+
hasFreshAdmissionReservation(lane, nowMs) ||
|
|
1230
|
+
hasActivePendingResume(lane, nowMs)) &&
|
|
1231
|
+
laneRunsHere(lane, deps.host));
|
|
1106
1232
|
for (const lane of awaiting) {
|
|
1233
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
1234
|
+
if (current.state !== "awaiting_input")
|
|
1235
|
+
continue;
|
|
1107
1236
|
const pending = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "pending");
|
|
1108
1237
|
const answered = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "answered");
|
|
1109
1238
|
if (pending.length > 0 || answered.length === 0)
|
|
1110
1239
|
continue;
|
|
1111
1240
|
try {
|
|
1241
|
+
let profileMaxConcurrent;
|
|
1242
|
+
if (current.worker_profile !== undefined) {
|
|
1243
|
+
profileMaxConcurrent = loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent;
|
|
1244
|
+
}
|
|
1245
|
+
if (laneAdmissionCapReason(running, current, config, deps.host, profileMaxConcurrent, nowMs) !== undefined) {
|
|
1246
|
+
continue;
|
|
1247
|
+
}
|
|
1112
1248
|
if (deps.dryRun) {
|
|
1113
1249
|
summary.resumed.push({
|
|
1114
1250
|
id: lane.lane_id,
|
|
1115
1251
|
from: "awaiting_input",
|
|
1116
1252
|
to: "running",
|
|
1117
1253
|
});
|
|
1254
|
+
running.push({ ...current, state: "running" });
|
|
1118
1255
|
continue;
|
|
1119
1256
|
}
|
|
1120
1257
|
writeAnsweredQuestions(hqRoot, lane, questions);
|
|
1121
1258
|
const started = spawnFn(hqRoot, lane);
|
|
1259
|
+
if (started.state === "running")
|
|
1260
|
+
running.push(started);
|
|
1122
1261
|
summary.resumed.push({
|
|
1123
1262
|
id: started.lane_id,
|
|
1124
1263
|
from: "awaiting_input",
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type SessionCopyResult } from "./move-account.js";
|
|
2
|
+
import type { LaneProvider } from "./types.js";
|
|
3
|
+
export interface EnsureSessionHomeInput {
|
|
4
|
+
provider: Exclude<LaneProvider, "grok">;
|
|
5
|
+
sessionId: string;
|
|
6
|
+
cwd: string;
|
|
7
|
+
targetHome: string;
|
|
8
|
+
registeredHomes: string[];
|
|
9
|
+
providerHomeAtSpawn?: string;
|
|
10
|
+
}
|
|
11
|
+
export interface EnsureSessionHomeResult extends SessionCopyResult {
|
|
12
|
+
source_home: string;
|
|
13
|
+
transcript_path: string;
|
|
14
|
+
homes_searched: string[];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Older Claude account homes can share a registered `account-N/.claude`
|
|
18
|
+
* parent even after one account is re-registered elsewhere. Discover only
|
|
19
|
+
* those sibling homes, then inspect the exact session path; credential files
|
|
20
|
+
* are never opened.
|
|
21
|
+
*/
|
|
22
|
+
export declare function discoverSiblingAccountHomes(provider: EnsureSessionHomeInput["provider"], registeredHomes: string[]): string[];
|
|
23
|
+
/** Find, copy, and verify a provider transcript in the home the next run uses. */
|
|
24
|
+
export declare function ensureSessionHome(input: EnsureSessionHomeInput): EnsureSessionHomeResult;
|
|
25
|
+
//# sourceMappingURL=session-relink.d.ts.map
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { copySessionFiles, encodeClaudeProjectDir, planSessionCopy, } from "./move-account.js";
|
|
4
|
+
import { LanesError } from "./errors.js";
|
|
5
|
+
function statDirectory(target) {
|
|
6
|
+
try {
|
|
7
|
+
return fs.statSync(target).isDirectory();
|
|
8
|
+
}
|
|
9
|
+
catch (err) {
|
|
10
|
+
const code = err.code;
|
|
11
|
+
if (code === "ENOENT" || code === "ENOTDIR")
|
|
12
|
+
return false;
|
|
13
|
+
throw err;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
function claudeTranscriptPath(home, cwd, sessionId) {
|
|
17
|
+
return path.join(home, "projects", encodeClaudeProjectDir(cwd), `${sessionId}.jsonl`);
|
|
18
|
+
}
|
|
19
|
+
function findCodexRollouts(home, sessionId) {
|
|
20
|
+
const root = path.join(home, "sessions");
|
|
21
|
+
if (!statDirectory(root))
|
|
22
|
+
return [];
|
|
23
|
+
const suffix = `-${sessionId}.jsonl`;
|
|
24
|
+
const stack = [{ dir: root, depth: 0 }];
|
|
25
|
+
const matches = [];
|
|
26
|
+
while (stack.length > 0) {
|
|
27
|
+
const { dir, depth } = stack.pop();
|
|
28
|
+
let entries;
|
|
29
|
+
try {
|
|
30
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
const code = err.code;
|
|
34
|
+
if (code === "ENOENT" || code === "ENOTDIR")
|
|
35
|
+
continue;
|
|
36
|
+
throw new LanesError("provider_home_search_failed", `Could not search provider home ${home}: ${err instanceof Error ? err.message : String(err)}`, { provider: "codex", home });
|
|
37
|
+
}
|
|
38
|
+
for (const entry of entries) {
|
|
39
|
+
const full = path.join(dir, entry.name);
|
|
40
|
+
if (entry.isDirectory()) {
|
|
41
|
+
if (depth < 3)
|
|
42
|
+
stack.push({ dir: full, depth: depth + 1 });
|
|
43
|
+
}
|
|
44
|
+
else if (entry.isFile() &&
|
|
45
|
+
entry.name.startsWith("rollout-") &&
|
|
46
|
+
entry.name.endsWith(suffix)) {
|
|
47
|
+
matches.push(full);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return matches;
|
|
52
|
+
}
|
|
53
|
+
function sessionTranscripts(provider, home, cwd, sessionId) {
|
|
54
|
+
if (provider === "codex")
|
|
55
|
+
return findCodexRollouts(home, sessionId);
|
|
56
|
+
const transcript = claudeTranscriptPath(home, cwd, sessionId);
|
|
57
|
+
try {
|
|
58
|
+
return fs.statSync(transcript).isFile() ? [transcript] : [];
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
const code = err.code;
|
|
62
|
+
if (code === "ENOENT" || code === "ENOTDIR")
|
|
63
|
+
return [];
|
|
64
|
+
throw new LanesError("provider_home_search_failed", `Could not check provider home ${home}: ${err instanceof Error ? err.message : String(err)}`, { provider, home });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Older Claude account homes can share a registered `account-N/.claude`
|
|
69
|
+
* parent even after one account is re-registered elsewhere. Discover only
|
|
70
|
+
* those sibling homes, then inspect the exact session path; credential files
|
|
71
|
+
* are never opened.
|
|
72
|
+
*/
|
|
73
|
+
export function discoverSiblingAccountHomes(provider, registeredHomes) {
|
|
74
|
+
const roots = new Set();
|
|
75
|
+
for (const home of registeredHomes) {
|
|
76
|
+
if (provider === "claude") {
|
|
77
|
+
const accountDir = path.dirname(home);
|
|
78
|
+
if (path.basename(home) === ".claude" &&
|
|
79
|
+
path.basename(accountDir).startsWith("account-")) {
|
|
80
|
+
roots.add(path.dirname(accountDir));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
else if (path.basename(home).startsWith("account-")) {
|
|
84
|
+
roots.add(path.dirname(home));
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const homes = [];
|
|
88
|
+
for (const root of [...roots].sort()) {
|
|
89
|
+
let entries;
|
|
90
|
+
try {
|
|
91
|
+
entries = fs.readdirSync(root, { withFileTypes: true });
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
const code = err.code;
|
|
95
|
+
if (code === "ENOENT" || code === "ENOTDIR")
|
|
96
|
+
continue;
|
|
97
|
+
throw new LanesError("provider_home_search_failed", `Could not search provider account directory ${root}: ${err instanceof Error ? err.message : String(err)}`, { provider, home: root });
|
|
98
|
+
}
|
|
99
|
+
for (const entry of entries) {
|
|
100
|
+
if (!entry.isDirectory() || !entry.name.startsWith("account-"))
|
|
101
|
+
continue;
|
|
102
|
+
const home = provider === "claude"
|
|
103
|
+
? path.join(root, entry.name, ".claude")
|
|
104
|
+
: path.join(root, entry.name);
|
|
105
|
+
if (statDirectory(home))
|
|
106
|
+
homes.push(home);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return homes;
|
|
110
|
+
}
|
|
111
|
+
function uniqueHomes(homes) {
|
|
112
|
+
const unique = new Set();
|
|
113
|
+
for (const home of homes) {
|
|
114
|
+
if (typeof home !== "string" || home.trim() === "")
|
|
115
|
+
continue;
|
|
116
|
+
unique.add(path.resolve(home));
|
|
117
|
+
}
|
|
118
|
+
return [...unique];
|
|
119
|
+
}
|
|
120
|
+
function missingTranscript(input, homes) {
|
|
121
|
+
return new LanesError("provider_transcript_missing", `Provider transcript for session ${input.sessionId} was not found. Homes searched: ${homes.join(", ")}`, { provider: input.provider, session_id: input.sessionId, homes_searched: homes });
|
|
122
|
+
}
|
|
123
|
+
function copyVerifiedTranscript(input, sourceHome, targetHome, searched) {
|
|
124
|
+
let copied;
|
|
125
|
+
try {
|
|
126
|
+
copied = copySessionFiles(planSessionCopy(input.provider, {
|
|
127
|
+
sourceHome,
|
|
128
|
+
targetHome,
|
|
129
|
+
cwd: input.cwd,
|
|
130
|
+
sessionId: input.sessionId,
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
if (err instanceof LanesError && err.error === "provider_session_missing") {
|
|
135
|
+
throw missingTranscript(input, searched);
|
|
136
|
+
}
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
const target = sessionTranscripts(input.provider, targetHome, input.cwd, input.sessionId);
|
|
140
|
+
if (target.length !== 1) {
|
|
141
|
+
throw new LanesError("provider_transcript_relink_failed", `Could not verify session ${input.sessionId} in target home ${targetHome}`, { provider: input.provider, session_id: input.sessionId, target_home: targetHome });
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
...copied,
|
|
145
|
+
source_home: sourceHome,
|
|
146
|
+
transcript_path: target[0],
|
|
147
|
+
homes_searched: searched,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/** Find, copy, and verify a provider transcript in the home the next run uses. */
|
|
151
|
+
export function ensureSessionHome(input) {
|
|
152
|
+
const targetHome = path.resolve(input.targetHome);
|
|
153
|
+
const homes = uniqueHomes([
|
|
154
|
+
targetHome,
|
|
155
|
+
...input.registeredHomes,
|
|
156
|
+
...(input.providerHomeAtSpawn ? [input.providerHomeAtSpawn] : []),
|
|
157
|
+
...discoverSiblingAccountHomes(input.provider, input.registeredHomes),
|
|
158
|
+
]);
|
|
159
|
+
const searched = [targetHome];
|
|
160
|
+
const current = sessionTranscripts(input.provider, targetHome, input.cwd, input.sessionId);
|
|
161
|
+
if (current.length > 1) {
|
|
162
|
+
throw new LanesError("provider_transcript_ambiguous", `Multiple ${input.provider} transcripts for session ${input.sessionId} exist in ${targetHome}`, { provider: input.provider, session_id: input.sessionId, homes_searched: searched, matches: current });
|
|
163
|
+
}
|
|
164
|
+
const lastRunHome = input.providerHomeAtSpawn
|
|
165
|
+
? path.resolve(input.providerHomeAtSpawn)
|
|
166
|
+
: undefined;
|
|
167
|
+
if (lastRunHome !== undefined && lastRunHome !== targetHome) {
|
|
168
|
+
searched.push(lastRunHome);
|
|
169
|
+
const lastRun = sessionTranscripts(input.provider, lastRunHome, input.cwd, input.sessionId);
|
|
170
|
+
if (lastRun.length > 1) {
|
|
171
|
+
throw new LanesError("provider_transcript_ambiguous", `Multiple ${input.provider} transcripts for session ${input.sessionId} exist in the last run home ${lastRunHome}`, { provider: input.provider, session_id: input.sessionId, homes_searched: searched, matches: lastRun });
|
|
172
|
+
}
|
|
173
|
+
if (lastRun.length === 1) {
|
|
174
|
+
return copyVerifiedTranscript(input, lastRunHome, targetHome, searched);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
if (current.length === 1) {
|
|
178
|
+
const plan = planSessionCopy(input.provider, {
|
|
179
|
+
sourceHome: targetHome,
|
|
180
|
+
targetHome,
|
|
181
|
+
cwd: input.cwd,
|
|
182
|
+
sessionId: input.sessionId,
|
|
183
|
+
});
|
|
184
|
+
return {
|
|
185
|
+
...plan,
|
|
186
|
+
copied: [],
|
|
187
|
+
source_home: targetHome,
|
|
188
|
+
transcript_path: current[0],
|
|
189
|
+
homes_searched: searched,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
const matches = [];
|
|
193
|
+
for (const home of homes.slice(1)) {
|
|
194
|
+
if (home === lastRunHome && searched.includes(home))
|
|
195
|
+
continue;
|
|
196
|
+
searched.push(home);
|
|
197
|
+
const found = sessionTranscripts(input.provider, home, input.cwd, input.sessionId);
|
|
198
|
+
if (found.length > 1) {
|
|
199
|
+
throw new LanesError("provider_transcript_ambiguous", `Multiple ${input.provider} transcripts for session ${input.sessionId} exist in ${home}`, { provider: input.provider, session_id: input.sessionId, homes_searched: searched, matches: found });
|
|
200
|
+
}
|
|
201
|
+
if (found.length === 1)
|
|
202
|
+
matches.push({ home, transcript: found[0] });
|
|
203
|
+
}
|
|
204
|
+
if (matches.length === 0)
|
|
205
|
+
throw missingTranscript(input, searched);
|
|
206
|
+
if (matches.length > 1) {
|
|
207
|
+
throw new LanesError("provider_transcript_ambiguous", `Provider transcript for session ${input.sessionId} exists in multiple homes: ${matches.map((match) => match.home).join(", ")}`, {
|
|
208
|
+
provider: input.provider,
|
|
209
|
+
session_id: input.sessionId,
|
|
210
|
+
homes_searched: searched,
|
|
211
|
+
matching_homes: matches.map((match) => match.home),
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
return copyVerifiedTranscript(input, matches[0].home, targetHome, searched);
|
|
215
|
+
}
|
|
216
|
+
//# sourceMappingURL=session-relink.js.map
|
|
@@ -235,6 +235,7 @@ export interface SpawnManifestFile {
|
|
|
235
235
|
resume?: {
|
|
236
236
|
session_id: string;
|
|
237
237
|
} | null;
|
|
238
|
+
provider_home_at_spawn?: string;
|
|
238
239
|
/** Byte length of logs/worker.out before this run started; its result is after this. */
|
|
239
240
|
worker_out_offset?: number;
|
|
240
241
|
fast_mode?: boolean;
|