@indigoai-us/hq-cli 5.154.0 → 5.155.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/commands/lanes.d.ts +3 -11
- package/dist/commands/lanes.js +68 -37
- package/dist/lib/lanes/accounts.d.ts +16 -11
- package/dist/lib/lanes/accounts.js +50 -8
- package/dist/lib/lanes/admission-host.d.ts +19 -0
- package/dist/lib/lanes/admission-host.js +33 -0
- package/dist/lib/lanes/config.js +25 -0
- package/dist/lib/lanes/provider-rejection.d.ts +4 -4
- package/dist/lib/lanes/provider-rejection.js +24 -6
- package/dist/lib/lanes/reconcile.js +30 -1
- package/dist/lib/lanes/spawn.d.ts +9 -1
- package/dist/lib/lanes/spawn.js +420 -14
- package/dist/lib/lanes/types.d.ts +7 -3
- package/dist/lib/lanes/types.js +3 -2
- package/dist/lib/lanes/watcher.d.ts +8 -1
- package/dist/lib/lanes/watcher.js +39 -3
- package/package.json +1 -1
|
@@ -15,7 +15,7 @@ import { MAX_LANE_DEPTH } from "./constants.js";
|
|
|
15
15
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
16
16
|
import { loadWorkerProfile } from "./profile.js";
|
|
17
17
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
18
|
-
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
18
|
+
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
19
19
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
20
20
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
21
21
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
@@ -916,6 +916,11 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
916
916
|
const repairedClassification = invalidEnvelope
|
|
917
917
|
? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
|
|
918
918
|
: classification;
|
|
919
|
+
const accountRetryReason = quota
|
|
920
|
+
? "usage_limit"
|
|
921
|
+
: providerRejection && classification.reason === "account_not_logged_in"
|
|
922
|
+
? "account_unavailable"
|
|
923
|
+
: null;
|
|
919
924
|
if (!deps.dryRun &&
|
|
920
925
|
quota &&
|
|
921
926
|
classified.account &&
|
|
@@ -923,6 +928,30 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
923
928
|
classification.reason === "account_model_unavailable")) {
|
|
924
929
|
recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
|
|
925
930
|
}
|
|
931
|
+
if (!deps.dryRun && accountRetryReason && classified.account) {
|
|
932
|
+
try {
|
|
933
|
+
const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
|
|
934
|
+
...(deps.spawnLaneDeps ?? {}),
|
|
935
|
+
admissionHost: deps.host,
|
|
936
|
+
});
|
|
937
|
+
summary.dead_host.push({
|
|
938
|
+
id: lane.lane_id,
|
|
939
|
+
from: "running",
|
|
940
|
+
to: recovered.state,
|
|
941
|
+
reason: accountRetryReason,
|
|
942
|
+
});
|
|
943
|
+
summary.heartbeats.push(`${lane.lane_id}:account-retry`);
|
|
944
|
+
continue;
|
|
945
|
+
}
|
|
946
|
+
catch (err) {
|
|
947
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
948
|
+
process.stderr.write(`hq lanes reconcile: account failover failed for ${lane.lane_id}: ${message}\n`);
|
|
949
|
+
if (readLane(hqRoot, lane.lane_id).state !== "running") {
|
|
950
|
+
summary.errors.push({ message: `${lane.lane_id}: account failover left a non-running record after ${message}` });
|
|
951
|
+
continue;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
926
955
|
if (!deps.dryRun &&
|
|
927
956
|
classification.state === "awaiting_input" &&
|
|
928
957
|
latest &&
|
|
@@ -12,7 +12,8 @@ import { type RunCompactionFn } from "./compaction.js";
|
|
|
12
12
|
import { type KnownModelsDeps } from "./models.js";
|
|
13
13
|
import { type LaunchTrigger } from "./delivery.js";
|
|
14
14
|
import { type MachineIdentity } from "./machine-identity.js";
|
|
15
|
-
import { type
|
|
15
|
+
import { type AdmissionHost } from "./admission.js";
|
|
16
|
+
import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
|
|
16
17
|
export declare const LANES_RUNS_REL: string;
|
|
17
18
|
/** Written by reconcile before a resume spawn; the worker prompt names this path. */
|
|
18
19
|
export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
|
|
@@ -32,6 +33,9 @@ export declare function resolveLaneModel(input: {
|
|
|
32
33
|
accountModel?: string;
|
|
33
34
|
providerModel?: string;
|
|
34
35
|
}): string | undefined;
|
|
36
|
+
type AutomaticAccountChangeReason = Exclude<AccountHistoryEntry["reason"], "operator_move">;
|
|
37
|
+
/** Change account only after a provider/runtime account failure, then re-admit. */
|
|
38
|
+
export declare function retryLaneOnAlternateAccount(hqRoot: string, lane: LaneRecord, reason: Extract<AutomaticAccountChangeReason, "usage_limit" | "account_unavailable">, writer?: "watcher" | "reconcile", deps?: SpawnLaneDeps): Promise<LaneRecord>;
|
|
35
39
|
export declare function appendModelFlag(provider: LaneProvider, args: string[], model: string | undefined): string[];
|
|
36
40
|
/** Add Codex fast-mode flags before the prompt/positional separator. */
|
|
37
41
|
export declare function appendFastModeFlags(provider: LaneProvider, args: string[], enabled: boolean | undefined): string[];
|
|
@@ -63,8 +67,12 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
|
|
|
63
67
|
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
|
|
64
68
|
export interface SpawnLaneDeps {
|
|
65
69
|
env?: NodeJS.ProcessEnv;
|
|
70
|
+
/** The caller's host scope for admission checks; reconcile supplies its resolved host. */
|
|
71
|
+
admissionHost?: AdmissionHost;
|
|
66
72
|
/** Provider model catalog dependencies, shared with create in tests. */
|
|
67
73
|
modelDeps?: KnownModelsDeps;
|
|
74
|
+
/** Create validated this exact account/model candidate before its admission lock. */
|
|
75
|
+
currentModelValidated?: boolean;
|
|
68
76
|
existsSync?: (p: string) => boolean;
|
|
69
77
|
spawnProcess?: SpawnProcessFn;
|
|
70
78
|
proveDetached?: ProveDetachedFn;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -11,18 +11,21 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
11
11
|
import * as fs from "node:fs";
|
|
12
12
|
import * as path from "node:path";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
|
-
import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, } from "./accounts.js";
|
|
14
|
+
import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
|
|
15
15
|
import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
|
|
16
16
|
import { installDrainHook } from "./hooks-install.js";
|
|
17
17
|
import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
|
|
18
18
|
import { appendEffortFlag, resolveLaneEffort } from "./effort.js";
|
|
19
|
-
import { validateProfileLaneModel, } from "./models.js";
|
|
19
|
+
import { defaultProbe, validateLaneModel, validateProfileLaneModel, } from "./models.js";
|
|
20
20
|
import { claimableStatesFor, planLaneLaunch, providerSupportsSessionResume, } from "./delivery.js";
|
|
21
21
|
import { AccountResolutionError, LanesError } from "./errors.js";
|
|
22
22
|
import { ignoredWorkerProfileFields, loadWorkerProfile, materializeWorkerProfile, workerProfilePrompt, } from "./profile.js";
|
|
23
23
|
import { resolveMachineIdentity, } from "./machine-identity.js";
|
|
24
|
-
import { readLane, retryUpdateLane } from "./store.js";
|
|
24
|
+
import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
|
|
25
|
+
import { copySessionFiles, planSessionCopy } from "./move-account.js";
|
|
25
26
|
import { ensureSessionHome } from "./session-relink.js";
|
|
27
|
+
import { hasEarlierEligibleQueuedLane, laneAdmissionCapReason, withLaneAdmissionLock, } from "./admission.js";
|
|
28
|
+
import { resolveAdmissionHost } from "./admission-host.js";
|
|
26
29
|
import { LANE_ENVELOPE_JSON_SCHEMA, parseLaneProvider, } from "./types.js";
|
|
27
30
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
28
31
|
/** Written by reconcile before a resume spawn; the worker prompt names this path. */
|
|
@@ -101,6 +104,365 @@ export function resolveLaneModel(input) {
|
|
|
101
104
|
}
|
|
102
105
|
return undefined;
|
|
103
106
|
}
|
|
107
|
+
function accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel) {
|
|
108
|
+
return resolveLaneModel({
|
|
109
|
+
cliModel: pinnedModel,
|
|
110
|
+
profileModel,
|
|
111
|
+
accountModel: config.providers[provider].accounts[accountId]?.model,
|
|
112
|
+
providerModel: config.providers[provider].model,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
function supportsSpawnModel(provider, accountId, config, pinnedModel, profileModel, modelDeps) {
|
|
116
|
+
const account = config.providers[provider].accounts[accountId];
|
|
117
|
+
if (account === undefined)
|
|
118
|
+
return false;
|
|
119
|
+
const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
|
|
120
|
+
if (model === undefined)
|
|
121
|
+
return true;
|
|
122
|
+
try {
|
|
123
|
+
validateLaneModel(provider, model, account.home, modelDeps);
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
if (err instanceof LanesError && err.error === "model_unsupported")
|
|
128
|
+
return false;
|
|
129
|
+
throw err;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function spawnModelFailure(provider, accountId, config, pinnedModel, profileModel, profileId, modelDeps) {
|
|
133
|
+
const account = config.providers[provider].accounts[accountId];
|
|
134
|
+
if (account === undefined)
|
|
135
|
+
return new LanesError("unknown_account", `Unknown account ${accountId} for ${provider}`, { provider, account: accountId });
|
|
136
|
+
const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
|
|
137
|
+
if (model === undefined)
|
|
138
|
+
return null;
|
|
139
|
+
try {
|
|
140
|
+
validateLaneModel(provider, model, account.home, modelDeps);
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
if (!(err instanceof LanesError) || err.error !== "model_unsupported")
|
|
145
|
+
throw err;
|
|
146
|
+
if (profileModel !== undefined && pinnedModel === undefined && profileId !== undefined) {
|
|
147
|
+
return new LanesError(err.error, err.message, {
|
|
148
|
+
...err.details,
|
|
149
|
+
profile_id: profileId,
|
|
150
|
+
worker_profile: profileId,
|
|
151
|
+
model_source: "profile",
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return err;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function selectSpawnAccount(input) {
|
|
158
|
+
const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated } = input;
|
|
159
|
+
const context = {
|
|
160
|
+
hqRoot,
|
|
161
|
+
existsSync,
|
|
162
|
+
accountDeps,
|
|
163
|
+
...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
|
|
164
|
+
};
|
|
165
|
+
if (lane.account === undefined) {
|
|
166
|
+
const modelSpecified = pinnedModel !== undefined || profileModel !== undefined;
|
|
167
|
+
const allowedAccounts = modelSpecified
|
|
168
|
+
? Object.keys(config.providers[lane.provider].accounts).filter((id) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
|
|
169
|
+
: undefined;
|
|
170
|
+
return {
|
|
171
|
+
account: selectBalancedAccount(lane.provider, config, {
|
|
172
|
+
...context,
|
|
173
|
+
...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
|
|
174
|
+
preferredAccounts: config.providers[lane.provider].preferred_accounts ?? [],
|
|
175
|
+
}),
|
|
176
|
+
changeReason: "first_assignment",
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
const current = config.providers[lane.provider].accounts[lane.account];
|
|
180
|
+
let reason;
|
|
181
|
+
let refusal;
|
|
182
|
+
const operatorMovePending = lane.pending_resume !== undefined &&
|
|
183
|
+
lane.account_history?.at(-1)?.reason === "operator_move";
|
|
184
|
+
if (current === undefined || !current.enabled) {
|
|
185
|
+
reason = "account_unavailable";
|
|
186
|
+
refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
const status = accountStatus(lane.provider, lane.account, config, {
|
|
190
|
+
existsSync,
|
|
191
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
192
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
193
|
+
});
|
|
194
|
+
if (isLoggedOut(status)) {
|
|
195
|
+
reason = "account_unavailable";
|
|
196
|
+
refusal = new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in`, { provider: lane.provider, account: lane.account });
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
try {
|
|
200
|
+
resolveAccount(lane.provider, lane.account, config, context);
|
|
201
|
+
const modelFailure = currentModelValidated
|
|
202
|
+
? null
|
|
203
|
+
: spawnModelFailure(lane.provider, lane.account, config, pinnedModel, profileModel, lane.worker_profile, modelDeps);
|
|
204
|
+
if (modelFailure !== null) {
|
|
205
|
+
reason = "model_change";
|
|
206
|
+
refusal = modelFailure;
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
return { account: resolveAccount(lane.provider, lane.account, config, context) };
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
if (err instanceof AccountResolutionError && err.error === "account_quota_exhausted") {
|
|
214
|
+
reason = "usage_limit";
|
|
215
|
+
refusal = err;
|
|
216
|
+
}
|
|
217
|
+
else if (err instanceof AccountResolutionError && err.error === "account_model_unavailable") {
|
|
218
|
+
reason = "usage_limit";
|
|
219
|
+
refusal = err;
|
|
220
|
+
}
|
|
221
|
+
else if (err instanceof LanesError && err.error === "model_unsupported") {
|
|
222
|
+
reason = "model_change";
|
|
223
|
+
refusal = err;
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
throw err;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (operatorMovePending) {
|
|
232
|
+
if (refusal instanceof Error)
|
|
233
|
+
throw refusal;
|
|
234
|
+
throw new LanesError("account_unavailable", `Explicit account move target ${lane.account} is unavailable`, {
|
|
235
|
+
provider: lane.provider,
|
|
236
|
+
account: lane.account,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
240
|
+
.filter(([id, account]) => id !== lane.account && account.enabled)
|
|
241
|
+
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
242
|
+
existsSync,
|
|
243
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
244
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
245
|
+
}).logged_in === true)
|
|
246
|
+
.filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
|
|
247
|
+
.map(([id]) => id);
|
|
248
|
+
try {
|
|
249
|
+
return {
|
|
250
|
+
account: selectBalancedAccount(lane.provider, config, { ...context, allowedAccounts }),
|
|
251
|
+
changeReason: reason,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
if (reason === "account_unavailable") {
|
|
256
|
+
if (refusal instanceof Error)
|
|
257
|
+
throw refusal;
|
|
258
|
+
throw new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in or available, and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account });
|
|
259
|
+
}
|
|
260
|
+
if (refusal instanceof Error)
|
|
261
|
+
throw refusal;
|
|
262
|
+
throw err;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function previouslyUnavailableAccounts(lane) {
|
|
266
|
+
const excluded = new Set();
|
|
267
|
+
for (const entry of [...(lane.account_history ?? [])].reverse()) {
|
|
268
|
+
if (entry.reason === "first_assignment" || entry.reason === "operator_move")
|
|
269
|
+
break;
|
|
270
|
+
if (entry.reason === "account_unavailable" && entry.from !== null)
|
|
271
|
+
excluded.add(entry.from);
|
|
272
|
+
}
|
|
273
|
+
return excluded;
|
|
274
|
+
}
|
|
275
|
+
function archivedResumeMessageText(hqRoot, laneId, messageId) {
|
|
276
|
+
if (!messageId || /[\\/\0]/.test(messageId))
|
|
277
|
+
return undefined;
|
|
278
|
+
const file = path.join(laneRunDir(hqRoot, laneId), "dropbox", "delivered", `${messageId}.resume.jsonl`);
|
|
279
|
+
let raw;
|
|
280
|
+
try {
|
|
281
|
+
raw = fs.readFileSync(file, "utf8");
|
|
282
|
+
}
|
|
283
|
+
catch (err) {
|
|
284
|
+
if (err.code === "ENOENT")
|
|
285
|
+
return undefined;
|
|
286
|
+
throw new LanesError("account_retry_context_unreadable", `Could not read the archived message needed to resume lane ${laneId}`, { lane_id: laneId, message_id: messageId });
|
|
287
|
+
}
|
|
288
|
+
const lines = raw.split(/\r?\n/).filter((line) => line.trim().length > 0);
|
|
289
|
+
if (lines.length !== 1) {
|
|
290
|
+
throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
|
|
291
|
+
}
|
|
292
|
+
let value;
|
|
293
|
+
try {
|
|
294
|
+
value = JSON.parse(lines[0]);
|
|
295
|
+
}
|
|
296
|
+
catch {
|
|
297
|
+
throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
|
|
298
|
+
}
|
|
299
|
+
if (value === null ||
|
|
300
|
+
typeof value !== "object" ||
|
|
301
|
+
value.message_id !== messageId ||
|
|
302
|
+
typeof value.text !== "string") {
|
|
303
|
+
throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
|
|
304
|
+
}
|
|
305
|
+
return value.text;
|
|
306
|
+
}
|
|
307
|
+
function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
|
|
308
|
+
const last = lane.last_resume;
|
|
309
|
+
if (last === undefined || last.session_id !== sessionId) {
|
|
310
|
+
return { session_id: sessionId, trigger: "resume", resumed_from: "interrupted", at };
|
|
311
|
+
}
|
|
312
|
+
const compact = last.compact === undefined ? {} : { compact: last.compact };
|
|
313
|
+
if (last.trigger === "answered_questions") {
|
|
314
|
+
return { session_id: sessionId, trigger: "answered_questions", at, ...compact };
|
|
315
|
+
}
|
|
316
|
+
if (last.trigger === "message") {
|
|
317
|
+
if (last.message_id === undefined) {
|
|
318
|
+
throw new LanesError("account_retry_context_missing", `The message id needed to resume lane ${lane.lane_id} is missing`, { lane_id: lane.lane_id, trigger: last.trigger });
|
|
319
|
+
}
|
|
320
|
+
const text = last.text ?? archivedResumeMessageText(hqRoot, lane.lane_id, last.message_id);
|
|
321
|
+
if (text === undefined) {
|
|
322
|
+
throw new LanesError("account_retry_context_missing", `The operator message needed to resume lane ${lane.lane_id} is unavailable`, { lane_id: lane.lane_id, message_id: last.message_id });
|
|
323
|
+
}
|
|
324
|
+
return {
|
|
325
|
+
session_id: sessionId,
|
|
326
|
+
trigger: "message",
|
|
327
|
+
text,
|
|
328
|
+
message_id: last.message_id,
|
|
329
|
+
at,
|
|
330
|
+
...compact,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
return {
|
|
334
|
+
session_id: sessionId,
|
|
335
|
+
trigger: "resume",
|
|
336
|
+
resumed_from: last.resumed_from ?? "interrupted",
|
|
337
|
+
...(last.text !== undefined ? { text: last.text } : {}),
|
|
338
|
+
...(last.message_id !== undefined ? { message_id: last.message_id } : {}),
|
|
339
|
+
at,
|
|
340
|
+
...compact,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
/** Change account only after a provider/runtime account failure, then re-admit. */
|
|
344
|
+
export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer = "watcher", deps = {}) {
|
|
345
|
+
const config = loadLanesConfig(hqRoot);
|
|
346
|
+
const existsSync = deps.existsSync ?? ((p) => fs.existsSync(p));
|
|
347
|
+
const env = deps.env ?? process.env;
|
|
348
|
+
const modelDeps = {
|
|
349
|
+
...deps.modelDeps,
|
|
350
|
+
env: deps.modelDeps?.env ?? env,
|
|
351
|
+
};
|
|
352
|
+
const accountDeps = {
|
|
353
|
+
probe: modelDeps.probe ?? defaultProbe,
|
|
354
|
+
env,
|
|
355
|
+
};
|
|
356
|
+
const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
|
|
357
|
+
const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
|
|
358
|
+
const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
|
|
359
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" ? lane.model : undefined;
|
|
360
|
+
const unavailableAccounts = previouslyUnavailableAccounts(lane);
|
|
361
|
+
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
362
|
+
.filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
|
|
363
|
+
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
364
|
+
existsSync,
|
|
365
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
366
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
367
|
+
}).logged_in === true)
|
|
368
|
+
.filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
|
|
369
|
+
.map(([id]) => id);
|
|
370
|
+
const modelContext = {
|
|
371
|
+
hqRoot,
|
|
372
|
+
existsSync,
|
|
373
|
+
accountDeps,
|
|
374
|
+
...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
|
|
375
|
+
allowedAccounts,
|
|
376
|
+
};
|
|
377
|
+
let target;
|
|
378
|
+
try {
|
|
379
|
+
target = selectBalancedAccount(lane.provider, config, modelContext);
|
|
380
|
+
}
|
|
381
|
+
catch (err) {
|
|
382
|
+
if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
|
|
383
|
+
throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
|
|
384
|
+
}
|
|
385
|
+
throw err;
|
|
386
|
+
}
|
|
387
|
+
const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
|
|
388
|
+
const sessionId = providerSupportsSessionResume(lane.provider) &&
|
|
389
|
+
typeof lane.provider_session_id === "string" &&
|
|
390
|
+
lane.provider_session_id.trim().length > 0
|
|
391
|
+
? lane.provider_session_id
|
|
392
|
+
: undefined;
|
|
393
|
+
if (source && sessionId) {
|
|
394
|
+
copySessionFiles(planSessionCopy(lane.provider, {
|
|
395
|
+
sourceHome: source.home,
|
|
396
|
+
targetHome: target.home,
|
|
397
|
+
cwd: hqRoot,
|
|
398
|
+
sessionId,
|
|
399
|
+
}));
|
|
400
|
+
}
|
|
401
|
+
const now = (deps.now ?? (() => new Date()))();
|
|
402
|
+
const at = now.toISOString();
|
|
403
|
+
const nowMs = now.getTime();
|
|
404
|
+
return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
405
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
406
|
+
if (current.state !== "running" ||
|
|
407
|
+
current.account !== lane.account ||
|
|
408
|
+
current.provider_session_id !== lane.provider_session_id) {
|
|
409
|
+
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
|
410
|
+
lane_id: lane.lane_id,
|
|
411
|
+
state: current.state,
|
|
412
|
+
account: current.account ?? null,
|
|
413
|
+
provider_session_id_changed: current.provider_session_id !== lane.provider_session_id,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
const moveToRetryQueue = (row) => {
|
|
417
|
+
const entry = {
|
|
418
|
+
from: row.account ?? null,
|
|
419
|
+
to: target.id,
|
|
420
|
+
at,
|
|
421
|
+
reason,
|
|
422
|
+
};
|
|
423
|
+
return {
|
|
424
|
+
...row,
|
|
425
|
+
state: "queued",
|
|
426
|
+
queued_reason: "account_retry",
|
|
427
|
+
queued_at: at,
|
|
428
|
+
admission_reserved_at: undefined,
|
|
429
|
+
terminal_reason: undefined,
|
|
430
|
+
terminal_details: undefined,
|
|
431
|
+
account: target.id,
|
|
432
|
+
account_history: appendCappedHistory(row.account_history ?? [], entry),
|
|
433
|
+
pending_resume: sessionId
|
|
434
|
+
? pendingResumeForRetry(hqRoot, row, sessionId, at)
|
|
435
|
+
: undefined,
|
|
436
|
+
};
|
|
437
|
+
};
|
|
438
|
+
const candidate = moveToRetryQueue(current);
|
|
439
|
+
const admissionLanes = listLaneIds(hqRoot)
|
|
440
|
+
.filter((id) => id !== current.lane_id)
|
|
441
|
+
.map((id) => readLane(hqRoot, id));
|
|
442
|
+
admissionLanes.push(candidate);
|
|
443
|
+
const profileMaxFor = (row) => row.worker_profile === undefined
|
|
444
|
+
? undefined
|
|
445
|
+
: loadWorkerProfile(hqRoot, row.worker_profile).meta.max_concurrent;
|
|
446
|
+
const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
|
|
447
|
+
const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
|
|
448
|
+
const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
|
|
449
|
+
if (row.state !== "running" ||
|
|
450
|
+
row.account !== lane.account ||
|
|
451
|
+
row.provider_session_id !== lane.provider_session_id) {
|
|
452
|
+
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
|
453
|
+
lane_id: lane.lane_id,
|
|
454
|
+
state: row.state,
|
|
455
|
+
account: row.account ?? null,
|
|
456
|
+
provider_session_id_changed: row.provider_session_id !== lane.provider_session_id,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
return moveToRetryQueue(row);
|
|
460
|
+
});
|
|
461
|
+
if (capacityReason !== undefined || earlierEligibleLane)
|
|
462
|
+
return moved;
|
|
463
|
+
return spawnLaneWorker(hqRoot, moved, deps);
|
|
464
|
+
});
|
|
465
|
+
}
|
|
104
466
|
export function appendModelFlag(provider, args, model) {
|
|
105
467
|
if (model === undefined)
|
|
106
468
|
return args;
|
|
@@ -1157,16 +1519,28 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1157
1519
|
const pinnedModel = existing.model_source === undefined || existing.model_source === "cli"
|
|
1158
1520
|
? existing.model
|
|
1159
1521
|
: undefined;
|
|
1160
|
-
|
|
1522
|
+
const accountDeps = {
|
|
1523
|
+
probe: modelDeps.probe ?? defaultProbe,
|
|
1524
|
+
env,
|
|
1525
|
+
};
|
|
1526
|
+
let selection;
|
|
1161
1527
|
try {
|
|
1162
|
-
|
|
1528
|
+
selection = selectSpawnAccount({
|
|
1163
1529
|
hqRoot,
|
|
1530
|
+
lane: existing,
|
|
1531
|
+
config,
|
|
1164
1532
|
existsSync,
|
|
1165
|
-
|
|
1533
|
+
accountDeps,
|
|
1534
|
+
modelDeps,
|
|
1535
|
+
pinnedModel,
|
|
1536
|
+
profileModel,
|
|
1537
|
+
currentModelValidated: deps.currentModelValidated === true,
|
|
1166
1538
|
});
|
|
1167
1539
|
}
|
|
1168
1540
|
catch (err) {
|
|
1169
|
-
if (err instanceof AccountResolutionError
|
|
1541
|
+
if (err instanceof AccountResolutionError ||
|
|
1542
|
+
(err instanceof LanesError &&
|
|
1543
|
+
(err.error === "account_not_logged_in" || err.error === "model_unsupported"))) {
|
|
1170
1544
|
if (markPreSpawnFailure) {
|
|
1171
1545
|
markSpawnFailure(hqRoot, lane.lane_id, err.error, err.details);
|
|
1172
1546
|
}
|
|
@@ -1175,6 +1549,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1175
1549
|
process.stderr.write(`hq lanes: account bind failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
1176
1550
|
throw err;
|
|
1177
1551
|
}
|
|
1552
|
+
const resolved = selection.account;
|
|
1178
1553
|
const status = accountStatus(lane.provider, resolved.id, config, { existsSync });
|
|
1179
1554
|
if (isLoggedOut(status)) {
|
|
1180
1555
|
const failed = markPreSpawnFailure
|
|
@@ -1185,8 +1560,20 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1185
1560
|
: existing;
|
|
1186
1561
|
throw new LanesError("account_not_logged_in", `Account ${resolved.id} for ${lane.provider} is not logged in (no credential file)`, { provider: lane.provider, account: resolved.id, lane_id: lane.lane_id, lane: failed });
|
|
1187
1562
|
}
|
|
1188
|
-
if (
|
|
1189
|
-
|
|
1563
|
+
if (existing.account !== resolved.id) {
|
|
1564
|
+
const source = existing.account
|
|
1565
|
+
? config.providers[existing.provider].accounts[existing.account]
|
|
1566
|
+
: undefined;
|
|
1567
|
+
if (source &&
|
|
1568
|
+
existing.provider_session_id &&
|
|
1569
|
+
providerSupportsSessionResume(existing.provider)) {
|
|
1570
|
+
copySessionFiles(planSessionCopy(existing.provider, {
|
|
1571
|
+
sourceHome: source.home,
|
|
1572
|
+
targetHome: resolved.home,
|
|
1573
|
+
cwd: hqRoot,
|
|
1574
|
+
sessionId: existing.provider_session_id,
|
|
1575
|
+
}));
|
|
1576
|
+
}
|
|
1190
1577
|
}
|
|
1191
1578
|
if (plan.mode === "resume" && existing.provider !== "grok") {
|
|
1192
1579
|
ensureSessionHome({
|
|
@@ -1199,11 +1586,26 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1199
1586
|
providerHomeAtSpawn: existing.provider_home_at_spawn,
|
|
1200
1587
|
});
|
|
1201
1588
|
}
|
|
1202
|
-
if (
|
|
1589
|
+
if (existing.account !== resolved.id) {
|
|
1590
|
+
const at = (deps.now ?? (() => new Date()))().toISOString();
|
|
1591
|
+
const reason = selection.changeReason ?? "first_assignment";
|
|
1203
1592
|
retryUpdateLane(hqRoot, lane.lane_id, "spawn", (current) => {
|
|
1204
1593
|
if (current.account === resolved.id)
|
|
1205
1594
|
return current;
|
|
1206
|
-
|
|
1595
|
+
if (current.account !== existing.account) {
|
|
1596
|
+
throw new LanesError("account_bind_conflict", `Lane ${lane.lane_id} changed accounts while its worker spawn was prepared`, { lane_id: lane.lane_id, expected_account: existing.account ?? null, account: current.account ?? null });
|
|
1597
|
+
}
|
|
1598
|
+
const entry = {
|
|
1599
|
+
from: current.account ?? null,
|
|
1600
|
+
to: resolved.id,
|
|
1601
|
+
at,
|
|
1602
|
+
reason,
|
|
1603
|
+
};
|
|
1604
|
+
return {
|
|
1605
|
+
...current,
|
|
1606
|
+
account: resolved.id,
|
|
1607
|
+
account_history: appendCappedHistory(current.account_history ?? [], entry),
|
|
1608
|
+
};
|
|
1207
1609
|
});
|
|
1208
1610
|
}
|
|
1209
1611
|
try {
|
|
@@ -1255,10 +1657,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1255
1657
|
? trigger.text
|
|
1256
1658
|
: pendingResume?.text,
|
|
1257
1659
|
resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
|
|
1258
|
-
answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
|
|
1660
|
+
answersPath: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
|
|
1259
1661
|
? answersPath
|
|
1260
1662
|
: undefined,
|
|
1261
|
-
answers: trigger.kind === "answered_questions" && existsSync(answersPath)
|
|
1663
|
+
answers: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
|
|
1262
1664
|
? readAnsweredQuestionLines(answersPath)
|
|
1263
1665
|
: undefined,
|
|
1264
1666
|
}),
|
|
@@ -1425,6 +1827,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1425
1827
|
...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
|
|
1426
1828
|
...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
|
|
1427
1829
|
...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
|
|
1830
|
+
...(pendingResume.compact !== undefined ? { compact: pendingResume.compact } : {}),
|
|
1428
1831
|
};
|
|
1429
1832
|
}
|
|
1430
1833
|
else if (resume && trigger.kind !== "admit") {
|
|
@@ -1433,7 +1836,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1433
1836
|
at: nowIso,
|
|
1434
1837
|
account: resolved.id,
|
|
1435
1838
|
trigger: trigger.kind,
|
|
1436
|
-
...(trigger.kind === "message"
|
|
1839
|
+
...(trigger.kind === "message"
|
|
1840
|
+
? { message_id: trigger.message_id, text: trigger.text }
|
|
1841
|
+
: {}),
|
|
1842
|
+
...(launch.compact !== undefined ? { compact: launch.compact } : {}),
|
|
1437
1843
|
...(trigger.kind === "resume"
|
|
1438
1844
|
? {
|
|
1439
1845
|
resumed_from: trigger.from,
|
|
@@ -166,7 +166,7 @@ export declare const TERMINAL_LANE_STATES: readonly ["done", "failed", "interrup
|
|
|
166
166
|
export type TerminalLaneState = (typeof TERMINAL_LANE_STATES)[number];
|
|
167
167
|
export declare const LANE_PROVIDERS: readonly ["claude", "codex", "grok"];
|
|
168
168
|
export type LaneProvider = (typeof LANE_PROVIDERS)[number];
|
|
169
|
-
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing"];
|
|
169
|
+
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry"];
|
|
170
170
|
export type QueuedReason = (typeof QUEUED_REASONS)[number];
|
|
171
171
|
export declare function isQueuedReason(value: unknown): value is QueuedReason;
|
|
172
172
|
export declare const HOST_KINDS: readonly ["local", "outpost"];
|
|
@@ -436,11 +436,11 @@ export interface AccountHistoryEntry {
|
|
|
436
436
|
from: string | null;
|
|
437
437
|
to: string;
|
|
438
438
|
at: string;
|
|
439
|
-
reason: "operator_move";
|
|
439
|
+
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "operator_move";
|
|
440
440
|
}
|
|
441
441
|
export interface PendingResume {
|
|
442
442
|
session_id: string;
|
|
443
|
-
trigger: "message" | "resume";
|
|
443
|
+
trigger: "answered_questions" | "message" | "resume";
|
|
444
444
|
resumed_from?: "interrupted" | "done" | "blocked";
|
|
445
445
|
text?: string;
|
|
446
446
|
message_id?: string;
|
|
@@ -461,6 +461,8 @@ export interface LaneResumeRecord {
|
|
|
461
461
|
*/
|
|
462
462
|
resumed_from?: "interrupted" | "done" | "blocked";
|
|
463
463
|
text?: string;
|
|
464
|
+
/** False when the continued turn explicitly skipped compaction. */
|
|
465
|
+
compact?: boolean;
|
|
464
466
|
}
|
|
465
467
|
export interface LaneRunUsage {
|
|
466
468
|
cache_read_input_tokens: number | null;
|
|
@@ -575,6 +577,8 @@ export interface ProviderConfig {
|
|
|
575
577
|
* Account pick is least-loaded among enabled, logged-in accounts.
|
|
576
578
|
*/
|
|
577
579
|
default_account?: string;
|
|
580
|
+
/** Ordered account preferences used only for a lane's first assignment. */
|
|
581
|
+
preferred_accounts?: string[];
|
|
578
582
|
/** Default model for this provider when the account does not set one. */
|
|
579
583
|
model?: string;
|
|
580
584
|
/** Default effort for this provider when the account does not set one. */
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -103,6 +103,7 @@ export const QUEUED_REASONS = [
|
|
|
103
103
|
"queue_position",
|
|
104
104
|
"throttle",
|
|
105
105
|
"provider_binary_missing",
|
|
106
|
+
"account_retry",
|
|
106
107
|
];
|
|
107
108
|
export function isQueuedReason(value) {
|
|
108
109
|
return typeof value === "string" && QUEUED_REASONS.includes(value);
|
|
@@ -215,11 +216,11 @@ export const LANE_FIELD_WRITERS = {
|
|
|
215
216
|
schema_version: ["creator"], lane_id: ["creator"], company: ["creator"],
|
|
216
217
|
project_id: ["command"], story_id: ["command"], stories: ["creator", "command", "watcher", "reconcile", "spawn"], thread_id: ["creator", "command", "spawn"], mesh_company: ["creator"],
|
|
217
218
|
depth: ["command"], senior: ["command"], senior_history: ["command"],
|
|
218
|
-
state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "reconcile"], queued_at: ["creator", "command", "spawn"], admission_reserved_at: ["creator", "command", "reconcile", "spawn"],
|
|
219
|
+
state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "watcher", "reconcile"], queued_at: ["creator", "command", "spawn", "watcher", "reconcile"], admission_reserved_at: ["creator", "command", "reconcile", "spawn", "watcher"],
|
|
219
220
|
pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
|
|
220
221
|
pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
|
|
221
222
|
priority: ["command"], worker: ["creator"], worker_path: ["creator"], worker_pack: ["creator"], worker_profile: ["creator"], worker_profile_hash: ["creator", "spawn"], worker_profile_ignored: ["creator", "spawn"],
|
|
222
|
-
provider: ["creator"], account: ["creator", "spawn", "command", "reconcile"], admission_accounts: ["creator", "command"], account_history: ["command"], pending_resume: ["command", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
223
|
+
provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
|
|
223
224
|
version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
|
|
224
225
|
last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"], fast_mode: ["spawn"],
|
|
225
226
|
last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
|