@indigoai-us/hq-cli 5.153.0 → 5.155.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/commands/agent-mcp.d.ts +4 -1
- package/dist/commands/agent-mcp.js +43 -2
- package/dist/commands/files-browse.d.ts +6 -0
- package/dist/commands/files-browse.js +3 -2
- package/dist/commands/lanes.d.ts +3 -11
- package/dist/commands/lanes.js +68 -37
- package/dist/lib/agent-kit/mcp/attachments.d.ts +99 -0
- package/dist/lib/agent-kit/mcp/attachments.js +188 -0
- package/dist/lib/agent-kit/mcp/tools.d.ts +13 -2
- package/dist/lib/agent-kit/mcp/tools.js +66 -3
- package/dist/lib/agent-kit/paths.d.ts +3 -0
- package/dist/lib/agent-kit/paths.js +2 -0
- package/dist/lib/lanes/accounts.d.ts +16 -11
- package/dist/lib/lanes/accounts.js +50 -8
- package/dist/lib/lanes/admission-host.d.ts +19 -0
- package/dist/lib/lanes/admission-host.js +33 -0
- package/dist/lib/lanes/config.js +25 -0
- package/dist/lib/lanes/provider-rejection.d.ts +4 -4
- package/dist/lib/lanes/provider-rejection.js +24 -6
- package/dist/lib/lanes/reconcile.js +30 -1
- package/dist/lib/lanes/spawn.d.ts +9 -1
- package/dist/lib/lanes/spawn.js +420 -14
- package/dist/lib/lanes/types.d.ts +7 -3
- package/dist/lib/lanes/types.js +3 -2
- package/dist/lib/lanes/watcher.d.ts +8 -1
- package/dist/lib/lanes/watcher.js +39 -3
- package/dist/utils/vault-api.d.ts +7 -1
- package/dist/utils/vault-api.js +22 -12
- package/package.json +1 -1
|
@@ -24,6 +24,25 @@ export const PROVIDER_REJECTED_REASON = "provider_rejected";
|
|
|
24
24
|
export const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
|
|
25
25
|
/** Status the watcher routes to the quota path instead of `provider_rejected`. */
|
|
26
26
|
export const QUOTA_STATUS = 429;
|
|
27
|
+
/** Exact auth failures observed from each installed CLI in an empty home. */
|
|
28
|
+
function isProviderLoginMessage(provider, message) {
|
|
29
|
+
switch (provider) {
|
|
30
|
+
case "claude":
|
|
31
|
+
return /\bnot logged in\b/i.test(message);
|
|
32
|
+
case "codex":
|
|
33
|
+
return /\bnot logged in\b/i.test(message) ||
|
|
34
|
+
/\bmissing bearer or basic authentication in header\b/i.test(message);
|
|
35
|
+
case "grok":
|
|
36
|
+
return /^\s*not signed in\b/i.test(message);
|
|
37
|
+
default: {
|
|
38
|
+
const _exhaustive = provider;
|
|
39
|
+
return _exhaustive;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function isProviderLoginFailure(failure) {
|
|
44
|
+
return failure !== null && isProviderLoginMessage(failure.provider, failure.message);
|
|
45
|
+
}
|
|
27
46
|
function isPlainObject(value) {
|
|
28
47
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
29
48
|
}
|
|
@@ -134,10 +153,9 @@ export function parseCodexTurnFailed(text) {
|
|
|
134
153
|
}
|
|
135
154
|
/**
|
|
136
155
|
* Claude `-p --output-format json`: the last `type: "result"` object with
|
|
137
|
-
* `is_error: true` and a numeric `api_error_status
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
* existing classification.
|
|
156
|
+
* `is_error: true` and a numeric `api_error_status`, or the CLI's explicit
|
|
157
|
+
* unauthenticated message. Other status-less errors keep the existing
|
|
158
|
+
* classification because they may be quota text or a worker failure.
|
|
141
159
|
*/
|
|
142
160
|
export function parseClaudeResultRejection(text) {
|
|
143
161
|
// The last `result` decides: a success or a status-less error there is
|
|
@@ -148,9 +166,9 @@ export function parseClaudeResultRejection(text) {
|
|
|
148
166
|
if (rec.is_error !== true)
|
|
149
167
|
return NOT_REJECTED;
|
|
150
168
|
const status = statusOf(rec.api_error_status);
|
|
151
|
-
if (status === null)
|
|
152
|
-
return NOT_REJECTED;
|
|
153
169
|
const message = typeof rec.result === "string" ? rec.result : "";
|
|
170
|
+
if (status === null && !isProviderLoginMessage("claude", message))
|
|
171
|
+
return NOT_REJECTED;
|
|
154
172
|
return {
|
|
155
173
|
reason: PROVIDER_REJECTED_REASON,
|
|
156
174
|
provider: "claude",
|
|
@@ -15,7 +15,7 @@ import { MAX_LANE_DEPTH } from "./constants.js";
|
|
|
15
15
|
import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
|
|
16
16
|
import { loadWorkerProfile } from "./profile.js";
|
|
17
17
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
18
|
-
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
18
|
+
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
19
19
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
20
20
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
21
21
|
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
@@ -916,6 +916,11 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
916
916
|
const repairedClassification = invalidEnvelope
|
|
917
917
|
? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
|
|
918
918
|
: classification;
|
|
919
|
+
const accountRetryReason = quota
|
|
920
|
+
? "usage_limit"
|
|
921
|
+
: providerRejection && classification.reason === "account_not_logged_in"
|
|
922
|
+
? "account_unavailable"
|
|
923
|
+
: null;
|
|
919
924
|
if (!deps.dryRun &&
|
|
920
925
|
quota &&
|
|
921
926
|
classified.account &&
|
|
@@ -923,6 +928,30 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
923
928
|
classification.reason === "account_model_unavailable")) {
|
|
924
929
|
recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
|
|
925
930
|
}
|
|
931
|
+
if (!deps.dryRun && accountRetryReason && classified.account) {
|
|
932
|
+
try {
|
|
933
|
+
const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
|
|
934
|
+
...(deps.spawnLaneDeps ?? {}),
|
|
935
|
+
admissionHost: deps.host,
|
|
936
|
+
});
|
|
937
|
+
summary.dead_host.push({
|
|
938
|
+
id: lane.lane_id,
|
|
939
|
+
from: "running",
|
|
940
|
+
to: recovered.state,
|
|
941
|
+
reason: accountRetryReason,
|
|
942
|
+
});
|
|
943
|
+
summary.heartbeats.push(`${lane.lane_id}:account-retry`);
|
|
944
|
+
continue;
|
|
945
|
+
}
|
|
946
|
+
catch (err) {
|
|
947
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
948
|
+
process.stderr.write(`hq lanes reconcile: account failover failed for ${lane.lane_id}: ${message}\n`);
|
|
949
|
+
if (readLane(hqRoot, lane.lane_id).state !== "running") {
|
|
950
|
+
summary.errors.push({ message: `${lane.lane_id}: account failover left a non-running record after ${message}` });
|
|
951
|
+
continue;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
926
955
|
if (!deps.dryRun &&
|
|
927
956
|
classification.state === "awaiting_input" &&
|
|
928
957
|
latest &&
|
|
@@ -12,7 +12,8 @@ import { type RunCompactionFn } from "./compaction.js";
|
|
|
12
12
|
import { type KnownModelsDeps } from "./models.js";
|
|
13
13
|
import { type LaunchTrigger } from "./delivery.js";
|
|
14
14
|
import { type MachineIdentity } from "./machine-identity.js";
|
|
15
|
-
import { type
|
|
15
|
+
import { type AdmissionHost } from "./admission.js";
|
|
16
|
+
import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
|
|
16
17
|
export declare const LANES_RUNS_REL: string;
|
|
17
18
|
/** Written by reconcile before a resume spawn; the worker prompt names this path. */
|
|
18
19
|
export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
|
|
@@ -32,6 +33,9 @@ export declare function resolveLaneModel(input: {
|
|
|
32
33
|
accountModel?: string;
|
|
33
34
|
providerModel?: string;
|
|
34
35
|
}): string | undefined;
|
|
36
|
+
type AutomaticAccountChangeReason = Exclude<AccountHistoryEntry["reason"], "operator_move">;
|
|
37
|
+
/** Change account only after a provider/runtime account failure, then re-admit. */
|
|
38
|
+
export declare function retryLaneOnAlternateAccount(hqRoot: string, lane: LaneRecord, reason: Extract<AutomaticAccountChangeReason, "usage_limit" | "account_unavailable">, writer?: "watcher" | "reconcile", deps?: SpawnLaneDeps): Promise<LaneRecord>;
|
|
35
39
|
export declare function appendModelFlag(provider: LaneProvider, args: string[], model: string | undefined): string[];
|
|
36
40
|
/** Add Codex fast-mode flags before the prompt/positional separator. */
|
|
37
41
|
export declare function appendFastModeFlags(provider: LaneProvider, args: string[], enabled: boolean | undefined): string[];
|
|
@@ -63,8 +67,12 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
|
|
|
63
67
|
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
|
|
64
68
|
export interface SpawnLaneDeps {
|
|
65
69
|
env?: NodeJS.ProcessEnv;
|
|
70
|
+
/** The caller's host scope for admission checks; reconcile supplies its resolved host. */
|
|
71
|
+
admissionHost?: AdmissionHost;
|
|
66
72
|
/** Provider model catalog dependencies, shared with create in tests. */
|
|
67
73
|
modelDeps?: KnownModelsDeps;
|
|
74
|
+
/** Create validated this exact account/model candidate before its admission lock. */
|
|
75
|
+
currentModelValidated?: boolean;
|
|
68
76
|
existsSync?: (p: string) => boolean;
|
|
69
77
|
spawnProcess?: SpawnProcessFn;
|
|
70
78
|
proveDetached?: ProveDetachedFn;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -11,18 +11,21 @@ import { spawn, spawnSync } from "node:child_process";
|
|
|
11
11
|
import * as fs from "node:fs";
|
|
12
12
|
import * as path from "node:path";
|
|
13
13
|
import { fileURLToPath } from "node:url";
|
|
14
|
-
import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, } from "./accounts.js";
|
|
14
|
+
import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
|
|
15
15
|
import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
|
|
16
16
|
import { installDrainHook } from "./hooks-install.js";
|
|
17
17
|
import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
|
|
18
18
|
import { appendEffortFlag, resolveLaneEffort } from "./effort.js";
|
|
19
|
-
import { validateProfileLaneModel, } from "./models.js";
|
|
19
|
+
import { defaultProbe, validateLaneModel, validateProfileLaneModel, } from "./models.js";
|
|
20
20
|
import { claimableStatesFor, planLaneLaunch, providerSupportsSessionResume, } from "./delivery.js";
|
|
21
21
|
import { AccountResolutionError, LanesError } from "./errors.js";
|
|
22
22
|
import { ignoredWorkerProfileFields, loadWorkerProfile, materializeWorkerProfile, workerProfilePrompt, } from "./profile.js";
|
|
23
23
|
import { resolveMachineIdentity, } from "./machine-identity.js";
|
|
24
|
-
import { readLane, retryUpdateLane } from "./store.js";
|
|
24
|
+
import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
|
|
25
|
+
import { copySessionFiles, planSessionCopy } from "./move-account.js";
|
|
25
26
|
import { ensureSessionHome } from "./session-relink.js";
|
|
27
|
+
import { hasEarlierEligibleQueuedLane, laneAdmissionCapReason, withLaneAdmissionLock, } from "./admission.js";
|
|
28
|
+
import { resolveAdmissionHost } from "./admission-host.js";
|
|
26
29
|
import { LANE_ENVELOPE_JSON_SCHEMA, parseLaneProvider, } from "./types.js";
|
|
27
30
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
28
31
|
/** Written by reconcile before a resume spawn; the worker prompt names this path. */
|
|
@@ -101,6 +104,365 @@ export function resolveLaneModel(input) {
|
|
|
101
104
|
}
|
|
102
105
|
return undefined;
|
|
103
106
|
}
|
|
107
|
+
function accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel) {
|
|
108
|
+
return resolveLaneModel({
|
|
109
|
+
cliModel: pinnedModel,
|
|
110
|
+
profileModel,
|
|
111
|
+
accountModel: config.providers[provider].accounts[accountId]?.model,
|
|
112
|
+
providerModel: config.providers[provider].model,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
function supportsSpawnModel(provider, accountId, config, pinnedModel, profileModel, modelDeps) {
|
|
116
|
+
const account = config.providers[provider].accounts[accountId];
|
|
117
|
+
if (account === undefined)
|
|
118
|
+
return false;
|
|
119
|
+
const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
|
|
120
|
+
if (model === undefined)
|
|
121
|
+
return true;
|
|
122
|
+
try {
|
|
123
|
+
validateLaneModel(provider, model, account.home, modelDeps);
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
if (err instanceof LanesError && err.error === "model_unsupported")
|
|
128
|
+
return false;
|
|
129
|
+
throw err;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function spawnModelFailure(provider, accountId, config, pinnedModel, profileModel, profileId, modelDeps) {
|
|
133
|
+
const account = config.providers[provider].accounts[accountId];
|
|
134
|
+
if (account === undefined)
|
|
135
|
+
return new LanesError("unknown_account", `Unknown account ${accountId} for ${provider}`, { provider, account: accountId });
|
|
136
|
+
const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
|
|
137
|
+
if (model === undefined)
|
|
138
|
+
return null;
|
|
139
|
+
try {
|
|
140
|
+
validateLaneModel(provider, model, account.home, modelDeps);
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
if (!(err instanceof LanesError) || err.error !== "model_unsupported")
|
|
145
|
+
throw err;
|
|
146
|
+
if (profileModel !== undefined && pinnedModel === undefined && profileId !== undefined) {
|
|
147
|
+
return new LanesError(err.error, err.message, {
|
|
148
|
+
...err.details,
|
|
149
|
+
profile_id: profileId,
|
|
150
|
+
worker_profile: profileId,
|
|
151
|
+
model_source: "profile",
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return err;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function selectSpawnAccount(input) {
|
|
158
|
+
const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated } = input;
|
|
159
|
+
const context = {
|
|
160
|
+
hqRoot,
|
|
161
|
+
existsSync,
|
|
162
|
+
accountDeps,
|
|
163
|
+
...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
|
|
164
|
+
};
|
|
165
|
+
if (lane.account === undefined) {
|
|
166
|
+
const modelSpecified = pinnedModel !== undefined || profileModel !== undefined;
|
|
167
|
+
const allowedAccounts = modelSpecified
|
|
168
|
+
? Object.keys(config.providers[lane.provider].accounts).filter((id) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
|
|
169
|
+
: undefined;
|
|
170
|
+
return {
|
|
171
|
+
account: selectBalancedAccount(lane.provider, config, {
|
|
172
|
+
...context,
|
|
173
|
+
...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
|
|
174
|
+
preferredAccounts: config.providers[lane.provider].preferred_accounts ?? [],
|
|
175
|
+
}),
|
|
176
|
+
changeReason: "first_assignment",
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
const current = config.providers[lane.provider].accounts[lane.account];
|
|
180
|
+
let reason;
|
|
181
|
+
let refusal;
|
|
182
|
+
const operatorMovePending = lane.pending_resume !== undefined &&
|
|
183
|
+
lane.account_history?.at(-1)?.reason === "operator_move";
|
|
184
|
+
if (current === undefined || !current.enabled) {
|
|
185
|
+
reason = "account_unavailable";
|
|
186
|
+
refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
const status = accountStatus(lane.provider, lane.account, config, {
|
|
190
|
+
existsSync,
|
|
191
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
192
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
193
|
+
});
|
|
194
|
+
if (isLoggedOut(status)) {
|
|
195
|
+
reason = "account_unavailable";
|
|
196
|
+
refusal = new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in`, { provider: lane.provider, account: lane.account });
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
try {
|
|
200
|
+
resolveAccount(lane.provider, lane.account, config, context);
|
|
201
|
+
const modelFailure = currentModelValidated
|
|
202
|
+
? null
|
|
203
|
+
: spawnModelFailure(lane.provider, lane.account, config, pinnedModel, profileModel, lane.worker_profile, modelDeps);
|
|
204
|
+
if (modelFailure !== null) {
|
|
205
|
+
reason = "model_change";
|
|
206
|
+
refusal = modelFailure;
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
return { account: resolveAccount(lane.provider, lane.account, config, context) };
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
if (err instanceof AccountResolutionError && err.error === "account_quota_exhausted") {
|
|
214
|
+
reason = "usage_limit";
|
|
215
|
+
refusal = err;
|
|
216
|
+
}
|
|
217
|
+
else if (err instanceof AccountResolutionError && err.error === "account_model_unavailable") {
|
|
218
|
+
reason = "usage_limit";
|
|
219
|
+
refusal = err;
|
|
220
|
+
}
|
|
221
|
+
else if (err instanceof LanesError && err.error === "model_unsupported") {
|
|
222
|
+
reason = "model_change";
|
|
223
|
+
refusal = err;
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
throw err;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (operatorMovePending) {
|
|
232
|
+
if (refusal instanceof Error)
|
|
233
|
+
throw refusal;
|
|
234
|
+
throw new LanesError("account_unavailable", `Explicit account move target ${lane.account} is unavailable`, {
|
|
235
|
+
provider: lane.provider,
|
|
236
|
+
account: lane.account,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
240
|
+
.filter(([id, account]) => id !== lane.account && account.enabled)
|
|
241
|
+
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
242
|
+
existsSync,
|
|
243
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
244
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
245
|
+
}).logged_in === true)
|
|
246
|
+
.filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
|
|
247
|
+
.map(([id]) => id);
|
|
248
|
+
try {
|
|
249
|
+
return {
|
|
250
|
+
account: selectBalancedAccount(lane.provider, config, { ...context, allowedAccounts }),
|
|
251
|
+
changeReason: reason,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
if (reason === "account_unavailable") {
|
|
256
|
+
if (refusal instanceof Error)
|
|
257
|
+
throw refusal;
|
|
258
|
+
throw new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in or available, and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account });
|
|
259
|
+
}
|
|
260
|
+
if (refusal instanceof Error)
|
|
261
|
+
throw refusal;
|
|
262
|
+
throw err;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function previouslyUnavailableAccounts(lane) {
|
|
266
|
+
const excluded = new Set();
|
|
267
|
+
for (const entry of [...(lane.account_history ?? [])].reverse()) {
|
|
268
|
+
if (entry.reason === "first_assignment" || entry.reason === "operator_move")
|
|
269
|
+
break;
|
|
270
|
+
if (entry.reason === "account_unavailable" && entry.from !== null)
|
|
271
|
+
excluded.add(entry.from);
|
|
272
|
+
}
|
|
273
|
+
return excluded;
|
|
274
|
+
}
|
|
275
|
+
function archivedResumeMessageText(hqRoot, laneId, messageId) {
|
|
276
|
+
if (!messageId || /[\\/\0]/.test(messageId))
|
|
277
|
+
return undefined;
|
|
278
|
+
const file = path.join(laneRunDir(hqRoot, laneId), "dropbox", "delivered", `${messageId}.resume.jsonl`);
|
|
279
|
+
let raw;
|
|
280
|
+
try {
|
|
281
|
+
raw = fs.readFileSync(file, "utf8");
|
|
282
|
+
}
|
|
283
|
+
catch (err) {
|
|
284
|
+
if (err.code === "ENOENT")
|
|
285
|
+
return undefined;
|
|
286
|
+
throw new LanesError("account_retry_context_unreadable", `Could not read the archived message needed to resume lane ${laneId}`, { lane_id: laneId, message_id: messageId });
|
|
287
|
+
}
|
|
288
|
+
const lines = raw.split(/\r?\n/).filter((line) => line.trim().length > 0);
|
|
289
|
+
if (lines.length !== 1) {
|
|
290
|
+
throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
|
|
291
|
+
}
|
|
292
|
+
let value;
|
|
293
|
+
try {
|
|
294
|
+
value = JSON.parse(lines[0]);
|
|
295
|
+
}
|
|
296
|
+
catch {
|
|
297
|
+
throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
|
|
298
|
+
}
|
|
299
|
+
if (value === null ||
|
|
300
|
+
typeof value !== "object" ||
|
|
301
|
+
value.message_id !== messageId ||
|
|
302
|
+
typeof value.text !== "string") {
|
|
303
|
+
throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
|
|
304
|
+
}
|
|
305
|
+
return value.text;
|
|
306
|
+
}
|
|
307
|
+
function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
|
|
308
|
+
const last = lane.last_resume;
|
|
309
|
+
if (last === undefined || last.session_id !== sessionId) {
|
|
310
|
+
return { session_id: sessionId, trigger: "resume", resumed_from: "interrupted", at };
|
|
311
|
+
}
|
|
312
|
+
const compact = last.compact === undefined ? {} : { compact: last.compact };
|
|
313
|
+
if (last.trigger === "answered_questions") {
|
|
314
|
+
return { session_id: sessionId, trigger: "answered_questions", at, ...compact };
|
|
315
|
+
}
|
|
316
|
+
if (last.trigger === "message") {
|
|
317
|
+
if (last.message_id === undefined) {
|
|
318
|
+
throw new LanesError("account_retry_context_missing", `The message id needed to resume lane ${lane.lane_id} is missing`, { lane_id: lane.lane_id, trigger: last.trigger });
|
|
319
|
+
}
|
|
320
|
+
const text = last.text ?? archivedResumeMessageText(hqRoot, lane.lane_id, last.message_id);
|
|
321
|
+
if (text === undefined) {
|
|
322
|
+
throw new LanesError("account_retry_context_missing", `The operator message needed to resume lane ${lane.lane_id} is unavailable`, { lane_id: lane.lane_id, message_id: last.message_id });
|
|
323
|
+
}
|
|
324
|
+
return {
|
|
325
|
+
session_id: sessionId,
|
|
326
|
+
trigger: "message",
|
|
327
|
+
text,
|
|
328
|
+
message_id: last.message_id,
|
|
329
|
+
at,
|
|
330
|
+
...compact,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
return {
|
|
334
|
+
session_id: sessionId,
|
|
335
|
+
trigger: "resume",
|
|
336
|
+
resumed_from: last.resumed_from ?? "interrupted",
|
|
337
|
+
...(last.text !== undefined ? { text: last.text } : {}),
|
|
338
|
+
...(last.message_id !== undefined ? { message_id: last.message_id } : {}),
|
|
339
|
+
at,
|
|
340
|
+
...compact,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
/** Change account only after a provider/runtime account failure, then re-admit. */
|
|
344
|
+
export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer = "watcher", deps = {}) {
|
|
345
|
+
const config = loadLanesConfig(hqRoot);
|
|
346
|
+
const existsSync = deps.existsSync ?? ((p) => fs.existsSync(p));
|
|
347
|
+
const env = deps.env ?? process.env;
|
|
348
|
+
const modelDeps = {
|
|
349
|
+
...deps.modelDeps,
|
|
350
|
+
env: deps.modelDeps?.env ?? env,
|
|
351
|
+
};
|
|
352
|
+
const accountDeps = {
|
|
353
|
+
probe: modelDeps.probe ?? defaultProbe,
|
|
354
|
+
env,
|
|
355
|
+
};
|
|
356
|
+
const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
|
|
357
|
+
const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
|
|
358
|
+
const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
|
|
359
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" ? lane.model : undefined;
|
|
360
|
+
const unavailableAccounts = previouslyUnavailableAccounts(lane);
|
|
361
|
+
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
362
|
+
.filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
|
|
363
|
+
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
364
|
+
existsSync,
|
|
365
|
+
...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
|
|
366
|
+
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
367
|
+
}).logged_in === true)
|
|
368
|
+
.filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
|
|
369
|
+
.map(([id]) => id);
|
|
370
|
+
const modelContext = {
|
|
371
|
+
hqRoot,
|
|
372
|
+
existsSync,
|
|
373
|
+
accountDeps,
|
|
374
|
+
...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
|
|
375
|
+
allowedAccounts,
|
|
376
|
+
};
|
|
377
|
+
let target;
|
|
378
|
+
try {
|
|
379
|
+
target = selectBalancedAccount(lane.provider, config, modelContext);
|
|
380
|
+
}
|
|
381
|
+
catch (err) {
|
|
382
|
+
if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
|
|
383
|
+
throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
|
|
384
|
+
}
|
|
385
|
+
throw err;
|
|
386
|
+
}
|
|
387
|
+
const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
|
|
388
|
+
const sessionId = providerSupportsSessionResume(lane.provider) &&
|
|
389
|
+
typeof lane.provider_session_id === "string" &&
|
|
390
|
+
lane.provider_session_id.trim().length > 0
|
|
391
|
+
? lane.provider_session_id
|
|
392
|
+
: undefined;
|
|
393
|
+
if (source && sessionId) {
|
|
394
|
+
copySessionFiles(planSessionCopy(lane.provider, {
|
|
395
|
+
sourceHome: source.home,
|
|
396
|
+
targetHome: target.home,
|
|
397
|
+
cwd: hqRoot,
|
|
398
|
+
sessionId,
|
|
399
|
+
}));
|
|
400
|
+
}
|
|
401
|
+
const now = (deps.now ?? (() => new Date()))();
|
|
402
|
+
const at = now.toISOString();
|
|
403
|
+
const nowMs = now.getTime();
|
|
404
|
+
return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
405
|
+
const current = readLane(hqRoot, lane.lane_id);
|
|
406
|
+
if (current.state !== "running" ||
|
|
407
|
+
current.account !== lane.account ||
|
|
408
|
+
current.provider_session_id !== lane.provider_session_id) {
|
|
409
|
+
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
|
410
|
+
lane_id: lane.lane_id,
|
|
411
|
+
state: current.state,
|
|
412
|
+
account: current.account ?? null,
|
|
413
|
+
provider_session_id_changed: current.provider_session_id !== lane.provider_session_id,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
const moveToRetryQueue = (row) => {
|
|
417
|
+
const entry = {
|
|
418
|
+
from: row.account ?? null,
|
|
419
|
+
to: target.id,
|
|
420
|
+
at,
|
|
421
|
+
reason,
|
|
422
|
+
};
|
|
423
|
+
return {
|
|
424
|
+
...row,
|
|
425
|
+
state: "queued",
|
|
426
|
+
queued_reason: "account_retry",
|
|
427
|
+
queued_at: at,
|
|
428
|
+
admission_reserved_at: undefined,
|
|
429
|
+
terminal_reason: undefined,
|
|
430
|
+
terminal_details: undefined,
|
|
431
|
+
account: target.id,
|
|
432
|
+
account_history: appendCappedHistory(row.account_history ?? [], entry),
|
|
433
|
+
pending_resume: sessionId
|
|
434
|
+
? pendingResumeForRetry(hqRoot, row, sessionId, at)
|
|
435
|
+
: undefined,
|
|
436
|
+
};
|
|
437
|
+
};
|
|
438
|
+
const candidate = moveToRetryQueue(current);
|
|
439
|
+
const admissionLanes = listLaneIds(hqRoot)
|
|
440
|
+
.filter((id) => id !== current.lane_id)
|
|
441
|
+
.map((id) => readLane(hqRoot, id));
|
|
442
|
+
admissionLanes.push(candidate);
|
|
443
|
+
const profileMaxFor = (row) => row.worker_profile === undefined
|
|
444
|
+
? undefined
|
|
445
|
+
: loadWorkerProfile(hqRoot, row.worker_profile).meta.max_concurrent;
|
|
446
|
+
const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
|
|
447
|
+
const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
|
|
448
|
+
const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
|
|
449
|
+
if (row.state !== "running" ||
|
|
450
|
+
row.account !== lane.account ||
|
|
451
|
+
row.provider_session_id !== lane.provider_session_id) {
|
|
452
|
+
throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
|
|
453
|
+
lane_id: lane.lane_id,
|
|
454
|
+
state: row.state,
|
|
455
|
+
account: row.account ?? null,
|
|
456
|
+
provider_session_id_changed: row.provider_session_id !== lane.provider_session_id,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
return moveToRetryQueue(row);
|
|
460
|
+
});
|
|
461
|
+
if (capacityReason !== undefined || earlierEligibleLane)
|
|
462
|
+
return moved;
|
|
463
|
+
return spawnLaneWorker(hqRoot, moved, deps);
|
|
464
|
+
});
|
|
465
|
+
}
|
|
104
466
|
export function appendModelFlag(provider, args, model) {
|
|
105
467
|
if (model === undefined)
|
|
106
468
|
return args;
|
|
@@ -1157,16 +1519,28 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1157
1519
|
const pinnedModel = existing.model_source === undefined || existing.model_source === "cli"
|
|
1158
1520
|
? existing.model
|
|
1159
1521
|
: undefined;
|
|
1160
|
-
|
|
1522
|
+
const accountDeps = {
|
|
1523
|
+
probe: modelDeps.probe ?? defaultProbe,
|
|
1524
|
+
env,
|
|
1525
|
+
};
|
|
1526
|
+
let selection;
|
|
1161
1527
|
try {
|
|
1162
|
-
|
|
1528
|
+
selection = selectSpawnAccount({
|
|
1163
1529
|
hqRoot,
|
|
1530
|
+
lane: existing,
|
|
1531
|
+
config,
|
|
1164
1532
|
existsSync,
|
|
1165
|
-
|
|
1533
|
+
accountDeps,
|
|
1534
|
+
modelDeps,
|
|
1535
|
+
pinnedModel,
|
|
1536
|
+
profileModel,
|
|
1537
|
+
currentModelValidated: deps.currentModelValidated === true,
|
|
1166
1538
|
});
|
|
1167
1539
|
}
|
|
1168
1540
|
catch (err) {
|
|
1169
|
-
if (err instanceof AccountResolutionError
|
|
1541
|
+
if (err instanceof AccountResolutionError ||
|
|
1542
|
+
(err instanceof LanesError &&
|
|
1543
|
+
(err.error === "account_not_logged_in" || err.error === "model_unsupported"))) {
|
|
1170
1544
|
if (markPreSpawnFailure) {
|
|
1171
1545
|
markSpawnFailure(hqRoot, lane.lane_id, err.error, err.details);
|
|
1172
1546
|
}
|
|
@@ -1175,6 +1549,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1175
1549
|
process.stderr.write(`hq lanes: account bind failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
1176
1550
|
throw err;
|
|
1177
1551
|
}
|
|
1552
|
+
const resolved = selection.account;
|
|
1178
1553
|
const status = accountStatus(lane.provider, resolved.id, config, { existsSync });
|
|
1179
1554
|
if (isLoggedOut(status)) {
|
|
1180
1555
|
const failed = markPreSpawnFailure
|
|
@@ -1185,8 +1560,20 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1185
1560
|
: existing;
|
|
1186
1561
|
throw new LanesError("account_not_logged_in", `Account ${resolved.id} for ${lane.provider} is not logged in (no credential file)`, { provider: lane.provider, account: resolved.id, lane_id: lane.lane_id, lane: failed });
|
|
1187
1562
|
}
|
|
1188
|
-
if (
|
|
1189
|
-
|
|
1563
|
+
if (existing.account !== resolved.id) {
|
|
1564
|
+
const source = existing.account
|
|
1565
|
+
? config.providers[existing.provider].accounts[existing.account]
|
|
1566
|
+
: undefined;
|
|
1567
|
+
if (source &&
|
|
1568
|
+
existing.provider_session_id &&
|
|
1569
|
+
providerSupportsSessionResume(existing.provider)) {
|
|
1570
|
+
copySessionFiles(planSessionCopy(existing.provider, {
|
|
1571
|
+
sourceHome: source.home,
|
|
1572
|
+
targetHome: resolved.home,
|
|
1573
|
+
cwd: hqRoot,
|
|
1574
|
+
sessionId: existing.provider_session_id,
|
|
1575
|
+
}));
|
|
1576
|
+
}
|
|
1190
1577
|
}
|
|
1191
1578
|
if (plan.mode === "resume" && existing.provider !== "grok") {
|
|
1192
1579
|
ensureSessionHome({
|
|
@@ -1199,11 +1586,26 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1199
1586
|
providerHomeAtSpawn: existing.provider_home_at_spawn,
|
|
1200
1587
|
});
|
|
1201
1588
|
}
|
|
1202
|
-
if (
|
|
1589
|
+
if (existing.account !== resolved.id) {
|
|
1590
|
+
const at = (deps.now ?? (() => new Date()))().toISOString();
|
|
1591
|
+
const reason = selection.changeReason ?? "first_assignment";
|
|
1203
1592
|
retryUpdateLane(hqRoot, lane.lane_id, "spawn", (current) => {
|
|
1204
1593
|
if (current.account === resolved.id)
|
|
1205
1594
|
return current;
|
|
1206
|
-
|
|
1595
|
+
if (current.account !== existing.account) {
|
|
1596
|
+
throw new LanesError("account_bind_conflict", `Lane ${lane.lane_id} changed accounts while its worker spawn was prepared`, { lane_id: lane.lane_id, expected_account: existing.account ?? null, account: current.account ?? null });
|
|
1597
|
+
}
|
|
1598
|
+
const entry = {
|
|
1599
|
+
from: current.account ?? null,
|
|
1600
|
+
to: resolved.id,
|
|
1601
|
+
at,
|
|
1602
|
+
reason,
|
|
1603
|
+
};
|
|
1604
|
+
return {
|
|
1605
|
+
...current,
|
|
1606
|
+
account: resolved.id,
|
|
1607
|
+
account_history: appendCappedHistory(current.account_history ?? [], entry),
|
|
1608
|
+
};
|
|
1207
1609
|
});
|
|
1208
1610
|
}
|
|
1209
1611
|
try {
|
|
@@ -1255,10 +1657,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1255
1657
|
? trigger.text
|
|
1256
1658
|
: pendingResume?.text,
|
|
1257
1659
|
resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
|
|
1258
|
-
answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
|
|
1660
|
+
answersPath: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
|
|
1259
1661
|
? answersPath
|
|
1260
1662
|
: undefined,
|
|
1261
|
-
answers: trigger.kind === "answered_questions" && existsSync(answersPath)
|
|
1663
|
+
answers: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
|
|
1262
1664
|
? readAnsweredQuestionLines(answersPath)
|
|
1263
1665
|
: undefined,
|
|
1264
1666
|
}),
|
|
@@ -1425,6 +1827,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1425
1827
|
...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
|
|
1426
1828
|
...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
|
|
1427
1829
|
...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
|
|
1830
|
+
...(pendingResume.compact !== undefined ? { compact: pendingResume.compact } : {}),
|
|
1428
1831
|
};
|
|
1429
1832
|
}
|
|
1430
1833
|
else if (resume && trigger.kind !== "admit") {
|
|
@@ -1433,7 +1836,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1433
1836
|
at: nowIso,
|
|
1434
1837
|
account: resolved.id,
|
|
1435
1838
|
trigger: trigger.kind,
|
|
1436
|
-
...(trigger.kind === "message"
|
|
1839
|
+
...(trigger.kind === "message"
|
|
1840
|
+
? { message_id: trigger.message_id, text: trigger.text }
|
|
1841
|
+
: {}),
|
|
1842
|
+
...(launch.compact !== undefined ? { compact: launch.compact } : {}),
|
|
1437
1843
|
...(trigger.kind === "resume"
|
|
1438
1844
|
? {
|
|
1439
1845
|
resumed_from: trigger.from,
|