@indigoai-us/hq-cli 5.222.0 → 5.223.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/commands/lanes.js +156 -50
- package/dist/lib/lanes/reconcile.js +1 -1
- package/dist/lib/lanes/spawn.js +62 -11
- package/dist/lib/lanes/types.d.ts +3 -3
- package/dist/lib/lanes/watch.d.ts +18 -1
- package/dist/lib/lanes/watch.js +122 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.223.0] — 2026-09-26
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Manual resume and explicitly opted-in keep-alive reconcile can move a lane
|
|
10
|
+
off a disabled or logged-out Claude account on the same host, preserving the
|
|
11
|
+
provider session, provider, and model. `hq lanes watch` reports each
|
|
12
|
+
logged-in-to-logged-out transition once with the affected lane ids. The
|
|
13
|
+
watcher stops polling terminal keep-alive lanes when no resume can run and
|
|
14
|
+
shares one local account probe across matching lanes per poll. Quota and
|
|
15
|
+
model failures keep their existing account-selection path and reason.
|
|
16
|
+
|
|
5
17
|
## [5.222.0] — 2026-09-26
|
|
6
18
|
|
|
7
19
|
- Checkpoint ontology, behind `checkpoint.ontology.multi-writer` (off by
|
package/dist/commands/lanes.js
CHANGED
|
@@ -42,7 +42,7 @@ import { effectiveAccountHome } from "../lib/lanes/account-home.js";
|
|
|
42
42
|
import { AccountResolutionError, compareLaneQueueOrder, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, createQuestion, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, hireGrantCapacityError, LANE_HIRE_GRANTS_FLAG, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseLivenessState, parseQuestionRecord, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, runReconcilePass, wakeAnsweredLaneNow, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, withLaneHireLocks, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
43
43
|
import { activeLaneStories, laneStories, laneStory, storyWarning, } from "../lib/lanes/stories.js";
|
|
44
44
|
import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
|
|
45
|
-
import { keepAliveExclusion, loadRetiredLaneIds, } from "../lib/lanes/keep-alive.js";
|
|
45
|
+
import { keepAliveExclusion, LEAD_LANE_KEEP_ALIVE_FLAG, loadRetiredLaneIds, } from "../lib/lanes/keep-alive.js";
|
|
46
46
|
import { questionBodyAbsPath, resolveQuestionCompanyWithFallback, resolveProjectQuestionCompany, } from "../lib/lanes/questions.js";
|
|
47
47
|
import { assertQuestionBodySecretSafe } from "../lib/lanes/questions.js";
|
|
48
48
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
@@ -1262,10 +1262,75 @@ function resolveMonitorEngine(requested, env) {
|
|
|
1262
1262
|
}
|
|
1263
1263
|
async function runWatch(ctx, laneIds, opts) {
|
|
1264
1264
|
const rawInterval = opts.intervalMs === undefined ? undefined : Number(opts.intervalMs);
|
|
1265
|
-
|
|
1265
|
+
const hqRoot = ctx.hqRoot();
|
|
1266
|
+
await runLaneWatch(hqRoot, laneIds, {
|
|
1266
1267
|
intervalMs: rawInterval,
|
|
1267
1268
|
sleep: ctx.sleep,
|
|
1269
|
+
keepAliveEnabled: (lane) => readLanesRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, {
|
|
1270
|
+
company: lane.company,
|
|
1271
|
+
env: ctx.env,
|
|
1272
|
+
}),
|
|
1273
|
+
readAccountStatuses: async (lanes) => {
|
|
1274
|
+
if (lanes.length === 0)
|
|
1275
|
+
return [];
|
|
1276
|
+
const config = loadLanesConfig(hqRoot);
|
|
1277
|
+
const currentHost = laneHostFromCtx(ctx);
|
|
1278
|
+
const currentHostname = ctx.hostname().trim();
|
|
1279
|
+
const isCurrentHost = (host) => (host.kind === currentHost.kind && host.id === currentHost.id) ||
|
|
1280
|
+
(host.kind === "local" && host.id === currentHostname);
|
|
1281
|
+
const remoteOutpostIds = [...new Set(lanes
|
|
1282
|
+
.filter((lane) => lane.host.kind === "outpost" && !isCurrentHost(lane.host))
|
|
1283
|
+
.map((lane) => lane.host.id))];
|
|
1284
|
+
const remoteOutposts = await Promise.all(remoteOutpostIds.map(async (id) => [id, await readOutpostCapacity(ctx, id)]));
|
|
1285
|
+
const outpostSnapshots = new Map(remoteOutposts);
|
|
1286
|
+
const remoteLocalLanes = lanes.filter((lane) => lane.host.kind === "local" && !isCurrentHost(lane.host));
|
|
1287
|
+
const remoteLocalHosts = remoteLocalLanes.length > 0
|
|
1288
|
+
? await loadClaudeBalanceHostSnapshots(ctx, hqRoot, config)
|
|
1289
|
+
: undefined;
|
|
1290
|
+
const localStatusCache = new Map();
|
|
1291
|
+
return lanes.map((lane) => {
|
|
1292
|
+
if (config.providers[lane.provider].accounts[lane.account] === undefined) {
|
|
1293
|
+
throw new LanesError("watch_account_unconfigured", `Lane ${lane.lane_id} records unconfigured ${lane.provider} account ${lane.account}`, { lane_id: lane.lane_id, provider: lane.provider, account: lane.account, host: lane.host });
|
|
1294
|
+
}
|
|
1295
|
+
let loggedIn;
|
|
1296
|
+
if (isCurrentHost(lane.host)) {
|
|
1297
|
+
const key = JSON.stringify([lane.provider, lane.account, currentHost.kind, currentHost.id]);
|
|
1298
|
+
if (!localStatusCache.has(key)) {
|
|
1299
|
+
localStatusCache.set(key, accountStatus(lane.provider, lane.account, config, {
|
|
1300
|
+
existsSync: ctx.existsSync,
|
|
1301
|
+
probe: ctx.probeProvider,
|
|
1302
|
+
env: ctx.env,
|
|
1303
|
+
hostname: currentHostname,
|
|
1304
|
+
}).logged_in);
|
|
1305
|
+
}
|
|
1306
|
+
loggedIn = localStatusCache.get(key);
|
|
1307
|
+
}
|
|
1308
|
+
else if (lane.host.kind === "outpost") {
|
|
1309
|
+
loggedIn = outpostSnapshots.get(lane.host.id)?.providers[lane.provider].accounts[lane.account]?.logged_in;
|
|
1310
|
+
}
|
|
1311
|
+
else {
|
|
1312
|
+
const snapshot = remoteLocalHosts?.snapshots.find((candidate) => candidate.capacity.host.kind === lane.host.kind &&
|
|
1313
|
+
candidate.capacity.host.id === lane.host.id);
|
|
1314
|
+
if (!snapshot) {
|
|
1315
|
+
const gap = remoteLocalHosts?.unavailableHosts.find((candidate) => candidate.host.kind === lane.host.kind && candidate.host.id === lane.host.id);
|
|
1316
|
+
throw new LanesError("watch_account_host_unavailable", `Cannot check ${lane.provider} account ${lane.account} on ${lane.host.kind}:${lane.host.id}${gap ? `: ${gap.reason}` : ""}`, { lane_id: lane.lane_id, provider: lane.provider, account: lane.account, host: lane.host });
|
|
1317
|
+
}
|
|
1318
|
+
loggedIn = snapshot.capacity.providers[lane.provider].accounts[lane.account]?.logged_in;
|
|
1319
|
+
}
|
|
1320
|
+
if (loggedIn === undefined) {
|
|
1321
|
+
throw new LanesError("watch_account_unconfigured", `Lane ${lane.lane_id} records unconfigured ${lane.provider} account ${lane.account} on ${lane.host.kind}:${lane.host.id}`, { lane_id: lane.lane_id, provider: lane.provider, account: lane.account, host: lane.host });
|
|
1322
|
+
}
|
|
1323
|
+
return {
|
|
1324
|
+
lane_id: lane.lane_id,
|
|
1325
|
+
provider: lane.provider,
|
|
1326
|
+
account: lane.account,
|
|
1327
|
+
host: lane.host,
|
|
1328
|
+
logged_in: loggedIn,
|
|
1329
|
+
};
|
|
1330
|
+
});
|
|
1331
|
+
},
|
|
1268
1332
|
writeLine: (line) => console.log(opts.json ? JSON.stringify({ event: line }) : line),
|
|
1333
|
+
writeAccountEvent: (event) => console.log(opts.json ? JSON.stringify(event) : `account_logged_out provider=${event.provider} account=${event.account} host=${event.host.kind}:${event.host.id} lane_ids=${event.lane_ids.join(",")}`),
|
|
1269
1334
|
writeError: (line) => process.stderr.write(`${line}\n`),
|
|
1270
1335
|
});
|
|
1271
1336
|
return 0;
|
|
@@ -5002,7 +5067,8 @@ function resolveResumeProfile(ctx, lane, workerId) {
|
|
|
5002
5067
|
function validateResumeProfileModel(ctx, lane, profile) {
|
|
5003
5068
|
if (profile.meta.provider !== lane.provider ||
|
|
5004
5069
|
profile.meta.model === undefined ||
|
|
5005
|
-
lane.model_source === "cli"
|
|
5070
|
+
lane.model_source === "cli" ||
|
|
5071
|
+
lane.model_source === "preserved") {
|
|
5006
5072
|
return;
|
|
5007
5073
|
}
|
|
5008
5074
|
const hqRoot = ctx.hqRoot();
|
|
@@ -5036,7 +5102,7 @@ function applyResumeProfileAndEffort(current, next, input) {
|
|
|
5036
5102
|
: undefined;
|
|
5037
5103
|
next.keep_alive_blocked_reason = undefined;
|
|
5038
5104
|
}
|
|
5039
|
-
if (next.model_source !== "cli") {
|
|
5105
|
+
if (next.model_source !== "cli" && next.model_source !== "preserved") {
|
|
5040
5106
|
if (profile.meta.provider === current.provider && profile.meta.model !== undefined) {
|
|
5041
5107
|
next.model = profile.meta.model;
|
|
5042
5108
|
next.model_source = "profile";
|
|
@@ -5130,7 +5196,7 @@ function resolveResumeEffort(ctx, lane, input) {
|
|
|
5130
5196
|
? undefined
|
|
5131
5197
|
: providerConfig.accounts[accountId];
|
|
5132
5198
|
const profileDefaultsApply = input.profile?.profile.meta.provider === lane.provider;
|
|
5133
|
-
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
|
|
5199
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
5134
5200
|
? lane.model
|
|
5135
5201
|
: undefined;
|
|
5136
5202
|
const model = resolveLaneModel({
|
|
@@ -5225,11 +5291,11 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5225
5291
|
? loadWorkerProfile(hqRoot, lane.worker_profile)
|
|
5226
5292
|
: undefined);
|
|
5227
5293
|
const profileDefaultsApply = profile !== undefined && profile.meta.provider === lane.provider;
|
|
5228
|
-
const pinnedModel = input.workerProfile !== undefined
|
|
5229
|
-
? lane.model_source === "cli" ? lane.model : undefined
|
|
5230
|
-
: lane.model_source === undefined || lane.model_source === "cli"
|
|
5294
|
+
const pinnedModel = input.preservedModel ?? (input.workerProfile !== undefined
|
|
5295
|
+
? lane.model_source === "cli" || lane.model_source === "preserved" ? lane.model : undefined
|
|
5296
|
+
: lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
5231
5297
|
? lane.model
|
|
5232
|
-
: undefined;
|
|
5298
|
+
: undefined);
|
|
5233
5299
|
const profileModel = profileDefaultsApply ? profile?.meta.model : undefined;
|
|
5234
5300
|
if (input.cliModel === undefined && profileDefaultsApply && profileModel !== undefined && pinnedModel === undefined) {
|
|
5235
5301
|
validateProfileLaneModel(lane.provider, profileModel, targetHome, profile.id, {
|
|
@@ -5313,12 +5379,14 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5313
5379
|
provider_session_id_absent: undefined,
|
|
5314
5380
|
...(input.cliModel !== undefined
|
|
5315
5381
|
? { model: input.cliModel, model_source: "cli" }
|
|
5316
|
-
:
|
|
5382
|
+
: input.preservedModel !== undefined
|
|
5383
|
+
? { model: input.preservedModel, model_source: "preserved" }
|
|
5384
|
+
: {}),
|
|
5317
5385
|
account_history: appendCappedHistory(current.account_history ?? [], {
|
|
5318
5386
|
from,
|
|
5319
5387
|
to: target,
|
|
5320
5388
|
at,
|
|
5321
|
-
reason: SESSION_MOVE_REASON,
|
|
5389
|
+
reason: input.accountHistoryReason ?? SESSION_MOVE_REASON,
|
|
5322
5390
|
}),
|
|
5323
5391
|
pending_resume: input.deferTurnStart
|
|
5324
5392
|
? undefined
|
|
@@ -5345,31 +5413,39 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5345
5413
|
return { lane: moved, from, to: target, copy, queued };
|
|
5346
5414
|
}, ctx.sleep);
|
|
5347
5415
|
}
|
|
5348
|
-
async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile) {
|
|
5416
|
+
async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile, options = {}) {
|
|
5349
5417
|
if (lane.provider !== "claude")
|
|
5350
5418
|
return undefined;
|
|
5351
|
-
|
|
5419
|
+
const balanceEnabled = await ctx.laneClaudeAccountBalanceEnabled(lane.company);
|
|
5420
|
+
const hqRoot = ctx.hqRoot();
|
|
5421
|
+
const config = loadLanesConfig(hqRoot);
|
|
5422
|
+
const currentConfig = lane.account === undefined
|
|
5423
|
+
? undefined
|
|
5424
|
+
: config.providers.claude.accounts[lane.account];
|
|
5425
|
+
const currentLogin = lane.account === undefined || currentConfig === undefined
|
|
5426
|
+
? false
|
|
5427
|
+
: accountStatus("claude", lane.account, config, {
|
|
5428
|
+
existsSync: ctx.existsSync,
|
|
5429
|
+
hostname: ctx.hostname(),
|
|
5430
|
+
probe: ctx.probeProvider,
|
|
5431
|
+
env: ctx.env,
|
|
5432
|
+
}).logged_in;
|
|
5433
|
+
const accountNeedsRepair = lane.account !== undefined &&
|
|
5434
|
+
(currentConfig?.enabled !== true || currentLogin === false);
|
|
5435
|
+
const repairUnavailableAccount = options.repairUnavailableAccount === true && accountNeedsRepair;
|
|
5436
|
+
if (!balanceEnabled && !(options.repairUnavailableAccount && accountNeedsRepair)) {
|
|
5352
5437
|
if (lane.account === undefined)
|
|
5353
5438
|
return undefined;
|
|
5354
|
-
const config = loadLanesConfig(ctx.hqRoot());
|
|
5355
|
-
const currentConfig = config.providers.claude.accounts[lane.account];
|
|
5356
5439
|
if (currentConfig === undefined || !currentConfig.enabled) {
|
|
5357
5440
|
throw new AccountResolutionError(currentConfig === undefined ? "unknown_account" : "account_disabled", currentConfig === undefined
|
|
5358
5441
|
? `Lane ${lane.lane_id} records account ${JSON.stringify(lane.account)} which is not configured for claude`
|
|
5359
5442
|
: `Account ${JSON.stringify(lane.account)} is unavailable for claude`, { provider: "claude", account: lane.account });
|
|
5360
5443
|
}
|
|
5361
|
-
if (
|
|
5362
|
-
existsSync: ctx.existsSync,
|
|
5363
|
-
hostname: ctx.hostname(),
|
|
5364
|
-
probe: ctx.probeProvider,
|
|
5365
|
-
env: ctx.env,
|
|
5366
|
-
}).logged_in === false) {
|
|
5444
|
+
if (currentLogin === false) {
|
|
5367
5445
|
throw new LanesError("account_not_logged_in", `Account ${lane.account} for claude is not logged in`, { provider: "claude", account: lane.account, lane_id: lane.lane_id });
|
|
5368
5446
|
}
|
|
5369
5447
|
return undefined;
|
|
5370
5448
|
}
|
|
5371
|
-
const hqRoot = ctx.hqRoot();
|
|
5372
|
-
const config = loadLanesConfig(hqRoot);
|
|
5373
5449
|
const profile = workerProfile?.profile ?? (lane.worker_profile !== undefined
|
|
5374
5450
|
? loadWorkerProfile(hqRoot, lane.worker_profile)
|
|
5375
5451
|
: undefined);
|
|
@@ -5380,33 +5456,45 @@ async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile) {
|
|
|
5380
5456
|
const allowedAccounts = profilePin !== undefined && lane.admission_accounts !== undefined
|
|
5381
5457
|
? profilePin.filter((account) => lane.admission_accounts.includes(account))
|
|
5382
5458
|
: profilePin ?? lane.admission_accounts;
|
|
5383
|
-
const currentConfig = lane.account === undefined
|
|
5384
|
-
? undefined
|
|
5385
|
-
: config.providers.claude.accounts[lane.account];
|
|
5386
|
-
const currentLogin = lane.account === undefined || currentConfig === undefined
|
|
5387
|
-
? false
|
|
5388
|
-
: accountStatus("claude", lane.account, config, {
|
|
5389
|
-
existsSync: ctx.existsSync,
|
|
5390
|
-
hostname: ctx.hostname(),
|
|
5391
|
-
probe: ctx.probeProvider,
|
|
5392
|
-
env: ctx.env,
|
|
5393
|
-
}).logged_in;
|
|
5394
5459
|
const pinnedModel = workerProfile !== undefined
|
|
5395
|
-
? lane.model_source === "cli" ? lane.model : undefined
|
|
5396
|
-
: lane.model_source === undefined || lane.model_source === "cli"
|
|
5460
|
+
? lane.model_source === "cli" || lane.model_source === "preserved" ? lane.model : undefined
|
|
5461
|
+
: lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
5397
5462
|
? lane.model
|
|
5398
5463
|
: undefined;
|
|
5464
|
+
const preservedModel = repairUnavailableAccount && lane.model_source === "account"
|
|
5465
|
+
? lane.model ?? currentConfig?.model
|
|
5466
|
+
: undefined;
|
|
5399
5467
|
const model = resolveLaneModel({
|
|
5400
|
-
cliModel: pinnedModel,
|
|
5468
|
+
cliModel: preservedModel ?? pinnedModel,
|
|
5401
5469
|
profileModel: profileDefaultsApply ? profile?.meta.model : undefined,
|
|
5402
5470
|
accountModel: lane.account === undefined ? undefined : config.providers.claude.accounts[lane.account]?.model,
|
|
5403
5471
|
providerModel: config.providers.claude.model,
|
|
5404
5472
|
});
|
|
5405
|
-
const accountNeedsRepair = currentConfig?.enabled !== true || currentLogin === false;
|
|
5406
5473
|
const mayRebalance = BALANCE_MOVABLE_STATES.includes(lane.state);
|
|
5407
5474
|
if (!accountNeedsRepair && !mayRebalance)
|
|
5408
5475
|
return undefined;
|
|
5409
|
-
const balanceHosts =
|
|
5476
|
+
const balanceHosts = balanceEnabled
|
|
5477
|
+
? await loadClaudeBalanceHostSnapshots(ctx, hqRoot, config)
|
|
5478
|
+
: (() => {
|
|
5479
|
+
const lanes = loadAllLanes(hqRoot).filter((candidate) => candidate.host.kind === lane.host.kind && candidate.host.id === lane.host.id);
|
|
5480
|
+
return {
|
|
5481
|
+
snapshots: [{
|
|
5482
|
+
capacity: buildHostCapacitySnapshot({
|
|
5483
|
+
host: lane.host,
|
|
5484
|
+
lanes,
|
|
5485
|
+
config,
|
|
5486
|
+
existsSync: ctx.existsSync,
|
|
5487
|
+
probe: ctx.probeProvider,
|
|
5488
|
+
env: ctx.env,
|
|
5489
|
+
hostname: ctx.hostname(),
|
|
5490
|
+
now: ctx.now(),
|
|
5491
|
+
}),
|
|
5492
|
+
lanes,
|
|
5493
|
+
laneListAvailable: true,
|
|
5494
|
+
}],
|
|
5495
|
+
unavailableHosts: [],
|
|
5496
|
+
};
|
|
5497
|
+
})();
|
|
5410
5498
|
const targetHost = balanceHosts.snapshots.find((host) => host.capacity.host.kind === lane.host.kind && host.capacity.host.id === lane.host.id);
|
|
5411
5499
|
const currentTargetAccount = lane.account === undefined
|
|
5412
5500
|
? undefined
|
|
@@ -5423,11 +5511,15 @@ async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile) {
|
|
|
5423
5511
|
hosts: balanceHosts.snapshots,
|
|
5424
5512
|
targetHost: lane.host,
|
|
5425
5513
|
config,
|
|
5426
|
-
...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
|
|
5514
|
+
...(!repairUnavailableAccount && allowedAccounts !== undefined ? { allowedAccounts } : {}),
|
|
5427
5515
|
...(model !== undefined ? { model } : {}),
|
|
5428
5516
|
unavailableHosts: balanceHosts.unavailableHosts,
|
|
5429
5517
|
});
|
|
5430
|
-
return {
|
|
5518
|
+
return {
|
|
5519
|
+
selection: { ...selected, selected_at: ctx.now().toISOString() },
|
|
5520
|
+
...(repairUnavailableAccount ? { historyReason: "recorded_account_logged_out" } : {}),
|
|
5521
|
+
...(preservedModel !== undefined ? { preservedModel } : {}),
|
|
5522
|
+
};
|
|
5431
5523
|
}
|
|
5432
5524
|
function accountRequestKeepsCurrentLane(lane, rawAccount) {
|
|
5433
5525
|
const raw = rawAccount.trim();
|
|
@@ -5525,7 +5617,7 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
5525
5617
|
const resumeEffort = resolveResumeEffort(ctx, lane, {
|
|
5526
5618
|
cliEffort,
|
|
5527
5619
|
profile: resumeProfile,
|
|
5528
|
-
account: keepCurrentAccount ? lane.account : opts.account ?? automaticAccountSelection?.account,
|
|
5620
|
+
account: keepCurrentAccount ? lane.account : opts.account ?? automaticAccountSelection?.selection.account,
|
|
5529
5621
|
model: opts.model,
|
|
5530
5622
|
});
|
|
5531
5623
|
const messageId = mintMessageId();
|
|
@@ -5585,12 +5677,14 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
5585
5677
|
}
|
|
5586
5678
|
}
|
|
5587
5679
|
else if (mechanism === "resume") {
|
|
5588
|
-
const
|
|
5589
|
-
if (
|
|
5680
|
+
const automatic = automaticAccountSelection;
|
|
5681
|
+
if (automatic !== undefined) {
|
|
5682
|
+
const selection = automatic.selection;
|
|
5590
5683
|
if (selection.account !== lane.account) {
|
|
5591
5684
|
move = await prepareAccountMove(ctx, lane, {
|
|
5592
5685
|
account: selection.account,
|
|
5593
5686
|
cliModel: undefined,
|
|
5687
|
+
...(automatic.historyReason !== undefined ? { accountHistoryReason: automatic.historyReason } : {}),
|
|
5594
5688
|
...(resumeProfile !== undefined ? { workerProfile: resumeProfile } : {}),
|
|
5595
5689
|
...(resumeEffort !== undefined ? { effort: resumeEffort } : {}),
|
|
5596
5690
|
pending: {
|
|
@@ -6063,12 +6157,12 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6063
6157
|
validateResumeProfileModel(ctx, lane, resumeProfile.profile);
|
|
6064
6158
|
}
|
|
6065
6159
|
const automaticAccountSelection = opts.account === undefined
|
|
6066
|
-
? await selectAutomaticClaudeResumeAccount(ctx, lane, resumeProfile)
|
|
6160
|
+
? await selectAutomaticClaudeResumeAccount(ctx, lane, resumeProfile, { repairUnavailableAccount: true })
|
|
6067
6161
|
: undefined;
|
|
6068
6162
|
const resumeEffort = resolveResumeEffort(ctx, lane, {
|
|
6069
6163
|
cliEffort,
|
|
6070
6164
|
profile: resumeProfile,
|
|
6071
|
-
account: keepCurrentAccount ? lane.account : opts.account ?? automaticAccountSelection?.account,
|
|
6165
|
+
account: keepCurrentAccount ? lane.account : opts.account ?? automaticAccountSelection?.selection.account,
|
|
6072
6166
|
model: opts.model,
|
|
6073
6167
|
});
|
|
6074
6168
|
let move = null;
|
|
@@ -6089,10 +6183,17 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6089
6183
|
lane = move.lane;
|
|
6090
6184
|
}
|
|
6091
6185
|
else if (automaticAccountSelection !== undefined) {
|
|
6092
|
-
|
|
6186
|
+
const selection = automaticAccountSelection.selection;
|
|
6187
|
+
if (selection.account !== lane.account) {
|
|
6093
6188
|
move = await prepareAccountMove(ctx, lane, {
|
|
6094
|
-
account:
|
|
6189
|
+
account: selection.account,
|
|
6095
6190
|
cliModel: undefined,
|
|
6191
|
+
...(automaticAccountSelection.historyReason !== undefined
|
|
6192
|
+
? { accountHistoryReason: automaticAccountSelection.historyReason }
|
|
6193
|
+
: {}),
|
|
6194
|
+
...(automaticAccountSelection.preservedModel !== undefined
|
|
6195
|
+
? { preservedModel: automaticAccountSelection.preservedModel }
|
|
6196
|
+
: {}),
|
|
6096
6197
|
...(resumeProfile !== undefined ? { workerProfile: resumeProfile } : {}),
|
|
6097
6198
|
...(resumeEffort !== undefined ? { effort: resumeEffort } : {}),
|
|
6098
6199
|
pending: {
|
|
@@ -6102,12 +6203,12 @@ async function runResume(ctx, laneId, opts) {
|
|
|
6102
6203
|
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
6103
6204
|
compact: opts.compact !== false,
|
|
6104
6205
|
},
|
|
6105
|
-
accountSelection:
|
|
6206
|
+
accountSelection: selection,
|
|
6106
6207
|
});
|
|
6107
6208
|
lane = move.lane;
|
|
6108
6209
|
}
|
|
6109
6210
|
else {
|
|
6110
|
-
lane = persistClaudeAccountSelection(hqRoot, lane,
|
|
6211
|
+
lane = persistClaudeAccountSelection(hqRoot, lane, selection);
|
|
6111
6212
|
}
|
|
6112
6213
|
}
|
|
6113
6214
|
lane = recordResumeSetup(ctx, lane, {
|
|
@@ -8412,6 +8513,11 @@ async function runReconcile(ctx, opts) {
|
|
|
8412
8513
|
dryRun: Boolean(opts.dryRun),
|
|
8413
8514
|
spawnLaneDeps: {
|
|
8414
8515
|
env: ctx.env,
|
|
8516
|
+
modelDeps: {
|
|
8517
|
+
readFileSync: ctx.readFileSync,
|
|
8518
|
+
probe: ctx.probeProvider,
|
|
8519
|
+
env: ctx.env,
|
|
8520
|
+
},
|
|
8415
8521
|
existsSync: ctx.existsSync,
|
|
8416
8522
|
spawnProcess: ctx.spawnProcess,
|
|
8417
8523
|
proveDetached: ctx.proveDetached,
|
|
@@ -329,7 +329,7 @@ function availableBalancedAccount(hqRoot, lane, occupying, config, host, profile
|
|
|
329
329
|
if (laneAdmissionCapReason(occupying, lane, config, host, profileMaxConcurrent, nowMs) !== "account_cap") {
|
|
330
330
|
return undefined;
|
|
331
331
|
}
|
|
332
|
-
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
|
|
332
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
333
333
|
? lane.model
|
|
334
334
|
: lane.model_source === "profile"
|
|
335
335
|
? lane.model
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -14,6 +14,8 @@ import * as path from "node:path";
|
|
|
14
14
|
import { fileURLToPath } from "node:url";
|
|
15
15
|
import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
|
|
16
16
|
import { effectiveAccountHome } from "./account-home.js";
|
|
17
|
+
import { buildHostCapacitySnapshot } from "./host-capacity.js";
|
|
18
|
+
import { selectClaudeAccount } from "./claude-account-balance.js";
|
|
17
19
|
import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
|
|
18
20
|
import { installDrainHook } from "./hooks-install.js";
|
|
19
21
|
import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
|
|
@@ -159,7 +161,7 @@ function spawnModelFailure(provider, accountId, config, pinnedModel, profileMode
|
|
|
159
161
|
}
|
|
160
162
|
}
|
|
161
163
|
function selectSpawnAccount(input) {
|
|
162
|
-
const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated, hostname } = input;
|
|
164
|
+
const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated, hostname, repairRecordedAccount, recordedModel, now } = input;
|
|
163
165
|
const context = {
|
|
164
166
|
hqRoot,
|
|
165
167
|
existsSync,
|
|
@@ -187,7 +189,7 @@ function selectSpawnAccount(input) {
|
|
|
187
189
|
const operatorMovePending = lane.pending_resume !== undefined &&
|
|
188
190
|
lane.account_history?.at(-1)?.reason === "operator_move";
|
|
189
191
|
if (current === undefined || !current.enabled) {
|
|
190
|
-
reason = "account_unavailable";
|
|
192
|
+
reason = repairRecordedAccount ? "recorded_account_logged_out" : "account_unavailable";
|
|
191
193
|
refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
|
|
192
194
|
}
|
|
193
195
|
else {
|
|
@@ -198,7 +200,7 @@ function selectSpawnAccount(input) {
|
|
|
198
200
|
...(accountDeps?.env ? { env: accountDeps.env } : {}),
|
|
199
201
|
});
|
|
200
202
|
if (isLoggedOut(status)) {
|
|
201
|
-
reason = "account_unavailable";
|
|
203
|
+
reason = repairRecordedAccount ? "recorded_account_logged_out" : "account_unavailable";
|
|
202
204
|
refusal = new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in`, { provider: lane.provider, account: lane.account });
|
|
203
205
|
}
|
|
204
206
|
else {
|
|
@@ -242,6 +244,33 @@ function selectSpawnAccount(input) {
|
|
|
242
244
|
account: lane.account,
|
|
243
245
|
});
|
|
244
246
|
}
|
|
247
|
+
if (reason === "recorded_account_logged_out" && lane.provider === "claude") {
|
|
248
|
+
const lanes = listLaneIds(hqRoot)
|
|
249
|
+
.map((laneId) => readLane(hqRoot, laneId))
|
|
250
|
+
.filter((record) => record.host.kind === lane.host.kind && record.host.id === lane.host.id);
|
|
251
|
+
const capacity = buildHostCapacitySnapshot({
|
|
252
|
+
host: lane.host,
|
|
253
|
+
lanes,
|
|
254
|
+
config,
|
|
255
|
+
existsSync,
|
|
256
|
+
probe: accountDeps?.probe,
|
|
257
|
+
env: accountDeps?.env,
|
|
258
|
+
hostname,
|
|
259
|
+
now,
|
|
260
|
+
});
|
|
261
|
+
const selection = selectClaudeAccount({
|
|
262
|
+
hosts: [{ capacity, lanes, laneListAvailable: true }],
|
|
263
|
+
targetHost: lane.host,
|
|
264
|
+
config,
|
|
265
|
+
...((recordedModel ?? pinnedModel ?? profileModel) !== undefined
|
|
266
|
+
? { model: recordedModel ?? pinnedModel ?? profileModel }
|
|
267
|
+
: {}),
|
|
268
|
+
});
|
|
269
|
+
return {
|
|
270
|
+
account: resolveAccount(lane.provider, selection.account, config, context),
|
|
271
|
+
changeReason: "recorded_account_logged_out",
|
|
272
|
+
};
|
|
273
|
+
}
|
|
245
274
|
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
246
275
|
.filter(([id, account]) => id !== lane.account && account.enabled)
|
|
247
276
|
.filter(([id]) => accountStatus(lane.provider, id, config, {
|
|
@@ -254,12 +283,15 @@ function selectSpawnAccount(input) {
|
|
|
254
283
|
.map(([id]) => id);
|
|
255
284
|
try {
|
|
256
285
|
return {
|
|
257
|
-
account: selectBalancedAccount(lane.provider, config, {
|
|
286
|
+
account: selectBalancedAccount(lane.provider, config, {
|
|
287
|
+
...context,
|
|
288
|
+
allowedAccounts,
|
|
289
|
+
}),
|
|
258
290
|
changeReason: reason,
|
|
259
291
|
};
|
|
260
292
|
}
|
|
261
293
|
catch (err) {
|
|
262
|
-
if (reason === "account_unavailable") {
|
|
294
|
+
if (reason === "account_unavailable" || reason === "recorded_account_logged_out") {
|
|
263
295
|
if (refusal instanceof Error)
|
|
264
296
|
throw refusal;
|
|
265
297
|
throw new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in or available, and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account });
|
|
@@ -274,7 +306,8 @@ function previouslyUnavailableAccounts(lane) {
|
|
|
274
306
|
for (const entry of [...(lane.account_history ?? [])].reverse()) {
|
|
275
307
|
if (entry.reason === "first_assignment" || entry.reason === "operator_move")
|
|
276
308
|
break;
|
|
277
|
-
if (entry.reason === "account_unavailable"
|
|
309
|
+
if ((entry.reason === "account_unavailable" || entry.reason === "recorded_account_logged_out") &&
|
|
310
|
+
entry.from !== null)
|
|
278
311
|
excluded.add(entry.from);
|
|
279
312
|
}
|
|
280
313
|
return excluded;
|
|
@@ -364,7 +397,9 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
|
|
|
364
397
|
};
|
|
365
398
|
const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
|
|
366
399
|
const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
|
|
367
|
-
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
|
|
400
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
401
|
+
? lane.model
|
|
402
|
+
: undefined;
|
|
368
403
|
const unavailableAccounts = previouslyUnavailableAccounts(lane);
|
|
369
404
|
const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
|
|
370
405
|
.filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
|
|
@@ -1649,7 +1684,12 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1649
1684
|
: undefined;
|
|
1650
1685
|
const profileChanged = profile !== undefined && existing.worker_profile_hash !== profile.contentHash;
|
|
1651
1686
|
const config = loadLanesConfig(hqRoot);
|
|
1652
|
-
const
|
|
1687
|
+
const repairRecordedAccount = plan.mode === "resume" &&
|
|
1688
|
+
(trigger.kind === "resume" || pendingResume?.trigger === "resume");
|
|
1689
|
+
const recordedModel = repairRecordedAccount && existing.model_source === "account"
|
|
1690
|
+
? existing.model
|
|
1691
|
+
: undefined;
|
|
1692
|
+
let pinnedModel = existing.model_source === undefined || existing.model_source === "cli" || existing.model_source === "preserved"
|
|
1653
1693
|
? existing.model
|
|
1654
1694
|
: undefined;
|
|
1655
1695
|
const accountDeps = {
|
|
@@ -1669,6 +1709,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1669
1709
|
profileModel,
|
|
1670
1710
|
currentModelValidated: deps.currentModelValidated === true,
|
|
1671
1711
|
hostname,
|
|
1712
|
+
repairRecordedAccount,
|
|
1713
|
+
...(recordedModel !== undefined ? { recordedModel } : {}),
|
|
1714
|
+
now: (deps.now ?? (() => new Date()))(),
|
|
1672
1715
|
});
|
|
1673
1716
|
}
|
|
1674
1717
|
catch (err) {
|
|
@@ -1683,6 +1726,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1683
1726
|
process.stderr.write(`hq lanes: account bind failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
1684
1727
|
throw err;
|
|
1685
1728
|
}
|
|
1729
|
+
const preservedAccountModel = selection.changeReason === "recorded_account_logged_out" ? recordedModel : undefined;
|
|
1730
|
+
if (preservedAccountModel !== undefined)
|
|
1731
|
+
pinnedModel = preservedAccountModel;
|
|
1686
1732
|
const resolved = selection.account;
|
|
1687
1733
|
const status = accountStatus(lane.provider, resolved.id, config, { existsSync, hostname });
|
|
1688
1734
|
if (isLoggedOut(status)) {
|
|
@@ -1738,6 +1784,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1738
1784
|
return {
|
|
1739
1785
|
...current,
|
|
1740
1786
|
account: resolved.id,
|
|
1787
|
+
...(preservedAccountModel !== undefined
|
|
1788
|
+
? { model: preservedAccountModel, model_source: "preserved" }
|
|
1789
|
+
: {}),
|
|
1741
1790
|
account_history: appendCappedHistory(current.account_history ?? [], entry),
|
|
1742
1791
|
};
|
|
1743
1792
|
});
|
|
@@ -1934,7 +1983,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1934
1983
|
...(model !== undefined
|
|
1935
1984
|
? {
|
|
1936
1985
|
model_source: pinnedModel !== undefined
|
|
1937
|
-
? "
|
|
1986
|
+
? preservedAccountModel !== undefined || existing.model_source === "preserved"
|
|
1987
|
+
? "preserved"
|
|
1988
|
+
: "cli"
|
|
1938
1989
|
: profileModel !== undefined
|
|
1939
1990
|
? "profile"
|
|
1940
1991
|
: accountCfg?.model !== undefined
|
|
@@ -2195,7 +2246,7 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
|
|
|
2195
2246
|
ignoredFields: ignoredProfileFields,
|
|
2196
2247
|
}
|
|
2197
2248
|
: undefined;
|
|
2198
|
-
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
|
|
2249
|
+
const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
|
|
2199
2250
|
? lane.model
|
|
2200
2251
|
: undefined;
|
|
2201
2252
|
const resolved = resolveAccount(lane.provider, lane.account, config, {
|
|
@@ -2307,7 +2358,7 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
|
|
|
2307
2358
|
if (model !== undefined) {
|
|
2308
2359
|
next.model = model;
|
|
2309
2360
|
next.model_source = pinnedModel !== undefined
|
|
2310
|
-
? "cli"
|
|
2361
|
+
? lane.model_source === "preserved" ? "preserved" : "cli"
|
|
2311
2362
|
: profileModel !== undefined
|
|
2312
2363
|
? "profile"
|
|
2313
2364
|
: accountCfg?.model !== undefined
|
|
@@ -412,8 +412,8 @@ export interface LaneRecord {
|
|
|
412
412
|
last_envelope?: LastEnvelopeRef;
|
|
413
413
|
/** Effective model id recorded at create/spawn. Omitted when the provider chooses. */
|
|
414
414
|
model?: string;
|
|
415
|
-
/** Where the effective model came from;
|
|
416
|
-
model_source?: "cli" | "profile" | "account" | "provider" | "none";
|
|
415
|
+
/** Where the effective model came from; repaired models stay pinned across the account move. */
|
|
416
|
+
model_source?: "cli" | "profile" | "account" | "provider" | "preserved" | "none";
|
|
417
417
|
/** Effective effort level passed to the provider at the last spawn (effort.ts). */
|
|
418
418
|
effort?: string;
|
|
419
419
|
/** Where `effort` came from. `cli` pins it across later spawns. */
|
|
@@ -523,7 +523,7 @@ export interface AccountHistoryEntry {
|
|
|
523
523
|
from: string | null;
|
|
524
524
|
to: string;
|
|
525
525
|
at: string;
|
|
526
|
-
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "operator_move";
|
|
526
|
+
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "operator_move";
|
|
527
527
|
}
|
|
528
528
|
export interface PendingResume {
|
|
529
529
|
/** Fresh answered-question launches need no previous session id. */
|
|
@@ -1,16 +1,33 @@
|
|
|
1
|
-
import { type LaneRecord, type QuestionRecord } from "./types.js";
|
|
1
|
+
import { type LaneHost, type LaneProvider, type LaneRecord, type QuestionRecord } from "./types.js";
|
|
2
2
|
export declare const DEFAULT_LANE_WATCH_INTERVAL_MS = 5000;
|
|
3
3
|
export declare const MAX_LANE_WATCH_INTERVAL_MS = 30000;
|
|
4
4
|
export declare const MAX_WATCHED_LANES = 64;
|
|
5
5
|
export declare const MAX_WATCH_OUTPUT_LINES = 512;
|
|
6
|
+
export interface LaneWatchAccountStatus {
|
|
7
|
+
lane_id: string;
|
|
8
|
+
provider: LaneProvider;
|
|
9
|
+
account: string;
|
|
10
|
+
host: LaneHost;
|
|
11
|
+
logged_in: boolean | null;
|
|
12
|
+
}
|
|
13
|
+
export interface LaneAccountLoggedOutEvent {
|
|
14
|
+
event: "account_logged_out";
|
|
15
|
+
provider: LaneProvider;
|
|
16
|
+
account: string;
|
|
17
|
+
host: LaneHost;
|
|
18
|
+
lane_ids: string[];
|
|
19
|
+
}
|
|
6
20
|
export interface LaneWatchDeps {
|
|
7
21
|
readLane?: (hqRoot: string, laneId: string) => LaneRecord | null;
|
|
8
22
|
listQuestionIds?: (hqRoot: string) => string[];
|
|
9
23
|
readQuestion?: (hqRoot: string, questionId: string) => QuestionRecord | null;
|
|
24
|
+
readAccountStatuses?: (lanes: readonly LaneRecord[]) => Promise<readonly LaneWatchAccountStatus[]>;
|
|
25
|
+
keepAliveEnabled?: (lane: LaneRecord) => boolean | Promise<boolean>;
|
|
10
26
|
sleep?: (ms: number) => Promise<void>;
|
|
11
27
|
intervalMs?: number;
|
|
12
28
|
maxOutputLines?: number;
|
|
13
29
|
writeLine?: (line: string) => void;
|
|
30
|
+
writeAccountEvent?: (event: LaneAccountLoggedOutEvent) => void;
|
|
14
31
|
writeError?: (line: string) => void;
|
|
15
32
|
}
|
|
16
33
|
export interface LaneWatchResult {
|
package/dist/lib/lanes/watch.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { listQuestionIds, readLane, readQuestion, } from "./store.js";
|
|
2
2
|
import { LaneNotFoundError, LanesError } from "./errors.js";
|
|
3
|
-
import {
|
|
3
|
+
import { providerSupportsSessionResume } from "./delivery.js";
|
|
4
|
+
import { LEAD_LANE_KEEP_ALIVE_MAX_RESUMES } from "./keep-alive.js";
|
|
5
|
+
import { isTerminalLaneState, } from "./types.js";
|
|
4
6
|
export const DEFAULT_LANE_WATCH_INTERVAL_MS = 5_000;
|
|
5
7
|
export const MAX_LANE_WATCH_INTERVAL_MS = 30_000;
|
|
6
8
|
export const MAX_WATCHED_LANES = 64;
|
|
@@ -30,6 +32,30 @@ function validateLaneIds(laneIds) {
|
|
|
30
32
|
}
|
|
31
33
|
return unique;
|
|
32
34
|
}
|
|
35
|
+
function canResumeKeepAliveLane(lane) {
|
|
36
|
+
if (lane.keep_alive !== true ||
|
|
37
|
+
lane.keep_alive_opted_in_at === undefined ||
|
|
38
|
+
!Number.isFinite(Date.parse(lane.keep_alive_opted_in_at)) ||
|
|
39
|
+
lane.keep_alive_exhausted_at !== undefined ||
|
|
40
|
+
(lane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES ||
|
|
41
|
+
lane.keep_alive_blocked_reason === "retired" ||
|
|
42
|
+
lane.keep_alive_blocked_reason === "superseded" ||
|
|
43
|
+
lane.keep_alive_blocked_reason === "retired_list_invalid" ||
|
|
44
|
+
lane.keep_alive_blocked_reason === "terminal_state" ||
|
|
45
|
+
!providerSupportsSessionResume(lane.provider) ||
|
|
46
|
+
!(lane.provider_session_id ?? lane.last_resume?.session_id))
|
|
47
|
+
return false;
|
|
48
|
+
if (lane.state === "interrupted") {
|
|
49
|
+
return lane.terminal_reason === "exited_with_partial_output";
|
|
50
|
+
}
|
|
51
|
+
if (lane.state === "done") {
|
|
52
|
+
return lane.last_envelope?.decision === "done" &&
|
|
53
|
+
lane.story_complete !== true &&
|
|
54
|
+
lane.terminal_reason !== "provider_rejected" &&
|
|
55
|
+
lane.terminal_reason !== "account_not_logged_in";
|
|
56
|
+
}
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
33
59
|
export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
34
60
|
const watchedIds = validateLaneIds(laneIds);
|
|
35
61
|
const readLaneRecord = deps.readLane ?? readLane;
|
|
@@ -37,11 +63,15 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
37
63
|
const readQuestionRecord = deps.readQuestion ?? readQuestion;
|
|
38
64
|
const sleep = deps.sleep ?? defaultSleep;
|
|
39
65
|
const writeLine = deps.writeLine ?? ((line) => console.log(line));
|
|
66
|
+
const writeAccountEvent = deps.writeAccountEvent ?? ((event) => {
|
|
67
|
+
writeLine(`account_logged_out provider=${event.provider} account=${event.account} host=${event.host.kind}:${event.host.id} lane_ids=${event.lane_ids.join(",")}`);
|
|
68
|
+
});
|
|
40
69
|
const writeError = deps.writeError ?? ((line) => process.stderr.write(`${line}\n`));
|
|
41
70
|
const maxOutputLines = Math.max(1, Math.floor(deps.maxOutputLines ?? MAX_WATCH_OUTPUT_LINES));
|
|
42
71
|
const intervalMs = boundedInterval(deps.intervalMs);
|
|
43
72
|
const previousStates = new Map();
|
|
44
73
|
const previousQuestionStatuses = new Map();
|
|
74
|
+
const previousAccountLogin = new Map();
|
|
45
75
|
let polls = 0;
|
|
46
76
|
let lines = 0;
|
|
47
77
|
let suppressed = false;
|
|
@@ -57,6 +87,18 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
57
87
|
suppressed = true;
|
|
58
88
|
}
|
|
59
89
|
};
|
|
90
|
+
const emitAccountEvent = (event) => {
|
|
91
|
+
if (lines < maxOutputLines - 1) {
|
|
92
|
+
writeAccountEvent(event);
|
|
93
|
+
lines += 1;
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (!suppressed) {
|
|
97
|
+
writeLine("lane watch: output limit reached; further events suppressed");
|
|
98
|
+
lines = maxOutputLines;
|
|
99
|
+
suppressed = true;
|
|
100
|
+
}
|
|
101
|
+
};
|
|
60
102
|
const unreadable = (kind, id, err) => {
|
|
61
103
|
const message = `${kind} ${id}: unreadable record (${readableError(err)})`;
|
|
62
104
|
writeError(`lane watch: ${message}`);
|
|
@@ -87,6 +129,15 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
87
129
|
previousStates.set(laneId, record.state);
|
|
88
130
|
}
|
|
89
131
|
}
|
|
132
|
+
const keepAliveWaiting = new Set();
|
|
133
|
+
if (deps.keepAliveEnabled !== undefined) {
|
|
134
|
+
for (const record of records) {
|
|
135
|
+
if (isTerminalLaneState(record.state) &&
|
|
136
|
+
canResumeKeepAliveLane(record) &&
|
|
137
|
+
await deps.keepAliveEnabled(record))
|
|
138
|
+
keepAliveWaiting.add(record.lane_id);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
90
141
|
let questionIds = [];
|
|
91
142
|
try {
|
|
92
143
|
questionIds = listQuestions(hqRoot);
|
|
@@ -113,7 +164,76 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
|
|
|
113
164
|
}
|
|
114
165
|
previousQuestionStatuses.set(questionId, record.status);
|
|
115
166
|
}
|
|
116
|
-
if (
|
|
167
|
+
if (deps.readAccountStatuses !== undefined) {
|
|
168
|
+
const accountLanes = records.filter((record) => record.account !== undefined &&
|
|
169
|
+
(!isTerminalLaneState(record.state) || keepAliveWaiting.has(record.lane_id)));
|
|
170
|
+
let statuses;
|
|
171
|
+
try {
|
|
172
|
+
statuses = await deps.readAccountStatuses(accountLanes);
|
|
173
|
+
}
|
|
174
|
+
catch (err) {
|
|
175
|
+
const message = `account login status: ${readableError(err)}`;
|
|
176
|
+
writeError(`lane watch: ${message}`);
|
|
177
|
+
throw new LanesError("watch_account_status_unavailable", message, {
|
|
178
|
+
cause: readableError(err),
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
const statusLanes = new Map(accountLanes.map((record) => [record.lane_id, record]));
|
|
182
|
+
const grouped = new Map();
|
|
183
|
+
const seenLaneIds = new Set();
|
|
184
|
+
for (const status of statuses) {
|
|
185
|
+
const lane = statusLanes.get(status.lane_id);
|
|
186
|
+
if (!lane || lane.provider !== status.provider || lane.account !== status.account ||
|
|
187
|
+
lane.host.kind !== status.host.kind || lane.host.id !== status.host.id ||
|
|
188
|
+
(status.logged_in !== true && status.logged_in !== false && status.logged_in !== null) ||
|
|
189
|
+
seenLaneIds.has(status.lane_id)) {
|
|
190
|
+
const message = `malformed account login status for lane ${JSON.stringify(status.lane_id)}`;
|
|
191
|
+
writeError(`lane watch: ${message}`);
|
|
192
|
+
throw new LanesError("watch_account_status_invalid", message, {
|
|
193
|
+
lane_id: status.lane_id,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
seenLaneIds.add(status.lane_id);
|
|
197
|
+
const key = JSON.stringify([status.provider, status.account, status.host.kind, status.host.id]);
|
|
198
|
+
const existing = grouped.get(key);
|
|
199
|
+
if (existing && existing.status.logged_in !== status.logged_in) {
|
|
200
|
+
const message = `conflicting account login statuses for ${status.provider} account ${status.account} on ${status.host.kind}:${status.host.id}`;
|
|
201
|
+
writeError(`lane watch: ${message}`);
|
|
202
|
+
throw new LanesError("watch_account_status_invalid", message, {
|
|
203
|
+
provider: status.provider,
|
|
204
|
+
account: status.account,
|
|
205
|
+
host: status.host,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
const group = existing ?? { status, laneIds: new Set() };
|
|
209
|
+
group.laneIds.add(status.lane_id);
|
|
210
|
+
grouped.set(key, group);
|
|
211
|
+
}
|
|
212
|
+
const missingLane = accountLanes.find((lane) => !seenLaneIds.has(lane.lane_id));
|
|
213
|
+
if (missingLane) {
|
|
214
|
+
const message = `account login status is missing for lane ${missingLane.lane_id}`;
|
|
215
|
+
writeError(`lane watch: ${message}`);
|
|
216
|
+
throw new LanesError("watch_account_status_invalid", message, {
|
|
217
|
+
lane_id: missingLane.lane_id,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
for (const [key, group] of grouped) {
|
|
221
|
+
const { status } = group;
|
|
222
|
+
const prior = previousAccountLogin.get(key);
|
|
223
|
+
if (prior === true && status.logged_in === false) {
|
|
224
|
+
emitAccountEvent({
|
|
225
|
+
event: "account_logged_out",
|
|
226
|
+
provider: status.provider,
|
|
227
|
+
account: status.account,
|
|
228
|
+
host: status.host,
|
|
229
|
+
lane_ids: [...group.laneIds].sort(),
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
if (status.logged_in !== null)
|
|
233
|
+
previousAccountLogin.set(key, status.logged_in);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
if (records.every((record) => isTerminalLaneState(record.state) && !keepAliveWaiting.has(record.lane_id))) {
|
|
117
237
|
return { polls, lines, suppressed };
|
|
118
238
|
}
|
|
119
239
|
await sleep(intervalMs);
|