@indigoai-us/hq-cli 5.278.0 → 5.280.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/commands/lanes-shared.d.ts +5 -0
- package/dist/commands/lanes.d.ts +2 -1
- package/dist/commands/lanes.js +212 -32
- package/dist/lib/lanes/reconcile.d.ts +5 -0
- package/dist/lib/lanes/reconcile.js +148 -3
- package/dist/lib/lanes/types.d.ts +3 -4
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.280.0] — 2026-09-29
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Single-lane commands for Outpost-hosted lanes use a configurable pickup wait,
|
|
10
|
+
defaulting to 120 seconds. `hq lanes list` capacity probes use a separate
|
|
11
|
+
15-second timeout. A lane ID with a registered host suffix routes to that
|
|
12
|
+
Outpost first.
|
|
13
|
+
|
|
14
|
+
## [5.279.0] — 2026-09-29
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- Claude keep-alive resumes use the same account selection as manual resumes,
|
|
19
|
+
including profile pins and the optional recent-use tiebreak. If no eligible
|
|
20
|
+
account can be selected, the lane resumes on its current account and records
|
|
21
|
+
the failure.
|
|
22
|
+
|
|
5
23
|
## [5.278.0] — 2026-09-29
|
|
6
24
|
|
|
7
25
|
### Changed
|
|
@@ -195,6 +195,7 @@ export interface Ctx {
|
|
|
195
195
|
resolveCompany: (company?: string) => Promise<LaneCompanyRef>;
|
|
196
196
|
resolveActiveCompany: (company: string) => Promise<LaneCompanyRef | null>;
|
|
197
197
|
listOutpostIds: () => Promise<string[]>;
|
|
198
|
+
listOutpostRoutes: () => Promise<OutpostRouteIdentity[]>;
|
|
198
199
|
readOutpostIdentity: () => OutpostIdentity;
|
|
199
200
|
spawnLogin: (opts: {
|
|
200
201
|
command: string;
|
|
@@ -244,6 +245,10 @@ export interface Ctx {
|
|
|
244
245
|
mergeHoldEnabled?: (company: LaneCompanyRef) => boolean | Promise<boolean>;
|
|
245
246
|
mergeReviewEnabled?: (company: LaneCompanyRef) => boolean | Promise<boolean>;
|
|
246
247
|
}
|
|
248
|
+
export interface OutpostRouteIdentity {
|
|
249
|
+
outpostId: string;
|
|
250
|
+
instanceName?: string;
|
|
251
|
+
}
|
|
247
252
|
export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
248
253
|
export declare const CLAUDE_ACCOUNT_BALANCE_SIGNALS_FLAG = "lanes.claude-account-balance-signals";
|
|
249
254
|
export declare function defaultLanePlacementEnabled(company: LaneCompanyRef | undefined): Promise<boolean>;
|
package/dist/commands/lanes.d.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `watch` and the monitor reminder are the line-oriented text surfaces.
|
|
7
7
|
*/
|
|
8
8
|
import { Command } from "commander";
|
|
9
|
-
import { type LanesMesh, type SpawnLoginResult } from "./lanes-shared.js";
|
|
9
|
+
import { type LanesMesh, type OutpostRouteIdentity, type SpawnLoginResult } from "./lanes-shared.js";
|
|
10
10
|
export type { LanesMesh, SpawnLoginResult } from "./lanes-shared.js";
|
|
11
11
|
export { CLAUDE_ACCOUNT_BALANCE_SIGNALS_FLAG, LANE_OUTPOST_LOAD_BALANCING_FLAG } from "./lanes-shared.js";
|
|
12
12
|
export { resolveAnswerChoice } from "./lanes-questions.js";
|
|
@@ -38,6 +38,7 @@ export interface LanesCommandDeps {
|
|
|
38
38
|
/** Membership-backed explicit company selection for account-home writers. */
|
|
39
39
|
resolveActiveCompany?: (company: string) => Promise<LaneCompanyRef | null>;
|
|
40
40
|
listOutpostIds?: () => Promise<string[]>;
|
|
41
|
+
listOutpostRoutes?: () => Promise<OutpostRouteIdentity[]>;
|
|
41
42
|
readOutpostIdentity?: () => OutpostIdentity;
|
|
42
43
|
spawnLogin?: (opts: {
|
|
43
44
|
command: string;
|
package/dist/commands/lanes.js
CHANGED
|
@@ -36,7 +36,7 @@ import { createOutpostProfileRepinContext, parseOutpostProfileRepinContext, } fr
|
|
|
36
36
|
import { createOutpostRegistryWorkerApprovalContext, parseOutpostRegistryWorkerApprovalContext, } from "../lib/lanes/outpost-registry-worker-approval.js";
|
|
37
37
|
import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
|
|
38
38
|
export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
|
|
39
|
-
import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
39
|
+
import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, PICKUP_WAIT_MS, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
40
40
|
import { outpostHireSummaryCacheKey, readOutpostHireSummaryCache, writeOutpostHireSummaryCache, } from "../lib/lanes/outpost-hire-cache.js";
|
|
41
41
|
import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
|
|
42
42
|
import { BALANCE_MOVABLE_STATES, selectClaudeAccount, } from "../lib/lanes/claude-balance.js";
|
|
@@ -159,6 +159,10 @@ async function defaultListOutpostIds() {
|
|
|
159
159
|
const rows = await defaultOutpostsClient().list();
|
|
160
160
|
return rows.map((row) => row.outpostId);
|
|
161
161
|
}
|
|
162
|
+
async function defaultListOutpostRoutes() {
|
|
163
|
+
const rows = await defaultOutpostsClient().list();
|
|
164
|
+
return rows.map(({ outpostId, instanceName }) => ({ outpostId, instanceName }));
|
|
165
|
+
}
|
|
162
166
|
async function defaultStageOutpostPayload(outpostId, bytes) {
|
|
163
167
|
return stageOutpostPayload(defaultOutpostsClient(), outpostId, bytes);
|
|
164
168
|
}
|
|
@@ -218,6 +222,9 @@ function resolveDeps(deps = {}) {
|
|
|
218
222
|
};
|
|
219
223
|
const bindProcessCwd = deps.hqRoot === undefined;
|
|
220
224
|
const resolveRoot = deps.hqRoot ?? (() => resolveLanesHqRoot({ env, cwd: cwd() }));
|
|
225
|
+
const listOutpostRoutes = deps.listOutpostRoutes ?? (deps.listOutpostIds
|
|
226
|
+
? async () => (await deps.listOutpostIds()).map((outpostId) => ({ outpostId }))
|
|
227
|
+
: defaultListOutpostRoutes);
|
|
221
228
|
const hqRoot = () => {
|
|
222
229
|
callerCwd();
|
|
223
230
|
const root = resolveRoot();
|
|
@@ -236,6 +243,7 @@ function resolveDeps(deps = {}) {
|
|
|
236
243
|
resolveCompany: deps.resolveCompany ?? defaultResolveCompany,
|
|
237
244
|
resolveActiveCompany: deps.resolveActiveCompany ?? defaultResolveActiveCompany,
|
|
238
245
|
listOutpostIds: deps.listOutpostIds ?? defaultListOutpostIds,
|
|
246
|
+
listOutpostRoutes,
|
|
239
247
|
readOutpostIdentity: deps.readOutpostIdentity ?? (() => readOutpostIdentity()),
|
|
240
248
|
spawnLogin: deps.spawnLogin ?? defaultSpawnLogin,
|
|
241
249
|
throttleDeps: deps.throttleDeps ?? defaultThrottleDeps(env),
|
|
@@ -611,6 +619,7 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
|
|
|
611
619
|
}
|
|
612
620
|
const outpostId = lane.host.id;
|
|
613
621
|
const hqBinary = outpostCliOverride(ctx, outpostId);
|
|
622
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
614
623
|
let dispatched;
|
|
615
624
|
try {
|
|
616
625
|
dispatched = await ctx.dispatchOutpostExec({
|
|
@@ -620,12 +629,15 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
|
|
|
620
629
|
laneId: lane.lane_id,
|
|
621
630
|
hqBinary,
|
|
622
631
|
}),
|
|
623
|
-
timeoutSeconds:
|
|
624
|
-
waitMs
|
|
632
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
633
|
+
waitMs,
|
|
625
634
|
});
|
|
626
635
|
}
|
|
627
636
|
catch (err) {
|
|
628
|
-
|
|
637
|
+
const message = dispatchErrorMessage(err);
|
|
638
|
+
throw new LanesError("outpost_lane_hire_summary_unavailable", /timed out waiting for outpost pickup/i.test(message)
|
|
639
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `reading hire summary for lane ${lane.lane_id}`)
|
|
640
|
+
: `Could not read hire summary for lane ${lane.lane_id} from Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId });
|
|
629
641
|
}
|
|
630
642
|
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
631
643
|
throw new LanesError("outpost_lane_hire_summary_unavailable", `Outpost ${outpostId} did not confirm the hire summary for lane ${lane.lane_id}`, { lane_id: lane.lane_id, outpost_id: outpostId, command_id: dispatched.commandId });
|
|
@@ -808,8 +820,6 @@ async function batchOwnerLaneHireFields(ctx, lanes) {
|
|
|
808
820
|
writeOutpostHireSummaryCache(ctx.hqRoot(), cacheUpdates, ctx.now().getTime(), cached);
|
|
809
821
|
return outcomes;
|
|
810
822
|
}
|
|
811
|
-
const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
|
|
812
|
-
const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
|
|
813
823
|
const OUTPOST_HIRE_SUMMARY_CONCURRENCY = 4;
|
|
814
824
|
async function laneHirePresentation(ctx, lane, lanes) {
|
|
815
825
|
const local = laneHireFields(lane, lanes);
|
|
@@ -886,6 +896,28 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
|
|
|
886
896
|
}
|
|
887
897
|
const CAPACITY_PROBE_TIMEOUT_MS = 15_000;
|
|
888
898
|
const MAX_PLACEMENT_OUTPOSTS = 16;
|
|
899
|
+
const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
|
|
900
|
+
const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
|
|
901
|
+
const OUTPOST_SINGLE_LANE_WAIT_ENV = "HQ_LANES_OUTPOST_SINGLE_LANE_WAIT_MS";
|
|
902
|
+
const OUTPOST_SINGLE_LANE_WAIT_MAX_MS = 600_000;
|
|
903
|
+
function outpostSingleLaneWaitMs(env) {
|
|
904
|
+
const raw = env[OUTPOST_SINGLE_LANE_WAIT_ENV]?.trim() || String(PICKUP_WAIT_MS);
|
|
905
|
+
const value = /^\d+$/.test(raw) ? Number(raw) : Number.NaN;
|
|
906
|
+
if (!Number.isSafeInteger(value) ||
|
|
907
|
+
value < CAPACITY_PROBE_TIMEOUT_MS ||
|
|
908
|
+
value > OUTPOST_SINGLE_LANE_WAIT_MAX_MS) {
|
|
909
|
+
throw new LanesError("outpost_wait_invalid", `${OUTPOST_SINGLE_LANE_WAIT_ENV} must be an integer from ${CAPACITY_PROBE_TIMEOUT_MS} to ${OUTPOST_SINGLE_LANE_WAIT_MAX_MS} milliseconds`, { env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
910
|
+
}
|
|
911
|
+
return value;
|
|
912
|
+
}
|
|
913
|
+
function outpostRouteMatchesHost(route, host) {
|
|
914
|
+
return typeof route.instanceName === "string" &&
|
|
915
|
+
route.instanceName.trim() !== "" &&
|
|
916
|
+
sanitizeHostComponent(route.instanceName.trim()) === sanitizeHostComponent(host.trim());
|
|
917
|
+
}
|
|
918
|
+
function outpostPickupTimeoutMessage(outpostId, waitMs, operation) {
|
|
919
|
+
return `Outpost ${outpostId} pickup timed out after ${waitMs}ms while ${operation}. Raise ${OUTPOST_SINGLE_LANE_WAIT_ENV} (milliseconds) and retry.`;
|
|
920
|
+
}
|
|
889
921
|
function outpostIdentityUnknownError(message, expectedId, details = {}) {
|
|
890
922
|
const expected = expectedId ?? "<expected-outpost-id>";
|
|
891
923
|
const command = `hq lanes outpost-identity backfill --id ${expected}`;
|
|
@@ -1025,14 +1057,20 @@ function normalizedLaneListQuery(query) {
|
|
|
1025
1057
|
};
|
|
1026
1058
|
}
|
|
1027
1059
|
async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
|
|
1060
|
+
const targetedLookup = query.hierarchy !== undefined || query.laneIds !== undefined;
|
|
1061
|
+
const waitMs = targetedLookup ? outpostSingleLaneWaitMs(ctx.env) : CAPACITY_PROBE_TIMEOUT_MS;
|
|
1028
1062
|
let result;
|
|
1029
1063
|
try {
|
|
1030
|
-
result = await dispatchOutpostLaneList(ctx, outpostId, query,
|
|
1064
|
+
result = await dispatchOutpostLaneList(ctx, outpostId, query, Math.ceil(waitMs / 1000), waitMs);
|
|
1031
1065
|
}
|
|
1032
1066
|
catch (err) {
|
|
1067
|
+
const message = dispatchErrorMessage(err);
|
|
1068
|
+
if (/timed out waiting for outpost pickup/i.test(message)) {
|
|
1069
|
+
throw new LanesError("outpost_lane_list_unavailable", outpostPickupTimeoutMessage(outpostId, waitMs, "reading the requested lane"), { outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
1070
|
+
}
|
|
1033
1071
|
if (isLanesError(err))
|
|
1034
1072
|
throw err;
|
|
1035
|
-
throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${
|
|
1073
|
+
throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${message}`, { outpost_id: outpostId });
|
|
1036
1074
|
}
|
|
1037
1075
|
if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
|
|
1038
1076
|
throw new LanesError("outpost_lane_list_unavailable", `Outpost ${outpostId} did not return a successful lane list`, { outpost_id: outpostId, status: result.status, done: result.done ?? false, exit_code: result.exitCode });
|
|
@@ -1119,17 +1157,34 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1119
1157
|
if (!hostSuffix || sanitizeHostComponent(hostSuffix) !== hostSuffix) {
|
|
1120
1158
|
throw commandLaneNotFound(laneId, options.outpostHint === true);
|
|
1121
1159
|
}
|
|
1122
|
-
let
|
|
1160
|
+
let rawOutpostRoutes;
|
|
1123
1161
|
try {
|
|
1124
|
-
|
|
1162
|
+
rawOutpostRoutes = await ctx.listOutpostRoutes();
|
|
1125
1163
|
}
|
|
1126
1164
|
catch (err) {
|
|
1127
1165
|
throw new LanesError("outpost_list_unavailable", `Could not resolve registered Outposts for lane ${laneId}: ${dispatchErrorMessage(err)}`, { lane_id: laneId });
|
|
1128
1166
|
}
|
|
1129
|
-
if (!Array.isArray(
|
|
1167
|
+
if (!Array.isArray(rawOutpostRoutes)) {
|
|
1130
1168
|
throw new LanesError("outpost_list_invalid", "Registered Outpost list is not an array", { lane_id: laneId });
|
|
1131
1169
|
}
|
|
1132
|
-
const
|
|
1170
|
+
const outpostRoutes = rawOutpostRoutes.map((raw, index) => {
|
|
1171
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw) ||
|
|
1172
|
+
typeof raw.outpostId !== "string" ||
|
|
1173
|
+
!raw.outpostId ||
|
|
1174
|
+
(raw.instanceName !== undefined &&
|
|
1175
|
+
typeof raw.instanceName !== "string")) {
|
|
1176
|
+
throw new LanesError("outpost_list_invalid", `Registered Outpost row ${index} is malformed`, {
|
|
1177
|
+
lane_id: laneId,
|
|
1178
|
+
index,
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1181
|
+
const row = raw;
|
|
1182
|
+
return {
|
|
1183
|
+
outpostId: row.outpostId.trim(),
|
|
1184
|
+
...(typeof row.instanceName === "string" ? { instanceName: row.instanceName.trim() } : {}),
|
|
1185
|
+
};
|
|
1186
|
+
});
|
|
1187
|
+
const outpostIds = outpostRoutes.map(({ outpostId }) => outpostId);
|
|
1133
1188
|
if (outpostIds.length > MAX_PLACEMENT_OUTPOSTS ||
|
|
1134
1189
|
outpostIds.some((id) => id === "") ||
|
|
1135
1190
|
new Set(outpostIds).size !== outpostIds.length) {
|
|
@@ -1142,9 +1197,23 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1142
1197
|
if (requestedOutpostId !== undefined && !outpostIds.includes(requestedOutpostId)) {
|
|
1143
1198
|
throw new LanesError("outpost_not_registered", `Outpost ${requestedOutpostId} is not registered`, { lane_id: laneId, outpost_id: requestedOutpostId });
|
|
1144
1199
|
}
|
|
1200
|
+
const hostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, hostSuffix));
|
|
1201
|
+
const identity = ctx.readOutpostIdentity();
|
|
1202
|
+
const localHostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, ctx.hostname()));
|
|
1203
|
+
if (localHostRoutes.length > 1 || (identity.kind === "id" && localHostRoutes.some((route) => route.outpostId !== identity.id))) {
|
|
1204
|
+
throw new LanesError("outpost_host_ambiguous", "Registered Outpost host identity conflicts with this host's identity", { lane_id: laneId, local_outpost_id: identity.kind === "id" ? identity.id : undefined });
|
|
1205
|
+
}
|
|
1206
|
+
const localOutpostId = identity.kind === "id" ? identity.id : localHostRoutes[0]?.outpostId;
|
|
1207
|
+
const hostOutpostIds = hostRoutes.map(({ outpostId }) => outpostId);
|
|
1208
|
+
const hostOutpostIdSet = new Set(hostOutpostIds);
|
|
1145
1209
|
const matches = [];
|
|
1146
1210
|
const failedOutpostIds = [];
|
|
1147
|
-
const lookupOutpostIds = requestedOutpostId
|
|
1211
|
+
const lookupOutpostIds = requestedOutpostId !== undefined
|
|
1212
|
+
? [requestedOutpostId]
|
|
1213
|
+
: [
|
|
1214
|
+
...hostOutpostIds.filter((id) => id !== localOutpostId),
|
|
1215
|
+
...outpostIds.filter((id) => id !== localOutpostId && !hostOutpostIdSet.has(id)),
|
|
1216
|
+
];
|
|
1148
1217
|
for (const outpostId of lookupOutpostIds) {
|
|
1149
1218
|
let ownerLanes;
|
|
1150
1219
|
try {
|
|
@@ -1156,6 +1225,8 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1156
1225
|
catch (err) {
|
|
1157
1226
|
if (requestedOutpostId !== undefined)
|
|
1158
1227
|
throw err;
|
|
1228
|
+
if (isLanesError(err) && err.error === "outpost_wait_invalid")
|
|
1229
|
+
throw err;
|
|
1159
1230
|
failedOutpostIds.push(outpostId);
|
|
1160
1231
|
continue;
|
|
1161
1232
|
}
|
|
@@ -3547,6 +3618,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3547
3618
|
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch to`, { lane_id: lane.lane_id });
|
|
3548
3619
|
}
|
|
3549
3620
|
const outpostId = lane.host.id;
|
|
3621
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
3550
3622
|
const profileRepinContext = action === "resume" &&
|
|
3551
3623
|
options.worker !== undefined &&
|
|
3552
3624
|
callerMatchesSenior(ctx, lane.senior)
|
|
@@ -3583,6 +3655,8 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3583
3655
|
try {
|
|
3584
3656
|
const dispatchCommand = (contexts) => ctx.dispatchOutpostExec({
|
|
3585
3657
|
outpostId,
|
|
3658
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
3659
|
+
waitMs,
|
|
3586
3660
|
command: buildOutpostLaneCommand({
|
|
3587
3661
|
action: action === "grant"
|
|
3588
3662
|
? "grant-target"
|
|
@@ -3636,7 +3710,10 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3636
3710
|
}
|
|
3637
3711
|
}
|
|
3638
3712
|
catch (err) {
|
|
3639
|
-
|
|
3713
|
+
const message = dispatchErrorMessage(err);
|
|
3714
|
+
throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
|
|
3715
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `dispatching ${action} for lane ${lane.lane_id}`)
|
|
3716
|
+
: `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
3640
3717
|
}
|
|
3641
3718
|
if (grantContextRejected) {
|
|
3642
3719
|
throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing this fix before it can resume lane ${lane.lane_id} under its primary-hosted senior`, {
|
|
@@ -3834,6 +3911,7 @@ function parseReparentDescendantIds(value, laneId, outpostId) {
|
|
|
3834
3911
|
return ids;
|
|
3835
3912
|
}
|
|
3836
3913
|
async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
|
|
3914
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
3837
3915
|
let dispatched;
|
|
3838
3916
|
try {
|
|
3839
3917
|
dispatched = await ctx.dispatchOutpostExec({
|
|
@@ -3843,12 +3921,15 @@ async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
|
|
|
3843
3921
|
laneId,
|
|
3844
3922
|
hqBinary: outpostCliOverride(ctx, outpostId),
|
|
3845
3923
|
}),
|
|
3846
|
-
timeoutSeconds:
|
|
3847
|
-
waitMs
|
|
3924
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
3925
|
+
waitMs,
|
|
3848
3926
|
});
|
|
3849
3927
|
}
|
|
3850
3928
|
catch (err) {
|
|
3851
|
-
|
|
3929
|
+
const message = dispatchErrorMessage(err);
|
|
3930
|
+
throw new LanesError("outpost_lane_sync_unavailable", /timed out waiting for outpost pickup/i.test(message)
|
|
3931
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `reading lane ${laneId}`)
|
|
3932
|
+
: `Could not refresh descendant lane ${laneId} from Outpost ${outpostId}: ${message}`, { lane_id: laneId, outpost_id: outpostId });
|
|
3852
3933
|
}
|
|
3853
3934
|
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
3854
3935
|
throw new LanesError("outpost_lane_sync_unavailable", `Outpost ${outpostId} did not confirm the snapshot for descendant lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, command_id: dispatched.commandId });
|
|
@@ -5534,12 +5615,12 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5534
5615
|
const at = now.toISOString();
|
|
5535
5616
|
const from = lane.account ?? null;
|
|
5536
5617
|
// The capacity decision and the record write run under the target
|
|
5537
|
-
// account's admission lock,
|
|
5538
|
-
// slot is taken by a running lane or
|
|
5539
|
-
// pointed at
|
|
5540
|
-
// (`pending_resume` set by this write
|
|
5541
|
-
// into one free slot cannot both start.
|
|
5542
|
-
|
|
5618
|
+
// account's admission lock, or while reconcile holds all provider
|
|
5619
|
+
// admission locks. A slot is taken by a running lane or a lane another move
|
|
5620
|
+
// has pointed at its target account but whose spawn has not claimed it yet
|
|
5621
|
+
// (`pending_resume` is set by this write and cleared by the claim), so two
|
|
5622
|
+
// moves into one free slot cannot both start.
|
|
5623
|
+
const writeMove = async () => {
|
|
5543
5624
|
const admission = input.deferTurnStart
|
|
5544
5625
|
? undefined
|
|
5545
5626
|
: await assessLaneAdmission(ctx, lane, {
|
|
@@ -5602,7 +5683,10 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5602
5683
|
return next;
|
|
5603
5684
|
}), input.grantContext);
|
|
5604
5685
|
return { lane: moved, from, to: target, copy, queued };
|
|
5605
|
-
}
|
|
5686
|
+
};
|
|
5687
|
+
return input.admissionLocksHeld
|
|
5688
|
+
? writeMove()
|
|
5689
|
+
: withLaneAdmissionLock(hqRoot, lane.provider, writeMove, ctx.sleep);
|
|
5606
5690
|
}
|
|
5607
5691
|
async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile, options = {}) {
|
|
5608
5692
|
if (lane.provider !== "claude")
|
|
@@ -6370,22 +6454,27 @@ function parseRemoteCancelledQuestionIds(value, laneId, outpostId, commandId) {
|
|
|
6370
6454
|
}
|
|
6371
6455
|
return value;
|
|
6372
6456
|
}
|
|
6373
|
-
async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId) {
|
|
6457
|
+
async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId, waitMs) {
|
|
6374
6458
|
let dispatched;
|
|
6375
6459
|
try {
|
|
6376
6460
|
dispatched = await ctx.dispatchOutpostExec({
|
|
6377
6461
|
outpostId,
|
|
6378
6462
|
command: buildOutpostLaneCommand({ action: "show", laneId }),
|
|
6379
|
-
timeoutSeconds:
|
|
6380
|
-
waitMs
|
|
6463
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
6464
|
+
waitMs,
|
|
6381
6465
|
});
|
|
6382
6466
|
}
|
|
6383
6467
|
catch (err) {
|
|
6384
|
-
|
|
6468
|
+
const message = dispatchErrorMessage(err);
|
|
6469
|
+
throw new LanesError("outpost_delivery_unconfirmed", /timed out waiting for outpost pickup/i.test(message)
|
|
6470
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `confirming terminal lane ${laneId}`)
|
|
6471
|
+
: `Could not read terminal lane ${laneId} from Outpost ${outpostId}: ${message}`, {
|
|
6385
6472
|
lane_id: laneId,
|
|
6386
6473
|
outpost_id: outpostId,
|
|
6387
6474
|
command_id: stopCommandId,
|
|
6388
6475
|
sync_command_id: null,
|
|
6476
|
+
wait_ms: waitMs,
|
|
6477
|
+
wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV,
|
|
6389
6478
|
});
|
|
6390
6479
|
}
|
|
6391
6480
|
const commandId = dispatched.commandId?.trim();
|
|
@@ -6454,17 +6543,21 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
|
6454
6543
|
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch stop to`, { lane_id: lane.lane_id });
|
|
6455
6544
|
}
|
|
6456
6545
|
const outpostId = lane.host.id;
|
|
6546
|
+
const waitMs = outpostSingleLaneWaitMs(ctx.env);
|
|
6457
6547
|
let dispatched;
|
|
6458
6548
|
try {
|
|
6459
6549
|
dispatched = await ctx.dispatchOutpostExec({
|
|
6460
6550
|
outpostId,
|
|
6461
6551
|
command: buildOutpostLaneCommand({ action: "stop", laneId: lane.lane_id, reason }),
|
|
6462
|
-
timeoutSeconds:
|
|
6463
|
-
waitMs
|
|
6552
|
+
timeoutSeconds: Math.ceil(waitMs / 1000),
|
|
6553
|
+
waitMs,
|
|
6464
6554
|
});
|
|
6465
6555
|
}
|
|
6466
6556
|
catch (err) {
|
|
6467
|
-
|
|
6557
|
+
const message = dispatchErrorMessage(err);
|
|
6558
|
+
throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
|
|
6559
|
+
? outpostPickupTimeoutMessage(outpostId, waitMs, `stopping lane ${lane.lane_id}`)
|
|
6560
|
+
: `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
6468
6561
|
}
|
|
6469
6562
|
if (reason !== undefined && outpostRejectedStopReasonOption(dispatched)) {
|
|
6470
6563
|
throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing stop reason support before it can stop lane ${lane.lane_id}`, {
|
|
@@ -6512,7 +6605,7 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
|
|
|
6512
6605
|
}
|
|
6513
6606
|
state = remote.state;
|
|
6514
6607
|
alreadyTerminal = true;
|
|
6515
|
-
const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId);
|
|
6608
|
+
const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId, waitMs);
|
|
6516
6609
|
if (owner.lane.state !== state) {
|
|
6517
6610
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} returned a different terminal state for lane ${lane.lane_id}`, {
|
|
6518
6611
|
lane_id: lane.lane_id,
|
|
@@ -6993,6 +7086,93 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
|
|
|
6993
7086
|
recoverQueuedMessages,
|
|
6994
7087
|
mesh: ctx.mesh,
|
|
6995
7088
|
env: ctx.env,
|
|
7089
|
+
prepareKeepAliveClaudeResume: async (lane) => {
|
|
7090
|
+
const recoveredLane = laneWithRecoveredProviderSession(hqRoot, lane);
|
|
7091
|
+
const pendingModel = recoveredLane.pending_settings?.model;
|
|
7092
|
+
const selectionLane = pendingModel === undefined
|
|
7093
|
+
? recoveredLane
|
|
7094
|
+
: { ...recoveredLane, model: pendingModel, model_source: undefined };
|
|
7095
|
+
let automatic;
|
|
7096
|
+
try {
|
|
7097
|
+
automatic = await selectAutomaticClaudeResumeAccount(ctx, selectionLane, undefined, {
|
|
7098
|
+
repairUnavailableAccount: true,
|
|
7099
|
+
});
|
|
7100
|
+
}
|
|
7101
|
+
catch (err) {
|
|
7102
|
+
return {
|
|
7103
|
+
lane: recoveredLane,
|
|
7104
|
+
selectionFailure: err instanceof LanesError
|
|
7105
|
+
? `${err.error}: ${err.message}`
|
|
7106
|
+
: err instanceof Error
|
|
7107
|
+
? err.message
|
|
7108
|
+
: String(err),
|
|
7109
|
+
};
|
|
7110
|
+
}
|
|
7111
|
+
if (automatic === undefined)
|
|
7112
|
+
return { lane: recoveredLane };
|
|
7113
|
+
const { selection } = automatic;
|
|
7114
|
+
if (opts.dryRun) {
|
|
7115
|
+
return {
|
|
7116
|
+
lane: {
|
|
7117
|
+
...selectionLane,
|
|
7118
|
+
account: selection.account,
|
|
7119
|
+
account_selection: selection,
|
|
7120
|
+
...(pendingModel !== undefined
|
|
7121
|
+
? { model: pendingModel, model_source: "cli" }
|
|
7122
|
+
: automatic.preservedModel !== undefined
|
|
7123
|
+
? { model: automatic.preservedModel, model_source: "preserved" }
|
|
7124
|
+
: {}),
|
|
7125
|
+
},
|
|
7126
|
+
};
|
|
7127
|
+
}
|
|
7128
|
+
let latest;
|
|
7129
|
+
try {
|
|
7130
|
+
latest = readLane(hqRoot, lane.lane_id);
|
|
7131
|
+
}
|
|
7132
|
+
catch (err) {
|
|
7133
|
+
throw new LanesError("keep_alive_account_selection_lane_unreadable", `Cannot re-read lane ${lane.lane_id} before applying Claude account selection: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id });
|
|
7134
|
+
}
|
|
7135
|
+
if (latest.version !== lane.version ||
|
|
7136
|
+
latest.state !== lane.state ||
|
|
7137
|
+
latest.keep_alive !== true ||
|
|
7138
|
+
latest.keep_alive_opted_in_at !== lane.keep_alive_opted_in_at ||
|
|
7139
|
+
latest.provider_session_id !== lane.provider_session_id ||
|
|
7140
|
+
latest.last_resume?.session_id !== lane.last_resume?.session_id)
|
|
7141
|
+
return { lane: latest };
|
|
7142
|
+
if (selection.account === lane.account) {
|
|
7143
|
+
return { lane: persistClaudeAccountSelection(hqRoot, recoveredLane, selection) };
|
|
7144
|
+
}
|
|
7145
|
+
try {
|
|
7146
|
+
const moved = await prepareAccountMove(ctx, recoveredLane, {
|
|
7147
|
+
account: selection.account,
|
|
7148
|
+
cliModel: pendingModel,
|
|
7149
|
+
accountHistoryReason: "keep_alive_resume",
|
|
7150
|
+
...(pendingModel === undefined && automatic.preservedModel !== undefined
|
|
7151
|
+
? { preservedModel: automatic.preservedModel }
|
|
7152
|
+
: {}),
|
|
7153
|
+
deferTurnStart: true,
|
|
7154
|
+
admissionLocksHeld: true,
|
|
7155
|
+
accountSelection: selection,
|
|
7156
|
+
});
|
|
7157
|
+
return { lane: moved.lane };
|
|
7158
|
+
}
|
|
7159
|
+
catch (err) {
|
|
7160
|
+
let latest;
|
|
7161
|
+
try {
|
|
7162
|
+
latest = readLane(hqRoot, lane.lane_id);
|
|
7163
|
+
}
|
|
7164
|
+
catch (readErr) {
|
|
7165
|
+
return {
|
|
7166
|
+
lane,
|
|
7167
|
+
selectionFailure: `account move failed: ${err instanceof Error ? err.message : String(err)}; lane re-read failed: ${readErr instanceof Error ? readErr.message : String(readErr)}`,
|
|
7168
|
+
};
|
|
7169
|
+
}
|
|
7170
|
+
return {
|
|
7171
|
+
lane: latest,
|
|
7172
|
+
selectionFailure: `account move failed: ${err instanceof LanesError ? `${err.error}: ${err.message}` : err instanceof Error ? err.message : String(err)}`,
|
|
7173
|
+
};
|
|
7174
|
+
}
|
|
7175
|
+
},
|
|
6996
7176
|
});
|
|
6997
7177
|
printJson(summary);
|
|
6998
7178
|
if (!summary.ok) {
|
|
@@ -92,6 +92,11 @@ export interface ReconcileDeps {
|
|
|
92
92
|
meshEmitTimeoutMs?: number;
|
|
93
93
|
/** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
|
|
94
94
|
readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
|
|
95
|
+
/** Reuse the command's Claude resume selection and session-move path for keep-alive. */
|
|
96
|
+
prepareKeepAliveClaudeResume?: (lane: LaneRecord) => Promise<{
|
|
97
|
+
lane: LaneRecord;
|
|
98
|
+
selectionFailure?: string;
|
|
99
|
+
}>;
|
|
95
100
|
/** Process-table snapshot seam for the opt-in runaway-search guard. */
|
|
96
101
|
runawayToolProcessLister?: () => readonly RunawayProcessInfo[];
|
|
97
102
|
/** Test seam: prevent real process signals while exercising the guard. */
|
|
@@ -1001,7 +1001,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1001
1001
|
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
1002
1002
|
const freshLanes = loadLaneRecords(hqRoot, summary);
|
|
1003
1003
|
const freshQuestions = loadQuestionRecords(hqRoot, summary);
|
|
1004
|
-
resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
|
|
1004
|
+
await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
|
|
1005
1005
|
const drainLanes = loadLaneRecords(hqRoot, summary);
|
|
1006
1006
|
await admitQueued(hqRoot, drainLanes, summary, startDeps, now);
|
|
1007
1007
|
await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, startDeps);
|
|
@@ -1723,7 +1723,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
|
|
|
1723
1723
|
throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
|
|
1724
1724
|
}
|
|
1725
1725
|
}
|
|
1726
|
-
function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany) {
|
|
1726
|
+
async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany) {
|
|
1727
1727
|
const nowMs = now.getTime();
|
|
1728
1728
|
const nowIso = now.toISOString();
|
|
1729
1729
|
const config = loadLanesConfig(hqRoot);
|
|
@@ -2108,6 +2108,44 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
2108
2108
|
});
|
|
2109
2109
|
continue;
|
|
2110
2110
|
}
|
|
2111
|
+
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
2112
|
+
if (seniorId !== undefined) {
|
|
2113
|
+
let grantPreflight;
|
|
2114
|
+
try {
|
|
2115
|
+
grantPreflight = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
|
|
2116
|
+
const latestLane = readLane(hqRoot, current.lane_id);
|
|
2117
|
+
if (latestLane.version !== current.version ||
|
|
2118
|
+
latestLane.state !== current.state ||
|
|
2119
|
+
latestLane.keep_alive !== true ||
|
|
2120
|
+
latestLane.keep_alive_opted_in_at !== current.keep_alive_opted_in_at ||
|
|
2121
|
+
latestLane.senior.kind !== "lane" ||
|
|
2122
|
+
latestLane.senior.id !== seniorId)
|
|
2123
|
+
return { kind: "stale" };
|
|
2124
|
+
const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
|
|
2125
|
+
return refusal === null
|
|
2126
|
+
? { kind: "eligible" }
|
|
2127
|
+
: { kind: "refused", error: refusal.error, message: refusal.message };
|
|
2128
|
+
});
|
|
2129
|
+
}
|
|
2130
|
+
catch (err) {
|
|
2131
|
+
markDirty(summary, {
|
|
2132
|
+
kind: "lane",
|
|
2133
|
+
id: current.lane_id,
|
|
2134
|
+
why: `keep-alive hire admission preflight failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2135
|
+
});
|
|
2136
|
+
continue;
|
|
2137
|
+
}
|
|
2138
|
+
if (grantPreflight.kind === "stale")
|
|
2139
|
+
continue;
|
|
2140
|
+
if (grantPreflight.kind === "refused") {
|
|
2141
|
+
summary.skipped.push({
|
|
2142
|
+
kind: "lane",
|
|
2143
|
+
id: current.lane_id,
|
|
2144
|
+
why: `keep_alive_hire_admission: ${grantPreflight.error}: ${grantPreflight.message}`,
|
|
2145
|
+
});
|
|
2146
|
+
continue;
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2111
2149
|
const resumedFrom = current.state === "done" ? "done" : "interrupted";
|
|
2112
2150
|
const resumeText = resumedFrom === "interrupted"
|
|
2113
2151
|
? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
|
|
@@ -2118,6 +2156,114 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
2118
2156
|
if (carriesAnsweredQuestions && !deps.dryRun) {
|
|
2119
2157
|
writeAnsweredQuestions(hqRoot, current, questions);
|
|
2120
2158
|
}
|
|
2159
|
+
let accountSelectionFailure;
|
|
2160
|
+
if (current.provider === "claude" && deps.prepareKeepAliveClaudeResume !== undefined) {
|
|
2161
|
+
const optInBeforePreparation = current.keep_alive_opted_in_at;
|
|
2162
|
+
let preparedLane;
|
|
2163
|
+
try {
|
|
2164
|
+
const prepared = await deps.prepareKeepAliveClaudeResume(current);
|
|
2165
|
+
preparedLane = prepared.lane;
|
|
2166
|
+
accountSelectionFailure = prepared.selectionFailure;
|
|
2167
|
+
}
|
|
2168
|
+
catch (err) {
|
|
2169
|
+
// Account balancing is best-effort for keep-alive. A failure to read
|
|
2170
|
+
// balance inputs must not prevent the existing session from resuming.
|
|
2171
|
+
try {
|
|
2172
|
+
preparedLane = readLane(hqRoot, current.lane_id);
|
|
2173
|
+
}
|
|
2174
|
+
catch (readErr) {
|
|
2175
|
+
markDirty(summary, {
|
|
2176
|
+
kind: "lane",
|
|
2177
|
+
id: current.lane_id,
|
|
2178
|
+
why: `keep-alive account selection failed and lane could not be re-read: ${readErr instanceof Error ? readErr.message : String(readErr)}`,
|
|
2179
|
+
});
|
|
2180
|
+
continue;
|
|
2181
|
+
}
|
|
2182
|
+
accountSelectionFailure = err instanceof Error ? err.message : String(err);
|
|
2183
|
+
}
|
|
2184
|
+
const preparedSessionId = preparedLane.provider_session_id ?? preparedLane.last_resume?.session_id;
|
|
2185
|
+
if (preparedLane.state !== initialState ||
|
|
2186
|
+
preparedLane.keep_alive !== true ||
|
|
2187
|
+
!laneRunsHere(preparedLane, deps.host) ||
|
|
2188
|
+
preparedSessionId !== currentSessionId)
|
|
2189
|
+
continue;
|
|
2190
|
+
let latest;
|
|
2191
|
+
try {
|
|
2192
|
+
latest = deps.dryRun
|
|
2193
|
+
? (deps.readKeepAliveRecheckLane?.(hqRoot, current.lane_id) ?? readLane(hqRoot, current.lane_id))
|
|
2194
|
+
: readLane(hqRoot, current.lane_id);
|
|
2195
|
+
}
|
|
2196
|
+
catch (err) {
|
|
2197
|
+
markDirty(summary, {
|
|
2198
|
+
kind: "lane",
|
|
2199
|
+
id: current.lane_id,
|
|
2200
|
+
why: `keep-alive eligibility recheck failed after account selection: ${err instanceof Error ? err.message : String(err)}`,
|
|
2201
|
+
});
|
|
2202
|
+
continue;
|
|
2203
|
+
}
|
|
2204
|
+
if (latest.state !== initialState || !laneRunsHere(latest, deps.host))
|
|
2205
|
+
continue;
|
|
2206
|
+
if (latest.keep_alive !== true) {
|
|
2207
|
+
recordKeepAliveSkip(hqRoot, latest, "not_opted_in", summary, deps);
|
|
2208
|
+
continue;
|
|
2209
|
+
}
|
|
2210
|
+
if (latest.keep_alive_opted_in_at !== optInBeforePreparation) {
|
|
2211
|
+
summary.skipped.push({
|
|
2212
|
+
kind: "lane",
|
|
2213
|
+
id: latest.lane_id,
|
|
2214
|
+
why: "keep_alive_eligibility_changed_during_account_selection",
|
|
2215
|
+
});
|
|
2216
|
+
continue;
|
|
2217
|
+
}
|
|
2218
|
+
const latestSessionId = latest.provider_session_id ?? latest.last_resume?.session_id;
|
|
2219
|
+
if (latestSessionId !== currentSessionId)
|
|
2220
|
+
continue;
|
|
2221
|
+
let latestEligibilityAtMs;
|
|
2222
|
+
if (initialState === "done") {
|
|
2223
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, latest.lane_id);
|
|
2224
|
+
if (!envelope || envelope.envelope.decision !== "done")
|
|
2225
|
+
continue;
|
|
2226
|
+
if (doneKeepAliveVerdict(latest, envelope.envelope, latest.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, latest, questions)) !== "eligible")
|
|
2227
|
+
continue;
|
|
2228
|
+
try {
|
|
2229
|
+
latestEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, latest, envelope.ref);
|
|
2230
|
+
}
|
|
2231
|
+
catch (err) {
|
|
2232
|
+
markDirty(summary, {
|
|
2233
|
+
kind: "lane",
|
|
2234
|
+
id: latest.lane_id,
|
|
2235
|
+
why: `keep-alive envelope timestamp unavailable after account selection: ${err instanceof Error ? err.message : String(err)}`,
|
|
2236
|
+
});
|
|
2237
|
+
continue;
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
else {
|
|
2241
|
+
latestEligibilityAtMs = Date.parse(latest.interrupted_at ?? "");
|
|
2242
|
+
}
|
|
2243
|
+
const latestOptedInAtMs = Date.parse(latest.keep_alive_opted_in_at ?? "");
|
|
2244
|
+
if (!Number.isFinite(latestEligibilityAtMs) || !Number.isFinite(latestOptedInAtMs))
|
|
2245
|
+
continue;
|
|
2246
|
+
if (latestEligibilityAtMs <= latestOptedInAtMs) {
|
|
2247
|
+
recordKeepAliveSkip(hqRoot, latest, initialState === "done" ? "envelope_before_opt_in" : "interruption_before_opt_in", summary, deps);
|
|
2248
|
+
continue;
|
|
2249
|
+
}
|
|
2250
|
+
current = deps.dryRun ? preparedLane : latest;
|
|
2251
|
+
}
|
|
2252
|
+
if (accountSelectionFailure !== undefined) {
|
|
2253
|
+
summary.skipped.push({
|
|
2254
|
+
kind: "lane",
|
|
2255
|
+
id: current.lane_id,
|
|
2256
|
+
why: `keep_alive_account_selection_failed: ${accountSelectionFailure}`,
|
|
2257
|
+
});
|
|
2258
|
+
if (!deps.dryRun) {
|
|
2259
|
+
recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
|
|
2260
|
+
event: "keep_alive_account_selection_failed",
|
|
2261
|
+
at: nowIso,
|
|
2262
|
+
account: current.account ?? null,
|
|
2263
|
+
reason: accountSelectionFailure,
|
|
2264
|
+
});
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2121
2267
|
const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
|
|
2122
2268
|
latestLane.state === current.state &&
|
|
2123
2269
|
latestLane.keep_alive === true &&
|
|
@@ -2145,7 +2291,6 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
|
|
|
2145
2291
|
keep_alive_exhausted_at: undefined,
|
|
2146
2292
|
keep_alive_blocked_reason: undefined,
|
|
2147
2293
|
}), deps.dryRun);
|
|
2148
|
-
const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
|
|
2149
2294
|
let queued;
|
|
2150
2295
|
if (seniorId) {
|
|
2151
2296
|
try {
|
|
@@ -500,9 +500,8 @@ export interface LaneRecord {
|
|
|
500
500
|
/** Eligible accounts for a queue entry created without an explicit account. */
|
|
501
501
|
admission_accounts?: string[];
|
|
502
502
|
/**
|
|
503
|
-
*
|
|
504
|
-
*
|
|
505
|
-
* like the other histories.
|
|
503
|
+
* Moves of the lane's session to another account of the same provider,
|
|
504
|
+
* newest last and capped like the other histories.
|
|
506
505
|
*/
|
|
507
506
|
account_history?: AccountHistoryEntry[];
|
|
508
507
|
/** Bounded audit history for explicitly authorized resumes from failed. */
|
|
@@ -665,7 +664,7 @@ export interface AccountHistoryEntry {
|
|
|
665
664
|
from: string | null;
|
|
666
665
|
to: string;
|
|
667
666
|
at: string;
|
|
668
|
-
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "operator_move";
|
|
667
|
+
reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "keep_alive_resume" | "operator_move";
|
|
669
668
|
}
|
|
670
669
|
export interface FailedResumeHistoryEntry {
|
|
671
670
|
at: string;
|