@indigoai-us/hq-cli 5.187.0 → 5.188.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/command-catalog.generated.d.ts +65 -0
- package/dist/command-catalog.generated.js +85 -0
- package/dist/commands/lanes.d.ts +7 -1
- package/dist/commands/lanes.js +894 -45
- package/dist/lib/lanes/claude-account-balance.d.ts +82 -0
- package/dist/lib/lanes/claude-account-balance.js +253 -0
- package/dist/lib/lanes/hire-grant-dispatch.d.ts +21 -0
- package/dist/lib/lanes/hire-grant-dispatch.js +167 -0
- package/dist/lib/lanes/hire-grants.d.ts +3 -2
- package/dist/lib/lanes/host-capacity.d.ts +19 -0
- package/dist/lib/lanes/host-capacity.js +68 -0
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/outpost-dispatch.d.ts +11 -1
- package/dist/lib/lanes/outpost-dispatch.js +20 -0
- package/dist/lib/lanes/ssm-command-context.d.ts +12 -0
- package/dist/lib/lanes/ssm-command-context.js +52 -0
- package/dist/lib/lanes/store.js +86 -0
- package/dist/lib/lanes/types.d.ts +48 -0
- package/dist/lib/lanes/types.js +1 -1
- package/package.json +1 -1
package/dist/commands/lanes.js
CHANGED
|
@@ -13,13 +13,16 @@ import * as path from "node:path";
|
|
|
13
13
|
import { registerOutpostPrunerCommand } from "./outpost-pruner.js";
|
|
14
14
|
import { ensureCognitoToken } from "../utils/cognito-session.js";
|
|
15
15
|
import { resolveLanesHqRoot } from "../lib/lanes/hq-root.js";
|
|
16
|
+
import { isSsmRunCommandProcess } from "../lib/lanes/ssm-command-context.js";
|
|
16
17
|
import { readLanesRegistryFlag } from "../lib/lanes/registry-flag.js";
|
|
17
18
|
import { resolveProcessFlagGate } from "../lib/flag-registry.js";
|
|
19
|
+
import { createOutpostHireGrantContext, parseOutpostHireGrantContext, sameHireGrantSenior, } from "../lib/lanes/hire-grant-dispatch.js";
|
|
18
20
|
import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
|
|
19
21
|
export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
|
|
20
22
|
export { LANE_HIRE_GRANTS_FLAG };
|
|
21
|
-
import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostMoveTargetCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
23
|
+
import { buildOutpostCapacityCommand, buildOutpostBalanceMoveCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostMoveTargetCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
22
24
|
import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
|
|
25
|
+
import { BALANCE_MOVABLE_STATES, CLAUDE_ACCOUNT_BALANCE_FLAG, accountLoadsForReport, assertBalanceApplyEnabled, planClaudeBalanceMoves, selectClaudeAccount, } from "../lib/lanes/claude-account-balance.js";
|
|
23
26
|
import { parseUserYaml } from "../utils/user-yaml-error.js";
|
|
24
27
|
import { backfillOutpostIdentityMarker as backfillOutpostIdentityMarkerImpl, createOutpostIdentityMarkerIfMissing, validateOutpostIdentityId, } from "../lib/lanes/outpost-identity.js";
|
|
25
28
|
import { vaultApiFetch } from "../utils/vault-api.js";
|
|
@@ -53,6 +56,9 @@ export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
|
|
|
53
56
|
function defaultLanePlacementEnabled() {
|
|
54
57
|
return resolveProcessFlagGate(LANE_OUTPOST_LOAD_BALANCING_FLAG, { fallback: false }, () => false);
|
|
55
58
|
}
|
|
59
|
+
function defaultLaneClaudeAccountBalanceEnabled() {
|
|
60
|
+
return resolveProcessFlagGate(CLAUDE_ACCOUNT_BALANCE_FLAG, { fallback: false }, () => false);
|
|
61
|
+
}
|
|
56
62
|
function defaultLaneHireGrantsEnabled(company) {
|
|
57
63
|
return readLanesRegistryFlag(LANE_HIRE_GRANTS_FLAG, { company });
|
|
58
64
|
}
|
|
@@ -77,7 +83,7 @@ function failJson(err) {
|
|
|
77
83
|
if (err.error === "hire_grant_exhausted") {
|
|
78
84
|
process.stderr.write(`hq lanes: ${err.message}\n`);
|
|
79
85
|
}
|
|
80
|
-
printJson(err.toJSON());
|
|
86
|
+
printJson({ ...err.toJSON(), message: err.message });
|
|
81
87
|
process.exit(1);
|
|
82
88
|
}
|
|
83
89
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -251,6 +257,9 @@ function resolveDeps(deps = {}) {
|
|
|
251
257
|
fetchText: deps.fetchText ?? defaultFetchText,
|
|
252
258
|
lanePlacementEnabled: deps.lanePlacementEnabled ?? defaultLanePlacementEnabled,
|
|
253
259
|
laneHireGrantsEnabled: deps.laneHireGrantsEnabled ?? defaultLaneHireGrantsEnabled,
|
|
260
|
+
isSsmOwnerDispatch: deps.isSsmOwnerDispatch ?? (() => isSsmRunCommandProcess()),
|
|
261
|
+
withGrantHireLocks: deps.withGrantHireLocks ?? withLaneHireLocks,
|
|
262
|
+
laneClaudeAccountBalanceEnabled: deps.laneClaudeAccountBalanceEnabled ?? defaultLaneClaudeAccountBalanceEnabled,
|
|
254
263
|
};
|
|
255
264
|
}
|
|
256
265
|
function livenessDepsFromCtx(ctx) {
|
|
@@ -446,6 +455,81 @@ function laneHireFields(lane, lanes) {
|
|
|
446
455
|
hire_summary: hireSummary,
|
|
447
456
|
};
|
|
448
457
|
}
|
|
458
|
+
function outpostCliOverride(ctx, outpostId) {
|
|
459
|
+
const envKey = `HQ_LANES_OUTPOST_CLI_${outpostId.toUpperCase().replace(/[^A-Z0-9_]/g, "_")}`;
|
|
460
|
+
const configured = ctx.env[envKey];
|
|
461
|
+
const hqBinary = configured === undefined ? undefined : configured.trim();
|
|
462
|
+
if (configured !== undefined &&
|
|
463
|
+
(!hqBinary ||
|
|
464
|
+
!path.posix.isAbsolute(hqBinary) ||
|
|
465
|
+
hqBinary.includes("\0") ||
|
|
466
|
+
hqBinary.includes("\r") ||
|
|
467
|
+
hqBinary.includes("\n"))) {
|
|
468
|
+
throw new LanesError("outpost_lane_cli_invalid", `${envKey} must be an absolute remote executable path`, { outpost_id: outpostId });
|
|
469
|
+
}
|
|
470
|
+
return hqBinary;
|
|
471
|
+
}
|
|
472
|
+
function safeOwnerErrorValue(value, key = "") {
|
|
473
|
+
if (/(?:env|environment|token|secret|password|credential|authorization|api[_-]?key|access[_-]?key|private[_-]?key)/i.test(key)) {
|
|
474
|
+
return undefined;
|
|
475
|
+
}
|
|
476
|
+
if (typeof value === "string") {
|
|
477
|
+
return value
|
|
478
|
+
.replace(/\b(Bearer|Basic)\s+[^\s,;]+/gi, "$1 [redacted]")
|
|
479
|
+
.replace(/\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[A-Z0-9]{16})\b/g, "[redacted]")
|
|
480
|
+
.replace(/\b["']?([A-Z][A-Z0-9_]{1,})["']?\s*[:=]\s*["']?([^\s"',;}]+)["']?/g, "$1=[redacted]");
|
|
481
|
+
}
|
|
482
|
+
if (Array.isArray(value)) {
|
|
483
|
+
return value.map((item) => safeOwnerErrorValue(item));
|
|
484
|
+
}
|
|
485
|
+
if (value === null || typeof value !== "object")
|
|
486
|
+
return value;
|
|
487
|
+
const safe = {};
|
|
488
|
+
for (const [childKey, childValue] of Object.entries(value)) {
|
|
489
|
+
const nested = safeOwnerErrorValue(childValue, childKey);
|
|
490
|
+
if (nested !== undefined)
|
|
491
|
+
safe[childKey] = nested;
|
|
492
|
+
}
|
|
493
|
+
return safe;
|
|
494
|
+
}
|
|
495
|
+
function ownerErrorFromOutput(stdout) {
|
|
496
|
+
const trimmed = stdout.trim();
|
|
497
|
+
if (!trimmed)
|
|
498
|
+
return undefined;
|
|
499
|
+
let parsed;
|
|
500
|
+
try {
|
|
501
|
+
parsed = JSON.parse(trimmed);
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
process.stderr.write("hq lanes: Outpost lanes dispatch returned malformed JSON\n");
|
|
505
|
+
return undefined;
|
|
506
|
+
}
|
|
507
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
508
|
+
return undefined;
|
|
509
|
+
}
|
|
510
|
+
const owner = parsed;
|
|
511
|
+
if (typeof owner.error !== "string" || !/^[a-z][a-z0-9_]{0,99}$/.test(owner.error)) {
|
|
512
|
+
return undefined;
|
|
513
|
+
}
|
|
514
|
+
if (safeOwnerErrorValue(owner.error) !== owner.error)
|
|
515
|
+
return undefined;
|
|
516
|
+
if (owner.error === "already_terminal" || owner.error === "outpost_self_target") {
|
|
517
|
+
return undefined;
|
|
518
|
+
}
|
|
519
|
+
const rawDetails = {};
|
|
520
|
+
for (const [key, value] of Object.entries(owner)) {
|
|
521
|
+
if (key !== "error" && key !== "message" && key !== "details")
|
|
522
|
+
rawDetails[key] = value;
|
|
523
|
+
}
|
|
524
|
+
if (owner.details !== null && typeof owner.details === "object" && !Array.isArray(owner.details)) {
|
|
525
|
+
Object.assign(rawDetails, owner.details);
|
|
526
|
+
}
|
|
527
|
+
const details = safeOwnerErrorValue(rawDetails);
|
|
528
|
+
const message = typeof owner.message === "string" && owner.message.trim()
|
|
529
|
+
? String(safeOwnerErrorValue(owner.message))
|
|
530
|
+
: `Outpost owner returned ${owner.error}`;
|
|
531
|
+
return { error: owner.error, message, details };
|
|
532
|
+
}
|
|
449
533
|
async function ownerLaneHireFields(ctx, lane) {
|
|
450
534
|
if (lane.host.kind !== "outpost") {
|
|
451
535
|
throw new LanesError("outpost_lane_hire_summary_invalid", "Owner summary requires an Outpost-hosted lane", {
|
|
@@ -453,17 +537,7 @@ async function ownerLaneHireFields(ctx, lane) {
|
|
|
453
537
|
});
|
|
454
538
|
}
|
|
455
539
|
const outpostId = lane.host.id;
|
|
456
|
-
const
|
|
457
|
-
const configuredHqBinary = ctx.env[outpostCliEnvKey];
|
|
458
|
-
const hqBinary = configuredHqBinary === undefined ? undefined : configuredHqBinary.trim();
|
|
459
|
-
if (configuredHqBinary !== undefined &&
|
|
460
|
-
(!hqBinary ||
|
|
461
|
-
!path.posix.isAbsolute(hqBinary) ||
|
|
462
|
-
hqBinary.includes("\0") ||
|
|
463
|
-
hqBinary.includes("\r") ||
|
|
464
|
-
hqBinary.includes("\n"))) {
|
|
465
|
-
throw new LanesError("outpost_lane_cli_invalid", `${outpostCliEnvKey} must be an absolute remote executable path`, { outpost_id: outpostId });
|
|
466
|
-
}
|
|
540
|
+
const hqBinary = outpostCliOverride(ctx, outpostId);
|
|
467
541
|
let dispatched;
|
|
468
542
|
try {
|
|
469
543
|
dispatched = await ctx.dispatchOutpostExec({
|
|
@@ -755,6 +829,198 @@ async function loadHostCapacitySnapshots(ctx, hqRoot, config = loadLanesConfig(h
|
|
|
755
829
|
skippedHosts: unavailable.map(({ outpost_id, error }) => ({ outpost_id, error })),
|
|
756
830
|
};
|
|
757
831
|
}
|
|
832
|
+
async function readOutpostLaneRecords(ctx, outpostId) {
|
|
833
|
+
let result;
|
|
834
|
+
try {
|
|
835
|
+
result = await ctx.dispatchOutpostExec({
|
|
836
|
+
outpostId,
|
|
837
|
+
command: buildOutpostLaneListCommand(),
|
|
838
|
+
timeoutSeconds: CAPACITY_PROBE_TIMEOUT_MS / 1000,
|
|
839
|
+
waitMs: CAPACITY_PROBE_TIMEOUT_MS,
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
catch (err) {
|
|
843
|
+
throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
|
|
844
|
+
}
|
|
845
|
+
if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
|
|
846
|
+
throw new LanesError("outpost_lane_list_unavailable", `Outpost ${outpostId} did not return a successful lane list`, { outpost_id: outpostId, status: result.status, done: result.done ?? false, exit_code: result.exitCode });
|
|
847
|
+
}
|
|
848
|
+
let body;
|
|
849
|
+
try {
|
|
850
|
+
body = JSON.parse(result.stdout);
|
|
851
|
+
}
|
|
852
|
+
catch (err) {
|
|
853
|
+
throw new LanesError("outpost_lane_list_invalid", `Outpost ${outpostId} returned invalid lane-list JSON: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
|
|
854
|
+
}
|
|
855
|
+
if (body === null || typeof body !== "object" || Array.isArray(body) ||
|
|
856
|
+
body.ok !== true ||
|
|
857
|
+
body.action !== "list" ||
|
|
858
|
+
!Array.isArray(body.lanes)) {
|
|
859
|
+
throw new LanesError("outpost_lane_list_invalid", `Outpost ${outpostId} returned a malformed lane list`, {
|
|
860
|
+
outpost_id: outpostId,
|
|
861
|
+
});
|
|
862
|
+
}
|
|
863
|
+
const listed = body.lanes;
|
|
864
|
+
const lanes = [];
|
|
865
|
+
for (const [index, value] of listed.entries()) {
|
|
866
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
867
|
+
throw new LanesError("outpost_lane_list_invalid", `Outpost ${outpostId} lane row ${index} is malformed`, {
|
|
868
|
+
outpost_id: outpostId,
|
|
869
|
+
index,
|
|
870
|
+
});
|
|
871
|
+
}
|
|
872
|
+
const raw = value;
|
|
873
|
+
const host = raw.host;
|
|
874
|
+
if (host === null || typeof host !== "object" || Array.isArray(host)) {
|
|
875
|
+
throw new LanesError("outpost_lane_list_invalid", `Outpost ${outpostId} lane row ${index} has no host`, {
|
|
876
|
+
outpost_id: outpostId,
|
|
877
|
+
index,
|
|
878
|
+
});
|
|
879
|
+
}
|
|
880
|
+
const rowHost = host;
|
|
881
|
+
const belongsToOutpost = rowHost.kind === "outpost" && rowHost.id === outpostId;
|
|
882
|
+
// Lanes created before an Outpost received its identity marker retain the
|
|
883
|
+
// real local hostname in their record. The remote list comes from this
|
|
884
|
+
// Outpost's own store, so preserve those local identities for accounting.
|
|
885
|
+
const hasLocalHostIdentity = rowHost.kind === "local" &&
|
|
886
|
+
typeof rowHost.id === "string" && rowHost.id.trim() !== "";
|
|
887
|
+
if (!belongsToOutpost && !hasLocalHostIdentity)
|
|
888
|
+
continue;
|
|
889
|
+
const normalized = raw.state === "remote" && raw.remote_state !== undefined
|
|
890
|
+
? { ...raw, state: raw.remote_state }
|
|
891
|
+
: raw;
|
|
892
|
+
try {
|
|
893
|
+
lanes.push(parseLaneRecord(normalized));
|
|
894
|
+
}
|
|
895
|
+
catch (err) {
|
|
896
|
+
throw new LanesError("outpost_lane_list_invalid", `Outpost ${outpostId} returned an invalid owned lane record: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId, index });
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
return lanes;
|
|
900
|
+
}
|
|
901
|
+
async function loadClaudeBalanceHostSnapshots(ctx, hqRoot, config) {
|
|
902
|
+
const localHost = laneHostFromCtx(ctx);
|
|
903
|
+
const localLanes = loadAllLanes(hqRoot);
|
|
904
|
+
const local = buildHostCapacitySnapshot({
|
|
905
|
+
host: localHost,
|
|
906
|
+
lanes: localLanes,
|
|
907
|
+
config,
|
|
908
|
+
existsSync: ctx.existsSync,
|
|
909
|
+
probe: ctx.probeProvider,
|
|
910
|
+
env: ctx.env,
|
|
911
|
+
now: ctx.now(),
|
|
912
|
+
});
|
|
913
|
+
const snapshots = [{
|
|
914
|
+
capacity: local,
|
|
915
|
+
lanes: localLanes.filter((lane) => lane.host.kind === localHost.kind && lane.host.id === localHost.id),
|
|
916
|
+
laneListAvailable: true,
|
|
917
|
+
}];
|
|
918
|
+
if (localHost.kind === "outpost") {
|
|
919
|
+
const hostname = ctx.hostname().trim();
|
|
920
|
+
const actualLocalHost = { kind: "local", id: hostname };
|
|
921
|
+
const actualLocalLanes = hostname
|
|
922
|
+
? localLanes.filter((lane) => lane.host.kind === "local" && lane.host.id === hostname)
|
|
923
|
+
: [];
|
|
924
|
+
if (actualLocalLanes.length > 0) {
|
|
925
|
+
snapshots.push({
|
|
926
|
+
capacity: buildHostCapacitySnapshot({
|
|
927
|
+
host: actualLocalHost,
|
|
928
|
+
lanes: actualLocalLanes,
|
|
929
|
+
config,
|
|
930
|
+
existsSync: ctx.existsSync,
|
|
931
|
+
probe: ctx.probeProvider,
|
|
932
|
+
env: ctx.env,
|
|
933
|
+
now: ctx.now(),
|
|
934
|
+
}),
|
|
935
|
+
lanes: actualLocalLanes,
|
|
936
|
+
laneListAvailable: true,
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
const unavailableHosts = [];
|
|
941
|
+
let outpostIds;
|
|
942
|
+
try {
|
|
943
|
+
outpostIds = await ctx.listOutpostIds();
|
|
944
|
+
}
|
|
945
|
+
catch (err) {
|
|
946
|
+
unavailableHosts.push({
|
|
947
|
+
host: { kind: "unknown", id: "unknown" },
|
|
948
|
+
reason: `outpost_list_unavailable: ${dispatchErrorMessage(err)}`,
|
|
949
|
+
});
|
|
950
|
+
return { snapshots, unavailableHosts };
|
|
951
|
+
}
|
|
952
|
+
if (outpostIds.length > MAX_PLACEMENT_OUTPOSTS ||
|
|
953
|
+
outpostIds.some((id) => typeof id !== "string" || id.trim() === "") ||
|
|
954
|
+
new Set(outpostIds).size !== outpostIds.length ||
|
|
955
|
+
new Set(outpostIds.map((id) => id.trim())).size !== outpostIds.length) {
|
|
956
|
+
throw new LanesError("outpost_list_invalid", "Outpost list is malformed or exceeds the placement limit", {
|
|
957
|
+
count: outpostIds.length,
|
|
958
|
+
maximum: MAX_PLACEMENT_OUTPOSTS,
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
const remoteIds = outpostIds.map((id) => id.trim())
|
|
962
|
+
.filter((id) => id !== "primary" || (localHost.kind === "outpost" && localHost.id !== "primary"))
|
|
963
|
+
.filter((id) => localHost.kind !== "outpost" || id !== localHost.id)
|
|
964
|
+
.sort();
|
|
965
|
+
const remoteResults = await mapWithConcurrency(remoteIds, OUTPOST_HIRE_SUMMARY_CONCURRENCY, async (id) => {
|
|
966
|
+
let capacity;
|
|
967
|
+
try {
|
|
968
|
+
capacity = await readOutpostCapacity(ctx, id);
|
|
969
|
+
}
|
|
970
|
+
catch (err) {
|
|
971
|
+
if (isLanesError(err) && err.error !== "outpost_capacity_unavailable") {
|
|
972
|
+
throw new LanesError("claude_account_balance_host_invalid", `Outpost ${id} returned an invalid account-capacity snapshot: ${err.message}`, { outpost_id: id, source_error: err.error });
|
|
973
|
+
}
|
|
974
|
+
return { gap: { host: { kind: "outpost", id }, reason: isLanesError(err) ? err.error : "outpost_capacity_unavailable" } };
|
|
975
|
+
}
|
|
976
|
+
let lanes = [];
|
|
977
|
+
let laneListAvailable = false;
|
|
978
|
+
try {
|
|
979
|
+
lanes = await readOutpostLaneRecords(ctx, id);
|
|
980
|
+
laneListAvailable = true;
|
|
981
|
+
}
|
|
982
|
+
catch (err) {
|
|
983
|
+
if (isLanesError(err) && err.error === "outpost_lane_list_invalid")
|
|
984
|
+
throw err;
|
|
985
|
+
unavailableHosts.push({
|
|
986
|
+
host: { kind: "outpost", id },
|
|
987
|
+
reason: isLanesError(err) ? err.error : "outpost_lane_list_unavailable",
|
|
988
|
+
});
|
|
989
|
+
}
|
|
990
|
+
const hostSnapshots = [{
|
|
991
|
+
capacity,
|
|
992
|
+
lanes: laneListAvailable
|
|
993
|
+
? lanes.filter((lane) => lane.host.kind === "outpost" && lane.host.id === id)
|
|
994
|
+
: lanes,
|
|
995
|
+
laneListAvailable,
|
|
996
|
+
}];
|
|
997
|
+
if (laneListAvailable) {
|
|
998
|
+
const localGroups = new Map();
|
|
999
|
+
for (const lane of lanes) {
|
|
1000
|
+
if (lane.host.kind !== "local")
|
|
1001
|
+
continue;
|
|
1002
|
+
const group = localGroups.get(lane.host.id) ?? [];
|
|
1003
|
+
group.push(lane);
|
|
1004
|
+
localGroups.set(lane.host.id, group);
|
|
1005
|
+
}
|
|
1006
|
+
for (const [hostname, localHostLanes] of localGroups) {
|
|
1007
|
+
hostSnapshots.push({
|
|
1008
|
+
capacity: { ...capacity, host: { kind: "local", id: hostname } },
|
|
1009
|
+
lanes: localHostLanes,
|
|
1010
|
+
laneListAvailable: true,
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
return { snapshots: hostSnapshots };
|
|
1015
|
+
});
|
|
1016
|
+
for (const result of remoteResults) {
|
|
1017
|
+
if ("snapshots" in result)
|
|
1018
|
+
snapshots.push(...result.snapshots);
|
|
1019
|
+
else
|
|
1020
|
+
unavailableHosts.push(result.gap);
|
|
1021
|
+
}
|
|
1022
|
+
return { snapshots, unavailableHosts };
|
|
1023
|
+
}
|
|
758
1024
|
function parsePlacementPolicy(raw) {
|
|
759
1025
|
const value = raw.trim();
|
|
760
1026
|
if (value === "local")
|
|
@@ -1955,6 +2221,8 @@ async function runCreate(ctx, opts) {
|
|
|
1955
2221
|
let placementHostOverride;
|
|
1956
2222
|
let autoPlacementChoice;
|
|
1957
2223
|
let autoPlacementSkippedHosts = [];
|
|
2224
|
+
let balanceHostLoad;
|
|
2225
|
+
let balanceTargetHost;
|
|
1958
2226
|
let inheritedPlacement;
|
|
1959
2227
|
const callerLaneId = ctx.env.HQ_LANE_ID?.trim() || undefined;
|
|
1960
2228
|
if (requestedPlacement === undefined &&
|
|
@@ -2059,6 +2327,11 @@ async function runCreate(ctx, opts) {
|
|
|
2059
2327
|
}
|
|
2060
2328
|
let pinnedAccount;
|
|
2061
2329
|
const accountWasExplicit = Boolean(opts.account && opts.account.trim() !== "");
|
|
2330
|
+
const explicitBalanceAccount = accountWasExplicit
|
|
2331
|
+
? opts.account.trim()
|
|
2332
|
+
: profileAccounts === undefined
|
|
2333
|
+
? profileAccount
|
|
2334
|
+
: undefined;
|
|
2062
2335
|
if (autoPlacementChoice !== undefined) {
|
|
2063
2336
|
pinnedAccount = autoPlacementChoice.account;
|
|
2064
2337
|
}
|
|
@@ -2227,6 +2500,24 @@ async function runCreate(ctx, opts) {
|
|
|
2227
2500
|
}
|
|
2228
2501
|
}
|
|
2229
2502
|
const allowedAccounts = validatedAccounts.size > 0 ? [...validatedAccounts] : profileAccounts;
|
|
2503
|
+
let accountSelection;
|
|
2504
|
+
if (provider === "claude" && ctx.laneClaudeAccountBalanceEnabled()) {
|
|
2505
|
+
const targetHost = placementHostOverride ?? (outpostId !== undefined
|
|
2506
|
+
? { kind: "outpost", id: outpostId }
|
|
2507
|
+
: laneHostFromCtx(ctx));
|
|
2508
|
+
balanceTargetHost = targetHost;
|
|
2509
|
+
balanceHostLoad = await loadClaudeBalanceHostSnapshots(ctx, hqRoot, config);
|
|
2510
|
+
const selected = selectClaudeAccount({
|
|
2511
|
+
hosts: balanceHostLoad.snapshots,
|
|
2512
|
+
targetHost,
|
|
2513
|
+
config,
|
|
2514
|
+
...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
|
|
2515
|
+
...(explicitBalanceAccount !== undefined ? { explicitAccount: explicitBalanceAccount } : {}),
|
|
2516
|
+
unavailableHosts: balanceHostLoad.unavailableHosts,
|
|
2517
|
+
});
|
|
2518
|
+
pinnedAccount = selected.account;
|
|
2519
|
+
accountSelection = { ...selected, selected_at: ctx.now().toISOString() };
|
|
2520
|
+
}
|
|
2230
2521
|
const balancedAdmissionAccounts = !outpostId && pinnedAccount === undefined
|
|
2231
2522
|
? [...new Set(allowedAccounts ??
|
|
2232
2523
|
Object.entries(config.providers[provider].accounts)
|
|
@@ -2239,6 +2530,38 @@ async function runCreate(ctx, opts) {
|
|
|
2239
2530
|
const persist = async () => {
|
|
2240
2531
|
let admission;
|
|
2241
2532
|
let validation;
|
|
2533
|
+
if (!outpostId && provider === "claude" && explicitBalanceAccount === undefined &&
|
|
2534
|
+
balanceHostLoad !== undefined && balanceTargetHost !== undefined) {
|
|
2535
|
+
const localHost = laneHostFromCtx(ctx);
|
|
2536
|
+
if (localHost.kind === balanceTargetHost.kind && localHost.id === balanceTargetHost.id) {
|
|
2537
|
+
const localLanes = loadAllLanes(hqRoot).filter((lane) => lane.host.kind === localHost.kind && lane.host.id === localHost.id);
|
|
2538
|
+
const refreshedLocal = {
|
|
2539
|
+
capacity: buildHostCapacitySnapshot({
|
|
2540
|
+
host: localHost,
|
|
2541
|
+
lanes: localLanes,
|
|
2542
|
+
config,
|
|
2543
|
+
existsSync: ctx.existsSync,
|
|
2544
|
+
probe: ctx.probeProvider,
|
|
2545
|
+
env: ctx.env,
|
|
2546
|
+
now: ctx.now(),
|
|
2547
|
+
}),
|
|
2548
|
+
lanes: localLanes,
|
|
2549
|
+
laneListAvailable: true,
|
|
2550
|
+
};
|
|
2551
|
+
const hosts = balanceHostLoad.snapshots.map((host) => host.capacity.host.kind === localHost.kind && host.capacity.host.id === localHost.id
|
|
2552
|
+
? refreshedLocal
|
|
2553
|
+
: host);
|
|
2554
|
+
const selected = selectClaudeAccount({
|
|
2555
|
+
hosts,
|
|
2556
|
+
targetHost: balanceTargetHost,
|
|
2557
|
+
config,
|
|
2558
|
+
...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
|
|
2559
|
+
unavailableHosts: balanceHostLoad.unavailableHosts,
|
|
2560
|
+
});
|
|
2561
|
+
pinnedAccount = selected.account;
|
|
2562
|
+
accountSelection = { ...selected, selected_at: ctx.now().toISOString() };
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2242
2565
|
if (!outpostId && pinnedAccount === undefined) {
|
|
2243
2566
|
pinnedAccount = resolveAccount(provider, undefined, config, {
|
|
2244
2567
|
hqRoot,
|
|
@@ -2337,6 +2660,7 @@ async function runCreate(ctx, opts) {
|
|
|
2337
2660
|
: {}),
|
|
2338
2661
|
provider,
|
|
2339
2662
|
account: pinnedAccount,
|
|
2663
|
+
...(accountSelection !== undefined ? { account_selection: accountSelection } : {}),
|
|
2340
2664
|
...(defaultPlacement !== undefined
|
|
2341
2665
|
? { placement_default: defaultPlacement }
|
|
2342
2666
|
: {}),
|
|
@@ -2528,7 +2852,8 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
2528
2852
|
const staged = await ctx.stageOutpostPayload(outpostId, bytes);
|
|
2529
2853
|
const command = buildOutpostPickupCommand(staged.getUrl);
|
|
2530
2854
|
const dispatched = await ctx.dispatchOutpostExec({ outpostId, command });
|
|
2531
|
-
|
|
2855
|
+
const dispatchedOwnerError = ownerErrorFromOutput(dispatched.stdout);
|
|
2856
|
+
if (preserveQueuedOnFailure && dispatched.done !== true && !dispatchedOwnerError) {
|
|
2532
2857
|
throw new LanesError("outpost_dispatch_failed", "Outpost pickup exec has not finished; its outcome is unknown", {
|
|
2533
2858
|
outpost_id: outpostId,
|
|
2534
2859
|
lane_id: lane.lane_id,
|
|
@@ -2558,6 +2883,40 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
2558
2883
|
await emitLaneTerminalStateSoft(ctx, cancelled);
|
|
2559
2884
|
return cancelled;
|
|
2560
2885
|
}
|
|
2886
|
+
const ownerError = dispatchedOwnerError;
|
|
2887
|
+
if (ownerError) {
|
|
2888
|
+
if (preserveQueuedOnFailure) {
|
|
2889
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected lane pickup: ${ownerError.message}`, {
|
|
2890
|
+
outpost_id: outpostId,
|
|
2891
|
+
lane_id: lane.lane_id,
|
|
2892
|
+
command_id: dispatched.commandId,
|
|
2893
|
+
status: dispatched.status,
|
|
2894
|
+
done: dispatched.done ?? false,
|
|
2895
|
+
exit_code: dispatched.exitCode,
|
|
2896
|
+
...(remote?.pickup_persisted === false ||
|
|
2897
|
+
ownerError.details.pickup_persisted === false
|
|
2898
|
+
? { pickup_persisted: false }
|
|
2899
|
+
: {}),
|
|
2900
|
+
owner_error: ownerError,
|
|
2901
|
+
});
|
|
2902
|
+
}
|
|
2903
|
+
const failed = markDispatchFailure(hqRoot, lane.lane_id, {
|
|
2904
|
+
outpost_id: outpostId,
|
|
2905
|
+
command_id: dispatched.commandId,
|
|
2906
|
+
exit_code: dispatched.exitCode,
|
|
2907
|
+
status: dispatched.status,
|
|
2908
|
+
owner_error: ownerError,
|
|
2909
|
+
});
|
|
2910
|
+
await emitLaneTerminalStateSoft(ctx, failed);
|
|
2911
|
+
throw new LanesError(DISPATCH_TERMINAL_REASON, `Outpost ${outpostId} rejected lane pickup: ${ownerError.message}`, {
|
|
2912
|
+
outpost_id: outpostId,
|
|
2913
|
+
lane_id: failed.lane_id,
|
|
2914
|
+
state: failed.state,
|
|
2915
|
+
terminal_reason: DISPATCH_TERMINAL_REASON,
|
|
2916
|
+
command_id: dispatched.commandId,
|
|
2917
|
+
owner_error: ownerError,
|
|
2918
|
+
});
|
|
2919
|
+
}
|
|
2561
2920
|
if (preserveQueuedOnFailure && remote?.pickup_persisted === false) {
|
|
2562
2921
|
throw new LanesError("outpost_dispatch_failed", "Outpost pickup finished and reported that it did not persist the lane", {
|
|
2563
2922
|
outpost_id: outpostId,
|
|
@@ -2725,19 +3084,24 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
2725
3084
|
throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch to`, { lane_id: lane.lane_id });
|
|
2726
3085
|
}
|
|
2727
3086
|
const outpostId = lane.host.id;
|
|
3087
|
+
const grantContext = action === "grant"
|
|
3088
|
+
? createOutpostHireGrantContext(lane, loadAllLanes(ctx.hqRoot()), await ctx.laneHireGrantsEnabled(lane.company))
|
|
3089
|
+
: undefined;
|
|
2728
3090
|
let dispatched;
|
|
2729
3091
|
try {
|
|
2730
3092
|
dispatched = await ctx.dispatchOutpostExec({
|
|
2731
3093
|
outpostId,
|
|
2732
3094
|
command: buildOutpostLaneCommand({
|
|
2733
|
-
action,
|
|
3095
|
+
action: action === "grant" ? "grant-target" : action,
|
|
2734
3096
|
laneId: lane.lane_id,
|
|
3097
|
+
hqBinary: action === "grant" ? outpostCliOverride(ctx, outpostId) : undefined,
|
|
2735
3098
|
text: options.text,
|
|
2736
3099
|
project: options.project,
|
|
2737
3100
|
story: options.story,
|
|
2738
3101
|
account: options.account,
|
|
2739
3102
|
model: options.model,
|
|
2740
3103
|
hire: options.hire,
|
|
3104
|
+
grantContext,
|
|
2741
3105
|
compact: options.compact,
|
|
2742
3106
|
}),
|
|
2743
3107
|
});
|
|
@@ -2745,6 +3109,18 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
2745
3109
|
catch (err) {
|
|
2746
3110
|
throw new LanesError("outpost_dispatch_failed", `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
|
|
2747
3111
|
}
|
|
3112
|
+
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
3113
|
+
if (ownerError) {
|
|
3114
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected the ${action} command: ${ownerError.message}`, {
|
|
3115
|
+
lane_id: lane.lane_id,
|
|
3116
|
+
outpost_id: outpostId,
|
|
3117
|
+
command_id: dispatched.commandId,
|
|
3118
|
+
status: dispatched.status,
|
|
3119
|
+
done: dispatched.done ?? false,
|
|
3120
|
+
exit_code: dispatched.exitCode,
|
|
3121
|
+
owner_error: ownerError,
|
|
3122
|
+
});
|
|
3123
|
+
}
|
|
2748
3124
|
if (dispatched.done !== true ||
|
|
2749
3125
|
dispatched.status !== "Success" ||
|
|
2750
3126
|
dispatched.exitCode !== 0) {
|
|
@@ -2759,13 +3135,11 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options) {
|
|
|
2759
3135
|
}
|
|
2760
3136
|
const remote = parseRemoteJson(dispatched.stdout, "Outpost lane command");
|
|
2761
3137
|
if (remote?.error !== undefined) {
|
|
2762
|
-
throw new LanesError("
|
|
2763
|
-
? remote.message
|
|
2764
|
-
: `Outpost ${outpostId} rejected the ${action} command`, {
|
|
3138
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected the ${action} command`, {
|
|
2765
3139
|
lane_id: lane.lane_id,
|
|
2766
3140
|
outpost_id: outpostId,
|
|
2767
3141
|
command_id: dispatched.commandId,
|
|
2768
|
-
|
|
3142
|
+
owner_error: ownerErrorFromOutput(dispatched.stdout),
|
|
2769
3143
|
});
|
|
2770
3144
|
}
|
|
2771
3145
|
const resultConfirmed = remote !== null &&
|
|
@@ -3113,6 +3487,8 @@ async function runCapacity(ctx) {
|
|
|
3113
3487
|
lanes: loadAllLanes(hqRoot),
|
|
3114
3488
|
config: loadLanesConfig(hqRoot),
|
|
3115
3489
|
existsSync: ctx.existsSync,
|
|
3490
|
+
probe: ctx.probeProvider,
|
|
3491
|
+
env: ctx.env,
|
|
3116
3492
|
now: ctx.now(),
|
|
3117
3493
|
});
|
|
3118
3494
|
printJson(snapshot);
|
|
@@ -3385,6 +3761,9 @@ async function commitLaneMove(ctx, laneId, target, account) {
|
|
|
3385
3761
|
outcome: "unknown",
|
|
3386
3762
|
...(commandId !== undefined ? { command_id: commandId } : {}),
|
|
3387
3763
|
check_command: checkCommand,
|
|
3764
|
+
...(dispatchDetails.owner_error !== undefined
|
|
3765
|
+
? { owner_error: dispatchDetails.owner_error }
|
|
3766
|
+
: {}),
|
|
3388
3767
|
});
|
|
3389
3768
|
}
|
|
3390
3769
|
let restored = false;
|
|
@@ -3425,6 +3804,9 @@ async function commitLaneMove(ctx, laneId, target, account) {
|
|
|
3425
3804
|
source_host: before.host,
|
|
3426
3805
|
target_host: target,
|
|
3427
3806
|
source_restored: restored,
|
|
3807
|
+
...(dispatchDetails.owner_error !== undefined
|
|
3808
|
+
? { owner_error: dispatchDetails.owner_error }
|
|
3809
|
+
: {}),
|
|
3428
3810
|
...(rollbackError !== undefined
|
|
3429
3811
|
? { rollback_error: dispatchErrorMessage(rollbackError) }
|
|
3430
3812
|
: {}),
|
|
@@ -3454,6 +3836,18 @@ async function dispatchRemoteLaneMove(ctx, source, target, account) {
|
|
|
3454
3836
|
catch (err) {
|
|
3455
3837
|
throw new LanesError("lane_move_dispatch_failed", `Could not ask Outpost ${source.host.id} to move lane ${source.lane_id}: ${dispatchErrorMessage(err)}`, { lane_id: source.lane_id, source_outpost: source.host.id });
|
|
3456
3838
|
}
|
|
3839
|
+
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
3840
|
+
if (ownerError) {
|
|
3841
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${source.host.id} rejected the lane move: ${ownerError.message}`, {
|
|
3842
|
+
lane_id: source.lane_id,
|
|
3843
|
+
source_outpost: source.host.id,
|
|
3844
|
+
command_id: dispatched.commandId,
|
|
3845
|
+
status: dispatched.status,
|
|
3846
|
+
done: dispatched.done ?? false,
|
|
3847
|
+
exit_code: dispatched.exitCode,
|
|
3848
|
+
owner_error: ownerError,
|
|
3849
|
+
});
|
|
3850
|
+
}
|
|
3457
3851
|
if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
|
|
3458
3852
|
throw new LanesError("lane_move_dispatch_failed", `Outpost ${source.host.id} did not confirm the lane move`, {
|
|
3459
3853
|
lane_id: source.lane_id,
|
|
@@ -3573,40 +3967,54 @@ function callerMatchesSenior(ctx, seniorRef) {
|
|
|
3573
3967
|
function forceAdmissionAuthorized(ctx, lane) {
|
|
3574
3968
|
return callerMatchesSenior(ctx, lane.senior);
|
|
3575
3969
|
}
|
|
3576
|
-
async function
|
|
3577
|
-
if (opts.hire === undefined) {
|
|
3578
|
-
throw new LanesError("missing_hire_grant", "--hire is required", {});
|
|
3579
|
-
}
|
|
3580
|
-
const hireGrant = parseHireGrant(opts.hire);
|
|
3970
|
+
async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContext) {
|
|
3581
3971
|
const hqRoot = ctx.hqRoot();
|
|
3582
3972
|
const initial = readLane(hqRoot, laneId);
|
|
3583
|
-
if (!callerMatchesSenior(ctx, initial.senior)) {
|
|
3973
|
+
if (authorization === "senior" && !callerMatchesSenior(ctx, initial.senior)) {
|
|
3584
3974
|
throw new LanesError("hire_grant_unauthorized", `Only senior ${seniorKey(initial.senior)} may change lane ${laneId}'s hire grant`, { lane_id: laneId, senior: initial.senior });
|
|
3585
3975
|
}
|
|
3586
|
-
if (
|
|
3587
|
-
|
|
3588
|
-
|
|
3976
|
+
if (authorization === "ssm-owner-dispatch" &&
|
|
3977
|
+
(initial.host.kind !== "outpost" || !laneIsHostedHere(ctx, initial))) {
|
|
3978
|
+
throw new LanesError("lane_remote", `Outpost grant dispatch cannot mutate lane ${laneId} on this host`, { lane_id: laneId, host: initial.host });
|
|
3979
|
+
}
|
|
3980
|
+
if (authorization === "ssm-owner-dispatch" &&
|
|
3981
|
+
(!grantContext || !sameHireGrantSenior(initial.senior, grantContext.expected_senior))) {
|
|
3982
|
+
throw new LanesError("hire_grant_senior_changed", `Lane ${laneId}'s senior changed after the primary authorized its grant update`, { lane_id: laneId, current_senior: initial.senior });
|
|
3589
3983
|
}
|
|
3590
|
-
const enforce =
|
|
3984
|
+
const enforce = grantContext?.enforcement_enabled ??
|
|
3985
|
+
await ctx.laneHireGrantsEnabled(initial.company);
|
|
3591
3986
|
const lockedLaneIds = [
|
|
3592
3987
|
laneId,
|
|
3593
3988
|
...(initial.senior.kind === "lane" && initial.senior.id
|
|
3594
3989
|
? [initial.senior.id]
|
|
3595
3990
|
: []),
|
|
3596
3991
|
];
|
|
3597
|
-
const updated =
|
|
3992
|
+
const updated = ctx.withGrantHireLocks(hqRoot, lockedLaneIds, () => {
|
|
3598
3993
|
const current = readLane(hqRoot, laneId);
|
|
3599
|
-
if (!callerMatchesSenior(ctx, current.senior)) {
|
|
3994
|
+
if (authorization === "senior" && !callerMatchesSenior(ctx, current.senior)) {
|
|
3600
3995
|
throw new LanesError("hire_grant_unauthorized", `Only senior ${seniorKey(current.senior)} may change lane ${laneId}'s hire grant`, { lane_id: laneId, senior: current.senior });
|
|
3601
3996
|
}
|
|
3602
|
-
|
|
3997
|
+
if (authorization === "ssm-owner-dispatch" &&
|
|
3998
|
+
(current.host.kind !== "outpost" || !laneIsHostedHere(ctx, current))) {
|
|
3999
|
+
throw new LanesError("lane_remote", `Outpost grant dispatch cannot mutate lane ${laneId} on this host`, { lane_id: laneId, host: current.host });
|
|
4000
|
+
}
|
|
4001
|
+
if (authorization === "ssm-owner-dispatch" &&
|
|
4002
|
+
(!grantContext || !sameHireGrantSenior(current.senior, grantContext.expected_senior))) {
|
|
4003
|
+
throw new LanesError("hire_grant_senior_changed", `Lane ${laneId}'s senior changed before the Outpost applied the grant update`, { lane_id: laneId, current_senior: current.senior });
|
|
4004
|
+
}
|
|
4005
|
+
const hierarchy = grantContext?.hierarchy ?? loadAllLanes(hqRoot);
|
|
3603
4006
|
if (enforce && current.senior.kind === "lane") {
|
|
3604
4007
|
if (!current.senior.id) {
|
|
3605
4008
|
throw new LanesError("hire_grant_senior_invalid", `Lane ${laneId} has a lane senior without an id`, {
|
|
3606
4009
|
lane_id: laneId,
|
|
3607
4010
|
});
|
|
3608
4011
|
}
|
|
3609
|
-
const parent =
|
|
4012
|
+
const parent = hierarchy.find((lane) => lane.lane_id === current.senior.id);
|
|
4013
|
+
if (!parent) {
|
|
4014
|
+
throw new LanesError(authorization === "ssm-owner-dispatch"
|
|
4015
|
+
? "hire_grant_context_invalid"
|
|
4016
|
+
: "hire_grant_senior_invalid", `Lane ${laneId}'s senior lane is missing from the grant hierarchy`, { lane_id: laneId, senior_lane_id: current.senior.id });
|
|
4017
|
+
}
|
|
3610
4018
|
if (isTerminalLaneState(parent.state)) {
|
|
3611
4019
|
throw new LanesError("hire_grant_senior_inactive", `Senior lane ${parent.lane_id} is ${parent.state} and cannot change grants`, { senior_lane_id: parent.lane_id, state: parent.state });
|
|
3612
4020
|
}
|
|
@@ -3644,9 +4052,44 @@ async function runGrant(ctx, laneId, opts) {
|
|
|
3644
4052
|
hire_grant: hireGrant,
|
|
3645
4053
|
}));
|
|
3646
4054
|
});
|
|
3647
|
-
|
|
4055
|
+
return updated;
|
|
4056
|
+
}
|
|
4057
|
+
function printGrantResult(hqRoot, updated, hierarchy) {
|
|
4058
|
+
const fields = laneHireFields(updated, hierarchy ?? loadAllLanes(hqRoot));
|
|
3648
4059
|
printJson({ ok: true, action: "grant", lane_id: updated.lane_id, lane: updated, ...fields });
|
|
3649
4060
|
}
|
|
4061
|
+
async function runGrant(ctx, laneId, opts) {
|
|
4062
|
+
if (opts.hire === undefined) {
|
|
4063
|
+
throw new LanesError("missing_hire_grant", "--hire is required", {});
|
|
4064
|
+
}
|
|
4065
|
+
const hireGrant = parseHireGrant(opts.hire);
|
|
4066
|
+
const hqRoot = ctx.hqRoot();
|
|
4067
|
+
const initial = readLane(hqRoot, laneId);
|
|
4068
|
+
if (!callerMatchesSenior(ctx, initial.senior)) {
|
|
4069
|
+
throw new LanesError("hire_grant_unauthorized", `Only senior ${seniorKey(initial.senior)} may change lane ${laneId}'s hire grant`, { lane_id: laneId, senior: initial.senior });
|
|
4070
|
+
}
|
|
4071
|
+
if (initial.host.kind === "outpost" && !laneRunsHere(initial, reconcileHostFromCtx(ctx))) {
|
|
4072
|
+
await dispatchOutpostLaneCommand(ctx, initial, "grant", { hire: String(hireGrant) });
|
|
4073
|
+
return;
|
|
4074
|
+
}
|
|
4075
|
+
const updated = await applyHireGrant(ctx, laneId, hireGrant, "senior");
|
|
4076
|
+
printGrantResult(hqRoot, updated);
|
|
4077
|
+
}
|
|
4078
|
+
async function runGrantOwnerTarget(ctx, laneId, opts) {
|
|
4079
|
+
if (!ctx.isSsmOwnerDispatch()) {
|
|
4080
|
+
throw new LanesError("hire_grant_dispatch_unauthorized", "The Outpost grant target is available only to an SSM Run Command dispatch", { lane_id: laneId });
|
|
4081
|
+
}
|
|
4082
|
+
if (opts.hire === undefined) {
|
|
4083
|
+
throw new LanesError("missing_hire_grant", "--hire is required", {});
|
|
4084
|
+
}
|
|
4085
|
+
if (opts.grantContext === undefined) {
|
|
4086
|
+
throw new LanesError("hire_grant_context_invalid", "--grant-context is required", { lane_id: laneId });
|
|
4087
|
+
}
|
|
4088
|
+
const hireGrant = parseHireGrant(opts.hire);
|
|
4089
|
+
const grantContext = parseOutpostHireGrantContext(opts.grantContext, laneId);
|
|
4090
|
+
const hqRoot = ctx.hqRoot();
|
|
4091
|
+
printGrantResult(hqRoot, await applyHireGrant(ctx, laneId, hireGrant, "ssm-owner-dispatch", grantContext), grantContext.hierarchy);
|
|
4092
|
+
}
|
|
3650
4093
|
async function runBackfillGrants(ctx, opts) {
|
|
3651
4094
|
if (opts.apply && opts.dryRun) {
|
|
3652
4095
|
throw new LanesError("hire_grant_backfill_mode_conflict", "Pass either --dry-run or --apply, not both", {});
|
|
@@ -4062,13 +4505,15 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
4062
4505
|
// (`pending_resume` set by this write, cleared by the claim), so two moves
|
|
4063
4506
|
// into one free slot cannot both start.
|
|
4064
4507
|
return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
|
|
4065
|
-
const admission =
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4508
|
+
const admission = input.deferTurnStart
|
|
4509
|
+
? undefined
|
|
4510
|
+
: await assessLaneAdmission(ctx, lane, {
|
|
4511
|
+
account: target,
|
|
4512
|
+
queueAt: at,
|
|
4513
|
+
});
|
|
4514
|
+
if (admission?.action === "refuse" && admission.error)
|
|
4070
4515
|
throw admission.error;
|
|
4071
|
-
const queued = admission
|
|
4516
|
+
const queued = admission?.action === "queue";
|
|
4072
4517
|
const moved = await withResumeHireAdmission(ctx, lane, () => casUpdateLane(hqRoot, laneId, (current) => {
|
|
4073
4518
|
// The copy and every check above were made from `lane`; a record
|
|
4074
4519
|
// that moved on since (another move, a spawn, a cancel) invalidates
|
|
@@ -4080,6 +4525,7 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
4080
4525
|
const next = {
|
|
4081
4526
|
...current,
|
|
4082
4527
|
account: target,
|
|
4528
|
+
...(input.accountSelection !== undefined ? { account_selection: input.accountSelection } : {}),
|
|
4083
4529
|
admission_accounts: undefined,
|
|
4084
4530
|
provider_session_id: sessionId,
|
|
4085
4531
|
provider_session_id_absent: undefined,
|
|
@@ -4092,11 +4538,16 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
4092
4538
|
at,
|
|
4093
4539
|
reason: SESSION_MOVE_REASON,
|
|
4094
4540
|
}),
|
|
4095
|
-
pending_resume:
|
|
4541
|
+
pending_resume: input.deferTurnStart
|
|
4542
|
+
? undefined
|
|
4543
|
+
: { session_id: sessionId, at, ...input.pending },
|
|
4096
4544
|
};
|
|
4097
|
-
if (
|
|
4545
|
+
if (input.deferTurnStart) {
|
|
4546
|
+
next.state = current.state;
|
|
4547
|
+
}
|
|
4548
|
+
else if (queued) {
|
|
4098
4549
|
next.state = "queued";
|
|
4099
|
-
next.queued_reason = admission
|
|
4550
|
+
next.queued_reason = admission?.reason ?? "throttle";
|
|
4100
4551
|
next.queued_at = at;
|
|
4101
4552
|
}
|
|
4102
4553
|
else {
|
|
@@ -4107,6 +4558,54 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
4107
4558
|
return { lane: moved, from, to: target, copy, queued };
|
|
4108
4559
|
}, ctx.sleep);
|
|
4109
4560
|
}
|
|
4561
|
+
async function selectAutomaticClaudeResumeAccount(ctx, lane) {
|
|
4562
|
+
if (lane.provider !== "claude")
|
|
4563
|
+
return undefined;
|
|
4564
|
+
if (!ctx.laneClaudeAccountBalanceEnabled()) {
|
|
4565
|
+
if (lane.account === undefined)
|
|
4566
|
+
return undefined;
|
|
4567
|
+
const config = loadLanesConfig(ctx.hqRoot());
|
|
4568
|
+
const currentConfig = config.providers.claude.accounts[lane.account];
|
|
4569
|
+
if (currentConfig === undefined || !currentConfig.enabled) {
|
|
4570
|
+
throw new AccountResolutionError(currentConfig === undefined ? "unknown_account" : "account_disabled", currentConfig === undefined
|
|
4571
|
+
? `Lane ${lane.lane_id} records account ${JSON.stringify(lane.account)} which is not configured for claude`
|
|
4572
|
+
: `Account ${JSON.stringify(lane.account)} is unavailable for claude`, { provider: "claude", account: lane.account });
|
|
4573
|
+
}
|
|
4574
|
+
if (accountStatus("claude", lane.account, config, {
|
|
4575
|
+
existsSync: ctx.existsSync,
|
|
4576
|
+
probe: ctx.probeProvider,
|
|
4577
|
+
env: ctx.env,
|
|
4578
|
+
}).logged_in === false) {
|
|
4579
|
+
throw new LanesError("account_not_logged_in", `Account ${lane.account} for claude is not logged in`, { provider: "claude", account: lane.account, lane_id: lane.lane_id });
|
|
4580
|
+
}
|
|
4581
|
+
return undefined;
|
|
4582
|
+
}
|
|
4583
|
+
const hqRoot = ctx.hqRoot();
|
|
4584
|
+
const config = loadLanesConfig(hqRoot);
|
|
4585
|
+
const currentConfig = lane.account === undefined
|
|
4586
|
+
? undefined
|
|
4587
|
+
: config.providers.claude.accounts[lane.account];
|
|
4588
|
+
const currentLogin = lane.account === undefined || currentConfig === undefined
|
|
4589
|
+
? false
|
|
4590
|
+
: accountStatus("claude", lane.account, config, {
|
|
4591
|
+
existsSync: ctx.existsSync,
|
|
4592
|
+
probe: ctx.probeProvider,
|
|
4593
|
+
env: ctx.env,
|
|
4594
|
+
}).logged_in;
|
|
4595
|
+
const accountNeedsRepair = currentConfig?.enabled !== true || currentLogin === false;
|
|
4596
|
+
const mayRebalance = BALANCE_MOVABLE_STATES.includes(lane.state);
|
|
4597
|
+
if (!accountNeedsRepair && !mayRebalance)
|
|
4598
|
+
return undefined;
|
|
4599
|
+
const balanceHosts = await loadClaudeBalanceHostSnapshots(ctx, hqRoot, config);
|
|
4600
|
+
const selected = selectClaudeAccount({
|
|
4601
|
+
hosts: balanceHosts.snapshots,
|
|
4602
|
+
targetHost: lane.host,
|
|
4603
|
+
config,
|
|
4604
|
+
...(lane.admission_accounts !== undefined ? { allowedAccounts: lane.admission_accounts } : {}),
|
|
4605
|
+
unavailableHosts: balanceHosts.unavailableHosts,
|
|
4606
|
+
});
|
|
4607
|
+
return { ...selected, selected_at: ctx.now().toISOString() };
|
|
4608
|
+
}
|
|
4110
4609
|
async function runMessage(ctx, laneId, opts) {
|
|
4111
4610
|
const hqRoot = ctx.hqRoot();
|
|
4112
4611
|
const text = opts.text;
|
|
@@ -4184,6 +4683,31 @@ async function runMessage(ctx, laneId, opts) {
|
|
|
4184
4683
|
archiveResumeMessage(hqRoot, laneId, dropboxMessage);
|
|
4185
4684
|
}
|
|
4186
4685
|
}
|
|
4686
|
+
else if (mechanism === "resume") {
|
|
4687
|
+
const selection = await selectAutomaticClaudeResumeAccount(ctx, lane);
|
|
4688
|
+
if (selection !== undefined) {
|
|
4689
|
+
if (selection.account !== lane.account) {
|
|
4690
|
+
move = await prepareAccountMove(ctx, lane, {
|
|
4691
|
+
account: selection.account,
|
|
4692
|
+
cliModel: undefined,
|
|
4693
|
+
...(resumeProfile !== undefined ? { workerProfile: resumeProfile.profile } : {}),
|
|
4694
|
+
pending: {
|
|
4695
|
+
trigger: "message",
|
|
4696
|
+
text,
|
|
4697
|
+
message_id: messageId,
|
|
4698
|
+
compact: opts.compact !== false,
|
|
4699
|
+
},
|
|
4700
|
+
accountSelection: selection,
|
|
4701
|
+
});
|
|
4702
|
+
lane = move.lane;
|
|
4703
|
+
if (move.queued)
|
|
4704
|
+
archiveResumeMessage(hqRoot, laneId, dropboxMessage);
|
|
4705
|
+
}
|
|
4706
|
+
else {
|
|
4707
|
+
lane = persistClaudeAccountSelection(hqRoot, lane, selection);
|
|
4708
|
+
}
|
|
4709
|
+
}
|
|
4710
|
+
}
|
|
4187
4711
|
if (resumeProfile !== undefined) {
|
|
4188
4712
|
if (opts.account === undefined)
|
|
4189
4713
|
validateResumeProfileModel(ctx, lane, resumeProfile.profile);
|
|
@@ -4357,6 +4881,150 @@ function accountMoveSummary(move) {
|
|
|
4357
4881
|
target_dir: move.copy.target_dir,
|
|
4358
4882
|
};
|
|
4359
4883
|
}
|
|
4884
|
+
function persistClaudeAccountSelection(hqRoot, lane, selection) {
|
|
4885
|
+
return casUpdateLane(hqRoot, lane.lane_id, (current) => {
|
|
4886
|
+
if (current.version !== lane.version || current.state !== lane.state || current.account !== lane.account) {
|
|
4887
|
+
throw new LanesError("claude_account_balance_selection_conflict", `Lane ${lane.lane_id} changed while Claude account balance was being selected`, {
|
|
4888
|
+
lane_id: lane.lane_id,
|
|
4889
|
+
prepared_version: lane.version,
|
|
4890
|
+
version: current.version,
|
|
4891
|
+
state: current.state,
|
|
4892
|
+
account: current.account ?? null,
|
|
4893
|
+
});
|
|
4894
|
+
}
|
|
4895
|
+
return { ...current, account_selection: selection };
|
|
4896
|
+
});
|
|
4897
|
+
}
|
|
4898
|
+
async function applyClaudeBalanceMove(ctx, suggestion) {
|
|
4899
|
+
const currentHost = reconcileHostFromCtx(ctx);
|
|
4900
|
+
const suggestionRunsHere = suggestion.host.kind === "outpost"
|
|
4901
|
+
? currentHost.outpostId === suggestion.host.id
|
|
4902
|
+
: currentHost.outpostId === null && currentHost.hostname === suggestion.host.id;
|
|
4903
|
+
if (suggestion.host.kind === "outpost" && !suggestionRunsHere) {
|
|
4904
|
+
let result;
|
|
4905
|
+
try {
|
|
4906
|
+
result = await ctx.dispatchOutpostExec({
|
|
4907
|
+
outpostId: suggestion.host.id,
|
|
4908
|
+
command: buildOutpostBalanceMoveCommand({
|
|
4909
|
+
laneId: suggestion.lane_id,
|
|
4910
|
+
account: suggestion.to,
|
|
4911
|
+
fromAccount: suggestion.from,
|
|
4912
|
+
expectedState: suggestion.state,
|
|
4913
|
+
}),
|
|
4914
|
+
timeoutSeconds: 120,
|
|
4915
|
+
waitMs: 120_000,
|
|
4916
|
+
});
|
|
4917
|
+
}
|
|
4918
|
+
catch (err) {
|
|
4919
|
+
throw new LanesError("claude_account_balance_move_failed", `Could not move lane ${suggestion.lane_id} on Outpost ${suggestion.host.id}: ${dispatchErrorMessage(err)}`, { lane_id: suggestion.lane_id, host: suggestion.host, account: suggestion.to });
|
|
4920
|
+
}
|
|
4921
|
+
if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
|
|
4922
|
+
throw new LanesError("claude_account_balance_move_failed", `Outpost ${suggestion.host.id} did not confirm the account move for lane ${suggestion.lane_id}`, { lane_id: suggestion.lane_id, host: suggestion.host, account: suggestion.to, status: result.status, exit_code: result.exitCode });
|
|
4923
|
+
}
|
|
4924
|
+
const remote = parseRemoteJson(result.stdout, "Claude account balance move");
|
|
4925
|
+
if (remote?.ok !== true || remote.action !== "balance-move-target" ||
|
|
4926
|
+
remote.lane_id !== suggestion.lane_id || remote.from !== suggestion.from ||
|
|
4927
|
+
remote.account !== suggestion.to ||
|
|
4928
|
+
remote.state !== suggestion.state) {
|
|
4929
|
+
throw new LanesError("claude_account_balance_move_unconfirmed", `Outpost ${suggestion.host.id} returned an invalid account-move result`, {
|
|
4930
|
+
lane_id: suggestion.lane_id,
|
|
4931
|
+
host: suggestion.host,
|
|
4932
|
+
account: suggestion.to,
|
|
4933
|
+
});
|
|
4934
|
+
}
|
|
4935
|
+
return { lane_id: suggestion.lane_id, host: suggestion.host, from: suggestion.from, to: suggestion.to, state: suggestion.state };
|
|
4936
|
+
}
|
|
4937
|
+
const lane = laneWithRecoveredProviderSession(ctx.hqRoot(), readLane(ctx.hqRoot(), suggestion.lane_id));
|
|
4938
|
+
if (lane.state === "running") {
|
|
4939
|
+
throw new LanesError("lane_running", `Lane ${lane.lane_id} is running; balance never moves a running turn`, {
|
|
4940
|
+
lane_id: lane.lane_id,
|
|
4941
|
+
state: lane.state,
|
|
4942
|
+
});
|
|
4943
|
+
}
|
|
4944
|
+
if (lane.host.kind !== suggestion.host.kind || lane.host.id !== suggestion.host.id ||
|
|
4945
|
+
lane.account !== suggestion.from || lane.state !== suggestion.state) {
|
|
4946
|
+
throw new LanesError("claude_account_balance_lane_changed", `Lane ${lane.lane_id} changed after balance was calculated`, {
|
|
4947
|
+
lane_id: lane.lane_id,
|
|
4948
|
+
host: lane.host,
|
|
4949
|
+
account: lane.account ?? null,
|
|
4950
|
+
});
|
|
4951
|
+
}
|
|
4952
|
+
if (!BALANCE_MOVABLE_STATES.includes(lane.state)) {
|
|
4953
|
+
throw new LanesError("claude_account_balance_state_not_movable", `Lane ${lane.lane_id} is no longer between turns`, {
|
|
4954
|
+
lane_id: lane.lane_id,
|
|
4955
|
+
state: lane.state,
|
|
4956
|
+
});
|
|
4957
|
+
}
|
|
4958
|
+
const moved = await prepareAccountMove(ctx, lane, {
|
|
4959
|
+
account: suggestion.to,
|
|
4960
|
+
cliModel: undefined,
|
|
4961
|
+
deferTurnStart: true,
|
|
4962
|
+
});
|
|
4963
|
+
return { lane_id: moved.lane.lane_id, host: moved.lane.host, from: moved.from ?? suggestion.from, to: moved.to, state: moved.lane.state };
|
|
4964
|
+
}
|
|
4965
|
+
async function runClaudeBalanceMoveTarget(ctx, laneId, opts) {
|
|
4966
|
+
assertBalanceApplyEnabled(ctx.laneClaudeAccountBalanceEnabled());
|
|
4967
|
+
const lane = laneWithRecoveredProviderSession(ctx.hqRoot(), readLane(ctx.hqRoot(), laneId));
|
|
4968
|
+
assertLaneRunsHereForOperatorMutation(ctx, lane, "balance-move-target");
|
|
4969
|
+
if (lane.provider !== "claude") {
|
|
4970
|
+
throw new LanesError("claude_account_balance_provider_mismatch", `Lane ${laneId} does not use Claude`, {
|
|
4971
|
+
lane_id: laneId,
|
|
4972
|
+
provider: lane.provider,
|
|
4973
|
+
});
|
|
4974
|
+
}
|
|
4975
|
+
const expectedFrom = opts.fromAccount?.trim();
|
|
4976
|
+
if (!expectedFrom) {
|
|
4977
|
+
throw new LanesError("claude_account_balance_source_invalid", "Balance move requires --from-account <id>", {
|
|
4978
|
+
lane_id: laneId,
|
|
4979
|
+
});
|
|
4980
|
+
}
|
|
4981
|
+
if (lane.account !== expectedFrom) {
|
|
4982
|
+
throw new LanesError("claude_account_balance_source_changed", `Lane ${laneId} no longer uses the planned source account`, {
|
|
4983
|
+
lane_id: laneId,
|
|
4984
|
+
expected_account: expectedFrom,
|
|
4985
|
+
actual_account: lane.account ?? null,
|
|
4986
|
+
});
|
|
4987
|
+
}
|
|
4988
|
+
const expectedState = opts.expectedState?.trim();
|
|
4989
|
+
if (!expectedState) {
|
|
4990
|
+
throw new LanesError("claude_account_balance_state_invalid", "Balance move requires --expected-state <state>", {
|
|
4991
|
+
lane_id: laneId,
|
|
4992
|
+
});
|
|
4993
|
+
}
|
|
4994
|
+
if (lane.state !== expectedState) {
|
|
4995
|
+
throw new LanesError("claude_account_balance_state_changed", `Lane ${laneId} changed state after balance was calculated`, {
|
|
4996
|
+
lane_id: laneId,
|
|
4997
|
+
expected_state: expectedState,
|
|
4998
|
+
actual_state: lane.state,
|
|
4999
|
+
});
|
|
5000
|
+
}
|
|
5001
|
+
if (!BALANCE_MOVABLE_STATES.includes(lane.state) || lane.pending_resume !== undefined) {
|
|
5002
|
+
throw new LanesError("claude_account_balance_state_not_movable", `Lane ${laneId} is not between turns`, {
|
|
5003
|
+
lane_id: laneId,
|
|
5004
|
+
state: lane.state,
|
|
5005
|
+
pending_resume: lane.pending_resume !== undefined,
|
|
5006
|
+
});
|
|
5007
|
+
}
|
|
5008
|
+
const account = opts.account?.trim();
|
|
5009
|
+
if (!account) {
|
|
5010
|
+
throw new LanesError("claude_account_balance_account_invalid", "Balance move requires --account <id>", {
|
|
5011
|
+
lane_id: laneId,
|
|
5012
|
+
});
|
|
5013
|
+
}
|
|
5014
|
+
const moved = await prepareAccountMove(ctx, lane, {
|
|
5015
|
+
account,
|
|
5016
|
+
cliModel: undefined,
|
|
5017
|
+
deferTurnStart: true,
|
|
5018
|
+
});
|
|
5019
|
+
printJson({
|
|
5020
|
+
ok: true,
|
|
5021
|
+
action: "balance-move-target",
|
|
5022
|
+
lane_id: laneId,
|
|
5023
|
+
from: moved.from,
|
|
5024
|
+
state: moved.lane.state,
|
|
5025
|
+
account: moved.lane.account,
|
|
5026
|
+
});
|
|
5027
|
+
}
|
|
4360
5028
|
function laneWithRecoveredProviderSession(hqRoot, lane) {
|
|
4361
5029
|
if (lane.provider === "grok")
|
|
4362
5030
|
return lane;
|
|
@@ -4456,6 +5124,9 @@ async function runResume(ctx, laneId, opts) {
|
|
|
4456
5124
|
if (resumeProfile !== undefined && opts.account === undefined) {
|
|
4457
5125
|
validateResumeProfileModel(ctx, lane, resumeProfile.profile);
|
|
4458
5126
|
}
|
|
5127
|
+
const automaticAccountSelection = opts.account === undefined
|
|
5128
|
+
? await selectAutomaticClaudeResumeAccount(ctx, lane)
|
|
5129
|
+
: undefined;
|
|
4459
5130
|
let move = null;
|
|
4460
5131
|
if (opts.account !== undefined) {
|
|
4461
5132
|
move = await prepareAccountMove(ctx, lane, {
|
|
@@ -4472,6 +5143,27 @@ async function runResume(ctx, laneId, opts) {
|
|
|
4472
5143
|
});
|
|
4473
5144
|
lane = move.lane;
|
|
4474
5145
|
}
|
|
5146
|
+
else if (automaticAccountSelection !== undefined) {
|
|
5147
|
+
if (automaticAccountSelection.account !== lane.account) {
|
|
5148
|
+
move = await prepareAccountMove(ctx, lane, {
|
|
5149
|
+
account: automaticAccountSelection.account,
|
|
5150
|
+
cliModel: undefined,
|
|
5151
|
+
...(resumeProfile !== undefined ? { workerProfile: resumeProfile.profile } : {}),
|
|
5152
|
+
pending: {
|
|
5153
|
+
trigger: "resume",
|
|
5154
|
+
resumed_from: resumedFrom,
|
|
5155
|
+
...(text !== undefined ? { text } : {}),
|
|
5156
|
+
...(messageId !== undefined ? { message_id: messageId } : {}),
|
|
5157
|
+
compact: opts.compact !== false,
|
|
5158
|
+
},
|
|
5159
|
+
accountSelection: automaticAccountSelection,
|
|
5160
|
+
});
|
|
5161
|
+
lane = move.lane;
|
|
5162
|
+
}
|
|
5163
|
+
else {
|
|
5164
|
+
lane = persistClaudeAccountSelection(hqRoot, lane, automaticAccountSelection);
|
|
5165
|
+
}
|
|
5166
|
+
}
|
|
4475
5167
|
lane = recordResumeSetup(ctx, lane, {
|
|
4476
5168
|
...(resumeProfile !== undefined ? { profile: resumeProfile } : {}),
|
|
4477
5169
|
...(requestedMeshCompany !== undefined
|
|
@@ -5212,6 +5904,16 @@ async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId) {
|
|
|
5212
5904
|
!commandId ||
|
|
5213
5905
|
dispatched.status !== "Success" ||
|
|
5214
5906
|
dispatched.exitCode !== 0) {
|
|
5907
|
+
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
5908
|
+
if (ownerError) {
|
|
5909
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected the lane read after stop: ${ownerError.message}`, {
|
|
5910
|
+
lane_id: laneId,
|
|
5911
|
+
outpost_id: outpostId,
|
|
5912
|
+
command_id: stopCommandId,
|
|
5913
|
+
sync_command_id: commandId || null,
|
|
5914
|
+
owner_error: ownerError,
|
|
5915
|
+
});
|
|
5916
|
+
}
|
|
5215
5917
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not confirm a lane read after stop ${laneId}`, {
|
|
5216
5918
|
lane_id: laneId,
|
|
5217
5919
|
outpost_id: outpostId,
|
|
@@ -5235,6 +5937,16 @@ async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId) {
|
|
|
5235
5937
|
cause: err instanceof Error ? err.message : String(err),
|
|
5236
5938
|
});
|
|
5237
5939
|
}
|
|
5940
|
+
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
5941
|
+
if (ownerError) {
|
|
5942
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected the lane read after stop: ${ownerError.message}`, {
|
|
5943
|
+
lane_id: laneId,
|
|
5944
|
+
outpost_id: outpostId,
|
|
5945
|
+
command_id: stopCommandId,
|
|
5946
|
+
sync_command_id: commandId,
|
|
5947
|
+
owner_error: ownerError,
|
|
5948
|
+
});
|
|
5949
|
+
}
|
|
5238
5950
|
if (remote?.ok !== true || remote.action !== "show") {
|
|
5239
5951
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not return lane ${laneId} after stop`, {
|
|
5240
5952
|
lane_id: laneId,
|
|
@@ -5265,6 +5977,18 @@ async function dispatchOutpostLaneStop(ctx, lane) {
|
|
|
5265
5977
|
catch (err) {
|
|
5266
5978
|
throw new LanesError("outpost_dispatch_failed", `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
|
|
5267
5979
|
}
|
|
5980
|
+
const ownerError = ownerErrorFromOutput(dispatched.stdout);
|
|
5981
|
+
if (ownerError) {
|
|
5982
|
+
throw new LanesError("outpost_dispatch_failed", `Outpost ${outpostId} rejected the stop command: ${ownerError.message}`, {
|
|
5983
|
+
lane_id: lane.lane_id,
|
|
5984
|
+
outpost_id: outpostId,
|
|
5985
|
+
command_id: dispatched.commandId,
|
|
5986
|
+
status: dispatched.status,
|
|
5987
|
+
done: dispatched.done ?? false,
|
|
5988
|
+
exit_code: dispatched.exitCode,
|
|
5989
|
+
owner_error: ownerError,
|
|
5990
|
+
});
|
|
5991
|
+
}
|
|
5268
5992
|
const commandId = dispatched.commandId?.trim();
|
|
5269
5993
|
if (dispatched.done !== true || !commandId) {
|
|
5270
5994
|
throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} did not return a completed stop command for lane ${lane.lane_id}`, {
|
|
@@ -5308,13 +6032,13 @@ async function dispatchOutpostLaneStop(ctx, lane) {
|
|
|
5308
6032
|
syncCommandId = owner.commandId;
|
|
5309
6033
|
}
|
|
5310
6034
|
else if (remote?.error !== undefined) {
|
|
5311
|
-
throw new LanesError("
|
|
6035
|
+
throw new LanesError("outpost_dispatch_failed", typeof remote.message === "string"
|
|
5312
6036
|
? remote.message
|
|
5313
6037
|
: `Outpost ${outpostId} rejected the stop command for lane ${lane.lane_id}`, {
|
|
5314
6038
|
lane_id: lane.lane_id,
|
|
5315
6039
|
outpost_id: outpostId,
|
|
5316
6040
|
command_id: commandId,
|
|
5317
|
-
|
|
6041
|
+
owner_error: ownerErrorFromOutput(dispatched.stdout),
|
|
5318
6042
|
});
|
|
5319
6043
|
}
|
|
5320
6044
|
else {
|
|
@@ -5750,6 +6474,105 @@ async function runAccountsList(ctx, opts) {
|
|
|
5750
6474
|
accounts,
|
|
5751
6475
|
});
|
|
5752
6476
|
}
|
|
6477
|
+
async function runAccountsBalance(ctx, opts) {
|
|
6478
|
+
const provider = parseLaneProvider(opts.provider ?? "");
|
|
6479
|
+
if (provider !== "claude") {
|
|
6480
|
+
throw new LanesError("claude_account_balance_provider_unsupported", "Account balance is available only for --provider claude", {
|
|
6481
|
+
provider,
|
|
6482
|
+
accepted: ["claude"],
|
|
6483
|
+
});
|
|
6484
|
+
}
|
|
6485
|
+
if (opts.apply)
|
|
6486
|
+
assertBalanceApplyEnabled(ctx.laneClaudeAccountBalanceEnabled());
|
|
6487
|
+
const hqRoot = ctx.hqRoot();
|
|
6488
|
+
const config = loadLanesConfig(hqRoot);
|
|
6489
|
+
const loaded = await loadClaudeBalanceHostSnapshots(ctx, hqRoot, config);
|
|
6490
|
+
const loads = accountLoadsForReport({ hosts: loaded.snapshots, config });
|
|
6491
|
+
const suggestions = planClaudeBalanceMoves({ hosts: loaded.snapshots, config });
|
|
6492
|
+
const selections = loaded.snapshots.map((host) => {
|
|
6493
|
+
try {
|
|
6494
|
+
const selection = selectClaudeAccount({
|
|
6495
|
+
hosts: loaded.snapshots,
|
|
6496
|
+
targetHost: host.capacity.host,
|
|
6497
|
+
config,
|
|
6498
|
+
unavailableHosts: loaded.unavailableHosts,
|
|
6499
|
+
});
|
|
6500
|
+
return {
|
|
6501
|
+
target_host: host.capacity.host,
|
|
6502
|
+
account: selection.account,
|
|
6503
|
+
signal: selection.signal,
|
|
6504
|
+
skipped_accounts: selection.skipped_accounts,
|
|
6505
|
+
};
|
|
6506
|
+
}
|
|
6507
|
+
catch (err) {
|
|
6508
|
+
return {
|
|
6509
|
+
target_host: host.capacity.host,
|
|
6510
|
+
error: isLanesError(err) ? err.error : "claude_account_balance_selection_failed",
|
|
6511
|
+
message: dispatchErrorMessage(err),
|
|
6512
|
+
};
|
|
6513
|
+
}
|
|
6514
|
+
});
|
|
6515
|
+
const accounts = Object.keys(config.providers.claude.accounts).sort().map((account) => {
|
|
6516
|
+
const load = loads.find((row) => row.account === account);
|
|
6517
|
+
return {
|
|
6518
|
+
account,
|
|
6519
|
+
enabled: config.providers.claude.accounts[account]?.enabled === true,
|
|
6520
|
+
live_lanes: load?.live_lanes ?? 0,
|
|
6521
|
+
counts_complete: load?.counts_complete ?? false,
|
|
6522
|
+
hosts: loaded.snapshots.map((host) => {
|
|
6523
|
+
const accountStatus = host.capacity.providers.claude.accounts[account];
|
|
6524
|
+
const accountLoad = load?.hosts.find((row) => row.host.kind === host.capacity.host.kind && row.host.id === host.capacity.host.id);
|
|
6525
|
+
return {
|
|
6526
|
+
host: host.capacity.host,
|
|
6527
|
+
enabled: accountStatus?.enabled ?? false,
|
|
6528
|
+
logged_in: accountStatus?.logged_in ?? null,
|
|
6529
|
+
live_lanes: accountLoad?.live_lanes ?? null,
|
|
6530
|
+
lane_counts: accountLoad?.lane_counts ?? { running: null, queued: null, awaiting_input: null },
|
|
6531
|
+
count_source: accountLoad?.count_source ?? "unavailable",
|
|
6532
|
+
usage: accountStatus?.usage ?? null,
|
|
6533
|
+
};
|
|
6534
|
+
}),
|
|
6535
|
+
};
|
|
6536
|
+
});
|
|
6537
|
+
const moved = [];
|
|
6538
|
+
const skipped = [];
|
|
6539
|
+
if (opts.apply) {
|
|
6540
|
+
for (const suggestion of suggestions) {
|
|
6541
|
+
if (!suggestion.apply) {
|
|
6542
|
+
skipped.push({
|
|
6543
|
+
lane_id: suggestion.lane_id,
|
|
6544
|
+
host: suggestion.host,
|
|
6545
|
+
from: suggestion.from,
|
|
6546
|
+
to: suggestion.to,
|
|
6547
|
+
reason: suggestion.skip_reason ?? "not_movable",
|
|
6548
|
+
});
|
|
6549
|
+
continue;
|
|
6550
|
+
}
|
|
6551
|
+
try {
|
|
6552
|
+
moved.push(await applyClaudeBalanceMove(ctx, suggestion));
|
|
6553
|
+
}
|
|
6554
|
+
catch (err) {
|
|
6555
|
+
skipped.push({
|
|
6556
|
+
lane_id: suggestion.lane_id,
|
|
6557
|
+
host: suggestion.host,
|
|
6558
|
+
from: suggestion.from,
|
|
6559
|
+
to: suggestion.to,
|
|
6560
|
+
reason: isLanesError(err) ? `${err.error}: ${err.message}` : dispatchErrorMessage(err),
|
|
6561
|
+
});
|
|
6562
|
+
}
|
|
6563
|
+
}
|
|
6564
|
+
}
|
|
6565
|
+
printJson({
|
|
6566
|
+
ok: true,
|
|
6567
|
+
action: "accounts.balance",
|
|
6568
|
+
provider: "claude",
|
|
6569
|
+
accounts,
|
|
6570
|
+
selections,
|
|
6571
|
+
unavailable_hosts: loaded.unavailableHosts,
|
|
6572
|
+
suggestions,
|
|
6573
|
+
...(opts.apply ? { moved, skipped } : {}),
|
|
6574
|
+
});
|
|
6575
|
+
}
|
|
5753
6576
|
async function runAccountsLogin(ctx, providerRaw, accountId, opts) {
|
|
5754
6577
|
const callerCwd = ctx.callerCwd();
|
|
5755
6578
|
const hqRoot = ctx.hqRoot();
|
|
@@ -6056,6 +6879,14 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
6056
6879
|
.action(wrap(async (opts) => {
|
|
6057
6880
|
await runCreate(ctx, opts);
|
|
6058
6881
|
}));
|
|
6882
|
+
lanes
|
|
6883
|
+
.command("grant-target", { hidden: true })
|
|
6884
|
+
.argument("<lane-id>", "Lane id")
|
|
6885
|
+
.option("--hire <n>", "Non-negative safe integer hire grant")
|
|
6886
|
+
.option("--grant-context <base64url>", "Primary-authorized hire hierarchy and gate decision")
|
|
6887
|
+
.action(wrap(async (laneId, opts) => {
|
|
6888
|
+
await runGrantOwnerTarget(ctx, laneId, opts);
|
|
6889
|
+
}));
|
|
6059
6890
|
lanes
|
|
6060
6891
|
.command("grant")
|
|
6061
6892
|
.description("Set a lane's alive child-lane grant as its senior")
|
|
@@ -6123,6 +6954,15 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
6123
6954
|
.action(wrap(async (laneId, opts) => {
|
|
6124
6955
|
await runMoveTarget(ctx, laneId, opts);
|
|
6125
6956
|
}));
|
|
6957
|
+
lanes
|
|
6958
|
+
.command("balance-move-target", { hidden: true })
|
|
6959
|
+
.argument("<lane-id>", "Lane id")
|
|
6960
|
+
.requiredOption("--account <id>", "Target Claude account id")
|
|
6961
|
+
.requiredOption("--from-account <id>", "Expected source Claude account id")
|
|
6962
|
+
.requiredOption("--expected-state <state>", "Expected lane state from the balance plan")
|
|
6963
|
+
.action(wrap(async (laneId, opts) => {
|
|
6964
|
+
await runClaudeBalanceMoveTarget(ctx, laneId, opts);
|
|
6965
|
+
}));
|
|
6126
6966
|
lanes
|
|
6127
6967
|
.command("admit")
|
|
6128
6968
|
.description("Try to admit the oldest eligible queued lane")
|
|
@@ -6321,6 +7161,15 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
6321
7161
|
.action(wrap(async (opts) => {
|
|
6322
7162
|
await runAccountsList(ctx, opts);
|
|
6323
7163
|
}));
|
|
7164
|
+
accounts
|
|
7165
|
+
.command("balance")
|
|
7166
|
+
.description("Report Claude account load and suggest safe between-turn moves")
|
|
7167
|
+
.requiredOption("--provider <name>", "claude")
|
|
7168
|
+
.option("--apply", "Move eligible between-turn sessions (requires the Claude account balance gate)")
|
|
7169
|
+
.option("--json", "JSON on stdout (always)")
|
|
7170
|
+
.action(wrap(async (opts) => {
|
|
7171
|
+
await runAccountsBalance(ctx, opts);
|
|
7172
|
+
}));
|
|
6324
7173
|
accounts
|
|
6325
7174
|
.command("login")
|
|
6326
7175
|
.description("Create a home, run the vendor login, and register the account")
|