@indigoai-us/hq-cli 5.157.0 → 5.158.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,10 +12,13 @@ import * as os from "node:os";
12
12
  import * as path from "node:path";
13
13
  import { ensureCognitoToken } from "../utils/cognito-session.js";
14
14
  import { resolveLanesHqRoot } from "../lib/lanes/hq-root.js";
15
- import { readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
15
+ import { resolveProcessFlagGate } from "../lib/flag-registry.js";
16
+ import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
16
17
  export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
17
- import { buildOutpostLaneCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
18
+ import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostMoveTargetCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
19
+ import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
18
20
  import { parseUserYaml } from "../utils/user-yaml-error.js";
21
+ import { backfillOutpostIdentityMarker as backfillOutpostIdentityMarkerImpl, createOutpostIdentityMarkerIfMissing, validateOutpostIdentityId, } from "../lib/lanes/outpost-identity.js";
19
22
  import { vaultApiFetch } from "../utils/vault-api.js";
20
23
  import { openMeshTransport } from "../lib/mesh/client.js";
21
24
  import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshEventIdentity, } from "../lib/lanes/mesh-events.js";
@@ -42,6 +45,10 @@ const VENDOR_LOGIN = {
42
45
  grok: { command: "grok", args: ["login"] },
43
46
  };
44
47
  export { emitLaneDecisionEvents };
48
+ export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
49
+ function defaultLanePlacementEnabled() {
50
+ return resolveProcessFlagGate(LANE_OUTPOST_LOAD_BALANCING_FLAG, { fallback: false }, () => false);
51
+ }
45
52
  function withoutProviderHome(value) {
46
53
  if (Array.isArray(value))
47
54
  return value.map(withoutProviderHome);
@@ -227,7 +234,12 @@ function resolveDeps(deps = {}) {
227
234
  })),
228
235
  stageOutpostPayload: deps.stageOutpostPayload ?? defaultStageOutpostPayload,
229
236
  dispatchOutpostExec: deps.dispatchOutpostExec ?? defaultDispatchOutpostExec,
237
+ backfillOutpostIdentity: deps.backfillOutpostIdentity ?? ((expectedId) => backfillOutpostIdentityMarkerImpl(expectedId, {
238
+ read: () => deps.readOutpostIdentity?.() ?? readOutpostIdentity(),
239
+ createIfMissing: createOutpostIdentityMarkerIfMissing,
240
+ })),
230
241
  fetchText: deps.fetchText ?? defaultFetchText,
242
+ lanePlacementEnabled: deps.lanePlacementEnabled ?? defaultLanePlacementEnabled,
231
243
  };
232
244
  }
233
245
  function livenessDepsFromCtx(ctx) {
@@ -402,6 +414,150 @@ function requireQuestionBody(hqRoot, question) {
402
414
  function loadAllLanes(hqRoot) {
403
415
  return listLaneIds(hqRoot).map((id) => readLane(hqRoot, id));
404
416
  }
417
+ const CAPACITY_PROBE_TIMEOUT_MS = 15_000;
418
+ const MAX_PLACEMENT_OUTPOSTS = 16;
419
+ function outpostIdentityUnknownError(message, expectedId, details = {}) {
420
+ const expected = expectedId ?? "<expected-outpost-id>";
421
+ const command = `hq lanes outpost-identity backfill --id ${expected}`;
422
+ return new LanesError("outpost_identity_unknown", message, {
423
+ ...details,
424
+ ...(expectedId !== undefined ? { expected_id: expectedId } : {}),
425
+ marker_path: OUTPOST_ID_PATH,
426
+ fix_command: command,
427
+ fix: `Run ${command} from a registered host, or write ${JSON.stringify(`${expected}\n`)} to ${OUTPOST_ID_PATH}.`,
428
+ });
429
+ }
430
+ function laneHostFromCtx(ctx, expectedId) {
431
+ const identity = ctx.readOutpostIdentity();
432
+ if (identity.kind === "unknown") {
433
+ throw outpostIdentityUnknownError("Cannot resolve this host for lane placement", expectedId);
434
+ }
435
+ if (identity.kind === "id")
436
+ return { kind: "outpost", id: identity.id };
437
+ const hostname = ctx.hostname().trim();
438
+ if (!hostname) {
439
+ throw new LanesError("host_identity_unknown", "Cannot resolve this host for lane placement", {});
440
+ }
441
+ return { kind: "local", id: hostname };
442
+ }
443
+ async function readOutpostCapacity(ctx, outpostId) {
444
+ let result;
445
+ try {
446
+ result = await ctx.dispatchOutpostExec({
447
+ outpostId,
448
+ command: buildOutpostCapacityCommand(),
449
+ timeoutSeconds: CAPACITY_PROBE_TIMEOUT_MS / 1000,
450
+ waitMs: CAPACITY_PROBE_TIMEOUT_MS,
451
+ });
452
+ }
453
+ catch (err) {
454
+ throw new LanesError("outpost_capacity_unavailable", `Could not read capacity from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
455
+ }
456
+ if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
457
+ throw new LanesError("outpost_capacity_unavailable", `Outpost ${outpostId} did not return a successful capacity snapshot`, {
458
+ outpost_id: outpostId,
459
+ status: result.status,
460
+ done: result.done ?? false,
461
+ exit_code: result.exitCode,
462
+ });
463
+ }
464
+ let raw;
465
+ try {
466
+ raw = JSON.parse(result.stdout);
467
+ }
468
+ catch (err) {
469
+ throw new LanesError("outpost_capacity_unavailable", `Outpost ${outpostId} returned invalid capacity JSON: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
470
+ }
471
+ const snapshot = parseHostCapacitySnapshot(raw);
472
+ if (snapshot.host.kind !== "outpost" || snapshot.host.id !== outpostId) {
473
+ throw new LanesError("outpost_capacity_identity_mismatch", `Capacity response from Outpost ${outpostId} identified a different host`, { requested_outpost_id: outpostId, reported_host: snapshot.host });
474
+ }
475
+ return snapshot;
476
+ }
477
+ async function loadHostCapacitySnapshots(ctx, hqRoot, config = loadLanesConfig(hqRoot)) {
478
+ const localHost = laneHostFromCtx(ctx);
479
+ let outpostIds;
480
+ try {
481
+ outpostIds = await ctx.listOutpostIds();
482
+ }
483
+ catch (err) {
484
+ throw new LanesError("outpost_list_unavailable", `Failed to list Outpost ids: ${dispatchErrorMessage(err)}`, {});
485
+ }
486
+ if (outpostIds.length > MAX_PLACEMENT_OUTPOSTS ||
487
+ outpostIds.some((id) => typeof id !== "string" || id.trim() === "") ||
488
+ new Set(outpostIds).size !== outpostIds.length) {
489
+ throw new LanesError("outpost_list_invalid", "Outpost list is malformed or exceeds the placement limit", {
490
+ count: outpostIds.length,
491
+ maximum: MAX_PLACEMENT_OUTPOSTS,
492
+ });
493
+ }
494
+ const remoteIds = outpostIds
495
+ .map((id) => id.trim())
496
+ // `primary` aliases the controller when running on the primary box. From
497
+ // another Outpost it identifies a separate host and must be measured.
498
+ .filter((id) => id !== "primary" ||
499
+ (localHost.kind === "outpost" && localHost.id !== "primary"))
500
+ .filter((id) => localHost.kind !== "outpost" || id !== localHost.id)
501
+ .sort();
502
+ const local = buildHostCapacitySnapshot({
503
+ host: localHost,
504
+ lanes: loadAllLanes(hqRoot),
505
+ config,
506
+ existsSync: ctx.existsSync,
507
+ now: ctx.now(),
508
+ });
509
+ const remote = await Promise.all(remoteIds.map(async (id) => {
510
+ try {
511
+ return {
512
+ ok: true,
513
+ snapshot: await readOutpostCapacity(ctx, id),
514
+ };
515
+ }
516
+ catch (err) {
517
+ const error = isLanesError(err) ? err.error : "outpost_capacity_unavailable";
518
+ return {
519
+ ok: false,
520
+ unavailable: {
521
+ host: { kind: "outpost", id },
522
+ outpost_id: id,
523
+ error,
524
+ },
525
+ };
526
+ }
527
+ }));
528
+ const snapshots = [
529
+ local,
530
+ ...remote.flatMap((result) => result.ok ? [result.snapshot] : []),
531
+ ];
532
+ const unavailable = remote.flatMap((result) => result.ok ? [] : [result.unavailable]);
533
+ return {
534
+ snapshots,
535
+ entries: [
536
+ local,
537
+ ...remote.map((result) => result.ok
538
+ ? result.snapshot
539
+ : { host: result.unavailable.host, error: result.unavailable.error }),
540
+ ],
541
+ skippedHosts: unavailable.map(({ outpost_id, error }) => ({ outpost_id, error })),
542
+ };
543
+ }
544
+ function parsePlacementPolicy(raw) {
545
+ const value = raw.trim();
546
+ if (value === "local")
547
+ return { kind: "local" };
548
+ if (value === "auto")
549
+ return { kind: "auto" };
550
+ const match = /^outpost:(.+)$/.exec(value);
551
+ const id = match?.[1]?.trim();
552
+ if (id)
553
+ return { kind: "outpost", id };
554
+ throw new LanesError("lane_placement_invalid", `Invalid placement ${JSON.stringify(raw)}. Accepted: local, auto, outpost:<id>`, { value: raw, accepted: ["local", "auto", "outpost:<id>"] });
555
+ }
556
+ function assertPlacementEnabled(ctx) {
557
+ if (ctx.lanePlacementEnabled())
558
+ return;
559
+ throw new LanesError("lane_placement_disabled", `Lane placement options require hq-flags gate ${LANE_OUTPOST_LOAD_BALANCING_FLAG}`, { flag: LANE_OUTPOST_LOAD_BALANCING_FLAG });
560
+ }
405
561
  function loadAllQuestions(hqRoot) {
406
562
  return listQuestionIds(hqRoot).map((id) => readQuestion(hqRoot, id));
407
563
  }
@@ -731,6 +887,11 @@ function assertSameTenant(laneCompany, senior, seniorCompany, extra = {}) {
731
887
  function recordMatchesCompany(lane, filter) {
732
888
  return companyRefMatches(lane.company, filter) || companyRefMatches(lane.mesh_company, filter);
733
889
  }
890
+ function sameLaneCompany(a, b) {
891
+ if (a === null || b === null)
892
+ return a === b;
893
+ return (a.uid !== "" && a.uid === b.uid) || (a.slug !== "" && a.slug === b.slug);
894
+ }
734
895
  function companyRefMatches(recordCompany, filter) {
735
896
  return (recordCompany !== null &&
736
897
  recordCompany !== undefined &&
@@ -1384,6 +1545,23 @@ function mintMessageId() {
1384
1545
  }
1385
1546
  async function runCreate(ctx, opts) {
1386
1547
  const hqRoot = ctx.hqRoot();
1548
+ const requestedPlacement = opts.placement !== undefined
1549
+ ? parsePlacementPolicy(opts.placement)
1550
+ : undefined;
1551
+ const defaultPlacement = opts.defaultPlacement !== undefined
1552
+ ? parsePlacementPolicy(opts.defaultPlacement)
1553
+ : undefined;
1554
+ if (opts.outpost !== undefined && requestedPlacement !== undefined) {
1555
+ throw new LanesError("lane_placement_conflict", "Pass either --outpost or --placement, not both", {});
1556
+ }
1557
+ let placementGate;
1558
+ const placementEnabled = () => {
1559
+ placementGate ??= ctx.lanePlacementEnabled();
1560
+ return placementGate;
1561
+ };
1562
+ if ((requestedPlacement !== undefined || defaultPlacement !== undefined) && !placementEnabled()) {
1563
+ assertPlacementEnabled(ctx);
1564
+ }
1387
1565
  const companyFlag = opts.company?.trim();
1388
1566
  const meshCompanyFlag = opts.meshCompany?.trim();
1389
1567
  const project = opts.project?.trim();
@@ -1460,6 +1638,13 @@ async function runCreate(ctx, opts) {
1460
1638
  profile?.meta.provider ??
1461
1639
  parseLaneProvider("codex");
1462
1640
  const profileDefaultsApply = profile !== undefined && profile.meta.provider === provider;
1641
+ const cliModel = parseCliModel(opts.model);
1642
+ // Validated here because `claude --effort <unknown>` only warns and runs.
1643
+ const cliEffort = parseCliEffort(provider, opts.effort);
1644
+ const profileModel = profileDefaultsApply ? profile?.meta.model : undefined;
1645
+ const profileEffort = profileDefaultsApply ? profile?.meta.effort : undefined;
1646
+ const profileAccount = profileDefaultsApply ? profile?.meta.account : undefined;
1647
+ const profileAccounts = profileDefaultsApply ? profile?.meta.accounts : undefined;
1463
1648
  const briefAbs = resolveCallerPath(ctx.callerCwd(), briefFile);
1464
1649
  if (!ctx.existsSync(briefAbs)) {
1465
1650
  throw new LanesError("brief_not_found", `Brief file not found: ${briefAbs}`, {
@@ -1467,7 +1652,102 @@ async function runCreate(ctx, opts) {
1467
1652
  });
1468
1653
  }
1469
1654
  const config = loadLanesConfig(hqRoot);
1470
- const outpostId = opts.outpost?.trim() || undefined;
1655
+ let outpostId = opts.outpost?.trim() || undefined;
1656
+ let placementHostOverride;
1657
+ let autoPlacementChoice;
1658
+ let autoPlacementSkippedHosts = [];
1659
+ let inheritedPlacement;
1660
+ const callerLaneId = ctx.env.HQ_LANE_ID?.trim() || undefined;
1661
+ if (requestedPlacement === undefined &&
1662
+ outpostId === undefined &&
1663
+ callerLaneId !== undefined &&
1664
+ placementEnabled()) {
1665
+ const callerLane = tryReadLane(hqRoot, callerLaneId);
1666
+ if (!callerLane) {
1667
+ throw new LanesError("lane_not_found", `Caller lane ${callerLaneId} not found`, {
1668
+ lane_id: callerLaneId,
1669
+ });
1670
+ }
1671
+ if (callerLane.placement_default !== undefined) {
1672
+ if (!sameLaneCompany(company, callerLane.company)) {
1673
+ throw new LanesError("lane_placement_tenant_mismatch", "Cannot inherit placement from a manager lane in another tenant", { caller_lane_id: callerLaneId });
1674
+ }
1675
+ inheritedPlacement = callerLane.placement_default;
1676
+ }
1677
+ }
1678
+ const placementToApply = requestedPlacement ?? inheritedPlacement;
1679
+ const outpostTargets = new Set();
1680
+ if (placementToApply?.kind === "outpost")
1681
+ outpostTargets.add(placementToApply.id);
1682
+ if (defaultPlacement?.kind === "outpost")
1683
+ outpostTargets.add(defaultPlacement.id);
1684
+ for (const target of outpostTargets) {
1685
+ if (target === opts.outpost?.trim())
1686
+ continue;
1687
+ let known;
1688
+ try {
1689
+ known = await ctx.listOutpostIds();
1690
+ }
1691
+ catch (err) {
1692
+ throw new LanesError("outpost_list_unavailable", `Failed to list Outpost ids: ${dispatchErrorMessage(err)}`, {});
1693
+ }
1694
+ assertKnownOutpost(target, known);
1695
+ }
1696
+ if (placementToApply !== undefined) {
1697
+ const currentHost = laneHostFromCtx(ctx);
1698
+ if (placementToApply.kind === "local") {
1699
+ placementHostOverride = currentHost;
1700
+ outpostId = undefined;
1701
+ }
1702
+ else if (placementToApply.kind === "outpost") {
1703
+ if (currentHost.kind === "outpost" && currentHost.id === placementToApply.id) {
1704
+ placementHostOverride = currentHost;
1705
+ outpostId = undefined;
1706
+ }
1707
+ else {
1708
+ placementHostOverride = { kind: "outpost", id: placementToApply.id };
1709
+ outpostId = placementToApply.id;
1710
+ }
1711
+ }
1712
+ else {
1713
+ const capacity = await loadHostCapacitySnapshots(ctx, hqRoot, config);
1714
+ autoPlacementSkippedHosts = capacity.skippedHosts;
1715
+ const requestedAccount = opts.account !== undefined
1716
+ ? opts.account.trim()
1717
+ : profileAccount !== undefined && profileAccounts === undefined
1718
+ ? profileAccount
1719
+ : undefined;
1720
+ if (opts.account !== undefined && requestedAccount === "") {
1721
+ throw new AccountResolutionError("unknown_account", `Unknown account ${JSON.stringify(opts.account)} for ${provider}`, { provider, account: opts.account, enabled: [] });
1722
+ }
1723
+ try {
1724
+ autoPlacementChoice = chooseHostForLane(capacity.snapshots, {
1725
+ provider,
1726
+ ...(requestedAccount !== undefined ? { account: requestedAccount } : {}),
1727
+ ...(profileAccounts !== undefined ? { allowedAccounts: profileAccounts } : {}),
1728
+ });
1729
+ }
1730
+ catch (err) {
1731
+ if (autoPlacementSkippedHosts.length > 0 &&
1732
+ isLanesError(err) &&
1733
+ err.error === "lane_placement_no_capacity") {
1734
+ throw new LanesError(err.error, `${err.message}; skipped Outposts: ${autoPlacementSkippedHosts.map((host) => host.outpost_id).join(", ")}`, { ...err.details, skipped_hosts: autoPlacementSkippedHosts });
1735
+ }
1736
+ throw err;
1737
+ }
1738
+ placementHostOverride = autoPlacementChoice.host;
1739
+ if (autoPlacementChoice.host.kind === currentHost.kind &&
1740
+ autoPlacementChoice.host.id === currentHost.id) {
1741
+ outpostId = undefined;
1742
+ }
1743
+ else if (autoPlacementChoice.host.kind === "outpost") {
1744
+ outpostId = autoPlacementChoice.host.id;
1745
+ }
1746
+ else {
1747
+ outpostId = undefined;
1748
+ }
1749
+ }
1750
+ }
1471
1751
  let originHost;
1472
1752
  let originHostSource;
1473
1753
  if (outpostId) {
@@ -1478,16 +1758,12 @@ async function runCreate(ctx, opts) {
1478
1758
  originHost = identity.id;
1479
1759
  originHostSource = identity.source;
1480
1760
  }
1481
- const cliModel = parseCliModel(opts.model);
1482
- // Validated here because `claude --effort <unknown>` only warns and runs.
1483
- const cliEffort = parseCliEffort(provider, opts.effort);
1484
- const profileModel = profileDefaultsApply ? profile?.meta.model : undefined;
1485
- const profileEffort = profileDefaultsApply ? profile?.meta.effort : undefined;
1486
- const profileAccount = profileDefaultsApply ? profile?.meta.account : undefined;
1487
- const profileAccounts = profileDefaultsApply ? profile?.meta.accounts : undefined;
1488
1761
  let pinnedAccount;
1489
1762
  const accountWasExplicit = Boolean(opts.account && opts.account.trim() !== "");
1490
- if (outpostId) {
1763
+ if (autoPlacementChoice !== undefined) {
1764
+ pinnedAccount = autoPlacementChoice.account;
1765
+ }
1766
+ else if (outpostId) {
1491
1767
  if (accountWasExplicit) {
1492
1768
  pinnedAccount = opts.account.trim();
1493
1769
  }
@@ -1515,7 +1791,9 @@ async function runCreate(ctx, opts) {
1515
1791
  });
1516
1792
  pinnedAccount = resolved.id;
1517
1793
  }
1518
- if (outpostId !== undefined && profile !== undefined) {
1794
+ if (outpostId !== undefined &&
1795
+ profile !== undefined &&
1796
+ !placementEnabled()) {
1519
1797
  throw new LanesError("worker_profile_outpost_unsupported", `Worker profile ${JSON.stringify(profile.id)} cannot be used with --outpost; personal profiles are local-only`, {
1520
1798
  worker: profile.id,
1521
1799
  outpost: outpostId,
@@ -1719,9 +1997,9 @@ async function runCreate(ctx, opts) {
1719
1997
  throw admission.error;
1720
1998
  }
1721
1999
  }
1722
- const host = outpostId !== undefined
2000
+ const host = placementHostOverride ?? (outpostId !== undefined
1723
2001
  ? { kind: "outpost", id: outpostId }
1724
- : { kind: "local", id: ctx.hostname() };
2002
+ : { kind: "local", id: ctx.hostname() });
1725
2003
  const queuedReason = admission.action === "queue" ? admission.reason : undefined;
1726
2004
  const admissionReservedAt = admission.action === "start" ? ctx.now().toISOString() : undefined;
1727
2005
  const lane = createLane(hqRoot, {
@@ -1759,6 +2037,9 @@ async function runCreate(ctx, opts) {
1759
2037
  : {}),
1760
2038
  provider,
1761
2039
  account: pinnedAccount,
2040
+ ...(defaultPlacement !== undefined
2041
+ ? { placement_default: defaultPlacement }
2042
+ : {}),
1762
2043
  ...(balancedAdmissionAccounts !== undefined
1763
2044
  ? { admission_accounts: balancedAdmissionAccounts }
1764
2045
  : {}),
@@ -1884,6 +2165,9 @@ async function runCreate(ctx, opts) {
1884
2165
  effort: started.effort ?? null,
1885
2166
  effort_source: started.effort_source ?? null,
1886
2167
  host: started.host,
2168
+ ...(autoPlacementSkippedHosts.length > 0
2169
+ ? { skipped_hosts: autoPlacementSkippedHosts }
2170
+ : {}),
1887
2171
  company: started.company,
1888
2172
  project_id: started.project_id,
1889
2173
  story_id: started.story_id,
@@ -1904,7 +2188,7 @@ async function runCreate(ctx, opts) {
1904
2188
  });
1905
2189
  }
1906
2190
  async function dispatchCreatedOutpostLane(ctx, input) {
1907
- const { hqRoot, lane, outpostId, briefText, timeout } = input;
2191
+ const { hqRoot, lane, outpostId, briefText, timeout, preserveQueuedOnFailure = false, } = input;
1908
2192
  const payload = {
1909
2193
  lane,
1910
2194
  brief: briefText,
@@ -1915,8 +2199,26 @@ async function dispatchCreatedOutpostLane(ctx, input) {
1915
2199
  const staged = await ctx.stageOutpostPayload(outpostId, bytes);
1916
2200
  const command = buildOutpostPickupCommand(staged.getUrl);
1917
2201
  const dispatched = await ctx.dispatchOutpostExec({ outpostId, command });
2202
+ if (preserveQueuedOnFailure && dispatched.done !== true) {
2203
+ throw new LanesError("outpost_dispatch_failed", "Outpost pickup exec has not finished; its outcome is unknown", {
2204
+ outpost_id: outpostId,
2205
+ lane_id: lane.lane_id,
2206
+ command_id: dispatched.commandId,
2207
+ status: dispatched.status,
2208
+ done: false,
2209
+ outcome_unknown: true,
2210
+ });
2211
+ }
1918
2212
  const remote = parseRemotePickupJson(dispatched.stdout);
1919
2213
  if (remote?.error === "outpost_self_target") {
2214
+ if (preserveQueuedOnFailure) {
2215
+ throw new LanesError("outpost_self_target", `Outpost ${outpostId} resolved to the lane's current host`, {
2216
+ outpost_id: outpostId,
2217
+ lane_id: lane.lane_id,
2218
+ command_id: dispatched.commandId,
2219
+ done: dispatched.done,
2220
+ });
2221
+ }
1920
2222
  const cancelled = casUpdateLane(hqRoot, lane.lane_id, (current) => ({
1921
2223
  ...current,
1922
2224
  state: "cancelled",
@@ -1927,8 +2229,28 @@ async function dispatchCreatedOutpostLane(ctx, input) {
1927
2229
  await emitLaneTerminalStateSoft(ctx, cancelled);
1928
2230
  return cancelled;
1929
2231
  }
2232
+ if (preserveQueuedOnFailure && remote?.pickup_persisted === false) {
2233
+ throw new LanesError("outpost_dispatch_failed", "Outpost pickup finished and reported that it did not persist the lane", {
2234
+ outpost_id: outpostId,
2235
+ lane_id: lane.lane_id,
2236
+ command_id: dispatched.commandId,
2237
+ done: true,
2238
+ pickup_persisted: false,
2239
+ status: dispatched.status,
2240
+ exit_code: dispatched.exitCode,
2241
+ });
2242
+ }
1930
2243
  const stdoutTrimmed = dispatched.stdout.trim();
1931
2244
  if (remote === null && stdoutTrimmed !== "" && dispatched.status !== "Success") {
2245
+ if (preserveQueuedOnFailure) {
2246
+ throw new LanesError("outpost_dispatch_failed", "Outpost pickup command failed", {
2247
+ outpost_id: outpostId,
2248
+ lane_id: lane.lane_id,
2249
+ command_id: dispatched.commandId,
2250
+ status: dispatched.status,
2251
+ exit_code: dispatched.exitCode,
2252
+ });
2253
+ }
1932
2254
  const failed = markDispatchFailure(hqRoot, lane.lane_id, {
1933
2255
  outpost_id: outpostId,
1934
2256
  command_id: dispatched.commandId,
@@ -1948,6 +2270,15 @@ async function dispatchCreatedOutpostLane(ctx, input) {
1948
2270
  }
1949
2271
  const dispatchOk = dispatched.status === "Success" || dispatched.exitCode === 0;
1950
2272
  if (!dispatchOk) {
2273
+ if (preserveQueuedOnFailure) {
2274
+ throw new LanesError("outpost_dispatch_failed", "Outpost pickup command failed", {
2275
+ outpost_id: outpostId,
2276
+ lane_id: lane.lane_id,
2277
+ command_id: dispatched.commandId,
2278
+ status: dispatched.status,
2279
+ exit_code: dispatched.exitCode,
2280
+ });
2281
+ }
1951
2282
  const failed = markDispatchFailure(hqRoot, lane.lane_id, {
1952
2283
  outpost_id: outpostId,
1953
2284
  command_id: dispatched.commandId,
@@ -1964,12 +2295,29 @@ async function dispatchCreatedOutpostLane(ctx, input) {
1964
2295
  command_id: dispatched.commandId,
1965
2296
  });
1966
2297
  }
1967
- return casUpdateLane(hqRoot, lane.lane_id, (current) => ({
1968
- ...current,
1969
- outpost_command_id: dispatched.commandId,
1970
- }));
2298
+ try {
2299
+ return casUpdateLane(hqRoot, lane.lane_id, (current) => ({
2300
+ ...current,
2301
+ outpost_command_id: dispatched.commandId,
2302
+ }));
2303
+ }
2304
+ catch (err) {
2305
+ if (!preserveQueuedOnFailure)
2306
+ throw err;
2307
+ const message = dispatchErrorMessage(err);
2308
+ throw new LanesError("outpost_dispatch_failed", `Outpost pickup completed but its command id could not be recorded: ${message}`, {
2309
+ outpost_id: outpostId,
2310
+ lane_id: lane.lane_id,
2311
+ command_id: dispatched.commandId,
2312
+ done: true,
2313
+ outcome_unknown: true,
2314
+ message,
2315
+ });
2316
+ }
1971
2317
  }
1972
2318
  catch (err) {
2319
+ if (preserveQueuedOnFailure)
2320
+ throw err;
1973
2321
  if (isLanesError(err) &&
1974
2322
  err.error === DISPATCH_TERMINAL_REASON &&
1975
2323
  readLane(hqRoot, lane.lane_id).state === "failed") {
@@ -2009,12 +2357,27 @@ function parseRemoteJson(stdout, surface) {
2009
2357
  }
2010
2358
  return null;
2011
2359
  }
2360
+ function verifyOutpostProfileContent(hqRoot, lane) {
2361
+ if (lane.host.kind !== "outpost" || lane.worker_profile === undefined)
2362
+ return;
2363
+ const expectedHash = lane.worker_profile_hash;
2364
+ const profile = loadWorkerProfile(hqRoot, lane.worker_profile);
2365
+ if (expectedHash !== profile.contentHash) {
2366
+ throw new LanesError("worker_profile_hash_mismatch", `Worker profile ${JSON.stringify(lane.worker_profile)} on Outpost ${lane.host.id} does not match the hash recorded by the creator`, {
2367
+ lane_id: lane.lane_id,
2368
+ worker: lane.worker_profile,
2369
+ outpost: lane.host.id,
2370
+ expected_hash: expectedHash ?? null,
2371
+ actual_hash: profile.contentHash,
2372
+ });
2373
+ }
2374
+ }
2012
2375
  function laneIsHostedHere(ctx, lane) {
2013
2376
  if (lane.host.kind !== "outpost")
2014
2377
  return true;
2015
2378
  const identity = ctx.readOutpostIdentity();
2016
2379
  if (identity.kind === "unknown") {
2017
- throw new LanesError("outpost_identity_unknown", `Cannot determine whether this process is Outpost ${lane.host.id}; refusing local lane delivery`, { lane_id: lane.lane_id, outpost_id: lane.host.id });
2380
+ throw outpostIdentityUnknownError(`Cannot determine whether this process is Outpost ${lane.host.id}; refusing local lane delivery`, lane.host.id, { lane_id: lane.lane_id, outpost_id: lane.host.id });
2018
2381
  }
2019
2382
  return identity.kind === "id" && identity.id === lane.host.id;
2020
2383
  }
@@ -2122,37 +2485,58 @@ function ingestPickupPayload(hqRoot, payload, now) {
2122
2485
  }
2123
2486
  return lane;
2124
2487
  }
2488
+ function pickupNotPersistedError(err) {
2489
+ const message = dispatchErrorMessage(err);
2490
+ if (isLanesError(err)) {
2491
+ return new LanesError(err.error, message, {
2492
+ ...err.details,
2493
+ message,
2494
+ pickup_persisted: false,
2495
+ });
2496
+ }
2497
+ return new LanesError("outpost_pickup_failed", message, {
2498
+ message,
2499
+ pickup_persisted: false,
2500
+ });
2501
+ }
2125
2502
  async function runPickup(ctx, opts) {
2126
2503
  const hqRoot = ctx.hqRoot();
2127
- const payloadFile = opts.payloadFile?.trim();
2128
- const payloadUrl = opts.payloadUrl?.trim();
2129
- if (!payloadFile && !payloadUrl) {
2130
- throw new LanesError("missing_payload", "--payload-file or --payload-url is required", {});
2131
- }
2132
- if (payloadFile && payloadUrl) {
2133
- throw new LanesError("invalid_payload", "Pass only one of --payload-file or --payload-url", {});
2134
- }
2135
- let rawText;
2136
- if (payloadUrl) {
2137
- rawText = await ctx.fetchText(payloadUrl);
2138
- }
2139
- else {
2140
- const abs = resolveCallerPath(ctx.callerCwd(), payloadFile);
2141
- if (!ctx.existsSync(abs)) {
2142
- throw new LanesError("brief_not_found", `Payload file not found: ${abs}`, {
2143
- path: abs,
2144
- });
2145
- }
2146
- rawText = ctx.readFileSync(abs, "utf8");
2147
- }
2148
- let parsedJson;
2504
+ let payload;
2149
2505
  try {
2150
- parsedJson = JSON.parse(rawText);
2506
+ const payloadFile = opts.payloadFile?.trim();
2507
+ const payloadUrl = opts.payloadUrl?.trim();
2508
+ if (!payloadFile && !payloadUrl) {
2509
+ throw new LanesError("missing_payload", "--payload-file or --payload-url is required", {});
2510
+ }
2511
+ if (payloadFile && payloadUrl) {
2512
+ throw new LanesError("invalid_payload", "Pass only one of --payload-file or --payload-url", {});
2513
+ }
2514
+ let rawText;
2515
+ if (payloadUrl) {
2516
+ rawText = await ctx.fetchText(payloadUrl);
2517
+ }
2518
+ else {
2519
+ const abs = resolveCallerPath(ctx.callerCwd(), payloadFile);
2520
+ if (!ctx.existsSync(abs)) {
2521
+ throw new LanesError("brief_not_found", `Payload file not found: ${abs}`, {
2522
+ path: abs,
2523
+ });
2524
+ }
2525
+ rawText = ctx.readFileSync(abs, "utf8");
2526
+ }
2527
+ let parsedJson;
2528
+ try {
2529
+ parsedJson = JSON.parse(rawText);
2530
+ }
2531
+ catch (err) {
2532
+ throw new LanesError("outpost_payload_invalid", `Pickup payload is not JSON: ${err instanceof Error ? err.message : String(err)}`, {});
2533
+ }
2534
+ payload = parseOutpostPickupPayload(parsedJson);
2535
+ verifyOutpostProfileContent(hqRoot, payload.lane);
2151
2536
  }
2152
2537
  catch (err) {
2153
- throw new LanesError("outpost_payload_invalid", `Pickup payload is not JSON: ${err instanceof Error ? err.message : String(err)}`, {});
2538
+ throw pickupNotPersistedError(err);
2154
2539
  }
2155
- const payload = parseOutpostPickupPayload(parsedJson);
2156
2540
  const ingested = ingestPickupPayload(hqRoot, payload, ctx.now());
2157
2541
  if (ingested.host.kind === "outpost") {
2158
2542
  try {
@@ -2340,7 +2724,428 @@ async function runList(ctx, opts) {
2340
2724
  : false,
2341
2725
  };
2342
2726
  });
2343
- printJson({ ok: true, action: "list", lanes });
2727
+ const host_capacity = ctx.lanePlacementEnabled()
2728
+ ? (await loadHostCapacitySnapshots(ctx, hqRoot)).entries
2729
+ : undefined;
2730
+ printJson({
2731
+ ok: true,
2732
+ action: "list",
2733
+ lanes,
2734
+ ...(host_capacity !== undefined ? { host_capacity } : {}),
2735
+ });
2736
+ }
2737
+ async function runCapacity(ctx) {
2738
+ const hqRoot = ctx.hqRoot();
2739
+ const host = laneHostFromCtx(ctx);
2740
+ const snapshot = buildHostCapacitySnapshot({
2741
+ host,
2742
+ lanes: loadAllLanes(hqRoot),
2743
+ config: loadLanesConfig(hqRoot),
2744
+ existsSync: ctx.existsSync,
2745
+ now: ctx.now(),
2746
+ });
2747
+ printJson(snapshot);
2748
+ }
2749
+ async function runOutpostIdentityBackfill(ctx, opts) {
2750
+ const id = opts.id?.trim();
2751
+ if (!id) {
2752
+ throw new LanesError("outpost_identity_id_invalid", "Outpost identity backfill requires --id <outpost-id>", {});
2753
+ }
2754
+ validateOutpostIdentityId(id);
2755
+ const identity = ctx.readOutpostIdentity();
2756
+ if (identity.kind === "unknown" || (identity.kind === "id" && identity.id === id)) {
2757
+ const result = ctx.backfillOutpostIdentity(id);
2758
+ printJson({
2759
+ ok: true,
2760
+ action: "outpost-identity-backfill",
2761
+ outpost_id: id,
2762
+ changed: result === "created",
2763
+ marker_path: OUTPOST_ID_PATH,
2764
+ });
2765
+ return;
2766
+ }
2767
+ let known;
2768
+ try {
2769
+ known = await ctx.listOutpostIds();
2770
+ }
2771
+ catch (err) {
2772
+ throw new LanesError("outpost_list_unavailable", `Failed to list Outpost ids: ${dispatchErrorMessage(err)}`, {});
2773
+ }
2774
+ assertKnownOutpost(id, known);
2775
+ let dispatched;
2776
+ try {
2777
+ dispatched = await ctx.dispatchOutpostExec({
2778
+ outpostId: id,
2779
+ command: buildOutpostIdentityBackfillCommand(id),
2780
+ timeoutSeconds: 30,
2781
+ waitMs: 30_000,
2782
+ });
2783
+ }
2784
+ catch (err) {
2785
+ throw new LanesError("outpost_identity_backfill_failed", `Could not backfill Outpost ${id}: ${dispatchErrorMessage(err)}`, { expected_id: id, marker_path: OUTPOST_ID_PATH, outpost_id: id });
2786
+ }
2787
+ if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
2788
+ throw new LanesError("outpost_identity_backfill_failed", `Outpost ${id} did not confirm the identity marker backfill`, {
2789
+ expected_id: id,
2790
+ marker_path: OUTPOST_ID_PATH,
2791
+ outpost_id: id,
2792
+ command_id: dispatched.commandId,
2793
+ status: dispatched.status,
2794
+ done: dispatched.done ?? false,
2795
+ exit_code: dispatched.exitCode,
2796
+ });
2797
+ }
2798
+ const remote = parseRemoteJson(dispatched.stdout, "Outpost identity backfill");
2799
+ if (remote?.ok !== true ||
2800
+ remote.action !== "outpost-identity-backfill-target" ||
2801
+ remote.outpost_id !== id ||
2802
+ typeof remote.changed !== "boolean" ||
2803
+ remote.marker_path !== OUTPOST_ID_PATH) {
2804
+ throw new LanesError("outpost_identity_backfill_unconfirmed", `Outpost ${id} returned an invalid identity backfill result`, { outpost_id: id, expected_id: id, marker_path: OUTPOST_ID_PATH });
2805
+ }
2806
+ printJson({
2807
+ ok: true,
2808
+ action: "outpost-identity-backfill",
2809
+ outpost_id: id,
2810
+ changed: remote.changed,
2811
+ marker_path: OUTPOST_ID_PATH,
2812
+ outpost_command_id: dispatched.commandId,
2813
+ });
2814
+ }
2815
+ async function runOutpostIdentityBackfillTarget(ctx, opts) {
2816
+ const expectedId = opts.expectedId?.trim();
2817
+ if (!expectedId) {
2818
+ throw new LanesError("outpost_identity_id_invalid", "Outpost identity target requires --expected-id <outpost-id>", {});
2819
+ }
2820
+ validateOutpostIdentityId(expectedId);
2821
+ const identity = ctx.readOutpostIdentity();
2822
+ if (identity.kind === "not_outpost") {
2823
+ throw new LanesError("outpost_identity_not_host", `Identity backfill target is not an Outpost; expected ${OUTPOST_DIR} to exist`, { expected_id: expectedId, marker_path: OUTPOST_ID_PATH });
2824
+ }
2825
+ const result = ctx.backfillOutpostIdentity(expectedId);
2826
+ printJson({
2827
+ ok: true,
2828
+ action: "outpost-identity-backfill-target",
2829
+ outpost_id: expectedId,
2830
+ changed: result === "created",
2831
+ marker_path: OUTPOST_ID_PATH,
2832
+ });
2833
+ }
2834
+ function laneCanMoveBeforeStart(lane) {
2835
+ return lane.state === "queued" &&
2836
+ lane.pickup_validation_pending !== true &&
2837
+ lane.admission_reserved_at === undefined &&
2838
+ lane.provider_session_id === undefined &&
2839
+ lane.provider_session_id_absent === undefined &&
2840
+ lane.provider_home_at_spawn === undefined &&
2841
+ lane.last_resume === undefined &&
2842
+ lane.last_run_usage === undefined &&
2843
+ lane.last_host_report_at === undefined &&
2844
+ lane.pending_resume === undefined;
2845
+ }
2846
+ function readLaneTimeout(hqRoot, laneId) {
2847
+ const raw = readJsonFile(timeoutPath(hqRoot, laneId));
2848
+ if (raw === null)
2849
+ return undefined;
2850
+ if (typeof raw !== "object" || Array.isArray(raw) ||
2851
+ typeof raw.timeout_secs !== "number" ||
2852
+ typeof raw.deadline_at !== "string" ||
2853
+ !Number.isSafeInteger(raw.timeout_secs) ||
2854
+ raw.timeout_secs <= 0 ||
2855
+ !Number.isFinite(Date.parse(raw.deadline_at))) {
2856
+ throw new LanesError("lane_timeout_invalid", `Lane ${laneId} has a malformed timeout record`, {
2857
+ lane_id: laneId,
2858
+ });
2859
+ }
2860
+ const record = raw;
2861
+ return { timeout_secs: record.timeout_secs, deadline_at: record.deadline_at };
2862
+ }
2863
+ async function runMove(ctx, laneId, opts) {
2864
+ assertPlacementEnabled(ctx);
2865
+ const hqRoot = ctx.hqRoot();
2866
+ const policy = parsePlacementPolicy(opts.placement ?? "");
2867
+ const source = readLane(hqRoot, laneId);
2868
+ if (!laneCanMoveBeforeStart(source)) {
2869
+ throw new LanesError(source.state === "queued" ? "lane_move_started" : "lane_not_queued", `Lane ${laneId} cannot move after it has started; current state is ${source.state}`, { lane_id: laneId, state: source.state });
2870
+ }
2871
+ const sourceHost = source.host;
2872
+ const config = loadLanesConfig(hqRoot);
2873
+ const capacity = await loadHostCapacitySnapshots(ctx, hqRoot, config);
2874
+ let targetSnapshots = capacity.snapshots.filter((snapshot) => snapshot.host.kind !== sourceHost.kind || snapshot.host.id !== sourceHost.id);
2875
+ if (policy.kind === "outpost") {
2876
+ targetSnapshots = targetSnapshots.filter((snapshot) => snapshot.host.kind === "outpost" && snapshot.host.id === policy.id);
2877
+ }
2878
+ else if (policy.kind === "local") {
2879
+ targetSnapshots = targetSnapshots.filter((snapshot) => snapshot.host.kind === "local");
2880
+ }
2881
+ const profileAccounts = source.worker_profile === undefined
2882
+ ? undefined
2883
+ : loadWorkerProfile(hqRoot, source.worker_profile).meta.accounts;
2884
+ const allowedAccounts = source.admission_accounts ?? profileAccounts;
2885
+ const choice = chooseHostForLane(targetSnapshots, {
2886
+ provider: source.provider,
2887
+ ...(source.account !== undefined ? { account: source.account } : {}),
2888
+ ...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
2889
+ });
2890
+ let moved;
2891
+ if (laneRunsHere(source, reconcileHostFromCtx(ctx))) {
2892
+ moved = await commitLaneMove(ctx, laneId, choice.host, choice.account);
2893
+ }
2894
+ else if (source.host.kind === "outpost") {
2895
+ moved = await dispatchRemoteLaneMove(ctx, source, choice.host, choice.account);
2896
+ }
2897
+ else {
2898
+ const currentHost = laneHostFromCtx(ctx);
2899
+ throw new LanesError("lane_move_wrong_host", `Move lane ${laneId} from its current host (${source.host.kind}:${source.host.id})`, { lane_id: laneId, lane_host: source.host, current_host: currentHost });
2900
+ }
2901
+ printJson({
2902
+ ok: true,
2903
+ action: "move",
2904
+ lane_id: laneId,
2905
+ state: moved.state,
2906
+ host: moved.host,
2907
+ account: moved.account ?? null,
2908
+ lane: moved,
2909
+ });
2910
+ }
2911
+ async function commitLaneMove(ctx, laneId, target, account) {
2912
+ assertPlacementEnabled(ctx);
2913
+ const hqRoot = ctx.hqRoot();
2914
+ const sourceHost = laneHostFromCtx(ctx);
2915
+ const admissionHost = reconcileHostFromCtx(ctx);
2916
+ const before = readLane(hqRoot, laneId);
2917
+ if (!laneRunsHere(before, admissionHost)) {
2918
+ throw new LanesError("lane_move_wrong_host", `Move lane ${laneId} from its current host (${before.host.kind}:${before.host.id})`, { lane_id: laneId, lane_host: before.host, current_host: sourceHost });
2919
+ }
2920
+ if (before.host.kind === target.kind && before.host.id === target.id) {
2921
+ throw new LanesError("lane_move_same_host", `Lane ${laneId} is already on ${target.kind}:${target.id}`, {
2922
+ lane_id: laneId,
2923
+ host: target,
2924
+ });
2925
+ }
2926
+ if (!laneCanMoveBeforeStart(before)) {
2927
+ throw new LanesError(before.state === "queued" ? "lane_move_started" : "lane_not_queued", `Lane ${laneId} cannot move after it has started; current state is ${before.state}`, { lane_id: laneId, state: before.state });
2928
+ }
2929
+ if (target.kind === "outpost") {
2930
+ let known;
2931
+ try {
2932
+ known = await ctx.listOutpostIds();
2933
+ }
2934
+ catch (err) {
2935
+ throw new LanesError("outpost_list_unavailable", `Failed to list Outpost ids: ${dispatchErrorMessage(err)}`, {});
2936
+ }
2937
+ assertKnownOutpost(target.id, known);
2938
+ }
2939
+ const accountConfig = loadLanesConfig(hqRoot).providers[before.provider].accounts[account];
2940
+ if (accountConfig === undefined || !accountConfig.enabled) {
2941
+ throw new LanesError("lane_move_account_invalid", `Account ${account} is unavailable for ${before.provider}`, {
2942
+ lane_id: laneId,
2943
+ provider: before.provider,
2944
+ account,
2945
+ });
2946
+ }
2947
+ const profileAccounts = before.worker_profile === undefined
2948
+ ? undefined
2949
+ : loadWorkerProfile(hqRoot, before.worker_profile).meta.accounts;
2950
+ const allowedAccounts = before.admission_accounts ?? profileAccounts;
2951
+ if (allowedAccounts !== undefined &&
2952
+ !allowedAccounts.includes(account)) {
2953
+ throw new LanesError("lane_move_account_invalid", `Account ${account} is not allowed by lane ${laneId}`, {
2954
+ lane_id: laneId,
2955
+ account,
2956
+ allowed_accounts: allowedAccounts,
2957
+ });
2958
+ }
2959
+ const briefText = target.kind === "outpost"
2960
+ ? (() => {
2961
+ const briefPath = path.join(runDir(hqRoot, laneId), "brief.md");
2962
+ if (!ctx.existsSync(briefPath)) {
2963
+ throw new LanesError("lane_move_brief_missing", `Lane ${laneId} has no stored brief to dispatch`, {
2964
+ lane_id: laneId,
2965
+ });
2966
+ }
2967
+ return ctx.readFileSync(briefPath, "utf8");
2968
+ })()
2969
+ : undefined;
2970
+ const timeout = target.kind === "outpost" ? readLaneTimeout(hqRoot, laneId) : undefined;
2971
+ const moved = await withLaneAdmissionLock(hqRoot, before.provider, async () => {
2972
+ const current = readLane(hqRoot, laneId);
2973
+ if (!laneRunsHere(current, admissionHost)) {
2974
+ throw new LanesError("lane_move_conflict", `Lane ${laneId} changed hosts before the move`, {
2975
+ lane_id: laneId,
2976
+ host: current.host,
2977
+ });
2978
+ }
2979
+ if (!laneCanMoveBeforeStart(current)) {
2980
+ throw new LanesError(current.state === "queued" ? "lane_move_started" : "lane_not_queued", `Lane ${laneId} started before the move could commit`, { lane_id: laneId, state: current.state });
2981
+ }
2982
+ return updateLane(hqRoot, laneId, current.version, "command", (live) => ({
2983
+ ...live,
2984
+ host: target,
2985
+ account,
2986
+ queued_reason: undefined,
2987
+ admission_reserved_at: undefined,
2988
+ }));
2989
+ }, ctx.sleep);
2990
+ if (target.kind !== "outpost")
2991
+ return moved;
2992
+ try {
2993
+ return await dispatchCreatedOutpostLane(ctx, {
2994
+ hqRoot,
2995
+ lane: moved,
2996
+ outpostId: target.id,
2997
+ briefText: briefText ?? "",
2998
+ timeout,
2999
+ preserveQueuedOnFailure: true,
3000
+ });
3001
+ }
3002
+ catch (err) {
3003
+ const dispatchDetails = isLanesError(err) ? err.details : {};
3004
+ const commandId = typeof dispatchDetails.command_id === "string" &&
3005
+ dispatchDetails.command_id.trim() !== ""
3006
+ ? dispatchDetails.command_id
3007
+ : undefined;
3008
+ const definitelyNotSubmitted = commandId === undefined &&
3009
+ dispatchDetails.outcome_unknown !== true &&
3010
+ dispatchDetails.done !== false;
3011
+ const knownNoPersistence = (isLanesError(err) && err.error === "outpost_self_target") ||
3012
+ (dispatchDetails.done === true && dispatchDetails.pickup_persisted === false);
3013
+ if (!definitelyNotSubmitted && !knownNoPersistence) {
3014
+ const checkCommand = `hq lanes show ${laneId}`;
3015
+ const message = `Failed to dispatch lane ${laneId} to Outpost ${target.id}; outcome is unknown and the lane remains assigned to Outpost ${target.id}. ` +
3016
+ `Check the target copy by running \`${checkCommand}\` on Outpost ${target.id}. ` +
3017
+ dispatchErrorMessage(err);
3018
+ throw new LanesError("lane_move_dispatch_failed", message, {
3019
+ lane_id: laneId,
3020
+ message,
3021
+ source_host: before.host,
3022
+ target_host: target,
3023
+ source_restored: false,
3024
+ outcome: "unknown",
3025
+ ...(commandId !== undefined ? { command_id: commandId } : {}),
3026
+ check_command: checkCommand,
3027
+ });
3028
+ }
3029
+ let restored = false;
3030
+ let rollbackError;
3031
+ try {
3032
+ restored = await withLaneAdmissionLock(hqRoot, before.provider, async () => {
3033
+ const current = readLane(hqRoot, laneId);
3034
+ if (current.version !== moved.version ||
3035
+ current.state !== "queued" ||
3036
+ current.host.kind !== target.kind ||
3037
+ current.host.id !== target.id ||
3038
+ current.account !== account) {
3039
+ return false;
3040
+ }
3041
+ updateLane(hqRoot, laneId, current.version, "command", (live) => ({
3042
+ ...live,
3043
+ host: before.host,
3044
+ account: before.account,
3045
+ queued_reason: before.queued_reason,
3046
+ admission_reserved_at: before.admission_reserved_at,
3047
+ }));
3048
+ return true;
3049
+ }, ctx.sleep);
3050
+ }
3051
+ catch (restoreErr) {
3052
+ rollbackError = restoreErr;
3053
+ }
3054
+ const dispatchMessage = dispatchErrorMessage(err);
3055
+ const outcome = restored
3056
+ ? "the lane stayed queued on its source"
3057
+ : rollbackError === undefined
3058
+ ? "the lane changed after the move and was not overwritten"
3059
+ : `source restoration failed: ${dispatchErrorMessage(rollbackError)}`;
3060
+ const message = `Failed to dispatch lane ${laneId} to Outpost ${target.id}; ${outcome}: ${dispatchMessage}`;
3061
+ throw new LanesError("lane_move_dispatch_failed", message, {
3062
+ lane_id: laneId,
3063
+ message,
3064
+ source_host: before.host,
3065
+ target_host: target,
3066
+ source_restored: restored,
3067
+ ...(rollbackError !== undefined
3068
+ ? { rollback_error: dispatchErrorMessage(rollbackError) }
3069
+ : {}),
3070
+ });
3071
+ }
3072
+ }
3073
+ async function dispatchRemoteLaneMove(ctx, source, target, account) {
3074
+ if (source.host.kind !== "outpost") {
3075
+ throw new LanesError("lane_move_wrong_host", `Lane ${source.lane_id} is not Outpost-hosted`, {
3076
+ lane_id: source.lane_id,
3077
+ host: source.host,
3078
+ });
3079
+ }
3080
+ let dispatched;
3081
+ try {
3082
+ dispatched = await ctx.dispatchOutpostExec({
3083
+ outpostId: source.host.id,
3084
+ command: buildOutpostMoveTargetCommand({
3085
+ laneId: source.lane_id,
3086
+ host: target,
3087
+ account,
3088
+ }),
3089
+ timeoutSeconds: 120,
3090
+ waitMs: 120_000,
3091
+ });
3092
+ }
3093
+ catch (err) {
3094
+ throw new LanesError("lane_move_dispatch_failed", `Could not ask Outpost ${source.host.id} to move lane ${source.lane_id}: ${dispatchErrorMessage(err)}`, { lane_id: source.lane_id, source_outpost: source.host.id });
3095
+ }
3096
+ if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
3097
+ throw new LanesError("lane_move_dispatch_failed", `Outpost ${source.host.id} did not confirm the lane move`, {
3098
+ lane_id: source.lane_id,
3099
+ source_outpost: source.host.id,
3100
+ command_id: dispatched.commandId,
3101
+ status: dispatched.status,
3102
+ done: dispatched.done ?? false,
3103
+ exit_code: dispatched.exitCode,
3104
+ });
3105
+ }
3106
+ const remote = parseRemoteJson(dispatched.stdout, "Outpost lane move");
3107
+ const remoteHost = remote?.host;
3108
+ if (remote?.ok !== true ||
3109
+ remote.action !== "move-target" ||
3110
+ remote.lane_id !== source.lane_id ||
3111
+ remote.state !== "queued" ||
3112
+ remote.account !== account ||
3113
+ remoteHost === null || typeof remoteHost !== "object" || Array.isArray(remoteHost) ||
3114
+ remoteHost.kind !== target.kind ||
3115
+ remoteHost.id !== target.id) {
3116
+ throw new LanesError("lane_move_unconfirmed", `Outpost ${source.host.id} returned an invalid lane move result`, {
3117
+ lane_id: source.lane_id,
3118
+ source_outpost: source.host.id,
3119
+ target,
3120
+ });
3121
+ }
3122
+ return {
3123
+ ...source,
3124
+ state: "queued",
3125
+ host: target,
3126
+ account,
3127
+ };
3128
+ }
3129
+ async function runMoveTarget(ctx, laneId, opts) {
3130
+ const kind = opts.hostKind?.trim();
3131
+ const id = opts.hostId?.trim();
3132
+ const account = opts.account?.trim();
3133
+ if ((kind !== "local" && kind !== "outpost") || !id || !account) {
3134
+ throw new LanesError("lane_move_target_invalid", "Move target requires a valid host kind, host id, and account", {
3135
+ host_kind: kind ?? null,
3136
+ host_id: id ?? null,
3137
+ account: account ?? null,
3138
+ });
3139
+ }
3140
+ const moved = await commitLaneMove(ctx, laneId, { kind, id }, account);
3141
+ printJson({
3142
+ ok: true,
3143
+ action: "move-target",
3144
+ lane_id: moved.lane_id,
3145
+ state: moved.state,
3146
+ host: moved.host,
3147
+ account: moved.account ?? null,
3148
+ });
2344
3149
  }
2345
3150
  async function runShow(ctx, laneId) {
2346
3151
  const hqRoot = ctx.hqRoot();
@@ -4236,6 +5041,8 @@ export function registerLanesCommand(parent, deps = {}) {
4236
5041
  .option("--timeout <secs>", "Deadline in seconds (warns, does not kill)")
4237
5042
  .option("--priority <n>", "Queue priority; higher runs first", "1")
4238
5043
  .option("--outpost <id>", "Run on this Outpost instead of the local host")
5044
+ .option("--placement <mode>", "local | auto | outpost:<id>")
5045
+ .option("--default-placement <mode>", "Default placement for lanes launched by this lane: local | auto | outpost:<id>")
4239
5046
  .action(wrap(async (opts) => {
4240
5047
  await runCreate(ctx, opts);
4241
5048
  }));
@@ -4251,6 +5058,45 @@ export function registerLanesCommand(parent, deps = {}) {
4251
5058
  .action(wrap(async (opts) => {
4252
5059
  await runList(ctx, opts);
4253
5060
  }));
5061
+ lanes
5062
+ .command("capacity")
5063
+ .description("Report this host's lane capacity by provider and account")
5064
+ .option("--json", "JSON on stdout (always)")
5065
+ .action(wrap(async () => {
5066
+ await runCapacity(ctx);
5067
+ }));
5068
+ const outpostIdentity = lanes
5069
+ .command("outpost-identity")
5070
+ .description("Repair this host's Outpost identity marker")
5071
+ .command("backfill")
5072
+ .description("Write a missing Outpost identity marker without replacing an existing value")
5073
+ .requiredOption("--id <outpost-id>", "Expected id of this host's Outpost");
5074
+ outpostIdentity.action(wrap(async (opts) => {
5075
+ await runOutpostIdentityBackfill(ctx, opts);
5076
+ }));
5077
+ lanes
5078
+ .command("outpost-identity-backfill-target", { hidden: true })
5079
+ .requiredOption("--expected-id <outpost-id>", "Expected id of this Outpost")
5080
+ .action(wrap(async (opts) => {
5081
+ await runOutpostIdentityBackfillTarget(ctx, opts);
5082
+ }));
5083
+ lanes
5084
+ .command("move")
5085
+ .description("Move an unstarted queued lane to a host with capacity")
5086
+ .argument("<lane-id>", "Lane id")
5087
+ .requiredOption("--placement <mode>", "local | auto | outpost:<id>")
5088
+ .action(wrap(async (laneId, opts) => {
5089
+ await runMove(ctx, laneId, opts);
5090
+ }));
5091
+ lanes
5092
+ .command("move-target", { hidden: true })
5093
+ .argument("<lane-id>", "Lane id")
5094
+ .requiredOption("--host-kind <kind>", "local | outpost")
5095
+ .requiredOption("--host-id <id>", "Target host id")
5096
+ .requiredOption("--account <id>", "Provider account id")
5097
+ .action(wrap(async (laneId, opts) => {
5098
+ await runMoveTarget(ctx, laneId, opts);
5099
+ }));
4254
5100
  lanes
4255
5101
  .command("admit")
4256
5102
  .description("Try to admit the oldest eligible queued lane")