@indigoai-us/hq-cli 5.132.0 → 5.134.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/dist/commands/agent-kit.d.ts +2 -0
  3. package/dist/commands/agent-kit.js +4 -1
  4. package/dist/commands/lanes.d.ts +2 -2
  5. package/dist/commands/lanes.js +98 -20
  6. package/dist/commands/mesh.js +4 -1
  7. package/dist/lib/agent-kit/run/heartbeat.d.ts +14 -2
  8. package/dist/lib/agent-kit/run/heartbeat.js +38 -6
  9. package/dist/lib/bot/api.d.ts +3 -1
  10. package/dist/lib/bot/api.js +7 -2
  11. package/dist/lib/bot/run.d.ts +4 -0
  12. package/dist/lib/bot/run.js +30 -1
  13. package/dist/lib/bot/status.d.ts +4 -0
  14. package/dist/lib/doctor/checks/bots.js +4 -1
  15. package/dist/lib/heartbeat-identity.d.ts +29 -0
  16. package/dist/lib/heartbeat-identity.js +71 -0
  17. package/dist/lib/lanes/constants.d.ts +3 -0
  18. package/dist/lib/lanes/constants.js +3 -0
  19. package/dist/lib/lanes/index.d.ts +1 -0
  20. package/dist/lib/lanes/index.js +1 -0
  21. package/dist/lib/lanes/quota.d.ts +6 -3
  22. package/dist/lib/lanes/quota.js +65 -15
  23. package/dist/lib/lanes/reconcile.d.ts +5 -2
  24. package/dist/lib/lanes/reconcile.js +94 -2
  25. package/dist/lib/lanes/types.d.ts +2 -0
  26. package/dist/lib/lanes/types.js +2 -1
  27. package/dist/lib/mesh/live/daemon/doctor.d.ts +6 -0
  28. package/dist/lib/mesh/live/daemon/doctor.js +11 -0
  29. package/dist/lib/mesh/live/daemon/install.d.ts +7 -0
  30. package/dist/lib/mesh/live/daemon/install.js +16 -6
  31. package/dist/lib/mesh/live/daemon/legacy-listen.d.ts +43 -0
  32. package/dist/lib/mesh/live/daemon/legacy-listen.js +89 -0
  33. package/dist/lib/mesh/live/daemon/live-cache.d.ts +26 -0
  34. package/dist/lib/mesh/live/daemon/live-cache.js +116 -0
  35. package/dist/lib/mesh/live/daemon/presence.d.ts +7 -3
  36. package/dist/lib/mesh/live/daemon/presence.js +10 -9
  37. package/dist/lib/mesh/live/daemon/run.d.ts +13 -0
  38. package/dist/lib/mesh/live/daemon/run.js +51 -2
  39. package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +31 -0
  40. package/dist/lib/mesh/live/daemon/work-subscribe.js +124 -0
  41. package/package.json +2 -2
package/CHANGELOG.md CHANGED
@@ -2,6 +2,40 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.134.0] — 2026-09-22
6
+
7
+ ### Fixed
8
+
9
+ - Agent-kit and local-bot heartbeat loops now stop after the server reports an
10
+ agent identity mismatch instead of retrying indefinitely. `hq doctor` shows
11
+ the configured and token agent IDs for the mismatch.
12
+
13
+ ## [5.133.0] — 2026-09-22
14
+
15
+ ### Fixed
16
+
17
+ - Lane follow-ups now probe all enabled, logged-in accounts for an unpinned
18
+ model, preserve the later quota reset and its source, expire abandoned pickup
19
+ validation markers with a terminal reason, and parse Codex's ordinal
20
+ `try again at` reset time.
21
+
22
+ - `hq sync push` no longer fails every file when the target vault was
23
+ provisioned on or after 2026-09-19. Those vaults carry an S3 Object Lock
24
+ default retention rule, and S3 refuses a PutObject into them unless the
25
+ request carries a content checksum; the CLI's declared `@indigoai-us/hq-cloud`
26
+ range still allowed a client that sends none, so an affected push reported
27
+ `400 InvalidRequest: Content-MD5 OR x-amz-checksum- HTTP header is required
28
+ for Put Object requests with Object Lock parameters` on every file. The
29
+ dependency floor now starts at the release that attaches the checksum, so no
30
+ install can resolve a client that cannot write to a locked vault. A CI guard
31
+ holds the floor.
32
+
33
+ - Lane creation and re-parenting now allow depth 3. A depth-3 caller and any
34
+ derived depth 4 lane are refused with `max_lane_depth`.
35
+
36
+ - Question ageing now derives its two-advance cap from the depth-3 lane cap, so
37
+ an unanswered worker-lane question reaches the user after two hourly hops.
38
+
5
39
  ## [5.132.0] — 2026-09-22
6
40
 
7
41
  ### Added
@@ -104,6 +138,16 @@
104
138
 
105
139
  ## [Unreleased]
106
140
 
141
+ - `hq mesh daemon install` and `hq mesh daemon run` unload and remove the
142
+ legacy `ai.getindigo.hq-work-mesh-listen` unit (LaunchAgent, systemd user
143
+ unit, or Scheduled Task) so only the daemon subscribes. The daemon writes
144
+ `~/.hq/work-mesh/live-cache.json` and bumps its cursor once per project-view
145
+ push. MQTT reconnect logs once per transition into reconnecting instead of
146
+ on every close. `hq mesh daemon doctor` reports "legacy listener still
147
+ installed" while that unit remains (US-006).
148
+
149
+ ## [Unreleased]
150
+
107
151
  - Held-event repair no longer treats a later event timestamp as proof that
108
152
  metadata belongs to the current session instance. A reused session id stays
109
153
  unresolved when the held event's binding episode does not match the current
@@ -23,6 +23,7 @@ import { type AgentKitPaths } from "../lib/agent-kit/paths.js";
23
23
  import { type KitService } from "../lib/agent-kit/services.js";
24
24
  import { type CronRunner, type RebootCronResult, type DetachedSpawn, type KillFn } from "../lib/agent-kit/fallback.js";
25
25
  import { type ServiceHostPaths, type ServiceManagerDeps, type ServiceSetResult } from "../lib/service-manager/index.js";
26
+ import type { HeartbeatCredsSource } from "../lib/heartbeat-identity.js";
26
27
  export declare class KitError extends Error {
27
28
  constructor(message: string);
28
29
  }
@@ -73,6 +74,7 @@ export interface KitRuntime {
73
74
  config: KitConfig;
74
75
  getToken: () => Promise<string>;
75
76
  log: KitLogger;
77
+ credsSource?: HeartbeatCredsSource;
76
78
  }
77
79
  export declare function buildKitRuntime(service: KitService | "all"): KitRuntime;
78
80
  export declare function runKitService(service: KitService, rt: KitRuntime): Promise<void>;
@@ -135,6 +135,7 @@ export function formatServiceSet(result) {
135
135
  }
136
136
  export function buildKitRuntime(service) {
137
137
  const paths = agentKitPaths(os.homedir(), process.env);
138
+ const credsSource = process.env.HQ_MACHINE_CREDS_FILE?.trim() ? "explicit" : "default";
138
139
  const creds = requireExternalCreds(paths);
139
140
  ensureKitDirs(paths);
140
141
  const config = readKitConfig(paths, CLI_VERSION);
@@ -145,7 +146,7 @@ export function buildKitRuntime(service) {
145
146
  process.env.HQ_VAULT_API_URL = creds.apiBaseUrl;
146
147
  const log = createKitLogger(paths, service, { echo: process.stdout.isTTY === true });
147
148
  const getToken = () => ensureCognitoToken({ tokenSource: "machine", interactive: false });
148
- return { paths, creds, config, getToken, log };
149
+ return { paths, creds, config, getToken, log, credsSource };
149
150
  }
150
151
  export async function runKitService(service, rt) {
151
152
  rt.log("info", `${service} starting agent=${rt.creds.entityUid} cli=${CLI_VERSION}`);
@@ -167,6 +168,8 @@ export async function runKitService(service, rt) {
167
168
  getToken: rt.getToken,
168
169
  log: rt.log,
169
170
  components: enabledKitComponents(rt.config),
171
+ agentUidSource: rt.credsSource === "explicit" ? "env" : "config_file",
172
+ credsSource: rt.credsSource ?? "default",
170
173
  });
171
174
  return;
172
175
  case "inbox":
@@ -7,14 +7,14 @@
7
7
  import { Command } from "commander";
8
8
  import { type OutpostDispatchResult } from "../lib/lanes/outpost-dispatch.js";
9
9
  import { emitLaneDecisionEvents, type LaneMesh } from "../lib/lanes/mesh-events.js";
10
- import { type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
10
+ import { MAX_LANE_DEPTH, type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
11
11
  import { questionBodyAbsPath } from "../lib/lanes/questions.js";
12
12
  import { type RunCompactionFn } from "../lib/lanes/compaction.js";
13
13
  import { type KnownModelsDeps, type ProbeCommand } from "../lib/lanes/models.js";
14
14
  import { spawnLaneWorker, type KillProcessFn, type ProveDetachedFn, type SpawnProcessFn } from "../lib/lanes/spawn.js";
15
15
  export { spawnLaneWorker };
16
16
  export { questionBodyAbsPath };
17
- export declare const MAX_LANE_DEPTH = 2;
17
+ export { MAX_LANE_DEPTH };
18
18
  export declare const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
19
19
  export declare const OUTPOST_DIR = "/etc/outpost";
20
20
  export declare const LANES_RUNS_REL: string;
@@ -17,7 +17,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
17
17
  import { openMeshTransport } from "../lib/mesh/client.js";
18
18
  import { createLaneMesh, emitLaneDecisionEvents, } from "../lib/lanes/mesh-events.js";
19
19
  import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
20
- import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
20
+ import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, MAX_LANE_DEPTH, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
21
21
  import { questionBodyAbsPath } from "../lib/lanes/questions.js";
22
22
  import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
23
23
  import { providerCompactAfterMs } from "../lib/lanes/config.js";
@@ -29,7 +29,7 @@ import { defaultProbe, validateLaneModel, } from "../lib/lanes/models.js";
29
29
  import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, stopLaneProcesses, } from "../lib/lanes/spawn.js";
30
30
  export { spawnLaneWorker };
31
31
  export { questionBodyAbsPath };
32
- export const MAX_LANE_DEPTH = 2;
32
+ export { MAX_LANE_DEPTH };
33
33
  export const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
34
34
  export const OUTPOST_DIR = "/etc/outpost";
35
35
  export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
@@ -1092,6 +1092,7 @@ export function pickupFailureUpdater(reasonRaw, details) {
1092
1092
  state: "failed",
1093
1093
  queued_reason: undefined,
1094
1094
  pickup_validation_pending: undefined,
1095
+ pickup_validation_pending_since: undefined,
1095
1096
  terminal_reason: reason,
1096
1097
  terminal_details: details,
1097
1098
  };
@@ -1166,7 +1167,12 @@ export function applyOutpostAccountPickup(hqRoot, laneId, modelDeps = {}) {
1166
1167
  if (current.account === resolved.id && current.pickup_validation_pending === undefined) {
1167
1168
  return current;
1168
1169
  }
1169
- return { ...current, account: resolved.id, pickup_validation_pending: undefined };
1170
+ return {
1171
+ ...current,
1172
+ account: resolved.id,
1173
+ pickup_validation_pending: undefined,
1174
+ pickup_validation_pending_since: undefined,
1175
+ };
1170
1176
  });
1171
1177
  }
1172
1178
  catch (err) {
@@ -1315,8 +1321,8 @@ async function runCreate(ctx, opts) {
1315
1321
  // creates wait on. A pinned create checks the pinned account's home. An
1316
1322
  // unpinned create is not bound to an account here (persist re-resolves the
1317
1323
  // balanced account under the lock, so overlapping creates still spread),
1318
- // so it checks the balanced candidate first and then every other enabled
1319
- // account the locked selection could land on (codex catalogs are per home
1324
+ // so it checks the balanced candidate first and then every other enabled,
1325
+ // logged-in account the locked selection could land on (codex catalogs are per home
1320
1326
  // and can differ). `validatedAccounts` records which accepted the model
1321
1327
  // or had no readable list, and the balanced pick under the lock is
1322
1328
  // restricted to that set (resolveAccount's allowedAccounts): an account
@@ -1344,26 +1350,68 @@ async function runCreate(ctx, opts) {
1344
1350
  : [
1345
1351
  candidate,
1346
1352
  ...Object.keys(accounts)
1347
- .filter((id) => id !== candidate && accounts[id]?.enabled)
1353
+ .filter((id) => id !== candidate &&
1354
+ accounts[id]?.enabled &&
1355
+ accountStatus(provider, id, config, { existsSync: ctx.existsSync }).logged_in === true)
1348
1356
  .sort(),
1349
1357
  ];
1358
+ const checkedAccounts = [];
1359
+ const unsupported = [];
1360
+ const quotaBlocked = [];
1361
+ const quota = loadQuotaState(hqRoot);
1350
1362
  for (const id of toValidate) {
1351
1363
  const home = accounts[id]?.home;
1352
1364
  if (home === undefined)
1353
1365
  continue;
1366
+ checkedAccounts.push(id);
1354
1367
  try {
1355
1368
  validateLaneModel(provider, cliModel, home, modelDeps);
1356
- validatedAccounts.add(id);
1357
1369
  }
1358
1370
  catch (err) {
1359
1371
  if (!(isLanesError(err) && err.error === "model_unsupported"))
1360
1372
  throw err;
1361
- // The account that would run the lane refuses it: refuse now, before
1362
- // the other lists are read. Another account's refusal only drops it
1363
- // from the locked selection.
1364
- if (id === candidate)
1365
- throw err;
1373
+ unsupported.push({ account: id, error: err });
1374
+ continue;
1375
+ }
1376
+ const block = quotaBlockFor(quota, provider, id, cliModel, now);
1377
+ if (block.blocked) {
1378
+ quotaBlocked.push({ account: id, block });
1379
+ continue;
1380
+ }
1381
+ validatedAccounts.add(id);
1382
+ }
1383
+ if (validatedAccounts.size === 0 && quotaBlocked.length > 0) {
1384
+ const blocked = quotaBlocked[0];
1385
+ const model = blocked.block.model ?? cliModel;
1386
+ const checkedSuffix = ` Checked accounts: ${checkedAccounts.join(", ")}`;
1387
+ if (blocked.block.reason === "account_model_unavailable") {
1388
+ throw new AccountResolutionError("account_model_unavailable", `Account ${JSON.stringify(blocked.account)} cannot run model ${model ?? "(no model)"}: ${blocked.block.marker ?? "needs credits"}${checkedSuffix}`, {
1389
+ provider,
1390
+ account: blocked.account,
1391
+ model: model ?? null,
1392
+ marker: blocked.block.marker ?? null,
1393
+ checked_accounts: checkedAccounts,
1394
+ });
1366
1395
  }
1396
+ throw new AccountResolutionError("account_quota_exhausted", `Account ${JSON.stringify(blocked.account)} is out of quota${blocked.block.quota_reset_at ? ` until ${blocked.block.quota_reset_at}` : ""} for model ${model ?? "(no model)"}${checkedSuffix}`, {
1397
+ provider,
1398
+ account: blocked.account,
1399
+ model: model ?? null,
1400
+ quota_reset_at: blocked.block.quota_reset_at ?? null,
1401
+ checked_accounts: checkedAccounts,
1402
+ });
1403
+ }
1404
+ if (validatedAccounts.size === 0 && unsupported.length > 0) {
1405
+ const first = unsupported[0].error;
1406
+ throw new LanesError(first.error, `${first.message} Checked accounts: ${checkedAccounts.join(", ")}`, {
1407
+ ...first.details,
1408
+ checked_accounts: checkedAccounts,
1409
+ account_errors: unsupported.map(({ account, error }) => ({
1410
+ account,
1411
+ accepted: error.details.accepted,
1412
+ source: error.details.source,
1413
+ })),
1414
+ });
1367
1415
  }
1368
1416
  }
1369
1417
  const threadId = company
@@ -1637,7 +1685,7 @@ function parseRemotePickupJson(stdout) {
1637
1685
  }
1638
1686
  return null;
1639
1687
  }
1640
- function ingestPickupPayload(hqRoot, payload) {
1688
+ function ingestPickupPayload(hqRoot, payload, now) {
1641
1689
  const laneId = payload.lane.lane_id;
1642
1690
  if (!laneId) {
1643
1691
  throw new LanesError("outpost_payload_invalid", "Pickup payload lane.lane_id is required", {});
@@ -1646,9 +1694,20 @@ function ingestPickupPayload(hqRoot, payload) {
1646
1694
  // The lane is stored ineligible for admission until the account bind
1647
1695
  // clears the marker (see LaneRecord.pickup_validation_pending).
1648
1696
  const lane = existing === null
1649
- ? createLane(hqRoot, { ...payload.lane, pickup_validation_pending: true })
1650
- : existing.state === "queued" && existing.pickup_validation_pending === undefined
1651
- ? retryUpdateLane(hqRoot, laneId, "command", (current) => current.state === "queued" ? { ...current, pickup_validation_pending: true } : current)
1697
+ ? createLane(hqRoot, {
1698
+ ...payload.lane,
1699
+ updated_at: now.toISOString(),
1700
+ pickup_validation_pending: true,
1701
+ pickup_validation_pending_since: now.toISOString(),
1702
+ })
1703
+ : existing.state === "queued"
1704
+ ? retryUpdateLane(hqRoot, laneId, "command", (current) => current.state === "queued"
1705
+ ? {
1706
+ ...current,
1707
+ pickup_validation_pending: true,
1708
+ pickup_validation_pending_since: now.toISOString(),
1709
+ }
1710
+ : current)
1652
1711
  : existing;
1653
1712
  const dest = runDir(hqRoot, lane.lane_id);
1654
1713
  ensureDir(path.join(dest, "logs"));
@@ -1692,7 +1751,7 @@ async function runPickup(ctx, opts) {
1692
1751
  throw new LanesError("outpost_payload_invalid", `Pickup payload is not JSON: ${err instanceof Error ? err.message : String(err)}`, {});
1693
1752
  }
1694
1753
  const payload = parseOutpostPickupPayload(parsedJson);
1695
- const ingested = ingestPickupPayload(hqRoot, payload);
1754
+ const ingested = ingestPickupPayload(hqRoot, payload, ctx.now());
1696
1755
  if (ingested.host.kind === "outpost") {
1697
1756
  try {
1698
1757
  refuseOutpostSelfTarget(hqRoot, ingested, {
@@ -2373,15 +2432,26 @@ async function runReparent(ctx, laneId, opts) {
2373
2432
  throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", { lane_id: laneId, from_depth: oldDepth, to_depth: newDepth });
2374
2433
  }
2375
2434
  const override = { laneId, senior: nextSenior, depth: newDepth };
2435
+ const descendants = [];
2376
2436
  for (const other of loadAllLanes(hqRoot)) {
2377
2437
  if (other.lane_id === laneId)
2378
2438
  continue;
2379
2439
  if (!walkHitsLane(hqRoot, other.senior, laneId, ctx))
2380
2440
  continue;
2381
- const d = derivedLaneDepth(hqRoot, other.senior, ctx, override);
2382
- if (d > MAX_LANE_DEPTH) {
2383
- throw new LanesError("max_lane_depth", `Re-parent would put descendant ${other.lane_id} at depth ${d}`, { lane_id: other.lane_id, depth: d, max: MAX_LANE_DEPTH });
2441
+ const oldDescendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx);
2442
+ const descendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx, override);
2443
+ if (descendantDepth > MAX_LANE_DEPTH) {
2444
+ throw new LanesError("max_lane_depth", `Re-parent would put descendant ${other.lane_id} at depth ${descendantDepth}`, { lane_id: other.lane_id, depth: descendantDepth, max: MAX_LANE_DEPTH });
2384
2445
  }
2446
+ if (descendantDepth > oldDescendantDepth &&
2447
+ pendingQuestionsForLane(hqRoot, other.lane_id).length > 0) {
2448
+ throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", {
2449
+ lane_id: other.lane_id,
2450
+ from_depth: oldDescendantDepth,
2451
+ to_depth: descendantDepth,
2452
+ });
2453
+ }
2454
+ descendants.push({ laneId: other.lane_id, depth: descendantDepth });
2385
2455
  }
2386
2456
  const at = ctx.now().toISOString();
2387
2457
  const updated = casUpdateLane(hqRoot, laneId, (current) => {
@@ -2401,6 +2471,13 @@ async function runReparent(ctx, laneId, opts) {
2401
2471
  }),
2402
2472
  };
2403
2473
  });
2474
+ for (const descendant of descendants) {
2475
+ casUpdateLane(hqRoot, descendant.laneId, (current) => {
2476
+ if (current.depth === descendant.depth)
2477
+ return current;
2478
+ return { ...current, depth: descendant.depth };
2479
+ });
2480
+ }
2404
2481
  for (const question of pending) {
2405
2482
  casUpdateQuestion(hqRoot, question.question_id, (current) => {
2406
2483
  const already = current.routing_history.some((entry) => entry.reason === "reparented" && entry.at === at);
@@ -3019,6 +3096,7 @@ async function runReconcile(ctx, opts) {
3019
3096
  throttleDeps: ctx.throttleDeps,
3020
3097
  livenessDeps: livenessDepsFromCtx(ctx),
3021
3098
  presenceConnected: ctx.presenceConnected,
3099
+ mesh: ctx.mesh,
3022
3100
  env: ctx.env,
3023
3101
  });
3024
3102
  printJson(summary);
@@ -1545,7 +1545,10 @@ export function registerMeshCommand(program) {
1545
1545
  }
1546
1546
  async function runDaemonRun(opts = {}) {
1547
1547
  const hqRoot = opts.hqRoot?.trim() || undefined;
1548
- const handle = await runMeshDaemon(hqRoot ? { hqRoot } : {});
1548
+ const handle = await runMeshDaemon({
1549
+ ...(hqRoot ? { hqRoot } : {}),
1550
+ enableWorkCache: true,
1551
+ });
1549
1552
  // Stay alive until signal handlers call stop + exit.
1550
1553
  await new Promise(() => {
1551
1554
  void handle;
@@ -11,6 +11,7 @@
11
11
  * The last successful beat is recorded at state/last-heartbeat.json so
12
12
  * `hq agent probe` can report its age.
13
13
  */
14
+ import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../../heartbeat-identity.js";
14
15
  import { type ComponentStatus } from "../creds.js";
15
16
  import type { KitLogger } from "../log.js";
16
17
  import { type AgentKitPaths, type KitComponent } from "../paths.js";
@@ -36,9 +37,13 @@ export interface HeartbeatBody {
36
37
  }
37
38
  export declare function buildHeartbeatBody(paths: Pick<AgentKitPaths, "stateDir">, now?: () => Date, hostInfo?: HeartbeatBody["hostInfo"], components_?: readonly KitComponent[]): HeartbeatBody;
38
39
  export interface LastHeartbeat {
40
+ /** The most recent heartbeat attempt, successful or a terminal identity mismatch. */
39
41
  at: string;
40
42
  components: Partial<Record<KitComponent, ComponentStatus>>;
41
43
  status: number;
44
+ reason?: "identity_mismatch";
45
+ configuredAgentUid?: string;
46
+ tokenAgentUid?: string;
42
47
  }
43
48
  export declare function writeLastHeartbeat(paths: Pick<AgentKitPaths, "stateDir" | "lastHeartbeatPath">, record: LastHeartbeat): void;
44
49
  export declare function readLastHeartbeat(paths: Pick<AgentKitPaths, "lastHeartbeatPath">): LastHeartbeat | null;
@@ -51,12 +56,19 @@ export interface HeartbeatDeps {
51
56
  log: KitLogger;
52
57
  /** Components this kit runs; defaults to all of them. */
53
58
  components?: readonly KitComponent[];
54
- post?: (token: string, body: HeartbeatBody) => Promise<number>;
59
+ post?: (token: string, body: HeartbeatBody) => Promise<number | HeartbeatPostResult>;
55
60
  now?: () => Date;
56
61
  sleep?: (ms: number) => Promise<void>;
57
62
  maxBeats?: number;
63
+ agentUidSource?: HeartbeatAgentUidSource;
64
+ credsSource?: HeartbeatCredsSource;
65
+ reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
58
66
  }
59
- export declare function defaultHeartbeatPost(agentUid: string, apiBaseUrl: string): (token: string, body: HeartbeatBody) => Promise<number>;
67
+ export interface HeartbeatPostResult {
68
+ status: number;
69
+ bodyText?: string;
70
+ }
71
+ export declare function defaultHeartbeatPost(agentUid: string, apiBaseUrl: string): (token: string, body: HeartbeatBody) => Promise<HeartbeatPostResult>;
60
72
  export declare function beatOnce(deps: HeartbeatDeps): Promise<boolean>;
61
73
  export declare function runHeartbeatLoop(deps: HeartbeatDeps): Promise<void>;
62
74
  //# sourceMappingURL=heartbeat.d.ts.map
@@ -14,6 +14,7 @@
14
14
  import * as fs from "node:fs";
15
15
  import * as os from "node:os";
16
16
  import { CLI_VERSION } from "../../../cli-version.js";
17
+ import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, parseHeartbeatFailureBody, tokenAgentUid, } from "../../heartbeat-identity.js";
17
18
  import { vaultApiFetch } from "../../../utils/vault-api.js";
18
19
  import { readComponentStatus } from "../creds.js";
19
20
  import { KIT_COMPONENTS } from "../paths.js";
@@ -60,36 +61,67 @@ export function defaultHeartbeatPost(agentUid, apiBaseUrl) {
60
61
  method: "POST",
61
62
  body: body,
62
63
  });
63
- return res.status;
64
+ return { status: res.status, bodyText: await res.text().catch(() => "") };
64
65
  };
65
66
  }
66
- export async function beatOnce(deps) {
67
+ async function beatOnceResult(deps) {
67
68
  const now = deps.now ?? (() => new Date());
68
69
  const post = deps.post ?? defaultHeartbeatPost(deps.agentUid, deps.apiBaseUrl);
69
70
  const body = buildHeartbeatBody(deps.paths, now, undefined, deps.components ?? KIT_COMPONENTS);
70
71
  try {
71
72
  const token = await deps.getToken();
72
- const status = await post(token, body);
73
+ const response = await post(token, body);
74
+ const result = typeof response === "number" ? { status: response } : response;
75
+ const status = result.status;
73
76
  const ok = status >= 200 && status < 300;
74
77
  if (ok) {
75
78
  writeLastHeartbeat(deps.paths, { at: now().toISOString(), components: body.components, status });
76
79
  }
80
+ else {
81
+ const failure = parseHeartbeatFailureBody(result.bodyText);
82
+ if (isHeartbeatIdentityMismatch({ status, code: failure.code, message: failure.message })) {
83
+ const tokenUid = tokenAgentUid(token);
84
+ writeLastHeartbeat(deps.paths, {
85
+ at: now().toISOString(),
86
+ components: body.components,
87
+ status,
88
+ reason: "identity_mismatch",
89
+ configuredAgentUid: deps.agentUid,
90
+ ...(tokenUid ? { tokenAgentUid: tokenUid } : {}),
91
+ });
92
+ const tokenLabel = tokenUid ?? "unknown";
93
+ deps.log("warn", `heartbeat stopped: identity mismatch configured agentUid=${deps.agentUid} token agentUid=${tokenLabel}`);
94
+ (deps.reportIdentityMismatch ?? captureHeartbeatIdentityMismatch)({
95
+ runner: "agent-kit",
96
+ configuredAgentUid: deps.agentUid,
97
+ ...(tokenUid ? { tokenAgentUid: tokenUid } : {}),
98
+ agentUidSource: deps.agentUidSource ?? "config_file",
99
+ credsSource: deps.credsSource ?? "default",
100
+ });
101
+ return { ok: false, stop: true };
102
+ }
103
+ }
77
104
  deps.log(ok ? "info" : "warn", `heartbeat ${status} ${Object.entries(body.components)
78
105
  .map(([name, health]) => `${name}=${health}`)
79
106
  .join(" ")}`);
80
- return ok;
107
+ return { ok, stop: false };
81
108
  }
82
109
  catch (err) {
83
110
  deps.log("error", `heartbeat failed: ${err instanceof Error ? err.message : String(err)}`);
84
- return false;
111
+ return { ok: false, stop: false };
85
112
  }
86
113
  }
114
+ export async function beatOnce(deps) {
115
+ return (await beatOnceResult(deps)).ok;
116
+ }
87
117
  export async function runHeartbeatLoop(deps) {
88
118
  const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
89
119
  let beats = 0;
90
120
  for (;;) {
91
- await beatOnce(deps);
121
+ const result = await beatOnceResult(deps);
92
122
  beats += 1;
123
+ if (result.stop)
124
+ return;
93
125
  if (deps.maxBeats !== undefined && beats >= deps.maxBeats)
94
126
  return;
95
127
  await sleep(deps.intervalMs);
@@ -85,8 +85,10 @@ export declare class BotApiError extends Error {
85
85
  readonly status: number;
86
86
  readonly path: string;
87
87
  readonly code?: string | undefined;
88
+ readonly detail?: string | undefined;
88
89
  readonly retryAfterMs?: number | undefined;
89
- constructor(status: number, path: string, code?: string | undefined, detail?: string, retryAfterMs?: number | undefined);
90
+ readonly tokenAgentUid?: string | undefined;
91
+ constructor(status: number, path: string, code?: string | undefined, detail?: string | undefined, retryAfterMs?: number | undefined, tokenAgentUid?: string | undefined);
90
92
  }
91
93
  /** Parse a Retry-After header (delta-seconds or HTTP-date) to milliseconds. */
92
94
  export declare function retryAfterHeaderMs(value: string | null | undefined, nowMs?: number): number | undefined;
@@ -6,18 +6,23 @@
6
6
  * existing Cognito machinery. `fetch` is injectable for tests.
7
7
  */
8
8
  import { vaultApiFetch } from "../../utils/vault-api.js";
9
+ import { tokenAgentUid } from "../heartbeat-identity.js";
9
10
  import { assertNoSecretKeys } from "./local-config.js";
10
11
  export class BotApiError extends Error {
11
12
  status;
12
13
  path;
13
14
  code;
15
+ detail;
14
16
  retryAfterMs;
15
- constructor(status, path, code, detail, retryAfterMs) {
17
+ tokenAgentUid;
18
+ constructor(status, path, code, detail, retryAfterMs, tokenAgentUid) {
16
19
  super(`HQ API ${path} → ${status}${code ? ` ${code}` : ""}${detail ? `: ${detail}` : ""}`);
17
20
  this.status = status;
18
21
  this.path = path;
19
22
  this.code = code;
23
+ this.detail = detail;
20
24
  this.retryAfterMs = retryAfterMs;
25
+ this.tokenAgentUid = tokenAgentUid;
21
26
  this.name = "BotApiError";
22
27
  }
23
28
  }
@@ -72,7 +77,7 @@ export class BotApi {
72
77
  });
73
78
  if (!accept.includes(res.status)) {
74
79
  const err = await readError(res);
75
- throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")));
80
+ throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")), path.endsWith("/heartbeat") ? tokenAgentUid(token) : undefined);
76
81
  }
77
82
  const text = await res.text();
78
83
  return { status: res.status, body: (text ? JSON.parse(text) : {}) };
@@ -42,6 +42,7 @@ import type { BotConfig, BotKind } from "./config.js";
42
42
  import { type ProgressPosterOptions } from "./progress.js";
43
43
  import { type BotLogger } from "./log.js";
44
44
  import { type BotRuntime, type RuntimeTurnInput, type RuntimeTurnResult } from "./runtime/index.js";
45
+ import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../heartbeat-identity.js";
45
46
  import { type BotState, type BotStatusFile } from "./status.js";
46
47
  export declare const INBOX_POLL_INTERVAL_MS = 2000;
47
48
  export declare const HEARTBEAT_INTERVAL_MS = 30000;
@@ -119,6 +120,9 @@ export interface BotRunDeps {
119
120
  maxParallelTurns?: number;
120
121
  /** Wait between retries while the coding tool needs a sign-in (default SIGN_IN_RETRY_MS). */
121
122
  signInRetryMs?: number;
123
+ agentUidSource?: HeartbeatAgentUidSource;
124
+ credsSource?: HeartbeatCredsSource;
125
+ reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
122
126
  }
123
127
  export declare const KICKOFF_MESSAGE_ID = "kickoff";
124
128
  /**
@@ -59,6 +59,7 @@ import { dmSessionScope } from "./paths.js";
59
59
  import { isDmItem, resolveReplyTarget } from "./room-policy.js";
60
60
  import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
61
61
  import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
62
+ import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
62
63
  import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
63
64
  import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
64
65
  export const INBOX_POLL_INTERVAL_MS = 2_000;
@@ -415,10 +416,38 @@ export async function runBot(deps) {
415
416
  // runtime → model, inbox → task, sync → sync.
416
417
  await api.heartbeat(config.agentUid, { model: modelHealth, task: "ok", sync: "ok" });
417
418
  heartbeatFails = 0;
418
- setStatus({ lastHeartbeatAt: now().toISOString(), lastHeartbeatOk: true, lastHeartbeatError: undefined });
419
+ setStatus({
420
+ lastHeartbeatAt: now().toISOString(),
421
+ lastHeartbeatOk: true,
422
+ lastHeartbeatError: undefined,
423
+ lastHeartbeatReason: undefined,
424
+ lastHeartbeatConfiguredAgentUid: undefined,
425
+ lastHeartbeatTokenAgentUid: undefined,
426
+ });
419
427
  await sleep(heartbeatIntervalMs);
420
428
  }
421
429
  catch (err) {
430
+ if (err instanceof BotApiError &&
431
+ isHeartbeatIdentityMismatch({ status: err.status, code: err.code, message: err.detail })) {
432
+ const tokenLabel = err.tokenAgentUid ?? "unknown";
433
+ const reason = `heartbeat stopped: identity mismatch configured agentUid=${config.agentUid} token agentUid=${tokenLabel}`;
434
+ setStatus({
435
+ lastHeartbeatOk: false,
436
+ lastHeartbeatError: reason.slice(0, 400),
437
+ lastHeartbeatReason: "identity_mismatch",
438
+ lastHeartbeatConfiguredAgentUid: config.agentUid,
439
+ ...(err.tokenAgentUid ? { lastHeartbeatTokenAgentUid: err.tokenAgentUid } : {}),
440
+ });
441
+ log("warn", reason);
442
+ (deps.reportIdentityMismatch ?? captureHeartbeatIdentityMismatch)({
443
+ runner: "bot",
444
+ configuredAgentUid: config.agentUid,
445
+ ...(err.tokenAgentUid ? { tokenAgentUid: err.tokenAgentUid } : {}),
446
+ agentUidSource: deps.agentUidSource ?? "config_file",
447
+ credsSource: deps.credsSource ?? "default",
448
+ });
449
+ break;
450
+ }
422
451
  heartbeatFails += 1;
423
452
  const message = err instanceof Error ? err.message : String(err);
424
453
  setStatus({ lastHeartbeatOk: false, lastHeartbeatError: message.slice(0, 400) });
@@ -19,6 +19,10 @@ export interface BotStatusFile {
19
19
  lastHeartbeatOk?: boolean;
20
20
  /** Last heartbeat error, when lastHeartbeatOk is false. */
21
21
  lastHeartbeatError?: string;
22
+ /** A heartbeat stopped permanently until the bot is restarted with corrected identity config. */
23
+ lastHeartbeatReason?: "identity_mismatch";
24
+ lastHeartbeatConfiguredAgentUid?: string;
25
+ lastHeartbeatTokenAgentUid?: string;
22
26
  lastInboxPollAt?: string;
23
27
  lastInboxPollOk?: boolean;
24
28
  /** Last inbox poll error, when lastInboxPollOk is false. */
@@ -76,11 +76,14 @@ function runChecks(deps) {
76
76
  const status = readBotStatus(dir);
77
77
  const alive = !!status && status.state === "running" && isPidAlive(status.pid);
78
78
  if (alive && status.lastHeartbeatOk === false) {
79
+ const heartbeatMessage = status.lastHeartbeatReason === "identity_mismatch"
80
+ ? `${name} heartbeat stopped: identity mismatch (configured agent ${status.lastHeartbeatConfiguredAgentUid ?? config.agentUid}; token agent ${status.lastHeartbeatTokenAgentUid ?? "unknown"}).`
81
+ : `${name} is running but cannot heartbeat to HQ${status.lastHeartbeatError ? ` (${status.lastHeartbeatError})` : ""}.`;
79
82
  results.push({
80
83
  status: "FAIL",
81
84
  checkId: `bots.${name}.heartbeat`,
82
85
  target: dir,
83
- message: `${name} is running but cannot heartbeat to HQ${status.lastHeartbeatError ? ` (${status.lastHeartbeatError})` : ""}.`,
86
+ message: heartbeatMessage,
84
87
  remediation: `hq bot logs ${name}`,
85
88
  });
86
89
  }