@indigoai-us/hq-cli 5.345.155 → 5.345.159

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/CHANGELOG.md +289 -0
  2. package/assets/scaffold/core/scripts/rebuild-company-knowledge-index.sh +110 -2
  3. package/assets/scaffold/core/scripts/rebuild-projects-index.sh +101 -5
  4. package/assets/scaffold/core/scripts/worktree.sh +4 -0
  5. package/dist/command-catalog.generated.d.ts +536 -7
  6. package/dist/command-catalog.generated.js +676 -24
  7. package/dist/command-registration-plan.d.ts +18 -0
  8. package/dist/command-registration-plan.js +4 -0
  9. package/dist/commands/_patterns.d.ts +17 -0
  10. package/dist/commands/_patterns.js +22 -0
  11. package/dist/commands/agent-inbox.js +2 -2
  12. package/dist/commands/agent-kit.js +8 -6
  13. package/dist/commands/bot.js +7 -4
  14. package/dist/commands/channel-suggest.d.ts +56 -0
  15. package/dist/commands/channel-suggest.js +170 -0
  16. package/dist/commands/channels.js +7 -1
  17. package/dist/commands/cloud.d.ts +45 -0
  18. package/dist/commands/cloud.js +150 -10
  19. package/dist/commands/company.js +4 -3
  20. package/dist/commands/core-checkpoint-native.d.ts +18 -0
  21. package/dist/commands/core-checkpoint-native.js +67 -2
  22. package/dist/commands/core-pipeline-driver-native.d.ts +12 -1
  23. package/dist/commands/core-pipeline-driver-native.js +131 -5
  24. package/dist/commands/core.js +13 -1
  25. package/dist/commands/crm.js +83 -6
  26. package/dist/commands/deploy-access.d.ts +33 -0
  27. package/dist/commands/deploy-access.js +322 -0
  28. package/dist/commands/dm-attach.d.ts +106 -0
  29. package/dist/commands/dm-attach.js +184 -0
  30. package/dist/commands/dm.d.ts +37 -0
  31. package/dist/commands/dm.js +172 -10
  32. package/dist/commands/feedback-show.d.ts +40 -0
  33. package/dist/commands/feedback-show.js +261 -0
  34. package/dist/commands/feedback.d.ts +21 -0
  35. package/dist/commands/feedback.js +152 -2
  36. package/dist/commands/files-browse.d.ts +16 -0
  37. package/dist/commands/files-browse.js +52 -1
  38. package/dist/commands/files.d.ts +24 -0
  39. package/dist/commands/files.js +52 -1
  40. package/dist/commands/integrations-connect.js +7 -3
  41. package/dist/commands/integrations-core.d.ts +13 -1
  42. package/dist/commands/integrations-core.js +49 -4
  43. package/dist/commands/integrations.js +4 -2
  44. package/dist/commands/lanes-accounts.js +16 -1
  45. package/dist/commands/lanes-loop.js +14 -4
  46. package/dist/commands/lanes-message.js +6 -4
  47. package/dist/commands/lanes-resume.js +36 -13
  48. package/dist/commands/lanes-shared.d.ts +5 -0
  49. package/dist/commands/lanes-shared.js +13 -3
  50. package/dist/commands/lanes-watch.js +1 -0
  51. package/dist/commands/lanes.d.ts +20 -1
  52. package/dist/commands/lanes.js +201 -51
  53. package/dist/commands/mesh.js +149 -1
  54. package/dist/commands/monitor.d.ts +1 -0
  55. package/dist/commands/monitor.js +4 -1
  56. package/dist/commands/repo.d.ts +41 -0
  57. package/dist/commands/repo.js +331 -0
  58. package/dist/commands/scan.d.ts +3 -1
  59. package/dist/commands/scan.js +7 -3
  60. package/dist/commands/secrets.d.ts +37 -0
  61. package/dist/commands/secrets.js +185 -16
  62. package/dist/commands/signals.js +17 -5
  63. package/dist/commands/staff/listings.d.ts +41 -0
  64. package/dist/commands/staff/listings.js +160 -0
  65. package/dist/commands/staff/reset-test-account.d.ts +3 -2
  66. package/dist/commands/staff/reset-test-account.js +4 -3
  67. package/dist/hq-cloud-sync-runner/bin/sync-runner.js +2629 -1036
  68. package/dist/hq-cloud-sync-runner/package.json +1 -1
  69. package/dist/lib/agent-kit/inbox-state.d.ts +7 -0
  70. package/dist/lib/agent-kit/inbox-state.js +5 -1
  71. package/dist/lib/agent-kit/mcp/tools.js +2 -1
  72. package/dist/lib/agent-kit/run/inbox.js +2 -1
  73. package/dist/lib/agent-kit/wake-secrets.d.ts +7 -0
  74. package/dist/lib/agent-kit/wake-secrets.js +32 -0
  75. package/dist/lib/agent-kit/wake.d.ts +4 -0
  76. package/dist/lib/agent-kit/wake.js +49 -3
  77. package/dist/lib/bot/inflight.d.ts +7 -1
  78. package/dist/lib/bot/inflight.js +10 -2
  79. package/dist/lib/bot/prompt.js +3 -0
  80. package/dist/lib/bot/run.js +72 -20
  81. package/dist/lib/bot/session.d.ts +14 -0
  82. package/dist/lib/bot/session.js +70 -0
  83. package/dist/lib/capacity/collectors/claude-oauth.d.ts +6 -0
  84. package/dist/lib/capacity/collectors/claude-oauth.js +12 -0
  85. package/dist/lib/capacity/lane-login.d.ts +6 -0
  86. package/dist/lib/capacity/lane-login.js +5 -0
  87. package/dist/lib/core-utils/worktree.js +2 -0
  88. package/dist/lib/daemon/exit-cause.d.ts +43 -0
  89. package/dist/lib/daemon/exit-cause.js +88 -0
  90. package/dist/lib/daemon/handoff.d.ts +2 -0
  91. package/dist/lib/daemon/handoff.js +6 -1
  92. package/dist/lib/daemon/hq-child-bootstrap.mjs +20 -3
  93. package/dist/lib/daemon/paths.d.ts +2 -0
  94. package/dist/lib/daemon/paths.js +1 -0
  95. package/dist/lib/daemon/run.js +13 -0
  96. package/dist/lib/daemon/supervisor.d.ts +54 -1
  97. package/dist/lib/daemon/supervisor.js +297 -21
  98. package/dist/lib/daemon/unit-holds.d.ts +22 -0
  99. package/dist/lib/daemon/unit-holds.js +86 -0
  100. package/dist/lib/doctor/checks/sync-health.d.ts +9 -0
  101. package/dist/lib/doctor/checks/sync-health.js +66 -1
  102. package/dist/lib/doctor/company-resolution-state.d.ts +30 -0
  103. package/dist/lib/doctor/company-resolution-state.js +89 -0
  104. package/dist/lib/doctor/fix/apply.d.ts +36 -6
  105. package/dist/lib/doctor/fix/apply.js +128 -12
  106. package/dist/lib/doctor/types.d.ts +1 -1
  107. package/dist/lib/doctor/types.js +1 -0
  108. package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
  109. package/dist/lib/flag-registry-worker-source.generated.js +1 -1
  110. package/dist/lib/hook-engine/resolve.js +6 -2
  111. package/dist/lib/index-render/company-knowledge.d.ts +1 -1
  112. package/dist/lib/index-render/company-knowledge.js +4 -5
  113. package/dist/lib/index-render/index.js +44 -3
  114. package/dist/lib/index-render/projects.d.ts +1 -1
  115. package/dist/lib/index-render/projects.js +5 -5
  116. package/dist/lib/lanes/account-company.d.ts +28 -0
  117. package/dist/lib/lanes/account-company.js +53 -0
  118. package/dist/lib/lanes/accounts.d.ts +22 -3
  119. package/dist/lib/lanes/accounts.js +38 -2
  120. package/dist/lib/lanes/claude-balance.d.ts +6 -1
  121. package/dist/lib/lanes/claude-balance.js +6 -1
  122. package/dist/lib/lanes/claude-credential.d.ts +78 -0
  123. package/dist/lib/lanes/claude-credential.js +163 -0
  124. package/dist/lib/lanes/compaction.d.ts +2 -0
  125. package/dist/lib/lanes/compaction.js +9 -0
  126. package/dist/lib/lanes/config.js +6 -0
  127. package/dist/lib/lanes/envelope-repair-dispatch.js +2 -1
  128. package/dist/lib/lanes/hooks-install.js +2 -1
  129. package/dist/lib/lanes/host-capacity.d.ts +2 -0
  130. package/dist/lib/lanes/host-capacity.js +1 -0
  131. package/dist/lib/lanes/keep-alive.d.ts +1 -1
  132. package/dist/lib/lanes/keep-alive.js +3 -1
  133. package/dist/lib/lanes/links.js +10 -4
  134. package/dist/lib/lanes/loop.js +16 -1
  135. package/dist/lib/lanes/node-binary.d.ts +17 -0
  136. package/dist/lib/lanes/node-binary.js +30 -0
  137. package/dist/lib/lanes/outpost-dispatch.d.ts +2 -1
  138. package/dist/lib/lanes/outpost-dispatch.js +1 -0
  139. package/dist/lib/lanes/reconcile.js +4 -0
  140. package/dist/lib/lanes/remote-control-lane.js +2 -0
  141. package/dist/lib/lanes/spawn.d.ts +3 -0
  142. package/dist/lib/lanes/spawn.js +119 -9
  143. package/dist/lib/lanes/store.d.ts +6 -0
  144. package/dist/lib/lanes/store.js +107 -18
  145. package/dist/lib/lanes/types.d.ts +2 -0
  146. package/dist/lib/lanes/watch.js +28 -5
  147. package/dist/lib/lanes/watcher-main.js +4 -3
  148. package/dist/lib/lanes/watcher.d.ts +4 -0
  149. package/dist/lib/lanes/watcher.js +110 -21
  150. package/dist/lib/mesh/api.d.ts +32 -0
  151. package/dist/lib/mesh/api.js +75 -0
  152. package/dist/lib/secret-scan/detect.js +9 -0
  153. package/dist/lib/secret-scan/repo-config.d.ts +41 -0
  154. package/dist/lib/secret-scan/repo-config.js +237 -0
  155. package/dist/lib/secret-scan/rules.d.ts +7 -1
  156. package/dist/lib/secret-scan/rules.js +3 -1
  157. package/dist/lib/secret-scan/staged.d.ts +5 -0
  158. package/dist/lib/secret-scan/staged.js +19 -2
  159. package/dist/lib/secret-scan/types.d.ts +2 -0
  160. package/dist/lib/sync/server-owned-local-only.d.ts +29 -0
  161. package/dist/lib/sync/server-owned-local-only.js +75 -0
  162. package/dist/sentry-before-send.js +1 -0
  163. package/dist/utils/access-denied-hint.js +6 -0
  164. package/dist/utils/client-health.d.ts +34 -0
  165. package/dist/utils/client-health.js +96 -3
  166. package/dist/utils/package-use-lease.js +27 -9
  167. package/dist/utils/version-gate.d.ts +7 -0
  168. package/dist/utils/version-gate.js +66 -0
  169. package/package.json +8 -8
@@ -3,6 +3,11 @@
3
3
  *
4
4
  * - long-running units are restarted after they exit, with per-unit backoff
5
5
  * (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
6
+ * a unit that fails EPISODE_MAX_FAILURES times without a run of
7
+ * EPISODE_HEALTHY_RESET_MS, or fails for a permanent cause (see exit-cause.ts), is held
8
+ * instead. Permanent holds need a restart request, command change or CLI update.
9
+ * Exhausted transient sync/mesh units make one durable supervised probe after
10
+ * a 30-minute cooldown; a failed probe re-holds, and an explicit stop stays stopped;
6
11
  * `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
7
12
  * outside, and any unit stays down after a stop request (`hq bot stop`),
8
13
  * until a restart request;
@@ -22,11 +27,20 @@
22
27
  *
23
28
  * Processes, time, and state output are injected; `run.ts` wires the real ones.
24
29
  */
30
+ import { createHash } from "node:crypto";
25
31
  import { StringDecoder } from "node:string_decoder";
32
+ import { classifyExit } from "./exit-cause.js";
26
33
  import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
27
34
  const BACKOFF_BASE_MS = 1_000;
28
35
  const BACKOFF_MAX_MS = 60_000;
29
- const HEALTHY_RUN_MS = 60_000;
36
+ export const HEALTHY_RUN_MS = 60_000;
37
+ /**
38
+ * Failures in one episode before the unit is held: the first start plus five
39
+ * automatic retries. Matches the Desktop app's MAX_FAILURES.
40
+ */
41
+ export const EPISODE_MAX_FAILURES = 6;
42
+ /** A run this long ends the failure episode. Matches the Desktop app's HEALTHY_RUN_WINDOW. */
43
+ export const EPISODE_HEALTHY_RESET_MS = 30 * 60_000;
30
44
  export const STOP_ESCALATION_MS = 10_000;
31
45
  export const SYNC_RUNNER_PERSIST_GRACE_MS = 60_000;
32
46
  const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
@@ -69,6 +83,17 @@ const BLOCKED_READS_BEFORE_PAUSE = 2;
69
83
  function backoffMs(attempt) {
70
84
  return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
71
85
  }
86
+ function keyHash(key) {
87
+ return createHash("sha256").update(key).digest("hex");
88
+ }
89
+ function recoverableHold(plan, hold) {
90
+ return plan.kind === "long-running" && (plan.service === "sync" || plan.service === "mesh")
91
+ && hold?.kind === "exhausted" && hold.cause === "transient";
92
+ }
93
+ function heldReason(unitId, hold) {
94
+ const what = hold.kind === "permanent" ? `${hold.cause}: ${hold.detail}` : hold.detail;
95
+ return `held (${what}); \`hq daemon restart ${unitId}\` retries`;
96
+ }
72
97
  /**
73
98
  * What a unit's process was started from; a unit whose key changes is
74
99
  * restarted. The CLI version counts only for `hq` commands, so a CLI update
@@ -109,6 +134,15 @@ export function createSupervisor(deps) {
109
134
  const reconcileMs = deps.reconcileMs ?? 15_000;
110
135
  const requestPollMs = deps.requestPollMs ?? 1_000;
111
136
  const startedAt = now();
137
+ let persistedHolds = {};
138
+ /** Holds dropped at plan time (a changed command) that holds.json still lists. */
139
+ let holdsDirty = false;
140
+ try {
141
+ persistedHolds = deps.holds?.read() ?? {};
142
+ }
143
+ catch (err) {
144
+ deps.log("error", `could not read unit holds; starting without them: ${errorMessage(err)}`);
145
+ }
112
146
  let slots = new Map();
113
147
  let stopped = false;
114
148
  let started = false;
@@ -144,7 +178,13 @@ export function createSupervisor(deps) {
144
178
  deps.log("error", `could not write daemon state: ${errorMessage(err)}`);
145
179
  }
146
180
  };
181
+ const clearRecoveryWindow = (slot) => {
182
+ if (slot.recoveryHealthyTimer !== undefined)
183
+ clearT(slot.recoveryHealthyTimer);
184
+ slot.recoveryHealthyTimer = undefined;
185
+ };
147
186
  const clearTimer = (slot) => {
187
+ clearRecoveryWindow(slot);
148
188
  if (slot.timer !== undefined)
149
189
  clearT(slot.timer);
150
190
  slot.timer = undefined;
@@ -269,6 +309,89 @@ export function createSupervisor(deps) {
269
309
  delete slot.state.pid;
270
310
  delete slot.state.nextRunAt;
271
311
  };
312
+ const writeHolds = () => {
313
+ if (!deps.holds)
314
+ return false;
315
+ const next = {};
316
+ // Keep holds for units not planned right now (for example a paused bot).
317
+ for (const [id, hold] of Object.entries(persistedHolds))
318
+ if (!slots.has(id))
319
+ next[id] = hold;
320
+ for (const [id, slot] of slots)
321
+ if (slot.state.hold)
322
+ next[id] = slot.state.hold;
323
+ persistedHolds = next;
324
+ holdsDirty = false;
325
+ try {
326
+ deps.holds.write(next);
327
+ return true;
328
+ }
329
+ catch (err) {
330
+ deps.log("error", `could not write unit holds: ${errorMessage(err)}`);
331
+ return false;
332
+ }
333
+ };
334
+ const showHeld = (slot) => {
335
+ if (recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.retryStopped === true) {
336
+ slot.state.status = "exited";
337
+ slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
338
+ delete slot.state.pid;
339
+ delete slot.state.nextRunAt;
340
+ return;
341
+ }
342
+ slot.state.status = "held";
343
+ slot.state.reason = heldReason(slot.plan.id, slot.state.hold);
344
+ delete slot.state.pid;
345
+ delete slot.state.nextRunAt;
346
+ if (recoverableHold(slot.plan, slot.state.hold) && Number.isSafeInteger(slot.state.hold.retryAt)
347
+ && slot.state.hold.retryAt >= 0 && slot.state.hold.retryAt <= 8_640_000_000_000_000) {
348
+ slot.state.nextRunAt = slot.state.hold.retryAt;
349
+ slot.state.reason += `; one supervised retry at ${new Date(slot.state.hold.retryAt).toISOString()}`;
350
+ }
351
+ };
352
+ const holdUnit = (slot, kind, cause, detail) => {
353
+ clearTimer(slot);
354
+ slot.state.hold = {
355
+ kind,
356
+ cause,
357
+ detail,
358
+ since: now(),
359
+ failures: slot.failures,
360
+ ...(slot.state.lastExit ? { lastExit: { ...slot.state.lastExit } } : {}),
361
+ keyHash: keyHash(slot.key),
362
+ };
363
+ if (deps.holds && recoverableHold(slot.plan, slot.state.hold)) {
364
+ slot.state.hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
365
+ }
366
+ persistedHolds[slot.plan.id] = slot.state.hold;
367
+ showHeld(slot);
368
+ deps.log("error", `holding ${slot.plan.id}: ${slot.state.reason}`);
369
+ writeHolds();
370
+ armRecoveryProbe(slot);
371
+ };
372
+ const clearHold = (slot) => {
373
+ clearRecoveryWindow(slot);
374
+ slot.attempts = 0;
375
+ slot.failures = 0;
376
+ if (!slot.state.hold && !persistedHolds[slot.plan.id])
377
+ return;
378
+ delete slot.state.hold;
379
+ if (slot.state.status === "held") {
380
+ slot.state.status = "idle";
381
+ delete slot.state.reason;
382
+ }
383
+ delete persistedHolds[slot.plan.id];
384
+ writeHolds();
385
+ };
386
+ /** Hold the unit when its exit has a permanent cause; true when it was held. */
387
+ const holdIfPermanent = (slot, evidence) => {
388
+ const cause = classifyExit(evidence);
389
+ if (cause.kind !== "permanent")
390
+ return false;
391
+ slot.failures += 1;
392
+ holdUnit(slot, "permanent", cause.cause, cause.detail);
393
+ return true;
394
+ };
272
395
  const signalChild = (slot, child, signal) => {
273
396
  try {
274
397
  child.kill(signal);
@@ -344,6 +467,11 @@ export function createSupervisor(deps) {
344
467
  const scheduleRestart = (slot) => {
345
468
  if (stopped)
346
469
  return;
470
+ slot.failures += 1;
471
+ if (slot.failures >= EPISODE_MAX_FAILURES) {
472
+ holdUnit(slot, "exhausted", "transient", `${slot.failures} failures without staying up for ${EPISODE_HEALTHY_RESET_MS / 60_000} min`);
473
+ return;
474
+ }
347
475
  const delay = backoffMs(slot.attempts);
348
476
  slot.attempts += 1;
349
477
  slot.state.restarts += 1;
@@ -469,12 +597,19 @@ export function createSupervisor(deps) {
469
597
  clearT(slot.killTimer);
470
598
  slot.killTimer = undefined;
471
599
  }
600
+ clearRecoveryWindow(slot);
472
601
  const ranFor = now() - (slot.childStartedAt ?? now());
473
602
  slot.state.lastExit = { code, signal, at: now() };
474
603
  delete slot.state.pid;
475
604
  if (slot.expectingExit) {
476
605
  slot.expectingExit = false;
477
606
  slot.busy = false;
607
+ // A unit we stopped (for example a stalled sync runner) after a healthy
608
+ // run starts a new failure episode, as an unexpected exit would.
609
+ if (ranFor >= HEALTHY_RUN_MS)
610
+ slot.attempts = 0;
611
+ if (ranFor >= EPISODE_HEALTHY_RESET_MS)
612
+ slot.failures = 0;
478
613
  const then = slot.afterExit;
479
614
  slot.afterExit = undefined;
480
615
  then?.();
@@ -492,6 +627,17 @@ export function createSupervisor(deps) {
492
627
  emit();
493
628
  return;
494
629
  }
630
+ const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
631
+ const recoveryOwnerReason = slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT
632
+ ? deps.syncOwnerBlockReason?.() ?? "sync owned by another runner"
633
+ : meshOwnerReason;
634
+ if (recoveryOwnerReason && recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.probing === true) {
635
+ // The ordinary ownership retry is 15 s. A recovery probe must not use it
636
+ // to bypass its persisted cooldown and start a second probe early.
637
+ holdUnit(slot, "exhausted", "transient", `recovery probe deferred: ${recoveryOwnerReason}`);
638
+ emit();
639
+ return;
640
+ }
495
641
  if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
496
642
  const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
497
643
  slot.state.status = "waiting";
@@ -506,7 +652,6 @@ export function createSupervisor(deps) {
506
652
  emit();
507
653
  return;
508
654
  }
509
- const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
510
655
  if (meshOwnerReason) {
511
656
  let lastLoggedReason;
512
657
  let lastLoggedAt = Number.NEGATIVE_INFINITY;
@@ -541,6 +686,8 @@ export function createSupervisor(deps) {
541
686
  deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
542
687
  if (ranFor >= HEALTHY_RUN_MS)
543
688
  slot.attempts = 0;
689
+ if (ranFor >= EPISODE_HEALTHY_RESET_MS)
690
+ slot.failures = 0;
544
691
  const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
545
692
  if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
546
693
  slot.state.status = "exited";
@@ -550,7 +697,11 @@ export function createSupervisor(deps) {
550
697
  delete slot.state.nextRunAt;
551
698
  }
552
699
  else {
553
- scheduleRestart(slot);
700
+ const spawnError = slot.lastSpawnError;
701
+ delete slot.lastSpawnError;
702
+ const outputTail = spawnError === undefined ? readExitOutputTail(slot) : undefined;
703
+ if (!holdIfPermanent(slot, { code, signal, spawnError, outputTail }))
704
+ scheduleRestart(slot);
554
705
  }
555
706
  emit();
556
707
  };
@@ -559,10 +710,22 @@ export function createSupervisor(deps) {
559
710
  deps.log("error", `${slot.plan.id}: ${errorMessage(err)}`);
560
711
  // A child that never got a pid failed to launch and may never emit
561
712
  // "exit". Any other error (for example a failed kill) leaves it running.
562
- if (child.pid === undefined)
713
+ if (child.pid === undefined) {
714
+ if (slot.child === child)
715
+ slot.lastSpawnError = err;
563
716
  onEnd(null, null);
717
+ }
564
718
  });
565
719
  };
720
+ const readExitOutputTail = (slot) => {
721
+ try {
722
+ return deps.exitOutputTail?.(slot.plan.id);
723
+ }
724
+ catch (err) {
725
+ deps.log("warn", `could not read ${slot.plan.id} output to classify its exit: ${errorMessage(err)}`);
726
+ return undefined;
727
+ }
728
+ };
566
729
  const startLong = (slot) => {
567
730
  if (stopped)
568
731
  return;
@@ -582,7 +745,8 @@ export function createSupervisor(deps) {
582
745
  catch (err) {
583
746
  slot.state.reason = `failed to start: ${errorMessage(err)}`;
584
747
  deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
585
- scheduleRestart(slot);
748
+ if (!holdIfPermanent(slot, { code: null, signal: null, spawnError: err }))
749
+ scheduleRestart(slot);
586
750
  return;
587
751
  }
588
752
  slot.child = child;
@@ -598,6 +762,57 @@ export function createSupervisor(deps) {
598
762
  else
599
763
  delete slot.state.pid;
600
764
  watch(slot, child);
765
+ armRecoveryHealthyWindow(slot, child);
766
+ };
767
+ const armRecoveryHealthyWindow = (slot, child) => {
768
+ clearRecoveryWindow(slot);
769
+ const hold = slot.state.hold;
770
+ if (!recoverableHold(slot.plan, hold) || hold.probing !== true || hold.retryStopped === true)
771
+ return;
772
+ // This timer belongs to this continuously running child, never to time spent held.
773
+ slot.recoveryHealthyTimer = setT(() => {
774
+ slot.recoveryHealthyTimer = undefined;
775
+ if (stopped || slot.child !== child || slot.state.hold !== hold || slot.expectingExit)
776
+ return;
777
+ clearHold(slot);
778
+ emit();
779
+ }, EPISODE_HEALTHY_RESET_MS);
780
+ };
781
+ const armRecoveryProbe = (slot) => {
782
+ const hold = slot.state.hold;
783
+ if (!recoverableHold(slot.plan, hold) || hold.retryStopped === true || !deps.holds
784
+ || stopped || slot.child || slot.plan.blocked)
785
+ return;
786
+ clearTimer(slot);
787
+ const at = now();
788
+ // Legacy/corrupt deadlines and a clock moving backwards get one full cooldown,
789
+ // rather than an immediate retry or an unbounded wait.
790
+ if (!Number.isSafeInteger(hold.retryAt) || hold.retryAt < 0 || hold.retryAt > at + EPISODE_HEALTHY_RESET_MS) {
791
+ hold.retryAt = at + EPISODE_HEALTHY_RESET_MS;
792
+ persistedHolds[slot.plan.id] = hold;
793
+ writeHolds();
794
+ }
795
+ showHeld(slot);
796
+ slot.timer = setT(() => {
797
+ slot.timer = undefined;
798
+ if (stopped || slots.get(slot.plan.id) !== slot || slot.child || slot.plan.blocked
799
+ || slot.state.hold !== hold || hold.retryStopped === true)
800
+ return;
801
+ // Advance the durable claim BEFORE spawning. A daemon crash during a probe
802
+ // cannot turn its old overdue deadline into a restart storm.
803
+ hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
804
+ hold.probing = true;
805
+ persistedHolds[slot.plan.id] = hold;
806
+ if (!writeHolds()) {
807
+ delete hold.probing;
808
+ armRecoveryProbe(slot);
809
+ emit();
810
+ return;
811
+ }
812
+ slot.state.restarts += 1;
813
+ startLong(slot);
814
+ emit();
815
+ }, Math.max(0, hold.retryAt - at));
601
816
  };
602
817
  const scheduleNextRun = (slot, delayMs) => {
603
818
  clearTimer(slot);
@@ -703,23 +918,47 @@ export function createSupervisor(deps) {
703
918
  scheduleNextRun(slot, delay);
704
919
  return;
705
920
  }
921
+ if (slot.state.hold) {
922
+ showHeld(slot);
923
+ armRecoveryProbe(slot);
924
+ return;
925
+ }
706
926
  startLong(slot);
707
927
  };
708
- const newSlot = (plan) => ({
709
- plan,
710
- key: planKey(plan, deps.cliVersion),
711
- attempts: 0,
712
- expectingExit: false,
713
- busy: false,
714
- state: {
715
- id: plan.id,
716
- service: plan.service,
717
- kind: plan.kind,
718
- description: plan.description,
719
- status: "idle",
720
- restarts: 0,
721
- },
722
- });
928
+ const newSlot = (plan) => {
929
+ const key = planKey(plan, deps.cliVersion);
930
+ const slot = {
931
+ plan,
932
+ key,
933
+ attempts: 0,
934
+ failures: 0,
935
+ expectingExit: false,
936
+ busy: false,
937
+ state: {
938
+ id: plan.id,
939
+ service: plan.service,
940
+ kind: plan.kind,
941
+ description: plan.description,
942
+ status: "idle",
943
+ restarts: 0,
944
+ },
945
+ };
946
+ const saved = persistedHolds[plan.id];
947
+ if (saved && plan.kind === "long-running") {
948
+ if (saved.keyHash === keyHash(key) || (recoverableHold(plan, saved) && saved.retryStopped === true)) {
949
+ slot.state.hold = saved;
950
+ slot.failures = saved.failures;
951
+ deps.log("info", `${plan.id} stays held from an earlier daemon: ${heldReason(plan.id, saved)}`);
952
+ }
953
+ else {
954
+ // The command changed (for example a CLI update), so the old cause may be fixed.
955
+ delete persistedHolds[plan.id];
956
+ holdsDirty = true;
957
+ deps.log("info", `releasing the hold on ${plan.id}: its command changed`);
958
+ }
959
+ }
960
+ return slot;
961
+ };
723
962
  const reconcile = () => {
724
963
  if (stopped)
725
964
  return;
@@ -774,6 +1013,12 @@ export function createSupervisor(deps) {
774
1013
  const changed = key !== existing.key;
775
1014
  existing.plan = plan;
776
1015
  existing.key = key;
1016
+ if (changed)
1017
+ existing.failures = 0;
1018
+ if (changed && existing.state.hold && existing.state.hold.retryStopped !== true) {
1019
+ deps.log("info", `releasing the hold on ${plan.id}: its command changed`);
1020
+ clearHold(existing);
1021
+ }
777
1022
  existing.state.description = plan.description;
778
1023
  existing.state.kind = plan.kind;
779
1024
  if (plan.blocked) {
@@ -786,6 +1031,11 @@ export function createSupervisor(deps) {
786
1031
  else if (wasBlocked) {
787
1032
  deps.log("info", `starting ${plan.id}: its requirement is now met`);
788
1033
  existing.attempts = 0;
1034
+ if (existing.state.hold && !changed) {
1035
+ // Still the same command that was held; keep it held.
1036
+ activate(existing);
1037
+ continue;
1038
+ }
789
1039
  // The copy stopped when it was paused may still be exiting; start the
790
1040
  // new one only after it is gone, so two never run side by side.
791
1041
  if (existing.child)
@@ -836,6 +1086,8 @@ export function createSupervisor(deps) {
836
1086
  }
837
1087
  }
838
1088
  slots = next;
1089
+ if (holdsDirty)
1090
+ writeHolds();
839
1091
  emit();
840
1092
  pumpHandoff();
841
1093
  };
@@ -911,6 +1163,16 @@ export function createSupervisor(deps) {
911
1163
  }
912
1164
  // Recorded as runnable, so the first plan read pauses it when it is now blocked.
913
1165
  const slot = newSlot({ ...plan, blocked: undefined });
1166
+ const manualRecoveryStop = recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.retryStopped === true;
1167
+ const keepProbe = recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.probing === true
1168
+ && slot.state.hold.retryStopped !== true && slot.state.hold.keyHash === keyHash(unit.key);
1169
+ if (!keepProbe && !manualRecoveryStop && (slot.state.hold || persistedHolds[unit.id])) {
1170
+ // A running copy drops stale holds, except a claimed recovery probe whose
1171
+ // failure episode must survive the daemon handoff.
1172
+ delete slot.state.hold;
1173
+ delete persistedHolds[unit.id];
1174
+ holdsDirty = true;
1175
+ }
914
1176
  slot.key = unit.key;
915
1177
  slot.adopted = true;
916
1178
  slot.child = unit.child;
@@ -922,7 +1184,10 @@ export function createSupervisor(deps) {
922
1184
  if (unit.child.pid !== undefined)
923
1185
  slot.state.pid = unit.child.pid;
924
1186
  watch(slot, unit.child);
1187
+ armRecoveryHealthyWindow(slot, unit.child);
925
1188
  slots.set(unit.id, slot);
1189
+ if (manualRecoveryStop)
1190
+ terminate(slot, () => showHeld(slot));
926
1191
  deps.log("info", `adopted ${unit.id} (pid ${unit.child.pid ?? "?"}) from the previous daemon`);
927
1192
  }
928
1193
  };
@@ -972,6 +1237,15 @@ export function createSupervisor(deps) {
972
1237
  }
973
1238
  if (req.action === "stop") {
974
1239
  deps.log("info", `stopping ${slot.plan.id} (requested)`);
1240
+ if (recoverableHold(slot.plan, slot.state.hold)) {
1241
+ slot.state.hold.retryStopped = true;
1242
+ delete slot.state.hold.retryAt;
1243
+ delete slot.state.hold.probing;
1244
+ persistedHolds[slot.plan.id] = slot.state.hold;
1245
+ writeHolds();
1246
+ }
1247
+ else
1248
+ clearHold(slot);
975
1249
  // An expected exit is never restarted, including one that needed SIGKILL.
976
1250
  terminate(slot, () => {
977
1251
  slot.state.status = "exited";
@@ -995,7 +1269,9 @@ export function createSupervisor(deps) {
995
1269
  continue;
996
1270
  }
997
1271
  deps.log("info", `restarting ${slot.plan.id} (requested)`);
998
- slot.attempts = 0;
1272
+ if (slot.state.hold)
1273
+ deps.log("info", `releasing the hold on ${slot.plan.id} (requested)`);
1274
+ clearHold(slot);
999
1275
  if (slot.plan.kind === "scheduled")
1000
1276
  continue;
1001
1277
  if (slot.child)
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Host-side storage for the supervisor's unit holds and the per-run output
3
+ * read used to classify a unit's exit.
4
+ *
5
+ * holds.json (`DaemonPaths.holdsPath`, ~/.hq/daemon/holds.json) maps a unit id
6
+ * to its UnitHold. The daemon rewrites it whenever a hold starts or clears; a
7
+ * missing or unreadable file means no holds.
8
+ */
9
+ import type { PersistedHolds } from "./supervisor.js";
10
+ /** How much of a run's output the exit classifier sees. */
11
+ export declare const EXIT_OUTPUT_TAIL_BYTES: number;
12
+ export declare function readUnitHolds(file: string): PersistedHolds;
13
+ export declare function writeUnitHolds(file: string, holds: PersistedHolds): void;
14
+ /** The size of a log file now, so a later read sees only what was written after. */
15
+ export declare function logOffset(file: string): number;
16
+ /**
17
+ * What was appended to `file` since `offset`, at most the last `maxBytes`.
18
+ * A file smaller than `offset` was rotated (copy-and-truncate), so it is read
19
+ * from the start.
20
+ */
21
+ export declare function readLogSince(file: string, offset: number, maxBytes?: number): string;
22
+ //# sourceMappingURL=unit-holds.d.ts.map
@@ -0,0 +1,86 @@
1
+ /**
2
+ * Host-side storage for the supervisor's unit holds and the per-run output
3
+ * read used to classify a unit's exit.
4
+ *
5
+ * holds.json (`DaemonPaths.holdsPath`, ~/.hq/daemon/holds.json) maps a unit id
6
+ * to its UnitHold. The daemon rewrites it whenever a hold starts or clears; a
7
+ * missing or unreadable file means no holds.
8
+ */
9
+ import * as fs from "node:fs";
10
+ import * as path from "node:path";
11
+ import { ensurePrivateDir } from "./paths.js";
12
+ /** How much of a run's output the exit classifier sees. */
13
+ export const EXIT_OUTPUT_TAIL_BYTES = 16 * 1024;
14
+ function isHold(value) {
15
+ if (!value || typeof value !== "object")
16
+ return false;
17
+ const h = value;
18
+ return ((h.kind === "permanent" || h.kind === "exhausted")
19
+ && typeof h.cause === "string"
20
+ && typeof h.detail === "string"
21
+ && typeof h.since === "number"
22
+ && typeof h.failures === "number"
23
+ && typeof h.keyHash === "string");
24
+ }
25
+ export function readUnitHolds(file) {
26
+ let raw;
27
+ try {
28
+ raw = fs.readFileSync(file, "utf8");
29
+ }
30
+ catch (err) {
31
+ if (err.code === "ENOENT")
32
+ return {};
33
+ throw err;
34
+ }
35
+ const parsed = JSON.parse(raw);
36
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
37
+ return {};
38
+ const holds = {};
39
+ for (const [id, hold] of Object.entries(parsed))
40
+ if (isHold(hold))
41
+ holds[id] = hold;
42
+ return holds;
43
+ }
44
+ export function writeUnitHolds(file, holds) {
45
+ ensurePrivateDir(path.dirname(file));
46
+ const tmp = `${file}.${process.pid}.tmp`;
47
+ fs.writeFileSync(tmp, `${JSON.stringify(holds, null, 2)}\n`, { mode: 0o600 });
48
+ fs.renameSync(tmp, file);
49
+ }
50
+ /** The size of a log file now, so a later read sees only what was written after. */
51
+ export function logOffset(file) {
52
+ try {
53
+ return fs.statSync(file).size;
54
+ }
55
+ catch {
56
+ return 0;
57
+ }
58
+ }
59
+ /**
60
+ * What was appended to `file` since `offset`, at most the last `maxBytes`.
61
+ * A file smaller than `offset` was rotated (copy-and-truncate), so it is read
62
+ * from the start.
63
+ */
64
+ export function readLogSince(file, offset, maxBytes = EXIT_OUTPUT_TAIL_BYTES) {
65
+ let fd;
66
+ try {
67
+ fd = fs.openSync(file, "r");
68
+ }
69
+ catch {
70
+ return "";
71
+ }
72
+ try {
73
+ const size = fs.fstatSync(fd).size;
74
+ const from = Math.max(size < offset ? 0 : offset, size - maxBytes);
75
+ const length = size - from;
76
+ if (length <= 0)
77
+ return "";
78
+ const buf = Buffer.alloc(length);
79
+ const read = fs.readSync(fd, buf, 0, length, from);
80
+ return buf.subarray(0, read).toString("utf8");
81
+ }
82
+ finally {
83
+ fs.closeSync(fd);
84
+ }
85
+ }
86
+ //# sourceMappingURL=unit-holds.js.map
@@ -100,6 +100,15 @@ export interface SyncHealthDeps {
100
100
  * unhealthy-sync signal, and `hq doctor --fix` has nothing local to refresh.
101
101
  */
102
102
  localCompanyExists?: (slug: string) => boolean;
103
+ /**
104
+ * Whether the sync service can resolve the company for this machine's
105
+ * identity. Defaults to true unless a recent `hq doctor --fix` pull of it
106
+ * failed because the service has no such company for the caller or refused
107
+ * access (see company-resolution-state.ts). A stale journal for a company
108
+ * that fails this has nothing `--fix` can pull, so it is NA ("not on this
109
+ * machine") rather than an auto-fixable WARN.
110
+ */
111
+ companySyncResolvable?: (slug: string, lastSync?: string | null) => boolean;
103
112
  /**
104
113
  * Whether HQ_SYNC_MANIFEST_DISABLED turns manifest uploads off on this
105
114
  * machine. Defaults to reading the environment the way `hq sync manifest`