@indigoai-us/hq-cli 5.345.174 → 5.345.176

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/dist/command-catalog.generated.d.ts +65 -0
  3. package/dist/command-catalog.generated.js +80 -0
  4. package/dist/commands/bot.d.ts +28 -2
  5. package/dist/commands/bot.js +32 -6
  6. package/dist/commands/daemon.js +16 -0
  7. package/dist/commands/dm.d.ts +7 -0
  8. package/dist/commands/dm.js +16 -4
  9. package/dist/commands/lanes-loop.d.ts +4 -1
  10. package/dist/commands/lanes-loop.js +23 -4
  11. package/dist/commands/lanes-shared.d.ts +5 -0
  12. package/dist/commands/lanes.d.ts +5 -7
  13. package/dist/commands/lanes.js +65 -5
  14. package/dist/commands/sync-core.d.ts +17 -0
  15. package/dist/commands/sync-core.js +46 -5
  16. package/dist/commands/sync-manifest.d.ts +6 -0
  17. package/dist/commands/sync-manifest.js +13 -0
  18. package/dist/hq-cloud-sync-runner/bin/sync-runner.js +448 -226
  19. package/dist/hq-cloud-sync-runner/package.json +1 -1
  20. package/dist/lib/bot/cli-update.d.ts +36 -0
  21. package/dist/lib/bot/cli-update.js +160 -42
  22. package/dist/lib/bot/run.d.ts +23 -0
  23. package/dist/lib/bot/run.js +174 -23
  24. package/dist/lib/bot/runtime/claude.d.ts +7 -1
  25. package/dist/lib/bot/runtime/claude.js +29 -0
  26. package/dist/lib/bot/runtime/codex.d.ts +5 -0
  27. package/dist/lib/bot/runtime/codex.js +13 -0
  28. package/dist/lib/bot/runtime/index.js +18 -2
  29. package/dist/lib/bot/runtime/types.d.ts +23 -0
  30. package/dist/lib/bot/runtime/types.js +13 -0
  31. package/dist/lib/bot/runtime-failure.d.ts +17 -6
  32. package/dist/lib/bot/runtime-failure.js +54 -9
  33. package/dist/lib/bot/runtime-sign-in-check.d.ts +32 -0
  34. package/dist/lib/bot/runtime-sign-in-check.js +53 -0
  35. package/dist/lib/bot/runtime-sign-in.d.ts +32 -3
  36. package/dist/lib/bot/runtime-sign-in.js +17 -2
  37. package/dist/lib/bot/status.d.ts +35 -1
  38. package/dist/lib/bot/status.js +28 -0
  39. package/dist/lib/daemon/catalog.d.ts +4 -0
  40. package/dist/lib/daemon/catalog.js +47 -19
  41. package/dist/lib/daemon/run.js +1 -0
  42. package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
  43. package/dist/lib/flag-registry-worker-source.generated.js +1 -1
  44. package/dist/lib/lanes/loop-disposition.d.ts +44 -0
  45. package/dist/lib/lanes/loop-disposition.js +302 -0
  46. package/dist/lib/lanes/loop.d.ts +16 -3
  47. package/dist/lib/lanes/loop.js +167 -60
  48. package/dist/lib/lanes/reconcile.d.ts +5 -0
  49. package/dist/lib/lanes/reconcile.js +79 -4
  50. package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +3 -0
  51. package/dist/lib/mesh/live/daemon/work-subscribe.js +41 -12
  52. package/dist/lib/sync-core/engine-home.d.ts +11 -4
  53. package/dist/lib/sync-core/engine-home.js +22 -21
  54. package/dist/lib/sync-core/manifest-scopes.d.ts +15 -0
  55. package/dist/lib/sync-core/manifest-scopes.js +26 -0
  56. package/dist/lib/sync-core/reconcile.d.ts +2 -2
  57. package/dist/lib/sync-core/reconcile.js +70 -46
  58. package/dist/lib/sync-core/release.d.ts +2 -2
  59. package/dist/lib/sync-core/release.js +31 -6
  60. package/dist/lib/sync-core/rollout-state.d.ts +6 -3
  61. package/dist/lib/sync-core/rollout-state.js +26 -22
  62. package/package.json +8 -8
@@ -50,6 +50,7 @@ import { ownerContextBlock } from "./owner-context.js";
50
50
  import { randomUUID } from "node:crypto";
51
51
  import { hasPromotionHold } from "./promotion-hold.js";
52
52
  import * as fs from "node:fs";
53
+ import * as os from "node:os";
53
54
  import * as path from "node:path";
54
55
  import { fullJitterDelayMs } from "../mesh/live/backoff.js";
55
56
  import { acquirePidLock, defaultPidLockDeps, releasePidLock } from "../mesh/live/daemon/pid-lock.js";
@@ -82,14 +83,15 @@ import { resolveRoomMentionsForBot, roomPostHasMentionTokens, } from "./room-men
82
83
  import { buildJobTurnPrompt, isCatchUpRun, isJobItem, jobReportHeader, jobRunOutput, machineTimeZone, planJobRuns, stripJobHeader, SUPERSEDED_DISPOSITION } from "./job-run.js";
83
84
  import { botJobQuietOptionEnabled } from "./job-quiet-flag.js";
84
85
  import { clearPendingJobCompletion, isCompletionDue, isCompletionExpired, isCompletionForOtherAgent, readPendingJobCompletions, recordPendingJobCompletion } from "./job-completions.js";
85
- import { SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
86
+ import { SIGN_IN_CHECK_FRESH_MS, SIGN_IN_CHECK_MS, SIGN_IN_RETRY_MS, signInNeededText, signedOutExcerpt, } from "./runtime-sign-in.js";
87
+ import { checkRuntimeSignIn } from "./runtime-sign-in-check.js";
86
88
  import { classifyRuntimeError, classifyRuntimeFailure, runtimeDidNotFinishText, runtimeFailureReason, runtimeFailureText, runtimeRepairNotice, withRetrySuggestion, } from "./runtime-failure.js";
87
89
  import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
88
90
  import { captureHeartbeatIdentityMismatch, identityMismatchMessage, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
89
91
  import { clearBotSession, ensureBotCompanyScope, readBotSession, writeBotSession } from "./session.js";
90
92
  import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
91
93
  import { CLI_VERSION } from "../../cli-version.js";
92
- import { CLI_UPDATE_RESTART_EXIT_CODE } from "./cli-update.js";
94
+ import { CLI_UPDATE_RESTART_EXIT_CODE, describeCliUpdate } from "./cli-update.js";
93
95
  import { createInboxMqttWakeFlagReader, createInboxPollBackoffFlagReader, safeErrorClass } from "./inbox-poll-backoff-flag.js";
94
96
  import { createInboxWakeWatcher } from "./inbox-wake.js";
95
97
  import { workMeshRoot } from "../mesh/live/paths.js";
@@ -129,6 +131,8 @@ export const FAILURE_WINDOW_MS = 10 * 60 * 1000;
129
131
  export const FAILURE_LIMIT = 5;
130
132
  /** Resumed-session failures with the model CLI's unknown error, in a row, before the session is cleared. */
131
133
  export const RESUMED_UNKNOWN_ERROR_LIMIT = 2;
134
+ /** Wait before the one automatic re-run of a turn whose runtime crashed before it started. */
135
+ export const RUNTIME_CRASH_RETRY_DELAY_MS = 3_000;
132
136
  /** The model CLI's own message, exactly; "Unexpected token" in a stack trace is not it. */
133
137
  const RESUMED_UNKNOWN_ERROR = /an unknown error occurred \(unexpected\)/i;
134
138
  export const MAX_ATTEMPTS_PER_MESSAGE = 3;
@@ -193,6 +197,29 @@ export function turnEnv(base, kind, agentUid, companySlug) {
193
197
  }
194
198
  return env;
195
199
  }
200
+ /** The account name this process runs as, or null when the OS cannot say. */
201
+ export function osUserName() {
202
+ try {
203
+ const name = os.userInfo().username?.trim();
204
+ return name || null;
205
+ }
206
+ catch {
207
+ return null;
208
+ }
209
+ }
210
+ /**
211
+ * Claude Code keys its macOS Keychain login on `$USER` (falling back to the
212
+ * literal "unknown"), so a tool spawned from an environment without USER
213
+ * reads a different Keychain item than the owner's terminal and is signed
214
+ * out. launchd and the HQ app's launcher may leave it unset; fill it (and
215
+ * LOGNAME) from the account this process runs as. Never overrides a USER
216
+ * already set. Pure.
217
+ */
218
+ export function withOsUser(env, name = osUserName()) {
219
+ if (env.USER?.trim() || !name)
220
+ return env;
221
+ return { ...env, USER: name, ...(env.LOGNAME?.trim() ? {} : { LOGNAME: name }) };
222
+ }
196
223
  /** The core worker HQ's setup bot runs. */
197
224
  export const SETUP_WORKER_ID = SETUP_BOT_WORKER_ID;
198
225
  /**
@@ -493,7 +520,7 @@ export async function runBot(deps) {
493
520
  cliUpdatePending = update;
494
521
  stopping = true;
495
522
  const busy = running.size;
496
- log("info", `hq-cli updated ${update.from} -> ${update.to}; restarting to load it${busy ? ` after ${busy} running turn${busy === 1 ? "" : "s"} finish${busy === 1 ? "es" : ""}` : ""}`);
523
+ log("info", `${describeCliUpdate(update)}${busy ? ` after ${busy} running turn${busy === 1 ? "" : "s"} finish${busy === 1 ? "es" : ""}` : ""}`);
497
524
  return true;
498
525
  };
499
526
  // ── Preflight: creds, pid lock, owner verification ─────────────────────────
@@ -693,7 +720,7 @@ export async function runBot(deps) {
693
720
  // nothing while their next messages wait out the hold.
694
721
  if (tell) {
695
722
  try {
696
- await tell(holdNoticeText(modelHoldMs, messagePending, cls ? runtimeFailureText(cls, runtime.id) : undefined));
723
+ await tell(holdNoticeText(modelHoldMs, messagePending, cls ? runtimeFailureText(cls, runtime.id, "message", config.name) : undefined));
697
724
  }
698
725
  catch (err) {
699
726
  log("warn", `could not say that turns are on hold: ${err instanceof Error ? err.message : String(err)}`);
@@ -712,14 +739,17 @@ export async function runBot(deps) {
712
739
  /** Conversations already told this time that the bot needs a sign-in. */
713
740
  const toldAboutSignIn = new Set();
714
741
  const isSignInFailure = (excerpt) => classifyRuntimeFailure(excerpt) === "signed-out";
715
- const markSignInBroken = (excerpt) => {
716
- nextSignInRetryAt = now().getTime() + signInRetryMs;
742
+ const markSignInBroken = (excerpt, state = "expired", bump = true) => {
743
+ if (bump)
744
+ nextSignInRetryAt = now().getTime() + signInRetryMs;
717
745
  modelHealth = "degraded";
718
746
  if (signInIssue)
719
747
  return;
720
- signInIssue = { state: "expired", runtime: config.runtime, since: now().toISOString() };
748
+ signInIssue = { state, runtime: config.runtime, since: now().toISOString() };
721
749
  setStatus({ runtimeSignIn: signInIssue, lastError: excerpt.slice(0, 400) });
722
- log("warn", `${runtime.id} needs to sign in again on this computer; holding turns and retrying every ${Math.round(signInRetryMs / 1000)}s`);
750
+ log("warn", state === "signed-out"
751
+ ? `${runtime.id} is not signed in on this computer (its own check); holding turns and checking every ${Math.round(signInCheckMs / 1000)}s`
752
+ : `${runtime.id} needs to sign in again on this computer; holding turns and retrying every ${Math.round(signInRetryMs / 1000)}s`);
723
753
  };
724
754
  const markSignInWorks = () => {
725
755
  if (!signInIssue)
@@ -729,6 +759,70 @@ export async function runBot(deps) {
729
759
  setStatus({ runtimeSignIn: null });
730
760
  log("info", `${runtime.id} sign-in works again; resuming`);
731
761
  };
762
+ // ── The bot's own sign-in check ────────────────────────────────────────────
763
+ // The tool is asked in the turn's environment (not the owner's shell): at
764
+ // start, before a turn, and on a timer while it says signed out, so a
765
+ // message is never spent on a turn that cannot start and the app can read
766
+ // the answer before it sends a job.
767
+ const signInCheckMs = Math.max(0, deps.signInCheckMs ?? SIGN_IN_CHECK_MS);
768
+ const checkSignIn = deps.checkSignIn ?? ((env) => checkRuntimeSignIn({ runtime, env, cwd: turnCwd, now }));
769
+ let lastSignInCheck = null;
770
+ let nextSignInCheckAt = 0;
771
+ /** Ask the tool; record the answer in status.json; return it. */
772
+ const runSignInCheck = async (reason) => {
773
+ const env = withOsUser(turnEnv(process.env, kind, config.agentUid, companySlug));
774
+ let check;
775
+ try {
776
+ check = await checkSignIn(env);
777
+ }
778
+ catch (err) {
779
+ check = { signedIn: null, checkedAt: now().toISOString(), detail: err instanceof Error ? err.message : String(err) };
780
+ }
781
+ lastSignInCheck = check;
782
+ nextSignInCheckAt = now().getTime() + signInCheckMs;
783
+ setStatus({ runtimeSignInCheck: check });
784
+ if (check.signedIn === false) {
785
+ // A repeat "no" while already held does not push the retry out again:
786
+ // the hold's own timer decides when a real turn gets to disagree.
787
+ markSignInBroken(signedOutExcerpt(runtime.id), "signed-out", signInIssue == null);
788
+ }
789
+ else if (check.signedIn === true) {
790
+ if (signInIssue) {
791
+ // The tool answers for itself now: the held messages run at once.
792
+ nextSignInRetryAt = 0;
793
+ markSignInWorks();
794
+ }
795
+ }
796
+ else if (check.detail) {
797
+ log("debug", `sign-in check (${reason}) could not tell: ${check.detail}`);
798
+ }
799
+ return check;
800
+ };
801
+ /** When a real turn last ran despite the tool saying "signed out". */
802
+ let lastTurnDespiteSignedOutAt = now().getTime();
803
+ /**
804
+ * Before a turn: a fresh "signed in" stands; anything else is asked again.
805
+ * The tool's status command only knows about its own account sign-in, so a
806
+ * tool that works some other way (an API key, Bedrock, Vertex) says "no"
807
+ * and would be held forever. Once per retry window a real turn runs anyway
808
+ * and its outcome decides: success clears the hold, a sign-in failure
809
+ * keeps it.
810
+ */
811
+ const signedOutBeforeTurn = async () => {
812
+ const fresh = lastSignInCheck?.signedIn === true && now().getTime() - new Date(lastSignInCheck.checkedAt).getTime() < SIGN_IN_CHECK_FRESH_MS;
813
+ if (fresh)
814
+ return false;
815
+ const check = await runSignInCheck("turn");
816
+ if (check.signedIn !== false)
817
+ return false;
818
+ const t = now().getTime();
819
+ if (signInRetryMs > 0 && t - lastTurnDespiteSignedOutAt >= signInRetryMs) {
820
+ lastTurnDespiteSignedOutAt = t;
821
+ log("info", `${runtime.id} still says it is not signed in; letting one turn try so a tool signed in another way is not held forever`);
822
+ return false;
823
+ }
824
+ return true;
825
+ };
732
826
  /** Say once per conversation that the bot cannot work until its tool signs in. */
733
827
  const tellSignInNeeded = async (conversation, post, kickoff = false, retryOf) => {
734
828
  if (toldAboutSignIn.has(conversation))
@@ -739,7 +833,7 @@ export async function runBot(deps) {
739
833
  runtime: config.runtime,
740
834
  botName: config.name,
741
835
  context: kickoff ? "kickoff" : "message",
742
- text: signInNeededText(config.runtime, kickoff),
836
+ text: signInNeededText(config.runtime, kickoff, signInIssue?.state ?? "expired"),
743
837
  ...(retryOf ? { retryOf } : {}),
744
838
  }));
745
839
  toldAboutSignIn.add(conversation);
@@ -953,7 +1047,7 @@ export async function runBot(deps) {
953
1047
  // resets). Claude gets the full hook set via its payload session id;
954
1048
  // Codex/Grok only see HQ_SESSION_ID.
955
1049
  const sid = session?.sessionId ?? randomUUID();
956
- const env = turnEnv(process.env, kind, config.agentUid, companySlug);
1050
+ const env = withOsUser(turnEnv(process.env, kind, config.agentUid, companySlug));
957
1051
  let bind = { env };
958
1052
  if (companySlug) {
959
1053
  try {
@@ -964,6 +1058,12 @@ export async function runBot(deps) {
964
1058
  }
965
1059
  bind = { newSessionId: sid, env: { ...env, HQ_SESSION_ID: sid } };
966
1060
  }
1061
+ // The tool's own word first: a turn on a signed-out tool cannot start,
1062
+ // and its failure text varies by version (on 2026-10-28 it was a bare
1063
+ // "unknown error" that read as a crash). Held like any sign-in failure.
1064
+ if (await signedOutBeforeTurn()) {
1065
+ throw new RuntimeError(`${runtime.id} is not signed in on this computer`, signedOutExcerpt(runtime.id), 1, { started: false, durationMs: 0 });
1066
+ }
967
1067
  const tTurnStart = now().getTime();
968
1068
  // Model and thinking level are read fresh each turn, so a change from the
969
1069
  // bot's profile (`hq bot set`) applies to the next message without a restart.
@@ -1018,6 +1118,14 @@ export async function runBot(deps) {
1018
1118
  setStatus({ runtimeModelFallback: null });
1019
1119
  }
1020
1120
  log("info", `model turn done in ${Math.round((now().getTime() - tTurnStart) / 1000)}s (${Math.round((tTurnStart - tReceived) / 1000)}s before it)`);
1121
+ // Clears a reported runtime crash (runtimeCrashSinceStart): the tool works again.
1122
+ // Stamped with when the turn STARTED, so a long turn begun before a crash
1123
+ // that finishes after it does not clear the crash for turns that follow.
1124
+ // Never moves backwards: two turns run side by side, and an older one
1125
+ // finishing last must not bring back a crash a newer turn already cleared.
1126
+ const prevOk = Date.parse(readStatus()?.lastTurnOkAt ?? "");
1127
+ if (!Number.isFinite(prevOk) || tTurnStart > prevOk)
1128
+ setStatus({ lastTurnOkAt: new Date(tTurnStart).toISOString() });
1021
1129
  // A brief job's session is one run long; nothing resumes it.
1022
1130
  if (turn.sessionId && !isBriefJobSessionScope(sessionScope)) {
1023
1131
  writeBotSession(dir, {
@@ -1039,16 +1147,35 @@ export async function runBot(deps) {
1039
1147
  }
1040
1148
  return turn;
1041
1149
  };
1042
- const session = readBotSession(dir, config.runtime, sessionScope);
1150
+ const attemptOnce = async () => {
1151
+ const session = readBotSession(dir, config.runtime, sessionScope);
1152
+ try {
1153
+ return await runAttempt(session, await composePrompt(session));
1154
+ }
1155
+ catch (err) {
1156
+ if (!missingResumedSession(err))
1157
+ throw err;
1158
+ clearBotSession(dir, sessionScope);
1159
+ log("warn", "saved model session is gone; starting a new one with recent DM history");
1160
+ return await runAttempt(null, await composePrompt(null));
1161
+ }
1162
+ };
1043
1163
  try {
1044
- return await runAttempt(session, await composePrompt(session));
1164
+ return await attemptOnce();
1045
1165
  }
1046
1166
  catch (err) {
1047
- if (!missingResumedSession(err))
1167
+ // The coding tool's process died before the turn started (Bun's
1168
+ // "error: An unknown error occurred (Unexpected)" after a long idle,
1169
+ // October 2026). A fresh process usually works: wait a moment and run
1170
+ // the same turn once more. If that fails too, the caller's normal
1171
+ // failure path tells the person (with the Heal bot card).
1172
+ if (stream.signal?.aborted || classifyRuntimeError(err) !== "runtime-crashed")
1048
1173
  throw err;
1049
- clearBotSession(dir, sessionScope);
1050
- log("warn", "saved model session is gone; starting a new one with recent DM history");
1051
- return await runAttempt(null, await composePrompt(null));
1174
+ log("info", "runtime crashed before the turn started; retrying once");
1175
+ await sleep(RUNTIME_CRASH_RETRY_DELAY_MS, stream.signal);
1176
+ if (stream.signal?.aborted)
1177
+ throw err;
1178
+ return await attemptOnce();
1052
1179
  }
1053
1180
  };
1054
1181
  /** Failure bookkeeping shared by every model turn: window, health, status, log, stale session. */
@@ -1059,7 +1186,20 @@ export async function runBot(deps) {
1059
1186
  const count = failures.record(now().getTime());
1060
1187
  modelHealth = count >= FAILURE_LIMIT ? "failed" : "degraded";
1061
1188
  setStatus({ restarts: (readStatus()?.restarts ?? 0) + 1, recentFailures: count, lastError: excerpt.slice(0, 400) });
1062
- log("error", `model turn failed (${count}/${FAILURE_LIMIT} in window, ${classifyRuntimeError(err)}): ${excerpt}`);
1189
+ const cls = classifyRuntimeError(err);
1190
+ log("error", `model turn failed (${count}/${FAILURE_LIMIT} in window, ${cls}): ${excerpt}`);
1191
+ if (cls === "runtime-crashed" && err instanceof RuntimeError) {
1192
+ const stderr = (err.stderrFull ?? err.stderrExcerpt).slice(0, 2000);
1193
+ log("error", `runtime crash evidence: exit ${err.exitCode ?? "?"}, lived ${err.durationMs ?? "?"} ms, binary ${err.binaryPath ?? runtime.binary}; stderr: ${JSON.stringify(stderr)}; stdout head: ${JSON.stringify((err.stdoutHead ?? "").slice(0, 300))}`);
1194
+ setStatus({
1195
+ lastCrash: {
1196
+ at: now().toISOString(),
1197
+ code: err.exitCode,
1198
+ ...(err.durationMs !== undefined ? { durationMs: err.durationMs } : {}),
1199
+ stderr,
1200
+ },
1201
+ });
1202
+ }
1063
1203
  // A resumed session that no longer exists must not poison every turn: the
1064
1204
  // next turn starts fresh with the recent DM history prepended. The model
1065
1205
  // CLI's "An unknown error occurred (Unexpected)" (every turn of a
@@ -1683,7 +1823,7 @@ export async function runBot(deps) {
1683
1823
  runtime: runtime.id,
1684
1824
  botName: config.name,
1685
1825
  context: "kickoff",
1686
- ...(outcome.started && poster.postedCount > 0 ? { text: runtimeDidNotFinishText(cls, runtime.id, true) } : {}),
1826
+ ...(outcome.started && poster.postedCount > 0 ? { text: runtimeDidNotFinishText(cls, runtime.id, true, config.name) } : {}),
1687
1827
  });
1688
1828
  try {
1689
1829
  await postRepair(target, notice);
@@ -1852,7 +1992,7 @@ export async function runBot(deps) {
1852
1992
  }
1853
1993
  // A missing tool does not come back between retries: say so now.
1854
1994
  if (attempt >= MAX_ATTEMPTS_PER_MESSAGE || cls === "not-installed") {
1855
- const plain = runtimeFailureText(cls, runtime.id);
1995
+ const plain = runtimeFailureText(cls, runtime.id, "message", config.name);
1856
1996
  await sendToPerson(`I couldn't run this one. ${plain}`).catch(() => undefined);
1857
1997
  await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(plain), disposition: `${runtime.id} could not be started` });
1858
1998
  markReported("failed");
@@ -1872,12 +2012,12 @@ export async function runBot(deps) {
1872
2012
  // reports and says this real reason, not "the bot was restarted".
1873
2013
  // What the cloud stores as the run's output: plain words, never the raw
1874
2014
  // runtime text (that is in the bot log and status.json).
1875
- const jobOutput = outcome.aborted ? `${STOPPED_BY_OWNER_REASON}.` : runtimeFailureText(failureClass(outcome), runtime.id);
2015
+ const jobOutput = outcome.aborted ? `${STOPPED_BY_OWNER_REASON}.` : runtimeFailureText(failureClass(outcome), runtime.id, "message", config.name);
1876
2016
  patchInflight(dir, id, { jobFailure: { reason, output: jobOutput } });
1877
2017
  try {
1878
2018
  await sendToPerson(outcome.aborted
1879
2019
  ? didNotFinishText(reason, poster.postedCount > 0)
1880
- : runtimeDidNotFinishText(failureClass(outcome), runtime.id, poster.postedCount > 0));
2020
+ : runtimeDidNotFinishText(failureClass(outcome), runtime.id, poster.postedCount > 0, config.name));
1881
2021
  }
1882
2022
  catch (err) {
1883
2023
  // Leave it un-acked with the marker: the next poll reports it.
@@ -2381,7 +2521,7 @@ export async function runBot(deps) {
2381
2521
  runtime: runtime.id,
2382
2522
  botName: config.name,
2383
2523
  context: "message",
2384
- text: runtimeDidNotFinishText(cls, runtime.id, poster.postedCount > 0),
2524
+ text: runtimeDidNotFinishText(cls, runtime.id, poster.postedCount > 0, config.name),
2385
2525
  retryOf: item.eventId ?? id,
2386
2526
  }), item.rootEventId);
2387
2527
  }
@@ -2397,6 +2537,10 @@ export async function runBot(deps) {
2397
2537
  attempts.delete(id);
2398
2538
  log("error", `turn on ${id} ${outcome.aborted ? "stopped by the owner" : "failed"} after it started; told the owner, not re-run`);
2399
2539
  };
2540
+ // The bot's first look at its tool's sign-in, so `hq bot status` and the
2541
+ // app know before any message arrives. A tool that cannot be asked changes
2542
+ // nothing; the first turn then reports for itself.
2543
+ await runSignInCheck("start");
2400
2544
  // The kickoff runs alongside the heartbeat, before the first inbox poll, so
2401
2545
  // an owner message sent meanwhile is answered after it in the same session.
2402
2546
  const kickoff = kickoffAfterIntro
@@ -2498,6 +2642,13 @@ export async function runBot(deps) {
2498
2642
  }
2499
2643
  if (modelHoldUntil)
2500
2644
  liftModelHold();
2645
+ // While the tool says it is signed out, ask it again on a timer (cheap,
2646
+ // no turn): the moment it is signed in, the held messages run.
2647
+ if (signInIssue && busyLanes.size === 0 && now().getTime() >= nextSignInCheckAt) {
2648
+ await runSignInCheck("watch");
2649
+ if (!signInIssue)
2650
+ pump();
2651
+ }
2501
2652
  // A kickoff held for a sign-in retries on the same timer as messages.
2502
2653
  if (busyLanes.size === 0 &&
2503
2654
  (!signInIssue || now().getTime() >= nextSignInRetryAt) &&
@@ -2628,7 +2779,7 @@ export async function runBot(deps) {
2628
2779
  catch (err) {
2629
2780
  log("warn", `could not record the update restart: ${err instanceof Error ? err.message : String(err)}`);
2630
2781
  }
2631
- void finish("stopped", CLI_UPDATE_RESTART_EXIT_CODE, `hq-cli updated ${update.from} -> ${update.to}; restarting to load it`, () => deps.onCliUpdateRestart?.(update, log));
2782
+ void finish("stopped", CLI_UPDATE_RESTART_EXIT_CODE, describeCliUpdate(update), () => deps.onCliUpdateRestart?.(update, log));
2632
2783
  });
2633
2784
  return { stop, done, status: readStatus, idle: () => pending.size === 0 };
2634
2785
  }
@@ -15,7 +15,7 @@
15
15
  * carries the final text and session_id.
16
16
  * Verified against Claude Code 2.1.258 on 2026-09-12 (fixtures/claude-stream.ndjson).
17
17
  */
18
- import type { BotRuntime } from "./types.js";
18
+ import type { BotRuntime, SpawnResult } from "./types.js";
19
19
  export declare const CLAUDE_PERMISSION_MODE_ENV = "HQ_BOT_CLAUDE_PERMISSION_MODE";
20
20
  export declare const DEFAULT_CLAUDE_PERMISSION_MODE = "bypassPermissions";
21
21
  export declare const GATED_CLAUDE_PERMISSION_MODE = "acceptEdits";
@@ -84,4 +84,10 @@ export interface ClaudeRuntimeOptions {
84
84
  systemBinDirs?: readonly string[];
85
85
  }
86
86
  export declare function createClaudeRuntime(env?: NodeJS.ProcessEnv, opts?: ClaudeRuntimeOptions): BotRuntime;
87
+ /**
88
+ * `claude auth status --json` prints `{"loggedIn": true|false, ...}` and
89
+ * exits 0 either way. Null when the output is not that (an older CLI, or a
90
+ * tool that could not run at all).
91
+ */
92
+ export declare function parseClaudeAuthStatus(result: SpawnResult): boolean | null;
87
93
  //# sourceMappingURL=claude.d.ts.map
@@ -230,6 +230,35 @@ export function createClaudeRuntime(env = process.env, opts = {}) {
230
230
  failureReason(stdout) {
231
231
  return messagesStreamFailure(stdout);
232
232
  },
233
+ signInStatus: { args: ["auth", "status", "--json"], parse: parseClaudeAuthStatus },
233
234
  };
234
235
  }
236
+ /**
237
+ * `claude auth status --json` prints `{"loggedIn": true|false, ...}` and
238
+ * exits 0 either way. Null when the output is not that (an older CLI, or a
239
+ * tool that could not run at all).
240
+ */
241
+ export function parseClaudeAuthStatus(result) {
242
+ const text = result.stdout ?? "";
243
+ const start = text.indexOf("{");
244
+ if (start >= 0) {
245
+ try {
246
+ const parsed = JSON.parse(text.slice(start));
247
+ if (typeof parsed.loggedIn === "boolean")
248
+ return parsed.loggedIn;
249
+ }
250
+ catch {
251
+ /* fall through to the loose match */
252
+ }
253
+ }
254
+ const m = /"loggedIn"\s*:\s*(true|false)/.exec(text);
255
+ if (m)
256
+ return m[1] === "true";
257
+ const plain = `${text}\n${result.stderr ?? ""}`;
258
+ if (/\bnot logged in\b/i.test(plain))
259
+ return false;
260
+ if (/\blogged in\b/i.test(plain))
261
+ return true;
262
+ return null;
263
+ }
235
264
  //# sourceMappingURL=claude.js.map
@@ -90,5 +90,10 @@ export declare function bundledCodexBin(homeDir?: string, isExecutable?: IsExecu
90
90
  * usual "codex is not installed or not on PATH" error.
91
91
  */
92
92
  export declare function resolveCodexBinary(env: NodeJS.ProcessEnv, homeDir?: string, platform?: NodeJS.Platform, cwd?: string, isExecutable?: IsExecutableFile): string;
93
+ /**
94
+ * `codex login status` prints "Logged in using ChatGPT" (exit 0) or "Not
95
+ * logged in" (exit 1). Null when it said neither.
96
+ */
97
+ export declare function parseCodexLoginStatus(result: SpawnResult): boolean | null;
93
98
  export declare function createCodexRuntime(): BotRuntime;
94
99
  //# sourceMappingURL=codex.d.ts.map
@@ -270,11 +270,24 @@ export function resolveCodexBinary(env, homeDir, platform = process.platform, cw
270
270
  // `||`, not `??`: an empty HOME must not turn ~/Applications into a relative path.
271
271
  return bundledCodexBin(homeDir || env.HOME || os.homedir(), isExecutable) ?? "codex";
272
272
  }
273
+ /**
274
+ * `codex login status` prints "Logged in using ChatGPT" (exit 0) or "Not
275
+ * logged in" (exit 1). Null when it said neither.
276
+ */
277
+ export function parseCodexLoginStatus(result) {
278
+ const text = `${result.stdout ?? ""}\n${result.stderr ?? ""}`;
279
+ if (/\bnot logged in\b/i.test(text))
280
+ return false;
281
+ if (/\blogged in\b/i.test(text))
282
+ return true;
283
+ return null;
284
+ }
273
285
  export function createCodexRuntime() {
274
286
  return {
275
287
  id: "codex",
276
288
  binary: "codex",
277
289
  resolveBinary: (env, cwd, platform) => resolveCodexBinary(env, undefined, platform ?? process.platform, cwd),
290
+ signInStatus: { args: ["login", "status"], parse: parseCodexLoginStatus },
278
291
  buildArgs(input, scratch) {
279
292
  const promptArg = input.platform === "win32" ? "-" : codexPrompt(input);
280
293
  const common = ["--json", "--skip-git-repo-check", "-o", scratch.outputFile];
@@ -394,12 +394,15 @@ export async function runRuntimeTurn(runtime, input, deps = {}) {
394
394
  let modelFallback = remembered ? { from: remembered.from, to: remembered.to } : undefined;
395
395
  /** The retries on supported models failed too: report the first failure (its cause is the old CLI). */
396
396
  let fallbackFailure;
397
+ let spawnedAt = nowMs();
397
398
  try {
398
399
  result = await spawnImpl(binary, args, spawnOptions);
399
400
  if (runtime.id === "grok" && result.code !== 0 && !input.signal?.aborted) {
400
401
  const retryArgs = grokRetryArgsAfterInvalidOutputFormat(args, result.stderr);
401
- if (retryArgs)
402
+ if (retryArgs) {
403
+ spawnedAt = nowMs();
402
404
  result = await spawnImpl(binary, retryArgs, spawnOptions);
405
+ }
403
406
  }
404
407
  if (remembered) {
405
408
  // A turn on the remembered model: it either still works or the memo is wrong.
@@ -439,6 +442,7 @@ export async function runRuntimeTurn(runtime, input, deps = {}) {
439
442
  continue;
440
443
  tried.push(to);
441
444
  const retryArgs = runtime.buildArgs({ ...turnInput, model: to }, { outputFile, newSessionId });
445
+ spawnedAt = nowMs();
442
446
  const retry = await spawnImpl(binary, retryArgs, spawnOptions);
443
447
  if (input.signal?.aborted) {
444
448
  // The owner stopped it: report the stop, remember nothing.
@@ -493,6 +497,11 @@ export async function runRuntimeTurn(runtime, input, deps = {}) {
493
497
  }
494
498
  }
495
499
  if (result.code !== 0 || result.signal) {
500
+ const durationMs = Math.max(0, nowMs() - spawnedAt);
501
+ const sawStreamEvent = result.stdout.split("\n").some((line) => {
502
+ const ev = parseJsonLine(line);
503
+ return ev !== null && (ev.type === "system" || ev.type === "assistant" || ev.type === "result");
504
+ });
496
505
  const reason = runtime.failureReason?.(result.stdout, result.stderr);
497
506
  // Streaming runtimes write JSON events to stdout: never show those to a
498
507
  // person as the reason. stderr, then a plain description, instead.
@@ -505,7 +514,14 @@ export async function runRuntimeTurn(runtime, input, deps = {}) {
505
514
  : result.signal
506
515
  ? `the ${runtime.binary} process was killed (${result.signal})`
507
516
  : `${runtime.binary} exited with code ${result.code}`;
508
- throw new RuntimeError(`${runtime.binary} exited with ${result.code ?? `signal ${result.signal}`}${reason ? `: ${reason.slice(0, 200)}` : ""}${fallbackFailure ? ` (${fallbackFailure})` : ""}`, reason ? reason.slice(0, 400) : fallback, result.code, !reason && stdoutTail ? { excerptIsOutputTail: true } : {});
517
+ throw new RuntimeError(`${runtime.binary} exited with ${result.code ?? `signal ${result.signal}`}${reason ? `: ${reason.slice(0, 200)}` : ""}${fallbackFailure ? ` (${fallbackFailure})` : ""}`, reason ? reason.slice(0, 400) : fallback, result.code, {
518
+ ...(!reason && stdoutTail ? { excerptIsOutputTail: true } : {}),
519
+ durationMs,
520
+ sawStreamEvent,
521
+ binaryPath: binary,
522
+ stderrFull: result.stderr.slice(0, 2000),
523
+ stdoutHead: result.stdout.slice(0, 300),
524
+ });
509
525
  }
510
526
  const parsed = runtime.parseOutput(result.stdout, { outputFile, outputFileText });
511
527
  // A turn that ran on the requested model and succeeded: nothing to remember.
@@ -128,10 +128,23 @@ export declare class RuntimeError extends Error {
128
128
  * must never decide what went wrong; only the message may.
129
129
  */
130
130
  readonly excerptIsOutputTail: boolean;
131
+ /** How long the runtime process lived, in ms, when known. */
132
+ readonly durationMs: number | undefined;
133
+ /** True when stdout carried at least one stream event (`system`, `assistant`, `result`): the turn started. */
134
+ readonly sawStreamEvent: boolean;
135
+ /** Crash evidence for the bot log: the binary spawned, full stderr and the head of stdout (bounded). */
136
+ readonly binaryPath: string | undefined;
137
+ readonly stderrFull: string | undefined;
138
+ readonly stdoutHead: string | undefined;
131
139
  constructor(message: string, stderrExcerpt: string, exitCode: number | null, opts?: {
132
140
  started?: boolean;
133
141
  aborted?: boolean;
134
142
  excerptIsOutputTail?: boolean;
143
+ durationMs?: number;
144
+ sawStreamEvent?: boolean;
145
+ binaryPath?: string;
146
+ stderrFull?: string;
147
+ stdoutHead?: string;
135
148
  });
136
149
  }
137
150
  export interface BotRuntime {
@@ -189,6 +202,16 @@ export interface BotRuntime {
189
202
  requestedModel: string | undefined;
190
203
  run: (args: string[]) => Promise<SpawnResult>;
191
204
  }): Promise<string[]>;
205
+ /**
206
+ * Optional: the tool's own "am I signed in?" command (`claude auth status
207
+ * --json`, `codex login status`), run in the bot's environment before a turn
208
+ * and at start. `parse` reads its answer: true/false, or null when the
209
+ * output does not say. Pure; needs no network.
210
+ */
211
+ signInStatus?: {
212
+ args: string[];
213
+ parse(result: SpawnResult): boolean | null;
214
+ };
192
215
  }
193
216
  /**
194
217
  * Turns have no time limit: a turn runs until the model process exits (the
@@ -19,6 +19,14 @@ export class RuntimeError extends Error {
19
19
  * must never decide what went wrong; only the message may.
20
20
  */
21
21
  excerptIsOutputTail;
22
+ /** How long the runtime process lived, in ms, when known. */
23
+ durationMs;
24
+ /** True when stdout carried at least one stream event (`system`, `assistant`, `result`): the turn started. */
25
+ sawStreamEvent;
26
+ /** Crash evidence for the bot log: the binary spawned, full stderr and the head of stdout (bounded). */
27
+ binaryPath;
28
+ stderrFull;
29
+ stdoutHead;
22
30
  constructor(message, stderrExcerpt, exitCode, opts = {}) {
23
31
  super(message);
24
32
  this.stderrExcerpt = stderrExcerpt;
@@ -27,6 +35,11 @@ export class RuntimeError extends Error {
27
35
  this.started = opts.started ?? true;
28
36
  this.aborted = opts.aborted ?? false;
29
37
  this.excerptIsOutputTail = opts.excerptIsOutputTail ?? false;
38
+ this.durationMs = opts.durationMs;
39
+ this.sawStreamEvent = opts.sawStreamEvent ?? false;
40
+ this.binaryPath = opts.binaryPath;
41
+ this.stderrFull = opts.stderrFull;
42
+ this.stdoutHead = opts.stdoutHead;
30
43
  }
31
44
  }
32
45
  /**
@@ -27,16 +27,27 @@ export type RuntimeFailureClass =
27
27
  | "transient"
28
28
  /** The tool is not installed (or not on PATH) on this computer. */
29
29
  | "not-installed"
30
+ /** The tool's process exited before the turn started (e.g. Bun's "error: ... (Unexpected)"). A restart heals it. */
31
+ | "runtime-crashed"
30
32
  /** Anything else. Neutral copy; never blames the sign-in. */
31
33
  | "unknown";
32
- /** The four classes that get a repair card in the DM. */
33
- export type RuntimeRepairClass = Extract<RuntimeFailureClass, "signed-out" | "cli-outdated" | "model-unsupported" | "transient">;
34
+ /** The classes that get a repair card in the DM. */
35
+ export type RuntimeRepairClass = Extract<RuntimeFailureClass, "signed-out" | "cli-outdated" | "model-unsupported" | "transient" | "runtime-crashed">;
34
36
  /** The one thing the person can do about it. The desktop maps each to a button. */
35
- export type RuntimeRepairAction = "signIn" | "update" | "switchModel" | "tryAgain";
37
+ export type RuntimeRepairAction = "signIn" | "update" | "switchModel" | "tryAgain" | "restart";
36
38
  export declare const RUNTIME_REPAIR_ACTIONS: Record<RuntimeRepairClass, RuntimeRepairAction>;
37
39
  export declare function isRuntimeRepairClass(cls: RuntimeFailureClass): cls is RuntimeRepairClass;
38
40
  /** Sort a runtime's failure text into one class. Order matters: most specific first. */
39
41
  export declare function classifyRuntimeFailure(text: string | undefined | null): RuntimeFailureClass;
42
+ export declare function isBunProcessError(text: string | undefined | null): boolean;
43
+ /** A process that exits non-zero this soon, with no stream event, never started its turn. */
44
+ export declare const RUNTIME_CRASH_MAX_MS = 5000;
45
+ /**
46
+ * True when the coding tool's process exited before the turn started: a
47
+ * non-zero exit code, not stopped by the owner, no stream event on stdout,
48
+ * and either Bun's process error shape on stderr or a life under 5 s.
49
+ */
50
+ export declare function isRuntimeCrash(err: unknown): boolean;
40
51
  /**
41
52
  * Class of a failed turn's error. A RuntimeError's excerpt (the runtime's own
42
53
  * reported error: failureReason, turn.failed, stderr) is the better evidence;
@@ -57,11 +68,11 @@ export type RuntimeFailureContext = "kickoff" | "message";
57
68
  */
58
69
  export declare function runtimeFailureReason(cls: RuntimeFailureClass, runtime: BotRuntimeId | string): string;
59
70
  /** The one next step, as a full sentence. */
60
- export declare function runtimeFailureNextStep(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, context?: RuntimeFailureContext): string;
71
+ export declare function runtimeFailureNextStep(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, context?: RuntimeFailureContext, botName?: string): string;
61
72
  /** "My Codex app is out of date. Update Codex, then send me any message." */
62
- export declare function runtimeFailureText(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, context?: RuntimeFailureContext): string;
73
+ export declare function runtimeFailureText(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, context?: RuntimeFailureContext, botName?: string): string;
63
74
  /** A failure after the model started (and maybe posted part of the answer). */
64
- export declare function runtimeDidNotFinishText(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, progressPosted: boolean): string;
75
+ export declare function runtimeDidNotFinishText(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, progressPosted: boolean, botName?: string): string;
65
76
  /**
66
77
  * The machine-readable repair payload a failure DM carries. The DM transport
67
78
  * (POST /v1/notify/dm) stores structured data only in `richContent`