@indigoai-us/hq-cli 5.113.1 → 5.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,7 +5,7 @@
5
5
  * - refuses to start unless the creds file names this bot's agt_ AND the
6
6
  * server-side record is owned by the configured owner
7
7
  * - heartbeat every 30s (POST /v1/agents/{uid}/heartbeat, agent JWT)
8
- * - inbox poll every 5s: owner DMs go to ONE headless model turn (resumed by
8
+ * - inbox poll every 2s: owner DMs go to ONE headless model turn (resumed by
9
9
  * session id); the reply is sent as the bot; the message is acked only after
10
10
  * the reply is sent; processed ids persist so a crash between reply and ack
11
11
  * never answers twice
@@ -16,12 +16,21 @@
16
16
  * - room items (channels / group chats): answered when the bot is @mentioned,
17
17
  * or when the owner posts in a small group (see room-policy.ts); claimed
18
18
  * first (first-claim-wins), one session per room, reply posted in the room
19
+ * - conversations run side by side: up to MAX_PARALLEL_TURNS turns at once,
20
+ * one lane per model session (the owner DM session, or one room). Messages
21
+ * in the same lane run in order, so a session is never resumed by two turns
22
+ * at once and a thread's answers never arrive out of sequence.
19
23
  * - while a turn runs, each finished assistant message is posted as its own
20
24
  * message (progress.ts); turns have no time limit
21
25
  * - every received message gets a reply: a turn that fails after the model
22
26
  * started (error, crash, owner stop, restart) posts one "did not finish"
23
27
  * message and is NOT run again — it may already have done outward things.
24
28
  * inflight.json marks the turn in progress so a restart can say so.
29
+ * - a turn that fails because the coding tool's sign-in no longer works
30
+ * (runtime-sign-in.ts) is not a failure of the message: the bot says once
31
+ * per conversation that it needs a sign-in, keeps the message un-acked,
32
+ * holds new turns, and tries again every SIGN_IN_RETRY_MS (and at once
33
+ * after a restart). A turn that works clears it.
25
34
  * - failures before the model starts (missing binary) retry with backoff;
26
35
  * 5 model failures in 10 minutes → state `failed` and a clean exit (launchd
27
36
  * does not respawn a clean exit). Owner stops never count.
@@ -35,17 +44,19 @@ import * as fs from "node:fs";
35
44
  import * as path from "node:path";
36
45
  import { fullJitterDelayMs } from "../mesh/live/backoff.js";
37
46
  import { acquirePidLock, defaultPidLockDeps, releasePidLock } from "../mesh/live/daemon/pid-lock.js";
38
- import { patchBotConfig, readBotConfig } from "./config.js";
47
+ import { effectiveBotCompanies, effectiveBotKind, patchBotConfig, readBotConfig, SETUP_BOT_WORKER_ID } from "./config.js";
39
48
  import { ensureBotSessionMeta } from "./company-bind.js";
40
49
  import { readBotCredsIdentity } from "./creds.js";
41
50
  import { isProcessed, markProcessed, readInboxState } from "./inbox-state.js";
42
- import { clearInflight, patchInflight, readInflight, writeInflight } from "./inflight.js";
51
+ import { clearInflight, patchInflight, readInflight, readInflightTurns, writeInflight } from "./inflight.js";
43
52
  import { ProgressPoster } from "./progress.js";
44
53
  import { createBotLogger } from "./log.js";
45
54
  import { buildSystemPrompt, introDmText, nonOwnerRefusalText, readPromptSources } from "./prompt.js";
46
55
  import { FALLBACK_CLAUDE_PERMISSION_MODE, isClaudeBypassDisabled } from "./runtime/claude.js";
47
56
  import { runRuntimeTurn, RuntimeError } from "./runtime/index.js";
48
57
  import { isDmItem, resolveReplyTarget } from "./room-policy.js";
58
+ import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
59
+ import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
49
60
  import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
50
61
  import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
51
62
  export const INBOX_POLL_INTERVAL_MS = 2_000;
@@ -57,9 +68,41 @@ export const THINKING_EMOJI = "👀";
57
68
  export const ROOM_THINKING_STATUS = "is thinking…";
58
69
  export const RECENT_MESSAGES_LIMIT = 15;
59
70
  export const RECENT_MESSAGE_MAX_CHARS = 400;
71
+ /** Model turns a bot runs at once (each on a different conversation). */
72
+ export const MAX_PARALLEL_TURNS = 2;
60
73
  export const KICKOFF_MESSAGE_ID = "kickoff";
74
+ /**
75
+ * The environment a model turn's `hq` commands run in.
76
+ *
77
+ * The bot process carries its own machine identity in HQ_MACHINE_CREDS_FILE /
78
+ * HQ_MACHINE_TOKEN_STATE_DIR so its inbox, heartbeat and posts are the bot's.
79
+ * What a turn inherits depends on the bot's kind:
80
+ * - A PERSONAL bot acts as its owner on every turn (DMs, the kickoff, rooms):
81
+ * everything the owner asks for — a company, a bot, an invite, a share —
82
+ * belongs to the owner's account, and the cloud rightly refuses a bot that
83
+ * tries to own it ("No person entity found"). Stripping the marker makes
84
+ * every `hq` command in the turn use the owner's sign-in, the same one the
85
+ * desktop app shares, so no command needs a special case. The bot's id
86
+ * (never a credential) rides along in HQ_BOT_AGENT_UID so what the owner
87
+ * creates in the turn — a company — can add the bot as a member afterwards.
88
+ * - A COMPANY bot acts as itself on every turn, like a cloud agent: the
89
+ * machine identity stays, and the cloud evaluates it through its own
90
+ * company memberships. It never acts as the owner, in DMs or in rooms.
91
+ * Pure so the rule is unit-testable.
92
+ */
93
+ export function turnEnv(base, kind, agentUid) {
94
+ const env = { ...base };
95
+ delete env[HQ_BOT_AGENT_UID_ENV];
96
+ if (kind === "personal") {
97
+ delete env.HQ_MACHINE_CREDS_FILE;
98
+ delete env.HQ_MACHINE_TOKEN_STATE_DIR;
99
+ if (agentUid)
100
+ env[HQ_BOT_AGENT_UID_ENV] = agentUid;
101
+ }
102
+ return env;
103
+ }
61
104
  /** The core worker HQ's setup bot runs. */
62
- export const SETUP_WORKER_ID = "setup";
105
+ export const SETUP_WORKER_ID = SETUP_BOT_WORKER_ID;
63
106
  /** The one message a person gets when a turn on their message did not finish. */
64
107
  export function didNotFinishText(reason, progressPosted) {
65
108
  const r = reason.trim().replace(/[.\s]+$/, "");
@@ -68,6 +111,17 @@ export function didNotFinishText(reason, progressPosted) {
68
111
  /** Owner stop, `hq bot stop` or `hq bot restart`: all arrive as the same signal. */
69
112
  export const STOPPED_BY_OWNER_REASON = "I was stopped before I was done";
70
113
  export const RESTARTED_REASON = "I was restarted while I was working on it";
114
+ /**
115
+ * The lane an inbox item runs in: the model session its turn resumes. Owner
116
+ * DMs share the DM session; each room has its own. An item that never reaches
117
+ * a model turn (refusal, not addressed to the bot) gets a lane of its own.
118
+ */
119
+ export function inboxLaneKey(item, config) {
120
+ if (isDmItem(item))
121
+ return "dm";
122
+ const target = resolveReplyTarget(item, config);
123
+ return target?.kind === "room" ? target.sessionScope : `item:${item.messageId}`;
124
+ }
71
125
  class FailureWindow {
72
126
  windowMs;
73
127
  limit;
@@ -117,8 +171,9 @@ export async function runBot(deps) {
117
171
  const setStatus = (patch) => patchBotStatus(dir, patch, fallback, now);
118
172
  let state = "starting";
119
173
  let stopping = false;
120
- /** Aborts the model turn in progress (owner stop). */
121
- let currentTurn = null;
174
+ /** Aborts the model turns in progress (owner stop). */
175
+ const currentTurns = new Set();
176
+ const maxParallelTurns = Math.max(1, deps.maxParallelTurns ?? MAX_PARALLEL_TURNS);
122
177
  let resolveDone;
123
178
  const done = new Promise((r) => (resolveDone = r));
124
179
  // Only override isProcessAlive when a caller injected one: spreading an
@@ -130,6 +185,9 @@ export async function runBot(deps) {
130
185
  const turnCwd = config.hqRoot;
131
186
  // HQ's setup bot is a guided conversation: no "working on it" filler between steps.
132
187
  const isSetupBot = config.workerId === SETUP_WORKER_ID;
188
+ // Personal: every turn acts as the owner. Company: every turn acts as the bot.
189
+ const kind = effectiveBotKind(config);
190
+ const companies = effectiveBotCompanies(config);
133
191
  const lockDeps = defaultPidLockDeps({
134
192
  pid,
135
193
  ...(deps.isProcessAlive ? { isProcessAlive: deps.isProcessAlive } : {}),
@@ -152,14 +210,14 @@ export async function runBot(deps) {
152
210
  log("error", credsProblem);
153
211
  resolveDone("failed");
154
212
  exit(0);
155
- return { stop: async () => { }, done, status: () => null };
213
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
156
214
  }
157
215
  if (config.enabled === false) {
158
216
  setStatus({ state: "stopped", pid });
159
217
  log("info", "bot is disabled (hq bot start to enable)");
160
218
  resolveDone("stopped");
161
219
  exit(0);
162
- return { stop: async () => { }, done, status: () => null };
220
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
163
221
  }
164
222
  if (!deps.skipPidLock) {
165
223
  const lock = acquirePidLock(dir, lockDeps);
@@ -168,32 +226,33 @@ export async function runBot(deps) {
168
226
  log("warn", msg);
169
227
  resolveDone("stopped");
170
228
  exit(0);
171
- return { stop: async () => { }, done, status: () => null };
229
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
172
230
  }
173
231
  }
174
232
  if (hasPromotionHold(dir)) {
175
233
  await finish("stopped", 0, "local bot held for cloud promotion");
176
- return { stop: async () => { }, done, status: () => null };
234
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
177
235
  }
178
- setStatus({ ...fallback(), pid, state: "starting", canDrainForPromotion: false });
179
- log("info", `starting ${config.name} (${config.runtime}) as ${config.agentUid} for ${config.ownerUid}`);
236
+ // A restart is the owner's "try again" after signing in: start clean.
237
+ setStatus({ ...fallback(), pid, state: "starting", canDrainForPromotion: false, runtimeSignIn: null });
238
+ log("info", `starting ${config.name} (${config.runtime}, ${kind} bot${companies.length ? ` of ${companies.join(", ")}` : ""}) as ${config.agentUid} for ${config.ownerUid}`);
180
239
  try {
181
240
  const record = await api.getAgent(config.agentUid);
182
241
  // Promotion keeps the identity and owner. Ownership alone must not let
183
242
  // an old local installation start consuming the cloud bot's inbox.
184
- if (record.uid !== config.agentUid || record.computeMode !== "local" || record.botKind !== "personal") {
185
- await finish("failed", 0, `identity ${config.agentUid} is not a personal local bot; this installation cannot run it`);
186
- return { stop: async () => { }, done, status: () => null };
243
+ if (record.uid !== config.agentUid || record.computeMode !== "local" || record.botKind !== kind) {
244
+ await finish("failed", 0, `identity ${config.agentUid} is not a ${kind} local bot; this installation cannot run it`);
245
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
187
246
  }
188
247
  const owner = record.ownerUid ?? null;
189
248
  if (owner !== config.ownerUid) {
190
249
  await finish("failed", 0, `identity ${config.agentUid} is owned by ${owner ?? "nobody"}, not ${config.ownerUid}`);
191
- return { stop: async () => { }, done, status: () => null };
250
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
192
251
  }
193
252
  }
194
253
  catch (err) {
195
254
  await finish("failed", 0, `could not verify bot identity: ${err instanceof Error ? err.message : String(err)}`);
196
- return { stop: async () => { }, done, status: () => null };
255
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
197
256
  }
198
257
  // ── System prompt ──────────────────────────────────────────────────────────
199
258
  const systemPrompt = deps.systemPromptOverride ??
@@ -206,6 +265,8 @@ export async function runBot(deps) {
206
265
  workerSource: config.workerSource,
207
266
  workerId: config.workerId,
208
267
  companySlug: config.companySlug,
268
+ kind,
269
+ companies,
209
270
  memoryDir: config.memoryDir,
210
271
  sources: readPromptSources(config.hqRoot, config.workerDir, undefined, {
211
272
  memoryDir: config.memoryDir,
@@ -218,20 +279,24 @@ export async function runBot(deps) {
218
279
  fs.writeFileSync(systemPromptFile, systemPrompt, { mode: 0o600 });
219
280
  if (hasPromotionHold(dir)) {
220
281
  await finish("stopped", 0, "local bot held for cloud promotion");
221
- return { stop: async () => { }, done, status: () => null };
282
+ return { stop: async () => { }, done, status: () => null, idle: () => true };
222
283
  }
223
284
  state = "running";
224
285
  setStatus({ state: "running", lastActivityAt: now().toISOString() });
225
286
  log("info", "running");
226
287
  // Read before the kickoff can write its own marker: whatever is here now was
227
288
  // left by a previous run that was cut off mid-turn.
228
- const leftover = readInflight(dir);
289
+ const leftovers = readInflightTurns(dir);
229
290
  // ── Intro DM (once) ────────────────────────────────────────────────────────
230
291
  // The kickoff turn is tied to the intro: it runs only in the start that sent
231
292
  // the intro, so introSentAt (persisted before the turn) guarantees it never
232
293
  // runs again on a restart.
233
294
  let kickoffAfterIntro = null;
234
- if (!config.introSentAt) {
295
+ if (config.introSentAt && config.kickoffPendingSignIn && config.kickoff?.trim()) {
296
+ // The kickoff was held for a sign-in on an earlier run; this start retries it.
297
+ kickoffAfterIntro = {};
298
+ }
299
+ else if (!config.introSentAt) {
235
300
  try {
236
301
  const sent = await api.sendDm({ toPersonUid: config.ownerUid, body: introDmText(config.name, config.runtime, config.intro) });
237
302
  patchBotConfig(dir, { introSentAt: now().toISOString() });
@@ -246,6 +311,41 @@ export async function runBot(deps) {
246
311
  // ── Heartbeat loop ─────────────────────────────────────────────────────────
247
312
  const failures = new FailureWindow(FAILURE_WINDOW_MS, FAILURE_LIMIT);
248
313
  let modelHealth = "ok";
314
+ // ── Coding tool sign-in ────────────────────────────────────────────────────
315
+ const signInRetryMs = Math.max(0, deps.signInRetryMs ?? SIGN_IN_RETRY_MS);
316
+ let signInIssue = null;
317
+ let nextSignInRetryAt = 0;
318
+ /** Conversations already told this time that the bot needs a sign-in. */
319
+ const toldAboutSignIn = new Set();
320
+ const markSignInBroken = (excerpt) => {
321
+ nextSignInRetryAt = now().getTime() + signInRetryMs;
322
+ modelHealth = "degraded";
323
+ if (signInIssue)
324
+ return;
325
+ signInIssue = { state: "expired", runtime: config.runtime, since: now().toISOString() };
326
+ setStatus({ runtimeSignIn: signInIssue, lastError: excerpt.slice(0, 400) });
327
+ log("warn", `${runtime.id} needs to sign in again on this computer; holding turns and retrying every ${Math.round(signInRetryMs / 1000)}s`);
328
+ };
329
+ const markSignInWorks = () => {
330
+ if (!signInIssue)
331
+ return;
332
+ signInIssue = null;
333
+ toldAboutSignIn.clear();
334
+ setStatus({ runtimeSignIn: null });
335
+ log("info", `${runtime.id} sign-in works again; resuming`);
336
+ };
337
+ /** Say once per conversation that the bot cannot work until its tool signs in. */
338
+ const tellSignInNeeded = async (conversation, post, kickoff = false) => {
339
+ if (toldAboutSignIn.has(conversation))
340
+ return;
341
+ try {
342
+ await post(signInNeededText(config.runtime, kickoff));
343
+ toldAboutSignIn.add(conversation);
344
+ }
345
+ catch (err) {
346
+ log("warn", `could not say that ${runtime.id} needs a sign-in: ${err instanceof Error ? err.message : String(err)}`);
347
+ }
348
+ };
249
349
  const heartbeatLoop = (async () => {
250
350
  while (!stopping && state === "running") {
251
351
  try {
@@ -296,20 +396,22 @@ export async function runBot(deps) {
296
396
  // resets). Claude gets the full hook set via its payload session id;
297
397
  // Codex/Grok only see HQ_SESSION_ID.
298
398
  const sid = session?.sessionId ?? randomUUID();
299
- let bind = {};
399
+ const env = turnEnv(process.env, kind, config.agentUid);
400
+ let bind = { env };
300
401
  if (config.companySlug) {
301
402
  try {
302
403
  ensureBotSessionMeta(config.hqRoot, sid, config.companySlug, undefined, now);
303
- bind = { newSessionId: sid, env: { ...process.env, HQ_SESSION_ID: sid } };
404
+ bind = { newSessionId: sid, env: { ...env, HQ_SESSION_ID: sid } };
304
405
  }
305
406
  catch (err) {
306
407
  log("warn", `company bind failed for ${config.companySlug}: ${err instanceof Error ? err.message : String(err)}`);
307
408
  }
308
409
  }
309
- // Only the owner's own DM session (DMs and the kickoff). Room prompts must
310
- // keep their [#channel …] header first, and a room turn can be triggered
311
- // by other people, who must not be handed the owner's companies.
312
- if (deps.ownerContext && sessionScope === "dm") {
410
+ // Only a personal bot, and only in the owner's own DM session (DMs and the
411
+ // kickoff). Room prompts must keep their [#channel …] header first, and a
412
+ // room turn can be triggered by other people, who must not be handed the
413
+ // owner's companies. A company bot acts as itself and never gets it.
414
+ if (deps.ownerContext && sessionScope === "dm" && kind === "personal") {
313
415
  try {
314
416
  const owner = await deps.ownerContext();
315
417
  if (owner.status === "unavailable") {
@@ -366,6 +468,7 @@ export async function runBot(deps) {
366
468
  }
367
469
  }
368
470
  modelHealth = "ok";
471
+ markSignInWorks();
369
472
  log("info", `model turn done in ${Math.round((now().getTime() - tTurnStart) / 1000)}s (${Math.round((tTurnStart - tReceived) / 1000)}s before it)`);
370
473
  if (turn.sessionId) {
371
474
  writeBotSession(dir, {
@@ -419,7 +522,7 @@ export async function runBot(deps) {
419
522
  },
420
523
  });
421
524
  const controller = new AbortController();
422
- currentTurn = controller;
525
+ currentTurns.add(controller);
423
526
  poster.start();
424
527
  try {
425
528
  const turn = await executeTurn(args.prompt, args.sessionScope, args.tReceived, {
@@ -442,8 +545,7 @@ export async function runBot(deps) {
442
545
  return { outcome: { ok: false, started, aborted, excerpt, error: err }, poster };
443
546
  }
444
547
  finally {
445
- if (currentTurn === controller)
446
- currentTurn = null;
548
+ currentTurns.delete(controller);
447
549
  }
448
550
  };
449
551
  const failureReason = (outcome) => outcome.aborted ? STOPPED_BY_OWNER_REASON : `${runtime.id} stopped with an error: ${outcome.excerpt.slice(0, 160)}`;
@@ -526,11 +628,23 @@ export async function runBot(deps) {
526
628
  return;
527
629
  }
528
630
  clearInflight(dir, KICKOFF_MESSAGE_ID);
631
+ if (readBotConfig(dir)?.kickoffPendingSignIn)
632
+ patchBotConfig(dir, { kickoffPendingSignIn: false });
529
633
  const st = readStatus();
530
634
  setStatus({ repliesSent: (st?.repliesSent ?? 0) + 1, lastActivityAt: now().toISOString() });
531
635
  log("info", `kickoff reply sent (${outcome.reply.length} chars)`);
532
636
  return;
533
637
  }
638
+ if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
639
+ // Nothing ran: hold the kickoff until the tool can sign in, then run it.
640
+ clearInflight(dir, KICKOFF_MESSAGE_ID);
641
+ patchBotConfig(dir, { kickoffPendingSignIn: true });
642
+ markSignInBroken(outcome.excerpt);
643
+ await tellSignInNeeded("dm", (body) => api.sendDm({ toPersonUid: peerUid, body }), true);
644
+ return;
645
+ }
646
+ if (readBotConfig(dir)?.kickoffPendingSignIn)
647
+ patchBotConfig(dir, { kickoffPendingSignIn: false });
534
648
  let count = 0;
535
649
  if (!outcome.aborted)
536
650
  count = recordTurnFailure(outcome.error, "dm").count;
@@ -561,8 +675,8 @@ export async function runBot(deps) {
561
675
  }
562
676
  // A turn on this message already started once and was cut off: say so,
563
677
  // never run it again.
564
- const inflight = readInflight(dir);
565
- if (inflight?.messageId === id) {
678
+ const inflight = readInflight(dir, id);
679
+ if (inflight) {
566
680
  await recoverInflight(inflight);
567
681
  return;
568
682
  }
@@ -572,7 +686,7 @@ export async function runBot(deps) {
572
686
  // 404 on the server by design), then ack so it never comes back.
573
687
  try {
574
688
  if (sender)
575
- await api.sendDm({ toPersonUid: sender, body: nonOwnerRefusalText(config.name) });
689
+ await api.sendDm({ toPersonUid: sender, body: nonOwnerRefusalText(config.name, kind) });
576
690
  }
577
691
  catch {
578
692
  /* refusal delivery is best-effort */
@@ -670,6 +784,17 @@ export async function runBot(deps) {
670
784
  return;
671
785
  }
672
786
  await clearThinking();
787
+ if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
788
+ // The tool refused before doing anything: keep the message for when it
789
+ // can sign in again, and say so once in this conversation.
790
+ clearInflight(dir, id);
791
+ attempts.delete(id);
792
+ markSignInBroken(outcome.excerpt);
793
+ const conversation = markerTarget.kind === "dm" ? "dm" : `room:${markerTarget.channelId}:${item.rootEventId ?? ""}`;
794
+ await tellSignInNeeded(conversation, send);
795
+ log("warn", `kept ${id} until ${runtime.id} can sign in again`);
796
+ return; // un-acked: answered once the sign-in works
797
+ }
673
798
  if (!outcome.started) {
674
799
  // The model never ran (e.g. the binary is missing): nothing happened
675
800
  // outside this Mac, so retrying is safe.
@@ -721,9 +846,59 @@ export async function runBot(deps) {
721
846
  log("error", `kickoff turn crashed: ${err instanceof Error ? err.message : String(err)}`);
722
847
  })
723
848
  : Promise.resolve();
849
+ // ── Lanes: conversations side by side, each conversation in order ─────────
850
+ /** Messages waiting per lane, in arrival order. */
851
+ const lanes = new Map();
852
+ /** Lanes with a message being handled right now. */
853
+ const busyLanes = new Set();
854
+ /** Every message queued or being handled (the inbox re-delivers until acked). */
855
+ const pending = new Set();
856
+ const running = new Set();
857
+ const canStart = () => !stopping && state === "running" && !hasPromotionHold(dir);
858
+ const pump = () => {
859
+ for (const [lane, queue] of lanes) {
860
+ if (busyLanes.size >= maxParallelTurns || !canStart())
861
+ return;
862
+ // While the sign-in is broken, one message at a time tries it, on a timer.
863
+ if (signInIssue && (busyLanes.size > 0 || now().getTime() < nextSignInRetryAt))
864
+ return;
865
+ if (busyLanes.has(lane))
866
+ continue;
867
+ const item = queue.shift();
868
+ if (queue.length === 0)
869
+ lanes.delete(lane);
870
+ if (!item)
871
+ continue;
872
+ busyLanes.add(lane);
873
+ if (signInIssue)
874
+ nextSignInRetryAt = now().getTime() + signInRetryMs;
875
+ const task = handleItem(item)
876
+ .catch((err) => {
877
+ log("warn", `inbox item ${item.messageId} failed: ${err instanceof Error ? err.message : String(err)}`);
878
+ })
879
+ .finally(() => {
880
+ busyLanes.delete(lane);
881
+ pending.delete(item.messageId);
882
+ running.delete(task);
883
+ pump();
884
+ });
885
+ running.add(task);
886
+ }
887
+ };
888
+ const enqueue = (item) => {
889
+ if (pending.has(item.messageId))
890
+ return;
891
+ pending.add(item.messageId);
892
+ const lane = inboxLaneKey(item, config);
893
+ const queue = lanes.get(lane);
894
+ if (queue)
895
+ queue.push(item);
896
+ else
897
+ lanes.set(lane, [item]);
898
+ };
724
899
  const inboxLoop = (async () => {
725
- // A turn cut off by the last restart is reported before anything new runs.
726
- if (leftover) {
900
+ // Turns cut off by the last restart are reported before anything new runs.
901
+ for (const leftover of leftovers) {
727
902
  try {
728
903
  await recoverInflight(leftover);
729
904
  }
@@ -735,19 +910,22 @@ export async function runBot(deps) {
735
910
  while (!stopping && state === "running") {
736
911
  if (hasPromotionHold(dir))
737
912
  break;
913
+ // A kickoff held for a sign-in retries on the same timer as messages.
914
+ if (busyLanes.size === 0 &&
915
+ (!signInIssue || now().getTime() >= nextSignInRetryAt) &&
916
+ readBotConfig(dir)?.kickoffPendingSignIn) {
917
+ if (signInIssue)
918
+ nextSignInRetryAt = now().getTime() + signInRetryMs;
919
+ await runKickoff().catch((err) => {
920
+ log("error", `kickoff retry crashed: ${err instanceof Error ? err.message : String(err)}`);
921
+ });
922
+ }
738
923
  try {
739
924
  const items = await api.pullInbox(config.agentUid);
740
925
  setStatus({ lastInboxPollAt: now().toISOString() });
741
- for (const item of items) {
742
- if (stopping || state !== "running" || hasPromotionHold(dir))
743
- break;
744
- try {
745
- await handleItem(item);
746
- }
747
- catch (err) {
748
- log("warn", `inbox item ${item.messageId} failed: ${err instanceof Error ? err.message : String(err)}`);
749
- }
750
- }
926
+ for (const item of items)
927
+ enqueue(item);
928
+ pump();
751
929
  }
752
930
  catch (err) {
753
931
  log("warn", `inbox poll failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -756,6 +934,10 @@ export async function runBot(deps) {
756
934
  break;
757
935
  await sleep(inboxIntervalMs);
758
936
  }
937
+ // Stop, drain and promotion wait for the turns already running. Queued
938
+ // messages were never started or acked, so the inbox delivers them again.
939
+ while (running.size > 0)
940
+ await Promise.allSettled([...running]);
759
941
  })();
760
942
  function readStatus() {
761
943
  return readBotStatus(dir);
@@ -763,7 +945,8 @@ export async function runBot(deps) {
763
945
  const stop = async (mode = "abort") => {
764
946
  if (stopping) {
765
947
  if (mode === "abort")
766
- currentTurn?.abort();
948
+ for (const turn of currentTurns)
949
+ turn.abort();
767
950
  return;
768
951
  }
769
952
  stopping = true;
@@ -771,7 +954,8 @@ export async function runBot(deps) {
771
954
  // Promotion instead drains that turn, including its reply and ACK, before
772
955
  // the coordinator snapshots memory and starts the cloud consumer.
773
956
  if (mode === "abort")
774
- currentTurn?.abort();
957
+ for (const turn of currentTurns)
958
+ turn.abort();
775
959
  await Promise.allSettled([heartbeatLoop, inboxLoop]);
776
960
  await finish("stopped", 0, mode === "drain" ? "promotion drain completed" : "stop requested");
777
961
  };
@@ -789,6 +973,6 @@ export async function runBot(deps) {
789
973
  if (state === "running")
790
974
  void finish("stopped", 0, "loops ended");
791
975
  });
792
- return { stop, done, status: readStatus };
976
+ return { stop, done, status: readStatus, idle: () => pending.size === 0 };
793
977
  }
794
978
  //# sourceMappingURL=run.js.map
@@ -0,0 +1,24 @@
1
+ /**
2
+ * The bot's coding tool (Claude Code, Codex, Grok) is installed but its login
3
+ * no longer works: never signed in, expired, or revoked. The vendor CLI can
4
+ * still report itself as logged in (a saved but dead token), so the only
5
+ * reliable evidence is a turn that fails this way.
6
+ *
7
+ * Such a turn fails before the model does anything, so the message is safe to
8
+ * answer later. The bot says once that it needs a sign-in, keeps the message,
9
+ * and tries again on its own (and at once after `hq bot restart`).
10
+ */
11
+ import type { BotRuntimeId } from "./config.js";
12
+ /** Time between the bot's own retries while its sign-in is broken. */
13
+ export declare const SIGN_IN_RETRY_MS = 60000;
14
+ export declare function isRuntimeSignInFailure(text: string | undefined | null): boolean;
15
+ export declare function runtimeDisplayName(runtime: BotRuntimeId | string): string;
16
+ /** Reported in status.json and `hq bot list --json` while the sign-in is broken. */
17
+ export interface RuntimeSignInIssue {
18
+ state: "expired";
19
+ runtime: BotRuntimeId;
20
+ since: string;
21
+ }
22
+ /** The one message a conversation gets when the bot cannot work until its tool signs in again. */
23
+ export declare function signInNeededText(runtime: BotRuntimeId | string, kickoff?: boolean): string;
24
+ //# sourceMappingURL=runtime-sign-in.d.ts.map
@@ -0,0 +1,31 @@
1
+ /**
2
+ * The bot's coding tool (Claude Code, Codex, Grok) is installed but its login
3
+ * no longer works: never signed in, expired, or revoked. The vendor CLI can
4
+ * still report itself as logged in (a saved but dead token), so the only
5
+ * reliable evidence is a turn that fails this way.
6
+ *
7
+ * Such a turn fails before the model does anything, so the message is safe to
8
+ * answer later. The bot says once that it needs a sign-in, keeps the message,
9
+ * and tries again on its own (and at once after `hq bot restart`).
10
+ */
11
+ /** Time between the bot's own retries while its sign-in is broken. */
12
+ export const SIGN_IN_RETRY_MS = 60_000;
13
+ const SIGN_IN_FAILURE = /failed to authenticate|authentication[_ ]failed|not logged in|not signed in|please (?:run \/login|log ?in|sign ?in)|(?:oauth|access|refresh) token (?:has been |was )?(?:expired|revoked|invalid)|oauth session expired|invalid api key|\b401\b|unauthori[sz]ed|login (?:required|expired)|sign-?in (?:required|expired)/i;
14
+ /** Usage and rate limits are a different problem; a new sign-in does not fix them. */
15
+ const NOT_SIGN_IN = /usage[_ ]limit|rate[_ ]limit|quota|too many requests|credits/i;
16
+ export function isRuntimeSignInFailure(text) {
17
+ const t = (text ?? "").trim();
18
+ if (!t)
19
+ return false;
20
+ return SIGN_IN_FAILURE.test(t) && !NOT_SIGN_IN.test(t);
21
+ }
22
+ export function runtimeDisplayName(runtime) {
23
+ return runtime === "claude" ? "Claude Code" : runtime === "codex" ? "Codex" : runtime === "grok" ? "Grok" : runtime;
24
+ }
25
+ /** The one message a conversation gets when the bot cannot work until its tool signs in again. */
26
+ export function signInNeededText(runtime, kickoff = false) {
27
+ const tool = runtimeDisplayName(runtime);
28
+ return (`I can't work right now: ${tool} needs you to sign in again on this computer. ` +
29
+ `Use "Sign in to ${tool}" in HQ, and I'll ${kickoff ? "get started" : "answer this"} as soon as it works again — no need to send it twice.`);
30
+ }
31
+ //# sourceMappingURL=runtime-sign-in.js.map
@@ -2,6 +2,7 @@
2
2
  * status.json — atomic, metadata-only runtime status (local-bots US-003).
3
3
  */
4
4
  import type { BotRuntimeId } from "./config.js";
5
+ import type { RuntimeSignInIssue } from "./runtime-sign-in.js";
5
6
  export type BotState = "starting" | "running" | "stopped" | "failed";
6
7
  export interface BotStatusFile {
7
8
  v: 1;
@@ -29,6 +30,11 @@ export interface BotStatusFile {
29
30
  * Persisted so later turns skip the failed attempt.
30
31
  */
31
32
  claudePermissionMode?: string;
33
+ /**
34
+ * Set while the coding tool's sign-in is broken (a turn failed for that
35
+ * reason); cleared by the next turn that works. `null` = known to work.
36
+ */
37
+ runtimeSignIn?: RuntimeSignInIssue | null;
32
38
  updatedAt: string;
33
39
  }
34
40
  export declare function defaultBotStatus(pid: number, runtime: BotRuntimeId, now?: () => Date): BotStatusFile;