@indigoai-us/hq-cli 5.113.1 → 5.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/dist/command-catalog.generated.d.ts +28 -0
- package/dist/command-catalog.generated.js +37 -0
- package/dist/commands/bot-companies.d.ts +40 -0
- package/dist/commands/bot-companies.js +64 -0
- package/dist/commands/bot.d.ts +33 -1
- package/dist/commands/bot.js +155 -7
- package/dist/commands/onboard-bot-membership.d.ts +32 -0
- package/dist/commands/onboard-bot-membership.js +40 -0
- package/dist/commands/onboard.js +10 -0
- package/dist/lib/bot/api.d.ts +6 -0
- package/dist/lib/bot/api.js +6 -1
- package/dist/lib/bot/config.d.ts +35 -0
- package/dist/lib/bot/config.js +69 -2
- package/dist/lib/bot/inflight.d.ts +18 -7
- package/dist/lib/bot/inflight.js +57 -18
- package/dist/lib/bot/prompt.d.ts +13 -1
- package/dist/lib/bot/prompt.js +37 -12
- package/dist/lib/bot/run.d.ts +48 -5
- package/dist/lib/bot/run.js +231 -47
- package/dist/lib/bot/runtime-sign-in.d.ts +24 -0
- package/dist/lib/bot/runtime-sign-in.js +31 -0
- package/dist/lib/bot/status.d.ts +6 -0
- package/dist/utils/feedback-log-bundle.d.ts +61 -11
- package/dist/utils/feedback-log-bundle.js +209 -29
- package/package.json +1 -1
package/dist/lib/bot/run.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* - refuses to start unless the creds file names this bot's agt_ AND the
|
|
6
6
|
* server-side record is owned by the configured owner
|
|
7
7
|
* - heartbeat every 30s (POST /v1/agents/{uid}/heartbeat, agent JWT)
|
|
8
|
-
* - inbox poll every
|
|
8
|
+
* - inbox poll every 2s: owner DMs go to ONE headless model turn (resumed by
|
|
9
9
|
* session id); the reply is sent as the bot; the message is acked only after
|
|
10
10
|
* the reply is sent; processed ids persist so a crash between reply and ack
|
|
11
11
|
* never answers twice
|
|
@@ -16,12 +16,21 @@
|
|
|
16
16
|
* - room items (channels / group chats): answered when the bot is @mentioned,
|
|
17
17
|
* or when the owner posts in a small group (see room-policy.ts); claimed
|
|
18
18
|
* first (first-claim-wins), one session per room, reply posted in the room
|
|
19
|
+
* - conversations run side by side: up to MAX_PARALLEL_TURNS turns at once,
|
|
20
|
+
* one lane per model session (the owner DM session, or one room). Messages
|
|
21
|
+
* in the same lane run in order, so a session is never resumed by two turns
|
|
22
|
+
* at once and a thread's answers never arrive out of sequence.
|
|
19
23
|
* - while a turn runs, each finished assistant message is posted as its own
|
|
20
24
|
* message (progress.ts); turns have no time limit
|
|
21
25
|
* - every received message gets a reply: a turn that fails after the model
|
|
22
26
|
* started (error, crash, owner stop, restart) posts one "did not finish"
|
|
23
27
|
* message and is NOT run again — it may already have done outward things.
|
|
24
28
|
* inflight.json marks the turn in progress so a restart can say so.
|
|
29
|
+
* - a turn that fails because the coding tool's sign-in no longer works
|
|
30
|
+
* (runtime-sign-in.ts) is not a failure of the message: the bot says once
|
|
31
|
+
* per conversation that it needs a sign-in, keeps the message un-acked,
|
|
32
|
+
* holds new turns, and tries again every SIGN_IN_RETRY_MS (and at once
|
|
33
|
+
* after a restart). A turn that works clears it.
|
|
25
34
|
* - failures before the model starts (missing binary) retry with backoff;
|
|
26
35
|
* 5 model failures in 10 minutes → state `failed` and a clean exit (launchd
|
|
27
36
|
* does not respawn a clean exit). Owner stops never count.
|
|
@@ -35,17 +44,19 @@ import * as fs from "node:fs";
|
|
|
35
44
|
import * as path from "node:path";
|
|
36
45
|
import { fullJitterDelayMs } from "../mesh/live/backoff.js";
|
|
37
46
|
import { acquirePidLock, defaultPidLockDeps, releasePidLock } from "../mesh/live/daemon/pid-lock.js";
|
|
38
|
-
import { patchBotConfig, readBotConfig } from "./config.js";
|
|
47
|
+
import { effectiveBotCompanies, effectiveBotKind, patchBotConfig, readBotConfig, SETUP_BOT_WORKER_ID } from "./config.js";
|
|
39
48
|
import { ensureBotSessionMeta } from "./company-bind.js";
|
|
40
49
|
import { readBotCredsIdentity } from "./creds.js";
|
|
41
50
|
import { isProcessed, markProcessed, readInboxState } from "./inbox-state.js";
|
|
42
|
-
import { clearInflight, patchInflight, readInflight, writeInflight } from "./inflight.js";
|
|
51
|
+
import { clearInflight, patchInflight, readInflight, readInflightTurns, writeInflight } from "./inflight.js";
|
|
43
52
|
import { ProgressPoster } from "./progress.js";
|
|
44
53
|
import { createBotLogger } from "./log.js";
|
|
45
54
|
import { buildSystemPrompt, introDmText, nonOwnerRefusalText, readPromptSources } from "./prompt.js";
|
|
46
55
|
import { FALLBACK_CLAUDE_PERMISSION_MODE, isClaudeBypassDisabled } from "./runtime/claude.js";
|
|
47
56
|
import { runRuntimeTurn, RuntimeError } from "./runtime/index.js";
|
|
48
57
|
import { isDmItem, resolveReplyTarget } from "./room-policy.js";
|
|
58
|
+
import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
|
|
59
|
+
import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
|
|
49
60
|
import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
|
|
50
61
|
import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
|
|
51
62
|
export const INBOX_POLL_INTERVAL_MS = 2_000;
|
|
@@ -57,9 +68,41 @@ export const THINKING_EMOJI = "👀";
|
|
|
57
68
|
export const ROOM_THINKING_STATUS = "is thinking…";
|
|
58
69
|
export const RECENT_MESSAGES_LIMIT = 15;
|
|
59
70
|
export const RECENT_MESSAGE_MAX_CHARS = 400;
|
|
71
|
+
/** Model turns a bot runs at once (each on a different conversation). */
|
|
72
|
+
export const MAX_PARALLEL_TURNS = 2;
|
|
60
73
|
export const KICKOFF_MESSAGE_ID = "kickoff";
|
|
74
|
+
/**
|
|
75
|
+
* The environment a model turn's `hq` commands run in.
|
|
76
|
+
*
|
|
77
|
+
* The bot process carries its own machine identity in HQ_MACHINE_CREDS_FILE /
|
|
78
|
+
* HQ_MACHINE_TOKEN_STATE_DIR so its inbox, heartbeat and posts are the bot's.
|
|
79
|
+
* What a turn inherits depends on the bot's kind:
|
|
80
|
+
* - A PERSONAL bot acts as its owner on every turn (DMs, the kickoff, rooms):
|
|
81
|
+
* everything the owner asks for — a company, a bot, an invite, a share —
|
|
82
|
+
* belongs to the owner's account, and the cloud rightly refuses a bot that
|
|
83
|
+
* tries to own it ("No person entity found"). Stripping the marker makes
|
|
84
|
+
* every `hq` command in the turn use the owner's sign-in, the same one the
|
|
85
|
+
* desktop app shares, so no command needs a special case. The bot's id
|
|
86
|
+
* (never a credential) rides along in HQ_BOT_AGENT_UID so what the owner
|
|
87
|
+
* creates in the turn — a company — can add the bot as a member afterwards.
|
|
88
|
+
* - A COMPANY bot acts as itself on every turn, like a cloud agent: the
|
|
89
|
+
* machine identity stays, and the cloud evaluates it through its own
|
|
90
|
+
* company memberships. It never acts as the owner, in DMs or in rooms.
|
|
91
|
+
* Pure so the rule is unit-testable.
|
|
92
|
+
*/
|
|
93
|
+
export function turnEnv(base, kind, agentUid) {
|
|
94
|
+
const env = { ...base };
|
|
95
|
+
delete env[HQ_BOT_AGENT_UID_ENV];
|
|
96
|
+
if (kind === "personal") {
|
|
97
|
+
delete env.HQ_MACHINE_CREDS_FILE;
|
|
98
|
+
delete env.HQ_MACHINE_TOKEN_STATE_DIR;
|
|
99
|
+
if (agentUid)
|
|
100
|
+
env[HQ_BOT_AGENT_UID_ENV] = agentUid;
|
|
101
|
+
}
|
|
102
|
+
return env;
|
|
103
|
+
}
|
|
61
104
|
/** The core worker HQ's setup bot runs. */
|
|
62
|
-
export const SETUP_WORKER_ID =
|
|
105
|
+
export const SETUP_WORKER_ID = SETUP_BOT_WORKER_ID;
|
|
63
106
|
/** The one message a person gets when a turn on their message did not finish. */
|
|
64
107
|
export function didNotFinishText(reason, progressPosted) {
|
|
65
108
|
const r = reason.trim().replace(/[.\s]+$/, "");
|
|
@@ -68,6 +111,17 @@ export function didNotFinishText(reason, progressPosted) {
|
|
|
68
111
|
/** Owner stop, `hq bot stop` or `hq bot restart`: all arrive as the same signal. */
|
|
69
112
|
export const STOPPED_BY_OWNER_REASON = "I was stopped before I was done";
|
|
70
113
|
export const RESTARTED_REASON = "I was restarted while I was working on it";
|
|
114
|
+
/**
|
|
115
|
+
* The lane an inbox item runs in: the model session its turn resumes. Owner
|
|
116
|
+
* DMs share the DM session; each room has its own. An item that never reaches
|
|
117
|
+
* a model turn (refusal, not addressed to the bot) gets a lane of its own.
|
|
118
|
+
*/
|
|
119
|
+
export function inboxLaneKey(item, config) {
|
|
120
|
+
if (isDmItem(item))
|
|
121
|
+
return "dm";
|
|
122
|
+
const target = resolveReplyTarget(item, config);
|
|
123
|
+
return target?.kind === "room" ? target.sessionScope : `item:${item.messageId}`;
|
|
124
|
+
}
|
|
71
125
|
class FailureWindow {
|
|
72
126
|
windowMs;
|
|
73
127
|
limit;
|
|
@@ -117,8 +171,9 @@ export async function runBot(deps) {
|
|
|
117
171
|
const setStatus = (patch) => patchBotStatus(dir, patch, fallback, now);
|
|
118
172
|
let state = "starting";
|
|
119
173
|
let stopping = false;
|
|
120
|
-
/** Aborts the model
|
|
121
|
-
|
|
174
|
+
/** Aborts the model turns in progress (owner stop). */
|
|
175
|
+
const currentTurns = new Set();
|
|
176
|
+
const maxParallelTurns = Math.max(1, deps.maxParallelTurns ?? MAX_PARALLEL_TURNS);
|
|
122
177
|
let resolveDone;
|
|
123
178
|
const done = new Promise((r) => (resolveDone = r));
|
|
124
179
|
// Only override isProcessAlive when a caller injected one: spreading an
|
|
@@ -130,6 +185,9 @@ export async function runBot(deps) {
|
|
|
130
185
|
const turnCwd = config.hqRoot;
|
|
131
186
|
// HQ's setup bot is a guided conversation: no "working on it" filler between steps.
|
|
132
187
|
const isSetupBot = config.workerId === SETUP_WORKER_ID;
|
|
188
|
+
// Personal: every turn acts as the owner. Company: every turn acts as the bot.
|
|
189
|
+
const kind = effectiveBotKind(config);
|
|
190
|
+
const companies = effectiveBotCompanies(config);
|
|
133
191
|
const lockDeps = defaultPidLockDeps({
|
|
134
192
|
pid,
|
|
135
193
|
...(deps.isProcessAlive ? { isProcessAlive: deps.isProcessAlive } : {}),
|
|
@@ -152,14 +210,14 @@ export async function runBot(deps) {
|
|
|
152
210
|
log("error", credsProblem);
|
|
153
211
|
resolveDone("failed");
|
|
154
212
|
exit(0);
|
|
155
|
-
return { stop: async () => { }, done, status: () => null };
|
|
213
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
156
214
|
}
|
|
157
215
|
if (config.enabled === false) {
|
|
158
216
|
setStatus({ state: "stopped", pid });
|
|
159
217
|
log("info", "bot is disabled (hq bot start to enable)");
|
|
160
218
|
resolveDone("stopped");
|
|
161
219
|
exit(0);
|
|
162
|
-
return { stop: async () => { }, done, status: () => null };
|
|
220
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
163
221
|
}
|
|
164
222
|
if (!deps.skipPidLock) {
|
|
165
223
|
const lock = acquirePidLock(dir, lockDeps);
|
|
@@ -168,32 +226,33 @@ export async function runBot(deps) {
|
|
|
168
226
|
log("warn", msg);
|
|
169
227
|
resolveDone("stopped");
|
|
170
228
|
exit(0);
|
|
171
|
-
return { stop: async () => { }, done, status: () => null };
|
|
229
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
172
230
|
}
|
|
173
231
|
}
|
|
174
232
|
if (hasPromotionHold(dir)) {
|
|
175
233
|
await finish("stopped", 0, "local bot held for cloud promotion");
|
|
176
|
-
return { stop: async () => { }, done, status: () => null };
|
|
234
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
177
235
|
}
|
|
178
|
-
|
|
179
|
-
|
|
236
|
+
// A restart is the owner's "try again" after signing in: start clean.
|
|
237
|
+
setStatus({ ...fallback(), pid, state: "starting", canDrainForPromotion: false, runtimeSignIn: null });
|
|
238
|
+
log("info", `starting ${config.name} (${config.runtime}, ${kind} bot${companies.length ? ` of ${companies.join(", ")}` : ""}) as ${config.agentUid} for ${config.ownerUid}`);
|
|
180
239
|
try {
|
|
181
240
|
const record = await api.getAgent(config.agentUid);
|
|
182
241
|
// Promotion keeps the identity and owner. Ownership alone must not let
|
|
183
242
|
// an old local installation start consuming the cloud bot's inbox.
|
|
184
|
-
if (record.uid !== config.agentUid || record.computeMode !== "local" || record.botKind !==
|
|
185
|
-
await finish("failed", 0, `identity ${config.agentUid} is not a
|
|
186
|
-
return { stop: async () => { }, done, status: () => null };
|
|
243
|
+
if (record.uid !== config.agentUid || record.computeMode !== "local" || record.botKind !== kind) {
|
|
244
|
+
await finish("failed", 0, `identity ${config.agentUid} is not a ${kind} local bot; this installation cannot run it`);
|
|
245
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
187
246
|
}
|
|
188
247
|
const owner = record.ownerUid ?? null;
|
|
189
248
|
if (owner !== config.ownerUid) {
|
|
190
249
|
await finish("failed", 0, `identity ${config.agentUid} is owned by ${owner ?? "nobody"}, not ${config.ownerUid}`);
|
|
191
|
-
return { stop: async () => { }, done, status: () => null };
|
|
250
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
192
251
|
}
|
|
193
252
|
}
|
|
194
253
|
catch (err) {
|
|
195
254
|
await finish("failed", 0, `could not verify bot identity: ${err instanceof Error ? err.message : String(err)}`);
|
|
196
|
-
return { stop: async () => { }, done, status: () => null };
|
|
255
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
197
256
|
}
|
|
198
257
|
// ── System prompt ──────────────────────────────────────────────────────────
|
|
199
258
|
const systemPrompt = deps.systemPromptOverride ??
|
|
@@ -206,6 +265,8 @@ export async function runBot(deps) {
|
|
|
206
265
|
workerSource: config.workerSource,
|
|
207
266
|
workerId: config.workerId,
|
|
208
267
|
companySlug: config.companySlug,
|
|
268
|
+
kind,
|
|
269
|
+
companies,
|
|
209
270
|
memoryDir: config.memoryDir,
|
|
210
271
|
sources: readPromptSources(config.hqRoot, config.workerDir, undefined, {
|
|
211
272
|
memoryDir: config.memoryDir,
|
|
@@ -218,20 +279,24 @@ export async function runBot(deps) {
|
|
|
218
279
|
fs.writeFileSync(systemPromptFile, systemPrompt, { mode: 0o600 });
|
|
219
280
|
if (hasPromotionHold(dir)) {
|
|
220
281
|
await finish("stopped", 0, "local bot held for cloud promotion");
|
|
221
|
-
return { stop: async () => { }, done, status: () => null };
|
|
282
|
+
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
222
283
|
}
|
|
223
284
|
state = "running";
|
|
224
285
|
setStatus({ state: "running", lastActivityAt: now().toISOString() });
|
|
225
286
|
log("info", "running");
|
|
226
287
|
// Read before the kickoff can write its own marker: whatever is here now was
|
|
227
288
|
// left by a previous run that was cut off mid-turn.
|
|
228
|
-
const
|
|
289
|
+
const leftovers = readInflightTurns(dir);
|
|
229
290
|
// ── Intro DM (once) ────────────────────────────────────────────────────────
|
|
230
291
|
// The kickoff turn is tied to the intro: it runs only in the start that sent
|
|
231
292
|
// the intro, so introSentAt (persisted before the turn) guarantees it never
|
|
232
293
|
// runs again on a restart.
|
|
233
294
|
let kickoffAfterIntro = null;
|
|
234
|
-
if (
|
|
295
|
+
if (config.introSentAt && config.kickoffPendingSignIn && config.kickoff?.trim()) {
|
|
296
|
+
// The kickoff was held for a sign-in on an earlier run; this start retries it.
|
|
297
|
+
kickoffAfterIntro = {};
|
|
298
|
+
}
|
|
299
|
+
else if (!config.introSentAt) {
|
|
235
300
|
try {
|
|
236
301
|
const sent = await api.sendDm({ toPersonUid: config.ownerUid, body: introDmText(config.name, config.runtime, config.intro) });
|
|
237
302
|
patchBotConfig(dir, { introSentAt: now().toISOString() });
|
|
@@ -246,6 +311,41 @@ export async function runBot(deps) {
|
|
|
246
311
|
// ── Heartbeat loop ─────────────────────────────────────────────────────────
|
|
247
312
|
const failures = new FailureWindow(FAILURE_WINDOW_MS, FAILURE_LIMIT);
|
|
248
313
|
let modelHealth = "ok";
|
|
314
|
+
// ── Coding tool sign-in ────────────────────────────────────────────────────
|
|
315
|
+
const signInRetryMs = Math.max(0, deps.signInRetryMs ?? SIGN_IN_RETRY_MS);
|
|
316
|
+
let signInIssue = null;
|
|
317
|
+
let nextSignInRetryAt = 0;
|
|
318
|
+
/** Conversations already told this time that the bot needs a sign-in. */
|
|
319
|
+
const toldAboutSignIn = new Set();
|
|
320
|
+
const markSignInBroken = (excerpt) => {
|
|
321
|
+
nextSignInRetryAt = now().getTime() + signInRetryMs;
|
|
322
|
+
modelHealth = "degraded";
|
|
323
|
+
if (signInIssue)
|
|
324
|
+
return;
|
|
325
|
+
signInIssue = { state: "expired", runtime: config.runtime, since: now().toISOString() };
|
|
326
|
+
setStatus({ runtimeSignIn: signInIssue, lastError: excerpt.slice(0, 400) });
|
|
327
|
+
log("warn", `${runtime.id} needs to sign in again on this computer; holding turns and retrying every ${Math.round(signInRetryMs / 1000)}s`);
|
|
328
|
+
};
|
|
329
|
+
const markSignInWorks = () => {
|
|
330
|
+
if (!signInIssue)
|
|
331
|
+
return;
|
|
332
|
+
signInIssue = null;
|
|
333
|
+
toldAboutSignIn.clear();
|
|
334
|
+
setStatus({ runtimeSignIn: null });
|
|
335
|
+
log("info", `${runtime.id} sign-in works again; resuming`);
|
|
336
|
+
};
|
|
337
|
+
/** Say once per conversation that the bot cannot work until its tool signs in. */
|
|
338
|
+
const tellSignInNeeded = async (conversation, post, kickoff = false) => {
|
|
339
|
+
if (toldAboutSignIn.has(conversation))
|
|
340
|
+
return;
|
|
341
|
+
try {
|
|
342
|
+
await post(signInNeededText(config.runtime, kickoff));
|
|
343
|
+
toldAboutSignIn.add(conversation);
|
|
344
|
+
}
|
|
345
|
+
catch (err) {
|
|
346
|
+
log("warn", `could not say that ${runtime.id} needs a sign-in: ${err instanceof Error ? err.message : String(err)}`);
|
|
347
|
+
}
|
|
348
|
+
};
|
|
249
349
|
const heartbeatLoop = (async () => {
|
|
250
350
|
while (!stopping && state === "running") {
|
|
251
351
|
try {
|
|
@@ -296,20 +396,22 @@ export async function runBot(deps) {
|
|
|
296
396
|
// resets). Claude gets the full hook set via its payload session id;
|
|
297
397
|
// Codex/Grok only see HQ_SESSION_ID.
|
|
298
398
|
const sid = session?.sessionId ?? randomUUID();
|
|
299
|
-
|
|
399
|
+
const env = turnEnv(process.env, kind, config.agentUid);
|
|
400
|
+
let bind = { env };
|
|
300
401
|
if (config.companySlug) {
|
|
301
402
|
try {
|
|
302
403
|
ensureBotSessionMeta(config.hqRoot, sid, config.companySlug, undefined, now);
|
|
303
|
-
bind = { newSessionId: sid, env: { ...
|
|
404
|
+
bind = { newSessionId: sid, env: { ...env, HQ_SESSION_ID: sid } };
|
|
304
405
|
}
|
|
305
406
|
catch (err) {
|
|
306
407
|
log("warn", `company bind failed for ${config.companySlug}: ${err instanceof Error ? err.message : String(err)}`);
|
|
307
408
|
}
|
|
308
409
|
}
|
|
309
|
-
// Only the owner's own DM session (DMs and the
|
|
310
|
-
// keep their [#channel …] header first, and a
|
|
311
|
-
// by other people, who must not be handed the
|
|
312
|
-
|
|
410
|
+
// Only a personal bot, and only in the owner's own DM session (DMs and the
|
|
411
|
+
// kickoff). Room prompts must keep their [#channel …] header first, and a
|
|
412
|
+
// room turn can be triggered by other people, who must not be handed the
|
|
413
|
+
// owner's companies. A company bot acts as itself and never gets it.
|
|
414
|
+
if (deps.ownerContext && sessionScope === "dm" && kind === "personal") {
|
|
313
415
|
try {
|
|
314
416
|
const owner = await deps.ownerContext();
|
|
315
417
|
if (owner.status === "unavailable") {
|
|
@@ -366,6 +468,7 @@ export async function runBot(deps) {
|
|
|
366
468
|
}
|
|
367
469
|
}
|
|
368
470
|
modelHealth = "ok";
|
|
471
|
+
markSignInWorks();
|
|
369
472
|
log("info", `model turn done in ${Math.round((now().getTime() - tTurnStart) / 1000)}s (${Math.round((tTurnStart - tReceived) / 1000)}s before it)`);
|
|
370
473
|
if (turn.sessionId) {
|
|
371
474
|
writeBotSession(dir, {
|
|
@@ -419,7 +522,7 @@ export async function runBot(deps) {
|
|
|
419
522
|
},
|
|
420
523
|
});
|
|
421
524
|
const controller = new AbortController();
|
|
422
|
-
|
|
525
|
+
currentTurns.add(controller);
|
|
423
526
|
poster.start();
|
|
424
527
|
try {
|
|
425
528
|
const turn = await executeTurn(args.prompt, args.sessionScope, args.tReceived, {
|
|
@@ -442,8 +545,7 @@ export async function runBot(deps) {
|
|
|
442
545
|
return { outcome: { ok: false, started, aborted, excerpt, error: err }, poster };
|
|
443
546
|
}
|
|
444
547
|
finally {
|
|
445
|
-
|
|
446
|
-
currentTurn = null;
|
|
548
|
+
currentTurns.delete(controller);
|
|
447
549
|
}
|
|
448
550
|
};
|
|
449
551
|
const failureReason = (outcome) => outcome.aborted ? STOPPED_BY_OWNER_REASON : `${runtime.id} stopped with an error: ${outcome.excerpt.slice(0, 160)}`;
|
|
@@ -526,11 +628,23 @@ export async function runBot(deps) {
|
|
|
526
628
|
return;
|
|
527
629
|
}
|
|
528
630
|
clearInflight(dir, KICKOFF_MESSAGE_ID);
|
|
631
|
+
if (readBotConfig(dir)?.kickoffPendingSignIn)
|
|
632
|
+
patchBotConfig(dir, { kickoffPendingSignIn: false });
|
|
529
633
|
const st = readStatus();
|
|
530
634
|
setStatus({ repliesSent: (st?.repliesSent ?? 0) + 1, lastActivityAt: now().toISOString() });
|
|
531
635
|
log("info", `kickoff reply sent (${outcome.reply.length} chars)`);
|
|
532
636
|
return;
|
|
533
637
|
}
|
|
638
|
+
if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
|
|
639
|
+
// Nothing ran: hold the kickoff until the tool can sign in, then run it.
|
|
640
|
+
clearInflight(dir, KICKOFF_MESSAGE_ID);
|
|
641
|
+
patchBotConfig(dir, { kickoffPendingSignIn: true });
|
|
642
|
+
markSignInBroken(outcome.excerpt);
|
|
643
|
+
await tellSignInNeeded("dm", (body) => api.sendDm({ toPersonUid: peerUid, body }), true);
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
if (readBotConfig(dir)?.kickoffPendingSignIn)
|
|
647
|
+
patchBotConfig(dir, { kickoffPendingSignIn: false });
|
|
534
648
|
let count = 0;
|
|
535
649
|
if (!outcome.aborted)
|
|
536
650
|
count = recordTurnFailure(outcome.error, "dm").count;
|
|
@@ -561,8 +675,8 @@ export async function runBot(deps) {
|
|
|
561
675
|
}
|
|
562
676
|
// A turn on this message already started once and was cut off: say so,
|
|
563
677
|
// never run it again.
|
|
564
|
-
const inflight = readInflight(dir);
|
|
565
|
-
if (inflight
|
|
678
|
+
const inflight = readInflight(dir, id);
|
|
679
|
+
if (inflight) {
|
|
566
680
|
await recoverInflight(inflight);
|
|
567
681
|
return;
|
|
568
682
|
}
|
|
@@ -572,7 +686,7 @@ export async function runBot(deps) {
|
|
|
572
686
|
// 404 on the server by design), then ack so it never comes back.
|
|
573
687
|
try {
|
|
574
688
|
if (sender)
|
|
575
|
-
await api.sendDm({ toPersonUid: sender, body: nonOwnerRefusalText(config.name) });
|
|
689
|
+
await api.sendDm({ toPersonUid: sender, body: nonOwnerRefusalText(config.name, kind) });
|
|
576
690
|
}
|
|
577
691
|
catch {
|
|
578
692
|
/* refusal delivery is best-effort */
|
|
@@ -670,6 +784,17 @@ export async function runBot(deps) {
|
|
|
670
784
|
return;
|
|
671
785
|
}
|
|
672
786
|
await clearThinking();
|
|
787
|
+
if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
|
|
788
|
+
// The tool refused before doing anything: keep the message for when it
|
|
789
|
+
// can sign in again, and say so once in this conversation.
|
|
790
|
+
clearInflight(dir, id);
|
|
791
|
+
attempts.delete(id);
|
|
792
|
+
markSignInBroken(outcome.excerpt);
|
|
793
|
+
const conversation = markerTarget.kind === "dm" ? "dm" : `room:${markerTarget.channelId}:${item.rootEventId ?? ""}`;
|
|
794
|
+
await tellSignInNeeded(conversation, send);
|
|
795
|
+
log("warn", `kept ${id} until ${runtime.id} can sign in again`);
|
|
796
|
+
return; // un-acked: answered once the sign-in works
|
|
797
|
+
}
|
|
673
798
|
if (!outcome.started) {
|
|
674
799
|
// The model never ran (e.g. the binary is missing): nothing happened
|
|
675
800
|
// outside this Mac, so retrying is safe.
|
|
@@ -721,9 +846,59 @@ export async function runBot(deps) {
|
|
|
721
846
|
log("error", `kickoff turn crashed: ${err instanceof Error ? err.message : String(err)}`);
|
|
722
847
|
})
|
|
723
848
|
: Promise.resolve();
|
|
849
|
+
// ── Lanes: conversations side by side, each conversation in order ─────────
|
|
850
|
+
/** Messages waiting per lane, in arrival order. */
|
|
851
|
+
const lanes = new Map();
|
|
852
|
+
/** Lanes with a message being handled right now. */
|
|
853
|
+
const busyLanes = new Set();
|
|
854
|
+
/** Every message queued or being handled (the inbox re-delivers until acked). */
|
|
855
|
+
const pending = new Set();
|
|
856
|
+
const running = new Set();
|
|
857
|
+
const canStart = () => !stopping && state === "running" && !hasPromotionHold(dir);
|
|
858
|
+
const pump = () => {
|
|
859
|
+
for (const [lane, queue] of lanes) {
|
|
860
|
+
if (busyLanes.size >= maxParallelTurns || !canStart())
|
|
861
|
+
return;
|
|
862
|
+
// While the sign-in is broken, one message at a time tries it, on a timer.
|
|
863
|
+
if (signInIssue && (busyLanes.size > 0 || now().getTime() < nextSignInRetryAt))
|
|
864
|
+
return;
|
|
865
|
+
if (busyLanes.has(lane))
|
|
866
|
+
continue;
|
|
867
|
+
const item = queue.shift();
|
|
868
|
+
if (queue.length === 0)
|
|
869
|
+
lanes.delete(lane);
|
|
870
|
+
if (!item)
|
|
871
|
+
continue;
|
|
872
|
+
busyLanes.add(lane);
|
|
873
|
+
if (signInIssue)
|
|
874
|
+
nextSignInRetryAt = now().getTime() + signInRetryMs;
|
|
875
|
+
const task = handleItem(item)
|
|
876
|
+
.catch((err) => {
|
|
877
|
+
log("warn", `inbox item ${item.messageId} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
878
|
+
})
|
|
879
|
+
.finally(() => {
|
|
880
|
+
busyLanes.delete(lane);
|
|
881
|
+
pending.delete(item.messageId);
|
|
882
|
+
running.delete(task);
|
|
883
|
+
pump();
|
|
884
|
+
});
|
|
885
|
+
running.add(task);
|
|
886
|
+
}
|
|
887
|
+
};
|
|
888
|
+
const enqueue = (item) => {
|
|
889
|
+
if (pending.has(item.messageId))
|
|
890
|
+
return;
|
|
891
|
+
pending.add(item.messageId);
|
|
892
|
+
const lane = inboxLaneKey(item, config);
|
|
893
|
+
const queue = lanes.get(lane);
|
|
894
|
+
if (queue)
|
|
895
|
+
queue.push(item);
|
|
896
|
+
else
|
|
897
|
+
lanes.set(lane, [item]);
|
|
898
|
+
};
|
|
724
899
|
const inboxLoop = (async () => {
|
|
725
|
-
//
|
|
726
|
-
|
|
900
|
+
// Turns cut off by the last restart are reported before anything new runs.
|
|
901
|
+
for (const leftover of leftovers) {
|
|
727
902
|
try {
|
|
728
903
|
await recoverInflight(leftover);
|
|
729
904
|
}
|
|
@@ -735,19 +910,22 @@ export async function runBot(deps) {
|
|
|
735
910
|
while (!stopping && state === "running") {
|
|
736
911
|
if (hasPromotionHold(dir))
|
|
737
912
|
break;
|
|
913
|
+
// A kickoff held for a sign-in retries on the same timer as messages.
|
|
914
|
+
if (busyLanes.size === 0 &&
|
|
915
|
+
(!signInIssue || now().getTime() >= nextSignInRetryAt) &&
|
|
916
|
+
readBotConfig(dir)?.kickoffPendingSignIn) {
|
|
917
|
+
if (signInIssue)
|
|
918
|
+
nextSignInRetryAt = now().getTime() + signInRetryMs;
|
|
919
|
+
await runKickoff().catch((err) => {
|
|
920
|
+
log("error", `kickoff retry crashed: ${err instanceof Error ? err.message : String(err)}`);
|
|
921
|
+
});
|
|
922
|
+
}
|
|
738
923
|
try {
|
|
739
924
|
const items = await api.pullInbox(config.agentUid);
|
|
740
925
|
setStatus({ lastInboxPollAt: now().toISOString() });
|
|
741
|
-
for (const item of items)
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
try {
|
|
745
|
-
await handleItem(item);
|
|
746
|
-
}
|
|
747
|
-
catch (err) {
|
|
748
|
-
log("warn", `inbox item ${item.messageId} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
749
|
-
}
|
|
750
|
-
}
|
|
926
|
+
for (const item of items)
|
|
927
|
+
enqueue(item);
|
|
928
|
+
pump();
|
|
751
929
|
}
|
|
752
930
|
catch (err) {
|
|
753
931
|
log("warn", `inbox poll failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -756,6 +934,10 @@ export async function runBot(deps) {
|
|
|
756
934
|
break;
|
|
757
935
|
await sleep(inboxIntervalMs);
|
|
758
936
|
}
|
|
937
|
+
// Stop, drain and promotion wait for the turns already running. Queued
|
|
938
|
+
// messages were never started or acked, so the inbox delivers them again.
|
|
939
|
+
while (running.size > 0)
|
|
940
|
+
await Promise.allSettled([...running]);
|
|
759
941
|
})();
|
|
760
942
|
function readStatus() {
|
|
761
943
|
return readBotStatus(dir);
|
|
@@ -763,7 +945,8 @@ export async function runBot(deps) {
|
|
|
763
945
|
const stop = async (mode = "abort") => {
|
|
764
946
|
if (stopping) {
|
|
765
947
|
if (mode === "abort")
|
|
766
|
-
|
|
948
|
+
for (const turn of currentTurns)
|
|
949
|
+
turn.abort();
|
|
767
950
|
return;
|
|
768
951
|
}
|
|
769
952
|
stopping = true;
|
|
@@ -771,7 +954,8 @@ export async function runBot(deps) {
|
|
|
771
954
|
// Promotion instead drains that turn, including its reply and ACK, before
|
|
772
955
|
// the coordinator snapshots memory and starts the cloud consumer.
|
|
773
956
|
if (mode === "abort")
|
|
774
|
-
|
|
957
|
+
for (const turn of currentTurns)
|
|
958
|
+
turn.abort();
|
|
775
959
|
await Promise.allSettled([heartbeatLoop, inboxLoop]);
|
|
776
960
|
await finish("stopped", 0, mode === "drain" ? "promotion drain completed" : "stop requested");
|
|
777
961
|
};
|
|
@@ -789,6 +973,6 @@ export async function runBot(deps) {
|
|
|
789
973
|
if (state === "running")
|
|
790
974
|
void finish("stopped", 0, "loops ended");
|
|
791
975
|
});
|
|
792
|
-
return { stop, done, status: readStatus };
|
|
976
|
+
return { stop, done, status: readStatus, idle: () => pending.size === 0 };
|
|
793
977
|
}
|
|
794
978
|
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bot's coding tool (Claude Code, Codex, Grok) is installed but its login
|
|
3
|
+
* no longer works: never signed in, expired, or revoked. The vendor CLI can
|
|
4
|
+
* still report itself as logged in (a saved but dead token), so the only
|
|
5
|
+
* reliable evidence is a turn that fails this way.
|
|
6
|
+
*
|
|
7
|
+
* Such a turn fails before the model does anything, so the message is safe to
|
|
8
|
+
* answer later. The bot says once that it needs a sign-in, keeps the message,
|
|
9
|
+
* and tries again on its own (and at once after `hq bot restart`).
|
|
10
|
+
*/
|
|
11
|
+
import type { BotRuntimeId } from "./config.js";
|
|
12
|
+
/** Time between the bot's own retries while its sign-in is broken. */
|
|
13
|
+
export declare const SIGN_IN_RETRY_MS = 60000;
|
|
14
|
+
export declare function isRuntimeSignInFailure(text: string | undefined | null): boolean;
|
|
15
|
+
export declare function runtimeDisplayName(runtime: BotRuntimeId | string): string;
|
|
16
|
+
/** Reported in status.json and `hq bot list --json` while the sign-in is broken. */
|
|
17
|
+
export interface RuntimeSignInIssue {
|
|
18
|
+
state: "expired";
|
|
19
|
+
runtime: BotRuntimeId;
|
|
20
|
+
since: string;
|
|
21
|
+
}
|
|
22
|
+
/** The one message a conversation gets when the bot cannot work until its tool signs in again. */
|
|
23
|
+
export declare function signInNeededText(runtime: BotRuntimeId | string, kickoff?: boolean): string;
|
|
24
|
+
//# sourceMappingURL=runtime-sign-in.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bot's coding tool (Claude Code, Codex, Grok) is installed but its login
|
|
3
|
+
* no longer works: never signed in, expired, or revoked. The vendor CLI can
|
|
4
|
+
* still report itself as logged in (a saved but dead token), so the only
|
|
5
|
+
* reliable evidence is a turn that fails this way.
|
|
6
|
+
*
|
|
7
|
+
* Such a turn fails before the model does anything, so the message is safe to
|
|
8
|
+
* answer later. The bot says once that it needs a sign-in, keeps the message,
|
|
9
|
+
* and tries again on its own (and at once after `hq bot restart`).
|
|
10
|
+
*/
|
|
11
|
+
/** Time between the bot's own retries while its sign-in is broken. */
|
|
12
|
+
export const SIGN_IN_RETRY_MS = 60_000;
|
|
13
|
+
const SIGN_IN_FAILURE = /failed to authenticate|authentication[_ ]failed|not logged in|not signed in|please (?:run \/login|log ?in|sign ?in)|(?:oauth|access|refresh) token (?:has been |was )?(?:expired|revoked|invalid)|oauth session expired|invalid api key|\b401\b|unauthori[sz]ed|login (?:required|expired)|sign-?in (?:required|expired)/i;
|
|
14
|
+
/** Usage and rate limits are a different problem; a new sign-in does not fix them. */
|
|
15
|
+
const NOT_SIGN_IN = /usage[_ ]limit|rate[_ ]limit|quota|too many requests|credits/i;
|
|
16
|
+
export function isRuntimeSignInFailure(text) {
|
|
17
|
+
const t = (text ?? "").trim();
|
|
18
|
+
if (!t)
|
|
19
|
+
return false;
|
|
20
|
+
return SIGN_IN_FAILURE.test(t) && !NOT_SIGN_IN.test(t);
|
|
21
|
+
}
|
|
22
|
+
export function runtimeDisplayName(runtime) {
|
|
23
|
+
return runtime === "claude" ? "Claude Code" : runtime === "codex" ? "Codex" : runtime === "grok" ? "Grok" : runtime;
|
|
24
|
+
}
|
|
25
|
+
/** The one message a conversation gets when the bot cannot work until its tool signs in again. */
|
|
26
|
+
export function signInNeededText(runtime, kickoff = false) {
|
|
27
|
+
const tool = runtimeDisplayName(runtime);
|
|
28
|
+
return (`I can't work right now: ${tool} needs you to sign in again on this computer. ` +
|
|
29
|
+
`Use "Sign in to ${tool}" in HQ, and I'll ${kickoff ? "get started" : "answer this"} as soon as it works again — no need to send it twice.`);
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=runtime-sign-in.js.map
|
package/dist/lib/bot/status.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* status.json — atomic, metadata-only runtime status (local-bots US-003).
|
|
3
3
|
*/
|
|
4
4
|
import type { BotRuntimeId } from "./config.js";
|
|
5
|
+
import type { RuntimeSignInIssue } from "./runtime-sign-in.js";
|
|
5
6
|
export type BotState = "starting" | "running" | "stopped" | "failed";
|
|
6
7
|
export interface BotStatusFile {
|
|
7
8
|
v: 1;
|
|
@@ -29,6 +30,11 @@ export interface BotStatusFile {
|
|
|
29
30
|
* Persisted so later turns skip the failed attempt.
|
|
30
31
|
*/
|
|
31
32
|
claudePermissionMode?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Set while the coding tool's sign-in is broken (a turn failed for that
|
|
35
|
+
* reason); cleared by the next turn that works. `null` = known to work.
|
|
36
|
+
*/
|
|
37
|
+
runtimeSignIn?: RuntimeSignInIssue | null;
|
|
32
38
|
updatedAt: string;
|
|
33
39
|
}
|
|
34
40
|
export declare function defaultBotStatus(pid: number, runtime: BotRuntimeId, now?: () => Date): BotStatusFile;
|