@indigoai-us/hq-cli 5.122.0 → 5.122.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/commands/bot.d.ts +3 -0
- package/dist/commands/bot.js +14 -2
- package/dist/commands/cloud-provision.js +46 -0
- package/dist/lib/bot/api.d.ts +4 -1
- package/dist/lib/bot/api.js +20 -2
- package/dist/lib/bot/daemon.d.ts +11 -1
- package/dist/lib/bot/daemon.js +44 -2
- package/dist/lib/bot/run.d.ts +19 -1
- package/dist/lib/bot/run.js +51 -6
- package/dist/lib/bot/status.d.ts +5 -0
- package/dist/lib/doctor/checks/bots.d.ts +10 -0
- package/dist/lib/doctor/checks/bots.js +149 -0
- package/dist/lib/doctor/registry.js +4 -0
- package/dist/lib/mesh/live/daemon/install.d.ts +7 -0
- package/dist/lib/mesh/live/daemon/install.js +17 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,8 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.122.2] — 2026-09-19
|
|
6
|
+
|
|
5
7
|
### Fixed
|
|
6
8
|
|
|
9
|
+
- Local bots no longer retry inbox polls and heartbeats every couple of seconds
|
|
10
|
+
after HQ returns 429. They honour Retry-After when present and otherwise back
|
|
11
|
+
off, so a rate limit can clear instead of being held forever. `hq bot status`
|
|
12
|
+
now says unreachable / cannot receive when those calls are failing, instead of
|
|
13
|
+
HQ online from the last attempt, and `hq doctor` checks the same.
|
|
14
|
+
|
|
15
|
+
- `hq bot start` / `restart` rewrites a LaunchAgent that still points at the
|
|
16
|
+
old toolchain `node/bin/hq` path after an HQ update moved the CLI to
|
|
17
|
+
`npm-global/bin/hq`, which previously failed with MODULE_NOT_FOUND until
|
|
18
|
+
someone ran `hq bot daemon install` by hand.
|
|
19
|
+
|
|
20
|
+
## [5.122.1] — 2026-09-19
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
|
|
24
|
+
- `hq cloud provision company` no longer silently accepts a server-suffixed
|
|
25
|
+
slug. When another owner already holds the requested name, the vault assigns
|
|
26
|
+
`<slug>-<tag>` and used to write local config as if the original name had
|
|
27
|
+
been granted — sync then cannot push, because it looks for a folder matching
|
|
28
|
+
the cloud slug. Provision now fails before writing, and retires the
|
|
29
|
+
just-created entity so it does not leave a suffixed orphan.
|
|
30
|
+
|
|
7
31
|
- Work Mesh now keeps the company you selected when its session context is
|
|
8
32
|
acknowledged or recovered after a restart. Mesh activity stays attached to
|
|
9
33
|
the right company instead of being held for missing attribution.
|
package/dist/commands/bot.d.ts
CHANGED
|
@@ -24,6 +24,9 @@ export interface BotRow {
|
|
|
24
24
|
} | null;
|
|
25
25
|
online: boolean | null;
|
|
26
26
|
lastHeartbeatAt: string | null;
|
|
27
|
+
lastHeartbeatOk: boolean | null;
|
|
28
|
+
lastInboxPollAt: string | null;
|
|
29
|
+
lastInboxPollOk: boolean | null;
|
|
27
30
|
/** Present while the coding tool needs a new sign-in before the bot can work. */
|
|
28
31
|
runtimeSignIn: RuntimeSignInIssue | null;
|
|
29
32
|
daemonInstalled: boolean;
|
package/dist/commands/bot.js
CHANGED
|
@@ -36,7 +36,7 @@ import { peekHqApiKey } from "../utils/resolve-vault-credential.js";
|
|
|
36
36
|
import { getCompanyUid, resolveCallerPersonUid } from "../utils/vault-api.js";
|
|
37
37
|
import { resolveHqBinary } from "../lib/mesh/live/daemon/install.js";
|
|
38
38
|
import { runtimeDisplayName } from "../lib/bot/runtime-sign-in.js";
|
|
39
|
-
import { BotApi, BotApiError, BOT_RESTORE_DEFAULTS, buildBotLocalConfig, describeBotLocalConfig, orphanBotRunMessage, parseBotLocalConfig, removeOrphanBotDaemon, botDaemonStatus, botDir, botCredsPath, botLogPath, botsRoot, botTokenStateDir, botWorkerRelDir, buildBotDaemonPaths, createBotLogger, deleteBotCreds, installBotDaemon, introDmText, isBotMemoryMode, botLocalMemoryDir, botMemoryMode, resolveBotMemoryDir, validateBotIntro, validateBotKickoff, validateBotEffort, validateBotModel, effectiveBotEffort, BOT_EFFORT_LEVELS, DEFAULT_BOT_EFFORT, BOT_MEMORY_MODES, BOT_KINDS, isBotKind, effectiveBotKind, effectiveBotCompanies, normalizeBotCompanies, SETUP_BOT_WORKER_ID, isBotRuntimeId, localNameForRemoteBot, remoteBotRunnability, isPidAlive, listBotWorkerOptions, patchBotConfig, readBotConfig, readBotCredsIdentity, readBotStatus, resolveBotWorker, runBot, runtimeFor, scaffoldBotMemory, scaffoldBotWorker, startBotDaemon, stopBotDaemon, uninstallBotDaemon, validateBotName, writeBotConfig, writeBotCreds, BOT_RUNTIMES, BOT_INTRO_MAX_CHARS, BOT_KICKOFF_MAX_CHARS, } from "../lib/bot/index.js";
|
|
39
|
+
import { BotApi, BotApiError, BOT_RESTORE_DEFAULTS, buildBotLocalConfig, describeBotLocalConfig, orphanBotRunMessage, parseBotLocalConfig, removeOrphanBotDaemon, botDaemonStatus, botDir, botCredsPath, botLogPath, botsRoot, botTokenStateDir, botWorkerRelDir, buildBotDaemonPaths, createBotLogger, deleteBotCreds, installBotDaemon, repairBotDaemonIfStale, introDmText, isBotMemoryMode, botLocalMemoryDir, botMemoryMode, resolveBotMemoryDir, validateBotIntro, validateBotKickoff, validateBotEffort, validateBotModel, effectiveBotEffort, BOT_EFFORT_LEVELS, DEFAULT_BOT_EFFORT, BOT_MEMORY_MODES, BOT_KINDS, isBotKind, effectiveBotKind, effectiveBotCompanies, normalizeBotCompanies, SETUP_BOT_WORKER_ID, isBotRuntimeId, localNameForRemoteBot, remoteBotRunnability, isPidAlive, listBotWorkerOptions, patchBotConfig, readBotConfig, readBotCredsIdentity, readBotStatus, resolveBotWorker, runBot, runtimeFor, scaffoldBotMemory, scaffoldBotWorker, startBotDaemon, stopBotDaemon, uninstallBotDaemon, validateBotName, writeBotConfig, writeBotCreds, BOT_RUNTIMES, BOT_INTRO_MAX_CHARS, BOT_KICKOFF_MAX_CHARS, } from "../lib/bot/index.js";
|
|
40
40
|
function collectRepeated(value, previous) {
|
|
41
41
|
return [...previous, value];
|
|
42
42
|
}
|
|
@@ -144,6 +144,9 @@ function rowFor(name, opts = {}) {
|
|
|
144
144
|
promotionHold: promotionHoldSummary(dir, config.agentUid, config.ownerUid),
|
|
145
145
|
online: null,
|
|
146
146
|
lastHeartbeatAt: status?.lastHeartbeatAt ?? null,
|
|
147
|
+
lastHeartbeatOk: status?.lastHeartbeatOk ?? null,
|
|
148
|
+
lastInboxPollAt: status?.lastInboxPollAt ?? null,
|
|
149
|
+
lastInboxPollOk: status?.lastInboxPollOk ?? null,
|
|
147
150
|
runtimeSignIn: status?.runtimeSignIn ?? null,
|
|
148
151
|
daemonInstalled,
|
|
149
152
|
daemonLoaded,
|
|
@@ -184,6 +187,8 @@ async function decorateOnline(rows) {
|
|
|
184
187
|
}));
|
|
185
188
|
}
|
|
186
189
|
function humanOnline(row) {
|
|
190
|
+
if (row.lastHeartbeatOk === false)
|
|
191
|
+
return chalk.red("unreachable");
|
|
187
192
|
if (row.online === true)
|
|
188
193
|
return chalk.green("online");
|
|
189
194
|
if (row.online === false)
|
|
@@ -241,6 +246,7 @@ async function startBot(name, dir, config, output, extra = {}) {
|
|
|
241
246
|
const d = botDaemonStatus(paths, daemonDeps());
|
|
242
247
|
let how;
|
|
243
248
|
if (d.installed && process.platform === "darwin") {
|
|
249
|
+
repairBotDaemonIfStale(paths, daemonDeps());
|
|
244
250
|
const r = startBotDaemon(paths, daemonDeps());
|
|
245
251
|
how = r.message;
|
|
246
252
|
}
|
|
@@ -710,7 +716,13 @@ async function runBotStatus(nameArg, opts) {
|
|
|
710
716
|
console.log(` worker: ${row.workerId}${row.companySlug ? ` (company ${row.companySlug})` : ""} ${chalk.dim(row.workerDir)}`);
|
|
711
717
|
console.log(` memory: ${row.memory} ${chalk.dim(row.memoryDir)}`);
|
|
712
718
|
console.log(` process: ${row.processAlive ? chalk.green(`running (pid ${row.pid})`) : chalk.gray(row.state)}`);
|
|
713
|
-
console.log(` HQ online: ${humanOnline(row)}${row.lastHeartbeatAt ? chalk.dim(` last heartbeat ${row.lastHeartbeatAt}`) : ""}`);
|
|
719
|
+
console.log(` HQ online: ${humanOnline(row)}${row.lastHeartbeatAt ? chalk.dim(` last heartbeat ${row.lastHeartbeatAt}`) : ""}${row.lastHeartbeatOk === false && status?.lastHeartbeatError ? chalk.dim(` (${status.lastHeartbeatError})`) : ""}`);
|
|
720
|
+
if (status?.lastInboxPollOk === false) {
|
|
721
|
+
console.log(` inbox: ${chalk.red("cannot receive")}${status.lastInboxError ? chalk.dim(` ${status.lastInboxError}`) : ""}`);
|
|
722
|
+
}
|
|
723
|
+
else if (status?.lastInboxPollAt) {
|
|
724
|
+
console.log(` inbox: last poll ${status.lastInboxPollAt}`);
|
|
725
|
+
}
|
|
714
726
|
console.log(` launchd: ${row.daemonInstalled ? (row.daemonLoaded ? "installed, loaded" : "installed, not loaded") : "not installed"}`);
|
|
715
727
|
console.log(` creds: ${creds ? `present for ${creds.entityUid}` : chalk.red("missing")}`);
|
|
716
728
|
if (status) {
|
|
@@ -513,6 +513,52 @@ export async function provisionCompany(options) {
|
|
|
513
513
|
createdEntity = true;
|
|
514
514
|
log(`created vault entity uid=${entity.uid}`);
|
|
515
515
|
}
|
|
516
|
+
// hq-pro auto-suffixes a company slug when another owner already holds the
|
|
517
|
+
// bare name (`<slug>-<ownerTag>`). `/entity/check-slug/me` only looks at the
|
|
518
|
+
// caller's namespace, so it reports "available" and we POST; the server then
|
|
519
|
+
// returns a different slug. Writing companies/<requested>/.hq/config.json
|
|
520
|
+
// with the requested slug (or syncing with --company <assigned>) leaves the
|
|
521
|
+
// local folder and the cloud slug pointing at two different paths — share()
|
|
522
|
+
// derives syncRoot from ctx.slug, so the company can never be pushed.
|
|
523
|
+
// Fail before any local write. Retire a just-created entity so we do not
|
|
524
|
+
// leave a suffixed orphan.
|
|
525
|
+
if (entity.slug !== options.slug) {
|
|
526
|
+
const assigned = entity.slug || "(missing)";
|
|
527
|
+
let retired = false;
|
|
528
|
+
if (createdEntity) {
|
|
529
|
+
try {
|
|
530
|
+
await vaultClient.softDeleteCompany(entity.uid);
|
|
531
|
+
retired = true;
|
|
532
|
+
log(`retired vault entity uid=${entity.uid} — assigned slug ${assigned} differs from requested ${options.slug}`);
|
|
533
|
+
}
|
|
534
|
+
catch (err) {
|
|
535
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
536
|
+
log(`failed to retire mismatched vault entity uid=${entity.uid}: ${msg}`);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
const retireNote = retired
|
|
540
|
+
? `The just-created entity ${entity.uid} was retired.`
|
|
541
|
+
: createdEntity
|
|
542
|
+
? `Retire entity ${entity.uid} with \`hq cloud retire company\`, then retry after the original slug is free.`
|
|
543
|
+
: `Existing entity ${entity.uid} was not modified.`;
|
|
544
|
+
throw new ProvisionError(1, `Cloud assigned slug "${assigned}" instead of requested "${options.slug}". ` +
|
|
545
|
+
`Local folder is companies/${options.slug}; sync derives the company folder from the cloud slug, so this company cannot be pushed. ` +
|
|
546
|
+
`${retireNote} Ask an admin to release the original slug, or rename the local folder to "${assigned}" and re-run.`, {
|
|
547
|
+
ok: false,
|
|
548
|
+
company_slug: options.slug,
|
|
549
|
+
cloud_uid: entity.uid,
|
|
550
|
+
bucket_name: entity.bucketName ?? "",
|
|
551
|
+
vault_api_url: options.vaultApiUrl,
|
|
552
|
+
kms_key_id: entity.kmsKeyId ?? null,
|
|
553
|
+
created_entity: createdEntity,
|
|
554
|
+
manifest_patched: false,
|
|
555
|
+
config_written: false,
|
|
556
|
+
initial_sync: {
|
|
557
|
+
ok: false,
|
|
558
|
+
error: `assigned slug "${assigned}" differs from requested "${options.slug}"`,
|
|
559
|
+
},
|
|
560
|
+
});
|
|
561
|
+
}
|
|
516
562
|
if (!entity.bucketName) {
|
|
517
563
|
// Vault returned an entity without a bucket — this would happen if the
|
|
518
564
|
// provisioning Lambda asynchronously failed. We have a `cloud_uid` but
|
package/dist/lib/bot/api.d.ts
CHANGED
|
@@ -85,8 +85,11 @@ export declare class BotApiError extends Error {
|
|
|
85
85
|
readonly status: number;
|
|
86
86
|
readonly path: string;
|
|
87
87
|
readonly code?: string | undefined;
|
|
88
|
-
|
|
88
|
+
readonly retryAfterMs?: number | undefined;
|
|
89
|
+
constructor(status: number, path: string, code?: string | undefined, detail?: string, retryAfterMs?: number | undefined);
|
|
89
90
|
}
|
|
91
|
+
/** Parse a Retry-After header (delta-seconds or HTTP-date) to milliseconds. */
|
|
92
|
+
export declare function retryAfterHeaderMs(value: string | null | undefined, nowMs?: number): number | undefined;
|
|
90
93
|
export interface BotApiOptions {
|
|
91
94
|
token: TokenSupplier;
|
|
92
95
|
baseUrl?: string;
|
package/dist/lib/bot/api.js
CHANGED
|
@@ -11,14 +11,32 @@ export class BotApiError extends Error {
|
|
|
11
11
|
status;
|
|
12
12
|
path;
|
|
13
13
|
code;
|
|
14
|
-
|
|
14
|
+
retryAfterMs;
|
|
15
|
+
constructor(status, path, code, detail, retryAfterMs) {
|
|
15
16
|
super(`HQ API ${path} → ${status}${code ? ` ${code}` : ""}${detail ? `: ${detail}` : ""}`);
|
|
16
17
|
this.status = status;
|
|
17
18
|
this.path = path;
|
|
18
19
|
this.code = code;
|
|
20
|
+
this.retryAfterMs = retryAfterMs;
|
|
19
21
|
this.name = "BotApiError";
|
|
20
22
|
}
|
|
21
23
|
}
|
|
24
|
+
/** Parse a Retry-After header (delta-seconds or HTTP-date) to milliseconds. */
|
|
25
|
+
export function retryAfterHeaderMs(value, nowMs = Date.now()) {
|
|
26
|
+
if (!value)
|
|
27
|
+
return undefined;
|
|
28
|
+
const trimmed = value.trim();
|
|
29
|
+
if (!trimmed)
|
|
30
|
+
return undefined;
|
|
31
|
+
const seconds = Number(trimmed);
|
|
32
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
33
|
+
return Math.round(seconds * 1000);
|
|
34
|
+
}
|
|
35
|
+
const dateMs = Date.parse(trimmed);
|
|
36
|
+
if (!Number.isNaN(dateMs))
|
|
37
|
+
return Math.max(0, dateMs - nowMs);
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
22
40
|
async function readError(res) {
|
|
23
41
|
try {
|
|
24
42
|
const body = (await res.json());
|
|
@@ -54,7 +72,7 @@ export class BotApi {
|
|
|
54
72
|
});
|
|
55
73
|
if (!accept.includes(res.status)) {
|
|
56
74
|
const err = await readError(res);
|
|
57
|
-
throw new BotApiError(res.status, path, err.code, err.detail);
|
|
75
|
+
throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")));
|
|
58
76
|
}
|
|
59
77
|
const text = await res.text();
|
|
60
78
|
return { status: res.status, body: (text ? JSON.parse(text) : {}) };
|
package/dist/lib/bot/daemon.d.ts
CHANGED
|
@@ -81,7 +81,17 @@ export interface BotDaemonDeps {
|
|
|
81
81
|
*/
|
|
82
82
|
announce?: (result: BotDaemonResult) => void;
|
|
83
83
|
}
|
|
84
|
-
|
|
84
|
+
export declare function launchdPlistProgramArguments(plist: string): string[];
|
|
85
|
+
/** The hq script path baked into an installed unit, or null if none. */
|
|
86
|
+
export declare function installedDaemonHqBinary(p: BotDaemonPaths, io?: {
|
|
87
|
+
existsSync(path: string): boolean;
|
|
88
|
+
readFileSync(path: string, encoding: "utf8"): string;
|
|
89
|
+
}): string | null;
|
|
90
|
+
/**
|
|
91
|
+
* Rewrite the unit when its hq path is missing or is the pre-move
|
|
92
|
+
* toolchain/node/bin/hq while toolchain/npm-global/bin/hq exists.
|
|
93
|
+
*/
|
|
94
|
+
export declare function repairBotDaemonIfStale(p: BotDaemonPaths, deps?: BotDaemonDeps): BotDaemonResult | null;
|
|
85
95
|
export declare function installBotDaemon(p: BotDaemonPaths, deps?: BotDaemonDeps): BotDaemonResult;
|
|
86
96
|
export declare function uninstallBotDaemon(p: BotDaemonPaths, deps?: BotDaemonDeps): BotDaemonResult;
|
|
87
97
|
export declare function botDaemonStatus(p: BotDaemonPaths, deps?: BotDaemonDeps): BotDaemonResult;
|
package/dist/lib/bot/daemon.js
CHANGED
|
@@ -13,7 +13,7 @@ import * as fs from "node:fs";
|
|
|
13
13
|
import * as os from "node:os";
|
|
14
14
|
import * as path from "node:path";
|
|
15
15
|
import { spawn } from "node:child_process";
|
|
16
|
-
import { resolveHqBinary, resolveNodeBinary } from "../mesh/live/daemon/install.js";
|
|
16
|
+
import { resolveHqBinary, resolveNodeBinary, preferStableHqBinary } from "../mesh/live/daemon/install.js";
|
|
17
17
|
import { botLogPath, launchdLabel } from "./paths.js";
|
|
18
18
|
export function detectBotPlatform(platform = process.platform) {
|
|
19
19
|
if (platform === "darwin")
|
|
@@ -192,7 +192,49 @@ function launchdLoaded(p, deps) {
|
|
|
192
192
|
return false;
|
|
193
193
|
return deps.launchctl(["print", `${launchdDomain(deps.uid)}/${p.label}`]).ok;
|
|
194
194
|
}
|
|
195
|
-
|
|
195
|
+
export function launchdPlistProgramArguments(plist) {
|
|
196
|
+
const block = plist.match(/<key>ProgramArguments<\/key>\s*<array>([\s\S]*?)<\/array>/);
|
|
197
|
+
if (!block)
|
|
198
|
+
return [];
|
|
199
|
+
return [...block[1].matchAll(/<string>([^<]*)<\/string>/g)].map((m) => m[1]);
|
|
200
|
+
}
|
|
201
|
+
/** The hq script path baked into an installed unit, or null if none. */
|
|
202
|
+
export function installedDaemonHqBinary(p, io = fs) {
|
|
203
|
+
if (io.existsSync(p.plistPath)) {
|
|
204
|
+
const args = launchdPlistProgramArguments(io.readFileSync(p.plistPath, "utf8"));
|
|
205
|
+
return args[1] ?? null;
|
|
206
|
+
}
|
|
207
|
+
if (io.existsSync(p.systemdUnitPath)) {
|
|
208
|
+
const unit = io.readFileSync(p.systemdUnitPath, "utf8");
|
|
209
|
+
const m = unit.match(/^ExecStart=\S+\s+(\S+)/m);
|
|
210
|
+
return m?.[1] ?? null;
|
|
211
|
+
}
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Rewrite the unit when its hq path is missing or is the pre-move
|
|
216
|
+
* toolchain/node/bin/hq while toolchain/npm-global/bin/hq exists.
|
|
217
|
+
*/
|
|
218
|
+
export function repairBotDaemonIfStale(p, deps = {}) {
|
|
219
|
+
const io = deps.fsImpl ?? fs;
|
|
220
|
+
let current;
|
|
221
|
+
try {
|
|
222
|
+
current = installedDaemonHqBinary(p, {
|
|
223
|
+
existsSync: (q) => io.existsSync(q),
|
|
224
|
+
readFileSync: (q) => fs.readFileSync(q, "utf8"),
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
if (!current)
|
|
231
|
+
return null;
|
|
232
|
+
const missing = !io.existsSync(current);
|
|
233
|
+
const stale = preferStableHqBinary(current) !== current;
|
|
234
|
+
if (!missing && !stale)
|
|
235
|
+
return null;
|
|
236
|
+
return installBotDaemon(p, deps);
|
|
237
|
+
}
|
|
196
238
|
export function installBotDaemon(p, deps = {}) {
|
|
197
239
|
const platform = deps.platform ?? detectBotPlatform();
|
|
198
240
|
const io = deps.fsImpl ?? fs;
|
package/dist/lib/bot/run.d.ts
CHANGED
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
* Everything with a side effect is injectable so the loop is unit-testable.
|
|
39
39
|
*/
|
|
40
40
|
import { type OwnerContext } from "./owner-context.js";
|
|
41
|
-
import type
|
|
41
|
+
import { type BotApi, type InboxItem } from "./api.js";
|
|
42
42
|
import type { BotConfig, BotKind } from "./config.js";
|
|
43
43
|
import { type ProgressPosterOptions } from "./progress.js";
|
|
44
44
|
import { type BotLogger } from "./log.js";
|
|
@@ -46,6 +46,24 @@ import { type BotRuntime, type RuntimeTurnInput, type RuntimeTurnResult } from "
|
|
|
46
46
|
import { type BotState, type BotStatusFile } from "./status.js";
|
|
47
47
|
export declare const INBOX_POLL_INTERVAL_MS = 2000;
|
|
48
48
|
export declare const HEARTBEAT_INTERVAL_MS = 30000;
|
|
49
|
+
/** First 429/5xx wait when Retry-After is absent (inbox polls at 2s). */
|
|
50
|
+
export declare const API_BACKOFF_BASE_MS = 4000;
|
|
51
|
+
/** Ceiling for exponential poll/heartbeat backoff. */
|
|
52
|
+
export declare const API_BACKOFF_CAP_MS: number;
|
|
53
|
+
/** Honor Retry-After up to this; a bogus 30-day header must not park the bot. */
|
|
54
|
+
export declare const API_RETRY_AFTER_CAP_MS: number;
|
|
55
|
+
export declare const API_RETRY_AFTER_FLOOR_MS = 1000;
|
|
56
|
+
/**
|
|
57
|
+
* How long to wait after a failed inbox poll or heartbeat.
|
|
58
|
+
* 429/5xx (and transport errors) back off exponentially, honouring Retry-After
|
|
59
|
+
* when present. Other 4xx keep the normal interval so a 401 is not stretched.
|
|
60
|
+
*/
|
|
61
|
+
export declare function apiPollDelayMs(opts: {
|
|
62
|
+
err: unknown;
|
|
63
|
+
attempt: number;
|
|
64
|
+
intervalMs: number;
|
|
65
|
+
random?: () => number;
|
|
66
|
+
}): number;
|
|
49
67
|
export declare const FAILURE_WINDOW_MS: number;
|
|
50
68
|
export declare const FAILURE_LIMIT = 5;
|
|
51
69
|
export declare const MAX_ATTEMPTS_PER_MESSAGE = 3;
|
package/dist/lib/bot/run.js
CHANGED
|
@@ -44,6 +44,7 @@ import * as fs from "node:fs";
|
|
|
44
44
|
import * as path from "node:path";
|
|
45
45
|
import { fullJitterDelayMs } from "../mesh/live/backoff.js";
|
|
46
46
|
import { acquirePidLock, defaultPidLockDeps, releasePidLock } from "../mesh/live/daemon/pid-lock.js";
|
|
47
|
+
import { BotApiError } from "./api.js";
|
|
47
48
|
import { effectiveBotCompanies, effectiveBotKind, patchBotConfig, readBotConfig, SETUP_BOT_WORKER_ID } from "./config.js";
|
|
48
49
|
import { ensureBotSessionMeta } from "./company-bind.js";
|
|
49
50
|
import { readBotCredsIdentity } from "./creds.js";
|
|
@@ -62,6 +63,35 @@ import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
|
|
|
62
63
|
import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
|
|
63
64
|
export const INBOX_POLL_INTERVAL_MS = 2_000;
|
|
64
65
|
export const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
66
|
+
/** First 429/5xx wait when Retry-After is absent (inbox polls at 2s). */
|
|
67
|
+
export const API_BACKOFF_BASE_MS = 4_000;
|
|
68
|
+
/** Ceiling for exponential poll/heartbeat backoff. */
|
|
69
|
+
export const API_BACKOFF_CAP_MS = 5 * 60_000;
|
|
70
|
+
/** Honor Retry-After up to this; a bogus 30-day header must not park the bot. */
|
|
71
|
+
export const API_RETRY_AFTER_CAP_MS = 15 * 60_000;
|
|
72
|
+
export const API_RETRY_AFTER_FLOOR_MS = 1_000;
|
|
73
|
+
/**
|
|
74
|
+
* How long to wait after a failed inbox poll or heartbeat.
|
|
75
|
+
* 429/5xx (and transport errors) back off exponentially, honouring Retry-After
|
|
76
|
+
* when present. Other 4xx keep the normal interval so a 401 is not stretched.
|
|
77
|
+
*/
|
|
78
|
+
export function apiPollDelayMs(opts) {
|
|
79
|
+
const intervalMs = Math.max(0, opts.intervalMs);
|
|
80
|
+
const status = opts.err instanceof BotApiError ? opts.err.status : undefined;
|
|
81
|
+
const retryable = status === 429 || (typeof status === "number" && status >= 500) || !(opts.err instanceof BotApiError);
|
|
82
|
+
if (!retryable)
|
|
83
|
+
return intervalMs;
|
|
84
|
+
const retryAfter = opts.err instanceof BotApiError ? opts.err.retryAfterMs : undefined;
|
|
85
|
+
const header = typeof retryAfter === "number" && Number.isFinite(retryAfter)
|
|
86
|
+
? Math.min(API_RETRY_AFTER_CAP_MS, Math.max(API_RETRY_AFTER_FLOOR_MS, retryAfter))
|
|
87
|
+
: 0;
|
|
88
|
+
const base = Math.max(intervalMs * 2, API_BACKOFF_BASE_MS);
|
|
89
|
+
const attempt = Math.max(0, opts.attempt);
|
|
90
|
+
const exp = Math.min(API_BACKOFF_CAP_MS, base * 2 ** attempt);
|
|
91
|
+
const random = opts.random ?? Math.random;
|
|
92
|
+
const jittered = Math.floor(exp * (1 + 0.2 * random()));
|
|
93
|
+
return Math.min(API_BACKOFF_CAP_MS, Math.max(intervalMs, exp, header, jittered));
|
|
94
|
+
}
|
|
65
95
|
export const FAILURE_WINDOW_MS = 10 * 60 * 1000;
|
|
66
96
|
export const FAILURE_LIMIT = 5;
|
|
67
97
|
export const MAX_ATTEMPTS_PER_MESSAGE = 3;
|
|
@@ -187,6 +217,7 @@ export function preflightCreds(dir, config) {
|
|
|
187
217
|
export async function runBot(deps) {
|
|
188
218
|
const now = deps.now ?? (() => new Date());
|
|
189
219
|
const sleep = deps.sleep ?? defaultSleep;
|
|
220
|
+
const random = deps.random ?? Math.random;
|
|
190
221
|
const log = deps.log ?? createBotLogger(deps.dir, now);
|
|
191
222
|
const pid = deps.pid ?? process.pid;
|
|
192
223
|
const { dir, config, api, runtime } = deps;
|
|
@@ -374,18 +405,23 @@ export async function runBot(deps) {
|
|
|
374
405
|
}
|
|
375
406
|
};
|
|
376
407
|
const heartbeatLoop = (async () => {
|
|
408
|
+
let heartbeatFails = 0;
|
|
377
409
|
while (!stopping && state === "running") {
|
|
378
410
|
try {
|
|
379
411
|
// The server's component vocabulary is the fleet one (sync/model/task…);
|
|
380
412
|
// runtime → model, inbox → task, sync → sync.
|
|
381
413
|
await api.heartbeat(config.agentUid, { model: modelHealth, task: "ok", sync: "ok" });
|
|
382
|
-
|
|
414
|
+
heartbeatFails = 0;
|
|
415
|
+
setStatus({ lastHeartbeatAt: now().toISOString(), lastHeartbeatOk: true, lastHeartbeatError: undefined });
|
|
416
|
+
await sleep(heartbeatIntervalMs);
|
|
383
417
|
}
|
|
384
418
|
catch (err) {
|
|
385
|
-
|
|
386
|
-
|
|
419
|
+
heartbeatFails += 1;
|
|
420
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
421
|
+
setStatus({ lastHeartbeatOk: false, lastHeartbeatError: message.slice(0, 400) });
|
|
422
|
+
log("warn", `heartbeat failed: ${message}`);
|
|
423
|
+
await sleep(apiPollDelayMs({ err, attempt: heartbeatFails - 1, intervalMs: heartbeatIntervalMs, random }));
|
|
387
424
|
}
|
|
388
|
-
await sleep(heartbeatIntervalMs);
|
|
389
425
|
}
|
|
390
426
|
})();
|
|
391
427
|
// ── Inbox loop ─────────────────────────────────────────────────────────────
|
|
@@ -1000,6 +1036,7 @@ export async function runBot(deps) {
|
|
|
1000
1036
|
}
|
|
1001
1037
|
}
|
|
1002
1038
|
await kickoff;
|
|
1039
|
+
let inboxFails = 0;
|
|
1003
1040
|
while (!stopping && state === "running") {
|
|
1004
1041
|
if (hasPromotionHold(dir))
|
|
1005
1042
|
break;
|
|
@@ -1015,13 +1052,21 @@ export async function runBot(deps) {
|
|
|
1015
1052
|
}
|
|
1016
1053
|
try {
|
|
1017
1054
|
const items = await api.pullInbox(config.agentUid);
|
|
1018
|
-
|
|
1055
|
+
inboxFails = 0;
|
|
1056
|
+
setStatus({ lastInboxPollAt: now().toISOString(), lastInboxPollOk: true, lastInboxError: undefined });
|
|
1019
1057
|
for (const item of items)
|
|
1020
1058
|
enqueue(item);
|
|
1021
1059
|
pump();
|
|
1022
1060
|
}
|
|
1023
1061
|
catch (err) {
|
|
1024
|
-
|
|
1062
|
+
inboxFails += 1;
|
|
1063
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1064
|
+
setStatus({ lastInboxPollOk: false, lastInboxError: message.slice(0, 400) });
|
|
1065
|
+
log("warn", `inbox poll failed: ${message}`);
|
|
1066
|
+
if (stopping || state !== "running")
|
|
1067
|
+
break;
|
|
1068
|
+
await sleep(apiPollDelayMs({ err, attempt: inboxFails - 1, intervalMs: inboxIntervalMs, random }));
|
|
1069
|
+
continue;
|
|
1025
1070
|
}
|
|
1026
1071
|
if (stopping || state !== "running")
|
|
1027
1072
|
break;
|
package/dist/lib/bot/status.d.ts
CHANGED
|
@@ -17,7 +17,12 @@ export interface BotStatusFile {
|
|
|
17
17
|
recentFailures: number;
|
|
18
18
|
lastHeartbeatAt?: string;
|
|
19
19
|
lastHeartbeatOk?: boolean;
|
|
20
|
+
/** Last heartbeat error, when lastHeartbeatOk is false. */
|
|
21
|
+
lastHeartbeatError?: string;
|
|
20
22
|
lastInboxPollAt?: string;
|
|
23
|
+
lastInboxPollOk?: boolean;
|
|
24
|
+
/** Last inbox poll error, when lastInboxPollOk is false. */
|
|
25
|
+
lastInboxError?: string;
|
|
21
26
|
/** Bounded, redacted stderr excerpt of the last model-CLI failure. */
|
|
22
27
|
lastError?: string;
|
|
23
28
|
/** Inbound DMs answered since start. */
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { CheckContext, CheckFamily, CheckResult } from "../types.js";
|
|
2
|
+
export declare const BOTS_FAMILY_ID = "bots";
|
|
3
|
+
export declare const BOTS_FAMILY_TITLE = "Local bots";
|
|
4
|
+
export interface BotsDoctorDeps {
|
|
5
|
+
home: () => string;
|
|
6
|
+
root: (home?: string) => string;
|
|
7
|
+
}
|
|
8
|
+
export declare const botsFamily: CheckFamily;
|
|
9
|
+
export declare function checkBots(_context: CheckContext, deps?: BotsDoctorDeps): CheckResult[];
|
|
10
|
+
//# sourceMappingURL=bots.d.ts.map
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local-bot doctor family: a running bot that cannot poll its inbox or
|
|
3
|
+
* heartbeat still looked healthy (`hq bot status` said online, `hq doctor`
|
|
4
|
+
* had no bot check). Offline, from ~/.hq/bots and installed LaunchAgents.
|
|
5
|
+
*/
|
|
6
|
+
import * as fs from "node:fs";
|
|
7
|
+
import * as os from "node:os";
|
|
8
|
+
import * as path from "node:path";
|
|
9
|
+
import { botsRoot, botDir, launchdLabel } from "../../bot/paths.js";
|
|
10
|
+
import { readBotConfig } from "../../bot/config.js";
|
|
11
|
+
import { isPidAlive, readBotStatus } from "../../bot/status.js";
|
|
12
|
+
export const BOTS_FAMILY_ID = "bots";
|
|
13
|
+
export const BOTS_FAMILY_TITLE = "Local bots";
|
|
14
|
+
const DEFAULT_DEPS = {
|
|
15
|
+
home: () => os.homedir(),
|
|
16
|
+
root: (home) => botsRoot(home ?? os.homedir()),
|
|
17
|
+
};
|
|
18
|
+
export const botsFamily = {
|
|
19
|
+
id: BOTS_FAMILY_ID,
|
|
20
|
+
title: BOTS_FAMILY_TITLE,
|
|
21
|
+
run: (context) => Promise.resolve(checkBots(context)),
|
|
22
|
+
};
|
|
23
|
+
export function checkBots(_context, deps = DEFAULT_DEPS) {
|
|
24
|
+
try {
|
|
25
|
+
return runChecks(deps);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
return [
|
|
29
|
+
{
|
|
30
|
+
status: "UNKNOWN",
|
|
31
|
+
checkId: "bots.error",
|
|
32
|
+
message: `local bot checks could not run: ${error.message}`,
|
|
33
|
+
},
|
|
34
|
+
];
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function launchdPlistHqBinary(plist) {
|
|
38
|
+
const block = plist.match(/<key>ProgramArguments<\/key>\s*<array>([\s\S]*?)<\/array>/);
|
|
39
|
+
if (!block)
|
|
40
|
+
return null;
|
|
41
|
+
const args = [...block[1].matchAll(/<string>([^<]*)<\/string>/g)].map((m) => m[1]);
|
|
42
|
+
return args[1] ?? null;
|
|
43
|
+
}
|
|
44
|
+
function runChecks(deps) {
|
|
45
|
+
const home = deps.home();
|
|
46
|
+
const root = deps.root(home);
|
|
47
|
+
if (!fs.existsSync(root)) {
|
|
48
|
+
return [
|
|
49
|
+
{
|
|
50
|
+
status: "NA",
|
|
51
|
+
checkId: "bots.none",
|
|
52
|
+
message: "No local bots on this computer.",
|
|
53
|
+
},
|
|
54
|
+
];
|
|
55
|
+
}
|
|
56
|
+
const names = fs
|
|
57
|
+
.readdirSync(root, { withFileTypes: true })
|
|
58
|
+
.filter((d) => d.isDirectory() && readBotConfig(path.join(root, d.name)) !== null)
|
|
59
|
+
.map((d) => d.name)
|
|
60
|
+
.sort();
|
|
61
|
+
if (names.length === 0) {
|
|
62
|
+
return [
|
|
63
|
+
{
|
|
64
|
+
status: "NA",
|
|
65
|
+
checkId: "bots.none",
|
|
66
|
+
message: "No local bots on this computer.",
|
|
67
|
+
},
|
|
68
|
+
];
|
|
69
|
+
}
|
|
70
|
+
const results = [];
|
|
71
|
+
for (const name of names) {
|
|
72
|
+
const dir = botDir(name, root);
|
|
73
|
+
const config = readBotConfig(dir);
|
|
74
|
+
if (!config)
|
|
75
|
+
continue;
|
|
76
|
+
const status = readBotStatus(dir);
|
|
77
|
+
const alive = !!status && status.state === "running" && isPidAlive(status.pid);
|
|
78
|
+
if (alive && status.lastHeartbeatOk === false) {
|
|
79
|
+
results.push({
|
|
80
|
+
status: "FAIL",
|
|
81
|
+
checkId: `bots.${name}.heartbeat`,
|
|
82
|
+
target: dir,
|
|
83
|
+
message: `${name} is running but cannot heartbeat to HQ${status.lastHeartbeatError ? ` (${status.lastHeartbeatError})` : ""}.`,
|
|
84
|
+
remediation: `hq bot logs ${name}`,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
else if (alive) {
|
|
88
|
+
results.push({
|
|
89
|
+
status: "PASS",
|
|
90
|
+
checkId: `bots.${name}.heartbeat`,
|
|
91
|
+
target: dir,
|
|
92
|
+
message: `${name} last heartbeat succeeded.`,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
if (alive && status.lastInboxPollOk === false) {
|
|
96
|
+
results.push({
|
|
97
|
+
status: "FAIL",
|
|
98
|
+
checkId: `bots.${name}.inbox`,
|
|
99
|
+
target: dir,
|
|
100
|
+
message: `${name} is running but cannot receive messages${status.lastInboxError ? ` (${status.lastInboxError})` : ""}.`,
|
|
101
|
+
remediation: `hq bot logs ${name}`,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
else if (alive) {
|
|
105
|
+
results.push({
|
|
106
|
+
status: "PASS",
|
|
107
|
+
checkId: `bots.${name}.inbox`,
|
|
108
|
+
target: dir,
|
|
109
|
+
message: `${name} last inbox poll succeeded.`,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
const plistPath = path.join(home, "Library", "LaunchAgents", `${launchdLabel(name)}.plist`);
|
|
113
|
+
if (fs.existsSync(plistPath)) {
|
|
114
|
+
let hqPath;
|
|
115
|
+
try {
|
|
116
|
+
hqPath = launchdPlistHqBinary(fs.readFileSync(plistPath, "utf8"));
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
hqPath = null;
|
|
120
|
+
}
|
|
121
|
+
if (hqPath && !fs.existsSync(hqPath)) {
|
|
122
|
+
results.push({
|
|
123
|
+
status: "FAIL",
|
|
124
|
+
checkId: `bots.${name}.daemon-binary`,
|
|
125
|
+
target: hqPath,
|
|
126
|
+
message: `${name} startup agent points at a missing hq binary.`,
|
|
127
|
+
remediation: `hq bot daemon install ${name}`,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
else if (hqPath) {
|
|
131
|
+
results.push({
|
|
132
|
+
status: "PASS",
|
|
133
|
+
checkId: `bots.${name}.daemon-binary`,
|
|
134
|
+
target: hqPath,
|
|
135
|
+
message: `${name} startup agent hq binary is present.`,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (results.length === 0) {
|
|
141
|
+
results.push({
|
|
142
|
+
status: "NA",
|
|
143
|
+
checkId: "bots.idle",
|
|
144
|
+
message: `${names.length} local bot(s) are installed and not running.`,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
return results;
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=bots.js.map
|
|
@@ -22,6 +22,7 @@ import { runtimeHealthFamily } from "./checks/runtime-health.js";
|
|
|
22
22
|
import { integrationsFamily } from "./checks/integrations.js";
|
|
23
23
|
import { syncHealthFamily } from "./checks/sync-health.js";
|
|
24
24
|
import { workContextFamily } from "./checks/work-context.js";
|
|
25
|
+
import { botsFamily } from "./checks/bots.js";
|
|
25
26
|
import { fixtureCoverageFamily } from "./fixtures/discover.js";
|
|
26
27
|
import { checkClaudeWiring } from "./checks/claude-wiring.js";
|
|
27
28
|
/**
|
|
@@ -256,6 +257,9 @@ export function createDefaultRegistry() {
|
|
|
256
257
|
// Work context (US-007A): session state files, outbox depth/quarantine,
|
|
257
258
|
// and default-company mode. Local-only reads under ~/.hq/work-context.
|
|
258
259
|
registry.register(workContextFamily);
|
|
260
|
+
// Local bots: inbox/heartbeat reachability and a LaunchAgent whose hq
|
|
261
|
+
// path survived an HQ toolchain move. Local-only reads.
|
|
262
|
+
registry.register(botsFamily);
|
|
259
263
|
return registry;
|
|
260
264
|
}
|
|
261
265
|
//# sourceMappingURL=registry.js.map
|
|
@@ -34,6 +34,13 @@ export interface InstallResult {
|
|
|
34
34
|
message: string;
|
|
35
35
|
}
|
|
36
36
|
export declare function detectPlatform(platform?: NodeJS.Platform): DaemonInstallPlatform;
|
|
37
|
+
/**
|
|
38
|
+
* HQ updates moved the CLI from toolchain/node/bin/hq to
|
|
39
|
+
* toolchain/npm-global/bin/hq. A LaunchAgent that still names the old path
|
|
40
|
+
* dies with MODULE_NOT_FOUND on the next restart. Prefer the live npm-global
|
|
41
|
+
* binary when both exist.
|
|
42
|
+
*/
|
|
43
|
+
export declare function preferStableHqBinary(entry: string): string;
|
|
37
44
|
/** Resolve the hq binary the current shell uses (argv[1] or PATH). */
|
|
38
45
|
export declare function resolveHqBinary(argv?: string[], env?: NodeJS.ProcessEnv): string;
|
|
39
46
|
export declare function resolveNodeBinary(execPath?: string, env?: NodeJS.ProcessEnv): string;
|
|
@@ -21,11 +21,27 @@ export function detectPlatform(platform = process.platform) {
|
|
|
21
21
|
return "linux";
|
|
22
22
|
return "other";
|
|
23
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* HQ updates moved the CLI from toolchain/node/bin/hq to
|
|
26
|
+
* toolchain/npm-global/bin/hq. A LaunchAgent that still names the old path
|
|
27
|
+
* dies with MODULE_NOT_FOUND on the next restart. Prefer the live npm-global
|
|
28
|
+
* binary when both exist.
|
|
29
|
+
*/
|
|
30
|
+
export function preferStableHqBinary(entry) {
|
|
31
|
+
const stale = `${path.sep}toolchain${path.sep}node${path.sep}bin${path.sep}`;
|
|
32
|
+
const stable = `${path.sep}toolchain${path.sep}npm-global${path.sep}bin${path.sep}`;
|
|
33
|
+
if (!entry.includes(stale))
|
|
34
|
+
return entry;
|
|
35
|
+
const candidate = entry.replace(stale, stable);
|
|
36
|
+
if (candidate !== entry && fs.existsSync(candidate))
|
|
37
|
+
return path.resolve(candidate);
|
|
38
|
+
return entry;
|
|
39
|
+
}
|
|
24
40
|
/** Resolve the hq binary the current shell uses (argv[1] or PATH). */
|
|
25
41
|
export function resolveHqBinary(argv = process.argv, env = process.env) {
|
|
26
42
|
const entry = argv[1];
|
|
27
43
|
if (entry && fs.existsSync(entry)) {
|
|
28
|
-
return path.resolve(entry);
|
|
44
|
+
return preferStableHqBinary(path.resolve(entry));
|
|
29
45
|
}
|
|
30
46
|
const pathEnv = env.PATH ?? "";
|
|
31
47
|
for (const dir of pathEnv.split(path.delimiter)) {
|