@indigoai-us/hq-cli 5.170.0 → 5.171.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/commands/bot.js +11 -0
- package/dist/lib/bot/cli-update.d.ts +146 -0
- package/dist/lib/bot/cli-update.js +278 -0
- package/dist/lib/bot/run.d.ts +12 -0
- package/dist/lib/bot/run.js +42 -4
- package/dist/lib/bot/status.d.ts +2 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.171.0] — 2026-09-24
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Local bots (`hq bot run`) now restart onto an updated hq-cli install. A
|
|
10
|
+
running bot checks the installed package behind its launch path every
|
|
11
|
+
minute. When a complete, valid, different version is seen on two checks in a
|
|
12
|
+
row, it stops taking new inbox items, lets running turns finish (queued items
|
|
13
|
+
stay un-acked for the next process), logs
|
|
14
|
+
`hq-cli updated <old> -> <new>; restarting to load it`, and exits 75 so
|
|
15
|
+
launchd, systemd, or `hq daemon` starts it again. A bot with no supervisor
|
|
16
|
+
starts its own replacement first. The new process logs its version at
|
|
17
|
+
startup and whether the update converged. A restart that comes back on the
|
|
18
|
+
old version blocks that target for 6 hours, and update restarts are capped
|
|
19
|
+
at 4 per hour. `hq bot status` shows the hq-cli version a bot is running.
|
|
20
|
+
Set `HQ_BOT_NO_CLI_UPDATE_RESTART=1` to turn this off.
|
|
21
|
+
|
|
5
22
|
## [5.170.0] — 2026-09-24
|
|
6
23
|
|
|
7
24
|
### Fixed
|
package/dist/commands/bot.js
CHANGED
|
@@ -25,6 +25,7 @@ import { hasPromotionHold, promotionHoldSummary } from "../lib/bot/promotion-hol
|
|
|
25
25
|
* list (online flag), and rm. Secrets are never printed.
|
|
26
26
|
*/
|
|
27
27
|
import { promoteLocalBot } from "../lib/bot/promote.js";
|
|
28
|
+
import { CLI_UPDATE_DISABLE_ENV, createCliUpdateWatcher, handOffToUpdatedCli } from "../lib/bot/cli-update.js";
|
|
28
29
|
import { createVaultObjectDownloader } from "../lib/agent-kit/attachment-download.js";
|
|
29
30
|
import { addBotToCompanies, describeCompanyFailures, removeBotFromCompanies } from "./bot-companies.js";
|
|
30
31
|
import { registerBotContinuityCommand } from "./bot-continuity.js";
|
|
@@ -743,6 +744,14 @@ async function runBotResident(nameArg) {
|
|
|
743
744
|
// Chat attachments are fetched with the bot's own machine identity against
|
|
744
745
|
// the same API its inbox and posts use (BotApi has no baseUrl override either).
|
|
745
746
|
downloadAttachment: createVaultObjectDownloader({ getToken: token }),
|
|
747
|
+
// Reload onto an updated hq-cli: the watched path is the script this
|
|
748
|
+
// process was launched with, which is what its supervisor launches next.
|
|
749
|
+
...(process.env[CLI_UPDATE_DISABLE_ENV] === "1"
|
|
750
|
+
? {}
|
|
751
|
+
: { cliUpdate: createCliUpdateWatcher({ dir, entry: process.argv[1] ?? null }) }),
|
|
752
|
+
onCliUpdateRestart: (_update, log) => {
|
|
753
|
+
handOffToUpdatedCli(name, log, { spawnReplacement: () => spawnDetachedRun(name, dir) });
|
|
754
|
+
},
|
|
746
755
|
});
|
|
747
756
|
await handle.done;
|
|
748
757
|
void name;
|
|
@@ -796,6 +805,8 @@ async function runBotStatus(nameArg, opts) {
|
|
|
796
805
|
console.log(` creds: ${creds ? `present for ${creds.entityUid}` : chalk.red("missing")}`);
|
|
797
806
|
if (status) {
|
|
798
807
|
console.log(` started: ${status.startedAt} restarts: ${status.restarts} replies: ${status.repliesSent}`);
|
|
808
|
+
if (status.cliVersion)
|
|
809
|
+
console.log(` hq-cli: ${status.cliVersion}`);
|
|
799
810
|
if (status.runtimeSignIn?.state === "expired") {
|
|
800
811
|
const tool = runtimeDisplayName(status.runtimeSignIn.runtime);
|
|
801
812
|
console.log(` ${chalk.yellow(`${tool} needs to sign in again`)} — sign in from HQ, or run \`${status.runtimeSignIn.runtime} ${status.runtimeSignIn.runtime === "claude" ? "auth login" : "login"}\`, then \`hq bot restart ${name}\``);
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A resident bot notices that the hq-cli install it was started from has been
|
|
3
|
+
* updated on disk, and restarts itself onto the new code.
|
|
4
|
+
*
|
|
5
|
+
* `hq bot run <name>` loads the CLI once and runs for days. The install under
|
|
6
|
+
* it moves on (the attended self-updater, the `hq daemon` updater, a manual
|
|
7
|
+
* `npm i -g`), but nothing reloaded a running bot: on 2026-09-24 the bot
|
|
8
|
+
* dr-love was still running code from 2026-09-19 while its install had moved
|
|
9
|
+
* to 5.163.0, so a shipped attachment fix never reached it.
|
|
10
|
+
*
|
|
11
|
+
* The watcher re-resolves the entry script the process was launched with
|
|
12
|
+
* (process.argv[1], the same path launchd / systemd / `hq daemon` will launch
|
|
13
|
+
* again) to its package.json on a timer. Once a complete, valid, different
|
|
14
|
+
* version is seen on consecutive checks, the run loop stops taking new inbox
|
|
15
|
+
* items, lets the running turns finish, and exits with
|
|
16
|
+
* CLI_UPDATE_RESTART_EXIT_CODE so its supervisor starts it again on the new
|
|
17
|
+
* code (or, with no supervisor, starts its own replacement first).
|
|
18
|
+
*
|
|
19
|
+
* Convergence (policy hq-self-updater-must-verify-convergence-after-install):
|
|
20
|
+
* the restart is recorded in <botDir>/cli-update.json before the exit. The next
|
|
21
|
+
* process checks the version it actually loaded against that record: a match
|
|
22
|
+
* is logged as converged; a process that came back on the OLD version blocks
|
|
23
|
+
* that target for CLI_UPDATE_NOT_CONVERGED_COOLDOWN_MS instead of restarting
|
|
24
|
+
* again. Restarts are also capped per window, so an install that keeps
|
|
25
|
+
* flipping cannot turn into a restart loop.
|
|
26
|
+
*/
|
|
27
|
+
import type { BotLogger } from "./log.js";
|
|
28
|
+
/**
|
|
29
|
+
* Exit code for "restart me onto the updated CLI". Non-zero on purpose: every
|
|
30
|
+
* supervisor a bot runs under restarts only a failed exit — launchd
|
|
31
|
+
* KeepAlive{SuccessfulExit:false}, systemd Restart=on-failure, and the
|
|
32
|
+
* `hq daemon` "on-failure" bot units — while a clean exit 0 (a stopped or
|
|
33
|
+
* disabled bot) stays down. Same value as the daemon's RESTART_EXIT_CODE.
|
|
34
|
+
*/
|
|
35
|
+
export declare const CLI_UPDATE_RESTART_EXIT_CODE = 75;
|
|
36
|
+
/** How often the installed package.json is read. */
|
|
37
|
+
export declare const CLI_UPDATE_CHECK_INTERVAL_MS = 60000;
|
|
38
|
+
/** Consecutive checks that must agree on the new version before acting. */
|
|
39
|
+
export declare const CLI_UPDATE_CONFIRMATIONS = 2;
|
|
40
|
+
/** At most this many update restarts per window, whatever the versions. */
|
|
41
|
+
export declare const CLI_UPDATE_RESTART_LIMIT = 4;
|
|
42
|
+
export declare const CLI_UPDATE_RESTART_WINDOW_MS: number;
|
|
43
|
+
/** A restart that came back on the old version blocks that target this long. */
|
|
44
|
+
export declare const CLI_UPDATE_NOT_CONVERGED_COOLDOWN_MS: number;
|
|
45
|
+
/** Set to "1" to turn the watcher off (development runs). */
|
|
46
|
+
export declare const CLI_UPDATE_DISABLE_ENV = "HQ_BOT_NO_CLI_UPDATE_RESTART";
|
|
47
|
+
export declare const CLI_UPDATE_LEDGER_FILE = "cli-update.json";
|
|
48
|
+
export type InstalledCliRead = {
|
|
49
|
+
ok: true;
|
|
50
|
+
version: string;
|
|
51
|
+
packageRoot: string;
|
|
52
|
+
} | {
|
|
53
|
+
ok: false;
|
|
54
|
+
reason: string;
|
|
55
|
+
packageRoot?: string;
|
|
56
|
+
};
|
|
57
|
+
export interface InstalledCliFs {
|
|
58
|
+
realpathSync(p: string): string;
|
|
59
|
+
readFileSync(p: string, encoding: "utf8"): string;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The hq-cli version installed behind `entry`, resolved the way the next launch
|
|
63
|
+
* of the same command line will resolve it: follow the symlinks now (a pnpm or
|
|
64
|
+
* versioned layout moves the link, npm rewrites the directory in place), then
|
|
65
|
+
* walk up to the package.json that names this CLI. Anything short of a
|
|
66
|
+
* readable manifest with a valid semver version is not a version.
|
|
67
|
+
*/
|
|
68
|
+
export declare function readInstalledCli(entry: string, io?: InstalledCliFs): InstalledCliRead;
|
|
69
|
+
export interface CliUpdate {
|
|
70
|
+
from: string;
|
|
71
|
+
to: string;
|
|
72
|
+
packageRoot: string;
|
|
73
|
+
}
|
|
74
|
+
interface CliUpdateLedger {
|
|
75
|
+
v: 1;
|
|
76
|
+
/** Written just before an update restart; settled by the next process. */
|
|
77
|
+
pending?: {
|
|
78
|
+
from: string;
|
|
79
|
+
to: string;
|
|
80
|
+
at: string;
|
|
81
|
+
};
|
|
82
|
+
/** ISO times of recent update restarts (for the per-window cap). */
|
|
83
|
+
restarts: string[];
|
|
84
|
+
/** A target that did not converge, and when it may be tried again. */
|
|
85
|
+
blocked?: {
|
|
86
|
+
version: string;
|
|
87
|
+
until: string;
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
export declare function readCliUpdateLedger(dir: string): CliUpdateLedger;
|
|
91
|
+
/** What the run loop needs from the watcher (injectable in tests). */
|
|
92
|
+
export interface CliUpdateWatch {
|
|
93
|
+
/** The version of the code this process loaded. */
|
|
94
|
+
readonly runningVersion: string;
|
|
95
|
+
/** Settle the previous restart (converged or not) and say what is watched. */
|
|
96
|
+
startup(log: BotLogger): void;
|
|
97
|
+
/** A confirmed update to restart onto, or null. Cheap; rate-limits itself. */
|
|
98
|
+
poll(log: BotLogger): CliUpdate | null;
|
|
99
|
+
/** Record the restart about to happen, before the process exits. */
|
|
100
|
+
recordRestart(update: CliUpdate): void;
|
|
101
|
+
}
|
|
102
|
+
export interface CliUpdateWatcherOptions {
|
|
103
|
+
dir: string;
|
|
104
|
+
/** The script this process was launched with (process.argv[1]); null = do not watch. */
|
|
105
|
+
entry: string | null;
|
|
106
|
+
runningVersion?: string;
|
|
107
|
+
now?: () => number;
|
|
108
|
+
read?: (entry: string) => InstalledCliRead;
|
|
109
|
+
/** True while an installer is provably rewriting the package dir. */
|
|
110
|
+
rewriteInProgress?: (packageRoot: string) => boolean;
|
|
111
|
+
checkIntervalMs?: number;
|
|
112
|
+
confirmations?: number;
|
|
113
|
+
restartLimit?: number;
|
|
114
|
+
restartWindowMs?: number;
|
|
115
|
+
cooldownMs?: number;
|
|
116
|
+
}
|
|
117
|
+
export declare function createCliUpdateWatcher(opts: CliUpdateWatcherOptions): CliUpdateWatch;
|
|
118
|
+
/**
|
|
119
|
+
* Who starts this bot again after it exits.
|
|
120
|
+
*
|
|
121
|
+
* - "hq-daemon": `hq daemon` sets HQ_DAEMON_UNIT=bots:<name> on its children;
|
|
122
|
+
* the unit is "on-failure" (supervisor.ts).
|
|
123
|
+
* - "launchd": launchd sets XPC_SERVICE_NAME to the job label for every job it
|
|
124
|
+
* spawns, including LaunchAgents installed before this code existed.
|
|
125
|
+
* - "systemd": the unit `hq-bot-<name>.service` shows up in the process cgroup.
|
|
126
|
+
* - "none": `hq bot start` without a service (Windows, --no-daemon, a Linux
|
|
127
|
+
* unit never enabled) or a foreground run. Nobody restarts this process.
|
|
128
|
+
*/
|
|
129
|
+
export type BotSupervisor = "hq-daemon" | "launchd" | "systemd" | "none";
|
|
130
|
+
export declare function detectBotSupervisor(name: string, opts?: {
|
|
131
|
+
env?: NodeJS.ProcessEnv;
|
|
132
|
+
platform?: NodeJS.Platform;
|
|
133
|
+
readCgroup?: () => string | null;
|
|
134
|
+
}): BotSupervisor;
|
|
135
|
+
/**
|
|
136
|
+
* The last step of an update restart, run after the pid lock is released and
|
|
137
|
+
* just before the exit. A supervised bot only has to exit; an unsupervised one
|
|
138
|
+
* starts its replacement on the same entry script (now the new code) first,
|
|
139
|
+
* because nothing else will.
|
|
140
|
+
*/
|
|
141
|
+
export declare function handOffToUpdatedCli(name: string, log: BotLogger, deps: {
|
|
142
|
+
supervisor?: BotSupervisor;
|
|
143
|
+
spawnReplacement: () => number;
|
|
144
|
+
}): BotSupervisor;
|
|
145
|
+
export {};
|
|
146
|
+
//# sourceMappingURL=cli-update.d.ts.map
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A resident bot notices that the hq-cli install it was started from has been
|
|
3
|
+
* updated on disk, and restarts itself onto the new code.
|
|
4
|
+
*
|
|
5
|
+
* `hq bot run <name>` loads the CLI once and runs for days. The install under
|
|
6
|
+
* it moves on (the attended self-updater, the `hq daemon` updater, a manual
|
|
7
|
+
* `npm i -g`), but nothing reloaded a running bot: on 2026-09-24 the bot
|
|
8
|
+
* dr-love was still running code from 2026-09-19 while its install had moved
|
|
9
|
+
* to 5.163.0, so a shipped attachment fix never reached it.
|
|
10
|
+
*
|
|
11
|
+
* The watcher re-resolves the entry script the process was launched with
|
|
12
|
+
* (process.argv[1], the same path launchd / systemd / `hq daemon` will launch
|
|
13
|
+
* again) to its package.json on a timer. Once a complete, valid, different
|
|
14
|
+
* version is seen on consecutive checks, the run loop stops taking new inbox
|
|
15
|
+
* items, lets the running turns finish, and exits with
|
|
16
|
+
* CLI_UPDATE_RESTART_EXIT_CODE so its supervisor starts it again on the new
|
|
17
|
+
* code (or, with no supervisor, starts its own replacement first).
|
|
18
|
+
*
|
|
19
|
+
* Convergence (policy hq-self-updater-must-verify-convergence-after-install):
|
|
20
|
+
* the restart is recorded in <botDir>/cli-update.json before the exit. The next
|
|
21
|
+
* process checks the version it actually loaded against that record: a match
|
|
22
|
+
* is logged as converged; a process that came back on the OLD version blocks
|
|
23
|
+
* that target for CLI_UPDATE_NOT_CONVERGED_COOLDOWN_MS instead of restarting
|
|
24
|
+
* again. Restarts are also capped per window, so an install that keeps
|
|
25
|
+
* flipping cannot turn into a restart loop.
|
|
26
|
+
*/
|
|
27
|
+
import * as fs from "node:fs";
|
|
28
|
+
import * as path from "node:path";
|
|
29
|
+
import semver from "semver";
|
|
30
|
+
import { CLI_NAME, CLI_VERSION } from "../../cli-version.js";
|
|
31
|
+
import { installRewriteInProgress } from "../../utils/install-tree-torn.js";
|
|
32
|
+
import { updateLockPath } from "../../utils/update-lock.js";
|
|
33
|
+
import { launchdLabel } from "./paths.js";
|
|
34
|
+
/**
|
|
35
|
+
* Exit code for "restart me onto the updated CLI". Non-zero on purpose: every
|
|
36
|
+
* supervisor a bot runs under restarts only a failed exit — launchd
|
|
37
|
+
* KeepAlive{SuccessfulExit:false}, systemd Restart=on-failure, and the
|
|
38
|
+
* `hq daemon` "on-failure" bot units — while a clean exit 0 (a stopped or
|
|
39
|
+
* disabled bot) stays down. Same value as the daemon's RESTART_EXIT_CODE.
|
|
40
|
+
*/
|
|
41
|
+
export const CLI_UPDATE_RESTART_EXIT_CODE = 75;
|
|
42
|
+
/** How often the installed package.json is read. */
|
|
43
|
+
export const CLI_UPDATE_CHECK_INTERVAL_MS = 60_000;
|
|
44
|
+
/** Consecutive checks that must agree on the new version before acting. */
|
|
45
|
+
export const CLI_UPDATE_CONFIRMATIONS = 2;
|
|
46
|
+
/** At most this many update restarts per window, whatever the versions. */
|
|
47
|
+
export const CLI_UPDATE_RESTART_LIMIT = 4;
|
|
48
|
+
export const CLI_UPDATE_RESTART_WINDOW_MS = 60 * 60_000;
|
|
49
|
+
/** A restart that came back on the old version blocks that target this long. */
|
|
50
|
+
export const CLI_UPDATE_NOT_CONVERGED_COOLDOWN_MS = 6 * 60 * 60_000;
|
|
51
|
+
/** Set to "1" to turn the watcher off (development runs). */
|
|
52
|
+
export const CLI_UPDATE_DISABLE_ENV = "HQ_BOT_NO_CLI_UPDATE_RESTART";
|
|
53
|
+
export const CLI_UPDATE_LEDGER_FILE = "cli-update.json";
|
|
54
|
+
/**
|
|
55
|
+
* The hq-cli version installed behind `entry`, resolved the way the next launch
|
|
56
|
+
* of the same command line will resolve it: follow the symlinks now (a pnpm or
|
|
57
|
+
* versioned layout moves the link, npm rewrites the directory in place), then
|
|
58
|
+
* walk up to the package.json that names this CLI. Anything short of a
|
|
59
|
+
* readable manifest with a valid semver version is not a version.
|
|
60
|
+
*/
|
|
61
|
+
export function readInstalledCli(entry, io = fs) {
|
|
62
|
+
let real;
|
|
63
|
+
try {
|
|
64
|
+
real = io.realpathSync(entry);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return { ok: false, reason: `${entry} does not resolve` };
|
|
68
|
+
}
|
|
69
|
+
let dir = path.dirname(real);
|
|
70
|
+
for (let i = 0; i < 8; i++) {
|
|
71
|
+
const manifest = path.join(dir, "package.json");
|
|
72
|
+
let raw;
|
|
73
|
+
try {
|
|
74
|
+
raw = io.readFileSync(manifest, "utf8");
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
raw = null;
|
|
78
|
+
}
|
|
79
|
+
if (raw !== null) {
|
|
80
|
+
let parsed;
|
|
81
|
+
try {
|
|
82
|
+
parsed = JSON.parse(raw);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return { ok: false, reason: `${manifest} is not valid JSON`, packageRoot: dir };
|
|
86
|
+
}
|
|
87
|
+
if (parsed.name === CLI_NAME) {
|
|
88
|
+
const version = typeof parsed.version === "string" ? semver.valid(parsed.version) : null;
|
|
89
|
+
if (!version)
|
|
90
|
+
return { ok: false, reason: `${manifest} has no valid version`, packageRoot: dir };
|
|
91
|
+
return { ok: true, version, packageRoot: dir };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
const parent = path.dirname(dir);
|
|
95
|
+
if (parent === dir)
|
|
96
|
+
break;
|
|
97
|
+
dir = parent;
|
|
98
|
+
}
|
|
99
|
+
return { ok: false, reason: `no ${CLI_NAME} package.json above ${real}` };
|
|
100
|
+
}
|
|
101
|
+
function ledgerPath(dir) {
|
|
102
|
+
return path.join(dir, CLI_UPDATE_LEDGER_FILE);
|
|
103
|
+
}
|
|
104
|
+
export function readCliUpdateLedger(dir) {
|
|
105
|
+
try {
|
|
106
|
+
const raw = JSON.parse(fs.readFileSync(ledgerPath(dir), "utf8"));
|
|
107
|
+
return {
|
|
108
|
+
v: 1,
|
|
109
|
+
restarts: Array.isArray(raw.restarts) ? raw.restarts.filter((s) => typeof s === "string") : [],
|
|
110
|
+
...(raw.pending && typeof raw.pending.from === "string" && typeof raw.pending.to === "string" ? { pending: raw.pending } : {}),
|
|
111
|
+
...(raw.blocked && typeof raw.blocked.version === "string" && typeof raw.blocked.until === "string" ? { blocked: raw.blocked } : {}),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return { v: 1, restarts: [] };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function writeCliUpdateLedger(dir, ledger) {
|
|
119
|
+
const target = ledgerPath(dir);
|
|
120
|
+
const tmp = `${target}.${process.pid}.tmp`;
|
|
121
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
122
|
+
fs.writeFileSync(tmp, `${JSON.stringify(ledger, null, 2)}\n`, { mode: 0o600 });
|
|
123
|
+
fs.renameSync(tmp, target);
|
|
124
|
+
}
|
|
125
|
+
export function createCliUpdateWatcher(opts) {
|
|
126
|
+
const runningVersion = opts.runningVersion ?? CLI_VERSION;
|
|
127
|
+
const now = opts.now ?? Date.now;
|
|
128
|
+
const read = opts.read ?? ((entry) => readInstalledCli(entry));
|
|
129
|
+
const rewriteInProgress = opts.rewriteInProgress ?? ((packageRoot) => installRewriteInProgress({ packageRoot, lockPath: updateLockPath() }));
|
|
130
|
+
const checkIntervalMs = opts.checkIntervalMs ?? CLI_UPDATE_CHECK_INTERVAL_MS;
|
|
131
|
+
const confirmations = Math.max(1, opts.confirmations ?? CLI_UPDATE_CONFIRMATIONS);
|
|
132
|
+
const restartLimit = Math.max(1, opts.restartLimit ?? CLI_UPDATE_RESTART_LIMIT);
|
|
133
|
+
const restartWindowMs = opts.restartWindowMs ?? CLI_UPDATE_RESTART_WINDOW_MS;
|
|
134
|
+
const cooldownMs = opts.cooldownMs ?? CLI_UPDATE_NOT_CONVERGED_COOLDOWN_MS;
|
|
135
|
+
const { dir, entry } = opts;
|
|
136
|
+
let nextCheckAt = now() + checkIntervalMs;
|
|
137
|
+
let candidate = null;
|
|
138
|
+
let seen = 0;
|
|
139
|
+
/** Last "why not" line, so a condition that persists is logged once. */
|
|
140
|
+
let lastNote = null;
|
|
141
|
+
const note = (log, level, message) => {
|
|
142
|
+
if (message === lastNote)
|
|
143
|
+
return;
|
|
144
|
+
lastNote = message;
|
|
145
|
+
log(level, message);
|
|
146
|
+
};
|
|
147
|
+
const recentRestarts = (ledger, at) => ledger.restarts.filter((iso) => {
|
|
148
|
+
const t = Date.parse(iso);
|
|
149
|
+
return Number.isFinite(t) && at - t < restartWindowMs;
|
|
150
|
+
});
|
|
151
|
+
return {
|
|
152
|
+
runningVersion,
|
|
153
|
+
startup(log) {
|
|
154
|
+
if (!entry) {
|
|
155
|
+
log("info", "not watching for hq-cli updates (no entry script)");
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const ledger = readCliUpdateLedger(dir);
|
|
159
|
+
const pending = ledger.pending;
|
|
160
|
+
if (pending) {
|
|
161
|
+
delete ledger.pending;
|
|
162
|
+
if (runningVersion === pending.to) {
|
|
163
|
+
log("info", `hq-cli update converged: now running ${runningVersion} (was ${pending.from})`);
|
|
164
|
+
if (ledger.blocked?.version === runningVersion)
|
|
165
|
+
delete ledger.blocked;
|
|
166
|
+
}
|
|
167
|
+
else if (runningVersion === pending.from) {
|
|
168
|
+
const until = new Date(now() + cooldownMs).toISOString();
|
|
169
|
+
ledger.blocked = { version: pending.to, until };
|
|
170
|
+
log("error", `restarted to load hq-cli ${pending.to} but this process still loaded ${runningVersion}; ` +
|
|
171
|
+
`not restarting for ${pending.to} again until ${until} (check which install ${entry} resolves to)`);
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
log("info", `restarted for hq-cli ${pending.to}; now running ${runningVersion}`);
|
|
175
|
+
}
|
|
176
|
+
try {
|
|
177
|
+
writeCliUpdateLedger(dir, ledger);
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
log("warn", `could not update ${CLI_UPDATE_LEDGER_FILE}: ${err instanceof Error ? err.message : String(err)}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
log("info", `watching ${entry} for hq-cli updates (running ${runningVersion})`);
|
|
184
|
+
},
|
|
185
|
+
poll(log) {
|
|
186
|
+
if (!entry)
|
|
187
|
+
return null;
|
|
188
|
+
const at = now();
|
|
189
|
+
if (at < nextCheckAt)
|
|
190
|
+
return null;
|
|
191
|
+
nextCheckAt = at + checkIntervalMs;
|
|
192
|
+
const installed = read(entry);
|
|
193
|
+
if (!installed.ok) {
|
|
194
|
+
candidate = null;
|
|
195
|
+
seen = 0;
|
|
196
|
+
note(log, "warn", `hq-cli install not readable, not restarting: ${installed.reason}`);
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
if (installed.version === runningVersion) {
|
|
200
|
+
candidate = null;
|
|
201
|
+
seen = 0;
|
|
202
|
+
lastNote = null;
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
if (rewriteInProgress(installed.packageRoot)) {
|
|
206
|
+
candidate = null;
|
|
207
|
+
seen = 0;
|
|
208
|
+
note(log, "info", `hq-cli ${installed.version} is still being installed; waiting for it to finish`);
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
if (candidate === installed.version)
|
|
212
|
+
seen += 1;
|
|
213
|
+
else {
|
|
214
|
+
candidate = installed.version;
|
|
215
|
+
seen = 1;
|
|
216
|
+
}
|
|
217
|
+
if (seen < confirmations) {
|
|
218
|
+
note(log, "info", `hq-cli ${installed.version} is installed (running ${runningVersion}); confirming before restarting`);
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
const ledger = readCliUpdateLedger(dir);
|
|
222
|
+
if (ledger.blocked?.version === installed.version && at < Date.parse(ledger.blocked.until)) {
|
|
223
|
+
note(log, "warn", `not restarting for hq-cli ${installed.version}: an earlier restart for it did not load it (retry after ${ledger.blocked.until})`);
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
if (recentRestarts(ledger, at).length >= restartLimit) {
|
|
227
|
+
note(log, "warn", `not restarting for hq-cli ${installed.version}: already restarted ${restartLimit} times for updates in the last ${Math.round(restartWindowMs / 60_000)} minutes`);
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
return { from: runningVersion, to: installed.version, packageRoot: installed.packageRoot };
|
|
231
|
+
},
|
|
232
|
+
recordRestart(update) {
|
|
233
|
+
const at = now();
|
|
234
|
+
const ledger = readCliUpdateLedger(dir);
|
|
235
|
+
ledger.restarts = [...recentRestarts(ledger, at), new Date(at).toISOString()];
|
|
236
|
+
ledger.pending = { from: update.from, to: update.to, at: new Date(at).toISOString() };
|
|
237
|
+
writeCliUpdateLedger(dir, ledger);
|
|
238
|
+
},
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
export function detectBotSupervisor(name, opts = {}) {
|
|
242
|
+
const env = opts.env ?? process.env;
|
|
243
|
+
const platform = opts.platform ?? process.platform;
|
|
244
|
+
if (env.HQ_DAEMON_UNIT === `bots:${name}`)
|
|
245
|
+
return "hq-daemon";
|
|
246
|
+
if (platform === "darwin" && env.XPC_SERVICE_NAME === launchdLabel(name))
|
|
247
|
+
return "launchd";
|
|
248
|
+
if (platform === "linux") {
|
|
249
|
+
const readCgroup = opts.readCgroup ??
|
|
250
|
+
(() => {
|
|
251
|
+
try {
|
|
252
|
+
return fs.readFileSync("/proc/self/cgroup", "utf8");
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
});
|
|
258
|
+
if (readCgroup()?.includes(`/hq-bot-${name}.service`))
|
|
259
|
+
return "systemd";
|
|
260
|
+
}
|
|
261
|
+
return "none";
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* The last step of an update restart, run after the pid lock is released and
|
|
265
|
+
* just before the exit. A supervised bot only has to exit; an unsupervised one
|
|
266
|
+
* starts its replacement on the same entry script (now the new code) first,
|
|
267
|
+
* because nothing else will.
|
|
268
|
+
*/
|
|
269
|
+
export function handOffToUpdatedCli(name, log, deps) {
|
|
270
|
+
const supervisor = deps.supervisor ?? detectBotSupervisor(name);
|
|
271
|
+
if (supervisor !== "none") {
|
|
272
|
+
log("info", `exiting for ${supervisor} to start the updated hq-cli`);
|
|
273
|
+
return supervisor;
|
|
274
|
+
}
|
|
275
|
+
log("info", `no supervisor; started replacement pid ${deps.spawnReplacement()}`);
|
|
276
|
+
return supervisor;
|
|
277
|
+
}
|
|
278
|
+
//# sourceMappingURL=cli-update.js.map
|
package/dist/lib/bot/run.d.ts
CHANGED
|
@@ -53,6 +53,7 @@ import { type BotRuntime, type RuntimeTurnInput, type RuntimeTurnResult } from "
|
|
|
53
53
|
import type { VaultObjectDownloader } from "../agent-kit/attachment-download.js";
|
|
54
54
|
import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../heartbeat-identity.js";
|
|
55
55
|
import { type BotState, type BotStatusFile } from "./status.js";
|
|
56
|
+
import { type CliUpdate, type CliUpdateWatch } from "./cli-update.js";
|
|
56
57
|
export declare const INBOX_POLL_INTERVAL_MS = 2000;
|
|
57
58
|
export declare const HEARTBEAT_INTERVAL_MS = 30000;
|
|
58
59
|
/** First 429/5xx wait when Retry-After is absent (inbox polls at 2s). */
|
|
@@ -150,6 +151,17 @@ export interface BotRunDeps {
|
|
|
150
151
|
agentUidSource?: HeartbeatAgentUidSource;
|
|
151
152
|
credsSource?: HeartbeatCredsSource;
|
|
152
153
|
reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
|
|
154
|
+
/**
|
|
155
|
+
* Watches the hq-cli install this process was started from (cli-update.ts).
|
|
156
|
+
* When it reports an update, the bot stops taking inbox items, lets running
|
|
157
|
+
* turns finish, and exits CLI_UPDATE_RESTART_EXIT_CODE. Unset = no watch.
|
|
158
|
+
*/
|
|
159
|
+
cliUpdate?: CliUpdateWatch;
|
|
160
|
+
/**
|
|
161
|
+
* Called after the pid lock is released and just before the update-restart
|
|
162
|
+
* exit: the place to start a replacement when no supervisor will.
|
|
163
|
+
*/
|
|
164
|
+
onCliUpdateRestart?: (update: CliUpdate, log: BotLogger) => void;
|
|
153
165
|
}
|
|
154
166
|
export declare const KICKOFF_MESSAGE_ID = "kickoff";
|
|
155
167
|
/**
|
package/dist/lib/bot/run.js
CHANGED
|
@@ -74,6 +74,8 @@ import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
|
|
|
74
74
|
import { captureHeartbeatIdentityMismatch, identityMismatchMessage, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
|
|
75
75
|
import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
|
|
76
76
|
import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
|
|
77
|
+
import { CLI_VERSION } from "../../cli-version.js";
|
|
78
|
+
import { CLI_UPDATE_RESTART_EXIT_CODE } from "./cli-update.js";
|
|
77
79
|
export const INBOX_POLL_INTERVAL_MS = 2_000;
|
|
78
80
|
export const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
79
81
|
/** First 429/5xx wait when Retry-After is absent (inbox polls at 2s). */
|
|
@@ -285,7 +287,7 @@ export async function runBot(deps) {
|
|
|
285
287
|
pid,
|
|
286
288
|
...(deps.isProcessAlive ? { isProcessAlive: deps.isProcessAlive } : {}),
|
|
287
289
|
});
|
|
288
|
-
const finish = async (final, code, reason) => {
|
|
290
|
+
const finish = async (final, code, reason, beforeExit) => {
|
|
289
291
|
if (state === "stopped" || state === "failed")
|
|
290
292
|
return;
|
|
291
293
|
state = final;
|
|
@@ -293,9 +295,18 @@ export async function runBot(deps) {
|
|
|
293
295
|
log(final === "failed" ? "error" : "info", `bot ${final}: ${reason}`);
|
|
294
296
|
if (!deps.skipPidLock)
|
|
295
297
|
releasePidLock(dir, lockDeps);
|
|
298
|
+
try {
|
|
299
|
+
beforeExit?.();
|
|
300
|
+
}
|
|
301
|
+
catch (err) {
|
|
302
|
+
log("error", `before exit: ${err instanceof Error ? err.message : String(err)}`);
|
|
303
|
+
}
|
|
296
304
|
resolveDone(final);
|
|
297
305
|
exit(code);
|
|
298
306
|
};
|
|
307
|
+
const cliVersion = deps.cliUpdate?.runningVersion ?? CLI_VERSION;
|
|
308
|
+
/** Set once an update restart is due: no new turns, then exit for the supervisor. */
|
|
309
|
+
let cliUpdatePending = null;
|
|
299
310
|
// ── Preflight: creds, pid lock, owner verification ─────────────────────────
|
|
300
311
|
const credsProblem = preflightCreds(dir, config);
|
|
301
312
|
if (credsProblem) {
|
|
@@ -327,8 +338,9 @@ export async function runBot(deps) {
|
|
|
327
338
|
return { stop: async () => { }, done, status: () => null, idle: () => true };
|
|
328
339
|
}
|
|
329
340
|
// A restart is the owner's "try again" after signing in: start clean.
|
|
330
|
-
setStatus({ ...fallback(), pid, state: "starting", canDrainForPromotion: false, runtimeSignIn: null });
|
|
331
|
-
log("info", `starting ${config.name} (${config.runtime}, ${kind} bot${companies.length ? ` of ${companies.join(", ")}` : ""}) as ${config.agentUid} for ${config.ownerUid}`);
|
|
341
|
+
setStatus({ ...fallback(), pid, state: "starting", canDrainForPromotion: false, runtimeSignIn: null, cliVersion });
|
|
342
|
+
log("info", `starting ${config.name} on hq-cli ${cliVersion} (${config.runtime}, ${kind} bot${companies.length ? ` of ${companies.join(", ")}` : ""}) as ${config.agentUid} for ${config.ownerUid}`);
|
|
343
|
+
deps.cliUpdate?.startup(log);
|
|
332
344
|
try {
|
|
333
345
|
const record = await api.getAgent(config.agentUid);
|
|
334
346
|
// Promotion keeps the identity and owner. Ownership alone must not let
|
|
@@ -1476,6 +1488,17 @@ export async function runBot(deps) {
|
|
|
1476
1488
|
while (!stopping && state === "running") {
|
|
1477
1489
|
if (hasPromotionHold(dir))
|
|
1478
1490
|
break;
|
|
1491
|
+
// The install under this process was updated: take nothing new, let the
|
|
1492
|
+
// turns already running finish, then exit so the supervisor loads it.
|
|
1493
|
+
// Queued items were never started or acked; the next process gets them.
|
|
1494
|
+
const update = deps.cliUpdate?.poll(log) ?? null;
|
|
1495
|
+
if (update) {
|
|
1496
|
+
cliUpdatePending = update;
|
|
1497
|
+
stopping = true;
|
|
1498
|
+
const busy = running.size;
|
|
1499
|
+
log("info", `hq-cli updated ${update.from} -> ${update.to}; restarting to load it${busy ? ` after ${busy} running turn${busy === 1 ? "" : "s"} finish${busy === 1 ? "es" : ""}` : ""}`);
|
|
1500
|
+
break;
|
|
1501
|
+
}
|
|
1479
1502
|
// A kickoff held for a sign-in retries on the same timer as messages.
|
|
1480
1503
|
if (busyLanes.size === 0 &&
|
|
1481
1504
|
(!signInIssue || now().getTime() >= nextSignInRetryAt) &&
|
|
@@ -1525,6 +1548,9 @@ export async function runBot(deps) {
|
|
|
1525
1548
|
return readBotStatus(dir);
|
|
1526
1549
|
}
|
|
1527
1550
|
const stop = async (mode = "abort") => {
|
|
1551
|
+
// An owner stop or a promotion while an update restart drains wins: the
|
|
1552
|
+
// bot then exits 0 and stays down instead of being restarted.
|
|
1553
|
+
cliUpdatePending = null;
|
|
1528
1554
|
if (stopping) {
|
|
1529
1555
|
if (mode === "abort")
|
|
1530
1556
|
for (const turn of currentTurns)
|
|
@@ -1552,8 +1578,20 @@ export async function runBot(deps) {
|
|
|
1552
1578
|
void stop("drain"); });
|
|
1553
1579
|
setStatus({ canDrainForPromotion: true });
|
|
1554
1580
|
void Promise.allSettled([heartbeatLoop, inboxLoop]).then(() => {
|
|
1555
|
-
if (state
|
|
1581
|
+
if (state !== "running")
|
|
1582
|
+
return;
|
|
1583
|
+
const update = cliUpdatePending;
|
|
1584
|
+
if (!update) {
|
|
1556
1585
|
void finish("stopped", 0, "loops ended");
|
|
1586
|
+
return;
|
|
1587
|
+
}
|
|
1588
|
+
try {
|
|
1589
|
+
deps.cliUpdate?.recordRestart(update);
|
|
1590
|
+
}
|
|
1591
|
+
catch (err) {
|
|
1592
|
+
log("warn", `could not record the update restart: ${err instanceof Error ? err.message : String(err)}`);
|
|
1593
|
+
}
|
|
1594
|
+
void finish("stopped", CLI_UPDATE_RESTART_EXIT_CODE, `hq-cli updated ${update.from} -> ${update.to}; restarting to load it`, () => deps.onCliUpdateRestart?.(update, log));
|
|
1557
1595
|
});
|
|
1558
1596
|
return { stop, done, status: readStatus, idle: () => pending.size === 0 };
|
|
1559
1597
|
}
|
package/dist/lib/bot/status.d.ts
CHANGED
|
@@ -44,6 +44,8 @@ export interface BotStatusFile {
|
|
|
44
44
|
* reason); cleared by the next turn that works. `null` = known to work.
|
|
45
45
|
*/
|
|
46
46
|
runtimeSignIn?: RuntimeSignInIssue | null;
|
|
47
|
+
/** hq-cli version this process loaded (a bot keeps it until it restarts). */
|
|
48
|
+
cliVersion?: string;
|
|
47
49
|
updatedAt: string;
|
|
48
50
|
}
|
|
49
51
|
export declare function defaultBotStatus(pid: number, runtime: BotRuntimeId, now?: () => Date): BotStatusFile;
|