@indigoai-us/hq-cli 5.274.1 → 5.275.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -4
- package/assets/scaffold/core/scripts/checkpoint-stop-gate.sh +74 -18
- package/dist/command-catalog.generated.d.ts +34 -1
- package/dist/command-catalog.generated.js +43 -1
- package/dist/commands/cloud.js +22 -9
- package/dist/commands/company.js +66 -3
- package/dist/commands/core-checkpoint-gate-setting.d.ts +69 -0
- package/dist/commands/core-checkpoint-gate-setting.js +163 -0
- package/dist/commands/core.js +2 -0
- package/dist/commands/daemon.js +7 -2
- package/dist/commands/lanes.js +30 -9
- package/dist/lib/daemon/catalog.d.ts +7 -0
- package/dist/lib/daemon/catalog.js +6 -0
- package/dist/lib/daemon/flag.d.ts +23 -0
- package/dist/lib/daemon/flag.js +57 -0
- package/dist/lib/daemon/install.js +6 -1
- package/dist/lib/daemon/remote-control.d.ts +6 -0
- package/dist/lib/daemon/remote-control.js +12 -0
- package/dist/lib/daemon/run.js +115 -2
- package/dist/lib/daemon/succession.d.ts +116 -0
- package/dist/lib/daemon/succession.js +262 -0
- package/dist/lib/daemon/supervisor.d.ts +34 -1
- package/dist/lib/daemon/supervisor.js +190 -23
- package/dist/lib/lanes/host-capacity.js +7 -1
- package/dist/lib/lanes/models.d.ts +2 -0
- package/dist/lib/lanes/models.js +6 -1
- package/package.json +1 -1
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hq core checkpoint-gate-setting` — keeps the local cache that lets a
|
|
3
|
+
* company owner turn the checkpoint Stop gate off for their company (CO-58).
|
|
4
|
+
*
|
|
5
|
+
* The owner sets `companySettings.checkpointGateEnabled` on hq-pro
|
|
6
|
+
* (`hq company settings set --checkpoint-gate-enabled false`, or the console
|
|
7
|
+
* Settings page). The Stop gate script never calls the network: it reads
|
|
8
|
+
* `~/.hq/cache/checkpoint-gate/<slug>.json` and, when that is missing or older
|
|
9
|
+
* than ten minutes, starts this command detached with `--refresh`.
|
|
10
|
+
*
|
|
11
|
+
* `--refresh` reads the setting through the member-readable
|
|
12
|
+
* `GET /company-settings/checkpoint-gate` route and writes the cache only while
|
|
13
|
+
* the hq-flag `checkpoint.gate.company-setting` is on for the company. With the
|
|
14
|
+
* flag off it removes the cache, so the gate keeps today's behaviour. Any
|
|
15
|
+
* failure (no login, API unreachable, timeout) leaves the cache as it was: the
|
|
16
|
+
* gate then uses the last known value, or today's behaviour when there is none.
|
|
17
|
+
* Only the named company's setting is ever read.
|
|
18
|
+
*/
|
|
19
|
+
import * as fs from "node:fs";
|
|
20
|
+
import * as os from "node:os";
|
|
21
|
+
import * as path from "node:path";
|
|
22
|
+
import { ensureCognitoToken } from "../utils/cognito-session.js";
|
|
23
|
+
import { getEntityUid, vaultApiFetch } from "../utils/vault-api.js";
|
|
24
|
+
import { readLanesRegistryFlag } from "../lib/lanes/registry-flag.js";
|
|
25
|
+
import { describeCoreCommand } from "./core-command-catalog.js";
|
|
26
|
+
export const CHECKPOINT_GATE_SETTING_FLAG = "checkpoint.gate.company-setting";
|
|
27
|
+
/** The refresh gives up after this long; the gate keeps the cached value. */
|
|
28
|
+
export const CHECKPOINT_GATE_REFRESH_TIMEOUT_MS = 3_000;
|
|
29
|
+
/** Same character set the gate script accepts; anything else is refused. */
|
|
30
|
+
const SLUG_RE = /^[a-z0-9][a-z0-9-]*$/;
|
|
31
|
+
export function assertCompanySlug(slug) {
|
|
32
|
+
if (!SLUG_RE.test(slug) || slug === "personal") {
|
|
33
|
+
throw new Error(`checkpoint-gate-setting: '${slug}' is not a company slug`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export function checkpointGateCacheDir(home = os.homedir()) {
|
|
37
|
+
return path.join(home, ".hq", "cache", "checkpoint-gate");
|
|
38
|
+
}
|
|
39
|
+
export function checkpointGateCachePath(slug, home = os.homedir()) {
|
|
40
|
+
return path.join(checkpointGateCacheDir(home), `${slug}.json`);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* `readLanesRegistryFlag` resolves a failed registry read to `false`. The
|
|
44
|
+
* refresh must not treat that as "flag off" and delete a valid cache, so a
|
|
45
|
+
* reported failure becomes null (unknown).
|
|
46
|
+
*/
|
|
47
|
+
export async function readCheckpointGateSettingFlag(slug, companyUid) {
|
|
48
|
+
let failed = false;
|
|
49
|
+
const enabled = await readLanesRegistryFlag(CHECKPOINT_GATE_SETTING_FLAG, {
|
|
50
|
+
company: { slug, uid: companyUid },
|
|
51
|
+
reportFailure: () => {
|
|
52
|
+
failed = true;
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
return failed ? null : enabled;
|
|
56
|
+
}
|
|
57
|
+
async function defaultFetchSetting(token, companyUid, signal) {
|
|
58
|
+
const res = await vaultApiFetch({
|
|
59
|
+
token,
|
|
60
|
+
path: "/company-settings/checkpoint-gate",
|
|
61
|
+
query: { companyUid },
|
|
62
|
+
signal,
|
|
63
|
+
});
|
|
64
|
+
if (!res.ok)
|
|
65
|
+
throw new Error(`GET /company-settings/checkpoint-gate returned ${res.status}`);
|
|
66
|
+
const body = (await res.json());
|
|
67
|
+
if (typeof body.checkpointGateEnabled !== "boolean") {
|
|
68
|
+
throw new Error("GET /company-settings/checkpoint-gate returned no boolean checkpointGateEnabled");
|
|
69
|
+
}
|
|
70
|
+
return body.checkpointGateEnabled;
|
|
71
|
+
}
|
|
72
|
+
function writeCacheAtomic(file, entry) {
|
|
73
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
74
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
75
|
+
fs.writeFileSync(tmp, `${JSON.stringify(entry)}\n`, { mode: 0o600 });
|
|
76
|
+
fs.renameSync(tmp, file);
|
|
77
|
+
}
|
|
78
|
+
function withTimeout(work, ms) {
|
|
79
|
+
const controller = new AbortController();
|
|
80
|
+
let timer;
|
|
81
|
+
const timeout = new Promise((_, reject) => {
|
|
82
|
+
timer = setTimeout(() => {
|
|
83
|
+
controller.abort();
|
|
84
|
+
reject(new Error(`timed out after ${ms} ms`));
|
|
85
|
+
}, ms);
|
|
86
|
+
});
|
|
87
|
+
return Promise.race([work(controller.signal), timeout]).finally(() => clearTimeout(timer));
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Refresh one company's cache entry. Never throws for a network or auth
|
|
91
|
+
* failure: those return `kept` and leave the cache untouched.
|
|
92
|
+
*/
|
|
93
|
+
export async function refreshCheckpointGateSetting(slug, deps = {}) {
|
|
94
|
+
assertCompanySlug(slug);
|
|
95
|
+
const file = checkpointGateCachePath(slug, deps.home);
|
|
96
|
+
// Never interactive: a detached refresh must not open a browser login.
|
|
97
|
+
const getToken = deps.getToken ?? (() => ensureCognitoToken({ interactive: false }));
|
|
98
|
+
const resolveCompanyUid = deps.resolveCompanyUid ?? ((token, companySlug) => getEntityUid(token, { companySlug }));
|
|
99
|
+
const fetchSetting = deps.fetchSetting ?? defaultFetchSetting;
|
|
100
|
+
const readFlag = deps.readFlag ?? readCheckpointGateSettingFlag;
|
|
101
|
+
// One timeout covers every network step, the flag read included.
|
|
102
|
+
let fetched;
|
|
103
|
+
try {
|
|
104
|
+
fetched = await withTimeout(async (signal) => {
|
|
105
|
+
const token = await getToken();
|
|
106
|
+
const uid = await resolveCompanyUid(token, slug);
|
|
107
|
+
const enabled = await fetchSetting(token, uid, signal);
|
|
108
|
+
return { enabled, flag: await readFlag(slug, uid) };
|
|
109
|
+
}, deps.timeoutMs ?? CHECKPOINT_GATE_REFRESH_TIMEOUT_MS);
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
return { status: "kept", company: slug, reason: error instanceof Error ? error.message : String(error) };
|
|
113
|
+
}
|
|
114
|
+
if (fetched.flag === null) {
|
|
115
|
+
return { status: "kept", company: slug, reason: `could not read ${CHECKPOINT_GATE_SETTING_FLAG}` };
|
|
116
|
+
}
|
|
117
|
+
if (!fetched.flag) {
|
|
118
|
+
fs.rmSync(file, { force: true });
|
|
119
|
+
return { status: "removed", company: slug, reason: "flag_off" };
|
|
120
|
+
}
|
|
121
|
+
const now = (deps.now?.() ?? new Date()).getTime();
|
|
122
|
+
writeCacheAtomic(file, { company: slug, checkpointGateEnabled: fetched.enabled, fetched_at: Math.floor(now / 1000) });
|
|
123
|
+
return { status: "written", company: slug, checkpointGateEnabled: fetched.enabled };
|
|
124
|
+
}
|
|
125
|
+
export function describeRefreshOutcome(outcome) {
|
|
126
|
+
switch (outcome.status) {
|
|
127
|
+
case "written":
|
|
128
|
+
return `checkpoint gate for ${outcome.company}: ${outcome.checkpointGateEnabled ? "on" : "off"} (cached)`;
|
|
129
|
+
case "removed":
|
|
130
|
+
return `checkpoint gate for ${outcome.company}: company setting not in use (${CHECKPOINT_GATE_SETTING_FLAG} is off); cache cleared`;
|
|
131
|
+
case "kept":
|
|
132
|
+
return `checkpoint gate for ${outcome.company}: could not refresh (${outcome.reason}); cache unchanged`;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
export function registerCoreCheckpointGateSettingCommand(core) {
|
|
136
|
+
const command = core
|
|
137
|
+
.command("checkpoint-gate-setting")
|
|
138
|
+
.description("Refresh the local cache of a company's checkpoint gate setting (used by the Stop gate)")
|
|
139
|
+
.requiredOption("--company <slug>", "company whose setting to read")
|
|
140
|
+
.option("--refresh", "read the setting from HQ and update the local cache")
|
|
141
|
+
.option("--json", "print the result as JSON")
|
|
142
|
+
.action(async (options) => {
|
|
143
|
+
try {
|
|
144
|
+
assertCompanySlug(options.company);
|
|
145
|
+
if (!options.refresh) {
|
|
146
|
+
const file = checkpointGateCachePath(options.company);
|
|
147
|
+
const cached = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : null;
|
|
148
|
+
process.stdout.write(`${options.json ? JSON.stringify({ company: options.company, cached }) : cached ? `checkpoint gate for ${options.company}: ${cached.checkpointGateEnabled ? "on" : "off"} (cached at ${new Date(cached.fetched_at * 1000).toISOString()})` : `checkpoint gate for ${options.company}: no cached setting`}\n`);
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const outcome = await refreshCheckpointGateSetting(options.company);
|
|
152
|
+
process.stdout.write(`${options.json ? JSON.stringify(outcome) : describeRefreshOutcome(outcome)}\n`);
|
|
153
|
+
// A timed-out request can keep sockets open; the answer is already out.
|
|
154
|
+
process.exit(0);
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
158
|
+
process.exit(1);
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
describeCoreCommand(command, { execution: "native", root: "cwd" });
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=core-checkpoint-gate-setting.js.map
|
package/dist/commands/core.js
CHANGED
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
import { Option } from "commander";
|
|
32
32
|
import { registerCoreCheckpointCommand } from "./core-checkpoint.js";
|
|
33
33
|
import { registerCoreOntologyCommand } from "./core-ontology.js";
|
|
34
|
+
import { registerCoreCheckpointGateSettingCommand } from "./core-checkpoint-gate-setting.js";
|
|
34
35
|
import { resolveLiveRoot } from "../utils/hq-roots.js";
|
|
35
36
|
import { interpreterFor, runBundledScript } from "../utils/run-bundled-script.js";
|
|
36
37
|
import { renderIndexTarget } from "../lib/index-render/index.js";
|
|
@@ -286,6 +287,7 @@ export function registerCoreCommands(program) {
|
|
|
286
287
|
// hosts native TypeScript plumbing when a scaffold contract needs it.
|
|
287
288
|
registerCoreCheckpointCommand(core);
|
|
288
289
|
registerCoreOntologyCommand(core);
|
|
290
|
+
registerCoreCheckpointGateSettingCommand(core);
|
|
289
291
|
// These command names intentionally own native implementations. Their
|
|
290
292
|
// similarly named scaffold assets are forwarders, so sending them through
|
|
291
293
|
// runEntry would recurse back into this command forever.
|
package/dist/commands/daemon.js
CHANGED
|
@@ -489,8 +489,13 @@ function binaryFingerprint(bin) {
|
|
|
489
489
|
}
|
|
490
490
|
}
|
|
491
491
|
async function runTmuxService(d, opts) {
|
|
492
|
-
const { runRemoteControlHost } = await import("../lib/daemon/remote-control.js");
|
|
492
|
+
const { installFingerprint, runRemoteControlHost } = await import("../lib/daemon/remote-control.js");
|
|
493
493
|
const { findOnPath } = await import("../lib/daemon/run.js");
|
|
494
|
+
const { resolveRunningInstall } = await import("../utils/version-gate.js");
|
|
495
|
+
// The daemon keeps this host running across a CLI update, so a new hq-cli
|
|
496
|
+
// install also counts as an update and restarts the session once it is quiet.
|
|
497
|
+
const cliRoot = resolveRunningInstall().packageRoot;
|
|
498
|
+
const watched = cliRoot ? [opts.watch, path.join(cliRoot, "package.json")] : [opts.watch];
|
|
494
499
|
const hqRoot = d.env.HQ_ROOT?.trim();
|
|
495
500
|
if (!hqRoot)
|
|
496
501
|
throw new DaemonCommandError(`${opts.label} needs HQ_ROOT (the daemon sets it from the saved HQ folder)`);
|
|
@@ -508,7 +513,7 @@ async function runTmuxService(d, opts) {
|
|
|
508
513
|
cwd: hqRoot,
|
|
509
514
|
argv: opts.argv,
|
|
510
515
|
label: opts.label,
|
|
511
|
-
binaryFingerprint: () => binaryFingerprint
|
|
516
|
+
binaryFingerprint: () => installFingerprint(watched, binaryFingerprint),
|
|
512
517
|
now: () => Date.now(),
|
|
513
518
|
run: d.run,
|
|
514
519
|
pollMs: 5_000,
|
package/dist/commands/lanes.js
CHANGED
|
@@ -1158,7 +1158,7 @@ async function loadClaudeBalanceHostSnapshots(ctx, hqRoot, config) {
|
|
|
1158
1158
|
const actualLocalLanes = hostname
|
|
1159
1159
|
? localLanes.filter((lane) => lane.host.kind === "local" && lane.host.id === hostname)
|
|
1160
1160
|
: [];
|
|
1161
|
-
if (
|
|
1161
|
+
if (hostname) {
|
|
1162
1162
|
snapshots.push({
|
|
1163
1163
|
capacity: buildHostCapacitySnapshot({
|
|
1164
1164
|
host: actualLocalHost,
|
|
@@ -2402,6 +2402,11 @@ async function runCreate(ctx, opts) {
|
|
|
2402
2402
|
let placementHostOverride;
|
|
2403
2403
|
let autoPlacementChoice;
|
|
2404
2404
|
let autoPlacementSkippedHosts = [];
|
|
2405
|
+
let placementMachineIdentity;
|
|
2406
|
+
const resolvePlacementMachineIdentity = () => {
|
|
2407
|
+
placementMachineIdentity ??= ctx.resolveMachineIdentity();
|
|
2408
|
+
return placementMachineIdentity;
|
|
2409
|
+
};
|
|
2405
2410
|
let balanceHostLoad;
|
|
2406
2411
|
let balanceTargetHost;
|
|
2407
2412
|
let inheritedPlacement;
|
|
@@ -2485,23 +2490,39 @@ async function runCreate(ctx, opts) {
|
|
|
2485
2490
|
}
|
|
2486
2491
|
throw err;
|
|
2487
2492
|
}
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2493
|
+
const chosenHost = autoPlacementChoice.host;
|
|
2494
|
+
const localHostname = ctx.hostname().trim();
|
|
2495
|
+
if (!localHostname) {
|
|
2496
|
+
throw new LanesError("host_identity_unknown", "Cannot resolve this host for automatic lane placement", {});
|
|
2497
|
+
}
|
|
2498
|
+
const localHost = { kind: "local", id: localHostname };
|
|
2499
|
+
let chosenHostIsCaller = chosenHost.id === currentHost.id ||
|
|
2500
|
+
sanitizeHostComponent(chosenHost.id) === sanitizeHostComponent(localHostname);
|
|
2501
|
+
if (!chosenHostIsCaller && chosenHost.kind === "outpost") {
|
|
2502
|
+
const identity = resolvePlacementMachineIdentity();
|
|
2503
|
+
if (identity.status === "ok") {
|
|
2504
|
+
const machineIds = [identity.id, identity.imds, identity.hostname]
|
|
2505
|
+
.filter((id) => typeof id === "string" && id.trim() !== "");
|
|
2506
|
+
chosenHostIsCaller = machineIds.some((id) => sanitizeHostComponent(chosenHost.id) === sanitizeHostComponent(id.trim()));
|
|
2507
|
+
}
|
|
2492
2508
|
}
|
|
2493
|
-
|
|
2494
|
-
|
|
2509
|
+
if (chosenHostIsCaller) {
|
|
2510
|
+
// A capacity snapshot can identify this process as an Outpost (or an
|
|
2511
|
+
// alias of its machine). Auto placement must spawn locally so the
|
|
2512
|
+
// Outpost self-target guard is never reached.
|
|
2513
|
+
placementHostOverride = localHost;
|
|
2514
|
+
outpostId = undefined;
|
|
2495
2515
|
}
|
|
2496
2516
|
else {
|
|
2497
|
-
|
|
2517
|
+
placementHostOverride = chosenHost;
|
|
2518
|
+
outpostId = chosenHost.kind === "outpost" ? chosenHost.id : undefined;
|
|
2498
2519
|
}
|
|
2499
2520
|
}
|
|
2500
2521
|
}
|
|
2501
2522
|
let originHost;
|
|
2502
2523
|
let originHostSource;
|
|
2503
2524
|
if (outpostId) {
|
|
2504
|
-
const identity =
|
|
2525
|
+
const identity = resolvePlacementMachineIdentity();
|
|
2505
2526
|
if (identity.status !== "ok") {
|
|
2506
2527
|
throw new LanesError("origin_host_unresolvable", "Cannot record origin_host; this machine's identity is unresolvable", { identity });
|
|
2507
2528
|
}
|
|
@@ -54,6 +54,13 @@ export interface UnitPlan {
|
|
|
54
54
|
command: UnitCommand | null;
|
|
55
55
|
env: Record<string, string>;
|
|
56
56
|
blocked?: UnitBlock;
|
|
57
|
+
/**
|
|
58
|
+
* Signal that asks the unit to finish its current work and exit, used when a
|
|
59
|
+
* new daemon replaces it after a CLI update. Without one it gets SIGTERM.
|
|
60
|
+
*/
|
|
61
|
+
drainSignal?: "SIGUSR2";
|
|
62
|
+
/** A new CLI version alone does not restart this unit; it restarts itself when idle. */
|
|
63
|
+
keepAcrossCliUpdates?: true;
|
|
57
64
|
}
|
|
58
65
|
export interface PlanContext {
|
|
59
66
|
machineType: MachineType;
|
|
@@ -183,6 +183,8 @@ function planService(service, ctx) {
|
|
|
183
183
|
description: `runs the local bot ${name}`,
|
|
184
184
|
restart: "on-failure",
|
|
185
185
|
command: { type: "hq", args: ["bot", "run", name] },
|
|
186
|
+
// A bot drains on SIGUSR2: running turns finish, then it exits.
|
|
187
|
+
drainSignal: "SIGUSR2",
|
|
186
188
|
}));
|
|
187
189
|
}
|
|
188
190
|
case "session-host":
|
|
@@ -267,6 +269,8 @@ function planService(service, ctx) {
|
|
|
267
269
|
type: "hq",
|
|
268
270
|
args: ["daemon", "service", "codex-remote-control", "--script", script ?? "", "--codex", ctx.codexPath ?? ""],
|
|
269
271
|
},
|
|
272
|
+
// Restarting the host ends the tmux session people are using.
|
|
273
|
+
keepAcrossCliUpdates: true,
|
|
270
274
|
env: ctx.hqRoot ? { HQ_ROOT: ctx.hqRoot } : {},
|
|
271
275
|
blocked: firstBlock([
|
|
272
276
|
[!script, `this machine has no Codex remote-control runner (${CODEX_RUNNER_SCRIPT})`],
|
|
@@ -290,6 +294,8 @@ function planService(service, ctx) {
|
|
|
290
294
|
base(service, {
|
|
291
295
|
command: { type: "hq", args },
|
|
292
296
|
env: ctx.hqRoot ? { HQ_ROOT: ctx.hqRoot } : {},
|
|
297
|
+
// Restarting the host ends the tmux session people are using.
|
|
298
|
+
keepAcrossCliUpdates: true,
|
|
293
299
|
blocked: firstBlock([
|
|
294
300
|
[!ctx.claudePath, "the claude CLI is not installed"],
|
|
295
301
|
[!ctx.tmuxPath, "tmux is not installed (remote control runs Claude inside tmux)"],
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feature flags read by the daemon. The daemon runs for weeks, so each read
|
|
3
|
+
* asks the registry again instead of holding a per-process answer, and it
|
|
4
|
+
* reads as off unless the registry itself says on (or a local override does).
|
|
5
|
+
*/
|
|
6
|
+
import { type FlagClientOptions, type FlagResolution } from "@indigoai-us/hq-flags-client";
|
|
7
|
+
/** Outposts hand running services to the next daemon on a CLI update instead of restarting them all. */
|
|
8
|
+
export declare const SUPERVISOR_HANDOFF_FLAG = "daemon.supervisor-handoff";
|
|
9
|
+
export interface DaemonFlagClient {
|
|
10
|
+
ready(): Promise<void>;
|
|
11
|
+
explain(flagKey: string, options?: {
|
|
12
|
+
fallback?: boolean;
|
|
13
|
+
}): FlagResolution;
|
|
14
|
+
close(): void;
|
|
15
|
+
}
|
|
16
|
+
export interface ReadDaemonFlagOptions {
|
|
17
|
+
env?: NodeJS.ProcessEnv;
|
|
18
|
+
createClient?: (options: FlagClientOptions) => DaemonFlagClient;
|
|
19
|
+
getToken?: () => Promise<string>;
|
|
20
|
+
onError?: (error: unknown) => void;
|
|
21
|
+
}
|
|
22
|
+
export declare function readDaemonFlag(flagKey: string, options?: ReadDaemonFlagOptions): Promise<boolean>;
|
|
23
|
+
//# sourceMappingURL=flag.d.ts.map
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Feature flags read by the daemon. The daemon runs for weeks, so each read
|
|
3
|
+
* asks the registry again instead of holding a per-process answer, and it
|
|
4
|
+
* reads as off unless the registry itself says on (or a local override does).
|
|
5
|
+
*/
|
|
6
|
+
import { createFlagClient, readLocalOverride, } from "@indigoai-us/hq-flags-client";
|
|
7
|
+
import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
|
|
8
|
+
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
9
|
+
import { resolveProcessFlagCompanyContext } from "../flag-registry.js";
|
|
10
|
+
/** Outposts hand running services to the next daemon on a CLI update instead of restarting them all. */
|
|
11
|
+
export const SUPERVISOR_HANDOFF_FLAG = "daemon.supervisor-handoff";
|
|
12
|
+
const READ_TIMEOUT_MS = 5_000;
|
|
13
|
+
export async function readDaemonFlag(flagKey, options = {}) {
|
|
14
|
+
const env = options.env ?? process.env;
|
|
15
|
+
const override = readLocalOverride(flagKey, env);
|
|
16
|
+
if (override !== null)
|
|
17
|
+
return override;
|
|
18
|
+
let failed = false;
|
|
19
|
+
const onError = (error) => {
|
|
20
|
+
failed = true;
|
|
21
|
+
options.onError?.(error);
|
|
22
|
+
};
|
|
23
|
+
let client;
|
|
24
|
+
try {
|
|
25
|
+
const company = resolveProcessFlagCompanyContext(env);
|
|
26
|
+
// Signed out reads as off without reaching the registry.
|
|
27
|
+
const token = await (options.getToken ?? (() => ensureCognitoIdToken({ interactive: false })))();
|
|
28
|
+
if (typeof token !== "string" || token.trim() === "")
|
|
29
|
+
return false;
|
|
30
|
+
client = (options.createClient ?? createFlagClient)({
|
|
31
|
+
endpoint: resolveFlagRegistryEndpoint(undefined, env),
|
|
32
|
+
getToken: () => token,
|
|
33
|
+
env: {},
|
|
34
|
+
refreshIntervalMs: 0,
|
|
35
|
+
requestTimeoutMs: READ_TIMEOUT_MS,
|
|
36
|
+
...(company.companyUid ? { companyUid: company.companyUid } : {}),
|
|
37
|
+
...(company.companySlug ? { companyIdentifiers: [company.companySlug] } : {}),
|
|
38
|
+
onError,
|
|
39
|
+
});
|
|
40
|
+
await client.ready();
|
|
41
|
+
const resolution = client.explain(flagKey, { fallback: false });
|
|
42
|
+
return !failed && resolution.source === "snapshot" && resolution.value === true;
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
onError(error);
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
try {
|
|
50
|
+
client?.close();
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// closing a one-shot client cannot change the answer
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=flag.js.map
|
|
@@ -61,6 +61,11 @@ function serviceHost(host) {
|
|
|
61
61
|
* Output goes to the journal: an `append:` log file would be created by
|
|
62
62
|
* systemd as root inside the user's home, and the daemon (running as the user)
|
|
63
63
|
* writes its own logs under ~/.hq/daemon/logs.
|
|
64
|
+
*
|
|
65
|
+
* `KillMode=process`: systemd signals only the daemon. The daemon stops its
|
|
66
|
+
* services itself on SIGTERM, and after a CLI update it can leave them running
|
|
67
|
+
* for the next daemon to adopt (see succession.ts). A daemon that starts and
|
|
68
|
+
* finds services it did not adopt stops them first.
|
|
64
69
|
*/
|
|
65
70
|
function renderSystemdSystemUnit(host, user, userHome) {
|
|
66
71
|
const exec = [host.nodeBinary, host.hqBinary, "daemon", "run", "--managed"].map(shellQuote).join(" ");
|
|
@@ -87,7 +92,7 @@ ExecStart=${exec}
|
|
|
87
92
|
WorkingDirectory=${userHome}
|
|
88
93
|
Restart=always
|
|
89
94
|
RestartSec=10
|
|
90
|
-
KillMode=
|
|
95
|
+
KillMode=process
|
|
91
96
|
TimeoutStopSec=20
|
|
92
97
|
${envLines}
|
|
93
98
|
|
|
@@ -31,6 +31,12 @@ export declare function remoteControlArgs(opts: {
|
|
|
31
31
|
* (`--name outpost-1-2`), so taking the runner over keeps the name people see.
|
|
32
32
|
*/
|
|
33
33
|
export declare function runnerSessionName(script: string): string | null;
|
|
34
|
+
/**
|
|
35
|
+
* Fingerprint of what a host runs: the tool first, then other installs it
|
|
36
|
+
* depends on (the hq-cli package). Null while the tool cannot be read; an
|
|
37
|
+
* unreadable extra install is left out rather than hiding a tool update.
|
|
38
|
+
*/
|
|
39
|
+
export declare function installFingerprint(paths: readonly string[], fingerprint: (file: string) => string | null): string | null;
|
|
34
40
|
/** A tool updated on disk restarts its session once quiet this long... */
|
|
35
41
|
export declare const IDLE_BEFORE_RESTART_MS: number;
|
|
36
42
|
/** ...or after this long, however busy the session is. */
|
|
@@ -35,6 +35,18 @@ export function runnerSessionName(script) {
|
|
|
35
35
|
const m = /remote-control\b[^\n]*?--name[= ]+"?([^\s"]+)"?/.exec(script);
|
|
36
36
|
return m ? m[1] : null;
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* Fingerprint of what a host runs: the tool first, then other installs it
|
|
40
|
+
* depends on (the hq-cli package). Null while the tool cannot be read; an
|
|
41
|
+
* unreadable extra install is left out rather than hiding a tool update.
|
|
42
|
+
*/
|
|
43
|
+
export function installFingerprint(paths, fingerprint) {
|
|
44
|
+
const [tool, ...rest] = paths;
|
|
45
|
+
const toolFp = tool === undefined ? null : fingerprint(tool);
|
|
46
|
+
if (toolFp === null)
|
|
47
|
+
return null;
|
|
48
|
+
return [toolFp, ...rest.map(fingerprint).filter((fp) => fp !== null)].join("|");
|
|
49
|
+
}
|
|
38
50
|
/** A tool updated on disk restarts its session once quiet this long... */
|
|
39
51
|
export const IDLE_BEFORE_RESTART_MS = 10 * 60_000;
|
|
40
52
|
/** ...or after this long, however busy the session is. */
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -11,12 +11,14 @@ import { createRequire } from "node:module";
|
|
|
11
11
|
import * as os from "node:os";
|
|
12
12
|
import * as path from "node:path";
|
|
13
13
|
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
14
|
+
import { CLI_VERSION } from "../../cli-version.js";
|
|
14
15
|
import { isLocalDependencyInstall, resolveRunningInstall } from "../../utils/version-gate.js";
|
|
15
16
|
import { agentKitPaths } from "../agent-kit/paths.js";
|
|
16
17
|
import { readBotConfig } from "../bot/config.js";
|
|
17
18
|
import { botsRoot } from "../bot/paths.js";
|
|
18
19
|
import { CLAUDE_RUNNER_SCRIPT, CODEX_RUNNER_SCRIPT, enabledServices, planUnits } from "./catalog.js";
|
|
19
20
|
import { readDaemonConfig } from "./config.js";
|
|
21
|
+
import { SUPERVISOR_HANDOFF_FLAG, readDaemonFlag } from "./flag.js";
|
|
20
22
|
import { findLegacyUnits, legacyOwnerLookup } from "./legacy.js";
|
|
21
23
|
import { readLegacySyncEnv } from "./legacy-sync-env.js";
|
|
22
24
|
import { acquireDaemonLock, releaseDaemonLock } from "./lock.js";
|
|
@@ -26,12 +28,15 @@ import { appendPrivate, daemonPaths, ensurePrivateDir, openPrivateAppend } from
|
|
|
26
28
|
import { takeControlRequests } from "./requests.js";
|
|
27
29
|
import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
|
|
28
30
|
import { runnerSessionName } from "./remote-control.js";
|
|
29
|
-
import {
|
|
31
|
+
import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
|
|
32
|
+
import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
|
|
30
33
|
import { userToolDirs } from "./tools.js";
|
|
31
34
|
export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
|
|
32
35
|
/** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
|
|
33
36
|
export const RESTART_EXIT_CODE = 75;
|
|
34
37
|
const LEGACY_PROBE_TTL_MS = 60_000;
|
|
38
|
+
/** A flag read that has not answered by then counts as off. */
|
|
39
|
+
const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
|
|
35
40
|
export function resolveSyncRunnerPath() {
|
|
36
41
|
try {
|
|
37
42
|
const require = createRequire(import.meta.url);
|
|
@@ -246,6 +251,32 @@ export async function runDaemon(opts) {
|
|
|
246
251
|
}
|
|
247
252
|
const hostNote = opts.host ? `, hosted by the ${opts.host} app` : "";
|
|
248
253
|
log("info", `starting hq daemon (pid ${process.pid}) as ${machineType} (${detected.detail})${hostNote}`);
|
|
254
|
+
// On Linux, services can outlive a daemon (KillMode=process). Adopt the ones
|
|
255
|
+
// the previous daemon handed over and stop any others it left behind before
|
|
256
|
+
// starting anything, so a service never runs twice.
|
|
257
|
+
const succession = platform === "linux" && opts.host === undefined ? successionPaths(paths.root) : null;
|
|
258
|
+
const signalPid = (pid, sig) => {
|
|
259
|
+
process.kill(pid, sig);
|
|
260
|
+
};
|
|
261
|
+
let adopt = [];
|
|
262
|
+
if (succession) {
|
|
263
|
+
const taken = takeSuccession(succession, { now: Date.now, readStart: readProcStart, log });
|
|
264
|
+
await stopOrphans(taken.orphans, {
|
|
265
|
+
readStart: readProcStart,
|
|
266
|
+
signal: signalPid,
|
|
267
|
+
sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
268
|
+
log,
|
|
269
|
+
});
|
|
270
|
+
adopt = taken.adopt.map((u) => ({ id: u.id, key: u.key, child: adoptChild(u, { readStart: readProcStart, signal: signalPid }) }));
|
|
271
|
+
}
|
|
272
|
+
const tracker = succession ? createChildTracker(succession.childrenPath, { readStart: readProcStart, log }) : null;
|
|
273
|
+
for (const unit of adopt)
|
|
274
|
+
tracker?.track(unit.id, unit.child);
|
|
275
|
+
// Until now a crash leaves the previous records for the next daemon; from here the tracker owns them.
|
|
276
|
+
if (succession && tracker)
|
|
277
|
+
finishSuccession(succession, tracker.records(), log);
|
|
278
|
+
/** Service processes this daemon spawned, so a handoff can let go of them. */
|
|
279
|
+
const spawned = new Set();
|
|
249
280
|
if (!resolveSyncRunnerPath())
|
|
250
281
|
log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
|
|
251
282
|
const updatable = cliSelfUpdatable();
|
|
@@ -320,6 +351,11 @@ export async function runDaemon(opts) {
|
|
|
320
351
|
stdio: ["ignore", fd, fd],
|
|
321
352
|
windowsHide: true,
|
|
322
353
|
});
|
|
354
|
+
spawned.add(child);
|
|
355
|
+
child.once("exit", () => spawned.delete(child));
|
|
356
|
+
// Only long-running services are handed over; a scheduled run left behind finishes on its own.
|
|
357
|
+
if (unit.kind === "long-running")
|
|
358
|
+
tracker?.track(unit.id, child);
|
|
323
359
|
return child;
|
|
324
360
|
}
|
|
325
361
|
finally {
|
|
@@ -346,9 +382,36 @@ export async function runDaemon(opts) {
|
|
|
346
382
|
restartDaemon: () => restartDaemon(),
|
|
347
383
|
writeState: (state) => writeJsonAtomic(paths.statePath, state),
|
|
348
384
|
log,
|
|
385
|
+
cliVersion: CLI_VERSION,
|
|
386
|
+
adopt,
|
|
349
387
|
});
|
|
388
|
+
/** Hand services over on this restart: the unit leaves them running and the flag is on. */
|
|
389
|
+
const handoffWanted = async () => {
|
|
390
|
+
const killMode = platform === "linux"
|
|
391
|
+
? systemdKillMode((file, args) => {
|
|
392
|
+
const r = spawnSync(file, args, { encoding: "utf8" });
|
|
393
|
+
return { status: r.status, stdout: r.stdout ?? "" };
|
|
394
|
+
})
|
|
395
|
+
: null;
|
|
396
|
+
if (!handoffEligible({ platform, managed: opts.managed === true, host: opts.host, killMode }))
|
|
397
|
+
return false;
|
|
398
|
+
let timer;
|
|
399
|
+
const timeout = new Promise((r) => {
|
|
400
|
+
timer = setTimeout(() => r(false), HANDOFF_FLAG_TIMEOUT_MS);
|
|
401
|
+
});
|
|
402
|
+
const on = await Promise.race([
|
|
403
|
+
readDaemonFlag(SUPERVISOR_HANDOFF_FLAG, {
|
|
404
|
+
env,
|
|
405
|
+
onError: (err) => log("warn", `could not read ${SUPERVISOR_HANDOFF_FLAG}: ${err.message ?? err}`),
|
|
406
|
+
}),
|
|
407
|
+
timeout,
|
|
408
|
+
]);
|
|
409
|
+
clearTimeout(timer);
|
|
410
|
+
return on;
|
|
411
|
+
};
|
|
350
412
|
return await new Promise((resolve) => {
|
|
351
413
|
let stopping = false;
|
|
414
|
+
let restartPending = false;
|
|
352
415
|
const shutdown = (why, exitCode = 0) => {
|
|
353
416
|
if (stopping)
|
|
354
417
|
return;
|
|
@@ -370,12 +433,62 @@ export async function runDaemon(opts) {
|
|
|
370
433
|
resolve(exitCode);
|
|
371
434
|
});
|
|
372
435
|
};
|
|
436
|
+
/** Exit for a restart but leave the services running for the next daemon to adopt. */
|
|
437
|
+
const handOver = () => {
|
|
438
|
+
if (stopping)
|
|
439
|
+
return;
|
|
440
|
+
stopping = true;
|
|
441
|
+
const released = supervisor.release();
|
|
442
|
+
const starts = new Map((tracker?.records() ?? []).map((r) => [r.pid, r.start]));
|
|
443
|
+
const units = released.flatMap((u) => {
|
|
444
|
+
const start = starts.get(u.pid);
|
|
445
|
+
if (start !== undefined)
|
|
446
|
+
return [{ ...u, start }];
|
|
447
|
+
// Unverifiable later, so it cannot be adopted; stop it rather than leave a second copy.
|
|
448
|
+
log("warn", `stopping ${u.id}: its process could not be recorded for the next daemon`);
|
|
449
|
+
try {
|
|
450
|
+
process.kill(u.pid, "SIGTERM");
|
|
451
|
+
}
|
|
452
|
+
catch {
|
|
453
|
+
// already gone
|
|
454
|
+
}
|
|
455
|
+
return [];
|
|
456
|
+
});
|
|
457
|
+
try {
|
|
458
|
+
writeHandoff(succession.handoffPath, { releasedAt: Date.now(), units });
|
|
459
|
+
log("info", `restarting onto the updated CLI; handing ${units.length} running service(s) to the next daemon`);
|
|
460
|
+
}
|
|
461
|
+
catch (err) {
|
|
462
|
+
// The children record still lists them, so the next daemon stops and restarts them.
|
|
463
|
+
log("error", `could not write the handoff; the next daemon will restart the services: ${err.message}`);
|
|
464
|
+
}
|
|
465
|
+
// The released processes must not keep this one alive.
|
|
466
|
+
for (const child of spawned)
|
|
467
|
+
child.unref();
|
|
468
|
+
releaseDaemonLock(paths.lockPath);
|
|
469
|
+
resolve(RESTART_EXIT_CODE);
|
|
470
|
+
// Anything else still holding the event loop open must not delay the restart.
|
|
471
|
+
setTimeout(() => process.exit(RESTART_EXIT_CODE), 5_000).unref();
|
|
472
|
+
};
|
|
373
473
|
restartDaemon = () => {
|
|
374
474
|
if (!restartsForParent(opts)) {
|
|
375
475
|
log("warn", "a restart was requested, but no service manager would start this daemon again; restart it yourself");
|
|
376
476
|
return;
|
|
377
477
|
}
|
|
378
|
-
|
|
478
|
+
if (stopping || restartPending)
|
|
479
|
+
return;
|
|
480
|
+
restartPending = true;
|
|
481
|
+
void handoffWanted()
|
|
482
|
+
.catch((err) => {
|
|
483
|
+
log("warn", `could not decide on a handoff; restarting every service: ${err.message}`);
|
|
484
|
+
return false;
|
|
485
|
+
})
|
|
486
|
+
.then((handoff) => {
|
|
487
|
+
if (handoff && succession)
|
|
488
|
+
handOver();
|
|
489
|
+
else
|
|
490
|
+
shutdown("restarting onto the updated CLI", RESTART_EXIT_CODE);
|
|
491
|
+
});
|
|
379
492
|
};
|
|
380
493
|
process.on("SIGTERM", () => shutdown("received SIGTERM"));
|
|
381
494
|
process.on("SIGINT", () => shutdown("received SIGINT"));
|