@indigoai-us/hq-cli 5.274.0 → 5.275.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/assets/scaffold/core/scripts/checkpoint-stop-gate.sh +74 -18
  3. package/dist/command-catalog.generated.d.ts +34 -1
  4. package/dist/command-catalog.generated.js +43 -1
  5. package/dist/commands/cloud.js +22 -9
  6. package/dist/commands/company.js +66 -3
  7. package/dist/commands/core-checkpoint-gate-setting.d.ts +69 -0
  8. package/dist/commands/core-checkpoint-gate-setting.js +163 -0
  9. package/dist/commands/core.js +2 -0
  10. package/dist/commands/daemon.js +7 -2
  11. package/dist/commands/lanes-message.js +40 -7
  12. package/dist/commands/lanes-shared.d.ts +4 -0
  13. package/dist/commands/lanes-shared.js +5 -0
  14. package/dist/commands/lanes.js +41 -17
  15. package/dist/lib/daemon/catalog.d.ts +7 -0
  16. package/dist/lib/daemon/catalog.js +6 -0
  17. package/dist/lib/daemon/flag.d.ts +23 -0
  18. package/dist/lib/daemon/flag.js +57 -0
  19. package/dist/lib/daemon/install.js +6 -1
  20. package/dist/lib/daemon/remote-control.d.ts +6 -0
  21. package/dist/lib/daemon/remote-control.js +12 -0
  22. package/dist/lib/daemon/run.js +115 -2
  23. package/dist/lib/daemon/succession.d.ts +116 -0
  24. package/dist/lib/daemon/succession.js +262 -0
  25. package/dist/lib/daemon/supervisor.d.ts +34 -1
  26. package/dist/lib/daemon/supervisor.js +190 -23
  27. package/dist/lib/lanes/hire-grant-dispatch.d.ts +2 -1
  28. package/dist/lib/lanes/hire-grant-dispatch.js +37 -2
  29. package/dist/lib/lanes/host-capacity.js +7 -1
  30. package/dist/lib/lanes/keep-alive.js +2 -2
  31. package/dist/lib/lanes/models.d.ts +2 -0
  32. package/dist/lib/lanes/models.js +6 -1
  33. package/package.json +1 -1
@@ -11,12 +11,14 @@ import { createRequire } from "node:module";
11
11
  import * as os from "node:os";
12
12
  import * as path from "node:path";
13
13
  import { loadCachedTokens } from "@indigoai-us/hq-cloud";
14
+ import { CLI_VERSION } from "../../cli-version.js";
14
15
  import { isLocalDependencyInstall, resolveRunningInstall } from "../../utils/version-gate.js";
15
16
  import { agentKitPaths } from "../agent-kit/paths.js";
16
17
  import { readBotConfig } from "../bot/config.js";
17
18
  import { botsRoot } from "../bot/paths.js";
18
19
  import { CLAUDE_RUNNER_SCRIPT, CODEX_RUNNER_SCRIPT, enabledServices, planUnits } from "./catalog.js";
19
20
  import { readDaemonConfig } from "./config.js";
21
+ import { SUPERVISOR_HANDOFF_FLAG, readDaemonFlag } from "./flag.js";
20
22
  import { findLegacyUnits, legacyOwnerLookup } from "./legacy.js";
21
23
  import { readLegacySyncEnv } from "./legacy-sync-env.js";
22
24
  import { acquireDaemonLock, releaseDaemonLock } from "./lock.js";
@@ -26,12 +28,15 @@ import { appendPrivate, daemonPaths, ensurePrivateDir, openPrivateAppend } from
26
28
  import { takeControlRequests } from "./requests.js";
27
29
  import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
28
30
  import { runnerSessionName } from "./remote-control.js";
29
- import { STOP_ESCALATION_MS, createSupervisor } from "./supervisor.js";
31
+ import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
32
+ import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
30
33
  import { userToolDirs } from "./tools.js";
31
34
  export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
32
35
  /** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
33
36
  export const RESTART_EXIT_CODE = 75;
34
37
  const LEGACY_PROBE_TTL_MS = 60_000;
38
+ /** A flag read that has not answered by then counts as off. */
39
+ const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
35
40
  export function resolveSyncRunnerPath() {
36
41
  try {
37
42
  const require = createRequire(import.meta.url);
@@ -246,6 +251,32 @@ export async function runDaemon(opts) {
246
251
  }
247
252
  const hostNote = opts.host ? `, hosted by the ${opts.host} app` : "";
248
253
  log("info", `starting hq daemon (pid ${process.pid}) as ${machineType} (${detected.detail})${hostNote}`);
254
+ // On Linux, services can outlive a daemon (KillMode=process). Adopt the ones
255
+ // the previous daemon handed over and stop any others it left behind before
256
+ // starting anything, so a service never runs twice.
257
+ const succession = platform === "linux" && opts.host === undefined ? successionPaths(paths.root) : null;
258
+ const signalPid = (pid, sig) => {
259
+ process.kill(pid, sig);
260
+ };
261
+ let adopt = [];
262
+ if (succession) {
263
+ const taken = takeSuccession(succession, { now: Date.now, readStart: readProcStart, log });
264
+ await stopOrphans(taken.orphans, {
265
+ readStart: readProcStart,
266
+ signal: signalPid,
267
+ sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
268
+ log,
269
+ });
270
+ adopt = taken.adopt.map((u) => ({ id: u.id, key: u.key, child: adoptChild(u, { readStart: readProcStart, signal: signalPid }) }));
271
+ }
272
+ const tracker = succession ? createChildTracker(succession.childrenPath, { readStart: readProcStart, log }) : null;
273
+ for (const unit of adopt)
274
+ tracker?.track(unit.id, unit.child);
275
+ // Until now a crash leaves the previous records for the next daemon; from here the tracker owns them.
276
+ if (succession && tracker)
277
+ finishSuccession(succession, tracker.records(), log);
278
+ /** Service processes this daemon spawned, so a handoff can let go of them. */
279
+ const spawned = new Set();
249
280
  if (!resolveSyncRunnerPath())
250
281
  log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
251
282
  const updatable = cliSelfUpdatable();
@@ -320,6 +351,11 @@ export async function runDaemon(opts) {
320
351
  stdio: ["ignore", fd, fd],
321
352
  windowsHide: true,
322
353
  });
354
+ spawned.add(child);
355
+ child.once("exit", () => spawned.delete(child));
356
+ // Only long-running services are handed over; a scheduled run left behind finishes on its own.
357
+ if (unit.kind === "long-running")
358
+ tracker?.track(unit.id, child);
323
359
  return child;
324
360
  }
325
361
  finally {
@@ -346,9 +382,36 @@ export async function runDaemon(opts) {
346
382
  restartDaemon: () => restartDaemon(),
347
383
  writeState: (state) => writeJsonAtomic(paths.statePath, state),
348
384
  log,
385
+ cliVersion: CLI_VERSION,
386
+ adopt,
349
387
  });
388
+ /** Hand services over on this restart: the unit leaves them running and the flag is on. */
389
+ const handoffWanted = async () => {
390
+ const killMode = platform === "linux"
391
+ ? systemdKillMode((file, args) => {
392
+ const r = spawnSync(file, args, { encoding: "utf8" });
393
+ return { status: r.status, stdout: r.stdout ?? "" };
394
+ })
395
+ : null;
396
+ if (!handoffEligible({ platform, managed: opts.managed === true, host: opts.host, killMode }))
397
+ return false;
398
+ let timer;
399
+ const timeout = new Promise((r) => {
400
+ timer = setTimeout(() => r(false), HANDOFF_FLAG_TIMEOUT_MS);
401
+ });
402
+ const on = await Promise.race([
403
+ readDaemonFlag(SUPERVISOR_HANDOFF_FLAG, {
404
+ env,
405
+ onError: (err) => log("warn", `could not read ${SUPERVISOR_HANDOFF_FLAG}: ${err.message ?? err}`),
406
+ }),
407
+ timeout,
408
+ ]);
409
+ clearTimeout(timer);
410
+ return on;
411
+ };
350
412
  return await new Promise((resolve) => {
351
413
  let stopping = false;
414
+ let restartPending = false;
352
415
  const shutdown = (why, exitCode = 0) => {
353
416
  if (stopping)
354
417
  return;
@@ -370,12 +433,62 @@ export async function runDaemon(opts) {
370
433
  resolve(exitCode);
371
434
  });
372
435
  };
436
+ /** Exit for a restart but leave the services running for the next daemon to adopt. */
437
+ const handOver = () => {
438
+ if (stopping)
439
+ return;
440
+ stopping = true;
441
+ const released = supervisor.release();
442
+ const starts = new Map((tracker?.records() ?? []).map((r) => [r.pid, r.start]));
443
+ const units = released.flatMap((u) => {
444
+ const start = starts.get(u.pid);
445
+ if (start !== undefined)
446
+ return [{ ...u, start }];
447
+ // Unverifiable later, so it cannot be adopted; stop it rather than leave a second copy.
448
+ log("warn", `stopping ${u.id}: its process could not be recorded for the next daemon`);
449
+ try {
450
+ process.kill(u.pid, "SIGTERM");
451
+ }
452
+ catch {
453
+ // already gone
454
+ }
455
+ return [];
456
+ });
457
+ try {
458
+ writeHandoff(succession.handoffPath, { releasedAt: Date.now(), units });
459
+ log("info", `restarting onto the updated CLI; handing ${units.length} running service(s) to the next daemon`);
460
+ }
461
+ catch (err) {
462
+ // The children record still lists them, so the next daemon stops and restarts them.
463
+ log("error", `could not write the handoff; the next daemon will restart the services: ${err.message}`);
464
+ }
465
+ // The released processes must not keep this one alive.
466
+ for (const child of spawned)
467
+ child.unref();
468
+ releaseDaemonLock(paths.lockPath);
469
+ resolve(RESTART_EXIT_CODE);
470
+ // Anything else still holding the event loop open must not delay the restart.
471
+ setTimeout(() => process.exit(RESTART_EXIT_CODE), 5_000).unref();
472
+ };
373
473
  restartDaemon = () => {
374
474
  if (!restartsForParent(opts)) {
375
475
  log("warn", "a restart was requested, but no service manager would start this daemon again; restart it yourself");
376
476
  return;
377
477
  }
378
- shutdown("restarting onto the updated CLI", RESTART_EXIT_CODE);
478
+ if (stopping || restartPending)
479
+ return;
480
+ restartPending = true;
481
+ void handoffWanted()
482
+ .catch((err) => {
483
+ log("warn", `could not decide on a handoff; restarting every service: ${err.message}`);
484
+ return false;
485
+ })
486
+ .then((handoff) => {
487
+ if (handoff && succession)
488
+ handOver();
489
+ else
490
+ shutdown("restarting onto the updated CLI", RESTART_EXIT_CODE);
491
+ });
379
492
  };
380
493
  process.on("SIGTERM", () => shutdown("received SIGTERM"));
381
494
  process.on("SIGINT", () => shutdown("received SIGINT"));
@@ -0,0 +1,116 @@
1
+ /**
2
+ * Handing a Linux outpost's services from one daemon to the next.
3
+ *
4
+ * Under `KillMode=process` systemd signals only the daemon, so its services
5
+ * outlive it. The daemon therefore keeps two records under its folder:
6
+ *
7
+ * - `children.json`: every service process it started, with the process start
8
+ * time from /proc, kept current as they start and exit;
9
+ * - `handoff.json`: written only when the daemon restarts onto an updated CLI
10
+ * with the `daemon.supervisor-handoff` flag on, listing the long-running
11
+ * services it left running and the key each was started with.
12
+ *
13
+ * On start the next daemon adopts the services named in a fresh handoff whose
14
+ * pid still belongs to the same process, and stops every other recorded
15
+ * process that is still alive (after a crash, or a restart with the handoff
16
+ * off) before it starts anything, so two copies of a service never run. Both
17
+ * records stay on disk until the adopted services are recorded as its own
18
+ * (finishSuccession), so a daemon that dies mid-takeover leaves them for the
19
+ * next one.
20
+ */
21
+ import { type HostApp, type SupervisedChild } from "./supervisor.js";
22
+ /** A handoff older than this is not trusted; its services are stopped instead. */
23
+ export declare const HANDOFF_MAX_AGE_MS: number;
24
+ export interface ChildRecord {
25
+ id: string;
26
+ pid: number;
27
+ /** Process start time (/proc/<pid>/stat field 22), so a reused pid is never mistaken for it. */
28
+ start: number;
29
+ }
30
+ export interface HandoffUnit extends ChildRecord {
31
+ key: string;
32
+ }
33
+ export interface HandoffRecord {
34
+ releasedAt: number;
35
+ units: HandoffUnit[];
36
+ }
37
+ export interface SuccessionPaths {
38
+ childrenPath: string;
39
+ handoffPath: string;
40
+ }
41
+ type ReadStart = (pid: number) => number | null;
42
+ type Log = (level: "info" | "warn" | "error", msg: string) => void;
43
+ export declare function successionPaths(daemonRoot: string): SuccessionPaths;
44
+ /**
45
+ * The start time (field 22) from a /proc/<pid>/stat line, or null when the
46
+ * line is unreadable or the process is a zombie (it has exited).
47
+ */
48
+ export declare function parseProcStart(text: string): number | null;
49
+ /** The start time of `pid` on Linux, or null when no such live process exists. */
50
+ export declare function readProcStart(pid: number): number | null;
51
+ /** Keeps `children.json` in step with the service processes this daemon started. */
52
+ export declare function createChildTracker(file: string, deps: {
53
+ readStart: ReadStart;
54
+ log?: Log;
55
+ }): {
56
+ track(id: string, child: SupervisedChild): void;
57
+ records(): ChildRecord[];
58
+ };
59
+ export declare function writeHandoff(file: string, record: HandoffRecord): void;
60
+ /**
61
+ * Read the previous daemon's records. Returns the services to adopt and the
62
+ * live processes to stop; a pid whose start time no longer matches belongs to
63
+ * another process and appears in neither. The records stay until
64
+ * finishSuccession.
65
+ */
66
+ export declare function takeSuccession(paths: SuccessionPaths, deps: {
67
+ now: () => number;
68
+ readStart: ReadStart;
69
+ log: Log;
70
+ }): {
71
+ adopt: HandoffUnit[];
72
+ orphans: ChildRecord[];
73
+ };
74
+ /**
75
+ * Complete a takeover: `records` (the adopted services, now this daemon's)
76
+ * replace the children record, then the handoff is dropped. Call it once the
77
+ * orphans are stopped.
78
+ */
79
+ export declare function finishSuccession(paths: SuccessionPaths, records: readonly ChildRecord[], log?: Log): void;
80
+ /** SIGTERM each orphan, wait up to `graceMs` for it to go, then SIGKILL what is left. */
81
+ export declare function stopOrphans(orphans: readonly ChildRecord[], deps: {
82
+ readStart: ReadStart;
83
+ signal: (pid: number, sig: NodeJS.Signals) => void;
84
+ sleep: (ms: number) => Promise<void>;
85
+ log: Log;
86
+ graceMs?: number;
87
+ }): Promise<void>;
88
+ /**
89
+ * A service process started by the previous daemon. It is not this process's
90
+ * child, so its exit is noticed by polling and its exit code is unknown.
91
+ */
92
+ export declare function adoptChild(record: ChildRecord, deps: {
93
+ readStart: ReadStart;
94
+ signal: (pid: number, sig: NodeJS.Signals) => void;
95
+ setInterval?: (fn: () => void, ms: number) => unknown;
96
+ clearInterval?: (handle: unknown) => void;
97
+ pollMs?: number;
98
+ }): SupervisedChild;
99
+ /** The loaded daemon unit's KillMode, or null when systemctl cannot tell. */
100
+ export declare function systemdKillMode(run: (file: string, args: string[]) => {
101
+ status: number | null;
102
+ stdout: string;
103
+ }): string | null;
104
+ /**
105
+ * Services survive the daemon only on Linux under the systemd system unit
106
+ * with `KillMode=process` (a unit installed by this CLI or later). Anywhere
107
+ * else the service manager or the host app stops them with the daemon.
108
+ */
109
+ export declare function handoffEligible(opts: {
110
+ platform: NodeJS.Platform;
111
+ managed: boolean;
112
+ host: HostApp | undefined;
113
+ killMode: string | null;
114
+ }): boolean;
115
+ export {};
116
+ //# sourceMappingURL=succession.d.ts.map
@@ -0,0 +1,262 @@
1
+ /**
2
+ * Handing a Linux outpost's services from one daemon to the next.
3
+ *
4
+ * Under `KillMode=process` systemd signals only the daemon, so its services
5
+ * outlive it. The daemon therefore keeps two records under its folder:
6
+ *
7
+ * - `children.json`: every service process it started, with the process start
8
+ * time from /proc, kept current as they start and exit;
9
+ * - `handoff.json`: written only when the daemon restarts onto an updated CLI
10
+ * with the `daemon.supervisor-handoff` flag on, listing the long-running
11
+ * services it left running and the key each was started with.
12
+ *
13
+ * On start the next daemon adopts the services named in a fresh handoff whose
14
+ * pid still belongs to the same process, and stops every other recorded
15
+ * process that is still alive (after a crash, or a restart with the handoff
16
+ * off) before it starts anything, so two copies of a service never run. Both
17
+ * records stay on disk until the adopted services are recorded as its own
18
+ * (finishSuccession), so a daemon that dies mid-takeover leaves them for the
19
+ * next one.
20
+ */
21
+ import { EventEmitter } from "node:events";
22
+ import * as fs from "node:fs";
23
+ import * as path from "node:path";
24
+ import { DAEMON_SYSTEMD_UNIT } from "./install.js";
25
+ import { ensurePrivateDir } from "./paths.js";
26
+ import { STOP_ESCALATION_MS } from "./supervisor.js";
27
+ /** A handoff older than this is not trusted; its services are stopped instead. */
28
+ export const HANDOFF_MAX_AGE_MS = 5 * 60_000;
29
+ const ADOPTED_POLL_MS = 1_000;
30
+ const ORPHAN_POLL_MS = 200;
31
+ export function successionPaths(daemonRoot) {
32
+ return {
33
+ childrenPath: path.join(daemonRoot, "children.json"),
34
+ handoffPath: path.join(daemonRoot, "handoff.json"),
35
+ };
36
+ }
37
+ function writeJsonAtomic(file, value) {
38
+ ensurePrivateDir(path.dirname(file));
39
+ const tmp = `${file}.${process.pid}.tmp`;
40
+ fs.writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
41
+ fs.renameSync(tmp, file);
42
+ }
43
+ function isChildRecord(value) {
44
+ const r = value;
45
+ return (typeof r === "object" &&
46
+ r !== null &&
47
+ typeof r.id === "string" &&
48
+ Number.isInteger(r.pid) &&
49
+ r.pid > 0 &&
50
+ typeof r.start === "number" &&
51
+ Number.isFinite(r.start));
52
+ }
53
+ function removeRecord(file, log) {
54
+ try {
55
+ fs.rmSync(file, { force: true });
56
+ }
57
+ catch (err) {
58
+ log("warn", `could not remove ${file}: ${err.message}`);
59
+ }
60
+ }
61
+ /** Read a record file; null when it is missing. An unreadable one is removed, since no later read can use it. */
62
+ function readRecord(file, log) {
63
+ let text;
64
+ try {
65
+ text = fs.readFileSync(file, "utf8");
66
+ }
67
+ catch (err) {
68
+ if (err.code !== "ENOENT")
69
+ log("warn", `could not read ${file}: ${err.message}`);
70
+ return null;
71
+ }
72
+ try {
73
+ return JSON.parse(text);
74
+ }
75
+ catch (err) {
76
+ log("warn", `ignoring unreadable ${file}: ${err.message}`);
77
+ removeRecord(file, log);
78
+ return null;
79
+ }
80
+ }
81
+ /**
82
+ * The start time (field 22) from a /proc/<pid>/stat line, or null when the
83
+ * line is unreadable or the process is a zombie (it has exited).
84
+ */
85
+ export function parseProcStart(text) {
86
+ const close = text.lastIndexOf(")");
87
+ if (close < 0)
88
+ return null;
89
+ // Fields after the command name start at field 3 (state), so field 22 is index 19.
90
+ const rest = text.slice(close + 1).trim().split(/\s+/);
91
+ if (rest[0] === "Z" || rest[0] === "X")
92
+ return null;
93
+ const start = Number(rest[19]);
94
+ return Number.isInteger(start) ? start : null;
95
+ }
96
+ /** The start time of `pid` on Linux, or null when no such live process exists. */
97
+ export function readProcStart(pid) {
98
+ try {
99
+ return parseProcStart(fs.readFileSync(`/proc/${pid}/stat`, "utf8"));
100
+ }
101
+ catch {
102
+ return null;
103
+ }
104
+ }
105
+ /** Keeps `children.json` in step with the service processes this daemon started. */
106
+ export function createChildTracker(file, deps) {
107
+ const current = new Map();
108
+ const save = () => {
109
+ try {
110
+ writeJsonAtomic(file, [...current.values()]);
111
+ }
112
+ catch (err) {
113
+ deps.log?.("error", `could not record service processes: ${err.message}`);
114
+ }
115
+ };
116
+ return {
117
+ track(id, child) {
118
+ if (child.pid === undefined)
119
+ return;
120
+ const start = deps.readStart(child.pid);
121
+ if (start === null)
122
+ return;
123
+ current.set(child, { id, pid: child.pid, start });
124
+ save();
125
+ child.on("exit", () => {
126
+ if (current.delete(child))
127
+ save();
128
+ });
129
+ },
130
+ records: () => [...current.values()],
131
+ };
132
+ }
133
+ export function writeHandoff(file, record) {
134
+ writeJsonAtomic(file, { version: 1, ...record });
135
+ }
136
+ /**
137
+ * Read the previous daemon's records. Returns the services to adopt and the
138
+ * live processes to stop; a pid whose start time no longer matches belongs to
139
+ * another process and appears in neither. The records stay until
140
+ * finishSuccession.
141
+ */
142
+ export function takeSuccession(paths, deps) {
143
+ const children = readRecord(paths.childrenPath, deps.log);
144
+ const handoff = readRecord(paths.handoffPath, deps.log);
145
+ const recorded = Array.isArray(children) ? children.filter(isChildRecord) : [];
146
+ let handoffUnits = [];
147
+ let fresh = false;
148
+ if (handoff !== null) {
149
+ if (typeof handoff.releasedAt !== "number" || !Array.isArray(handoff.units)) {
150
+ deps.log("warn", "ignoring a malformed handoff record");
151
+ removeRecord(paths.handoffPath, deps.log);
152
+ }
153
+ else {
154
+ handoffUnits = handoff.units.filter((u) => isChildRecord(u) && typeof u.key === "string");
155
+ fresh = deps.now() - handoff.releasedAt <= HANDOFF_MAX_AGE_MS;
156
+ if (!fresh)
157
+ deps.log("warn", "the previous daemon's handoff is too old to trust; stopping its services");
158
+ }
159
+ }
160
+ const alive = (r) => deps.readStart(r.pid) === r.start;
161
+ const adopt = fresh ? handoffUnits.filter(alive) : [];
162
+ const adopted = new Set(adopt.map((u) => u.pid));
163
+ const orphans = new Map();
164
+ for (const r of [...recorded, ...handoffUnits]) {
165
+ if (adopted.has(r.pid) || orphans.has(r.pid) || !alive(r))
166
+ continue;
167
+ orphans.set(r.pid, { id: r.id, pid: r.pid, start: r.start });
168
+ }
169
+ return { adopt, orphans: [...orphans.values()] };
170
+ }
171
+ /**
172
+ * Complete a takeover: `records` (the adopted services, now this daemon's)
173
+ * replace the children record, then the handoff is dropped. Call it once the
174
+ * orphans are stopped.
175
+ */
176
+ export function finishSuccession(paths, records, log) {
177
+ writeJsonAtomic(paths.childrenPath, records);
178
+ removeRecord(paths.handoffPath, log ?? (() => { }));
179
+ }
180
+ /** SIGTERM each orphan, wait up to `graceMs` for it to go, then SIGKILL what is left. */
181
+ export async function stopOrphans(orphans, deps) {
182
+ const graceMs = deps.graceMs ?? STOP_ESCALATION_MS;
183
+ const same = (r) => deps.readStart(r.pid) === r.start;
184
+ const send = (r, sig) => {
185
+ try {
186
+ deps.signal(r.pid, sig);
187
+ }
188
+ catch (err) {
189
+ deps.log("warn", `could not send ${sig} to ${r.id} (pid ${r.pid}): ${err.message}`);
190
+ }
191
+ };
192
+ let remaining = orphans.filter(same);
193
+ for (const r of remaining) {
194
+ deps.log("info", `stopping ${r.id} (pid ${r.pid}), left running by a previous daemon`);
195
+ send(r, "SIGTERM");
196
+ }
197
+ let waited = 0;
198
+ remaining = remaining.filter(same);
199
+ while (remaining.length > 0 && waited < graceMs) {
200
+ await deps.sleep(ORPHAN_POLL_MS);
201
+ waited += ORPHAN_POLL_MS;
202
+ remaining = remaining.filter(same);
203
+ }
204
+ for (const r of remaining) {
205
+ deps.log("warn", `${r.id} (pid ${r.pid}) ignored SIGTERM; sending SIGKILL`);
206
+ send(r, "SIGKILL");
207
+ }
208
+ }
209
+ /**
210
+ * A service process started by the previous daemon. It is not this process's
211
+ * child, so its exit is noticed by polling and its exit code is unknown.
212
+ */
213
+ export function adoptChild(record, deps) {
214
+ const events = new EventEmitter();
215
+ const setI = deps.setInterval ?? ((fn, ms) => setInterval(fn, ms));
216
+ const clearI = deps.clearInterval ?? ((h) => clearInterval(h));
217
+ const same = () => deps.readStart(record.pid) === record.start;
218
+ let ended = false;
219
+ const poll = setI(() => {
220
+ if (ended || same())
221
+ return;
222
+ ended = true;
223
+ clearI(poll);
224
+ events.emit("exit", null, null);
225
+ }, deps.pollMs ?? ADOPTED_POLL_MS);
226
+ poll.unref?.();
227
+ const child = {
228
+ pid: record.pid,
229
+ on(event, handler) {
230
+ events.on(event, handler);
231
+ return child;
232
+ },
233
+ kill(signal = "SIGTERM") {
234
+ if (ended || !same())
235
+ return false;
236
+ deps.signal(record.pid, (typeof signal === "number" ? "SIGTERM" : signal));
237
+ return true;
238
+ },
239
+ };
240
+ return child;
241
+ }
242
+ /** The loaded daemon unit's KillMode, or null when systemctl cannot tell. */
243
+ export function systemdKillMode(run) {
244
+ try {
245
+ const r = run("systemctl", ["show", "--property=KillMode", "--value", DAEMON_SYSTEMD_UNIT]);
246
+ if (r.status !== 0)
247
+ return null;
248
+ return r.stdout.trim() || null;
249
+ }
250
+ catch {
251
+ return null;
252
+ }
253
+ }
254
+ /**
255
+ * Services survive the daemon only on Linux under the systemd system unit
256
+ * with `KillMode=process` (a unit installed by this CLI or later). Anywhere
257
+ * else the service manager or the host app stops them with the daemon.
258
+ */
259
+ export function handoffEligible(opts) {
260
+ return opts.platform === "linux" && opts.managed && opts.host === undefined && opts.killMode === "process";
261
+ }
262
+ //# sourceMappingURL=succession.js.map
@@ -13,7 +13,11 @@
13
13
  * - restart/trigger/stop requests from the CLI are polled and applied; a
14
14
  * request for a unit that is not planned yet re-reads the plan first;
15
15
  * - stop() sends SIGTERM to every child, escalates to SIGKILL after
16
- * STOP_ESCALATION_MS, and resolves once every child has exited.
16
+ * STOP_ESCALATION_MS, and resolves once every child has exited;
17
+ * - release() instead leaves every child running and reports the long-running
18
+ * ones, so the next daemon can adopt them (`adopt`). Adopted units whose
19
+ * command is unchanged keep running; changed ones are replaced one at a
20
+ * time, draining first when the unit has a drain signal.
17
21
  *
18
22
  * Processes, time, and state output are injected; `run.ts` wires the real ones.
19
23
  */
@@ -24,6 +28,10 @@ export declare const BACKOFF_BASE_MS = 1000;
24
28
  export declare const BACKOFF_MAX_MS = 60000;
25
29
  export declare const HEALTHY_RUN_MS = 60000;
26
30
  export declare const STOP_ESCALATION_MS = 10000;
31
+ /** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
32
+ export declare const HANDOFF_STAGGER_MS = 15000;
33
+ /** How long a draining unit may take to finish its work before it gets SIGTERM. */
34
+ export declare const HANDOFF_DRAIN_MAX_MS: number;
27
35
  export interface SupervisedChild {
28
36
  pid?: number;
29
37
  on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
@@ -62,6 +70,18 @@ export interface DaemonState {
62
70
  host?: HostApp;
63
71
  units: UnitState[];
64
72
  }
73
+ /** A running service process left by the previous daemon, and the key it was started with. */
74
+ export interface AdoptedUnit {
75
+ id: string;
76
+ key: string;
77
+ child: SupervisedChild;
78
+ }
79
+ /** A long-running service left running by release(). */
80
+ export interface ReleasedUnit {
81
+ id: string;
82
+ pid: number;
83
+ key: string;
84
+ }
65
85
  export interface SupervisorDeps {
66
86
  machineType: MachineType;
67
87
  host?: HostApp;
@@ -79,6 +99,10 @@ export interface SupervisorDeps {
79
99
  clearTimeout?: (handle: unknown) => void;
80
100
  reconcileMs?: number;
81
101
  requestPollMs?: number;
102
+ /** The running CLI's version; part of the key of units that run `hq` commands. */
103
+ cliVersion?: string;
104
+ /** Services to take over from the previous daemon instead of starting them. */
105
+ adopt?: readonly AdoptedUnit[];
82
106
  }
83
107
  export interface Supervisor {
84
108
  start(): void;
@@ -86,7 +110,16 @@ export interface Supervisor {
86
110
  stop(): Promise<void>;
87
111
  reconcile(): void;
88
112
  state(): DaemonState;
113
+ /** Stop supervising without signalling any child; returns the long-running units left running. */
114
+ release(): ReleasedUnit[];
89
115
  }
90
116
  export declare function backoffMs(attempt: number): number;
117
+ /**
118
+ * What a unit's process was started from; a unit whose key changes is
119
+ * restarted. The CLI version counts only for `hq` commands, so a CLI update
120
+ * leaves hq-cloud runners, other programs, and units marked
121
+ * `keepAcrossCliUpdates` running.
122
+ */
123
+ export declare function planKey(plan: UnitPlan, cliVersion?: string): string;
91
124
  export declare function createSupervisor(deps: SupervisorDeps): Supervisor;
92
125
  //# sourceMappingURL=supervisor.d.ts.map