@indigoai-us/hq-cli 5.275.0 → 5.277.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +112 -0
  2. package/README.md +12 -0
  3. package/assets/bot-workers/setup/context/USER-GUIDE.md +4 -4
  4. package/assets/scaffold/core/scripts/hq-delegate-pickup.sh +99 -0
  5. package/assets/scaffold/core/scripts/hq-delegate-repo.sh +158 -0
  6. package/assets/scaffold/core/scripts/hq-delegate-resolve.sh +145 -0
  7. package/assets/scaffold/core/scripts/hq-delegate-secrets.sh +129 -0
  8. package/assets/scaffold/core/scripts/hq-delegate-transfer.sh +175 -0
  9. package/assets/scaffold/core/scripts/lib/secret-patterns.sh +58 -0
  10. package/dist/command-catalog.generated.d.ts +239 -1
  11. package/dist/command-catalog.generated.js +302 -1
  12. package/dist/commands/core-checkpoint.d.ts +4 -2
  13. package/dist/commands/core-checkpoint.js +34 -3
  14. package/dist/commands/core-command-catalog.d.ts +4 -2
  15. package/dist/commands/core-command-catalog.js +5 -2
  16. package/dist/commands/core-delegate.d.ts +6 -0
  17. package/dist/commands/core-delegate.js +987 -0
  18. package/dist/commands/core-ontology.js +3 -3
  19. package/dist/commands/core.js +11 -1
  20. package/dist/commands/daemon.d.ts +4 -0
  21. package/dist/commands/daemon.js +175 -2
  22. package/dist/commands/dm.d.ts +2 -0
  23. package/dist/commands/dm.js +11 -4
  24. package/dist/commands/feedback.d.ts +6 -0
  25. package/dist/commands/feedback.js +86 -37
  26. package/dist/commands/lanes-accounts.js +44 -2
  27. package/dist/commands/lanes-message.js +69 -12
  28. package/dist/commands/lanes-resume.js +2 -0
  29. package/dist/commands/lanes-shared.d.ts +3 -0
  30. package/dist/commands/lanes-shared.js +12 -2
  31. package/dist/commands/lanes.d.ts +5 -1
  32. package/dist/commands/lanes.js +402 -49
  33. package/dist/commands/run-feature-telemetry.d.ts +3 -0
  34. package/dist/commands/run-feature-telemetry.js +6 -0
  35. package/dist/commands/run.js +3 -2
  36. package/dist/commands/secrets.d.ts +16 -1
  37. package/dist/commands/secrets.js +80 -45
  38. package/dist/commands/sync-mode.d.ts +11 -0
  39. package/dist/commands/sync-mode.js +30 -0
  40. package/dist/commands/whoami.d.ts +4 -6
  41. package/dist/commands/whoami.js +39 -8
  42. package/dist/fast-core.d.ts +6 -7
  43. package/dist/fast-core.js +22 -8
  44. package/dist/fast-scaffold.d.ts +33 -7
  45. package/dist/fast-scaffold.js +104 -8
  46. package/dist/index.js +32 -1
  47. package/dist/lanes-drain-hook.js +154 -7
  48. package/dist/lib/agent-kit/run/inbox.js +3 -2
  49. package/dist/lib/billing/plan-lock.d.ts +3 -1
  50. package/dist/lib/billing/plan-lock.js +7 -1
  51. package/dist/lib/checkpoint-ontology/apply.d.ts +6 -0
  52. package/dist/lib/checkpoint-ontology/apply.js +60 -3
  53. package/dist/lib/checkpoint-ontology/company.d.ts +8 -4
  54. package/dist/lib/checkpoint-ontology/company.js +15 -7
  55. package/dist/lib/checkpoint-ontology/plan.d.ts +36 -0
  56. package/dist/lib/checkpoint-ontology/plan.js +66 -4
  57. package/dist/lib/checkpoint-ontology/queued-touches.d.ts +32 -0
  58. package/dist/lib/checkpoint-ontology/queued-touches.js +193 -0
  59. package/dist/lib/checkpoint-ontology/tree.d.ts +2 -0
  60. package/dist/lib/checkpoint-ontology/tree.js +34 -0
  61. package/dist/lib/core-utils/jq-json-serializer.d.ts +5 -0
  62. package/dist/lib/core-utils/jq-json-serializer.js +10 -0
  63. package/dist/lib/core-utils/secret-patterns.d.ts +41 -0
  64. package/dist/lib/core-utils/secret-patterns.js +33 -0
  65. package/dist/lib/core-utils/timeout-guard-native.d.ts +7 -0
  66. package/dist/lib/core-utils/timeout-guard-native.js +40 -0
  67. package/dist/lib/core-utils/timeout-guard.d.ts +3 -3
  68. package/dist/lib/core-utils/timeout-guard.js +28 -13
  69. package/dist/lib/daemon/catalog.d.ts +18 -2
  70. package/dist/lib/daemon/catalog.js +18 -2
  71. package/dist/lib/daemon/config.d.ts +6 -0
  72. package/dist/lib/daemon/config.js +15 -2
  73. package/dist/lib/daemon/requests.d.ts +4 -0
  74. package/dist/lib/daemon/requests.js +2 -0
  75. package/dist/lib/daemon/run.js +94 -8
  76. package/dist/lib/daemon/supervisor.d.ts +8 -0
  77. package/dist/lib/daemon/supervisor.js +101 -1
  78. package/dist/lib/daemon/sync-runner.d.ts +58 -0
  79. package/dist/lib/daemon/sync-runner.js +277 -0
  80. package/dist/lib/doctor/__testing__/run-doctor.d.ts +8 -0
  81. package/dist/lib/doctor/__testing__/run-doctor.js +52 -0
  82. package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
  83. package/dist/lib/flag-registry-worker-source.generated.js +1 -1
  84. package/dist/lib/flag-registry.d.ts +12 -7
  85. package/dist/lib/flag-registry.js +66 -75
  86. package/dist/lib/flag-snapshot-file.d.ts +20 -0
  87. package/dist/lib/flag-snapshot-file.js +91 -0
  88. package/dist/lib/lanes/claude-balance.d.ts +6 -1
  89. package/dist/lib/lanes/claude-balance.js +46 -8
  90. package/dist/lib/lanes/config.js +15 -0
  91. package/dist/lib/lanes/hire-grant-dispatch.d.ts +2 -0
  92. package/dist/lib/lanes/hire-grant-dispatch.js +41 -0
  93. package/dist/lib/lanes/index.d.ts +1 -1
  94. package/dist/lib/lanes/keep-alive.d.ts +3 -0
  95. package/dist/lib/lanes/keep-alive.js +3 -0
  96. package/dist/lib/lanes/outpost-dispatch.d.ts +4 -0
  97. package/dist/lib/lanes/outpost-dispatch.js +11 -2
  98. package/dist/lib/lanes/reconcile.js +20 -4
  99. package/dist/lib/lanes/spawn.js +17 -10
  100. package/dist/lib/lanes/store.js +45 -1
  101. package/dist/lib/lanes/types.d.ts +30 -1
  102. package/dist/lib/lanes/types.js +2 -2
  103. package/dist/lib/lanes/watcher.js +3 -2
  104. package/dist/lib/process-flag-company-context.d.ts +8 -0
  105. package/dist/lib/process-flag-company-context.js +67 -0
  106. package/dist/main.js +3 -1
  107. package/dist/native-fast-route.d.ts +3 -0
  108. package/dist/native-fast-route.js +39 -0
  109. package/dist/secrets-env-fast-path.d.ts +5 -0
  110. package/dist/secrets-env-fast-path.js +16 -0
  111. package/dist/secrets-env-fast-route.d.ts +18 -0
  112. package/dist/secrets-env-fast-route.js +54 -0
  113. package/dist/utils/cli-telemetry.d.ts +26 -2
  114. package/dist/utils/cli-telemetry.js +178 -3
  115. package/dist/utils/hq-root-resolver.d.ts +13 -0
  116. package/dist/utils/hq-root-resolver.js +103 -0
  117. package/dist/utils/scaffold-fast-import-guard.js +6 -1
  118. package/dist/utils/vault-api.d.ts +8 -0
  119. package/dist/utils/vault-api.js +13 -1
  120. package/package.json +2 -2
@@ -20,9 +20,6 @@
20
20
  *
21
21
  * Exit codes: 0 = allow, 2 = block.
22
22
  */
23
- import { loadCachedTokens } from "@indigoai-us/hq-cloud";
24
- import { peekIdToken } from "../../utils/id-token.js";
25
- import { resolveFlagGate, resolveProcessFlagGate, } from "../flag-registry.js";
26
23
  // Observed Claude Code Bash-tool bounds. The tool `timeout` (ms) raises the
27
24
  // deadline up to the max; with none declared the default applies.
28
25
  const HARNESS_DEFAULT_MS = 120_000; // 2 min
@@ -153,8 +150,14 @@ supervise yourself.)
153
150
 
154
151
  Policy: hq-foreground-timeout-killed-by-harness-deadline.
155
152
  Bypass for one sanctioned foreground run: prefix HQ_ALLOW_LONG_FOREGROUND=1.`;
156
- function defaultEmail() {
153
+ async function defaultEmail() {
157
154
  try {
155
+ // The cached identity is needed only for commands that would otherwise
156
+ // block. Keep hq-cloud and the JWT decoder out of the common allow path.
157
+ const [{ loadCachedTokens }, { peekIdToken }] = await Promise.all([
158
+ import("@indigoai-us/hq-cloud"),
159
+ import("../../utils/id-token.js"),
160
+ ]);
158
161
  const cached = loadCachedTokens();
159
162
  if (!cached)
160
163
  return undefined;
@@ -166,23 +169,35 @@ function defaultEmail() {
166
169
  }
167
170
  }
168
171
  /** Whether the guard is active for this identity (rollout gate). */
169
- export function isGatedUser(email, flagReader) {
172
+ export async function isGatedUser(email, flagReader) {
170
173
  const legacyFallback = () => typeof email === "string" && email.toLowerCase().endsWith(GATE_DOMAIN);
171
174
  const lookup = {
172
175
  defaultEmailDomains: ["getindigo.ai"],
173
176
  personEmail: email,
174
177
  fallback: legacyFallback(),
175
178
  };
176
- return flagReader
177
- ? resolveFlagGate(flagReader, "cli.foreground-timeout-guard", lookup, legacyFallback)
178
- : resolveProcessFlagGate("cli.foreground-timeout-guard", lookup, legacyFallback);
179
+ if (flagReader) {
180
+ try {
181
+ return flagReader.isEnabled("cli.foreground-timeout-guard", lookup);
182
+ }
183
+ catch {
184
+ return legacyFallback();
185
+ }
186
+ }
187
+ try {
188
+ const { resolveProcessFlagGate } = await import("../flag-registry.js");
189
+ return resolveProcessFlagGate("cli.foreground-timeout-guard", lookup, legacyFallback);
190
+ }
191
+ catch {
192
+ return legacyFallback();
193
+ }
179
194
  }
180
195
  /**
181
196
  * Command entry: read the PreToolUse JSON, apply the gate + short-circuits, and
182
197
  * return the exit code (0 allow / 2 block). Never throws — a parse failure or a
183
198
  * missing identity fails OPEN (allow), because a hook must not break tool calls.
184
199
  */
185
- export function timeoutGuardCommand(stdin, deps = {}) {
200
+ export async function timeoutGuardCommand(stdin, deps = {}) {
186
201
  const env = deps.env ?? process.env;
187
202
  const err = deps.stderr ?? process.stderr;
188
203
  let payload;
@@ -201,15 +216,15 @@ export function timeoutGuardCommand(stdin, deps = {}) {
201
216
  return 0;
202
217
  if (/(^|\s)HQ_ALLOW_LONG_FOREGROUND=1(\s|$)/.test(command))
203
218
  return 0;
204
- // Rollout gate: only act for @getindigo.ai identities.
205
- const getEmail = deps.getEmail ?? defaultEmail;
206
- if (!isGatedUser(getEmail(), deps.flagReader))
207
- return 0;
208
219
  const runInBackground = ti.run_in_background === true;
209
220
  const toolTimeoutMs = typeof ti.timeout === "number" ? ti.timeout : undefined;
210
221
  const verdict = evaluateTimeoutGuard({ command, toolTimeoutMs, runInBackground });
211
222
  if (!verdict.block)
212
223
  return 0;
224
+ // Rollout gate: only act for @getindigo.ai identities.
225
+ const getEmail = deps.getEmail ?? defaultEmail;
226
+ if (!await isGatedUser(await getEmail(), deps.flagReader))
227
+ return 0;
213
228
  err.write(`${BLOCK_MESSAGE(verdict.reason ?? "declared timeout exceeds the harness ceiling")}\n`);
214
229
  return 2;
215
230
  }
@@ -10,6 +10,8 @@
10
10
  * them instead of silently dropping them.
11
11
  */
12
12
  import type { MachineType } from "./machine-type.js";
13
+ import type { SyncOwner } from "./config.js";
14
+ import { type SyncRunnerProcess } from "./sync-runner.js";
13
15
  export declare const SERVICE_NAMES: readonly ["sync", "mesh", "heartbeat", "inbox", "bots", "remote-control", "codex-remote-control", "session-host", "lanes-reconcile", "search-index", "jobs-reconcile", "updater", "housekeeping"];
14
16
  export type ServiceName = (typeof SERVICE_NAMES)[number];
15
17
  export declare function isServiceName(value: unknown): value is ServiceName;
@@ -51,6 +53,8 @@ export interface UnitPlan {
51
53
  restart: "always" | "on-failure";
52
54
  intervalSec?: number;
53
55
  initialDelaySec?: number;
56
+ /** Kill and restart this child after silence on its liveness stream. */
57
+ heartbeatTimeoutMs?: number;
54
58
  command: UnitCommand | null;
55
59
  env: Record<string, string>;
56
60
  blocked?: UnitBlock;
@@ -75,11 +79,23 @@ export interface PlanContext {
75
79
  sessionHostRunnerPath: string | null;
76
80
  /** Environment retained from legacy outpost sync systemd drop-ins. */
77
81
  legacySyncEnv: Record<string, string>;
82
+ /** Keep the local sync runner stopped until the operator resumes it. */
83
+ syncPaused: boolean;
84
+ /** Persisted/effective owner selected for this machine. */
85
+ syncOwner: SyncOwner;
86
+ /** A legacy desktop app or sync unit is still managing sync. */
87
+ desktopAppManagesSync: boolean;
88
+ /** Full process list, or null when the daemon could not read it. */
89
+ syncRunnerProcesses: readonly SyncRunnerProcess[] | null;
90
+ /** PID recorded in the watch-owner status, when it is readable. */
91
+ syncRunnerOwnerPid: number | null;
92
+ /** Sync runner children already started by this daemon before their lease is written. */
93
+ daemonSyncRunnerPids?: ReadonlySet<number>;
94
+ /** The protocol-event heartbeat watchdog timeout for the sync runner. */
95
+ syncHeartbeatTimeoutMs: number;
78
96
  signedIn: boolean;
79
97
  /** ~/.hq-agent/kit.json exists. */
80
98
  kitConfigured: boolean;
81
- /** The HQ desktop app runs its own sync on this machine. */
82
- desktopAppManagesSync: boolean;
83
99
  bots: readonly string[];
84
100
  /** Name of a pre-daemon unit that still runs this unit id, or null. */
85
101
  legacyOwner: (unitId: string) => string | null;
@@ -9,6 +9,7 @@
9
9
  * with no entrypoint yet stay in the catalog as `unavailable` so status shows
10
10
  * them instead of silently dropping them.
11
11
  */
12
+ import { SYNC_RUNNER_OWNER, syncRunnerUnleasedBlockReason } from "./sync-runner.js";
12
13
  export const SERVICE_NAMES = [
13
14
  "sync",
14
15
  "mesh",
@@ -127,14 +128,29 @@ function planService(service, ctx) {
127
128
  command: {
128
129
  type: "node",
129
130
  script: ctx.syncRunnerPath,
130
- args: ["--companies", "--direction", "both", "--on-conflict", "keep", "--hq-root", ctx.hqRoot ?? "", "--watch", "--event-push"],
131
+ args: ["--companies", "--direction", "both", "--on-conflict", "keep", "--hq-root", ctx.hqRoot ?? "", "--watch", "--event-push", "--owner", SYNC_RUNNER_OWNER],
131
132
  },
132
133
  env: { ...ctx.legacySyncEnv },
134
+ heartbeatTimeoutMs: ctx.syncHeartbeatTimeoutMs,
133
135
  });
136
+ let ownerBlock;
137
+ if (ctx.machineType === "desktop" && ctx.syncOwner === "desktop-app" && ctx.desktopAppManagesSync) {
138
+ ownerBlock = "the desktop app is still managing sync (run `hq daemon sync take-ownership` after it stops its runner)";
139
+ }
140
+ else if (ctx.machineType === "desktop" &&
141
+ ctx.syncOwner === "daemon" &&
142
+ !ctx.syncPaused &&
143
+ ctx.hqRoot &&
144
+ ctx.signedIn) {
145
+ ownerBlock = ctx.syncRunnerProcesses === null
146
+ ? "could not check for an existing sync runner; waiting to avoid starting a duplicate"
147
+ : syncRunnerUnleasedBlockReason(ctx.hqRoot, ctx.syncRunnerProcesses, ctx.syncRunnerOwnerPid, ctx.platform, ctx.daemonSyncRunnerPids);
148
+ }
134
149
  unit.blocked = firstBlock([
135
- [ctx.desktopAppManagesSync, "the HQ desktop app is handling sync on this machine"],
150
+ [ctx.syncPaused, "sync is paused (run `hq daemon sync resume`)"],
136
151
  [!ctx.hqRoot, NO_HQ_ROOT],
137
152
  [!ctx.signedIn, NOT_SIGNED_IN],
153
+ [!!ownerBlock, ownerBlock ?? ""],
138
154
  ]);
139
155
  return [unit];
140
156
  }
@@ -12,11 +12,17 @@ export interface DaemonConfig {
12
12
  hqRoot?: string;
13
13
  /** Exact hq-cloud package version preferred by this daemon, including prereleases. */
14
14
  hqCloudVersion?: string;
15
+ /** Persisted machine-local sync pause. */
16
+ syncPaused?: boolean;
17
+ /** Explicit desktop handoff; absent values preserve the legacy per-machine owner. */
18
+ syncOwner?: SyncOwner;
15
19
  /** Services turned on here beyond the machine type's defaults. */
16
20
  enable: string[];
17
21
  /** Default services turned off here. */
18
22
  disable: string[];
19
23
  }
24
+ export type SyncOwner = "daemon" | "desktop-app";
25
+ export declare function effectiveSyncOwner(config: Pick<DaemonConfig, "syncOwner">, machineType: MachineType): SyncOwner;
20
26
  export declare function emptyDaemonConfig(): DaemonConfig;
21
27
  export declare function readDaemonConfig(file: string): DaemonConfig;
22
28
  export declare function writeDaemonConfig(file: string, config: DaemonConfig): void;
@@ -10,8 +10,11 @@ import { valid as validSemver } from "semver";
10
10
  import { isServiceName } from "./catalog.js";
11
11
  import { isMachineType } from "./machine-type.js";
12
12
  import { ensurePrivateDir } from "./paths.js";
13
+ export function effectiveSyncOwner(config, machineType) {
14
+ return config.syncOwner ?? (machineType === "desktop" ? "desktop-app" : "daemon");
15
+ }
13
16
  export function emptyDaemonConfig() {
14
- return { version: 1, enable: [], disable: [] };
17
+ return { version: 1, syncPaused: false, enable: [], disable: [] };
15
18
  }
16
19
  function stringList(value) {
17
20
  return Array.isArray(value) ? value.filter((v) => typeof v === "string") : [];
@@ -36,9 +39,16 @@ export function readDaemonConfig(file) {
36
39
  if (!raw || typeof raw !== "object")
37
40
  throw new Error(`hq daemon config at ${file} is not a JSON object`);
38
41
  const obj = raw;
39
- const config = { version: 1, enable: stringList(obj.enable), disable: stringList(obj.disable) };
42
+ const config = {
43
+ version: 1,
44
+ syncPaused: obj.syncPaused === true,
45
+ enable: stringList(obj.enable),
46
+ disable: stringList(obj.disable),
47
+ };
40
48
  if (isMachineType(obj.machineType))
41
49
  config.machineType = obj.machineType;
50
+ if (obj.syncOwner === "daemon" || obj.syncOwner === "desktop-app")
51
+ config.syncOwner = obj.syncOwner;
42
52
  if (typeof obj.hqRoot === "string" && obj.hqRoot.trim())
43
53
  config.hqRoot = obj.hqRoot;
44
54
  if (typeof obj.hqCloudVersion === "string" && validSemver(obj.hqCloudVersion) === obj.hqCloudVersion) {
@@ -56,6 +66,9 @@ export function writeDaemonConfig(file, config) {
56
66
  ordered.hqRoot = config.hqRoot;
57
67
  if (config.hqCloudVersion)
58
68
  ordered.hqCloudVersion = config.hqCloudVersion;
69
+ ordered.syncPaused = config.syncPaused === true;
70
+ if (config.syncOwner)
71
+ ordered.syncOwner = config.syncOwner;
59
72
  ordered.enable = config.enable;
60
73
  ordered.disable = config.disable;
61
74
  fs.writeFileSync(tmp, `${JSON.stringify(ordered, null, 2)}\n`, { mode: 0o600 });
@@ -15,6 +15,10 @@ export type ControlRequest = {
15
15
  action: "stop";
16
16
  unit: string;
17
17
  }
18
+ /** Re-read persisted configuration and apply any changed blocks immediately. */
19
+ | {
20
+ action: "reconcile";
21
+ }
18
22
  /** Restart the whole daemon, for example after the updater installed a new CLI. */
19
23
  | {
20
24
  action: "restart-daemon";
@@ -19,6 +19,8 @@ function isControlRequest(value) {
19
19
  return typeof v.unit === "string";
20
20
  if (v.action === "restart-daemon")
21
21
  return true;
22
+ if (v.action === "reconcile")
23
+ return true;
22
24
  return false;
23
25
  }
24
26
  // Orders requests written by one process within the same millisecond; the
@@ -17,7 +17,7 @@ import { agentKitPaths } from "../agent-kit/paths.js";
17
17
  import { readBotConfig } from "../bot/config.js";
18
18
  import { botsRoot } from "../bot/paths.js";
19
19
  import { CLAUDE_RUNNER_SCRIPT, CODEX_RUNNER_SCRIPT, enabledServices, planUnits } from "./catalog.js";
20
- import { readDaemonConfig } from "./config.js";
20
+ import { effectiveSyncOwner, readDaemonConfig } from "./config.js";
21
21
  import { SUPERVISOR_HANDOFF_FLAG, readDaemonFlag } from "./flag.js";
22
22
  import { findLegacyUnits, legacyOwnerLookup } from "./legacy.js";
23
23
  import { readLegacySyncEnv } from "./legacy-sync-env.js";
@@ -29,6 +29,7 @@ import { takeControlRequests } from "./requests.js";
29
29
  import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
30
30
  import { runnerSessionName } from "./remote-control.js";
31
31
  import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
32
+ import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
32
33
  import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
33
34
  import { userToolDirs } from "./tools.js";
34
35
  export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
@@ -218,7 +219,10 @@ function makeLogger(paths, foreground) {
218
219
  * an on-failure unit down. The daemon's updater handles CLI updates.
219
220
  */
220
221
  export function unitProcessEnv(base, unit) {
221
- return { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
222
+ const env = { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
223
+ if (unit.service === "sync")
224
+ env.NODE_OPTIONS = syncRunnerNodeOptions(env);
225
+ return env;
222
226
  }
223
227
  export async function runDaemon(opts) {
224
228
  const env = opts.env ?? process.env;
@@ -289,10 +293,11 @@ export async function runDaemon(opts) {
289
293
  return legacyCache.units;
290
294
  };
291
295
  let warnedMissingPin = null;
296
+ const daemonSyncRunnerPids = new Set();
292
297
  const plan = () => {
293
298
  const config = readDaemonConfig(paths.configPath);
294
299
  const legacy = legacyUnits();
295
- const configured = config.hqRoot ?? env.HQ_ROOT?.trim() ?? undefined;
300
+ const configured = config.hqRoot ?? (env.HQ_ROOT?.trim() || undefined);
296
301
  const hqRoot = configured && fs.existsSync(configured) ? configured : undefined;
297
302
  // Re-picked every reconcile, so a newer hq-cloud from the updater is used as soon as it lands.
298
303
  const hqCloud = currentHqCloud(paths.root);
@@ -304,7 +309,33 @@ export async function runDaemon(opts) {
304
309
  }
305
310
  }
306
311
  const runners = hqCloud ? hqCloudRunners(hqCloud.root) : { syncRunner: null, heartbeat: null, sessionHost: null };
307
- return planUnits(enabledServices(machineType, config), {
312
+ const services = enabledServices(machineType, config);
313
+ const isSignedIn = signedIn(env, home, log);
314
+ const syncOwner = effectiveSyncOwner(config, machineType);
315
+ let syncRunnerProcesses = [];
316
+ let syncRunnerOwnerPid = null;
317
+ if (machineType === "desktop" &&
318
+ syncOwner === "daemon" &&
319
+ services.includes("sync") &&
320
+ !config.syncPaused &&
321
+ hqRoot &&
322
+ isSignedIn) {
323
+ const processList = listSyncRunnerProcesses(platform);
324
+ if (!processList.ok) {
325
+ syncRunnerProcesses = null;
326
+ log("warn", `${processList.error}; sync will wait before taking ownership`);
327
+ }
328
+ else {
329
+ syncRunnerProcesses = processList.processes;
330
+ try {
331
+ syncRunnerOwnerPid = readWatchRunnerOwnerStatus(watchRunnerOwnerStatusPath(hqRoot, home, env))?.pid ?? null;
332
+ }
333
+ catch (error) {
334
+ log("warn", `could not verify the sync watch-owner status: ${error instanceof Error ? error.message : String(error)}`);
335
+ }
336
+ }
337
+ }
338
+ return planUnits(services, {
308
339
  machineType,
309
340
  platform,
310
341
  home,
@@ -313,14 +344,20 @@ export async function runDaemon(opts) {
313
344
  heartbeatRunnerPath: runners.heartbeat,
314
345
  sessionHostRunnerPath: runners.sessionHost,
315
346
  legacySyncEnv: readLegacySyncEnv(platform),
316
- signedIn: signedIn(env, home, log),
347
+ signedIn: isSignedIn,
317
348
  kitConfigured: fs.existsSync(agentKitPaths(home, env).kitConfigPath),
349
+ syncPaused: config.syncPaused === true,
350
+ syncOwner,
318
351
  desktopAppManagesSync: desktopAppManagesSync({
319
352
  machineType,
320
353
  host: opts.host,
321
354
  legacy,
322
355
  appRunning: () => desktopAppRunning(platform),
323
356
  }),
357
+ syncRunnerProcesses,
358
+ syncRunnerOwnerPid,
359
+ daemonSyncRunnerPids,
360
+ syncHeartbeatTimeoutMs: syncHeartbeatTimeoutMs(env),
324
361
  bots: listLocalBots(),
325
362
  legacyOwner: legacyOwnerLookup(legacy),
326
363
  cliSelfUpdatable: updatable,
@@ -342,9 +379,37 @@ export async function runDaemon(opts) {
342
379
  : command.type === "node"
343
380
  ? [opts.nodeBinary, [command.script, ...command.args]]
344
381
  : [command.file, command.args];
345
- const fd = openPrivateAppend(paths.unitLogPath(unit.id));
382
+ const logPath = paths.unitLogPath(unit.id);
383
+ const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
384
+ if (unit.heartbeatTimeoutMs) {
385
+ // The supervisor reads this unit's protocol heartbeats from stdout, so its
386
+ // output goes through the daemon. A piped child cannot outlive the daemon,
387
+ // so it is never handed over: a handoff stops it and the next daemon
388
+ // starts it again.
389
+ const logFd = openPrivateAppend(logPath);
390
+ const logStream = fs.createWriteStream(logPath, { fd: logFd, autoClose: true });
391
+ logStream.on("error", (err) => log("error", `could not write ${unit.id} log ${logPath}: ${err.message}`));
392
+ const child = spawn(file, args, {
393
+ cwd,
394
+ env: unitProcessEnv(env, unit),
395
+ stdio: ["ignore", "pipe", "pipe"],
396
+ windowsHide: true,
397
+ });
398
+ spawned.add(child);
399
+ if (unit.service === "sync" && child.pid !== undefined)
400
+ daemonSyncRunnerPids.add(child.pid);
401
+ child.stdout?.pipe(logStream, { end: false });
402
+ child.stderr?.pipe(logStream, { end: false });
403
+ child.on("close", () => {
404
+ spawned.delete(child);
405
+ if (child.pid !== undefined)
406
+ daemonSyncRunnerPids.delete(child.pid);
407
+ logStream.end();
408
+ });
409
+ return child;
410
+ }
411
+ const fd = openPrivateAppend(logPath);
346
412
  try {
347
- const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
348
413
  const child = spawn(file, args, {
349
414
  cwd,
350
415
  env: unitProcessEnv(env, unit),
@@ -352,7 +417,13 @@ export async function runDaemon(opts) {
352
417
  windowsHide: true,
353
418
  });
354
419
  spawned.add(child);
355
- child.once("exit", () => spawned.delete(child));
420
+ if (unit.service === "sync" && child.pid !== undefined)
421
+ daemonSyncRunnerPids.add(child.pid);
422
+ child.once("exit", () => {
423
+ spawned.delete(child);
424
+ if (child.pid !== undefined)
425
+ daemonSyncRunnerPids.delete(child.pid);
426
+ });
356
427
  // Only long-running services are handed over; a scheduled run left behind finishes on its own.
357
428
  if (unit.kind === "long-running")
358
429
  tracker?.track(unit.id, child);
@@ -382,6 +453,21 @@ export async function runDaemon(opts) {
382
453
  restartDaemon: () => restartDaemon(),
383
454
  writeState: (state) => writeJsonAtomic(paths.statePath, state),
384
455
  log,
456
+ syncOwnerBlockReason: () => {
457
+ const config = readDaemonConfig(paths.configPath);
458
+ const hqRoot = config.hqRoot ?? env.HQ_ROOT?.trim();
459
+ if (!hqRoot)
460
+ return undefined;
461
+ const statusPath = watchRunnerOwnerStatusPath(hqRoot, home, env);
462
+ try {
463
+ const owner = readWatchRunnerOwnerStatus(statusPath);
464
+ return owner ? `sync owned by ${owner.owner} pid ${owner.pid}` : undefined;
465
+ }
466
+ catch (err) {
467
+ log("error", `could not read sync owner status at ${statusPath}: ${err instanceof Error ? err.message : String(err)}`);
468
+ return undefined;
469
+ }
470
+ },
385
471
  cliVersion: CLI_VERSION,
386
472
  adopt,
387
473
  });
@@ -24,6 +24,7 @@
24
24
  import type { UnitPlan } from "./catalog.js";
25
25
  import type { MachineType } from "./machine-type.js";
26
26
  import type { ControlRequest } from "./requests.js";
27
+ import { syncPassResult } from "./sync-runner.js";
27
28
  export declare const BACKOFF_BASE_MS = 1000;
28
29
  export declare const BACKOFF_MAX_MS = 60000;
29
30
  export declare const HEALTHY_RUN_MS = 60000;
@@ -34,6 +35,9 @@ export declare const HANDOFF_STAGGER_MS = 15000;
34
35
  export declare const HANDOFF_DRAIN_MAX_MS: number;
35
36
  export interface SupervisedChild {
36
37
  pid?: number;
38
+ stdout?: {
39
+ on(event: "data" | "end", handler: (chunk?: Buffer | string) => void): unknown;
40
+ };
37
41
  on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
38
42
  on(event: "error", handler: (err: Error) => void): unknown;
39
43
  kill(signal?: NodeJS.Signals | number): boolean;
@@ -57,6 +61,8 @@ export interface UnitState {
57
61
  lastRunAt?: number;
58
62
  nextRunAt?: number;
59
63
  lastResult?: "ok" | "failed";
64
+ lastHeartbeatAt?: string;
65
+ lastPassResult?: ReturnType<typeof syncPassResult>;
60
66
  }
61
67
  /** An app that runs the daemon as its own child process instead of an OS unit. */
62
68
  export type HostApp = "desktop";
@@ -99,6 +105,8 @@ export interface SupervisorDeps {
99
105
  clearTimeout?: (handle: unknown) => void;
100
106
  reconcileMs?: number;
101
107
  requestPollMs?: number;
108
+ /** Read hq-cloud's lease status after it exits with the lease-conflict code. */
109
+ syncOwnerBlockReason?: () => string | undefined;
102
110
  /** The running CLI's version; part of the key of units that run `hq` commands. */
103
111
  cliVersion?: string;
104
112
  /** Services to take over from the previous daemon instead of starting them. */
@@ -21,6 +21,8 @@
21
21
  *
22
22
  * Processes, time, and state output are injected; `run.ts` wires the real ones.
23
23
  */
24
+ import { StringDecoder } from "node:string_decoder";
25
+ import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
24
26
  export const BACKOFF_BASE_MS = 1_000;
25
27
  export const BACKOFF_MAX_MS = 60_000;
26
28
  export const HEALTHY_RUN_MS = 60_000;
@@ -39,7 +41,15 @@ export function backoffMs(attempt) {
39
41
  * `keepAcrossCliUpdates` running.
40
42
  */
41
43
  export function planKey(plan, cliVersion) {
42
- const fields = [plan.kind, plan.restart, plan.intervalSec, plan.initialDelaySec, plan.command, plan.env];
44
+ const fields = [
45
+ plan.kind,
46
+ plan.restart,
47
+ plan.intervalSec,
48
+ plan.initialDelaySec,
49
+ plan.heartbeatTimeoutMs,
50
+ plan.command,
51
+ plan.env,
52
+ ];
43
53
  if (cliVersion !== undefined && plan.command?.type === "hq" && !plan.keepAcrossCliUpdates)
44
54
  fields.push(cliVersion);
45
55
  return JSON.stringify(fields);
@@ -89,6 +99,11 @@ export function createSupervisor(deps) {
89
99
  clearT(slot.timer);
90
100
  slot.timer = undefined;
91
101
  };
102
+ const clearHeartbeatTimer = (slot) => {
103
+ if (slot.heartbeatTimer !== undefined)
104
+ clearT(slot.heartbeatTimer);
105
+ slot.heartbeatTimer = undefined;
106
+ };
92
107
  const setBlocked = (slot) => {
93
108
  const block = slot.plan.blocked;
94
109
  slot.state.status = block.state;
@@ -111,6 +126,7 @@ export function createSupervisor(deps) {
111
126
  */
112
127
  const terminate = (slot, then, drain = false) => {
113
128
  clearTimer(slot);
129
+ clearHeartbeatTimer(slot);
114
130
  const child = slot.child;
115
131
  if (!child) {
116
132
  then?.();
@@ -163,6 +179,69 @@ export function createSupervisor(deps) {
163
179
  const watch = (slot, child) => {
164
180
  live.add(child);
165
181
  let ended = false;
182
+ const timeoutMs = slot.plan.heartbeatTimeoutMs;
183
+ const decoder = timeoutMs ? new StringDecoder("utf8") : null;
184
+ let stdoutRemainder = "";
185
+ let lastHeartbeatPersistedAt = now() - 15_000;
186
+ const armHeartbeatWatchdog = () => {
187
+ if (!timeoutMs)
188
+ return;
189
+ clearHeartbeatTimer(slot);
190
+ slot.heartbeatTimer = setT(() => {
191
+ slot.heartbeatTimer = undefined;
192
+ if (stopped || slot.child !== child || slot.expectingExit)
193
+ return;
194
+ deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
195
+ terminate(slot, () => scheduleRestart(slot));
196
+ emit();
197
+ }, timeoutMs);
198
+ };
199
+ const acceptProtocolLine = (line) => {
200
+ let event;
201
+ try {
202
+ event = parseSyncRunnerProtocolLine(line);
203
+ }
204
+ catch (err) {
205
+ deps.log("error", `${slot.plan.id} emitted invalid sync protocol output (${Buffer.byteLength(line)} bytes): ${errorMessage(err)}`);
206
+ return;
207
+ }
208
+ if (!event)
209
+ return;
210
+ const at = now();
211
+ slot.state.lastHeartbeatAt = new Date(at).toISOString();
212
+ const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
213
+ if (pass)
214
+ slot.state.lastPassResult = pass;
215
+ armHeartbeatWatchdog();
216
+ if (pass || at - lastHeartbeatPersistedAt >= 15_000) {
217
+ lastHeartbeatPersistedAt = at;
218
+ emit();
219
+ }
220
+ };
221
+ if (decoder && child.stdout) {
222
+ child.stdout.on("data", (chunk) => {
223
+ if (chunk === undefined)
224
+ return;
225
+ stdoutRemainder += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
226
+ let newline = stdoutRemainder.indexOf("\n");
227
+ while (newline >= 0) {
228
+ acceptProtocolLine(stdoutRemainder.slice(0, newline).trim());
229
+ stdoutRemainder = stdoutRemainder.slice(newline + 1);
230
+ newline = stdoutRemainder.indexOf("\n");
231
+ }
232
+ // A malformed or unexpectedly large line is not a heartbeat. Bound
233
+ // retained data so a broken child cannot grow daemon memory forever.
234
+ if (Buffer.byteLength(stdoutRemainder) > 1024 * 1024)
235
+ stdoutRemainder = "";
236
+ });
237
+ child.stdout.on("end", () => {
238
+ stdoutRemainder += decoder.end();
239
+ if (stdoutRemainder)
240
+ acceptProtocolLine(stdoutRemainder.trim());
241
+ stdoutRemainder = "";
242
+ });
243
+ }
244
+ armHeartbeatWatchdog();
166
245
  const onEnd = (code, signal) => {
167
246
  if (ended)
168
247
  return;
@@ -173,6 +252,7 @@ export function createSupervisor(deps) {
173
252
  if (slot.child !== child)
174
253
  return;
175
254
  slot.child = undefined;
255
+ clearHeartbeatTimer(slot);
176
256
  if (slot.killTimer !== undefined) {
177
257
  clearT(slot.killTimer);
178
258
  slot.killTimer = undefined;
@@ -200,6 +280,20 @@ export function createSupervisor(deps) {
200
280
  emit();
201
281
  return;
202
282
  }
283
+ if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
284
+ const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
285
+ slot.state.status = "waiting";
286
+ slot.state.reason = reason;
287
+ slot.state.nextRunAt = now() + SYNC_RUNNER_LEASE_RETRY_MS;
288
+ deps.log("info", `${reason}; retrying in ${SYNC_RUNNER_LEASE_RETRY_MS}ms`);
289
+ slot.timer = setT(() => {
290
+ slot.timer = undefined;
291
+ startLong(slot);
292
+ emit();
293
+ }, SYNC_RUNNER_LEASE_RETRY_MS);
294
+ emit();
295
+ return;
296
+ }
203
297
  deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
204
298
  if (ranFor >= HEALTHY_RUN_MS)
205
299
  slot.attempts = 0;
@@ -505,6 +599,11 @@ export function createSupervisor(deps) {
505
599
  }
506
600
  };
507
601
  const handleRequest = (req) => {
602
+ if (req.action === "reconcile") {
603
+ deps.log("info", "reconciling daemon configuration (requested)");
604
+ reconcile();
605
+ return;
606
+ }
508
607
  if (req.action === "restart-daemon") {
509
608
  deps.log("info", "restarting the daemon (requested)");
510
609
  if (deps.restartDaemon)
@@ -614,6 +713,7 @@ export function createSupervisor(deps) {
614
713
  clearT(handoffTimer);
615
714
  for (const slot of slots.values()) {
616
715
  clearTimer(slot);
716
+ clearHeartbeatTimer(slot);
617
717
  slot.child = undefined;
618
718
  if (slot.killTimer !== undefined)
619
719
  clearT(slot.killTimer);