@indigoai-us/hq-cli 5.275.0 → 5.277.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -0
- package/README.md +12 -0
- package/assets/bot-workers/setup/context/USER-GUIDE.md +4 -4
- package/assets/scaffold/core/scripts/hq-delegate-pickup.sh +99 -0
- package/assets/scaffold/core/scripts/hq-delegate-repo.sh +158 -0
- package/assets/scaffold/core/scripts/hq-delegate-resolve.sh +145 -0
- package/assets/scaffold/core/scripts/hq-delegate-secrets.sh +129 -0
- package/assets/scaffold/core/scripts/hq-delegate-transfer.sh +175 -0
- package/assets/scaffold/core/scripts/lib/secret-patterns.sh +58 -0
- package/dist/command-catalog.generated.d.ts +239 -1
- package/dist/command-catalog.generated.js +302 -1
- package/dist/commands/core-checkpoint.d.ts +4 -2
- package/dist/commands/core-checkpoint.js +34 -3
- package/dist/commands/core-command-catalog.d.ts +4 -2
- package/dist/commands/core-command-catalog.js +5 -2
- package/dist/commands/core-delegate.d.ts +6 -0
- package/dist/commands/core-delegate.js +987 -0
- package/dist/commands/core-ontology.js +3 -3
- package/dist/commands/core.js +11 -1
- package/dist/commands/daemon.d.ts +4 -0
- package/dist/commands/daemon.js +175 -2
- package/dist/commands/dm.d.ts +2 -0
- package/dist/commands/dm.js +11 -4
- package/dist/commands/feedback.d.ts +6 -0
- package/dist/commands/feedback.js +86 -37
- package/dist/commands/lanes-accounts.js +44 -2
- package/dist/commands/lanes-message.js +69 -12
- package/dist/commands/lanes-resume.js +2 -0
- package/dist/commands/lanes-shared.d.ts +3 -0
- package/dist/commands/lanes-shared.js +12 -2
- package/dist/commands/lanes.d.ts +5 -1
- package/dist/commands/lanes.js +402 -49
- package/dist/commands/run-feature-telemetry.d.ts +3 -0
- package/dist/commands/run-feature-telemetry.js +6 -0
- package/dist/commands/run.js +3 -2
- package/dist/commands/secrets.d.ts +16 -1
- package/dist/commands/secrets.js +80 -45
- package/dist/commands/sync-mode.d.ts +11 -0
- package/dist/commands/sync-mode.js +30 -0
- package/dist/commands/whoami.d.ts +4 -6
- package/dist/commands/whoami.js +39 -8
- package/dist/fast-core.d.ts +6 -7
- package/dist/fast-core.js +22 -8
- package/dist/fast-scaffold.d.ts +33 -7
- package/dist/fast-scaffold.js +104 -8
- package/dist/index.js +32 -1
- package/dist/lanes-drain-hook.js +154 -7
- package/dist/lib/agent-kit/run/inbox.js +3 -2
- package/dist/lib/billing/plan-lock.d.ts +3 -1
- package/dist/lib/billing/plan-lock.js +7 -1
- package/dist/lib/checkpoint-ontology/apply.d.ts +6 -0
- package/dist/lib/checkpoint-ontology/apply.js +60 -3
- package/dist/lib/checkpoint-ontology/company.d.ts +8 -4
- package/dist/lib/checkpoint-ontology/company.js +15 -7
- package/dist/lib/checkpoint-ontology/plan.d.ts +36 -0
- package/dist/lib/checkpoint-ontology/plan.js +66 -4
- package/dist/lib/checkpoint-ontology/queued-touches.d.ts +32 -0
- package/dist/lib/checkpoint-ontology/queued-touches.js +193 -0
- package/dist/lib/checkpoint-ontology/tree.d.ts +2 -0
- package/dist/lib/checkpoint-ontology/tree.js +34 -0
- package/dist/lib/core-utils/jq-json-serializer.d.ts +5 -0
- package/dist/lib/core-utils/jq-json-serializer.js +10 -0
- package/dist/lib/core-utils/secret-patterns.d.ts +41 -0
- package/dist/lib/core-utils/secret-patterns.js +33 -0
- package/dist/lib/core-utils/timeout-guard-native.d.ts +7 -0
- package/dist/lib/core-utils/timeout-guard-native.js +40 -0
- package/dist/lib/core-utils/timeout-guard.d.ts +3 -3
- package/dist/lib/core-utils/timeout-guard.js +28 -13
- package/dist/lib/daemon/catalog.d.ts +18 -2
- package/dist/lib/daemon/catalog.js +18 -2
- package/dist/lib/daemon/config.d.ts +6 -0
- package/dist/lib/daemon/config.js +15 -2
- package/dist/lib/daemon/requests.d.ts +4 -0
- package/dist/lib/daemon/requests.js +2 -0
- package/dist/lib/daemon/run.js +94 -8
- package/dist/lib/daemon/supervisor.d.ts +8 -0
- package/dist/lib/daemon/supervisor.js +101 -1
- package/dist/lib/daemon/sync-runner.d.ts +58 -0
- package/dist/lib/daemon/sync-runner.js +277 -0
- package/dist/lib/doctor/__testing__/run-doctor.d.ts +8 -0
- package/dist/lib/doctor/__testing__/run-doctor.js +52 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/flag-registry.d.ts +12 -7
- package/dist/lib/flag-registry.js +66 -75
- package/dist/lib/flag-snapshot-file.d.ts +20 -0
- package/dist/lib/flag-snapshot-file.js +91 -0
- package/dist/lib/lanes/claude-balance.d.ts +6 -1
- package/dist/lib/lanes/claude-balance.js +46 -8
- package/dist/lib/lanes/config.js +15 -0
- package/dist/lib/lanes/hire-grant-dispatch.d.ts +2 -0
- package/dist/lib/lanes/hire-grant-dispatch.js +41 -0
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/keep-alive.d.ts +3 -0
- package/dist/lib/lanes/keep-alive.js +3 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +4 -0
- package/dist/lib/lanes/outpost-dispatch.js +11 -2
- package/dist/lib/lanes/reconcile.js +20 -4
- package/dist/lib/lanes/spawn.js +17 -10
- package/dist/lib/lanes/store.js +45 -1
- package/dist/lib/lanes/types.d.ts +30 -1
- package/dist/lib/lanes/types.js +2 -2
- package/dist/lib/lanes/watcher.js +3 -2
- package/dist/lib/process-flag-company-context.d.ts +8 -0
- package/dist/lib/process-flag-company-context.js +67 -0
- package/dist/main.js +3 -1
- package/dist/native-fast-route.d.ts +3 -0
- package/dist/native-fast-route.js +39 -0
- package/dist/secrets-env-fast-path.d.ts +5 -0
- package/dist/secrets-env-fast-path.js +16 -0
- package/dist/secrets-env-fast-route.d.ts +18 -0
- package/dist/secrets-env-fast-route.js +54 -0
- package/dist/utils/cli-telemetry.d.ts +26 -2
- package/dist/utils/cli-telemetry.js +178 -3
- package/dist/utils/hq-root-resolver.d.ts +13 -0
- package/dist/utils/hq-root-resolver.js +103 -0
- package/dist/utils/scaffold-fast-import-guard.js +6 -1
- package/dist/utils/vault-api.d.ts +8 -0
- package/dist/utils/vault-api.js +13 -1
- package/package.json +2 -2
|
@@ -20,9 +20,6 @@
|
|
|
20
20
|
*
|
|
21
21
|
* Exit codes: 0 = allow, 2 = block.
|
|
22
22
|
*/
|
|
23
|
-
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
24
|
-
import { peekIdToken } from "../../utils/id-token.js";
|
|
25
|
-
import { resolveFlagGate, resolveProcessFlagGate, } from "../flag-registry.js";
|
|
26
23
|
// Observed Claude Code Bash-tool bounds. The tool `timeout` (ms) raises the
|
|
27
24
|
// deadline up to the max; with none declared the default applies.
|
|
28
25
|
const HARNESS_DEFAULT_MS = 120_000; // 2 min
|
|
@@ -153,8 +150,14 @@ supervise yourself.)
|
|
|
153
150
|
|
|
154
151
|
Policy: hq-foreground-timeout-killed-by-harness-deadline.
|
|
155
152
|
Bypass for one sanctioned foreground run: prefix HQ_ALLOW_LONG_FOREGROUND=1.`;
|
|
156
|
-
function defaultEmail() {
|
|
153
|
+
async function defaultEmail() {
|
|
157
154
|
try {
|
|
155
|
+
// The cached identity is needed only for commands that would otherwise
|
|
156
|
+
// block. Keep hq-cloud and the JWT decoder out of the common allow path.
|
|
157
|
+
const [{ loadCachedTokens }, { peekIdToken }] = await Promise.all([
|
|
158
|
+
import("@indigoai-us/hq-cloud"),
|
|
159
|
+
import("../../utils/id-token.js"),
|
|
160
|
+
]);
|
|
158
161
|
const cached = loadCachedTokens();
|
|
159
162
|
if (!cached)
|
|
160
163
|
return undefined;
|
|
@@ -166,23 +169,35 @@ function defaultEmail() {
|
|
|
166
169
|
}
|
|
167
170
|
}
|
|
168
171
|
/** Whether the guard is active for this identity (rollout gate). */
|
|
169
|
-
export function isGatedUser(email, flagReader) {
|
|
172
|
+
export async function isGatedUser(email, flagReader) {
|
|
170
173
|
const legacyFallback = () => typeof email === "string" && email.toLowerCase().endsWith(GATE_DOMAIN);
|
|
171
174
|
const lookup = {
|
|
172
175
|
defaultEmailDomains: ["getindigo.ai"],
|
|
173
176
|
personEmail: email,
|
|
174
177
|
fallback: legacyFallback(),
|
|
175
178
|
};
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
+
if (flagReader) {
|
|
180
|
+
try {
|
|
181
|
+
return flagReader.isEnabled("cli.foreground-timeout-guard", lookup);
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
return legacyFallback();
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
const { resolveProcessFlagGate } = await import("../flag-registry.js");
|
|
189
|
+
return resolveProcessFlagGate("cli.foreground-timeout-guard", lookup, legacyFallback);
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return legacyFallback();
|
|
193
|
+
}
|
|
179
194
|
}
|
|
180
195
|
/**
|
|
181
196
|
* Command entry: read the PreToolUse JSON, apply the gate + short-circuits, and
|
|
182
197
|
* return the exit code (0 allow / 2 block). Never throws — a parse failure or a
|
|
183
198
|
* missing identity fails OPEN (allow), because a hook must not break tool calls.
|
|
184
199
|
*/
|
|
185
|
-
export function timeoutGuardCommand(stdin, deps = {}) {
|
|
200
|
+
export async function timeoutGuardCommand(stdin, deps = {}) {
|
|
186
201
|
const env = deps.env ?? process.env;
|
|
187
202
|
const err = deps.stderr ?? process.stderr;
|
|
188
203
|
let payload;
|
|
@@ -201,15 +216,15 @@ export function timeoutGuardCommand(stdin, deps = {}) {
|
|
|
201
216
|
return 0;
|
|
202
217
|
if (/(^|\s)HQ_ALLOW_LONG_FOREGROUND=1(\s|$)/.test(command))
|
|
203
218
|
return 0;
|
|
204
|
-
// Rollout gate: only act for @getindigo.ai identities.
|
|
205
|
-
const getEmail = deps.getEmail ?? defaultEmail;
|
|
206
|
-
if (!isGatedUser(getEmail(), deps.flagReader))
|
|
207
|
-
return 0;
|
|
208
219
|
const runInBackground = ti.run_in_background === true;
|
|
209
220
|
const toolTimeoutMs = typeof ti.timeout === "number" ? ti.timeout : undefined;
|
|
210
221
|
const verdict = evaluateTimeoutGuard({ command, toolTimeoutMs, runInBackground });
|
|
211
222
|
if (!verdict.block)
|
|
212
223
|
return 0;
|
|
224
|
+
// Rollout gate: only act for @getindigo.ai identities.
|
|
225
|
+
const getEmail = deps.getEmail ?? defaultEmail;
|
|
226
|
+
if (!await isGatedUser(await getEmail(), deps.flagReader))
|
|
227
|
+
return 0;
|
|
213
228
|
err.write(`${BLOCK_MESSAGE(verdict.reason ?? "declared timeout exceeds the harness ceiling")}\n`);
|
|
214
229
|
return 2;
|
|
215
230
|
}
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
* them instead of silently dropping them.
|
|
11
11
|
*/
|
|
12
12
|
import type { MachineType } from "./machine-type.js";
|
|
13
|
+
import type { SyncOwner } from "./config.js";
|
|
14
|
+
import { type SyncRunnerProcess } from "./sync-runner.js";
|
|
13
15
|
export declare const SERVICE_NAMES: readonly ["sync", "mesh", "heartbeat", "inbox", "bots", "remote-control", "codex-remote-control", "session-host", "lanes-reconcile", "search-index", "jobs-reconcile", "updater", "housekeeping"];
|
|
14
16
|
export type ServiceName = (typeof SERVICE_NAMES)[number];
|
|
15
17
|
export declare function isServiceName(value: unknown): value is ServiceName;
|
|
@@ -51,6 +53,8 @@ export interface UnitPlan {
|
|
|
51
53
|
restart: "always" | "on-failure";
|
|
52
54
|
intervalSec?: number;
|
|
53
55
|
initialDelaySec?: number;
|
|
56
|
+
/** Kill and restart this child after silence on its liveness stream. */
|
|
57
|
+
heartbeatTimeoutMs?: number;
|
|
54
58
|
command: UnitCommand | null;
|
|
55
59
|
env: Record<string, string>;
|
|
56
60
|
blocked?: UnitBlock;
|
|
@@ -75,11 +79,23 @@ export interface PlanContext {
|
|
|
75
79
|
sessionHostRunnerPath: string | null;
|
|
76
80
|
/** Environment retained from legacy outpost sync systemd drop-ins. */
|
|
77
81
|
legacySyncEnv: Record<string, string>;
|
|
82
|
+
/** Keep the local sync runner stopped until the operator resumes it. */
|
|
83
|
+
syncPaused: boolean;
|
|
84
|
+
/** Persisted/effective owner selected for this machine. */
|
|
85
|
+
syncOwner: SyncOwner;
|
|
86
|
+
/** A legacy desktop app or sync unit is still managing sync. */
|
|
87
|
+
desktopAppManagesSync: boolean;
|
|
88
|
+
/** Full process list, or null when the daemon could not read it. */
|
|
89
|
+
syncRunnerProcesses: readonly SyncRunnerProcess[] | null;
|
|
90
|
+
/** PID recorded in the watch-owner status, when it is readable. */
|
|
91
|
+
syncRunnerOwnerPid: number | null;
|
|
92
|
+
/** Sync runner children already started by this daemon before their lease is written. */
|
|
93
|
+
daemonSyncRunnerPids?: ReadonlySet<number>;
|
|
94
|
+
/** The protocol-event heartbeat watchdog timeout for the sync runner. */
|
|
95
|
+
syncHeartbeatTimeoutMs: number;
|
|
78
96
|
signedIn: boolean;
|
|
79
97
|
/** ~/.hq-agent/kit.json exists. */
|
|
80
98
|
kitConfigured: boolean;
|
|
81
|
-
/** The HQ desktop app runs its own sync on this machine. */
|
|
82
|
-
desktopAppManagesSync: boolean;
|
|
83
99
|
bots: readonly string[];
|
|
84
100
|
/** Name of a pre-daemon unit that still runs this unit id, or null. */
|
|
85
101
|
legacyOwner: (unitId: string) => string | null;
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* with no entrypoint yet stay in the catalog as `unavailable` so status shows
|
|
10
10
|
* them instead of silently dropping them.
|
|
11
11
|
*/
|
|
12
|
+
import { SYNC_RUNNER_OWNER, syncRunnerUnleasedBlockReason } from "./sync-runner.js";
|
|
12
13
|
export const SERVICE_NAMES = [
|
|
13
14
|
"sync",
|
|
14
15
|
"mesh",
|
|
@@ -127,14 +128,29 @@ function planService(service, ctx) {
|
|
|
127
128
|
command: {
|
|
128
129
|
type: "node",
|
|
129
130
|
script: ctx.syncRunnerPath,
|
|
130
|
-
args: ["--companies", "--direction", "both", "--on-conflict", "keep", "--hq-root", ctx.hqRoot ?? "", "--watch", "--event-push"],
|
|
131
|
+
args: ["--companies", "--direction", "both", "--on-conflict", "keep", "--hq-root", ctx.hqRoot ?? "", "--watch", "--event-push", "--owner", SYNC_RUNNER_OWNER],
|
|
131
132
|
},
|
|
132
133
|
env: { ...ctx.legacySyncEnv },
|
|
134
|
+
heartbeatTimeoutMs: ctx.syncHeartbeatTimeoutMs,
|
|
133
135
|
});
|
|
136
|
+
let ownerBlock;
|
|
137
|
+
if (ctx.machineType === "desktop" && ctx.syncOwner === "desktop-app" && ctx.desktopAppManagesSync) {
|
|
138
|
+
ownerBlock = "the desktop app is still managing sync (run `hq daemon sync take-ownership` after it stops its runner)";
|
|
139
|
+
}
|
|
140
|
+
else if (ctx.machineType === "desktop" &&
|
|
141
|
+
ctx.syncOwner === "daemon" &&
|
|
142
|
+
!ctx.syncPaused &&
|
|
143
|
+
ctx.hqRoot &&
|
|
144
|
+
ctx.signedIn) {
|
|
145
|
+
ownerBlock = ctx.syncRunnerProcesses === null
|
|
146
|
+
? "could not check for an existing sync runner; waiting to avoid starting a duplicate"
|
|
147
|
+
: syncRunnerUnleasedBlockReason(ctx.hqRoot, ctx.syncRunnerProcesses, ctx.syncRunnerOwnerPid, ctx.platform, ctx.daemonSyncRunnerPids);
|
|
148
|
+
}
|
|
134
149
|
unit.blocked = firstBlock([
|
|
135
|
-
[ctx.
|
|
150
|
+
[ctx.syncPaused, "sync is paused (run `hq daemon sync resume`)"],
|
|
136
151
|
[!ctx.hqRoot, NO_HQ_ROOT],
|
|
137
152
|
[!ctx.signedIn, NOT_SIGNED_IN],
|
|
153
|
+
[!!ownerBlock, ownerBlock ?? ""],
|
|
138
154
|
]);
|
|
139
155
|
return [unit];
|
|
140
156
|
}
|
|
@@ -12,11 +12,17 @@ export interface DaemonConfig {
|
|
|
12
12
|
hqRoot?: string;
|
|
13
13
|
/** Exact hq-cloud package version preferred by this daemon, including prereleases. */
|
|
14
14
|
hqCloudVersion?: string;
|
|
15
|
+
/** Persisted machine-local sync pause. */
|
|
16
|
+
syncPaused?: boolean;
|
|
17
|
+
/** Explicit desktop handoff; absent values preserve the legacy per-machine owner. */
|
|
18
|
+
syncOwner?: SyncOwner;
|
|
15
19
|
/** Services turned on here beyond the machine type's defaults. */
|
|
16
20
|
enable: string[];
|
|
17
21
|
/** Default services turned off here. */
|
|
18
22
|
disable: string[];
|
|
19
23
|
}
|
|
24
|
+
export type SyncOwner = "daemon" | "desktop-app";
|
|
25
|
+
export declare function effectiveSyncOwner(config: Pick<DaemonConfig, "syncOwner">, machineType: MachineType): SyncOwner;
|
|
20
26
|
export declare function emptyDaemonConfig(): DaemonConfig;
|
|
21
27
|
export declare function readDaemonConfig(file: string): DaemonConfig;
|
|
22
28
|
export declare function writeDaemonConfig(file: string, config: DaemonConfig): void;
|
|
@@ -10,8 +10,11 @@ import { valid as validSemver } from "semver";
|
|
|
10
10
|
import { isServiceName } from "./catalog.js";
|
|
11
11
|
import { isMachineType } from "./machine-type.js";
|
|
12
12
|
import { ensurePrivateDir } from "./paths.js";
|
|
13
|
+
export function effectiveSyncOwner(config, machineType) {
|
|
14
|
+
return config.syncOwner ?? (machineType === "desktop" ? "desktop-app" : "daemon");
|
|
15
|
+
}
|
|
13
16
|
export function emptyDaemonConfig() {
|
|
14
|
-
return { version: 1, enable: [], disable: [] };
|
|
17
|
+
return { version: 1, syncPaused: false, enable: [], disable: [] };
|
|
15
18
|
}
|
|
16
19
|
function stringList(value) {
|
|
17
20
|
return Array.isArray(value) ? value.filter((v) => typeof v === "string") : [];
|
|
@@ -36,9 +39,16 @@ export function readDaemonConfig(file) {
|
|
|
36
39
|
if (!raw || typeof raw !== "object")
|
|
37
40
|
throw new Error(`hq daemon config at ${file} is not a JSON object`);
|
|
38
41
|
const obj = raw;
|
|
39
|
-
const config = {
|
|
42
|
+
const config = {
|
|
43
|
+
version: 1,
|
|
44
|
+
syncPaused: obj.syncPaused === true,
|
|
45
|
+
enable: stringList(obj.enable),
|
|
46
|
+
disable: stringList(obj.disable),
|
|
47
|
+
};
|
|
40
48
|
if (isMachineType(obj.machineType))
|
|
41
49
|
config.machineType = obj.machineType;
|
|
50
|
+
if (obj.syncOwner === "daemon" || obj.syncOwner === "desktop-app")
|
|
51
|
+
config.syncOwner = obj.syncOwner;
|
|
42
52
|
if (typeof obj.hqRoot === "string" && obj.hqRoot.trim())
|
|
43
53
|
config.hqRoot = obj.hqRoot;
|
|
44
54
|
if (typeof obj.hqCloudVersion === "string" && validSemver(obj.hqCloudVersion) === obj.hqCloudVersion) {
|
|
@@ -56,6 +66,9 @@ export function writeDaemonConfig(file, config) {
|
|
|
56
66
|
ordered.hqRoot = config.hqRoot;
|
|
57
67
|
if (config.hqCloudVersion)
|
|
58
68
|
ordered.hqCloudVersion = config.hqCloudVersion;
|
|
69
|
+
ordered.syncPaused = config.syncPaused === true;
|
|
70
|
+
if (config.syncOwner)
|
|
71
|
+
ordered.syncOwner = config.syncOwner;
|
|
59
72
|
ordered.enable = config.enable;
|
|
60
73
|
ordered.disable = config.disable;
|
|
61
74
|
fs.writeFileSync(tmp, `${JSON.stringify(ordered, null, 2)}\n`, { mode: 0o600 });
|
|
@@ -15,6 +15,10 @@ export type ControlRequest = {
|
|
|
15
15
|
action: "stop";
|
|
16
16
|
unit: string;
|
|
17
17
|
}
|
|
18
|
+
/** Re-read persisted configuration and apply any changed blocks immediately. */
|
|
19
|
+
| {
|
|
20
|
+
action: "reconcile";
|
|
21
|
+
}
|
|
18
22
|
/** Restart the whole daemon, for example after the updater installed a new CLI. */
|
|
19
23
|
| {
|
|
20
24
|
action: "restart-daemon";
|
|
@@ -19,6 +19,8 @@ function isControlRequest(value) {
|
|
|
19
19
|
return typeof v.unit === "string";
|
|
20
20
|
if (v.action === "restart-daemon")
|
|
21
21
|
return true;
|
|
22
|
+
if (v.action === "reconcile")
|
|
23
|
+
return true;
|
|
22
24
|
return false;
|
|
23
25
|
}
|
|
24
26
|
// Orders requests written by one process within the same millisecond; the
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -17,7 +17,7 @@ import { agentKitPaths } from "../agent-kit/paths.js";
|
|
|
17
17
|
import { readBotConfig } from "../bot/config.js";
|
|
18
18
|
import { botsRoot } from "../bot/paths.js";
|
|
19
19
|
import { CLAUDE_RUNNER_SCRIPT, CODEX_RUNNER_SCRIPT, enabledServices, planUnits } from "./catalog.js";
|
|
20
|
-
import { readDaemonConfig } from "./config.js";
|
|
20
|
+
import { effectiveSyncOwner, readDaemonConfig } from "./config.js";
|
|
21
21
|
import { SUPERVISOR_HANDOFF_FLAG, readDaemonFlag } from "./flag.js";
|
|
22
22
|
import { findLegacyUnits, legacyOwnerLookup } from "./legacy.js";
|
|
23
23
|
import { readLegacySyncEnv } from "./legacy-sync-env.js";
|
|
@@ -29,6 +29,7 @@ import { takeControlRequests } from "./requests.js";
|
|
|
29
29
|
import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
|
|
30
30
|
import { runnerSessionName } from "./remote-control.js";
|
|
31
31
|
import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
|
|
32
|
+
import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
|
|
32
33
|
import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
|
|
33
34
|
import { userToolDirs } from "./tools.js";
|
|
34
35
|
export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
|
|
@@ -218,7 +219,10 @@ function makeLogger(paths, foreground) {
|
|
|
218
219
|
* an on-failure unit down. The daemon's updater handles CLI updates.
|
|
219
220
|
*/
|
|
220
221
|
export function unitProcessEnv(base, unit) {
|
|
221
|
-
|
|
222
|
+
const env = { ...base, ...unit.env, HQ_DAEMON_UNIT: unit.id, HQ_NO_UPDATE_CHECK: "1" };
|
|
223
|
+
if (unit.service === "sync")
|
|
224
|
+
env.NODE_OPTIONS = syncRunnerNodeOptions(env);
|
|
225
|
+
return env;
|
|
222
226
|
}
|
|
223
227
|
export async function runDaemon(opts) {
|
|
224
228
|
const env = opts.env ?? process.env;
|
|
@@ -289,10 +293,11 @@ export async function runDaemon(opts) {
|
|
|
289
293
|
return legacyCache.units;
|
|
290
294
|
};
|
|
291
295
|
let warnedMissingPin = null;
|
|
296
|
+
const daemonSyncRunnerPids = new Set();
|
|
292
297
|
const plan = () => {
|
|
293
298
|
const config = readDaemonConfig(paths.configPath);
|
|
294
299
|
const legacy = legacyUnits();
|
|
295
|
-
const configured = config.hqRoot ?? env.HQ_ROOT?.trim()
|
|
300
|
+
const configured = config.hqRoot ?? (env.HQ_ROOT?.trim() || undefined);
|
|
296
301
|
const hqRoot = configured && fs.existsSync(configured) ? configured : undefined;
|
|
297
302
|
// Re-picked every reconcile, so a newer hq-cloud from the updater is used as soon as it lands.
|
|
298
303
|
const hqCloud = currentHqCloud(paths.root);
|
|
@@ -304,7 +309,33 @@ export async function runDaemon(opts) {
|
|
|
304
309
|
}
|
|
305
310
|
}
|
|
306
311
|
const runners = hqCloud ? hqCloudRunners(hqCloud.root) : { syncRunner: null, heartbeat: null, sessionHost: null };
|
|
307
|
-
|
|
312
|
+
const services = enabledServices(machineType, config);
|
|
313
|
+
const isSignedIn = signedIn(env, home, log);
|
|
314
|
+
const syncOwner = effectiveSyncOwner(config, machineType);
|
|
315
|
+
let syncRunnerProcesses = [];
|
|
316
|
+
let syncRunnerOwnerPid = null;
|
|
317
|
+
if (machineType === "desktop" &&
|
|
318
|
+
syncOwner === "daemon" &&
|
|
319
|
+
services.includes("sync") &&
|
|
320
|
+
!config.syncPaused &&
|
|
321
|
+
hqRoot &&
|
|
322
|
+
isSignedIn) {
|
|
323
|
+
const processList = listSyncRunnerProcesses(platform);
|
|
324
|
+
if (!processList.ok) {
|
|
325
|
+
syncRunnerProcesses = null;
|
|
326
|
+
log("warn", `${processList.error}; sync will wait before taking ownership`);
|
|
327
|
+
}
|
|
328
|
+
else {
|
|
329
|
+
syncRunnerProcesses = processList.processes;
|
|
330
|
+
try {
|
|
331
|
+
syncRunnerOwnerPid = readWatchRunnerOwnerStatus(watchRunnerOwnerStatusPath(hqRoot, home, env))?.pid ?? null;
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
log("warn", `could not verify the sync watch-owner status: ${error instanceof Error ? error.message : String(error)}`);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
return planUnits(services, {
|
|
308
339
|
machineType,
|
|
309
340
|
platform,
|
|
310
341
|
home,
|
|
@@ -313,14 +344,20 @@ export async function runDaemon(opts) {
|
|
|
313
344
|
heartbeatRunnerPath: runners.heartbeat,
|
|
314
345
|
sessionHostRunnerPath: runners.sessionHost,
|
|
315
346
|
legacySyncEnv: readLegacySyncEnv(platform),
|
|
316
|
-
signedIn:
|
|
347
|
+
signedIn: isSignedIn,
|
|
317
348
|
kitConfigured: fs.existsSync(agentKitPaths(home, env).kitConfigPath),
|
|
349
|
+
syncPaused: config.syncPaused === true,
|
|
350
|
+
syncOwner,
|
|
318
351
|
desktopAppManagesSync: desktopAppManagesSync({
|
|
319
352
|
machineType,
|
|
320
353
|
host: opts.host,
|
|
321
354
|
legacy,
|
|
322
355
|
appRunning: () => desktopAppRunning(platform),
|
|
323
356
|
}),
|
|
357
|
+
syncRunnerProcesses,
|
|
358
|
+
syncRunnerOwnerPid,
|
|
359
|
+
daemonSyncRunnerPids,
|
|
360
|
+
syncHeartbeatTimeoutMs: syncHeartbeatTimeoutMs(env),
|
|
324
361
|
bots: listLocalBots(),
|
|
325
362
|
legacyOwner: legacyOwnerLookup(legacy),
|
|
326
363
|
cliSelfUpdatable: updatable,
|
|
@@ -342,9 +379,37 @@ export async function runDaemon(opts) {
|
|
|
342
379
|
: command.type === "node"
|
|
343
380
|
? [opts.nodeBinary, [command.script, ...command.args]]
|
|
344
381
|
: [command.file, command.args];
|
|
345
|
-
const
|
|
382
|
+
const logPath = paths.unitLogPath(unit.id);
|
|
383
|
+
const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
|
|
384
|
+
if (unit.heartbeatTimeoutMs) {
|
|
385
|
+
// The supervisor reads this unit's protocol heartbeats from stdout, so its
|
|
386
|
+
// output goes through the daemon. A piped child cannot outlive the daemon,
|
|
387
|
+
// so it is never handed over: a handoff stops it and the next daemon
|
|
388
|
+
// starts it again.
|
|
389
|
+
const logFd = openPrivateAppend(logPath);
|
|
390
|
+
const logStream = fs.createWriteStream(logPath, { fd: logFd, autoClose: true });
|
|
391
|
+
logStream.on("error", (err) => log("error", `could not write ${unit.id} log ${logPath}: ${err.message}`));
|
|
392
|
+
const child = spawn(file, args, {
|
|
393
|
+
cwd,
|
|
394
|
+
env: unitProcessEnv(env, unit),
|
|
395
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
396
|
+
windowsHide: true,
|
|
397
|
+
});
|
|
398
|
+
spawned.add(child);
|
|
399
|
+
if (unit.service === "sync" && child.pid !== undefined)
|
|
400
|
+
daemonSyncRunnerPids.add(child.pid);
|
|
401
|
+
child.stdout?.pipe(logStream, { end: false });
|
|
402
|
+
child.stderr?.pipe(logStream, { end: false });
|
|
403
|
+
child.on("close", () => {
|
|
404
|
+
spawned.delete(child);
|
|
405
|
+
if (child.pid !== undefined)
|
|
406
|
+
daemonSyncRunnerPids.delete(child.pid);
|
|
407
|
+
logStream.end();
|
|
408
|
+
});
|
|
409
|
+
return child;
|
|
410
|
+
}
|
|
411
|
+
const fd = openPrivateAppend(logPath);
|
|
346
412
|
try {
|
|
347
|
-
const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
|
|
348
413
|
const child = spawn(file, args, {
|
|
349
414
|
cwd,
|
|
350
415
|
env: unitProcessEnv(env, unit),
|
|
@@ -352,7 +417,13 @@ export async function runDaemon(opts) {
|
|
|
352
417
|
windowsHide: true,
|
|
353
418
|
});
|
|
354
419
|
spawned.add(child);
|
|
355
|
-
|
|
420
|
+
if (unit.service === "sync" && child.pid !== undefined)
|
|
421
|
+
daemonSyncRunnerPids.add(child.pid);
|
|
422
|
+
child.once("exit", () => {
|
|
423
|
+
spawned.delete(child);
|
|
424
|
+
if (child.pid !== undefined)
|
|
425
|
+
daemonSyncRunnerPids.delete(child.pid);
|
|
426
|
+
});
|
|
356
427
|
// Only long-running services are handed over; a scheduled run left behind finishes on its own.
|
|
357
428
|
if (unit.kind === "long-running")
|
|
358
429
|
tracker?.track(unit.id, child);
|
|
@@ -382,6 +453,21 @@ export async function runDaemon(opts) {
|
|
|
382
453
|
restartDaemon: () => restartDaemon(),
|
|
383
454
|
writeState: (state) => writeJsonAtomic(paths.statePath, state),
|
|
384
455
|
log,
|
|
456
|
+
syncOwnerBlockReason: () => {
|
|
457
|
+
const config = readDaemonConfig(paths.configPath);
|
|
458
|
+
const hqRoot = config.hqRoot ?? env.HQ_ROOT?.trim();
|
|
459
|
+
if (!hqRoot)
|
|
460
|
+
return undefined;
|
|
461
|
+
const statusPath = watchRunnerOwnerStatusPath(hqRoot, home, env);
|
|
462
|
+
try {
|
|
463
|
+
const owner = readWatchRunnerOwnerStatus(statusPath);
|
|
464
|
+
return owner ? `sync owned by ${owner.owner} pid ${owner.pid}` : undefined;
|
|
465
|
+
}
|
|
466
|
+
catch (err) {
|
|
467
|
+
log("error", `could not read sync owner status at ${statusPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
468
|
+
return undefined;
|
|
469
|
+
}
|
|
470
|
+
},
|
|
385
471
|
cliVersion: CLI_VERSION,
|
|
386
472
|
adopt,
|
|
387
473
|
});
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
import type { UnitPlan } from "./catalog.js";
|
|
25
25
|
import type { MachineType } from "./machine-type.js";
|
|
26
26
|
import type { ControlRequest } from "./requests.js";
|
|
27
|
+
import { syncPassResult } from "./sync-runner.js";
|
|
27
28
|
export declare const BACKOFF_BASE_MS = 1000;
|
|
28
29
|
export declare const BACKOFF_MAX_MS = 60000;
|
|
29
30
|
export declare const HEALTHY_RUN_MS = 60000;
|
|
@@ -34,6 +35,9 @@ export declare const HANDOFF_STAGGER_MS = 15000;
|
|
|
34
35
|
export declare const HANDOFF_DRAIN_MAX_MS: number;
|
|
35
36
|
export interface SupervisedChild {
|
|
36
37
|
pid?: number;
|
|
38
|
+
stdout?: {
|
|
39
|
+
on(event: "data" | "end", handler: (chunk?: Buffer | string) => void): unknown;
|
|
40
|
+
};
|
|
37
41
|
on(event: "exit", handler: (code: number | null, signal: string | null) => void): unknown;
|
|
38
42
|
on(event: "error", handler: (err: Error) => void): unknown;
|
|
39
43
|
kill(signal?: NodeJS.Signals | number): boolean;
|
|
@@ -57,6 +61,8 @@ export interface UnitState {
|
|
|
57
61
|
lastRunAt?: number;
|
|
58
62
|
nextRunAt?: number;
|
|
59
63
|
lastResult?: "ok" | "failed";
|
|
64
|
+
lastHeartbeatAt?: string;
|
|
65
|
+
lastPassResult?: ReturnType<typeof syncPassResult>;
|
|
60
66
|
}
|
|
61
67
|
/** An app that runs the daemon as its own child process instead of an OS unit. */
|
|
62
68
|
export type HostApp = "desktop";
|
|
@@ -99,6 +105,8 @@ export interface SupervisorDeps {
|
|
|
99
105
|
clearTimeout?: (handle: unknown) => void;
|
|
100
106
|
reconcileMs?: number;
|
|
101
107
|
requestPollMs?: number;
|
|
108
|
+
/** Read hq-cloud's lease status after it exits with the lease-conflict code. */
|
|
109
|
+
syncOwnerBlockReason?: () => string | undefined;
|
|
102
110
|
/** The running CLI's version; part of the key of units that run `hq` commands. */
|
|
103
111
|
cliVersion?: string;
|
|
104
112
|
/** Services to take over from the previous daemon instead of starting them. */
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
*
|
|
22
22
|
* Processes, time, and state output are injected; `run.ts` wires the real ones.
|
|
23
23
|
*/
|
|
24
|
+
import { StringDecoder } from "node:string_decoder";
|
|
25
|
+
import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
|
|
24
26
|
export const BACKOFF_BASE_MS = 1_000;
|
|
25
27
|
export const BACKOFF_MAX_MS = 60_000;
|
|
26
28
|
export const HEALTHY_RUN_MS = 60_000;
|
|
@@ -39,7 +41,15 @@ export function backoffMs(attempt) {
|
|
|
39
41
|
* `keepAcrossCliUpdates` running.
|
|
40
42
|
*/
|
|
41
43
|
export function planKey(plan, cliVersion) {
|
|
42
|
-
const fields = [
|
|
44
|
+
const fields = [
|
|
45
|
+
plan.kind,
|
|
46
|
+
plan.restart,
|
|
47
|
+
plan.intervalSec,
|
|
48
|
+
plan.initialDelaySec,
|
|
49
|
+
plan.heartbeatTimeoutMs,
|
|
50
|
+
plan.command,
|
|
51
|
+
plan.env,
|
|
52
|
+
];
|
|
43
53
|
if (cliVersion !== undefined && plan.command?.type === "hq" && !plan.keepAcrossCliUpdates)
|
|
44
54
|
fields.push(cliVersion);
|
|
45
55
|
return JSON.stringify(fields);
|
|
@@ -89,6 +99,11 @@ export function createSupervisor(deps) {
|
|
|
89
99
|
clearT(slot.timer);
|
|
90
100
|
slot.timer = undefined;
|
|
91
101
|
};
|
|
102
|
+
const clearHeartbeatTimer = (slot) => {
|
|
103
|
+
if (slot.heartbeatTimer !== undefined)
|
|
104
|
+
clearT(slot.heartbeatTimer);
|
|
105
|
+
slot.heartbeatTimer = undefined;
|
|
106
|
+
};
|
|
92
107
|
const setBlocked = (slot) => {
|
|
93
108
|
const block = slot.plan.blocked;
|
|
94
109
|
slot.state.status = block.state;
|
|
@@ -111,6 +126,7 @@ export function createSupervisor(deps) {
|
|
|
111
126
|
*/
|
|
112
127
|
const terminate = (slot, then, drain = false) => {
|
|
113
128
|
clearTimer(slot);
|
|
129
|
+
clearHeartbeatTimer(slot);
|
|
114
130
|
const child = slot.child;
|
|
115
131
|
if (!child) {
|
|
116
132
|
then?.();
|
|
@@ -163,6 +179,69 @@ export function createSupervisor(deps) {
|
|
|
163
179
|
const watch = (slot, child) => {
|
|
164
180
|
live.add(child);
|
|
165
181
|
let ended = false;
|
|
182
|
+
const timeoutMs = slot.plan.heartbeatTimeoutMs;
|
|
183
|
+
const decoder = timeoutMs ? new StringDecoder("utf8") : null;
|
|
184
|
+
let stdoutRemainder = "";
|
|
185
|
+
let lastHeartbeatPersistedAt = now() - 15_000;
|
|
186
|
+
const armHeartbeatWatchdog = () => {
|
|
187
|
+
if (!timeoutMs)
|
|
188
|
+
return;
|
|
189
|
+
clearHeartbeatTimer(slot);
|
|
190
|
+
slot.heartbeatTimer = setT(() => {
|
|
191
|
+
slot.heartbeatTimer = undefined;
|
|
192
|
+
if (stopped || slot.child !== child || slot.expectingExit)
|
|
193
|
+
return;
|
|
194
|
+
deps.log("warn", `${slot.plan.id} emitted no sync protocol heartbeat for ${timeoutMs}ms; restarting it`);
|
|
195
|
+
terminate(slot, () => scheduleRestart(slot));
|
|
196
|
+
emit();
|
|
197
|
+
}, timeoutMs);
|
|
198
|
+
};
|
|
199
|
+
const acceptProtocolLine = (line) => {
|
|
200
|
+
let event;
|
|
201
|
+
try {
|
|
202
|
+
event = parseSyncRunnerProtocolLine(line);
|
|
203
|
+
}
|
|
204
|
+
catch (err) {
|
|
205
|
+
deps.log("error", `${slot.plan.id} emitted invalid sync protocol output (${Buffer.byteLength(line)} bytes): ${errorMessage(err)}`);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (!event)
|
|
209
|
+
return;
|
|
210
|
+
const at = now();
|
|
211
|
+
slot.state.lastHeartbeatAt = new Date(at).toISOString();
|
|
212
|
+
const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
|
|
213
|
+
if (pass)
|
|
214
|
+
slot.state.lastPassResult = pass;
|
|
215
|
+
armHeartbeatWatchdog();
|
|
216
|
+
if (pass || at - lastHeartbeatPersistedAt >= 15_000) {
|
|
217
|
+
lastHeartbeatPersistedAt = at;
|
|
218
|
+
emit();
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
if (decoder && child.stdout) {
|
|
222
|
+
child.stdout.on("data", (chunk) => {
|
|
223
|
+
if (chunk === undefined)
|
|
224
|
+
return;
|
|
225
|
+
stdoutRemainder += decoder.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
226
|
+
let newline = stdoutRemainder.indexOf("\n");
|
|
227
|
+
while (newline >= 0) {
|
|
228
|
+
acceptProtocolLine(stdoutRemainder.slice(0, newline).trim());
|
|
229
|
+
stdoutRemainder = stdoutRemainder.slice(newline + 1);
|
|
230
|
+
newline = stdoutRemainder.indexOf("\n");
|
|
231
|
+
}
|
|
232
|
+
// A malformed or unexpectedly large line is not a heartbeat. Bound
|
|
233
|
+
// retained data so a broken child cannot grow daemon memory forever.
|
|
234
|
+
if (Buffer.byteLength(stdoutRemainder) > 1024 * 1024)
|
|
235
|
+
stdoutRemainder = "";
|
|
236
|
+
});
|
|
237
|
+
child.stdout.on("end", () => {
|
|
238
|
+
stdoutRemainder += decoder.end();
|
|
239
|
+
if (stdoutRemainder)
|
|
240
|
+
acceptProtocolLine(stdoutRemainder.trim());
|
|
241
|
+
stdoutRemainder = "";
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
armHeartbeatWatchdog();
|
|
166
245
|
const onEnd = (code, signal) => {
|
|
167
246
|
if (ended)
|
|
168
247
|
return;
|
|
@@ -173,6 +252,7 @@ export function createSupervisor(deps) {
|
|
|
173
252
|
if (slot.child !== child)
|
|
174
253
|
return;
|
|
175
254
|
slot.child = undefined;
|
|
255
|
+
clearHeartbeatTimer(slot);
|
|
176
256
|
if (slot.killTimer !== undefined) {
|
|
177
257
|
clearT(slot.killTimer);
|
|
178
258
|
slot.killTimer = undefined;
|
|
@@ -200,6 +280,20 @@ export function createSupervisor(deps) {
|
|
|
200
280
|
emit();
|
|
201
281
|
return;
|
|
202
282
|
}
|
|
283
|
+
if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
|
|
284
|
+
const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
|
|
285
|
+
slot.state.status = "waiting";
|
|
286
|
+
slot.state.reason = reason;
|
|
287
|
+
slot.state.nextRunAt = now() + SYNC_RUNNER_LEASE_RETRY_MS;
|
|
288
|
+
deps.log("info", `${reason}; retrying in ${SYNC_RUNNER_LEASE_RETRY_MS}ms`);
|
|
289
|
+
slot.timer = setT(() => {
|
|
290
|
+
slot.timer = undefined;
|
|
291
|
+
startLong(slot);
|
|
292
|
+
emit();
|
|
293
|
+
}, SYNC_RUNNER_LEASE_RETRY_MS);
|
|
294
|
+
emit();
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
203
297
|
deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
|
|
204
298
|
if (ranFor >= HEALTHY_RUN_MS)
|
|
205
299
|
slot.attempts = 0;
|
|
@@ -505,6 +599,11 @@ export function createSupervisor(deps) {
|
|
|
505
599
|
}
|
|
506
600
|
};
|
|
507
601
|
const handleRequest = (req) => {
|
|
602
|
+
if (req.action === "reconcile") {
|
|
603
|
+
deps.log("info", "reconciling daemon configuration (requested)");
|
|
604
|
+
reconcile();
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
508
607
|
if (req.action === "restart-daemon") {
|
|
509
608
|
deps.log("info", "restarting the daemon (requested)");
|
|
510
609
|
if (deps.restartDaemon)
|
|
@@ -614,6 +713,7 @@ export function createSupervisor(deps) {
|
|
|
614
713
|
clearT(handoffTimer);
|
|
615
714
|
for (const slot of slots.values()) {
|
|
616
715
|
clearTimer(slot);
|
|
716
|
+
clearHeartbeatTimer(slot);
|
|
617
717
|
slot.child = undefined;
|
|
618
718
|
if (slot.killTimer !== undefined)
|
|
619
719
|
clearT(slot.killTimer);
|