@indigoai-us/hq-cli 5.345.155 → 5.345.159
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +289 -0
- package/assets/scaffold/core/scripts/rebuild-company-knowledge-index.sh +110 -2
- package/assets/scaffold/core/scripts/rebuild-projects-index.sh +101 -5
- package/assets/scaffold/core/scripts/worktree.sh +4 -0
- package/dist/command-catalog.generated.d.ts +536 -7
- package/dist/command-catalog.generated.js +676 -24
- package/dist/command-registration-plan.d.ts +18 -0
- package/dist/command-registration-plan.js +4 -0
- package/dist/commands/_patterns.d.ts +17 -0
- package/dist/commands/_patterns.js +22 -0
- package/dist/commands/agent-inbox.js +2 -2
- package/dist/commands/agent-kit.js +8 -6
- package/dist/commands/bot.js +7 -4
- package/dist/commands/channel-suggest.d.ts +56 -0
- package/dist/commands/channel-suggest.js +170 -0
- package/dist/commands/channels.js +7 -1
- package/dist/commands/cloud.d.ts +45 -0
- package/dist/commands/cloud.js +150 -10
- package/dist/commands/company.js +4 -3
- package/dist/commands/core-checkpoint-native.d.ts +18 -0
- package/dist/commands/core-checkpoint-native.js +67 -2
- package/dist/commands/core-pipeline-driver-native.d.ts +12 -1
- package/dist/commands/core-pipeline-driver-native.js +131 -5
- package/dist/commands/core.js +13 -1
- package/dist/commands/crm.js +83 -6
- package/dist/commands/deploy-access.d.ts +33 -0
- package/dist/commands/deploy-access.js +322 -0
- package/dist/commands/dm-attach.d.ts +106 -0
- package/dist/commands/dm-attach.js +184 -0
- package/dist/commands/dm.d.ts +37 -0
- package/dist/commands/dm.js +172 -10
- package/dist/commands/feedback-show.d.ts +40 -0
- package/dist/commands/feedback-show.js +261 -0
- package/dist/commands/feedback.d.ts +21 -0
- package/dist/commands/feedback.js +152 -2
- package/dist/commands/files-browse.d.ts +16 -0
- package/dist/commands/files-browse.js +52 -1
- package/dist/commands/files.d.ts +24 -0
- package/dist/commands/files.js +52 -1
- package/dist/commands/integrations-connect.js +7 -3
- package/dist/commands/integrations-core.d.ts +13 -1
- package/dist/commands/integrations-core.js +49 -4
- package/dist/commands/integrations.js +4 -2
- package/dist/commands/lanes-accounts.js +16 -1
- package/dist/commands/lanes-loop.js +14 -4
- package/dist/commands/lanes-message.js +6 -4
- package/dist/commands/lanes-resume.js +36 -13
- package/dist/commands/lanes-shared.d.ts +5 -0
- package/dist/commands/lanes-shared.js +13 -3
- package/dist/commands/lanes-watch.js +1 -0
- package/dist/commands/lanes.d.ts +20 -1
- package/dist/commands/lanes.js +201 -51
- package/dist/commands/mesh.js +149 -1
- package/dist/commands/monitor.d.ts +1 -0
- package/dist/commands/monitor.js +4 -1
- package/dist/commands/repo.d.ts +41 -0
- package/dist/commands/repo.js +331 -0
- package/dist/commands/scan.d.ts +3 -1
- package/dist/commands/scan.js +7 -3
- package/dist/commands/secrets.d.ts +37 -0
- package/dist/commands/secrets.js +185 -16
- package/dist/commands/signals.js +17 -5
- package/dist/commands/staff/listings.d.ts +41 -0
- package/dist/commands/staff/listings.js +160 -0
- package/dist/commands/staff/reset-test-account.d.ts +3 -2
- package/dist/commands/staff/reset-test-account.js +4 -3
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +2629 -1036
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/agent-kit/inbox-state.d.ts +7 -0
- package/dist/lib/agent-kit/inbox-state.js +5 -1
- package/dist/lib/agent-kit/mcp/tools.js +2 -1
- package/dist/lib/agent-kit/run/inbox.js +2 -1
- package/dist/lib/agent-kit/wake-secrets.d.ts +7 -0
- package/dist/lib/agent-kit/wake-secrets.js +32 -0
- package/dist/lib/agent-kit/wake.d.ts +4 -0
- package/dist/lib/agent-kit/wake.js +49 -3
- package/dist/lib/bot/inflight.d.ts +7 -1
- package/dist/lib/bot/inflight.js +10 -2
- package/dist/lib/bot/prompt.js +3 -0
- package/dist/lib/bot/run.js +72 -20
- package/dist/lib/bot/session.d.ts +14 -0
- package/dist/lib/bot/session.js +70 -0
- package/dist/lib/capacity/collectors/claude-oauth.d.ts +6 -0
- package/dist/lib/capacity/collectors/claude-oauth.js +12 -0
- package/dist/lib/capacity/lane-login.d.ts +6 -0
- package/dist/lib/capacity/lane-login.js +5 -0
- package/dist/lib/core-utils/worktree.js +2 -0
- package/dist/lib/daemon/exit-cause.d.ts +43 -0
- package/dist/lib/daemon/exit-cause.js +88 -0
- package/dist/lib/daemon/handoff.d.ts +2 -0
- package/dist/lib/daemon/handoff.js +6 -1
- package/dist/lib/daemon/hq-child-bootstrap.mjs +20 -3
- package/dist/lib/daemon/paths.d.ts +2 -0
- package/dist/lib/daemon/paths.js +1 -0
- package/dist/lib/daemon/run.js +13 -0
- package/dist/lib/daemon/supervisor.d.ts +54 -1
- package/dist/lib/daemon/supervisor.js +297 -21
- package/dist/lib/daemon/unit-holds.d.ts +22 -0
- package/dist/lib/daemon/unit-holds.js +86 -0
- package/dist/lib/doctor/checks/sync-health.d.ts +9 -0
- package/dist/lib/doctor/checks/sync-health.js +66 -1
- package/dist/lib/doctor/company-resolution-state.d.ts +30 -0
- package/dist/lib/doctor/company-resolution-state.js +89 -0
- package/dist/lib/doctor/fix/apply.d.ts +36 -6
- package/dist/lib/doctor/fix/apply.js +128 -12
- package/dist/lib/doctor/types.d.ts +1 -1
- package/dist/lib/doctor/types.js +1 -0
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/hook-engine/resolve.js +6 -2
- package/dist/lib/index-render/company-knowledge.d.ts +1 -1
- package/dist/lib/index-render/company-knowledge.js +4 -5
- package/dist/lib/index-render/index.js +44 -3
- package/dist/lib/index-render/projects.d.ts +1 -1
- package/dist/lib/index-render/projects.js +5 -5
- package/dist/lib/lanes/account-company.d.ts +28 -0
- package/dist/lib/lanes/account-company.js +53 -0
- package/dist/lib/lanes/accounts.d.ts +22 -3
- package/dist/lib/lanes/accounts.js +38 -2
- package/dist/lib/lanes/claude-balance.d.ts +6 -1
- package/dist/lib/lanes/claude-balance.js +6 -1
- package/dist/lib/lanes/claude-credential.d.ts +78 -0
- package/dist/lib/lanes/claude-credential.js +163 -0
- package/dist/lib/lanes/compaction.d.ts +2 -0
- package/dist/lib/lanes/compaction.js +9 -0
- package/dist/lib/lanes/config.js +6 -0
- package/dist/lib/lanes/envelope-repair-dispatch.js +2 -1
- package/dist/lib/lanes/hooks-install.js +2 -1
- package/dist/lib/lanes/host-capacity.d.ts +2 -0
- package/dist/lib/lanes/host-capacity.js +1 -0
- package/dist/lib/lanes/keep-alive.d.ts +1 -1
- package/dist/lib/lanes/keep-alive.js +3 -1
- package/dist/lib/lanes/links.js +10 -4
- package/dist/lib/lanes/loop.js +16 -1
- package/dist/lib/lanes/node-binary.d.ts +17 -0
- package/dist/lib/lanes/node-binary.js +30 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +2 -1
- package/dist/lib/lanes/outpost-dispatch.js +1 -0
- package/dist/lib/lanes/reconcile.js +4 -0
- package/dist/lib/lanes/remote-control-lane.js +2 -0
- package/dist/lib/lanes/spawn.d.ts +3 -0
- package/dist/lib/lanes/spawn.js +119 -9
- package/dist/lib/lanes/store.d.ts +6 -0
- package/dist/lib/lanes/store.js +107 -18
- package/dist/lib/lanes/types.d.ts +2 -0
- package/dist/lib/lanes/watch.js +28 -5
- package/dist/lib/lanes/watcher-main.js +4 -3
- package/dist/lib/lanes/watcher.d.ts +4 -0
- package/dist/lib/lanes/watcher.js +110 -21
- package/dist/lib/mesh/api.d.ts +32 -0
- package/dist/lib/mesh/api.js +75 -0
- package/dist/lib/secret-scan/detect.js +9 -0
- package/dist/lib/secret-scan/repo-config.d.ts +41 -0
- package/dist/lib/secret-scan/repo-config.js +237 -0
- package/dist/lib/secret-scan/rules.d.ts +7 -1
- package/dist/lib/secret-scan/rules.js +3 -1
- package/dist/lib/secret-scan/staged.d.ts +5 -0
- package/dist/lib/secret-scan/staged.js +19 -2
- package/dist/lib/secret-scan/types.d.ts +2 -0
- package/dist/lib/sync/server-owned-local-only.d.ts +29 -0
- package/dist/lib/sync/server-owned-local-only.js +75 -0
- package/dist/sentry-before-send.js +1 -0
- package/dist/utils/access-denied-hint.js +6 -0
- package/dist/utils/client-health.d.ts +34 -0
- package/dist/utils/client-health.js +96 -3
- package/dist/utils/package-use-lease.js +27 -9
- package/dist/utils/version-gate.d.ts +7 -0
- package/dist/utils/version-gate.js +66 -0
- package/package.json +8 -8
|
@@ -3,6 +3,11 @@
|
|
|
3
3
|
*
|
|
4
4
|
* - long-running units are restarted after they exit, with per-unit backoff
|
|
5
5
|
* (1 s doubling to 60 s, reset once a unit has stayed up for a minute);
|
|
6
|
+
* a unit that fails EPISODE_MAX_FAILURES times without a run of
|
|
7
|
+
* EPISODE_HEALTHY_RESET_MS, or fails for a permanent cause (see exit-cause.ts), is held
|
|
8
|
+
* instead. Permanent holds need a restart request, command change or CLI update.
|
|
9
|
+
* Exhausted transient sync/mesh units make one durable supervised probe after
|
|
10
|
+
* a 30-minute cooldown; a failed probe re-holds, and an explicit stop stays stopped;
|
|
6
11
|
* `on-failure` units stay down after a clean exit or a SIGTERM/SIGINT from
|
|
7
12
|
* outside, and any unit stays down after a stop request (`hq bot stop`),
|
|
8
13
|
* until a restart request;
|
|
@@ -22,11 +27,20 @@
|
|
|
22
27
|
*
|
|
23
28
|
* Processes, time, and state output are injected; `run.ts` wires the real ones.
|
|
24
29
|
*/
|
|
30
|
+
import { createHash } from "node:crypto";
|
|
25
31
|
import { StringDecoder } from "node:string_decoder";
|
|
32
|
+
import { classifyExit } from "./exit-cause.js";
|
|
26
33
|
import { SYNC_RUNNER_LEASE_CONFLICT_EXIT, SYNC_RUNNER_LEASE_RETRY_MS, parseSyncRunnerProtocolLine, syncPassResult, } from "./sync-runner.js";
|
|
27
34
|
const BACKOFF_BASE_MS = 1_000;
|
|
28
35
|
const BACKOFF_MAX_MS = 60_000;
|
|
29
|
-
const HEALTHY_RUN_MS = 60_000;
|
|
36
|
+
export const HEALTHY_RUN_MS = 60_000;
|
|
37
|
+
/**
|
|
38
|
+
* Failures in one episode before the unit is held: the first start plus five
|
|
39
|
+
* automatic retries. Matches the Desktop app's MAX_FAILURES.
|
|
40
|
+
*/
|
|
41
|
+
export const EPISODE_MAX_FAILURES = 6;
|
|
42
|
+
/** A run this long ends the failure episode. Matches the Desktop app's HEALTHY_RUN_WINDOW. */
|
|
43
|
+
export const EPISODE_HEALTHY_RESET_MS = 30 * 60_000;
|
|
30
44
|
export const STOP_ESCALATION_MS = 10_000;
|
|
31
45
|
export const SYNC_RUNNER_PERSIST_GRACE_MS = 60_000;
|
|
32
46
|
const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
|
|
@@ -69,6 +83,17 @@ const BLOCKED_READS_BEFORE_PAUSE = 2;
|
|
|
69
83
|
function backoffMs(attempt) {
|
|
70
84
|
return Math.min(BACKOFF_MAX_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt));
|
|
71
85
|
}
|
|
86
|
+
function keyHash(key) {
|
|
87
|
+
return createHash("sha256").update(key).digest("hex");
|
|
88
|
+
}
|
|
89
|
+
function recoverableHold(plan, hold) {
|
|
90
|
+
return plan.kind === "long-running" && (plan.service === "sync" || plan.service === "mesh")
|
|
91
|
+
&& hold?.kind === "exhausted" && hold.cause === "transient";
|
|
92
|
+
}
|
|
93
|
+
function heldReason(unitId, hold) {
|
|
94
|
+
const what = hold.kind === "permanent" ? `${hold.cause}: ${hold.detail}` : hold.detail;
|
|
95
|
+
return `held (${what}); \`hq daemon restart ${unitId}\` retries`;
|
|
96
|
+
}
|
|
72
97
|
/**
|
|
73
98
|
* What a unit's process was started from; a unit whose key changes is
|
|
74
99
|
* restarted. The CLI version counts only for `hq` commands, so a CLI update
|
|
@@ -109,6 +134,15 @@ export function createSupervisor(deps) {
|
|
|
109
134
|
const reconcileMs = deps.reconcileMs ?? 15_000;
|
|
110
135
|
const requestPollMs = deps.requestPollMs ?? 1_000;
|
|
111
136
|
const startedAt = now();
|
|
137
|
+
let persistedHolds = {};
|
|
138
|
+
/** Holds dropped at plan time (a changed command) that holds.json still lists. */
|
|
139
|
+
let holdsDirty = false;
|
|
140
|
+
try {
|
|
141
|
+
persistedHolds = deps.holds?.read() ?? {};
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
deps.log("error", `could not read unit holds; starting without them: ${errorMessage(err)}`);
|
|
145
|
+
}
|
|
112
146
|
let slots = new Map();
|
|
113
147
|
let stopped = false;
|
|
114
148
|
let started = false;
|
|
@@ -144,7 +178,13 @@ export function createSupervisor(deps) {
|
|
|
144
178
|
deps.log("error", `could not write daemon state: ${errorMessage(err)}`);
|
|
145
179
|
}
|
|
146
180
|
};
|
|
181
|
+
const clearRecoveryWindow = (slot) => {
|
|
182
|
+
if (slot.recoveryHealthyTimer !== undefined)
|
|
183
|
+
clearT(slot.recoveryHealthyTimer);
|
|
184
|
+
slot.recoveryHealthyTimer = undefined;
|
|
185
|
+
};
|
|
147
186
|
const clearTimer = (slot) => {
|
|
187
|
+
clearRecoveryWindow(slot);
|
|
148
188
|
if (slot.timer !== undefined)
|
|
149
189
|
clearT(slot.timer);
|
|
150
190
|
slot.timer = undefined;
|
|
@@ -269,6 +309,89 @@ export function createSupervisor(deps) {
|
|
|
269
309
|
delete slot.state.pid;
|
|
270
310
|
delete slot.state.nextRunAt;
|
|
271
311
|
};
|
|
312
|
+
const writeHolds = () => {
|
|
313
|
+
if (!deps.holds)
|
|
314
|
+
return false;
|
|
315
|
+
const next = {};
|
|
316
|
+
// Keep holds for units not planned right now (for example a paused bot).
|
|
317
|
+
for (const [id, hold] of Object.entries(persistedHolds))
|
|
318
|
+
if (!slots.has(id))
|
|
319
|
+
next[id] = hold;
|
|
320
|
+
for (const [id, slot] of slots)
|
|
321
|
+
if (slot.state.hold)
|
|
322
|
+
next[id] = slot.state.hold;
|
|
323
|
+
persistedHolds = next;
|
|
324
|
+
holdsDirty = false;
|
|
325
|
+
try {
|
|
326
|
+
deps.holds.write(next);
|
|
327
|
+
return true;
|
|
328
|
+
}
|
|
329
|
+
catch (err) {
|
|
330
|
+
deps.log("error", `could not write unit holds: ${errorMessage(err)}`);
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
};
|
|
334
|
+
const showHeld = (slot) => {
|
|
335
|
+
if (recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.retryStopped === true) {
|
|
336
|
+
slot.state.status = "exited";
|
|
337
|
+
slot.state.reason = `stopped on request; \`hq daemon restart ${slot.plan.id}\` starts it again`;
|
|
338
|
+
delete slot.state.pid;
|
|
339
|
+
delete slot.state.nextRunAt;
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
slot.state.status = "held";
|
|
343
|
+
slot.state.reason = heldReason(slot.plan.id, slot.state.hold);
|
|
344
|
+
delete slot.state.pid;
|
|
345
|
+
delete slot.state.nextRunAt;
|
|
346
|
+
if (recoverableHold(slot.plan, slot.state.hold) && Number.isSafeInteger(slot.state.hold.retryAt)
|
|
347
|
+
&& slot.state.hold.retryAt >= 0 && slot.state.hold.retryAt <= 8_640_000_000_000_000) {
|
|
348
|
+
slot.state.nextRunAt = slot.state.hold.retryAt;
|
|
349
|
+
slot.state.reason += `; one supervised retry at ${new Date(slot.state.hold.retryAt).toISOString()}`;
|
|
350
|
+
}
|
|
351
|
+
};
|
|
352
|
+
const holdUnit = (slot, kind, cause, detail) => {
|
|
353
|
+
clearTimer(slot);
|
|
354
|
+
slot.state.hold = {
|
|
355
|
+
kind,
|
|
356
|
+
cause,
|
|
357
|
+
detail,
|
|
358
|
+
since: now(),
|
|
359
|
+
failures: slot.failures,
|
|
360
|
+
...(slot.state.lastExit ? { lastExit: { ...slot.state.lastExit } } : {}),
|
|
361
|
+
keyHash: keyHash(slot.key),
|
|
362
|
+
};
|
|
363
|
+
if (deps.holds && recoverableHold(slot.plan, slot.state.hold)) {
|
|
364
|
+
slot.state.hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
|
|
365
|
+
}
|
|
366
|
+
persistedHolds[slot.plan.id] = slot.state.hold;
|
|
367
|
+
showHeld(slot);
|
|
368
|
+
deps.log("error", `holding ${slot.plan.id}: ${slot.state.reason}`);
|
|
369
|
+
writeHolds();
|
|
370
|
+
armRecoveryProbe(slot);
|
|
371
|
+
};
|
|
372
|
+
const clearHold = (slot) => {
|
|
373
|
+
clearRecoveryWindow(slot);
|
|
374
|
+
slot.attempts = 0;
|
|
375
|
+
slot.failures = 0;
|
|
376
|
+
if (!slot.state.hold && !persistedHolds[slot.plan.id])
|
|
377
|
+
return;
|
|
378
|
+
delete slot.state.hold;
|
|
379
|
+
if (slot.state.status === "held") {
|
|
380
|
+
slot.state.status = "idle";
|
|
381
|
+
delete slot.state.reason;
|
|
382
|
+
}
|
|
383
|
+
delete persistedHolds[slot.plan.id];
|
|
384
|
+
writeHolds();
|
|
385
|
+
};
|
|
386
|
+
/** Hold the unit when its exit has a permanent cause; true when it was held. */
|
|
387
|
+
const holdIfPermanent = (slot, evidence) => {
|
|
388
|
+
const cause = classifyExit(evidence);
|
|
389
|
+
if (cause.kind !== "permanent")
|
|
390
|
+
return false;
|
|
391
|
+
slot.failures += 1;
|
|
392
|
+
holdUnit(slot, "permanent", cause.cause, cause.detail);
|
|
393
|
+
return true;
|
|
394
|
+
};
|
|
272
395
|
const signalChild = (slot, child, signal) => {
|
|
273
396
|
try {
|
|
274
397
|
child.kill(signal);
|
|
@@ -344,6 +467,11 @@ export function createSupervisor(deps) {
|
|
|
344
467
|
const scheduleRestart = (slot) => {
|
|
345
468
|
if (stopped)
|
|
346
469
|
return;
|
|
470
|
+
slot.failures += 1;
|
|
471
|
+
if (slot.failures >= EPISODE_MAX_FAILURES) {
|
|
472
|
+
holdUnit(slot, "exhausted", "transient", `${slot.failures} failures without staying up for ${EPISODE_HEALTHY_RESET_MS / 60_000} min`);
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
347
475
|
const delay = backoffMs(slot.attempts);
|
|
348
476
|
slot.attempts += 1;
|
|
349
477
|
slot.state.restarts += 1;
|
|
@@ -469,12 +597,19 @@ export function createSupervisor(deps) {
|
|
|
469
597
|
clearT(slot.killTimer);
|
|
470
598
|
slot.killTimer = undefined;
|
|
471
599
|
}
|
|
600
|
+
clearRecoveryWindow(slot);
|
|
472
601
|
const ranFor = now() - (slot.childStartedAt ?? now());
|
|
473
602
|
slot.state.lastExit = { code, signal, at: now() };
|
|
474
603
|
delete slot.state.pid;
|
|
475
604
|
if (slot.expectingExit) {
|
|
476
605
|
slot.expectingExit = false;
|
|
477
606
|
slot.busy = false;
|
|
607
|
+
// A unit we stopped (for example a stalled sync runner) after a healthy
|
|
608
|
+
// run starts a new failure episode, as an unexpected exit would.
|
|
609
|
+
if (ranFor >= HEALTHY_RUN_MS)
|
|
610
|
+
slot.attempts = 0;
|
|
611
|
+
if (ranFor >= EPISODE_HEALTHY_RESET_MS)
|
|
612
|
+
slot.failures = 0;
|
|
478
613
|
const then = slot.afterExit;
|
|
479
614
|
slot.afterExit = undefined;
|
|
480
615
|
then?.();
|
|
@@ -492,6 +627,17 @@ export function createSupervisor(deps) {
|
|
|
492
627
|
emit();
|
|
493
628
|
return;
|
|
494
629
|
}
|
|
630
|
+
const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
|
|
631
|
+
const recoveryOwnerReason = slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT
|
|
632
|
+
? deps.syncOwnerBlockReason?.() ?? "sync owned by another runner"
|
|
633
|
+
: meshOwnerReason;
|
|
634
|
+
if (recoveryOwnerReason && recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.probing === true) {
|
|
635
|
+
// The ordinary ownership retry is 15 s. A recovery probe must not use it
|
|
636
|
+
// to bypass its persisted cooldown and start a second probe early.
|
|
637
|
+
holdUnit(slot, "exhausted", "transient", `recovery probe deferred: ${recoveryOwnerReason}`);
|
|
638
|
+
emit();
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
495
641
|
if (slot.plan.service === "sync" && code === SYNC_RUNNER_LEASE_CONFLICT_EXIT) {
|
|
496
642
|
const reason = deps.syncOwnerBlockReason?.() ?? "sync owned by another runner";
|
|
497
643
|
slot.state.status = "waiting";
|
|
@@ -506,7 +652,6 @@ export function createSupervisor(deps) {
|
|
|
506
652
|
emit();
|
|
507
653
|
return;
|
|
508
654
|
}
|
|
509
|
-
const meshOwnerReason = slot.plan.service === "mesh" ? deps.meshOwnerReason?.() : undefined;
|
|
510
655
|
if (meshOwnerReason) {
|
|
511
656
|
let lastLoggedReason;
|
|
512
657
|
let lastLoggedAt = Number.NEGATIVE_INFINITY;
|
|
@@ -541,6 +686,8 @@ export function createSupervisor(deps) {
|
|
|
541
686
|
deps.log("warn", `${slot.plan.id} exited (code=${code ?? "null"} signal=${signal ?? "none"})`);
|
|
542
687
|
if (ranFor >= HEALTHY_RUN_MS)
|
|
543
688
|
slot.attempts = 0;
|
|
689
|
+
if (ranFor >= EPISODE_HEALTHY_RESET_MS)
|
|
690
|
+
slot.failures = 0;
|
|
544
691
|
const stoppedFromOutside = signal === "SIGTERM" || signal === "SIGINT";
|
|
545
692
|
if (slot.plan.restart === "on-failure" && (code === 0 || stoppedFromOutside)) {
|
|
546
693
|
slot.state.status = "exited";
|
|
@@ -550,7 +697,11 @@ export function createSupervisor(deps) {
|
|
|
550
697
|
delete slot.state.nextRunAt;
|
|
551
698
|
}
|
|
552
699
|
else {
|
|
553
|
-
|
|
700
|
+
const spawnError = slot.lastSpawnError;
|
|
701
|
+
delete slot.lastSpawnError;
|
|
702
|
+
const outputTail = spawnError === undefined ? readExitOutputTail(slot) : undefined;
|
|
703
|
+
if (!holdIfPermanent(slot, { code, signal, spawnError, outputTail }))
|
|
704
|
+
scheduleRestart(slot);
|
|
554
705
|
}
|
|
555
706
|
emit();
|
|
556
707
|
};
|
|
@@ -559,10 +710,22 @@ export function createSupervisor(deps) {
|
|
|
559
710
|
deps.log("error", `${slot.plan.id}: ${errorMessage(err)}`);
|
|
560
711
|
// A child that never got a pid failed to launch and may never emit
|
|
561
712
|
// "exit". Any other error (for example a failed kill) leaves it running.
|
|
562
|
-
if (child.pid === undefined)
|
|
713
|
+
if (child.pid === undefined) {
|
|
714
|
+
if (slot.child === child)
|
|
715
|
+
slot.lastSpawnError = err;
|
|
563
716
|
onEnd(null, null);
|
|
717
|
+
}
|
|
564
718
|
});
|
|
565
719
|
};
|
|
720
|
+
const readExitOutputTail = (slot) => {
|
|
721
|
+
try {
|
|
722
|
+
return deps.exitOutputTail?.(slot.plan.id);
|
|
723
|
+
}
|
|
724
|
+
catch (err) {
|
|
725
|
+
deps.log("warn", `could not read ${slot.plan.id} output to classify its exit: ${errorMessage(err)}`);
|
|
726
|
+
return undefined;
|
|
727
|
+
}
|
|
728
|
+
};
|
|
566
729
|
const startLong = (slot) => {
|
|
567
730
|
if (stopped)
|
|
568
731
|
return;
|
|
@@ -582,7 +745,8 @@ export function createSupervisor(deps) {
|
|
|
582
745
|
catch (err) {
|
|
583
746
|
slot.state.reason = `failed to start: ${errorMessage(err)}`;
|
|
584
747
|
deps.log("error", `${slot.plan.id} ${slot.state.reason}`);
|
|
585
|
-
|
|
748
|
+
if (!holdIfPermanent(slot, { code: null, signal: null, spawnError: err }))
|
|
749
|
+
scheduleRestart(slot);
|
|
586
750
|
return;
|
|
587
751
|
}
|
|
588
752
|
slot.child = child;
|
|
@@ -598,6 +762,57 @@ export function createSupervisor(deps) {
|
|
|
598
762
|
else
|
|
599
763
|
delete slot.state.pid;
|
|
600
764
|
watch(slot, child);
|
|
765
|
+
armRecoveryHealthyWindow(slot, child);
|
|
766
|
+
};
|
|
767
|
+
const armRecoveryHealthyWindow = (slot, child) => {
|
|
768
|
+
clearRecoveryWindow(slot);
|
|
769
|
+
const hold = slot.state.hold;
|
|
770
|
+
if (!recoverableHold(slot.plan, hold) || hold.probing !== true || hold.retryStopped === true)
|
|
771
|
+
return;
|
|
772
|
+
// This timer belongs to this continuously running child, never to time spent held.
|
|
773
|
+
slot.recoveryHealthyTimer = setT(() => {
|
|
774
|
+
slot.recoveryHealthyTimer = undefined;
|
|
775
|
+
if (stopped || slot.child !== child || slot.state.hold !== hold || slot.expectingExit)
|
|
776
|
+
return;
|
|
777
|
+
clearHold(slot);
|
|
778
|
+
emit();
|
|
779
|
+
}, EPISODE_HEALTHY_RESET_MS);
|
|
780
|
+
};
|
|
781
|
+
const armRecoveryProbe = (slot) => {
|
|
782
|
+
const hold = slot.state.hold;
|
|
783
|
+
if (!recoverableHold(slot.plan, hold) || hold.retryStopped === true || !deps.holds
|
|
784
|
+
|| stopped || slot.child || slot.plan.blocked)
|
|
785
|
+
return;
|
|
786
|
+
clearTimer(slot);
|
|
787
|
+
const at = now();
|
|
788
|
+
// Legacy/corrupt deadlines and a clock moving backwards get one full cooldown,
|
|
789
|
+
// rather than an immediate retry or an unbounded wait.
|
|
790
|
+
if (!Number.isSafeInteger(hold.retryAt) || hold.retryAt < 0 || hold.retryAt > at + EPISODE_HEALTHY_RESET_MS) {
|
|
791
|
+
hold.retryAt = at + EPISODE_HEALTHY_RESET_MS;
|
|
792
|
+
persistedHolds[slot.plan.id] = hold;
|
|
793
|
+
writeHolds();
|
|
794
|
+
}
|
|
795
|
+
showHeld(slot);
|
|
796
|
+
slot.timer = setT(() => {
|
|
797
|
+
slot.timer = undefined;
|
|
798
|
+
if (stopped || slots.get(slot.plan.id) !== slot || slot.child || slot.plan.blocked
|
|
799
|
+
|| slot.state.hold !== hold || hold.retryStopped === true)
|
|
800
|
+
return;
|
|
801
|
+
// Advance the durable claim BEFORE spawning. A daemon crash during a probe
|
|
802
|
+
// cannot turn its old overdue deadline into a restart storm.
|
|
803
|
+
hold.retryAt = now() + EPISODE_HEALTHY_RESET_MS;
|
|
804
|
+
hold.probing = true;
|
|
805
|
+
persistedHolds[slot.plan.id] = hold;
|
|
806
|
+
if (!writeHolds()) {
|
|
807
|
+
delete hold.probing;
|
|
808
|
+
armRecoveryProbe(slot);
|
|
809
|
+
emit();
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
slot.state.restarts += 1;
|
|
813
|
+
startLong(slot);
|
|
814
|
+
emit();
|
|
815
|
+
}, Math.max(0, hold.retryAt - at));
|
|
601
816
|
};
|
|
602
817
|
const scheduleNextRun = (slot, delayMs) => {
|
|
603
818
|
clearTimer(slot);
|
|
@@ -703,23 +918,47 @@ export function createSupervisor(deps) {
|
|
|
703
918
|
scheduleNextRun(slot, delay);
|
|
704
919
|
return;
|
|
705
920
|
}
|
|
921
|
+
if (slot.state.hold) {
|
|
922
|
+
showHeld(slot);
|
|
923
|
+
armRecoveryProbe(slot);
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
706
926
|
startLong(slot);
|
|
707
927
|
};
|
|
708
|
-
const newSlot = (plan) =>
|
|
709
|
-
plan,
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
928
|
+
const newSlot = (plan) => {
|
|
929
|
+
const key = planKey(plan, deps.cliVersion);
|
|
930
|
+
const slot = {
|
|
931
|
+
plan,
|
|
932
|
+
key,
|
|
933
|
+
attempts: 0,
|
|
934
|
+
failures: 0,
|
|
935
|
+
expectingExit: false,
|
|
936
|
+
busy: false,
|
|
937
|
+
state: {
|
|
938
|
+
id: plan.id,
|
|
939
|
+
service: plan.service,
|
|
940
|
+
kind: plan.kind,
|
|
941
|
+
description: plan.description,
|
|
942
|
+
status: "idle",
|
|
943
|
+
restarts: 0,
|
|
944
|
+
},
|
|
945
|
+
};
|
|
946
|
+
const saved = persistedHolds[plan.id];
|
|
947
|
+
if (saved && plan.kind === "long-running") {
|
|
948
|
+
if (saved.keyHash === keyHash(key) || (recoverableHold(plan, saved) && saved.retryStopped === true)) {
|
|
949
|
+
slot.state.hold = saved;
|
|
950
|
+
slot.failures = saved.failures;
|
|
951
|
+
deps.log("info", `${plan.id} stays held from an earlier daemon: ${heldReason(plan.id, saved)}`);
|
|
952
|
+
}
|
|
953
|
+
else {
|
|
954
|
+
// The command changed (for example a CLI update), so the old cause may be fixed.
|
|
955
|
+
delete persistedHolds[plan.id];
|
|
956
|
+
holdsDirty = true;
|
|
957
|
+
deps.log("info", `releasing the hold on ${plan.id}: its command changed`);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
return slot;
|
|
961
|
+
};
|
|
723
962
|
const reconcile = () => {
|
|
724
963
|
if (stopped)
|
|
725
964
|
return;
|
|
@@ -774,6 +1013,12 @@ export function createSupervisor(deps) {
|
|
|
774
1013
|
const changed = key !== existing.key;
|
|
775
1014
|
existing.plan = plan;
|
|
776
1015
|
existing.key = key;
|
|
1016
|
+
if (changed)
|
|
1017
|
+
existing.failures = 0;
|
|
1018
|
+
if (changed && existing.state.hold && existing.state.hold.retryStopped !== true) {
|
|
1019
|
+
deps.log("info", `releasing the hold on ${plan.id}: its command changed`);
|
|
1020
|
+
clearHold(existing);
|
|
1021
|
+
}
|
|
777
1022
|
existing.state.description = plan.description;
|
|
778
1023
|
existing.state.kind = plan.kind;
|
|
779
1024
|
if (plan.blocked) {
|
|
@@ -786,6 +1031,11 @@ export function createSupervisor(deps) {
|
|
|
786
1031
|
else if (wasBlocked) {
|
|
787
1032
|
deps.log("info", `starting ${plan.id}: its requirement is now met`);
|
|
788
1033
|
existing.attempts = 0;
|
|
1034
|
+
if (existing.state.hold && !changed) {
|
|
1035
|
+
// Still the same command that was held; keep it held.
|
|
1036
|
+
activate(existing);
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
789
1039
|
// The copy stopped when it was paused may still be exiting; start the
|
|
790
1040
|
// new one only after it is gone, so two never run side by side.
|
|
791
1041
|
if (existing.child)
|
|
@@ -836,6 +1086,8 @@ export function createSupervisor(deps) {
|
|
|
836
1086
|
}
|
|
837
1087
|
}
|
|
838
1088
|
slots = next;
|
|
1089
|
+
if (holdsDirty)
|
|
1090
|
+
writeHolds();
|
|
839
1091
|
emit();
|
|
840
1092
|
pumpHandoff();
|
|
841
1093
|
};
|
|
@@ -911,6 +1163,16 @@ export function createSupervisor(deps) {
|
|
|
911
1163
|
}
|
|
912
1164
|
// Recorded as runnable, so the first plan read pauses it when it is now blocked.
|
|
913
1165
|
const slot = newSlot({ ...plan, blocked: undefined });
|
|
1166
|
+
const manualRecoveryStop = recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.retryStopped === true;
|
|
1167
|
+
const keepProbe = recoverableHold(slot.plan, slot.state.hold) && slot.state.hold.probing === true
|
|
1168
|
+
&& slot.state.hold.retryStopped !== true && slot.state.hold.keyHash === keyHash(unit.key);
|
|
1169
|
+
if (!keepProbe && !manualRecoveryStop && (slot.state.hold || persistedHolds[unit.id])) {
|
|
1170
|
+
// A running copy drops stale holds, except a claimed recovery probe whose
|
|
1171
|
+
// failure episode must survive the daemon handoff.
|
|
1172
|
+
delete slot.state.hold;
|
|
1173
|
+
delete persistedHolds[unit.id];
|
|
1174
|
+
holdsDirty = true;
|
|
1175
|
+
}
|
|
914
1176
|
slot.key = unit.key;
|
|
915
1177
|
slot.adopted = true;
|
|
916
1178
|
slot.child = unit.child;
|
|
@@ -922,7 +1184,10 @@ export function createSupervisor(deps) {
|
|
|
922
1184
|
if (unit.child.pid !== undefined)
|
|
923
1185
|
slot.state.pid = unit.child.pid;
|
|
924
1186
|
watch(slot, unit.child);
|
|
1187
|
+
armRecoveryHealthyWindow(slot, unit.child);
|
|
925
1188
|
slots.set(unit.id, slot);
|
|
1189
|
+
if (manualRecoveryStop)
|
|
1190
|
+
terminate(slot, () => showHeld(slot));
|
|
926
1191
|
deps.log("info", `adopted ${unit.id} (pid ${unit.child.pid ?? "?"}) from the previous daemon`);
|
|
927
1192
|
}
|
|
928
1193
|
};
|
|
@@ -972,6 +1237,15 @@ export function createSupervisor(deps) {
|
|
|
972
1237
|
}
|
|
973
1238
|
if (req.action === "stop") {
|
|
974
1239
|
deps.log("info", `stopping ${slot.plan.id} (requested)`);
|
|
1240
|
+
if (recoverableHold(slot.plan, slot.state.hold)) {
|
|
1241
|
+
slot.state.hold.retryStopped = true;
|
|
1242
|
+
delete slot.state.hold.retryAt;
|
|
1243
|
+
delete slot.state.hold.probing;
|
|
1244
|
+
persistedHolds[slot.plan.id] = slot.state.hold;
|
|
1245
|
+
writeHolds();
|
|
1246
|
+
}
|
|
1247
|
+
else
|
|
1248
|
+
clearHold(slot);
|
|
975
1249
|
// An expected exit is never restarted, including one that needed SIGKILL.
|
|
976
1250
|
terminate(slot, () => {
|
|
977
1251
|
slot.state.status = "exited";
|
|
@@ -995,7 +1269,9 @@ export function createSupervisor(deps) {
|
|
|
995
1269
|
continue;
|
|
996
1270
|
}
|
|
997
1271
|
deps.log("info", `restarting ${slot.plan.id} (requested)`);
|
|
998
|
-
slot.
|
|
1272
|
+
if (slot.state.hold)
|
|
1273
|
+
deps.log("info", `releasing the hold on ${slot.plan.id} (requested)`);
|
|
1274
|
+
clearHold(slot);
|
|
999
1275
|
if (slot.plan.kind === "scheduled")
|
|
1000
1276
|
continue;
|
|
1001
1277
|
if (slot.child)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-side storage for the supervisor's unit holds and the per-run output
|
|
3
|
+
* read used to classify a unit's exit.
|
|
4
|
+
*
|
|
5
|
+
* holds.json (`DaemonPaths.holdsPath`, ~/.hq/daemon/holds.json) maps a unit id
|
|
6
|
+
* to its UnitHold. The daemon rewrites it whenever a hold starts or clears; a
|
|
7
|
+
* missing or unreadable file means no holds.
|
|
8
|
+
*/
|
|
9
|
+
import type { PersistedHolds } from "./supervisor.js";
|
|
10
|
+
/** How much of a run's output the exit classifier sees. */
|
|
11
|
+
export declare const EXIT_OUTPUT_TAIL_BYTES: number;
|
|
12
|
+
export declare function readUnitHolds(file: string): PersistedHolds;
|
|
13
|
+
export declare function writeUnitHolds(file: string, holds: PersistedHolds): void;
|
|
14
|
+
/** The size of a log file now, so a later read sees only what was written after. */
|
|
15
|
+
export declare function logOffset(file: string): number;
|
|
16
|
+
/**
|
|
17
|
+
* What was appended to `file` since `offset`, at most the last `maxBytes`.
|
|
18
|
+
* A file smaller than `offset` was rotated (copy-and-truncate), so it is read
|
|
19
|
+
* from the start.
|
|
20
|
+
*/
|
|
21
|
+
export declare function readLogSince(file: string, offset: number, maxBytes?: number): string;
|
|
22
|
+
//# sourceMappingURL=unit-holds.d.ts.map
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host-side storage for the supervisor's unit holds and the per-run output
|
|
3
|
+
* read used to classify a unit's exit.
|
|
4
|
+
*
|
|
5
|
+
* holds.json (`DaemonPaths.holdsPath`, ~/.hq/daemon/holds.json) maps a unit id
|
|
6
|
+
* to its UnitHold. The daemon rewrites it whenever a hold starts or clears; a
|
|
7
|
+
* missing or unreadable file means no holds.
|
|
8
|
+
*/
|
|
9
|
+
import * as fs from "node:fs";
|
|
10
|
+
import * as path from "node:path";
|
|
11
|
+
import { ensurePrivateDir } from "./paths.js";
|
|
12
|
+
/** How much of a run's output the exit classifier sees. */
|
|
13
|
+
export const EXIT_OUTPUT_TAIL_BYTES = 16 * 1024;
|
|
14
|
+
function isHold(value) {
|
|
15
|
+
if (!value || typeof value !== "object")
|
|
16
|
+
return false;
|
|
17
|
+
const h = value;
|
|
18
|
+
return ((h.kind === "permanent" || h.kind === "exhausted")
|
|
19
|
+
&& typeof h.cause === "string"
|
|
20
|
+
&& typeof h.detail === "string"
|
|
21
|
+
&& typeof h.since === "number"
|
|
22
|
+
&& typeof h.failures === "number"
|
|
23
|
+
&& typeof h.keyHash === "string");
|
|
24
|
+
}
|
|
25
|
+
export function readUnitHolds(file) {
|
|
26
|
+
let raw;
|
|
27
|
+
try {
|
|
28
|
+
raw = fs.readFileSync(file, "utf8");
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
if (err.code === "ENOENT")
|
|
32
|
+
return {};
|
|
33
|
+
throw err;
|
|
34
|
+
}
|
|
35
|
+
const parsed = JSON.parse(raw);
|
|
36
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
37
|
+
return {};
|
|
38
|
+
const holds = {};
|
|
39
|
+
for (const [id, hold] of Object.entries(parsed))
|
|
40
|
+
if (isHold(hold))
|
|
41
|
+
holds[id] = hold;
|
|
42
|
+
return holds;
|
|
43
|
+
}
|
|
44
|
+
export function writeUnitHolds(file, holds) {
|
|
45
|
+
ensurePrivateDir(path.dirname(file));
|
|
46
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
47
|
+
fs.writeFileSync(tmp, `${JSON.stringify(holds, null, 2)}\n`, { mode: 0o600 });
|
|
48
|
+
fs.renameSync(tmp, file);
|
|
49
|
+
}
|
|
50
|
+
/** The size of a log file now, so a later read sees only what was written after. */
|
|
51
|
+
export function logOffset(file) {
|
|
52
|
+
try {
|
|
53
|
+
return fs.statSync(file).size;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* What was appended to `file` since `offset`, at most the last `maxBytes`.
|
|
61
|
+
* A file smaller than `offset` was rotated (copy-and-truncate), so it is read
|
|
62
|
+
* from the start.
|
|
63
|
+
*/
|
|
64
|
+
export function readLogSince(file, offset, maxBytes = EXIT_OUTPUT_TAIL_BYTES) {
|
|
65
|
+
let fd;
|
|
66
|
+
try {
|
|
67
|
+
fd = fs.openSync(file, "r");
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return "";
|
|
71
|
+
}
|
|
72
|
+
try {
|
|
73
|
+
const size = fs.fstatSync(fd).size;
|
|
74
|
+
const from = Math.max(size < offset ? 0 : offset, size - maxBytes);
|
|
75
|
+
const length = size - from;
|
|
76
|
+
if (length <= 0)
|
|
77
|
+
return "";
|
|
78
|
+
const buf = Buffer.alloc(length);
|
|
79
|
+
const read = fs.readSync(fd, buf, 0, length, from);
|
|
80
|
+
return buf.subarray(0, read).toString("utf8");
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
fs.closeSync(fd);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=unit-holds.js.map
|
|
@@ -100,6 +100,15 @@ export interface SyncHealthDeps {
|
|
|
100
100
|
* unhealthy-sync signal, and `hq doctor --fix` has nothing local to refresh.
|
|
101
101
|
*/
|
|
102
102
|
localCompanyExists?: (slug: string) => boolean;
|
|
103
|
+
/**
|
|
104
|
+
* Whether the sync service can resolve the company for this machine's
|
|
105
|
+
* identity. Defaults to true unless a recent `hq doctor --fix` pull of it
|
|
106
|
+
* failed because the service has no such company for the caller or refused
|
|
107
|
+
* access (see company-resolution-state.ts). A stale journal for a company
|
|
108
|
+
* that fails this has nothing `--fix` can pull, so it is NA ("not on this
|
|
109
|
+
* machine") rather than an auto-fixable WARN.
|
|
110
|
+
*/
|
|
111
|
+
companySyncResolvable?: (slug: string, lastSync?: string | null) => boolean;
|
|
103
112
|
/**
|
|
104
113
|
* Whether HQ_SYNC_MANIFEST_DISABLED turns manifest uploads off on this
|
|
105
114
|
* machine. Defaults to reading the environment the way `hq sync manifest`
|