@indigoai-us/hq-cli 5.345.47 → 5.345.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +2 -1
  3. package/assets/scaffold/core/scripts/hq-session.sh +113 -7
  4. package/assets/scaffold/core/scripts/lib/session-scope-capability.sh +101 -0
  5. package/dist/command-catalog.generated.d.ts +12 -0
  6. package/dist/command-catalog.generated.js +15 -0
  7. package/dist/commands/anywhere-runtime-flag.js +5 -3
  8. package/dist/commands/core-hq-session-native.d.ts +2 -0
  9. package/dist/commands/core-hq-session-native.js +242 -52
  10. package/dist/commands/core-resolve-company-native.d.ts +1 -0
  11. package/dist/commands/core-resolve-company-native.js +41 -11
  12. package/dist/commands/feedback.d.ts +10 -0
  13. package/dist/commands/feedback.js +45 -0
  14. package/dist/commands/lanes-message.js +2 -2
  15. package/dist/commands/lanes.js +2 -2
  16. package/dist/commands/mcp-serve.js +17 -4
  17. package/dist/commands/secrets.d.ts +5 -0
  18. package/dist/commands/secrets.js +181 -2
  19. package/dist/commands/workers.d.ts +4 -1
  20. package/dist/commands/workers.js +57 -15
  21. package/dist/lib/anywhere-person-setting.d.ts +1 -0
  22. package/dist/lib/anywhere-person-setting.js +12 -0
  23. package/dist/lib/anywhere-runtime-fast-flag.js +5 -3
  24. package/dist/lib/anywhere-runtime-flag-key.d.ts +3 -1
  25. package/dist/lib/anywhere-runtime-flag-key.js +6 -1
  26. package/dist/lib/cli-flag-keys.d.ts +1 -0
  27. package/dist/lib/cli-flag-keys.js +1 -0
  28. package/dist/lib/core-utils/session-id.d.ts +6 -0
  29. package/dist/lib/core-utils/session-id.js +35 -0
  30. package/dist/lib/daemon/catalog.js +1 -1
  31. package/dist/lib/daemon/flag.d.ts +1 -1
  32. package/dist/lib/daemon/flag.js +20 -9
  33. package/dist/lib/daemon/lanes-reconcile-guard.d.ts +6 -5
  34. package/dist/lib/daemon/lanes-reconcile-guard.js +26 -7
  35. package/dist/lib/daemon/run.d.ts +6 -0
  36. package/dist/lib/daemon/run.js +36 -9
  37. package/dist/lib/daemon/server.d.ts +1 -1
  38. package/dist/lib/daemon/supervisor.d.ts +7 -1
  39. package/dist/lib/daemon/supervisor.js +50 -4
  40. package/dist/lib/lanes/reconcile.js +6 -4
  41. package/dist/lib/lanes/registry-flag.d.ts +11 -1
  42. package/dist/lib/lanes/registry-flag.js +68 -29
  43. package/dist/lib/lanes/watcher.js +63 -2
  44. package/dist/utils/resident-package-preload.js +8 -1
  45. package/dist/utils/self-update.js +6 -5
  46. package/dist/utils/version-gate.d.ts +11 -4
  47. package/dist/utils/version-gate.js +132 -41
  48. package/docs/anywhere-runtime.md +5 -3
  49. package/package.json +1 -1
@@ -1,12 +1,13 @@
1
1
  /**
2
2
  * Feature flags read by the daemon. The daemon runs for weeks, so each read
3
3
  * asks the registry again instead of holding a per-process answer, and it
4
- * reads as off unless the registry itself says on (or a local override does).
4
+ * reads as each flag's fallback unless the registry or a local override decides it.
5
5
  */
6
6
  import { createFlagClient, readLocalOverride, } from "@indigoai-us/hq-flags-client";
7
7
  import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
8
8
  import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
9
9
  import { resolveProcessFlagCompanyContext } from "../flag-registry.js";
10
+ import { HQ_ANYWHERE_RUNTIME_FLAG, hqAnywhereRuntimeLocallyDisabled } from "../anywhere-runtime-flag-key.js";
10
11
  /** Outposts hand running services to the next daemon on a CLI update instead of restarting them all. */
11
12
  export const SUPERVISOR_HANDOFF_FLAG = "daemon.supervisor-handoff";
12
13
  /** Self-updates build the new CLI beside the running one and swap it in, so `hq` never disappears mid-update. */
@@ -14,6 +15,9 @@ export const STAGED_SELF_UPDATE_FLAG = "cli.staged-self-update";
14
15
  const READ_TIMEOUT_MS = 5_000;
15
16
  export async function readDaemonFlag(flagKey, options = {}) {
16
17
  const env = options.env ?? process.env;
18
+ if (flagKey === HQ_ANYWHERE_RUNTIME_FLAG && hqAnywhereRuntimeLocallyDisabled(env))
19
+ return false;
20
+ const fallback = flagKey === HQ_ANYWHERE_RUNTIME_FLAG;
17
21
  const override = readLocalOverride(flagKey, env);
18
22
  const localOverrideMode = options.localOverrideMode ?? "respect";
19
23
  if (override !== null && (localOverrideMode === "respect" || (localOverrideMode === "off-only" && !override)))
@@ -28,10 +32,10 @@ export async function readDaemonFlag(flagKey, options = {}) {
28
32
  let client;
29
33
  try {
30
34
  const company = resolveProcessFlagCompanyContext(env);
31
- // Signed out reads as off without reaching the registry.
35
+ // Signed out reads as the flag fallback without reaching the registry.
32
36
  const token = await (options.getToken ?? (() => ensureCognitoIdToken({ interactive: false })))();
33
37
  if (typeof token !== "string" || token.trim() === "")
34
- return false;
38
+ return fallback;
35
39
  client = (options.createClient ?? createFlagClient)({
36
40
  endpoint: resolveFlagRegistryEndpoint(undefined, env),
37
41
  getToken: () => token,
@@ -42,20 +46,27 @@ export async function readDaemonFlag(flagKey, options = {}) {
42
46
  ...(company.companySlug ? { companyIdentifiers: [company.companySlug] } : {}),
43
47
  onError,
44
48
  });
45
- await client.ready();
46
- const resolution = client.explain(flagKey, { fallback: false });
47
- if (failed && options.throwOnError) {
49
+ try {
50
+ await client.ready();
51
+ }
52
+ catch (error) {
53
+ onError(error);
54
+ }
55
+ const resolution = client.explain(flagKey, { fallback });
56
+ if (failed && options.throwOnError && !fallback) {
48
57
  throw new Error("hq-flags registry read failed", { cause: reportedError });
49
58
  }
50
- return !failed && resolution.source === "snapshot" && resolution.value === true;
59
+ if (resolution.source === "snapshot")
60
+ return resolution.value === true;
61
+ return fallback && resolution.value === true;
51
62
  }
52
63
  catch (error) {
53
64
  if (!failed)
54
65
  onError(error);
55
- if (options.throwOnError && failed) {
66
+ if (options.throwOnError && failed && !fallback) {
56
67
  throw new Error("hq-flags registry read failed", { cause: reportedError ?? error });
57
68
  }
58
- return false;
69
+ return fallback;
59
70
  }
60
71
  finally {
61
72
  try {
@@ -7,12 +7,13 @@
7
7
  * JSON reads, no child process) and tells the supervisor how urgent the next
8
8
  * pass is:
9
9
  *
10
- * - `spawn`: something the 15-second cadence exists for may be due — lane
11
- * admission, a running lane's heartbeat, or a keep-alive resume (#1396).
10
+ * - `spawn`: something the desktop's 15-second cadence exists for may be due —
11
+ * local lane admission/heartbeat, a local keep-alive resume (#1396), or a
12
+ * pending question.
12
13
  * - `idle`: only work that ran on the pre-#1396 60-second cadence remains
13
- * (terminal lanes, worker-profile resumes, question aging, Codex rollout
14
- * retention, lead-autopilot registry). The supervisor runs it at most once
15
- * per `idleIntervalSec`.
14
+ * (terminal lanes, worker-profile resumes, question aging, outpost mirror
15
+ * refresh, Codex retention, lead-autopilot registry). The supervisor runs it
16
+ * at most once per `idleIntervalSec`.
16
17
  * - `skip`: the store holds no lane or question records, no lead registry and
17
18
  * no lanes config, so a reconcile pass has no input to act on.
18
19
  *
@@ -7,12 +7,13 @@
7
7
  * JSON reads, no child process) and tells the supervisor how urgent the next
8
8
  * pass is:
9
9
  *
10
- * - `spawn`: something the 15-second cadence exists for may be due — lane
11
- * admission, a running lane's heartbeat, or a keep-alive resume (#1396).
10
+ * - `spawn`: something the desktop's 15-second cadence exists for may be due —
11
+ * local lane admission/heartbeat, a local keep-alive resume (#1396), or a
12
+ * pending question.
12
13
  * - `idle`: only work that ran on the pre-#1396 60-second cadence remains
13
- * (terminal lanes, worker-profile resumes, question aging, Codex rollout
14
- * retention, lead-autopilot registry). The supervisor runs it at most once
15
- * per `idleIntervalSec`.
14
+ * (terminal lanes, worker-profile resumes, question aging, outpost mirror
15
+ * refresh, Codex retention, lead-autopilot registry). The supervisor runs it
16
+ * at most once per `idleIntervalSec`.
16
17
  * - `skip`: the store holds no lane or question records, no lead registry and
17
18
  * no lanes config, so a reconcile pass has no input to act on.
18
19
  *
@@ -38,6 +39,9 @@ export const LEAD_REGISTRY_FILE = path.join("workspace", "lane-briefs", "juniors
38
39
  * keep-alive, which is checked separately.
39
40
  */
40
41
  const QUIET_LANE_STATES = new Set(["done", "failed", "interrupted", "cancelled", "killed", "blocked"]);
42
+ const KNOWN_LANE_STATES = new Set([
43
+ "queued", "running", "awaiting_input", "blocked", "done", "failed", "interrupted", "cancelled", "killed",
44
+ ]);
41
45
  const realFs = {
42
46
  readdirSync: (dir) => fs.readdirSync(dir),
43
47
  readFileSync: (file, encoding) => fs.readFileSync(file, encoding),
@@ -63,9 +67,24 @@ function readRecord(fsDeps, file) {
63
67
  }
64
68
  return parsed;
65
69
  }
66
- /** A lane that may need the 15-second cadence. Unknown shapes count as urgent. */
70
+ /** A lane that may need the desktop's 15-second cadence. Unknown shapes count as urgent. */
67
71
  function laneIsUrgent(lane) {
68
- if (typeof lane.state !== "string" || !QUIET_LANE_STATES.has(lane.state))
72
+ if (typeof lane.state !== "string" || !KNOWN_LANE_STATES.has(lane.state))
73
+ return true;
74
+ const host = lane.host;
75
+ if (typeof host !== "object" || host === null || Array.isArray(host))
76
+ return true;
77
+ const hostRecord = host;
78
+ if (typeof hostRecord.id !== "string" || hostRecord.id.length === 0)
79
+ return true;
80
+ // This guard is installed only on desktops. The outpost's own daemon keeps
81
+ // its 15-second cadence; the desktop's outpost mirror/registry refresh can
82
+ // use the 60-second idle cadence. Pending questions remain urgent below.
83
+ if (hostRecord.kind === "outpost")
84
+ return false;
85
+ if (hostRecord.kind !== "local")
86
+ return true;
87
+ if (!QUIET_LANE_STATES.has(lane.state))
69
88
  return true;
70
89
  if (lane.keep_alive === true)
71
90
  return true;
@@ -14,6 +14,12 @@ import { type HostApp } from "./supervisor.js";
14
14
  export declare const LOG_ROTATE_BYTES: number;
15
15
  /** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
16
16
  export declare const RESTART_EXIT_CODE = 75;
17
+ /** A bounded shutdown deadline that can account for sync push persistence. */
18
+ export declare function createShutdownCap(): {
19
+ promise: Promise<"timeout">;
20
+ extendForSyncRunnerPersist(): void;
21
+ clear(): void;
22
+ };
17
23
  export interface DaemonLockConflictOutcome {
18
24
  exitCode: 0 | 1;
19
25
  stdout?: string;
@@ -45,12 +45,36 @@ import { stageHqChildBootstrap } from "./hq-child-bootstrap-stage.js";
45
45
  import { runnerSessionName } from "./remote-control.js";
46
46
  import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
47
47
  import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
48
- import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
48
+ import { STOP_ESCALATION_MS, SYNC_RUNNER_PERSIST_GRACE_MS, createSupervisor, } from "./supervisor.js";
49
49
  import { userToolDirs } from "./tools.js";
50
50
  import { daemonRunGuard } from "./lanes-reconcile-guard.js";
51
51
  export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
52
52
  /** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
53
53
  export const RESTART_EXIT_CODE = 75;
54
+ const NON_SYNC_SHUTDOWN_CAP_MS = STOP_ESCALATION_MS + 5_000;
55
+ const SYNC_RUNNER_SHUTDOWN_CAP_MS = STOP_ESCALATION_MS + SYNC_RUNNER_PERSIST_GRACE_MS + 5_000;
56
+ /** A bounded shutdown deadline that can account for sync push persistence. */
57
+ export function createShutdownCap() {
58
+ const startedAt = Date.now();
59
+ let deadline = startedAt + NON_SYNC_SHUTDOWN_CAP_MS;
60
+ let resolveCap;
61
+ let timer;
62
+ const promise = new Promise((resolve) => { resolveCap = resolve; });
63
+ const arm = () => {
64
+ if (timer)
65
+ clearTimeout(timer);
66
+ timer = setTimeout(() => resolveCap("timeout"), Math.max(0, deadline - Date.now()));
67
+ };
68
+ arm();
69
+ return {
70
+ promise,
71
+ extendForSyncRunnerPersist() {
72
+ deadline = Math.max(deadline, startedAt + SYNC_RUNNER_SHUTDOWN_CAP_MS);
73
+ arm();
74
+ },
75
+ clear() { clearTimeout(timer); },
76
+ };
77
+ }
54
78
  /** Preserve the legacy lock failure unless Anywhere explicitly enables the no-op behavior. */
55
79
  export function daemonLockConflictOutcome(lock, anywhereEnabled) {
56
80
  if (lock.ok)
@@ -772,6 +796,7 @@ export async function runDaemon(opts) {
772
796
  };
773
797
  // Set once the shutdown path exists below; a restart request before then is dropped.
774
798
  let restartDaemon = () => false;
799
+ let extendSyncShutdownCap;
775
800
  const supervisor = createSupervisor({
776
801
  machineType,
777
802
  host: opts.host,
@@ -794,6 +819,7 @@ export async function runDaemon(opts) {
794
819
  runGuard: daemonRunGuard,
795
820
  writeState: (state) => writeJsonAtomic(paths.statePath, state),
796
821
  log,
822
+ onSyncRunnerStopping: () => extendSyncShutdownCap?.(),
797
823
  syncOwnerBlockReason: () => {
798
824
  const config = readDaemonConfig(paths.configPath);
799
825
  const hqRoot = config.hqRoot ?? env.HQ_ROOT?.trim();
@@ -891,14 +917,15 @@ export async function runDaemon(opts) {
891
917
  stopping = true;
892
918
  log("info", `${why}; stopping services`);
893
919
  closeHqd();
894
- // The supervisor escalates to SIGKILL after STOP_ESCALATION_MS; the extra
895
- // margin only covers a child that survives even that.
896
- let capTimer;
897
- const cap = new Promise((r) => {
898
- capTimer = setTimeout(() => r("timeout"), STOP_ESCALATION_MS + 5_000);
899
- });
900
- void Promise.race([supervisor.stop(), cap]).then((result) => {
901
- clearTimeout(capTimer);
920
+ // Keep shutdown bounded, but let a stopping sync runner reach its persist deadline.
921
+ const cap = createShutdownCap();
922
+ extendSyncShutdownCap = cap.extendForSyncRunnerPersist;
923
+ if (supervisor.hasStoppingSyncRunner())
924
+ cap.extendForSyncRunnerPersist();
925
+ const servicesStopped = supervisor.stop();
926
+ void Promise.race([servicesStopped, cap.promise]).then((result) => {
927
+ cap.clear();
928
+ extendSyncShutdownCap = undefined;
902
929
  if (result === "timeout")
903
930
  log("error", "some services did not exit after SIGKILL; releasing the lock anyway");
904
931
  else
@@ -39,7 +39,7 @@ export interface HqdServerOptions {
39
39
  env?: NodeJS.ProcessEnv;
40
40
  /** Work-mesh root override for mesh.presence (tests). */
41
41
  workMeshRoot?: string;
42
- /** Test seam for the hq-anywhere-runtime default-off gate. */
42
+ /** Test seam for the hq-anywhere-runtime fail-safe gate. */
43
43
  mcpEnabled?: () => boolean | Promise<boolean>;
44
44
  /** Test seam for the live hq-flags read used by the daemon's bounded runtime monitor. */
45
45
  runtimeFlagReader?: (options: ReadDaemonFlagOptions) => Promise<boolean>;
@@ -13,7 +13,8 @@
13
13
  * - restart/trigger/stop requests from the CLI are polled and applied; a
14
14
  * request for a unit that is not planned yet re-reads the plan first;
15
15
  * - stop() sends SIGTERM to every child, escalates to SIGKILL after
16
- * STOP_ESCALATION_MS, and resolves once every child has exited;
16
+ * STOP_ESCALATION_MS except for a sync runner already stopping, which it lets
17
+ * finish persisting pending pushes before resolving;
17
18
  * - release() instead leaves every child running and reports the long-running
18
19
  * ones, so the next daemon can adopt them (`adopt`). Adopted units whose
19
20
  * command is unchanged keep running; changed ones are replaced one at a
@@ -26,6 +27,7 @@ import type { MachineType } from "./machine-type.js";
26
27
  import type { ControlRequest } from "./requests.js";
27
28
  import { syncPassResult } from "./sync-runner.js";
28
29
  export declare const STOP_ESCALATION_MS = 10000;
30
+ export declare const SYNC_RUNNER_PERSIST_GRACE_MS = 60000;
29
31
  export declare const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60000;
30
32
  /** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
31
33
  export declare const HANDOFF_STAGGER_MS = 15000;
@@ -98,6 +100,8 @@ export interface SupervisorDeps {
98
100
  restartDaemon?: () => void | boolean | Promise<boolean | void>;
99
101
  writeState: (state: DaemonState) => void;
100
102
  log: (level: "info" | "warn" | "error", msg: string) => void;
103
+ /** Extend the daemon shutdown deadline when a sync runner reports stopping. */
104
+ onSyncRunnerStopping?: () => void;
101
105
  now?: () => number;
102
106
  setTimeout?: (fn: () => void, ms: number) => unknown;
103
107
  clearTimeout?: (handle: unknown) => void;
@@ -120,6 +124,8 @@ export interface Supervisor {
120
124
  stop(): Promise<void>;
121
125
  reconcile(): void;
122
126
  state(): DaemonState;
127
+ /** True while a live sync runner has reported that it is persisting pushes. */
128
+ hasStoppingSyncRunner(): boolean;
123
129
  /** Stop supervising without signalling any child; returns the long-running units left running. */
124
130
  release(): ReleasedUnit[];
125
131
  }
@@ -13,7 +13,8 @@
13
13
  * - restart/trigger/stop requests from the CLI are polled and applied; a
14
14
  * request for a unit that is not planned yet re-reads the plan first;
15
15
  * - stop() sends SIGTERM to every child, escalates to SIGKILL after
16
- * STOP_ESCALATION_MS, and resolves once every child has exited;
16
+ * STOP_ESCALATION_MS except for a sync runner already stopping, which it lets
17
+ * finish persisting pending pushes before resolving;
17
18
  * - release() instead leaves every child running and reports the long-running
18
19
  * ones, so the next daemon can adopt them (`adopt`). Adopted units whose
19
20
  * command is unchanged keep running; changed ones are replaced one at a
@@ -27,6 +28,7 @@ const BACKOFF_BASE_MS = 1_000;
27
28
  const BACKOFF_MAX_MS = 60_000;
28
29
  const HEALTHY_RUN_MS = 60_000;
29
30
  export const STOP_ESCALATION_MS = 10_000;
31
+ export const SYNC_RUNNER_PERSIST_GRACE_MS = 60_000;
30
32
  const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
31
33
  export const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60_000;
32
34
  const MESH_OWNER_RECHECK_MS = 15_000;
@@ -91,6 +93,7 @@ export function createSupervisor(deps) {
91
93
  let handoffTimer;
92
94
  /** Every child process not yet known to have exited, including ones the slots have let go of. */
93
95
  const live = new Set();
96
+ const liveOwners = new Map();
94
97
  let drained = null;
95
98
  let onDrained = null;
96
99
  let deferredCliRestart = false;
@@ -247,6 +250,16 @@ export function createSupervisor(deps) {
247
250
  deps.log("error", `could not send ${signal} to ${slot.plan.id}: ${errorMessage(err)}`);
248
251
  }
249
252
  };
253
+ const syncRunnerIsStopping = (slot) => slot?.plan.service === "sync" && slot.lastProtocolStatus === "stopping";
254
+ const scheduleSyncRunnerPersistEscalation = (slot, child) => {
255
+ slot.killTimer = setT(() => {
256
+ slot.killTimer = undefined;
257
+ if (slot.child !== child)
258
+ return;
259
+ deps.log("warn", `${slot.plan.id} did not finish persisting pending pushes within ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s; sending SIGKILL${lastProtocolHint(slot)}`);
260
+ signalChild(slot, child, "SIGKILL");
261
+ }, SYNC_RUNNER_PERSIST_GRACE_MS);
262
+ };
250
263
  /**
251
264
  * Stop the slot's child (if any), then run `then` once it has exited. With
252
265
  * `drain`, a unit that has a drain signal gets it first and SIGTERM only
@@ -276,6 +289,11 @@ export function createSupervisor(deps) {
276
289
  slot.killTimer = undefined;
277
290
  if (slot.child !== child)
278
291
  return;
292
+ if (syncRunnerIsStopping(slot)) {
293
+ deps.log("info", `${slot.plan.id} is stopping; allowing ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s for pending pushes to persist`);
294
+ scheduleSyncRunnerPersistEscalation(slot, child);
295
+ return;
296
+ }
279
297
  deps.log("warn", `${slot.plan.id} did not exit within ${killGraceMs / 1_000} s of SIGTERM; sending SIGKILL${lastProtocolHint(slot)}`);
280
298
  signalChild(slot, child, "SIGKILL");
281
299
  }, killGraceMs);
@@ -313,6 +331,7 @@ export function createSupervisor(deps) {
313
331
  };
314
332
  const watch = (slot, child) => {
315
333
  live.add(child);
334
+ liveOwners.set(child, slot);
316
335
  let ended = false;
317
336
  const timeoutMs = slot.plan.heartbeatTimeoutMs;
318
337
  const decoder = timeoutMs ? new StringDecoder("utf8") : null;
@@ -350,6 +369,8 @@ export function createSupervisor(deps) {
350
369
  slot.syncPassActive = false;
351
370
  else if (event.type === "status" && typeof event.state === "string") {
352
371
  slot.lastProtocolStatus = event.state;
372
+ if (event.state === "stopping")
373
+ deps.onSyncRunnerStopping?.();
353
374
  if (event.state === "stopped" || event.state === "waiting-for-lock")
354
375
  slot.syncPassActive = false;
355
376
  else if (slot.syncPassActive === undefined && (event.state === "starting" || event.state === "running")) {
@@ -405,6 +426,7 @@ export function createSupervisor(deps) {
405
426
  return;
406
427
  ended = true;
407
428
  live.delete(child);
429
+ liveOwners.delete(child);
408
430
  if (live.size === 0)
409
431
  onDrained?.();
410
432
  if (slot.child !== child)
@@ -991,10 +1013,8 @@ export function createSupervisor(deps) {
991
1013
  clearT(handoffTimer);
992
1014
  clearDeferredCliRestart();
993
1015
  clearPendingSyncCommandRestart();
994
- const childOwners = new Map();
1016
+ const childOwners = new Map(liveOwners);
995
1017
  for (const slot of slots.values()) {
996
- if (slot.child)
997
- childOwners.set(slot.child, slot);
998
1018
  clearTimer(slot);
999
1019
  clearHeartbeatTimer(slot);
1000
1020
  slot.child = undefined;
@@ -1020,9 +1040,16 @@ export function createSupervisor(deps) {
1020
1040
  resolve();
1021
1041
  return;
1022
1042
  }
1043
+ const persistingSyncRunners = new Set();
1044
+ let persistEscalation;
1023
1045
  const escalation = setT(() => {
1024
1046
  for (const child of live) {
1025
1047
  const owner = childOwners.get(child);
1048
+ if (owner && syncRunnerIsStopping(owner)) {
1049
+ deps.log("info", `${owner.plan.id} is stopping; allowing ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s for pending pushes to persist`);
1050
+ persistingSyncRunners.add(child);
1051
+ continue;
1052
+ }
1026
1053
  deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not exit within ${STOP_ESCALATION_MS / 1_000} s of SIGTERM; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
1027
1054
  try {
1028
1055
  child.kill("SIGKILL");
@@ -1031,10 +1058,28 @@ export function createSupervisor(deps) {
1031
1058
  deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
1032
1059
  }
1033
1060
  }
1061
+ if (persistingSyncRunners.size > 0) {
1062
+ persistEscalation = setT(() => {
1063
+ for (const child of persistingSyncRunners) {
1064
+ if (!live.has(child))
1065
+ continue;
1066
+ const owner = childOwners.get(child);
1067
+ deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not finish persisting pending pushes within ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
1068
+ try {
1069
+ child.kill("SIGKILL");
1070
+ }
1071
+ catch (err) {
1072
+ deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
1073
+ }
1074
+ }
1075
+ }, SYNC_RUNNER_PERSIST_GRACE_MS);
1076
+ }
1034
1077
  }, STOP_ESCALATION_MS);
1035
1078
  onDrained = () => {
1036
1079
  onDrained = null;
1037
1080
  clearT(escalation);
1081
+ if (persistEscalation !== undefined)
1082
+ clearT(persistEscalation);
1038
1083
  resolve();
1039
1084
  };
1040
1085
  });
@@ -1042,6 +1087,7 @@ export function createSupervisor(deps) {
1042
1087
  },
1043
1088
  reconcile,
1044
1089
  state: snapshot,
1090
+ hasStoppingSyncRunner: () => [...liveOwners.values()].some(syncRunnerIsStopping),
1045
1091
  release() {
1046
1092
  const released = [];
1047
1093
  for (const slot of slots.values()) {
@@ -22,7 +22,7 @@ import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDa
22
22
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
23
23
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
24
24
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
25
- import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
25
+ import { readLanesRegistryFlag, readLanesRegistryFlagFailOff, readLanesRegistryFlagFailOpen } from "./registry-flag.js";
26
26
  import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
27
27
  import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
28
28
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
@@ -1216,6 +1216,7 @@ export async function runPassBody(hqRoot, deps) {
1216
1216
  }
1217
1217
  }
1218
1218
  const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
1219
+ const readRegistryFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
1219
1220
  const storyCloseCompanies = new Map();
1220
1221
  for (const lane of lanes) {
1221
1222
  if (lane.depth < 1 || lane.state !== "interrupted" || lane.keep_alive === true ||
@@ -1303,10 +1304,10 @@ export async function runPassBody(hqRoot, deps) {
1303
1304
  readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
1304
1305
  readRegistryFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
1305
1306
  hasBlockedKeepAliveLane
1306
- ? readRegistryFlag(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
1307
+ ? readRegistryFlagFailOpen(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
1307
1308
  : Promise.resolve(false),
1308
1309
  readRegistryFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
1309
- readRegistryFlag(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
1310
+ readRegistryFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
1310
1311
  ]);
1311
1312
  keepAliveFlagByCompany.set(key, keepAliveEnabled);
1312
1313
  blockedResumeFlagByCompany.set(key, blockedResumeEnabled);
@@ -3585,6 +3586,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
3585
3586
  if (company === null)
3586
3587
  return summary;
3587
3588
  const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
3589
+ const readFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
3588
3590
  const options = { company, env: deps.env ?? process.env };
3589
3591
  let keepAliveEnabled;
3590
3592
  let lineageAware;
@@ -3595,7 +3597,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
3595
3597
  readFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
3596
3598
  readFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
3597
3599
  readFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
3598
- readFlag(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
3600
+ readFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
3599
3601
  ]);
3600
3602
  }
3601
3603
  catch (err) {
@@ -19,12 +19,22 @@ export interface ReadLanesFlagOptions {
19
19
  reportFailure?: (error: unknown) => void;
20
20
  }
21
21
  export declare function readLanesFlagLocalOverride(flagKey: string, env?: NodeJS.ProcessEnv): boolean | null;
22
+ export type LanesRegistryFlagResult = {
23
+ state: "resolved";
24
+ value: boolean;
25
+ } | {
26
+ state: "unknown";
27
+ };
22
28
  /**
23
29
  * Read a lanes flag once for each company identity in this CLI process. The
24
- * cache contains only an in-flight or settled boolean promise; it is never
30
+ * cache contains only an in-flight or settled decision result; it is never
25
31
  * written to disk. A local override is checked on every call and wins first.
26
32
  */
27
33
  export declare function readLanesRegistryFlag(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
34
+ /** Read a lanes flag while preserving registry failure as an explicit unknown state. */
35
+ export declare function readLanesRegistryFlagResult(flagKey: string, options?: ReadLanesFlagOptions): Promise<LanesRegistryFlagResult>;
36
+ /** Delivery and wake call sites opt in to availability when the registry is unreadable. */
37
+ export declare function readLanesRegistryFlagFailOpen(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
28
38
  /**
29
39
  * Flag readers normally report their own failures and return the safe default.
30
40
  * Keep enforcement callers safe if an injected or unexpected reader rejects.