@indigoai-us/hq-cli 5.345.47 → 5.345.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +2 -1
- package/assets/scaffold/core/scripts/hq-session.sh +113 -7
- package/assets/scaffold/core/scripts/lib/session-scope-capability.sh +101 -0
- package/dist/command-catalog.generated.d.ts +12 -0
- package/dist/command-catalog.generated.js +15 -0
- package/dist/commands/anywhere-runtime-flag.js +5 -3
- package/dist/commands/core-hq-session-native.d.ts +2 -0
- package/dist/commands/core-hq-session-native.js +242 -52
- package/dist/commands/core-resolve-company-native.d.ts +1 -0
- package/dist/commands/core-resolve-company-native.js +41 -11
- package/dist/commands/feedback.d.ts +10 -0
- package/dist/commands/feedback.js +45 -0
- package/dist/commands/lanes-message.js +2 -2
- package/dist/commands/lanes.js +2 -2
- package/dist/commands/mcp-serve.js +17 -4
- package/dist/commands/secrets.d.ts +5 -0
- package/dist/commands/secrets.js +181 -2
- package/dist/commands/workers.d.ts +4 -1
- package/dist/commands/workers.js +57 -15
- package/dist/lib/anywhere-person-setting.d.ts +1 -0
- package/dist/lib/anywhere-person-setting.js +12 -0
- package/dist/lib/anywhere-runtime-fast-flag.js +5 -3
- package/dist/lib/anywhere-runtime-flag-key.d.ts +3 -1
- package/dist/lib/anywhere-runtime-flag-key.js +6 -1
- package/dist/lib/cli-flag-keys.d.ts +1 -0
- package/dist/lib/cli-flag-keys.js +1 -0
- package/dist/lib/core-utils/session-id.d.ts +6 -0
- package/dist/lib/core-utils/session-id.js +35 -0
- package/dist/lib/daemon/catalog.js +1 -1
- package/dist/lib/daemon/flag.d.ts +1 -1
- package/dist/lib/daemon/flag.js +20 -9
- package/dist/lib/daemon/lanes-reconcile-guard.d.ts +6 -5
- package/dist/lib/daemon/lanes-reconcile-guard.js +26 -7
- package/dist/lib/daemon/run.d.ts +6 -0
- package/dist/lib/daemon/run.js +36 -9
- package/dist/lib/daemon/server.d.ts +1 -1
- package/dist/lib/daemon/supervisor.d.ts +7 -1
- package/dist/lib/daemon/supervisor.js +50 -4
- package/dist/lib/lanes/reconcile.js +6 -4
- package/dist/lib/lanes/registry-flag.d.ts +11 -1
- package/dist/lib/lanes/registry-flag.js +68 -29
- package/dist/lib/lanes/watcher.js +63 -2
- package/dist/utils/resident-package-preload.js +8 -1
- package/dist/utils/self-update.js +6 -5
- package/dist/utils/version-gate.d.ts +11 -4
- package/dist/utils/version-gate.js +132 -41
- package/docs/anywhere-runtime.md +5 -3
- package/package.json +1 -1
package/dist/lib/daemon/flag.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Feature flags read by the daemon. The daemon runs for weeks, so each read
|
|
3
3
|
* asks the registry again instead of holding a per-process answer, and it
|
|
4
|
-
* reads as
|
|
4
|
+
* reads as each flag's fallback unless the registry or a local override decides it.
|
|
5
5
|
*/
|
|
6
6
|
import { createFlagClient, readLocalOverride, } from "@indigoai-us/hq-flags-client";
|
|
7
7
|
import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
|
|
8
8
|
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
9
9
|
import { resolveProcessFlagCompanyContext } from "../flag-registry.js";
|
|
10
|
+
import { HQ_ANYWHERE_RUNTIME_FLAG, hqAnywhereRuntimeLocallyDisabled } from "../anywhere-runtime-flag-key.js";
|
|
10
11
|
/** Outposts hand running services to the next daemon on a CLI update instead of restarting them all. */
|
|
11
12
|
export const SUPERVISOR_HANDOFF_FLAG = "daemon.supervisor-handoff";
|
|
12
13
|
/** Self-updates build the new CLI beside the running one and swap it in, so `hq` never disappears mid-update. */
|
|
@@ -14,6 +15,9 @@ export const STAGED_SELF_UPDATE_FLAG = "cli.staged-self-update";
|
|
|
14
15
|
const READ_TIMEOUT_MS = 5_000;
|
|
15
16
|
export async function readDaemonFlag(flagKey, options = {}) {
|
|
16
17
|
const env = options.env ?? process.env;
|
|
18
|
+
if (flagKey === HQ_ANYWHERE_RUNTIME_FLAG && hqAnywhereRuntimeLocallyDisabled(env))
|
|
19
|
+
return false;
|
|
20
|
+
const fallback = flagKey === HQ_ANYWHERE_RUNTIME_FLAG;
|
|
17
21
|
const override = readLocalOverride(flagKey, env);
|
|
18
22
|
const localOverrideMode = options.localOverrideMode ?? "respect";
|
|
19
23
|
if (override !== null && (localOverrideMode === "respect" || (localOverrideMode === "off-only" && !override)))
|
|
@@ -28,10 +32,10 @@ export async function readDaemonFlag(flagKey, options = {}) {
|
|
|
28
32
|
let client;
|
|
29
33
|
try {
|
|
30
34
|
const company = resolveProcessFlagCompanyContext(env);
|
|
31
|
-
// Signed out reads as
|
|
35
|
+
// Signed out reads as the flag fallback without reaching the registry.
|
|
32
36
|
const token = await (options.getToken ?? (() => ensureCognitoIdToken({ interactive: false })))();
|
|
33
37
|
if (typeof token !== "string" || token.trim() === "")
|
|
34
|
-
return
|
|
38
|
+
return fallback;
|
|
35
39
|
client = (options.createClient ?? createFlagClient)({
|
|
36
40
|
endpoint: resolveFlagRegistryEndpoint(undefined, env),
|
|
37
41
|
getToken: () => token,
|
|
@@ -42,20 +46,27 @@ export async function readDaemonFlag(flagKey, options = {}) {
|
|
|
42
46
|
...(company.companySlug ? { companyIdentifiers: [company.companySlug] } : {}),
|
|
43
47
|
onError,
|
|
44
48
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
try {
|
|
50
|
+
await client.ready();
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
onError(error);
|
|
54
|
+
}
|
|
55
|
+
const resolution = client.explain(flagKey, { fallback });
|
|
56
|
+
if (failed && options.throwOnError && !fallback) {
|
|
48
57
|
throw new Error("hq-flags registry read failed", { cause: reportedError });
|
|
49
58
|
}
|
|
50
|
-
|
|
59
|
+
if (resolution.source === "snapshot")
|
|
60
|
+
return resolution.value === true;
|
|
61
|
+
return fallback && resolution.value === true;
|
|
51
62
|
}
|
|
52
63
|
catch (error) {
|
|
53
64
|
if (!failed)
|
|
54
65
|
onError(error);
|
|
55
|
-
if (options.throwOnError && failed) {
|
|
66
|
+
if (options.throwOnError && failed && !fallback) {
|
|
56
67
|
throw new Error("hq-flags registry read failed", { cause: reportedError ?? error });
|
|
57
68
|
}
|
|
58
|
-
return
|
|
69
|
+
return fallback;
|
|
59
70
|
}
|
|
60
71
|
finally {
|
|
61
72
|
try {
|
|
@@ -7,12 +7,13 @@
|
|
|
7
7
|
* JSON reads, no child process) and tells the supervisor how urgent the next
|
|
8
8
|
* pass is:
|
|
9
9
|
*
|
|
10
|
-
* - `spawn`: something the 15-second cadence exists for may be due —
|
|
11
|
-
*
|
|
10
|
+
* - `spawn`: something the desktop's 15-second cadence exists for may be due —
|
|
11
|
+
* local lane admission/heartbeat, a local keep-alive resume (#1396), or a
|
|
12
|
+
* pending question.
|
|
12
13
|
* - `idle`: only work that ran on the pre-#1396 60-second cadence remains
|
|
13
|
-
* (terminal lanes, worker-profile resumes, question aging,
|
|
14
|
-
* retention, lead-autopilot registry). The supervisor runs it
|
|
15
|
-
* per `idleIntervalSec`.
|
|
14
|
+
* (terminal lanes, worker-profile resumes, question aging, outpost mirror
|
|
15
|
+
* refresh, Codex retention, lead-autopilot registry). The supervisor runs it
|
|
16
|
+
* at most once per `idleIntervalSec`.
|
|
16
17
|
* - `skip`: the store holds no lane or question records, no lead registry and
|
|
17
18
|
* no lanes config, so a reconcile pass has no input to act on.
|
|
18
19
|
*
|
|
@@ -7,12 +7,13 @@
|
|
|
7
7
|
* JSON reads, no child process) and tells the supervisor how urgent the next
|
|
8
8
|
* pass is:
|
|
9
9
|
*
|
|
10
|
-
* - `spawn`: something the 15-second cadence exists for may be due —
|
|
11
|
-
*
|
|
10
|
+
* - `spawn`: something the desktop's 15-second cadence exists for may be due —
|
|
11
|
+
* local lane admission/heartbeat, a local keep-alive resume (#1396), or a
|
|
12
|
+
* pending question.
|
|
12
13
|
* - `idle`: only work that ran on the pre-#1396 60-second cadence remains
|
|
13
|
-
* (terminal lanes, worker-profile resumes, question aging,
|
|
14
|
-
* retention, lead-autopilot registry). The supervisor runs it
|
|
15
|
-
* per `idleIntervalSec`.
|
|
14
|
+
* (terminal lanes, worker-profile resumes, question aging, outpost mirror
|
|
15
|
+
* refresh, Codex retention, lead-autopilot registry). The supervisor runs it
|
|
16
|
+
* at most once per `idleIntervalSec`.
|
|
16
17
|
* - `skip`: the store holds no lane or question records, no lead registry and
|
|
17
18
|
* no lanes config, so a reconcile pass has no input to act on.
|
|
18
19
|
*
|
|
@@ -38,6 +39,9 @@ export const LEAD_REGISTRY_FILE = path.join("workspace", "lane-briefs", "juniors
|
|
|
38
39
|
* keep-alive, which is checked separately.
|
|
39
40
|
*/
|
|
40
41
|
const QUIET_LANE_STATES = new Set(["done", "failed", "interrupted", "cancelled", "killed", "blocked"]);
|
|
42
|
+
const KNOWN_LANE_STATES = new Set([
|
|
43
|
+
"queued", "running", "awaiting_input", "blocked", "done", "failed", "interrupted", "cancelled", "killed",
|
|
44
|
+
]);
|
|
41
45
|
const realFs = {
|
|
42
46
|
readdirSync: (dir) => fs.readdirSync(dir),
|
|
43
47
|
readFileSync: (file, encoding) => fs.readFileSync(file, encoding),
|
|
@@ -63,9 +67,24 @@ function readRecord(fsDeps, file) {
|
|
|
63
67
|
}
|
|
64
68
|
return parsed;
|
|
65
69
|
}
|
|
66
|
-
/** A lane that may need the 15-second cadence. Unknown shapes count as urgent. */
|
|
70
|
+
/** A lane that may need the desktop's 15-second cadence. Unknown shapes count as urgent. */
|
|
67
71
|
function laneIsUrgent(lane) {
|
|
68
|
-
if (typeof lane.state !== "string" || !
|
|
72
|
+
if (typeof lane.state !== "string" || !KNOWN_LANE_STATES.has(lane.state))
|
|
73
|
+
return true;
|
|
74
|
+
const host = lane.host;
|
|
75
|
+
if (typeof host !== "object" || host === null || Array.isArray(host))
|
|
76
|
+
return true;
|
|
77
|
+
const hostRecord = host;
|
|
78
|
+
if (typeof hostRecord.id !== "string" || hostRecord.id.length === 0)
|
|
79
|
+
return true;
|
|
80
|
+
// This guard is installed only on desktops. The outpost's own daemon keeps
|
|
81
|
+
// its 15-second cadence; the desktop's outpost mirror/registry refresh can
|
|
82
|
+
// use the 60-second idle cadence. Pending questions remain urgent below.
|
|
83
|
+
if (hostRecord.kind === "outpost")
|
|
84
|
+
return false;
|
|
85
|
+
if (hostRecord.kind !== "local")
|
|
86
|
+
return true;
|
|
87
|
+
if (!QUIET_LANE_STATES.has(lane.state))
|
|
69
88
|
return true;
|
|
70
89
|
if (lane.keep_alive === true)
|
|
71
90
|
return true;
|
package/dist/lib/daemon/run.d.ts
CHANGED
|
@@ -14,6 +14,12 @@ import { type HostApp } from "./supervisor.js";
|
|
|
14
14
|
export declare const LOG_ROTATE_BYTES: number;
|
|
15
15
|
/** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
|
|
16
16
|
export declare const RESTART_EXIT_CODE = 75;
|
|
17
|
+
/** A bounded shutdown deadline that can account for sync push persistence. */
|
|
18
|
+
export declare function createShutdownCap(): {
|
|
19
|
+
promise: Promise<"timeout">;
|
|
20
|
+
extendForSyncRunnerPersist(): void;
|
|
21
|
+
clear(): void;
|
|
22
|
+
};
|
|
17
23
|
export interface DaemonLockConflictOutcome {
|
|
18
24
|
exitCode: 0 | 1;
|
|
19
25
|
stdout?: string;
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -45,12 +45,36 @@ import { stageHqChildBootstrap } from "./hq-child-bootstrap-stage.js";
|
|
|
45
45
|
import { runnerSessionName } from "./remote-control.js";
|
|
46
46
|
import { adoptChild, createChildTracker, finishSuccession, handoffEligible, readProcStart, stopOrphans, successionPaths, systemdKillMode, takeSuccession, writeHandoff, } from "./succession.js";
|
|
47
47
|
import { listSyncRunnerProcesses, readWatchRunnerOwnerStatus, syncHeartbeatTimeoutMs, syncRunnerNodeOptions, watchRunnerOwnerStatusPath, } from "./sync-runner.js";
|
|
48
|
-
import { STOP_ESCALATION_MS, createSupervisor, } from "./supervisor.js";
|
|
48
|
+
import { STOP_ESCALATION_MS, SYNC_RUNNER_PERSIST_GRACE_MS, createSupervisor, } from "./supervisor.js";
|
|
49
49
|
import { userToolDirs } from "./tools.js";
|
|
50
50
|
import { daemonRunGuard } from "./lanes-reconcile-guard.js";
|
|
51
51
|
export const LOG_ROTATE_BYTES = 5 * 1024 * 1024;
|
|
52
52
|
/** Exit code for "restart me": non-zero so every service manager relaunches (Windows tasks only restart on failure). */
|
|
53
53
|
export const RESTART_EXIT_CODE = 75;
|
|
54
|
+
const NON_SYNC_SHUTDOWN_CAP_MS = STOP_ESCALATION_MS + 5_000;
|
|
55
|
+
const SYNC_RUNNER_SHUTDOWN_CAP_MS = STOP_ESCALATION_MS + SYNC_RUNNER_PERSIST_GRACE_MS + 5_000;
|
|
56
|
+
/** A bounded shutdown deadline that can account for sync push persistence. */
|
|
57
|
+
export function createShutdownCap() {
|
|
58
|
+
const startedAt = Date.now();
|
|
59
|
+
let deadline = startedAt + NON_SYNC_SHUTDOWN_CAP_MS;
|
|
60
|
+
let resolveCap;
|
|
61
|
+
let timer;
|
|
62
|
+
const promise = new Promise((resolve) => { resolveCap = resolve; });
|
|
63
|
+
const arm = () => {
|
|
64
|
+
if (timer)
|
|
65
|
+
clearTimeout(timer);
|
|
66
|
+
timer = setTimeout(() => resolveCap("timeout"), Math.max(0, deadline - Date.now()));
|
|
67
|
+
};
|
|
68
|
+
arm();
|
|
69
|
+
return {
|
|
70
|
+
promise,
|
|
71
|
+
extendForSyncRunnerPersist() {
|
|
72
|
+
deadline = Math.max(deadline, startedAt + SYNC_RUNNER_SHUTDOWN_CAP_MS);
|
|
73
|
+
arm();
|
|
74
|
+
},
|
|
75
|
+
clear() { clearTimeout(timer); },
|
|
76
|
+
};
|
|
77
|
+
}
|
|
54
78
|
/** Preserve the legacy lock failure unless Anywhere explicitly enables the no-op behavior. */
|
|
55
79
|
export function daemonLockConflictOutcome(lock, anywhereEnabled) {
|
|
56
80
|
if (lock.ok)
|
|
@@ -772,6 +796,7 @@ export async function runDaemon(opts) {
|
|
|
772
796
|
};
|
|
773
797
|
// Set once the shutdown path exists below; a restart request before then is dropped.
|
|
774
798
|
let restartDaemon = () => false;
|
|
799
|
+
let extendSyncShutdownCap;
|
|
775
800
|
const supervisor = createSupervisor({
|
|
776
801
|
machineType,
|
|
777
802
|
host: opts.host,
|
|
@@ -794,6 +819,7 @@ export async function runDaemon(opts) {
|
|
|
794
819
|
runGuard: daemonRunGuard,
|
|
795
820
|
writeState: (state) => writeJsonAtomic(paths.statePath, state),
|
|
796
821
|
log,
|
|
822
|
+
onSyncRunnerStopping: () => extendSyncShutdownCap?.(),
|
|
797
823
|
syncOwnerBlockReason: () => {
|
|
798
824
|
const config = readDaemonConfig(paths.configPath);
|
|
799
825
|
const hqRoot = config.hqRoot ?? env.HQ_ROOT?.trim();
|
|
@@ -891,14 +917,15 @@ export async function runDaemon(opts) {
|
|
|
891
917
|
stopping = true;
|
|
892
918
|
log("info", `${why}; stopping services`);
|
|
893
919
|
closeHqd();
|
|
894
|
-
//
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
void Promise.race([
|
|
901
|
-
|
|
920
|
+
// Keep shutdown bounded, but let a stopping sync runner reach its persist deadline.
|
|
921
|
+
const cap = createShutdownCap();
|
|
922
|
+
extendSyncShutdownCap = cap.extendForSyncRunnerPersist;
|
|
923
|
+
if (supervisor.hasStoppingSyncRunner())
|
|
924
|
+
cap.extendForSyncRunnerPersist();
|
|
925
|
+
const servicesStopped = supervisor.stop();
|
|
926
|
+
void Promise.race([servicesStopped, cap.promise]).then((result) => {
|
|
927
|
+
cap.clear();
|
|
928
|
+
extendSyncShutdownCap = undefined;
|
|
902
929
|
if (result === "timeout")
|
|
903
930
|
log("error", "some services did not exit after SIGKILL; releasing the lock anyway");
|
|
904
931
|
else
|
|
@@ -39,7 +39,7 @@ export interface HqdServerOptions {
|
|
|
39
39
|
env?: NodeJS.ProcessEnv;
|
|
40
40
|
/** Work-mesh root override for mesh.presence (tests). */
|
|
41
41
|
workMeshRoot?: string;
|
|
42
|
-
/** Test seam for the hq-anywhere-runtime
|
|
42
|
+
/** Test seam for the hq-anywhere-runtime fail-safe gate. */
|
|
43
43
|
mcpEnabled?: () => boolean | Promise<boolean>;
|
|
44
44
|
/** Test seam for the live hq-flags read used by the daemon's bounded runtime monitor. */
|
|
45
45
|
runtimeFlagReader?: (options: ReadDaemonFlagOptions) => Promise<boolean>;
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
* - restart/trigger/stop requests from the CLI are polled and applied; a
|
|
14
14
|
* request for a unit that is not planned yet re-reads the plan first;
|
|
15
15
|
* - stop() sends SIGTERM to every child, escalates to SIGKILL after
|
|
16
|
-
* STOP_ESCALATION_MS
|
|
16
|
+
* STOP_ESCALATION_MS except for a sync runner already stopping, which it lets
|
|
17
|
+
* finish persisting pending pushes before resolving;
|
|
17
18
|
* - release() instead leaves every child running and reports the long-running
|
|
18
19
|
* ones, so the next daemon can adopt them (`adopt`). Adopted units whose
|
|
19
20
|
* command is unchanged keep running; changed ones are replaced one at a
|
|
@@ -26,6 +27,7 @@ import type { MachineType } from "./machine-type.js";
|
|
|
26
27
|
import type { ControlRequest } from "./requests.js";
|
|
27
28
|
import { syncPassResult } from "./sync-runner.js";
|
|
28
29
|
export declare const STOP_ESCALATION_MS = 10000;
|
|
30
|
+
export declare const SYNC_RUNNER_PERSIST_GRACE_MS = 60000;
|
|
29
31
|
export declare const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60000;
|
|
30
32
|
/** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
|
|
31
33
|
export declare const HANDOFF_STAGGER_MS = 15000;
|
|
@@ -98,6 +100,8 @@ export interface SupervisorDeps {
|
|
|
98
100
|
restartDaemon?: () => void | boolean | Promise<boolean | void>;
|
|
99
101
|
writeState: (state: DaemonState) => void;
|
|
100
102
|
log: (level: "info" | "warn" | "error", msg: string) => void;
|
|
103
|
+
/** Extend the daemon shutdown deadline when a sync runner reports stopping. */
|
|
104
|
+
onSyncRunnerStopping?: () => void;
|
|
101
105
|
now?: () => number;
|
|
102
106
|
setTimeout?: (fn: () => void, ms: number) => unknown;
|
|
103
107
|
clearTimeout?: (handle: unknown) => void;
|
|
@@ -120,6 +124,8 @@ export interface Supervisor {
|
|
|
120
124
|
stop(): Promise<void>;
|
|
121
125
|
reconcile(): void;
|
|
122
126
|
state(): DaemonState;
|
|
127
|
+
/** True while a live sync runner has reported that it is persisting pushes. */
|
|
128
|
+
hasStoppingSyncRunner(): boolean;
|
|
123
129
|
/** Stop supervising without signalling any child; returns the long-running units left running. */
|
|
124
130
|
release(): ReleasedUnit[];
|
|
125
131
|
}
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
* - restart/trigger/stop requests from the CLI are polled and applied; a
|
|
14
14
|
* request for a unit that is not planned yet re-reads the plan first;
|
|
15
15
|
* - stop() sends SIGTERM to every child, escalates to SIGKILL after
|
|
16
|
-
* STOP_ESCALATION_MS
|
|
16
|
+
* STOP_ESCALATION_MS except for a sync runner already stopping, which it lets
|
|
17
|
+
* finish persisting pending pushes before resolving;
|
|
17
18
|
* - release() instead leaves every child running and reports the long-running
|
|
18
19
|
* ones, so the next daemon can adopt them (`adopt`). Adopted units whose
|
|
19
20
|
* command is unchanged keep running; changed ones are replaced one at a
|
|
@@ -27,6 +28,7 @@ const BACKOFF_BASE_MS = 1_000;
|
|
|
27
28
|
const BACKOFF_MAX_MS = 60_000;
|
|
28
29
|
const HEALTHY_RUN_MS = 60_000;
|
|
29
30
|
export const STOP_ESCALATION_MS = 10_000;
|
|
31
|
+
export const SYNC_RUNNER_PERSIST_GRACE_MS = 60_000;
|
|
30
32
|
const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
|
|
31
33
|
export const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60_000;
|
|
32
34
|
const MESH_OWNER_RECHECK_MS = 15_000;
|
|
@@ -91,6 +93,7 @@ export function createSupervisor(deps) {
|
|
|
91
93
|
let handoffTimer;
|
|
92
94
|
/** Every child process not yet known to have exited, including ones the slots have let go of. */
|
|
93
95
|
const live = new Set();
|
|
96
|
+
const liveOwners = new Map();
|
|
94
97
|
let drained = null;
|
|
95
98
|
let onDrained = null;
|
|
96
99
|
let deferredCliRestart = false;
|
|
@@ -247,6 +250,16 @@ export function createSupervisor(deps) {
|
|
|
247
250
|
deps.log("error", `could not send ${signal} to ${slot.plan.id}: ${errorMessage(err)}`);
|
|
248
251
|
}
|
|
249
252
|
};
|
|
253
|
+
const syncRunnerIsStopping = (slot) => slot?.plan.service === "sync" && slot.lastProtocolStatus === "stopping";
|
|
254
|
+
const scheduleSyncRunnerPersistEscalation = (slot, child) => {
|
|
255
|
+
slot.killTimer = setT(() => {
|
|
256
|
+
slot.killTimer = undefined;
|
|
257
|
+
if (slot.child !== child)
|
|
258
|
+
return;
|
|
259
|
+
deps.log("warn", `${slot.plan.id} did not finish persisting pending pushes within ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s; sending SIGKILL${lastProtocolHint(slot)}`);
|
|
260
|
+
signalChild(slot, child, "SIGKILL");
|
|
261
|
+
}, SYNC_RUNNER_PERSIST_GRACE_MS);
|
|
262
|
+
};
|
|
250
263
|
/**
|
|
251
264
|
* Stop the slot's child (if any), then run `then` once it has exited. With
|
|
252
265
|
* `drain`, a unit that has a drain signal gets it first and SIGTERM only
|
|
@@ -276,6 +289,11 @@ export function createSupervisor(deps) {
|
|
|
276
289
|
slot.killTimer = undefined;
|
|
277
290
|
if (slot.child !== child)
|
|
278
291
|
return;
|
|
292
|
+
if (syncRunnerIsStopping(slot)) {
|
|
293
|
+
deps.log("info", `${slot.plan.id} is stopping; allowing ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s for pending pushes to persist`);
|
|
294
|
+
scheduleSyncRunnerPersistEscalation(slot, child);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
279
297
|
deps.log("warn", `${slot.plan.id} did not exit within ${killGraceMs / 1_000} s of SIGTERM; sending SIGKILL${lastProtocolHint(slot)}`);
|
|
280
298
|
signalChild(slot, child, "SIGKILL");
|
|
281
299
|
}, killGraceMs);
|
|
@@ -313,6 +331,7 @@ export function createSupervisor(deps) {
|
|
|
313
331
|
};
|
|
314
332
|
const watch = (slot, child) => {
|
|
315
333
|
live.add(child);
|
|
334
|
+
liveOwners.set(child, slot);
|
|
316
335
|
let ended = false;
|
|
317
336
|
const timeoutMs = slot.plan.heartbeatTimeoutMs;
|
|
318
337
|
const decoder = timeoutMs ? new StringDecoder("utf8") : null;
|
|
@@ -350,6 +369,8 @@ export function createSupervisor(deps) {
|
|
|
350
369
|
slot.syncPassActive = false;
|
|
351
370
|
else if (event.type === "status" && typeof event.state === "string") {
|
|
352
371
|
slot.lastProtocolStatus = event.state;
|
|
372
|
+
if (event.state === "stopping")
|
|
373
|
+
deps.onSyncRunnerStopping?.();
|
|
353
374
|
if (event.state === "stopped" || event.state === "waiting-for-lock")
|
|
354
375
|
slot.syncPassActive = false;
|
|
355
376
|
else if (slot.syncPassActive === undefined && (event.state === "starting" || event.state === "running")) {
|
|
@@ -405,6 +426,7 @@ export function createSupervisor(deps) {
|
|
|
405
426
|
return;
|
|
406
427
|
ended = true;
|
|
407
428
|
live.delete(child);
|
|
429
|
+
liveOwners.delete(child);
|
|
408
430
|
if (live.size === 0)
|
|
409
431
|
onDrained?.();
|
|
410
432
|
if (slot.child !== child)
|
|
@@ -991,10 +1013,8 @@ export function createSupervisor(deps) {
|
|
|
991
1013
|
clearT(handoffTimer);
|
|
992
1014
|
clearDeferredCliRestart();
|
|
993
1015
|
clearPendingSyncCommandRestart();
|
|
994
|
-
const childOwners = new Map();
|
|
1016
|
+
const childOwners = new Map(liveOwners);
|
|
995
1017
|
for (const slot of slots.values()) {
|
|
996
|
-
if (slot.child)
|
|
997
|
-
childOwners.set(slot.child, slot);
|
|
998
1018
|
clearTimer(slot);
|
|
999
1019
|
clearHeartbeatTimer(slot);
|
|
1000
1020
|
slot.child = undefined;
|
|
@@ -1020,9 +1040,16 @@ export function createSupervisor(deps) {
|
|
|
1020
1040
|
resolve();
|
|
1021
1041
|
return;
|
|
1022
1042
|
}
|
|
1043
|
+
const persistingSyncRunners = new Set();
|
|
1044
|
+
let persistEscalation;
|
|
1023
1045
|
const escalation = setT(() => {
|
|
1024
1046
|
for (const child of live) {
|
|
1025
1047
|
const owner = childOwners.get(child);
|
|
1048
|
+
if (owner && syncRunnerIsStopping(owner)) {
|
|
1049
|
+
deps.log("info", `${owner.plan.id} is stopping; allowing ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s for pending pushes to persist`);
|
|
1050
|
+
persistingSyncRunners.add(child);
|
|
1051
|
+
continue;
|
|
1052
|
+
}
|
|
1026
1053
|
deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not exit within ${STOP_ESCALATION_MS / 1_000} s of SIGTERM; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
|
|
1027
1054
|
try {
|
|
1028
1055
|
child.kill("SIGKILL");
|
|
@@ -1031,10 +1058,28 @@ export function createSupervisor(deps) {
|
|
|
1031
1058
|
deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
|
|
1032
1059
|
}
|
|
1033
1060
|
}
|
|
1061
|
+
if (persistingSyncRunners.size > 0) {
|
|
1062
|
+
persistEscalation = setT(() => {
|
|
1063
|
+
for (const child of persistingSyncRunners) {
|
|
1064
|
+
if (!live.has(child))
|
|
1065
|
+
continue;
|
|
1066
|
+
const owner = childOwners.get(child);
|
|
1067
|
+
deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not finish persisting pending pushes within ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
|
|
1068
|
+
try {
|
|
1069
|
+
child.kill("SIGKILL");
|
|
1070
|
+
}
|
|
1071
|
+
catch (err) {
|
|
1072
|
+
deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}, SYNC_RUNNER_PERSIST_GRACE_MS);
|
|
1076
|
+
}
|
|
1034
1077
|
}, STOP_ESCALATION_MS);
|
|
1035
1078
|
onDrained = () => {
|
|
1036
1079
|
onDrained = null;
|
|
1037
1080
|
clearT(escalation);
|
|
1081
|
+
if (persistEscalation !== undefined)
|
|
1082
|
+
clearT(persistEscalation);
|
|
1038
1083
|
resolve();
|
|
1039
1084
|
};
|
|
1040
1085
|
});
|
|
@@ -1042,6 +1087,7 @@ export function createSupervisor(deps) {
|
|
|
1042
1087
|
},
|
|
1043
1088
|
reconcile,
|
|
1044
1089
|
state: snapshot,
|
|
1090
|
+
hasStoppingSyncRunner: () => [...liveOwners.values()].some(syncRunnerIsStopping),
|
|
1045
1091
|
release() {
|
|
1046
1092
|
const released = [];
|
|
1047
1093
|
for (const slot of slots.values()) {
|
|
@@ -22,7 +22,7 @@ import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDa
|
|
|
22
22
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
23
23
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
24
24
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
25
|
-
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
25
|
+
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff, readLanesRegistryFlagFailOpen } from "./registry-flag.js";
|
|
26
26
|
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
|
|
27
27
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
|
|
28
28
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
@@ -1216,6 +1216,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1216
1216
|
}
|
|
1217
1217
|
}
|
|
1218
1218
|
const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
1219
|
+
const readRegistryFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
|
|
1219
1220
|
const storyCloseCompanies = new Map();
|
|
1220
1221
|
for (const lane of lanes) {
|
|
1221
1222
|
if (lane.depth < 1 || lane.state !== "interrupted" || lane.keep_alive === true ||
|
|
@@ -1303,10 +1304,10 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1303
1304
|
readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
|
|
1304
1305
|
readRegistryFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
|
|
1305
1306
|
hasBlockedKeepAliveLane
|
|
1306
|
-
?
|
|
1307
|
+
? readRegistryFlagFailOpen(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
|
|
1307
1308
|
: Promise.resolve(false),
|
|
1308
1309
|
readRegistryFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
|
|
1309
|
-
|
|
1310
|
+
readRegistryFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
|
|
1310
1311
|
]);
|
|
1311
1312
|
keepAliveFlagByCompany.set(key, keepAliveEnabled);
|
|
1312
1313
|
blockedResumeFlagByCompany.set(key, blockedResumeEnabled);
|
|
@@ -3585,6 +3586,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
|
|
|
3585
3586
|
if (company === null)
|
|
3586
3587
|
return summary;
|
|
3587
3588
|
const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
3589
|
+
const readFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
|
|
3588
3590
|
const options = { company, env: deps.env ?? process.env };
|
|
3589
3591
|
let keepAliveEnabled;
|
|
3590
3592
|
let lineageAware;
|
|
@@ -3595,7 +3597,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
|
|
|
3595
3597
|
readFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
|
|
3596
3598
|
readFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
|
|
3597
3599
|
readFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
|
|
3598
|
-
|
|
3600
|
+
readFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
|
|
3599
3601
|
]);
|
|
3600
3602
|
}
|
|
3601
3603
|
catch (err) {
|
|
@@ -19,12 +19,22 @@ export interface ReadLanesFlagOptions {
|
|
|
19
19
|
reportFailure?: (error: unknown) => void;
|
|
20
20
|
}
|
|
21
21
|
export declare function readLanesFlagLocalOverride(flagKey: string, env?: NodeJS.ProcessEnv): boolean | null;
|
|
22
|
+
export type LanesRegistryFlagResult = {
|
|
23
|
+
state: "resolved";
|
|
24
|
+
value: boolean;
|
|
25
|
+
} | {
|
|
26
|
+
state: "unknown";
|
|
27
|
+
};
|
|
22
28
|
/**
|
|
23
29
|
* Read a lanes flag once for each company identity in this CLI process. The
|
|
24
|
-
* cache contains only an in-flight or settled
|
|
30
|
+
* cache contains only an in-flight or settled decision result; it is never
|
|
25
31
|
* written to disk. A local override is checked on every call and wins first.
|
|
26
32
|
*/
|
|
27
33
|
export declare function readLanesRegistryFlag(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
|
|
34
|
+
/** Read a lanes flag while preserving registry failure as an explicit unknown state. */
|
|
35
|
+
export declare function readLanesRegistryFlagResult(flagKey: string, options?: ReadLanesFlagOptions): Promise<LanesRegistryFlagResult>;
|
|
36
|
+
/** Delivery and wake call sites opt in to availability when the registry is unreadable. */
|
|
37
|
+
export declare function readLanesRegistryFlagFailOpen(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
|
|
28
38
|
/**
|
|
29
39
|
* Flag readers normally report their own failures and return the safe default.
|
|
30
40
|
* Keep enforcement callers safe if an injected or unexpected reader rejects.
|