@indigoai-us/hq-cli 5.345.48 → 5.345.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +2 -1
  3. package/dist/command-catalog.generated.d.ts +64 -11
  4. package/dist/command-catalog.generated.js +77 -11
  5. package/dist/commands/anywhere-runtime-flag.js +5 -3
  6. package/dist/commands/db-destroy.js +13 -11
  7. package/dist/commands/db-dump.js +3 -3
  8. package/dist/commands/db-remote-deps.d.ts +3 -0
  9. package/dist/commands/db-remote-deps.js +13 -0
  10. package/dist/commands/db-sql.js +4 -3
  11. package/dist/commands/db-usage.js +4 -3
  12. package/dist/commands/dm-forward-flag.d.ts +4 -0
  13. package/dist/commands/dm-forward-flag.js +7 -0
  14. package/dist/commands/dm.d.ts +8 -0
  15. package/dist/commands/dm.js +219 -27
  16. package/dist/commands/feedback.d.ts +10 -0
  17. package/dist/commands/feedback.js +45 -0
  18. package/dist/commands/lanes-message.js +2 -2
  19. package/dist/commands/lanes.js +2 -2
  20. package/dist/commands/mcp-serve.js +17 -4
  21. package/dist/lib/anywhere-person-setting.d.ts +1 -0
  22. package/dist/lib/anywhere-person-setting.js +12 -0
  23. package/dist/lib/anywhere-runtime-fast-flag.js +5 -3
  24. package/dist/lib/anywhere-runtime-flag-key.d.ts +3 -1
  25. package/dist/lib/anywhere-runtime-flag-key.js +6 -1
  26. package/dist/lib/daemon/flag.d.ts +1 -1
  27. package/dist/lib/daemon/flag.js +20 -9
  28. package/dist/lib/daemon/hq-cloud.d.ts +2 -2
  29. package/dist/lib/daemon/hq-cloud.js +38 -9
  30. package/dist/lib/daemon/run.js +45 -19
  31. package/dist/lib/daemon/server.d.ts +1 -1
  32. package/dist/lib/db/app-ref.d.ts +21 -0
  33. package/dist/lib/db/app-ref.js +45 -0
  34. package/dist/lib/lanes/keep-alive-audit.d.ts +5 -1
  35. package/dist/lib/lanes/keep-alive-audit.js +85 -3
  36. package/dist/lib/lanes/reconcile.js +6 -4
  37. package/dist/lib/lanes/registry-flag.d.ts +11 -1
  38. package/dist/lib/lanes/registry-flag.js +68 -29
  39. package/dist/lib/lanes/watcher.js +63 -2
  40. package/dist/utils/resident-package-preload.js +4 -1
  41. package/dist/utils/self-update.js +6 -5
  42. package/dist/utils/version-gate.d.ts +11 -4
  43. package/dist/utils/version-gate.js +132 -41
  44. package/docs/anywhere-runtime.md +5 -3
  45. package/package.json +1 -1
@@ -1,12 +1,13 @@
1
1
  /**
2
2
  * Feature flags read by the daemon. The daemon runs for weeks, so each read
3
3
  * asks the registry again instead of holding a per-process answer, and it
4
- * reads as off unless the registry itself says on (or a local override does).
4
+ * reads as each flag's fallback unless the registry or a local override decides it.
5
5
  */
6
6
  import { createFlagClient, readLocalOverride, } from "@indigoai-us/hq-flags-client";
7
7
  import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
8
8
  import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
9
9
  import { resolveProcessFlagCompanyContext } from "../flag-registry.js";
10
+ import { HQ_ANYWHERE_RUNTIME_FLAG, hqAnywhereRuntimeLocallyDisabled } from "../anywhere-runtime-flag-key.js";
10
11
  /** Outposts hand running services to the next daemon on a CLI update instead of restarting them all. */
11
12
  export const SUPERVISOR_HANDOFF_FLAG = "daemon.supervisor-handoff";
12
13
  /** Self-updates build the new CLI beside the running one and swap it in, so `hq` never disappears mid-update. */
@@ -14,6 +15,9 @@ export const STAGED_SELF_UPDATE_FLAG = "cli.staged-self-update";
14
15
  const READ_TIMEOUT_MS = 5_000;
15
16
  export async function readDaemonFlag(flagKey, options = {}) {
16
17
  const env = options.env ?? process.env;
18
+ if (flagKey === HQ_ANYWHERE_RUNTIME_FLAG && hqAnywhereRuntimeLocallyDisabled(env))
19
+ return false;
20
+ const fallback = flagKey === HQ_ANYWHERE_RUNTIME_FLAG;
17
21
  const override = readLocalOverride(flagKey, env);
18
22
  const localOverrideMode = options.localOverrideMode ?? "respect";
19
23
  if (override !== null && (localOverrideMode === "respect" || (localOverrideMode === "off-only" && !override)))
@@ -28,10 +32,10 @@ export async function readDaemonFlag(flagKey, options = {}) {
28
32
  let client;
29
33
  try {
30
34
  const company = resolveProcessFlagCompanyContext(env);
31
- // Signed out reads as off without reaching the registry.
35
+ // Signed out reads as the flag fallback without reaching the registry.
32
36
  const token = await (options.getToken ?? (() => ensureCognitoIdToken({ interactive: false })))();
33
37
  if (typeof token !== "string" || token.trim() === "")
34
- return false;
38
+ return fallback;
35
39
  client = (options.createClient ?? createFlagClient)({
36
40
  endpoint: resolveFlagRegistryEndpoint(undefined, env),
37
41
  getToken: () => token,
@@ -42,20 +46,27 @@ export async function readDaemonFlag(flagKey, options = {}) {
42
46
  ...(company.companySlug ? { companyIdentifiers: [company.companySlug] } : {}),
43
47
  onError,
44
48
  });
45
- await client.ready();
46
- const resolution = client.explain(flagKey, { fallback: false });
47
- if (failed && options.throwOnError) {
49
+ try {
50
+ await client.ready();
51
+ }
52
+ catch (error) {
53
+ onError(error);
54
+ }
55
+ const resolution = client.explain(flagKey, { fallback });
56
+ if (failed && options.throwOnError && !fallback) {
48
57
  throw new Error("hq-flags registry read failed", { cause: reportedError });
49
58
  }
50
- return !failed && resolution.source === "snapshot" && resolution.value === true;
59
+ if (resolution.source === "snapshot")
60
+ return resolution.value === true;
61
+ return fallback && resolution.value === true;
51
62
  }
52
63
  catch (error) {
53
64
  if (!failed)
54
65
  onError(error);
55
- if (options.throwOnError && failed) {
66
+ if (options.throwOnError && failed && !fallback) {
56
67
  throw new Error("hq-flags registry read failed", { cause: reportedError ?? error });
57
68
  }
58
- return false;
69
+ return fallback;
59
70
  }
60
71
  finally {
61
72
  try {
@@ -14,8 +14,6 @@ export interface HqCloudCopy {
14
14
  }
15
15
  export interface PickedHqCloud extends HqCloudCopy {
16
16
  source: "bundled" | "managed";
17
- /** Set when a version is pinned but that version is not the installed managed copy. */
18
- pinnedVersionMissing?: string;
19
17
  }
20
18
  export type ManagedHqCloud = PickedHqCloud & {
21
19
  source: "managed";
@@ -85,6 +83,8 @@ export interface UpdateHqCloudDeps {
85
83
  run?: (file: string, args: string[], shell: boolean, env: NodeJS.ProcessEnv) => NpmRunResult;
86
84
  bundled?: HqCloudCopy | null;
87
85
  platform?: NodeJS.Platform;
86
+ /** Test seam for bounded install retry delays. */
87
+ sleep?: (ms: number) => void;
88
88
  }
89
89
  /** Install or update the updater's copy, as this (the login) user. */
90
90
  export declare function updateManagedHqCloud(daemonRoot: string, env: NodeJS.ProcessEnv, deps?: UpdateHqCloudDeps): HqCloudUpdateOutcome;
@@ -54,10 +54,11 @@ export function pickHqCloud(bundled, managed, usable = () => true, pinnedVersion
54
54
  if (managed && managed.version === pinnedVersion && b && m && m[0] === b[0] && usable(managed)) {
55
55
  return { ...managed, source: "managed" };
56
56
  }
57
- // The pinned version is not installed yet (the updater installs it). Keep
58
- // running the newest usable copy meanwhile instead of dropping to an older
59
- // bundled one, and report the missing pin.
60
- return { ...pickNewest(bundled, managed, b, m, usable), pinnedVersionMissing: pinnedVersion };
57
+ if (bundled.version === pinnedVersion && b && usable(bundled))
58
+ return { ...bundled, source: "bundled" };
59
+ // A pin is an exact version contract. Do not silently run another installed
60
+ // package while npm has not yet delivered the pinned version.
61
+ return null;
61
62
  }
62
63
  return pickNewest(bundled, managed, b, m, usable);
63
64
  }
@@ -160,6 +161,17 @@ export function currentHqCloudAfterLeaseRelease(daemonRoot, expectedVersion) {
160
161
  return null;
161
162
  return { ...managed, source: "managed" };
162
163
  }
164
+ function isTransientNpmFailure(result) {
165
+ // spawnSync's ETIMEDOUT means this attempt hit our full 10-minute process
166
+ // timeout. Retrying that result would multiply one stall across every attempt.
167
+ if (result.error?.code === "ETIMEDOUT")
168
+ return false;
169
+ const text = `${result.stderr ?? ""}\n${result.stdout ?? ""}\n${result.error?.message ?? ""}`;
170
+ return /\b(?:ETARGET|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENETUNREACH|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|ENETWORK|ECONNABORTED)\b|\bnotarget\b|network timeout|fetch failed|socket hang up/i.test(text);
171
+ }
172
+ function sleepSync(ms) {
173
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
174
+ }
163
175
  function runNpm(file, args, shell, env) {
164
176
  return spawnSync(file, args, {
165
177
  encoding: "utf8",
@@ -180,7 +192,7 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
180
192
  const platform = deps.platform ?? process.platform;
181
193
  const pinnedVersion = readDaemonConfig(path.join(daemonRoot, "config.json")).hqCloudVersion;
182
194
  const bundled = deps.bundled !== undefined ? deps.bundled : bundledHqCloud();
183
- const from = currentHqCloud(daemonRoot)?.version ?? null;
195
+ const from = managedHqCloud(daemonRoot)?.version ?? bundled?.version ?? null;
184
196
  if (!bundled)
185
197
  return { ok: false, from, to: null, detail: "this CLI install has no bundled hq-cloud to match" };
186
198
  const prefix = managedHqCloudDir(daemonRoot);
@@ -203,9 +215,26 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
203
215
  version = latest;
204
216
  }
205
217
  const cmd = hqCloudInstallCommand(prefix, version, platform);
206
- const r = run(cmd.file, cmd.args, cmd.shell, env);
207
- if (r.status !== 0)
208
- return { ok: false, from, to: null, detail: npmFailure(r) };
209
- return { ok: true, from, to: currentHqCloud(daemonRoot)?.version ?? null };
218
+ const retryDelays = [500, 1_000, 2_000];
219
+ const sleep = deps.sleep ?? sleepSync;
220
+ let r;
221
+ for (let attempt = 0;; attempt++) {
222
+ r = run(cmd.file, cmd.args, cmd.shell, env);
223
+ if (r.status === 0)
224
+ break;
225
+ const retryable = pinnedVersion !== undefined && pinnedVersion !== null && isTransientNpmFailure(r);
226
+ const delay = retryDelays[attempt];
227
+ if (!retryable || delay === undefined) {
228
+ const retries = retryable ? attempt : 0;
229
+ return {
230
+ ok: false,
231
+ from,
232
+ to: null,
233
+ detail: retries > 0 ? `pinned hq-cloud install failed after ${retries + 1} attempts: ${npmFailure(r)}` : npmFailure(r),
234
+ };
235
+ }
236
+ sleep(delay);
237
+ }
238
+ return { ok: true, from, to: managedHqCloud(daemonRoot)?.version ?? null };
210
239
  }
211
240
  //# sourceMappingURL=hq-cloud.js.map
@@ -105,17 +105,26 @@ const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
105
105
  const HQD_PERSON_SETTING_GRACE_MS = 30 * 60_000;
106
106
  export function syncRunnerSourceForPlan(stagedRunnerPath, hqCloud, legacyBundledRunnerPath = null, staged = { version: null }, currentSyncRunnerSource = null) {
107
107
  // A managed copy lives in the daemon directory, so it survives a CLI
108
- // replacement. Before lease release pickHqCloud returns it only when it is
109
- // pinned or newer than the bundled copy; after release it is the only
110
- // eligible package, so it must not displace a newer staged bundle unless it
111
- // is the pinned version. The staged bundle is built from the bundled copy and
112
- // otherwise only stands in for that copy (hq-cli#1547).
108
+ // replacement. A configured pin is an exact version contract: only that
109
+ // installed package or an exactly matching staged bundle may start.
113
110
  const pin = staged.pinnedVersion;
114
- const hasRunnerAtOrAbovePin = pin != null &&
115
- ((staged.version !== null && !hqCloudVersionOlder(staged.version, pin)) ||
116
- (hqCloud !== null && !hqCloudVersionOlder(hqCloud.version, pin)));
117
- if (pin && !hasRunnerAtOrAbovePin && currentSyncRunnerSource?.runnerPath) {
118
- return currentSyncRunnerSource;
111
+ if (pin) {
112
+ if (hqCloud?.version === pin) {
113
+ if (hqCloud.source === "managed") {
114
+ return { kind: "managed-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
115
+ }
116
+ if (stagedRunnerPath && staged.version === pin)
117
+ return { kind: "staged-sync", runnerPath: stagedRunnerPath };
118
+ return { kind: "bundled-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
119
+ }
120
+ if (stagedRunnerPath && staged.version === pin)
121
+ return { kind: "staged-sync", runnerPath: stagedRunnerPath };
122
+ // A transient install gap must not stop a runner that is already serving
123
+ // sync. On cold start there is no current source, so mismatched bundles
124
+ // remain unavailable and cannot be started.
125
+ if (currentSyncRunnerSource?.runnerPath)
126
+ return currentSyncRunnerSource;
127
+ return { kind: "unavailable", runnerPath: null };
119
128
  }
120
129
  const managedYieldsToStaged = stagedRunnerPath !== null &&
121
130
  hqCloud?.source === "managed" &&
@@ -408,6 +417,7 @@ export async function runDaemon(opts) {
408
417
  let legacySyncRunnerPathChecked = false;
409
418
  let legacySyncRunnerPath = null;
410
419
  let lastPlannedSyncRunnerSource = null;
420
+ let lastPlannedSyncRunnerVersion = null;
411
421
  let packageUseLeaseReleased = false;
412
422
  let hqCloudVersionBeforeRelease = null;
413
423
  let startupPlanOpen = true;
@@ -580,6 +590,7 @@ export async function runDaemon(opts) {
580
590
  let warnedMissingPin = null;
581
591
  let warnedMissingSyncRunner = false;
582
592
  const daemonSyncRunnerPids = new Set();
593
+ let supervisor = null;
583
594
  const plan = () => {
584
595
  const config = readDaemonConfig(paths.configPath);
585
596
  const legacy = legacyUnits();
@@ -594,19 +605,34 @@ export async function runDaemon(opts) {
594
605
  : currentHqCloud(paths.root);
595
606
  if (!packageUseLeaseReleased)
596
607
  hqCloudVersionBeforeRelease = hqCloud?.version ?? staged?.metadata.version ?? null;
597
- const missingPin = hqCloud?.pinnedVersionMissing ?? null;
598
- if (missingPin !== warnedMissingPin) {
599
- warnedMissingPin = missingPin;
600
- if (missingPin && hqCloud) {
601
- log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; running ${hqCloud.version} (${hqCloud.source}) until the updater installs it`);
602
- }
603
- }
604
608
  if (!packageUseLeaseReleased && !legacySyncRunnerPathChecked) {
605
609
  legacySyncRunnerPath = resolveSyncRunnerPath();
606
610
  legacySyncRunnerPathChecked = true;
607
611
  }
608
- const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null }, lastPlannedSyncRunnerSource);
612
+ const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null }, supervisor?.state().units.some((unit) => unit.id === "sync" && unit.status === "running" && typeof unit.pid === "number")
613
+ ? lastPlannedSyncRunnerSource
614
+ : null);
615
+ const retainedLiveSource = syncSource === lastPlannedSyncRunnerSource;
616
+ const syncRunnerVersion = retainedLiveSource
617
+ ? lastPlannedSyncRunnerVersion
618
+ : syncSource.kind === "staged-sync"
619
+ ? staged?.metadata.version ?? null
620
+ : hqCloud?.version ?? null;
621
+ const missingPin = config.hqCloudVersion && syncRunnerVersion !== config.hqCloudVersion ? config.hqCloudVersion : null;
622
+ const pinWarningKey = missingPin
623
+ ? `${missingPin}:${retainedLiveSource ? syncRunnerVersion ?? "unknown" : "unavailable"}`
624
+ : null;
625
+ if (pinWarningKey !== warnedMissingPin) {
626
+ warnedMissingPin = pinWarningKey;
627
+ if (missingPin && retainedLiveSource) {
628
+ log("warn", `hq-cloud is pinned to ${missingPin}, but the running sync runner is ${syncRunnerVersion ?? "unknown"}; keeping it running while the updater retries`);
629
+ }
630
+ else if (missingPin && syncSource.kind === "unavailable") {
631
+ log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; sync is unavailable until the updater installs it`);
632
+ }
633
+ }
609
634
  lastPlannedSyncRunnerSource = syncSource.runnerPath ? syncSource : null;
635
+ lastPlannedSyncRunnerVersion = syncSource.runnerPath ? syncRunnerVersion : null;
610
636
  if (!syncSource.runnerPath && !warnedMissingSyncRunner) {
611
637
  warnedMissingSyncRunner = true;
612
638
  log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
@@ -797,7 +823,7 @@ export async function runDaemon(opts) {
797
823
  // Set once the shutdown path exists below; a restart request before then is dropped.
798
824
  let restartDaemon = () => false;
799
825
  let extendSyncShutdownCap;
800
- const supervisor = createSupervisor({
826
+ supervisor = createSupervisor({
801
827
  machineType,
802
828
  host: opts.host,
803
829
  pid: process.pid,
@@ -39,7 +39,7 @@ export interface HqdServerOptions {
39
39
  env?: NodeJS.ProcessEnv;
40
40
  /** Work-mesh root override for mesh.presence (tests). */
41
41
  workMeshRoot?: string;
42
- /** Test seam for the hq-anywhere-runtime default-off gate. */
42
+ /** Test seam for the hq-anywhere-runtime fail-safe gate. */
43
43
  mcpEnabled?: () => boolean | Promise<boolean>;
44
44
  /** Test seam for the live hq-flags read used by the daemon's bounded runtime monitor. */
45
45
  runtimeFlagReader?: (options: ReadDaemonFlagOptions) => Promise<boolean>;
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Resolve the `--app` value of the `hq db` commands to an app id. The value can
3
+ * be the app id, its name or its subdomain. Only apps in the given company are
4
+ * searched, so a name never resolves to another company's app.
5
+ */
6
+ import { deployApiRequest } from "../deploy-api.js";
7
+ export interface AppRefRow {
8
+ id?: unknown;
9
+ name?: unknown;
10
+ subdomain?: unknown;
11
+ }
12
+ export declare class AppRefError extends Error {
13
+ constructor(message: string);
14
+ }
15
+ export declare function matchAppRef(apps: AppRefRow[], ref: string): string;
16
+ export declare function resolveAppRef(input: {
17
+ token: string;
18
+ companySlug: string;
19
+ ref: string;
20
+ }, request?: typeof deployApiRequest): Promise<string>;
21
+ //# sourceMappingURL=app-ref.d.ts.map
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Resolve the `--app` value of the `hq db` commands to an app id. The value can
3
+ * be the app id, its name or its subdomain. Only apps in the given company are
4
+ * searched, so a name never resolves to another company's app.
5
+ */
6
+ import { deployApiRequest } from "../deploy-api.js";
7
+ import { listDeployApps } from "../deploy-apps.js";
8
+ export class AppRefError extends Error {
9
+ constructor(message) {
10
+ super(message);
11
+ this.name = "AppRefError";
12
+ }
13
+ }
14
+ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
15
+ export function matchAppRef(apps, ref) {
16
+ const ids = (rows) => [
17
+ ...new Set(rows.map((row) => row.id).filter((id) => typeof id === "string")),
18
+ ];
19
+ const byId = ids(apps.filter((row) => row.id === ref));
20
+ if (byId.length === 1)
21
+ return byId[0];
22
+ const byName = ids(apps.filter((row) => row.name === ref || row.subdomain === ref));
23
+ if (byName.length === 1)
24
+ return byName[0];
25
+ if (byName.length > 1) {
26
+ throw new AppRefError(`More than one app in this company is called "${ref}" (${byName.join(", ")}). Use the app id.`);
27
+ }
28
+ throw new AppRefError(`No app called "${ref}" was found in this company.`);
29
+ }
30
+ export async function resolveAppRef(input, request = deployApiRequest) {
31
+ const ref = input.ref.trim();
32
+ if (UUID_RE.test(ref))
33
+ return ref;
34
+ const headers = { "X-Org-Slug": input.companySlug };
35
+ const apps = await listDeployApps(async (cursor) => {
36
+ const route = cursor ? `/api/apps?cursor=${encodeURIComponent(cursor)}` : "/api/apps";
37
+ const res = await request(input.token, route, "GET", undefined, headers);
38
+ if (!res.ok) {
39
+ throw new AppRefError(`Could not look up the app "${ref}" (hq-deploy returned ${res.status}). Use the app id.`);
40
+ }
41
+ return (await res.json());
42
+ });
43
+ return matchAppRef(apps, ref);
44
+ }
45
+ //# sourceMappingURL=app-ref.js.map
@@ -1,4 +1,4 @@
1
- type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "waiting";
1
+ type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "finished_story" | "idle_no_queued_work" | "waiting";
2
2
  interface KeepAliveAuditRestart {
3
3
  at: string | null;
4
4
  trigger: string;
@@ -17,6 +17,7 @@ interface KeepAliveAuditRow {
17
17
  blocked_outcome?: "woken" | "skipped";
18
18
  blocked_outcome_reason?: string;
19
19
  verdict: KeepAliveAuditVerdict;
20
+ queued_work_at_audit_time?: boolean;
20
21
  remediation?: string;
21
22
  terminal_reason?: string;
22
23
  }
@@ -25,6 +26,8 @@ interface KeepAliveAuditSummary {
25
26
  not_restarted: number;
26
27
  stranded: number;
27
28
  restarted_untimed: number;
29
+ finished_story: number;
30
+ idle_no_queued_work: number;
28
31
  late_restarts_after_window: number;
29
32
  late_over_10m: number;
30
33
  early_wake_by_others: number;
@@ -53,6 +56,7 @@ export interface StopEnvelope {
53
56
  name: string;
54
57
  mtimeMs: number;
55
58
  decision: "done" | "blocked";
59
+ storyComplete: boolean;
56
60
  nextCheckAt?: string;
57
61
  }
58
62
  export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
@@ -1,6 +1,7 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
- import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS } from "./keep-alive.js";
3
+ import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, doneKeepAliveVerdict, hasPendingKeepAliveWork, undeliveredAnsweredQuestions } from "./keep-alive.js";
4
+ import { dropboxDir, dropboxInboxPath } from "./dropbox.js";
4
5
  import { LanesError } from "./errors.js";
5
6
  import { laneRunDir } from "./spawn.js";
6
7
  import { compareEnvelopeNames } from "./envelope-sequence.js";
@@ -63,6 +64,8 @@ const VERDICTS = [
63
64
  "keep_alive_off",
64
65
  "not_restarted",
65
66
  "stranded",
67
+ "finished_story",
68
+ "idle_no_queued_work",
66
69
  "waiting",
67
70
  ];
68
71
  function parseTime(value) {
@@ -174,7 +177,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
174
177
  if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
175
178
  throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
176
179
  }
177
- const doneStop = raw.decision === "done" && raw.story_complete === false;
180
+ const doneStop = raw.decision === "done";
178
181
  const blockedStop = raw.decision === "blocked" && typeof raw.next_check_at === "string" &&
179
182
  isIsoDateTime(raw.next_check_at) && Date.parse(raw.next_check_at) > stat.mtimeMs;
180
183
  if (!doneStop && !blockedStop)
@@ -183,6 +186,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
183
186
  name,
184
187
  mtimeMs: stat.mtimeMs,
185
188
  decision: blockedStop ? "blocked" : "done",
189
+ storyComplete: raw.story_complete === true,
186
190
  ...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
187
191
  });
188
192
  }
@@ -428,6 +432,48 @@ function firstRestartAfter(lane, stop, nextEnvelopeMs, workerResumeEvents) {
428
432
  // untimed instead of using the lane's mutable current state as a proxy.
429
433
  return nextEnvelopeMs === undefined ? null : { at: null, trigger: "unknown", caller: null };
430
434
  }
435
+ /**
436
+ * When the work a finished standing lane is now waiting on first appeared.
437
+ * The earliest signal wins: if work was queued at 10:05 and more arrived at
438
+ * 11:59, keep-alive has owed a launch since 10:05. Returns undefined when no
439
+ * signal carries a time, which leaves the caller unable to open a window.
440
+ */
441
+ function keepAliveQueuedWorkAt(hqRoot, lane) {
442
+ const candidates = [];
443
+ const pendingAt = parseTime(lane.pending_resume?.at);
444
+ if (pendingAt !== undefined)
445
+ candidates.push(pendingAt);
446
+ for (const candidate of [dropboxInboxPath(hqRoot, lane.lane_id), dropboxDir(hqRoot, lane.lane_id)]) {
447
+ try {
448
+ candidates.push(fs.statSync(candidate).mtimeMs);
449
+ break;
450
+ }
451
+ catch (err) {
452
+ if (err.code !== "ENOENT")
453
+ throw err;
454
+ }
455
+ }
456
+ const questions = [];
457
+ for (const questionId of listQuestionIds(hqRoot)) {
458
+ try {
459
+ const question = tryReadQuestion(hqRoot, questionId);
460
+ if (question)
461
+ questions.push(question);
462
+ }
463
+ catch {
464
+ // One unreadable question file must not fail the whole audit.
465
+ continue;
466
+ }
467
+ }
468
+ for (const question of undeliveredAnsweredQuestions(lane, questions)) {
469
+ const answeredAt = parseTime(question.answer?.answered_at);
470
+ if (answeredAt !== undefined)
471
+ candidates.push(answeredAt);
472
+ }
473
+ if (candidates.length === 0)
474
+ return undefined;
475
+ return Math.min(...candidates);
476
+ }
431
477
  function classifyVerdict(input) {
432
478
  const { dueAt, restart, lane, nowMs } = input;
433
479
  if (restart?.trigger === "unknown" && laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
@@ -478,6 +524,34 @@ function classifyVerdict(input) {
478
524
  }
479
525
  if (laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
480
526
  return "keep_alive_off";
527
+ // Split the unrestarted stops by whether keep-alive owes this lane a wake at
528
+ // all. The discriminator is the shared done-state policy, not whether the
529
+ // envelope declared a next_check_at: a stop with story_complete false is owed
530
+ // a wake with or without one. Only done stops with no later envelope reach
531
+ // this branch, because a later envelope means the lane carried on.
532
+ if (input.decision === "done" && input.nextEnvelopeAt === undefined) {
533
+ const eligibility = doneKeepAliveVerdict(lane, {
534
+ decision: "done",
535
+ story_complete: input.storyComplete,
536
+ ...(input.nextCheckAt === undefined ? {} : { next_check_at: input.nextCheckAt }),
537
+ }, input.hasQueuedWork, { nowMs });
538
+ // Keep-alive is specified not to wake either case, so neither is a
539
+ // keep-alive liveness failure and neither counts against the KPI.
540
+ if (eligibility === "story_complete")
541
+ return "finished_story";
542
+ if (eligibility === "standing_idle")
543
+ return "idle_no_queued_work";
544
+ // A finished story becomes eligible again only once work is queued, which
545
+ // can be long after the stop. Open the failure window when that work
546
+ // appeared, not at the stop's own due time, or every normal exit from
547
+ // idle_no_queued_work reads as a liveness failure the moment it happens.
548
+ if (input.storyComplete) {
549
+ if (input.queuedWorkAt === undefined)
550
+ return "waiting";
551
+ const anchor = Math.max(dueAt, input.queuedWorkAt);
552
+ return nowMs > anchor + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS ? "not_restarted" : "waiting";
553
+ }
554
+ }
481
555
  if (nowMs > dueAt + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS)
482
556
  return "not_restarted";
483
557
  return "waiting";
@@ -546,6 +620,10 @@ export function buildKeepAliveAudit(input) {
546
620
  try {
547
621
  const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
548
622
  const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
623
+ // Read at audit time, not stop time: for a stop with no restart the
624
+ // question is whether anything is going to resume the lane now.
625
+ const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(input.hqRoot, lane);
626
+ const queuedWorkAt = hasQueuedWork ? keepAliveQueuedWorkAt(input.hqRoot, lane) : undefined;
549
627
  const laneWorkerResumeEvents = readLaneWorkerResumeEvents(input.hqRoot, lane.lane_id);
550
628
  const terminalClassifications = readTerminalClassifications(input.hqRoot, lane.lane_id);
551
629
  for (const stop of stops) {
@@ -572,7 +650,8 @@ export function buildKeepAliveAudit(input) {
572
650
  blocked_outcome: blockedOutcome.status,
573
651
  ...(blockedOutcome.reason === undefined ? {} : { blocked_outcome_reason: blockedOutcome.reason }),
574
652
  } : {}),
575
- verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
653
+ verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, decision: stop.decision, storyComplete: stop.storyComplete, hasQueuedWork, ...(queuedWorkAt === undefined ? {} : { queuedWorkAt }), lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
654
+ ...(stop.decision === "done" ? { queued_work_at_audit_time: hasQueuedWork } : {}),
576
655
  });
577
656
  }
578
657
  }
@@ -611,6 +690,8 @@ export function buildKeepAliveAudit(input) {
611
690
  not_restarted: byVerdict.not_restarted,
612
691
  stranded: byVerdict.stranded,
613
692
  restarted_untimed: byVerdict.restarted_untimed,
693
+ finished_story: byVerdict.finished_story,
694
+ idle_no_queued_work: byVerdict.idle_no_queued_work,
614
695
  late_restarts_after_window: lateRestartsAfterWindow,
615
696
  late_over_10m: lateOverTenMinutes,
616
697
  early_wake_by_others: byVerdict.early_wake,
@@ -642,6 +723,7 @@ export function formatKeepAliveAuditText(result) {
642
723
  "",
643
724
  `Since ${result.since}; ${counts}`,
644
725
  `not_restarted=${result.summary.not_restarted}, stranded=${result.summary.stranded}, late_restarts_after_window=${result.summary.late_restarts_after_window}, late_over_10m=${result.summary.late_over_10m}, early_wake_by_others=${result.summary.early_wake_by_others}, self_wake=${result.summary.self_wake}`,
726
+ `finished_story=${result.summary.finished_story}, idle_no_queued_work=${result.summary.idle_no_queued_work} (neither counts against the keep-alive KPI)`,
645
727
  `Stopped and not restarted on their own: ${result.summary.stopped_and_not_restarted_on_their_own}`,
646
728
  ...result.rows.filter((row) => row.remediation).map((row) => `Stranded ${row.lane}: ${row.remediation}`),
647
729
  ...result.companyless_lead_lanes.map((lane) => `Company-less lead ${lane.lane} (${lane.project}): ${lane.reason}; repair with hq lanes repair-company ${lane.lane} --company <slug>`),
@@ -22,7 +22,7 @@ import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDa
22
22
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
23
23
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
24
24
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
25
- import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
25
+ import { readLanesRegistryFlag, readLanesRegistryFlagFailOff, readLanesRegistryFlagFailOpen } from "./registry-flag.js";
26
26
  import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
27
27
  import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
28
28
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
@@ -1216,6 +1216,7 @@ export async function runPassBody(hqRoot, deps) {
1216
1216
  }
1217
1217
  }
1218
1218
  const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
1219
+ const readRegistryFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
1219
1220
  const storyCloseCompanies = new Map();
1220
1221
  for (const lane of lanes) {
1221
1222
  if (lane.depth < 1 || lane.state !== "interrupted" || lane.keep_alive === true ||
@@ -1303,10 +1304,10 @@ export async function runPassBody(hqRoot, deps) {
1303
1304
  readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
1304
1305
  readRegistryFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
1305
1306
  hasBlockedKeepAliveLane
1306
- ? readRegistryFlag(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
1307
+ ? readRegistryFlagFailOpen(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
1307
1308
  : Promise.resolve(false),
1308
1309
  readRegistryFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
1309
- readRegistryFlag(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
1310
+ readRegistryFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
1310
1311
  ]);
1311
1312
  keepAliveFlagByCompany.set(key, keepAliveEnabled);
1312
1313
  blockedResumeFlagByCompany.set(key, blockedResumeEnabled);
@@ -3585,6 +3586,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
3585
3586
  if (company === null)
3586
3587
  return summary;
3587
3588
  const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
3589
+ const readFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
3588
3590
  const options = { company, env: deps.env ?? process.env };
3589
3591
  let keepAliveEnabled;
3590
3592
  let lineageAware;
@@ -3595,7 +3597,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
3595
3597
  readFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
3596
3598
  readFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
3597
3599
  readFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
3598
- readFlag(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
3600
+ readFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
3599
3601
  ]);
3600
3602
  }
3601
3603
  catch (err) {
@@ -19,12 +19,22 @@ export interface ReadLanesFlagOptions {
19
19
  reportFailure?: (error: unknown) => void;
20
20
  }
21
21
  export declare function readLanesFlagLocalOverride(flagKey: string, env?: NodeJS.ProcessEnv): boolean | null;
22
+ export type LanesRegistryFlagResult = {
23
+ state: "resolved";
24
+ value: boolean;
25
+ } | {
26
+ state: "unknown";
27
+ };
22
28
  /**
23
29
  * Read a lanes flag once for each company identity in this CLI process. The
24
- * cache contains only an in-flight or settled boolean promise; it is never
30
+ * cache contains only an in-flight or settled decision result; it is never
25
31
  * written to disk. A local override is checked on every call and wins first.
26
32
  */
27
33
  export declare function readLanesRegistryFlag(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
34
+ /** Read a lanes flag while preserving registry failure as an explicit unknown state. */
35
+ export declare function readLanesRegistryFlagResult(flagKey: string, options?: ReadLanesFlagOptions): Promise<LanesRegistryFlagResult>;
36
+ /** Delivery and wake call sites opt in to availability when the registry is unreadable. */
37
+ export declare function readLanesRegistryFlagFailOpen(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
28
38
  /**
29
39
  * Flag readers normally report their own failures and return the safe default.
30
40
  * Keep enforcement callers safe if an injected or unexpected reader rejects.