@indigoai-us/hq-cli 5.345.49 → 5.345.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/dist/command-catalog.generated.d.ts +56 -11
  3. package/dist/command-catalog.generated.js +67 -11
  4. package/dist/commands/core-hq-session-native.js +2 -2
  5. package/dist/commands/core-resolve-company-native.js +2 -2
  6. package/dist/commands/db-destroy.js +13 -11
  7. package/dist/commands/db-dump.js +3 -3
  8. package/dist/commands/db-remote-deps.d.ts +3 -0
  9. package/dist/commands/db-remote-deps.js +13 -0
  10. package/dist/commands/db-sql.js +4 -3
  11. package/dist/commands/db-usage.js +4 -3
  12. package/dist/commands/dm-forward-flag.d.ts +4 -0
  13. package/dist/commands/dm-forward-flag.js +7 -0
  14. package/dist/commands/dm.d.ts +8 -0
  15. package/dist/commands/dm.js +219 -27
  16. package/dist/commands/lanes.js +52 -11
  17. package/dist/commands/secrets.js +2 -2
  18. package/dist/commands/workers.js +2 -2
  19. package/dist/lib/bot/daemon.d.ts +1 -1
  20. package/dist/lib/bot/daemon.js +35 -29
  21. package/dist/lib/daemon/hq-cloud.d.ts +2 -2
  22. package/dist/lib/daemon/hq-cloud.js +38 -9
  23. package/dist/lib/daemon/install.js +34 -3
  24. package/dist/lib/daemon/run.js +45 -19
  25. package/dist/lib/db/app-ref.d.ts +21 -0
  26. package/dist/lib/db/app-ref.js +45 -0
  27. package/dist/lib/doctor/checks/bots.js +3 -8
  28. package/dist/lib/lanes/keep-alive-audit.d.ts +5 -1
  29. package/dist/lib/lanes/keep-alive-audit.js +85 -3
  30. package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
  31. package/dist/lib/lanes/outpost-dispatch.js +2 -0
  32. package/dist/lib/mesh/live/daemon/install.js +28 -15
  33. package/dist/lib/service-manager/index.d.ts +3 -0
  34. package/dist/lib/service-manager/index.js +22 -6
  35. package/dist/lib/service-manager/launchd.d.ts +40 -3
  36. package/dist/lib/service-manager/launchd.js +106 -5
  37. package/dist/lib/service-manager/migration-state.d.ts +12 -0
  38. package/dist/lib/service-manager/migration-state.js +46 -0
  39. package/dist/lib/service-manager/migration.d.ts +22 -0
  40. package/dist/lib/service-manager/migration.js +85 -0
  41. package/dist/lib/service-manager/startup-migration.d.ts +24 -0
  42. package/dist/lib/service-manager/startup-migration.js +27 -0
  43. package/dist/lib/service-manager/types.d.ts +3 -0
  44. package/dist/main.js +22 -0
  45. package/dist/native-fast-route.d.ts +4 -0
  46. package/dist/native-fast-route.js +20 -0
  47. package/dist/utils/resident-package-preload.js +2 -1
  48. package/package.json +1 -1
@@ -54,10 +54,11 @@ export function pickHqCloud(bundled, managed, usable = () => true, pinnedVersion
54
54
  if (managed && managed.version === pinnedVersion && b && m && m[0] === b[0] && usable(managed)) {
55
55
  return { ...managed, source: "managed" };
56
56
  }
57
- // The pinned version is not installed yet (the updater installs it). Keep
58
- // running the newest usable copy meanwhile instead of dropping to an older
59
- // bundled one, and report the missing pin.
60
- return { ...pickNewest(bundled, managed, b, m, usable), pinnedVersionMissing: pinnedVersion };
57
+ if (bundled.version === pinnedVersion && b && usable(bundled))
58
+ return { ...bundled, source: "bundled" };
59
+ // A pin is an exact version contract. Do not silently run another installed
60
+ // package while npm has not yet delivered the pinned version.
61
+ return null;
61
62
  }
62
63
  return pickNewest(bundled, managed, b, m, usable);
63
64
  }
@@ -160,6 +161,17 @@ export function currentHqCloudAfterLeaseRelease(daemonRoot, expectedVersion) {
160
161
  return null;
161
162
  return { ...managed, source: "managed" };
162
163
  }
164
+ function isTransientNpmFailure(result) {
165
+ // spawnSync's ETIMEDOUT means this attempt hit our full 10-minute process
166
+ // timeout. Retrying that result would multiply one stall across every attempt.
167
+ if (result.error?.code === "ETIMEDOUT")
168
+ return false;
169
+ const text = `${result.stderr ?? ""}\n${result.stdout ?? ""}\n${result.error?.message ?? ""}`;
170
+ return /\b(?:ETARGET|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENETUNREACH|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|ENETWORK|ECONNABORTED)\b|\bnotarget\b|network timeout|fetch failed|socket hang up/i.test(text);
171
+ }
172
+ function sleepSync(ms) {
173
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
174
+ }
163
175
  function runNpm(file, args, shell, env) {
164
176
  return spawnSync(file, args, {
165
177
  encoding: "utf8",
@@ -180,7 +192,7 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
180
192
  const platform = deps.platform ?? process.platform;
181
193
  const pinnedVersion = readDaemonConfig(path.join(daemonRoot, "config.json")).hqCloudVersion;
182
194
  const bundled = deps.bundled !== undefined ? deps.bundled : bundledHqCloud();
183
- const from = currentHqCloud(daemonRoot)?.version ?? null;
195
+ const from = managedHqCloud(daemonRoot)?.version ?? bundled?.version ?? null;
184
196
  if (!bundled)
185
197
  return { ok: false, from, to: null, detail: "this CLI install has no bundled hq-cloud to match" };
186
198
  const prefix = managedHqCloudDir(daemonRoot);
@@ -203,9 +215,26 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
203
215
  version = latest;
204
216
  }
205
217
  const cmd = hqCloudInstallCommand(prefix, version, platform);
206
- const r = run(cmd.file, cmd.args, cmd.shell, env);
207
- if (r.status !== 0)
208
- return { ok: false, from, to: null, detail: npmFailure(r) };
209
- return { ok: true, from, to: currentHqCloud(daemonRoot)?.version ?? null };
218
+ const retryDelays = [500, 1_000, 2_000];
219
+ const sleep = deps.sleep ?? sleepSync;
220
+ let r;
221
+ for (let attempt = 0;; attempt++) {
222
+ r = run(cmd.file, cmd.args, cmd.shell, env);
223
+ if (r.status === 0)
224
+ break;
225
+ const retryable = pinnedVersion !== undefined && pinnedVersion !== null && isTransientNpmFailure(r);
226
+ const delay = retryDelays[attempt];
227
+ if (!retryable || delay === undefined) {
228
+ const retries = retryable ? attempt : 0;
229
+ return {
230
+ ok: false,
231
+ from,
232
+ to: null,
233
+ detail: retries > 0 ? `pinned hq-cloud install failed after ${retries + 1} attempts: ${npmFailure(r)}` : npmFailure(r),
234
+ };
235
+ }
236
+ sleep(delay);
237
+ }
238
+ return { ok: true, from, to: managedHqCloud(daemonRoot)?.version ?? null };
210
239
  }
211
240
  //# sourceMappingURL=hq-cloud.js.map
@@ -17,7 +17,7 @@
17
17
  */
18
18
  import * as fs from "node:fs";
19
19
  import * as path from "node:path";
20
- import { activateLaunchd, deactivateLaunchd, launchdPlistPath, renderLaunchdPlist, } from "../service-manager/launchd.js";
20
+ import { activateLaunchd, deactivateLaunchd, defaultRunCommand, installLaunchdPlist, launchdIsLoaded, launchdPlistPath, renderLaunchdPlist, } from "../service-manager/launchd.js";
21
21
  import { activateSystemd, deactivateSystemd, renderSystemdUserUnit, systemdUnitPath, } from "../service-manager/systemd.js";
22
22
  import { escapeXml, shellQuote } from "../service-manager/types.js";
23
23
  import { readPidLock } from "../mesh/live/daemon/pid-lock.js";
@@ -251,12 +251,43 @@ export function installDaemonUnit(host, options, deps) {
251
251
  fs.mkdirSync(path.dirname(plan.unitPath), { recursive: true });
252
252
  if (plan.kind !== "systemd-system")
253
253
  fs.mkdirSync(path.join(host.daemonDir, "logs"), { recursive: true });
254
- if (plan.kind === "schtasks")
254
+ if (plan.kind === "launchd") {
255
+ let plist;
256
+ try {
257
+ plist = installLaunchdPlist(plan.unitPath, DAEMON_LAUNCHD_LABEL, plan.rendered, {
258
+ run: deps.run ?? defaultRunCommand,
259
+ uid: host.uid,
260
+ sleep: deps.sleep,
261
+ isLoaded: () => launchdIsLoaded(DAEMON_LAUNCHD_LABEL, deps.run ?? defaultRunCommand, host.uid),
262
+ files: {
263
+ existsSync: fs.existsSync,
264
+ readFileSync: (file) => fs.readFileSync(file, "utf8"),
265
+ writeFileSync: fs.writeFileSync,
266
+ chmodSync: fs.chmodSync,
267
+ renameSync: fs.renameSync,
268
+ unlinkSync: fs.unlinkSync,
269
+ },
270
+ });
271
+ }
272
+ catch (err) {
273
+ return {
274
+ kind: plan.kind,
275
+ unitPath: plan.unitPath,
276
+ installed: fs.existsSync(plan.unitPath),
277
+ running: launchdIsLoaded(DAEMON_LAUNCHD_LABEL, deps.run ?? defaultRunCommand, host.uid),
278
+ error: err instanceof Error ? err.message : String(err),
279
+ };
280
+ }
281
+ if (!plist.loaded)
282
+ return { kind: plan.kind, unitPath: plan.unitPath, installed: true, running: false };
283
+ }
284
+ else if (plan.kind === "schtasks")
255
285
  fs.writeFileSync(plan.unitPath, encodeWindowsTaskXml(plan.rendered));
256
286
  else
257
287
  fs.writeFileSync(plan.unitPath, plan.rendered, { mode: plan.kind === "systemd-system" ? 0o644 : 0o600 });
258
288
  try {
259
- activate(plan, host, deps.run, deps.sleep);
289
+ if (plan.kind !== "launchd")
290
+ activate(plan, host, deps.run, deps.sleep);
260
291
  return { kind: plan.kind, unitPath: plan.unitPath, installed: true, running: true };
261
292
  }
262
293
  catch (err) {
@@ -105,17 +105,26 @@ const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
105
105
  const HQD_PERSON_SETTING_GRACE_MS = 30 * 60_000;
106
106
  export function syncRunnerSourceForPlan(stagedRunnerPath, hqCloud, legacyBundledRunnerPath = null, staged = { version: null }, currentSyncRunnerSource = null) {
107
107
  // A managed copy lives in the daemon directory, so it survives a CLI
108
- // replacement. Before lease release pickHqCloud returns it only when it is
109
- // pinned or newer than the bundled copy; after release it is the only
110
- // eligible package, so it must not displace a newer staged bundle unless it
111
- // is the pinned version. The staged bundle is built from the bundled copy and
112
- // otherwise only stands in for that copy (hq-cli#1547).
108
+ // replacement. A configured pin is an exact version contract: only that
109
+ // installed package or an exactly matching staged bundle may start.
113
110
  const pin = staged.pinnedVersion;
114
- const hasRunnerAtOrAbovePin = pin != null &&
115
- ((staged.version !== null && !hqCloudVersionOlder(staged.version, pin)) ||
116
- (hqCloud !== null && !hqCloudVersionOlder(hqCloud.version, pin)));
117
- if (pin && !hasRunnerAtOrAbovePin && currentSyncRunnerSource?.runnerPath) {
118
- return currentSyncRunnerSource;
111
+ if (pin) {
112
+ if (hqCloud?.version === pin) {
113
+ if (hqCloud.source === "managed") {
114
+ return { kind: "managed-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
115
+ }
116
+ if (stagedRunnerPath && staged.version === pin)
117
+ return { kind: "staged-sync", runnerPath: stagedRunnerPath };
118
+ return { kind: "bundled-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
119
+ }
120
+ if (stagedRunnerPath && staged.version === pin)
121
+ return { kind: "staged-sync", runnerPath: stagedRunnerPath };
122
+ // A transient install gap must not stop a runner that is already serving
123
+ // sync. On cold start there is no current source, so mismatched bundles
124
+ // remain unavailable and cannot be started.
125
+ if (currentSyncRunnerSource?.runnerPath)
126
+ return currentSyncRunnerSource;
127
+ return { kind: "unavailable", runnerPath: null };
119
128
  }
120
129
  const managedYieldsToStaged = stagedRunnerPath !== null &&
121
130
  hqCloud?.source === "managed" &&
@@ -408,6 +417,7 @@ export async function runDaemon(opts) {
408
417
  let legacySyncRunnerPathChecked = false;
409
418
  let legacySyncRunnerPath = null;
410
419
  let lastPlannedSyncRunnerSource = null;
420
+ let lastPlannedSyncRunnerVersion = null;
411
421
  let packageUseLeaseReleased = false;
412
422
  let hqCloudVersionBeforeRelease = null;
413
423
  let startupPlanOpen = true;
@@ -580,6 +590,7 @@ export async function runDaemon(opts) {
580
590
  let warnedMissingPin = null;
581
591
  let warnedMissingSyncRunner = false;
582
592
  const daemonSyncRunnerPids = new Set();
593
+ let supervisor = null;
583
594
  const plan = () => {
584
595
  const config = readDaemonConfig(paths.configPath);
585
596
  const legacy = legacyUnits();
@@ -594,19 +605,34 @@ export async function runDaemon(opts) {
594
605
  : currentHqCloud(paths.root);
595
606
  if (!packageUseLeaseReleased)
596
607
  hqCloudVersionBeforeRelease = hqCloud?.version ?? staged?.metadata.version ?? null;
597
- const missingPin = hqCloud?.pinnedVersionMissing ?? null;
598
- if (missingPin !== warnedMissingPin) {
599
- warnedMissingPin = missingPin;
600
- if (missingPin && hqCloud) {
601
- log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; running ${hqCloud.version} (${hqCloud.source}) until the updater installs it`);
602
- }
603
- }
604
608
  if (!packageUseLeaseReleased && !legacySyncRunnerPathChecked) {
605
609
  legacySyncRunnerPath = resolveSyncRunnerPath();
606
610
  legacySyncRunnerPathChecked = true;
607
611
  }
608
- const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null }, lastPlannedSyncRunnerSource);
612
+ const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null }, supervisor?.state().units.some((unit) => unit.id === "sync" && unit.status === "running" && typeof unit.pid === "number")
613
+ ? lastPlannedSyncRunnerSource
614
+ : null);
615
+ const retainedLiveSource = syncSource === lastPlannedSyncRunnerSource;
616
+ const syncRunnerVersion = retainedLiveSource
617
+ ? lastPlannedSyncRunnerVersion
618
+ : syncSource.kind === "staged-sync"
619
+ ? staged?.metadata.version ?? null
620
+ : hqCloud?.version ?? null;
621
+ const missingPin = config.hqCloudVersion && syncRunnerVersion !== config.hqCloudVersion ? config.hqCloudVersion : null;
622
+ const pinWarningKey = missingPin
623
+ ? `${missingPin}:${retainedLiveSource ? syncRunnerVersion ?? "unknown" : "unavailable"}`
624
+ : null;
625
+ if (pinWarningKey !== warnedMissingPin) {
626
+ warnedMissingPin = pinWarningKey;
627
+ if (missingPin && retainedLiveSource) {
628
+ log("warn", `hq-cloud is pinned to ${missingPin}, but the running sync runner is ${syncRunnerVersion ?? "unknown"}; keeping it running while the updater retries`);
629
+ }
630
+ else if (missingPin && syncSource.kind === "unavailable") {
631
+ log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; sync is unavailable until the updater installs it`);
632
+ }
633
+ }
609
634
  lastPlannedSyncRunnerSource = syncSource.runnerPath ? syncSource : null;
635
+ lastPlannedSyncRunnerVersion = syncSource.runnerPath ? syncRunnerVersion : null;
610
636
  if (!syncSource.runnerPath && !warnedMissingSyncRunner) {
611
637
  warnedMissingSyncRunner = true;
612
638
  log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
@@ -797,7 +823,7 @@ export async function runDaemon(opts) {
797
823
  // Set once the shutdown path exists below; a restart request before then is dropped.
798
824
  let restartDaemon = () => false;
799
825
  let extendSyncShutdownCap;
800
- const supervisor = createSupervisor({
826
+ supervisor = createSupervisor({
801
827
  machineType,
802
828
  host: opts.host,
803
829
  pid: process.pid,
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Resolve the `--app` value of the `hq db` commands to an app id. The value can
3
+ * be the app id, its name or its subdomain. Only apps in the given company are
4
+ * searched, so a name never resolves to another company's app.
5
+ */
6
+ import { deployApiRequest } from "../deploy-api.js";
7
+ export interface AppRefRow {
8
+ id?: unknown;
9
+ name?: unknown;
10
+ subdomain?: unknown;
11
+ }
12
+ export declare class AppRefError extends Error {
13
+ constructor(message: string);
14
+ }
15
+ export declare function matchAppRef(apps: AppRefRow[], ref: string): string;
16
+ export declare function resolveAppRef(input: {
17
+ token: string;
18
+ companySlug: string;
19
+ ref: string;
20
+ }, request?: typeof deployApiRequest): Promise<string>;
21
+ //# sourceMappingURL=app-ref.d.ts.map
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Resolve the `--app` value of the `hq db` commands to an app id. The value can
3
+ * be the app id, its name or its subdomain. Only apps in the given company are
4
+ * searched, so a name never resolves to another company's app.
5
+ */
6
+ import { deployApiRequest } from "../deploy-api.js";
7
+ import { listDeployApps } from "../deploy-apps.js";
8
+ export class AppRefError extends Error {
9
+ constructor(message) {
10
+ super(message);
11
+ this.name = "AppRefError";
12
+ }
13
+ }
14
+ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
15
+ export function matchAppRef(apps, ref) {
16
+ const ids = (rows) => [
17
+ ...new Set(rows.map((row) => row.id).filter((id) => typeof id === "string")),
18
+ ];
19
+ const byId = ids(apps.filter((row) => row.id === ref));
20
+ if (byId.length === 1)
21
+ return byId[0];
22
+ const byName = ids(apps.filter((row) => row.name === ref || row.subdomain === ref));
23
+ if (byName.length === 1)
24
+ return byName[0];
25
+ if (byName.length > 1) {
26
+ throw new AppRefError(`More than one app in this company is called "${ref}" (${byName.join(", ")}). Use the app id.`);
27
+ }
28
+ throw new AppRefError(`No app called "${ref}" was found in this company.`);
29
+ }
30
+ export async function resolveAppRef(input, request = deployApiRequest) {
31
+ const ref = input.ref.trim();
32
+ if (UUID_RE.test(ref))
33
+ return ref;
34
+ const headers = { "X-Org-Slug": input.companySlug };
35
+ const apps = await listDeployApps(async (cursor) => {
36
+ const route = cursor ? `/api/apps?cursor=${encodeURIComponent(cursor)}` : "/api/apps";
37
+ const res = await request(input.token, route, "GET", undefined, headers);
38
+ if (!res.ok) {
39
+ throw new AppRefError(`Could not look up the app "${ref}" (hq-deploy returned ${res.status}). Use the app id.`);
40
+ }
41
+ return (await res.json());
42
+ });
43
+ return matchAppRef(apps, ref);
44
+ }
45
+ //# sourceMappingURL=app-ref.js.map
@@ -10,6 +10,7 @@ import { botsRoot, botDir, launchdLabel } from "../../bot/paths.js";
10
10
  import { readBotConfig } from "../../bot/config.js";
11
11
  import { isPidAlive, readBotStatus } from "../../bot/status.js";
12
12
  import { nodeBinaryProblem } from "../../node-binary.js";
13
+ import { launchdPlistProgramArguments, parseLaunchdCommand } from "../../service-manager/launchd.js";
13
14
  export const BOTS_FAMILY_ID = "bots";
14
15
  export const BOTS_FAMILY_TITLE = "Local bots";
15
16
  const DEFAULT_DEPS = {
@@ -35,14 +36,8 @@ export function checkBots(_context, deps = DEFAULT_DEPS) {
35
36
  ];
36
37
  }
37
38
  }
38
- function launchdPlistArgs(plist) {
39
- const block = plist.match(/<key>ProgramArguments<\/key>\s*<array>([\s\S]*?)<\/array>/);
40
- if (!block)
41
- return [];
42
- return [...block[1].matchAll(/<string>([^<]*)<\/string>/g)].map((m) => m[1]);
43
- }
44
39
  function launchdPlistHqBinary(plist) {
45
- return launchdPlistArgs(plist)[1] ?? null;
40
+ return parseLaunchdCommand(launchdPlistProgramArguments(plist))?.hq ?? null;
46
41
  }
47
42
  function runChecks(deps) {
48
43
  const home = deps.home();
@@ -122,7 +117,7 @@ function runChecks(deps) {
122
117
  try {
123
118
  const text = fs.readFileSync(plistPath, "utf8");
124
119
  hqPath = launchdPlistHqBinary(text);
125
- nodePath = launchdPlistArgs(text)[0] ?? null;
120
+ nodePath = parseLaunchdCommand(launchdPlistProgramArguments(text))?.node ?? null;
126
121
  }
127
122
  catch {
128
123
  hqPath = null;
@@ -1,4 +1,4 @@
1
- type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "waiting";
1
+ type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "finished_story" | "idle_no_queued_work" | "waiting";
2
2
  interface KeepAliveAuditRestart {
3
3
  at: string | null;
4
4
  trigger: string;
@@ -17,6 +17,7 @@ interface KeepAliveAuditRow {
17
17
  blocked_outcome?: "woken" | "skipped";
18
18
  blocked_outcome_reason?: string;
19
19
  verdict: KeepAliveAuditVerdict;
20
+ queued_work_at_audit_time?: boolean;
20
21
  remediation?: string;
21
22
  terminal_reason?: string;
22
23
  }
@@ -25,6 +26,8 @@ interface KeepAliveAuditSummary {
25
26
  not_restarted: number;
26
27
  stranded: number;
27
28
  restarted_untimed: number;
29
+ finished_story: number;
30
+ idle_no_queued_work: number;
28
31
  late_restarts_after_window: number;
29
32
  late_over_10m: number;
30
33
  early_wake_by_others: number;
@@ -53,6 +56,7 @@ export interface StopEnvelope {
53
56
  name: string;
54
57
  mtimeMs: number;
55
58
  decision: "done" | "blocked";
59
+ storyComplete: boolean;
56
60
  nextCheckAt?: string;
57
61
  }
58
62
  export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
@@ -1,6 +1,7 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
- import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS } from "./keep-alive.js";
3
+ import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, doneKeepAliveVerdict, hasPendingKeepAliveWork, undeliveredAnsweredQuestions } from "./keep-alive.js";
4
+ import { dropboxDir, dropboxInboxPath } from "./dropbox.js";
4
5
  import { LanesError } from "./errors.js";
5
6
  import { laneRunDir } from "./spawn.js";
6
7
  import { compareEnvelopeNames } from "./envelope-sequence.js";
@@ -63,6 +64,8 @@ const VERDICTS = [
63
64
  "keep_alive_off",
64
65
  "not_restarted",
65
66
  "stranded",
67
+ "finished_story",
68
+ "idle_no_queued_work",
66
69
  "waiting",
67
70
  ];
68
71
  function parseTime(value) {
@@ -174,7 +177,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
174
177
  if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
175
178
  throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
176
179
  }
177
- const doneStop = raw.decision === "done" && raw.story_complete === false;
180
+ const doneStop = raw.decision === "done";
178
181
  const blockedStop = raw.decision === "blocked" && typeof raw.next_check_at === "string" &&
179
182
  isIsoDateTime(raw.next_check_at) && Date.parse(raw.next_check_at) > stat.mtimeMs;
180
183
  if (!doneStop && !blockedStop)
@@ -183,6 +186,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
183
186
  name,
184
187
  mtimeMs: stat.mtimeMs,
185
188
  decision: blockedStop ? "blocked" : "done",
189
+ storyComplete: raw.story_complete === true,
186
190
  ...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
187
191
  });
188
192
  }
@@ -428,6 +432,48 @@ function firstRestartAfter(lane, stop, nextEnvelopeMs, workerResumeEvents) {
428
432
  // untimed instead of using the lane's mutable current state as a proxy.
429
433
  return nextEnvelopeMs === undefined ? null : { at: null, trigger: "unknown", caller: null };
430
434
  }
435
+ /**
436
+ * When the work a finished standing lane is now waiting on first appeared.
437
+ * The earliest signal wins: if work was queued at 10:05 and more arrived at
438
+ * 11:59, keep-alive has owed a launch since 10:05. Returns undefined when no
439
+ * signal carries a time, which leaves the caller unable to open a window.
440
+ */
441
+ function keepAliveQueuedWorkAt(hqRoot, lane) {
442
+ const candidates = [];
443
+ const pendingAt = parseTime(lane.pending_resume?.at);
444
+ if (pendingAt !== undefined)
445
+ candidates.push(pendingAt);
446
+ for (const candidate of [dropboxInboxPath(hqRoot, lane.lane_id), dropboxDir(hqRoot, lane.lane_id)]) {
447
+ try {
448
+ candidates.push(fs.statSync(candidate).mtimeMs);
449
+ break;
450
+ }
451
+ catch (err) {
452
+ if (err.code !== "ENOENT")
453
+ throw err;
454
+ }
455
+ }
456
+ const questions = [];
457
+ for (const questionId of listQuestionIds(hqRoot)) {
458
+ try {
459
+ const question = tryReadQuestion(hqRoot, questionId);
460
+ if (question)
461
+ questions.push(question);
462
+ }
463
+ catch {
464
+ // One unreadable question file must not fail the whole audit.
465
+ continue;
466
+ }
467
+ }
468
+ for (const question of undeliveredAnsweredQuestions(lane, questions)) {
469
+ const answeredAt = parseTime(question.answer?.answered_at);
470
+ if (answeredAt !== undefined)
471
+ candidates.push(answeredAt);
472
+ }
473
+ if (candidates.length === 0)
474
+ return undefined;
475
+ return Math.min(...candidates);
476
+ }
431
477
  function classifyVerdict(input) {
432
478
  const { dueAt, restart, lane, nowMs } = input;
433
479
  if (restart?.trigger === "unknown" && laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
@@ -478,6 +524,34 @@ function classifyVerdict(input) {
478
524
  }
479
525
  if (laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
480
526
  return "keep_alive_off";
527
+ // Split the unrestarted stops by whether keep-alive owes this lane a wake at
528
+ // all. The discriminator is the shared done-state policy, not whether the
529
+ // envelope declared a next_check_at: a stop with story_complete false is owed
530
+ // a wake with or without one. Only done stops with no later envelope reach
531
+ // this branch, because a later envelope means the lane carried on.
532
+ if (input.decision === "done" && input.nextEnvelopeAt === undefined) {
533
+ const eligibility = doneKeepAliveVerdict(lane, {
534
+ decision: "done",
535
+ story_complete: input.storyComplete,
536
+ ...(input.nextCheckAt === undefined ? {} : { next_check_at: input.nextCheckAt }),
537
+ }, input.hasQueuedWork, { nowMs });
538
+ // Keep-alive is specified not to wake either case, so neither is a
539
+ // keep-alive liveness failure and neither counts against the KPI.
540
+ if (eligibility === "story_complete")
541
+ return "finished_story";
542
+ if (eligibility === "standing_idle")
543
+ return "idle_no_queued_work";
544
+ // A finished story becomes eligible again only once work is queued, which
545
+ // can be long after the stop. Open the failure window when that work
546
+ // appeared, not at the stop's own due time, or every normal exit from
547
+ // idle_no_queued_work reads as a liveness failure the moment it happens.
548
+ if (input.storyComplete) {
549
+ if (input.queuedWorkAt === undefined)
550
+ return "waiting";
551
+ const anchor = Math.max(dueAt, input.queuedWorkAt);
552
+ return nowMs > anchor + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS ? "not_restarted" : "waiting";
553
+ }
554
+ }
481
555
  if (nowMs > dueAt + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS)
482
556
  return "not_restarted";
483
557
  return "waiting";
@@ -546,6 +620,10 @@ export function buildKeepAliveAudit(input) {
546
620
  try {
547
621
  const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
548
622
  const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
623
+ // Read at audit time, not stop time: for a stop with no restart the
624
+ // question is whether anything is going to resume the lane now.
625
+ const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(input.hqRoot, lane);
626
+ const queuedWorkAt = hasQueuedWork ? keepAliveQueuedWorkAt(input.hqRoot, lane) : undefined;
549
627
  const laneWorkerResumeEvents = readLaneWorkerResumeEvents(input.hqRoot, lane.lane_id);
550
628
  const terminalClassifications = readTerminalClassifications(input.hqRoot, lane.lane_id);
551
629
  for (const stop of stops) {
@@ -572,7 +650,8 @@ export function buildKeepAliveAudit(input) {
572
650
  blocked_outcome: blockedOutcome.status,
573
651
  ...(blockedOutcome.reason === undefined ? {} : { blocked_outcome_reason: blockedOutcome.reason }),
574
652
  } : {}),
575
- verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
653
+ verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, decision: stop.decision, storyComplete: stop.storyComplete, hasQueuedWork, ...(queuedWorkAt === undefined ? {} : { queuedWorkAt }), lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
654
+ ...(stop.decision === "done" ? { queued_work_at_audit_time: hasQueuedWork } : {}),
576
655
  });
577
656
  }
578
657
  }
@@ -611,6 +690,8 @@ export function buildKeepAliveAudit(input) {
611
690
  not_restarted: byVerdict.not_restarted,
612
691
  stranded: byVerdict.stranded,
613
692
  restarted_untimed: byVerdict.restarted_untimed,
693
+ finished_story: byVerdict.finished_story,
694
+ idle_no_queued_work: byVerdict.idle_no_queued_work,
614
695
  late_restarts_after_window: lateRestartsAfterWindow,
615
696
  late_over_10m: lateOverTenMinutes,
616
697
  early_wake_by_others: byVerdict.early_wake,
@@ -642,6 +723,7 @@ export function formatKeepAliveAuditText(result) {
642
723
  "",
643
724
  `Since ${result.since}; ${counts}`,
644
725
  `not_restarted=${result.summary.not_restarted}, stranded=${result.summary.stranded}, late_restarts_after_window=${result.summary.late_restarts_after_window}, late_over_10m=${result.summary.late_over_10m}, early_wake_by_others=${result.summary.early_wake_by_others}, self_wake=${result.summary.self_wake}`,
726
+ `finished_story=${result.summary.finished_story}, idle_no_queued_work=${result.summary.idle_no_queued_work} (neither counts against the keep-alive KPI)`,
645
727
  `Stopped and not restarted on their own: ${result.summary.stopped_and_not_restarted_on_their_own}`,
646
728
  ...result.rows.filter((row) => row.remediation).map((row) => `Stranded ${row.lane}: ${row.remediation}`),
647
729
  ...result.companyless_lead_lanes.map((lane) => `Company-less lead ${lane.lane} (${lane.project}): ${lane.reason}; repair with hq lanes repair-company ${lane.lane} --company <slug>`),
@@ -100,6 +100,7 @@ export declare function buildOutpostLaneListCommand(input?: {
100
100
  laneUlids?: readonly string[];
101
101
  active?: boolean;
102
102
  hierarchy?: string | readonly string[];
103
+ senior?: string;
103
104
  snapshot?: boolean;
104
105
  }): string;
105
106
  /**
@@ -93,6 +93,8 @@ export function buildOutpostLaneListCommand(input = {}) {
93
93
  ? [input.hierarchy]
94
94
  : input.hierarchy ?? [])
95
95
  args.push("--hierarchy", laneId);
96
+ if (input.senior !== undefined)
97
+ args.push("--senior", input.senior);
96
98
  if (input.snapshot)
97
99
  args.push("--snapshot");
98
100
  return withRemoteHqDir(joinCommandParts(args));
@@ -23,7 +23,7 @@ import { isInstallableHqRoot, readDaemonConfig, registerTrustedHqRoot, verifyHqR
23
23
  import { defaultPidLockDeps, pidLockStatus, readPidLock } from "./pid-lock.js";
24
24
  import { retireLegacyListen } from "./legacy-listen.js";
25
25
  import { readDaemonState } from "./state.js";
26
- import { activateLaunchd, defaultRunCommand, launchdIsLoaded, launchdServiceTarget, launchdPlistProgramArguments, launchdProgramArgumentsDiffer, parseLaunchctlPrintArguments, } from "../../../service-manager/launchd.js";
26
+ import { activateLaunchd, defaultRunCommand, discoverLaunchdIdentity, installLaunchdPlist, launchdProgramArguments, launchdIsLoaded, launchdServiceTarget, launchdPlistProgramArguments, parseLaunchdCommand, launchdProgramArgumentsDiffer, parseLaunchctlPrintArguments, } from "../../../service-manager/launchd.js";
27
27
  import { activateSystemd, systemdIsActive } from "../../../service-manager/systemd.js";
28
28
  import { nodeBinaryProblem, preferStableNodeBinary } from "../../../node-binary.js";
29
29
  /**
@@ -428,9 +428,7 @@ export function installedDaemonNodeBinary(paths, platform, readUnit = (file) =>
428
428
  const text = readUnit(paths.plistPath);
429
429
  if (!text)
430
430
  return null;
431
- const block = text.match(/<key>ProgramArguments<\/key>\s*<array>([\s\S]*?)<\/array>/);
432
- const first = block?.[1]?.match(/<string>([^<]*)<\/string>/);
433
- return first?.[1] ?? null;
431
+ return parseLaunchdCommand(launchdPlistProgramArguments(text))?.node ?? null;
434
432
  }
435
433
  if (platform === "linux") {
436
434
  const text = readUnit(paths.systemdUnitPath);
@@ -544,19 +542,21 @@ function safeUsername() {
544
542
  }
545
543
  }
546
544
  export function renderLaunchdPlist(paths) {
545
+ const identity = discoverLaunchdIdentity(paths.home);
546
+ const args = launchdProgramArguments({
547
+ nodeBinary: paths.nodeBinary,
548
+ hqBinary: paths.hqBinary,
549
+ launchdLauncherBinary: identity.launcherBinary,
550
+ }, ["mesh", "daemon", "run"]);
547
551
  return `<?xml version="1.0" encoding="UTF-8"?>
548
552
  <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
549
553
  <plist version="1.0">
550
554
  <dict>
551
555
  <key>Label</key>
552
556
  <string>${LAUNCHD_LABEL}</string>
553
- <key>ProgramArguments</key>
557
+ ${identity.associatedBundleIdentifier ? ` <key>AssociatedBundleIdentifiers</key>\n <array>\n <string>${escapeXml(identity.associatedBundleIdentifier)}</string>\n </array>\n` : ""} <key>ProgramArguments</key>
554
558
  <array>
555
- <string>${escapeXml(paths.nodeBinary)}</string>
556
- <string>${escapeXml(paths.hqBinary)}</string>
557
- <string>mesh</string>
558
- <string>daemon</string>
559
- <string>run</string>
559
+ ${args.map((arg) => ` <string>${escapeXml(arg)}</string>`).join("\n")}
560
560
  </array>
561
561
  <key>WorkingDirectory</key>
562
562
  <string>${escapeXml(paths.daemonDir)}</string>
@@ -920,11 +920,24 @@ export function installDaemonService(opts) {
920
920
  if (platform === "darwin") {
921
921
  const rendered = renderLaunchdPlist(paths);
922
922
  mkdirSync(paths.launchAgentsDir, { recursive: true });
923
- replaceUnitFile(paths.plistPath, rendered, 0o600, () => {
924
- if (opts.run)
925
- activateLaunchd(paths.plistPath, opts.run, opts.uid, { sleep: opts.sleep });
926
- else
927
- opts.activate?.(platform, paths.plistPath);
923
+ installLaunchdPlist(paths.plistPath, LAUNCHD_LABEL, rendered, {
924
+ files: {
925
+ existsSync: fs.existsSync,
926
+ readFileSync: (file) => fs.readFileSync(file, "utf8"),
927
+ writeFileSync: fs.writeFileSync,
928
+ chmodSync: fs.chmodSync,
929
+ renameSync: fs.renameSync,
930
+ unlinkSync: fs.unlinkSync,
931
+ },
932
+ activate: Boolean(opts.run || opts.activate),
933
+ uid: opts.uid,
934
+ rollbackOnReloadError: true,
935
+ reload: () => {
936
+ if (opts.run)
937
+ activateLaunchd(paths.plistPath, opts.run, opts.uid, { sleep: opts.sleep });
938
+ else if (opts.activate)
939
+ opts.activate(platform, paths.plistPath);
940
+ },
928
941
  });
929
942
  return {
930
943
  platform,
@@ -15,9 +15,12 @@ export interface ServiceManagerDeps {
15
15
  platform?: ServicePlatform;
16
16
  run?: RunCommand;
17
17
  writeFileSync?: typeof fs.writeFileSync;
18
+ chmodSync?: typeof fs.chmodSync;
19
+ renameSync?: typeof fs.renameSync;
18
20
  mkdirSync?: typeof fs.mkdirSync;
19
21
  unlinkSync?: typeof fs.unlinkSync;
20
22
  existsSync?: typeof fs.existsSync;
23
+ readFileSync?: typeof fs.readFileSync;
21
24
  /** When false, render + write units but do not (de)activate them. */
22
25
  activate?: boolean;
23
26
  }