@indigoai-us/hq-cli 5.345.49 → 5.345.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/command-catalog.generated.d.ts +56 -11
- package/dist/command-catalog.generated.js +67 -11
- package/dist/commands/core-hq-session-native.js +2 -2
- package/dist/commands/core-resolve-company-native.js +2 -2
- package/dist/commands/db-destroy.js +13 -11
- package/dist/commands/db-dump.js +3 -3
- package/dist/commands/db-remote-deps.d.ts +3 -0
- package/dist/commands/db-remote-deps.js +13 -0
- package/dist/commands/db-sql.js +4 -3
- package/dist/commands/db-usage.js +4 -3
- package/dist/commands/dm-forward-flag.d.ts +4 -0
- package/dist/commands/dm-forward-flag.js +7 -0
- package/dist/commands/dm.d.ts +8 -0
- package/dist/commands/dm.js +219 -27
- package/dist/commands/lanes.js +52 -11
- package/dist/commands/secrets.js +2 -2
- package/dist/commands/workers.js +2 -2
- package/dist/lib/bot/daemon.d.ts +1 -1
- package/dist/lib/bot/daemon.js +35 -29
- package/dist/lib/daemon/hq-cloud.d.ts +2 -2
- package/dist/lib/daemon/hq-cloud.js +38 -9
- package/dist/lib/daemon/install.js +34 -3
- package/dist/lib/daemon/run.js +45 -19
- package/dist/lib/db/app-ref.d.ts +21 -0
- package/dist/lib/db/app-ref.js +45 -0
- package/dist/lib/doctor/checks/bots.js +3 -8
- package/dist/lib/lanes/keep-alive-audit.d.ts +5 -1
- package/dist/lib/lanes/keep-alive-audit.js +85 -3
- package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
- package/dist/lib/lanes/outpost-dispatch.js +2 -0
- package/dist/lib/mesh/live/daemon/install.js +28 -15
- package/dist/lib/service-manager/index.d.ts +3 -0
- package/dist/lib/service-manager/index.js +22 -6
- package/dist/lib/service-manager/launchd.d.ts +40 -3
- package/dist/lib/service-manager/launchd.js +106 -5
- package/dist/lib/service-manager/migration-state.d.ts +12 -0
- package/dist/lib/service-manager/migration-state.js +46 -0
- package/dist/lib/service-manager/migration.d.ts +22 -0
- package/dist/lib/service-manager/migration.js +85 -0
- package/dist/lib/service-manager/startup-migration.d.ts +24 -0
- package/dist/lib/service-manager/startup-migration.js +27 -0
- package/dist/lib/service-manager/types.d.ts +3 -0
- package/dist/main.js +22 -0
- package/dist/native-fast-route.d.ts +4 -0
- package/dist/native-fast-route.js +20 -0
- package/dist/utils/resident-package-preload.js +2 -1
- package/package.json +1 -1
|
@@ -54,10 +54,11 @@ export function pickHqCloud(bundled, managed, usable = () => true, pinnedVersion
|
|
|
54
54
|
if (managed && managed.version === pinnedVersion && b && m && m[0] === b[0] && usable(managed)) {
|
|
55
55
|
return { ...managed, source: "managed" };
|
|
56
56
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
//
|
|
60
|
-
|
|
57
|
+
if (bundled.version === pinnedVersion && b && usable(bundled))
|
|
58
|
+
return { ...bundled, source: "bundled" };
|
|
59
|
+
// A pin is an exact version contract. Do not silently run another installed
|
|
60
|
+
// package while npm has not yet delivered the pinned version.
|
|
61
|
+
return null;
|
|
61
62
|
}
|
|
62
63
|
return pickNewest(bundled, managed, b, m, usable);
|
|
63
64
|
}
|
|
@@ -160,6 +161,17 @@ export function currentHqCloudAfterLeaseRelease(daemonRoot, expectedVersion) {
|
|
|
160
161
|
return null;
|
|
161
162
|
return { ...managed, source: "managed" };
|
|
162
163
|
}
|
|
164
|
+
function isTransientNpmFailure(result) {
|
|
165
|
+
// spawnSync's ETIMEDOUT means this attempt hit our full 10-minute process
|
|
166
|
+
// timeout. Retrying that result would multiply one stall across every attempt.
|
|
167
|
+
if (result.error?.code === "ETIMEDOUT")
|
|
168
|
+
return false;
|
|
169
|
+
const text = `${result.stderr ?? ""}\n${result.stdout ?? ""}\n${result.error?.message ?? ""}`;
|
|
170
|
+
return /\b(?:ETARGET|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENETUNREACH|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|ENETWORK|ECONNABORTED)\b|\bnotarget\b|network timeout|fetch failed|socket hang up/i.test(text);
|
|
171
|
+
}
|
|
172
|
+
function sleepSync(ms) {
|
|
173
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
174
|
+
}
|
|
163
175
|
function runNpm(file, args, shell, env) {
|
|
164
176
|
return spawnSync(file, args, {
|
|
165
177
|
encoding: "utf8",
|
|
@@ -180,7 +192,7 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
|
|
|
180
192
|
const platform = deps.platform ?? process.platform;
|
|
181
193
|
const pinnedVersion = readDaemonConfig(path.join(daemonRoot, "config.json")).hqCloudVersion;
|
|
182
194
|
const bundled = deps.bundled !== undefined ? deps.bundled : bundledHqCloud();
|
|
183
|
-
const from =
|
|
195
|
+
const from = managedHqCloud(daemonRoot)?.version ?? bundled?.version ?? null;
|
|
184
196
|
if (!bundled)
|
|
185
197
|
return { ok: false, from, to: null, detail: "this CLI install has no bundled hq-cloud to match" };
|
|
186
198
|
const prefix = managedHqCloudDir(daemonRoot);
|
|
@@ -203,9 +215,26 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
|
|
|
203
215
|
version = latest;
|
|
204
216
|
}
|
|
205
217
|
const cmd = hqCloudInstallCommand(prefix, version, platform);
|
|
206
|
-
const
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
218
|
+
const retryDelays = [500, 1_000, 2_000];
|
|
219
|
+
const sleep = deps.sleep ?? sleepSync;
|
|
220
|
+
let r;
|
|
221
|
+
for (let attempt = 0;; attempt++) {
|
|
222
|
+
r = run(cmd.file, cmd.args, cmd.shell, env);
|
|
223
|
+
if (r.status === 0)
|
|
224
|
+
break;
|
|
225
|
+
const retryable = pinnedVersion !== undefined && pinnedVersion !== null && isTransientNpmFailure(r);
|
|
226
|
+
const delay = retryDelays[attempt];
|
|
227
|
+
if (!retryable || delay === undefined) {
|
|
228
|
+
const retries = retryable ? attempt : 0;
|
|
229
|
+
return {
|
|
230
|
+
ok: false,
|
|
231
|
+
from,
|
|
232
|
+
to: null,
|
|
233
|
+
detail: retries > 0 ? `pinned hq-cloud install failed after ${retries + 1} attempts: ${npmFailure(r)}` : npmFailure(r),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
sleep(delay);
|
|
237
|
+
}
|
|
238
|
+
return { ok: true, from, to: managedHqCloud(daemonRoot)?.version ?? null };
|
|
210
239
|
}
|
|
211
240
|
//# sourceMappingURL=hq-cloud.js.map
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import * as fs from "node:fs";
|
|
19
19
|
import * as path from "node:path";
|
|
20
|
-
import { activateLaunchd, deactivateLaunchd, launchdPlistPath, renderLaunchdPlist, } from "../service-manager/launchd.js";
|
|
20
|
+
import { activateLaunchd, deactivateLaunchd, defaultRunCommand, installLaunchdPlist, launchdIsLoaded, launchdPlistPath, renderLaunchdPlist, } from "../service-manager/launchd.js";
|
|
21
21
|
import { activateSystemd, deactivateSystemd, renderSystemdUserUnit, systemdUnitPath, } from "../service-manager/systemd.js";
|
|
22
22
|
import { escapeXml, shellQuote } from "../service-manager/types.js";
|
|
23
23
|
import { readPidLock } from "../mesh/live/daemon/pid-lock.js";
|
|
@@ -251,12 +251,43 @@ export function installDaemonUnit(host, options, deps) {
|
|
|
251
251
|
fs.mkdirSync(path.dirname(plan.unitPath), { recursive: true });
|
|
252
252
|
if (plan.kind !== "systemd-system")
|
|
253
253
|
fs.mkdirSync(path.join(host.daemonDir, "logs"), { recursive: true });
|
|
254
|
-
if (plan.kind === "
|
|
254
|
+
if (plan.kind === "launchd") {
|
|
255
|
+
let plist;
|
|
256
|
+
try {
|
|
257
|
+
plist = installLaunchdPlist(plan.unitPath, DAEMON_LAUNCHD_LABEL, plan.rendered, {
|
|
258
|
+
run: deps.run ?? defaultRunCommand,
|
|
259
|
+
uid: host.uid,
|
|
260
|
+
sleep: deps.sleep,
|
|
261
|
+
isLoaded: () => launchdIsLoaded(DAEMON_LAUNCHD_LABEL, deps.run ?? defaultRunCommand, host.uid),
|
|
262
|
+
files: {
|
|
263
|
+
existsSync: fs.existsSync,
|
|
264
|
+
readFileSync: (file) => fs.readFileSync(file, "utf8"),
|
|
265
|
+
writeFileSync: fs.writeFileSync,
|
|
266
|
+
chmodSync: fs.chmodSync,
|
|
267
|
+
renameSync: fs.renameSync,
|
|
268
|
+
unlinkSync: fs.unlinkSync,
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
catch (err) {
|
|
273
|
+
return {
|
|
274
|
+
kind: plan.kind,
|
|
275
|
+
unitPath: plan.unitPath,
|
|
276
|
+
installed: fs.existsSync(plan.unitPath),
|
|
277
|
+
running: launchdIsLoaded(DAEMON_LAUNCHD_LABEL, deps.run ?? defaultRunCommand, host.uid),
|
|
278
|
+
error: err instanceof Error ? err.message : String(err),
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
if (!plist.loaded)
|
|
282
|
+
return { kind: plan.kind, unitPath: plan.unitPath, installed: true, running: false };
|
|
283
|
+
}
|
|
284
|
+
else if (plan.kind === "schtasks")
|
|
255
285
|
fs.writeFileSync(plan.unitPath, encodeWindowsTaskXml(plan.rendered));
|
|
256
286
|
else
|
|
257
287
|
fs.writeFileSync(plan.unitPath, plan.rendered, { mode: plan.kind === "systemd-system" ? 0o644 : 0o600 });
|
|
258
288
|
try {
|
|
259
|
-
|
|
289
|
+
if (plan.kind !== "launchd")
|
|
290
|
+
activate(plan, host, deps.run, deps.sleep);
|
|
260
291
|
return { kind: plan.kind, unitPath: plan.unitPath, installed: true, running: true };
|
|
261
292
|
}
|
|
262
293
|
catch (err) {
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -105,17 +105,26 @@ const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
|
|
|
105
105
|
const HQD_PERSON_SETTING_GRACE_MS = 30 * 60_000;
|
|
106
106
|
export function syncRunnerSourceForPlan(stagedRunnerPath, hqCloud, legacyBundledRunnerPath = null, staged = { version: null }, currentSyncRunnerSource = null) {
|
|
107
107
|
// A managed copy lives in the daemon directory, so it survives a CLI
|
|
108
|
-
// replacement.
|
|
109
|
-
//
|
|
110
|
-
// eligible package, so it must not displace a newer staged bundle unless it
|
|
111
|
-
// is the pinned version. The staged bundle is built from the bundled copy and
|
|
112
|
-
// otherwise only stands in for that copy (hq-cli#1547).
|
|
108
|
+
// replacement. A configured pin is an exact version contract: only that
|
|
109
|
+
// installed package or an exactly matching staged bundle may start.
|
|
113
110
|
const pin = staged.pinnedVersion;
|
|
114
|
-
|
|
115
|
-
(
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
111
|
+
if (pin) {
|
|
112
|
+
if (hqCloud?.version === pin) {
|
|
113
|
+
if (hqCloud.source === "managed") {
|
|
114
|
+
return { kind: "managed-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
|
|
115
|
+
}
|
|
116
|
+
if (stagedRunnerPath && staged.version === pin)
|
|
117
|
+
return { kind: "staged-sync", runnerPath: stagedRunnerPath };
|
|
118
|
+
return { kind: "bundled-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
|
|
119
|
+
}
|
|
120
|
+
if (stagedRunnerPath && staged.version === pin)
|
|
121
|
+
return { kind: "staged-sync", runnerPath: stagedRunnerPath };
|
|
122
|
+
// A transient install gap must not stop a runner that is already serving
|
|
123
|
+
// sync. On cold start there is no current source, so mismatched bundles
|
|
124
|
+
// remain unavailable and cannot be started.
|
|
125
|
+
if (currentSyncRunnerSource?.runnerPath)
|
|
126
|
+
return currentSyncRunnerSource;
|
|
127
|
+
return { kind: "unavailable", runnerPath: null };
|
|
119
128
|
}
|
|
120
129
|
const managedYieldsToStaged = stagedRunnerPath !== null &&
|
|
121
130
|
hqCloud?.source === "managed" &&
|
|
@@ -408,6 +417,7 @@ export async function runDaemon(opts) {
|
|
|
408
417
|
let legacySyncRunnerPathChecked = false;
|
|
409
418
|
let legacySyncRunnerPath = null;
|
|
410
419
|
let lastPlannedSyncRunnerSource = null;
|
|
420
|
+
let lastPlannedSyncRunnerVersion = null;
|
|
411
421
|
let packageUseLeaseReleased = false;
|
|
412
422
|
let hqCloudVersionBeforeRelease = null;
|
|
413
423
|
let startupPlanOpen = true;
|
|
@@ -580,6 +590,7 @@ export async function runDaemon(opts) {
|
|
|
580
590
|
let warnedMissingPin = null;
|
|
581
591
|
let warnedMissingSyncRunner = false;
|
|
582
592
|
const daemonSyncRunnerPids = new Set();
|
|
593
|
+
let supervisor = null;
|
|
583
594
|
const plan = () => {
|
|
584
595
|
const config = readDaemonConfig(paths.configPath);
|
|
585
596
|
const legacy = legacyUnits();
|
|
@@ -594,19 +605,34 @@ export async function runDaemon(opts) {
|
|
|
594
605
|
: currentHqCloud(paths.root);
|
|
595
606
|
if (!packageUseLeaseReleased)
|
|
596
607
|
hqCloudVersionBeforeRelease = hqCloud?.version ?? staged?.metadata.version ?? null;
|
|
597
|
-
const missingPin = hqCloud?.pinnedVersionMissing ?? null;
|
|
598
|
-
if (missingPin !== warnedMissingPin) {
|
|
599
|
-
warnedMissingPin = missingPin;
|
|
600
|
-
if (missingPin && hqCloud) {
|
|
601
|
-
log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; running ${hqCloud.version} (${hqCloud.source}) until the updater installs it`);
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
608
|
if (!packageUseLeaseReleased && !legacySyncRunnerPathChecked) {
|
|
605
609
|
legacySyncRunnerPath = resolveSyncRunnerPath();
|
|
606
610
|
legacySyncRunnerPathChecked = true;
|
|
607
611
|
}
|
|
608
|
-
const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null },
|
|
612
|
+
const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null }, supervisor?.state().units.some((unit) => unit.id === "sync" && unit.status === "running" && typeof unit.pid === "number")
|
|
613
|
+
? lastPlannedSyncRunnerSource
|
|
614
|
+
: null);
|
|
615
|
+
const retainedLiveSource = syncSource === lastPlannedSyncRunnerSource;
|
|
616
|
+
const syncRunnerVersion = retainedLiveSource
|
|
617
|
+
? lastPlannedSyncRunnerVersion
|
|
618
|
+
: syncSource.kind === "staged-sync"
|
|
619
|
+
? staged?.metadata.version ?? null
|
|
620
|
+
: hqCloud?.version ?? null;
|
|
621
|
+
const missingPin = config.hqCloudVersion && syncRunnerVersion !== config.hqCloudVersion ? config.hqCloudVersion : null;
|
|
622
|
+
const pinWarningKey = missingPin
|
|
623
|
+
? `${missingPin}:${retainedLiveSource ? syncRunnerVersion ?? "unknown" : "unavailable"}`
|
|
624
|
+
: null;
|
|
625
|
+
if (pinWarningKey !== warnedMissingPin) {
|
|
626
|
+
warnedMissingPin = pinWarningKey;
|
|
627
|
+
if (missingPin && retainedLiveSource) {
|
|
628
|
+
log("warn", `hq-cloud is pinned to ${missingPin}, but the running sync runner is ${syncRunnerVersion ?? "unknown"}; keeping it running while the updater retries`);
|
|
629
|
+
}
|
|
630
|
+
else if (missingPin && syncSource.kind === "unavailable") {
|
|
631
|
+
log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; sync is unavailable until the updater installs it`);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
609
634
|
lastPlannedSyncRunnerSource = syncSource.runnerPath ? syncSource : null;
|
|
635
|
+
lastPlannedSyncRunnerVersion = syncSource.runnerPath ? syncRunnerVersion : null;
|
|
610
636
|
if (!syncSource.runnerPath && !warnedMissingSyncRunner) {
|
|
611
637
|
warnedMissingSyncRunner = true;
|
|
612
638
|
log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
|
|
@@ -797,7 +823,7 @@ export async function runDaemon(opts) {
|
|
|
797
823
|
// Set once the shutdown path exists below; a restart request before then is dropped.
|
|
798
824
|
let restartDaemon = () => false;
|
|
799
825
|
let extendSyncShutdownCap;
|
|
800
|
-
|
|
826
|
+
supervisor = createSupervisor({
|
|
801
827
|
machineType,
|
|
802
828
|
host: opts.host,
|
|
803
829
|
pid: process.pid,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the `--app` value of the `hq db` commands to an app id. The value can
|
|
3
|
+
* be the app id, its name or its subdomain. Only apps in the given company are
|
|
4
|
+
* searched, so a name never resolves to another company's app.
|
|
5
|
+
*/
|
|
6
|
+
import { deployApiRequest } from "../deploy-api.js";
|
|
7
|
+
export interface AppRefRow {
|
|
8
|
+
id?: unknown;
|
|
9
|
+
name?: unknown;
|
|
10
|
+
subdomain?: unknown;
|
|
11
|
+
}
|
|
12
|
+
export declare class AppRefError extends Error {
|
|
13
|
+
constructor(message: string);
|
|
14
|
+
}
|
|
15
|
+
export declare function matchAppRef(apps: AppRefRow[], ref: string): string;
|
|
16
|
+
export declare function resolveAppRef(input: {
|
|
17
|
+
token: string;
|
|
18
|
+
companySlug: string;
|
|
19
|
+
ref: string;
|
|
20
|
+
}, request?: typeof deployApiRequest): Promise<string>;
|
|
21
|
+
//# sourceMappingURL=app-ref.d.ts.map
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the `--app` value of the `hq db` commands to an app id. The value can
|
|
3
|
+
* be the app id, its name or its subdomain. Only apps in the given company are
|
|
4
|
+
* searched, so a name never resolves to another company's app.
|
|
5
|
+
*/
|
|
6
|
+
import { deployApiRequest } from "../deploy-api.js";
|
|
7
|
+
import { listDeployApps } from "../deploy-apps.js";
|
|
8
|
+
export class AppRefError extends Error {
|
|
9
|
+
constructor(message) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.name = "AppRefError";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
15
|
+
export function matchAppRef(apps, ref) {
|
|
16
|
+
const ids = (rows) => [
|
|
17
|
+
...new Set(rows.map((row) => row.id).filter((id) => typeof id === "string")),
|
|
18
|
+
];
|
|
19
|
+
const byId = ids(apps.filter((row) => row.id === ref));
|
|
20
|
+
if (byId.length === 1)
|
|
21
|
+
return byId[0];
|
|
22
|
+
const byName = ids(apps.filter((row) => row.name === ref || row.subdomain === ref));
|
|
23
|
+
if (byName.length === 1)
|
|
24
|
+
return byName[0];
|
|
25
|
+
if (byName.length > 1) {
|
|
26
|
+
throw new AppRefError(`More than one app in this company is called "${ref}" (${byName.join(", ")}). Use the app id.`);
|
|
27
|
+
}
|
|
28
|
+
throw new AppRefError(`No app called "${ref}" was found in this company.`);
|
|
29
|
+
}
|
|
30
|
+
export async function resolveAppRef(input, request = deployApiRequest) {
|
|
31
|
+
const ref = input.ref.trim();
|
|
32
|
+
if (UUID_RE.test(ref))
|
|
33
|
+
return ref;
|
|
34
|
+
const headers = { "X-Org-Slug": input.companySlug };
|
|
35
|
+
const apps = await listDeployApps(async (cursor) => {
|
|
36
|
+
const route = cursor ? `/api/apps?cursor=${encodeURIComponent(cursor)}` : "/api/apps";
|
|
37
|
+
const res = await request(input.token, route, "GET", undefined, headers);
|
|
38
|
+
if (!res.ok) {
|
|
39
|
+
throw new AppRefError(`Could not look up the app "${ref}" (hq-deploy returned ${res.status}). Use the app id.`);
|
|
40
|
+
}
|
|
41
|
+
return (await res.json());
|
|
42
|
+
});
|
|
43
|
+
return matchAppRef(apps, ref);
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=app-ref.js.map
|
|
@@ -10,6 +10,7 @@ import { botsRoot, botDir, launchdLabel } from "../../bot/paths.js";
|
|
|
10
10
|
import { readBotConfig } from "../../bot/config.js";
|
|
11
11
|
import { isPidAlive, readBotStatus } from "../../bot/status.js";
|
|
12
12
|
import { nodeBinaryProblem } from "../../node-binary.js";
|
|
13
|
+
import { launchdPlistProgramArguments, parseLaunchdCommand } from "../../service-manager/launchd.js";
|
|
13
14
|
export const BOTS_FAMILY_ID = "bots";
|
|
14
15
|
export const BOTS_FAMILY_TITLE = "Local bots";
|
|
15
16
|
const DEFAULT_DEPS = {
|
|
@@ -35,14 +36,8 @@ export function checkBots(_context, deps = DEFAULT_DEPS) {
|
|
|
35
36
|
];
|
|
36
37
|
}
|
|
37
38
|
}
|
|
38
|
-
function launchdPlistArgs(plist) {
|
|
39
|
-
const block = plist.match(/<key>ProgramArguments<\/key>\s*<array>([\s\S]*?)<\/array>/);
|
|
40
|
-
if (!block)
|
|
41
|
-
return [];
|
|
42
|
-
return [...block[1].matchAll(/<string>([^<]*)<\/string>/g)].map((m) => m[1]);
|
|
43
|
-
}
|
|
44
39
|
function launchdPlistHqBinary(plist) {
|
|
45
|
-
return
|
|
40
|
+
return parseLaunchdCommand(launchdPlistProgramArguments(plist))?.hq ?? null;
|
|
46
41
|
}
|
|
47
42
|
function runChecks(deps) {
|
|
48
43
|
const home = deps.home();
|
|
@@ -122,7 +117,7 @@ function runChecks(deps) {
|
|
|
122
117
|
try {
|
|
123
118
|
const text = fs.readFileSync(plistPath, "utf8");
|
|
124
119
|
hqPath = launchdPlistHqBinary(text);
|
|
125
|
-
nodePath =
|
|
120
|
+
nodePath = parseLaunchdCommand(launchdPlistProgramArguments(text))?.node ?? null;
|
|
126
121
|
}
|
|
127
122
|
catch {
|
|
128
123
|
hqPath = null;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "waiting";
|
|
1
|
+
type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "finished_story" | "idle_no_queued_work" | "waiting";
|
|
2
2
|
interface KeepAliveAuditRestart {
|
|
3
3
|
at: string | null;
|
|
4
4
|
trigger: string;
|
|
@@ -17,6 +17,7 @@ interface KeepAliveAuditRow {
|
|
|
17
17
|
blocked_outcome?: "woken" | "skipped";
|
|
18
18
|
blocked_outcome_reason?: string;
|
|
19
19
|
verdict: KeepAliveAuditVerdict;
|
|
20
|
+
queued_work_at_audit_time?: boolean;
|
|
20
21
|
remediation?: string;
|
|
21
22
|
terminal_reason?: string;
|
|
22
23
|
}
|
|
@@ -25,6 +26,8 @@ interface KeepAliveAuditSummary {
|
|
|
25
26
|
not_restarted: number;
|
|
26
27
|
stranded: number;
|
|
27
28
|
restarted_untimed: number;
|
|
29
|
+
finished_story: number;
|
|
30
|
+
idle_no_queued_work: number;
|
|
28
31
|
late_restarts_after_window: number;
|
|
29
32
|
late_over_10m: number;
|
|
30
33
|
early_wake_by_others: number;
|
|
@@ -53,6 +56,7 @@ export interface StopEnvelope {
|
|
|
53
56
|
name: string;
|
|
54
57
|
mtimeMs: number;
|
|
55
58
|
decision: "done" | "blocked";
|
|
59
|
+
storyComplete: boolean;
|
|
56
60
|
nextCheckAt?: string;
|
|
57
61
|
}
|
|
58
62
|
export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS } from "./keep-alive.js";
|
|
3
|
+
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, doneKeepAliveVerdict, hasPendingKeepAliveWork, undeliveredAnsweredQuestions } from "./keep-alive.js";
|
|
4
|
+
import { dropboxDir, dropboxInboxPath } from "./dropbox.js";
|
|
4
5
|
import { LanesError } from "./errors.js";
|
|
5
6
|
import { laneRunDir } from "./spawn.js";
|
|
6
7
|
import { compareEnvelopeNames } from "./envelope-sequence.js";
|
|
@@ -63,6 +64,8 @@ const VERDICTS = [
|
|
|
63
64
|
"keep_alive_off",
|
|
64
65
|
"not_restarted",
|
|
65
66
|
"stranded",
|
|
67
|
+
"finished_story",
|
|
68
|
+
"idle_no_queued_work",
|
|
66
69
|
"waiting",
|
|
67
70
|
];
|
|
68
71
|
function parseTime(value) {
|
|
@@ -174,7 +177,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
|
|
|
174
177
|
if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
|
|
175
178
|
throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
|
|
176
179
|
}
|
|
177
|
-
const doneStop = raw.decision === "done"
|
|
180
|
+
const doneStop = raw.decision === "done";
|
|
178
181
|
const blockedStop = raw.decision === "blocked" && typeof raw.next_check_at === "string" &&
|
|
179
182
|
isIsoDateTime(raw.next_check_at) && Date.parse(raw.next_check_at) > stat.mtimeMs;
|
|
180
183
|
if (!doneStop && !blockedStop)
|
|
@@ -183,6 +186,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
|
|
|
183
186
|
name,
|
|
184
187
|
mtimeMs: stat.mtimeMs,
|
|
185
188
|
decision: blockedStop ? "blocked" : "done",
|
|
189
|
+
storyComplete: raw.story_complete === true,
|
|
186
190
|
...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
|
|
187
191
|
});
|
|
188
192
|
}
|
|
@@ -428,6 +432,48 @@ function firstRestartAfter(lane, stop, nextEnvelopeMs, workerResumeEvents) {
|
|
|
428
432
|
// untimed instead of using the lane's mutable current state as a proxy.
|
|
429
433
|
return nextEnvelopeMs === undefined ? null : { at: null, trigger: "unknown", caller: null };
|
|
430
434
|
}
|
|
435
|
+
/**
|
|
436
|
+
* When the work a finished standing lane is now waiting on first appeared.
|
|
437
|
+
* The earliest signal wins: if work was queued at 10:05 and more arrived at
|
|
438
|
+
* 11:59, keep-alive has owed a launch since 10:05. Returns undefined when no
|
|
439
|
+
* signal carries a time, which leaves the caller unable to open a window.
|
|
440
|
+
*/
|
|
441
|
+
function keepAliveQueuedWorkAt(hqRoot, lane) {
|
|
442
|
+
const candidates = [];
|
|
443
|
+
const pendingAt = parseTime(lane.pending_resume?.at);
|
|
444
|
+
if (pendingAt !== undefined)
|
|
445
|
+
candidates.push(pendingAt);
|
|
446
|
+
for (const candidate of [dropboxInboxPath(hqRoot, lane.lane_id), dropboxDir(hqRoot, lane.lane_id)]) {
|
|
447
|
+
try {
|
|
448
|
+
candidates.push(fs.statSync(candidate).mtimeMs);
|
|
449
|
+
break;
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
if (err.code !== "ENOENT")
|
|
453
|
+
throw err;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
const questions = [];
|
|
457
|
+
for (const questionId of listQuestionIds(hqRoot)) {
|
|
458
|
+
try {
|
|
459
|
+
const question = tryReadQuestion(hqRoot, questionId);
|
|
460
|
+
if (question)
|
|
461
|
+
questions.push(question);
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
// One unreadable question file must not fail the whole audit.
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
for (const question of undeliveredAnsweredQuestions(lane, questions)) {
|
|
469
|
+
const answeredAt = parseTime(question.answer?.answered_at);
|
|
470
|
+
if (answeredAt !== undefined)
|
|
471
|
+
candidates.push(answeredAt);
|
|
472
|
+
}
|
|
473
|
+
if (candidates.length === 0)
|
|
474
|
+
return undefined;
|
|
475
|
+
return Math.min(...candidates);
|
|
476
|
+
}
|
|
431
477
|
function classifyVerdict(input) {
|
|
432
478
|
const { dueAt, restart, lane, nowMs } = input;
|
|
433
479
|
if (restart?.trigger === "unknown" && laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
|
|
@@ -478,6 +524,34 @@ function classifyVerdict(input) {
|
|
|
478
524
|
}
|
|
479
525
|
if (laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
|
|
480
526
|
return "keep_alive_off";
|
|
527
|
+
// Split the unrestarted stops by whether keep-alive owes this lane a wake at
|
|
528
|
+
// all. The discriminator is the shared done-state policy, not whether the
|
|
529
|
+
// envelope declared a next_check_at: a stop with story_complete false is owed
|
|
530
|
+
// a wake with or without one. Only done stops with no later envelope reach
|
|
531
|
+
// this branch, because a later envelope means the lane carried on.
|
|
532
|
+
if (input.decision === "done" && input.nextEnvelopeAt === undefined) {
|
|
533
|
+
const eligibility = doneKeepAliveVerdict(lane, {
|
|
534
|
+
decision: "done",
|
|
535
|
+
story_complete: input.storyComplete,
|
|
536
|
+
...(input.nextCheckAt === undefined ? {} : { next_check_at: input.nextCheckAt }),
|
|
537
|
+
}, input.hasQueuedWork, { nowMs });
|
|
538
|
+
// Keep-alive is specified not to wake either case, so neither is a
|
|
539
|
+
// keep-alive liveness failure and neither counts against the KPI.
|
|
540
|
+
if (eligibility === "story_complete")
|
|
541
|
+
return "finished_story";
|
|
542
|
+
if (eligibility === "standing_idle")
|
|
543
|
+
return "idle_no_queued_work";
|
|
544
|
+
// A finished story becomes eligible again only once work is queued, which
|
|
545
|
+
// can be long after the stop. Open the failure window when that work
|
|
546
|
+
// appeared, not at the stop's own due time, or every normal exit from
|
|
547
|
+
// idle_no_queued_work reads as a liveness failure the moment it happens.
|
|
548
|
+
if (input.storyComplete) {
|
|
549
|
+
if (input.queuedWorkAt === undefined)
|
|
550
|
+
return "waiting";
|
|
551
|
+
const anchor = Math.max(dueAt, input.queuedWorkAt);
|
|
552
|
+
return nowMs > anchor + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS ? "not_restarted" : "waiting";
|
|
553
|
+
}
|
|
554
|
+
}
|
|
481
555
|
if (nowMs > dueAt + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS)
|
|
482
556
|
return "not_restarted";
|
|
483
557
|
return "waiting";
|
|
@@ -546,6 +620,10 @@ export function buildKeepAliveAudit(input) {
|
|
|
546
620
|
try {
|
|
547
621
|
const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
|
|
548
622
|
const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
|
|
623
|
+
// Read at audit time, not stop time: for a stop with no restart the
|
|
624
|
+
// question is whether anything is going to resume the lane now.
|
|
625
|
+
const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(input.hqRoot, lane);
|
|
626
|
+
const queuedWorkAt = hasQueuedWork ? keepAliveQueuedWorkAt(input.hqRoot, lane) : undefined;
|
|
549
627
|
const laneWorkerResumeEvents = readLaneWorkerResumeEvents(input.hqRoot, lane.lane_id);
|
|
550
628
|
const terminalClassifications = readTerminalClassifications(input.hqRoot, lane.lane_id);
|
|
551
629
|
for (const stop of stops) {
|
|
@@ -572,7 +650,8 @@ export function buildKeepAliveAudit(input) {
|
|
|
572
650
|
blocked_outcome: blockedOutcome.status,
|
|
573
651
|
...(blockedOutcome.reason === undefined ? {} : { blocked_outcome_reason: blockedOutcome.reason }),
|
|
574
652
|
} : {}),
|
|
575
|
-
verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
|
|
653
|
+
verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, decision: stop.decision, storyComplete: stop.storyComplete, hasQueuedWork, ...(queuedWorkAt === undefined ? {} : { queuedWorkAt }), lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
|
|
654
|
+
...(stop.decision === "done" ? { queued_work_at_audit_time: hasQueuedWork } : {}),
|
|
576
655
|
});
|
|
577
656
|
}
|
|
578
657
|
}
|
|
@@ -611,6 +690,8 @@ export function buildKeepAliveAudit(input) {
|
|
|
611
690
|
not_restarted: byVerdict.not_restarted,
|
|
612
691
|
stranded: byVerdict.stranded,
|
|
613
692
|
restarted_untimed: byVerdict.restarted_untimed,
|
|
693
|
+
finished_story: byVerdict.finished_story,
|
|
694
|
+
idle_no_queued_work: byVerdict.idle_no_queued_work,
|
|
614
695
|
late_restarts_after_window: lateRestartsAfterWindow,
|
|
615
696
|
late_over_10m: lateOverTenMinutes,
|
|
616
697
|
early_wake_by_others: byVerdict.early_wake,
|
|
@@ -642,6 +723,7 @@ export function formatKeepAliveAuditText(result) {
|
|
|
642
723
|
"",
|
|
643
724
|
`Since ${result.since}; ${counts}`,
|
|
644
725
|
`not_restarted=${result.summary.not_restarted}, stranded=${result.summary.stranded}, late_restarts_after_window=${result.summary.late_restarts_after_window}, late_over_10m=${result.summary.late_over_10m}, early_wake_by_others=${result.summary.early_wake_by_others}, self_wake=${result.summary.self_wake}`,
|
|
726
|
+
`finished_story=${result.summary.finished_story}, idle_no_queued_work=${result.summary.idle_no_queued_work} (neither counts against the keep-alive KPI)`,
|
|
645
727
|
`Stopped and not restarted on their own: ${result.summary.stopped_and_not_restarted_on_their_own}`,
|
|
646
728
|
...result.rows.filter((row) => row.remediation).map((row) => `Stranded ${row.lane}: ${row.remediation}`),
|
|
647
729
|
...result.companyless_lead_lanes.map((lane) => `Company-less lead ${lane.lane} (${lane.project}): ${lane.reason}; repair with hq lanes repair-company ${lane.lane} --company <slug>`),
|
|
@@ -93,6 +93,8 @@ export function buildOutpostLaneListCommand(input = {}) {
|
|
|
93
93
|
? [input.hierarchy]
|
|
94
94
|
: input.hierarchy ?? [])
|
|
95
95
|
args.push("--hierarchy", laneId);
|
|
96
|
+
if (input.senior !== undefined)
|
|
97
|
+
args.push("--senior", input.senior);
|
|
96
98
|
if (input.snapshot)
|
|
97
99
|
args.push("--snapshot");
|
|
98
100
|
return withRemoteHqDir(joinCommandParts(args));
|
|
@@ -23,7 +23,7 @@ import { isInstallableHqRoot, readDaemonConfig, registerTrustedHqRoot, verifyHqR
|
|
|
23
23
|
import { defaultPidLockDeps, pidLockStatus, readPidLock } from "./pid-lock.js";
|
|
24
24
|
import { retireLegacyListen } from "./legacy-listen.js";
|
|
25
25
|
import { readDaemonState } from "./state.js";
|
|
26
|
-
import { activateLaunchd, defaultRunCommand, launchdIsLoaded, launchdServiceTarget, launchdPlistProgramArguments, launchdProgramArgumentsDiffer, parseLaunchctlPrintArguments, } from "../../../service-manager/launchd.js";
|
|
26
|
+
import { activateLaunchd, defaultRunCommand, discoverLaunchdIdentity, installLaunchdPlist, launchdProgramArguments, launchdIsLoaded, launchdServiceTarget, launchdPlistProgramArguments, parseLaunchdCommand, launchdProgramArgumentsDiffer, parseLaunchctlPrintArguments, } from "../../../service-manager/launchd.js";
|
|
27
27
|
import { activateSystemd, systemdIsActive } from "../../../service-manager/systemd.js";
|
|
28
28
|
import { nodeBinaryProblem, preferStableNodeBinary } from "../../../node-binary.js";
|
|
29
29
|
/**
|
|
@@ -428,9 +428,7 @@ export function installedDaemonNodeBinary(paths, platform, readUnit = (file) =>
|
|
|
428
428
|
const text = readUnit(paths.plistPath);
|
|
429
429
|
if (!text)
|
|
430
430
|
return null;
|
|
431
|
-
|
|
432
|
-
const first = block?.[1]?.match(/<string>([^<]*)<\/string>/);
|
|
433
|
-
return first?.[1] ?? null;
|
|
431
|
+
return parseLaunchdCommand(launchdPlistProgramArguments(text))?.node ?? null;
|
|
434
432
|
}
|
|
435
433
|
if (platform === "linux") {
|
|
436
434
|
const text = readUnit(paths.systemdUnitPath);
|
|
@@ -544,19 +542,21 @@ function safeUsername() {
|
|
|
544
542
|
}
|
|
545
543
|
}
|
|
546
544
|
export function renderLaunchdPlist(paths) {
|
|
545
|
+
const identity = discoverLaunchdIdentity(paths.home);
|
|
546
|
+
const args = launchdProgramArguments({
|
|
547
|
+
nodeBinary: paths.nodeBinary,
|
|
548
|
+
hqBinary: paths.hqBinary,
|
|
549
|
+
launchdLauncherBinary: identity.launcherBinary,
|
|
550
|
+
}, ["mesh", "daemon", "run"]);
|
|
547
551
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
548
552
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
549
553
|
<plist version="1.0">
|
|
550
554
|
<dict>
|
|
551
555
|
<key>Label</key>
|
|
552
556
|
<string>${LAUNCHD_LABEL}</string>
|
|
553
|
-
<key>ProgramArguments</key>
|
|
557
|
+
${identity.associatedBundleIdentifier ? ` <key>AssociatedBundleIdentifiers</key>\n <array>\n <string>${escapeXml(identity.associatedBundleIdentifier)}</string>\n </array>\n` : ""} <key>ProgramArguments</key>
|
|
554
558
|
<array>
|
|
555
|
-
<string>${escapeXml(
|
|
556
|
-
<string>${escapeXml(paths.hqBinary)}</string>
|
|
557
|
-
<string>mesh</string>
|
|
558
|
-
<string>daemon</string>
|
|
559
|
-
<string>run</string>
|
|
559
|
+
${args.map((arg) => ` <string>${escapeXml(arg)}</string>`).join("\n")}
|
|
560
560
|
</array>
|
|
561
561
|
<key>WorkingDirectory</key>
|
|
562
562
|
<string>${escapeXml(paths.daemonDir)}</string>
|
|
@@ -920,11 +920,24 @@ export function installDaemonService(opts) {
|
|
|
920
920
|
if (platform === "darwin") {
|
|
921
921
|
const rendered = renderLaunchdPlist(paths);
|
|
922
922
|
mkdirSync(paths.launchAgentsDir, { recursive: true });
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
923
|
+
installLaunchdPlist(paths.plistPath, LAUNCHD_LABEL, rendered, {
|
|
924
|
+
files: {
|
|
925
|
+
existsSync: fs.existsSync,
|
|
926
|
+
readFileSync: (file) => fs.readFileSync(file, "utf8"),
|
|
927
|
+
writeFileSync: fs.writeFileSync,
|
|
928
|
+
chmodSync: fs.chmodSync,
|
|
929
|
+
renameSync: fs.renameSync,
|
|
930
|
+
unlinkSync: fs.unlinkSync,
|
|
931
|
+
},
|
|
932
|
+
activate: Boolean(opts.run || opts.activate),
|
|
933
|
+
uid: opts.uid,
|
|
934
|
+
rollbackOnReloadError: true,
|
|
935
|
+
reload: () => {
|
|
936
|
+
if (opts.run)
|
|
937
|
+
activateLaunchd(paths.plistPath, opts.run, opts.uid, { sleep: opts.sleep });
|
|
938
|
+
else if (opts.activate)
|
|
939
|
+
opts.activate(platform, paths.plistPath);
|
|
940
|
+
},
|
|
928
941
|
});
|
|
929
942
|
return {
|
|
930
943
|
platform,
|
|
@@ -15,9 +15,12 @@ export interface ServiceManagerDeps {
|
|
|
15
15
|
platform?: ServicePlatform;
|
|
16
16
|
run?: RunCommand;
|
|
17
17
|
writeFileSync?: typeof fs.writeFileSync;
|
|
18
|
+
chmodSync?: typeof fs.chmodSync;
|
|
19
|
+
renameSync?: typeof fs.renameSync;
|
|
18
20
|
mkdirSync?: typeof fs.mkdirSync;
|
|
19
21
|
unlinkSync?: typeof fs.unlinkSync;
|
|
20
22
|
existsSync?: typeof fs.existsSync;
|
|
23
|
+
readFileSync?: typeof fs.readFileSync;
|
|
21
24
|
/** When false, render + write units but do not (de)activate them. */
|
|
22
25
|
activate?: boolean;
|
|
23
26
|
}
|