@indigoai-us/hq-cli 5.284.0 → 5.286.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/command-catalog.generated.d.ts +21 -1
- package/dist/command-catalog.generated.js +27 -1
- package/dist/commands/agents.d.ts +11 -1
- package/dist/commands/agents.js +39 -6
- package/dist/commands/lanes.js +17 -9
- package/dist/lib/daemon/paths.d.ts +2 -0
- package/dist/lib/daemon/paths.js +1 -0
- package/dist/lib/daemon/protocol-tail.d.ts +33 -0
- package/dist/lib/daemon/protocol-tail.js +131 -0
- package/dist/lib/daemon/run.js +79 -5
- package/dist/lib/daemon/succession.d.ts +2 -0
- package/dist/lib/daemon/succession.js +1 -0
- package/dist/lib/doctor/report.js +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,8 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.286.0] — 2026-09-29
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- On Linux the daemon's sync runner writes its protocol output to a named
|
|
10
|
+
pipe, `~/.hq/daemon/protocol/sync.fifo`, instead of an anonymous pipe. With
|
|
11
|
+
`daemon.supervisor-handoff` on, an outpost daemon restart now hands sync to
|
|
12
|
+
the next daemon like the other services instead of stopping and restarting
|
|
13
|
+
it, and output the old daemon had not read reaches the new one. Other
|
|
14
|
+
platforms keep the anonymous pipe.
|
|
15
|
+
|
|
16
|
+
## [5.285.0] — 2026-09-29
|
|
17
|
+
|
|
18
|
+
- Route eligible Linux CI jobs onto ARM runners while retaining existing test and platform gates.
|
|
19
|
+
|
|
5
20
|
## [5.284.0] — 2026-09-29
|
|
6
21
|
- The existing Work Mesh MQTT subscriber can signal agent inbox wakes through a private local marker without persisting MQTT message contents.
|
|
22
|
+
- Add `hq agents jobs resume <uid> <jobId>` to re-enable paused agent job schedules.
|
|
7
23
|
|
|
8
24
|
### Fixed
|
|
9
25
|
|
|
@@ -11,6 +27,14 @@
|
|
|
11
27
|
capacity probes, or mirror snapshots to other Outposts. Two Outposts that
|
|
12
28
|
each held a lane placed on the other ran `hq lanes list --json` on each other
|
|
13
29
|
without end, which filled both boxes with SSM commands.
|
|
30
|
+
- `hq doctor` labels the detected agent runtime value as `Agent platform` in human output. The JSON platform field and schema are unchanged.
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
|
|
34
|
+
- Refresh remote lane mirrors on hosts with an Outpost identity, while excluding
|
|
35
|
+
mirrors owned by that host. Resolve company attribution before checking the
|
|
36
|
+
mirror gate for refreshes and remote logs; unresolved lanes stay local.
|
|
37
|
+
Remote-read invocations skip refresh to prevent reciprocal Outpost fan-out.
|
|
14
38
|
|
|
15
39
|
## [5.283.0] — 2026-09-29
|
|
16
40
|
|
|
@@ -4257,7 +4257,7 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
4257
4257
|
readonly subcommands: readonly [];
|
|
4258
4258
|
}, {
|
|
4259
4259
|
readonly name: "jobs";
|
|
4260
|
-
readonly description: "List, pause, or cancel an agent's scheduled jobs";
|
|
4260
|
+
readonly description: "List, pause, resume, or cancel an agent's scheduled jobs";
|
|
4261
4261
|
readonly aliases: readonly [];
|
|
4262
4262
|
readonly hidden: false;
|
|
4263
4263
|
readonly usage: "[options] [command]";
|
|
@@ -4302,6 +4302,26 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
4302
4302
|
readonly description: "Company slug (to resolve an agent by name/slug)";
|
|
4303
4303
|
}];
|
|
4304
4304
|
readonly subcommands: readonly [];
|
|
4305
|
+
}, {
|
|
4306
|
+
readonly name: "resume";
|
|
4307
|
+
readonly description: "Resume a paused job's schedule";
|
|
4308
|
+
readonly aliases: readonly [];
|
|
4309
|
+
readonly hidden: false;
|
|
4310
|
+
readonly usage: "[options] <agentUid> <jobId>";
|
|
4311
|
+
readonly arguments: readonly [{
|
|
4312
|
+
readonly name: "agentUid";
|
|
4313
|
+
readonly required: true;
|
|
4314
|
+
readonly variadic: false;
|
|
4315
|
+
}, {
|
|
4316
|
+
readonly name: "jobId";
|
|
4317
|
+
readonly required: true;
|
|
4318
|
+
readonly variadic: false;
|
|
4319
|
+
}];
|
|
4320
|
+
readonly options: readonly [{
|
|
4321
|
+
readonly flags: "--company <slug>";
|
|
4322
|
+
readonly description: "Company slug (to resolve an agent by name/slug)";
|
|
4323
|
+
}];
|
|
4324
|
+
readonly subcommands: readonly [];
|
|
4305
4325
|
}, {
|
|
4306
4326
|
readonly name: "cancel";
|
|
4307
4327
|
readonly description: "Cancel a job and delete its schedule";
|
|
@@ -5512,7 +5512,7 @@ export const COMMAND_CATALOG = [
|
|
|
5512
5512
|
},
|
|
5513
5513
|
{
|
|
5514
5514
|
"name": "jobs",
|
|
5515
|
-
"description": "List, pause, or cancel an agent's scheduled jobs",
|
|
5515
|
+
"description": "List, pause, resume, or cancel an agent's scheduled jobs",
|
|
5516
5516
|
"aliases": [],
|
|
5517
5517
|
"hidden": false,
|
|
5518
5518
|
"usage": "[options] [command]",
|
|
@@ -5570,6 +5570,32 @@ export const COMMAND_CATALOG = [
|
|
|
5570
5570
|
],
|
|
5571
5571
|
"subcommands": []
|
|
5572
5572
|
},
|
|
5573
|
+
{
|
|
5574
|
+
"name": "resume",
|
|
5575
|
+
"description": "Resume a paused job's schedule",
|
|
5576
|
+
"aliases": [],
|
|
5577
|
+
"hidden": false,
|
|
5578
|
+
"usage": "[options] <agentUid> <jobId>",
|
|
5579
|
+
"arguments": [
|
|
5580
|
+
{
|
|
5581
|
+
"name": "agentUid",
|
|
5582
|
+
"required": true,
|
|
5583
|
+
"variadic": false
|
|
5584
|
+
},
|
|
5585
|
+
{
|
|
5586
|
+
"name": "jobId",
|
|
5587
|
+
"required": true,
|
|
5588
|
+
"variadic": false
|
|
5589
|
+
}
|
|
5590
|
+
],
|
|
5591
|
+
"options": [
|
|
5592
|
+
{
|
|
5593
|
+
"flags": "--company <slug>",
|
|
5594
|
+
"description": "Company slug (to resolve an agent by name/slug)"
|
|
5595
|
+
}
|
|
5596
|
+
],
|
|
5597
|
+
"subcommands": []
|
|
5598
|
+
},
|
|
5573
5599
|
{
|
|
5574
5600
|
"name": "cancel",
|
|
5575
5601
|
"description": "Cancel a job and delete its schedule",
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
* hq agents revoke <uid> --yes — revoke an external agent (--remove deletes it)
|
|
20
20
|
* hq agents jobs list <uid> — off-box job roster (schedule + rate)
|
|
21
21
|
* hq agents jobs pause <uid> <jobId> — flip schedule State=DISABLED
|
|
22
|
+
* hq agents jobs resume <uid> <jobId> — flip schedule State=ENABLED
|
|
22
23
|
* hq agents jobs cancel <uid> <jobId> — delete schedule + drop the job
|
|
23
24
|
*
|
|
24
25
|
* Agents are company-scoped. `--company <slug>` may sit on the group
|
|
@@ -320,12 +321,20 @@ export interface PauseJobResult {
|
|
|
320
321
|
changed: boolean;
|
|
321
322
|
previousState?: string;
|
|
322
323
|
}
|
|
324
|
+
export interface ResumeJobResult {
|
|
325
|
+
ok: true;
|
|
326
|
+
jobId: string;
|
|
327
|
+
scheduleState: "ENABLED" | string;
|
|
328
|
+
changed: boolean;
|
|
329
|
+
previousState?: string;
|
|
330
|
+
}
|
|
323
331
|
export interface CancelJobResult {
|
|
324
332
|
ok: true;
|
|
325
333
|
jobId: string;
|
|
326
334
|
}
|
|
327
335
|
export declare function listAgentJobs(token: string, agentUid: string): Promise<AgentJobView[]>;
|
|
328
336
|
export declare function pauseAgentJob(token: string, agentUid: string, jobId: string): Promise<PauseJobResult>;
|
|
337
|
+
export declare function resumeAgentJob(token: string, agentUid: string, jobId: string): Promise<ResumeJobResult>;
|
|
329
338
|
export declare function cancelAgentJob(token: string, agentUid: string, jobId: string): Promise<CancelJobResult>;
|
|
330
339
|
/**
|
|
331
340
|
* Map a jobs-control HTTP error to a single operator-facing line. Pure so the
|
|
@@ -336,10 +345,11 @@ export declare function cancelAgentJob(token: string, agentUid: string, jobId: s
|
|
|
336
345
|
* the agent is cross-company) must print a clear message and exit non-zero,
|
|
337
346
|
* never throw a stack trace.
|
|
338
347
|
*/
|
|
339
|
-
export declare function formatJobsHttpError(err: AgentsHttpError, jobId?: string): string;
|
|
348
|
+
export declare function formatJobsHttpError(err: AgentsHttpError, jobId?: string, action?: "resume"): string;
|
|
340
349
|
/** Stable padEnd table: jobId, scheduleState, rate, lastRunOutcome, status. */
|
|
341
350
|
export declare function formatJobsTable(jobs: AgentJobView[]): string;
|
|
342
351
|
export declare function formatPauseResult(result: PauseJobResult): string;
|
|
352
|
+
export declare function formatResumeResult(result: ResumeJobResult): string;
|
|
343
353
|
/** Human-readable "hot-applied to the running box" vs "saved for next launch". */
|
|
344
354
|
export declare function appliedHint(applied: boolean): string;
|
|
345
355
|
/** Stable human table shared by the all-provider and provider-filtered forms. */
|
package/dist/commands/agents.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
* hq agents revoke <uid> --yes — revoke an external agent (--remove deletes it)
|
|
20
20
|
* hq agents jobs list <uid> — off-box job roster (schedule + rate)
|
|
21
21
|
* hq agents jobs pause <uid> <jobId> — flip schedule State=DISABLED
|
|
22
|
+
* hq agents jobs resume <uid> <jobId> — flip schedule State=ENABLED
|
|
22
23
|
* hq agents jobs cancel <uid> <jobId> — delete schedule + drop the job
|
|
23
24
|
*
|
|
24
25
|
* Agents are company-scoped. `--company <slug>` may sit on the group
|
|
@@ -790,6 +791,13 @@ export async function pauseAgentJob(token, agentUid, jobId) {
|
|
|
790
791
|
method: "POST",
|
|
791
792
|
});
|
|
792
793
|
}
|
|
794
|
+
export async function resumeAgentJob(token, agentUid, jobId) {
|
|
795
|
+
return agentsRequest({
|
|
796
|
+
token,
|
|
797
|
+
path: `/v1/agents/${encodeURIComponent(agentUid)}/jobs/${encodeURIComponent(jobId)}/resume`,
|
|
798
|
+
method: "POST",
|
|
799
|
+
});
|
|
800
|
+
}
|
|
793
801
|
export async function cancelAgentJob(token, agentUid, jobId) {
|
|
794
802
|
return agentsRequest({
|
|
795
803
|
token,
|
|
@@ -806,7 +814,7 @@ export async function cancelAgentJob(token, agentUid, jobId) {
|
|
|
806
814
|
* the agent is cross-company) must print a clear message and exit non-zero,
|
|
807
815
|
* never throw a stack trace.
|
|
808
816
|
*/
|
|
809
|
-
export function formatJobsHttpError(err, jobId) {
|
|
817
|
+
export function formatJobsHttpError(err, jobId, action) {
|
|
810
818
|
if (err.status === 401)
|
|
811
819
|
return "Not authenticated — run `hq login`.";
|
|
812
820
|
// A 403 CROSS_AGENT_JOB is hq-pro's self-only jobs guard, not a role problem:
|
|
@@ -816,12 +824,17 @@ export function formatJobsHttpError(err, jobId) {
|
|
|
816
824
|
// closed locally before the request, see crossAgentJobsRefusal).
|
|
817
825
|
if (err.status === 403 && err.code === "CROSS_AGENT_JOB") {
|
|
818
826
|
return ("Jobs are scoped to the calling agent's own identity — an agent may only " +
|
|
819
|
-
"list, pause, or cancel its OWN jobs (use the on-box `hq-agent-jobs` tool). " +
|
|
827
|
+
"list, pause, resume, or cancel its OWN jobs (use the on-box `hq-agent-jobs` tool). " +
|
|
820
828
|
"Managing another agent's jobs needs a person session with owner/admin on " +
|
|
821
829
|
"the company.");
|
|
822
830
|
}
|
|
823
831
|
if (err.status === 403)
|
|
824
832
|
return "You need owner/admin on this company.";
|
|
833
|
+
if (action === "resume" &&
|
|
834
|
+
err.status === 404 &&
|
|
835
|
+
(err.code === "JOB_NOT_FOUND" || err.code === "SCHEDULE_NOT_FOUND")) {
|
|
836
|
+
return "job not found or its schedule was deleted; it cannot be resumed";
|
|
837
|
+
}
|
|
825
838
|
if (err.status === 404 && err.code === "JOB_NOT_FOUND") {
|
|
826
839
|
const id = jobId ?? extractQuotedJobId(err.message) ?? "unknown";
|
|
827
840
|
return `Job '${id}' not found.`;
|
|
@@ -860,6 +873,12 @@ export function formatPauseResult(result) {
|
|
|
860
873
|
const from = result.previousState ?? "ENABLED";
|
|
861
874
|
return `Paused ${result.jobId} (${from} → ${result.scheduleState})`;
|
|
862
875
|
}
|
|
876
|
+
export function formatResumeResult(result) {
|
|
877
|
+
if (!result.changed)
|
|
878
|
+
return "Already active";
|
|
879
|
+
const from = result.previousState ?? "DISABLED";
|
|
880
|
+
return `Resumed ${result.jobId} (${from} → ${result.scheduleState})`;
|
|
881
|
+
}
|
|
863
882
|
/** Human-readable "hot-applied to the running box" vs "saved for next launch". */
|
|
864
883
|
export function appliedHint(applied) {
|
|
865
884
|
return applied
|
|
@@ -1119,9 +1138,9 @@ function fail(err) {
|
|
|
1119
1138
|
}
|
|
1120
1139
|
process.exit(1);
|
|
1121
1140
|
}
|
|
1122
|
-
function failJobs(err, jobId) {
|
|
1141
|
+
function failJobs(err, jobId, action) {
|
|
1123
1142
|
if (err instanceof AgentsHttpError) {
|
|
1124
|
-
console.error(chalk.red(formatJobsHttpError(err, jobId)));
|
|
1143
|
+
console.error(chalk.red(formatJobsHttpError(err, jobId, action)));
|
|
1125
1144
|
process.exit(1);
|
|
1126
1145
|
}
|
|
1127
1146
|
fail(err);
|
|
@@ -1922,11 +1941,11 @@ export function registerAgentsCommand(program) {
|
|
|
1922
1941
|
}
|
|
1923
1942
|
});
|
|
1924
1943
|
// Off-box job control (US-003). Nested group so `hq agents jobs --help`
|
|
1925
|
-
// lists list|pause|cancel. Same vaultApiFetch + person JWT as the rest of
|
|
1944
|
+
// lists list|pause|resume|cancel. Same vaultApiFetch + person JWT as the rest of
|
|
1926
1945
|
// this file; keyed HQ_API_KEY rewrites /v1/agents → /v1/keys/agents.
|
|
1927
1946
|
const jobs = agents
|
|
1928
1947
|
.command("jobs")
|
|
1929
|
-
.description("List, pause, or cancel an agent's scheduled jobs");
|
|
1948
|
+
.description("List, pause, resume, or cancel an agent's scheduled jobs");
|
|
1930
1949
|
jobs
|
|
1931
1950
|
.command("list <agentUid>")
|
|
1932
1951
|
.description("List an agent's scheduled jobs")
|
|
@@ -1964,6 +1983,20 @@ export function registerAgentsCommand(program) {
|
|
|
1964
1983
|
failJobs(err, jobId);
|
|
1965
1984
|
}
|
|
1966
1985
|
});
|
|
1986
|
+
jobs
|
|
1987
|
+
.command("resume <agentUid> <jobId>")
|
|
1988
|
+
.description("Resume a paused job's schedule")
|
|
1989
|
+
.option("--company <slug>", "Company slug (to resolve an agent by name/slug)")
|
|
1990
|
+
.action(async function (agentUid, jobId) {
|
|
1991
|
+
const { token, targetUid } = await resolveJobsTargetOrExit(agentUid, companyOf(this));
|
|
1992
|
+
try {
|
|
1993
|
+
const result = await resumeAgentJob(token, targetUid, jobId);
|
|
1994
|
+
console.log(chalk.green(formatResumeResult(result)));
|
|
1995
|
+
}
|
|
1996
|
+
catch (err) {
|
|
1997
|
+
failJobs(err, jobId, "resume");
|
|
1998
|
+
}
|
|
1999
|
+
});
|
|
1967
2000
|
jobs
|
|
1968
2001
|
.command("cancel <agentUid> <jobId>")
|
|
1969
2002
|
.description("Cancel a job and delete its schedule")
|
package/dist/commands/lanes.js
CHANGED
|
@@ -1238,17 +1238,17 @@ async function readOutpostLaneRecords(ctx, outpostId, requestedQuery = {}, dispa
|
|
|
1238
1238
|
}
|
|
1239
1239
|
return invocationMemo.getOrCreate(outpostId, query, () => readOutpostSnapshotWithCache(ctx, outpostLaneListSnapshotKey(outpostId, query), isLaneRecordArray, () => readOutpostLaneRecordsUncached(ctx, outpostId, query, dispatchOptions)));
|
|
1240
1240
|
}
|
|
1241
|
-
/** Refresh only
|
|
1241
|
+
/** Refresh only this host's non-terminal Outpost mirrors in one bounded owner query per Outpost. */
|
|
1242
1242
|
async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
1243
1243
|
const result = {
|
|
1244
1244
|
updated: new Map(),
|
|
1245
1245
|
unverifiedRemote: new Map(),
|
|
1246
1246
|
};
|
|
1247
1247
|
const host = reconcileHostFromCtx(ctx);
|
|
1248
|
-
if (
|
|
1248
|
+
if (ctx.env.HQ_LANES_REMOTE_READ === "1" || parentLaneIds?.size === 0)
|
|
1249
1249
|
return result;
|
|
1250
1250
|
let primaryIdentity;
|
|
1251
|
-
const
|
|
1251
|
+
const originMatchesThisHost = (lane) => {
|
|
1252
1252
|
if (lane.origin_host_source !== "imds") {
|
|
1253
1253
|
return lane.origin_host === host.hostname;
|
|
1254
1254
|
}
|
|
@@ -1261,7 +1261,8 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1261
1261
|
};
|
|
1262
1262
|
const candidates = records
|
|
1263
1263
|
.filter((lane) => lane.host.kind === "outpost" &&
|
|
1264
|
-
|
|
1264
|
+
(host.outpostId === null || lane.host.id !== host.outpostId) &&
|
|
1265
|
+
originMatchesThisHost(lane) &&
|
|
1265
1266
|
(parentLaneIds === undefined || (lane.senior.kind === "lane" &&
|
|
1266
1267
|
lane.senior.id !== undefined &&
|
|
1267
1268
|
parentLaneIds.has(lane.senior.id))) &&
|
|
@@ -1272,11 +1273,14 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1272
1273
|
const enabledByCompany = new Map();
|
|
1273
1274
|
const enabledCandidates = [];
|
|
1274
1275
|
for (const lane of candidates) {
|
|
1275
|
-
const
|
|
1276
|
+
const company = resolveLaneCompany(hqRoot, lane).company;
|
|
1277
|
+
if (company === null)
|
|
1278
|
+
continue;
|
|
1279
|
+
const companyKey = JSON.stringify([company.uid, company.slug]);
|
|
1276
1280
|
let enabled = enabledByCompany.get(companyKey);
|
|
1277
1281
|
if (enabled === undefined) {
|
|
1278
1282
|
enabled = await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
|
|
1279
|
-
company
|
|
1283
|
+
company,
|
|
1280
1284
|
env: ctx.env,
|
|
1281
1285
|
});
|
|
1282
1286
|
enabledByCompany.set(companyKey, enabled);
|
|
@@ -1346,7 +1350,7 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1346
1350
|
: undefined;
|
|
1347
1351
|
const before = readLane(hqRoot, mirror.lane_id);
|
|
1348
1352
|
if (before.host.kind !== "outpost" || before.host.id !== outpostId ||
|
|
1349
|
-
!
|
|
1353
|
+
!originMatchesThisHost(before) || isTerminalLaneState(before.state)) {
|
|
1350
1354
|
result.updated.set(mirror.lane_id, before);
|
|
1351
1355
|
continue;
|
|
1352
1356
|
}
|
|
@@ -1365,7 +1369,7 @@ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
|
1365
1369
|
}
|
|
1366
1370
|
const updated = retryUpdateLane(hqRoot, mirror.lane_id, "reconcile", (current) => {
|
|
1367
1371
|
if (current.host.kind !== "outpost" || current.host.id !== outpostId ||
|
|
1368
|
-
!
|
|
1372
|
+
!originMatchesThisHost(current) || isTerminalLaneState(current.state))
|
|
1369
1373
|
return current;
|
|
1370
1374
|
return {
|
|
1371
1375
|
...current,
|
|
@@ -6805,14 +6809,18 @@ const CANONICAL_LOG_STREAMS = [
|
|
|
6805
6809
|
async function runLogs(ctx, laneId, opts) {
|
|
6806
6810
|
const hqRoot = ctx.hqRoot();
|
|
6807
6811
|
const lane = readLane(hqRoot, laneId);
|
|
6812
|
+
const remoteMirrorCompany = lane.host.kind === "outpost"
|
|
6813
|
+
? resolveLaneCompany(hqRoot, lane).company
|
|
6814
|
+
: null;
|
|
6808
6815
|
const tail = parseOptionalInt(opts.tail, "--tail");
|
|
6809
6816
|
if (tail === 0) {
|
|
6810
6817
|
throw new LanesError("invalid_integer", "--tail must be a positive integer", { flag: "--tail", value: opts.tail });
|
|
6811
6818
|
}
|
|
6812
6819
|
if (lane.host.kind === "outpost" &&
|
|
6813
6820
|
!laneRunsHere(lane, reconcileHostFromCtx(ctx)) &&
|
|
6821
|
+
remoteMirrorCompany !== null &&
|
|
6814
6822
|
await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
|
|
6815
|
-
company:
|
|
6823
|
+
company: remoteMirrorCompany,
|
|
6816
6824
|
env: ctx.env,
|
|
6817
6825
|
})) {
|
|
6818
6826
|
const outpostId = lane.host.id;
|
|
@@ -12,6 +12,8 @@ export interface DaemonPaths {
|
|
|
12
12
|
requestsDir: string;
|
|
13
13
|
supervisorLogPath: string;
|
|
14
14
|
unitLogPath: (unitId: string) => string;
|
|
15
|
+
/** The FIFO a unit that speaks the sync protocol writes its stdout to (see protocol-tail.ts); kept out of logs/ so nothing reading logs drains it. */
|
|
16
|
+
unitProtocolPath: (unitId: string) => string;
|
|
15
17
|
}
|
|
16
18
|
export declare function daemonPaths(home?: string, env?: NodeJS.ProcessEnv): DaemonPaths;
|
|
17
19
|
/** Create `dir` (and any missing parents) readable only by its owner, and tighten it if it already exists. */
|
package/dist/lib/daemon/paths.js
CHANGED
|
@@ -18,6 +18,7 @@ export function daemonPaths(home = os.homedir(), env = process.env) {
|
|
|
18
18
|
requestsDir: path.join(root, "requests"),
|
|
19
19
|
supervisorLogPath: path.join(logsDir, "daemon.log"),
|
|
20
20
|
unitLogPath: (unitId) => path.join(logsDir, `${unitId.replace(/[^A-Za-z0-9_.-]/g, "-")}.log`),
|
|
21
|
+
unitProtocolPath: (unitId) => path.join(root, "protocol", `${unitId.replace(/[^A-Za-z0-9_.-]/g, "-")}.fifo`),
|
|
21
22
|
};
|
|
22
23
|
}
|
|
23
24
|
/** Create `dir` (and any missing parents) readable only by its owner, and tighten it if it already exists. */
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A service's protocol output read from a named pipe instead of an anonymous
|
|
3
|
+
* one.
|
|
4
|
+
*
|
|
5
|
+
* An anonymous pipe loses its reader when the daemon exits, so a service
|
|
6
|
+
* writing to one cannot be handed to the next daemon. Here the service's
|
|
7
|
+
* stdout is a FIFO it holds open for reading and writing, so it always has a
|
|
8
|
+
* reader and never gets EPIPE. Each daemon polls the FIFO without blocking and
|
|
9
|
+
* feeds what it reads to the supervisor as the child's stdout. On a handoff
|
|
10
|
+
* the daemon stops reading without draining, so output it has not read stays
|
|
11
|
+
* in the kernel buffer for the next daemon, and nothing accumulates on disk.
|
|
12
|
+
*/
|
|
13
|
+
export interface ProtocolTail {
|
|
14
|
+
/** "data" carries each chunk read; "end" fires once, after close(). */
|
|
15
|
+
on(event: "data" | "end", handler: (chunk?: Buffer | string) => void): ProtocolTail;
|
|
16
|
+
/** The service exited: read what is left, emit "end" and stop polling. */
|
|
17
|
+
close(): void;
|
|
18
|
+
/** Handing over: stop polling without reading, leaving the rest for the next daemon. */
|
|
19
|
+
detach(): void;
|
|
20
|
+
}
|
|
21
|
+
/** Make `file` a FIFO, replacing anything else found there. */
|
|
22
|
+
export declare function ensureProtocolFifo(file: string): void;
|
|
23
|
+
/**
|
|
24
|
+
* The service's stdout. Opened for reading and writing, so the FIFO keeps a
|
|
25
|
+
* reader while no daemon has it open and the service's writes never fail.
|
|
26
|
+
*/
|
|
27
|
+
export declare function openProtocolWriter(file: string): number;
|
|
28
|
+
export declare function createProtocolTail(file: string, opts?: {
|
|
29
|
+
pollMs?: number;
|
|
30
|
+
setInterval?: (fn: () => void, ms: number) => unknown;
|
|
31
|
+
clearInterval?: (handle: unknown) => void;
|
|
32
|
+
}): ProtocolTail;
|
|
33
|
+
//# sourceMappingURL=protocol-tail.d.ts.map
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A service's protocol output read from a named pipe instead of an anonymous
|
|
3
|
+
* one.
|
|
4
|
+
*
|
|
5
|
+
* An anonymous pipe loses its reader when the daemon exits, so a service
|
|
6
|
+
* writing to one cannot be handed to the next daemon. Here the service's
|
|
7
|
+
* stdout is a FIFO it holds open for reading and writing, so it always has a
|
|
8
|
+
* reader and never gets EPIPE. Each daemon polls the FIFO without blocking and
|
|
9
|
+
* feeds what it reads to the supervisor as the child's stdout. On a handoff
|
|
10
|
+
* the daemon stops reading without draining, so output it has not read stays
|
|
11
|
+
* in the kernel buffer for the next daemon, and nothing accumulates on disk.
|
|
12
|
+
*/
|
|
13
|
+
import { spawnSync } from "node:child_process";
|
|
14
|
+
import { EventEmitter } from "node:events";
|
|
15
|
+
import * as fs from "node:fs";
|
|
16
|
+
import * as path from "node:path";
|
|
17
|
+
import { ensurePrivateDir } from "./paths.js";
|
|
18
|
+
const POLL_MS = 1_000;
|
|
19
|
+
const READ_CHUNK_BYTES = 64 * 1024;
|
|
20
|
+
/** Make `file` a FIFO, replacing anything else found there. */
|
|
21
|
+
export function ensureProtocolFifo(file) {
|
|
22
|
+
ensurePrivateDir(path.dirname(file));
|
|
23
|
+
try {
|
|
24
|
+
if (fs.lstatSync(file).isFIFO())
|
|
25
|
+
return;
|
|
26
|
+
fs.rmSync(file, { force: true });
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// Missing: created below.
|
|
30
|
+
}
|
|
31
|
+
const r = spawnSync("mkfifo", ["-m", "600", file], { encoding: "utf8" });
|
|
32
|
+
if (r.error)
|
|
33
|
+
throw r.error;
|
|
34
|
+
if (r.status !== 0)
|
|
35
|
+
throw new Error(`mkfifo ${file} failed: ${(r.stderr ?? "").trim() || `exit ${r.status}`}`);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The service's stdout. Opened for reading and writing, so the FIFO keeps a
|
|
39
|
+
* reader while no daemon has it open and the service's writes never fail.
|
|
40
|
+
*/
|
|
41
|
+
export function openProtocolWriter(file) {
|
|
42
|
+
return fs.openSync(file, fs.constants.O_RDWR);
|
|
43
|
+
}
|
|
44
|
+
export function createProtocolTail(file, opts = {}) {
|
|
45
|
+
const events = new EventEmitter();
|
|
46
|
+
const setI = opts.setInterval ?? ((fn, ms) => setInterval(fn, ms));
|
|
47
|
+
const clearI = opts.clearInterval ?? ((h) => clearInterval(h));
|
|
48
|
+
let fd = null;
|
|
49
|
+
let done = false;
|
|
50
|
+
const open = () => {
|
|
51
|
+
if (fd !== null)
|
|
52
|
+
return fd;
|
|
53
|
+
try {
|
|
54
|
+
fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NONBLOCK);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
fd = null;
|
|
58
|
+
}
|
|
59
|
+
return fd;
|
|
60
|
+
};
|
|
61
|
+
const read = () => {
|
|
62
|
+
const handle = open();
|
|
63
|
+
if (handle === null)
|
|
64
|
+
return;
|
|
65
|
+
const buf = Buffer.alloc(READ_CHUNK_BYTES);
|
|
66
|
+
for (;;) {
|
|
67
|
+
let n;
|
|
68
|
+
try {
|
|
69
|
+
n = fs.readSync(handle, buf, 0, buf.length, null);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
// EAGAIN: nothing written since the last read.
|
|
73
|
+
if (err.code === "EAGAIN")
|
|
74
|
+
return;
|
|
75
|
+
throw err;
|
|
76
|
+
}
|
|
77
|
+
if (n <= 0)
|
|
78
|
+
return;
|
|
79
|
+
events.emit("data", Buffer.from(buf.subarray(0, n)));
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const stop = () => {
|
|
83
|
+
done = true;
|
|
84
|
+
clearI(timer);
|
|
85
|
+
if (fd !== null) {
|
|
86
|
+
try {
|
|
87
|
+
fs.closeSync(fd);
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
// already closed
|
|
91
|
+
}
|
|
92
|
+
fd = null;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
const timer = setI(() => {
|
|
96
|
+
try {
|
|
97
|
+
read();
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
// An unreadable FIFO reads as silence; the heartbeat watchdog restarts the service.
|
|
101
|
+
}
|
|
102
|
+
}, opts.pollMs ?? POLL_MS);
|
|
103
|
+
timer.unref?.();
|
|
104
|
+
// Open now, so the FIFO has a reader before the service starts writing.
|
|
105
|
+
open();
|
|
106
|
+
const tail = {
|
|
107
|
+
on(event, handler) {
|
|
108
|
+
events.on(event, handler);
|
|
109
|
+
return tail;
|
|
110
|
+
},
|
|
111
|
+
close() {
|
|
112
|
+
if (done)
|
|
113
|
+
return;
|
|
114
|
+
try {
|
|
115
|
+
read();
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
// nothing more to read
|
|
119
|
+
}
|
|
120
|
+
stop();
|
|
121
|
+
events.emit("end");
|
|
122
|
+
},
|
|
123
|
+
detach() {
|
|
124
|
+
if (done)
|
|
125
|
+
return;
|
|
126
|
+
stop();
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
return tail;
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=protocol-tail.js.map
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -25,6 +25,7 @@ import { acquireDaemonLock, releaseDaemonLock } from "./lock.js";
|
|
|
25
25
|
import { daemonRootRefusal } from "./install.js";
|
|
26
26
|
import { detectMachineType } from "./machine-type.js";
|
|
27
27
|
import { appendPrivate, daemonPaths, ensurePrivateDir, openPrivateAppend } from "./paths.js";
|
|
28
|
+
import { createProtocolTail, ensureProtocolFifo, openProtocolWriter } from "./protocol-tail.js";
|
|
28
29
|
import { takeControlRequests } from "./requests.js";
|
|
29
30
|
import { currentHqCloud, hqCloudRunners } from "./hq-cloud.js";
|
|
30
31
|
import { runnerSessionName } from "./remote-control.js";
|
|
@@ -224,6 +225,14 @@ export function unitProcessEnv(base, unit) {
|
|
|
224
225
|
env.NODE_OPTIONS = syncRunnerNodeOptions(env);
|
|
225
226
|
return env;
|
|
226
227
|
}
|
|
228
|
+
function isFifo(file) {
|
|
229
|
+
try {
|
|
230
|
+
return fs.lstatSync(file).isFIFO();
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
227
236
|
export async function runDaemon(opts) {
|
|
228
237
|
const env = opts.env ?? process.env;
|
|
229
238
|
const home = opts.home ?? os.homedir();
|
|
@@ -263,6 +272,8 @@ export async function runDaemon(opts) {
|
|
|
263
272
|
process.kill(pid, sig);
|
|
264
273
|
};
|
|
265
274
|
let adopt = [];
|
|
275
|
+
/** Readers of units' protocol FIFOs; a handoff detaches them so the next daemon reads what is left. */
|
|
276
|
+
const protocolTails = new Set();
|
|
266
277
|
if (succession) {
|
|
267
278
|
const taken = takeSuccession(succession, { now: Date.now, readStart: readProcStart, log });
|
|
268
279
|
await stopOrphans(taken.orphans, {
|
|
@@ -271,7 +282,20 @@ export async function runDaemon(opts) {
|
|
|
271
282
|
sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
272
283
|
log,
|
|
273
284
|
});
|
|
274
|
-
adopt = taken.adopt.map((u) =>
|
|
285
|
+
adopt = taken.adopt.map((u) => {
|
|
286
|
+
// A unit that speaks the sync protocol writes it to a FIFO; read what the previous daemon left.
|
|
287
|
+
const protocolPath = paths.unitProtocolPath(u.id);
|
|
288
|
+
const stdout = isFifo(protocolPath) ? createProtocolTail(protocolPath) : undefined;
|
|
289
|
+
const child = adoptChild(u, { readStart: readProcStart, signal: signalPid, ...(stdout ? { stdout } : {}) });
|
|
290
|
+
if (stdout) {
|
|
291
|
+
protocolTails.add(stdout);
|
|
292
|
+
child.on("exit", () => {
|
|
293
|
+
protocolTails.delete(stdout);
|
|
294
|
+
stdout.close();
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
return { id: u.id, key: u.key, child };
|
|
298
|
+
});
|
|
275
299
|
}
|
|
276
300
|
const tracker = succession ? createChildTracker(succession.childrenPath, { readStart: readProcStart, log }) : null;
|
|
277
301
|
for (const unit of adopt)
|
|
@@ -381,11 +405,57 @@ export async function runDaemon(opts) {
|
|
|
381
405
|
: [command.file, command.args];
|
|
382
406
|
const logPath = paths.unitLogPath(unit.id);
|
|
383
407
|
const cwd = unit.env.HQ_ROOT && fs.existsSync(unit.env.HQ_ROOT) ? unit.env.HQ_ROOT : home;
|
|
408
|
+
if (unit.heartbeatTimeoutMs && succession) {
|
|
409
|
+
// The supervisor reads this unit's protocol heartbeats from stdout. Where
|
|
410
|
+
// services can outlive the daemon, stdout is a FIFO (see protocol-tail.ts)
|
|
411
|
+
// rather than a pipe, so the unit can be handed to the next daemon.
|
|
412
|
+
const protocolPath = paths.unitProtocolPath(unit.id);
|
|
413
|
+
ensureProtocolFifo(protocolPath);
|
|
414
|
+
const stdout = createProtocolTail(protocolPath);
|
|
415
|
+
const outFd = openProtocolWriter(protocolPath);
|
|
416
|
+
const errFd = openPrivateAppend(logPath);
|
|
417
|
+
try {
|
|
418
|
+
const child = spawn(file, args, {
|
|
419
|
+
cwd,
|
|
420
|
+
env: unitProcessEnv(env, unit),
|
|
421
|
+
stdio: ["ignore", outFd, errFd],
|
|
422
|
+
windowsHide: true,
|
|
423
|
+
});
|
|
424
|
+
spawned.add(child);
|
|
425
|
+
if (unit.service === "sync" && child.pid !== undefined)
|
|
426
|
+
daemonSyncRunnerPids.add(child.pid);
|
|
427
|
+
protocolTails.add(stdout);
|
|
428
|
+
const finish = () => {
|
|
429
|
+
protocolTails.delete(stdout);
|
|
430
|
+
stdout.close();
|
|
431
|
+
};
|
|
432
|
+
child.once("exit", () => {
|
|
433
|
+
spawned.delete(child);
|
|
434
|
+
if (child.pid !== undefined)
|
|
435
|
+
daemonSyncRunnerPids.delete(child.pid);
|
|
436
|
+
finish();
|
|
437
|
+
});
|
|
438
|
+
child.once("error", finish);
|
|
439
|
+
const supervised = {
|
|
440
|
+
pid: child.pid,
|
|
441
|
+
stdout,
|
|
442
|
+
on(event, handler) {
|
|
443
|
+
child.on(event, handler);
|
|
444
|
+
return supervised;
|
|
445
|
+
},
|
|
446
|
+
kill: (signal) => child.kill(signal),
|
|
447
|
+
};
|
|
448
|
+
if (unit.kind === "long-running")
|
|
449
|
+
tracker?.track(unit.id, supervised);
|
|
450
|
+
return supervised;
|
|
451
|
+
}
|
|
452
|
+
finally {
|
|
453
|
+
fs.closeSync(outFd);
|
|
454
|
+
fs.closeSync(errFd);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
384
457
|
if (unit.heartbeatTimeoutMs) {
|
|
385
|
-
//
|
|
386
|
-
// output goes through the daemon. A piped child cannot outlive the daemon,
|
|
387
|
-
// so it is never handed over: a handoff stops it and the next daemon
|
|
388
|
-
// starts it again.
|
|
458
|
+
// Elsewhere the service stops with the daemon, so its output can go through a pipe.
|
|
389
459
|
const logFd = openPrivateAppend(logPath);
|
|
390
460
|
const logStream = fs.createWriteStream(logPath, { fd: logFd, autoClose: true });
|
|
391
461
|
logStream.on("error", (err) => log("error", `could not write ${unit.id} log ${logPath}: ${err.message}`));
|
|
@@ -525,6 +595,10 @@ export async function runDaemon(opts) {
|
|
|
525
595
|
return;
|
|
526
596
|
stopping = true;
|
|
527
597
|
const released = supervisor.release();
|
|
598
|
+
// What these readers have not read stays in the FIFO for the next daemon.
|
|
599
|
+
for (const tail of protocolTails)
|
|
600
|
+
tail.detach();
|
|
601
|
+
protocolTails.clear();
|
|
528
602
|
const starts = new Map((tracker?.records() ?? []).map((r) => [r.pid, r.start]));
|
|
529
603
|
const units = released.flatMap((u) => {
|
|
530
604
|
const start = starts.get(u.pid);
|
|
@@ -95,6 +95,8 @@ export declare function adoptChild(record: ChildRecord, deps: {
|
|
|
95
95
|
setInterval?: (fn: () => void, ms: number) => unknown;
|
|
96
96
|
clearInterval?: (handle: unknown) => void;
|
|
97
97
|
pollMs?: number;
|
|
98
|
+
/** Its protocol output, read from the file it writes to (see protocol-tail.ts). */
|
|
99
|
+
stdout?: SupervisedChild["stdout"];
|
|
98
100
|
}): SupervisedChild;
|
|
99
101
|
/** The loaded daemon unit's KillMode, or null when systemctl cannot tell. */
|
|
100
102
|
export declare function systemdKillMode(run: (file: string, args: string[]) => {
|
|
@@ -103,7 +103,7 @@ export function renderText(options) {
|
|
|
103
103
|
const pad = " ".repeat(STATUS_LABEL_WIDTH);
|
|
104
104
|
const lines = [];
|
|
105
105
|
lines.push(`hq doctor — HQ root: ${hqRoot}`);
|
|
106
|
-
lines.push(`
|
|
106
|
+
lines.push(`Agent platform: ${platform.id}`);
|
|
107
107
|
for (const run of families) {
|
|
108
108
|
lines.push("");
|
|
109
109
|
lines.push(`${run.family.title} (${run.family.id})`);
|