@indigoai-us/hq-cli 5.133.0 → 5.135.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/command-catalog.generated.d.ts +40 -0
- package/dist/command-catalog.generated.js +52 -0
- package/dist/commands/agent-kit.d.ts +2 -0
- package/dist/commands/agent-kit.js +4 -1
- package/dist/commands/lanes.d.ts +4 -2
- package/dist/commands/lanes.js +150 -2
- package/dist/lib/agent-kit/run/heartbeat.d.ts +14 -2
- package/dist/lib/agent-kit/run/heartbeat.js +38 -6
- package/dist/lib/bot/api.d.ts +3 -1
- package/dist/lib/bot/api.js +7 -2
- package/dist/lib/bot/run.d.ts +4 -0
- package/dist/lib/bot/run.js +30 -1
- package/dist/lib/bot/status.d.ts +4 -0
- package/dist/lib/doctor/checks/bots.js +4 -1
- package/dist/lib/heartbeat-identity.d.ts +29 -0
- package/dist/lib/heartbeat-identity.js +71 -0
- package/dist/lib/lanes/config.d.ts +7 -0
- package/dist/lib/lanes/config.js +28 -1
- package/dist/lib/lanes/index.d.ts +5 -1
- package/dist/lib/lanes/index.js +3 -1
- package/dist/lib/lanes/monitor.d.ts +48 -0
- package/dist/lib/lanes/monitor.js +409 -0
- package/dist/lib/lanes/spawn.d.ts +10 -1
- package/dist/lib/lanes/spawn.js +60 -5
- package/dist/lib/lanes/types.d.ts +6 -0
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watch.d.ts +22 -0
- package/dist/lib/lanes/watch.js +122 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,36 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.135.0] — 2026-09-23
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Claude lanes now set Claude Code's `CLAUDE_CODE_AUTO_COMPACT_WINDOW=200000`
|
|
10
|
+
control on fresh launches and session resumes.
|
|
11
|
+
`providers.claude.auto_compact_tokens` sets a positive-integer override, and
|
|
12
|
+
Codex and Grok launch environments are unchanged.
|
|
13
|
+
|
|
14
|
+
- Added `hq lanes watch` for bounded state and question polling, plus
|
|
15
|
+
`hq lanes monitor-check` for Claude Monitor coverage with distinct exit codes
|
|
16
|
+
for uncovered lanes and unreadable evidence. The reminder text is available
|
|
17
|
+
from `hq lanes monitor-check --reminder`.
|
|
18
|
+
- Corrected watch JSONL output and question status transitions, bounded Monitor
|
|
19
|
+
recommendations to the watch limit, and made stale or unreadable Monitor
|
|
20
|
+
evidence fail closed.
|
|
21
|
+
|
|
22
|
+
- Lanes can opt selected Codex model ids into fast mode with
|
|
23
|
+
`providers.codex.fast_mode_models`. Matching fresh and resumed runs pass
|
|
24
|
+
Codex's `service_tier="fast"` and `fast_mode` settings and record the
|
|
25
|
+
decision in the lane record and `spawn.json`.
|
|
26
|
+
|
|
27
|
+
## [5.134.0] — 2026-09-22
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- Agent-kit and local-bot heartbeat loops now stop after the server reports an
|
|
32
|
+
agent identity mismatch instead of retrying indefinitely. `hq doctor` shows
|
|
33
|
+
the configured and token agent IDs for the mismatch.
|
|
34
|
+
|
|
5
35
|
## [5.133.0] — 2026-09-22
|
|
6
36
|
|
|
7
37
|
### Fixed
|
|
@@ -6744,6 +6744,46 @@ export declare const COMMAND_CATALOG: readonly [{
|
|
|
6744
6744
|
readonly description: "JSON on stdout (always)";
|
|
6745
6745
|
}];
|
|
6746
6746
|
readonly subcommands: readonly [];
|
|
6747
|
+
}, {
|
|
6748
|
+
readonly name: "watch";
|
|
6749
|
+
readonly description: "Watch lane state and question changes until all lanes are terminal";
|
|
6750
|
+
readonly aliases: readonly [];
|
|
6751
|
+
readonly hidden: false;
|
|
6752
|
+
readonly usage: "[options] <lane-id...>";
|
|
6753
|
+
readonly arguments: readonly [{
|
|
6754
|
+
readonly name: "lane-id";
|
|
6755
|
+
readonly required: true;
|
|
6756
|
+
readonly variadic: true;
|
|
6757
|
+
}];
|
|
6758
|
+
readonly options: readonly [{
|
|
6759
|
+
readonly flags: "--interval-ms <ms>";
|
|
6760
|
+
readonly description: "Polling interval, bounded to 100ms–30s";
|
|
6761
|
+
}, {
|
|
6762
|
+
readonly flags: "--json";
|
|
6763
|
+
readonly description: "JSON on stdout (always)";
|
|
6764
|
+
}];
|
|
6765
|
+
readonly subcommands: readonly [];
|
|
6766
|
+
}, {
|
|
6767
|
+
readonly name: "monitor-check";
|
|
6768
|
+
readonly description: "Check that active Claude senior lanes have live Monitor watch tasks";
|
|
6769
|
+
readonly aliases: readonly [];
|
|
6770
|
+
readonly hidden: false;
|
|
6771
|
+
readonly usage: "[options]";
|
|
6772
|
+
readonly arguments: readonly [];
|
|
6773
|
+
readonly options: readonly [{
|
|
6774
|
+
readonly flags: "--session <id>";
|
|
6775
|
+
readonly description: "Senior session id";
|
|
6776
|
+
}, {
|
|
6777
|
+
readonly flags: "--engine <engine>";
|
|
6778
|
+
readonly description: "claude | codex | grok; defaults to HQ_CONDUCT_ENGINE or claude";
|
|
6779
|
+
}, {
|
|
6780
|
+
readonly flags: "--reminder";
|
|
6781
|
+
readonly description: "Print the senior Monitor reminder text";
|
|
6782
|
+
}, {
|
|
6783
|
+
readonly flags: "--json";
|
|
6784
|
+
readonly description: "JSON on stdout";
|
|
6785
|
+
}];
|
|
6786
|
+
readonly subcommands: readonly [];
|
|
6747
6787
|
}, {
|
|
6748
6788
|
readonly name: "show";
|
|
6749
6789
|
readonly description: "Show one lane";
|
|
@@ -8716,6 +8716,58 @@ export const COMMAND_CATALOG = [
|
|
|
8716
8716
|
],
|
|
8717
8717
|
"subcommands": []
|
|
8718
8718
|
},
|
|
8719
|
+
{
|
|
8720
|
+
"name": "watch",
|
|
8721
|
+
"description": "Watch lane state and question changes until all lanes are terminal",
|
|
8722
|
+
"aliases": [],
|
|
8723
|
+
"hidden": false,
|
|
8724
|
+
"usage": "[options] <lane-id...>",
|
|
8725
|
+
"arguments": [
|
|
8726
|
+
{
|
|
8727
|
+
"name": "lane-id",
|
|
8728
|
+
"required": true,
|
|
8729
|
+
"variadic": true
|
|
8730
|
+
}
|
|
8731
|
+
],
|
|
8732
|
+
"options": [
|
|
8733
|
+
{
|
|
8734
|
+
"flags": "--interval-ms <ms>",
|
|
8735
|
+
"description": "Polling interval, bounded to 100ms–30s"
|
|
8736
|
+
},
|
|
8737
|
+
{
|
|
8738
|
+
"flags": "--json",
|
|
8739
|
+
"description": "JSON on stdout (always)"
|
|
8740
|
+
}
|
|
8741
|
+
],
|
|
8742
|
+
"subcommands": []
|
|
8743
|
+
},
|
|
8744
|
+
{
|
|
8745
|
+
"name": "monitor-check",
|
|
8746
|
+
"description": "Check that active Claude senior lanes have live Monitor watch tasks",
|
|
8747
|
+
"aliases": [],
|
|
8748
|
+
"hidden": false,
|
|
8749
|
+
"usage": "[options]",
|
|
8750
|
+
"arguments": [],
|
|
8751
|
+
"options": [
|
|
8752
|
+
{
|
|
8753
|
+
"flags": "--session <id>",
|
|
8754
|
+
"description": "Senior session id"
|
|
8755
|
+
},
|
|
8756
|
+
{
|
|
8757
|
+
"flags": "--engine <engine>",
|
|
8758
|
+
"description": "claude | codex | grok; defaults to HQ_CONDUCT_ENGINE or claude"
|
|
8759
|
+
},
|
|
8760
|
+
{
|
|
8761
|
+
"flags": "--reminder",
|
|
8762
|
+
"description": "Print the senior Monitor reminder text"
|
|
8763
|
+
},
|
|
8764
|
+
{
|
|
8765
|
+
"flags": "--json",
|
|
8766
|
+
"description": "JSON on stdout"
|
|
8767
|
+
}
|
|
8768
|
+
],
|
|
8769
|
+
"subcommands": []
|
|
8770
|
+
},
|
|
8719
8771
|
{
|
|
8720
8772
|
"name": "show",
|
|
8721
8773
|
"description": "Show one lane",
|
|
@@ -23,6 +23,7 @@ import { type AgentKitPaths } from "../lib/agent-kit/paths.js";
|
|
|
23
23
|
import { type KitService } from "../lib/agent-kit/services.js";
|
|
24
24
|
import { type CronRunner, type RebootCronResult, type DetachedSpawn, type KillFn } from "../lib/agent-kit/fallback.js";
|
|
25
25
|
import { type ServiceHostPaths, type ServiceManagerDeps, type ServiceSetResult } from "../lib/service-manager/index.js";
|
|
26
|
+
import type { HeartbeatCredsSource } from "../lib/heartbeat-identity.js";
|
|
26
27
|
export declare class KitError extends Error {
|
|
27
28
|
constructor(message: string);
|
|
28
29
|
}
|
|
@@ -73,6 +74,7 @@ export interface KitRuntime {
|
|
|
73
74
|
config: KitConfig;
|
|
74
75
|
getToken: () => Promise<string>;
|
|
75
76
|
log: KitLogger;
|
|
77
|
+
credsSource?: HeartbeatCredsSource;
|
|
76
78
|
}
|
|
77
79
|
export declare function buildKitRuntime(service: KitService | "all"): KitRuntime;
|
|
78
80
|
export declare function runKitService(service: KitService, rt: KitRuntime): Promise<void>;
|
|
@@ -135,6 +135,7 @@ export function formatServiceSet(result) {
|
|
|
135
135
|
}
|
|
136
136
|
export function buildKitRuntime(service) {
|
|
137
137
|
const paths = agentKitPaths(os.homedir(), process.env);
|
|
138
|
+
const credsSource = process.env.HQ_MACHINE_CREDS_FILE?.trim() ? "explicit" : "default";
|
|
138
139
|
const creds = requireExternalCreds(paths);
|
|
139
140
|
ensureKitDirs(paths);
|
|
140
141
|
const config = readKitConfig(paths, CLI_VERSION);
|
|
@@ -145,7 +146,7 @@ export function buildKitRuntime(service) {
|
|
|
145
146
|
process.env.HQ_VAULT_API_URL = creds.apiBaseUrl;
|
|
146
147
|
const log = createKitLogger(paths, service, { echo: process.stdout.isTTY === true });
|
|
147
148
|
const getToken = () => ensureCognitoToken({ tokenSource: "machine", interactive: false });
|
|
148
|
-
return { paths, creds, config, getToken, log };
|
|
149
|
+
return { paths, creds, config, getToken, log, credsSource };
|
|
149
150
|
}
|
|
150
151
|
export async function runKitService(service, rt) {
|
|
151
152
|
rt.log("info", `${service} starting agent=${rt.creds.entityUid} cli=${CLI_VERSION}`);
|
|
@@ -167,6 +168,8 @@ export async function runKitService(service, rt) {
|
|
|
167
168
|
getToken: rt.getToken,
|
|
168
169
|
log: rt.log,
|
|
169
170
|
components: enabledKitComponents(rt.config),
|
|
171
|
+
agentUidSource: rt.credsSource === "explicit" ? "env" : "config_file",
|
|
172
|
+
credsSource: rt.credsSource ?? "default",
|
|
170
173
|
});
|
|
171
174
|
return;
|
|
172
175
|
case "inbox":
|
package/dist/commands/lanes.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `hq lanes` — worker lane management and decision queue (spec §5–§7, §10–§11).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Existing commands print JSON on stdout and take no interactive input, except
|
|
5
5
|
* `accounts login`, which hands the terminal to the vendor's own sign-in.
|
|
6
|
+
* `watch` and the monitor reminder are the line-oriented text surfaces.
|
|
6
7
|
*/
|
|
7
8
|
import { Command } from "commander";
|
|
8
9
|
import { type OutpostDispatchResult } from "../lib/lanes/outpost-dispatch.js";
|
|
9
10
|
import { emitLaneDecisionEvents, type LaneMesh } from "../lib/lanes/mesh-events.js";
|
|
10
|
-
import { MAX_LANE_DEPTH, type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
|
|
11
|
+
import { MAX_LANE_DEPTH, type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type MonitorEvidenceReader, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
|
|
11
12
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
12
13
|
import { type RunCompactionFn } from "../lib/lanes/compaction.js";
|
|
13
14
|
import { type KnownModelsDeps, type ProbeCommand } from "../lib/lanes/models.js";
|
|
@@ -76,6 +77,7 @@ export interface LanesCommandDeps {
|
|
|
76
77
|
fetchText?: (url: string) => Promise<string>;
|
|
77
78
|
runCompaction?: RunCompactionFn;
|
|
78
79
|
locateTranscript?: (home: string, sessionId: string) => string | null;
|
|
80
|
+
monitorEvidence?: MonitorEvidenceReader;
|
|
79
81
|
}
|
|
80
82
|
export declare function readOutpostIdentity(existsSync?: (p: string) => boolean, readFileSync?: (p: string) => string): OutpostIdentity;
|
|
81
83
|
export declare function parseSeniorRef(raw: string): SeniorRef;
|
package/dist/commands/lanes.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `hq lanes` — worker lane management and decision queue (spec §5–§7, §10–§11).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Existing commands print JSON on stdout and take no interactive input, except
|
|
5
5
|
* `accounts login`, which hands the terminal to the vendor's own sign-in.
|
|
6
|
+
* `watch` and the monitor reminder are the line-oriented text surfaces.
|
|
6
7
|
*/
|
|
7
8
|
import { spawn, spawnSync } from "node:child_process";
|
|
8
9
|
import { randomBytes } from "node:crypto";
|
|
@@ -17,7 +18,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
|
|
|
17
18
|
import { openMeshTransport } from "../lib/mesh/client.js";
|
|
18
19
|
import { createLaneMesh, emitLaneDecisionEvents, } from "../lib/lanes/mesh-events.js";
|
|
19
20
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
20
|
-
import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, MAX_LANE_DEPTH, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
21
|
+
import { AccountResolutionError, coveredLaneIds, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, MONITOR_ENGINES, MONITOR_MAX_TIMEOUT_MS, MONITOR_UNCOVERED_EXIT_CODE, MAX_WATCHED_LANES, monitorReminderText, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, runLaneWatch, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
21
22
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
22
23
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
23
24
|
import { providerCompactAfterMs } from "../lib/lanes/config.js";
|
|
@@ -62,6 +63,24 @@ function wrap(action) {
|
|
|
62
63
|
}
|
|
63
64
|
};
|
|
64
65
|
}
|
|
66
|
+
function failText(err) {
|
|
67
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
68
|
+
process.stderr.write(`hq lanes: ${message}\n`);
|
|
69
|
+
process.exit(MONITOR_ERROR_EXIT_CODE);
|
|
70
|
+
}
|
|
71
|
+
function wrapText(action) {
|
|
72
|
+
return async (...args) => {
|
|
73
|
+
let code;
|
|
74
|
+
try {
|
|
75
|
+
code = await action(...args);
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
failText(err);
|
|
79
|
+
}
|
|
80
|
+
if (code !== 0)
|
|
81
|
+
process.exit(code);
|
|
82
|
+
};
|
|
83
|
+
}
|
|
65
84
|
function collectRepeatable(value, previous = []) {
|
|
66
85
|
return previous.concat(value);
|
|
67
86
|
}
|
|
@@ -206,6 +225,12 @@ function resolveDeps(deps = {}) {
|
|
|
206
225
|
probeProvider: deps.probeProvider ?? defaultProbe,
|
|
207
226
|
runCompaction: deps.runCompaction,
|
|
208
227
|
locateTranscript: deps.locateTranscript,
|
|
228
|
+
monitorEvidence: deps.monitorEvidence ??
|
|
229
|
+
((sessionId) => readLiveMonitorEvidence(sessionId, {
|
|
230
|
+
homedir: () => deps.homedir?.() ?? os.homedir(),
|
|
231
|
+
env,
|
|
232
|
+
now: deps.now ?? (() => new Date()),
|
|
233
|
+
})),
|
|
209
234
|
stageOutpostPayload: deps.stageOutpostPayload ?? defaultStageOutpostPayload,
|
|
210
235
|
dispatchOutpostExec: deps.dispatchOutpostExec ?? defaultDispatchOutpostExec,
|
|
211
236
|
fetchText: deps.fetchText ?? defaultFetchText,
|
|
@@ -387,6 +412,115 @@ function loadAllLanes(hqRoot) {
|
|
|
387
412
|
function loadAllQuestions(hqRoot) {
|
|
388
413
|
return listQuestionIds(hqRoot).map((id) => readQuestion(hqRoot, id));
|
|
389
414
|
}
|
|
415
|
+
const MONITOR_ACTIVE_LANE_STATES = new Set([
|
|
416
|
+
"running",
|
|
417
|
+
"queued",
|
|
418
|
+
"awaiting_input",
|
|
419
|
+
"blocked",
|
|
420
|
+
]);
|
|
421
|
+
function resolveMonitorEngine(requested, env) {
|
|
422
|
+
const value = requested?.trim().toLowerCase() ||
|
|
423
|
+
env.HQ_CONDUCT_ENGINE?.trim().toLowerCase() ||
|
|
424
|
+
"claude";
|
|
425
|
+
if (!MONITOR_ENGINES.includes(value)) {
|
|
426
|
+
throw new LanesError("monitor_invalid_engine", `Unknown monitor-check engine ${JSON.stringify(value)}. Accepted: ${MONITOR_ENGINES.join(", ")}`, { accepted: [...MONITOR_ENGINES], value });
|
|
427
|
+
}
|
|
428
|
+
return value;
|
|
429
|
+
}
|
|
430
|
+
async function runWatch(ctx, laneIds, opts) {
|
|
431
|
+
const rawInterval = opts.intervalMs === undefined ? undefined : Number(opts.intervalMs);
|
|
432
|
+
await runLaneWatch(ctx.hqRoot(), laneIds, {
|
|
433
|
+
intervalMs: rawInterval,
|
|
434
|
+
sleep: ctx.sleep,
|
|
435
|
+
writeLine: (line) => console.log(opts.json ? JSON.stringify({ event: line }) : line),
|
|
436
|
+
writeError: (line) => process.stderr.write(`${line}\n`),
|
|
437
|
+
});
|
|
438
|
+
return 0;
|
|
439
|
+
}
|
|
440
|
+
function monitorCallsForLaneIds(laneIds) {
|
|
441
|
+
const calls = [];
|
|
442
|
+
for (let offset = 0; offset < laneIds.length; offset += MAX_WATCHED_LANES) {
|
|
443
|
+
const chunk = laneIds.slice(offset, offset + MAX_WATCHED_LANES);
|
|
444
|
+
calls.push({
|
|
445
|
+
command: `hq lanes watch ${chunk.join(" ")}`,
|
|
446
|
+
timeout_ms: MONITOR_MAX_TIMEOUT_MS,
|
|
447
|
+
persistent: true,
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
return calls;
|
|
451
|
+
}
|
|
452
|
+
async function runMonitorCheck(ctx, opts) {
|
|
453
|
+
if (opts.reminder) {
|
|
454
|
+
console.log(monitorReminderText());
|
|
455
|
+
return 0;
|
|
456
|
+
}
|
|
457
|
+
const sessionId = opts.session?.trim();
|
|
458
|
+
if (!sessionId) {
|
|
459
|
+
throw new LanesError("monitor_session_required", "hq lanes monitor-check requires --session <id> unless --reminder is used");
|
|
460
|
+
}
|
|
461
|
+
const engine = resolveMonitorEngine(opts.engine, ctx.env);
|
|
462
|
+
const activeLanes = loadAllLanes(ctx.hqRoot()).filter((lane) => lane.senior.kind === "session" &&
|
|
463
|
+
lane.senior.id === sessionId &&
|
|
464
|
+
MONITOR_ACTIVE_LANE_STATES.has(lane.state));
|
|
465
|
+
if (engine !== "claude") {
|
|
466
|
+
if (opts.json) {
|
|
467
|
+
printJson({
|
|
468
|
+
ok: true,
|
|
469
|
+
action: "monitor-check",
|
|
470
|
+
session_id: sessionId,
|
|
471
|
+
engine,
|
|
472
|
+
active_lane_ids: activeLanes.map((lane) => lane.lane_id),
|
|
473
|
+
covered_lane_ids: [],
|
|
474
|
+
uncovered_lane_ids: [],
|
|
475
|
+
skipped: "engine_has_no_claude_monitor",
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
return 0;
|
|
479
|
+
}
|
|
480
|
+
if (activeLanes.length === 0) {
|
|
481
|
+
if (opts.json) {
|
|
482
|
+
printJson({
|
|
483
|
+
ok: true,
|
|
484
|
+
action: "monitor-check",
|
|
485
|
+
session_id: sessionId,
|
|
486
|
+
engine,
|
|
487
|
+
active_lane_ids: [],
|
|
488
|
+
covered_lane_ids: [],
|
|
489
|
+
uncovered_lane_ids: [],
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
return 0;
|
|
493
|
+
}
|
|
494
|
+
const evidence = await ctx.monitorEvidence(sessionId);
|
|
495
|
+
if (evidence.sessionId !== sessionId) {
|
|
496
|
+
throw new LanesError("monitor_evidence_session_mismatch", `Monitor evidence was for ${JSON.stringify(evidence.sessionId)}, expected ${JSON.stringify(sessionId)}`);
|
|
497
|
+
}
|
|
498
|
+
const covered = coveredLaneIds(evidence, ctx.now());
|
|
499
|
+
const activeIds = activeLanes.map((lane) => lane.lane_id);
|
|
500
|
+
const uncovered = activeIds.filter((laneId) => !covered.has(laneId));
|
|
501
|
+
const monitorCalls = monitorCallsForLaneIds(uncovered);
|
|
502
|
+
if (opts.json) {
|
|
503
|
+
printJson({
|
|
504
|
+
ok: uncovered.length === 0,
|
|
505
|
+
action: "monitor-check",
|
|
506
|
+
session_id: sessionId,
|
|
507
|
+
engine,
|
|
508
|
+
active_lane_ids: activeIds,
|
|
509
|
+
covered_lane_ids: activeIds.filter((laneId) => covered.has(laneId)),
|
|
510
|
+
uncovered_lane_ids: uncovered,
|
|
511
|
+
monitor_call: monitorCalls.length === 1 ? monitorCalls[0] : null,
|
|
512
|
+
monitor_calls: monitorCalls,
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
else if (uncovered.length > 0) {
|
|
516
|
+
for (const laneId of uncovered)
|
|
517
|
+
console.log(`uncovered lane: ${laneId}`);
|
|
518
|
+
for (const monitorCall of monitorCalls) {
|
|
519
|
+
console.log(`Monitor(command=${JSON.stringify(monitorCall.command)}, timeout_ms=${monitorCall.timeout_ms}, persistent=true)`);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return uncovered.length > 0 ? MONITOR_UNCOVERED_EXIT_CODE : 0;
|
|
523
|
+
}
|
|
390
524
|
function sessionMetaPath(hqRoot, sessionId) {
|
|
391
525
|
return path.join(hqRoot, "workspace", "sessions", sessionId, "meta.yaml");
|
|
392
526
|
}
|
|
@@ -3193,6 +3327,20 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
3193
3327
|
.action(wrap(async (opts) => {
|
|
3194
3328
|
await runList(ctx, opts);
|
|
3195
3329
|
}));
|
|
3330
|
+
lanes
|
|
3331
|
+
.command("watch")
|
|
3332
|
+
.description("Watch lane state and question changes until all lanes are terminal")
|
|
3333
|
+
.argument("<lane-id...>", "Lane ids to watch")
|
|
3334
|
+
.option("--interval-ms <ms>", "Polling interval, bounded to 100ms–30s", "5000")
|
|
3335
|
+
.action(wrapText(async (laneIds, opts) => runWatch(ctx, laneIds, opts)));
|
|
3336
|
+
lanes
|
|
3337
|
+
.command("monitor-check")
|
|
3338
|
+
.description("Check that active Claude senior lanes have live Monitor watch tasks")
|
|
3339
|
+
.option("--session <id>", "Senior session id")
|
|
3340
|
+
.option("--engine <engine>", "claude | codex | grok; defaults to HQ_CONDUCT_ENGINE or claude")
|
|
3341
|
+
.option("--reminder", "Print the senior Monitor reminder text")
|
|
3342
|
+
.option("--json", "JSON on stdout")
|
|
3343
|
+
.action(wrapText(async (opts) => runMonitorCheck(ctx, opts)));
|
|
3196
3344
|
lanes
|
|
3197
3345
|
.command("show")
|
|
3198
3346
|
.description("Show one lane")
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* The last successful beat is recorded at state/last-heartbeat.json so
|
|
12
12
|
* `hq agent probe` can report its age.
|
|
13
13
|
*/
|
|
14
|
+
import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../../heartbeat-identity.js";
|
|
14
15
|
import { type ComponentStatus } from "../creds.js";
|
|
15
16
|
import type { KitLogger } from "../log.js";
|
|
16
17
|
import { type AgentKitPaths, type KitComponent } from "../paths.js";
|
|
@@ -36,9 +37,13 @@ export interface HeartbeatBody {
|
|
|
36
37
|
}
|
|
37
38
|
export declare function buildHeartbeatBody(paths: Pick<AgentKitPaths, "stateDir">, now?: () => Date, hostInfo?: HeartbeatBody["hostInfo"], components_?: readonly KitComponent[]): HeartbeatBody;
|
|
38
39
|
export interface LastHeartbeat {
|
|
40
|
+
/** The most recent heartbeat attempt, successful or a terminal identity mismatch. */
|
|
39
41
|
at: string;
|
|
40
42
|
components: Partial<Record<KitComponent, ComponentStatus>>;
|
|
41
43
|
status: number;
|
|
44
|
+
reason?: "identity_mismatch";
|
|
45
|
+
configuredAgentUid?: string;
|
|
46
|
+
tokenAgentUid?: string;
|
|
42
47
|
}
|
|
43
48
|
export declare function writeLastHeartbeat(paths: Pick<AgentKitPaths, "stateDir" | "lastHeartbeatPath">, record: LastHeartbeat): void;
|
|
44
49
|
export declare function readLastHeartbeat(paths: Pick<AgentKitPaths, "lastHeartbeatPath">): LastHeartbeat | null;
|
|
@@ -51,12 +56,19 @@ export interface HeartbeatDeps {
|
|
|
51
56
|
log: KitLogger;
|
|
52
57
|
/** Components this kit runs; defaults to all of them. */
|
|
53
58
|
components?: readonly KitComponent[];
|
|
54
|
-
post?: (token: string, body: HeartbeatBody) => Promise<number>;
|
|
59
|
+
post?: (token: string, body: HeartbeatBody) => Promise<number | HeartbeatPostResult>;
|
|
55
60
|
now?: () => Date;
|
|
56
61
|
sleep?: (ms: number) => Promise<void>;
|
|
57
62
|
maxBeats?: number;
|
|
63
|
+
agentUidSource?: HeartbeatAgentUidSource;
|
|
64
|
+
credsSource?: HeartbeatCredsSource;
|
|
65
|
+
reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
|
|
58
66
|
}
|
|
59
|
-
export
|
|
67
|
+
export interface HeartbeatPostResult {
|
|
68
|
+
status: number;
|
|
69
|
+
bodyText?: string;
|
|
70
|
+
}
|
|
71
|
+
export declare function defaultHeartbeatPost(agentUid: string, apiBaseUrl: string): (token: string, body: HeartbeatBody) => Promise<HeartbeatPostResult>;
|
|
60
72
|
export declare function beatOnce(deps: HeartbeatDeps): Promise<boolean>;
|
|
61
73
|
export declare function runHeartbeatLoop(deps: HeartbeatDeps): Promise<void>;
|
|
62
74
|
//# sourceMappingURL=heartbeat.d.ts.map
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import * as fs from "node:fs";
|
|
15
15
|
import * as os from "node:os";
|
|
16
16
|
import { CLI_VERSION } from "../../../cli-version.js";
|
|
17
|
+
import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, parseHeartbeatFailureBody, tokenAgentUid, } from "../../heartbeat-identity.js";
|
|
17
18
|
import { vaultApiFetch } from "../../../utils/vault-api.js";
|
|
18
19
|
import { readComponentStatus } from "../creds.js";
|
|
19
20
|
import { KIT_COMPONENTS } from "../paths.js";
|
|
@@ -60,36 +61,67 @@ export function defaultHeartbeatPost(agentUid, apiBaseUrl) {
|
|
|
60
61
|
method: "POST",
|
|
61
62
|
body: body,
|
|
62
63
|
});
|
|
63
|
-
return res.status;
|
|
64
|
+
return { status: res.status, bodyText: await res.text().catch(() => "") };
|
|
64
65
|
};
|
|
65
66
|
}
|
|
66
|
-
|
|
67
|
+
async function beatOnceResult(deps) {
|
|
67
68
|
const now = deps.now ?? (() => new Date());
|
|
68
69
|
const post = deps.post ?? defaultHeartbeatPost(deps.agentUid, deps.apiBaseUrl);
|
|
69
70
|
const body = buildHeartbeatBody(deps.paths, now, undefined, deps.components ?? KIT_COMPONENTS);
|
|
70
71
|
try {
|
|
71
72
|
const token = await deps.getToken();
|
|
72
|
-
const
|
|
73
|
+
const response = await post(token, body);
|
|
74
|
+
const result = typeof response === "number" ? { status: response } : response;
|
|
75
|
+
const status = result.status;
|
|
73
76
|
const ok = status >= 200 && status < 300;
|
|
74
77
|
if (ok) {
|
|
75
78
|
writeLastHeartbeat(deps.paths, { at: now().toISOString(), components: body.components, status });
|
|
76
79
|
}
|
|
80
|
+
else {
|
|
81
|
+
const failure = parseHeartbeatFailureBody(result.bodyText);
|
|
82
|
+
if (isHeartbeatIdentityMismatch({ status, code: failure.code, message: failure.message })) {
|
|
83
|
+
const tokenUid = tokenAgentUid(token);
|
|
84
|
+
writeLastHeartbeat(deps.paths, {
|
|
85
|
+
at: now().toISOString(),
|
|
86
|
+
components: body.components,
|
|
87
|
+
status,
|
|
88
|
+
reason: "identity_mismatch",
|
|
89
|
+
configuredAgentUid: deps.agentUid,
|
|
90
|
+
...(tokenUid ? { tokenAgentUid: tokenUid } : {}),
|
|
91
|
+
});
|
|
92
|
+
const tokenLabel = tokenUid ?? "unknown";
|
|
93
|
+
deps.log("warn", `heartbeat stopped: identity mismatch configured agentUid=${deps.agentUid} token agentUid=${tokenLabel}`);
|
|
94
|
+
(deps.reportIdentityMismatch ?? captureHeartbeatIdentityMismatch)({
|
|
95
|
+
runner: "agent-kit",
|
|
96
|
+
configuredAgentUid: deps.agentUid,
|
|
97
|
+
...(tokenUid ? { tokenAgentUid: tokenUid } : {}),
|
|
98
|
+
agentUidSource: deps.agentUidSource ?? "config_file",
|
|
99
|
+
credsSource: deps.credsSource ?? "default",
|
|
100
|
+
});
|
|
101
|
+
return { ok: false, stop: true };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
77
104
|
deps.log(ok ? "info" : "warn", `heartbeat ${status} ${Object.entries(body.components)
|
|
78
105
|
.map(([name, health]) => `${name}=${health}`)
|
|
79
106
|
.join(" ")}`);
|
|
80
|
-
return ok;
|
|
107
|
+
return { ok, stop: false };
|
|
81
108
|
}
|
|
82
109
|
catch (err) {
|
|
83
110
|
deps.log("error", `heartbeat failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
84
|
-
return false;
|
|
111
|
+
return { ok: false, stop: false };
|
|
85
112
|
}
|
|
86
113
|
}
|
|
114
|
+
export async function beatOnce(deps) {
|
|
115
|
+
return (await beatOnceResult(deps)).ok;
|
|
116
|
+
}
|
|
87
117
|
export async function runHeartbeatLoop(deps) {
|
|
88
118
|
const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
89
119
|
let beats = 0;
|
|
90
120
|
for (;;) {
|
|
91
|
-
await
|
|
121
|
+
const result = await beatOnceResult(deps);
|
|
92
122
|
beats += 1;
|
|
123
|
+
if (result.stop)
|
|
124
|
+
return;
|
|
93
125
|
if (deps.maxBeats !== undefined && beats >= deps.maxBeats)
|
|
94
126
|
return;
|
|
95
127
|
await sleep(deps.intervalMs);
|
package/dist/lib/bot/api.d.ts
CHANGED
|
@@ -85,8 +85,10 @@ export declare class BotApiError extends Error {
|
|
|
85
85
|
readonly status: number;
|
|
86
86
|
readonly path: string;
|
|
87
87
|
readonly code?: string | undefined;
|
|
88
|
+
readonly detail?: string | undefined;
|
|
88
89
|
readonly retryAfterMs?: number | undefined;
|
|
89
|
-
|
|
90
|
+
readonly tokenAgentUid?: string | undefined;
|
|
91
|
+
constructor(status: number, path: string, code?: string | undefined, detail?: string | undefined, retryAfterMs?: number | undefined, tokenAgentUid?: string | undefined);
|
|
90
92
|
}
|
|
91
93
|
/** Parse a Retry-After header (delta-seconds or HTTP-date) to milliseconds. */
|
|
92
94
|
export declare function retryAfterHeaderMs(value: string | null | undefined, nowMs?: number): number | undefined;
|
package/dist/lib/bot/api.js
CHANGED
|
@@ -6,18 +6,23 @@
|
|
|
6
6
|
* existing Cognito machinery. `fetch` is injectable for tests.
|
|
7
7
|
*/
|
|
8
8
|
import { vaultApiFetch } from "../../utils/vault-api.js";
|
|
9
|
+
import { tokenAgentUid } from "../heartbeat-identity.js";
|
|
9
10
|
import { assertNoSecretKeys } from "./local-config.js";
|
|
10
11
|
export class BotApiError extends Error {
|
|
11
12
|
status;
|
|
12
13
|
path;
|
|
13
14
|
code;
|
|
15
|
+
detail;
|
|
14
16
|
retryAfterMs;
|
|
15
|
-
|
|
17
|
+
tokenAgentUid;
|
|
18
|
+
constructor(status, path, code, detail, retryAfterMs, tokenAgentUid) {
|
|
16
19
|
super(`HQ API ${path} → ${status}${code ? ` ${code}` : ""}${detail ? `: ${detail}` : ""}`);
|
|
17
20
|
this.status = status;
|
|
18
21
|
this.path = path;
|
|
19
22
|
this.code = code;
|
|
23
|
+
this.detail = detail;
|
|
20
24
|
this.retryAfterMs = retryAfterMs;
|
|
25
|
+
this.tokenAgentUid = tokenAgentUid;
|
|
21
26
|
this.name = "BotApiError";
|
|
22
27
|
}
|
|
23
28
|
}
|
|
@@ -72,7 +77,7 @@ export class BotApi {
|
|
|
72
77
|
});
|
|
73
78
|
if (!accept.includes(res.status)) {
|
|
74
79
|
const err = await readError(res);
|
|
75
|
-
throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")));
|
|
80
|
+
throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")), path.endsWith("/heartbeat") ? tokenAgentUid(token) : undefined);
|
|
76
81
|
}
|
|
77
82
|
const text = await res.text();
|
|
78
83
|
return { status: res.status, body: (text ? JSON.parse(text) : {}) };
|
package/dist/lib/bot/run.d.ts
CHANGED
|
@@ -42,6 +42,7 @@ import type { BotConfig, BotKind } from "./config.js";
|
|
|
42
42
|
import { type ProgressPosterOptions } from "./progress.js";
|
|
43
43
|
import { type BotLogger } from "./log.js";
|
|
44
44
|
import { type BotRuntime, type RuntimeTurnInput, type RuntimeTurnResult } from "./runtime/index.js";
|
|
45
|
+
import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../heartbeat-identity.js";
|
|
45
46
|
import { type BotState, type BotStatusFile } from "./status.js";
|
|
46
47
|
export declare const INBOX_POLL_INTERVAL_MS = 2000;
|
|
47
48
|
export declare const HEARTBEAT_INTERVAL_MS = 30000;
|
|
@@ -119,6 +120,9 @@ export interface BotRunDeps {
|
|
|
119
120
|
maxParallelTurns?: number;
|
|
120
121
|
/** Wait between retries while the coding tool needs a sign-in (default SIGN_IN_RETRY_MS). */
|
|
121
122
|
signInRetryMs?: number;
|
|
123
|
+
agentUidSource?: HeartbeatAgentUidSource;
|
|
124
|
+
credsSource?: HeartbeatCredsSource;
|
|
125
|
+
reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
|
|
122
126
|
}
|
|
123
127
|
export declare const KICKOFF_MESSAGE_ID = "kickoff";
|
|
124
128
|
/**
|
package/dist/lib/bot/run.js
CHANGED
|
@@ -59,6 +59,7 @@ import { dmSessionScope } from "./paths.js";
|
|
|
59
59
|
import { isDmItem, resolveReplyTarget } from "./room-policy.js";
|
|
60
60
|
import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
|
|
61
61
|
import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
|
|
62
|
+
import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
|
|
62
63
|
import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
|
|
63
64
|
import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
|
|
64
65
|
export const INBOX_POLL_INTERVAL_MS = 2_000;
|
|
@@ -415,10 +416,38 @@ export async function runBot(deps) {
|
|
|
415
416
|
// runtime → model, inbox → task, sync → sync.
|
|
416
417
|
await api.heartbeat(config.agentUid, { model: modelHealth, task: "ok", sync: "ok" });
|
|
417
418
|
heartbeatFails = 0;
|
|
418
|
-
setStatus({
|
|
419
|
+
setStatus({
|
|
420
|
+
lastHeartbeatAt: now().toISOString(),
|
|
421
|
+
lastHeartbeatOk: true,
|
|
422
|
+
lastHeartbeatError: undefined,
|
|
423
|
+
lastHeartbeatReason: undefined,
|
|
424
|
+
lastHeartbeatConfiguredAgentUid: undefined,
|
|
425
|
+
lastHeartbeatTokenAgentUid: undefined,
|
|
426
|
+
});
|
|
419
427
|
await sleep(heartbeatIntervalMs);
|
|
420
428
|
}
|
|
421
429
|
catch (err) {
|
|
430
|
+
if (err instanceof BotApiError &&
|
|
431
|
+
isHeartbeatIdentityMismatch({ status: err.status, code: err.code, message: err.detail })) {
|
|
432
|
+
const tokenLabel = err.tokenAgentUid ?? "unknown";
|
|
433
|
+
const reason = `heartbeat stopped: identity mismatch configured agentUid=${config.agentUid} token agentUid=${tokenLabel}`;
|
|
434
|
+
setStatus({
|
|
435
|
+
lastHeartbeatOk: false,
|
|
436
|
+
lastHeartbeatError: reason.slice(0, 400),
|
|
437
|
+
lastHeartbeatReason: "identity_mismatch",
|
|
438
|
+
lastHeartbeatConfiguredAgentUid: config.agentUid,
|
|
439
|
+
...(err.tokenAgentUid ? { lastHeartbeatTokenAgentUid: err.tokenAgentUid } : {}),
|
|
440
|
+
});
|
|
441
|
+
log("warn", reason);
|
|
442
|
+
(deps.reportIdentityMismatch ?? captureHeartbeatIdentityMismatch)({
|
|
443
|
+
runner: "bot",
|
|
444
|
+
configuredAgentUid: config.agentUid,
|
|
445
|
+
...(err.tokenAgentUid ? { tokenAgentUid: err.tokenAgentUid } : {}),
|
|
446
|
+
agentUidSource: deps.agentUidSource ?? "config_file",
|
|
447
|
+
credsSource: deps.credsSource ?? "default",
|
|
448
|
+
});
|
|
449
|
+
break;
|
|
450
|
+
}
|
|
422
451
|
heartbeatFails += 1;
|
|
423
452
|
const message = err instanceof Error ? err.message : String(err);
|
|
424
453
|
setStatus({ lastHeartbeatOk: false, lastHeartbeatError: message.slice(0, 400) });
|
package/dist/lib/bot/status.d.ts
CHANGED
|
@@ -19,6 +19,10 @@ export interface BotStatusFile {
|
|
|
19
19
|
lastHeartbeatOk?: boolean;
|
|
20
20
|
/** Last heartbeat error, when lastHeartbeatOk is false. */
|
|
21
21
|
lastHeartbeatError?: string;
|
|
22
|
+
/** A heartbeat stopped permanently until the bot is restarted with corrected identity config. */
|
|
23
|
+
lastHeartbeatReason?: "identity_mismatch";
|
|
24
|
+
lastHeartbeatConfiguredAgentUid?: string;
|
|
25
|
+
lastHeartbeatTokenAgentUid?: string;
|
|
22
26
|
lastInboxPollAt?: string;
|
|
23
27
|
lastInboxPollOk?: boolean;
|
|
24
28
|
/** Last inbox poll error, when lastInboxPollOk is false. */
|