@indigoai-us/hq-cli 5.132.0 → 5.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/dist/commands/agent-kit.d.ts +2 -0
- package/dist/commands/agent-kit.js +4 -1
- package/dist/commands/lanes.d.ts +2 -2
- package/dist/commands/lanes.js +98 -20
- package/dist/commands/mesh.js +4 -1
- package/dist/lib/agent-kit/run/heartbeat.d.ts +14 -2
- package/dist/lib/agent-kit/run/heartbeat.js +38 -6
- package/dist/lib/bot/api.d.ts +3 -1
- package/dist/lib/bot/api.js +7 -2
- package/dist/lib/bot/run.d.ts +4 -0
- package/dist/lib/bot/run.js +30 -1
- package/dist/lib/bot/status.d.ts +4 -0
- package/dist/lib/doctor/checks/bots.js +4 -1
- package/dist/lib/heartbeat-identity.d.ts +29 -0
- package/dist/lib/heartbeat-identity.js +71 -0
- package/dist/lib/lanes/constants.d.ts +3 -0
- package/dist/lib/lanes/constants.js +3 -0
- package/dist/lib/lanes/index.d.ts +1 -0
- package/dist/lib/lanes/index.js +1 -0
- package/dist/lib/lanes/quota.d.ts +6 -3
- package/dist/lib/lanes/quota.js +65 -15
- package/dist/lib/lanes/reconcile.d.ts +5 -2
- package/dist/lib/lanes/reconcile.js +94 -2
- package/dist/lib/lanes/types.d.ts +2 -0
- package/dist/lib/lanes/types.js +2 -1
- package/dist/lib/mesh/live/daemon/doctor.d.ts +6 -0
- package/dist/lib/mesh/live/daemon/doctor.js +11 -0
- package/dist/lib/mesh/live/daemon/install.d.ts +7 -0
- package/dist/lib/mesh/live/daemon/install.js +16 -6
- package/dist/lib/mesh/live/daemon/legacy-listen.d.ts +43 -0
- package/dist/lib/mesh/live/daemon/legacy-listen.js +89 -0
- package/dist/lib/mesh/live/daemon/live-cache.d.ts +26 -0
- package/dist/lib/mesh/live/daemon/live-cache.js +116 -0
- package/dist/lib/mesh/live/daemon/presence.d.ts +7 -3
- package/dist/lib/mesh/live/daemon/presence.js +10 -9
- package/dist/lib/mesh/live/daemon/run.d.ts +13 -0
- package/dist/lib/mesh/live/daemon/run.js +51 -2
- package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +31 -0
- package/dist/lib/mesh/live/daemon/work-subscribe.js +124 -0
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,40 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.134.0] — 2026-09-22
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Agent-kit and local-bot heartbeat loops now stop after the server reports an
|
|
10
|
+
agent identity mismatch instead of retrying indefinitely. `hq doctor` shows
|
|
11
|
+
the configured and token agent IDs for the mismatch.
|
|
12
|
+
|
|
13
|
+
## [5.133.0] — 2026-09-22
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- Lane follow-ups now probe all enabled, logged-in accounts for an unpinned
|
|
18
|
+
model, preserve the later quota reset and its source, expire abandoned pickup
|
|
19
|
+
validation markers with a terminal reason, and parse Codex's ordinal
|
|
20
|
+
`try again at` reset time.
|
|
21
|
+
|
|
22
|
+
- `hq sync push` no longer fails every file when the target vault was
|
|
23
|
+
provisioned on or after 2026-09-19. Those vaults carry an S3 Object Lock
|
|
24
|
+
default retention rule, and S3 refuses a PutObject into them unless the
|
|
25
|
+
request carries a content checksum; the CLI's declared `@indigoai-us/hq-cloud`
|
|
26
|
+
range still allowed a client that sends none, so an affected push reported
|
|
27
|
+
`400 InvalidRequest: Content-MD5 OR x-amz-checksum- HTTP header is required
|
|
28
|
+
for Put Object requests with Object Lock parameters` on every file. The
|
|
29
|
+
dependency floor now starts at the release that attaches the checksum, so no
|
|
30
|
+
install can resolve a client that cannot write to a locked vault. A CI guard
|
|
31
|
+
holds the floor.
|
|
32
|
+
|
|
33
|
+
- Lane creation and re-parenting now allow depth 3. A depth-3 caller and any
|
|
34
|
+
derived depth 4 lane are refused with `max_lane_depth`.
|
|
35
|
+
|
|
36
|
+
- Question ageing now derives its two-advance cap from the depth-3 lane cap, so
|
|
37
|
+
an unanswered worker-lane question reaches the user after two hourly hops.
|
|
38
|
+
|
|
5
39
|
## [5.132.0] — 2026-09-22
|
|
6
40
|
|
|
7
41
|
### Added
|
|
@@ -104,6 +138,16 @@
|
|
|
104
138
|
|
|
105
139
|
## [Unreleased]
|
|
106
140
|
|
|
141
|
+
- `hq mesh daemon install` and `hq mesh daemon run` unload and remove the
|
|
142
|
+
legacy `ai.getindigo.hq-work-mesh-listen` unit (LaunchAgent, systemd user
|
|
143
|
+
unit, or Scheduled Task) so only the daemon subscribes. The daemon writes
|
|
144
|
+
`~/.hq/work-mesh/live-cache.json` and bumps its cursor once per project-view
|
|
145
|
+
push. MQTT reconnect logs once per transition into reconnecting instead of
|
|
146
|
+
on every close. `hq mesh daemon doctor` reports "legacy listener still
|
|
147
|
+
installed" while that unit remains (US-006).
|
|
148
|
+
|
|
149
|
+
## [Unreleased]
|
|
150
|
+
|
|
107
151
|
- Held-event repair no longer treats a later event timestamp as proof that
|
|
108
152
|
metadata belongs to the current session instance. A reused session id stays
|
|
109
153
|
unresolved when the held event's binding episode does not match the current
|
|
@@ -23,6 +23,7 @@ import { type AgentKitPaths } from "../lib/agent-kit/paths.js";
|
|
|
23
23
|
import { type KitService } from "../lib/agent-kit/services.js";
|
|
24
24
|
import { type CronRunner, type RebootCronResult, type DetachedSpawn, type KillFn } from "../lib/agent-kit/fallback.js";
|
|
25
25
|
import { type ServiceHostPaths, type ServiceManagerDeps, type ServiceSetResult } from "../lib/service-manager/index.js";
|
|
26
|
+
import type { HeartbeatCredsSource } from "../lib/heartbeat-identity.js";
|
|
26
27
|
export declare class KitError extends Error {
|
|
27
28
|
constructor(message: string);
|
|
28
29
|
}
|
|
@@ -73,6 +74,7 @@ export interface KitRuntime {
|
|
|
73
74
|
config: KitConfig;
|
|
74
75
|
getToken: () => Promise<string>;
|
|
75
76
|
log: KitLogger;
|
|
77
|
+
credsSource?: HeartbeatCredsSource;
|
|
76
78
|
}
|
|
77
79
|
export declare function buildKitRuntime(service: KitService | "all"): KitRuntime;
|
|
78
80
|
export declare function runKitService(service: KitService, rt: KitRuntime): Promise<void>;
|
|
@@ -135,6 +135,7 @@ export function formatServiceSet(result) {
|
|
|
135
135
|
}
|
|
136
136
|
export function buildKitRuntime(service) {
|
|
137
137
|
const paths = agentKitPaths(os.homedir(), process.env);
|
|
138
|
+
const credsSource = process.env.HQ_MACHINE_CREDS_FILE?.trim() ? "explicit" : "default";
|
|
138
139
|
const creds = requireExternalCreds(paths);
|
|
139
140
|
ensureKitDirs(paths);
|
|
140
141
|
const config = readKitConfig(paths, CLI_VERSION);
|
|
@@ -145,7 +146,7 @@ export function buildKitRuntime(service) {
|
|
|
145
146
|
process.env.HQ_VAULT_API_URL = creds.apiBaseUrl;
|
|
146
147
|
const log = createKitLogger(paths, service, { echo: process.stdout.isTTY === true });
|
|
147
148
|
const getToken = () => ensureCognitoToken({ tokenSource: "machine", interactive: false });
|
|
148
|
-
return { paths, creds, config, getToken, log };
|
|
149
|
+
return { paths, creds, config, getToken, log, credsSource };
|
|
149
150
|
}
|
|
150
151
|
export async function runKitService(service, rt) {
|
|
151
152
|
rt.log("info", `${service} starting agent=${rt.creds.entityUid} cli=${CLI_VERSION}`);
|
|
@@ -167,6 +168,8 @@ export async function runKitService(service, rt) {
|
|
|
167
168
|
getToken: rt.getToken,
|
|
168
169
|
log: rt.log,
|
|
169
170
|
components: enabledKitComponents(rt.config),
|
|
171
|
+
agentUidSource: rt.credsSource === "explicit" ? "env" : "config_file",
|
|
172
|
+
credsSource: rt.credsSource ?? "default",
|
|
170
173
|
});
|
|
171
174
|
return;
|
|
172
175
|
case "inbox":
|
package/dist/commands/lanes.d.ts
CHANGED
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
import { Command } from "commander";
|
|
8
8
|
import { type OutpostDispatchResult } from "../lib/lanes/outpost-dispatch.js";
|
|
9
9
|
import { emitLaneDecisionEvents, type LaneMesh } from "../lib/lanes/mesh-events.js";
|
|
10
|
-
import { type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
|
|
10
|
+
import { MAX_LANE_DEPTH, type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
|
|
11
11
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
12
12
|
import { type RunCompactionFn } from "../lib/lanes/compaction.js";
|
|
13
13
|
import { type KnownModelsDeps, type ProbeCommand } from "../lib/lanes/models.js";
|
|
14
14
|
import { spawnLaneWorker, type KillProcessFn, type ProveDetachedFn, type SpawnProcessFn } from "../lib/lanes/spawn.js";
|
|
15
15
|
export { spawnLaneWorker };
|
|
16
16
|
export { questionBodyAbsPath };
|
|
17
|
-
export
|
|
17
|
+
export { MAX_LANE_DEPTH };
|
|
18
18
|
export declare const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
|
|
19
19
|
export declare const OUTPOST_DIR = "/etc/outpost";
|
|
20
20
|
export declare const LANES_RUNS_REL: string;
|
package/dist/commands/lanes.js
CHANGED
|
@@ -17,7 +17,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
|
|
|
17
17
|
import { openMeshTransport } from "../lib/mesh/client.js";
|
|
18
18
|
import { createLaneMesh, emitLaneDecisionEvents, } from "../lib/lanes/mesh-events.js";
|
|
19
19
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
20
|
-
import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
20
|
+
import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, MAX_LANE_DEPTH, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
21
21
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
22
22
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
23
23
|
import { providerCompactAfterMs } from "../lib/lanes/config.js";
|
|
@@ -29,7 +29,7 @@ import { defaultProbe, validateLaneModel, } from "../lib/lanes/models.js";
|
|
|
29
29
|
import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, stopLaneProcesses, } from "../lib/lanes/spawn.js";
|
|
30
30
|
export { spawnLaneWorker };
|
|
31
31
|
export { questionBodyAbsPath };
|
|
32
|
-
export
|
|
32
|
+
export { MAX_LANE_DEPTH };
|
|
33
33
|
export const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
|
|
34
34
|
export const OUTPOST_DIR = "/etc/outpost";
|
|
35
35
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
@@ -1092,6 +1092,7 @@ export function pickupFailureUpdater(reasonRaw, details) {
|
|
|
1092
1092
|
state: "failed",
|
|
1093
1093
|
queued_reason: undefined,
|
|
1094
1094
|
pickup_validation_pending: undefined,
|
|
1095
|
+
pickup_validation_pending_since: undefined,
|
|
1095
1096
|
terminal_reason: reason,
|
|
1096
1097
|
terminal_details: details,
|
|
1097
1098
|
};
|
|
@@ -1166,7 +1167,12 @@ export function applyOutpostAccountPickup(hqRoot, laneId, modelDeps = {}) {
|
|
|
1166
1167
|
if (current.account === resolved.id && current.pickup_validation_pending === undefined) {
|
|
1167
1168
|
return current;
|
|
1168
1169
|
}
|
|
1169
|
-
return {
|
|
1170
|
+
return {
|
|
1171
|
+
...current,
|
|
1172
|
+
account: resolved.id,
|
|
1173
|
+
pickup_validation_pending: undefined,
|
|
1174
|
+
pickup_validation_pending_since: undefined,
|
|
1175
|
+
};
|
|
1170
1176
|
});
|
|
1171
1177
|
}
|
|
1172
1178
|
catch (err) {
|
|
@@ -1315,8 +1321,8 @@ async function runCreate(ctx, opts) {
|
|
|
1315
1321
|
// creates wait on. A pinned create checks the pinned account's home. An
|
|
1316
1322
|
// unpinned create is not bound to an account here (persist re-resolves the
|
|
1317
1323
|
// balanced account under the lock, so overlapping creates still spread),
|
|
1318
|
-
// so it checks the balanced candidate first and then every other enabled
|
|
1319
|
-
// account the locked selection could land on (codex catalogs are per home
|
|
1324
|
+
// so it checks the balanced candidate first and then every other enabled,
|
|
1325
|
+
// logged-in account the locked selection could land on (codex catalogs are per home
|
|
1320
1326
|
// and can differ). `validatedAccounts` records which accepted the model
|
|
1321
1327
|
// or had no readable list, and the balanced pick under the lock is
|
|
1322
1328
|
// restricted to that set (resolveAccount's allowedAccounts): an account
|
|
@@ -1344,26 +1350,68 @@ async function runCreate(ctx, opts) {
|
|
|
1344
1350
|
: [
|
|
1345
1351
|
candidate,
|
|
1346
1352
|
...Object.keys(accounts)
|
|
1347
|
-
.filter((id) => id !== candidate &&
|
|
1353
|
+
.filter((id) => id !== candidate &&
|
|
1354
|
+
accounts[id]?.enabled &&
|
|
1355
|
+
accountStatus(provider, id, config, { existsSync: ctx.existsSync }).logged_in === true)
|
|
1348
1356
|
.sort(),
|
|
1349
1357
|
];
|
|
1358
|
+
const checkedAccounts = [];
|
|
1359
|
+
const unsupported = [];
|
|
1360
|
+
const quotaBlocked = [];
|
|
1361
|
+
const quota = loadQuotaState(hqRoot);
|
|
1350
1362
|
for (const id of toValidate) {
|
|
1351
1363
|
const home = accounts[id]?.home;
|
|
1352
1364
|
if (home === undefined)
|
|
1353
1365
|
continue;
|
|
1366
|
+
checkedAccounts.push(id);
|
|
1354
1367
|
try {
|
|
1355
1368
|
validateLaneModel(provider, cliModel, home, modelDeps);
|
|
1356
|
-
validatedAccounts.add(id);
|
|
1357
1369
|
}
|
|
1358
1370
|
catch (err) {
|
|
1359
1371
|
if (!(isLanesError(err) && err.error === "model_unsupported"))
|
|
1360
1372
|
throw err;
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1373
|
+
unsupported.push({ account: id, error: err });
|
|
1374
|
+
continue;
|
|
1375
|
+
}
|
|
1376
|
+
const block = quotaBlockFor(quota, provider, id, cliModel, now);
|
|
1377
|
+
if (block.blocked) {
|
|
1378
|
+
quotaBlocked.push({ account: id, block });
|
|
1379
|
+
continue;
|
|
1380
|
+
}
|
|
1381
|
+
validatedAccounts.add(id);
|
|
1382
|
+
}
|
|
1383
|
+
if (validatedAccounts.size === 0 && quotaBlocked.length > 0) {
|
|
1384
|
+
const blocked = quotaBlocked[0];
|
|
1385
|
+
const model = blocked.block.model ?? cliModel;
|
|
1386
|
+
const checkedSuffix = ` Checked accounts: ${checkedAccounts.join(", ")}`;
|
|
1387
|
+
if (blocked.block.reason === "account_model_unavailable") {
|
|
1388
|
+
throw new AccountResolutionError("account_model_unavailable", `Account ${JSON.stringify(blocked.account)} cannot run model ${model ?? "(no model)"}: ${blocked.block.marker ?? "needs credits"}${checkedSuffix}`, {
|
|
1389
|
+
provider,
|
|
1390
|
+
account: blocked.account,
|
|
1391
|
+
model: model ?? null,
|
|
1392
|
+
marker: blocked.block.marker ?? null,
|
|
1393
|
+
checked_accounts: checkedAccounts,
|
|
1394
|
+
});
|
|
1366
1395
|
}
|
|
1396
|
+
throw new AccountResolutionError("account_quota_exhausted", `Account ${JSON.stringify(blocked.account)} is out of quota${blocked.block.quota_reset_at ? ` until ${blocked.block.quota_reset_at}` : ""} for model ${model ?? "(no model)"}${checkedSuffix}`, {
|
|
1397
|
+
provider,
|
|
1398
|
+
account: blocked.account,
|
|
1399
|
+
model: model ?? null,
|
|
1400
|
+
quota_reset_at: blocked.block.quota_reset_at ?? null,
|
|
1401
|
+
checked_accounts: checkedAccounts,
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
if (validatedAccounts.size === 0 && unsupported.length > 0) {
|
|
1405
|
+
const first = unsupported[0].error;
|
|
1406
|
+
throw new LanesError(first.error, `${first.message} Checked accounts: ${checkedAccounts.join(", ")}`, {
|
|
1407
|
+
...first.details,
|
|
1408
|
+
checked_accounts: checkedAccounts,
|
|
1409
|
+
account_errors: unsupported.map(({ account, error }) => ({
|
|
1410
|
+
account,
|
|
1411
|
+
accepted: error.details.accepted,
|
|
1412
|
+
source: error.details.source,
|
|
1413
|
+
})),
|
|
1414
|
+
});
|
|
1367
1415
|
}
|
|
1368
1416
|
}
|
|
1369
1417
|
const threadId = company
|
|
@@ -1637,7 +1685,7 @@ function parseRemotePickupJson(stdout) {
|
|
|
1637
1685
|
}
|
|
1638
1686
|
return null;
|
|
1639
1687
|
}
|
|
1640
|
-
function ingestPickupPayload(hqRoot, payload) {
|
|
1688
|
+
function ingestPickupPayload(hqRoot, payload, now) {
|
|
1641
1689
|
const laneId = payload.lane.lane_id;
|
|
1642
1690
|
if (!laneId) {
|
|
1643
1691
|
throw new LanesError("outpost_payload_invalid", "Pickup payload lane.lane_id is required", {});
|
|
@@ -1646,9 +1694,20 @@ function ingestPickupPayload(hqRoot, payload) {
|
|
|
1646
1694
|
// The lane is stored ineligible for admission until the account bind
|
|
1647
1695
|
// clears the marker (see LaneRecord.pickup_validation_pending).
|
|
1648
1696
|
const lane = existing === null
|
|
1649
|
-
? createLane(hqRoot, {
|
|
1650
|
-
|
|
1651
|
-
|
|
1697
|
+
? createLane(hqRoot, {
|
|
1698
|
+
...payload.lane,
|
|
1699
|
+
updated_at: now.toISOString(),
|
|
1700
|
+
pickup_validation_pending: true,
|
|
1701
|
+
pickup_validation_pending_since: now.toISOString(),
|
|
1702
|
+
})
|
|
1703
|
+
: existing.state === "queued"
|
|
1704
|
+
? retryUpdateLane(hqRoot, laneId, "command", (current) => current.state === "queued"
|
|
1705
|
+
? {
|
|
1706
|
+
...current,
|
|
1707
|
+
pickup_validation_pending: true,
|
|
1708
|
+
pickup_validation_pending_since: now.toISOString(),
|
|
1709
|
+
}
|
|
1710
|
+
: current)
|
|
1652
1711
|
: existing;
|
|
1653
1712
|
const dest = runDir(hqRoot, lane.lane_id);
|
|
1654
1713
|
ensureDir(path.join(dest, "logs"));
|
|
@@ -1692,7 +1751,7 @@ async function runPickup(ctx, opts) {
|
|
|
1692
1751
|
throw new LanesError("outpost_payload_invalid", `Pickup payload is not JSON: ${err instanceof Error ? err.message : String(err)}`, {});
|
|
1693
1752
|
}
|
|
1694
1753
|
const payload = parseOutpostPickupPayload(parsedJson);
|
|
1695
|
-
const ingested = ingestPickupPayload(hqRoot, payload);
|
|
1754
|
+
const ingested = ingestPickupPayload(hqRoot, payload, ctx.now());
|
|
1696
1755
|
if (ingested.host.kind === "outpost") {
|
|
1697
1756
|
try {
|
|
1698
1757
|
refuseOutpostSelfTarget(hqRoot, ingested, {
|
|
@@ -2373,15 +2432,26 @@ async function runReparent(ctx, laneId, opts) {
|
|
|
2373
2432
|
throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", { lane_id: laneId, from_depth: oldDepth, to_depth: newDepth });
|
|
2374
2433
|
}
|
|
2375
2434
|
const override = { laneId, senior: nextSenior, depth: newDepth };
|
|
2435
|
+
const descendants = [];
|
|
2376
2436
|
for (const other of loadAllLanes(hqRoot)) {
|
|
2377
2437
|
if (other.lane_id === laneId)
|
|
2378
2438
|
continue;
|
|
2379
2439
|
if (!walkHitsLane(hqRoot, other.senior, laneId, ctx))
|
|
2380
2440
|
continue;
|
|
2381
|
-
const
|
|
2382
|
-
|
|
2383
|
-
|
|
2441
|
+
const oldDescendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx);
|
|
2442
|
+
const descendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx, override);
|
|
2443
|
+
if (descendantDepth > MAX_LANE_DEPTH) {
|
|
2444
|
+
throw new LanesError("max_lane_depth", `Re-parent would put descendant ${other.lane_id} at depth ${descendantDepth}`, { lane_id: other.lane_id, depth: descendantDepth, max: MAX_LANE_DEPTH });
|
|
2384
2445
|
}
|
|
2446
|
+
if (descendantDepth > oldDescendantDepth &&
|
|
2447
|
+
pendingQuestionsForLane(hqRoot, other.lane_id).length > 0) {
|
|
2448
|
+
throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", {
|
|
2449
|
+
lane_id: other.lane_id,
|
|
2450
|
+
from_depth: oldDescendantDepth,
|
|
2451
|
+
to_depth: descendantDepth,
|
|
2452
|
+
});
|
|
2453
|
+
}
|
|
2454
|
+
descendants.push({ laneId: other.lane_id, depth: descendantDepth });
|
|
2385
2455
|
}
|
|
2386
2456
|
const at = ctx.now().toISOString();
|
|
2387
2457
|
const updated = casUpdateLane(hqRoot, laneId, (current) => {
|
|
@@ -2401,6 +2471,13 @@ async function runReparent(ctx, laneId, opts) {
|
|
|
2401
2471
|
}),
|
|
2402
2472
|
};
|
|
2403
2473
|
});
|
|
2474
|
+
for (const descendant of descendants) {
|
|
2475
|
+
casUpdateLane(hqRoot, descendant.laneId, (current) => {
|
|
2476
|
+
if (current.depth === descendant.depth)
|
|
2477
|
+
return current;
|
|
2478
|
+
return { ...current, depth: descendant.depth };
|
|
2479
|
+
});
|
|
2480
|
+
}
|
|
2404
2481
|
for (const question of pending) {
|
|
2405
2482
|
casUpdateQuestion(hqRoot, question.question_id, (current) => {
|
|
2406
2483
|
const already = current.routing_history.some((entry) => entry.reason === "reparented" && entry.at === at);
|
|
@@ -3019,6 +3096,7 @@ async function runReconcile(ctx, opts) {
|
|
|
3019
3096
|
throttleDeps: ctx.throttleDeps,
|
|
3020
3097
|
livenessDeps: livenessDepsFromCtx(ctx),
|
|
3021
3098
|
presenceConnected: ctx.presenceConnected,
|
|
3099
|
+
mesh: ctx.mesh,
|
|
3022
3100
|
env: ctx.env,
|
|
3023
3101
|
});
|
|
3024
3102
|
printJson(summary);
|
package/dist/commands/mesh.js
CHANGED
|
@@ -1545,7 +1545,10 @@ export function registerMeshCommand(program) {
|
|
|
1545
1545
|
}
|
|
1546
1546
|
async function runDaemonRun(opts = {}) {
|
|
1547
1547
|
const hqRoot = opts.hqRoot?.trim() || undefined;
|
|
1548
|
-
const handle = await runMeshDaemon(
|
|
1548
|
+
const handle = await runMeshDaemon({
|
|
1549
|
+
...(hqRoot ? { hqRoot } : {}),
|
|
1550
|
+
enableWorkCache: true,
|
|
1551
|
+
});
|
|
1549
1552
|
// Stay alive until signal handlers call stop + exit.
|
|
1550
1553
|
await new Promise(() => {
|
|
1551
1554
|
void handle;
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* The last successful beat is recorded at state/last-heartbeat.json so
|
|
12
12
|
* `hq agent probe` can report its age.
|
|
13
13
|
*/
|
|
14
|
+
import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../../heartbeat-identity.js";
|
|
14
15
|
import { type ComponentStatus } from "../creds.js";
|
|
15
16
|
import type { KitLogger } from "../log.js";
|
|
16
17
|
import { type AgentKitPaths, type KitComponent } from "../paths.js";
|
|
@@ -36,9 +37,13 @@ export interface HeartbeatBody {
|
|
|
36
37
|
}
|
|
37
38
|
export declare function buildHeartbeatBody(paths: Pick<AgentKitPaths, "stateDir">, now?: () => Date, hostInfo?: HeartbeatBody["hostInfo"], components_?: readonly KitComponent[]): HeartbeatBody;
|
|
38
39
|
export interface LastHeartbeat {
|
|
40
|
+
/** The most recent heartbeat attempt, successful or a terminal identity mismatch. */
|
|
39
41
|
at: string;
|
|
40
42
|
components: Partial<Record<KitComponent, ComponentStatus>>;
|
|
41
43
|
status: number;
|
|
44
|
+
reason?: "identity_mismatch";
|
|
45
|
+
configuredAgentUid?: string;
|
|
46
|
+
tokenAgentUid?: string;
|
|
42
47
|
}
|
|
43
48
|
export declare function writeLastHeartbeat(paths: Pick<AgentKitPaths, "stateDir" | "lastHeartbeatPath">, record: LastHeartbeat): void;
|
|
44
49
|
export declare function readLastHeartbeat(paths: Pick<AgentKitPaths, "lastHeartbeatPath">): LastHeartbeat | null;
|
|
@@ -51,12 +56,19 @@ export interface HeartbeatDeps {
|
|
|
51
56
|
log: KitLogger;
|
|
52
57
|
/** Components this kit runs; defaults to all of them. */
|
|
53
58
|
components?: readonly KitComponent[];
|
|
54
|
-
post?: (token: string, body: HeartbeatBody) => Promise<number>;
|
|
59
|
+
post?: (token: string, body: HeartbeatBody) => Promise<number | HeartbeatPostResult>;
|
|
55
60
|
now?: () => Date;
|
|
56
61
|
sleep?: (ms: number) => Promise<void>;
|
|
57
62
|
maxBeats?: number;
|
|
63
|
+
agentUidSource?: HeartbeatAgentUidSource;
|
|
64
|
+
credsSource?: HeartbeatCredsSource;
|
|
65
|
+
reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
|
|
58
66
|
}
|
|
59
|
-
export
|
|
67
|
+
export interface HeartbeatPostResult {
|
|
68
|
+
status: number;
|
|
69
|
+
bodyText?: string;
|
|
70
|
+
}
|
|
71
|
+
export declare function defaultHeartbeatPost(agentUid: string, apiBaseUrl: string): (token: string, body: HeartbeatBody) => Promise<HeartbeatPostResult>;
|
|
60
72
|
export declare function beatOnce(deps: HeartbeatDeps): Promise<boolean>;
|
|
61
73
|
export declare function runHeartbeatLoop(deps: HeartbeatDeps): Promise<void>;
|
|
62
74
|
//# sourceMappingURL=heartbeat.d.ts.map
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import * as fs from "node:fs";
|
|
15
15
|
import * as os from "node:os";
|
|
16
16
|
import { CLI_VERSION } from "../../../cli-version.js";
|
|
17
|
+
import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, parseHeartbeatFailureBody, tokenAgentUid, } from "../../heartbeat-identity.js";
|
|
17
18
|
import { vaultApiFetch } from "../../../utils/vault-api.js";
|
|
18
19
|
import { readComponentStatus } from "../creds.js";
|
|
19
20
|
import { KIT_COMPONENTS } from "../paths.js";
|
|
@@ -60,36 +61,67 @@ export function defaultHeartbeatPost(agentUid, apiBaseUrl) {
|
|
|
60
61
|
method: "POST",
|
|
61
62
|
body: body,
|
|
62
63
|
});
|
|
63
|
-
return res.status;
|
|
64
|
+
return { status: res.status, bodyText: await res.text().catch(() => "") };
|
|
64
65
|
};
|
|
65
66
|
}
|
|
66
|
-
|
|
67
|
+
async function beatOnceResult(deps) {
|
|
67
68
|
const now = deps.now ?? (() => new Date());
|
|
68
69
|
const post = deps.post ?? defaultHeartbeatPost(deps.agentUid, deps.apiBaseUrl);
|
|
69
70
|
const body = buildHeartbeatBody(deps.paths, now, undefined, deps.components ?? KIT_COMPONENTS);
|
|
70
71
|
try {
|
|
71
72
|
const token = await deps.getToken();
|
|
72
|
-
const
|
|
73
|
+
const response = await post(token, body);
|
|
74
|
+
const result = typeof response === "number" ? { status: response } : response;
|
|
75
|
+
const status = result.status;
|
|
73
76
|
const ok = status >= 200 && status < 300;
|
|
74
77
|
if (ok) {
|
|
75
78
|
writeLastHeartbeat(deps.paths, { at: now().toISOString(), components: body.components, status });
|
|
76
79
|
}
|
|
80
|
+
else {
|
|
81
|
+
const failure = parseHeartbeatFailureBody(result.bodyText);
|
|
82
|
+
if (isHeartbeatIdentityMismatch({ status, code: failure.code, message: failure.message })) {
|
|
83
|
+
const tokenUid = tokenAgentUid(token);
|
|
84
|
+
writeLastHeartbeat(deps.paths, {
|
|
85
|
+
at: now().toISOString(),
|
|
86
|
+
components: body.components,
|
|
87
|
+
status,
|
|
88
|
+
reason: "identity_mismatch",
|
|
89
|
+
configuredAgentUid: deps.agentUid,
|
|
90
|
+
...(tokenUid ? { tokenAgentUid: tokenUid } : {}),
|
|
91
|
+
});
|
|
92
|
+
const tokenLabel = tokenUid ?? "unknown";
|
|
93
|
+
deps.log("warn", `heartbeat stopped: identity mismatch configured agentUid=${deps.agentUid} token agentUid=${tokenLabel}`);
|
|
94
|
+
(deps.reportIdentityMismatch ?? captureHeartbeatIdentityMismatch)({
|
|
95
|
+
runner: "agent-kit",
|
|
96
|
+
configuredAgentUid: deps.agentUid,
|
|
97
|
+
...(tokenUid ? { tokenAgentUid: tokenUid } : {}),
|
|
98
|
+
agentUidSource: deps.agentUidSource ?? "config_file",
|
|
99
|
+
credsSource: deps.credsSource ?? "default",
|
|
100
|
+
});
|
|
101
|
+
return { ok: false, stop: true };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
77
104
|
deps.log(ok ? "info" : "warn", `heartbeat ${status} ${Object.entries(body.components)
|
|
78
105
|
.map(([name, health]) => `${name}=${health}`)
|
|
79
106
|
.join(" ")}`);
|
|
80
|
-
return ok;
|
|
107
|
+
return { ok, stop: false };
|
|
81
108
|
}
|
|
82
109
|
catch (err) {
|
|
83
110
|
deps.log("error", `heartbeat failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
84
|
-
return false;
|
|
111
|
+
return { ok: false, stop: false };
|
|
85
112
|
}
|
|
86
113
|
}
|
|
114
|
+
export async function beatOnce(deps) {
|
|
115
|
+
return (await beatOnceResult(deps)).ok;
|
|
116
|
+
}
|
|
87
117
|
export async function runHeartbeatLoop(deps) {
|
|
88
118
|
const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
89
119
|
let beats = 0;
|
|
90
120
|
for (;;) {
|
|
91
|
-
await
|
|
121
|
+
const result = await beatOnceResult(deps);
|
|
92
122
|
beats += 1;
|
|
123
|
+
if (result.stop)
|
|
124
|
+
return;
|
|
93
125
|
if (deps.maxBeats !== undefined && beats >= deps.maxBeats)
|
|
94
126
|
return;
|
|
95
127
|
await sleep(deps.intervalMs);
|
package/dist/lib/bot/api.d.ts
CHANGED
|
@@ -85,8 +85,10 @@ export declare class BotApiError extends Error {
|
|
|
85
85
|
readonly status: number;
|
|
86
86
|
readonly path: string;
|
|
87
87
|
readonly code?: string | undefined;
|
|
88
|
+
readonly detail?: string | undefined;
|
|
88
89
|
readonly retryAfterMs?: number | undefined;
|
|
89
|
-
|
|
90
|
+
readonly tokenAgentUid?: string | undefined;
|
|
91
|
+
constructor(status: number, path: string, code?: string | undefined, detail?: string | undefined, retryAfterMs?: number | undefined, tokenAgentUid?: string | undefined);
|
|
90
92
|
}
|
|
91
93
|
/** Parse a Retry-After header (delta-seconds or HTTP-date) to milliseconds. */
|
|
92
94
|
export declare function retryAfterHeaderMs(value: string | null | undefined, nowMs?: number): number | undefined;
|
package/dist/lib/bot/api.js
CHANGED
|
@@ -6,18 +6,23 @@
|
|
|
6
6
|
* existing Cognito machinery. `fetch` is injectable for tests.
|
|
7
7
|
*/
|
|
8
8
|
import { vaultApiFetch } from "../../utils/vault-api.js";
|
|
9
|
+
import { tokenAgentUid } from "../heartbeat-identity.js";
|
|
9
10
|
import { assertNoSecretKeys } from "./local-config.js";
|
|
10
11
|
export class BotApiError extends Error {
|
|
11
12
|
status;
|
|
12
13
|
path;
|
|
13
14
|
code;
|
|
15
|
+
detail;
|
|
14
16
|
retryAfterMs;
|
|
15
|
-
|
|
17
|
+
tokenAgentUid;
|
|
18
|
+
constructor(status, path, code, detail, retryAfterMs, tokenAgentUid) {
|
|
16
19
|
super(`HQ API ${path} → ${status}${code ? ` ${code}` : ""}${detail ? `: ${detail}` : ""}`);
|
|
17
20
|
this.status = status;
|
|
18
21
|
this.path = path;
|
|
19
22
|
this.code = code;
|
|
23
|
+
this.detail = detail;
|
|
20
24
|
this.retryAfterMs = retryAfterMs;
|
|
25
|
+
this.tokenAgentUid = tokenAgentUid;
|
|
21
26
|
this.name = "BotApiError";
|
|
22
27
|
}
|
|
23
28
|
}
|
|
@@ -72,7 +77,7 @@ export class BotApi {
|
|
|
72
77
|
});
|
|
73
78
|
if (!accept.includes(res.status)) {
|
|
74
79
|
const err = await readError(res);
|
|
75
|
-
throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")));
|
|
80
|
+
throw new BotApiError(res.status, path, err.code, err.detail, retryAfterHeaderMs(res.headers.get("Retry-After")), path.endsWith("/heartbeat") ? tokenAgentUid(token) : undefined);
|
|
76
81
|
}
|
|
77
82
|
const text = await res.text();
|
|
78
83
|
return { status: res.status, body: (text ? JSON.parse(text) : {}) };
|
package/dist/lib/bot/run.d.ts
CHANGED
|
@@ -42,6 +42,7 @@ import type { BotConfig, BotKind } from "./config.js";
|
|
|
42
42
|
import { type ProgressPosterOptions } from "./progress.js";
|
|
43
43
|
import { type BotLogger } from "./log.js";
|
|
44
44
|
import { type BotRuntime, type RuntimeTurnInput, type RuntimeTurnResult } from "./runtime/index.js";
|
|
45
|
+
import { type HeartbeatAgentUidSource, type HeartbeatCredsSource, type HeartbeatIdentityMismatchReporter } from "../heartbeat-identity.js";
|
|
45
46
|
import { type BotState, type BotStatusFile } from "./status.js";
|
|
46
47
|
export declare const INBOX_POLL_INTERVAL_MS = 2000;
|
|
47
48
|
export declare const HEARTBEAT_INTERVAL_MS = 30000;
|
|
@@ -119,6 +120,9 @@ export interface BotRunDeps {
|
|
|
119
120
|
maxParallelTurns?: number;
|
|
120
121
|
/** Wait between retries while the coding tool needs a sign-in (default SIGN_IN_RETRY_MS). */
|
|
121
122
|
signInRetryMs?: number;
|
|
123
|
+
agentUidSource?: HeartbeatAgentUidSource;
|
|
124
|
+
credsSource?: HeartbeatCredsSource;
|
|
125
|
+
reportIdentityMismatch?: HeartbeatIdentityMismatchReporter;
|
|
122
126
|
}
|
|
123
127
|
export declare const KICKOFF_MESSAGE_ID = "kickoff";
|
|
124
128
|
/**
|
package/dist/lib/bot/run.js
CHANGED
|
@@ -59,6 +59,7 @@ import { dmSessionScope } from "./paths.js";
|
|
|
59
59
|
import { isDmItem, resolveReplyTarget } from "./room-policy.js";
|
|
60
60
|
import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
|
|
61
61
|
import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
|
|
62
|
+
import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
|
|
62
63
|
import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
|
|
63
64
|
import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
|
|
64
65
|
export const INBOX_POLL_INTERVAL_MS = 2_000;
|
|
@@ -415,10 +416,38 @@ export async function runBot(deps) {
|
|
|
415
416
|
// runtime → model, inbox → task, sync → sync.
|
|
416
417
|
await api.heartbeat(config.agentUid, { model: modelHealth, task: "ok", sync: "ok" });
|
|
417
418
|
heartbeatFails = 0;
|
|
418
|
-
setStatus({
|
|
419
|
+
setStatus({
|
|
420
|
+
lastHeartbeatAt: now().toISOString(),
|
|
421
|
+
lastHeartbeatOk: true,
|
|
422
|
+
lastHeartbeatError: undefined,
|
|
423
|
+
lastHeartbeatReason: undefined,
|
|
424
|
+
lastHeartbeatConfiguredAgentUid: undefined,
|
|
425
|
+
lastHeartbeatTokenAgentUid: undefined,
|
|
426
|
+
});
|
|
419
427
|
await sleep(heartbeatIntervalMs);
|
|
420
428
|
}
|
|
421
429
|
catch (err) {
|
|
430
|
+
if (err instanceof BotApiError &&
|
|
431
|
+
isHeartbeatIdentityMismatch({ status: err.status, code: err.code, message: err.detail })) {
|
|
432
|
+
const tokenLabel = err.tokenAgentUid ?? "unknown";
|
|
433
|
+
const reason = `heartbeat stopped: identity mismatch configured agentUid=${config.agentUid} token agentUid=${tokenLabel}`;
|
|
434
|
+
setStatus({
|
|
435
|
+
lastHeartbeatOk: false,
|
|
436
|
+
lastHeartbeatError: reason.slice(0, 400),
|
|
437
|
+
lastHeartbeatReason: "identity_mismatch",
|
|
438
|
+
lastHeartbeatConfiguredAgentUid: config.agentUid,
|
|
439
|
+
...(err.tokenAgentUid ? { lastHeartbeatTokenAgentUid: err.tokenAgentUid } : {}),
|
|
440
|
+
});
|
|
441
|
+
log("warn", reason);
|
|
442
|
+
(deps.reportIdentityMismatch ?? captureHeartbeatIdentityMismatch)({
|
|
443
|
+
runner: "bot",
|
|
444
|
+
configuredAgentUid: config.agentUid,
|
|
445
|
+
...(err.tokenAgentUid ? { tokenAgentUid: err.tokenAgentUid } : {}),
|
|
446
|
+
agentUidSource: deps.agentUidSource ?? "config_file",
|
|
447
|
+
credsSource: deps.credsSource ?? "default",
|
|
448
|
+
});
|
|
449
|
+
break;
|
|
450
|
+
}
|
|
422
451
|
heartbeatFails += 1;
|
|
423
452
|
const message = err instanceof Error ? err.message : String(err);
|
|
424
453
|
setStatus({ lastHeartbeatOk: false, lastHeartbeatError: message.slice(0, 400) });
|
package/dist/lib/bot/status.d.ts
CHANGED
|
@@ -19,6 +19,10 @@ export interface BotStatusFile {
|
|
|
19
19
|
lastHeartbeatOk?: boolean;
|
|
20
20
|
/** Last heartbeat error, when lastHeartbeatOk is false. */
|
|
21
21
|
lastHeartbeatError?: string;
|
|
22
|
+
/** A heartbeat stopped permanently until the bot is restarted with corrected identity config. */
|
|
23
|
+
lastHeartbeatReason?: "identity_mismatch";
|
|
24
|
+
lastHeartbeatConfiguredAgentUid?: string;
|
|
25
|
+
lastHeartbeatTokenAgentUid?: string;
|
|
22
26
|
lastInboxPollAt?: string;
|
|
23
27
|
lastInboxPollOk?: boolean;
|
|
24
28
|
/** Last inbox poll error, when lastInboxPollOk is false. */
|
|
@@ -76,11 +76,14 @@ function runChecks(deps) {
|
|
|
76
76
|
const status = readBotStatus(dir);
|
|
77
77
|
const alive = !!status && status.state === "running" && isPidAlive(status.pid);
|
|
78
78
|
if (alive && status.lastHeartbeatOk === false) {
|
|
79
|
+
const heartbeatMessage = status.lastHeartbeatReason === "identity_mismatch"
|
|
80
|
+
? `${name} heartbeat stopped: identity mismatch (configured agent ${status.lastHeartbeatConfiguredAgentUid ?? config.agentUid}; token agent ${status.lastHeartbeatTokenAgentUid ?? "unknown"}).`
|
|
81
|
+
: `${name} is running but cannot heartbeat to HQ${status.lastHeartbeatError ? ` (${status.lastHeartbeatError})` : ""}.`;
|
|
79
82
|
results.push({
|
|
80
83
|
status: "FAIL",
|
|
81
84
|
checkId: `bots.${name}.heartbeat`,
|
|
82
85
|
target: dir,
|
|
83
|
-
message:
|
|
86
|
+
message: heartbeatMessage,
|
|
84
87
|
remediation: `hq bot logs ${name}`,
|
|
85
88
|
});
|
|
86
89
|
}
|