@bli-cockpit/cli 0.2.45 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-git-diff.js +50 -0
- package/dist/adapters/raw-evidence-keys.js +4 -1
- package/dist/adapters/raw-evidence-pack-store.js +13 -4
- package/dist/adapters/raw-evidence.js +43 -1
- package/dist/autostart-node-path.js +141 -0
- package/dist/autostart-self-heal.js +115 -11
- package/dist/autostart.js +213 -41
- package/dist/commands/autostart-heal.js +162 -0
- package/dist/commands/brief.js +95 -1
- package/dist/commands/collection-roots.js +4 -4
- package/dist/commands/doctor.js +47 -1
- package/dist/commands/heartbeat.js +175 -0
- package/dist/commands/install-receipts.js +11 -19
- package/dist/commands/install-update.js +3 -3
- package/dist/commands/jarvis.js +5 -0
- package/dist/commands/local-args.js +70 -4
- package/dist/commands/local-help.js +13 -4
- package/dist/commands/local.js +116 -10
- package/dist/commands/ops-render.js +29 -0
- package/dist/commands/ops.js +6 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +146 -25
- package/dist/evidence-upload-client.js +112 -4
- package/dist/evidence-upload-rekey.js +40 -0
- package/dist/log-rotation.js +144 -0
- package/dist/onboarding-roots.js +23 -6
- package/dist/raw-evidence-gc.js +9 -23
- package/dist/scheduled-self-update.js +1 -0
- package/dist/second-install.js +160 -0
- package/dist/sync-health-class.js +242 -0
- package/dist/upload.js +2 -0
- package/package.json +2 -2
package/dist/commands/local.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* local-help.ts what --help prints, and the recognised command names
|
|
10
10
|
* cli-io.ts stdout/stderr/stdin plumbing and the production CliIo
|
|
11
11
|
* install-receipts.ts the named step results this machine reports back
|
|
12
|
+
* heartbeat.ts the per-tick "I am alive, and these are my roots"
|
|
12
13
|
* local-auth.ts which email, the OTP exchange, pairing with fallback
|
|
13
14
|
* collection-roots.ts which folders may be collected, saved and read back
|
|
14
15
|
* local-discovery.ts finding git worktrees inside those roots
|
|
@@ -36,6 +37,8 @@
|
|
|
36
37
|
* settings.ts `cockpit settings` and `cockpit model`
|
|
37
38
|
* settings-render.ts how a settings answer reads in a terminal
|
|
38
39
|
* team.ts `cockpit team` — members, invite, role
|
|
40
|
+
* autostart-heal.ts the internal detached one-shot the macOS tick spawns
|
|
41
|
+
* to re-register launchd after the tick itself exits
|
|
39
42
|
*
|
|
40
43
|
* What stays here is orchestration: onboard, login/logout/start, the sync tick,
|
|
41
44
|
* analyze, serve, autostart and agent-rules.
|
|
@@ -44,7 +47,8 @@ import path from "node:path";
|
|
|
44
47
|
import { bufferedWritable, defaultExec, defaultIo, errorMessage, parseCapturedJson, replayCaptured, writeLine, } from "./cli-io.js";
|
|
45
48
|
import { isLocalHelpRequest, localCommandHelp } from "./local-help.js";
|
|
46
49
|
import { describeError, isMissingFileFailure } from "../health-detail.js";
|
|
47
|
-
import {
|
|
50
|
+
import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
51
|
+
import { addInstallEvent, classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
48
52
|
import { canReuseOnboardSession, pairLocalCollectorWithAuthFallback, readOnboardSessionReuseCandidate, requestPairingAccessToken, requestPairingAccessTokenDetailed, resolveInteractiveLoginEmail, resolveOnboardEmail, } from "./local-auth.js";
|
|
49
53
|
import { collectionRootConsentAliases, persistOnboardingRootConfig, resolveOnboardingRootsForCommand, } from "./collection-roots.js";
|
|
50
54
|
import { discoverCommandWorktrees, rememberDiscoveryLimits } from "./local-discovery.js";
|
|
@@ -75,16 +79,19 @@ import { autostartStatus, installAutostartAgent, uninstallAutostartAgent, } from
|
|
|
75
79
|
import { DEFAULT_DASHBOARD_URL, ensureLocalCollectorConfig, getCollectorRuntimePaths, inspectLocalCollectorStatus, logoutLocalCollector, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, startLocalWorkContext, } from "../local-state.js";
|
|
76
80
|
import { acquireSyncLock } from "../sync-lock.js";
|
|
77
81
|
import { runAttributedWorktreeSync, } from "./session-sync.js";
|
|
78
|
-
import { COLLECTION_ROOT_REQUIRED, missingCollectionRootMessage, } from "../onboarding-roots.js";
|
|
82
|
+
import { COLLECTION_ROOT_REQUIRED, CollectionRootRequiredError, missingCollectionRootMessage, } from "../onboarding-roots.js";
|
|
79
83
|
import { rawEvidenceDedupSummary, rawEvidenceGcSummary, runRawEvidenceLocalGc, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
|
|
80
84
|
import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
|
|
81
85
|
import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
|
|
86
|
+
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
87
|
+
import { runAutostartHealDetached } from "./autostart-heal.js";
|
|
82
88
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
83
89
|
// `./local.js` is the published entry point for this command surface: the
|
|
84
90
|
// public CLI's generated root, commands/root.ts, doctor.ts and the test suite
|
|
85
91
|
// all import from here. Splitting the file must not move a name off it.
|
|
86
92
|
export { rootCommandNames, localCommandHelp } from "./local-help.js";
|
|
87
|
-
export {
|
|
93
|
+
export { buildCollectorHeartbeat, collectionRootLabel, collectionRootLabels, sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
|
|
94
|
+
export { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, SYNC_ERROR_DETAIL_MAX_CHARS, } from "./install-receipts.js";
|
|
88
95
|
export { assertCollectionRootPersisted } from "./collection-roots.js";
|
|
89
96
|
export { runSelfUpdate, SelfUpdateError, } from "./install-update.js";
|
|
90
97
|
export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
@@ -1091,6 +1098,11 @@ async function runStart(command, io) {
|
|
|
1091
1098
|
}
|
|
1092
1099
|
async function runSync(command, io) {
|
|
1093
1100
|
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
1101
|
+
// BLI-3553, first thing in the tick: cap the scheduler's own logs. Nothing
|
|
1102
|
+
// rotated them before, and two months of a 15-minute tick left 211 MB of
|
|
1103
|
+
// sync.log on the reference Mac. Best-effort by construction — a rotation
|
|
1104
|
+
// problem is its own log line, never a reason collection does not run.
|
|
1105
|
+
await rotateCollectorLogsBestEffort(paths);
|
|
1094
1106
|
const config = await readLocalCollectorConfig(paths).catch(() => null);
|
|
1095
1107
|
const dashboardUrl = command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
|
|
1096
1108
|
const minCliVersionAtStart = await reportInstallEventsBestEffort({
|
|
@@ -1103,6 +1115,10 @@ async function runSync(command, io) {
|
|
|
1103
1115
|
});
|
|
1104
1116
|
try {
|
|
1105
1117
|
const result = await runSyncWithHealthReceipt(command, io);
|
|
1118
|
+
// BLI-3551: every tick checks in, including one that collected nothing.
|
|
1119
|
+
// This is the only writer of `last_seen_at` that does not need an envelope,
|
|
1120
|
+
// so it is what separates a quiet machine from a dead one.
|
|
1121
|
+
await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
|
|
1106
1122
|
const minCliVersion = await reportInstallEventsBestEffort({
|
|
1107
1123
|
homeDir: command.homeDir,
|
|
1108
1124
|
dashboardUrl,
|
|
@@ -1118,6 +1134,14 @@ async function runSync(command, io) {
|
|
|
1118
1134
|
return result.exitCode;
|
|
1119
1135
|
}
|
|
1120
1136
|
catch (error) {
|
|
1137
|
+
const errorCode = classifySyncHealthError(error);
|
|
1138
|
+
// A machine whose sync THREW is still alive, and that is worth knowing —
|
|
1139
|
+
// a device that stops checking in entirely is a different problem from one
|
|
1140
|
+
// checking in with a failure every fifteen minutes.
|
|
1141
|
+
await sendSyncHeartbeat(command, io, dashboardUrl, {
|
|
1142
|
+
status: "fail",
|
|
1143
|
+
reason: errorCode,
|
|
1144
|
+
});
|
|
1121
1145
|
const minCliVersion = await reportInstallEventsBestEffort({
|
|
1122
1146
|
homeDir: command.homeDir,
|
|
1123
1147
|
dashboardUrl,
|
|
@@ -1126,7 +1150,7 @@ async function runSync(command, io) {
|
|
|
1126
1150
|
{
|
|
1127
1151
|
step: "sync_complete",
|
|
1128
1152
|
status: "fail",
|
|
1129
|
-
error_code:
|
|
1153
|
+
error_code: errorCode,
|
|
1130
1154
|
error_detail: redactedSyncErrorDetail(error),
|
|
1131
1155
|
},
|
|
1132
1156
|
],
|
|
@@ -1138,6 +1162,28 @@ async function runSync(command, io) {
|
|
|
1138
1162
|
throw error;
|
|
1139
1163
|
}
|
|
1140
1164
|
}
|
|
1165
|
+
/**
|
|
1166
|
+
* The tick's check-in (BLI-3551).
|
|
1167
|
+
*
|
|
1168
|
+
* Resolving the roots is best-effort on purpose: a machine with NO approved
|
|
1169
|
+
* root is exactly the machine whose silence needs explaining, so it still
|
|
1170
|
+
* checks in — with an empty root list, which is itself the finding.
|
|
1171
|
+
*/
|
|
1172
|
+
async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
|
|
1173
|
+
const roots = await resolveSyncCollectionRoots(command).catch(() => []);
|
|
1174
|
+
await sendCollectorHeartbeatBestEffort({
|
|
1175
|
+
homeDir: command.homeDir,
|
|
1176
|
+
dashboardUrl,
|
|
1177
|
+
roots,
|
|
1178
|
+
facts,
|
|
1179
|
+
io,
|
|
1180
|
+
}).catch((error) => {
|
|
1181
|
+
// The sender already swallows everything it knows about; this is the net
|
|
1182
|
+
// for anything it does not, because a heartbeat must never fail a sync.
|
|
1183
|
+
console.error("[heartbeat] the check-in threw and was dropped", JSON.stringify({ reason: "heartbeat_threw", ...describeError(error) }));
|
|
1184
|
+
return false;
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1141
1187
|
/**
|
|
1142
1188
|
* BLI-2721: after the tick's collection and self-update are done and
|
|
1143
1189
|
* reported, repair a broken/legacy autostart registration in place (Windows
|
|
@@ -1199,7 +1245,10 @@ async function reportAutostartSelfHealOutcome(command, io, dashboardUrl, result)
|
|
|
1199
1245
|
command: "sync",
|
|
1200
1246
|
events: [
|
|
1201
1247
|
{
|
|
1202
|
-
|
|
1248
|
+
// Windows repairs in place and keeps the name already in the receipts
|
|
1249
|
+
// and the runbook; the macOS path only SCHEDULES a detached repair, so
|
|
1250
|
+
// it reports under its own step (BLI-3553).
|
|
1251
|
+
step: result.step ?? "autostart_repair",
|
|
1203
1252
|
status: result.status,
|
|
1204
1253
|
...(result.status === "ok" ? {} : { error_code: result.reason }),
|
|
1205
1254
|
...(result.detail ? { error_detail: result.detail } : {}),
|
|
@@ -1298,10 +1347,24 @@ function scheduledSelfUpdateInstallEvent(result) {
|
|
|
1298
1347
|
? `forced_min_version ${result.min_version}`
|
|
1299
1348
|
: null;
|
|
1300
1349
|
if (result.status === "ok") {
|
|
1350
|
+
// BLI-3551: this used to be `update ok` with an empty detail unless the
|
|
1351
|
+
// floor forced it. One machine posted that receipt daily for nine releases
|
|
1352
|
+
// while sitting on 0.2.37, and nobody could tell "already current" from
|
|
1353
|
+
// "installed something" from "npm answered nothing" — three different
|
|
1354
|
+
// situations wearing one word. The success branch names itself now.
|
|
1355
|
+
const okDetail = [
|
|
1356
|
+
forcedDetail,
|
|
1357
|
+
result.reason === "updated" && result.previous_version && result.installed_version
|
|
1358
|
+
? `installed ${result.previous_version}→${result.installed_version}`
|
|
1359
|
+
: result.reason,
|
|
1360
|
+
result.target_version ? `target ${result.target_version}` : null,
|
|
1361
|
+
]
|
|
1362
|
+
.filter((part) => Boolean(part))
|
|
1363
|
+
.join("; ");
|
|
1301
1364
|
return {
|
|
1302
1365
|
step: "update",
|
|
1303
1366
|
status: "ok",
|
|
1304
|
-
...(
|
|
1367
|
+
...(okDetail ? { error_detail: okDetail } : {}),
|
|
1305
1368
|
};
|
|
1306
1369
|
}
|
|
1307
1370
|
const detail = [
|
|
@@ -1341,6 +1404,10 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
1341
1404
|
status: "skipped",
|
|
1342
1405
|
error_code: "live_sync_paused_during_backfill",
|
|
1343
1406
|
},
|
|
1407
|
+
heartbeat: {
|
|
1408
|
+
status: "skipped",
|
|
1409
|
+
reason: "live_sync_paused_during_backfill",
|
|
1410
|
+
},
|
|
1344
1411
|
};
|
|
1345
1412
|
}
|
|
1346
1413
|
// Single-flight: a launchd timer and a manual sync must not interleave the
|
|
@@ -1367,14 +1434,28 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
1367
1434
|
status: "skipped",
|
|
1368
1435
|
error_code: "sync_already_running",
|
|
1369
1436
|
},
|
|
1437
|
+
heartbeat: { status: "skipped", reason: "sync_already_running" },
|
|
1370
1438
|
};
|
|
1371
1439
|
}
|
|
1372
1440
|
try {
|
|
1373
|
-
const { exitCode, failureReasons } = await runSyncLocked(command, io);
|
|
1441
|
+
const { exitCode, failureReasons, failureRecords, notice, sessionsObserved, sessionsOutsideRoot, } = await runSyncLocked(command, io);
|
|
1442
|
+
const counts = {
|
|
1443
|
+
sessionsObserved,
|
|
1444
|
+
sessionsOutsideRoot,
|
|
1445
|
+
};
|
|
1374
1446
|
if (exitCode === 0) {
|
|
1447
|
+
// BLI-3551: an `ok` tick can still have something to say. `nothing_in_root`
|
|
1448
|
+
// is the receipt that separates "this machine is alive and its operator
|
|
1449
|
+
// works outside the approved roots" from "this machine is dead", which
|
|
1450
|
+
// until now looked identical from the dashboard.
|
|
1375
1451
|
return {
|
|
1376
1452
|
exitCode,
|
|
1377
|
-
completion: {
|
|
1453
|
+
completion: {
|
|
1454
|
+
step: "sync_complete",
|
|
1455
|
+
status: "ok",
|
|
1456
|
+
...(notice ? { error_detail: notice } : {}),
|
|
1457
|
+
},
|
|
1458
|
+
heartbeat: { status: "ok", reason: notice, ...counts },
|
|
1378
1459
|
};
|
|
1379
1460
|
}
|
|
1380
1461
|
// A sync that fails by exit code says exactly as much as one that throws.
|
|
@@ -1382,14 +1463,18 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
1382
1463
|
// failure rows carried a null detail and the real reason was reachable only
|
|
1383
1464
|
// by running `cockpit status` on the machine itself (BLI-2526).
|
|
1384
1465
|
const reasonText = failureReasons.join("; ");
|
|
1466
|
+
// The bucket comes from the records the deciding branches wrote, not from
|
|
1467
|
+
// this sentence (BLI-3551). The sentence is still the detail.
|
|
1468
|
+
const errorCode = classifySyncFailureRecords(failureRecords);
|
|
1385
1469
|
return {
|
|
1386
1470
|
exitCode,
|
|
1387
1471
|
completion: {
|
|
1388
1472
|
step: "sync_complete",
|
|
1389
1473
|
status: "fail",
|
|
1390
|
-
error_code:
|
|
1474
|
+
error_code: errorCode,
|
|
1391
1475
|
error_detail: redactedSyncErrorDetail(reasonText),
|
|
1392
1476
|
},
|
|
1477
|
+
heartbeat: { status: "fail", reason: errorCode, ...counts },
|
|
1393
1478
|
};
|
|
1394
1479
|
}
|
|
1395
1480
|
finally {
|
|
@@ -1407,6 +1492,10 @@ function syncResult(run) {
|
|
|
1407
1492
|
return {
|
|
1408
1493
|
exitCode: run.ok ? 0 : 1,
|
|
1409
1494
|
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
1495
|
+
failureRecords: run.ok ? [] : run.failure_records,
|
|
1496
|
+
notice: run.notice,
|
|
1497
|
+
sessionsObserved: run.sessions_observed,
|
|
1498
|
+
sessionsOutsideRoot: run.sessions_outside_root,
|
|
1410
1499
|
};
|
|
1411
1500
|
}
|
|
1412
1501
|
/**
|
|
@@ -1492,6 +1581,7 @@ async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
1492
1581
|
mode: "no_worktrees",
|
|
1493
1582
|
status: collectionRunStatus,
|
|
1494
1583
|
collection_complete: run.ok,
|
|
1584
|
+
...(run.notice ? { notice: run.notice } : {}),
|
|
1495
1585
|
codex_sessions: run.summary,
|
|
1496
1586
|
raw_evidence_gc: gc,
|
|
1497
1587
|
raw_evidence_dedup: dedup,
|
|
@@ -1499,6 +1589,11 @@ async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
1499
1589
|
return syncResult(run);
|
|
1500
1590
|
}
|
|
1501
1591
|
writeLine(run.ok ? io.stdout : io.stderr, `Tower sync ${collectionRunStatus}: no git worktrees under this root; session scan ran.`);
|
|
1592
|
+
if (run.notice) {
|
|
1593
|
+
// Says out loud what the receipt now says to the dashboard: the sessions
|
|
1594
|
+
// this machine ran were all outside the folders it is allowed to look at.
|
|
1595
|
+
writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). Nothing was collected, and nothing is broken.`);
|
|
1596
|
+
}
|
|
1502
1597
|
writeAgentSessionSummary(io, run.summary);
|
|
1503
1598
|
if (gc && !gc.skipped)
|
|
1504
1599
|
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
@@ -1558,7 +1653,7 @@ async function resolveSyncCollectionRoots(command) {
|
|
|
1558
1653
|
if (savedRoots.length > 0) {
|
|
1559
1654
|
return collectionRootConsentAliases(savedRoots);
|
|
1560
1655
|
}
|
|
1561
|
-
throw new
|
|
1656
|
+
throw new CollectionRootRequiredError(`no explicit or saved collection root is available.`);
|
|
1562
1657
|
}
|
|
1563
1658
|
async function runAnalyze(command, io) {
|
|
1564
1659
|
const syncStdout = [];
|
|
@@ -1701,6 +1796,17 @@ async function runServe(command, io) {
|
|
|
1701
1796
|
return 0;
|
|
1702
1797
|
}
|
|
1703
1798
|
async function runAutostart(command, io) {
|
|
1799
|
+
if (command.action === "heal-detached") {
|
|
1800
|
+
// BLI-3553: the internal one-shot the scheduled macOS tick spawns. It has
|
|
1801
|
+
// its own receipt and its own exit code; nothing else in this function
|
|
1802
|
+
// applies to it.
|
|
1803
|
+
return runAutostartHealDetached({
|
|
1804
|
+
homeDir: command.homeDir,
|
|
1805
|
+
dashboardUrl: command.dashboardUrl,
|
|
1806
|
+
parentPid: command.parentPid ?? null,
|
|
1807
|
+
json: command.json,
|
|
1808
|
+
}, io);
|
|
1809
|
+
}
|
|
1704
1810
|
const exec = io.exec ?? defaultExec();
|
|
1705
1811
|
const repoRoots = command.action === "install" || command.action === "status"
|
|
1706
1812
|
? await resolveAutostartRoots(command.homeDir, command.repoRoot)
|
|
@@ -80,6 +80,8 @@ export function renderOpsStatus(payload, dim) {
|
|
|
80
80
|
lines.push(dim(` scheduled by ${row.configFile} (${row.cron ?? "?"})`));
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
|
+
if (payload.fleet)
|
|
84
|
+
lines.push(...renderFleet(payload.fleet, dim));
|
|
83
85
|
const slack = payload.skips?.slack;
|
|
84
86
|
const external = payload.skips?.external;
|
|
85
87
|
if (slack || external) {
|
|
@@ -93,6 +95,33 @@ export function renderOpsStatus(payload, dim) {
|
|
|
93
95
|
}
|
|
94
96
|
return lines;
|
|
95
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* The laptops. Red first, because a person reading this at 9am should not have
|
|
100
|
+
* to scroll past nine healthy machines to find the dead one.
|
|
101
|
+
*/
|
|
102
|
+
export function renderFleet(fleet, dim) {
|
|
103
|
+
const lines = ["", `FLEET ${fleet.summary ?? "(no summary)"}`];
|
|
104
|
+
if (fleet.readError) {
|
|
105
|
+
lines.push(` the fleet could not be read (${fleet.readError}); nothing is known about any machine`);
|
|
106
|
+
return lines;
|
|
107
|
+
}
|
|
108
|
+
const devices = fleet.devices ?? [];
|
|
109
|
+
const rank = (colour) => colour === "red" ? 0 : colour === "amber" ? 1 : 2;
|
|
110
|
+
const ordered = [...devices].sort((left, right) => rank(left.colour) - rank(right.colour));
|
|
111
|
+
if (ordered.length === 0) {
|
|
112
|
+
lines.push(dim(" no live collector device is registered at all"));
|
|
113
|
+
}
|
|
114
|
+
for (const device of ordered) {
|
|
115
|
+
const text = ` ${device.line ?? device.deviceId ?? "(device)"}`;
|
|
116
|
+
// A healthy machine is dimmed, never dropped: "which laptops are fine" is
|
|
117
|
+
// the other half of the question, and a list that only shows failures
|
|
118
|
+
// cannot answer "is everybody else collecting?".
|
|
119
|
+
lines.push(device.colour === "red" || device.colour === "amber" ? text : dim(text));
|
|
120
|
+
}
|
|
121
|
+
if (fleet.noDeviceLine)
|
|
122
|
+
lines.push(dim(` ${fleet.noDeviceLine}`));
|
|
123
|
+
return lines;
|
|
124
|
+
}
|
|
96
125
|
function renderSkipLedger(ledger, dim) {
|
|
97
126
|
const lines = [];
|
|
98
127
|
const name = ledger.relation ?? "skips";
|
package/dist/commands/ops.js
CHANGED
|
@@ -78,6 +78,12 @@ async function runOpsStatus(command, io, tower) {
|
|
|
78
78
|
unhealthy: unhealthy.length,
|
|
79
79
|
unhealthy_ids: unhealthy.map((row) => row.id ?? "?"),
|
|
80
80
|
with_skips: Boolean(command.skips),
|
|
81
|
+
// The laptops (BLI-3550), counted on the same line: a run that shows a
|
|
82
|
+
// green board and says nothing about the fleet cannot answer "was
|
|
83
|
+
// anybody's machine dead this morning?".
|
|
84
|
+
fleet_devices: payload.fleet?.counts?.devices ?? null,
|
|
85
|
+
fleet_red: payload.fleet?.counts?.red ?? null,
|
|
86
|
+
fleet_amber: payload.fleet?.counts?.amber ?? null,
|
|
81
87
|
})}`);
|
|
82
88
|
return unhealthy.length > 0 ? 1 : 0;
|
|
83
89
|
}
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.47");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -143,17 +143,34 @@ function claudeAttributionReadFailureCount(scan) {
|
|
|
143
143
|
}
|
|
144
144
|
export function sourceScanRetryReason(source, scan) {
|
|
145
145
|
const reasons = new Set();
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
146
|
+
const failure = sourceScanFailureReason(source, scan);
|
|
147
|
+
if (failure)
|
|
148
|
+
reasons.add(failure);
|
|
149
|
+
// Kept HERE and nowhere else (BLI-3551): a repo that is not on disk is a
|
|
150
|
+
// reason to widen the next scan window, because the transcript fallback can
|
|
151
|
+
// still attribute it. It is not a reason to call this sync failed — see
|
|
152
|
+
// `sourceScanFailureReason`.
|
|
152
153
|
if (scan.results.some((result) => result.reason === "repo_not_on_disk")) {
|
|
153
154
|
reasons.add("repo_not_on_disk");
|
|
154
155
|
}
|
|
155
156
|
return reasons.size > 0 ? [...reasons].sort().join(",") : null;
|
|
156
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* The part of the scan outcome that is a genuine FAILURE: the session store
|
|
160
|
+
* itself could not be read, so sessions that exist were not seen.
|
|
161
|
+
*
|
|
162
|
+
* Split from {@link sourceScanRetryReason} in BLI-3551. The two used to be one
|
|
163
|
+
* function, so `repo_not_on_disk` — a label the attribution umbrella finding
|
|
164
|
+
* already established is not a defect (nothing was deleted; the transcript
|
|
165
|
+
* names a path git no longer tracks) — failed the sync on every tick for three
|
|
166
|
+
* operators. A retry hint and a failure are different claims.
|
|
167
|
+
*/
|
|
168
|
+
export function sourceScanFailureReason(source, scan) {
|
|
169
|
+
const readFailureCount = source === "codex"
|
|
170
|
+
? codexAttributionReadFailureCount(scan)
|
|
171
|
+
: claudeAttributionReadFailureCount(scan);
|
|
172
|
+
return readFailureCount > 0 ? `${source}_session_store_read_failed` : null;
|
|
173
|
+
}
|
|
157
174
|
async function reconcileSourceScanRetry(options) {
|
|
158
175
|
if (options.reason) {
|
|
159
176
|
await recordSourceRetryFailure(options.paths, {
|
|
@@ -447,25 +464,47 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
447
464
|
});
|
|
448
465
|
// Same conditions as before, one per line, each writing down its own reason.
|
|
449
466
|
// The old version was a single boolean chain: correct, and completely mute.
|
|
450
|
-
|
|
451
|
-
|
|
467
|
+
//
|
|
468
|
+
// Since BLI-3551 each condition also writes down the LABEL it is classified
|
|
469
|
+
// by, beside the rendered string a person reads. The health receipt reads the
|
|
470
|
+
// label; nothing parses the sentence back apart.
|
|
471
|
+
const failureRecords = new Map();
|
|
472
|
+
const add = (record) => {
|
|
473
|
+
if (!failureRecords.has(record.rendered)) {
|
|
474
|
+
failureRecords.set(record.rendered, record);
|
|
475
|
+
}
|
|
476
|
+
};
|
|
477
|
+
const fail = (condition, label, rendered = label) => {
|
|
452
478
|
if (condition)
|
|
453
|
-
|
|
479
|
+
add({ label, rendered });
|
|
454
480
|
};
|
|
455
481
|
for (const { worktree, sync } of outcomes) {
|
|
456
482
|
if (sync.status !== "uploaded") {
|
|
457
483
|
// The spooled reason is the most specific thing anyone has, so lead with
|
|
458
484
|
// it and name the worktree it belongs to — a fleet failure is usually one
|
|
459
485
|
// repo, and "which one" is the first question asked.
|
|
460
|
-
|
|
461
|
-
?
|
|
462
|
-
|
|
486
|
+
add(sync.status === "spooled" && sync.failure_reason
|
|
487
|
+
? {
|
|
488
|
+
label: sync.failure_class,
|
|
489
|
+
rendered: `${worktree.worktree_label}:${sync.failure_reason}`,
|
|
490
|
+
http_status: sync.failure_http_status,
|
|
491
|
+
}
|
|
492
|
+
: {
|
|
493
|
+
label: "upload_not_completed",
|
|
494
|
+
rendered: `${worktree.worktree_label}:upload_${sync.status}`,
|
|
495
|
+
});
|
|
463
496
|
}
|
|
464
497
|
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
465
|
-
|
|
498
|
+
add({
|
|
499
|
+
label: "raw_evidence_upload_failed",
|
|
500
|
+
rendered: `raw_evidence:${reason}`,
|
|
501
|
+
});
|
|
466
502
|
}
|
|
467
503
|
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
468
|
-
|
|
504
|
+
add({
|
|
505
|
+
label: "raw_evidence_retry_required",
|
|
506
|
+
rendered: `raw_evidence_retry:${reason}`,
|
|
507
|
+
});
|
|
469
508
|
}
|
|
470
509
|
fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
|
|
471
510
|
fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
|
|
@@ -474,28 +513,110 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
474
513
|
fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
|
|
475
514
|
fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
|
|
476
515
|
fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
516
|
+
// BLI-3551: the scan's RETRY reason and the scan's FAILURE reason are two
|
|
517
|
+
// different questions, and answering both with one function is what put
|
|
518
|
+
// `claude_scan:repo_not_on_disk` on every tick of three machines. A repo that
|
|
519
|
+
// is not on disk is a label on the session (the attribution umbrella finding:
|
|
520
|
+
// nothing was deleted, the transcript simply names a path git no longer
|
|
521
|
+
// knows). It still widens the next scan window; it is not a failed sync.
|
|
522
|
+
const codexScanFailure = sourceScanFailureReason("codex", codexAttribution);
|
|
523
|
+
if (codexScanFailure) {
|
|
524
|
+
add({ label: "codex_scan_read_failed", rendered: `codex_scan:${codexScanFailure}` });
|
|
525
|
+
}
|
|
526
|
+
const claudeScanFailure = sourceScanFailureReason("claude_code", claudeAttribution);
|
|
527
|
+
if (claudeScanFailure) {
|
|
528
|
+
add({
|
|
529
|
+
label: "claude_scan_read_failed",
|
|
530
|
+
rendered: `claude_scan:${claudeScanFailure}`,
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
// BLI-3551: a tick that observed only sessions from outside the operator's
|
|
534
|
+
// approved roots has nothing to post, and that is the consent boundary
|
|
535
|
+
// working — not a failure. It used to fail as
|
|
536
|
+
// `session_report_unposted:no_successful_sync`, whose word "session" then
|
|
537
|
+
// classified as `auth_failed`; one machine reported a broken credential 377
|
|
538
|
+
// times in 38 hours while its token had eleven weeks left. The withhold
|
|
539
|
+
// decision itself is untouched (adapters/attribution-core.ts) — only what it
|
|
540
|
+
// is CALLED.
|
|
541
|
+
const sessionsOutsideRoot = sessions.filter((session) => OUTSIDE_APPROVED_ROOT_REASONS.has(session.attribution_reason)).length;
|
|
542
|
+
const nothingInRoot = nothingInRootCount({
|
|
543
|
+
sessionCount: sessions.length,
|
|
544
|
+
outsideRootCount: sessionsOutsideRoot,
|
|
545
|
+
outcomes,
|
|
546
|
+
reportPosted: report.posted,
|
|
547
|
+
reportReason: report.reason,
|
|
548
|
+
});
|
|
549
|
+
fail(reportRequired && !report.posted && nothingInRoot === null, "session_report_unposted", `session_report_unposted:${report.reason ?? "unknown"}`);
|
|
484
550
|
// `ok` may already be false from the per-worktree loop above; the outcome
|
|
485
551
|
// scan re-derives that, so the two agree by construction.
|
|
486
|
-
ok = ok &&
|
|
487
|
-
if (!ok &&
|
|
488
|
-
|
|
552
|
+
ok = ok && failureRecords.size === 0;
|
|
553
|
+
if (!ok && failureRecords.size === 0) {
|
|
554
|
+
add({
|
|
555
|
+
label: SYNC_FAILED_WITHOUT_REASON,
|
|
556
|
+
rendered: SYNC_FAILED_WITHOUT_REASON,
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
const notice = ok && nothingInRoot !== null ? `nothing_in_root:${nothingInRoot}` : null;
|
|
560
|
+
if (notice) {
|
|
561
|
+
// The success branch says something too: this is the receipt that proves a
|
|
562
|
+
// quiet machine is a working machine, and the count is what tells a coach
|
|
563
|
+
// that someone is working entirely outside the approved boundary.
|
|
564
|
+
console.error("[session-sync] nothing to collect inside the approved roots", JSON.stringify({
|
|
565
|
+
reason: "nothing_in_root",
|
|
566
|
+
sessions_outside_root: nothingInRoot,
|
|
567
|
+
collection_root_count: collectionRoots.length,
|
|
568
|
+
next_action: "widen the approved roots (an operator decision) if this machine should be collecting here",
|
|
569
|
+
}));
|
|
489
570
|
}
|
|
571
|
+
const records = [...failureRecords.values()].sort((a, b) => a.rendered.localeCompare(b.rendered));
|
|
490
572
|
return {
|
|
491
573
|
ok,
|
|
492
|
-
failure_reasons:
|
|
574
|
+
failure_reasons: records.map((record) => record.rendered),
|
|
575
|
+
failure_records: records,
|
|
576
|
+
notice,
|
|
577
|
+
sessions_observed: sessions.length,
|
|
578
|
+
sessions_outside_root: sessionsOutsideRoot,
|
|
493
579
|
outcomes,
|
|
494
580
|
codexAttribution,
|
|
495
581
|
claudeAttribution,
|
|
496
582
|
summary,
|
|
497
583
|
};
|
|
498
584
|
}
|
|
585
|
+
/**
|
|
586
|
+
* Reasons attribution gives when a session's working directory is not inside
|
|
587
|
+
* any approved collection root.
|
|
588
|
+
*
|
|
589
|
+
* Exact labels, not a pattern — the same discipline the classifier now follows.
|
|
590
|
+
* `attribution-core.ts` writes both of these and nothing else means
|
|
591
|
+
* "outside the boundary".
|
|
592
|
+
*/
|
|
593
|
+
const OUTSIDE_APPROVED_ROOT_REASONS = new Set([
|
|
594
|
+
"cwd_outside_scanned_worktrees",
|
|
595
|
+
"no_matching_worktree_signals",
|
|
596
|
+
]);
|
|
597
|
+
/**
|
|
598
|
+
* How many observed sessions were outside the approved roots, when that
|
|
599
|
+
* accounts for ALL of them and nothing else went wrong — otherwise `null`.
|
|
600
|
+
*
|
|
601
|
+
* Deliberately narrow. It requires that no worktree was synced at all (so no
|
|
602
|
+
* upload could have succeeded or failed), that every session observed this tick
|
|
603
|
+
* names an outside-the-root reason, and that the unposted report is the
|
|
604
|
+
* `no_successful_sync` shape rather than a spooled report that failed to flush.
|
|
605
|
+
* Anything else keeps its failure.
|
|
606
|
+
*/
|
|
607
|
+
export function nothingInRootCount(options) {
|
|
608
|
+
if (options.reportPosted)
|
|
609
|
+
return null;
|
|
610
|
+
if (options.reportReason !== "no_successful_sync")
|
|
611
|
+
return null;
|
|
612
|
+
if (options.outcomes.length > 0)
|
|
613
|
+
return null;
|
|
614
|
+
if (options.sessionCount === 0)
|
|
615
|
+
return null;
|
|
616
|
+
return options.outsideRootCount === options.sessionCount
|
|
617
|
+
? options.outsideRootCount
|
|
618
|
+
: null;
|
|
619
|
+
}
|
|
499
620
|
export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
|
|
500
621
|
function normalizeCodexResult(result) {
|
|
501
622
|
return {
|