@bli-cockpit/cli 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/autostart.js +11 -2
- package/dist/commands/backfill.js +23 -3
- package/dist/commands/doctor.js +6 -1
- package/dist/commands/local.js +81 -16
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +103 -20
- package/dist/discovery-limits.js +89 -0
- package/dist/evidence-upload-client.js +27 -1
- package/dist/repo-identity.js +30 -1
- package/dist/upload.js +54 -6
- package/package.json +2 -2
package/dist/autostart.js
CHANGED
|
@@ -2,6 +2,7 @@ import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "./local-state.js";
|
|
5
|
+
import { savedDiscoveryLimitArgs } from "./discovery-limits.js";
|
|
5
6
|
/** launchd LaunchAgent label; matches docs/runbooks/cockpit-launchd-sync.md. */
|
|
6
7
|
export const AUTOSTART_LABEL = "com.bli.cockpit.sync";
|
|
7
8
|
export const WINDOWS_AUTOSTART_TASK_NAME = "BLI Cockpit Sync";
|
|
@@ -53,6 +54,7 @@ export async function installAutostartAgent(options) {
|
|
|
53
54
|
await mkdir(path.dirname(plistPath), { recursive: true });
|
|
54
55
|
await mkdir(paths.state_dir, { recursive: true });
|
|
55
56
|
await writeFile(plistPath, renderPlist({
|
|
57
|
+
discoveryArgs: await savedDiscoveryLimitArgs(homeDir),
|
|
56
58
|
workDir,
|
|
57
59
|
workDirs: resolvedWorkDirs,
|
|
58
60
|
dashboardUrl,
|
|
@@ -127,6 +129,7 @@ export async function autostartStatus(options) {
|
|
|
127
129
|
const cliEntryPoint = path.resolve(options.cliEntryPoint ?? process.argv[1] ?? "");
|
|
128
130
|
const plist = await readFile(plistPath, "utf8").catch(() => "");
|
|
129
131
|
const registrationProblems = darwinAgentRegistrationProblems(plist, {
|
|
132
|
+
discoveryArgs: await savedDiscoveryLimitArgs(homeDir),
|
|
130
133
|
workDirs: resolvedWorkDirs,
|
|
131
134
|
dashboardUrl,
|
|
132
135
|
intervalSeconds,
|
|
@@ -165,6 +168,7 @@ async function installWindowsTask(options) {
|
|
|
165
168
|
const cliEntryPoint = path.win32.resolve(options.cliEntryPoint ?? process.argv[1] ?? "");
|
|
166
169
|
await mkdir(path.dirname(scriptPath), { recursive: true });
|
|
167
170
|
await writeFile(scriptPath, `${UTF8_BOM}${renderWindowsSyncScript({
|
|
171
|
+
discoveryArgs: await savedDiscoveryLimitArgs(homeDir),
|
|
168
172
|
workDirs: resolvedWorkDirs,
|
|
169
173
|
dashboardUrl,
|
|
170
174
|
nodeExecutable,
|
|
@@ -290,6 +294,7 @@ async function windowsTaskStatus(options) {
|
|
|
290
294
|
}
|
|
291
295
|
else if (options.repoRoots && options.repoRoots.length > 0) {
|
|
292
296
|
const expectedScript = `${UTF8_BOM}${renderWindowsSyncScript({
|
|
297
|
+
discoveryArgs: await savedDiscoveryLimitArgs(options.homeDir),
|
|
293
298
|
workDirs: normalizeWindowsWorkDirs(options.repoRoots),
|
|
294
299
|
dashboardUrl: options.dashboardUrl ?? DEFAULT_DASHBOARD_URL,
|
|
295
300
|
nodeExecutable: path.win32.resolve(options.nodeExecutable ?? process.execPath),
|
|
@@ -337,9 +342,10 @@ function renderWindowsSyncScript(options) {
|
|
|
337
342
|
const dashboardArgs = options.dashboardUrl === DEFAULT_DASHBOARD_URL
|
|
338
343
|
? ""
|
|
339
344
|
: ` --dashboard-url ${powershellLiteral(options.dashboardUrl)}`;
|
|
345
|
+
const discoveryArgs = options.discoveryArgs.length > 0 ? ` ${options.discoveryArgs.join(" ")}` : "";
|
|
340
346
|
const commands = options.workDirs.flatMap((root) => [
|
|
341
347
|
"try {",
|
|
342
|
-
` & $nodeExecutable $cliEntryPoint sync --workspace ${powershellLiteral(root)}${dashboardArgs} --json`,
|
|
348
|
+
` & $nodeExecutable $cliEntryPoint sync --workspace ${powershellLiteral(root)}${dashboardArgs}${discoveryArgs} --json`,
|
|
343
349
|
" if ($LASTEXITCODE -ne 0) { $exitCode = $LASTEXITCODE }",
|
|
344
350
|
"} catch {",
|
|
345
351
|
" [Console]::Error.WriteLine($_.Exception.Message)",
|
|
@@ -532,6 +538,7 @@ function renderPlist(options) {
|
|
|
532
538
|
dashboardUrl: options.dashboardUrl,
|
|
533
539
|
nodeExecutable: options.nodeExecutable,
|
|
534
540
|
cliEntryPoint: options.cliEntryPoint,
|
|
541
|
+
discoveryArgs: options.discoveryArgs,
|
|
535
542
|
});
|
|
536
543
|
return [
|
|
537
544
|
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
@@ -569,7 +576,8 @@ function renderDarwinSyncCommand(options) {
|
|
|
569
576
|
const dashboardArg = options.dashboardUrl === DEFAULT_DASHBOARD_URL
|
|
570
577
|
? ""
|
|
571
578
|
: ` --dashboard-url ${shellQuote(options.dashboardUrl)}`;
|
|
572
|
-
const
|
|
579
|
+
const discoveryArg = options.discoveryArgs.length > 0 ? ` ${options.discoveryArgs.join(" ")}` : "";
|
|
580
|
+
const commands = options.workDirs.map((root) => `${shellQuote(options.nodeExecutable)} ${shellQuote(options.cliEntryPoint)} sync --workspace ${shellQuote(root)}${dashboardArg}${discoveryArg} --json || exit_code=1`);
|
|
573
581
|
return ["exit_code=0", ...commands, 'exit "$exit_code"'].join("; ");
|
|
574
582
|
}
|
|
575
583
|
function darwinAgentRegistrationProblems(plist, expected) {
|
|
@@ -583,6 +591,7 @@ function darwinAgentRegistrationProblems(plist, expected) {
|
|
|
583
591
|
problems.push(`cadence is not ${expected.intervalSeconds} seconds`);
|
|
584
592
|
}
|
|
585
593
|
const expectedCommand = renderDarwinSyncCommand({
|
|
594
|
+
discoveryArgs: expected.discoveryArgs,
|
|
586
595
|
workDirs: expected.workDirs,
|
|
587
596
|
dashboardUrl: expected.dashboardUrl,
|
|
588
597
|
nodeExecutable: expected.nodeExecutable,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { containsSecretLikeContent, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { containsSecretLikeContent, NO_UPLOAD_ATTEMPT_RECORDED, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
@@ -1081,11 +1081,14 @@ async function syncBackfillBatch(options) {
|
|
|
1081
1081
|
}
|
|
1082
1082
|
function buildBackfillSessionReport(options) {
|
|
1083
1083
|
const uploadByKey = new Map();
|
|
1084
|
+
// BLI-2107: an outcome that names a failure but has no pointer used to be
|
|
1085
|
+
// dropped on the floor here, taking its reason with it.
|
|
1086
|
+
const noUploadReasonBySessionId = new Map();
|
|
1084
1087
|
for (const sync of options.syncResults) {
|
|
1085
1088
|
if (sync.status !== "uploaded")
|
|
1086
1089
|
continue;
|
|
1087
1090
|
for (const outcome of sync.raw_evidence_outcomes) {
|
|
1088
|
-
if (!outcome.codex_session_id
|
|
1091
|
+
if (!outcome.codex_session_id)
|
|
1089
1092
|
continue;
|
|
1090
1093
|
const source = outcome.kind === "claude_jsonl"
|
|
1091
1094
|
? "claude_code"
|
|
@@ -1094,9 +1097,16 @@ function buildBackfillSessionReport(options) {
|
|
|
1094
1097
|
: null;
|
|
1095
1098
|
if (!source)
|
|
1096
1099
|
continue;
|
|
1100
|
+
if (!outcome.raw_evidence_pointer_id) {
|
|
1101
|
+
if (outcome.reason) {
|
|
1102
|
+
noUploadReasonBySessionId.set(outcome.codex_session_id, outcome.reason);
|
|
1103
|
+
}
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1097
1106
|
uploadByKey.set(`${source}:${outcome.codex_session_id}`, {
|
|
1098
1107
|
upload_state: outcome.upload_state,
|
|
1099
1108
|
raw_evidence_pointer_id: outcome.raw_evidence_pointer_id,
|
|
1109
|
+
reason: outcome.reason,
|
|
1100
1110
|
});
|
|
1101
1111
|
}
|
|
1102
1112
|
}
|
|
@@ -1143,9 +1153,19 @@ function buildBackfillSessionReport(options) {
|
|
|
1143
1153
|
? {
|
|
1144
1154
|
raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
|
|
1145
1155
|
upload_state: upload.upload_state,
|
|
1156
|
+
...(upload.upload_state === "upload_failed"
|
|
1157
|
+
? { upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED }
|
|
1158
|
+
: {}),
|
|
1146
1159
|
}
|
|
1147
1160
|
: isRawEvidenceUploadableAttributionState(candidate.state)
|
|
1148
|
-
? {
|
|
1161
|
+
? {
|
|
1162
|
+
upload_state: "not_uploaded",
|
|
1163
|
+
// BLI-2107: same rule as live sync — an attributed session with
|
|
1164
|
+
// no pointer always says why, even when the answer is that this
|
|
1165
|
+
// path never recorded one.
|
|
1166
|
+
upload_reason: noUploadReasonBySessionId.get(candidate.session_id) ??
|
|
1167
|
+
NO_UPLOAD_ATTEMPT_RECORDED,
|
|
1168
|
+
}
|
|
1149
1169
|
: {}),
|
|
1150
1170
|
};
|
|
1151
1171
|
});
|
package/dist/commands/doctor.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { autostartStatus, installAutostartAgent } from "../autostart.js";
|
|
4
|
+
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
4
5
|
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
5
6
|
import { backfillCompletionCovers, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
6
7
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
@@ -314,8 +315,12 @@ async function fixSyncState(context) {
|
|
|
314
315
|
if (roots.length === 0) {
|
|
315
316
|
return fail("sync-fresh", "no_roots", "sync has no saved workspace roots");
|
|
316
317
|
}
|
|
318
|
+
// The remembered limits have to ride along, or the doctor's own verification
|
|
319
|
+
// sync scans differently from every other run and can fail closed on a
|
|
320
|
+
// machine the operator already fixed by hand (BLI-2362).
|
|
321
|
+
const discoveryArgs = await savedDiscoveryLimitArgs(context.command.homeDir);
|
|
317
322
|
for (const repoRoot of roots) {
|
|
318
|
-
const args = ["sync", "--json", "--workspace", repoRoot];
|
|
323
|
+
const args = ["sync", "--json", "--workspace", repoRoot, ...discoveryArgs];
|
|
319
324
|
if (context.command.dashboardUrl !== DEFAULT_DASHBOARD_URL) {
|
|
320
325
|
args.push("--dashboard-url", context.command.dashboardUrl);
|
|
321
326
|
}
|
package/dist/commands/local.js
CHANGED
|
@@ -15,7 +15,8 @@ import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WIN
|
|
|
15
15
|
import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
|
|
16
16
|
import { backfillCompletionCovers, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
17
17
|
import { acquireSyncLock } from "../sync-lock.js";
|
|
18
|
-
import {
|
|
18
|
+
import { collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } from "../repo-identity.js";
|
|
19
|
+
import { resolveDiscoveryLimits, saveDiscoveryLimits, } from "../discovery-limits.js";
|
|
19
20
|
import { runAttributedWorktreeSync, matchesLiveSyncWorktree, } from "./session-sync.js";
|
|
20
21
|
import { COLLECTION_ROOT_REQUIRED, missingCollectionRootMessage, normalizeRootsDetailed, resolveOnboardingRoots, rootRejectionExplanation, } from "../onboarding-roots.js";
|
|
21
22
|
import { rawEvidenceGcSummary, runRawEvidenceLocalGc, } from "../raw-evidence-gc.js";
|
|
@@ -58,6 +59,10 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
|
58
59
|
writeLine(io.stderr, localCommandHelp());
|
|
59
60
|
return 1;
|
|
60
61
|
}
|
|
62
|
+
// A limit typed on the command line is remembered for every later run,
|
|
63
|
+
// including the scheduled one nobody types into (BLI-2362). Done here so it
|
|
64
|
+
// applies to whichever command carried the flag.
|
|
65
|
+
await rememberDiscoveryLimits(command);
|
|
61
66
|
try {
|
|
62
67
|
switch (command.kind) {
|
|
63
68
|
case "install":
|
|
@@ -1430,6 +1435,7 @@ async function runOnboard(command, io) {
|
|
|
1430
1435
|
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
1431
1436
|
maxDepth: command.maxDepth,
|
|
1432
1437
|
maxRepos: command.maxRepos,
|
|
1438
|
+
homeDir: command.homeDir,
|
|
1433
1439
|
allowEmpty: true,
|
|
1434
1440
|
}, io);
|
|
1435
1441
|
if (worktrees.length > 1) {
|
|
@@ -1782,14 +1788,22 @@ function cursorStatusLine(sync) {
|
|
|
1782
1788
|
const ALL_SESSION_SCAN_WINDOW_MINUTES = 20 * 365 * 24 * 60;
|
|
1783
1789
|
const SESSION_SCAN_OVERRIDE_LIMIT = 10_000;
|
|
1784
1790
|
async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
|
|
1785
|
-
|
|
1791
|
+
// What the operator typed this run, else what they typed some previous run,
|
|
1792
|
+
// else the built-in defaults (BLI-2362).
|
|
1793
|
+
const limits = await resolveDiscoveryLimits(discovery, discovery.homeDir);
|
|
1794
|
+
const maxWorktrees = limits.maxRepos;
|
|
1786
1795
|
const roots = Array.isArray(repoRoot)
|
|
1787
1796
|
? repoRoot
|
|
1788
1797
|
: [repoRoot ?? process.cwd()];
|
|
1789
1798
|
const result = await discoverGitWorktreesInRootsWithStatus(roots, {
|
|
1790
|
-
maxDepth:
|
|
1799
|
+
maxDepth: limits.maxDepth,
|
|
1791
1800
|
maxWorktrees,
|
|
1792
1801
|
});
|
|
1802
|
+
if (io && result.unreadable_dirs.length > 0) {
|
|
1803
|
+
// Never silently dropped: anything under these folders is missing from the
|
|
1804
|
+
// scan, so say so even when the run otherwise succeeds.
|
|
1805
|
+
writeLine(io.stderr, unreadableDirectoriesMessage(result.unreadable_dirs));
|
|
1806
|
+
}
|
|
1793
1807
|
const worktrees = result.worktrees;
|
|
1794
1808
|
if (!result.complete) {
|
|
1795
1809
|
// Sync fails closed here ON PURPOSE, and that is not the bug. Advancing a
|
|
@@ -1801,7 +1815,7 @@ async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
|
|
|
1801
1815
|
const message = incompleteDiscoveryMessage({
|
|
1802
1816
|
result,
|
|
1803
1817
|
roots,
|
|
1804
|
-
maxDepth:
|
|
1818
|
+
maxDepth: limits.maxDepth,
|
|
1805
1819
|
maxRepos: maxWorktrees,
|
|
1806
1820
|
found: worktrees.length,
|
|
1807
1821
|
});
|
|
@@ -1814,6 +1828,31 @@ async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
|
|
|
1814
1828
|
}
|
|
1815
1829
|
return worktrees;
|
|
1816
1830
|
}
|
|
1831
|
+
/**
|
|
1832
|
+
* Persists `--max-depth` / `--max-repos` when a command carried them, so the
|
|
1833
|
+
* number survives into the background sync and the doctor's own sync — neither
|
|
1834
|
+
* of which has anywhere to type one (BLI-2362). Best-effort: failing to record
|
|
1835
|
+
* a preference must never fail the command the operator actually asked for.
|
|
1836
|
+
*/
|
|
1837
|
+
async function rememberDiscoveryLimits(command) {
|
|
1838
|
+
const limits = command;
|
|
1839
|
+
if (limits.maxDepth === undefined && limits.maxRepos === undefined)
|
|
1840
|
+
return;
|
|
1841
|
+
await saveDiscoveryLimits({ maxDepth: limits.maxDepth, maxRepos: limits.maxRepos }, limits.homeDir).catch(() => undefined);
|
|
1842
|
+
}
|
|
1843
|
+
/**
|
|
1844
|
+
* Says which folders could not be opened, and therefore what the scan could not
|
|
1845
|
+
* see. Reported without failing the run — an unreadable folder cannot be fixed
|
|
1846
|
+
* by retrying, so blocking on one would strand the machine (BLI-2362).
|
|
1847
|
+
*/
|
|
1848
|
+
function unreadableDirectoriesMessage(unreadable) {
|
|
1849
|
+
return [
|
|
1850
|
+
`WARNING: ${unreadable.length} folder(s) could not be opened, so anything inside them was not scanned:`,
|
|
1851
|
+
...unreadable.map((dir) => ` ${dir.path} (${dir.code})`),
|
|
1852
|
+
"Collection continued for everything else. If a repo is missing from Cockpit,",
|
|
1853
|
+
"check the permissions on the folders above.",
|
|
1854
|
+
].join("\n");
|
|
1855
|
+
}
|
|
1817
1856
|
/**
|
|
1818
1857
|
* Names the roots that could not be covered and hands back a command that
|
|
1819
1858
|
* actually fixes it, with this machine's numbers already filled in.
|
|
@@ -2039,7 +2078,7 @@ async function runLogout(command, io) {
|
|
|
2039
2078
|
return 0;
|
|
2040
2079
|
}
|
|
2041
2080
|
async function runStart(command, io) {
|
|
2042
|
-
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos }, io);
|
|
2081
|
+
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
|
|
2043
2082
|
if (worktrees.length > 1) {
|
|
2044
2083
|
const contexts = await Promise.all(worktrees.map((worktree) => startLocalWorkContext({
|
|
2045
2084
|
homeDir: command.homeDir,
|
|
@@ -2176,13 +2215,25 @@ async function runSyncWithHealthReceipt(command, io) {
|
|
|
2176
2215
|
};
|
|
2177
2216
|
}
|
|
2178
2217
|
try {
|
|
2179
|
-
const exitCode = await runSyncLocked(command, io);
|
|
2218
|
+
const { exitCode, failureReasons } = await runSyncLocked(command, io);
|
|
2219
|
+
if (exitCode === 0) {
|
|
2220
|
+
return {
|
|
2221
|
+
exitCode,
|
|
2222
|
+
completion: { step: "sync_complete", status: "ok" },
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
// A sync that fails by exit code says exactly as much as one that throws.
|
|
2226
|
+
// It used to say `sync_failed` and nothing else, so 100% of recorded
|
|
2227
|
+
// failure rows carried a null detail and the real reason was reachable only
|
|
2228
|
+
// by running `cockpit status` on the machine itself (BLI-2526).
|
|
2229
|
+
const reasonText = failureReasons.join("; ");
|
|
2180
2230
|
return {
|
|
2181
2231
|
exitCode,
|
|
2182
2232
|
completion: {
|
|
2183
2233
|
step: "sync_complete",
|
|
2184
|
-
status:
|
|
2185
|
-
|
|
2234
|
+
status: "fail",
|
|
2235
|
+
error_code: classifySyncHealthError(reasonText),
|
|
2236
|
+
error_detail: redactedSyncErrorDetail(reasonText),
|
|
2186
2237
|
},
|
|
2187
2238
|
};
|
|
2188
2239
|
}
|
|
@@ -2225,11 +2276,25 @@ export function redactedSyncErrorDetail(error) {
|
|
|
2225
2276
|
? `${text.slice(0, SYNC_ERROR_DETAIL_MAX_CHARS - 1)}…`
|
|
2226
2277
|
: text;
|
|
2227
2278
|
}
|
|
2279
|
+
/**
|
|
2280
|
+
* Turn a finished run into an exit code and the reasons behind it.
|
|
2281
|
+
*
|
|
2282
|
+
* One place, so a future return path cannot reintroduce a code with no reason.
|
|
2283
|
+
* The reasons come from the run itself — the code that decided `ok` is false is
|
|
2284
|
+
* the only code that knows why.
|
|
2285
|
+
*/
|
|
2286
|
+
function syncResult(run) {
|
|
2287
|
+
return {
|
|
2288
|
+
exitCode: run.ok ? 0 : 1,
|
|
2289
|
+
failureReasons: run.ok ? [] : run.failure_reasons,
|
|
2290
|
+
};
|
|
2291
|
+
}
|
|
2228
2292
|
async function runSyncLocked(command, io) {
|
|
2229
2293
|
const collectionRoots = await resolveSyncCollectionRoots(command);
|
|
2230
2294
|
const worktrees = await discoverCommandWorktrees(collectionRoots, {
|
|
2231
2295
|
maxDepth: command.maxDepth,
|
|
2232
2296
|
maxRepos: command.maxRepos,
|
|
2297
|
+
homeDir: command.homeDir,
|
|
2233
2298
|
allowEmpty: true,
|
|
2234
2299
|
}, io);
|
|
2235
2300
|
const run = await runAttributedWorktreeSync({
|
|
@@ -2254,7 +2319,7 @@ async function runSyncLocked(command, io) {
|
|
|
2254
2319
|
codex_sessions: run.summary,
|
|
2255
2320
|
raw_evidence_gc: gc,
|
|
2256
2321
|
}, null, 2));
|
|
2257
|
-
return run
|
|
2322
|
+
return syncResult(run);
|
|
2258
2323
|
}
|
|
2259
2324
|
writeLine(run.ok ? io.stdout : io.stderr, `Cockpit parent sync ${runStatus} ${run.outcomes.filter((outcome) => outcome.sync.status === "uploaded").length}/${run.outcomes.length} worktree(s).`);
|
|
2260
2325
|
for (const outcome of run.outcomes) {
|
|
@@ -2266,7 +2331,7 @@ async function runSyncLocked(command, io) {
|
|
|
2266
2331
|
writeAgentSessionSummary(io, run.summary);
|
|
2267
2332
|
if (gc && !gc.skipped)
|
|
2268
2333
|
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
2269
|
-
return run
|
|
2334
|
+
return syncResult(run);
|
|
2270
2335
|
}
|
|
2271
2336
|
const result = run.outcomes[0]?.sync;
|
|
2272
2337
|
if (!result) {
|
|
@@ -2282,7 +2347,7 @@ async function runSyncLocked(command, io) {
|
|
|
2282
2347
|
codex_sessions: run.summary,
|
|
2283
2348
|
raw_evidence_gc: gc,
|
|
2284
2349
|
}, null, 2));
|
|
2285
|
-
return run
|
|
2350
|
+
return syncResult(run);
|
|
2286
2351
|
}
|
|
2287
2352
|
if (run.ok) {
|
|
2288
2353
|
writeLine(io.stdout, "Cockpit ambient envelope uploaded.");
|
|
@@ -2297,18 +2362,18 @@ async function runSyncLocked(command, io) {
|
|
|
2297
2362
|
writeLine(io.stdout, cursorStatusLine(result));
|
|
2298
2363
|
if (gc && !gc.skipped)
|
|
2299
2364
|
writeLine(io.stdout, rawEvidenceGcSummary(gc));
|
|
2300
|
-
return
|
|
2365
|
+
return syncResult(run);
|
|
2301
2366
|
}
|
|
2302
2367
|
if (result.status === "uploaded") {
|
|
2303
2368
|
writeLine(io.stderr, "Cockpit ambient upload was accepted, but session collection is partial; retry `cockpit sync`.");
|
|
2304
2369
|
writeAgentSessionSummary(io, run.summary);
|
|
2305
|
-
return
|
|
2370
|
+
return syncResult(run);
|
|
2306
2371
|
}
|
|
2307
2372
|
writeLine(io.stderr, "Cockpit ambient upload failed; safe retry metadata was spooled.");
|
|
2308
2373
|
writeLine(io.stderr, `Ticket: ${displayTicketId(result.ticket_id)}`);
|
|
2309
2374
|
writeLine(io.stderr, `Failure: ${result.failure_reason}`);
|
|
2310
2375
|
writeLine(io.stderr, `Retry: ${result.retry_command}`);
|
|
2311
|
-
return
|
|
2376
|
+
return syncResult(run);
|
|
2312
2377
|
}
|
|
2313
2378
|
async function resolveSyncCollectionRoots(command) {
|
|
2314
2379
|
const explicitRoots = normalizeCollectionRoots(command.repoRoot ? [command.repoRoot] : []);
|
|
@@ -2472,7 +2537,7 @@ async function runSyncRawEvidenceGc(command, io) {
|
|
|
2472
2537
|
}
|
|
2473
2538
|
async function runStatus(command, io) {
|
|
2474
2539
|
const backfillCursor = await inspectBackfillCursor(command.homeDir);
|
|
2475
|
-
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos }, io);
|
|
2540
|
+
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
|
|
2476
2541
|
if (worktrees.length > 1) {
|
|
2477
2542
|
const statuses = await Promise.all(worktrees.map(async (worktree) => ({
|
|
2478
2543
|
...(await inspectLocalCollectorStatus({
|
|
@@ -2703,7 +2768,7 @@ function sessionsWindowLine(window) {
|
|
|
2703
2768
|
async function runSessions(command, io) {
|
|
2704
2769
|
const now = new Date();
|
|
2705
2770
|
const homeDir = command.homeDir ?? os.homedir();
|
|
2706
|
-
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos }, io);
|
|
2771
|
+
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
|
|
2707
2772
|
const window = await sessionsScanWindow(command, now);
|
|
2708
2773
|
const wantCodex = command.source !== "claude";
|
|
2709
2774
|
const wantClaude = command.source !== "codex";
|
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.13");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import os from "node:os";
|
|
7
7
|
import path from "node:path";
|
|
8
8
|
import { getCollectorRuntimePaths, readLocalCollectorConfig, startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "../local-state.js";
|
|
9
|
-
import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, } from "@bli-cockpit/telemetry-core";
|
|
9
|
+
import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, NO_UPLOAD_ATTEMPT_RECORDED, } from "@bli-cockpit/telemetry-core";
|
|
10
10
|
import { flushPendingCodexSessionReports, LocalUploadBlockedError, queueCodexSessionReport, syncLocalAmbientEnvelope, } from "../upload.js";
|
|
11
11
|
import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
|
|
12
12
|
import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
|
|
@@ -15,6 +15,14 @@ import { CLAUDE_CURSOR_FILENAME, countStaleSessions, emptyRawEvidenceCursorState
|
|
|
15
15
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
16
16
|
import { clearSourceRetryFailure, readLocalUploadSpoolState, recordSourceRetryFailure, } from "../spool/local-spool.js";
|
|
17
17
|
import { isLiveRawEvidenceSyncAttribution, } from "../raw-evidence-attribution-policy.js";
|
|
18
|
+
/**
|
|
19
|
+
* The label used when a sync fails and nothing on the way there said why.
|
|
20
|
+
*
|
|
21
|
+
* A deliberate sentinel rather than a fallback to `sync_failed`: it means the
|
|
22
|
+
* gate is real but its reason is unrecorded, which is a bug in this file, and
|
|
23
|
+
* it should be visible as one instead of blending into the generic bucket.
|
|
24
|
+
*/
|
|
25
|
+
export const SYNC_FAILED_WITHOUT_REASON = "sync_failed_reason_not_recorded";
|
|
18
26
|
const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
|
|
19
27
|
const CLAUDE_DAMP_GROWTH_BYTES = 256 * 1024;
|
|
20
28
|
const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
@@ -414,20 +422,56 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
414
422
|
growthDamped: dampedClaudePointers.size,
|
|
415
423
|
report,
|
|
416
424
|
});
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
(
|
|
430
|
-
|
|
425
|
+
// Same conditions as before, one per line, each writing down its own reason.
|
|
426
|
+
// The old version was a single boolean chain: correct, and completely mute.
|
|
427
|
+
const failureReasons = new Set();
|
|
428
|
+
const fail = (condition, reason) => {
|
|
429
|
+
if (condition)
|
|
430
|
+
failureReasons.add(reason);
|
|
431
|
+
};
|
|
432
|
+
for (const { worktree, sync } of outcomes) {
|
|
433
|
+
if (sync.status !== "uploaded") {
|
|
434
|
+
// The spooled reason is the most specific thing anyone has, so lead with
|
|
435
|
+
// it and name the worktree it belongs to — a fleet failure is usually one
|
|
436
|
+
// repo, and "which one" is the first question asked.
|
|
437
|
+
failureReasons.add(sync.status === "spooled" && sync.failure_reason
|
|
438
|
+
? `${worktree.worktree_label}:${sync.failure_reason}`
|
|
439
|
+
: `${worktree.worktree_label}:upload_${sync.status}`);
|
|
440
|
+
}
|
|
441
|
+
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
442
|
+
failureReasons.add(`raw_evidence:${reason}`);
|
|
443
|
+
}
|
|
444
|
+
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
445
|
+
failureReasons.add(`raw_evidence_retry:${reason}`);
|
|
446
|
+
}
|
|
447
|
+
fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
|
|
448
|
+
fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
|
|
449
|
+
}
|
|
450
|
+
fail(codexAttribution.session_limit_applied, "codex_session_limit_applied");
|
|
451
|
+
fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
|
|
452
|
+
fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
|
|
453
|
+
fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
|
|
454
|
+
const codexScanRetry = sourceScanRetryReason("codex", codexAttribution);
|
|
455
|
+
if (codexScanRetry)
|
|
456
|
+
failureReasons.add(`codex_scan:${codexScanRetry}`);
|
|
457
|
+
const claudeScanRetry = sourceScanRetryReason("claude_code", claudeAttribution);
|
|
458
|
+
if (claudeScanRetry)
|
|
459
|
+
failureReasons.add(`claude_scan:${claudeScanRetry}`);
|
|
460
|
+
fail(reportRequired && !report.posted, `session_report_unposted:${report.reason ?? "unknown"}`);
|
|
461
|
+
// `ok` may already be false from the per-worktree loop above; the outcome
|
|
462
|
+
// scan re-derives that, so the two agree by construction.
|
|
463
|
+
ok = ok && failureReasons.size === 0;
|
|
464
|
+
if (!ok && failureReasons.size === 0) {
|
|
465
|
+
failureReasons.add(SYNC_FAILED_WITHOUT_REASON);
|
|
466
|
+
}
|
|
467
|
+
return {
|
|
468
|
+
ok,
|
|
469
|
+
failure_reasons: [...failureReasons].sort(),
|
|
470
|
+
outcomes,
|
|
471
|
+
codexAttribution,
|
|
472
|
+
claudeAttribution,
|
|
473
|
+
summary,
|
|
474
|
+
};
|
|
431
475
|
}
|
|
432
476
|
export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
|
|
433
477
|
function normalizeCodexResult(result) {
|
|
@@ -474,7 +518,7 @@ function normalizeClaudeResult(result) {
|
|
|
474
518
|
* session's upload state (D3). Damped Claude sessions report `reused_existing`
|
|
475
519
|
* carrying their prior durable pointer.
|
|
476
520
|
*/
|
|
477
|
-
function buildAgentSessionReport(options) {
|
|
521
|
+
export function buildAgentSessionReport(options) {
|
|
478
522
|
const normalized = [
|
|
479
523
|
...options.codexResults.map(normalizeCodexResult),
|
|
480
524
|
...options.claudeResults.map(normalizeClaudeResult),
|
|
@@ -495,11 +539,23 @@ function buildAgentSessionReport(options) {
|
|
|
495
539
|
// Main-file outcomes only (D3): a sidecar making it must never mark a session
|
|
496
540
|
// uploaded when the main did not.
|
|
497
541
|
const uploadByKey = new Map();
|
|
542
|
+
// BLI-2107: why a session got no pointer, keyed the same way. The reasons
|
|
543
|
+
// already existed — file_too_large, deferred_byte_budget, the redaction
|
|
544
|
+
// guards — but only as aggregate skip counts in the health report, so no
|
|
545
|
+
// individual session could say what happened to it.
|
|
546
|
+
const noUploadReasonByKey = new Map();
|
|
547
|
+
// A worktree sync that did not finish carries no per-session outcomes, and
|
|
548
|
+
// WorktreeSyncOutcome does not record which sessions it was going to cover.
|
|
549
|
+
// So this pass knows only that it was degraded, not which session each
|
|
550
|
+
// failure belonged to — and says exactly that rather than picking one.
|
|
551
|
+
let anySyncIncomplete = false;
|
|
498
552
|
for (const outcome of options.outcomes) {
|
|
499
|
-
if (outcome.sync.status !== "uploaded")
|
|
553
|
+
if (outcome.sync.status !== "uploaded") {
|
|
554
|
+
anySyncIncomplete = true;
|
|
500
555
|
continue;
|
|
556
|
+
}
|
|
501
557
|
for (const upload of outcome.sync.raw_evidence_outcomes) {
|
|
502
|
-
if (!upload.codex_session_id
|
|
558
|
+
if (!upload.codex_session_id)
|
|
503
559
|
continue;
|
|
504
560
|
const source = upload.kind === "claude_jsonl"
|
|
505
561
|
? "claude_code"
|
|
@@ -508,7 +564,15 @@ function buildAgentSessionReport(options) {
|
|
|
508
564
|
: null;
|
|
509
565
|
if (!source)
|
|
510
566
|
continue; // sidecars and other kinds do not set session state
|
|
511
|
-
|
|
567
|
+
const key = `${source}:${upload.codex_session_id}`;
|
|
568
|
+
if (!upload.raw_evidence_pointer_id) {
|
|
569
|
+
// An outcome with no pointer is a failure that named itself. Keep the
|
|
570
|
+
// reason even though there is nothing to point at.
|
|
571
|
+
if (upload.reason)
|
|
572
|
+
noUploadReasonByKey.set(key, upload.reason);
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
uploadByKey.set(key, upload);
|
|
512
576
|
}
|
|
513
577
|
}
|
|
514
578
|
return [...bestByKey.values()].map((result) => {
|
|
@@ -548,6 +612,13 @@ function buildAgentSessionReport(options) {
|
|
|
548
612
|
? {
|
|
549
613
|
raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
|
|
550
614
|
upload_state: upload.upload_state,
|
|
615
|
+
// A failed upload names itself; a successful one has nothing to
|
|
616
|
+
// explain.
|
|
617
|
+
...(upload.upload_state === "upload_failed"
|
|
618
|
+
? {
|
|
619
|
+
upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED,
|
|
620
|
+
}
|
|
621
|
+
: {}),
|
|
551
622
|
}
|
|
552
623
|
: priorDurablePointer
|
|
553
624
|
? {
|
|
@@ -556,7 +627,19 @@ function buildAgentSessionReport(options) {
|
|
|
556
627
|
}
|
|
557
628
|
: result.state === "attributed" ||
|
|
558
629
|
result.state === "attributed_fallback"
|
|
559
|
-
? {
|
|
630
|
+
? {
|
|
631
|
+
upload_state: "not_uploaded",
|
|
632
|
+
// BLI-2107: `not_uploaded` used to be the branch of last
|
|
633
|
+
// resort, recording that nothing happened and never why. It
|
|
634
|
+
// now always carries a cause, even when the cause is that we
|
|
635
|
+
// have none — a session labelled NO_UPLOAD_ATTEMPT_RECORDED
|
|
636
|
+
// is a path that still needs instrumenting, and saying so is
|
|
637
|
+
// the point.
|
|
638
|
+
upload_reason: noUploadReasonByKey.get(key) ??
|
|
639
|
+
(anySyncIncomplete
|
|
640
|
+
? "sync_incomplete_this_pass"
|
|
641
|
+
: NO_UPLOAD_ATTEMPT_RECORDED),
|
|
642
|
+
}
|
|
560
643
|
: {}),
|
|
561
644
|
};
|
|
562
645
|
});
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { DEFAULT_DISCOVERY_MAX_DEPTH, DEFAULT_DISCOVERY_MAX_REPOS, } from "./repo-identity.js";
|
|
5
|
+
import { getCollectorRuntimePaths } from "./local-state.js";
|
|
6
|
+
const SCHEMA_VERSION = "cockpit-discovery-limits.v1";
|
|
7
|
+
// A depth beyond this is a typo rather than a workspace, and an unbounded walk
|
|
8
|
+
// on a huge tree is its own outage. Same for the repo cap.
|
|
9
|
+
const MAX_ALLOWED_DEPTH = 64;
|
|
10
|
+
const MAX_ALLOWED_REPOS = 5000;
|
|
11
|
+
function discoveryLimitsFile(homeDir) {
|
|
12
|
+
return path.join(getCollectorRuntimePaths(homeDir).state_dir, "discovery-limits.json");
|
|
13
|
+
}
|
|
14
|
+
function sanitizeLimit(value, ceiling) {
|
|
15
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value))
|
|
16
|
+
return undefined;
|
|
17
|
+
if (value < 1 || value > ceiling)
|
|
18
|
+
return undefined;
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Reads the remembered limits. Never throws: a corrupt file falls back to the
|
|
23
|
+
* defaults rather than taking collection down, because failing to read a tuning
|
|
24
|
+
* knob is not a reason to stop collecting.
|
|
25
|
+
*/
|
|
26
|
+
export async function readSavedDiscoveryLimits(homeDir = os.homedir()) {
|
|
27
|
+
try {
|
|
28
|
+
const raw = await fs.readFile(discoveryLimitsFile(homeDir), "utf8");
|
|
29
|
+
const parsed = JSON.parse(raw);
|
|
30
|
+
return {
|
|
31
|
+
max_depth: sanitizeLimit(parsed.max_depth, MAX_ALLOWED_DEPTH),
|
|
32
|
+
max_repos: sanitizeLimit(parsed.max_repos, MAX_ALLOWED_REPOS),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return {};
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Remembers limits an operator passed explicitly. Only writes the ones actually
|
|
41
|
+
* supplied, so raising the depth does not quietly reset a repo cap someone set
|
|
42
|
+
* earlier for their own reasons.
|
|
43
|
+
*/
|
|
44
|
+
export async function saveDiscoveryLimits(limits, homeDir = os.homedir()) {
|
|
45
|
+
const maxDepth = sanitizeLimit(limits.maxDepth, MAX_ALLOWED_DEPTH);
|
|
46
|
+
const maxRepos = sanitizeLimit(limits.maxRepos, MAX_ALLOWED_REPOS);
|
|
47
|
+
if (maxDepth === undefined && maxRepos === undefined)
|
|
48
|
+
return;
|
|
49
|
+
const existing = await readSavedDiscoveryLimits(homeDir);
|
|
50
|
+
const next = {
|
|
51
|
+
schema_version: SCHEMA_VERSION,
|
|
52
|
+
max_depth: maxDepth ?? existing.max_depth,
|
|
53
|
+
max_repos: maxRepos ?? existing.max_repos,
|
|
54
|
+
updated_at: new Date().toISOString(),
|
|
55
|
+
};
|
|
56
|
+
const file = discoveryLimitsFile(homeDir);
|
|
57
|
+
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
58
|
+
await fs.writeFile(file, `${JSON.stringify(next, null, 2)}\n`, {
|
|
59
|
+
encoding: "utf8",
|
|
60
|
+
mode: 0o600,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* The limits a scan should actually use: what the operator typed this run,
|
|
65
|
+
* else what they typed some previous run, else the built-in defaults.
|
|
66
|
+
*/
|
|
67
|
+
export async function resolveDiscoveryLimits(command, homeDir = os.homedir()) {
|
|
68
|
+
const saved = await readSavedDiscoveryLimits(homeDir);
|
|
69
|
+
return {
|
|
70
|
+
maxDepth: command.maxDepth ?? saved.max_depth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
|
|
71
|
+
maxRepos: command.maxRepos ?? saved.max_repos ?? DEFAULT_DISCOVERY_MAX_REPOS,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The flags a generated background command needs so a scheduled run scans the
|
|
76
|
+
* same way an operator's manual run did. Empty when the machine is on the
|
|
77
|
+
* defaults, which keeps the common plist/Task Scheduler command unchanged.
|
|
78
|
+
*/
|
|
79
|
+
export async function savedDiscoveryLimitArgs(homeDir = os.homedir()) {
|
|
80
|
+
const saved = await readSavedDiscoveryLimits(homeDir);
|
|
81
|
+
const args = [];
|
|
82
|
+
if (saved.max_depth !== undefined) {
|
|
83
|
+
args.push("--max-depth", String(saved.max_depth));
|
|
84
|
+
}
|
|
85
|
+
if (saved.max_repos !== undefined) {
|
|
86
|
+
args.push("--max-repos", String(saved.max_repos));
|
|
87
|
+
}
|
|
88
|
+
return args;
|
|
89
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
@@ -214,6 +214,14 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
|
214
214
|
object_key: objectKey,
|
|
215
215
|
});
|
|
216
216
|
if (!commit.ok) {
|
|
217
|
+
// The server can tell us this object will be refused again. Take its reason
|
|
218
|
+
// verbatim so the label names the cause ("storage rejected 116 MB") instead
|
|
219
|
+
// of the transport ("commit_failed_http_500"), which is all the fleet could
|
|
220
|
+
// say for the 57 days of BLI-2528.
|
|
221
|
+
const permanent = permanentCommitRejection(commit.body);
|
|
222
|
+
if (permanent) {
|
|
223
|
+
return failedOutcome(entry.file, permanent, uploadedChunks);
|
|
224
|
+
}
|
|
217
225
|
const detail = safeFailureDetail(commit.body);
|
|
218
226
|
return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
|
|
219
227
|
}
|
|
@@ -376,6 +384,24 @@ function failedOutcome(file, reason, uploadedChunks = 0) {
|
|
|
376
384
|
uploaded_chunk_count: uploadedChunks,
|
|
377
385
|
};
|
|
378
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* The server's verdict that repeating this commit is pointless, or null.
|
|
389
|
+
*
|
|
390
|
+
* Two conditions, both required: the response says `retryable: false`, and the
|
|
391
|
+
* reason is one this collector has classified as permanent. The second check is
|
|
392
|
+
* the important one. A newer dashboard could declare a reason this CLI has never
|
|
393
|
+
* heard of, and quietly abandoning an object on a word we cannot interpret is
|
|
394
|
+
* exactly the silent drop the fleet contract forbids — so an unclassified reason
|
|
395
|
+
* falls through to the ordinary retry path and stays visible.
|
|
396
|
+
*/
|
|
397
|
+
function permanentCommitRejection(body) {
|
|
398
|
+
if (!body || typeof body !== "object")
|
|
399
|
+
return null;
|
|
400
|
+
if (body.retryable !== false)
|
|
401
|
+
return null;
|
|
402
|
+
const reason = safeFailureDetail(body);
|
|
403
|
+
return reason && isPermanentUploadFailure(reason) ? reason : null;
|
|
404
|
+
}
|
|
379
405
|
function safeFailureDetail(body) {
|
|
380
406
|
if (!body || typeof body !== "object")
|
|
381
407
|
return null;
|
package/dist/repo-identity.js
CHANGED
|
@@ -31,6 +31,12 @@ const SKIPPED_DIR_NAMES = new Set([
|
|
|
31
31
|
*/
|
|
32
32
|
export const DEFAULT_DISCOVERY_MAX_DEPTH = 20;
|
|
33
33
|
export const DEFAULT_DISCOVERY_MAX_REPOS = 200;
|
|
34
|
+
/** Caps the reported list so one pathological tree cannot dominate a receipt. */
|
|
35
|
+
const MAX_REPORTED_UNREADABLE_DIRS = 25;
|
|
36
|
+
function errorCode(error) {
|
|
37
|
+
const code = error?.code;
|
|
38
|
+
return typeof code === "string" ? code : "UNKNOWN";
|
|
39
|
+
}
|
|
34
40
|
export async function resolveRepoWorktreeIdentity(repoRoot) {
|
|
35
41
|
const requestedPath = path.resolve(repoRoot);
|
|
36
42
|
const gitRoot = await runGit(["rev-parse", "--show-toplevel"], requestedPath);
|
|
@@ -89,6 +95,7 @@ export async function discoverGitWorktreesWithStatus(root, options = {}) {
|
|
|
89
95
|
const stack = [{ dir: resolvedRoot, depth: 0 }];
|
|
90
96
|
const visited = new Set();
|
|
91
97
|
const incompleteReasons = new Set();
|
|
98
|
+
const unreadableDirs = [];
|
|
92
99
|
while (stack.length > 0) {
|
|
93
100
|
if (discovered.size >= maxWorktrees) {
|
|
94
101
|
incompleteReasons.add("max_worktrees_reached");
|
|
@@ -107,7 +114,18 @@ export async function discoverGitWorktreesWithStatus(root, options = {}) {
|
|
|
107
114
|
discovered.set(identity.worktree_fingerprint, identity);
|
|
108
115
|
continue;
|
|
109
116
|
}
|
|
110
|
-
|
|
117
|
+
// A failed read is recorded, never swallowed: anything beneath this folder
|
|
118
|
+
// is missing from the scan, and the operator has to be able to see that.
|
|
119
|
+
let entries;
|
|
120
|
+
try {
|
|
121
|
+
entries = await fs.readdir(current.dir, { withFileTypes: true });
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
if (unreadableDirs.length < MAX_REPORTED_UNREADABLE_DIRS) {
|
|
125
|
+
unreadableDirs.push({ path: current.dir, code: errorCode(error) });
|
|
126
|
+
}
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
111
129
|
if (current.depth >= maxDepth) {
|
|
112
130
|
if (entries.some((entry) => entry.isDirectory() && !shouldSkipDirectory(entry.name))) {
|
|
113
131
|
incompleteReasons.add("max_depth_reached");
|
|
@@ -130,6 +148,7 @@ export async function discoverGitWorktreesWithStatus(root, options = {}) {
|
|
|
130
148
|
incomplete_reasons: [...incompleteReasons].sort(),
|
|
131
149
|
// Single-root scanner: the only root in play is the one it was handed.
|
|
132
150
|
incomplete_roots: incompleteReasons.size === 0 ? [] : [path.resolve(root)],
|
|
151
|
+
unreadable_dirs: unreadableDirs,
|
|
133
152
|
};
|
|
134
153
|
}
|
|
135
154
|
/**
|
|
@@ -147,6 +166,7 @@ export async function discoverGitWorktreesInRootsWithStatus(roots, options = {})
|
|
|
147
166
|
const discovered = new Map();
|
|
148
167
|
const incompleteReasons = new Set();
|
|
149
168
|
const incompleteRoots = new Set();
|
|
169
|
+
const unreadableDirs = [];
|
|
150
170
|
for (const root of roots) {
|
|
151
171
|
const result = await discoverGitWorktreesWithStatus(root, {
|
|
152
172
|
...options,
|
|
@@ -156,6 +176,11 @@ export async function discoverGitWorktreesInRootsWithStatus(roots, options = {})
|
|
|
156
176
|
incompleteReasons.add(reason);
|
|
157
177
|
incompleteRoots.add(root);
|
|
158
178
|
}
|
|
179
|
+
for (const dir of result.unreadable_dirs) {
|
|
180
|
+
if (unreadableDirs.length >= MAX_REPORTED_UNREADABLE_DIRS)
|
|
181
|
+
break;
|
|
182
|
+
unreadableDirs.push(dir);
|
|
183
|
+
}
|
|
159
184
|
for (const worktree of result.worktrees) {
|
|
160
185
|
if (discovered.size >= maxWorktrees) {
|
|
161
186
|
if (!discovered.has(worktree.worktree_fingerprint)) {
|
|
@@ -174,6 +199,7 @@ export async function discoverGitWorktreesInRootsWithStatus(roots, options = {})
|
|
|
174
199
|
complete: incompleteReasons.size === 0,
|
|
175
200
|
incomplete_reasons: [...incompleteReasons].sort(),
|
|
176
201
|
incomplete_roots: [...incompleteRoots].sort(),
|
|
202
|
+
unreadable_dirs: unreadableDirs,
|
|
177
203
|
};
|
|
178
204
|
}
|
|
179
205
|
/**
|
|
@@ -242,6 +268,9 @@ async function expandLinkedWorktrees(identities, maxWorktrees, allowedRoots) {
|
|
|
242
268
|
worktrees,
|
|
243
269
|
complete: incompleteReasons.length === 0,
|
|
244
270
|
incomplete_reasons: incompleteReasons,
|
|
271
|
+
// Expansion asks git for its own worktree list; it never walks folders, so
|
|
272
|
+
// it has no unreadable directories of its own to report.
|
|
273
|
+
unreadable_dirs: [],
|
|
245
274
|
// Linked-worktree expansion is not scoped to one root; callers merge this
|
|
246
275
|
// into a result that already knows which roots were involved.
|
|
247
276
|
incomplete_roots: [],
|
package/dist/upload.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, } from "./local-state.js";
|
|
4
4
|
import { runLocalSourceCollectors } from "./adapters/local-sources.js";
|
|
@@ -236,6 +236,27 @@ export async function syncLocalAmbientEnvelope(options = {}) {
|
|
|
236
236
|
retry_command: "cockpit sync",
|
|
237
237
|
});
|
|
238
238
|
}
|
|
239
|
+
else {
|
|
240
|
+
// No retry to queue, but a permanently rejected object still has to say
|
|
241
|
+
// so. `recordUploadBlocked` names the reason without spooling a retry —
|
|
242
|
+
// the machine reports the failure instead of promising a fix it cannot
|
|
243
|
+
// deliver, and `cockpit status` stops reading clean.
|
|
244
|
+
const permanentReason = permanentEvidenceFailureReason(uploadOutcomes);
|
|
245
|
+
if (permanentReason) {
|
|
246
|
+
await recordUploadBlocked(paths, {
|
|
247
|
+
attemptedAt,
|
|
248
|
+
reason: permanentReason,
|
|
249
|
+
});
|
|
250
|
+
// stderr, which launchd captures to `sync.err.log`, so an unattended
|
|
251
|
+
// machine leaves a dated record of the objects it gave up on. Reason
|
|
252
|
+
// labels and counts only — never a path or a byte of content.
|
|
253
|
+
console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
|
|
254
|
+
attempted_at: attemptedAt,
|
|
255
|
+
reason: permanentReason,
|
|
256
|
+
object_count: permanentFailedOutcomes(uploadOutcomes).length,
|
|
257
|
+
}));
|
|
258
|
+
}
|
|
259
|
+
}
|
|
239
260
|
return {
|
|
240
261
|
status: "uploaded",
|
|
241
262
|
dashboard_url: built.dashboard_url,
|
|
@@ -307,8 +328,37 @@ function retrySourcesForFailedSync(options, facts) {
|
|
|
307
328
|
}
|
|
308
329
|
return [...sources];
|
|
309
330
|
}
|
|
331
|
+
/**
|
|
332
|
+
* The failed uploads a later attempt could still rescue.
|
|
333
|
+
*
|
|
334
|
+
* An object storage has already refused on its own terms is not one of them,
|
|
335
|
+
* and counting it as one is what kept Edward's Mac in `retry_pending` through
|
|
336
|
+
* 13 consecutive syncs that were never going to end differently (BLI-2528).
|
|
337
|
+
*/
|
|
338
|
+
function retryableFailedOutcomes(outcomes) {
|
|
339
|
+
return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
|
|
340
|
+
!isPermanentUploadFailure(outcome.reason));
|
|
341
|
+
}
|
|
342
|
+
/** Failed uploads that no retry can rescue, kept so they can still be named. */
|
|
343
|
+
function permanentFailedOutcomes(outcomes) {
|
|
344
|
+
return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
|
|
345
|
+
isPermanentUploadFailure(outcome.reason));
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* The reason to show for objects that failed for good.
|
|
349
|
+
*
|
|
350
|
+
* Returns null when there are none. These never queue a retry, but they must
|
|
351
|
+
* never disappear either: a machine with a permanently rejected object has
|
|
352
|
+
* missing collection, and a status that reads clean would hide it.
|
|
353
|
+
*/
|
|
354
|
+
function permanentEvidenceFailureReason(outcomes) {
|
|
355
|
+
const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
|
|
356
|
+
if (reasons.size === 0)
|
|
357
|
+
return null;
|
|
358
|
+
return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
|
|
359
|
+
}
|
|
310
360
|
function hasRetryableEvidenceGap(facts, outcomes) {
|
|
311
|
-
if (outcomes
|
|
361
|
+
if (retryableFailedOutcomes(outcomes).length > 0) {
|
|
312
362
|
return true;
|
|
313
363
|
}
|
|
314
364
|
if (!facts)
|
|
@@ -326,10 +376,8 @@ function hasRetryableEvidenceGap(facts, outcomes) {
|
|
|
326
376
|
}
|
|
327
377
|
function retryableEvidenceGapReason(facts, outcomes) {
|
|
328
378
|
const reasons = new Set();
|
|
329
|
-
for (const outcome of outcomes) {
|
|
330
|
-
|
|
331
|
-
reasons.add(outcome.reason ?? "upload_failed");
|
|
332
|
-
}
|
|
379
|
+
for (const outcome of retryableFailedOutcomes(outcomes)) {
|
|
380
|
+
reasons.add(outcome.reason ?? "upload_failed");
|
|
333
381
|
}
|
|
334
382
|
if (facts) {
|
|
335
383
|
if (facts.deferred_byte_budget_count > 0)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.13",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -26,6 +26,6 @@
|
|
|
26
26
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
29
|
+
"@bli-cockpit/telemetry-core": "0.1.17"
|
|
30
30
|
}
|
|
31
31
|
}
|