@bli-cockpit/cli 0.2.11 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/autostart.js CHANGED
@@ -2,6 +2,7 @@ import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
2
2
  import os from "node:os";
3
3
  import path from "node:path";
4
4
  import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "./local-state.js";
5
+ import { savedDiscoveryLimitArgs } from "./discovery-limits.js";
5
6
  /** launchd LaunchAgent label; matches docs/runbooks/cockpit-launchd-sync.md. */
6
7
  export const AUTOSTART_LABEL = "com.bli.cockpit.sync";
7
8
  export const WINDOWS_AUTOSTART_TASK_NAME = "BLI Cockpit Sync";
@@ -53,6 +54,7 @@ export async function installAutostartAgent(options) {
53
54
  await mkdir(path.dirname(plistPath), { recursive: true });
54
55
  await mkdir(paths.state_dir, { recursive: true });
55
56
  await writeFile(plistPath, renderPlist({
57
+ discoveryArgs: await savedDiscoveryLimitArgs(homeDir),
56
58
  workDir,
57
59
  workDirs: resolvedWorkDirs,
58
60
  dashboardUrl,
@@ -127,6 +129,7 @@ export async function autostartStatus(options) {
127
129
  const cliEntryPoint = path.resolve(options.cliEntryPoint ?? process.argv[1] ?? "");
128
130
  const plist = await readFile(plistPath, "utf8").catch(() => "");
129
131
  const registrationProblems = darwinAgentRegistrationProblems(plist, {
132
+ discoveryArgs: await savedDiscoveryLimitArgs(homeDir),
130
133
  workDirs: resolvedWorkDirs,
131
134
  dashboardUrl,
132
135
  intervalSeconds,
@@ -165,6 +168,7 @@ async function installWindowsTask(options) {
165
168
  const cliEntryPoint = path.win32.resolve(options.cliEntryPoint ?? process.argv[1] ?? "");
166
169
  await mkdir(path.dirname(scriptPath), { recursive: true });
167
170
  await writeFile(scriptPath, `${UTF8_BOM}${renderWindowsSyncScript({
171
+ discoveryArgs: await savedDiscoveryLimitArgs(homeDir),
168
172
  workDirs: resolvedWorkDirs,
169
173
  dashboardUrl,
170
174
  nodeExecutable,
@@ -290,6 +294,7 @@ async function windowsTaskStatus(options) {
290
294
  }
291
295
  else if (options.repoRoots && options.repoRoots.length > 0) {
292
296
  const expectedScript = `${UTF8_BOM}${renderWindowsSyncScript({
297
+ discoveryArgs: await savedDiscoveryLimitArgs(options.homeDir),
293
298
  workDirs: normalizeWindowsWorkDirs(options.repoRoots),
294
299
  dashboardUrl: options.dashboardUrl ?? DEFAULT_DASHBOARD_URL,
295
300
  nodeExecutable: path.win32.resolve(options.nodeExecutable ?? process.execPath),
@@ -337,9 +342,10 @@ function renderWindowsSyncScript(options) {
337
342
  const dashboardArgs = options.dashboardUrl === DEFAULT_DASHBOARD_URL
338
343
  ? ""
339
344
  : ` --dashboard-url ${powershellLiteral(options.dashboardUrl)}`;
345
+ const discoveryArgs = options.discoveryArgs.length > 0 ? ` ${options.discoveryArgs.join(" ")}` : "";
340
346
  const commands = options.workDirs.flatMap((root) => [
341
347
  "try {",
342
- ` & $nodeExecutable $cliEntryPoint sync --workspace ${powershellLiteral(root)}${dashboardArgs} --json`,
348
+ ` & $nodeExecutable $cliEntryPoint sync --workspace ${powershellLiteral(root)}${dashboardArgs}${discoveryArgs} --json`,
343
349
  " if ($LASTEXITCODE -ne 0) { $exitCode = $LASTEXITCODE }",
344
350
  "} catch {",
345
351
  " [Console]::Error.WriteLine($_.Exception.Message)",
@@ -532,6 +538,7 @@ function renderPlist(options) {
532
538
  dashboardUrl: options.dashboardUrl,
533
539
  nodeExecutable: options.nodeExecutable,
534
540
  cliEntryPoint: options.cliEntryPoint,
541
+ discoveryArgs: options.discoveryArgs,
535
542
  });
536
543
  return [
537
544
  '<?xml version="1.0" encoding="UTF-8"?>',
@@ -569,7 +576,8 @@ function renderDarwinSyncCommand(options) {
569
576
  const dashboardArg = options.dashboardUrl === DEFAULT_DASHBOARD_URL
570
577
  ? ""
571
578
  : ` --dashboard-url ${shellQuote(options.dashboardUrl)}`;
572
- const commands = options.workDirs.map((root) => `${shellQuote(options.nodeExecutable)} ${shellQuote(options.cliEntryPoint)} sync --workspace ${shellQuote(root)}${dashboardArg} --json || exit_code=1`);
579
+ const discoveryArg = options.discoveryArgs.length > 0 ? ` ${options.discoveryArgs.join(" ")}` : "";
580
+ const commands = options.workDirs.map((root) => `${shellQuote(options.nodeExecutable)} ${shellQuote(options.cliEntryPoint)} sync --workspace ${shellQuote(root)}${dashboardArg}${discoveryArg} --json || exit_code=1`);
573
581
  return ["exit_code=0", ...commands, 'exit "$exit_code"'].join("; ");
574
582
  }
575
583
  function darwinAgentRegistrationProblems(plist, expected) {
@@ -583,6 +591,7 @@ function darwinAgentRegistrationProblems(plist, expected) {
583
591
  problems.push(`cadence is not ${expected.intervalSeconds} seconds`);
584
592
  }
585
593
  const expectedCommand = renderDarwinSyncCommand({
594
+ discoveryArgs: expected.discoveryArgs,
586
595
  workDirs: expected.workDirs,
587
596
  dashboardUrl: expected.dashboardUrl,
588
597
  nodeExecutable: expected.nodeExecutable,
@@ -1,4 +1,4 @@
1
- import { containsSecretLikeContent, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
1
+ import { containsSecretLikeContent, NO_UPLOAD_ATTEMPT_RECORDED, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs/promises";
4
4
  import os from "node:os";
@@ -1081,11 +1081,14 @@ async function syncBackfillBatch(options) {
1081
1081
  }
1082
1082
  function buildBackfillSessionReport(options) {
1083
1083
  const uploadByKey = new Map();
1084
+ // BLI-2107: an outcome that names a failure but has no pointer used to be
1085
+ // dropped on the floor here, taking its reason with it.
1086
+ const noUploadReasonBySessionId = new Map();
1084
1087
  for (const sync of options.syncResults) {
1085
1088
  if (sync.status !== "uploaded")
1086
1089
  continue;
1087
1090
  for (const outcome of sync.raw_evidence_outcomes) {
1088
- if (!outcome.codex_session_id || !outcome.raw_evidence_pointer_id)
1091
+ if (!outcome.codex_session_id)
1089
1092
  continue;
1090
1093
  const source = outcome.kind === "claude_jsonl"
1091
1094
  ? "claude_code"
@@ -1094,9 +1097,16 @@ function buildBackfillSessionReport(options) {
1094
1097
  : null;
1095
1098
  if (!source)
1096
1099
  continue;
1100
+ if (!outcome.raw_evidence_pointer_id) {
1101
+ if (outcome.reason) {
1102
+ noUploadReasonBySessionId.set(outcome.codex_session_id, outcome.reason);
1103
+ }
1104
+ continue;
1105
+ }
1097
1106
  uploadByKey.set(`${source}:${outcome.codex_session_id}`, {
1098
1107
  upload_state: outcome.upload_state,
1099
1108
  raw_evidence_pointer_id: outcome.raw_evidence_pointer_id,
1109
+ reason: outcome.reason,
1100
1110
  });
1101
1111
  }
1102
1112
  }
@@ -1143,9 +1153,19 @@ function buildBackfillSessionReport(options) {
1143
1153
  ? {
1144
1154
  raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
1145
1155
  upload_state: upload.upload_state,
1156
+ ...(upload.upload_state === "upload_failed"
1157
+ ? { upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED }
1158
+ : {}),
1146
1159
  }
1147
1160
  : isRawEvidenceUploadableAttributionState(candidate.state)
1148
- ? { upload_state: "not_uploaded" }
1161
+ ? {
1162
+ upload_state: "not_uploaded",
1163
+ // BLI-2107: same rule as live sync — an attributed session with
1164
+ // no pointer always says why, even when the answer is that this
1165
+ // path never recorded one.
1166
+ upload_reason: noUploadReasonBySessionId.get(candidate.session_id) ??
1167
+ NO_UPLOAD_ATTEMPT_RECORDED,
1168
+ }
1149
1169
  : {}),
1150
1170
  };
1151
1171
  });
@@ -1,6 +1,7 @@
1
1
  import fs from "node:fs/promises";
2
2
  import path from "node:path";
3
3
  import { autostartStatus, installAutostartAgent } from "../autostart.js";
4
+ import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
4
5
  import { inspectBackfillLock } from "../backfill-lock.js";
5
6
  import { backfillCompletionCovers, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
6
7
  import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
@@ -314,8 +315,12 @@ async function fixSyncState(context) {
314
315
  if (roots.length === 0) {
315
316
  return fail("sync-fresh", "no_roots", "sync has no saved workspace roots");
316
317
  }
318
+ // The remembered limits have to ride along, or the doctor's own verification
319
+ // sync scans differently from every other run and can fail closed on a
320
+ // machine the operator already fixed by hand (BLI-2362).
321
+ const discoveryArgs = await savedDiscoveryLimitArgs(context.command.homeDir);
317
322
  for (const repoRoot of roots) {
318
- const args = ["sync", "--json", "--workspace", repoRoot];
323
+ const args = ["sync", "--json", "--workspace", repoRoot, ...discoveryArgs];
319
324
  if (context.command.dashboardUrl !== DEFAULT_DASHBOARD_URL) {
320
325
  args.push("--dashboard-url", context.command.dashboardUrl);
321
326
  }
@@ -15,7 +15,8 @@ import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WIN
15
15
  import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
16
16
  import { backfillCompletionCovers, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
17
17
  import { acquireSyncLock } from "../sync-lock.js";
18
- import { DEFAULT_DISCOVERY_MAX_DEPTH, DEFAULT_DISCOVERY_MAX_REPOS, collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } from "../repo-identity.js";
18
+ import { collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } from "../repo-identity.js";
19
+ import { resolveDiscoveryLimits, saveDiscoveryLimits, } from "../discovery-limits.js";
19
20
  import { runAttributedWorktreeSync, matchesLiveSyncWorktree, } from "./session-sync.js";
20
21
  import { COLLECTION_ROOT_REQUIRED, missingCollectionRootMessage, normalizeRootsDetailed, resolveOnboardingRoots, rootRejectionExplanation, } from "../onboarding-roots.js";
21
22
  import { rawEvidenceGcSummary, runRawEvidenceLocalGc, } from "../raw-evidence-gc.js";
@@ -58,6 +59,10 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
58
59
  writeLine(io.stderr, localCommandHelp());
59
60
  return 1;
60
61
  }
62
+ // A limit typed on the command line is remembered for every later run,
63
+ // including the scheduled one nobody types into (BLI-2362). Done here so it
64
+ // applies to whichever command carried the flag.
65
+ await rememberDiscoveryLimits(command);
61
66
  try {
62
67
  switch (command.kind) {
63
68
  case "install":
@@ -1430,6 +1435,7 @@ async function runOnboard(command, io) {
1430
1435
  const worktrees = await discoverCommandWorktrees(collectionRoots, {
1431
1436
  maxDepth: command.maxDepth,
1432
1437
  maxRepos: command.maxRepos,
1438
+ homeDir: command.homeDir,
1433
1439
  allowEmpty: true,
1434
1440
  }, io);
1435
1441
  if (worktrees.length > 1) {
@@ -1782,14 +1788,22 @@ function cursorStatusLine(sync) {
1782
1788
  const ALL_SESSION_SCAN_WINDOW_MINUTES = 20 * 365 * 24 * 60;
1783
1789
  const SESSION_SCAN_OVERRIDE_LIMIT = 10_000;
1784
1790
  async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
1785
- const maxWorktrees = discovery.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS;
1791
+ // What the operator typed this run, else what they typed some previous run,
1792
+ // else the built-in defaults (BLI-2362).
1793
+ const limits = await resolveDiscoveryLimits(discovery, discovery.homeDir);
1794
+ const maxWorktrees = limits.maxRepos;
1786
1795
  const roots = Array.isArray(repoRoot)
1787
1796
  ? repoRoot
1788
1797
  : [repoRoot ?? process.cwd()];
1789
1798
  const result = await discoverGitWorktreesInRootsWithStatus(roots, {
1790
- maxDepth: discovery.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
1799
+ maxDepth: limits.maxDepth,
1791
1800
  maxWorktrees,
1792
1801
  });
1802
+ if (io && result.unreadable_dirs.length > 0) {
1803
+ // Never silently dropped: anything under these folders is missing from the
1804
+ // scan, so say so even when the run otherwise succeeds.
1805
+ writeLine(io.stderr, unreadableDirectoriesMessage(result.unreadable_dirs));
1806
+ }
1793
1807
  const worktrees = result.worktrees;
1794
1808
  if (!result.complete) {
1795
1809
  // Sync fails closed here ON PURPOSE, and that is not the bug. Advancing a
@@ -1801,7 +1815,7 @@ async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
1801
1815
  const message = incompleteDiscoveryMessage({
1802
1816
  result,
1803
1817
  roots,
1804
- maxDepth: discovery.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
1818
+ maxDepth: limits.maxDepth,
1805
1819
  maxRepos: maxWorktrees,
1806
1820
  found: worktrees.length,
1807
1821
  });
@@ -1814,6 +1828,31 @@ async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
1814
1828
  }
1815
1829
  return worktrees;
1816
1830
  }
1831
+ /**
1832
+ * Persists `--max-depth` / `--max-repos` when a command carried them, so the
1833
+ * number survives into the background sync and the doctor's own sync — neither
1834
+ * of which has anywhere to type one (BLI-2362). Best-effort: failing to record
1835
+ * a preference must never fail the command the operator actually asked for.
1836
+ */
1837
+ async function rememberDiscoveryLimits(command) {
1838
+ const limits = command;
1839
+ if (limits.maxDepth === undefined && limits.maxRepos === undefined)
1840
+ return;
1841
+ await saveDiscoveryLimits({ maxDepth: limits.maxDepth, maxRepos: limits.maxRepos }, limits.homeDir).catch(() => undefined);
1842
+ }
1843
+ /**
1844
+ * Says which folders could not be opened, and therefore what the scan could not
1845
+ * see. Reported without failing the run — an unreadable folder cannot be fixed
1846
+ * by retrying, so blocking on one would strand the machine (BLI-2362).
1847
+ */
1848
+ function unreadableDirectoriesMessage(unreadable) {
1849
+ return [
1850
+ `WARNING: ${unreadable.length} folder(s) could not be opened, so anything inside them was not scanned:`,
1851
+ ...unreadable.map((dir) => ` ${dir.path} (${dir.code})`),
1852
+ "Collection continued for everything else. If a repo is missing from Cockpit,",
1853
+ "check the permissions on the folders above.",
1854
+ ].join("\n");
1855
+ }
1817
1856
  /**
1818
1857
  * Names the roots that could not be covered and hands back a command that
1819
1858
  * actually fixes it, with this machine's numbers already filled in.
@@ -2039,7 +2078,7 @@ async function runLogout(command, io) {
2039
2078
  return 0;
2040
2079
  }
2041
2080
  async function runStart(command, io) {
2042
- const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos }, io);
2081
+ const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
2043
2082
  if (worktrees.length > 1) {
2044
2083
  const contexts = await Promise.all(worktrees.map((worktree) => startLocalWorkContext({
2045
2084
  homeDir: command.homeDir,
@@ -2176,13 +2215,25 @@ async function runSyncWithHealthReceipt(command, io) {
2176
2215
  };
2177
2216
  }
2178
2217
  try {
2179
- const exitCode = await runSyncLocked(command, io);
2218
+ const { exitCode, failureReasons } = await runSyncLocked(command, io);
2219
+ if (exitCode === 0) {
2220
+ return {
2221
+ exitCode,
2222
+ completion: { step: "sync_complete", status: "ok" },
2223
+ };
2224
+ }
2225
+ // A sync that fails by exit code says exactly as much as one that throws.
2226
+ // It used to say `sync_failed` and nothing else, so 100% of recorded
2227
+ // failure rows carried a null detail and the real reason was reachable only
2228
+ // by running `cockpit status` on the machine itself (BLI-2526).
2229
+ const reasonText = failureReasons.join("; ");
2180
2230
  return {
2181
2231
  exitCode,
2182
2232
  completion: {
2183
2233
  step: "sync_complete",
2184
- status: exitCode === 0 ? "ok" : "fail",
2185
- ...(exitCode === 0 ? {} : { error_code: "sync_failed" }),
2234
+ status: "fail",
2235
+ error_code: classifySyncHealthError(reasonText),
2236
+ error_detail: redactedSyncErrorDetail(reasonText),
2186
2237
  },
2187
2238
  };
2188
2239
  }
@@ -2225,11 +2276,25 @@ export function redactedSyncErrorDetail(error) {
2225
2276
  ? `${text.slice(0, SYNC_ERROR_DETAIL_MAX_CHARS - 1)}…`
2226
2277
  : text;
2227
2278
  }
2279
+ /**
2280
+ * Turn a finished run into an exit code and the reasons behind it.
2281
+ *
2282
+ * One place, so a future return path cannot reintroduce a code with no reason.
2283
+ * The reasons come from the run itself — the code that decided `ok` is false is
2284
+ * the only code that knows why.
2285
+ */
2286
+ function syncResult(run) {
2287
+ return {
2288
+ exitCode: run.ok ? 0 : 1,
2289
+ failureReasons: run.ok ? [] : run.failure_reasons,
2290
+ };
2291
+ }
2228
2292
  async function runSyncLocked(command, io) {
2229
2293
  const collectionRoots = await resolveSyncCollectionRoots(command);
2230
2294
  const worktrees = await discoverCommandWorktrees(collectionRoots, {
2231
2295
  maxDepth: command.maxDepth,
2232
2296
  maxRepos: command.maxRepos,
2297
+ homeDir: command.homeDir,
2233
2298
  allowEmpty: true,
2234
2299
  }, io);
2235
2300
  const run = await runAttributedWorktreeSync({
@@ -2254,7 +2319,7 @@ async function runSyncLocked(command, io) {
2254
2319
  codex_sessions: run.summary,
2255
2320
  raw_evidence_gc: gc,
2256
2321
  }, null, 2));
2257
- return run.ok ? 0 : 1;
2322
+ return syncResult(run);
2258
2323
  }
2259
2324
  writeLine(run.ok ? io.stdout : io.stderr, `Cockpit parent sync ${runStatus} ${run.outcomes.filter((outcome) => outcome.sync.status === "uploaded").length}/${run.outcomes.length} worktree(s).`);
2260
2325
  for (const outcome of run.outcomes) {
@@ -2266,7 +2331,7 @@ async function runSyncLocked(command, io) {
2266
2331
  writeAgentSessionSummary(io, run.summary);
2267
2332
  if (gc && !gc.skipped)
2268
2333
  writeLine(io.stdout, rawEvidenceGcSummary(gc));
2269
- return run.ok ? 0 : 1;
2334
+ return syncResult(run);
2270
2335
  }
2271
2336
  const result = run.outcomes[0]?.sync;
2272
2337
  if (!result) {
@@ -2282,7 +2347,7 @@ async function runSyncLocked(command, io) {
2282
2347
  codex_sessions: run.summary,
2283
2348
  raw_evidence_gc: gc,
2284
2349
  }, null, 2));
2285
- return run.ok ? 0 : 1;
2350
+ return syncResult(run);
2286
2351
  }
2287
2352
  if (run.ok) {
2288
2353
  writeLine(io.stdout, "Cockpit ambient envelope uploaded.");
@@ -2297,18 +2362,18 @@ async function runSyncLocked(command, io) {
2297
2362
  writeLine(io.stdout, cursorStatusLine(result));
2298
2363
  if (gc && !gc.skipped)
2299
2364
  writeLine(io.stdout, rawEvidenceGcSummary(gc));
2300
- return 0;
2365
+ return syncResult(run);
2301
2366
  }
2302
2367
  if (result.status === "uploaded") {
2303
2368
  writeLine(io.stderr, "Cockpit ambient upload was accepted, but session collection is partial; retry `cockpit sync`.");
2304
2369
  writeAgentSessionSummary(io, run.summary);
2305
- return 1;
2370
+ return syncResult(run);
2306
2371
  }
2307
2372
  writeLine(io.stderr, "Cockpit ambient upload failed; safe retry metadata was spooled.");
2308
2373
  writeLine(io.stderr, `Ticket: ${displayTicketId(result.ticket_id)}`);
2309
2374
  writeLine(io.stderr, `Failure: ${result.failure_reason}`);
2310
2375
  writeLine(io.stderr, `Retry: ${result.retry_command}`);
2311
- return 1;
2376
+ return syncResult(run);
2312
2377
  }
2313
2378
  async function resolveSyncCollectionRoots(command) {
2314
2379
  const explicitRoots = normalizeCollectionRoots(command.repoRoot ? [command.repoRoot] : []);
@@ -2472,7 +2537,7 @@ async function runSyncRawEvidenceGc(command, io) {
2472
2537
  }
2473
2538
  async function runStatus(command, io) {
2474
2539
  const backfillCursor = await inspectBackfillCursor(command.homeDir);
2475
- const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos }, io);
2540
+ const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
2476
2541
  if (worktrees.length > 1) {
2477
2542
  const statuses = await Promise.all(worktrees.map(async (worktree) => ({
2478
2543
  ...(await inspectLocalCollectorStatus({
@@ -2703,7 +2768,7 @@ function sessionsWindowLine(window) {
2703
2768
  async function runSessions(command, io) {
2704
2769
  const now = new Date();
2705
2770
  const homeDir = command.homeDir ?? os.homedir();
2706
- const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos }, io);
2771
+ const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
2707
2772
  const window = await sessionsScanWindow(command, now);
2708
2773
  const wantCodex = command.source !== "claude";
2709
2774
  const wantClaude = command.source !== "codex";
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.11");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.13");
19
19
  return 0;
20
20
  }
21
21
 
@@ -6,7 +6,7 @@
6
6
  import os from "node:os";
7
7
  import path from "node:path";
8
8
  import { getCollectorRuntimePaths, readLocalCollectorConfig, startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "../local-state.js";
9
- import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, } from "@bli-cockpit/telemetry-core";
9
+ import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, NO_UPLOAD_ATTEMPT_RECORDED, } from "@bli-cockpit/telemetry-core";
10
10
  import { flushPendingCodexSessionReports, LocalUploadBlockedError, queueCodexSessionReport, syncLocalAmbientEnvelope, } from "../upload.js";
11
11
  import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
12
12
  import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
@@ -15,6 +15,14 @@ import { CLAUDE_CURSOR_FILENAME, countStaleSessions, emptyRawEvidenceCursorState
15
15
  import { normalizeCollectionRoots } from "../root-normalization.js";
16
16
  import { clearSourceRetryFailure, readLocalUploadSpoolState, recordSourceRetryFailure, } from "../spool/local-spool.js";
17
17
  import { isLiveRawEvidenceSyncAttribution, } from "../raw-evidence-attribution-policy.js";
18
+ /**
19
+ * The label used when a sync fails and nothing on the way there said why.
20
+ *
21
+ * A deliberate sentinel rather than a fallback to `sync_failed`: it means the
22
+ * gate is real but its reason is unrecorded, which is a bug in this file, and
23
+ * it should be visible as one instead of blending into the generic bucket.
24
+ */
25
+ export const SYNC_FAILED_WITHOUT_REASON = "sync_failed_reason_not_recorded";
18
26
  const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
19
27
  const CLAUDE_DAMP_GROWTH_BYTES = 256 * 1024;
20
28
  const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
@@ -414,20 +422,56 @@ export async function runAttributedWorktreeSync(options) {
414
422
  growthDamped: dampedClaudePointers.size,
415
423
  report,
416
424
  });
417
- ok =
418
- ok &&
419
- !codexAttribution.session_limit_applied &&
420
- !claudeAttribution.session_limit_applied &&
421
- outcomes.every(({ sync }) => sync.raw_evidence_failed_count === 0 &&
422
- !sync.raw_evidence_retry_required &&
423
- sync.raw_evidence_deferred_byte_budget === 0 &&
424
- sync.raw_evidence_deferred_object_budget === 0) &&
425
- codexAttributionReadFailureCount(codexAttribution) === 0 &&
426
- claudeAttributionReadFailureCount(claudeAttribution) === 0 &&
427
- sourceScanRetryReason("codex", codexAttribution) === null &&
428
- sourceScanRetryReason("claude_code", claudeAttribution) === null &&
429
- (!reportRequired || report.posted);
430
- return { ok, outcomes, codexAttribution, claudeAttribution, summary };
425
+ // Same conditions as before, one per line, each writing down its own reason.
426
+ // The old version was a single boolean chain: correct, and completely mute.
427
+ const failureReasons = new Set();
428
+ const fail = (condition, reason) => {
429
+ if (condition)
430
+ failureReasons.add(reason);
431
+ };
432
+ for (const { worktree, sync } of outcomes) {
433
+ if (sync.status !== "uploaded") {
434
+ // The spooled reason is the most specific thing anyone has, so lead with
435
+ // it and name the worktree it belongs to — a fleet failure is usually one
436
+ // repo, and "which one" is the first question asked.
437
+ failureReasons.add(sync.status === "spooled" && sync.failure_reason
438
+ ? `${worktree.worktree_label}:${sync.failure_reason}`
439
+ : `${worktree.worktree_label}:upload_${sync.status}`);
440
+ }
441
+ for (const reason of sync.raw_evidence_failure_reasons ?? []) {
442
+ failureReasons.add(`raw_evidence:${reason}`);
443
+ }
444
+ for (const reason of sync.raw_evidence_retry_reasons ?? []) {
445
+ failureReasons.add(`raw_evidence_retry:${reason}`);
446
+ }
447
+ fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
448
+ fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
449
+ }
450
+ fail(codexAttribution.session_limit_applied, "codex_session_limit_applied");
451
+ fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
452
+ fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
453
+ fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
454
+ const codexScanRetry = sourceScanRetryReason("codex", codexAttribution);
455
+ if (codexScanRetry)
456
+ failureReasons.add(`codex_scan:${codexScanRetry}`);
457
+ const claudeScanRetry = sourceScanRetryReason("claude_code", claudeAttribution);
458
+ if (claudeScanRetry)
459
+ failureReasons.add(`claude_scan:${claudeScanRetry}`);
460
+ fail(reportRequired && !report.posted, `session_report_unposted:${report.reason ?? "unknown"}`);
461
+ // `ok` may already be false from the per-worktree loop above; the outcome
462
+ // scan re-derives that, so the two agree by construction.
463
+ ok = ok && failureReasons.size === 0;
464
+ if (!ok && failureReasons.size === 0) {
465
+ failureReasons.add(SYNC_FAILED_WITHOUT_REASON);
466
+ }
467
+ return {
468
+ ok,
469
+ failure_reasons: [...failureReasons].sort(),
470
+ outcomes,
471
+ codexAttribution,
472
+ claudeAttribution,
473
+ summary,
474
+ };
431
475
  }
432
476
  export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
433
477
  function normalizeCodexResult(result) {
@@ -474,7 +518,7 @@ function normalizeClaudeResult(result) {
474
518
  * session's upload state (D3). Damped Claude sessions report `reused_existing`
475
519
  * carrying their prior durable pointer.
476
520
  */
477
- function buildAgentSessionReport(options) {
521
+ export function buildAgentSessionReport(options) {
478
522
  const normalized = [
479
523
  ...options.codexResults.map(normalizeCodexResult),
480
524
  ...options.claudeResults.map(normalizeClaudeResult),
@@ -495,11 +539,23 @@ function buildAgentSessionReport(options) {
495
539
  // Main-file outcomes only (D3): a sidecar making it must never mark a session
496
540
  // uploaded when the main did not.
497
541
  const uploadByKey = new Map();
542
+ // BLI-2107: why a session got no pointer, keyed the same way. The reasons
543
+ // already existed — file_too_large, deferred_byte_budget, the redaction
544
+ // guards — but only as aggregate skip counts in the health report, so no
545
+ // individual session could say what happened to it.
546
+ const noUploadReasonByKey = new Map();
547
+ // A worktree sync that did not finish carries no per-session outcomes, and
548
+ // WorktreeSyncOutcome does not record which sessions it was going to cover.
549
+ // So this pass knows only that it was degraded, not which session each
550
+ // failure belonged to — and says exactly that rather than picking one.
551
+ let anySyncIncomplete = false;
498
552
  for (const outcome of options.outcomes) {
499
- if (outcome.sync.status !== "uploaded")
553
+ if (outcome.sync.status !== "uploaded") {
554
+ anySyncIncomplete = true;
500
555
  continue;
556
+ }
501
557
  for (const upload of outcome.sync.raw_evidence_outcomes) {
502
- if (!upload.codex_session_id || !upload.raw_evidence_pointer_id)
558
+ if (!upload.codex_session_id)
503
559
  continue;
504
560
  const source = upload.kind === "claude_jsonl"
505
561
  ? "claude_code"
@@ -508,7 +564,15 @@ function buildAgentSessionReport(options) {
508
564
  : null;
509
565
  if (!source)
510
566
  continue; // sidecars and other kinds do not set session state
511
- uploadByKey.set(`${source}:${upload.codex_session_id}`, upload);
567
+ const key = `${source}:${upload.codex_session_id}`;
568
+ if (!upload.raw_evidence_pointer_id) {
569
+ // An outcome with no pointer is a failure that named itself. Keep the
570
+ // reason even though there is nothing to point at.
571
+ if (upload.reason)
572
+ noUploadReasonByKey.set(key, upload.reason);
573
+ continue;
574
+ }
575
+ uploadByKey.set(key, upload);
512
576
  }
513
577
  }
514
578
  return [...bestByKey.values()].map((result) => {
@@ -548,6 +612,13 @@ function buildAgentSessionReport(options) {
548
612
  ? {
549
613
  raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
550
614
  upload_state: upload.upload_state,
615
+ // A failed upload names itself; a successful one has nothing to
616
+ // explain.
617
+ ...(upload.upload_state === "upload_failed"
618
+ ? {
619
+ upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED,
620
+ }
621
+ : {}),
551
622
  }
552
623
  : priorDurablePointer
553
624
  ? {
@@ -556,7 +627,19 @@ function buildAgentSessionReport(options) {
556
627
  }
557
628
  : result.state === "attributed" ||
558
629
  result.state === "attributed_fallback"
559
- ? { upload_state: "not_uploaded" }
630
+ ? {
631
+ upload_state: "not_uploaded",
632
+ // BLI-2107: `not_uploaded` used to be the branch of last
633
+ // resort, recording that nothing happened and never why. It
634
+ // now always carries a cause, even when the cause is that we
635
+ // have none — a session labelled NO_UPLOAD_ATTEMPT_RECORDED
636
+ // is a path that still needs instrumenting, and saying so is
637
+ // the point.
638
+ upload_reason: noUploadReasonByKey.get(key) ??
639
+ (anySyncIncomplete
640
+ ? "sync_incomplete_this_pass"
641
+ : NO_UPLOAD_ATTEMPT_RECORDED),
642
+ }
560
643
  : {}),
561
644
  };
562
645
  });
@@ -0,0 +1,89 @@
1
+ import fs from "node:fs/promises";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import { DEFAULT_DISCOVERY_MAX_DEPTH, DEFAULT_DISCOVERY_MAX_REPOS, } from "./repo-identity.js";
5
+ import { getCollectorRuntimePaths } from "./local-state.js";
6
+ const SCHEMA_VERSION = "cockpit-discovery-limits.v1";
7
+ // A depth beyond this is a typo rather than a workspace, and an unbounded walk
8
+ // on a huge tree is its own outage. Same for the repo cap.
9
+ const MAX_ALLOWED_DEPTH = 64;
10
+ const MAX_ALLOWED_REPOS = 5000;
11
+ function discoveryLimitsFile(homeDir) {
12
+ return path.join(getCollectorRuntimePaths(homeDir).state_dir, "discovery-limits.json");
13
+ }
14
+ function sanitizeLimit(value, ceiling) {
15
+ if (typeof value !== "number" || !Number.isSafeInteger(value))
16
+ return undefined;
17
+ if (value < 1 || value > ceiling)
18
+ return undefined;
19
+ return value;
20
+ }
21
+ /**
22
+ * Reads the remembered limits. Never throws: a corrupt file falls back to the
23
+ * defaults rather than taking collection down, because failing to read a tuning
24
+ * knob is not a reason to stop collecting.
25
+ */
26
+ export async function readSavedDiscoveryLimits(homeDir = os.homedir()) {
27
+ try {
28
+ const raw = await fs.readFile(discoveryLimitsFile(homeDir), "utf8");
29
+ const parsed = JSON.parse(raw);
30
+ return {
31
+ max_depth: sanitizeLimit(parsed.max_depth, MAX_ALLOWED_DEPTH),
32
+ max_repos: sanitizeLimit(parsed.max_repos, MAX_ALLOWED_REPOS),
33
+ };
34
+ }
35
+ catch {
36
+ return {};
37
+ }
38
+ }
39
+ /**
40
+ * Remembers limits an operator passed explicitly. Only writes the ones actually
41
+ * supplied, so raising the depth does not quietly reset a repo cap someone set
42
+ * earlier for their own reasons.
43
+ */
44
+ export async function saveDiscoveryLimits(limits, homeDir = os.homedir()) {
45
+ const maxDepth = sanitizeLimit(limits.maxDepth, MAX_ALLOWED_DEPTH);
46
+ const maxRepos = sanitizeLimit(limits.maxRepos, MAX_ALLOWED_REPOS);
47
+ if (maxDepth === undefined && maxRepos === undefined)
48
+ return;
49
+ const existing = await readSavedDiscoveryLimits(homeDir);
50
+ const next = {
51
+ schema_version: SCHEMA_VERSION,
52
+ max_depth: maxDepth ?? existing.max_depth,
53
+ max_repos: maxRepos ?? existing.max_repos,
54
+ updated_at: new Date().toISOString(),
55
+ };
56
+ const file = discoveryLimitsFile(homeDir);
57
+ await fs.mkdir(path.dirname(file), { recursive: true });
58
+ await fs.writeFile(file, `${JSON.stringify(next, null, 2)}\n`, {
59
+ encoding: "utf8",
60
+ mode: 0o600,
61
+ });
62
+ }
63
+ /**
64
+ * The limits a scan should actually use: what the operator typed this run,
65
+ * else what they typed some previous run, else the built-in defaults.
66
+ */
67
+ export async function resolveDiscoveryLimits(command, homeDir = os.homedir()) {
68
+ const saved = await readSavedDiscoveryLimits(homeDir);
69
+ return {
70
+ maxDepth: command.maxDepth ?? saved.max_depth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
71
+ maxRepos: command.maxRepos ?? saved.max_repos ?? DEFAULT_DISCOVERY_MAX_REPOS,
72
+ };
73
+ }
74
+ /**
75
+ * The flags a generated background command needs so a scheduled run scans the
76
+ * same way an operator's manual run did. Empty when the machine is on the
77
+ * defaults, which keeps the common plist/Task Scheduler command unchanged.
78
+ */
79
+ export async function savedDiscoveryLimitArgs(homeDir = os.homedir()) {
80
+ const saved = await readSavedDiscoveryLimits(homeDir);
81
+ const args = [];
82
+ if (saved.max_depth !== undefined) {
83
+ args.push("--max-depth", String(saved.max_depth));
84
+ }
85
+ if (saved.max_repos !== undefined) {
86
+ args.push("--max-repos", String(saved.max_repos));
87
+ }
88
+ return args;
89
+ }
@@ -1,4 +1,4 @@
1
- import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, } from "@bli-cockpit/telemetry-core";
1
+ import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs/promises";
4
4
  const DEFAULT_MAX_ATTEMPTS = 3;
@@ -214,6 +214,14 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
214
214
  object_key: objectKey,
215
215
  });
216
216
  if (!commit.ok) {
217
+ // The server can tell us this object will be refused again. Take its reason
218
+ // verbatim so the label names the cause ("storage rejected 116 MB") instead
219
+ // of the transport ("commit_failed_http_500"), which is all the fleet could
220
+ // say for the 57 days of BLI-2528.
221
+ const permanent = permanentCommitRejection(commit.body);
222
+ if (permanent) {
223
+ return failedOutcome(entry.file, permanent, uploadedChunks);
224
+ }
217
225
  const detail = safeFailureDetail(commit.body);
218
226
  return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
219
227
  }
@@ -376,6 +384,24 @@ function failedOutcome(file, reason, uploadedChunks = 0) {
376
384
  uploaded_chunk_count: uploadedChunks,
377
385
  };
378
386
  }
387
+ /**
388
+ * The server's verdict that repeating this commit is pointless, or null.
389
+ *
390
+ * Two conditions, both required: the response says `retryable: false`, and the
391
+ * reason is one this collector has classified as permanent. The second check is
392
+ * the important one. A newer dashboard could declare a reason this CLI has never
393
+ * heard of, and quietly abandoning an object on a word we cannot interpret is
394
+ * exactly the silent drop the fleet contract forbids — so an unclassified reason
395
+ * falls through to the ordinary retry path and stays visible.
396
+ */
397
+ function permanentCommitRejection(body) {
398
+ if (!body || typeof body !== "object")
399
+ return null;
400
+ if (body.retryable !== false)
401
+ return null;
402
+ const reason = safeFailureDetail(body);
403
+ return reason && isPermanentUploadFailure(reason) ? reason : null;
404
+ }
379
405
  function safeFailureDetail(body) {
380
406
  if (!body || typeof body !== "object")
381
407
  return null;
@@ -31,6 +31,12 @@ const SKIPPED_DIR_NAMES = new Set([
31
31
  */
32
32
  export const DEFAULT_DISCOVERY_MAX_DEPTH = 20;
33
33
  export const DEFAULT_DISCOVERY_MAX_REPOS = 200;
34
+ /** Caps the reported list so one pathological tree cannot dominate a receipt. */
35
+ const MAX_REPORTED_UNREADABLE_DIRS = 25;
36
+ function errorCode(error) {
37
+ const code = error?.code;
38
+ return typeof code === "string" ? code : "UNKNOWN";
39
+ }
34
40
  export async function resolveRepoWorktreeIdentity(repoRoot) {
35
41
  const requestedPath = path.resolve(repoRoot);
36
42
  const gitRoot = await runGit(["rev-parse", "--show-toplevel"], requestedPath);
@@ -89,6 +95,7 @@ export async function discoverGitWorktreesWithStatus(root, options = {}) {
89
95
  const stack = [{ dir: resolvedRoot, depth: 0 }];
90
96
  const visited = new Set();
91
97
  const incompleteReasons = new Set();
98
+ const unreadableDirs = [];
92
99
  while (stack.length > 0) {
93
100
  if (discovered.size >= maxWorktrees) {
94
101
  incompleteReasons.add("max_worktrees_reached");
@@ -107,7 +114,18 @@ export async function discoverGitWorktreesWithStatus(root, options = {}) {
107
114
  discovered.set(identity.worktree_fingerprint, identity);
108
115
  continue;
109
116
  }
110
- const entries = await fs.readdir(current.dir, { withFileTypes: true }).catch(() => []);
117
+ // A failed read is recorded, never swallowed: anything beneath this folder
118
+ // is missing from the scan, and the operator has to be able to see that.
119
+ let entries;
120
+ try {
121
+ entries = await fs.readdir(current.dir, { withFileTypes: true });
122
+ }
123
+ catch (error) {
124
+ if (unreadableDirs.length < MAX_REPORTED_UNREADABLE_DIRS) {
125
+ unreadableDirs.push({ path: current.dir, code: errorCode(error) });
126
+ }
127
+ continue;
128
+ }
111
129
  if (current.depth >= maxDepth) {
112
130
  if (entries.some((entry) => entry.isDirectory() && !shouldSkipDirectory(entry.name))) {
113
131
  incompleteReasons.add("max_depth_reached");
@@ -130,6 +148,7 @@ export async function discoverGitWorktreesWithStatus(root, options = {}) {
130
148
  incomplete_reasons: [...incompleteReasons].sort(),
131
149
  // Single-root scanner: the only root in play is the one it was handed.
132
150
  incomplete_roots: incompleteReasons.size === 0 ? [] : [path.resolve(root)],
151
+ unreadable_dirs: unreadableDirs,
133
152
  };
134
153
  }
135
154
  /**
@@ -147,6 +166,7 @@ export async function discoverGitWorktreesInRootsWithStatus(roots, options = {})
147
166
  const discovered = new Map();
148
167
  const incompleteReasons = new Set();
149
168
  const incompleteRoots = new Set();
169
+ const unreadableDirs = [];
150
170
  for (const root of roots) {
151
171
  const result = await discoverGitWorktreesWithStatus(root, {
152
172
  ...options,
@@ -156,6 +176,11 @@ export async function discoverGitWorktreesInRootsWithStatus(roots, options = {})
156
176
  incompleteReasons.add(reason);
157
177
  incompleteRoots.add(root);
158
178
  }
179
+ for (const dir of result.unreadable_dirs) {
180
+ if (unreadableDirs.length >= MAX_REPORTED_UNREADABLE_DIRS)
181
+ break;
182
+ unreadableDirs.push(dir);
183
+ }
159
184
  for (const worktree of result.worktrees) {
160
185
  if (discovered.size >= maxWorktrees) {
161
186
  if (!discovered.has(worktree.worktree_fingerprint)) {
@@ -174,6 +199,7 @@ export async function discoverGitWorktreesInRootsWithStatus(roots, options = {})
174
199
  complete: incompleteReasons.size === 0,
175
200
  incomplete_reasons: [...incompleteReasons].sort(),
176
201
  incomplete_roots: [...incompleteRoots].sort(),
202
+ unreadable_dirs: unreadableDirs,
177
203
  };
178
204
  }
179
205
  /**
@@ -242,6 +268,9 @@ async function expandLinkedWorktrees(identities, maxWorktrees, allowedRoots) {
242
268
  worktrees,
243
269
  complete: incompleteReasons.length === 0,
244
270
  incomplete_reasons: incompleteReasons,
271
+ // Expansion asks git for its own worktree list; it never walks folders, so
272
+ // it has no unreadable directories of its own to report.
273
+ unreadable_dirs: [],
245
274
  // Linked-worktree expansion is not scoped to one root; callers merge this
246
275
  // into a result that already knows which roots were involved.
247
276
  incomplete_roots: [],
package/dist/upload.js CHANGED
@@ -1,4 +1,4 @@
1
- import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, } from "@bli-cockpit/telemetry-core";
1
+ import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
2
2
  import path from "node:path";
3
3
  import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, } from "./local-state.js";
4
4
  import { runLocalSourceCollectors } from "./adapters/local-sources.js";
@@ -236,6 +236,27 @@ export async function syncLocalAmbientEnvelope(options = {}) {
236
236
  retry_command: "cockpit sync",
237
237
  });
238
238
  }
239
+ else {
240
+ // No retry to queue, but a permanently rejected object still has to say
241
+ // so. `recordUploadBlocked` names the reason without spooling a retry —
242
+ // the machine reports the failure instead of promising a fix it cannot
243
+ // deliver, and `cockpit status` stops reading clean.
244
+ const permanentReason = permanentEvidenceFailureReason(uploadOutcomes);
245
+ if (permanentReason) {
246
+ await recordUploadBlocked(paths, {
247
+ attemptedAt,
248
+ reason: permanentReason,
249
+ });
250
+ // stderr, which launchd captures to `sync.err.log`, so an unattended
251
+ // machine leaves a dated record of the objects it gave up on. Reason
252
+ // labels and counts only — never a path or a byte of content.
253
+ console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
254
+ attempted_at: attemptedAt,
255
+ reason: permanentReason,
256
+ object_count: permanentFailedOutcomes(uploadOutcomes).length,
257
+ }));
258
+ }
259
+ }
239
260
  return {
240
261
  status: "uploaded",
241
262
  dashboard_url: built.dashboard_url,
@@ -307,8 +328,37 @@ function retrySourcesForFailedSync(options, facts) {
307
328
  }
308
329
  return [...sources];
309
330
  }
331
+ /**
332
+ * The failed uploads a later attempt could still rescue.
333
+ *
334
+ * An object storage has already refused on its own terms is not one of them,
335
+ * and counting it as one is what kept Edward's Mac in `retry_pending` through
336
+ * 13 consecutive syncs that were never going to end differently (BLI-2528).
337
+ */
338
+ function retryableFailedOutcomes(outcomes) {
339
+ return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
340
+ !isPermanentUploadFailure(outcome.reason));
341
+ }
342
+ /** Failed uploads that no retry can rescue, kept so they can still be named. */
343
+ function permanentFailedOutcomes(outcomes) {
344
+ return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
345
+ isPermanentUploadFailure(outcome.reason));
346
+ }
347
+ /**
348
+ * The reason to show for objects that failed for good.
349
+ *
350
+ * Returns null when there are none. These never queue a retry, but they must
351
+ * never disappear either: a machine with a permanently rejected object has
352
+ * missing collection, and a status that reads clean would hide it.
353
+ */
354
+ function permanentEvidenceFailureReason(outcomes) {
355
+ const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
356
+ if (reasons.size === 0)
357
+ return null;
358
+ return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
359
+ }
310
360
  function hasRetryableEvidenceGap(facts, outcomes) {
311
- if (outcomes.some((outcome) => outcome.upload_state === "upload_failed")) {
361
+ if (retryableFailedOutcomes(outcomes).length > 0) {
312
362
  return true;
313
363
  }
314
364
  if (!facts)
@@ -326,10 +376,8 @@ function hasRetryableEvidenceGap(facts, outcomes) {
326
376
  }
327
377
  function retryableEvidenceGapReason(facts, outcomes) {
328
378
  const reasons = new Set();
329
- for (const outcome of outcomes) {
330
- if (outcome.upload_state === "upload_failed") {
331
- reasons.add(outcome.reason ?? "upload_failed");
332
- }
379
+ for (const outcome of retryableFailedOutcomes(outcomes)) {
380
+ reasons.add(outcome.reason ?? "upload_failed");
333
381
  }
334
382
  if (facts) {
335
383
  if (facts.deferred_byte_budget_count > 0)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.11",
3
+ "version": "0.2.13",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -26,6 +26,6 @@
26
26
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
27
27
  },
28
28
  "dependencies": {
29
- "@bli-cockpit/telemetry-core": "0.1.15"
29
+ "@bli-cockpit/telemetry-core": "0.1.17"
30
30
  }
31
31
  }