@bli-cockpit/cli 0.2.12 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/autostart.js CHANGED
@@ -270,6 +270,13 @@ async function windowsTaskStatus(options) {
270
270
  const queryMessage = `${query.stderr}\n${query.stdout}`;
271
271
  const taskIsKnownAbsent = query.code === 1 &&
272
272
  /cannot find|not found|does not exist/iu.test(queryMessage);
273
+ // Metadata only: schtasks stderr can carry the state directory, so the exit
274
+ // code and the classification travel, never the text.
275
+ console.error("[autostart] windows task query failed", JSON.stringify({
276
+ task_name: WINDOWS_AUTOSTART_TASK_NAME,
277
+ exit_code: query.code,
278
+ status: taskIsKnownAbsent ? "absent" : "not_loaded",
279
+ }));
273
280
  return {
274
281
  status: taskIsKnownAbsent ? "absent" : "not_loaded",
275
282
  label: AUTOSTART_LABEL,
@@ -305,6 +312,23 @@ async function windowsTaskStatus(options) {
305
312
  registrationProblems.push("sync script does not match the current roots or Cockpit runtime");
306
313
  }
307
314
  }
315
+ // Both branches log. A line that only fires on failure cannot answer "did
316
+ // background collection validate at all today?", which is the question that
317
+ // would have caught a validator rejecting every healthy Windows task.
318
+ // Problem labels carry no paths, so they are safe to emit verbatim.
319
+ if (registrationProblems.length > 0) {
320
+ console.error("[autostart] windows task needs repair", JSON.stringify({
321
+ task_name: WINDOWS_AUTOSTART_TASK_NAME,
322
+ problem_count: registrationProblems.length,
323
+ problems: registrationProblems,
324
+ }));
325
+ }
326
+ else {
327
+ console.error("[autostart] windows task validated", JSON.stringify({
328
+ task_name: WINDOWS_AUTOSTART_TASK_NAME,
329
+ interval_minutes: intervalMinutes,
330
+ }));
331
+ }
308
332
  return {
309
333
  status: registrationProblems.length > 0 ? "not_loaded" : "loaded",
310
334
  label: AUTOSTART_LABEL,
@@ -415,7 +439,7 @@ function windowsTaskRegistrationProblems(taskXml, expected) {
415
439
  const actionArguments = exactExecBlock
416
440
  .match(/<Arguments>\s*([^<]*?)\s*<\/Arguments>/iu)?.[1]
417
441
  ?.trim() ?? "";
418
- const expectedActionArguments = `-NoProfile -NonInteractive -ExecutionPolicy Bypass -File "${expected.scriptPath}"`;
442
+ const expectedArgumentFlags = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File";
419
443
  const requirements = [
420
444
  [
421
445
  /<DisallowStartIfOnBatteries>\s*false\s*<\/DisallowStartIfOnBatteries>/iu,
@@ -444,8 +468,18 @@ function windowsTaskRegistrationProblems(taskXml, expected) {
444
468
  if (!/<LogonType>\s*InteractiveToken\s*<\/LogonType>/iu.test(exactPrincipalBlock)) {
445
469
  problems.push("logon type is not InteractiveToken");
446
470
  }
447
- if (!/<RunLevel>\s*LeastPrivilege\s*<\/RunLevel>/iu.test(exactPrincipalBlock)) {
448
- problems.push("task does not run with limited privileges");
471
+ // Windows omits <RunLevel> entirely when a task runs at the default
472
+ // LeastPrivilege, so `schtasks /Create ... /RL LIMITED` stores a principal
473
+ // block with no RunLevel element at all. Demanding the element asks Windows
474
+ // to state a default it never states, which is why this check failed on
475
+ // every Windows machine in the fleet and passed on none (0 of 4
476
+ // autostart-alive events ok, 2026-08-12). Absent means LeastPrivilege; only
477
+ // an explicit elevated value is a real problem.
478
+ const runLevel = exactPrincipalBlock
479
+ .match(/<RunLevel>\s*([^<]*?)\s*<\/RunLevel>/iu)?.[1]
480
+ ?.trim() || "LeastPrivilege";
481
+ if (!/^LeastPrivilege$/iu.test(runLevel)) {
482
+ problems.push(`task does not run with limited privileges (RunLevel=${runLevel})`);
449
483
  }
450
484
  }
451
485
  if (actionContext && actionContext !== principalId) {
@@ -477,8 +511,9 @@ function windowsTaskRegistrationProblems(taskXml, expected) {
477
511
  if (!sameWindowsPath(command, expected.powershellPath)) {
478
512
  problems.push("task action does not run the expected Windows PowerShell");
479
513
  }
480
- if (actionArguments !== expectedActionArguments) {
481
- problems.push("task action arguments do not exactly run the Cockpit sync script");
514
+ const actionArgumentProblem = windowsActionArgumentProblem(actionArguments, expectedArgumentFlags, expected.scriptPath);
515
+ if (actionArgumentProblem) {
516
+ problems.push(actionArgumentProblem);
482
517
  }
483
518
  if (/<Duration>/iu.test(repetition)) {
484
519
  problems.push("repetition has a finite duration");
@@ -489,6 +524,38 @@ function windowsPowerShellPath(env = process.env) {
489
524
  const windowsRoot = env.SystemRoot ?? env.WINDIR ?? "C:\\Windows";
490
525
  return path.win32.join(windowsRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe");
491
526
  }
527
+ /**
528
+ * Task Scheduler stores a normalized form of the action it was given, not the
529
+ * string we passed: `schtasks /Create /TR "<powershell> <args>"` is split into
530
+ * <Command> plus <Arguments>, and the quotes around a space-free script path
531
+ * are dropped. Comparing <Arguments> against the exact string we built
532
+ * therefore fails on a task that is registered perfectly — observed on
533
+ * DESKTOP-G2UO1GK (CLI 0.2.13, 2026-08-12), where the stored value differed
534
+ * from the expected one only by those quotes.
535
+ *
536
+ * Compare the flags exactly, because those are ours, and compare the script as
537
+ * a path, because that is what Windows normalizes. Each failure returns its own
538
+ * reason so a repair message says which half is wrong rather than "arguments
539
+ * differ".
540
+ */
541
+ function windowsActionArgumentProblem(actionArguments, expectedFlags, expectedScriptPath) {
542
+ if (!actionArguments.startsWith(expectedFlags)) {
543
+ return "task action does not run PowerShell with the expected Cockpit sync flags";
544
+ }
545
+ const scriptArgument = unquoteWindowsArgument(actionArguments.slice(expectedFlags.length));
546
+ if (!scriptArgument) {
547
+ return "task action names no sync script to run";
548
+ }
549
+ if (!sameWindowsPath(scriptArgument, expectedScriptPath)) {
550
+ return "task action runs a script other than the Cockpit sync script";
551
+ }
552
+ return null;
553
+ }
554
+ function unquoteWindowsArgument(value) {
555
+ const trimmed = value.trim();
556
+ const quoted = trimmed.match(/^"([\s\S]*)"$/u) ?? trimmed.match(/^'([\s\S]*)'$/u);
557
+ return (quoted?.[1] ?? trimmed).trim();
558
+ }
492
559
  function sameWindowsPath(left, right) {
493
560
  if (!left || !right)
494
561
  return false;
@@ -1,4 +1,4 @@
1
- import { containsSecretLikeContent, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
1
+ import { containsSecretLikeContent, NO_UPLOAD_ATTEMPT_RECORDED, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs/promises";
4
4
  import os from "node:os";
@@ -1081,11 +1081,14 @@ async function syncBackfillBatch(options) {
1081
1081
  }
1082
1082
  function buildBackfillSessionReport(options) {
1083
1083
  const uploadByKey = new Map();
1084
+ // BLI-2107: an outcome that names a failure but has no pointer used to be
1085
+ // dropped on the floor here, taking its reason with it.
1086
+ const noUploadReasonBySessionId = new Map();
1084
1087
  for (const sync of options.syncResults) {
1085
1088
  if (sync.status !== "uploaded")
1086
1089
  continue;
1087
1090
  for (const outcome of sync.raw_evidence_outcomes) {
1088
- if (!outcome.codex_session_id || !outcome.raw_evidence_pointer_id)
1091
+ if (!outcome.codex_session_id)
1089
1092
  continue;
1090
1093
  const source = outcome.kind === "claude_jsonl"
1091
1094
  ? "claude_code"
@@ -1094,9 +1097,16 @@ function buildBackfillSessionReport(options) {
1094
1097
  : null;
1095
1098
  if (!source)
1096
1099
  continue;
1100
+ if (!outcome.raw_evidence_pointer_id) {
1101
+ if (outcome.reason) {
1102
+ noUploadReasonBySessionId.set(outcome.codex_session_id, outcome.reason);
1103
+ }
1104
+ continue;
1105
+ }
1097
1106
  uploadByKey.set(`${source}:${outcome.codex_session_id}`, {
1098
1107
  upload_state: outcome.upload_state,
1099
1108
  raw_evidence_pointer_id: outcome.raw_evidence_pointer_id,
1109
+ reason: outcome.reason,
1100
1110
  });
1101
1111
  }
1102
1112
  }
@@ -1143,9 +1153,19 @@ function buildBackfillSessionReport(options) {
1143
1153
  ? {
1144
1154
  raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
1145
1155
  upload_state: upload.upload_state,
1156
+ ...(upload.upload_state === "upload_failed"
1157
+ ? { upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED }
1158
+ : {}),
1146
1159
  }
1147
1160
  : isRawEvidenceUploadableAttributionState(candidate.state)
1148
- ? { upload_state: "not_uploaded" }
1161
+ ? {
1162
+ upload_state: "not_uploaded",
1163
+ // BLI-2107: same rule as live sync — an attributed session with
1164
+ // no pointer always says why, even when the answer is that this
1165
+ // path never recorded one.
1166
+ upload_reason: noUploadReasonBySessionId.get(candidate.session_id) ??
1167
+ NO_UPLOAD_ATTEMPT_RECORDED,
1168
+ }
1149
1169
  : {}),
1150
1170
  };
1151
1171
  });
@@ -2215,13 +2215,25 @@ async function runSyncWithHealthReceipt(command, io) {
2215
2215
  };
2216
2216
  }
2217
2217
  try {
2218
- const exitCode = await runSyncLocked(command, io);
2218
+ const { exitCode, failureReasons } = await runSyncLocked(command, io);
2219
+ if (exitCode === 0) {
2220
+ return {
2221
+ exitCode,
2222
+ completion: { step: "sync_complete", status: "ok" },
2223
+ };
2224
+ }
2225
+ // A sync that fails by exit code says exactly as much as one that throws.
2226
+ // It used to say `sync_failed` and nothing else, so 100% of recorded
2227
+ // failure rows carried a null detail and the real reason was reachable only
2228
+ // by running `cockpit status` on the machine itself (BLI-2526).
2229
+ const reasonText = failureReasons.join("; ");
2219
2230
  return {
2220
2231
  exitCode,
2221
2232
  completion: {
2222
2233
  step: "sync_complete",
2223
- status: exitCode === 0 ? "ok" : "fail",
2224
- ...(exitCode === 0 ? {} : { error_code: "sync_failed" }),
2234
+ status: "fail",
2235
+ error_code: classifySyncHealthError(reasonText),
2236
+ error_detail: redactedSyncErrorDetail(reasonText),
2225
2237
  },
2226
2238
  };
2227
2239
  }
@@ -2264,6 +2276,19 @@ export function redactedSyncErrorDetail(error) {
2264
2276
  ? `${text.slice(0, SYNC_ERROR_DETAIL_MAX_CHARS - 1)}…`
2265
2277
  : text;
2266
2278
  }
2279
+ /**
2280
+ * Turn a finished run into an exit code and the reasons behind it.
2281
+ *
2282
+ * One place, so a future return path cannot reintroduce a code with no reason.
2283
+ * The reasons come from the run itself — the code that decided `ok` is false is
2284
+ * the only code that knows why.
2285
+ */
2286
+ function syncResult(run) {
2287
+ return {
2288
+ exitCode: run.ok ? 0 : 1,
2289
+ failureReasons: run.ok ? [] : run.failure_reasons,
2290
+ };
2291
+ }
2267
2292
  async function runSyncLocked(command, io) {
2268
2293
  const collectionRoots = await resolveSyncCollectionRoots(command);
2269
2294
  const worktrees = await discoverCommandWorktrees(collectionRoots, {
@@ -2294,7 +2319,7 @@ async function runSyncLocked(command, io) {
2294
2319
  codex_sessions: run.summary,
2295
2320
  raw_evidence_gc: gc,
2296
2321
  }, null, 2));
2297
- return run.ok ? 0 : 1;
2322
+ return syncResult(run);
2298
2323
  }
2299
2324
  writeLine(run.ok ? io.stdout : io.stderr, `Cockpit parent sync ${runStatus} ${run.outcomes.filter((outcome) => outcome.sync.status === "uploaded").length}/${run.outcomes.length} worktree(s).`);
2300
2325
  for (const outcome of run.outcomes) {
@@ -2306,7 +2331,7 @@ async function runSyncLocked(command, io) {
2306
2331
  writeAgentSessionSummary(io, run.summary);
2307
2332
  if (gc && !gc.skipped)
2308
2333
  writeLine(io.stdout, rawEvidenceGcSummary(gc));
2309
- return run.ok ? 0 : 1;
2334
+ return syncResult(run);
2310
2335
  }
2311
2336
  const result = run.outcomes[0]?.sync;
2312
2337
  if (!result) {
@@ -2322,7 +2347,7 @@ async function runSyncLocked(command, io) {
2322
2347
  codex_sessions: run.summary,
2323
2348
  raw_evidence_gc: gc,
2324
2349
  }, null, 2));
2325
- return run.ok ? 0 : 1;
2350
+ return syncResult(run);
2326
2351
  }
2327
2352
  if (run.ok) {
2328
2353
  writeLine(io.stdout, "Cockpit ambient envelope uploaded.");
@@ -2337,18 +2362,18 @@ async function runSyncLocked(command, io) {
2337
2362
  writeLine(io.stdout, cursorStatusLine(result));
2338
2363
  if (gc && !gc.skipped)
2339
2364
  writeLine(io.stdout, rawEvidenceGcSummary(gc));
2340
- return 0;
2365
+ return syncResult(run);
2341
2366
  }
2342
2367
  if (result.status === "uploaded") {
2343
2368
  writeLine(io.stderr, "Cockpit ambient upload was accepted, but session collection is partial; retry `cockpit sync`.");
2344
2369
  writeAgentSessionSummary(io, run.summary);
2345
- return 1;
2370
+ return syncResult(run);
2346
2371
  }
2347
2372
  writeLine(io.stderr, "Cockpit ambient upload failed; safe retry metadata was spooled.");
2348
2373
  writeLine(io.stderr, `Ticket: ${displayTicketId(result.ticket_id)}`);
2349
2374
  writeLine(io.stderr, `Failure: ${result.failure_reason}`);
2350
2375
  writeLine(io.stderr, `Retry: ${result.retry_command}`);
2351
- return 1;
2376
+ return syncResult(run);
2352
2377
  }
2353
2378
  async function resolveSyncCollectionRoots(command) {
2354
2379
  const explicitRoots = normalizeCollectionRoots(command.repoRoot ? [command.repoRoot] : []);
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.12");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.14");
19
19
  return 0;
20
20
  }
21
21
 
@@ -6,7 +6,7 @@
6
6
  import os from "node:os";
7
7
  import path from "node:path";
8
8
  import { getCollectorRuntimePaths, readLocalCollectorConfig, startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from "../local-state.js";
9
- import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, } from "@bli-cockpit/telemetry-core";
9
+ import { CODEX_SESSION_ATTRIBUTION_STATE_RANK, NO_UPLOAD_ATTEMPT_RECORDED, } from "@bli-cockpit/telemetry-core";
10
10
  import { flushPendingCodexSessionReports, LocalUploadBlockedError, queueCodexSessionReport, syncLocalAmbientEnvelope, } from "../upload.js";
11
11
  import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
12
12
  import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
@@ -15,6 +15,14 @@ import { CLAUDE_CURSOR_FILENAME, countStaleSessions, emptyRawEvidenceCursorState
15
15
  import { normalizeCollectionRoots } from "../root-normalization.js";
16
16
  import { clearSourceRetryFailure, readLocalUploadSpoolState, recordSourceRetryFailure, } from "../spool/local-spool.js";
17
17
  import { isLiveRawEvidenceSyncAttribution, } from "../raw-evidence-attribution-policy.js";
18
+ /**
19
+ * The label used when a sync fails and nothing on the way there said why.
20
+ *
21
+ * A deliberate sentinel rather than a fallback to `sync_failed`: it means the
22
+ * gate is real but its reason is unrecorded, which is a bug in this file, and
23
+ * it should be visible as one instead of blending into the generic bucket.
24
+ */
25
+ export const SYNC_FAILED_WITHOUT_REASON = "sync_failed_reason_not_recorded";
18
26
  const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
19
27
  const CLAUDE_DAMP_GROWTH_BYTES = 256 * 1024;
20
28
  const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
@@ -414,20 +422,56 @@ export async function runAttributedWorktreeSync(options) {
414
422
  growthDamped: dampedClaudePointers.size,
415
423
  report,
416
424
  });
417
- ok =
418
- ok &&
419
- !codexAttribution.session_limit_applied &&
420
- !claudeAttribution.session_limit_applied &&
421
- outcomes.every(({ sync }) => sync.raw_evidence_failed_count === 0 &&
422
- !sync.raw_evidence_retry_required &&
423
- sync.raw_evidence_deferred_byte_budget === 0 &&
424
- sync.raw_evidence_deferred_object_budget === 0) &&
425
- codexAttributionReadFailureCount(codexAttribution) === 0 &&
426
- claudeAttributionReadFailureCount(claudeAttribution) === 0 &&
427
- sourceScanRetryReason("codex", codexAttribution) === null &&
428
- sourceScanRetryReason("claude_code", claudeAttribution) === null &&
429
- (!reportRequired || report.posted);
430
- return { ok, outcomes, codexAttribution, claudeAttribution, summary };
425
+ // Same conditions as before, one per line, each writing down its own reason.
426
+ // The old version was a single boolean chain: correct, and completely mute.
427
+ const failureReasons = new Set();
428
+ const fail = (condition, reason) => {
429
+ if (condition)
430
+ failureReasons.add(reason);
431
+ };
432
+ for (const { worktree, sync } of outcomes) {
433
+ if (sync.status !== "uploaded") {
434
+ // The spooled reason is the most specific thing anyone has, so lead with
435
+ // it and name the worktree it belongs to — a fleet failure is usually one
436
+ // repo, and "which one" is the first question asked.
437
+ failureReasons.add(sync.status === "spooled" && sync.failure_reason
438
+ ? `${worktree.worktree_label}:${sync.failure_reason}`
439
+ : `${worktree.worktree_label}:upload_${sync.status}`);
440
+ }
441
+ for (const reason of sync.raw_evidence_failure_reasons ?? []) {
442
+ failureReasons.add(`raw_evidence:${reason}`);
443
+ }
444
+ for (const reason of sync.raw_evidence_retry_reasons ?? []) {
445
+ failureReasons.add(`raw_evidence_retry:${reason}`);
446
+ }
447
+ fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
448
+ fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
449
+ }
450
+ fail(codexAttribution.session_limit_applied, "codex_session_limit_applied");
451
+ fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
452
+ fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
453
+ fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
454
+ const codexScanRetry = sourceScanRetryReason("codex", codexAttribution);
455
+ if (codexScanRetry)
456
+ failureReasons.add(`codex_scan:${codexScanRetry}`);
457
+ const claudeScanRetry = sourceScanRetryReason("claude_code", claudeAttribution);
458
+ if (claudeScanRetry)
459
+ failureReasons.add(`claude_scan:${claudeScanRetry}`);
460
+ fail(reportRequired && !report.posted, `session_report_unposted:${report.reason ?? "unknown"}`);
461
+ // `ok` may already be false from the per-worktree loop above; the outcome
462
+ // scan re-derives that, so the two agree by construction.
463
+ ok = ok && failureReasons.size === 0;
464
+ if (!ok && failureReasons.size === 0) {
465
+ failureReasons.add(SYNC_FAILED_WITHOUT_REASON);
466
+ }
467
+ return {
468
+ ok,
469
+ failure_reasons: [...failureReasons].sort(),
470
+ outcomes,
471
+ codexAttribution,
472
+ claudeAttribution,
473
+ summary,
474
+ };
431
475
  }
432
476
  export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
433
477
  function normalizeCodexResult(result) {
@@ -474,7 +518,7 @@ function normalizeClaudeResult(result) {
474
518
  * session's upload state (D3). Damped Claude sessions report `reused_existing`
475
519
  * carrying their prior durable pointer.
476
520
  */
477
- function buildAgentSessionReport(options) {
521
+ export function buildAgentSessionReport(options) {
478
522
  const normalized = [
479
523
  ...options.codexResults.map(normalizeCodexResult),
480
524
  ...options.claudeResults.map(normalizeClaudeResult),
@@ -495,11 +539,23 @@ function buildAgentSessionReport(options) {
495
539
  // Main-file outcomes only (D3): a sidecar making it must never mark a session
496
540
  // uploaded when the main did not.
497
541
  const uploadByKey = new Map();
542
+ // BLI-2107: why a session got no pointer, keyed the same way. The reasons
543
+ // already existed — file_too_large, deferred_byte_budget, the redaction
544
+ // guards — but only as aggregate skip counts in the health report, so no
545
+ // individual session could say what happened to it.
546
+ const noUploadReasonByKey = new Map();
547
+ // A worktree sync that did not finish carries no per-session outcomes, and
548
+ // WorktreeSyncOutcome does not record which sessions it was going to cover.
549
+ // So this pass knows only that it was degraded, not which session each
550
+ // failure belonged to — and says exactly that rather than picking one.
551
+ let anySyncIncomplete = false;
498
552
  for (const outcome of options.outcomes) {
499
- if (outcome.sync.status !== "uploaded")
553
+ if (outcome.sync.status !== "uploaded") {
554
+ anySyncIncomplete = true;
500
555
  continue;
556
+ }
501
557
  for (const upload of outcome.sync.raw_evidence_outcomes) {
502
- if (!upload.codex_session_id || !upload.raw_evidence_pointer_id)
558
+ if (!upload.codex_session_id)
503
559
  continue;
504
560
  const source = upload.kind === "claude_jsonl"
505
561
  ? "claude_code"
@@ -508,7 +564,15 @@ function buildAgentSessionReport(options) {
508
564
  : null;
509
565
  if (!source)
510
566
  continue; // sidecars and other kinds do not set session state
511
- uploadByKey.set(`${source}:${upload.codex_session_id}`, upload);
567
+ const key = `${source}:${upload.codex_session_id}`;
568
+ if (!upload.raw_evidence_pointer_id) {
569
+ // An outcome with no pointer is a failure that named itself. Keep the
570
+ // reason even though there is nothing to point at.
571
+ if (upload.reason)
572
+ noUploadReasonByKey.set(key, upload.reason);
573
+ continue;
574
+ }
575
+ uploadByKey.set(key, upload);
512
576
  }
513
577
  }
514
578
  return [...bestByKey.values()].map((result) => {
@@ -548,6 +612,13 @@ function buildAgentSessionReport(options) {
548
612
  ? {
549
613
  raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
550
614
  upload_state: upload.upload_state,
615
+ // A failed upload names itself; a successful one has nothing to
616
+ // explain.
617
+ ...(upload.upload_state === "upload_failed"
618
+ ? {
619
+ upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED,
620
+ }
621
+ : {}),
551
622
  }
552
623
  : priorDurablePointer
553
624
  ? {
@@ -556,7 +627,19 @@ function buildAgentSessionReport(options) {
556
627
  }
557
628
  : result.state === "attributed" ||
558
629
  result.state === "attributed_fallback"
559
- ? { upload_state: "not_uploaded" }
630
+ ? {
631
+ upload_state: "not_uploaded",
632
+ // BLI-2107: `not_uploaded` used to be the branch of last
633
+ // resort, recording that nothing happened and never why. It
634
+ // now always carries a cause, even when the cause is that we
635
+ // have none — a session labelled NO_UPLOAD_ATTEMPT_RECORDED
636
+ // is a path that still needs instrumenting, and saying so is
637
+ // the point.
638
+ upload_reason: noUploadReasonByKey.get(key) ??
639
+ (anySyncIncomplete
640
+ ? "sync_incomplete_this_pass"
641
+ : NO_UPLOAD_ATTEMPT_RECORDED),
642
+ }
560
643
  : {}),
561
644
  };
562
645
  });
@@ -1,4 +1,4 @@
1
- import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, } from "@bli-cockpit/telemetry-core";
1
+ import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
2
2
  import crypto from "node:crypto";
3
3
  import fs from "node:fs/promises";
4
4
  const DEFAULT_MAX_ATTEMPTS = 3;
@@ -214,6 +214,14 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
214
214
  object_key: objectKey,
215
215
  });
216
216
  if (!commit.ok) {
217
+ // The server can tell us this object will be refused again. Take its reason
218
+ // verbatim so the label names the cause ("storage rejected 116 MB") instead
219
+ // of the transport ("commit_failed_http_500"), which is all the fleet could
220
+ // say for the 57 days of BLI-2528.
221
+ const permanent = permanentCommitRejection(commit.body);
222
+ if (permanent) {
223
+ return failedOutcome(entry.file, permanent, uploadedChunks);
224
+ }
217
225
  const detail = safeFailureDetail(commit.body);
218
226
  return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
219
227
  }
@@ -376,6 +384,24 @@ function failedOutcome(file, reason, uploadedChunks = 0) {
376
384
  uploaded_chunk_count: uploadedChunks,
377
385
  };
378
386
  }
387
+ /**
388
+ * The server's verdict that repeating this commit is pointless, or null.
389
+ *
390
+ * Two conditions, both required: the response says `retryable: false`, and the
391
+ * reason is one this collector has classified as permanent. The second check is
392
+ * the important one. A newer dashboard could declare a reason this CLI has never
393
+ * heard of, and quietly abandoning an object on a word we cannot interpret is
394
+ * exactly the silent drop the fleet contract forbids — so an unclassified reason
395
+ * falls through to the ordinary retry path and stays visible.
396
+ */
397
+ function permanentCommitRejection(body) {
398
+ if (!body || typeof body !== "object")
399
+ return null;
400
+ if (body.retryable !== false)
401
+ return null;
402
+ const reason = safeFailureDetail(body);
403
+ return reason && isPermanentUploadFailure(reason) ? reason : null;
404
+ }
379
405
  function safeFailureDetail(body) {
380
406
  if (!body || typeof body !== "object")
381
407
  return null;
package/dist/upload.js CHANGED
@@ -1,4 +1,4 @@
1
- import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, } from "@bli-cockpit/telemetry-core";
1
+ import { AgentImageArtifactReportRequestSchema, CODEX_SESSION_REPORT_MAX_SESSIONS, CodexSessionAttributionReportResponseSchema, EvidenceCompletenessPayloadSchema, TelemetryIngestEnvelopeSchema, TelemetryIngestEventDtoSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
2
2
  import path from "node:path";
3
3
  import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, } from "./local-state.js";
4
4
  import { runLocalSourceCollectors } from "./adapters/local-sources.js";
@@ -236,6 +236,27 @@ export async function syncLocalAmbientEnvelope(options = {}) {
236
236
  retry_command: "cockpit sync",
237
237
  });
238
238
  }
239
+ else {
240
+ // No retry to queue, but a permanently rejected object still has to say
241
+ // so. `recordUploadBlocked` names the reason without spooling a retry —
242
+ // the machine reports the failure instead of promising a fix it cannot
243
+ // deliver, and `cockpit status` stops reading clean.
244
+ const permanentReason = permanentEvidenceFailureReason(uploadOutcomes);
245
+ if (permanentReason) {
246
+ await recordUploadBlocked(paths, {
247
+ attemptedAt,
248
+ reason: permanentReason,
249
+ });
250
+ // stderr, which launchd captures to `sync.err.log`, so an unattended
251
+ // machine leaves a dated record of the objects it gave up on. Reason
252
+ // labels and counts only — never a path or a byte of content.
253
+ console.error("[cockpit-sync] raw evidence permanently rejected", JSON.stringify({
254
+ attempted_at: attemptedAt,
255
+ reason: permanentReason,
256
+ object_count: permanentFailedOutcomes(uploadOutcomes).length,
257
+ }));
258
+ }
259
+ }
239
260
  return {
240
261
  status: "uploaded",
241
262
  dashboard_url: built.dashboard_url,
@@ -307,8 +328,37 @@ function retrySourcesForFailedSync(options, facts) {
307
328
  }
308
329
  return [...sources];
309
330
  }
331
+ /**
332
+ * The failed uploads a later attempt could still rescue.
333
+ *
334
+ * An object storage has already refused on its own terms is not one of them,
335
+ * and counting it as one is what kept Edward's Mac in `retry_pending` through
336
+ * 13 consecutive syncs that were never going to end differently (BLI-2528).
337
+ */
338
+ function retryableFailedOutcomes(outcomes) {
339
+ return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
340
+ !isPermanentUploadFailure(outcome.reason));
341
+ }
342
+ /** Failed uploads that no retry can rescue, kept so they can still be named. */
343
+ function permanentFailedOutcomes(outcomes) {
344
+ return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
345
+ isPermanentUploadFailure(outcome.reason));
346
+ }
347
+ /**
348
+ * The reason to show for objects that failed for good.
349
+ *
350
+ * Returns null when there are none. These never queue a retry, but they must
351
+ * never disappear either: a machine with a permanently rejected object has
352
+ * missing collection, and a status that reads clean would hide it.
353
+ */
354
+ function permanentEvidenceFailureReason(outcomes) {
355
+ const reasons = new Set(permanentFailedOutcomes(outcomes).map((outcome) => outcome.reason ?? "unknown"));
356
+ if (reasons.size === 0)
357
+ return null;
358
+ return `raw_evidence_permanently_rejected:${[...reasons].sort().join(",")}`;
359
+ }
310
360
  function hasRetryableEvidenceGap(facts, outcomes) {
311
- if (outcomes.some((outcome) => outcome.upload_state === "upload_failed")) {
361
+ if (retryableFailedOutcomes(outcomes).length > 0) {
312
362
  return true;
313
363
  }
314
364
  if (!facts)
@@ -326,10 +376,8 @@ function hasRetryableEvidenceGap(facts, outcomes) {
326
376
  }
327
377
  function retryableEvidenceGapReason(facts, outcomes) {
328
378
  const reasons = new Set();
329
- for (const outcome of outcomes) {
330
- if (outcome.upload_state === "upload_failed") {
331
- reasons.add(outcome.reason ?? "upload_failed");
332
- }
379
+ for (const outcome of retryableFailedOutcomes(outcomes)) {
380
+ reasons.add(outcome.reason ?? "upload_failed");
333
381
  }
334
382
  if (facts) {
335
383
  if (facts.deferred_byte_budget_count > 0)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.12",
3
+ "version": "0.2.14",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -26,6 +26,6 @@
26
26
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
27
27
  },
28
28
  "dependencies": {
29
- "@bli-cockpit/telemetry-core": "0.1.16"
29
+ "@bli-cockpit/telemetry-core": "0.1.17"
30
30
  }
31
31
  }