@bli-cockpit/cli 0.2.104 → 0.2.108

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,7 +18,14 @@ const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
18
18
  */
19
19
  export async function uploadBackfillBatches(command, io, lock, ctx) {
20
20
  const uploadable = uploadableCandidates(ctx.scan.candidates);
21
- const batches = buildBackfillBatches(uploadable);
21
+ // Doctor needs a boundary after each session. Ordinary backfill retains
22
+ // its existing batching, and scheduled sync does not use this command.
23
+ const plannedBatches = buildBackfillBatches(uploadable);
24
+ const batches = command.deadlineMs !== undefined || command.onProgress
25
+ ? plannedBatches.flatMap((batch) => batch.candidates.map((candidate) => ({
26
+ worktree: batch.worktree, candidates: [candidate],
27
+ })))
28
+ : plannedBatches;
22
29
  const rawEvidenceBudget = {
23
30
  remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET,
24
31
  remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET,
@@ -34,10 +41,33 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
34
41
  done: tally.done,
35
42
  total: uploadable.length,
36
43
  });
44
+ const reportProgress = () => {
45
+ if (!command.onProgress)
46
+ return;
47
+ const unresolved = new Set([
48
+ ...uploadable.map(candidateCursorKey).filter((key) => !tally.durableCandidateKeys.has(key)),
49
+ ...ctx.scan.retryable_candidate_keys,
50
+ ]);
51
+ command.onProgress({
52
+ processed: tally.done,
53
+ uploaded: tally.backfilledSessions,
54
+ remaining: unresolved.size + ctx.scan.omitted_candidate_count + ctx.scan.issues
55
+ .filter((issue) => issue.scope === "global")
56
+ .reduce((total, issue) => total + issue.count, 0),
57
+ elapsedMs: Date.now() - ctx.now.getTime(),
58
+ });
59
+ };
60
+ reportProgress();
37
61
  for (const [index, batch] of batches.entries()) {
38
62
  await lock.handle.heartbeat();
63
+ if (command.deadlineMs !== undefined && Date.now() >= command.deadlineMs) {
64
+ blockedAt = stoppedAt("backfill time budget reached", index);
65
+ failureReason = "deferred_budget_exhausted";
66
+ break;
67
+ }
39
68
  const outcome = await uploadOneBackfillBatch(command, io, ctx, batch, rawEvidenceBudget);
40
69
  foldBatchIntoTally(tally, batch, outcome);
70
+ reportProgress();
41
71
  consecutiveFailures = outcome.failed ? consecutiveFailures + 1 : 0;
42
72
  if (!command.json) {
43
73
  writeLine(io.stdout, `Uploaded ${tally.done}/${uploadable.length} (batch ${index + 1}/${batches.length})`);
@@ -49,7 +49,7 @@ export async function runClean(command, io) {
49
49
  writeLine(io.stdout, line);
50
50
  }
51
51
  const footprint = await readDiskFootprint(paths, now);
52
- const plan = await planFromDisk(paths, env, { allCommitted: command.allCommitted }, now);
52
+ const plan = await planFromDisk(paths, env, { allCommitted: command.allCommitted, reclaimTerminal: true }, now);
53
53
  const vaults = command.allCommitted
54
54
  ? await vaultsSafeToRemove(paths, footprint.vaults)
55
55
  : [];
@@ -69,6 +69,7 @@ export async function runClean(command, io) {
69
69
  now,
70
70
  force: true,
71
71
  allCommitted: command.allCommitted,
72
+ reclaimTerminal: true,
72
73
  });
73
74
  // The two smaller footprints go with it: a rotation pass sweeps the archives
74
75
  // outside the keep window, and each removed vault is named on its own line.
@@ -84,7 +84,7 @@ export function rawEvidenceSyncLine(sync) {
84
84
  const stuck = sync.raw_evidence_stuck_object_count > 0
85
85
  ? `, ${sync.raw_evidence_stuck_object_count} stuck (${sync.raw_evidence_max_delivery_attempts} tries since ${sync.raw_evidence_oldest_delivery_failure_at ?? "unknown"})`
86
86
  : "";
87
- return `Files: ${sync.raw_evidence_uploaded_object_count} uploaded, ${sync.raw_evidence_reused_count} already there, ${sync.raw_evidence_failed_count} failed${held}${stuck}${failures}${retries}`;
87
+ return `Files: ${sync.raw_evidence_uploaded_object_count} uploaded, ${sync.raw_evidence_reused_count} already there, ${sync.raw_evidence_failed_count} failed, reconciled_committed_by_409=${sync.reconciled_committed_by_409 ?? 0}${held}${stuck}${failures}${retries}`;
88
88
  }
89
89
  export function cursorStatusLine(sync) {
90
90
  return `Tracked so far: ${sync.cursor_tracked_object_count} uploaded item(s)`;
@@ -118,6 +118,7 @@ export function worktreeSyncRow(outcome, run) {
118
118
  raw_evidence_uploaded_object_count: sync.raw_evidence_uploaded_object_count,
119
119
  raw_evidence_uploaded_chunk_count: sync.raw_evidence_uploaded_chunk_count,
120
120
  raw_evidence_reused_count: sync.raw_evidence_reused_count,
121
+ reconciled_committed_by_409: sync.reconciled_committed_by_409 ?? 0,
121
122
  raw_evidence_failed_count: sync.raw_evidence_failed_count,
122
123
  raw_evidence_failure_reasons: sync.raw_evidence_failure_reasons,
123
124
  raw_evidence_retry_required: sync.raw_evidence_retry_required,
@@ -38,6 +38,9 @@ export function diskRowMessage(footprint, capBytes) {
38
38
  // have been here — on the reference Mac the answer was
39
39
  // `reconciled_unknown_to_server` on 234 objects whose oldest was 45 days old,
40
40
  // which is a different problem from a delivery that failed this morning.
41
+ if (staging.terminal_count) {
42
+ parts.push(`${staging.terminal_count} terminal object(s), ${mib(staging.terminal_bytes ?? 0)} MB excluded from cap; reclaimable with \`cockpit clean\`: ${Object.keys(staging.terminal_reasons ?? {}).join(", ")}`);
43
+ }
41
44
  const dominant = staging.dominant_uncommitted_reason;
42
45
  if (dominant) {
43
46
  parts.push(`oldest undelivered ${days(dominant.oldest_disk_age_ms)}d; biggest reason ${dominant.reason} on ${dominant.count} object(s), ${mib(dominant.bytes)} MB`);
@@ -235,4 +235,14 @@ function positiveNumberOrZero(value) {
235
235
  return typeof value === "number" && Number.isFinite(value) && value > 0
236
236
  ? value
237
237
  : 0;
238
+ }
239
+ /** Names only the sync receipt's recorded causes. Missing evidence names itself. */
240
+ export function syncFailureMessage(parsed) {
241
+ const reasons = typeof parsed?.failure_reason === "string"
242
+ ? [parsed.failure_reason]
243
+ : Array.isArray(parsed?.failure_reasons)
244
+ ? parsed.failure_reasons.filter((value) => typeof value === "string" && value.length > 0)
245
+ : [];
246
+ const reason = reasons.join("; ") || "sync_failure_receipt_missing";
247
+ return /staging_cap/u.test(reason) ? `${reason}; see disk-bounded and run \`cockpit clean --dry-run\`` : reason;
238
248
  }
@@ -13,7 +13,7 @@ import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.
13
13
  import { runBackfillCommand } from "./backfill.js";
14
14
  import { diskRowMessage, mib, redeliveryLine } from "./doctor-disk-words.js";
15
15
  import { doctorRoots } from "./doctor-access.js";
16
- import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
16
+ import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
17
17
  import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
18
18
  /**
19
19
  * The `backfill-complete`, `gc-checked`, `disk-bounded`, and `sync-fresh`
@@ -50,15 +50,29 @@ export async function fixBackfillState(context) {
50
50
  const started = Date.now();
51
51
  const deadline = started + (context.command.backfillBudgetSeconds ?? 900) * 1000;
52
52
  let caughtUp = 0;
53
+ let chunks = 0;
54
+ let live = null;
55
+ const elapsed = () => {
56
+ const seconds = Math.floor((Date.now() - started) / 1000);
57
+ return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, "0")}s`;
58
+ };
59
+ const receipt = () => `caught up on ${caughtUp} old sessions in ${chunks} chunk${chunks === 1 ? "" : "s"} over ${elapsed()}`;
53
60
  let remaining = null;
54
61
  const unfinished = (code, reason) => ({
55
- ...needsFix("backfill-complete", code, `caught up on ${caughtUp} old sessions this run, ${remaining ?? "unknown"} still to go; ${reason}; run \`cockpit doctor\` again to continue`),
62
+ ...needsFix("backfill-complete", code, `${receipt()}, ${remaining ?? "unknown"} still to go; ${reason}; run \`cockpit doctor\` again to continue`),
56
63
  nextAction: "cockpit doctor",
57
64
  });
58
65
  const progress = setInterval(() => {
59
66
  const seconds = Math.floor((Date.now() - started) / 1000);
60
- context.io.stderr.write(`backfill: ${caughtUp} caught up, ${remaining ?? "unknown"} to go, ${Math.floor(seconds / 60)}m${seconds % 60}s\n`);
61
- console.error("[doctor] backfill-complete progress", JSON.stringify({ caught_up: caughtUp, remaining, elapsed_seconds: seconds }));
67
+ const uploaded = live === null ? null : caughtUp + live.uploaded;
68
+ context.io.stderr.write(live === null
69
+ ? `backfill: starting, ${elapsed()}\n`
70
+ : `backfill: ${uploaded} caught up, ${live.processed} processed in this chunk, ${live.remaining} to go, ${elapsed()}\n`);
71
+ console.error("[doctor] backfill-complete progress", JSON.stringify({
72
+ status: live === null ? "starting" : "running", processed: live?.processed ?? null,
73
+ uploaded, caught_up: uploaded, remaining: live?.remaining ?? null,
74
+ chunks, elapsed_seconds: seconds,
75
+ }));
62
76
  }, 30_000);
63
77
  try {
64
78
  while (Date.now() < deadline) {
@@ -71,20 +85,25 @@ export async function fixBackfillState(context) {
71
85
  if (Date.now() >= deadline)
72
86
  return unfinished("backfill_budget_timeout", "backfill time budget reached");
73
87
  const capture = capturedIo(context.io, false);
88
+ live = null;
89
+ chunks += 1;
74
90
  const code = await runBackfillCommand({
75
91
  homeDir: context.command.homeDir, repoRoot: context.command.repoRoot,
76
92
  all: true, dryRun: false, yes: true, json: true,
93
+ deadlineMs: deadline,
94
+ onProgress: (progress) => { live = progress; },
77
95
  }, capture.io);
78
96
  const parsed = parseDoctorBackfillJson(capture.stdout());
79
97
  const counts = asRecord(parsed?.counts);
80
98
  const count = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
81
99
  caughtUp += count(counts?.["backfilled"]) ?? 0;
82
100
  remaining = count(counts?.["remaining"]) ?? remaining;
101
+ live = null;
83
102
  if (code === 0 && parsed?.status === "complete") {
84
103
  // Reporting totals include permanent caps. Preserve their marker
85
104
  // note instead of retrying history the collector says is complete.
86
105
  const checked = await checkBackfillState(context);
87
- return ok("backfill-complete", checked.status === "ok" ? checked.code : "complete", `caught up on ${caughtUp} old sessions this run` + (checked.status === "ok" ? `; ${checked.message}` : ""));
106
+ return ok("backfill-complete", checked.status === "ok" ? checked.code : "complete", receipt() + (checked.status === "ok" ? `; ${checked.message}` : ""));
88
107
  }
89
108
  const verdict = backfillFixVerdict(parsed, jsonField(capture.stderr(), "failure_reason"));
90
109
  if (parsed?.status === "partial" &&
@@ -94,7 +113,7 @@ export async function fixBackfillState(context) {
94
113
  return { ...verdict, message: typeof parsed?.failure_reason === "string" ? parsed.failure_reason : "backfill failed without a valid completion receipt" };
95
114
  });
96
115
  if (row.status === "ok") {
97
- console.error("[doctor] backfill-complete finished", JSON.stringify({ caught_up: caughtUp, remaining, elapsed_seconds: Math.floor((Date.now() - started) / 1000) }));
116
+ console.error("[doctor] backfill-complete finished", JSON.stringify({ caught_up: caughtUp, remaining, chunks, elapsed_seconds: Math.floor((Date.now() - started) / 1000) }));
98
117
  return row;
99
118
  }
100
119
  if (row.code !== "backfill_chunk_pending") {
@@ -146,7 +165,7 @@ export async function checkDiskState(context) {
146
165
  const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
147
166
  const { capBytes } = retentionOptionsFromEnv(context.io.env);
148
167
  const message = diskRowMessage(footprint, capBytes);
149
- return footprint.staging.total_bytes > capBytes
168
+ return footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes
150
169
  ? needsFix("disk-bounded", "over_staging_cap", message)
151
170
  : ok("disk-bounded", "within_staging_cap", message);
152
171
  }
@@ -179,10 +198,8 @@ export async function fixDiskState(context) {
179
198
  if (!pruned.cap_blocked_by_uncommitted) {
180
199
  return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
181
200
  }
182
- // Deliberately still `ok`: staging over the cap because evidence has not been
183
- // accepted yet is the collector working, not a machine to repair. The row
184
- // names the blockage and the one command that goes further.
185
- return ok("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest — ${redelivered} — or \`cockpit clean --all-committed\` to drop every accepted copy now`);
201
+ // Unaccepted evidence still occupies the cap. Keep the disk row actionable.
202
+ return needsFix("disk-bounded", "staging_cap_blocked_by_uncommitted", `freed ${mib(pruned.deleted_bytes)} MB; ${message}; ${pruned.cap_blocked_count} object(s) the upload ledger cannot vouch for are holding the rest; ${redelivered}; or \`cockpit clean --all-committed\` to drop every accepted copy now`);
186
203
  }
187
204
  function capturedIo(io, forward) {
188
205
  const stdoutChunks = [];
@@ -278,7 +295,11 @@ async function runSyncRepair(context) {
278
295
  `file${draining.remainingObjects === 1 ? "" : "s"} left for later this run); ` +
279
296
  "rerun `cockpit sync` to continue");
280
297
  }
281
- return fail("sync-fresh", status ?? "sync_failed", `sync failed for ${repoRoot}`);
298
+ return fail("sync-fresh", status ?? "sync_failed", `${repoRoot}: ${syncFailureMessage(parsed)}`);
299
+ }
300
+ if (parsed?.collection_complete === false && typeof parsed.discovery_notice === "string") {
301
+ console.error("[doctor] partial discovery", JSON.stringify({ reason: "discovery_partial" }));
302
+ return needsFix("sync-fresh", "discovery_partial", parsed.discovery_notice);
282
303
  }
283
304
  if (status !== "uploaded") {
284
305
  return fail("sync-fresh", status ?? "sync_unverified", `sync did not return an uploaded receipt for ${repoRoot}`);
@@ -1,8 +1,13 @@
1
1
  import { optionalNonEmpty, optionalUrl, parseNamedArgs } from "./local-arg-values.js";
2
2
  export function parseUsageArgs(args) {
3
- const values = parseNamedArgs(args, { allowedFlags: ["--by-repo", "--person", "--all", "--since", "--until", "--include-automated", "--detail", "--home", "--dashboard-url", "--json"], valueFlags: ["--person", "--since", "--until", "--home", "--dashboard-url"] });
3
+ const values = parseNamedArgs(args, { allowedFlags: ["--min-confidence", "--by-topic", "--by-repo", "--person", "--all", "--since", "--until", "--include-automated", "--detail", "--home", "--dashboard-url", "--json"], valueFlags: ["--min-confidence", "--person", "--since", "--until", "--home", "--dashboard-url"] });
4
+ if (values.booleans.has("--by-repo") && values.booleans.has("--by-topic"))
5
+ throw new Error("--by-repo and --by-topic cannot be used together.");
6
+ const minConfidence = values.flags.get("--min-confidence");
7
+ if (minConfidence !== undefined && !["high", "medium", "low"].includes(minConfidence))
8
+ throw new Error("--min-confidence accepts high, medium or low.");
4
9
  const action = values.positionals[0] ?? "people";
5
10
  if (action !== "people" || values.positionals.length > 1)
6
11
  throw new Error("usage takes one verb: people.");
7
- return { kind: "usage", byRepo: values.booleans.has("--by-repo"), person: optionalNonEmpty(values.flags.get("--person")), all: values.booleans.has("--all"), action: "people", since: optionalNonEmpty(values.flags.get("--since")) ?? "30d", until: optionalNonEmpty(values.flags.get("--until")), detail: values.booleans.has("--detail"), includeAutomated: values.booleans.has("--include-automated"), homeDir: optionalNonEmpty(values.flags.get("--home")), dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")), json: values.booleans.has("--json") };
12
+ return { kind: "usage", ...(minConfidence ? { minConfidence: minConfidence } : {}), byTopic: values.booleans.has("--by-topic"), byRepo: values.booleans.has("--by-repo"), person: optionalNonEmpty(values.flags.get("--person")), all: values.booleans.has("--all"), action: "people", since: optionalNonEmpty(values.flags.get("--since")) ?? "30d", until: optionalNonEmpty(values.flags.get("--until")), detail: values.booleans.has("--detail"), includeAutomated: values.booleans.has("--include-automated"), homeDir: optionalNonEmpty(values.flags.get("--home")), dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")), json: values.booleans.has("--json") };
8
13
  }
@@ -20,12 +20,6 @@ export async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
20
20
  }
21
21
  const worktrees = result.worktrees;
22
22
  if (!result.complete) {
23
- // Sync fails closed here ON PURPOSE, and that is not the bug. Advancing a
24
- // cursor after a partial scan would mark the run as covering repos it
25
- // never saw, permanently skipping their sessions — backfill can tolerate
26
- // partial only because it keeps per-scope completion markers, and sync
27
- // does not. The bug (BLI-2362) was that the refusal named no roots and
28
- // gave no runnable command, so a big workspace just stayed red forever.
29
23
  const message = incompleteDiscoveryMessage({
30
24
  result,
31
25
  roots,
@@ -35,9 +29,10 @@ export async function discoverCommandWorktrees(repoRoot, discovery = {}, io) {
35
29
  });
36
30
  if (io)
37
31
  writeLine(io.stderr, message);
38
- throw new Error(message);
32
+ discovery.onPartial?.(message);
33
+ console.error("[local-discovery] partial collection", JSON.stringify({ reason: result.incomplete_reasons.join(","), roots: roots.length, incomplete_roots: result.incomplete_roots.length, max_depth: limits.maxDepth, found: worktrees.length }));
39
34
  }
40
- if (worktrees.length === 0 && !discovery.allowEmpty) {
35
+ if (worktrees.length === 0 && result.complete && !discovery.allowEmpty) {
41
36
  // "No git repos found" is only true when the scan could actually SEE
42
37
  // everywhere it looked. With unreadable folders in hand, the sentence sent
43
38
  // the operator to the wrong repair — "run from a git repo", when the repos
@@ -104,17 +99,17 @@ function incompleteDiscoveryMessage(input) {
104
99
  `--max-repos ${nextRepos}`,
105
100
  ].join(" ");
106
101
  return [
107
- `Tower could not finish scanning for repos, so it stopped instead of collecting a partial picture (${result.incomplete_reasons.join(", ")}).`,
108
- "It stops rather than continuing because a partial scan would mark these repos as already checked and skip them from now on.",
102
+ `TOWER collection continued with partial repo discovery (${result.incomplete_reasons.join(", ")}).`,
103
+ `Fully scanned ${roots.length - blocked.length} of ${roots.length} roots to depth ${maxDepth}. Sessions under all approved roots remain eligible for collection.`,
109
104
  "",
110
105
  "Could not fully scan:",
111
106
  ...blocked.map((root) => ` ${root}`),
112
107
  "",
113
- `Found ${found} repo(s) before stopping, with --max-depth ${maxDepth} and --max-repos ${maxRepos}.`,
108
+ `Found ${found} repo(s) before the cap, with --max-depth ${maxDepth} and --max-repos ${maxRepos}.`,
114
109
  "",
115
110
  "Run this to raise the limits and try again:",
116
111
  ` ${retry}`,
117
112
  "",
118
- "If that still stops, the folder is deeper or larger than expected — raise the numbers again, or point --workspace at the specific project folders instead of a parent.",
113
+ "If the scan is still partial, raise the numbers again, or point --workspace at the specific project folders instead of a parent.",
119
114
  ].join("\n");
120
115
  }
@@ -143,9 +143,9 @@ export const TOWER_COMMAND_HELP = [
143
143
  [
144
144
  "usage",
145
145
  [
146
- "Usage: cockpit usage people [--by-repo] [--person <email|me>] [--all] [--since <n>d|<n>h|<iso>] [--detail] [--until <iso>] [--include-automated] [--json]",
146
+ "Usage: cockpit usage people [--by-repo | --by-topic] [--min-confidence high|medium|low] [--person <email|me>] [--all] [--since <n>d|<n>h|<iso>] [--detail] [--until <iso>] [--include-automated] [--json]",
147
147
  "",
148
- "--by-repo adds project rows under each person (top 10; --all shows every repo). --person me uses your signed-in email.",
148
+ "--by-topic adds topic rows, including (unlabelled). Choose only one grouping. --by-repo adds project rows under each person (top 10; --all shows every repo). --person me uses your signed-in email.",
149
149
  "Claude Code and Codex usage per person: sessions observed and extracted, tokens (total, output,",
150
150
  "and the input / cache split when the row carries it), and an API list-price equivalent that is",
151
151
  "labelled as such and is never actual spend. A super_admin sees everyone; a member sees their own row.",
@@ -102,7 +102,7 @@ export function localCommandHelp(command) {
102
102
  ` cockpit search "<words>" [--kind ${SEARCH_KINDS.join(",")}] [--limit <n>] [--dashboard-url <url>] [--json]`,
103
103
  " cockpit release [--dry-run] [--skip-checks] [--no-floor] [--tag <tag>] [--access <public|restricted>] [--otp <code>]",
104
104
  " cockpit careers [list|show <id>|rescreen <id>] [--role <slug>] [--min-score <n>] [--since <date>] [--json]",
105
- " cockpit usage people [--by-repo] [--person <email|me>] [--all] [--detail] [--since <n>d|<n>h|<iso>] [--until <iso>] [--include-automated] [--dashboard-url <url>] [--json]",
105
+ " cockpit usage people [--by-repo | --by-topic] [--min-confidence high|medium|low] [--person <email|me>] [--all] [--detail] [--since <n>d|<n>h|<iso>] [--until <iso>] [--include-automated] [--dashboard-url <url>] [--json]",
106
106
  "",
107
107
  `Default dashboard: ${DEFAULT_DASHBOARD_URL}. Omit --dashboard-url for normal production use; pass it only for staging/custom dashboards or to force a different pairing.`,
108
108
  ].join("\n");
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
15
15
  }
16
16
 
17
17
  if (command === "--version" || command === "-V" || command === "version") {
18
- writeLine(io?.stdout ?? process.stdout, "0.2.104");
18
+ writeLine(io?.stdout ?? process.stdout, "0.2.108");
19
19
  return 0;
20
20
  }
21
21
 
@@ -5,17 +5,20 @@ import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-
5
5
  /**
6
6
  * BLI-2601: the fleet keeps itself current on npm `latest` without anyone
7
7
  * re-running `npm i -g @bli-cockpit/cli && cockpit doctor` by hand after day 0. This always
8
- * runs AFTER `runSync` has already decided and reported collection's own
9
- * outcome above a stuck or failing self-update can never block or delay
10
- * collection, and a collection failure never blocks the chance to
11
- * self-update. Every error path here is swallowed on purpose: a failure is
8
+ * runs before collection, so a failing or slow scan cannot delay the floor pull.
9
+ * The replacement process skips this check and continues the same tick. Every error path here is swallowed on purpose: a failure is
12
10
  * reported as its own named `update` receipt, never surfaced as a `sync`
13
11
  * failure or thrown from this function.
14
12
  */
15
13
  export async function runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion) {
14
+ if (io.env?.COCKPIT_SYNC_REEXEC === "1")
15
+ return false;
16
+ let updated = false;
16
17
  let event;
17
18
  try {
18
- event = await runScheduledSelfUpdateForSync(command, io, minCliVersion);
19
+ const result = await runScheduledSelfUpdateForSync(command, io, minCliVersion);
20
+ updated = result?.reason === "updated";
21
+ event = result ? scheduledSelfUpdateInstallEvent(result) : null;
19
22
  }
20
23
  catch (error) {
21
24
  // The throttle/probe/install machinery below is defensive already; this
@@ -29,7 +32,7 @@ export async function runScheduledSelfUpdateAfterSync(command, io, dashboardUrl,
29
32
  };
30
33
  }
31
34
  if (!event)
32
- return;
35
+ return false;
33
36
  await reportInstallEventsBestEffort({
34
37
  homeDir: command.homeDir,
35
38
  dashboardUrl,
@@ -38,6 +41,7 @@ export async function runScheduledSelfUpdateAfterSync(command, io, dashboardUrl,
38
41
  json: command.json,
39
42
  io,
40
43
  });
44
+ return updated;
41
45
  }
42
46
  async function runScheduledSelfUpdateForSync(command, io, minCliVersion) {
43
47
  const rawExec = io.exec;
@@ -60,7 +64,7 @@ async function runScheduledSelfUpdateForSync(command, io, minCliVersion) {
60
64
  currentVersion: LOCAL_COLLECTOR_VERSION,
61
65
  install: (tag) => attemptScheduledSelfUpdateInstall(scheduledIo, tag),
62
66
  }, { env: io.env, minVersion: minCliVersion });
63
- return scheduledSelfUpdateInstallEvent(result);
67
+ return result;
64
68
  }
65
69
  async function attemptScheduledSelfUpdateInstall(io, tag) {
66
70
  try {
@@ -125,7 +125,7 @@ export function failedReceipt(run) {
125
125
  // an operator reading a red row still has to be able to see what is merely
126
126
  // waiting — otherwise the held objects become invisible the moment anything
127
127
  // else goes wrong, which is the opposite of the fix.
128
- const reasonText = [run.holdNotice, ...run.failureReasons]
128
+ const reasonText = [run.notice, run.holdNotice, ...run.failureReasons]
129
129
  .filter(Boolean)
130
130
  .join("; ");
131
131
  // The bucket comes from the records the deciding branches wrote, not from
@@ -16,7 +16,10 @@ export async function reportMultiRepoSync(command, io, run, dedup) {
16
16
  writeLine(io.stdout, JSON.stringify({
17
17
  mode: "multi_repo",
18
18
  status: collectionRunStatus,
19
- collection_complete: run.ok,
19
+ collection_complete: run.ok && !run.discovery_partial,
20
+ discovery_notice: run.notice,
21
+ failure_reasons: run.failure_reasons,
22
+ failure_records: run.failure_records,
20
23
  results: run.outcomes.map((outcome) => outcome.sync),
21
24
  repos: rows,
22
25
  codex_sessions: run.summary,
@@ -45,7 +48,10 @@ export async function reportNoWorktreeSync(command, io, run, dedup) {
45
48
  writeLine(io.stdout, JSON.stringify({
46
49
  mode: "no_worktrees",
47
50
  status: collectionRunStatus,
48
- collection_complete: run.ok,
51
+ collection_complete: run.ok && !run.discovery_partial,
52
+ discovery_notice: run.notice,
53
+ failure_reasons: run.failure_reasons,
54
+ failure_records: run.failure_records,
49
55
  ...(run.notice ? { notice: run.notice } : {}),
50
56
  codex_sessions: run.summary,
51
57
  raw_evidence_gc: gc,
@@ -54,7 +60,7 @@ export async function reportNoWorktreeSync(command, io, run, dedup) {
54
60
  return syncResult(run);
55
61
  }
56
62
  writeLine(run.ok ? io.stdout : io.stderr, `Tower sync ${collectionRunStatus}: no git worktrees under this root; session scan ran.`);
57
- if (run.notice) {
63
+ if (run.notice?.startsWith("nothing_in_root")) {
58
64
  // Says out loud what the receipt now says to the dashboard: the sessions
59
65
  // this machine ran were all outside the folders it is allowed to look at.
60
66
  writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). Nothing was collected, and nothing is broken.`);
@@ -82,7 +88,10 @@ export async function reportSingleRepoSync(command, io, run, dedup) {
82
88
  writeLine(io.stdout, JSON.stringify({
83
89
  ...result,
84
90
  status: collectionRunStatus,
85
- collection_complete: run.ok,
91
+ collection_complete: run.ok && !run.discovery_partial,
92
+ discovery_notice: run.notice,
93
+ failure_reasons: run.failure_reasons,
94
+ failure_records: run.failure_records,
86
95
  codex_sessions: run.summary,
87
96
  raw_evidence_gc: gc,
88
97
  raw_evidence_dedup: dedup,
@@ -22,11 +22,13 @@ export async function runSyncLocked(command, io) {
22
22
  if (!dedup.skipped && dedup.removed_dirs > 0) {
23
23
  writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
24
24
  }
25
+ let discoveryNotice = null;
25
26
  const worktrees = await discoverCommandWorktrees(collectionRoots, {
26
27
  maxDepth: command.maxDepth,
27
28
  maxRepos: command.maxRepos,
28
29
  homeDir: command.homeDir,
29
30
  allowEmpty: true,
31
+ onPartial: (notice) => { discoveryNotice = notice; },
30
32
  }, io);
31
33
  const run = await runAttributedWorktreeSync({
32
34
  homeDir: command.homeDir,
@@ -36,6 +38,10 @@ export async function runSyncLocked(command, io) {
36
38
  worktrees,
37
39
  fetchImpl: io.fetch,
38
40
  });
41
+ if (discoveryNotice) {
42
+ run.discovery_partial = true;
43
+ run.notice = [discoveryNotice, run.notice].filter(Boolean).join("; ");
44
+ }
39
45
  if (run.outcomes.length > 1) {
40
46
  return reportMultiRepoSync(command, io, run, dedup);
41
47
  }
@@ -7,6 +7,7 @@
7
7
  *
8
8
  * rotate the scheduler's own logs ../log-rotation.ts
9
9
  * say the tick started reportTickStarted, below
10
+ * update and re-exec if needed sync-followups-self-update.ts
10
11
  * collect behind both locks sync-receipt.ts
11
12
  * check in with the heartbeat door sync-heartbeat.ts
12
13
  * say how the tick went reportTickOutcome / reportTickThrew
@@ -32,6 +33,8 @@
32
33
  * `scripts/build-public-cli.mjs` `runtimeFiles`, or the repo tests stay green
33
34
  * while the packed CLI breaks.
34
35
  */
36
+ import { createInteractiveExecRunner } from "../process-runner.js";
37
+ import { envWithNodeRuntimeOnPath } from "../scheduled-self-update.js";
35
38
  import { runMemoryExperienceAfterSync } from "./memory-log.js";
36
39
  import { classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
37
40
  import { sendSyncHeartbeat } from "./sync-heartbeat.js";
@@ -48,14 +51,33 @@ export async function runSync(command, io) {
48
51
  await rotateCollectorLogsBestEffort(paths);
49
52
  const dashboardUrl = await tickDashboardUrl(command, paths);
50
53
  const minCliVersionAtStart = await reportTickStarted(command, io, dashboardUrl);
54
+ const updated = await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersionAtStart);
55
+ if (updated && io.exec) {
56
+ const args = ["sync"];
57
+ if (command.homeDir)
58
+ args.push("--home", command.homeDir);
59
+ if (command.repoRoot)
60
+ args.push("--workspace", command.repoRoot);
61
+ if (command.dashboardUrl)
62
+ args.push("--dashboard-url", command.dashboardUrl);
63
+ if (command.maxDepth)
64
+ args.push("--max-depth", String(command.maxDepth));
65
+ if (command.maxRepos)
66
+ args.push("--max-repos", String(command.maxRepos));
67
+ if (command.json)
68
+ args.push("--json");
69
+ const exec = io.interactiveExec ?? createInteractiveExecRunner();
70
+ const result = await exec("cockpit", args, { env: envWithNodeRuntimeOnPath({ ...process.env, ...io.env, COCKPIT_SYNC_REEXEC: "1" }) });
71
+ return result.code;
72
+ }
51
73
  try {
52
74
  const result = await runSyncWithHealthReceipt(command, io);
53
75
  // BLI-3551: every tick checks in, including one that collected nothing.
54
76
  // This is the only writer of `last_seen_at` that does not need an envelope,
55
77
  // so it is what separates a quiet machine from a dead one.
56
78
  await sendSyncHeartbeat(command, io, dashboardUrl, result.heartbeat);
57
- const minCliVersion = await reportTickOutcome(command, io, dashboardUrl, result.completion);
58
- await convergeAfterTick(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
79
+ await reportTickOutcome(command, io, dashboardUrl, result.completion);
80
+ await convergeAfterTick(command, io, dashboardUrl);
59
81
  // BLI-3619: and the disk stops growing without bound — same daily cadence,
60
82
  // same rule that a follow-up never blocks or fails collection.
61
83
  await runStagingPruneAfterSync(command, io, dashboardUrl);
@@ -70,8 +92,8 @@ export async function runSync(command, io) {
70
92
  status: "fail",
71
93
  reason: errorCode,
72
94
  });
73
- const minCliVersion = await reportTickThrew(command, io, dashboardUrl, error, errorCode);
74
- await convergeAfterTick(command, io, dashboardUrl, minCliVersion ?? minCliVersionAtStart);
95
+ await reportTickThrew(command, io, dashboardUrl, error, errorCode);
96
+ await convergeAfterTick(command, io, dashboardUrl);
75
97
  throw error;
76
98
  }
77
99
  }
@@ -137,8 +159,7 @@ async function reportTickThrew(command, io, dashboardUrl, error, errorCode) {
137
159
  * Memory's registration — each at most once a day, each with its own receipt,
138
160
  * and none of them able to block or fail collection (BLI-3580).
139
161
  */
140
- async function convergeAfterTick(command, io, dashboardUrl, minCliVersion) {
141
- await runScheduledSelfUpdateAfterSync(command, io, dashboardUrl, minCliVersion);
162
+ async function convergeAfterTick(command, io, dashboardUrl) {
142
163
  await runAutostartSelfHealAfterSync(command, io, dashboardUrl);
143
164
  await runMemoryInstallAfterSync(command, io, dashboardUrl);
144
165
  await runMemoryExperienceAfterSync({ ...command, dashboardUrl }, io);
@@ -5,6 +5,10 @@ import { writeLine } from "./cli-io.js";
5
5
  export async function runUsage(command, io) {
6
6
  const door = await openAgentDoor("usage", command, io);
7
7
  const query = new URLSearchParams({ since: command.since });
8
+ if (command.minConfidence)
9
+ query.set("minConfidence", command.minConfidence);
10
+ if (command.byTopic)
11
+ query.set("groupBy", "task_type");
8
12
  if (command.byRepo)
9
13
  query.set("groupBy", "repo");
10
14
  if (command.until)
@@ -20,17 +24,19 @@ export async function runUsage(command, io) {
20
24
  if (!email)
21
25
  return failAgentDoor(door, "[usage]", "caller_email_unavailable", "The paired session has no email. Sign in again or pass --person <email>.");
22
26
  body.people = (body.people ?? []).filter((person) => person.email?.toLowerCase() === email.toLowerCase());
23
- body.coverage = { sessions_extracted: body.people.reduce((total, person) => total + person.sessions_extracted, 0), sessions_observed: body.people.reduce((total, person) => total + person.sessions_observed, 0) };
27
+ body.coverage = { sessions_labelled: body.people.reduce((total, person) => total + (person.sessions_labelled ?? 0), 0), sessions_extracted: body.people.reduce((total, person) => total + person.sessions_extracted, 0), sessions_observed: body.people.reduce((total, person) => total + person.sessions_observed, 0) };
24
28
  }
25
29
  if (command.byRepo && body.people?.some((person) => !Array.isArray(person.repos))) {
26
30
  return failAgentDoor(door, "[usage]", "repo_grouping_unavailable", "The dashboard returned person totals without project rows. Deploy the dashboard project split before using --by-repo.");
27
31
  }
32
+ if (command.byTopic && body.people?.some((person) => !Array.isArray(person.task_types)))
33
+ return failAgentDoor(door, "[usage]", "topic_grouping_unavailable", "The dashboard returned person totals without topic rows. Deploy the dashboard topic split before using --by-topic.");
28
34
  if (door.json)
29
35
  return emitAgentDoor(door, body);
30
36
  let nameWidth = 20;
31
- if (command.byRepo) {
37
+ if (command.byRepo || command.byTopic) {
32
38
  for (const person of body.people ?? []) {
33
- const repos = person.repos ?? [];
39
+ const repos = (command.byTopic ? person.task_types : person.repos) ?? [];
34
40
  for (const repo of command.all ? repos : repos.slice(0, 10))
35
41
  nameWidth = Math.max(nameWidth, repo.repo_label.length + 2);
36
42
  }
@@ -46,17 +52,21 @@ export async function runUsage(command, io) {
46
52
  `${row.sessions_extracted}/${row.sessions_observed}`,
47
53
  ...(command.detail ? [row.output_tokens, row.input_tokens, row.cache_read_input_tokens, row.cache_creation_input_tokens].map(formatCount) : []),
48
54
  ]);
49
- if (command.byRepo) {
50
- const repos = row.repos ?? [];
55
+ if (command.byRepo || command.byTopic) {
56
+ const repos = (command.byTopic ? row.task_types : row.repos) ?? [];
51
57
  for (const repo of command.all ? repos : repos.slice(0, 10)) {
58
+ if (command.byTopic && repo.confidence)
59
+ writeLine(io.stdout, ` Confidence: ${repo.confidence.high} high, ${repo.confidence.medium} medium, ${repo.confidence.low} low`);
52
60
  printRow([` ${repo.repo_label}`, formatCount(repo.tokens), formatUsageDollars(repo.api_list_price_equivalent_usd), `${repo.extracted_sessions}/${repo.sessions}`, ...(command.detail ? [repo.output, repo.input, repo.cache_read, repo.cache_creation].map(formatCount) : [])]);
53
61
  }
54
62
  if (!command.all && repos.length > 10)
55
- writeLine(io.stdout, ` and ${repos.length - 10} more repos`);
63
+ writeLine(io.stdout, ` and ${repos.length - 10} more ${command.byTopic ? "topics" : "repos"}`);
56
64
  }
57
65
  }
58
66
  writeLine(io.stdout, "");
59
67
  writeLine(io.stdout, body.api_list_price_equivalent_label ?? "API list-price equivalent (not actual spend)");
60
68
  writeLine(io.stdout, `${body.coverage?.sessions_extracted ?? 0} of ${body.coverage?.sessions_observed ?? 0} sessions extracted`);
69
+ if (command.byTopic)
70
+ writeLine(io.stdout, `${body.coverage?.sessions_labelled ?? 0} of ${body.coverage?.sessions_observed ?? 0} sessions labelled by topic`);
61
71
  return 0;
62
72
  }
@@ -63,6 +63,7 @@ export async function planFromDisk(paths, env, options, now) {
63
63
  retentionMs: options.retentionMs ?? fromEnv.retentionMs,
64
64
  capBytes: options.capBytes ?? fromEnv.capBytes,
65
65
  allCommitted: options.allCommitted,
66
+ reclaimTerminal: options.reclaimTerminal,
66
67
  });
67
68
  }
68
69
  async function applyPlan(paths, plan) {
@@ -23,6 +23,11 @@ export function planStagingRetention(inventory, options = {}) {
23
23
  const committedInWindow = [];
24
24
  for (const pack of inventory.packs) {
25
25
  for (const object of pack.objects) {
26
+ if (object.state === "terminal") {
27
+ if (options.reclaimTerminal)
28
+ pushDeletion(plan, object, `terminal:${object.reason}`);
29
+ continue;
30
+ }
26
31
  if (object.state !== "committed") {
27
32
  keepUnvouched(plan, object);
28
33
  continue;
@@ -35,7 +40,8 @@ export function planStagingRetention(inventory, options = {}) {
35
40
  addToBucket(plan.kept_in_window, object);
36
41
  }
37
42
  }
38
- applyByteCap(plan, committedInWindow, capBytes, inventory);
43
+ const retainedTerminalBytes = options.reclaimTerminal ? 0 : (inventory.terminal_bytes ?? 0);
44
+ applyByteCap(plan, committedInWindow, capBytes + retainedTerminalBytes, inventory);
39
45
  markEmptyPacks(plan, inventory.packs);
40
46
  plan.bytes_after =
41
47
  inventory.total_bytes - plan.deleted_bytes - plan.empty_pack_manifest_bytes;
@@ -44,7 +50,7 @@ export function planStagingRetention(inventory, options = {}) {
44
50
  // sentence — the remaining manifests alone exceed it — and saying the wrong
45
51
  // one would send an operator hunting evidence that is not missing.
46
52
  const unvouched = plan.kept_uncommitted.count + plan.kept_unknown.count;
47
- plan.cap_blocked_by_uncommitted = plan.bytes_after > capBytes && unvouched > 0;
53
+ plan.cap_blocked_by_uncommitted = plan.bytes_after - retainedTerminalBytes > capBytes && unvouched > 0;
48
54
  plan.cap_blocked_count = plan.cap_blocked_by_uncommitted ? unvouched : 0;
49
55
  return plan;
50
56
  }
@@ -84,7 +90,7 @@ function applyByteCap(plan, committedInWindow, capBytes, inventory) {
84
90
  function unremovableFloor(inventory) {
85
91
  let floor = 0;
86
92
  for (const pack of inventory.packs) {
87
- const unvouched = pack.objects.filter((object) => object.state !== "committed");
93
+ const unvouched = pack.objects.filter((object) => object.state !== "committed" && object.state !== "terminal");
88
94
  if (unvouched.length === 0)
89
95
  continue;
90
96
  floor += pack.manifest_bytes;
@@ -2,14 +2,13 @@
2
2
  * The three-state question, asked once per staged file: committed, uncommitted,
3
3
  * or unknown — and the reason label that says which record answered.
4
4
  *
5
- * The order of the answers is the whole safety property. This machine's commit
6
- * ledger wins first; a server answer an earlier `cockpit clean --reconcile`
7
- * recorded wins next, because the server was actually asked; and only when
8
- * neither can speak does the object fall to `unknown` with the reason its
9
- * silence has. Everything downstream — what may be deleted, what is offered
5
+ * A local commit wins unless a newer server reconciliation disputes it.
6
+ * Explicit object refusals remain terminal until a later successful delivery.
7
+ * When neither ledger can speak, the object is unknown and stays on disk. Everything downstream — what may be deleted, what is offered
10
8
  * again, what an operator reads — is this label.
11
9
  */
12
10
  import path from "node:path";
11
+ import { terminalDeliveryReason } from "./raw-evidence-staging.js";
13
12
  import { hashUnnamedFile } from "./disk-usage-files.js";
14
13
  export async function classifyObject(dir, packId, relative, size, manifest, options) {
15
14
  const hash = manifest?.get(relative) ??
@@ -23,21 +22,28 @@ export async function classifyObject(dir, packId, relative, size, manifest, opti
23
22
  staged_on_disk_at: options.stagedOnDiskAt,
24
23
  disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
25
24
  };
26
- if (committedAt) {
25
+ const reconciled = hash ? options.reconciled.get(hash) : undefined;
26
+ const serverDisputesCommit = reconciled && committedAt &&
27
+ Date.parse(reconciled.checked_at) > Date.parse(committedAt) &&
28
+ reconciled.verdict !== "committed";
29
+ if (committedAt && !serverDisputesCommit) {
27
30
  return {
28
- ...base,
29
- content_hash: hash ?? null,
30
- state: "committed",
31
- reason: "committed_in_ledger",
32
- decided_at: committedAt,
31
+ ...base, content_hash: hash ?? null, state: "committed",
32
+ reason: "committed_in_ledger", decided_at: committedAt,
33
33
  age_ms: ageMs(committedAt, options.now),
34
34
  };
35
35
  }
36
- if (hash) {
37
- const reconciled = options.reconciled.get(hash);
38
- if (reconciled) {
39
- return classifyFromReconcileAnswer(base, hash, reconciled, options.now);
40
- }
36
+ const attempt = hash ? options.deliveryAttempts?.[hash] : undefined;
37
+ const serverConfirmedAfterRefusal = reconciled?.verdict === "committed" &&
38
+ attempt && Date.parse(reconciled.checked_at) >= Date.parse(attempt.last_attempt_at);
39
+ if (attempt && terminalDeliveryReason(attempt.last_reason) && !serverConfirmedAfterRefusal) {
40
+ return {
41
+ ...base, content_hash: hash ?? null, state: "terminal", reason: attempt.last_reason,
42
+ decided_at: attempt.last_attempt_at, age_ms: ageMs(attempt.last_attempt_at, options.now),
43
+ };
44
+ }
45
+ if (hash && reconciled) {
46
+ return classifyFromReconcileAnswer(base, hash, reconciled, options.now);
41
47
  }
42
48
  const unknownReason = unknownStateReason(hash, manifest, stagedAt, options);
43
49
  return {
@@ -8,6 +8,7 @@
8
8
  * each pack's own manifest, and a single byte budget shared across the run.
9
9
  */
10
10
  import fs from "node:fs/promises";
11
+ import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
11
12
  import path from "node:path";
12
13
  import { classifyObject, } from "./disk-usage-classify.js";
13
14
  import { RAW_EVIDENCE_DIR, } from "./disk-usage-facts.js";
@@ -19,6 +20,7 @@ export async function readStagingInventory(paths, now = new Date()) {
19
20
  const ledger = await readCommitLedger(paths);
20
21
  const reconciled = await readReconcileLedger(paths);
21
22
  const stagedAt = await readStagedAtIndex(paths);
23
+ const { delivery_attempts: deliveryAttempts } = await readRawEvidenceStagingState(paths.state_dir);
22
24
  const entries = await fs
23
25
  .readdir(root, { withFileTypes: true })
24
26
  .catch(() => []);
@@ -36,6 +38,7 @@ export async function readStagingInventory(paths, now = new Date()) {
36
38
  if (!entry.name.startsWith("work-"))
37
39
  continue;
38
40
  const pack = await readPack(dir, entry.name, {
41
+ deliveryAttempts,
39
42
  ledger,
40
43
  reconciled,
41
44
  stagedAt,
@@ -26,6 +26,13 @@ export function accumulatePack(inventory, pack) {
26
26
  for (const object of pack.objects) {
27
27
  inventory.object_count += 1;
28
28
  inventory.total_bytes += object.byte_size;
29
+ if (object.state === "terminal") {
30
+ inventory.terminal_count = (inventory.terminal_count ?? 0) + 1;
31
+ inventory.terminal_bytes = (inventory.terminal_bytes ?? 0) + object.byte_size;
32
+ const reasons = inventory.terminal_reasons ??= {};
33
+ reasons[object.reason] = (reasons[object.reason] ?? 0) + 1;
34
+ continue;
35
+ }
29
36
  if (object.state === "committed") {
30
37
  inventory.committed_count += 1;
31
38
  inventory.committed_bytes += object.byte_size;
@@ -28,7 +28,7 @@ import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } f
28
28
  import { readStagingInventory } from "./disk-usage.js";
29
29
  import { describeError } from "./health-detail.js";
30
30
  import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "./local-state.js";
31
- import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
31
+ import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, terminalDeliveryReason, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
32
32
  /**
33
33
  * The drain the sync tick runs and `cockpit doctor`'s disk fix reuses.
34
34
  * Never throws.
@@ -46,7 +46,7 @@ export async function runEvidenceRedelivery(options) {
46
46
  // The steady state, 96 ticks a day. Saying so every time would bury the
47
47
  // ticks that matter; `cockpit doctor`'s disk row still says it out loud
48
48
  // to a person who asks.
49
- return emptyResult("nothing_uncommitted", "ok");
49
+ return { ...emptyResult("nothing_uncommitted", "ok"), terminal: inventory.terminal_count ?? 0, terminal_bytes: inventory.terminal_bytes ?? 0 };
50
50
  }
51
51
  const staging = await readRawEvidenceStagingState(paths.state_dir);
52
52
  const plan = await planEvidenceRedelivery({
@@ -58,7 +58,7 @@ export async function runEvidenceRedelivery(options) {
58
58
  ...(options.maxObjects != null ? { maxObjects: options.maxObjects } : {}),
59
59
  });
60
60
  if (plan.object_count === 0) {
61
- const result = emptyPlanResult(plan);
61
+ const result = { ...emptyPlanResult(plan), terminal: inventory.terminal_count ?? 0, terminal_bytes: inventory.terminal_bytes ?? 0 };
62
62
  reportRedelivery(result, plan);
63
63
  return result;
64
64
  }
@@ -86,14 +86,18 @@ export async function runEvidenceRedelivery(options) {
86
86
  now,
87
87
  });
88
88
  await recordOutcomes(paths, staging, outcome.outcomes, now);
89
+ const terminalOutcomes = outcome.outcomes.filter(entry => entry.upload_state === "upload_failed" && terminalDeliveryReason(entry.reason));
89
90
  const result = {
90
91
  status: outcome.failed > 0 && outcome.uploaded === 0 ? "fail" : "ok",
91
92
  reason: "redelivered",
93
+ terminal: (inventory.terminal_count ?? 0) + terminalOutcomes.length,
94
+ terminal_bytes: (inventory.terminal_bytes ?? 0) + terminalOutcomes.reduce((bytes, entry) => bytes + (entry.pointer.byte_size ?? 0), 0),
92
95
  offered: plan.object_count,
93
96
  offered_bytes: plan.bytes,
94
97
  uploaded: outcome.uploaded,
95
98
  uploaded_bytes: outcome.uploadedBytes,
96
99
  reused: outcome.reused,
100
+ reconciled_committed_by_409: outcome.outcomes.filter(entry => entry.reason === "reconciled_committed_by_409").length,
97
101
  failed: outcome.failed,
98
102
  held: plan.held_count,
99
103
  deferred: plan.deferred_count,
@@ -229,11 +233,14 @@ function reportRedelivery(result, plan) {
229
233
  const fields = {
230
234
  reason: result.reason,
231
235
  status: result.status,
236
+ terminal: result.terminal ?? 0,
237
+ terminal_bytes: result.terminal_bytes ?? 0,
232
238
  offered: result.offered,
233
239
  offered_bytes: result.offered_bytes,
234
240
  uploaded: result.uploaded,
235
241
  uploaded_bytes: result.uploaded_bytes,
236
242
  reused: result.reused,
243
+ reconciled_committed_by_409: result.reconciled_committed_by_409 ?? 0,
237
244
  failed: result.failed,
238
245
  held_by_backoff: result.held,
239
246
  deferred_to_next_tick: result.deferred,
@@ -243,7 +250,7 @@ function reportRedelivery(result, plan) {
243
250
  failure_reasons: result.failure_reasons,
244
251
  };
245
252
  if (result.status === "fail") {
246
- console.error("[evidence-redelivery] every re-offered object was refused", JSON.stringify(fields));
253
+ console.error("[evidence-redelivery] staged evidence could not be delivered", JSON.stringify(fields));
247
254
  return;
248
255
  }
249
256
  console.error(result.uploaded > 0
@@ -251,11 +258,11 @@ function reportRedelivery(result, plan) {
251
258
  : "[evidence-redelivery] undelivered staged evidence was not re-offered this tick", JSON.stringify(fields));
252
259
  }
253
260
  function emptyPlanResult(plan) {
254
- const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0
261
+ const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0 && Object.keys(plan.unplannable).length === 0
255
262
  ? "all_held_by_backoff"
256
263
  : "nothing_plannable";
257
264
  return {
258
- ...emptyResult(nothingPlannable, "ok"),
265
+ ...emptyResult(nothingPlannable, Object.keys(plan.unplannable).length ? "fail" : "ok"),
259
266
  held: plan.held_count,
260
267
  deferred: plan.deferred_count,
261
268
  deferred_bytes: plan.deferred_bytes,
@@ -1,11 +1,12 @@
1
1
  import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
2
+ import { beginOneObject } from "./evidence-upload-terminal.js";
2
3
  import { isNonJsonResponseBody, requestJson, safeFailureDetail, sha256, } from "./evidence-upload-transport.js";
3
4
  /**
4
5
  * The `chunk` and `commit` stages for one already-`begin`-accepted object:
5
6
  * send every chunk the server does not already have, then commit and turn
6
7
  * the receipt into a `ChunkedUploadFileOutcome`.
7
8
  */
8
- export async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
9
+ export async function uploadOneObject(options, entry, disposition, chunkSizeBytes, restageAttempted = false) {
9
10
  const objectKey = entry.file.pointer.object_key ?? "";
10
11
  if (disposition.disposition === "already_committed") {
11
12
  if (!disposition.upload_id) {
@@ -18,7 +19,22 @@ export async function uploadOneObject(options, entry, disposition, chunkSizeByte
18
19
  const chunked = await uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey);
19
20
  if (!chunked.ok)
20
21
  return chunked.outcome;
21
- return commitUploadedObject(options, entry, disposition, objectKey, chunked.uploadedChunks);
22
+ const committed = await commitUploadedObject(options, entry, disposition, objectKey, chunked.uploadedChunks);
23
+ if (committed.reason !== "commit_failed_http_409_staging_missing_between_passes")
24
+ return committed;
25
+ if (restageAttempted)
26
+ return { ...committed, reason: "staging_missing_after_restage" };
27
+ console.error("[evidence-upload] restaging missing chunks", JSON.stringify({
28
+ reason: "staging_missing_between_passes", upload_id: disposition.upload_id,
29
+ }));
30
+ const restarted = await beginOneObject(options, { ...entry, duplicates: [] }, chunkSizeBytes);
31
+ if (!restarted)
32
+ return failedOutcome(entry.file, "staging_restage_failed:begin_failed", chunked.uploadedChunks);
33
+ const retried = await uploadOneObject(options, entry, restarted, chunkSizeBytes, true);
34
+ if (retried.upload_state === "upload_failed" && retried.reason !== "staging_missing_after_restage") {
35
+ retried.reason = `staging_restage_failed:${retried.reason ?? "unknown"}`;
36
+ }
37
+ return retried;
22
38
  }
23
39
  async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey) {
24
40
  const received = new Set(disposition.received_chunk_indexes);
@@ -47,6 +63,15 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
47
63
  // `chunk_3_failed_http_413_object_too_large` and the ledger row names
48
64
  // the cause instead of the transport (BLI-3483).
49
65
  const chunkDetail = safeFailureDetail(chunkResponse.body);
66
+ if (chunkResponse.status === 409 && chunkDetail === "upload_not_pending:committed") {
67
+ console.error("[evidence-upload] committed receipt reconciled", JSON.stringify({
68
+ reason: "reconciled_committed_by_409", upload_id: disposition.upload_id,
69
+ }));
70
+ return { ok: false, outcome: {
71
+ ...failedOutcome(entry.file, "reconciled_committed_by_409", uploadedChunks),
72
+ upload_state: "reused_existing",
73
+ } };
74
+ }
50
75
  console.error("[evidence-upload] chunk rejected", JSON.stringify({
51
76
  reason: "chunk_rejected",
52
77
  upload_id: disposition.upload_id,
@@ -73,7 +73,7 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
73
73
  * pointer id that is not the one we sent — and the caller then keeps the
74
74
  * original conflict rather than acting on a guess.
75
75
  */
76
- async function beginOneObject(options, entry, chunkSizeBytes) {
76
+ export async function beginOneObject(options, entry, chunkSizeBytes) {
77
77
  const objectKey = entry.file.pointer.object_key ?? "";
78
78
  const response = await requestJson(options, "/api/ambient/evidence/upload/begin", {
79
79
  schema_version: "ambient-raw-evidence-upload-begin.v1",
@@ -160,6 +160,7 @@ export function recordDeliveryFailure(state, contentHash, options) {
160
160
  const attemptedAtIso = options.attemptedAt.toISOString();
161
161
  const entry = {
162
162
  attempts,
163
+ ...(terminalDeliveryReason(options.reason) ? { terminal_reason: options.reason } : {}),
163
164
  first_failed_at: previous?.first_failed_at ?? attemptedAtIso,
164
165
  last_attempt_at: attemptedAtIso,
165
166
  last_reason: options.reason,
@@ -177,13 +178,20 @@ export function clearDeliveryAttempt(state, contentHash) {
177
178
  delete state.delivery_attempts[contentHash];
178
179
  return true;
179
180
  }
181
+ export function terminalDeliveryReason(reason) {
182
+ return Boolean(reason && (reason.startsWith("staging_restage_failed:") ||
183
+ ["staging_missing_after_restage", "secret_guard_rejected", "storage_rejected_object_too_large", "storage_rejected_media_type"].includes(reason) ||
184
+ /^(?:begin|commit|chunk_\d+)_failed_http_413(?:_|$)/u.test(reason)));
185
+ }
180
186
  export function deliveryHold(state, contentHash, now) {
181
187
  if (!contentHash)
182
188
  return null;
183
189
  const entry = state.delivery_attempts[contentHash];
184
190
  if (!entry)
185
191
  return null;
186
- return Date.parse(entry.next_attempt_at) > now.getTime() ? entry : null;
192
+ if (terminalDeliveryReason(entry.last_reason))
193
+ return entry;
194
+ return deliveryRetryAtMs(entry) > now.getTime() ? entry : null;
187
195
  }
188
196
  /**
189
197
  * Source keys currently held by backoff.
@@ -194,10 +202,10 @@ export function deliveryHold(state, contentHash, now) {
194
202
  */
195
203
  export function heldSourceKeys(state, now) {
196
204
  const held = new Set();
197
- for (const entry of Object.values(state.delivery_attempts)) {
198
- if (!entry.source_key)
205
+ for (const [hash, entry] of Object.entries(state.delivery_attempts)) {
206
+ if (!entry.source_key || terminalDeliveryReason(entry.last_reason))
199
207
  continue;
200
- if (Date.parse(entry.next_attempt_at) > now.getTime()) {
208
+ if (deliveryHold(state, hash, now)) {
201
209
  held.add(entry.source_key);
202
210
  }
203
211
  }
@@ -286,6 +294,7 @@ function parseAttemptEntry(value) {
286
294
  return null;
287
295
  const attempts = optionalNumber(record["attempts"]);
288
296
  return {
297
+ ...(terminalDeliveryReason(optionalString(record["last_reason"])) ? { terminal_reason: String(record["last_reason"]) } : {}),
289
298
  attempts: attempts > 0 ? attempts : 1,
290
299
  first_failed_at: optionalString(record["first_failed_at"]) ?? lastAttemptAt,
291
300
  last_attempt_at: lastAttemptAt,
@@ -307,4 +316,8 @@ function shortHash(value) {
307
316
  .update(value, "utf8")
308
317
  .digest("hex")
309
318
  .slice(0, 12);
319
+ }
320
+ /** A persisted clock cannot extend a retry beyond the last attempt's ceiling. */
321
+ export function deliveryRetryAtMs(entry) {
322
+ return Math.min(Date.parse(entry.next_attempt_at), Date.parse(entry.last_attempt_at) + EVIDENCE_DELIVERY_BACKOFF_MAX_MS);
310
323
  }
@@ -14,12 +14,13 @@
14
14
  * Metadata only, like the rest of this family: content hashes, byte sizes,
15
15
  * reason labels, timestamps. Never content, never an absolute path.
16
16
  */
17
+ import { deliveryRetryAtMs, terminalDeliveryReason } from "./raw-evidence-staging.js";
17
18
  /**
18
19
  * What `cockpit status` and the health receipt need in order to be unable to
19
20
  * read green while an object has been failing for nine days.
20
21
  */
21
22
  export function summarizeStuckEvidence(state, now) {
22
- const entries = Object.values(state.delivery_attempts);
23
+ const entries = Object.values(state.delivery_attempts).filter(entry => !terminalDeliveryReason(entry.last_reason));
23
24
  const reasons = new Set();
24
25
  let heldCount = 0;
25
26
  let maxAttempts = 0;
@@ -31,15 +32,16 @@ export function summarizeStuckEvidence(state, now) {
31
32
  let heldNextAttemptAt = null;
32
33
  for (const entry of entries) {
33
34
  reasons.add(entry.last_reason);
34
- if (Date.parse(entry.next_attempt_at) > now.getTime()) {
35
+ const retryAt = deliveryRetryAtMs(entry);
36
+ if (retryAt > now.getTime()) {
35
37
  heldCount += 1;
36
38
  if (!heldOldest ||
37
39
  entry.first_failed_at.localeCompare(heldOldest.first_failed_at) < 0) {
38
40
  heldOldest = entry;
39
41
  }
40
42
  if (!heldNextAttemptAt ||
41
- entry.next_attempt_at.localeCompare(heldNextAttemptAt) < 0) {
42
- heldNextAttemptAt = entry.next_attempt_at;
43
+ retryAt < Date.parse(heldNextAttemptAt)) {
44
+ heldNextAttemptAt = new Date(retryAt).toISOString();
43
45
  }
44
46
  }
45
47
  maxAttempts = Math.max(maxAttempts, entry.attempts);
@@ -28,11 +28,9 @@ export function envWithNodeRuntimeOnPath(env, nodeExecutable = process.execPath)
28
28
  * post-collection tail, so the fleet converges on npm `latest` without
29
29
  * anyone re-running `npm i -g @bli-cockpit/cli && cockpit doctor` by hand after day 0.
30
30
  *
31
- * Deliberately does not re-exec: this process already has the OLD code
32
- * loaded in memory, so nothing in-process could prove a re-exec actually
33
- * picked up the new build. The next scheduled tick launches a fresh
34
- * `cockpit` process from disk and picks up whatever npm actually installed —
35
- * that is the verification, not a re-exec here.
31
+ * Installation is verified with npm's installed-version probe. The sync caller
32
+ * re-executes the tick on `updated`, carrying a guard that skips another update
33
+ * attempt in the replacement process.
36
34
  */
37
35
  export async function runScheduledSelfUpdate(paths, deps, options = {}) {
38
36
  const env = options.env ?? process.env;
@@ -1,4 +1,4 @@
1
- import { clearDeliveryAttempt, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, deliveryHold, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
1
+ import { clearDeliveryAttempt, DELIVERY_BACKOFF_BYPASS_REASON, DELIVERY_BACKOFF_HOLDING_REASON, deliveryBackoffApplies, deliveryHold, recordDeliveryFailure, terminalDeliveryReason, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
2
2
  /**
3
3
  * 1. What may we offer this sync? 2. What just happened to each object?
4
4
  *
@@ -25,7 +25,7 @@ export function partitionHeldEvidenceFiles(files, staging, now, mode) {
25
25
  const backoffApplies = deliveryBackoffApplies(mode);
26
26
  for (const file of files) {
27
27
  const hold = deliveryHold(staging, file.pointer.content_hash_sha256, now);
28
- if (hold && !backoffApplies) {
28
+ if (hold && !backoffApplies && !terminalDeliveryReason(hold.last_reason)) {
29
29
  // BLI-3118: a person asked for this one now. Offering it is the whole
30
30
  // point of the retry command Cockpit printed, and the bypass is logged
31
31
  // rather than assumed.
@@ -46,7 +46,8 @@ export function partitionHeldEvidenceFiles(files, staging, now, mode) {
46
46
  ? { artifact_metadata: file.artifact_metadata }
47
47
  : {}),
48
48
  upload_state: "upload_failed",
49
- reason: DELIVERY_BACKOFF_HOLDING_REASON,
49
+ reason: terminalDeliveryReason(hold.last_reason) ? hold.last_reason : DELIVERY_BACKOFF_HOLDING_REASON,
50
+ delivery_attempted: false,
50
51
  uploaded_chunk_count: 0,
51
52
  });
52
53
  }
@@ -103,7 +104,7 @@ export async function persistDeliveryAttempts(stateDir, staging, outcomes, attem
103
104
  const contentHash = outcome.pointer.content_hash_sha256;
104
105
  if (!contentHash)
105
106
  continue;
106
- if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON)
107
+ if (outcome.reason === DELIVERY_BACKOFF_HOLDING_REASON || outcome.delivery_attempted === false)
107
108
  continue;
108
109
  if (outcome.upload_state === "upload_failed") {
109
110
  const entry = recordDeliveryFailure(staging, contentHash, {
@@ -1,5 +1,5 @@
1
1
  import { isPermanentUploadFailure } from "@bli-cockpit/telemetry-core";
2
- import { DELIVERY_BACKOFF_HOLDING_REASON, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
2
+ import { DELIVERY_BACKOFF_HOLDING_REASON, summarizeStuckEvidence, terminalDeliveryReason, } from "./raw-evidence-staging.js";
3
3
  /**
4
4
  * Which adapters a queued retry has to re-run.
5
5
  *
@@ -36,12 +36,12 @@ export function retrySourcesForFailedSync(options, facts) {
36
36
  */
37
37
  function retryableFailedOutcomes(outcomes) {
38
38
  return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
39
- !isPermanentUploadFailure(outcome.reason));
39
+ !isPermanentUploadFailure(outcome.reason) && !terminalDeliveryReason(outcome.reason));
40
40
  }
41
41
  /** Failed uploads that no retry can rescue, kept so they can still be named. */
42
42
  function permanentFailedOutcomes(outcomes) {
43
43
  return outcomes.filter((outcome) => outcome.upload_state === "upload_failed" &&
44
- isPermanentUploadFailure(outcome.reason));
44
+ (isPermanentUploadFailure(outcome.reason) || terminalDeliveryReason(outcome.reason)));
45
45
  }
46
46
  /**
47
47
  * The reason to show for objects that failed for good.
@@ -174,6 +174,7 @@ export function summarizeRawEvidenceDelivery(built, outcomes, uploadedChunkCount
174
174
  raw_evidence_uploaded_object_count: outcomes.filter((outcome) => outcome.upload_state === "uploaded").length,
175
175
  raw_evidence_uploaded_chunk_count: uploadedChunkCount,
176
176
  raw_evidence_reused_count: cursorReused.length + serverReusedCount,
177
+ reconciled_committed_by_409: outcomes.filter((outcome) => outcome.reason === "reconciled_committed_by_409").length,
177
178
  raw_evidence_failed_count: failed.length,
178
179
  raw_evidence_sanitized_count: built.raw_evidence_facts?.sanitized_count ?? 0,
179
180
  raw_evidence_failure_reasons: [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.104",
3
+ "version": "0.2.108",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -28,7 +28,7 @@
28
28
  },
29
29
  "dependencies": {
30
30
  "@bli-cockpit/memory-mcp": "0.1.27",
31
- "@bli-cockpit/mcp": "0.1.34",
31
+ "@bli-cockpit/mcp": "0.1.36",
32
32
  "@bli-cockpit/telemetry-core": "0.1.43"
33
33
  }
34
34
  }