@bli-cockpit/cli 0.2.48 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/adapters/raw-evidence-attribution-gaps.js +133 -0
  2. package/dist/adapters/raw-evidence-claude-reader.js +108 -0
  3. package/dist/adapters/raw-evidence-codex-reader.js +147 -0
  4. package/dist/adapters/raw-evidence-collection-state.js +199 -0
  5. package/dist/adapters/raw-evidence-facts.js +338 -0
  6. package/dist/adapters/raw-evidence-git-diff-reader.js +187 -0
  7. package/dist/adapters/raw-evidence-image-reader.js +107 -0
  8. package/dist/adapters/raw-evidence-sanitize.js +56 -0
  9. package/dist/adapters/raw-evidence-transcript-file.js +182 -0
  10. package/dist/adapters/raw-evidence.js +94 -1203
  11. package/dist/autostart-contract.js +79 -0
  12. package/dist/autostart-darwin-plist.js +265 -0
  13. package/dist/autostart-darwin.js +171 -0
  14. package/dist/autostart-windows-scripts.js +310 -0
  15. package/dist/autostart-windows-task-xml.js +260 -0
  16. package/dist/autostart-windows.js +237 -0
  17. package/dist/autostart-xml.js +23 -0
  18. package/dist/autostart.js +35 -1148
  19. package/dist/commands/agent-rules-command.js +55 -0
  20. package/dist/commands/agent-session-report.js +290 -0
  21. package/dist/commands/analyze.js +131 -0
  22. package/dist/commands/autostart-command.js +105 -0
  23. package/dist/commands/backfill-batches.js +34 -0
  24. package/dist/commands/backfill-candidates.js +54 -0
  25. package/dist/commands/backfill-checkpoint.js +101 -0
  26. package/dist/commands/backfill-command-line.js +70 -0
  27. package/dist/commands/backfill-evidence-outcomes.js +104 -0
  28. package/dist/commands/backfill-issues.js +265 -0
  29. package/dist/commands/backfill-output.js +75 -0
  30. package/dist/commands/backfill-plan.js +71 -0
  31. package/dist/commands/backfill-reasons.js +107 -0
  32. package/dist/commands/backfill-report.js +298 -0
  33. package/dist/commands/backfill-result.js +150 -0
  34. package/dist/commands/backfill-scan.js +274 -0
  35. package/dist/commands/backfill-scope.js +114 -0
  36. package/dist/commands/backfill-session-report.js +145 -0
  37. package/dist/commands/backfill-types.js +1 -0
  38. package/dist/commands/backfill-upload.js +212 -0
  39. package/dist/commands/backfill.js +58 -1705
  40. package/dist/commands/cli-io.js +13 -0
  41. package/dist/commands/doctor.js +57 -0
  42. package/dist/commands/jarvis-trace.js +184 -0
  43. package/dist/commands/jarvis.js +323 -7
  44. package/dist/commands/local-arg-values.js +169 -0
  45. package/dist/commands/local-args-collector.js +604 -0
  46. package/dist/commands/local-args-tower.js +891 -0
  47. package/dist/commands/local-args.js +10 -1549
  48. package/dist/commands/local-help.js +30 -5
  49. package/dist/commands/local.js +21 -1786
  50. package/dist/commands/login.js +53 -0
  51. package/dist/commands/logout.js +66 -0
  52. package/dist/commands/memory-install-claude.js +294 -0
  53. package/dist/commands/memory-install-codex.js +205 -0
  54. package/dist/commands/memory-install-contract.js +231 -0
  55. package/dist/commands/memory-install-files.js +63 -0
  56. package/dist/commands/memory-install-skills.js +121 -0
  57. package/dist/commands/memory-install-toml.js +265 -0
  58. package/dist/commands/memory-install.js +378 -0
  59. package/dist/commands/onboard-receipts.js +66 -0
  60. package/dist/commands/onboard-report.js +274 -0
  61. package/dist/commands/onboard.js +449 -0
  62. package/dist/commands/ops-render.js +36 -0
  63. package/dist/commands/public-root.js +1 -1
  64. package/dist/commands/serve.js +13 -0
  65. package/dist/commands/session-sync.js +513 -534
  66. package/dist/commands/settings-render.js +28 -0
  67. package/dist/commands/settings.js +66 -2
  68. package/dist/commands/start.js +47 -0
  69. package/dist/commands/sync-followups.js +308 -0
  70. package/dist/commands/sync.js +387 -0
  71. package/dist/local-state-attributed-target.js +75 -0
  72. package/dist/local-state-config.js +147 -0
  73. package/dist/local-state-files.js +59 -0
  74. package/dist/local-state-identity.js +73 -0
  75. package/dist/local-state-pairing.js +263 -0
  76. package/dist/local-state-paths.js +61 -0
  77. package/dist/local-state-session.js +68 -0
  78. package/dist/local-state-status.js +163 -0
  79. package/dist/local-state-work-context.js +190 -0
  80. package/dist/local-state.js +34 -848
  81. package/dist/tower-client.js +3 -2
  82. package/dist/tower-stream.js +76 -6
  83. package/package.json +2 -1
@@ -1,28 +1,54 @@
1
- import { NO_UPLOAD_ATTEMPT_RECORDED, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, notUploadableAttributionStateReason } from "@bli-cockpit/telemetry-core";
2
- import crypto from "node:crypto";
3
- import fs from "node:fs/promises";
4
- import os from "node:os";
5
- import path from "node:path";
6
- import { describeError } from "../health-detail.js";
7
- import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
8
- import { defaultCodexSessionDirs, scanAndAttributeCodexSessions } from "../adapters/codex-attribution.js";
9
- import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET } from "../adapters/raw-evidence.js";
10
- import { acquireBackfillLock, } from "../backfill-lock.js";
11
- import { BACKFILL_COMPLETION_RECHECK_MS, BACKFILL_COVERAGE_VERSION, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCursor, recordBackfillCursorObservations, recordBackfillScanCoverage, writeBackfillCompletionMarker, writeBackfillCursor, } from "../cursors/backfill-cursor.js";
12
- import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
13
- import { getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, startLocalWorkContext } from "../local-state.js";
14
- import { DEFAULT_DISCOVERY_MAX_DEPTH, DEFAULT_DISCOVERY_MAX_REPOS, collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } from "../repo-identity.js";
15
- import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
16
- import { normalizeCollectionRoots } from "../root-normalization.js";
1
+ /**
2
+ * `cockpit backfill` — collect the archived Codex and Claude history that the
3
+ * every-15-minutes sync was never running to see.
4
+ *
5
+ * Read `runBackfill` below as the table of contents. It is four stages, in
6
+ * order, and every part of this command lives in exactly one of them:
7
+ *
8
+ * SCAN backfill-scan.ts which roots, stores and window; read both
9
+ * stores into candidates; census the gaps
10
+ * PLAN backfill-plan.ts confirm `--all`, or answer a `--dry-run`
11
+ * UPLOAD backfill-upload.ts sync in batches under the shared lock
12
+ * REPORT backfill-report.ts post, checkpoint, decide, assemble
13
+ *
14
+ * The supporting modules each own one thing the stages share:
15
+ *
16
+ * backfill-types.ts the shapes every stage passes along
17
+ * backfill-command-line.ts the flag surface and the retry command
18
+ * backfill-scope.ts paired? which roots, sources and window
19
+ * backfill-candidates.ts one session's identity and ordering
20
+ * backfill-batches.ts what is uploadable, cut into batches
21
+ * backfill-issues.ts the ledger of what could not be accounted for
22
+ * backfill-reasons.ts the skip-reason table an operator reads
23
+ * backfill-evidence-outcomes.ts what one upload achieved per session
24
+ * backfill-session-report.ts one attribution row per session seen
25
+ * backfill-checkpoint.ts the durable cursor and pointer writes
26
+ * backfill-result.ts the `--json` payload
27
+ * backfill-output.ts every line and prompt a person sees
28
+ *
29
+ * Two rules run through all four stages and explain most of the apparent
30
+ * complexity. First, a session may never be silently dropped: everything the
31
+ * scan could not account for lands on the issue ledger and everything a person
32
+ * reads names its own reason. Second, the cursor is the only irreversible
33
+ * thing here — it advances solely through a contiguous prefix of sessions that
34
+ * both earned a durable pointer and were acknowledged by the server, so a
35
+ * misjudged "resolved" loses that history for good.
36
+ */
37
+ import { acquireBackfillLock } from "../backfill-lock.js";
38
+ import { writeBackfillCursor } from "../cursors/backfill-cursor.js";
17
39
  import { acquireSyncLock } from "../sync-lock.js";
18
- import { LocalUploadBlockedError, postCodexSessionReport, syncLocalAmbientEnvelope } from "../upload.js";
19
- const BACKFILL_UPLOAD_BATCH_SESSIONS = 25;
20
- const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
21
- const ALL_BACKFILL_SINCE_MINUTES = 20 * 365 * 24 * 60;
22
- // Window a bare `cockpit backfill` uses. Wide enough to cover a new machine's
23
- // recent history and an intern who went quiet for a few weeks, narrow enough
24
- // that it is not the whole-history scan `--all` deliberately gates.
25
- export const DEFAULT_BACKFILL_SINCE_DAYS = 30;
40
+ import { DEFAULT_BACKFILL_SINCE_DAYS, backfillRetryCommand, defaultBackfillWindowNotice, } from "./backfill-command-line.js";
41
+ import { isInteractiveStdin, writeHumanBackfillResult, writeLine, } from "./backfill-output.js";
42
+ import { planBackfillRun } from "./backfill-plan.js";
43
+ import { reportBackfillOutcome } from "./backfill-report.js";
44
+ import { lockHeldResult } from "./backfill-result.js";
45
+ import { scanBackfillRun } from "./backfill-scan.js";
46
+ import { uploadBackfillBatches } from "./backfill-upload.js";
47
+ // The command's public surface. Callers (doctor, onboard, local, the CLI
48
+ // router, the BLI-3272 regression test) import these from `./backfill.js`;
49
+ // which sibling implements one is this module's business, not theirs.
50
+ export { DEFAULT_BACKFILL_SINCE_DAYS, backfillRetryCommand } from "./backfill-command-line.js";
51
+ export { buildBackfillSessionReport } from "./backfill-session-report.js";
26
52
  export async function runBackfillCommand(command, io) {
27
53
  // A bare `cockpit backfill` used to refuse and print three lines telling the
28
54
  // operator to pick a window. That put a mandatory flag on the command that
@@ -75,19 +101,10 @@ export async function runBackfill(command, io) {
75
101
  const { paths } = scanned;
76
102
  const lock = await acquireBackfillLock(paths, now);
77
103
  if (!lock.acquired) {
78
- return {
79
- ...baseBackfillResult(command, backfillResultBaseArgs(scanned)),
80
- status: "blocked",
81
- retry_command: scanned.retryCommand,
82
- failure_reason: "backfill_already_running",
83
- blocked_at: {
84
- what: "backfill lock held",
85
- batch_index: 0,
86
- batch_total: 0,
87
- done: 0,
88
- total: scanned.scan.candidates.length,
89
- },
90
- };
104
+ return lockHeldResult(command, scanned, {
105
+ failureReason: "backfill_already_running",
106
+ what: "backfill lock held",
107
+ });
91
108
  }
92
109
  let collectionLock;
93
110
  try {
@@ -99,19 +116,10 @@ export async function runBackfill(command, io) {
99
116
  }
100
117
  if (!collectionLock.acquired) {
101
118
  await lock.handle.release();
102
- return {
103
- ...baseBackfillResult(command, backfillResultBaseArgs(scanned)),
104
- status: "blocked",
105
- retry_command: scanned.retryCommand,
106
- failure_reason: "sync_already_running",
107
- blocked_at: {
108
- what: "sync collection lock held",
109
- batch_index: 0,
110
- batch_total: 0,
111
- done: 0,
112
- total: scanned.scan.candidates.length,
113
- },
114
- };
119
+ return lockHeldResult(command, scanned, {
120
+ failureReason: "sync_already_running",
121
+ what: "sync collection lock held",
122
+ });
115
123
  }
116
124
  try {
117
125
  // Backfill and scheduled/manual sync share upload spool and raw-evidence
@@ -128,1659 +136,4 @@ export async function runBackfill(command, io) {
128
136
  await collectionLock.handle.release();
129
137
  await lock.handle.release();
130
138
  }
131
- }
132
- /** The `{now, dashboardUrl, sources, window, cursor, scan, reasonCounts}` bag every `baseBackfillResult` call needs. */
133
- function backfillResultBaseArgs(ctx) {
134
- return {
135
- now: ctx.now,
136
- dashboardUrl: ctx.dashboardUrl,
137
- sources: ctx.sources,
138
- window: ctx.window,
139
- cursor: ctx.cursor,
140
- scan: ctx.scan,
141
- reasonCounts: ctx.reasonCounts,
142
- };
143
- }
144
- /**
145
- * SCAN: is this machine paired, what roots/sources/window apply, and what did
146
- * the archived Codex + Claude session stores actually contain. Returns either
147
- * a terminal "not paired" result or everything PLAN/UPLOAD/REPORT need next.
148
- */
149
- async function scanBackfillRun(command, io, now) {
150
- const paths = getCollectorRuntimePaths(command.homeDir);
151
- const config = await readLocalCollectorConfig(paths);
152
- const sessionFile = await readLocalCollectorSessionFile(paths);
153
- const session = await readLocalSessionReference(paths);
154
- if (session.session_state !== "valid") {
155
- return {
156
- kind: "blocked",
157
- result: blockedBackfillResult(command, {
158
- now,
159
- dashboardUrl: sessionFile.dashboard_url ?? config.dashboard_url,
160
- reason: "collector_not_paired",
161
- cursor: await readBackfillCursor(paths),
162
- }),
163
- };
164
- }
165
- const pairedAt = parseRequiredDate(sessionFile.paired_at, "paired_at");
166
- const dashboardUrl = normalizeDashboardUrl(sessionFile.dashboard_url ?? config.dashboard_url);
167
- const roots = backfillCollectionRoots(command, config.default_repo_paths);
168
- const collectionRoots = normalizeCollectionRoots(await collectionRootPathAliases(roots));
169
- const worktreeDiscovery = await discoverBackfillWorktrees(collectionRoots, command);
170
- const retryCommand = backfillDiscoveryRetryCommand(command, worktreeDiscovery);
171
- const worktrees = worktreeDiscovery.worktrees;
172
- const sources = selectedSources(command.source);
173
- const window = backfillWindow(command, now, pairedAt);
174
- await validateBackfillReachability({
175
- fetchImpl: io.fetch,
176
- dashboardUrl,
177
- dryRun: command.dryRun,
178
- });
179
- const storedCursor = command.dryRun
180
- ? emptyBackfillCursorState()
181
- : await readBackfillCursor(paths);
182
- const scopedCursor = prepareBackfillCursorForScope(storedCursor, collectionRoots, sources);
183
- const cursor = scopedCursor.cursor;
184
- const pointerlessTerminalSessions = await loadPointerlessTerminalSessions(paths, sources);
185
- const scan = await scanBackfillSessions({
186
- command,
187
- homeDir: command.homeDir ?? os.homedir(),
188
- worktrees,
189
- collectionRoots,
190
- sources,
191
- window,
192
- cursor,
193
- pointerlessTerminalSessions,
194
- now,
195
- });
196
- // The scan issue ledger: every write below goes through a named operation
197
- // (see "Scan issue ledger" further down) instead of a bare push/sort.
198
- addRepoDiscoveryIssues(scan.issues, worktreeDiscovery.incomplete_reasons);
199
- sortScanIssuesByPriority(scan.issues);
200
- const guards = await countReadOnlyGuards(scan.candidates);
201
- scan.retryable_candidate_keys = new Set([
202
- ...scan.retryable_candidate_keys,
203
- ...guards.retryable_candidate_keys,
204
- ]);
205
- addReadOnlyGuardIssues(scan.issues, guards.counts);
206
- const reasonCounts = reasonCountsFor(scan.candidates, guards.counts, scan.issues);
207
- const oversizedCandidateKeys = oversizedBackfillCandidateKeys(scan.candidates);
208
- return {
209
- kind: "scanned",
210
- now,
211
- paths,
212
- dashboardUrl,
213
- sources,
214
- window,
215
- collectionRoots,
216
- worktrees,
217
- retryCommand,
218
- scopedCursor,
219
- cursor,
220
- scan,
221
- reasonCounts,
222
- oversizedCandidateKeys,
223
- };
224
- }
225
- /**
226
- * PLAN: given the scan, should this run actually upload anything right now?
227
- * `--all` without `--yes` needs interactive confirmation; `--dry-run` reports
228
- * what would happen and stops there. Either returns a terminal result;
229
- * anything else proceeds to UPLOAD.
230
- */
231
- async function planBackfillRun(command, io, ctx) {
232
- const { dashboardUrl, scan, reasonCounts, retryCommand } = ctx;
233
- if (command.all && !command.yes) {
234
- if (!command.json) {
235
- writeDryRunSummary(io, {
236
- candidates: scan.candidates,
237
- reasonCounts,
238
- dashboardUrl,
239
- dryRunOnly: false,
240
- });
241
- }
242
- const confirmed = await confirmAllBackfill(io);
243
- if (!confirmed) {
244
- return {
245
- kind: "result",
246
- result: {
247
- ...baseBackfillResult(command, backfillResultBaseArgs(ctx)),
248
- status: "blocked",
249
- counts: {
250
- ...baseBackfillResult(command, backfillResultBaseArgs(ctx)).counts,
251
- remaining: scan.candidates.length,
252
- },
253
- failure_reason: "confirmation_declined",
254
- },
255
- };
256
- }
257
- }
258
- if (command.dryRun) {
259
- if (!command.json) {
260
- writeDryRunSummary(io, {
261
- candidates: scan.candidates,
262
- reasonCounts,
263
- dashboardUrl,
264
- dryRunOnly: true,
265
- });
266
- }
267
- return {
268
- kind: "result",
269
- result: {
270
- ...baseBackfillResult(command, backfillResultBaseArgs(ctx)),
271
- status: blockingScanIssues(scan.issues).length > 0 ? "partial" : "complete",
272
- dry_run: true,
273
- retry_command: retryCommand,
274
- ...(blockingScanIssues(scan.issues).length > 0
275
- ? { failure_reason: blockingScanIssues(scan.issues)[0]?.reason }
276
- : {}),
277
- },
278
- };
279
- }
280
- return { kind: "proceed" };
281
- }
282
- /**
283
- * UPLOAD: sync every uploadable candidate in fixed-size batches, heartbeating
284
- * the backfill lock between batches, stopping early on an exhausted budget or
285
- * three consecutive failed batches. A candidate whose main is durably
286
- * oversized-skipped (BLI-2727) is accounted for, not missing — it will never
287
- * earn a durable pointer under the current cap, and treating it as "still
288
- * missing" would fail every batch it happens to share with genuinely uploaded
289
- * siblings, so it is excluded via `oversizedCandidateKeys` throughout.
290
- */
291
- async function uploadBackfillBatches(command, io, lock, ctx) {
292
- const { worktrees, scan, oversizedCandidateKeys } = ctx;
293
- const uploadable = uploadableCandidates(scan.candidates);
294
- const batches = buildBackfillBatches(uploadable);
295
- const rawEvidenceBudget = {
296
- remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET,
297
- remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET,
298
- };
299
- const syncResults = [];
300
- let completedBatches = 0;
301
- let failedBatches = 0;
302
- let consecutiveFailures = 0;
303
- let done = 0;
304
- let failed = 0;
305
- let deferred = 0;
306
- let uploadedObjects = 0;
307
- let uploadedChunks = 0;
308
- let backfilledSessions = 0;
309
- const durableCandidateKeys = new Set();
310
- let blockedAt;
311
- let failureReason = blockingScanIssues(scan.issues)[0]?.reason;
312
- for (const [index, batch] of batches.entries()) {
313
- await lock.handle.heartbeat();
314
- const sync = await syncBackfillBatch({
315
- command,
316
- batch,
317
- worktrees,
318
- codexAttribution: scan.codexAttribution,
319
- claudeAttribution: scan.claudeAttribution,
320
- rawEvidenceBudget,
321
- fetchImpl: io.fetch,
322
- });
323
- syncResults.push(sync);
324
- done += batch.candidates.length;
325
- uploadedObjects += sync.raw_evidence_uploaded_object_count;
326
- uploadedChunks += sync.raw_evidence_uploaded_chunk_count;
327
- failed += countSessionUploadFailures(sync);
328
- deferred +=
329
- sync.raw_evidence_deferred_byte_budget +
330
- sync.raw_evidence_deferred_object_budget;
331
- const durableInBatch = durableBackfillCandidateKeys(batch, sync);
332
- for (const key of durableInBatch)
333
- durableCandidateKeys.add(key);
334
- backfilledSessions = durableCandidateKeys.size;
335
- const missingDurableMain = batch.candidates.filter((candidate) => !durableInBatch.has(candidateCursorKey(candidate)) &&
336
- !oversizedCandidateKeys.has(candidateCursorKey(candidate))).length;
337
- const batchFailed = sync.status !== "uploaded" ||
338
- countSessionUploadFailures(sync) > 0 ||
339
- missingDurableMain > 0;
340
- const batchDeferred = sync.raw_evidence_deferred_byte_budget +
341
- sync.raw_evidence_deferred_object_budget >
342
- 0;
343
- if (batchFailed) {
344
- failedBatches += 1;
345
- consecutiveFailures += 1;
346
- }
347
- else {
348
- consecutiveFailures = 0;
349
- }
350
- if (!batchFailed && !batchDeferred) {
351
- completedBatches += 1;
352
- }
353
- if (!command.json) {
354
- writeLine(io.stdout, `Uploaded ${done}/${uploadable.length} (batch ${index + 1}/${batches.length})`);
355
- }
356
- await yieldToEventLoop();
357
- if (batchDeferred) {
358
- blockedAt = {
359
- what: "raw evidence budget exhausted",
360
- batch_index: index + 1,
361
- batch_total: batches.length,
362
- done,
363
- total: uploadable.length,
364
- };
365
- failureReason = "deferred_budget_exhausted";
366
- break;
367
- }
368
- if (consecutiveFailures >= BACKFILL_MAX_CONSECUTIVE_FAILURES) {
369
- blockedAt = {
370
- what: "consecutive upload failures",
371
- batch_index: index + 1,
372
- batch_total: batches.length,
373
- done,
374
- total: uploadable.length,
375
- };
376
- failureReason =
377
- sync.status === "spooled"
378
- ? sync.failure_reason
379
- : missingDurableMain > 0
380
- ? "durable_session_pointer_missing"
381
- : "upload_failed";
382
- break;
383
- }
384
- }
385
- return {
386
- syncResults,
387
- uploadable,
388
- batches,
389
- completedBatches,
390
- failedBatches,
391
- done,
392
- failed,
393
- deferred,
394
- uploadedObjects,
395
- uploadedChunks,
396
- backfilledSessions,
397
- durableCandidateKeys,
398
- blockedAt,
399
- failureReason,
400
- };
401
- }
402
- /**
403
- * REPORT: post the session report, record durable pointers, advance the
404
- * cursor through the resolved contiguous prefix, decide completion, write the
405
- * all-history completion marker when this run actually finished it, and
406
- * assemble the final result. `remaining`/`completionBlocked` deliberately
407
- * treat an oversized skip differently (BLI-2727): it still counts toward
408
- * `remaining` so the JSON output never goes silent about it, but it never
409
- * blocks completion on its own — every completion-gating computation below
410
- * excludes it explicitly via `oversizedCandidateKeys`/`blockingScanIssues`.
411
- */
412
- async function reportBackfillOutcome(command, io, ctx, upload) {
413
- const { now, paths, dashboardUrl, sources, collectionRoots, worktrees, retryCommand, scopedCursor, cursor, scan, oversizedCandidateKeys, } = ctx;
414
- const { syncResults, uploadable, batches, completedBatches, failedBatches, done, deferred, uploadedObjects, uploadedChunks, backfilledSessions, durableCandidateKeys, } = upload;
415
- let { blockedAt, failureReason, failed } = upload;
416
- const sessions = buildBackfillSessionReport({
417
- candidates: scan.candidates,
418
- syncResults,
419
- now,
420
- });
421
- const reportContext = sessions.length
422
- ? await ensureBackfillReportContext({
423
- homeDir: command.homeDir,
424
- paths,
425
- collectionRoots,
426
- worktrees,
427
- candidates: scan.candidates,
428
- })
429
- : null;
430
- const report = sessions.length
431
- ? await postCodexSessionReport({
432
- homeDir: command.homeDir,
433
- repoRoot: reportContext?.repoRoot,
434
- dashboardUrl,
435
- sessions,
436
- fetch: io.fetch,
437
- now,
438
- })
439
- : emptyReport("no_sessions_observed");
440
- const reportAcknowledged = sessions.length === 0 ||
441
- (report.posted && report.recorded_count >= sessions.length);
442
- if (!reportAcknowledged && !blockedAt) {
443
- blockedAt = {
444
- what: "session report failed",
445
- batch_index: completedBatches,
446
- batch_total: batches.length,
447
- done,
448
- total: uploadable.length,
449
- };
450
- failureReason ??=
451
- report.posted && report.recorded_count < sessions.length
452
- ? "session_report_ack_incomplete"
453
- : report.reason;
454
- }
455
- if (reportAcknowledged && durableCandidateKeys.size > 0) {
456
- await recordBackfillDurableSessionPointers({
457
- paths,
458
- candidates: scan.candidates,
459
- syncResults,
460
- now,
461
- });
462
- }
463
- // Raw evidence durability is necessary but not sufficient: the server must
464
- // also acknowledge the session attribution rows before their historical
465
- // cursor positions become irreversible.
466
- if (reportAcknowledged) {
467
- const cursorAdvanced = advanceBackfillCursorThroughResolvedPrefix({
468
- cursor,
469
- candidates: scan.candidates,
470
- durableCandidateKeys,
471
- retryableCandidateKeys: scan.retryable_candidate_keys,
472
- discoveryComplete: !scan.issues.some((issue) => issue.scope === "global"),
473
- now,
474
- });
475
- if (cursorAdvanced)
476
- await writeBackfillCursor(paths, cursor);
477
- }
478
- const unresolvedUploadableKeys = new Set(uploadable
479
- .filter((candidate) => !durableCandidateKeys.has(candidateCursorKey(candidate)))
480
- .map(candidateCursorKey));
481
- // "Blocking" views exclude a deterministic oversized skip (BLI-2727): it is
482
- // still unresolved (it never gets a durable pointer), but it is a permanent,
483
- // labeled fact rather than something completion should wait on forever.
484
- const unresolvedUploadableBlocking = [...unresolvedUploadableKeys].filter((key) => !oversizedCandidateKeys.has(key)).length;
485
- const unresolvedRetryableBlocking = [...scan.retryable_candidate_keys].filter((key) => !oversizedCandidateKeys.has(key)).length;
486
- const unresolvedKnownKeys = new Set([
487
- ...unresolvedUploadableKeys,
488
- ...scan.retryable_candidate_keys,
489
- ]);
490
- const unseenGlobalFailures = scan.issues
491
- .filter((issue) => issue.scope === "global")
492
- .reduce((total, issue) => total + issue.count, 0);
493
- const reportRetryable = reportAcknowledged
494
- ? 0
495
- // The cursor is intentionally all-or-nothing for the report. Even
496
- // chunks already accepted by the server are retried idempotently when
497
- // another required chunk lacks an acknowledgement.
498
- : Math.max(1, sessions.length);
499
- // `remaining` is a reporting total, not a completion gate: it still counts
500
- // every unresolved candidate, including oversized skips, so the JSON output
501
- // never goes silent about them (BLI-2727).
502
- const remaining = unresolvedKnownKeys.size +
503
- scan.omitted_candidate_count +
504
- unseenGlobalFailures +
505
- reportRetryable;
506
- failed = Math.max(failed, unresolvedUploadableBlocking);
507
- const blockingIssues = blockingScanIssues(scan.issues);
508
- const completionBlocked = blockingIssues.length > 0 ||
509
- unresolvedUploadableBlocking > 0 ||
510
- unresolvedRetryableBlocking > 0 ||
511
- deferred > 0 ||
512
- failed > 0 ||
513
- !reportAcknowledged;
514
- if (!failureReason && completionBlocked) {
515
- const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)) &&
516
- !oversizedCandidateKeys.has(candidateCursorKey(candidate)))?.reason;
517
- failureReason =
518
- blockingIssues[0]?.reason ??
519
- (unresolvedUploadableBlocking > 0
520
- ? "durable_session_pointer_missing"
521
- : retryableCandidateReason ?? "backfill_incomplete");
522
- }
523
- const status = blockedAt || completionBlocked ? "partial" : "complete";
524
- // The marker is consumed by doctor/status as proof that archived history
525
- // is covered. A bounded --since-days run may complete its requested
526
- // window, but it is not proof of an all-history backfill.
527
- if (status === "complete" && command.all) {
528
- recordBackfillScanCoverage(cursor, sources, now, now);
529
- await writeBackfillCursor(paths, cursor);
530
- const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
531
- await writeBackfillCompletionMarker(paths, {
532
- schema_version: "cockpit-backfill-complete.v2",
533
- coverage_version: BACKFILL_COVERAGE_VERSION,
534
- collection_scope_id: scopedCursor.collection_scope_id,
535
- sources,
536
- completed_at: now.toISOString(),
537
- revalidate_after: new Date(now.getTime() + BACKFILL_COMPLETION_RECHECK_MS).toISOString(),
538
- cursor,
539
- ...(oversizedCandidates.length > 0
540
- ? {
541
- oversized_skips: {
542
- reason: "file_too_large",
543
- count: oversizedCandidates.length,
544
- byte_sizes: oversizedCandidates.map((candidate) => candidate.byte_size),
545
- },
546
- }
547
- : {}),
548
- });
549
- }
550
- return {
551
- ...baseBackfillResult(command, backfillResultBaseArgs(ctx)),
552
- status,
553
- retry_command: retryCommand,
554
- counts: {
555
- ...baseBackfillResult(command, backfillResultBaseArgs(ctx)).counts,
556
- backfilled: backfilledSessions,
557
- failed,
558
- deferred,
559
- remaining,
560
- },
561
- batches: {
562
- total: batches.length,
563
- completed: completedBatches,
564
- failed: failedBatches,
565
- },
566
- report,
567
- server_acknowledged: {
568
- codex_session_report_recorded_count: report.recorded_count,
569
- raw_evidence_uploaded_object_count: uploadedObjects,
570
- raw_evidence_uploaded_chunk_count: uploadedChunks,
571
- },
572
- blocked_at: blockedAt,
573
- failure_reason: failureReason,
574
- };
575
- }
576
- function defaultBackfillWindowNotice() {
577
- return [
578
- `No window given — backfilling the last ${DEFAULT_BACKFILL_SINCE_DAYS} days.`,
579
- "The effective start is capped at the collector paired_at timestamp.",
580
- "Use `cockpit backfill --since-days N` for a different window, or `cockpit backfill --all` for the full local history (review a dry-run first; add `--yes` on headless agent runs).",
581
- ].join("\n");
582
- }
583
- /**
584
- * Produces a copyable retry that preserves the requested history window and
585
- * every discovery/selection override. Values that need quoting use syntax
586
- * accepted by the supported native shells: POSIX shells on macOS and
587
- * PowerShell on Windows.
588
- */
589
- export function backfillRetryCommand(command) {
590
- const parts = ["cockpit", "backfill"];
591
- if (command.all) {
592
- parts.push("--all", "--yes");
593
- }
594
- else if (command.sinceDays !== undefined) {
595
- parts.push("--since-days", String(command.sinceDays));
596
- }
597
- if (command.source)
598
- parts.push("--source", command.source);
599
- if (command.maxFiles !== undefined) {
600
- parts.push("--max-files", String(command.maxFiles));
601
- }
602
- if (command.maxDepth !== undefined) {
603
- parts.push("--max-depth", String(command.maxDepth));
604
- }
605
- if (command.maxRepos !== undefined) {
606
- parts.push("--max-repos", String(command.maxRepos));
607
- }
608
- if (command.repoRoot) {
609
- parts.push("--workspace", quoteCliArgument(command.repoRoot));
610
- }
611
- if (command.dryRun)
612
- parts.push("--dry-run");
613
- return parts.join(" ");
614
- }
615
- function backfillDiscoveryRetryCommand(command, discovery) {
616
- const retry = { ...command };
617
- if (discovery.incomplete_reasons.includes("max_depth_reached")) {
618
- retry.maxDepth =
619
- (command.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH) + 1;
620
- }
621
- if (discovery.incomplete_reasons.includes("max_worktrees_reached")) {
622
- retry.maxRepos =
623
- (command.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS) * 2;
624
- }
625
- return backfillRetryCommand(retry);
626
- }
627
- function quoteCliArgument(value) {
628
- if (/^[a-z0-9_./:\\-]+$/iu.test(value))
629
- return value;
630
- const escaped = process.platform === "win32"
631
- ? value.replaceAll("'", "''")
632
- : value.replaceAll("'", `'\"'\"'`);
633
- return `'${escaped}'`;
634
- }
635
- function backfillCollectionRoots(command, savedRoots) {
636
- if (command.repoRoot)
637
- return [path.resolve(command.repoRoot)];
638
- const roots = normalizeCollectionRoots(savedRoots);
639
- if (roots.length === 0) {
640
- throw new Error("No saved collection roots. Run `cockpit onboard --workspace <path>` or pass `cockpit backfill --workspace <path>`.");
641
- }
642
- return roots;
643
- }
644
- async function discoverBackfillWorktrees(roots, command) {
645
- return discoverGitWorktreesInRootsWithStatus(roots, {
646
- maxDepth: command.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
647
- maxWorktrees: command.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS,
648
- });
649
- }
650
- function selectedSources(source) {
651
- if (source === "codex")
652
- return ["codex"];
653
- if (source === "claude")
654
- return ["claude_code"];
655
- return ["codex", "claude_code"];
656
- }
657
- function backfillWindow(command, now, pairedAt) {
658
- if (command.all) {
659
- return {
660
- mode: "all",
661
- since_days: null,
662
- started_at: new Date(now.getTime() - ALL_BACKFILL_SINCE_MINUTES * 60_000)
663
- .toISOString(),
664
- paired_at: pairedAt.toISOString(),
665
- since_minutes: ALL_BACKFILL_SINCE_MINUTES,
666
- };
667
- }
668
- const requestedMs = now.getTime() - (command.sinceDays ?? 1) * 24 * 60 * 60_000;
669
- const startedAtMs = Math.max(requestedMs, pairedAt.getTime());
670
- const sinceMinutes = Math.max(1, Math.ceil((now.getTime() - startedAtMs) / 60_000));
671
- return {
672
- mode: "since_days",
673
- since_days: command.sinceDays ?? null,
674
- started_at: new Date(startedAtMs).toISOString(),
675
- paired_at: pairedAt.toISOString(),
676
- since_minutes: sinceMinutes,
677
- };
678
- }
679
- async function validateBackfillReachability(options) {
680
- if (!options.dryRun)
681
- return;
682
- const response = await options.fetchImpl(options.dashboardUrl, {
683
- method: "HEAD",
684
- });
685
- if (response.status >= 500) {
686
- throw new Error(`Dashboard reachability failed with HTTP ${response.status}.`);
687
- }
688
- }
689
- async function scanBackfillSessions(options) {
690
- const scanLimit = options.command.maxFiles ?? 10_000;
691
- const codexSessionDirs = defaultCodexSessionDirs(options.homeDir);
692
- const claudeProjectsDir = path.join(options.homeDir, ".claude", "projects");
693
- let codexAttribution = options.sources.includes("codex")
694
- ? await scanAndAttributeCodexSessions({
695
- sessionsDirs: codexSessionDirs,
696
- worktrees: options.worktrees,
697
- now: options.now,
698
- sinceMinutes: options.window.since_minutes,
699
- limit: scanLimit,
700
- collectionRoots: options.collectionRoots,
701
- })
702
- : null;
703
- if (codexAttribution?.session_limit_applied) {
704
- // The adapter limit is a memory guard, not permission to silently leave
705
- // history undiscovered. Re-scan the exact discovered population so a
706
- // user-facing --max-files cap can resume from a truthful ordering.
707
- codexAttribution = await scanAndAttributeCodexSessions({
708
- sessionsDirs: codexSessionDirs,
709
- worktrees: options.worktrees,
710
- now: options.now,
711
- sinceMinutes: options.window.since_minutes,
712
- limit: Math.max(scanLimit + 1, codexAttribution.discovered_file_count),
713
- collectionRoots: options.collectionRoots,
714
- });
715
- }
716
- let claudeAttribution = options.sources.includes("claude_code")
717
- ? await scanAndAttributeClaudeSessions({
718
- projectsDir: claudeProjectsDir,
719
- worktrees: options.worktrees,
720
- now: options.now,
721
- sinceMinutes: options.window.since_minutes,
722
- limit: scanLimit,
723
- collectionRoots: options.collectionRoots,
724
- })
725
- : null;
726
- if (claudeAttribution?.session_limit_applied) {
727
- claudeAttribution = await scanAndAttributeClaudeSessions({
728
- projectsDir: claudeProjectsDir,
729
- worktrees: options.worktrees,
730
- now: options.now,
731
- sinceMinutes: options.window.since_minutes,
732
- limit: Math.max(scanLimit + 1, claudeAttribution.discovered_session_count),
733
- collectionRoots: options.collectionRoots,
734
- });
735
- }
736
- const allCandidates = [
737
- ...(codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
738
- ...(claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
739
- ]
740
- .filter((candidate) => isAfterCursor(candidate, options.cursor) ||
741
- isPointerlessTerminalRetry(candidate, options.pointerlessTerminalSessions))
742
- .sort((a, b) => compareBackfillCandidatesForRetry(a, b, options.pointerlessTerminalSessions));
743
- const candidates = allCandidates.slice(0, options.command.maxFiles ?? Number.MAX_SAFE_INTEGER);
744
- const omittedCandidateCount = allCandidates.length - candidates.length;
745
- const issues = await backfillScanIssues({
746
- codexAttribution,
747
- claudeAttribution,
748
- codexSessionDirs,
749
- claudeProjectsDir,
750
- candidates,
751
- omittedCandidateCount,
752
- });
753
- return {
754
- candidates,
755
- codexAttribution,
756
- claudeAttribution,
757
- issues,
758
- retryable_candidate_keys: retryableCandidateKeys(candidates),
759
- omitted_candidate_count: omittedCandidateCount,
760
- };
761
- }
762
- async function loadPointerlessTerminalSessions(paths, sources) {
763
- const [codexCursor, claudeCursor] = await Promise.all([
764
- sources.includes("codex")
765
- ? readRawEvidenceCursor(paths)
766
- : Promise.resolve(null),
767
- sources.includes("claude_code")
768
- ? readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME })
769
- : Promise.resolve(null),
770
- ]);
771
- const terminalIds = (sessions) => new Set(Object.entries(sessions ?? {})
772
- .filter(([, entry]) => isPointerlessTerminalCursorEntry(entry))
773
- .map(([sessionId]) => sessionId));
774
- const result = {
775
- codex: terminalIds(codexCursor?.sessions),
776
- claude_code: terminalIds(claudeCursor?.sessions),
777
- };
778
- const count = result.codex.size + result.claude_code.size;
779
- if (count > 0) {
780
- console.error("[backfill] pointer-less terminal sessions reopened", JSON.stringify({
781
- count,
782
- codex_count: result.codex.size,
783
- claude_count: result.claude_code.size,
784
- }));
785
- }
786
- return result;
787
- }
788
- function isPointerlessTerminalCursorEntry(entry) {
789
- return (!entry.uploaded_object_key &&
790
- (entry.state === "ambiguous" ||
791
- entry.state === "unattributed" ||
792
- entry.state === "skipped"));
793
- }
794
- function isPointerlessTerminalRetry(candidate, sessions) {
795
- return sessions[candidate.source].has(candidate.session_id);
796
- }
797
- function compareBackfillCandidatesForRetry(a, b, sessions) {
798
- const aRetry = isPointerlessTerminalRetry(a, sessions);
799
- const bRetry = isPointerlessTerminalRetry(b, sessions);
800
- if (aRetry !== bRetry)
801
- return aRetry ? -1 : 1;
802
- if (aRetry && bRetry) {
803
- return (a.session_file_mtime_ms - b.session_file_mtime_ms ||
804
- candidateCursorKey(a).localeCompare(candidateCursorKey(b)));
805
- }
806
- return compareBackfillCandidates(a, b);
807
- }
808
- function normalizeCodexCandidate(result) {
809
- return {
810
- source: "codex",
811
- session_id: result.codex_session_id,
812
- file_path: result.file_path,
813
- state: result.state,
814
- reason: result.reason,
815
- signals: result.signals,
816
- attribution_score: result.attribution_score,
817
- path_score: result.path_score,
818
- content_hash_sha256: result.content_hash_sha256,
819
- byte_size: result.byte_size,
820
- session_file_mtime: result.session_file_mtime,
821
- session_file_mtime_ms: result.session_file_mtime_ms,
822
- worktree: result.worktree,
823
- cwd_basename: result.cwd_basename,
824
- cwd_hash: result.cwd_hash,
825
- };
826
- }
827
- function normalizeClaudeCandidate(result) {
828
- return {
829
- source: "claude_code",
830
- session_id: result.claude_session_id,
831
- file_path: result.file_path,
832
- state: result.state,
833
- reason: result.reason,
834
- signals: result.signals,
835
- attribution_score: result.attribution_score,
836
- path_score: result.path_score,
837
- content_hash_sha256: result.content_hash_sha256,
838
- byte_size: result.byte_size,
839
- session_file_mtime: result.session_file_mtime,
840
- session_file_mtime_ms: result.session_file_mtime_ms,
841
- worktree: result.worktree,
842
- cwd_basename: result.cwd_basename,
843
- cwd_hash: result.cwd_hash,
844
- claude: result,
845
- };
846
- }
847
- function isAfterCursor(candidate, cursor) {
848
- const source = cursor.sources[candidate.source];
849
- const oldest = source?.oldest_mtime_ms_processed;
850
- if (oldest === null || oldest === undefined)
851
- return true;
852
- if (candidate.session_file_mtime_ms < oldest)
853
- return true;
854
- const key = candidateCursorKey(candidate);
855
- if (candidate.session_file_mtime_ms === oldest) {
856
- return !source.processed_keys_at_oldest_mtime.includes(key);
857
- }
858
- const newest = source.newest_mtime_ms_covered;
859
- // A cursor written before upper-edge coverage existed gets one safe
860
- // migration pass across the previously processed interval. Once that pass is
861
- // acknowledged, subsequent --all runs only inspect genuinely newer files.
862
- if (newest === null)
863
- return true;
864
- if (candidate.session_file_mtime_ms > newest)
865
- return true;
866
- if (candidate.session_file_mtime_ms < newest)
867
- return false;
868
- // A legacy cursor has no boundary identities. Rechecking the equal-time
869
- // boundary is safe because durable evidence is content-addressed.
870
- return !source.processed_keys_at_newest_mtime.includes(key);
871
- }
872
- function compareBackfillCandidates(a, b) {
873
- return (b.session_file_mtime_ms - a.session_file_mtime_ms ||
874
- candidateCursorKey(a).localeCompare(candidateCursorKey(b)));
875
- }
876
- function candidateCursorKey(candidate) {
877
- return crypto
878
- .createHash("sha256")
879
- .update(JSON.stringify([
880
- candidate.source,
881
- candidate.session_id,
882
- normalizedCursorPath(candidate.file_path),
883
- ]))
884
- .digest("hex");
885
- }
886
- function normalizedCursorPath(filePath) {
887
- const windowsStyle = path.win32.isAbsolute(filePath) && !path.posix.isAbsolute(filePath);
888
- if (windowsStyle)
889
- return path.win32.normalize(filePath).toLowerCase();
890
- const normalized = path.resolve(filePath);
891
- return process.platform === "win32" ? normalized.toLowerCase() : normalized;
892
- }
893
- async function backfillScanIssues(options) {
894
- const issues = [];
895
- const add = (reason, count, scope) => {
896
- if (count > 0)
897
- issues.push({ reason, count, scope });
898
- };
899
- if (options.codexAttribution) {
900
- add("codex_session_limit_applied", options.codexAttribution.session_limit_applied
901
- ? Math.max(1, options.codexAttribution.discovered_file_count -
902
- options.codexAttribution.scanned_file_count)
903
- : 0, "global");
904
- const missingTopLevelDirs = (await Promise.all(options.codexSessionDirs.map(isMissingPath))).filter(Boolean).length;
905
- add("codex_directory_read_failed", Math.max(0, options.codexAttribution.directory_read_failed_count -
906
- missingTopLevelDirs), "global");
907
- add("codex_session_stat_failed", options.codexAttribution.stat_failed_count, "global");
908
- }
909
- if (options.claudeAttribution) {
910
- add("claude_session_limit_applied", options.claudeAttribution.session_limit_applied
911
- ? Math.max(1, options.claudeAttribution.discovered_session_count -
912
- options.claudeAttribution.scanned_session_count)
913
- : 0, "global");
914
- const projectsDirMissing = await isMissingPath(options.claudeProjectsDir);
915
- add("claude_project_dir_read_failed", Math.max(0, options.claudeAttribution.project_dir_read_failed_count -
916
- (projectsDirMissing ? 1 : 0)), "global");
917
- add("claude_session_stat_failed", options.claudeAttribution.session_stat_failed_count, "global");
918
- add("claude_sidecar_stat_failed", options.claudeAttribution.sidecar_stat_failed_count, "global");
919
- add("claude_sidecar_dir_read_failed", await countUnreadableClaudeSidecarDirs(options.candidates), "global");
920
- }
921
- add("backfill_max_files_applied", options.omittedCandidateCount, "selection");
922
- add("candidate_file_read_failed", options.candidates.filter((candidate) => candidate.reason === "file_read_failed").length, "candidate");
923
- add("candidate_worktree_unavailable", options.candidates.filter((candidate) => isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
924
- candidate.worktree === null).length, "candidate");
925
- add("claude_main_file_too_large", options.candidates.filter((candidate) => candidate.source === "claude_code" &&
926
- candidate.claude?.main_file_oversized).length, "candidate");
927
- add("claude_sidecar_limit_applied", options.candidates.reduce((total, candidate) => total + (candidate.claude?.sidecars_capped ?? 0), 0), "candidate");
928
- add("claude_sidecar_file_unreadable", options.candidates.reduce((total, candidate) => total +
929
- (candidate.claude?.sidecar_files.filter((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
930
- sidecar.skipped_reason === "file_too_large").length ?? 0), 0), "candidate");
931
- return issues.sort((a, b) => scanIssuePriority(a.scope) - scanIssuePriority(b.scope) ||
932
- a.reason.localeCompare(b.reason));
933
- }
934
- function scanIssuePriority(scope) {
935
- if (scope === "global")
936
- return 0;
937
- if (scope === "candidate")
938
- return 1;
939
- return 2;
940
- }
941
- function retryableCandidateKeys(candidates) {
942
- return new Set(candidates
943
- .filter((candidate) => candidate.reason === "file_read_failed" ||
944
- candidate.reason === "repo_not_on_disk" ||
945
- (isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
946
- !candidate.worktree) ||
947
- Boolean(candidate.claude?.main_file_oversized) ||
948
- (candidate.claude?.sidecars_capped ?? 0) > 0 ||
949
- Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
950
- sidecar.skipped_reason === "file_too_large")))
951
- .map(candidateCursorKey));
952
- }
953
- /**
954
- * A main session file whose only story is "too large to upload under the
955
- * current cap" (BLI-2727). This mirrors exactly the two branches in
956
- * `countReadOnlyGuards` that emit the `file_too_large` reason, so a candidate
957
- * is in this set if and only if it contributed to that scan issue's count —
958
- * one predicate, no drift between "why the issue fired" and "which candidate
959
- * caused it". Deterministic and non-retryable: rerunning backfill cannot
960
- * resolve it (only a larger cap or a smaller file can), so unlike a transient
961
- * read failure it must never poison completion or a batch's success.
962
- */
963
- function oversizedBackfillCandidateKeys(candidates) {
964
- const keys = new Set();
965
- for (const candidate of candidates) {
966
- if ((candidate.source === "claude_code" &&
967
- candidate.claude?.main_file_oversized) ||
968
- candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
969
- keys.add(candidateCursorKey(candidate));
970
- }
971
- }
972
- return keys;
973
- }
974
- // --- Scan issue ledger --------------------------------------------------
975
- //
976
- // `scan.issues` is the running ledger of everything the scan could not
977
- // account for cleanly. Every write to it goes through one of the named
978
- // operations below instead of a bare `.push`/`.sort`, so the ledger's shape
979
- // (global vs candidate vs selection scope, priority order) has one owner.
980
- /** Record one issue on the ledger. */
981
- function addScanIssue(issues, issue) {
982
- issues.push(issue);
983
- }
984
- /** Repo discovery could not fully enumerate a root: one global issue per reason. */
985
- function addRepoDiscoveryIssues(issues, incompleteReasons) {
986
- for (const reason of incompleteReasons) {
987
- addScanIssue(issues, {
988
- reason: `repo_discovery_${reason}`,
989
- count: 1,
990
- scope: "global",
991
- });
992
- }
993
- }
994
- /** Stable read order: global issues first, then candidate, then selection; alphabetical within a scope. */
995
- function sortScanIssuesByPriority(issues) {
996
- issues.sort((a, b) => scanIssuePriority(a.scope) - scanIssuePriority(b.scope) ||
997
- a.reason.localeCompare(b.reason));
998
- }
999
- /** The read-only guard pass's two aggregate outcomes, each recorded once if it fired at all. */
1000
- function addReadOnlyGuardIssues(issues, guardCounts) {
1001
- const readFailed = guardCounts.get("file_read_failed");
1002
- if (readFailed) {
1003
- addScanIssue(issues, {
1004
- reason: "candidate_file_read_failed",
1005
- count: readFailed,
1006
- scope: "candidate",
1007
- });
1008
- }
1009
- const tooLarge = guardCounts.get("file_too_large");
1010
- if (tooLarge) {
1011
- addScanIssue(issues, {
1012
- reason: "file_too_large",
1013
- count: tooLarge,
1014
- scope: "candidate",
1015
- });
1016
- }
1017
- }
1018
- // BLI-2727: a deterministic, labeled oversized skip must never poison
1019
- // completion — it is a permanent, non-retryable fact about the file, not an
1020
- // in-flight problem a rerun can fix. `scan.issues`/`retryable_candidate_keys`
1021
- // still carry it (so it's never silently dropped from reporting); every
1022
- // completion-gating computation excludes it explicitly instead, by filtering
1023
- // through `blockingScanIssues` below.
1024
- //
1025
- // Two scan issues describe the exact same oversized-main candidates:
1026
- // `backfillScanIssues` pushes the Claude-specific `claude_main_file_too_large`
1027
- // (from `claude?.main_file_oversized`) and `countReadOnlyGuards` (via
1028
- // `addReadOnlyGuardIssues` above) separately pushes the source-agnostic
1029
- // `file_too_large` (same predicate as `oversizedBackfillCandidateKeys`, so
1030
- // this list can never drift from it). Both must be excluded from
1031
- // completion-gating together.
1032
- const OVERSIZED_SCAN_ISSUE_REASONS = new Set([
1033
- "file_too_large",
1034
- "claude_main_file_too_large",
1035
- ]);
1036
- function blockingScanIssues(issues) {
1037
- return issues.filter((issue) => !OVERSIZED_SCAN_ISSUE_REASONS.has(issue.reason));
1038
- }
1039
- async function countUnreadableClaudeSidecarDirs(candidates) {
1040
- let unreadable = 0;
1041
- for (const candidate of candidates) {
1042
- if (candidate.source !== "claude_code")
1043
- continue;
1044
- const subagentsDir = path.join(path.dirname(candidate.file_path), path.basename(candidate.file_path).replace(/\.jsonl$/i, ""), "subagents");
1045
- try {
1046
- await fs.readdir(subagentsDir);
1047
- }
1048
- catch (error) {
1049
- if (!isMissingFsError(error))
1050
- unreadable += 1;
1051
- }
1052
- }
1053
- return unreadable;
1054
- }
1055
- async function isMissingPath(filePath) {
1056
- try {
1057
- await fs.stat(filePath);
1058
- return false;
1059
- }
1060
- catch (error) {
1061
- return isMissingFsError(error);
1062
- }
1063
- }
1064
- function isMissingFsError(error) {
1065
- if (!error || typeof error !== "object")
1066
- return false;
1067
- const code = error.code;
1068
- return code === "ENOENT" || code === "ENOTDIR";
1069
- }
1070
- async function countReadOnlyGuards(candidates) {
1071
- const counts = new Map();
1072
- const retryableCandidateKeys = new Set();
1073
- // Aggregated: this runs over the whole archived history, so a per-candidate
1074
- // line could be thousands. The count already travels; the reason did not
1075
- // (BLI-3238).
1076
- let firstReadFailure = null;
1077
- for (const candidate of candidates) {
1078
- if (candidate.reason === "repo_not_on_disk") {
1079
- increment(counts, "repo_not_on_disk");
1080
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1081
- }
1082
- if (candidate.source === "claude_code" && candidate.claude?.main_file_oversized) {
1083
- increment(counts, "file_too_large");
1084
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1085
- continue;
1086
- }
1087
- if (candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
1088
- increment(counts, "file_too_large");
1089
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1090
- continue;
1091
- }
1092
- if (!isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null))
1093
- continue;
1094
- try {
1095
- await fs.readFile(candidate.file_path);
1096
- }
1097
- catch (error) {
1098
- increment(counts, "file_read_failed");
1099
- firstReadFailure ??= describeError(error);
1100
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1101
- }
1102
- }
1103
- const readFailedCount = counts.get("file_read_failed") ?? 0;
1104
- if (readFailedCount > 0) {
1105
- console.error("[cockpit-backfill] archived sessions could not be read", JSON.stringify({
1106
- reason: "file_read_failed",
1107
- read_failed_count: readFailedCount,
1108
- candidate_count: candidates.length,
1109
- ...firstReadFailure,
1110
- }));
1111
- }
1112
- return {
1113
- counts,
1114
- retryable_candidate_keys: retryableCandidateKeys,
1115
- };
1116
- }
1117
- function reasonCountsFor(candidates, guardCounts, scanIssues) {
1118
- const counts = new Map();
1119
- for (const candidate of candidates)
1120
- increment(counts, candidate.reason);
1121
- for (const [reason, count] of guardCounts) {
1122
- counts.set(reason, Math.max(counts.get(reason) ?? 0, count));
1123
- }
1124
- for (const issue of scanIssues) {
1125
- counts.set(issue.reason, Math.max(counts.get(issue.reason) ?? 0, issue.count));
1126
- }
1127
- for (const required of ["file_too_large", "repo_not_on_disk"]) {
1128
- counts.set(required, counts.get(required) ?? 0);
1129
- }
1130
- return [...counts.entries()]
1131
- .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
1132
- .map(([reason, count]) => ({
1133
- reason,
1134
- count,
1135
- ...reasonClassification(reason),
1136
- }));
1137
- }
1138
- function reasonClassification(reason) {
1139
- if (reason === "secret_like_content_guard" || reason === "secret_redaction_failed") {
1140
- return {
1141
- classification: "retryable",
1142
- note: "historical guard result; collector now masks and retries",
1143
- };
1144
- }
1145
- if (reason === "file_too_large") {
1146
- return {
1147
- classification: "retryable",
1148
- note: "until the evidence file cap is raised",
1149
- };
1150
- }
1151
- if (reason === "repo_not_on_disk") {
1152
- return {
1153
- classification: "retryable",
1154
- note: "repo must exist on disk",
1155
- };
1156
- }
1157
- if (reason === "cwd_not_a_repo") {
1158
- return {
1159
- classification: "permanent",
1160
- note: "cwd exists but is not a repo or folder workspace",
1161
- };
1162
- }
1163
- if (reason === "multiple_transcript_origins") {
1164
- return {
1165
- classification: "permanent",
1166
- note: "multiple transcript origins; attribution is ambiguous",
1167
- };
1168
- }
1169
- if (reason === "single_repo_folder_fallback") {
1170
- return {
1171
- classification: "permanent",
1172
- note: "uploadable single-repo folder workspace fallback",
1173
- };
1174
- }
1175
- if (reason === "multi_repo_folder_workspace") {
1176
- return {
1177
- classification: "permanent",
1178
- note: "uploadable multi-repo folder workspace fallback",
1179
- };
1180
- }
1181
- if (reason.startsWith("deferred_")) {
1182
- return {
1183
- classification: "retryable",
1184
- note: "rerun cockpit backfill to continue",
1185
- };
1186
- }
1187
- return {
1188
- classification: "retryable",
1189
- note: "rerun after fixing source or collector state",
1190
- };
1191
- }
1192
- function uploadableCandidates(candidates) {
1193
- return candidates.filter((candidate) => isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
1194
- candidate.worktree);
1195
- }
1196
- function buildBackfillBatches(candidates) {
1197
- const byWorktree = new Map();
1198
- for (const candidate of candidates) {
1199
- if (!candidate.worktree)
1200
- continue;
1201
- const key = candidate.worktree.worktree_fingerprint;
1202
- byWorktree.set(key, [...(byWorktree.get(key) ?? []), candidate]);
1203
- }
1204
- const batches = [];
1205
- for (const group of byWorktree.values()) {
1206
- const worktree = group[0]?.worktree;
1207
- if (!worktree)
1208
- continue;
1209
- for (let offset = 0; offset < group.length; offset += BACKFILL_UPLOAD_BATCH_SESSIONS) {
1210
- batches.push({
1211
- worktree,
1212
- candidates: group.slice(offset, offset + BACKFILL_UPLOAD_BATCH_SESSIONS),
1213
- });
1214
- }
1215
- }
1216
- return batches;
1217
- }
1218
- async function ensureBackfillReportContext(options) {
1219
- const candidateWorktree = options.candidates.find((candidate) => candidate.worktree)?.worktree;
1220
- const representative = candidateWorktree ?? options.worktrees[0] ?? null;
1221
- // A deleted repo or an approved folder workspace can legitimately produce
1222
- // report-only rows with no current git worktree. Use the exact approved root
1223
- // as a synthetic local context instead of falling through to process.cwd().
1224
- const repoRoot = representative?.repo_root ??
1225
- options.collectionRoots[0] ??
1226
- process.cwd();
1227
- try {
1228
- await readLocalWorkContextForRepo(options.paths, repoRoot);
1229
- }
1230
- catch {
1231
- // Reading it can legitimately fail — there is no context yet, which is
1232
- // precisely why the next line creates one. That read is a probe and stays
1233
- // silent; the CREATE is the branch that has to speak (BLI-3238).
1234
- await startLocalWorkContext({
1235
- homeDir: options.homeDir,
1236
- repoRoot,
1237
- branch: representative?.branch,
1238
- }).catch((error) => {
1239
- // Context creation is best-effort here: postCodexSessionReport converts
1240
- // a remaining local-context failure into a retryable report reason. But
1241
- // that reason is `collector_not_ready`, which points the operator at
1242
- // setup rather than at whatever actually failed here.
1243
- console.error("[cockpit-backfill] could not start a local work context for the batch", JSON.stringify({
1244
- reason: "work_context_start_failed",
1245
- ...describeError(error),
1246
- }));
1247
- });
1248
- }
1249
- return { repoRoot };
1250
- }
1251
- async function syncBackfillBatch(options) {
1252
- const syncOptions = {
1253
- homeDir: options.command.homeDir,
1254
- repoRoot: options.batch.worktree.repo_root,
1255
- worktreeInventory: worktreeInventoryForRepo(options.batch.worktree, options.worktrees),
1256
- codexSessionFiles: options.batch.candidates
1257
- .filter((candidate) => candidate.source === "codex")
1258
- .map((candidate) => ({
1259
- local_path: candidate.file_path,
1260
- codex_session_id: candidate.session_id,
1261
- })),
1262
- codexAttributionScan: options.codexAttribution ?? undefined,
1263
- claudeSessionFiles: options.batch.candidates
1264
- .filter((candidate) => candidate.source === "claude_code")
1265
- .map((candidate) => ({
1266
- local_path: candidate.file_path,
1267
- claude_session_id: candidate.session_id,
1268
- main_file_oversized: Boolean(candidate.claude?.main_file_oversized),
1269
- skip_main: false,
1270
- sidecar_files: candidate.claude?.sidecar_files
1271
- .filter((sidecar) => !sidecar.skipped_reason)
1272
- .map((sidecar) => ({ local_path: sidecar.local_path })) ?? [],
1273
- })),
1274
- claudeAttributionScan: options.claudeAttribution ?? undefined,
1275
- rawEvidenceBudget: options.rawEvidenceBudget,
1276
- // `cockpit backfill` is only ever an operator asking — by hand, through
1277
- // onboarding, through doctor, or by running the retry command Cockpit
1278
- // printed. The delivery-backoff window is the scheduler's cadence, so it
1279
- // does not gate this pass: honouring it here made the retry that follows a
1280
- // failed commit a 15-minute no-op that reported
1281
- // `durable_session_pointer_missing` and never named the hold (BLI-3118).
1282
- evidenceDeliveryMode: "operator_retry",
1283
- fetch: options.fetchImpl,
1284
- };
1285
- try {
1286
- return await syncLocalAmbientEnvelope(syncOptions);
1287
- }
1288
- catch (error) {
1289
- if (error instanceof LocalUploadBlockedError &&
1290
- error.blocker === "missing_context") {
1291
- await startLocalWorkContext({
1292
- homeDir: options.command.homeDir,
1293
- repoRoot: options.batch.worktree.repo_root,
1294
- branch: options.batch.worktree.branch,
1295
- });
1296
- return await syncLocalAmbientEnvelope(syncOptions);
1297
- }
1298
- throw error;
1299
- }
1300
- }
1301
- /** Exported for the BLI-3272 regression test; not part of the CLI surface. */
1302
- export function buildBackfillSessionReport(options) {
1303
- const uploadByKey = new Map();
1304
- // BLI-2107: an outcome that names a failure but has no pointer used to be
1305
- // dropped on the floor here, taking its reason with it.
1306
- const noUploadReasonBySessionId = new Map();
1307
- for (const sync of options.syncResults) {
1308
- if (sync.status !== "uploaded")
1309
- continue;
1310
- for (const outcome of sync.raw_evidence_outcomes) {
1311
- if (!outcome.codex_session_id)
1312
- continue;
1313
- const source = outcome.kind === "claude_jsonl"
1314
- ? "claude_code"
1315
- : outcome.kind === "codex_jsonl"
1316
- ? "codex"
1317
- : null;
1318
- if (!source)
1319
- continue;
1320
- if (!outcome.raw_evidence_pointer_id) {
1321
- if (outcome.reason) {
1322
- noUploadReasonBySessionId.set(outcome.codex_session_id, outcome.reason);
1323
- }
1324
- continue;
1325
- }
1326
- uploadByKey.set(`${source}:${outcome.codex_session_id}`, {
1327
- upload_state: outcome.upload_state,
1328
- raw_evidence_pointer_id: outcome.raw_evidence_pointer_id,
1329
- reason: outcome.reason,
1330
- });
1331
- }
1332
- }
1333
- const bestByKey = new Map();
1334
- for (const candidate of options.candidates) {
1335
- const key = `${candidate.source}:${candidate.session_id}`;
1336
- const existing = bestByKey.get(key);
1337
- if (!existing || rank(candidate.state) > rank(existing.state)) {
1338
- bestByKey.set(key, candidate);
1339
- }
1340
- else if (existing &&
1341
- rank(candidate.state) === rank(existing.state) &&
1342
- candidate.session_file_mtime_ms > existing.session_file_mtime_ms) {
1343
- bestByKey.set(key, candidate);
1344
- }
1345
- }
1346
- return [...bestByKey.values()].map((candidate) => {
1347
- const upload = uploadByKey.get(`${candidate.source}:${candidate.session_id}`);
1348
- return {
1349
- codex_session_id: candidate.session_id,
1350
- source: candidate.source,
1351
- observed_at: options.now.toISOString(),
1352
- attribution_state: candidate.state,
1353
- attribution_reason: candidate.reason,
1354
- attribution_score: candidate.attribution_score,
1355
- path_score: candidate.path_score,
1356
- signals: candidate.signals,
1357
- ...(candidate.content_hash_sha256
1358
- ? { session_file_hash_sha256: candidate.content_hash_sha256 }
1359
- : {}),
1360
- session_file_byte_size: candidate.byte_size,
1361
- session_file_mtime: candidate.session_file_mtime,
1362
- ...(candidate.worktree
1363
- ? {
1364
- repo_fingerprint: candidate.worktree.repo_fingerprint,
1365
- worktree_fingerprint: candidate.worktree.worktree_fingerprint,
1366
- repo_label: candidate.worktree.repo_label,
1367
- branch: candidate.worktree.branch,
1368
- }
1369
- : {}),
1370
- ...(candidate.cwd_basename ? { cwd_basename: candidate.cwd_basename } : {}),
1371
- ...(candidate.cwd_hash ? { cwd_hash: candidate.cwd_hash } : {}),
1372
- ...(upload
1373
- ? {
1374
- raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
1375
- upload_state: upload.upload_state,
1376
- ...(upload.upload_state === "upload_failed"
1377
- ? { upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED }
1378
- : {}),
1379
- }
1380
- : isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null)
1381
- ? {
1382
- upload_state: "not_uploaded",
1383
- // BLI-2107: same rule as live sync — an attributed session with
1384
- // no pointer always says why, even when the answer is that this
1385
- // path never recorded one.
1386
- upload_reason: noUploadReasonBySessionId.get(candidate.session_id) ??
1387
- NO_UPLOAD_ATTEMPT_RECORDED,
1388
- }
1389
- : {
1390
- // BLI-3272: and the refused-attribution branch says why too. Same
1391
- // NULL/NULL hole as live sync, same fix — backfill is the path
1392
- // that revisits old sessions, so leaving it silent would keep
1393
- // rewriting the very rows this ticket found.
1394
- upload_state: "not_uploaded",
1395
- upload_reason: noUploadReasonBySessionId.get(candidate.session_id) ??
1396
- notUploadableAttributionStateReason(candidate.reason),
1397
- }),
1398
- };
1399
- });
1400
- }
1401
- function rank(state) {
1402
- switch (state) {
1403
- case "attributed":
1404
- return 5;
1405
- case "attributed_fallback":
1406
- return 4;
1407
- case "ambiguous":
1408
- return 3;
1409
- case "unattributed":
1410
- return 2;
1411
- case "skipped":
1412
- return 1;
1413
- default:
1414
- return 0;
1415
- }
1416
- }
1417
- function durableBackfillCandidateKeys(batch, sync) {
1418
- const durable = new Set();
1419
- if (sync.status !== "uploaded")
1420
- return durable;
1421
- if (sync.raw_evidence_deferred_byte_budget > 0 ||
1422
- sync.raw_evidence_deferred_object_budget > 0) {
1423
- // Deferred outcomes have no per-file identity. Advancing any candidate in
1424
- // this batch could therefore strand the deferred main transcript.
1425
- return durable;
1426
- }
1427
- const outcomesBySession = new Map();
1428
- for (const outcome of sync.raw_evidence_outcomes) {
1429
- const source = backfillSourceForEvidenceKind(outcome.kind);
1430
- if (!source || !outcome.codex_session_id)
1431
- continue;
1432
- const key = `${source}:${outcome.codex_session_id}`;
1433
- const summary = outcomesBySession.get(key) ?? {
1434
- durableMainCount: 0,
1435
- durableSidecarCount: 0,
1436
- failed: false,
1437
- };
1438
- if (outcome.upload_state === "upload_failed") {
1439
- summary.failed = true;
1440
- }
1441
- else if (outcome.raw_evidence_pointer_id &&
1442
- (outcome.upload_state === "uploaded" ||
1443
- outcome.upload_state === "reused_existing")) {
1444
- if (outcome.kind === "codex_jsonl" ||
1445
- outcome.kind === "claude_jsonl") {
1446
- summary.durableMainCount += 1;
1447
- }
1448
- else if (outcome.kind === "claude_jsonl_sidecar") {
1449
- summary.durableSidecarCount += 1;
1450
- }
1451
- }
1452
- outcomesBySession.set(key, summary);
1453
- }
1454
- for (const candidate of batch.candidates) {
1455
- const summary = outcomesBySession.get(`${candidate.source}:${candidate.session_id}`);
1456
- const expectedSidecars = candidate.source === "claude_code"
1457
- ? (candidate.claude?.sidecar_files.filter((sidecar) => !sidecar.skipped_reason).length ?? 0)
1458
- : 0;
1459
- if (summary &&
1460
- !summary.failed &&
1461
- summary.durableMainCount > 0 &&
1462
- summary.durableSidecarCount >= expectedSidecars) {
1463
- durable.add(candidateCursorKey(candidate));
1464
- }
1465
- }
1466
- return durable;
1467
- }
1468
- async function recordBackfillDurableSessionPointers(options) {
1469
- const durableObjectBySession = new Map();
1470
- for (const sync of options.syncResults) {
1471
- for (const outcome of sync.raw_evidence_outcomes) {
1472
- if (!outcome.codex_session_id ||
1473
- (outcome.kind !== "codex_jsonl" && outcome.kind !== "claude_jsonl") ||
1474
- (outcome.upload_state !== "uploaded" &&
1475
- outcome.upload_state !== "reused_existing") ||
1476
- !outcome.object_key) {
1477
- continue;
1478
- }
1479
- const source = outcome.kind === "codex_jsonl" ? "codex" : "claude_code";
1480
- durableObjectBySession.set(`${source}:${outcome.codex_session_id}`, outcome.object_key);
1481
- }
1482
- }
1483
- let recordedCount = 0;
1484
- for (const source of ["codex", "claude_code"]) {
1485
- const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
1486
- const cursor = await readRawEvidenceCursor(options.paths, { filename });
1487
- let changed = false;
1488
- for (const candidate of options.candidates) {
1489
- if (candidate.source !== source)
1490
- continue;
1491
- const objectKey = durableObjectBySession.get(`${source}:${candidate.session_id}`);
1492
- const prior = cursor.sessions[candidate.session_id];
1493
- if (!objectKey || !prior || prior.uploaded_object_key)
1494
- continue;
1495
- cursor.sessions[candidate.session_id] = {
1496
- ...prior,
1497
- file_hash_sha256: candidate.content_hash_sha256,
1498
- file_mtime_ms: candidate.session_file_mtime_ms,
1499
- byte_size: candidate.byte_size,
1500
- byte_offset: candidate.byte_size,
1501
- state: candidate.state,
1502
- reason: candidate.reason,
1503
- worktree_fingerprint: candidate.worktree?.worktree_fingerprint ?? null,
1504
- uploaded_object_key: objectKey,
1505
- uploaded_at: options.now.toISOString(),
1506
- uploaded_byte_size: candidate.byte_size,
1507
- last_seen_at: options.now.toISOString(),
1508
- };
1509
- changed = true;
1510
- recordedCount += 1;
1511
- }
1512
- if (changed) {
1513
- await writeRawEvidenceCursor(options.paths, cursor, {
1514
- filename,
1515
- sessionsOnly: source === "claude_code",
1516
- });
1517
- }
1518
- }
1519
- if (recordedCount > 0) {
1520
- console.error("[backfill] terminal session pointers recorded", JSON.stringify({ count: recordedCount }));
1521
- }
1522
- }
1523
- function countSessionUploadFailures(sync) {
1524
- return sync.raw_evidence_outcomes.filter((outcome) => Boolean(outcome.codex_session_id) &&
1525
- outcome.upload_state === "upload_failed" &&
1526
- Boolean(backfillSourceForEvidenceKind(outcome.kind))).length;
1527
- }
1528
- function backfillSourceForEvidenceKind(kind) {
1529
- if (kind === "codex_jsonl" || kind === "codex_image_attachment") {
1530
- return "codex";
1531
- }
1532
- if (kind === "claude_jsonl" ||
1533
- kind === "claude_jsonl_sidecar" ||
1534
- kind === "claude_image_attachment") {
1535
- return "claude_code";
1536
- }
1537
- return null;
1538
- }
1539
- function advanceBackfillCursorThroughResolvedPrefix(options) {
1540
- if (!options.discoveryComplete)
1541
- return false;
1542
- const observations = [];
1543
- for (const source of ["codex", "claude_code"]) {
1544
- const remaining = options.candidates
1545
- .filter((candidate) => candidate.source === source &&
1546
- isAfterCursor(candidate, options.cursor))
1547
- .sort(compareBackfillCandidates);
1548
- for (const candidate of remaining) {
1549
- const key = candidateCursorKey(candidate);
1550
- if (options.retryableCandidateKeys.has(key))
1551
- break;
1552
- const resolved = isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null)
1553
- ? options.durableCandidateKeys.has(key)
1554
- : true;
1555
- if (!resolved)
1556
- break;
1557
- observations.push({
1558
- source: candidate.source,
1559
- cursor_key: key,
1560
- state: candidate.state,
1561
- reason: candidate.reason,
1562
- session_file_mtime_ms: candidate.session_file_mtime_ms,
1563
- session_file_mtime: candidate.session_file_mtime,
1564
- });
1565
- }
1566
- }
1567
- if (observations.length === 0)
1568
- return false;
1569
- recordBackfillCursorObservations(options.cursor, observations, options.now);
1570
- return true;
1571
- }
1572
- function worktreeInventoryForRepo(current, worktrees) {
1573
- return worktrees
1574
- .filter((worktree) => worktree.repo_fingerprint
1575
- ? worktree.repo_fingerprint === current.repo_fingerprint
1576
- : worktree.repo_label === current.repo_label)
1577
- .map((worktree) => ({
1578
- repo: worktree.repo_root,
1579
- repo_label: worktree.repo_label,
1580
- repo_fingerprint: worktree.repo_fingerprint,
1581
- repo_origin_url: worktree.repo_origin_url ?? undefined,
1582
- head_sha: worktree.head_sha ?? undefined,
1583
- worktree_label: worktree.worktree_label,
1584
- worktree_fingerprint: worktree.worktree_fingerprint,
1585
- worktree_is_primary: worktree.worktree_is_primary,
1586
- branch: worktree.branch,
1587
- }));
1588
- }
1589
- function baseBackfillResult(command, options) {
1590
- const states = countBy(options.scan.candidates, (candidate) => candidate.state);
1591
- const uploadable = uploadableCandidates(options.scan.candidates);
1592
- const perSource = {
1593
- codex: sourceCounts("codex", options.scan),
1594
- claude_code: sourceCounts("claude_code", options.scan),
1595
- };
1596
- return {
1597
- status: "complete",
1598
- dry_run: command.dryRun,
1599
- dashboard_url: options.dashboardUrl,
1600
- verify_url: `${options.dashboardUrl}/my-work`,
1601
- sources: options.sources,
1602
- window: options.window,
1603
- counts: {
1604
- total: options.scan.candidates.length,
1605
- uploadable: uploadable.length,
1606
- backfilled: 0,
1607
- skipped: options.scan.candidates.length - uploadable.length,
1608
- failed: 0,
1609
- deferred: 0,
1610
- remaining: 0,
1611
- states,
1612
- reasons: options.reasonCounts,
1613
- per_source: perSource,
1614
- },
1615
- batches: {
1616
- total: Math.ceil(uploadable.length / BACKFILL_UPLOAD_BATCH_SESSIONS),
1617
- completed: 0,
1618
- failed: 0,
1619
- },
1620
- resume_cursor: options.cursor,
1621
- retry_command: backfillRetryCommand(command),
1622
- report: emptyReport("not_posted"),
1623
- server_acknowledged: {
1624
- codex_session_report_recorded_count: 0,
1625
- raw_evidence_uploaded_object_count: 0,
1626
- raw_evidence_uploaded_chunk_count: 0,
1627
- },
1628
- };
1629
- }
1630
- function blockedBackfillResult(command, options) {
1631
- return {
1632
- status: "blocked",
1633
- dry_run: command.dryRun,
1634
- dashboard_url: options.dashboardUrl,
1635
- verify_url: `${options.dashboardUrl}/my-work`,
1636
- sources: selectedSources(command.source),
1637
- window: {
1638
- mode: command.all ? "all" : "since_days",
1639
- since_days: command.sinceDays ?? null,
1640
- started_at: options.now.toISOString(),
1641
- paired_at: options.now.toISOString(),
1642
- since_minutes: 0,
1643
- },
1644
- counts: {
1645
- total: 0,
1646
- uploadable: 0,
1647
- backfilled: 0,
1648
- skipped: 0,
1649
- failed: 0,
1650
- deferred: 0,
1651
- remaining: 0,
1652
- states: {},
1653
- reasons: [],
1654
- per_source: {
1655
- codex: emptySourceCounts(),
1656
- claude_code: emptySourceCounts(),
1657
- },
1658
- },
1659
- batches: { total: 0, completed: 0, failed: 0 },
1660
- resume_cursor: options.cursor,
1661
- retry_command: backfillRetryCommand(command),
1662
- report: emptyReport("not_posted"),
1663
- server_acknowledged: {
1664
- codex_session_report_recorded_count: 0,
1665
- raw_evidence_uploaded_object_count: 0,
1666
- raw_evidence_uploaded_chunk_count: 0,
1667
- },
1668
- failure_reason: options.reason,
1669
- };
1670
- }
1671
- function sourceCounts(source, scan) {
1672
- const candidates = scan.candidates.filter((candidate) => candidate.source === source);
1673
- const scanned = source === "codex"
1674
- ? (scan.codexAttribution?.scanned_file_count ?? 0)
1675
- : (scan.claudeAttribution?.scanned_session_count ?? 0);
1676
- return {
1677
- scanned,
1678
- selected: candidates.length,
1679
- uploadable: uploadableCandidates(candidates).length,
1680
- states: countBy(candidates, (candidate) => candidate.state),
1681
- };
1682
- }
1683
- function emptySourceCounts() {
1684
- return { scanned: 0, selected: 0, uploadable: 0, states: {} };
1685
- }
1686
- function emptyReport(reason) {
1687
- return {
1688
- posted: false,
1689
- reason,
1690
- chunk_count: 0,
1691
- recorded_count: 0,
1692
- failed_count: 0,
1693
- chunks: [],
1694
- };
1695
- }
1696
- function countBy(items, keyOf) {
1697
- const counts = {};
1698
- for (const item of items) {
1699
- const key = keyOf(item);
1700
- counts[key] = (counts[key] ?? 0) + 1;
1701
- }
1702
- return counts;
1703
- }
1704
- function increment(counts, key) {
1705
- counts.set(key, (counts.get(key) ?? 0) + 1);
1706
- }
1707
- function writeDryRunSummary(io, options) {
1708
- const uploadable = uploadableCandidates(options.candidates);
1709
- writeLine(io.stdout, `${options.dryRunOnly ? "DRY-RUN" : "Review"}: ${options.candidates.length} session(s), ${uploadable.length} uploadable, ${options.candidates.length - uploadable.length} skipped.`);
1710
- writeReasonTable(io, options.reasonCounts);
1711
- if (options.dryRunOnly) {
1712
- writeLine(io.stdout, "DRY-RUN: wrote nothing (no cursor, marker, report, or upload).");
1713
- }
1714
- writeLine(io.stdout, `Verify after upload: ${options.dashboardUrl}/my-work`);
1715
- }
1716
- function writeHumanBackfillResult(result, io) {
1717
- if (result.status === "complete") {
1718
- writeLine(io.stdout, `PASS: ${result.counts.backfilled} backfilled, ${result.counts.skipped} skipped (table). Verify: ${result.verify_url}`);
1719
- writeReasonTable(io, result.counts.reasons);
1720
- return;
1721
- }
1722
- if (result.status === "partial") {
1723
- const at = result.blocked_at;
1724
- if (at) {
1725
- writeLine(io.stderr, `BLOCKED: ${at.what} at batch ${at.batch_index}/${at.batch_total}, ${at.done}/${at.total} done`);
1726
- }
1727
- writeLine(io.stderr, `Failure: ${result.failure_reason ?? "partial_backfill"}`);
1728
- writeLine(io.stderr, `Retry: ${result.retry_command}`);
1729
- writeLine(io.stderr, `Stopped: ${result.counts.remaining} remaining — rerun cockpit backfill to continue`);
1730
- writeReasonTable(io, result.counts.reasons);
1731
- return;
1732
- }
1733
- const at = result.blocked_at;
1734
- if (at) {
1735
- writeLine(io.stderr, `BLOCKED: ${at.what} at batch ${at.batch_index}/${at.batch_total}, ${at.done}/${at.total} done`);
1736
- }
1737
- else {
1738
- writeLine(io.stderr, "BLOCKED: backfill could not run.");
1739
- }
1740
- writeLine(io.stderr, `Failure: ${result.failure_reason ?? "no_sessions"}`);
1741
- writeLine(io.stderr, `Retry: ${result.retry_command}`);
1742
- }
1743
- function writeReasonTable(io, reasons) {
1744
- if (reasons.length === 0)
1745
- return;
1746
- writeLine(io.stdout, "Skip/reason table:");
1747
- for (const reason of reasons) {
1748
- writeLine(io.stdout, `- ${reason.reason}: ${reason.count} (${reason.classification}; ${reason.note})`);
1749
- }
1750
- }
1751
- async function confirmAllBackfill(io) {
1752
- const answer = await readLine(io, "Proceed with --all backfill upload? [y/N] ");
1753
- return answer.trim().toLowerCase() === "y" || answer.trim().toLowerCase() === "yes";
1754
- }
1755
- async function readLine(io, prompt) {
1756
- io.stdout.write(prompt);
1757
- io.stdin.setEncoding("utf8");
1758
- return new Promise((resolve) => {
1759
- const onData = (chunk) => {
1760
- io.stdin.removeListener("data", onData);
1761
- io.stdin.pause();
1762
- resolve(chunk);
1763
- };
1764
- io.stdin.resume();
1765
- io.stdin.on("data", onData);
1766
- });
1767
- }
1768
- function isInteractiveStdin(io) {
1769
- return Boolean(io.stdin.isTTY);
1770
- }
1771
- function parseRequiredDate(value, label) {
1772
- const date = new Date(value);
1773
- if (!Number.isFinite(date.getTime())) {
1774
- throw new Error(`Collector session ${label} is invalid.`);
1775
- }
1776
- return date;
1777
- }
1778
- function normalizeDashboardUrl(value) {
1779
- return value.trim().replace(/\/+$/, "");
1780
- }
1781
- function writeLine(stream, text) {
1782
- stream.write(`${text}\n`);
1783
- }
1784
- function yieldToEventLoop() {
1785
- return new Promise((resolve) => setTimeout(resolve, 0));
1786
139
  }